From 849d995346713b57fe9de59f86003eb3f42a21e8 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 6 Jul 2022 15:10:54 +0300 Subject: [PATCH 001/524] Add getvalidatorsessions query to validator-engine-console --- test/test-validator-session-state.cpp | 4 ++ tl/generate/scheme/ton_api.tl | 4 ++ tl/generate/scheme/ton_api.tlo | Bin 68244 -> 68852 bytes .../validator-engine-console-query.cpp | 23 ++++++++++++ .../validator-engine-console-query.h | 20 ++++++++++ .../validator-engine-console.cpp | 1 + validator-engine/validator-engine.cpp | 27 ++++++++++++++ validator-engine/validator-engine.hpp | 2 + .../validator-session-description.cpp | 5 +++ .../validator-session-description.h | 1 + .../validator-session-description.hpp | 1 + validator-session/validator-session.cpp | 13 +++++++ validator-session/validator-session.h | 2 + validator-session/validator-session.hpp | 2 + validator/manager-disk.hpp | 4 ++ validator/manager-hardfork.hpp | 4 ++ validator/manager.cpp | 35 +++++++++++++++++- validator/manager.hpp | 4 +- validator/validator-group.cpp | 28 +++++++++++++- validator/validator-group.hpp | 3 ++ validator/validator.h | 2 + 21 files changed, 181 insertions(+), 4 deletions(-) diff --git a/test/test-validator-session-state.cpp b/test/test-validator-session-state.cpp index 819c1cc2de..02ec4866a2 100644 --- a/test/test-validator-session-state.cpp +++ b/test/test-validator-session-state.cpp @@ -103,6 +103,10 @@ class Description : public ton::validatorsession::ValidatorSessionDescription { td::uint32 get_max_priority() const override { return opts_.round_candidates - 1; } + td::uint32 get_node_by_priority(td::uint32 round, td::uint32 priority) const override { + CHECK(priority <= get_max_priority()); + return (round + priority) % get_total_nodes(); + } td::uint32 get_unixtime(td::uint64 ts) const override { return static_cast(ts >> 32); } diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 7dc6d8ca47..1909c71a01 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -630,6 +630,8 @@ engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string bdcst_errors:int engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) = engine.validator.OverlayStats; engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; +engine.validator.validatorSessionInfo current_block:tonNode.blockId self:int256 current_round:int next_producers:(vector int256) = engine.validator.ValidatorSessionInfo; +engine.validator.validatorSessionsInfo sessions:(vector engine.validator.validatorSessionInfo) = engine.validator.ValidatorSessionsInfo; ---functions--- @@ -680,6 +682,8 @@ engine.validator.importCertificate overlay_id:int256 local_id:adnl.id.short sign engine.validator.signShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash expire_at:int max_size:int = overlay.Certificate; engine.validator.importShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash cert:overlay.Certificate = engine.validator.Success; +engine.validator.getValidatorSessionsInfo = engine.validator.ValidatorSessionsInfo; + ---types--- storage.pong = storage.Pong; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index c49612788983ca6776a9977c3b0bbb6bc2701e05..9af19e27884de50bfd6dae0e45cdbc98835e7dab 100644 GIT binary patch delta 415 zcmbO-mF3G!7T!m*^{p77;Pgh`Y;i3M!LQd;Q}fa@^HTN75_2+B5=-)n^ul1Q;MC&c z%=|phytMp@>!h@=E?s&{jiBmckn+v(;;wv*j+>XN{?G8nZVTK#BwH95Kx%Iu6Y^mJ zx>NeueNEov(xRf&yps5&oc!c$hRN?QNKV$dsK5yqpWJv|o|i8#wW1`xpeR43G&yy$ zVw3#j6MiBNAhTfpy(-5QgWdh;?f|==0i #include @@ -1005,3 +1008,23 @@ td::Status ImportShardOverlayCertificateQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "successfully sent certificate to overlay manager\n"; return td::Status::OK(); } + +td::Status GetValidatorSessionsInfoQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status GetValidatorSessionsInfoQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetValidatorSessionsInfoQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX( + f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + std::string s = td::json_encode(td::ToJson(*f), true); + td::TerminalIO::out() << "---------\n" << s << "--------\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 15b77314a4..dea2273dbd 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1075,3 +1075,23 @@ class ImportShardOverlayCertificateQuery : public Query { std::string in_file_; }; +class GetValidatorSessionsInfoQuery : public Query { + public: + GetValidatorSessionsInfoQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "getvalidatorsessions"; + } + static std::string get_help() { + return "getvalidatorsessions\tprint info about validator session"; + } + std::string name() const override { + return get_name(); + } + + private: +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 1b3f337601..2c04971b57 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -140,6 +140,7 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index f380b003c4..7783dc38ea 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -3280,6 +3280,33 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getOverla }); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getValidatorSessionsInfo &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + + if (validator_manager_.empty()) { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); + return; + } + + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise)]( + td::Result> R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::serialize_tl_object(R.move_as_ok(), true)); + } + }); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::get_validator_sessions_info, + std::move(P)); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 3f4fea2c65..e0364baba6 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -407,6 +407,8 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getValidatorSessionsInfo &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator-session/validator-session-description.cpp b/validator-session/validator-session-description.cpp index 42059053ae..001f04058f 100644 --- a/validator-session/validator-session-description.cpp +++ b/validator-session/validator-session-description.cpp @@ -79,6 +79,11 @@ td::uint32 ValidatorSessionDescriptionImpl::get_max_priority() const { return opts_.round_candidates - 1; } +td::uint32 ValidatorSessionDescriptionImpl::get_node_by_priority(td::uint32 round, td::uint32 priority) const { + CHECK(priority <= get_max_priority()); + return (round + priority) % get_total_nodes(); +} + ValidatorSessionCandidateId ValidatorSessionDescriptionImpl::candidate_id( td::uint32 src_idx, ValidatorSessionRootHash root_hash, ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash) const { diff --git a/validator-session/validator-session-description.h b/validator-session/validator-session-description.h index 60e006d081..ac50431258 100644 --- a/validator-session/validator-session-description.h +++ b/validator-session/validator-session-description.h @@ -85,6 +85,7 @@ class ValidatorSessionDescription { virtual ValidatorWeight get_total_weight() const = 0; virtual td::int32 get_node_priority(td::uint32 src_idx, td::uint32 round) const = 0; virtual td::uint32 get_max_priority() const = 0; + virtual td::uint32 get_node_by_priority(td::uint32 round, td::uint32 priority) const = 0; virtual td::uint32 get_unixtime(td::uint64 t) const = 0; virtual td::uint32 get_attempt_seqno(td::uint64 t) const = 0; virtual td::uint32 get_self_idx() const = 0; diff --git a/validator-session/validator-session-description.hpp b/validator-session/validator-session-description.hpp index d65369ace6..d7c12b1018 100644 --- a/validator-session/validator-session-description.hpp +++ b/validator-session/validator-session-description.hpp @@ -101,6 +101,7 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription { } td::int32 get_node_priority(td::uint32 src_idx, td::uint32 round) const override; td::uint32 get_max_priority() const override; + td::uint32 get_node_by_priority(td::uint32 round, td::uint32 priority) const override; td::uint32 get_unixtime(td::uint64 ts) const override { return static_cast(ts >> 32); } diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 3f56e3a31c..5d4d4fd03d 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -19,6 +19,7 @@ #include "validator-session.hpp" #include "td/utils/Random.h" #include "td/utils/crypto.h" +#include "ton/ton-tl.hpp" namespace ton { @@ -937,6 +938,18 @@ void ValidatorSessionImpl::stats_set_candidate_status(td::uint32 round, PublicKe it->block_status = status; } +void ValidatorSessionImpl::get_session_info( + td::Promise> promise) { + std::vector next_producers; + for (td::uint32 round = cur_round_; round < cur_round_ + 20; ++round) { + td::uint32 node = description().get_node_by_priority(round, 0); + next_producers.push_back(description().get_source_id(node).bits256_value()); + } + promise.set_result(create_tl_object( + create_tl_block_id_simple(BlockId{}), description().get_source_id(local_idx()).bits256_value(), + cur_round_, std::move(next_producers))); +} + td::actor::ActorOwn ValidatorSession::create( catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index 376cac45e2..fed75020e9 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -92,6 +92,8 @@ class ValidatorSession : public td::actor::Actor { virtual void start() = 0; virtual void destroy() = 0; + virtual void get_session_info(td::Promise> promise) = 0; + static td::actor::ActorOwn create( catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 54bba33d18..a4e6bf03f2 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -159,6 +159,8 @@ class ValidatorSessionImpl : public ValidatorSession { void stats_add_round(); void stats_set_candidate_status(td::uint32 round, PublicKeyHash src, int status); + void get_session_info(td::Promise> promise) override; + public: ValidatorSessionImpl(catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index b56f8b3ea3..3f98273e0d 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -367,6 +367,10 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { UNREACHABLE(); } + void get_validator_sessions_info( + td::Promise> promise) override { + UNREACHABLE(); + } private: PublicKeyHash local_id_; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 11470cfd33..17042dbf12 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -426,6 +426,10 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { UNREACHABLE(); } + void get_validator_sessions_info( + td::Promise> promise) override { + UNREACHABLE(); + } private: td::Ref opts_; diff --git a/validator/manager.cpp b/validator/manager.cpp index d6a9b4dda5..81cd2e0eaf 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2539,7 +2539,7 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, stats.creator.bits256_value(), stats.total_validators, stats.total_weight, stats.signatures, stats.signatures_weight, stats.approve_signatures, stats.approve_signatures_weight, stats.first_round, std::move(rounds)); - std::string s = td::json_encode(td::ToJson(*obj.get()), false); + std::string s = td::json_encode(td::ToJson(*obj), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); std::ofstream file; @@ -2550,6 +2550,39 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, LOG(INFO) << "Writing validator session stats for " << block_id.id; } +void ValidatorManagerImpl::get_validator_sessions_info( + td::Promise> promise) { + std::vector> groups; + for (const auto& g : validator_groups_) { + groups.push_back(g.second.get()); + } + struct IntermediateData { + std::vector> groups; + std::vector> result; + td::Promise> promise; + + static void step(IntermediateData data) { + if (data.groups.empty()) { + data.promise.set_result( + create_tl_object(std::move(data.result))); + return; + } + auto group = std::move(data.groups.back()); + data.groups.pop_back(); + auto P = td::PromiseCreator::lambda( + [data = + std::move(data)](td::Result> R) mutable { + if (R.is_ok()) { + data.result.push_back(R.move_as_ok()); + } + step(std::move(data)); + }); + td::actor::send_closure(group, &ValidatorGroup::get_session_info, std::move(P)); + } + }; + IntermediateData::step({std::move(groups), {}, std::move(promise)}); +} + td::actor::ActorOwn ValidatorManagerFactory::create( td::Ref opts, std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, diff --git a/validator/manager.hpp b/validator/manager.hpp index d7f2c8a337..c855e1cc7e 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -531,8 +531,10 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override; void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override; + void get_validator_sessions_info( + td::Promise> promise) override; - private: + private: td::Timestamp resend_shard_blocks_at_; td::Timestamp check_waiters_at_; td::Timestamp check_shard_clients_; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 22ee35a521..333259b4c4 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -21,6 +21,7 @@ #include "ton/ton-io.hpp" #include "td/utils/overloaded.h" #include "common/delay.h" +#include "ton/ton-tl.hpp" namespace ton { @@ -159,14 +160,18 @@ void ValidatorGroup::get_approved_candidate(PublicKey source, RootHash root_hash std::move(promise)); } -BlockIdExt ValidatorGroup::create_next_block_id(RootHash root_hash, FileHash file_hash) const { +BlockId ValidatorGroup::create_next_block_id_simple() const { BlockSeqno seqno = 0; for (auto &p : prev_block_ids_) { if (seqno < p.id.seqno) { seqno = p.id.seqno; } } - return BlockIdExt{shard_.workchain, shard_.shard, seqno + 1, root_hash, file_hash}; + return BlockId{shard_.workchain, shard_.shard, seqno + 1}; +} + +BlockIdExt ValidatorGroup::create_next_block_id(RootHash root_hash, FileHash file_hash) const { + return BlockIdExt{create_next_block_id_simple(), root_hash, file_hash}; } std::unique_ptr ValidatorGroup::make_validator_session_callback() { @@ -307,6 +312,25 @@ void ValidatorGroup::destroy() { stop(); } +void ValidatorGroup::get_session_info( + td::Promise> promise) { + if (session_.empty() || !started_) { + promise.set_error(td::Status::Error(ErrorCode::notready, "session not started")); + } + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise), block_id = create_next_block_id_simple()]( + td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + auto info = R.move_as_ok(); + info->current_block_ = create_tl_block_id_simple(block_id); + promise.set_result(std::move(info)); + }); + td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_session_info, std::move(P)); +} + } // namespace validator } // namespace ton diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 15256984b9..3216f5fe67 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -44,6 +44,7 @@ class ValidatorGroup : public td::actor::Actor { td::Promise promise); void get_approved_candidate(PublicKey source, RootHash root_hash, FileHash file_hash, FileHash collated_data_file_hash, td::Promise promise); + BlockId create_next_block_id_simple() const; BlockIdExt create_next_block_id(RootHash root_hash, FileHash file_hash) const; void start(std::vector prev, BlockIdExt min_masterchain_block_id, UnixTime min_ts); @@ -56,6 +57,8 @@ class ValidatorGroup : public td::actor::Actor { } } + void get_session_info(td::Promise> promise); + ValidatorGroup(ShardIdFull shard, PublicKeyHash local_id, ValidatorSessionId session_id, td::Ref validator_set, validatorsession::ValidatorSessionOptions config, td::actor::ActorId keyring, td::actor::ActorId adnl, diff --git a/validator/validator.h b/validator/validator.h index fce119a49d..b3d32e307f 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -214,6 +214,8 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void run_ext_query(td::BufferSlice data, td::Promise promise) = 0; virtual void prepare_stats(td::Promise>> promise) = 0; + virtual void get_validator_sessions_info( + td::Promise> promise) = 0; }; } // namespace validator From 625516c5682f2ed12a27cd50b338b27dfda60a46 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 12 Jul 2022 11:04:31 +0300 Subject: [PATCH 002/524] Lite-mode for validate-query, more collated data in collator --- validator/fabric.h | 4 +- validator/impl/collator-impl.h | 7 +- validator/impl/collator.cpp | 89 +++++++++++++++++-- validator/impl/fabric.cpp | 5 +- validator/impl/validate-query.cpp | 138 +++++++++++++++++++++++++----- validator/impl/validate-query.hpp | 6 +- validator/manager-disk.cpp | 2 +- 7 files changed, 214 insertions(+), 37 deletions(-) diff --git a/validator/fabric.h b/validator/fabric.h index 67b6ae9ee4..3515c0da84 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -25,6 +25,8 @@ namespace ton { namespace validator { +enum ValidateMode { fake = 1, lite = 2 }; + td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_); td::actor::ActorOwn create_liteserver_cache_actor(td::actor::ActorId manager, std::string db_root); @@ -73,7 +75,7 @@ void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::act void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, bool is_fake = false); + td::Promise promise, unsigned mode = 0); void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& min_masterchain_block_id, std::vector prev, Ed25519_PublicKey local_id, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 2dced7495e..997fea3663 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -205,7 +205,11 @@ class Collator final : public td::actor::Actor { unsigned block_create_total_{0}; std::vector bad_ext_msgs_, delay_ext_msgs_; Ref shard_account_blocks_; // ShardAccountBlocks + + std::map> blocks_with_state_proofs_; + std::vector neighbor_proof_builders_; std::vector> collated_roots_; + std::unique_ptr block_candidate; td::PerfWarningTimer perf_timer_{"collate", 0.1}; @@ -235,7 +239,8 @@ class Collator final : public td::actor::Actor { void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res); bool fix_one_processed_upto(block::MsgProcessedUpto& proc, const ton::ShardIdFull& owner); bool fix_processed_upto(block::MsgProcessedUptoCollection& upto); - void got_neighbor_out_queue(int i, td::Result> res); + void got_neighbor_block_data(td::Result> res); + void got_neighbor_block_state(int i, td::Result> res); bool adjust_shard_config(); bool store_shard_fees(ShardIdFull shard, const block::CurrencyCollection& fees, const block::CurrencyCollection& created); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 6aedcc21ae..4d26f4b5d7 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -502,6 +502,7 @@ void Collator::after_get_block_data(int idx, td::Result> res) { prev_mc_block = prev_block_data[0]; mc_block_root = prev_mc_block->root_cell(); } + blocks_with_state_proofs_[prev_block_data[idx]->root_cell()->get_hash().bits()] = prev_block_data[idx]; } check_pending(); } @@ -614,26 +615,59 @@ bool Collator::request_neighbor_msg_queues() { neighbors_.emplace_back(*shard_ptr); } int i = 0; + neighbor_proof_builders_.resize(neighbors_.size()); for (block::McShardDescr& descr : neighbors_) { LOG(DEBUG) << "neighbor #" << i << " : " << descr.blk_.to_str(); + if (descr.blk_.seqno() != 0) { + ++pending; + send_closure_later(manager, &ValidatorManager::wait_block_data_short, descr.blk_, priority(), timeout, + [self = get_self(), i](td::Result> res) { + LOG(DEBUG) << "got answer to wait_block_data for neighbor #" << i; + send_closure_later(std::move(self), &Collator::got_neighbor_block_data, std::move(res)); + }); + } ++pending; - send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, - [self = get_self(), i](td::Result> res) { - td::actor::send_closure(std::move(self), &Collator::got_neighbor_out_queue, i, std::move(res)); + send_closure_later(manager, &ValidatorManager::wait_block_state_short, descr.blk_, priority(), timeout, + [self = get_self(), i](td::Result> res) { + LOG(DEBUG) << "got answer to wait_block_state for neighbor #" << i; + send_closure_later(std::move(self), &Collator::got_neighbor_block_state, i, std::move(res)); }); ++i; } return true; } -void Collator::got_neighbor_out_queue(int i, td::Result> res) { - LOG(DEBUG) << "obtained outbound queue for neighbor #" << i; +void Collator::got_neighbor_block_data(td::Result> res) { + --pending; + if (res.is_error()) { + fatal_error(res.move_as_error()); + return; + } + auto block_data = res.move_as_ok(); + blocks_with_state_proofs_[block_data->root_cell()->get_hash().bits()] = block_data; + check_pending(); +} + +void Collator::got_neighbor_block_state(int i, td::Result> res) { --pending; if (res.is_error()) { fatal_error(res.move_as_error()); return; } - Ref outq_descr = res.move_as_ok(); + Ref state = res.move_as_ok(); + neighbor_proof_builders_.at(i) = vm::MerkleProofBuilder{state->root_cell()}; + auto new_state = ShardStateQ::fetch(state->get_block_id(), {}, neighbor_proof_builders_.at(i).root()); + if (new_state.is_error()) { + fatal_error(new_state.move_as_error()); + return; + } + auto outq_descr_res = new_state.move_as_ok()->message_queue(); + if (outq_descr_res.is_error()) { + fatal_error(outq_descr_res.move_as_error()); + return; + } + LOG(DEBUG) << "obtained outbound queue for neighbor #" << i; + Ref outq_descr = outq_descr_res.move_as_ok(); block::McShardDescr& descr = neighbors_.at(i); if (outq_descr->get_block_id() != descr.blk_) { LOG(DEBUG) << "outq_descr->id = " << outq_descr->get_block_id().to_str() << " ; descr.id = " << descr.blk_.to_str(); @@ -3949,7 +3983,6 @@ Ref Collator::collate_shard_block_descr_set() { } bool Collator::create_collated_data() { - // TODO: store something into collated_roots_ // 1. store the set of used shard block descriptions if (!used_shard_block_descr_.empty()) { auto cell = collate_shard_block_descr_set(); @@ -3959,7 +3992,47 @@ bool Collator::create_collated_data() { } collated_roots_.push_back(std::move(cell)); } - // 2. ... + // 2. Proofs for hashes of states: previous states + neighbors + for (const auto& p : blocks_with_state_proofs_) { + vm::MerkleProofBuilder mpb{p.second->root_cell()}; + block::gen::Block::Record block; + if (!tlb::unpack_cell(mpb.root(), block) || block.state_update->load_cell().is_error()) { + return fatal_error("cannot generate Merkle proof for previous block"); + } + Ref proof = mpb.extract_proof(); + if (proof.is_null()) { + return fatal_error("cannot generate Merkle proof for previous block"); + } + collated_roots_.push_back(std::move(proof)); + } + // 3. Previous state proof (only shadchains) + std::map> proofs; + if (!is_masterchain()) { + state_usage_tree_->set_use_mark_for_is_loaded(false); + Ref state_proof = vm::MerkleProof::generate(prev_state_root_, state_usage_tree_.get()); + if (state_proof.is_null()) { + return fatal_error("cannot generate Merkle proof for previous state"); + } + proofs[prev_state_root_->get_hash().bits()] = std::move(state_proof); + } + // 4. Proofs for message queues + for (vm::MerkleProofBuilder &mpb : neighbor_proof_builders_) { + Ref proof = mpb.extract_proof(); + if (proof.is_null()) { + return fatal_error("cannot generate Merkle proof for neighbor"); + } + auto it = proofs.emplace(mpb.root()->get_hash().bits(), proof); + if (!it.second) { + it.first->second = vm::MerkleProof::combine(it.first->second, std::move(proof)); + if (it.first->second.is_null()) { + return fatal_error("cannot combine merkle proofs"); + } + } + } + + for (auto& p : proofs) { + collated_roots_.push_back(std::move(p.second)); + } return true; } diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 196595aedc..c3f4338d4f 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -191,17 +191,18 @@ void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::act void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, bool is_fake) { + td::Promise promise, unsigned mode) { BlockSeqno seqno = 0; for (auto& p : prev) { if (p.seqno() > seqno) { seqno = p.seqno(); } } + bool is_fake = mode & ValidateMode::fake; td::actor::create_actor( PSTRING() << (is_fake ? "fakevalidate" : "validateblock") << shard.to_str() << ":" << (seqno + 1), shard, min_ts, min_masterchain_block_id, std::move(prev), std::move(candidate), std::move(validator_set), std::move(manager), - timeout, std::move(promise), is_fake) + timeout, std::move(promise), mode) .release(); } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 97a4bab968..b9347fcf76 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -31,6 +31,7 @@ #include "vm/cells/MerkleProof.h" #include "vm/cells/MerkleUpdate.h" #include "common/errorlog.h" +#include "fabric.h" #include namespace ton { @@ -51,7 +52,7 @@ std::string ErrorCtx::as_string() const { ValidateQuery::ValidateQuery(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, bool is_fake) + td::Promise promise, unsigned mode) : shard_(shard) , id_(candidate.id) , min_ts(min_ts) @@ -62,7 +63,8 @@ ValidateQuery::ValidateQuery(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_ , manager(std::move(manager)) , timeout(timeout) , main_promise(std::move(promise)) - , is_fake_(is_fake) + , is_fake_(mode & ValidateMode::fake) + , is_lite_(mode & ValidateMode::lite) , shard_pfx_(shard_.shard) , shard_pfx_len_(ton::shard_prefix_length(shard_)) { proc_hash_.zero(); @@ -257,18 +259,20 @@ void ValidateQuery::start_up() { td::actor::send_closure_later( std::move(self), &ValidateQuery::after_get_latest_mc_state, std::move(res)); }); - // 3. load state(s) corresponding to previous block(s) + // 3. load state(s) corresponding to previous block(s) (non-lite mode or masterchain) prev_states.resize(prev_blocks.size()); - for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { - // 3.1. load state - LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; - ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), - timeout, [self = get_self(), i](td::Result> res) -> void { - LOG(DEBUG) << "got answer to wait_block_state_short query #" << i; - td::actor::send_closure_later( - std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res)); - }); + if (is_masterchain() || !is_lite_) { + for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { + // 3.1. load state + LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; + ++pending; + td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), + timeout, [self = get_self(), i](td::Result> res) -> void { + LOG(DEBUG) << "got answer to wait_block_state_short query #" << i; + td::actor::send_closure_later( + std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res)); + }); + } } // 4. unpack block candidate (while necessary data is being loaded) if (!unpack_block_candidate()) { @@ -988,6 +992,9 @@ bool ValidateQuery::check_this_shard_mc_info() { */ bool ValidateQuery::compute_prev_state() { + if (!is_masterchain() && is_lite_) { + return compute_prev_state_lite_mode(); + } CHECK(prev_states.size() == 1u + after_merge_); prev_state_root_ = prev_states[0]->root_cell(); CHECK(prev_state_root_.not_null()); @@ -1006,6 +1013,46 @@ bool ValidateQuery::compute_prev_state() { return true; } +bool ValidateQuery::compute_prev_state_lite_mode() { + td::Bits256 state_hash; + if (id_.seqno() == 1) { + if (prev_blocks.size() != 1) { + return reject_query("seqno is 1, but number of previous blocks is not 1"); + } + state_hash = prev_blocks[0].root_hash; + } else { + std::vector> prev_state_roots(prev_blocks.size()); + for (size_t i = 0; i < prev_blocks.size(); ++i) { + prev_state_roots[i] = get_virt_state_root(prev_blocks[i].root_hash); + if (prev_state_roots[i].is_null()) { + return reject_query(PSTRING() << "cannot get hash of previous state root: " << prev_blocks[i]); + } + } + + if (prev_state_roots.size() == 1) { + state_hash = prev_state_roots[0]->get_hash().bits(); + } else { + CHECK(prev_state_roots.size() == 2); + Ref merged; + if (!block::gen::t_ShardState.cell_pack_split_state(merged, prev_state_roots[0], prev_state_roots[1])) { + return fatal_error(-667, "cannot construct mechanically merged previously state"); + } + state_hash = merged->get_hash().bits(); + } + } + if (state_hash != prev_state_hash_) { + return reject_query("previous state hash mismatch for block "s + id_.to_str() + " : block header declares " + + prev_state_hash_.to_hex() + " , actual " + state_hash.to_hex()); + } + auto it = virt_roots_.find(state_hash); + if (it == virt_roots_.end()) { + return reject_query(PSTRING() << "no state root for previous block in collated data (hash = " + << state_hash.to_hex() << ")"); + } + prev_state_root_ = it->second; + return true; +} + bool ValidateQuery::compute_next_state() { LOG(DEBUG) << "computing next state"; auto res = vm::MerkleUpdate::validate(state_update_); @@ -1209,15 +1256,42 @@ bool ValidateQuery::request_neighbor_queues() { neighbors_.emplace_back(*shard_ptr); } int i = 0; - for (block::McShardDescr& descr : neighbors_) { - LOG(DEBUG) << "requesting outbound queue of neighbor #" << i << " : " << descr.blk_.to_str(); - ++pending; - send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, - [self = get_self(), i](td::Result> res) { - td::actor::send_closure(std::move(self), &ValidateQuery::got_neighbor_out_queue, i, - std::move(res)); - }); - ++i; + if (is_lite_) { + for (block::McShardDescr& descr : neighbors_) { + LOG(DEBUG) << "getting outbound queue of neighbor #" << i << " from collated data : " << descr.blk_.to_str(); + td::Bits256 state_root_hash; + if (descr.blk_.seqno() == 0) { + state_root_hash = descr.blk_.root_hash; + } else { + Ref state_root = get_virt_state_root(descr.blk_.root_hash); + if (state_root.is_null()) { + return reject_query(PSTRING() << "cannot get hash of state root: " << descr.blk_); + } + state_root_hash = state_root->get_hash().bits(); + } + auto it = virt_roots_.find(state_root_hash); + if (it == virt_roots_.end()) { + return reject_query(PSTRING() << "cannot get state root form collated data: " << descr.blk_); + } + auto state = ShardStateQ::fetch(descr.blk_, {}, it->second); + if (state.is_error()) { + return reject_query("cannot fetch shard state from collated data", state.move_as_error()); + } + ++pending; + send_closure_later(get_self(), &ValidateQuery::got_neighbor_out_queue, i, state.move_as_ok()->message_queue()); + ++i; + } + } else { + for (block::McShardDescr& descr : neighbors_) { + LOG(DEBUG) << "requesting outbound queue of neighbor #" << i << " : " << descr.blk_.to_str(); + ++pending; + send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, + [self = get_self(), i](td::Result> res) { + td::actor::send_closure(std::move(self), &ValidateQuery::got_neighbor_out_queue, i, + std::move(res)); + }); + ++i; + } } return true; } @@ -5430,6 +5504,24 @@ bool ValidateQuery::check_mc_block_extra() { return true; } +Ref ValidateQuery::get_virt_state_root(td::Bits256 block_root_hash) { + auto it = virt_roots_.find(block_root_hash); + if (it == virt_roots_.end()) { + return {}; + } + Ref root = it->second; + block::gen::Block::Record block; + if (!tlb::unpack_cell(root, block)) { + return {}; + } + vm::CellSlice upd_cs{vm::NoVmSpec(), block.state_update}; + if (!(upd_cs.is_special() && upd_cs.prefetch_long(8) == 4 // merkle update + && upd_cs.size_ext() == 0x20228)) { + return {}; + } + return vm::MerkleProof::virtualize_raw(upd_cs.prefetch_ref(1), {0, 1}); +} + /* * * MAIN VALIDATOR FUNCTION @@ -5533,7 +5625,7 @@ bool ValidateQuery::try_validate() { } catch (vm::VmError& err) { return fatal_error(-666, err.get_msg()); } catch (vm::VmVirtError& err) { - return fatal_error(-666, err.get_msg()); + return reject_query(err.get_msg()); } return save_candidate(); } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 2aef04afad..3a400e7553 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -118,7 +118,7 @@ class ValidateQuery : public td::actor::Actor { ValidateQuery(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, bool is_fake = false); + td::Promise promise, unsigned mode = 0); private: int verbosity{3 * 1}; @@ -142,6 +142,7 @@ class ValidateQuery : public td::actor::Actor { bool is_key_block_{false}; bool update_shard_cc_{false}; bool is_fake_{false}; + bool is_lite_{false}; bool prev_key_block_exists_{false}; bool debug_checks_{false}; bool outq_cleanup_partial_{false}; @@ -286,6 +287,7 @@ class ValidateQuery : public td::actor::Actor { bool extract_collated_data(); bool try_validate(); bool compute_prev_state(); + bool compute_prev_state_lite_mode(); bool compute_next_state(); bool unpack_merge_prev_state(); bool unpack_prev_state(); @@ -368,6 +370,8 @@ class ValidateQuery : public td::actor::Actor { bool check_one_shard_fee(ShardIdFull shard, const block::CurrencyCollection& fees, const block::CurrencyCollection& create); bool check_mc_block_extra(); + + Ref get_virt_state_root(td::Bits256 block_root_hash); }; } // namespace validator diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 46ad50faaa..7563718d52 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -153,7 +153,7 @@ void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector prev, BlockIdExt last, From 415ace3da94ce3c73ab0dee3a50935cac4e4ab60 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 12 Jul 2022 21:15:04 +0300 Subject: [PATCH 003/524] Exporting and importing candidates using validator-engine-console; lite mode for validator group --- tl/generate/scheme/ton_api.tl | 6 + tl/generate/scheme/ton_api.tlo | Bin 68852 -> 69376 bytes .../validator-engine-console-query.cpp | 63 ++++++++++ .../validator-engine-console-query.h | 68 +++++++++++ .../validator-engine-console.cpp | 3 + validator-engine/validator-engine.cpp | 88 ++++++++++++++ validator-engine/validator-engine.hpp | 6 + validator/impl/validate-query.cpp | 1 + validator/interfaces/validator-manager.h | 2 + validator/manager-disk.hpp | 12 ++ validator/manager-hardfork.hpp | 13 ++ validator/manager.cpp | 115 +++++++++++++++++- validator/manager.hpp | 10 +- validator/validator-group.cpp | 18 ++- validator/validator-group.hpp | 6 +- validator/validator.h | 4 + 16 files changed, 410 insertions(+), 5 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 1909c71a01..eca1523543 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -633,6 +633,8 @@ engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.validatorSessionInfo current_block:tonNode.blockId self:int256 current_round:int next_producers:(vector int256) = engine.validator.ValidatorSessionInfo; engine.validator.validatorSessionsInfo sessions:(vector engine.validator.validatorSessionInfo) = engine.validator.ValidatorSessionsInfo; +engine.validator.requiredBlockCandidates block_ids:(vector tonNode.blockId) = engine.validator.RequiredBlockCandidates; + ---functions--- engine.validator.getTime = engine.validator.Time; @@ -684,6 +686,10 @@ engine.validator.importShardOverlayCertificate workchain:int shard:long signed_k engine.validator.getValidatorSessionsInfo = engine.validator.ValidatorSessionsInfo; +engine.validator.generateBlockCandidate block_id:tonNode.BlockId = db.Candidate; +engine.validator.getRequiredBlockCandidates = engine.validator.RequiredBlockCandidates; +engine.validator.importBlockCandidate block:db.candidate = engine.validator.Success; + ---types--- storage.pong = storage.Pong; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 9af19e27884de50bfd6dae0e45cdbc98835e7dab..252c6d0564df80fb0a38fe6fab78645f0e29d6ad 100644 GIT binary patch delta 372 zcmew|lcixE3-6=Z`c@23aAqU#a*oLvj3S#)aNOY4-CSOrsF9kNo|%`bSC*KQnUYwN zU!)h5T3DJ{l$zp{lb@XJoS2sa5>G9joX9D?d4X6DKcmxTUA6jjPwcvju()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(seqno_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(file_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status GenerateBlockCandidateQuery::send() { + auto b = ton::create_serialize_tl_object( + ton::create_tl_block_id_simple(ton::BlockId(wc_, shard_, seqno_))); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GenerateBlockCandidateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + TRY_STATUS_PREFIX(td::write_file(file_, data.as_slice()), "failed to write block to file"); + td::TerminalIO::out() << "successfully written candidate to file\n"; + return td::Status::OK(); +} + +td::Status GetRequiredBlockCandidatesQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status GetRequiredBlockCandidatesQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetRequiredBlockCandidatesQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX( + f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << td::json_encode(td::ToJson(*f), true); + return td::Status::OK(); +} + +td::Status ImportBlockCandidateQuery::run() { + TRY_RESULT_ASSIGN(file_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status ImportBlockCandidateQuery::send() { + TRY_RESULT(data, td::read_file(file_)); + TRY_RESULT_PREFIX(candidate, ton::fetch_tl_object(data.as_slice(), true), + "invalid file: "); + auto b = ton::create_serialize_tl_object(std::move(candidate)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ImportBlockCandidateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully imported a block candidate\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index dea2273dbd..e4236f6dc8 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1092,6 +1092,74 @@ class GetValidatorSessionsInfoQuery : public Query { std::string name() const override { return get_name(); } +}; + +class GenerateBlockCandidateQuery : public Query { + public: + GenerateBlockCandidateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "genblock"; + } + static std::string get_help() { + return "genblock \t" + "generate a block candidate for a given shard (seqno mush match the next seqno for the shard), " + "candidate is saved to "; + } + std::string name() const override { + return get_name(); + } + + private: + td::int32 wc_; + td::int64 shard_; + td::int32 seqno_; + std::string file_; +}; + +class GetRequiredBlockCandidatesQuery : public Query { + public: + GetRequiredBlockCandidatesQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "getrequiredblockcandidates"; + } + static std::string get_help() { + return "getrequiredblockcandidates\t" + "get a list of block candidates that the validator is currently waiting for"; + } + std::string name() const override { + return get_name(); + } +}; + +class ImportBlockCandidateQuery : public Query { + public: + ImportBlockCandidateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "importblockcandidate"; + } + static std::string get_help() { + return "importblockcandidate \t" + "load a block candidate from a given file"; + } + std::string name() const override { + return get_name(); + } private: + std::string file_; }; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 2c04971b57..b4dcad22b7 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -141,6 +141,9 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 7783dc38ea..2e158e079b 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -3307,6 +3307,94 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getValida std::move(P)); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_generateBlockCandidate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (validator_manager_.empty()) { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); + return; + } + ton::BlockId block_id = ton::create_block_id_simple(query.block_id_); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::generate_block_candidate, + block_id, [promise = std::move(promise)](td::Result R) mutable { + if (R.is_ok()) { + auto block = R.move_as_ok(); + auto result = ton::create_serialize_tl_object( + ton::PublicKey{ton::pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), + ton::create_tl_block_id(block.id), std::move(block.data), + std::move(block.collated_data)); + promise.set_result(std::move(result)); + } else { + promise.set_value(create_control_query_error(R.move_as_error())); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getRequiredBlockCandidates &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (validator_manager_.empty()) { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); + return; + } + td::actor::send_closure( + validator_manager_, &ton::validator::ValidatorManagerInterface::get_required_block_candidates, + [promise = std::move(promise)](td::Result> R) mutable { + if (R.is_ok()) { + std::vector> block_ids; + for (const ton::BlockId &block_id : R.move_as_ok()) { + block_ids.push_back(ton::create_tl_block_id_simple(block_id)); + } + auto result = ton::create_serialize_tl_object( + std::move(block_ids)); + promise.set_result(std::move(result)); + } else { + promise.set_value(create_control_query_error(R.move_as_error())); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importBlockCandidate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (validator_manager_.empty()) { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); + return; + } + + auto collated_data_hash = td::sha256_bits256(query.block_->collated_data_); + auto key = ton::PublicKey{query.block_->source_}; + auto e_key = ton::Ed25519_PublicKey{key.ed25519_value().raw()}; + ton::BlockCandidate candidate{e_key, ton::create_block_id(query.block_->id_), collated_data_hash, + std::move(query.block_->data_), std::move(query.block_->collated_data_)}; + + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::import_block_candidate, + std::move(candidate), + [promise = std::move(promise)](td::Result R) mutable { + if (R.is_ok()) { + promise.set_result(ton::serialize_tl_object( + ton::create_tl_object(), true)); + } else { + promise.set_value(create_control_query_error(R.move_as_error())); + } + }); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index e0364baba6..beb24237d3 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -409,6 +409,12 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getValidatorSessionsInfo &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_generateBlockCandidate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getRequiredBlockCandidates &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_importBlockCandidate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index b9347fcf76..bd0d3b2a46 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -449,6 +449,7 @@ bool ValidateQuery::init_parse() { return reject_query("after_merge value mismatch in block header"); } rand_seed_ = extra.rand_seed; + created_by_ = extra.created_by; if (created_by_ != extra.created_by) { return reject_query("block candidate "s + id_.to_str() + " has creator " + created_by_.to_hex() + " but the block header contains different value " + extra.created_by.to_hex()); diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 4ee276d385..c72ec307cb 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -169,6 +169,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0; + virtual void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise promise) = 0; + static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 3f98273e0d..6bad117158 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -371,6 +371,18 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { UNREACHABLE(); } + void generate_block_candidate(BlockId block_id, td::Promise promise) override { + UNREACHABLE(); + } + void get_required_block_candidates(td::Promise> promise) override { + UNREACHABLE(); + } + void import_block_candidate(BlockCandidate candidate, td::Promise promise) override { + UNREACHABLE(); + } + void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise promise) override { + UNREACHABLE(); + } private: PublicKeyHash local_id_; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 17042dbf12..a8aaecfc3d 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -430,6 +430,19 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { UNREACHABLE(); } + void generate_block_candidate(BlockId block_id, td::Promise promise) override { + UNREACHABLE(); + } + void get_required_block_candidates(td::Promise> promise) override { + UNREACHABLE(); + } + void import_block_candidate(BlockCandidate candidate, td::Promise promise) override { + UNREACHABLE(); + } + void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise promise) override { + UNREACHABLE(); + } + private: td::Ref opts_; diff --git a/validator/manager.cpp b/validator/manager.cpp index 81cd2e0eaf..0d8929ace3 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2031,7 +2031,7 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto G = td::actor::create_actor( "validatorgroup", shard, validator_id, session_id, validator_set, opts, keyring_, adnl_, rldp_, overlays_, db_root_, actor_id(this), init_session, - opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno())); + opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), true); return G; } } @@ -2583,6 +2583,119 @@ void ValidatorManagerImpl::get_validator_sessions_info( IntermediateData::step({std::move(groups), {}, std::move(promise)}); } +void ValidatorManagerImpl::generate_block_candidate(BlockId block_id, td::Promise promise) { + if (!block_id.is_valid_full()) { + promise.set_error(td::Status::Error("invalid block id")); + return; + } + if (last_masterchain_state_.is_null()) { + promise.set_error(td::Status::Error("not started")); + return; + } + ShardIdFull shard_id = block_id.shard_full(); + std::vector prev; + auto shard = last_masterchain_state_->get_shard_from_config(shard_id); + if (shard.not_null()) { + if (shard->before_split()) { + promise.set_error(td::Status::Error("shard is before_split")); + return; + } + if (shard->before_merge()) { + promise.set_error(td::Status::Error("shard is before_merge")); + return; + } + prev.push_back(shard->top_block_id()); + } else { + auto parent = shard_id.pfx_len() == 0 ? td::Ref() + : last_masterchain_state_->get_shard_from_config(shard_parent(shard_id)); + if (parent.not_null() && parent->before_split()) { + prev.push_back(parent->top_block_id()); + } else { + auto child_l = last_masterchain_state_->get_shard_from_config(shard_child(shard_id, true)); + auto child_r = last_masterchain_state_->get_shard_from_config(shard_child(shard_id, false)); + if (child_l.not_null() && child_r.not_null() && child_l->before_merge() && child_r->before_merge()) { + prev.push_back(child_l->top_block_id()); + prev.push_back(child_r->top_block_id()); + } + } + if (prev.empty()) { + promise.set_error(td::Status::Error("no such shard")); + return; + } + } + + BlockSeqno next_seqno = 0; + for (const BlockIdExt& prev_id : prev) { + next_seqno = std::max(next_seqno, prev_id.seqno() + 1); + } + if (next_seqno != block_id.seqno) { + promise.set_error(td::Status::Error(PSTRING() << "seqno mismatch: asked for seqno " << block_id.seqno + << ", but actual next seqno is " << next_seqno)); + return; + } + + Ed25519_PublicKey local_id{Bits256::zero()}; + td::Ref validator_set = last_masterchain_state_->get_validator_set(shard_id); + if (validator_set.is_null()) { + promise.set_error(td::Status::Error("cannot get validator set")); + return; + } + run_collate_query(shard_id, last_masterchain_state_->get_unix_time(), last_masterchain_block_id_, std::move(prev), + local_id, std::move(validator_set), actor_id(this), td::Timestamp::in(10.0), std::move(promise)); +} + +void ValidatorManagerImpl::get_required_block_candidates(td::Promise> promise) { + std::vector block_ids; + for (const auto& p : pending_block_candidates_) { + block_ids.push_back(p.first); + } + promise.set_result(std::move(block_ids)); +} + +void ValidatorManagerImpl::import_block_candidate(BlockCandidate candidate, td::Promise promise) { + auto it = pending_block_candidates_.find(candidate.id.id); + if (it != pending_block_candidates_.end()) { + while (!it->second.empty()) { + auto promise = std::move(it->second.back().first); + it->second.pop_back(); + if (it->second.empty()) { + promise.set_result(std::move(candidate)); + } else { + promise.set_result(candidate.clone()); + } + } + pending_block_candidates_.erase(it); + } + promise.set_result(td::Unit()); +} + +void ValidatorManagerImpl::wait_block_candidate(BlockId block_id, td::Timestamp timeout, + td::Promise promise) { + pending_block_candidates_[block_id].emplace_back(std::move(promise), timeout); + delay_action([SelfId = actor_id(this), block_id, timeout]() { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::cleanup_old_pending_candidates, block_id, timeout); + }, timeout); +} + +void ValidatorManagerImpl::cleanup_old_pending_candidates(BlockId block_id, td::Timestamp now) { + auto it = pending_block_candidates_.find(block_id); + if (it == pending_block_candidates_.end()) { + return; + } + it->second.erase(std::remove_if(it->second.begin(), it->second.end(), + [&](std::pair, td::Timestamp> &p) { + if (p.second.is_in_past(now)) { + p.first.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); + return true; + } + return false; + }), + it->second.end()); + if (it->second.empty()) { + pending_block_candidates_.erase(it); + } +} + td::actor::ActorOwn ValidatorManagerFactory::create( td::Ref opts, std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, diff --git a/validator/manager.hpp b/validator/manager.hpp index c855e1cc7e..1fbf60a9d0 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -534,7 +534,12 @@ class ValidatorManagerImpl : public ValidatorManager { void get_validator_sessions_info( td::Promise> promise) override; - private: + void generate_block_candidate(BlockId block_id, td::Promise promise) override; + void get_required_block_candidates(td::Promise> promise) override; + void import_block_candidate(BlockCandidate candidate, td::Promise promise) override; + void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise promise) override; + + private: td::Timestamp resend_shard_blocks_at_; td::Timestamp check_waiters_at_; td::Timestamp check_shard_clients_; @@ -598,6 +603,9 @@ class ValidatorManagerImpl : public ValidatorManager { private: std::map> shard_client_waiters_; + + std::map, td::Timestamp>>> pending_block_candidates_; + void cleanup_old_pending_candidates(BlockId block_id, td::Timestamp now); }; } // namespace validator diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 333259b4c4..f5c3579823 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -35,6 +35,22 @@ void ValidatorGroup::generate_block_candidate(td::uint32 round_id, td::Promise R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + BlockCandidate candidate = R.move_as_ok(); + candidate.pubkey = pubkey; + promise.set_result(std::move(candidate)); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::wait_block_candidate, create_next_block_id_simple(), + td::Timestamp::in(15.0), std::move(P)); + return; + } run_collate_query(shard_, min_ts_, min_masterchain_block_id_, prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_, manager_, td::Timestamp::in(10.0), std::move(promise)); @@ -79,7 +95,7 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id; block.id = next_block_id; run_validate_query(shard_, min_ts_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, - manager_, td::Timestamp::in(10.0), std::move(P)); + manager_, td::Timestamp::in(10.0), std::move(P), lite_mode_ ? ValidateMode::lite : 0); } void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash src, td::BufferSlice block_data, diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 3216f5fe67..5d760899a3 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -64,7 +64,7 @@ class ValidatorGroup : public td::actor::Actor { td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, td::actor::ActorId validator_manager, bool create_session, - bool allow_unsafe_self_blocks_resync) + bool allow_unsafe_self_blocks_resync, bool lite_mode = false) : shard_(shard) , local_id_(std::move(local_id)) , session_id_(session_id) @@ -77,7 +77,8 @@ class ValidatorGroup : public td::actor::Actor { , db_root_(std::move(db_root)) , manager_(validator_manager) , init_(create_session) - , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) { + , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) + , lite_mode_(lite_mode) { } private: @@ -118,6 +119,7 @@ class ValidatorGroup : public td::actor::Actor { bool init_ = false; bool started_ = false; bool allow_unsafe_self_blocks_resync_; + bool lite_mode_ = false; td::uint32 last_known_round_id_ = 0; }; diff --git a/validator/validator.h b/validator/validator.h index b3d32e307f..09081f1e31 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -216,6 +216,10 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void prepare_stats(td::Promise>> promise) = 0; virtual void get_validator_sessions_info( td::Promise> promise) = 0; + + virtual void generate_block_candidate(BlockId block_id, td::Promise promise) = 0; + virtual void get_required_block_candidates(td::Promise> promise) = 0; + virtual void import_block_candidate(BlockCandidate candidate, td::Promise promise) = 0; }; } // namespace validator From bdfca7afef41b90705900b9166334b567bec916b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 13 Jul 2022 12:13:51 +0300 Subject: [PATCH 004/524] Remove unused code --- validator/fabric.h | 8 ++-- validator/impl/CMakeLists.txt | 1 - validator/impl/collate-query-impl.h | 63 ----------------------------- validator/impl/collator-impl.h | 3 +- validator/impl/collator.cpp | 3 +- validator/impl/collator.h | 19 --------- validator/impl/fabric.cpp | 14 +++---- validator/impl/validate-query.cpp | 3 +- validator/impl/validate-query.hpp | 3 +- validator/manager-disk.cpp | 4 +- validator/manager.cpp | 16 +++----- validator/validator-group.cpp | 9 ++--- validator/validator-group.hpp | 3 +- 13 files changed, 28 insertions(+), 121 deletions(-) delete mode 100644 validator/impl/collate-query-impl.h diff --git a/validator/fabric.h b/validator/fabric.h index 3515c0da84..158ea7e00b 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -72,12 +72,12 @@ void run_check_proof_query(BlockIdExt id, td::Ref proof, td::actor::Actor td::Ref rel_key_block_proof, bool skip_check_signatures = false); void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); -void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, - std::vector prev, BlockCandidate candidate, td::Ref validator_set, +void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, + BlockCandidate candidate, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, unsigned mode = 0); -void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& min_masterchain_block_id, - std::vector prev, Ed25519_PublicKey local_id, td::Ref validator_set, +void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, + Ed25519_PublicKey local_id, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, diff --git a/validator/impl/CMakeLists.txt b/validator/impl/CMakeLists.txt index 459e772443..224599f864 100644 --- a/validator/impl/CMakeLists.txt +++ b/validator/impl/CMakeLists.txt @@ -25,7 +25,6 @@ set(TON_VALIDATOR_SOURCE accept-block.hpp block.hpp check-proof.hpp - collate-query-impl.h collator-impl.h collator.h config.hpp diff --git a/validator/impl/collate-query-impl.h b/validator/impl/collate-query-impl.h deleted file mode 100644 index 8383264684..0000000000 --- a/validator/impl/collate-query-impl.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -#include "validator/interfaces/validator-manager.h" - -namespace ton { - -namespace validator { - -class CollateQuery : public td::actor::Actor { - public: - CollateQuery(ShardIdFull shard, td::uint32 min_ts, BlockIdExt min_masterchain_block_id, std::vector prev, - td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise); - CollateQuery(ShardIdFull shard, td::uint32 min_ts, BlockIdExt min_masterchain_block_id, ZeroStateIdExt zero_state_id, - td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise); - - void alarm() override; - - void abort_query(td::Status reason); - void finish_query(); - - void start_up() override; - void got_prev_state(td::Ref state); - void written_block_data(); - void written_block_collated_data(); - - private: - ShardIdFull shard_; - UnixTime min_ts_; - BlockIdExt min_masterchain_block_id_; - std::vector prev_; - ZeroStateIdExt zero_state_id_; - td::Ref validator_set_; - td::actor::ActorId manager_; - td::Timestamp timeout_; - td::Promise promise_; - - BlockCandidate candidate_; - UnixTime ts_; -}; - -} // namespace validator - -} // namespace ton diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 997fea3663..8a17e95317 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -64,7 +64,6 @@ class Collator final : public td::actor::Actor { bool libraries_changed_{false}; bool prev_key_block_exists_{false}; bool is_hardfork_{false}; - UnixTime min_ts; BlockIdExt min_mc_block_id; std::vector prev_blocks; std::vector> prev_states; @@ -86,7 +85,7 @@ class Collator final : public td::actor::Actor { static constexpr bool shard_splitting_enabled = true; public: - Collator(ShardIdFull shard, bool is_hardfork, td::uint32 min_ts, BlockIdExt min_masterchain_block_id, + Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, Ref validator_set, Ed25519_PublicKey collator_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); ~Collator() override = default; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 4d26f4b5d7..49c9459614 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -54,13 +54,12 @@ static inline bool dbg(int c) { return true; } -Collator::Collator(ShardIdFull shard, bool is_hardfork, UnixTime min_ts, BlockIdExt min_masterchain_block_id, +Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, td::Ref validator_set, Ed25519_PublicKey collator_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise) : shard_(shard) , is_hardfork_(is_hardfork) - , min_ts(min_ts) , min_mc_block_id{min_masterchain_block_id} , prev_blocks(std::move(prev)) , created_by_(collator_id) diff --git a/validator/impl/collator.h b/validator/impl/collator.h index c92fa80e3b..5acc20e8d8 100644 --- a/validator/impl/collator.h +++ b/validator/impl/collator.h @@ -24,26 +24,7 @@ #include "vm/cells.h" namespace ton { -using td::Ref; extern int collator_settings; // +1 = force want_split, +2 = force want_merge -class Collator : public td::actor::Actor { - protected: - Collator() = default; - - public: - virtual ~Collator() = default; - static td::actor::ActorOwn create_collator( - td::actor::ActorId block_db, - ShardIdFull shard /* , td::actor::ActorId validator_manager */); - virtual void generate_block_candidate(ShardIdFull shard, td::Promise promise) = 0; - virtual td::Result register_external_message_cell(Ref ext_msg) = 0; - virtual td::Result register_external_message(td::Slice ext_msg_boc) = 0; - virtual td::Result register_ihr_message_cell(Ref ihr_msg) = 0; - virtual td::Result register_ihr_message(td::Slice ihr_msg_boc) = 0; - virtual td::Result register_shard_signatures_cell(Ref shard_blk_signatures) = 0; - virtual td::Result register_shard_signatures(td::Slice shard_blk_signatures_boc) = 0; -}; - } // namespace ton diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index c3f4338d4f..6958ad8906 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -188,7 +188,7 @@ void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::act .release(); } -void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, +void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, unsigned mode) { @@ -200,14 +200,14 @@ void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_maste } bool is_fake = mode & ValidateMode::fake; td::actor::create_actor( - PSTRING() << (is_fake ? "fakevalidate" : "validateblock") << shard.to_str() << ":" << (seqno + 1), shard, min_ts, + PSTRING() << (is_fake ? "fakevalidate" : "validateblock") << shard.to_str() << ":" << (seqno + 1), shard, min_masterchain_block_id, std::move(prev), std::move(candidate), std::move(validator_set), std::move(manager), timeout, std::move(promise), mode) .release(); } -void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& min_masterchain_block_id, - std::vector prev, Ed25519_PublicKey collator_id, td::Ref validator_set, +void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, + Ed25519_PublicKey collator_id, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise) { BlockSeqno seqno = 0; @@ -217,8 +217,8 @@ void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& m } } td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, false, - min_ts, min_masterchain_block_id, std::move(prev), std::move(validator_set), - collator_id, std::move(manager), timeout, std::move(promise)) + min_masterchain_block_id, std::move(prev), std::move(validator_set), collator_id, + std::move(manager), timeout, std::move(promise)) .release(); } @@ -231,7 +231,7 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b seqno = p.seqno(); } } - td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, 0, + td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, min_masterchain_block_id, std::move(prev), td::Ref{}, Ed25519_PublicKey{Bits256::zero()}, std::move(manager), timeout, std::move(promise)) .release(); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index bd0d3b2a46..aac85c8f1f 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -49,13 +49,12 @@ std::string ErrorCtx::as_string() const { return a; } -ValidateQuery::ValidateQuery(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, +ValidateQuery::ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, unsigned mode) : shard_(shard) , id_(candidate.id) - , min_ts(min_ts) , min_mc_block_id(min_masterchain_block_id) , prev_blocks(std::move(prev)) , block_candidate(std::move(candidate)) diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 3a400e7553..6fe8010f5b 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -115,7 +115,7 @@ class ValidateQuery : public td::actor::Actor { } public: - ValidateQuery(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, std::vector prev, + ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, unsigned mode = 0); @@ -125,7 +125,6 @@ class ValidateQuery : public td::actor::Actor { int pending{0}; const ShardIdFull shard_; const BlockIdExt id_; - UnixTime min_ts; BlockIdExt min_mc_block_id; std::vector prev_blocks; std::vector> prev_states; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 7563718d52..26da9e165c 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -128,7 +128,7 @@ void ValidatorManagerImpl::sync_complete(td::Promise promise) { } Ed25519_PublicKey created_by{td::Bits256::zero()}; td::as(created_by.as_bits256().data() + 32 - 4) = ((unsigned)std::time(nullptr) >> 8); - run_collate_query(shard_id, 0, last_masterchain_block_id_, prev, created_by, val_set, actor_id(this), + run_collate_query(shard_id, last_masterchain_block_id_, prev, created_by, val_set, actor_id(this), td::Timestamp::in(10.0), std::move(P)); } @@ -152,7 +152,7 @@ void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vectorsecond.empty()) { - td::actor::send_closure(it2->second, &ValidatorGroup::start, prev, last_masterchain_block_id_, - last_masterchain_state_->get_unix_time()); + td::actor::send_closure(it2->second, &ValidatorGroup::start, prev, last_masterchain_block_id_); } new_validator_groups_.emplace(val_group_id, std::move(it2->second)); } else { auto G = create_validator_group(val_group_id, shard, val_set, opts, started_); if (!G.empty()) { - td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_, - last_masterchain_state_->get_unix_time()); + td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_); } new_validator_groups_.emplace(val_group_id, std::move(G)); } @@ -1887,15 +1885,13 @@ void ValidatorManagerImpl::update_shards() { auto it2 = next_validator_groups_.find(val_group_id); if (it2 != next_validator_groups_.end()) { if (!it2->second.empty()) { - td::actor::send_closure(it2->second, &ValidatorGroup::start, prev, last_masterchain_block_id_, - last_masterchain_state_->get_unix_time()); + td::actor::send_closure(it2->second, &ValidatorGroup::start, prev, last_masterchain_block_id_); } new_validator_groups_.emplace(val_group_id, std::move(it2->second)); } else { auto G = create_validator_group(val_group_id, shard, val_set, opts, started_); if (!G.empty()) { - td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_, - last_masterchain_state_->get_unix_time()); + td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_); } new_validator_groups_.emplace(val_group_id, std::move(G)); } @@ -2640,8 +2636,8 @@ void ValidatorManagerImpl::generate_block_candidate(BlockId block_id, td::Promis promise.set_error(td::Status::Error("cannot get validator set")); return; } - run_collate_query(shard_id, last_masterchain_state_->get_unix_time(), last_masterchain_block_id_, std::move(prev), - local_id, std::move(validator_set), actor_id(this), td::Timestamp::in(10.0), std::move(promise)); + run_collate_query(shard_id, last_masterchain_block_id_, std::move(prev), local_id, std::move(validator_set), + actor_id(this), td::Timestamp::in(10.0), std::move(promise)); } void ValidatorManagerImpl::get_required_block_candidates(td::Promise> promise) { diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index f5c3579823..ab7f206277 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -51,7 +51,7 @@ void ValidatorGroup::generate_block_candidate(td::uint32 round_id, td::Promise prev, BlockIdExt min_masterchain_block_id, UnixTime min_ts) { +void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterchain_block_id) { prev_block_ids_ = prev; min_masterchain_block_id_ = min_masterchain_block_id; - min_ts_ = min_ts; started_ = true; if (init_) { diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 5d760899a3..91d801cd09 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -47,7 +47,7 @@ class ValidatorGroup : public td::actor::Actor { BlockId create_next_block_id_simple() const; BlockIdExt create_next_block_id(RootHash root_hash, FileHash file_hash) const; - void start(std::vector prev, BlockIdExt min_masterchain_block_id, UnixTime min_ts); + void start(std::vector prev, BlockIdExt min_masterchain_block_id); void create_session(); void destroy(); void start_up() override { @@ -103,7 +103,6 @@ class ValidatorGroup : public td::actor::Actor { std::vector prev_block_ids_; BlockIdExt min_masterchain_block_id_; - UnixTime min_ts_; td::Ref validator_set_; validatorsession::ValidatorSessionOptions config_; From 996c23e506bdb930bc8517f6c6bb91809a89b051 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 18 Jul 2022 18:35:06 +0300 Subject: [PATCH 005/524] Shardchain validation without monitoring shardchains --- create-hardfork/create-hardfork.cpp | 4 +- overlay/overlay-manager.cpp | 9 +++ overlay/overlay-manager.h | 2 + overlay/overlay.cpp | 16 ++++- overlay/overlay.hpp | 7 ++ overlay/overlays.h | 2 + test/test-ton-collator.cpp | 4 +- validator-engine/validator-engine.cpp | 7 +- validator-engine/validator-engine.hpp | 4 ++ validator/fabric.h | 4 ++ validator/full-node-shard.cpp | 92 +++++++++++++++++++-------- validator/full-node-shard.h | 4 +- validator/full-node-shard.hpp | 6 +- validator/full-node.cpp | 24 +++---- validator/full-node.hpp | 2 +- validator/impl/accept-block.cpp | 31 +++++++++ validator/impl/accept-block.hpp | 6 ++ validator/impl/fabric.cpp | 11 ++++ validator/manager.cpp | 4 +- validator/validator-group.cpp | 49 ++++++-------- validator/validator-group.hpp | 6 +- validator/validator.h | 4 +- 22 files changed, 210 insertions(+), 88 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 15533768d1..849a31b125 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -236,9 +236,7 @@ class HardforkCreator : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void add_shard(ton::ShardIdFull) override { - } - void del_shard(ton::ShardIdFull) override { + void subscribe_to_shard(ton::ShardIdFull) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index 41750b4fae..2eac5a0c25 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -98,6 +98,15 @@ void OverlayManager::create_public_overlay(adnl::AdnlNodeIdShort local_id, Overl std::move(callback), std::move(rules), scope)); } +void OverlayManager::create_public_overlay_no_subscribe(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + OverlayPrivacyRules rules, td::string scope) { + CHECK(!dht_node_.empty()); + auto id = overlay_id.compute_short_id(); + register_overlay(local_id, id, + Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), nullptr, + std::move(rules), scope)); +} + void OverlayManager::create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules) { diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index d9739bf382..b03261e806 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -52,6 +52,8 @@ class OverlayManager : public Overlays { void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) override; + void create_public_overlay_no_subscribe(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + OverlayPrivacyRules rules, td::string scope) override; void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules) override; diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 3c707e6015..a6d0bc89f4 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -149,6 +149,9 @@ void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, if (R.is_error()) { // allow custom query to be here + if (!subscribed()) { + return; + } callback_->receive_query(src, overlay_id_, std::move(data), std::move(promise)); return; } @@ -225,6 +228,9 @@ void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice dat } auto X = fetch_tl_object(data.clone(), true); if (X.is_error()) { + if (!subscribed()) { + return; + } VLOG(OVERLAY_DEBUG) << this << ": received custom message"; callback_->receive_message(src, overlay_id_, std::move(data)); return; @@ -268,7 +274,7 @@ void OverlayImpl::alarm() { } if (public_) { - if (peers_.size() > 0) { + if (peers_.size() > 0 && subscribed()) { auto P = get_random_peer(); if (P) { send_random_peers(P->get_id(), {}); @@ -336,7 +342,7 @@ void OverlayImpl::receive_dht_nodes(td::Result res, bool dummy) { } void OverlayImpl::update_dht_nodes(OverlayNode node) { - if (!public_) { + if (!public_ || !subscribed()) { return; } @@ -497,6 +503,9 @@ void OverlayImpl::send_new_fec_broadcast_part(PublicKeyHash local_id, Overlay::B } void OverlayImpl::deliver_broadcast(PublicKeyHash source, td::BufferSlice data) { + if (!subscribed()) { + return; + } callback_->receive_broadcast(source, overlay_id_, std::move(data)); } @@ -569,6 +578,9 @@ void OverlayImpl::set_privacy_rules(OverlayPrivacyRules rules) { } void OverlayImpl::check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise promise) { + if (!subscribed()) { + return; + } callback_->check_broadcast(src, overlay_id_, std::move(data), std::move(promise)); } diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 1d5f986113..ddcb026713 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -251,8 +251,15 @@ class OverlayImpl : public Overlay { } private: + bool subscribed() const { + return (bool)callback_; + } + template void process_query(adnl::AdnlNodeIdShort src, T &query, td::Promise promise) { + if (!subscribed()) { + return; + } callback_->receive_query(src, overlay_id_, serialize_tl_object(&query, true), std::move(promise)); } diff --git a/overlay/overlays.h b/overlay/overlays.h index 45316254ed..6781b6c32b 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -194,6 +194,8 @@ class Overlays : public td::actor::Actor { virtual void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) = 0; + virtual void create_public_overlay_no_subscribe(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + OverlayPrivacyRules rules, td::string scope) = 0; virtual void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules) = 0; diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 9ed5c7814a..1e46bc7df6 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -323,9 +323,7 @@ class TestNode : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void add_shard(ton::ShardIdFull) override { - } - void del_shard(ton::ShardIdFull) override { + void subscribe_to_shard(ton::ShardIdFull) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 2e158e079b..113455249c 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1300,10 +1300,10 @@ td::Status ValidatorEngine::load_global_config() { validator_options_ = ton::validator::ValidatorManagerOptions::create(zero_state, init_block); validator_options_.write().set_shard_check_function( - [](ton::ShardIdFull shard, ton::CatchainSeqno cc_seqno, + [masterchain_only = masterchain_only_](ton::ShardIdFull shard, ton::CatchainSeqno cc_seqno, ton::validator::ValidatorManagerOptions::ShardCheckMode mode) -> bool { if (mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_monitor) { - return true; + return shard.is_masterchain() || !masterchain_only; } CHECK(mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_validate); return true; @@ -3648,6 +3648,9 @@ int main(int argc, char *argv[]) { }); return td::Status::OK(); }); + p.add_option('M', "masterchain-only", "don't track shardchains", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_masterchain_only); }); + }); td::uint32 threads = 7; p.add_checked_option( 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index beb24237d3..0a7f8e2642 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -202,6 +202,7 @@ class ValidatorEngine : public td::actor::Actor { bool started_ = false; ton::BlockSeqno truncate_seqno_{0}; std::string session_logs_file_; + bool masterchain_only_ = false; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -253,6 +254,9 @@ class ValidatorEngine : public td::actor::Actor { void add_key_to_set(ton::PublicKey key) { keys_[key.compute_short_id()] = key; } + void set_masterchain_only() { + masterchain_only_ = true; + } void start_up() override; ValidatorEngine() { } diff --git a/validator/fabric.h b/validator/fabric.h index 158ea7e00b..6f06dcc9dc 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -59,6 +59,10 @@ void run_fake_accept_block_query(BlockIdExt id, td::Ref data, std::ve td::Promise promise); void run_hardfork_accept_block_query(BlockIdExt id, td::Ref data, td::actor::ActorId manager, td::Promise promise); +void run_broadcast_only_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, + td::Ref validator_set, td::Ref signatures, + td::Ref approve_signatures, + td::actor::ActorId manager, td::Promise promise); void run_apply_block_query(BlockIdExt id, td::Ref block, BlockIdExt masterchain_block_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 1094c89680..8e579f3c8c 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -69,31 +69,41 @@ void Neighbour::update_roundtrip(double t) { } void FullNodeShardImpl::create_overlay() { - class Callback : public overlay::Overlays::Callback { - public: - void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { - // just ignore - } - void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, - td::Promise promise) override { - td::actor::send_closure(node_, &FullNodeShardImpl::receive_query, src, std::move(data), std::move(promise)); - } - void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { - td::actor::send_closure(node_, &FullNodeShardImpl::receive_broadcast, src, std::move(data)); - } - void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, - td::Promise promise) override { - td::actor::send_closure(node_, &FullNodeShardImpl::check_broadcast, src, std::move(data), std::move(promise)); - } - Callback(td::actor::ActorId node) : node_(node) { - } + if (subscribed_) { + class Callback : public overlay::Overlays::Callback { + public: + void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, + td::BufferSlice data) override { + // just ignore + } + void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(node_, &FullNodeShardImpl::receive_query, src, std::move(data), std::move(promise)); + } + void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + td::actor::send_closure(node_, &FullNodeShardImpl::receive_broadcast, src, std::move(data)); + } + void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(node_, &FullNodeShardImpl::check_broadcast, src, std::move(data), std::move(promise)); + } + Callback(td::actor::ActorId node) : node_(node) { + } - private: - td::actor::ActorId node_; - }; + private: + td::actor::ActorId node_; + }; - td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay, adnl_id_, overlay_id_full_.clone(), - std::make_unique(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() << ", \"workchain_id\": " << get_workchain() << " }"); + td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay, adnl_id_, overlay_id_full_.clone(), + std::make_unique(actor_id(this)), rules_, + PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() + << ", \"workchain_id\": " << get_workchain() << " }"); + } else { + td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay_no_subscribe, adnl_id_, + overlay_id_full_.clone(), rules_, + PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() + << ", \"workchain_id\": " << get_workchain() << " }"); + } td::actor::send_closure(rldp_, &rldp::Rldp::add_id, adnl_id_); if (cert_) { @@ -119,6 +129,15 @@ void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promis create_overlay(); } +void FullNodeShardImpl::subscribe_to_shard() { + if (subscribed_ || !client_.empty()) { + return; + } + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); + subscribed_ = true; + create_overlay(); +} + void FullNodeShardImpl::try_get_next_block(td::Timestamp timeout, td::Promise promise) { if (timeout.is_in_past()) { promise.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); @@ -576,8 +595,14 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ex } void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { + auto block_id = create_block_id(query.block_->block_); + if (block_id.shard_full() != shard_) { + LOG(WARNING) << "dropping newShardBlockBroadcast: expected shard " << shard_.to_str() << ", got shard " + << block_id.shard_full().to_str(); + return; + } td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, - create_block_id(query.block_->block_), query.block_->cc_seqno_, + block_id, query.block_->cc_seqno_, std::move(query.block_->data_)); } @@ -588,6 +613,11 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_bl } BlockIdExt block_id = create_block_id(query.id_); + if (block_id.shard_full() != shard_) { + LOG(WARNING) << "dropping blockBroadcast: expected shard " << shard_.to_str() << ", got shard " + << block_id.shard_full().to_str(); + return; + } BlockBroadcast B{block_id, std::move(signatures), static_cast(query.catchain_seqno_), @@ -817,6 +847,10 @@ void FullNodeShardImpl::start_up() { } } +void FullNodeShardImpl::tear_down() { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); +} + void FullNodeShardImpl::sign_new_certificate(PublicKeyHash sign_by) { if (sign_by.is_zero()) { return; @@ -1053,7 +1087,7 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client) + td::actor::ActorId client, bool subscribe) : shard_(shard) , local_id_(local_id) , adnl_id_(adnl_id) @@ -1063,16 +1097,18 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, , rldp_(rldp) , overlays_(overlays) , validator_manager_(validator_manager) - , client_(client) { + , client_(client) + , subscribed_(subscribe) { } td::actor::ActorOwn FullNodeShard::create( ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, - td::actor::ActorId validator_manager, td::actor::ActorId client) { + td::actor::ActorId validator_manager, td::actor::ActorId client, + bool subscribe) { return td::actor::create_actor("tonnode", shard, local_id, adnl_id, zero_state_file_hash, keyring, - adnl, rldp, overlays, validator_manager, client); + adnl, rldp, overlays, validator_manager, client, subscribe); } } // namespace fullnode diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index c1712baf26..e45e2325c3 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -36,6 +36,7 @@ class FullNodeShard : public td::actor::Actor { virtual ShardIdFull get_shard_full() const = 0; virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; + virtual void subscribe_to_shard() = 0; virtual void send_ihr_message(td::BufferSlice data) = 0; virtual void send_external_message(td::BufferSlice data) = 0; @@ -70,7 +71,8 @@ class FullNodeShard : public td::actor::Actor { ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, - td::actor::ActorId validator_manager, td::actor::ActorId client); + td::actor::ActorId validator_manager, td::actor::ActorId client, + bool subscribe); }; } // namespace fullnode diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index a2dd5cc461..33705bc799 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -80,6 +80,7 @@ class FullNodeShardImpl : public FullNodeShard { void create_overlay(); void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; + virtual void subscribe_to_shard() override; //td::Result fetch_block(td::BufferSlice data); void prevalidate_block(BlockIdExt block_id, td::BufferSlice data, td::BufferSlice proof, @@ -167,6 +168,7 @@ class FullNodeShardImpl : public FullNodeShard { void set_handle(BlockHandle handle, td::Promise promise) override; void start_up() override; + void tear_down() override; void alarm() override; void update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) override; @@ -202,7 +204,7 @@ class FullNodeShardImpl : public FullNodeShard { td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client); + td::actor::ActorId client, bool subscribe); private: bool use_new_download() const { @@ -239,6 +241,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Timestamp reload_neighbours_at_; td::Timestamp ping_neighbours_at_; adnl::AdnlNodeIdShort last_pinged_neighbour_ = adnl::AdnlNodeIdShort::zero(); + + bool subscribed_ = false; }; } // namespace fullnode diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 6606f21528..5712d3cf71 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -120,14 +120,17 @@ void FullNodeImpl::initial_read_complete(BlockHandle top_handle) { td::actor::send_closure(it->second, &FullNodeShard::set_handle, top_handle, std::move(P)); } -void FullNodeImpl::add_shard(ShardIdFull shard) { +void FullNodeImpl::add_shard(ShardIdFull shard, bool subscribe) { while (true) { - if (shards_.count(shard) == 0) { + auto it = shards_.find(shard); + if (it == shards_.end()) { shards_.emplace(shard, FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, keyring_, adnl_, - rldp_, overlays_, validator_manager_, client_)); + rldp_, overlays_, validator_manager_, client_, subscribe)); if (all_validators_.size() > 0) { td::actor::send_closure(shards_[shard], &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } + } else if (subscribe) { + td::actor::send_closure(it->second, &FullNodeShard::subscribe_to_shard); } else { break; } @@ -148,7 +151,7 @@ void FullNodeImpl::sync_completed() { } void FullNodeImpl::send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) { - auto shard = get_shard(ShardIdFull{masterchainId}); + auto shard = get_shard(dst); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT ihr message to unknown shard"; return; @@ -166,7 +169,7 @@ void FullNodeImpl::send_ext_message(AccountIdPrefixFull dst, td::BufferSlice dat } void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - auto shard = get_shard(ShardIdFull{masterchainId, shardIdAll}); + auto shard = get_shard(block_id.shard_full()); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; return; @@ -175,7 +178,7 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s } void FullNodeImpl::send_broadcast(BlockBroadcast broadcast) { - auto shard = get_shard(ShardIdFull{masterchainId}); + auto shard = get_shard(broadcast.block_id.shard_full()); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; return; @@ -381,11 +384,8 @@ void FullNodeImpl::start_up() { void initial_read_complete(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::initial_read_complete, handle); } - void add_shard(ShardIdFull shard) override { - td::actor::send_closure(id_, &FullNodeImpl::add_shard, shard); - } - void del_shard(ShardIdFull shard) override { - td::actor::send_closure(id_, &FullNodeImpl::del_shard, shard); + void subscribe_to_shard(ShardIdFull shard) override { + td::actor::send_closure(id_, &FullNodeImpl::add_shard, shard, true); } void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) override { td::actor::send_closure(id_, &FullNodeImpl::send_ihr_message, dst, std::move(data)); @@ -465,7 +465,7 @@ FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id , validator_manager_(validator_manager) , client_(client) , db_root_(db_root) { - add_shard(ShardIdFull{masterchainId}); + add_shard(ShardIdFull{masterchainId}, true); } td::actor::ActorOwn FullNode::create(ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 6d57f4a82a..f3ce9d0505 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -52,7 +52,7 @@ class FullNodeImpl : public FullNode { void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; - void add_shard(ShardIdFull shard); + void add_shard(ShardIdFull shard, bool subscribe = false); void del_shard(ShardIdFull shard); void sync_completed(); diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index a5c4d9a125..74adcd696e 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -92,6 +92,28 @@ AcceptBlockQuery::AcceptBlockQuery(ForceFork ffork, BlockIdExt id, td::Ref data, std::vector prev, + td::Ref validator_set, td::Ref signatures, + td::Ref approve_signatures, + td::actor::ActorId manager, td::Promise promise) + : id_(id) + , data_(std::move(data)) + , prev_(std::move(prev)) + , validator_set_(std::move(validator_set)) + , signatures_(std::move(signatures)) + , approve_signatures_(std::move(approve_signatures)) + , is_fake_(false) + , is_fork_(false) + , send_broadcast_(true) + , broadcast_only_(false) + , manager_(manager) + , promise_(std::move(promise)) { + state_keep_old_hash_.clear(); + state_old_hash_.clear(); + state_hash_.clear(); + CHECK(prev_.size() > 0); +} + bool AcceptBlockQuery::precheck_header() { VLOG(VALIDATOR_DEBUG) << "precheck_header()"; // 0. sanity check @@ -384,6 +406,15 @@ void AcceptBlockQuery::start_up() { return; } + if (broadcast_only_) { + if (!create_new_proof()) { + fatal_error("cannot generate proof for block "s + id_.to_str()); + return; + } + applied(); + return; + } + td::actor::send_closure( manager_, &ValidatorManager::get_block_handle, id_, true, [SelfId = actor_id(this)](td::Result R) { check_send_error(SelfId, R) || diff --git a/validator/impl/accept-block.hpp b/validator/impl/accept-block.hpp index 0b8cf2c328..b45095b4f1 100644 --- a/validator/impl/accept-block.hpp +++ b/validator/impl/accept-block.hpp @@ -48,6 +48,7 @@ class AcceptBlockQuery : public td::actor::Actor { public: struct IsFake {}; struct ForceFork {}; + struct BroadcastOnly{}; AcceptBlockQuery(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, td::Ref approve_signatures, bool send_broadcast, @@ -57,6 +58,10 @@ class AcceptBlockQuery : public td::actor::Actor { td::Promise promise); AcceptBlockQuery(ForceFork ffork, BlockIdExt id, td::Ref data, td::actor::ActorId manager, td::Promise promise); + AcceptBlockQuery(BroadcastOnly, BlockIdExt id, td::Ref data, std::vector prev, + td::Ref validator_set, td::Ref signatures, + td::Ref approve_signatures, td::actor::ActorId manager, + td::Promise promise); private: static constexpr td::uint32 priority() { @@ -98,6 +103,7 @@ class AcceptBlockQuery : public td::actor::Actor { bool is_fake_; bool is_fork_; bool send_broadcast_; + bool broadcast_only_{false}; bool ancestors_split_{false}, is_key_block_{false}; td::Timestamp timeout_ = td::Timestamp::in(600.0); td::actor::ActorId manager_; diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 6958ad8906..e86b9b90de 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -151,6 +151,17 @@ void run_hardfork_accept_block_query(BlockIdExt id, td::Ref data, .release(); } +void run_broadcast_only_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, + td::Ref validator_set, td::Ref signatures, + td::Ref approve_signatures, + td::actor::ActorId manager, + td::Promise promise) { + td::actor::create_actor("broadcastaccept", AcceptBlockQuery::BroadcastOnly(), id, std::move(data), + prev, std::move(validator_set), std::move(signatures), + std::move(approve_signatures), manager, std::move(promise)) + .release(); +} + void run_apply_block_query(BlockIdExt id, td::Ref block, BlockIdExt masterchain_block_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise) { diff --git a/validator/manager.cpp b/validator/manager.cpp index 182ba9c3b8..1d639291ee 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2367,7 +2367,7 @@ void ValidatorManagerImpl::get_shard_client_state(bool from_db, td::Promiseadd_shard(shard); + callback_->subscribe_to_shard(shard); } void ValidatorManagerImpl::update_async_serializer_state(AsyncSerializerState state, td::Promise promise) { @@ -2396,7 +2396,7 @@ void ValidatorManagerImpl::get_archive_slice(td::uint64 archive_id, td::uint64 o } bool ValidatorManagerImpl::is_validator() { - return temp_keys_.size() > 0 || permanent_keys_.size() > 0; + return true; // temp_keys_.size() > 0 || permanent_keys_.size() > 0; } PublicKeyHash ValidatorManagerImpl::get_validator(ShardIdFull shard, td::Ref val_set) { diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index ab7f206277..9efda4131f 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -121,9 +121,15 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, next_block_id, std::move(stats)); auto block = block_data.size() > 0 ? create_block(next_block_id, std::move(block_data)).move_as_ok() : td::Ref{}; + accept_block_query(next_block_id, std::move(block), std::move(prev_block_ids_), std::move(sig_set), + std::move(approve_sig_set), src == local_id_, std::move(promise)); + prev_block_ids_ = std::vector{next_block_id}; +} - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = next_block_id, block, prev = prev_block_ids_, - sig_set, approve_sig_set, +void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, + td::Ref sig_set, td::Ref approve_sig_set, + bool send_broadcast, td::Promise promise) { + auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { if (R.error().code() == ErrorCode::cancelled) { @@ -131,35 +137,22 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s return; } LOG_CHECK(R.error().code() == ErrorCode::timeout || R.error().code() == ErrorCode::notready) << R.move_as_error(); - td::actor::send_closure(SelfId, &ValidatorGroup::retry_accept_block_query, block_id, std::move(block), - std::move(prev), std::move(sig_set), std::move(approve_sig_set), std::move(promise)); - } else { - promise.set_value(R.move_as_ok()); - } - }); - - run_accept_block_query(next_block_id, std::move(block), prev_block_ids_, validator_set_, std::move(sig_set), - std::move(approve_sig_set), src == local_id_, manager_, std::move(P)); - prev_block_ids_ = std::vector{next_block_id}; -} - -void ValidatorGroup::retry_accept_block_query(BlockIdExt block_id, td::Ref block, - std::vector prev, td::Ref sig_set, - td::Ref approve_sig_set, - td::Promise promise) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id, block, prev, sig_set, approve_sig_set, - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - LOG_CHECK(R.error().code() == ErrorCode::timeout) << R.move_as_error(); - td::actor::send_closure(SelfId, &ValidatorGroup::retry_accept_block_query, block_id, std::move(block), - std::move(prev), std::move(sig_set), std::move(approve_sig_set), std::move(promise)); + td::actor::send_closure(SelfId, &ValidatorGroup::accept_block_query, block_id, std::move(block), std::move(prev), + std::move(sig_set), std::move(approve_sig_set), false, std::move(promise)); } else { promise.set_value(R.move_as_ok()); } }); - run_accept_block_query(block_id, std::move(block), prev, validator_set_, std::move(sig_set), - std::move(approve_sig_set), false, manager_, std::move(P)); + if (shard_.is_masterchain() || lite_mode_) { + run_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, std::move(sig_set), + std::move(approve_sig_set), send_broadcast, manager_, std::move(P)); + } else if (send_broadcast) { + run_broadcast_only_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, + std::move(sig_set), std::move(approve_sig_set), manager_, std::move(P)); + } else { + promise.set_value(td::Unit()); + } } void ValidatorGroup::skip_round(td::uint32 round_id) { @@ -311,8 +304,8 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch auto block = p.block.size() > 0 ? create_block(next_block_id, std::move(p.block)).move_as_ok() : td::Ref{}; - retry_accept_block_query(next_block_id, std::move(block), prev_block_ids_, std::move(p.sigs), - std::move(p.approve_sigs), std::move(p.promise)); + accept_block_query(next_block_id, std::move(block), std::move(prev_block_ids_), std::move(p.sigs), + std::move(p.approve_sigs), false, std::move(p.promise)); prev_block_ids_ = std::vector{next_block_id}; } postponed_accept_.clear(); diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 91d801cd09..bd5e104305 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -39,9 +39,9 @@ class ValidatorGroup : public td::actor::Actor { std::vector approve_signatures, validatorsession::ValidatorSessionStats stats, td::Promise promise); void skip_round(td::uint32 round); - void retry_accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, - td::Ref sigs, td::Ref approve_sigs, - td::Promise promise); + void accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, + td::Ref sigs, td::Ref approve_sigs, + bool send_broadcast, td::Promise promise); void get_approved_candidate(PublicKey source, RootHash root_hash, FileHash file_hash, FileHash collated_data_file_hash, td::Promise promise); BlockId create_next_block_id_simple() const; diff --git a/validator/validator.h b/validator/validator.h index 09081f1e31..132ff06e6f 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -111,8 +111,8 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual ~Callback() = default; virtual void initial_read_complete(BlockHandle top_masterchain_blocks) = 0; - virtual void add_shard(ShardIdFull shard) = 0; - virtual void del_shard(ShardIdFull shard) = 0; + virtual void subscribe_to_shard(ShardIdFull shard) = 0; + //virtual void del_shard(ShardIdFull shard) = 0; virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; From 53270a00e660a7ae492aef93ca90c382a20101d6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 20 Jul 2022 15:39:50 +0300 Subject: [PATCH 006/524] Add CollatorNode and make validators request blocks from it --- crypto/block/block.tlb | 4 + keys/keys.hpp | 4 + tl/generate/scheme/ton_api.tl | 12 +- tl/generate/scheme/ton_api.tlo | Bin 69376 -> 70276 bytes ton/ton-types.h | 8 + .../validator-engine-console-query.cpp | 21 +++ .../validator-engine-console-query.h | 24 +++ .../validator-engine-console.cpp | 1 + validator-engine/validator-engine.cpp | 99 ++++++++++-- validator-engine/validator-engine.hpp | 14 ++ validator/CMakeLists.txt | 2 + validator/collator-node.cpp | 145 ++++++++++++++++++ validator/collator-node.hpp | 51 ++++++ validator/impl/shard.cpp | 24 ++- validator/impl/shard.hpp | 4 + validator/impl/validate-query.cpp | 1 - validator/interfaces/shard.h | 2 + validator/manager-disk.hpp | 4 + validator/manager-hardfork.hpp | 3 + validator/manager.cpp | 12 +- validator/manager.hpp | 5 + validator/shard-client.cpp | 16 ++ validator/validator-group.cpp | 134 +++++++++++++--- validator/validator-group.hpp | 18 ++- validator/validator.h | 2 + 25 files changed, 577 insertions(+), 33 deletions(-) create mode 100644 validator/collator-node.cpp create mode 100644 validator/collator-node.hpp diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 20bacaa490..c9a95535f9 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -739,6 +739,10 @@ _ OracleBridgeParams = ConfigParam 71; // Ethereum bridge _ OracleBridgeParams = ConfigParam 72; // Binance Smart Chain bridge _ OracleBridgeParams = ConfigParam 73; // Polygon bridge +// Set of collators: each collator is (workchain:int32 shard:int64 adnl_id:int256) +colator_set#a0 collators:(HashmapE 352 Unit) = CollatorSet; +_ CollatorSet = ConfigParam 81; + // // PROOFS // diff --git a/keys/keys.hpp b/keys/keys.hpp index cf883bbe61..6ed51f08f0 100644 --- a/keys/keys.hpp +++ b/keys/keys.hpp @@ -253,6 +253,10 @@ class PublicKey { td::BufferSlice export_as_slice() const; static td::Result import(td::Slice s); + bool is_ed25519() const { + return pub_key_.get_offset() == pub_key_.offset(); + } + pubkeys::Ed25519 ed25519_value() const { CHECK(pub_key_.get_offset() == pub_key_.offset()); return pub_key_.get(); diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index eca1523543..3a81ab5858 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -572,6 +572,7 @@ engine.dht id:int256 = engine.Dht; engine.validatorTempKey key:int256 expire_at:int = engine.ValidatorTempKey; engine.validatorAdnlAddress id:int256 expire_at:int = engine.ValidatorAdnlAddress; engine.validator id:int256 temp_keys:(vector engine.validatorTempKey) adnl_addrs:(vector engine.validatorAdnlAddress) election_date:int expire_at:int = engine.Validator; +engine.collator adnl_id:int256 workchain:int shard:long = engine.Validator; engine.liteServer id:int256 port:int = engine.LiteServer; engine.controlProcess id:int256 permissions:int = engine.ControlProcess; engine.controlInterface id:int256 port:int allowed:(vector engine.controlProcess) = engine.ControlInterface; @@ -582,7 +583,7 @@ engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNode engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave; engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) - validators:(vector engine.validator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) + validators:(vector engine.Validator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) fullnodemasters:(vector engine.validator.fullNodeMaster) liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) gc:engine.gc = engine.validator.Config; @@ -690,6 +691,8 @@ engine.validator.generateBlockCandidate block_id:tonNode.BlockId = db.Candidate; engine.validator.getRequiredBlockCandidates = engine.validator.RequiredBlockCandidates; engine.validator.importBlockCandidate block:db.candidate = engine.validator.Success; +engine.validator.addCollator adnl_id:int256 workchain:int shard:long = engine.validator.Success; + ---types--- storage.pong = storage.Pong; @@ -741,3 +744,10 @@ validatorSession.statsRound timestamp:long producers:(vector validatorSession.st validatorSession.stats id:tonNode.blockId timestamp:long self:int256 creator:int256 total_validators:int total_weight:long signatures:int signatures_weight:long approve_signatures:int approve_signatures_weight:long first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats; + +collatorNode.generateBlockSuccess candidate:db.Candidate = collatorNode.GenerateBlockResult; +collatorNode.generateBlockError code:int message:string = collatorNode.GenerateBlockResult; + +---functions--- +collatorNode.generateBlock workchain:int shard:long min_mc_id:tonNode.blockIdExt prev_blocks:(vector tonNode.blockIdExt) + creator:int256 = collatorNode.GenerateBlockResult; \ No newline at end of file diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 252c6d0564df80fb0a38fe6fab78645f0e29d6ad..479455feb679ece457175e95b6b44e8c7807cfa9 100644 GIT binary patch delta 520 zcmZpe$I`Nth4;~HeJchiIJ=RTnM3M`LEd$R(RgNoW z*3-qm<#C(rT2z!@1ky9Pr6qs`#Mo>Q@vH6`y?Zl?Z!UW;%#7S)u6zE7;`CqRqz8 z@bGC_6Umso`R84KCQZ!?kJd@Sj0XoRN_fD-fpxOrQEy((+|0cA+~oMo6rf8dJ4T4} za2FJ%mc=K5!mfB~1Ea=dfpC_|I#*;SA28*B_|oRV9wyeyU(dA2OrH2yOb8NzK<(Yj To@~bO%j5?y1vdY9#GwNKPASF; delta 170 zcmZo!%F-~8h4;~HeJchiIJ1$LnPYPR#~n@<%~$aqlMRGKCV%0R+#D;!%*e7t^3b`> zDZ&B5j82=ksvXzeToQUmlM|%w<}o23*2#j6DwAvL_k(qAzSHo7i4~+ne6nGi2v1sO zI*0>O$UV8YT6^*jE0xI}Epu2w+8H;$Y%zwKuh6z8k}+}f+57%Xn-v~!VA@>sj6(+i DhTB6J diff --git a/ton/ton-types.h b/ton/ton-types.h index 8cb73de6dc..754fb86a17 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -397,6 +397,9 @@ struct Ed25519_PublicKey { bool operator==(const Ed25519_PublicKey& other) const { return _pubkey == other._pubkey; } + bool operator!=(const Ed25519_PublicKey& other) const { + return _pubkey != other._pubkey; + } bool clear() { _pubkey.set_zero(); return true; @@ -479,4 +482,9 @@ struct ValidatorSessionConfig { static const td::uint32 BLOCK_HASH_COVERS_DATA_FROM_VERSION = 2; }; +struct CollatorNodeDescr { + ShardIdFull shard; + NodeIdShort adnl_id; +}; + } // namespace ton diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 028c5b1ae0..9c5b6572d1 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1091,3 +1091,24 @@ td::Status ImportBlockCandidateQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "successfully imported a block candidate\n"; return td::Status::OK(); } + +td::Status AddCollatorQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status AddCollatorQuery::send() { + auto b = ton::create_serialize_tl_object( + adnl_id_.tl(), wc_, shard_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status AddCollatorQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully added collator\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index e4236f6dc8..34b2ede3e6 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1163,3 +1163,27 @@ class ImportBlockCandidateQuery : public Query { private: std::string file_; }; + +class AddCollatorQuery : public Query { + public: + AddCollatorQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "addcollator"; + } + static std::string get_help() { + return "addcollator \tadd collator with given adnl_id and shard"; + } + std::string name() const override { + return get_name(); + } + + private: + ton::PublicKeyHash adnl_id_; + td::int32 wc_; + td::int64 shard_; +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index b4dcad22b7..d70e2cf2e5 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -144,6 +144,7 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 113455249c..09526279f7 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -125,15 +125,24 @@ Config::Config(ton::ton_api::engine_validator_config &config) { for (auto &dht : config.dht_) { config_add_dht_node(ton::PublicKeyHash{dht->id_}).ensure(); } - for (auto &val : config.validators_) { - auto key = ton::PublicKeyHash{val->id_}; - config_add_validator_permanent_key(key, val->election_date_, val->expire_at_).ensure(); - for (auto &temp : val->temp_keys_) { - config_add_validator_temp_key(key, ton::PublicKeyHash{temp->key_}, temp->expire_at_).ensure(); - } - for (auto &adnl : val->adnl_addrs_) { - config_add_validator_adnl_id(key, ton::PublicKeyHash{adnl->id_}, adnl->expire_at_).ensure(); - } + for (auto &v : config.validators_) { + ton::ton_api::downcast_call( + *v, td::overloaded( + [&](ton::ton_api::engine_validator &val) { + auto key = ton::PublicKeyHash{val.id_}; + config_add_validator_permanent_key(key, val.election_date_, val.expire_at_).ensure(); + for (auto &temp : val.temp_keys_) { + config_add_validator_temp_key(key, ton::PublicKeyHash{temp->key_}, temp->expire_at_).ensure(); + } + for (auto &adnl : val.adnl_addrs_) { + config_add_validator_adnl_id(key, ton::PublicKeyHash{adnl->id_}, adnl->expire_at_).ensure(); + } + }, + [&](ton::ton_api::engine_collator &col) { + auto key = ton::PublicKeyHash{col.adnl_id_}; + ton::ShardIdFull shard(col.workchain_, col.shard_); + config_add_collator(key, shard).ensure(); + })); } config_add_full_node_adnl_id(ton::PublicKeyHash{config.fullnode_}).ensure(); @@ -192,7 +201,7 @@ ton::tl_object_ptr Config::tl() const { dht_vec.push_back(ton::create_tl_object(x.tl())); } - std::vector> val_vec; + std::vector> val_vec; for (auto &val : validators) { std::vector> temp_vec; for (auto &t : val.second.temp_keys) { @@ -205,6 +214,10 @@ ton::tl_object_ptr Config::tl() const { val_vec.push_back(ton::create_tl_object( val.first.tl(), std::move(temp_vec), std::move(adnl_val_vec), val.second.election_date, val.second.expire_at)); } + for (auto &col : collators) { + val_vec.push_back(ton::create_tl_object( + col.adnl_id.tl(), col.shard.workchain, col.shard.shard)); + } std::vector> full_node_slaves_vec; for (auto &x : full_node_slaves) { @@ -383,6 +396,18 @@ td::Result Config::config_add_validator_adnl_id(ton::PublicKeyHash perm_ke } } +td::Result Config::config_add_collator(ton::PublicKeyHash addr, ton::ShardIdFull shard) { + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard: " << shard.to_str()); + } + Collator c{addr, shard}; + if (std::find(collators.begin(), collators.end(), c) != collators.end()) { + return false; + } + collators.push_back(c); + return true; +} + td::Result Config::config_add_full_node_adnl_id(ton::PublicKeyHash id) { if (full_node == id) { return false; @@ -1836,6 +1861,19 @@ void ValidatorEngine::start_lite_server() { } void ValidatorEngine::started_lite_server() { + start_collator(); +} + +void ValidatorEngine::start_collator() { + for (auto &c : config_.collators) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, + ton::adnl::AdnlNodeIdShort(c.adnl_id), c.shard); + } + + started_collator(); +} + +void ValidatorEngine::started_collator() { start_control_interface(); } @@ -3395,6 +3433,47 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importBlo }); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollator &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto id = ton::PublicKeyHash{query.adnl_id_}; + auto shard = ton::ShardIdFull(query.workchain_, query.shard_); + if (!shard.is_valid_ext()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "invalid shard"))); + return; + } + + auto R = config_.config_add_collator(id, shard); + if (R.is_error()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!R.move_as_ok()) { + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + return; + } + if (!validator_manager_.empty()) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, + ton::adnl::AdnlNodeIdShort(id), shard); + } + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 0a7f8e2642..037288cee2 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -66,6 +66,14 @@ struct Config { ton::UnixTime election_date; ton::UnixTime expire_at; }; + struct Collator { + ton::PublicKeyHash adnl_id; + ton::ShardIdFull shard; + + bool operator==(const Collator& b) const { + return adnl_id == b.adnl_id && shard == b.shard; + } + }; struct Control { ton::PublicKeyHash key; std::map clients; @@ -81,6 +89,7 @@ struct Config { std::map adnl_ids; std::set dht_ids; std::map validators; + std::vector collators; ton::PublicKeyHash full_node = ton::PublicKeyHash::zero(); std::vector full_node_slaves; std::map full_node_masters; @@ -104,6 +113,7 @@ struct Config { ton::UnixTime expire_at); td::Result config_add_validator_adnl_id(ton::PublicKeyHash perm_key, ton::PublicKeyHash adnl_id, ton::UnixTime expire_at); + td::Result config_add_collator(ton::PublicKeyHash addr, ton::ShardIdFull shard); td::Result config_add_full_node_adnl_id(ton::PublicKeyHash id); td::Result config_add_full_node_slave(td::IPAddress addr, ton::PublicKey id); td::Result config_add_full_node_master(td::int32 port, ton::PublicKeyHash id); @@ -293,6 +303,8 @@ class ValidatorEngine : public td::actor::Actor { void add_lite_server(ton::PublicKeyHash id, td::uint16 port); void start_lite_server(); void started_lite_server(); + void start_collator(); + void started_collator(); void add_control_interface(ton::PublicKeyHash id, td::uint16 port); void add_control_process(ton::PublicKeyHash id, td::uint16 port, ton::PublicKeyHash pub, td::int32 permissions); @@ -419,6 +431,8 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_importBlockCandidate &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_addCollator &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 65d7b34859..443ae23ea2 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -53,6 +53,7 @@ set(VALIDATOR_HEADERS import-db-slice.hpp + collator-node.hpp manager-disk.h manager-disk.hpp manager-init.h @@ -68,6 +69,7 @@ set(VALIDATOR_HEADERS set(VALIDATOR_SOURCE apply-block.cpp block-handle.cpp + collator-node.cpp get-next-key-blocks.cpp import-db-slice.cpp shard-client.cpp diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp new file mode 100644 index 0000000000..958b5f1362 --- /dev/null +++ b/validator/collator-node.cpp @@ -0,0 +1,145 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "collator-node.hpp" +#include "ton/ton-tl.hpp" +#include "fabric.h" + +namespace ton { + +namespace validator { + +CollatorNode::CollatorNode(adnl::AdnlNodeIdShort local_id, td::actor::ActorId manager, + td::actor::ActorId adnl, td::actor::ActorId rldp) + : local_id_(local_id), manager_(std::move(manager)), adnl_(std::move(adnl)), rldp_(std::move(rldp)) { +} + +void CollatorNode::start_up() { + class Cb : public adnl::Adnl::Callback { + public: + Cb(td::actor::ActorId id) : id_(std::move(id)) { + } + void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(id_, &CollatorNode::receive_query, src, std::move(data), std::move(promise)); + } + + private: + td::actor::ActorId id_; + }; + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlock::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, adnl::AdnlNodeIdShort(local_id_)); +} + +void CollatorNode::tear_down() { + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlock::ID)); +} + +void CollatorNode::add_shard(ShardIdFull shard) { + LOG(INFO) << "Collator node: local_id=" << local_id_ << " , shard=" << shard.to_str(); + shards_.push_back(shard); +} + +static td::BufferSlice serialize_error(td::Status error) { + return create_serialize_tl_object(error.code(), error.message().c_str()); +} + +void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, + td::Promise promise) { + auto status = [&]() -> td::Status { + TRY_RESULT(f, fetch_tl_object(std::move(data), true)); + ShardIdFull shard(f->workchain_, f->shard_); + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << shard.to_str()); + } + bool found = false; + for (ShardIdFull our_shard : shards_) { + if (shard_is_ancestor(shard, our_shard)) { + found = true; + break; + } + } + if (!found) { + return td::Status::Error(PSTRING() << "this node doesn't collate shard " << shard.to_str()); + } + if (f->prev_blocks_.size() != 1 && f->prev_blocks_.size() != 2) { + return td::Status::Error(PSTRING() << "invalid size of prev_blocks: " << f->prev_blocks_.size()); + } + std::vector prev_blocks; + for (const auto& b : f->prev_blocks_) { + prev_blocks.push_back(create_block_id(b)); + } + BlockIdExt min_mc_id = create_block_id(f->min_mc_id_); + Ed25519_PublicKey creator(f->creator_); + + LOG(INFO) << "Query from " << src << ": shard=" << shard.to_str() << ", min_mc_id=" << min_mc_id.to_str(); + + auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), prev_blocks = std::move(prev_blocks), + promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + LOG(WARNING) << "Query from " << src << ", error: " << R.error(); + promise.set_result(serialize_error(R.move_as_error_prefix("failed to get masterchain state: "))); + } else { + td::Ref state(R.move_as_ok()); + if (state.is_null()) { + LOG(WARNING) << "Query from " << src << ", error: failed to get masterchain state"; + promise.set_result(serialize_error(R.move_as_error_prefix("failed to get masterchain state: "))); + return; + } + td::actor::send_closure(SelfId, &CollatorNode::receive_query_cont, src, shard, std::move(state), + std::move(prev_blocks), creator, std::move(promise)); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, min_mc_id, 1, td::Timestamp::in(5.0), + std::move(P)); + return td::Status::OK(); + }(); + if (status.is_error()) { + LOG(WARNING) << "Query from " << src << ", error: " << status; + promise.set_result(serialize_error(std::move(status))); + } +} + +void CollatorNode::receive_query_cont(adnl::AdnlNodeIdShort src, ShardIdFull shard, + td::Ref min_mc_state, std::vector prev_blocks, + Ed25519_PublicKey creator, td::Promise promise) { + auto P = td::PromiseCreator::lambda([promise = std::move(promise), src](td::Result R) mutable { + if (R.is_error()) { + LOG(WARNING) << "Query from " << src << ", error: " << R.error(); + promise.set_result(serialize_error(R.move_as_error())); + } else { + LOG(INFO) << "Query from " << src << ", success"; + auto block = R.move_as_ok(); + auto result = create_serialize_tl_object( + create_tl_object(PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), + create_tl_block_id(block.id), std::move(block.data), + std::move(block.collated_data))); + promise.set_result(std::move(result)); + } + }); + + run_collate_query(shard, min_mc_state->get_block_id(), std::move(prev_blocks), creator, + min_mc_state->get_validator_set(shard), manager_, td::Timestamp::in(10.0), std::move(P)); +} + +} // namespace validator + +} // namespace ton diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp new file mode 100644 index 0000000000..854e98ce2a --- /dev/null +++ b/validator/collator-node.hpp @@ -0,0 +1,51 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "interfaces/validator-manager.h" +#include "rldp/rldp.h" + +namespace ton { + +namespace validator { + +class ValidatorManager; + +class CollatorNode : public td::actor::Actor { + public: + CollatorNode(adnl::AdnlNodeIdShort local_id, td::actor::ActorId manager, + td::actor::ActorId adnl, td::actor::ActorId rldp); + void start_up() override; + void tear_down() override; + void add_shard(ShardIdFull shard); + + private: + void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); + void receive_query_cont(adnl::AdnlNodeIdShort src, ShardIdFull shard, td::Ref min_mc_state, + std::vector prev_blocks, Ed25519_PublicKey creator, + td::Promise promise); + + adnl::AdnlNodeIdShort local_id_; + td::actor::ActorId manager_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + std::vector shards_; +}; + +} // namespace validator + +} // namespace ton diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index a96f1a819d..dbf2440029 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -364,7 +364,8 @@ td::Status MasterchainStateQ::mc_init() { td::Status MasterchainStateQ::mc_reinit() { auto res = block::ConfigInfo::extract_config( root_cell(), block::ConfigInfo::needStateRoot | block::ConfigInfo::needValidatorSet | - block::ConfigInfo::needShardHashes | block::ConfigInfo::needPrevBlocks); + block::ConfigInfo::needShardHashes | block::ConfigInfo::needPrevBlocks | + block::ConfigInfo::needWorkchainInfo); cur_validators_.reset(); next_validators_.reset(); if (res.is_error()) { @@ -510,6 +511,27 @@ bool MasterchainStateQ::check_old_mc_block_id(const ton::BlockIdExt& blkid, bool return config_ && config_->check_old_mc_block_id(blkid, strict); } +std::vector MasterchainStateQ::get_collator_set() const { + block::gen::CollatorSet::Record rec; + auto cell = config_->get_config_param(81); + if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { + return {}; + } + vm::Dictionary dict{rec.collators->prefetch_ref(), 32 + 64 + 256}; + std::vector collators; + dict.check_for_each([&](Ref, td::ConstBitPtr key, int n) { + CHECK(n == 32 + 64 + 256); + auto workchain = (td::int32)key.get_int(32); + key.advance(32); + td::uint64 shard = key.get_uint(64); + key.advance(64); + td::Bits256 adnl_id(key); + collators.push_back({ShardIdFull(workchain, shard), adnl_id}); + return true; + }); + return collators; +} + td::uint32 MasterchainStateQ::min_split_depth(WorkchainId workchain_id) const { if (!config_) { return 0; diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index 0056df1190..2b411d9442 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -147,6 +147,10 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { return td::make_ref(config_); } } + block::WorkchainSet get_workchain_list() const override { + return config_ ? config_->get_workchain_list() : block::WorkchainSet(); + } + std::vector get_collator_set() const override; private: ZeroStateIdExt zerostate_id_; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index aac85c8f1f..369f1f2a8b 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -448,7 +448,6 @@ bool ValidateQuery::init_parse() { return reject_query("after_merge value mismatch in block header"); } rand_seed_ = extra.rand_seed; - created_by_ = extra.created_by; if (created_by_ != extra.created_by) { return reject_query("block candidate "s + id_.to_str() + " has creator " + created_by_.to_hex() + " but the block header contains different value " + extra.created_by.to_hex()); diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 96ca8da260..ff458d604c 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -81,6 +81,8 @@ class MasterchainState : virtual public ShardState { ton::LogicalTime* end_lt = nullptr) const = 0; virtual bool check_old_mc_block_id(const ton::BlockIdExt& blkid, bool strict = false) const = 0; virtual td::Result> get_config_holder() const = 0; + virtual block::WorkchainSet get_workchain_list() const = 0; + virtual std::vector get_collator_set() const = 0; virtual td::Status prepare() { return td::Status::OK(); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 6bad117158..91235ebf62 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -384,6 +384,10 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { + UNREACHABLE(); + } + private: PublicKeyHash local_id_; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index a8aaecfc3d..80f6df93ca 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -443,6 +443,9 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { + UNREACHABLE(); + } private: td::Ref opts_; diff --git a/validator/manager.cpp b/validator/manager.cpp index 1d639291ee..9af0077264 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2025,7 +2025,8 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto validator_id = get_validator(shard, validator_set); CHECK(!validator_id.is_zero()); auto G = td::actor::create_actor( - "validatorgroup", shard, validator_id, session_id, validator_set, opts, keyring_, adnl_, rldp_, overlays_, + "validatorgroup", shard, validator_id, session_id, validator_set, last_masterchain_state_->get_collator_set(), + opts, keyring_, adnl_, rldp_, overlays_, db_root_, actor_id(this), init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), true); return G; @@ -2692,6 +2693,15 @@ void ValidatorManagerImpl::cleanup_old_pending_candidates(BlockId block_id, td:: } } +void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) { + auto it = collator_nodes_.find(id); + if (it == collator_nodes_.end()) { + auto actor = td::actor::create_actor("collatornode", id, actor_id(this), adnl_, rldp_); + it = collator_nodes_.emplace(id, std::move(actor)).first; + } + td::actor::send_closure(it->second, &CollatorNode::add_shard, shard); +} + td::actor::ActorOwn ValidatorManagerFactory::create( td::Ref opts, std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, diff --git a/validator/manager.hpp b/validator/manager.hpp index 1fbf60a9d0..94657d49d3 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -28,6 +28,7 @@ #include "state-serializer.hpp" #include "rldp/rldp.h" #include "token-manager.h" +#include "collator-node.hpp" #include #include @@ -539,6 +540,8 @@ class ValidatorManagerImpl : public ValidatorManager { void import_block_candidate(BlockCandidate candidate, td::Promise promise) override; void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise promise) override; + void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; + private: td::Timestamp resend_shard_blocks_at_; td::Timestamp check_waiters_at_; @@ -606,6 +609,8 @@ class ValidatorManagerImpl : public ValidatorManager { std::map, td::Timestamp>>> pending_block_candidates_; void cleanup_old_pending_candidates(BlockId block_id, td::Timestamp now); + + std::map> collator_nodes_; }; } // namespace validator diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index c7d87fb428..af44f16e84 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -248,9 +248,11 @@ void ShardClient::get_processed_masterchain_block_id(td::Promise pro void ShardClient::build_shard_overlays() { auto v = masterchain_state_->get_shards(); + std::set workchains; for (auto &x : v) { auto shard = x->shard(); + workchains.insert(shard.workchain); if (opts_->need_monitor(shard)) { auto d = masterchain_state_->soft_min_split_depth(shard.workchain); auto l = shard_prefix_length(shard.shard); @@ -264,6 +266,20 @@ void ShardClient::build_shard_overlays() { } } } + + for (const auto &wpair : masterchain_state_->get_workchain_list()) { + ton::WorkchainId wc = wpair.first; + const block::WorkchainInfo *winfo = wpair.second.get(); + if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= masterchain_state_->get_unix_time()) { + auto shard = ShardIdFull(wc); + if (opts_->need_monitor(shard) && created_overlays_.count(shard) == 0) { + td::actor::send_closure(manager_, &ValidatorManager::subscribe_to_shard, shard); + BlockIdExt block_id(shard.workchain, shard.shard, 0, winfo->zerostate_root_hash, winfo->zerostate_file_hash); + td::actor::send_closure_later(manager_, &ValidatorManager::wait_block_state_short, block_id, 0, + td::Timestamp::in(5.0), [](td::Result>) {}); + } + } + } } void ShardClient::force_update_shard_client(BlockHandle handle, td::Promise promise) { diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 9efda4131f..bb493f9120 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -22,6 +22,7 @@ #include "td/utils/overloaded.h" #include "common/delay.h" #include "ton/ton-tl.hpp" +#include "td/utils/Random.h" namespace ton { @@ -36,19 +37,7 @@ void ValidatorGroup::generate_block_candidate(td::uint32 round_id, td::Promise R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - BlockCandidate candidate = R.move_as_ok(); - candidate.pubkey = pubkey; - promise.set_result(std::move(candidate)); - } - }); - td::actor::send_closure(manager_, &ValidatorManager::wait_block_candidate, create_next_block_id_simple(), - td::Timestamp::in(15.0), std::move(P)); + send_collate_query(round_id, td::Timestamp::in(10.0), std::move(promise)); return; } run_collate_query(shard_, min_masterchain_block_id_, prev_block_ids_, @@ -65,6 +54,17 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat promise.set_error(td::Status::Error(ErrorCode::notready, "too old")); return; } + + if (approved_candidates_cache_round_ != round_id) { + approved_candidates_cache_round_ = round_id; + approved_candidates_cache_.clear(); + } + CacheKey cache_key = block_to_cache_key(block); + auto it = approved_candidates_cache_.find(cache_key); + if (it != approved_candidates_cache_.end()) { + promise.set_result(it->second); + } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), round_id, block = block.clone(), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -80,11 +80,16 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat td::Timestamp::in(0.1)); } else { auto v = R.move_as_ok(); - v.visit(td::overloaded([&](UnixTime ts) { promise.set_result(ts); }, - [&](CandidateReject reject) { - promise.set_error(td::Status::Error(ErrorCode::protoviolation, - PSTRING() << "bad candidate: " << reject.reason)); - })); + v.visit(td::overloaded( + [&](UnixTime ts) { + td::actor::send_closure(SelfId, &ValidatorGroup::update_approve_cache, round_id, block_to_cache_key(block), + ts); + promise.set_result(ts); + }, + [&](CandidateReject reject) { + promise.set_error( + td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad candidate: " << reject.reason)); + })); } }); if (!started_) { @@ -98,6 +103,14 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat td::Timestamp::in(10.0), std::move(P), lite_mode_ ? ValidateMode::lite : 0); } +void ValidatorGroup::update_approve_cache(td::uint32 round_id, CacheKey key, UnixTime value) { + if (approved_candidates_cache_round_ != round_id) { + approved_candidates_cache_round_ = round_id; + approved_candidates_cache_.clear(); + } + approved_candidates_cache_[key] = value; +} + void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash src, td::BufferSlice block_data, RootHash root_hash, FileHash file_hash, std::vector signatures, @@ -339,6 +352,91 @@ void ValidatorGroup::get_session_info( td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_session_info, std::move(P)); } +void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeout, + td::Promise promise) { + adnl::AdnlNodeIdShort collator = adnl::AdnlNodeIdShort::zero(); + // TODO: some other way for storing and choosing collators for real network + int cnt = 0; + for (const CollatorNodeDescr& c : collator_set_) { + if (shard_is_ancestor(shard_, c.shard) && td::Random::fast(0, cnt) == 0) { + collator = adnl::AdnlNodeIdShort(c.adnl_id); + ++cnt; + } + } + + if (collator.is_zero()) { + promise.set_error(td::Status::Error(PSTRING() << "no collator for shard " << shard_.to_str())); + return; + } + + std::vector> prev_blocks; + for (const BlockIdExt &p : prev_block_ids_) { + prev_blocks.push_back(create_tl_block_id(p)); + } + td::BufferSlice query = create_serialize_tl_object( + shard_.workchain, shard_.shard, create_tl_block_id(min_masterchain_block_id_), std::move(prev_blocks), + local_id_full_.ed25519_value().raw()); + + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), round_id, promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("rldp query failed: ")); + return; + } + td::actor::send_closure(SelfId, &ValidatorGroup::receive_collate_query_response, round_id, R.move_as_ok(), + std::move(promise)); + }); + LOG(INFO) << "collate query for " << shard_.to_str() << ": send query to " << collator; + size_t max_answer_size = config_.max_block_size + config_.max_collated_data_size + 256; + td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, adnl::AdnlNodeIdShort(local_id_), collator, "collatequery", + std::move(P), timeout, std::move(query), max_answer_size); +} + +void ValidatorGroup::receive_collate_query_response(td::uint32 round_id, td::BufferSlice data, + td::Promise promise) { + if (round_id < last_known_round_id_) { + promise.set_error(td::Status::Error("too old")); + return; + } + TRY_RESULT_PROMISE(promise, f, fetch_tl_object(data, true)); + tl_object_ptr b; + ton_api::downcast_call(*f, td::overloaded( + [&](ton_api::collatorNode_generateBlockError &r) { + td::Status error = td::Status::Error(r.code_, r.message_); + promise.set_error(error.move_as_error_prefix("collate query: ")); + }, + [&](ton_api::collatorNode_generateBlockSuccess &r) { b = std::move(r.candidate_); })); + if (!b) { + return; + } + auto key = PublicKey{b->source_}; + if (!key.is_ed25519()) { + promise.set_error(td::Status::Error("collate query: block candidate source mismatch")); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + if (e_key != Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}) { + promise.set_error(td::Status::Error("collate query: block candidate source mismatch")); + return; + } + auto block_id = ton::create_block_id(b->id_); + if (block_id.shard_full() != shard_) { + promise.set_error(td::Status::Error("collate query: shard mismatch")); + return; + } + auto collated_data_hash = td::sha256_bits256(b->collated_data_); + BlockCandidate candidate(e_key, block_id, collated_data_hash, std::move(b->data_), std::move(b->collated_data_)); + + auto P = td::PromiseCreator::lambda( + [candidate = candidate.clone(), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("validate received block error: ")); + return; + } + promise.set_result(std::move(candidate)); + }); + validate_block_candidate(round_id, std::move(candidate), std::move(P)); +} + } // namespace validator } // namespace ton diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index bd5e104305..d0a96d659d 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -55,12 +55,14 @@ class ValidatorGroup : public td::actor::Actor { init_ = false; create_session(); } + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, adnl::AdnlNodeIdShort(local_id_)); } void get_session_info(td::Promise> promise); ValidatorGroup(ShardIdFull shard, PublicKeyHash local_id, ValidatorSessionId session_id, - td::Ref validator_set, validatorsession::ValidatorSessionOptions config, + td::Ref validator_set, std::vector collator_set, + validatorsession::ValidatorSessionOptions config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, td::actor::ActorId validator_manager, bool create_session, @@ -69,6 +71,7 @@ class ValidatorGroup : public td::actor::Actor { , local_id_(std::move(local_id)) , session_id_(session_id) , validator_set_(std::move(validator_set)) + , collator_set_(std::move(collator_set)) , config_(std::move(config)) , keyring_(keyring) , adnl_(adnl) @@ -83,6 +86,8 @@ class ValidatorGroup : public td::actor::Actor { private: std::unique_ptr make_validator_session_callback(); + void send_collate_query(td::uint32 round_id, td::Timestamp timeout, td::Promise promise); + void receive_collate_query_response(td::uint32 round_id, td::BufferSlice data, td::Promise promise); struct PostponedAccept { RootHash root_hash; @@ -105,6 +110,7 @@ class ValidatorGroup : public td::actor::Actor { BlockIdExt min_masterchain_block_id_; td::Ref validator_set_; + std::vector collator_set_; validatorsession::ValidatorSessionOptions config_; td::actor::ActorId keyring_; @@ -120,6 +126,16 @@ class ValidatorGroup : public td::actor::Actor { bool allow_unsafe_self_blocks_resync_; bool lite_mode_ = false; td::uint32 last_known_round_id_ = 0; + + typedef std::tuple CacheKey; + std::map approved_candidates_cache_; + td::uint32 approved_candidates_cache_round_ = 0; + + void update_approve_cache(td::uint32 round_id, CacheKey key, UnixTime value); + + static CacheKey block_to_cache_key(const BlockCandidate& block) { + return std::make_tuple(block.pubkey.as_bits256(), block.id, sha256_bits256(block.data), block.collated_file_hash); + } }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 132ff06e6f..66007910b6 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -220,6 +220,8 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void generate_block_candidate(BlockId block_id, td::Promise promise) = 0; virtual void get_required_block_candidates(td::Promise> promise) = 0; virtual void import_block_candidate(BlockCandidate candidate, td::Promise promise) = 0; + + virtual void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; }; } // namespace validator From 7ac60bea7d5858f91fc218ca13a59c4388647f0e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 29 Jul 2022 10:39:02 +0300 Subject: [PATCH 007/524] New shard overlays --- create-hardfork/create-hardfork.cpp | 2 +- overlay/overlay-manager.cpp | 19 +- overlay/overlay-manager.h | 4 +- overlay/overlay-peers.cpp | 4 +- overlay/overlay.cpp | 78 ++++++-- overlay/overlay.hpp | 8 +- overlay/overlays.h | 4 +- test/test-ton-collator.cpp | 2 +- ton/ton-types.h | 2 + validator-engine/validator-engine.cpp | 67 +++++-- validator-engine/validator-engine.hpp | 1 + validator/fabric.h | 4 +- validator/full-node-shard.cpp | 44 +++-- validator/full-node-shard.h | 4 +- validator/full-node-shard.hpp | 6 +- validator/full-node.cpp | 239 +++++++++++++++++------ validator/full-node.h | 5 +- validator/full-node.hpp | 27 ++- validator/impl/accept-block.cpp | 44 +---- validator/impl/accept-block.hpp | 9 +- validator/impl/fabric.cpp | 15 +- validator/interfaces/validator-manager.h | 6 +- validator/manager-disk.hpp | 2 +- validator/manager-hardfork.hpp | 3 +- validator/manager.cpp | 16 +- validator/manager.hpp | 3 +- validator/net/download-block-new.cpp | 4 +- validator/shard-client.cpp | 34 +--- validator/validator-group.cpp | 18 +- validator/validator.h | 8 +- 30 files changed, 411 insertions(+), 271 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 849a31b125..27260ee02b 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -236,7 +236,7 @@ class HardforkCreator : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void subscribe_to_shard(ton::ShardIdFull) override { + void update_shard_configuration(td::Ref state) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index 2eac5a0c25..62d98eb00e 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -92,14 +92,15 @@ void OverlayManager::delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdSho void OverlayManager::create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) { CHECK(!dht_node_.empty()); + CHECK(callback != nullptr); auto id = overlay_id.compute_short_id(); register_overlay(local_id, id, Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), std::move(callback), std::move(rules), scope)); } -void OverlayManager::create_public_overlay_no_subscribe(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - OverlayPrivacyRules rules, td::string scope) { +void OverlayManager::create_public_overlay_external(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + OverlayPrivacyRules rules, td::string scope) { CHECK(!dht_node_.empty()); auto id = overlay_id.compute_short_id(); register_overlay(local_id, id, @@ -266,12 +267,16 @@ void OverlayManager::get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, Ov td::uint32 max_peers, td::Promise> promise) { auto it = overlays_.find(local_id); - if (it != overlays_.end()) { - auto it2 = it->second.find(overlay_id); - if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::get_overlay_random_peers, max_peers, std::move(promise)); - } + if (it == overlays_.end()) { + promise.set_error(td::Status::Error(PSTRING() << "no such local id " << local_id)); + return; + } + auto it2 = it->second.find(overlay_id); + if (it2 == it->second.end()) { + promise.set_error(td::Status::Error(PSTRING() << "no such overlay " << overlay_id)); + return; } + td::actor::send_closure(it2->second, &Overlay::get_overlay_random_peers, max_peers, std::move(promise)); } td::actor::ActorOwn Overlays::create(std::string db_root, td::actor::ActorId keyring, diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index b03261e806..a3ed4a54ab 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -52,8 +52,8 @@ class OverlayManager : public Overlays { void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) override; - void create_public_overlay_no_subscribe(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - OverlayPrivacyRules rules, td::string scope) override; + void create_public_overlay_external(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + OverlayPrivacyRules rules, td::string scope) override; void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules) override; diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index 0539383f94..dbb6f32a87 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -142,7 +142,9 @@ void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::BufferSlic void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode node, td::Promise promise) { std::vector> vec; - vec.emplace_back(node.tl()); + if (!is_external()) { + vec.emplace_back(node.tl()); + } for (td::uint32 i = 0; i < nodes_to_send(); i++) { auto P = get_random_peer(); diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index a6d0bc89f4..8a655c6e59 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -18,6 +18,7 @@ */ #include "auto/tl/ton_api.h" #include "td/utils/Random.h" +#include "common/delay.h" #include "adnl/utils.hpp" #include "dht/dht.h" @@ -73,7 +74,12 @@ OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor , scope_(scope) { overlay_id_ = id_full_.compute_short_id(); - VLOG(OVERLAY_INFO) << this << ": creating " << (public_ ? "public" : "private"); + if (is_external()) { + CHECK(public_); + VLOG(OVERLAY_INFO) << this << ": creating public external"; + } else { + VLOG(OVERLAY_INFO) << this << ": creating " << (public_ ? "public" : "private"); + } for (auto &node : nodes) { CHECK(!public_); @@ -86,6 +92,7 @@ OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeers &query, td::Promise promise) { + CHECK(!is_external()); if (public_) { VLOG(OVERLAY_DEBUG) << this << ": received " << query.peers_->nodes_.size() << " nodes from " << src << " in getRandomPeers query"; @@ -106,6 +113,7 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getR void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, td::Promise promise) { + CHECK(!is_external()); auto it = broadcasts_.find(query.hash_); if (it == broadcasts_.end()) { VLOG(OVERLAY_NOTICE) << this << ": received getBroadcastQuery(" << query.hash_ << ") from " << src @@ -127,16 +135,17 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getB void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcastList &query, td::Promise promise) { + CHECK(!is_external()); VLOG(OVERLAY_WARNING) << this << ": DROPPING getBroadcastList query"; promise.set_error(td::Status::Error(ErrorCode::protoviolation, "dropping get broadcast list query")); } -/*void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, adnl::AdnlQueryId query_id, ton_api::overlay_customQuery &query) { - callback_->receive_query(src, query_id, id_, std::move(query.data_)); -} -*/ - void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) { + if (is_external()) { + LOG(OVERLAY_WARNING) << "dropping query in external overlay " << overlay_id_; + promise.set_error(td::Status::Error("overlay is external")); + return; + } if (!public_) { auto P = peers_.get(src); if (P == nullptr) { @@ -149,9 +158,6 @@ void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, if (R.is_error()) { // allow custom query to be here - if (!subscribed()) { - return; - } callback_->receive_query(src, overlay_id_, std::move(data), std::move(promise)); return; } @@ -165,27 +171,32 @@ void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr bcast) { + CHECK(!is_external()); return BroadcastSimple::create(this, message_from, std::move(bcast)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr b) { + CHECK(!is_external()); return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr b) { + CHECK(!is_external()); return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr bcast) { + CHECK(!is_external()); return td::Status::Error(ErrorCode::protoviolation, PSTRING() << "received strange message broadcastNotFound from " << message_from); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { + CHECK(!is_external()); auto it = fec_broadcasts_.find(msg->hash_); if (it != fec_broadcasts_.end()) { VLOG(OVERLAY_DEBUG) << this << ": received fec opt-out message from " << message_from << " for broadcast " @@ -200,6 +211,7 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { + CHECK(!is_external()); auto it = fec_broadcasts_.find(msg->hash_); if (it != fec_broadcasts_.end()) { VLOG(OVERLAY_DEBUG) << this << ": received fec completed message from " << message_from << " for broadcast " @@ -214,12 +226,17 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { + CHECK(!is_external()); VLOG(OVERLAY_DEBUG) << this << ": received unicast from " << message_from; callback_->receive_message(message_from, overlay_id_, std::move(msg->data_)); return td::Status::OK(); } void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) { + if (is_external()) { + LOG(OVERLAY_WARNING) << "dropping message in external overlay " << overlay_id_; + return; + } if (!public_) { if (peers_.get(src) == nullptr) { VLOG(OVERLAY_WARNING) << this << ": received query in private overlay from unknown source " << src; @@ -228,9 +245,6 @@ void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice dat } auto X = fetch_tl_object(data.clone(), true); if (X.is_error()) { - if (!subscribed()) { - return; - } VLOG(OVERLAY_DEBUG) << this << ": received custom message"; callback_->receive_message(src, overlay_id_, std::move(data)); return; @@ -274,7 +288,7 @@ void OverlayImpl::alarm() { } if (public_) { - if (peers_.size() > 0 && subscribed()) { + if (peers_.size() > 0) { auto P = get_random_peer(); if (P) { send_random_peers(P->get_id(), {}); @@ -330,6 +344,10 @@ void OverlayImpl::receive_dht_nodes(td::Result res, bool dummy) { VLOG(OVERLAY_NOTICE) << this << ": can not get value from DHT: " << res.move_as_error(); } + if (is_external()) { + return; + } + VLOG(OVERLAY_INFO) << this << ": adding self node to DHT overlay's nodes"; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), oid = print_id()](td::Result R) { if (R.is_error()) { @@ -342,7 +360,7 @@ void OverlayImpl::receive_dht_nodes(td::Result res, bool dummy) { } void OverlayImpl::update_dht_nodes(OverlayNode node) { - if (!public_ || !subscribed()) { + if (!public_) { return; } @@ -399,12 +417,30 @@ void OverlayImpl::bcast_gc() { } void OverlayImpl::send_message_to_neighbours(td::BufferSlice data) { + if (neighbours_.empty()) { + // TODO: limit retries + delay_action( + [SelfId = actor_id(this), data = std::move(data)]() mutable { + td::actor::send_closure(SelfId, &OverlayImpl::send_message_to_neighbours, std::move(data)); + }, + td::Timestamp::in(0.5)); + return; + } for (auto &n : neighbours_) { td::actor::send_closure(manager_, &OverlayManager::send_message, n, local_id_, overlay_id_, data.clone()); } } void OverlayImpl::send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { + if (neighbours_.empty()) { + // TODO: limit retries + delay_action( + [SelfId = actor_id(this), send_as, flags, data = std::move(data)]() mutable { + td::actor::send_closure(SelfId, &OverlayImpl::send_broadcast, send_as, flags, std::move(data)); + }, + td::Timestamp::in(0.5)); + return; + } auto S = BroadcastSimple::create_new(actor_id(this), keyring_, send_as, std::move(data), flags); if (S.is_error()) { LOG(WARNING) << "failed to send broadcast: " << S; @@ -412,6 +448,15 @@ void OverlayImpl::send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::Bu } void OverlayImpl::send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { + if (neighbours_.empty()) { + // TODO: limit retries + delay_action( + [SelfId = actor_id(this), send_as, flags, data = std::move(data)]() mutable { + td::actor::send_closure(SelfId, &OverlayImpl::send_broadcast_fec, send_as, flags, std::move(data)); + }, + td::Timestamp::in(0.5)); + return; + } OverlayOutboundFecBroadcast::create(std::move(data), flags, actor_id(this), send_as); } @@ -503,7 +548,7 @@ void OverlayImpl::send_new_fec_broadcast_part(PublicKeyHash local_id, Overlay::B } void OverlayImpl::deliver_broadcast(PublicKeyHash source, td::BufferSlice data) { - if (!subscribed()) { + if (is_external()) { return; } callback_->receive_broadcast(source, overlay_id_, std::move(data)); @@ -578,7 +623,8 @@ void OverlayImpl::set_privacy_rules(OverlayPrivacyRules rules) { } void OverlayImpl::check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise promise) { - if (!subscribed()) { + if (is_external()) { + promise.set_result(td::Unit()); return; } callback_->check_broadcast(src, overlay_id_, std::move(data), std::move(promise)); diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index ddcb026713..59b3ec06ac 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -251,15 +251,13 @@ class OverlayImpl : public Overlay { } private: - bool subscribed() const { - return (bool)callback_; + bool is_external() const { + return callback_ == nullptr; } template void process_query(adnl::AdnlNodeIdShort src, T &query, td::Promise promise) { - if (!subscribed()) { - return; - } + CHECK(!is_external()); callback_->receive_query(src, overlay_id_, serialize_tl_object(&query, true), std::move(promise)); } diff --git a/overlay/overlays.h b/overlay/overlays.h index 6781b6c32b..82617fab63 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -194,8 +194,8 @@ class Overlays : public td::actor::Actor { virtual void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) = 0; - virtual void create_public_overlay_no_subscribe(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - OverlayPrivacyRules rules, td::string scope) = 0; + virtual void create_public_overlay_external(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + OverlayPrivacyRules rules, td::string scope) = 0; virtual void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules) = 0; diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 1e46bc7df6..b3d3f9db42 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -323,7 +323,7 @@ class TestNode : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void subscribe_to_shard(ton::ShardIdFull) override { + void update_shard_configuration(td::Ref state) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/ton/ton-types.h b/ton/ton-types.h index 754fb86a17..7d0101d10b 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -57,6 +57,8 @@ constexpr unsigned min_split_merge_interval = 30; // split/merge interval must constexpr unsigned max_split_merge_delay = 1000; // end of split/merge interval must be at most 1000 seconds in the future +constexpr int max_shard_pfx_len = 60; + enum GlobalCapabilities { capIhrEnabled = 1, capCreateStatsEnabled = 2, diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 09526279f7..1bb820a42a 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1324,18 +1324,6 @@ td::Status ValidatorEngine::load_global_config() { } validator_options_ = ton::validator::ValidatorManagerOptions::create(zero_state, init_block); - validator_options_.write().set_shard_check_function( - [masterchain_only = masterchain_only_](ton::ShardIdFull shard, ton::CatchainSeqno cc_seqno, - ton::validator::ValidatorManagerOptions::ShardCheckMode mode) -> bool { - if (mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_monitor) { - return shard.is_masterchain() || !masterchain_only; - } - CHECK(mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_validate); - return true; - /*ton::ShardIdFull p{ton::basechainId, ((cc_seqno * 1ull % 4) << 62) + 1}; - auto s = ton::shard_prefix(p, 2); - return shard.is_masterchain() || ton::shard_intersects(shard, s);*/ - }); if (state_ttl_ != 0) { validator_options_.write().set_state_ttl(state_ttl_); } @@ -1389,6 +1377,39 @@ td::Status ValidatorEngine::load_global_config() { return td::Status::OK(); } +void ValidatorEngine::init_validator_options() { + if (!masterchain_only_) { + validator_options_.write().set_shard_check_function( + [](ton::ShardIdFull shard, ton::CatchainSeqno cc_seqno, + ton::validator::ValidatorManagerOptions::ShardCheckMode mode) -> bool { + if (mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_monitor) { + return true; + } + CHECK(mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_validate); + return true; + }); + } else { + std::vector shards = {ton::ShardIdFull(ton::masterchainId)}; + for (const auto& c : config_.collators) { + shards.push_back(c.shard); + } + validator_options_.write().set_shard_check_function( + [shards = std::move(shards)](ton::ShardIdFull shard, ton::CatchainSeqno cc_seqno, + ton::validator::ValidatorManagerOptions::ShardCheckMode mode) -> bool { + if (mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_monitor) { + for (auto s : shards) { + if (shard_is_ancestor(shard, s)) { + return true; + } + } + return false; + } + CHECK(mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_validate); + return true; + }); + } +} + void ValidatorEngine::load_empty_local_config(td::Promise promise) { auto ret_promise = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { @@ -1671,6 +1692,7 @@ void ValidatorEngine::got_key(ton::PublicKey key) { } void ValidatorEngine::start() { + init_validator_options(); read_config_ = true; start_adnl(); } @@ -1825,19 +1847,22 @@ void ValidatorEngine::start_full_node() { }; full_node_client_ = ton::adnl::AdnlExtMultiClient::create(std::move(vec), std::make_unique()); } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &ValidatorEngine::started_full_node); + }); full_node_ = ton::validator::fullnode::FullNode::create( short_id, ton::adnl::AdnlNodeIdShort{config_.full_node}, validator_options_->zero_block_id().file_hash, - keyring_.get(), adnl_.get(), rldp_.get(), + validator_options_, keyring_.get(), adnl_.get(), rldp_.get(), default_dht_node_.is_zero() ? td::actor::ActorId{} : dht_nodes_[default_dht_node_].get(), - overlay_manager_.get(), validator_manager_.get(), full_node_client_.get(), db_root_); - } - - for (auto &v : config_.validators) { - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_permanent_key, v.first, - [](td::Unit) {}); + overlay_manager_.get(), validator_manager_.get(), full_node_client_.get(), db_root_, std::move(P)); + for (auto &v : config_.validators) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_permanent_key, v.first, + [](td::Unit) {}); + } + } else { + started_full_node(); } - - started_full_node(); } void ValidatorEngine::started_full_node() { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 037288cee2..bf47e2d793 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -276,6 +276,7 @@ class ValidatorEngine : public td::actor::Actor { void load_empty_local_config(td::Promise promise); void load_local_config(td::Promise promise); void load_config(td::Promise promise); + void init_validator_options(); void start(); diff --git a/validator/fabric.h b/validator/fabric.h index 6f06dcc9dc..12452ec55b 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -52,7 +52,7 @@ void run_check_external_message(td::BufferSlice data, td::actor::ActorId data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, bool send_broadcast, + td::Ref approve_signatures, bool send_broadcast, bool apply, td::actor::ActorId manager, td::Promise promise); void run_fake_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::actor::ActorId manager, @@ -61,7 +61,7 @@ void run_hardfork_accept_block_query(BlockIdExt id, td::Ref data, td::actor::ActorId manager, td::Promise promise); void run_broadcast_only_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, + td::Ref approve_signatures, bool send_block_broadcast, td::actor::ActorId manager, td::Promise promise); void run_apply_block_query(BlockIdExt id, td::Ref block, BlockIdExt masterchain_block_id, td::actor::ActorId manager, td::Timestamp timeout, diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 8e579f3c8c..7eb00db8f8 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -69,7 +69,7 @@ void Neighbour::update_roundtrip(double t) { } void FullNodeShardImpl::create_overlay() { - if (subscribed_) { + if (active_) { class Callback : public overlay::Overlays::Callback { public: void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, @@ -99,7 +99,7 @@ void FullNodeShardImpl::create_overlay() { PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() << ", \"workchain_id\": " << get_workchain() << " }"); } else { - td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay_no_subscribe, adnl_id_, + td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay_external, adnl_id_, overlay_id_full_.clone(), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() << ", \"workchain_id\": " << get_workchain() << " }"); @@ -129,12 +129,12 @@ void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promis create_overlay(); } -void FullNodeShardImpl::subscribe_to_shard() { - if (subscribed_ || !client_.empty()) { +void FullNodeShardImpl::set_active(bool active) { + if (active_ == active || shard_.is_masterchain()) { return; } td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); - subscribed_ = true; + active_ = active; create_overlay(); } @@ -576,6 +576,10 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { + if (!active_) { + promise.set_error(td::Status::Error("shard is inactive")); + return; + } auto B = fetch_tl_object(std::move(query), true); if (B.is_error()) { promise.set_error(td::Status::Error(ErrorCode::protoviolation, "cannot parse tonnode query")); @@ -596,28 +600,23 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ex void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { auto block_id = create_block_id(query.block_->block_); - if (block_id.shard_full() != shard_) { - LOG(WARNING) << "dropping newShardBlockBroadcast: expected shard " << shard_.to_str() << ", got shard " - << block_id.shard_full().to_str(); - return; - } td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, block_id, query.block_->cc_seqno_, std::move(query.block_->data_)); } void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { - std::vector signatures; - for (auto &sig : query.signatures_) { - signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); - } - BlockIdExt block_id = create_block_id(query.id_); if (block_id.shard_full() != shard_) { - LOG(WARNING) << "dropping blockBroadcast: expected shard " << shard_.to_str() << ", got shard " - << block_id.shard_full().to_str(); + LOG(FULL_NODE_WARNING) << "dropping block broadcast: shard mismatch. overlay=" << shard_.to_str() + << " block=" << block_id.to_str(); return; } + + std::vector signatures; + for (auto &sig : query.signatures_) { + signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); + } BlockBroadcast B{block_id, std::move(signatures), static_cast(query.catchain_seqno_), @@ -639,6 +638,9 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_bl } void FullNodeShardImpl::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { + if (!active_) { + return; + } auto B = fetch_tl_object(std::move(broadcast), true); if (B.is_error()) { return; @@ -1087,7 +1089,7 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, bool subscribe) + td::actor::ActorId client, bool active) : shard_(shard) , local_id_(local_id) , adnl_id_(adnl_id) @@ -1098,7 +1100,7 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, , overlays_(overlays) , validator_manager_(validator_manager) , client_(client) - , subscribed_(subscribe) { + , active_(shard.is_masterchain() || active) { } td::actor::ActorOwn FullNodeShard::create( @@ -1106,9 +1108,9 @@ td::actor::ActorOwn FullNodeShard::create( td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, - bool subscribe) { + bool active) { return td::actor::create_actor("tonnode", shard, local_id, adnl_id, zero_state_file_hash, keyring, - adnl, rldp, overlays, validator_manager, client, subscribe); + adnl, rldp, overlays, validator_manager, client, active); } } // namespace fullnode diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index e45e2325c3..2b2098a690 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -36,7 +36,7 @@ class FullNodeShard : public td::actor::Actor { virtual ShardIdFull get_shard_full() const = 0; virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; - virtual void subscribe_to_shard() = 0; + virtual void set_active(bool active) = 0; virtual void send_ihr_message(td::BufferSlice data) = 0; virtual void send_external_message(td::BufferSlice data) = 0; @@ -72,7 +72,7 @@ class FullNodeShard : public td::actor::Actor { td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, - bool subscribe); + bool active = true); }; } // namespace fullnode diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 33705bc799..9a022cbf08 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -80,7 +80,7 @@ class FullNodeShardImpl : public FullNodeShard { void create_overlay(); void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; - virtual void subscribe_to_shard() override; + void set_active(bool active) override; //td::Result fetch_block(td::BufferSlice data); void prevalidate_block(BlockIdExt block_id, td::BufferSlice data, td::BufferSlice proof, @@ -204,7 +204,7 @@ class FullNodeShardImpl : public FullNodeShard { td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, bool subscribe); + td::actor::ActorId client, bool active = true); private: bool use_new_download() const { @@ -242,7 +242,7 @@ class FullNodeShardImpl : public FullNodeShard { td::Timestamp ping_neighbours_at_; adnl::AdnlNodeIdShort last_pinged_neighbour_ = adnl::AdnlNodeIdShort::zero(); - bool subscribed_ = false; + bool active_ = false; }; } // namespace fullnode diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 5712d3cf71..9c26cf3c4d 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -20,6 +20,7 @@ #include "ton/ton-shard.h" #include "ton/ton-io.hpp" #include "td/actor/MultiPromise.h" +#include "ton/ton-types.h" namespace ton { @@ -27,6 +28,8 @@ namespace validator { namespace fullnode { +static const double INACTIVE_SHARD_TTL = 120.0; + void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise promise) { if (local_keys_.count(key)) { promise.set_value(td::Unit()); @@ -47,7 +50,9 @@ void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise pr } for (auto &shard : shards_) { - td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + if (!shard.second.actor.empty()) { + td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } } promise.set_value(td::Unit()); } @@ -71,7 +76,9 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr } for (auto &shard : shards_) { - td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + if (!shard.second.actor.empty()) { + td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } } promise.set_value(td::Unit()); } @@ -79,22 +86,25 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr void FullNodeImpl::sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) { - auto it = shards_.find(shard_id); - if(it == shards_.end()) { - promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); - return; - } - td::actor::send_closure(it->second, &FullNodeShard::sign_overlay_certificate, signed_key, expiry_at, max_size, std::move(promise)); + auto it = shards_.find(shard_id); + if(it == shards_.end() || it->second.actor.empty()) { + promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); + return; + } + td::actor::send_closure(it->second.actor, &FullNodeShard::sign_overlay_certificate, signed_key, expiry_at, max_size, + std::move(promise)); } void FullNodeImpl::import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) { - auto it = shards_.find(shard_id); - if(it == shards_.end()) { - promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); - } - td::actor::send_closure(it->second, &FullNodeShard::import_overlay_certificate, signed_key, cert, std::move(promise)); + auto it = shards_.find(shard_id); + if(it == shards_.end() || it->second.actor.empty()) { + promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); + return; + } + td::actor::send_closure(it->second.actor, &FullNodeShard::import_overlay_certificate, signed_key, cert, + std::move(promise)); } void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { @@ -105,7 +115,9 @@ void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promisesecond, &FullNodeShard::set_handle, top_handle, std::move(P)); + CHECK(it != shards_.end() && !it->second.actor.empty()); + td::actor::send_closure(it->second.actor, &FullNodeShard::set_handle, top_handle, std::move(P)); } -void FullNodeImpl::add_shard(ShardIdFull shard, bool subscribe) { - while (true) { - auto it = shards_.find(shard); - if (it == shards_.end()) { - shards_.emplace(shard, FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, keyring_, adnl_, - rldp_, overlays_, validator_manager_, client_, subscribe)); - if (all_validators_.size() > 0) { - td::actor::send_closure(shards_[shard], &FullNodeShard::update_validators, all_validators_, sign_cert_by_); +void FullNodeImpl::update_shard_configuration(td::Ref state) { + std::map new_shards; + std::set new_active; + new_shards[ShardIdFull(masterchainId)] = state->get_block_id(); + new_active.insert(ShardIdFull(masterchainId)); + std::set workchains; + auto cur_time = state->get_unix_time(); + + auto set_active = [&](ShardIdFull shard) { + while (new_active.insert(shard).second && shard.pfx_len() > 0) { + shard = shard_parent(shard); + } + }; + + for (auto &info : state->get_shards()) { + auto shard = info->shard(); + workchains.insert(shard.workchain); + new_shards[shard] = info->top_block_id(); + bool will_split = shard.pfx_len() < max_shard_pfx_len && ((info->fsm_state() == McShardHash::FsmState::fsm_split && + info->fsm_utime() < cur_time + 60) || info->before_split()); + bool will_merge = shard.pfx_len() > 0 && ((info->fsm_state() == McShardHash::FsmState::fsm_merge && + info->fsm_utime() < cur_time + 60) || info->before_merge()); + if (opts_->need_monitor(shard)) { + set_active(shard); + } + if (will_merge && opts_->need_monitor(shard_parent(shard))) { + set_active(shard); + set_active(shard_sibling(shard)); + } + for (int id = 0; id < 2; ++id) { + if (will_split && opts_->need_monitor(shard_child(shard, id))) { + set_active(shard_child(shard, id)); } - } else if (subscribe) { - td::actor::send_closure(it->second, &FullNodeShard::subscribe_to_shard); - } else { - break; } - if (shard.shard == shardIdAll) { - break; + } + for (const auto &wpair : state->get_workchain_list()) { + ton::WorkchainId wc = wpair.first; + const block::WorkchainInfo *winfo = wpair.second.get(); + if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= cur_time) { + auto shard = ShardIdFull(wc); + new_shards[shard] = BlockIdExt(wc, shard.shard, 0, winfo->zerostate_root_hash, winfo->zerostate_file_hash); + if (opts_->need_monitor(shard)) { + set_active(shard); + } + } + } + + auto info_set_active = [&](ShardIdFull shard, ShardInfo& info, bool active) { + if (info.active == active) { + return; + } + if (info.actor.empty()) { + add_shard_actor(shard, active); + return; + } + info.active = active; + td::actor::send_closure(info.actor, &FullNodeShard::set_active, active); + info.delete_at = active ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); + }; + + for (auto shard : new_shards) { + auto &info = shards_[shard.first]; + info.exists = true; + if (!info.active && new_active.count(shard.first)) { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::wait_block_state_short, shard.second, 0, + td::Timestamp::in(60.0), [](td::Result>){}); + } + } + + for (auto& p : shards_) { + ShardIdFull shard = p.first; + ShardInfo &info = p.second; + info.exists = new_shards.count(shard); + info_set_active(shard, info, new_active.count(shard)); + } + + for (ShardIdFull shard : new_active) { + info_set_active(shard, shards_[shard], true); + } + + auto it = shards_.begin(); + while (it != shards_.end()) { + if (!it->second.active && it->second.delete_at && it->second.delete_at.is_in_past()) { + it->second.actor.reset(); + it->second.delete_at = td::Timestamp::never(); + } + if (!it->second.exists && it->second.actor.empty()) { + it = shards_.erase(it); + } else { + ++it; } - shard = shard_parent(shard); } } -void FullNodeImpl::del_shard(ShardIdFull shard) { - LOG(FATAL) << "deleting shards not implemented: shard=" << shard; - shards_.erase(shard); +void FullNodeImpl::add_shard_actor(ShardIdFull shard, bool active) { + ShardInfo &info = shards_[shard]; + if (!info.actor.empty()) { + return; + } + info.actor = FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, keyring_, adnl_, rldp_, + overlays_, validator_manager_, client_, active); + info.active = active; + info.delete_at = active ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); + if (all_validators_.size() > 0) { + td::actor::send_closure(info.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } } void FullNodeImpl::sync_completed() { @@ -169,7 +263,7 @@ void FullNodeImpl::send_ext_message(AccountIdPrefixFull dst, td::BufferSlice dat } void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - auto shard = get_shard(block_id.shard_full()); + auto shard = get_shard(ShardIdFull{masterchainId}); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; return; @@ -178,7 +272,7 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s } void FullNodeImpl::send_broadcast(BlockBroadcast broadcast) { - auto shard = get_shard(broadcast.block_id.shard_full()); + auto shard = get_shard(broadcast.block_id.shard_full(), true); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; return; @@ -262,19 +356,38 @@ void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, std::string tm std::move(promise)); } -td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { - add_shard(ShardIdFull{shard.workchain, shardIdAll}); - while (shards_.count(shard) == 0) { - if (shard.shard == shardIdAll) { - return td::actor::ActorId{}; +td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard, bool exact) { + if (!exact) { + ShardIdFull s = shard; + while (true) { + auto it = shards_.find(s); + if (it != shards_.end() && it->second.exists) { + if (it->second.actor.empty()) { + add_shard_actor(s, false); + } + if (!it->second.active) { + it->second.delete_at = td::Timestamp::in(INACTIVE_SHARD_TTL); + } + return it->second.actor.get(); + } + if (s.pfx_len() == 0) { + break; + } + s = shard_parent(s); } - shard = shard_parent(shard); } - return shards_[shard].get(); + auto &info = shards_[shard]; + if (info.actor.empty()) { + add_shard_actor(shard, false); + } + if (!info.active) { + info.delete_at = td::Timestamp::in(INACTIVE_SHARD_TTL); + } + return info.actor.get(); } td::actor::ActorId FullNodeImpl::get_shard(AccountIdPrefixFull dst) { - return get_shard(shard_prefix(dst, 60)); + return get_shard(shard_prefix(dst, max_shard_pfx_len)); } void FullNodeImpl::got_key_block_proof(td::Ref proof) { @@ -307,7 +420,7 @@ void FullNodeImpl::got_key_block_proof(td::Ref proof) { CHECK(all_validators_.size() > 0); for (auto &shard : shards_) { - td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } } @@ -339,7 +452,9 @@ void FullNodeImpl::got_zero_block_state(td::Ref state) { CHECK(all_validators_.size() > 0); for (auto &shard : shards_) { - td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + if (!shard.second.actor.empty()) { + td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } } } @@ -384,8 +499,8 @@ void FullNodeImpl::start_up() { void initial_read_complete(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::initial_read_complete, handle); } - void subscribe_to_shard(ShardIdFull shard) override { - td::actor::send_closure(id_, &FullNodeImpl::add_shard, shard, true); + void update_shard_configuration(td::Ref state) override { + td::actor::send_closure(id_, &FullNodeImpl::update_shard_configuration, std::move(state)); } void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) override { td::actor::send_closure(id_, &FullNodeImpl::send_ihr_message, dst, std::move(data)); @@ -443,20 +558,21 @@ void FullNodeImpl::start_up() { td::actor::ActorId id_; }; - auto P = td::PromiseCreator::lambda([](td::Unit R) {}); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::install_callback, - std::make_unique(actor_id(this)), std::move(P)); + std::make_unique(actor_id(this)), std::move(started_promise_)); } FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId dht, - td::actor::ActorId overlays, + td::Ref opts, td::actor::ActorId keyring, + td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root) + td::actor::ActorId client, std::string db_root, + td::Promise started_promise) : local_id_(local_id) , adnl_id_(adnl_id) , zero_state_file_hash_(zero_state_file_hash) + , opts_(opts) , keyring_(keyring) , adnl_(adnl) , rldp_(rldp) @@ -464,20 +580,23 @@ FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id , overlays_(overlays) , validator_manager_(validator_manager) , client_(client) - , db_root_(db_root) { - add_shard(ShardIdFull{masterchainId}, true); + , db_root_(db_root) + , started_promise_(std::move(started_promise)) { + add_shard_actor(ShardIdFull{masterchainId}, true); } td::actor::ActorOwn FullNode::create(ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, + FileHash zero_state_file_hash, td::Ref opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root) { - return td::actor::create_actor("fullnode", local_id, adnl_id, zero_state_file_hash, keyring, adnl, rldp, - dht, overlays, validator_manager, client, db_root); + td::actor::ActorId client, std::string db_root, + td::Promise started_promise) { + return td::actor::create_actor("fullnode", local_id, adnl_id, zero_state_file_hash, opts, keyring, adnl, + rldp, dht, overlays, validator_manager, client, db_root, + std::move(started_promise)); } } // namespace fullnode diff --git a/validator/full-node.h b/validator/full-node.h index cdf39d6fac..7342a82295 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -73,13 +73,14 @@ class FullNode : public td::actor::Actor { } static td::actor::ActorOwn create(ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, + FileHash zero_state_file_hash, td::Ref opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root); + td::actor::ActorId client, std::string db_root, + td::Promise started_promise); }; } // namespace fullnode diff --git a/validator/full-node.hpp b/validator/full-node.hpp index f3ce9d0505..47cca0f57b 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -52,8 +52,7 @@ class FullNodeImpl : public FullNode { void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; - void add_shard(ShardIdFull shard, bool subscribe = false); - void del_shard(ShardIdFull shard); + void update_shard_configuration(td::Ref state); void sync_completed(); @@ -82,21 +81,31 @@ class FullNodeImpl : public FullNode { void start_up() override; FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId dht, - td::actor::ActorId overlays, + td::Ref opts, td::actor::ActorId keyring, + td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root); + td::actor::ActorId client, std::string db_root, + td::Promise started_promise); private: + void add_shard_actor(ShardIdFull shard, bool active); + PublicKeyHash local_id_; adnl::AdnlNodeIdShort adnl_id_; FileHash zero_state_file_hash_; + td::Ref opts_; td::actor::ActorId get_shard(AccountIdPrefixFull dst); - td::actor::ActorId get_shard(ShardIdFull dst); + td::actor::ActorId get_shard(ShardIdFull shard, bool exact = false); - std::map> shards_; + struct ShardInfo { + bool exists = false; + td::actor::ActorOwn actor; + bool active = false; + td::Timestamp delete_at = td::Timestamp::never(); + }; + std::map shards_; td::actor::ActorId keyring_; td::actor::ActorId adnl_; @@ -112,6 +121,8 @@ class FullNodeImpl : public FullNode { std::vector all_validators_; std::set local_keys_; + + td::Promise started_promise_; }; } // namespace fullnode diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index 74adcd696e..6491a71167 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -41,7 +41,7 @@ using namespace std::literals::string_literals; AcceptBlockQuery::AcceptBlockQuery(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, bool send_broadcast, + td::Ref approve_signatures, bool send_broadcast, bool apply, td::actor::ActorId manager, td::Promise promise) : id_(id) , data_(std::move(data)) @@ -52,6 +52,7 @@ AcceptBlockQuery::AcceptBlockQuery(BlockIdExt id, td::Ref data, std:: , is_fake_(false) , is_fork_(false) , send_broadcast_(send_broadcast) + , apply_(apply) , manager_(manager) , promise_(std::move(promise)) { state_keep_old_hash_.clear(); @@ -92,28 +93,6 @@ AcceptBlockQuery::AcceptBlockQuery(ForceFork ffork, BlockIdExt id, td::Ref data, std::vector prev, - td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, - td::actor::ActorId manager, td::Promise promise) - : id_(id) - , data_(std::move(data)) - , prev_(std::move(prev)) - , validator_set_(std::move(validator_set)) - , signatures_(std::move(signatures)) - , approve_signatures_(std::move(approve_signatures)) - , is_fake_(false) - , is_fork_(false) - , send_broadcast_(true) - , broadcast_only_(false) - , manager_(manager) - , promise_(std::move(promise)) { - state_keep_old_hash_.clear(); - state_old_hash_.clear(); - state_hash_.clear(); - CHECK(prev_.size() > 0); -} - bool AcceptBlockQuery::precheck_header() { VLOG(VALIDATOR_DEBUG) << "precheck_header()"; // 0. sanity check @@ -357,7 +336,9 @@ bool AcceptBlockQuery::check_send_error(td::actor::ActorId Sel } void AcceptBlockQuery::finish_query() { - ValidatorInvariants::check_post_accept(handle_); + if (apply_) { + ValidatorInvariants::check_post_accept(handle_); + } if (is_masterchain()) { CHECK(handle_->inited_proof()); } else { @@ -406,15 +387,6 @@ void AcceptBlockQuery::start_up() { return; } - if (broadcast_only_) { - if (!create_new_proof()) { - fatal_error("cannot generate proof for block "s + id_.to_str()); - return; - } - applied(); - return; - } - td::actor::send_closure( manager_, &ValidatorManager::get_block_handle, id_, true, [SelfId = actor_id(this)](td::Result R) { check_send_error(SelfId, R) || @@ -479,6 +451,10 @@ void AcceptBlockQuery::written_block_signatures() { void AcceptBlockQuery::written_block_info() { VLOG(VALIDATOR_DEBUG) << "written block info"; if (data_.not_null()) { + if (!apply_) { + written_state({}); + return; + } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::got_prev_state, R.move_as_ok()); @@ -555,7 +531,7 @@ void AcceptBlockQuery::written_state(td::Ref upd_state) { return; } - if (state_keep_old_hash_ != state_old_hash_) { + if (apply_ && state_keep_old_hash_ != state_old_hash_) { fatal_error(PSTRING() << "invalid previous state hash in newly-created proof: expected " << state_->root_hash().to_hex() << ", found in update " << state_old_hash_.to_hex()); return; diff --git a/validator/impl/accept-block.hpp b/validator/impl/accept-block.hpp index b45095b4f1..1a1f2a543a 100644 --- a/validator/impl/accept-block.hpp +++ b/validator/impl/accept-block.hpp @@ -48,20 +48,15 @@ class AcceptBlockQuery : public td::actor::Actor { public: struct IsFake {}; struct ForceFork {}; - struct BroadcastOnly{}; AcceptBlockQuery(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, bool send_broadcast, + td::Ref approve_signatures, bool send_broadcast, bool apply, td::actor::ActorId manager, td::Promise promise); AcceptBlockQuery(IsFake fake, BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::actor::ActorId manager, td::Promise promise); AcceptBlockQuery(ForceFork ffork, BlockIdExt id, td::Ref data, td::actor::ActorId manager, td::Promise promise); - AcceptBlockQuery(BroadcastOnly, BlockIdExt id, td::Ref data, std::vector prev, - td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, td::actor::ActorId manager, - td::Promise promise); private: static constexpr td::uint32 priority() { @@ -103,7 +98,7 @@ class AcceptBlockQuery : public td::actor::Actor { bool is_fake_; bool is_fork_; bool send_broadcast_; - bool broadcast_only_{false}; + bool apply_ = true; bool ancestors_split_{false}, is_key_block_{false}; td::Timestamp timeout_ = td::Timestamp::in(600.0); td::actor::ActorId manager_; diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index e86b9b90de..47979af8bf 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -127,10 +127,10 @@ td::Result> create_ihr_message(td::BufferSlice data) { void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, bool send_broadcast, + td::Ref approve_signatures, bool send_broadcast, bool apply, td::actor::ActorId manager, td::Promise promise) { td::actor::create_actor("accept", id, std::move(data), prev, std::move(validator_set), - std::move(signatures), std::move(approve_signatures), send_broadcast, + std::move(signatures), std::move(approve_signatures), send_broadcast, apply, manager, std::move(promise)) .release(); } @@ -151,17 +151,6 @@ void run_hardfork_accept_block_query(BlockIdExt id, td::Ref data, .release(); } -void run_broadcast_only_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, - td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, - td::actor::ActorId manager, - td::Promise promise) { - td::actor::create_actor("broadcastaccept", AcceptBlockQuery::BroadcastOnly(), id, std::move(data), - prev, std::move(validator_set), std::move(signatures), - std::move(approve_signatures), manager, std::move(promise)) - .release(); -} - void run_apply_block_query(BlockIdExt id, td::Ref block, BlockIdExt masterchain_block_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise) { diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index c72ec307cb..209f871f29 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -64,10 +64,6 @@ class ValidatorManager : public ValidatorManagerInterface { std::function write_data, td::Promise promise) = 0; virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) = 0; - virtual void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) = 0; - virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) = 0; virtual void set_block_data(BlockHandle handle, td::Ref data, td::Promise promise) = 0; virtual void wait_block_data(BlockHandle handle, td::uint32 priority, td::Timestamp, @@ -135,7 +131,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; - virtual void subscribe_to_shard(ShardIdFull shard) = 0; + virtual void update_shard_configuration(td::Ref state) = 0; virtual void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) = 0; virtual void get_async_serializer_state(td::Promise promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 91235ebf62..501ee078c4 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -253,7 +253,7 @@ class ValidatorManagerImpl : public ValidatorManager { void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; - void subscribe_to_shard(ShardIdFull shard) override { + void update_shard_configuration(td::Ref state) override { } void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override { diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 80f6df93ca..6778277143 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -321,7 +321,8 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_client_state(bool from_db, td::Promise promise) override { UNREACHABLE(); } - void subscribe_to_shard(ShardIdFull shard) override { + void update_shard_configuration(td::Ref state) override { + UNREACHABLE(); } void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override { diff --git a/validator/manager.cpp b/validator/manager.cpp index 9af0077264..7c5e7d3f10 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -377,7 +377,7 @@ void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { } auto R = create_ext_message(std::move(data)); if (R.is_error()) { - VLOG(VALIDATOR_NOTICE) << "dropping bad ihr message: " << R.move_as_error(); + VLOG(VALIDATOR_NOTICE) << "dropping bad external message: " << R.move_as_error(); return; } add_external_message(R.move_as_ok()); @@ -453,9 +453,9 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id().shard_full(), desc->catchain_seqno()}] = desc; VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); - if (last_masterchain_block_handle_ && last_masterchain_seqno_ > 0 && - desc->generated_at() < last_masterchain_block_handle_->unix_time() + 60) { - delay_action( + if (opts_->need_monitor(desc->block_id().shard_full()) && last_masterchain_block_handle_ && + last_masterchain_seqno_ > 0 && desc->generated_at() < last_masterchain_block_handle_->unix_time() + 60) { + delay_action( [SelfId = actor_id(this), desc]() { auto P = td::PromiseCreator::lambda([](td::Result> R) { if (R.is_error()) { @@ -2367,8 +2367,8 @@ void ValidatorManagerImpl::get_shard_client_state(bool from_db, td::Promisesubscribe_to_shard(shard); +void ValidatorManagerImpl::update_shard_configuration(td::Ref state) { + callback_->update_shard_configuration(state); } void ValidatorManagerImpl::update_async_serializer_state(AsyncSerializerState state, td::Promise promise) { @@ -2397,6 +2397,7 @@ void ValidatorManagerImpl::get_archive_slice(td::uint64 archive_id, td::uint64 o } bool ValidatorManagerImpl::is_validator() { + // TODO: change is_vaidator to other condition in some cases return true; // temp_keys_.size() > 0 || permanent_keys_.size() > 0; } @@ -2700,6 +2701,9 @@ void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh it = collator_nodes_.emplace(id, std::move(actor)).first; } td::actor::send_closure(it->second, &CollatorNode::add_shard, shard); + if (shard.is_masterchain()) { + collating_masterchain_ = true; + } } td::actor::ActorOwn ValidatorManagerFactory::create( diff --git a/validator/manager.hpp b/validator/manager.hpp index 94657d49d3..1105b2b62a 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -447,7 +447,7 @@ class ValidatorManagerImpl : public ValidatorManager { void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; - void subscribe_to_shard(ShardIdFull shard) override; + void update_shard_configuration(td::Ref state) override; void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override; void get_async_serializer_state(td::Promise promise) override; @@ -611,6 +611,7 @@ class ValidatorManagerImpl : public ValidatorManager { void cleanup_old_pending_candidates(BlockId block_id, td::Timestamp now); std::map> collator_nodes_; + bool collating_masterchain_ = false; }; } // namespace validator diff --git a/validator/net/download-block-new.cpp b/validator/net/download-block-new.cpp index ef5ed7e54c..1063d8a35b 100644 --- a/validator/net/download-block-new.cpp +++ b/validator/net/download-block-new.cpp @@ -201,10 +201,10 @@ void DownloadBlockNew::got_node_to_download(adnl::AdnlNodeIdShort node) { } if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, - "get_proof", std::move(P), td::Timestamp::in(3.0), std::move(q), + "get_block_full", std::move(P), td::Timestamp::in(3.0), std::move(q), FullNode::max_proof_size() + FullNode::max_block_size() + 128, rldp_); } else { - td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get_prepare", + td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get_block_full", create_serialize_tl_object_suffix(std::move(q)), td::Timestamp::in(1.0), std::move(P)); } diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index af44f16e84..50c6075d82 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -247,39 +247,7 @@ void ShardClient::get_processed_masterchain_block_id(td::Promise pro } void ShardClient::build_shard_overlays() { - auto v = masterchain_state_->get_shards(); - std::set workchains; - - for (auto &x : v) { - auto shard = x->shard(); - workchains.insert(shard.workchain); - if (opts_->need_monitor(shard)) { - auto d = masterchain_state_->soft_min_split_depth(shard.workchain); - auto l = shard_prefix_length(shard.shard); - if (l > d) { - shard = shard_prefix(shard, d); - } - - if (created_overlays_.count(shard) == 0) { - created_overlays_.insert(shard); - td::actor::send_closure(manager_, &ValidatorManager::subscribe_to_shard, shard); - } - } - } - - for (const auto &wpair : masterchain_state_->get_workchain_list()) { - ton::WorkchainId wc = wpair.first; - const block::WorkchainInfo *winfo = wpair.second.get(); - if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= masterchain_state_->get_unix_time()) { - auto shard = ShardIdFull(wc); - if (opts_->need_monitor(shard) && created_overlays_.count(shard) == 0) { - td::actor::send_closure(manager_, &ValidatorManager::subscribe_to_shard, shard); - BlockIdExt block_id(shard.workchain, shard.shard, 0, winfo->zerostate_root_hash, winfo->zerostate_file_hash); - td::actor::send_closure_later(manager_, &ValidatorManager::wait_block_state_short, block_id, 0, - td::Timestamp::in(5.0), [](td::Result>) {}); - } - } - } + td::actor::send_closure(manager_, &ValidatorManager::update_shard_configuration, masterchain_state_); } void ShardClient::force_update_shard_client(BlockHandle handle, td::Promise promise) { diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index bb493f9120..8eae613329 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -105,8 +105,7 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat void ValidatorGroup::update_approve_cache(td::uint32 round_id, CacheKey key, UnixTime value) { if (approved_candidates_cache_round_ != round_id) { - approved_candidates_cache_round_ = round_id; - approved_candidates_cache_.clear(); + return; } approved_candidates_cache_[key] = value; } @@ -157,15 +156,9 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref } }); - if (shard_.is_masterchain() || lite_mode_) { - run_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, std::move(sig_set), - std::move(approve_sig_set), send_broadcast, manager_, std::move(P)); - } else if (send_broadcast) { - run_broadcast_only_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, - std::move(sig_set), std::move(approve_sig_set), manager_, std::move(P)); - } else { - promise.set_value(td::Unit()); - } + run_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, std::move(sig_set), + std::move(approve_sig_set), send_broadcast, shard_.is_masterchain() || !lite_mode_, manager_, + std::move(P)); } void ValidatorGroup::skip_round(td::uint32 round_id) { @@ -386,7 +379,7 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo td::actor::send_closure(SelfId, &ValidatorGroup::receive_collate_query_response, round_id, R.move_as_ok(), std::move(promise)); }); - LOG(INFO) << "collate query for " << shard_.to_str() << ": send query to " << collator; + LOG(INFO) << "collate query for " << create_next_block_id_simple().to_str() << ": send query to " << collator; size_t max_answer_size = config_.max_block_size + config_.max_collated_data_size + 256; td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, adnl::AdnlNodeIdShort(local_id_), collator, "collatequery", std::move(P), timeout, std::move(query), max_answer_size); @@ -412,6 +405,7 @@ void ValidatorGroup::receive_collate_query_response(td::uint32 round_id, td::Buf auto key = PublicKey{b->source_}; if (!key.is_ed25519()) { promise.set_error(td::Status::Error("collate query: block candidate source mismatch")); + return; } auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; if (e_key != Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}) { diff --git a/validator/validator.h b/validator/validator.h index 66007910b6..ab013cb085 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -111,8 +111,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual ~Callback() = default; virtual void initial_read_complete(BlockHandle top_masterchain_blocks) = 0; - virtual void subscribe_to_shard(ShardIdFull shard) = 0; - //virtual void del_shard(ShardIdFull shard) = 0; + virtual void update_shard_configuration(td::Ref state) = 0; virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; @@ -208,6 +207,11 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise promise) = 0; + virtual void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) = 0; + virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) = 0; + virtual void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) = 0; virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) = 0; From 1869a2506287dbc66e04ac52ab7b620e9960371e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 29 Jul 2022 12:08:17 +0300 Subject: [PATCH 008/524] Process adnl query errors --- adnl/adnl-message.cpp | 3 +++ adnl/adnl-message.h | 21 ++++++++++++++++++++- adnl/adnl-peer.cpp | 16 ++++++++++++++++ adnl/adnl-peer.hpp | 1 + adnl/adnl-query.cpp | 6 +++++- adnl/adnl-query.h | 1 + rldp/rldp-in.hpp | 4 ++++ rldp/rldp.cpp | 22 ++++++++++++++++++++++ rldp2/rldp-in.hpp | 4 ++++ rldp2/rldp.cpp | 22 ++++++++++++++++++++++ tl/generate/scheme/ton_api.tl | 2 ++ tl/generate/scheme/ton_api.tlo | Bin 70276 -> 70468 bytes 12 files changed, 100 insertions(+), 2 deletions(-) diff --git a/adnl/adnl-message.cpp b/adnl/adnl-message.cpp index 0d7129783f..1d3ba5f8a2 100644 --- a/adnl/adnl-message.cpp +++ b/adnl/adnl-message.cpp @@ -46,6 +46,9 @@ AdnlMessage::AdnlMessage(tl_object_ptr message) { [&](ton_api::adnl_message_part &msg) { message_ = adnlmessage::AdnlMessagePart{msg.hash_, static_cast(msg.total_size_), static_cast(msg.offset_), std::move(msg.data_)}; + }, + [&](ton_api::adnl_message_queryError &msg) { + message_ = adnlmessage::AdnlMessageQueryError{msg.query_id_}; })); } diff --git a/adnl/adnl-message.h b/adnl/adnl-message.h index 43849e9822..d557c31953 100644 --- a/adnl/adnl-message.h +++ b/adnl/adnl-message.h @@ -170,6 +170,24 @@ class AdnlMessageAnswer { td::BufferSlice data_; }; +class AdnlMessageQueryError { + public: + explicit AdnlMessageQueryError(AdnlQueryId query_id) : query_id_(query_id) { + } + const auto &query_id() const { + return query_id_; + } + td::uint32 size() const { + return 36; + } + tl_object_ptr tl() const { + return create_tl_object(query_id_); + } + + private: + AdnlQueryId query_id_; +}; + class AdnlMessagePart { public: AdnlMessagePart(td::Bits256 hash, td::uint32 total_size, td::uint32 offset, td::BufferSlice data) @@ -220,7 +238,8 @@ class AdnlMessage { private: td::Variant + adnlmessage::AdnlMessageQuery, adnlmessage::AdnlMessageAnswer, adnlmessage::AdnlMessagePart, + adnlmessage::AdnlMessageQueryError> message_{Empty{}}; public: diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index 35ba2a115a..1d73b8f6c2 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -524,10 +524,14 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQuery &mess flags = static_cast(0)](td::Result R) { if (R.is_error()) { LOG(WARNING) << "failed to answer query: " << R.move_as_error(); + td::actor::send_closure(SelfId, &AdnlPeerPairImpl::send_message, + OutboundAdnlMessage{adnlmessage::AdnlMessageQueryError{query_id}, flags}); } else { auto data = R.move_as_ok(); if (data.size() > Adnl::huge_packet_max_size()) { LOG(WARNING) << "dropping too big answer query: size=" << data.size(); + td::actor::send_closure(SelfId, &AdnlPeerPairImpl::send_message, + OutboundAdnlMessage{adnlmessage::AdnlMessageQueryError{query_id}, flags}); } else { td::actor::send_closure(SelfId, &AdnlPeerPairImpl::send_message, OutboundAdnlMessage{adnlmessage::AdnlMessageAnswer{query_id, std::move(data)}, flags}); @@ -609,6 +613,18 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessagePart &messa } } +void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQueryError &message) { + auto Q = out_queries_.find(message.query_id()); + + if (Q == out_queries_.end()) { + VLOG(ADNL_NOTICE) << this << ": dropping IN query error: unknown query id " << message.query_id(); + return; + } + + td::actor::send_closure_later(Q->second, &AdnlQuery::reject_query); + out_queries_.erase(Q); +} + void AdnlPeerPairImpl::delete_query(AdnlQueryId id) { auto Q = out_queries_.find(id); diff --git a/adnl/adnl-peer.hpp b/adnl/adnl-peer.hpp index 410c0f75f8..ec7166ed6d 100644 --- a/adnl/adnl-peer.hpp +++ b/adnl/adnl-peer.hpp @@ -104,6 +104,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair { void process_message(const adnlmessage::AdnlMessageQuery &message); void process_message(const adnlmessage::AdnlMessageAnswer &message); void process_message(const adnlmessage::AdnlMessagePart &message); + void process_message(const adnlmessage::AdnlMessageQueryError &message); void process_message(const AdnlMessage::Empty &message) { UNREACHABLE(); } diff --git a/adnl/adnl-query.cpp b/adnl/adnl-query.cpp index 5bc767d2bf..40e9be8038 100644 --- a/adnl/adnl-query.cpp +++ b/adnl/adnl-query.cpp @@ -25,13 +25,17 @@ namespace ton { namespace adnl { void AdnlQuery::alarm() { - promise_.set_error(td::Status::Error(ErrorCode::timeout, "adnl query timeout")); + promise_.set_error(td::Status::Error(ErrorCode::timeout, PSTRING() << "timeout for adnl query " << name_)); stop(); } void AdnlQuery::result(td::BufferSlice data) { promise_.set_value(std::move(data)); stop(); } +void AdnlQuery::reject_query() { + promise_.set_error(td::Status::Error(ErrorCode::timeout, PSTRING() << "rejected adnl query " << name_)); + stop(); +} AdnlQueryId AdnlQuery::random_query_id() { AdnlQueryId q_id; diff --git a/adnl/adnl-query.h b/adnl/adnl-query.h index 6e24a49fe7..2a78615a6d 100644 --- a/adnl/adnl-query.h +++ b/adnl/adnl-query.h @@ -48,6 +48,7 @@ class AdnlQuery : public td::actor::Actor { } void alarm() override; void result(td::BufferSlice data); + void reject_query(); void start_up() override { alarm_timestamp() = timeout_; } diff --git a/rldp/rldp-in.hpp b/rldp/rldp-in.hpp index b498199932..8850ccf9db 100644 --- a/rldp/rldp-in.hpp +++ b/rldp/rldp-in.hpp @@ -78,6 +78,8 @@ class RldpIn : public RldpImpl { td::uint64 max_answer_size) override; void answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, adnl::AdnlQueryId query_id, TransferId transfer_id, td::BufferSlice data); + void reject_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, + adnl::AdnlQueryId query_id, TransferId transfer_id); void alarm_query(adnl::AdnlQueryId query_id, TransferId transfer_id); @@ -93,6 +95,8 @@ class RldpIn : public RldpImpl { ton_api::rldp_query &message); void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, ton_api::rldp_answer &message); + void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, + ton_api::rldp_queryError &message); void receive_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, td::BufferSlice data); diff --git a/rldp/rldp.cpp b/rldp/rldp.cpp index 9b38dcb8c4..a1d4592607 100644 --- a/rldp/rldp.cpp +++ b/rldp/rldp.cpp @@ -87,6 +87,13 @@ void RldpIn::answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, transfer(src, dst, timeout, std::move(B), transfer_id); } +void RldpIn::reject_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, + adnl::AdnlQueryId query_id, TransferId transfer_id) { + auto B = serialize_tl_object(create_tl_object(query_id), true); + + transfer(src, dst, timeout, std::move(B), transfer_id); +} + void RldpIn::alarm_query(adnl::AdnlQueryId query_id, TransferId transfer_id) { queries_.erase(query_id); max_size_.erase(transfer_id); @@ -199,12 +206,16 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort auto data = R.move_as_ok(); if (data.size() > max_answer_size) { VLOG(RLDP_NOTICE) << "rldp query failed: answer too big"; + td::actor::send_closure(SelfId, &RldpIn::reject_query, local_id, source, timeout, query_id, + transfer_id ^ TransferId::ones()); } else { td::actor::send_closure(SelfId, &RldpIn::answer_query, local_id, source, timeout, query_id, transfer_id ^ TransferId::ones(), std::move(data)); } } else { VLOG(RLDP_NOTICE) << "rldp query failed: " << R.move_as_error(); + td::actor::send_closure(SelfId, &RldpIn::reject_query, local_id, source, timeout, query_id, + transfer_id ^ TransferId::ones()); } }); VLOG(RLDP_DEBUG) << "delivering rldp query"; @@ -223,6 +234,17 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort } } +void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, + ton_api::rldp_queryError &message) { + auto it = queries_.find(message.query_id_); + if (it != queries_.end()) { + td::actor::send_closure(it->second, &adnl::AdnlQuery::reject_query); + queries_.erase(it); + } else { + VLOG(RLDP_INFO) << "received reject to unknown query " << message.query_id_; + } +} + void RldpIn::transfer_completed(TransferId transfer_id) { senders_.erase(transfer_id); VLOG(RLDP_DEBUG) << "rldp: completed transfer " << transfer_id << "; " << senders_.size() << " out transfer pending "; diff --git a/rldp2/rldp-in.hpp b/rldp2/rldp-in.hpp index c2e46d2ab2..a815004c98 100644 --- a/rldp2/rldp-in.hpp +++ b/rldp2/rldp-in.hpp @@ -76,6 +76,8 @@ class RldpIn : public RldpImpl { td::uint64 max_answer_size) override; void answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, adnl::AdnlQueryId query_id, TransferId transfer_id, td::BufferSlice data); + void reject_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, + adnl::AdnlQueryId query_id, TransferId transfer_id); void receive_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, td::BufferSlice data); @@ -85,6 +87,8 @@ class RldpIn : public RldpImpl { ton_api::rldp_query &message); void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, ton_api::rldp_answer &message); + void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, + ton_api::rldp_queryError &message); void receive_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, td::Result data); diff --git a/rldp2/rldp.cpp b/rldp2/rldp.cpp index 765e38a5b6..34abd78e06 100644 --- a/rldp2/rldp.cpp +++ b/rldp2/rldp.cpp @@ -118,6 +118,13 @@ void RldpIn::answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, send_closure(create_connection(src, dst), &RldpConnectionActor::send, transfer_id, std::move(B), timeout); } +void RldpIn::reject_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, + adnl::AdnlQueryId query_id, TransferId transfer_id) { + auto B = serialize_tl_object(create_tl_object(query_id), true); + + send_closure(create_connection(src, dst), &RldpConnectionActor::send, transfer_id, std::move(B), timeout); +} + void RldpIn::receive_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, td::BufferSlice data) { send_closure(create_connection(local_id, source), &RldpConnectionActor::receive_raw, std::move(data)); } @@ -174,12 +181,16 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort auto data = R.move_as_ok(); if (data.size() > max_answer_size) { VLOG(RLDP_NOTICE) << "rldp query failed: answer too big"; + td::actor::send_closure(SelfId, &RldpIn::reject_query, local_id, source, timeout, query_id, + transfer_id ^ TransferId::ones()); } else { td::actor::send_closure(SelfId, &RldpIn::answer_query, local_id, source, timeout, query_id, transfer_id ^ TransferId::ones(), std::move(data)); } } else { VLOG(RLDP_NOTICE) << "rldp query failed: " << R.move_as_error(); + td::actor::send_closure(SelfId, &RldpIn::reject_query, local_id, source, timeout, query_id, + transfer_id ^ TransferId::ones()); } }); VLOG(RLDP_DEBUG) << "delivering rldp query"; @@ -198,6 +209,17 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort } } +void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, + ton_api::rldp_queryError &message) { + auto it = queries_.find(transfer_id); + if (it != queries_.end()) { + it->second.set_error(td::Status::Error("rejected")); + queries_.erase(it); + } else { + VLOG(RLDP_INFO) << "received reject to unknown query " << message.query_id_; + } +} + void RldpIn::on_sent(TransferId transfer_id, td::Result state) { //TODO: completed transfer } diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 3a81ab5858..9f1072ce72 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -136,6 +136,7 @@ adnl.message.reinit date:int = adnl.Message; adnl.message.query query_id:int256 query:bytes = adnl.Message; adnl.message.answer query_id:int256 answer:bytes = adnl.Message; +adnl.message.queryError query_id:int256 = adnl.Message; adnl.message.part hash:int256 total_size:int offset:int data:bytes = adnl.Message; @@ -161,6 +162,7 @@ rldp.complete transfer_id:int256 part:int = rldp.MessagePart; rldp.message id:int256 data:bytes = rldp.Message; rldp.query query_id:int256 max_answer_size:long timeout:int data:bytes = rldp.Message; rldp.answer query_id:int256 data:bytes = rldp.Message; +rldp.queryError query_id:int256 = rldp.Message; ---functions--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 479455feb679ece457175e95b6b44e8c7807cfa9..95a2144a69edb475430ee6965b8a84f5b497eddb 100644 GIT binary patch delta 481 zcmZo!%5r2G%LWx@(Y1dr9Og+($;;96O)V}?OiyKCVBnnSD84y>`HiIL`#%{~JViMv z1xT_in*(K&IT_tIZ&foFWdZ4*{K47>!f>$BfiMbe3LuOpHYyOtgM9{2HMS}eAYC_) z3HbmmV<>WF)8VjfI|w$Z##V(BCdoKi&_)}g=Y}oBlu33W5P=9gsId)pA*LV=Ad^7g zsvK90>V`ZOaj>6qA%4;;EKMz{bS)~%FG3jpbCvoeGY+WvP~#@qPXP;To?(B05h4}i zXafgLTmjo~`grj~`($*fStCpkQR24D}31U^CB}iZlSc*U)tU delta 322 zcmX@IjHP8M%LWx@(GH(|&v_D4@^bWiQ;Ule(^DB37&s<6if;~Jej_Pba%Wx$Pf<=v z0g^29=0MqGPDabxnT=2Ws+S8L?FTrYHR~c#Ks<>lVx)5 za`nj*>@&cMH~+CezzETHz|jWG*yQvig9YTd$qx0Jlh>|QhHxxa%Rm?rtLH!%KcFI$ S> Date: Mon, 1 Aug 2022 17:48:22 +0300 Subject: [PATCH 009/524] Get neighbors' msg queues from other nodes --- tl/generate/scheme/ton_api.tl | 4 + tl/generate/scheme/ton_api.tlo | Bin 70468 -> 70924 bytes validator/CMakeLists.txt | 1 + validator/full-node-shard.cpp | 53 +++++ validator/full-node-shard.h | 2 + validator/full-node-shard.hpp | 4 + validator/full-node.cpp | 17 ++ validator/full-node.hpp | 2 + validator/impl/CMakeLists.txt | 5 +- validator/impl/collator-impl.h | 5 +- validator/impl/collator.cpp | 70 +++--- validator/impl/out-msg-queue-proof.cpp | 245 +++++++++++++++++++++ validator/impl/out-msg-queue-proof.hpp | 109 +++++++++ validator/impl/proof.cpp | 35 +++ validator/interfaces/out-msg-queue-proof.h | 42 ++++ validator/interfaces/proof.h | 3 + validator/interfaces/validator-manager.h | 3 + validator/manager-disk.hpp | 8 + validator/manager-hardfork.hpp | 8 + validator/manager.cpp | 94 ++++++-- validator/manager.hpp | 7 + validator/validator.h | 7 + 22 files changed, 657 insertions(+), 67 deletions(-) create mode 100644 validator/impl/out-msg-queue-proof.cpp create mode 100644 validator/impl/out-msg-queue-proof.hpp create mode 100644 validator/interfaces/out-msg-queue-proof.h diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 9f1072ce72..f3ac4ee91b 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -410,6 +410,9 @@ tonNode.success = tonNode.Success; tonNode.archiveNotFound = tonNode.ArchiveInfo; tonNode.archiveInfo id:long = tonNode.ArchiveInfo; +tonNode.outMsgQueueProof queue_proof:bytes block_state_proof:bytes = tonNode.OutMsgQueueProof; +tonNode.outMsgQueueProofEmpty = tonNode.OutMsgQueueProof; + ---functions--- tonNode.getNextBlockDescription prev_block:tonNode.blockIdExt = tonNode.BlockDescription; @@ -441,6 +444,7 @@ tonNode.downloadBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.Dat tonNode.downloadKeyBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.DataList; tonNode.getArchiveInfo masterchain_seqno:int = tonNode.ArchiveInfo; tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data; +tonNode.getOutMsgQueueProof block_id:tonNode.blockIdExt dst_workchain:int dst_shard:long = tonNode.OutMsgQueueProof; tonNode.getCapabilities = tonNode.Capabilities; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 95a2144a69edb475430ee6965b8a84f5b497eddb..2960fa72c13d97c0b874d959604f413a38afe08c 100644 GIT binary patch delta 326 zcmX@IjHPE43-6=Z`c@23aBd^-X+_y2t+*)?CHZ-N`6;P-{-q_p#p!{isimm_Mfv$@ zlO2VmH{Vda!@}sfd8=CUESmbFCBiC@ zz9MHf9S+;JgCKJ@>l{pwXH41L`@mmElZAIxjx@}m^wbh`55nBbJo#dx`g8|AMpl-T a;*$8u509uo7>wD promise) { + BlockIdExt block_id = create_block_id(query.block_id_); + ShardIdFull dst_shard(query.dst_workchain_, query.dst_shard_); + if (!block_id.is_valid_ext()) { + promise.set_error(td::Status::Error("invalid block_id")); + return; + } + if (!dst_shard.is_valid_ext()) { + promise.set_error(td::Status::Error("invalid shard")); + return; + } + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_result(create_serialize_tl_object()); + } else { + promise.set_result(serialize_tl_object(R.move_as_ok(), true)); + } + }); + td::actor::create_actor("buildqueueproof", block_id, dst_shard, validator_manager_, + std::move(P)) + .release(); +} + void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { if (!active_) { @@ -791,6 +818,32 @@ void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, std::stri .release(); } +void FullNodeShardImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, + td::Promise> promise) { + // TODO: maybe more complex download (like other requests here) + // TODO: estimate max size + auto &b = choose_neighbour(); + auto P = td::PromiseCreator::lambda( + [=, promise = create_neighbour_promise(b, std::move(promise))](td::Result R) mutable { + if (R.is_error()) { + promise.set_result(R.move_as_error()); + return; + } + TRY_RESULT_PROMISE(promise, f, fetch_tl_object(R.move_as_ok(), true)); + ton_api::downcast_call(*f, td::overloaded( + [&](ton_api::tonNode_outMsgQueueProofEmpty &x) { + promise.set_error(td::Status::Error("node doesn't have this block")); + }, + [&](ton_api::tonNode_outMsgQueueProof &x) { + promise.set_result(OutMsgQueueProof::fetch(block_id, dst_shard, x)); + })); + }); + td::BufferSlice query = create_serialize_tl_object( + create_tl_block_id(block_id), dst_shard.workchain, dst_shard.shard); + td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, b.adnl_id, adnl_id_, overlay_id_, + "get_msg_queue", std::move(P), timeout, std::move(query), 1 << 20, rldp_); +} + void FullNodeShardImpl::set_handle(BlockHandle handle, td::Promise promise) { CHECK(!handle_); handle_ = std::move(handle); diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 2b2098a690..7da46af18e 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -62,6 +62,8 @@ class FullNodeShard : public td::actor::Actor { td::Promise> promise) = 0; virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; + virtual void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, + td::Promise> promise) = 0; virtual void set_handle(BlockHandle handle, td::Promise promise) = 0; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 9a022cbf08..ead8cafd58 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -133,6 +133,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getOutMsgQueueProof &query, + td::Promise promise); // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query, // td::Promise promise); void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise); @@ -164,6 +166,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise> promise) override; void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) override; + void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, + td::Promise> promise) override; void set_handle(BlockHandle handle, td::Promise promise) override; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 9c26cf3c4d..9194b9a4c5 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -356,6 +356,18 @@ void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, std::string tm std::move(promise)); } +void FullNodeImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, + td::Promise> promise) { + auto shard = get_shard(block_id.shard_full()); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping download msg queue query to unknown shard"; + promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); + return; + } + td::actor::send_closure(shard, &FullNodeShard::download_out_msg_queue_proof, block_id, dst_shard, timeout, + std::move(promise)); +} + td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard, bool exact) { if (!exact) { ShardIdFull s = shard; @@ -546,6 +558,11 @@ void FullNodeImpl::start_up() { td::actor::send_closure(id_, &FullNodeImpl::download_archive, masterchain_seqno, std::move(tmp_dir), timeout, std::move(promise)); } + void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, + td::Promise> promise) override { + td::actor::send_closure(id_, &FullNodeImpl::download_out_msg_queue_proof, block_id, dst_shard, timeout, + std::move(promise)); + } void new_key_block(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::new_key_block, std::move(handle)); diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 47cca0f57b..49dad3ba3d 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -73,6 +73,8 @@ class FullNodeImpl : public FullNode { void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise); void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise); + void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, + td::Promise> promise); void got_key_block_proof(td::Ref proof); void got_zero_block_state(td::Ref state); diff --git a/validator/impl/CMakeLists.txt b/validator/impl/CMakeLists.txt index 224599f864..4e3fd04f1e 100644 --- a/validator/impl/CMakeLists.txt +++ b/validator/impl/CMakeLists.txt @@ -15,6 +15,7 @@ set(TON_VALIDATOR_SOURCE ihr-message.cpp liteserver.cpp message-queue.cpp + out-msg-queue-proof.cpp proof.cpp shard.cpp signature-set.cpp @@ -32,13 +33,13 @@ set(TON_VALIDATOR_SOURCE ihr-message.hpp liteserver.hpp message-queue.hpp + out-msg-queue-proof.hpp proof.hpp shard.hpp signature-set.hpp top-shard-descr.hpp validate-query.hpp - validator-set.hpp -) + validator-set.hpp) add_library(ton_validator STATIC ${TON_VALIDATOR_SOURCE}) diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 8a17e95317..c19199c827 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -205,7 +205,7 @@ class Collator final : public td::actor::Actor { std::vector bad_ext_msgs_, delay_ext_msgs_; Ref shard_account_blocks_; // ShardAccountBlocks - std::map> blocks_with_state_proofs_; + std::map> block_state_proofs_; std::vector neighbor_proof_builders_; std::vector> collated_roots_; @@ -238,8 +238,7 @@ class Collator final : public td::actor::Actor { void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res); bool fix_one_processed_upto(block::MsgProcessedUpto& proc, const ton::ShardIdFull& owner); bool fix_processed_upto(block::MsgProcessedUptoCollection& upto); - void got_neighbor_block_data(td::Result> res); - void got_neighbor_block_state(int i, td::Result> res); + void got_neighbor_msg_queue(unsigned i, td::Result> R); bool adjust_shard_config(); bool store_shard_fees(ShardIdFull shard, const block::CurrencyCollection& fees, const block::CurrencyCollection& created); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 49c9459614..e116a93583 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -501,7 +501,13 @@ void Collator::after_get_block_data(int idx, td::Result> res) { prev_mc_block = prev_block_data[0]; mc_block_root = prev_mc_block->root_cell(); } - blocks_with_state_proofs_[prev_block_data[idx]->root_cell()->get_hash().bits()] = prev_block_data[idx]; + Ref root = prev_block_data[idx]->root_cell(); + auto proof = create_block_state_proof(root); + if (proof.is_error()) { + fatal_error(proof.move_as_error()); + return; + } + block_state_proofs_.emplace(root->get_hash().bits(), proof.move_as_ok()); } check_pending(); } @@ -613,54 +619,39 @@ bool Collator::request_neighbor_msg_queues() { } neighbors_.emplace_back(*shard_ptr); } - int i = 0; + unsigned i = 0; neighbor_proof_builders_.resize(neighbors_.size()); for (block::McShardDescr& descr : neighbors_) { LOG(DEBUG) << "neighbor #" << i << " : " << descr.blk_.to_str(); - if (descr.blk_.seqno() != 0) { - ++pending; - send_closure_later(manager, &ValidatorManager::wait_block_data_short, descr.blk_, priority(), timeout, - [self = get_self(), i](td::Result> res) { - LOG(DEBUG) << "got answer to wait_block_data for neighbor #" << i; - send_closure_later(std::move(self), &Collator::got_neighbor_block_data, std::move(res)); - }); - } ++pending; - send_closure_later(manager, &ValidatorManager::wait_block_state_short, descr.blk_, priority(), timeout, - [self = get_self(), i](td::Result> res) { - LOG(DEBUG) << "got answer to wait_block_state for neighbor #" << i; - send_closure_later(std::move(self), &Collator::got_neighbor_block_state, i, std::move(res)); + send_closure_later(manager, &ValidatorManager::wait_out_msg_queue_proof, descr.blk_, shard_, priority(), timeout, + [self = get_self(), i](td::Result> res) { + LOG(DEBUG) << "got msg queue for neighbor #" << i; + send_closure_later(std::move(self), &Collator::got_neighbor_msg_queue, i, std::move(res)); }); ++i; } return true; } -void Collator::got_neighbor_block_data(td::Result> res) { +void Collator::got_neighbor_msg_queue(unsigned i, td::Result> R) { --pending; - if (res.is_error()) { - fatal_error(res.move_as_error()); + if (R.is_error()) { + fatal_error(R.move_as_error()); return; } - auto block_data = res.move_as_ok(); - blocks_with_state_proofs_[block_data->root_cell()->get_hash().bits()] = block_data; - check_pending(); -} - -void Collator::got_neighbor_block_state(int i, td::Result> res) { - --pending; - if (res.is_error()) { - fatal_error(res.move_as_error()); - return; + auto res = R.move_as_ok(); + BlockIdExt block_id = neighbors_.at(i).blk_; + if (res->block_state_proof_.not_null()) { + block_state_proofs_.emplace(block_id.root_hash, res->block_state_proof_); } - Ref state = res.move_as_ok(); - neighbor_proof_builders_.at(i) = vm::MerkleProofBuilder{state->root_cell()}; - auto new_state = ShardStateQ::fetch(state->get_block_id(), {}, neighbor_proof_builders_.at(i).root()); - if (new_state.is_error()) { - fatal_error(new_state.move_as_error()); + neighbor_proof_builders_.at(i) = vm::MerkleProofBuilder{res->state_root_}; + auto state = ShardStateQ::fetch(block_id, {}, neighbor_proof_builders_.at(i).root()); + if (state.is_error()) { + fatal_error(state.move_as_error()); return; } - auto outq_descr_res = new_state.move_as_ok()->message_queue(); + auto outq_descr_res = state.move_as_ok()->message_queue(); if (outq_descr_res.is_error()) { fatal_error(outq_descr_res.move_as_error()); return; @@ -3992,17 +3983,8 @@ bool Collator::create_collated_data() { collated_roots_.push_back(std::move(cell)); } // 2. Proofs for hashes of states: previous states + neighbors - for (const auto& p : blocks_with_state_proofs_) { - vm::MerkleProofBuilder mpb{p.second->root_cell()}; - block::gen::Block::Record block; - if (!tlb::unpack_cell(mpb.root(), block) || block.state_update->load_cell().is_error()) { - return fatal_error("cannot generate Merkle proof for previous block"); - } - Ref proof = mpb.extract_proof(); - if (proof.is_null()) { - return fatal_error("cannot generate Merkle proof for previous block"); - } - collated_roots_.push_back(std::move(proof)); + for (const auto& p : block_state_proofs_) { + collated_roots_.push_back(p.second); } // 3. Previous state proof (only shadchains) std::map> proofs; diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp new file mode 100644 index 0000000000..0c3c993cdc --- /dev/null +++ b/validator/impl/out-msg-queue-proof.cpp @@ -0,0 +1,245 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "out-msg-queue-proof.hpp" +#include "interfaces/proof.h" +#include "shard.hpp" +#include "vm/cells/MerkleProof.h" +#include "common/delay.h" +#include "interfaces/validator-manager.h" + +namespace ton { + +namespace validator { + +td::Result> OutMsgQueueProof::fetch(BlockIdExt block_id, ShardIdFull dst_shard, + const ton_api::tonNode_outMsgQueueProof &f) { + Ref block_state_proof; + td::Bits256 state_root_hash; + if (block_id.seqno() == 0) { + if (!f.block_state_proof_.empty()) { + return td::Status::Error("expected empty block state proof"); + } + state_root_hash = block_id.root_hash; + } else { + TRY_RESULT_ASSIGN(block_state_proof, vm::std_boc_deserialize(f.block_state_proof_.as_slice())); + TRY_RESULT_ASSIGN(state_root_hash, unpack_block_state_proof(block_id, block_state_proof)); + } + + TRY_RESULT(queue_proof, vm::std_boc_deserialize(f.queue_proof_.as_slice())); + auto state_root = vm::MerkleProof::virtualize(queue_proof, 1); + if (state_root.is_null()) { + return td::Status::Error("invalid queue proof"); + } + if (state_root->get_hash().as_slice() != state_root_hash.as_slice()) { + return td::Status::Error("state root hash mismatch"); + } + + // TODO: validate + return Ref(true, std::move(state_root), std::move(block_state_proof)); +} + +td::Result> OutMsgQueueProof::serialize( + BlockIdExt block_id, ShardIdFull dst_shard, Ref state_root, Ref block_root) { + vm::MerkleProofBuilder mpb{std::move(state_root)}; + TRY_RESULT(state, ShardStateQ::fetch(block_id, {}, mpb.root())); + TRY_RESULT(outq_descr, state->message_queue()); + + // TODO: add only required part of msg queue + td::HashSet visited; + std::function)> dfs = [&](Ref cell) { + if (!visited.insert(cell->get_hash()).second) { + return; + } + vm::CellSlice cs(vm::NoVm(), cell); + for (unsigned i = 0; i < cs.size_refs(); i++) { + dfs(cs.prefetch_ref(i)); + } + }; + dfs(outq_descr->root_cell()); + + TRY_RESULT(queue_proof, vm::std_boc_serialize(mpb.extract_proof())); + + td::BufferSlice block_state_proof; + if (block_id.seqno() != 0) { + TRY_RESULT(proof, create_block_state_proof(std::move(block_root))); + TRY_RESULT_ASSIGN(block_state_proof, vm::std_boc_serialize(std::move(proof), 31)); + } + + return create_tl_object(std::move(queue_proof), std::move(block_state_proof)); +} + +void WaitOutMsgQueueProof::alarm() { + abort_query(td::Status::Error(ErrorCode::timeout, "timeout")); +} + +void WaitOutMsgQueueProof::abort_query(td::Status reason) { + if (promise_) { + if (priority_ > 0 || (reason.code() != ErrorCode::timeout && reason.code() != ErrorCode::notready)) { + LOG(WARNING) << "aborting wait msg queue query for " << block_id_.to_str() << " priority=" << priority_ << ": " + << reason; + } else { + LOG(DEBUG) << "aborting wait msg queue query for " << block_id_.to_str() << " priority=" << priority_ << ": " + << reason; + } + promise_.set_error( + reason.move_as_error_prefix(PSTRING() << "failed to get msg queue for " << block_id_.to_str() << ": ")); + } + stop(); +} + +void WaitOutMsgQueueProof::finish_query(Ref result) { + promise_.set_result(std::move(result)); + stop(); +} + +void WaitOutMsgQueueProof::start_up() { + alarm_timestamp() = timeout_; + if (local_) { + run_local(); + } else { + run_net(); + } +} + +void WaitOutMsgQueueProof::run_local() { + ++pending; + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, block_id_, priority_, timeout_, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::abort_query, + R.move_as_error_prefix("failed to get shard state")); + } else { + td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::got_state_root, + R.move_as_ok()->root_cell()); + } + }); + if (block_id_.seqno() != 0) { + ++pending; + td::actor::send_closure(manager_, &ValidatorManager::wait_block_data_short, block_id_, priority_, timeout_, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::abort_query, + R.move_as_error_prefix("failed to get block data")); + } else { + td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::got_block_root, + R.move_as_ok()->root_cell()); + } + }); + } +} + +void WaitOutMsgQueueProof::got_state_root(Ref root) { + state_root_ = std::move(root); + if (--pending == 0) { + run_local_cont(); + } +} + +void WaitOutMsgQueueProof::got_block_root(Ref root) { + block_root_ = std::move(root); + if (--pending == 0) { + run_local_cont(); + } +} + +void WaitOutMsgQueueProof::run_local_cont() { + Ref block_state_proof; + if (block_id_.seqno() != 0) { + auto R = create_block_state_proof(std::move(block_root_)); + if (R.is_error()) { + abort_query(R.move_as_error_prefix("failed to create block state proof")); + return; + } + block_state_proof = R.move_as_ok(); + } + finish_query(td::Ref(true, std::move(state_root_), std::move(block_state_proof))); +} + +void WaitOutMsgQueueProof::run_net() { + auto P = + td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = block_id_](td::Result> R) { + if (R.is_error()) { + LOG(DEBUG) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); + delay_action([SelfId]() mutable { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::run_net); }, + td::Timestamp::in(0.1)); + } else { + td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::finish_query, R.move_as_ok()); + } + }); + + td::actor::send_closure(manager_, &ValidatorManager::send_get_out_msg_queue_proof_request, block_id_, dst_shard_, + priority_, std::move(P)); +} + +void BuildOutMsgQueueProof::abort_query(td::Status reason) { + if (promise_) { + LOG(WARNING) << "failed to build msg queue proof for " << block_id_.to_str() << ": " << reason; + promise_.set_error( + reason.move_as_error_prefix(PSTRING() << "failed to build msg queue proof for " << block_id_.to_str() << ": ")); + } + stop(); +} + +void BuildOutMsgQueueProof::start_up() { + ++pending; + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_shard_state_from_db_short, block_id_, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, + R.move_as_error_prefix("failed to get shard state")); + } else { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_state_root, + R.move_as_ok()->root_cell()); + } + }); + if (block_id_.seqno() != 0) { + ++pending; + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_block_data_from_db_short, block_id_, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, + R.move_as_error_prefix("failed to get block data")); + } else { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_block_root, + R.move_as_ok()->root_cell()); + } + }); + } +} + +void BuildOutMsgQueueProof::got_state_root(Ref root) { + state_root_ = std::move(root); + if (--pending == 0) { + build_proof(); + } +} + +void BuildOutMsgQueueProof::got_block_root(Ref root) { + block_root_ = std::move(root); + if (--pending == 0) { + build_proof(); + } +} + +void BuildOutMsgQueueProof::build_proof() { + promise_.set_result( + OutMsgQueueProof::serialize(block_id_, dst_shard_, std::move(state_root_), std::move(block_root_))); + stop(); +} + +} // namespace validator +} // namespace ton diff --git a/validator/impl/out-msg-queue-proof.hpp b/validator/impl/out-msg-queue-proof.hpp new file mode 100644 index 0000000000..6a18ca6cb6 --- /dev/null +++ b/validator/impl/out-msg-queue-proof.hpp @@ -0,0 +1,109 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "vm/cells.h" +#include "ton/ton-types.h" +#include "auto/tl/ton_api.h" +#include "interfaces/out-msg-queue-proof.h" +#include "td/actor/actor.h" + +namespace ton { + +namespace validator { +using td::Ref; + +class ValidatorManager; +class ValidatorManagerInterface; + +class WaitOutMsgQueueProof : public td::actor::Actor { + public: + WaitOutMsgQueueProof(BlockIdExt block_id, ShardIdFull dst_shard, bool local, td::uint32 priority, + td::actor::ActorId manager, td::Timestamp timeout, + td::Promise> promise) + : block_id_(std::move(block_id)) + , dst_shard_(dst_shard) + , local_(local) + , priority_(priority) + , manager_(manager) + , timeout_(timeout) + , promise_(std::move(promise)) { + } + + void update_timeout(td::Timestamp timeout, td::uint32 priority) { + timeout_ = timeout; + alarm_timestamp() = timeout_; + priority_ = priority; + } + + void abort_query(td::Status reason); + void finish_query(Ref result); + void alarm() override; + + void start_up() override; + + void run_local(); + void got_state_root(Ref root); + void got_block_root(Ref root); + void run_local_cont(); + + void run_net(); + + + private: + BlockIdExt block_id_; + ShardIdFull dst_shard_; + bool local_; + td::uint32 priority_; + + td::actor::ActorId manager_; + td::Timestamp timeout_; + td::Promise> promise_; + + Ref state_root_, block_root_; + unsigned pending = 0; +}; + +class BuildOutMsgQueueProof : public td::actor::Actor { + public: + BuildOutMsgQueueProof(BlockIdExt block_id, ShardIdFull dst_shard, + td::actor::ActorId manager, + td::Promise> promise) + : block_id_(std::move(block_id)) + , dst_shard_(dst_shard) + , manager_(manager) + , promise_(std::move(promise)) { + } + + void abort_query(td::Status reason); + void start_up() override; + void got_state_root(Ref root); + void got_block_root(Ref root); + void build_proof(); + + private: + BlockIdExt block_id_; + ShardIdFull dst_shard_; + + td::actor::ActorId manager_; + td::Promise> promise_; + + Ref state_root_, block_root_; + unsigned pending = 0; +}; + +} // namespace validator +} // namespace ton diff --git a/validator/impl/proof.cpp b/validator/impl/proof.cpp index 033a1ab126..2297b48c18 100644 --- a/validator/impl/proof.cpp +++ b/validator/impl/proof.cpp @@ -162,5 +162,40 @@ td::Result> ProofQ::get_signatures_root() const { return proof.signatures->prefetch_ref(); } +td::Result> create_block_state_proof(td::Ref root) { + if (root.is_null()) { + return td::Status::Error("root is null"); + } + vm::MerkleProofBuilder mpb{std::move(root)}; + block::gen::Block::Record block; + if (!tlb::unpack_cell(mpb.root(), block) || block.state_update->load_cell().is_error()) { + return td::Status::Error("invalid block"); + } + Ref proof = mpb.extract_proof(); + if (proof.is_null()) { + return td::Status::Error("failed to create proof"); + } + return proof; +} + +td::Result unpack_block_state_proof(BlockIdExt block_id, td::Ref proof) { + auto virt_root = vm::MerkleProof::virtualize(proof, 1); + if (virt_root.is_null()) { + return td::Status::Error("invalid Merkle proof"); + } + if (virt_root->get_hash().as_slice() != block_id.root_hash.as_slice()) { + return td::Status::Error("hash mismatch"); + } + block::gen::Block::Record block; + if (!tlb::unpack_cell(virt_root, block)) { + return td::Status::Error("invalid block"); + } + vm::CellSlice upd_cs{vm::NoVmSpec(), block.state_update}; + if (!(upd_cs.is_special() && upd_cs.prefetch_long(8) == 4 && upd_cs.size_ext() == 0x20228)) { + return td::Status::Error("invalid Merkle update"); + } + return upd_cs.prefetch_ref(1)->get_hash(0).bits(); +} + } // namespace validator } // namespace ton diff --git a/validator/interfaces/out-msg-queue-proof.h b/validator/interfaces/out-msg-queue-proof.h new file mode 100644 index 0000000000..1fece66a5c --- /dev/null +++ b/validator/interfaces/out-msg-queue-proof.h @@ -0,0 +1,42 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "vm/cells.h" +#include "ton/ton-types.h" +#include "auto/tl/ton_api.h" + +namespace ton { + +namespace validator { +using td::Ref; + +struct OutMsgQueueProof : public td::CntObject { + OutMsgQueueProof(Ref state_root, Ref block_state_proof) + : state_root_(std::move(state_root)), block_state_proof_(std::move(block_state_proof)) { + } + + Ref state_root_; + Ref block_state_proof_; + + static td::Result> fetch(BlockIdExt block_id, ShardIdFull dst_shard, + const ton_api::tonNode_outMsgQueueProof &f); + static td::Result> serialize( + BlockIdExt block_id, ShardIdFull dst_shard, Ref state_root, Ref block_root); +}; + +} // namespace validator +} // namespace ton diff --git a/validator/interfaces/proof.h b/validator/interfaces/proof.h index 99471a1ff3..6665ad0801 100644 --- a/validator/interfaces/proof.h +++ b/validator/interfaces/proof.h @@ -48,6 +48,9 @@ class Proof : virtual public ProofLink { virtual td::Result> export_as_proof_link() const = 0; }; +td::Result> create_block_state_proof(td::Ref root); +td::Result unpack_block_state_proof(BlockIdExt block_id, td::Ref proof); + } // namespace validator } // namespace ton diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 209f871f29..362c259f9e 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -29,6 +29,7 @@ #include "liteserver.h" #include "crypto/vm/db/DynamicBagOfCellsDb.h" #include "validator-session/validator-session-types.h" +#include "impl/out-msg-queue-proof.hpp" namespace ton { @@ -128,6 +129,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_ihr_message(td::Ref message) = 0; virtual void send_top_shard_block_description(td::Ref desc) = 0; virtual void send_block_broadcast(BlockBroadcast broadcast) = 0; + virtual void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, td::uint32 priority, + td::Promise> promise) = 0; virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 501ee078c4..d8f75eef60 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -152,6 +152,10 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; + void wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) override { + UNREACHABLE(); + } void set_block_data(BlockHandle handle, td::Ref data, td::Promise promise) override; void wait_block_data(BlockHandle handle, td::uint32 priority, td::Timestamp, @@ -250,6 +254,10 @@ class ValidatorManagerImpl : public ValidatorManager { void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast) override { } + void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, td::uint32 priority, + td::Promise> promise) override { + UNREACHABLE(); + } void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 6778277143..87ec8c2bac 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -182,6 +182,10 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; + void wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) override { + UNREACHABLE(); + } void set_block_data(BlockHandle handle, td::Ref data, td::Promise promise) override { UNREACHABLE(); @@ -314,6 +318,10 @@ class ValidatorManagerImpl : public ValidatorManager { } void send_block_broadcast(BlockBroadcast broadcast) override { } + void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, td::uint32 priority, + td::Promise> promise) override { + UNREACHABLE(); + } void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override { UNREACHABLE(); diff --git a/validator/manager.cpp b/validator/manager.cpp index 7c5e7d3f10..64ee327346 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -453,24 +453,18 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id().shard_full(), desc->catchain_seqno()}] = desc; VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); - if (opts_->need_monitor(desc->block_id().shard_full()) && last_masterchain_block_handle_ && - last_masterchain_seqno_ > 0 && desc->generated_at() < last_masterchain_block_handle_->unix_time() + 60) { - delay_action( - [SelfId = actor_id(this), desc]() { - auto P = td::PromiseCreator::lambda([](td::Result> R) { - if (R.is_error()) { - auto S = R.move_as_error(); - if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { - VLOG(VALIDATOR_NOTICE) << "failed to get shard state: " << S; - } else { - VLOG(VALIDATOR_DEBUG) << "failed to get shard state: " << S; - } - } - }); - td::actor::send_closure(SelfId, &ValidatorManager::wait_block_state_short, desc->block_id(), 0, - td::Timestamp::in(60.0), std::move(P)); - }, - td::Timestamp::in(1.0)); + if (opts_->need_monitor(desc->block_id().shard_full())) { + auto P = td::PromiseCreator::lambda([](td::Result> R) { + if (R.is_error()) { + auto S = R.move_as_error(); + if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { + VLOG(VALIDATOR_NOTICE) << "failed to get shard state: " << S; + } else { + VLOG(VALIDATOR_DEBUG) << "failed to get shard state: " << S; + } + } + }); + wait_block_state_short(desc->block_id(), 0, td::Timestamp::in(60.0), std::move(P)); } } } @@ -606,6 +600,30 @@ void ValidatorManagerImpl::wait_block_state_short(BlockIdExt block_id, td::uint3 get_block_handle(block_id, true, std::move(P)); } +void ValidatorManagerImpl::wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, + td::Timestamp timeout, + td::Promise> promise) { + auto key = std::make_pair(block_id, dst_shard); + auto it = wait_out_msg_queue_proof_.find(key); + if (it == wait_out_msg_queue_proof_.end()) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), block_id, dst_shard](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_msg_queue, block_id, dst_shard, + std::move(R)); + }); + auto id = td::actor::create_actor("waitmsgqueue", block_id, dst_shard, + opts_->need_monitor(block_id.shard_full()), priority, + actor_id(this), td::Timestamp::in(10.0), std::move(P)) + .release(); + wait_out_msg_queue_proof_[key].actor_ = id; + it = wait_out_msg_queue_proof_.find(key); + } + + it->second.waiting_.emplace_back(timeout, priority, std::move(promise)); + auto X = it->second.get_timeout(); + td::actor::send_closure(it->second.actor_, &WaitOutMsgQueueProof::update_timeout, X.first, X.second); +} + void ValidatorManagerImpl::wait_block_data(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) { auto it = wait_block_data_.find(handle->id()); @@ -1037,6 +1055,40 @@ void ValidatorManagerImpl::finished_wait_data(BlockHandle handle, td::Result> R) { + auto it = wait_out_msg_queue_proof_.find({block_id, dst_shard}); + if (it != wait_out_msg_queue_proof_.end()) { + if (R.is_error()) { + auto S = R.move_as_error(); + if (S.code() != ErrorCode::timeout) { + for (auto &X : it->second.waiting_) { + X.promise.set_error(S.clone()); + } + } else { + auto X = it->second.get_timeout(); + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), block_id, dst_shard](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_msg_queue, block_id, dst_shard, + std::move(R)); + }); + auto id = td::actor::create_actor("waitmsgqueue", block_id, dst_shard, + opts_->need_monitor(block_id.shard_full()), X.second, + actor_id(this), X.first, std::move(P)) + .release(); + it->second.actor_ = id; + return; + } + } else { + auto r = R.move_as_ok(); + for (auto &X : it->second.waiting_) { + X.promise.set_result(r); + } + } + wait_out_msg_queue_proof_.erase(it); + } +} + void ValidatorManagerImpl::set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) { auto P = td::PromiseCreator::lambda( @@ -1410,6 +1462,12 @@ void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast) { callback_->send_broadcast(std::move(broadcast)); } +void ValidatorManagerImpl::send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, + td::uint32 priority, + td::Promise> promise) { + callback_->download_out_msg_queue_proof(id, dst_shard, td::Timestamp::in(10.0), std::move(promise)); +} + void ValidatorManagerImpl::start_up() { db_ = create_db_actor(actor_id(this), db_root_); lite_server_cache_ = create_liteserver_cache_actor(actor_id(this), db_root_); diff --git a/validator/manager.hpp b/validator/manager.hpp index 1105b2b62a..327aa3c51a 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -183,6 +183,8 @@ class ValidatorManagerImpl : public ValidatorManager { }; std::map>> wait_state_; std::map>> wait_block_data_; + std::map, WaitList>> + wait_out_msg_queue_proof_; struct WaitBlockHandle { std::vector> waiting_; @@ -357,6 +359,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; + void wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) override; void set_block_data(BlockHandle handle, td::Ref data, td::Promise promise) override; void wait_block_data(BlockHandle handle, td::uint32 priority, td::Timestamp, @@ -444,6 +448,8 @@ class ValidatorManagerImpl : public ValidatorManager { void send_ihr_message(td::Ref message) override; void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast) override; + void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, td::uint32 priority, + td::Promise> promise) override; void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; @@ -478,6 +484,7 @@ class ValidatorManagerImpl : public ValidatorManager { void finished_wait_state(BlockHandle handle, td::Result> R); void finished_wait_data(BlockHandle handle, td::Result> R); + void finished_wait_msg_queue(BlockIdExt block_id, ShardIdFull dst_shard, td::Result> R); void start_up() override; void started(ValidatorManagerInitResult result); diff --git a/validator/validator.h b/validator/validator.h index ab013cb085..6804ae8043 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -34,6 +34,7 @@ #include "interfaces/proof.h" #include "interfaces/shard.h" #include "catchain/catchain-types.h" +#include "interfaces/out-msg-queue-proof.h" namespace ton { @@ -131,6 +132,8 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise> promise) = 0; virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; + virtual void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, + td::Promise> promise) = 0; virtual void new_key_block(BlockHandle handle) = 0; }; @@ -212,6 +215,10 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) = 0; + virtual void wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, + td::Timestamp timeout, + td::Promise> promise) = 0; + virtual void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) = 0; virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) = 0; From cffffbab79b29a8b91d6a908f099bc81c6072115 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 1 Aug 2022 18:19:50 +0300 Subject: [PATCH 010/524] Fix CE --- create-hardfork/create-hardfork.cpp | 3 +++ dht-server/dht-server.cpp | 2 +- test/test-ton-collator.cpp | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 27260ee02b..5455a984e7 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -269,6 +269,9 @@ class HardforkCreator : public td::actor::Actor { td::Promise promise) override { } + void download_out_msg_queue_proof(ton::BlockIdExt block_id, ton::ShardIdFull dst_shard, td::Timestamp timeout, + td::Promise> promise) override { + } void new_key_block(ton::validator::BlockHandle handle) override { } diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index 212a6faee8..24b7b7e30d 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -146,7 +146,7 @@ ton::tl_object_ptr Config::tl() const { dht_vec.push_back(ton::create_tl_object(x.tl())); } - std::vector> val_vec; + std::vector> val_vec; std::vector> full_node_slaves_vec; std::vector> full_node_masters_vec; diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index b3d3f9db42..a68422d597 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -370,6 +370,9 @@ class TestNode : public td::actor::Actor { td::Promise promise) override { } + void download_out_msg_queue_proof(ton::BlockIdExt block_id, ton::ShardIdFull dst_shard, td::Timestamp timeout, + td::Promise> promise) override { + } void new_key_block(ton::validator::BlockHandle handle) override { } From ac7a33497b74371cd2b843942a7ee4176c3c5909 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 2 Aug 2022 18:22:56 +0300 Subject: [PATCH 011/524] Validate msg queue proof --- validator/impl/out-msg-queue-proof.cpp | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 0c3c993cdc..4a8731d39c 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -26,7 +26,7 @@ namespace ton { namespace validator { td::Result> OutMsgQueueProof::fetch(BlockIdExt block_id, ShardIdFull dst_shard, - const ton_api::tonNode_outMsgQueueProof &f) { + const ton_api::tonNode_outMsgQueueProof& f) { Ref block_state_proof; td::Bits256 state_root_hash; if (block_id.seqno() == 0) { @@ -40,20 +40,30 @@ td::Result> OutMsgQueueProof::fetch(BlockIdExt block_i } TRY_RESULT(queue_proof, vm::std_boc_deserialize(f.queue_proof_.as_slice())); - auto state_root = vm::MerkleProof::virtualize(queue_proof, 1); - if (state_root.is_null()) { + auto virtual_root = vm::MerkleProof::virtualize(queue_proof, 1); + if (virtual_root.is_null()) { return td::Status::Error("invalid queue proof"); } - if (state_root->get_hash().as_slice() != state_root_hash.as_slice()) { + if (virtual_root->get_hash().as_slice() != state_root_hash.as_slice()) { return td::Status::Error("state root hash mismatch"); } - // TODO: validate - return Ref(true, std::move(state_root), std::move(block_state_proof)); + // Validate proof + auto state_root = vm::CellSlice(vm::NoVm(), queue_proof).prefetch_ref(0); + TRY_RESULT_PREFIX(state, ShardStateQ::fetch(block_id, {}, state_root), "invalid proof: "); + TRY_RESULT_PREFIX(queue, state->message_queue(), "invalid proof: "); + auto queue_root = queue->root_cell(); + if (queue_root->get_level() != 0) { + return td::Status::Error("invalid proof: msg queue has prunned branches"); + } + + return Ref(true, std::move(virtual_root), std::move(block_state_proof)); } -td::Result> OutMsgQueueProof::serialize( - BlockIdExt block_id, ShardIdFull dst_shard, Ref state_root, Ref block_root) { +td::Result> OutMsgQueueProof::serialize(BlockIdExt block_id, + ShardIdFull dst_shard, + Ref state_root, + Ref block_root) { vm::MerkleProofBuilder mpb{std::move(state_root)}; TRY_RESULT(state, ShardStateQ::fetch(block_id, {}, mpb.root())); TRY_RESULT(outq_descr, state->message_queue()); From 212c07f2f0408c40efd9db442d6f9bb8657a685d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 3 Aug 2022 15:15:42 +0300 Subject: [PATCH 012/524] Change update_shard_configuration --- create-hardfork/create-hardfork.cpp | 3 +- test/test-ton-collator.cpp | 3 +- validator-engine/validator-engine.cpp | 2 +- validator/full-node.cpp | 74 ++++++++---------------- validator/full-node.h | 15 ++--- validator/full-node.hpp | 9 ++- validator/interfaces/validator-manager.h | 2 +- validator/manager-disk.hpp | 2 +- validator/manager-hardfork.hpp | 2 +- validator/manager.cpp | 16 ++--- validator/manager.hpp | 4 +- validator/shard-client.cpp | 53 ++++++++++++++++- validator/shard-client.hpp | 2 +- validator/validator.h | 3 +- 14 files changed, 106 insertions(+), 84 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 5455a984e7..a24b7fc183 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -236,7 +236,8 @@ class HardforkCreator : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void update_shard_configuration(td::Ref state) override { + void update_shard_configuration(td::Ref state, + std::set shards_to_monitor) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index a68422d597..d9ec82e5bd 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -323,7 +323,8 @@ class TestNode : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void update_shard_configuration(td::Ref state) override { + void update_shard_configuration(td::Ref state, + std::set shards_to_monitor) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 1bb820a42a..b2617769eb 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1853,7 +1853,7 @@ void ValidatorEngine::start_full_node() { }); full_node_ = ton::validator::fullnode::FullNode::create( short_id, ton::adnl::AdnlNodeIdShort{config_.full_node}, validator_options_->zero_block_id().file_hash, - validator_options_, keyring_.get(), adnl_.get(), rldp_.get(), + keyring_.get(), adnl_.get(), rldp_.get(), default_dht_node_.is_zero() ? td::actor::ActorId{} : dht_nodes_[default_dht_node_].get(), overlay_manager_.get(), validator_manager_.get(), full_node_client_.get(), db_root_, std::move(P)); for (auto &v : config_.validators) { diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 9194b9a4c5..63891502d9 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -132,52 +132,31 @@ void FullNodeImpl::initial_read_complete(BlockHandle top_handle) { td::actor::send_closure(it->second.actor, &FullNodeShard::set_handle, top_handle, std::move(P)); } -void FullNodeImpl::update_shard_configuration(td::Ref state) { +void FullNodeImpl::update_shard_configuration(td::Ref state, + std::set shards_to_monitor) { std::map new_shards; std::set new_active; new_shards[ShardIdFull(masterchainId)] = state->get_block_id(); - new_active.insert(ShardIdFull(masterchainId)); std::set workchains; - auto cur_time = state->get_unix_time(); - auto set_active = [&](ShardIdFull shard) { while (new_active.insert(shard).second && shard.pfx_len() > 0) { shard = shard_parent(shard); } }; - for (auto &info : state->get_shards()) { - auto shard = info->shard(); - workchains.insert(shard.workchain); - new_shards[shard] = info->top_block_id(); - bool will_split = shard.pfx_len() < max_shard_pfx_len && ((info->fsm_state() == McShardHash::FsmState::fsm_split && - info->fsm_utime() < cur_time + 60) || info->before_split()); - bool will_merge = shard.pfx_len() > 0 && ((info->fsm_state() == McShardHash::FsmState::fsm_merge && - info->fsm_utime() < cur_time + 60) || info->before_merge()); - if (opts_->need_monitor(shard)) { - set_active(shard); - } - if (will_merge && opts_->need_monitor(shard_parent(shard))) { - set_active(shard); - set_active(shard_sibling(shard)); - } - for (int id = 0; id < 2; ++id) { - if (will_split && opts_->need_monitor(shard_child(shard, id))) { - set_active(shard_child(shard, id)); - } - } + workchains.insert(info->shard().workchain); + new_shards[info->shard()] = info->top_block_id(); } for (const auto &wpair : state->get_workchain_list()) { ton::WorkchainId wc = wpair.first; const block::WorkchainInfo *winfo = wpair.second.get(); - if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= cur_time) { - auto shard = ShardIdFull(wc); - new_shards[shard] = BlockIdExt(wc, shard.shard, 0, winfo->zerostate_root_hash, winfo->zerostate_file_hash); - if (opts_->need_monitor(shard)) { - set_active(shard); - } + if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= state->get_unix_time()) { + new_shards[ShardIdFull(wc)] = BlockIdExt(wc, shardIdAll, 0, winfo->zerostate_root_hash, winfo->zerostate_file_hash); } } + for (ShardIdFull shard : shards_to_monitor) { + set_active(shard); + } auto info_set_active = [&](ShardIdFull shard, ShardInfo& info, bool active) { if (info.active == active) { @@ -195,10 +174,6 @@ void FullNodeImpl::update_shard_configuration(td::Ref state) { for (auto shard : new_shards) { auto &info = shards_[shard.first]; info.exists = true; - if (!info.active && new_active.count(shard.first)) { - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::wait_block_state_short, shard.second, 0, - td::Timestamp::in(60.0), [](td::Result>){}); - } } for (auto& p : shards_) { @@ -511,8 +486,9 @@ void FullNodeImpl::start_up() { void initial_read_complete(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::initial_read_complete, handle); } - void update_shard_configuration(td::Ref state) override { - td::actor::send_closure(id_, &FullNodeImpl::update_shard_configuration, std::move(state)); + void update_shard_configuration(td::Ref state, std::set shards_to_monitor) override { + td::actor::send_closure(id_, &FullNodeImpl::update_shard_configuration, std::move(state), + std::move(shards_to_monitor)); } void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) override { td::actor::send_closure(id_, &FullNodeImpl::send_ihr_message, dst, std::move(data)); @@ -580,16 +556,15 @@ void FullNodeImpl::start_up() { } FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - td::Ref opts, td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId dht, td::actor::ActorId overlays, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId dht, + td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, std::string db_root, td::Promise started_promise) : local_id_(local_id) , adnl_id_(adnl_id) , zero_state_file_hash_(zero_state_file_hash) - , opts_(opts) , keyring_(keyring) , adnl_(adnl) , rldp_(rldp) @@ -602,17 +577,14 @@ FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id add_shard_actor(ShardIdFull{masterchainId}, true); } -td::actor::ActorOwn FullNode::create(ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, td::Ref opts, - td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId dht, - td::actor::ActorId overlays, - td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root, - td::Promise started_promise) { - return td::actor::create_actor("fullnode", local_id, adnl_id, zero_state_file_hash, opts, keyring, adnl, - rldp, dht, overlays, validator_manager, client, db_root, +td::actor::ActorOwn FullNode::create( + ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId dht, + td::actor::ActorId overlays, td::actor::ActorId validator_manager, + td::actor::ActorId client, std::string db_root, td::Promise started_promise) { + return td::actor::create_actor("fullnode", local_id, adnl_id, zero_state_file_hash, keyring, adnl, rldp, + dht, overlays, validator_manager, client, db_root, std::move(started_promise)); } diff --git a/validator/full-node.h b/validator/full-node.h index 7342a82295..9f268666c7 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -72,15 +72,12 @@ class FullNode : public td::actor::Actor { return 4ull << 30; } - static td::actor::ActorOwn create(ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, td::Ref opts, - td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId dht, - td::actor::ActorId overlays, - td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root, - td::Promise started_promise); + static td::actor::ActorOwn create( + ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId dht, + td::actor::ActorId overlays, td::actor::ActorId validator_manager, + td::actor::ActorId client, std::string db_root, td::Promise started_promise); }; } // namespace fullnode diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 49dad3ba3d..2f82545878 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -52,7 +52,7 @@ class FullNodeImpl : public FullNode { void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; - void update_shard_configuration(td::Ref state); + void update_shard_configuration(td::Ref state, std::set shards_to_monitor); void sync_completed(); @@ -83,9 +83,9 @@ class FullNodeImpl : public FullNode { void start_up() override; FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - td::Ref opts, td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId dht, td::actor::ActorId overlays, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId dht, + td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, std::string db_root, td::Promise started_promise); @@ -96,7 +96,6 @@ class FullNodeImpl : public FullNode { PublicKeyHash local_id_; adnl::AdnlNodeIdShort adnl_id_; FileHash zero_state_file_hash_; - td::Ref opts_; td::actor::ActorId get_shard(AccountIdPrefixFull dst); td::actor::ActorId get_shard(ShardIdFull shard, bool exact = false); diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 362c259f9e..345ae679f6 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -134,7 +134,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; - virtual void update_shard_configuration(td::Ref state) = 0; + virtual void update_shard_configuration(td::Ref state, std::set shards_to_monitor) = 0; virtual void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) = 0; virtual void get_async_serializer_state(td::Promise promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index d8f75eef60..2e331c7208 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -261,7 +261,7 @@ class ValidatorManagerImpl : public ValidatorManager { void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; - void update_shard_configuration(td::Ref state) override { + void update_shard_configuration(td::Ref state, std::set shards_to_monitor) override { } void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override { diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 87ec8c2bac..79e742c4b7 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -329,7 +329,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_client_state(bool from_db, td::Promise promise) override { UNREACHABLE(); } - void update_shard_configuration(td::Ref state) override { + void update_shard_configuration(td::Ref state, std::set shards_to_monitor) override { UNREACHABLE(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 64ee327346..4427969813 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -372,7 +372,7 @@ void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { if (!is_validator()) { return; } - if( ext_messages_.size() > max_mempool_num() ) { + if ((double)ext_messages_.size() > max_mempool_num()) { return; } auto R = create_ext_message(std::move(data)); @@ -453,7 +453,7 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id().shard_full(), desc->catchain_seqno()}] = desc; VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); - if (opts_->need_monitor(desc->block_id().shard_full())) { + if (shards_to_monitor_.count(desc->block_id().shard_full())) { auto P = td::PromiseCreator::lambda([](td::Result> R) { if (R.is_error()) { auto S = R.move_as_error(); @@ -612,7 +612,7 @@ void ValidatorManagerImpl::wait_out_msg_queue_proof(BlockIdExt block_id, ShardId std::move(R)); }); auto id = td::actor::create_actor("waitmsgqueue", block_id, dst_shard, - opts_->need_monitor(block_id.shard_full()), priority, + shards_to_monitor_.count(block_id.shard_full()), priority, actor_id(this), td::Timestamp::in(10.0), std::move(P)) .release(); wait_out_msg_queue_proof_[key].actor_ = id; @@ -1073,8 +1073,8 @@ void ValidatorManagerImpl::finished_wait_msg_queue(BlockIdExt block_id, ShardIdF std::move(R)); }); auto id = td::actor::create_actor("waitmsgqueue", block_id, dst_shard, - opts_->need_monitor(block_id.shard_full()), X.second, - actor_id(this), X.first, std::move(P)) + shards_to_monitor_.count(block_id.shard_full()), + X.second, actor_id(this), X.first, std::move(P)) .release(); it->second.actor_ = id; return; @@ -2425,8 +2425,10 @@ void ValidatorManagerImpl::get_shard_client_state(bool from_db, td::Promise state) { - callback_->update_shard_configuration(state); +void ValidatorManagerImpl::update_shard_configuration(td::Ref state, + std::set shards_to_monitor) { + shards_to_monitor_ = shards_to_monitor; + callback_->update_shard_configuration(std::move(state), std::move(shards_to_monitor)); } void ValidatorManagerImpl::update_async_serializer_state(AsyncSerializerState state, td::Promise promise) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 327aa3c51a..6eb192a71b 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -453,7 +453,7 @@ class ValidatorManagerImpl : public ValidatorManager { void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; - void update_shard_configuration(td::Ref state) override; + void update_shard_configuration(td::Ref state, std::set shards_to_monitor) override; void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override; void get_async_serializer_state(td::Promise promise) override; @@ -619,6 +619,8 @@ class ValidatorManagerImpl : public ValidatorManager { std::map> collator_nodes_; bool collating_masterchain_ = false; + + std::set shards_to_monitor_ = {ShardIdFull(masterchainId)}; }; } // namespace validator diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index 50c6075d82..65adf34d4a 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -91,7 +91,7 @@ void ShardClient::start_up_init_mode() { auto vec = masterchain_state_->get_shards(); for (auto &shard : vec) { - if (opts_->need_monitor(shard->shard())) { + if (shards_to_monitor_.count(shard->shard())) { auto P = td::PromiseCreator::lambda([promise = ig.get_promise()](td::Result> R) mutable { R.ensure(); promise.set_value(td::Unit()); @@ -192,7 +192,7 @@ void ShardClient::apply_all_shards() { auto vec = masterchain_state_->get_shards(); for (auto &shard : vec) { - if (opts_->need_monitor(shard->shard())) { + if (shards_to_monitor_.count(shard->shard())) { auto Q = td::PromiseCreator::lambda([SelfId = actor_id(this), promise = ig.get_promise(), shard = shard->shard()](td::Result> R) mutable { if (R.is_error()) { @@ -247,7 +247,54 @@ void ShardClient::get_processed_masterchain_block_id(td::Promise pro } void ShardClient::build_shard_overlays() { - td::actor::send_closure(manager_, &ValidatorManager::update_shard_configuration, masterchain_state_); + std::set new_shards_to_monitor; + std::set workchains; + auto cur_time = masterchain_state_->get_unix_time(); + new_shards_to_monitor.insert(ShardIdFull(masterchainId)); + for (const auto &info : masterchain_state_->get_shards()) { + auto shard = info->shard(); + workchains.insert(shard.workchain); + bool will_split = shard.pfx_len() < max_shard_pfx_len && + ((info->fsm_state() == McShardHash::FsmState::fsm_split && info->fsm_utime() < cur_time + 60) || + info->before_split()); + bool will_merge = shard.pfx_len() > 0 && + ((info->fsm_state() == McShardHash::FsmState::fsm_merge && info->fsm_utime() < cur_time + 60) || + info->before_merge()); + if (opts_->need_monitor(shard) || (will_merge && opts_->need_monitor(shard_parent(shard)))) { + new_shards_to_monitor.insert(shard); + } + if (will_merge && opts_->need_monitor(shard_parent(shard))) { + new_shards_to_monitor.insert(shard_parent(shard)); + } + if (will_split) { + for (int id = 0; id < 2; ++id) { + if (opts_->need_monitor(shard_child(shard, id))) { + new_shards_to_monitor.insert(shard_child(shard, id)); + } + } + } + } + + std::vector new_workchains; + for (const auto &wpair : masterchain_state_->get_workchain_list()) { + ton::WorkchainId wc = wpair.first; + const block::WorkchainInfo *winfo = wpair.second.get(); + auto shard = ShardIdFull(wc); + if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= cur_time && opts_->need_monitor(shard)) { + new_shards_to_monitor.insert(shard); + if (shards_to_monitor_.count(shard) == 0) { + new_workchains.push_back(BlockIdExt(wc, shardIdAll, 0, winfo->zerostate_root_hash, winfo->zerostate_file_hash)); + } + } + } + + td::actor::send_closure(manager_, &ValidatorManager::update_shard_configuration, masterchain_state_, + new_shards_to_monitor); + shards_to_monitor_ = std::move(new_shards_to_monitor); + for (BlockIdExt block_id : new_workchains) { + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, block_id, shard_client_priority(), + td::Timestamp::in(60.0), [](td::Result>) {}); + } } void ShardClient::force_update_shard_client(BlockHandle handle, td::Promise promise) { diff --git a/validator/shard-client.hpp b/validator/shard-client.hpp index 455b67e7cb..701a387e88 100644 --- a/validator/shard-client.hpp +++ b/validator/shard-client.hpp @@ -42,7 +42,7 @@ class ShardClient : public td::actor::Actor { td::Promise promise_; - std::set created_overlays_; + std::set shards_to_monitor_ = {ShardIdFull(masterchainId)}; public: ShardClient(td::Ref opts, BlockHandle masterchain_block_handle, diff --git a/validator/validator.h b/validator/validator.h index 6804ae8043..c5212abb48 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -112,7 +112,8 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual ~Callback() = default; virtual void initial_read_complete(BlockHandle top_masterchain_blocks) = 0; - virtual void update_shard_configuration(td::Ref state) = 0; + virtual void update_shard_configuration(td::Ref state, + std::set shards_to_monitor) = 0; virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; From 5be10b1e6bfeb1f505eaf2d7b564221da5f411a4 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 4 Aug 2022 13:55:36 +0300 Subject: [PATCH 013/524] Add retries in validator group + bugfix --- validator/full-node.cpp | 4 +++- validator/validator-group.cpp | 39 ++++++++++++++++++++++++++++------- validator/validator-group.hpp | 5 +++-- 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 63891502d9..e4a2cf5762 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -407,7 +407,9 @@ void FullNodeImpl::got_key_block_proof(td::Ref proof) { CHECK(all_validators_.size() > 0); for (auto &shard : shards_) { - td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + if (!shard.second.actor.empty()) { + td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } } } diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 8eae613329..6c0e242d5f 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -263,16 +263,17 @@ void ValidatorGroup::create_session() { validatorsession::ValidatorSessionNode n; n.pub_key = ValidatorFullId{el.key}; n.weight = el.weight; - if (n.pub_key.compute_short_id() == local_id_) { - CHECK(!found); - found = true; - local_id_full_ = n.pub_key; - } if (el.addr.is_zero()) { n.adnl_id = adnl::AdnlNodeIdShort{n.pub_key.compute_short_id()}; } else { n.adnl_id = adnl::AdnlNodeIdShort{el.addr}; } + if (n.pub_key.compute_short_id() == local_id_) { + CHECK(!found); + found = true; + local_id_full_ = n.pub_key; + local_adnl_id_ = n.adnl_id; + } vec.emplace_back(std::move(n)); } CHECK(found); @@ -292,6 +293,8 @@ void ValidatorGroup::create_session() { if (started_) { td::actor::send_closure(session_, &validatorsession::ValidatorSession::start); } + + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_adnl_id_); } void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterchain_block_id) { @@ -346,7 +349,11 @@ void ValidatorGroup::get_session_info( } void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeout, - td::Promise promise) { + td::Promise promise, unsigned max_retries) { + if (round_id < last_known_round_id_) { + promise.set_error(td::Status::Error("too old")); + return; + } adnl::AdnlNodeIdShort collator = adnl::AdnlNodeIdShort::zero(); // TODO: some other way for storing and choosing collators for real network int cnt = 0; @@ -356,12 +363,26 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo ++cnt; } } - if (collator.is_zero()) { promise.set_error(td::Status::Error(PSTRING() << "no collator for shard " << shard_.to_str())); return; } + if (max_retries > 0) { + promise = td::PromiseCreator::lambda( + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_ok()) { + promise.set_result(R.move_as_ok()); + } else if (timeout && timeout.is_in_past()) { + promise.set_result(R.move_as_error()); + } else { + LOG(WARNING) << "collate query error, retrying: " << R.move_as_error(); + td::actor::send_closure(SelfId, &ValidatorGroup::send_collate_query, round_id, timeout, std::move(promise), + max_retries - 1); + } + }); + } + std::vector> prev_blocks; for (const BlockIdExt &p : prev_block_ids_) { prev_blocks.push_back(create_tl_block_id(p)); @@ -381,7 +402,9 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo }); LOG(INFO) << "collate query for " << create_next_block_id_simple().to_str() << ": send query to " << collator; size_t max_answer_size = config_.max_block_size + config_.max_collated_data_size + 256; - td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, adnl::AdnlNodeIdShort(local_id_), collator, "collatequery", + td::Timestamp query_timeout = td::Timestamp::in(10.0); + query_timeout.relax(timeout); + td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_adnl_id_, collator, "collatequery", std::move(P), timeout, std::move(query), max_answer_size); } diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index d0a96d659d..76e6ea9922 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -55,7 +55,6 @@ class ValidatorGroup : public td::actor::Actor { init_ = false; create_session(); } - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, adnl::AdnlNodeIdShort(local_id_)); } void get_session_info(td::Promise> promise); @@ -86,7 +85,8 @@ class ValidatorGroup : public td::actor::Actor { private: std::unique_ptr make_validator_session_callback(); - void send_collate_query(td::uint32 round_id, td::Timestamp timeout, td::Promise promise); + void send_collate_query(td::uint32 round_id, td::Timestamp timeout, td::Promise promise, + unsigned max_retries = 4); void receive_collate_query_response(td::uint32 round_id, td::BufferSlice data, td::Promise promise); struct PostponedAccept { @@ -120,6 +120,7 @@ class ValidatorGroup : public td::actor::Actor { std::string db_root_; td::actor::ActorId manager_; td::actor::ActorOwn session_; + adnl::AdnlNodeIdShort local_adnl_id_; bool init_ = false; bool started_ = false; From e5718c499f6ba6c47952efe311a989b5cebaf0d4 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 4 Aug 2022 23:21:55 +0300 Subject: [PATCH 014/524] Increase number of neighbours in BroadcastSimple::distribute (make it consistent with BroadcastFec) --- overlay/overlay-broadcast.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overlay/overlay-broadcast.cpp b/overlay/overlay-broadcast.cpp index 03991b76b4..ecd21e7afb 100644 --- a/overlay/overlay-broadcast.cpp +++ b/overlay/overlay-broadcast.cpp @@ -68,7 +68,7 @@ td::Status BroadcastSimple::run_checks() { td::Status BroadcastSimple::distribute() { auto B = serialize(); - auto nodes = overlay_->get_neighbours(3); + auto nodes = overlay_->get_neighbours(5); auto manager = overlay_->overlay_manager(); for (auto &n : nodes) { From a792565afa4aaf33c078cdc14e32762cd4073105 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 5 Aug 2022 11:11:21 +0300 Subject: [PATCH 015/524] Fix MSVC compilation error --- validator/collator-node.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 958b5f1362..84ef9eaa82 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -64,6 +64,7 @@ static td::BufferSlice serialize_error(td::Status error) { void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) { + auto SelfId = actor_id(this); auto status = [&]() -> td::Status { TRY_RESULT(f, fetch_tl_object(std::move(data), true)); ShardIdFull shard(f->workchain_, f->shard_); @@ -92,7 +93,7 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data LOG(INFO) << "Query from " << src << ": shard=" << shard.to_str() << ", min_mc_id=" << min_mc_id.to_str(); - auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), prev_blocks = std::move(prev_blocks), + auto P = td::PromiseCreator::lambda([=, prev_blocks = std::move(prev_blocks), promise = std::move(promise)](td::Result> R) mutable { if (R.is_error()) { LOG(WARNING) << "Query from " << src << ", error: " << R.error(); From 51e6885f2cc3da9bc5b82f1477fee91ae5f342bd Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 8 Aug 2022 12:21:15 +0300 Subject: [PATCH 016/524] Update validator config format; partial liteservers and their support in liteclient --- dht-server/dht-server.cpp | 15 +- dht-server/dht-server.hpp | 4 +- lite-client/lite-client.cpp | 701 +++++++++++------- lite-client/lite-client.h | 77 +- tl/generate/scheme/ton_api.tl | 20 +- tl/generate/scheme/ton_api.tlo | Bin 70924 -> 72088 bytes ton/ton-tl.hpp | 30 +- .../validator-engine-console-query.cpp | 22 +- .../validator-engine-console-query.h | 23 + .../validator-engine-console.cpp | 1 + validator-engine/validator-engine.cpp | 119 ++- validator-engine/validator-engine.hpp | 8 +- 12 files changed, 657 insertions(+), 363 deletions(-) diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index 24b7b7e30d..d1f439534c 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -38,6 +38,9 @@ #include "td/utils/TsFileLog.h" #include "td/utils/Random.h" +#include "ton/ton-tl.hpp" +#include "tl/tl_json.h" + #include "memprof/memprof.h" #if TD_DARWIN || TD_LINUX @@ -54,7 +57,7 @@ Config::Config() { out_port = 3278; } -Config::Config(ton::ton_api::engine_validator_config &config) { +Config::Config(const ton::ton_api::engine_validator_config_v2 &config) { out_port = static_cast(config.out_port_); if (!out_port) { out_port = 3278; @@ -121,7 +124,7 @@ Config::Config(ton::ton_api::engine_validator_config &config) { } } -ton::tl_object_ptr Config::tl() const { +ton::tl_object_ptr Config::tl() const { std::vector> addrs_vec; for (auto &x : addrs) { if (x.second.proxy) { @@ -146,7 +149,7 @@ ton::tl_object_ptr Config::tl() const { dht_vec.push_back(ton::create_tl_object(x.tl())); } - std::vector> val_vec; + std::vector> val_vec; std::vector> full_node_slaves_vec; std::vector> full_node_masters_vec; @@ -597,14 +600,14 @@ void DhtServer::load_config(td::Promise promise) { } auto conf_json = conf_json_R.move_as_ok(); - ton::ton_api::engine_validator_config conf; - auto S = ton::ton_api::from_json(conf, conf_json.get_object()); + ton::tl_object_ptr conf; + auto S = td::from_json(conf, std::move(conf_json)); if (S.is_error()) { promise.set_error(S.move_as_error_prefix("json does not fit TL scheme")); return; } - config_ = Config{conf}; + config_ = Config{*ton::unpack_engine_validator_config(std::move(conf))}; td::MultiPromise mp; auto ig = mp.init_guard(); diff --git a/dht-server/dht-server.hpp b/dht-server/dht-server.hpp index bf24d6216a..724e7bbfa4 100644 --- a/dht-server/dht-server.hpp +++ b/dht-server/dht-server.hpp @@ -91,10 +91,10 @@ struct Config { td::Result config_del_control_process(td::int32 port, ton::PublicKeyHash id); td::Result config_del_gc(ton::PublicKeyHash key); - ton::tl_object_ptr tl() const; + ton::tl_object_ptr tl() const; Config(); - Config(ton::ton_api::engine_validator_config &config); + Config(const ton::ton_api::engine_validator_config_v2 &config); }; class DhtServer : public td::actor::Actor { diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index aad3ae6e36..970cdad94c 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -41,8 +41,6 @@ #include "td/utils/crypto.h" #include "td/utils/overloaded.h" #include "td/utils/port/signals.h" -#include "td/utils/port/stacktrace.h" -#include "td/utils/port/StdStreams.h" #include "td/utils/port/FileFd.h" #include "terminal/terminal.h" #include "ton/lite-tl.hpp" @@ -59,17 +57,14 @@ #include "vm/cp0.h" #include "vm/memo.h" #include "ton/ton-shard.h" -#include "openssl/rand.hpp" #include "crypto/vm/utils.h" #include "crypto/common/util.h" #include "common/checksum.h" #if TD_DARWIN || TD_LINUX #include -#include #endif #include -#include #include "git.h" using namespace std::literals::string_literals; @@ -77,22 +72,27 @@ using td::Ref; int verbosity; -std::unique_ptr TestNode::make_callback() { - class Callback : public ton::adnl::AdnlExtClient::Callback { - public: - void on_ready() override { - td::actor::send_closure(id_, &TestNode::conn_ready); - } - void on_stop_ready() override { - td::actor::send_closure(id_, &TestNode::conn_closed); - } - Callback(td::actor::ActorId id) : id_(std::move(id)) { +int TestNode::LiteServer::max_common_prefix(ton::ShardIdFull shard) const { + if (shard.is_masterchain()) { + return 0; + } + if (is_full) { + return shard.pfx_len(); + } + int res = -1; + for (const ton::ShardIdFull &our_shard : shards) { + if (shard.workchain == our_shard.workchain) { + int x = std::min({shard.pfx_len(), our_shard.pfx_len(), ton::count_matching_bits(shard.shard, our_shard.shard)}); + res = std::max(res, x); } + } + return res; +} - private: - td::actor::ActorId id_; - }; - return std::make_unique(actor_id(this)); +bool TestNode::LiteServer::supports_shard(ton::ShardIdFull shard) const { + return is_full || shard.is_masterchain() || + std::any_of(shards.begin(), shards.end(), + [&](const ton::ShardIdFull& our_shard) { return ton::shard_is_ancestor(shard, our_shard); }); } void TestNode::run() { @@ -110,31 +110,58 @@ void TestNode::run() { io_ = td::TerminalIO::create("> ", readline_enabled_, ex_mode_, std::make_unique(actor_id(this))); td::actor::send_closure(io_, &td::TerminalIO::set_log_interface); - if (remote_public_key_.empty()) { - auto G = td::read_file(global_config_).move_as_ok(); - auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); - ton::ton_api::liteclient_config_global gc; - ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - CHECK(gc.liteservers_.size() > 0); - auto idx = liteserver_idx_ >= 0 ? liteserver_idx_ - : td::Random::fast(0, static_cast(gc.liteservers_.size() - 1)); - CHECK(idx >= 0 && static_cast(idx) <= gc.liteservers_.size()); - auto& cli = gc.liteservers_[idx]; - remote_addr_.init_host_port(td::IPAddress::ipv4_to_str(cli->ip_), cli->port_).ensure(); - remote_public_key_ = ton::PublicKey{cli->id_}; - td::TerminalIO::out() << "using liteserver " << idx << " with addr " << remote_addr_ << "\n"; - if (gc.validator_ && gc.validator_->zero_state_) { - zstate_id_.workchain = gc.validator_->zero_state_->workchain_; - if (zstate_id_.workchain != ton::workchainInvalid) { - zstate_id_.root_hash = gc.validator_->zero_state_->root_hash_; - zstate_id_.file_hash = gc.validator_->zero_state_->file_hash_; - td::TerminalIO::out() << "zerostate set to " << zstate_id_.to_str() << "\n"; - } + if (!single_remote_public_key_.empty()) { // Use single provided liteserver + LiteServer s; + s.addr = single_remote_addr_; + s.public_key = single_remote_public_key_; + single_liteserver_idx_ = 0; + servers_.push_back(std::move(s)); + run_init_queries(); + return; + } + + auto G = td::read_file(global_config_).move_as_ok(); + auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); + ton::ton_api::liteclient_config_global gc; + ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); + CHECK(gc.liteservers_.size() > 0); + + if (gc.validator_ && gc.validator_->zero_state_) { + zstate_id_.workchain = gc.validator_->zero_state_->workchain_; + if (zstate_id_.workchain != ton::workchainInvalid) { + zstate_id_.root_hash = gc.validator_->zero_state_->root_hash_; + zstate_id_.file_hash = gc.validator_->zero_state_->file_hash_; + td::TerminalIO::out() << "zerostate set to " << zstate_id_.to_str() << "\n"; } } - client_ = - ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull{remote_public_key_}, remote_addr_, make_callback()); + for (auto& server : gc.liteservers_) { + LiteServer s; + ton::ton_api::downcast_call(*server, + td::overloaded( + [&](ton::ton_api::liteserver_desc& obj) { + s.addr.init_host_port(td::IPAddress::ipv4_to_str(obj.ip_), obj.port_).ensure(); + s.public_key = ton::PublicKey{obj.id_}; + }, + [&](ton::ton_api::liteserver_descV2& obj) { + s.addr.init_host_port(td::IPAddress::ipv4_to_str(obj.ip_), obj.port_).ensure(); + s.public_key = ton::PublicKey{obj.id_}; + s.is_full = false; + for (const auto& shard : obj.shards_) { + s.shards.emplace_back(shard->workchain_, shard->shard_); + CHECK(s.shards.back().is_valid_ext()); + } + })); + servers_.push_back(std::move(s)); + } + + if (single_liteserver_idx_ != -1) { // Use single liteserver from config + CHECK(single_liteserver_idx_ >= 0 && (size_t)single_liteserver_idx_ < gc.liteservers_.size()); + td::TerminalIO::out() << "using liteserver " << single_liteserver_idx_ << " with addr " + << servers_[single_liteserver_idx_].addr << "\n"; + } + + run_init_queries(); } void TestNode::got_result(td::Result R, td::Promise promise) { @@ -179,23 +206,156 @@ void TestNode::after_got_result(bool ok) { } } -bool TestNode::envelope_send_query(td::BufferSlice query, td::Promise promise) { - running_queries_++; - if (!ready_ || client_.empty()) { - got_result(td::Status::Error("failed to send query to server: not ready"), std::move(promise)); +bool TestNode::envelope_send_query_to_any(td::BufferSlice query, td::Promise promise) { + return envelope_send_query_to_shard(ton::ShardIdFull(ton::masterchainId), std::move(query), std::move(promise)); +} + +bool TestNode::envelope_send_query_to_account(ton::AccountIdPrefixFull prefix, td::BufferSlice query, + td::Promise promise) { + if (single_liteserver_idx_ >= 0) { + return envelope_send_query_to_server(single_liteserver_idx_, std::move(query), std::move(promise)); + } + // TODO: maybe use current shard configuration? + int max_prefix_len = -1; + for (const LiteServer &server : servers_) { + max_prefix_len = std::max(max_prefix_len, server.max_common_prefix(prefix.as_leaf_shard())); + } + max_prefix_len = std::min(max_prefix_len, ton::max_shard_pfx_len); + if (max_prefix_len == -1) { + running_queries_++; + got_result(td::Status::Error("failed to select a suitable server"), std::move(promise)); + return false; + } + ton::ShardIdFull shard = shard_prefix(prefix.as_leaf_shard(), max_prefix_len); + return envelope_send_query_to_shard(shard, std::move(query), std::move(promise)); +} + +bool TestNode::envelope_send_query_to_shard(ton::ShardIdFull shard, td::BufferSlice query, + td::Promise promise) { + if (single_liteserver_idx_ >= 0) { + return envelope_send_query_to_server(single_liteserver_idx_, std::move(query), std::move(promise)); + } + if (shard.is_masterchain() && mc_server_idx_ != -1) { + return envelope_send_query_to_server(mc_server_idx_, std::move(query), std::move(promise)); + } + auto it = shard_server_idx_cached_.find(shard); + if (it != shard_server_idx_cached_.end()) { + return envelope_send_query_to_server(it->second, std::move(query), std::move(promise)); + } + int server_idx = -1; + int random_idx = -1; + int cnt = 0; + bool selected_full = false; + for (int i = 0; i < (int)servers_.size(); ++i) { + const LiteServer &server = servers_[i]; + if (!server.supports_shard(shard)) { + continue; + } + if (server.is_full && !selected_full) { + selected_full = true; + server_idx = -1; + cnt = 0; + } + if (!server.is_full && selected_full) { + continue; + } + if (!server.client.empty()) { + server_idx = i; + } + if (td::Random::fast(0, cnt) == 0) { + random_idx = i; + } + ++cnt; + } + if (server_idx == -1) { + server_idx = random_idx; + } + if (server_idx == -1) { + running_queries_++; + got_result(td::Status::Error("failed to select a suitable server"), std::move(promise)); return false; } + shard_server_idx_cached_[shard] = server_idx; + if (shard.is_masterchain()) { + mc_server_idx_ = server_idx; + } + return envelope_send_query_to_server(server_idx, std::move(query), std::move(promise)); +} + +bool TestNode::envelope_send_query_to_server(td::int32 server_idx, td::BufferSlice query, + td::Promise promise) { + running_queries_++; + LiteServer &server = servers_.at(server_idx); + if (server.client.empty()) { + start_client(server_idx); + } + CHECK(!server.client.empty()); + auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { td::actor::send_closure(SelfId, &TestNode::got_result, std::move(R), std::move(promise)); }); td::BufferSlice b = ton::serialize_tl_object(ton::create_tl_object(std::move(query)), true); - td::actor::send_closure(client_, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b), - td::Timestamp::in(10.0), std::move(P)); + if (server.client_ready) { + td::actor::send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b), + td::Timestamp::in(10.0), std::move(P)); + } else { + server.wait_client_ready.push_back( + [client = server.client.get(), b = std::move(b), P = std::move(P)](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(client, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b), + td::Timestamp::in(10.0), std::move(P)); + } else { + P.set_error(R.move_as_error_prefix("failed to connect: ")); + } + }); + } return true; } +void TestNode::start_client(int server_idx) { + LiteServer &server = servers_[server_idx]; + CHECK(server.client.empty()); + class Callback : public ton::adnl::AdnlExtClient::Callback { + public: + void on_ready() override { + td::actor::send_closure(id_, &TestNode::conn_ready, server_idx_); + } + void on_stop_ready() override { + td::actor::send_closure(id_, &TestNode::conn_closed, server_idx_); + } + Callback(td::actor::ActorId id, int server_idx) : id_(std::move(id)), server_idx_(server_idx) { + } + + private: + td::actor::ActorId id_; + int server_idx_; + }; + server.client_ready = false; + server.wait_client_ready.clear(); + LOG(INFO) << "Connecting to " << server.addr << " (liteserver #" << server_idx << ")"; + server.client = ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull{server.public_key}, server.addr, + std::make_unique(actor_id(this), server_idx)); +} + +void TestNode::conn_ready(int server_idx) { + LiteServer &server = servers_[server_idx]; + LOG(INFO) << "Connection to " << server.addr << " (liteserver #" << server_idx << ") is ready"; + server.client_ready = true; + for (auto &p : server.wait_client_ready) { + p.set_result(td::Unit()); + } + server.wait_client_ready.clear(); +} + +void TestNode::conn_closed(int server_idx) { + LiteServer &server = servers_[server_idx]; + LOG(INFO) << "Connection to " << server.addr << " (liteserver #" << server_idx << ") closed"; + server.client_ready = false; + server.wait_client_ready.clear(); +} + td::Promise TestNode::trivial_promise() { return td::PromiseCreator::lambda([Self = actor_id(this)](td::Result res) { if (res.is_error()) { @@ -310,7 +470,7 @@ bool TestNode::dump_cached_cell(td::Slice hash_pfx, td::Slice type_name) { bool TestNode::get_server_time() { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); - return envelope_send_query(std::move(b), [&, Self = actor_id(this)](td::Result res) -> void { + return envelope_send_query_to_any(std::move(b), [&, Self = actor_id(this)](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot get server time"; return; @@ -319,9 +479,10 @@ bool TestNode::get_server_time() { if (F.is_error()) { LOG(ERROR) << "cannot parse answer to liteServer.getTime"; } else { - server_time_ = F.move_as_ok()->now_; - server_time_got_at_ = now(); - LOG(INFO) << "server time is " << server_time_ << " (delta " << server_time_ - server_time_got_at_ << ")"; + mc_server_time_ = F.move_as_ok()->now_; + mc_server_time_got_at_ = now(); + LOG(INFO) << "server time is " << mc_server_time_ << " (delta " << mc_server_time_ - mc_server_time_got_at_ + << ")"; } } }); @@ -329,13 +490,13 @@ bool TestNode::get_server_time() { bool TestNode::get_server_version(int mode) { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), mode](td::Result res) { + return envelope_send_query_to_any(std::move(b), [Self = actor_id(this), mode](td::Result res) { td::actor::send_closure_later(Self, &TestNode::got_server_version, std::move(res), mode); }); }; void TestNode::got_server_version(td::Result res, int mode) { - server_ok_ = false; + mc_server_ok_ = false; if (res.is_error()) { LOG(ERROR) << "cannot get server version and time (server too old?)"; } else { @@ -344,11 +505,11 @@ void TestNode::got_server_version(td::Result res, int mode) { LOG(ERROR) << "cannot parse answer to liteServer.getVersion"; } else { auto a = F.move_as_ok(); - set_server_version(a->version_, a->capabilities_); - set_server_time(a->now_); + set_mc_server_version(a->version_, a->capabilities_); + set_mc_server_time(a->now_); } } - if (!server_ok_) { + if (!mc_server_ok_) { LOG(ERROR) << "server version is too old (at least " << (min_ls_version >> 8) << "." << (min_ls_version & 0xff) << " with capabilities " << min_ls_capabilities << " required), some queries are unavailable"; } @@ -357,27 +518,27 @@ void TestNode::got_server_version(td::Result res, int mode) { } } -void TestNode::set_server_version(td::int32 version, td::int64 capabilities) { - if (server_version_ != version || server_capabilities_ != capabilities) { - server_version_ = version; - server_capabilities_ = capabilities; - LOG(WARNING) << "server version is " << (server_version_ >> 8) << "." << (server_version_ & 0xff) - << ", capabilities " << server_capabilities_; +void TestNode::set_mc_server_version(td::int32 version, td::int64 capabilities) { + if (mc_server_version_ != version || mc_server_capabilities_ != capabilities) { + mc_server_version_ = version; + mc_server_capabilities_ = capabilities; + LOG(WARNING) << "server version is " << (mc_server_version_ >> 8) << "." << (mc_server_version_ & 0xff) + << ", capabilities " << mc_server_capabilities_; } - server_ok_ = (server_version_ >= min_ls_version) && !(~server_capabilities_ & min_ls_capabilities); + mc_server_ok_ = (mc_server_version_ >= min_ls_version) && !(~mc_server_capabilities_ & min_ls_capabilities); } -void TestNode::set_server_time(int server_utime) { - server_time_ = server_utime; - server_time_got_at_ = now(); - LOG(INFO) << "server time is " << server_time_ << " (delta " << server_time_ - server_time_got_at_ << ")"; +void TestNode::set_mc_server_time(int server_utime) { + mc_server_time_ = server_utime; + mc_server_time_got_at_ = now(); + LOG(INFO) << "server time is " << mc_server_time_ << " (delta " << mc_server_time_ - mc_server_time_got_at_ << ")"; } bool TestNode::get_server_mc_block_id() { - int mode = (server_capabilities_ & 2) ? 0 : -1; + int mode = (mc_server_capabilities_ & 2) ? 0 : -1; if (mode < 0) { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); - return envelope_send_query(std::move(b), [Self = actor_id(this)](td::Result res) -> void { + return envelope_send_query_to_any(std::move(b), [Self = actor_id(this)](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot get masterchain info from server"; return; @@ -397,24 +558,25 @@ bool TestNode::get_server_mc_block_id() { } else { auto b = ton::serialize_tl_object(ton::create_tl_object(mode), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), mode](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot get extended masterchain info from server"; - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getMasterchainInfoExt"; - } else { - auto f = F.move_as_ok(); - auto blk_id = create_block_id(f->last_); - auto zstate_id = create_zero_state_id(f->init_); - LOG(INFO) << "last masterchain block is " << blk_id.to_str(); - td::actor::send_closure_later(Self, &TestNode::got_server_mc_block_id_ext, blk_id, zstate_id, mode, - f->version_, f->capabilities_, f->last_utime_, f->now_); - } - } - }); + return envelope_send_query_to_any( + std::move(b), [Self = actor_id(this), mode](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "cannot get extended masterchain info from server"; + return; + } else { + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getMasterchainInfoExt"; + } else { + auto f = F.move_as_ok(); + auto blk_id = create_block_id(f->last_); + auto zstate_id = create_zero_state_id(f->init_); + LOG(INFO) << "last masterchain block is " << blk_id.to_str(); + td::actor::send_closure_later(Self, &TestNode::got_server_mc_block_id_ext, blk_id, zstate_id, mode, + f->version_, f->capabilities_, f->last_utime_, f->now_); + } + } + }); } } @@ -448,8 +610,8 @@ void TestNode::got_server_mc_block_id(ton::BlockIdExt blkid, ton::ZeroStateIdExt void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateIdExt zstateid, int mode, int version, long long capabilities, int last_utime, int server_now) { - set_server_version(version, capabilities); - set_server_time(server_now); + set_mc_server_version(version, capabilities); + set_mc_server_time(server_now); if (last_utime > server_now) { LOG(WARNING) << "server claims to have a masterchain block " << blkid.to_str() << " created at " << last_utime << " (" << last_utime - server_now << " seconds in the future)"; @@ -457,10 +619,10 @@ void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateI LOG(WARNING) << "server appears to be out of sync: its newest masterchain block is " << blkid.to_str() << " created at " << last_utime << " (" << server_now - last_utime << " seconds ago according to the server's clock)"; - } else if (last_utime < server_time_got_at_ - 60) { + } else if (last_utime < mc_server_time_got_at_ - 60) { LOG(WARNING) << "either the server is out of sync, or the local clock is set incorrectly: the newest masterchain " "block known to server is " - << blkid.to_str() << " created at " << last_utime << " (" << server_now - server_time_got_at_ + << blkid.to_str() << " created at " << last_utime << " (" << server_now - mc_server_time_got_at_ << " seconds ago according to the local clock)"; } got_server_mc_block_id(blkid, zstateid, last_utime); @@ -469,52 +631,54 @@ void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateI bool TestNode::request_block(ton::BlockIdExt blkid) { auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot obtain block " << blkid.to_str() << " from server"; - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getBlock"; - } else { - auto f = F.move_as_ok(); - auto blk_id = ton::create_block_id(f->id_); - LOG(INFO) << "obtained block " << blk_id.to_str() << " from server"; - if (blk_id != blkid) { - LOG(ERROR) << "block id mismatch: expected data for block " << blkid.to_str() << ", obtained for " - << blk_id.to_str(); + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "cannot obtain block " << blkid.to_str() << " from server"; + return; + } else { + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getBlock"; + } else { + auto f = F.move_as_ok(); + auto blk_id = ton::create_block_id(f->id_); + LOG(INFO) << "obtained block " << blk_id.to_str() << " from server"; + if (blk_id != blkid) { + LOG(ERROR) << "block id mismatch: expected data for block " << blkid.to_str() << ", obtained for " + << blk_id.to_str(); + } + td::actor::send_closure_later(Self, &TestNode::got_mc_block, blk_id, std::move(f->data_)); + } } - td::actor::send_closure_later(Self, &TestNode::got_mc_block, blk_id, std::move(f->data_)); - } - } - }); + }); } bool TestNode::request_state(ton::BlockIdExt blkid) { auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot obtain state " << blkid.to_str() << " from server"; - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getState"; - } else { - auto f = F.move_as_ok(); - auto blk_id = ton::create_block_id(f->id_); - LOG(INFO) << "obtained state " << blk_id.to_str() << " from server"; - if (blk_id != blkid) { - LOG(ERROR) << "block id mismatch: expected state for block " << blkid.to_str() << ", obtained for " - << blk_id.to_str(); + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "cannot obtain state " << blkid.to_str() << " from server"; + return; + } else { + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getState"; + } else { + auto f = F.move_as_ok(); + auto blk_id = ton::create_block_id(f->id_); + LOG(INFO) << "obtained state " << blk_id.to_str() << " from server"; + if (blk_id != blkid) { + LOG(ERROR) << "block id mismatch: expected state for block " << blkid.to_str() << ", obtained for " + << blk_id.to_str(); + } + td::actor::send_closure_later(Self, &TestNode::got_mc_state, blk_id, f->root_hash_, f->file_hash_, + std::move(f->data_)); + } } - td::actor::send_closure_later(Self, &TestNode::got_mc_state, blk_id, f->root_hash_, f->file_hash_, - std::move(f->data_)); - } - } - }); + }); } void TestNode::got_mc_block(ton::BlockIdExt blkid, td::BufferSlice data) { @@ -1149,27 +1313,34 @@ td::Status TestNode::send_ext_msg_from_filename(std::string filename) { LOG(ERROR) << "failed to read file `" << filename << "`: " << err.to_string(); return err; } - if (ready_ && !client_.empty()) { - LOG(ERROR) << "sending query from file " << filename; - auto P = td::PromiseCreator::lambda([](td::Result R) { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.sendMessage"; - } else { - int status = F.move_as_ok()->status_; - LOG(INFO) << "external message status is " << status; - } - }); - auto b = - ton::serialize_tl_object(ton::create_tl_object(F.move_as_ok()), true); - return envelope_send_query(std::move(b), std::move(P)) ? td::Status::OK() - : td::Status::Error("cannot send query to server"); - } else { - return td::Status::Error("server connection not ready"); + LOG(ERROR) << "sending query from file " << filename; + + TRY_RESULT_PREFIX(root, vm::std_boc_deserialize(F.ok().as_slice()), "invalid boc: "); + block::gen::CommonMsgInfo::Record_ext_in_msg_info info; + if (!tlb::unpack_cell_inexact(root, info)) { + return td::Status::Error("failed to unpack external message header"); + } + auto dest_prefix = block::tlb::t_MsgAddressInt.get_prefix(info.dest); + if (!dest_prefix.is_valid()) { + return td::Status::Error("destination of the message is invalid"); } + + auto P = td::PromiseCreator::lambda([](td::Result R) { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.sendMessage"; + } else { + int status = F.move_as_ok()->status_; + LOG(INFO) << "external message status is " << status; + } + }); + auto b = + ton::serialize_tl_object(ton::create_tl_object(F.move_as_ok()), true); + return envelope_send_query_to_account(dest_prefix, std::move(b), std::move(P)) ? td::Status::OK() + : td::Status::Error("cannot send query to server"); } bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::BlockIdExt ref_blkid, @@ -1177,9 +1348,6 @@ bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress if (!ref_blkid.is_valid()) { return set_error("must obtain last block information before making other queries"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } if (addr_ext) { return get_special_smc_addr(addr_ext, [this, ref_blkid, filename, mode](td::Result res) { if (res.is_error()) { @@ -1193,10 +1361,12 @@ bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(ref_blkid), std::move(a)), true); + ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); LOG(INFO) << "requesting account state for " << workchain << ":" << addr.to_hex() << " with respect to " << ref_blkid.to_str() << " with savefile `" << filename << "` and mode " << mode; - return envelope_send_query( - std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, filename, mode](td::Result R) { + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, ref_blkid, filename, mode](td::Result R) { if (R.is_error()) { return; } @@ -1275,19 +1445,18 @@ bool TestNode::start_run_method(ton::WorkchainId workchain, ton::StdSmcAddress a if (!ref_blkid.is_valid()) { return set_error("must obtain last block information before making other queries"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto a = ton::create_tl_object(workchain, addr); + ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); if (!mode) { auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(ref_blkid), std::move(a)), true); LOG(INFO) << "requesting account state for " << workchain << ":" << addr.to_hex() << " with respect to " << ref_blkid.to_str() << " to run method " << method_name << " with " << params.size() << " parameters"; - return envelope_send_query( - std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, method_name, params = std::move(params), - promise = std::move(promise)](td::Result R) mutable { + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, ref_blkid, method_name, params = std::move(params), + promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); return; @@ -1327,26 +1496,27 @@ bool TestNode::start_run_method(ton::WorkchainId workchain, ton::StdSmcAddress a LOG(INFO) << "requesting remote get-method execution for " << workchain << ":" << addr.to_hex() << " with respect to " << ref_blkid.to_str() << " to run method " << method_name << " with " << params.size() << " parameters"; - return envelope_send_query(std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, method_name, mode, - params = std::move(params), - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.runSmcMethod"; - promise.set_error(td::Status::Error("cannot parse answer to liteServer.runSmcMethod")); - } else { - auto f = F.move_as_ok(); - td::actor::send_closure_later(Self, &TestNode::run_smc_method, mode, ref_blkid, ton::create_block_id(f->id_), - ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), - std::move(f->proof_), std::move(f->state_proof_), workchain, addr, method_name, - std::move(params), std::move(f->init_c7_), std::move(f->lib_extras_), - std::move(f->result_), f->exit_code_, std::move(promise)); - } - }); + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, ref_blkid, method_name, mode, params = std::move(params), + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.runSmcMethod"; + promise.set_error(td::Status::Error("cannot parse answer to liteServer.runSmcMethod")); + } else { + auto f = F.move_as_ok(); + td::actor::send_closure_later( + Self, &TestNode::run_smc_method, mode, ref_blkid, ton::create_block_id(f->id_), + ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), std::move(f->proof_), + std::move(f->state_proof_), workchain, addr, method_name, std::move(params), std::move(f->init_c7_), + std::move(f->lib_extras_), std::move(f->result_), f->exit_code_, std::move(promise)); + } + }); } } @@ -1577,7 +1747,7 @@ void TestNode::send_compute_complaint_price_query(ton::StdSmcAddress elector_add params.emplace_back(td::make_refint(refs)); params.emplace_back(td::make_refint(expires_in)); auto P = td::PromiseCreator::lambda( - [this, expires_in, bits, refs, chash, filename](td::Result> R) { + [expires_in, bits, refs, chash, filename](td::Result> R) { if (R.is_error()) { LOG(ERROR) << R.move_as_error(); return; @@ -1639,10 +1809,6 @@ bool TestNode::dns_resolve_start(ton::WorkchainId workchain, ton::StdSmcAddress return set_error("domain name too long"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } - if (workchain == ton::workchainInvalid) { if (dns_root_queried_) { workchain = ton::masterchainId; @@ -1843,17 +2009,16 @@ bool TestNode::get_one_transaction(ton::BlockIdExt blkid, ton::WorkchainId workc if (!ton::shard_contains(blkid.shard_full(), ton::extract_addr_prefix(workchain, addr))) { return set_error("the shard of this block cannot contain this account"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto a = ton::create_tl_object(workchain, addr); auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(blkid), std::move(a), lt), true); + ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); LOG(INFO) << "requesting transaction " << lt << " of " << workchain << ":" << addr.to_hex() << " from block " << blkid.to_str(); - return envelope_send_query( - std::move(b), [Self = actor_id(this), workchain, addr, lt, blkid, dump](td::Result R) -> void { + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, lt, blkid, dump](td::Result R) -> void { if (R.is_error()) { return; } @@ -1870,16 +2035,15 @@ bool TestNode::get_one_transaction(ton::BlockIdExt blkid, ton::WorkchainId workc bool TestNode::get_last_transactions(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::LogicalTime lt, ton::Bits256 hash, unsigned count, bool dump) { - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto a = ton::create_tl_object(workchain, addr); auto b = ton::serialize_tl_object( ton::create_tl_object(count, std::move(a), lt, hash), true); + ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); LOG(INFO) << "requesting " << count << " last transactions from " << lt << ":" << hash.to_hex() << " of " << workchain << ":" << addr.to_hex(); - return envelope_send_query( - std::move(b), [Self = actor_id(this), workchain, addr, lt, hash, count, dump](td::Result R) { + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, lt, hash, count, dump](td::Result R) { if (R.is_error()) { return; } @@ -2236,10 +2400,10 @@ void TestNode::got_one_transaction(ton::BlockIdExt req_blkid, ton::BlockIdExt bl << " but received data has " << root->get_hash().bits().to_hex(256); return; } - } catch (vm::VmError err) { + } catch (vm::VmError &err) { LOG(ERROR) << "error while traversing block transaction proof : " << err.get_msg(); return; - } catch (vm::VmVirtError err) { + } catch (vm::VmVirtError &err) { LOG(ERROR) << "virtualization error while traversing block transaction proof : " << err.get_msg(); return; } @@ -2418,32 +2582,30 @@ void TestNode::got_last_transactions(std::vector blkids, td::Bu bool TestNode::get_block_transactions(ton::BlockIdExt blkid, int mode, unsigned count, ton::Bits256 acc_addr, ton::LogicalTime lt) { - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto a = ton::create_tl_object(acc_addr, lt); auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(blkid), mode, count, std::move(a), false, false), true); LOG(INFO) << "requesting " << count << " transactions from block " << blkid.to_str() << " starting from account " << acc_addr.to_hex() << " lt " << lt; - return envelope_send_query(std::move(b), [Self = actor_id(this), mode](td::Result R) { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.listBlockTransactions"; - } else { - auto f = F.move_as_ok(); - std::vector transactions; - for (auto& id : f->ids_) { - transactions.emplace_back(id->account_, id->lt_, id->hash_); - } - td::actor::send_closure_later(Self, &TestNode::got_block_transactions, ton::create_block_id(f->id_), mode, - f->req_count_, f->incomplete_, std::move(transactions), std::move(f->proof_)); - } - }); + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), mode](td::Result R) { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.listBlockTransactions"; + } else { + auto f = F.move_as_ok(); + std::vector transactions; + for (auto& id : f->ids_) { + transactions.emplace_back(id->account_, id->lt_, id->hash_); + } + td::actor::send_closure_later(Self, &TestNode::got_block_transactions, ton::create_block_id(f->id_), mode, + f->req_count_, f->incomplete_, std::move(transactions), std::move(f->proof_)); + } + }); } void TestNode::got_block_transactions(ton::BlockIdExt blkid, int mode, unsigned req_count, bool incomplete, @@ -2469,25 +2631,23 @@ bool TestNode::get_all_shards(std::string filename, bool use_last, ton::BlockIdE if (!blkid.is_masterchain()) { return set_error("only masterchain blocks contain shard configuration"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); LOG(INFO) << "requesting recent shard configuration"; - return envelope_send_query(std::move(b), [Self = actor_id(this), filename](td::Result R) -> void { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getAllShardsInfo"; - } else { - auto f = F.move_as_ok(); - td::actor::send_closure_later(Self, &TestNode::got_all_shards, ton::create_block_id(f->id_), std::move(f->proof_), - std::move(f->data_), filename); - } - }); + return envelope_send_query_to_any( + std::move(b), [Self = actor_id(this), filename](td::Result R) -> void { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getAllShardsInfo"; + } else { + auto f = F.move_as_ok(); + td::actor::send_closure_later(Self, &TestNode::got_all_shards, ton::create_block_id(f->id_), + std::move(f->proof_), std::move(f->data_), filename); + } + }); } void TestNode::got_all_shards(ton::BlockIdExt blk, td::BufferSlice proof, td::BufferSlice data, std::string filename) { @@ -2551,9 +2711,6 @@ bool TestNode::parse_get_config_params(ton::BlockIdExt blkid, int mode, std::str params.push_back(x); } } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } if (!blkid.is_masterchain_ext()) { return set_error("only masterchain blocks contain configuration"); } @@ -2572,10 +2729,6 @@ bool TestNode::get_config_params(ton::BlockIdExt blkid, td::Promise promise, int mode, std::string filename, std::vector params) { - if (!(ready_ && !client_.empty())) { - promise.set_error(td::Status::Error("server connection not ready")); - return false; - } if (!blkid.is_masterchain_ext()) { promise.set_error(td::Status::Error("masterchain reference block expected")); return false; @@ -2593,11 +2746,12 @@ bool TestNode::get_config_params_ext(ton::BlockIdExt blkid, td::Promise R) mutable { - td::actor::send_closure_later(Self, &TestNode::got_config_params, blkid, mode, filename, std::move(params), - std::move(R), std::move(promise)); - }); + return envelope_send_query_to_any( + std::move(b), [Self = actor_id(this), mode, filename, blkid, params = std::move(params), + promise = std::move(promise)](td::Result R) mutable { + td::actor::send_closure_later(Self, &TestNode::got_config_params, blkid, mode, filename, std::move(params), + std::move(R), std::move(promise)); + }); } void TestNode::got_config_params(ton::BlockIdExt req_blkid, int mode, std::string filename, std::vector params, @@ -2785,8 +2939,8 @@ bool TestNode::get_block(ton::BlockIdExt blkid, bool dump) { LOG(INFO) << "got block download request for " << blkid.to_str(); auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query( - std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot obtain block " << blkid.to_str() << " from server : " << res.move_as_error().to_string(); @@ -2814,8 +2968,8 @@ bool TestNode::get_state(ton::BlockIdExt blkid, bool dump) { LOG(INFO) << "got state download request for " << blkid.to_str(); auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query( - std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot obtain state " << blkid.to_str() << " from server : " << res.move_as_error().to_string(); @@ -2952,7 +3106,7 @@ void TestNode::got_state(ton::BlockIdExt blkid, ton::RootHash root_hash, ton::Fi } bool TestNode::get_show_block_header(ton::BlockIdExt blkid, int mode) { - return get_block_header(blkid, mode, [this, blkid](td::Result R) { + return get_block_header(blkid, mode, [this](td::Result R) { if (R.is_error()) { LOG(ERROR) << "unable to fetch block header: " << R.move_as_error(); } else { @@ -2967,8 +3121,9 @@ bool TestNode::get_block_header(ton::BlockIdExt blkid, int mode, td::Promise(ton::create_tl_lite_block_id(blkid), mode), true); - return envelope_send_query( - std::move(b), [this, blkid, promise = std::move(promise)](td::Result R) mutable -> void { + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), + [this, blkid, promise = std::move(promise)](td::Result R) mutable -> void { TRY_RESULT_PROMISE_PREFIX(promise, res, std::move(R), PSLICE() << "cannot obtain block header for " << blkid.to_str() << " from server :"); got_block_header_raw(std::move(res), std::move(promise), blkid); @@ -2994,8 +3149,8 @@ bool TestNode::lookup_block(ton::ShardIdFull shard, int mode, td::uint64 arg, auto b = ton::serialize_tl_object(ton::create_tl_object( mode, ton::create_tl_lite_block_id_simple(id), arg, (td::uint32)arg), true); - return envelope_send_query( - std::move(b), [this, id, mode, arg, promise = std::move(promise)](td::Result R) mutable -> void { + return envelope_send_query_to_shard( + shard, std::move(b), [this, id, mode, arg, promise = std::move(promise)](td::Result R) mutable -> void { TRY_RESULT_PROMISE_PREFIX(promise, res, std::move(R), PSLICE() << "cannot look up block header for " << id.to_str() << " with mode " << mode << " and argument " << arg << " from server :"); @@ -3109,9 +3264,9 @@ void TestNode::got_block_header(ton::BlockIdExt blkid, td::BufferSlice data, int return; } show_block_header(blkid, std::move(virt_root), mode); - } catch (vm::VmError err) { + } catch (vm::VmError &err) { LOG(ERROR) << "error processing header for " << blkid.to_str() << " : " << err.get_msg(); - } catch (vm::VmVirtError err) { + } catch (vm::VmVirtError &err) { LOG(ERROR) << "error processing header for " << blkid.to_str() << " : " << err.get_msg(); } show_new_blkids(); @@ -3140,14 +3295,15 @@ bool TestNode::get_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mod ton::serialize_tl_object(ton::create_tl_object( mode & 0xfff, ton::create_tl_lite_block_id(from), ton::create_tl_lite_block_id(to)), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), from, to, mode](td::Result res) { - if (res.is_error()) { - LOG(ERROR) << "cannot obtain block proof for " << ((mode & 1) ? to.to_str() : "last masterchain block") - << " starting from " << from.to_str() << " from server : " << res.move_as_error().to_string(); - } else { - td::actor::send_closure_later(Self, &TestNode::got_block_proof, from, to, mode, res.move_as_ok()); - } - }); + return envelope_send_query_to_any( + std::move(b), [Self = actor_id(this), from, to, mode](td::Result res) { + if (res.is_error()) { + LOG(ERROR) << "cannot obtain block proof for " << ((mode & 1) ? to.to_str() : "last masterchain block") + << " starting from " << from.to_str() << " from server : " << res.move_as_error().to_string(); + } else { + td::actor::send_closure_later(Self, &TestNode::got_block_proof, from, to, mode, res.move_as_ok()); + } + }); } void TestNode::got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mode, td::BufferSlice pchain) { @@ -3201,9 +3357,6 @@ void TestNode::got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mod bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_count, ton::Bits256 start_after, ton::UnixTime min_utime) { - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } if (!blkid.is_masterchain_ext()) { return set_error("only masterchain blocks contain block creator statistics"); } @@ -3214,7 +3367,7 @@ bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_c auto& os = *osp; return get_creator_stats( blkid, mode, req_count, start_after, min_utime, - [min_utime, &os](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, + [&os](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, const block::DiscountedCounter& shard_cnt) -> bool { os << key.to_hex() << " mc_cnt:" << mc_cnt << " shard_cnt:" << shard_cnt << std::endl; return true; @@ -3244,10 +3397,6 @@ bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_c bool TestNode::get_creator_stats(ton::BlockIdExt blkid, unsigned req_count, ton::UnixTime min_utime, TestNode::creator_stats_func_t func, std::unique_ptr state, td::Promise> promise) { - if (!(ready_ && !client_.empty())) { - promise.set_error(td::Status::Error("server connection not ready")); - return false; - } if (!state) { promise.set_error(td::Status::Error("null CreatorStatsRes")); return false; @@ -3266,7 +3415,7 @@ bool TestNode::get_creator_stats(ton::BlockIdExt blkid, unsigned req_count, ton: LOG(INFO) << "requesting up to " << req_count << " block creator stats records with respect to masterchain block " << blkid.to_str() << " starting from validator public key " << state->last_key.to_hex() << " created after " << min_utime << " (mode=" << state->mode << ")"; - return envelope_send_query( + return envelope_send_query_to_any( std::move(b), [this, blkid, req_count, state = std::move(state), min_utime, func = std::move(func), promise = std::move(promise)](td::Result R) mutable { TRY_RESULT_PROMISE(promise, res, std::move(R)); @@ -3483,7 +3632,7 @@ bool TestNode::load_creator_stats(std::unique_ptr l ton::UnixTime min_utime = info.valid_since - 1000; return get_creator_stats( info.blk_id, 1000, min_utime, - [min_utime, &info](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, + [&info](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, const block::DiscountedCounter& shard_cnt) -> bool { info.store_record(key, mc_cnt, shard_cnt); return true; @@ -3878,7 +4027,7 @@ td::Result> TestNode::ValidatorLoadInfo::build_proof(int idx, td:: block::gen::ValidatorDescr::Record_validator_addr rec2; if (tlb::csr_unpack(entry, rec1)) { pk = std::move(rec1.public_key); - } else if (tlb::csr_unpack(std::move(entry), rec2)) { + } else if (tlb::csr_unpack(entry, rec2)) { pk = std::move(rec2.public_key); } else { return td::Status::Error("cannot unpack ValidatorDescr"); diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index e62e801ce7..adfa9a04a6 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -45,22 +45,41 @@ class TestNode : public td::actor::Actor { min_ls_version = 0x101, min_ls_capabilities = 1 }; // server version >= 1.1, capabilities at least +1 = build proof chains - td::actor::ActorOwn client_; td::actor::ActorOwn io_; + struct LiteServer { + td::IPAddress addr; + ton::PublicKey public_key; + bool is_full = true; + std::vector shards; + + td::actor::ActorOwn client; + bool client_ready = false; + std::vector> wait_client_ready; + + int max_common_prefix(ton::ShardIdFull shard) const; + bool supports_shard(ton::ShardIdFull shard) const; + }; + std::vector servers_; + + td::int32 single_liteserver_idx_ = -1; + td::IPAddress single_remote_addr_; + ton::PublicKey single_remote_public_key_; + + std::map shard_server_idx_cached_; + bool readline_enabled_ = true; - bool server_ok_ = false; - td::int32 liteserver_idx_ = -1; int print_limit_ = 1024; - bool ready_ = false; - bool inited_ = false; std::string db_root_; - int server_time_ = 0; - int server_time_got_at_ = 0; - int server_version_ = 0; - long long server_capabilities_ = 0; + // mc_server is the server for queries to masterchain + int mc_server_idx_ = -1; + int mc_server_time_ = 0; + int mc_server_time_got_at_ = 0; + int mc_server_version_ = 0; + long long mc_server_capabilities_ = 0; + bool mc_server_ok_ = false; ton::ZeroStateIdExt zstate_id_; ton::BlockIdExt mc_last_id_; @@ -75,9 +94,6 @@ class TestNode : public td::actor::Actor { const char *parse_ptr_, *parse_end_; td::Status error_; - td::IPAddress remote_addr_; - ton::PublicKey remote_public_key_; - std::vector known_blk_ids_; std::size_t shown_blk_ids_ = 0; @@ -88,8 +104,6 @@ class TestNode : public td::actor::Actor { std::map> cell_cache_; - std::unique_ptr make_callback(); - using creator_stats_func_t = std::function; @@ -182,8 +196,8 @@ class TestNode : public td::actor::Actor { void got_server_mc_block_id(ton::BlockIdExt blkid, ton::ZeroStateIdExt zstateid, int created_at); void got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateIdExt zstateid, int mode, int version, long long capabilities, int last_utime, int server_now); - void set_server_version(td::int32 version, td::int64 capabilities); - void set_server_time(int server_utime); + void set_mc_server_version(td::int32 version, td::int64 capabilities); + void set_mc_server_time(int server_utime); bool request_block(ton::BlockIdExt blkid); bool request_state(ton::BlockIdExt blkid); void got_mc_block(ton::BlockIdExt blkid, td::BufferSlice data); @@ -358,16 +372,6 @@ class TestNode : public td::actor::Actor { static const tlb::TypenameLookup& get_tlb_dict(); public: - void conn_ready() { - LOG(ERROR) << "conn ready"; - ready_ = true; - if (!inited_) { - run_init_queries(); - } - } - void conn_closed() { - ready_ = false; - } void set_global_config(std::string str) { global_config_ = str; } @@ -378,10 +382,10 @@ class TestNode : public td::actor::Actor { readline_enabled_ = value; } void set_liteserver_idx(td::int32 idx) { - liteserver_idx_ = idx; + single_liteserver_idx_ = idx; } void set_remote_addr(td::IPAddress addr) { - remote_addr_ = addr; + single_remote_addr_ = addr; } void set_public_key(td::BufferSlice file_name) { auto R = [&]() -> td::Result { @@ -392,7 +396,7 @@ class TestNode : public td::actor::Actor { if (R.is_error()) { LOG(FATAL) << "bad server public key: " << R.move_as_error(); } - remote_public_key_ = R.move_as_ok(); + single_remote_public_key_ = R.move_as_ok(); } void decode_public_key(td::BufferSlice b64_key) { auto R = [&]() -> td::Result { @@ -404,7 +408,7 @@ class TestNode : public td::actor::Actor { if (R.is_error()) { LOG(FATAL) << "bad b64 server public key: " << R.move_as_error(); } - remote_public_key_ = R.move_as_ok(); + single_remote_public_key_ = R.move_as_ok(); } void set_fail_timeout(td::Timestamp ts) { fail_timeout_ = ts; @@ -439,7 +443,18 @@ class TestNode : public td::actor::Actor { void got_result(td::Result R, td::Promise promise); void after_got_result(bool ok); - bool envelope_send_query(td::BufferSlice query, td::Promise promise); + bool envelope_send_query_to_any(td::BufferSlice query, td::Promise promise); + bool envelope_send_query_to_shard(ton::ShardIdFull shard, td::BufferSlice query, + td::Promise promise); + bool envelope_send_query_to_account(ton::AccountIdPrefixFull prefix, td::BufferSlice query, + td::Promise promise); + + bool envelope_send_query_to_server(td::int32 server_idx, td::BufferSlice query, td::Promise promise); + + void start_client(int server_idx); + void conn_ready(int server_idx); + void conn_closed(int server_idx); + void parse_line(td::BufferSlice data); TestNode() { diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index f3ac4ee91b..187c4952ce 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -364,6 +364,7 @@ tonNode.blockSignature who:int256 signature:bytes = tonNode.BlockSignature; tonNode.blockId workchain:int shard:long seqno:int = tonNode.BlockId; tonNode.blockIdExt workchain:int shard:long seqno:int root_hash:int256 file_hash:int256 = tonNode.BlockIdExt; tonNode.zeroStateIdExt workchain:int root_hash:int256 file_hash:int256 = tonNode.ZeroStateIdExt; +tonNode.shardId workchain:int shard:long = tonNode.ShardId; tonNode.blockDescriptionEmpty = tonNode.BlockDescription; tonNode.blockDescription id:tonNode.blockIdExt = tonNode.BlockDescription; @@ -568,7 +569,8 @@ validator.config.global zero_state:tonNode.blockIdExt init_block:tonNode.blockId config.global adnl:adnl.config.global dht:dht.config.global validator:validator.config.global = config.Global; liteserver.desc id:PublicKey ip:int port:int = liteserver.Desc; -liteclient.config.global liteservers:(vector liteserver.desc) validator:validator.config.global = liteclient.config.Global; +liteserver.descV2 id:PublicKey ip:int port:int shards:(vector tonNode.shardId) = liteserver.Desc; +liteclient.config.global liteservers:(vector liteserver.Desc) validator:validator.config.global = liteclient.config.Global; engine.adnl id:int256 category:int = engine.Adnl; engine.addr ip:int port:int categories:(vector int) priority_categories:(vector int) = engine.Addr; @@ -578,7 +580,7 @@ engine.dht id:int256 = engine.Dht; engine.validatorTempKey key:int256 expire_at:int = engine.ValidatorTempKey; engine.validatorAdnlAddress id:int256 expire_at:int = engine.ValidatorAdnlAddress; engine.validator id:int256 temp_keys:(vector engine.validatorTempKey) adnl_addrs:(vector engine.validatorAdnlAddress) election_date:int expire_at:int = engine.Validator; -engine.collator adnl_id:int256 workchain:int shard:long = engine.Validator; +engine.collator adnl_id:int256 shard:tonNode.shardId = engine.Collator; engine.liteServer id:int256 port:int = engine.LiteServer; engine.controlProcess id:int256 permissions:int = engine.ControlProcess; engine.controlInterface id:int256 port:int allowed:(vector engine.controlProcess) = engine.ControlInterface; @@ -592,7 +594,16 @@ engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector eng validators:(vector engine.Validator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) fullnodemasters:(vector engine.validator.fullNodeMaster) liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) - gc:engine.gc = engine.validator.Config; + gc:engine.gc = engine.validator.Config; + +engine.validator.config_v2 out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) + dht:(vector engine.dht) + validators:(vector engine.validator) collators:(vector engine.collator) + fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) + fullnodemasters:(vector engine.validator.fullNodeMaster) + liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) + shards_to_monitor:(vector tonNode.shardId) + gc:engine.gc = engine.validator.Config; ---functions--- ---types--- @@ -697,7 +708,8 @@ engine.validator.generateBlockCandidate block_id:tonNode.BlockId = db.Candidate; engine.validator.getRequiredBlockCandidates = engine.validator.RequiredBlockCandidates; engine.validator.importBlockCandidate block:db.candidate = engine.validator.Success; -engine.validator.addCollator adnl_id:int256 workchain:int shard:long = engine.validator.Success; +engine.validator.addCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; +engine.validator.addShard shard:tonNode.shardId = engine.validator.Success; ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 2960fa72c13d97c0b874d959604f413a38afe08c..2bb8d64c32f5762201a2aab2bffd1bed74b12bc8 100644 GIT binary patch delta 709 zcmeBK#WG_v3-6=Z`c@23aA6~_xu8gdlkGeH)V%b}yi`5s{G6P`lKi5H8zd&X2)AhCIhZ~`04qGNl{ zPyQikB4Ycceg=O|W=U#sYEfBgk)BIxaWY6%gj55ENIAzZ4gQk+Jiq*uRK4Jg#G(|> z6rjbMZzzedF#2qMtJWPRi|n{$sN+ClH;)PVFipPrTz#@khJXafBu1#Gik#VWIBeSv zf>dqJscm3_c;Z7n#5bEO8bz2{K)#qfq2&WuZgWek38N)Q2o^9`<+x&Ux(!}PVR$qd z?9uo#BL@Z^oD@~j~DFm7&b6=oC`ECzYD zIKCu5J~uxv6BvLDlh6CfvVgoXxu;DB?7Yo;+A@S8A*nG-2Ey=|Wx@&yQijQmvy34^ zJ7y`cfP^L+L`zIg@DZ4N!qPzptOn{CkY79l%whzQq9Fws4PizM3}B5HmTFA4@UUTJ z1G#Fl;!Sz1!3k2j*<*H(rYuUdfqmkc0uqBn$;apZV1H~rapZ;!W9sIG51H9mC0JI? zlbL+*rO4#H&xIM`@di<7@ZtdrOJu^O$CD4d6cqrc0|tifWluJP3}l?_m?jQU_u-9< F1^`Gk54r#V delta 367 zcmbQSnWbkH3-6=Z`c@23aBd^7x!~ji!55;BvK|@>q~@h(=B4U|CFW$NB$nhCP5!7L zy}3h(g^^{7aX`@I3BnSSA23U7J|LXH#`5DpFUMpRDHD;BC7wt4b23X(i&KlrQj7Fl zQj3!(ZjjjABGtgL*+5x@h0$}ft9p0XW{DI&bTVa6?3{_-)?@W z1CsM|azOTj)Z9EKE5RnU65|by)^O$@< zn*%JcSz>;V=4OSHXJi;tHotw!%*J~8>zNjr$%1diCUd_Lob2workchain_, B->root_hash_, B->file_hash_}; } +inline ShardIdFull create_shard_id(const tl_object_ptr &s) { + return ShardIdFull{s->workchain_, static_cast(s->shard_)}; +} + +inline tl_object_ptr create_tl_shard_id(const ShardIdFull &s) { + return create_tl_object(s.workchain, s.shard); +} + +inline tl_object_ptr unpack_engine_validator_config( + tl_object_ptr config) { + tl_object_ptr res; + ton_api::downcast_call(*config, td::overloaded( + [&](ton_api::engine_validator_config &c) { + res = create_tl_object( + c.out_port_, std::move(c.addrs_), std::move(c.adnl_), std::move(c.dht_), + std::move(c.validators_), + std::vector>(), c.fullnode_, + std::move(c.fullnodeslaves_), std::move(c.fullnodemasters_), + std::move(c.liteservers_), std::move(c.control_), + std::vector>(), std::move(c.gc_)); + }, + [&](ton_api::engine_validator_config_v2 &c) { + res = std::make_unique(std::move(c)); + })); + return res; +} + } // namespace ton diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 9c5b6572d1..d1bc8d5ca6 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1101,7 +1101,7 @@ td::Status AddCollatorQuery::run() { td::Status AddCollatorQuery::send() { auto b = ton::create_serialize_tl_object( - adnl_id_.tl(), wc_, shard_); + adnl_id_.tl(), ton::create_tl_shard_id(ton::ShardIdFull(wc_, shard_))); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1112,3 +1112,23 @@ td::Status AddCollatorQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "successfully added collator\n"; return td::Status::OK(); } + +td::Status AddShardQuery::run() { + TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status AddShardQuery::send() { + auto b = ton::create_serialize_tl_object( + ton::create_tl_shard_id(ton::ShardIdFull(wc_, shard_))); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status AddShardQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully added shard\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 34b2ede3e6..daa19ab1a2 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1187,3 +1187,26 @@ class AddCollatorQuery : public Query { td::int32 wc_; td::int64 shard_; }; + +class AddShardQuery : public Query { + public: + AddShardQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "addshard"; + } + static std::string get_help() { + return "addshard \tstart monitoring shard"; + } + std::string name() const override { + return get_name(); + } + + private: + td::int32 wc_; + td::int64 shard_; +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index d70e2cf2e5..83375b2099 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -145,6 +145,7 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index b2617769eb..fda610afe4 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -54,6 +54,7 @@ #include "td/utils/Random.h" #include "auto/tl/lite_api.h" +#include "tl/tl_json.h" #include "memprof/memprof.h" @@ -74,7 +75,7 @@ Config::Config() { full_node = ton::PublicKeyHash::zero(); } -Config::Config(ton::ton_api::engine_validator_config &config) { +Config::Config(const ton::ton_api::engine_validator_config_v2 &config) { full_node = ton::PublicKeyHash::zero(); out_port = static_cast(config.out_port_); if (!out_port) { @@ -87,7 +88,7 @@ Config::Config(ton::ton_api::engine_validator_config &config) { std::vector categories; std::vector priority_categories; ton::ton_api::downcast_call( - *addr.get(), + *addr, td::overloaded( [&](const ton::ton_api::engine_addr &obj) { in_ip.init_ipv4_port(td::IPAddress::ipv4_to_str(obj.ip_), static_cast(obj.port_)).ensure(); @@ -125,24 +126,20 @@ Config::Config(ton::ton_api::engine_validator_config &config) { for (auto &dht : config.dht_) { config_add_dht_node(ton::PublicKeyHash{dht->id_}).ensure(); } - for (auto &v : config.validators_) { - ton::ton_api::downcast_call( - *v, td::overloaded( - [&](ton::ton_api::engine_validator &val) { - auto key = ton::PublicKeyHash{val.id_}; - config_add_validator_permanent_key(key, val.election_date_, val.expire_at_).ensure(); - for (auto &temp : val.temp_keys_) { - config_add_validator_temp_key(key, ton::PublicKeyHash{temp->key_}, temp->expire_at_).ensure(); - } - for (auto &adnl : val.adnl_addrs_) { - config_add_validator_adnl_id(key, ton::PublicKeyHash{adnl->id_}, adnl->expire_at_).ensure(); - } - }, - [&](ton::ton_api::engine_collator &col) { - auto key = ton::PublicKeyHash{col.adnl_id_}; - ton::ShardIdFull shard(col.workchain_, col.shard_); - config_add_collator(key, shard).ensure(); - })); + for (auto &val : config.validators_) { + auto key = ton::PublicKeyHash{val->id_}; + config_add_validator_permanent_key(key, val->election_date_, val->expire_at_).ensure(); + for (auto &temp : val->temp_keys_) { + config_add_validator_temp_key(key, ton::PublicKeyHash{temp->key_}, temp->expire_at_).ensure(); + } + for (auto &adnl : val->adnl_addrs_) { + config_add_validator_adnl_id(key, ton::PublicKeyHash{adnl->id_}, adnl->expire_at_).ensure(); + } + } + for (auto &col : config.collators_) { + auto key = ton::PublicKeyHash{col->adnl_id_}; + ton::ShardIdFull shard = ton::create_shard_id(col->shard_); + config_add_collator(key, shard).ensure(); } config_add_full_node_adnl_id(ton::PublicKeyHash{config.fullnode_}).ensure(); @@ -169,6 +166,10 @@ Config::Config(ton::ton_api::engine_validator_config &config) { } } + for (auto &shard : config.shards_to_monitor_) { + config_add_shard(ton::create_shard_id(shard)).ensure(); + } + if (config.gc_) { for (auto &gc : config.gc_->ids_) { config_add_gc(ton::PublicKeyHash{gc}).ensure(); @@ -176,7 +177,7 @@ Config::Config(ton::ton_api::engine_validator_config &config) { } } -ton::tl_object_ptr Config::tl() const { +ton::tl_object_ptr Config::tl() const { std::vector> addrs_vec; for (auto &x : addrs) { if (x.second.proxy) { @@ -201,7 +202,7 @@ ton::tl_object_ptr Config::tl() const { dht_vec.push_back(ton::create_tl_object(x.tl())); } - std::vector> val_vec; + std::vector> val_vec; for (auto &val : validators) { std::vector> temp_vec; for (auto &t : val.second.temp_keys) { @@ -214,9 +215,10 @@ ton::tl_object_ptr Config::tl() const { val_vec.push_back(ton::create_tl_object( val.first.tl(), std::move(temp_vec), std::move(adnl_val_vec), val.second.election_date, val.second.expire_at)); } + std::vector> col_vec; for (auto &col : collators) { - val_vec.push_back(ton::create_tl_object( - col.adnl_id.tl(), col.shard.workchain, col.shard.shard)); + col_vec.push_back( + ton::create_tl_object(col.adnl_id.tl(), ton::create_tl_shard_id(col.shard))); } std::vector> full_node_slaves_vec; @@ -245,14 +247,19 @@ ton::tl_object_ptr Config::tl() const { std::move(control_proc_vec))); } + std::vector> shards_vec; + for (auto &shard : shards_to_monitor) { + shards_vec.push_back(ton::create_tl_shard_id(shard)); + } + auto gc_vec = ton::create_tl_object(std::vector{}); for (auto &id : gc) { gc_vec->ids_.push_back(id.tl()); } - return ton::create_tl_object( - out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), full_node.tl(), - std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), - std::move(control_vec), std::move(gc_vec)); + return ton::create_tl_object( + out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), + full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), + std::move(control_vec), std::move(shards_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, @@ -527,6 +534,17 @@ td::Result Config::config_add_control_process(ton::PublicKeyHash key, td:: } } +td::Result Config::config_add_shard(ton::ShardIdFull shard) { + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << shard.to_str()); + } + if (std::find(shards_to_monitor.begin(), shards_to_monitor.end(), shard) != shards_to_monitor.end()) { + return false; + } + shards_to_monitor.push_back(shard); + return true; +} + td::Result Config::config_add_gc(ton::PublicKeyHash key) { return gc.insert(key).second; } @@ -1393,6 +1411,11 @@ void ValidatorEngine::init_validator_options() { for (const auto& c : config_.collators) { shards.push_back(c.shard); } + for (const auto& s : config_.shards_to_monitor) { + shards.push_back(s); + } + std::sort(shards.begin(), shards.end()); + shards.erase(std::unique(shards.begin(), shards.end()), shards.end()); validator_options_.write().set_shard_check_function( [shards = std::move(shards)](ton::ShardIdFull shard, ton::CatchainSeqno cc_seqno, ton::validator::ValidatorManagerOptions::ShardCheckMode mode) -> bool { @@ -1643,14 +1666,14 @@ void ValidatorEngine::load_config(td::Promise promise) { } auto conf_json = conf_json_R.move_as_ok(); - ton::ton_api::engine_validator_config conf; - auto S = ton::ton_api::from_json(conf, conf_json.get_object()); + ton::tl_object_ptr conf; + auto S = td::from_json(conf, std::move(conf_json)); if (S.is_error()) { promise.set_error(S.move_as_error_prefix("json does not fit TL scheme")); return; } - config_ = Config{conf}; + config_ = Config{*ton::unpack_engine_validator_config(std::move(conf))}; td::MultiPromise mp; auto ig = mp.init_guard(); @@ -3471,15 +3494,10 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollat } auto id = ton::PublicKeyHash{query.adnl_id_}; - auto shard = ton::ShardIdFull(query.workchain_, query.shard_); - if (!shard.is_valid_ext()) { - promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "invalid shard"))); - return; - } - + auto shard = ton::create_shard_id(query.shard_); auto R = config_.config_add_collator(id, shard); if (R.is_error()) { - promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + promise.set_value(create_control_query_error(R.move_as_error())); return; } if (!R.move_as_ok()) { @@ -3499,6 +3517,27 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollat }); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addShard &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto shard = ton::create_shard_id(query.shard_); + auto R = config_.config_add_shard(shard); + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { @@ -3532,7 +3571,7 @@ void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNode } auto f = F.move_as_ok(); - ton::ton_api::downcast_call(*f.get(), [&](auto &obj) { + ton::ton_api::downcast_call(*f, [&](auto &obj) { run_control_query(obj, std::move(data), src.pubkey_hash(), it->second, std::move(promise)); }); } @@ -3752,7 +3791,7 @@ int main(int argc, char *argv[]) { }); return td::Status::OK(); }); - p.add_option('M', "masterchain-only", "don't track shardchains", [&]() { + p.add_option('M', "not-all-shards", "monitor only a necessary set of shards instead of all", [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_masterchain_only); }); }); td::uint32 threads = 7; diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index bf47e2d793..46de7e86a7 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -96,6 +96,7 @@ struct Config { std::map liteservers; std::map controls; std::set gc; + std::vector shards_to_monitor; void decref(ton::PublicKeyHash key); void incref(ton::PublicKeyHash key) { @@ -121,6 +122,7 @@ struct Config { td::Result config_add_control_interface(ton::PublicKeyHash key, td::int32 port); td::Result config_add_control_process(ton::PublicKeyHash key, td::int32 port, ton::PublicKeyHash id, td::uint32 permissions); + td::Result config_add_shard(ton::ShardIdFull shard); td::Result config_add_gc(ton::PublicKeyHash key); td::Result config_del_network_addr(td::IPAddress addr, std::vector cats, std::vector prio_cats); @@ -135,10 +137,10 @@ struct Config { td::Result config_del_control_process(td::int32 port, ton::PublicKeyHash id); td::Result config_del_gc(ton::PublicKeyHash key); - ton::tl_object_ptr tl() const; + ton::tl_object_ptr tl() const; Config(); - Config(ton::ton_api::engine_validator_config &config); + Config(const ton::ton_api::engine_validator_config_v2 &config); }; class ValidatorEngine : public td::actor::Actor { @@ -434,6 +436,8 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_addCollator &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_addShard &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { From 7241522de2c4ae243375c94cae8b44c0fe9cbb5e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 8 Aug 2022 12:41:28 +0300 Subject: [PATCH 017/524] Remove obsolete interface for importing blocks --- tl/generate/scheme/ton_api.tl | 4 - tl/generate/scheme/ton_api.tlo | Bin 72088 -> 71776 bytes .../validator-engine-console-query.cpp | 63 ---------- .../validator-engine-console-query.h | 70 ----------- .../validator-engine-console.cpp | 3 - validator-engine/validator-engine.cpp | 88 -------------- validator-engine/validator-engine.hpp | 6 - validator/interfaces/validator-manager.h | 2 - validator/manager-disk.hpp | 12 -- validator/manager-hardfork.hpp | 12 -- validator/manager.cpp | 113 ------------------ validator/manager.hpp | 8 -- validator/validator.h | 4 - 13 files changed, 385 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 187c4952ce..837e575416 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -704,10 +704,6 @@ engine.validator.importShardOverlayCertificate workchain:int shard:long signed_k engine.validator.getValidatorSessionsInfo = engine.validator.ValidatorSessionsInfo; -engine.validator.generateBlockCandidate block_id:tonNode.BlockId = db.Candidate; -engine.validator.getRequiredBlockCandidates = engine.validator.RequiredBlockCandidates; -engine.validator.importBlockCandidate block:db.candidate = engine.validator.Success; - engine.validator.addCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; engine.validator.addShard shard:tonNode.shardId = engine.validator.Success; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 2bb8d64c32f5762201a2aab2bffd1bed74b12bc8..36c21e21f7ebcbdc819c4ab0da29bca718a36353 100644 GIT binary patch delta 38 wcmV+>0NMYTvjpI<1hDi7lc)(4vj_>~+yP>^V_GvVyp$P*mkNXY!rNGLvQg-eG<4d&+Os$$$Tfi=e7xVCY`()); - TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); - TRY_RESULT_ASSIGN(seqno_, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(file_, tokenizer_.get_token()); - return td::Status::OK(); -} - -td::Status GenerateBlockCandidateQuery::send() { - auto b = ton::create_serialize_tl_object( - ton::create_tl_block_id_simple(ton::BlockId(wc_, shard_, seqno_))); - td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); - return td::Status::OK(); -} - -td::Status GenerateBlockCandidateQuery::receive(td::BufferSlice data) { - TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), - "received incorrect answer: "); - TRY_STATUS_PREFIX(td::write_file(file_, data.as_slice()), "failed to write block to file"); - td::TerminalIO::out() << "successfully written candidate to file\n"; - return td::Status::OK(); -} - -td::Status GetRequiredBlockCandidatesQuery::run() { - TRY_STATUS(tokenizer_.check_endl()); - return td::Status::OK(); -} - -td::Status GetRequiredBlockCandidatesQuery::send() { - auto b = ton::create_serialize_tl_object(); - td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); - return td::Status::OK(); -} - -td::Status GetRequiredBlockCandidatesQuery::receive(td::BufferSlice data) { - TRY_RESULT_PREFIX( - f, ton::fetch_tl_object(data.as_slice(), true), - "received incorrect answer: "); - td::TerminalIO::out() << td::json_encode(td::ToJson(*f), true); - return td::Status::OK(); -} - -td::Status ImportBlockCandidateQuery::run() { - TRY_RESULT_ASSIGN(file_, tokenizer_.get_token()); - return td::Status::OK(); -} - -td::Status ImportBlockCandidateQuery::send() { - TRY_RESULT(data, td::read_file(file_)); - TRY_RESULT_PREFIX(candidate, ton::fetch_tl_object(data.as_slice(), true), - "invalid file: "); - auto b = ton::create_serialize_tl_object(std::move(candidate)); - td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); - return td::Status::OK(); -} - -td::Status ImportBlockCandidateQuery::receive(td::BufferSlice data) { - TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), - "received incorrect answer: "); - td::TerminalIO::out() << "successfully imported a block candidate\n"; - return td::Status::OK(); -} - td::Status AddCollatorQuery::run() { TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index daa19ab1a2..cfb8850d60 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1094,76 +1094,6 @@ class GetValidatorSessionsInfoQuery : public Query { } }; -class GenerateBlockCandidateQuery : public Query { - public: - GenerateBlockCandidateQuery(td::actor::ActorId console, Tokenizer tokenizer) - : Query(console, std::move(tokenizer)) { - } - td::Status run() override; - td::Status send() override; - td::Status receive(td::BufferSlice data) override; - static std::string get_name() { - return "genblock"; - } - static std::string get_help() { - return "genblock \t" - "generate a block candidate for a given shard (seqno mush match the next seqno for the shard), " - "candidate is saved to "; - } - std::string name() const override { - return get_name(); - } - - private: - td::int32 wc_; - td::int64 shard_; - td::int32 seqno_; - std::string file_; -}; - -class GetRequiredBlockCandidatesQuery : public Query { - public: - GetRequiredBlockCandidatesQuery(td::actor::ActorId console, Tokenizer tokenizer) - : Query(console, std::move(tokenizer)) { - } - td::Status run() override; - td::Status send() override; - td::Status receive(td::BufferSlice data) override; - static std::string get_name() { - return "getrequiredblockcandidates"; - } - static std::string get_help() { - return "getrequiredblockcandidates\t" - "get a list of block candidates that the validator is currently waiting for"; - } - std::string name() const override { - return get_name(); - } -}; - -class ImportBlockCandidateQuery : public Query { - public: - ImportBlockCandidateQuery(td::actor::ActorId console, Tokenizer tokenizer) - : Query(console, std::move(tokenizer)) { - } - td::Status run() override; - td::Status send() override; - td::Status receive(td::BufferSlice data) override; - static std::string get_name() { - return "importblockcandidate"; - } - static std::string get_help() { - return "importblockcandidate \t" - "load a block candidate from a given file"; - } - std::string name() const override { - return get_name(); - } - - private: - std::string file_; -}; - class AddCollatorQuery : public Query { public: AddCollatorQuery(td::actor::ActorId console, Tokenizer tokenizer) diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 83375b2099..0ce81fb411 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -141,9 +141,6 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); - add_query_runner(std::make_unique>()); - add_query_runner(std::make_unique>()); - add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); } diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index fda610afe4..9bafd347fd 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -3393,94 +3393,6 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getValida std::move(P)); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_generateBlockCandidate &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { - if (!(perm & ValidatorEnginePermissions::vep_default)) { - promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); - return; - } - if (validator_manager_.empty()) { - promise.set_value( - create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); - return; - } - ton::BlockId block_id = ton::create_block_id_simple(query.block_id_); - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::generate_block_candidate, - block_id, [promise = std::move(promise)](td::Result R) mutable { - if (R.is_ok()) { - auto block = R.move_as_ok(); - auto result = ton::create_serialize_tl_object( - ton::PublicKey{ton::pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), - ton::create_tl_block_id(block.id), std::move(block.data), - std::move(block.collated_data)); - promise.set_result(std::move(result)); - } else { - promise.set_value(create_control_query_error(R.move_as_error())); - } - }); -} - -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getRequiredBlockCandidates &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { - if (!(perm & ValidatorEnginePermissions::vep_default)) { - promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); - return; - } - if (validator_manager_.empty()) { - promise.set_value( - create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); - return; - } - td::actor::send_closure( - validator_manager_, &ton::validator::ValidatorManagerInterface::get_required_block_candidates, - [promise = std::move(promise)](td::Result> R) mutable { - if (R.is_ok()) { - std::vector> block_ids; - for (const ton::BlockId &block_id : R.move_as_ok()) { - block_ids.push_back(ton::create_tl_block_id_simple(block_id)); - } - auto result = ton::create_serialize_tl_object( - std::move(block_ids)); - promise.set_result(std::move(result)); - } else { - promise.set_value(create_control_query_error(R.move_as_error())); - } - }); -} - -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importBlockCandidate &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { - if (!(perm & ValidatorEnginePermissions::vep_modify)) { - promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); - return; - } - if (validator_manager_.empty()) { - promise.set_value( - create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); - return; - } - - auto collated_data_hash = td::sha256_bits256(query.block_->collated_data_); - auto key = ton::PublicKey{query.block_->source_}; - auto e_key = ton::Ed25519_PublicKey{key.ed25519_value().raw()}; - ton::BlockCandidate candidate{e_key, ton::create_block_id(query.block_->id_), collated_data_hash, - std::move(query.block_->data_), std::move(query.block_->collated_data_)}; - - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::import_block_candidate, - std::move(candidate), - [promise = std::move(promise)](td::Result R) mutable { - if (R.is_ok()) { - promise.set_result(ton::serialize_tl_object( - ton::create_tl_object(), true)); - } else { - promise.set_value(create_control_query_error(R.move_as_error())); - } - }); -} - void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollator &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 46de7e86a7..dd1c8fcabf 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -428,12 +428,6 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getValidatorSessionsInfo &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); - void run_control_query(ton::ton_api::engine_validator_generateBlockCandidate &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); - void run_control_query(ton::ton_api::engine_validator_getRequiredBlockCandidates &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); - void run_control_query(ton::ton_api::engine_validator_importBlockCandidate &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_addCollator &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_addShard &query, td::BufferSlice data, diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 345ae679f6..46fe887095 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -168,8 +168,6 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0; - virtual void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise promise) = 0; - static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 2e331c7208..3c45b48aa5 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -379,18 +379,6 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { UNREACHABLE(); } - void generate_block_candidate(BlockId block_id, td::Promise promise) override { - UNREACHABLE(); - } - void get_required_block_candidates(td::Promise> promise) override { - UNREACHABLE(); - } - void import_block_candidate(BlockCandidate candidate, td::Promise promise) override { - UNREACHABLE(); - } - void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise promise) override { - UNREACHABLE(); - } void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { UNREACHABLE(); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 79e742c4b7..bc8913b89c 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -439,18 +439,6 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { UNREACHABLE(); } - void generate_block_candidate(BlockId block_id, td::Promise promise) override { - UNREACHABLE(); - } - void get_required_block_candidates(td::Promise> promise) override { - UNREACHABLE(); - } - void import_block_candidate(BlockCandidate candidate, td::Promise promise) override { - UNREACHABLE(); - } - void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise promise) override { - UNREACHABLE(); - } void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { UNREACHABLE(); diff --git a/validator/manager.cpp b/validator/manager.cpp index 4427969813..ab5968943a 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2641,119 +2641,6 @@ void ValidatorManagerImpl::get_validator_sessions_info( IntermediateData::step({std::move(groups), {}, std::move(promise)}); } -void ValidatorManagerImpl::generate_block_candidate(BlockId block_id, td::Promise promise) { - if (!block_id.is_valid_full()) { - promise.set_error(td::Status::Error("invalid block id")); - return; - } - if (last_masterchain_state_.is_null()) { - promise.set_error(td::Status::Error("not started")); - return; - } - ShardIdFull shard_id = block_id.shard_full(); - std::vector prev; - auto shard = last_masterchain_state_->get_shard_from_config(shard_id); - if (shard.not_null()) { - if (shard->before_split()) { - promise.set_error(td::Status::Error("shard is before_split")); - return; - } - if (shard->before_merge()) { - promise.set_error(td::Status::Error("shard is before_merge")); - return; - } - prev.push_back(shard->top_block_id()); - } else { - auto parent = shard_id.pfx_len() == 0 ? td::Ref() - : last_masterchain_state_->get_shard_from_config(shard_parent(shard_id)); - if (parent.not_null() && parent->before_split()) { - prev.push_back(parent->top_block_id()); - } else { - auto child_l = last_masterchain_state_->get_shard_from_config(shard_child(shard_id, true)); - auto child_r = last_masterchain_state_->get_shard_from_config(shard_child(shard_id, false)); - if (child_l.not_null() && child_r.not_null() && child_l->before_merge() && child_r->before_merge()) { - prev.push_back(child_l->top_block_id()); - prev.push_back(child_r->top_block_id()); - } - } - if (prev.empty()) { - promise.set_error(td::Status::Error("no such shard")); - return; - } - } - - BlockSeqno next_seqno = 0; - for (const BlockIdExt& prev_id : prev) { - next_seqno = std::max(next_seqno, prev_id.seqno() + 1); - } - if (next_seqno != block_id.seqno) { - promise.set_error(td::Status::Error(PSTRING() << "seqno mismatch: asked for seqno " << block_id.seqno - << ", but actual next seqno is " << next_seqno)); - return; - } - - Ed25519_PublicKey local_id{Bits256::zero()}; - td::Ref validator_set = last_masterchain_state_->get_validator_set(shard_id); - if (validator_set.is_null()) { - promise.set_error(td::Status::Error("cannot get validator set")); - return; - } - run_collate_query(shard_id, last_masterchain_block_id_, std::move(prev), local_id, std::move(validator_set), - actor_id(this), td::Timestamp::in(10.0), std::move(promise)); -} - -void ValidatorManagerImpl::get_required_block_candidates(td::Promise> promise) { - std::vector block_ids; - for (const auto& p : pending_block_candidates_) { - block_ids.push_back(p.first); - } - promise.set_result(std::move(block_ids)); -} - -void ValidatorManagerImpl::import_block_candidate(BlockCandidate candidate, td::Promise promise) { - auto it = pending_block_candidates_.find(candidate.id.id); - if (it != pending_block_candidates_.end()) { - while (!it->second.empty()) { - auto promise = std::move(it->second.back().first); - it->second.pop_back(); - if (it->second.empty()) { - promise.set_result(std::move(candidate)); - } else { - promise.set_result(candidate.clone()); - } - } - pending_block_candidates_.erase(it); - } - promise.set_result(td::Unit()); -} - -void ValidatorManagerImpl::wait_block_candidate(BlockId block_id, td::Timestamp timeout, - td::Promise promise) { - pending_block_candidates_[block_id].emplace_back(std::move(promise), timeout); - delay_action([SelfId = actor_id(this), block_id, timeout]() { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::cleanup_old_pending_candidates, block_id, timeout); - }, timeout); -} - -void ValidatorManagerImpl::cleanup_old_pending_candidates(BlockId block_id, td::Timestamp now) { - auto it = pending_block_candidates_.find(block_id); - if (it == pending_block_candidates_.end()) { - return; - } - it->second.erase(std::remove_if(it->second.begin(), it->second.end(), - [&](std::pair, td::Timestamp> &p) { - if (p.second.is_in_past(now)) { - p.first.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); - return true; - } - return false; - }), - it->second.end()); - if (it->second.empty()) { - pending_block_candidates_.erase(it); - } -} - void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) { auto it = collator_nodes_.find(id); if (it == collator_nodes_.end()) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 6eb192a71b..5447c05e18 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -542,11 +542,6 @@ class ValidatorManagerImpl : public ValidatorManager { void get_validator_sessions_info( td::Promise> promise) override; - void generate_block_candidate(BlockId block_id, td::Promise promise) override; - void get_required_block_candidates(td::Promise> promise) override; - void import_block_candidate(BlockCandidate candidate, td::Promise promise) override; - void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise promise) override; - void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; private: @@ -614,9 +609,6 @@ class ValidatorManagerImpl : public ValidatorManager { private: std::map> shard_client_waiters_; - std::map, td::Timestamp>>> pending_block_candidates_; - void cleanup_old_pending_candidates(BlockId block_id, td::Timestamp now); - std::map> collator_nodes_; bool collating_masterchain_ = false; diff --git a/validator/validator.h b/validator/validator.h index c5212abb48..363c837f7b 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -229,10 +229,6 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_validator_sessions_info( td::Promise> promise) = 0; - virtual void generate_block_candidate(BlockId block_id, td::Promise promise) = 0; - virtual void get_required_block_candidates(td::Promise> promise) = 0; - virtual void import_block_candidate(BlockCandidate candidate, td::Promise promise) = 0; - virtual void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; }; From 5ba2a5571663fe5187a5615a60610223fb922990 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 8 Aug 2022 14:08:57 +0300 Subject: [PATCH 018/524] Add --lite-validator flag, rework is_validator checks --- tdutils/td/utils/OptionParser.cpp | 2 +- validator-engine/validator-engine.cpp | 33 ++++++++++----------------- validator-engine/validator-engine.hpp | 11 ++++++--- validator/manager.cpp | 23 ++++++++----------- validator/manager.hpp | 2 +- validator/validator-options.cpp | 2 +- validator/validator-options.hpp | 24 ++++++++++--------- validator/validator.h | 11 ++++----- 8 files changed, 50 insertions(+), 58 deletions(-) diff --git a/tdutils/td/utils/OptionParser.cpp b/tdutils/td/utils/OptionParser.cpp index b95848562b..634570e1b0 100644 --- a/tdutils/td/utils/OptionParser.cpp +++ b/tdutils/td/utils/OptionParser.cpp @@ -33,7 +33,7 @@ void OptionParser::set_description(string description) { void OptionParser::add_option(Option::Type type, char short_key, Slice long_key, Slice description, std::function callback) { for (auto &option : options_) { - if (option.short_key == short_key || (!long_key.empty() && long_key == option.long_key)) { + if ((short_key != '\0' && option.short_key == short_key) || (!long_key.empty() && long_key == option.long_key)) { LOG(ERROR) << "Ignore duplicated option '" << short_key << "' '" << long_key << "'"; } } diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 9bafd347fd..26c485bdf7 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1391,21 +1391,14 @@ td::Status ValidatorEngine::load_global_config() { h.push_back(b); } validator_options_.write().set_hardforks(std::move(h)); + validator_options_.write().set_validator_lite_mode(validator_lite_mode_); return td::Status::OK(); } void ValidatorEngine::init_validator_options() { - if (!masterchain_only_) { - validator_options_.write().set_shard_check_function( - [](ton::ShardIdFull shard, ton::CatchainSeqno cc_seqno, - ton::validator::ValidatorManagerOptions::ShardCheckMode mode) -> bool { - if (mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_monitor) { - return true; - } - CHECK(mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_validate); - return true; - }); + if (!not_all_shards_) { + validator_options_.write().set_shard_check_function([](ton::ShardIdFull shard) -> bool { return true; }); } else { std::vector shards = {ton::ShardIdFull(ton::masterchainId)}; for (const auto& c : config_.collators) { @@ -1417,18 +1410,13 @@ void ValidatorEngine::init_validator_options() { std::sort(shards.begin(), shards.end()); shards.erase(std::unique(shards.begin(), shards.end()), shards.end()); validator_options_.write().set_shard_check_function( - [shards = std::move(shards)](ton::ShardIdFull shard, ton::CatchainSeqno cc_seqno, - ton::validator::ValidatorManagerOptions::ShardCheckMode mode) -> bool { - if (mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_monitor) { - for (auto s : shards) { - if (shard_is_ancestor(shard, s)) { - return true; - } + [shards = std::move(shards)](ton::ShardIdFull shard) -> bool { + for (auto s : shards) { + if (shard_is_ancestor(shard, s)) { + return true; } - return false; } - CHECK(mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_validate); - return true; + return false; }); } } @@ -3704,7 +3692,10 @@ int main(int argc, char *argv[]) { return td::Status::OK(); }); p.add_option('M', "not-all-shards", "monitor only a necessary set of shards instead of all", [&]() { - acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_masterchain_only); }); + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_not_all_shards); }); + }); + p.add_option('\0', "lite-validator", "lite-mode validator (don't collate blocks, use collator nodes instead)", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_validator_lite_mode); }); }); td::uint32 threads = 7; p.add_checked_option( diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index dd1c8fcabf..8e333791af 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -214,7 +214,8 @@ class ValidatorEngine : public td::actor::Actor { bool started_ = false; ton::BlockSeqno truncate_seqno_{0}; std::string session_logs_file_; - bool masterchain_only_ = false; + bool not_all_shards_ = false; + bool validator_lite_mode_ = false; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -266,9 +267,13 @@ class ValidatorEngine : public td::actor::Actor { void add_key_to_set(ton::PublicKey key) { keys_[key.compute_short_id()] = key; } - void set_masterchain_only() { - masterchain_only_ = true; + void set_not_all_shards() { + not_all_shards_ = true; } + void set_validator_lite_mode() { + validator_lite_mode_ = true; + } + void start_up() override; ValidatorEngine() { } diff --git a/validator/manager.cpp b/validator/manager.cpp index ab5968943a..c2bb706197 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -369,7 +369,7 @@ void ValidatorManagerImpl::get_key_block_proof_link(BlockIdExt block_id, td::Pro } void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { - if (!is_validator()) { + if (!is_collator()) { return; } if ((double)ext_messages_.size() > max_mempool_num()) { @@ -388,7 +388,7 @@ void ValidatorManagerImpl::add_external_message(td::Ref msg) { auto id = message->ext_id(); auto address = message->address(); unsigned long per_address_limit = 256; - if(ext_addr_messages_.count(address) < per_address_limit) { + if (ext_addr_messages_.count(address) < per_address_limit) { if (ext_messages_hashes_.count(id.hash) == 0) { ext_messages_.emplace(id, std::move(message)); ext_messages_hashes_.emplace(id.hash, id); @@ -401,7 +401,7 @@ void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Prom } void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { - if (!is_validator()) { + if (!is_collator()) { return; } auto R = create_ihr_message(std::move(data)); @@ -418,7 +418,7 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - if (!is_validator()) { + if (!is_collator()) { return; } if (!last_masterchain_block_handle_) { @@ -2086,7 +2086,7 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group "validatorgroup", shard, validator_id, session_id, validator_set, last_masterchain_state_->get_collator_set(), opts, keyring_, adnl_, rldp_, overlays_, db_root_, actor_id(this), init_session, - opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), true); + opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_->validator_lite_mode()); return G; } } @@ -2457,14 +2457,14 @@ void ValidatorManagerImpl::get_archive_slice(td::uint64 archive_id, td::uint64 o } bool ValidatorManagerImpl::is_validator() { - // TODO: change is_vaidator to other condition in some cases - return true; // temp_keys_.size() > 0 || permanent_keys_.size() > 0; + return temp_keys_.size() > 0 || permanent_keys_.size() > 0; +} + +bool ValidatorManagerImpl::is_collator() { + return !collator_nodes_.empty() || (!opts_->validator_lite_mode() && is_validator()); } PublicKeyHash ValidatorManagerImpl::get_validator(ShardIdFull shard, td::Ref val_set) { - if (!opts_->need_validate(shard, val_set->get_catchain_seqno())) { - return PublicKeyHash::zero(); - } for (auto &key : temp_keys_) { if (val_set->is_validator(key.bits256_value())) { return key; @@ -2648,9 +2648,6 @@ void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh it = collator_nodes_.emplace(id, std::move(actor)).first; } td::actor::send_closure(it->second, &CollatorNode::add_shard, shard); - if (shard.is_masterchain()) { - collating_masterchain_ = true; - } } td::actor::ActorOwn ValidatorManagerFactory::create( diff --git a/validator/manager.hpp b/validator/manager.hpp index 5447c05e18..9e55abf586 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -491,6 +491,7 @@ class ValidatorManagerImpl : public ValidatorManager { void read_gc_list(std::vector list); bool is_validator(); + bool is_collator(); PublicKeyHash get_validator(ShardIdFull shard, td::Ref val_set); ValidatorManagerImpl(td::Ref opts, std::string db_root, @@ -610,7 +611,6 @@ class ValidatorManagerImpl : public ValidatorManager { std::map> shard_client_waiters_; std::map> collator_nodes_; - bool collating_masterchain_ = false; std::set shards_to_monitor_ = {ShardIdFull(masterchainId)}; }; diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index 93fe05e6cd..cb26fe44d6 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -26,7 +26,7 @@ namespace validator { td::Ref ValidatorManagerOptions::create( BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, bool allow_blockchain_init, + std::function check_shard, bool allow_blockchain_init, double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) { return td::make_ref(zero_block_id, init_block_id, std::move(check_shard), diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index d23d8cc91b..b4a94a39bf 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -33,10 +33,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { return init_block_id_; } bool need_monitor(ShardIdFull shard) const override { - return check_shard_(shard, 0, ShardCheckMode::m_monitor); - } - bool need_validate(ShardIdFull shard, CatchainSeqno cc_seqno) const override { - return check_shard_(shard, cc_seqno, ShardCheckMode::m_validate); + return check_shard_(shard); } bool allow_blockchain_init() const override { return allow_blockchain_init_; @@ -114,6 +111,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { std::string get_session_logs_file() const override { return session_logs_file_; } + bool validator_lite_mode() const override { + return validator_lite_mode_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -121,7 +121,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_init_block_id(BlockIdExt block_id) override { init_block_id_ = block_id; } - void set_shard_check_function(std::function check_shard) override { + void set_shard_check_function(std::function check_shard) override { check_shard_ = std::move(check_shard); } void set_allow_blockchain_init(bool value) override { @@ -167,17 +167,18 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_session_logs_file(std::string f) override { session_logs_file_ = std::move(f); } + void set_validator_lite_mode(bool value) override { + validator_lite_mode_ = value; + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); } ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, - bool allow_blockchain_init, double sync_blocks_before, - double block_ttl, double state_ttl, double max_mempool_num, - double archive_ttl, double key_proof_ttl, - bool initial_sync_disabled) + std::function check_shard, bool allow_blockchain_init, + double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, + double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) : zero_block_id_(zero_block_id) , init_block_id_(init_block_id) , check_shard_(std::move(check_shard)) @@ -194,7 +195,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { private: BlockIdExt zero_block_id_; BlockIdExt init_block_id_; - std::function check_shard_; + std::function check_shard_; bool allow_blockchain_init_; double sync_blocks_before_; double block_ttl_; @@ -209,6 +210,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { BlockSeqno truncate_{0}; BlockSeqno sync_upto_{0}; std::string session_logs_file_; + bool validator_lite_mode_ = false; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 363c837f7b..545c8a81c7 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -47,12 +47,9 @@ class DownloadToken { struct ValidatorManagerOptions : public td::CntObject { public: - enum class ShardCheckMode { m_monitor, m_validate }; - virtual BlockIdExt zero_block_id() const = 0; virtual BlockIdExt init_block_id() const = 0; virtual bool need_monitor(ShardIdFull shard) const = 0; - virtual bool need_validate(ShardIdFull shard, CatchainSeqno cc_seqno) const = 0; virtual bool allow_blockchain_init() const = 0; virtual double sync_blocks_before() const = 0; virtual double block_ttl() const = 0; @@ -75,11 +72,11 @@ struct ValidatorManagerOptions : public td::CntObject { virtual BlockSeqno get_truncate_seqno() const = 0; virtual BlockSeqno sync_upto() const = 0; virtual std::string get_session_logs_file() const = 0; + virtual bool validator_lite_mode() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; - virtual void set_shard_check_function( - std::function check_shard) = 0; + virtual void set_shard_check_function(std::function check_shard) = 0; virtual void set_allow_blockchain_init(bool value) = 0; virtual void set_sync_blocks_before(double value) = 0; virtual void set_block_ttl(double value) = 0; @@ -94,11 +91,11 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void truncate_db(BlockSeqno seqno) = 0; virtual void set_sync_upto(BlockSeqno seqno) = 0; virtual void set_session_logs_file(std::string f) = 0; + virtual void set_validator_lite_mode(bool value) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard = [](ShardIdFull, CatchainSeqno, - ShardCheckMode) { return true; }, + std::function check_shard = [](ShardIdFull) { return true; }, bool allow_blockchain_init = false, double sync_blocks_before = 86400, double block_ttl = 86400 * 7, double state_ttl = 3600, double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650, double max_mempool_num = 999999, From 7749cbfa1ffb0de45f853cb39c744e061d3bf13e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 9 Aug 2022 17:37:01 +0300 Subject: [PATCH 019/524] Change config serialization for compatibility --- validator-engine/validator-engine.cpp | 18 +++++++++++++----- validator-engine/validator-engine.hpp | 2 +- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 26c485bdf7..65fb0e8f2f 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -177,7 +177,7 @@ Config::Config(const ton::ton_api::engine_validator_config_v2 &config) { } } -ton::tl_object_ptr Config::tl() const { +ton::tl_object_ptr Config::tl() const { std::vector> addrs_vec; for (auto &x : addrs) { if (x.second.proxy) { @@ -256,10 +256,18 @@ ton::tl_object_ptr Config::tl() const for (auto &id : gc) { gc_vec->ids_.push_back(id.tl()); } - return ton::create_tl_object( - out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), - full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), - std::move(control_vec), std::move(shards_vec), std::move(gc_vec)); + + if (col_vec.empty() && shards_vec.empty()) { + return ton::create_tl_object( + out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), + full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), + std::move(control_vec), std::move(gc_vec)); + } else { + return ton::create_tl_object( + out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), + full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), + std::move(control_vec), std::move(shards_vec), std::move(gc_vec)); + } } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 8e333791af..47d612b7ba 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -137,7 +137,7 @@ struct Config { td::Result config_del_control_process(td::int32 port, ton::PublicKeyHash id); td::Result config_del_gc(ton::PublicKeyHash key); - ton::tl_object_ptr tl() const; + ton::tl_object_ptr tl() const; Config(); Config(const ton::ton_api::engine_validator_config_v2 &config); From 662435462e16b5e9489ecfb881a73e59ede9166b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 10 Aug 2022 20:59:49 +0300 Subject: [PATCH 020/524] Validators temporary join shard overlays --- overlay/overlay-manager.cpp | 20 +++++--- overlay/overlay-manager.h | 3 +- overlay/overlay-peers.cpp | 2 +- overlay/overlay.cpp | 53 +++++++++++---------- overlay/overlay.h | 2 +- overlay/overlay.hpp | 8 +--- overlay/overlays.h | 7 ++- tl/generate/scheme/ton_api.tl | 2 + tl/generate/scheme/ton_api.tlo | Bin 71776 -> 71904 bytes validator/full-node-shard.cpp | 56 +++++++++++++---------- validator/full-node-shard.hpp | 1 + validator/interfaces/validator-manager.h | 2 + validator/manager-disk.hpp | 2 + validator/manager-hardfork.hpp | 2 + validator/manager.cpp | 30 +++++++++++- validator/manager.hpp | 9 ++++ validator/validator-group.cpp | 13 ++++-- validator/validator-group.hpp | 2 +- 18 files changed, 140 insertions(+), 74 deletions(-) diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index 62d98eb00e..2134bbc0c6 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -30,7 +30,6 @@ #include "td/utils/Status.h" #include "td/utils/overloaded.h" -#include "keys/encryptor.h" #include "td/utils/port/Poll.h" #include @@ -92,7 +91,6 @@ void OverlayManager::delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdSho void OverlayManager::create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) { CHECK(!dht_node_.empty()); - CHECK(callback != nullptr); auto id = overlay_id.compute_short_id(); register_overlay(local_id, id, Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), @@ -100,12 +98,13 @@ void OverlayManager::create_public_overlay(adnl::AdnlNodeIdShort local_id, Overl } void OverlayManager::create_public_overlay_external(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - OverlayPrivacyRules rules, td::string scope) { + std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope) { CHECK(!dht_node_.empty()); auto id = overlay_id.compute_short_id(); register_overlay(local_id, id, - Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), nullptr, - std::move(rules), scope)); + Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), + std::move(callback), std::move(rules), scope, true)); } void OverlayManager::create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, @@ -155,16 +154,23 @@ void OverlayManager::receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdSh auto M = R.move_as_ok(); + OverlayIdShort overlay_id{M->overlay_}; + auto send_remove_peer = [&, this]() { + send_message(src, dst, overlay_id, create_serialize_tl_object()); + }; + auto it = overlays_.find(dst); if (it == overlays_.end()) { VLOG(OVERLAY_NOTICE) << this << ": query to unknown overlay " << M->overlay_ << "@" << dst << " from " << src; promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad local_id " << dst)); + send_remove_peer(); return; } - auto it2 = it->second.find(OverlayIdShort{M->overlay_}); + auto it2 = it->second.find(overlay_id); if (it2 == it->second.end()) { VLOG(OVERLAY_NOTICE) << this << ": query to localid not in overlay " << M->overlay_ << "@" << dst << " from " << src; promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad overlay_id " << M->overlay_)); + send_remove_peer(); return; } @@ -394,7 +400,7 @@ td::BufferSlice Certificate::to_sign(OverlayIdShort overlay_id, PublicKeyHash is } } -const PublicKeyHash Certificate::issuer_hash() const { +PublicKeyHash Certificate::issuer_hash() const { PublicKeyHash r; issued_by_.visit( td::overloaded([&](const PublicKeyHash &x) { r = x; }, [&](const PublicKey &x) { r = x.compute_short_id(); })); diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index a3ed4a54ab..ada37303d9 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -53,7 +53,8 @@ class OverlayManager : public Overlays { void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) override; void create_public_overlay_external(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - OverlayPrivacyRules rules, td::string scope) override; + std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope) override; void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules) override; diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index dbb6f32a87..e1e4a3ce83 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -142,7 +142,7 @@ void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::BufferSlic void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode node, td::Promise promise) { std::vector> vec; - if (!is_external()) { + if (!is_external_) { vec.emplace_back(node.tl()); } diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 8a655c6e59..ff86d837ec 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -38,10 +38,10 @@ td::actor::ActorOwn Overlay::create(td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope) { + OverlayPrivacyRules rules, td::string scope, bool is_external) { auto R = td::actor::create_actor("overlay", keyring, adnl, manager, dht_node, local_id, std::move(overlay_id), true, std::vector(), - std::move(callback), std::move(rules), scope); + std::move(callback), std::move(rules), scope, is_external); return td::actor::ActorOwn(std::move(R)); } @@ -61,7 +61,7 @@ OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub, std::vector nodes, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope) + OverlayPrivacyRules rules, td::string scope, bool is_external) : keyring_(keyring) , adnl_(adnl) , manager_(manager) @@ -71,10 +71,11 @@ OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor , callback_(std::move(callback)) , public_(pub) , rules_(std::move(rules)) - , scope_(scope) { + , scope_(scope) + , is_external_(is_external) { overlay_id_ = id_full_.compute_short_id(); - if (is_external()) { + if (is_external_) { CHECK(public_); VLOG(OVERLAY_INFO) << this << ": creating public external"; } else { @@ -92,7 +93,6 @@ OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeers &query, td::Promise promise) { - CHECK(!is_external()); if (public_) { VLOG(OVERLAY_DEBUG) << this << ": received " << query.peers_->nodes_.size() << " nodes from " << src << " in getRandomPeers query"; @@ -113,7 +113,6 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getR void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, td::Promise promise) { - CHECK(!is_external()); auto it = broadcasts_.find(query.hash_); if (it == broadcasts_.end()) { VLOG(OVERLAY_NOTICE) << this << ": received getBroadcastQuery(" << query.hash_ << ") from " << src @@ -135,15 +134,16 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getB void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcastList &query, td::Promise promise) { - CHECK(!is_external()); VLOG(OVERLAY_WARNING) << this << ": DROPPING getBroadcastList query"; promise.set_error(td::Status::Error(ErrorCode::protoviolation, "dropping get broadcast list query")); } void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) { - if (is_external()) { + if (is_external_) { LOG(OVERLAY_WARNING) << "dropping query in external overlay " << overlay_id_; promise.set_error(td::Status::Error("overlay is external")); + td::actor::send_closure(manager_, &Overlays::send_message, src, local_id_, overlay_id_, + create_serialize_tl_object()); return; } if (!public_) { @@ -171,32 +171,27 @@ void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr bcast) { - CHECK(!is_external()); return BroadcastSimple::create(this, message_from, std::move(bcast)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr b) { - CHECK(!is_external()); return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr b) { - CHECK(!is_external()); return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr bcast) { - CHECK(!is_external()); return td::Status::Error(ErrorCode::protoviolation, PSTRING() << "received strange message broadcastNotFound from " << message_from); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { - CHECK(!is_external()); auto it = fec_broadcasts_.find(msg->hash_); if (it != fec_broadcasts_.end()) { VLOG(OVERLAY_DEBUG) << this << ": received fec opt-out message from " << message_from << " for broadcast " @@ -211,7 +206,6 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { - CHECK(!is_external()); auto it = fec_broadcasts_.find(msg->hash_); if (it != fec_broadcasts_.end()) { VLOG(OVERLAY_DEBUG) << this << ": received fec completed message from " << message_from << " for broadcast " @@ -226,17 +220,12 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { - CHECK(!is_external()); VLOG(OVERLAY_DEBUG) << this << ": received unicast from " << message_from; callback_->receive_message(message_from, overlay_id_, std::move(msg->data_)); return td::Status::OK(); } void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) { - if (is_external()) { - LOG(OVERLAY_WARNING) << "dropping message in external overlay " << overlay_id_; - return; - } if (!public_) { if (peers_.get(src) == nullptr) { VLOG(OVERLAY_WARNING) << this << ": received query in private overlay from unknown source " << src; @@ -244,13 +233,29 @@ void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice dat } } auto X = fetch_tl_object(data.clone(), true); + if (X.is_error()) { + auto Y = fetch_tl_object(data.clone(), true); + if (Y.is_ok()) { + VLOG(OVERLAY_DEBUG) << this << ": received removePeer message from " << src; + if (peers_.exists(src)) { + del_peer(src); + callback_->on_remove_peer(src); + } + return; + } + } + + if (is_external_) { + LOG(OVERLAY_WARNING) << "dropping message in external overlay " << overlay_id_; + return; + } if (X.is_error()) { VLOG(OVERLAY_DEBUG) << this << ": received custom message"; callback_->receive_message(src, overlay_id_, std::move(data)); return; } auto Q = X.move_as_ok(); - ton_api::downcast_call(*Q.get(), [Self = this, &Q, &src](auto &object) { + ton_api::downcast_call(*Q, [Self = this, &Q, &src](auto &object) { Self->process_broadcast(src, move_tl_object_as>(Q)); }); } @@ -344,7 +349,7 @@ void OverlayImpl::receive_dht_nodes(td::Result res, bool dummy) { VLOG(OVERLAY_NOTICE) << this << ": can not get value from DHT: " << res.move_as_error(); } - if (is_external()) { + if (is_external_) { return; } @@ -548,7 +553,7 @@ void OverlayImpl::send_new_fec_broadcast_part(PublicKeyHash local_id, Overlay::B } void OverlayImpl::deliver_broadcast(PublicKeyHash source, td::BufferSlice data) { - if (is_external()) { + if (is_external_) { return; } callback_->receive_broadcast(source, overlay_id_, std::move(data)); @@ -623,7 +628,7 @@ void OverlayImpl::set_privacy_rules(OverlayPrivacyRules rules) { } void OverlayImpl::check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise promise) { - if (is_external()) { + if (is_external_) { promise.set_result(td::Unit()); return; } diff --git a/overlay/overlay.h b/overlay/overlay.h index 51eab1b435..146be73836 100644 --- a/overlay/overlay.h +++ b/overlay/overlay.h @@ -42,7 +42,7 @@ class Overlay : public td::actor::Actor { td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope); + OverlayPrivacyRules rules, td::string scope, bool is_external = false); static td::actor::ActorOwn create(td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId manager, diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 59b3ec06ac..197864946a 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -113,7 +113,7 @@ class OverlayImpl : public Overlay { td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub, std::vector nodes, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope = "{ \"type\": \"undefined\" }"); + OverlayPrivacyRules rules, td::string scope = "{ \"type\": \"undefined\" }", bool is_external = false); void update_dht_node(td::actor::ActorId dht) override { dht_node_ = dht; } @@ -251,13 +251,8 @@ class OverlayImpl : public Overlay { } private: - bool is_external() const { - return callback_ == nullptr; - } - template void process_query(adnl::AdnlNodeIdShort src, T &query, td::Promise promise) { - CHECK(!is_external()); callback_->receive_query(src, overlay_id_, serialize_tl_object(&query, true), std::move(promise)); } @@ -357,6 +352,7 @@ class OverlayImpl : public Overlay { bool semi_public_ = false; OverlayPrivacyRules rules_; td::string scope_; + bool is_external_ = false; std::map> certs_; class CachedEncryptor : public td::ListNode { diff --git a/overlay/overlays.h b/overlay/overlays.h index 82617fab63..0a9c7765c8 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -145,7 +145,7 @@ class Certificate { bool is_fec) const; tl_object_ptr tl() const; const PublicKey &issuer() const; - const PublicKeyHash issuer_hash() const; + PublicKeyHash issuer_hash() const; static td::Result> create(tl_object_ptr cert); static tl_object_ptr empty_tl(); @@ -170,6 +170,8 @@ class Overlays : public td::actor::Actor { td::Promise promise) { promise.set_value(td::Unit()); } + virtual void on_remove_peer(adnl::AdnlNodeIdShort src) { + } virtual ~Callback() = default; }; @@ -195,7 +197,8 @@ class Overlays : public td::actor::Actor { virtual void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) = 0; virtual void create_public_overlay_external(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - OverlayPrivacyRules rules, td::string scope) = 0; + std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope) = 0; virtual void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules) = 0; diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 837e575416..733c034a3a 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -211,6 +211,8 @@ overlay.message overlay:int256 = overlay.Message; //overlay.randomPeers peers:(vector adnl.node) = overlay.RandomPeers; overlay.broadcastList hashes:(vector int256) = overlay.BroadcastList; +overlay.message.removePeer = overlay.message.InternalMessage; + overlay.fec.received hash:int256 = overlay.Broadcast; overlay.fec.completed hash:int256 = overlay.Broadcast; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 36c21e21f7ebcbdc819c4ab0da29bca718a36353..c62bf28ef2a0a6bcd2314d8069c5af25cad09281 100644 GIT binary patch delta 132 zcmaE`f#tzQ7T!m*^{p77;NnJJZCTkTGgp6=&o4_Y%1Nx$%S|mVPE1eL^UNzrEy_#G zncOQazu7{Thl|m7v#a{=HVcqyDRk9EskuO*fYj6?29W&CV?sV4D~g=ibU19=4uZrs Jo2=8x0s!WpGt>Y8 delta 41 scmaE`k>$Y#7T!m*^{p77;KD{;ZQ0EwvOHXjKARiWf46NuuudTh030t4ApigX diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index a4eac57a96..fea4916dfa 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -25,7 +25,6 @@ #include "ton/ton-shard.h" #include "ton/ton-tl.hpp" -#include "ton/ton-io.hpp" #include "adnl/utils.hpp" #include "net/download-block-new.hpp" @@ -71,38 +70,41 @@ void Neighbour::update_roundtrip(double t) { } void FullNodeShardImpl::create_overlay() { - if (active_) { - class Callback : public overlay::Overlays::Callback { - public: - void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, - td::BufferSlice data) override { - // just ignore - } - void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, - td::Promise promise) override { - td::actor::send_closure(node_, &FullNodeShardImpl::receive_query, src, std::move(data), std::move(promise)); - } - void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { - td::actor::send_closure(node_, &FullNodeShardImpl::receive_broadcast, src, std::move(data)); - } - void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, - td::Promise promise) override { - td::actor::send_closure(node_, &FullNodeShardImpl::check_broadcast, src, std::move(data), std::move(promise)); - } - Callback(td::actor::ActorId node) : node_(node) { - } + class Callback : public overlay::Overlays::Callback { + public: + void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, + td::BufferSlice data) override { + // just ignore + } + void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(node_, &FullNodeShardImpl::receive_query, src, std::move(data), std::move(promise)); + } + void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + td::actor::send_closure(node_, &FullNodeShardImpl::receive_broadcast, src, std::move(data)); + } + void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(node_, &FullNodeShardImpl::check_broadcast, src, std::move(data), std::move(promise)); + } + void on_remove_peer(adnl::AdnlNodeIdShort src) override { + td::actor::send_closure(node_, &FullNodeShardImpl::remove_neighbour, src); + } + Callback(td::actor::ActorId node) : node_(node) { + } - private: - td::actor::ActorId node_; - }; + private: + td::actor::ActorId node_; + }; + if (active_) { td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay, adnl_id_, overlay_id_full_.clone(), std::make_unique(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() << ", \"workchain_id\": " << get_workchain() << " }"); } else { td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay_external, adnl_id_, - overlay_id_full_.clone(), rules_, + overlay_id_full_.clone(), std::make_unique(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() << ", \"workchain_id\": " << get_workchain() << " }"); } @@ -124,6 +126,10 @@ void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broad std::move(q->message_->data_), std::move(promise)); } +void FullNodeShardImpl::remove_neighbour(adnl::AdnlNodeIdShort id) { + neighbours_.erase(id); +} + void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); adnl_id_ = adnl_id; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index ead8cafd58..a9b7618153 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -145,6 +145,7 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void check_broadcast(PublicKeyHash src, td::BufferSlice query, td::Promise promise); + void remove_neighbour(adnl::AdnlNodeIdShort id); void send_ihr_message(td::BufferSlice data) override; void send_external_message(td::BufferSlice data) override; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 46fe887095..0477b64873 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -168,6 +168,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0; + virtual void validated_new_block(BlockIdExt block_id) = 0; + static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 3c45b48aa5..53467ff323 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -375,6 +375,8 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { UNREACHABLE(); } + void validated_new_block(BlockIdExt block_id) override { + } void get_validator_sessions_info( td::Promise> promise) override { UNREACHABLE(); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index bc8913b89c..9aa302263b 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -435,6 +435,8 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { UNREACHABLE(); } + void validated_new_block(BlockIdExt block_id) override { + } void get_validator_sessions_info( td::Promise> promise) override { UNREACHABLE(); diff --git a/validator/manager.cpp b/validator/manager.cpp index c2bb706197..04edefdc63 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -418,7 +418,7 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - if (!is_collator()) { + if (!is_collator() && !is_validator()) { return; } if (!last_masterchain_block_handle_) { @@ -1794,6 +1794,7 @@ void ValidatorManagerImpl::update_shards() { opts.proto_version = std::max(opts.proto_version, 1); } auto opts_hash = opts.get_hash(); + extra_active_shards_.clear(); std::map> new_shards; std::set future_shards; @@ -1848,6 +1849,11 @@ void ValidatorManagerImpl::update_shards() { default: LOG(FATAL) << "state=" << static_cast(v->fsm_state()); } + cleanup_last_validated_blocks(v->top_block_id().id); + } + + for (const auto& s : last_validated_blocks_) { + extra_active_shards_.insert(s.first); } new_shards.emplace(ShardIdFull{masterchainId, shardIdAll}, std::vector{last_masterchain_block_id_}); @@ -1920,6 +1926,7 @@ void ValidatorManagerImpl::update_shards() { auto validator_id = get_validator(shard, val_set); if (!validator_id.is_zero()) { + extra_active_shards_.insert(shard); auto val_group_id = get_validator_set_id(shard, val_set, opts_hash, key_seqno, opts); if (force_recover) { @@ -2012,7 +2019,25 @@ void ValidatorManagerImpl::update_shards() { }); td::actor::send_closure(db_, &Db::update_destroyed_validator_sessions, gc_list_, std::move(P)); } -} // namespace validator +} + +void ValidatorManagerImpl::cleanup_last_validated_blocks(BlockId new_block) { + auto process_shard = [&, this](ShardIdFull shard) { + auto it = last_validated_blocks_.find(shard); + if (it != last_validated_blocks_.end() && it->second < new_block.seqno) { + last_validated_blocks_.erase(it); + } + }; + ShardIdFull shard = new_block.shard_full(); + process_shard(shard); + if (shard.pfx_len() > 0) { + process_shard(shard_parent(shard)); + } + if (shard.pfx_len() < max_shard_pfx_len) { + process_shard(shard_child(shard, true)); + process_shard(shard_child(shard, false)); + } +} void ValidatorManagerImpl::written_destroyed_validator_sessions(std::vector> list) { for (auto &v : list) { @@ -2428,6 +2453,7 @@ void ValidatorManagerImpl::get_shard_client_state(bool from_db, td::Promise state, std::set shards_to_monitor) { shards_to_monitor_ = shards_to_monitor; + shards_to_monitor.insert(extra_active_shards_.begin(), extra_active_shards_.end()); callback_->update_shard_configuration(std::move(state), std::move(shards_to_monitor)); } diff --git a/validator/manager.hpp b/validator/manager.hpp index 9e55abf586..c65914aae7 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -543,6 +543,11 @@ class ValidatorManagerImpl : public ValidatorManager { void get_validator_sessions_info( td::Promise> promise) override; + void validated_new_block(BlockIdExt block_id) override { + BlockSeqno &last = last_validated_blocks_[block_id.shard_full()]; + last = std::max(last, block_id.seqno()); + } + void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; private: @@ -606,13 +611,17 @@ class ValidatorManagerImpl : public ValidatorManager { double max_mempool_num() const { return opts_->max_mempool_num(); } + void cleanup_last_validated_blocks(BlockId new_block); private: + std::map> shard_client_waiters_; std::map> collator_nodes_; std::set shards_to_monitor_ = {ShardIdFull(masterchainId)}; + std::set extra_active_shards_; + std::map last_validated_blocks_; }; } // namespace validator diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 6c0e242d5f..a72bded100 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -59,10 +59,14 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat approved_candidates_cache_round_ = round_id; approved_candidates_cache_.clear(); } + auto next_block_id = create_next_block_id(block.id.root_hash, block.id.file_hash); + block.id = next_block_id; + CacheKey cache_key = block_to_cache_key(block); auto it = approved_candidates_cache_.find(cache_key); if (it != approved_candidates_cache_.end()) { promise.set_result(it->second); + return; } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), round_id, block = block.clone(), @@ -96,9 +100,7 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat P.set_error(td::Status::Error(ErrorCode::notready, "validator group not started")); return; } - auto next_block_id = create_next_block_id(block.id.root_hash, block.id.file_hash); VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id; - block.id = next_block_id; run_validate_query(shard_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, manager_, td::Timestamp::in(10.0), std::move(P), lite_mode_ ? ValidateMode::lite : 0); } @@ -140,7 +142,10 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, td::Ref sig_set, td::Ref approve_sig_set, - bool send_broadcast, td::Promise promise) { + bool send_broadcast, td::Promise promise, bool is_retry) { + if (!is_retry) { + td::actor::send_closure(manager_, &ValidatorManager::validated_new_block, block_id); + } auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -150,7 +155,7 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref } LOG_CHECK(R.error().code() == ErrorCode::timeout || R.error().code() == ErrorCode::notready) << R.move_as_error(); td::actor::send_closure(SelfId, &ValidatorGroup::accept_block_query, block_id, std::move(block), std::move(prev), - std::move(sig_set), std::move(approve_sig_set), false, std::move(promise)); + std::move(sig_set), std::move(approve_sig_set), false, std::move(promise), true); } else { promise.set_value(R.move_as_ok()); } diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 76e6ea9922..4614ea12ad 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -41,7 +41,7 @@ class ValidatorGroup : public td::actor::Actor { void skip_round(td::uint32 round); void accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, td::Ref sigs, td::Ref approve_sigs, - bool send_broadcast, td::Promise promise); + bool send_broadcast, td::Promise promise, bool is_retry = false); void get_approved_candidate(PublicKey source, RootHash root_hash, FileHash file_hash, FileHash collated_data_file_hash, td::Promise promise); BlockId create_next_block_id_simple() const; From 910398da92af2cbabef208755e4cc90c0e572308 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 12 Aug 2022 12:10:46 +0300 Subject: [PATCH 021/524] Improved neighbor choosing in full-node-shard --- create-hardfork/create-hardfork.cpp | 3 +- dht-server/dht-server.cpp | 2 +- dht-server/dht-server.hpp | 2 +- overlay/overlay.cpp | 4 +- test/test-ton-collator.cpp | 3 +- tl/generate/scheme/ton_api.tl | 4 +- tl/generate/scheme/ton_api.tlo | Bin 71904 -> 72144 bytes ton/ton-tl.hpp | 10 +- validator-engine/validator-engine.cpp | 4 +- validator-engine/validator-engine.hpp | 2 +- validator/full-node-shard.cpp | 135 ++++++++++++++++++-------- validator/full-node-shard.h | 11 ++- validator/full-node-shard.hpp | 36 ++++--- validator/full-node.cpp | 61 ++++++------ validator/full-node.hpp | 7 +- validator/manager.cpp | 3 +- validator/validator.h | 3 +- 17 files changed, 185 insertions(+), 105 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index a24b7fc183..3fb57f41b5 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -237,7 +237,8 @@ class HardforkCreator : public td::actor::Actor { td::PromiseCreator::lambda([](td::Unit) {})); } void update_shard_configuration(td::Ref state, - std::set shards_to_monitor) override { + std::set shards_to_monitor, + std::set temporary_shards) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index d1f439534c..3bd65cc6e3 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -57,7 +57,7 @@ Config::Config() { out_port = 3278; } -Config::Config(const ton::ton_api::engine_validator_config_v2 &config) { +Config::Config(const ton::ton_api::engine_validator_configV2 &config) { out_port = static_cast(config.out_port_); if (!out_port) { out_port = 3278; diff --git a/dht-server/dht-server.hpp b/dht-server/dht-server.hpp index 724e7bbfa4..724232fa74 100644 --- a/dht-server/dht-server.hpp +++ b/dht-server/dht-server.hpp @@ -94,7 +94,7 @@ struct Config { ton::tl_object_ptr tl() const; Config(); - Config(const ton::ton_api::engine_validator_config_v2 &config); + Config(const ton::ton_api::engine_validator_configV2 &config); }; class DhtServer : public td::actor::Actor { diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index ff86d837ec..8e35e2a252 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -235,12 +235,12 @@ void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice dat auto X = fetch_tl_object(data.clone(), true); if (X.is_error()) { auto Y = fetch_tl_object(data.clone(), true); - if (Y.is_ok()) { + if (Y.is_ok() && public_) { VLOG(OVERLAY_DEBUG) << this << ": received removePeer message from " << src; if (peers_.exists(src)) { del_peer(src); - callback_->on_remove_peer(src); } + callback_->on_remove_peer(src); return; } } diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index d9ec82e5bd..8ef469b5f0 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -324,7 +324,8 @@ class TestNode : public td::actor::Actor { td::PromiseCreator::lambda([](td::Unit) {})); } void update_shard_configuration(td::Ref state, - std::set shards_to_monitor) override { + std::set shards_to_monitor, + std::set temporary_shards) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 733c034a3a..39cfb5eec7 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -407,6 +407,7 @@ tonNode.dataFull id:tonNode.blockIdExt proof:bytes block:bytes is_link:Bool = to tonNode.dataFullEmpty = tonNode.DataFull; tonNode.capabilities version:int capabilities:long = tonNode.Capabilities; +tonNode.capabilitiesV2 version:int capabilities:long have_state:Bool = tonNode.Capabilities; tonNode.success = tonNode.Success; @@ -450,6 +451,7 @@ tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data; tonNode.getOutMsgQueueProof block_id:tonNode.blockIdExt dst_workchain:int dst_shard:long = tonNode.OutMsgQueueProof; tonNode.getCapabilities = tonNode.Capabilities; +tonNode.getCapabilitiesV2 = tonNode.Capabilities; tonNode.slave.sendExtMessage message:tonNode.externalMessage = tonNode.Success; @@ -598,7 +600,7 @@ engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector eng liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) gc:engine.gc = engine.validator.Config; -engine.validator.config_v2 out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) +engine.validator.configV2 out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) validators:(vector engine.validator) collators:(vector engine.collator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index c62bf28ef2a0a6bcd2314d8069c5af25cad09281..413fbd8769514b2bed2ffc52cc34f4679ea70c69 100644 GIT binary patch delta 319 zcmaE`k>$c>mJL0^EXSne_%=@v*5GBC(A5|+d5?m`LECku9oaHnOahZ!+2fX#|&U%&*>)74lvKUs#Dc`6%$&@U%+z9_i7?&FP~9q%1@5wPWh9oR z#ut|)mZVOWpCt=+_vQx&dsG;cH|IWEuNec=1+t~cnN5eow(TIgt%)9Nk0oKYrl*!T Rqgo8N8QqNSCIXCWH~=yFY=r;- delta 171 zcmcbxndQMomJL0^EI)Q6?Akm*Sc8}4K*Ij7llLe{OkTn%vH6X{6gEcR&5i2Sv0x>e zS~ZwNL24PG;Hn%~Ois7K3#rM1T_W6Rnd$LmMhsxXV%irlLG*Mrm~(pH;+YW-nwyOv-uHgUxU~NcV diff --git a/ton/ton-tl.hpp b/ton/ton-tl.hpp index f6bda4d92d..5df54ec65d 100644 --- a/ton/ton-tl.hpp +++ b/ton/ton-tl.hpp @@ -62,12 +62,12 @@ inline tl_object_ptr create_tl_shard_id(const ShardIdF return create_tl_object(s.workchain, s.shard); } -inline tl_object_ptr unpack_engine_validator_config( +inline tl_object_ptr unpack_engine_validator_config( tl_object_ptr config) { - tl_object_ptr res; + tl_object_ptr res; ton_api::downcast_call(*config, td::overloaded( [&](ton_api::engine_validator_config &c) { - res = create_tl_object( + res = create_tl_object( c.out_port_, std::move(c.addrs_), std::move(c.adnl_), std::move(c.dht_), std::move(c.validators_), std::vector>(), c.fullnode_, @@ -75,8 +75,8 @@ inline tl_object_ptr unpack_engine_validato std::move(c.liteservers_), std::move(c.control_), std::vector>(), std::move(c.gc_)); }, - [&](ton_api::engine_validator_config_v2 &c) { - res = std::make_unique(std::move(c)); + [&](ton_api::engine_validator_configV2 &c) { + res = std::make_unique(std::move(c)); })); return res; } diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 65fb0e8f2f..c0f85d0395 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -75,7 +75,7 @@ Config::Config() { full_node = ton::PublicKeyHash::zero(); } -Config::Config(const ton::ton_api::engine_validator_config_v2 &config) { +Config::Config(const ton::ton_api::engine_validator_configV2 &config) { full_node = ton::PublicKeyHash::zero(); out_port = static_cast(config.out_port_); if (!out_port) { @@ -263,7 +263,7 @@ ton::tl_object_ptr Config::tl() const { full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); } else { - return ton::create_tl_object( + return ton::create_tl_object( out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), std::move(control_vec), std::move(shards_vec), std::move(gc_vec)); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 47d612b7ba..b5d770a4e9 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -140,7 +140,7 @@ struct Config { ton::tl_object_ptr tl() const; Config(); - Config(const ton::ton_api::engine_validator_config_v2 &config); + Config(const ton::ton_api::engine_validator_configV2 &config); }; class ValidatorEngine : public td::actor::Actor { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index fea4916dfa..4bd7bf3629 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -46,11 +46,25 @@ namespace validator { namespace fullnode { +static const td::uint32 PROTO_CAPABILITIES_V2 = 2; + Neighbour Neighbour::zero = Neighbour{adnl::AdnlNodeIdShort::zero()}; -void Neighbour::update_proto_version(const ton_api::tonNode_capabilities &q) { - proto_version = q.version_; - capabilities = q.capabilities_; +void Neighbour::update_proto_version(ton_api::tonNode_Capabilities &q) { + ton_api::downcast_call(q, td::overloaded( + [&](ton_api::tonNode_capabilities &x) { + proto_version = x.version_; + capabilities = x.capabilities_; + if (capabilities < PROTO_CAPABILITIES_V2) { + has_state_known = has_state = true; + } + }, + [&](ton_api::tonNode_capabilitiesV2 &x) { + proto_version = x.version_; + capabilities = x.capabilities_; + has_state_known = true; + has_state = x.have_state_; + })); } void Neighbour::query_success(double t) { @@ -97,7 +111,7 @@ void FullNodeShardImpl::create_overlay() { td::actor::ActorId node_; }; - if (active_) { + if (is_active()) { td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay, adnl_id_, overlay_id_full_.clone(), std::make_unique(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() @@ -137,13 +151,16 @@ void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promis create_overlay(); } -void FullNodeShardImpl::set_active(bool active) { - if (active_ == active || shard_.is_masterchain()) { +void FullNodeShardImpl::set_mode(FullNodeShardMode mode) { + if (shard_.is_masterchain()) { return; } - td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); - active_ = active; - create_overlay(); + bool was_active = is_active(); + mode_ = mode; + if (was_active != is_active()) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); + create_overlay(); + } } void FullNodeShardImpl::try_get_next_block(td::Timestamp timeout, td::Promise promise) { @@ -562,6 +579,12 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_value(create_serialize_tl_object(proto_version(), proto_capabilities())); } +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilitiesV2 &query, + td::Promise promise) { + promise.set_value(create_serialize_tl_object(proto_version(), proto_capabilities(), + mode_ == FullNodeShardMode::active)); +} + void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, td::Promise promise) { auto P = td::PromiseCreator::lambda( @@ -609,7 +632,7 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { - if (!active_) { + if (!is_active()) { promise.set_error(td::Status::Error("shard is inactive")); return; } @@ -671,7 +694,7 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_bl } void FullNodeShardImpl::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { - if (!active_) { + if (!is_active()) { return; } auto B = fetch_tl_object(std::move(broadcast), true); @@ -801,7 +824,7 @@ void FullNodeShardImpl::download_block_proof_link(BlockIdExt block_id, td::uint3 td::Promise promise) { auto &b = choose_neighbour(); td::actor::create_actor("downloadproofreq", block_id, true, false, adnl_id_, overlay_id_, - adnl::AdnlNodeIdShort::zero(), priority, timeout, validator_manager_, rldp_, + b.adnl_id, priority, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) .release(); } @@ -809,7 +832,7 @@ void FullNodeShardImpl::download_block_proof_link(BlockIdExt block_id, td::uint3 void FullNodeShardImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) { auto &b = choose_neighbour(); - td::actor::create_actor("next", block_id, 16, adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), + td::actor::create_actor("next", block_id, 16, adnl_id_, overlay_id_, b.adnl_id, 1, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) .release(); @@ -817,10 +840,10 @@ void FullNodeShardImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp t void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) { - auto &b = choose_neighbour(); + auto &b = choose_neighbour(true); td::actor::create_actor( - "archive", masterchain_seqno, std::move(tmp_dir), adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), timeout, - validator_manager_, rldp_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) + "archive", masterchain_seqno, std::move(tmp_dir), adnl_id_, overlay_id_, b.adnl_id, timeout, + validator_manager_, rldp_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise), true)) .release(); } @@ -828,9 +851,9 @@ void FullNodeShardImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardI td::Promise> promise) { // TODO: maybe more complex download (like other requests here) // TODO: estimate max size - auto &b = choose_neighbour(); + auto &b = choose_neighbour(true); auto P = td::PromiseCreator::lambda( - [=, promise = create_neighbour_promise(b, std::move(promise))](td::Result R) mutable { + [=, promise = create_neighbour_promise(b, std::move(promise), true)](td::Result R) mutable { if (R.is_error()) { promise.set_result(R.move_as_error()); return; @@ -1026,11 +1049,21 @@ void FullNodeShardImpl::got_neighbours(std::vector vec) { continue; } if (neighbours_.size() == max_neighbours()) { + td::uint32 neighbours_with_state = 0; + for (const auto &n : neighbours_) { + if (n.second.has_state) { + ++neighbours_with_state; + } + } + adnl::AdnlNodeIdShort a = adnl::AdnlNodeIdShort::zero(); adnl::AdnlNodeIdShort b = adnl::AdnlNodeIdShort::zero(); td::uint32 cnt = 0; double u = 0; for (auto &n : neighbours_) { + if (neighbours_with_state <= min_neighbours_with_state() && n.second.has_state) { + continue; + } if (n.second.unreliability > u) { u = n.second.unreliability; a = n.first; @@ -1054,34 +1087,47 @@ void FullNodeShardImpl::got_neighbours(std::vector vec) { } } -const Neighbour &FullNodeShardImpl::choose_neighbour() const { +const Neighbour &FullNodeShardImpl::choose_neighbour(bool require_state) const { if (neighbours_.size() == 0) { return Neighbour::zero; } - const Neighbour *best = nullptr; - td::uint32 sum = 0; + for (int attempt = 0; attempt < (require_state ? 2 : 1); ++attempt) { + const Neighbour *best = nullptr; + td::uint32 sum = 0; - for (auto &x : neighbours_) { - td::uint32 unr = static_cast(x.second.unreliability); + for (auto &x : neighbours_) { + if (require_state) { + if (attempt == 0 && !x.second.has_state) { + continue; + } + if (attempt == 1 && x.second.has_state_known) { + continue; + } + } + auto unr = static_cast(x.second.unreliability); - if (x.second.proto_version < proto_version()) { - unr += 4; - } else if (x.second.proto_version == proto_version() && x.second.capabilities < proto_capabilities()) { - unr += 2; - } + if (x.second.proto_version < proto_version()) { + unr += 4; + } else if (x.second.proto_version == proto_version() && x.second.capabilities < proto_capabilities()) { + unr += 2; + } - auto f = static_cast(fail_unreliability()); + auto f = static_cast(fail_unreliability()); - if (unr <= f) { - auto w = 1 << (f - unr); - sum += w; - if (td::Random::fast(0, sum - 1) <= w - 1) { - best = &x.second; + if (unr <= f) { + auto w = 1 << (f - unr); + sum += w; + if (td::Random::fast(0, sum - 1) <= w - 1) { + best = &x.second; + } } } + if (best) { + return *best; + } } - return best ? *best : Neighbour::zero; + return Neighbour::zero; } void FullNodeShardImpl::update_neighbour_stats(adnl::AdnlNodeIdShort adnl_id, double t, bool success) { @@ -1100,11 +1146,11 @@ void FullNodeShardImpl::got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id if (it == neighbours_.end()) { return; } - auto F = fetch_tl_object(std::move(data), true); + auto F = fetch_tl_object(std::move(data), true); if (F.is_error()) { it->second.query_failed(); } else { - it->second.update_proto_version(*F.move_as_ok().get()); + it->second.update_proto_version(*F.ok()); it->second.query_success(t); } } @@ -1133,7 +1179,12 @@ void FullNodeShardImpl::ping_neighbours() { td::Time::now() - start_time, R.move_as_ok()); } }); - auto q = create_serialize_tl_object(); + td::BufferSlice q; + if (it->second.capabilities >= PROTO_CAPABILITIES_V2) { + q = create_serialize_tl_object(); + } else { + q = create_serialize_tl_object(); + } td::actor::send_closure(overlays_, &overlay::Overlays::send_query, it->first, adnl_id_, overlay_id_, "get_prepare_block", std::move(P), td::Timestamp::in(1.0), std::move(q)); @@ -1148,7 +1199,7 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, bool active) + td::actor::ActorId client, FullNodeShardMode mode) : shard_(shard) , local_id_(local_id) , adnl_id_(adnl_id) @@ -1159,7 +1210,7 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, , overlays_(overlays) , validator_manager_(validator_manager) , client_(client) - , active_(shard.is_masterchain() || active) { + , mode_(shard.is_masterchain() ? FullNodeShardMode::active : mode) { } td::actor::ActorOwn FullNodeShard::create( @@ -1167,9 +1218,9 @@ td::actor::ActorOwn FullNodeShard::create( td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, - bool active) { + FullNodeShardMode mode) { return td::actor::create_actor("tonnode", shard, local_id, adnl_id, zero_state_file_hash, keyring, - adnl, rldp, overlays, validator_manager, client, active); + adnl, rldp, overlays, validator_manager, client, mode); } } // namespace fullnode diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 7da46af18e..e24873aacd 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -28,6 +28,13 @@ namespace validator { namespace fullnode { +enum FullNodeShardMode { + active, // Node can answer queries about the shard + active_temp, // Like 'active', but queries about shard state are not allowed (only blocks) + inactive // Node is not a part of the overlay (overlay is_external) +}; + + class FullNodeShard : public td::actor::Actor { public: virtual ~FullNodeShard() = default; @@ -36,7 +43,7 @@ class FullNodeShard : public td::actor::Actor { virtual ShardIdFull get_shard_full() const = 0; virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; - virtual void set_active(bool active) = 0; + virtual void set_mode(FullNodeShardMode mode) = 0; virtual void send_ihr_message(td::BufferSlice data) = 0; virtual void send_external_message(td::BufferSlice data) = 0; @@ -74,7 +81,7 @@ class FullNodeShard : public td::actor::Actor { td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, - bool active = true); + FullNodeShardMode mode = FullNodeShardMode::active); }; } // namespace fullnode diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index a9b7618153..cd7c20a95e 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -36,10 +36,12 @@ struct Neighbour { double roundtrip_relax_at = 0; double roundtrip_weight = 0; double unreliability = 0; + bool has_state_known = false; + bool has_state = false; Neighbour(adnl::AdnlNodeIdShort adnl_id) : adnl_id(std::move(adnl_id)) { } - void update_proto_version(const ton_api::tonNode_capabilities &q); + void update_proto_version(ton_api::tonNode_Capabilities &q); void query_success(double t); void query_failed(); void update_roundtrip(double t); @@ -66,11 +68,14 @@ class FullNodeShardImpl : public FullNodeShard { return 2; } static constexpr td::uint64 proto_capabilities() { - return 1; + return 2; } static constexpr td::uint32 max_neighbours() { return 16; } + static constexpr td::uint32 min_neighbours_with_state() { + return 10; + } static constexpr double stop_unreliability() { return 5.0; } @@ -80,11 +85,8 @@ class FullNodeShardImpl : public FullNodeShard { void create_overlay(); void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; - void set_active(bool active) override; + void set_mode(FullNodeShardMode mode) override; - //td::Result fetch_block(td::BufferSlice data); - void prevalidate_block(BlockIdExt block_id, td::BufferSlice data, td::BufferSlice proof, - td::Promise promise); void try_get_next_block(td::Timestamp timestamp, td::Promise promise); void got_next_block(td::Result block); void get_next_block(); @@ -129,6 +131,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilitiesV2 &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, @@ -189,14 +193,16 @@ class FullNodeShardImpl : public FullNodeShard { void got_neighbours(std::vector res); void update_neighbour_stats(adnl::AdnlNodeIdShort adnl_id, double t, bool success); void got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id, double t, td::BufferSlice data); - const Neighbour &choose_neighbour() const; + const Neighbour &choose_neighbour(bool require_state = false) const; template - td::Promise create_neighbour_promise(const Neighbour &x, td::Promise p) { - return td::PromiseCreator::lambda([id = x.adnl_id, SelfId = actor_id(this), p = std::move(p), - ts = td::Time::now()](td::Result R) mutable { + td::Promise create_neighbour_promise(const Neighbour &x, td::Promise p, bool require_state = false) { + return td::PromiseCreator::lambda([id = x.adnl_id, SelfId = actor_id(this), p = std::move(p), ts = td::Time::now(), + ignore_error = require_state && !x.has_state_known](td::Result R) mutable { if (R.is_error() && R.error().code() != ErrorCode::notready && R.error().code() != ErrorCode::cancelled) { - td::actor::send_closure(SelfId, &FullNodeShardImpl::update_neighbour_stats, id, td::Time::now() - ts, false); + if (!ignore_error) { + td::actor::send_closure(SelfId, &FullNodeShardImpl::update_neighbour_stats, id, td::Time::now() - ts, false); + } } else { td::actor::send_closure(SelfId, &FullNodeShardImpl::update_neighbour_stats, id, td::Time::now() - ts, true); } @@ -209,13 +215,17 @@ class FullNodeShardImpl : public FullNodeShard { td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, bool active = true); + td::actor::ActorId client, FullNodeShardMode mode = FullNodeShardMode::active); private: bool use_new_download() const { return false; } + bool is_active() const { + return mode_ != FullNodeShardMode::inactive; + } + ShardIdFull shard_; BlockHandle handle_; td::Promise promise_; @@ -247,7 +257,7 @@ class FullNodeShardImpl : public FullNodeShard { td::Timestamp ping_neighbours_at_; adnl::AdnlNodeIdShort last_pinged_neighbour_ = adnl::AdnlNodeIdShort::zero(); - bool active_ = false; + FullNodeShardMode mode_; }; } // namespace fullnode diff --git a/validator/full-node.cpp b/validator/full-node.cpp index e4a2cf5762..6792f969fc 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -132,14 +132,15 @@ void FullNodeImpl::initial_read_complete(BlockHandle top_handle) { td::actor::send_closure(it->second.actor, &FullNodeShard::set_handle, top_handle, std::move(P)); } -void FullNodeImpl::update_shard_configuration(td::Ref state, - std::set shards_to_monitor) { +void FullNodeImpl::update_shard_configuration(td::Ref state, std::set shards_to_monitor, + std::set temporary_shards) { + CHECK(shards_to_monitor.count(ShardIdFull(masterchainId))); std::map new_shards; - std::set new_active; + std::map new_active; new_shards[ShardIdFull(masterchainId)] = state->get_block_id(); std::set workchains; - auto set_active = [&](ShardIdFull shard) { - while (new_active.insert(shard).second && shard.pfx_len() > 0) { + auto set_active = [&](ShardIdFull shard, FullNodeShardMode mode) { + while (new_active.emplace(shard, mode).second && shard.pfx_len() > 0) { shard = shard_parent(shard); } }; @@ -155,20 +156,24 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, } } for (ShardIdFull shard : shards_to_monitor) { - set_active(shard); + set_active(shard, FullNodeShardMode::active); + } + for (ShardIdFull shard : temporary_shards) { + set_active(shard, FullNodeShardMode::active_temp); } - auto info_set_active = [&](ShardIdFull shard, ShardInfo& info, bool active) { - if (info.active == active) { + auto info_set_mode = [&](ShardIdFull shard, ShardInfo& info, FullNodeShardMode mode) { + if (info.mode == mode) { return; } if (info.actor.empty()) { - add_shard_actor(shard, active); + add_shard_actor(shard, mode); return; } - info.active = active; - td::actor::send_closure(info.actor, &FullNodeShard::set_active, active); - info.delete_at = active ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); + info.mode = mode; + td::actor::send_closure(info.actor, &FullNodeShard::set_mode, mode); + info.delete_at = + mode != FullNodeShardMode::inactive ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); }; for (auto shard : new_shards) { @@ -180,16 +185,17 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, ShardIdFull shard = p.first; ShardInfo &info = p.second; info.exists = new_shards.count(shard); - info_set_active(shard, info, new_active.count(shard)); + auto it = new_active.find(shard); + info_set_mode(shard, info, it == new_active.end() ? FullNodeShardMode::inactive : it->second); } - for (ShardIdFull shard : new_active) { - info_set_active(shard, shards_[shard], true); + for (const auto& s : new_active) { + info_set_mode(s.first, shards_[s.first], s.second); } auto it = shards_.begin(); while (it != shards_.end()) { - if (!it->second.active && it->second.delete_at && it->second.delete_at.is_in_past()) { + if (it->second.mode == FullNodeShardMode::inactive && it->second.delete_at && it->second.delete_at.is_in_past()) { it->second.actor.reset(); it->second.delete_at = td::Timestamp::never(); } @@ -201,15 +207,15 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, } } -void FullNodeImpl::add_shard_actor(ShardIdFull shard, bool active) { +void FullNodeImpl::add_shard_actor(ShardIdFull shard, FullNodeShardMode mode) { ShardInfo &info = shards_[shard]; if (!info.actor.empty()) { return; } info.actor = FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, keyring_, adnl_, rldp_, - overlays_, validator_manager_, client_, active); - info.active = active; - info.delete_at = active ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); + overlays_, validator_manager_, client_, mode); + info.mode = mode; + info.delete_at = mode != FullNodeShardMode::inactive ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); if (all_validators_.size() > 0) { td::actor::send_closure(info.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } @@ -350,9 +356,9 @@ td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard, boo auto it = shards_.find(s); if (it != shards_.end() && it->second.exists) { if (it->second.actor.empty()) { - add_shard_actor(s, false); + add_shard_actor(s, FullNodeShardMode::inactive); } - if (!it->second.active) { + if (it->second.mode == FullNodeShardMode::inactive) { it->second.delete_at = td::Timestamp::in(INACTIVE_SHARD_TTL); } return it->second.actor.get(); @@ -365,9 +371,9 @@ td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard, boo } auto &info = shards_[shard]; if (info.actor.empty()) { - add_shard_actor(shard, false); + add_shard_actor(shard, FullNodeShardMode::inactive); } - if (!info.active) { + if (info.mode == FullNodeShardMode::inactive) { info.delete_at = td::Timestamp::in(INACTIVE_SHARD_TTL); } return info.actor.get(); @@ -488,9 +494,10 @@ void FullNodeImpl::start_up() { void initial_read_complete(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::initial_read_complete, handle); } - void update_shard_configuration(td::Ref state, std::set shards_to_monitor) override { + void update_shard_configuration(td::Ref state, std::set shards_to_monitor, + std::set temporary_shards) override { td::actor::send_closure(id_, &FullNodeImpl::update_shard_configuration, std::move(state), - std::move(shards_to_monitor)); + std::move(shards_to_monitor), std::move(temporary_shards)); } void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) override { td::actor::send_closure(id_, &FullNodeImpl::send_ihr_message, dst, std::move(data)); @@ -576,7 +583,7 @@ FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id , client_(client) , db_root_(db_root) , started_promise_(std::move(started_promise)) { - add_shard_actor(ShardIdFull{masterchainId}, true); + add_shard_actor(ShardIdFull{masterchainId}, FullNodeShardMode::active); } td::actor::ActorOwn FullNode::create( diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 2f82545878..fbe60d1b22 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -52,7 +52,8 @@ class FullNodeImpl : public FullNode { void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; - void update_shard_configuration(td::Ref state, std::set shards_to_monitor); + void update_shard_configuration(td::Ref state, std::set shards_to_monitor, + std::set temporary_shards); void sync_completed(); @@ -91,7 +92,7 @@ class FullNodeImpl : public FullNode { td::Promise started_promise); private: - void add_shard_actor(ShardIdFull shard, bool active); + void add_shard_actor(ShardIdFull shard, FullNodeShardMode mode); PublicKeyHash local_id_; adnl::AdnlNodeIdShort adnl_id_; @@ -103,7 +104,7 @@ class FullNodeImpl : public FullNode { struct ShardInfo { bool exists = false; td::actor::ActorOwn actor; - bool active = false; + FullNodeShardMode mode = FullNodeShardMode::inactive; td::Timestamp delete_at = td::Timestamp::never(); }; std::map shards_; diff --git a/validator/manager.cpp b/validator/manager.cpp index 04edefdc63..7e7e1e8404 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2453,8 +2453,7 @@ void ValidatorManagerImpl::get_shard_client_state(bool from_db, td::Promise state, std::set shards_to_monitor) { shards_to_monitor_ = shards_to_monitor; - shards_to_monitor.insert(extra_active_shards_.begin(), extra_active_shards_.end()); - callback_->update_shard_configuration(std::move(state), std::move(shards_to_monitor)); + callback_->update_shard_configuration(std::move(state), std::move(shards_to_monitor), extra_active_shards_); } void ValidatorManagerImpl::update_async_serializer_state(AsyncSerializerState state, td::Promise promise) { diff --git a/validator/validator.h b/validator/validator.h index 545c8a81c7..0595949fb9 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -110,7 +110,8 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void initial_read_complete(BlockHandle top_masterchain_blocks) = 0; virtual void update_shard_configuration(td::Ref state, - std::set shards_to_monitor) = 0; + std::set shards_to_monitor, + std::set temporary_shards) = 0; virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; From 597fd8443d877eae070aaebce85c22e2e4928730 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 12 Aug 2022 16:14:03 +0300 Subject: [PATCH 022/524] Collator nodes preload msg queues --- validator/collator-node.cpp | 52 +++++++++++++++++++++++++++++++------ validator/collator-node.hpp | 4 +++ validator/manager.cpp | 23 +++++++++++++++- validator/manager.hpp | 9 ++++++- 4 files changed, 78 insertions(+), 10 deletions(-) diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 84ef9eaa82..c11b01c824 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -58,6 +58,40 @@ void CollatorNode::add_shard(ShardIdFull shard) { shards_.push_back(shard); } +void CollatorNode::new_masterchain_block_notification(td::Ref state) { + std::vector top_blocks = {state->get_block_id()}; + std::vector next_shards; + if (collate_shard(ShardIdFull(masterchainId))) { + next_shards.push_back(ShardIdFull(masterchainId)); + } + for (const auto& desc : state->get_shards()) { + top_blocks.push_back(desc->top_block_id()); + ShardIdFull shard = desc->shard(); + if (desc->before_split()) { + if (collate_shard(shard_child(shard, true))) { + next_shards.push_back(shard_child(shard, true)); + } + if (collate_shard(shard_child(shard, false))) { + next_shards.push_back(shard_child(shard, false)); + } + } else if (desc->before_merge()) { + if (is_left_child(shard) && collate_shard(shard_parent(shard))) { + next_shards.push_back(shard_parent(shard)); + } + } else if (collate_shard(shard)) { + next_shards.push_back(shard); + } + } + for (const ShardIdFull& shard : next_shards) { + for (const BlockIdExt& neighbor : top_blocks) { + if (neighbor.shard_full() != shard && block::ShardConfig::is_neighbor(shard, neighbor.shard_full())) { + td::actor::send_closure(manager_, &ValidatorManager::wait_out_msg_queue_proof, neighbor, shard, 0, + td::Timestamp::in(10.0), [](td::Ref) {}); + } + } + } +} + static td::BufferSlice serialize_error(td::Status error) { return create_serialize_tl_object(error.code(), error.message().c_str()); } @@ -71,14 +105,7 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data if (!shard.is_valid_ext()) { return td::Status::Error(PSTRING() << "invalid shard " << shard.to_str()); } - bool found = false; - for (ShardIdFull our_shard : shards_) { - if (shard_is_ancestor(shard, our_shard)) { - found = true; - break; - } - } - if (!found) { + if (!collate_shard(shard)) { return td::Status::Error(PSTRING() << "this node doesn't collate shard " << shard.to_str()); } if (f->prev_blocks_.size() != 1 && f->prev_blocks_.size() != 2) { @@ -141,6 +168,15 @@ void CollatorNode::receive_query_cont(adnl::AdnlNodeIdShort src, ShardIdFull sha min_mc_state->get_validator_set(shard), manager_, td::Timestamp::in(10.0), std::move(P)); } +bool CollatorNode::collate_shard(ShardIdFull shard) const { + for (ShardIdFull our_shard : shards_) { + if (shard_is_ancestor(shard, our_shard)) { + return true; + } + } + return false; +} + } // namespace validator } // namespace ton diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index 854e98ce2a..b0a2ad9037 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -33,12 +33,16 @@ class CollatorNode : public td::actor::Actor { void tear_down() override; void add_shard(ShardIdFull shard); + void new_masterchain_block_notification(td::Ref state); + private: void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); void receive_query_cont(adnl::AdnlNodeIdShort src, ShardIdFull shard, td::Ref min_mc_state, std::vector prev_blocks, Ed25519_PublicKey creator, td::Promise promise); + bool collate_shard(ShardIdFull shard) const; + adnl::AdnlNodeIdShort local_id_; td::actor::ActorId manager_; td::actor::ActorId adnl_; diff --git a/validator/manager.cpp b/validator/manager.cpp index 7e7e1e8404..6323c4f2f3 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -617,6 +617,9 @@ void ValidatorManagerImpl::wait_out_msg_queue_proof(BlockIdExt block_id, ShardId .release(); wait_out_msg_queue_proof_[key].actor_ = id; it = wait_out_msg_queue_proof_.find(key); + } else if (it->second.done_) { + promise.set_result(it->second.result_); + it->second.remove_at_ = td::Timestamp::in(30.0); } it->second.waiting_.emplace_back(timeout, priority, std::move(promise)); @@ -1079,13 +1082,16 @@ void ValidatorManagerImpl::finished_wait_msg_queue(BlockIdExt block_id, ShardIdF it->second.actor_ = id; return; } + wait_out_msg_queue_proof_.erase(it); } else { auto r = R.move_as_ok(); for (auto &X : it->second.waiting_) { X.promise.set_result(r); } + it->second.done_ = true; + it->second.result_ = std::move(r); + it->second.remove_at_ = td::Timestamp::in(30.0); } - wait_out_msg_queue_proof_.erase(it); } } @@ -1773,6 +1779,9 @@ void ValidatorManagerImpl::new_masterchain_block() { td::actor::send_closure(shard_client_, &ShardClient::new_masterchain_block_notification, last_masterchain_block_handle_, last_masterchain_state_); } + for (auto &c : collator_nodes_) { + td::actor::send_closure(c.second, &CollatorNode::new_masterchain_block_notification, last_masterchain_state_); + } if (last_masterchain_seqno_ % 1024 == 0) { LOG(WARNING) << "applied masterchain block " << last_masterchain_block_id_; @@ -2436,6 +2445,18 @@ void ValidatorManagerImpl::alarm() { } } alarm_timestamp().relax(check_shard_clients_); + if (cleanup_wait_caches_at_.is_in_past()) { + auto it = wait_out_msg_queue_proof_.begin(); + while (it != wait_out_msg_queue_proof_.end()) { + if (it->second.done_ && it->second.remove_at_.is_in_past()) { + it = wait_out_msg_queue_proof_.erase(it); + } else { + ++it; + } + } + cleanup_wait_caches_at_ = td::Timestamp::in(10.0); + } + alarm_timestamp().relax(cleanup_wait_caches_at_); } void ValidatorManagerImpl::update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) { diff --git a/validator/manager.hpp b/validator/manager.hpp index c65914aae7..15598f4cd7 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -181,10 +181,17 @@ class ValidatorManagerImpl : public ValidatorManager { waiting_.resize(j); } }; + template + struct WaitListCaching : public WaitList { + bool done_ = false; + ResType result_; + td::Timestamp remove_at_; + }; std::map>> wait_state_; std::map>> wait_block_data_; - std::map, WaitList>> + std::map, WaitListCaching>> wait_out_msg_queue_proof_; + td::Timestamp cleanup_wait_caches_at_ = td::Timestamp::now(); struct WaitBlockHandle { std::vector> waiting_; From 81d32ba5d64ffb6ab5bea99fb0ec794c2d93ebfd Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 15 Aug 2022 13:59:49 +0300 Subject: [PATCH 023/524] Out msg queue proof: send only the required part --- validator/full-node-shard.cpp | 4 ++ validator/impl/out-msg-queue-proof.cpp | 75 ++++++++++++++++++++++---- 2 files changed, 68 insertions(+), 11 deletions(-) diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 4bd7bf3629..7bf44f33b6 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -852,6 +852,10 @@ void FullNodeShardImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardI // TODO: maybe more complex download (like other requests here) // TODO: estimate max size auto &b = choose_neighbour(true); + if (b.adnl_id == adnl::AdnlNodeIdShort::zero()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "no nodes")); + return; + } auto P = td::PromiseCreator::lambda( [=, promise = create_neighbour_promise(b, std::move(promise), true)](td::Result R) mutable { if (R.is_error()) { diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 4a8731d39c..4ba102c538 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -20,6 +20,8 @@ #include "vm/cells/MerkleProof.h" #include "common/delay.h" #include "interfaces/validator-manager.h" +#include "block/block-parse.h" +#include "block/block-auto.h" namespace ton { @@ -51,9 +53,31 @@ td::Result> OutMsgQueueProof::fetch(BlockIdExt block_i // Validate proof auto state_root = vm::CellSlice(vm::NoVm(), queue_proof).prefetch_ref(0); TRY_RESULT_PREFIX(state, ShardStateQ::fetch(block_id, {}, state_root), "invalid proof: "); - TRY_RESULT_PREFIX(queue, state->message_queue(), "invalid proof: "); - auto queue_root = queue->root_cell(); - if (queue_root->get_level() != 0) { + TRY_RESULT_PREFIX(outq_descr, state->message_queue(), "invalid proof: "); + + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { + return td::Status::Error("invalid proof: invalid message queue"); + } + td::Ref proc_info = qinfo.proc_info->prefetch_ref(0); + if (proc_info.not_null() && proc_info->get_level() != 0) { + return td::Status::Error("invalid proof: proc_info has prunned branches"); + } + td::Ref ihr_pending = qinfo.ihr_pending->prefetch_ref(0); + if (ihr_pending.not_null() && ihr_pending->get_level() != 0) { + return td::Status::Error("invalid proof: ihr_pending has prunned branches"); + } + auto queue = + std::make_unique(qinfo.out_queue->prefetch_ref(0), 352, block::tlb::aug_OutMsgQueue); + td::BitArray<96> prefix; + td::BitPtr ptr = prefix.bits(); + ptr.store_int(dst_shard.workchain, 32); + ptr.advance(32); + ptr.store_uint(dst_shard.shard, 64); + if (!queue->cut_prefix_subdict(prefix.bits(), 32 + dst_shard.pfx_len())) { + return td::Status::Error("invalid proof: failed to cut queue dict"); + } + if (queue->get_root_cell().not_null() && queue->get_root_cell()->get_level() != 0) { return td::Status::Error("invalid proof: msg queue has prunned branches"); } @@ -64,14 +88,20 @@ td::Result> OutMsgQueueProof::s ShardIdFull dst_shard, Ref state_root, Ref block_root) { + if (!dst_shard.is_valid_ext()) { + return td::Status::Error("invalid shard"); + } vm::MerkleProofBuilder mpb{std::move(state_root)}; TRY_RESULT(state, ShardStateQ::fetch(block_id, {}, mpb.root())); TRY_RESULT(outq_descr, state->message_queue()); + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { + return td::Status::Error("invalid message queue"); + } - // TODO: add only required part of msg queue td::HashSet visited; std::function)> dfs = [&](Ref cell) { - if (!visited.insert(cell->get_hash()).second) { + if (cell.is_null() || !visited.insert(cell->get_hash()).second) { return; } vm::CellSlice cs(vm::NoVm(), cell); @@ -79,16 +109,32 @@ td::Result> OutMsgQueueProof::s dfs(cs.prefetch_ref(i)); } }; - dfs(outq_descr->root_cell()); + auto dfs_cs = [&](const vm::CellSlice &cs) { + for (unsigned i = 0; i < cs.size_refs(); i++) { + dfs(cs.prefetch_ref(i)); + } + }; + dfs_cs(*qinfo.proc_info); + dfs_cs(*qinfo.ihr_pending); - TRY_RESULT(queue_proof, vm::std_boc_serialize(mpb.extract_proof())); + auto queue = + std::make_unique(qinfo.out_queue->prefetch_ref(0), 352, block::tlb::aug_OutMsgQueue); + td::BitArray<96> prefix; + td::BitPtr ptr = prefix.bits(); + ptr.store_int(dst_shard.workchain, 32); + ptr.advance(32); + ptr.store_uint(dst_shard.shard, 64); + if (!queue->cut_prefix_subdict(prefix.bits(), 32 + dst_shard.pfx_len())) { + return td::Status::Error("invalid message queue"); + } + dfs(queue->get_root_cell()); + TRY_RESULT(queue_proof, vm::std_boc_serialize(mpb.extract_proof())); td::BufferSlice block_state_proof; if (block_id.seqno() != 0) { TRY_RESULT(proof, create_block_state_proof(std::move(block_root))); TRY_RESULT_ASSIGN(block_state_proof, vm::std_boc_serialize(std::move(proof), 31)); } - return create_tl_object(std::move(queue_proof), std::move(block_state_proof)); } @@ -183,7 +229,11 @@ void WaitOutMsgQueueProof::run_net() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = block_id_](td::Result> R) { if (R.is_error()) { - LOG(DEBUG) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); + if (R.error().code() == ErrorCode::notready) { + LOG(DEBUG) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); + } else { + LOG(WARNING) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); + } delay_action([SelfId]() mutable { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::run_net); }, td::Timestamp::in(0.1)); } else { @@ -246,8 +296,11 @@ void BuildOutMsgQueueProof::got_block_root(Ref root) { } void BuildOutMsgQueueProof::build_proof() { - promise_.set_result( - OutMsgQueueProof::serialize(block_id_, dst_shard_, std::move(state_root_), std::move(block_root_))); + auto result = OutMsgQueueProof::serialize(block_id_, dst_shard_, std::move(state_root_), std::move(block_root_)); + if (result.is_error()) { + LOG(ERROR) << "Failed to build msg queue proof: " << result.error(); + } + promise_.set_result(std::move(result)); stop(); } From 011e97f53c1610ead70e59f662f14d4a7be268d6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 15 Aug 2022 19:27:19 +0300 Subject: [PATCH 024/524] Flag "full collated data" in mc config; fix accept-block --- crypto/block/block.tlb | 8 ++++---- crypto/block/mc-config.cpp | 24 ++++++++++++++++++++++++ crypto/block/mc-config.h | 11 +++++++++++ ton/ton-types.h | 5 ----- validator/fabric.h | 2 +- validator/impl/accept-block.cpp | 2 ++ validator/impl/accept-block.hpp | 1 + validator/impl/collator-impl.h | 1 + validator/impl/collator.cpp | 5 +++++ validator/impl/shard.cpp | 21 --------------------- validator/impl/shard.hpp | 4 +++- validator/impl/validate-query.cpp | 16 ++++++++-------- validator/impl/validate-query.hpp | 6 +++--- validator/interfaces/shard.h | 2 +- validator/manager.cpp | 8 ++++---- validator/validator-group.cpp | 9 +++++---- validator/validator-group.hpp | 6 +++--- 17 files changed, 76 insertions(+), 55 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index c9a95535f9..8cb6c2f0d5 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -734,15 +734,15 @@ misbehaviour_punishment_config_v1#01 = MisbehaviourPunishmentConfig; _ MisbehaviourPunishmentConfig = ConfigParam 40; +// collator_nodes: each collator is (workchain:int32 shard:uint64 adnl_id:uint256) +colator_config#a0 full_collated_data:Bool collator_nodes:(HashmapE 352 Unit) = CollatorConfig; +_ CollatorConfig = ConfigParam 41; + oracle_bridge_params#_ bridge_address:bits256 oracle_mutlisig_address:bits256 oracles:(HashmapE 256 uint256) external_chain_address:bits256 = OracleBridgeParams; _ OracleBridgeParams = ConfigParam 71; // Ethereum bridge _ OracleBridgeParams = ConfigParam 72; // Binance Smart Chain bridge _ OracleBridgeParams = ConfigParam 73; // Polygon bridge -// Set of collators: each collator is (workchain:int32 shard:int64 adnl_id:int256) -colator_set#a0 collators:(HashmapE 352 Unit) = CollatorSet; -_ CollatorSet = ConfigParam 81; - // // PROOFS // diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index b5409a6204..a1ddfb3ef7 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2132,4 +2132,28 @@ Ref ConfigInfo::lookup_library(td::ConstBitPtr root_hash) const { return lib; } +CollatorConfig Config::get_collator_config(bool need_collator_nodes) const { + CollatorConfig collator_config; + gen::CollatorConfig::Record rec; + auto cell = get_config_param(41); + if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { + return collator_config; + } + collator_config.full_collated_data = rec.full_collated_data; + if (need_collator_nodes) { + vm::Dictionary dict{rec.collator_nodes->prefetch_ref(), 32 + 64 + 256}; + dict.check_for_each([&](Ref, td::ConstBitPtr key, int n) { + CHECK(n == 32 + 64 + 256); + auto workchain = (td::int32)key.get_int(32); + key.advance(32); + td::uint64 shard = key.get_uint(64); + key.advance(64); + td::Bits256 adnl_id(key); + collator_config.collator_nodes.push_back({ton::ShardIdFull(workchain, shard), adnl_id}); + return true; + }); + } + return collator_config; +} + } // namespace block diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 8be3f6e6fb..7058117ba9 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -482,6 +482,16 @@ class ShardConfig { bool set_shard_info(ton::ShardIdFull shard, Ref value); }; +struct CollatorNodeDescr { + ton::ShardIdFull shard; + ton::NodeIdShort adnl_id; +}; + +struct CollatorConfig { + bool full_collated_data = false; + std::vector collator_nodes; +}; + class Config { enum { default_mc_catchain_lifetime = 200, @@ -593,6 +603,7 @@ class Config { std::vector compute_validator_set(ton::ShardIdFull shard, ton::UnixTime time, ton::CatchainSeqno cc_seqno) const; std::vector compute_total_validator_set(int next) const; + CollatorConfig get_collator_config(bool need_collator_nodes) const; static std::vector do_compute_validator_set(const block::CatchainValidatorsConfig& ccv_conf, ton::ShardIdFull shard, const block::ValidatorSet& vset, ton::UnixTime time, diff --git a/ton/ton-types.h b/ton/ton-types.h index 7d0101d10b..36c137b91b 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -484,9 +484,4 @@ struct ValidatorSessionConfig { static const td::uint32 BLOCK_HASH_COVERS_DATA_FROM_VERSION = 2; }; -struct CollatorNodeDescr { - ShardIdFull shard; - NodeIdShort adnl_id; -}; - } // namespace ton diff --git a/validator/fabric.h b/validator/fabric.h index 12452ec55b..069a0470bc 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -25,7 +25,7 @@ namespace ton { namespace validator { -enum ValidateMode { fake = 1, lite = 2 }; +enum ValidateMode { fake = 1, full_collated_data = 2 }; td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_); td::actor::ActorOwn create_liteserver_cache_actor(td::actor::ActorId manager, diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index 6491a71167..c3e20c0e08 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -144,6 +144,7 @@ bool AcceptBlockQuery::precheck_header() { if (is_fork_ && !info.key_block) { return fatal_error("fork block is not a key block"); } + before_split_ = info.before_split; return true; } @@ -538,6 +539,7 @@ void AcceptBlockQuery::written_state(td::Ref upd_state) { } //handle_->set_masterchain_block(prev_[0]); + handle_->set_split(before_split_); handle_->set_state_root_hash(state_hash_); handle_->set_logical_time(lt_); handle_->set_unix_time(created_at_); diff --git a/validator/impl/accept-block.hpp b/validator/impl/accept-block.hpp index 1a1f2a543a..e0e27ec20b 100644 --- a/validator/impl/accept-block.hpp +++ b/validator/impl/accept-block.hpp @@ -115,6 +115,7 @@ class AcceptBlockQuery : public td::actor::Actor { UnixTime created_at_; RootHash state_keep_old_hash_, state_old_hash_, state_hash_; BlockIdExt mc_blkid_, prev_mc_blkid_; + bool before_split_; Ref last_mc_state_; BlockIdExt last_mc_id_; diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index c19199c827..b886173a78 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -128,6 +128,7 @@ class Collator final : public td::actor::Actor { void alarm() override; int verbosity{3 * 0}; int verify{1}; + bool full_collated_data_ = false; ton::LogicalTime start_lt, max_lt; ton::UnixTime now_; ton::UnixTime prev_now_; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index e116a93583..3468bc5ded 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -572,6 +572,8 @@ bool Collator::unpack_last_mc_state() { << " have been enabled in global configuration, but we support only " << supported_version() << " (upgrade validator software?)"; } + full_collated_data_ = config_->get_collator_config(false).full_collated_data; + LOG(DEBUG) << "full_collated_data is " << full_collated_data_; // TODO: extract start_lt and end_lt from prev_mc_block as well // std::cerr << " block::gen::ShardState::print_ref(mc_state_root) = "; // block::gen::t_ShardState.print_ref(std::cerr, mc_state_root, 2); @@ -3982,6 +3984,9 @@ bool Collator::create_collated_data() { } collated_roots_.push_back(std::move(cell)); } + if (!full_collated_data_) { + return true; + } // 2. Proofs for hashes of states: previous states + neighbors for (const auto& p : block_state_proofs_) { collated_roots_.push_back(p.second); diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index dbf2440029..019ecbeb02 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -511,27 +511,6 @@ bool MasterchainStateQ::check_old_mc_block_id(const ton::BlockIdExt& blkid, bool return config_ && config_->check_old_mc_block_id(blkid, strict); } -std::vector MasterchainStateQ::get_collator_set() const { - block::gen::CollatorSet::Record rec; - auto cell = config_->get_config_param(81); - if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { - return {}; - } - vm::Dictionary dict{rec.collators->prefetch_ref(), 32 + 64 + 256}; - std::vector collators; - dict.check_for_each([&](Ref, td::ConstBitPtr key, int n) { - CHECK(n == 32 + 64 + 256); - auto workchain = (td::int32)key.get_int(32); - key.advance(32); - td::uint64 shard = key.get_uint(64); - key.advance(64); - td::Bits256 adnl_id(key); - collators.push_back({ShardIdFull(workchain, shard), adnl_id}); - return true; - }); - return collators; -} - td::uint32 MasterchainStateQ::min_split_depth(WorkchainId workchain_id) const { if (!config_) { return 0; diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index 2b411d9442..6b95c7e409 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -150,7 +150,9 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { block::WorkchainSet get_workchain_list() const override { return config_ ? config_->get_workchain_list() : block::WorkchainSet(); } - std::vector get_collator_set() const override; + block::CollatorConfig get_collator_config(bool need_collator_nodes) const override { + return config_ ? config_->get_collator_config(need_collator_nodes) : block::CollatorConfig(); + } private: ZeroStateIdExt zerostate_id_; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 369f1f2a8b..eeb2726c66 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -63,10 +63,9 @@ ValidateQuery::ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block , timeout(timeout) , main_promise(std::move(promise)) , is_fake_(mode & ValidateMode::fake) - , is_lite_(mode & ValidateMode::lite) + , full_collated_data_(mode & ValidateMode::full_collated_data) , shard_pfx_(shard_.shard) , shard_pfx_len_(ton::shard_prefix_length(shard_)) { - proc_hash_.zero(); } void ValidateQuery::alarm() { @@ -158,6 +157,7 @@ void ValidateQuery::finish_query() { void ValidateQuery::start_up() { LOG(INFO) << "validate query for " << block_candidate.id.to_str() << " started"; + LOG(DEBUG) << "full_collated_data is " << full_collated_data_; alarm_timestamp() = timeout; rand_seed_.set_zero(); created_by_ = block_candidate.pubkey; @@ -258,9 +258,9 @@ void ValidateQuery::start_up() { td::actor::send_closure_later( std::move(self), &ValidateQuery::after_get_latest_mc_state, std::move(res)); }); - // 3. load state(s) corresponding to previous block(s) (non-lite mode or masterchain) + // 3. load state(s) corresponding to previous block(s) (not full-collaoted-data or masterchain) prev_states.resize(prev_blocks.size()); - if (is_masterchain() || !is_lite_) { + if (is_masterchain() || !full_collated_data_) { for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { // 3.1. load state LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; @@ -991,8 +991,8 @@ bool ValidateQuery::check_this_shard_mc_info() { */ bool ValidateQuery::compute_prev_state() { - if (!is_masterchain() && is_lite_) { - return compute_prev_state_lite_mode(); + if (!is_masterchain() && full_collated_data_) { + return compute_prev_state_from_collated_data(); } CHECK(prev_states.size() == 1u + after_merge_); prev_state_root_ = prev_states[0]->root_cell(); @@ -1012,7 +1012,7 @@ bool ValidateQuery::compute_prev_state() { return true; } -bool ValidateQuery::compute_prev_state_lite_mode() { +bool ValidateQuery::compute_prev_state_from_collated_data() { td::Bits256 state_hash; if (id_.seqno() == 1) { if (prev_blocks.size() != 1) { @@ -1255,7 +1255,7 @@ bool ValidateQuery::request_neighbor_queues() { neighbors_.emplace_back(*shard_ptr); } int i = 0; - if (is_lite_) { + if (full_collated_data_) { for (block::McShardDescr& descr : neighbors_) { LOG(DEBUG) << "getting outbound queue of neighbor #" << i << " from collated data : " << descr.blk_.to_str(); td::Bits256 state_root_hash; diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 6fe8010f5b..48c20e6ceb 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -141,7 +141,7 @@ class ValidateQuery : public td::actor::Actor { bool is_key_block_{false}; bool update_shard_cc_{false}; bool is_fake_{false}; - bool is_lite_{false}; + bool full_collated_data_{false}; bool prev_key_block_exists_{false}; bool debug_checks_{false}; bool outq_cleanup_partial_{false}; @@ -221,7 +221,7 @@ class ValidateQuery : public td::actor::Actor { td::RefInt256 import_fees_; ton::LogicalTime proc_lt_{0}, claimed_proc_lt_{0}, min_enq_lt_{~0ULL}; - ton::Bits256 proc_hash_, claimed_proc_hash_, min_enq_hash_; + ton::Bits256 proc_hash_ = ton::Bits256::zero(), claimed_proc_hash_, min_enq_hash_; bool inbound_queues_empty_{false}; std::vector> msg_proc_lt_; @@ -286,7 +286,7 @@ class ValidateQuery : public td::actor::Actor { bool extract_collated_data(); bool try_validate(); bool compute_prev_state(); - bool compute_prev_state_lite_mode(); + bool compute_prev_state_from_collated_data(); bool compute_next_state(); bool unpack_merge_prev_state(); bool unpack_prev_state(); diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index ff458d604c..15caa7b270 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -82,7 +82,7 @@ class MasterchainState : virtual public ShardState { virtual bool check_old_mc_block_id(const ton::BlockIdExt& blkid, bool strict = false) const = 0; virtual td::Result> get_config_holder() const = 0; virtual block::WorkchainSet get_workchain_list() const = 0; - virtual std::vector get_collator_set() const = 0; + virtual block::CollatorConfig get_collator_config(bool need_collator_nodes) const = 0; virtual td::Status prepare() { return td::Status::OK(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 6323c4f2f3..10f0ca488a 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2117,10 +2117,10 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto validator_id = get_validator(shard, validator_set); CHECK(!validator_id.is_zero()); auto G = td::actor::create_actor( - "validatorgroup", shard, validator_id, session_id, validator_set, last_masterchain_state_->get_collator_set(), - opts, keyring_, adnl_, rldp_, overlays_, - db_root_, actor_id(this), init_session, - opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_->validator_lite_mode()); + "validatorgroup", shard, validator_id, session_id, validator_set, + last_masterchain_state_->get_collator_config(true), opts, keyring_, adnl_, rldp_, overlays_, db_root_, + actor_id(this), init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), + opts_->validator_lite_mode()); return G; } } diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index a72bded100..88a473ff63 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -42,7 +42,7 @@ void ValidatorGroup::generate_block_candidate(td::uint32 round_id, td::Promise> promise); ValidatorGroup(ShardIdFull shard, PublicKeyHash local_id, ValidatorSessionId session_id, - td::Ref validator_set, std::vector collator_set, + td::Ref validator_set, block::CollatorConfig collator_config, validatorsession::ValidatorSessionOptions config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, @@ -70,7 +70,7 @@ class ValidatorGroup : public td::actor::Actor { , local_id_(std::move(local_id)) , session_id_(session_id) , validator_set_(std::move(validator_set)) - , collator_set_(std::move(collator_set)) + , collator_config_(std::move(collator_config)) , config_(std::move(config)) , keyring_(keyring) , adnl_(adnl) @@ -110,7 +110,7 @@ class ValidatorGroup : public td::actor::Actor { BlockIdExt min_masterchain_block_id_; td::Ref validator_set_; - std::vector collator_set_; + block::CollatorConfig collator_config_; validatorsession::ValidatorSessionOptions config_; td::actor::ActorId keyring_; From be2169e523398564fbb112b85bf124ec2b142e81 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 16 Aug 2022 13:54:15 +0300 Subject: [PATCH 025/524] Reload validator options when needed --- validator-engine/validator-engine.cpp | 20 +++++++++++++++++--- validator-engine/validator-engine.hpp | 2 +- validator/manager-disk.hpp | 4 ++++ validator/manager-hardfork.hpp | 3 +++ validator/manager.cpp | 10 ++++++++++ validator/manager.hpp | 1 + validator/shard-client.cpp | 4 ++++ validator/shard-client.hpp | 2 ++ validator/state-serializer.hpp | 4 ++++ validator/validator.h | 1 + 10 files changed, 47 insertions(+), 4 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index c0f85d0395..e56dfb7be1 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1404,7 +1404,7 @@ td::Status ValidatorEngine::load_global_config() { return td::Status::OK(); } -void ValidatorEngine::init_validator_options() { +void ValidatorEngine::set_shard_check_function() { if (!not_all_shards_) { validator_options_.write().set_shard_check_function([](ton::ShardIdFull shard) -> bool { return true; }); } else { @@ -1711,7 +1711,7 @@ void ValidatorEngine::got_key(ton::PublicKey key) { } void ValidatorEngine::start() { - init_validator_options(); + set_shard_check_function(); read_config_ = true; start_adnl(); } @@ -3412,7 +3412,10 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollat promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); return; } + set_shard_check_function(); if (!validator_manager_.empty()) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, ton::adnl::AdnlNodeIdShort(id), shard); } @@ -3443,7 +3446,18 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addShard promise.set_value(create_control_query_error(R.move_as_error())); return; } - promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + set_shard_check_function(); + if (!validator_manager_.empty()) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + } + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); } void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index b5d770a4e9..40a04c703d 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -283,7 +283,7 @@ class ValidatorEngine : public td::actor::Actor { void load_empty_local_config(td::Promise promise); void load_local_config(td::Promise promise); void load_config(td::Promise promise); - void init_validator_options(); + void set_shard_check_function(); void start(); diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 53467ff323..ea443d4667 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -386,6 +386,10 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void update_options(td::Ref opts) override { + opts_ = std::move(opts); + } + private: PublicKeyHash local_id_; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 9aa302263b..ce92f8a7b3 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -445,6 +445,9 @@ class ValidatorManagerImpl : public ValidatorManager { void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { UNREACHABLE(); } + void update_options(td::Ref opts) override { + opts_ = std::move(opts); + } private: td::Ref opts_; diff --git a/validator/manager.cpp b/validator/manager.cpp index 10f0ca488a..710edec567 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2696,6 +2696,16 @@ void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh td::actor::send_closure(it->second, &CollatorNode::add_shard, shard); } +void ValidatorManagerImpl::update_options(td::Ref opts) { + if (!shard_client_.empty()) { + td::actor::send_closure(shard_client_, &ShardClient::update_options, opts); + } + if (!serializer_.empty()) { + td::actor::send_closure(serializer_, &AsyncStateSerializer::update_options, opts); + } + opts_ = std::move(opts); +} + td::actor::ActorOwn ValidatorManagerFactory::create( td::Ref opts, std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, diff --git a/validator/manager.hpp b/validator/manager.hpp index 15598f4cd7..d61351f29b 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -556,6 +556,7 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; + void update_options(td::Ref opts) override; private: td::Timestamp resend_shard_blocks_at_; diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index 65adf34d4a..eff8aced3a 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -331,6 +331,10 @@ void ShardClient::force_update_shard_client_ex(BlockHandle handle, td::Ref opts) { + opts_ = std::move(opts); +} + } // namespace validator } // namespace ton diff --git a/validator/shard-client.hpp b/validator/shard-client.hpp index 701a387e88..d7330b368d 100644 --- a/validator/shard-client.hpp +++ b/validator/shard-client.hpp @@ -97,6 +97,8 @@ class ShardClient : public td::actor::Actor { void force_update_shard_client(BlockHandle handle, td::Promise promise); void force_update_shard_client_ex(BlockHandle handle, td::Ref state, td::Promise promise); + + void update_options(td::Ref opts); }; } // namespace validator diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index e971a3f849..89db2fc6c1 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -91,6 +91,10 @@ class AsyncStateSerializer : public td::actor::Actor { void fail_handler(td::Status reason); void fail_handler_cont(); void success_handler(); + + void update_options(td::Ref opts) { + opts_ = std::move(opts); + } }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 0595949fb9..0daccc1ae4 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -228,6 +228,7 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise> promise) = 0; virtual void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; + virtual void update_options(td::Ref opts) = 0; }; } // namespace validator From ea7a5776fecdab61301106e462ea98e55c38ca7c Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 17 Aug 2022 16:29:50 +0300 Subject: [PATCH 026/524] Download persistent states when syncing new shards --- tl/generate/scheme/ton_api.tl | 5 ++ tl/generate/scheme/ton_api.tlo | Bin 72144 -> 73056 bytes ton/ton-types.h | 10 +++ validator/db/rootdb.cpp | 8 ++ validator/db/rootdb.hpp | 3 + validator/db/statedb.cpp | 95 +++++++++++++++++++++ validator/db/statedb.hpp | 4 +- validator/downloaders/wait-block-state.cpp | 14 +++ validator/downloaders/wait-block-state.hpp | 15 +++- validator/interfaces/db.h | 5 ++ validator/interfaces/validator-manager.h | 2 + validator/manager-disk.hpp | 3 + validator/manager-hardfork.hpp | 2 + validator/manager.cpp | 67 +++++++++++++-- validator/manager.hpp | 10 ++- validator/shard-client.cpp | 10 +-- validator/state-serializer.cpp | 35 +++++++- validator/state-serializer.hpp | 2 + 18 files changed, 273 insertions(+), 17 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 39cfb5eec7..ff03764a79 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -514,6 +514,9 @@ db.state.shardClient block:tonNode.blockIdExt = db.state.ShardClient; db.state.asyncSerializer block:tonNode.blockIdExt last:tonNode.blockIdExt last_ts:int = db.state.AsyncSerializer; db.state.hardforks blocks:(vector tonNode.blockIdExt) = db.state.Hardforks; db.state.dbVersion version:int = db.state.DbVersion; +db.state.persistentStateDescriptionShards shard_blocks:(vector tonNode.blockIdExt) = db.state.PersistentStateDescriptionShards; +db.state.persistentStateDescriptionHeader masterchain_id:tonNode.blockIdExt start_time:int end_time:int = db.state.PersistentStateDescriptionHeader; +db.state.persistentStateDescriptionsList list:(vector db.state.persistentStateDescriptionHeader) = db.state.PersistentStateDescriptionsList; db.state.key.destroyedSessions = db.state.Key; db.state.key.initBlockId = db.state.Key; @@ -522,6 +525,8 @@ db.state.key.shardClient = db.state.Key; db.state.key.asyncSerializer = db.state.Key; db.state.key.hardforks = db.state.Key; db.state.key.dbVersion = db.state.Key; +db.state.key.persistentStateDescriptionShards masterchain_seqno:int = db.state.Key; +db.state.key.persistentStateDescriptionsList = db.state.Key; db.lt.el.key workchain:int shard:long idx:int = db.lt.Key; db.lt.desc.key workchain:int shard:long = db.lt.Key; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 413fbd8769514b2bed2ffc52cc34f4679ea70c69..12399020f6b0a164ee8f20686f7629f4a83f5545 100644 GIT binary patch delta 915 zcmcbxndQMQ7T!m*^{p77;L1kc-@Kv$m3$9*Qj+wFOA<>`^}JInfwG(v9mVxRlvy@v z!X*Myi;6RgOH%Vnfxf9mPICyC*N?l-_K@=Of7&ytz^RzNZAp={JuF`GEXWy2AH{w(-(>}icG$^Nx2>*glz1oO7SQiB8>$HHBd9mSdcl$fs1S|Y7&7v)eJ`( zVPF7h0L2(H&=l!s_ci%)6M<=}C^;iBGcSH}Vx%-T7cdsMS2hpi~&2l#`#FT|D{WcKPWE+>9bFAY+lO oK~15!t$_s_Nd5E2vV_`qxi%!0QlWLng9R* delta 265 zcmaE`i{-**7T!m*^{p77;NnK!-@Kwi$D6cxQj+wFOA<>`^}JInfwJrq9mO~E@cT$I z`fZNXxbG_ku#ePhi%(Ikl5saA|2e4Q;IZjOJ2dJQ=-@ax6XuOJjO1- LBe{7-*@ shard_blocks; + UnixTime start_time, end_time; + + virtual CntObject* make_copy() const { + return new PersistentStateDescription(*this); + } +}; + } // namespace ton diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index a7a1becf39..1c8c8e591d 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -497,6 +497,14 @@ void RootDb::set_async_mode(bool mode, td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::set_async_mode, mode, std::move(promise)); } +void RootDb::add_persistent_state_description(td::Ref desc, td::Promise promise) { + td::actor::send_closure(state_db_, &StateDb::add_persistent_state_description, std::move(desc), std::move(promise)); +} + +void RootDb::get_persistent_state_descriptions(td::Promise>> promise) { + td::actor::send_closure(state_db_, &StateDb::get_persistent_state_descriptions, std::move(promise)); +} + void RootDb::run_gc(UnixTime ts, UnixTime archive_ttl) { td::actor::send_closure(archive_db_, &ArchiveManager::run_gc, ts, archive_ttl); } diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 9b0d52a6b5..6206bbfe6d 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -132,6 +132,9 @@ class RootDb : public Db { td::Promise promise) override; void set_async_mode(bool mode, td::Promise promise) override; + void add_persistent_state_description(td::Ref desc, td::Promise promise) override; + void get_persistent_state_descriptions(td::Promise>> promise) override; + void run_gc(UnixTime ts, UnixTime archive_ttl) override; private: diff --git a/validator/db/statedb.cpp b/validator/db/statedb.cpp index 5d49ae2bd0..fe3b9d7351 100644 --- a/validator/db/statedb.cpp +++ b/validator/db/statedb.cpp @@ -240,6 +240,101 @@ void StateDb::start_up() { } } +void StateDb::add_persistent_state_description(td::Ref desc, + td::Promise promise) { + std::string value; + auto list_key = create_hash_tl_object(); + auto R = kv_->get(list_key.as_slice(), value); + R.ensure(); + tl_object_ptr list; + if (R.ok() == td::KeyValue::GetStatus::Ok) { + auto F = fetch_tl_object(value, true); + F.ensure(); + list = F.move_as_ok(); + } else { + list = create_tl_object( + std::vector>()); + } + for (const auto& obj : list->list_) { + if ((BlockSeqno)obj->masterchain_id_->seqno_ == desc->masterchain_id.seqno()) { + promise.set_error(td::Status::Error("duplicate masterchain seqno")); + return; + } + } + + auto now = (UnixTime)td::Clocks::system(); + size_t new_size = 0; + kv_->begin_write_batch().ensure(); + for (auto& obj : list->list_) { + auto end_time = (UnixTime)obj->end_time_; + if (end_time <= now) { + auto key = + create_hash_tl_object(obj->masterchain_id_->seqno_); + kv_->erase(key.as_slice()).ensure(); + } else { + list->list_[new_size++] = std::move(obj); + } + } + list->list_.resize(new_size); + + std::vector> shard_blocks; + for (const BlockIdExt& block_id : desc->shard_blocks) { + shard_blocks.push_back(create_tl_block_id(block_id)); + } + auto key = + create_hash_tl_object(desc->masterchain_id.seqno()); + kv_->set(key.as_slice(), + create_serialize_tl_object(std::move(shard_blocks)) + .as_slice()) + .ensure(); + + list->list_.push_back(create_tl_object( + create_tl_block_id(desc->masterchain_id), desc->start_time, desc->end_time)); + kv_->set(list_key.as_slice(), serialize_tl_object(list, true).as_slice()).ensure(); + + kv_->commit_write_batch().ensure(); + + promise.set_result(td::Unit()); +} + +void StateDb::get_persistent_state_descriptions(td::Promise>> promise) { + std::string value; + auto R = kv_->get(create_hash_tl_object().as_slice(), value); + R.ensure(); + if (R.ok() == td::KeyValue::GetStatus::NotFound) { + promise.set_value({}); + return; + } + auto F = fetch_tl_object(value, true); + F.ensure(); + std::vector> result; + auto now = (UnixTime)td::Clocks::system(); + for (const auto& obj : F.ok()->list_) { + auto end_time = (UnixTime)obj->end_time_; + if (end_time <= now) { + continue; + } + PersistentStateDescription desc; + desc.start_time = (UnixTime)obj->start_time_; + desc.end_time = end_time; + desc.masterchain_id = create_block_id(obj->masterchain_id_); + auto key = + create_hash_tl_object(desc.masterchain_id.seqno()); + auto R2 = kv_->get(key.as_slice(), value); + R2.ensure(); + if (R2.ok() == td::KeyValue::GetStatus::NotFound) { + continue; + } + auto F2 = fetch_tl_object(value, true); + F2.ensure(); + for (const auto& block_id : F2.ok()->shard_blocks_) { + desc.shard_blocks.push_back(create_block_id(block_id)); + } + result.push_back(td::Ref(true, std::move(desc))); + } + promise.set_result(std::move(result)); +} + void StateDb::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise) { { auto key = create_hash_tl_object(); diff --git a/validator/db/statedb.hpp b/validator/db/statedb.hpp index 75382d61c6..fe23898f4f 100644 --- a/validator/db/statedb.hpp +++ b/validator/db/statedb.hpp @@ -50,8 +50,8 @@ class StateDb : public td::actor::Actor { void update_hardforks(std::vector blocks, td::Promise promise); void get_hardforks(td::Promise> promise); - void update_db_version(td::uint32 version, td::Promise promise); - void get_db_version(td::Promise promise); + void add_persistent_state_description(td::Ref desc, td::Promise promise); + void get_persistent_state_descriptions(td::Promise>> promise); StateDb(td::actor::ActorId root_db, std::string path); diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index 56137fc394..8d033c1126 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -21,6 +21,7 @@ #include "ton/ton-io.hpp" #include "common/checksum.h" #include "common/delay.h" +#include "validator/downloaders/download-state.hpp" namespace ton { @@ -106,6 +107,19 @@ void WaitBlockState::start() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_zero_state_request, handle_->id(), priority_, std::move(P)); + } else if (check_persistent_state_desc()) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + LOG(WARNING) << "failed to get persistent state: " << R.move_as_error(); + td::actor::send_closure(SelfId, &WaitBlockState::start); + } else { + td::actor::send_closure(SelfId, &WaitBlockState::written_state, R.move_as_ok()); + } + }); + BlockIdExt masterchain_id = persistent_state_desc_->masterchain_id; + td::actor::create_actor("downloadstate", handle_->id(), masterchain_id, priority_, manager_, + timeout_, std::move(P)) + .release(); } else if (!handle_->inited_prev() || (!handle_->inited_proof() && !handle_->inited_proof_link())) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { if (R.is_error()) { diff --git a/validator/downloaders/wait-block-state.hpp b/validator/downloaders/wait-block-state.hpp index a93f297178..fc25eeec4e 100644 --- a/validator/downloaders/wait-block-state.hpp +++ b/validator/downloaders/wait-block-state.hpp @@ -27,12 +27,14 @@ namespace validator { class WaitBlockState : public td::actor::Actor { public: WaitBlockState(BlockHandle handle, td::uint32 priority, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise> promise) + td::Timestamp timeout, td::Promise> promise, + td::Ref persistent_state_desc = {}) : handle_(std::move(handle)) , priority_(priority) , manager_(manager) , timeout_(timeout) - , promise_(std::move(promise)) { + , promise_(std::move(promise)) + , persistent_state_desc_(std::move(persistent_state_desc)) { } void abort_query(td::Status reason); @@ -73,6 +75,7 @@ class WaitBlockState : public td::actor::Actor { td::actor::ActorId manager_; td::Timestamp timeout_; td::Promise> promise_; + td::Ref persistent_state_desc_; td::Ref prev_state_; td::Ref block_; @@ -81,6 +84,14 @@ class WaitBlockState : public td::actor::Actor { td::Timestamp next_static_file_attempt_; //td::PerfWarningTimer perf_timer_{"waitstate", 1.0}; + + bool check_persistent_state_desc() const { + if (persistent_state_desc_.is_null()) { + return false; + } + auto now = (UnixTime)td::Clocks::system(); + return persistent_state_desc_->end_time > now + 3600 && persistent_state_desc_->start_time < now - 6 * 3600; + } }; } // namespace validator diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index ba4d9dda12..2df3da5918 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -119,6 +119,11 @@ class Db : public td::actor::Actor { td::Promise promise) = 0; virtual void set_async_mode(bool mode, td::Promise promise) = 0; + virtual void add_persistent_state_description(td::Ref desc, + td::Promise promise) = 0; + virtual void get_persistent_state_descriptions( + td::Promise>> promise) = 0; + virtual void run_gc(UnixTime ts, UnixTime archive_ttl) = 0; }; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 0477b64873..38de746764 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -170,6 +170,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void validated_new_block(BlockIdExt block_id) = 0; + virtual void add_persistent_state_description(td::Ref desc) = 0; + static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index ea443d4667..1ab6fb0922 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -377,6 +377,9 @@ class ValidatorManagerImpl : public ValidatorManager { } void validated_new_block(BlockIdExt block_id) override { } + void add_persistent_state_description(td::Ref desc) override { + } + void get_validator_sessions_info( td::Promise> promise) override { UNREACHABLE(); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index ce92f8a7b3..3d20e494cf 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -437,6 +437,8 @@ class ValidatorManagerImpl : public ValidatorManager { } void validated_new_block(BlockIdExt block_id) override { } + void add_persistent_state_description(td::Ref desc) override { + } void get_validator_sessions_info( td::Promise> promise) override { UNREACHABLE(); diff --git a/validator/manager.cpp b/validator/manager.cpp index 710edec567..2f2e5f0242 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -574,9 +574,10 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); }); - auto id = td::actor::create_actor("waitstate", handle, priority, actor_id(this), - td::Timestamp::in(10.0), std::move(P)) - .release(); + auto id = + td::actor::create_actor("waitstate", handle, priority, actor_id(this), td::Timestamp::in(10.0), + std::move(P), get_block_persistent_state(handle->id())) + .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); } @@ -1013,7 +1014,7 @@ void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Result("waitstate", handle, X.second, actor_id(this), X.first, - std::move(P)) + std::move(P), get_block_persistent_state(handle->id())) .release(); it->second.actor_ = id; return; @@ -1568,8 +1569,17 @@ void ValidatorManagerImpl::started(ValidatorManagerInitResult R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::read_gc_list, R.move_as_ok()); } }); - td::actor::send_closure(db_, &Db::get_destroyed_validator_sessions, std::move(P)); + + auto Q = td::PromiseCreator::lambda( + [SelfId = actor_id(this)](td::Result>> R) { + if (R.is_error()) { + LOG(FATAL) << "db error: " << R.move_as_error(); + } else { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::got_persistent_state_descriptions, R.move_as_ok()); + } + }); + td::actor::send_closure(db_, &Db::get_persistent_state_descriptions, std::move(Q)); } void ValidatorManagerImpl::read_gc_list(std::vector list) { @@ -2706,6 +2716,53 @@ void ValidatorManagerImpl::update_options(td::Ref opts) opts_ = std::move(opts); } +void ValidatorManagerImpl::add_persistent_state_description(td::Ref desc) { + auto now = (UnixTime)td::Clocks::system(); + if (desc->end_time <= now) { + return; + } + td::actor::send_closure(db_, &Db::add_persistent_state_description, desc, [](td::Result) {}); + auto it = persistent_state_descriptions_.begin(); + while (it != persistent_state_descriptions_.end()) { + const auto &prev_desc = it->second; + if (prev_desc->end_time <= now) { + for (const BlockIdExt &block_id : prev_desc->shard_blocks) { + persistent_state_blocks_.erase(block_id); + } + it = persistent_state_descriptions_.erase(it); + } else { + ++it; + } + } + add_persistent_state_description_impl(std::move(desc)); +} + +void ValidatorManagerImpl::add_persistent_state_description_impl(td::Ref desc) { + if (!persistent_state_descriptions_.emplace(desc->masterchain_id.seqno(), desc).second) { + return; + } + LOG(DEBUG) << "Add persistent state description for mc block " << desc->masterchain_id.to_str() + << " start_time=" << desc->start_time << " end_time=" << desc->end_time; + for (const BlockIdExt &block_id : desc->shard_blocks) { + persistent_state_blocks_[block_id] = desc; + LOG(DEBUG) << "Persistent state description: shard block " << block_id.to_str(); + } +} + +void ValidatorManagerImpl::got_persistent_state_descriptions(std::vector> descs) { + for (auto &desc : descs) { + add_persistent_state_description_impl(std::move(desc)); + } +} + +td::Ref ValidatorManagerImpl::get_block_persistent_state(BlockIdExt block_id) { + auto it = persistent_state_blocks_.find(block_id); + if (it == persistent_state_blocks_.end()) { + return {}; + } + return it->second; +} + td::actor::ActorOwn ValidatorManagerFactory::create( td::Ref opts, std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, diff --git a/validator/manager.hpp b/validator/manager.hpp index d61351f29b..ddef344147 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -555,6 +555,8 @@ class ValidatorManagerImpl : public ValidatorManager { last = std::max(last, block_id.seqno()); } + void add_persistent_state_description(td::Ref desc) override; + void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; void update_options(td::Ref opts) override; @@ -621,8 +623,11 @@ class ValidatorManagerImpl : public ValidatorManager { } void cleanup_last_validated_blocks(BlockId new_block); - private: + void got_persistent_state_descriptions(std::vector> descs); + void add_persistent_state_description_impl(td::Ref desc); + td::Ref get_block_persistent_state(BlockIdExt block_id); + private: std::map> shard_client_waiters_; std::map> collator_nodes_; @@ -630,6 +635,9 @@ class ValidatorManagerImpl : public ValidatorManager { std::set shards_to_monitor_ = {ShardIdFull(masterchainId)}; std::set extra_active_shards_; std::map last_validated_blocks_; + + std::map> persistent_state_descriptions_; + std::map> persistent_state_blocks_; }; } // namespace validator diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index eff8aced3a..431d772165 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -202,7 +202,7 @@ void ShardClient::apply_all_shards() { } }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, shard->top_block_id(), - shard_client_priority(), td::Timestamp::in(600), std::move(Q)); + shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); } } } @@ -255,11 +255,9 @@ void ShardClient::build_shard_overlays() { auto shard = info->shard(); workchains.insert(shard.workchain); bool will_split = shard.pfx_len() < max_shard_pfx_len && - ((info->fsm_state() == McShardHash::FsmState::fsm_split && info->fsm_utime() < cur_time + 60) || - info->before_split()); - bool will_merge = shard.pfx_len() > 0 && - ((info->fsm_state() == McShardHash::FsmState::fsm_merge && info->fsm_utime() < cur_time + 60) || - info->before_merge()); + (info->fsm_state() == McShardHash::FsmState::fsm_split || info->before_split()); + bool will_merge = + shard.pfx_len() > 0 && (info->fsm_state() == McShardHash::FsmState::fsm_merge || info->before_merge()); if (opts_->need_monitor(shard) || (will_merge && opts_->need_monitor(shard_parent(shard)))) { new_shards_to_monitor.insert(shard); } diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index d5d2f0009b..fc8cd29aaf 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -18,7 +18,6 @@ */ #include "state-serializer.hpp" #include "td/utils/Random.h" -#include "adnl/utils.hpp" #include "ton/ton-io.hpp" #include "common/delay.h" @@ -122,6 +121,21 @@ void AsyncStateSerializer::next_iteration() { CHECK(masterchain_handle_->id() == last_block_id_); if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno && need_serialize(masterchain_handle_)) { + if (!stored_persistent_state_description_) { + LOG(INFO) << "storing persistent state description for " << masterchain_handle_->id().id; + running_ = true; + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &AsyncStateSerializer::fail_handler, + R.move_as_error_prefix("failed to get masterchain state: ")); + } else { + td::actor::send_closure(SelfId, &AsyncStateSerializer::store_persistent_state_description, + td::Ref(R.move_as_ok())); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_handle_, std::move(P)); + return; + } if (!cell_db_reader_) { running_ = true; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { @@ -175,6 +189,7 @@ void AsyncStateSerializer::next_iteration() { if (masterchain_handle_->inited_next_left()) { last_block_id_ = masterchain_handle_->one_next(true); have_masterchain_state_ = false; + stored_persistent_state_description_ = false; masterchain_handle_ = nullptr; saved_to_db_ = false; shards_.clear(); @@ -189,6 +204,24 @@ void AsyncStateSerializer::got_top_masterchain_handle(BlockIdExt block_id) { } } +void AsyncStateSerializer::store_persistent_state_description(td::Ref state) { + stored_persistent_state_description_ = true; + attempt_ = 0; + running_ = false; + + PersistentStateDescription desc; + desc.masterchain_id = state->get_block_id(); + desc.start_time = state->get_unix_time(); + desc.end_time = ValidatorManager::persistent_state_ttl(desc.start_time); + for (const auto &v : state->get_shards()) { + desc.shard_blocks.push_back(v->top_block_id()); + } + td::actor::send_closure(manager_, &ValidatorManager::add_persistent_state_description, + td::Ref(true, std::move(desc))); + + next_iteration(); +} + void AsyncStateSerializer::got_cell_db_reader(std::shared_ptr cell_db_reader) { cell_db_reader_ = std::move(cell_db_reader); running_ = false; diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index 89db2fc6c1..0d6e955050 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -44,6 +44,7 @@ class AsyncStateSerializer : public td::actor::Actor { std::shared_ptr cell_db_reader_ = nullptr; BlockHandle masterchain_handle_; + bool stored_persistent_state_description_ = false; bool have_masterchain_state_ = false; std::vector shards_; @@ -71,6 +72,7 @@ class AsyncStateSerializer : public td::actor::Actor { void next_iteration(); void got_top_masterchain_handle(BlockIdExt block_id); + void store_persistent_state_description(td::Ref state); void got_cell_db_reader(std::shared_ptr cell_db_reader); void got_masterchain_handle(BlockHandle handle_); void got_masterchain_state(td::Ref state); From 1efd4254031e8d1d37d98a81e0f47d42180120b3 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 19 Aug 2022 18:20:16 +0300 Subject: [PATCH 027/524] Fix compatibility --- dht-server/dht-server.cpp | 19 ++++++++-------- dht-server/dht-server.hpp | 4 ++-- lite-client/lite-client.cpp | 30 ++++++++++++-------------- tl/generate/scheme/ton_api.tl | 12 +++-------- tl/generate/scheme/ton_api.tlo | Bin 73056 -> 72560 bytes ton/ton-tl.hpp | 19 ---------------- validator-engine/validator-engine.cpp | 25 ++++++++------------- validator-engine/validator-engine.hpp | 4 ++-- validator/full-node-shard.cpp | 10 ++++----- 9 files changed, 44 insertions(+), 79 deletions(-) diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index 3bd65cc6e3..01ee105d24 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -38,9 +38,6 @@ #include "td/utils/TsFileLog.h" #include "td/utils/Random.h" -#include "ton/ton-tl.hpp" -#include "tl/tl_json.h" - #include "memprof/memprof.h" #if TD_DARWIN || TD_LINUX @@ -57,7 +54,7 @@ Config::Config() { out_port = 3278; } -Config::Config(const ton::ton_api::engine_validator_configV2 &config) { +Config::Config(const ton::ton_api::engine_validator_config &config) { out_port = static_cast(config.out_port_); if (!out_port) { out_port = 3278; @@ -124,7 +121,7 @@ Config::Config(const ton::ton_api::engine_validator_configV2 &config) { } } -ton::tl_object_ptr Config::tl() const { +ton::tl_object_ptr Config::tl() const { std::vector> addrs_vec; for (auto &x : addrs) { if (x.second.proxy) { @@ -150,6 +147,7 @@ ton::tl_object_ptr Config::tl() const { } std::vector> val_vec; + std::vector> col_vec; std::vector> full_node_slaves_vec; std::vector> full_node_masters_vec; @@ -165,15 +163,16 @@ ton::tl_object_ptr Config::tl() const { control_vec.push_back(ton::create_tl_object(x.second.key.tl(), x.first, std::move(control_proc_vec))); } + std::vector> shard_vec; auto gc_vec = ton::create_tl_object(std::vector{}); for (auto &id : gc) { gc_vec->ids_.push_back(id.tl()); } return ton::create_tl_object( - out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), + out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), - std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); + std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, @@ -600,14 +599,14 @@ void DhtServer::load_config(td::Promise promise) { } auto conf_json = conf_json_R.move_as_ok(); - ton::tl_object_ptr conf; - auto S = td::from_json(conf, std::move(conf_json)); + ton::ton_api::engine_validator_config conf; + auto S = ton::ton_api::from_json(conf, conf_json.get_object()); if (S.is_error()) { promise.set_error(S.move_as_error_prefix("json does not fit TL scheme")); return; } - config_ = Config{*ton::unpack_engine_validator_config(std::move(conf))}; + config_ = Config{conf}; td::MultiPromise mp; auto ig = mp.init_guard(); diff --git a/dht-server/dht-server.hpp b/dht-server/dht-server.hpp index 724232fa74..bf81520260 100644 --- a/dht-server/dht-server.hpp +++ b/dht-server/dht-server.hpp @@ -91,10 +91,10 @@ struct Config { td::Result config_del_control_process(td::int32 port, ton::PublicKeyHash id); td::Result config_del_gc(ton::PublicKeyHash key); - ton::tl_object_ptr tl() const; + ton::tl_object_ptr tl() const; Config(); - Config(const ton::ton_api::engine_validator_configV2 &config); + Config(const ton::ton_api::engine_validator_config &config); }; class DhtServer : public td::actor::Actor { diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 970cdad94c..7165ca53c8 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -124,7 +124,7 @@ void TestNode::run() { auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); ton::ton_api::liteclient_config_global gc; ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - CHECK(gc.liteservers_.size() > 0); + CHECK(gc.liteservers_.size() + gc.liteservers_v2_.size() > 0); if (gc.validator_ && gc.validator_->zero_state_) { zstate_id_.workchain = gc.validator_->zero_state_->workchain_; @@ -137,21 +137,19 @@ void TestNode::run() { for (auto& server : gc.liteservers_) { LiteServer s; - ton::ton_api::downcast_call(*server, - td::overloaded( - [&](ton::ton_api::liteserver_desc& obj) { - s.addr.init_host_port(td::IPAddress::ipv4_to_str(obj.ip_), obj.port_).ensure(); - s.public_key = ton::PublicKey{obj.id_}; - }, - [&](ton::ton_api::liteserver_descV2& obj) { - s.addr.init_host_port(td::IPAddress::ipv4_to_str(obj.ip_), obj.port_).ensure(); - s.public_key = ton::PublicKey{obj.id_}; - s.is_full = false; - for (const auto& shard : obj.shards_) { - s.shards.emplace_back(shard->workchain_, shard->shard_); - CHECK(s.shards.back().is_valid_ext()); - } - })); + s.addr.init_host_port(td::IPAddress::ipv4_to_str(server->ip_), server->port_).ensure(); + s.public_key = ton::PublicKey{server->id_}; + servers_.push_back(std::move(s)); + } + for (auto& server : gc.liteservers_v2_) { + LiteServer s; + s.addr.init_host_port(td::IPAddress::ipv4_to_str(server->ip_), server->port_).ensure(); + s.public_key = ton::PublicKey{server->id_}; + s.is_full = false; + for (const auto& shard : server->shards_) { + s.shards.emplace_back(shard->workchain_, shard->shard_); + CHECK(s.shards.back().is_valid_ext()); + } servers_.push_back(std::move(s)); } diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index ff03764a79..67b123c8b1 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -578,8 +578,8 @@ validator.config.global zero_state:tonNode.blockIdExt init_block:tonNode.blockId config.global adnl:adnl.config.global dht:dht.config.global validator:validator.config.global = config.Global; liteserver.desc id:PublicKey ip:int port:int = liteserver.Desc; -liteserver.descV2 id:PublicKey ip:int port:int shards:(vector tonNode.shardId) = liteserver.Desc; -liteclient.config.global liteservers:(vector liteserver.Desc) validator:validator.config.global = liteclient.config.Global; +liteserver.descV2 id:PublicKey ip:int port:int shards:(vector tonNode.shardId) = liteserver.DescV2; +liteclient.config.global liteservers:(vector liteserver.desc) liteservers_v2:(vector liteserver.descV2) validator:validator.config.global = liteclient.config.Global; engine.adnl id:int256 category:int = engine.Adnl; engine.addr ip:int port:int categories:(vector int) priority_categories:(vector int) = engine.Addr; @@ -598,14 +598,8 @@ engine.gc ids:(vector int256) = engine.Gc; engine.dht.config dht:(vector engine.dht) gc:engine.gc = engine.dht.Config; engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNodeMaster; engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave; -engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) - dht:(vector engine.dht) - validators:(vector engine.Validator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) - fullnodemasters:(vector engine.validator.fullNodeMaster) - liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) - gc:engine.gc = engine.validator.Config; -engine.validator.configV2 out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) +engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) validators:(vector engine.validator) collators:(vector engine.collator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 12399020f6b0a164ee8f20686f7629f4a83f5545..25b64d2125744eb1c87f0ccc2bca800b5edbe326 100644 GIT binary patch delta 308 zcmaE`i{-;M7T!m*^{p77;Oa)+a1l|1+w1R$r{<++=B4VDCFW$NB$nhCO@1gWzj=m8 z0vpTP`py@V|45mLlq~T)!k?2_l3JWvRF+z#=aO2SJaL1Bn6iVtz95=-n9;;_Qk!F> z|L`ydZI09sPhbVxGI?RI2zy#)I*7CRP1gb@7BFi6*M@VTKNf0r##PR}8vi dQ-GcbGh$!>8?m86V{*=-60mujpDe1W0ss;Jb=3d> delta 314 zcmeycjpe~E7T!m*^{p77;L1kca1ohfQgVFasd?#{d8vA3i8+}mi6!|(dd~TIX_@I0 z9V9l-5J_NTS#)gg`N@BzOhjzI)X(70$t+1NPAw`+Ez)yIElvih>XH7#!x+4|Q$sv~ zb&7F7koe@r?;<=vBS6OAJSODBJ-M)5d-8{lBa952-*yT!PA=$B6amV?9CKBUD<(+O zFGdoiksGKn%!q*jq create_tl_shard_id(const ShardIdF return create_tl_object(s.workchain, s.shard); } -inline tl_object_ptr unpack_engine_validator_config( - tl_object_ptr config) { - tl_object_ptr res; - ton_api::downcast_call(*config, td::overloaded( - [&](ton_api::engine_validator_config &c) { - res = create_tl_object( - c.out_port_, std::move(c.addrs_), std::move(c.adnl_), std::move(c.dht_), - std::move(c.validators_), - std::vector>(), c.fullnode_, - std::move(c.fullnodeslaves_), std::move(c.fullnodemasters_), - std::move(c.liteservers_), std::move(c.control_), - std::vector>(), std::move(c.gc_)); - }, - [&](ton_api::engine_validator_configV2 &c) { - res = std::make_unique(std::move(c)); - })); - return res; -} - } // namespace ton diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index e56dfb7be1..2aaff67411 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -75,7 +75,7 @@ Config::Config() { full_node = ton::PublicKeyHash::zero(); } -Config::Config(const ton::ton_api::engine_validator_configV2 &config) { +Config::Config(const ton::ton_api::engine_validator_config &config) { full_node = ton::PublicKeyHash::zero(); out_port = static_cast(config.out_port_); if (!out_port) { @@ -177,7 +177,7 @@ Config::Config(const ton::ton_api::engine_validator_configV2 &config) { } } -ton::tl_object_ptr Config::tl() const { +ton::tl_object_ptr Config::tl() const { std::vector> addrs_vec; for (auto &x : addrs) { if (x.second.proxy) { @@ -257,17 +257,10 @@ ton::tl_object_ptr Config::tl() const { gc_vec->ids_.push_back(id.tl()); } - if (col_vec.empty() && shards_vec.empty()) { - return ton::create_tl_object( - out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), - full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), - std::move(control_vec), std::move(gc_vec)); - } else { - return ton::create_tl_object( - out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), - full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), - std::move(control_vec), std::move(shards_vec), std::move(gc_vec)); - } + return ton::create_tl_object( + out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), + full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), + std::move(control_vec), std::move(shards_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, @@ -1662,14 +1655,14 @@ void ValidatorEngine::load_config(td::Promise promise) { } auto conf_json = conf_json_R.move_as_ok(); - ton::tl_object_ptr conf; - auto S = td::from_json(conf, std::move(conf_json)); + ton::ton_api::engine_validator_config conf; + auto S = ton::ton_api::from_json(conf, conf_json.get_object()); if (S.is_error()) { promise.set_error(S.move_as_error_prefix("json does not fit TL scheme")); return; } - config_ = Config{*ton::unpack_engine_validator_config(std::move(conf))}; + config_ = Config{conf}; td::MultiPromise mp; auto ig = mp.init_guard(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 40a04c703d..69ef5cb002 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -137,10 +137,10 @@ struct Config { td::Result config_del_control_process(td::int32 port, ton::PublicKeyHash id); td::Result config_del_gc(ton::PublicKeyHash key); - ton::tl_object_ptr tl() const; + ton::tl_object_ptr tl() const; Config(); - Config(const ton::ton_api::engine_validator_configV2 &config); + Config(const ton::ton_api::engine_validator_config &config); }; class ValidatorEngine : public td::actor::Actor { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 7bf44f33b6..2101435fd7 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -663,11 +663,11 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ne void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { BlockIdExt block_id = create_block_id(query.id_); - if (block_id.shard_full() != shard_) { - LOG(FULL_NODE_WARNING) << "dropping block broadcast: shard mismatch. overlay=" << shard_.to_str() - << " block=" << block_id.to_str(); - return; - } + //if (block_id.shard_full() != shard_) { + // LOG(FULL_NODE_WARNING) << "dropping block broadcast: shard mismatch. overlay=" << shard_.to_str() + // << " block=" << block_id.to_str(); + // return; + //} std::vector signatures; for (auto &sig : query.signatures_) { From c2e7d0b0dec18b26c8f6d751a36f48109e879731 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 23 Aug 2022 17:31:37 +0300 Subject: [PATCH 028/524] Bugfixes in manager.cpp --- validator/manager.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index 2f2e5f0242..2b6018364d 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -214,6 +214,10 @@ void ValidatorManagerImpl::prevalidate_block(BlockBroadcast broadcast, td::Promi promise.set_error(td::Status::Error(ErrorCode::notready, "node not started")); return; } + if (!shards_to_monitor_.count(broadcast.block_id.shard_full())) { + promise.set_error(td::Status::Error("not monitoring shard")); + return; + } td::actor::create_actor("broadcast", std::move(broadcast), last_masterchain_block_handle_, last_masterchain_state_, last_known_key_block_handle_, actor_id(this), td::Timestamp::in(2.0), std::move(promise)) @@ -575,8 +579,9 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); }); auto id = - td::actor::create_actor("waitstate", handle, priority, actor_id(this), td::Timestamp::in(10.0), - std::move(P), get_block_persistent_state(handle->id())) + td::actor::create_actor("waitstate", handle, priority, actor_id(this), + td::Timestamp::at(timeout.at() + 10.0), std::move(P), + get_block_persistent_state(handle->id())) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); @@ -614,7 +619,8 @@ void ValidatorManagerImpl::wait_out_msg_queue_proof(BlockIdExt block_id, ShardId }); auto id = td::actor::create_actor("waitmsgqueue", block_id, dst_shard, shards_to_monitor_.count(block_id.shard_full()), priority, - actor_id(this), td::Timestamp::in(10.0), std::move(P)) + actor_id(this), td::Timestamp::at(timeout.at() + 10.0), + std::move(P)) .release(); wait_out_msg_queue_proof_[key].actor_ = id; it = wait_out_msg_queue_proof_.find(key); @@ -636,7 +642,7 @@ void ValidatorManagerImpl::wait_block_data(BlockHandle handle, td::uint32 priori td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_data, handle, std::move(R)); }); auto id = td::actor::create_actor("waitdata", handle, priority, actor_id(this), - td::Timestamp::in(10.0), std::move(P)) + td::Timestamp::at(timeout.at() + 10.0), std::move(P)) .release(); wait_block_data_[handle->id()].actor_ = id; it = wait_block_data_.find(handle->id()); From 86250706b8ce8529c8e9f3e24baf6b69aff523b7 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 13 Sep 2022 16:02:55 +0300 Subject: [PATCH 029/524] Change format of specifying shards to monitor --- lite-client/lite-client.cpp | 5 ++++- validator-engine/validator-engine.cpp | 2 +- validator/collator-node.cpp | 2 +- validator/validator-group.cpp | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 7165ca53c8..2b559bc783 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -81,6 +81,9 @@ int TestNode::LiteServer::max_common_prefix(ton::ShardIdFull shard) const { } int res = -1; for (const ton::ShardIdFull &our_shard : shards) { + if (ton::shard_is_ancestor(our_shard, shard)) { + return shard.pfx_len(); + } if (shard.workchain == our_shard.workchain) { int x = std::min({shard.pfx_len(), our_shard.pfx_len(), ton::count_matching_bits(shard.shard, our_shard.shard)}); res = std::max(res, x); @@ -92,7 +95,7 @@ int TestNode::LiteServer::max_common_prefix(ton::ShardIdFull shard) const { bool TestNode::LiteServer::supports_shard(ton::ShardIdFull shard) const { return is_full || shard.is_masterchain() || std::any_of(shards.begin(), shards.end(), - [&](const ton::ShardIdFull& our_shard) { return ton::shard_is_ancestor(shard, our_shard); }); + [&](const ton::ShardIdFull& our_shard) { return ton::shard_intersects(shard, our_shard); }); } void TestNode::run() { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 2aaff67411..bb3bd05904 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1413,7 +1413,7 @@ void ValidatorEngine::set_shard_check_function() { validator_options_.write().set_shard_check_function( [shards = std::move(shards)](ton::ShardIdFull shard) -> bool { for (auto s : shards) { - if (shard_is_ancestor(shard, s)) { + if (shard_intersects(shard, s)) { return true; } } diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index c11b01c824..0a2a050ebd 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -170,7 +170,7 @@ void CollatorNode::receive_query_cont(adnl::AdnlNodeIdShort src, ShardIdFull sha bool CollatorNode::collate_shard(ShardIdFull shard) const { for (ShardIdFull our_shard : shards_) { - if (shard_is_ancestor(shard, our_shard)) { + if (shard_intersects(shard, our_shard)) { return true; } } diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 88a473ff63..21517a2055 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -364,7 +364,7 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo // TODO: some way to choose node (similar to "unreliability" in full-node) int cnt = 0; for (const block::CollatorNodeDescr& c : collator_config_.collator_nodes) { - if (shard_is_ancestor(shard_, c.shard) && td::Random::fast(0, cnt) == 0) { + if (shard_intersects(shard_, c.shard) && td::Random::fast(0, cnt) == 0) { collator = adnl::AdnlNodeIdShort(c.adnl_id); ++cnt; } From 62a412ac13627162a22cb5887b7d68572e87c55b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 15 Sep 2022 20:14:34 +0300 Subject: [PATCH 030/524] Add some required collated data --- validator/full-node-shard.cpp | 3 ++ validator/impl/collator-impl.h | 7 +++- validator/impl/collator.cpp | 57 ++++++++++++++++++++++++++++--- validator/impl/validate-query.cpp | 2 +- 4 files changed, 62 insertions(+), 7 deletions(-) diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 2101435fd7..56e0c1db19 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -130,6 +130,9 @@ void FullNodeShardImpl::create_overlay() { } void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broadcast, td::Promise promise) { + if (mode_ != FullNodeShardMode::active) { + return promise.set_error(td::Status::Error("cannot check broadcast: shard is not active")); + } auto B = fetch_tl_object(std::move(broadcast), true); if (B.is_error()) { return promise.set_error(B.move_as_error_prefix("failed to parse external message broadcast: ")); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index b886173a78..58f87fceea 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -197,7 +197,8 @@ class Collator final : public td::actor::Actor { std::vector, ExtMessage::Hash>> ext_msg_list_; std::priority_queue, std::greater> new_msgs; std::pair last_proc_int_msg_, first_unproc_int_msg_; - std::unique_ptr in_msg_dict, out_msg_dict, out_msg_queue_, sibling_out_msg_queue_; + std::unique_ptr in_msg_dict, out_msg_dict, old_out_msg_queue_, out_msg_queue_, + sibling_out_msg_queue_; std::unique_ptr ihr_pending; std::shared_ptr processed_upto_, sibling_processed_upto_; std::unique_ptr block_create_stats_; @@ -292,6 +293,7 @@ class Collator final : public td::actor::Actor { bool process_new_messages(bool enqueue_only = false); int process_one_new_message(block::NewOutMsg msg, bool enqueue_only = false, Ref* is_special = nullptr); bool process_inbound_internal_messages(); + bool precheck_inbound_message(Ref msg, ton::LogicalTime lt); bool process_inbound_message(Ref msg, ton::LogicalTime lt, td::ConstBitPtr key, const block::McShardDescr& src_nb); bool process_inbound_external_messages(); @@ -333,8 +335,11 @@ class Collator final : public td::actor::Actor { bool update_cc); bool create_mc_block_extra(Ref& mc_block_extra); bool create_block(); + Ref collate_shard_block_descr_set(); + bool prepare_msg_queue_proof(); bool create_collated_data(); + bool create_block_candidate(); void return_block_candidate(td::Result saved); bool update_last_proc_int_msg(const std::pair& new_lt_hash); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 3468bc5ded..98f0799c5b 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -825,6 +825,7 @@ bool Collator::import_shard_state_data(block::ShardState& ss) { total_validator_fees_ = std::move(ss.total_validator_fees_); old_global_balance_ = std::move(ss.global_balance_); out_msg_queue_ = std::move(ss.out_msg_queue_); + old_out_msg_queue_ = std::make_unique(*out_msg_queue_); processed_upto_ = std::move(ss.processed_upto_); ihr_pending = std::move(ss.ihr_pending_); block_create_stats_ = std::move(ss.block_create_stats_); @@ -2619,8 +2620,7 @@ bool Collator::delete_out_msg_queue_msg(td::ConstBitPtr key) { return register_out_msg_queue_op(); } -bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalTime lt, td::ConstBitPtr key, - const block::McShardDescr& src_nb) { +bool Collator::precheck_inbound_message(Ref enq_msg, ton::LogicalTime lt) { ton::LogicalTime enqueued_lt = 0; if (enq_msg.is_null() || enq_msg->size_ext() != 0x10040 || (enqueued_lt = enq_msg->prefetch_ulong(64)) < /* 0 */ 1 * lt) { // DEBUG @@ -2646,6 +2646,13 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT LOG(ERROR) << "inbound internal MsgEnvelope is invalid according to automated checks"; return false; } + return true; +} + +bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalTime lt, td::ConstBitPtr key, + const block::McShardDescr& src_nb) { + ton::LogicalTime enqueued_lt = enq_msg->prefetch_ulong(64); + auto msg_env = enq_msg->prefetch_ref(); // 1. unpack MsgEnvelope block::tlb::MsgEnvelope::Record_std env; if (!tlb::unpack_cell(msg_env, env)) { @@ -2784,14 +2791,22 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT } bool Collator::process_inbound_internal_messages() { - while (!block_full_ && !nb_out_msgs_->is_eof()) { + while (!nb_out_msgs_->is_eof()) { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); + auto kv = nb_out_msgs_->extract_cur(); + CHECK(kv && kv->msg.not_null()); + if (!precheck_inbound_message(kv->msg, kv->lt)) { + if (verbosity > 1) { + std::cerr << "invalid inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() + << " msg="; + block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg)); + } + return fatal_error("error processing inbound internal message"); + } if (block_full_) { LOG(INFO) << "BLOCK FULL, stop processing inbound internal messages"; break; } - auto kv = nb_out_msgs_->extract_cur(); - CHECK(kv && kv->msg.not_null()); LOG(DEBUG) << "processing inbound message with (lt,hash)=(" << kv->lt << "," << kv->key.to_hex() << ") from neighbor #" << kv->source; if (verbosity > 2) { @@ -3974,6 +3989,34 @@ Ref Collator::collate_shard_block_descr_set() { return cell; } +bool Collator::prepare_msg_queue_proof() { + auto res = old_out_msg_queue_->scan_diff( + *out_msg_queue_, + [this](td::ConstBitPtr key, int key_len, Ref old_value, Ref new_value) { + old_value = old_out_msg_queue_->extract_value(std::move(old_value)); + new_value = out_msg_queue_->extract_value(std::move(new_value)); + if (new_value.not_null()) { + if (!block::gen::t_EnqueuedMsg.validate_csr(new_value)) { + return false; + } + if (!block::tlb::t_EnqueuedMsg.validate_csr(new_value)) { + return false; + } + } + if (old_value.not_null()) { + if (!block::gen::t_EnqueuedMsg.validate_csr(old_value)) { + return false; + } + if (!block::tlb::t_EnqueuedMsg.validate_csr(old_value)) { + return false; + } + } + return true; + }, + 3); + return res; +} + bool Collator::create_collated_data() { // 1. store the set of used shard block descriptions if (!used_shard_block_descr_.empty()) { @@ -3994,6 +4037,10 @@ bool Collator::create_collated_data() { // 3. Previous state proof (only shadchains) std::map> proofs; if (!is_masterchain()) { + if (!prepare_msg_queue_proof()) { + return fatal_error("cannot prepare message queue proof"); + } + state_usage_tree_->set_use_mark_for_is_loaded(false); Ref state_proof = vm::MerkleProof::generate(prev_state_root_, state_usage_tree_.get()); if (state_proof.is_null()) { diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index eeb2726c66..aa342fc440 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -2678,7 +2678,7 @@ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id } ton::LogicalTime enqueued_lt = old_value->prefetch_ulong(64); if (enqueued_lt >= start_lt_) { - return reject_query(PSTRING() << "new EnqueuedMsg with key "s + out_msg_id.to_hex(352) + " has enqueued_lt=" + return reject_query(PSTRING() << "old EnqueuedMsg with key "s + out_msg_id.to_hex(352) + " has enqueued_lt=" << enqueued_lt << " greater than or equal to this block's start_lt=" << start_lt_); } } From b0c2c6c525f58f882c3b3f3cb9b9639c8da0d802 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 23 Sep 2022 17:23:00 +0300 Subject: [PATCH 031/524] Don't send proof of masterchain message queue --- validator/impl/collator.cpp | 28 +++++++++++++++++----------- validator/impl/validate-query.cpp | 9 +++++++++ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 98f0799c5b..5d0d2972d8 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -500,14 +500,15 @@ void Collator::after_get_block_data(int idx, td::Result> res) { CHECK(!idx); prev_mc_block = prev_block_data[0]; mc_block_root = prev_mc_block->root_cell(); + } else { + Ref root = prev_block_data[idx]->root_cell(); + auto proof = create_block_state_proof(root); + if (proof.is_error()) { + fatal_error(proof.move_as_error()); + return; + } + block_state_proofs_.emplace(root->get_hash().bits(), proof.move_as_ok()); } - Ref root = prev_block_data[idx]->root_cell(); - auto proof = create_block_state_proof(root); - if (proof.is_error()) { - fatal_error(proof.move_as_error()); - return; - } - block_state_proofs_.emplace(root->get_hash().bits(), proof.move_as_ok()); } check_pending(); } @@ -622,7 +623,6 @@ bool Collator::request_neighbor_msg_queues() { neighbors_.emplace_back(*shard_ptr); } unsigned i = 0; - neighbor_proof_builders_.resize(neighbors_.size()); for (block::McShardDescr& descr : neighbors_) { LOG(DEBUG) << "neighbor #" << i << " : " << descr.blk_.to_str(); ++pending; @@ -644,11 +644,17 @@ void Collator::got_neighbor_msg_queue(unsigned i, td::Resultblock_state_proof_.not_null()) { + if (res->block_state_proof_.not_null() && !block_id.is_masterchain()) { block_state_proofs_.emplace(block_id.root_hash, res->block_state_proof_); } - neighbor_proof_builders_.at(i) = vm::MerkleProofBuilder{res->state_root_}; - auto state = ShardStateQ::fetch(block_id, {}, neighbor_proof_builders_.at(i).root()); + Ref state_root; + if (block_id.is_masterchain()) { + state_root = res->state_root_; + } else { + neighbor_proof_builders_.push_back(vm::MerkleProofBuilder{res->state_root_}); + state_root = neighbor_proof_builders_.back().root(); + } + auto state = ShardStateQ::fetch(block_id, {}, state_root); if (state.is_error()) { fatal_error(state.move_as_error()); return; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index aa342fc440..f8a2708d8d 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1258,6 +1258,15 @@ bool ValidateQuery::request_neighbor_queues() { if (full_collated_data_) { for (block::McShardDescr& descr : neighbors_) { LOG(DEBUG) << "getting outbound queue of neighbor #" << i << " from collated data : " << descr.blk_.to_str(); + if (descr.blk_.is_masterchain()) { + if (descr.blk_ != mc_state_->get_block_id()) { + return fatal_error("neighbor from masterchain is not the last mc block"); + } + ++pending; + send_closure_later(get_self(), &ValidateQuery::got_neighbor_out_queue, i, mc_state_->message_queue()); + ++i; + continue; + } td::Bits256 state_root_hash; if (descr.blk_.seqno() == 0) { state_root_hash = descr.blk_.root_hash; From 69cee95abbe64d621f6e35d3ec88095b7e08d989 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 4 Oct 2022 11:07:38 +0300 Subject: [PATCH 032/524] Bugfixes in rldp-http-proxy and http parser --- http/http-inbound-connection.cpp | 2 +- http/http-outbound-connection.cpp | 2 +- http/http.cpp | 2 +- rldp-http-proxy/rldp-http-proxy.cpp | 6 ++++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/http/http-inbound-connection.cpp b/http/http-inbound-connection.cpp index 89c9c12303..57064dddc3 100644 --- a/http/http-inbound-connection.cpp +++ b/http/http-inbound-connection.cpp @@ -79,10 +79,10 @@ td::Status HttpInboundConnection::receive(td::ChainBufferReader &input) { send_client_error(); return td::Status::OK(); } + cur_request_ = R.move_as_ok(); if (exit_loop) { return td::Status::OK(); } - cur_request_ = R.move_as_ok(); } auto payload = cur_request_->create_empty_payload().move_as_ok(); diff --git a/http/http-outbound-connection.cpp b/http/http-outbound-connection.cpp index a98efbc93a..63621df875 100644 --- a/http/http-outbound-connection.cpp +++ b/http/http-outbound-connection.cpp @@ -42,10 +42,10 @@ td::Status HttpOutboundConnection::receive(td::ChainBufferReader &input) { answer_error(HttpStatusCode::status_bad_request, "", std::move(promise_)); return td::Status::OK(); } + cur_response_ = R.move_as_ok(); if (exit_loop) { return td::Status::OK(); } - cur_response_ = R.move_as_ok(); } if (cur_response_->code() == 100) { diff --git a/http/http.cpp b/http/http.cpp index cefe1a4759..cc1fd889d5 100644 --- a/http/http.cpp +++ b/http/http.cpp @@ -587,7 +587,7 @@ tl_object_ptr HttpPayload::store_tl(size_t max_size) max_size -= s.size(); } obj->data_.truncate(obj->data_.size() - S.size()); - if (chunks_.size() != 0) { + if (chunks_.size() != 0 || !parse_completed()) { return obj; } if (!written_zero_chunk_) { diff --git a/rldp-http-proxy/rldp-http-proxy.cpp b/rldp-http-proxy/rldp-http-proxy.cpp index a5faa002c7..0b679c2710 100644 --- a/rldp-http-proxy/rldp-http-proxy.cpp +++ b/rldp-http-proxy/rldp-http-proxy.cpp @@ -117,7 +117,7 @@ class HttpRemote : public td::actor::Actor { } }); td::actor::send_closure(client_, &ton::http::HttpClient::send_request, std::move(request), std::move(payload), - td::Timestamp::in(30.0), std::move(P)); + td::Timestamp::never(), std::move(P)); } else { ton::http::answer_error(ton::http::HttpStatusCode::status_bad_request, "", std::move(promise)); } @@ -801,6 +801,7 @@ class RldpToTcpRequestSender : public td::actor::Actor { , dst_(dst) , request_(std::move(request)) , request_payload_(std::move(request_payload)) + , proto_version_(request_->proto_version()) , promise_(std::move(promise)) , adnl_(adnl) , rldp_(rldp) @@ -836,7 +837,7 @@ class RldpToTcpRequestSender : public td::actor::Actor { void abort_query(td::Status error) { LOG(INFO) << "aborting http over rldp query: " << error; - promise_.set_result(create_error_response(request_->proto_version(), 502, "Bad Gateway")); + promise_.set_result(create_error_response(proto_version_, 502, "Bad Gateway")); stop(); } @@ -848,6 +849,7 @@ class RldpToTcpRequestSender : public td::actor::Actor { std::unique_ptr request_; std::shared_ptr request_payload_; + std::string proto_version_; td::Promise promise_; From 33a079962fca393b06168cc497cf2fcdbf3ca854 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 4 Oct 2022 15:12:13 +0300 Subject: [PATCH 033/524] Tonlib: change liteservers on query timeout or connection close --- adnl/adnl-ext-client.hpp | 3 ++ adnl/adnl-query.cpp | 7 ++- adnl/adnl-query.h | 1 + rldp-http-proxy/DNSResolver.cpp | 13 ++++- rldp-http-proxy/DNSResolver.h | 1 + tonlib/tonlib/ExtClient.cpp | 6 +-- tonlib/tonlib/ExtClient.h | 9 +++- tonlib/tonlib/ExtClientLazy.cpp | 77 +++++++++++++++++++++++------ tonlib/tonlib/ExtClientLazy.h | 11 +++-- tonlib/tonlib/ExtClientOutbound.cpp | 3 ++ tonlib/tonlib/ExtClientOutbound.h | 4 +- tonlib/tonlib/LastBlock.cpp | 1 + tonlib/tonlib/LastConfig.cpp | 1 + tonlib/tonlib/TonlibClient.cpp | 13 +++-- tonlib/tonlib/TonlibClient.h | 2 +- tonlib/tonlib/tonlib-cli.cpp | 2 +- 16 files changed, 117 insertions(+), 37 deletions(-) diff --git a/adnl/adnl-ext-client.hpp b/adnl/adnl-ext-client.hpp index a4df818e72..1333972557 100644 --- a/adnl/adnl-ext-client.hpp +++ b/adnl/adnl-ext-client.hpp @@ -80,6 +80,9 @@ class AdnlExtClientImpl : public AdnlExtClient { if (!conn_.empty() && conn_.get() == conn) { callback_->on_stop_ready(); conn_ = {}; + for (auto& q : out_queries_) { + td::actor::send_closure(q.second, &AdnlQuery::set_error, td::Status::Error(ErrorCode::cancelled)); + } alarm_timestamp() = next_create_at_; try_stop(); } diff --git a/adnl/adnl-query.cpp b/adnl/adnl-query.cpp index 5bc767d2bf..e098c13442 100644 --- a/adnl/adnl-query.cpp +++ b/adnl/adnl-query.cpp @@ -25,13 +25,16 @@ namespace ton { namespace adnl { void AdnlQuery::alarm() { - promise_.set_error(td::Status::Error(ErrorCode::timeout, "adnl query timeout")); - stop(); + set_error(td::Status::Error(ErrorCode::timeout, "adnl query timeout")); } void AdnlQuery::result(td::BufferSlice data) { promise_.set_value(std::move(data)); stop(); } +void AdnlQuery::set_error(td::Status error) { + promise_.set_error(std::move(error)); + stop(); +} AdnlQueryId AdnlQuery::random_query_id() { AdnlQueryId q_id; diff --git a/adnl/adnl-query.h b/adnl/adnl-query.h index 6e24a49fe7..3db8c754c1 100644 --- a/adnl/adnl-query.h +++ b/adnl/adnl-query.h @@ -48,6 +48,7 @@ class AdnlQuery : public td::actor::Actor { } void alarm() override; void result(td::BufferSlice data); + void set_error(td::Status error); void start_up() override { alarm_timestamp() = timeout_; } diff --git a/rldp-http-proxy/DNSResolver.cpp b/rldp-http-proxy/DNSResolver.cpp index a75197420d..1fb197a57b 100644 --- a/rldp-http-proxy/DNSResolver.cpp +++ b/rldp-http-proxy/DNSResolver.cpp @@ -25,6 +25,7 @@ */ #include "DNSResolver.h" #include "td/utils/overloaded.h" +#include "common/delay.h" static const double CACHE_TIMEOUT_HARD = 300.0; static const double CACHE_TIMEOUT_SOFT = 270.0; @@ -33,8 +34,18 @@ DNSResolver::DNSResolver(td::actor::ActorId tonlib_client) : tonli } void DNSResolver::start_up() { + sync(); +} + +void DNSResolver::sync() { auto obj = tonlib_api::make_object(); - auto P = td::PromiseCreator::lambda([](td::Result>) {}); + auto P = td::PromiseCreator::lambda([SelfId = + actor_id(this)](td::Result> R) { + if (R.is_error()) { + LOG(WARNING) << "Sync error: " << R.move_as_error(); + ton::delay_action([SelfId]() { td::actor::send_closure(SelfId, &DNSResolver::sync); }, td::Timestamp::in(5.0)); + } + }); td::actor::send_closure(tonlib_client_, &TonlibClient::send_request, std::move(obj), std::move(P)); } diff --git a/rldp-http-proxy/DNSResolver.h b/rldp-http-proxy/DNSResolver.h index f87b5e5a81..ba52a67e3c 100644 --- a/rldp-http-proxy/DNSResolver.h +++ b/rldp-http-proxy/DNSResolver.h @@ -37,6 +37,7 @@ class DNSResolver : public td::actor::Actor { void resolve(std::string host, td::Promise promise); private: + void sync(); void save_to_cache(std::string host, ton::adnl::AdnlNodeIdShort id); td::actor::ActorId tonlib_client_; diff --git a/tonlib/tonlib/ExtClient.cpp b/tonlib/tonlib/ExtClient.cpp index 8c8b5d0747..30a29b59c8 100644 --- a/tonlib/tonlib/ExtClient.cpp +++ b/tonlib/tonlib/ExtClient.cpp @@ -35,7 +35,7 @@ void ExtClient::with_last_config(td::Promise promise) { self->last_config_queries_.extract(query_id).set_result(std::move(result)); }); }; - if (client_.last_block_actor_.empty()) { + if (client_.last_config_actor_.empty()) { return P.set_error(TonlibError::NoLiteServers()); } td::actor::send_closure(client_.last_config_actor_, &LastConfig::get_last_config, std::move(P)); @@ -62,10 +62,10 @@ void ExtClient::send_raw_query(td::BufferSlice query, td::Promisequeries_.extract(query_id).set_result(std::move(result)); }); }; - if (client_.andl_ext_client_.empty()) { + if (client_.adnl_ext_client_.empty()) { return P.set_error(TonlibError::NoLiteServers()); } - td::actor::send_closure(client_.andl_ext_client_, &ton::adnl::AdnlExtClient::send_query, "query", std::move(query), + td::actor::send_closure(client_.adnl_ext_client_, &ton::adnl::AdnlExtClient::send_query, "query", std::move(query), td::Timestamp::in(10.0), std::move(P)); } } // namespace tonlib diff --git a/tonlib/tonlib/ExtClient.h b/tonlib/tonlib/ExtClient.h index 882b4794af..707ca74bd3 100644 --- a/tonlib/tonlib/ExtClient.h +++ b/tonlib/tonlib/ExtClient.h @@ -28,6 +28,7 @@ #include "td/utils/Container.h" #include "td/utils/Random.h" +#include "ExtClientLazy.h" #include "TonlibError.h" #include "utils.h" @@ -37,7 +38,7 @@ class LastConfig; struct LastBlockState; struct LastConfigState; struct ExtClientRef { - td::actor::ActorId andl_ext_client_; + td::actor::ActorId adnl_ext_client_; td::actor::ActorId last_block_actor_; td::actor::ActorId last_config_actor_; }; @@ -94,6 +95,12 @@ class ExtClient { }); } + void force_change_liteserver() { + if (!client_.adnl_ext_client_.empty()) { + td::actor::send_closure(client_.adnl_ext_client_, &ExtClientLazy::force_change_liteserver); + } + } + private: ExtClientRef client_; td::Container> queries_; diff --git a/tonlib/tonlib/ExtClientLazy.cpp b/tonlib/tonlib/ExtClientLazy.cpp index 1ab7a24f86..335a0ff968 100644 --- a/tonlib/tonlib/ExtClientLazy.cpp +++ b/tonlib/tonlib/ExtClientLazy.cpp @@ -18,13 +18,20 @@ */ #include "ExtClientLazy.h" #include "TonlibError.h" +#include "td/utils/Random.h" namespace tonlib { -class ExtClientLazyImp : public ton::adnl::AdnlExtClient { +class ExtClientLazyImp : public ExtClientLazy { public: - ExtClientLazyImp(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + ExtClientLazyImp(std::vector> servers, td::unique_ptr callback) - : dst_(std::move(dst)), dst_addr_(std::move(dst_addr)), callback_(std::move(callback)) { + : servers_(std::move(servers)), callback_(std::move(callback)) { + CHECK(!servers_.empty()); + } + + void start_up() override { + td::Random::Fast rnd; + td::random_shuffle(td::as_mutable_span(servers_), rnd); } void check_ready(td::Promise promise) override { @@ -41,37 +48,66 @@ class ExtClientLazyImp : public ton::adnl::AdnlExtClient { if (client_.empty()) { return promise.set_error(TonlibError::Cancelled()); } + td::Promise P = [SelfId = actor_id(this), idx = cur_server_idx_, + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error() && + (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { + td::actor::send_closure(SelfId, &ExtClientLazyImp::set_server_bad, idx, true); + } + promise.set_result(std::move(R)); + }; send_closure(client_, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, - std::move(promise)); + std::move(P)); + } + + void force_change_liteserver() override { + if (servers_.size() == 1) { + return; + } + cur_server_bad_ = cur_server_bad_force_ = true; } + private: void before_query() { if (is_closing_) { return; } - if (!client_.empty()) { - alarm_timestamp() = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT); + alarm_timestamp() = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT); + if (cur_server_bad_) { + ++cur_server_idx_; + } else if (!client_.empty()) { return; } class Callback : public ton::adnl::AdnlExtClient::Callback { public: - explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { + explicit Callback(td::actor::ActorShared parent, size_t idx) + : parent_(std::move(parent)), idx_(idx) { } void on_ready() override { + td::actor::send_closure(parent_, &ExtClientLazyImp::set_server_bad, idx_, false); } void on_stop_ready() override { + td::actor::send_closure(parent_, &ExtClientLazyImp::set_server_bad, idx_, true); } private: - td::actor::ActorShared<> parent_; + td::actor::ActorShared parent_; + size_t idx_; }; ref_cnt_++; - client_ = ton::adnl::AdnlExtClient::create(dst_, dst_addr_, std::make_unique(td::actor::actor_shared())); + cur_server_bad_ = false; + cur_server_bad_force_ = false; + const auto& s = servers_[cur_server_idx_ % servers_.size()]; + LOG(INFO) << "Connecting to liteserver " << s.second; + client_ = ton::adnl::AdnlExtClient::create( + s.first, s.second, std::make_unique(td::actor::actor_shared(this), cur_server_idx_)); } - private: - ton::adnl::AdnlNodeIdFull dst_; - td::IPAddress dst_addr_; + std::vector> servers_; + size_t cur_server_idx_ = 0; + bool cur_server_bad_ = false; + bool cur_server_bad_force_ = false; + td::actor::ActorOwn client_; td::unique_ptr callback_; static constexpr double MAX_NO_QUERIES_TIMEOUT = 100; @@ -79,6 +115,11 @@ class ExtClientLazyImp : public ton::adnl::AdnlExtClient { bool is_closing_{false}; td::uint32 ref_cnt_{1}; + void set_server_bad(size_t idx, bool bad) { + if (idx == cur_server_idx_ && servers_.size() > 1 && !cur_server_bad_force_) { + cur_server_bad_ = bad; + } + } void alarm() override { client_.reset(); } @@ -99,9 +140,13 @@ class ExtClientLazyImp : public ton::adnl::AdnlExtClient { } }; -td::actor::ActorOwn ExtClientLazy::create(ton::adnl::AdnlNodeIdFull dst, - td::IPAddress dst_addr, - td::unique_ptr callback) { - return td::actor::create_actor("ExtClientLazy", dst, dst_addr, std::move(callback)); +td::actor::ActorOwn ExtClientLazy::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + td::unique_ptr callback) { + return create({std::make_pair(dst, dst_addr)}, std::move(callback)); +} + +td::actor::ActorOwn ExtClientLazy::create( + std::vector> servers, td::unique_ptr callback) { + return td::actor::create_actor("ExtClientLazy", std::move(servers), std::move(callback)); } } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientLazy.h b/tonlib/tonlib/ExtClientLazy.h index 6014abd3d7..dc4490b3a3 100644 --- a/tonlib/tonlib/ExtClientLazy.h +++ b/tonlib/tonlib/ExtClientLazy.h @@ -22,15 +22,20 @@ #include "adnl/adnl-ext-client.h" namespace tonlib { -class ExtClientLazy { +class ExtClientLazy : public ton::adnl::AdnlExtClient { public: class Callback { public: virtual ~Callback() { } }; - static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, - td::unique_ptr callback); + + virtual void force_change_liteserver() = 0; + + static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + td::unique_ptr callback); + static td::actor::ActorOwn create( + std::vector> servers, td::unique_ptr callback); }; } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientOutbound.cpp b/tonlib/tonlib/ExtClientOutbound.cpp index fcccc4d4b5..025ba848ed 100644 --- a/tonlib/tonlib/ExtClientOutbound.cpp +++ b/tonlib/tonlib/ExtClientOutbound.cpp @@ -38,6 +38,9 @@ class ExtClientOutboundImp : public ExtClientOutbound { callback_->request(query_id, data.as_slice().str()); } + void force_change_liteserver() override { + } + void on_query_result(td::int64 id, td::Result r_data, td::Promise promise) override { auto it = queries_.find(id); if (it == queries_.end()) { diff --git a/tonlib/tonlib/ExtClientOutbound.h b/tonlib/tonlib/ExtClientOutbound.h index 7bab49030a..87b73b9e01 100644 --- a/tonlib/tonlib/ExtClientOutbound.h +++ b/tonlib/tonlib/ExtClientOutbound.h @@ -19,10 +19,10 @@ #pragma once #include "td/actor/actor.h" -#include "adnl/adnl-ext-client.h" +#include "ExtClientLazy.h" namespace tonlib { -class ExtClientOutbound : public ton::adnl::AdnlExtClient { +class ExtClientOutbound : public ExtClientLazy { public: class Callback { public: diff --git a/tonlib/tonlib/LastBlock.cpp b/tonlib/tonlib/LastBlock.cpp index bc2b74ba81..10a4d10fba 100644 --- a/tonlib/tonlib/LastBlock.cpp +++ b/tonlib/tonlib/LastBlock.cpp @@ -374,6 +374,7 @@ void LastBlock::on_sync_error(td::Status status) { promise.set_error(status.clone()); } promises_.clear(); + client_.force_change_liteserver(); } void LastBlock::on_fatal_error(td::Status status) { VLOG(last_block) << "sync: fatal error " << status; diff --git a/tonlib/tonlib/LastConfig.cpp b/tonlib/tonlib/LastConfig.cpp index 0111095bca..960d59946b 100644 --- a/tonlib/tonlib/LastConfig.cpp +++ b/tonlib/tonlib/LastConfig.cpp @@ -141,6 +141,7 @@ void LastConfig::on_error(td::Status status) { promise.set_error(status.clone()); } promises_.clear(); + get_config_state_ = QueryState::Empty; } void LastConfig::tear_down() { diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 842ea5fdfb..f9e984bb9e 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -1649,7 +1649,7 @@ void TonlibClient::hangup() { ExtClientRef TonlibClient::get_client_ref() { ExtClientRef ref; - ref.andl_ext_client_ = raw_client_.get(); + ref.adnl_ext_client_ = raw_client_.get(); ref.last_block_actor_ = raw_last_block_.get(); ref.last_config_actor_ = raw_last_config_.get(); @@ -1683,10 +1683,10 @@ void TonlibClient::init_ext_client() { ext_client_outbound_ = client.get(); raw_client_ = std::move(client); } else { - auto lite_clients_size = config_.lite_clients.size(); - CHECK(lite_clients_size != 0); - auto lite_client_id = td::Random::fast(0, td::narrow_cast(lite_clients_size) - 1); - auto& lite_client = config_.lite_clients[lite_client_id]; + std::vector> servers; + for (const auto& s : config_.lite_clients) { + servers.emplace_back(s.adnl_id, s.address); + } class Callback : public ExtClientLazy::Callback { public: explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { @@ -1697,8 +1697,7 @@ void TonlibClient::init_ext_client() { }; ext_client_outbound_ = {}; ref_cnt_++; - raw_client_ = ExtClientLazy::create(lite_client.adnl_id, lite_client.address, - td::make_unique(td::actor::actor_shared())); + raw_client_ = ExtClientLazy::create(std::move(servers), td::make_unique(td::actor::actor_shared())); } } diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 800d8c806b..e781929098 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -110,7 +110,7 @@ class TonlibClient : public td::actor::Actor { vm::Dictionary libraries{256}; // network - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; td::actor::ActorId ext_client_outbound_; td::actor::ActorOwn raw_last_block_; td::actor::ActorOwn raw_last_config_; diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index e889234ac4..0a042eb1dd 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -174,7 +174,7 @@ class TonlibCli : public td::actor::Actor { std::map>> query_handlers_; - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; bool is_closing_{false}; td::uint32 ref_cnt_{1}; From 836184e566de6a3e6ee44e0a06f9c5f85b98dfbf Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 4 Oct 2022 18:25:25 +0300 Subject: [PATCH 034/524] Increase maximum size of http request --- rldp-http-proxy/rldp-http-proxy.cpp | 9 ++++----- rldp/rldp-in.hpp | 7 ++++++- rldp/rldp.cpp | 4 ++-- rldp/rldp.h | 6 ++---- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/rldp-http-proxy/rldp-http-proxy.cpp b/rldp-http-proxy/rldp-http-proxy.cpp index 0b679c2710..a45e6433d4 100644 --- a/rldp-http-proxy/rldp-http-proxy.cpp +++ b/rldp-http-proxy/rldp-http-proxy.cpp @@ -825,11 +825,9 @@ class RldpToTcpRequestSender : public td::actor::Actor { } void got_result(std::pair, std::shared_ptr> R) { - if (R.first->need_payload()) { - td::actor::create_actor("HttpPayloadSender(R)", std::move(R.second), id_, local_id_, adnl_, - rldp_) - .release(); - } + td::actor::create_actor("HttpPayloadSender(R)", std::move(R.second), id_, local_id_, adnl_, + rldp_) + .release(); auto f = ton::serialize_tl_object(R.first->store_tl(), true); promise_.set_value(std::move(f)); stop(); @@ -1092,6 +1090,7 @@ class RldpHttpProxy : public td::actor::Actor { } rldp_ = ton::rldp::Rldp::create(adnl_.get()); + td::actor::send_closure(rldp_, &ton::rldp::Rldp::set_default_mtu, 16 << 10); td::actor::send_closure(rldp_, &ton::rldp::Rldp::add_id, local_id_); for (auto &serv_id : server_ids_) { td::actor::send_closure(rldp_, &ton::rldp::Rldp::add_id, serv_id); diff --git a/rldp/rldp-in.hpp b/rldp/rldp-in.hpp index b498199932..9640fd98e2 100644 --- a/rldp/rldp-in.hpp +++ b/rldp/rldp-in.hpp @@ -71,7 +71,7 @@ class RldpIn : public RldpImpl { void send_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice data) override { - send_query_ex(src, dst, name, std::move(promise), timeout, std::move(data), default_mtu()); + send_query_ex(src, dst, name, std::move(promise), timeout, std::move(data), default_mtu_); } void send_query_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice data, @@ -101,6 +101,10 @@ class RldpIn : public RldpImpl { void add_id(adnl::AdnlNodeIdShort local_id) override; void get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, td::Promise promise) override; + void set_default_mtu(td::uint64 mtu) override { + default_mtu_ = mtu; + } + RldpIn(td::actor::ActorId adnl) : adnl_(adnl) { } @@ -116,6 +120,7 @@ class RldpIn : public RldpImpl { std::set lru_set_; RldpLru lru_; td::uint32 lru_size_ = 0; + td::uint64 default_mtu_ = adnl::Adnl::get_mtu(); std::map max_size_; diff --git a/rldp/rldp.cpp b/rldp/rldp.cpp index 9b38dcb8c4..1a772a688c 100644 --- a/rldp/rldp.cpp +++ b/rldp/rldp.cpp @@ -116,9 +116,9 @@ void RldpIn::process_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeId } auto ite = max_size_.find(part.transfer_id_); if (ite == max_size_.end()) { - if (static_cast(part.total_size_) > default_mtu()) { + if (static_cast(part.total_size_) > default_mtu_) { VLOG(RLDP_NOTICE) << "dropping too big rldp packet of size=" << part.total_size_ - << " default_mtu=" << default_mtu(); + << " default_mtu=" << default_mtu_; return; } } else { diff --git a/rldp/rldp.h b/rldp/rldp.h index edb19d7a4f..b8ce3623be 100644 --- a/rldp/rldp.h +++ b/rldp/rldp.h @@ -28,15 +28,13 @@ class Rldp : public adnl::AdnlSenderInterface { public: virtual ~Rldp() = default; - static constexpr td::uint64 default_mtu() { - return adnl::Adnl::get_mtu(); - } - virtual void add_id(adnl::AdnlNodeIdShort local_id) = 0; virtual void send_message_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, td::BufferSlice data) = 0; + virtual void set_default_mtu(td::uint64 mtu) = 0; + static td::actor::ActorOwn create(td::actor::ActorId adnl); }; From 5500acd1ba9fcc6782107ff0d10fc69ec48ae2c6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 5 Oct 2022 23:05:23 +0300 Subject: [PATCH 035/524] Minor bugfixes in http --- http/http.cpp | 45 +++++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/http/http.cpp b/http/http.cpp index cc1fd889d5..dd8ab80931 100644 --- a/http/http.cpp +++ b/http/http.cpp @@ -279,25 +279,20 @@ td::Status HttpPayload::parse(td::ChainBufferReader &input) { } break; case ParseState::reading_chunk_data: { if (cur_chunk_size_ == 0) { - switch (type_) { - case PayloadType::pt_empty: - UNREACHABLE(); - case PayloadType::pt_eof: - case PayloadType::pt_tunnel: - cur_chunk_size_ = 1LL << 60; - break; - case PayloadType::pt_chunked: - state_ = ParseState::reading_crlf; - break; - case PayloadType::pt_content_length: { - LOG(INFO) << "payload parse success"; - const std::lock_guard lock{mutex_}; - state_ = ParseState::completed; - run_callbacks(); - return td::Status::OK(); - } break; + if (type_ == PayloadType::pt_eof || type_ == PayloadType::pt_tunnel) { + cur_chunk_size_ = 1LL << 60; + } else if (type_ == PayloadType::pt_chunked) { + state_ = ParseState::reading_crlf; + break; + } else if (type_ == PayloadType::pt_content_length) { + LOG(INFO) << "payload parse success"; + const std::lock_guard lock{mutex_}; + state_ = ParseState::completed; + run_callbacks(); + return td::Status::OK(); + } else { + UNREACHABLE(); } - break; } if (input.size() == 0) { return td::Status::OK(); @@ -502,7 +497,7 @@ bool HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, Htt char buf[64]; ::sprintf(buf, "%lx\r\n", s.size()); auto slice = td::Slice(buf, strlen(buf)); - wrote |= !slice.empty(); + wrote = true; output.append(slice); } @@ -514,7 +509,8 @@ bool HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, Htt wrote = true; } } - if (chunks_.size() != 0 || !parse_completed()) { + auto cur_state = state_.load(std::memory_order_consume); + if (chunks_.size() != 0 || (cur_state != ParseState::reading_trailer && cur_state != ParseState::completed)) { return wrote; } if (!written_zero_chunk_) { @@ -531,7 +527,7 @@ bool HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, Htt } while (max_size > 0) { - auto cur_state = state_.load(std::memory_order_consume); + cur_state = state_.load(std::memory_order_consume); HttpHeader h = get_header(); if (h.empty()) { if (cur_state != ParseState::completed) { @@ -587,7 +583,8 @@ tl_object_ptr HttpPayload::store_tl(size_t max_size) max_size -= s.size(); } obj->data_.truncate(obj->data_.size() - S.size()); - if (chunks_.size() != 0 || !parse_completed()) { + auto cur_state = state_.load(std::memory_order_consume); + if (chunks_.size() != 0 || (cur_state != ParseState::reading_trailer && cur_state != ParseState::completed)) { return obj; } if (!written_zero_chunk_) { @@ -597,7 +594,7 @@ tl_object_ptr HttpPayload::store_tl(size_t max_size) LOG(INFO) << "data completed"; while (max_size > 0) { - auto cur_state = state_.load(std::memory_order_consume); + cur_state = state_.load(std::memory_order_consume); HttpHeader h = get_header(); if (h.empty()) { if (cur_state != ParseState::completed) { @@ -869,7 +866,7 @@ td::Status HttpHeader::basic_check() { } for (auto &c : value) { if (c == '\r' || c == '\n') { - return td::Status::Error("bad character in header name"); + return td::Status::Error("bad character in header value"); } } return td::Status::OK(); From 699a56b951b196aa96d819889e08ea9a2fd83563 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 27 Sep 2022 16:33:27 +0300 Subject: [PATCH 036/524] Send block broadcasts directly to collators --- crypto/block/block.tlb | 3 ++- crypto/block/mc-config.cpp | 9 +++++++-- crypto/block/mc-config.h | 1 + overlay/overlay-broadcast.cpp | 6 +++++- overlay/overlay-broadcast.hpp | 6 ++++-- overlay/overlay-fec-broadcast.cpp | 20 +++++++++++++------- overlay/overlay-fec-broadcast.hpp | 15 ++++++++++----- overlay/overlay-manager.cpp | 13 +++++++++++++ overlay/overlay-manager.h | 3 +++ overlay/overlay.h | 1 + overlay/overlay.hpp | 19 ++++++++++++++++++- overlay/overlays.h | 3 +++ validator/full-node-shard.cpp | 13 +++++++++++++ validator/full-node-shard.h | 1 + validator/full-node-shard.hpp | 2 ++ validator/full-node.cpp | 30 ++++++++++++++++++++++++++++++ validator/full-node.hpp | 18 +++++++++++------- validator/impl/shard.cpp | 4 ++++ validator/impl/shard.hpp | 1 + validator/interfaces/shard.h | 1 + 20 files changed, 143 insertions(+), 26 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 8cb6c2f0d5..f1a1c9485a 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -735,7 +735,8 @@ misbehaviour_punishment_config_v1#01 _ MisbehaviourPunishmentConfig = ConfigParam 40; // collator_nodes: each collator is (workchain:int32 shard:uint64 adnl_id:uint256) -colator_config#a0 full_collated_data:Bool collator_nodes:(HashmapE 352 Unit) = CollatorConfig; +collator_info#0 full_node_id:(Maybe uint256) = CollatorInfo; +colator_config#a0 full_collated_data:Bool collator_nodes:(HashmapE 352 CollatorInfo) = CollatorConfig; _ CollatorConfig = ConfigParam 41; oracle_bridge_params#_ bridge_address:bits256 oracle_mutlisig_address:bits256 oracles:(HashmapE 256 uint256) external_chain_address:bits256 = OracleBridgeParams; diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index a1ddfb3ef7..ed79c8b633 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2142,14 +2142,19 @@ CollatorConfig Config::get_collator_config(bool need_collator_nodes) const { collator_config.full_collated_data = rec.full_collated_data; if (need_collator_nodes) { vm::Dictionary dict{rec.collator_nodes->prefetch_ref(), 32 + 64 + 256}; - dict.check_for_each([&](Ref, td::ConstBitPtr key, int n) { + dict.check_for_each([&](Ref value, td::ConstBitPtr key, int n) { CHECK(n == 32 + 64 + 256); auto workchain = (td::int32)key.get_int(32); key.advance(32); td::uint64 shard = key.get_uint(64); key.advance(64); td::Bits256 adnl_id(key); - collator_config.collator_nodes.push_back({ton::ShardIdFull(workchain, shard), adnl_id}); + td::Bits256 full_node_id = td::Bits256::zero(); + gen::CollatorInfo::Record info; + if (tlb::csr_unpack(std::move(value), info) && info.full_node_id->size() == 257) { + full_node_id = td::Bits256(info.full_node_id->data_bits() + 1); + } + collator_config.collator_nodes.push_back({ton::ShardIdFull(workchain, shard), adnl_id, full_node_id}); return true; }); } diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 7058117ba9..e00a045ef5 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -485,6 +485,7 @@ class ShardConfig { struct CollatorNodeDescr { ton::ShardIdFull shard; ton::NodeIdShort adnl_id; + ton::NodeIdShort full_node_id; }; struct CollatorConfig { diff --git a/overlay/overlay-broadcast.cpp b/overlay/overlay-broadcast.cpp index ecd21e7afb..bcb7c28587 100644 --- a/overlay/overlay-broadcast.cpp +++ b/overlay/overlay-broadcast.cpp @@ -69,6 +69,10 @@ td::Status BroadcastSimple::run_checks() { td::Status BroadcastSimple::distribute() { auto B = serialize(); auto nodes = overlay_->get_neighbours(5); + if (is_ours_) { + auto priority_nodes = overlay_->get_priority_broadcast_receivers(3); + nodes.insert(nodes.end(), priority_nodes.begin(), priority_nodes.end()); + } auto manager = overlay_->overlay_manager(); for (auto &n : nodes) { @@ -140,7 +144,7 @@ td::Status BroadcastSimple::create_new(td::actor::ActorId overlay, auto date = static_cast(td::Clocks::system()); auto B = std::make_unique(broadcast_hash, PublicKey{}, nullptr, flags, std::move(data), date, - td::BufferSlice{}, false, nullptr, adnl::AdnlNodeIdShort::zero()); + td::BufferSlice{}, false, nullptr, adnl::AdnlNodeIdShort::zero(), true); auto to_sign = B->to_sign(); auto P = td::PromiseCreator::lambda( diff --git a/overlay/overlay-broadcast.hpp b/overlay/overlay-broadcast.hpp index e7b39eecc0..906a94a0d6 100644 --- a/overlay/overlay-broadcast.hpp +++ b/overlay/overlay-broadcast.hpp @@ -46,6 +46,7 @@ class BroadcastSimple : public td::ListNode { td::uint32 date_; td::BufferSlice signature_; bool is_valid_{false}; + bool is_ours_{false}; OverlayImpl *overlay_; @@ -63,7 +64,7 @@ class BroadcastSimple : public td::ListNode { public: BroadcastSimple(Overlay::BroadcastHash broadcast_hash, PublicKey source, std::shared_ptr cert, td::uint32 flags, td::BufferSlice data, td::uint32 date, td::BufferSlice signature, bool is_valid, - OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id) + OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, bool is_ours = false) : broadcast_hash_(broadcast_hash) , source_(std::move(source)) , cert_(std::move(cert)) @@ -73,7 +74,8 @@ class BroadcastSimple : public td::ListNode { , signature_(std::move(signature)) , is_valid_(is_valid) , overlay_(overlay) - , src_peer_id_(src_peer_id) { + , src_peer_id_(src_peer_id) + , is_ours_(is_ours) { } Overlay::BroadcastHash get_hash() const { diff --git a/overlay/overlay-fec-broadcast.cpp b/overlay/overlay-fec-broadcast.cpp index 7ff08309d1..4297ea0cc8 100644 --- a/overlay/overlay-fec-broadcast.cpp +++ b/overlay/overlay-fec-broadcast.cpp @@ -26,8 +26,8 @@ namespace overlay { td::Result> BroadcastFec::create(Overlay::BroadcastHash hash, PublicKey src, Overlay::BroadcastDataHash data_hash, td::uint32 flags, - td::uint32 date, fec::FecType fec_type) { - auto F = std::make_unique(hash, std::move(src), data_hash, flags, date, std::move(fec_type)); + td::uint32 date, fec::FecType fec_type, bool is_ours) { + auto F = std::make_unique(hash, std::move(src), data_hash, flags, date, std::move(fec_type), is_ours); TRY_STATUS(F->init_fec_type()); TRY_STATUS(F->run_checks()); return std::move(F); @@ -94,12 +94,12 @@ void BroadcastFec::broadcast_checked(td::Result R) { overlay_->deliver_broadcast(get_source().compute_short_id(), data_.clone()); auto manager = overlay_->overlay_manager(); while (!parts_.empty()) { - distribute_part(parts_.begin()->first); + distribute_part(parts_.begin()->first); } } // Do we need status here?? -td::Status BroadcastFec::distribute_part(td::uint32 seqno) { +td::Status BroadcastFec::distribute_part(td::uint32 seqno) { auto i = parts_.find(seqno); if (i == parts_.end()) { // should not get here @@ -111,8 +111,12 @@ td::Status BroadcastFec::distribute_part(td::uint32 seqno) { td::BufferSlice data = std::move(tls.second); auto nodes = overlay_->get_neighbours(5); - auto manager = overlay_->overlay_manager(); + if (is_ours_) { + auto priority_nodes = overlay_->get_priority_broadcast_receivers(3); + nodes.insert(nodes.end(), priority_nodes.begin(), priority_nodes.end()); + } + auto manager = overlay_->overlay_manager(); for (auto &n : nodes) { if (neighbour_completed(n)) { continue; @@ -140,7 +144,8 @@ td::Status OverlayFecBroadcastPart::apply() { if (is_short_) { return td::Status::Error(ErrorCode::protoviolation, "short broadcast part for incomplete broadcast"); } - TRY_RESULT(B, BroadcastFec::create(broadcast_hash_, source_, broadcast_data_hash_, flags_, date_, fec_type_)); + TRY_RESULT( + B, BroadcastFec::create(broadcast_hash_, source_, broadcast_data_hash_, flags_, date_, fec_type_, is_ours_)); bcast_ = B.get(); overlay_->register_fec_broadcast(std::move(B)); } @@ -304,7 +309,8 @@ td::Status OverlayFecBroadcastPart::create_new(OverlayImpl *overlay, td::actor:: auto B = std::make_unique( broadcast_hash, part_hash, PublicKey{}, overlay->get_certificate(local_id), data_hash, size, flags, - part_data_hash, std::move(part), seqno, std::move(fec_type), date, td::BufferSlice{}, false, nullptr, overlay, adnl::AdnlNodeIdShort::zero()); + part_data_hash, std::move(part), seqno, std::move(fec_type), date, td::BufferSlice{}, false, nullptr, overlay, + adnl::AdnlNodeIdShort::zero(), true); auto to_sign = B->to_sign(); auto P = td::PromiseCreator::lambda( diff --git a/overlay/overlay-fec-broadcast.hpp b/overlay/overlay-fec-broadcast.hpp index 612af22fbc..7f038b41dd 100644 --- a/overlay/overlay-fec-broadcast.hpp +++ b/overlay/overlay-fec-broadcast.hpp @@ -131,18 +131,19 @@ class BroadcastFec : public td::ListNode { td::Status run_checks(); BroadcastFec(Overlay::BroadcastHash hash, PublicKey src, Overlay::BroadcastDataHash data_hash, td::uint32 flags, - td::uint32 date, fec::FecType fec_type) + td::uint32 date, fec::FecType fec_type, bool is_ours = false) : hash_(hash) , data_hash_(data_hash) , flags_(flags) , date_(date) , src_(std::move(src)) - , fec_type_(std::move(fec_type)) { + , fec_type_(std::move(fec_type)) + , is_ours_(is_ours) { } static td::Result> create(Overlay::BroadcastHash hash, PublicKey src, Overlay::BroadcastDataHash data_hash, td::uint32 flags, - td::uint32 date, fec::FecType fec_type); + td::uint32 date, fec::FecType fec_type, bool is_ours); bool neighbour_received(adnl::AdnlNodeIdShort id) const { return received_neighbours_.find(id) != received_neighbours_.end(); @@ -225,6 +226,7 @@ class BroadcastFec : public td::ListNode { OverlayImpl *overlay_; adnl::AdnlNodeIdShort src_peer_id_ = adnl::AdnlNodeIdShort::zero(); td::BufferSlice data_; + bool is_ours_ = false; }; class OverlayFecBroadcastPart : public td::ListNode { @@ -246,6 +248,7 @@ class OverlayFecBroadcastPart : public td::ListNode { bool is_short_; bool untrusted_{false}; + bool is_ours_; BroadcastFec *bcast_; OverlayImpl *overlay_; @@ -265,7 +268,8 @@ class OverlayFecBroadcastPart : public td::ListNode { std::shared_ptr cert, Overlay::BroadcastDataHash data_hash, td::uint32 data_size, td::uint32 flags, Overlay::BroadcastDataHash part_data_hash, td::BufferSlice data, td::uint32 seqno, fec::FecType fec_type, td::uint32 date, td::BufferSlice signature, - bool is_short, BroadcastFec *bcast, OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id) + bool is_short, BroadcastFec *bcast, OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + bool is_ours = false) : broadcast_hash_(broadcast_hash) , part_hash_(part_hash) , source_(std::move(source)) @@ -282,7 +286,8 @@ class OverlayFecBroadcastPart : public td::ListNode { , is_short_(is_short) , bcast_(bcast) , overlay_(overlay) - , src_peer_id_(src_peer_id) { + , src_peer_id_(src_peer_id) + , is_ours_(is_ours) { } td::uint32 data_size() const { diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index 2134bbc0c6..96e9fc2a9e 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -359,6 +359,19 @@ void OverlayManager::get_stats(td::Promise nodes) { + auto it = overlays_.find(local_id); + if (it == overlays_.end()) { + return; + } + auto it2 = it->second.find(overlay); + if (it2 == it->second.end()) { + return; + } + td::actor::send_closure(it2->second, &Overlay::set_priority_broadcast_receivers, std::move(nodes)); +} + Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, td::BufferSlice signature) : issued_by_(issued_by) diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index ada37303d9..ed9da23b56 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -96,6 +96,9 @@ class OverlayManager : public Overlays { td::actor::ActorOwn overlay); void get_stats(td::Promise> promise) override; + void set_priority_broadcast_receivers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, + std::vector nodes) override; + struct PrintId {}; PrintId print_id() const { diff --git a/overlay/overlay.h b/overlay/overlay.h index 146be73836..fcf61ea841 100644 --- a/overlay/overlay.h +++ b/overlay/overlay.h @@ -69,6 +69,7 @@ class Overlay : public td::actor::Actor { virtual void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) = 0; //virtual void receive_broadcast(td::BufferSlice data) = 0; //virtual void subscribe(std::unique_ptr callback) = 0; + virtual void set_priority_broadcast_receivers(std::vector nodes) = 0; }; } // namespace overlay diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 197864946a..07003e0986 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -192,7 +192,19 @@ class OverlayImpl : public Overlay { } else { std::vector vec; for (td::uint32 i = 0; i < max_size; i++) { - vec.push_back(neighbours_[td::Random::fast(0, static_cast(neighbours_.size()))]); + vec.push_back(neighbours_[td::Random::fast(0, static_cast(neighbours_.size()) - 1)]); + } + return vec; + } + } + std::vector get_priority_broadcast_receivers(td::uint32 max_size = 0) const { + if (max_size == 0 || max_size >= priority_broadcast_receivers_.size()) { + return priority_broadcast_receivers_; + } else { + std::vector vec; + for (td::uint32 i = 0; i < max_size; i++) { + vec.push_back(priority_broadcast_receivers_[td::Random::fast( + 0, static_cast(priority_broadcast_receivers_.size()) - 1)]); } return vec; } @@ -250,6 +262,10 @@ class OverlayImpl : public Overlay { } } + void set_priority_broadcast_receivers(std::vector nodes) override { + priority_broadcast_receivers_ = std::move(nodes); + } + private: template void process_query(adnl::AdnlNodeIdShort src, T &query, td::Promise promise) { @@ -309,6 +325,7 @@ class OverlayImpl : public Overlay { std::queue bcast_lru_; std::map> out_fec_bcasts_; + std::vector priority_broadcast_receivers_; void bcast_gc(); diff --git a/overlay/overlays.h b/overlay/overlays.h index 0a9c7765c8..a7b29faf88 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -238,6 +238,9 @@ class Overlays : public td::actor::Actor { virtual void get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, td::uint32 max_peers, td::Promise> promise) = 0; virtual void get_stats(td::Promise> promise) = 0; + + virtual void set_priority_broadcast_receivers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, + std::vector nodes) = 0; }; } // namespace overlay diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 56e0c1db19..e491061ed0 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -127,6 +127,10 @@ void FullNodeShardImpl::create_overlay() { if (cert_) { td::actor::send_closure(overlays_, &overlay::Overlays::update_certificate, adnl_id_, overlay_id_, local_id_, cert_); } + if (!collator_nodes_.empty()) { + td::actor::send_closure(overlays_, &overlay::Overlays::set_priority_broadcast_receivers, adnl_id_, overlay_id_, + collator_nodes_); + } } void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broadcast, td::Promise promise) { @@ -1031,6 +1035,15 @@ void FullNodeShardImpl::update_validators(std::vector public_key_ } } +void FullNodeShardImpl::update_collators(std::vector nodes) { + if (!client_.empty()) { + return; + } + collator_nodes_ = std::move(nodes); + td::actor::send_closure(overlays_, &overlay::Overlays::set_priority_broadcast_receivers, adnl_id_, overlay_id_, + collator_nodes_); +} + void FullNodeShardImpl::reload_neighbours() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index e24873aacd..847091c652 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -75,6 +75,7 @@ class FullNodeShard : public td::actor::Actor { virtual void set_handle(BlockHandle handle, td::Promise promise) = 0; virtual void update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) = 0; + virtual void update_collators(std::vector nodes) = 0; static td::actor::ActorOwn create( ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index cd7c20a95e..1cb2957de1 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -181,6 +181,7 @@ class FullNodeShardImpl : public FullNodeShard { void alarm() override; void update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) override; + void update_collators(std::vector nodes) override; void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) override; void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) override; @@ -258,6 +259,7 @@ class FullNodeShardImpl : public FullNodeShard { adnl::AdnlNodeIdShort last_pinged_neighbour_ = adnl::AdnlNodeIdShort::zero(); FullNodeShardMode mode_; + std::vector collator_nodes_; }; } // namespace fullnode diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 6792f969fc..52d3cfb977 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -205,6 +205,33 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s ++it; } } + + if (!collators_inited_ || state->is_key_state()) { + update_collators(state); + } +} + +void FullNodeImpl::update_collators(td::Ref state) { + collators_inited_ = true; + collator_config_ = state->get_collator_config(true); + for (auto& s : shards_) { + update_shard_collators(s.first, s.second); + } +} + +void FullNodeImpl::update_shard_collators(ShardIdFull shard, ShardInfo& info) { + if (info.actor.empty()) { + return; + } + std::vector nodes; + for (const block::CollatorNodeDescr& desc : collator_config_.collator_nodes) { + if (!desc.full_node_id.is_zero() && shard_intersects(shard, desc.shard)) { + nodes.emplace_back(desc.full_node_id); + } + } + std::sort(nodes.begin(), nodes.end()); + nodes.erase(std::unique(nodes.begin(), nodes.end()), nodes.end()); + td::actor::send_closure(info.actor, &FullNodeShard::update_collators, std::move(nodes)); } void FullNodeImpl::add_shard_actor(ShardIdFull shard, FullNodeShardMode mode) { @@ -219,6 +246,9 @@ void FullNodeImpl::add_shard_actor(ShardIdFull shard, FullNodeShardMode mode) { if (all_validators_.size() > 0) { td::actor::send_closure(info.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } + if (collators_inited_) { + update_shard_collators(shard, info); + } } void FullNodeImpl::sync_completed() { diff --git a/validator/full-node.hpp b/validator/full-node.hpp index fbe60d1b22..8cc8994b1d 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -92,7 +92,16 @@ class FullNodeImpl : public FullNode { td::Promise started_promise); private: + struct ShardInfo { + bool exists = false; + td::actor::ActorOwn actor; + FullNodeShardMode mode = FullNodeShardMode::inactive; + td::Timestamp delete_at = td::Timestamp::never(); + }; + void add_shard_actor(ShardIdFull shard, FullNodeShardMode mode); + void update_collators(td::Ref state); + void update_shard_collators(ShardIdFull shard, ShardInfo& info); PublicKeyHash local_id_; adnl::AdnlNodeIdShort adnl_id_; @@ -100,13 +109,6 @@ class FullNodeImpl : public FullNode { td::actor::ActorId get_shard(AccountIdPrefixFull dst); td::actor::ActorId get_shard(ShardIdFull shard, bool exact = false); - - struct ShardInfo { - bool exists = false; - td::actor::ActorOwn actor; - FullNodeShardMode mode = FullNodeShardMode::inactive; - td::Timestamp delete_at = td::Timestamp::never(); - }; std::map shards_; td::actor::ActorId keyring_; @@ -125,6 +127,8 @@ class FullNodeImpl : public FullNode { std::set local_keys_; td::Promise started_promise_; + bool collators_inited_ = false; + block::CollatorConfig collator_config_; }; } // namespace fullnode diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 019ecbeb02..a09bd1df83 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -556,5 +556,9 @@ BlockIdExt MasterchainStateQ::prev_key_block_id(BlockSeqno seqno) const { return block_id; } +bool MasterchainStateQ::is_key_state() const { + return config_ ? config_->is_key_state() : false; +} + } // namespace validator } // namespace ton diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index 6b95c7e409..c726447fcf 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -129,6 +129,7 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { BlockIdExt last_key_block_id() const override; BlockIdExt next_key_block_id(BlockSeqno seqno) const override; BlockIdExt prev_key_block_id(BlockSeqno seqno) const override; + bool is_key_state() const override; MasterchainStateQ* make_copy() const override; static td::Result> fetch(const BlockIdExt& _id, td::BufferSlice _data, diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 15caa7b270..59211a8551 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -77,6 +77,7 @@ class MasterchainState : virtual public ShardState { virtual BlockIdExt last_key_block_id() const = 0; virtual BlockIdExt next_key_block_id(BlockSeqno seqno) const = 0; virtual BlockIdExt prev_key_block_id(BlockSeqno seqno) const = 0; + virtual bool is_key_state() const = 0; virtual bool get_old_mc_block_id(ton::BlockSeqno seqno, ton::BlockIdExt& blkid, ton::LogicalTime* end_lt = nullptr) const = 0; virtual bool check_old_mc_block_id(const ton::BlockIdExt& blkid, bool strict = false) const = 0; From 9fb986f6f5a65ef573ba383503649565cae9c9f1 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 28 Sep 2022 13:16:22 +0300 Subject: [PATCH 037/524] Disable downloading archives during sync --- validator/manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index 2b6018364d..95ef6bf06a 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1680,7 +1680,7 @@ void ValidatorManagerImpl::prestart_sync() { } void ValidatorManagerImpl::download_next_archive() { - if (!out_of_sync()) { + if (true) { finish_prestart_sync(); return; } From c2dde00459183caf39941bb8c45fb687b0e8c2ff Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 28 Sep 2022 14:33:17 +0300 Subject: [PATCH 038/524] Improve retries in overlay.cpp --- overlay/overlay.cpp | 66 +++++++++++++++++++++++---------------------- overlay/overlay.hpp | 2 ++ 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 8e35e2a252..3e10daec23 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -421,48 +421,50 @@ void OverlayImpl::bcast_gc() { CHECK(delivered_broadcasts_.size() == bcast_lru_.size()); } -void OverlayImpl::send_message_to_neighbours(td::BufferSlice data) { - if (neighbours_.empty()) { - // TODO: limit retries +void OverlayImpl::wait_neighbours_not_empty(td::Promise promise, int max_retries) { + if (!neighbours_.empty()) { + promise.set_result(td::Unit()); + } else if (max_retries > 0) { delay_action( - [SelfId = actor_id(this), data = std::move(data)]() mutable { - td::actor::send_closure(SelfId, &OverlayImpl::send_message_to_neighbours, std::move(data)); + [SelfId = actor_id(this), promise = std::move(promise), max_retries]() mutable { + td::actor::send_closure(SelfId, &OverlayImpl::wait_neighbours_not_empty, std::move(promise), max_retries - 1); }, td::Timestamp::in(0.5)); - return; - } - for (auto &n : neighbours_) { - td::actor::send_closure(manager_, &OverlayManager::send_message, n, local_id_, overlay_id_, data.clone()); + } else { + promise.set_error(td::Status::Error(ErrorCode::timeout)); } } +void OverlayImpl::send_message_to_neighbours(td::BufferSlice data) { + wait_neighbours_not_empty([this, data = std::move(data)](td::Result R) { + if (R.is_error()) { + return; + } + for (auto &n : neighbours_) { + td::actor::send_closure(manager_, &OverlayManager::send_message, n, local_id_, overlay_id_, data.clone()); + } + }); +} + void OverlayImpl::send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { - if (neighbours_.empty()) { - // TODO: limit retries - delay_action( - [SelfId = actor_id(this), send_as, flags, data = std::move(data)]() mutable { - td::actor::send_closure(SelfId, &OverlayImpl::send_broadcast, send_as, flags, std::move(data)); - }, - td::Timestamp::in(0.5)); - return; - } - auto S = BroadcastSimple::create_new(actor_id(this), keyring_, send_as, std::move(data), flags); - if (S.is_error()) { - LOG(WARNING) << "failed to send broadcast: " << S; - } + wait_neighbours_not_empty([this, send_as, flags, data = std::move(data)](td::Result R) mutable { + if (R.is_error()) { + return; + } + auto S = BroadcastSimple::create_new(actor_id(this), keyring_, send_as, std::move(data), flags); + if (S.is_error()) { + LOG(WARNING) << "failed to send broadcast: " << S; + } + }); } void OverlayImpl::send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { - if (neighbours_.empty()) { - // TODO: limit retries - delay_action( - [SelfId = actor_id(this), send_as, flags, data = std::move(data)]() mutable { - td::actor::send_closure(SelfId, &OverlayImpl::send_broadcast_fec, send_as, flags, std::move(data)); - }, - td::Timestamp::in(0.5)); - return; - } - OverlayOutboundFecBroadcast::create(std::move(data), flags, actor_id(this), send_as); + wait_neighbours_not_empty([this, send_as, flags, data = std::move(data)](td::Result R) mutable { + if (R.is_error()) { + return; + } + OverlayOutboundFecBroadcast::create(std::move(data), flags, actor_id(this), send_as); + }); } void OverlayImpl::print(td::StringBuilder &sb) { diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 07003e0986..f52de86d6b 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -266,6 +266,8 @@ class OverlayImpl : public Overlay { priority_broadcast_receivers_ = std::move(nodes); } + void wait_neighbours_not_empty(td::Promise promise, int max_retries = 10); + private: template void process_query(adnl::AdnlNodeIdShort src, T &query, td::Promise promise) { From 81c0e920c5d13764b8bf40a7e5e69efa296eb16d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 3 Oct 2022 17:55:37 +0300 Subject: [PATCH 039/524] Simplify selecting shards for monitor --- validator/full-node-shard.cpp | 2 +- validator/full-node.cpp | 59 +++++++++++++++++++-------------- validator/full-node.hpp | 2 +- validator/manager-init.cpp | 2 +- validator/manager.cpp | 16 ++++----- validator/manager.hpp | 5 ++- validator/shard-client.cpp | 23 +++---------- validator/state-serializer.cpp | 24 +++++--------- validator/state-serializer.hpp | 1 - validator/validator-options.hpp | 5 +-- validator/validator.h | 2 +- 11 files changed, 66 insertions(+), 75 deletions(-) diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index e491061ed0..d048f61125 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -670,7 +670,7 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ne void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { BlockIdExt block_id = create_block_id(query.id_); - //if (block_id.shard_full() != shard_) { + //if (!shard_is_ancestor(shard_, block_id.shard_full())) { // LOG(FULL_NODE_WARNING) << "dropping block broadcast: shard mismatch. overlay=" << shard_.to_str() // << " block=" << block_id.to_str(); // return; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 52d3cfb977..72af6fe8e8 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -135,10 +135,18 @@ void FullNodeImpl::initial_read_complete(BlockHandle top_handle) { void FullNodeImpl::update_shard_configuration(td::Ref state, std::set shards_to_monitor, std::set temporary_shards) { CHECK(shards_to_monitor.count(ShardIdFull(masterchainId))); - std::map new_shards; + std::set new_shards; std::map new_active; - new_shards[ShardIdFull(masterchainId)] = state->get_block_id(); + new_shards.insert(ShardIdFull(masterchainId)); std::set workchains; + auto cut_shard = [&](ShardIdFull shard) -> ShardIdFull { + unsigned pfx_len = shard.pfx_len(); + unsigned min_split = state->soft_min_split_depth(shard.workchain); + if (min_split < pfx_len) { + return shard_prefix(shard, min_split); + } + return shard; + }; auto set_active = [&](ShardIdFull shard, FullNodeShardMode mode) { while (new_active.emplace(shard, mode).second && shard.pfx_len() > 0) { shard = shard_parent(shard); @@ -146,20 +154,20 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s }; for (auto &info : state->get_shards()) { workchains.insert(info->shard().workchain); - new_shards[info->shard()] = info->top_block_id(); + new_shards.insert(cut_shard(info->shard())); } for (const auto &wpair : state->get_workchain_list()) { ton::WorkchainId wc = wpair.first; const block::WorkchainInfo *winfo = wpair.second.get(); if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= state->get_unix_time()) { - new_shards[ShardIdFull(wc)] = BlockIdExt(wc, shardIdAll, 0, winfo->zerostate_root_hash, winfo->zerostate_file_hash); + new_shards.insert(ShardIdFull(wc)); } } for (ShardIdFull shard : shards_to_monitor) { - set_active(shard, FullNodeShardMode::active); + set_active(cut_shard(shard), FullNodeShardMode::active); } for (ShardIdFull shard : temporary_shards) { - set_active(shard, FullNodeShardMode::active_temp); + set_active(cut_shard(shard), FullNodeShardMode::active_temp); } auto info_set_mode = [&](ShardIdFull shard, ShardInfo& info, FullNodeShardMode mode) { @@ -177,7 +185,7 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s }; for (auto shard : new_shards) { - auto &info = shards_[shard.first]; + auto &info = shards_[shard]; info.exists = true; } @@ -283,7 +291,7 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s } void FullNodeImpl::send_broadcast(BlockBroadcast broadcast) { - auto shard = get_shard(broadcast.block_id.shard_full(), true); + auto shard = get_shard(broadcast.block_id.shard_full()); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; return; @@ -379,27 +387,28 @@ void FullNodeImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull std::move(promise)); } -td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard, bool exact) { - if (!exact) { - ShardIdFull s = shard; - while (true) { - auto it = shards_.find(s); - if (it != shards_.end() && it->second.exists) { - if (it->second.actor.empty()) { - add_shard_actor(s, FullNodeShardMode::inactive); - } - if (it->second.mode == FullNodeShardMode::inactive) { - it->second.delete_at = td::Timestamp::in(INACTIVE_SHARD_TTL); - } - return it->second.actor.get(); +td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { + while (true) { + auto it = shards_.find(shard); + if (it != shards_.end() && it->second.exists) { + if (it->second.actor.empty()) { + add_shard_actor(shard, FullNodeShardMode::inactive); } - if (s.pfx_len() == 0) { - break; + if (it->second.mode == FullNodeShardMode::inactive) { + it->second.delete_at = td::Timestamp::in(INACTIVE_SHARD_TTL); } - s = shard_parent(s); + return it->second.actor.get(); + } + if (shard.pfx_len() == 0) { + break; } + shard = shard_parent(shard); + } + auto it = shards_.find(shard); + if (it == shards_.end()) { + it = shards_.emplace(shard = ShardIdFull(shard.workchain), ShardInfo{}).first; } - auto &info = shards_[shard]; + auto &info = it->second; if (info.actor.empty()) { add_shard_actor(shard, FullNodeShardMode::inactive); } diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 8cc8994b1d..503ba2866c 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -108,7 +108,7 @@ class FullNodeImpl : public FullNode { FileHash zero_state_file_hash_; td::actor::ActorId get_shard(AccountIdPrefixFull dst); - td::actor::ActorId get_shard(ShardIdFull shard, bool exact = false); + td::actor::ActorId get_shard(ShardIdFull shard); std::map shards_; td::actor::ActorId keyring_; diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index 1a322c02c5..8935664e36 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -566,7 +566,7 @@ void ValidatorManagerMasterchainStarter::truncated() { truncate_shard_next(handle_->id(), ig.get_promise()); auto s = state_->get_shards(); for (auto &shard : s) { - if (opts_->need_monitor(shard->shard())) { + if (opts_->need_monitor(shard->shard(), state_)) { truncate_shard_next(shard->top_block_id(), ig.get_promise()); } } diff --git a/validator/manager.cpp b/validator/manager.cpp index 95ef6bf06a..2a17af5ed1 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -214,7 +214,7 @@ void ValidatorManagerImpl::prevalidate_block(BlockBroadcast broadcast, td::Promi promise.set_error(td::Status::Error(ErrorCode::notready, "node not started")); return; } - if (!shards_to_monitor_.count(broadcast.block_id.shard_full())) { + if (!need_monitor(broadcast.block_id.shard_full())) { promise.set_error(td::Status::Error("not monitoring shard")); return; } @@ -457,7 +457,7 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id().shard_full(), desc->catchain_seqno()}] = desc; VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); - if (shards_to_monitor_.count(desc->block_id().shard_full())) { + if (need_monitor(desc->block_id().shard_full())) { auto P = td::PromiseCreator::lambda([](td::Result> R) { if (R.is_error()) { auto S = R.move_as_error(); @@ -617,10 +617,9 @@ void ValidatorManagerImpl::wait_out_msg_queue_proof(BlockIdExt block_id, ShardId td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_msg_queue, block_id, dst_shard, std::move(R)); }); - auto id = td::actor::create_actor("waitmsgqueue", block_id, dst_shard, - shards_to_monitor_.count(block_id.shard_full()), priority, - actor_id(this), td::Timestamp::at(timeout.at() + 10.0), - std::move(P)) + auto id = td::actor::create_actor( + "waitmsgqueue", block_id, dst_shard, need_monitor(block_id.shard_full()), priority, actor_id(this), + td::Timestamp::at(timeout.at() + 10.0), std::move(P)) .release(); wait_out_msg_queue_proof_[key].actor_ = id; it = wait_out_msg_queue_proof_.find(key); @@ -1083,8 +1082,8 @@ void ValidatorManagerImpl::finished_wait_msg_queue(BlockIdExt block_id, ShardIdF std::move(R)); }); auto id = td::actor::create_actor("waitmsgqueue", block_id, dst_shard, - shards_to_monitor_.count(block_id.shard_full()), - X.second, actor_id(this), X.first, std::move(P)) + need_monitor(block_id.shard_full()), X.second, + actor_id(this), X.first, std::move(P)) .release(); it->second.actor_ = id; return; @@ -2489,7 +2488,6 @@ void ValidatorManagerImpl::get_shard_client_state(bool from_db, td::Promise state, std::set shards_to_monitor) { - shards_to_monitor_ = shards_to_monitor; callback_->update_shard_configuration(std::move(state), std::move(shards_to_monitor), extra_active_shards_); } diff --git a/validator/manager.hpp b/validator/manager.hpp index ddef344147..3b32cfcdf2 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -628,11 +628,14 @@ class ValidatorManagerImpl : public ValidatorManager { td::Ref get_block_persistent_state(BlockIdExt block_id); private: + bool need_monitor(ShardIdFull shard) const { + return opts_->need_monitor(shard, last_masterchain_state_); + } + std::map> shard_client_waiters_; std::map> collator_nodes_; - std::set shards_to_monitor_ = {ShardIdFull(masterchainId)}; std::set extra_active_shards_; std::map last_validated_blocks_; diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index 431d772165..5c25b78305 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -91,7 +91,7 @@ void ShardClient::start_up_init_mode() { auto vec = masterchain_state_->get_shards(); for (auto &shard : vec) { - if (shards_to_monitor_.count(shard->shard())) { + if (opts_->need_monitor(shard->shard(), masterchain_state_)) { auto P = td::PromiseCreator::lambda([promise = ig.get_promise()](td::Result> R) mutable { R.ensure(); promise.set_value(td::Unit()); @@ -192,7 +192,7 @@ void ShardClient::apply_all_shards() { auto vec = masterchain_state_->get_shards(); for (auto &shard : vec) { - if (shards_to_monitor_.count(shard->shard())) { + if (opts_->need_monitor(shard->shard(), masterchain_state_)) { auto Q = td::PromiseCreator::lambda([SelfId = actor_id(this), promise = ig.get_promise(), shard = shard->shard()](td::Result> R) mutable { if (R.is_error()) { @@ -254,23 +254,9 @@ void ShardClient::build_shard_overlays() { for (const auto &info : masterchain_state_->get_shards()) { auto shard = info->shard(); workchains.insert(shard.workchain); - bool will_split = shard.pfx_len() < max_shard_pfx_len && - (info->fsm_state() == McShardHash::FsmState::fsm_split || info->before_split()); - bool will_merge = - shard.pfx_len() > 0 && (info->fsm_state() == McShardHash::FsmState::fsm_merge || info->before_merge()); - if (opts_->need_monitor(shard) || (will_merge && opts_->need_monitor(shard_parent(shard)))) { + if (opts_->need_monitor(shard, masterchain_state_)) { new_shards_to_monitor.insert(shard); } - if (will_merge && opts_->need_monitor(shard_parent(shard))) { - new_shards_to_monitor.insert(shard_parent(shard)); - } - if (will_split) { - for (int id = 0; id < 2; ++id) { - if (opts_->need_monitor(shard_child(shard, id))) { - new_shards_to_monitor.insert(shard_child(shard, id)); - } - } - } } std::vector new_workchains; @@ -278,7 +264,8 @@ void ShardClient::build_shard_overlays() { ton::WorkchainId wc = wpair.first; const block::WorkchainInfo *winfo = wpair.second.get(); auto shard = ShardIdFull(wc); - if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= cur_time && opts_->need_monitor(shard)) { + if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= cur_time && + opts_->need_monitor(shard, masterchain_state_)) { new_shards_to_monitor.insert(shard); if (shards_to_monitor_.count(shard) == 0) { new_workchains.push_back(BlockIdExt(wc, shardIdAll, 0, winfo->zerostate_root_hash, winfo->zerostate_file_hash)); diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index fc8cd29aaf..1c8d677c2c 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -159,16 +159,12 @@ void AsyncStateSerializer::next_iteration() { return; } while (next_idx_ < shards_.size()) { - if (!need_monitor(shards_[next_idx_].shard_full())) { - next_idx_++; - } else { - // block next attempts immediately, but send actual request later - running_ = true; - delay_action( - [SelfId = actor_id(this), shard = shards_[next_idx_]]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_shard_state, shard); }, - td::Timestamp::in(td::Random::fast(0, 4 * 3600))); - return; - } + // block next attempts immediately, but send actual request later + running_ = true; + delay_action( + [SelfId = actor_id(this), shard = shards_[next_idx_]]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_shard_state, shard); }, + td::Timestamp::in(td::Random::fast(0, 4 * 3600))); + return; } LOG(INFO) << "finished serializing persistent state for " << masterchain_handle_->id().id; last_key_block_ts_ = masterchain_handle_->unix_time(); @@ -245,7 +241,9 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state auto vec = state->get_shards(); for (auto &v : vec) { - shards_.push_back(v->top_block_id()); + if (opts_->need_monitor(v->shard(), state)) { + shards_.push_back(v->top_block_id()); + } } auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader = cell_db_reader_] (td::FileFd& fd) { @@ -311,10 +309,6 @@ void AsyncStateSerializer::success_handler() { next_iteration(); } -bool AsyncStateSerializer::need_monitor(ShardIdFull shard) { - return opts_->need_monitor(shard); -} - bool AsyncStateSerializer::need_serialize(BlockHandle handle) { if (handle->id().id.seqno == 0 || !handle->is_key_block()) { return false; diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index 0d6e955050..1472c3b4bf 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -60,7 +60,6 @@ class AsyncStateSerializer : public td::actor::Actor { } bool need_serialize(BlockHandle handle); - bool need_monitor(ShardIdFull shard); void alarm() override; void start_up() override; diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index b4a94a39bf..e96292ad3e 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -32,8 +32,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { BlockIdExt init_block_id() const override { return init_block_id_; } - bool need_monitor(ShardIdFull shard) const override { - return check_shard_(shard); + bool need_monitor(ShardIdFull shard, const td::Ref& state) const override { + td::uint32 min_split = state->min_split_depth(shard.workchain); + return check_shard_((td::uint32)shard.pfx_len() <= min_split ? shard : shard_prefix(shard, min_split)); } bool allow_blockchain_init() const override { return allow_blockchain_init_; diff --git a/validator/validator.h b/validator/validator.h index 0daccc1ae4..c52cbad7d8 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -49,7 +49,7 @@ struct ValidatorManagerOptions : public td::CntObject { public: virtual BlockIdExt zero_block_id() const = 0; virtual BlockIdExt init_block_id() const = 0; - virtual bool need_monitor(ShardIdFull shard) const = 0; + virtual bool need_monitor(ShardIdFull shard, const td::Ref& state) const = 0; virtual bool allow_blockchain_init() const = 0; virtual double sync_blocks_before() const = 0; virtual double block_ttl() const = 0; From 89771e2a2c354a4ead5ee160876b4e29e9ba5ab7 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 6 Oct 2022 13:53:53 +0300 Subject: [PATCH 040/524] Generate tlo and other changes after merge --- adnl/adnl-peer.cpp | 2 +- overlay/overlay-broadcast.hpp | 4 ++-- overlay/overlay-fec-broadcast.hpp | 6 +++--- rldp/rldp.cpp | 2 +- tl/generate/scheme/ton_api.tlo | Bin 69060 -> 73376 bytes 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index 1d73b8f6c2..46a097d956 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -621,7 +621,7 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQueryError return; } - td::actor::send_closure_later(Q->second, &AdnlQuery::reject_query); + td::actor::send_closure_later(Q->second, &AdnlQuery::set_error, td::Status::Error("adnl query rejected")); out_queries_.erase(Q); } diff --git a/overlay/overlay-broadcast.hpp b/overlay/overlay-broadcast.hpp index 906a94a0d6..8a5d132497 100644 --- a/overlay/overlay-broadcast.hpp +++ b/overlay/overlay-broadcast.hpp @@ -73,9 +73,9 @@ class BroadcastSimple : public td::ListNode { , date_(date) , signature_(std::move(signature)) , is_valid_(is_valid) + , is_ours_(is_ours) , overlay_(overlay) - , src_peer_id_(src_peer_id) - , is_ours_(is_ours) { + , src_peer_id_(src_peer_id) { } Overlay::BroadcastHash get_hash() const { diff --git a/overlay/overlay-fec-broadcast.hpp b/overlay/overlay-fec-broadcast.hpp index 7f038b41dd..c97ad88581 100644 --- a/overlay/overlay-fec-broadcast.hpp +++ b/overlay/overlay-fec-broadcast.hpp @@ -284,10 +284,10 @@ class OverlayFecBroadcastPart : public td::ListNode { , date_(date) , signature_(std::move(signature)) , is_short_(is_short) + , is_ours_(is_ours) , bcast_(bcast) - , overlay_(overlay) - , src_peer_id_(src_peer_id) - , is_ours_(is_ours) { + , overlay_(overlay) + , src_peer_id_(src_peer_id) { } td::uint32 data_size() const { diff --git a/rldp/rldp.cpp b/rldp/rldp.cpp index 952a7d6924..1834aac3fa 100644 --- a/rldp/rldp.cpp +++ b/rldp/rldp.cpp @@ -238,7 +238,7 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort ton_api::rldp_queryError &message) { auto it = queries_.find(message.query_id_); if (it != queries_.end()) { - td::actor::send_closure(it->second, &adnl::AdnlQuery::reject_query); + td::actor::send_closure(it->second, &adnl::AdnlQuery::set_error, td::Status::Error("adnl query rejected")); queries_.erase(it); } else { VLOG(RLDP_INFO) << "received reject to unknown query " << message.query_id_; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index d89402883f7b46eb1700a121d1d38ff367ee2df2..d535668aa423585bb2be3319dbf46d3f4f610efc 100644 GIT binary patch delta 3357 zcma)8e{2(V6z^@t$5uTfk7#5#1JH`guD{elGL0uCgu$a=@J%4?%a&P%l zxDfF!ms7rHD$KSlo>6tX0C9~G4*6=5H*4#($+X##MOXa%`4!nc)K8CLTzC%Gz_%SL z8FO*Zp39^DS6c*i*=c)+q7SZq?{~bfxZYM)J#Cyh8X>Nc(2N8hM03%2old@6d?u^( z#z&vsck)CYWm-Hl#jB3+bl|<>R3aXOasYyx2)Zj$004Vq_J~Md9 z$WG2}Rmwt(---khT4o6XFFZmw?0*(_PRvbu(C|8^wkr~O_Oz2EXK+2^biXR9oqlad z<@r}Akx1`SOH$KLkq?th#E zBP+)T8!W94(6EqQ!P6x+DB;Xtf^0MI9dHlNm7?(|pCimWA;rzy{*wO9nb@ zdK_B6?PwdD>I=(Ji!VN4`%#`Hi8kPF`GltYt6`On_FaqhHSbfRiM6E35#y&hW#0yw zI^lEuc1R@XbYK8^$|er%H-GWu#~9L4g>VxS;X-&B52(k|HsLof}GM#IYf4 zZYx^?rs;R;+iO$4A9J0G4(ZLxTTHq(iqM2Z%b*eBO<@VG~G$x_B^l?Eoh*h z?u3q#VB)s0(+0<2N_^xVbiTLTt8orpKmCzK|{*K0ZqB} z&8D1@(EhEDHRLeQHAEU=5AaO5Yt0y3z4n~Jpt-_-1f8^kLa(z>L(EEO#E`^?Vmuci zyn$~|dWHtq*tp)Sl8k?SYu=0flZ?2#nAPUvK&P2q$f5ZYFAt>yr_EqKH?e()Mn3b?1+Y0Br&Dk&5fcs3D6 zxM!^sx6Zdz186dRN`LL2`Z5osP@fKTadm8h4L~yQ2%GZcw)z~E9JgHvhn#bcilS*lzPrN2LLACL2-99{%P}wA*f?Np(X6BaRRqy%-<*3;OoPJ*u zSjfk>mKRJ08ZODCVFAU6D1b;g!#-d`T(mb0GTgoQC=Kkqd1g7T-b3S)`&@4LPY+;Z zUdzV83VSQWq=S7-@8ABIV)l}UyYm!9h7(sJK~e0is?dS*INW-vzYmY51f<^if1GlE`8ip&6S&U>fflW0j3y#gjYcA+>o0w)4BJB$A zIO0@5&L;K}lS!tVS+Tf8##qf2*l7s@nf%t5?O96lQ?rfoDCV$Ajqhv- zD#b3-UEL0fm9Ji%pXKqK>pfJWH4J60>cjaduGeEz!(5XgMjCo79Ez~SHLhH2HRi_8?*&giL4l!rDQh}yLTHXhFFl84+RB1NG}CRR4Dln)c$vy2%`U|Kflla`~Cm^zx&?! zb82anD*7^7cRDY-R%VJyNqfTo;;TgnD>Bz6s;UqXGnBsXD@6e^O`!lipKrIl9yo3Z zcX6sFXd-uB?8_WEgpXK?tt2{Ya}d;2*{fC5Q{txJ2J;d5%Vj`qr|;2lp(2F*j#&b_ zD|x&2`Ek$^m&37LSCOc{@*h!_4!zte(W1(QXjtT$g_zJK$&x0CDx4^&K(j}K0D9Rs z=u~wH1)9*g+Io8dZv4}-%}>CS^_7sHqhYeKio#X+zR|UVNgW3U^5F?jtAVe3SYtEp z;j;n{%jEp7k%c=qV8t(vO8`73QvzgFHajfc^4w!BVP?ndz}W_*n_;jEL!tr z@89-~=!em8QyLhuV2j%*jkK)Wkle9Q&fxLmhwg3+*PvH*?GJCO&i&!iK}uTV_g4nA z18j#6fGE*RnGtoX(^QSaQYtDDNfEhgf-HpvEgI*#qsnC|f*F3S8UEUG!I{B#Xnu$W zkRPlvGREFbm{G&Sn*g~-gQi*>GA5H0v0IizMa!CS5jol=$eDp*ZiHm`X!ekQzL#@^aYbL Date: Thu, 6 Oct 2022 15:13:10 +0300 Subject: [PATCH 041/524] Tonlib uses partial liteservers --- tl/generate/scheme/tonlib_api.tl | 2 +- tl/generate/scheme/tonlib_api.tlo | Bin 30836 -> 30904 bytes tonlib/CMakeLists.txt | 4 +- tonlib/tonlib/Config.cpp | 51 +++-- tonlib/tonlib/Config.h | 2 + tonlib/tonlib/ExtClient.cpp | 6 +- tonlib/tonlib/ExtClient.h | 15 +- tonlib/tonlib/ExtClientLazy.cpp | 24 +-- tonlib/tonlib/ExtClientMulti.cpp | 169 +++++++++++++++ tonlib/tonlib/ExtClientMulti.h | 61 ++++++ tonlib/tonlib/ExtClientOutbound.cpp | 8 +- tonlib/tonlib/ExtClientOutbound.h | 6 +- .../{ExtClientLazy.h => ExtClientRaw.h} | 15 +- tonlib/tonlib/QueryTraits.h | 202 ++++++++++++++++++ tonlib/tonlib/TonlibClient.cpp | 38 ++-- tonlib/tonlib/TonlibClient.h | 4 +- tonlib/tonlib/tonlib-cli.cpp | 31 ++- 17 files changed, 555 insertions(+), 83 deletions(-) create mode 100644 tonlib/tonlib/ExtClientMulti.cpp create mode 100644 tonlib/tonlib/ExtClientMulti.h rename tonlib/tonlib/{ExtClientLazy.h => ExtClientRaw.h} (68%) create mode 100644 tonlib/tonlib/QueryTraits.h diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index 79185d5bc1..e83ee78429 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -183,7 +183,7 @@ smc.runResult gas_used:int53 stack:vector exit_code:int32 = smc. smc.libraryEntry hash:int256 data:bytes = smc.LibraryEntry; smc.libraryResult result:(vector smc.libraryEntry) = smc.LibraryResult; -updateSendLiteServerQuery id:int64 data:bytes = Update; +updateSendLiteServerQuery id:int64 data:bytes workchain:int32 shard:int64 = Update; updateSyncState sync_state:SyncState = Update; //@class LogStream @description Describes a stream to which tonlib internal log is written diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index ee457609da33cca7655480b0531417fd19ccf958..5f3970a1197f2a1c2cdabfbcb6e8638fcc202586 100644 GIT binary patch delta 94 zcmezJfpN!2#tjSDc<1 Config::parse(std::string str) { if (json.type() != td::JsonValue::Type::Object) { return td::Status::Error("Invalid config (1)"); } - //TRY_RESULT(main_type, td::get_json_object_string_field(json.get_object(), "@type", false)); - //if (main_type != "config.global") { - //return td::Status::Error("Invalid config (3)"); - //} + td::JsonArray empty_array; TRY_RESULT(lite_clients_obj, - td::get_json_object_field(json.get_object(), "liteservers", td::JsonValue::Type::Array, false)); - auto &lite_clients = lite_clients_obj.get_array(); + td::get_json_object_field(json.get_object(), "liteservers", td::JsonValue::Type::Array, true)); + auto &lite_clients = + lite_clients_obj.type() == td::JsonValue::Type::Array ? lite_clients_obj.get_array() : empty_array; + TRY_RESULT(lite_clients_v2_obj, + td::get_json_object_field(json.get_object(), "liteservers_v2", td::JsonValue::Type::Array, true)); + auto &lite_clients_v2 = + lite_clients_v2_obj.type() == td::JsonValue::Type::Array ? lite_clients_v2_obj.get_array() : empty_array; - Config res; - for (auto &value : lite_clients) { + auto parse_desc = [&](td::JsonValue& value) -> td::Result { if (value.type() != td::JsonValue::Type::Object) { return td::Status::Error("Invalid config (2)"); } auto &object = value.get_object(); - //TRY_RESULT(value_type, td::get_json_object_string_field(object, "@type", false)); - //if (value_type != "liteclient.config.global") { - //return td::Status::Error("Invalid config (4)"); - //} TRY_RESULT(ip, td::get_json_object_long_field(object, "ip", false)); TRY_RESULT(port, td::get_json_object_int_field(object, "port", false)); @@ -93,15 +90,41 @@ td::Result Config::parse(std::string str) { auto &id = id_obj.get_object(); TRY_RESULT(id_type, td::get_json_object_string_field(id, "@type", false)); if (id_type != "pub.ed25519") { - return td::Status::Error("Invalid config (5)"); + return td::Status::Error("Invalid config (3)"); } TRY_RESULT(key_base64, td::get_json_object_string_field(id, "key", false)); TRY_RESULT(key, td::base64_decode(key_base64)); if (key.size() != 32) { - return td::Status::Error("Invalid config (6)"); + return td::Status::Error("Invalid config (4)"); } client.adnl_id = ton::adnl::AdnlNodeIdFull(ton::pubkeys::Ed25519(td::Bits256(td::Slice(key).ubegin()))); + return client; + }; + + Config res; + for (auto &value : lite_clients) { + TRY_RESULT(client, parse_desc(value)); + res.lite_clients.push_back(std::move(client)); + } + for (auto &value : lite_clients_v2) { + TRY_RESULT(client, parse_desc(value)); + client.is_full = false; + TRY_RESULT(shards_obj, + td::get_json_object_field(value.get_object(), "shards", td::JsonValue::Type::Array, false)); + for (auto &shard : shards_obj.get_array()) { + if (shard.type() != td::JsonValue::Type::Object) { + return td::Status::Error("Invalid config (5)"); + } + auto &shard_obj = shard.get_object(); + TRY_RESULT(workchain, td::get_json_object_int_field(shard_obj, "workchain", false)); + TRY_RESULT(shard_id, td::get_json_object_long_field(shard_obj, "shard", false)); + if (shard_id == 0) { + return td::Status::Error("Invalid config (6)"); + } + client.shards.emplace_back(workchain, shard_id); + } + res.lite_clients.push_back(std::move(client)); } diff --git a/tonlib/tonlib/Config.h b/tonlib/tonlib/Config.h index 3902c3419a..420889dfdb 100644 --- a/tonlib/tonlib/Config.h +++ b/tonlib/tonlib/Config.h @@ -26,6 +26,8 @@ struct Config { struct LiteClient { ton::adnl::AdnlNodeIdFull adnl_id; td::IPAddress address; + bool is_full = true; + std::vector shards; }; ton::BlockIdExt zero_state_id; ton::BlockIdExt init_block_id; diff --git a/tonlib/tonlib/ExtClient.cpp b/tonlib/tonlib/ExtClient.cpp index 30a29b59c8..fdd1606df8 100644 --- a/tonlib/tonlib/ExtClient.cpp +++ b/tonlib/tonlib/ExtClient.cpp @@ -54,7 +54,7 @@ void ExtClient::with_last_block(td::Promise promise) { td::actor::send_closure(client_.last_block_actor_, &LastBlock::get_last_block, std::move(P)); } -void ExtClient::send_raw_query(td::BufferSlice query, td::Promise promise) { +void ExtClient::send_raw_query(td::BufferSlice query, ton::ShardIdFull shard, td::Promise promise) { auto query_id = queries_.create(std::move(promise)); td::Promise P = [query_id, self = this, actor_id = td::actor::actor_id()](td::Result result) { @@ -62,10 +62,10 @@ void ExtClient::send_raw_query(td::BufferSlice query, td::Promisequeries_.extract(query_id).set_result(std::move(result)); }); }; - if (client_.adnl_ext_client_.empty()) { + if (client_.raw_client_.empty()) { return P.set_error(TonlibError::NoLiteServers()); } - td::actor::send_closure(client_.adnl_ext_client_, &ton::adnl::AdnlExtClient::send_query, "query", std::move(query), + td::actor::send_closure(client_.raw_client_, &ExtClientRaw::send_query, "query", std::move(query), shard, td::Timestamp::in(10.0), std::move(P)); } } // namespace tonlib diff --git a/tonlib/tonlib/ExtClient.h b/tonlib/tonlib/ExtClient.h index 707ca74bd3..4881862229 100644 --- a/tonlib/tonlib/ExtClient.h +++ b/tonlib/tonlib/ExtClient.h @@ -28,9 +28,11 @@ #include "td/utils/Container.h" #include "td/utils/Random.h" -#include "ExtClientLazy.h" +#include "ExtClientRaw.h" #include "TonlibError.h" #include "utils.h" +#include "QueryTraits.h" +#include "ExtClientRaw.h" namespace tonlib { class LastBlock; @@ -38,7 +40,7 @@ class LastConfig; struct LastBlockState; struct LastConfigState; struct ExtClientRef { - td::actor::ActorId adnl_ext_client_; + td::actor::ActorId raw_client_; td::actor::ActorId last_block_actor_; td::actor::ActorId last_config_actor_; }; @@ -64,6 +66,7 @@ class ExtClient { template void send_query(QueryT query, td::Promise promise, td::int32 seq_no = -1) { + ton::ShardIdFull shard = QueryTraits::get_shard(query); auto raw_query = ton::serialize_tl_object(&query, true); td::uint32 tag = td::Random::fast_uint32(); VLOG(lite_server) << "send query to liteserver: " << tag << " " << to_string(query); @@ -77,7 +80,7 @@ class ExtClient { ton::serialize_tl_object(ton::create_tl_object(std::move(raw_query)), true); send_raw_query( - std::move(liteserver_query), [promise = std::move(promise), tag](td::Result R) mutable { + std::move(liteserver_query), shard, [promise = std::move(promise), tag](td::Result R) mutable { auto res = [&]() -> td::Result { TRY_RESULT_PREFIX(data, std::move(R), TonlibError::LiteServerNetwork()); auto r_error = ton::fetch_tl_object(data.clone(), true); @@ -96,8 +99,8 @@ class ExtClient { } void force_change_liteserver() { - if (!client_.adnl_ext_client_.empty()) { - td::actor::send_closure(client_.adnl_ext_client_, &ExtClientLazy::force_change_liteserver); + if (!client_.raw_client_.empty()) { + td::actor::send_closure(client_.raw_client_, &ExtClientRaw::force_change_liteserver); } } @@ -107,6 +110,6 @@ class ExtClient { td::Container> last_block_queries_; td::Container> last_config_queries_; - void send_raw_query(td::BufferSlice query, td::Promise promise); + void send_raw_query(td::BufferSlice query, ton::ShardIdFull shard, td::Promise promise); }; } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientLazy.cpp b/tonlib/tonlib/ExtClientLazy.cpp index 335a0ff968..625d7eaa76 100644 --- a/tonlib/tonlib/ExtClientLazy.cpp +++ b/tonlib/tonlib/ExtClientLazy.cpp @@ -16,15 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "ExtClientLazy.h" +#include "ExtClientRaw.h" #include "TonlibError.h" #include "td/utils/Random.h" namespace tonlib { -class ExtClientLazyImp : public ExtClientLazy { +class ExtClientLazyImp : public ExtClientRaw { public: ExtClientLazyImp(std::vector> servers, - td::unique_ptr callback) + td::unique_ptr callback) : servers_(std::move(servers)), callback_(std::move(callback)) { CHECK(!servers_.empty()); } @@ -34,15 +34,7 @@ class ExtClientLazyImp : public ExtClientLazy { td::random_shuffle(td::as_mutable_span(servers_), rnd); } - void check_ready(td::Promise promise) override { - before_query(); - if (client_.empty()) { - return promise.set_error(TonlibError::Cancelled()); - } - send_closure(client_, &ton::adnl::AdnlExtClient::check_ready, std::move(promise)); - } - - void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, + void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, td::Promise promise) override { before_query(); if (client_.empty()) { @@ -109,7 +101,7 @@ class ExtClientLazyImp : public ExtClientLazy { bool cur_server_bad_force_ = false; td::actor::ActorOwn client_; - td::unique_ptr callback_; + td::unique_ptr callback_; static constexpr double MAX_NO_QUERIES_TIMEOUT = 100; bool is_closing_{false}; @@ -140,12 +132,12 @@ class ExtClientLazyImp : public ExtClientLazy { } }; -td::actor::ActorOwn ExtClientLazy::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, - td::unique_ptr callback) { +td::actor::ActorOwn ExtClientRaw::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + td::unique_ptr callback) { return create({std::make_pair(dst, dst_addr)}, std::move(callback)); } -td::actor::ActorOwn ExtClientLazy::create( +td::actor::ActorOwn ExtClientRaw::create( std::vector> servers, td::unique_ptr callback) { return td::actor::create_actor("ExtClientLazy", std::move(servers), std::move(callback)); } diff --git a/tonlib/tonlib/ExtClientMulti.cpp b/tonlib/tonlib/ExtClientMulti.cpp new file mode 100644 index 0000000000..3a6cdf595d --- /dev/null +++ b/tonlib/tonlib/ExtClientMulti.cpp @@ -0,0 +1,169 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "ExtClientMulti.h" +#include "ton/ton-shard.h" +#include "td/utils/Random.h" + +namespace tonlib { + +static const double MAX_NO_QUERIES_TIMEOUT = 120; + +ExtClientMulti::ExtClientMulti(std::vector clients, td::unique_ptr callback) + : callback_(std::move(callback)) { + for (auto &desc : clients) { + servers_.emplace_back(); + servers_.back().desc = std::move(desc); + } +} + +void ExtClientMulti::start_up() { + alarm_timestamp() = td::Timestamp::in(60.0); +} + +void ExtClientMulti::send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, + td::Promise promise) { + if (shard.is_masterchain() && mc_server_idx_ != -1 && !servers_[mc_server_idx_].is_bad()) { + send_query_to_server(std::move(name), std::move(data), mc_server_idx_, timeout, std::move(promise)); + return; + } + auto it = shard_server_idx_cached_.find(shard); + if (it != shard_server_idx_cached_.end() && !servers_[it->second].is_bad()) { + send_query_to_server(std::move(name), std::move(data), it->second, timeout, std::move(promise)); + return; + } + int server_idx = -1; + int random_idx = -1; + int cnt = 0; + int best_prefix = -1; + for (int i = 0; i < (int)servers_.size(); ++i) { + const Server &server = servers_[i]; + if (server.is_bad()) { + continue; + } + int len = server.desc.is_full ? 65 : server.max_supported_prefix(shard); + if (len > best_prefix) { + best_prefix = len; + server_idx = -1; + random_idx = -1; + cnt = 0; + } else if (len < best_prefix) { + continue; + } + if (!server.client.empty()) { + server_idx = i; + } + if (td::Random::fast(0, cnt) == 0) { + random_idx = i; + } + ++cnt; + } + if (server_idx == -1) { + server_idx = random_idx; + } + if (server_idx == -1) { + promise.set_error(td::Status::Error("failed to select a suitable server")); + return; + } + if (shard.pfx_len() <= ton::max_shard_pfx_len) { + shard_server_idx_cached_[shard] = server_idx; + } + if (shard.is_masterchain() || servers_[server_idx].desc.is_full) { + mc_server_idx_ = server_idx; + } + send_query_to_server(std::move(name), std::move(data), server_idx, timeout, std::move(promise)); +} + +void ExtClientMulti::send_query_to_server(std::string name, td::BufferSlice data, int server_idx, td::Timestamp timeout, + td::Promise promise) { + Server &server = servers_.at(server_idx); + if (server.client.empty()) { + start_client(server_idx); + } + server.ttl = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT); + td::Promise P = [SelfId = actor_id(this), idx = server_idx, + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error() && + (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { + td::actor::send_closure(SelfId, &ExtClientMulti::set_server_bad, idx); + } + promise.set_result(std::move(R)); + }; + send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, + std::move(P)); +} + +void ExtClientMulti::force_change_liteserver() { + if (mc_server_idx_ != -1) { + set_server_bad(mc_server_idx_); + mc_server_idx_ = -1; + } +} + +void ExtClientMulti::start_client(int server_idx) { + class Callback : public ton::adnl::AdnlExtClient::Callback { + public: + Callback(td::actor::ActorId parent, int idx) : parent_(std::move(parent)), idx_(idx) { + } + void on_ready() override { + } + void on_stop_ready() override { + td::actor::send_closure(parent_, &ExtClientMulti::set_server_bad, idx_); + } + + private: + td::actor::ActorId parent_; + int idx_; + }; + Server &server = servers_.at(server_idx); + server.client = ton::adnl::AdnlExtClient::create(server.desc.adnl_id, server.desc.address, + std::make_unique(actor_id(this), server_idx)); +} + +void ExtClientMulti::alarm() { + for (Server& server : servers_) { + if (server.ttl && server.ttl.is_in_past()) { + server.client.reset(); + } + } + alarm_timestamp() = td::Timestamp::in(60.0); +} + +void ExtClientMulti::set_server_bad(int idx) { + Server& server = servers_.at(idx); + server.client.reset(); + server.ttl = td::Timestamp::never(); + server.ignore_until = td::Timestamp::in(10.0); +} + +int ExtClientMulti::Server::max_supported_prefix(ton::ShardIdFull shard) const { + if (desc.is_full || shard.is_masterchain()) { + return shard.pfx_len(); + } + int res = -1; + for (const ton::ShardIdFull &our_shard : desc.shards) { + if (ton::shard_is_ancestor(our_shard, shard)) { + return shard.pfx_len(); + } + if (shard.workchain == our_shard.workchain) { + int x = std::min({shard.pfx_len(), our_shard.pfx_len(), ton::count_matching_bits(shard.shard, our_shard.shard)}); + res = std::max(res, x); + } + } + return res; +} + +} // namespace tonlib diff --git a/tonlib/tonlib/ExtClientMulti.h b/tonlib/tonlib/ExtClientMulti.h new file mode 100644 index 0000000000..4c93b00ee5 --- /dev/null +++ b/tonlib/tonlib/ExtClientMulti.h @@ -0,0 +1,61 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "td/actor/actor.h" +#include "adnl/adnl-ext-client.h" +#include "Config.h" +#include "ExtClientRaw.h" +#include + +namespace tonlib { + +class ExtClientMulti : public ExtClientRaw { + public: + ExtClientMulti(std::vector clients, td::unique_ptr callback); + + void start_up() override; + void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, + td::Promise promise) override; + void alarm() override; + void force_change_liteserver() override; + + private: + void send_query_to_server(std::string name, td::BufferSlice data, int server_idx, td::Timestamp timeout, + td::Promise promise); + void start_client(int server_idx); + + struct Server { + Config::LiteClient desc; + td::actor::ActorOwn client; + td::Timestamp ttl; + td::Timestamp ignore_until = td::Timestamp::never(); + + int max_supported_prefix(ton::ShardIdFull shard) const; + bool is_bad() const { + return ignore_until && !ignore_until.is_in_past(); + } + }; + + void set_server_bad(int idx); + + td::unique_ptr callback_; + std::vector servers_; + int mc_server_idx_ = -1; + std::map shard_server_idx_cached_; +}; + +} // namespace tonlib diff --git a/tonlib/tonlib/ExtClientOutbound.cpp b/tonlib/tonlib/ExtClientOutbound.cpp index 025ba848ed..e057368d7e 100644 --- a/tonlib/tonlib/ExtClientOutbound.cpp +++ b/tonlib/tonlib/ExtClientOutbound.cpp @@ -27,15 +27,11 @@ class ExtClientOutboundImp : public ExtClientOutbound { ExtClientOutboundImp(td::unique_ptr callback) : callback_(std::move(callback)) { } - void check_ready(td::Promise promise) override { - promise.set_error(td::Status::Error("Not supported")); - } - - void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, + void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, td::Promise promise) override { auto query_id = next_query_id_++; queries_[query_id] = std::move(promise); - callback_->request(query_id, data.as_slice().str()); + callback_->request(query_id, data.as_slice().str(), shard); } void force_change_liteserver() override { diff --git a/tonlib/tonlib/ExtClientOutbound.h b/tonlib/tonlib/ExtClientOutbound.h index 87b73b9e01..9121786849 100644 --- a/tonlib/tonlib/ExtClientOutbound.h +++ b/tonlib/tonlib/ExtClientOutbound.h @@ -19,16 +19,16 @@ #pragma once #include "td/actor/actor.h" -#include "ExtClientLazy.h" +#include "ExtClientRaw.h" namespace tonlib { -class ExtClientOutbound : public ExtClientLazy { +class ExtClientOutbound : public ExtClientRaw { public: class Callback { public: virtual ~Callback() { } - virtual void request(td::int64 id, std::string data) = 0; + virtual void request(td::int64 id, std::string data, ton::ShardIdFull shard) = 0; }; virtual void on_query_result(td::int64 id, td::Result r_data, td::Promise promise) = 0; static td::actor::ActorOwn create(td::unique_ptr callback); diff --git a/tonlib/tonlib/ExtClientLazy.h b/tonlib/tonlib/ExtClientRaw.h similarity index 68% rename from tonlib/tonlib/ExtClientLazy.h rename to tonlib/tonlib/ExtClientRaw.h index dc4490b3a3..0a55bdc17b 100644 --- a/tonlib/tonlib/ExtClientLazy.h +++ b/tonlib/tonlib/ExtClientRaw.h @@ -18,23 +18,24 @@ */ #pragma once #include "td/actor/actor.h" - #include "adnl/adnl-ext-client.h" +#include "ton/ton-types.h" namespace tonlib { -class ExtClientLazy : public ton::adnl::AdnlExtClient { +class ExtClientRaw : public td::actor::Actor { public: class Callback { public: - virtual ~Callback() { - } + virtual ~Callback() = default; }; + virtual void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, + td::Promise promise) = 0; virtual void force_change_liteserver() = 0; - static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, - td::unique_ptr callback); - static td::actor::ActorOwn create( + static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + td::unique_ptr callback); + static td::actor::ActorOwn create( std::vector> servers, td::unique_ptr callback); }; diff --git a/tonlib/tonlib/QueryTraits.h b/tonlib/tonlib/QueryTraits.h new file mode 100644 index 0000000000..bf480f64c8 --- /dev/null +++ b/tonlib/tonlib/QueryTraits.h @@ -0,0 +1,202 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "ton/ton-types.h" +#include "auto/tl/lite_api.h" +#include "vm/boc.h" +#include "vm/cellslice.h" +#include "block/block-auto.h" +#include "block/block-parse.h" + +namespace tonlib { + +template +struct QueryTraits { + static ton::ShardIdFull get_shard(const Query& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getMasterchainInfo& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getMasterchainInfoExt& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getTime& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getVersion& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getBlock& q) { + return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getState& q) { + return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getBlockHeader& q) { + return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_sendMessage& q) { + auto shard = [&]() -> td::Result { + vm::BagOfCells boc; + TRY_STATUS(boc.deserialize(q.body_.as_slice())); + if (boc.get_root_count() != 1) { + return td::Status::Error("external message is not a valid bag of cells"); + } + block::gen::CommonMsgInfo::Record_ext_in_msg_info info; + if (!tlb::unpack_cell_inexact(boc.get_root_cell(), info)) { + return td::Status::Error("cannot unpack external message header"); + } + auto dest_prefix = block::tlb::t_MsgAddressInt.get_prefix(info.dest); + if (!dest_prefix.is_valid()) { + return td::Status::Error("destination of an inbound external message is an invalid blockchain address"); + } + return dest_prefix.as_leaf_shard(); + }(); + if (shard.is_error()) { + LOG(DEBUG) << "Failed to get shard from query liteServer.sendMessage: " << shard.move_as_error(); + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } + return shard.move_as_ok(); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getAccountState& q) { + return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_runSmcMethod& q) { + return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getShardInfo& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getAllShardsInfo& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getOneTransaction& q) { + return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getTransactions& q) { + return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_lookupBlock& q) { + return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_listBlockTransactions& q) { + return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getBlockProof& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getConfigAll& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getConfigParams& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getValidatorStats& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getLibraries& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + + +} // namespace tonlib diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index f9e984bb9e..b4b3869784 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -18,7 +18,7 @@ */ #include "TonlibClient.h" -#include "tonlib/ExtClientLazy.h" +#include "tonlib/ExtClientRaw.h" #include "tonlib/ExtClientOutbound.h" #include "tonlib/LastBlock.h" #include "tonlib/LastConfig.h" @@ -27,6 +27,7 @@ #include "tonlib/keys/Mnemonic.h" #include "tonlib/keys/SimpleEncryption.h" #include "tonlib/TonlibError.h" +#include "tonlib/ExtClientMulti.h" #include "smc-envelope/GenericAccount.h" #include "smc-envelope/ManualDns.h" @@ -1649,15 +1650,16 @@ void TonlibClient::hangup() { ExtClientRef TonlibClient::get_client_ref() { ExtClientRef ref; - ref.adnl_ext_client_ = raw_client_.get(); + ref.raw_client_ = raw_client_.get(); ref.last_block_actor_ = raw_last_block_.get(); ref.last_config_actor_ = raw_last_config_.get(); return ref; } -void TonlibClient::proxy_request(td::int64 query_id, std::string data) { - on_update(tonlib_api::make_object(query_id, data)); +void TonlibClient::proxy_request(td::int64 query_id, std::string data, ton::ShardIdFull shard) { + on_update( + tonlib_api::make_object(query_id, data, shard.workchain, shard.shard)); } void TonlibClient::init_ext_client() { @@ -1668,9 +1670,9 @@ void TonlibClient::init_ext_client() { : parent_(std::move(parent)), config_generation_(config_generation) { } - void request(td::int64 id, std::string data) override { + void request(td::int64 id, std::string data, ton::ShardIdFull shard) override { send_closure(parent_, &TonlibClient::proxy_request, (id << 16) | (config_generation_ & 0xffff), - std::move(data)); + std::move(data), shard); } private: @@ -1683,21 +1685,29 @@ void TonlibClient::init_ext_client() { ext_client_outbound_ = client.get(); raw_client_ = std::move(client); } else { - std::vector> servers; - for (const auto& s : config_.lite_clients) { - servers.emplace_back(s.adnl_id, s.address); - } - class Callback : public ExtClientLazy::Callback { + class Callback : public ExtClientRaw::Callback { public: explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { } - private: td::actor::ActorShared<> parent_; }; + std::vector> full_servers; + for (const auto& s : config_.lite_clients) { + if (s.is_full) { + full_servers.emplace_back(s.adnl_id, s.address); + } + } + if (!full_servers.empty()) { + raw_client_ = + ExtClientRaw::create(std::move(full_servers), td::make_unique(td::actor::actor_shared())); + } else { + CHECK(!config_.lite_clients.empty()); + raw_client_ = td::actor::create_actor("ExtClientMulti", config_.lite_clients, + td::make_unique(td::actor::actor_shared())); + } ext_client_outbound_ = {}; ref_cnt_++; - raw_client_ = ExtClientLazy::create(std::move(servers), td::make_unique(td::actor::actor_shared())); } } @@ -2826,7 +2836,7 @@ td::Status TonlibClient::do_request(tonlib_api::raw_getTransactionsV2& request, auto actor_id = actor_id_++; actors_[actor_id] = td::actor::create_actor( - "GetTransactionHistory", client_.get_client(), account_address, lt, hash, count, actor_shared(this, actor_id), + "GetTransactionHistory", client_.get_client(), account_address, lt, hash, count, actor_shared(this, actor_id), promise.wrap([private_key = std::move(private_key), try_decode_messages = request.try_decode_messages_](auto&& x) mutable { return ToRawTransactions(std::move(private_key), try_decode_messages).to_raw_transactions(std::move(x)); })); diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index e781929098..8c07687625 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -110,7 +110,7 @@ class TonlibClient : public td::actor::Actor { vm::Dictionary libraries{256}; // network - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; td::actor::ActorId ext_client_outbound_; td::actor::ActorOwn raw_last_block_; td::actor::ActorOwn raw_last_config_; @@ -372,7 +372,7 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(const tonlib_api::getConfigParam& request, td::Promise>&& promise); - void proxy_request(td::int64 query_id, std::string data); + void proxy_request(td::int64 query_id, std::string data, ton::ShardIdFull shard); void load_libs_from_disk(); void store_libs_to_disk(); diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 0a042eb1dd..c256bf1eef 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -47,7 +47,7 @@ #include "tonlib/TonlibClient.h" #include "tonlib/TonlibCallback.h" -#include "tonlib/ExtClientLazy.h" +#include "tonlib/ExtClientRaw.h" #include "smc-envelope/ManualDns.h" #include "smc-envelope/PaymentChannel.h" @@ -62,6 +62,7 @@ #include #include #include "git.h" +#include "ExtClientMulti.h" using tonlib_api::make_object; @@ -174,7 +175,7 @@ class TonlibCli : public td::actor::Actor { std::map>> query_handlers_; - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; bool is_closing_{false}; td::uint32 ref_cnt_{1}; @@ -223,11 +224,7 @@ class TonlibCli : public td::actor::Actor { if (options_.use_callbacks_for_network) { auto config = tonlib::Config::parse(options_.config).move_as_ok(); - auto lite_clients_size = config.lite_clients.size(); - CHECK(lite_clients_size != 0); - auto lite_client_id = td::Random::fast(0, td::narrow_cast(lite_clients_size) - 1); - auto& lite_client = config.lite_clients[lite_client_id]; - class Callback : public tonlib::ExtClientLazy::Callback { + class Callback : public tonlib::ExtClientRaw::Callback { public: explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { } @@ -235,9 +232,22 @@ class TonlibCli : public td::actor::Actor { private: td::actor::ActorShared<> parent_; }; + std::vector> full_servers; + int lite_client_id = -1, cnt = 0; + for (const auto& s : config.lite_clients) { + if (s.is_full) { + full_servers.emplace_back(s.adnl_id, s.address); + } + } + if (!full_servers.empty()) { + raw_client_ = tonlib::ExtClientRaw::create(std::move(full_servers), + td::make_unique(td::actor::actor_shared())); + } else { + CHECK(!config.lite_clients.empty()); + raw_client_ = td::actor::create_actor( + "ExtClientMulti", config.lite_clients, td::make_unique(td::actor::actor_shared())); + } ref_cnt_++; - raw_client_ = tonlib::ExtClientLazy::create(lite_client.adnl_id, lite_client.address, - td::make_unique(td::actor::actor_shared())); } auto config = !options_.config.empty() @@ -1534,7 +1544,8 @@ class TonlibCli : public td::actor::Actor { auto update = tonlib_api::move_object_as(std::move(result)); CHECK(!raw_client_.empty()); snd_bytes_ += update->data_.size(); - send_closure(raw_client_, &ton::adnl::AdnlExtClient::send_query, "query", td::BufferSlice(update->data_), + ton::ShardIdFull shard(update->workchain_, update->shard_); + send_closure(raw_client_, &tonlib::ExtClientRaw::send_query, "query", td::BufferSlice(update->data_), shard, td::Timestamp::in(5), [actor_id = actor_id(this), id = update->id_](td::Result res) { send_closure(actor_id, &TonlibCli::on_adnl_result, id, std::move(res)); From 23dbf2d834256ef7fafca6f536eead1ebb594c78 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Sat, 15 Oct 2022 11:58:01 +0300 Subject: [PATCH 042/524] Move CollatorConfig to -41 Config Param --- crypto/block/mc-config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index ed79c8b633..6b3e51ffaa 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2135,7 +2135,7 @@ Ref ConfigInfo::lookup_library(td::ConstBitPtr root_hash) const { CollatorConfig Config::get_collator_config(bool need_collator_nodes) const { CollatorConfig collator_config; gen::CollatorConfig::Record rec; - auto cell = get_config_param(41); + auto cell = get_config_param(-41); if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { return collator_config; } From d4339b839c966dd423862bd328216da73df41169 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 17 Oct 2022 13:24:59 +0300 Subject: [PATCH 043/524] Add "delshard", "delcollator" to validator-engine-console --- tl/generate/scheme/ton_api.tl | 2 + tl/generate/scheme/ton_api.tlo | Bin 73376 -> 73612 bytes .../validator-engine-console-query.cpp | 41 +++++++ .../validator-engine-console-query.h | 47 ++++++++ validator-engine/validator-engine.cpp | 104 ++++++++++++++++++ validator-engine/validator-engine.hpp | 6 + validator/collator-node.cpp | 10 ++ validator/collator-node.hpp | 1 + validator/manager-disk.hpp | 3 + validator/manager-hardfork.hpp | 3 + validator/manager.cpp | 21 +++- validator/manager.hpp | 7 +- validator/validator.h | 1 + 13 files changed, 241 insertions(+), 5 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index f04f7863c4..3a2ad1fc05 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -715,6 +715,8 @@ engine.validator.getValidatorSessionsInfo = engine.validator.ValidatorSessionsIn engine.validator.addCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; engine.validator.addShard shard:tonNode.shardId = engine.validator.Success; +engine.validator.delCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; +engine.validator.delShard shard:tonNode.shardId = engine.validator.Success; ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index d535668aa423585bb2be3319dbf46d3f4f610efc..2820a7898588afc696e5aef67cb15bbddf789087 100644 GIT binary patch delta 53 zcmV-50LuTMy#$QE1h9PT0cNv(>rW8{T*-tH9Fx)f6qA|pZ?nw&XaWSgb}CPk(EJsX L;r2_jH2-WLyuuo? delta 26 icmeC#&$3`I%Z8fQj47LIU;7Jg4*8qGv{ivIM;-vFhzoZB diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index cab347774e..7bad3c974c 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1119,3 +1119,44 @@ td::Status AddShardQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "successfully added shard\n"; return td::Status::OK(); } + +td::Status DelCollatorQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status DelCollatorQuery::send() { + auto b = ton::create_serialize_tl_object( + adnl_id_.tl(), ton::create_tl_shard_id(ton::ShardIdFull(wc_, shard_))); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status DelCollatorQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully removed collator\n"; + return td::Status::OK(); +} + +td::Status DelShardQuery::run() { + TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status DelShardQuery::send() { + auto b = ton::create_serialize_tl_object( + ton::create_tl_shard_id(ton::ShardIdFull(wc_, shard_))); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status DelShardQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully removed shard\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 316cc5e77f..3c2c219399 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1162,3 +1162,50 @@ class AddShardQuery : public Query { td::int32 wc_; td::int64 shard_; }; + +class DelCollatorQuery : public Query { + public: + DelCollatorQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "delcollator"; + } + static std::string get_help() { + return "delcollator \tremove collator with given adnl_id and shard"; + } + std::string name() const override { + return get_name(); + } + + private: + ton::PublicKeyHash adnl_id_; + td::int32 wc_; + td::int64 shard_; +}; + +class DelShardQuery : public Query { + public: + DelShardQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "delshard"; + } + static std::string get_help() { + return "delshard \tstop monitoring shard"; + } + std::string name() const override { + return get_name(); + } + + private: + td::int32 wc_; + td::int64 shard_; +}; diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 14a6961fad..cd14ae1edd 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -418,6 +418,19 @@ td::Result Config::config_add_collator(ton::PublicKeyHash addr, ton::Shard return true; } +td::Result Config::config_del_collator(ton::PublicKeyHash addr, ton::ShardIdFull shard) { + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard: " << shard.to_str()); + } + Collator c{addr, shard}; + auto it = std::find(collators.begin(), collators.end(), c); + if (it == collators.end()) { + return false; + } + collators.erase(it); + return true; +} + td::Result Config::config_add_full_node_adnl_id(ton::PublicKeyHash id) { if (full_node == id) { return false; @@ -548,6 +561,18 @@ td::Result Config::config_add_shard(ton::ShardIdFull shard) { return true; } +td::Result Config::config_del_shard(ton::ShardIdFull shard) { + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << shard.to_str()); + } + auto it = std::find(shards_to_monitor.begin(), shards_to_monitor.end(), shard); + if (it == shards_to_monitor.end()) { + return false; + } + shards_to_monitor.erase(it); + return true; +} + td::Result Config::config_add_gc(ton::PublicKeyHash key) { return gc.insert(key).second; } @@ -3506,6 +3531,85 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addShard }); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCollator &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto id = ton::PublicKeyHash{query.adnl_id_}; + auto shard = ton::create_shard_id(query.shard_); + auto R = config_.config_del_collator(id, shard); + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + if (!R.move_as_ok()) { + promise.set_value(create_control_query_error(td::Status::Error("No such collator"))); + return; + } + if (!R.move_as_ok()) { + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + return; + } + set_shard_check_function(); + if (!validator_manager_.empty()) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::del_collator, + ton::adnl::AdnlNodeIdShort(id), shard); + } + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delShard &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto shard = ton::create_shard_id(query.shard_); + auto R = config_.config_del_shard(shard); + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + if (!R.move_as_ok()) { + promise.set_value(create_control_query_error(td::Status::Error("No such shard"))); + return; + } + set_shard_check_function(); + if (!validator_manager_.empty()) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + } + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 71276e5dfa..49010bc476 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -115,6 +115,7 @@ struct Config { td::Result config_add_validator_adnl_id(ton::PublicKeyHash perm_key, ton::PublicKeyHash adnl_id, ton::UnixTime expire_at); td::Result config_add_collator(ton::PublicKeyHash addr, ton::ShardIdFull shard); + td::Result config_del_collator(ton::PublicKeyHash addr, ton::ShardIdFull shard); td::Result config_add_full_node_adnl_id(ton::PublicKeyHash id); td::Result config_add_full_node_slave(td::IPAddress addr, ton::PublicKey id); td::Result config_add_full_node_master(td::int32 port, ton::PublicKeyHash id); @@ -123,6 +124,7 @@ struct Config { td::Result config_add_control_process(ton::PublicKeyHash key, td::int32 port, ton::PublicKeyHash id, td::uint32 permissions); td::Result config_add_shard(ton::ShardIdFull shard); + td::Result config_del_shard(ton::ShardIdFull shard); td::Result config_add_gc(ton::PublicKeyHash key); td::Result config_del_network_addr(td::IPAddress addr, std::vector cats, std::vector prio_cats); @@ -437,6 +439,10 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_addShard &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_delCollator &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_delShard &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getPerfTimerStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 0a2a050ebd..c6afa54186 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -54,10 +54,20 @@ void CollatorNode::tear_down() { } void CollatorNode::add_shard(ShardIdFull shard) { + if (std::find(shards_.begin(), shards_.end(), shard) != shards_.end()) { + return; + } LOG(INFO) << "Collator node: local_id=" << local_id_ << " , shard=" << shard.to_str(); shards_.push_back(shard); } +void CollatorNode::del_shard(ShardIdFull shard) { + auto it = std::find(shards_.begin(), shards_.end(), shard); + if (it != shards_.end()) { + shards_.erase(it); + } +} + void CollatorNode::new_masterchain_block_notification(td::Ref state) { std::vector top_blocks = {state->get_block_id()}; std::vector next_shards; diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index b0a2ad9037..b3b232cf9e 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -32,6 +32,7 @@ class CollatorNode : public td::actor::Actor { void start_up() override; void tear_down() override; void add_shard(ShardIdFull shard); + void del_shard(ShardIdFull shard); void new_masterchain_block_notification(td::Ref state); diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index c4403e9417..e0aad62ed0 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -395,6 +395,9 @@ class ValidatorManagerImpl : public ValidatorManager { void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { UNREACHABLE(); } + void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { + UNREACHABLE(); + } void update_options(td::Ref opts) override { opts_ = std::move(opts); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index bd30b1de3f..e8fa327b06 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -454,6 +454,9 @@ class ValidatorManagerImpl : public ValidatorManager { void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { UNREACHABLE(); } + void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { + UNREACHABLE(); + } void update_options(td::Ref opts) override { opts_ = std::move(opts); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 183dc8c1eb..f56d690172 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1795,7 +1795,7 @@ void ValidatorManagerImpl::new_masterchain_block() { last_masterchain_block_handle_, last_masterchain_state_); } for (auto &c : collator_nodes_) { - td::actor::send_closure(c.second, &CollatorNode::new_masterchain_block_notification, last_masterchain_state_); + td::actor::send_closure(c.second.actor, &CollatorNode::new_masterchain_block_notification, last_masterchain_state_); } if (last_masterchain_seqno_ % 1024 == 0) { @@ -2722,10 +2722,23 @@ void ValidatorManagerImpl::get_validator_sessions_info( void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) { auto it = collator_nodes_.find(id); if (it == collator_nodes_.end()) { - auto actor = td::actor::create_actor("collatornode", id, actor_id(this), adnl_, rldp_); - it = collator_nodes_.emplace(id, std::move(actor)).first; + it = collator_nodes_.emplace(id, Collator()).first; + it->second.actor = td::actor::create_actor("collatornode", id, actor_id(this), adnl_, rldp_); + } + it->second.shards.insert(shard); + td::actor::send_closure(it->second.actor, &CollatorNode::add_shard, shard); +} + +void ValidatorManagerImpl::del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) { + auto it = collator_nodes_.find(id); + if (it == collator_nodes_.end()) { + return; + } + td::actor::send_closure(it->second.actor, &CollatorNode::del_shard, shard); + it->second.shards.erase(shard); + if (it->second.shards.empty()) { + collator_nodes_.erase(it); } - td::actor::send_closure(it->second, &CollatorNode::add_shard, shard); } void ValidatorManagerImpl::update_options(td::Ref opts) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 136e5e8ac2..6fca117857 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -563,6 +563,7 @@ class ValidatorManagerImpl : public ValidatorManager { void add_persistent_state_description(td::Ref desc) override; void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; + void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; void update_options(td::Ref opts) override; private: @@ -639,7 +640,11 @@ class ValidatorManagerImpl : public ValidatorManager { std::map> shard_client_waiters_; - std::map> collator_nodes_; + struct Collator { + td::actor::ActorOwn actor; + std::set shards; + }; + std::map collator_nodes_; std::set extra_active_shards_; std::map last_validated_blocks_; diff --git a/validator/validator.h b/validator/validator.h index c387caffc8..0a5a22b5f2 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -238,6 +238,7 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise> promise) = 0; virtual void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; + virtual void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; virtual void update_options(td::Ref opts) = 0; }; From 2ea17ec03b8b116e24c800b3389e34b5ae2e21f9 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 11 Jan 2023 19:14:03 +0300 Subject: [PATCH 044/524] Use partial liteservers in tonlib and lite-client --- lite-client/lite-client.cpp | 757 ++++++++++-------- lite-client/lite-client.h | 9 +- tl/generate/scheme/ton_api.tlo | Bin 83196 -> 87748 bytes tl/generate/scheme/tonlib_api.tlo | Bin 31016 -> 31084 bytes tonlib/CMakeLists.txt | 4 +- tonlib/tonlib/Config.cpp | 41 +- tonlib/tonlib/Config.h | 4 +- tonlib/tonlib/ExtClient.cpp | 6 +- tonlib/tonlib/ExtClient.h | 1 - tonlib/tonlib/ExtClientLazy.cpp | 174 ++-- .../{ExtClientRaw.h => ExtClientLazy.h} | 14 +- tonlib/tonlib/ExtClientMulti.cpp | 169 ---- tonlib/tonlib/ExtClientMulti.h | 61 -- tonlib/tonlib/ExtClientOutbound.cpp | 10 +- tonlib/tonlib/ExtClientOutbound.h | 1 + tonlib/tonlib/TonlibClient.cpp | 581 +++++++------- tonlib/tonlib/tonlib-cli.cpp | 33 +- 17 files changed, 891 insertions(+), 974 deletions(-) rename tonlib/tonlib/{ExtClientRaw.h => ExtClientLazy.h} (72%) delete mode 100644 tonlib/tonlib/ExtClientMulti.cpp delete mode 100644 tonlib/tonlib/ExtClientMulti.h diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index feedbe40b8..c6988ef3ef 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -29,22 +29,16 @@ #include "lite-client-common.h" -#include "adnl/adnl-ext-client.h" #include "tl-utils/lite-utils.hpp" #include "auto/tl/ton_api_json.h" #include "auto/tl/lite_api.hpp" #include "td/utils/OptionParser.h" #include "td/utils/Time.h" #include "td/utils/filesystem.h" -#include "td/utils/format.h" #include "td/utils/Random.h" #include "td/utils/crypto.h" -#include "td/utils/overloaded.h" #include "td/utils/port/signals.h" -#include "td/utils/port/stacktrace.h" -#include "td/utils/port/StdStreams.h" #include "td/utils/port/FileFd.h" -#include "terminal/terminal.h" #include "ton/lite-tl.hpp" #include "block/block-db.h" #include "block/block.h" @@ -58,18 +52,14 @@ #include "vm/vm.h" #include "vm/cp0.h" #include "vm/memo.h" -#include "ton/ton-shard.h" -#include "openssl/rand.hpp" #include "crypto/vm/utils.h" #include "crypto/common/util.h" #include "common/checksum.h" #if TD_DARWIN || TD_LINUX #include -#include #endif #include -#include #include "git.h" using namespace std::literals::string_literals; @@ -77,22 +67,10 @@ using td::Ref; int verbosity; -std::unique_ptr TestNode::make_callback() { - class Callback : public ton::adnl::AdnlExtClient::Callback { - public: - void on_ready() override { - td::actor::send_closure(id_, &TestNode::conn_ready); - } - void on_stop_ready() override { - td::actor::send_closure(id_, &TestNode::conn_closed); - } - Callback(td::actor::ActorId id) : id_(std::move(id)) { - } - - private: - td::actor::ActorId id_; - }; - return std::make_unique(actor_id(this)); +bool TestNode::LiteServer::supports(ton::ShardIdFull shard) const { + return is_full || shard.is_masterchain() || + std::any_of(shards.begin(), shards.end(), + [&](const ton::ShardIdFull& our_shard) { return ton::shard_intersects(shard, our_shard); }); } void TestNode::run() { @@ -110,31 +88,57 @@ void TestNode::run() { io_ = td::TerminalIO::create("> ", readline_enabled_, ex_mode_, std::make_unique(actor_id(this))); td::actor::send_closure(io_, &td::TerminalIO::set_log_interface); - if (remote_public_key_.empty()) { - auto G = td::read_file(global_config_).move_as_ok(); - auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); - ton::ton_api::liteclient_config_global gc; - ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - CHECK(gc.liteservers_.size() > 0); - auto idx = liteserver_idx_ >= 0 ? liteserver_idx_ - : td::Random::fast(0, static_cast(gc.liteservers_.size() - 1)); - CHECK(idx >= 0 && static_cast(idx) <= gc.liteservers_.size()); - auto& cli = gc.liteservers_[idx]; - remote_addr_.init_host_port(td::IPAddress::ipv4_to_str(cli->ip_), cli->port_).ensure(); - remote_public_key_ = ton::PublicKey{cli->id_}; - td::TerminalIO::out() << "using liteserver " << idx << " with addr " << remote_addr_ << "\n"; - if (gc.validator_ && gc.validator_->zero_state_) { - zstate_id_.workchain = gc.validator_->zero_state_->workchain_; - if (zstate_id_.workchain != ton::workchainInvalid) { - zstate_id_.root_hash = gc.validator_->zero_state_->root_hash_; - zstate_id_.file_hash = gc.validator_->zero_state_->file_hash_; - td::TerminalIO::out() << "zerostate set to " << zstate_id_.to_str() << "\n"; - } + if (!single_remote_public_key_.empty()) { // Use single provided liteserver + LiteServer s; + s.addr = single_remote_addr_; + s.public_key = single_remote_public_key_; + single_liteserver_idx_ = 0; + td::TerminalIO::out() << "using liteserver " << s.addr << "\n"; + servers_.push_back(std::move(s)); + run_init_queries(); + return; + } + + auto G = td::read_file(global_config_).move_as_ok(); + auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); + ton::ton_api::liteclient_config_global gc; + ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); + CHECK(gc.liteservers_.size() + gc.liteservers_v2_.size() > 0); + + if (gc.validator_ && gc.validator_->zero_state_) { + zstate_id_.workchain = gc.validator_->zero_state_->workchain_; + if (zstate_id_.workchain != ton::workchainInvalid) { + zstate_id_.root_hash = gc.validator_->zero_state_->root_hash_; + zstate_id_.file_hash = gc.validator_->zero_state_->file_hash_; + td::TerminalIO::out() << "zerostate set to " << zstate_id_.to_str() << "\n"; } } - client_ = - ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull{remote_public_key_}, remote_addr_, make_callback()); + for (auto& server : gc.liteservers_) { + LiteServer s; + s.addr.init_host_port(td::IPAddress::ipv4_to_str(server->ip_), server->port_).ensure(); + s.public_key = ton::PublicKey{server->id_}; + servers_.push_back(std::move(s)); + } + for (auto& server : gc.liteservers_v2_) { + LiteServer s; + s.addr.init_host_port(td::IPAddress::ipv4_to_str(server->ip_), server->port_).ensure(); + s.public_key = ton::PublicKey{server->id_}; + s.is_full = false; + for (const auto& shard : server->shards_) { + s.shards.emplace_back(shard->workchain_, shard->shard_); + CHECK(s.shards.back().is_valid_ext()); + } + servers_.push_back(std::move(s)); + } + + if (single_liteserver_idx_ != -1) { // Use single liteserver from config + CHECK(single_liteserver_idx_ >= 0 && (size_t)single_liteserver_idx_ < gc.liteservers_.size()); + td::TerminalIO::out() << "using liteserver " << single_liteserver_idx_ << " with addr " + << servers_[single_liteserver_idx_].addr << "\n"; + } + + run_init_queries(); } void TestNode::got_result(td::Result R, td::Promise promise) { @@ -179,23 +183,142 @@ void TestNode::after_got_result(bool ok) { } } -bool TestNode::envelope_send_query(td::BufferSlice query, td::Promise promise) { - running_queries_++; - if (!ready_ || client_.empty()) { - got_result(td::Status::Error("failed to send query to server: not ready"), std::move(promise)); +bool TestNode::envelope_send_query_to_any(td::BufferSlice query, td::Promise promise) { + return envelope_send_query_to_shard(ton::ShardIdFull(ton::masterchainId), std::move(query), std::move(promise)); +} + +bool TestNode::envelope_send_query_to_account(ton::AccountIdPrefixFull prefix, td::BufferSlice query, + td::Promise promise) { + return envelope_send_query_to_shard(prefix.as_leaf_shard(), std::move(query), std::move(promise)); +} + +bool TestNode::envelope_send_query_to_shard(ton::ShardIdFull shard, td::BufferSlice query, + td::Promise promise) { + if (single_liteserver_idx_ >= 0) { + return envelope_send_query_to_server(single_liteserver_idx_, std::move(query), std::move(promise)); + } + if (shard.is_masterchain() && mc_server_idx_ != -1) { + return envelope_send_query_to_server(mc_server_idx_, std::move(query), std::move(promise)); + } + auto it = shard_server_idx_cached_.find(shard); + if (it != shard_server_idx_cached_.end()) { + return envelope_send_query_to_server(it->second, std::move(query), std::move(promise)); + } + int server_idx = -1; + int random_idx = -1; + int cnt = 0; + bool selected_full = false; + for (int i = 0; i < (int)servers_.size(); ++i) { + const LiteServer& server = servers_[i]; + if (!server.supports(shard)) { + continue; + } + if (server.is_full && !selected_full) { + selected_full = true; + server_idx = -1; + cnt = 0; + } + if (!server.is_full && selected_full) { + continue; + } + if (!server.client.empty()) { + server_idx = i; + } + if (td::Random::fast(0, cnt) == 0) { + random_idx = i; + } + ++cnt; + } + if (server_idx == -1) { + server_idx = random_idx; + } + if (server_idx == -1) { + running_queries_++; + got_result(td::Status::Error(PSTRING() << "failed to select a suitable server for " << shard.to_str()), + std::move(promise)); return false; } + shard_server_idx_cached_[shard] = server_idx; + if (shard.is_masterchain()) { + mc_server_idx_ = server_idx; + } + return envelope_send_query_to_server(server_idx, std::move(query), std::move(promise)); +} + +bool TestNode::envelope_send_query_to_server(td::int32 server_idx, td::BufferSlice query, + td::Promise promise) { + running_queries_++; + LiteServer& server = servers_.at(server_idx); + if (server.client.empty()) { + start_client(server_idx); + } + CHECK(!server.client.empty()); + auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { td::actor::send_closure(SelfId, &TestNode::got_result, std::move(R), std::move(promise)); }); td::BufferSlice b = ton::serialize_tl_object(ton::create_tl_object(std::move(query)), true); - td::actor::send_closure(client_, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b), - td::Timestamp::in(10.0), std::move(P)); + if (server.client_ready) { + td::actor::send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b), + td::Timestamp::in(10.0), std::move(P)); + } else { + server.wait_client_ready.push_back( + [client = server.client.get(), b = std::move(b), P = std::move(P)](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(client, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b), + td::Timestamp::in(10.0), std::move(P)); + } else { + P.set_error(R.move_as_error_prefix("failed to connect: ")); + } + }); + } return true; } +void TestNode::start_client(int server_idx) { + LiteServer& server = servers_[server_idx]; + CHECK(server.client.empty()); + class Callback : public ton::adnl::AdnlExtClient::Callback { + public: + void on_ready() override { + td::actor::send_closure(id_, &TestNode::conn_ready, server_idx_); + } + void on_stop_ready() override { + td::actor::send_closure(id_, &TestNode::conn_closed, server_idx_); + } + Callback(td::actor::ActorId id, int server_idx) : id_(std::move(id)), server_idx_(server_idx) { + } + + private: + td::actor::ActorId id_; + int server_idx_; + }; + server.client_ready = false; + server.wait_client_ready.clear(); + LOG(INFO) << "Connecting to " << server.addr << " (liteserver #" << server_idx << ")"; + server.client = ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull{server.public_key}, server.addr, + std::make_unique(actor_id(this), server_idx)); +} + +void TestNode::conn_ready(int server_idx) { + LiteServer& server = servers_[server_idx]; + LOG(INFO) << "Connection to " << server.addr << " (liteserver #" << server_idx << ") is ready"; + server.client_ready = true; + for (auto& p : server.wait_client_ready) { + p.set_result(td::Unit()); + } + server.wait_client_ready.clear(); +} + +void TestNode::conn_closed(int server_idx) { + LiteServer& server = servers_[server_idx]; + LOG(INFO) << "Connection to " << server.addr << " (liteserver #" << server_idx << ") closed"; + server.client_ready = false; + server.wait_client_ready.clear(); +} + td::Promise TestNode::trivial_promise() { return td::PromiseCreator::lambda([Self = actor_id(this)](td::Result res) { if (res.is_error()) { @@ -310,7 +433,7 @@ bool TestNode::dump_cached_cell(td::Slice hash_pfx, td::Slice type_name) { bool TestNode::get_server_time() { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); - return envelope_send_query(std::move(b), [&, Self = actor_id(this)](td::Result res) -> void { + return envelope_send_query_to_any(std::move(b), [&, Self = actor_id(this)](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot get server time"; return; @@ -319,9 +442,10 @@ bool TestNode::get_server_time() { if (F.is_error()) { LOG(ERROR) << "cannot parse answer to liteServer.getTime"; } else { - server_time_ = F.move_as_ok()->now_; - server_time_got_at_ = now(); - LOG(INFO) << "server time is " << server_time_ << " (delta " << server_time_ - server_time_got_at_ << ")"; + mc_server_time_ = F.move_as_ok()->now_; + mc_server_time_got_at_ = now(); + LOG(INFO) << "server time is " << mc_server_time_ << " (delta " << mc_server_time_ - mc_server_time_got_at_ + << ")"; } } }); @@ -329,13 +453,13 @@ bool TestNode::get_server_time() { bool TestNode::get_server_version(int mode) { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), mode](td::Result res) { + return envelope_send_query_to_any(std::move(b), [Self = actor_id(this), mode](td::Result res) { td::actor::send_closure_later(Self, &TestNode::got_server_version, std::move(res), mode); }); }; void TestNode::got_server_version(td::Result res, int mode) { - server_ok_ = false; + mc_server_ok_ = false; if (res.is_error()) { LOG(ERROR) << "cannot get server version and time (server too old?)"; } else { @@ -344,11 +468,11 @@ void TestNode::got_server_version(td::Result res, int mode) { LOG(ERROR) << "cannot parse answer to liteServer.getVersion"; } else { auto a = F.move_as_ok(); - set_server_version(a->version_, a->capabilities_); - set_server_time(a->now_); + set_mc_server_version(a->version_, a->capabilities_); + set_mc_server_time(a->now_); } } - if (!server_ok_) { + if (!mc_server_ok_) { LOG(ERROR) << "server version is too old (at least " << (min_ls_version >> 8) << "." << (min_ls_version & 0xff) << " with capabilities " << min_ls_capabilities << " required), some queries are unavailable"; } @@ -357,27 +481,27 @@ void TestNode::got_server_version(td::Result res, int mode) { } } -void TestNode::set_server_version(td::int32 version, td::int64 capabilities) { - if (server_version_ != version || server_capabilities_ != capabilities) { - server_version_ = version; - server_capabilities_ = capabilities; - LOG(WARNING) << "server version is " << (server_version_ >> 8) << "." << (server_version_ & 0xff) - << ", capabilities " << server_capabilities_; +void TestNode::set_mc_server_version(td::int32 version, td::int64 capabilities) { + if (mc_server_version_ != version || mc_server_capabilities_ != capabilities) { + mc_server_version_ = version; + mc_server_capabilities_ = capabilities; + LOG(WARNING) << "server version is " << (mc_server_version_ >> 8) << "." << (mc_server_version_ & 0xff) + << ", capabilities " << mc_server_capabilities_; } - server_ok_ = (server_version_ >= min_ls_version) && !(~server_capabilities_ & min_ls_capabilities); + mc_server_ok_ = (mc_server_version_ >= min_ls_version) && !(~mc_server_capabilities_ & min_ls_capabilities); } -void TestNode::set_server_time(int server_utime) { - server_time_ = server_utime; - server_time_got_at_ = now(); - LOG(INFO) << "server time is " << server_time_ << " (delta " << server_time_ - server_time_got_at_ << ")"; +void TestNode::set_mc_server_time(int server_utime) { + mc_server_time_ = server_utime; + mc_server_time_got_at_ = now(); + LOG(INFO) << "server time is " << mc_server_time_ << " (delta " << mc_server_time_ - mc_server_time_got_at_ << ")"; } bool TestNode::get_server_mc_block_id() { - int mode = (server_capabilities_ & 2) ? 0 : -1; + int mode = (mc_server_capabilities_ & 2) ? 0 : -1; if (mode < 0) { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); - return envelope_send_query(std::move(b), [Self = actor_id(this)](td::Result res) -> void { + return envelope_send_query_to_any(std::move(b), [Self = actor_id(this)](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot get masterchain info from server"; return; @@ -397,24 +521,25 @@ bool TestNode::get_server_mc_block_id() { } else { auto b = ton::serialize_tl_object(ton::create_tl_object(mode), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), mode](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot get extended masterchain info from server"; - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getMasterchainInfoExt"; - } else { - auto f = F.move_as_ok(); - auto blk_id = create_block_id(f->last_); - auto zstate_id = create_zero_state_id(f->init_); - LOG(INFO) << "last masterchain block is " << blk_id.to_str(); - td::actor::send_closure_later(Self, &TestNode::got_server_mc_block_id_ext, blk_id, zstate_id, mode, - f->version_, f->capabilities_, f->last_utime_, f->now_); - } - } - }); + return envelope_send_query_to_any( + std::move(b), [Self = actor_id(this), mode](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "cannot get extended masterchain info from server"; + return; + } else { + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getMasterchainInfoExt"; + } else { + auto f = F.move_as_ok(); + auto blk_id = create_block_id(f->last_); + auto zstate_id = create_zero_state_id(f->init_); + LOG(INFO) << "last masterchain block is " << blk_id.to_str(); + td::actor::send_closure_later(Self, &TestNode::got_server_mc_block_id_ext, blk_id, zstate_id, mode, + f->version_, f->capabilities_, f->last_utime_, f->now_); + } + } + }); } } @@ -448,8 +573,8 @@ void TestNode::got_server_mc_block_id(ton::BlockIdExt blkid, ton::ZeroStateIdExt void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateIdExt zstateid, int mode, int version, long long capabilities, int last_utime, int server_now) { - set_server_version(version, capabilities); - set_server_time(server_now); + set_mc_server_version(version, capabilities); + set_mc_server_time(server_now); if (last_utime > server_now) { LOG(WARNING) << "server claims to have a masterchain block " << blkid.to_str() << " created at " << last_utime << " (" << last_utime - server_now << " seconds in the future)"; @@ -457,10 +582,10 @@ void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateI LOG(WARNING) << "server appears to be out of sync: its newest masterchain block is " << blkid.to_str() << " created at " << last_utime << " (" << server_now - last_utime << " seconds ago according to the server's clock)"; - } else if (last_utime < server_time_got_at_ - 60) { + } else if (last_utime < mc_server_time_got_at_ - 60) { LOG(WARNING) << "either the server is out of sync, or the local clock is set incorrectly: the newest masterchain " "block known to server is " - << blkid.to_str() << " created at " << last_utime << " (" << server_now - server_time_got_at_ + << blkid.to_str() << " created at " << last_utime << " (" << server_now - mc_server_time_got_at_ << " seconds ago according to the local clock)"; } got_server_mc_block_id(blkid, zstateid, last_utime); @@ -469,52 +594,54 @@ void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateI bool TestNode::request_block(ton::BlockIdExt blkid) { auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot obtain block " << blkid.to_str() << " from server"; - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getBlock"; - } else { - auto f = F.move_as_ok(); - auto blk_id = ton::create_block_id(f->id_); - LOG(INFO) << "obtained block " << blk_id.to_str() << " from server"; - if (blk_id != blkid) { - LOG(ERROR) << "block id mismatch: expected data for block " << blkid.to_str() << ", obtained for " - << blk_id.to_str(); + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "cannot obtain block " << blkid.to_str() << " from server"; + return; + } else { + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getBlock"; + } else { + auto f = F.move_as_ok(); + auto blk_id = ton::create_block_id(f->id_); + LOG(INFO) << "obtained block " << blk_id.to_str() << " from server"; + if (blk_id != blkid) { + LOG(ERROR) << "block id mismatch: expected data for block " << blkid.to_str() << ", obtained for " + << blk_id.to_str(); + } + td::actor::send_closure_later(Self, &TestNode::got_mc_block, blk_id, std::move(f->data_)); + } } - td::actor::send_closure_later(Self, &TestNode::got_mc_block, blk_id, std::move(f->data_)); - } - } - }); + }); } bool TestNode::request_state(ton::BlockIdExt blkid) { auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot obtain state " << blkid.to_str() << " from server"; - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getState"; - } else { - auto f = F.move_as_ok(); - auto blk_id = ton::create_block_id(f->id_); - LOG(INFO) << "obtained state " << blk_id.to_str() << " from server"; - if (blk_id != blkid) { - LOG(ERROR) << "block id mismatch: expected state for block " << blkid.to_str() << ", obtained for " - << blk_id.to_str(); + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "cannot obtain state " << blkid.to_str() << " from server"; + return; + } else { + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getState"; + } else { + auto f = F.move_as_ok(); + auto blk_id = ton::create_block_id(f->id_); + LOG(INFO) << "obtained state " << blk_id.to_str() << " from server"; + if (blk_id != blkid) { + LOG(ERROR) << "block id mismatch: expected state for block " << blkid.to_str() << ", obtained for " + << blk_id.to_str(); + } + td::actor::send_closure_later(Self, &TestNode::got_mc_state, blk_id, f->root_hash_, f->file_hash_, + std::move(f->data_)); + } } - td::actor::send_closure_later(Self, &TestNode::got_mc_state, blk_id, f->root_hash_, f->file_hash_, - std::move(f->data_)); - } - } - }); + }); } void TestNode::got_mc_block(ton::BlockIdExt blkid, td::BufferSlice data) { @@ -1150,27 +1277,34 @@ td::Status TestNode::send_ext_msg_from_filename(std::string filename) { LOG(ERROR) << "failed to read file `" << filename << "`: " << err.to_string(); return err; } - if (ready_ && !client_.empty()) { - LOG(ERROR) << "sending query from file " << filename; - auto P = td::PromiseCreator::lambda([](td::Result R) { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.sendMessage"; - } else { - int status = F.move_as_ok()->status_; - LOG(INFO) << "external message status is " << status; - } - }); - auto b = - ton::serialize_tl_object(ton::create_tl_object(F.move_as_ok()), true); - return envelope_send_query(std::move(b), std::move(P)) ? td::Status::OK() - : td::Status::Error("cannot send query to server"); - } else { - return td::Status::Error("server connection not ready"); + LOG(ERROR) << "sending query from file " << filename; + + TRY_RESULT_PREFIX(root, vm::std_boc_deserialize(F.ok().as_slice()), "invalid boc: "); + block::gen::CommonMsgInfo::Record_ext_in_msg_info info; + if (!tlb::unpack_cell_inexact(root, info)) { + return td::Status::Error("failed to unpack external message header"); + } + auto dest_prefix = block::tlb::MsgAddressInt::get_prefix(info.dest); + if (!dest_prefix.is_valid()) { + return td::Status::Error("destination of the message is invalid"); } + + auto P = td::PromiseCreator::lambda([](td::Result R) { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.sendMessage"; + } else { + int status = F.move_as_ok()->status_; + LOG(INFO) << "external message status is " << status; + } + }); + auto b = ton::serialize_tl_object(ton::create_tl_object(F.move_as_ok()), true); + return envelope_send_query_to_account(dest_prefix, std::move(b), std::move(P)) + ? td::Status::OK() + : td::Status::Error("cannot send query to server"); } bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::BlockIdExt ref_blkid, @@ -1178,9 +1312,6 @@ bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress if (!ref_blkid.is_valid()) { return set_error("must obtain last block information before making other queries"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } if (addr_ext) { return get_special_smc_addr( addr_ext, [this, ref_blkid, filename, mode, prunned](td::Result res) { @@ -1202,24 +1333,26 @@ bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress ton::create_tl_lite_block_id(ref_blkid), std::move(a)), true); } + ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); LOG(INFO) << "requesting " << (prunned ? "prunned " : "") << "account state for " << workchain << ":" << addr.to_hex() << " with respect to " << ref_blkid.to_str() << " with savefile `" << filename << "` and mode " << mode; - return envelope_send_query(std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, filename, mode, - prunned](td::Result R) { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getAccountState"; - } else { - auto f = F.move_as_ok(); - td::actor::send_closure_later(Self, &TestNode::got_account_state, ref_blkid, ton::create_block_id(f->id_), - ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), - std::move(f->proof_), std::move(f->state_), workchain, addr, filename, mode, - prunned); - } - }); + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, ref_blkid, filename, mode, prunned](td::Result R) { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getAccountState"; + } else { + auto f = F.move_as_ok(); + td::actor::send_closure_later(Self, &TestNode::got_account_state, ref_blkid, ton::create_block_id(f->id_), + ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), + std::move(f->proof_), std::move(f->state_), workchain, addr, filename, mode, + prunned); + } + }); } td::int64 TestNode::compute_method_id(std::string method) { @@ -1285,19 +1418,18 @@ bool TestNode::start_run_method(ton::WorkchainId workchain, ton::StdSmcAddress a if (!ref_blkid.is_valid()) { return set_error("must obtain last block information before making other queries"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto a = ton::create_tl_object(workchain, addr); + ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); if (!mode) { auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(ref_blkid), std::move(a)), true); LOG(INFO) << "requesting account state for " << workchain << ":" << addr.to_hex() << " with respect to " << ref_blkid.to_str() << " to run method " << method_name << " with " << params.size() << " parameters"; - return envelope_send_query( - std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, method_name, params = std::move(params), - promise = std::move(promise)](td::Result R) mutable { + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, ref_blkid, method_name, params = std::move(params), + promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); return; @@ -1337,26 +1469,27 @@ bool TestNode::start_run_method(ton::WorkchainId workchain, ton::StdSmcAddress a LOG(INFO) << "requesting remote get-method execution for " << workchain << ":" << addr.to_hex() << " with respect to " << ref_blkid.to_str() << " to run method " << method_name << " with " << params.size() << " parameters"; - return envelope_send_query(std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, method_name, mode, - params = std::move(params), - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.runSmcMethod"; - promise.set_error(td::Status::Error("cannot parse answer to liteServer.runSmcMethod")); - } else { - auto f = F.move_as_ok(); - td::actor::send_closure_later(Self, &TestNode::run_smc_method, mode, ref_blkid, ton::create_block_id(f->id_), - ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), - std::move(f->proof_), std::move(f->state_proof_), workchain, addr, method_name, - std::move(params), std::move(f->init_c7_), std::move(f->lib_extras_), - std::move(f->result_), f->exit_code_, std::move(promise)); - } - }); + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, ref_blkid, method_name, mode, params = std::move(params), + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.runSmcMethod"; + promise.set_error(td::Status::Error("cannot parse answer to liteServer.runSmcMethod")); + } else { + auto f = F.move_as_ok(); + td::actor::send_closure_later( + Self, &TestNode::run_smc_method, mode, ref_blkid, ton::create_block_id(f->id_), + ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), std::move(f->proof_), + std::move(f->state_proof_), workchain, addr, method_name, std::move(params), std::move(f->init_c7_), + std::move(f->lib_extras_), std::move(f->result_), f->exit_code_, std::move(promise)); + } + }); } } @@ -1586,8 +1719,8 @@ void TestNode::send_compute_complaint_price_query(ton::StdSmcAddress elector_add params.emplace_back(td::make_refint(bits)); params.emplace_back(td::make_refint(refs)); params.emplace_back(td::make_refint(expires_in)); - auto P = td::PromiseCreator::lambda( - [this, expires_in, bits, refs, chash, filename](td::Result> R) { + auto P = + td::PromiseCreator::lambda([expires_in, bits, refs, chash, filename](td::Result> R) { if (R.is_error()) { LOG(ERROR) << R.move_as_error(); return; @@ -1649,10 +1782,6 @@ bool TestNode::dns_resolve_start(ton::WorkchainId workchain, ton::StdSmcAddress return set_error("domain name too long"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } - if (workchain == ton::workchainInvalid) { if (dns_root_queried_) { workchain = ton::masterchainId; @@ -1861,17 +1990,16 @@ bool TestNode::get_one_transaction(ton::BlockIdExt blkid, ton::WorkchainId workc if (!ton::shard_contains(blkid.shard_full(), ton::extract_addr_prefix(workchain, addr))) { return set_error("the shard of this block cannot contain this account"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto a = ton::create_tl_object(workchain, addr); auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(blkid), std::move(a), lt), true); + ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); LOG(INFO) << "requesting transaction " << lt << " of " << workchain << ":" << addr.to_hex() << " from block " << blkid.to_str(); - return envelope_send_query( - std::move(b), [Self = actor_id(this), workchain, addr, lt, blkid, dump](td::Result R) -> void { + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, lt, blkid, dump](td::Result R) -> void { if (R.is_error()) { return; } @@ -1888,16 +2016,15 @@ bool TestNode::get_one_transaction(ton::BlockIdExt blkid, ton::WorkchainId workc bool TestNode::get_last_transactions(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::LogicalTime lt, ton::Bits256 hash, unsigned count, bool dump) { - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto a = ton::create_tl_object(workchain, addr); auto b = ton::serialize_tl_object( ton::create_tl_object(count, std::move(a), lt, hash), true); + ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); LOG(INFO) << "requesting " << count << " last transactions from " << lt << ":" << hash.to_hex() << " of " << workchain << ":" << addr.to_hex(); - return envelope_send_query( - std::move(b), [Self = actor_id(this), workchain, addr, lt, hash, count, dump](td::Result R) { + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, lt, hash, count, dump](td::Result R) { if (R.is_error()) { return; } @@ -2257,10 +2384,10 @@ void TestNode::got_one_transaction(ton::BlockIdExt req_blkid, ton::BlockIdExt bl << " but received data has " << root->get_hash().bits().to_hex(256); return; } - } catch (vm::VmError err) { + } catch (vm::VmError& err) { LOG(ERROR) << "error while traversing block transaction proof : " << err.get_msg(); return; - } catch (vm::VmVirtError err) { + } catch (vm::VmVirtError& err) { LOG(ERROR) << "virtualization error while traversing block transaction proof : " << err.get_msg(); return; } @@ -2439,32 +2566,30 @@ void TestNode::got_last_transactions(std::vector blkids, td::Bu bool TestNode::get_block_transactions(ton::BlockIdExt blkid, int mode, unsigned count, ton::Bits256 acc_addr, ton::LogicalTime lt) { - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto a = ton::create_tl_object(acc_addr, lt); auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(blkid), mode, count, std::move(a), false, false), true); LOG(INFO) << "requesting " << count << " transactions from block " << blkid.to_str() << " starting from account " << acc_addr.to_hex() << " lt " << lt; - return envelope_send_query(std::move(b), [Self = actor_id(this), mode](td::Result R) { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.listBlockTransactions"; - } else { - auto f = F.move_as_ok(); - std::vector transactions; - for (auto& id : f->ids_) { - transactions.emplace_back(id->account_, id->lt_, id->hash_); - } - td::actor::send_closure_later(Self, &TestNode::got_block_transactions, ton::create_block_id(f->id_), mode, - f->req_count_, f->incomplete_, std::move(transactions), std::move(f->proof_)); - } - }); + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), mode](td::Result R) { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.listBlockTransactions"; + } else { + auto f = F.move_as_ok(); + std::vector transactions; + for (auto& id : f->ids_) { + transactions.emplace_back(id->account_, id->lt_, id->hash_); + } + td::actor::send_closure_later(Self, &TestNode::got_block_transactions, ton::create_block_id(f->id_), mode, + f->req_count_, f->incomplete_, std::move(transactions), std::move(f->proof_)); + } + }); } void TestNode::got_block_transactions(ton::BlockIdExt blkid, int mode, unsigned req_count, bool incomplete, @@ -2490,25 +2615,23 @@ bool TestNode::get_all_shards(std::string filename, bool use_last, ton::BlockIdE if (!blkid.is_masterchain()) { return set_error("only masterchain blocks contain shard configuration"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); LOG(INFO) << "requesting recent shard configuration"; - return envelope_send_query(std::move(b), [Self = actor_id(this), filename](td::Result R) -> void { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getAllShardsInfo"; - } else { - auto f = F.move_as_ok(); - td::actor::send_closure_later(Self, &TestNode::got_all_shards, ton::create_block_id(f->id_), std::move(f->proof_), - std::move(f->data_), filename); - } - }); + return envelope_send_query_to_any( + std::move(b), [Self = actor_id(this), filename](td::Result R) -> void { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getAllShardsInfo"; + } else { + auto f = F.move_as_ok(); + td::actor::send_closure_later(Self, &TestNode::got_all_shards, ton::create_block_id(f->id_), + std::move(f->proof_), std::move(f->data_), filename); + } + }); } void TestNode::got_all_shards(ton::BlockIdExt blk, td::BufferSlice proof, td::BufferSlice data, std::string filename) { @@ -2572,9 +2695,6 @@ bool TestNode::parse_get_config_params(ton::BlockIdExt blkid, int mode, std::str params.push_back(x); } } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } if (!blkid.is_masterchain_ext()) { return set_error("only masterchain blocks contain configuration"); } @@ -2593,10 +2713,6 @@ bool TestNode::get_config_params(ton::BlockIdExt blkid, td::Promise promise, int mode, std::string filename, std::vector params) { - if (!(ready_ && !client_.empty())) { - promise.set_error(td::Status::Error("server connection not ready")); - return false; - } if (!blkid.is_masterchain_ext()) { promise.set_error(td::Status::Error("masterchain reference block expected")); return false; @@ -2614,11 +2730,12 @@ bool TestNode::get_config_params_ext(ton::BlockIdExt blkid, td::Promise R) mutable { - td::actor::send_closure_later(Self, &TestNode::got_config_params, blkid, mode, filename, std::move(params), - std::move(R), std::move(promise)); - }); + return envelope_send_query_to_any( + std::move(b), [Self = actor_id(this), mode, filename, blkid, params = std::move(params), + promise = std::move(promise)](td::Result R) mutable { + td::actor::send_closure_later(Self, &TestNode::got_config_params, blkid, mode, filename, std::move(params), + std::move(R), std::move(promise)); + }); } void TestNode::got_config_params(ton::BlockIdExt req_blkid, int mode, std::string filename, std::vector params, @@ -2806,8 +2923,8 @@ bool TestNode::get_block(ton::BlockIdExt blkid, bool dump) { LOG(INFO) << "got block download request for " << blkid.to_str(); auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query( - std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot obtain block " << blkid.to_str() << " from server : " << res.move_as_error().to_string(); @@ -2835,8 +2952,8 @@ bool TestNode::get_state(ton::BlockIdExt blkid, bool dump) { LOG(INFO) << "got state download request for " << blkid.to_str(); auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query( - std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot obtain state " << blkid.to_str() << " from server : " << res.move_as_error().to_string(); @@ -2973,7 +3090,7 @@ void TestNode::got_state(ton::BlockIdExt blkid, ton::RootHash root_hash, ton::Fi } bool TestNode::get_show_block_header(ton::BlockIdExt blkid, int mode) { - return get_block_header(blkid, mode, [this, blkid](td::Result R) { + return get_block_header(blkid, mode, [this](td::Result R) { if (R.is_error()) { LOG(ERROR) << "unable to fetch block header: " << R.move_as_error(); } else { @@ -2988,8 +3105,9 @@ bool TestNode::get_block_header(ton::BlockIdExt blkid, int mode, td::Promise(ton::create_tl_lite_block_id(blkid), mode), true); - return envelope_send_query( - std::move(b), [this, blkid, promise = std::move(promise)](td::Result R) mutable -> void { + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), + [this, blkid, promise = std::move(promise)](td::Result R) mutable -> void { TRY_RESULT_PROMISE_PREFIX(promise, res, std::move(R), PSLICE() << "cannot obtain block header for " << blkid.to_str() << " from server :"); got_block_header_raw(std::move(res), std::move(promise), blkid); @@ -3015,8 +3133,9 @@ bool TestNode::lookup_block(ton::ShardIdFull shard, int mode, td::uint64 arg, auto b = ton::serialize_tl_object(ton::create_tl_object( mode, ton::create_tl_lite_block_id_simple(id), arg, (td::uint32)arg), true); - return envelope_send_query( - std::move(b), [this, id, mode, arg, promise = std::move(promise)](td::Result R) mutable -> void { + return envelope_send_query_to_shard( + shard, std::move(b), + [this, id, mode, arg, promise = std::move(promise)](td::Result R) mutable -> void { TRY_RESULT_PROMISE_PREFIX(promise, res, std::move(R), PSLICE() << "cannot look up block header for " << id.to_str() << " with mode " << mode << " and argument " << arg << " from server :"); @@ -3130,9 +3249,9 @@ void TestNode::got_block_header(ton::BlockIdExt blkid, td::BufferSlice data, int return; } show_block_header(blkid, std::move(virt_root), mode); - } catch (vm::VmError err) { + } catch (vm::VmError& err) { LOG(ERROR) << "error processing header for " << blkid.to_str() << " : " << err.get_msg(); - } catch (vm::VmVirtError err) { + } catch (vm::VmVirtError& err) { LOG(ERROR) << "error processing header for " << blkid.to_str() << " : " << err.get_msg(); } show_new_blkids(); @@ -3161,14 +3280,15 @@ bool TestNode::get_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mod ton::serialize_tl_object(ton::create_tl_object( mode & 0xfff, ton::create_tl_lite_block_id(from), ton::create_tl_lite_block_id(to)), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), from, to, mode](td::Result res) { - if (res.is_error()) { - LOG(ERROR) << "cannot obtain block proof for " << ((mode & 1) ? to.to_str() : "last masterchain block") - << " starting from " << from.to_str() << " from server : " << res.move_as_error().to_string(); - } else { - td::actor::send_closure_later(Self, &TestNode::got_block_proof, from, to, mode, res.move_as_ok()); - } - }); + return envelope_send_query_to_any( + std::move(b), [Self = actor_id(this), from, to, mode](td::Result res) { + if (res.is_error()) { + LOG(ERROR) << "cannot obtain block proof for " << ((mode & 1) ? to.to_str() : "last masterchain block") + << " starting from " << from.to_str() << " from server : " << res.move_as_error().to_string(); + } else { + td::actor::send_closure_later(Self, &TestNode::got_block_proof, from, to, mode, res.move_as_ok()); + } + }); } void TestNode::got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mode, td::BufferSlice pchain) { @@ -3222,9 +3342,6 @@ void TestNode::got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mod bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_count, ton::Bits256 start_after, ton::UnixTime min_utime) { - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } if (!blkid.is_masterchain_ext()) { return set_error("only masterchain blocks contain block creator statistics"); } @@ -3235,8 +3352,8 @@ bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_c auto& os = *osp; return get_creator_stats( blkid, mode, req_count, start_after, min_utime, - [min_utime, &os](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, - const block::DiscountedCounter& shard_cnt) -> bool { + [&os](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, + const block::DiscountedCounter& shard_cnt) -> bool { os << key.to_hex() << " mc_cnt:" << mc_cnt << " shard_cnt:" << shard_cnt << std::endl; return true; }, @@ -3265,10 +3382,6 @@ bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_c bool TestNode::get_creator_stats(ton::BlockIdExt blkid, unsigned req_count, ton::UnixTime min_utime, TestNode::creator_stats_func_t func, std::unique_ptr state, td::Promise> promise) { - if (!(ready_ && !client_.empty())) { - promise.set_error(td::Status::Error("server connection not ready")); - return false; - } if (!state) { promise.set_error(td::Status::Error("null CreatorStatsRes")); return false; @@ -3287,7 +3400,7 @@ bool TestNode::get_creator_stats(ton::BlockIdExt blkid, unsigned req_count, ton: LOG(INFO) << "requesting up to " << req_count << " block creator stats records with respect to masterchain block " << blkid.to_str() << " starting from validator public key " << state->last_key.to_hex() << " created after " << min_utime << " (mode=" << state->mode << ")"; - return envelope_send_query( + return envelope_send_query_to_any( std::move(b), [this, blkid, req_count, state = std::move(state), min_utime, func = std::move(func), promise = std::move(promise)](td::Result R) mutable { TRY_RESULT_PROMISE(promise, res, std::move(R)); @@ -3504,8 +3617,8 @@ bool TestNode::load_creator_stats(std::unique_ptr l ton::UnixTime min_utime = info.valid_since - 1000; return get_creator_stats( info.blk_id, 1000, min_utime, - [min_utime, &info](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, - const block::DiscountedCounter& shard_cnt) -> bool { + [&info](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, + const block::DiscountedCounter& shard_cnt) -> bool { info.store_record(key, mc_cnt, shard_cnt); return true; }, @@ -3708,7 +3821,7 @@ bool compute_punishment_default(int interval, bool severe, td::RefInt256& fine, fine = td::make_refint(101 * 1000000000LL); // 101 fine_part = 0; - return true; // todo: (tolya-yanot) temporary reduction of fine + return true; // todo: (tolya-yanot) temporary reduction of fine if (severe) { fine = td::make_refint(2500 * 1000000000LL); // GR$2500 @@ -3730,41 +3843,49 @@ bool compute_punishment_default(int interval, bool severe, td::RefInt256& fine, return true; } -bool compute_punishment(int interval, bool severe, td::RefInt256& fine, unsigned& fine_part, Ref punishment_params) { - if(punishment_params.is_null()) { +bool compute_punishment(int interval, bool severe, td::RefInt256& fine, unsigned& fine_part, + Ref punishment_params) { + if (punishment_params.is_null()) { return compute_punishment_default(interval, severe, fine, fine_part); } block::gen::MisbehaviourPunishmentConfig::Record rec; if (!tlb::unpack_cell(punishment_params, rec)) { - return false; + return false; } - if(interval <= rec.unpunishable_interval) { - return false; + if (interval <= rec.unpunishable_interval) { + return false; } fine = block::tlb::t_Grams.as_integer(rec.default_flat_fine); fine_part = rec.default_proportional_fine; if (severe) { - fine = fine * rec.severity_flat_mult; fine >>= 8; - fine_part = fine_part * rec.severity_proportional_mult; fine_part >>= 8; + fine = fine * rec.severity_flat_mult; + fine >>= 8; + fine_part = fine_part * rec.severity_proportional_mult; + fine_part >>= 8; } if (interval >= rec.long_interval) { - fine = fine * rec.long_flat_mult; fine >>= 8; - fine_part = fine_part * rec.long_proportional_mult; fine_part >>= 8; + fine = fine * rec.long_flat_mult; + fine >>= 8; + fine_part = fine_part * rec.long_proportional_mult; + fine_part >>= 8; return true; } if (interval >= rec.medium_interval) { - fine = fine * rec.medium_flat_mult; fine >>= 8; - fine_part = fine_part * rec.medium_proportional_mult; fine_part >>= 8; + fine = fine * rec.medium_flat_mult; + fine >>= 8; + fine_part = fine_part * rec.medium_proportional_mult; + fine_part >>= 8; return true; } return true; } -bool check_punishment(int interval, bool severe, td::RefInt256 fine, unsigned fine_part, Ref punishment_params) { +bool check_punishment(int interval, bool severe, td::RefInt256 fine, unsigned fine_part, + Ref punishment_params) { td::RefInt256 computed_fine; unsigned computed_fine_part; return compute_punishment(interval, severe, computed_fine, computed_fine_part, punishment_params) && @@ -3800,7 +3921,7 @@ td::Status TestNode::write_val_create_proof(TestNode::ValidatorLoadInfo& info1, int severity = (severe ? 2 : 1); td::RefInt256 fine = td::make_refint(101000000000); - unsigned fine_part = 0; // todo: (tolya-yanot) temporary reduction of fine // 0xffffffff / 16; // 1/16 + unsigned fine_part = 0; // todo: (tolya-yanot) temporary reduction of fine // 0xffffffff / 16; // 1/16 if (!compute_punishment(interval, severe, fine, fine_part, punishment_params)) { return td::Status::Error("cannot compute adequate punishment"); } @@ -3899,7 +4020,7 @@ td::Result> TestNode::ValidatorLoadInfo::build_proof(int idx, td:: block::gen::ValidatorDescr::Record_validator_addr rec2; if (tlb::csr_unpack(entry, rec1)) { pk = std::move(rec1.public_key); - } else if (tlb::csr_unpack(std::move(entry), rec2)) { + } else if (tlb::csr_unpack(entry, rec2)) { pk = std::move(rec2.public_key); } else { return td::Status::Error("cannot unpack ValidatorDescr"); @@ -4115,7 +4236,8 @@ td::Status TestNode::continue_check_validator_load_proof(std::unique_ptrconfig->get_config_param(40))) { + if (!check_punishment(interval, severe, suggested_fine, rec.suggested_fine_part, + info2->config->get_config_param(40))) { LOG(ERROR) << "proposed punishment (fine " << td::dec_string(suggested_fine) << ", fine_part=" << (double)rec.suggested_fine_part / (1LL << 32) << " is too harsh"; show_vote(root->get_hash().bits(), false); @@ -4264,8 +4386,9 @@ int main(int argc, char* argv[]) { return (verbosity >= 0 && verbosity <= 9) ? td::Status::OK() : td::Status::Error("verbosity must be 0..9"); }); p.add_option('V', "version", "shows lite-client build information", [&]() { - std::cout << "lite-client build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; - + std::cout << "lite-client build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::exit(0); }); p.add_option('i', "idx", "set liteserver idx", [&](td::Slice arg) { diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 34d02011df..018a9298af 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -57,8 +57,7 @@ class TestNode : public td::actor::Actor { bool client_ready = false; std::vector> wait_client_ready; - int max_common_prefix(ton::ShardIdFull shard) const; - bool supports_shard(ton::ShardIdFull shard) const; + bool supports(ton::ShardIdFull shard) const; }; std::vector servers_; @@ -352,9 +351,6 @@ class TestNode : public td::actor::Actor { bool parse_shard_id(ton::ShardIdFull& shard); bool parse_block_id_ext(ton::BlockIdExt& blkid, bool allow_incomplete = false); bool parse_block_id_ext(std::string blk_id_string, ton::BlockIdExt& blkid, bool allow_incomplete = false) const; - bool parse_stack_value(td::Slice str, vm::StackEntry& value); - bool parse_stack_value(vm::StackEntry& value); - bool parse_stack_values(std::vector& values); bool register_blkid(const ton::BlockIdExt& blkid); bool show_new_blkids(bool all = false); bool complete_blkid(ton::BlockId partial_blkid, ton::BlockIdExt& complete_blkid) const; @@ -458,8 +454,7 @@ class TestNode : public td::actor::Actor { void parse_line(td::BufferSlice data); - TestNode() { - } + TestNode() = default; void run(); }; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 6a2e18f1640f473c854f3eb2dabdf8f076537438..97c2bda2df4125bec8281603581b892696097361 100644 GIT binary patch delta 3518 zcma)8Z){Ul6z^SMXUo>D!;ZCUYl|zcV_P>6lqkc=fNe6QIzp68y0^Z|R_)%audf>= z7$F(~K^hgPp{{m z^E-d;x#xFo?lGJ^W+=KeSoeS~{nMl2qIvq#V>idYC}cT4;$MejF}54iW7GbJ%jAH5 zj43}^&;Q^F36TgZ3F4yyhyBf%$093XGAI8Y*<$>}P#Q=SO%-xo{+L9?{Rr_{YXPR{@ zzeS)qSD!t4^uj&=s~rP%a#3%WkJuuJIrjXXYs6aKIj^Gep}LYtn7YSe9~S*k){ToP zt}K<0H3e-j&JyTa6Am?pf_=a5fTZ9uNBg_{rBRF(B{XlR>-Wzw7!-8R$o{W3P`iuJNY+oLpUF0_piL z=eFs|RD+w;?=leAn&#ATkE%DYrao=3#%ogzpxMQy{2F7!OD8~fpxK)NL!@N5I%U$K z5$BPqk!mv1oOVBP+tHF1?^3`5Z1|nMYfi_4A>Lc2JR`Nr&&IKsSS^ZzsK$aQaOHs& z#dc{zP$w!u_M4q*ia~Ewsd!}aowAa|3evL{iw#P#u zdc`^&yjrLhO1cnov|V`mJiKb=s@w)| zA3On+3Jo&y-PW4ihqi<-_u&(X|FArisLySA`M+Sj+k73lqSY2+_g91aYcm7o8JzG( za}F-0hiSb5H#zYjTK+tIY00Wn+Mr+_|eKUttPz`W2fAC<6@5$rEL$n%G_) zH=5F*WXCEGqnJ#*YDIHPl*D5}NeK1|e3W(du@3AmXq}mS#F@#5 zrV0Am5}TCb2E!cLxyo(p9zHXgod{US1B3g}+L77%NNGKF znL6UWVJ96oJf@61IgF6IxQNGFrC?MPxOj-ZDDu#giy8q+d*3s2VRje_7wSNlSo@dQ z03*~H5!#H-VRT&WIuxV3@=jw~bNEr$Hmr32KA9R)`6 z!6UxnWkA^F*@P|PUMdP;Eb8DK;Gt@2PlJpM$)7O5;|KQol4EY-AJXg&8yD+TYkk_-8-`o~%_5GW=jawP{7|#3zxUnCayqco&{F~%9VkGP4D)2?w?8L6 zW>jivhk<=d8<9nPSxPftL1P9($JZ~GP;mRKz<)1AyYmd6_AOj*S4?uFCutm>(XBQV zVGM&WIIdPDStqjAWpC4|nV<{>$VGcCFbhU$k*%H%wdJ(O*)7Yq z#^xPTIgBhn4)k(N){sfqyhCOS)8rNM2BIZ*=7sPS<)joKS;Y)8hebhvNAf_z{;wh> z`FVc%DXDtSi3N#CnK_vynW@E-7jjB(_RtAooP0(1#O4!vA}ow0o9`Nxi?V>+KKa8W zjmZrjDiF>J4;@aB=*?q7KFpH^9p$G#P-JAC{#Ke%8KKC>Q-v9*X!3eTL#QrK6$zLw zQ0Npnv*~cywjBhU|HKn&HjkG9M7e{P3fvN=$%+TGC(rPLsyTomCE*Q~S^{I7@ty)U zXLE|r0Y-?48h$nqp$xwVLYp0u_DD&90v+TFWcPr?CX3YS;FgT2)xa$|1E0=2wFbC# zs?^~zHU*F5=9oqf-pzOVV%We*WU|dv2~JR20r`x3a_rRKnHK!wE`EAU#ZzC$6=h%(G4dB4x2|&gMDmW%wum*|&xTl)NSv zOwyP>!GKW(!uzvd2EvdzFbBfea{wxG=RgQpWb=fB2edgsi4S7_!?)g(N;z0S0-Iyr zJz(A}@wtO(v&^?MGK|&RUkWk$Gi|pKW0YXoen*P&0LwN7#s#d~HPje0n6`h>V9a6J ezD1XjLqP&&9&+q~#I{TLF-~CI9uvTr!3F^N)?rWp diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 075c5a3abdf705679b48dd79a799a1f70a9d67a9..1f4dab07974697338e4e299f46e17cda26580488 100644 GIT binary patch delta 101 zcmZ4SiSf-R#tjSDc<1Iw@Fk`cgwt*i2F8LvG diff --git a/tonlib/CMakeLists.txt b/tonlib/CMakeLists.txt index d9984368be..f27e00f21c 100644 --- a/tonlib/CMakeLists.txt +++ b/tonlib/CMakeLists.txt @@ -8,7 +8,6 @@ set(TONLIB_SOURCE tonlib/Client.cpp tonlib/Config.cpp tonlib/ExtClient.cpp - tonlib/ExtClientMulti.cpp tonlib/ExtClientLazy.cpp tonlib/ExtClientOutbound.cpp tonlib/KeyStorage.cpp @@ -24,8 +23,7 @@ set(TONLIB_SOURCE tonlib/Client.h tonlib/Config.h tonlib/ExtClient.h - tonlib/ExtClientMulti.h - tonlib/ExtClientRaw.h + tonlib/ExtClientLazy.h tonlib/ExtClientOutbound.h tonlib/KeyStorage.h tonlib/KeyValue.h diff --git a/tonlib/tonlib/Config.cpp b/tonlib/tonlib/Config.cpp index dfa9e3f754..73d4e033ce 100644 --- a/tonlib/tonlib/Config.cpp +++ b/tonlib/tonlib/Config.cpp @@ -66,16 +66,16 @@ td::Result Config::parse(std::string str) { return td::Status::Error("Invalid config (1)"); } td::JsonArray empty_array; - TRY_RESULT(lite_clients_obj, + TRY_RESULT(lite_servers_obj, td::get_json_object_field(json.get_object(), "liteservers", td::JsonValue::Type::Array, true)); - auto &lite_clients = - lite_clients_obj.type() == td::JsonValue::Type::Array ? lite_clients_obj.get_array() : empty_array; - TRY_RESULT(lite_clients_v2_obj, + auto &lite_servers = + lite_servers_obj.type() == td::JsonValue::Type::Array ? lite_servers_obj.get_array() : empty_array; + TRY_RESULT(lite_servers_v2_obj, td::get_json_object_field(json.get_object(), "liteservers_v2", td::JsonValue::Type::Array, true)); - auto &lite_clients_v2 = - lite_clients_v2_obj.type() == td::JsonValue::Type::Array ? lite_clients_v2_obj.get_array() : empty_array; + auto &lite_servers_v2 = + lite_servers_v2_obj.type() == td::JsonValue::Type::Array ? lite_servers_v2_obj.get_array() : empty_array; - auto parse_desc = [&](td::JsonValue& value) -> td::Result { + auto parse_desc = [&](td::JsonValue &value) -> td::Result { if (value.type() != td::JsonValue::Type::Object) { return td::Status::Error("Invalid config (2)"); } @@ -83,8 +83,8 @@ td::Result Config::parse(std::string str) { TRY_RESULT(ip, td::get_json_object_long_field(object, "ip", false)); TRY_RESULT(port, td::get_json_object_int_field(object, "port", false)); - Config::LiteClient client; - TRY_STATUS(client.address.init_host_port(td::IPAddress::ipv4_to_str(static_cast(ip)), port)); + Config::LiteServer server; + TRY_STATUS(server.address.init_host_port(td::IPAddress::ipv4_to_str(static_cast(ip)), port)); TRY_RESULT(id_obj, td::get_json_object_field(object, "id", td::JsonValue::Type::Object, false)); auto &id = id_obj.get_object(); @@ -98,20 +98,19 @@ td::Result Config::parse(std::string str) { return td::Status::Error("Invalid config (4)"); } - client.adnl_id = ton::adnl::AdnlNodeIdFull(ton::pubkeys::Ed25519(td::Bits256(td::Slice(key).ubegin()))); - return client; + server.adnl_id = ton::adnl::AdnlNodeIdFull(ton::pubkeys::Ed25519(td::Bits256(td::Slice(key).ubegin()))); + return server; }; Config res; - for (auto &value : lite_clients) { - TRY_RESULT(client, parse_desc(value)); - res.lite_clients.push_back(std::move(client)); + for (auto &value : lite_servers) { + TRY_RESULT(server, parse_desc(value)); + res.lite_servers.push_back(std::move(server)); } - for (auto &value : lite_clients_v2) { - TRY_RESULT(client, parse_desc(value)); - client.is_full = false; - TRY_RESULT(shards_obj, - td::get_json_object_field(value.get_object(), "shards", td::JsonValue::Type::Array, false)); + for (auto &value : lite_servers_v2) { + TRY_RESULT(server, parse_desc(value)); + server.is_full = false; + TRY_RESULT(shards_obj, td::get_json_object_field(value.get_object(), "shards", td::JsonValue::Type::Array, false)); for (auto &shard : shards_obj.get_array()) { if (shard.type() != td::JsonValue::Type::Object) { return td::Status::Error("Invalid config (5)"); @@ -122,10 +121,10 @@ td::Result Config::parse(std::string str) { if (shard_id == 0) { return td::Status::Error("Invalid config (6)"); } - client.shards.emplace_back(workchain, shard_id); + server.shards.emplace_back(workchain, shard_id); } - res.lite_clients.push_back(std::move(client)); + res.lite_servers.push_back(std::move(server)); } TRY_RESULT(validator_obj, diff --git a/tonlib/tonlib/Config.h b/tonlib/tonlib/Config.h index 420889dfdb..7af1b1c5aa 100644 --- a/tonlib/tonlib/Config.h +++ b/tonlib/tonlib/Config.h @@ -23,7 +23,7 @@ namespace tonlib { struct Config { - struct LiteClient { + struct LiteServer { ton::adnl::AdnlNodeIdFull adnl_id; td::IPAddress address; bool is_full = true; @@ -32,7 +32,7 @@ struct Config { ton::BlockIdExt zero_state_id; ton::BlockIdExt init_block_id; std::vector hardforks; - std::vector lite_clients; + std::vector lite_servers; std::string name; static td::Result parse(std::string str); }; diff --git a/tonlib/tonlib/ExtClient.cpp b/tonlib/tonlib/ExtClient.cpp index 30a29b59c8..755be170d0 100644 --- a/tonlib/tonlib/ExtClient.cpp +++ b/tonlib/tonlib/ExtClient.cpp @@ -54,7 +54,7 @@ void ExtClient::with_last_block(td::Promise promise) { td::actor::send_closure(client_.last_block_actor_, &LastBlock::get_last_block, std::move(P)); } -void ExtClient::send_raw_query(td::BufferSlice query, td::Promise promise) { +void ExtClient::send_raw_query(td::BufferSlice query, ton::ShardIdFull shard, td::Promise promise) { auto query_id = queries_.create(std::move(promise)); td::Promise P = [query_id, self = this, actor_id = td::actor::actor_id()](td::Result result) { @@ -65,7 +65,7 @@ void ExtClient::send_raw_query(td::BufferSlice query, td::Promise + namespace tonlib { -class ExtClientLazyImp : public ExtClientLazy { +class ExtClientLazyImpl : public ExtClientLazy { public: - ExtClientLazyImp(std::vector> servers, - td::unique_ptr callback) - : servers_(std::move(servers)), callback_(std::move(callback)) { - CHECK(!servers_.empty()); + ExtClientLazyImpl(std::vector servers, td::unique_ptr callback) + : callback_(std::move(callback)) { + CHECK(!servers.empty()); + servers_.resize(servers.size()); + for (size_t i = 0; i < servers_.size(); ++i) { + servers_[i].s = std::move(servers[i]); + if (!servers_[i].s.is_full) { + for (auto shard : servers_[i].s.shards) { + CHECK(shard.is_valid_ext()); + max_server_shard_depth_ = std::max(max_server_shard_depth_, shard.pfx_len()); + } + } + } } void start_up() override { @@ -34,29 +46,21 @@ class ExtClientLazyImp : public ExtClientLazy { td::random_shuffle(td::as_mutable_span(servers_), rnd); } - void check_ready(td::Promise promise) override { - before_query(); - if (client_.empty()) { - return promise.set_error(TonlibError::Cancelled()); - } - send_closure(client_, &ton::adnl::AdnlExtClient::check_ready, std::move(promise)); - } - - void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, + void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, td::Promise promise) override { - before_query(); - if (client_.empty()) { - return promise.set_error(TonlibError::Cancelled()); - } - td::Promise P = [SelfId = actor_id(this), idx = cur_server_idx_, + TRY_RESULT_PROMISE(promise, server_idx, before_query(shard)); + auto& server = servers_[server_idx]; + CHECK(!server.client.empty()); + alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); + td::Promise P = [SelfId = actor_id(this), server_idx, promise = std::move(promise)](td::Result R) mutable { if (R.is_error() && (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { - td::actor::send_closure(SelfId, &ExtClientLazyImp::set_server_bad, idx, true); + td::actor::send_closure(SelfId, &ExtClientLazyImpl::set_server_bad, server_idx); } promise.set_result(std::move(R)); }; - send_closure(client_, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, + send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, std::move(P)); } @@ -64,64 +68,124 @@ class ExtClientLazyImp : public ExtClientLazy { if (servers_.size() == 1) { return; } - cur_server_bad_ = cur_server_bad_force_ = true; + auto it = shard_to_server_.find(ton::ShardIdFull(ton::masterchainId)); + if (it != shard_to_server_.end()) { + set_server_bad(it->second); + } } private: - void before_query() { + td::Result before_query(ton::ShardIdFull shard) { + if (!shard.is_valid_ext()) { + return td::Status::Error("Invalid shard"); + } if (is_closing_) { - return; + return td::Status::Error("Client is closing"); } - alarm_timestamp() = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT); - if (cur_server_bad_) { - ++cur_server_idx_; - } else if (!client_.empty()) { - return; + if (shard.pfx_len() > max_server_shard_depth_) { + shard = shard_prefix(shard, max_server_shard_depth_); + } + auto it = shard_to_server_.find(shard); + if (it != shard_to_server_.end()) { + size_t server_idx = it->second; + if (!servers_[server_idx].client.empty()) { + return server_idx; + } + shard_to_server_.erase(it); + } + + size_t server_idx = servers_.size(); + int cnt = 0; + int best_priority = -1; + for (size_t i = 0; i < servers_.size(); ++i) { + Server& server = servers_[i]; + if (!server.supports(shard)) { + continue; + } + int priority = 0; + priority += (server.client.empty() ? 0 : 100); + priority += (server.ignore_until && !server.ignore_until.is_in_past() ? 0 : 10); + priority += (server.s.is_full ? 1 : 0); + if (priority < best_priority) { + continue; + } + if (priority > best_priority) { + best_priority = priority; + cnt = 0; + } + if (td::Random::fast(0, cnt) == 0) { + server_idx = i; + } + ++cnt; } + if (server_idx == servers_.size()) { + return td::Status::Error(PSTRING() << "No liteserver for shard " << shard.to_str()); + } + Server& server = servers_[server_idx]; + if (!server.client.empty()) { + return server_idx; + } + class Callback : public ton::adnl::AdnlExtClient::Callback { public: - explicit Callback(td::actor::ActorShared parent, size_t idx) + explicit Callback(td::actor::ActorShared parent, size_t idx) : parent_(std::move(parent)), idx_(idx) { } void on_ready() override { - td::actor::send_closure(parent_, &ExtClientLazyImp::set_server_bad, idx_, false); } void on_stop_ready() override { - td::actor::send_closure(parent_, &ExtClientLazyImp::set_server_bad, idx_, true); + td::actor::send_closure(parent_, &ExtClientLazyImpl::set_server_bad, idx_); } private: - td::actor::ActorShared parent_; + td::actor::ActorShared parent_; size_t idx_; }; ref_cnt_++; - cur_server_bad_ = false; - cur_server_bad_force_ = false; - const auto& s = servers_[cur_server_idx_ % servers_.size()]; - LOG(INFO) << "Connecting to liteserver " << s.second; - client_ = ton::adnl::AdnlExtClient::create( - s.first, s.second, std::make_unique(td::actor::actor_shared(this), cur_server_idx_)); + if (shard.is_masterchain()) { + LOG(INFO) << "Connecting to liteserver " << server.s.address << " for masterchain"; + } else { + LOG(INFO) << "Connecting to liteserver " << server.s.address << " for shard " << shard.to_str(); + } + server.client = ton::adnl::AdnlExtClient::create( + server.s.adnl_id, server.s.address, std::make_unique(td::actor::actor_shared(this), server_idx)); + alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); + return server_idx; } - std::vector> servers_; - size_t cur_server_idx_ = 0; - bool cur_server_bad_ = false; - bool cur_server_bad_force_ = false; + struct Server { + Config::LiteServer s; + td::actor::ActorOwn client; + td::Timestamp timeout = td::Timestamp::never(); + td::Timestamp ignore_until = td::Timestamp::never(); + + bool supports(const ton::ShardIdFull& shard) const { + return s.is_full || shard.is_masterchain() || + std::any_of(s.shards.begin(), s.shards.end(), + [&](const ton::ShardIdFull s_shard) { return ton::shard_intersects(shard, s_shard); }); + } + }; + std::vector servers_; + std::map shard_to_server_; + int max_server_shard_depth_ = 0; - td::actor::ActorOwn client_; td::unique_ptr callback_; static constexpr double MAX_NO_QUERIES_TIMEOUT = 100; bool is_closing_{false}; td::uint32 ref_cnt_{1}; - void set_server_bad(size_t idx, bool bad) { - if (idx == cur_server_idx_ && servers_.size() > 1 && !cur_server_bad_force_) { - cur_server_bad_ = bad; + void alarm() override { + for (Server& server : servers_) { + if (server.timeout && server.timeout.is_in_past()) { + server.client.reset(); + } } } - void alarm() override { - client_.reset(); + void set_server_bad(size_t idx) { + servers_[idx].client.reset(); + servers_[idx].timeout = td::Timestamp::never(); + servers_[idx].ignore_until = td::Timestamp::in(60.0); } void hangup_shared() override { ref_cnt_--; @@ -130,7 +194,9 @@ class ExtClientLazyImp : public ExtClientLazy { void hangup() override { is_closing_ = true; ref_cnt_--; - client_.reset(); + for (Server& server : servers_) { + server.client.reset(); + } try_stop(); } void try_stop() { @@ -142,11 +208,11 @@ class ExtClientLazyImp : public ExtClientLazy { td::actor::ActorOwn ExtClientLazy::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, td::unique_ptr callback) { - return create({std::make_pair(dst, dst_addr)}, std::move(callback)); + return create({Config::LiteServer{dst, dst_addr, true, {}}}, std::move(callback)); } -td::actor::ActorOwn ExtClientLazy::create( - std::vector> servers, td::unique_ptr callback) { - return td::actor::create_actor("ExtClientLazy", std::move(servers), std::move(callback)); +td::actor::ActorOwn ExtClientLazy::create(std::vector servers, + td::unique_ptr callback) { + return td::actor::create_actor("ExtClientLazy", std::move(servers), std::move(callback)); } } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientRaw.h b/tonlib/tonlib/ExtClientLazy.h similarity index 72% rename from tonlib/tonlib/ExtClientRaw.h rename to tonlib/tonlib/ExtClientLazy.h index 612995af06..6383cec55b 100644 --- a/tonlib/tonlib/ExtClientRaw.h +++ b/tonlib/tonlib/ExtClientLazy.h @@ -18,23 +18,27 @@ */ #pragma once #include "td/actor/actor.h" -#include "adnl/adnl-ext-client.h" #include "ton/ton-types.h" +#include "adnl/adnl-ext-client.h" +#include "Config.h" namespace tonlib { -class ExtClientLazy : public ton::adnl::AdnlExtClient { +class ExtClientLazy : public td::actor::Actor { public: class Callback { public: - virtual ~Callback() = default; + virtual ~Callback() { + } }; + virtual void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, + td::Promise promise) = 0; virtual void force_change_liteserver() = 0; static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, td::unique_ptr callback); - static td::actor::ActorOwn create( - std::vector> servers, td::unique_ptr callback); + static td::actor::ActorOwn create(std::vector servers, + td::unique_ptr callback); }; } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientMulti.cpp b/tonlib/tonlib/ExtClientMulti.cpp deleted file mode 100644 index 3a6cdf595d..0000000000 --- a/tonlib/tonlib/ExtClientMulti.cpp +++ /dev/null @@ -1,169 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . -*/ -#include "ExtClientMulti.h" -#include "ton/ton-shard.h" -#include "td/utils/Random.h" - -namespace tonlib { - -static const double MAX_NO_QUERIES_TIMEOUT = 120; - -ExtClientMulti::ExtClientMulti(std::vector clients, td::unique_ptr callback) - : callback_(std::move(callback)) { - for (auto &desc : clients) { - servers_.emplace_back(); - servers_.back().desc = std::move(desc); - } -} - -void ExtClientMulti::start_up() { - alarm_timestamp() = td::Timestamp::in(60.0); -} - -void ExtClientMulti::send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, - td::Promise promise) { - if (shard.is_masterchain() && mc_server_idx_ != -1 && !servers_[mc_server_idx_].is_bad()) { - send_query_to_server(std::move(name), std::move(data), mc_server_idx_, timeout, std::move(promise)); - return; - } - auto it = shard_server_idx_cached_.find(shard); - if (it != shard_server_idx_cached_.end() && !servers_[it->second].is_bad()) { - send_query_to_server(std::move(name), std::move(data), it->second, timeout, std::move(promise)); - return; - } - int server_idx = -1; - int random_idx = -1; - int cnt = 0; - int best_prefix = -1; - for (int i = 0; i < (int)servers_.size(); ++i) { - const Server &server = servers_[i]; - if (server.is_bad()) { - continue; - } - int len = server.desc.is_full ? 65 : server.max_supported_prefix(shard); - if (len > best_prefix) { - best_prefix = len; - server_idx = -1; - random_idx = -1; - cnt = 0; - } else if (len < best_prefix) { - continue; - } - if (!server.client.empty()) { - server_idx = i; - } - if (td::Random::fast(0, cnt) == 0) { - random_idx = i; - } - ++cnt; - } - if (server_idx == -1) { - server_idx = random_idx; - } - if (server_idx == -1) { - promise.set_error(td::Status::Error("failed to select a suitable server")); - return; - } - if (shard.pfx_len() <= ton::max_shard_pfx_len) { - shard_server_idx_cached_[shard] = server_idx; - } - if (shard.is_masterchain() || servers_[server_idx].desc.is_full) { - mc_server_idx_ = server_idx; - } - send_query_to_server(std::move(name), std::move(data), server_idx, timeout, std::move(promise)); -} - -void ExtClientMulti::send_query_to_server(std::string name, td::BufferSlice data, int server_idx, td::Timestamp timeout, - td::Promise promise) { - Server &server = servers_.at(server_idx); - if (server.client.empty()) { - start_client(server_idx); - } - server.ttl = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT); - td::Promise P = [SelfId = actor_id(this), idx = server_idx, - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error() && - (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { - td::actor::send_closure(SelfId, &ExtClientMulti::set_server_bad, idx); - } - promise.set_result(std::move(R)); - }; - send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, - std::move(P)); -} - -void ExtClientMulti::force_change_liteserver() { - if (mc_server_idx_ != -1) { - set_server_bad(mc_server_idx_); - mc_server_idx_ = -1; - } -} - -void ExtClientMulti::start_client(int server_idx) { - class Callback : public ton::adnl::AdnlExtClient::Callback { - public: - Callback(td::actor::ActorId parent, int idx) : parent_(std::move(parent)), idx_(idx) { - } - void on_ready() override { - } - void on_stop_ready() override { - td::actor::send_closure(parent_, &ExtClientMulti::set_server_bad, idx_); - } - - private: - td::actor::ActorId parent_; - int idx_; - }; - Server &server = servers_.at(server_idx); - server.client = ton::adnl::AdnlExtClient::create(server.desc.adnl_id, server.desc.address, - std::make_unique(actor_id(this), server_idx)); -} - -void ExtClientMulti::alarm() { - for (Server& server : servers_) { - if (server.ttl && server.ttl.is_in_past()) { - server.client.reset(); - } - } - alarm_timestamp() = td::Timestamp::in(60.0); -} - -void ExtClientMulti::set_server_bad(int idx) { - Server& server = servers_.at(idx); - server.client.reset(); - server.ttl = td::Timestamp::never(); - server.ignore_until = td::Timestamp::in(10.0); -} - -int ExtClientMulti::Server::max_supported_prefix(ton::ShardIdFull shard) const { - if (desc.is_full || shard.is_masterchain()) { - return shard.pfx_len(); - } - int res = -1; - for (const ton::ShardIdFull &our_shard : desc.shards) { - if (ton::shard_is_ancestor(our_shard, shard)) { - return shard.pfx_len(); - } - if (shard.workchain == our_shard.workchain) { - int x = std::min({shard.pfx_len(), our_shard.pfx_len(), ton::count_matching_bits(shard.shard, our_shard.shard)}); - res = std::max(res, x); - } - } - return res; -} - -} // namespace tonlib diff --git a/tonlib/tonlib/ExtClientMulti.h b/tonlib/tonlib/ExtClientMulti.h deleted file mode 100644 index 4c93b00ee5..0000000000 --- a/tonlib/tonlib/ExtClientMulti.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . -*/ -#pragma once -#include "td/actor/actor.h" -#include "adnl/adnl-ext-client.h" -#include "Config.h" -#include "ExtClientRaw.h" -#include - -namespace tonlib { - -class ExtClientMulti : public ExtClientRaw { - public: - ExtClientMulti(std::vector clients, td::unique_ptr callback); - - void start_up() override; - void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, - td::Promise promise) override; - void alarm() override; - void force_change_liteserver() override; - - private: - void send_query_to_server(std::string name, td::BufferSlice data, int server_idx, td::Timestamp timeout, - td::Promise promise); - void start_client(int server_idx); - - struct Server { - Config::LiteClient desc; - td::actor::ActorOwn client; - td::Timestamp ttl; - td::Timestamp ignore_until = td::Timestamp::never(); - - int max_supported_prefix(ton::ShardIdFull shard) const; - bool is_bad() const { - return ignore_until && !ignore_until.is_in_past(); - } - }; - - void set_server_bad(int idx); - - td::unique_ptr callback_; - std::vector servers_; - int mc_server_idx_ = -1; - std::map shard_server_idx_cached_; -}; - -} // namespace tonlib diff --git a/tonlib/tonlib/ExtClientOutbound.cpp b/tonlib/tonlib/ExtClientOutbound.cpp index d2715b8b5f..eb923734e5 100644 --- a/tonlib/tonlib/ExtClientOutbound.cpp +++ b/tonlib/tonlib/ExtClientOutbound.cpp @@ -20,11 +20,12 @@ #include "ExtClientOutbound.h" #include "TonlibError.h" #include + namespace tonlib { -class ExtClientOutboundImp : public ExtClientOutbound { +class ExtClientOutboundImpl : public ExtClientOutbound { public: - ExtClientOutboundImp(td::unique_ptr callback) : callback_(std::move(callback)) { + ExtClientOutboundImpl(td::unique_ptr callback) : callback_(std::move(callback)) { } void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, @@ -37,9 +38,6 @@ class ExtClientOutboundImp : public ExtClientOutbound { void force_change_liteserver() override { } - void force_change_liteserver() override { - } - void on_query_result(td::int64 id, td::Result r_data, td::Promise promise) override { auto it = queries_.find(id); if (it == queries_.end()) { @@ -65,6 +63,6 @@ class ExtClientOutboundImp : public ExtClientOutbound { }; td::actor::ActorOwn ExtClientOutbound::create(td::unique_ptr callback) { - return td::actor::create_actor("ExtClientOutbound", std::move(callback)); + return td::actor::create_actor("ExtClientOutbound", std::move(callback)); } } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientOutbound.h b/tonlib/tonlib/ExtClientOutbound.h index 4721e8729d..31f76b9773 100644 --- a/tonlib/tonlib/ExtClientOutbound.h +++ b/tonlib/tonlib/ExtClientOutbound.h @@ -30,6 +30,7 @@ class ExtClientOutbound : public ExtClientLazy { } virtual void request(td::int64 id, std::string data, ton::ShardIdFull shard) = 0; }; + virtual void on_query_result(td::int64 id, td::Result r_data, td::Promise promise) = 0; static td::actor::ActorOwn create(td::unique_ptr callback); }; diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 0680cebc67..05406d5098 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -27,7 +27,6 @@ #include "tonlib/keys/Mnemonic.h" #include "tonlib/keys/SimpleEncryption.h" #include "tonlib/TonlibError.h" -#include "tonlib/ExtClientMulti.h" #include "smc-envelope/GenericAccount.h" #include "smc-envelope/ManualDns.h" @@ -863,7 +862,9 @@ class Query { } return res; } - td::Result>> estimate_fees(bool ignore_chksig, std::shared_ptr& cfg, vm::Dictionary& libraries) { + td::Result>> estimate_fees(bool ignore_chksig, + std::shared_ptr& cfg, + vm::Dictionary& libraries) { // gas fees bool is_masterchain = raw_.source->get_address().workchain == ton::masterchainId; TRY_RESULT(gas_limits_prices, cfg->get_gas_limits_prices(is_masterchain)); @@ -892,7 +893,8 @@ class Query { .set_now(raw_.source->get_sync_time()) .set_ignore_chksig(ignore_chksig) .set_address(raw_.source->get_address()) - .set_config(cfg).set_libraries(libraries)); + .set_config(cfg) + .set_libraries(libraries)); td::int64 fwd_fee = 0; if (res.success) { LOG(DEBUG) << "output actions:\n" @@ -962,8 +964,9 @@ td::Result to_balance(td::Ref balance_ref) { class GetTransactionHistory : public td::actor::Actor { public: - GetTransactionHistory(ExtClientRef ext_client_ref, block::StdAddress address, ton::LogicalTime lt, ton::Bits256 hash, td::int32 count, - td::actor::ActorShared<> parent, td::Promise promise) + GetTransactionHistory(ExtClientRef ext_client_ref, block::StdAddress address, ton::LogicalTime lt, ton::Bits256 hash, + td::int32 count, td::actor::ActorShared<> parent, + td::Promise promise) : address_(std::move(address)) , lt_(std::move(lt)) , hash_(std::move(hash)) @@ -1671,8 +1674,8 @@ void TonlibClient::init_ext_client() { } void request(td::int64 id, std::string data, ton::ShardIdFull shard) override { - send_closure(parent_, &TonlibClient::proxy_request, (id << 16) | (config_generation_ & 0xffff), - std::move(data), shard); + send_closure(parent_, &TonlibClient::proxy_request, (id << 16) | (config_generation_ & 0xffff), std::move(data), + shard); } private: @@ -1685,34 +1688,17 @@ void TonlibClient::init_ext_client() { ext_client_outbound_ = client.get(); raw_client_ = std::move(client); } else { - std::vector> servers; - for (const auto& s : config_.lite_clients) { - servers.emplace_back(s.adnl_id, s.address); - } class Callback : public ExtClientLazy::Callback { public: explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { } + private: td::actor::ActorShared<> parent_; }; - std::vector> full_servers; - for (const auto& s : config_.lite_clients) { - if (s.is_full) { - full_servers.emplace_back(s.adnl_id, s.address); - } - } - if (!full_servers.empty()) { - raw_client_ = - ExtClientRaw::create(std::move(full_servers), td::make_unique(td::actor::actor_shared())); - } else { - CHECK(!config_.lite_clients.empty()); - raw_client_ = td::actor::create_actor("ExtClientMulti", config_.lite_clients, - td::make_unique(td::actor::actor_shared())); - } ext_client_outbound_ = {}; ref_cnt_++; - raw_client_ = ExtClientLazy::create(std::move(servers), td::make_unique(td::actor::actor_shared())); + raw_client_ = ExtClientLazy::create(config_.lite_servers, td::make_unique(td::actor::actor_shared())); } } @@ -2360,7 +2346,7 @@ td::Result TonlibClient::validate_config(tonlib_api::o TRY_RESULT_PREFIX(new_config, Config::parse(std::move(config->config_)), TonlibError::InvalidConfig("can't parse config")); - if (new_config.lite_clients.empty() && !config->use_callbacks_for_network_) { + if (new_config.lite_servers.empty() && !config->use_callbacks_for_network_) { return TonlibError::InvalidConfig("no lite clients"); } td::optional o_master_config; @@ -2427,8 +2413,7 @@ td::Result TonlibClient::validate_config(tonlib_api::o state.vert_seqno = vert_seqno; bool user_defined_init_block = false; - if (new_config.init_block_id.is_valid() && - state.last_key_block_id.id.seqno < new_config.init_block_id.id.seqno) { + if (new_config.init_block_id.is_valid() && state.last_key_block_id.id.seqno < new_config.init_block_id.id.seqno) { state.last_key_block_id = new_config.init_block_id; user_defined_init_block = true; LOG(INFO) << "Use init block from USER config: " << new_config.init_block_id.to_str(); @@ -2524,8 +2509,7 @@ td::Result to_std_address(td::Ref cs) { } struct ToRawTransactions { explicit ToRawTransactions(td::optional private_key, bool try_decode_messages = true) - : private_key_(std::move(private_key)) - , try_decode_messages_(try_decode_messages) { + : private_key_(std::move(private_key)), try_decode_messages_(try_decode_messages) { } td::optional private_key_; @@ -2588,7 +2572,8 @@ struct ToRawTransactions { } } if (!data) { - data = tonlib_api::make_object(to_bytes(std::move(body_cell)), to_bytes(std::move(init_state_cell))); + data = tonlib_api::make_object(to_bytes(std::move(body_cell)), + to_bytes(std::move(init_state_cell))); } return data; }; @@ -2636,7 +2621,8 @@ struct ToRawTransactions { auto created_lt = static_cast(msg_info.created_lt); return tonlib_api::make_object( tonlib_api::make_object(src), - tonlib_api::make_object(), 0, 0, 0, created_lt, std::move(body_hash), get_data(src)); + tonlib_api::make_object(), 0, 0, 0, created_lt, std::move(body_hash), + get_data(src)); } } @@ -2746,10 +2732,9 @@ td::Status TonlibClient::do_request(const tonlib_api::raw_sendMessageReturnHash& td::Promise>&& promise) { TRY_RESULT_PREFIX(body, vm::std_boc_deserialize(request.body_), TonlibError::InvalidBagOfCells("body")); auto hash = body->get_hash().as_slice().str(); - make_request(int_api::SendMessage{std::move(body)}, - promise.wrap([hash = std::move(hash)](auto res) { - return tonlib_api::make_object(std::move(hash)); - })); + make_request(int_api::SendMessage{std::move(body)}, promise.wrap([hash = std::move(hash)](auto res) { + return tonlib_api::make_object(std::move(hash)); + })); return td::Status::OK(); } @@ -2839,7 +2824,7 @@ td::Status TonlibClient::do_request(tonlib_api::raw_getTransactions& request, } td::Status TonlibClient::do_request(tonlib_api::raw_getTransactionsV2& request, - td::Promise>&& promise) { + td::Promise>&& promise) { if (!request.account_address_) { return TonlibError::EmptyField("account_address"); } @@ -2875,9 +2860,10 @@ td::Status TonlibClient::do_request(tonlib_api::raw_getTransactionsV2& request, auto actor_id = actor_id_++; actors_[actor_id] = td::actor::create_actor( "GetTransactionHistory", client_.get_client(), account_address, lt, hash, count, actor_shared(this, actor_id), - promise.wrap([private_key = std::move(private_key), try_decode_messages = request.try_decode_messages_](auto&& x) mutable { - return ToRawTransactions(std::move(private_key), try_decode_messages).to_raw_transactions(std::move(x)); - })); + promise.wrap( + [private_key = std::move(private_key), try_decode_messages = request.try_decode_messages_](auto&& x) mutable { + return ToRawTransactions(std::move(private_key), try_decode_messages).to_raw_transactions(std::move(x)); + })); return td::Status::OK(); } @@ -3044,26 +3030,25 @@ class GenericCreateSendGrams : public TonlibQueryActor { td::Result to_dns_action(tonlib_api::dns_Action& action) { using R = td::Result; - return downcast_call2(action, - td::overloaded( - [&](tonlib_api::dns_actionDeleteAll& del_all) -> R { - return ton::ManualDns::Action{"", td::Bits256::zero(), {}}; - }, - [&](tonlib_api::dns_actionDelete& del) -> R { - return ton::ManualDns::Action{del.name_, del.category_, {}}; - }, - [&](tonlib_api::dns_actionSet& set) -> R { - if (!set.entry_) { - return TonlibError::EmptyField("entry"); - } - if (!set.entry_->entry_) { - return TonlibError::EmptyField("entry.entry"); - } - TRY_RESULT(entry_data, to_dns_entry_data(*set.entry_->entry_)); - TRY_RESULT(data_cell, entry_data.as_cell()); - return ton::ManualDns::Action{set.entry_->name_, set.entry_->category_, - std::move(data_cell)}; - })); + return downcast_call2( + action, td::overloaded( + [&](tonlib_api::dns_actionDeleteAll& del_all) -> R { + return ton::ManualDns::Action{"", td::Bits256::zero(), {}}; + }, + [&](tonlib_api::dns_actionDelete& del) -> R { + return ton::ManualDns::Action{del.name_, del.category_, {}}; + }, + [&](tonlib_api::dns_actionSet& set) -> R { + if (!set.entry_) { + return TonlibError::EmptyField("entry"); + } + if (!set.entry_->entry_) { + return TonlibError::EmptyField("entry.entry"); + } + TRY_RESULT(entry_data, to_dns_entry_data(*set.entry_->entry_)); + TRY_RESULT(data_cell, entry_data.as_cell()); + return ton::ManualDns::Action{set.entry_->name_, set.entry_->category_, std::move(data_cell)}; + })); } td::Status parse_action(tonlib_api::Action& action) { @@ -3471,9 +3456,9 @@ class GenericCreateSendGrams : public TonlibQueryActor { } } -// if (!o_public_key) { // todo: (tolya-yanot) temporary disable msg comment encryption (The exchanges/payment services needs to read the comment of incoming messages). This will be uncommented when a general standard is developed. - return TonlibError::MessageEncryption("Get public key (in destination)"); -// } + // if (!o_public_key) { // todo: (tolya-yanot) temporary disable msg comment encryption (The exchanges/payment services needs to read the comment of incoming messages). This will be uncommented when a general standard is developed. + return TonlibError::MessageEncryption("Get public key (in destination)"); + // } auto addr = source_->get_address(); addr.bounceable = true; @@ -3868,8 +3853,8 @@ td::Result from_tonlib_api(tonlib_api::tvm_StackEntry& entry) { })); } -void deep_library_search(std::set& set, std::set& visited, - vm::Dictionary& libs, td::Ref cell, int depth) { +void deep_library_search(std::set& set, std::set& visited, vm::Dictionary& libs, + td::Ref cell, int depth) { if (depth <= 0 || set.size() >= 16 || visited.size() >= 256) { return; } @@ -3895,7 +3880,7 @@ void deep_library_search(std::set& set, std::set& v } return; } - for (unsigned int i=0; iget_refs_cnt(); i++) { + for (unsigned int i = 0; i < loaded_cell.data_cell->get_refs_cnt(); i++) { deep_library_search(set, visited, libs, loaded_cell.data_cell->get_ref(i), depth - 1); } } @@ -3919,36 +3904,38 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_getLibraries& request, return td::Status::OK(); } - client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(not_cached_hashes)), - promise.wrap([self=this, result_entries = std::move(result_entries)] - (td::Result> r_libraries) mutable - { - if (r_libraries.is_error()) { - LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); - } else { - auto libraries = r_libraries.move_as_ok(); - bool updated = false; - for (auto& lr : libraries->result_) { - auto contents = vm::std_boc_deserialize(lr->data_); - if (contents.is_ok() && contents.ok().not_null()) { - if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { - LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); - continue; + client_.send_query( + ton::lite_api::liteServer_getLibraries(std::move(not_cached_hashes)), + promise.wrap( + [self = this, result_entries = std::move(result_entries)]( + td::Result> r_libraries) mutable { + if (r_libraries.is_error()) { + LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); + } else { + auto libraries = r_libraries.move_as_ok(); + bool updated = false; + for (auto& lr : libraries->result_) { + auto contents = vm::std_boc_deserialize(lr->data_); + if (contents.is_ok() && contents.ok().not_null()) { + if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { + LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); + continue; + } + result_entries.push_back( + tonlib_api::make_object(lr->hash_, lr->data_.as_slice().str())); + self->libraries.set_ref(lr->hash_, contents.move_as_ok()); + updated = true; + LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); + } else { + LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); + } + if (updated) { + self->store_libs_to_disk(); + } + } } - result_entries.push_back(tonlib_api::make_object(lr->hash_, lr->data_.as_slice().str())); - self->libraries.set_ref(lr->hash_, contents.move_as_ok()); - updated = true; - LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); - } else { - LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); - } - if (updated) { - self->store_libs_to_disk(); - } - } - } - return tonlib_api::make_object(std::move(result_entries)); - })); + return tonlib_api::make_object(std::move(result_entries)); + })); return td::Status::OK(); } @@ -3975,8 +3962,8 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, args.set_now(it->second->get_sync_time()); args.set_address(it->second->get_address()); - client_.with_last_config([self = this, smc = std::move(smc), args = std::move(args), promise = std::move(promise) - ](td::Result r_state) mutable { + client_.with_last_config([self = this, smc = std::move(smc), args = std::move(args), + promise = std::move(promise)](td::Result r_state) mutable { TRY_RESULT_PROMISE(promise, state, std::move(r_state)); args.set_config(state.config); @@ -3988,41 +3975,39 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, std::vector libraryList{librarySet.begin(), librarySet.end()}; if (libraryList.size() > 0) { LOG(DEBUG) << "Requesting found libraries in code (" << libraryList.size() << ")"; - self->client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(libraryList)), - [self, smc = std::move(smc), args = std::move(args), promise = std::move(promise)] - (td::Result> r_libraries) mutable - { - if (r_libraries.is_error()) { - LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); - } else { - auto libraries = r_libraries.move_as_ok(); - bool updated = false; - for (auto& lr : libraries->result_) { - auto contents = vm::std_boc_deserialize(lr->data_); - if (contents.is_ok() && contents.ok().not_null()) { - if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { - LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); - continue; - } - self->libraries.set_ref(lr->hash_, contents.move_as_ok()); - updated = true; - LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); + self->client_.send_query( + ton::lite_api::liteServer_getLibraries(std::move(libraryList)), + [self, smc = std::move(smc), args = std::move(args), promise = std::move(promise)]( + td::Result> r_libraries) mutable { + if (r_libraries.is_error()) { + LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); } else { - LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); + auto libraries = r_libraries.move_as_ok(); + bool updated = false; + for (auto& lr : libraries->result_) { + auto contents = vm::std_boc_deserialize(lr->data_); + if (contents.is_ok() && contents.ok().not_null()) { + if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { + LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); + continue; + } + self->libraries.set_ref(lr->hash_, contents.move_as_ok()); + updated = true; + LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); + } else { + LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); + } + } + if (updated) { + self->store_libs_to_disk(); + } } - } - if (updated) { - self->store_libs_to_disk(); - } - } - self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); - }); - } - else { + self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); + }); + } else { self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); } - } - else { + } else { self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); } }); @@ -4031,7 +4016,6 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, void TonlibClient::perform_smc_execution(td::Ref smc, ton::SmartContract::Args args, td::Promise>&& promise) { - args.set_libraries(libraries); auto res = smc->run_get_method(args); @@ -4046,45 +4030,46 @@ void TonlibClient::perform_smc_execution(td::Ref smc, ton::S td::Bits256 hash = res.missing_library; LOG(DEBUG) << "Requesting missing library: " << hash.to_hex(); std::vector req = {std::move(hash)}; - client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(req)), - [self = this, res = std::move(res), res_stack = std::move(res_stack), hash = std::move(hash), - smc = std::move(smc), args = std::move(args), promise = std::move(promise)] - (td::Result> r_libraries) mutable - { - if (r_libraries.is_error()) { - LOG(WARNING) << "cannot obtain missing library: " << r_libraries.move_as_error().to_string(); - promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); - return; - } - bool found = false, updated = false; - auto libraries = r_libraries.move_as_ok(); - for (auto& lr : libraries->result_) { - auto contents = vm::std_boc_deserialize(lr->data_); - if (contents.is_ok() && contents.ok().not_null()) { - if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { - LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); - continue; + client_.send_query( + ton::lite_api::liteServer_getLibraries(std::move(req)), + [self = this, res = std::move(res), res_stack = std::move(res_stack), hash = std::move(hash), + smc = std::move(smc), args = std::move(args), promise = std::move(promise)]( + td::Result> r_libraries) mutable { + if (r_libraries.is_error()) { + LOG(WARNING) << "cannot obtain missing library: " << r_libraries.move_as_error().to_string(); + promise.set_value( + tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); + return; } - found |= (lr->hash_ == hash); - updated = true; - self->libraries.set_ref(lr->hash_, contents.move_as_ok()); - LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); - } else { - LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); - } - } - if (updated) { - self->store_libs_to_disk(); - } - if (!found) { - LOG(WARNING) << "cannot obtain library " << hash.to_hex() << ", it may not exist"; - promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); - } else { - self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); - } - }); - } - else { + bool found = false, updated = false; + auto libraries = r_libraries.move_as_ok(); + for (auto& lr : libraries->result_) { + auto contents = vm::std_boc_deserialize(lr->data_); + if (contents.is_ok() && contents.ok().not_null()) { + if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { + LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); + continue; + } + found |= (lr->hash_ == hash); + updated = true; + self->libraries.set_ref(lr->hash_, contents.move_as_ok()); + LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); + } else { + LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); + } + } + if (updated) { + self->store_libs_to_disk(); + } + if (!found) { + LOG(WARNING) << "cannot obtain library " << hash.to_hex() << ", it may not exist"; + promise.set_value( + tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); + } else { + self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); + } + }); + } else { promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); } } @@ -4160,9 +4145,8 @@ void TonlibClient::do_dns_request(std::string name, td::Bits256 category, td::in td::optional block_id, block::StdAddress address, td::Promise>&& promise) { auto block_id_copy = block_id.copy(); - td::Promise new_promise = - promise.send_closure(actor_id(this), &TonlibClient::finish_dns_resolve, name, category, ttl, std::move(block_id), - address); + td::Promise new_promise = promise.send_closure(actor_id(this), &TonlibClient::finish_dns_resolve, name, + category, ttl, std::move(block_id), address); if (0) { make_request(int_api::GetAccountState{address, std::move(block_id_copy), {}}, @@ -4200,8 +4184,7 @@ td::Status TonlibClient::do_request(const tonlib_api::dns_resolve& request, name += '.'; } TRY_RESULT(account_address, get_account_address(request.account_address_->account_address_)); - do_dns_request(name, request.category_, request.ttl_, std::move(block_id), account_address, - std::move(promise)); + do_dns_request(name, request.category_, request.ttl_, std::move(block_id), account_address, std::move(promise)); return td::Status::OK(); } @@ -4639,15 +4622,15 @@ auto to_tonlib_api(const ton::lite_api::tonNode_blockIdExt& blk) -> tonlib_api_p auto to_tonlib_api(const ton::lite_api::tonNode_zeroStateIdExt& zeroStateId) -> tonlib_api_ptr { - return tonlib_api::make_object( //TODO check wether shard indeed 0??? + return tonlib_api::make_object( //TODO check wether shard indeed 0??? zeroStateId.workchain_, 0, 0, zeroStateId.root_hash_.as_slice().str(), zeroStateId.file_hash_.as_slice().str()); } auto to_lite_api(const tonlib_api::ton_blockIdExt& blk) -> td::Result> { TRY_RESULT(root_hash, to_bits256(blk.root_hash_, "blk.root_hash")) TRY_RESULT(file_hash, to_bits256(blk.file_hash_, "blk.file_hash")) - return ton::lite_api::make_object( - blk.workchain_, blk.shard_, blk.seqno_, root_hash, file_hash); + return ton::lite_api::make_object(blk.workchain_, blk.shard_, blk.seqno_, + root_hash, file_hash); } td::Result to_block_id(const tonlib_api::ton_blockIdExt& blk) { @@ -4657,33 +4640,34 @@ td::Result to_block_id(const tonlib_api::ton_blockIdExt& blk) { } td::Status TonlibClient::do_request(const tonlib_api::getConfigParam& request, - td::Promise>&& promise) { + td::Promise>&& promise) { TRY_RESULT(lite_block, to_lite_api(*request.id_)) auto block = create_block_id(std::move(lite_block)); auto param = request.param_; - std::vector params = { param }; + std::vector params = {param}; client_.send_query(ton::lite_api::liteServer_getConfigParams(0, std::move(lite_block), std::move(params)), promise.wrap([block, param](auto r_config) { - auto state = block::check_extract_state_proof(block, r_config->state_proof_.as_slice(), - r_config->config_proof_.as_slice()); - if (state.is_error()) { - LOG(ERROR) << "block::check_extract_state_proof failed: " << state.error(); - } - auto config = block::Config::extract_from_state(std::move(state.move_as_ok()), 0); - if (config.is_error()) { - LOG(ERROR) << "block::Config::extract_from_state failed: " << config.error(); - } - tonlib_api::configInfo config_result; - config_result.config_ = tonlib_api::make_object(to_bytes(config.move_as_ok()->get_config_param(param))); - return tonlib_api::make_object(std::move(config_result)); - })); + auto state = block::check_extract_state_proof(block, r_config->state_proof_.as_slice(), + r_config->config_proof_.as_slice()); + if (state.is_error()) { + LOG(ERROR) << "block::check_extract_state_proof failed: " << state.error(); + } + auto config = block::Config::extract_from_state(std::move(state.move_as_ok()), 0); + if (config.is_error()) { + LOG(ERROR) << "block::Config::extract_from_state failed: " << config.error(); + } + tonlib_api::configInfo config_result; + config_result.config_ = tonlib_api::make_object( + to_bytes(config.move_as_ok()->get_config_param(param))); + return tonlib_api::make_object(std::move(config_result)); + })); return td::Status::OK(); } td::Status TonlibClient::do_request(const tonlib_api::blocks_getMasterchainInfo& masterchain_info, - td::Promise>&& promise) { + td::Promise>&& promise) { client_.send_query(ton::lite_api::liteServer_getMasterchainInfo(), promise.wrap([](lite_api_ptr&& masterchain_info) { return tonlib_api::make_object( @@ -4728,133 +4712,126 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getShards& request, return td::Status::OK(); } - td::Status TonlibClient::do_request(const tonlib_api::blocks_lookupBlock& request, - td::Promise>&& promise) { + td::Promise>&& promise) { client_.send_query(ton::lite_api::liteServer_lookupBlock( - request.mode_, - ton::lite_api::make_object((*request.id_).workchain_, (*request.id_).shard_, (*request.id_).seqno_), - (td::uint64)(request.lt_), - (td::uint32)(request.utime_)), + request.mode_, + ton::lite_api::make_object( + (*request.id_).workchain_, (*request.id_).shard_, (*request.id_).seqno_), + (td::uint64)(request.lt_), (td::uint32)(request.utime_)), promise.wrap([](lite_api_ptr&& header) { - const auto& id = header->id_; - return to_tonlib_api(*id); - //tonlib_api::make_object( - // ton::tonlib_api::ton_blockIdExt(id->workchain_, id->) - //); + const auto& id = header->id_; + return to_tonlib_api(*id); + //tonlib_api::make_object( + // ton::tonlib_api::ton_blockIdExt(id->workchain_, id->) + //); })); return td::Status::OK(); } auto to_tonlib_api(const ton::lite_api::liteServer_transactionId& txid) -> tonlib_api_ptr { - return tonlib_api::make_object( - txid.mode_, txid.account_.as_slice().str(), txid.lt_, txid.hash_.as_slice().str()); + return tonlib_api::make_object(txid.mode_, txid.account_.as_slice().str(), txid.lt_, + txid.hash_.as_slice().str()); } td::Status TonlibClient::do_request(const tonlib_api::blocks_getTransactions& request, - td::Promise>&& promise) { + td::Promise>&& promise) { TRY_RESULT(block, to_lite_api(*request.id_)) TRY_RESULT(account, to_bits256((*request.after_).account_, "account")); auto after = ton::lite_api::make_object(account, (*request.after_).lt_); - client_.send_query(ton::lite_api::liteServer_listBlockTransactions( - std::move(block), - request.mode_, - request.count_, - std::move(after), - false, - false), + client_.send_query(ton::lite_api::liteServer_listBlockTransactions(std::move(block), request.mode_, request.count_, + std::move(after), false, false), promise.wrap([](lite_api_ptr&& bTxes) { - const auto& id = bTxes->id_; - //for (auto id : ids) { - tonlib_api::blocks_transactions r; - r.id_ = to_tonlib_api(*id); - r.req_count_ = bTxes->req_count_; - r.incomplete_ = bTxes->incomplete_; - for (auto& id: bTxes->ids_) { - //tonlib_api::blocks_shortTxId txid = tonlib_api::blocks_shortTxId(id->mode_, id->account_.as_slice().str(), id->lt_, id->hash_.as_slice().str()); - //r.transactions_.push_back(txid); - r.transactions_.push_back(to_tonlib_api(*id)); - } - return tonlib_api::make_object(std::move(r)); + const auto& id = bTxes->id_; + //for (auto id : ids) { + tonlib_api::blocks_transactions r; + r.id_ = to_tonlib_api(*id); + r.req_count_ = bTxes->req_count_; + r.incomplete_ = bTxes->incomplete_; + for (auto& id : bTxes->ids_) { + //tonlib_api::blocks_shortTxId txid = tonlib_api::blocks_shortTxId(id->mode_, id->account_.as_slice().str(), id->lt_, id->hash_.as_slice().str()); + //r.transactions_.push_back(txid); + r.transactions_.push_back(to_tonlib_api(*id)); + } + return tonlib_api::make_object(std::move(r)); })); return td::Status::OK(); } td::Status TonlibClient::do_request(const tonlib_api::blocks_getBlockHeader& request, - td::Promise>&& promise) { + td::Promise>&& promise) { TRY_RESULT(block, to_lite_api(*request.id_)) - client_.send_query(ton::lite_api::liteServer_getBlockHeader( - std::move(block), - 0xffff), - promise.wrap([](lite_api_ptr&& hdr) { - auto blk_id = ton::create_block_id(hdr->id_); - auto R = vm::std_boc_deserialize(std::move(hdr->header_proof_)); - tonlib_api::blocks_header header; - if (R.is_error()) { - LOG(WARNING) << "R.is_error() "; - } else { - auto root = R.move_as_ok(); - try { - ton::RootHash vhash{root->get_hash().bits()}; - auto virt_root = vm::MerkleProof::virtualize(root, 1); - if (virt_root.is_null()) { - LOG(WARNING) << "virt root is null"; - } else { - std::vector prev; - ton::BlockIdExt mc_blkid; - bool after_split; - auto res = block::unpack_block_prev_blk_ext(virt_root, blk_id, prev, mc_blkid, after_split); - if (res.is_error()) { - LOG(WARNING) << "res.is_error() "; - } else { - block::gen::Block::Record blk; - block::gen::BlockInfo::Record info; - if (!(tlb::unpack_cell(virt_root, blk) && tlb::unpack_cell(blk.info, info))) { - LOG(WARNING) << "unpack failed"; - } else { - header.id_ = to_tonlib_api(blk_id); - header.global_id_ = blk.global_id; - header.version_ = info.version; - header.flags_ = info.flags; - header.after_merge_ = info.after_merge; - header.after_split_ = info.after_split; - header.before_split_ = info.before_split; - header.want_merge_ = info.want_merge; - header.want_split_ = info.want_split; - header.validator_list_hash_short_ = info.gen_validator_list_hash_short; - header.catchain_seqno_ = info.gen_catchain_seqno; - header.min_ref_mc_seqno_ = info.min_ref_mc_seqno; - header.start_lt_ = info.start_lt; - header.end_lt_ = info.end_lt; - header.gen_utime_ = info.gen_utime; - header.is_key_block_ = info.key_block; - header.vert_seqno_ = info.vert_seq_no; - if(!info.not_master) { - header.prev_key_block_seqno_ = info.prev_key_block_seqno; - } - for (auto id : prev) { - header.prev_blocks_.push_back(to_tonlib_api(id)); - } - //if(info.before_split) { - //} else { - //} - return tonlib_api::make_object(std::move(header)); - } - } - } - } catch (vm::VmError& err) { - auto E = err.as_status(PSLICE() << "error processing header for " << blk_id.to_str() << " :"); - LOG(ERROR) << std::move(E); - } catch (vm::VmVirtError& err) { - auto E = err.as_status(PSLICE() << "error processing header for " << blk_id.to_str() << " :"); - LOG(ERROR) << std::move(E); - } catch (...) { - LOG(WARNING) << "exception catched "; - } - } - return tonlib_api::make_object(std::move(header)); - })); + client_.send_query( + ton::lite_api::liteServer_getBlockHeader(std::move(block), 0xffff), + promise.wrap([](lite_api_ptr&& hdr) { + auto blk_id = ton::create_block_id(hdr->id_); + auto R = vm::std_boc_deserialize(std::move(hdr->header_proof_)); + tonlib_api::blocks_header header; + if (R.is_error()) { + LOG(WARNING) << "R.is_error() "; + } else { + auto root = R.move_as_ok(); + try { + ton::RootHash vhash{root->get_hash().bits()}; + auto virt_root = vm::MerkleProof::virtualize(root, 1); + if (virt_root.is_null()) { + LOG(WARNING) << "virt root is null"; + } else { + std::vector prev; + ton::BlockIdExt mc_blkid; + bool after_split; + auto res = block::unpack_block_prev_blk_ext(virt_root, blk_id, prev, mc_blkid, after_split); + if (res.is_error()) { + LOG(WARNING) << "res.is_error() "; + } else { + block::gen::Block::Record blk; + block::gen::BlockInfo::Record info; + if (!(tlb::unpack_cell(virt_root, blk) && tlb::unpack_cell(blk.info, info))) { + LOG(WARNING) << "unpack failed"; + } else { + header.id_ = to_tonlib_api(blk_id); + header.global_id_ = blk.global_id; + header.version_ = info.version; + header.flags_ = info.flags; + header.after_merge_ = info.after_merge; + header.after_split_ = info.after_split; + header.before_split_ = info.before_split; + header.want_merge_ = info.want_merge; + header.want_split_ = info.want_split; + header.validator_list_hash_short_ = info.gen_validator_list_hash_short; + header.catchain_seqno_ = info.gen_catchain_seqno; + header.min_ref_mc_seqno_ = info.min_ref_mc_seqno; + header.start_lt_ = info.start_lt; + header.end_lt_ = info.end_lt; + header.gen_utime_ = info.gen_utime; + header.is_key_block_ = info.key_block; + header.vert_seqno_ = info.vert_seq_no; + if (!info.not_master) { + header.prev_key_block_seqno_ = info.prev_key_block_seqno; + } + for (auto id : prev) { + header.prev_blocks_.push_back(to_tonlib_api(id)); + } + //if(info.before_split) { + //} else { + //} + return tonlib_api::make_object(std::move(header)); + } + } + } + } catch (vm::VmError& err) { + auto E = err.as_status(PSLICE() << "error processing header for " << blk_id.to_str() << " :"); + LOG(ERROR) << std::move(E); + } catch (vm::VmVirtError& err) { + auto E = err.as_status(PSLICE() << "error processing header for " << blk_id.to_str() << " :"); + LOG(ERROR) << std::move(E); + } catch (...) { + LOG(WARNING) << "exception catched "; + } + } + return tonlib_api::make_object(std::move(header)); + })); return td::Status::OK(); } @@ -4890,15 +4867,17 @@ void TonlibClient::load_libs_from_disk() { if (r_dict.is_error()) { return; } - libraries = vm::Dictionary(vm::load_cell_slice(vm::CellBuilder().append_cellslice(vm::load_cell_slice( - r_dict.move_as_ok())).finalize()), 256); + libraries = vm::Dictionary( + vm::load_cell_slice(vm::CellBuilder().append_cellslice(vm::load_cell_slice(r_dict.move_as_ok())).finalize()), + 256); // int n = 0; for (auto&& lr : libraries) n++; LOG(DEBUG) << "loaded libraries from disk cache"; } void TonlibClient::store_libs_to_disk() { // NB: Dictionary.get_root_cell does not compute_root, and it is protected - kv_->set("tonlib.libcache", vm::std_boc_serialize(vm::CellBuilder().append_cellslice(libraries.get_root()) - .finalize()).move_as_ok().as_slice()); + kv_->set("tonlib.libcache", vm::std_boc_serialize(vm::CellBuilder().append_cellslice(libraries.get_root()).finalize()) + .move_as_ok() + .as_slice()); // int n = 0; for (auto&& lr : libraries) n++; LOG(DEBUG) << "stored libraries to disk cache"; } diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 9430b6725e..1f4d8f5b46 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -47,7 +47,7 @@ #include "tonlib/TonlibClient.h" #include "tonlib/TonlibCallback.h" -#include "tonlib/ExtClientRaw.h" +#include "tonlib/ExtClientLazy.h" #include "smc-envelope/ManualDns.h" #include "smc-envelope/PaymentChannel.h" @@ -62,7 +62,6 @@ #include #include #include "git.h" -#include "ExtClientMulti.h" using tonlib_api::make_object; @@ -224,7 +223,7 @@ class TonlibCli : public td::actor::Actor { if (options_.use_callbacks_for_network) { auto config = tonlib::Config::parse(options_.config).move_as_ok(); - class Callback : public tonlib::ExtClientRaw::Callback { + class Callback : public tonlib::ExtClientLazy::Callback { public: explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { } @@ -232,28 +231,15 @@ class TonlibCli : public td::actor::Actor { private: td::actor::ActorShared<> parent_; }; - std::vector> full_servers; - int lite_client_id = -1, cnt = 0; - for (const auto& s : config.lite_clients) { - if (s.is_full) { - full_servers.emplace_back(s.adnl_id, s.address); - } - } - if (!full_servers.empty()) { - raw_client_ = tonlib::ExtClientRaw::create(std::move(full_servers), - td::make_unique(td::actor::actor_shared())); - } else { - CHECK(!config.lite_clients.empty()); - raw_client_ = td::actor::create_actor( - "ExtClientMulti", config.lite_clients, td::make_unique(td::actor::actor_shared())); - } ref_cnt_++; + raw_client_ = tonlib::ExtClientLazy::create(config.lite_servers, + td::make_unique(td::actor::actor_shared())); } auto config = !options_.config.empty() - ? make_object(options_.config, options_.name, - options_.use_callbacks_for_network, options_.ignore_cache) - : nullptr; + ? make_object(options_.config, options_.name, + options_.use_callbacks_for_network, options_.ignore_cache) + : nullptr; tonlib_api::object_ptr ks_type; if (options_.in_memory) { @@ -1544,9 +1530,8 @@ class TonlibCli : public td::actor::Actor { auto update = tonlib_api::move_object_as(std::move(result)); CHECK(!raw_client_.empty()); snd_bytes_ += update->data_.size(); - ton::ShardIdFull shard(update->workchain_, update->shard_); - send_closure(raw_client_, &tonlib::ExtClientRaw::send_query, "query", td::BufferSlice(update->data_), shard, - td::Timestamp::in(5), + send_closure(raw_client_, &tonlib::ExtClientLazy::send_query, "query", td::BufferSlice(update->data_), + ton::ShardIdFull(update->workchain_, update->shard_), td::Timestamp::in(5), [actor_id = actor_id(this), id = update->id_](td::Result res) { send_closure(actor_id, &TonlibCli::on_adnl_result, id, std::move(res)); }); From 29851c38ef91ce42215f5e8965084c9a3f9aca7c Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 12 Jan 2023 17:32:59 +0300 Subject: [PATCH 045/524] Various changes after merge --- crypto/block/mc-config.cpp | 2 +- overlay/overlay-manager.cpp | 34 ++++++++---------- overlay/overlay-manager.h | 1 + overlay/overlay.cpp | 2 +- overlay/overlay.h | 1 + overlay/overlay.hpp | 6 +++- overlay/overlays.h | 5 ++- tl/generate/scheme/ton_api.tl | 4 +-- tl/generate/scheme/ton_api.tlo | Bin 87748 -> 87728 bytes tonlib/tonlib/QueryTraits.h | 14 ++++++++ .../validator-engine-console.cpp | 2 ++ validator/full-node-shard.cpp | 25 ++++++++----- validator/full-node-shard.h | 2 +- validator/full-node-shard.hpp | 1 + validator/full-node.cpp | 12 +++---- validator/manager.cpp | 9 +++++ validator/validator-group.cpp | 29 ++++++++------- 17 files changed, 93 insertions(+), 56 deletions(-) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 31826acf02..8d592c210e 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2203,7 +2203,7 @@ Ref ConfigInfo::lookup_library(td::ConstBitPtr root_hash) const { CollatorConfig Config::get_collator_config(bool need_collator_nodes) const { CollatorConfig collator_config; gen::CollatorConfig::Record rec; - auto cell = get_config_param(-41); + auto cell = get_config_param(41, -41); if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { return collator_config; } diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index b7ca1270ee..f9a798781a 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -105,16 +105,6 @@ void OverlayManager::create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, Ov std::move(callback), std::move(rules), scope, announce_self)); } -void OverlayManager::create_public_overlay_external(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - std::unique_ptr callback, OverlayPrivacyRules rules, - td::string scope) { - CHECK(!dht_node_.empty()); - auto id = overlay_id.compute_short_id(); - register_overlay(local_id, id, - Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), - std::move(callback), std::move(rules), scope, true)); -} - void OverlayManager::create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules) { @@ -162,23 +152,16 @@ void OverlayManager::receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdSh auto M = R.move_as_ok(); - OverlayIdShort overlay_id{M->overlay_}; - auto send_remove_peer = [&, this]() { - send_message(src, dst, overlay_id, create_serialize_tl_object()); - }; - auto it = overlays_.find(dst); if (it == overlays_.end()) { VLOG(OVERLAY_NOTICE) << this << ": query to unknown overlay " << M->overlay_ << "@" << dst << " from " << src; promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad local_id " << dst)); - send_remove_peer(); return; } - auto it2 = it->second.find(overlay_id); + auto it2 = it->second.find(OverlayIdShort{M->overlay_}); if (it2 == it->second.end()) { VLOG(OVERLAY_NOTICE) << this << ": query to localid not in overlay " << M->overlay_ << "@" << dst << " from " << src; promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad overlay_id " << M->overlay_)); - send_remove_peer(); return; } @@ -380,6 +363,19 @@ void OverlayManager::set_priority_broadcast_receivers(adnl::AdnlNodeIdShort loca td::actor::send_closure(it2->second, &Overlay::set_priority_broadcast_receivers, std::move(nodes)); } +void OverlayManager::forget_peer(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, + adnl::AdnlNodeIdShort peer_id) { + auto it = overlays_.find(local_id); + if (it == overlays_.end()) { + return; + } + auto it2 = it->second.find(overlay); + if (it2 == it->second.end()) { + return; + } + td::actor::send_closure(it2->second, &Overlay::forget_peer, peer_id); +} + Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, td::BufferSlice signature) : issued_by_(issued_by) @@ -421,7 +417,7 @@ td::BufferSlice Certificate::to_sign(OverlayIdShort overlay_id, PublicKeyHash is } } -PublicKeyHash Certificate::issuer_hash() const { +const PublicKeyHash Certificate::issuer_hash() const { PublicKeyHash r; issued_by_.visit( td::overloaded([&](const PublicKeyHash &x) { r = x; }, [&](const PublicKey &x) { r = x.compute_short_id(); })); diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index f7ed32bac5..5b59b3715f 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -98,6 +98,7 @@ class OverlayManager : public Overlays { void set_priority_broadcast_receivers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, std::vector nodes) override; + void forget_peer(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, adnl::AdnlNodeIdShort peer_id) override; struct PrintId {}; diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index f0bd3b9ca9..c4cf8428b1 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -231,7 +231,7 @@ void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice dat return; } auto Q = X.move_as_ok(); - ton_api::downcast_call(*Q, [Self = this, &Q, &src](auto &object) { + ton_api::downcast_call(*Q.get(), [Self = this, &Q, &src](auto &object) { Self->process_broadcast(src, move_tl_object_as>(Q)); }); } diff --git a/overlay/overlay.h b/overlay/overlay.h index 4289142cb9..5141ae5380 100644 --- a/overlay/overlay.h +++ b/overlay/overlay.h @@ -70,6 +70,7 @@ class Overlay : public td::actor::Actor { //virtual void receive_broadcast(td::BufferSlice data) = 0; //virtual void subscribe(std::unique_ptr callback) = 0; virtual void set_priority_broadcast_receivers(std::vector nodes) = 0; + virtual void forget_peer(adnl::AdnlNodeIdShort peer_id) = 0; }; } // namespace overlay diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 208aba7f3d..a8fd7f3035 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -278,7 +278,11 @@ class OverlayImpl : public Overlay { priority_broadcast_receivers_ = std::move(nodes); } - void wait_neighbours_not_empty(td::Promise promise, int max_retries = 10); + void forget_peer(adnl::AdnlNodeIdShort peer_id) override { + del_peer(peer_id); + } + + void wait_neighbours_not_empty(td::Promise promise, int max_retries = 20); private: template diff --git a/overlay/overlays.h b/overlay/overlays.h index def80d7589..1141efee24 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -145,7 +145,7 @@ class Certificate { bool is_fec) const; tl_object_ptr tl() const; const PublicKey &issuer() const; - PublicKeyHash issuer_hash() const; + const PublicKeyHash issuer_hash() const; static td::Result> create(tl_object_ptr cert); static tl_object_ptr empty_tl(); @@ -170,8 +170,6 @@ class Overlays : public td::actor::Actor { td::Promise promise) { promise.set_value(td::Unit()); } - virtual void on_remove_peer(adnl::AdnlNodeIdShort src) { - } virtual ~Callback() = default; }; @@ -242,6 +240,7 @@ class Overlays : public td::actor::Actor { virtual void set_priority_broadcast_receivers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, std::vector nodes) = 0; + virtual void forget_peer(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, adnl::AdnlNodeIdShort peer_id) = 0; }; } // namespace overlay diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 004b1dd294..08d4c9dfb6 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -218,8 +218,6 @@ overlay.message overlay:int256 = overlay.Message; //overlay.randomPeers peers:(vector adnl.node) = overlay.RandomPeers; overlay.broadcastList hashes:(vector int256) = overlay.BroadcastList; -overlay.message.removePeer = overlay.message.InternalMessage; - overlay.fec.received hash:int256 = overlay.Broadcast; overlay.fec.completed hash:int256 = overlay.Broadcast; @@ -424,6 +422,8 @@ tonNode.archiveInfo id:long = tonNode.ArchiveInfo; tonNode.outMsgQueueProof queue_proof:bytes block_state_proof:bytes = tonNode.OutMsgQueueProof; tonNode.outMsgQueueProofEmpty = tonNode.OutMsgQueueProof; +tonNode.forgetPeer = tonNode.ForgetPeer; + ---functions--- tonNode.getNextBlockDescription prev_block:tonNode.blockIdExt = tonNode.BlockDescription; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 97c2bda2df4125bec8281603581b892696097361..9dc8c51b2a5dad856a6abbf39aeedffcc767d724 100644 GIT binary patch delta 106 zcmX@ImUROV<;ZVdA)ml2-uOjnwNOcZo?m`Ss-9bZQF>}gKx%5ylnhOx*lu mUyq>yimEiYDh81F&0|773_wui%%;O(+jbBnwz=kmM;ZV##4DTt delta 120 zcmdn6mi5S5)(tuGvQK8N{wkkemRgjPSgDtrT3no%o~q}WSCU$kmzXp8psf7n9Qg#^ z&3|+@Otb*0l|olrl$r|^3P?>YVgSkCJSODB00c$OY&smaZ3jVOn-}bxVYs>EgH0L$ DeYiDE diff --git a/tonlib/tonlib/QueryTraits.h b/tonlib/tonlib/QueryTraits.h index bf480f64c8..b583cd05b2 100644 --- a/tonlib/tonlib/QueryTraits.h +++ b/tonlib/tonlib/QueryTraits.h @@ -114,6 +114,13 @@ struct QueryTraits { } }; +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getAccountStatePrunned& q) { + return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); + } +}; + template<> struct QueryTraits { static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_runSmcMethod& q) { @@ -198,5 +205,12 @@ struct QueryTraits { } }; +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getShardBlockProof& q) { + return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); + } +}; + } // namespace tonlib diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index a1a0135b5e..97e7690a37 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -144,6 +144,8 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 35a9b85180..69804a07a8 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -88,7 +88,7 @@ void FullNodeShardImpl::create_overlay() { public: void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { - // just ignore + td::actor::send_closure(node_, &FullNodeShardImpl::receive_message, src, std::move(data)); } void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, td::Promise promise) override { @@ -101,26 +101,23 @@ void FullNodeShardImpl::create_overlay() { td::Promise promise) override { td::actor::send_closure(node_, &FullNodeShardImpl::check_broadcast, src, std::move(data), std::move(promise)); } - void on_remove_peer(adnl::AdnlNodeIdShort src) override { - td::actor::send_closure(node_, &FullNodeShardImpl::remove_neighbour, src); - } Callback(td::actor::ActorId node) : node_(node) { } private: td::actor::ActorId node_; }; - if (is_active()) { td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay, adnl_id_, overlay_id_full_.clone(), std::make_unique(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() << ", \"workchain_id\": " << get_workchain() << " }"); } else { - td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay_external, adnl_id_, - overlay_id_full_.clone(), std::make_unique(actor_id(this)), rules_, + td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay_ex, adnl_id_, overlay_id_full_.clone(), + std::make_unique(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() - << ", \"workchain_id\": " << get_workchain() << " }"); + << ", \"workchain_id\": " << get_workchain() << " }", + false); } td::actor::send_closure(rldp_, &rldp::Rldp::add_id, adnl_id_); @@ -647,6 +644,8 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { if (!is_active()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_message, src, adnl_id_, overlay_id_, + create_serialize_tl_object()); promise.set_error(td::Status::Error("shard is inactive")); return; } @@ -658,6 +657,16 @@ void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice ton_api::downcast_call(*B.move_as_ok().get(), [&](auto &obj) { this->process_query(src, obj, std::move(promise)); }); } +void FullNodeShardImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) { + auto B = fetch_tl_object(std::move(data), true); + if (B.is_error()) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Got tonNode.forgetPeer from " << src; + neighbours_.erase(src); + td::actor::send_closure(overlays_, &overlay::Overlays::forget_peer, adnl_id_, overlay_id_, src); +} + void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query) { td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_ihr_message, std::move(query.message_->data_)); diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 847091c652..633c93409c 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -31,7 +31,7 @@ namespace fullnode { enum FullNodeShardMode { active, // Node can answer queries about the shard active_temp, // Like 'active', but queries about shard state are not allowed (only blocks) - inactive // Node is not a part of the overlay (overlay is_external) + inactive // Node is not a part of the overlay }; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 1cb2957de1..7d80792bf9 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -142,6 +142,7 @@ class FullNodeShardImpl : public FullNodeShard { // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query, // td::Promise promise); void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise); + void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query); diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 72af6fe8e8..42e8511fe6 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -17,10 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "full-node.hpp" -#include "ton/ton-shard.h" #include "ton/ton-io.hpp" #include "td/actor/MultiPromise.h" -#include "ton/ton-types.h" namespace ton { @@ -140,12 +138,8 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s new_shards.insert(ShardIdFull(masterchainId)); std::set workchains; auto cut_shard = [&](ShardIdFull shard) -> ShardIdFull { - unsigned pfx_len = shard.pfx_len(); - unsigned min_split = state->soft_min_split_depth(shard.workchain); - if (min_split < pfx_len) { - return shard_prefix(shard, min_split); - } - return shard; + int min_split = state->soft_min_split_depth(shard.workchain); + return min_split < shard.pfx_len() ? shard_prefix(shard, min_split) : shard; }; auto set_active = [&](ShardIdFull shard, FullNodeShardMode mode) { while (new_active.emplace(shard, mode).second && shard.pfx_len() > 0) { @@ -388,6 +382,7 @@ void FullNodeImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull } td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { + ShardIdFull shard0 = shard; while (true) { auto it = shards_.find(shard); if (it != shards_.end() && it->second.exists) { @@ -404,6 +399,7 @@ td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { } shard = shard_parent(shard); } + shard = shard0; auto it = shards_.find(shard); if (it == shards_.end()) { it = shards_.emplace(shard = ShardIdFull(shard.workchain), ShardInfo{}).first; diff --git a/validator/manager.cpp b/validator/manager.cpp index 3015e8827e..c19e96f41b 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -583,6 +583,10 @@ void ValidatorManagerImpl::run_ext_query(td::BufferSlice data, td::Promise> promise) { + if (last_masterchain_state_.not_null() && !opts_->need_monitor(handle->id().shard_full(), last_masterchain_state_)) { + return promise.set_error( + td::Status::Error(PSTRING() << "not monitoring shard " << handle->id().shard_full().to_str())); + } auto it = wait_state_.find(handle->id()); if (it == wait_state_.end()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { @@ -678,6 +682,10 @@ void ValidatorManagerImpl::wait_block_data_short(BlockIdExt block_id, td::uint32 void ValidatorManagerImpl::wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) { + if (last_masterchain_state_.not_null() && !opts_->need_monitor(left_id.shard_full(), last_masterchain_state_)) { + return promise.set_error( + td::Status::Error(PSTRING() << "not monitoring shard " << left_id.shard_full().to_str())); + } td::actor::create_actor("merge", left_id, right_id, priority, actor_id(this), timeout, std::move(promise)) .release(); @@ -1604,6 +1612,7 @@ void ValidatorManagerImpl::started(ValidatorManagerInitResult R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::read_gc_list, R.move_as_ok()); } }); + td::actor::send_closure(db_, &Db::get_destroyed_validator_sessions, std::move(P)); auto Q = td::PromiseCreator::lambda( diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 601fc52648..b4f665c39c 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -139,6 +139,13 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat collator_config_.full_collated_data ? ValidateMode::full_collated_data : 0); } +void ValidatorGroup::update_approve_cache(td::uint32 round_id, CacheKey key, UnixTime value) { + if (approved_candidates_cache_round_ != round_id) { + return; + } + approved_candidates_cache_[key] = value; +} + void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash src, td::BufferSlice block_data, RootHash root_hash, FileHash file_hash, std::vector signatures, @@ -382,8 +389,8 @@ void ValidatorGroup::get_session_info( td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_session_info, std::move(P)); } -void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeout, - td::Promise promise, unsigned max_retries) { +void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeout, td::Promise promise, + unsigned max_retries) { if (round_id < last_known_round_id_) { promise.set_error(td::Status::Error("too old")); return; @@ -391,9 +398,11 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo adnl::AdnlNodeIdShort collator = adnl::AdnlNodeIdShort::zero(); // TODO: some way to choose node (similar to "unreliability" in full-node) int cnt = 0; - for (const block::CollatorNodeDescr& c : collator_config_.collator_nodes) { - if (shard_intersects(shard_, c.shard) && td::Random::fast(0, cnt) == 0) { - collator = adnl::AdnlNodeIdShort(c.adnl_id); + for (const block::CollatorNodeDescr &c : collator_config_.collator_nodes) { + if (shard_intersects(shard_, c.shard)) { + if (td::Random::fast(0, cnt) == 0) { + collator = adnl::AdnlNodeIdShort(c.adnl_id); + } ++cnt; } } @@ -438,8 +447,8 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo size_t max_answer_size = config_.max_block_size + config_.max_collated_data_size + 256; td::Timestamp query_timeout = td::Timestamp::in(10.0); query_timeout.relax(timeout); - td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_adnl_id_, collator, "collatequery", - std::move(P), timeout, std::move(query), max_answer_size); + td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_adnl_id_, collator, "collatequery", std::move(P), + timeout, std::move(query), max_answer_size); } void ValidatorGroup::receive_collate_query_response(td::uint32 round_id, td::BufferSlice data, @@ -460,15 +469,11 @@ void ValidatorGroup::receive_collate_query_response(td::uint32 round_id, td::Buf return; } auto key = PublicKey{b->source_}; - if (!key.is_ed25519()) { + if (key != local_id_full_) { promise.set_error(td::Status::Error("collate query: block candidate source mismatch")); return; } auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - if (e_key != Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}) { - promise.set_error(td::Status::Error("collate query: block candidate source mismatch")); - return; - } auto block_id = ton::create_block_id(b->id_); if (block_id.shard_full() != shard_) { promise.set_error(td::Status::Error("collate query: shard mismatch")); From d324fa58208e69179b473ccb1cc2a8fc1c5e6ce0 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 13 Jan 2023 19:04:11 +0300 Subject: [PATCH 046/524] Bugfix --- overlay/overlay-peers.cpp | 4 +++- validator/impl/out-msg-queue-proof.cpp | 10 +++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index 409f099352..28dac8f05f 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -24,7 +24,9 @@ namespace overlay { void OverlayImpl::del_peer(adnl::AdnlNodeIdShort id) { auto P = peers_.get(id); - CHECK(P != nullptr); + if (P == nullptr) { + return; + } VLOG(OVERLAY_DEBUG) << this << ": deleting peer " << id; diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 4ba102c538..7591ab179f 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -177,7 +177,7 @@ void WaitOutMsgQueueProof::run_local() { [SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::abort_query, - R.move_as_error_prefix("failed to get shard state")); + R.move_as_error_prefix("failed to get shard state: ")); } else { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::got_state_root, R.move_as_ok()->root_cell()); @@ -189,7 +189,7 @@ void WaitOutMsgQueueProof::run_local() { [SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::abort_query, - R.move_as_error_prefix("failed to get block data")); + R.move_as_error_prefix("failed to get block data: ")); } else { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::got_block_root, R.move_as_ok()->root_cell()); @@ -247,7 +247,7 @@ void WaitOutMsgQueueProof::run_net() { void BuildOutMsgQueueProof::abort_query(td::Status reason) { if (promise_) { - LOG(WARNING) << "failed to build msg queue proof for " << block_id_.to_str() << ": " << reason; + LOG(DEBUG) << "failed to build msg queue proof for " << block_id_.to_str() << ": " << reason; promise_.set_error( reason.move_as_error_prefix(PSTRING() << "failed to build msg queue proof for " << block_id_.to_str() << ": ")); } @@ -260,7 +260,7 @@ void BuildOutMsgQueueProof::start_up() { [SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, - R.move_as_error_prefix("failed to get shard state")); + R.move_as_error_prefix("failed to get shard state: ")); } else { td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_state_root, R.move_as_ok()->root_cell()); @@ -272,7 +272,7 @@ void BuildOutMsgQueueProof::start_up() { [SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, - R.move_as_error_prefix("failed to get block data")); + R.move_as_error_prefix("failed to get block data: ")); } else { td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_block_root, R.move_as_ok()->root_cell()); From 225f71238b61694ca42fc7af95415098b9914d0e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 16 Jan 2023 19:30:15 +0300 Subject: [PATCH 047/524] Option for collating masterchain on validators --- validator-engine/validator-engine.cpp | 19 ++++++++++++---- validator-engine/validator-engine.hpp | 7 +++--- validator/collator-node.cpp | 26 +++++++++++----------- validator/manager.cpp | 31 +++++++++++++++++++-------- validator/manager.hpp | 1 + validator/validator-group.cpp | 6 ++++-- validator/validator-group.hpp | 7 +++--- validator/validator-options.hpp | 10 ++++----- validator/validator.h | 6 ++++-- 9 files changed, 72 insertions(+), 41 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index cd14ae1edd..4ed6ae56bc 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1419,7 +1419,7 @@ td::Status ValidatorEngine::load_global_config() { h.push_back(b); } validator_options_.write().set_hardforks(std::move(h)); - validator_options_.write().set_validator_lite_mode(validator_lite_mode_); + validator_options_.write().set_validator_mode(validator_mode_); return td::Status::OK(); } @@ -3866,9 +3866,20 @@ int main(int argc, char *argv[]) { p.add_option('M', "not-all-shards", "monitor only a necessary set of shards instead of all", [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_not_all_shards); }); }); - p.add_option('\0', "lite-validator", "lite-mode validator (don't collate blocks, use collator nodes instead)", [&]() { - acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_validator_lite_mode); }); - }); + p.add_option('\0', "lite-validator-shards", + "lite-mode validator for shard blocks (don't collate blocks, use collator nodes instead)", [&]() { + acts.push_back([&x]() { + td::actor::send_closure(x, &ValidatorEngine::set_validator_mode, + ton::validator::ValidatorManagerOptions::validator_lite_shards); + }); + }); + p.add_option('\0', "lite-validator-all", + "lite-mode validator for all blocks (don't collate blocks, use collator nodes instead)", [&]() { + acts.push_back([&x]() { + td::actor::send_closure(x, &ValidatorEngine::set_validator_mode, + ton::validator::ValidatorManagerOptions::validator_lite_all); + }); + }); td::uint32 threads = 7; p.add_checked_option( 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 49010bc476..7e0db66141 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -217,7 +217,8 @@ class ValidatorEngine : public td::actor::Actor { ton::BlockSeqno truncate_seqno_{0}; std::string session_logs_file_; bool not_all_shards_ = false; - bool validator_lite_mode_ = false; + ton::validator::ValidatorManagerOptions::ValidatorMode validator_mode_ = + ton::validator::ValidatorManagerOptions::validator_normal; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -272,8 +273,8 @@ class ValidatorEngine : public td::actor::Actor { void set_not_all_shards() { not_all_shards_ = true; } - void set_validator_lite_mode() { - validator_lite_mode_ = true; + void set_validator_mode(ton::validator::ValidatorManagerOptions::ValidatorMode mode) { + validator_mode_ = mode; } void start_up() override; diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index c6afa54186..193613c742 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -160,19 +160,19 @@ void CollatorNode::receive_query_cont(adnl::AdnlNodeIdShort src, ShardIdFull sha td::Ref min_mc_state, std::vector prev_blocks, Ed25519_PublicKey creator, td::Promise promise) { auto P = td::PromiseCreator::lambda([promise = std::move(promise), src](td::Result R) mutable { - if (R.is_error()) { - LOG(WARNING) << "Query from " << src << ", error: " << R.error(); - promise.set_result(serialize_error(R.move_as_error())); - } else { - LOG(INFO) << "Query from " << src << ", success"; - auto block = R.move_as_ok(); - auto result = create_serialize_tl_object( - create_tl_object(PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), - create_tl_block_id(block.id), std::move(block.data), - std::move(block.collated_data))); - promise.set_result(std::move(result)); - } - }); + if (R.is_error()) { + LOG(WARNING) << "Query from " << src << ", error: " << R.error(); + promise.set_result(serialize_error(R.move_as_error())); + } else { + LOG(INFO) << "Query from " << src << ", success"; + auto block = R.move_as_ok(); + auto result = create_serialize_tl_object( + create_tl_object(PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), + create_tl_block_id(block.id), std::move(block.data), + std::move(block.collated_data))); + promise.set_result(std::move(result)); + } + }); run_collate_query(shard, min_mc_state->get_block_id(), std::move(prev_blocks), creator, min_mc_state->get_validator_set(shard), manager_, td::Timestamp::in(10.0), std::move(P)); diff --git a/validator/manager.cpp b/validator/manager.cpp index c19e96f41b..07266fb149 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -18,11 +18,9 @@ */ #include "manager.hpp" #include "validator-group.hpp" -#include "adnl/utils.hpp" #include "downloaders/wait-block-state.hpp" #include "downloaders/wait-block-state-merge.hpp" #include "downloaders/wait-block-data.hpp" -#include "validator-group.hpp" #include "fabric.h" #include "manager.h" #include "validate-broadcast.hpp" @@ -199,7 +197,7 @@ void ValidatorManagerImpl::validate_block(ReceivedBlock block, td::Promise R) mutable { + [SelfId = actor_id(this), promise = std::move(promise), id = blkid](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); } else { @@ -1241,7 +1239,7 @@ void ValidatorManagerImpl::write_handle(BlockHandle handle, td::Promise promise) { bool received = handle->received(); bool inited_state = handle->received_state(); - bool inited_proof = handle->id().is_masterchain() ? handle->inited_proof() : handle->inited_proof(); + bool inited_proof = handle->inited_proof(); if (handle->need_flush()) { handle->flush(actor_id(this), handle, std::move(promise)); @@ -1712,13 +1710,14 @@ bool ValidatorManagerImpl::out_of_sync() { void ValidatorManagerImpl::prestart_sync() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); - td::actor::send_closure(SelfId, &ValidatorManagerImpl::download_next_archive); + // Don't download archives + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finish_prestart_sync); }); td::actor::send_closure(db_, &Db::set_async_mode, false, std::move(P)); } void ValidatorManagerImpl::download_next_archive() { - if (true) { + if (!out_of_sync()) { finish_prestart_sync(); return; } @@ -1836,6 +1835,13 @@ void ValidatorManagerImpl::new_masterchain_block() { for (auto &c : collator_nodes_) { td::actor::send_closure(c.second.actor, &CollatorNode::new_masterchain_block_notification, last_masterchain_state_); } + if (opts_->validator_mode() == ValidatorManagerOptions::validator_lite_shards && validating_masterchain()) { + // Prepare neighboours' queues for collating masterchain + for (const auto &desc : last_masterchain_state_->get_shards()) { + wait_out_msg_queue_proof(desc->top_block_id(), ShardIdFull(masterchainId), 0, td::Timestamp::in(10.0), + [](td::Ref) {}); + } + } if (last_masterchain_seqno_ % 1024 == 0) { LOG(WARNING) << "applied masterchain block " << last_masterchain_block_id_; @@ -2174,7 +2180,7 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group "validatorgroup", shard, validator_id, session_id, validator_set, last_masterchain_state_->get_collator_config(true), opts, keyring_, adnl_, rldp_, overlays_, db_root_, actor_id(this), init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), - opts_->validator_lite_mode()); + opts_->validator_mode()); return G; } } @@ -2348,7 +2354,7 @@ void ValidatorManagerImpl::allow_block_state_gc(BlockIdExt block_id, td::Promise return; } auto shards = gc_masterchain_state_->get_shards(); - for (auto shard : shards) { + for (const auto &shard : shards) { if (shard_intersects(shard->shard(), block_id.shard_full())) { promise.set_result(block_id.id.seqno < shard->top_block_id().id.seqno); return; @@ -2564,7 +2570,14 @@ bool ValidatorManagerImpl::is_validator() { } bool ValidatorManagerImpl::is_collator() { - return !collator_nodes_.empty() || (!opts_->validator_lite_mode() && is_validator()); + return !collator_nodes_.empty() || + (opts_->validator_mode() != ValidatorManagerOptions::validator_lite_all && is_validator()); +} + +bool ValidatorManagerImpl::validating_masterchain() { + return !get_validator(ShardIdFull(masterchainId), + last_masterchain_state_->get_validator_set(ShardIdFull(masterchainId))) + .is_zero(); } PublicKeyHash ValidatorManagerImpl::get_validator(ShardIdFull shard, td::Ref val_set) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 835f7de070..3debccb39d 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -506,6 +506,7 @@ class ValidatorManagerImpl : public ValidatorManager { bool is_validator(); bool is_collator(); + bool validating_masterchain(); PublicKeyHash get_validator(ShardIdFull shard, td::Ref val_set); ValidatorManagerImpl(td::Ref opts, std::string db_root, diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index b4f665c39c..7cc92f8041 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -50,7 +50,8 @@ void ValidatorGroup::generate_block_candidate(td::uint32 round_id, td::Promise R) { td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, std::move(cache), std::move(R)); }; - if (lite_mode_) { + if (mode_ == ValidatorManagerOptions::validator_lite_all || + (mode_ == ValidatorManagerOptions::validator_lite_shards && !shard_.is_masterchain())) { send_collate_query(round_id, td::Timestamp::in(10.0), std::move(P)); return; } @@ -197,7 +198,8 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref }); run_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, std::move(sig_set), - std::move(approve_sig_set), send_broadcast, shard_.is_masterchain() || !lite_mode_, manager_, + std::move(approve_sig_set), send_broadcast, + shard_.is_masterchain() || mode_ == ValidatorManagerOptions::validator_normal, manager_, std::move(P)); } diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index c7964ae549..6a78827d5d 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -65,7 +65,8 @@ class ValidatorGroup : public td::actor::Actor { td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, td::actor::ActorId validator_manager, bool create_session, - bool allow_unsafe_self_blocks_resync, bool lite_mode = false) + bool allow_unsafe_self_blocks_resync, + ValidatorManagerOptions::ValidatorMode mode = ValidatorManagerOptions::validator_normal) : shard_(shard) , local_id_(std::move(local_id)) , session_id_(session_id) @@ -80,7 +81,7 @@ class ValidatorGroup : public td::actor::Actor { , manager_(validator_manager) , init_(create_session) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) - , lite_mode_(lite_mode) { + , mode_(mode) { } private: @@ -125,7 +126,7 @@ class ValidatorGroup : public td::actor::Actor { bool init_ = false; bool started_ = false; bool allow_unsafe_self_blocks_resync_; - bool lite_mode_ = false; + ValidatorManagerOptions::ValidatorMode mode_; td::uint32 last_known_round_id_ = 0; struct CachedCollatedBlock { diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index e96292ad3e..ea100a50bc 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -112,8 +112,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { std::string get_session_logs_file() const override { return session_logs_file_; } - bool validator_lite_mode() const override { - return validator_lite_mode_; + ValidatorMode validator_mode() const override { + return validator_mode_; } void set_zero_block_id(BlockIdExt block_id) override { @@ -168,8 +168,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_session_logs_file(std::string f) override { session_logs_file_ = std::move(f); } - void set_validator_lite_mode(bool value) override { - validator_lite_mode_ = value; + void set_validator_mode(ValidatorMode value) override { + validator_mode_ = value; } ValidatorManagerOptionsImpl *make_copy() const override { @@ -211,7 +211,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { BlockSeqno truncate_{0}; BlockSeqno sync_upto_{0}; std::string session_logs_file_; - bool validator_lite_mode_ = false; + ValidatorMode validator_mode_ = validator_normal; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 343de0824d..4ce3f4d021 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -54,6 +54,8 @@ struct PerfTimerStats { struct ValidatorManagerOptions : public td::CntObject { public: + enum ValidatorMode { validator_normal, validator_lite_shards, validator_lite_all }; + virtual BlockIdExt zero_block_id() const = 0; virtual BlockIdExt init_block_id() const = 0; virtual bool need_monitor(ShardIdFull shard, const td::Ref& state) const = 0; @@ -79,7 +81,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual BlockSeqno get_truncate_seqno() const = 0; virtual BlockSeqno sync_upto() const = 0; virtual std::string get_session_logs_file() const = 0; - virtual bool validator_lite_mode() const = 0; + virtual ValidatorMode validator_mode() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; @@ -98,7 +100,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void truncate_db(BlockSeqno seqno) = 0; virtual void set_sync_upto(BlockSeqno seqno) = 0; virtual void set_session_logs_file(std::string f) = 0; - virtual void set_validator_lite_mode(bool value) = 0; + virtual void set_validator_mode(ValidatorMode value) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, From 5ad7f8c246646afa819ea95b30fe026183019bf3 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 17 Jan 2023 21:02:47 +0300 Subject: [PATCH 048/524] Allow queries to collator from validators only --- validator/collator-node.cpp | 19 +++++++++++++++++++ validator/collator-node.hpp | 1 + 2 files changed, 20 insertions(+) diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 193613c742..654e4e7428 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -100,6 +100,22 @@ void CollatorNode::new_masterchain_block_notification(td::Ref } } } + if (validators_.empty() || state->is_key_state()) { + validators_.clear(); + for (int next : {-1, 0, 1}) { + td::Ref vals = state->get_total_validator_set(next); + if (vals.not_null()) { + for (const ValidatorDescr& descr : vals->export_vector()) { + if (descr.addr.is_zero()) { + validators_.insert( + adnl::AdnlNodeIdShort(PublicKey(pubkeys::Ed25519{descr.key.as_bits256()}).compute_short_id())); + } else { + validators_.insert(adnl::AdnlNodeIdShort(descr.addr)); + } + } + } + } + } } static td::BufferSlice serialize_error(td::Status error) { @@ -110,6 +126,9 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data td::Promise promise) { auto SelfId = actor_id(this); auto status = [&]() -> td::Status { + if (!validators_.count(src)) { + return td::Status::Error("src is not a validator"); + } TRY_RESULT(f, fetch_tl_object(std::move(data), true)); ShardIdFull shard(f->workchain_, f->shard_); if (!shard.is_valid_ext()) { diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index b3b232cf9e..46848fcfef 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -49,6 +49,7 @@ class CollatorNode : public td::actor::Actor { td::actor::ActorId adnl_; td::actor::ActorId rldp_; std::vector shards_; + std::set validators_; }; } // namespace validator From 5dd0c15d072d016b15b8952c9bbd119953a0accf Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 4 Jul 2023 23:34:34 +0300 Subject: [PATCH 049/524] Limit imported msg queue size --- crypto/block/block.tlb | 1 + crypto/block/output-queue-merger.cpp | 42 +++-- crypto/block/output-queue-merger.h | 18 +- tl/generate/scheme/ton_api.tl | 2 +- tl/generate/scheme/ton_api.tlo | Bin 90648 -> 90684 bytes validator/impl/collator-impl.h | 2 + validator/impl/collator.cpp | 29 ++- validator/impl/out-msg-queue-proof.cpp | 208 +++++++++++++-------- validator/impl/out-msg-queue-proof.hpp | 10 +- validator/impl/validate-query.cpp | 21 ++- validator/impl/validate-query.hpp | 1 + validator/interfaces/out-msg-queue-proof.h | 13 +- 12 files changed, 226 insertions(+), 121 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index cb78604974..b316d1c649 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -829,6 +829,7 @@ top_block_descr#d5 proof_for:BlockIdExt signatures:(Maybe ^BlockSignatures) // COLLATED DATA // top_block_descr_set#4ac789f3 collection:(HashmapE 96 ^TopBlockDescr) = TopBlockDescrSet; +neighbor_msg_queue_limits#7e549333 neighbors:(HashmapE 96 int32) = NeighborMsgQueueLimits; // // VALIDATOR MISBEHAVIOR COMPLAINTS diff --git a/crypto/block/output-queue-merger.cpp b/crypto/block/output-queue-merger.cpp index 1084bb1a8c..ca7021a890 100644 --- a/crypto/block/output-queue-merger.cpp +++ b/crypto/block/output-queue-merger.cpp @@ -134,34 +134,33 @@ bool OutputQueueMerger::MsgKeyValue::split(MsgKeyValue& second) { return true; } -bool OutputQueueMerger::add_root(int src, Ref outmsg_root) { +void OutputQueueMerger::add_root(int src, Ref outmsg_root, td::int32 msg_limit) { if (outmsg_root.is_null()) { - return true; + return; } //block::gen::HashmapAug{352, block::gen::t_EnqueuedMsg, block::gen::t_uint64}.print_ref(std::cerr, outmsg_root); auto kv = std::make_unique(src, std::move(outmsg_root)); if (kv->replace_by_prefix(common_pfx.cbits(), common_pfx_len)) { heap.push_back(std::move(kv)); } - return true; -} - -OutputQueueMerger::OutputQueueMerger(ton::ShardIdFull _queue_for, std::vector _neighbors) - : queue_for(_queue_for), neighbors(std::move(_neighbors)), eof(false), failed(false) { - init(); + if ((int)src_remaining_msgs_.size() < src + 1) { + src_remaining_msgs_.resize(src + 1); + } + src_remaining_msgs_[src] = msg_limit; } -void OutputQueueMerger::init() { +OutputQueueMerger::OutputQueueMerger(ton::ShardIdFull queue_for, std::vector neighbors) + : eof(false), failed(false) { common_pfx.bits().store_int(queue_for.workchain, 32); int l = queue_for.pfx_len(); td::bitstring::bits_store_long_top(common_pfx.bits() + 32, queue_for.shard, l); common_pfx_len = 32 + l; int i = 0; - for (block::McShardDescr& neighbor : neighbors) { - if (!neighbor.is_disabled()) { - LOG(DEBUG) << "adding " << (neighbor.outmsg_root.is_null() ? "" : "non-") << "empty output queue for neighbor #" - << i << " (" << neighbor.blk_.to_str() << ")"; - add_root(i++, neighbor.outmsg_root); + for (Neighbor& neighbor : neighbors) { + if (!neighbor.disabled_) { + LOG(DEBUG) << "adding " << (neighbor.outmsg_root_.is_null() ? "" : "non-") << "empty output queue for neighbor #" + << i << " (" << neighbor.block_id_.to_str() << ")"; + add_root(i++, neighbor.outmsg_root_, neighbor.msg_limit_); } else { LOG(DEBUG) << "skipping output queue for disabled neighbor #" << i; i++; @@ -200,6 +199,10 @@ bool OutputQueueMerger::load() { unsigned long long lt = heap[0]->lt; std::size_t orig_size = msg_list.size(); do { + if (src_remaining_msgs_[heap[0]->source] == 0) { + std::pop_heap(heap.begin(), heap.end(), MsgKeyValue::greater); + continue; + } while (heap[0]->is_fork()) { auto other = std::make_unique(); if (!heap[0]->split(*other)) { @@ -215,6 +218,17 @@ bool OutputQueueMerger::load() { heap.pop_back(); } while (!heap.empty() && heap[0]->lt <= lt); std::sort(msg_list.begin() + orig_size, msg_list.end(), MsgKeyValue::less); + size_t j = orig_size; + for (size_t i = orig_size; i < msg_list.size(); ++i) { + td::int32 &remaining = src_remaining_msgs_[msg_list[i]->source]; + if (remaining != 0) { + if (remaining > 0) { + --remaining; + } + msg_list[j++] = std::move(msg_list[i]); + } + } + msg_list.resize(j); return true; } diff --git a/crypto/block/output-queue-merger.h b/crypto/block/output-queue-merger.h index bf3d85868c..f5c15ab4d7 100644 --- a/crypto/block/output-queue-merger.h +++ b/crypto/block/output-queue-merger.h @@ -52,12 +52,20 @@ struct OutputQueueMerger { bool split(MsgKeyValue& second); }; // - ton::ShardIdFull queue_for; std::vector> msg_list; - std::vector neighbors; public: - OutputQueueMerger(ton::ShardIdFull _queue_for, std::vector _neighbors); + struct Neighbor { + ton::BlockIdExt block_id_; + td::Ref outmsg_root_; + bool disabled_; + td::int32 msg_limit_; // -1 - unlimited + Neighbor(ton::BlockIdExt block_id, td::Ref outmsg_root, bool disabled = false, td::int32 msg_limit = -1) + : block_id_(block_id), outmsg_root_(std::move(outmsg_root)), disabled_(disabled), msg_limit_(msg_limit) { + } + }; + + OutputQueueMerger(ton::ShardIdFull queue_for, std::vector neighbors); bool is_eof() const { return eof; } @@ -70,10 +78,10 @@ struct OutputQueueMerger { int common_pfx_len; std::vector> heap; std::size_t pos{0}; + std::vector src_remaining_msgs_; bool eof; bool failed; - void init(); - bool add_root(int src, Ref outmsg_root); + void add_root(int src, Ref outmsg_root, td::int32 msg_limit); bool load(); }; diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 2a11819728..6d52b8454b 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -415,7 +415,7 @@ tonNode.success = tonNode.Success; tonNode.archiveNotFound = tonNode.ArchiveInfo; tonNode.archiveInfo id:long = tonNode.ArchiveInfo; -tonNode.outMsgQueueProof queue_proof:bytes block_state_proof:bytes = tonNode.OutMsgQueueProof; +tonNode.outMsgQueueProof queue_proof:bytes block_state_proof:bytes msg_count:int = tonNode.OutMsgQueueProof; tonNode.outMsgQueueProofEmpty = tonNode.OutMsgQueueProof; tonNode.forgetPeer = tonNode.ForgetPeer; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index f85ddc6b49657a9b9477515e82ad3d1775fe4197..1eed452846aa645ab1b76b678dde81080e579487 100644 GIT binary patch delta 138 zcmbPngmupm)(txPEI$+_I5!*UPdKG1rQfttq9i}hFFz$!FTb?Jw>UkpG_^D}peR2- zje&syr1Iu5As^<+FMkG4R`6$G;mj>gpRBN21#HM X_|=mDRyJKBny~@Q+P);3v4jl(pQkWp delta 119 zcmdmUgmuOd)(txPEJ<2%Q#KpuPdLTepmM}SVsh;-2_6Op29WU0V?sVmlVAP}o~+=; z0aiFU=a+#5ObV!qp~#s{hr_n*AlQ^Gzj_ilVY1AV8}qcMmqarLfOT#^5zSb_1^{`V BEK2|Y diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index dab3f6a952..80a8e9a7de 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -133,6 +133,8 @@ class Collator final : public td::actor::Actor { std::unique_ptr config_; std::unique_ptr shard_conf_; std::map> aux_mc_states_; + std::vector neighbor_queues_; + vm::Dictionary neighbor_msg_queues_limits_{32 + 64}; std::vector neighbors_; std::unique_ptr nb_out_msgs_; std::vector special_smcs; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index a2f3f7bd3c..c6ac6f2fbb 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -77,7 +77,7 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha } void Collator::start_up() { - LOG(DEBUG) << "Collator for shard " << shard_.to_str() << " started"; + LOG(INFO) << "Collator for shard " << shard_.to_str() << " started"; LOG(DEBUG) << "Previous block #1 is " << prev_blocks.at(0).to_str(); if (prev_blocks.size() > 1) { LOG(DEBUG) << "Previous block #2 is " << prev_blocks.at(1).to_str(); @@ -690,7 +690,15 @@ void Collator::got_neighbor_msg_queue(unsigned i, td::Resultprefetch_ref(0)); + auto queue_root = qinfo.out_queue->prefetch_ref(0); + descr.set_queue_root(queue_root); + neighbor_queues_.emplace_back(descr.top_block_id(), queue_root, descr.is_disabled(), res->msg_count_); + if (res->msg_count_ != -1) { + td::BitArray<96> key; + key.bits().store_int(block_id.id.workchain, 32); + (key.bits() + 32).store_uint(block_id.id.shard, 64); + neighbor_msg_queues_limits_.set_builder(key, vm::CellBuilder().store_long(res->msg_count_, 32)); + } // comment the next two lines in the future when the output queues become huge // CHECK(block::gen::t_OutMsgQueueInfo.validate_ref(1000000, outq_descr->root_cell())); // CHECK(block::tlb::t_OutMsgQueueInfo.validate_ref(1000000, outq_descr->root_cell())); @@ -1723,7 +1731,7 @@ bool Collator::do_collate() { // 1.3. create OutputQueueMerger from adjusted neighbors CHECK(!nb_out_msgs_); LOG(DEBUG) << "creating OutputQueueMerger"; - nb_out_msgs_ = std::make_unique(shard_, neighbors_); + nb_out_msgs_ = std::make_unique(shard_, neighbor_queues_); // 1.4. compute created / minted / recovered if (!init_value_create()) { return fatal_error("cannot compute the value to be created / minted / recovered"); @@ -4013,18 +4021,25 @@ bool Collator::create_collated_data() { auto cell = collate_shard_block_descr_set(); if (cell.is_null()) { return true; - return fatal_error("cannot collate the collection of used shard block descriptions"); + // return fatal_error("cannot collate the collection of used shard block descriptions"); } collated_roots_.push_back(std::move(cell)); } + // 2. Message count for neighbors' out queues + if (!neighbor_msg_queues_limits_.is_empty()) { + vm::CellBuilder cb; + cb.store_long(block::gen::t_NeighborMsgQueueLimits.cons_tag[0], 32); + cb.store_maybe_ref(neighbor_msg_queues_limits_.get_root_cell()); + collated_roots_.push_back(cb.finalize_novm()); + } if (!full_collated_data_) { return true; } - // 2. Proofs for hashes of states: previous states + neighbors + // 3. Proofs for hashes of states: previous states + neighbors for (const auto& p : block_state_proofs_) { collated_roots_.push_back(p.second); } - // 3. Previous state proof (only shadchains) + // 4. Previous state proof (only shadchains) std::map> proofs; if (!is_masterchain()) { if (!prepare_msg_queue_proof()) { @@ -4038,7 +4053,7 @@ bool Collator::create_collated_data() { } proofs[prev_state_root_->get_hash().bits()] = std::move(state_proof); } - // 4. Proofs for message queues + // 5. Proofs for message queues for (vm::MerkleProofBuilder &mpb : neighbor_proof_builders_) { auto r_proof = mpb.extract_proof(); if (r_proof.is_error()) { diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 56a4edd34d..329da607dd 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -22,94 +22,45 @@ #include "interfaces/validator-manager.h" #include "block/block-parse.h" #include "block/block-auto.h" +#include "output-queue-merger.h" namespace ton { namespace validator { -td::Result> OutMsgQueueProof::fetch(BlockIdExt block_id, ShardIdFull dst_shard, - const ton_api::tonNode_outMsgQueueProof& f) { - Ref block_state_proof; - td::Bits256 state_root_hash; - if (block_id.seqno() == 0) { - if (!f.block_state_proof_.empty()) { - return td::Status::Error("expected empty block state proof"); - } - state_root_hash = block_id.root_hash; - } else { - TRY_RESULT_ASSIGN(block_state_proof, vm::std_boc_deserialize(f.block_state_proof_.as_slice())); - TRY_RESULT_ASSIGN(state_root_hash, unpack_block_state_proof(block_id, block_state_proof)); - } - - TRY_RESULT(queue_proof, vm::std_boc_deserialize(f.queue_proof_.as_slice())); - auto virtual_root = vm::MerkleProof::virtualize(queue_proof, 1); - if (virtual_root.is_null()) { - return td::Status::Error("invalid queue proof"); +static td::Status check_no_prunned(const Ref& cell) { + if (cell.is_null()) { + return td::Status::OK(); } - if (virtual_root->get_hash().as_slice() != state_root_hash.as_slice()) { - return td::Status::Error("state root hash mismatch"); + TRY_RESULT(loaded_cell, cell->load_cell()); + if (loaded_cell.data_cell->get_level() > 0) { + return td::Status::Error("prunned branch"); } + return td::Status::OK(); +} - // Validate proof - auto state_root = vm::CellSlice(vm::NoVm(), queue_proof).prefetch_ref(0); - TRY_RESULT_PREFIX(state, ShardStateQ::fetch(block_id, {}, state_root), "invalid proof: "); - TRY_RESULT_PREFIX(outq_descr, state->message_queue(), "invalid proof: "); - - block::gen::OutMsgQueueInfo::Record qinfo; - if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { - return td::Status::Error("invalid proof: invalid message queue"); - } - td::Ref proc_info = qinfo.proc_info->prefetch_ref(0); - if (proc_info.not_null() && proc_info->get_level() != 0) { - return td::Status::Error("invalid proof: proc_info has prunned branches"); +static td::Status check_no_prunned(const vm::CellSlice& cs) { + for (unsigned i = 0; i < cs.size_refs(); ++i) { + TRY_STATUS(check_no_prunned(cs.prefetch_ref(i))); } - td::Ref ihr_pending = qinfo.ihr_pending->prefetch_ref(0); - if (ihr_pending.not_null() && ihr_pending->get_level() != 0) { - return td::Status::Error("invalid proof: ihr_pending has prunned branches"); - } - auto queue = - std::make_unique(qinfo.out_queue->prefetch_ref(0), 352, block::tlb::aug_OutMsgQueue); - td::BitArray<96> prefix; - td::BitPtr ptr = prefix.bits(); - ptr.store_int(dst_shard.workchain, 32); - ptr.advance(32); - ptr.store_uint(dst_shard.shard, 64); - if (!queue->cut_prefix_subdict(prefix.bits(), 32 + dst_shard.pfx_len())) { - return td::Status::Error("invalid proof: failed to cut queue dict"); - } - if (queue->get_root_cell().not_null() && queue->get_root_cell()->get_level() != 0) { - return td::Status::Error("invalid proof: msg queue has prunned branches"); - } - - return Ref(true, std::move(virtual_root), std::move(block_state_proof)); + return td::Status::OK(); } -td::Result> OutMsgQueueProof::serialize(BlockIdExt block_id, - ShardIdFull dst_shard, - Ref state_root, - Ref block_root) { - if (!dst_shard.is_valid_ext()) { - return td::Status::Error("invalid shard"); - } - vm::MerkleProofBuilder mpb{std::move(state_root)}; - TRY_RESULT(state, ShardStateQ::fetch(block_id, {}, mpb.root())); - TRY_RESULT(outq_descr, state->message_queue()); - block::gen::OutMsgQueueInfo::Record qinfo; - if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { - return td::Status::Error("invalid message queue"); - } +static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_shard, + const block::gen::OutMsgQueueInfo::Record& qinfo) { + td::uint64 estimated_proof_size = 0; td::HashSet visited; - std::function)> dfs = [&](Ref cell) { + std::function dfs_cs; + auto dfs = [&](Ref cell) { if (cell.is_null() || !visited.insert(cell->get_hash()).second) { return; } - vm::CellSlice cs(vm::NoVm(), cell); - for (unsigned i = 0; i < cs.size_refs(); i++) { - dfs(cs.prefetch_ref(i)); - } + dfs_cs(vm::CellSlice(vm::NoVm(), cell)); }; - auto dfs_cs = [&](const vm::CellSlice &cs) { + dfs_cs = [&](const vm::CellSlice& cs) { + // Based on BlockLimitStatus::estimate_block_size + estimated_proof_size += 12 + (cs.size() + 7) / 8 + cs.size_refs() * 3; for (unsigned i = 0; i < cs.size_refs(); i++) { dfs(cs.prefetch_ref(i)); } @@ -117,17 +68,65 @@ td::Result> OutMsgQueueProof::s dfs_cs(*qinfo.proc_info); dfs_cs(*qinfo.ihr_pending); - auto queue = - std::make_unique(qinfo.out_queue->prefetch_ref(0), 352, block::tlb::aug_OutMsgQueue); - td::BitArray<96> prefix; - td::BitPtr ptr = prefix.bits(); - ptr.store_int(dst_shard.workchain, 32); - ptr.advance(32); - ptr.store_uint(dst_shard.shard, 64); - if (!queue->cut_prefix_subdict(prefix.bits(), 32 + dst_shard.pfx_len())) { + block::OutputQueueMerger queue_merger{ + dst_shard, {block::OutputQueueMerger::Neighbor{block_id, qinfo.out_queue->prefetch_ref()}}}; + td::int32 msg_count = 0; + bool limit_reached = false; + + while (!queue_merger.is_eof()) { + auto kv = queue_merger.extract_cur(); + queue_merger.next(); + ++msg_count; + + // TODO: Get processed_upto from destination shard (in request?) + /* + // Parse message to check if it was processed (as in Collator::process_inbound_message) + ton::LogicalTime enqueued_lt = kv->msg->prefetch_ulong(64); + auto msg_env = kv->msg->prefetch_ref(); + block::tlb::MsgEnvelope::Record_std env; + if (!tlb::unpack_cell(msg_env, env)) { + return td::Status::Error("cannot unpack MsgEnvelope of an internal message"); + } + vm::CellSlice cs{vm::NoVmOrd{}, env.msg}; + block::gen::CommonMsgInfo::Record_int_msg_info info; + if (!tlb::unpack(cs, info)) { + return td::Status::Error("cannot unpack CommonMsgInfo of an internal message"); + } + auto src_prefix = block::tlb::MsgAddressInt::get_prefix(info.src); + auto dest_prefix = block::tlb::MsgAddressInt::get_prefix(info.dest); + auto cur_prefix = block::interpolate_addr(src_prefix, dest_prefix, env.cur_addr); + auto next_prefix = block::interpolate_addr(src_prefix, dest_prefix, env.next_addr); + block::EnqueuedMsgDescr descr{cur_prefix, next_prefix, kv->lt, enqueued_lt, env.msg->get_hash().bits()}; + if (dst_processed_upto->already_processed(descr)) { + } else { + }*/ + + dfs_cs(*kv->msg); + TRY_STATUS_PREFIX(check_no_prunned(*kv->msg), "invalid message proof: ") + if (estimated_proof_size > OutMsgQueueProof::QUEUE_SIZE_THRESHOLD) { + limit_reached = true; + break; + } + } + return limit_reached ? msg_count : -1; +} + +td::Result> OutMsgQueueProof::build(BlockIdExt block_id, + ShardIdFull dst_shard, + Ref state_root, + Ref block_root) { + if (!dst_shard.is_valid_ext()) { + return td::Status::Error("invalid shard"); + } + + vm::MerkleProofBuilder mpb{std::move(state_root)}; + TRY_RESULT(state, ShardStateQ::fetch(block_id, {}, mpb.root())); + TRY_RESULT(outq_descr, state->message_queue()); + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { return td::Status::Error("invalid message queue"); } - dfs(queue->get_root_cell()); + TRY_RESULT(cnt, process_queue(block_id, dst_shard, qinfo)); TRY_RESULT(queue_proof, mpb.extract_proof_boc()); td::BufferSlice block_state_proof; @@ -135,7 +134,52 @@ td::Result> OutMsgQueueProof::s TRY_RESULT(proof, create_block_state_proof(std::move(block_root))); TRY_RESULT_ASSIGN(block_state_proof, vm::std_boc_serialize(std::move(proof), 31)); } - return create_tl_object(std::move(queue_proof), std::move(block_state_proof)); + return create_tl_object(std::move(queue_proof), std::move(block_state_proof), + cnt); +} + +td::Result> OutMsgQueueProof::fetch(BlockIdExt block_id, ShardIdFull dst_shard, + const ton_api::tonNode_outMsgQueueProof& f) { + try { + Ref block_state_proof; + td::Bits256 state_root_hash; + if (block_id.seqno() == 0) { + if (!f.block_state_proof_.empty()) { + return td::Status::Error("expected empty block state proof"); + } + state_root_hash = block_id.root_hash; + } else { + TRY_RESULT_ASSIGN(block_state_proof, vm::std_boc_deserialize(f.block_state_proof_.as_slice())); + TRY_RESULT_ASSIGN(state_root_hash, unpack_block_state_proof(block_id, block_state_proof)); + } + + TRY_RESULT(queue_proof, vm::std_boc_deserialize(f.queue_proof_.as_slice())); + auto virtual_root = vm::MerkleProof::virtualize(queue_proof, 1); + if (virtual_root.is_null()) { + return td::Status::Error("invalid queue proof"); + } + if (virtual_root->get_hash().as_slice() != state_root_hash.as_slice()) { + return td::Status::Error("state root hash mismatch"); + } + + // Validate proof + TRY_RESULT_PREFIX(state, ShardStateQ::fetch(block_id, {}, virtual_root), "invalid proof: "); + TRY_RESULT_PREFIX(outq_descr, state->message_queue(), "invalid proof: "); + + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { + return td::Status::Error("invalid proof: invalid message queue"); + } + TRY_STATUS_PREFIX(check_no_prunned(qinfo.proc_info->prefetch_ref(0)), "invalid proc_info: ") + TRY_STATUS_PREFIX(check_no_prunned(qinfo.ihr_pending->prefetch_ref(0)), "invalid ihr_pending: ") + TRY_RESULT(cnt, process_queue(block_id, dst_shard, qinfo)); + if (cnt != f.msg_count_) { + return td::Status::Error(PSTRING() << "invalid msg_count: expected=" << f.msg_count_ << ", found=" << cnt); + } + return Ref(true, std::move(virtual_root), std::move(block_state_proof), cnt); + } catch (vm::VmVirtError& err) { + return td::Status::Error(PSTRING() << "invalid proof: " << err.get_msg()); + } } void WaitOutMsgQueueProof::alarm() { @@ -296,7 +340,7 @@ void BuildOutMsgQueueProof::got_block_root(Ref root) { } void BuildOutMsgQueueProof::build_proof() { - auto result = OutMsgQueueProof::serialize(block_id_, dst_shard_, std::move(state_root_), std::move(block_root_)); + auto result = OutMsgQueueProof::build(block_id_, dst_shard_, std::move(state_root_), std::move(block_root_)); if (result.is_error()) { LOG(ERROR) << "Failed to build msg queue proof: " << result.error(); } diff --git a/validator/impl/out-msg-queue-proof.hpp b/validator/impl/out-msg-queue-proof.hpp index 6a18ca6cb6..e94f805f40 100644 --- a/validator/impl/out-msg-queue-proof.hpp +++ b/validator/impl/out-msg-queue-proof.hpp @@ -62,7 +62,6 @@ class WaitOutMsgQueueProof : public td::actor::Actor { void run_net(); - private: BlockIdExt block_id_; ShardIdFull dst_shard_; @@ -80,12 +79,9 @@ class WaitOutMsgQueueProof : public td::actor::Actor { class BuildOutMsgQueueProof : public td::actor::Actor { public: BuildOutMsgQueueProof(BlockIdExt block_id, ShardIdFull dst_shard, - td::actor::ActorId manager, - td::Promise> promise) - : block_id_(std::move(block_id)) - , dst_shard_(dst_shard) - , manager_(manager) - , promise_(std::move(promise)) { + td::actor::ActorId manager, + td::Promise> promise) + : block_id_(std::move(block_id)), dst_shard_(dst_shard), manager_(manager), promise_(std::move(promise)) { } void abort_query(td::Status reason); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index f43bad2979..3329e4fd59 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -522,6 +522,13 @@ bool ValidateQuery::extract_collated_data_from(Ref croot, int idx) { top_shard_descr_dict_ = std::make_unique(cs.prefetch_ref(), 96); return true; } + if (block::gen::t_NeighborMsgQueueLimits.has_valid_tag(cs)) { + LOG(DEBUG) << "collated datum # " << idx << " is a NeighborMsgQueueLimits"; + if (!block::gen::t_NeighborMsgQueueLimits.validate_upto(10000, cs)) { + return reject_query("invalid NeighborMsgQueueLimits"); + } + neighbor_msg_queues_limits_ = vm::Dictionary{cs.prefetch_ref(0), 32 + 64}; + } LOG(WARNING) << "collated datum # " << idx << " has unknown type (magic " << cs.prefetch_ulong(32) << "), ignoring"; return true; } @@ -4106,7 +4113,19 @@ bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, } bool ValidateQuery::check_in_queue() { - block::OutputQueueMerger nb_out_msgs(shard_, neighbors_); + std::vector neighbor_queues; + for (const auto& descr : neighbors_) { + td::BitArray<96> key; + key.bits().store_int(descr.workchain(), 32); + (key.bits() + 32).store_uint(descr.shard().shard, 64); + auto r = neighbor_msg_queues_limits_.lookup(key); + td::int32 msg_limit = r.is_null() ? -1 : (td::int32)r->prefetch_long(32); + if (msg_limit < -1) { + return reject_query("invalid value in NeighborMsgQueueLimits"); + } + neighbor_queues.emplace_back(descr.top_block_id(), descr.outmsg_root, descr.disabled_, msg_limit); + } + block::OutputQueueMerger nb_out_msgs(shard_, std::move(neighbor_queues)); while (!nb_out_msgs.is_eof()) { auto kv = nb_out_msgs.extract_cur(); CHECK(kv && kv->msg.not_null()); diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 942b32a5f9..cd14504e61 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -206,6 +206,7 @@ class ValidateQuery : public td::actor::Actor { block::ActionPhaseConfig action_phase_cfg_; td::RefInt256 masterchain_create_fee_, basechain_create_fee_; + vm::Dictionary neighbor_msg_queues_limits_{32 + 64}; std::vector neighbors_; std::map> aux_mc_states_; diff --git a/validator/interfaces/out-msg-queue-proof.h b/validator/interfaces/out-msg-queue-proof.h index 1fece66a5c..5d79249137 100644 --- a/validator/interfaces/out-msg-queue-proof.h +++ b/validator/interfaces/out-msg-queue-proof.h @@ -18,6 +18,7 @@ #include "vm/cells.h" #include "ton/ton-types.h" #include "auto/tl/ton_api.h" +#include "block/block.h" namespace ton { @@ -25,17 +26,21 @@ namespace validator { using td::Ref; struct OutMsgQueueProof : public td::CntObject { - OutMsgQueueProof(Ref state_root, Ref block_state_proof) - : state_root_(std::move(state_root)), block_state_proof_(std::move(block_state_proof)) { + OutMsgQueueProof(Ref state_root, Ref block_state_proof, td::int32 msg_count = -1) + : state_root_(std::move(state_root)), block_state_proof_(std::move(block_state_proof)), msg_count_(msg_count) { } Ref state_root_; Ref block_state_proof_; + td::int32 msg_count_; // -1 - up to end of queue static td::Result> fetch(BlockIdExt block_id, ShardIdFull dst_shard, const ton_api::tonNode_outMsgQueueProof &f); - static td::Result> serialize( - BlockIdExt block_id, ShardIdFull dst_shard, Ref state_root, Ref block_root); + static td::Result> build(BlockIdExt block_id, ShardIdFull dst_shard, + Ref state_root, + Ref block_root); + + static const td::uint64 QUEUE_SIZE_THRESHOLD = 128 * 1024; }; } // namespace validator From f10c7f54a82bc8ead6aaae9e6fd01db02493e3c0 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 6 Jul 2023 11:44:21 +0300 Subject: [PATCH 050/524] Bugfix in processing message queues; improve out_msg_queue_cleanup --- crypto/block/output-queue-merger.cpp | 2 +- crypto/block/output-queue-merger.h | 1 + validator/impl/collator-impl.h | 5 +- validator/impl/collator.cpp | 69 ++++++++++++++++++++++------ validator/impl/validate-query.cpp | 8 ++++ 5 files changed, 68 insertions(+), 17 deletions(-) diff --git a/crypto/block/output-queue-merger.cpp b/crypto/block/output-queue-merger.cpp index ca7021a890..2955b3e012 100644 --- a/crypto/block/output-queue-merger.cpp +++ b/crypto/block/output-queue-merger.cpp @@ -229,7 +229,7 @@ bool OutputQueueMerger::load() { } } msg_list.resize(j); - return true; + return msg_list.size() > orig_size; } } // namespace block diff --git a/crypto/block/output-queue-merger.h b/crypto/block/output-queue-merger.h index f5c15ab4d7..3eb4b37994 100644 --- a/crypto/block/output-queue-merger.h +++ b/crypto/block/output-queue-merger.h @@ -60,6 +60,7 @@ struct OutputQueueMerger { td::Ref outmsg_root_; bool disabled_; td::int32 msg_limit_; // -1 - unlimited + Neighbor() = default; Neighbor(ton::BlockIdExt block_id, td::Ref outmsg_root, bool disabled = false, td::int32 msg_limit = -1) : block_id_(block_id), outmsg_root_(std::move(outmsg_root)), disabled_(disabled), msg_limit_(msg_limit) { } diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 80a8e9a7de..5981189664 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -133,8 +133,8 @@ class Collator final : public td::actor::Actor { std::unique_ptr config_; std::unique_ptr shard_conf_; std::map> aux_mc_states_; - std::vector neighbor_queues_; - vm::Dictionary neighbor_msg_queues_limits_{32 + 64}; + std::map neighbor_msg_queues_limits_; + vm::Dictionary neighbor_msg_queues_limits_dict_{32 + 64}; std::vector neighbors_; std::unique_ptr nb_out_msgs_; std::vector special_smcs; @@ -241,6 +241,7 @@ class Collator final : public td::actor::Actor { bool register_shard_block_creators(std::vector creator_list); bool init_block_limits(); bool compute_minted_amount(block::CurrencyCollection& to_mint); + bool create_output_queue_merger(); bool init_value_create(); bool try_collate(); bool do_preinit(); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index c6ac6f2fbb..1ce0a4b4c3 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -672,9 +672,9 @@ void Collator::got_neighbor_msg_queue(unsigned i, td::Result outq_descr = outq_descr_res.move_as_ok(); block::McShardDescr& descr = neighbors_.at(i); + LOG(DEBUG) << "obtained outbound queue for neighbor #" << i << "(" << descr.shard().to_str() << ")"; if (outq_descr->get_block_id() != descr.blk_) { LOG(DEBUG) << "outq_descr->id = " << outq_descr->get_block_id().to_str() << " ; descr.id = " << descr.blk_.to_str(); fatal_error( @@ -692,12 +692,13 @@ void Collator::got_neighbor_msg_queue(unsigned i, td::Resultprefetch_ref(0); descr.set_queue_root(queue_root); - neighbor_queues_.emplace_back(descr.top_block_id(), queue_root, descr.is_disabled(), res->msg_count_); if (res->msg_count_ != -1) { + LOG(DEBUG) << "Neighbor " << descr.shard().to_str() << " has msg_limit=" << res->msg_count_; td::BitArray<96> key; key.bits().store_int(block_id.id.workchain, 32); (key.bits() + 32).store_uint(block_id.id.shard, 64); - neighbor_msg_queues_limits_.set_builder(key, vm::CellBuilder().store_long(res->msg_count_, 32)); + neighbor_msg_queues_limits_dict_.set_builder(key, vm::CellBuilder().store_long(res->msg_count_, 32)); + neighbor_msg_queues_limits_[block_id.shard_full()] = res->msg_count_; } // comment the next two lines in the future when the output queues become huge // CHECK(block::gen::t_OutMsgQueueInfo.validate_ref(1000000, outq_descr->root_cell())); @@ -1674,6 +1675,17 @@ bool Collator::compute_minted_amount(block::CurrencyCollection& to_mint) { return true; } +bool Collator::create_output_queue_merger() { + std::vector neighbor_queues; + for (const auto& descr : neighbors_) { + auto it = neighbor_msg_queues_limits_.find(descr.shard()); + td::int32 msg_limit = it == neighbor_msg_queues_limits_.end() ? -1 : it->second; + neighbor_queues.emplace_back(descr.top_block_id(), descr.outmsg_root, descr.disabled_, msg_limit); + } + nb_out_msgs_ = std::make_unique(shard_, neighbor_queues); + return true; +} + bool Collator::init_value_create() { value_flow_.created.set_zero(); value_flow_.minted.set_zero(); @@ -1731,7 +1743,9 @@ bool Collator::do_collate() { // 1.3. create OutputQueueMerger from adjusted neighbors CHECK(!nb_out_msgs_); LOG(DEBUG) << "creating OutputQueueMerger"; - nb_out_msgs_ = std::make_unique(shard_, neighbor_queues_); + if (!create_output_queue_merger()) { + return fatal_error("cannot compute the value to be created / minted / recovered"); + } // 1.4. compute created / minted / recovered if (!init_value_create()) { return fatal_error("cannot compute the value to be created / minted / recovered"); @@ -1861,18 +1875,34 @@ bool Collator::out_msg_queue_cleanup() { } } - auto res = out_msg_queue_->filter([&](vm::CellSlice& cs, td::ConstBitPtr key, int n) -> int { + auto queue_root = out_msg_queue_->get_root_cell(); + if (queue_root.is_null()) { + LOG(DEBUG) << "out_msg_queue is empty"; + return true; + } + // Unwrap UsageCell: don't build proof for visiting output queue (unless something is deleted) + auto r_cell = queue_root->load_cell(); + if (r_cell.is_error()) { + return fatal_error(r_cell.move_as_error()); + } + auto pure_out_msg_queue = + std::make_unique(r_cell.move_as_ok().data_cell, 352, block::tlb::aug_OutMsgQueue); + + int deleted = 0; + bool fail = false; + pure_out_msg_queue->check_for_each([&](Ref value, td::ConstBitPtr key, int n) -> bool { assert(n == 352); + vm::CellSlice& cs = value.write(); // LOG(DEBUG) << "key is " << key.to_hex(n); if (queue_cleanup_timeout_.is_in_past(td::Timestamp::now())) { LOG(WARNING) << "cleaning up outbound queue takes too long, ending"; outq_cleanup_partial_ = true; - return (1 << 30) + 1; // retain all remaining outbound queue entries including this one without processing + return false; // retain all remaining outbound queue entries including this one without processing } if (block_full_) { LOG(WARNING) << "BLOCK FULL while cleaning up outbound queue, cleanup completed only partially"; outq_cleanup_partial_ = true; - return (1 << 30) + 1; // retain all remaining outbound queue entries including this one without processing + return false; // retain all remaining outbound queue entries including this one without processing } block::EnqueuedMsgDescr enq_msg_descr; unsigned long long created_lt; @@ -1881,7 +1911,8 @@ bool Collator::out_msg_queue_cleanup() { && enq_msg_descr.check_key(key) // check key && enq_msg_descr.lt_ == created_lt)) { LOG(ERROR) << "cannot unpack EnqueuedMsg with key " << key.to_hex(n); - return -1; + fail = true; + return false; } LOG(DEBUG) << "scanning outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," << enq_msg_descr.hash_.to_hex() << ") enqueued_lt=" << enq_msg_descr.enqueued_lt_; @@ -1899,20 +1930,30 @@ bool Collator::out_msg_queue_cleanup() { if (delivered) { LOG(DEBUG) << "outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," << enq_msg_descr.hash_.to_hex() << ") enqueued_lt=" << enq_msg_descr.enqueued_lt_ << " has been already delivered, dequeueing"; + // Get value from out_msg_queue_ instead of pure_out_msg_queue (for proof) + auto value2 = out_msg_queue_->lookup_delete_with_extra(key, n); + CHECK(value2.not_null()); + vm::CellSlice& cs2 = value2.write(); + CHECK(cs2.fetch_ulong_bool(64, created_lt) // augmentation + && enq_msg_descr.unpack(cs2) // unpack EnqueuedMsg + && enq_msg_descr.check_key(key) // check key + && enq_msg_descr.lt_ == created_lt); + if (!dequeue_message(std::move(enq_msg_descr.msg_env_), deliver_lt)) { fatal_error(PSTRING() << "cannot dequeue outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," << enq_msg_descr.hash_.to_hex() << ") by inserting a msg_export_deq record"); - return -1; + fail = true; + return false; } register_out_msg_queue_op(); if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) { block_full_ = true; } } - return !delivered; + return true; }); - LOG(DEBUG) << "deleted " << res << " messages from out_msg_queue"; - if (res < 0) { + LOG(DEBUG) << "deleted " << deleted << " messages from out_msg_queue"; + if (fail) { return fatal_error("error scanning/updating OutMsgQueue"); } auto rt = out_msg_queue_->get_root(); @@ -4026,10 +4067,10 @@ bool Collator::create_collated_data() { collated_roots_.push_back(std::move(cell)); } // 2. Message count for neighbors' out queues - if (!neighbor_msg_queues_limits_.is_empty()) { + if (!neighbor_msg_queues_limits_dict_.is_empty()) { vm::CellBuilder cb; cb.store_long(block::gen::t_NeighborMsgQueueLimits.cons_tag[0], 32); - cb.store_maybe_ref(neighbor_msg_queues_limits_.get_root_cell()); + cb.store_maybe_ref(neighbor_msg_queues_limits_dict_.get_root_cell()); collated_roots_.push_back(cb.finalize_novm()); } if (!full_collated_data_) { diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 3329e4fd59..f9e5be2757 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -528,6 +528,7 @@ bool ValidateQuery::extract_collated_data_from(Ref croot, int idx) { return reject_query("invalid NeighborMsgQueueLimits"); } neighbor_msg_queues_limits_ = vm::Dictionary{cs.prefetch_ref(0), 32 + 64}; + return true; } LOG(WARNING) << "collated datum # " << idx << " has unknown type (magic " << cs.prefetch_ulong(32) << "), ignoring"; return true; @@ -4123,7 +4124,14 @@ bool ValidateQuery::check_in_queue() { if (msg_limit < -1) { return reject_query("invalid value in NeighborMsgQueueLimits"); } + LOG(DEBUG) << "Neighbor " << descr.shard().to_str() << " has msg_limit=" << msg_limit; neighbor_queues.emplace_back(descr.top_block_id(), descr.outmsg_root, descr.disabled_, msg_limit); + if (msg_limit != -1 && descr.shard().is_masterchain()) { + return reject_query("masterchain out message queue cannot be limited"); + } + if (msg_limit != -1 && shard_intersects(descr.shard(), shard_)) { + return reject_query("prev block out message queue cannot be limited"); + } } block::OutputQueueMerger nb_out_msgs(shard_, std::move(neighbor_queues)); while (!nb_out_msgs.is_eof()) { From d5a56b7c2bcd57c08a27fedf45b701e96ce86634 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 12 Jul 2023 09:01:15 +0300 Subject: [PATCH 051/524] Changes in CollatorNode --- validator/collator-node.cpp | 225 +++++++++++++++++++++++++----------- validator/collator-node.hpp | 33 +++++- 2 files changed, 187 insertions(+), 71 deletions(-) diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 654e4e7428..516555fae7 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -17,6 +17,8 @@ #include "collator-node.hpp" #include "ton/ton-tl.hpp" #include "fabric.h" +#include "block-auto.h" +#include "block-db.h" namespace ton { @@ -69,33 +71,35 @@ void CollatorNode::del_shard(ShardIdFull shard) { } void CollatorNode::new_masterchain_block_notification(td::Ref state) { - std::vector top_blocks = {state->get_block_id()}; + last_masterchain_block_ = state->get_block_id(); + last_top_blocks_.clear(); + last_top_blocks_[ShardIdFull{masterchainId, shardIdAll}] = last_masterchain_block_; std::vector next_shards; - if (collate_shard(ShardIdFull(masterchainId))) { + if (can_collate_shard(ShardIdFull(masterchainId))) { next_shards.push_back(ShardIdFull(masterchainId)); } for (const auto& desc : state->get_shards()) { - top_blocks.push_back(desc->top_block_id()); + last_top_blocks_[desc->shard()] = desc->top_block_id(); ShardIdFull shard = desc->shard(); if (desc->before_split()) { - if (collate_shard(shard_child(shard, true))) { + if (can_collate_shard(shard_child(shard, true))) { next_shards.push_back(shard_child(shard, true)); } - if (collate_shard(shard_child(shard, false))) { + if (can_collate_shard(shard_child(shard, false))) { next_shards.push_back(shard_child(shard, false)); } } else if (desc->before_merge()) { - if (is_left_child(shard) && collate_shard(shard_parent(shard))) { + if (is_left_child(shard) && can_collate_shard(shard_parent(shard))) { next_shards.push_back(shard_parent(shard)); } - } else if (collate_shard(shard)) { + } else if (can_collate_shard(shard)) { next_shards.push_back(shard); } } for (const ShardIdFull& shard : next_shards) { - for (const BlockIdExt& neighbor : top_blocks) { - if (neighbor.shard_full() != shard && block::ShardConfig::is_neighbor(shard, neighbor.shard_full())) { - td::actor::send_closure(manager_, &ValidatorManager::wait_out_msg_queue_proof, neighbor, shard, 0, + for (const auto& neighbor : last_top_blocks_) { + if (neighbor.first != shard && block::ShardConfig::is_neighbor(shard, neighbor.first)) { + td::actor::send_closure(manager_, &ValidatorManager::wait_out_msg_queue_proof, neighbor.second, shard, 0, td::Timestamp::in(10.0), [](td::Ref) {}); } } @@ -116,88 +120,173 @@ void CollatorNode::new_masterchain_block_notification(td::Ref } } } + // Remove old cache entries + auto it = cache_.begin(); + while (it != cache_.end()) { + auto prev_block_id = std::get<2>(it->first)[0]; + auto top_block = get_shard_top_block(prev_block_id.shard_full()); + if (top_block && top_block.value().seqno() > prev_block_id.seqno()) { + it = cache_.erase(it); + } else { + ++it; + } + } } static td::BufferSlice serialize_error(td::Status error) { return create_serialize_tl_object(error.code(), error.message().c_str()); } +static td::BufferSlice serialize_response(BlockCandidate block) { + return create_serialize_tl_object(create_tl_object( + PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), std::move(block.data), + std::move(block.collated_data))); +} + +static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey creator) { + CHECK(!block.id.is_masterchain()); + if (block.pubkey == creator) { + return block; + } + auto root = vm::std_boc_deserialize(block.data).move_as_ok(); + block::gen::Block::Record blk; + block::gen::BlockExtra::Record extra; + CHECK(tlb::unpack_cell(root, blk)); + CHECK(tlb::unpack_cell(blk.extra, extra)); + extra.created_by = creator.as_bits256(); + CHECK(tlb::pack_cell(blk.extra, extra)); + CHECK(tlb::pack_cell(root, blk)); + block.data = vm::std_boc_serialize(root, 31).move_as_ok(); + + block.id.root_hash = root->get_hash().bits(); + block.id.file_hash = block::compute_file_hash(block.data.as_slice()); + block.pubkey = creator; + return block; +} + void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) { - auto SelfId = actor_id(this); - auto status = [&]() -> td::Status { - if (!validators_.count(src)) { - return td::Status::Error("src is not a validator"); - } - TRY_RESULT(f, fetch_tl_object(std::move(data), true)); - ShardIdFull shard(f->workchain_, f->shard_); - if (!shard.is_valid_ext()) { - return td::Status::Error(PSTRING() << "invalid shard " << shard.to_str()); - } - if (!collate_shard(shard)) { - return td::Status::Error(PSTRING() << "this node doesn't collate shard " << shard.to_str()); + td::Promise new_promise = [promise = std::move(promise), src](td::Result R) mutable { + if (R.is_error()) { + LOG(WARNING) << "Query from " << src << ", error: " << R.error(); + promise.set_result(serialize_error(R.move_as_error())); + } else { + LOG(INFO) << "Query from " << src << ", success"; + promise.set_result(serialize_response(R.move_as_ok())); } - if (f->prev_blocks_.size() != 1 && f->prev_blocks_.size() != 2) { - return td::Status::Error(PSTRING() << "invalid size of prev_blocks: " << f->prev_blocks_.size()); + }; + if (!last_masterchain_block_.is_valid()) { + new_promise.set_error(td::Status::Error("not ready")); + return; + } + if (!validators_.count(src)) { + new_promise.set_error(td::Status::Error("src is not a validator")); + } + TRY_RESULT_PROMISE(new_promise, f, fetch_tl_object(std::move(data), true)); + ShardIdFull shard(f->workchain_, f->shard_); + BlockIdExt min_mc_id = create_block_id(f->min_mc_id_); + LOG(INFO) << "Got query from " << src << ": shard=" << shard.to_str() << ", min_mc_seqno=" << min_mc_id.seqno(); + if (!shard.is_valid_ext()) { + new_promise.set_error(td::Status::Error(PSTRING() << "invalid shard " << shard.to_str())); + return; + } + if (!can_collate_shard(shard)) { + new_promise.set_error(td::Status::Error(PSTRING() << "this node doesn't collate shard " << shard.to_str())); + return; + } + if (f->prev_blocks_.size() != 1 && f->prev_blocks_.size() != 2) { + new_promise.set_error(td::Status::Error(PSTRING() << "invalid size of prev_blocks: " << f->prev_blocks_.size())); + return; + } + if (!min_mc_id.is_masterchain_ext()) { + new_promise.set_error(td::Status::Error("min_mc_id is not form masterchain")); + return; + } + std::vector prev_blocks; + for (const auto& b : f->prev_blocks_) { + auto id = create_block_id(b); + if (!id.is_valid_full()) { + new_promise.set_error(td::Status::Error("invalid prev_block")); + return; } - std::vector prev_blocks; - for (const auto& b : f->prev_blocks_) { - prev_blocks.push_back(create_block_id(b)); + auto top_block = get_shard_top_block(id.shard_full()); + if (top_block && top_block.value().seqno() > id.seqno()) { + new_promise.set_error(td::Status::Error("cannot collate block: already exists in blockchain")); + return; } - BlockIdExt min_mc_id = create_block_id(f->min_mc_id_); - Ed25519_PublicKey creator(f->creator_); - - LOG(INFO) << "Query from " << src << ": shard=" << shard.to_str() << ", min_mc_id=" << min_mc_id.to_str(); - - auto P = td::PromiseCreator::lambda([=, prev_blocks = std::move(prev_blocks), - promise = std::move(promise)](td::Result> R) mutable { + prev_blocks.push_back(id); + } + Ed25519_PublicKey creator(f->creator_); + if (!shard.is_masterchain()) { + // Collation of masterchain cannot be cached because changing "created_by" in masterchain is hard + // It does not really matter because validators can collate masterchain themselves + new_promise = [promise = std::move(new_promise), creator](td::Result R) mutable { if (R.is_error()) { - LOG(WARNING) << "Query from " << src << ", error: " << R.error(); - promise.set_result(serialize_error(R.move_as_error_prefix("failed to get masterchain state: "))); + promise.set_error(R.move_as_error()); } else { - td::Ref state(R.move_as_ok()); - if (state.is_null()) { - LOG(WARNING) << "Query from " << src << ", error: failed to get masterchain state"; - promise.set_result(serialize_error(R.move_as_error_prefix("failed to get masterchain state: "))); - return; - } - td::actor::send_closure(SelfId, &CollatorNode::receive_query_cont, src, shard, std::move(state), - std::move(prev_blocks), creator, std::move(promise)); + promise.set_result(change_creator(R.move_as_ok(), creator)); } - }); - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, min_mc_id, 1, td::Timestamp::in(5.0), - std::move(P)); - return td::Status::OK(); - }(); - if (status.is_error()) { - LOG(WARNING) << "Query from " << src << ", error: " << status; - promise.set_result(serialize_error(std::move(status))); + }; + auto cache_key = std::make_tuple(min_mc_id.seqno(), shard, prev_blocks); + auto cache_entry = cache_[cache_key]; + if (cache_entry == nullptr) { + cache_entry = cache_[cache_key] = std::make_shared(); + } + if (cache_entry->result) { + new_promise.set_result(cache_entry->result.value().clone()); + return; + } + cache_entry->promises.push_back(std::move(new_promise)); + if (cache_entry->started) { + return; + } + cache_entry->started = true; + new_promise = [SelfId = actor_id(this), cache_entry](td::Result R) mutable { + td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); + }; } -} -void CollatorNode::receive_query_cont(adnl::AdnlNodeIdShort src, ShardIdFull shard, - td::Ref min_mc_state, std::vector prev_blocks, - Ed25519_PublicKey creator, td::Promise promise) { - auto P = td::PromiseCreator::lambda([promise = std::move(promise), src](td::Result R) mutable { + auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), prev_blocks = std::move(prev_blocks), + promise = std::move(new_promise)](td::Result> R) mutable { if (R.is_error()) { - LOG(WARNING) << "Query from " << src << ", error: " << R.error(); - promise.set_result(serialize_error(R.move_as_error())); + promise.set_error(R.move_as_error_prefix("failed to get masterchain state: ")); } else { - LOG(INFO) << "Query from " << src << ", success"; - auto block = R.move_as_ok(); - auto result = create_serialize_tl_object( - create_tl_object(PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), - create_tl_block_id(block.id), std::move(block.data), - std::move(block.collated_data))); - promise.set_result(std::move(result)); + td::Ref state(R.move_as_ok()); + if (state.is_null()) { + promise.set_error(R.move_as_error_prefix("failed to get masterchain state: ")); + return; + } + td::actor::send_closure(SelfId, &CollatorNode::receive_query_cont, shard, std::move(state), + std::move(prev_blocks), creator, std::move(promise)); } }); + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, min_mc_id, 1, td::Timestamp::in(5.0), + std::move(P)); +} +void CollatorNode::receive_query_cont(ShardIdFull shard, td::Ref min_mc_state, + std::vector prev_blocks, Ed25519_PublicKey creator, + td::Promise promise) { run_collate_query(shard, min_mc_state->get_block_id(), std::move(prev_blocks), creator, - min_mc_state->get_validator_set(shard), manager_, td::Timestamp::in(10.0), std::move(P)); + min_mc_state->get_validator_set(shard), manager_, td::Timestamp::in(10.0), std::move(promise)); +} + +void CollatorNode::process_result(std::shared_ptr cache_entry, td::Result R) { + if (R.is_error()) { + cache_entry->started = false; + for (auto& p : cache_entry->promises) { + p.set_error(R.error().clone()); + } + } else { + cache_entry->result = R.move_as_ok(); + for (auto& p : cache_entry->promises) { + p.set_result(cache_entry->result.value().clone()); + } + } + cache_entry->promises.clear(); } -bool CollatorNode::collate_shard(ShardIdFull shard) const { +bool CollatorNode::can_collate_shard(ShardIdFull shard) const { for (ShardIdFull our_shard : shards_) { if (shard_intersects(shard, our_shard)) { return true; diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index 46848fcfef..d640244a7d 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -18,6 +18,7 @@ #include "interfaces/validator-manager.h" #include "rldp/rldp.h" +#include namespace ton { @@ -38,11 +39,11 @@ class CollatorNode : public td::actor::Actor { private: void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); - void receive_query_cont(adnl::AdnlNodeIdShort src, ShardIdFull shard, td::Ref min_mc_state, + void receive_query_cont(ShardIdFull shard, td::Ref min_mc_state, std::vector prev_blocks, Ed25519_PublicKey creator, - td::Promise promise); + td::Promise promise); - bool collate_shard(ShardIdFull shard) const; + bool can_collate_shard(ShardIdFull shard) const; adnl::AdnlNodeIdShort local_id_; td::actor::ActorId manager_; @@ -50,6 +51,32 @@ class CollatorNode : public td::actor::Actor { td::actor::ActorId rldp_; std::vector shards_; std::set validators_; + + BlockIdExt last_masterchain_block_{}; + std::map last_top_blocks_; + + struct CacheEntry { + bool started = false; + td::optional result; + std::vector> promises; + }; + std::map>, std::shared_ptr> cache_; + + td::optional get_shard_top_block(ShardIdFull shard) const { + auto it = last_top_blocks_.lower_bound(shard); + if (it != last_top_blocks_.end() && shard_intersects(it->first, shard)) { + return it->second; + } + if (it != last_top_blocks_.begin()) { + --it; + if (shard_intersects(it->first, shard)) { + return it->second; + } + } + return {}; + } + + void process_result(std::shared_ptr cache_entry, td::Result R); }; } // namespace validator From 21ce145af2aeb628c154ac5d805ac5f5b7c20941 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 12 Jul 2023 13:29:59 +0300 Subject: [PATCH 052/524] Limit collated data size in collator --- crypto/block/block.cpp | 23 ++++++++++++++++------- crypto/block/block.h | 7 +++++-- crypto/vm/boc.cpp | 31 +++++++++++++++++++++++++++++++ crypto/vm/boc.h | 13 +++++++++++++ crypto/vm/cells/CellUsageTree.cpp | 12 +++++++++--- crypto/vm/cells/CellUsageTree.h | 13 +++++++++++-- crypto/vm/cells/MerkleProof.h | 4 ++++ crypto/vm/cells/UsageCell.h | 2 +- validator/impl/collator.cpp | 28 ++++++++++++++++++++++++++-- 9 files changed, 116 insertions(+), 17 deletions(-) diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index 1131213c69..b9ecfe60ca 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -701,12 +701,19 @@ int BlockLimits::classify_lt(ton::LogicalTime lt) const { return lt_delta.classify(lt - start_lt); } -int BlockLimits::classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt) const { - return std::max(std::max(classify_size(size), classify_gas(gas)), classify_lt(lt)); +int BlockLimits::classify_collated_data_size(td::uint64 size) const { + return bytes.classify(size); // TODO: Maybe separate limits in config } -bool BlockLimits::fits(unsigned cls, td::uint64 size, td::uint64 gas_value, ton::LogicalTime lt) const { - return bytes.fits(cls, size) && gas.fits(cls, gas_value) && lt_delta.fits(cls, lt - start_lt); +int BlockLimits::classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt, td::uint64 collated_size) const { + return std::max( + {classify_size(size), classify_gas(gas), classify_lt(lt), classify_collated_data_size(collated_size)}); +} + +bool BlockLimits::fits(unsigned cls, td::uint64 size, td::uint64 gas_value, ton::LogicalTime lt, + td::uint64 collated_size) const { + return bytes.fits(cls, size) && gas.fits(cls, gas_value) && lt_delta.fits(cls, lt - start_lt) && + bytes.fits(cls, collated_size); } td::uint64 BlockLimitStatus::estimate_block_size(const vm::NewCellStorageStat::Stat* extra) const { @@ -719,20 +726,22 @@ td::uint64 BlockLimitStatus::estimate_block_size(const vm::NewCellStorageStat::S } int BlockLimitStatus::classify() const { - return limits.classify(estimate_block_size(), gas_used, cur_lt); + return limits.classify(estimate_block_size(), gas_used, cur_lt, collated_data_stat.estimate_proof_size()); } bool BlockLimitStatus::fits(unsigned cls) const { return cls >= ParamLimits::limits_cnt || (limits.gas.fits(cls, gas_used) && limits.lt_delta.fits(cls, cur_lt - limits.start_lt) && - limits.bytes.fits(cls, estimate_block_size())); + limits.bytes.fits(cls, estimate_block_size()) && + limits.bytes.fits(cls, collated_data_stat.estimate_proof_size())); } bool BlockLimitStatus::would_fit(unsigned cls, ton::LogicalTime end_lt, td::uint64 more_gas, const vm::NewCellStorageStat::Stat* extra) const { return cls >= ParamLimits::limits_cnt || (limits.gas.fits(cls, gas_used + more_gas) && limits.lt_delta.fits(cls, std::max(cur_lt, end_lt) - limits.start_lt) && - limits.bytes.fits(cls, estimate_block_size(extra))); + limits.bytes.fits(cls, estimate_block_size(extra)) && + limits.bytes.fits(cls, collated_data_stat.estimate_proof_size())); } // SETS: account_dict, shard_libraries_, mc_state_extra diff --git a/crypto/block/block.h b/crypto/block/block.h index 19d99e6a0d..b6b46a3d6a 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -252,8 +252,9 @@ struct BlockLimits { int classify_size(td::uint64 size) const; int classify_gas(td::uint64 gas) const; int classify_lt(ton::LogicalTime lt) const; - int classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt) const; - bool fits(unsigned cls, td::uint64 size, td::uint64 gas, ton::LogicalTime lt) const; + int classify_collated_data_size(td::uint64 size) const; + int classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt, td::uint64 collated_size) const; + bool fits(unsigned cls, td::uint64 size, td::uint64 gas, ton::LogicalTime lt, td::uint64 collated_size) const; }; struct BlockLimitStatus { @@ -262,6 +263,7 @@ struct BlockLimitStatus { td::uint64 gas_used{}; vm::NewCellStorageStat st_stat; unsigned accounts{}, transactions{}, extra_out_msgs{}; + vm::ProofStorageStat collated_data_stat; BlockLimitStatus(const BlockLimits& limits_, ton::LogicalTime lt = 0) : limits(limits_), cur_lt(std::max(limits_.start_lt, lt)) { } @@ -271,6 +273,7 @@ struct BlockLimitStatus { transactions = accounts = 0; gas_used = 0; extra_out_msgs = 0; + collated_data_stat = {}; } td::uint64 estimate_block_size(const vm::NewCellStorageStat::Stat* extra = nullptr) const; int classify() const; diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 11583ede6e..0d84c1189e 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -1214,4 +1214,35 @@ bool VmStorageStat::add_storage(const CellSlice& cs) { return true; } +static td::uint64 estimate_prunned_size() { + return 41; +} + +static td::uint64 estimate_serialized_size(const Ref& cell) { + return cell->get_serialized_size() + cell->size_refs() * 3 + 3; +} + +void ProofStorageStat::add_cell(const Ref& cell) { + auto& status = cells_[cell->get_hash()]; + if (status == c_loaded) { + return; + } + if (status == c_prunned) { + proof_size_ -= estimate_prunned_size(); + } + status = c_loaded; + proof_size_ += estimate_serialized_size(cell); + for (unsigned i = 0; i < cell->size_refs(); ++i) { + auto& child_status = cells_[cell->get_ref(i)->get_hash()]; + if (child_status == c_none) { + child_status = c_prunned; + proof_size_ += estimate_prunned_size(); + } + } +} + +td::uint64 ProofStorageStat::estimate_proof_size() const { + return proof_size_; +} + } // namespace vm diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index c7a1810d71..d6a7f9adcc 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -41,6 +41,7 @@ class NewCellStorageStat { Stat(td::uint64 cells_, td::uint64 bits_, td::uint64 internal_refs_ = 0, td::uint64 external_refs_ = 0) : cells(cells_), bits(bits_), internal_refs(internal_refs_), external_refs(external_refs_) { } + Stat(const Stat&) = default; td::uint64 cells{0}; td::uint64 bits{0}; td::uint64 internal_refs{0}; @@ -160,6 +161,18 @@ struct VmStorageStat { } }; +class ProofStorageStat { + public: + void add_cell(const Ref& cell); + td::uint64 estimate_proof_size() const; + private: + enum CellStatus { + c_none = 0, c_prunned = 1, c_loaded = 2 + }; + std::map cells_; + td::uint64 proof_size_ = 0; +}; + struct CellSerializationInfo { bool special; Cell::LevelMask level_mask; diff --git a/crypto/vm/cells/CellUsageTree.cpp b/crypto/vm/cells/CellUsageTree.cpp index 3f43ec6bd8..410b3fcd64 100644 --- a/crypto/vm/cells/CellUsageTree.cpp +++ b/crypto/vm/cells/CellUsageTree.cpp @@ -22,12 +22,12 @@ namespace vm { // // CellUsageTree::NodePtr // -bool CellUsageTree::NodePtr::on_load() const { +bool CellUsageTree::NodePtr::on_load(const td::Ref& cell) const { auto tree = tree_weak_.lock(); if (!tree) { return false; } - tree->on_load(node_id_); + tree->on_load(node_id_, cell); return true; } @@ -111,8 +111,14 @@ void CellUsageTree::set_use_mark_for_is_loaded(bool use_mark) { use_mark_ = use_mark; } -void CellUsageTree::on_load(NodeId node_id) { +void CellUsageTree::on_load(NodeId node_id, const td::Ref& cell) { + if (nodes_[node_id].is_loaded) { + return; + } nodes_[node_id].is_loaded = true; + if (cell_load_callback_) { + cell_load_callback_(cell); + } } CellUsageTree::NodeId CellUsageTree::create_child(NodeId node_id, unsigned ref_id) { diff --git a/crypto/vm/cells/CellUsageTree.h b/crypto/vm/cells/CellUsageTree.h index 150dd2bda8..af0f21f534 100644 --- a/crypto/vm/cells/CellUsageTree.h +++ b/crypto/vm/cells/CellUsageTree.h @@ -22,8 +22,12 @@ #include "td/utils/int_types.h" #include "td/utils/logging.h" +#include namespace vm { + +class DataCell; + class CellUsageTree : public std::enable_shared_from_this { public: using NodeId = td::uint32; @@ -38,7 +42,7 @@ class CellUsageTree : public std::enable_shared_from_this { return node_id_ == 0 || tree_weak_.expired(); } - bool on_load() const; + bool on_load(const td::Ref& cell) const; NodePtr create_child(unsigned ref_id) const; bool mark_path(CellUsageTree* master_tree) const; bool is_from_tree(const CellUsageTree* master_tree) const; @@ -59,6 +63,10 @@ class CellUsageTree : public std::enable_shared_from_this { void set_use_mark_for_is_loaded(bool use_mark = true); NodeId create_child(NodeId node_id, unsigned ref_id); + void set_cell_load_callback(std::function&)> f) { + cell_load_callback_ = std::move(f); + } + private: struct Node { bool is_loaded{false}; @@ -68,8 +76,9 @@ class CellUsageTree : public std::enable_shared_from_this { }; bool use_mark_{false}; std::vector nodes_{2}; + std::function&)> cell_load_callback_; - void on_load(NodeId node_id); + void on_load(NodeId node_id, const td::Ref& cell); NodeId create_node(NodeId parent); }; } // namespace vm diff --git a/crypto/vm/cells/MerkleProof.h b/crypto/vm/cells/MerkleProof.h index 9c50fd078f..fc2cb6ebde 100644 --- a/crypto/vm/cells/MerkleProof.h +++ b/crypto/vm/cells/MerkleProof.h @@ -66,6 +66,10 @@ class MerkleProofBuilder { td::Result> extract_proof() const; bool extract_proof_to(Ref &proof_root) const; td::Result extract_proof_boc() const; + + void set_cell_load_callback(std::function&)> f) { + usage_tree->set_cell_load_callback(std::move(f)); + } }; } // namespace vm diff --git a/crypto/vm/cells/UsageCell.h b/crypto/vm/cells/UsageCell.h index bf15bb56fd..3e6e88982a 100644 --- a/crypto/vm/cells/UsageCell.h +++ b/crypto/vm/cells/UsageCell.h @@ -39,7 +39,7 @@ class UsageCell : public Cell { // load interface td::Result load_cell() const override { TRY_RESULT(loaded_cell, cell_->load_cell()); - if (tree_node_.on_load()) { + if (tree_node_.on_load(loaded_cell.data_cell)) { CHECK(loaded_cell.tree_node.empty()); loaded_cell.tree_node = tree_node_; } diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 1ce0a4b4c3..fc31d2645c 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -661,6 +661,13 @@ void Collator::got_neighbor_msg_queue(unsigned i, td::Resultstate_root_}); state_root = neighbor_proof_builders_.back().root(); + if (full_collated_data_ && !block_id.is_masterchain()) { + neighbor_proof_builders_.back().set_cell_load_callback([&](const td::Ref& cell) { + if (block_limit_status_) { + block_limit_status_->collated_data_stat.add_cell(cell); + } + }); + } } auto state = ShardStateQ::fetch(block_id, {}, state_root); if (state.is_error()) { @@ -745,6 +752,13 @@ bool Collator::unpack_merge_last_state() { } // 1. prepare for creating a MerkleUpdate based on previous state state_usage_tree_ = std::make_shared(); + if (full_collated_data_ && !is_masterchain()) { + state_usage_tree_->set_cell_load_callback([&](const td::Ref& cell) { + if (block_limit_status_) { + block_limit_status_->collated_data_stat.add_cell(cell); + } + }); + } prev_state_root_ = vm::UsageCell::create(prev_state_root_pure_, state_usage_tree_->root_ptr()); // 2. extract back slightly virtualized roots of the two original states Ref root0, root1; @@ -783,6 +797,13 @@ bool Collator::unpack_last_state() { prev_state_root_pure_ = prev_states.at(0)->root_cell(); // prepare for creating a MerkleUpdate based on previous state state_usage_tree_ = std::make_shared(); + if (full_collated_data_ && !is_masterchain()) { + state_usage_tree_->set_cell_load_callback([&](const td::Ref& cell) { + if (block_limit_status_) { + block_limit_status_->collated_data_stat.add_cell(cell); + } + }); + } prev_state_root_ = vm::UsageCell::create(prev_state_root_pure_, state_usage_tree_->root_ptr()); // unpack previous state block::ShardState ss; @@ -3547,7 +3568,8 @@ bool Collator::check_block_overload() { block_size_estimate_ = block_limit_status_->estimate_block_size(); LOG(INFO) << "block load statistics: gas=" << block_limit_status_->gas_used << " lt_delta=" << block_limit_status_->cur_lt - block_limit_status_->limits.start_lt - << " size_estimate=" << block_size_estimate_; + << " size_estimate=" << block_size_estimate_ + << " collated_size_estimate=" << block_limit_status_->collated_data_stat.estimate_proof_size(); auto cl = block_limit_status_->classify(); if (cl <= block::ParamLimits::cl_underload) { underload_history_ |= 1; @@ -4153,11 +4175,13 @@ bool Collator::create_block_candidate() { cdata_slice = cdata_res.move_as_ok(); } LOG(INFO) << "serialized block size " << blk_slice.size() << " bytes (preliminary estimate was " - << block_size_estimate_ << "), collated data " << cdata_slice.size() << " bytes"; + << block_size_estimate_ << ")"; auto st = block_limit_status_->st_stat.get_total_stat(); LOG(INFO) << "size regression stats: " << blk_slice.size() << " " << st.cells << " " << st.bits << " " << st.internal_refs << " " << st.external_refs << " " << block_limit_status_->accounts << " " << block_limit_status_->transactions; + LOG(INFO) << "serialized collated data size " << cdata_slice.size() << " bytes (preliminary estimate was " + << block_limit_status_->collated_data_stat.estimate_proof_size() << ")"; // 3. create a BlockCandidate block_candidate = std::make_unique( created_by_, From d115807f6e4d5fc67481a3f9ccb40c19bacb07d0 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 14 Jul 2023 13:13:00 +0300 Subject: [PATCH 053/524] Move ExtClientLazy and QueryTraits to lite-client-common --- lite-client/CMakeLists.txt | 4 +-- {tonlib/tonlib => lite-client}/QueryTraits.h | 8 ++++- .../ext-client.cpp | 35 +++++++++---------- .../ext-client.h | 24 +++++++------ tonlib/CMakeLists.txt | 2 -- tonlib/tonlib/Config.h | 8 ++--- tonlib/tonlib/ExtClient.cpp | 2 +- tonlib/tonlib/ExtClient.h | 10 +++--- tonlib/tonlib/ExtClientOutbound.h | 5 ++- tonlib/tonlib/TonlibClient.cpp | 5 ++- tonlib/tonlib/TonlibClient.h | 3 +- tonlib/tonlib/tonlib-cli.cpp | 11 +++--- 12 files changed, 57 insertions(+), 60 deletions(-) rename {tonlib/tonlib => lite-client}/QueryTraits.h (97%) rename tonlib/tonlib/ExtClientLazy.cpp => lite-client/ext-client.cpp (84%) rename tonlib/tonlib/ExtClientLazy.h => lite-client/ext-client.h (67%) diff --git a/lite-client/CMakeLists.txt b/lite-client/CMakeLists.txt index b84495162f..fd5c812cff 100644 --- a/lite-client/CMakeLists.txt +++ b/lite-client/CMakeLists.txt @@ -1,9 +1,9 @@ cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) -add_library(lite-client-common lite-client-common.cpp lite-client-common.h) +add_library(lite-client-common lite-client-common.cpp lite-client-common.h ext-client.cpp ext-client.h QueryTraits.h) target_link_libraries(lite-client-common PUBLIC tdutils tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto ton_block) -add_executable(lite-client lite-client.cpp lite-client.h) +add_executable(lite-client lite-client.cpp lite-client.h ext-client.h ext-client.cpp) target_link_libraries(lite-client tdutils tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto ton_block terminal lite-client-common git) diff --git a/tonlib/tonlib/QueryTraits.h b/lite-client/QueryTraits.h similarity index 97% rename from tonlib/tonlib/QueryTraits.h rename to lite-client/QueryTraits.h index b583cd05b2..ffa683e629 100644 --- a/tonlib/tonlib/QueryTraits.h +++ b/lite-client/QueryTraits.h @@ -17,12 +17,14 @@ #pragma once #include "ton/ton-types.h" #include "auto/tl/lite_api.h" +#include "auto/tl/lite_api.hpp" #include "vm/boc.h" #include "vm/cellslice.h" #include "block/block-auto.h" #include "block/block-parse.h" +#include "auto/tl/lite_api.hpp" -namespace tonlib { +namespace liteclient { template struct QueryTraits { @@ -212,5 +214,9 @@ struct QueryTraits { } }; +template +inline ton::ShardIdFull get_query_shard(const Query& q) { + return QueryTraits::get_shard(q); +} } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientLazy.cpp b/lite-client/ext-client.cpp similarity index 84% rename from tonlib/tonlib/ExtClientLazy.cpp rename to lite-client/ext-client.cpp index 027198de4a..e79ae9a4fb 100644 --- a/tonlib/tonlib/ExtClientLazy.cpp +++ b/lite-client/ext-client.cpp @@ -13,20 +13,17 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP */ -#include "ExtClientLazy.h" -#include "TonlibError.h" +#include "ext-client.h" #include "td/utils/Random.h" #include "ton/ton-shard.h" #include -namespace tonlib { +namespace liteclient { -class ExtClientLazyImpl : public ExtClientLazy { +class ExtClientImpl : public ExtClient { public: - ExtClientLazyImpl(std::vector servers, td::unique_ptr callback) + ExtClientImpl(std::vector servers, td::unique_ptr callback) : callback_(std::move(callback)) { CHECK(!servers.empty()); servers_.resize(servers.size()); @@ -53,10 +50,10 @@ class ExtClientLazyImpl : public ExtClientLazy { CHECK(!server.client.empty()); alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); td::Promise P = [SelfId = actor_id(this), server_idx, - promise = std::move(promise)](td::Result R) mutable { + promise = std::move(promise)](td::Result R) mutable { if (R.is_error() && (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { - td::actor::send_closure(SelfId, &ExtClientLazyImpl::set_server_bad, server_idx); + td::actor::send_closure(SelfId, &ExtClientImpl::set_server_bad, server_idx); } promise.set_result(std::move(R)); }; @@ -128,17 +125,17 @@ class ExtClientLazyImpl : public ExtClientLazy { class Callback : public ton::adnl::AdnlExtClient::Callback { public: - explicit Callback(td::actor::ActorShared parent, size_t idx) + explicit Callback(td::actor::ActorShared parent, size_t idx) : parent_(std::move(parent)), idx_(idx) { } void on_ready() override { } void on_stop_ready() override { - td::actor::send_closure(parent_, &ExtClientLazyImpl::set_server_bad, idx_); + td::actor::send_closure(parent_, &ExtClientImpl::set_server_bad, idx_); } private: - td::actor::ActorShared parent_; + td::actor::ActorShared parent_; size_t idx_; }; ref_cnt_++; @@ -154,7 +151,7 @@ class ExtClientLazyImpl : public ExtClientLazy { } struct Server { - Config::LiteServer s; + LiteServer s; td::actor::ActorOwn client; td::Timestamp timeout = td::Timestamp::never(); td::Timestamp ignore_until = td::Timestamp::never(); @@ -169,7 +166,7 @@ class ExtClientLazyImpl : public ExtClientLazy { std::map shard_to_server_; int max_server_shard_depth_ = 0; - td::unique_ptr callback_; + td::unique_ptr callback_; static constexpr double MAX_NO_QUERIES_TIMEOUT = 100; bool is_closing_{false}; @@ -206,13 +203,13 @@ class ExtClientLazyImpl : public ExtClientLazy { } }; -td::actor::ActorOwn ExtClientLazy::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, +td::actor::ActorOwn ExtClient::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, td::unique_ptr callback) { - return create({Config::LiteServer{dst, dst_addr, true, {}}}, std::move(callback)); + return create({LiteServer{dst, dst_addr, true, {}}}, std::move(callback)); } -td::actor::ActorOwn ExtClientLazy::create(std::vector servers, +td::actor::ActorOwn ExtClient::create(std::vector servers, td::unique_ptr callback) { - return td::actor::create_actor("ExtClientLazy", std::move(servers), std::move(callback)); + return td::actor::create_actor("ExtClient", std::move(servers), std::move(callback)); } -} // namespace tonlib +} // namespace liteclient diff --git a/tonlib/tonlib/ExtClientLazy.h b/lite-client/ext-client.h similarity index 67% rename from tonlib/tonlib/ExtClientLazy.h rename to lite-client/ext-client.h index 6383cec55b..c8569964f9 100644 --- a/tonlib/tonlib/ExtClientLazy.h +++ b/lite-client/ext-client.h @@ -13,18 +13,22 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP */ #pragma once #include "td/actor/actor.h" #include "ton/ton-types.h" #include "adnl/adnl-ext-client.h" -#include "Config.h" -namespace tonlib { -class ExtClientLazy : public td::actor::Actor { +namespace liteclient { +class ExtClient : public td::actor::Actor { public: + struct LiteServer { + ton::adnl::AdnlNodeIdFull adnl_id; + td::IPAddress address; + bool is_full = true; + std::vector shards; + }; + class Callback { public: virtual ~Callback() { @@ -35,10 +39,8 @@ class ExtClientLazy : public td::actor::Actor { td::Promise promise) = 0; virtual void force_change_liteserver() = 0; - static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, - td::unique_ptr callback); - static td::actor::ActorOwn create(std::vector servers, - td::unique_ptr callback); + static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + td::unique_ptr callback); + static td::actor::ActorOwn create(std::vector servers, td::unique_ptr callback); }; - -} // namespace tonlib +} // namespace liteclient diff --git a/tonlib/CMakeLists.txt b/tonlib/CMakeLists.txt index 72c59c4da5..8ee8fcc697 100644 --- a/tonlib/CMakeLists.txt +++ b/tonlib/CMakeLists.txt @@ -10,7 +10,6 @@ set(TONLIB_SOURCE tonlib/Client.cpp tonlib/Config.cpp tonlib/ExtClient.cpp - tonlib/ExtClientLazy.cpp tonlib/ExtClientOutbound.cpp tonlib/KeyStorage.cpp tonlib/KeyValue.cpp @@ -25,7 +24,6 @@ set(TONLIB_SOURCE tonlib/Client.h tonlib/Config.h tonlib/ExtClient.h - tonlib/ExtClientLazy.h tonlib/ExtClientOutbound.h tonlib/KeyStorage.h tonlib/KeyValue.h diff --git a/tonlib/tonlib/Config.h b/tonlib/tonlib/Config.h index 7af1b1c5aa..5ae304ea85 100644 --- a/tonlib/tonlib/Config.h +++ b/tonlib/tonlib/Config.h @@ -20,15 +20,11 @@ #include "adnl/adnl-node-id.hpp" #include "td/utils/port/IPAddress.h" #include "ton/ton-types.h" +#include "lite-client/ext-client.h" namespace tonlib { struct Config { - struct LiteServer { - ton::adnl::AdnlNodeIdFull adnl_id; - td::IPAddress address; - bool is_full = true; - std::vector shards; - }; + using LiteServer = liteclient::ExtClient::LiteServer; ton::BlockIdExt zero_state_id; ton::BlockIdExt init_block_id; std::vector hardforks; diff --git a/tonlib/tonlib/ExtClient.cpp b/tonlib/tonlib/ExtClient.cpp index 755be170d0..f0c84d6163 100644 --- a/tonlib/tonlib/ExtClient.cpp +++ b/tonlib/tonlib/ExtClient.cpp @@ -65,7 +65,7 @@ void ExtClient::send_raw_query(td::BufferSlice query, ton::ShardIdFull shard, td if (client_.adnl_ext_client_.empty()) { return P.set_error(TonlibError::NoLiteServers()); } - td::actor::send_closure(client_.adnl_ext_client_, &ExtClientLazy::send_query, "query", std::move(query), + td::actor::send_closure(client_.adnl_ext_client_, &liteclient::ExtClient::send_query, "query", std::move(query), shard, td::Timestamp::in(10.0), std::move(P)); } } // namespace tonlib diff --git a/tonlib/tonlib/ExtClient.h b/tonlib/tonlib/ExtClient.h index 5cd7f4fd69..9db040bb17 100644 --- a/tonlib/tonlib/ExtClient.h +++ b/tonlib/tonlib/ExtClient.h @@ -28,10 +28,10 @@ #include "td/utils/Container.h" #include "td/utils/Random.h" -#include "ExtClientLazy.h" +#include "lite-client/ext-client.h" #include "TonlibError.h" #include "utils.h" -#include "QueryTraits.h" +#include "lite-client/QueryTraits.h" namespace tonlib { class LastBlock; @@ -39,7 +39,7 @@ class LastConfig; struct LastBlockState; struct LastConfigState; struct ExtClientRef { - td::actor::ActorId adnl_ext_client_; + td::actor::ActorId adnl_ext_client_; td::actor::ActorId last_block_actor_; td::actor::ActorId last_config_actor_; }; @@ -65,7 +65,7 @@ class ExtClient { template void send_query(QueryT query, td::Promise promise, td::int32 seq_no = -1) { - ton::ShardIdFull shard = QueryTraits::get_shard(query); + ton::ShardIdFull shard = liteclient::QueryTraits::get_shard(query); auto raw_query = ton::serialize_tl_object(&query, true); td::uint32 tag = td::Random::fast_uint32(); VLOG(lite_server) << "send query to liteserver: " << tag << " " << to_string(query); @@ -99,7 +99,7 @@ class ExtClient { void force_change_liteserver() { if (!client_.adnl_ext_client_.empty()) { - td::actor::send_closure(client_.adnl_ext_client_, &ExtClientLazy::force_change_liteserver); + td::actor::send_closure(client_.adnl_ext_client_, &liteclient::ExtClient::force_change_liteserver); } } diff --git a/tonlib/tonlib/ExtClientOutbound.h b/tonlib/tonlib/ExtClientOutbound.h index 31f76b9773..6eb6aa983b 100644 --- a/tonlib/tonlib/ExtClientOutbound.h +++ b/tonlib/tonlib/ExtClientOutbound.h @@ -18,11 +18,10 @@ */ #pragma once #include "td/actor/actor.h" - -#include "ExtClientLazy.h" +#include "lite-client/ext-client.h" namespace tonlib { -class ExtClientOutbound : public ExtClientLazy { +class ExtClientOutbound : public liteclient::ExtClient { public: class Callback { public: diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 1682c78030..3bb3f8a216 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -18,7 +18,6 @@ */ #include "TonlibClient.h" -#include "tonlib/ExtClientLazy.h" #include "tonlib/ExtClientOutbound.h" #include "tonlib/LastBlock.h" #include "tonlib/LastConfig.h" @@ -2022,7 +2021,7 @@ void TonlibClient::init_ext_client() { ext_client_outbound_ = client.get(); raw_client_ = std::move(client); } else { - class Callback : public ExtClientLazy::Callback { + class Callback : public liteclient::ExtClient::Callback { public: explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { } @@ -2032,7 +2031,7 @@ void TonlibClient::init_ext_client() { }; ext_client_outbound_ = {}; ref_cnt_++; - raw_client_ = ExtClientLazy::create(config_.lite_servers, td::make_unique(td::actor::actor_shared())); + raw_client_ = liteclient::ExtClient::create(config_.lite_servers, td::make_unique(td::actor::actor_shared())); } } diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index e31653693a..b1d649efec 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -33,6 +33,7 @@ #include "td/utils/optional.h" #include "smc-envelope/ManualDns.h" +#include "lite-client/ext-client.h" #include @@ -112,7 +113,7 @@ class TonlibClient : public td::actor::Actor { vm::Dictionary libraries{256}; // network - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; td::actor::ActorId ext_client_outbound_; td::actor::ActorOwn raw_last_block_; td::actor::ActorOwn raw_last_config_; diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 92749d089b..8669dda32d 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -47,8 +47,6 @@ #include "tonlib/TonlibClient.h" #include "tonlib/TonlibCallback.h" -#include "tonlib/ExtClientLazy.h" - #include "smc-envelope/ManualDns.h" #include "smc-envelope/PaymentChannel.h" @@ -57,6 +55,7 @@ #include "crypto/util/Miner.h" #include "vm/boc.h" #include "vm/cells/CellBuilder.h" +#include "lite-client/ext-client.h" #include #include @@ -174,7 +173,7 @@ class TonlibCli : public td::actor::Actor { std::map>> query_handlers_; - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; bool is_closing_{false}; td::uint32 ref_cnt_{1}; @@ -223,7 +222,7 @@ class TonlibCli : public td::actor::Actor { if (options_.use_callbacks_for_network) { auto config = tonlib::Config::parse(options_.config).move_as_ok(); - class Callback : public tonlib::ExtClientLazy::Callback { + class Callback : public liteclient::ExtClient::Callback { public: explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { } @@ -232,7 +231,7 @@ class TonlibCli : public td::actor::Actor { td::actor::ActorShared<> parent_; }; ref_cnt_++; - raw_client_ = tonlib::ExtClientLazy::create(config.lite_servers, + raw_client_ = liteclient::ExtClient::create(config.lite_servers, td::make_unique(td::actor::actor_shared())); } @@ -1533,7 +1532,7 @@ class TonlibCli : public td::actor::Actor { auto update = tonlib_api::move_object_as(std::move(result)); CHECK(!raw_client_.empty()); snd_bytes_ += update->data_.size(); - send_closure(raw_client_, &tonlib::ExtClientLazy::send_query, "query", td::BufferSlice(update->data_), + send_closure(raw_client_, &liteclient::ExtClient::send_query, "query", td::BufferSlice(update->data_), ton::ShardIdFull(update->workchain_, update->shard_), td::Timestamp::in(5), [actor_id = actor_id(this), id = update->id_](td::Result res) { send_closure(actor_id, &TonlibCli::on_adnl_result, id, std::move(res)); From b422d95b95d5389cc432e42e289687df7b6c56d4 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 14 Jul 2023 13:15:10 +0300 Subject: [PATCH 054/524] Add missing query to QueryTraits --- lite-client/QueryTraits.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lite-client/QueryTraits.h b/lite-client/QueryTraits.h index ffa683e629..54190244e9 100644 --- a/lite-client/QueryTraits.h +++ b/lite-client/QueryTraits.h @@ -172,6 +172,13 @@ struct QueryTraits { } }; +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_listBlockTransactionsExt& q) { + return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); + } +}; + template<> struct QueryTraits { static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getBlockProof& q) { From ac6cc3bafd5f45f97956db820f88a77806ec3726 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 14 Jul 2023 15:51:27 +0300 Subject: [PATCH 055/524] Use partial liteservers in blockchain-explorer --- blockchain-explorer/CMakeLists.txt | 4 +- .../blockchain-explorer-query.cpp | 145 ++++++++---------- blockchain-explorer/blockchain-explorer.cpp | 93 +++++------ blockchain-explorer/blockchain-explorer.hpp | 2 +- 4 files changed, 112 insertions(+), 132 deletions(-) diff --git a/blockchain-explorer/CMakeLists.txt b/blockchain-explorer/CMakeLists.txt index 11328a7a37..b1979e70ec 100644 --- a/blockchain-explorer/CMakeLists.txt +++ b/blockchain-explorer/CMakeLists.txt @@ -17,11 +17,11 @@ if (NIX) find_package(PkgConfig REQUIRED) pkg_check_modules(MHD libmicrohttpd) target_include_directories(blockchain-explorer PUBLIC ${MHD_INCLUDE_DIRS} ${MHD_STATIC_INCLUDE_DIRS}) - target_link_libraries(blockchain-explorer tdutils tdactor adnllite tl_lite_api tl-lite-utils ton_crypto ton_block ${MHD_LIBRARIES} ${MHD_STATIC_LIBRARIES}) + target_link_libraries(blockchain-explorer tdutils tdactor adnllite tl_lite_api tl-lite-utils ton_crypto ton_block lite-client-common ${MHD_LIBRARIES} ${MHD_STATIC_LIBRARIES}) else() find_package(MHD) target_include_directories(blockchain-explorer PUBLIC ${MHD_INCLUDE_DIRS}) - target_link_libraries(blockchain-explorer tdutils tdactor adnllite tl_lite_api tl-lite-utils ton_crypto ton_block ${MHD_LIBRARIES}) + target_link_libraries(blockchain-explorer tdutils tdactor adnllite tl_lite_api tl-lite-utils ton_crypto ton_block lite-client-common ${MHD_LIBRARIES}) endif() install(TARGETS blockchain-explorer RUNTIME DESTINATION bin) diff --git a/blockchain-explorer/blockchain-explorer-query.cpp b/blockchain-explorer/blockchain-explorer-query.cpp index b53e79696e..9f8ea74193 100644 --- a/blockchain-explorer/blockchain-explorer-query.cpp +++ b/blockchain-explorer/blockchain-explorer-query.cpp @@ -43,6 +43,7 @@ #include "block/block-auto.h" #include "crypto/vm/utils.h" #include "td/utils/crypto.h" +#include "lite-client/QueryTraits.h" #include "vm/boc.h" #include "vm/cellops.h" @@ -237,8 +238,7 @@ void HttpQueryBlockData::finish_query() { } void HttpQueryBlockData::start_up() { - auto query = ton::serialize_tl_object( - ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)), true); + auto query = ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { @@ -249,7 +249,7 @@ void HttpQueryBlockData::start_up() { }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } void HttpQueryBlockData::got_block_data(td::BufferSlice data) { @@ -300,8 +300,7 @@ void HttpQueryBlockView::finish_query() { } void HttpQueryBlockView::start_up_query() { - auto query = ton::serialize_tl_object( - ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)), true); + auto query = ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { @@ -312,7 +311,7 @@ void HttpQueryBlockView::start_up_query() { }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } void HttpQueryBlockView::got_block_data(td::BufferSlice data) { @@ -348,11 +347,10 @@ void HttpQueryBlockInfo::start_up_query() { td::actor::send_closure(SelfId, &HttpQueryBlockInfo::got_block_header, R.move_as_ok()); } }); - auto query = ton::serialize_tl_object( - ton::create_tl_object(ton::create_tl_lite_block_id(block_id_), 0), - true); + auto query = + ton::create_tl_object(ton::create_tl_lite_block_id(block_id_), 0); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); pending_queries_ = 1; if (block_id_.is_masterchain()) { @@ -364,16 +362,15 @@ void HttpQueryBlockInfo::start_up_query() { td::actor::send_closure(SelfId, &HttpQueryBlockInfo::got_shard_info, R.move_as_ok()); } }); - auto query_2 = ton::serialize_tl_object( - ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)), - true); + auto query_2 = + ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query_2), std::move(P_2)); + ton::serialize_tl_object(query_2, true), liteclient::get_query_shard(*query_2), + std::move(P_2)); pending_queries_++; } - auto query_3 = ton::serialize_tl_object(ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), 7, 1024, nullptr, false, false), - true); + auto query_3 = ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), 7, 1024, nullptr, false, false); auto P_3 = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &HttpQueryBlockInfo::abort_query, R.move_as_error_prefix("litequery failed: ")); @@ -382,7 +379,8 @@ void HttpQueryBlockInfo::start_up_query() { } }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query_3), std::move(P_3)); + ton::serialize_tl_object(query_3, true), liteclient::get_query_shard(*query_3), + std::move(P_3)); pending_queries_++; } @@ -435,11 +433,9 @@ void HttpQueryBlockInfo::got_transactions(td::BufferSlice data) { if (f->incomplete_ && transactions_.size() > 0) { const auto &T = *transactions_.rbegin(); - auto query_3 = ton::serialize_tl_object( - ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), 7 + 128, 1024, - ton::create_tl_object(T.addr.addr, T.lt), false, false), - true); + auto query_3 = ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), 7 + 128, 1024, + ton::create_tl_object(T.addr.addr, T.lt), false, false); auto P_3 = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &HttpQueryBlockInfo::abort_query, R.move_as_error_prefix("litequery failed: ")); @@ -448,7 +444,8 @@ void HttpQueryBlockInfo::got_transactions(td::BufferSlice data) { } }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query_3), std::move(P_3)); + ton::serialize_tl_object(query_3, true), liteclient::get_query_shard(*query_3), + std::move(P_3)); } else { if (!--pending_queries_) { finish_query(); @@ -568,14 +565,13 @@ void HttpQueryBlockSearch::start_up_query() { td::actor::send_closure(SelfId, &HttpQueryBlockSearch::got_block_header, R.move_as_ok()); } }); - auto query = ton::serialize_tl_object(ton::create_tl_object( - mode_, - ton::create_tl_lite_block_id_simple(ton::BlockId{ - account_prefix_.workchain, account_prefix_.account_id_prefix, seqno_}), - lt_, utime_), - true); + auto query = ton::create_tl_object( + mode_, + ton::create_tl_lite_block_id_simple( + ton::BlockId{account_prefix_.workchain, account_prefix_.account_id_prefix, seqno_}), + lt_, utime_); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } void HttpQueryBlockSearch::got_block_header(td::BufferSlice data) { @@ -597,17 +593,16 @@ void HttpQueryBlockSearch::got_block_header(td::BufferSlice data) { td::actor::send_closure(SelfId, &HttpQueryBlockSearch::got_shard_info, R.move_as_ok()); } }); - auto query_2 = ton::serialize_tl_object( - ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)), - true); + auto query_2 = + ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query_2), std::move(P_2)); + ton::serialize_tl_object(query_2, true), liteclient::get_query_shard(*query_2), + std::move(P_2)); pending_queries_++; } - auto query_3 = ton::serialize_tl_object(ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), 7, 1024, nullptr, false, false), - true); + auto query_3 = ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), 7, 1024, nullptr, false, false); auto P_3 = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &HttpQueryBlockSearch::abort_query, R.move_as_error_prefix("litequery failed: ")); @@ -616,7 +611,8 @@ void HttpQueryBlockSearch::got_block_header(td::BufferSlice data) { } }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query_3), std::move(P_3)); + ton::serialize_tl_object(query_3, true), liteclient::get_query_shard(*query_3), + std::move(P_3)); pending_queries_++; } @@ -656,11 +652,9 @@ void HttpQueryBlockSearch::got_transactions(td::BufferSlice data) { if (f->incomplete_ && transactions_.size() > 0) { const auto &T = *transactions_.rbegin(); - auto query_3 = ton::serialize_tl_object( - ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), 7 + 128, 1024, - ton::create_tl_object(T.addr.addr, T.lt), false, false), - true); + auto query_3 = ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), 7 + 128, 1024, + ton::create_tl_object(T.addr.addr, T.lt), false, false); auto P_3 = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &HttpQueryBlockSearch::abort_query, @@ -670,7 +664,8 @@ void HttpQueryBlockSearch::got_transactions(td::BufferSlice data) { } }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query_3), std::move(P_3)); + ton::serialize_tl_object(query_3, true), liteclient::get_query_shard(*query_3), + std::move(P_3)); } else { if (!--pending_queries_) { finish_query(); @@ -758,11 +753,10 @@ void HttpQueryViewAccount::start_up_query() { } }); auto a = ton::create_tl_object(addr_.workchain, addr_.addr); - auto query = ton::serialize_tl_object(ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), std::move(a)), - true); + auto query = ton::create_tl_object(ton::create_tl_lite_block_id(block_id_), + std::move(a)); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } void HttpQueryViewAccount::got_account(td::BufferSlice data) { @@ -855,10 +849,9 @@ void HttpQueryViewTransaction::start_up_query() { } }); auto a = ton::create_tl_object(addr_.workchain, addr_.addr); - auto query = ton::serialize_tl_object( - ton::create_tl_object(1, std::move(a), lt_, hash_), true); + auto query = ton::create_tl_object(1, std::move(a), lt_, hash_); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } void HttpQueryViewTransaction::got_transaction(td::BufferSlice data) { @@ -946,11 +939,10 @@ void HttpQueryViewTransaction2::start_up_query() { } }); auto a = ton::create_tl_object(addr_.workchain, addr_.addr); - auto query = ton::serialize_tl_object(ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), std::move(a), lt_), - true); + auto query = ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), std::move(a), lt_); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } void HttpQueryViewTransaction2::got_transaction(td::BufferSlice data) { @@ -1009,9 +1001,9 @@ void HttpQueryViewLastBlock::start_up() { } }); - auto query = ton::serialize_tl_object(ton::create_tl_object(), true); + auto query = ton::create_tl_object(); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } void HttpQueryViewLastBlock::got_result(td::BufferSlice data) { @@ -1075,9 +1067,9 @@ void HttpQueryConfig::start_up() { } }); - auto query = ton::serialize_tl_object(ton::create_tl_object(), true); + auto query = ton::create_tl_object(); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } } @@ -1101,16 +1093,17 @@ void HttpQueryConfig::send_main_query() { td::actor::send_closure(SelfId, &HttpQueryConfig::got_result, R.move_as_ok()); } }); - auto query = - params_.size() > 0 - ? ton::serialize_tl_object(ton::create_tl_object( - 0, ton::create_tl_lite_block_id(block_id_), std::vector(params_)), - true) - : ton::serialize_tl_object(ton::create_tl_object( - 0, ton::create_tl_lite_block_id(block_id_)), - true); - td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + if (params_.size() > 0) { + auto query = ton::create_tl_object( + 0, ton::create_tl_lite_block_id(block_id_), std::vector(params_)); + td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + } else { + auto query = + ton::create_tl_object(0, ton::create_tl_lite_block_id(block_id_)); + td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + } } void HttpQueryConfig::got_result(td::BufferSlice data) { @@ -1252,10 +1245,9 @@ void HttpQuerySend::start_up() { td::actor::send_closure(SelfId, &HttpQuerySend::got_result, R.move_as_ok()); } }); - auto query = - ton::serialize_tl_object(ton::create_tl_object(std::move(data_)), true); + auto query = ton::create_tl_object(std::move(data_)); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } void HttpQuerySend::got_result(td::BufferSlice data) { @@ -1347,11 +1339,10 @@ void HttpQueryRunMethod::start_up_query() { } }); auto a = ton::create_tl_object(addr_.workchain, addr_.addr); - auto query = ton::serialize_tl_object(ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), std::move(a)), - true); + auto query = ton::create_tl_object(ton::create_tl_lite_block_id(block_id_), + std::move(a)); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } void HttpQueryRunMethod::got_account(td::BufferSlice data) { diff --git a/blockchain-explorer/blockchain-explorer.cpp b/blockchain-explorer/blockchain-explorer.cpp index 035b84ea7d..9baee7d8d0 100644 --- a/blockchain-explorer/blockchain-explorer.cpp +++ b/blockchain-explorer/blockchain-explorer.cpp @@ -57,6 +57,7 @@ #include "auto/tl/lite_api.h" #include "ton/lite-tl.hpp" #include "tl-utils/lite-utils.hpp" +#include "lite-client/ext-client.h" #include @@ -126,7 +127,7 @@ class CoreActor : public CoreActorInterface { private: std::string global_config_ = "ton-global.config"; - std::vector> clients_; + td::actor::ActorOwn client_; td::uint32 http_port_ = 80; MHD_Daemon* daemon_ = nullptr; @@ -136,24 +137,17 @@ class CoreActor : public CoreActorInterface { bool hide_ips_ = false; - std::unique_ptr make_callback(td::uint32 idx) { - class Callback : public ton::adnl::AdnlExtClient::Callback { + td::unique_ptr make_callback() { + class Callback : public liteclient::ExtClient::Callback { public: - void on_ready() override { - td::actor::send_closure(id_, &CoreActor::conn_ready, idx_); - } - void on_stop_ready() override { - td::actor::send_closure(id_, &CoreActor::conn_closed, idx_); - } - Callback(td::actor::ActorId id, td::uint32 idx) : id_(std::move(id)), idx_(idx) { + Callback(td::actor::ActorId id) : id_(std::move(id)) { } private: td::actor::ActorId id_; - td::uint32 idx_; }; - return std::make_unique(actor_id(this), idx); + return td::make_unique(actor_id(this)); } std::shared_ptr new_result_; @@ -162,9 +156,8 @@ class CoreActor : public CoreActorInterface { std::vector ready_; - void run_queries(); + //void run_queries(); void got_result(td::uint32 idx, td::int32 attempt, td::Result data); - void send_query(td::uint32 idx); void add_result() { if (new_result_) { @@ -183,7 +176,7 @@ class CoreActor : public CoreActorInterface { add_result(); } attempt_ = t; - run_queries(); + //run_queries(); alarm_timestamp() = td::Timestamp::at_unix((attempt_ + 1) * 60); } @@ -225,10 +218,7 @@ class CoreActor : public CoreActorInterface { hide_ips_ = value; } - void send_lite_query(td::uint32 idx, td::BufferSlice query, td::Promise promise); - void send_lite_query(td::BufferSlice data, td::Promise promise) override { - return send_lite_query(0, std::move(data), std::move(promise)); - } + void send_lite_query(td::BufferSlice query, ton::ShardIdFull shard, td::Promise promise) override; void get_last_result(td::Promise> promise) override { } void get_results(td::uint32 max, td::Promise promise) override { @@ -448,23 +438,39 @@ class CoreActor : public CoreActorInterface { } void run() { + std::vector servers; if (remote_public_key_.empty()) { auto G = td::read_file(global_config_).move_as_ok(); auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); ton::ton_api::liteclient_config_global gc; ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - CHECK(gc.liteservers_.size() > 0); - td::uint32 size = static_cast(gc.liteservers_.size()); + size_t size = gc.liteservers_.size() + gc.liteservers_v2_.size(); + CHECK(size > 0); ready_.resize(size, false); - for (td::uint32 i = 0; i < size; i++) { - auto& cli = gc.liteservers_[i]; + for (auto& s : gc.liteservers_) { td::IPAddress addr; - addr.init_host_port(td::IPAddress::ipv4_to_str(cli->ip_), cli->port_).ensure(); + addr.init_host_port(td::IPAddress::ipv4_to_str(s->ip_), s->port_).ensure(); addrs_.push_back(addr); - clients_.emplace_back(ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull::create(cli->id_).move_as_ok(), - addr, make_callback(i))); + liteclient::ExtClient::LiteServer serv; + serv.address = addr; + serv.adnl_id = ton::adnl::AdnlNodeIdFull::create(s->id_).move_as_ok(); + servers.push_back(std::move(serv)); + } + for (auto& s : gc.liteservers_v2_) { + td::IPAddress addr; + addr.init_host_port(td::IPAddress::ipv4_to_str(s->ip_), s->port_).ensure(); + addrs_.push_back(addr); + liteclient::ExtClient::LiteServer serv; + serv.address = addr; + serv.adnl_id = ton::adnl::AdnlNodeIdFull::create(s->id_).move_as_ok(); + serv.is_full = false; + for (auto& shard : s->shards_) { + serv.shards.emplace_back(shard->workchain_, (ton::ShardId)shard->shard_); + CHECK(serv.shards.back().is_valid_ext()); + } + servers.push_back(std::move(serv)); } } else { if (!remote_addr_.is_valid()) { @@ -472,9 +478,12 @@ class CoreActor : public CoreActorInterface { } ready_.resize(1, false); addrs_.push_back(remote_addr_); - clients_.emplace_back(ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull{remote_public_key_}, - remote_addr_, make_callback(0))); + liteclient::ExtClient::LiteServer serv; + serv.address = remote_addr_; + serv.adnl_id = ton::adnl::AdnlNodeIdFull{remote_public_key_}; + servers.push_back(std::move(serv)); } + client_ = liteclient::ExtClient::create(std::move(servers), make_callback()); daemon_ = MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, static_cast(http_port_), nullptr, nullptr, &process_http_request, nullptr, MHD_OPTION_NOTIFY_COMPLETED, request_completed, nullptr, MHD_OPTION_THREAD_POOL_SIZE, 16, MHD_OPTION_END); @@ -523,23 +532,7 @@ void CoreActor::got_result(td::uint32 idx, td::int32 attempt, td::Result(); - auto q = ton::create_tl_object(serialize_tl_object(query, true)); - - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), idx, attempt = attempt_](td::Result R) { - td::actor::send_closure(SelfId, &CoreActor::got_result, idx, attempt, std::move(R)); - }); - td::actor::send_closure(clients_[idx], &ton::adnl::AdnlExtClient::send_query, "query", serialize_tl_object(q, true), - td::Timestamp::in(10.0), std::move(P)); -} - -void CoreActor::run_queries() { +/*void CoreActor::run_queries() { waiting_ = 0; new_result_ = std::make_shared(ready_.size(), td::Timestamp::at_unix(attempt_ * 60)); for (td::uint32 i = 0; i < ready_.size(); i++) { @@ -549,13 +542,9 @@ void CoreActor::run_queries() { if (waiting_ == 0) { add_result(); } -} +}*/ -void CoreActor::send_lite_query(td::uint32 idx, td::BufferSlice query, td::Promise promise) { - if (!ready_[idx]) { - promise.set_error(td::Status::Error(ton::ErrorCode::notready, "ext conn not ready")); - return; - } +void CoreActor::send_lite_query(td::BufferSlice query, ton::ShardIdFull shard, td::Promise promise) { auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); @@ -573,7 +562,7 @@ void CoreActor::send_lite_query(td::uint32 idx, td::BufferSlice query, td::Promi promise.set_value(std::move(B)); }); auto q = ton::create_tl_object(std::move(query)); - td::actor::send_closure(clients_[idx], &ton::adnl::AdnlExtClient::send_query, "query", serialize_tl_object(q, true), + td::actor::send_closure(client_, &liteclient::ExtClient::send_query, "query", serialize_tl_object(q, true), shard, td::Timestamp::in(10.0), std::move(P)); } diff --git a/blockchain-explorer/blockchain-explorer.hpp b/blockchain-explorer/blockchain-explorer.hpp index 1bae362dee..d3c53e451c 100644 --- a/blockchain-explorer/blockchain-explorer.hpp +++ b/blockchain-explorer/blockchain-explorer.hpp @@ -64,7 +64,7 @@ class CoreActorInterface : public td::actor::Actor { }; virtual ~CoreActorInterface() = default; - virtual void send_lite_query(td::BufferSlice data, td::Promise promise) = 0; + virtual void send_lite_query(td::BufferSlice data, ton::ShardIdFull shard, td::Promise promise) = 0; virtual void get_last_result(td::Promise> promise) = 0; virtual void get_results(td::uint32 max, td::Promise promise) = 0; From 96afdc12ac547a5b12541f0432f50e5ab83610ab Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 14 Jul 2023 17:16:49 +0300 Subject: [PATCH 056/524] Increase INACTIVE_SHARD_TTL Node in overlay should be available until peer is expired --- overlay/overlay-peers.cpp | 4 ++-- overlay/overlays.h | 4 ++++ validator/full-node.cpp | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index 28dac8f05f..8c4f5d228a 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -100,7 +100,7 @@ void OverlayImpl::add_peer_in(OverlayNode node) { return; } auto t = td::Clocks::system(); - if (node.version() + 600 < t || node.version() > t + 60) { + if (node.version() + Overlays::overlay_peer_ttl() < t || node.version() > t + 60) { VLOG(OVERLAY_INFO) << this << ": ignoring node of too old version " << node.version(); return; } @@ -231,7 +231,7 @@ void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) { continue; } - if (X->get_version() <= td::Clocks::system() - 600) { + if (X->get_version() <= td::Clocks::system() - Overlays::overlay_peer_ttl()) { if (X->is_neighbour()) { bool found = false; for (auto &n : neighbours_) { diff --git a/overlay/overlays.h b/overlay/overlays.h index 1141efee24..cf153c3a5c 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -187,6 +187,10 @@ class Overlays : public td::actor::Actor { return 1; } + static constexpr td::uint32 overlay_peer_ttl() { + return 600; + } + static td::actor::ActorOwn create(std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId dht); diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 669e2fd974..cfe4936674 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -27,7 +27,7 @@ namespace validator { namespace fullnode { -static const double INACTIVE_SHARD_TTL = 120.0; +static const double INACTIVE_SHARD_TTL = (double)overlay::Overlays::overlay_peer_ttl() + 60.0; void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise promise) { if (local_keys_.count(key)) { From b8f347231fd4246736da8300d81df0ff73861f08 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 17 Jul 2023 18:45:32 +0300 Subject: [PATCH 057/524] Fix shard overlays --- validator/full-node.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/full-node.cpp b/validator/full-node.cpp index cfe4936674..3897626f04 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -147,7 +147,7 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s new_shards.insert(ShardIdFull(masterchainId)); std::set workchains; auto cut_shard = [&](ShardIdFull shard) -> ShardIdFull { - int min_split = state->soft_min_split_depth(shard.workchain); + int min_split = state->min_split_depth(shard.workchain); return min_split < shard.pfx_len() ? shard_prefix(shard, min_split) : shard; }; auto set_active = [&](ShardIdFull shard, FullNodeShardMode mode) { From 3265e397f27d50b6d45b6e0d9e6b20f724467ba1 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 17 Jul 2023 19:14:39 +0300 Subject: [PATCH 058/524] Rename actual_min_split and soft_mis_split to avoid confusion --- crypto/block/block.tlb | 8 ++++---- crypto/block/mc-config.cpp | 2 +- crypto/block/mc-config.h | 2 +- validator/full-node.cpp | 2 +- validator/impl/shard.cpp | 6 +++--- validator/impl/shard.hpp | 2 +- validator/interfaces/shard.h | 2 +- validator/validator-options.hpp | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 14ec54f47c..f2f11fc71f 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -646,15 +646,15 @@ wc_split_merge_timings#0 //workchain#a5 enabled_since:uint32 min_split:(## 8) max_split:(## 8) // { min_split <= max_split } { max_split <= 60 } -workchain#a6 enabled_since:uint32 actual_min_split:(## 8) - min_split:(## 8) max_split:(## 8) { actual_min_split <= min_split } +workchain#a6 enabled_since:uint32 monitor_min_split:(## 8) + min_split:(## 8) max_split:(## 8) { monitor_min_split <= min_split } basic:(## 1) active:Bool accept_msgs:Bool flags:(## 13) { flags = 0 } zerostate_root_hash:bits256 zerostate_file_hash:bits256 version:uint32 format:(WorkchainFormat basic) = WorkchainDescr; -workchain_v2#a7 enabled_since:uint32 actual_min_split:(## 8) - min_split:(## 8) max_split:(## 8) { actual_min_split <= min_split } +workchain_v2#a7 enabled_since:uint32 monitor_min_split:(## 8) + min_split:(## 8) max_split:(## 8) { monitor_min_split <= min_split } basic:(## 1) active:Bool accept_msgs:Bool flags:(## 13) { flags = 0 } zerostate_root_hash:bits256 zerostate_file_hash:bits256 version:uint32 format:(WorkchainFormat basic) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index b4de693cfc..1ab907cf56 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2005,7 +2005,7 @@ bool WorkchainInfo::unpack(ton::WorkchainId wc, vm::CellSlice& cs) { } auto unpack_v1 = [this](auto& info) { enabled_since = info.enabled_since; - actual_min_split = info.actual_min_split; + monitor_min_split = info.monitor_min_split; min_split = info.min_split; max_split = info.max_split; basic = info.basic; diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 8eb8a7f9fb..bf5350b503 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -407,7 +407,7 @@ struct CatchainValidatorsConfig { struct WorkchainInfo : public td::CntObject { ton::WorkchainId workchain{ton::workchainInvalid}; ton::UnixTime enabled_since; - td::uint32 actual_min_split; + td::uint32 monitor_min_split; td::uint32 min_split, max_split; bool basic; bool active; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 3897626f04..3b275ea991 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -147,7 +147,7 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s new_shards.insert(ShardIdFull(masterchainId)); std::set workchains; auto cut_shard = [&](ShardIdFull shard) -> ShardIdFull { - int min_split = state->min_split_depth(shard.workchain); + int min_split = state->monitor_min_split_depth(shard.workchain); return min_split < shard.pfx_len() ? shard_prefix(shard, min_split) : shard; }; auto set_active = [&](ShardIdFull shard, FullNodeShardMode mode) { diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index a09bd1df83..a7ff371ea7 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -511,15 +511,15 @@ bool MasterchainStateQ::check_old_mc_block_id(const ton::BlockIdExt& blkid, bool return config_ && config_->check_old_mc_block_id(blkid, strict); } -td::uint32 MasterchainStateQ::min_split_depth(WorkchainId workchain_id) const { +td::uint32 MasterchainStateQ::monitor_min_split_depth(WorkchainId workchain_id) const { if (!config_) { return 0; } auto wc_info = config_->get_workchain_info(workchain_id); - return wc_info.not_null() ? wc_info->actual_min_split : 0; + return wc_info.not_null() ? wc_info->monitor_min_split : 0; } -td::uint32 MasterchainStateQ::soft_min_split_depth(WorkchainId workchain_id) const { +td::uint32 MasterchainStateQ::min_split_depth(WorkchainId workchain_id) const { if (!config_) { return 0; } diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index b55cac35c9..83aa1f9a45 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -116,8 +116,8 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { bool has_workchain(WorkchainId workchain) const { return config_ && config_->has_workchain(workchain); } + td::uint32 monitor_min_split_depth(WorkchainId workchain_id) const override; td::uint32 min_split_depth(WorkchainId workchain_id) const override; - td::uint32 soft_min_split_depth(WorkchainId workchain_id) const override; BlockSeqno min_ref_masterchain_seqno() const override; td::Status prepare() override; ZeroStateIdExt get_zerostate_id() const { diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index ac355d7dce..9ccfa2fb48 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -69,8 +69,8 @@ class MasterchainState : virtual public ShardState { virtual std::vector> get_shards() const = 0; virtual td::Ref get_shard_from_config(ShardIdFull shard) const = 0; virtual bool workchain_is_active(WorkchainId workchain_id) const = 0; + virtual td::uint32 monitor_min_split_depth(WorkchainId workchain_id) const = 0; virtual td::uint32 min_split_depth(WorkchainId workchain_id) const = 0; - virtual td::uint32 soft_min_split_depth(WorkchainId workchain_id) const = 0; virtual BlockSeqno min_ref_masterchain_seqno() const = 0; virtual bool ancestor_is_valid(BlockIdExt id) const = 0; virtual ValidatorSessionConfig get_consensus_config() const = 0; diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index ea100a50bc..fa7eb5f9b6 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -33,7 +33,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { return init_block_id_; } bool need_monitor(ShardIdFull shard, const td::Ref& state) const override { - td::uint32 min_split = state->min_split_depth(shard.workchain); + td::uint32 min_split = state->monitor_min_split_depth(shard.workchain); return check_shard_((td::uint32)shard.pfx_len() <= min_split ? shard : shard_prefix(shard, min_split)); } bool allow_blockchain_init() const override { From aa4f5769cac9707365e7ad79eab8bbae3a8010a2 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 18 Jul 2023 17:06:06 +0300 Subject: [PATCH 059/524] More logs for collators and validators --- tdutils/td/utils/Timer.cpp | 4 ++++ tdutils/td/utils/Timer.h | 1 + validator/collator-node.cpp | 2 ++ validator/full-node-shard.cpp | 1 - validator/impl/collator.cpp | 2 ++ validator/impl/validate-query.cpp | 4 ++++ validator/validator-group.cpp | 36 +++++++++++++++++-------------- 7 files changed, 33 insertions(+), 17 deletions(-) diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index 0f6a7d6edb..1f72fba96c 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -87,4 +87,8 @@ void PerfWarningTimer::reset() { start_at_ = 0; } +double PerfWarningTimer::elapsed() const { + return Time::now() - start_at_; +} + } // namespace td diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index 18ea35d7c2..3e0cafbf59 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -53,6 +53,7 @@ class PerfWarningTimer { PerfWarningTimer &operator=(PerfWarningTimer &&) = delete; ~PerfWarningTimer(); void reset(); + double elapsed() const; private: string name_; diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 516555fae7..2cc06b3403 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -233,11 +233,13 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data cache_entry = cache_[cache_key] = std::make_shared(); } if (cache_entry->result) { + LOG(INFO) << "Using cached result"; new_promise.set_result(cache_entry->result.value().clone()); return; } cache_entry->promises.push_back(std::move(new_promise)); if (cache_entry->started) { + LOG(INFO) << "Collating of this block is already in progress, waiting"; return; } cache_entry->started = true; diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 02bf0e39d6..b34432a932 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -927,7 +927,6 @@ void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, std::stri void FullNodeShardImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, td::Promise> promise) { // TODO: maybe more complex download (like other requests here) - // TODO: estimate max size auto &b = choose_neighbour(true); if (b.adnl_id == adnl::AdnlNodeIdShort::zero()) { promise.set_error(td::Status::Error(ErrorCode::notready, "no nodes")); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index fc31d2645c..2d91489e09 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -279,6 +279,7 @@ bool Collator::fatal_error(td::Status error) { error.ensure_error(); LOG(ERROR) << "cannot generate block candidate for " << show_shard(shard_) << " : " << error.to_string(); if (busy_) { + LOG(INFO) << "collation took " << perf_timer_.elapsed() << " s"; main_promise(std::move(error)); busy_ = false; } @@ -4226,6 +4227,7 @@ void Collator::return_block_candidate(td::Result saved) { } else { CHECK(block_candidate); LOG(INFO) << "sending new BlockCandidate to Promise"; + LOG(INFO) << "collation took " << perf_timer_.elapsed() << " s"; main_promise(block_candidate->clone()); busy_ = false; stop(); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index f9e5be2757..4c7384ee97 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -88,6 +88,7 @@ bool ValidateQuery::reject_query(std::string error, td::BufferSlice reason) { << " collated_data=" << block_candidate.collated_file_hash.to_hex()); errorlog::ErrorLog::log_file(block_candidate.data.clone()); errorlog::ErrorLog::log_file(block_candidate.collated_data.clone()); + LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s"; main_promise.set_result(CandidateReject{std::move(error), std::move(reason)}); } stop(); @@ -108,6 +109,7 @@ bool ValidateQuery::soft_reject_query(std::string error, td::BufferSlice reason) << " collated_data=" << block_candidate.collated_file_hash.to_hex()); errorlog::ErrorLog::log_file(block_candidate.data.clone()); errorlog::ErrorLog::log_file(block_candidate.collated_data.clone()); + LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s"; main_promise.set_result(CandidateReject{std::move(error), std::move(reason)}); } stop(); @@ -126,6 +128,7 @@ bool ValidateQuery::fatal_error(td::Status error) { errorlog::ErrorLog::log_file(block_candidate.data.clone()); errorlog::ErrorLog::log_file(block_candidate.collated_data.clone()); } + LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s"; main_promise(std::move(error)); } stop(); @@ -147,6 +150,7 @@ bool ValidateQuery::fatal_error(std::string err_msg, int err_code) { void ValidateQuery::finish_query() { if (main_promise) { + LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s"; main_promise.set_result(now_); } stop(); diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 7cc92f8041..bc6a9812c8 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -133,7 +133,7 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat P.set_error(td::Status::Error(ErrorCode::notready, "validator group not started")); return; } - VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id; + VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id.to_str(); block.id = next_block_id; run_validate_query(shard_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, manager_, td::Timestamp::in(15.0), std::move(P), @@ -397,6 +397,7 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo promise.set_error(td::Status::Error("too old")); return; } + BlockId next_block_id = create_next_block_id_simple(); adnl::AdnlNodeIdShort collator = adnl::AdnlNodeIdShort::zero(); // TODO: some way to choose node (similar to "unreliability" in full-node) int cnt = 0; @@ -413,20 +414,23 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo return; } - if (max_retries > 0) { - promise = td::PromiseCreator::lambda( - [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_ok()) { - promise.set_result(R.move_as_ok()); - } else if (timeout && timeout.is_in_past()) { - promise.set_result(R.move_as_error()); - } else { - LOG(WARNING) << "collate query error, retrying: " << R.move_as_error(); - td::actor::send_closure(SelfId, &ValidatorGroup::send_collate_query, round_id, timeout, std::move(promise), - max_retries - 1); - } - }); - } + promise = td::PromiseCreator::lambda([=, SelfId = actor_id(this), promise = std::move(promise), + timer = td::Timer()](td::Result R) mutable { + if (R.is_ok()) { + LOG(WARNING) << "collate query for " << next_block_id.to_str() << ": success, time=" << timer.elapsed() << "s"; + promise.set_result(R.move_as_ok()); + return; + } + bool retry = (!timeout || !timeout.is_in_past()) && max_retries > 0; + LOG(WARNING) << "collate query for " << next_block_id.to_str() << ": " << R.error() << ", time=" << timer.elapsed() + << "s, " << (retry ? "retrying" : "giving up"); + if (retry) { + td::actor::send_closure(SelfId, &ValidatorGroup::send_collate_query, round_id, timeout, std::move(promise), + max_retries - 1); + } else { + promise.set_result(td::Status::Error(ErrorCode::timeout, "timeout")); + } + }); std::vector> prev_blocks; for (const BlockIdExt &p : prev_block_ids_) { @@ -445,7 +449,7 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo td::actor::send_closure(SelfId, &ValidatorGroup::receive_collate_query_response, round_id, R.move_as_ok(), std::move(promise)); }); - LOG(INFO) << "collate query for " << create_next_block_id_simple().to_str() << ": send query to " << collator; + LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to " << collator; size_t max_answer_size = config_.max_block_size + config_.max_collated_data_size + 256; td::Timestamp query_timeout = td::Timestamp::in(10.0); query_timeout.relax(timeout); From e6b77ef71dcd71bbbc0e46b93e9c0a3d47dbc05e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 19 Jul 2023 14:53:10 +0300 Subject: [PATCH 060/524] Remove excessive check in check_neighbor_outbound_message --- validator/impl/validate-query.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 4c7384ee97..fcd869b1a8 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -3983,14 +3983,6 @@ bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, td::ConstBitPtr key, const block::McShardDescr& nb, bool& unprocessed) { unprocessed = false; - if (!block::gen::t_EnqueuedMsg.validate_csr(enq_msg)) { - return reject_query("EnqueuedMsg with key "s + key.to_hex(352) + " in outbound queue of our neighbor " + - nb.blk_.to_str() + " failed to pass automated validity tests"); - } - if (!block::tlb::t_EnqueuedMsg.validate_csr(enq_msg)) { - return reject_query("EnqueuedMsg with key "s + key.to_hex(352) + " in outbound queue of our neighbor " + - nb.blk_.to_str() + " failed to pass hand-written validity tests"); - } block::EnqueuedMsgDescr enq; if (!enq.unpack(enq_msg.write())) { // unpack EnqueuedMsg return reject_query("cannot unpack EnqueuedMsg with key "s + key.to_hex(352) + From 869c6fe675bb7ee936a5284d229eadb0ae0e40df Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 20 Jul 2023 17:48:52 +0300 Subject: [PATCH 061/524] Rework limiting imported msg queues --- crypto/block/block.tlb | 1 - crypto/block/output-queue-merger.cpp | 14 +++++--------- crypto/block/output-queue-merger.h | 2 ++ validator/impl/collator-impl.h | 1 - validator/impl/collator.cpp | 25 ++++++++++--------------- validator/impl/out-msg-queue-proof.cpp | 13 +++++++++++-- validator/impl/validate-query.cpp | 22 +--------------------- validator/impl/validate-query.hpp | 1 - validator/validator-group.cpp | 2 +- 9 files changed, 30 insertions(+), 51 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index f2f11fc71f..2176c0adb6 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -829,7 +829,6 @@ top_block_descr#d5 proof_for:BlockIdExt signatures:(Maybe ^BlockSignatures) // COLLATED DATA // top_block_descr_set#4ac789f3 collection:(HashmapE 96 ^TopBlockDescr) = TopBlockDescrSet; -neighbor_msg_queue_limits#7e549333 neighbors:(HashmapE 96 int32) = NeighborMsgQueueLimits; // // VALIDATOR MISBEHAVIOR COMPLAINTS diff --git a/crypto/block/output-queue-merger.cpp b/crypto/block/output-queue-merger.cpp index 2955b3e012..f3ec61430f 100644 --- a/crypto/block/output-queue-merger.cpp +++ b/crypto/block/output-queue-merger.cpp @@ -199,10 +199,6 @@ bool OutputQueueMerger::load() { unsigned long long lt = heap[0]->lt; std::size_t orig_size = msg_list.size(); do { - if (src_remaining_msgs_[heap[0]->source] == 0) { - std::pop_heap(heap.begin(), heap.end(), MsgKeyValue::greater); - continue; - } while (heap[0]->is_fork()) { auto other = std::make_unique(); if (!heap[0]->split(*other)) { @@ -218,17 +214,17 @@ bool OutputQueueMerger::load() { heap.pop_back(); } while (!heap.empty() && heap[0]->lt <= lt); std::sort(msg_list.begin() + orig_size, msg_list.end(), MsgKeyValue::less); - size_t j = orig_size; for (size_t i = orig_size; i < msg_list.size(); ++i) { td::int32 &remaining = src_remaining_msgs_[msg_list[i]->source]; - if (remaining != 0) { - if (remaining > 0) { + if (remaining != -1) { + if (remaining == 0) { + limit_exceeded = true; + } else { --remaining; } - msg_list[j++] = std::move(msg_list[i]); } + msg_list[i]->limit_exceeded = limit_exceeded; } - msg_list.resize(j); return msg_list.size() > orig_size; } diff --git a/crypto/block/output-queue-merger.h b/crypto/block/output-queue-merger.h index 3eb4b37994..26deb7ee47 100644 --- a/crypto/block/output-queue-merger.h +++ b/crypto/block/output-queue-merger.h @@ -32,6 +32,7 @@ struct OutputQueueMerger { int source; int key_len{0}; td::BitArray key; + bool limit_exceeded{false}; MsgKeyValue() = default; MsgKeyValue(int src, Ref node); MsgKeyValue(td::ConstBitPtr key_pfx, int key_pfx_len, int src, Ref node); @@ -82,6 +83,7 @@ struct OutputQueueMerger { std::vector src_remaining_msgs_; bool eof; bool failed; + bool limit_exceeded{false}; void add_root(int src, Ref outmsg_root, td::int32 msg_limit); bool load(); }; diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 59be8d4b4d..2839a0e5b7 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -134,7 +134,6 @@ class Collator final : public td::actor::Actor { std::unique_ptr shard_conf_; std::map> aux_mc_states_; std::map neighbor_msg_queues_limits_; - vm::Dictionary neighbor_msg_queues_limits_dict_{32 + 64}; std::vector neighbors_; std::unique_ptr nb_out_msgs_; std::vector special_smcs; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 2d91489e09..52d9c1c40b 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -701,11 +701,7 @@ void Collator::got_neighbor_msg_queue(unsigned i, td::Resultprefetch_ref(0); descr.set_queue_root(queue_root); if (res->msg_count_ != -1) { - LOG(DEBUG) << "Neighbor " << descr.shard().to_str() << " has msg_limit=" << res->msg_count_; - td::BitArray<96> key; - key.bits().store_int(block_id.id.workchain, 32); - (key.bits() + 32).store_uint(block_id.id.shard, 64); - neighbor_msg_queues_limits_dict_.set_builder(key, vm::CellBuilder().store_long(res->msg_count_, 32)); + LOG(INFO) << "neighbor " << descr.shard().to_str() << " has msg_limit=" << res->msg_count_; neighbor_msg_queues_limits_[block_id.shard_full()] = res->msg_count_; } // comment the next two lines in the future when the output queues become huge @@ -2833,6 +2829,12 @@ bool Collator::process_inbound_internal_messages() { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); auto kv = nb_out_msgs_->extract_cur(); CHECK(kv && kv->msg.not_null()); + if (kv->limit_exceeded) { + LOG(INFO) << "limit for imported messages is reached, stop processing inbound internal messages"; + block::EnqueuedMsgDescr enq; + enq.unpack(kv->msg.write()); // Visit cells to include it in proof + break; + } if (!precheck_inbound_message(kv->msg, kv->lt)) { if (verbosity > 1) { std::cerr << "invalid inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() @@ -4089,21 +4091,14 @@ bool Collator::create_collated_data() { } collated_roots_.push_back(std::move(cell)); } - // 2. Message count for neighbors' out queues - if (!neighbor_msg_queues_limits_dict_.is_empty()) { - vm::CellBuilder cb; - cb.store_long(block::gen::t_NeighborMsgQueueLimits.cons_tag[0], 32); - cb.store_maybe_ref(neighbor_msg_queues_limits_dict_.get_root_cell()); - collated_roots_.push_back(cb.finalize_novm()); - } if (!full_collated_data_) { return true; } - // 3. Proofs for hashes of states: previous states + neighbors + // 2. Proofs for hashes of states: previous states + neighbors for (const auto& p : block_state_proofs_) { collated_roots_.push_back(p.second); } - // 4. Previous state proof (only shadchains) + // 3. Previous state proof (only shadchains) std::map> proofs; if (!is_masterchain()) { if (!prepare_msg_queue_proof()) { @@ -4117,7 +4112,7 @@ bool Collator::create_collated_data() { } proofs[prev_state_root_->get_hash().bits()] = std::move(state_proof); } - // 5. Proofs for message queues + // 4. Proofs for message queues for (vm::MerkleProofBuilder &mpb : neighbor_proof_builders_) { auto r_proof = mpb.extract_proof(); if (r_proof.is_error()) { diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 329da607dd..59aa8aed69 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -52,7 +52,7 @@ static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_ td::HashSet visited; std::function dfs_cs; - auto dfs = [&](Ref cell) { + auto dfs = [&](const Ref& cell) { if (cell.is_null() || !visited.insert(cell->get_hash()).second) { return; } @@ -65,6 +65,8 @@ static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_ dfs(cs.prefetch_ref(i)); } }; + TRY_STATUS_PREFIX(check_no_prunned(*qinfo.proc_info), "invalid proc_info proof: ") + TRY_STATUS_PREFIX(check_no_prunned(*qinfo.ihr_pending), "invalid ihr_pending proof: ") dfs_cs(*qinfo.proc_info); dfs_cs(*qinfo.ihr_pending); @@ -76,6 +78,14 @@ static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_ while (!queue_merger.is_eof()) { auto kv = queue_merger.extract_cur(); queue_merger.next(); + block::EnqueuedMsgDescr enq; + auto msg = kv->msg; + if (!enq.unpack(msg.write())) { + return td::Status::Error("cannot unpack EnqueuedMsgDescr"); + } + if (limit_reached) { + break; + } ++msg_count; // TODO: Get processed_upto from destination shard (in request?) @@ -105,7 +115,6 @@ static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_ TRY_STATUS_PREFIX(check_no_prunned(*kv->msg), "invalid message proof: ") if (estimated_proof_size > OutMsgQueueProof::QUEUE_SIZE_THRESHOLD) { limit_reached = true; - break; } } return limit_reached ? msg_count : -1; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index fcd869b1a8..4f8c464c96 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -526,14 +526,6 @@ bool ValidateQuery::extract_collated_data_from(Ref croot, int idx) { top_shard_descr_dict_ = std::make_unique(cs.prefetch_ref(), 96); return true; } - if (block::gen::t_NeighborMsgQueueLimits.has_valid_tag(cs)) { - LOG(DEBUG) << "collated datum # " << idx << " is a NeighborMsgQueueLimits"; - if (!block::gen::t_NeighborMsgQueueLimits.validate_upto(10000, cs)) { - return reject_query("invalid NeighborMsgQueueLimits"); - } - neighbor_msg_queues_limits_ = vm::Dictionary{cs.prefetch_ref(0), 32 + 64}; - return true; - } LOG(WARNING) << "collated datum # " << idx << " has unknown type (magic " << cs.prefetch_ulong(32) << "), ignoring"; return true; } @@ -4115,19 +4107,7 @@ bool ValidateQuery::check_in_queue() { td::BitArray<96> key; key.bits().store_int(descr.workchain(), 32); (key.bits() + 32).store_uint(descr.shard().shard, 64); - auto r = neighbor_msg_queues_limits_.lookup(key); - td::int32 msg_limit = r.is_null() ? -1 : (td::int32)r->prefetch_long(32); - if (msg_limit < -1) { - return reject_query("invalid value in NeighborMsgQueueLimits"); - } - LOG(DEBUG) << "Neighbor " << descr.shard().to_str() << " has msg_limit=" << msg_limit; - neighbor_queues.emplace_back(descr.top_block_id(), descr.outmsg_root, descr.disabled_, msg_limit); - if (msg_limit != -1 && descr.shard().is_masterchain()) { - return reject_query("masterchain out message queue cannot be limited"); - } - if (msg_limit != -1 && shard_intersects(descr.shard(), shard_)) { - return reject_query("prev block out message queue cannot be limited"); - } + neighbor_queues.emplace_back(descr.top_block_id(), descr.outmsg_root, descr.disabled_); } block::OutputQueueMerger nb_out_msgs(shard_, std::move(neighbor_queues)); while (!nb_out_msgs.is_eof()) { diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index cd14504e61..942b32a5f9 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -206,7 +206,6 @@ class ValidateQuery : public td::actor::Actor { block::ActionPhaseConfig action_phase_cfg_; td::RefInt256 masterchain_create_fee_, basechain_create_fee_; - vm::Dictionary neighbor_msg_queues_limits_{32 + 64}; std::vector neighbors_; std::map> aux_mc_states_; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index bc6a9812c8..5c7bef5b3e 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -417,7 +417,7 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo promise = td::PromiseCreator::lambda([=, SelfId = actor_id(this), promise = std::move(promise), timer = td::Timer()](td::Result R) mutable { if (R.is_ok()) { - LOG(WARNING) << "collate query for " << next_block_id.to_str() << ": success, time=" << timer.elapsed() << "s"; + LOG(INFO) << "collate query for " << next_block_id.to_str() << ": success, time=" << timer.elapsed() << "s"; promise.set_result(R.move_as_ok()); return; } From 32b3fe748ab1799843b73595e1f9ebad597934ee Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 21 Jul 2023 19:29:29 +0300 Subject: [PATCH 062/524] Fix validating inbound msg queues --- validator/impl/collator.cpp | 3 +++ validator/impl/validate-query.cpp | 43 +++++++------------------------ validator/impl/validate-query.hpp | 2 +- 3 files changed, 14 insertions(+), 34 deletions(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 52d9c1c40b..f11bd1169f 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -3805,6 +3805,9 @@ bool Collator::store_master_ref(vm::CellBuilder& cb) { bool Collator::update_processed_upto() { auto ref_mc_seqno = is_masterchain() ? new_block_seqno : prev_mc_block_seqno; update_min_mc_seqno(ref_mc_seqno); + if (in_msg_dict->is_empty()) { + return true; + } if (last_proc_int_msg_.first) { if (!processed_upto_->insert(ref_mc_seqno, last_proc_int_msg_.first, last_proc_int_msg_.second.cbits())) { return fatal_error("cannot update our ProcessedUpto to reflect processed inbound message"); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 4f8c464c96..8d0b51731b 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -3927,6 +3927,7 @@ bool ValidateQuery::check_processed_upto() { if (!ok) { return reject_query("new ProcessedInfo is not obtained from old ProcessedInfo by adding at most one new entry"); } + processed_upto_updated_ = upd; if (upd) { if (upd->shard != shard_.shard) { return reject_query("newly-added ProcessedInfo entry refers to shard "s + @@ -4007,37 +4008,8 @@ bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, key.to_hex(352) + " of neighbor " + nb.blk_.to_str()); } if (shard_contains(shard_, enq.cur_prefix_)) { - // if this message comes from our own outbound queue, we must have dequeued it - if (out_entry.is_null()) { - return reject_query("our old outbound queue contains EnqueuedMsg with key "s + key.to_hex(352) + - " already processed by this shard, but there is no ext_message_deq OutMsg record for this " - "message in this block"); - } - int tag = block::gen::t_OutMsg.get_tag(*out_entry); - if (tag == block::gen::OutMsg::msg_export_deq_short) { - block::gen::OutMsg::Record_msg_export_deq_short deq; - if (!tlb::csr_unpack(std::move(out_entry), deq)) { - return reject_query( - "cannot unpack msg_export_deq_short OutMsg record for already processed EnqueuedMsg with key "s + - key.to_hex(352) + " of old outbound queue"); - } - if (deq.msg_env_hash != enq.msg_env_->get_hash().bits()) { - return reject_query("unpack ext_message_deq OutMsg record for already processed EnqueuedMsg with key "s + - key.to_hex(352) + " of old outbound queue refers to MsgEnvelope with different hash " + - deq.msg_env_hash.to_hex()); - } - } else { - block::gen::OutMsg::Record_msg_export_deq deq; - if (!tlb::csr_unpack(std::move(out_entry), deq)) { - return reject_query( - "cannot unpack msg_export_deq OutMsg record for already processed EnqueuedMsg with key "s + - key.to_hex(352) + " of old outbound queue"); - } - if (deq.out_msg->get_hash() != enq.msg_env_->get_hash()) { - return reject_query("unpack ext_message_deq OutMsg record for already processed EnqueuedMsg with key "s + - key.to_hex(352) + " of old outbound queue contains a different MsgEnvelope"); - } - } + // this message couldn't come from our own outbound queue because processed messages from our queue don't stay here + return fatal_error("have an already processed EnqueuedMsg from our shard: "s + key.to_hex(352)); } // next check is incorrect after a merge, when ns_.processed_upto has > 1 entries // we effectively comment it out @@ -4110,6 +4082,13 @@ bool ValidateQuery::check_in_queue() { neighbor_queues.emplace_back(descr.top_block_id(), descr.outmsg_root, descr.disabled_); } block::OutputQueueMerger nb_out_msgs(shard_, std::move(neighbor_queues)); + if (in_msg_dict_->is_empty()) { + LOG(DEBUG) << "in_msg_dict is empty, skip checking neighbors' message queues"; + if (processed_upto_updated_) { + return reject_query("processed_upto was updated, but no messages were processed"); + } + return true; + } while (!nb_out_msgs.is_eof()) { auto kv = nb_out_msgs.extract_cur(); CHECK(kv && kv->msg.not_null()); @@ -4130,12 +4109,10 @@ bool ValidateQuery::check_in_queue() { neighbors_.at(kv->source).blk_.to_str()); } if (unprocessed) { - inbound_queues_empty_ = false; return true; } nb_out_msgs.next(); } - inbound_queues_empty_ = true; return true; } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 942b32a5f9..b1d528197a 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -210,6 +210,7 @@ class ValidateQuery : public td::actor::Actor { std::map> aux_mc_states_; block::ShardState ps_, ns_; + bool processed_upto_updated_{false}; std::unique_ptr sibling_out_msg_queue_; std::shared_ptr sibling_processed_upto_; @@ -223,7 +224,6 @@ class ValidateQuery : public td::actor::Actor { ton::LogicalTime proc_lt_{0}, claimed_proc_lt_{0}, min_enq_lt_{~0ULL}; ton::Bits256 proc_hash_ = ton::Bits256::zero(), claimed_proc_hash_, min_enq_hash_; - bool inbound_queues_empty_{false}; std::vector> msg_proc_lt_; From f1e62d00757ad585c77b3a2b7d09496a55113dd3 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 24 Jul 2023 15:29:55 +0300 Subject: [PATCH 063/524] Move msg queue limits to config --- crypto/block/block.cpp | 20 ++++++++++++++++---- crypto/block/block.h | 8 ++++++++ crypto/block/block.tlb | 6 +++++- tl/generate/scheme/ton_api.tl | 4 +++- tl/generate/scheme/ton_api.tlo | Bin 90684 -> 90912 bytes validator/full-node-shard.cpp | 17 ++++++++++++----- validator/full-node-shard.h | 3 ++- validator/full-node-shard.hpp | 4 ++-- validator/full-node.cpp | 11 ++++++----- validator/full-node.hpp | 4 ++-- validator/impl/out-msg-queue-proof.cpp | 19 ++++++++++--------- validator/impl/out-msg-queue-proof.hpp | 15 +++++++++++---- validator/impl/shard.hpp | 7 +++++++ validator/interfaces/out-msg-queue-proof.h | 3 ++- validator/interfaces/shard.h | 1 + validator/interfaces/validator-manager.h | 3 ++- validator/manager-disk.hpp | 3 ++- validator/manager-hardfork.hpp | 3 ++- validator/manager.cpp | 20 ++++++++++++-------- validator/manager.hpp | 3 ++- validator/validator.h | 3 ++- 21 files changed, 109 insertions(+), 48 deletions(-) diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index b9ecfe60ca..14f2c18b07 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -655,6 +655,12 @@ bool EnqueuedMsgDescr::check_key(td::ConstBitPtr key) const { hash_ == key + 96; } +bool ImportedMsgQueueLimits::deserialize(vm::CellSlice& cs) { + return cs.fetch_ulong(8) == 0xd3 // imported_msg_queue_limits#d3 + && cs.fetch_uint_to(32, max_bytes) // max_bytes:# + && cs.fetch_uint_to(32, max_msgs); // max_msgs:# +} + bool ParamLimits::deserialize(vm::CellSlice& cs) { return cs.fetch_ulong(8) == 0xc3 // param_limits#c3 && cs.fetch_uint_to(32, limits_[0]) // underload:uint32 @@ -666,10 +672,16 @@ bool ParamLimits::deserialize(vm::CellSlice& cs) { } bool BlockLimits::deserialize(vm::CellSlice& cs) { - return cs.fetch_ulong(8) == 0x5d // block_limits#5d - && bytes.deserialize(cs) // bytes:ParamLimits - && gas.deserialize(cs) // gas:ParamLimits - && lt_delta.deserialize(cs); // lt_delta:ParamLimits + auto tag = cs.fetch_ulong(8); + if (tag != 0x5d && tag != 0x5e) { + return false; + } + // block_limits#5d + // block_limits_v2#5e + return bytes.deserialize(cs) // bytes:ParamLimits + && gas.deserialize(cs) // gas:ParamLimits + && lt_delta.deserialize(cs) // lt_delta:ParamLimits + && (tag == 0x5d || imported_msg_queue.deserialize(cs)); // imported_msg_queue:ImportedMsgQueueLimits } int ParamLimits::classify(td::uint64 value) const { diff --git a/crypto/block/block.h b/crypto/block/block.h index b6b46a3d6a..5f3f316304 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -216,6 +216,13 @@ static inline std::ostream& operator<<(std::ostream& os, const MsgProcessedUptoC return proc_coll.print(os); } +struct ImportedMsgQueueLimits { + // Default values + td::uint32 max_bytes = 1 << 18; + td::uint32 max_msgs = 40; + bool deserialize(vm::CellSlice& cs); +}; + struct ParamLimits { enum { limits_cnt = 4 }; enum { cl_underload = 0, cl_normal = 1, cl_soft = 2, cl_medium = 3, cl_hard = 4 }; @@ -247,6 +254,7 @@ struct ParamLimits { struct BlockLimits { ParamLimits bytes, gas, lt_delta; ton::LogicalTime start_lt{0}; + ImportedMsgQueueLimits imported_msg_queue; const vm::CellUsageTree* usage_tree{nullptr}; bool deserialize(vm::CellSlice& cs); int classify_size(td::uint64 size) const; diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 2176c0adb6..9e542999af 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -704,9 +704,13 @@ config_gas_prices#_ GasLimitsPrices = ConfigParam 21; param_limits#c3 underload:# soft_limit:# { underload <= soft_limit } hard_limit:# { soft_limit <= hard_limit } = ParamLimits; +imported_msg_queue_limits#d3 max_bytes:# max_msgs:# = ImportedMsgQueueLimits; block_limits#5d bytes:ParamLimits gas:ParamLimits lt_delta:ParamLimits = BlockLimits; - +block_limits_v2#5e bytes:ParamLimits gas:ParamLimits lt_delta:ParamLimits + imported_msg_queue:ImportedMsgQueueLimits + = BlockLimits; + config_mc_block_limits#_ BlockLimits = ConfigParam 22; config_block_limits#_ BlockLimits = ConfigParam 23; diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 6d52b8454b..5d636e9c31 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -415,6 +415,7 @@ tonNode.success = tonNode.Success; tonNode.archiveNotFound = tonNode.ArchiveInfo; tonNode.archiveInfo id:long = tonNode.ArchiveInfo; +tonNode.importedMsgQueueLimits max_bytes:int max_msgs:int = ImportedMsgQueueLimits; tonNode.outMsgQueueProof queue_proof:bytes block_state_proof:bytes msg_count:int = tonNode.OutMsgQueueProof; tonNode.outMsgQueueProofEmpty = tonNode.OutMsgQueueProof; @@ -451,7 +452,8 @@ tonNode.downloadBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.Dat tonNode.downloadKeyBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.DataList; tonNode.getArchiveInfo masterchain_seqno:int = tonNode.ArchiveInfo; tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data; -tonNode.getOutMsgQueueProof block_id:tonNode.blockIdExt dst_workchain:int dst_shard:long = tonNode.OutMsgQueueProof; +tonNode.getOutMsgQueueProof block_id:tonNode.blockIdExt dst_workchain:int dst_shard:long + limits:tonNode.importedMsgQueueLimits = tonNode.OutMsgQueueProof; tonNode.getCapabilities = tonNode.Capabilities; tonNode.getCapabilitiesV2 = tonNode.Capabilities; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 1eed452846aa645ab1b76b678dde81080e579487..60e942922e5b98a51f11eb69ef9e47aaf47f22ce 100644 GIT binary patch delta 213 zcmdmUgmu9&R^CUm^{p77fO{hEV`=WkXTOVi<`(1^m87Ql7N-Z6rk1AqWaeg;6i-|) zx$#4dG2?{IxhCJatwAc~O7ipk@>5dvGO_Al0I9ioOvr}`Xod8%`vw2kaEEi#crD4uZ5zj`*dp{Yo@r1{(k?AyH2N delta 83 zcmZ2*jCIcuR^CUm^{p77fNLV}EzI^8mc(S1@m$7G$~0^4`QFlMj;09oxDF#rGn diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index b34432a932..3c7b80638c 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -666,6 +666,7 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod td::Promise promise) { BlockIdExt block_id = create_block_id(query.block_id_); ShardIdFull dst_shard(query.dst_workchain_, query.dst_shard_); + block::ImportedMsgQueueLimits limits{(td::uint32)query.limits_->max_bytes_, (td::uint32)query.limits_->max_msgs_}; if (!block_id.is_valid_ext()) { promise.set_error(td::Status::Error("invalid block_id")); return; @@ -674,6 +675,10 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_error(td::Status::Error("invalid shard")); return; } + if (limits.max_bytes > (1 << 21)) { + promise.set_error(td::Status::Error("max_bytes is too big")); + return; + } auto P = td::PromiseCreator::lambda( [promise = std::move(promise)](td::Result> R) mutable { if (R.is_error()) { @@ -684,7 +689,7 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod }); VLOG(FULL_NODE_DEBUG) << "Got query getOutMsgQueueProof " << block_id.to_str() << " " << dst_shard.to_str() << " from " << src; - td::actor::create_actor("buildqueueproof", block_id, dst_shard, validator_manager_, + td::actor::create_actor("buildqueueproof", block_id, dst_shard, limits, validator_manager_, std::move(P)) .release(); } @@ -924,7 +929,8 @@ void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, std::stri .release(); } -void FullNodeShardImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, +void FullNodeShardImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise> promise) { // TODO: maybe more complex download (like other requests here) auto &b = choose_neighbour(true); @@ -944,13 +950,14 @@ void FullNodeShardImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardI promise.set_error(td::Status::Error("node doesn't have this block")); }, [&](ton_api::tonNode_outMsgQueueProof &x) { - promise.set_result(OutMsgQueueProof::fetch(block_id, dst_shard, x)); + promise.set_result(OutMsgQueueProof::fetch(block_id, dst_shard, limits, x)); })); }); td::BufferSlice query = create_serialize_tl_object( - create_tl_block_id(block_id), dst_shard.workchain, dst_shard.shard); + create_tl_block_id(block_id), dst_shard.workchain, dst_shard.shard, + create_tl_object(limits.max_bytes, limits.max_msgs)); td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, b.adnl_id, adnl_id_, overlay_id_, - "get_msg_queue", std::move(P), timeout, std::move(query), 1 << 20, rldp_); + "get_msg_queue", std::move(P), timeout, std::move(query), 1 << 22, rldp_); } void FullNodeShardImpl::set_handle(BlockHandle handle, td::Promise promise) { diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 59d00e2641..fca17bf4b4 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -70,7 +70,8 @@ class FullNodeShard : public td::actor::Actor { td::Promise> promise) = 0; virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; - virtual void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, + virtual void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise> promise) = 0; virtual void set_handle(BlockHandle handle, td::Promise promise) = 0; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index c0c7dd4bb6..b2be91081d 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -186,8 +186,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise> promise) override; void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) override; - void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, - td::Promise> promise) override; + void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, + td::Timestamp timeout, td::Promise> promise) override; void set_handle(BlockHandle handle, td::Promise promise) override; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 3b275ea991..eebc49f8b6 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -378,7 +378,8 @@ void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, std::string tm std::move(promise)); } -void FullNodeImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, +void FullNodeImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise> promise) { auto shard = get_shard(block_id.shard_full()); if (shard.empty()) { @@ -386,7 +387,7 @@ void FullNodeImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); return; } - td::actor::send_closure(shard, &FullNodeShard::download_out_msg_queue_proof, block_id, dst_shard, timeout, + td::actor::send_closure(shard, &FullNodeShard::download_out_msg_queue_proof, block_id, dst_shard, limits, timeout, std::move(promise)); } @@ -587,9 +588,9 @@ void FullNodeImpl::start_up() { td::actor::send_closure(id_, &FullNodeImpl::download_archive, masterchain_seqno, std::move(tmp_dir), timeout, std::move(promise)); } - void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, - td::Promise> promise) override { - td::actor::send_closure(id_, &FullNodeImpl::download_out_msg_queue_proof, block_id, dst_shard, timeout, + void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, + td::Timestamp timeout, td::Promise> promise) override { + td::actor::send_closure(id_, &FullNodeImpl::download_out_msg_queue_proof, block_id, dst_shard, limits, timeout, std::move(promise)); } diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 01bd19e4c0..99ea49cc3c 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -74,8 +74,8 @@ class FullNodeImpl : public FullNode { void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise); void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise); - void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, - td::Promise> promise); + void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, + td::Timestamp timeout, td::Promise> promise); void got_key_block_proof(td::Ref proof); void got_zero_block_state(td::Ref state); diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 59aa8aed69..cc8c7e1792 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -47,6 +47,7 @@ static td::Status check_no_prunned(const vm::CellSlice& cs) { } static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, const block::gen::OutMsgQueueInfo::Record& qinfo) { td::uint64 estimated_proof_size = 0; @@ -113,17 +114,16 @@ static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_ dfs_cs(*kv->msg); TRY_STATUS_PREFIX(check_no_prunned(*kv->msg), "invalid message proof: ") - if (estimated_proof_size > OutMsgQueueProof::QUEUE_SIZE_THRESHOLD) { + if (estimated_proof_size >= limits.max_bytes || msg_count >= (long long)limits.max_msgs) { limit_reached = true; } } return limit_reached ? msg_count : -1; } -td::Result> OutMsgQueueProof::build(BlockIdExt block_id, - ShardIdFull dst_shard, - Ref state_root, - Ref block_root) { +td::Result> OutMsgQueueProof::build( + BlockIdExt block_id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, Ref state_root, Ref block_root) { if (!dst_shard.is_valid_ext()) { return td::Status::Error("invalid shard"); } @@ -135,7 +135,7 @@ td::Result> OutMsgQueueProof::b if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { return td::Status::Error("invalid message queue"); } - TRY_RESULT(cnt, process_queue(block_id, dst_shard, qinfo)); + TRY_RESULT(cnt, process_queue(block_id, dst_shard, limits, qinfo)); TRY_RESULT(queue_proof, mpb.extract_proof_boc()); td::BufferSlice block_state_proof; @@ -148,6 +148,7 @@ td::Result> OutMsgQueueProof::b } td::Result> OutMsgQueueProof::fetch(BlockIdExt block_id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, const ton_api::tonNode_outMsgQueueProof& f) { try { Ref block_state_proof; @@ -181,7 +182,7 @@ td::Result> OutMsgQueueProof::fetch(BlockIdExt block_i } TRY_STATUS_PREFIX(check_no_prunned(qinfo.proc_info->prefetch_ref(0)), "invalid proc_info: ") TRY_STATUS_PREFIX(check_no_prunned(qinfo.ihr_pending->prefetch_ref(0)), "invalid ihr_pending: ") - TRY_RESULT(cnt, process_queue(block_id, dst_shard, qinfo)); + TRY_RESULT(cnt, process_queue(block_id, dst_shard, limits, qinfo)); if (cnt != f.msg_count_) { return td::Status::Error(PSTRING() << "invalid msg_count: expected=" << f.msg_count_ << ", found=" << cnt); } @@ -295,7 +296,7 @@ void WaitOutMsgQueueProof::run_net() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_out_msg_queue_proof_request, block_id_, dst_shard_, - priority_, std::move(P)); + limits_, priority_, std::move(P)); } void BuildOutMsgQueueProof::abort_query(td::Status reason) { @@ -349,7 +350,7 @@ void BuildOutMsgQueueProof::got_block_root(Ref root) { } void BuildOutMsgQueueProof::build_proof() { - auto result = OutMsgQueueProof::build(block_id_, dst_shard_, std::move(state_root_), std::move(block_root_)); + auto result = OutMsgQueueProof::build(block_id_, dst_shard_, limits_, std::move(state_root_), std::move(block_root_)); if (result.is_error()) { LOG(ERROR) << "Failed to build msg queue proof: " << result.error(); } diff --git a/validator/impl/out-msg-queue-proof.hpp b/validator/impl/out-msg-queue-proof.hpp index e94f805f40..9c5acfef99 100644 --- a/validator/impl/out-msg-queue-proof.hpp +++ b/validator/impl/out-msg-queue-proof.hpp @@ -31,11 +31,12 @@ class ValidatorManagerInterface; class WaitOutMsgQueueProof : public td::actor::Actor { public: - WaitOutMsgQueueProof(BlockIdExt block_id, ShardIdFull dst_shard, bool local, td::uint32 priority, - td::actor::ActorId manager, td::Timestamp timeout, + WaitOutMsgQueueProof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, bool local, + td::uint32 priority, td::actor::ActorId manager, td::Timestamp timeout, td::Promise> promise) : block_id_(std::move(block_id)) , dst_shard_(dst_shard) + , limits_(limits) , local_(local) , priority_(priority) , manager_(manager) @@ -65,6 +66,7 @@ class WaitOutMsgQueueProof : public td::actor::Actor { private: BlockIdExt block_id_; ShardIdFull dst_shard_; + block::ImportedMsgQueueLimits limits_; bool local_; td::uint32 priority_; @@ -78,10 +80,14 @@ class WaitOutMsgQueueProof : public td::actor::Actor { class BuildOutMsgQueueProof : public td::actor::Actor { public: - BuildOutMsgQueueProof(BlockIdExt block_id, ShardIdFull dst_shard, + BuildOutMsgQueueProof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, td::actor::ActorId manager, td::Promise> promise) - : block_id_(std::move(block_id)), dst_shard_(dst_shard), manager_(manager), promise_(std::move(promise)) { + : block_id_(std::move(block_id)) + , dst_shard_(dst_shard) + , limits_(limits) + , manager_(manager) + , promise_(std::move(promise)) { } void abort_query(td::Status reason); @@ -93,6 +99,7 @@ class BuildOutMsgQueueProof : public td::actor::Actor { private: BlockIdExt block_id_; ShardIdFull dst_shard_; + block::ImportedMsgQueueLimits limits_; td::actor::ActorId manager_; td::Promise> promise_; diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index 83aa1f9a45..a59f26d18e 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -130,6 +130,13 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { auto R = config_->get_size_limits_config(); return R.is_error() ? block::SizeLimitsConfig::ExtMsgLimits() : R.ok_ref().ext_msg_limits; } + block::ImportedMsgQueueLimits get_imported_msg_queue_limits(bool is_masterchain) const override { + auto R = config_->get_block_limits(is_masterchain); + if (R.is_ok() && R.ok()) { + return R.ok()->imported_msg_queue; + } + return {}; + } BlockIdExt last_key_block_id() const override; BlockIdExt next_key_block_id(BlockSeqno seqno) const override; BlockIdExt prev_key_block_id(BlockSeqno seqno) const override; diff --git a/validator/interfaces/out-msg-queue-proof.h b/validator/interfaces/out-msg-queue-proof.h index 5d79249137..524168d0d6 100644 --- a/validator/interfaces/out-msg-queue-proof.h +++ b/validator/interfaces/out-msg-queue-proof.h @@ -35,12 +35,13 @@ struct OutMsgQueueProof : public td::CntObject { td::int32 msg_count_; // -1 - up to end of queue static td::Result> fetch(BlockIdExt block_id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, const ton_api::tonNode_outMsgQueueProof &f); static td::Result> build(BlockIdExt block_id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, Ref state_root, Ref block_root); - static const td::uint64 QUEUE_SIZE_THRESHOLD = 128 * 1024; }; } // namespace validator diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 9ccfa2fb48..2e34374704 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -88,6 +88,7 @@ class MasterchainState : virtual public ShardState { return td::Status::OK(); } virtual block::SizeLimitsConfig::ExtMsgLimits get_ext_msg_limits() const = 0; + virtual block::ImportedMsgQueueLimits get_imported_msg_queue_limits(bool is_masterchain) const = 0; }; } // namespace validator diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index bcc5fd53e3..a4a1dd1672 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -129,7 +129,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_ihr_message(td::Ref message) = 0; virtual void send_top_shard_block_description(td::Ref desc) = 0; virtual void send_block_broadcast(BlockBroadcast broadcast) = 0; - virtual void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, td::uint32 priority, + virtual void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, td::uint32 priority, td::Promise> promise) = 0; virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index d41a6e3427..17a1bffaef 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -255,7 +255,8 @@ class ValidatorManagerImpl : public ValidatorManager { void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast) override { } - void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, td::uint32 priority, + void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, + td::uint32 priority, td::Promise> promise) override { UNREACHABLE(); } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 0a9e8a2b70..65d2b3c0d9 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -321,7 +321,8 @@ class ValidatorManagerImpl : public ValidatorManager { } void send_block_broadcast(BlockBroadcast broadcast) override { } - void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, td::uint32 priority, + void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, + td::uint32 priority, td::Promise> promise) override { UNREACHABLE(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 5c3691bd66..638b741f96 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -630,16 +630,18 @@ void ValidatorManagerImpl::wait_out_msg_queue_proof(BlockIdExt block_id, ShardId std::move(R)); }); auto id = td::actor::create_actor( - "waitmsgqueue", block_id, dst_shard, need_monitor(block_id.shard_full()), priority, actor_id(this), - td::Timestamp::at(timeout.at() + 10.0), std::move(P)) + "waitmsgqueue", block_id, dst_shard, + last_masterchain_state_->get_imported_msg_queue_limits(block_id.is_masterchain()), + need_monitor(block_id.shard_full()), priority, actor_id(this), td::Timestamp::at(timeout.at() + 10.0), + std::move(P)) .release(); wait_out_msg_queue_proof_[key].actor_ = id; it = wait_out_msg_queue_proof_.find(key); } else if (it->second.done_) { promise.set_result(it->second.result_); it->second.remove_at_ = td::Timestamp::in(30.0); + return; } - it->second.waiting_.emplace_back(timeout, priority, std::move(promise)); auto X = it->second.get_timeout(); td::actor::send_closure(it->second.actor_, &WaitOutMsgQueueProof::update_timeout, X.first, X.second); @@ -1097,9 +1099,10 @@ void ValidatorManagerImpl::finished_wait_msg_queue(BlockIdExt block_id, ShardIdF td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_msg_queue, block_id, dst_shard, std::move(R)); }); - auto id = td::actor::create_actor("waitmsgqueue", block_id, dst_shard, - need_monitor(block_id.shard_full()), X.second, - actor_id(this), X.first, std::move(P)) + auto id = td::actor::create_actor( + "waitmsgqueue", block_id, dst_shard, + last_masterchain_state_->get_imported_msg_queue_limits(block_id.is_masterchain()), + need_monitor(block_id.shard_full()), X.second, actor_id(this), X.first, std::move(P)) .release(); it->second.actor_ = id; return; @@ -1511,9 +1514,10 @@ void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast) { } void ValidatorManagerImpl::send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, td::uint32 priority, td::Promise> promise) { - callback_->download_out_msg_queue_proof(id, dst_shard, td::Timestamp::in(10.0), std::move(promise)); + callback_->download_out_msg_queue_proof(id, dst_shard, limits, td::Timestamp::in(10.0), std::move(promise)); } void ValidatorManagerImpl::start_up() { @@ -1839,7 +1843,7 @@ void ValidatorManagerImpl::new_masterchain_block() { // Prepare neighboours' queues for collating masterchain for (const auto &desc : last_masterchain_state_->get_shards()) { wait_out_msg_queue_proof(desc->top_block_id(), ShardIdFull(masterchainId), 0, td::Timestamp::in(10.0), - [](td::Ref) {}); + [](td::Result>) {}); } } diff --git a/validator/manager.hpp b/validator/manager.hpp index 3debccb39d..a37fcb4e0b 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -462,7 +462,8 @@ class ValidatorManagerImpl : public ValidatorManager { void send_ihr_message(td::Ref message) override; void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast) override; - void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, td::uint32 priority, + void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, + td::uint32 priority, td::Promise> promise) override; void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; diff --git a/validator/validator.h b/validator/validator.h index 4ce3f4d021..5fa23467f6 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -140,7 +140,8 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise> promise) = 0; virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; - virtual void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, + virtual void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise> promise) = 0; virtual void new_key_block(BlockHandle handle) = 0; From 8c4bc5b3f15ce0d137a8b4e0298f9d1304291d96 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 26 Jul 2023 12:21:19 +0300 Subject: [PATCH 064/524] Fix sending msg queue queries --- validator/collator-node.cpp | 52 +++++++++++++------------- validator/impl/out-msg-queue-proof.cpp | 28 +++++++------- validator/manager.cpp | 3 +- 3 files changed, 43 insertions(+), 40 deletions(-) diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 2cc06b3403..f5900e0a95 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -74,33 +74,35 @@ void CollatorNode::new_masterchain_block_notification(td::Ref last_masterchain_block_ = state->get_block_id(); last_top_blocks_.clear(); last_top_blocks_[ShardIdFull{masterchainId, shardIdAll}] = last_masterchain_block_; - std::vector next_shards; - if (can_collate_shard(ShardIdFull(masterchainId))) { - next_shards.push_back(ShardIdFull(masterchainId)); - } - for (const auto& desc : state->get_shards()) { - last_top_blocks_[desc->shard()] = desc->top_block_id(); - ShardIdFull shard = desc->shard(); - if (desc->before_split()) { - if (can_collate_shard(shard_child(shard, true))) { - next_shards.push_back(shard_child(shard, true)); - } - if (can_collate_shard(shard_child(shard, false))) { - next_shards.push_back(shard_child(shard, false)); - } - } else if (desc->before_merge()) { - if (is_left_child(shard) && can_collate_shard(shard_parent(shard))) { - next_shards.push_back(shard_parent(shard)); + if (state->get_unix_time() > (td::uint32)td::Clocks::system() - 20) { + std::vector next_shards; + if (can_collate_shard(ShardIdFull(masterchainId))) { + next_shards.push_back(ShardIdFull(masterchainId)); + } + for (const auto& desc : state->get_shards()) { + last_top_blocks_[desc->shard()] = desc->top_block_id(); + ShardIdFull shard = desc->shard(); + if (desc->before_split()) { + if (can_collate_shard(shard_child(shard, true))) { + next_shards.push_back(shard_child(shard, true)); + } + if (can_collate_shard(shard_child(shard, false))) { + next_shards.push_back(shard_child(shard, false)); + } + } else if (desc->before_merge()) { + if (is_left_child(shard) && can_collate_shard(shard_parent(shard))) { + next_shards.push_back(shard_parent(shard)); + } + } else if (can_collate_shard(shard)) { + next_shards.push_back(shard); } - } else if (can_collate_shard(shard)) { - next_shards.push_back(shard); } - } - for (const ShardIdFull& shard : next_shards) { - for (const auto& neighbor : last_top_blocks_) { - if (neighbor.first != shard && block::ShardConfig::is_neighbor(shard, neighbor.first)) { - td::actor::send_closure(manager_, &ValidatorManager::wait_out_msg_queue_proof, neighbor.second, shard, 0, - td::Timestamp::in(10.0), [](td::Ref) {}); + for (const ShardIdFull& shard : next_shards) { + for (const auto& neighbor : last_top_blocks_) { + if (neighbor.first != shard && block::ShardConfig::is_neighbor(shard, neighbor.first)) { + td::actor::send_closure(manager_, &ValidatorManager::wait_out_msg_queue_proof, neighbor.second, shard, 0, + td::Timestamp::in(10.0), [](td::Ref) {}); + } } } } diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index cc8c7e1792..52c59afa04 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -280,20 +280,20 @@ void WaitOutMsgQueueProof::run_local_cont() { } void WaitOutMsgQueueProof::run_net() { - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = block_id_](td::Result> R) { - if (R.is_error()) { - if (R.error().code() == ErrorCode::notready) { - LOG(DEBUG) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); - } else { - LOG(WARNING) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); - } - delay_action([SelfId]() mutable { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::run_net); }, - td::Timestamp::in(0.1)); - } else { - td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::finish_query, R.move_as_ok()); - } - }); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = block_id_, + retry_after = td::Timestamp::in(0.5)](td::Result> R) { + if (R.is_error()) { + if (R.error().code() == ErrorCode::notready) { + LOG(DEBUG) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); + } else { + LOG(WARNING) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); + } + delay_action([SelfId]() mutable { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::run_net); }, + retry_after); + } else { + td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::finish_query, R.move_as_ok()); + } + }); td::actor::send_closure(manager_, &ValidatorManager::send_get_out_msg_queue_proof_request, block_id_, dst_shard_, limits_, priority_, std::move(P)); diff --git a/validator/manager.cpp b/validator/manager.cpp index 638b741f96..d419f80ee3 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1839,7 +1839,8 @@ void ValidatorManagerImpl::new_masterchain_block() { for (auto &c : collator_nodes_) { td::actor::send_closure(c.second.actor, &CollatorNode::new_masterchain_block_notification, last_masterchain_state_); } - if (opts_->validator_mode() == ValidatorManagerOptions::validator_lite_shards && validating_masterchain()) { + if (opts_->validator_mode() == ValidatorManagerOptions::validator_lite_shards && validating_masterchain() && + last_masterchain_state_->get_unix_time() > (td::uint32)td::Clocks::system() - 20) { // Prepare neighboours' queues for collating masterchain for (const auto &desc : last_masterchain_state_->get_shards()) { wait_out_msg_queue_proof(desc->top_block_id(), ShardIdFull(masterchainId), 0, td::Timestamp::in(10.0), From da137fecf55b670d79df25390096396264da380c Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 26 Jul 2023 13:05:16 +0300 Subject: [PATCH 065/524] Extra shard overlay stats --- overlay/overlay.cpp | 7 +++- overlay/overlays.h | 3 ++ tl/generate/scheme/ton_api.tl | 7 +++- tl/generate/scheme/ton_api.tlo | Bin 90912 -> 91588 bytes .../validator-engine-console-query.cpp | 14 ++++++-- validator/full-node-shard.cpp | 34 ++++++++++++++++++ validator/full-node-shard.hpp | 1 + 7 files changed, 62 insertions(+), 4 deletions(-) diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index c4cf8428b1..88518eb7ef 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -661,7 +661,12 @@ void OverlayImpl::get_stats(td::Promisestats_.push_back( create_tl_object("neighbours_cnt", PSTRING() << neighbours_.size())); - promise.set_value(std::move(res)); + callback_->get_stats_extra([promise = std::move(promise), res = std::move(res)](td::Result R) mutable { + if (R.is_ok()) { + res->extra_ = R.move_as_ok(); + } + promise.set_value(std::move(res)); + }); } } // namespace overlay diff --git a/overlay/overlays.h b/overlay/overlays.h index cf153c3a5c..ee225c3b8a 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -170,6 +170,9 @@ class Overlays : public td::actor::Actor { td::Promise promise) { promise.set_value(td::Unit()); } + virtual void get_stats_extra(td::Promise promise) { + promise.set_result(""); + } virtual ~Callback() = default; }; diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 5d636e9c31..0a4134e5f5 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -656,9 +656,14 @@ engine.validator.dhtServersStatus servers:(vector engine.validator.dhtServerStat engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string bdcst_errors:int fec_bdcst_errors:int last_in_query:int last_out_query:int t_out_bytes:int t_in_bytes:int t_out_pckts:int t_in_pckts:int = engine.validator.OverlayStatsNode; -engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) = engine.validator.OverlayStats; +engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) extra:string = engine.validator.OverlayStats; engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; +engine.validator.shardOverlayStats.neighbour id:string proto_verison:int capabilities:long + roundtrip:double unreliability:double has_state:string = engine.validator.shardOverlayStats.Neighbour; +engine.validator.shardOverlayStats shard:string mode:string + neighbours:(vector engine.validator.shardOverlayStats.neighbour) = engine.validator.ShardOverlayStats; + engine.validator.onePerfTimerStat time:int min:double avg:double max:double = engine.validator.OnePerfTimerStat; engine.validator.perfTimerStatsByName name:string stats:(vector engine.validator.OnePerfTimerStat) = engine.validator.PerfTimerStatsByName; engine.validator.perfTimerStats stats:(vector engine.validator.PerfTimerStatsByName) = engine.validator.PerfTimerStats; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 60e942922e5b98a51f11eb69ef9e47aaf47f22ce..974e5ad6d42377302c711e0636525175d496910f 100644 GIT binary patch delta 407 zcmZ2*jP=NAR^CUm^{p77fOjMBWpP$M59vn zu`;+Mv80%Raq>YX@y&N66Bt?7T0Zg9nY?$IxG0hWJ-^h<^o*qZ(xS;1`K348NT*0L zPTIWCBsqr#Wc*}~g$kS?#?50wKJ1eX7k=lFes*7zHMOFoD3O6-@`PD3V0D{K7HKd+ zSP4sxFhSh1XPE@DRSX~pg3Mu_Tv#GLdBq|D7M9%nl*t#)%W!k$!JJV%Io@AZ0_0Xk zsAr3u*>pH;+YW-XZM&T46;w)MgaAJ^iTRM0kVXbx1cD$BtIUgG_yE=vSFF@ zNLXKqhaNS;evw0NxLf5&!@I delta 124 zcmX?dnsvc3R^CUm^{p77fO{kFWpUP@zZTq=ovbJ!yO~Mi7UO0asRYK&C!|v(87FMk zHBHWe%70uW%gw+5QhW25kPq8r#{JrpUo1SsQ{>F1!(rQY5G1qt!6FSN7BGuv=@F*Q OQ&zArZ$7i;#|{9nj5Zqp diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index d0f6ab290e..98a4d324cd 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -934,8 +934,18 @@ td::Status GetOverlaysStatsJsonQuery::receive(td::BufferSlice data) { sb << " \"" << t->key_ << "\": \"" << t->value_ << "\""; } - sb << "\n }\n"; - sb << "}\n"; + sb << "\n }"; + if (!s->extra_.empty()) { + sb << ",\n \"extra\": "; + for (char c : s->extra_) { + if (c == '\n') { + sb << "\n "; + } else { + sb << c; + } + } + } + sb << "\n}\n"; } sb << "]\n"; sb << std::flush; diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 3c7b80638c..a991975463 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -39,6 +39,9 @@ #include "td/utils/Random.h" #include "common/delay.h" +#include "td/utils/JsonBuilder.h" +#include "tl/tl_json.h" +#include "auto/tl/ton_api_json.h" namespace ton { @@ -99,6 +102,9 @@ void FullNodeShardImpl::create_overlay() { td::Promise promise) override { td::actor::send_closure(node_, &FullNodeShardImpl::check_broadcast, src, std::move(data), std::move(promise)); } + void get_stats_extra(td::Promise promise) override { + td::actor::send_closure(node_, &FullNodeShardImpl::get_stats_extra, std::move(promise)); + } Callback(td::actor::ActorId node) : node_(node) { } @@ -1290,6 +1296,34 @@ void FullNodeShardImpl::ping_neighbours() { } } +void FullNodeShardImpl::get_stats_extra(td::Promise promise) { + auto res = create_tl_object(); + res->shard_ = shard_.to_str(); + switch (mode_) { + case active: + res->mode_ = "active"; + break; + case active_temp: + res->mode_ = "active_temp"; + break; + case inactive: + res->mode_ = "inactive"; + break; + } + for (const auto &p : neighbours_) { + const auto &n = p.second; + auto f = create_tl_object(); + f->id_ = n.adnl_id.bits256_value().to_hex(); + f->proto_verison_ = n.proto_version; + f->capabilities_ = n.capabilities; + f->roundtrip_ = n.roundtrip; + f->unreliability_ = n.unreliability; + f->has_state_ = (n.has_state_known ? (n.has_state ? "true" : "false") : "undefined"); + res->neighbours_.push_back(std::move(f)); + } + promise.set_result(td::json_encode(td::ToJson(*res), true)); +} + FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index b2be91081d..af16ccc91a 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -164,6 +164,7 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void check_broadcast(PublicKeyHash src, td::BufferSlice query, td::Promise promise); + void get_stats_extra(td::Promise promise); void remove_neighbour(adnl::AdnlNodeIdShort id); void send_ihr_message(td::BufferSlice data) override; From 783c75fc8569e567c7028c5c929be62f2c7684d6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 26 Jul 2023 16:29:37 +0300 Subject: [PATCH 066/524] Decrease verbosity in out-msg-queue-proof.cpp --- validator/impl/out-msg-queue-proof.cpp | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 52c59afa04..365913760f 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -198,13 +198,8 @@ void WaitOutMsgQueueProof::alarm() { void WaitOutMsgQueueProof::abort_query(td::Status reason) { if (promise_) { - if (priority_ > 0 || (reason.code() != ErrorCode::timeout && reason.code() != ErrorCode::notready)) { - LOG(WARNING) << "aborting wait msg queue query for " << block_id_.to_str() << " priority=" << priority_ << ": " - << reason; - } else { - LOG(DEBUG) << "aborting wait msg queue query for " << block_id_.to_str() << " priority=" << priority_ << ": " - << reason; - } + LOG(DEBUG) << "aborting wait msg queue query for " << block_id_.to_str() << " priority=" << priority_ << ": " + << reason; promise_.set_error( reason.move_as_error_prefix(PSTRING() << "failed to get msg queue for " << block_id_.to_str() << ": ")); } @@ -283,11 +278,7 @@ void WaitOutMsgQueueProof::run_net() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = block_id_, retry_after = td::Timestamp::in(0.5)](td::Result> R) { if (R.is_error()) { - if (R.error().code() == ErrorCode::notready) { - LOG(DEBUG) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); - } else { - LOG(WARNING) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); - } + LOG(DEBUG) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); delay_action([SelfId]() mutable { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::run_net); }, retry_after); } else { From 503e10c05acd4c1d346f42d5ad4fd324cdf5b1e6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 26 Jul 2023 17:28:14 +0300 Subject: [PATCH 067/524] Fix choose_neighbour in full-node-shard --- validator/full-node-shard.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index a991975463..7938111a31 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -1194,6 +1194,10 @@ const Neighbour &FullNodeShardImpl::choose_neighbour(bool require_state) const { return Neighbour::zero; } + double min_unreliability = 1e9; + for (auto &x : neighbours_) { + min_unreliability = std::min(min_unreliability, x.second.unreliability); + } for (int attempt = 0; attempt < (require_state ? 2 : 1); ++attempt) { const Neighbour *best = nullptr; td::uint32 sum = 0; @@ -1207,7 +1211,7 @@ const Neighbour &FullNodeShardImpl::choose_neighbour(bool require_state) const { continue; } } - auto unr = static_cast(x.second.unreliability); + auto unr = static_cast(x.second.unreliability - min_unreliability); if (x.second.proto_version < proto_version()) { unr += 4; From 44ba04093475d6514903854751811a68acdaddd2 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 27 Jul 2023 17:15:11 +0300 Subject: [PATCH 068/524] Don't store candidates on collator nodes --- create-hardfork/create-hardfork.cpp | 3 ++- test/test-ton-collator.cpp | 3 ++- validator/collator-node.cpp | 3 ++- validator/fabric.h | 5 +++-- validator/impl/collator-impl.h | 4 +++- validator/impl/collator.cpp | 21 +++++++++++++-------- validator/impl/fabric.cpp | 11 ++++++----- 7 files changed, 31 insertions(+), 19 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 3fb57f41b5..b5c151a75b 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -271,7 +271,8 @@ class HardforkCreator : public td::actor::Actor { td::Promise promise) override { } - void download_out_msg_queue_proof(ton::BlockIdExt block_id, ton::ShardIdFull dst_shard, td::Timestamp timeout, + void download_out_msg_queue_proof(ton::BlockIdExt block_id, ton::ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise> promise) override { } diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 8ef469b5f0..1bd1cadae0 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -372,7 +372,8 @@ class TestNode : public td::actor::Actor { td::Promise promise) override { } - void download_out_msg_queue_proof(ton::BlockIdExt block_id, ton::ShardIdFull dst_shard, td::Timestamp timeout, + void download_out_msg_queue_proof(ton::BlockIdExt block_id, ton::ShardIdFull dst_shard, + block::ImportedMsgQueueLimits, td::Timestamp timeout, td::Promise> promise) override { } diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index f5900e0a95..b9992d4006 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -272,7 +272,8 @@ void CollatorNode::receive_query_cont(ShardIdFull shard, td::Ref prev_blocks, Ed25519_PublicKey creator, td::Promise promise) { run_collate_query(shard, min_mc_state->get_block_id(), std::move(prev_blocks), creator, - min_mc_state->get_validator_set(shard), manager_, td::Timestamp::in(10.0), std::move(promise)); + min_mc_state->get_validator_set(shard), manager_, td::Timestamp::in(10.0), std::move(promise), + CollateMode::skip_store_candidate); } void CollatorNode::process_result(std::shared_ptr cache_entry, td::Result R) { diff --git a/validator/fabric.h b/validator/fabric.h index d1611bea3a..bd26b0e390 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -26,6 +26,7 @@ namespace ton { namespace validator { enum ValidateMode { fake = 1, full_collated_data = 2 }; +enum CollateMode { skip_store_candidate = 1 }; td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_); td::actor::ActorOwn create_liteserver_cache_actor(td::actor::ActorId manager, @@ -83,9 +84,9 @@ void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, unsigned mode = 0); void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - Ed25519_PublicKey local_id, td::Ref validator_set, + Ed25519_PublicKey creator, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise); + td::Promise promise, unsigned mode = 0); void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 2839a0e5b7..0a0d59ca6b 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -75,6 +75,7 @@ class Collator final : public td::actor::Actor { td::Timestamp timeout; td::Timestamp queue_cleanup_timeout_, soft_timeout_, medium_timeout_; td::Promise main_promise; + unsigned mode_ = 0; ton::BlockSeqno last_block_seqno{0}; ton::BlockSeqno prev_mc_block_seqno{0}; ton::BlockSeqno new_block_seqno{0}; @@ -89,7 +90,8 @@ class Collator final : public td::actor::Actor { public: Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, Ref validator_set, Ed25519_PublicKey collator_id, - td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); + td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, + unsigned mode); ~Collator() override = default; bool is_busy() const { return busy_; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index f11bd1169f..b254b70c9e 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -57,7 +57,7 @@ static inline bool dbg(int c) { Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, td::Ref validator_set, Ed25519_PublicKey collator_id, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise) + td::Promise promise, unsigned mode) : shard_(shard) , is_hardfork_(is_hardfork) , min_mc_block_id{min_masterchain_block_id} @@ -71,6 +71,7 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha , soft_timeout_(td::Timestamp::at(timeout.at() - 3.0)) , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) , main_promise(std::move(promise)) + , mode_(mode) , perf_timer_("collate", 0.1, [manager](double duration) { send_closure(manager, &ValidatorManager::add_perf_timer_stat, "collate", duration); }) { @@ -4200,13 +4201,17 @@ bool Collator::create_block_candidate() { << consensus_config.max_collated_data_size << ")"); } // 4. save block candidate - LOG(INFO) << "saving new BlockCandidate"; - td::actor::send_closure_later(manager, &ValidatorManager::set_block_candidate, block_candidate->id, - block_candidate->clone(), [self = get_self()](td::Result saved) -> void { - LOG(DEBUG) << "got answer to set_block_candidate"; - td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, - std::move(saved)); - }); + if (mode_ & CollateMode::skip_store_candidate) { + td::actor::send_closure_later(actor_id(this), &Collator::return_block_candidate, td::Unit()); + } else { + LOG(INFO) << "saving new BlockCandidate"; + td::actor::send_closure_later(manager, &ValidatorManager::set_block_candidate, block_candidate->id, + block_candidate->clone(), [self = get_self()](td::Result saved) -> void { + LOG(DEBUG) << "got answer to set_block_candidate"; + td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, + std::move(saved)); + }); + } // 5. communicate about bad and delayed external messages if (!bad_ext_msgs_.empty() || !delay_ext_msgs_.empty()) { LOG(INFO) << "sending complete_external_messages() to Manager"; diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 5d8b644001..a5c52f7cc7 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -210,9 +210,9 @@ void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, } void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - Ed25519_PublicKey collator_id, td::Ref validator_set, + Ed25519_PublicKey creator, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise) { + td::Promise promise, unsigned mode) { BlockSeqno seqno = 0; for (auto& p : prev) { if (p.seqno() > seqno) { @@ -220,8 +220,8 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc } } td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, false, - min_masterchain_block_id, std::move(prev), std::move(validator_set), collator_id, - std::move(manager), timeout, std::move(promise)) + min_masterchain_block_id, std::move(prev), std::move(validator_set), creator, + std::move(manager), timeout, std::move(promise), mode) .release(); } @@ -236,7 +236,8 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b } td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, min_masterchain_block_id, std::move(prev), td::Ref{}, - Ed25519_PublicKey{Bits256::zero()}, std::move(manager), timeout, std::move(promise)) + Ed25519_PublicKey{Bits256::zero()}, std::move(manager), timeout, std::move(promise), + 0) .release(); } From 5c02459fd8a1cc07f03633c7d095982bc1ab9730 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 31 Jul 2023 18:12:09 +0300 Subject: [PATCH 069/524] Optimize importing out queues --- create-hardfork/create-hardfork.cpp | 7 +- crypto/block/block.h | 4 +- test/test-ton-collator.cpp | 6 +- tl/generate/scheme/ton_api.tl | 4 +- tl/generate/scheme/ton_api.tlo | Bin 91588 -> 91588 bytes validator/collator-node.cpp | 33 +- validator/full-node-shard.cpp | 82 ++-- validator/full-node-shard.h | 4 +- validator/full-node-shard.hpp | 5 +- validator/full-node.cpp | 24 +- validator/full-node.hpp | 5 +- validator/impl/collator-impl.h | 3 +- validator/impl/collator.cpp | 38 +- validator/impl/out-msg-queue-proof.cpp | 499 ++++++++++++++------- validator/impl/out-msg-queue-proof.hpp | 94 ++-- validator/interfaces/out-msg-queue-proof.h | 33 +- validator/interfaces/validator-manager.h | 6 +- validator/manager-disk.hpp | 10 +- validator/manager-hardfork.hpp | 10 +- validator/manager.cpp | 123 ++--- validator/manager.hpp | 16 +- validator/validator.h | 10 +- 22 files changed, 577 insertions(+), 439 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index b5c151a75b..165748f98c 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -268,12 +268,11 @@ class HardforkCreator : public td::actor::Actor { td::Promise> promise) override { } void download_archive(ton::BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) override { } - void download_out_msg_queue_proof(ton::BlockIdExt block_id, ton::ShardIdFull dst_shard, - block::ImportedMsgQueueLimits limits, td::Timestamp timeout, - td::Promise> promise) override { + void download_out_msg_queue_proof( + ton::ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, + td::Timestamp timeout, td::Promise>> promise) override { } void new_key_block(ton::validator::BlockHandle handle) override { diff --git a/crypto/block/block.h b/crypto/block/block.h index 5f3f316304..6a94f0aad7 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -218,8 +218,8 @@ static inline std::ostream& operator<<(std::ostream& os, const MsgProcessedUptoC struct ImportedMsgQueueLimits { // Default values - td::uint32 max_bytes = 1 << 18; - td::uint32 max_msgs = 40; + td::uint32 max_bytes = 1 << 19; + td::uint32 max_msgs = 500; bool deserialize(vm::CellSlice& cs); }; diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 1bd1cadae0..aa4c256b62 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -372,9 +372,9 @@ class TestNode : public td::actor::Actor { td::Promise promise) override { } - void download_out_msg_queue_proof(ton::BlockIdExt block_id, ton::ShardIdFull dst_shard, - block::ImportedMsgQueueLimits, td::Timestamp timeout, - td::Promise> promise) override { + void download_out_msg_queue_proof( + ton::ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, + td::Timestamp timeout, td::Promise>> promise) override { } void new_key_block(ton::validator::BlockHandle handle) override { diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 0a4134e5f5..6d64b692b1 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -416,7 +416,7 @@ tonNode.archiveNotFound = tonNode.ArchiveInfo; tonNode.archiveInfo id:long = tonNode.ArchiveInfo; tonNode.importedMsgQueueLimits max_bytes:int max_msgs:int = ImportedMsgQueueLimits; -tonNode.outMsgQueueProof queue_proof:bytes block_state_proof:bytes msg_count:int = tonNode.OutMsgQueueProof; +tonNode.outMsgQueueProof queue_proofs:bytes block_state_proofs:bytes msg_counts:(vector int) = tonNode.OutMsgQueueProof; tonNode.outMsgQueueProofEmpty = tonNode.OutMsgQueueProof; tonNode.forgetPeer = tonNode.ForgetPeer; @@ -452,7 +452,7 @@ tonNode.downloadBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.Dat tonNode.downloadKeyBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.DataList; tonNode.getArchiveInfo masterchain_seqno:int = tonNode.ArchiveInfo; tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data; -tonNode.getOutMsgQueueProof block_id:tonNode.blockIdExt dst_workchain:int dst_shard:long +tonNode.getOutMsgQueueProof dst_shard:tonNode.shardId blocks:(vector tonNode.blockIdExt) limits:tonNode.importedMsgQueueLimits = tonNode.OutMsgQueueProof; tonNode.getCapabilities = tonNode.Capabilities; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 974e5ad6d42377302c711e0636525175d496910f..9ecd272dc182524dcfa66469b1b53029f87bafa0 100644 GIT binary patch delta 193 zcmX?dn)S$O)(vm;SqwyH{Mr0L-{LIGC26LZ$vS^!WI;kVj|urOGcYhnKfABVQ&^f> zni^kFl%JngJb7Y^w5U*0PJVKBd~r!)2~3=UVe-cyac-{M;`I3B{L;LV;>r0XhG63- zU-@ew0W%b6CPR@kn+}I<+d;71GXHWCSQI0DTBi5JG0Jel)G<&08^{ zh9YM+9S+;JgJ8={{^cZSzO{6|DqWJF=a-+7s+XQx;$K?gTbv#UvM2y#4@@`9 last_masterchain_block_ = state->get_block_id(); last_top_blocks_.clear(); last_top_blocks_[ShardIdFull{masterchainId, shardIdAll}] = last_masterchain_block_; - if (state->get_unix_time() > (td::uint32)td::Clocks::system() - 20) { - std::vector next_shards; - if (can_collate_shard(ShardIdFull(masterchainId))) { - next_shards.push_back(ShardIdFull(masterchainId)); - } - for (const auto& desc : state->get_shards()) { - last_top_blocks_[desc->shard()] = desc->top_block_id(); - ShardIdFull shard = desc->shard(); - if (desc->before_split()) { - if (can_collate_shard(shard_child(shard, true))) { - next_shards.push_back(shard_child(shard, true)); - } - if (can_collate_shard(shard_child(shard, false))) { - next_shards.push_back(shard_child(shard, false)); - } - } else if (desc->before_merge()) { - if (is_left_child(shard) && can_collate_shard(shard_parent(shard))) { - next_shards.push_back(shard_parent(shard)); - } - } else if (can_collate_shard(shard)) { - next_shards.push_back(shard); - } - } - for (const ShardIdFull& shard : next_shards) { - for (const auto& neighbor : last_top_blocks_) { - if (neighbor.first != shard && block::ShardConfig::is_neighbor(shard, neighbor.first)) { - td::actor::send_closure(manager_, &ValidatorManager::wait_out_msg_queue_proof, neighbor.second, shard, 0, - td::Timestamp::in(10.0), [](td::Ref) {}); - } - } - } + for (const auto& desc : state->get_shards()) { + last_top_blocks_[desc->shard()] = desc->top_block_id(); } if (validators_.empty() || state->is_key_state()) { validators_.clear(); diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 7938111a31..5fcd62c648 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -670,18 +670,26 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getOutMsgQueueProof &query, td::Promise promise) { - BlockIdExt block_id = create_block_id(query.block_id_); - ShardIdFull dst_shard(query.dst_workchain_, query.dst_shard_); - block::ImportedMsgQueueLimits limits{(td::uint32)query.limits_->max_bytes_, (td::uint32)query.limits_->max_msgs_}; - if (!block_id.is_valid_ext()) { - promise.set_error(td::Status::Error("invalid block_id")); - return; + std::vector blocks; + for (const auto& x : query.blocks_) { + BlockIdExt id = create_block_id(x); + if (!id.is_valid_ext()) { + promise.set_error(td::Status::Error("invalid block_id")); + return; + } + if (!shard_is_ancestor(shard_, id.shard_full())) { + promise.set_error(td::Status::Error("query in wrong overlay")); + return; + } + blocks.push_back(create_block_id(x)); } + ShardIdFull dst_shard = create_shard_id(query.dst_shard_); if (!dst_shard.is_valid_ext()) { promise.set_error(td::Status::Error("invalid shard")); return; } - if (limits.max_bytes > (1 << 21)) { + block::ImportedMsgQueueLimits limits{(td::uint32)query.limits_->max_bytes_, (td::uint32)query.limits_->max_msgs_}; + if (limits.max_bytes > (1 << 24)) { promise.set_error(td::Status::Error("max_bytes is too big")); return; } @@ -693,10 +701,10 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_result(serialize_tl_object(R.move_as_ok(), true)); } }); - VLOG(FULL_NODE_DEBUG) << "Got query getOutMsgQueueProof " << block_id.to_str() << " " << dst_shard.to_str() - << " from " << src; - td::actor::create_actor("buildqueueproof", block_id, dst_shard, limits, validator_manager_, - std::move(P)) + VLOG(FULL_NODE_DEBUG) << "Got query getOutMsgQueueProof (" << blocks.size() << " blocks) to shard " + << dst_shard.to_str() << " from " << src; + td::actor::create_actor("buildqueueproof", dst_shard, std::move(blocks), limits, + validator_manager_, std::move(P)) .release(); } @@ -935,33 +943,43 @@ void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, std::stri .release(); } -void FullNodeShardImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, +void FullNodeShardImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Timestamp timeout, - td::Promise> promise) { + td::Promise>> promise) { // TODO: maybe more complex download (like other requests here) auto &b = choose_neighbour(true); if (b.adnl_id == adnl::AdnlNodeIdShort::zero()) { promise.set_error(td::Status::Error(ErrorCode::notready, "no nodes")); return; } - auto P = td::PromiseCreator::lambda( - [=, promise = create_neighbour_promise(b, std::move(promise), true)](td::Result R) mutable { - if (R.is_error()) { - promise.set_result(R.move_as_error()); - return; - } - TRY_RESULT_PROMISE(promise, f, fetch_tl_object(R.move_as_ok(), true)); - ton_api::downcast_call(*f, td::overloaded( - [&](ton_api::tonNode_outMsgQueueProofEmpty &x) { - promise.set_error(td::Status::Error("node doesn't have this block")); - }, - [&](ton_api::tonNode_outMsgQueueProof &x) { - promise.set_result(OutMsgQueueProof::fetch(block_id, dst_shard, limits, x)); - })); - }); + std::vector> blocks_tl; + for (const BlockIdExt &id : blocks) { + blocks_tl.push_back(create_tl_block_id(id)); + } td::BufferSlice query = create_serialize_tl_object( - create_tl_block_id(block_id), dst_shard.workchain, dst_shard.shard, + create_tl_shard_id(dst_shard), std::move(blocks_tl), create_tl_object(limits.max_bytes, limits.max_msgs)); + + auto P = td::PromiseCreator::lambda([=, promise = create_neighbour_promise(b, std::move(promise), true), + blocks = std::move(blocks)](td::Result R) mutable { + if (R.is_error()) { + promise.set_result(R.move_as_error()); + return; + } + TRY_RESULT_PROMISE(promise, f, fetch_tl_object(R.move_as_ok(), true)); + ton_api::downcast_call( + *f, td::overloaded( + [&](ton_api::tonNode_outMsgQueueProofEmpty &x) { + promise.set_error(td::Status::Error("node doesn't have this block")); + }, + [&](ton_api::tonNode_outMsgQueueProof &x) { + delay_action( + [=, promise = std::move(promise), blocks = std::move(blocks), x = std::move(x)]() mutable { + promise.set_result(OutMsgQueueProof::fetch(dst_shard, blocks, limits, x)); + }, + td::Timestamp::now()); + })); + }); td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, b.adnl_id, adnl_id_, overlay_id_, "get_msg_queue", std::move(P), timeout, std::move(query), 1 << 22, rldp_); } @@ -1356,9 +1374,9 @@ td::actor::ActorOwn FullNodeShard::create( td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, FullNodeShardMode mode) { - return td::actor::create_actor("tonnode", shard, local_id, adnl_id, zero_state_file_hash, config, - keyring, adnl, rldp, rldp2, overlays, validator_manager, client, - mode); + return td::actor::create_actor(PSTRING() << "fullnode" << shard.to_str(), shard, local_id, adnl_id, + zero_state_file_hash, config, keyring, adnl, rldp, rldp2, overlays, + validator_manager, client, mode); } } // namespace fullnode diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index fca17bf4b4..2131939fe8 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -70,9 +70,9 @@ class FullNodeShard : public td::actor::Actor { td::Promise> promise) = 0; virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; - virtual void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, + virtual void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Timestamp timeout, - td::Promise> promise) = 0; + td::Promise>> promise) = 0; virtual void set_handle(BlockHandle handle, td::Promise promise) = 0; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index af16ccc91a..f2b8bb6695 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -187,8 +187,9 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise> promise) override; void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) override; - void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, - td::Timestamp timeout, td::Promise> promise) override; + void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) override; void set_handle(BlockHandle handle, td::Promise promise) override; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index eebc49f8b6..bd41327057 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -378,17 +378,22 @@ void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, std::string tm std::move(promise)); } -void FullNodeImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, +void FullNodeImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Timestamp timeout, - td::Promise> promise) { - auto shard = get_shard(block_id.shard_full()); + td::Promise>> promise) { + if (blocks.empty()) { + promise.set_value({}); + return; + } + // All blocks are expected to have the same minsplit shard prefix + auto shard = get_shard(blocks[0].shard_full()); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping download msg queue query to unknown shard"; promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); return; } - td::actor::send_closure(shard, &FullNodeShard::download_out_msg_queue_proof, block_id, dst_shard, limits, timeout, - std::move(promise)); + td::actor::send_closure(shard, &FullNodeShard::download_out_msg_queue_proof, dst_shard, std::move(blocks), limits, + timeout, std::move(promise)); } td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { @@ -588,10 +593,11 @@ void FullNodeImpl::start_up() { td::actor::send_closure(id_, &FullNodeImpl::download_archive, masterchain_seqno, std::move(tmp_dir), timeout, std::move(promise)); } - void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, - td::Timestamp timeout, td::Promise> promise) override { - td::actor::send_closure(id_, &FullNodeImpl::download_out_msg_queue_proof, block_id, dst_shard, limits, timeout, - std::move(promise)); + void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) override { + td::actor::send_closure(id_, &FullNodeImpl::download_out_msg_queue_proof, dst_shard, std::move(blocks), limits, + timeout, std::move(promise)); } void new_key_block(BlockHandle handle) override { diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 99ea49cc3c..fd0240cd9d 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -74,8 +74,9 @@ class FullNodeImpl : public FullNode { void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise); void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise); - void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, - td::Timestamp timeout, td::Promise> promise); + void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise); void got_key_block_proof(td::Ref proof); void got_zero_block_state(td::Ref state); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 0a0d59ca6b..10953a97f4 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -233,7 +233,8 @@ class Collator final : public td::actor::Actor { void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res); bool fix_one_processed_upto(block::MsgProcessedUpto& proc, const ton::ShardIdFull& owner); bool fix_processed_upto(block::MsgProcessedUptoCollection& upto); - void got_neighbor_msg_queue(unsigned i, td::Result> R); + void got_neighbor_msg_queues(td::Result>> R); + void got_neighbor_msg_queue(unsigned i, Ref res); bool adjust_shard_config(); bool store_shard_fees(ShardIdFull shard, const block::CurrencyCollection& fees, const block::CurrencyCollection& created); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index b254b70c9e..d2ace04f63 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -632,27 +632,45 @@ bool Collator::request_neighbor_msg_queues() { } neighbors_.emplace_back(*shard_ptr); } + std::vector top_blocks; unsigned i = 0; for (block::McShardDescr& descr : neighbors_) { LOG(DEBUG) << "neighbor #" << i << " : " << descr.blk_.to_str(); - ++pending; - send_closure_later(manager, &ValidatorManager::wait_out_msg_queue_proof, descr.blk_, shard_, priority(), timeout, - [self = get_self(), i](td::Result> res) { - LOG(DEBUG) << "got msg queue for neighbor #" << i; - send_closure_later(std::move(self), &Collator::got_neighbor_msg_queue, i, std::move(res)); - }); + top_blocks.push_back(descr.blk_); ++i; } + ++pending; + td::actor::send_closure_later( + manager, &ValidatorManager::wait_neighbor_msg_queue_proofs, shard_, std::move(top_blocks), timeout, + [self = get_self()](td::Result>> res) { + td::actor::send_closure_later(std::move(self), &Collator::got_neighbor_msg_queues, std::move(res)); + }); return true; } -void Collator::got_neighbor_msg_queue(unsigned i, td::Result> R) { +void Collator::got_neighbor_msg_queues(td::Result>> R) { --pending; if (R.is_error()) { - fatal_error(R.move_as_error()); + fatal_error(R.move_as_error_prefix("failed to get neighbor msg queues: ")); return; } + LOG(INFO) << "neighbor output queues fetched"; auto res = R.move_as_ok(); + unsigned i = 0; + for (block::McShardDescr& descr : neighbors_) { + LOG(DEBUG) << "neighbor #" << i << " : " << descr.blk_.to_str(); + auto it = res.find(descr.blk_); + if (it == res.end()) { + fatal_error(PSTRING() << "no msg queue from neighbor #" << i); + return; + } + got_neighbor_msg_queue(i, it->second); + ++i; + } + check_pending(); +} + +void Collator::got_neighbor_msg_queue(unsigned i, Ref res) { BlockIdExt block_id = neighbors_.at(i).blk_; if (res->block_state_proof_.not_null() && !block_id.is_masterchain()) { block_state_proofs_.emplace(block_id.root_hash, res->block_state_proof_); @@ -732,10 +750,6 @@ void Collator::got_neighbor_msg_queue(unsigned i, td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_shard, - block::ImportedMsgQueueLimits limits, - const block::gen::OutMsgQueueInfo::Record& qinfo) { +static td::Result> process_queue( + ShardIdFull dst_shard, std::vector> blocks, + block::ImportedMsgQueueLimits limits) { td::uint64 estimated_proof_size = 0; td::HashSet visited; @@ -66,14 +66,18 @@ static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_ dfs(cs.prefetch_ref(i)); } }; - TRY_STATUS_PREFIX(check_no_prunned(*qinfo.proc_info), "invalid proc_info proof: ") - TRY_STATUS_PREFIX(check_no_prunned(*qinfo.ihr_pending), "invalid ihr_pending proof: ") - dfs_cs(*qinfo.proc_info); - dfs_cs(*qinfo.ihr_pending); - - block::OutputQueueMerger queue_merger{ - dst_shard, {block::OutputQueueMerger::Neighbor{block_id, qinfo.out_queue->prefetch_ref()}}}; - td::int32 msg_count = 0; + std::vector neighbors; + for (auto& b : blocks) { + TRY_STATUS_PREFIX(check_no_prunned(*b.second.proc_info), "invalid proc_info proof: ") + TRY_STATUS_PREFIX(check_no_prunned(*b.second.ihr_pending), "invalid ihr_pending proof: ") + dfs_cs(*b.second.proc_info); + dfs_cs(*b.second.ihr_pending); + neighbors.emplace_back(b.first, b.second.out_queue->prefetch_ref()); + } + + block::OutputQueueMerger queue_merger{dst_shard, std::move(neighbors)}; + std::vector msg_count(blocks.size()); + td::int32 msg_count_total = 0; bool limit_reached = false; while (!queue_merger.is_eof()) { @@ -87,7 +91,8 @@ static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_ if (limit_reached) { break; } - ++msg_count; + ++msg_count[kv->source]; + ++msg_count_total; // TODO: Get processed_upto from destination shard (in request?) /* @@ -114,234 +119,410 @@ static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_ dfs_cs(*kv->msg); TRY_STATUS_PREFIX(check_no_prunned(*kv->msg), "invalid message proof: ") - if (estimated_proof_size >= limits.max_bytes || msg_count >= (long long)limits.max_msgs) { + if (estimated_proof_size >= limits.max_bytes || msg_count_total >= (long long)limits.max_msgs) { limit_reached = true; } } - return limit_reached ? msg_count : -1; + if (!limit_reached) { + std::fill(msg_count.begin(), msg_count.end(), -1); + } + return msg_count; } td::Result> OutMsgQueueProof::build( - BlockIdExt block_id, ShardIdFull dst_shard, - block::ImportedMsgQueueLimits limits, Ref state_root, Ref block_root) { + ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits) { if (!dst_shard.is_valid_ext()) { return td::Status::Error("invalid shard"); } + if (blocks.empty()) { + return create_tl_object(td::BufferSlice{}, td::BufferSlice{}, + std::vector{}); + } + + std::vector> block_state_proofs; + for (auto& block : blocks) { + if (block.id.seqno() != 0) { + if (block.block_root.is_null()) { + return td::Status::Error("block is null"); + } + TRY_RESULT(proof, create_block_state_proof(block.block_root)); + block_state_proofs.push_back(std::move(proof)); + } + if (!block::ShardConfig::is_neighbor(dst_shard, block.id.shard_full())) { + return td::Status::Error("shards are not neighbors"); + } + } + TRY_RESULT(block_state_proof, vm::std_boc_serialize_multi(block_state_proofs)); - vm::MerkleProofBuilder mpb{std::move(state_root)}; - TRY_RESULT(state, ShardStateQ::fetch(block_id, {}, mpb.root())); - TRY_RESULT(outq_descr, state->message_queue()); - block::gen::OutMsgQueueInfo::Record qinfo; - if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { - return td::Status::Error("invalid message queue"); + vm::Dictionary states_dict_pure{32}; + for (size_t i = 0; i < blocks.size(); ++i) { + if (blocks[i].state_root.is_null()) { + return td::Status::Error("state is null"); + } + states_dict_pure.set_ref(td::BitArray<32>{(long long)i}, blocks[i].state_root); } - TRY_RESULT(cnt, process_queue(block_id, dst_shard, limits, qinfo)); - TRY_RESULT(queue_proof, mpb.extract_proof_boc()); - td::BufferSlice block_state_proof; - if (block_id.seqno() != 0) { - TRY_RESULT(proof, create_block_state_proof(std::move(block_root))); - TRY_RESULT_ASSIGN(block_state_proof, vm::std_boc_serialize(std::move(proof), 31)); + vm::MerkleProofBuilder mpb{states_dict_pure.get_root_cell()}; + vm::Dictionary states_dict{mpb.root(), 32}; + std::vector> data(blocks.size()); + for (size_t i = 0; i < blocks.size(); ++i) { + data[i].first = blocks[i].id; + TRY_RESULT(state, ShardStateQ::fetch(blocks[i].id, {}, states_dict.lookup_ref(td::BitArray<32>{(long long)i}))); + TRY_RESULT(outq_descr, state->message_queue()); + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), data[i].second)) { + return td::Status::Error("invalid message queue"); + } } + TRY_RESULT(msg_count, process_queue(dst_shard, std::move(data), limits)); + + TRY_RESULT(proof, mpb.extract_proof()); + vm::Dictionary states_dict_proof{vm::CellSlice{vm::NoVm(), proof}.prefetch_ref(), 32}; + std::vector> state_proofs; + for (size_t i = 0; i < blocks.size(); ++i) { + td::Ref proof_raw = states_dict_proof.lookup_ref(td::BitArray<32>{(long long)i}); + CHECK(proof_raw.not_null()); + state_proofs.push_back(vm::CellBuilder::create_merkle_proof(proof_raw)); + } + TRY_RESULT(queue_proof, vm::std_boc_serialize_multi(state_proofs)); return create_tl_object(std::move(queue_proof), std::move(block_state_proof), - cnt); + std::move(msg_count)); } -td::Result> OutMsgQueueProof::fetch(BlockIdExt block_id, ShardIdFull dst_shard, - block::ImportedMsgQueueLimits limits, - const ton_api::tonNode_outMsgQueueProof& f) { +td::Result>> OutMsgQueueProof::fetch(ShardIdFull dst_shard, + std::vector blocks, + block::ImportedMsgQueueLimits limits, + const ton_api::tonNode_outMsgQueueProof& f) { try { - Ref block_state_proof; - td::Bits256 state_root_hash; - if (block_id.seqno() == 0) { - if (!f.block_state_proof_.empty()) { - return td::Status::Error("expected empty block state proof"); - } - state_root_hash = block_id.root_hash; - } else { - TRY_RESULT_ASSIGN(block_state_proof, vm::std_boc_deserialize(f.block_state_proof_.as_slice())); - TRY_RESULT_ASSIGN(state_root_hash, unpack_block_state_proof(block_id, block_state_proof)); + std::vector> res; + TRY_RESULT(queue_proofs, vm::std_boc_deserialize_multi(f.queue_proofs_, (int)blocks.size())); + TRY_RESULT(block_state_proofs, vm::std_boc_deserialize_multi(f.block_state_proofs_, (int)blocks.size())); + if (queue_proofs.size() != blocks.size()) { + return td::Status::Error("invalid size of queue_proofs"); } - - TRY_RESULT(queue_proof, vm::std_boc_deserialize(f.queue_proof_.as_slice())); - auto virtual_root = vm::MerkleProof::virtualize(queue_proof, 1); - if (virtual_root.is_null()) { - return td::Status::Error("invalid queue proof"); + if (f.msg_counts_.size() != blocks.size()) { + return td::Status::Error("invalid size of msg_counts"); } - if (virtual_root->get_hash().as_slice() != state_root_hash.as_slice()) { - return td::Status::Error("state root hash mismatch"); + size_t j = 0; + std::vector> data(blocks.size()); + for (size_t i = 0; i < blocks.size(); ++i) { + td::Bits256 state_root_hash; + Ref block_state_proof = {}; + if (blocks[i].seqno() == 0) { + state_root_hash = blocks[i].root_hash; + } else { + if (j == block_state_proofs.size()) { + return td::Status::Error("invalid size of block_state_proofs"); + } + block_state_proof = block_state_proofs[j++]; + TRY_RESULT_ASSIGN(state_root_hash, unpack_block_state_proof(blocks[i], block_state_proof)); + } + auto state_root = vm::MerkleProof::virtualize(queue_proofs[i], 1); + if (state_root->get_hash().as_slice() != state_root_hash.as_slice()) { + return td::Status::Error("state root hash mismatch"); + } + res.emplace_back(true, blocks[i], state_root, block_state_proof, f.msg_counts_[i]); + + data[i].first = blocks[i]; + TRY_RESULT(state, ShardStateQ::fetch(blocks[i], {}, state_root)); + TRY_RESULT(outq_descr, state->message_queue()); + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), data[i].second)) { + return td::Status::Error("invalid message queue"); + } } - - // Validate proof - TRY_RESULT_PREFIX(state, ShardStateQ::fetch(block_id, {}, virtual_root), "invalid proof: "); - TRY_RESULT_PREFIX(outq_descr, state->message_queue(), "invalid proof: "); - - block::gen::OutMsgQueueInfo::Record qinfo; - if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { - return td::Status::Error("invalid proof: invalid message queue"); + if (j != block_state_proofs.size()) { + return td::Status::Error("invalid size of block_state_proofs"); } - TRY_STATUS_PREFIX(check_no_prunned(qinfo.proc_info->prefetch_ref(0)), "invalid proc_info: ") - TRY_STATUS_PREFIX(check_no_prunned(qinfo.ihr_pending->prefetch_ref(0)), "invalid ihr_pending: ") - TRY_RESULT(cnt, process_queue(block_id, dst_shard, limits, qinfo)); - if (cnt != f.msg_count_) { - return td::Status::Error(PSTRING() << "invalid msg_count: expected=" << f.msg_count_ << ", found=" << cnt); + TRY_RESULT(msg_count, process_queue(dst_shard, std::move(data), limits)); + if (msg_count != f.msg_counts_) { + return td::Status::Error("incorrect msg_count"); } - return Ref(true, std::move(virtual_root), std::move(block_state_proof), cnt); + return res; } catch (vm::VmVirtError& err) { return td::Status::Error(PSTRING() << "invalid proof: " << err.get_msg()); } } -void WaitOutMsgQueueProof::alarm() { - abort_query(td::Status::Error(ErrorCode::timeout, "timeout")); +void OutMsgQueueImporter::new_masterchain_block_notification(td::Ref state, + std::set collating_shards) { + last_masterchain_state_ = state; + if (collating_shards.empty() || state->get_unix_time() < (td::uint32)td::Clocks::system() - 20) { + return; + } + auto can_collate_shard = [&](const ShardIdFull& shard) -> bool { + return std::any_of(collating_shards.begin(), collating_shards.end(), + [&](ShardIdFull our_shard) { return shard_intersects(shard, our_shard); }); + }; + auto shards = state->get_shards(); + auto process_dst_shard = [&](const ShardIdFull& dst_shard) { + if (!can_collate_shard(dst_shard)) { + return; + } + std::vector top_blocks; + for (const auto& shard : shards) { + if (block::ShardConfig::is_neighbor(dst_shard, shard->shard())) { + top_blocks.push_back(shard->top_block_id()); + } + } + get_neighbor_msg_queue_proofs(dst_shard, std::move(top_blocks), td::Timestamp::in(15.0), + [](td::Result>>) {}); + }; + for (const auto& shard : shards) { + if (shard->before_merge()) { + if (is_left_child(shard->shard())) { + process_dst_shard(shard_parent(shard->shard())); + } + } else if (shard->before_split()) { + process_dst_shard(shard_child(shard->shard(), true)); + process_dst_shard(shard_child(shard->shard(), false)); + } else { + process_dst_shard(shard->shard()); + } + } } -void WaitOutMsgQueueProof::abort_query(td::Status reason) { - if (promise_) { - LOG(DEBUG) << "aborting wait msg queue query for " << block_id_.to_str() << " priority=" << priority_ << ": " - << reason; - promise_.set_error( - reason.move_as_error_prefix(PSTRING() << "failed to get msg queue for " << block_id_.to_str() << ": ")); +void OutMsgQueueImporter::get_neighbor_msg_queue_proofs( + ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, + td::Promise>> promise) { + std::sort(blocks.begin(), blocks.end()); + auto entry = cache_[{dst_shard, blocks}]; + if (entry) { + if (entry->done) { + promise.set_result(entry->result); + alarm_timestamp().relax(entry->timeout = td::Timestamp::in(CACHE_TTL)); + } else { + entry->timeout = std::max(entry->timeout, timeout); + entry->promises.emplace_back(std::move(promise), timeout); + alarm_timestamp().relax(timeout); + } + return; } - stop(); -} -void WaitOutMsgQueueProof::finish_query(Ref result) { - promise_.set_result(std::move(result)); - stop(); + LOG(DEBUG) << "Importing neighbor msg queues for shard " << dst_shard.to_str() << ", " << blocks.size() << " blocks"; + + cache_[{dst_shard, blocks}] = entry = std::make_shared(); + entry->dst_shard = dst_shard; + entry->blocks = blocks; + entry->promises.emplace_back(std::move(promise), timeout); + alarm_timestamp().relax(entry->timeout = timeout); + + std::map> new_queries; + for (const BlockIdExt& block : blocks) { + if (opts_->need_monitor(block.shard_full(), last_masterchain_state_)) { + ++entry->pending; + get_proof_local(entry, block); + } else { + ShardIdFull prefix = block.shard_full(); + int min_split = last_masterchain_state_->monitor_min_split_depth(prefix.workchain); + if (prefix.pfx_len() > min_split) { + prefix = shard_prefix(prefix, min_split); + } + new_queries[prefix].push_back(block); + } + }; + auto limits = last_masterchain_state_->get_imported_msg_queue_limits(dst_shard.workchain); + for (auto& p : new_queries) { + ++entry->pending; + get_proof_import(entry, std::move(p.second), limits); + } + if (entry->pending == 0) { + finish_query(entry); + } } -void WaitOutMsgQueueProof::start_up() { - alarm_timestamp() = timeout_; - if (local_) { - run_local(); - } else { - run_net(); +void OutMsgQueueImporter::get_proof_local(std::shared_ptr entry, BlockIdExt block) { + if (!check_timeout(entry)) { + return; } + td::actor::send_closure( + manager_, &ValidatorManager::wait_block_state_short, block, 0, entry->timeout, + [=, SelfId = actor_id(this), manager = manager_, timeout = entry->timeout, + retry_after = td::Timestamp::in(0.5)](td::Result> R) mutable { + if (R.is_error()) { + LOG(DEBUG) << "Failed to get block state for " << block.to_str() << ": " << R.move_as_error(); + delay_action([=]() { td::actor::send_closure(SelfId, &OutMsgQueueImporter::get_proof_local, entry, block); }, + retry_after); + return; + } + auto state = R.move_as_ok(); + if (block.seqno() == 0) { + std::vector> proof = { + td::Ref(true, block, state->root_cell(), td::Ref{})}; + td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, std::move(proof)); + return; + } + td::actor::send_closure( + manager, &ValidatorManager::wait_block_data_short, block, 0, timeout, + [=](td::Result> R) mutable { + if (R.is_error()) { + LOG(DEBUG) << "Failed to get block data for " << block.to_str() << ": " << R.move_as_error(); + delay_action( + [=]() { td::actor::send_closure(SelfId, &OutMsgQueueImporter::get_proof_local, entry, block); }, + retry_after); + return; + } + Ref block_state_proof = create_block_state_proof(R.ok()->root_cell()).move_as_ok(); + std::vector> proof = { + td::Ref(true, block, state->root_cell(), std::move(block_state_proof))}; + td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, std::move(proof)); + }); + }); } -void WaitOutMsgQueueProof::run_local() { - ++pending; - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, block_id_, priority_, timeout_, - [SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::abort_query, - R.move_as_error_prefix("failed to get shard state: ")); - } else { - td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::got_state_root, - R.move_as_ok()->root_cell()); - } - }); - if (block_id_.seqno() != 0) { - ++pending; - td::actor::send_closure(manager_, &ValidatorManager::wait_block_data_short, block_id_, priority_, timeout_, - [SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::abort_query, - R.move_as_error_prefix("failed to get block data: ")); - } else { - td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::got_block_root, - R.move_as_ok()->root_cell()); - } - }); +void OutMsgQueueImporter::get_proof_import(std::shared_ptr entry, std::vector blocks, + block::ImportedMsgQueueLimits limits) { + if (!check_timeout(entry)) { + return; } + td::actor::send_closure( + manager_, &ValidatorManager::send_get_out_msg_queue_proof_request, entry->dst_shard, blocks, limits, + [=, SelfId = actor_id(this), retry_after = td::Timestamp::in(0.5), + dst_shard = entry->dst_shard](td::Result>> R) { + if (R.is_error()) { + LOG(DEBUG) << "Failed to get out msg queue for " << dst_shard.to_str() << ": " << R.move_as_error(); + delay_action( + [=]() { + td::actor::send_closure(SelfId, &OutMsgQueueImporter::get_proof_import, entry, std::move(blocks), + limits); + }, + retry_after); + return; + } + td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, R.move_as_ok()); + }); } -void WaitOutMsgQueueProof::got_state_root(Ref root) { - state_root_ = std::move(root); - if (--pending == 0) { - run_local_cont(); +void OutMsgQueueImporter::got_proof(std::shared_ptr entry, std::vector> proofs) { + if (!check_timeout(entry)) { + return; + } + for (auto& p : proofs) { + entry->result[p->block_id_] = std::move(p); + } + CHECK(entry->pending > 0); + if (--entry->pending == 0) { + finish_query(entry); } } -void WaitOutMsgQueueProof::got_block_root(Ref root) { - block_root_ = std::move(root); - if (--pending == 0) { - run_local_cont(); +void OutMsgQueueImporter::finish_query(std::shared_ptr entry) { + LOG(DEBUG) << "Done importing neighbor msg queues for shard " << entry->dst_shard.to_str() << ", " + << entry->blocks.size() << " blocks in " << entry->timer.elapsed() << "s"; + entry->done = true; + alarm_timestamp().relax(entry->timeout = td::Timestamp::in(CACHE_TTL)); + for (auto& p : entry->promises) { + p.first.set_result(entry->result); } + entry->promises.clear(); } -void WaitOutMsgQueueProof::run_local_cont() { - Ref block_state_proof; - if (block_id_.seqno() != 0) { - auto R = create_block_state_proof(std::move(block_root_)); - if (R.is_error()) { - abort_query(R.move_as_error_prefix("failed to create block state proof")); - return; +bool OutMsgQueueImporter::check_timeout(std::shared_ptr entry) { + if (entry->timeout.is_in_past()) { + LOG(DEBUG) << "Aborting importing neighbor msg queues for shard " << entry->dst_shard.to_str() << ", " + << entry->blocks.size() << " blocks: timeout"; + for (auto& p : entry->promises) { + p.first.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); } - block_state_proof = R.move_as_ok(); + entry->promises.clear(); + auto it = cache_.find({entry->dst_shard, entry->blocks}); + if (it != cache_.end() && it->second == entry) { + cache_.erase(it); + } + return false; } - finish_query(td::Ref(true, std::move(state_root_), std::move(block_state_proof))); + return true; } -void WaitOutMsgQueueProof::run_net() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = block_id_, - retry_after = td::Timestamp::in(0.5)](td::Result> R) { - if (R.is_error()) { - LOG(DEBUG) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); - delay_action([SelfId]() mutable { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::run_net); }, - retry_after); - } else { - td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::finish_query, R.move_as_ok()); +void OutMsgQueueImporter::alarm() { + auto it = cache_.begin(); + while (it != cache_.end()) { + auto& promises = it->second->promises; + if (it->second->timeout.is_in_past()) { + if (!it->second->done) { + LOG(DEBUG) << "Aborting importing neighbor msg queues for shard " << it->second->dst_shard.to_str() << ", " + << it->second->blocks.size() << " blocks: timeout"; + for (auto& p : promises) { + p.first.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); + } + promises.clear(); + } + it = cache_.erase(it); + continue; } - }); - - td::actor::send_closure(manager_, &ValidatorManager::send_get_out_msg_queue_proof_request, block_id_, dst_shard_, - limits_, priority_, std::move(P)); + alarm_timestamp().relax(it->second->timeout); + size_t j = 0; + for (auto& p : promises) { + if (p.second.is_in_past()) { + p.first.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); + } else { + alarm_timestamp().relax(p.second); + promises[j++] = std::move(p); + } + } + promises.resize(j); + ++it; + } } void BuildOutMsgQueueProof::abort_query(td::Status reason) { if (promise_) { - LOG(DEBUG) << "failed to build msg queue proof for " << block_id_.to_str() << ": " << reason; + LOG(DEBUG) << "failed to build msg queue proof to " << dst_shard_.to_str() << ": " << reason; promise_.set_error( - reason.move_as_error_prefix(PSTRING() << "failed to build msg queue proof for " << block_id_.to_str() << ": ")); + reason.move_as_error_prefix(PSTRING() << "failed to build msg queue proof to " << dst_shard_.to_str() << ": ")); } stop(); } void BuildOutMsgQueueProof::start_up() { - ++pending; - td::actor::send_closure(manager_, &ValidatorManagerInterface::get_shard_state_from_db_short, block_id_, - [SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, - R.move_as_error_prefix("failed to get shard state: ")); - } else { - td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_state_root, - R.move_as_ok()->root_cell()); - } - }); - if (block_id_.seqno() != 0) { + for (size_t i = 0; i < blocks_.size(); ++i) { + BlockIdExt id = blocks_[i].id; ++pending; - td::actor::send_closure(manager_, &ValidatorManagerInterface::get_block_data_from_db_short, block_id_, - [SelfId = actor_id(this)](td::Result> R) { + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_shard_state_from_db_short, id, + [SelfId = actor_id(this), i](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, - R.move_as_error_prefix("failed to get block data: ")); + R.move_as_error_prefix("failed to get shard state: ")); } else { - td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_block_root, + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_state_root, i, R.move_as_ok()->root_cell()); } }); + if (id.seqno() != 0) { + ++pending; + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_block_data_from_db_short, id, + [SelfId = actor_id(this), i](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, + R.move_as_error_prefix("failed to get block data: ")); + } else { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_block_root, i, + R.move_as_ok()->root_cell()); + } + }); + } + } + if (pending == 0) { + build_proof(); } } -void BuildOutMsgQueueProof::got_state_root(Ref root) { - state_root_ = std::move(root); +void BuildOutMsgQueueProof::got_state_root(size_t i, Ref root) { + blocks_[i].state_root = std::move(root); if (--pending == 0) { build_proof(); } } -void BuildOutMsgQueueProof::got_block_root(Ref root) { - block_root_ = std::move(root); +void BuildOutMsgQueueProof::got_block_root(size_t i, Ref root) { + blocks_[i].block_root = std::move(root); if (--pending == 0) { build_proof(); } } void BuildOutMsgQueueProof::build_proof() { - auto result = OutMsgQueueProof::build(block_id_, dst_shard_, limits_, std::move(state_root_), std::move(block_root_)); + auto result = OutMsgQueueProof::build(dst_shard_, std::move(blocks_), limits_); if (result.is_error()) { LOG(ERROR) << "Failed to build msg queue proof: " << result.error(); } diff --git a/validator/impl/out-msg-queue-proof.hpp b/validator/impl/out-msg-queue-proof.hpp index 9c5acfef99..33f6c399e1 100644 --- a/validator/impl/out-msg-queue-proof.hpp +++ b/validator/impl/out-msg-queue-proof.hpp @@ -20,6 +20,8 @@ #include "auto/tl/ton_api.h" #include "interfaces/out-msg-queue-proof.h" #include "td/actor/actor.h" +#include "interfaces/shard.h" +#include "validator.h" namespace ton { @@ -29,83 +31,77 @@ using td::Ref; class ValidatorManager; class ValidatorManagerInterface; -class WaitOutMsgQueueProof : public td::actor::Actor { +class OutMsgQueueImporter : public td::actor::Actor { public: - WaitOutMsgQueueProof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, bool local, - td::uint32 priority, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise> promise) - : block_id_(std::move(block_id)) - , dst_shard_(dst_shard) - , limits_(limits) - , local_(local) - , priority_(priority) - , manager_(manager) - , timeout_(timeout) - , promise_(std::move(promise)) { + OutMsgQueueImporter(td::actor::ActorId manager, td::Ref opts, + td::Ref last_masterchain_state) + : manager_(manager), opts_(opts), last_masterchain_state_(last_masterchain_state) { } - void update_timeout(td::Timestamp timeout, td::uint32 priority) { - timeout_ = timeout; - alarm_timestamp() = timeout_; - priority_ = priority; + void new_masterchain_block_notification(td::Ref state, std::set collating_shards); + void get_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, + td::Promise>> promise); + + void update_options(td::Ref opts) { + opts_ = std::move(opts); } - void abort_query(td::Status reason); - void finish_query(Ref result); void alarm() override; - void start_up() override; - - void run_local(); - void got_state_root(Ref root); - void got_block_root(Ref root); - void run_local_cont(); - - void run_net(); - private: - BlockIdExt block_id_; - ShardIdFull dst_shard_; - block::ImportedMsgQueueLimits limits_; - bool local_; - td::uint32 priority_; - td::actor::ActorId manager_; - td::Timestamp timeout_; - td::Promise> promise_; - - Ref state_root_, block_root_; - unsigned pending = 0; + td::Ref opts_; + td::Ref last_masterchain_state_; + + struct CacheEntry { + ShardIdFull dst_shard; + std::vector blocks; + bool done = false; + std::map> result; + std::vector>>, td::Timestamp>> promises; + td::Timestamp timeout = td::Timestamp::never(); + td::Timer timer; + size_t pending = 0; + }; + std::map>, std::shared_ptr> cache_; + + void get_proof_local(std::shared_ptr entry, BlockIdExt block); + void get_proof_import(std::shared_ptr entry, std::vector blocks, + block::ImportedMsgQueueLimits limits); + void got_proof(std::shared_ptr entry, std::vector> proofs); + void finish_query(std::shared_ptr entry); + bool check_timeout(std::shared_ptr entry); + + constexpr static const double CACHE_TTL = 30.0; }; class BuildOutMsgQueueProof : public td::actor::Actor { public: - BuildOutMsgQueueProof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, + BuildOutMsgQueueProof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::actor::ActorId manager, td::Promise> promise) - : block_id_(std::move(block_id)) - , dst_shard_(dst_shard) - , limits_(limits) - , manager_(manager) - , promise_(std::move(promise)) { + : dst_shard_(dst_shard), limits_(limits), manager_(manager), promise_(std::move(promise)) { + blocks_.resize(blocks.size()); + for (size_t i = 0; i < blocks_.size(); ++i) { + blocks_[i].id = blocks[i]; + } } void abort_query(td::Status reason); void start_up() override; - void got_state_root(Ref root); - void got_block_root(Ref root); + void got_state_root(size_t i, Ref root); + void got_block_root(size_t i, Ref root); void build_proof(); private: - BlockIdExt block_id_; ShardIdFull dst_shard_; + std::vector blocks_; block::ImportedMsgQueueLimits limits_; td::actor::ActorId manager_; td::Promise> promise_; - Ref state_root_, block_root_; - unsigned pending = 0; + size_t pending = 0; }; } // namespace validator diff --git a/validator/interfaces/out-msg-queue-proof.h b/validator/interfaces/out-msg-queue-proof.h index 524168d0d6..c0aa56106b 100644 --- a/validator/interfaces/out-msg-queue-proof.h +++ b/validator/interfaces/out-msg-queue-proof.h @@ -26,22 +26,31 @@ namespace validator { using td::Ref; struct OutMsgQueueProof : public td::CntObject { - OutMsgQueueProof(Ref state_root, Ref block_state_proof, td::int32 msg_count = -1) - : state_root_(std::move(state_root)), block_state_proof_(std::move(block_state_proof)), msg_count_(msg_count) { + OutMsgQueueProof(BlockIdExt block_id, Ref state_root, Ref block_state_proof, + td::int32 msg_count = -1) + : block_id_(block_id) + , state_root_(std::move(state_root)) + , block_state_proof_(std::move(block_state_proof)) + , msg_count_(msg_count) { } + BlockIdExt block_id_; Ref state_root_; Ref block_state_proof_; - td::int32 msg_count_; // -1 - up to end of queue - - static td::Result> fetch(BlockIdExt block_id, ShardIdFull dst_shard, - block::ImportedMsgQueueLimits limits, - const ton_api::tonNode_outMsgQueueProof &f); - static td::Result> build(BlockIdExt block_id, ShardIdFull dst_shard, - block::ImportedMsgQueueLimits limits, - Ref state_root, - Ref block_root); - + td::int32 msg_count_; // -1 - no limit + + static td::Result>> fetch(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + const ton_api::tonNode_outMsgQueueProof &f); + + struct OneBlock { + BlockIdExt id; + Ref state_root; + Ref block_root; + }; + static td::Result> build(ShardIdFull dst_shard, + std::vector blocks, + block::ImportedMsgQueueLimits limits); }; } // namespace validator diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index a4a1dd1672..c09d42937c 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -129,9 +129,9 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_ihr_message(td::Ref message) = 0; virtual void send_top_shard_block_description(td::Ref desc) = 0; virtual void send_block_broadcast(BlockBroadcast broadcast) = 0; - virtual void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, - block::ImportedMsgQueueLimits limits, td::uint32 priority, - td::Promise> promise) = 0; + virtual void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) = 0; virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 17a1bffaef..bad3fc35f8 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -152,8 +152,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; - void wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) override { + void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, + td::Promise>> promise) override { UNREACHABLE(); } @@ -255,9 +255,9 @@ class ValidatorManagerImpl : public ValidatorManager { void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast) override { } - void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, - td::uint32 priority, - td::Promise> promise) override { + void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) override { UNREACHABLE(); } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 65d2b3c0d9..33b6a1a719 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -182,8 +182,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; - void wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) override { + void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, + td::Promise>> promise) override { UNREACHABLE(); } @@ -321,9 +321,9 @@ class ValidatorManagerImpl : public ValidatorManager { } void send_block_broadcast(BlockBroadcast broadcast) override { } - void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, - td::uint32 priority, - td::Promise> promise) override { + void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) override { UNREACHABLE(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index d419f80ee3..955334dbc8 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -618,33 +618,15 @@ void ValidatorManagerImpl::wait_block_state_short(BlockIdExt block_id, td::uint3 get_block_handle(block_id, true, std::move(P)); } -void ValidatorManagerImpl::wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, - td::Timestamp timeout, - td::Promise> promise) { - auto key = std::make_pair(block_id, dst_shard); - auto it = wait_out_msg_queue_proof_.find(key); - if (it == wait_out_msg_queue_proof_.end()) { - auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), block_id, dst_shard](td::Result> R) { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_msg_queue, block_id, dst_shard, - std::move(R)); - }); - auto id = td::actor::create_actor( - "waitmsgqueue", block_id, dst_shard, - last_masterchain_state_->get_imported_msg_queue_limits(block_id.is_masterchain()), - need_monitor(block_id.shard_full()), priority, actor_id(this), td::Timestamp::at(timeout.at() + 10.0), - std::move(P)) - .release(); - wait_out_msg_queue_proof_[key].actor_ = id; - it = wait_out_msg_queue_proof_.find(key); - } else if (it->second.done_) { - promise.set_result(it->second.result_); - it->second.remove_at_ = td::Timestamp::in(30.0); - return; +void ValidatorManagerImpl::wait_neighbor_msg_queue_proofs( + ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, + td::Promise>> promise) { + if (out_msg_queue_importer_.empty()) { + out_msg_queue_importer_ = td::actor::create_actor("outmsgqueueimporter", actor_id(this), opts_, + last_masterchain_state_); } - it->second.waiting_.emplace_back(timeout, priority, std::move(promise)); - auto X = it->second.get_timeout(); - td::actor::send_closure(it->second.actor_, &WaitOutMsgQueueProof::update_timeout, X.first, X.second); + td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::get_neighbor_msg_queue_proofs, dst_shard, + std::move(blocks), timeout, std::move(promise)); } void ValidatorManagerImpl::wait_block_data(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, @@ -1082,44 +1064,6 @@ void ValidatorManagerImpl::finished_wait_data(BlockHandle handle, td::Result> R) { - auto it = wait_out_msg_queue_proof_.find({block_id, dst_shard}); - if (it != wait_out_msg_queue_proof_.end()) { - if (R.is_error()) { - auto S = R.move_as_error(); - if (S.code() != ErrorCode::timeout) { - for (auto &X : it->second.waiting_) { - X.promise.set_error(S.clone()); - } - } else { - auto X = it->second.get_timeout(); - auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), block_id, dst_shard](td::Result> R) { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_msg_queue, block_id, dst_shard, - std::move(R)); - }); - auto id = td::actor::create_actor( - "waitmsgqueue", block_id, dst_shard, - last_masterchain_state_->get_imported_msg_queue_limits(block_id.is_masterchain()), - need_monitor(block_id.shard_full()), X.second, actor_id(this), X.first, std::move(P)) - .release(); - it->second.actor_ = id; - return; - } - wait_out_msg_queue_proof_.erase(it); - } else { - auto r = R.move_as_ok(); - for (auto &X : it->second.waiting_) { - X.promise.set_result(r); - } - it->second.done_ = true; - it->second.result_ = std::move(r); - it->second.remove_at_ = td::Timestamp::in(30.0); - } - } -} - void ValidatorManagerImpl::set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) { auto P = td::PromiseCreator::lambda( @@ -1513,11 +1457,11 @@ void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast) { callback_->send_broadcast(std::move(broadcast)); } -void ValidatorManagerImpl::send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, - block::ImportedMsgQueueLimits limits, - td::uint32 priority, - td::Promise> promise) { - callback_->download_out_msg_queue_proof(id, dst_shard, limits, td::Timestamp::in(10.0), std::move(promise)); +void ValidatorManagerImpl::send_get_out_msg_queue_proof_request( + ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, + td::Promise>> promise) { + callback_->download_out_msg_queue_proof(dst_shard, std::move(blocks), limits, td::Timestamp::in(10.0), + std::move(promise)); } void ValidatorManagerImpl::start_up() { @@ -1836,18 +1780,26 @@ void ValidatorManagerImpl::new_masterchain_block() { td::actor::send_closure(shard_client_, &ShardClient::new_masterchain_block_notification, last_masterchain_block_handle_, last_masterchain_state_); } + if (is_collator()) { + std::set collating_shards; + if (validating_masterchain()) { + collating_shards.emplace(masterchainId); + } + for (const auto &collator : collator_nodes_) { + collating_shards.insert(collator.second.shards.begin(), collator.second.shards.end()); + } + if (!collating_shards.empty()) { + if (out_msg_queue_importer_.empty()) { + out_msg_queue_importer_ = td::actor::create_actor("outmsgqueueimporter", actor_id(this), + opts_, last_masterchain_state_); + } + td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::new_masterchain_block_notification, + last_masterchain_state_, std::move(collating_shards)); + } + } for (auto &c : collator_nodes_) { td::actor::send_closure(c.second.actor, &CollatorNode::new_masterchain_block_notification, last_masterchain_state_); } - if (opts_->validator_mode() == ValidatorManagerOptions::validator_lite_shards && validating_masterchain() && - last_masterchain_state_->get_unix_time() > (td::uint32)td::Clocks::system() - 20) { - // Prepare neighboours' queues for collating masterchain - for (const auto &desc : last_masterchain_state_->get_shards()) { - wait_out_msg_queue_proof(desc->top_block_id(), ShardIdFull(masterchainId), 0, td::Timestamp::in(10.0), - [](td::Result>) {}); - } - } - if (last_masterchain_seqno_ % 1024 == 0) { LOG(WARNING) << "applied masterchain block " << last_masterchain_block_id_; } @@ -2514,18 +2466,6 @@ void ValidatorManagerImpl::alarm() { } } alarm_timestamp().relax(check_shard_clients_); - if (cleanup_wait_caches_at_.is_in_past()) { - auto it = wait_out_msg_queue_proof_.begin(); - while (it != wait_out_msg_queue_proof_.end()) { - if (it->second.done_ && it->second.remove_at_.is_in_past()) { - it = wait_out_msg_queue_proof_.erase(it); - } else { - ++it; - } - } - cleanup_wait_caches_at_ = td::Timestamp::in(10.0); - } - alarm_timestamp().relax(cleanup_wait_caches_at_); } void ValidatorManagerImpl::update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) { @@ -2809,6 +2749,9 @@ void ValidatorManagerImpl::update_options(td::Ref opts) if (!serializer_.empty()) { td::actor::send_closure(serializer_, &AsyncStateSerializer::update_options, opts); } + if (!out_msg_queue_importer_.empty()) { + td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::update_options, opts); + } opts_ = std::move(opts); } diff --git a/validator/manager.hpp b/validator/manager.hpp index a37fcb4e0b..ce5254acfe 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -189,15 +189,14 @@ class ValidatorManagerImpl : public ValidatorManager { }; std::map>> wait_state_; std::map>> wait_block_data_; - std::map, WaitListCaching>> - wait_out_msg_queue_proof_; - td::Timestamp cleanup_wait_caches_at_ = td::Timestamp::now(); struct WaitBlockHandle { std::vector> waiting_; }; std::map wait_block_handle_; + td::actor::ActorOwn out_msg_queue_importer_; + private: // HANDLES CACHE std::map> handles_; @@ -372,8 +371,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; - void wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) override; + void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, + td::Promise>> promise) override; void set_block_data(BlockHandle handle, td::Ref data, td::Promise promise) override; void wait_block_data(BlockHandle handle, td::uint32 priority, td::Timestamp, @@ -462,9 +461,9 @@ class ValidatorManagerImpl : public ValidatorManager { void send_ihr_message(td::Ref message) override; void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast) override; - void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, - td::uint32 priority, - td::Promise> promise) override; + void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) override; void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; @@ -499,7 +498,6 @@ class ValidatorManagerImpl : public ValidatorManager { void finished_wait_state(BlockHandle handle, td::Result> R); void finished_wait_data(BlockHandle handle, td::Result> R); - void finished_wait_msg_queue(BlockIdExt block_id, ShardIdFull dst_shard, td::Result> R); void start_up() override; void started(ValidatorManagerInitResult result); diff --git a/validator/validator.h b/validator/validator.h index 5fa23467f6..506950e7fe 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -140,9 +140,9 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise> promise) = 0; virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; - virtual void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, + virtual void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Timestamp timeout, - td::Promise> promise) = 0; + td::Promise>> promise) = 0; virtual void new_key_block(BlockHandle handle) = 0; }; @@ -226,9 +226,9 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) = 0; - virtual void wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, - td::Timestamp timeout, - td::Promise> promise) = 0; + virtual void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, + td::Timestamp timeout, + td::Promise>> promise) = 0; virtual void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) = 0; virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, From e814973749d97025e06aa61c8b2fa3cd501be6bf Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 31 Jul 2023 22:56:25 +0300 Subject: [PATCH 070/524] Limit query size in importing out queues --- validator-engine/validator-engine.cpp | 2 ++ validator/impl/out-msg-queue-proof.cpp | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index d43b9b7088..93bd23beeb 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1850,6 +1850,8 @@ void ValidatorEngine::started_dht() { void ValidatorEngine::start_rldp() { rldp_ = ton::rldp::Rldp::create(adnl_.get()); rldp2_ = ton::rldp2::Rldp::create(adnl_.get()); + td::actor::send_closure(rldp_, &ton::rldp::Rldp::set_default_mtu, 2048); + td::actor::send_closure(rldp2_, &ton::rldp2::Rldp::set_default_mtu, 2048); started_rldp(); } diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 601f083aac..d0c3be9574 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -324,7 +324,10 @@ void OutMsgQueueImporter::get_neighbor_msg_queue_proofs( auto limits = last_masterchain_state_->get_imported_msg_queue_limits(dst_shard.workchain); for (auto& p : new_queries) { ++entry->pending; - get_proof_import(entry, std::move(p.second), limits); + for (size_t i = 0; i < p.second.size(); i += 16) { + size_t j = std::min(i + 16, p.second.size()); + get_proof_import(entry, std::vector(p.second.begin() + i, p.second.begin() + j), limits); + } } if (entry->pending == 0) { finish_query(entry); From 9e02853cbb9283f50fe34a1c0e161eff0e8d862f Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 3 Aug 2023 12:06:41 +0300 Subject: [PATCH 071/524] Improve importing msg queues --- crypto/block/block.h | 7 +++- validator/full-node-shard.cpp | 40 +++++++++---------- validator/impl/out-msg-queue-proof.cpp | 55 ++++++++++++++++++++++++-- validator/manager.cpp | 31 +++++++++++++-- validator/manager.hpp | 2 + 5 files changed, 107 insertions(+), 28 deletions(-) diff --git a/crypto/block/block.h b/crypto/block/block.h index 6a94f0aad7..f2c89286f9 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -218,9 +218,12 @@ static inline std::ostream& operator<<(std::ostream& os, const MsgProcessedUptoC struct ImportedMsgQueueLimits { // Default values - td::uint32 max_bytes = 1 << 19; - td::uint32 max_msgs = 500; + td::uint32 max_bytes = 1 << 16; + td::uint32 max_msgs = 30; bool deserialize(vm::CellSlice& cs); + ImportedMsgQueueLimits operator*(td::uint32 x) const { + return {max_bytes * x, max_msgs * x}; + } }; struct ParamLimits { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 5fcd62c648..3d9d19235d 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -960,26 +960,26 @@ void FullNodeShardImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std: create_tl_shard_id(dst_shard), std::move(blocks_tl), create_tl_object(limits.max_bytes, limits.max_msgs)); - auto P = td::PromiseCreator::lambda([=, promise = create_neighbour_promise(b, std::move(promise), true), - blocks = std::move(blocks)](td::Result R) mutable { - if (R.is_error()) { - promise.set_result(R.move_as_error()); - return; - } - TRY_RESULT_PROMISE(promise, f, fetch_tl_object(R.move_as_ok(), true)); - ton_api::downcast_call( - *f, td::overloaded( - [&](ton_api::tonNode_outMsgQueueProofEmpty &x) { - promise.set_error(td::Status::Error("node doesn't have this block")); - }, - [&](ton_api::tonNode_outMsgQueueProof &x) { - delay_action( - [=, promise = std::move(promise), blocks = std::move(blocks), x = std::move(x)]() mutable { - promise.set_result(OutMsgQueueProof::fetch(dst_shard, blocks, limits, x)); - }, - td::Timestamp::now()); - })); - }); + auto P = td::PromiseCreator::lambda( + [=, promise = std::move(promise), blocks = std::move(blocks)](td::Result R) mutable { + if (R.is_error()) { + promise.set_result(R.move_as_error()); + return; + } + TRY_RESULT_PROMISE(promise, f, fetch_tl_object(R.move_as_ok(), true)); + ton_api::downcast_call( + *f, td::overloaded( + [&](ton_api::tonNode_outMsgQueueProofEmpty &x) { + promise.set_error(td::Status::Error("node doesn't have this block")); + }, + [&](ton_api::tonNode_outMsgQueueProof &x) { + delay_action( + [=, promise = std::move(promise), blocks = std::move(blocks), x = std::move(x)]() mutable { + promise.set_result(OutMsgQueueProof::fetch(dst_shard, blocks, limits, x)); + }, + td::Timestamp::now()); + })); + }); td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, b.adnl_id, adnl_id_, overlay_id_, "get_msg_queue", std::move(P), timeout, std::move(query), 1 << 22, rldp_); } diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index d0c3be9574..3512bf0d6b 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -285,6 +285,54 @@ void OutMsgQueueImporter::new_masterchain_block_notification(td::Ref blocks, td::Timestamp timeout, td::Promise>> promise) { + if (blocks.empty()) { + promise.set_value({}); + return; + } + if (dst_shard.is_masterchain() && blocks.size() != 1) { + // We spit queries for masterchain {dst_shard, {block_1, ..., block_n}} into separate queries + // {dst_shard, {block_1}}, ..., {dst_shard, {block_n}} + class Worker : public td::actor::Actor { + public: + Worker(size_t pending, td::Promise>> promise) + : pending_(pending), promise_(std::move(promise)) { + CHECK(pending_ > 0); + } + + void on_result(td::Ref res) { + result_[res->block_id_] = res; + if (--pending_ == 0) { + promise_.set_result(std::move(result_)); + stop(); + } + } + + void on_error(td::Status error) { + promise_.set_error(std::move(error)); + stop(); + } + + private: + size_t pending_; + td::Promise>> promise_; + std::map> result_; + }; + auto worker = td::actor::create_actor("queueworker", blocks.size(), std::move(promise)).release(); + for (const BlockIdExt& block : blocks) { + get_neighbor_msg_queue_proofs(dst_shard, {block}, timeout, + [=](td::Result>> R) { + if (R.is_error()) { + td::actor::send_closure(worker, &Worker::on_error, R.move_as_error()); + } else { + auto res = R.move_as_ok(); + CHECK(res.size() == 1); + td::actor::send_closure(worker, &Worker::on_result, res.begin()->second); + } + }); + } + return; + } + std::sort(blocks.begin(), blocks.end()); auto entry = cache_[{dst_shard, blocks}]; if (entry) { @@ -326,7 +374,8 @@ void OutMsgQueueImporter::get_neighbor_msg_queue_proofs( ++entry->pending; for (size_t i = 0; i < p.second.size(); i += 16) { size_t j = std::min(i + 16, p.second.size()); - get_proof_import(entry, std::vector(p.second.begin() + i, p.second.begin() + j), limits); + get_proof_import(entry, std::vector(p.second.begin() + i, p.second.begin() + j), + limits * (td::uint32)(j - i)); } } if (entry->pending == 0) { @@ -341,7 +390,7 @@ void OutMsgQueueImporter::get_proof_local(std::shared_ptr entry, Blo td::actor::send_closure( manager_, &ValidatorManager::wait_block_state_short, block, 0, entry->timeout, [=, SelfId = actor_id(this), manager = manager_, timeout = entry->timeout, - retry_after = td::Timestamp::in(0.5)](td::Result> R) mutable { + retry_after = td::Timestamp::in(0.1)](td::Result> R) mutable { if (R.is_error()) { LOG(DEBUG) << "Failed to get block state for " << block.to_str() << ": " << R.move_as_error(); delay_action([=]() { td::actor::send_closure(SelfId, &OutMsgQueueImporter::get_proof_local, entry, block); }, @@ -380,7 +429,7 @@ void OutMsgQueueImporter::get_proof_import(std::shared_ptr entry, st } td::actor::send_closure( manager_, &ValidatorManager::send_get_out_msg_queue_proof_request, entry->dst_shard, blocks, limits, - [=, SelfId = actor_id(this), retry_after = td::Timestamp::in(0.5), + [=, SelfId = actor_id(this), retry_after = td::Timestamp::in(0.1), dst_shard = entry->dst_shard](td::Result>> R) { if (R.is_error()) { LOG(DEBUG) << "Failed to get out msg queue for " << dst_shard.to_str() << ": " << R.move_as_error(); diff --git a/validator/manager.cpp b/validator/manager.cpp index 955334dbc8..af5a7c37b8 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -478,6 +478,10 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id(), 0, td::Timestamp::in(60.0), std::move(P)); } + if (collating_masterchain() && desc->generated_at() > td::Clocks::system() - 20) { + wait_neighbor_msg_queue_proofs(ShardIdFull{masterchainId}, {desc->block_id()}, td::Timestamp::in(15.0), + [](td::Result>>) {}); + } } } @@ -2525,6 +2529,16 @@ bool ValidatorManagerImpl::validating_masterchain() { .is_zero(); } +bool ValidatorManagerImpl::collating_masterchain() { + if (masterchain_collators_) { + return true; + } + if (opts_->validator_mode() == ValidatorManagerOptions::validator_lite_all) { + return false; + } + return validating_masterchain(); +} + PublicKeyHash ValidatorManagerImpl::get_validator(ShardIdFull shard, td::Ref val_set) { for (auto &key : temp_keys_) { if (val_set->is_validator(key.bits256_value())) { @@ -2726,7 +2740,12 @@ void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh it = collator_nodes_.emplace(id, Collator()).first; it->second.actor = td::actor::create_actor("collatornode", id, actor_id(this), adnl_, rldp_); } - it->second.shards.insert(shard); + if (!it->second.shards.insert(shard).second) { + return; + } + if (shard.is_masterchain()) { + ++masterchain_collators_; + } td::actor::send_closure(it->second.actor, &CollatorNode::add_shard, shard); } @@ -2735,10 +2754,16 @@ void ValidatorManagerImpl::del_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh if (it == collator_nodes_.end()) { return; } - td::actor::send_closure(it->second.actor, &CollatorNode::del_shard, shard); - it->second.shards.erase(shard); + if (!it->second.shards.erase(shard)) { + return; + } + if (shard.is_masterchain()) { + --masterchain_collators_; + } if (it->second.shards.empty()) { collator_nodes_.erase(it); + } else { + td::actor::send_closure(it->second.actor, &CollatorNode::del_shard, shard); } } diff --git a/validator/manager.hpp b/validator/manager.hpp index ce5254acfe..f8ca08f359 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -506,6 +506,7 @@ class ValidatorManagerImpl : public ValidatorManager { bool is_validator(); bool is_collator(); bool validating_masterchain(); + bool collating_masterchain(); PublicKeyHash get_validator(ShardIdFull shard, td::Ref val_set); ValidatorManagerImpl(td::Ref opts, std::string db_root, @@ -650,6 +651,7 @@ class ValidatorManagerImpl : public ValidatorManager { std::set shards; }; std::map collator_nodes_; + size_t masterchain_collators_ = 0; std::set extra_active_shards_; std::map last_validated_blocks_; From 7155bf5ecad50700bc4d50a6eba98ebd879fca34 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sun, 13 Aug 2023 20:37:59 +0300 Subject: [PATCH 072/524] Fix processing message queue in collator and validator --- validator/impl/collator.cpp | 3 --- validator/impl/validate-query.cpp | 35 +++++++++++++++++++++++-------- validator/impl/validate-query.hpp | 3 ++- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index d2ace04f63..4eada9fd9a 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -3820,9 +3820,6 @@ bool Collator::store_master_ref(vm::CellBuilder& cb) { bool Collator::update_processed_upto() { auto ref_mc_seqno = is_masterchain() ? new_block_seqno : prev_mc_block_seqno; update_min_mc_seqno(ref_mc_seqno); - if (in_msg_dict->is_empty()) { - return true; - } if (last_proc_int_msg_.first) { if (!processed_upto_->insert(ref_mc_seqno, last_proc_int_msg_.first, last_proc_int_msg_.second.cbits())) { return fatal_error("cannot update our ProcessedUpto to reflect processed inbound message"); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 8d0b51731b..ecce370d35 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -3974,7 +3974,7 @@ bool ValidateQuery::check_processed_upto() { // similar to Collator::process_inbound_message bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, ton::LogicalTime lt, td::ConstBitPtr key, const block::McShardDescr& nb, - bool& unprocessed) { + bool& unprocessed, bool& processed_here, td::Bits256& msg_hash) { unprocessed = false; block::EnqueuedMsgDescr enq; if (!enq.unpack(enq_msg.write())) { // unpack EnqueuedMsg @@ -3995,6 +3995,8 @@ bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, auto out_entry = out_msg_dict_->lookup(key + 96, 256); bool f0 = ps_.processed_upto_->already_processed(enq); bool f1 = ns_.processed_upto_->already_processed(enq); + processed_here = f1 && !f0; + msg_hash = enq.hash_; if (f0 && !f1) { return fatal_error( "a previously processed message has been un-processed (impossible situation after the validation of " @@ -4074,6 +4076,18 @@ bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, } bool ValidateQuery::check_in_queue() { + int imported_messages_count = 0; + in_msg_dict_->check_for_each_extra([&](Ref value, Ref, td::ConstBitPtr, int) { + int tag = block::gen::t_InMsg.get_tag(*value); + if (tag == block::gen::InMsg::msg_import_fin || tag == block::gen::InMsg::msg_import_tr) { + ++imported_messages_count; + } + return true; + }); + if (imported_messages_count == 0 && claimed_proc_lt_ == 0) { + return true; + } + std::vector neighbor_queues; for (const auto& descr : neighbors_) { td::BitArray<96> key; @@ -4082,13 +4096,6 @@ bool ValidateQuery::check_in_queue() { neighbor_queues.emplace_back(descr.top_block_id(), descr.outmsg_root, descr.disabled_); } block::OutputQueueMerger nb_out_msgs(shard_, std::move(neighbor_queues)); - if (in_msg_dict_->is_empty()) { - LOG(DEBUG) << "in_msg_dict is empty, skip checking neighbors' message queues"; - if (processed_upto_updated_) { - return reject_query("processed_upto was updated, but no messages were processed"); - } - return true; - } while (!nb_out_msgs.is_eof()) { auto kv = nb_out_msgs.extract_cur(); CHECK(kv && kv->msg.not_null()); @@ -4099,7 +4106,10 @@ bool ValidateQuery::check_in_queue() { block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg)); } bool unprocessed = false; - if (!check_neighbor_outbound_message(kv->msg, kv->lt, kv->key.cbits(), neighbors_.at(kv->source), unprocessed)) { + bool processed_here = false; + td::Bits256 msg_hash; + if (!check_neighbor_outbound_message(kv->msg, kv->lt, kv->key.cbits(), neighbors_.at(kv->source), unprocessed, + processed_here, msg_hash)) { if (verbosity > 1) { std::cerr << "invalid neighbor outbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg="; @@ -4108,6 +4118,13 @@ bool ValidateQuery::check_in_queue() { return reject_query("error processing outbound internal message "s + kv->key.to_hex() + " of neighbor " + neighbors_.at(kv->source).blk_.to_str()); } + if (processed_here) { + --imported_messages_count; + } + auto msg_lt = kv->lt; + if (imported_messages_count == 0 && msg_lt == claimed_proc_lt_ && msg_hash == claimed_proc_hash_) { + return true; + } if (unprocessed) { return true; } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index b1d528197a..eb052886df 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -340,7 +340,8 @@ class ValidateQuery : public td::actor::Actor { bool check_out_msg_descr(); bool check_processed_upto(); bool check_neighbor_outbound_message(Ref enq_msg, ton::LogicalTime lt, td::ConstBitPtr key, - const block::McShardDescr& src_nb, bool& unprocessed); + const block::McShardDescr& src_nb, bool& unprocessed, bool& processed_here, + td::Bits256& msg_hash); bool check_in_queue(); bool check_delivered_dequeued(); std::unique_ptr make_account_from(td::ConstBitPtr addr, Ref account, From 47c60d8bf01983089eeaa604ef26df70d9f581c5 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 21 Aug 2023 14:33:58 +0300 Subject: [PATCH 073/524] Bugfix in OutMsgQueueImporter --- validator/impl/out-msg-queue-proof.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 3512bf0d6b..2ffc52de89 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -371,8 +371,8 @@ void OutMsgQueueImporter::get_neighbor_msg_queue_proofs( }; auto limits = last_masterchain_state_->get_imported_msg_queue_limits(dst_shard.workchain); for (auto& p : new_queries) { - ++entry->pending; for (size_t i = 0; i < p.second.size(); i += 16) { + ++entry->pending; size_t j = std::min(i + 16, p.second.size()); get_proof_import(entry, std::vector(p.second.begin() + i, p.second.begin() + j), limits * (td::uint32)(j - i)); From 1e3a12259b41f51260cb9c3f131aeb18f3a283bc Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 30 Aug 2023 20:14:19 +0300 Subject: [PATCH 074/524] Optimize masterchain collation Use only shard blocks with ready msg queues --- validator/impl/collator.cpp | 23 ++-- validator/impl/out-msg-queue-proof.cpp | 44 -------- validator/impl/out-msg-queue-proof.hpp | 2 +- validator/interfaces/validator-manager.h | 4 +- validator/manager-disk.cpp | 4 +- validator/manager-disk.hpp | 4 +- validator/manager-hardfork.cpp | 4 +- validator/manager-hardfork.hpp | 2 +- validator/manager.cpp | 130 ++++++++++++++++++++--- validator/manager.hpp | 16 ++- 10 files changed, 151 insertions(+), 82 deletions(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 4eada9fd9a..d9def69062 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -236,12 +236,12 @@ void Collator::start_up() { } if (is_masterchain() && !is_hardfork_) { // 5. load shard block info messages - LOG(DEBUG) << "sending get_shard_blocks() query to Manager"; + LOG(DEBUG) << "sending get_shard_blocks_for_collator() query to Manager"; ++pending; td::actor::send_closure_later( - manager, &ValidatorManager::get_shard_blocks, prev_blocks[0], + manager, &ValidatorManager::get_shard_blocks_for_collator, prev_blocks[0], [self = get_self()](td::Result>> res) -> void { - LOG(DEBUG) << "got answer to get_shard_blocks() query"; + LOG(DEBUG) << "got answer to get_shard_blocks_for_collator() query"; td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_blocks, std::move(res)); }); } @@ -1405,8 +1405,8 @@ bool Collator::import_new_shard_top_blocks() { prev_descr.clear(); descr.clear(); } else { - LOG(INFO) << "updated top shard block information with " << sh_bd->block_id().to_str() << " and " - << prev_bd->block_id().to_str(); + LOG(DEBUG) << "updated top shard block information with " << sh_bd->block_id().to_str() << " and " + << prev_bd->block_id().to_str(); CHECK(ures.move_as_ok()); store_shard_fees(std::move(prev_descr)); store_shard_fees(std::move(descr)); @@ -1448,7 +1448,7 @@ bool Collator::import_new_shard_top_blocks() { store_shard_fees(std::move(descr)); register_shard_block_creators(sh_bd->get_creator_list(chain_len)); shards_max_end_lt_ = std::max(shards_max_end_lt_, end_lt); - LOG(INFO) << "updated top shard block information with " << sh_bd->block_id().to_str(); + LOG(DEBUG) << "updated top shard block information with " << sh_bd->block_id().to_str(); CHECK(ures.move_as_ok()); ++tb_act; used_shard_block_descr_.emplace_back(sh_bd); @@ -1456,10 +1456,13 @@ bool Collator::import_new_shard_top_blocks() { if (tb_act) { shard_conf_adjusted_ = true; } - if (tb_act && verbosity >= 0) { // DEBUG - LOG(INFO) << "updated shard block configuration to "; - auto csr = shard_conf_->get_root_csr(); - block::gen::t_ShardHashes.print(std::cerr, csr.write()); + if (tb_act) { + LOG(INFO) << "updated shard block configuration: " << tb_act << " new top shard blocks"; + if (verbosity >= 1) { + LOG(INFO) << "updated shard block configuration to "; + auto csr = shard_conf_->get_root_csr(); + block::gen::t_ShardHashes.print(std::cerr, csr.write()); + } } block::gen::ShardFeeCreated::Record fc; if (!(tlb::csr_unpack(fees_import_dict_->get_root_extra(), diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 2ffc52de89..d8e02653fd 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -289,50 +289,6 @@ void OutMsgQueueImporter::get_neighbor_msg_queue_proofs( promise.set_value({}); return; } - if (dst_shard.is_masterchain() && blocks.size() != 1) { - // We spit queries for masterchain {dst_shard, {block_1, ..., block_n}} into separate queries - // {dst_shard, {block_1}}, ..., {dst_shard, {block_n}} - class Worker : public td::actor::Actor { - public: - Worker(size_t pending, td::Promise>> promise) - : pending_(pending), promise_(std::move(promise)) { - CHECK(pending_ > 0); - } - - void on_result(td::Ref res) { - result_[res->block_id_] = res; - if (--pending_ == 0) { - promise_.set_result(std::move(result_)); - stop(); - } - } - - void on_error(td::Status error) { - promise_.set_error(std::move(error)); - stop(); - } - - private: - size_t pending_; - td::Promise>> promise_; - std::map> result_; - }; - auto worker = td::actor::create_actor("queueworker", blocks.size(), std::move(promise)).release(); - for (const BlockIdExt& block : blocks) { - get_neighbor_msg_queue_proofs(dst_shard, {block}, timeout, - [=](td::Result>> R) { - if (R.is_error()) { - td::actor::send_closure(worker, &Worker::on_error, R.move_as_error()); - } else { - auto res = R.move_as_ok(); - CHECK(res.size() == 1); - td::actor::send_closure(worker, &Worker::on_result, res.begin()->second); - } - }); - } - return; - } - std::sort(blocks.begin(), blocks.end()); auto entry = cache_[{dst_shard, blocks}]; if (entry) { diff --git a/validator/impl/out-msg-queue-proof.hpp b/validator/impl/out-msg-queue-proof.hpp index 33f6c399e1..c115a27630 100644 --- a/validator/impl/out-msg-queue-proof.hpp +++ b/validator/impl/out-msg-queue-proof.hpp @@ -72,7 +72,7 @@ class OutMsgQueueImporter : public td::actor::Actor { void finish_query(std::shared_ptr entry); bool check_timeout(std::shared_ptr entry); - constexpr static const double CACHE_TTL = 30.0; + constexpr static const double CACHE_TTL = 60.0; }; class BuildOutMsgQueueProof : public td::actor::Actor { diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index c09d42937c..f63e453dc8 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -102,8 +102,8 @@ class ValidatorManager : public ValidatorManagerInterface { td::Promise> promise) = 0; virtual void get_external_messages(ShardIdFull shard, td::Promise>> promise) = 0; virtual void get_ihr_messages(ShardIdFull shard, td::Promise>> promise) = 0; - virtual void get_shard_blocks(BlockIdExt masterchain_block_id, - td::Promise>> promise) = 0; + virtual void get_shard_blocks_for_collator(BlockIdExt masterchain_block_id, + td::Promise>> promise) = 0; virtual void complete_external_messages(std::vector to_delay, std::vector to_delete) = 0; virtual void complete_ihr_messages(std::vector to_delay, diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 5a314e2f23..15954e4727 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -516,8 +516,8 @@ void ValidatorManagerImpl::get_ihr_messages(ShardIdFull shard, td::Promise>> promise) { +void ValidatorManagerImpl::get_shard_blocks_for_collator( + BlockIdExt masterchain_block_id, td::Promise>> promise) { if (!last_masterchain_block_handle_) { promise.set_result(std::vector>{}); return; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index bad3fc35f8..53c399ed0f 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -193,8 +193,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void get_external_messages(ShardIdFull shard, td::Promise>> promise) override; void get_ihr_messages(ShardIdFull shard, td::Promise>> promise) override; - void get_shard_blocks(BlockIdExt masterchain_block_id, - td::Promise>> promise) override; + void get_shard_blocks_for_collator(BlockIdExt masterchain_block_id, + td::Promise>> promise) override; void complete_external_messages(std::vector to_delay, std::vector to_delete) override; void complete_ihr_messages(std::vector to_delay, std::vector to_delete) override; diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index 80a64d25fa..d1be057f0a 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -366,8 +366,8 @@ void ValidatorManagerImpl::get_ihr_messages(ShardIdFull shard, td::Promise>> promise) { +void ValidatorManagerImpl::get_shard_blocks_for_collator( + BlockIdExt masterchain_block_id, td::Promise>> promise) { } void ValidatorManagerImpl::get_block_data_from_db(ConstBlockHandle handle, td::Promise> promise) { diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 33b6a1a719..be8e08249c 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -233,7 +233,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void get_external_messages(ShardIdFull shard, td::Promise>> promise) override; void get_ihr_messages(ShardIdFull shard, td::Promise>> promise) override; - void get_shard_blocks(BlockIdExt masterchain_block_id, + void get_shard_blocks_for_collator(BlockIdExt masterchain_block_id, td::Promise>> promise) override; void complete_external_messages(std::vector to_delay, std::vector to_delete) override { diff --git a/validator/manager.cpp b/validator/manager.cpp index af5a7c37b8..5b5ad43ecc 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -441,7 +441,7 @@ void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc return; } auto it = shard_blocks_.find(ShardTopBlockDescriptionId{block_id.shard_full(), cc_seqno}); - if (it != shard_blocks_.end() && block_id.id.seqno <= it->second->block_id().id.seqno) { + if (it != shard_blocks_.end() && block_id.id.seqno <= it->second.latest_desc->block_id().id.seqno) { VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; return; } @@ -459,11 +459,11 @@ void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc void ValidatorManagerImpl::add_shard_block_description(td::Ref desc) { if (desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); - if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second->block_id().id.seqno) { + if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second.latest_desc->block_id().id.seqno) { VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; return; } - shard_blocks_[ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}] = desc; + shard_blocks_[ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}].latest_desc = desc; VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); if (need_monitor(desc->block_id().shard_full())) { auto P = td::PromiseCreator::lambda([](td::Result> R) { @@ -478,13 +478,53 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id(), 0, td::Timestamp::in(60.0), std::move(P)); } - if (collating_masterchain() && desc->generated_at() > td::Clocks::system() - 20) { - wait_neighbor_msg_queue_proofs(ShardIdFull{masterchainId}, {desc->block_id()}, td::Timestamp::in(15.0), - [](td::Result>>) {}); + if (collating_masterchain()) { + preload_msg_queue_to_masterchain(desc); } } } +void ValidatorManagerImpl::preload_msg_queue_to_masterchain(td::Ref desc) { + auto id = ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}; + auto it = shard_blocks_.find(id); + if (!collating_masterchain() || it == shard_blocks_.end() || it->second.latest_desc->block_id() != desc->block_id()) { + return; + } + wait_neighbor_msg_queue_proofs( + ShardIdFull{masterchainId}, {desc->block_id()}, td::Timestamp::in(10.0), + [=, SelfId = actor_id(this), + retry_at = td::Timestamp::in(1.0)](td::Result>> R) { + if (R.is_error()) { + delay_action( + [=]() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::preload_msg_queue_to_masterchain, desc); }, + retry_at); + return; + } + auto res = R.move_as_ok(); + auto &queue = res[desc->block_id()]; + CHECK(queue.not_null()); + td::actor::send_closure(SelfId, &ValidatorManagerImpl::loaded_msg_queue_to_masterchain, desc, std::move(queue)); + }); +} + +void ValidatorManagerImpl::loaded_msg_queue_to_masterchain(td::Ref desc, + td::Ref res) { + auto id = ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}; + auto it = shard_blocks_.find(id); + if (it == shard_blocks_.end()) { + return; + } + auto &info = it->second; + if (info.ready_desc.is_null() || info.ready_desc->block_id().seqno() < desc->block_id().seqno()) { + VLOG(VALIDATOR_DEBUG) << "loaded out msg queue to masterchain from " << desc->block_id(); + if (info.ready_desc.not_null()) { + cached_msg_queue_to_masterchain_.erase(info.ready_desc->block_id()); + } + info.ready_desc = desc; + cached_msg_queue_to_masterchain_[desc->block_id()] = std::move(res); + } +} + void ValidatorManagerImpl::add_ext_server_id(adnl::AdnlNodeIdShort id) { class Cb : public adnl::Adnl::Callback { private: @@ -629,6 +669,56 @@ void ValidatorManagerImpl::wait_neighbor_msg_queue_proofs( out_msg_queue_importer_ = td::actor::create_actor("outmsgqueueimporter", actor_id(this), opts_, last_masterchain_state_); } + if (dst_shard.is_masterchain()) { + // We spit queries for masterchain {dst_shard, {block_1, ..., block_n}} into separate queries + // {dst_shard, {block_1}}, ..., {dst_shard, {block_n}} + // Also, use cache + class Worker : public td::actor::Actor { + public: + Worker(size_t pending, td::Promise>> promise) + : pending_(pending), promise_(std::move(promise)) { + CHECK(pending_ > 0); + } + + void on_result(td::Ref res) { + result_[res->block_id_] = res; + if (--pending_ == 0) { + promise_.set_result(std::move(result_)); + stop(); + } + } + + void on_error(td::Status error) { + promise_.set_error(std::move(error)); + stop(); + } + + private: + size_t pending_; + td::Promise>> promise_; + std::map> result_; + }; + auto worker = td::actor::create_actor("queueworker", blocks.size(), std::move(promise)).release(); + for (const BlockIdExt &block : blocks) { + auto it = cached_msg_queue_to_masterchain_.find(block); + if (it != cached_msg_queue_to_masterchain_.end()) { + td::actor::send_closure(worker, &Worker::on_result, it->second); + continue; + } + td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::get_neighbor_msg_queue_proofs, dst_shard, + std::vector{1, block}, timeout, + [=](td::Result>> R) { + if (R.is_error()) { + td::actor::send_closure(worker, &Worker::on_error, R.move_as_error()); + } else { + auto res = R.move_as_ok(); + CHECK(res.size() == 1); + td::actor::send_closure(worker, &Worker::on_result, res.begin()->second); + } + }); + } + return; + } td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::get_neighbor_msg_queue_proofs, dst_shard, std::move(blocks), timeout, std::move(promise)); } @@ -843,11 +933,14 @@ void ValidatorManagerImpl::get_ihr_messages(ShardIdFull shard, td::Promise>> promise) { +void ValidatorManagerImpl::get_shard_blocks_for_collator( + BlockIdExt masterchain_block_id, td::Promise>> promise) { std::vector> v; for (auto &b : shard_blocks_) { - v.push_back(b.second); + if (b.second.ready_desc.is_null()) { + continue; + } + v.push_back(b.second.ready_desc); } promise.set_value(std::move(v)); } @@ -1786,7 +1879,7 @@ void ValidatorManagerImpl::new_masterchain_block() { } if (is_collator()) { std::set collating_shards; - if (validating_masterchain()) { + if (collating_masterchain()) { collating_shards.emplace(masterchainId); } for (const auto &collator : collator_nodes_) { @@ -2080,12 +2173,19 @@ void ValidatorManagerImpl::update_shard_blocks() { auto it = shard_blocks_.begin(); while (it != shard_blocks_.end()) { auto &B = it->second; - if (!B->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { - auto it2 = it++; - shard_blocks_.erase(it2); - } else { - ++it; + if (!B.latest_desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { + if (B.ready_desc.not_null()) { + cached_msg_queue_to_masterchain_.erase(B.ready_desc->block_id()); + } + it = shard_blocks_.erase(it); + continue; } + if (B.ready_desc.not_null() && + !B.ready_desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { + cached_msg_queue_to_masterchain_.erase(B.ready_desc->block_id()); + B.ready_desc = {}; + } + ++it; } } diff --git a/validator/manager.hpp b/validator/manager.hpp index f8ca08f359..45a0c1e636 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -220,7 +220,15 @@ class ValidatorManagerImpl : public ValidatorManager { } }; // DATA FOR COLLATOR - std::map> shard_blocks_; + // Shard block will not be used until queue is ready (to avoid too long masterchain collation) + // latest_desc - latest known block + // ready_desc - block with ready msg queue (may be null) + struct ShardTopBlock { + td::Ref latest_desc; + td::Ref ready_desc; + }; + std::map shard_blocks_; + std::map> cached_msg_queue_to_masterchain_; std::map, std::unique_ptr>> ext_messages_; std::map, std::map>> ext_addr_messages_; std::map> ext_messages_hashes_; @@ -410,8 +418,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void get_external_messages(ShardIdFull shard, td::Promise>> promise) override; void get_ihr_messages(ShardIdFull shard, td::Promise>> promise) override; - void get_shard_blocks(BlockIdExt masterchain_block_id, - td::Promise>> promise) override; + void get_shard_blocks_for_collator(BlockIdExt masterchain_block_id, + td::Promise>> promise) override; void complete_external_messages(std::vector to_delay, std::vector to_delete) override; void complete_ihr_messages(std::vector to_delay, std::vector to_delete) override; @@ -493,6 +501,8 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_shard_block_description(td::Ref desc); + void preload_msg_queue_to_masterchain(td::Ref desc); + void loaded_msg_queue_to_masterchain(td::Ref desc, td::Ref res); void register_block_handle(BlockHandle handle); From 98fe1f886d1fdd6780ddb5a8b9f2d7061b275af5 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 31 Aug 2023 11:57:02 +0300 Subject: [PATCH 075/524] Remove excessive logs in collator --- validator/impl/collator.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index d9def69062..7cf14eec9f 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1458,7 +1458,7 @@ bool Collator::import_new_shard_top_blocks() { } if (tb_act) { LOG(INFO) << "updated shard block configuration: " << tb_act << " new top shard blocks"; - if (verbosity >= 1) { + if (verbosity >= 3) { LOG(INFO) << "updated shard block configuration to "; auto csr = shard_conf_->get_root_csr(); block::gen::t_ShardHashes.print(std::cerr, csr.write()); @@ -3336,7 +3336,7 @@ bool Collator::create_mc_state_extra() { } // 3. save new shard_hashes state_extra.shard_hashes = shard_conf_->get_root_csr(); - if (verbosity >= 3 * 0) { // DEBUG + if (verbosity >= 3) { std::cerr << "updated shard configuration to "; block::gen::t_ShardHashes.print(std::cerr, *state_extra.shard_hashes); } @@ -3734,7 +3734,7 @@ bool Collator::update_public_libraries() { } } } - if (libraries_changed_ && verbosity >= 2 * 0) { + if (libraries_changed_ && verbosity >= 2) { std::cerr << "New public libraries: "; block::gen::t_HashmapE_256_LibDescr.print(std::cerr, shard_libraries_->get_root()); shard_libraries_->get_root()->print_rec(std::cerr); From 4d5b79224dd6f616923cf747373f19751009d8a2 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 21 Sep 2023 15:14:34 +0300 Subject: [PATCH 076/524] Proxy liteserver --- tl/generate/scheme/ton_api.tl | 3 + tl/generate/scheme/ton_api.tlo | Bin 91588 -> 91764 bytes utils/CMakeLists.txt | 3 + utils/proxy-liteserver.cpp | 323 +++++++++++++++++++++++++++++++++ 4 files changed, 329 insertions(+) create mode 100644 utils/proxy-liteserver.cpp diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 6d64b692b1..50e0aabbcd 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -931,3 +931,6 @@ storage.daemon.withdraw contract:string = storage.daemon.Success; storage.daemon.sendCoins address:string amount:string message:string = storage.daemon.Success; storage.daemon.closeStorageContract address:string = storage.daemon.Success; storage.daemon.removeStorageProvider = storage.daemon.Success; + +---types--- +proxyLiteserver.config port:int id:PublicKey = proxyLiteserver.Config; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 9ecd272dc182524dcfa66469b1b53029f87bafa0..4ff97e05b6900f7621443a2f1d5ccd95166714a1 100644 GIT binary patch delta 125 zcmX?dn)S;WR^CUm^{p77fNvx3Wd)JGR6_%?f};G2N}tS<)Z)~lvecr<2|Nm$A1L^U zFizgAYwA2f6`LZx|XoB(pi?Xu<*j DjteYF delta 41 scmexzhV{s4R^CUm^{p77fOjMBWrfWmiasKYlQt)sI#1ZFa4cj202#Xtj{pDw diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 3d029c9381..67c48851f6 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -25,4 +25,7 @@ target_link_libraries(opcode-timing ton_crypto) target_include_directories(pack-viewer PUBLIC $/..) +add_executable(proxy-liteserver proxy-liteserver.cpp) +target_link_libraries(proxy-liteserver tdutils tdactor adnl dht tl_api ton_crypto git lite-client-common) + install(TARGETS generate-random-id RUNTIME DESTINATION bin) diff --git a/utils/proxy-liteserver.cpp b/utils/proxy-liteserver.cpp new file mode 100644 index 0000000000..f7da39640f --- /dev/null +++ b/utils/proxy-liteserver.cpp @@ -0,0 +1,323 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "td/utils/filesystem.h" +#include "td/actor/actor.h" +#include "td/actor/MultiPromise.h" +#include "td/utils/OptionParser.h" +#include "td/utils/port/path.h" +#include "td/utils/port/signals.h" +#include "td/utils/port/user.h" +#include "td/utils/port/IPAddress.h" +#include "td/utils/Random.h" +#include "td/utils/FileLog.h" +#include "git.h" +#include "auto/tl/ton_api.h" +#include "auto/tl/lite_api.h" +#include "auto/tl/lite_api.hpp" +#include "tl-utils/lite-utils.hpp" +#include "ton/lite-tl.hpp" +#include "auto/tl/ton_api_json.h" +#include "adnl/adnl.h" +#include "lite-client/QueryTraits.h" +#include "lite-client/ext-client.h" + +#if TD_DARWIN || TD_LINUX +#include +#endif +#include + +using namespace ton; + +class ProxyLiteserver : public td::actor::Actor { + public: + ProxyLiteserver(std::string global_config, std::string db_root, td::uint16 port) + : global_config_(std::move(global_config)), db_root_(std::move(db_root)), port_(port) { + } + + void start_up() override { + LOG_CHECK(db_root_ != "") << "db root is not set"; + td::mkdir(db_root_).ensure(); + db_root_ = td::realpath(db_root_).move_as_ok(); + keyring_ = keyring::Keyring::create(db_root_ + "/keyring"); + + td::Status S = prepare_local_config(); + if (S.is_error()) { + LOG(FATAL) << "Local config error: " << S; + } + + S = create_ext_client(); + if (S.is_error()) { + LOG(FATAL) << S; + } + + create_ext_server(); + } + + td::Status prepare_local_config() { + auto r_conf_data = td::read_file(config_file()); + if (r_conf_data.is_ok()) { + auto conf_data = r_conf_data.move_as_ok(); + TRY_RESULT_PREFIX(conf_json, td::json_decode(conf_data.as_slice()), "failed to parse json: "); + TRY_STATUS_PREFIX(ton_api::from_json(*config_, conf_json.get_object()), "json does not fit TL scheme: "); + TRY_RESULT_PREFIX_ASSIGN(port_, td::narrow_cast_safe(config_->port_), "invalid port: "); + TRY_RESULT_PREFIX_ASSIGN(id_, adnl::AdnlNodeIdFull::create(config_->id_), "invalid id: "); + } else { + LOG(WARNING) << "First launch, creating local config"; + if (port_ == 0) { + return td::Status::Error("port is not set"); + } + config_->port_ = port_; + auto pk = PrivateKey{privkeys::Ed25519::random()}; + id_ = adnl::AdnlNodeIdFull{pk.compute_public_key()}; + config_->id_ = id_.tl(); + td::actor::send_closure(keyring_, &keyring::Keyring::add_key, std::move(pk), false, [](td::Result R) { + if (R.is_error()) { + LOG(FATAL) << "Failed to store private key"; + } + }); + + auto s = td::json_encode(td::ToJson(*config_), true); + TRY_STATUS_PREFIX(td::write_file(config_file(), s), "failed to write file: "); + } + return td::Status::OK(); + } + + td::Status create_ext_client() { + std::vector servers; + TRY_RESULT_PREFIX(global_config_data, td::read_file(global_config_), "Failed to read global config: "); + TRY_RESULT_PREFIX(global_config_json, td::json_decode(global_config_data.as_slice()), + "Failed to parse global config: "); + ton::ton_api::liteclient_config_global gc; + ton::ton_api::from_json(gc, global_config_json.get_object()).ensure(); + + size_t size = gc.liteservers_.size() + gc.liteservers_v2_.size(); + if (size == 0) { + return td::Status::Error("No liteservers in global config"); + } + + for (auto& s : gc.liteservers_) { + td::IPAddress addr; + addr.init_host_port(td::IPAddress::ipv4_to_str(s->ip_), s->port_).ensure(); + liteclient::ExtClient::LiteServer serv; + serv.address = addr; + serv.adnl_id = ton::adnl::AdnlNodeIdFull::create(s->id_).move_as_ok(); + servers.push_back(std::move(serv)); + } + for (auto& s : gc.liteservers_v2_) { + td::IPAddress addr; + addr.init_host_port(td::IPAddress::ipv4_to_str(s->ip_), s->port_).ensure(); + liteclient::ExtClient::LiteServer serv; + serv.address = addr; + serv.adnl_id = ton::adnl::AdnlNodeIdFull::create(s->id_).move_as_ok(); + serv.is_full = false; + for (auto& shard : s->shards_) { + serv.shards.emplace_back(shard->workchain_, (ton::ShardId)shard->shard_); + CHECK(serv.shards.back().is_valid_ext()); + } + servers.push_back(std::move(serv)); + } + class Callback : public liteclient::ExtClient::Callback {}; + ext_client_ = liteclient::ExtClient::create(std::move(servers), td::make_unique()); + return td::Status::OK(); + } + + void create_ext_server() { + adnl_ = adnl::Adnl::create("", keyring_.get()); + td::actor::send_closure(adnl_, &adnl::Adnl::add_id, id_, ton::adnl::AdnlAddressList{}, (td::uint8)255); + td::actor::send_closure(adnl_, &adnl::Adnl::create_ext_server, + std::vector{id_.compute_short_id()}, std::vector{port_}, + [SelfId = actor_id(this)](td::Result> R) { + R.ensure(); + td::actor::send_closure(SelfId, &ProxyLiteserver::created_ext_server, R.move_as_ok()); + }); + } + + void created_ext_server(td::actor::ActorOwn s) { + ext_server_ = std::move(s); + LOG(WARNING) << "Started proxy liteserver on port " << port_; + + class AdnlCallback : public adnl::Adnl::Callback { + public: + AdnlCallback(td::actor::ActorId client) : client_(client) { + } + + void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + td::actor::create_actor("worker", client_, std::move(data), std::move(promise)).release(); + } + + private: + td::actor::ActorId client_; + }; + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, id_.compute_short_id(), + adnl::Adnl::int_to_bytestring(lite_api::liteServer_query::ID), + std::make_unique(ext_client_.get())); + } + + class QueryWorker : public td::actor::Actor { + public: + QueryWorker(td::actor::ActorId client, td::BufferSlice data, + td::Promise promise) + : client_(std::move(client)), data_(std::move(data)), promise_(std::move(promise)) { + } + + void start_up() override { + auto data = data_.clone(); + auto F = fetch_tl_object(data, true); + if (F.is_ok()) { + data = std::move(F.move_as_ok()->data_); + } else { + auto G = fetch_tl_prefix(data, true); + if (G.is_error()) { + fatal_error(G.move_as_error()); + } + } + fetch_tl_prefix(data, true).ignore(); + auto F2 = fetch_tl_object(std::move(data), true); + if (F2.is_error()) { + fatal_error(F2.move_as_error()); + } + auto query = F2.move_as_ok(); + lite_api::downcast_call(*query, [&](auto& obj) { shard_ = liteclient::get_query_shard(obj); }); + + LOG(INFO) << "Got query: shard=" << shard_.to_str() << " size=" << data_.size(); + td::actor::send_closure(client_, &liteclient::ExtClient::send_query, "q", std::move(data_), shard_, + td::Timestamp::in(8.0), [SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &QueryWorker::got_result, std::move(R)); + }); + } + + void got_result(td::Result R) { + if (R.is_error()) { + LOG(INFO) << "Query to shard=" << shard_.to_str() << ": " << R.error(); + promise_.set_value(create_serialize_tl_object( + R.error().code(), "gateway error: " + R.error().message().str())); + } else { + td::BufferSlice response = R.move_as_ok(); + LOG(INFO) << "Query to shard=" << shard_.to_str() << ": OK, size=" << response.size() + << " time=" << timer_.elapsed(); + promise_.set_value(std::move(response)); + } + stop(); + } + + void fatal_error(td::Status S) { + promise_.set_error(std::move(S)); + stop(); + } + + private: + td::actor::ActorId client_; + td::BufferSlice data_; + td::Promise promise_; + td::Timer timer_ = {}; + ShardIdFull shard_; + }; + + private: + std::string global_config_; + std::string db_root_; + td::uint16 port_; + + tl_object_ptr config_ = create_tl_object(); + adnl::AdnlNodeIdFull id_; + + td::actor::ActorOwn keyring_; + td::actor::ActorOwn adnl_; + td::actor::ActorOwn ext_server_; + td::actor::ActorOwn ext_client_; + + std::string config_file() const { + return db_root_ + "/config.json"; + } +}; + +int main(int argc, char* argv[]) { + SET_VERBOSITY_LEVEL(verbosity_WARNING); + td::set_default_failure_signal_handler().ensure(); + + td::unique_ptr logger_; + SCOPE_EXIT { + td::log_interface = td::default_log_interface; + }; + + std::string global_config, db_root; + td::uint16 port = 0; + td::uint32 threads = 4; + + td::OptionParser p; + p.set_description("Proxy liteserver: distributes incoming queries to servers in global config\n"); + p.add_option('v', "verbosity", "set verbosity level", [&](td::Slice arg) { + int v = VERBOSITY_NAME(FATAL) + (td::to_integer(arg)); + SET_VERBOSITY_LEVEL(v); + }); + p.add_option('V', "version", "show build information", [&]() { + std::cout << "proxy-liteserver build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::exit(0); + }); + p.add_option('h', "help", "print help", [&]() { + char b[10240]; + td::StringBuilder sb(td::MutableSlice{b, 10000}); + sb << p; + std::cout << sb.as_cslice().c_str(); + std::exit(2); + }); + p.add_checked_option('p', "port", "liteserver port (use only on first launch)", [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(port, td::to_integer_safe(arg)); + return td::Status::OK(); + }); + p.add_option('C', "global-config", "global TON configuration file", + [&](td::Slice arg) { global_config = arg.str(); }); + p.add_option('D', "db", "db root", [&](td::Slice arg) { db_root = arg.str(); }); + p.add_option('d', "daemonize", "set SIGHUP", [&]() { + td::set_signal_handler(td::SignalType::HangUp, [](int sig) { +#if TD_DARWIN || TD_LINUX + close(0); + setsid(); +#endif + }).ensure(); + }); + p.add_option('l', "logname", "log to file", [&](td::Slice fname) { + logger_ = td::FileLog::create(fname.str()).move_as_ok(); + td::log_interface = logger_.get(); + }); + p.add_checked_option('t', "threads", PSTRING() << "number of threads (default=" << 4 << ")", + [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(threads, td::to_integer_safe(arg)); + return td::Status::OK(); + }); + + p.run(argc, argv).ensure(); + td::actor::Scheduler scheduler({threads}); + + scheduler.run_in_context( + [&] { td::actor::create_actor("proxy-liteserver", global_config, db_root, port).release(); }); + while (scheduler.run(1)) { + } +} From a198ca7fe3567596a0b8d8b0baffe779d9690994 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 10 Oct 2023 23:52:54 +0300 Subject: [PATCH 077/524] Bugfix in proxy-liteserver --- utils/proxy-liteserver.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/proxy-liteserver.cpp b/utils/proxy-liteserver.cpp index f7da39640f..7abaa1c7a0 100644 --- a/utils/proxy-liteserver.cpp +++ b/utils/proxy-liteserver.cpp @@ -195,12 +195,14 @@ class ProxyLiteserver : public td::actor::Actor { auto G = fetch_tl_prefix(data, true); if (G.is_error()) { fatal_error(G.move_as_error()); + return; } } fetch_tl_prefix(data, true).ignore(); auto F2 = fetch_tl_object(std::move(data), true); if (F2.is_error()) { fatal_error(F2.move_as_error()); + return; } auto query = F2.move_as_ok(); lite_api::downcast_call(*query, [&](auto& obj) { shard_ = liteclient::get_query_shard(obj); }); From 1eef6ed55ef812da1ee7828373f8c062f1fc7b4a Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 16 Oct 2023 11:22:41 +0300 Subject: [PATCH 078/524] Bugfix in check_neighbor_outbound_message --- validator/impl/validate-query.cpp | 33 +++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index ecce370d35..49885a5f8d 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -4010,8 +4010,37 @@ bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, key.to_hex(352) + " of neighbor " + nb.blk_.to_str()); } if (shard_contains(shard_, enq.cur_prefix_)) { - // this message couldn't come from our own outbound queue because processed messages from our queue don't stay here - return fatal_error("have an already processed EnqueuedMsg from our shard: "s + key.to_hex(352)); + // if this message comes from our own outbound queue, we must have dequeued it + if (out_entry.is_null()) { + return reject_query("our old outbound queue contains EnqueuedMsg with key "s + key.to_hex(352) + + " already processed by this shard, but there is no ext_message_deq OutMsg record for this " + "message in this block"); + } + int tag = block::gen::t_OutMsg.get_tag(*out_entry); + if (tag == block::gen::OutMsg::msg_export_deq_short) { + block::gen::OutMsg::Record_msg_export_deq_short deq; + if (!tlb::csr_unpack(std::move(out_entry), deq)) { + return reject_query( + "cannot unpack msg_export_deq_short OutMsg record for already processed EnqueuedMsg with key "s + + key.to_hex(352) + " of old outbound queue"); + } + if (deq.msg_env_hash != enq.msg_env_->get_hash().bits()) { + return reject_query("unpack ext_message_deq OutMsg record for already processed EnqueuedMsg with key "s + + key.to_hex(352) + " of old outbound queue refers to MsgEnvelope with different hash " + + deq.msg_env_hash.to_hex()); + } + } else { + block::gen::OutMsg::Record_msg_export_deq deq; + if (!tlb::csr_unpack(std::move(out_entry), deq)) { + return reject_query( + "cannot unpack msg_export_deq OutMsg record for already processed EnqueuedMsg with key "s + + key.to_hex(352) + " of old outbound queue"); + } + if (deq.out_msg->get_hash() != enq.msg_env_->get_hash()) { + return reject_query("unpack ext_message_deq OutMsg record for already processed EnqueuedMsg with key "s + + key.to_hex(352) + " of old outbound queue contains a different MsgEnvelope"); + } + } } // next check is incorrect after a merge, when ns_.processed_upto has > 1 entries // we effectively comment it out From 3c55abf2d21cc8cd67021bcca4920948862d68f1 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 17 Oct 2023 16:07:44 +0300 Subject: [PATCH 079/524] Add proxy-liteserver to github builds --- .github/workflows/macos-11.7-compile.yml | 3 ++- .github/workflows/macos-12.6-compile.yml | 3 ++- .github/workflows/ubuntu-22.04-compile.yml | 4 ++-- .github/workflows/ubuntu-compile.yml | 4 ++-- .github/workflows/win-2019-compile.yml | 4 ++-- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/macos-11.7-compile.yml b/.github/workflows/macos-11.7-compile.yml index 910b16fc19..d96cdb7b94 100644 --- a/.github/workflows/macos-11.7-compile.yml +++ b/.github/workflows/macos-11.7-compile.yml @@ -28,7 +28,7 @@ jobs: mkdir build cd build cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=11.7 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_BUILD_TYPE=Release .. - ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator + ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver - name: Find & copy binaries run: | @@ -52,6 +52,7 @@ jobs: cp build/utils/json2tlo artifacts/ cp build/adnl/adnl-proxy artifacts/ cp build/emulator/*emulator.* artifacts/ + cp build/utils/proxy-liteserver artifacts/ chmod +x artifacts/* rsync -r crypto/smartcont artifacts/ rsync -r crypto/fift/lib artifacts/ diff --git a/.github/workflows/macos-12.6-compile.yml b/.github/workflows/macos-12.6-compile.yml index 368abf02c9..75f3555953 100644 --- a/.github/workflows/macos-12.6-compile.yml +++ b/.github/workflows/macos-12.6-compile.yml @@ -39,7 +39,7 @@ jobs: mkdir build cd build cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=12.6 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_BUILD_TYPE=Release .. - ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator + ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver - name: Find & copy binaries run: | @@ -63,6 +63,7 @@ jobs: cp build/utils/json2tlo artifacts/ cp build/adnl/adnl-proxy artifacts/ cp build/emulator/*emulator.* artifacts/ + cp build/utils/proxy-liteserver artifacts/ chmod +x artifacts/* rsync -r crypto/smartcont artifacts/ rsync -r crypto/fift/lib artifacts/ diff --git a/.github/workflows/ubuntu-22.04-compile.yml b/.github/workflows/ubuntu-22.04-compile.yml index 397116ff66..fc13701e51 100644 --- a/.github/workflows/ubuntu-22.04-compile.yml +++ b/.github/workflows/ubuntu-22.04-compile.yml @@ -40,12 +40,12 @@ jobs: cd build cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. - ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state emulator + ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state emulator proxy-liteserver - name: Find & copy binaries run: | mkdir artifacts - cp build/storage/storage-daemon/storage-daemon build/storage/storage-daemon/storage-daemon-cli build/crypto/fift build/crypto/tlbc build/crypto/func build/crypto/create-state build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/tonlib/libtonlibjson.so.0.5 build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.* artifacts + cp build/storage/storage-daemon/storage-daemon build/storage/storage-daemon/storage-daemon-cli build/crypto/fift build/crypto/tlbc build/crypto/func build/crypto/create-state build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/tonlib/libtonlibjson.so.0.5 build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.* build/utils/proxy-liteserver artifacts chmod +x artifacts/* cp -R crypto/smartcont artifacts/ cp -R crypto/fift/lib artifacts/ diff --git a/.github/workflows/ubuntu-compile.yml b/.github/workflows/ubuntu-compile.yml index 171ba46d8d..6907606d4a 100644 --- a/.github/workflows/ubuntu-compile.yml +++ b/.github/workflows/ubuntu-compile.yml @@ -44,12 +44,12 @@ jobs: buildPath=`pwd` cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$buildPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$buildPath/openssl_1_1_1/libcrypto.a -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. - ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator + ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver - name: Find & copy binaries run: | mkdir artifacts-${{ matrix.os }} - cp build-${{ matrix.os }}/storage/storage-daemon/storage-daemon build-${{ matrix.os }}/storage/storage-daemon/storage-daemon-cli build-${{ matrix.os }}/crypto/fift build-${{ matrix.os }}/crypto/tlbc build-${{ matrix.os }}/crypto/func build-${{ matrix.os }}/crypto/create-state build-${{ matrix.os }}/validator-engine-console/validator-engine-console build-${{ matrix.os }}/tonlib/tonlib-cli build-${{ matrix.os }}/tonlib/libtonlibjson.so.0.5 build-${{ matrix.os }}/http/http-proxy build-${{ matrix.os }}/rldp-http-proxy/rldp-http-proxy build-${{ matrix.os }}/dht-server/dht-server build-${{ matrix.os }}/lite-client/lite-client build-${{ matrix.os }}/validator-engine/validator-engine build-${{ matrix.os }}/utils/generate-random-id build-${{ matrix.os }}/utils/json2tlo build-${{ matrix.os }}/adnl/adnl-proxy build-${{ matrix.os }}/emulator/libemulator.* artifacts-${{ matrix.os }} + cp build-${{ matrix.os }}/storage/storage-daemon/storage-daemon build-${{ matrix.os }}/storage/storage-daemon/storage-daemon-cli build-${{ matrix.os }}/crypto/fift build-${{ matrix.os }}/crypto/tlbc build-${{ matrix.os }}/crypto/func build-${{ matrix.os }}/crypto/create-state build-${{ matrix.os }}/validator-engine-console/validator-engine-console build-${{ matrix.os }}/tonlib/tonlib-cli build-${{ matrix.os }}/tonlib/libtonlibjson.so.0.5 build-${{ matrix.os }}/http/http-proxy build-${{ matrix.os }}/rldp-http-proxy/rldp-http-proxy build-${{ matrix.os }}/dht-server/dht-server build-${{ matrix.os }}/lite-client/lite-client build-${{ matrix.os }}/validator-engine/validator-engine build-${{ matrix.os }}/utils/generate-random-id build-${{ matrix.os }}/utils/json2tlo build-${{ matrix.os }}/adnl/adnl-proxy build-${{ matrix.os }}/emulator/libemulator.* build-${{ matrix.os }}/utils/proxy-liteserver artifacts-${{ matrix.os }} chmod +x artifacts-${{ matrix.os }}/* cp -R crypto/smartcont artifacts-${{ matrix.os }} cp -R crypto/fift/lib artifacts-${{ matrix.os }} diff --git a/.github/workflows/win-2019-compile.yml b/.github/workflows/win-2019-compile.yml index 1edcb0e165..62a560d2ac 100644 --- a/.github/workflows/win-2019-compile.yml +++ b/.github/workflows/win-2019-compile.yml @@ -73,7 +73,7 @@ jobs: mkdir build cd build cmake -DCMAKE_BUILD_TYPE=Release -DSODIUM_USE_STATIC_LIBS=1 -DSECP256K1_INCLUDE_DIR=%root%\secp256k1\include -DSECP256K1_LIBRARY=%root%\secp256k1\bin\x64\Release\v142\static\secp256k1.lib -DREADLINE_INCLUDE_DIR=%root%\readline-5.0-1-lib\include\readline -DREADLINE_LIBRARY=%root%\readline-5.0-1-lib\lib\readline.lib -DPORTABLE=1 -DZLIB_FOUND=1 -DMHD_FOUND=1 -DMHD_LIBRARY=%root%\libmicrohttpd-0.9.77-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib -DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.77-w32-bin\x86_64\VS2019\Release-static -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1.1j/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1.1j/lib/libcrypto_static.lib -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" .. - cmake --build . --target storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator --config Release + cmake --build . --target storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver --config Release - name: Show executables run: | @@ -92,7 +92,7 @@ jobs: mkdir artifacts\smartcont mkdir artifacts\lib - for %%I in (build\storage\storage-daemon\Release\storage-daemon.exe build\storage\storage-daemon\Release\storage-daemon-cli.exe build\blockchain-explorer\blockchain-explorer.exe build\crypto\Release\fift.exe build\crypto\Release\tlbc.exe build\crypto\Release\func.exe build\crypto\Release\create-state.exe build\validator-engine-console\Release\validator-engine-console.exe build\tonlib\Release\tonlib-cli.exe build\tonlib\Release\tonlibjson.dll build\http\Release\http-proxy.exe build\rldp-http-proxy\Release\rldp-http-proxy.exe build\dht-server\Release\dht-server.exe build\lite-client\Release\lite-client.exe build\validator-engine\Release\validator-engine.exe build\utils\Release\generate-random-id.exe build\utils\Release\json2tlo.exe build\adnl\Release\adnl-proxy.exe build\emulator\Release\emulator.dll) do copy %%I artifacts\ + for %%I in (build\storage\storage-daemon\Release\storage-daemon.exe build\storage\storage-daemon\Release\storage-daemon-cli.exe build\blockchain-explorer\blockchain-explorer.exe build\crypto\Release\fift.exe build\crypto\Release\tlbc.exe build\crypto\Release\func.exe build\crypto\Release\create-state.exe build\validator-engine-console\Release\validator-engine-console.exe build\tonlib\Release\tonlib-cli.exe build\tonlib\Release\tonlibjson.dll build\http\Release\http-proxy.exe build\rldp-http-proxy\Release\rldp-http-proxy.exe build\dht-server\Release\dht-server.exe build\lite-client\Release\lite-client.exe build\validator-engine\Release\validator-engine.exe build\utils\Release\generate-random-id.exe build\utils\Release\json2tlo.exe build\adnl\Release\adnl-proxy.exe build\emulator\Release\emulator.dll build\utils\proxy-liteserver.exe) do copy %%I artifacts\ xcopy /e /k /h /i crypto\smartcont artifacts\smartcont xcopy /e /k /h /i crypto\fift\lib artifacts\lib From 747b24aa18c3214a4072fe0e09a19917946745ce Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 19 Feb 2024 13:10:47 +0300 Subject: [PATCH 080/524] Fix QueueSizeCounter and ValidatorGroup after merge --- validator/full-node.cpp | 4 ++-- validator/manager-disk.hpp | 4 ++-- validator/manager-hardfork.hpp | 4 ++-- validator/manager.cpp | 3 +++ validator/manager.hpp | 16 ++++++++++------ validator/queue-size-counter.cpp | 4 +++- validator/queue-size-counter.hpp | 10 ++++++++-- validator/validator-group.cpp | 2 +- 8 files changed, 31 insertions(+), 16 deletions(-) diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 7ee8f1c58c..d679633784 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -654,7 +654,7 @@ void FullNodeImpl::update_private_block_overlays() { } void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { - CHECK(local_keys_.count(key)); + /*CHECK(local_keys_.count(key)); if (current_validators_.count(key)) { std::vector nodes; for (const auto &p : current_validators_) { @@ -663,7 +663,7 @@ void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { private_block_overlays_[key] = td::actor::create_actor( "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_); - } + }*/ } FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index add71ca5c4..df0c1453b2 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -388,8 +388,8 @@ class ValidatorManagerImpl : public ValidatorManager { } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { - queue_size_counter_ = - td::actor::create_actor("queuesizecounter", td::Ref{}, actor_id(this)); + queue_size_counter_ = td::actor::create_actor("queuesizecounter", td::Ref{}, + opts_, actor_id(this)); } td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::get_queue_size, block_id, std::move(promise)); } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 43c979eb9d..065cad62f0 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -450,8 +450,8 @@ class ValidatorManagerImpl : public ValidatorManager { } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { - queue_size_counter_ = - td::actor::create_actor("queuesizecounter", td::Ref{}, actor_id(this)); + queue_size_counter_ = td::actor::create_actor("queuesizecounter", td::Ref{}, + opts_, actor_id(this)); } td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::get_queue_size, block_id, std::move(promise)); } diff --git a/validator/manager.cpp b/validator/manager.cpp index f1d65127aa..1dceeee951 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3115,6 +3115,9 @@ void ValidatorManagerImpl::update_options(td::Ref opts) if (!out_msg_queue_importer_.empty()) { td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::update_options, opts); } + if (!queue_size_counter_.empty()) { + td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::update_options, opts); + } opts_ = std::move(opts); } diff --git a/validator/manager.hpp b/validator/manager.hpp index 045f69a255..df1cc77c6e 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -591,13 +591,17 @@ class ValidatorManagerImpl : public ValidatorManager { void update_options(td::Ref opts) override; void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { + if (last_masterchain_state_.is_null()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); + return; + } if (queue_size_counter_.empty()) { - if (last_masterchain_state_.is_null()) { - promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); - return; - } - queue_size_counter_ = td::actor::create_actor("queuesizecounter", - last_masterchain_state_, actor_id(this)); + queue_size_counter_ = + td::actor::create_actor("queuesizecounter", last_masterchain_state_, opts_, actor_id(this)); + } + if (!opts_->need_monitor(block_id.shard_full(), last_masterchain_state_)) { + return promise.set_error( + td::Status::Error(PSTRING() << "not monitoring shard " << block_id.shard_full().to_str())); } td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::get_queue_size, block_id, std::move(promise)); } diff --git a/validator/queue-size-counter.cpp b/validator/queue-size-counter.cpp index 4780f202c5..fd70cf2e7f 100644 --- a/validator/queue-size-counter.cpp +++ b/validator/queue-size-counter.cpp @@ -234,7 +234,9 @@ void QueueSizeCounter::process_top_shard_blocks_cont(td::Ref s last_top_blocks_.clear(); last_top_blocks_.push_back(state->get_block_id()); for (auto &shard : state->get_shards()) { - last_top_blocks_.push_back(shard->top_block_id()); + if (opts_->need_monitor(shard->shard(), state)) { + last_top_blocks_.push_back(shard->top_block_id()); + } } for (const BlockIdExt &block_id : last_top_blocks_) { get_queue_size_ex_retry(block_id, init, ig.get_promise()); diff --git a/validator/queue-size-counter.hpp b/validator/queue-size-counter.hpp index fabb0cec37..4c962fc173 100644 --- a/validator/queue-size-counter.hpp +++ b/validator/queue-size-counter.hpp @@ -21,16 +21,22 @@ namespace ton::validator { class QueueSizeCounter : public td::actor::Actor { public: - QueueSizeCounter(td::Ref last_masterchain_state, td::actor::ActorId manager) - : init_masterchain_state_(last_masterchain_state), manager_(std::move(manager)) { + QueueSizeCounter(td::Ref last_masterchain_state, td::Ref opts, + td::actor::ActorId manager) + : init_masterchain_state_(last_masterchain_state), opts_(std::move(opts)), manager_(std::move(manager)) { } void start_up() override; void get_queue_size(BlockIdExt block_id, td::Promise promise); void alarm() override; + void update_options(td::Ref opts) { + opts_ = std::move(opts); + } + private: td::Ref init_masterchain_state_; + td::Ref opts_; td::actor::ActorId manager_; bool simple_mode_ = false; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index dbbefa7dc5..79c1a2da4b 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -195,7 +195,7 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref } }); - run_accept_block_query(block_id, std::move(block), prev_block_ids_, validator_set_, std::move(sig_set), + run_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, std::move(sig_set), std::move(approve_sig_set), send_broadcast, shard_.is_masterchain() || mode_ == ValidatorManagerOptions::validator_normal, manager_, std::move(P)); From e575d273025add5c2a5232704c0ef85c3f3c15a9 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 19 Feb 2024 14:21:15 +0300 Subject: [PATCH 081/524] Remove unused priority_broadcast_receivers --- crypto/block/block.tlb | 2 +- crypto/block/mc-config.cpp | 7 +------ crypto/block/mc-config.h | 1 - overlay/overlay-broadcast.cpp | 8 ++------ overlay/overlay-broadcast.hpp | 4 +--- overlay/overlay-fec-broadcast.cpp | 20 +++++++------------- overlay/overlay-fec-broadcast.hpp | 13 ++++--------- overlay/overlay-manager.cpp | 13 ------------- overlay/overlay-manager.h | 2 -- overlay/overlay.h | 1 - overlay/overlay.hpp | 17 ----------------- overlay/overlays.h | 2 -- validator/full-node-shard.cpp | 13 ------------- validator/full-node-shard.h | 1 - validator/full-node-shard.hpp | 2 -- validator/full-node.cpp | 30 ------------------------------ validator/full-node.hpp | 4 ---- 17 files changed, 16 insertions(+), 124 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 9988f2d1c8..475276184c 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -782,7 +782,7 @@ misbehaviour_punishment_config_v1#01 _ MisbehaviourPunishmentConfig = ConfigParam 40; // collator_nodes: each collator is (workchain:int32 shard:uint64 adnl_id:uint256) -collator_info#0 full_node_id:(Maybe uint256) = CollatorInfo; +collator_info#00 = CollatorInfo; colator_config#a0 full_collated_data:Bool collator_nodes:(HashmapE 352 CollatorInfo) = CollatorConfig; _ CollatorConfig = ConfigParam 41; diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 5470693657..628093b0b3 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2333,12 +2333,7 @@ CollatorConfig Config::get_collator_config(bool need_collator_nodes) const { td::uint64 shard = key.get_uint(64); key.advance(64); td::Bits256 adnl_id(key); - td::Bits256 full_node_id = td::Bits256::zero(); - gen::CollatorInfo::Record info; - if (tlb::csr_unpack(std::move(value), info) && info.full_node_id->size() == 257) { - full_node_id = td::Bits256(info.full_node_id->data_bits() + 1); - } - collator_config.collator_nodes.push_back({ton::ShardIdFull(workchain, shard), adnl_id, full_node_id}); + collator_config.collator_nodes.push_back({ton::ShardIdFull(workchain, shard), adnl_id}); return true; }); } diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 1262a1d8e5..ef8f4dfa7c 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -529,7 +529,6 @@ struct BurningConfig { struct CollatorNodeDescr { ton::ShardIdFull shard; ton::NodeIdShort adnl_id; - ton::NodeIdShort full_node_id; }; struct CollatorConfig { diff --git a/overlay/overlay-broadcast.cpp b/overlay/overlay-broadcast.cpp index bcb7c28587..03991b76b4 100644 --- a/overlay/overlay-broadcast.cpp +++ b/overlay/overlay-broadcast.cpp @@ -68,11 +68,7 @@ td::Status BroadcastSimple::run_checks() { td::Status BroadcastSimple::distribute() { auto B = serialize(); - auto nodes = overlay_->get_neighbours(5); - if (is_ours_) { - auto priority_nodes = overlay_->get_priority_broadcast_receivers(3); - nodes.insert(nodes.end(), priority_nodes.begin(), priority_nodes.end()); - } + auto nodes = overlay_->get_neighbours(3); auto manager = overlay_->overlay_manager(); for (auto &n : nodes) { @@ -144,7 +140,7 @@ td::Status BroadcastSimple::create_new(td::actor::ActorId overlay, auto date = static_cast(td::Clocks::system()); auto B = std::make_unique(broadcast_hash, PublicKey{}, nullptr, flags, std::move(data), date, - td::BufferSlice{}, false, nullptr, adnl::AdnlNodeIdShort::zero(), true); + td::BufferSlice{}, false, nullptr, adnl::AdnlNodeIdShort::zero()); auto to_sign = B->to_sign(); auto P = td::PromiseCreator::lambda( diff --git a/overlay/overlay-broadcast.hpp b/overlay/overlay-broadcast.hpp index 8a5d132497..e7b39eecc0 100644 --- a/overlay/overlay-broadcast.hpp +++ b/overlay/overlay-broadcast.hpp @@ -46,7 +46,6 @@ class BroadcastSimple : public td::ListNode { td::uint32 date_; td::BufferSlice signature_; bool is_valid_{false}; - bool is_ours_{false}; OverlayImpl *overlay_; @@ -64,7 +63,7 @@ class BroadcastSimple : public td::ListNode { public: BroadcastSimple(Overlay::BroadcastHash broadcast_hash, PublicKey source, std::shared_ptr cert, td::uint32 flags, td::BufferSlice data, td::uint32 date, td::BufferSlice signature, bool is_valid, - OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, bool is_ours = false) + OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id) : broadcast_hash_(broadcast_hash) , source_(std::move(source)) , cert_(std::move(cert)) @@ -73,7 +72,6 @@ class BroadcastSimple : public td::ListNode { , date_(date) , signature_(std::move(signature)) , is_valid_(is_valid) - , is_ours_(is_ours) , overlay_(overlay) , src_peer_id_(src_peer_id) { } diff --git a/overlay/overlay-fec-broadcast.cpp b/overlay/overlay-fec-broadcast.cpp index e38ff78d70..aed5248b8d 100644 --- a/overlay/overlay-fec-broadcast.cpp +++ b/overlay/overlay-fec-broadcast.cpp @@ -26,8 +26,8 @@ namespace overlay { td::Result> BroadcastFec::create(Overlay::BroadcastHash hash, PublicKey src, Overlay::BroadcastDataHash data_hash, td::uint32 flags, - td::uint32 date, fec::FecType fec_type, bool is_ours) { - auto F = std::make_unique(hash, std::move(src), data_hash, flags, date, std::move(fec_type), is_ours); + td::uint32 date, fec::FecType fec_type) { + auto F = std::make_unique(hash, std::move(src), data_hash, flags, date, std::move(fec_type)); TRY_STATUS(F->run_checks()); TRY_STATUS(F->init_fec_type()); return std::move(F); @@ -94,12 +94,12 @@ void BroadcastFec::broadcast_checked(td::Result R) { overlay_->deliver_broadcast(get_source().compute_short_id(), data_.clone()); auto manager = overlay_->overlay_manager(); while (!parts_.empty()) { - distribute_part(parts_.begin()->first); + distribute_part(parts_.begin()->first); } } // Do we need status here?? -td::Status BroadcastFec::distribute_part(td::uint32 seqno) { +td::Status BroadcastFec::distribute_part(td::uint32 seqno) { auto i = parts_.find(seqno); if (i == parts_.end()) { // should not get here @@ -111,12 +111,8 @@ td::Status BroadcastFec::distribute_part(td::uint32 seqno) { td::BufferSlice data = std::move(tls.second); auto nodes = overlay_->get_neighbours(5); - if (is_ours_) { - auto priority_nodes = overlay_->get_priority_broadcast_receivers(3); - nodes.insert(nodes.end(), priority_nodes.begin(), priority_nodes.end()); - } - auto manager = overlay_->overlay_manager(); + for (auto &n : nodes) { if (neighbour_completed(n)) { continue; @@ -144,8 +140,7 @@ td::Status OverlayFecBroadcastPart::apply() { if (is_short_) { return td::Status::Error(ErrorCode::protoviolation, "short broadcast part for incomplete broadcast"); } - TRY_RESULT( - B, BroadcastFec::create(broadcast_hash_, source_, broadcast_data_hash_, flags_, date_, fec_type_, is_ours_)); + TRY_RESULT(B, BroadcastFec::create(broadcast_hash_, source_, broadcast_data_hash_, flags_, date_, fec_type_)); bcast_ = B.get(); overlay_->register_fec_broadcast(std::move(B)); } @@ -309,8 +304,7 @@ td::Status OverlayFecBroadcastPart::create_new(OverlayImpl *overlay, td::actor:: auto B = std::make_unique( broadcast_hash, part_hash, PublicKey{}, overlay->get_certificate(local_id), data_hash, size, flags, - part_data_hash, std::move(part), seqno, std::move(fec_type), date, td::BufferSlice{}, false, nullptr, overlay, - adnl::AdnlNodeIdShort::zero(), true); + part_data_hash, std::move(part), seqno, std::move(fec_type), date, td::BufferSlice{}, false, nullptr, overlay, adnl::AdnlNodeIdShort::zero()); auto to_sign = B->to_sign(); auto P = td::PromiseCreator::lambda( diff --git a/overlay/overlay-fec-broadcast.hpp b/overlay/overlay-fec-broadcast.hpp index c97ad88581..199574c31f 100644 --- a/overlay/overlay-fec-broadcast.hpp +++ b/overlay/overlay-fec-broadcast.hpp @@ -131,19 +131,18 @@ class BroadcastFec : public td::ListNode { td::Status run_checks(); BroadcastFec(Overlay::BroadcastHash hash, PublicKey src, Overlay::BroadcastDataHash data_hash, td::uint32 flags, - td::uint32 date, fec::FecType fec_type, bool is_ours = false) + td::uint32 date, fec::FecType fec_type) : hash_(hash) , data_hash_(data_hash) , flags_(flags) , date_(date) , src_(std::move(src)) - , fec_type_(std::move(fec_type)) - , is_ours_(is_ours) { + , fec_type_(std::move(fec_type)) { } static td::Result> create(Overlay::BroadcastHash hash, PublicKey src, Overlay::BroadcastDataHash data_hash, td::uint32 flags, - td::uint32 date, fec::FecType fec_type, bool is_ours); + td::uint32 date, fec::FecType fec_type); bool neighbour_received(adnl::AdnlNodeIdShort id) const { return received_neighbours_.find(id) != received_neighbours_.end(); @@ -226,7 +225,6 @@ class BroadcastFec : public td::ListNode { OverlayImpl *overlay_; adnl::AdnlNodeIdShort src_peer_id_ = adnl::AdnlNodeIdShort::zero(); td::BufferSlice data_; - bool is_ours_ = false; }; class OverlayFecBroadcastPart : public td::ListNode { @@ -248,7 +246,6 @@ class OverlayFecBroadcastPart : public td::ListNode { bool is_short_; bool untrusted_{false}; - bool is_ours_; BroadcastFec *bcast_; OverlayImpl *overlay_; @@ -268,8 +265,7 @@ class OverlayFecBroadcastPart : public td::ListNode { std::shared_ptr cert, Overlay::BroadcastDataHash data_hash, td::uint32 data_size, td::uint32 flags, Overlay::BroadcastDataHash part_data_hash, td::BufferSlice data, td::uint32 seqno, fec::FecType fec_type, td::uint32 date, td::BufferSlice signature, - bool is_short, BroadcastFec *bcast, OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, - bool is_ours = false) + bool is_short, BroadcastFec *bcast, OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id) : broadcast_hash_(broadcast_hash) , part_hash_(part_hash) , source_(std::move(source)) @@ -284,7 +280,6 @@ class OverlayFecBroadcastPart : public td::ListNode { , date_(date) , signature_(std::move(signature)) , is_short_(is_short) - , is_ours_(is_ours) , bcast_(bcast) , overlay_(overlay) , src_peer_id_(src_peer_id) { diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index 17e94aae79..0591e7320d 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -350,19 +350,6 @@ void OverlayManager::get_stats(td::Promise nodes) { - auto it = overlays_.find(local_id); - if (it == overlays_.end()) { - return; - } - auto it2 = it->second.find(overlay); - if (it2 == it->second.end()) { - return; - } - td::actor::send_closure(it2->second, &Overlay::set_priority_broadcast_receivers, std::move(nodes)); -} - void OverlayManager::forget_peer(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, adnl::AdnlNodeIdShort peer_id) { auto it = overlays_.find(local_id); diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index aeca69da73..659069bd8f 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -96,8 +96,6 @@ class OverlayManager : public Overlays { td::actor::ActorOwn overlay); void get_stats(td::Promise> promise) override; - void set_priority_broadcast_receivers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, - std::vector nodes) override; void forget_peer(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, adnl::AdnlNodeIdShort peer_id) override; struct PrintId {}; diff --git a/overlay/overlay.h b/overlay/overlay.h index 1b5264a850..b21de36289 100644 --- a/overlay/overlay.h +++ b/overlay/overlay.h @@ -69,7 +69,6 @@ class Overlay : public td::actor::Actor { virtual void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) = 0; //virtual void receive_broadcast(td::BufferSlice data) = 0; //virtual void subscribe(std::unique_ptr callback) = 0; - virtual void set_priority_broadcast_receivers(std::vector nodes) = 0; virtual void forget_peer(adnl::AdnlNodeIdShort peer_id) = 0; }; diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 1137aa8ddf..f3ef3ab41d 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -216,18 +216,6 @@ class OverlayImpl : public Overlay { return vec; } } - std::vector get_priority_broadcast_receivers(td::uint32 max_size = 0) const { - if (max_size == 0 || max_size >= priority_broadcast_receivers_.size()) { - return priority_broadcast_receivers_; - } else { - std::vector vec; - for (td::uint32 i = 0; i < max_size; i++) { - vec.push_back(priority_broadcast_receivers_[td::Random::fast( - 0, static_cast(priority_broadcast_receivers_.size()) - 1)]); - } - return vec; - } - } td::actor::ActorId overlay_manager() const { return manager_; } @@ -281,10 +269,6 @@ class OverlayImpl : public Overlay { } } - void set_priority_broadcast_receivers(std::vector nodes) override { - priority_broadcast_receivers_ = std::move(nodes); - } - void forget_peer(adnl::AdnlNodeIdShort peer_id) override { del_peer(peer_id); } @@ -364,7 +348,6 @@ class OverlayImpl : public Overlay { std::queue bcast_lru_; std::map> out_fec_bcasts_; - std::vector priority_broadcast_receivers_; void bcast_gc(); diff --git a/overlay/overlays.h b/overlay/overlays.h index cc93fae33f..62daa0d936 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -250,8 +250,6 @@ class Overlays : public td::actor::Actor { td::Promise> promise) = 0; virtual void get_stats(td::Promise> promise) = 0; - virtual void set_priority_broadcast_receivers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, - std::vector nodes) = 0; virtual void forget_peer(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, adnl::AdnlNodeIdShort peer_id) = 0; }; diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index fc2ae3a48b..026e8190aa 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -124,10 +124,6 @@ void FullNodeShardImpl::create_overlay() { if (cert_) { td::actor::send_closure(overlays_, &overlay::Overlays::update_certificate, adnl_id_, overlay_id_, local_id_, cert_); } - if (!collator_nodes_.empty()) { - td::actor::send_closure(overlays_, &overlay::Overlays::set_priority_broadcast_receivers, adnl_id_, overlay_id_, - collator_nodes_); - } } void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broadcast, td::Promise promise) { @@ -1141,15 +1137,6 @@ void FullNodeShardImpl::update_validators(std::vector public_key_ } } -void FullNodeShardImpl::update_collators(std::vector nodes) { - if (!client_.empty()) { - return; - } - collator_nodes_ = std::move(nodes); - td::actor::send_closure(overlays_, &overlay::Overlays::set_priority_broadcast_receivers, adnl_id_, overlay_id_, - collator_nodes_); -} - void FullNodeShardImpl::reload_neighbours() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 2131939fe8..d984aeadf9 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -77,7 +77,6 @@ class FullNodeShard : public td::actor::Actor { virtual void set_handle(BlockHandle handle, td::Promise promise) = 0; virtual void update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) = 0; - virtual void update_collators(std::vector nodes) = 0; static td::actor::ActorOwn create( ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index cf27cfeef5..102fc62129 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -199,7 +199,6 @@ class FullNodeShardImpl : public FullNodeShard { void alarm() override; void update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) override; - void update_collators(std::vector nodes) override; void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) override; void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) override; @@ -278,7 +277,6 @@ class FullNodeShardImpl : public FullNodeShard { adnl::AdnlNodeIdShort last_pinged_neighbour_ = adnl::AdnlNodeIdShort::zero(); FullNodeShardMode mode_; - std::vector collator_nodes_; FullNodeConfig config_; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index d679633784..b8842a61e1 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -218,33 +218,6 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s ++it; } } - - if (!collators_inited_ || state->is_key_state()) { - update_collators(state); - } -} - -void FullNodeImpl::update_collators(td::Ref state) { - collators_inited_ = true; - collator_config_ = state->get_collator_config(true); - for (auto& s : shards_) { - update_shard_collators(s.first, s.second); - } -} - -void FullNodeImpl::update_shard_collators(ShardIdFull shard, ShardInfo& info) { - if (info.actor.empty()) { - return; - } - std::vector nodes; - for (const block::CollatorNodeDescr& desc : collator_config_.collator_nodes) { - if (!desc.full_node_id.is_zero() && shard_intersects(shard, desc.shard)) { - nodes.emplace_back(desc.full_node_id); - } - } - std::sort(nodes.begin(), nodes.end()); - nodes.erase(std::unique(nodes.begin(), nodes.end()), nodes.end()); - td::actor::send_closure(info.actor, &FullNodeShard::update_collators, std::move(nodes)); } void FullNodeImpl::add_shard_actor(ShardIdFull shard, FullNodeShardMode mode) { @@ -259,9 +232,6 @@ void FullNodeImpl::add_shard_actor(ShardIdFull shard, FullNodeShardMode mode) { if (all_validators_.size() > 0) { td::actor::send_closure(info.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } - if (collators_inited_) { - update_shard_collators(shard, info); - } } void FullNodeImpl::sync_completed() { diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 5bb95186bf..5ed8101fbc 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -102,8 +102,6 @@ class FullNodeImpl : public FullNode { }; void add_shard_actor(ShardIdFull shard, FullNodeShardMode mode); - void update_collators(td::Ref state); - void update_shard_collators(ShardIdFull shard, ShardInfo& info); PublicKeyHash local_id_; adnl::AdnlNodeIdShort adnl_id_; @@ -131,8 +129,6 @@ class FullNodeImpl : public FullNode { std::set local_keys_; td::Promise started_promise_; - bool collators_inited_ = false; - block::CollatorConfig collator_config_; FullNodeConfig config_; std::map> private_block_overlays_; From 28699654fb35ad4f11204d0578832ba030850ed3 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 21 Feb 2024 18:38:02 +0300 Subject: [PATCH 082/524] New private block overlays --- tl/generate/scheme/ton_api.tl | 2 + tl/generate/scheme/ton_api.tlo | Bin 92172 -> 92852 bytes validator-engine/validator-engine.cpp | 12 + validator/CMakeLists.txt | 2 + validator/full-node-private-overlay-v2.cpp | 330 +++++++++++++++++++++ validator/full-node-private-overlay-v2.hpp | 109 +++++++ validator/full-node.cpp | 53 +++- validator/full-node.h | 2 + validator/full-node.hpp | 9 + 9 files changed, 505 insertions(+), 14 deletions(-) create mode 100644 validator/full-node-private-overlay-v2.cpp create mode 100644 validator/full-node-private-overlay-v2.hpp diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index e911eacfc9..aa761ad26a 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -398,6 +398,7 @@ tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast; tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int256 = tonNode.ShardPublicOverlayId; tonNode.privateBlockOverlayId zero_state_file_hash:int256 nodes:(vector int256) = tonNode.PrivateBlockOverlayId; +tonNode.privateBlockOverlayIdV2 zero_state_file_hash:int256 workchain:int shard:long nodes:(vector int256) senders:(vector int256) = tonNode.PrivateBlockOverlayIdV2; tonNode.keyBlocks blocks:(vector tonNode.blockIdExt) incomplete:Bool error:Bool = tonNode.KeyBlocks; @@ -665,6 +666,7 @@ engine.validator.shardOverlayStats.neighbour id:string proto_verison:int capabil roundtrip:double unreliability:double has_state:string = engine.validator.shardOverlayStats.Neighbour; engine.validator.shardOverlayStats shard:string mode:string neighbours:(vector engine.validator.shardOverlayStats.neighbour) = engine.validator.ShardOverlayStats; +engine.validator.privateBlockOverlayV2Stats shard:string nodes:(vector int256) senders:(vector int256) created_at:int = engine.validator.PrivateBlockOverlayV2Stats; engine.validator.onePerfTimerStat time:int min:double avg:double max:double = engine.validator.OnePerfTimerStat; engine.validator.perfTimerStatsByName name:string stats:(vector engine.validator.OnePerfTimerStat) = engine.validator.PerfTimerStatsByName; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index afd8e0fb16b8b5e5cfd48d072aa4604134a85ceb..3afe3f33e8e089d6d7691e781d0441ed40885811 100644 GIT binary patch delta 304 zcmeCVz`ErsEAOM(`c@23Ah?m2Thid_(t|zPsd?#{d8vA3i8+}mi6!|(dI3e5Wr-!J zPC5C>+5TmzMLCI;VMf6vi6zCGBy}WIjuy;dlP}57^UF_3#irIXCCq5DfT`qWUBi49 z#%Y`9n#vclf^C|7aG9_oF0(-@ZXOfzVVPXGLVfaqMJkhHR`X2Wu$YylI5jV2^1?KE zkic{W1x8k`nu02Z-!bN~PV delta 52 zcmdmTm9^&rEAOM(`c@23z`v1~TXJ)Pq=>}k4Tc3Qj8ir%n#mV#c35#IYP$mqV@VnS Dn7a@P diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 5abf913aa9..c3d3742482 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1937,6 +1937,10 @@ void ValidatorEngine::start_full_node() { td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_permanent_key, v.first, [](td::Unit) {}); } + for (auto &c : config_.collators) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_collator_adnl_id, + ton::adnl::AdnlNodeIdShort(c.adnl_id)); + } } else { started_full_node(); } @@ -3647,6 +3651,10 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollat td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, ton::adnl::AdnlNodeIdShort(id), shard); } + if (!full_node_.empty()) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_collator_adnl_id, + ton::adnl::AdnlNodeIdShort(id)); + } write_config([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); @@ -3722,6 +3730,10 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCollat td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::del_collator, ton::adnl::AdnlNodeIdShort(id), shard); } + if (!full_node_.empty()) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::del_collator_adnl_id, + ton::adnl::AdnlNodeIdShort(id)); + } write_config([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index afeb779346..0cde874b43 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -149,6 +149,8 @@ set(FULL_NODE_SOURCE full-node-master.cpp full-node-private-overlay.hpp full-node-private-overlay.cpp + full-node-private-overlay-v2.hpp + full-node-private-overlay-v2.cpp net/download-block.hpp net/download-block.cpp diff --git a/validator/full-node-private-overlay-v2.cpp b/validator/full-node-private-overlay-v2.cpp new file mode 100644 index 0000000000..fedb8a788b --- /dev/null +++ b/validator/full-node-private-overlay-v2.cpp @@ -0,0 +1,330 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "full-node-private-overlay-v2.hpp" +#include "ton/ton-tl.hpp" +#include "common/delay.h" +#include "td/utils/JsonBuilder.h" +#include "tl/tl_json.h" +#include "auto/tl/ton_api_json.h" + +namespace ton::validator::fullnode { + +void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash, ton_api::tonNode_blockBroadcast &query) { + std::vector signatures; + for (auto &sig : query.signatures_) { + signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); + } + + BlockIdExt block_id = create_block_id(query.id_); + BlockBroadcast B{block_id, + std::move(signatures), + static_cast(query.catchain_seqno_), + static_cast(query.validator_set_hash_), + std::move(query.data_), + std::move(query.proof_)}; + + auto P = td::PromiseCreator::lambda([](td::Result R) { + if (R.is_error()) { + if (R.error().code() == ErrorCode::notready) { + LOG(DEBUG) << "dropped broadcast: " << R.move_as_error(); + } else { + LOG(INFO) << "dropped broadcast: " << R.move_as_error(); + } + } + }); + LOG(FULL_NODE_DEBUG) << "Got block broadcast in private overlay: " << B.block_id.to_str(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(B), + std::move(P)); +} + +void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash, ton_api::tonNode_newShardBlockBroadcast &query) { + BlockIdExt block_id = create_block_id(query.block_->block_); + LOG(FULL_NODE_DEBUG) << "Got block description broadcast in private overlay: " << block_id.to_str(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, block_id, + query.block_->cc_seqno_, std::move(query.block_->data_)); +} + +void FullNodePrivateOverlayV2::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { + auto B = fetch_tl_object(std::move(broadcast), true); + if (B.is_error()) { + return; + } + + ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); }); +} + +void FullNodePrivateOverlayV2::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) { + if (!inited_) { + return; + } + auto B = create_serialize_tl_object( + create_tl_object(create_tl_block_id(block_id), cc_seqno, std::move(data))); + if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(B)); + } else { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); + } +} + +void FullNodePrivateOverlayV2::send_broadcast(BlockBroadcast broadcast) { + if (!inited_) { + return; + } + std::vector> sigs; + for (auto &sig : broadcast.signatures) { + sigs.emplace_back(create_tl_object(sig.node, sig.signature.clone())); + } + auto B = create_serialize_tl_object( + create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs), + broadcast.proof.clone(), broadcast.data.clone()); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); +} + +void FullNodePrivateOverlayV2::start_up() { + std::sort(nodes_.begin(), nodes_.end()); + nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end()); + + std::vector nodes, senders; + for (const adnl::AdnlNodeIdShort &id : nodes_) { + nodes.push_back(id.bits256_value()); + } + for (const adnl::AdnlNodeIdShort &id : senders_) { + senders.push_back(id.bits256_value()); + } + auto X = create_hash_tl_object( + zero_state_file_hash_, shard_.workchain, shard_.shard, std::move(nodes), std::move(senders)); + td::BufferSlice b{32}; + b.as_slice().copy_from(as_slice(X)); + overlay_id_full_ = overlay::OverlayIdFull{std::move(b)}; + overlay_id_ = overlay_id_full_.compute_short_id(); + + try_init(); +} + +void FullNodePrivateOverlayV2::try_init() { + // Sometimes adnl id is added to validator engine later (or not at all) + td::actor::send_closure( + adnl_, &adnl::Adnl::check_id_exists, local_id_, [SelfId = actor_id(this)](td::Result R) { + if (R.is_ok() && R.ok()) { + td::actor::send_closure(SelfId, &FullNodePrivateOverlayV2::init); + } else { + delay_action([SelfId]() { td::actor::send_closure(SelfId, &FullNodePrivateOverlayV2::try_init); }, + td::Timestamp::in(30.0)); + } + }); +} + +void FullNodePrivateOverlayV2::init() { + LOG(FULL_NODE_INFO) << "Creating private block overlay for shard " << shard_.to_str() << ", adnl_id=" << local_id_ + << " : " << nodes_.size() << " nodes"; + class Callback : public overlay::Overlays::Callback { + public: + void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + } + void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + td::actor::send_closure(node_, &FullNodePrivateOverlayV2::receive_broadcast, src, std::move(data)); + } + void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + } + void get_stats_extra(td::Promise promise) override { + td::actor::send_closure(node_, &FullNodePrivateOverlayV2::get_stats_extra, std::move(promise)); + } + Callback(td::actor::ActorId node) : node_(node) { + } + + private: + td::actor::ActorId node_; + }; + + std::map authorized_keys; + for (const adnl::AdnlNodeIdShort &sender : senders_) { + authorized_keys[sender.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size(); + } + overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), 0, std::move(authorized_keys)}; + td::actor::send_closure(overlays_, &overlay::Overlays::create_private_overlay, local_id_, overlay_id_full_.clone(), + nodes_, std::make_unique(actor_id(this)), rules); + + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); + td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); + inited_ = true; +} + +void FullNodePrivateOverlayV2::tear_down() { + if (inited_) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_); + } +} + +void FullNodePrivateOverlayV2::get_stats_extra(td::Promise promise) { + auto res = create_tl_object(); + res->shard_ = shard_.to_str(); + for (const auto &x : nodes_) { + res->nodes_.push_back(x.bits256_value()); + } + for (const auto &x : senders_) { + res->senders_.push_back(x.bits256_value()); + } + res->created_at_ = created_at_; + promise.set_result(td::json_encode(td::ToJson(*res), true)); +} + +td::actor::ActorId FullNodePrivateBlockOverlays::choose_overlay(ShardIdFull shard) { + for (auto &p : id_to_overlays_) { + auto &overlays = p.second.overlays_; + ShardIdFull cur_shard = shard; + while (true) { + auto it = overlays.find(cur_shard); + if (it != overlays.end() && it->second.is_sender_) { + return it->second.overlay_.get(); + } + if (cur_shard.pfx_len() == 0) { + break; + } + cur_shard = shard_parent(cur_shard); + } + } + return {}; +} + +void FullNodePrivateBlockOverlays::update_overlays( + td::Ref state, std::set my_adnl_ids, const FileHash &zero_state_file_hash, + const td::actor::ActorId &keyring, const td::actor::ActorId &adnl, + const td::actor::ActorId &rldp, const td::actor::ActorId &rldp2, + const td::actor::ActorId &overlays, + const td::actor::ActorId &validator_manager) { + if (my_adnl_ids.empty()) { + id_to_overlays_.clear(); + return; + } + auto collators = state->get_collator_config(true); + auto all_validators = state->get_total_validator_set(0); + + struct OverlayInfo { + std::vector nodes, senders; + }; + std::map pverlay_infos; + + // Masterchain overlay: all validators + collators + OverlayInfo &mc_overlay = pverlay_infos[ShardIdFull(masterchainId)]; + for (const auto &x : all_validators->export_vector()) { + td::Bits256 addr = x.addr.is_zero() ? ValidatorFullId(x.key).compute_short_id().bits256_value() : x.addr; + mc_overlay.nodes.emplace_back(addr); + mc_overlay.senders.emplace_back(addr); + } + for (const auto &x : collators.collator_nodes) { + mc_overlay.nodes.emplace_back(x.adnl_id); + } + + // Shard overlays: validators of the shard + collators of the shard + // See ValidatorManagerImpl::update_shards + std::set new_shards; + for (auto &v : state->get_shards()) { + ShardIdFull shard = v->shard(); + if (shard.is_masterchain()) { + continue; + } + if (v->before_split()) { + ShardIdFull l_shard{shard.workchain, shard_child(shard.shard, true)}; + ShardIdFull r_shard{shard.workchain, shard_child(shard.shard, false)}; + new_shards.insert(l_shard); + new_shards.insert(r_shard); + } else if (v->before_merge()) { + ShardIdFull p_shard{shard.workchain, shard_parent(shard.shard)}; + new_shards.insert(p_shard); + } else { + new_shards.insert(shard); + } + } + for (ShardIdFull shard : new_shards) { + auto val_set = state->get_validator_set(shard); + td::uint32 min_split = state->monitor_min_split_depth(shard.workchain); + OverlayInfo &overlay = + pverlay_infos[shard_prefix_length(shard) <= min_split ? shard : shard_prefix(shard, min_split)]; + for (const auto &x : val_set->export_vector()) { + td::Bits256 addr = x.addr.is_zero() ? ValidatorFullId(x.key).compute_short_id().bits256_value() : x.addr; + overlay.nodes.emplace_back(addr); + overlay.senders.emplace_back(addr); + } + } + for (auto &p : pverlay_infos) { + ShardIdFull shard = p.first; + OverlayInfo &overlay = p.second; + if (!shard.is_masterchain()) { + for (const auto &collator : collators.collator_nodes) { + if (shard_intersects(collator.shard, shard)) { + overlay.nodes.emplace_back(collator.adnl_id); + } + } + } + + std::sort(overlay.nodes.begin(), overlay.nodes.end()); + overlay.nodes.erase(std::unique(overlay.nodes.begin(), overlay.nodes.end()), overlay.nodes.end()); + std::sort(overlay.senders.begin(), overlay.senders.end()); + overlay.senders.erase(std::unique(overlay.senders.begin(), overlay.senders.end()), overlay.senders.end()); + } + + std::map old_private_block_overlays = std::move(id_to_overlays_); + id_to_overlays_.clear(); + + for (const auto &p : pverlay_infos) { + ShardIdFull shard = p.first; + const OverlayInfo &new_overlay_info = p.second; + for (adnl::AdnlNodeIdShort local_id : new_overlay_info.nodes) { + if (!my_adnl_ids.count(local_id)) { + continue; + } + Overlays::ShardOverlay &new_overlay = id_to_overlays_[local_id].overlays_[shard]; + Overlays::ShardOverlay &old_overlay = old_private_block_overlays[local_id].overlays_[shard]; + if (!old_overlay.overlay_.empty() && old_overlay.nodes_ == new_overlay_info.nodes && + old_overlay.senders_ == new_overlay_info.senders) { + new_overlay = std::move(old_overlay); + old_overlay = {}; + } else { + new_overlay.nodes_ = new_overlay_info.nodes; + new_overlay.senders_ = new_overlay_info.senders; + new_overlay.is_sender_ = std::binary_search(new_overlay.senders_.begin(), new_overlay.senders_.end(), local_id); + new_overlay.overlay_ = td::actor::create_actor( + "BlocksPrivateOverlay", local_id, shard, new_overlay.nodes_, new_overlay.senders_, zero_state_file_hash, + keyring, adnl, rldp, rldp2, overlays, validator_manager); + } + } + } + + // Delete old overlays, but not immediately + for (auto &p : old_private_block_overlays) { + for (auto &x : p.second.overlays_) { + if (x.second.overlay_.empty()) { + continue; + } + td::actor::ActorId id = x.second.overlay_.release(); + delay_action([id = std::move(id)]() { td::actor::send_closure(id, &FullNodePrivateOverlayV2::destroy); }, + td::Timestamp::in(60.0)); + } + } +} + +} // namespace ton::validator::fullnode diff --git a/validator/full-node-private-overlay-v2.hpp b/validator/full-node-private-overlay-v2.hpp new file mode 100644 index 0000000000..6fbcbbf717 --- /dev/null +++ b/validator/full-node-private-overlay-v2.hpp @@ -0,0 +1,109 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "full-node.h" + +namespace ton::validator::fullnode { + +class FullNodePrivateOverlayV2 : public td::actor::Actor { + public: + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + template + void process_broadcast(PublicKeyHash, T &) { + VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; + } + void receive_broadcast(PublicKeyHash src, td::BufferSlice query); + + void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data); + void send_broadcast(BlockBroadcast broadcast); + + void start_up() override; + void tear_down() override; + + void destroy() { + stop(); + } + + FullNodePrivateOverlayV2(adnl::AdnlNodeIdShort local_id, ShardIdFull shard, std::vector nodes, + std::vector senders, FileHash zero_state_file_hash, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId rldp2, + td::actor::ActorId overlays, + td::actor::ActorId validator_manager) + : local_id_(local_id) + , shard_(shard) + , nodes_(std::move(nodes)) + , senders_(std::move(senders)) + , zero_state_file_hash_(zero_state_file_hash) + , keyring_(keyring) + , adnl_(adnl) + , rldp_(rldp) + , rldp2_(rldp2) + , overlays_(overlays) + , validator_manager_(validator_manager) { + } + + private: + adnl::AdnlNodeIdShort local_id_; + ShardIdFull shard_; + std::vector nodes_; + std::vector senders_; + FileHash zero_state_file_hash_; + + td::actor::ActorId keyring_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + td::actor::ActorId rldp2_; + td::actor::ActorId overlays_; + td::actor::ActorId validator_manager_; + + bool inited_ = false; + overlay::OverlayIdFull overlay_id_full_; + overlay::OverlayIdShort overlay_id_; + UnixTime created_at_ = (UnixTime)td::Clocks::system(); + + void try_init(); + void init(); + void get_stats_extra(td::Promise promise); +}; + +class FullNodePrivateBlockOverlays { + public: + td::actor::ActorId choose_overlay(ShardIdFull shard); + void update_overlays(td::Ref state, std::set my_adnl_ids, + const FileHash& zero_state_file_hash, const td::actor::ActorId& keyring, + const td::actor::ActorId& adnl, const td::actor::ActorId& rldp, + const td::actor::ActorId& rldp2, + const td::actor::ActorId& overlays, + const td::actor::ActorId& validator_manager); + + private: + struct Overlays { + struct ShardOverlay { + td::actor::ActorOwn overlay_; + std::vector nodes_, senders_; + bool is_sender_ = false; + }; + std::map overlays_; + }; + + std::map id_to_overlays_; // local_id -> overlays +}; + +} // namespace ton::validator::fullnode diff --git a/validator/full-node.cpp b/validator/full-node.cpp index b8842a61e1..57323c4b18 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -20,6 +20,7 @@ #include "ton/ton-io.hpp" #include "td/actor/MultiPromise.h" #include "full-node.h" +#include "common/delay.h" namespace ton { @@ -53,7 +54,7 @@ void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise pr td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } } - create_private_block_overlay(key); + // create_private_block_overlay(key); promise.set_value(td::Unit()); } @@ -80,10 +81,20 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } } - private_block_overlays_.erase(key); + // private_block_overlays_.erase(key); promise.set_value(td::Unit()); } +void FullNodeImpl::add_collator_adnl_id(adnl::AdnlNodeIdShort id) { + ++local_collator_nodes_[id]; +} + +void FullNodeImpl::del_collator_adnl_id(adnl::AdnlNodeIdShort id) { + if (--local_collator_nodes_[id] == 0) { + local_collator_nodes_.erase(id); + } +} + void FullNodeImpl::sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) { auto it = shards_.find(shard_id); @@ -218,6 +229,19 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s ++it; } } + + std::set my_adnl_ids; + for (const auto &p : local_collator_nodes_) { + my_adnl_ids.insert(p.first); + } + for (auto key : local_keys_) { + auto it = current_validators_.find(key); + if (it != current_validators_.end()) { + my_adnl_ids.insert(it->second); + } + } + private_block_overlays_.update_overlays(state, std::move(my_adnl_ids), zero_state_file_hash_, keyring_, adnl_, rldp_, + rldp2_, overlays_, validator_manager_); } void FullNodeImpl::add_shard_actor(ShardIdFull shard, FullNodeShardMode mode) { @@ -262,9 +286,10 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; return; } - if (!private_block_overlays_.empty()) { - td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateOverlay::send_shard_block_info, - block_id, cc_seqno, data.clone()); + auto private_overlay = private_block_overlays_.choose_overlay(ShardIdFull(masterchainId)); + if (!private_overlay.empty()) { + td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_shard_block_info, block_id, cc_seqno, + data.clone()); } td::actor::send_closure(shard, &FullNodeShard::send_shard_block_info, block_id, cc_seqno, std::move(data)); } @@ -275,9 +300,9 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast) { VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; return; } - if (!private_block_overlays_.empty()) { - td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateOverlay::send_broadcast, - broadcast.clone()); + auto private_overlay = private_block_overlays_.choose_overlay(broadcast.block_id.shard_full()); + if (!private_overlay.empty()) { + td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_broadcast, broadcast.clone()); } td::actor::send_closure(shard, &FullNodeShard::send_broadcast, std::move(broadcast)); } @@ -440,7 +465,7 @@ void FullNodeImpl::got_key_block_proof(td::Ref proof) { if (current_validators != current_validators_) { current_validators_ = std::move(current_validators); - update_private_block_overlays(); + // update_private_block_overlays(); } if (keys == all_validators_) { @@ -483,7 +508,7 @@ void FullNodeImpl::got_zero_block_state(td::Ref state) { if (current_validators != current_validators_) { current_validators_ = std::move(current_validators); - update_private_block_overlays(); + // update_private_block_overlays(); } if (keys == all_validators_) { @@ -613,7 +638,7 @@ void FullNodeImpl::start_up() { std::make_unique(actor_id(this)), std::move(started_promise_)); } -void FullNodeImpl::update_private_block_overlays() { +/* void FullNodeImpl::update_private_block_overlays() { private_block_overlays_.clear(); if (local_keys_.empty()) { return; @@ -624,7 +649,7 @@ void FullNodeImpl::update_private_block_overlays() { } void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { - /*CHECK(local_keys_.count(key)); + CHECK(local_keys_.count(key)); if (current_validators_.count(key)) { std::vector nodes; for (const auto &p : current_validators_) { @@ -633,8 +658,8 @@ void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { private_block_overlays_[key] = td::actor::create_actor( "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_); - }*/ -} + } +} */ FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, diff --git a/validator/full-node.h b/validator/full-node.h index b10baeda7d..65e6fae0b7 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -63,6 +63,8 @@ class FullNode : public td::actor::Actor { virtual void add_permanent_key(PublicKeyHash key, td::Promise promise) = 0; virtual void del_permanent_key(PublicKeyHash key, td::Promise promise) = 0; + virtual void add_collator_adnl_id(adnl::AdnlNodeIdShort id) = 0; + virtual void del_collator_adnl_id(adnl::AdnlNodeIdShort id) = 0; virtual void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) = 0; diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 5ed8101fbc..ff998f320b 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -24,6 +24,7 @@ #include "interfaces/proof.h" #include "interfaces/shard.h" #include "full-node-private-overlay.hpp" +#include "full-node-private-overlay-v2.hpp" #include #include @@ -42,6 +43,8 @@ class FullNodeImpl : public FullNode { void add_permanent_key(PublicKeyHash key, td::Promise promise) override; void del_permanent_key(PublicKeyHash key, td::Promise promise) override; + void add_collator_adnl_id(adnl::AdnlNodeIdShort id) override; + void del_collator_adnl_id(adnl::AdnlNodeIdShort id) override; void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, @@ -127,14 +130,20 @@ class FullNodeImpl : public FullNode { std::map current_validators_; std::set local_keys_; + std::map local_collator_nodes_; td::Promise started_promise_; FullNodeConfig config_; + // TODO: Decide what to do with old private overlays. Maybe use old or new depending on some flag in config. + /* std::map> private_block_overlays_; void update_private_block_overlays(); void create_private_block_overlay(PublicKeyHash key); + */ + + FullNodePrivateBlockOverlays private_block_overlays_; }; } // namespace fullnode From 0fb781b4c3c7cabb675167f003cb1a3a6b81e8dd Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 22 Feb 2024 14:55:11 +0300 Subject: [PATCH 083/524] Improve queue cleanup (after merge) --- validator/impl/collator.cpp | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 50e01fd7f8..c0c3cf16ff 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -2323,8 +2323,22 @@ bool Collator::out_msg_queue_cleanup() { << nb.blk_.to_str()); } } + auto queue_root = out_msg_queue_->get_root_cell(); + if (queue_root.is_null()) { + LOG(DEBUG) << "out_msg_queue is empty"; + return true; + } + // Unwrap UsageCell: don't build proof for visiting output queue (unless something is deleted) + auto r_cell = queue_root->load_cell(); + if (r_cell.is_error()) { + return fatal_error(r_cell.move_as_error()); + } + auto pure_out_msg_queue = + std::make_unique(r_cell.move_as_ok().data_cell, 352, block::tlb::aug_OutMsgQueue); td::uint32 deleted = 0; - auto res = out_msg_queue_->filter([&](vm::CellSlice& cs, td::ConstBitPtr key, int n) -> int { + bool fail = false; + pure_out_msg_queue->check_for_each([&](Ref value, td::ConstBitPtr key, int n) -> bool { + vm::CellSlice& cs = value.write(); assert(n == 352); block::EnqueuedMsgDescr enq_msg_descr; unsigned long long created_lt; @@ -2333,7 +2347,8 @@ bool Collator::out_msg_queue_cleanup() { && enq_msg_descr.check_key(key) // check key && enq_msg_descr.lt_ == created_lt)) { LOG(ERROR) << "cannot unpack EnqueuedMsg with key " << key.to_hex(n); - return -1; + fail = true; + return false; } LOG(DEBUG) << "scanning outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," << enq_msg_descr.hash_.to_hex() << ") enqueued_lt=" << enq_msg_descr.enqueued_lt_; @@ -2354,10 +2369,20 @@ bool Collator::out_msg_queue_cleanup() { --out_msg_queue_size_; LOG(DEBUG) << "outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," << enq_msg_descr.hash_.to_hex() << ") enqueued_lt=" << enq_msg_descr.enqueued_lt_ << " has been already delivered, dequeueing"; + // Get value from out_msg_queue_ instead of pure_out_msg_queue (for proof) + auto value2 = out_msg_queue_->lookup_delete_with_extra(key, n); + CHECK(value2.not_null()); + vm::CellSlice& cs2 = value2.write(); + CHECK(cs2.fetch_ulong_bool(64, created_lt) // augmentation + && enq_msg_descr.unpack(cs2) // unpack EnqueuedMsg + && enq_msg_descr.check_key(key) // check key + && enq_msg_descr.lt_ == created_lt); + if (!dequeue_message(std::move(enq_msg_descr.msg_env_), deliver_lt)) { fatal_error(PSTRING() << "cannot dequeue outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," << enq_msg_descr.hash_.to_hex() << ") by inserting a msg_export_deq record"); - return -1; + fail = true; + return false; } register_out_msg_queue_op(); if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) { @@ -2368,7 +2393,7 @@ bool Collator::out_msg_queue_cleanup() { }); LOG(WARNING) << "deleted " << deleted << " messages from out_msg_queue after merge, remaining queue size is " << out_msg_queue_size_; - if (res < 0) { + if (fail) { return fatal_error("error scanning/updating OutMsgQueue"); } } else { From 6c680dbc4e9b106b7fed3a151919715bedd82d07 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 22 Feb 2024 15:00:41 +0300 Subject: [PATCH 084/524] Cleanup code in validator-group.cpp (after merge) --- validator/validator-group.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 79c1a2da4b..ed13599333 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -130,7 +130,6 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat return; } VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id; - block.id = next_block_id; run_validate_query(shard_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, manager_, td::Timestamp::in(15.0), std::move(P), collator_config_.full_collated_data ? ValidateMode::full_collated_data : 0); @@ -170,7 +169,6 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s prev_block_ids_ = std::vector{next_block_id}; cached_collated_block_ = nullptr; approved_candidates_cache_.clear(); - approved_candidates_cache_.clear(); } void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, From fc6e2ead29f5d141b22b6dca9c0f4ec9bd051a20 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 15 Mar 2024 14:13:07 +0300 Subject: [PATCH 085/524] Revert "Process adnl query errors" This reverts commit 1869a2506287dbc66e04ac52ab7b620e9960371e. --- adnl/adnl-message.cpp | 3 --- adnl/adnl-message.h | 21 +-------------------- adnl/adnl-peer.cpp | 16 ---------------- adnl/adnl-peer.hpp | 1 - rldp/rldp-in.hpp | 4 ---- rldp/rldp.cpp | 22 ---------------------- rldp2/rldp-in.hpp | 4 ---- rldp2/rldp.cpp | 22 ---------------------- tl/generate/scheme/ton_api.tl | 2 -- tl/generate/scheme/ton_api.tlo | Bin 92852 -> 92660 bytes 10 files changed, 1 insertion(+), 94 deletions(-) diff --git a/adnl/adnl-message.cpp b/adnl/adnl-message.cpp index 1d3ba5f8a2..0d7129783f 100644 --- a/adnl/adnl-message.cpp +++ b/adnl/adnl-message.cpp @@ -46,9 +46,6 @@ AdnlMessage::AdnlMessage(tl_object_ptr message) { [&](ton_api::adnl_message_part &msg) { message_ = adnlmessage::AdnlMessagePart{msg.hash_, static_cast(msg.total_size_), static_cast(msg.offset_), std::move(msg.data_)}; - }, - [&](ton_api::adnl_message_queryError &msg) { - message_ = adnlmessage::AdnlMessageQueryError{msg.query_id_}; })); } diff --git a/adnl/adnl-message.h b/adnl/adnl-message.h index d557c31953..43849e9822 100644 --- a/adnl/adnl-message.h +++ b/adnl/adnl-message.h @@ -170,24 +170,6 @@ class AdnlMessageAnswer { td::BufferSlice data_; }; -class AdnlMessageQueryError { - public: - explicit AdnlMessageQueryError(AdnlQueryId query_id) : query_id_(query_id) { - } - const auto &query_id() const { - return query_id_; - } - td::uint32 size() const { - return 36; - } - tl_object_ptr tl() const { - return create_tl_object(query_id_); - } - - private: - AdnlQueryId query_id_; -}; - class AdnlMessagePart { public: AdnlMessagePart(td::Bits256 hash, td::uint32 total_size, td::uint32 offset, td::BufferSlice data) @@ -238,8 +220,7 @@ class AdnlMessage { private: td::Variant + adnlmessage::AdnlMessageQuery, adnlmessage::AdnlMessageAnswer, adnlmessage::AdnlMessagePart> message_{Empty{}}; public: diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index 85632f1de0..3e21a7f53a 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -536,14 +536,10 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQuery &mess flags = static_cast(0)](td::Result R) { if (R.is_error()) { LOG(WARNING) << "failed to answer query: " << R.move_as_error(); - td::actor::send_closure(SelfId, &AdnlPeerPairImpl::send_message, - OutboundAdnlMessage{adnlmessage::AdnlMessageQueryError{query_id}, flags}); } else { auto data = R.move_as_ok(); if (data.size() > Adnl::huge_packet_max_size()) { LOG(WARNING) << "dropping too big answer query: size=" << data.size(); - td::actor::send_closure(SelfId, &AdnlPeerPairImpl::send_message, - OutboundAdnlMessage{adnlmessage::AdnlMessageQueryError{query_id}, flags}); } else { td::actor::send_closure(SelfId, &AdnlPeerPairImpl::send_message, OutboundAdnlMessage{adnlmessage::AdnlMessageAnswer{query_id, std::move(data)}, flags}); @@ -625,18 +621,6 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessagePart &messa } } -void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQueryError &message) { - auto Q = out_queries_.find(message.query_id()); - - if (Q == out_queries_.end()) { - VLOG(ADNL_NOTICE) << this << ": dropping IN query error: unknown query id " << message.query_id(); - return; - } - - td::actor::send_closure_later(Q->second, &AdnlQuery::set_error, td::Status::Error("adnl query rejected")); - out_queries_.erase(Q); -} - void AdnlPeerPairImpl::delete_query(AdnlQueryId id) { auto Q = out_queries_.find(id); diff --git a/adnl/adnl-peer.hpp b/adnl/adnl-peer.hpp index 041de23c4a..12ee01c6c8 100644 --- a/adnl/adnl-peer.hpp +++ b/adnl/adnl-peer.hpp @@ -104,7 +104,6 @@ class AdnlPeerPairImpl : public AdnlPeerPair { void process_message(const adnlmessage::AdnlMessageQuery &message); void process_message(const adnlmessage::AdnlMessageAnswer &message); void process_message(const adnlmessage::AdnlMessagePart &message); - void process_message(const adnlmessage::AdnlMessageQueryError &message); void process_message(const AdnlMessage::Empty &message) { UNREACHABLE(); } diff --git a/rldp/rldp-in.hpp b/rldp/rldp-in.hpp index 2073b59caf..266f128a5a 100644 --- a/rldp/rldp-in.hpp +++ b/rldp/rldp-in.hpp @@ -78,8 +78,6 @@ class RldpIn : public RldpImpl { td::uint64 max_answer_size) override; void answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, adnl::AdnlQueryId query_id, TransferId transfer_id, td::BufferSlice data); - void reject_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, - adnl::AdnlQueryId query_id, TransferId transfer_id); void alarm_query(adnl::AdnlQueryId query_id, TransferId transfer_id); @@ -95,8 +93,6 @@ class RldpIn : public RldpImpl { ton_api::rldp_query &message); void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, ton_api::rldp_answer &message); - void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, - ton_api::rldp_queryError &message); void receive_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, td::BufferSlice data); diff --git a/rldp/rldp.cpp b/rldp/rldp.cpp index 3c9e7b27f7..402740e19a 100644 --- a/rldp/rldp.cpp +++ b/rldp/rldp.cpp @@ -87,13 +87,6 @@ void RldpIn::answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, transfer(src, dst, timeout, std::move(B), transfer_id); } -void RldpIn::reject_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, - adnl::AdnlQueryId query_id, TransferId transfer_id) { - auto B = serialize_tl_object(create_tl_object(query_id), true); - - transfer(src, dst, timeout, std::move(B), transfer_id); -} - void RldpIn::alarm_query(adnl::AdnlQueryId query_id, TransferId transfer_id) { queries_.erase(query_id); max_size_.erase(transfer_id); @@ -206,16 +199,12 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort auto data = R.move_as_ok(); if (data.size() > max_answer_size) { VLOG(RLDP_NOTICE) << "rldp query failed: answer too big"; - td::actor::send_closure(SelfId, &RldpIn::reject_query, local_id, source, timeout, query_id, - transfer_id ^ TransferId::ones()); } else { td::actor::send_closure(SelfId, &RldpIn::answer_query, local_id, source, timeout, query_id, transfer_id ^ TransferId::ones(), std::move(data)); } } else { VLOG(RLDP_NOTICE) << "rldp query failed: " << R.move_as_error(); - td::actor::send_closure(SelfId, &RldpIn::reject_query, local_id, source, timeout, query_id, - transfer_id ^ TransferId::ones()); } }); VLOG(RLDP_DEBUG) << "delivering rldp query"; @@ -234,17 +223,6 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort } } -void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, - ton_api::rldp_queryError &message) { - auto it = queries_.find(message.query_id_); - if (it != queries_.end()) { - td::actor::send_closure(it->second, &adnl::AdnlQuery::set_error, td::Status::Error("adnl query rejected")); - queries_.erase(it); - } else { - VLOG(RLDP_INFO) << "received reject to unknown query " << message.query_id_; - } -} - void RldpIn::transfer_completed(TransferId transfer_id) { senders_.erase(transfer_id); VLOG(RLDP_DEBUG) << "rldp: completed transfer " << transfer_id << "; " << senders_.size() << " out transfer pending "; diff --git a/rldp2/rldp-in.hpp b/rldp2/rldp-in.hpp index 0092dae4fa..095c14796d 100644 --- a/rldp2/rldp-in.hpp +++ b/rldp2/rldp-in.hpp @@ -76,8 +76,6 @@ class RldpIn : public RldpImpl { td::uint64 max_answer_size) override; void answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, adnl::AdnlQueryId query_id, TransferId transfer_id, td::BufferSlice data); - void reject_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, - adnl::AdnlQueryId query_id, TransferId transfer_id); void receive_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, td::BufferSlice data); @@ -87,8 +85,6 @@ class RldpIn : public RldpImpl { ton_api::rldp_query &message); void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, ton_api::rldp_answer &message); - void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, - ton_api::rldp_queryError &message); void receive_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, td::Result data); diff --git a/rldp2/rldp.cpp b/rldp2/rldp.cpp index 8a3f77309f..95780b2375 100644 --- a/rldp2/rldp.cpp +++ b/rldp2/rldp.cpp @@ -121,13 +121,6 @@ void RldpIn::answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, send_closure(create_connection(src, dst), &RldpConnectionActor::send, transfer_id, std::move(B), timeout); } -void RldpIn::reject_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, - adnl::AdnlQueryId query_id, TransferId transfer_id) { - auto B = serialize_tl_object(create_tl_object(query_id), true); - - send_closure(create_connection(src, dst), &RldpConnectionActor::send, transfer_id, std::move(B), timeout); -} - void RldpIn::receive_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, td::BufferSlice data) { send_closure(create_connection(local_id, source), &RldpConnectionActor::receive_raw, std::move(data)); } @@ -187,16 +180,12 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort auto data = R.move_as_ok(); if (data.size() > max_answer_size) { VLOG(RLDP_NOTICE) << "rldp query failed: answer too big"; - td::actor::send_closure(SelfId, &RldpIn::reject_query, local_id, source, timeout, query_id, - transfer_id ^ TransferId::ones()); } else { td::actor::send_closure(SelfId, &RldpIn::answer_query, local_id, source, timeout, query_id, transfer_id ^ TransferId::ones(), std::move(data)); } } else { VLOG(RLDP_NOTICE) << "rldp query failed: " << R.move_as_error(); - td::actor::send_closure(SelfId, &RldpIn::reject_query, local_id, source, timeout, query_id, - transfer_id ^ TransferId::ones()); } }); VLOG(RLDP_DEBUG) << "delivering rldp query"; @@ -215,17 +204,6 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort } } -void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, - ton_api::rldp_queryError &message) { - auto it = queries_.find(transfer_id); - if (it != queries_.end()) { - it->second.set_error(td::Status::Error("rejected")); - queries_.erase(it); - } else { - VLOG(RLDP_INFO) << "received reject to unknown query " << message.query_id_; - } -} - void RldpIn::on_sent(TransferId transfer_id, td::Result state) { //TODO: completed transfer } diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index aa761ad26a..ef8679d65a 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -137,7 +137,6 @@ adnl.message.reinit date:int = adnl.Message; adnl.message.query query_id:int256 query:bytes = adnl.Message; adnl.message.answer query_id:int256 answer:bytes = adnl.Message; -adnl.message.queryError query_id:int256 = adnl.Message; adnl.message.part hash:int256 total_size:int offset:int data:bytes = adnl.Message; @@ -163,7 +162,6 @@ rldp.complete transfer_id:int256 part:int = rldp.MessagePart; rldp.message id:int256 data:bytes = rldp.Message; rldp.query query_id:int256 max_answer_size:long timeout:int data:bytes = rldp.Message; rldp.answer query_id:int256 data:bytes = rldp.Message; -rldp.queryError query_id:int256 = rldp.Message; ---functions--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 3afe3f33e8e089d6d7691e781d0441ed40885811..9680b01e9bd9d374e8925c5a05da6a11d883635f 100644 GIT binary patch delta 331 zcmdmTmG#SI)(sKNq8&c_p7SK8zvcd_L$qA7XlN-V~nL%PTJ~|M|H9iFphJvpO zgyG;~0AbYlsz`tgxOq&-2WT%tku#ePhi%(Iu%SC(${+Ybm5ca64Y2VGffz6aCb9=6 zQsEC(v%r4}Sj}dOfCG#W)qetQAVM}lJ4#tVew+Lu1mtoR2*=}u3@1nw0RVl0jE(>R delta 489 zcmexznRUxm)(sKNf@}X=ILwonl9!|Bn_66)n4UUWkxzPa3-b+G(f5Bcs(6ZWQVL*F z3=9k`n+p}!i!e^xtZ2qA#sbnjS;xl*!l?1lfiTwi6hIgXzA6xggNp%FjjxIXNY~9{ zLOwv_7>beej%nHT_96H;Hn%~jOvCw z6>+d5oiMu zx)Hdelm!$VlNBavOy9u6C^NahmqQaI0m!Va+4VD=!q1`^pUbMnRl01cVa A*#H0l From 3190c67f1817aa988675cd312db1116021aae413 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 26 Mar 2024 11:42:30 +0300 Subject: [PATCH 086/524] Simplify getCapabilities in fullnode --- tl/generate/scheme/ton_api.tl | 6 +-- tl/generate/scheme/ton_api.tlo | Bin 92660 -> 92500 bytes validator/full-node-shard.cpp | 66 +++++++++++++-------------------- validator/full-node-shard.hpp | 29 ++++++++------- 4 files changed, 42 insertions(+), 59 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index ef8679d65a..81f7192ed2 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -408,8 +408,7 @@ tonNode.dataList data:(vector bytes) = tonNode.DataList; tonNode.dataFull id:tonNode.blockIdExt proof:bytes block:bytes is_link:Bool = tonNode.DataFull; tonNode.dataFullEmpty = tonNode.DataFull; -tonNode.capabilities version:int capabilities:long = tonNode.Capabilities; -tonNode.capabilitiesV2 version:int capabilities:long have_state:Bool = tonNode.Capabilities; +tonNode.capabilities#f5bf60c0 version_major:int version_minor:int flags:# = tonNode.Capabilities; tonNode.success = tonNode.Success; @@ -457,7 +456,6 @@ tonNode.getOutMsgQueueProof dst_shard:tonNode.shardId blocks:(vector tonNode.blo limits:tonNode.importedMsgQueueLimits = tonNode.OutMsgQueueProof; tonNode.getCapabilities = tonNode.Capabilities; -tonNode.getCapabilitiesV2 = tonNode.Capabilities; tonNode.slave.sendExtMessage message:tonNode.externalMessage = tonNode.Success; @@ -660,7 +658,7 @@ engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string bdcst_errors:int engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) extra:string = engine.validator.OverlayStats; engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; -engine.validator.shardOverlayStats.neighbour id:string proto_verison:int capabilities:long +engine.validator.shardOverlayStats.neighbour id:string verison_major:int version_minor:int flags:# roundtrip:double unreliability:double has_state:string = engine.validator.shardOverlayStats.Neighbour; engine.validator.shardOverlayStats shard:string mode:string neighbours:(vector engine.validator.shardOverlayStats.neighbour) = engine.validator.ShardOverlayStats; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 9680b01e9bd9d374e8925c5a05da6a11d883635f..c44ee331648d597810cdcbf77ecae5532dc399d3 100644 GIT binary patch delta 269 zcmexznf1yg)(uCbSwxObc|Tcat;*&%(no|O4fY&0|77?2{*k2v4q9E67`xT9jFw zpBJB-n3Z2N+0I&u4GD|X3 ziznJjZ+@a@!_GKmv!dDkVz9Q&EUQ$QVFnm*f=s%3Ovs0A^289~$rWn_c?yd1OY-B( zQj0Q+^QTW_Wt5&S;l!vic|$47-Efj;fEE&mVVMYuv?aVOkDw6{qv2tZ3mZioQmn4>?PJaJVmIb70bHc9{b;kDX xPZb##+iAk|fvhQVX4B!YZ99l=Z=wg=W69}-35?=0K%3xJp=;T`Ac1iU2LQPvem(#I diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index d64dff2523..d70cea8fe1 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -51,21 +51,10 @@ namespace fullnode { Neighbour Neighbour::zero = Neighbour{adnl::AdnlNodeIdShort::zero()}; -void Neighbour::update_proto_version(ton_api::tonNode_Capabilities &q) { - ton_api::downcast_call(q, td::overloaded( - [&](ton_api::tonNode_capabilities &x) { - proto_version = x.version_; - capabilities = x.capabilities_; - if (!supports_v2()) { - has_state_known = has_state = true; - } - }, - [&](ton_api::tonNode_capabilitiesV2 &x) { - proto_version = x.version_; - capabilities = x.capabilities_; - has_state_known = true; - has_state = x.have_state_; - })); +void Neighbour::update_proto_version(ton_api::tonNode_capabilities &q) { + version_major = q.version_major_; + version_minor = q.version_minor_; + flags = q.flags_; } void Neighbour::query_success(double t) { @@ -188,7 +177,7 @@ void FullNodeShardImpl::try_get_next_block(td::Timestamp timeout, td::Promise= 1) { + if (!b.adnl_id.is_zero() && b.version_major >= 1) { VLOG(FULL_NODE_DEBUG) << "using new download method with adnlid=" << b.adnl_id; td::actor::create_actor("downloadnext", adnl_id_, overlay_id_, handle_->id(), b.adnl_id, download_next_priority(), timeout, validator_manager_, rldp_, overlays_, @@ -631,14 +620,12 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise) { VLOG(FULL_NODE_DEBUG) << "Got query getCapabilities from " << src; - promise.set_value(create_serialize_tl_object(proto_version(), proto_capabilities())); -} - -void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilitiesV2 &query, - td::Promise promise) { - VLOG(FULL_NODE_DEBUG) << "Got query getCapabilitiesV2 from " << src; - promise.set_value(create_serialize_tl_object(proto_version(), proto_capabilities(), - mode_ == FullNodeShardMode::active)); + td::uint32 flags = 0; + if (mode_ != FullNodeShardMode::active) { + flags |= Neighbour::FLAG_NO_STATE; + } + promise.set_value( + create_serialize_tl_object(proto_version_major(), proto_version_minor(), flags)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, @@ -878,7 +865,7 @@ void FullNodeShardImpl::send_broadcast(BlockBroadcast broadcast) { void FullNodeShardImpl::download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { auto &b = choose_neighbour(); - if (!b.adnl_id.is_zero() && b.proto_version >= 1) { + if (!b.adnl_id.is_zero() && b.version_major >= 1) { VLOG(FULL_NODE_DEBUG) << "new block download"; td::actor::create_actor("downloadreq", id, adnl_id_, overlay_id_, b.adnl_id, priority, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, @@ -1173,7 +1160,7 @@ void FullNodeShardImpl::got_neighbours(std::vector vec) { if (neighbours_.size() == max_neighbours()) { td::uint32 neighbours_with_state = 0; for (const auto &n : neighbours_) { - if (n.second.has_state) { + if (n.second.has_state_known() && n.second.has_state()) { ++neighbours_with_state; } } @@ -1183,7 +1170,8 @@ void FullNodeShardImpl::got_neighbours(std::vector vec) { td::uint32 cnt = 0; double u = 0; for (auto &n : neighbours_) { - if (neighbours_with_state <= min_neighbours_with_state() && n.second.has_state) { + if (neighbours_with_state <= min_neighbours_with_state() && n.second.has_state_known() && + n.second.has_state()) { continue; } if (n.second.unreliability > u) { @@ -1224,18 +1212,18 @@ const Neighbour &FullNodeShardImpl::choose_neighbour(bool require_state) const { for (auto &x : neighbours_) { if (require_state) { - if (attempt == 0 && !x.second.has_state) { + if (attempt == 0 && !(x.second.has_state_known() && x.second.has_state())) { continue; } - if (attempt == 1 && x.second.has_state_known) { + if (attempt == 1 && x.second.has_state_known()) { continue; } } auto unr = static_cast(x.second.unreliability - min_unreliability); - if (x.second.proto_version < proto_version()) { + if (x.second.version_major < proto_version_major()) { unr += 4; - } else if (x.second.proto_version == proto_version() && x.second.capabilities < proto_capabilities()) { + } else if (x.second.version_major == proto_version_major() && x.second.version_minor < proto_version_minor()) { unr += 2; } @@ -1272,7 +1260,7 @@ void FullNodeShardImpl::got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id if (it == neighbours_.end()) { return; } - auto F = fetch_tl_object(std::move(data), true); + auto F = fetch_tl_object(std::move(data), true); if (F.is_error()) { it->second.query_failed(); } else { @@ -1305,12 +1293,7 @@ void FullNodeShardImpl::ping_neighbours() { td::Time::now() - start_time, R.move_as_ok()); } }); - td::BufferSlice q; - if (it->second.supports_v2()) { - q = create_serialize_tl_object(); - } else { - q = create_serialize_tl_object(); - } + td::BufferSlice q = create_serialize_tl_object(); td::actor::send_closure(overlays_, &overlay::Overlays::send_query, it->first, adnl_id_, overlay_id_, "get_prepare_block", std::move(P), td::Timestamp::in(1.0), std::move(q)); @@ -1338,11 +1321,12 @@ void FullNodeShardImpl::get_stats_extra(td::Promise promise) { const auto &n = p.second; auto f = create_tl_object(); f->id_ = n.adnl_id.bits256_value().to_hex(); - f->proto_verison_ = n.proto_version; - f->capabilities_ = n.capabilities; + f->verison_major_ = n.version_major; + f->version_minor_ = n.version_minor; + f->flags_ = n.flags; f->roundtrip_ = n.roundtrip; f->unreliability_ = n.unreliability; - f->has_state_ = (n.has_state_known ? (n.has_state ? "true" : "false") : "undefined"); + f->has_state_ = (n.has_state_known() ? (n.has_state() ? "true" : "false") : "undefined"); res->neighbours_.push_back(std::move(f)); } promise.set_result(td::json_encode(td::ToJson(*res), true)); diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index b1417e223a..834fee8577 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -31,30 +31,33 @@ namespace fullnode { struct Neighbour { adnl::AdnlNodeIdShort adnl_id; - td::uint32 proto_version = 0; - td::uint64 capabilities = 0; + td::uint32 version_major = 0; + td::uint32 version_minor = 0; + td::uint32 flags = 0; double roundtrip = 0; double roundtrip_relax_at = 0; double roundtrip_weight = 0; double unreliability = 0; - bool has_state_known = false; - bool has_state = false; Neighbour(adnl::AdnlNodeIdShort adnl_id) : adnl_id(std::move(adnl_id)) { } - void update_proto_version(ton_api::tonNode_Capabilities &q); + void update_proto_version(ton_api::tonNode_capabilities &q); void query_success(double t); void query_failed(); void update_roundtrip(double t); bool use_rldp2() const { - return std::make_pair(proto_version, capabilities) >= std::make_pair(2, 2); + return std::make_pair(version_major, version_minor) >= std::make_pair(2, 2); } - bool supports_v2() const { - return proto_version >= 3; + bool has_state() const { + return !(flags & FLAG_NO_STATE); + } + bool has_state_known() const { + return version_major != 0; } static Neighbour zero; + static constexpr td::uint32 FLAG_NO_STATE = 1; }; class FullNodeShardImpl : public FullNodeShard { @@ -72,11 +75,11 @@ class FullNodeShardImpl : public FullNodeShard { static constexpr td::uint32 download_next_priority() { return 1; } - static constexpr td::uint32 proto_version() { + static constexpr td::uint32 proto_version_major() { return 3; } - static constexpr td::uint64 proto_capabilities() { - return 2; + static constexpr td::uint32 proto_version_minor() { + return 0; } static constexpr td::uint32 max_neighbours() { return 16; @@ -146,8 +149,6 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise); - void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilitiesV2 &query, - td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, @@ -216,7 +217,7 @@ class FullNodeShardImpl : public FullNodeShard { template td::Promise create_neighbour_promise(const Neighbour &x, td::Promise p, bool require_state = false) { return td::PromiseCreator::lambda([id = x.adnl_id, SelfId = actor_id(this), p = std::move(p), ts = td::Time::now(), - ignore_error = require_state && !x.has_state_known](td::Result R) mutable { + ignore_error = require_state && !x.has_state_known()](td::Result R) mutable { if (R.is_error() && R.error().code() != ErrorCode::notready && R.error().code() != ErrorCode::cancelled) { if (!ignore_error) { td::actor::send_closure(SelfId, &FullNodeShardImpl::update_neighbour_stats, id, td::Time::now() - ts, false); From 103eb9aad5d2ea72c3b2066e50b4f38968a7dd0d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 26 Mar 2024 12:50:01 +0300 Subject: [PATCH 087/524] Separate limits for collated data --- .github/workflows/macos-11.7-compile.yml | 0 crypto/block/block.cpp | 25 ++++++++++++++++-------- crypto/block/block.h | 2 +- crypto/block/block.tlb | 2 +- crypto/vm/boc.h | 2 +- 5 files changed, 20 insertions(+), 11 deletions(-) delete mode 100644 .github/workflows/macos-11.7-compile.yml diff --git a/.github/workflows/macos-11.7-compile.yml b/.github/workflows/macos-11.7-compile.yml deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index cb93441734..d17f25976e 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -678,10 +678,19 @@ bool BlockLimits::deserialize(vm::CellSlice& cs) { } // block_limits#5d // block_limits_v2#5e - return bytes.deserialize(cs) // bytes:ParamLimits - && gas.deserialize(cs) // gas:ParamLimits - && lt_delta.deserialize(cs) // lt_delta:ParamLimits - && (tag == 0x5d || imported_msg_queue.deserialize(cs)); // imported_msg_queue:ImportedMsgQueueLimits + bool ok = bytes.deserialize(cs) // bytes:ParamLimits + && gas.deserialize(cs) // gas:ParamLimits + && lt_delta.deserialize(cs); // lt_delta:ParamLimits + if (!ok) { + return false; + } + if (tag == 0x5d) { + return collated_data.deserialize(cs) && // collated_data:ParamLimits + imported_msg_queue.deserialize(cs); // imported_msg_queue:ImportedMsgQueueLimits + } else { + collated_data = bytes; + return true; + } } int ParamLimits::classify(td::uint64 value) const { @@ -714,7 +723,7 @@ int BlockLimits::classify_lt(ton::LogicalTime lt) const { } int BlockLimits::classify_collated_data_size(td::uint64 size) const { - return bytes.classify(size); // TODO: Maybe separate limits in config + return collated_data.classify(size); } int BlockLimits::classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt, td::uint64 collated_size) const { @@ -725,7 +734,7 @@ int BlockLimits::classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt, bool BlockLimits::fits(unsigned cls, td::uint64 size, td::uint64 gas_value, ton::LogicalTime lt, td::uint64 collated_size) const { return bytes.fits(cls, size) && gas.fits(cls, gas_value) && lt_delta.fits(cls, lt - start_lt) && - bytes.fits(cls, collated_size); + collated_data.fits(cls, collated_size); } td::uint64 BlockLimitStatus::estimate_block_size(const vm::NewCellStorageStat::Stat* extra) const { @@ -745,7 +754,7 @@ bool BlockLimitStatus::fits(unsigned cls) const { return cls >= ParamLimits::limits_cnt || (limits.gas.fits(cls, gas_used) && limits.lt_delta.fits(cls, cur_lt - limits.start_lt) && limits.bytes.fits(cls, estimate_block_size()) && - limits.bytes.fits(cls, collated_data_stat.estimate_proof_size())); + limits.collated_data.fits(cls, collated_data_stat.estimate_proof_size())); } bool BlockLimitStatus::would_fit(unsigned cls, ton::LogicalTime end_lt, td::uint64 more_gas, @@ -753,7 +762,7 @@ bool BlockLimitStatus::would_fit(unsigned cls, ton::LogicalTime end_lt, td::uint return cls >= ParamLimits::limits_cnt || (limits.gas.fits(cls, gas_used + more_gas) && limits.lt_delta.fits(cls, std::max(cur_lt, end_lt) - limits.start_lt) && limits.bytes.fits(cls, estimate_block_size(extra)) && - limits.bytes.fits(cls, collated_data_stat.estimate_proof_size())); + limits.collated_data.fits(cls, collated_data_stat.estimate_proof_size())); } // SETS: account_dict, shard_libraries_, mc_state_extra diff --git a/crypto/block/block.h b/crypto/block/block.h index 61b057c44e..6032654365 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -255,7 +255,7 @@ struct ParamLimits { }; struct BlockLimits { - ParamLimits bytes, gas, lt_delta; + ParamLimits bytes, gas, lt_delta, collated_data; ton::LogicalTime start_lt{0}; ImportedMsgQueueLimits imported_msg_queue; const vm::CellUsageTree* usage_tree{nullptr}; diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 6aababa1a5..e05b501801 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -708,7 +708,7 @@ imported_msg_queue_limits#d3 max_bytes:# max_msgs:# = ImportedMsgQueueLimits; block_limits#5d bytes:ParamLimits gas:ParamLimits lt_delta:ParamLimits = BlockLimits; block_limits_v2#5e bytes:ParamLimits gas:ParamLimits lt_delta:ParamLimits - imported_msg_queue:ImportedMsgQueueLimits + collated_data:ParamLimits imported_msg_queue:ImportedMsgQueueLimits = BlockLimits; config_mc_block_limits#_ BlockLimits = ConfigParam 22; diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index e9feb71e99..f81928cd14 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -41,7 +41,6 @@ class NewCellStorageStat { Stat(td::uint64 cells_, td::uint64 bits_, td::uint64 internal_refs_ = 0, td::uint64 external_refs_ = 0) : cells(cells_), bits(bits_), internal_refs(internal_refs_), external_refs(external_refs_) { } - Stat(const Stat&) = default; td::uint64 cells{0}; td::uint64 bits{0}; td::uint64 internal_refs{0}; @@ -53,6 +52,7 @@ class NewCellStorageStat { bool operator==(const Stat& other) const { return key() == other.key(); } + Stat(const Stat& other) = default; Stat& operator=(const Stat& other) = default; Stat& operator+=(const Stat& other) { cells += other.cells; From 036ce697869459e65ad4cfe19058048cb8f4853a Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 26 Mar 2024 16:22:52 +0300 Subject: [PATCH 088/524] Use compression in new private overlays --- validator/full-node-private-overlay-v2.cpp | 41 +++++++++++----------- validator/full-node-private-overlay-v2.hpp | 3 ++ 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/validator/full-node-private-overlay-v2.cpp b/validator/full-node-private-overlay-v2.cpp index fedb8a788b..a356825e24 100644 --- a/validator/full-node-private-overlay-v2.cpp +++ b/validator/full-node-private-overlay-v2.cpp @@ -22,23 +22,24 @@ #include "td/utils/JsonBuilder.h" #include "tl/tl_json.h" #include "auto/tl/ton_api_json.h" +#include "full-node-serializer.hpp" namespace ton::validator::fullnode { -void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash, ton_api::tonNode_blockBroadcast &query) { - std::vector signatures; - for (auto &sig : query.signatures_) { - signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); - } +void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { + process_block_broadcast(src, query); +} - BlockIdExt block_id = create_block_id(query.id_); - BlockBroadcast B{block_id, - std::move(signatures), - static_cast(query.catchain_seqno_), - static_cast(query.validator_set_hash_), - std::move(query.data_), - std::move(query.proof_)}; +void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) { + process_block_broadcast(src, query); +} +void FullNodePrivateOverlayV2::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); + if (B.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); + return; + } auto P = td::PromiseCreator::lambda([](td::Result R) { if (R.is_error()) { if (R.error().code() == ErrorCode::notready) { @@ -48,8 +49,8 @@ void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash, ton_api::tonNode } } }); - LOG(FULL_NODE_DEBUG) << "Got block broadcast in private overlay: " << B.block_id.to_str(); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(B), + LOG(FULL_NODE_DEBUG) << "Got block broadcast in private overlay: " << B.ok().block_id.to_str(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, B.move_as_ok(), std::move(P)); } @@ -89,15 +90,13 @@ void FullNodePrivateOverlayV2::send_broadcast(BlockBroadcast broadcast) { if (!inited_) { return; } - std::vector> sigs; - for (auto &sig : broadcast.signatures) { - sigs.emplace_back(create_tl_object(sig.node, sig.signature.clone())); + auto B = serialize_block_broadcast(broadcast, false); // compression_enabled = false + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); + return; } - auto B = create_serialize_tl_object( - create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs), - broadcast.proof.clone(), broadcast.data.clone()); td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); } void FullNodePrivateOverlayV2::start_up() { diff --git a/validator/full-node-private-overlay-v2.hpp b/validator/full-node-private-overlay-v2.hpp index 6fbcbbf717..a48dc4ccf4 100644 --- a/validator/full-node-private-overlay-v2.hpp +++ b/validator/full-node-private-overlay-v2.hpp @@ -23,6 +23,9 @@ namespace ton::validator::fullnode { class FullNodePrivateOverlayV2 : public td::actor::Actor { public: void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); template void process_broadcast(PublicKeyHash, T &) { From f5cedc3b6edadd00795cc9d198e85b72b130784e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 29 Mar 2024 15:04:07 +0300 Subject: [PATCH 089/524] Enable compression in private overlays v2 and in collator node --- tl/generate/scheme/ton_api.tl | 4 +- tl/generate/scheme/ton_api.tlo | Bin 92716 -> 93212 bytes validator-session/candidate-serializer.cpp | 57 +++++++++------ validator-session/candidate-serializer.h | 7 +- validator/collator-node.cpp | 78 ++++++++++++++++----- validator/collator-node.hpp | 13 ++-- validator/full-node-private-overlay-v2.cpp | 2 +- validator/validator-group.cpp | 32 ++++----- 8 files changed, 128 insertions(+), 65 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 4b038f6fbf..cafc09260a 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -789,7 +789,9 @@ validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:long self:in signatures:int signatures_weight:long approve_signatures:int approve_signatures_weight:long first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats; -collatorNode.generateBlockSuccess candidate:db.Candidate = collatorNode.GenerateBlockResult; +collatorNode.candidate source:PublicKey id:tonNode.blockIdExt data:bytes collated_data:bytes = collatorNode.Candidate; +collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt decompressed_size:int data:bytes = collatorNode.Candidate; +collatorNode.generateBlockSuccess candidate:collatorNode.Candidate = collatorNode.GenerateBlockResult; collatorNode.generateBlockError code:int message:string = collatorNode.GenerateBlockResult; ---functions--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index de65fb4086c32897f552c854bb1b9e970a2e498d..b3c6bf820a2fc331f453baff383bdcdd29dece7b 100644 GIT binary patch delta 270 zcmZ2;g>}viR^CUm^{p77KxiW`H;14P_olNTB*v;6CfGu|x2 zA)&`OW3#E5vz`L`4_gHkWqQepc`2DGi6yBFAR{JaOi={UFvG9Pam9Fl{=QQI+34vH z`547HLCS6(6Y^o5EEuRgc|oqt^d5f301nOs+{$;RLw_q@Hp5LPbVt31oFe t&TKjywrvMNN+#b)F#x-JvOsD8!~-d*EC<1?={ahQ7r?CT4(g0=oB;Q;TeAQF delta 133 zcmbPpgLTanR^CUm^{p77KyV{3HwViRgS_jT12`n~7^iK{G;`KtiMx7MXL3%O3MWYB z<}o23#_1E48Kos|W!+4LX)SVQ)8VjfI|x!T`AnJt*px}>0T9zd(pe6IS<_=Q7%zZX L+YK}s-#7sPVva8I diff --git a/validator-session/candidate-serializer.cpp b/validator-session/candidate-serializer.cpp index c220b4a95d..c2c0a81afe 100644 --- a/validator-session/candidate-serializer.cpp +++ b/validator-session/candidate-serializer.cpp @@ -23,27 +23,15 @@ namespace ton::validatorsession { -td::Result serialize_candidate(const tl_object_ptr &block, +td::Result serialize_candidate(const tl_object_ptr& block, bool compression_enabled) { if (!compression_enabled) { return serialize_tl_object(block, true); } - vm::BagOfCells boc1, boc2; - TRY_STATUS(boc1.deserialize(block->data_)); - if (boc1.get_root_count() != 1) { - return td::Status::Error("block candidate should have exactly one root"); - } - std::vector> roots = {boc1.get_root_cell()}; - TRY_STATUS(boc2.deserialize(block->collated_data_)); - for (int i = 0; i < boc2.get_root_count(); ++i) { - roots.push_back(boc2.get_root_cell(i)); - } - TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(roots), 2)); - td::BufferSlice compressed = td::lz4_compress(data); - LOG(VALIDATOR_SESSION_DEBUG) << "Compressing block candidate: " << block->data_.size() + block->collated_data_.size() - << " -> " << compressed.size(); + size_t decompressed_size; + TRY_RESULT(compressed, compress_candidate_data(block->data_, block->collated_data_, decompressed_size)) return create_serialize_tl_object( - 0, block->src_, block->round_, block->root_hash_, (int)data.size(), std::move(compressed)); + 0, block->src_, block->round_, block->root_hash_, (int)decompressed_size, std::move(compressed)); } td::Result> deserialize_candidate(td::Slice data, @@ -56,8 +44,34 @@ td::Result> deserialize_candi if (f->decompressed_size_ > max_decompressed_data_size) { return td::Status::Error("decompressed size is too big"); } - TRY_RESULT(decompressed, td::lz4_decompress(f->data_, f->decompressed_size_)); - if (decompressed.size() != (size_t)f->decompressed_size_) { + TRY_RESULT(p, decompress_candidate_data(f->data_, f->decompressed_size_)); + return create_tl_object(f->src_, f->round_, f->root_hash_, std::move(p.first), + std::move(p.second)); +} + +td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, + size_t& decompressed_size) { + vm::BagOfCells boc1, boc2; + TRY_STATUS(boc1.deserialize(block)); + if (boc1.get_root_count() != 1) { + return td::Status::Error("block candidate should have exactly one root"); + } + std::vector> roots = {boc1.get_root_cell()}; + TRY_STATUS(boc2.deserialize(collated_data)); + for (int i = 0; i < boc2.get_root_count(); ++i) { + roots.push_back(boc2.get_root_cell(i)); + } + TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(roots), 2)); + decompressed_size = data.size(); + td::BufferSlice compressed = td::lz4_compress(data); + LOG(DEBUG) << "Compressing block candidate: " << block.size() + collated_data.size() << " -> " << compressed.size(); + return compressed; +} + +td::Result> decompress_candidate_data(td::Slice compressed, + int decompressed_size) { + TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size)); + if (decompressed.size() != (size_t)decompressed_size) { return td::Status::Error("decompressed size mismatch"); } TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed)); @@ -67,10 +81,9 @@ td::Result> deserialize_candi TRY_RESULT(block_data, vm::std_boc_serialize(roots[0], 31)); roots.erase(roots.begin()); TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), 31)); - LOG(VALIDATOR_SESSION_DEBUG) << "Decompressing block candidate: " << f->data_.size() << " -> " - << block_data.size() + collated_data.size(); - return create_tl_object(f->src_, f->round_, f->root_hash_, std::move(block_data), - std::move(collated_data)); + LOG(DEBUG) << "Decompressing block candidate: " << compressed.size() << " -> " + << block_data.size() + collated_data.size(); + return std::make_pair(std::move(block_data), std::move(collated_data)); } } // namespace ton::validatorsession diff --git a/validator-session/candidate-serializer.h b/validator-session/candidate-serializer.h index 030a412c0d..e88376cd77 100644 --- a/validator-session/candidate-serializer.h +++ b/validator-session/candidate-serializer.h @@ -20,10 +20,15 @@ namespace ton::validatorsession { -td::Result serialize_candidate(const tl_object_ptr &block, +td::Result serialize_candidate(const tl_object_ptr& block, bool compression_enabled); td::Result> deserialize_candidate(td::Slice data, bool compression_enabled, int max_decompressed_data_size); +td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, + size_t& decompressed_size); +td::Result> decompress_candidate_data(td::Slice compressed, + int decompressed_size); + } // namespace ton::validatorsession diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 9bfd857add..ce3c4be4b2 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -19,10 +19,11 @@ #include "fabric.h" #include "block-auto.h" #include "block-db.h" +#include "td/utils/lz4.h" +#include "checksum.h" +#include "validator-session/candidate-serializer.h" -namespace ton { - -namespace validator { +namespace ton::validator { CollatorNode::CollatorNode(adnl::AdnlNodeIdShort local_id, td::actor::ActorId manager, td::actor::ActorId adnl, td::actor::ActorId rldp) @@ -110,12 +111,6 @@ static td::BufferSlice serialize_error(td::Status error) { return create_serialize_tl_object(error.code(), error.message().c_str()); } -static td::BufferSlice serialize_response(BlockCandidate block) { - return create_serialize_tl_object(create_tl_object( - PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), std::move(block.data), - std::move(block.collated_data))); -} - static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey creator) { CHECK(!block.id.is_masterchain()); if (block.pubkey == creator) { @@ -145,7 +140,8 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data promise.set_result(serialize_error(R.move_as_error())); } else { LOG(INFO) << "Query from " << src << ", success"; - promise.set_result(serialize_response(R.move_as_ok())); + promise.set_result(create_serialize_tl_object( + serialize_candidate(R.move_as_ok(), true))); } }; if (!last_masterchain_block_.is_valid()) { @@ -263,14 +259,62 @@ void CollatorNode::process_result(std::shared_ptr cache_entry, td::R } bool CollatorNode::can_collate_shard(ShardIdFull shard) const { - for (ShardIdFull our_shard : shards_) { - if (shard_intersects(shard, our_shard)) { - return true; - } + return std::any_of(shards_.begin(), shards_.end(), + [&](const ShardIdFull& our_shard) { return shard_intersects(shard, our_shard); }); +} + +tl_object_ptr CollatorNode::serialize_candidate(const BlockCandidate& block, + bool compress) { + if (!compress) { + return create_tl_object( + PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), block.data.clone(), + block.collated_data.clone()); } - return false; + size_t decompressed_size; + td::BufferSlice compressed = + validatorsession::compress_candidate_data(block.data, block.collated_data, decompressed_size).move_as_ok(); + return create_tl_object( + 0, PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), + (int)decompressed_size, std::move(compressed)); } -} // namespace validator +td::Result CollatorNode::deserialize_candidate(tl_object_ptr f, + int max_decompressed_data_size) { + td::Result res; + ton_api::downcast_call(*f, td::overloaded( + [&](ton_api::collatorNode_candidate& c) { + res = [&]() -> td::Result { + auto hash = td::sha256_bits256(c.collated_data_); + auto key = ton::PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), hash, std::move(c.data_), + std::move(c.collated_data_)}; + }(); + }, + [&](ton_api::collatorNode_compressedCandidate& c) { + res = [&]() -> td::Result { + if (c.decompressed_size_ <= 0) { + return td::Status::Error("invalid decompressed size"); + } + if (c.decompressed_size_ > max_decompressed_data_size) { + return td::Status::Error("decompressed size is too big"); + } + TRY_RESULT( + p, validatorsession::decompress_candidate_data(c.data_, c.decompressed_size_)); + auto collated_data_hash = td::sha256_bits256(p.second); + auto key = ton::PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, + std::move(p.first), std::move(p.second)}; + }(); + })); + return res; +} -} // namespace ton +} // namespace ton::validator diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index d640244a7d..ca2fd73d41 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -20,9 +20,7 @@ #include "rldp/rldp.h" #include -namespace ton { - -namespace validator { +namespace ton::validator { class ValidatorManager; @@ -77,8 +75,11 @@ class CollatorNode : public td::actor::Actor { } void process_result(std::shared_ptr cache_entry, td::Result R); -}; -} // namespace validator + public: + static tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress); + static td::Result deserialize_candidate(tl_object_ptr f, + int max_decompressed_data_size); +}; -} // namespace ton +} // namespace ton::validator diff --git a/validator/full-node-private-overlay-v2.cpp b/validator/full-node-private-overlay-v2.cpp index a356825e24..402c8d0930 100644 --- a/validator/full-node-private-overlay-v2.cpp +++ b/validator/full-node-private-overlay-v2.cpp @@ -90,7 +90,7 @@ void FullNodePrivateOverlayV2::send_broadcast(BlockBroadcast broadcast) { if (!inited_) { return; } - auto B = serialize_block_broadcast(broadcast, false); // compression_enabled = false + auto B = serialize_block_broadcast(broadcast, true); // compression_enabled = true if (B.is_error()) { VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); return; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 2ceffae555..68ac6d570c 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -24,6 +24,7 @@ #include "ton/lite-tl.hpp" #include "ton/ton-tl.hpp" #include "td/utils/Random.h" +#include "collator-node.hpp" namespace ton { @@ -506,7 +507,7 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo std::move(promise)); }); LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to " << collator; - size_t max_answer_size = config_.max_block_size + config_.max_collated_data_size + 256; + size_t max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; td::Timestamp query_timeout = td::Timestamp::in(10.0); query_timeout.relax(timeout); td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_adnl_id_, collator, "collatequery", std::move(P), @@ -520,29 +521,26 @@ void ValidatorGroup::receive_collate_query_response(td::uint32 round_id, td::Buf return; } TRY_RESULT_PROMISE(promise, f, fetch_tl_object(data, true)); - tl_object_ptr b; + td::Result res; ton_api::downcast_call(*f, td::overloaded( - [&](ton_api::collatorNode_generateBlockError &r) { - td::Status error = td::Status::Error(r.code_, r.message_); - promise.set_error(error.move_as_error_prefix("collate query: ")); - }, - [&](ton_api::collatorNode_generateBlockSuccess &r) { b = std::move(r.candidate_); })); - if (!b) { - return; - } - auto key = PublicKey{b->source_}; - if (key != local_id_full_) { + [&](ton_api::collatorNode_generateBlockError &r) { + td::Status error = td::Status::Error(r.code_, r.message_); + res = error.move_as_error_prefix("collate query: "); + }, + [&](ton_api::collatorNode_generateBlockSuccess &r) { + res = CollatorNode::deserialize_candidate( + std::move(r.candidate_), + config_.max_block_size + config_.max_collated_data_size + 1024); + })); + TRY_RESULT_PROMISE(promise, candidate, std::move(res)); + if (candidate.pubkey.as_bits256() != local_id_full_.ed25519_value().raw()) { promise.set_error(td::Status::Error("collate query: block candidate source mismatch")); return; } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - auto block_id = ton::create_block_id(b->id_); - if (block_id.shard_full() != shard_) { + if (candidate.id.shard_full() != shard_) { promise.set_error(td::Status::Error("collate query: shard mismatch")); return; } - auto collated_data_hash = td::sha256_bits256(b->collated_data_); - BlockCandidate candidate(e_key, block_id, collated_data_hash, std::move(b->data_), std::move(b->collated_data_)); auto P = td::PromiseCreator::lambda( [candidate = candidate.clone(), promise = std::move(promise)](td::Result R) mutable { From 86d3ed97c1dd06db97d3cbd8b4fd801fb1794b16 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 30 May 2024 18:19:38 +0300 Subject: [PATCH 090/524] Fix parsing block limits config --- crypto/block/block.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index d17f25976e..fdbe8e20eb 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -684,7 +684,7 @@ bool BlockLimits::deserialize(vm::CellSlice& cs) { if (!ok) { return false; } - if (tag == 0x5d) { + if (tag == 0x5e) { return collated_data.deserialize(cs) && // collated_data:ParamLimits imported_msg_queue.deserialize(cs); // imported_msg_queue:ImportedMsgQueueLimits } else { From c129a784c6b3930f40bbaa213fb57c35fbb95a70 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 30 May 2024 18:57:50 +0300 Subject: [PATCH 091/524] Prepare transition between old and new private overlays --- validator/full-node-private-overlay-v2.cpp | 26 +++++++----- validator/full-node-private-overlay-v2.hpp | 3 +- validator/full-node.cpp | 48 +++++++++++++--------- validator/full-node.h | 5 ++- validator/full-node.hpp | 17 ++++---- validator/manager.cpp | 4 ++ 6 files changed, 61 insertions(+), 42 deletions(-) diff --git a/validator/full-node-private-overlay-v2.cpp b/validator/full-node-private-overlay-v2.cpp index 8dd7f6dd74..20e9cc55fe 100644 --- a/validator/full-node-private-overlay-v2.cpp +++ b/validator/full-node-private-overlay-v2.cpp @@ -239,7 +239,7 @@ void FullNodePrivateOverlayV2::get_stats_extra(td::Promise promise) promise.set_result(td::json_encode(td::ToJson(*res), true)); } -td::actor::ActorId FullNodePrivateBlockOverlays::choose_overlay(ShardIdFull shard) { +td::actor::ActorId FullNodePrivateBlockOverlaysV2::choose_overlay(ShardIdFull shard) { for (auto &p : id_to_overlays_) { auto &overlays = p.second.overlays_; ShardIdFull cur_shard = shard; @@ -257,7 +257,7 @@ td::actor::ActorId FullNodePrivateBlockOverlays::choos return {}; } -void FullNodePrivateBlockOverlays::update_overlays( +void FullNodePrivateBlockOverlaysV2::update_overlays( td::Ref state, std::set my_adnl_ids, const FileHash &zero_state_file_hash, const td::actor::ActorId &keyring, const td::actor::ActorId &adnl, const td::actor::ActorId &rldp, const td::actor::ActorId &rldp2, @@ -274,10 +274,10 @@ void FullNodePrivateBlockOverlays::update_overlays( struct OverlayInfo { std::vector nodes, senders; }; - std::map pverlay_infos; + std::map overlay_infos; // Masterchain overlay: all validators + collators - OverlayInfo &mc_overlay = pverlay_infos[ShardIdFull(masterchainId)]; + OverlayInfo &mc_overlay = overlay_infos[ShardIdFull(masterchainId)]; for (const auto &x : all_validators->export_vector()) { td::Bits256 addr = x.addr.is_zero() ? ValidatorFullId(x.key).compute_short_id().bits256_value() : x.addr; mc_overlay.nodes.emplace_back(addr); @@ -311,14 +311,14 @@ void FullNodePrivateBlockOverlays::update_overlays( auto val_set = state->get_validator_set(shard); td::uint32 min_split = state->monitor_min_split_depth(shard.workchain); OverlayInfo &overlay = - pverlay_infos[shard_prefix_length(shard) <= min_split ? shard : shard_prefix(shard, min_split)]; + overlay_infos[shard_prefix_length(shard) <= min_split ? shard : shard_prefix(shard, min_split)]; for (const auto &x : val_set->export_vector()) { td::Bits256 addr = x.addr.is_zero() ? ValidatorFullId(x.key).compute_short_id().bits256_value() : x.addr; overlay.nodes.emplace_back(addr); overlay.senders.emplace_back(addr); } } - for (auto &p : pverlay_infos) { + for (auto &p : overlay_infos) { ShardIdFull shard = p.first; OverlayInfo &overlay = p.second; if (!shard.is_masterchain()) { @@ -338,7 +338,7 @@ void FullNodePrivateBlockOverlays::update_overlays( std::map old_private_block_overlays = std::move(id_to_overlays_); id_to_overlays_.clear(); - for (const auto &p : pverlay_infos) { + for (const auto &p : overlay_infos) { ShardIdFull shard = p.first; const OverlayInfo &new_overlay_info = p.second; for (adnl::AdnlNodeIdShort local_id : new_overlay_info.nodes) { @@ -356,8 +356,9 @@ void FullNodePrivateBlockOverlays::update_overlays( new_overlay.senders_ = new_overlay_info.senders; new_overlay.is_sender_ = std::binary_search(new_overlay.senders_.begin(), new_overlay.senders_.end(), local_id); new_overlay.overlay_ = td::actor::create_actor( - "BlocksPrivateOverlay", local_id, shard, new_overlay.nodes_, new_overlay.senders_, zero_state_file_hash, - keyring, adnl, rldp, rldp2, overlays, validator_manager, full_node); + PSTRING() << "BlocksPrivateOverlay" << shard.to_str(), local_id, shard, new_overlay.nodes_, + new_overlay.senders_, zero_state_file_hash, keyring, adnl, rldp, rldp2, overlays, validator_manager, + full_node); } } } @@ -370,9 +371,14 @@ void FullNodePrivateBlockOverlays::update_overlays( } td::actor::ActorId id = x.second.overlay_.release(); delay_action([id = std::move(id)]() { td::actor::send_closure(id, &FullNodePrivateOverlayV2::destroy); }, - td::Timestamp::in(60.0)); + td::Timestamp::in(30.0)); } } } +void FullNodePrivateBlockOverlaysV2::destroy_overlays() { + id_to_overlays_.clear(); +} + + } // namespace ton::validator::fullnode diff --git a/validator/full-node-private-overlay-v2.hpp b/validator/full-node-private-overlay-v2.hpp index 9a6ec932d5..abb88fb439 100644 --- a/validator/full-node-private-overlay-v2.hpp +++ b/validator/full-node-private-overlay-v2.hpp @@ -96,7 +96,7 @@ class FullNodePrivateOverlayV2 : public td::actor::Actor { void get_stats_extra(td::Promise promise); }; -class FullNodePrivateBlockOverlays { +class FullNodePrivateBlockOverlaysV2 { public: td::actor::ActorId choose_overlay(ShardIdFull shard); void update_overlays(td::Ref state, std::set my_adnl_ids, @@ -106,6 +106,7 @@ class FullNodePrivateBlockOverlays { const td::actor::ActorId& overlays, const td::actor::ActorId& validator_manager, const td::actor::ActorId& full_node); + void destroy_overlays(); private: struct Overlays { diff --git a/validator/full-node.cpp b/validator/full-node.cpp index e5ffa9c9da..a6eb121fb8 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -37,7 +37,7 @@ void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise pr } local_keys_.insert(key); - // create_private_block_overlay(key); + create_private_block_overlay(key); for (auto &p : custom_overlays_) { update_custom_overlay(p.second); } @@ -67,7 +67,7 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr return; } local_keys_.erase(key); - // private_block_overlays_.erase(key); + private_block_overlays_.erase(key); for (auto &p : custom_overlays_) { update_custom_overlay(p.second); } @@ -151,9 +151,9 @@ void FullNodeImpl::set_config(FullNodeConfig config) { td::actor::send_closure(s.second.actor, &FullNodeShard::set_config, config); } } - /*for (auto& overlay : private_block_overlays_) { + for (auto& overlay : private_block_overlays_) { td::actor::send_closure(overlay.second, &FullNodePrivateBlockOverlay::set_config, config); - }*/ + } for (auto& overlay : custom_overlays_) { for (auto &actor : overlay.second.actors_) { td::actor::send_closure(actor.second, &FullNodeCustomOverlay::set_config, config); @@ -286,8 +286,12 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s my_adnl_ids.insert(it->second); } } - private_block_overlays_.update_overlays(state, std::move(my_adnl_ids), zero_state_file_hash_, keyring_, adnl_, rldp_, - rldp2_, overlays_, validator_manager_, actor_id(this)); + if (use_old_private_overlays_) { + private_block_overlays_v2_.destroy_overlays(); + } else { + private_block_overlays_v2_.update_overlays(state, std::move(my_adnl_ids), zero_state_file_hash_, keyring_, adnl_, + rldp_, rldp2_, overlays_, validator_manager_, actor_id(this)); + } } void FullNodeImpl::add_shard_actor(ShardIdFull shard, FullNodeShardMode mode) { @@ -340,11 +344,11 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; return; } - /*if (!private_block_overlays_.empty()) { + if (!private_block_overlays_.empty()) { td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_shard_block_info, block_id, cc_seqno, data.clone()); - }*/ - auto private_overlay = private_block_overlays_.choose_overlay(ShardIdFull(masterchainId)); + } + auto private_overlay = private_block_overlays_v2_.choose_overlay(ShardIdFull(masterchainId)); if (!private_overlay.empty()) { td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_shard_block_info, block_id, cc_seqno, data.clone()); @@ -360,11 +364,11 @@ void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_se VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; return; } - /*if (!private_block_overlays_.empty()) { + if (!private_block_overlays_.empty()) { td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_block_candidate, block_id, cc_seqno, validator_set_hash, data.clone()); - }*/ - auto private_overlay = private_block_overlays_.choose_overlay(block_id.shard_full()); + } + auto private_overlay = private_block_overlays_v2_.choose_overlay(block_id.shard_full()); if (!private_overlay.empty()) { td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_block_candidate, block_id, cc_seqno, validator_set_hash, data.clone()); @@ -385,11 +389,11 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, bool custom_overlays VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; return; } - /*if (broadcast.block_id.is_masterchain() && !private_block_overlays_.empty()) { + if (broadcast.block_id.is_masterchain() && !private_block_overlays_.empty()) { td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, broadcast.clone()); - }*/ - auto private_overlay = private_block_overlays_.choose_overlay(broadcast.block_id.shard_full()); + } + auto private_overlay = private_block_overlays_v2_.choose_overlay(broadcast.block_id.shard_full()); if (!private_overlay.empty()) { td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_broadcast, broadcast.clone()); } @@ -550,7 +554,7 @@ void FullNodeImpl::got_key_block_state(td::Ref state) { } } - // set_private_block_overlays_enable_compression(m->get_consensus_config().proto_version >= 3); + set_private_block_overlays_enable_compression(m->get_consensus_config().proto_version >= 3); if (current_validators != current_validators_) { current_validators_ = std::move(current_validators); @@ -703,16 +707,17 @@ void FullNodeImpl::update_private_overlays() { for (auto &p : custom_overlays_) { update_custom_overlay(p.second); } - /*private_block_overlays_.clear(); + + private_block_overlays_.clear(); if (local_keys_.empty()) { return; } for (const auto &key : local_keys_) { create_private_block_overlay(key); - }*/ + } } -/*void FullNodeImpl::set_private_block_overlays_enable_compression(bool value) { +void FullNodeImpl::set_private_block_overlays_enable_compression(bool value) { if (private_block_overlays_enable_compression_ == value) { return; } @@ -723,6 +728,9 @@ void FullNodeImpl::update_private_overlays() { } void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { + if (!use_old_private_overlays_) { + return; + } CHECK(local_keys_.count(key)); if (current_validators_.count(key)) { std::vector nodes; @@ -734,7 +742,7 @@ void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { private_block_overlays_enable_compression_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_, actor_id(this)); } -}*/ +} void FullNodeImpl::update_custom_overlay(CustomOverlayInfo &overlay) { auto old_actors = std::move(overlay.actors_); diff --git a/validator/full-node.h b/validator/full-node.h index 477d961e5e..0545dfb98d 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -75,8 +75,9 @@ class FullNode : public td::actor::Actor { virtual void add_collator_adnl_id(adnl::AdnlNodeIdShort id) = 0; virtual void del_collator_adnl_id(adnl::AdnlNodeIdShort id) = 0; - virtual void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, - td::uint32 max_size, td::Promise promise) = 0; + virtual void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + td::uint32 expiry_at, td::uint32 max_size, + td::Promise promise) = 0; virtual void import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) = 0; diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 1a172993af..16f6fffbec 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -144,14 +144,14 @@ class FullNodeImpl : public FullNode { td::Promise started_promise_; FullNodeConfig config_; - // TODO: Decide what to do with old private overlays. Maybe use old or new depending on some flag in config. - /* - std::map> private_block_overlays_; + // Private overlays: + // Old overlays - one overlay for all validators + // New overlays (v2) - overlay per shard (monitor_min_split depth). + bool use_old_private_overlays_ = false; // TODO: set from config + std::map> private_block_overlays_; bool private_block_overlays_enable_compression_ = false; - void set_private_block_overlays_enable_compression(bool value); - void create_private_block_overlay(PublicKeyHash key); - */ bool broadcast_block_candidates_in_public_overlay_ = false; + FullNodePrivateBlockOverlaysV2 private_block_overlays_v2_; struct CustomOverlayInfo { CustomOverlayParams params_; @@ -162,13 +162,12 @@ class FullNodeImpl : public FullNode { std::queue custom_overlays_sent_broadcasts_lru_; void update_private_overlays(); - // void set_private_block_overlays_enable_compression(bool value); - // void create_private_block_overlay(PublicKeyHash key); + void set_private_block_overlays_enable_compression(bool value); + void create_private_block_overlay(PublicKeyHash key); void update_custom_overlay(CustomOverlayInfo& overlay); void send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast); void send_block_candidate_broadcast_to_custom_overlays(const BlockIdExt& block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, const td::BufferSlice& data); - FullNodePrivateBlockOverlays private_block_overlays_; }; } // namespace fullnode diff --git a/validator/manager.cpp b/validator/manager.cpp index 663cc7a11e..c2233e388f 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -504,6 +504,10 @@ void ValidatorManagerImpl::new_block_candidate(BlockIdExt block_id, td::BufferSl if (!started_) { return; } + if (!need_monitor(block_id.shard_full())) { + VLOG(VALIDATOR_DEBUG) << "dropping block candidate broadcast: not monitoring shard"; + return; + } add_cached_block_candidate(ReceivedBlock{block_id, std::move(data)}); } From b8999be2c0d8c36e339d28c977bb27800a4cfd58 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 5 Jun 2024 18:05:08 +0300 Subject: [PATCH 092/524] New archive format and importing archive slices --- create-hardfork/create-hardfork.cpp | 4 +- test/test-ton-collator.cpp | 5 +- tl/generate/scheme/ton_api.tl | 1 + tl/generate/scheme/ton_api.tlo | Bin 96776 -> 96928 bytes validator/db/archive-manager.cpp | 15 +- validator/db/archive-manager.hpp | 7 +- validator/db/archive-slice.cpp | 228 +++++++++++++----- validator/db/archive-slice.hpp | 24 +- validator/db/archiver.cpp | 20 +- validator/db/archiver.hpp | 5 +- validator/db/rootdb.cpp | 11 +- validator/db/rootdb.hpp | 2 +- validator/full-node-master.cpp | 2 +- validator/full-node-shard.cpp | 29 ++- validator/full-node-shard.h | 4 +- validator/full-node-shard.hpp | 6 +- validator/full-node.cpp | 23 +- validator/full-node.hpp | 4 +- validator/import-db-slice.cpp | 290 +++++++++++++++++------ validator/import-db-slice.hpp | 44 +++- validator/interfaces/db.h | 3 +- validator/interfaces/validator-manager.h | 2 + validator/manager-disk.hpp | 7 +- validator/manager-hardfork.hpp | 8 +- validator/manager.cpp | 86 +++---- validator/manager.hpp | 9 +- validator/net/download-archive-slice.cpp | 22 +- validator/net/download-archive-slice.hpp | 6 +- validator/validator.h | 7 +- 29 files changed, 607 insertions(+), 267 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 77fe7155ee..bb590960ca 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -270,8 +270,8 @@ class HardforkCreator : public td::actor::Actor { void get_next_key_blocks(ton::BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) override { } - void download_archive(ton::BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) override { + void download_archive(ton::BlockSeqno masterchain_seqno, ton::ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { } void download_out_msg_queue_proof( ton::ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index e3bac4e1c1..6deb369ade 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -371,9 +371,8 @@ class TestNode : public td::actor::Actor { void get_next_key_blocks(ton::BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) override { } - void download_archive(ton::BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - - td::Promise promise) override { + void download_archive(ton::BlockSeqno masterchain_seqno, ton::ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { } void download_out_msg_queue_proof( ton::ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index f7dfb93fa2..a4956780c3 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -456,6 +456,7 @@ tonNode.downloadKeyBlockProof block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadBlockProofLink block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadKeyBlockProofLink block:tonNode.blockIdExt = tonNode.Data; tonNode.getArchiveInfo masterchain_seqno:int = tonNode.ArchiveInfo; +tonNode.getShardArchiveInfo masterchain_seqno:int shard_prefix:tonNode.shardId = tonNode.ArchiveInfo; tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data; tonNode.getOutMsgQueueProof dst_shard:tonNode.shardId blocks:(vector tonNode.blockIdExt) limits:tonNode.importedMsgQueueLimits = tonNode.OutMsgQueueProof; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 83f7d32f9dd86c38cda1b789ece45c7586c918a8..9ffd9fd82766d1736b79d0a245f2275660325268 100644 GIT binary patch delta 97 zcmeD9!n)uqYr_`C6l2D=?Wx9$pB*$KtbYqhm*nU9<)@_TrKgqzXCxM-I2I*mWR|6R z=B4HHzdY7(^O%qi({#fEM*T?(S$T^=D&h-@QqwXk7$)~km784fOJI9M3F8J<0Aorf AUH||9 delta 25 hcmZ4Rm9^swYr_`C6l2EL?Wx9$pB=V;C}G^d3ILJ)3c>&Y diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index b87d04f787..88da8dc8cf 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -600,8 +600,8 @@ void ArchiveManager::load_package(PackageId id) { } } - desc.file = - td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get(), statistics_); + desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, 0, db_root_, + archive_lru_.get(), statistics_); m.emplace(id, std::move(desc)); update_permanent_slices(); @@ -635,8 +635,9 @@ const ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull FileDescription new_desc{id, false}; td::mkdir(db_root_ + id.path()).ensure(); std::string prefix = PSTRING() << db_root_ << id.path() << id.name(); - new_desc.file = - td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get(), statistics_); + new_desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, + id.key || id.temp ? 0 : cur_shard_split_depth_, db_root_, + archive_lru_.get(), statistics_); const FileDescription &desc = f.emplace(id, std::move(new_desc)); if (!id.temp) { update_desc(f, desc, shard, seqno, ts, lt); @@ -1091,14 +1092,16 @@ PackageId ArchiveManager::get_package_id_force(BlockSeqno masterchain_seqno, Sha return it->first; } -void ArchiveManager::get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) { +void ArchiveManager::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) { auto F = get_file_desc_by_seqno(ShardIdFull{masterchainId}, masterchain_seqno, false); if (!F) { promise.set_error(td::Status::Error(ErrorCode::notready, "archive not found")); return; } - td::actor::send_closure(F->file_actor_id(), &ArchiveSlice::get_archive_id, masterchain_seqno, std::move(promise)); + td::actor::send_closure(F->file_actor_id(), &ArchiveSlice::get_archive_id, masterchain_seqno, shard_prefix, + std::move(promise)); } void ArchiveManager::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index a1ed970228..e260b8dc76 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -65,7 +65,7 @@ class ArchiveManager : public td::actor::Actor { void get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise promise); void get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, td::Promise promise); - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise); + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise); void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise); @@ -75,6 +75,10 @@ class ArchiveManager : public td::actor::Actor { void commit_transaction(); void set_async_mode(bool mode, td::Promise promise); + void set_current_shard_split_depth(td::uint32 value) { + cur_shard_split_depth_ = value; + } + static constexpr td::uint32 archive_size() { return 20000; } @@ -173,6 +177,7 @@ class ArchiveManager : public td::actor::Actor { bool async_mode_ = false; bool huge_transaction_started_ = false; td::uint32 huge_transaction_size_ = 0; + td::uint32 cur_shard_split_depth_ = 0; DbStatistics statistics_; diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index d392431a94..0fcdf79249 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -39,7 +39,7 @@ class PackageStatistics { void record_close(uint64_t count = 1) { close_count.fetch_add(count, std::memory_order_relaxed); } - + void record_read(double time, uint64_t bytes) { read_bytes.fetch_add(bytes, std::memory_order_relaxed); std::lock_guard guard(read_mutex); @@ -56,10 +56,10 @@ class PackageStatistics { std::stringstream ss; ss.setf(std::ios::fixed); ss.precision(6); - + ss << "ton.pack.open COUNT : " << open_count.exchange(0, std::memory_order_relaxed) << "\n"; ss << "ton.pack.close COUNT : " << close_count.exchange(0, std::memory_order_relaxed) << "\n"; - + ss << "ton.pack.read.bytes COUNT : " << read_bytes.exchange(0, std::memory_order_relaxed) << "\n"; ss << "ton.pack.write.bytes COUNT : " << write_bytes.exchange(0, std::memory_order_relaxed) << "\n"; @@ -118,7 +118,7 @@ void PackageWriter::append(std::string filename, td::BufferSlice data, return; } start = td::Timestamp::now(); - offset = p->append(std::move(filename), std::move(data), !async_mode_); + offset = p->append(std::move(filename), std::move(data), !async_mode_); end = td::Timestamp::now(); size = p->size(); } @@ -152,6 +152,21 @@ class PackageReader : public td::actor::Actor { std::shared_ptr statistics_; }; +static std::string get_package_file_name(PackageId p_id, ShardIdFull shard_prefix) { + td::StringBuilder sb; + sb << p_id.name(); + if (!shard_prefix.is_masterchain()) { + sb << "."; + sb << shard_prefix.workchain << ":" << shard_to_str(shard_prefix.shard); + } + sb << ".pack"; + return sb.as_cslice().str(); +} + +static std::string package_info_to_str(BlockSeqno seqno, ShardIdFull shard_prefix) { + return PSTRING() << seqno << "." << shard_prefix.workchain << ":" << shard_to_str(shard_prefix.shard); +} + void ArchiveSlice::add_handle(BlockHandle handle, td::Promise promise) { if (destroyed_) { promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd")); @@ -271,7 +286,8 @@ void ArchiveSlice::add_file(BlockHandle handle, FileReference ref_id, td::Buffer TRY_RESULT_PROMISE( promise, p, choose_package( - handle ? handle->id().is_masterchain() ? handle->id().seqno() : handle->masterchain_ref_block() : 0, true)); + handle ? handle->id().is_masterchain() ? handle->id().seqno() : handle->masterchain_ref_block() : 0, + handle ? handle->id().shard_full() : ShardIdFull{masterchainId}, true)); std::string value; auto R = kv_->get(ref_id.hash().to_hex(), value); R.ensure(); @@ -376,7 +392,8 @@ void ArchiveSlice::get_file(ConstBlockHandle handle, FileReference ref_id, td::P TRY_RESULT_PROMISE( promise, p, choose_package( - handle ? handle->id().is_masterchain() ? handle->id().seqno() : handle->masterchain_ref_block() : 0, false)); + handle ? handle->id().is_masterchain() ? handle->id().seqno() : handle->masterchain_ref_block() : 0, + handle ? handle->id().shard_full() : ShardIdFull{masterchainId}, false)); promise = begin_async_query(std::move(promise)); auto P = td::PromiseCreator::lambda( [promise = std::move(promise)](td::Result> R) mutable { @@ -536,18 +553,32 @@ void ArchiveSlice::get_slice(td::uint64 archive_id, td::uint64 offset, td::uint3 } before_query(); auto value = static_cast(archive_id >> 32); - TRY_RESULT_PROMISE(promise, p, choose_package(value, false)); + PackageInfo *p; + if (shard_split_depth_ == 0) { + TRY_RESULT_PROMISE_ASSIGN(promise, p, choose_package(value, ShardIdFull{masterchainId}, false)); + } else { + if (value >= packages_.size()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "no such package")); + return; + } + p = &packages_[value]; + } promise = begin_async_query(std::move(promise)); td::actor::create_actor("readfile", p->path, offset, limit, 0, std::move(promise)).release(); } -void ArchiveSlice::get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) { +void ArchiveSlice::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) { before_query(); if (!sliced_mode_) { promise.set_result(archive_id_); } else { - TRY_RESULT_PROMISE(promise, p, choose_package(masterchain_seqno, false)); - promise.set_result(p->id * (1ull << 32) + archive_id_); + TRY_RESULT_PROMISE(promise, p, choose_package(masterchain_seqno, shard_prefix, false)); + if (shard_split_depth_ == 0) { + promise.set_result(p->seqno * (1ull << 32) + archive_id_); + } else { + promise.set_result(p->idx * (1ull << 32) + archive_id_); + } } } @@ -573,9 +604,18 @@ void ArchiveSlice::before_query() { R2.ensure(); slice_size_ = td::to_integer(value); CHECK(slice_size_ > 0); + R2 = kv_->get("shard_split_depth", value); + R2.ensure(); + if (R2.move_as_ok() == td::KeyValue::GetStatus::Ok) { + shard_split_depth_ = td::to_integer(value); + CHECK(shard_split_depth_ <= 60); + } else { + shard_split_depth_ = 0; + } for (td::uint32 i = 0; i < tot; i++) { R2 = kv_->get(PSTRING() << "status." << i, value); R2.ensure(); + CHECK(R2.move_as_ok() == td::KeyValue::GetStatus::Ok); auto len = td::to_integer(value); R2 = kv_->get(PSTRING() << "version." << i, value); R2.ensure(); @@ -583,12 +623,24 @@ void ArchiveSlice::before_query() { if (R2.move_as_ok() == td::KeyValue::GetStatus::Ok) { ver = td::to_integer(value); } - auto v = archive_id_ + slice_size_ * i; - add_package(v, len, ver); + td::uint32 seqno; + ShardIdFull shard_prefix; + if (shard_split_depth_ == 0) { + seqno = archive_id_ + slice_size_ * i; + shard_prefix = ShardIdFull{masterchainId}; + } else { + R2 = kv_->get(PSTRING() << "info." << i, value); + R2.ensure(); + CHECK(R2.move_as_ok() == td::KeyValue::GetStatus::Ok); + unsigned long long shard; + CHECK(sscanf(value.c_str(), "%u.%d:%016llx", &seqno, &shard_prefix.workchain, &shard) == 3); + shard_prefix.shard = shard; + } + add_package(seqno, shard_prefix, len, ver); } } else { auto len = td::to_integer(value); - add_package(archive_id_, len, 0); + add_package(archive_id_, ShardIdFull{masterchainId}, len, 0); } } else { if (!temp_ && !key_blocks_only_) { @@ -599,13 +651,17 @@ void ArchiveSlice::before_query() { kv_->set("slice_size", td::to_string(slice_size_)).ensure(); kv_->set("status.0", "0").ensure(); kv_->set("version.0", td::to_string(default_package_version())).ensure(); + if (shard_split_depth_ > 0) { + kv_->set("info.0", package_info_to_str(archive_id_, ShardIdFull{masterchainId})).ensure(); + kv_->set("shard_split_depth", td::to_string(shard_split_depth_)).ensure(); + } kv_->commit_transaction().ensure(); - add_package(archive_id_, 0, default_package_version()); + add_package(archive_id_, ShardIdFull{masterchainId}, 0, default_package_version()); } else { kv_->begin_transaction().ensure(); kv_->set("status", "0").ensure(); kv_->commit_transaction().ensure(); - add_package(archive_id_, 0, 0); + add_package(archive_id_, ShardIdFull{masterchainId}, 0, 0); } } } @@ -642,6 +698,7 @@ void ArchiveSlice::do_close() { statistics_.pack_statistics->record_close(packages_.size()); } packages_.clear(); + id_to_package_.clear(); } template @@ -697,48 +754,61 @@ void ArchiveSlice::set_async_mode(bool mode, td::Promise promise) { } } -ArchiveSlice::ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root, +ArchiveSlice::ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, + td::uint32 shard_split_depth, std::string db_root, td::actor::ActorId archive_lru, DbStatistics statistics) : archive_id_(archive_id) , key_blocks_only_(key_blocks_only) , temp_(temp) , finalized_(finalized) , p_id_(archive_id_, key_blocks_only_, temp_) + , shard_split_depth_(temp || key_blocks_only ? 0 : shard_split_depth) , db_root_(std::move(db_root)) , archive_lru_(std::move(archive_lru)) , statistics_(statistics) { db_path_ = PSTRING() << db_root_ << p_id_.path() << p_id_.name() << ".index"; } -td::Result ArchiveSlice::choose_package(BlockSeqno masterchain_seqno, bool force) { +td::Result ArchiveSlice::choose_package(BlockSeqno masterchain_seqno, + ShardIdFull shard_prefix, bool force) { if (temp_ || key_blocks_only_ || !sliced_mode_) { return &packages_[0]; } if (masterchain_seqno < archive_id_) { return td::Status::Error(ErrorCode::notready, "too small masterchain seqno"); } - auto v = (masterchain_seqno - archive_id_) / slice_size_; - if (v >= packages_.size()) { + masterchain_seqno -= (masterchain_seqno - archive_id_) % slice_size_; + CHECK((masterchain_seqno - archive_id_) % slice_size_ == 0); + if (shard_split_depth_ == 0) { + shard_prefix = ShardIdFull{masterchainId}; + } else if (!shard_prefix.is_masterchain()) { + shard_prefix.shard |= 1; // In case length is < split depth + shard_prefix = ton::shard_prefix(shard_prefix, shard_split_depth_); + } + auto it = id_to_package_.find({masterchain_seqno, shard_prefix}); + if (it == id_to_package_.end()) { if (!force) { - return td::Status::Error(ErrorCode::notready, "too big masterchain seqno"); + return td::Status::Error(ErrorCode::notready, "no such package"); } - CHECK(v == packages_.size()); begin_transaction(); + size_t v = packages_.size(); kv_->set("slices", td::to_string(v + 1)).ensure(); kv_->set(PSTRING() << "status." << v, "0").ensure(); kv_->set(PSTRING() << "version." << v, td::to_string(default_package_version())).ensure(); + if (shard_split_depth_ > 0) { + kv_->set(PSTRING() << "info." << v, package_info_to_str(masterchain_seqno, shard_prefix)).ensure(); + } commit_transaction(); - CHECK((masterchain_seqno - archive_id_) % slice_size_ == 0); - add_package(masterchain_seqno, 0, default_package_version()); + add_package(masterchain_seqno, shard_prefix, 0, default_package_version()); return &packages_[v]; } else { - return &packages_[v]; + return &packages_[it->second]; } } -void ArchiveSlice::add_package(td::uint32 seqno, td::uint64 size, td::uint32 version) { +void ArchiveSlice::add_package(td::uint32 seqno, ShardIdFull shard_prefix, td::uint64 size, td::uint32 version) { PackageId p_id{seqno, key_blocks_only_, temp_}; - std::string path = PSTRING() << db_root_ << p_id.path() << p_id.name() << ".pack"; + std::string path = PSTRING() << db_root_ << p_id.path() << get_package_file_name(p_id, shard_prefix); auto R = Package::open(path, false, true); if (R.is_error()) { LOG(FATAL) << "failed to open/create archive '" << path << "': " << R.move_as_error(); @@ -748,8 +818,9 @@ void ArchiveSlice::add_package(td::uint32 seqno, td::uint64 size, td::uint32 ver statistics_.pack_statistics->record_open(); } auto idx = td::narrow_cast(packages_.size()); + id_to_package_[{seqno, shard_prefix}] = idx; if (finalized_) { - packages_.emplace_back(nullptr, td::actor::ActorOwn(), seqno, path, idx, version); + packages_.emplace_back(nullptr, td::actor::ActorOwn(), seqno, shard_prefix, path, idx, version); return; } auto pack = std::make_shared(R.move_as_ok()); @@ -757,7 +828,7 @@ void ArchiveSlice::add_package(td::uint32 seqno, td::uint64 size, td::uint32 ver pack->truncate(size).ensure(); } auto writer = td::actor::create_actor("writer", pack, async_mode_, statistics_.pack_statistics); - packages_.emplace_back(std::move(pack), std::move(writer), seqno, path, idx, version); + packages_.emplace_back(std::move(pack), std::move(writer), seqno, shard_prefix, path, idx, version); } namespace { @@ -793,6 +864,7 @@ void ArchiveSlice::destroy(td::Promise promise) { statistics_.pack_statistics->record_close(packages_.size()); } packages_.clear(); + id_to_package_.clear(); kv_ = nullptr; PackageId p_id{archive_id_, key_blocks_only_, temp_}; @@ -866,7 +938,7 @@ void ArchiveSlice::move_handle(ConstBlockHandle handle, Package *old_pack, Packa move_file(fileref::Block{handle->id()}, old_pack, pack); } -bool ArchiveSlice::truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_idx, +bool ArchiveSlice::truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_seqno, Package *pack) { std::string value; auto R = kv_->get(get_db_key_block_info(block_id), value); @@ -881,18 +953,18 @@ bool ArchiveSlice::truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block return false; } - auto S = choose_package(seqno, false); + auto S = choose_package(seqno, block_id.shard_full(), false); S.ensure(); auto p = S.move_as_ok(); - CHECK(p->idx <= cutoff_idx); - if (p->idx == cutoff_idx) { + CHECK(p->seqno <= cutoff_seqno); + if (p->seqno == cutoff_seqno) { move_handle(std::move(handle), p->package.get(), pack); } return true; } -void ArchiveSlice::truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_idx, +void ArchiveSlice::truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_seqno, Package *pack) { auto key = get_db_key_lt_desc(shard); std::string value; @@ -918,7 +990,7 @@ void ArchiveSlice::truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shar E.ensure(); auto e = E.move_as_ok(); - if (truncate_block(masterchain_seqno, create_block_id(e->id_), cutoff_idx, pack)) { + if (truncate_block(masterchain_seqno, create_block_id(e->id_), cutoff_seqno, pack)) { CHECK(new_last_idx == i); new_last_idx = i + 1; } @@ -930,7 +1002,7 @@ void ArchiveSlice::truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shar } } -void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise) { +void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle, td::Promise promise) { if (temp_ || archive_id_ > masterchain_seqno) { destroy(std::move(promise)); return; @@ -943,15 +1015,8 @@ void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handl return; } - auto cutoff = choose_package(masterchain_seqno, false); - cutoff.ensure(); - auto pack = cutoff.move_as_ok(); - CHECK(pack); - - auto pack_r = Package::open(pack->path + ".new", false, true); - pack_r.ensure(); - auto new_package = std::make_shared(pack_r.move_as_ok()); - new_package->truncate(0).ensure(); + std::map old_packages; + std::map> new_packages; std::string value; auto status_key = create_serialize_tl_object(); @@ -972,38 +1037,71 @@ void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handl auto G = fetch_tl_object(value, true); G.ensure(); auto g = G.move_as_ok(); + ShardIdFull shard{g->workchain_, static_cast(g->shard_)}; + + auto package_r = choose_package(masterchain_seqno, shard, false); + if (package_r.is_error()) { + continue; + } + auto package = package_r.move_as_ok(); + CHECK(package); + if (!old_packages.count(package->shard_prefix)) { + old_packages[package->shard_prefix] = package; + auto new_package_r = Package::open(package->path + ".new", false, true); + new_package_r.ensure(); + auto new_package = std::make_shared(new_package_r.move_as_ok()); + new_package->truncate(0).ensure(); + new_packages[package->shard_prefix] = std::move(new_package); + } + truncate_shard(masterchain_seqno, shard, package->seqno, new_packages[package->shard_prefix].get()); + } - truncate_shard(masterchain_seqno, ShardIdFull{g->workchain_, static_cast(g->shard_)}, pack->idx, - new_package.get()); + for (auto& [shard_prefix, package] : old_packages) { + auto new_package = new_packages[shard_prefix]; + CHECK(new_package); + package->package = new_package; + package->writer.reset(); + td::unlink(package->path).ensure(); + td::rename(package->path + ".new", package->path).ensure(); + package->writer = td::actor::create_actor("writer", new_package, async_mode_); } + std::vector new_packages_info; + if (!sliced_mode_) { - kv_->set("status", td::to_string(new_package->size())).ensure(); + kv_->set("status", td::to_string(packages_.at(0).package->size())).ensure(); } else { - kv_->set(PSTRING() << "status." << pack->idx, td::to_string(new_package->size())).ensure(); - for (size_t i = pack->idx + 1; i < packages_.size(); i++) { + for (PackageInfo &package : packages_) { + if (package.seqno <= masterchain_seqno) { + new_packages_info.push_back(std::move(package)); + } else { + td::unlink(package.path).ensure(); + } + } + id_to_package_.clear(); + for (td::uint32 i = 0; i < new_packages_info.size(); ++i) { + PackageInfo &package = new_packages_info[i]; + package.idx = i; + kv_->set(PSTRING() << "status." << i, td::to_string(package.package->size())).ensure(); + kv_->set(PSTRING() << "version." << i, td::to_string(package.version)).ensure(); + if (shard_split_depth_ > 0) { + kv_->set(PSTRING() << "info." << i, package_info_to_str(package.seqno, package.shard_prefix)).ensure(); + } + id_to_package_[{package.seqno, package.shard_prefix}] = i; + } + for (size_t i = new_packages_info.size(); i < packages_.size(); i++) { kv_->erase(PSTRING() << "status." << i); kv_->erase(PSTRING() << "version." << i); + kv_->erase(PSTRING() << "info." << i); } - kv_->set("slices", td::to_string(pack->idx + 1)); - } - - pack->package = new_package; - pack->writer.reset(); - td::unlink(pack->path).ensure(); - td::rename(pack->path + ".new", pack->path).ensure(); - pack->writer = td::actor::create_actor("writer", new_package, async_mode_); - - for (auto idx = pack->idx + 1; idx < packages_.size(); idx++) { - td::unlink(packages_[idx].path).ensure(); - } - if (statistics_.pack_statistics) { - statistics_.pack_statistics->record_close(packages_.size() - pack->idx - 1); + kv_->set("slices", td::to_string(new_packages_info.size())); + if (statistics_.pack_statistics) { + statistics_.pack_statistics->record_close(packages_.size() - new_packages_info.size()); + } + packages_ = std::move(new_packages_info); } - packages_.erase(packages_.begin() + pack->idx + 1, packages_.end()); kv_->commit_transaction().ensure(); - promise.set_value(td::Unit()); } diff --git a/validator/db/archive-slice.hpp b/validator/db/archive-slice.hpp index faec2fb830..a027ec0ff5 100644 --- a/validator/db/archive-slice.hpp +++ b/validator/db/archive-slice.hpp @@ -96,10 +96,10 @@ class ArchiveLru; class ArchiveSlice : public td::actor::Actor { public: - ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root, - td::actor::ActorId archive_lru, DbStatistics statistics = {}); + ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, td::uint32 shard_split_depth, + std::string db_root, td::actor::ActorId archive_lru, DbStatistics statistics = {}); - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise); + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise); void add_handle(BlockHandle handle, td::Promise promise); void update_handle(BlockHandle handle, td::Promise promise); @@ -159,6 +159,7 @@ class ArchiveSlice : public td::actor::Actor { bool sliced_mode_{false}; td::uint32 huge_transaction_size_ = 0; td::uint32 slice_size_{100}; + td::uint32 shard_split_depth_ = 0; enum Status { st_closed, st_open, st_want_close @@ -171,28 +172,31 @@ class ArchiveSlice : public td::actor::Actor { std::unique_ptr kv_; struct PackageInfo { - PackageInfo(std::shared_ptr package, td::actor::ActorOwn writer, BlockSeqno id, + PackageInfo(std::shared_ptr package, td::actor::ActorOwn writer, BlockSeqno seqno, ShardIdFull shard_prefix, std::string path, td::uint32 idx, td::uint32 version) : package(std::move(package)) , writer(std ::move(writer)) - , id(id) + , seqno(seqno) + , shard_prefix(shard_prefix) , path(std::move(path)) , idx(idx) , version(version) { } std::shared_ptr package; td::actor::ActorOwn writer; - BlockSeqno id; + BlockSeqno seqno; + ShardIdFull shard_prefix; std::string path; td::uint32 idx; td::uint32 version; }; std::vector packages_; + std::map, td::uint32> id_to_package_; - td::Result choose_package(BlockSeqno masterchain_seqno, bool force); - void add_package(BlockSeqno masterchain_seqno, td::uint64 size, td::uint32 version); - void truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_idx, Package *pack); - bool truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_idx, Package *pack); + td::Result choose_package(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, bool force); + void add_package(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::uint64 size, td::uint32 version); + void truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_seqno, Package *pack); + bool truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_seqno, Package *pack); void delete_handle(ConstBlockHandle handle); void delete_file(FileReference ref_id); diff --git a/validator/db/archiver.cpp b/validator/db/archiver.cpp index c8cbd7b977..93ba18ca29 100644 --- a/validator/db/archiver.cpp +++ b/validator/db/archiver.cpp @@ -25,11 +25,27 @@ namespace ton { namespace validator { BlockArchiver::BlockArchiver(BlockHandle handle, td::actor::ActorId archive_db, - td::Promise promise) - : handle_(std::move(handle)), archive_(archive_db), promise_(std::move(promise)) { + td::actor::ActorId db, td::Promise promise) + : handle_(std::move(handle)), archive_(archive_db), db_(std::move(db)), promise_(std::move(promise)) { } void BlockArchiver::start_up() { + if (handle_->id().is_masterchain()) { + td::actor::send_closure(db_, &Db::get_block_state, handle_, + [SelfId = actor_id(this), archive = archive_](td::Result> R) { + R.ensure(); + td::Ref state{R.move_as_ok()}; + td::uint32 monitor_min_split = state->monitor_min_split_depth(basechainId); + td::actor::send_closure(archive, &ArchiveManager::set_current_shard_split_depth, + monitor_min_split); + td::actor::send_closure(SelfId, &BlockArchiver::move_handle); + }); + } else { + move_handle(); + } +} + +void BlockArchiver::move_handle() { if (handle_->handle_moved_to_archive()) { moved_handle(); } else { diff --git a/validator/db/archiver.hpp b/validator/db/archiver.hpp index 859f269cd5..9498977fdf 100644 --- a/validator/db/archiver.hpp +++ b/validator/db/archiver.hpp @@ -33,11 +33,13 @@ class FileDb; class BlockArchiver : public td::actor::Actor { public: - BlockArchiver(BlockHandle handle, td::actor::ActorId archive_db, td::Promise promise); + BlockArchiver(BlockHandle handle, td::actor::ActorId archive_db, td::actor::ActorId db, + td::Promise promise); void abort_query(td::Status error); void start_up() override; + void move_handle(); void moved_handle(); void got_proof(td::BufferSlice data); void written_proof(); @@ -50,6 +52,7 @@ class BlockArchiver : public td::actor::Actor { private: BlockHandle handle_; td::actor::ActorId archive_; + td::actor::ActorId db_; td::Promise promise_; }; diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 89868f7235..d1c1c24332 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -330,7 +330,8 @@ void RootDb::try_get_static_file(FileHash file_hash, td::Promise promise) { - td::actor::create_actor("archiver", std::move(handle), archive_db_.get(), std::move(promise)) + td::actor::create_actor("archiver", std::move(handle), archive_db_.get(), actor_id(this), + std::move(promise)) .release(); } @@ -404,7 +405,8 @@ void RootDb::start_up() { } void RootDb::archive(BlockHandle handle, td::Promise promise) { - td::actor::create_actor("archiveblock", std::move(handle), archive_db_.get(), std::move(promise)) + td::actor::create_actor("archiveblock", std::move(handle), archive_db_.get(), actor_id(this), + std::move(promise)) .release(); } @@ -483,8 +485,9 @@ void RootDb::check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise< std::move(P)); } -void RootDb::get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) { - td::actor::send_closure(archive_db_, &ArchiveManager::get_archive_id, masterchain_seqno, std::move(promise)); +void RootDb::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::get_archive_id, masterchain_seqno, shard_prefix, + std::move(promise)); } void RootDb::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 63509371a4..dc94c600d4 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -129,7 +129,7 @@ class RootDb : public Db { void check_key_block_proof_exists(BlockIdExt block_id, td::Promise promise) override; void check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise promise) override; - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override; + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise) override; void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) override; void set_async_mode(bool mode, td::Promise promise) override; diff --git a/validator/full-node-master.cpp b/validator/full-node-master.cpp index 8dca60d451..da49f0e2ea 100644 --- a/validator/full-node-master.cpp +++ b/validator/full-node-master.cpp @@ -386,7 +386,7 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo } }); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_, - std::move(P)); + ShardIdFull{masterchainId}, std::move(P)); } void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 1284cf9795..d0dbf1a64b 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -643,7 +643,24 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod }); VLOG(FULL_NODE_DEBUG) << "Got query getArchiveInfo " << query.masterchain_seqno_ << " from " << src; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_, - std::move(P)); + ShardIdFull{masterchainId}, std::move(P)); +} + +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getShardArchiveInfo &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_serialize_tl_object()); + } else { + promise.set_value(create_serialize_tl_object(R.move_as_ok())); + } + }); + ShardIdFull shard_prefix = create_shard_id(query.shard_prefix_); + VLOG(FULL_NODE_DEBUG) << "Got query getShardArchiveInfo " << query.masterchain_seqno_ << " " << shard_prefix.to_str() + << " from " << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_, + shard_prefix, std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, @@ -965,13 +982,13 @@ void FullNodeShardImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp t .release(); } -void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) { +void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) { auto &b = choose_neighbour(true); td::actor::create_actor( - "archive", masterchain_seqno, std::move(tmp_dir), adnl_id_, overlay_id_, b.adnl_id, timeout, validator_manager_, - b.use_rldp2() ? (td::actor::ActorId)rldp2_ : rldp_, overlays_, adnl_, client_, - create_neighbour_promise(b, std::move(promise))) + "archive", masterchain_seqno, shard_prefix, std::move(tmp_dir), adnl_id_, overlay_id_, b.adnl_id, timeout, + validator_manager_, b.use_rldp2() ? (td::actor::ActorId)rldp2_ : rldp_, overlays_, + adnl_, client_, create_neighbour_promise(b, std::move(promise))) .release(); } diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 4856989a0d..235e30514c 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -70,8 +70,8 @@ class FullNodeShard : public td::actor::Actor { td::Promise promise) = 0; virtual void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) = 0; - virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) = 0; + virtual void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) = 0; virtual void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise>> promise) = 0; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 393ee269e0..472abb1942 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -152,6 +152,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getShardArchiveInfo &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getOutMsgQueueProof &query, @@ -198,8 +200,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise) override; void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) override; - void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) override; + void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override; void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise>> promise) override; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 9f5fbf698a..335578cfb7 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -468,12 +468,17 @@ void FullNodeImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeou td::actor::send_closure(shard, &FullNodeShard::get_next_key_blocks, block_id, timeout, std::move(promise)); } -void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) { - auto shard = get_shard(ShardIdFull{masterchainId}); +void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) { + auto shard = get_shard(shard_prefix); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping download archive query to unknown shard"; + promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); + return; + } CHECK(!shard.empty()); - td::actor::send_closure(shard, &FullNodeShard::download_archive, masterchain_seqno, std::move(tmp_dir), timeout, - std::move(promise)); + td::actor::send_closure(shard, &FullNodeShard::download_archive, masterchain_seqno, shard_prefix, std::move(tmp_dir), + timeout, std::move(promise)); } void FullNodeImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, @@ -689,10 +694,10 @@ void FullNodeImpl::start_up() { td::Promise> promise) override { td::actor::send_closure(id_, &FullNodeImpl::get_next_key_blocks, block_id, timeout, std::move(promise)); } - void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) override { - td::actor::send_closure(id_, &FullNodeImpl::download_archive, masterchain_seqno, std::move(tmp_dir), timeout, - std::move(promise)); + void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { + td::actor::send_closure(id_, &FullNodeImpl::download_archive, masterchain_seqno, shard_prefix, std::move(tmp_dir), + timeout, std::move(promise)); } void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Timestamp timeout, diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 76daf1159e..33ccf2cd14 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -82,8 +82,8 @@ class FullNodeImpl : public FullNode { void download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise); - void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise); + void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise); void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise>> promise); diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index a93fb05be6..06573d347e 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -17,6 +17,7 @@ Copyright 2019-2020 Telegram Systems LLP */ #include "import-db-slice.hpp" + #include "validator/db/fileref.hpp" #include "td/utils/overloaded.h" #include "validator/fabric.h" @@ -26,35 +27,91 @@ #include "ton/ton-io.hpp" #include "downloaders/download-state.hpp" +#include + namespace ton { namespace validator { -ArchiveImporter::ArchiveImporter(std::string path, td::Ref state, BlockSeqno shard_client_seqno, +ArchiveImporter::ArchiveImporter(std::string db_root, td::Ref state, BlockSeqno shard_client_seqno, td::Ref opts, td::actor::ActorId manager, - td::Promise> promise) - : path_(std::move(path)) - , state_(std::move(state)) + std::vector to_import_files, + td::Promise> promise) + : db_root_(std::move(db_root)) + , last_masterchain_state_(std::move(state)) , shard_client_seqno_(shard_client_seqno) + , start_import_seqno_(shard_client_seqno + 1) , opts_(std::move(opts)) , manager_(manager) + , to_import_files_(std::move(to_import_files)) + , use_imported_files_(!to_import_files_.empty()) , promise_(std::move(promise)) { } void ArchiveImporter::start_up() { - auto R = Package::open(path_, false, false); - if (R.is_error()) { - abort_query(R.move_as_error()); + if (use_imported_files_) { + LOG(INFO) << "Importing archive for masterchain seqno #" << start_import_seqno_ << " from disk"; + for (const std::string& path : to_import_files_) { + LOG(INFO) << "Importing file from disk " << path; + td::Status S = process_package(path, true); + if (S.is_error()) { + LOG(INFO) << "Error processing package " << path << ": " << S; + } + } + files_to_cleanup_.clear(); + processed_mc_archive(); + return; + } + LOG(INFO) << "Importing archive for masterchain seqno #" << start_import_seqno_ << " from net"; + td::actor::send_closure(manager_, &ValidatorManager::send_download_archive_request, start_import_seqno_, + ShardIdFull{masterchainId}, db_root_ + "/tmp/", td::Timestamp::in(3600.0), + [SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporter::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveImporter::downloaded_mc_archive, R.move_as_ok()); + } + }); +} + +void ArchiveImporter::downloaded_mc_archive(std::string path) { + td::Status S = process_package(path, true); + if (S.is_error()) { + abort_query(std::move(S)); + return; + } + processed_mc_archive(); +} + +void ArchiveImporter::processed_mc_archive() { + if (masterchain_blocks_.empty()) { + LOG(DEBUG) << "No masterhchain blocks in archive"; + last_masterchain_seqno_ = last_masterchain_state_->get_seqno(); + checked_all_masterchain_blocks(); + return; + } + + auto seqno = masterchain_blocks_.begin()->first; + LOG(DEBUG) << "First mc seqno in archive = " << seqno; + if (seqno > last_masterchain_state_->get_seqno() + 1) { + abort_query(td::Status::Error(ErrorCode::notready, "too big first masterchain seqno")); return; } - package_ = std::make_shared(R.move_as_ok()); - bool fail = false; - package_->iterate([&](std::string filename, td::BufferSlice data, td::uint64 offset) -> bool { + check_masterchain_block(seqno); +} + +td::Status ArchiveImporter::process_package(std::string path, bool with_masterchain) { + LOG(DEBUG) << "Processing package " << path << " (with_masterchain=" << with_masterchain << ")"; + files_to_cleanup_.push_back(path); + TRY_RESULT(p, Package::open(path, false, false)); + auto package = std::make_shared(std::move(p)); + + td::Status S = td::Status::OK(); + package->iterate([&](std::string filename, td::BufferSlice, td::uint64 offset) -> bool { auto F = FileReference::create(filename); if (F.is_error()) { - abort_query(F.move_as_error()); - fail = true; + S = F.move_as_error(); return false; } auto f = F.move_as_ok(); @@ -79,33 +136,26 @@ void ArchiveImporter::start_up() { ignore = false; is_proof = false; }, - [&](const auto &p) { ignore = true; })); - - if (!ignore) { - blocks_[b][is_proof ? 0 : 1] = offset; + [&](const auto &) { ignore = true; })); + + if (!ignore && (with_masterchain || !b.is_masterchain())) { + if (is_proof) { + blocks_[b].proof_pkg = package; + blocks_[b].proof_offset = offset; + } else { + blocks_[b].data_pkg = package; + blocks_[b].data_offset = offset; + } if (b.is_masterchain()) { masterchain_blocks_[b.seqno()] = b; + last_masterchain_seqno_ = std::max(last_masterchain_seqno_, b.seqno()); + } else { + have_shard_blocks_ = true; } } return true; }); - - if (fail) { - return; - } - - if (masterchain_blocks_.size() == 0) { - abort_query(td::Status::Error(ErrorCode::notready, "archive does not contain any masterchain blocks")); - return; - } - - auto seqno = masterchain_blocks_.begin()->first; - if (seqno > state_->get_seqno() + 1) { - abort_query(td::Status::Error(ErrorCode::notready, "too big first masterchain seqno")); - return; - } - - check_masterchain_block(seqno); + return S; } void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { @@ -115,17 +165,17 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { abort_query(td::Status::Error(ErrorCode::notready, "no new blocks")); return; } - checked_all_masterchain_blocks(seqno - 1); + checked_all_masterchain_blocks(); return; } - while (seqno <= state_->get_block_id().seqno()) { - if (seqno < state_->get_block_id().seqno()) { - if (!state_->check_old_mc_block_id(it->second)) { + while (seqno <= last_masterchain_state_->get_block_id().seqno()) { + if (seqno < last_masterchain_state_->get_block_id().seqno()) { + if (!last_masterchain_state_->check_old_mc_block_id(it->second)) { abort_query(td::Status::Error(ErrorCode::protoviolation, "bad old masterchain block id")); return; } } else { - if (state_->get_block_id() != it->second) { + if (last_masterchain_state_->get_block_id() != it->second) { abort_query(td::Status::Error(ErrorCode::protoviolation, "bad old masterchain block id")); return; } @@ -133,18 +183,27 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { seqno++; it = masterchain_blocks_.find(seqno); if (it == masterchain_blocks_.end()) { - checked_all_masterchain_blocks(seqno - 1); + checked_all_masterchain_blocks(); return; } } - if (seqno != state_->get_block_id().seqno() + 1) { + LOG(DEBUG) << "Checking masterchain block #" << seqno; + if (seqno != last_masterchain_state_->get_block_id().seqno() + 1) { abort_query(td::Status::Error(ErrorCode::protoviolation, "hole in masterchain seqno")); return; } auto it2 = blocks_.find(it->second); CHECK(it2 != blocks_.end()); + if (!it2->second.proof_pkg) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block proof")); + return; + } + if (!it2->second.data_pkg) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block data")); + return; + } - auto R1 = package_->read(it2->second[0]); + auto R1 = it2->second.proof_pkg->read(it2->second.proof_offset); if (R1.is_error()) { abort_query(R1.move_as_error()); return; @@ -156,7 +215,7 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { return; } - auto R2 = package_->read(it2->second[1]); + auto R2 = it2->second.data_pkg->read(it2->second.data_offset); if (R2.is_error()) { abort_query(R2.move_as_error()); return; @@ -175,7 +234,7 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { auto proof = proofR.move_as_ok(); auto data = dataR.move_as_ok(); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id = state_->get_block_id(), + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id = last_masterchain_state_->get_block_id(), data](td::Result R) mutable { if (R.is_error()) { td::actor::send_closure(SelfId, &ArchiveImporter::abort_query, R.move_as_error()); @@ -191,11 +250,12 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { td::actor::send_closure(SelfId, &ArchiveImporter::checked_masterchain_proof, std::move(handle), std::move(data)); }); - run_check_proof_query(it->second, std::move(proof), manager_, td::Timestamp::in(2.0), std::move(P), state_, - opts_->is_hardfork(it->second)); + run_check_proof_query(it->second, std::move(proof), manager_, td::Timestamp::in(2.0), std::move(P), + last_masterchain_state_, opts_->is_hardfork(it->second)); } void ArchiveImporter::checked_masterchain_proof(BlockHandle handle, td::Ref data) { + LOG(DEBUG) << "Checked proof for masterchain block #" << handle->id().seqno(); CHECK(data.not_null()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { R.ensure(); @@ -205,6 +265,7 @@ void ArchiveImporter::checked_masterchain_proof(BlockHandle handle, td::Refid().seqno(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); td::actor::send_closure(SelfId, &ArchiveImporter::got_new_materchain_state, @@ -214,22 +275,87 @@ void ArchiveImporter::applied_masterchain_block(BlockHandle handle) { } void ArchiveImporter::got_new_materchain_state(td::Ref state) { - state_ = std::move(state); - check_masterchain_block(state_->get_block_id().seqno() + 1); + last_masterchain_state_ = std::move(state); + imported_any_ = true; + check_masterchain_block(last_masterchain_state_->get_block_id().seqno() + 1); } -void ArchiveImporter::checked_all_masterchain_blocks(BlockSeqno seqno) { - check_next_shard_client_seqno(shard_client_seqno_ + 1); +void ArchiveImporter::checked_all_masterchain_blocks() { + LOG(DEBUG) << "Done importing masterchain blocks. Last block seqno = " << last_masterchain_seqno_; + if (start_import_seqno_ > last_masterchain_state_->get_seqno()) { + abort_query(td::Status::Error("no new masterchain blocks were imported")); + return; + } + BlockIdExt block_id; + CHECK(last_masterchain_state_->get_old_mc_block_id(start_import_seqno_, block_id)); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db_short, block_id, + [SelfId = actor_id(this)](td::Result> R) { + R.ensure(); + td::Ref state{R.move_as_ok()}; + td::actor::send_closure(SelfId, &ArchiveImporter::download_shard_archives, + std::move(state)); + }); +} + +void ArchiveImporter::download_shard_archives(td::Ref start_state) { + start_state_ = start_state; + td::uint32 monitor_min_split = start_state->monitor_min_split_depth(basechainId); + LOG(DEBUG) << "Monitor min split = " << monitor_min_split; + // If monitor_min_split == 0, we use the old archive format (packages are not separated by shard) + // If masterchain package has shard blocks then it's old archive format, don't need to download shards + if (monitor_min_split > 0 && !have_shard_blocks_ && !use_imported_files_) { + for (td::uint64 i = 0; i < (1ULL << monitor_min_split); ++i) { + ShardIdFull shard_prefix{basechainId, (i * 2 + 1) << (64 - monitor_min_split - 1)}; + if (opts_->need_monitor(shard_prefix, start_state)) { + ++pending_shard_archives_; + LOG(DEBUG) << "Downloading shard archive #" << start_import_seqno_ << " " << shard_prefix.to_str(); + download_shard_archive(shard_prefix); + } + } + } else { + LOG(DEBUG) << "Skip downloading shard archives"; + } + if (pending_shard_archives_ == 0) { + check_next_shard_client_seqno(shard_client_seqno_ + 1); + } +} + +void ArchiveImporter::download_shard_archive(ShardIdFull shard_prefix) { + td::actor::send_closure( + manager_, &ValidatorManager::send_download_archive_request, start_import_seqno_, shard_prefix, db_root_ + "/tmp/", + td::Timestamp::in(3600.0), + [SelfId = actor_id(this), seqno = start_import_seqno_, shard_prefix](td::Result R) { + if (R.is_error()) { + LOG(WARNING) << "Failed to download archive slice #" << seqno << " for shard " << shard_prefix.to_str(); + delay_action( + [=]() { td::actor::send_closure(SelfId, &ArchiveImporter::download_shard_archive, shard_prefix); }, + td::Timestamp::in(2.0)); + } else { + LOG(DEBUG) << "Downloaded shard archive #" << seqno << " " << shard_prefix.to_str(); + td::actor::send_closure(SelfId, &ArchiveImporter::downloaded_shard_archive, R.move_as_ok()); + } + }); +} + +void ArchiveImporter::downloaded_shard_archive(std::string path) { + td::Status S = process_package(path, false); + if (S.is_error()) { + LOG(INFO) << "Error processing package: " << S; + } + --pending_shard_archives_; + if (pending_shard_archives_ == 0) { + check_next_shard_client_seqno(shard_client_seqno_ + 1); + } } void ArchiveImporter::check_next_shard_client_seqno(BlockSeqno seqno) { - if (seqno > state_->get_seqno()) { + if (seqno > last_masterchain_state_->get_seqno() || seqno > last_masterchain_seqno_) { finish_query(); - } else if (seqno == state_->get_seqno()) { - got_masterchain_state(state_); + } else if (seqno == last_masterchain_state_->get_seqno()) { + got_masterchain_state(last_masterchain_state_); } else { BlockIdExt b; - bool f = state_->get_old_mc_block_id(seqno, b); + bool f = last_masterchain_state_->get_old_mc_block_id(seqno, b); CHECK(f); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); @@ -241,33 +367,38 @@ void ArchiveImporter::check_next_shard_client_seqno(BlockSeqno seqno) { } void ArchiveImporter::got_masterchain_state(td::Ref state) { + if (state->get_seqno() != start_import_seqno_ && state->is_key_state()) { + finish_query(); + return; + } + LOG(DEBUG) << "Applying shard client seqno " << state->get_seqno(); auto s = state->get_shards(); - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), seqno = state->get_seqno()](td::Result R) { + td::MultiPromise mp; + auto ig = mp.init_guard(); + for (auto &shard : s) { + if (opts_->need_monitor(shard->shard(), state)) { + apply_shard_block(shard->top_block_id(), state->get_block_id(), ig.get_promise()); + } + } + ig.add_promise([SelfId = actor_id(this), seqno = state->get_seqno()](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &ArchiveImporter::abort_query, R.move_as_error()); } else { td::actor::send_closure(SelfId, &ArchiveImporter::checked_shard_client_seqno, seqno); } }); - - td::MultiPromise mp; - auto ig = mp.init_guard(); - ig.add_promise(std::move(P)); - - for (auto &shard : s) { - apply_shard_block(shard->top_block_id(), state->get_block_id(), ig.get_promise()); - } } void ArchiveImporter::checked_shard_client_seqno(BlockSeqno seqno) { CHECK(shard_client_seqno_ + 1 == seqno); shard_client_seqno_++; + imported_any_ = true; check_next_shard_client_seqno(seqno + 1); } void ArchiveImporter::apply_shard_block(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) { + LOG(DEBUG) << "Applying shard block " << block_id.id.to_str(); auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), masterchain_block_id, promise = std::move(promise)](td::Result R) mutable { R.ensure(); @@ -286,7 +417,7 @@ void ArchiveImporter::apply_shard_block_cont1(BlockHandle handle, BlockIdExt mas if (handle->id().seqno() == 0) { auto P = td::PromiseCreator::lambda( - [promise = std::move(promise)](td::Result> R) mutable { promise.set_value(td::Unit()); }); + [promise = std::move(promise)](td::Result>) mutable { promise.set_value(td::Unit()); }); td::actor::create_actor("downloadstate", handle->id(), masterchain_block_id, 2, manager_, td::Timestamp::in(3600), std::move(P)) .release(); @@ -294,12 +425,13 @@ void ArchiveImporter::apply_shard_block_cont1(BlockHandle handle, BlockIdExt mas } auto it = blocks_.find(handle->id()); - if (it == blocks_.end()) { - promise.set_error(td::Status::Error(ErrorCode::notready, PSTRING() << "no proof for shard block " << handle->id())); + if (it == blocks_.end() || !it->second.proof_pkg || !it->second.data_pkg) { + promise.set_error( + td::Status::Error(ErrorCode::notready, PSTRING() << "no data/proof for shard block " << handle->id())); return; } - TRY_RESULT_PROMISE(promise, data, package_->read(it->second[0])); - TRY_RESULT_PROMISE(promise, proof, create_proof_link(handle->id(), std::move(data.second))); + TRY_RESULT_PROMISE(promise, proof_data, it->second.proof_pkg->read(it->second.proof_offset)); + TRY_RESULT_PROMISE(promise, proof, create_proof_link(handle->id(), std::move(proof_data.second))); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle, masterchain_block_id, promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -345,8 +477,8 @@ void ArchiveImporter::apply_shard_block_cont2(BlockHandle handle, BlockIdExt mas void ArchiveImporter::apply_shard_block_cont3(BlockHandle handle, BlockIdExt masterchain_block_id, td::Promise promise) { auto it = blocks_.find(handle->id()); - CHECK(it != blocks_.end()); - TRY_RESULT_PROMISE(promise, data, package_->read(it->second[1])); + CHECK(it != blocks_.end() && it->second.data_pkg); + TRY_RESULT_PROMISE(promise, data, it->second.data_pkg->read(it->second.data_offset)); if (sha256_bits256(data.second.as_slice()) != handle->id().file_hash) { promise.set_error(td::Status::Error(ErrorCode::protoviolation, "bad block file hash")); return; @@ -367,6 +499,7 @@ void ArchiveImporter::check_shard_block_applied(BlockIdExt block_id, td::Promise if (!handle->is_applied()) { promise.set_error(td::Status::Error(ErrorCode::notready, "not applied")); } else { + LOG(DEBUG) << "Applied shard block " << handle->id().id.to_str(); promise.set_value(td::Unit()); } } @@ -375,13 +508,24 @@ void ArchiveImporter::check_shard_block_applied(BlockIdExt block_id, td::Promise } void ArchiveImporter::abort_query(td::Status error) { - LOG(INFO) << error; + if (!imported_any_) { + for (const std::string &f : files_to_cleanup_) { + td::unlink(f).ignore(); + } + promise_.set_error(std::move(error)); + return; + } + LOG(INFO) << "Archive import: " << error; finish_query(); } + void ArchiveImporter::finish_query() { + for (const std::string &f : files_to_cleanup_) { + td::unlink(f).ignore(); + } if (promise_) { - promise_.set_value( - std::vector{state_->get_seqno(), std::min(state_->get_seqno(), shard_client_seqno_)}); + promise_.set_value({last_masterchain_state_->get_seqno(), + std::min(last_masterchain_state_->get_seqno(), shard_client_seqno_)}); } stop(); } diff --git a/validator/import-db-slice.hpp b/validator/import-db-slice.hpp index 0993d4bba7..04f22642dc 100644 --- a/validator/import-db-slice.hpp +++ b/validator/import-db-slice.hpp @@ -19,6 +19,7 @@ #pragma once #include "td/actor/actor.h" +#include "td/utils/port/path.h" #include "validator/interfaces/validator-manager.h" #include "validator/db/package.hpp" @@ -28,19 +29,27 @@ namespace validator { class ArchiveImporter : public td::actor::Actor { public: - ArchiveImporter(std::string path, td::Ref state, BlockSeqno shard_client_seqno, + ArchiveImporter(std::string db_root, td::Ref state, BlockSeqno shard_client_seqno, td::Ref opts, td::actor::ActorId manager, - td::Promise> promise); + std::vector to_import_files, td::Promise> promise); void start_up() override; void abort_query(td::Status error); void finish_query(); + void downloaded_mc_archive(std::string path); + td::Status process_package(std::string path, bool with_masterchain); + + void processed_mc_archive(); void check_masterchain_block(BlockSeqno seqno); void checked_masterchain_proof(BlockHandle handle, td::Ref data); void applied_masterchain_block(BlockHandle handle); void got_new_materchain_state(td::Ref state); - void checked_all_masterchain_blocks(BlockSeqno seqno); + + void checked_all_masterchain_blocks(); + void download_shard_archives(td::Ref start_state); + void download_shard_archive(ShardIdFull shard_prefix); + void downloaded_shard_archive(std::string path); void check_next_shard_client_seqno(BlockSeqno seqno); void checked_shard_client_seqno(BlockSeqno seqno); @@ -52,19 +61,36 @@ class ArchiveImporter : public td::actor::Actor { void check_shard_block_applied(BlockIdExt block_id, td::Promise promise); private: - std::string path_; - td::Ref state_; + std::string db_root_; + td::Ref last_masterchain_state_; BlockSeqno shard_client_seqno_; + BlockSeqno start_import_seqno_; td::Ref opts_; - std::shared_ptr package_; - td::actor::ActorId manager_; - td::Promise> promise_; + + std::vector to_import_files_; + bool use_imported_files_; + td::Promise> promise_; std::map masterchain_blocks_; - std::map> blocks_; + BlockSeqno last_masterchain_seqno_ = 0; + + struct BlockInfo { + std::shared_ptr data_pkg; + td::uint64 data_offset = 0; + std::shared_ptr proof_pkg; + td::uint64 proof_offset = 0; + }; + std::map blocks_; + + td::Ref start_state_; + size_t pending_shard_archives_ = 0; + + bool imported_any_ = false; + bool have_shard_blocks_ = false; + std::vector files_to_cleanup_; }; } // namespace validator diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 66c1f07612..2d8566462d 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -114,7 +114,8 @@ class Db : public td::actor::Actor { virtual void check_key_block_proof_exists(BlockIdExt block_id, td::Promise promise) = 0; virtual void check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise promise) = 0; - virtual void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) = 0; + virtual void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) = 0; virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) = 0; virtual void set_async_mode(bool mode, td::Promise promise) = 0; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 5e567eeeb8..3b44e894fe 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -135,6 +135,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) = 0; + virtual void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) = 0; virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 695ac566f0..509e743d1e 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -265,6 +265,10 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise>> promise) override { UNREACHABLE(); } + void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { + UNREACHABLE(); + } void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; @@ -285,7 +289,8 @@ class ValidatorManagerImpl : public ValidatorManager { promise.set_error(td::Status::Error(ErrorCode::error, "download disabled")); } - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override { + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) override { UNREACHABLE(); } void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index f85b2f641c..c97ad1c8ad 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -20,6 +20,7 @@ #include "interfaces/validator-manager.h" #include "interfaces/db.h" +#include "ton/ton-types.h" #include "validator-group.hpp" #include "manager-init.h" #include "manager-hardfork.h" @@ -332,6 +333,10 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise>> promise) override { UNREACHABLE(); } + void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { + UNREACHABLE(); + } void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override { UNREACHABLE(); @@ -357,7 +362,8 @@ class ValidatorManagerImpl : public ValidatorManager { promise.set_error(td::Status::Error(ErrorCode::error, "download disabled")); } - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override { + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) override { UNREACHABLE(); } void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, diff --git a/validator/manager.cpp b/validator/manager.cpp index d79267eaf8..aa46e9e802 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1729,6 +1729,12 @@ void ValidatorManagerImpl::send_get_out_msg_queue_proof_request( std::move(promise)); } +void ValidatorManagerImpl::send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, + std::string tmp_dir, td::Timestamp timeout, + td::Promise promise) { + callback_->download_archive(mc_seqno, shard_prefix, std::move(tmp_dir), timeout, std::move(promise)); +} + void ValidatorManagerImpl::start_up() { db_ = create_db_actor(actor_id(this), db_root_, opts_); lite_server_cache_ = create_liteserver_cache_actor(actor_id(this), db_root_); @@ -1763,7 +1769,6 @@ void ValidatorManagerImpl::start_up() { if (fname.substr(fname.size() - 5) != ".pack") { return; } - fname = fname.substr(0, fname.size() - 5); if (fname.substr(0, 8) != "archive.") { return; } @@ -1772,13 +1777,18 @@ void ValidatorManagerImpl::start_up() { while (fname.size() > 1 && fname[0] == '0') { fname.remove_prefix(1); } + auto i = fname.find('.'); + if (i == td::Slice::npos) { + return; + } + fname = fname.substr(0, i); auto v = td::to_integer_safe(fname); if (v.is_error()) { return; } - auto pos = v.move_as_ok(); - LOG(INFO) << "found archive slice '" << cfname << "' for position " << pos; - to_import_[pos] = std::make_pair(cfname.str(), true); + auto seqno = v.move_as_ok(); + LOG(INFO) << "found archive slice '" << cfname << "' for seqno " << seqno; + to_import_[seqno].push_back(cfname.str()); } }); if (S.is_error()) { @@ -1926,8 +1936,7 @@ bool ValidatorManagerImpl::out_of_sync() { void ValidatorManagerImpl::prestart_sync() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); - // Don't download archives - td::actor::send_closure(SelfId, &ValidatorManagerImpl::finish_prestart_sync); + td::actor::send_closure(SelfId, &ValidatorManagerImpl::download_next_archive); }); td::actor::send_closure(db_, &Db::set_async_mode, false, std::move(P)); } @@ -1939,61 +1948,35 @@ void ValidatorManagerImpl::download_next_archive() { } auto seqno = std::min(last_masterchain_seqno_, shard_client_handle_->id().seqno()); + std::vector to_import_files; auto it = to_import_.upper_bound(seqno + 1); if (it != to_import_.begin()) { - it--; - if (it->second.second) { - it->second.second = false; - downloaded_archive_slice(it->second.first, false); - return; - } + --it; + to_import_files = std::move(it->second); + it->second.clear(); } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - LOG(INFO) << "failed to download archive slice: " << R.error(); - delay_action([SelfId]() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::download_next_archive); }, - td::Timestamp::in(2.0)); - } else { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::downloaded_archive_slice, R.move_as_ok(), true); - } - }); - callback_->download_archive(seqno + 1, db_root_ + "/tmp/", td::Timestamp::in(36000.0), std::move(P)); -} - -void ValidatorManagerImpl::downloaded_archive_slice(std::string name, bool is_tmp) { - LOG(INFO) << "downloaded archive slice: " << name; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), name, is_tmp](td::Result> R) { - if (is_tmp) { - td::unlink(name).ensure(); - } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { - LOG(INFO) << "failed to check downloaded archive slice: " << R.error(); + LOG(INFO) << "failed to download and import archive slice: " << R.error(); delay_action([SelfId]() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::download_next_archive); }, td::Timestamp::in(2.0)); } else { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::checked_archive_slice, R.move_as_ok()); + td::actor::send_closure(SelfId, &ValidatorManagerImpl::checked_archive_slice, R.ok().first, R.ok().second); } }); - - auto seqno = std::min(last_masterchain_seqno_, shard_client_handle_->id().seqno()); - - td::actor::create_actor("archiveimport", name, last_masterchain_state_, seqno, opts_, actor_id(this), - std::move(P)) + td::actor::create_actor("archiveimport", db_root_, last_masterchain_state_, seqno, opts_, + actor_id(this), std::move(to_import_files), std::move(P)) .release(); } -void ValidatorManagerImpl::checked_archive_slice(std::vector seqno) { - CHECK(seqno.size() == 2); - LOG(INFO) << "checked downloaded archive slice: mc_top_seqno=" << seqno[0] << " shard_top_seqno_=" << seqno[1]; - CHECK(seqno[0] <= last_masterchain_seqno_); - CHECK(seqno[1] <= last_masterchain_seqno_); +void ValidatorManagerImpl::checked_archive_slice(BlockSeqno new_last_mc_seqno, BlockSeqno new_shard_client_seqno) { + LOG(INFO) << "checked downloaded archive slice: mc_top_seqno=" << new_last_mc_seqno + << " shard_top_seqno_=" << new_shard_client_seqno; + CHECK(new_last_mc_seqno <= last_masterchain_seqno_); + CHECK(new_shard_client_seqno <= last_masterchain_seqno_); - BlockIdExt b; - if (seqno[1] < last_masterchain_seqno_) { - CHECK(last_masterchain_state_->get_old_mc_block_id(seqno[1], b)); - } else { - b = last_masterchain_block_id_; - } + BlockIdExt shard_client_block_id; + CHECK(last_masterchain_state_->get_old_mc_block_id(new_shard_client_seqno, shard_client_block_id)); auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), db = db_.get(), client = shard_client_.get()](td::Result R) { @@ -2009,7 +1992,7 @@ void ValidatorManagerImpl::checked_archive_slice(std::vector seqno) }); td::actor::send_closure(db, &Db::get_block_state, std::move(handle), std::move(P)); }); - get_block_handle(b, true, std::move(P)); + get_block_handle(shard_client_block_id, true, std::move(P)); } void ValidatorManagerImpl::finish_prestart_sync() { @@ -2844,12 +2827,13 @@ void ValidatorManagerImpl::try_get_static_file(FileHash file_hash, td::Promise promise) { +void ValidatorManagerImpl::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) { if (masterchain_seqno > last_masterchain_seqno_) { promise.set_error(td::Status::Error(ErrorCode::notready, "masterchain seqno too big")); return; } - td::actor::send_closure(db_, &Db::get_archive_id, masterchain_seqno, std::move(promise)); + td::actor::send_closure(db_, &Db::get_archive_id, masterchain_seqno, shard_prefix, std::move(promise)); } void ValidatorManagerImpl::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, diff --git a/validator/manager.hpp b/validator/manager.hpp index 37b447cc7b..dc18e6278b 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -337,8 +337,7 @@ class ValidatorManagerImpl : public ValidatorManager { void applied_hardfork(); void prestart_sync(); void download_next_archive(); - void downloaded_archive_slice(std::string name, bool is_tmp); - void checked_archive_slice(std::vector seqno); + void checked_archive_slice(BlockSeqno new_last_mc_seqno, BlockSeqno new_shard_client_seqno); void finish_prestart_sync(); void completed_prestart_sync(); @@ -516,6 +515,8 @@ class ValidatorManagerImpl : public ValidatorManager { void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) override; + void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override; void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; @@ -532,7 +533,7 @@ class ValidatorManagerImpl : public ValidatorManager { std::move(promise)); } - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override; + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise) override; void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) override; @@ -712,7 +713,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::actor::ActorOwn serializer_; - std::map> to_import_; + std::map> to_import_; private: std::unique_ptr callback_; diff --git a/validator/net/download-archive-slice.cpp b/validator/net/download-archive-slice.cpp index 6235b8b080..c2f8eceac6 100644 --- a/validator/net/download-archive-slice.cpp +++ b/validator/net/download-archive-slice.cpp @@ -20,6 +20,8 @@ #include "td/utils/port/path.h" #include "td/utils/overloaded.h" +#include + namespace ton { namespace validator { @@ -27,12 +29,13 @@ namespace validator { namespace fullnode { DownloadArchiveSlice::DownloadArchiveSlice( - BlockSeqno masterchain_seqno, std::string tmp_dir, adnl::AdnlNodeIdShort local_id, + BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::Timestamp timeout, td::actor::ActorId validator_manager, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId adnl, td::actor::ActorId client, td::Promise promise) : masterchain_seqno_(masterchain_seqno) + , shard_prefix_(shard_prefix) , tmp_dir_(std::move(tmp_dir)) , local_id_(local_id) , overlay_id_(overlay_id) @@ -114,7 +117,13 @@ void DownloadArchiveSlice::got_node_to_download(adnl::AdnlNodeIdShort download_f } }); - auto q = create_serialize_tl_object(masterchain_seqno_); + td::BufferSlice q; + if (shard_prefix_.is_masterchain()) { + q = create_serialize_tl_object(masterchain_seqno_); + } else { + q = create_serialize_tl_object(masterchain_seqno_, + create_tl_shard_id(shard_prefix_)); + } if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query, download_from_, local_id_, overlay_id_, "get_archive_info", std::move(P), td::Timestamp::in(3.0), std::move(q)); @@ -145,7 +154,8 @@ void DownloadArchiveSlice::got_archive_info(td::BufferSlice data) { } prev_logged_timer_ = td::Timer(); - LOG(INFO) << "downloading archive slice #" << masterchain_seqno_ << " from " << download_from_; + LOG(INFO) << "downloading archive slice #" << masterchain_seqno_ << " " << shard_prefix_.to_str() << " from " + << download_from_; get_archive_slice(); } @@ -186,13 +196,15 @@ void DownloadArchiveSlice::got_archive_slice(td::BufferSlice data) { double elapsed = prev_logged_timer_.elapsed(); if (elapsed > 10.0) { prev_logged_timer_ = td::Timer(); - LOG(INFO) << "downloading archive slice #" << masterchain_seqno_ << ": total=" << offset_ << " (" + LOG(INFO) << "downloading archive slice #" << masterchain_seqno_ << " " << shard_prefix_.to_str() + << ": total=" << offset_ << " (" << td::format::as_size((td::uint64)(double(offset_ - prev_logged_sum_) / elapsed)) << "/s)"; prev_logged_sum_ = offset_; } if (data.size() < slice_size()) { - LOG(INFO) << "finished downloading arcrive slice #" << masterchain_seqno_ << ": total=" << offset_; + LOG(INFO) << "finished downloading arcrive slice #" << masterchain_seqno_ << " " << shard_prefix_.to_str() + << ": total=" << offset_; finish_query(); } else { get_archive_slice(); diff --git a/validator/net/download-archive-slice.hpp b/validator/net/download-archive-slice.hpp index 0384ac8c90..42fd715f77 100644 --- a/validator/net/download-archive-slice.hpp +++ b/validator/net/download-archive-slice.hpp @@ -32,8 +32,9 @@ namespace fullnode { class DownloadArchiveSlice : public td::actor::Actor { public: - DownloadArchiveSlice(BlockSeqno masterchain_seqno, std::string tmp_dir, adnl::AdnlNodeIdShort local_id, - overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::Timestamp timeout, + DownloadArchiveSlice(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, + adnl::AdnlNodeIdShort download_from, td::Timestamp timeout, td::actor::ActorId validator_manager, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId adnl, @@ -55,6 +56,7 @@ class DownloadArchiveSlice : public td::actor::Actor { private: BlockSeqno masterchain_seqno_; + ShardIdFull shard_prefix_; std::string tmp_dir_; std::string tmp_name_; td::FileFd fd_; diff --git a/validator/validator.h b/validator/validator.h index afd884acc8..c7ef2ce874 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -160,8 +160,8 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise promise) = 0; virtual void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) = 0; - virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) = 0; + virtual void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) = 0; virtual void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise>> promise) = 0; @@ -253,7 +253,8 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Timestamp timeout, td::Promise>> promise) = 0; - virtual void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) = 0; + virtual void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) = 0; virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) = 0; From 38ab70c037a49faf2db8b5a874c26163ce90cbc5 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 11 Jun 2024 11:29:52 +0300 Subject: [PATCH 093/524] Changes in validation * Configure collator list in validator-console * Remove "lite-validator" flags * Better compatibility in validate-query.cpp --- tl/generate/scheme/ton_api.tl | 9 ++ tl/generate/scheme/ton_api.tlo | Bin 96928 -> 97828 bytes .../validator-engine-console-query.cpp | 77 +++++++++++++++ .../validator-engine-console-query.h | 60 ++++++++++++ .../validator-engine-console.cpp | 3 + validator-engine/validator-engine.cpp | 87 ++++++++++++++--- validator-engine/validator-engine.hpp | 14 ++- validator/fabric.h | 2 +- validator/impl/validate-query.cpp | 20 ++-- validator/manager.cpp | 48 ++++------ validator/manager.hpp | 3 - validator/validator-group.cpp | 88 +++++++++++------- validator/validator-group.hpp | 16 +++- validator/validator-options.cpp | 16 ++++ validator/validator-options.hpp | 10 +- validator/validator.h | 22 ++++- 16 files changed, 368 insertions(+), 107 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index a4956780c3..0d7f1249eb 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -626,6 +626,12 @@ engine.validator.customOverlayNode adnl_id:int256 msg_sender:Bool msg_sender_pri engine.validator.customOverlay name:string nodes:(vector engine.validator.customOverlayNode) = engine.validator.CustomOverlay; engine.validator.customOverlaysConfig overlays:(vector engine.validator.customOverlay) = engine.validator.CustomOverlaysConfig; +engine.validator.collatorsList.collator adnl_id:int256 trusted:Bool = engine.validator.collatorsList.Collator; +engine.validator.collatorsList.shard shard_id:tonNode.shardId collators:(vector engine.validator.collatorsList.collator) + = engine.validator.collatorsList.Shard; +engine.validator.collatorsList self_collate:Bool use_config_41:Bool shards:(vector engine.validator.collatorsList.shard) + = engine.validator.CollatorsList; + ---functions--- ---types--- @@ -752,6 +758,9 @@ engine.validator.addShard shard:tonNode.shardId = engine.validator.Success; engine.validator.delCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; engine.validator.delShard shard:tonNode.shardId = engine.validator.Success; +engine.validator.setCollatorsList list:engine.validator.collatorsList = engine.validator.Success; +engine.validator.showCollatorsList = engine.validator.CollatorsList; + ---types--- storage.pong = storage.Pong; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 9ffd9fd82766d1736b79d0a245f2275660325268..0d286bdc091a5b25e2a8deab3019aa84e0e73fe5 100644 GIT binary patch delta 587 zcmZ4Rm37G0TI4~`xT^i`UOiy9S-)A zqSE4$)Rf5umNKCrVOWS>mE(#bCLoGI0RhwmQVa4H6VzKAV2Sw5l<5ac7{w>eXPIm< zS3?411G*PLIyZN0zEjWGu|3t8v0aw6=S6X(;`EOJj3UC$$g#t~(7o)*=IM9B7-d*M t9-l5?#i+3TO#ov8Bdci80ez+EfkBL7swi4u!42{ka-eN@2x3go005-h!+`(* delta 71 zcmZ4ThjqbMR^CUm^{p77KyoARf04}|qA%Dt?~pZ-W?Z=Wt;PG~%`4{Ch;L5VcA=iJ XZM&fvW4r8jl@P`R#_bnE8ACJxv3MGm diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 660915bb78..3d22ff8146 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1316,3 +1316,80 @@ td::Status DelShardQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "successfully removed shard\n"; return td::Status::OK(); } + +td::Status SetCollatorsListQuery::run() { + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status SetCollatorsListQuery::send() { + TRY_RESULT(data, td::read_file(file_name_)); + TRY_RESULT(json, td::json_decode(data.as_slice())); + auto list = ton::create_tl_object(); + TRY_STATUS(ton::ton_api::from_json(*list, json.get_object())); + auto b = ton::create_serialize_tl_object(std::move(list)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SetCollatorsListQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status ClearCollatorsListQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status ClearCollatorsListQuery::send() { + auto list = ton::create_tl_object(); + list->self_collate_ = true; + auto b = ton::create_serialize_tl_object(std::move(list)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ClearCollatorsListQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status ShowCollatorsListQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status ShowCollatorsListQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ShowCollatorsListQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(list, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "Collators list:\n"; + if (list->self_collate_) { + td::TerminalIO::out() << "self_collate = true\n"; + } + if (list->use_config_41_) { + td::TerminalIO::out() << "use_config_41 = true\n"; + } + if (list->shards_.empty()) { + td::TerminalIO::out() << "Shard list is empty\n"; + return td::Status::OK(); + } + for (const auto &shard : list->shards_) { + td::TerminalIO::out() << "Shard " << create_shard_id(shard->shard_id_).to_str() << "\n"; + for (const auto &collator : shard->collators_) { + td::TerminalIO::out() << " Collator " << collator->adnl_id_ << (collator->trusted_ ? " (trusted)" : "") << "\n"; + } + } + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index ecdfa5e2b4..4d2a50a309 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1342,3 +1342,63 @@ class DelShardQuery : public Query { td::int32 wc_; td::int64 shard_; }; + +class SetCollatorsListQuery : public Query { + public: + SetCollatorsListQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "setcollatorslist"; + } + static std::string get_help() { + return "setcollatorslist \tset list of collators from file "; + } + std::string name() const override { + return get_name(); + } + + private: + std::string file_name_; +}; + +class ClearCollatorsListQuery : public Query { + public: + ClearCollatorsListQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "clearcollatorslist"; + } + static std::string get_help() { + return "clearcollatorslist\tclear list of collators"; + } + std::string name() const override { + return get_name(); + } +}; + +class ShowCollatorsListQuery : public Query { + public: + ShowCollatorsListQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "showcollatorslist"; + } + static std::string get_help() { + return "showcollatorslist\tshow list of collators"; + } + std::string name() const override { + return get_name(); + } +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index cd4c7c36b1..e9c26484d9 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -152,6 +152,9 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 585c4c74e8..4dd25030a0 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1472,7 +1472,6 @@ td::Status ValidatorEngine::load_global_config() { } validator_options_.write().set_hardforks(std::move(h)); validator_options_.write().set_state_serializer_enabled(config_.state_serializer_enabled); - validator_options_.write().set_validator_mode(validator_mode_); return td::Status::OK(); } @@ -1502,6 +1501,31 @@ void ValidatorEngine::set_shard_check_function() { } } +void ValidatorEngine::load_collators_list() { + collators_list_ = {}; + auto data_R = td::read_file(collators_list_file()); + if (data_R.is_error()) { + return; + } + auto data = data_R.move_as_ok(); + auto json_R = td::json_decode(data.as_slice()); + if (json_R.is_error()) { + LOG(ERROR) << "Failed to parse collators list: " << json_R.move_as_error(); + return; + } + auto json = json_R.move_as_ok(); + collators_list_ = ton::create_tl_object(); + auto S = ton::ton_api::from_json(*collators_list_, json.get_object()); + if (S.is_error()) { + LOG(ERROR) << "Failed to parse collators list: " << S; + collators_list_ = {}; + return; + } + td::Ref list{true}; + list.write().unpack(*collators_list_); + validator_options_.write().set_collators_list(std::move(list)); +} + void ValidatorEngine::load_empty_local_config(td::Promise promise) { auto ret_promise = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { @@ -1794,6 +1818,7 @@ void ValidatorEngine::got_key(ton::PublicKey key) { void ValidatorEngine::start() { set_shard_check_function(); + load_collators_list(); read_config_ = true; start_adnl(); } @@ -3792,6 +3817,52 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setStateS }); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setCollatorsList &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto s = td::json_encode(td::ToJson(*query.list_), true); + auto S = td::write_file(collators_list_file(), s); + if (S.is_error()) { + promise.set_value(create_control_query_error(std::move(S))); + return; + } + + collators_list_ = std::move(query.list_); + td::Ref list{true}; + list.write().unpack(*collators_list_); + validator_options_.write().set_collators_list(std::move(list)); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCollatorsList &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + if (collators_list_) { + promise.set_value(ton::serialize_tl_object(collators_list_, true)); + } else { + auto list = ton::create_tl_object(); + list->self_collate_ = true; + promise.set_value(ton::serialize_tl_object(list, true)); + } +} + void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getValidatorSessionsInfo &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { @@ -4260,20 +4331,6 @@ int main(int argc, char *argv[]) { p.add_option('M', "not-all-shards", "monitor only a necessary set of shards instead of all", [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_not_all_shards); }); }); - p.add_option('\0', "lite-validator-shards", - "lite-mode validator for shard blocks (don't collate blocks, use collator nodes instead)", [&]() { - acts.push_back([&x]() { - td::actor::send_closure(x, &ValidatorEngine::set_validator_mode, - ton::validator::ValidatorManagerOptions::validator_lite_shards); - }); - }); - p.add_option('\0', "lite-validator-all", - "lite-mode validator for all blocks (don't collate blocks, use collator nodes instead)", [&]() { - acts.push_back([&x]() { - td::actor::send_closure(x, &ValidatorEngine::set_validator_mode, - ton::validator::ValidatorManagerOptions::validator_lite_all); - }); - }); td::uint32 threads = 7; p.add_checked_option( 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index ebf8fb5812..64154718ab 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -184,6 +184,7 @@ class ValidatorEngine : public td::actor::Actor { td::Ref validator_options_; Config config_; ton::tl_object_ptr custom_overlays_config_; + ton::tl_object_ptr collators_list_; std::set running_gc_; @@ -235,8 +236,6 @@ class ValidatorEngine : public td::actor::Actor { ton::BlockSeqno truncate_seqno_{0}; std::string session_logs_file_; bool not_all_shards_ = false; - ton::validator::ValidatorManagerOptions::ValidatorMode validator_mode_ = - ton::validator::ValidatorManagerOptions::validator_normal; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -319,9 +318,6 @@ class ValidatorEngine : public td::actor::Actor { void set_not_all_shards() { not_all_shards_ = true; } - void set_validator_mode(ton::validator::ValidatorManagerOptions::ValidatorMode mode) { - validator_mode_ = mode; - } void start_up() override; ValidatorEngine() { @@ -333,6 +329,7 @@ class ValidatorEngine : public td::actor::Actor { void load_local_config(td::Promise promise); void load_config(td::Promise promise); void set_shard_check_function(); + void load_collators_list(); void start(); @@ -411,6 +408,9 @@ class ValidatorEngine : public td::actor::Actor { std::string custom_overlays_config_file() const { return db_root_ + "/custom-overlays.json"; } + std::string collators_list_file() const { + return db_root_ + "/collators-list.json"; + } void load_custom_overlays_config(); td::Status write_custom_overlays_config(); @@ -514,6 +514,10 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_setCollatorsList &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_showCollatorsList &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/fabric.h b/validator/fabric.h index e809be43fa..a38b405688 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -26,7 +26,7 @@ namespace ton { namespace validator { -enum ValidateMode { fake = 1, full_collated_data = 2 }; +enum ValidateMode { fake = 1 }; enum CollateMode { skip_store_candidate = 1 }; td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_, diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index c0c4a2e836..85ced27d71 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -82,7 +82,6 @@ ValidateQuery::ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block , timeout(timeout) , main_promise(std::move(promise)) , is_fake_(mode & ValidateMode::fake) - , full_collated_data_(mode & ValidateMode::full_collated_data) , shard_pfx_(shard_.shard) , shard_pfx_len_(ton::shard_prefix_length(shard_)) , perf_timer_("validateblock", 0.1, [manager](double duration) { @@ -258,7 +257,6 @@ void ValidateQuery::finish_query() { */ void ValidateQuery::start_up() { LOG(WARNING) << "validate query for " << block_candidate.id.to_str() << " started"; - LOG(DEBUG) << "full_collated_data is " << full_collated_data_; alarm_timestamp() = timeout; rand_seed_.set_zero(); created_by_ = block_candidate.pubkey; @@ -359,11 +357,16 @@ void ValidateQuery::start_up() { td::actor::send_closure_later( std::move(self), &ValidateQuery::after_get_latest_mc_state, std::move(res)); }); - // 3. load state(s) corresponding to previous block(s) (not full-collaoted-data or masterchain) + // 3. unpack block candidate (while necessary data is being loaded) + if (!unpack_block_candidate()) { + reject_query("error unpacking block candidate"); + return; + } + // 4. load state(s) corresponding to previous block(s) (not full-collated-data or masterchain) prev_states.resize(prev_blocks.size()); if (is_masterchain() || !full_collated_data_) { for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { - // 3.1. load state + // 4.1. load state LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; ++pending; td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), @@ -374,11 +377,6 @@ void ValidateQuery::start_up() { }); } } - // 4. unpack block candidate (while necessary data is being loaded) - if (!unpack_block_candidate()) { - reject_query("error unpacking block candidate"); - return; - } // 5. request masterchain state referred to in the block if (!is_masterchain()) { ++pending; @@ -629,6 +627,7 @@ bool ValidateQuery::extract_collated_data_from(Ref croot, int idx) { if (!ins.second) { return reject_query("Merkle proof with duplicate virtual root hash "s + virt_hash.to_hex()); } + full_collated_data_ = true; return true; } if (block::gen::t_TopBlockDescrSet.has_valid_tag(cs)) { @@ -666,6 +665,9 @@ bool ValidateQuery::extract_collated_data() { return reject_query(PSTRING() << "virtualization error " << err.get_msg()); } } + if (full_collated_data_) { + LOG(INFO) << "full_collated_data = true"; + } return true; } diff --git a/validator/manager.cpp b/validator/manager.cpp index aa46e9e802..bde7a8056a 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -373,7 +373,8 @@ void ValidatorManagerImpl::get_key_block_proof_link(BlockIdExt block_id, td::Pro } void ValidatorManagerImpl::new_external_message(td::BufferSlice data, int priority) { - if (!is_collator()) { + if (!validating_masterchain() && collator_nodes_.empty() && + (!is_validator() || !opts_->get_collators_list()->self_collate)) { return; } if (last_masterchain_state_.is_null()) { @@ -453,7 +454,7 @@ void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Prom } void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { - if (!is_collator()) { + if (collator_nodes_.empty() && (!is_validator() || !opts_->get_collators_list()->self_collate)) { return; } auto R = create_ihr_message(std::move(data)); @@ -470,7 +471,7 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - if (!is_collator() && !is_validator()) { + if (!is_validator()) { return; } if (!last_masterchain_block_handle_) { @@ -533,7 +534,7 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id(), 0, td::Timestamp::in(60.0), std::move(P)); } - if (collating_masterchain()) { + if (validating_masterchain()) { preload_msg_queue_to_masterchain(desc); } } @@ -542,7 +543,8 @@ void ValidatorManagerImpl::add_shard_block_description(td::Ref desc) { auto id = ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}; auto it = shard_blocks_.find(id); - if (!collating_masterchain() || it == shard_blocks_.end() || it->second.latest_desc->block_id() != desc->block_id()) { + if (!validating_masterchain() || it == shard_blocks_.end() || + it->second.latest_desc->block_id() != desc->block_id()) { return; } wait_neighbor_msg_queue_proofs( @@ -2031,9 +2033,9 @@ void ValidatorManagerImpl::new_masterchain_block() { td::actor::send_closure(shard_client_, &ShardClient::new_masterchain_block_notification, last_masterchain_block_handle_, last_masterchain_state_); } - if (is_collator()) { + if (validating_masterchain() || !collator_nodes_.empty()) { std::set collating_shards; - if (collating_masterchain()) { + if (validating_masterchain()) { collating_shards.emplace(masterchainId); } for (const auto &collator : collator_nodes_) { @@ -2398,7 +2400,8 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto G = td::actor::create_actor( "validatorgroup", shard, validator_id, session_id, validator_set, last_masterchain_state_->get_collator_config(true), opts, keyring_, adnl_, rldp_, overlays_, db_root_, - actor_id(this), init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_); + actor_id(this), init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_, + opts_->need_monitor(shard, last_masterchain_state_)); return G; } } @@ -2845,27 +2848,12 @@ bool ValidatorManagerImpl::is_validator() { return temp_keys_.size() > 0 || permanent_keys_.size() > 0; } -bool ValidatorManagerImpl::is_collator() { - return !collator_nodes_.empty() || - (opts_->validator_mode() != ValidatorManagerOptions::validator_lite_all && is_validator()); -} - bool ValidatorManagerImpl::validating_masterchain() { return !get_validator(ShardIdFull(masterchainId), last_masterchain_state_->get_validator_set(ShardIdFull(masterchainId))) .is_zero(); } -bool ValidatorManagerImpl::collating_masterchain() { - if (masterchain_collators_) { - return true; - } - if (opts_->validator_mode() == ValidatorManagerOptions::validator_lite_all) { - return false; - } - return validating_masterchain(); -} - PublicKeyHash ValidatorManagerImpl::get_validator(ShardIdFull shard, td::Ref val_set) { for (auto &key : temp_keys_) { if (val_set->is_validator(key.bits256_value())) { @@ -3331,6 +3319,14 @@ void ValidatorManagerImpl::update_options(td::Ref opts) if (!queue_size_counter_.empty()) { td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::update_options, opts); } + for (auto &group : validator_groups_) { + td::actor::send_closure(group.second.actor, &ValidatorGroup::update_options, opts, + opts->need_monitor(group.second.shard, last_masterchain_state_)); + } + for (auto &group : next_validator_groups_) { + td::actor::send_closure(group.second.actor, &ValidatorGroup::update_options, opts, + opts->need_monitor(group.second.shard, last_masterchain_state_)); + } opts_ = std::move(opts); } @@ -3376,9 +3372,6 @@ void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh if (!it->second.shards.insert(shard).second) { return; } - if (shard.is_masterchain()) { - ++masterchain_collators_; - } td::actor::send_closure(it->second.actor, &CollatorNode::add_shard, shard); } @@ -3390,9 +3383,6 @@ void ValidatorManagerImpl::del_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh if (!it->second.shards.erase(shard)) { return; } - if (shard.is_masterchain()) { - --masterchain_collators_; - } if (it->second.shards.empty()) { collator_nodes_.erase(it); } else { diff --git a/validator/manager.hpp b/validator/manager.hpp index dc18e6278b..4e5d1d3e73 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -560,9 +560,7 @@ class ValidatorManagerImpl : public ValidatorManager { void read_gc_list(std::vector list); bool is_validator(); - bool is_collator(); bool validating_masterchain(); - bool collating_masterchain(); PublicKeyHash get_validator(ShardIdFull shard, td::Ref val_set); ValidatorManagerImpl(td::Ref opts, std::string db_root, @@ -767,7 +765,6 @@ class ValidatorManagerImpl : public ValidatorManager { std::set shards; }; std::map collator_nodes_; - size_t masterchain_collators_ = 0; std::set extra_active_shards_; std::map last_validated_blocks_; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index fc1fb00901..e4ae88dd4d 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -57,14 +57,7 @@ void ValidatorGroup::generate_block_candidate( cache = cached_collated_block_](td::Result R) { td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, std::move(cache), std::move(R)); }; - if (opts_->validator_mode() == ValidatorManagerOptions::validator_lite_all || - (opts_->validator_mode() == ValidatorManagerOptions::validator_lite_shards && !shard_.is_masterchain())) { - send_collate_query(round_id, td::Timestamp::in(10.0), std::move(P)); - return; - } - run_collate_query(shard_, min_masterchain_block_id_, prev_block_ids_, - Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_, manager_, - td::Timestamp::in(10.0), std::move(P)); + collate_block(round_id, td::Timestamp::in(10.0), std::move(P)); } void ValidatorGroup::generated_block_candidate(std::shared_ptr cache, @@ -142,8 +135,7 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat } VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id; run_validate_query(shard_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, manager_, - td::Timestamp::in(15.0), std::move(P), - collator_config_.full_collated_data ? ValidateMode::full_collated_data : 0); + td::Timestamp::in(15.0), std::move(P)); } void ValidatorGroup::update_approve_cache(CacheKey key, UnixTime value) { @@ -204,10 +196,8 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref } }); - run_accept_block_query( - block_id, std::move(block), std::move(prev), validator_set_, std::move(sig_set), std::move(approve_sig_set), - send_broadcast, shard_.is_masterchain() || opts_->validator_mode() == ValidatorManagerOptions::validator_normal, - manager_, std::move(P)); + run_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, std::move(sig_set), + std::move(approve_sig_set), send_broadcast, apply_blocks_, manager_, std::move(P)); } void ValidatorGroup::skip_round(td::uint32 round_id) { @@ -480,25 +470,55 @@ void ValidatorGroup::get_session_info( td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_session_info, std::move(P)); } -void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeout, td::Promise promise, - unsigned max_retries) { +void ValidatorGroup::collate_block(td::uint32 round_id, td::Timestamp timeout, td::Promise promise, + unsigned max_retries) { if (round_id < last_known_round_id_) { promise.set_error(td::Status::Error("too old")); return; } BlockId next_block_id = create_next_block_id_simple(); - adnl::AdnlNodeIdShort collator = adnl::AdnlNodeIdShort::zero(); - // TODO: some way to choose node (similar to "unreliability" in full-node) - int cnt = 0; - for (const block::CollatorNodeDescr &c : collator_config_.collator_nodes) { - if (shard_intersects(shard_, c.shard)) { - if (td::Random::fast(0, cnt) == 0) { - collator = adnl::AdnlNodeIdShort(c.adnl_id); + adnl::AdnlNodeIdShort collator_adnl_id = adnl::AdnlNodeIdShort::zero(); + bool self_collate = false; + bool trusted_collator = false; + + if (shard_.is_masterchain()) { + self_collate = true; + } else { + for (const auto &s : opts_->get_collators_list()->shards) { + if (!shard_intersects(s.shard_id, shard_)) { + continue; + } + if (!s.collators.empty()) { + const CollatorsList::Collator &col = s.collators[td::Random::fast(0, s.collators.size() - 1)]; + collator_adnl_id = col.adnl_id; + trusted_collator = col.trusted; + break; } - ++cnt; } + if (collator_adnl_id.is_zero()) { + if (opts_->get_collators_list()->self_collate) { + self_collate = true; + } else if (opts_->get_collators_list()->use_config_41) { + // TODO: some way to choose node (similar to "unreliability" in full-node) + int cnt = 0; + for (const block::CollatorNodeDescr &c : collator_config_.collator_nodes) { + if (shard_intersects(shard_, c.shard)) { + if (td::Random::fast(0, cnt) == 0) { + collator_adnl_id = adnl::AdnlNodeIdShort(c.adnl_id); + } + ++cnt; + } + } + } + } + } + if (self_collate) { + run_collate_query(shard_, min_masterchain_block_id_, prev_block_ids_, + Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_, manager_, + td::Timestamp::in(10.0), std::move(promise)); + return; } - if (collator.is_zero()) { + if (collator_adnl_id.is_zero()) { promise.set_error(td::Status::Error(PSTRING() << "no collator for shard " << shard_.to_str())); return; } @@ -514,7 +534,7 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo LOG(WARNING) << "collate query for " << next_block_id.to_str() << ": " << R.error() << ", time=" << timer.elapsed() << "s, " << (retry ? "retrying" : "giving up"); if (retry) { - td::actor::send_closure(SelfId, &ValidatorGroup::send_collate_query, round_id, timeout, std::move(promise), + td::actor::send_closure(SelfId, &ValidatorGroup::collate_block, round_id, timeout, std::move(promise), max_retries - 1); } else { promise.set_result(td::Status::Error(ErrorCode::timeout, "timeout")); @@ -530,23 +550,23 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo local_id_full_.ed25519_value().raw()); auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), round_id, promise = std::move(promise)](td::Result R) mutable { + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error_prefix("rldp query failed: ")); return; } td::actor::send_closure(SelfId, &ValidatorGroup::receive_collate_query_response, round_id, R.move_as_ok(), - std::move(promise)); + trusted_collator, std::move(promise)); }); - LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to " << collator; + LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to " << collator_adnl_id; size_t max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; td::Timestamp query_timeout = td::Timestamp::in(10.0); query_timeout.relax(timeout); - td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_adnl_id_, collator, "collatequery", std::move(P), - timeout, std::move(query), max_answer_size); + td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_adnl_id_, collator_adnl_id, "collatequery", + std::move(P), timeout, std::move(query), max_answer_size); } -void ValidatorGroup::receive_collate_query_response(td::uint32 round_id, td::BufferSlice data, +void ValidatorGroup::receive_collate_query_response(td::uint32 round_id, td::BufferSlice data, bool trusted_collator, td::Promise promise) { if (round_id < last_known_round_id_) { promise.set_error(td::Status::Error("too old")); @@ -574,6 +594,10 @@ void ValidatorGroup::receive_collate_query_response(td::uint32 round_id, td::Buf return; } + if (trusted_collator) { + promise.set_result(std::move(candidate)); + return; + } auto P = td::PromiseCreator::lambda( [candidate = candidate.clone(), promise = std::move(promise)](td::Result> R) mutable { if (R.is_error()) { diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index d4080f6a10..6275734270 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -66,13 +66,18 @@ class ValidatorGroup : public td::actor::Actor { void get_session_info(td::Promise> promise); + void update_options(td::Ref opts, bool apply_blocks) { + opts_ = std::move(opts); + apply_blocks_ = apply_blocks; + } + ValidatorGroup(ShardIdFull shard, PublicKeyHash local_id, ValidatorSessionId session_id, td::Ref validator_set, block::CollatorConfig collator_config, validatorsession::ValidatorSessionOptions config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, td::actor::ActorId validator_manager, bool create_session, - bool allow_unsafe_self_blocks_resync, td::Ref opts) + bool allow_unsafe_self_blocks_resync, td::Ref opts, bool apply_blocks) : shard_(shard) , local_id_(std::move(local_id)) , session_id_(session_id) @@ -87,14 +92,16 @@ class ValidatorGroup : public td::actor::Actor { , manager_(validator_manager) , init_(create_session) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) - , opts_(std::move(opts)) { + , opts_(std::move(opts)) + , apply_blocks_(apply_blocks) { } private: std::unique_ptr make_validator_session_callback(); - void send_collate_query(td::uint32 round_id, td::Timestamp timeout, td::Promise promise, + void collate_block(td::uint32 round_id, td::Timestamp timeout, td::Promise promise, unsigned max_retries = 4); - void receive_collate_query_response(td::uint32 round_id, td::BufferSlice data, td::Promise promise); + void receive_collate_query_response(td::uint32 round_id, td::BufferSlice data, bool trusted_collator, + td::Promise promise); struct PostponedAccept { RootHash root_hash; @@ -134,6 +141,7 @@ class ValidatorGroup : public td::actor::Actor { bool allow_unsafe_self_blocks_resync_; td::Ref opts_; td::uint32 last_known_round_id_ = 0; + bool apply_blocks_ = true; struct CachedCollatedBlock { td::optional result; diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index cb26fe44d6..4f9b8c538a 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -20,10 +20,26 @@ #include "ton/ton-shard.h" +#include + namespace ton { namespace validator { +void CollatorsList::unpack(const ton_api::engine_validator_collatorsList& obj) { + shards.clear(); + self_collate = obj.self_collate_; + use_config_41 = obj.use_config_41_; + for (const auto& shard_obj : obj.shards_) { + shards.emplace_back(); + Shard& shard = shards.back(); + shard.shard_id = create_shard_id(shard_obj->shard_id_); + for (const auto& collator : shard_obj->collators_) { + shard.collators.push_back({adnl::AdnlNodeIdShort{collator->adnl_id_}, collator->trusted_}); + } + } +} + td::Ref ValidatorManagerOptions::create( BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard, bool allow_blockchain_init, diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 55b40a3395..e67c4e2d07 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -142,8 +142,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool get_state_serializer_enabled() const override { return state_serializer_enabled_; } - ValidatorMode validator_mode() const override { - return validator_mode_; + td::Ref get_collators_list() const override { + return collators_list_; } void set_zero_block_id(BlockIdExt block_id) override { @@ -228,8 +228,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_state_serializer_enabled(bool value) override { state_serializer_enabled_ = value; } - void set_validator_mode(ValidatorMode value) override { - validator_mode_ = value; + void set_collators_list(td::Ref list) override { + collators_list_ = std::move(list); } ValidatorManagerOptionsImpl *make_copy() const override { @@ -281,7 +281,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool celldb_preload_all_ = false; td::optional catchain_max_block_delay_; bool state_serializer_enabled_ = true; - ValidatorMode validator_mode_ = validator_normal; + td::Ref collators_list_{true, CollatorsList{}}; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index c7ef2ce874..77b39607db 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -52,10 +52,24 @@ struct PerfTimerStats { std::deque> stats; // }; +struct CollatorsList : public td::CntObject { + struct Collator { + adnl::AdnlNodeIdShort adnl_id; + bool trusted; + }; + struct Shard { + ShardIdFull shard_id; + std::vector collators; + }; + bool self_collate = true; + bool use_config_41 = false; + std::vector shards; + + void unpack(const ton_api::engine_validator_collatorsList& obj); +}; + struct ValidatorManagerOptions : public td::CntObject { public: - enum ValidatorMode { validator_normal, validator_lite_shards, validator_lite_all }; - virtual BlockIdExt zero_block_id() const = 0; virtual BlockIdExt init_block_id() const = 0; virtual bool need_monitor(ShardIdFull shard, const td::Ref& state) const = 0; @@ -91,7 +105,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual bool get_celldb_preload_all() const = 0; virtual td::optional get_catchain_max_block_delay() const = 0; virtual bool get_state_serializer_enabled() const = 0; - virtual ValidatorMode validator_mode() const = 0; + virtual td::Ref get_collators_list() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; @@ -120,7 +134,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_celldb_preload_all(bool value) = 0; virtual void set_catchain_max_block_delay(double value) = 0; virtual void set_state_serializer_enabled(bool value) = 0; - virtual void set_validator_mode(ValidatorMode value) = 0; + virtual void set_collators_list(td::Ref list) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, From 007f1fb1d71e221dc2a5a680bf997f0031a6bc94 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 12 Jun 2024 18:12:45 +0300 Subject: [PATCH 094/524] New liteserver config format * Specify shards and seqno/utime/lt limits for liteservers in global config * Support in lite-client, tonlib, blockchain-explorer * Rework proxy-liteserver --- .../blockchain-explorer-query.cpp | 146 ++-- blockchain-explorer/blockchain-explorer.cpp | 54 +- blockchain-explorer/blockchain-explorer.hpp | 2 +- lite-client/CMakeLists.txt | 6 +- lite-client/QueryTraits.h | 229 ------ lite-client/ext-client.cpp | 151 ++-- lite-client/ext-client.h | 19 +- lite-client/lite-client.cpp | 689 +++++++----------- lite-client/lite-client.h | 34 +- lite-client/query-utils.cpp | 394 ++++++++++ lite-client/query-utils.hpp | 89 +++ tl-utils/common-utils.hpp | 33 + tl/generate/scheme/ton_api.tl | 6 +- tl/generate/scheme/ton_api.tlo | Bin 97828 -> 98440 bytes tl/generate/scheme/tonlib_api.tl | 2 +- tl/generate/scheme/tonlib_api.tlo | Bin 33292 -> 33224 bytes tonlib/tonlib/Config.cpp | 63 +- tonlib/tonlib/Config.h | 3 +- tonlib/tonlib/ExtClient.cpp | 4 +- tonlib/tonlib/ExtClient.h | 8 +- tonlib/tonlib/ExtClientOutbound.cpp | 7 +- tonlib/tonlib/ExtClientOutbound.h | 3 +- tonlib/tonlib/TonlibClient.cpp | 26 +- tonlib/tonlib/TonlibClient.h | 2 +- tonlib/tonlib/tonlib-cli.cpp | 2 +- utils/proxy-liteserver.cpp | 321 ++++---- 26 files changed, 1175 insertions(+), 1118 deletions(-) delete mode 100644 lite-client/QueryTraits.h create mode 100644 lite-client/query-utils.cpp create mode 100644 lite-client/query-utils.hpp diff --git a/blockchain-explorer/blockchain-explorer-query.cpp b/blockchain-explorer/blockchain-explorer-query.cpp index 920ae2c7ff..1808a3c462 100644 --- a/blockchain-explorer/blockchain-explorer-query.cpp +++ b/blockchain-explorer/blockchain-explorer-query.cpp @@ -43,7 +43,6 @@ #include "block/block-auto.h" #include "crypto/vm/utils.h" #include "td/utils/crypto.h" -#include "lite-client/QueryTraits.h" #include "vm/boc.h" #include "vm/cellops.h" @@ -212,7 +211,8 @@ void HttpQueryBlockData::finish_query() { } void HttpQueryBlockData::start_up() { - auto query = ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)); + auto query = ton::serialize_tl_object( + ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)), true); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { @@ -223,7 +223,7 @@ void HttpQueryBlockData::start_up() { }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } void HttpQueryBlockData::got_block_data(td::BufferSlice data) { @@ -274,7 +274,8 @@ void HttpQueryBlockView::finish_query() { } void HttpQueryBlockView::start_up_query() { - auto query = ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)); + auto query = ton::serialize_tl_object( + ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)), true); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { @@ -285,7 +286,7 @@ void HttpQueryBlockView::start_up_query() { }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } void HttpQueryBlockView::got_block_data(td::BufferSlice data) { @@ -321,10 +322,11 @@ void HttpQueryBlockInfo::start_up_query() { td::actor::send_closure(SelfId, &HttpQueryBlockInfo::got_block_header, R.move_as_ok()); } }); - auto query = - ton::create_tl_object(ton::create_tl_lite_block_id(block_id_), 0); + auto query = ton::serialize_tl_object( + ton::create_tl_object(ton::create_tl_lite_block_id(block_id_), 0), + true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); pending_queries_ = 1; if (block_id_.is_masterchain()) { @@ -336,15 +338,16 @@ void HttpQueryBlockInfo::start_up_query() { td::actor::send_closure(SelfId, &HttpQueryBlockInfo::got_shard_info, R.move_as_ok()); } }); - auto query_2 = - ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)); + auto query_2 = ton::serialize_tl_object( + ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)), + true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query_2, true), liteclient::get_query_shard(*query_2), - std::move(P_2)); + std::move(query_2), std::move(P_2)); pending_queries_++; } - auto query_3 = ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), 7, 1024, nullptr, false, false); + auto query_3 = ton::serialize_tl_object(ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), 7, 1024, nullptr, false, false), + true); auto P_3 = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &HttpQueryBlockInfo::abort_query, R.move_as_error_prefix("litequery failed: ")); @@ -353,8 +356,7 @@ void HttpQueryBlockInfo::start_up_query() { } }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query_3, true), liteclient::get_query_shard(*query_3), - std::move(P_3)); + std::move(query_3), std::move(P_3)); pending_queries_++; } @@ -407,9 +409,11 @@ void HttpQueryBlockInfo::got_transactions(td::BufferSlice data) { if (f->incomplete_ && transactions_.size() > 0) { const auto &T = *transactions_.rbegin(); - auto query_3 = ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), 7 + 128, 1024, - ton::create_tl_object(T.addr.addr, T.lt), false, false); + auto query_3 = ton::serialize_tl_object( + ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), 7 + 128, 1024, + ton::create_tl_object(T.addr.addr, T.lt), false, false), + true); auto P_3 = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &HttpQueryBlockInfo::abort_query, R.move_as_error_prefix("litequery failed: ")); @@ -418,8 +422,7 @@ void HttpQueryBlockInfo::got_transactions(td::BufferSlice data) { } }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query_3, true), liteclient::get_query_shard(*query_3), - std::move(P_3)); + std::move(query_3), std::move(P_3)); } else { if (!--pending_queries_) { finish_query(); @@ -539,13 +542,14 @@ void HttpQueryBlockSearch::start_up_query() { td::actor::send_closure(SelfId, &HttpQueryBlockSearch::got_block_header, R.move_as_ok()); } }); - auto query = ton::create_tl_object( - mode_, - ton::create_tl_lite_block_id_simple( - ton::BlockId{account_prefix_.workchain, account_prefix_.account_id_prefix, seqno_}), - lt_, utime_); + auto query = ton::serialize_tl_object(ton::create_tl_object( + mode_, + ton::create_tl_lite_block_id_simple(ton::BlockId{ + account_prefix_.workchain, account_prefix_.account_id_prefix, seqno_}), + lt_, utime_), + true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } void HttpQueryBlockSearch::got_block_header(td::BufferSlice data) { @@ -567,16 +571,17 @@ void HttpQueryBlockSearch::got_block_header(td::BufferSlice data) { td::actor::send_closure(SelfId, &HttpQueryBlockSearch::got_shard_info, R.move_as_ok()); } }); - auto query_2 = - ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)); + auto query_2 = ton::serialize_tl_object( + ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)), + true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query_2, true), liteclient::get_query_shard(*query_2), - std::move(P_2)); + std::move(query_2), std::move(P_2)); pending_queries_++; } - auto query_3 = ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), 7, 1024, nullptr, false, false); + auto query_3 = ton::serialize_tl_object(ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), 7, 1024, nullptr, false, false), + true); auto P_3 = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &HttpQueryBlockSearch::abort_query, R.move_as_error_prefix("litequery failed: ")); @@ -585,8 +590,7 @@ void HttpQueryBlockSearch::got_block_header(td::BufferSlice data) { } }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query_3, true), liteclient::get_query_shard(*query_3), - std::move(P_3)); + std::move(query_3), std::move(P_3)); pending_queries_++; } @@ -626,9 +630,11 @@ void HttpQueryBlockSearch::got_transactions(td::BufferSlice data) { if (f->incomplete_ && transactions_.size() > 0) { const auto &T = *transactions_.rbegin(); - auto query_3 = ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), 7 + 128, 1024, - ton::create_tl_object(T.addr.addr, T.lt), false, false); + auto query_3 = ton::serialize_tl_object( + ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), 7 + 128, 1024, + ton::create_tl_object(T.addr.addr, T.lt), false, false), + true); auto P_3 = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &HttpQueryBlockSearch::abort_query, @@ -638,8 +644,7 @@ void HttpQueryBlockSearch::got_transactions(td::BufferSlice data) { } }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query_3, true), liteclient::get_query_shard(*query_3), - std::move(P_3)); + std::move(query_3), std::move(P_3)); } else { if (!--pending_queries_) { finish_query(); @@ -727,10 +732,11 @@ void HttpQueryViewAccount::start_up_query() { } }); auto a = ton::create_tl_object(addr_.workchain, addr_.addr); - auto query = ton::create_tl_object(ton::create_tl_lite_block_id(block_id_), - std::move(a)); + auto query = ton::serialize_tl_object(ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), std::move(a)), + true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } void HttpQueryViewAccount::got_account(td::BufferSlice data) { @@ -823,9 +829,10 @@ void HttpQueryViewTransaction::start_up_query() { } }); auto a = ton::create_tl_object(addr_.workchain, addr_.addr); - auto query = ton::create_tl_object(1, std::move(a), lt_, hash_); + auto query = ton::serialize_tl_object( + ton::create_tl_object(1, std::move(a), lt_, hash_), true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } void HttpQueryViewTransaction::got_transaction(td::BufferSlice data) { @@ -913,10 +920,11 @@ void HttpQueryViewTransaction2::start_up_query() { } }); auto a = ton::create_tl_object(addr_.workchain, addr_.addr); - auto query = ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), std::move(a), lt_); + auto query = ton::serialize_tl_object(ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), std::move(a), lt_), + true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } void HttpQueryViewTransaction2::got_transaction(td::BufferSlice data) { @@ -975,9 +983,9 @@ void HttpQueryViewLastBlock::start_up() { } }); - auto query = ton::create_tl_object(); + auto query = ton::serialize_tl_object(ton::create_tl_object(), true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } void HttpQueryViewLastBlock::got_result(td::BufferSlice data) { @@ -1041,9 +1049,9 @@ void HttpQueryConfig::start_up() { } }); - auto query = ton::create_tl_object(); + auto query = ton::serialize_tl_object(ton::create_tl_object(), true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } } @@ -1067,17 +1075,16 @@ void HttpQueryConfig::send_main_query() { td::actor::send_closure(SelfId, &HttpQueryConfig::got_result, R.move_as_ok()); } }); - if (params_.size() > 0) { - auto query = ton::create_tl_object( - 0, ton::create_tl_lite_block_id(block_id_), std::vector(params_)); - td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); - } else { - auto query = - ton::create_tl_object(0, ton::create_tl_lite_block_id(block_id_)); - td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); - } + auto query = + params_.size() > 0 + ? ton::serialize_tl_object(ton::create_tl_object( + 0, ton::create_tl_lite_block_id(block_id_), std::vector(params_)), + true) + : ton::serialize_tl_object(ton::create_tl_object( + 0, ton::create_tl_lite_block_id(block_id_)), + true); + td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, + std::move(query), std::move(P)); } void HttpQueryConfig::got_result(td::BufferSlice data) { @@ -1219,9 +1226,10 @@ void HttpQuerySend::start_up() { td::actor::send_closure(SelfId, &HttpQuerySend::got_result, R.move_as_ok()); } }); - auto query = ton::create_tl_object(std::move(data_)); + auto query = + ton::serialize_tl_object(ton::create_tl_object(std::move(data_)), true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } void HttpQuerySend::got_result(td::BufferSlice data) { @@ -1327,10 +1335,12 @@ void HttpQueryRunMethod::start_up_query() { return abort_query(params_serialized.move_as_error_prefix("cannot serialize stack with get-method parameters : ")); } - auto query = ton::create_tl_object( - 0x17, ton::create_tl_lite_block_id(block_id_), std::move(a), method_id, params_serialized.move_as_ok()); + auto query = ton::serialize_tl_object( + ton::create_tl_object( + 0x17, ton::create_tl_lite_block_id(block_id_), std::move(a), method_id, params_serialized.move_as_ok()), + true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } void HttpQueryRunMethod::got_result(td::BufferSlice data) { diff --git a/blockchain-explorer/blockchain-explorer.cpp b/blockchain-explorer/blockchain-explorer.cpp index c22ce1b3c5..029c718d62 100644 --- a/blockchain-explorer/blockchain-explorer.cpp +++ b/blockchain-explorer/blockchain-explorer.cpp @@ -155,8 +155,6 @@ class CoreActor : public CoreActorInterface { td::int32 attempt_ = 0; td::int32 waiting_ = 0; - std::vector ready_; - //void run_queries(); void got_result(td::uint32 idx, td::int32 attempt, td::Result data); @@ -189,12 +187,6 @@ class CoreActor : public CoreActorInterface { static CoreActor* instance_; td::actor::ActorId self_id_; - void conn_ready(td::uint32 idx) { - ready_.at(idx) = true; - } - void conn_closed(td::uint32 idx) { - ready_.at(idx) = false; - } void set_global_config(std::string str) { global_config_ = str; } @@ -219,7 +211,7 @@ class CoreActor : public CoreActorInterface { hide_ips_ = value; } - void send_lite_query(td::BufferSlice query, ton::ShardIdFull shard, td::Promise promise) override; + void send_lite_query(td::BufferSlice query, td::Promise promise) override; void get_last_result(td::Promise> promise) override { } void get_results(td::uint32 max, td::Promise promise) override { @@ -439,50 +431,24 @@ class CoreActor : public CoreActorInterface { } void run() { - std::vector servers; + std::vector servers; if (remote_public_key_.empty()) { auto G = td::read_file(global_config_).move_as_ok(); auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); ton::ton_api::liteclient_config_global gc; ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - - size_t size = gc.liteservers_.size() + gc.liteservers_v2_.size(); - CHECK(size > 0); - ready_.resize(size, false); - - for (auto& s : gc.liteservers_) { - td::IPAddress addr; - addr.init_host_port(td::IPAddress::ipv4_to_str(s->ip_), s->port_).ensure(); - addrs_.push_back(addr); - liteclient::ExtClient::LiteServer serv; - serv.address = addr; - serv.adnl_id = ton::adnl::AdnlNodeIdFull::create(s->id_).move_as_ok(); - servers.push_back(std::move(serv)); - } - for (auto& s : gc.liteservers_v2_) { - td::IPAddress addr; - addr.init_host_port(td::IPAddress::ipv4_to_str(s->ip_), s->port_).ensure(); - addrs_.push_back(addr); - liteclient::ExtClient::LiteServer serv; - serv.address = addr; - serv.adnl_id = ton::adnl::AdnlNodeIdFull::create(s->id_).move_as_ok(); - serv.is_full = false; - for (auto& shard : s->shards_) { - serv.shards.emplace_back(shard->workchain_, (ton::ShardId)shard->shard_); - CHECK(serv.shards.back().is_valid_ext()); - } - servers.push_back(std::move(serv)); + auto r_servers = liteclient::LiteServerConfig::parse_global_config(gc); + r_servers.ensure(); + servers = r_servers.move_as_ok(); + for (const auto& serv : servers) { + addrs_.push_back(serv.addr); } } else { if (!remote_addr_.is_valid()) { LOG(FATAL) << "remote addr not set"; } - ready_.resize(1, false); addrs_.push_back(remote_addr_); - liteclient::ExtClient::LiteServer serv; - serv.address = remote_addr_; - serv.adnl_id = ton::adnl::AdnlNodeIdFull{remote_public_key_}; - servers.push_back(std::move(serv)); + servers.push_back(liteclient::LiteServerConfig{ton::adnl::AdnlNodeIdFull{remote_public_key_}, remote_addr_}); } client_ = liteclient::ExtClient::create(std::move(servers), make_callback()); daemon_ = MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, static_cast(http_port_), nullptr, nullptr, @@ -545,7 +511,7 @@ void CoreActor::got_result(td::uint32 idx, td::int32 attempt, td::Result promise) { +void CoreActor::send_lite_query(td::BufferSlice query, td::Promise promise) { auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); @@ -563,7 +529,7 @@ void CoreActor::send_lite_query(td::BufferSlice query, ton::ShardIdFull shard, t promise.set_value(std::move(B)); }); auto q = ton::create_tl_object(std::move(query)); - td::actor::send_closure(client_, &liteclient::ExtClient::send_query, "query", serialize_tl_object(q, true), shard, + td::actor::send_closure(client_, &liteclient::ExtClient::send_query, "query", serialize_tl_object(q, true), td::Timestamp::in(10.0), std::move(P)); } diff --git a/blockchain-explorer/blockchain-explorer.hpp b/blockchain-explorer/blockchain-explorer.hpp index d3c53e451c..1bae362dee 100644 --- a/blockchain-explorer/blockchain-explorer.hpp +++ b/blockchain-explorer/blockchain-explorer.hpp @@ -64,7 +64,7 @@ class CoreActorInterface : public td::actor::Actor { }; virtual ~CoreActorInterface() = default; - virtual void send_lite_query(td::BufferSlice data, ton::ShardIdFull shard, td::Promise promise) = 0; + virtual void send_lite_query(td::BufferSlice data, td::Promise promise) = 0; virtual void get_last_result(td::Promise> promise) = 0; virtual void get_results(td::uint32 max, td::Promise promise) = 0; diff --git a/lite-client/CMakeLists.txt b/lite-client/CMakeLists.txt index 5774c050bb..5b0c0fa910 100644 --- a/lite-client/CMakeLists.txt +++ b/lite-client/CMakeLists.txt @@ -1,7 +1,9 @@ cmake_minimum_required(VERSION 3.5 FATAL_ERROR) -add_library(lite-client-common STATIC lite-client-common.cpp lite-client-common.h ext-client.cpp ext-client.h QueryTraits.h) -target_link_libraries(lite-client-common PUBLIC tdutils tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto ton_block) +add_library(lite-client-common STATIC lite-client-common.cpp lite-client-common.h ext-client.cpp ext-client.h + query-utils.hpp query-utils.cpp) +target_link_libraries(lite-client-common PUBLIC tdutils tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto + ton_block) add_executable(lite-client lite-client.cpp lite-client.h ext-client.h ext-client.cpp) target_link_libraries(lite-client tdutils tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto ton_block diff --git a/lite-client/QueryTraits.h b/lite-client/QueryTraits.h deleted file mode 100644 index 54190244e9..0000000000 --- a/lite-client/QueryTraits.h +++ /dev/null @@ -1,229 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . -*/ -#pragma once -#include "ton/ton-types.h" -#include "auto/tl/lite_api.h" -#include "auto/tl/lite_api.hpp" -#include "vm/boc.h" -#include "vm/cellslice.h" -#include "block/block-auto.h" -#include "block/block-parse.h" -#include "auto/tl/lite_api.hpp" - -namespace liteclient { - -template -struct QueryTraits { - static ton::ShardIdFull get_shard(const Query& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getMasterchainInfo& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getMasterchainInfoExt& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getTime& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getVersion& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getBlock& q) { - return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getState& q) { - return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getBlockHeader& q) { - return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_sendMessage& q) { - auto shard = [&]() -> td::Result { - vm::BagOfCells boc; - TRY_STATUS(boc.deserialize(q.body_.as_slice())); - if (boc.get_root_count() != 1) { - return td::Status::Error("external message is not a valid bag of cells"); - } - block::gen::CommonMsgInfo::Record_ext_in_msg_info info; - if (!tlb::unpack_cell_inexact(boc.get_root_cell(), info)) { - return td::Status::Error("cannot unpack external message header"); - } - auto dest_prefix = block::tlb::t_MsgAddressInt.get_prefix(info.dest); - if (!dest_prefix.is_valid()) { - return td::Status::Error("destination of an inbound external message is an invalid blockchain address"); - } - return dest_prefix.as_leaf_shard(); - }(); - if (shard.is_error()) { - LOG(DEBUG) << "Failed to get shard from query liteServer.sendMessage: " << shard.move_as_error(); - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } - return shard.move_as_ok(); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getAccountState& q) { - return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getAccountStatePrunned& q) { - return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_runSmcMethod& q) { - return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getShardInfo& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getAllShardsInfo& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getOneTransaction& q) { - return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getTransactions& q) { - return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_lookupBlock& q) { - return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_listBlockTransactions& q) { - return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_listBlockTransactionsExt& q) { - return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getBlockProof& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getConfigAll& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getConfigParams& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getValidatorStats& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getLibraries& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getShardBlockProof& q) { - return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); - } -}; - -template -inline ton::ShardIdFull get_query_shard(const Query& q) { - return QueryTraits::get_shard(q); -} - -} // namespace tonlib diff --git a/lite-client/ext-client.cpp b/lite-client/ext-client.cpp index e79ae9a4fb..0232c28f45 100644 --- a/lite-client/ext-client.cpp +++ b/lite-client/ext-client.cpp @@ -17,92 +17,75 @@ #include "ext-client.h" #include "td/utils/Random.h" #include "ton/ton-shard.h" -#include namespace liteclient { class ExtClientImpl : public ExtClient { public: - ExtClientImpl(std::vector servers, td::unique_ptr callback) + ExtClientImpl(std::vector liteservers, td::unique_ptr callback) : callback_(std::move(callback)) { - CHECK(!servers.empty()); - servers_.resize(servers.size()); + CHECK(!liteservers.empty()); + servers_.resize(liteservers.size()); for (size_t i = 0; i < servers_.size(); ++i) { - servers_[i].s = std::move(servers[i]); - if (!servers_[i].s.is_full) { - for (auto shard : servers_[i].s.shards) { - CHECK(shard.is_valid_ext()); - max_server_shard_depth_ = std::max(max_server_shard_depth_, shard.pfx_len()); - } - } + servers_[i].config = std::move(liteservers[i]); + servers_[i].idx = i; } } void start_up() override { + LOG(INFO) << "Started ext client, " << servers_.size() << " liteservers"; td::Random::Fast rnd; td::random_shuffle(td::as_mutable_span(servers_), rnd); } - void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, + void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, td::Promise promise) override { - TRY_RESULT_PROMISE(promise, server_idx, before_query(shard)); + QueryInfo query_info = get_query_info(data); + TRY_RESULT_PROMISE(promise, server_idx, select_server(query_info)); auto& server = servers_[server_idx]; CHECK(!server.client.empty()); alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); td::Promise P = [SelfId = actor_id(this), server_idx, - promise = std::move(promise)](td::Result R) mutable { + promise = std::move(promise)](td::Result R) mutable { if (R.is_error() && (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { - td::actor::send_closure(SelfId, &ExtClientImpl::set_server_bad, server_idx); + td::actor::send_closure(SelfId, &ExtClientImpl::on_server_error, server_idx); } promise.set_result(std::move(R)); }; + LOG(DEBUG) << "Sending query " << query_info.to_str() << " to server #" << server.idx << " (" + << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, std::move(P)); } - void force_change_liteserver() override { - if (servers_.size() == 1) { - return; - } - auto it = shard_to_server_.find(ton::ShardIdFull(ton::masterchainId)); - if (it != shard_to_server_.end()) { - set_server_bad(it->second); + void reset_servers() override { + LOG(INFO) << "Force resetting all liteservers"; + for (Server& server : servers_) { + server.alive = false; + server.timeout = {}; + server.ignore_until = {}; + server.client.reset(); } } private: - td::Result before_query(ton::ShardIdFull shard) { - if (!shard.is_valid_ext()) { - return td::Status::Error("Invalid shard"); - } - if (is_closing_) { - return td::Status::Error("Client is closing"); - } - if (shard.pfx_len() > max_server_shard_depth_) { - shard = shard_prefix(shard, max_server_shard_depth_); - } - auto it = shard_to_server_.find(shard); - if (it != shard_to_server_.end()) { - size_t server_idx = it->second; - if (!servers_[server_idx].client.empty()) { - return server_idx; + td::Result select_server(const QueryInfo& query_info) { + for (size_t i = 0; i < servers_.size(); ++i) { + if (servers_[i].alive && servers_[i].config.accepts_query(query_info)) { + return i; } - shard_to_server_.erase(it); } - size_t server_idx = servers_.size(); int cnt = 0; int best_priority = -1; for (size_t i = 0; i < servers_.size(); ++i) { Server& server = servers_[i]; - if (!server.supports(shard)) { + if (!server.config.accepts_query(query_info)) { continue; } int priority = 0; - priority += (server.client.empty() ? 0 : 100); priority += (server.ignore_until && !server.ignore_until.is_in_past() ? 0 : 10); - priority += (server.s.is_full ? 1 : 0); if (priority < best_priority) { continue; } @@ -116,100 +99,76 @@ class ExtClientImpl : public ExtClient { ++cnt; } if (server_idx == servers_.size()) { - return td::Status::Error(PSTRING() << "No liteserver for shard " << shard.to_str()); + return td::Status::Error(PSTRING() << "no liteserver for query " << query_info.to_str()); } Server& server = servers_[server_idx]; + server.alive = true; + server.ignore_until = {}; + alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); if (!server.client.empty()) { return server_idx; } class Callback : public ton::adnl::AdnlExtClient::Callback { public: - explicit Callback(td::actor::ActorShared parent, size_t idx) - : parent_(std::move(parent)), idx_(idx) { + explicit Callback(td::actor::ActorId parent, size_t idx) : parent_(std::move(parent)), idx_(idx) { } void on_ready() override { } void on_stop_ready() override { - td::actor::send_closure(parent_, &ExtClientImpl::set_server_bad, idx_); + td::actor::send_closure(parent_, &ExtClientImpl::on_server_error, idx_); } private: - td::actor::ActorShared parent_; + td::actor::ActorId parent_; size_t idx_; }; - ref_cnt_++; - if (shard.is_masterchain()) { - LOG(INFO) << "Connecting to liteserver " << server.s.address << " for masterchain"; - } else { - LOG(INFO) << "Connecting to liteserver " << server.s.address << " for shard " << shard.to_str(); - } - server.client = ton::adnl::AdnlExtClient::create( - server.s.adnl_id, server.s.address, std::make_unique(td::actor::actor_shared(this), server_idx)); - alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); + LOG(INFO) << "Connecting to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() << ":" + << server.config.addr.get_port() << ") for query " << query_info.to_str(); + server.client = ton::adnl::AdnlExtClient::create(server.config.adnl_id, server.config.addr, + std::make_unique(actor_id(this), server_idx)); return server_idx; } struct Server { - LiteServer s; + LiteServerConfig config; + size_t idx = 0; td::actor::ActorOwn client; + bool alive = false; td::Timestamp timeout = td::Timestamp::never(); td::Timestamp ignore_until = td::Timestamp::never(); - - bool supports(const ton::ShardIdFull& shard) const { - return s.is_full || shard.is_masterchain() || - std::any_of(s.shards.begin(), s.shards.end(), - [&](const ton::ShardIdFull s_shard) { return ton::shard_intersects(shard, s_shard); }); - } }; std::vector servers_; - std::map shard_to_server_; - int max_server_shard_depth_ = 0; - - td::unique_ptr callback_; - static constexpr double MAX_NO_QUERIES_TIMEOUT = 100; - bool is_closing_{false}; - td::uint32 ref_cnt_{1}; + td::unique_ptr callback_; + static constexpr double MAX_NO_QUERIES_TIMEOUT = 100.0; + static constexpr double BAD_SERVER_TIMEOUT = 30.0; void alarm() override { for (Server& server : servers_) { if (server.timeout && server.timeout.is_in_past()) { + LOG(INFO) << "Closing connection to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() + << ":" << server.config.addr.get_port() << ")"; server.client.reset(); + server.alive = false; + server.ignore_until = {}; } } } - void set_server_bad(size_t idx) { - servers_[idx].client.reset(); - servers_[idx].timeout = td::Timestamp::never(); - servers_[idx].ignore_until = td::Timestamp::in(60.0); - } - void hangup_shared() override { - ref_cnt_--; - try_stop(); - } - void hangup() override { - is_closing_ = true; - ref_cnt_--; - for (Server& server : servers_) { - server.client.reset(); - } - try_stop(); - } - void try_stop() { - if (is_closing_ && ref_cnt_ == 0) { - stop(); - } + + void on_server_error(size_t idx) { + servers_[idx].alive = false; + servers_[idx].ignore_until = td::Timestamp::in(BAD_SERVER_TIMEOUT); } }; td::actor::ActorOwn ExtClient::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, - td::unique_ptr callback) { - return create({LiteServer{dst, dst_addr, true, {}}}, std::move(callback)); + td::unique_ptr callback) { + return create({LiteServerConfig{dst, dst_addr}}, std::move(callback)); } -td::actor::ActorOwn ExtClient::create(std::vector servers, - td::unique_ptr callback) { - return td::actor::create_actor("ExtClient", std::move(servers), std::move(callback)); +td::actor::ActorOwn ExtClient::create(std::vector liteservers, + td::unique_ptr callback) { + return td::actor::create_actor("ExtClient", std::move(liteservers), std::move(callback)); } } // namespace liteclient diff --git a/lite-client/ext-client.h b/lite-client/ext-client.h index c8569964f9..adcff9bed1 100644 --- a/lite-client/ext-client.h +++ b/lite-client/ext-client.h @@ -18,29 +18,24 @@ #include "td/actor/actor.h" #include "ton/ton-types.h" #include "adnl/adnl-ext-client.h" +#include "query-utils.hpp" namespace liteclient { class ExtClient : public td::actor::Actor { public: - struct LiteServer { - ton::adnl::AdnlNodeIdFull adnl_id; - td::IPAddress address; - bool is_full = true; - std::vector shards; - }; - class Callback { public: - virtual ~Callback() { - } + virtual ~Callback() = default; }; - virtual void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, + virtual void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, td::Promise promise) = 0; - virtual void force_change_liteserver() = 0; + virtual void reset_servers() { + } static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, td::unique_ptr callback); - static td::actor::ActorOwn create(std::vector servers, td::unique_ptr callback); + static td::actor::ActorOwn create(std::vector liteservers, + td::unique_ptr callback); }; } // namespace liteclient diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 923ea617b3..2e2770a474 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -67,12 +67,6 @@ using td::Ref; int verbosity; -bool TestNode::LiteServer::supports(ton::ShardIdFull shard) const { - return is_full || shard.is_masterchain() || - std::any_of(shards.begin(), shards.end(), - [&](const ton::ShardIdFull& our_shard) { return ton::shard_intersects(shard, our_shard); }); -} - void TestNode::run() { class Cb : public td::TerminalIO::Callback { public: @@ -88,55 +82,39 @@ void TestNode::run() { io_ = td::TerminalIO::create("> ", readline_enabled_, ex_mode_, std::make_unique(actor_id(this))); td::actor::send_closure(io_, &td::TerminalIO::set_log_interface); + std::vector servers; if (!single_remote_public_key_.empty()) { // Use single provided liteserver - LiteServer s; - s.addr = single_remote_addr_; - s.public_key = single_remote_public_key_; - single_liteserver_idx_ = 0; - td::TerminalIO::out() << "using liteserver " << s.addr << "\n"; - servers_.push_back(std::move(s)); - run_init_queries(); - return; - } - - auto G = td::read_file(global_config_).move_as_ok(); - auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); - ton::ton_api::liteclient_config_global gc; - ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - CHECK(gc.liteservers_.size() + gc.liteservers_v2_.size() > 0); - - if (gc.validator_ && gc.validator_->zero_state_) { - zstate_id_.workchain = gc.validator_->zero_state_->workchain_; - if (zstate_id_.workchain != ton::workchainInvalid) { - zstate_id_.root_hash = gc.validator_->zero_state_->root_hash_; - zstate_id_.file_hash = gc.validator_->zero_state_->file_hash_; - td::TerminalIO::out() << "zerostate set to " << zstate_id_.to_str() << "\n"; + servers.push_back( + liteclient::LiteServerConfig{ton::adnl::AdnlNodeIdFull{single_remote_public_key_}, single_remote_addr_}); + td::TerminalIO::out() << "using liteserver " << single_remote_addr_ << "\n"; + } else { + auto G = td::read_file(global_config_).move_as_ok(); + auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); + ton::ton_api::liteclient_config_global gc; + ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); + auto r_servers = liteclient::LiteServerConfig::parse_global_config(gc); + r_servers.ensure(); + servers = r_servers.move_as_ok(); + + if (gc.validator_ && gc.validator_->zero_state_) { + zstate_id_.workchain = gc.validator_->zero_state_->workchain_; + if (zstate_id_.workchain != ton::workchainInvalid) { + zstate_id_.root_hash = gc.validator_->zero_state_->root_hash_; + zstate_id_.file_hash = gc.validator_->zero_state_->file_hash_; + td::TerminalIO::out() << "zerostate set to " << zstate_id_.to_str() << "\n"; + } } - } - for (auto& server : gc.liteservers_) { - LiteServer s; - s.addr.init_host_port(td::IPAddress::ipv4_to_str(server->ip_), server->port_).ensure(); - s.public_key = ton::PublicKey{server->id_}; - servers_.push_back(std::move(s)); - } - for (auto& server : gc.liteservers_v2_) { - LiteServer s; - s.addr.init_host_port(td::IPAddress::ipv4_to_str(server->ip_), server->port_).ensure(); - s.public_key = ton::PublicKey{server->id_}; - s.is_full = false; - for (const auto& shard : server->shards_) { - s.shards.emplace_back(shard->workchain_, shard->shard_); - CHECK(s.shards.back().is_valid_ext()); + if (single_liteserver_idx_ != -1) { // Use single liteserver from config + CHECK(single_liteserver_idx_ >= 0 && (size_t)single_liteserver_idx_ < servers.size()); + td::TerminalIO::out() << "using liteserver #" << single_liteserver_idx_ << " with addr " + << servers[single_liteserver_idx_].addr << "\n"; + servers = {servers[single_liteserver_idx_]}; } - servers_.push_back(std::move(s)); - } - - if (single_liteserver_idx_ != -1) { // Use single liteserver from config - CHECK(single_liteserver_idx_ >= 0 && (size_t)single_liteserver_idx_ < gc.liteservers_.size()); - td::TerminalIO::out() << "using liteserver " << single_liteserver_idx_ << " with addr " - << servers_[single_liteserver_idx_].addr << "\n"; } + CHECK(!servers.empty()); + client_ = liteclient::ExtClient::create(std::move(servers), nullptr); + ready_ = true; run_init_queries(); } @@ -183,142 +161,23 @@ void TestNode::after_got_result(bool ok) { } } -bool TestNode::envelope_send_query_to_any(td::BufferSlice query, td::Promise promise) { - return envelope_send_query_to_shard(ton::ShardIdFull(ton::masterchainId), std::move(query), std::move(promise)); -} - -bool TestNode::envelope_send_query_to_account(ton::AccountIdPrefixFull prefix, td::BufferSlice query, - td::Promise promise) { - return envelope_send_query_to_shard(prefix.as_leaf_shard(), std::move(query), std::move(promise)); -} - -bool TestNode::envelope_send_query_to_shard(ton::ShardIdFull shard, td::BufferSlice query, - td::Promise promise) { - if (single_liteserver_idx_ >= 0) { - return envelope_send_query_to_server(single_liteserver_idx_, std::move(query), std::move(promise)); - } - if (shard.is_masterchain() && mc_server_idx_ != -1) { - return envelope_send_query_to_server(mc_server_idx_, std::move(query), std::move(promise)); - } - auto it = shard_server_idx_cached_.find(shard); - if (it != shard_server_idx_cached_.end()) { - return envelope_send_query_to_server(it->second, std::move(query), std::move(promise)); - } - int server_idx = -1; - int random_idx = -1; - int cnt = 0; - bool selected_full = false; - for (int i = 0; i < (int)servers_.size(); ++i) { - const LiteServer& server = servers_[i]; - if (!server.supports(shard)) { - continue; - } - if (server.is_full && !selected_full) { - selected_full = true; - server_idx = -1; - cnt = 0; - } - if (!server.is_full && selected_full) { - continue; - } - if (!server.client.empty()) { - server_idx = i; - } - if (td::Random::fast(0, cnt) == 0) { - random_idx = i; - } - ++cnt; - } - if (server_idx == -1) { - server_idx = random_idx; - } - if (server_idx == -1) { - running_queries_++; - got_result(td::Status::Error(PSTRING() << "failed to select a suitable server for " << shard.to_str()), - std::move(promise)); - return false; - } - shard_server_idx_cached_[shard] = server_idx; - if (shard.is_masterchain()) { - mc_server_idx_ = server_idx; - } - return envelope_send_query_to_server(server_idx, std::move(query), std::move(promise)); -} - -bool TestNode::envelope_send_query_to_server(td::int32 server_idx, td::BufferSlice query, - td::Promise promise) { +bool TestNode::envelope_send_query(td::BufferSlice query, td::Promise promise) { running_queries_++; - LiteServer& server = servers_.at(server_idx); - if (server.client.empty()) { - start_client(server_idx); + if (!ready_ || client_.empty()) { + got_result(td::Status::Error("failed to send query to server: not ready"), std::move(promise)); + return false; } - CHECK(!server.client.empty()); - auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { td::actor::send_closure(SelfId, &TestNode::got_result, std::move(R), std::move(promise)); }); td::BufferSlice b = ton::serialize_tl_object(ton::create_tl_object(std::move(query)), true); - if (server.client_ready) { - td::actor::send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b), - td::Timestamp::in(10.0), std::move(P)); - } else { - server.wait_client_ready.push_back( - [client = server.client.get(), b = std::move(b), P = std::move(P)](td::Result R) mutable { - if (R.is_ok()) { - td::actor::send_closure(client, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b), - td::Timestamp::in(10.0), std::move(P)); - } else { - P.set_error(R.move_as_error_prefix("failed to connect: ")); - } - }); - } + td::actor::send_closure(client_, &liteclient::ExtClient::send_query, "query", std::move(b), td::Timestamp::in(10.0), + std::move(P)); return true; } -void TestNode::start_client(int server_idx) { - LiteServer& server = servers_[server_idx]; - CHECK(server.client.empty()); - class Callback : public ton::adnl::AdnlExtClient::Callback { - public: - void on_ready() override { - td::actor::send_closure(id_, &TestNode::conn_ready, server_idx_); - } - void on_stop_ready() override { - td::actor::send_closure(id_, &TestNode::conn_closed, server_idx_); - } - Callback(td::actor::ActorId id, int server_idx) : id_(std::move(id)), server_idx_(server_idx) { - } - - private: - td::actor::ActorId id_; - int server_idx_; - }; - server.client_ready = false; - server.wait_client_ready.clear(); - LOG(INFO) << "Connecting to " << server.addr << " (liteserver #" << server_idx << ")"; - server.client = ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull{server.public_key}, server.addr, - std::make_unique(actor_id(this), server_idx)); -} - -void TestNode::conn_ready(int server_idx) { - LiteServer& server = servers_[server_idx]; - LOG(INFO) << "Connection to " << server.addr << " (liteserver #" << server_idx << ") is ready"; - server.client_ready = true; - for (auto& p : server.wait_client_ready) { - p.set_result(td::Unit()); - } - server.wait_client_ready.clear(); -} - -void TestNode::conn_closed(int server_idx) { - LiteServer& server = servers_[server_idx]; - LOG(INFO) << "Connection to " << server.addr << " (liteserver #" << server_idx << ") closed"; - server.client_ready = false; - server.wait_client_ready.clear(); -} - td::Promise TestNode::trivial_promise() { return td::PromiseCreator::lambda([Self = actor_id(this)](td::Result res) { if (res.is_error()) { @@ -433,7 +292,7 @@ bool TestNode::dump_cached_cell(td::Slice hash_pfx, td::Slice type_name) { bool TestNode::get_server_time() { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); - return envelope_send_query_to_any(std::move(b), [&, Self = actor_id(this)](td::Result res) -> void { + return envelope_send_query(std::move(b), [&, Self = actor_id(this)](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot get server time"; return; @@ -453,7 +312,7 @@ bool TestNode::get_server_time() { bool TestNode::get_server_version(int mode) { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); - return envelope_send_query_to_any(std::move(b), [Self = actor_id(this), mode](td::Result res) { + return envelope_send_query(std::move(b), [Self = actor_id(this), mode](td::Result res) { td::actor::send_closure_later(Self, &TestNode::got_server_version, std::move(res), mode); }); }; @@ -501,7 +360,7 @@ bool TestNode::get_server_mc_block_id() { int mode = (mc_server_capabilities_ & 2) ? 0 : -1; if (mode < 0) { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); - return envelope_send_query_to_any(std::move(b), [Self = actor_id(this)](td::Result res) -> void { + return envelope_send_query(std::move(b), [Self = actor_id(this)](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot get masterchain info from server"; return; @@ -521,25 +380,24 @@ bool TestNode::get_server_mc_block_id() { } else { auto b = ton::serialize_tl_object(ton::create_tl_object(mode), true); - return envelope_send_query_to_any( - std::move(b), [Self = actor_id(this), mode](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot get extended masterchain info from server"; - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getMasterchainInfoExt"; - } else { - auto f = F.move_as_ok(); - auto blk_id = create_block_id(f->last_); - auto zstate_id = create_zero_state_id(f->init_); - LOG(INFO) << "last masterchain block is " << blk_id.to_str(); - td::actor::send_closure_later(Self, &TestNode::got_server_mc_block_id_ext, blk_id, zstate_id, mode, - f->version_, f->capabilities_, f->last_utime_, f->now_); - } - } - }); + return envelope_send_query(std::move(b), [Self = actor_id(this), mode](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "cannot get extended masterchain info from server"; + return; + } else { + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getMasterchainInfoExt"; + } else { + auto f = F.move_as_ok(); + auto blk_id = create_block_id(f->last_); + auto zstate_id = create_zero_state_id(f->init_); + LOG(INFO) << "last masterchain block is " << blk_id.to_str(); + td::actor::send_closure_later(Self, &TestNode::got_server_mc_block_id_ext, blk_id, zstate_id, mode, + f->version_, f->capabilities_, f->last_utime_, f->now_); + } + } + }); } } @@ -594,54 +452,52 @@ void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateI bool TestNode::request_block(ton::BlockIdExt blkid) { auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query_to_shard( - blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot obtain block " << blkid.to_str() << " from server"; - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getBlock"; - } else { - auto f = F.move_as_ok(); - auto blk_id = ton::create_block_id(f->id_); - LOG(INFO) << "obtained block " << blk_id.to_str() << " from server"; - if (blk_id != blkid) { - LOG(ERROR) << "block id mismatch: expected data for block " << blkid.to_str() << ", obtained for " - << blk_id.to_str(); - } - td::actor::send_closure_later(Self, &TestNode::got_mc_block, blk_id, std::move(f->data_)); - } + return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "cannot obtain block " << blkid.to_str() << " from server"; + return; + } else { + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getBlock"; + } else { + auto f = F.move_as_ok(); + auto blk_id = ton::create_block_id(f->id_); + LOG(INFO) << "obtained block " << blk_id.to_str() << " from server"; + if (blk_id != blkid) { + LOG(ERROR) << "block id mismatch: expected data for block " << blkid.to_str() << ", obtained for " + << blk_id.to_str(); } - }); + td::actor::send_closure_later(Self, &TestNode::got_mc_block, blk_id, std::move(f->data_)); + } + } + }); } bool TestNode::request_state(ton::BlockIdExt blkid) { auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query_to_shard( - blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot obtain state " << blkid.to_str() << " from server"; - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getState"; - } else { - auto f = F.move_as_ok(); - auto blk_id = ton::create_block_id(f->id_); - LOG(INFO) << "obtained state " << blk_id.to_str() << " from server"; - if (blk_id != blkid) { - LOG(ERROR) << "block id mismatch: expected state for block " << blkid.to_str() << ", obtained for " - << blk_id.to_str(); - } - td::actor::send_closure_later(Self, &TestNode::got_mc_state, blk_id, f->root_hash_, f->file_hash_, - std::move(f->data_)); - } + return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "cannot obtain state " << blkid.to_str() << " from server"; + return; + } else { + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getState"; + } else { + auto f = F.move_as_ok(); + auto blk_id = ton::create_block_id(f->id_); + LOG(INFO) << "obtained state " << blk_id.to_str() << " from server"; + if (blk_id != blkid) { + LOG(ERROR) << "block id mismatch: expected state for block " << blkid.to_str() << ", obtained for " + << blk_id.to_str(); } - }); + td::actor::send_closure_later(Self, &TestNode::got_mc_state, blk_id, f->root_hash_, f->file_hash_, + std::move(f->data_)); + } + } + }); } void TestNode::got_mc_block(ton::BlockIdExt blkid, td::BufferSlice data) { @@ -1280,34 +1136,27 @@ td::Status TestNode::send_ext_msg_from_filename(std::string filename) { LOG(ERROR) << "failed to read file `" << filename << "`: " << err.to_string(); return err; } - LOG(ERROR) << "sending query from file " << filename; - - TRY_RESULT_PREFIX(root, vm::std_boc_deserialize(F.ok().as_slice()), "invalid boc: "); - block::gen::CommonMsgInfo::Record_ext_in_msg_info info; - if (!tlb::unpack_cell_inexact(root, info)) { - return td::Status::Error("failed to unpack external message header"); - } - auto dest_prefix = block::tlb::MsgAddressInt::get_prefix(info.dest); - if (!dest_prefix.is_valid()) { - return td::Status::Error("destination of the message is invalid"); + if (ready_ && !client_.empty()) { + LOG(ERROR) << "sending query from file " << filename; + auto P = td::PromiseCreator::lambda([](td::Result R) { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.sendMessage"; + } else { + int status = F.move_as_ok()->status_; + LOG(INFO) << "external message status is " << status; + } + }); + auto b = + ton::serialize_tl_object(ton::create_tl_object(F.move_as_ok()), true); + return envelope_send_query(std::move(b), std::move(P)) ? td::Status::OK() + : td::Status::Error("cannot send query to server"); + } else { + return td::Status::Error("server connection not ready"); } - - auto P = td::PromiseCreator::lambda([](td::Result R) { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.sendMessage"; - } else { - int status = F.move_as_ok()->status_; - LOG(INFO) << "external message status is " << status; - } - }); - auto b = ton::serialize_tl_object(ton::create_tl_object(F.move_as_ok()), true); - return envelope_send_query_to_account(dest_prefix, std::move(b), std::move(P)) - ? td::Status::OK() - : td::Status::Error("cannot send query to server"); } bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::BlockIdExt ref_blkid, @@ -1315,6 +1164,9 @@ bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress if (!ref_blkid.is_valid()) { return set_error("must obtain last block information before making other queries"); } + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } if (addr_ext) { return get_special_smc_addr( addr_ext, [this, ref_blkid, filename, mode, prunned](td::Result res) { @@ -1336,26 +1188,24 @@ bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress ton::create_tl_lite_block_id(ref_blkid), std::move(a)), true); } - ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); LOG(INFO) << "requesting " << (prunned ? "prunned " : "") << "account state for " << workchain << ":" << addr.to_hex() << " with respect to " << ref_blkid.to_str() << " with savefile `" << filename << "` and mode " << mode; - return envelope_send_query_to_account( - account_prefix, std::move(b), - [Self = actor_id(this), workchain, addr, ref_blkid, filename, mode, prunned](td::Result R) { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getAccountState"; - } else { - auto f = F.move_as_ok(); - td::actor::send_closure_later(Self, &TestNode::got_account_state, ref_blkid, ton::create_block_id(f->id_), - ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), - std::move(f->proof_), std::move(f->state_), workchain, addr, filename, mode, - prunned); - } - }); + return envelope_send_query(std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, filename, mode, + prunned](td::Result R) { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getAccountState"; + } else { + auto f = F.move_as_ok(); + td::actor::send_closure_later(Self, &TestNode::got_account_state, ref_blkid, ton::create_block_id(f->id_), + ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), + std::move(f->proof_), std::move(f->state_), workchain, addr, filename, mode, + prunned); + } + }); } td::int64 TestNode::compute_method_id(std::string method) { @@ -1421,18 +1271,19 @@ bool TestNode::start_run_method(ton::WorkchainId workchain, ton::StdSmcAddress a if (!ref_blkid.is_valid()) { return set_error("must obtain last block information before making other queries"); } + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } auto a = ton::create_tl_object(workchain, addr); - ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); if (!mode) { auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(ref_blkid), std::move(a)), true); LOG(INFO) << "requesting account state for " << workchain << ":" << addr.to_hex() << " with respect to " << ref_blkid.to_str() << " to run method " << method_name << " with " << params.size() << " parameters"; - return envelope_send_query_to_account( - account_prefix, std::move(b), - [Self = actor_id(this), workchain, addr, ref_blkid, method_name, params = std::move(params), - promise = std::move(promise)](td::Result R) mutable { + return envelope_send_query( + std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, method_name, params = std::move(params), + promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); return; @@ -1472,27 +1323,26 @@ bool TestNode::start_run_method(ton::WorkchainId workchain, ton::StdSmcAddress a LOG(INFO) << "requesting remote get-method execution for " << workchain << ":" << addr.to_hex() << " with respect to " << ref_blkid.to_str() << " to run method " << method_name << " with " << params.size() << " parameters"; - return envelope_send_query_to_account( - account_prefix, std::move(b), - [Self = actor_id(this), workchain, addr, ref_blkid, method_name, mode, params = std::move(params), - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.runSmcMethod"; - promise.set_error(td::Status::Error("cannot parse answer to liteServer.runSmcMethod")); - } else { - auto f = F.move_as_ok(); - td::actor::send_closure_later( - Self, &TestNode::run_smc_method, mode, ref_blkid, ton::create_block_id(f->id_), - ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), std::move(f->proof_), - std::move(f->state_proof_), workchain, addr, method_name, std::move(params), std::move(f->init_c7_), - std::move(f->lib_extras_), std::move(f->result_), f->exit_code_, std::move(promise)); - } - }); + return envelope_send_query(std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, method_name, mode, + params = std::move(params), + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.runSmcMethod"; + promise.set_error(td::Status::Error("cannot parse answer to liteServer.runSmcMethod")); + } else { + auto f = F.move_as_ok(); + td::actor::send_closure_later(Self, &TestNode::run_smc_method, mode, ref_blkid, ton::create_block_id(f->id_), + ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), + std::move(f->proof_), std::move(f->state_proof_), workchain, addr, method_name, + std::move(params), std::move(f->init_c7_), std::move(f->lib_extras_), + std::move(f->result_), f->exit_code_, std::move(promise)); + } + }); } } @@ -1722,8 +1572,8 @@ void TestNode::send_compute_complaint_price_query(ton::StdSmcAddress elector_add params.emplace_back(td::make_refint(bits)); params.emplace_back(td::make_refint(refs)); params.emplace_back(td::make_refint(expires_in)); - auto P = - td::PromiseCreator::lambda([expires_in, bits, refs, chash, filename](td::Result> R) { + auto P = td::PromiseCreator::lambda( + [this, expires_in, bits, refs, chash, filename](td::Result> R) { if (R.is_error()) { LOG(ERROR) << R.move_as_error(); return; @@ -1748,9 +1598,8 @@ void TestNode::send_compute_complaint_price_query(ton::StdSmcAddress elector_add } bool TestNode::get_msg_queue_sizes() { - // TODO: rework for separated liteservers auto q = ton::serialize_tl_object(ton::create_tl_object(0, 0, 0), true); - return envelope_send_query_to_any(std::move(q), [Self = actor_id(this)](td::Result res) -> void { + return envelope_send_query(std::move(q), [Self = actor_id(this)](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "liteServer.getOutMsgQueueSizes error: " << res.move_as_error(); return; @@ -1810,6 +1659,10 @@ bool TestNode::dns_resolve_start(ton::WorkchainId workchain, ton::StdSmcAddress return set_error("domain name too long"); } + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } + if (workchain == ton::workchainInvalid) { if (dns_root_queried_) { workchain = ton::masterchainId; @@ -2018,16 +1871,17 @@ bool TestNode::get_one_transaction(ton::BlockIdExt blkid, ton::WorkchainId workc if (!ton::shard_contains(blkid.shard_full(), ton::extract_addr_prefix(workchain, addr))) { return set_error("the shard of this block cannot contain this account"); } + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } auto a = ton::create_tl_object(workchain, addr); auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(blkid), std::move(a), lt), true); - ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); LOG(INFO) << "requesting transaction " << lt << " of " << workchain << ":" << addr.to_hex() << " from block " << blkid.to_str(); - return envelope_send_query_to_account( - account_prefix, std::move(b), - [Self = actor_id(this), workchain, addr, lt, blkid, dump](td::Result R) -> void { + return envelope_send_query( + std::move(b), [Self = actor_id(this), workchain, addr, lt, blkid, dump](td::Result R) -> void { if (R.is_error()) { return; } @@ -2044,15 +1898,16 @@ bool TestNode::get_one_transaction(ton::BlockIdExt blkid, ton::WorkchainId workc bool TestNode::get_last_transactions(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::LogicalTime lt, ton::Bits256 hash, unsigned count, bool dump) { + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } auto a = ton::create_tl_object(workchain, addr); auto b = ton::serialize_tl_object( ton::create_tl_object(count, std::move(a), lt, hash), true); - ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); LOG(INFO) << "requesting " << count << " last transactions from " << lt << ":" << hash.to_hex() << " of " << workchain << ":" << addr.to_hex(); - return envelope_send_query_to_account( - account_prefix, std::move(b), - [Self = actor_id(this), workchain, addr, lt, hash, count, dump](td::Result R) { + return envelope_send_query( + std::move(b), [Self = actor_id(this), workchain, addr, lt, hash, count, dump](td::Result R) { if (R.is_error()) { return; } @@ -2420,10 +2275,10 @@ void TestNode::got_one_transaction(ton::BlockIdExt req_blkid, ton::BlockIdExt bl << " but received data has " << root->get_hash().bits().to_hex(256); return; } - } catch (vm::VmError& err) { + } catch (vm::VmError err) { LOG(ERROR) << "error while traversing block transaction proof : " << err.get_msg(); return; - } catch (vm::VmVirtError& err) { + } catch (vm::VmVirtError err) { LOG(ERROR) << "virtualization error while traversing block transaction proof : " << err.get_msg(); return; } @@ -2602,30 +2457,32 @@ void TestNode::got_last_transactions(std::vector blkids, td::Bu bool TestNode::get_block_transactions(ton::BlockIdExt blkid, int mode, unsigned count, ton::Bits256 acc_addr, ton::LogicalTime lt) { + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } auto a = ton::create_tl_object(acc_addr, lt); auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(blkid), mode, count, std::move(a), false, false), true); LOG(INFO) << "requesting " << count << " transactions from block " << blkid.to_str() << " starting from account " << acc_addr.to_hex() << " lt " << lt; - return envelope_send_query_to_shard( - blkid.shard_full(), std::move(b), [Self = actor_id(this), mode](td::Result R) { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.listBlockTransactions"; - } else { - auto f = F.move_as_ok(); - std::vector transactions; - for (auto& id : f->ids_) { - transactions.emplace_back(id->account_, id->lt_, id->hash_); - } - td::actor::send_closure_later(Self, &TestNode::got_block_transactions, ton::create_block_id(f->id_), mode, - f->req_count_, f->incomplete_, std::move(transactions), std::move(f->proof_)); - } - }); + return envelope_send_query(std::move(b), [Self = actor_id(this), mode](td::Result R) { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.listBlockTransactions"; + } else { + auto f = F.move_as_ok(); + std::vector transactions; + for (auto& id : f->ids_) { + transactions.emplace_back(id->account_, id->lt_, id->hash_); + } + td::actor::send_closure_later(Self, &TestNode::got_block_transactions, ton::create_block_id(f->id_), mode, + f->req_count_, f->incomplete_, std::move(transactions), std::move(f->proof_)); + } + }); } void TestNode::got_block_transactions(ton::BlockIdExt blkid, int mode, unsigned req_count, bool incomplete, @@ -2651,23 +2508,25 @@ bool TestNode::get_all_shards(std::string filename, bool use_last, ton::BlockIdE if (!blkid.is_masterchain()) { return set_error("only masterchain blocks contain shard configuration"); } + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); LOG(INFO) << "requesting recent shard configuration"; - return envelope_send_query_to_any( - std::move(b), [Self = actor_id(this), filename](td::Result R) -> void { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getAllShardsInfo"; - } else { - auto f = F.move_as_ok(); - td::actor::send_closure_later(Self, &TestNode::got_all_shards, ton::create_block_id(f->id_), - std::move(f->proof_), std::move(f->data_), filename); - } - }); + return envelope_send_query(std::move(b), [Self = actor_id(this), filename](td::Result R) -> void { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getAllShardsInfo"; + } else { + auto f = F.move_as_ok(); + td::actor::send_closure_later(Self, &TestNode::got_all_shards, ton::create_block_id(f->id_), std::move(f->proof_), + std::move(f->data_), filename); + } + }); } void TestNode::got_all_shards(ton::BlockIdExt blk, td::BufferSlice proof, td::BufferSlice data, std::string filename) { @@ -2731,6 +2590,9 @@ bool TestNode::parse_get_config_params(ton::BlockIdExt blkid, int mode, std::str params.push_back(x); } } + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } if (!blkid.is_masterchain_ext()) { return set_error("only masterchain blocks contain configuration"); } @@ -2749,6 +2611,10 @@ bool TestNode::get_config_params(ton::BlockIdExt blkid, td::Promise promise, int mode, std::string filename, std::vector params) { + if (!(ready_ && !client_.empty())) { + promise.set_error(td::Status::Error("server connection not ready")); + return false; + } if (!blkid.is_masterchain_ext()) { promise.set_error(td::Status::Error("masterchain reference block expected")); return false; @@ -2766,12 +2632,11 @@ bool TestNode::get_config_params_ext(ton::BlockIdExt blkid, td::Promise R) mutable { - td::actor::send_closure_later(Self, &TestNode::got_config_params, blkid, mode, filename, std::move(params), - std::move(R), std::move(promise)); - }); + return envelope_send_query(std::move(b), [Self = actor_id(this), mode, filename, blkid, params = std::move(params), + promise = std::move(promise)](td::Result R) mutable { + td::actor::send_closure_later(Self, &TestNode::got_config_params, blkid, mode, filename, std::move(params), + std::move(R), std::move(promise)); + }); } void TestNode::got_config_params(ton::BlockIdExt req_blkid, int mode, std::string filename, std::vector params, @@ -2959,8 +2824,8 @@ bool TestNode::get_block(ton::BlockIdExt blkid, bool dump) { LOG(INFO) << "got block download request for " << blkid.to_str(); auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query_to_shard( - blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { + return envelope_send_query( + std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot obtain block " << blkid.to_str() << " from server : " << res.move_as_error().to_string(); @@ -2988,8 +2853,8 @@ bool TestNode::get_state(ton::BlockIdExt blkid, bool dump) { LOG(INFO) << "got state download request for " << blkid.to_str(); auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query_to_shard( - blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { + return envelope_send_query( + std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot obtain state " << blkid.to_str() << " from server : " << res.move_as_error().to_string(); @@ -3126,7 +2991,7 @@ void TestNode::got_state(ton::BlockIdExt blkid, ton::RootHash root_hash, ton::Fi } bool TestNode::get_show_block_header(ton::BlockIdExt blkid, int mode) { - return get_block_header(blkid, mode, [this](td::Result R) { + return get_block_header(blkid, mode, [this, blkid](td::Result R) { if (R.is_error()) { LOG(ERROR) << "unable to fetch block header: " << R.move_as_error(); } else { @@ -3141,9 +3006,8 @@ bool TestNode::get_block_header(ton::BlockIdExt blkid, int mode, td::Promise(ton::create_tl_lite_block_id(blkid), mode), true); - return envelope_send_query_to_shard( - blkid.shard_full(), std::move(b), - [this, blkid, promise = std::move(promise)](td::Result R) mutable -> void { + return envelope_send_query( + std::move(b), [this, blkid, promise = std::move(promise)](td::Result R) mutable -> void { TRY_RESULT_PROMISE_PREFIX(promise, res, std::move(R), PSLICE() << "cannot obtain block header for " << blkid.to_str() << " from server :"); got_block_header_raw(std::move(res), std::move(promise), blkid); @@ -3169,9 +3033,8 @@ bool TestNode::lookup_block(ton::ShardIdFull shard, int mode, td::uint64 arg, auto b = ton::serialize_tl_object(ton::create_tl_object( mode, ton::create_tl_lite_block_id_simple(id), arg, (td::uint32)arg), true); - return envelope_send_query_to_shard( - shard, std::move(b), - [this, id, mode, arg, promise = std::move(promise)](td::Result R) mutable -> void { + return envelope_send_query( + std::move(b), [this, id, mode, arg, promise = std::move(promise)](td::Result R) mutable -> void { TRY_RESULT_PROMISE_PREFIX(promise, res, std::move(R), PSLICE() << "cannot look up block header for " << id.to_str() << " with mode " << mode << " and argument " << arg << " from server :"); @@ -3285,9 +3148,9 @@ void TestNode::got_block_header(ton::BlockIdExt blkid, td::BufferSlice data, int return; } show_block_header(blkid, std::move(virt_root), mode); - } catch (vm::VmError& err) { + } catch (vm::VmError err) { LOG(ERROR) << "error processing header for " << blkid.to_str() << " : " << err.get_msg(); - } catch (vm::VmVirtError& err) { + } catch (vm::VmVirtError err) { LOG(ERROR) << "error processing header for " << blkid.to_str() << " : " << err.get_msg(); } show_new_blkids(); @@ -3316,15 +3179,14 @@ bool TestNode::get_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mod ton::serialize_tl_object(ton::create_tl_object( mode & 0xfff, ton::create_tl_lite_block_id(from), ton::create_tl_lite_block_id(to)), true); - return envelope_send_query_to_any( - std::move(b), [Self = actor_id(this), from, to, mode](td::Result res) { - if (res.is_error()) { - LOG(ERROR) << "cannot obtain block proof for " << ((mode & 1) ? to.to_str() : "last masterchain block") - << " starting from " << from.to_str() << " from server : " << res.move_as_error().to_string(); - } else { - td::actor::send_closure_later(Self, &TestNode::got_block_proof, from, to, mode, res.move_as_ok()); - } - }); + return envelope_send_query(std::move(b), [Self = actor_id(this), from, to, mode](td::Result res) { + if (res.is_error()) { + LOG(ERROR) << "cannot obtain block proof for " << ((mode & 1) ? to.to_str() : "last masterchain block") + << " starting from " << from.to_str() << " from server : " << res.move_as_error().to_string(); + } else { + td::actor::send_closure_later(Self, &TestNode::got_block_proof, from, to, mode, res.move_as_ok()); + } + }); } void TestNode::got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mode, td::BufferSlice pchain) { @@ -3378,6 +3240,9 @@ void TestNode::got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mod bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_count, ton::Bits256 start_after, ton::UnixTime min_utime) { + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } if (!blkid.is_masterchain_ext()) { return set_error("only masterchain blocks contain block creator statistics"); } @@ -3388,8 +3253,8 @@ bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_c auto& os = *osp; return get_creator_stats( blkid, mode, req_count, start_after, min_utime, - [&os](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, - const block::DiscountedCounter& shard_cnt) -> bool { + [min_utime, &os](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, + const block::DiscountedCounter& shard_cnt) -> bool { os << key.to_hex() << " mc_cnt:" << mc_cnt << " shard_cnt:" << shard_cnt << std::endl; return true; }, @@ -3418,6 +3283,10 @@ bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_c bool TestNode::get_creator_stats(ton::BlockIdExt blkid, unsigned req_count, ton::UnixTime min_utime, TestNode::creator_stats_func_t func, std::unique_ptr state, td::Promise> promise) { + if (!(ready_ && !client_.empty())) { + promise.set_error(td::Status::Error("server connection not ready")); + return false; + } if (!state) { promise.set_error(td::Status::Error("null CreatorStatsRes")); return false; @@ -3436,7 +3305,7 @@ bool TestNode::get_creator_stats(ton::BlockIdExt blkid, unsigned req_count, ton: LOG(INFO) << "requesting up to " << req_count << " block creator stats records with respect to masterchain block " << blkid.to_str() << " starting from validator public key " << state->last_key.to_hex() << " created after " << min_utime << " (mode=" << state->mode << ")"; - return envelope_send_query_to_any( + return envelope_send_query( std::move(b), [this, blkid, req_count, state = std::move(state), min_utime, func = std::move(func), promise = std::move(promise)](td::Result R) mutable { TRY_RESULT_PROMISE(promise, res, std::move(R)); @@ -3653,8 +3522,8 @@ bool TestNode::load_creator_stats(std::unique_ptr l ton::UnixTime min_utime = info.valid_since - 1000; return get_creator_stats( info.blk_id, 1000, min_utime, - [&info](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, - const block::DiscountedCounter& shard_cnt) -> bool { + [min_utime, &info](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, + const block::DiscountedCounter& shard_cnt) -> bool { info.store_record(key, mc_cnt, shard_cnt); return true; }, @@ -3857,7 +3726,7 @@ bool compute_punishment_default(int interval, bool severe, td::RefInt256& fine, fine = td::make_refint(101 * 1000000000LL); // 101 fine_part = 0; - return true; // todo: (tolya-yanot) temporary reduction of fine + return true; // todo: (tolya-yanot) temporary reduction of fine if (severe) { fine = td::make_refint(2500 * 1000000000LL); // GR$2500 @@ -3879,49 +3748,41 @@ bool compute_punishment_default(int interval, bool severe, td::RefInt256& fine, return true; } -bool compute_punishment(int interval, bool severe, td::RefInt256& fine, unsigned& fine_part, - Ref punishment_params) { - if (punishment_params.is_null()) { +bool compute_punishment(int interval, bool severe, td::RefInt256& fine, unsigned& fine_part, Ref punishment_params) { + if(punishment_params.is_null()) { return compute_punishment_default(interval, severe, fine, fine_part); } block::gen::MisbehaviourPunishmentConfig::Record rec; if (!tlb::unpack_cell(punishment_params, rec)) { - return false; + return false; } - if (interval <= rec.unpunishable_interval) { - return false; + if(interval <= rec.unpunishable_interval) { + return false; } fine = block::tlb::t_Grams.as_integer(rec.default_flat_fine); fine_part = rec.default_proportional_fine; if (severe) { - fine = fine * rec.severity_flat_mult; - fine >>= 8; - fine_part = fine_part * rec.severity_proportional_mult; - fine_part >>= 8; + fine = fine * rec.severity_flat_mult; fine >>= 8; + fine_part = fine_part * rec.severity_proportional_mult; fine_part >>= 8; } if (interval >= rec.long_interval) { - fine = fine * rec.long_flat_mult; - fine >>= 8; - fine_part = fine_part * rec.long_proportional_mult; - fine_part >>= 8; + fine = fine * rec.long_flat_mult; fine >>= 8; + fine_part = fine_part * rec.long_proportional_mult; fine_part >>= 8; return true; } if (interval >= rec.medium_interval) { - fine = fine * rec.medium_flat_mult; - fine >>= 8; - fine_part = fine_part * rec.medium_proportional_mult; - fine_part >>= 8; + fine = fine * rec.medium_flat_mult; fine >>= 8; + fine_part = fine_part * rec.medium_proportional_mult; fine_part >>= 8; return true; } return true; } -bool check_punishment(int interval, bool severe, td::RefInt256 fine, unsigned fine_part, - Ref punishment_params) { +bool check_punishment(int interval, bool severe, td::RefInt256 fine, unsigned fine_part, Ref punishment_params) { td::RefInt256 computed_fine; unsigned computed_fine_part; return compute_punishment(interval, severe, computed_fine, computed_fine_part, punishment_params) && @@ -3957,7 +3818,7 @@ td::Status TestNode::write_val_create_proof(TestNode::ValidatorLoadInfo& info1, int severity = (severe ? 2 : 1); td::RefInt256 fine = td::make_refint(101000000000); - unsigned fine_part = 0; // todo: (tolya-yanot) temporary reduction of fine // 0xffffffff / 16; // 1/16 + unsigned fine_part = 0; // todo: (tolya-yanot) temporary reduction of fine // 0xffffffff / 16; // 1/16 if (!compute_punishment(interval, severe, fine, fine_part, punishment_params)) { return td::Status::Error("cannot compute adequate punishment"); } @@ -4056,7 +3917,7 @@ td::Result> TestNode::ValidatorLoadInfo::build_proof(int idx, td:: block::gen::ValidatorDescr::Record_validator_addr rec2; if (tlb::csr_unpack(entry, rec1)) { pk = std::move(rec1.public_key); - } else if (tlb::csr_unpack(entry, rec2)) { + } else if (tlb::csr_unpack(std::move(entry), rec2)) { pk = std::move(rec2.public_key); } else { return td::Status::Error("cannot unpack ValidatorDescr"); @@ -4272,8 +4133,7 @@ td::Status TestNode::continue_check_validator_load_proof(std::unique_ptrconfig->get_config_param(40))) { + if (!check_punishment(interval, severe, suggested_fine, rec.suggested_fine_part, info2->config->get_config_param(40))) { LOG(ERROR) << "proposed punishment (fine " << td::dec_string(suggested_fine) << ", fine_part=" << (double)rec.suggested_fine_part / (1LL << 32) << " is too harsh"; show_vote(root->get_hash().bits(), false); @@ -4422,8 +4282,7 @@ int main(int argc, char* argv[]) { return (verbosity >= 0 && verbosity <= 9) ? td::Status::OK() : td::Status::Error("verbosity must be 0..9"); }); p.add_option('V', "version", "shows lite-client build information", [&]() { - std::cout << "lite-client build information: [ Commit: " << GitMetadata::CommitSHA1() - << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "lite-client build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index da45ec92ee..59fc1165cb 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -26,6 +26,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "ext-client.h" #include "adnl/adnl-ext-client.h" #include "tl-utils/tl-utils.hpp" #include "ton/ton-types.h" @@ -46,35 +47,19 @@ class TestNode : public td::actor::Actor { min_ls_version = 0x101, min_ls_capabilities = 1 }; // server version >= 1.1, capabilities at least +1 = build proof chains + td::actor::ActorOwn client_; td::actor::ActorOwn io_; - - struct LiteServer { - td::IPAddress addr; - ton::PublicKey public_key; - bool is_full = true; - std::vector shards; - - td::actor::ActorOwn client; - bool client_ready = false; - std::vector> wait_client_ready; - - bool supports(ton::ShardIdFull shard) const; - }; - std::vector servers_; + bool ready_ = false; td::int32 single_liteserver_idx_ = -1; td::IPAddress single_remote_addr_; ton::PublicKey single_remote_public_key_; - std::map shard_server_idx_cached_; - bool readline_enabled_ = true; int print_limit_ = 1024; std::string db_root_; - // mc_server is the server for queries to masterchain - int mc_server_idx_ = -1; int mc_server_time_ = 0; int mc_server_time_got_at_ = 0; int mc_server_version_ = 0; @@ -443,18 +428,7 @@ class TestNode : public td::actor::Actor { void got_result(td::Result R, td::Promise promise); void after_got_result(bool ok); - bool envelope_send_query_to_any(td::BufferSlice query, td::Promise promise); - bool envelope_send_query_to_shard(ton::ShardIdFull shard, td::BufferSlice query, - td::Promise promise); - bool envelope_send_query_to_account(ton::AccountIdPrefixFull prefix, td::BufferSlice query, - td::Promise promise); - - bool envelope_send_query_to_server(td::int32 server_idx, td::BufferSlice query, td::Promise promise); - - void start_client(int server_idx); - void conn_ready(int server_idx); - void conn_closed(int server_idx); - + bool envelope_send_query(td::BufferSlice query, td::Promise promise); void parse_line(td::BufferSlice data); TestNode() = default; diff --git a/lite-client/query-utils.cpp b/lite-client/query-utils.cpp new file mode 100644 index 0000000000..eddaa72762 --- /dev/null +++ b/lite-client/query-utils.cpp @@ -0,0 +1,394 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "query-utils.hpp" + +#include "block-parse.h" +#include "td/utils/overloaded.h" +#include "tl-utils/common-utils.hpp" + +#include "block/block-auto.h" +#include "auto/tl/lite_api.hpp" +#include "overlay/overlay-broadcast.hpp" +#include "tl-utils/lite-utils.hpp" +#include "ton/lite-tl.hpp" +#include "ton/ton-shard.h" + +#include + +namespace liteclient { + +using namespace ton; + +std::string QueryInfo::to_str() const { + td::StringBuilder sb; + sb << "[ " << lite_query_name_by_id(query_id) << " " << shard_id.to_str(); + switch (type) { + case t_simple: + break; + case t_seqno: + sb << " seqno=" << value; + break; + case t_utime: + sb << " utime=" << value; + break; + case t_lt: + sb << " lt=" << value; + break; + case t_mc_seqno: + sb << " mc_seqno=" << value; + break; + } + sb << " ]"; + return sb.as_cslice().str(); +} + +QueryInfo get_query_info(td::Slice data) { + auto F = fetch_tl_object(data, true); + if (F.is_ok()) { + data = F.ok()->data_; + } else { + fetch_tl_prefix(data, true).ignore(); + } + fetch_tl_prefix(data, true).ignore(); + auto Q = fetch_tl_object(data, true); + if (Q.is_error()) { + return {}; + } + return get_query_info(*Q.ok()); +} + +QueryInfo get_query_info(const lite_api::Function& f) { + QueryInfo info; + info.query_id = f.get_id(); + auto from_block_id = [&](const tl_object_ptr& id) { + BlockIdExt block_id = create_block_id(id); + info.shard_id = block_id.shard_full(); + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno(); + }; + downcast_call( + const_cast(f), + td::overloaded([&](const lite_api::liteServer_getTime& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getVersion& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getMasterchainInfo& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getMasterchainInfoExt& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getBlock& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getBlockHeader& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getState& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getAccountState& q) { + BlockIdExt block_id = create_block_id(q.id_); + AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_); + info.shard_id = acc_id_prefix.as_leaf_shard(); + // See LiteQuery::perform_getAccountState + if (block_id.id.workchain != masterchainId) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno(); + } else if (block_id.id.seqno != ~0U) { + info.type = QueryInfo::t_mc_seqno; + info.value = block_id.seqno(); + } else { + info.type = QueryInfo::t_simple; + } + }, + [&](const lite_api::liteServer_getAccountStatePrunned& q) { + BlockIdExt block_id = create_block_id(q.id_); + AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_); + info.shard_id = acc_id_prefix.as_leaf_shard(); + // See LiteQuery::perform_getAccountState + if (block_id.id.workchain != masterchainId) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno(); + } else if (block_id.id.seqno != ~0U) { + info.type = QueryInfo::t_mc_seqno; + info.value = block_id.seqno(); + } else { + info.type = QueryInfo::t_simple; + } + }, + [&](const lite_api::liteServer_getOneTransaction& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getTransactions& q) { + AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_); + info.shard_id = acc_id_prefix.as_leaf_shard(); + info.type = QueryInfo::t_lt; + info.value = q.lt_; + }, + [&](const lite_api::liteServer_sendMessage& q) { + info.type = QueryInfo::t_simple; + auto r_root = vm::std_boc_deserialize(q.body_); + if (r_root.is_error()) { + return; + } + block::gen::CommonMsgInfo::Record_ext_in_msg_info msg_info; + if (!tlb::unpack_cell_inexact(r_root.ok(), msg_info)) { + return; + } + auto dest_prefix = block::tlb::MsgAddressInt::get_prefix(msg_info.dest); + if (!dest_prefix.is_valid()) { + return; + } + info.shard_id = dest_prefix.as_leaf_shard(); + }, + [&](const lite_api::liteServer_getShardInfo& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getAllShardsInfo& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_lookupBlock& q) { + BlockId block_id = create_block_id_simple(q.id_); + info.shard_id = block_id.shard_full(); + // See LiteQuery::perform_lookupBlock + if (q.mode_ & 1) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno; + } else if (q.mode_ == 2) { + info.type = QueryInfo::t_lt; + info.value = q.lt_; + } else if (q.mode_ == 4) { + info.type = QueryInfo::t_utime; + info.value = q.utime_; + } + }, + [&](const lite_api::liteServer_lookupBlockWithProof& q) { + BlockId block_id = create_block_id_simple(q.id_); + info.shard_id = block_id.shard_full(); + // See LiteQuery::perform_lookupBlockWithProof + if (q.mode_ & 1) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno; + } else if (q.mode_ == 2) { + info.type = QueryInfo::t_lt; + info.value = q.lt_; + } else if (q.mode_ == 4) { + info.type = QueryInfo::t_utime; + info.value = q.utime_; + } + }, + [&](const lite_api::liteServer_listBlockTransactions& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_listBlockTransactionsExt& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getConfigParams& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getConfigAll& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getBlockProof& q) { + info.shard_id = ShardIdFull{masterchainId}; + BlockIdExt from = create_block_id(q.known_block_); + BlockIdExt to = create_block_id(q.target_block_); + // See LiteQuery::perform_getBlockProof + if ((q.mode_ & 1) && (q.mode_ & 0x1000)) { + info.type = QueryInfo::t_seqno; + info.value = std::max(from.seqno(), to.seqno()); + } else { + info.type = QueryInfo::t_simple; + } + }, + [&](const lite_api::liteServer_getValidatorStats& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_runSmcMethod& q) { + BlockIdExt block_id = create_block_id(q.id_); + AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_); + info.shard_id = acc_id_prefix.as_leaf_shard(); + // See LiteQuery::perform_getAccountState + if (block_id.id.workchain != masterchainId) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno(); + } else if (block_id.id.seqno != ~0U) { + info.type = QueryInfo::t_mc_seqno; + info.value = block_id.seqno(); + } else { + info.type = QueryInfo::t_simple; + } + }, + [&](const lite_api::liteServer_getLibraries& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getLibrariesWithProof& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getShardBlockProof& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_nonfinal_getCandidate& q) { /* t_simple */ }, + [&](const lite_api::liteServer_nonfinal_getValidatorGroups& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getOutMsgQueueSizes& q) { /* t_simple */ }, + [&](const auto&) { /* t_simple */ })); + if (info.shard_id.workchain == masterchainId) { + info.shard_id.shard = shardIdAll; + } + if (!info.shard_id.is_valid_ext()) { + info.shard_id = ShardIdFull{masterchainId}; + info.type = QueryInfo::t_simple; + info.value = 0; + } + return info; +} + +bool LiteServerConfig::accepts_query(const QueryInfo& query_info) const { + if (is_full) { + return true; + } + for (const Slice& s : slices) { + if (s.accepts_query(query_info)) { + return true; + } + } + return false; +} + +bool LiteServerConfig::Slice::accepts_query(const QueryInfo& query_info) const { + if (unlimited) { + for (const ShardInfo& shard : shards_from) { + if (shard_intersects(shard.shard_id, query_info.shard_id)) { + return true; + } + } + return false; + } + if (!shards_from.empty()) { + bool from_ok = false; + DCHECK(shards_from[0].shard_id.is_masterchain()); + for (const ShardInfo& shard : shards_from) { + if (shard_intersects(shard.shard_id, query_info.shard_id)) { + switch (query_info.type) { + case QueryInfo::t_simple: + from_ok = true; + break; + case QueryInfo::t_seqno: + from_ok = shard.seqno <= query_info.value; + break; + case QueryInfo::t_utime: + from_ok = shard.utime <= query_info.value; + break; + case QueryInfo::t_lt: + from_ok = shard.lt <= query_info.value; + break; + case QueryInfo::t_mc_seqno: + from_ok = shards_from[0].seqno <= query_info.value; + break; + } + if (from_ok) { + break; + } + } + } + if (!from_ok) { + return false; + } + } + if (!shards_to.empty()) { + bool to_ok = false; + DCHECK(shards_to[0].shard_id.is_masterchain()); + for (const ShardInfo& shard : shards_to) { + if (shard_intersects(shard.shard_id, query_info.shard_id)) { + switch (query_info.type) { + case QueryInfo::t_simple: + break; + case QueryInfo::t_seqno: + to_ok = shard.seqno >= query_info.value; + break; + case QueryInfo::t_utime: + to_ok = shard.utime >= query_info.value; + break; + case QueryInfo::t_lt: + to_ok = shard.lt >= query_info.value; + break; + case QueryInfo::t_mc_seqno: + to_ok = shards_from[0].seqno >= query_info.value; + break; + } + if (to_ok) { + break; + } + } + } + if (!to_ok) { + return false; + } + } + return true; +} + +td::Result> LiteServerConfig::parse_global_config( + const ton_api::liteclient_config_global& config) { + std::vector servers; + for (const auto& f : config.liteservers_) { + LiteServerConfig server; + TRY_STATUS(server.addr.init_host_port(td::IPAddress::ipv4_to_str(f->ip_), f->port_)); + server.adnl_id = adnl::AdnlNodeIdFull{PublicKey{f->id_}}; + server.is_full = true; + servers.push_back(std::move(server)); + } + for (const auto& f : config.liteservers_v2_) { + LiteServerConfig server; + TRY_STATUS(server.addr.init_host_port(td::IPAddress::ipv4_to_str(f->ip_), f->port_)); + server.adnl_id = adnl::AdnlNodeIdFull{PublicKey{f->id_}}; + server.is_full = false; + for (const auto& slice_obj : f->slices_) { + Slice slice; + td::Status S = td::Status::OK(); + downcast_call(*slice_obj, + td::overloaded( + [&](const ton_api::liteserver_descV2_sliceSimple& s) { + slice.unlimited = true; + slice.shards_from.push_back({ShardIdFull{masterchainId}, 0, 0, 0}); + for (const auto& shard_obj : s.shards_) { + ShardIdFull shard_id = create_shard_id(shard_obj); + if (!shard_id.is_valid_ext()) { + S = td::Status::Error(PSTRING() << "invalid shard id " << shard_id.to_str()); + break; + } + if (!shard_id.is_masterchain()) { + slice.shards_from.push_back({shard_id, 0, 0, 0}); + } + } + }, + [&](const ton_api::liteserver_descV2_sliceTimed& s) { + auto parse_shards = + [](const std::vector>& shard_objs, + std::vector& shards) -> td::Status { + if (shard_objs.empty()) { + return td::Status::OK(); + } + size_t i = 0; + int mc_idx = -1; + for (const auto& shard_obj : shard_objs) { + ShardIdFull shard_id = create_shard_id(shard_obj->shard_id_); + if (!shard_id.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard id " << shard_id.to_str()); + } + if (shard_id.is_masterchain()) { + shard_id = ShardIdFull{masterchainId}; + if (mc_idx != -1) { + return td::Status::Error("duplicate masterchain shard in sliceTimed"); + } + mc_idx = (int)i; + } + shards.push_back({shard_id, (BlockSeqno)shard_obj->seqno_, (UnixTime)shard_obj->utime_, + (LogicalTime)shard_obj->lt_}); + ++i; + } + if (mc_idx == -1) { + return td::Status::Error("no masterchain shard in sliceTimed"); + } + std::swap(shards[0], shards[mc_idx]); + return td::Status::OK(); + }; + S = parse_shards(s.shards_from_, slice.shards_from); + if (S.is_ok()) { + S = parse_shards(s.shards_to_, slice.shards_to); + } + if (S.is_ok() && slice.shards_from.empty() && slice.shards_to.empty()) { + S = td::Status::Error("shards_from and shards_to are both empty"); + } + })); + TRY_STATUS(std::move(S)); + server.slices.push_back(slice); + } + + servers.push_back(std::move(server)); + } + return servers; +} + +} // namespace liteclient \ No newline at end of file diff --git a/lite-client/query-utils.hpp b/lite-client/query-utils.hpp new file mode 100644 index 0000000000..28500e2664 --- /dev/null +++ b/lite-client/query-utils.hpp @@ -0,0 +1,89 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "ton/ton-types.h" +#include "auto/tl/lite_api.h" +#include "td/utils/port/IPAddress.h" +#include "adnl/adnl-node-id.hpp" + +namespace liteclient { + +struct QueryInfo { + enum Type { t_simple, t_seqno, t_utime, t_lt, t_mc_seqno }; + int query_id = 0; + ton::ShardIdFull shard_id{ton::masterchainId}; + Type type = t_simple; + td::uint64 value = 0; + /* Query types and examples: + * t_simple - query to the recent blocks in a shard, or general info. value = 0. + * getTime, getMasterchainInfo (shard_id = masterchain) + * sendMessage + * getAccountState, runSmcMethod - when no block is given + * t_seqno - query to block with seqno in a shard. value = seqno. + * lookupBlock by seqno + * getBlock, getBlockHeader + * getAccountState, runSmcMethod - when shard block is given + * t_utime - query to a block with given unixtime in a shard. value = utime. + * lookupBlock by utime + * t_lt - query to a block with given lt in a shard. value = lt. + * lookupBlock by lt + * getTransactions + * t_mc_seqno - query to a block in a shard, masterchain seqno is given. value = mc_seqno. + * getAccountState, runSmcMethod - when mc block is given + */ + + std::string to_str() const; +}; + +QueryInfo get_query_info(td::Slice data); +QueryInfo get_query_info(const ton::lite_api::Function& f); + +struct LiteServerConfig { + private: + struct ShardInfo { + ton::ShardIdFull shard_id; + ton::BlockSeqno seqno; + ton::UnixTime utime; + ton::LogicalTime lt; + }; + + struct Slice { + std::vector shards_from, shards_to; + bool unlimited = false; + + bool accepts_query(const QueryInfo& query_info) const; + }; + + bool is_full = false; + std::vector slices; + + public: + ton::adnl::AdnlNodeIdFull adnl_id; + td::IPAddress addr; + + LiteServerConfig() = default; + LiteServerConfig(ton::adnl::AdnlNodeIdFull adnl_id, td::IPAddress addr) + : is_full(true), adnl_id(adnl_id), addr(addr) { + } + + bool accepts_query(const QueryInfo& query_info) const; + + static td::Result> parse_global_config( + const ton::ton_api::liteclient_config_global& config); +}; + +} // namespace liteclient diff --git a/tl-utils/common-utils.hpp b/tl-utils/common-utils.hpp index d61bd79c0e..05f8a825fe 100644 --- a/tl-utils/common-utils.hpp +++ b/tl-utils/common-utils.hpp @@ -148,6 +148,39 @@ td::Result::value, T>>> } } +template +td::Result::value, T>>> fetch_tl_prefix(td::Slice &data, + bool boxed) { + td::TlParser p(data); + tl_object_ptr R; + if (boxed) { + R = TlFetchBoxed, T::ID>::parse(p); + } else { + R = move_tl_object_as(T::fetch(p)); + } + if (p.get_status().is_ok()) { + data.remove_prefix(data.size() - p.get_left_len()); + return std::move(R); + } else { + return p.get_status(); + } +} + +template +td::Result::value, T>>> fetch_tl_prefix(td::Slice &data, + bool boxed) { + CHECK(boxed); + td::TlParser p(data); + tl_object_ptr R; + R = move_tl_object_as(T::fetch(p)); + if (p.get_status().is_ok()) { + data.remove_prefix(data.size() - p.get_left_len()); + return std::move(R); + } else { + return p.get_status(); + } +} + template [[deprecated]] tl_object_ptr clone_tl_object(const tl_object_ptr &obj) { auto B = serialize_tl_object(obj, true); diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 0d7f1249eb..3a46406fa8 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -588,8 +588,12 @@ dummyworkchain0.config.global zero_state_hash:int256 = dummyworkchain0.config.Gl validator.config.global zero_state:tonNode.blockIdExt init_block:tonNode.blockIdExt hardforks:(vector tonNode.blockIdExt) = validator.config.Global; config.global adnl:adnl.config.global dht:dht.config.Global validator:validator.config.global = config.Global; +liteserver.descV2.sliceSimple shards:(vector tonNode.shardId) = liteserver.descV2.Slice; +liteserver.descV2.shardInfo shard_id:tonNode.shardId seqno:int utime:int lt:long = liteserver.descV2.ShardInfo; +liteserver.descV2.sliceTimed shards_from:(vector liteserver.descV2.shardInfo) shards_to:(vector liteserver.descV2.shardInfo) = liteserver.descV2.Slice; + liteserver.desc id:PublicKey ip:int port:int = liteserver.Desc; -liteserver.descV2 id:PublicKey ip:int port:int shards:(vector tonNode.shardId) = liteserver.DescV2; +liteserver.descV2 id:PublicKey ip:int port:int slices:(vector liteserver.descV2.Slice) = liteserver.DescV2; liteclient.config.global liteservers:(vector liteserver.desc) liteservers_v2:(vector liteserver.descV2) validator:validator.config.global = liteclient.config.Global; engine.adnl id:int256 category:int = engine.Adnl; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 0d286bdc091a5b25e2a8deab3019aa84e0e73fe5..77bcff0c5f4e7a7914a3bc653e7cc4f5587d3720 100644 GIT binary patch delta 463 zcmZ4Thqa@TjrY-PeJchiklV=nTb`v`hTnTKk3xdXlI@yI(m9zWsl};9WvNAaDXGQD zVMcnv8Hq(HlO0rLWPVn(d5dFFkdv95I?+L5bB;oT1mp6}Z!M(Tpr-D>A_FpB5JM9K z1K8R>`_q_#x+e=xm6rgS4s%(NGn)>FZQDVxqB#dXXyR~NG01J6d1?6|`J2atd{`#G z+^Rja8wX`HNck)F~>B$XM3br75nBiCDxMH|NPj<*+xJs`WTR56j>%AtW<5H3euL%mk3t($DT|a)aGl9G_N{pF4R&qKpK{ eS&UG3ffR5;6--`}sDL5~3lNZrn;Q-;*#iK}$DR!U delta 107 zcmeBZWLxrwmG{wXeJchiklD!lTb@PP!Crqdk3z!c8ww2)j7v60T1vM;1S|GmfiMCN uh;V|G-8?4b!!p@0Sb6e<19p?o%~hKGVFAx%g~J>WWgG`TZ2oX~#vTA1R3_5c6? diff --git a/tonlib/tonlib/Config.cpp b/tonlib/tonlib/Config.cpp index 73d4e033ce..0f317f94ae 100644 --- a/tonlib/tonlib/Config.cpp +++ b/tonlib/tonlib/Config.cpp @@ -19,6 +19,7 @@ #include "Config.h" #include "adnl/adnl-node-id.hpp" #include "td/utils/JsonBuilder.h" +#include "auto/tl/ton_api_json.h" namespace tonlib { td::Result parse_block_id_ext(td::JsonObject &obj) { @@ -65,67 +66,11 @@ td::Result Config::parse(std::string str) { if (json.type() != td::JsonValue::Type::Object) { return td::Status::Error("Invalid config (1)"); } - td::JsonArray empty_array; - TRY_RESULT(lite_servers_obj, - td::get_json_object_field(json.get_object(), "liteservers", td::JsonValue::Type::Array, true)); - auto &lite_servers = - lite_servers_obj.type() == td::JsonValue::Type::Array ? lite_servers_obj.get_array() : empty_array; - TRY_RESULT(lite_servers_v2_obj, - td::get_json_object_field(json.get_object(), "liteservers_v2", td::JsonValue::Type::Array, true)); - auto &lite_servers_v2 = - lite_servers_v2_obj.type() == td::JsonValue::Type::Array ? lite_servers_v2_obj.get_array() : empty_array; - - auto parse_desc = [&](td::JsonValue &value) -> td::Result { - if (value.type() != td::JsonValue::Type::Object) { - return td::Status::Error("Invalid config (2)"); - } - auto &object = value.get_object(); - - TRY_RESULT(ip, td::get_json_object_long_field(object, "ip", false)); - TRY_RESULT(port, td::get_json_object_int_field(object, "port", false)); - Config::LiteServer server; - TRY_STATUS(server.address.init_host_port(td::IPAddress::ipv4_to_str(static_cast(ip)), port)); - - TRY_RESULT(id_obj, td::get_json_object_field(object, "id", td::JsonValue::Type::Object, false)); - auto &id = id_obj.get_object(); - TRY_RESULT(id_type, td::get_json_object_string_field(id, "@type", false)); - if (id_type != "pub.ed25519") { - return td::Status::Error("Invalid config (3)"); - } - TRY_RESULT(key_base64, td::get_json_object_string_field(id, "key", false)); - TRY_RESULT(key, td::base64_decode(key_base64)); - if (key.size() != 32) { - return td::Status::Error("Invalid config (4)"); - } - - server.adnl_id = ton::adnl::AdnlNodeIdFull(ton::pubkeys::Ed25519(td::Bits256(td::Slice(key).ubegin()))); - return server; - }; Config res; - for (auto &value : lite_servers) { - TRY_RESULT(server, parse_desc(value)); - res.lite_servers.push_back(std::move(server)); - } - for (auto &value : lite_servers_v2) { - TRY_RESULT(server, parse_desc(value)); - server.is_full = false; - TRY_RESULT(shards_obj, td::get_json_object_field(value.get_object(), "shards", td::JsonValue::Type::Array, false)); - for (auto &shard : shards_obj.get_array()) { - if (shard.type() != td::JsonValue::Type::Object) { - return td::Status::Error("Invalid config (5)"); - } - auto &shard_obj = shard.get_object(); - TRY_RESULT(workchain, td::get_json_object_int_field(shard_obj, "workchain", false)); - TRY_RESULT(shard_id, td::get_json_object_long_field(shard_obj, "shard", false)); - if (shard_id == 0) { - return td::Status::Error("Invalid config (6)"); - } - server.shards.emplace_back(workchain, shard_id); - } - - res.lite_servers.push_back(std::move(server)); - } + ton::ton_api::liteclient_config_global conf; + TRY_STATUS(ton::ton_api::from_json(conf, json.get_object())); + TRY_RESULT_ASSIGN(res.lite_servers, liteclient::LiteServerConfig::parse_global_config(conf)); TRY_RESULT(validator_obj, td::get_json_object_field(json.get_object(), "validator", td::JsonValue::Type::Object, false)); diff --git a/tonlib/tonlib/Config.h b/tonlib/tonlib/Config.h index 5ae304ea85..28f23881ba 100644 --- a/tonlib/tonlib/Config.h +++ b/tonlib/tonlib/Config.h @@ -24,11 +24,10 @@ namespace tonlib { struct Config { - using LiteServer = liteclient::ExtClient::LiteServer; ton::BlockIdExt zero_state_id; ton::BlockIdExt init_block_id; std::vector hardforks; - std::vector lite_servers; + std::vector lite_servers; std::string name; static td::Result parse(std::string str); }; diff --git a/tonlib/tonlib/ExtClient.cpp b/tonlib/tonlib/ExtClient.cpp index f0c84d6163..b66ca25c1d 100644 --- a/tonlib/tonlib/ExtClient.cpp +++ b/tonlib/tonlib/ExtClient.cpp @@ -54,7 +54,7 @@ void ExtClient::with_last_block(td::Promise promise) { td::actor::send_closure(client_.last_block_actor_, &LastBlock::get_last_block, std::move(P)); } -void ExtClient::send_raw_query(td::BufferSlice query, ton::ShardIdFull shard, td::Promise promise) { +void ExtClient::send_raw_query(td::BufferSlice query, td::Promise promise) { auto query_id = queries_.create(std::move(promise)); td::Promise P = [query_id, self = this, actor_id = td::actor::actor_id()](td::Result result) { @@ -66,6 +66,6 @@ void ExtClient::send_raw_query(td::BufferSlice query, ton::ShardIdFull shard, td return P.set_error(TonlibError::NoLiteServers()); } td::actor::send_closure(client_.adnl_ext_client_, &liteclient::ExtClient::send_query, "query", std::move(query), - shard, td::Timestamp::in(10.0), std::move(P)); + td::Timestamp::in(10.0), std::move(P)); } } // namespace tonlib diff --git a/tonlib/tonlib/ExtClient.h b/tonlib/tonlib/ExtClient.h index 9db040bb17..b0194c18fd 100644 --- a/tonlib/tonlib/ExtClient.h +++ b/tonlib/tonlib/ExtClient.h @@ -31,7 +31,6 @@ #include "lite-client/ext-client.h" #include "TonlibError.h" #include "utils.h" -#include "lite-client/QueryTraits.h" namespace tonlib { class LastBlock; @@ -65,7 +64,6 @@ class ExtClient { template void send_query(QueryT query, td::Promise promise, td::int32 seq_no = -1) { - ton::ShardIdFull shard = liteclient::QueryTraits::get_shard(query); auto raw_query = ton::serialize_tl_object(&query, true); td::uint32 tag = td::Random::fast_uint32(); VLOG(lite_server) << "send query to liteserver: " << tag << " " << to_string(query); @@ -79,7 +77,7 @@ class ExtClient { ton::serialize_tl_object(ton::create_tl_object(std::move(raw_query)), true); send_raw_query( - std::move(liteserver_query), shard, [promise = std::move(promise), tag](td::Result R) mutable { + std::move(liteserver_query), [promise = std::move(promise), tag](td::Result R) mutable { auto res = [&]() -> td::Result { TRY_RESULT_PREFIX(data, std::move(R), TonlibError::LiteServerNetwork()); auto r_error = ton::fetch_tl_object(data.clone(), true); @@ -99,7 +97,7 @@ class ExtClient { void force_change_liteserver() { if (!client_.adnl_ext_client_.empty()) { - td::actor::send_closure(client_.adnl_ext_client_, &liteclient::ExtClient::force_change_liteserver); + td::actor::send_closure(client_.adnl_ext_client_, &liteclient::ExtClient::reset_servers); } } @@ -109,6 +107,6 @@ class ExtClient { td::Container> last_block_queries_; td::Container> last_config_queries_; - void send_raw_query(td::BufferSlice query, ton::ShardIdFull shard, td::Promise promise); + void send_raw_query(td::BufferSlice query, td::Promise promise); }; } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientOutbound.cpp b/tonlib/tonlib/ExtClientOutbound.cpp index eb923734e5..e5fac8b477 100644 --- a/tonlib/tonlib/ExtClientOutbound.cpp +++ b/tonlib/tonlib/ExtClientOutbound.cpp @@ -28,14 +28,11 @@ class ExtClientOutboundImpl : public ExtClientOutbound { ExtClientOutboundImpl(td::unique_ptr callback) : callback_(std::move(callback)) { } - void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, + void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, td::Promise promise) override { auto query_id = next_query_id_++; queries_[query_id] = std::move(promise); - callback_->request(query_id, data.as_slice().str(), shard); - } - - void force_change_liteserver() override { + callback_->request(query_id, data.as_slice().str()); } void on_query_result(td::int64 id, td::Result r_data, td::Promise promise) override { diff --git a/tonlib/tonlib/ExtClientOutbound.h b/tonlib/tonlib/ExtClientOutbound.h index 6eb6aa983b..bf52c8c260 100644 --- a/tonlib/tonlib/ExtClientOutbound.h +++ b/tonlib/tonlib/ExtClientOutbound.h @@ -27,9 +27,8 @@ class ExtClientOutbound : public liteclient::ExtClient { public: virtual ~Callback() { } - virtual void request(td::int64 id, std::string data, ton::ShardIdFull shard) = 0; + virtual void request(td::int64 id, std::string data) = 0; }; - virtual void on_query_result(td::int64 id, td::Result r_data, td::Promise promise) = 0; static td::actor::ActorOwn create(td::unique_ptr callback); }; diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 3dda7feafd..13a69594ca 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -2078,9 +2078,8 @@ ExtClientRef TonlibClient::get_client_ref() { return ref; } -void TonlibClient::proxy_request(td::int64 query_id, std::string data, ton::ShardIdFull shard) { - on_update( - tonlib_api::make_object(query_id, data, shard.workchain, shard.shard)); +void TonlibClient::proxy_request(td::int64 query_id, std::string data) { + on_update(tonlib_api::make_object(query_id, data)); } void TonlibClient::init_ext_client() { @@ -2091,9 +2090,9 @@ void TonlibClient::init_ext_client() { : parent_(std::move(parent)), config_generation_(config_generation) { } - void request(td::int64 id, std::string data, ton::ShardIdFull shard) override { - send_closure(parent_, &TonlibClient::proxy_request, (id << 16) | (config_generation_ & 0xffff), std::move(data), - shard); + void request(td::int64 id, std::string data) override { + send_closure(parent_, &TonlibClient::proxy_request, (id << 16) | (config_generation_ & 0xffff), + std::move(data)); } private: @@ -2106,17 +2105,8 @@ void TonlibClient::init_ext_client() { ext_client_outbound_ = client.get(); raw_client_ = std::move(client); } else { - class Callback : public liteclient::ExtClient::Callback { - public: - explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { - } - - private: - td::actor::ActorShared<> parent_; - }; ext_client_outbound_ = {}; - ref_cnt_++; - raw_client_ = liteclient::ExtClient::create(config_.lite_servers, td::make_unique(td::actor::actor_shared())); + raw_client_ = liteclient::ExtClient::create(config_.lite_servers, nullptr); } } @@ -4491,8 +4481,8 @@ void deep_library_search(std::set& set, std::set& v } return; } - for (unsigned int i = 0; i < loaded_cell.data_cell->get_refs_cnt(); i++) { - deep_library_search(set, visited, libs, loaded_cell.data_cell->get_ref(i), depth - 1); + for (unsigned int i=0; iget_refs_cnt(); i++) { + deep_library_search(set, visited, libs, loaded_cell.data_cell->get_ref(i), depth - 1, max_libs); } } diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 32731e4e99..973ede94a3 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -401,7 +401,7 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(const tonlib_api::getConfigAll& request, td::Promise>&& promise); - void proxy_request(td::int64 query_id, std::string data, ton::ShardIdFull shard); + void proxy_request(td::int64 query_id, std::string data); void load_libs_from_disk(); void store_libs_to_disk(); diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index ba44481f53..eb0f5af4f5 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -1541,7 +1541,7 @@ class TonlibCli : public td::actor::Actor { CHECK(!raw_client_.empty()); snd_bytes_ += update->data_.size(); send_closure(raw_client_, &liteclient::ExtClient::send_query, "query", td::BufferSlice(update->data_), - ton::ShardIdFull(update->workchain_, update->shard_), td::Timestamp::in(5), + td::Timestamp::in(5), [actor_id = actor_id(this), id = update->id_](td::Result res) { send_closure(actor_id, &TonlibCli::on_adnl_result, id, std::move(res)); }); diff --git a/utils/proxy-liteserver.cpp b/utils/proxy-liteserver.cpp index 7abaa1c7a0..66161fc450 100644 --- a/utils/proxy-liteserver.cpp +++ b/utils/proxy-liteserver.cpp @@ -29,50 +29,71 @@ #include "td/utils/OptionParser.h" #include "td/utils/port/path.h" #include "td/utils/port/signals.h" -#include "td/utils/port/user.h" #include "td/utils/port/IPAddress.h" #include "td/utils/Random.h" #include "td/utils/FileLog.h" #include "git.h" #include "auto/tl/ton_api.h" #include "auto/tl/lite_api.h" -#include "auto/tl/lite_api.hpp" #include "tl-utils/lite-utils.hpp" -#include "ton/lite-tl.hpp" #include "auto/tl/ton_api_json.h" #include "adnl/adnl.h" -#include "lite-client/QueryTraits.h" #include "lite-client/ext-client.h" #if TD_DARWIN || TD_LINUX #include #endif #include +#include using namespace ton; class ProxyLiteserver : public td::actor::Actor { public: - ProxyLiteserver(std::string global_config, std::string db_root, td::uint16 port) - : global_config_(std::move(global_config)), db_root_(std::move(db_root)), port_(port) { + ProxyLiteserver(std::string global_config, std::string db_root, td::uint16 port, PublicKeyHash public_key_hash) + : global_config_(std::move(global_config)) + , db_root_(std::move(db_root)) + , port_(port) + , public_key_hash_(public_key_hash) { } void start_up() override { - LOG_CHECK(db_root_ != "") << "db root is not set"; + LOG_CHECK(!db_root_.empty()) << "db root is not set"; td::mkdir(db_root_).ensure(); db_root_ = td::realpath(db_root_).move_as_ok(); keyring_ = keyring::Keyring::create(db_root_ + "/keyring"); + if (public_key_hash_.is_zero()) { + id_ = {}; + run(); + } else { + td::actor::send_closure(keyring_, &keyring::Keyring::get_public_key, public_key_hash_, + [SelfId = actor_id(this)](td::Result R) mutable { + if (R.is_error()) { + LOG(FATAL) << "Failed to load public key: " << R.move_as_error(); + } + td::actor::send_closure(SelfId, &ProxyLiteserver::got_public_key, R.move_as_ok()); + }); + } + } + + void got_public_key(PublicKey pub) { + id_ = adnl::AdnlNodeIdFull{pub}; + run(); + } + + void run() { td::Status S = prepare_local_config(); if (S.is_error()) { LOG(FATAL) << "Local config error: " << S; } - S = create_ext_client(); + S = parse_global_config(); if (S.is_error()) { LOG(FATAL) << S; } + run_clients(); create_ext_server(); } @@ -82,169 +103,198 @@ class ProxyLiteserver : public td::actor::Actor { auto conf_data = r_conf_data.move_as_ok(); TRY_RESULT_PREFIX(conf_json, td::json_decode(conf_data.as_slice()), "failed to parse json: "); TRY_STATUS_PREFIX(ton_api::from_json(*config_, conf_json.get_object()), "json does not fit TL scheme: "); - TRY_RESULT_PREFIX_ASSIGN(port_, td::narrow_cast_safe(config_->port_), "invalid port: "); - TRY_RESULT_PREFIX_ASSIGN(id_, adnl::AdnlNodeIdFull::create(config_->id_), "invalid id: "); - } else { - LOG(WARNING) << "First launch, creating local config"; + TRY_RESULT_PREFIX(cfg_port, td::narrow_cast_safe(config_->port_), "invalid port: "); + TRY_RESULT_PREFIX(cfg_id, adnl::AdnlNodeIdFull::create(config_->id_), "invalid id: "); + bool rewrite_config = false; if (port_ == 0) { - return td::Status::Error("port is not set"); + port_ = cfg_port; + } else { + rewrite_config |= (port_ != cfg_port); } - config_->port_ = port_; + if (id_.empty()) { + id_ = std::move(cfg_id); + } else { + rewrite_config |= (id_ != cfg_id); + } + if (!rewrite_config) { + return td::Status::OK(); + } + } else { + LOG(WARNING) << "First launch, creating local config"; + } + if (port_ == 0) { + return td::Status::Error("port is not set"); + } + config_->port_ = port_; + if (id_.empty()) { auto pk = PrivateKey{privkeys::Ed25519::random()}; id_ = adnl::AdnlNodeIdFull{pk.compute_public_key()}; - config_->id_ = id_.tl(); td::actor::send_closure(keyring_, &keyring::Keyring::add_key, std::move(pk), false, [](td::Result R) { if (R.is_error()) { LOG(FATAL) << "Failed to store private key"; } }); - - auto s = td::json_encode(td::ToJson(*config_), true); - TRY_STATUS_PREFIX(td::write_file(config_file(), s), "failed to write file: "); } + config_->id_ = id_.tl(); + + auto s = td::json_encode(td::ToJson(*config_), true); + TRY_STATUS_PREFIX(td::write_file(config_file(), s), "failed to write file: "); + LOG(WARNING) << "Writing config.json"; return td::Status::OK(); } - td::Status create_ext_client() { - std::vector servers; + td::Status parse_global_config() { TRY_RESULT_PREFIX(global_config_data, td::read_file(global_config_), "Failed to read global config: "); TRY_RESULT_PREFIX(global_config_json, td::json_decode(global_config_data.as_slice()), "Failed to parse global config: "); - ton::ton_api::liteclient_config_global gc; - ton::ton_api::from_json(gc, global_config_json.get_object()).ensure(); - - size_t size = gc.liteservers_.size() + gc.liteservers_v2_.size(); - if (size == 0) { + ton_api::liteclient_config_global gc; + TRY_STATUS_PREFIX(ton_api::from_json(gc, global_config_json.get_object()), "Failed to parse global config: "); + TRY_RESULT_PREFIX(servers, liteclient::LiteServerConfig::parse_global_config(gc), + "Falied to parse liteservers in global config: "); + if (servers.empty()) { return td::Status::Error("No liteservers in global config"); } - - for (auto& s : gc.liteservers_) { - td::IPAddress addr; - addr.init_host_port(td::IPAddress::ipv4_to_str(s->ip_), s->port_).ensure(); - liteclient::ExtClient::LiteServer serv; - serv.address = addr; - serv.adnl_id = ton::adnl::AdnlNodeIdFull::create(s->id_).move_as_ok(); - servers.push_back(std::move(serv)); + for (auto& s : servers) { + servers_.emplace_back(); + servers_.back().config = std::move(s); } - for (auto& s : gc.liteservers_v2_) { - td::IPAddress addr; - addr.init_host_port(td::IPAddress::ipv4_to_str(s->ip_), s->port_).ensure(); - liteclient::ExtClient::LiteServer serv; - serv.address = addr; - serv.adnl_id = ton::adnl::AdnlNodeIdFull::create(s->id_).move_as_ok(); - serv.is_full = false; - for (auto& shard : s->shards_) { - serv.shards.emplace_back(shard->workchain_, (ton::ShardId)shard->shard_); - CHECK(serv.shards.back().is_valid_ext()); + return td::Status::OK(); + } + + void run_clients() { + class Callback : public adnl::AdnlExtClient::Callback { + public: + explicit Callback(td::actor::ActorId id, size_t idx) : id_(std::move(id)), idx_(idx) { + } + void on_ready() override { + td::actor::send_closure(id_, &ProxyLiteserver::on_client_status, idx_, true); } - servers.push_back(std::move(serv)); + void on_stop_ready() override { + td::actor::send_closure(id_, &ProxyLiteserver::on_client_status, idx_, false); + } + + private: + td::actor::ActorId id_; + size_t idx_; + }; + + for (size_t i = 0; i < servers_.size(); ++i) { + Server& server = servers_[i]; + server.client = adnl::AdnlExtClient::create(server.config.adnl_id, server.config.addr, + std::make_unique(actor_id(this), i)); + server.alive = false; } - class Callback : public liteclient::ExtClient::Callback {}; - ext_client_ = liteclient::ExtClient::create(std::move(servers), td::make_unique()); - return td::Status::OK(); } - void create_ext_server() { - adnl_ = adnl::Adnl::create("", keyring_.get()); - td::actor::send_closure(adnl_, &adnl::Adnl::add_id, id_, ton::adnl::AdnlAddressList{}, (td::uint8)255); - td::actor::send_closure(adnl_, &adnl::Adnl::create_ext_server, - std::vector{id_.compute_short_id()}, std::vector{port_}, - [SelfId = actor_id(this)](td::Result> R) { - R.ensure(); - td::actor::send_closure(SelfId, &ProxyLiteserver::created_ext_server, R.move_as_ok()); - }); + void on_client_status(size_t idx, bool ready) { + Server& server = servers_[idx]; + if (server.alive == ready) { + return; + } + server.alive = ready; + LOG(WARNING) << (ready ? "Connected to" : "Disconnected from") << " server #" << idx << " (" + << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; } - void created_ext_server(td::actor::ActorOwn s) { - ext_server_ = std::move(s); - LOG(WARNING) << "Started proxy liteserver on port " << port_; + void create_ext_server() { + adnl_ = adnl::Adnl::create("", keyring_.get()); + td::actor::send_closure(adnl_, &adnl::Adnl::add_id, id_, adnl::AdnlAddressList{}, (td::uint8)255); class AdnlCallback : public adnl::Adnl::Callback { public: - AdnlCallback(td::actor::ActorId client) : client_(client) { + explicit AdnlCallback(td::actor::ActorId id) : id_(id) { } void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { } void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) override { - td::actor::create_actor("worker", client_, std::move(data), std::move(promise)).release(); + td::actor::send_closure(id_, &ProxyLiteserver::receive_query, std::move(data), std::move(promise)); } private: - td::actor::ActorId client_; + td::actor::ActorId id_; }; td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, id_.compute_short_id(), adnl::Adnl::int_to_bytestring(lite_api::liteServer_query::ID), - std::make_unique(ext_client_.get())); + std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::create_ext_server, std::vector{id_.compute_short_id()}, + std::vector{port_}, + [SelfId = actor_id(this)](td::Result> R) { + R.ensure(); + td::actor::send_closure(SelfId, &ProxyLiteserver::created_ext_server, R.move_as_ok()); + }); } - class QueryWorker : public td::actor::Actor { - public: - QueryWorker(td::actor::ActorId client, td::BufferSlice data, - td::Promise promise) - : client_(std::move(client)), data_(std::move(data)), promise_(std::move(promise)) { - } + void created_ext_server(td::actor::ActorOwn s) { + ext_server_ = std::move(s); + LOG(WARNING) << "Started proxy liteserver on port " << port_; + alarm(); + } - void start_up() override { - auto data = data_.clone(); - auto F = fetch_tl_object(data, true); - if (F.is_ok()) { - data = std::move(F.move_as_ok()->data_); - } else { - auto G = fetch_tl_prefix(data, true); - if (G.is_error()) { - fatal_error(G.move_as_error()); - return; - } + td::Result select_server(const liteclient::QueryInfo& query_info) { + size_t best_idx = servers_.size(); + int cnt = 0; + for (size_t i = 0; i < servers_.size(); ++i) { + Server& server = servers_[i]; + if (!server.alive || !server.config.accepts_query(query_info)) { + continue; } - fetch_tl_prefix(data, true).ignore(); - auto F2 = fetch_tl_object(std::move(data), true); - if (F2.is_error()) { - fatal_error(F2.move_as_error()); - return; + ++cnt; + if (td::Random::fast(1, cnt) == 1) { + best_idx = i; } - auto query = F2.move_as_ok(); - lite_api::downcast_call(*query, [&](auto& obj) { shard_ = liteclient::get_query_shard(obj); }); - - LOG(INFO) << "Got query: shard=" << shard_.to_str() << " size=" << data_.size(); - td::actor::send_closure(client_, &liteclient::ExtClient::send_query, "q", std::move(data_), shard_, - td::Timestamp::in(8.0), [SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &QueryWorker::got_result, std::move(R)); - }); } + if (best_idx == servers_.size()) { + return td::Status::Error(PSTRING() << "no liteserver for query " << query_info.to_str()); + } + return best_idx; + } - void got_result(td::Result R) { - if (R.is_error()) { - LOG(INFO) << "Query to shard=" << shard_.to_str() << ": " << R.error(); - promise_.set_value(create_serialize_tl_object( - R.error().code(), "gateway error: " + R.error().message().str())); - } else { - td::BufferSlice response = R.move_as_ok(); - LOG(INFO) << "Query to shard=" << shard_.to_str() << ": OK, size=" << response.size() - << " time=" << timer_.elapsed(); - promise_.set_value(std::move(response)); + void receive_query(td::BufferSlice data, td::Promise promise) { + liteclient::QueryInfo query_info = liteclient::get_query_info(data); + ++ls_stats_[query_info.query_id]; + promise = [promise = std::move(promise), query_info, timer = td::Timer()](td::Result R) mutable { + if (R.is_ok()) { + LOG(INFO) << "Query " << query_info.to_str() << ": OK, time=" << timer.elapsed() + << ", response_size=" << R.ok().size(); + promise.set_value(R.move_as_ok()); + return; } - stop(); - } + LOG(INFO) << "Query " << query_info.to_str() << ": " << R.error(); + promise.set_value(create_serialize_tl_object( + R.error().code(), "Gateway error: " + R.error().message().str())); + }; + TRY_RESULT_PROMISE(promise, server_idx, select_server(query_info)); - void fatal_error(td::Status S) { - promise_.set_error(std::move(S)); - stop(); - } + Server& server = servers_[server_idx]; + LOG(INFO) << "Sending query " << query_info.to_str() << ", size=" << data.size() << ", to server #" << server_idx + << " (" << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; + td::actor::send_closure(server.client, &adnl::AdnlExtClient::send_query, "q", std::move(data), + td::Timestamp::in(8.0), std::move(promise)); + } - private: - td::actor::ActorId client_; - td::BufferSlice data_; - td::Promise promise_; - td::Timer timer_ = {}; - ShardIdFull shard_; - }; + void alarm() override { + alarm_timestamp() = td::Timestamp::in(60.0); + if (!ls_stats_.empty()) { + td::StringBuilder sb; + sb << "Liteserver stats (1 minute):"; + td::uint32 total = 0; + for (const auto& p : ls_stats_) { + sb << " " << lite_query_name_by_id(p.first) << ":" << p.second; + total += p.second; + } + sb << " TOTAL:" << total; + LOG(WARNING) << sb.as_cslice(); + ls_stats_.clear(); + } + } private: std::string global_config_; std::string db_root_; td::uint16 port_; + PublicKeyHash public_key_hash_; tl_object_ptr config_ = create_tl_object(); adnl::AdnlNodeIdFull id_; @@ -252,7 +302,15 @@ class ProxyLiteserver : public td::actor::Actor { td::actor::ActorOwn keyring_; td::actor::ActorOwn adnl_; td::actor::ActorOwn ext_server_; - td::actor::ActorOwn ext_client_; + + struct Server { + liteclient::LiteServerConfig config; + td::actor::ActorOwn client; + bool alive = false; + }; + std::vector servers_; + + std::map ls_stats_; // lite_api ID -> count, 0 for unknown std::string config_file() const { return db_root_ + "/config.json"; @@ -270,6 +328,7 @@ int main(int argc, char* argv[]) { std::string global_config, db_root; td::uint16 port = 0; + PublicKeyHash public_key_hash = PublicKeyHash::zero(); td::uint32 threads = 4; td::OptionParser p; @@ -290,15 +349,27 @@ int main(int argc, char* argv[]) { std::cout << sb.as_cslice().c_str(); std::exit(2); }); - p.add_checked_option('p', "port", "liteserver port (use only on first launch)", [&](td::Slice arg) -> td::Status { - TRY_RESULT_ASSIGN(port, td::to_integer_safe(arg)); - return td::Status::OK(); - }); + p.add_checked_option('p', "port", "liteserver port (required only on first launch)", + [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(port, td::to_integer_safe(arg)); + return td::Status::OK(); + }); + p.add_checked_option( + 'A', "adnl-id", + "liteserver public key hash in hex (optional). The corresponding private key is required in /keyring/", + [&](td::Slice arg) -> td::Status { + td::Bits256 value; + if (value.from_hex(arg) != 256) { + return td::Status::Error("invalid adnl-id"); + } + public_key_hash = PublicKeyHash{value}; + return td::Status::OK(); + }); p.add_option('C', "global-config", "global TON configuration file", [&](td::Slice arg) { global_config = arg.str(); }); p.add_option('D', "db", "db root", [&](td::Slice arg) { db_root = arg.str(); }); p.add_option('d', "daemonize", "set SIGHUP", [&]() { - td::set_signal_handler(td::SignalType::HangUp, [](int sig) { + td::set_signal_handler(td::SignalType::HangUp, [](int) { #if TD_DARWIN || TD_LINUX close(0); setsid(); @@ -318,8 +389,10 @@ int main(int argc, char* argv[]) { p.run(argc, argv).ensure(); td::actor::Scheduler scheduler({threads}); - scheduler.run_in_context( - [&] { td::actor::create_actor("proxy-liteserver", global_config, db_root, port).release(); }); + scheduler.run_in_context([&] { + td::actor::create_actor("proxy-liteserver", global_config, db_root, port, public_key_hash) + .release(); + }); while (scheduler.run(1)) { } } From 90d2edf5354c3f3664e10580e8815cf18565b859 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 25 Jun 2024 14:06:15 +0300 Subject: [PATCH 095/524] Improve CollatorNode * Keep track of validator groups * Pre-generate shard blocks --- tl/generate/scheme/ton_api.tl | 2 +- tl/generate/scheme/ton_api.tlo | Bin 98440 -> 98404 bytes validator-engine/validator-engine.cpp | 4 +- validator/collator-node.cpp | 362 ++++++++++++++++++-------- validator/collator-node.hpp | 50 ++-- validator/fabric.h | 3 +- validator/impl/collator-impl.h | 9 +- validator/impl/collator.cpp | 37 ++- validator/impl/fabric.cpp | 7 +- validator/manager.cpp | 88 +++++-- validator/manager.hpp | 4 + validator/validator-group.cpp | 2 +- 12 files changed, 402 insertions(+), 166 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 3a46406fa8..9c94e3ee2e 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -837,7 +837,7 @@ validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:lo self_idx:int nodes:(vector validatorSession.newValidatorGroupStats.node) = validatorSession.NewValidatorGroupStats; ---functions--- -collatorNode.generateBlock workchain:int shard:long min_mc_id:tonNode.blockIdExt prev_blocks:(vector tonNode.blockIdExt) +collatorNode.generateBlock shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt) creator:int256 = collatorNode.GenerateBlockResult; ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 77bcff0c5f4e7a7914a3bc653e7cc4f5587d3720..e4ec14379ddd72c2f9543763412a6b3791c194b5 100644 GIT binary patch delta 38 ucmeBZWP8%U*06_7E#Z6E^@Q<_+}# delta 48 zcmaFTz}C^o*06 Config::config_add_control_process(ton::PublicKeyHash key, td:: } td::Result Config::config_add_shard(ton::ShardIdFull shard) { - if (!shard.is_valid_ext()) { - return td::Status::Error(PSTRING() << "invalid shard " << shard.to_str()); + if (!shard.is_valid_ext() || shard.is_masterchain()) { + return td::Status::Error(PSTRING() << "invalid shard to collate " << shard.to_str()); } if (std::find(shards_to_monitor.begin(), shards_to_monitor.end(), shard) != shards_to_monitor.end()) { return false; diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index ce3c4be4b2..79455de440 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -21,6 +21,7 @@ #include "block-db.h" #include "td/utils/lz4.h" #include "checksum.h" +#include "impl/shard.hpp" #include "validator-session/candidate-serializer.h" namespace ton::validator { @@ -33,7 +34,7 @@ CollatorNode::CollatorNode(adnl::AdnlNodeIdShort local_id, td::actor::ActorId id) : id_(std::move(id)) { + explicit Cb(td::actor::ActorId id) : id_(std::move(id)) { } void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { } @@ -57,56 +58,197 @@ void CollatorNode::tear_down() { } void CollatorNode::add_shard(ShardIdFull shard) { - if (std::find(shards_.begin(), shards_.end(), shard) != shards_.end()) { + CHECK(shard.is_valid_ext() && !shard.is_masterchain()); + if (std::find(collating_shards_.begin(), collating_shards_.end(), shard) != collating_shards_.end()) { return; } LOG(INFO) << "Collator node: local_id=" << local_id_ << " , shard=" << shard.to_str(); - shards_.push_back(shard); + collating_shards_.push_back(shard); } void CollatorNode::del_shard(ShardIdFull shard) { - auto it = std::find(shards_.begin(), shards_.end(), shard); - if (it != shards_.end()) { - shards_.erase(it); + auto it = std::find(collating_shards_.begin(), collating_shards_.end(), shard); + if (it != collating_shards_.end()) { + collating_shards_.erase(it); } } void CollatorNode::new_masterchain_block_notification(td::Ref state) { - last_masterchain_block_ = state->get_block_id(); - last_top_blocks_.clear(); - last_top_blocks_[ShardIdFull{masterchainId, shardIdAll}] = last_masterchain_block_; - for (const auto& desc : state->get_shards()) { - last_top_blocks_[desc->shard()] = desc->top_block_id(); - } - if (validators_.empty() || state->is_key_state()) { - validators_.clear(); + last_masterchain_state_ = state; + if (validator_adnl_ids_.empty() || state->is_key_state()) { + validator_adnl_ids_.clear(); for (int next : {-1, 0, 1}) { td::Ref vals = state->get_total_validator_set(next); if (vals.not_null()) { for (const ValidatorDescr& descr : vals->export_vector()) { if (descr.addr.is_zero()) { - validators_.insert( + validator_adnl_ids_.insert( adnl::AdnlNodeIdShort(PublicKey(pubkeys::Ed25519{descr.key.as_bits256()}).compute_short_id())); } else { - validators_.insert(adnl::AdnlNodeIdShort(descr.addr)); + validator_adnl_ids_.insert(adnl::AdnlNodeIdShort(descr.addr)); } } } } } - // Remove old cache entries - auto it = cache_.begin(); - while (it != cache_.end()) { - auto prev_block_id = std::get<2>(it->first)[0]; - auto top_block = get_shard_top_block(prev_block_id.shard_full()); - if (top_block && top_block.value().seqno() > prev_block_id.seqno()) { - it = cache_.erase(it); + + std::map> new_shards; + for (auto& v : state->get_shards()) { + auto shard = v->shard(); + if (v->before_split()) { + CHECK(!v->before_merge()); + new_shards.emplace(shard_child(shard, true), std::vector{v->top_block_id()}); + new_shards.emplace(shard_child(shard, false), std::vector{v->top_block_id()}); + } else if (v->before_merge()) { + ShardIdFull p_shard = shard_parent(shard); + auto it = new_shards.find(p_shard); + if (it == new_shards.end()) { + new_shards.emplace(p_shard, std::vector(2)); + } + bool left = shard_child(p_shard.shard, true) == shard.shard; + new_shards[p_shard][left ? 0 : 1] = v->top_block_id(); + } else { + new_shards.emplace(shard, std::vector{v->top_block_id()}); + } + } + + for (auto& [shard, prev] : new_shards) { + CatchainSeqno cc_seqno = state->get_validator_set(shard)->get_catchain_seqno(); + auto it = validator_groups_.emplace(shard, ValidatorGroupInfo{}); + ValidatorGroupInfo& info = it.first->second; + if (it.second || info.cc_seqno != cc_seqno) { + info.cleanup(); + info.cc_seqno = cc_seqno; + } + } + for (auto it = validator_groups_.begin(); it != validator_groups_.end();) { + if (new_shards.count(it->first)) { + ++it; } else { + it->second.cleanup(); + it = validator_groups_.erase(it); + } + } + for (auto& [shard, prev] : new_shards) { + ValidatorGroupInfo& info = validator_groups_[shard]; + update_validator_group_info(shard, std::move(prev), info.cc_seqno); + auto it = future_validator_groups_.find({shard, info.cc_seqno}); + if (it != future_validator_groups_.end()) { + for (auto& new_prev : it->second.pending_blocks) { + update_validator_group_info(shard, std::move(new_prev), info.cc_seqno); + } + for (auto& promise : it->second.promises) { + promise.set_value(td::Unit()); + } + future_validator_groups_.erase(it); + } + } + + for (auto it = future_validator_groups_.begin(); it != future_validator_groups_.end(); ++it) { + if (get_future_validator_group(it->first.first, it->first.second).is_ok()) { ++it; + } else { + auto& future_group = it->second; + for (auto& promise : future_group.promises) { + promise.set_error(td::Status::Error("validator group is outdated")); + } + it = future_validator_groups_.erase(it); } } } +void CollatorNode::update_validator_group_info(ShardIdFull shard, std::vector prev, + CatchainSeqno cc_seqno) { + if (!can_collate_shard(shard)) { + return; + } + CHECK(prev.size() == 1 || prev.size() == 2); + BlockSeqno next_block_seqno = prev[0].seqno() + 1; + if (prev.size() == 2) { + next_block_seqno = std::max(next_block_seqno, prev[1].seqno() + 1); + } + auto it = validator_groups_.find(shard); + if (it != validator_groups_.end()) { + ValidatorGroupInfo& info = it->second; + if (info.cc_seqno == cc_seqno) { // block from currently known validator group + if (info.next_block_seqno < next_block_seqno) { + LOG(DEBUG) << "updated validator group info: shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno + << ", next_block_seqno=" << next_block_seqno; + info.next_block_seqno = next_block_seqno; + info.prev = std::move(prev); + for (auto cache_it = info.cache.begin(); cache_it != info.cache.end();) { + auto& [cached_prev, cache_entry] = *cache_it; + if (cache_entry->block_seqno < info.next_block_seqno) { + cache_entry->cancel(td::Status::Error(PSTRING() << "next block seqno " << cache_entry->block_seqno + << " is too small, expected " << info.next_block_seqno)); + cache_it = info.cache.erase(cache_it); + continue; + } + if (cache_entry->block_seqno == info.next_block_seqno && cached_prev != info.prev) { + cache_entry->cancel(td::Status::Error("invalid prev blocks")); + cache_it = info.cache.erase(cache_it); + continue; + } + ++cache_it; + } + if (do_pregen_) { + generate_block(shard, cc_seqno, info.prev, td::Timestamp::in(10.0), [](td::Result) {}); + } + } + return; + } + } + auto future_validator_group = get_future_validator_group(shard, cc_seqno); + if (future_validator_group.is_ok()) { + // future validator group, remember for later + future_validator_group.ok()->pending_blocks.push_back(std::move(prev)); + } +} + +td::Result CollatorNode::get_future_validator_group(ShardIdFull shard, + CatchainSeqno cc_seqno) { + auto it = validator_groups_.find(shard); + if (it == validator_groups_.end() && shard.pfx_len() != 0) { + it = validator_groups_.find(shard_parent(shard)); + } + if (it == validator_groups_.end() && shard.pfx_len() < max_shard_pfx_len) { + it = validator_groups_.find(shard_child(shard, true)); + } + if (it == validator_groups_.end() && shard.pfx_len() < max_shard_pfx_len) { + it = validator_groups_.find(shard_child(shard, false)); + } + if (it == validator_groups_.end()) { + return td::Status::Error("no such shard"); + } + if (cc_seqno < it->second.cc_seqno) { // past validator group + return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " is outdated (current is" << it->second.cc_seqno + << ")"); + } + if (cc_seqno - it->second.cc_seqno > 1) { // future validator group, cc_seqno too big + return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " is too big (currently known is" + << it->second.cc_seqno << ")"); + } + // future validator group + return &future_validator_groups_[{shard, cc_seqno}]; +} + +void CollatorNode::ValidatorGroupInfo::cleanup() { + prev.clear(); + next_block_seqno = 0; + for (auto& [_, cache_entry] : cache) { + cache_entry->cancel(td::Status::Error("validator group is outdated")); + } + cache.clear(); +} + +void CollatorNode::CacheEntry::cancel(td::Status reason) { + for (auto& promise : promises) { + promise.set_error(reason.clone()); + } + promises.clear(); + cancellation_token_source.cancel(); +} + static td::BufferSlice serialize_error(td::Status error) { return create_serialize_tl_object(error.code(), error.message().c_str()); } @@ -136,111 +278,121 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data td::Promise promise) { td::Promise new_promise = [promise = std::move(promise), src](td::Result R) mutable { if (R.is_error()) { - LOG(WARNING) << "Query from " << src << ", error: " << R.error(); - promise.set_result(serialize_error(R.move_as_error())); + LOG(INFO) << "adnl query from " << src << ", error: " << R.error(); + if (R.error().code() == ErrorCode::timeout) { + promise.set_error(R.move_as_error()); + } else { + promise.set_result(serialize_error(R.move_as_error())); + } } else { - LOG(INFO) << "Query from " << src << ", success"; + LOG(INFO) << "adnl query from " << src << ", success"; promise.set_result(create_serialize_tl_object( serialize_candidate(R.move_as_ok(), true))); } }; - if (!last_masterchain_block_.is_valid()) { - new_promise.set_error(td::Status::Error("not ready")); - return; - } - if (!validators_.count(src)) { + if (!validator_adnl_ids_.count(src)) { new_promise.set_error(td::Status::Error("src is not a validator")); } TRY_RESULT_PROMISE(new_promise, f, fetch_tl_object(std::move(data), true)); - ShardIdFull shard(f->workchain_, f->shard_); - BlockIdExt min_mc_id = create_block_id(f->min_mc_id_); - LOG(INFO) << "Got query from " << src << ": shard=" << shard.to_str() << ", min_mc_seqno=" << min_mc_id.seqno(); + ShardIdFull shard = create_shard_id(f->shard_); + CatchainSeqno cc_seqno = f->cc_seqno_; + std::vector prev_blocks; + for (const auto& b : f->prev_blocks_) { + prev_blocks.push_back(create_block_id(b)); + } + Ed25519_PublicKey creator(f->creator_); + new_promise = [new_promise = std::move(new_promise), creator](td::Result R) mutable { + TRY_RESULT_PROMISE(new_promise, block, std::move(R)); + new_promise.set_value(change_creator(std::move(block), creator)); + }; if (!shard.is_valid_ext()) { new_promise.set_error(td::Status::Error(PSTRING() << "invalid shard " << shard.to_str())); return; } - if (!can_collate_shard(shard)) { - new_promise.set_error(td::Status::Error(PSTRING() << "this node doesn't collate shard " << shard.to_str())); + if (prev_blocks.size() != 1 && prev_blocks.size() != 2) { + new_promise.set_error(td::Status::Error(PSTRING() << "invalid size of prev_blocks: " << prev_blocks.size())); return; } - if (f->prev_blocks_.size() != 1 && f->prev_blocks_.size() != 2) { - new_promise.set_error(td::Status::Error(PSTRING() << "invalid size of prev_blocks: " << f->prev_blocks_.size())); + LOG(INFO) << "got adnl query from " << src << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno; + generate_block(shard, cc_seqno, std::move(prev_blocks), td::Timestamp::in(10.0), std::move(new_promise)); +} + +void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, + td::Timestamp timeout, td::Promise promise) { + if (last_masterchain_state_.is_null()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); return; } - if (!min_mc_id.is_masterchain_ext()) { - new_promise.set_error(td::Status::Error("min_mc_id is not form masterchain")); + if (!can_collate_shard(shard)) { + promise.set_error(td::Status::Error(PSTRING() << "this node can't collate shard " << shard.to_str())); return; } - std::vector prev_blocks; - for (const auto& b : f->prev_blocks_) { - auto id = create_block_id(b); - if (!id.is_valid_full()) { - new_promise.set_error(td::Status::Error("invalid prev_block")); - return; - } - auto top_block = get_shard_top_block(id.shard_full()); - if (top_block && top_block.value().seqno() > id.seqno()) { - new_promise.set_error(td::Status::Error("cannot collate block: already exists in blockchain")); - return; - } - prev_blocks.push_back(id); - } - Ed25519_PublicKey creator(f->creator_); - if (!shard.is_masterchain()) { - // Collation of masterchain cannot be cached because changing "created_by" in masterchain is hard - // It does not really matter because validators can collate masterchain themselves - new_promise = [promise = std::move(new_promise), creator](td::Result R) mutable { + auto it = validator_groups_.find(shard); + if (it == validator_groups_.end() || it->second.cc_seqno != cc_seqno) { + TRY_RESULT_PROMISE(promise, future_validator_group, get_future_validator_group(shard, cc_seqno)); + future_validator_group->promises.push_back([=, SelfId = actor_id(this), prev_blocks = std::move(prev_blocks), + promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); - } else { - promise.set_result(change_creator(R.move_as_ok(), creator)); + return; } - }; - auto cache_key = std::make_tuple(min_mc_id.seqno(), shard, prev_blocks); - auto cache_entry = cache_[cache_key]; - if (cache_entry == nullptr) { - cache_entry = cache_[cache_key] = std::make_shared(); - } - if (cache_entry->result) { - LOG(INFO) << "Using cached result"; - new_promise.set_result(cache_entry->result.value().clone()); - return; - } - cache_entry->promises.push_back(std::move(new_promise)); - if (cache_entry->started) { - LOG(INFO) << "Collating of this block is already in progress, waiting"; - return; - } - cache_entry->started = true; - new_promise = [SelfId = actor_id(this), cache_entry](td::Result R) mutable { - td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); - }; - } - - auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), prev_blocks = std::move(prev_blocks), - promise = std::move(new_promise)](td::Result> R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error_prefix("failed to get masterchain state: ")); - } else { - td::Ref state(R.move_as_ok()); - if (state.is_null()) { - promise.set_error(R.move_as_error_prefix("failed to get masterchain state: ")); + if (timeout.is_in_past()) { + promise.set_error(td::Status::Error(ErrorCode::timeout)); return; } - td::actor::send_closure(SelfId, &CollatorNode::receive_query_cont, shard, std::move(state), - std::move(prev_blocks), creator, std::move(promise)); - } - }); - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, min_mc_id, 1, td::Timestamp::in(5.0), - std::move(P)); -} + td::actor::send_closure(SelfId, &CollatorNode::generate_block, shard, cc_seqno, std::move(prev_blocks), timeout, + std::move(promise)); + }); + return; + } + ValidatorGroupInfo& validator_group_info = it->second; + BlockSeqno block_seqno = prev_blocks.at(0).seqno() + 1; + if (prev_blocks.size() == 2) { + block_seqno = std::max(block_seqno, prev_blocks.at(1).seqno() + 1); + } + if (validator_group_info.next_block_seqno > block_seqno) { + promise.set_error(td::Status::Error(PSTRING() << "next block seqno " << block_seqno << " is too small, expected " + << validator_group_info.next_block_seqno)); + return; + } + if (validator_group_info.next_block_seqno == block_seqno && validator_group_info.prev != prev_blocks) { + promise.set_error(td::Status::Error("invalid prev_blocks")); + return; + } -void CollatorNode::receive_query_cont(ShardIdFull shard, td::Ref min_mc_state, - std::vector prev_blocks, Ed25519_PublicKey creator, - td::Promise promise) { - run_collate_query(shard, min_mc_state->get_block_id(), std::move(prev_blocks), creator, - min_mc_state->get_validator_set(shard), manager_, td::Timestamp::in(10.0), std::move(promise), - CollateMode::skip_store_candidate); + auto cache_entry = validator_group_info.cache[prev_blocks]; + if (cache_entry == nullptr) { + cache_entry = validator_group_info.cache[prev_blocks] = std::make_shared(); + } + if (cache_entry->result) { + LOG(INFO) << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno + << ": using cached result"; + promise.set_result(cache_entry->result.value().clone()); + return; + } + cache_entry->promises.push_back(std::move(promise)); + if (cache_entry->started) { + LOG(INFO) << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno + << ": collation in progress, waiting"; + return; + } + LOG(INFO) << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno + << ": starting collation"; + cache_entry->started = true; + cache_entry->block_seqno = block_seqno; + run_collate_query( + shard, last_masterchain_state_->get_block_id(), std::move(prev_blocks), Ed25519_PublicKey{td::Bits256::zero()}, + last_masterchain_state_->get_validator_set(shard), manager_, timeout, + [=, SelfId = actor_id(this), timer = td::Timer{}](td::Result R) { + LOG(INFO) << "generate block result" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno + << ", time=" << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); + td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); + }, + cache_entry->cancellation_token_source.get_cancellation_token(), CollateMode::skip_store_candidate); } void CollatorNode::process_result(std::shared_ptr cache_entry, td::Result R) { @@ -259,7 +411,7 @@ void CollatorNode::process_result(std::shared_ptr cache_entry, td::R } bool CollatorNode::can_collate_shard(ShardIdFull shard) const { - return std::any_of(shards_.begin(), shards_.end(), + return std::any_of(collating_shards_.begin(), collating_shards_.end(), [&](const ShardIdFull& our_shard) { return shard_intersects(shard, our_shard); }); } diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index ca2fd73d41..fe7ff43ac7 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -34,12 +34,10 @@ class CollatorNode : public td::actor::Actor { void del_shard(ShardIdFull shard); void new_masterchain_block_notification(td::Ref state); + void update_validator_group_info(ShardIdFull shard, std::vector prev, CatchainSeqno cc_seqno); private: void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); - void receive_query_cont(ShardIdFull shard, td::Ref min_mc_state, - std::vector prev_blocks, Ed25519_PublicKey creator, - td::Promise promise); bool can_collate_shard(ShardIdFull shard) const; @@ -47,33 +45,39 @@ class CollatorNode : public td::actor::Actor { td::actor::ActorId manager_; td::actor::ActorId adnl_; td::actor::ActorId rldp_; - std::vector shards_; - std::set validators_; - - BlockIdExt last_masterchain_block_{}; - std::map last_top_blocks_; + std::vector collating_shards_; + std::set validator_adnl_ids_; struct CacheEntry { bool started = false; + BlockSeqno block_seqno = 0; td::optional result; + td::CancellationTokenSource cancellation_token_source; std::vector> promises; + + void cancel(td::Status reason); + }; + struct ValidatorGroupInfo { + CatchainSeqno cc_seqno{0}; + std::vector prev; + BlockSeqno next_block_seqno{0}; + std::map, std::shared_ptr> cache; + + void cleanup(); }; - std::map>, std::shared_ptr> cache_; - - td::optional get_shard_top_block(ShardIdFull shard) const { - auto it = last_top_blocks_.lower_bound(shard); - if (it != last_top_blocks_.end() && shard_intersects(it->first, shard)) { - return it->second; - } - if (it != last_top_blocks_.begin()) { - --it; - if (shard_intersects(it->first, shard)) { - return it->second; - } - } - return {}; - } + struct FutureValidatorGroup { + std::vector> pending_blocks; + std::vector> promises; + }; + std::map validator_groups_; + std::map, FutureValidatorGroup> future_validator_groups_; + + td::Ref last_masterchain_state_; + + td::Result get_future_validator_group(ShardIdFull shard, CatchainSeqno cc_seqno); + void generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, + td::Timestamp timeout, td::Promise promise); void process_result(std::shared_ptr cache_entry, td::Result R); public: diff --git a/validator/fabric.h b/validator/fabric.h index a38b405688..1aa5a16706 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -88,7 +88,8 @@ void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, unsigned mode = 0); + td::Promise promise, td::CancellationToken cancellation_token = {}, + unsigned mode = 0); void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 1775d657ce..b8003277be 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -87,9 +87,9 @@ class Collator final : public td::actor::Actor { static constexpr bool shard_splitting_enabled = true; public: - Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, - std::vector prev, Ref validator_set, Ed25519_PublicKey collator_id, - td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, + Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, + Ref validator_set, Ed25519_PublicKey collator_id, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise, td::CancellationToken cancellation_token, unsigned mode); ~Collator() override = default; bool is_busy() const { @@ -343,6 +343,9 @@ class Collator final : public td::actor::Actor { void return_block_candidate(td::Result saved); bool update_last_proc_int_msg(const std::pair& new_lt_hash); + td::CancellationToken cancellation_token_; + bool check_cancelled(); + public: static td::uint32 get_skip_externals_queue_size(); }; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index fc031351c0..2f8d7bbd7d 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -77,7 +77,7 @@ static inline bool dbg(int c) { Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, td::Ref validator_set, Ed25519_PublicKey collator_id, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, unsigned mode) + td::Promise promise, td::CancellationToken cancellation_token, unsigned mode) : shard_(shard) , is_hardfork_(is_hardfork) , min_mc_block_id{min_masterchain_block_id} @@ -92,9 +92,11 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) , main_promise(std::move(promise)) , mode_(mode) - , perf_timer_("collate", 0.1, [manager](double duration) { - send_closure(manager, &ValidatorManager::add_perf_timer_stat, "collate", duration); - }) { + , perf_timer_("collate", 0.1, + [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "collate", duration); + }) + , cancellation_token_(std::move(cancellation_token)) { } /** @@ -382,6 +384,9 @@ bool Collator::fatal_error(std::string err_msg, int err_code) { */ void Collator::check_pending() { // LOG(DEBUG) << "pending = " << pending; + if (!check_cancelled()) { + return; + } if (!pending) { step = 2; try { @@ -2423,6 +2428,9 @@ bool Collator::out_msg_queue_cleanup() { LOG(WARNING) << "cleaning up outbound queue takes too long, ending"; break; } + if (!check_cancelled()) { + return false; + } if (i == queue_parts.size()) { i = 0; } @@ -3516,6 +3524,9 @@ bool Collator::process_inbound_internal_messages() { LOG(WARNING) << "soft timeout reached, stop processing inbound internal messages"; break; } + if (!check_cancelled()) { + return false; + } LOG(DEBUG) << "processing inbound message with (lt,hash)=(" << kv->lt << "," << kv->key.to_hex() << ") from neighbor #" << kv->source; if (verbosity > 2) { @@ -3565,6 +3576,9 @@ bool Collator::process_inbound_external_messages() { LOG(WARNING) << "medium timeout reached, stop processing inbound external messages"; break; } + if (!check_cancelled()) { + return false; + } auto ext_msg = ext_msg_struct.cell; ton::Bits256 hash{ext_msg->get_hash().bits()}; int r = process_external_message(std::move(ext_msg)); @@ -3819,6 +3833,9 @@ bool Collator::process_new_messages(bool enqueue_only) { LOG(INFO) << "BLOCK FULL, enqueue all remaining new messages"; enqueue_only = true; } + if (!check_cancelled()) { + return false; + } LOG(DEBUG) << "have message with lt=" << msg.lt; int res = process_one_new_message(std::move(msg), enqueue_only); if (res < 0) { @@ -5360,6 +5377,18 @@ void Collator::after_get_external_messages(td::Result prev, Ed25519_PublicKey creator, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, unsigned mode) { + td::Promise promise, td::CancellationToken cancellation_token, unsigned mode) { BlockSeqno seqno = 0; for (auto& p : prev) { if (p.seqno() > seqno) { @@ -224,7 +224,8 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc } td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, false, min_masterchain_block_id, std::move(prev), std::move(validator_set), creator, - std::move(manager), timeout, std::move(promise), mode) + std::move(manager), timeout, std::move(promise), std::move(cancellation_token), + mode) .release(); } @@ -240,7 +241,7 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, min_masterchain_block_id, std::move(prev), td::Ref{}, Ed25519_PublicKey{Bits256::zero()}, std::move(manager), timeout, std::move(promise), - 0) + td::CancellationToken{}, 0) .release(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index bde7a8056a..d3ce388e76 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -218,12 +218,28 @@ void ValidatorManagerImpl::prevalidate_block(BlockBroadcast broadcast, td::Promi promise.set_error(td::Status::Error("not monitoring shard")); return; } + promise = [SelfId = actor_id(this), promise = std::move(promise), block_id = broadcast.block_id, + cc_seqno = broadcast.catchain_seqno](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::validated_block_broadcast, block_id, cc_seqno); + } + promise.set_result(std::move(R)); + }; td::actor::create_actor("broadcast", std::move(broadcast), last_masterchain_block_handle_, last_masterchain_state_, last_known_key_block_handle_, actor_id(this), td::Timestamp::in(2.0), std::move(promise)) .release(); } +void ValidatorManagerImpl::validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno) { + for (auto &[_, collator_node] : collator_nodes_) { + if (collator_node.can_collate_shard(block_id.shard_full())) { + td::actor::send_closure(collator_node.actor, &CollatorNode::update_validator_group_info, block_id.shard_full(), + std::vector{block_id}, cc_seqno); + } + } +} + void ValidatorManagerImpl::sync_complete(td::Promise promise) { started_ = true; @@ -471,7 +487,7 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - if (!is_validator()) { + if (!is_validator() && !is_shard_collator(block_id.shard_full())) { return; } if (!last_masterchain_block_handle_) { @@ -513,29 +529,36 @@ void ValidatorManagerImpl::new_block_candidate(BlockIdExt block_id, td::BufferSl } void ValidatorManagerImpl::add_shard_block_description(td::Ref desc) { - if (desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { - auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); - if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second.latest_desc->block_id().id.seqno) { - VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; - return; - } - shard_blocks_[ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}].latest_desc = desc; - VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); - if (need_monitor(desc->block_id().shard_full())) { - auto P = td::PromiseCreator::lambda([](td::Result> R) { - if (R.is_error()) { - auto S = R.move_as_error(); - if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { - VLOG(VALIDATOR_NOTICE) << "failed to get shard state: " << S; - } else { - VLOG(VALIDATOR_DEBUG) << "failed to get shard state: " << S; - } + if (!desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { + return; + } + auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); + if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second.latest_desc->block_id().id.seqno) { + VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; + return; + } + shard_blocks_[ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}].latest_desc = desc; + VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); + if (need_monitor(desc->block_id().shard_full())) { + auto P = td::PromiseCreator::lambda([](td::Result> R) { + if (R.is_error()) { + auto S = R.move_as_error(); + if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { + VLOG(VALIDATOR_NOTICE) << "failed to get shard state: " << S; + } else { + VLOG(VALIDATOR_DEBUG) << "failed to get shard state: " << S; } - }); - wait_block_state_short(desc->block_id(), 0, td::Timestamp::in(60.0), std::move(P)); - } - if (validating_masterchain()) { - preload_msg_queue_to_masterchain(desc); + } + }); + wait_block_state_short(desc->block_id(), 0, td::Timestamp::in(60.0), std::move(P)); + } + if (validating_masterchain()) { + preload_msg_queue_to_masterchain(desc); + } + for (auto& [_, collator_node] : collator_nodes_) { + if (collator_node.can_collate_shard(desc->shard())) { + td::actor::send_closure(collator_node.actor, &CollatorNode::update_validator_group_info, desc->shard(), + std::vector{desc->block_id()}, desc->catchain_seqno()); } } } @@ -2863,6 +2886,21 @@ PublicKeyHash ValidatorManagerImpl::get_validator(ShardIdFull shard, td::Ref r) { if (r.size() == 0) { delay_action([SelfId = actor_id( @@ -3364,6 +3402,10 @@ void ValidatorManagerImpl::get_validator_sessions_info( } void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) { + if (shard.is_masterchain() || !shard.is_valid_ext()) { + LOG(WARNING) << "cannot collate shard " << shard.to_str(); + return; + } auto it = collator_nodes_.find(id); if (it == collator_nodes_.end()) { it = collator_nodes_.emplace(id, Collator()).first; diff --git a/validator/manager.hpp b/validator/manager.hpp index 4e5d1d3e73..b85808d3c1 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -372,6 +372,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise promise) override; void validate_block(ReceivedBlock block, td::Promise promise) override; void prevalidate_block(BlockBroadcast broadcast, td::Promise promise) override; + void validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno); //void create_validate_block(BlockId block, td::BufferSlice data, td::Promise promise) = 0; void sync_complete(td::Promise promise) override; @@ -562,6 +563,7 @@ class ValidatorManagerImpl : public ValidatorManager { bool is_validator(); bool validating_masterchain(); PublicKeyHash get_validator(ShardIdFull shard, td::Ref val_set); + bool is_shard_collator(ShardIdFull shard); ValidatorManagerImpl(td::Ref opts, std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, @@ -763,6 +765,8 @@ class ValidatorManagerImpl : public ValidatorManager { struct Collator { td::actor::ActorOwn actor; std::set shards; + + bool can_collate_shard(ShardIdFull shard) const; }; std::map collator_nodes_; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index e4ae88dd4d..94490e5a29 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -546,7 +546,7 @@ void ValidatorGroup::collate_block(td::uint32 round_id, td::Timestamp timeout, t prev_blocks.push_back(create_tl_block_id(p)); } td::BufferSlice query = create_serialize_tl_object( - shard_.workchain, shard_.shard, create_tl_block_id(min_masterchain_block_id_), std::move(prev_blocks), + create_tl_shard_id(shard_), validator_set_->get_catchain_seqno(), std::move(prev_blocks), local_id_full_.ed25519_value().raw()); auto P = td::PromiseCreator::lambda( From 81b7993d661c0a208048e6e173b1b022ab19867e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 28 Jun 2024 11:57:14 +0300 Subject: [PATCH 096/524] Update query-utils.cpp --- lite-client/query-utils.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lite-client/query-utils.cpp b/lite-client/query-utils.cpp index eddaa72762..f48e3e8dd7 100644 --- a/lite-client/query-utils.cpp +++ b/lite-client/query-utils.cpp @@ -211,7 +211,11 @@ QueryInfo get_query_info(const lite_api::Function& f) { [&](const lite_api::liteServer_getShardBlockProof& q) { from_block_id(q.id_); }, [&](const lite_api::liteServer_nonfinal_getCandidate& q) { /* t_simple */ }, [&](const lite_api::liteServer_nonfinal_getValidatorGroups& q) { /* t_simple */ }, - [&](const lite_api::liteServer_getOutMsgQueueSizes& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getOutMsgQueueSizes& q) { + // This query is expected to be removed, as it is not fully compatible with separated liteservers + /* t_simple */ + }, + [&](const lite_api::liteServer_getBlockOutMsgQueueSize& q) { from_block_id(q.id_); }, [&](const auto&) { /* t_simple */ })); if (info.shard_id.workchain == masterchainId) { info.shard_id.shard = shardIdAll; From 7f263b8b11eb71eb60df5f16900a1b7edbeb5329 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 28 Jun 2024 12:46:08 +0300 Subject: [PATCH 097/524] Don't request queue size if not needed in validate-query --- validator/impl/validate-query.cpp | 9 ++++++++- validator/impl/validate-query.hpp | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index e3a499a845..616fe31c72 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -2297,9 +2297,14 @@ bool ValidateQuery::prepare_out_msg_queue_size() { if (ps_.out_msg_queue_size_) { // if after_split then out_msg_queue_size is always present, since it is calculated during split old_out_msg_queue_size_ = ps_.out_msg_queue_size_.value(); + out_msg_queue_size_known_ = true; + return true; + } + if (!store_out_msg_queue_size_) { // Don't need it return true; } old_out_msg_queue_size_ = 0; + out_msg_queue_size_known_ = true; for (size_t i = 0; i < prev_blocks.size(); ++i) { ++pending; send_closure_later(manager, &ValidatorManager::get_out_msg_queue_size, prev_blocks[i], @@ -3363,7 +3368,9 @@ bool ValidateQuery::precheck_message_queue_update() { return reject_query("invalid OutMsgQueue dictionary difference between the old and the new state: "s + err.get_msg()); } - LOG(INFO) << "outbound message queue size: " << old_out_msg_queue_size_ << " -> " << new_out_msg_queue_size_; + if (out_msg_queue_size_known_) { + LOG(INFO) << "outbound message queue size: " << old_out_msg_queue_size_ << " -> " << new_out_msg_queue_size_; + } if (store_out_msg_queue_size_) { if (!ns_.out_msg_queue_size_) { return reject_query(PSTRING() << "outbound message queue size in the new state is not correct (expected: " diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 09ed7984eb..c039fec447 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -236,6 +236,7 @@ class ValidateQuery : public td::actor::Actor { std::map, Ref> new_dispatch_queue_messages_; std::set account_expected_defer_all_messages_; td::uint64 old_out_msg_queue_size_ = 0, new_out_msg_queue_size_ = 0; + bool out_msg_queue_size_known_ = false; bool msg_metadata_enabled_ = false; bool deferring_messages_enabled_ = false; From f4b23a257b2ba3200389d45ec6ba55bdff4d990e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 28 Jun 2024 15:05:54 +0300 Subject: [PATCH 098/524] Fix parsing config in tonlib --- tonlib/test/offline.cpp | 3 ++- tonlib/tonlib/Config.cpp | 38 ++++++++++++-------------------------- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/tonlib/test/offline.cpp b/tonlib/test/offline.cpp index b7423853c5..47d5d6a268 100644 --- a/tonlib/test/offline.cpp +++ b/tonlib/test/offline.cpp @@ -659,11 +659,12 @@ TEST(Tonlib, ConfigCache) { ], "validator": { "@type": "validator.config.global", - "zero_state": { + "init_block": { "workchain": -1, "shard": -9223372036854775808, "seqno": 0, "file_hash": "eh9yveSz1qMdJ7mOsO+I+H77jkLr9NpAuEkoJuseXBo=" + "root_hash": "ZXSXxDHhTALFxReyTZRd8E4Ya3ySOmpOWAS4rBX9XBY=", } } })abc"; diff --git a/tonlib/tonlib/Config.cpp b/tonlib/tonlib/Config.cpp index 0f317f94ae..063b34bbdf 100644 --- a/tonlib/tonlib/Config.cpp +++ b/tonlib/tonlib/Config.cpp @@ -20,6 +20,7 @@ #include "adnl/adnl-node-id.hpp" #include "td/utils/JsonBuilder.h" #include "auto/tl/ton_api_json.h" +#include "ton/ton-tl.hpp" namespace tonlib { td::Result parse_block_id_ext(td::JsonObject &obj) { @@ -64,41 +65,26 @@ td::Result parse_block_id_ext(td::JsonObject &obj) { td::Result Config::parse(std::string str) { TRY_RESULT(json, td::json_decode(str)); if (json.type() != td::JsonValue::Type::Object) { - return td::Status::Error("Invalid config (1)"); + return td::Status::Error("Invalid config: json is not an object"); } - Config res; ton::ton_api::liteclient_config_global conf; TRY_STATUS(ton::ton_api::from_json(conf, json.get_object())); TRY_RESULT_ASSIGN(res.lite_servers, liteclient::LiteServerConfig::parse_global_config(conf)); - TRY_RESULT(validator_obj, - td::get_json_object_field(json.get_object(), "validator", td::JsonValue::Type::Object, false)); - auto &validator = validator_obj.get_object(); - TRY_RESULT(validator_type, td::get_json_object_string_field(validator, "@type", false)); - if (validator_type != "validator.config.global") { - return td::Status::Error("Invalid config (7)"); + if (!conf.validator_) { + return td::Status::Error("Invalid config: no 'validator' section"); + } + if (!conf.validator_->zero_state_) { + return td::Status::Error("Invalid config: no zerostate"); } - TRY_RESULT(zero_state_obj, td::get_json_object_field(validator, "zero_state", td::JsonValue::Type::Object, false)); - TRY_RESULT(zero_state_id, parse_block_id_ext(zero_state_obj.get_object())); - res.zero_state_id = zero_state_id; - auto r_init_block_obj = td::get_json_object_field(validator, "init_block", td::JsonValue::Type::Object, false); - if (r_init_block_obj.is_ok()) { - TRY_RESULT(init_block_id, parse_block_id_ext(r_init_block_obj.move_as_ok().get_object())); - res.init_block_id = init_block_id; + res.zero_state_id = ton::create_block_id(conf.validator_->zero_state_); + if (conf.validator_->init_block_) { + res.init_block_id = ton::create_block_id(conf.validator_->init_block_); } - auto r_hardforks = td::get_json_object_field(validator, "hardforks", td::JsonValue::Type::Array, false); - if (r_hardforks.is_ok()) { - auto hardforks_obj = r_hardforks.move_as_ok(); - auto &hardforks = hardforks_obj.get_array(); - for (auto &fork : hardforks) { - if (fork.type() != td::JsonValue::Type::Object) { - return td::Status::Error("Invalid config (8)"); - } - TRY_RESULT(fork_block, parse_block_id_ext(fork.get_object())); - res.hardforks.push_back(std::move(fork_block)); - } + for (auto &fork : conf.validator_->hardforks_) { + res.hardforks.push_back(ton::create_block_id(fork)); } for (auto hardfork : res.hardforks) { From 3a8ef60bbb4c4dbf725eee9a2a8d3bba286a08ca Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sat, 13 Jul 2024 15:40:26 +0300 Subject: [PATCH 099/524] Fast-sync overlays * Semiprivate overlays * Fast sync overlays in full-node * Adjust shard overlays --- CMakeLists.txt | 2 + create-hardfork/create-hardfork.cpp | 4 - overlay/overlay-broadcast.cpp | 5 +- overlay/overlay-fec-broadcast.cpp | 2 +- overlay/overlay-id.hpp | 100 ++- overlay/overlay-manager.cpp | 277 ++++++-- overlay/overlay-manager.h | 25 +- overlay/overlay-peers.cpp | 590 +++++++++++++++--- overlay/overlay.cpp | 315 ++++++---- overlay/overlay.h | 40 +- overlay/overlay.hpp | 237 ++++--- overlay/overlays.h | 127 +++- test/test-overlay.cpp | 450 +++++++++++++ test/test-ton-collator.cpp | 4 - tl/generate/scheme/ton_api.tl | 23 +- tl/generate/scheme/ton_api.tlo | Bin 99024 -> 101484 bytes .../validator-engine-console-query.cpp | 48 ++ .../validator-engine-console-query.h | 52 ++ .../validator-engine-console.cpp | 2 + validator-engine/validator-engine.cpp | 110 +++- validator-engine/validator-engine.hpp | 6 + validator/CMakeLists.txt | 4 +- validator/full-node-fast-sync-overlays.cpp | 416 ++++++++++++ ...2.hpp => full-node-fast-sync-overlays.hpp} | 70 +-- validator/full-node-private-overlay-v2.cpp | 384 ------------ validator/full-node-shard.cpp | 6 +- validator/full-node.cpp | 60 +- validator/full-node.h | 3 + validator/full-node.hpp | 23 +- validator/interfaces/validator-manager.h | 3 +- validator/manager-disk.hpp | 2 - validator/manager-hardfork.hpp | 3 - validator/manager-init.cpp | 1 + validator/manager.cpp | 36 +- validator/manager.hpp | 3 +- validator/shard-client.cpp | 64 +- validator/shard-client.hpp | 4 - validator/validator.h | 6 +- 38 files changed, 2544 insertions(+), 963 deletions(-) create mode 100644 test/test-overlay.cpp create mode 100644 validator/full-node-fast-sync-overlays.cpp rename validator/{full-node-private-overlay-v2.hpp => full-node-fast-sync-overlays.hpp} (59%) delete mode 100644 validator/full-node-private-overlay-v2.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b92ff6f1b0..70cb8b8d8f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -529,6 +529,8 @@ target_link_libraries(test-rldp2 adnl adnltest dht rldp2 tl_api) add_executable(test-validator-session-state test/test-validator-session-state.cpp) target_link_libraries(test-validator-session-state adnl dht rldp validatorsession tl_api) +add_executable(test-overlay test/test-overlay.cpp) +target_link_libraries(test-overlay overlay tdutils tdactor adnl adnltest tl_api dht ) add_executable(test-catchain test/test-catchain.cpp) target_link_libraries(test-catchain overlay tdutils tdactor adnl adnltest rldp tl_api dht catchain ) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index bb590960ca..32aa01c324 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -236,10 +236,6 @@ class HardforkCreator : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void update_shard_configuration(td::Ref state, - std::set shards_to_monitor, - std::set temporary_shards) override { - } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } void send_ext_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { diff --git a/overlay/overlay-broadcast.cpp b/overlay/overlay-broadcast.cpp index 03991b76b4..615b3e7c81 100644 --- a/overlay/overlay-broadcast.cpp +++ b/overlay/overlay-broadcast.cpp @@ -68,7 +68,7 @@ td::Status BroadcastSimple::run_checks() { td::Status BroadcastSimple::distribute() { auto B = serialize(); - auto nodes = overlay_->get_neighbours(3); + auto nodes = overlay_->get_neighbours(overlay_->propagate_broadcast_to()); auto manager = overlay_->overlay_manager(); for (auto &n : nodes) { @@ -115,7 +115,8 @@ td::Status BroadcastSimple::run() { return run_continue(); } -td::Status BroadcastSimple::create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr broadcast) { +td::Status BroadcastSimple::create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + tl_object_ptr broadcast) { auto src = PublicKey{broadcast->src_}; auto data_hash = sha256_bits256(broadcast->data_.as_slice()); auto broadcast_hash = compute_broadcast_id(src, data_hash, broadcast->flags_); diff --git a/overlay/overlay-fec-broadcast.cpp b/overlay/overlay-fec-broadcast.cpp index cd030742ac..5a0ad10ddf 100644 --- a/overlay/overlay-fec-broadcast.cpp +++ b/overlay/overlay-fec-broadcast.cpp @@ -112,7 +112,7 @@ td::Status BroadcastFec::distribute_part(td::uint32 seqno) { td::BufferSlice data_short = std::move(tls.first); td::BufferSlice data = std::move(tls.second); - auto nodes = overlay_->get_neighbours(5); + auto nodes = overlay_->get_neighbours(overlay_->propagate_broadcast_to()); auto manager = overlay_->overlay_manager(); for (auto &n : nodes) { diff --git a/overlay/overlay-id.hpp b/overlay/overlay-id.hpp index c22b0faa74..2625773bd7 100644 --- a/overlay/overlay-id.hpp +++ b/overlay/overlay-id.hpp @@ -21,8 +21,14 @@ #include "auto/tl/ton_api.h" #include "adnl/adnl-node-id.hpp" #include "overlay/overlays.h" +#include "td/utils/SharedSlice.h" +#include "td/utils/buffer.h" #include "td/utils/overloaded.h" #include "keys/encryptor.h" +#include "td/utils/port/StdStreams.h" +#include "td/utils/unique_ptr.h" +#include +#include namespace ton { @@ -30,18 +36,30 @@ namespace overlay { class OverlayNode { public: - explicit OverlayNode(adnl::AdnlNodeIdShort self_id, OverlayIdShort overlay) { + explicit OverlayNode(adnl::AdnlNodeIdShort self_id, OverlayIdShort overlay, td::uint32 flags) { source_ = self_id; overlay_ = overlay; + flags_ = flags; version_ = static_cast(td::Clocks::system()); } static td::Result create(const tl_object_ptr &node) { TRY_RESULT(source, adnl::AdnlNodeIdFull::create(node->id_)); - return OverlayNode{source, OverlayIdShort{node->overlay_}, node->version_, node->signature_.as_slice()}; + return OverlayNode{source, OverlayIdShort{node->overlay_}, 0, node->version_, node->signature_.as_slice()}; } - OverlayNode(td::Variant source, OverlayIdShort overlay, + static td::Result create(const tl_object_ptr &node) { + TRY_RESULT(source, adnl::AdnlNodeIdFull::create(node->id_)); + auto res = OverlayNode{source, OverlayIdShort{node->overlay_}, (td::uint32)node->flags_, node->version_, + node->signature_.as_slice()}; + res.update_certificate(OverlayMemberCertificate(node->certificate_.get())); + return res; + } + OverlayNode(td::Variant source, OverlayIdShort overlay, td::uint32 flags, td::int32 version, td::Slice signature) - : source_(std::move(source)), overlay_(overlay), version_(version), signature_(td::SharedSlice(signature)) { + : source_(std::move(source)) + , overlay_(overlay) + , flags_(flags) + , version_(version) + , signature_(td::SharedSlice(signature)) { } OverlayNode(td::Variant source, OverlayIdShort overlay, td::int32 version, td::SharedSlice signature) @@ -64,10 +82,17 @@ class OverlayNode { } td::BufferSlice to_sign() const { - auto obj = create_tl_object(nullptr, overlay_.tl(), version_); - source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { obj->id_ = id.tl(); }, - [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.compute_short_id().tl(); })); - return serialize_tl_object(obj, true); + if (flags_ == 0) { + auto obj = create_tl_object(nullptr, overlay_.tl(), version_); + source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { obj->id_ = id.tl(); }, + [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.compute_short_id().tl(); })); + return serialize_tl_object(obj, true); + } else { + auto obj = create_tl_object(nullptr, overlay_.tl(), flags_, version_); + source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { obj->id_ = id.tl(); }, + [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.compute_short_id().tl(); })); + return serialize_tl_object(obj, true); + } } void update_adnl_id(adnl::AdnlNodeIdFull node_id) { source_ = node_id; @@ -81,6 +106,9 @@ class OverlayNode { td::int32 version() const { return version_; } + td::uint32 flags() const { + return flags_; + } td::BufferSlice signature() const { return signature_.clone_as_buffer_slice(); } @@ -103,15 +131,69 @@ class OverlayNode { [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.tl(); })); return obj; } + tl_object_ptr tl_v2() const { + tl_object_ptr cert; + if (cert_ && !cert_->empty()) { + cert = cert_->tl(); + } else { + cert = create_tl_object(); + } + auto obj = create_tl_object(nullptr, overlay_.tl(), flags_, version_, + signature_.clone_as_buffer_slice(), std::move(cert)); + source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { UNREACHABLE(); }, + [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.tl(); })); + return obj; + } OverlayNode clone() const { - return OverlayNode{source_, overlay_, version_, signature_.clone()}; + auto res = OverlayNode{source_, overlay_, version_, signature_.clone()}; + if (cert_) { + res.cert_ = td::make_unique(*cert_); + } + return res; + } + + const OverlayMemberCertificate *certificate() const { + if (cert_) { + return cert_.get(); + } + return &empty_certificate_; + } + + void update_certificate(OverlayMemberCertificate cert) { + if (!cert_ || cert_->empty() || cert_->is_expired() || cert.is_newer(*cert_)) { + cert_ = td::make_unique(std::move(cert)); + } + } + + void update(OverlayNode from) { + if (version_ < from.version_) { + source_ = from.source_; + overlay_ = from.overlay_; + flags_ = from.flags_; + version_ = from.version_; + signature_ = from.signature_.clone(); + } + if (from.cert_ && !from.cert_->empty()) { + update_certificate(std::move(*from.cert_)); + } + } + + void clear_certificate() { + cert_ = nullptr; + } + + bool has_full_id() const { + return source_.get_offset() == source_.offset(); } private: td::Variant source_; OverlayIdShort overlay_; + td::uint32 flags_; td::int32 version_; + td::unique_ptr cert_; td::SharedSlice signature_; + static const OverlayMemberCertificate empty_certificate_; }; } // namespace overlay diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index e10557615c..878f6637a7 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -18,6 +18,7 @@ */ #include "overlay-manager.h" #include "auto/tl/ton_api.h" +#include "auto/tl/ton_api.hpp" #include "overlay.h" #include "adnl/utils.hpp" @@ -28,6 +29,7 @@ #include "td/db/RocksDb.h" #include "td/utils/Status.h" +#include "td/utils/buffer.h" #include "td/utils/overloaded.h" #include "td/utils/port/Poll.h" @@ -41,13 +43,13 @@ void OverlayManager::update_dht_node(td::actor::ActorId dht) { dht_node_ = dht; for (auto &X : overlays_) { for (auto &Y : X.second) { - td::actor::send_closure(Y.second, &Overlay::update_dht_node, dht); + td::actor::send_closure(Y.second.overlay, &Overlay::update_dht_node, dht); } } } void OverlayManager::register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, - td::actor::ActorOwn overlay) { + OverlayMemberCertificate cert, td::actor::ActorOwn overlay) { auto it = overlays_.find(local_id); VLOG(OVERLAY_INFO) << this << ": registering overlay " << overlay_id << "@" << local_id; if (it == overlays_.end()) { @@ -57,19 +59,34 @@ void OverlayManager::register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdS td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id, adnl::Adnl::int_to_bytestring(ton_api::overlay_query::ID), std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id, + adnl::Adnl::int_to_bytestring(ton_api::overlay_messageWithExtra::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id, + adnl::Adnl::int_to_bytestring(ton_api::overlay_queryWithExtra::ID), + std::make_unique(actor_id(this))); } - overlays_[local_id][overlay_id] = std::move(overlay); - - auto P = td::PromiseCreator::lambda([id = overlays_[local_id][overlay_id].get()](td::Result R) { - R.ensure(); - auto value = R.move_as_ok(); - if (value.status == td::KeyValue::GetStatus::Ok) { - auto F = fetch_tl_object(std::move(value.value), true); - F.ensure(); - auto nodes = std::move(F.move_as_ok()->nodes_); - td::actor::send_closure(id, &Overlay::receive_nodes_from_db, std::move(nodes)); - } - }); + overlays_[local_id][overlay_id] = OverlayDescription{std::move(overlay), std::move(cert)}; + + auto P = + td::PromiseCreator::lambda([id = overlays_[local_id][overlay_id].overlay.get()](td::Result R) { + R.ensure(); + auto value = R.move_as_ok(); + if (value.status == td::KeyValue::GetStatus::Ok) { + auto F = fetch_tl_object(std::move(value.value), true); + F.ensure(); + ton_api::downcast_call( + *F.move_as_ok(), td::overloaded( + [&](ton_api::overlay_db_nodes &V) { + auto nodes = std::move(V.nodes_); + td::actor::send_closure(id, &Overlay::receive_nodes_from_db, std::move(nodes)); + }, + [&](ton_api::overlay_db_nodesV2 &V) { + auto nodes = std::move(V.nodes_); + td::actor::send_closure(id, &Overlay::receive_nodes_from_db_v2, std::move(nodes)); + })); + } + }); auto key = create_hash_tl_object(local_id.bits256_value(), overlay_id.bits256_value()); db_.get(key, std::move(P)); } @@ -83,6 +100,10 @@ void OverlayManager::delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdSho adnl::Adnl::int_to_bytestring(ton_api::overlay_message::ID)); td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id, adnl::Adnl::int_to_bytestring(ton_api::overlay_query::ID)); + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id, + adnl::Adnl::int_to_bytestring(ton_api::overlay_messageWithExtra::ID)); + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id, + adnl::Adnl::int_to_bytestring(ton_api::overlay_queryWithExtra::ID)); overlays_.erase(it); } } @@ -100,74 +121,113 @@ void OverlayManager::create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, Ov td::string scope, OverlayOptions opts) { CHECK(!dht_node_.empty()); auto id = overlay_id.compute_short_id(); - register_overlay(local_id, id, - Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), - std::move(callback), std::move(rules), scope, std::move(opts))); + register_overlay(local_id, id, OverlayMemberCertificate{}, + Overlay::create_public(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), + std::move(callback), std::move(rules), scope, std::move(opts))); } void OverlayManager::create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules, std::string scope) { + create_private_overlay_ex(local_id, std::move(overlay_id), std::move(nodes), std::move(callback), std::move(rules), + std::move(scope), {}); +} + +void OverlayManager::create_private_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, + std::unique_ptr callback, OverlayPrivacyRules rules, + std::string scope, OverlayOptions opts) { auto id = overlay_id.compute_short_id(); - register_overlay(local_id, id, - Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), - std::move(nodes), std::move(callback), std::move(rules), std::move(scope))); + register_overlay(local_id, id, OverlayMemberCertificate{}, + Overlay::create_private(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), + std::move(nodes), std::move(callback), std::move(rules), std::move(scope), + std::move(opts))); +} + +void OverlayManager::create_semiprivate_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, + std::vector root_public_keys, + OverlayMemberCertificate certificate, + std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope, OverlayOptions opts) { + auto id = overlay_id.compute_short_id(); + register_overlay( + local_id, id, certificate, + Overlay::create_semiprivate(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), + std::move(nodes), std::move(root_public_keys), certificate, std::move(callback), + std::move(rules), std::move(scope), std::move(opts))); } void OverlayManager::receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) { - auto R = fetch_tl_prefix(data, true); - - if (R.is_error()) { - VLOG(OVERLAY_WARNING) << this << ": can not parse overlay message: " << R.move_as_error(); - return; + OverlayIdShort overlay_id; + tl_object_ptr extra; + auto R = fetch_tl_prefix(data, true); + if (R.is_ok()) { + overlay_id = OverlayIdShort{R.ok()->overlay_}; + extra = std::move(R.ok()->extra_); + } else { + auto R2 = fetch_tl_prefix(data, true); + if (R2.is_ok()) { + overlay_id = OverlayIdShort{R2.ok()->overlay_}; + } else { + VLOG(OVERLAY_WARNING) << this << ": can not parse overlay message [" << src << "->" << dst + << "]: " << R2.move_as_error(); + return; + } } - auto M = R.move_as_ok(); - auto it = overlays_.find(dst); if (it == overlays_.end()) { - VLOG(OVERLAY_NOTICE) << this << ": message to unknown overlay " << M->overlay_ << "@" << dst; + VLOG(OVERLAY_NOTICE) << this << ": message to unknown overlay " << overlay_id << "@" << dst; return; } - auto it2 = it->second.find(OverlayIdShort{M->overlay_}); + auto it2 = it->second.find(overlay_id); if (it2 == it->second.end()) { - VLOG(OVERLAY_NOTICE) << this << ": message to localid is not in overlay " << M->overlay_ << "@" << dst; + VLOG(OVERLAY_NOTICE) << this << ": message to localid is not in overlay " << overlay_id << "@" << dst; return; } - td::actor::send_closure(it2->second, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), false); - td::actor::send_closure(it2->second, &Overlay::receive_message, src, std::move(data)); + td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), false); + td::actor::send_closure(it2->second.overlay, &Overlay::receive_message, src, std::move(extra), std::move(data)); } void OverlayManager::receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { - auto R = fetch_tl_prefix(data, true); - - if (R.is_error()) { - VLOG(OVERLAY_WARNING) << this << ": can not parse overlay query [" << src << "->" << dst - << "]: " << R.move_as_error(); - promise.set_error(td::Status::Error(ErrorCode::protoviolation, "bad overlay query header")); - return; + OverlayIdShort overlay_id; + tl_object_ptr extra; + auto R = fetch_tl_prefix(data, true); + if (R.is_ok()) { + overlay_id = OverlayIdShort{R.ok()->overlay_}; + extra = std::move(R.ok()->extra_); + } else { + auto R2 = fetch_tl_prefix(data, true); + if (R2.is_ok()) { + overlay_id = OverlayIdShort{R2.ok()->overlay_}; + } else { + VLOG(OVERLAY_WARNING) << this << ": can not parse overlay query [" << src << "->" << dst + << "]: " << R2.move_as_error(); + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "bad overlay query header")); + return; + } } - auto M = R.move_as_ok(); - auto it = overlays_.find(dst); if (it == overlays_.end()) { - VLOG(OVERLAY_NOTICE) << this << ": query to unknown overlay " << M->overlay_ << "@" << dst << " from " << src; + VLOG(OVERLAY_NOTICE) << this << ": query to unknown overlay " << overlay_id << "@" << dst << " from " << src; promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad local_id " << dst)); return; } - auto it2 = it->second.find(OverlayIdShort{M->overlay_}); + auto it2 = it->second.find(overlay_id); if (it2 == it->second.end()) { - VLOG(OVERLAY_NOTICE) << this << ": query to localid not in overlay " << M->overlay_ << "@" << dst << " from " << src; - promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad overlay_id " << M->overlay_)); + VLOG(OVERLAY_NOTICE) << this << ": query to localid not in overlay " << overlay_id << "@" << dst << " from " << src; + promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad overlay_id " << overlay_id)); return; } - td::actor::send_closure(it2->second, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), true); - td::actor::send_closure(it2->second, &Overlay::receive_query, src, std::move(data), std::move(promise)); + td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), true); + td::actor::send_closure(it2->second.overlay, &Overlay::receive_query, src, std::move(extra), std::move(data), + std::move(promise)); } void OverlayManager::send_query_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, @@ -175,35 +235,64 @@ void OverlayManager::send_query_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdS td::BufferSlice query, td::uint64 max_answer_size, td::actor::ActorId via) { CHECK(query.size() <= adnl::Adnl::huge_packet_max_size()); - + + auto extra = create_tl_object(); + extra->flags_ = 0; + auto it = overlays_.find(src); if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::update_throughput_out_ctr, dst, (td::uint32)query.size(), true); + td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_out_ctr, dst, (td::uint32)query.size(), + true); + if (!it2->second.member_certificate.empty()) { + extra->flags_ |= 1; + extra->certificate_ = it2->second.member_certificate.tl(); + } } } - - td::actor::send_closure( - via, &adnl::AdnlSenderInterface::send_query_ex, src, dst, std::move(name), std::move(promise), timeout, - create_serialize_tl_object_suffix(query.as_slice(), overlay_id.tl()), max_answer_size); + + auto extra_flags = extra->flags_; + td::BufferSlice serialized_query = + (extra_flags ? create_serialize_tl_object_suffix( + query.as_slice(), overlay_id.tl(), std::move(extra)) + : create_serialize_tl_object_suffix(query.as_slice(), overlay_id.tl())); + + td::actor::send_closure(via, &adnl::AdnlSenderInterface::send_query_ex, src, dst, std::move(name), std::move(promise), + timeout, std::move(serialized_query), max_answer_size); } void OverlayManager::send_message_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, td::BufferSlice object, td::actor::ActorId via) { CHECK(object.size() <= adnl::Adnl::huge_packet_max_size()); - + + auto extra = create_tl_object(); + extra->flags_ = 0; + auto it = overlays_.find(src); if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::update_throughput_out_ctr, dst, (td::uint32)object.size(), false); + td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_out_ctr, dst, (td::uint32)object.size(), + false); + if (!it2->second.member_certificate.empty()) { + // do not send certificate here, we hope that all our neighbours already know of out certificate + // we send it every second to some random nodes. Here we don't want to increase the size of the message + if (false) { + extra->flags_ |= 1; + extra->certificate_ = it2->second.member_certificate.tl(); + } + } } } - - td::actor::send_closure( - via, &adnl::AdnlSenderInterface::send_message, src, dst, - create_serialize_tl_object_suffix(object.as_slice(), overlay_id.tl())); + + auto extra_flags = extra->flags_; + td::BufferSlice serialized_message = + (extra_flags ? create_serialize_tl_object_suffix( + object.as_slice(), overlay_id.tl(), std::move(extra)) + : create_serialize_tl_object_suffix(object.as_slice(), overlay_id.tl())); + + td::actor::send_closure(via, &adnl::AdnlSenderInterface::send_message, src, dst, std::move(serialized_message)); } void OverlayManager::send_broadcast(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, td::BufferSlice object) { @@ -217,7 +306,7 @@ void OverlayManager::send_broadcast_ex(adnl::AdnlNodeIdShort local_id, OverlayId if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::send_broadcast, send_as, flags, std::move(object)); + td::actor::send_closure(it2->second.overlay, &Overlay::send_broadcast, send_as, flags, std::move(object)); } } } @@ -234,7 +323,7 @@ void OverlayManager::send_broadcast_fec_ex(adnl::AdnlNodeIdShort local_id, Overl if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::send_broadcast_fec, send_as, flags, std::move(object)); + td::actor::send_closure(it2->second.overlay, &Overlay::send_broadcast_fec, send_as, flags, std::move(object)); } } } @@ -245,7 +334,7 @@ void OverlayManager::set_privacy_rules(adnl::AdnlNodeIdShort local_id, OverlayId if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::set_privacy_rules, std::move(rules)); + td::actor::send_closure(it2->second.overlay, &Overlay::set_privacy_rules, std::move(rules)); } } } @@ -256,7 +345,34 @@ void OverlayManager::update_certificate(adnl::AdnlNodeIdShort local_id, OverlayI if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::add_certificate, key, std::move(cert)); + td::actor::send_closure(it2->second.overlay, &Overlay::add_certificate, key, std::move(cert)); + } + } +} + +void OverlayManager::update_member_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + OverlayMemberCertificate certificate) { + auto it = overlays_.find(local_id); + if (it != overlays_.end()) { + auto it2 = it->second.find(overlay_id); + if (it2 != it->second.end()) { + it2->second.member_certificate = certificate; + td::actor::send_closure(it2->second.overlay, &Overlay::update_member_certificate, certificate); + } + } +} + +void OverlayManager::update_root_member_list(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + std::vector nodes, + std::vector root_public_keys, + OverlayMemberCertificate certificate) { + auto it = overlays_.find(local_id); + if (it != overlays_.end()) { + auto it2 = it->second.find(overlay_id); + if (it2 != it->second.end()) { + it2->second.member_certificate = certificate; + td::actor::send_closure(it2->second.overlay, &Overlay::update_root_member_list, std::move(nodes), + std::move(root_public_keys), std::move(certificate)); } } } @@ -274,7 +390,7 @@ void OverlayManager::get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, Ov promise.set_error(td::Status::Error(PSTRING() << "no such overlay " << overlay_id)); return; } - td::actor::send_closure(it2->second, &Overlay::get_overlay_random_peers, max_peers, std::move(promise)); + td::actor::send_closure(it2->second.overlay, &Overlay::get_overlay_random_peers, max_peers, std::move(promise)); } td::actor::ActorOwn Overlays::create(std::string db_root, td::actor::ActorId keyring, @@ -337,7 +453,7 @@ void OverlayManager::get_stats(td::Promise> R) { if (R.is_ok()) { td::actor::send_closure(act, &Cb::receive_answer, R.move_as_ok()); @@ -361,7 +477,7 @@ void OverlayManager::forget_peer(adnl::AdnlNodeIdShort local_id, OverlayIdShort if (it2 == it->second.end()) { return; } - td::actor::send_closure(it2->second, &Overlay::forget_peer, peer_id); + td::actor::send_closure(it2->second.overlay, &Overlay::forget_peer, peer_id); } Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, @@ -470,6 +586,35 @@ tl_object_ptr Certificate::empty_tl() { return create_tl_object(); } +OverlayMemberCertificate::OverlayMemberCertificate(const ton_api::overlay_MemberCertificate *cert) { + if (!cert) { + expire_at_ = std::numeric_limits::max(); + return; + } + if (cert->get_id() == ton_api::overlay_emptyMemberCertificate::ID) { + expire_at_ = std::numeric_limits::max(); + return; + } + CHECK(cert->get_id() == ton_api::overlay_memberCertificate::ID); + const auto *real_cert = static_cast(cert); + signed_by_ = PublicKey(real_cert->issued_by_); + flags_ = real_cert->flags_; + slot_ = real_cert->slot_; + expire_at_ = real_cert->expire_at_; + signature_ = td::SharedSlice(real_cert->signature_.as_slice()); +} + +td::Status OverlayMemberCertificate::check_signature(const adnl::AdnlNodeIdShort &node) { + if (is_expired()) { + return td::Status::Error(ErrorCode::notready, "certificate is expired"); + } + td::BufferSlice data_to_sign = to_sign_data(node); + + TRY_RESULT(encryptor, signed_by_.create_encryptor()); + TRY_STATUS(encryptor->check_signature(data_to_sign.as_slice(), signature_.as_slice())); + return td::Status::OK(); +} + } // namespace overlay } // namespace ton diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index 2e25c944ee..12206e0481 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -53,11 +53,19 @@ class OverlayManager : public Overlays { void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) override; void create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, - OverlayOptions opts) override; + std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, + OverlayOptions opts) override; + void create_semiprivate_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate certificate, + std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, + OverlayOptions opts) override; void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules, std::string scope) override; + void create_private_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, std::unique_ptr callback, + OverlayPrivacyRules rules, std::string scope, OverlayOptions opts) override; void delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id) override; void send_query(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice query) override { @@ -84,6 +92,11 @@ class OverlayManager : public Overlays { void set_privacy_rules(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, OverlayPrivacyRules rules) override; void update_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, PublicKeyHash key, std::shared_ptr cert) override; + void update_member_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + OverlayMemberCertificate certificate) override; + void update_root_member_list(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate certificate) override; void get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, td::uint32 max_peers, td::Promise> promise) override; @@ -92,7 +105,7 @@ class OverlayManager : public Overlays { td::Promise promise); void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data); - void register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + void register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, OverlayMemberCertificate cert, td::actor::ActorOwn overlay); void get_stats(td::Promise> promise) override; @@ -105,7 +118,11 @@ class OverlayManager : public Overlays { } private: - std::map>> overlays_; + struct OverlayDescription { + td::actor::ActorOwn overlay; + OverlayMemberCertificate member_certificate; + }; + std::map> overlays_; std::string db_root_; diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index 77fd1bbe36..c61e49652e 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -16,85 +16,159 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "adnl/adnl-node-id.hpp" +#include "adnl/adnl-node.h" +#include "auto/tl/ton_api.h" #include "overlay.hpp" +#include "td/utils/Status.h" +#include "td/utils/Time.h" +#include "td/utils/port/signals.h" +#include +#include namespace ton { namespace overlay { -void OverlayImpl::del_peer(adnl::AdnlNodeIdShort id) { - auto P = peers_.get(id); +void OverlayImpl::del_peer(const adnl::AdnlNodeIdShort &id) { + auto P = peer_list_.peers_.get(id); if (P == nullptr) { return; } + if (P->is_permanent_member()) { + VLOG(OVERLAY_DEBUG) << this << ": not deleting peer " << id << ": a permanent member"; + return; + } VLOG(OVERLAY_DEBUG) << this << ": deleting peer " << id; if (P->is_neighbour()) { - VLOG(OVERLAY_INFO) << this << ": deleting neighbour " << id; - bool deleted = false; - for (auto &n : neighbours_) { - if (n == id) { - n = neighbours_[neighbours_.size() - 1]; - neighbours_.resize(neighbours_.size() - 1); - deleted = true; - break; - } + del_from_neighbour_list(P); + } + peer_list_.peers_.remove(id); + peer_list_.bad_peers_.erase(id); +} + +void OverlayImpl::del_from_neighbour_list(OverlayPeer *P) { + CHECK(P); + if (!P->is_neighbour()) { + return; + } + auto id = P->get_id(); + bool deleted = false; + auto &neighbours = peer_list_.neighbours_; + for (auto &n : neighbours) { + if (n == id) { + n = neighbours[neighbours.size() - 1]; + neighbours.resize(neighbours.size() - 1); + deleted = true; + break; } - CHECK(deleted); - P->set_neighbour(false); } - peers_.remove(id); - bad_peers_.erase(id); - update_neighbours(0); + CHECK(deleted); + P->set_neighbour(false); +} + +void OverlayImpl::del_from_neighbour_list(const adnl::AdnlNodeIdShort &id) { + auto P = peer_list_.peers_.get(id); + CHECK(P != nullptr); + return del_from_neighbour_list(P); } void OverlayImpl::del_some_peers() { - if (!public_) { + if (overlay_type_ == OverlayType::FixedMemberList) { return; } - while (peers_.size() > max_peers()) { + const size_t max_iterations = 10; + size_t iteration_seqno = 0; + while (peer_list_.peers_.size() > max_peers() && iteration_seqno++ < max_iterations) { OverlayPeer *P; - if (bad_peers_.empty()) { + if (peer_list_.bad_peers_.empty()) { P = get_random_peer(); } else { - auto it = bad_peers_.upper_bound(next_bad_peer_); - if (it == bad_peers_.end()) { - it = bad_peers_.begin(); + auto it = peer_list_.bad_peers_.upper_bound(peer_list_.next_bad_peer_); + if (it == peer_list_.bad_peers_.end()) { + it = peer_list_.bad_peers_.begin(); } - P = peers_.get(next_bad_peer_ = *it); + P = peer_list_.peers_.get(peer_list_.next_bad_peer_ = *it); } - if (P) { + if (P && !P->is_permanent_member()) { auto id = P->get_id(); del_peer(id); } } + update_neighbours(0); } -void OverlayImpl::do_add_peer(OverlayNode node) { - auto id = node.adnl_id_short(); - - auto V = peers_.get(id); - if (V) { - VLOG(OVERLAY_DEBUG) << this << ": updating peer " << id << " up to version " << node.version(); - V->update(std::move(node)); - } else { - VLOG(OVERLAY_DEBUG) << this << ": adding peer " << id << " of version " << node.version(); - peers_.insert(id, OverlayPeer(std::move(node))); - - del_some_peers(); - update_neighbours(0); +td::Status OverlayImpl::validate_peer_certificate(const adnl::AdnlNodeIdShort &node, + const OverlayMemberCertificate &cert) { + if (cert.empty()) { + if (is_persistent_node(node) || overlay_type_ == OverlayType::Public) { + return td::Status::OK(); + } + return td::Status::Error(ErrorCode::protoviolation, "no member certificate found"); + } + if (cert.is_expired()) { + return td::Status::Error(ErrorCode::timeout, "member certificate is expired"); + } + if (cert.slot() < 0 || cert.slot() >= opts_.max_slaves_in_semiprivate_overlay_) { + return td::Status::Error(ErrorCode::timeout, "member certificate has invalid slot"); } + const auto &issued_by = cert.issued_by(); + auto it = peer_list_.root_public_keys_.find(issued_by.compute_short_id()); + if (it == peer_list_.root_public_keys_.end()) { + return td::Status::Error(ErrorCode::protoviolation, "member certificate is signed by unknown public key"); + } + if (it->second.size() > (size_t)cert.slot()) { + auto &el = it->second[cert.slot()]; + if (cert.expire_at() < el.expire_at) { + return td::Status::Error(ErrorCode::protoviolation, + "member certificate rejected, because we know of newer certificate at the same slot"); + } else if (cert.expire_at() == el.expire_at) { + if (node < el.node) { + return td::Status::Error(ErrorCode::protoviolation, + "member certificate rejected, because we know of newer certificate at the same slot"); + } else if (el.node == node) { + // we could return OK here, but we must make sure, that the unchecked signature will not be used for updating PeerNode. + } + } + } + auto R = get_encryptor(issued_by); + if (R.is_error()) { + return R.move_as_error_prefix("failed to check member certificate: failed to create encryptor: "); + } + auto enc = R.move_as_ok(); + auto S = enc->check_signature(cert.to_sign_data(node).as_slice(), cert.signature()); + if (S.is_error()) { + return S.move_as_error_prefix("failed to check member certificate: bad signature: "); + } + if (it->second.size() <= (size_t)cert.slot()) { + it->second.resize((size_t)cert.slot() + 1); + } + it->second[cert.slot()].expire_at = cert.expire_at(); + it->second[cert.slot()].node = node; + return td::Status::OK(); } -void OverlayImpl::add_peer_in_cont(OverlayNode node) { - CHECK(public_); +td::Status OverlayImpl::validate_peer_certificate(const adnl::AdnlNodeIdShort &node, + ton_api::overlay_MemberCertificate *cert) { + OverlayMemberCertificate ncert(cert); + return validate_peer_certificate(node, ncert); +} - do_add_peer(std::move(node)); +td::Status OverlayImpl::validate_peer_certificate(const adnl::AdnlNodeIdShort &node, + const OverlayMemberCertificate *cert) { + if (!cert) { + if (is_persistent_node(node) || overlay_type_ == OverlayType::Public) { + return td::Status::OK(); + } + return td::Status::Error(ErrorCode::protoviolation, "no member certificate found"); + } + return validate_peer_certificate(node, *cert); } -void OverlayImpl::add_peer_in(OverlayNode node) { - CHECK(public_); +void OverlayImpl::add_peer(OverlayNode node) { + CHECK(overlay_type_ != OverlayType::FixedMemberList); if (node.overlay_id() != overlay_id_) { VLOG(OVERLAY_WARNING) << this << ": received node with bad overlay"; return; @@ -117,35 +191,82 @@ void OverlayImpl::add_peer_in(OverlayNode node) { return; } - add_peer_in_cont(std::move(node)); + if (overlay_type_ == OverlayType::CertificatedMembers) { + auto R = validate_peer_certificate(node.adnl_id_short(), *node.certificate()); + if (R.is_error()) { + VLOG(OVERLAY_WARNING) << this << ": bad peer certificate node=" << node.adnl_id_short() << ": " + << R.move_as_error(); + UNREACHABLE(); + return; + } + } + + auto id = node.adnl_id_short(); + + auto V = peer_list_.peers_.get(id); + if (V) { + VLOG(OVERLAY_DEBUG) << this << ": updating peer " << id << " up to version " << node.version(); + V->update(std::move(node)); + } else { + VLOG(OVERLAY_DEBUG) << this << ": adding peer " << id << " of version " << node.version(); + CHECK(overlay_type_ != OverlayType::CertificatedMembers || (node.certificate() && !node.certificate()->empty())); + peer_list_.peers_.insert(id, OverlayPeer(std::move(node))); + del_some_peers(); + auto X = peer_list_.peers_.get(id); + CHECK(X); + + if (peer_list_.neighbours_.size() < max_neighbours() && + !(X->get_node()->flags() & OverlayMemberFlags::DoNotReceiveBroadcasts)) { + peer_list_.neighbours_.push_back(X->get_id()); + X->set_neighbour(true); + } + + update_neighbours(0); + } } void OverlayImpl::add_peers(std::vector peers) { for (auto &node : peers) { - add_peer_in(std::move(node)); + add_peer(std::move(node)); } } -void OverlayImpl::add_peer(OverlayNode P) { - add_peer_in(std::move(P)); +void OverlayImpl::add_peers(const tl_object_ptr &nodes) { + for (auto &n : nodes->nodes_) { + auto N = OverlayNode::create(n); + if (N.is_ok()) { + add_peer(N.move_as_ok()); + } + } +} + +void OverlayImpl::add_peers(const tl_object_ptr &nodes) { + for (auto &n : nodes->nodes_) { + auto N = OverlayNode::create(n); + if (N.is_ok()) { + add_peer(N.move_as_ok()); + } + } } void OverlayImpl::on_ping_result(adnl::AdnlNodeIdShort peer, bool success) { - if (!public_) { + if (overlay_type_ == OverlayType::FixedMemberList) { return; } - if (OverlayPeer *p = peers_.get(peer)) { + if (OverlayPeer *p = peer_list_.peers_.get(peer)) { p->on_ping_result(success); - if (p->is_alive()) { - bad_peers_.erase(peer); - } else { - bad_peers_.insert(peer); + if (!p->is_permanent_member()) { + if (p->is_alive()) { + peer_list_.bad_peers_.erase(peer); + } else { + peer_list_.bad_peers_.insert(peer); + } } } } void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R) { - CHECK(public_); + CHECK(overlay_type_ != OverlayType::FixedMemberList); on_ping_result(src, R.is_ok()); if (R.is_error()) { VLOG(OVERLAY_NOTICE) << this << ": failed getRandomPeers query: " << R.move_as_error(); @@ -158,16 +279,24 @@ void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result nodes; - for (auto &n : res->nodes_) { - auto N = OverlayNode::create(n); - if (N.is_ok()) { - nodes.emplace_back(N.move_as_ok()); - } +void OverlayImpl::receive_random_peers_v2(adnl::AdnlNodeIdShort src, td::Result R) { + CHECK(overlay_type_ != OverlayType::FixedMemberList); + on_ping_result(src, R.is_ok()); + if (R.is_error()) { + VLOG(OVERLAY_NOTICE) << this << ": failed getRandomPeersV2 query: " << R.move_as_error(); + return; + } + auto R2 = fetch_tl_object(R.move_as_ok(), true); + if (R2.is_error()) { + VLOG(OVERLAY_WARNING) << this << ": dropping incorrect answer to overlay.getRandomPeers query from " << src << ": " + << R2.move_as_error(); + return; } - add_peers(std::move(nodes)); + + add_peers(R2.move_as_ok()); } void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode node, @@ -177,10 +306,13 @@ void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode vec.emplace_back(node.tl()); } - for (td::uint32 i = 0; i < nodes_to_send(); i++) { + td::uint32 max_iterations = nodes_to_send() + 16; + for (td::uint32 i = 0; i < max_iterations && vec.size() < nodes_to_send(); i++) { auto P = get_random_peer(true); if (P) { - vec.emplace_back(P->get().tl()); + if (P->has_full_id()) { + vec.emplace_back(P->get_node()->tl()); + } } else { break; } @@ -215,58 +347,110 @@ void OverlayImpl::send_random_peers(adnl::AdnlNodeIdShort src, td::Promise promise) { + std::vector> vec; + if (announce_self_) { + CHECK(is_persistent_node(node.adnl_id_short()) || !node.certificate()->empty()); + vec.emplace_back(node.tl_v2()); + } + + td::uint32 max_iterations = nodes_to_send() + 16; + for (td::uint32 i = 0; i < max_iterations && vec.size() < nodes_to_send(); i++) { + auto P = get_random_peer(true); + if (P) { + if (P->has_full_id() && !P->is_permanent_member()) { + vec.emplace_back(P->get_node()->tl_v2()); + } + } else { + break; + } + } + + if (promise) { + auto Q = create_tl_object(std::move(vec)); + promise.set_value(serialize_tl_object(Q, true)); + } else { + auto P = + td::PromiseCreator::lambda([SelfId = actor_id(this), src, oid = print_id()](td::Result res) { + td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers_v2, src, std::move(res)); + }); + auto Q = + create_tl_object(create_tl_object(std::move(vec))); + td::actor::send_closure(manager_, &OverlayManager::send_query, src, local_id_, overlay_id_, + "overlay getRandomPeers", std::move(P), + td::Timestamp::in(5.0 + td::Random::fast(0, 50) * 0.1), serialize_tl_object(Q, true)); + } +} + +void OverlayImpl::send_random_peers_v2(adnl::AdnlNodeIdShort src, td::Promise promise) { + auto P = td::PromiseCreator::lambda([src, promise = std::move(promise), + SelfId = actor_id(this)](td::Result res) mutable { + if (res.is_error()) { + promise.set_error(td::Status::Error(ErrorCode::error, "cannot get self node")); + return; + } + td::actor::send_closure(SelfId, &OverlayImpl::send_random_peers_v2_cont, src, res.move_as_ok(), std::move(promise)); + }); + + get_self_node(std::move(P)); +} + void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) { - if (peers_.size() == 0) { + if (peer_list_.peers_.size() == 0) { return; } td::uint32 iter = 0; - while (iter < 10 && (nodes_to_change > 0 || neighbours_.size() < max_neighbours())) { - auto X = peers_.get_random(); + while (iter++ < 10 && (nodes_to_change > 0 || peer_list_.neighbours_.size() < max_neighbours())) { + auto X = peer_list_.peers_.get_random(); if (!X) { break; } if (X->get_id() == local_id_) { - iter++; continue; } - if (public_ && X->get_version() <= td::Clocks::system() - Overlays::overlay_peer_ttl()) { + if (X->get_version() <= td::Clocks::system() - Overlays::overlay_peer_ttl()) { + if (X->is_permanent_member()) { + del_from_neighbour_list(X); + } else { + auto id = X->get_id(); + del_peer(id); + } + continue; + } + + if (overlay_type_ == OverlayType::CertificatedMembers && !X->is_permanent_member() && + X->certificate()->is_expired()) { + auto id = X->get_id(); + del_peer(id); + continue; + } + + if (X->get_node()->flags() & OverlayMemberFlags::DoNotReceiveBroadcasts) { if (X->is_neighbour()) { - bool found = false; - for (auto &n : neighbours_) { - if (n == X->get_id()) { - n = *neighbours_.rbegin(); - found = true; - break; - } - } - CHECK(found); - neighbours_.pop_back(); - X->set_neighbour(false); + del_from_neighbour_list(X); } - bad_peers_.erase(X->get_id()); - peers_.remove(X->get_id()); continue; } if (X->is_neighbour()) { - iter++; continue; } - if (neighbours_.size() < max_neighbours()) { + if (peer_list_.neighbours_.size() < max_neighbours()) { VLOG(OVERLAY_INFO) << this << ": adding new neighbour " << X->get_id(); - neighbours_.push_back(X->get_id()); + peer_list_.neighbours_.push_back(X->get_id()); X->set_neighbour(true); } else { CHECK(nodes_to_change > 0); - auto i = td::Random::fast(0, static_cast(neighbours_.size()) - 1); - auto Y = peers_.get(neighbours_[i]); + auto i = td::Random::fast(0, static_cast(peer_list_.neighbours_.size()) - 1); + auto Y = peer_list_.peers_.get(peer_list_.neighbours_[i]); CHECK(Y != nullptr); CHECK(Y->is_neighbour()); Y->set_neighbour(false); - neighbours_[i] = X->get_id(); + peer_list_.neighbours_[i] = X->get_id(); X->set_neighbour(true); nodes_to_change--; VLOG(OVERLAY_INFO) << this << ": changing neighbour " << Y->get_id() << " -> " << X->get_id(); @@ -276,9 +460,11 @@ void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) { OverlayPeer *OverlayImpl::get_random_peer(bool only_alive) { size_t skip_bad = 3; - while (peers_.size() > (only_alive ? bad_peers_.size() : 0)) { - auto P = peers_.get_random(); - if (public_ && P->get_version() + 3600 < td::Clocks::system()) { + OverlayPeer *res = nullptr; + while (!res && peer_list_.peers_.size() > (only_alive ? peer_list_.bad_peers_.size() : 0)) { + auto P = peer_list_.peers_.get_random(); + if (!P->is_permanent_member() && + (P->get_version() + 3600 < td::Clocks::system() || P->certificate()->is_expired())) { VLOG(OVERLAY_INFO) << this << ": deleting outdated peer " << P->get_id(); del_peer(P->get_id()); continue; @@ -292,18 +478,19 @@ OverlayPeer *OverlayImpl::get_random_peer(bool only_alive) { continue; } } - return P; + res = P; } - return nullptr; + update_neighbours(0); + return res; } void OverlayImpl::get_overlay_random_peers(td::uint32 max_peers, td::Promise> promise) { std::vector v; auto t = td::Clocks::system(); - while (v.size() < max_peers && v.size() < peers_.size() - bad_peers_.size()) { - auto P = peers_.get_random(); - if (public_ && P->get_version() + 3600 < t) { + while (v.size() < max_peers && v.size() < peer_list_.peers_.size() - peer_list_.bad_peers_.size()) { + auto P = peer_list_.peers_.get_random(); + if (!P->is_permanent_member() && (P->get_version() + 3600 < t || P->certificate()->is_expired(t))) { VLOG(OVERLAY_INFO) << this << ": deleting outdated peer " << P->get_id(); del_peer(P->get_id()); } else if (P->is_alive()) { @@ -319,22 +506,221 @@ void OverlayImpl::get_overlay_random_peers(td::uint32 max_peers, } } } + update_neighbours(0); promise.set_result(std::move(v)); } void OverlayImpl::receive_nodes_from_db(tl_object_ptr tl_nodes) { - if (public_) { - std::vector nodes; - for (auto &n : tl_nodes->nodes_) { - auto N = OverlayNode::create(n); - if (N.is_ok()) { - nodes.emplace_back(N.move_as_ok()); + if (overlay_type_ != OverlayType::FixedMemberList) { + add_peers(tl_nodes); + } +} + +void OverlayImpl::receive_nodes_from_db_v2(tl_object_ptr tl_nodes) { + if (overlay_type_ != OverlayType::FixedMemberList) { + add_peers(tl_nodes); + } +} + +bool OverlayImpl::is_persistent_node(const adnl::AdnlNodeIdShort &id) { + auto P = peer_list_.peers_.get(id); + if (!P) { + return false; + } + return P->is_permanent_member(); +} + +bool OverlayImpl::is_valid_peer(const adnl::AdnlNodeIdShort &src, + const ton_api::overlay_MemberCertificate *certificate) { + if (overlay_type_ == OverlayType::Public) { + on_ping_result(src, true); + return true; + } else if (overlay_type_ == OverlayType::FixedMemberList) { + return peer_list_.peers_.get(src); + } else { + OverlayMemberCertificate cert(certificate); + if (cert.empty()) { + auto P = peer_list_.peers_.get(src); + if (P && !P->is_permanent_member()) { + auto C = P->certificate(); + if (C) { + cert = *C; + } } } - add_peers(std::move(nodes)); + + auto S = validate_peer_certificate(src, cert); + if (S.is_error()) { + VLOG(OVERLAY_WARNING) << "adnl=" << src << ": certificate is invalid: " << S; + return false; + } + auto P = peer_list_.peers_.get(src); + if (P) { + CHECK(P->is_permanent_member() || !cert.empty()); + P->update_certificate(std::move(cert)); + } + return true; + } +} + +void OverlayImpl::iterate_all_peers(std::function cb) { + peer_list_.peers_.iterate([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) { cb(key, peer); }); +} + +void OverlayImpl::update_peer_err_ctr(adnl::AdnlNodeIdShort peer_id, bool is_fec) { + auto src_peer = peer_list_.peers_.get(peer_id); + if (src_peer) { + if (is_fec) { + src_peer->fec_broadcast_errors++; + } else { + src_peer->broadcast_errors++; + } } } +void OverlayImpl::update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) { + auto out_peer = peer_list_.peers_.get(peer_id); + if (out_peer) { + out_peer->throughput_out_bytes_ctr += msg_size; + out_peer->throughput_out_packets_ctr++; + + if (is_query) { + out_peer->last_out_query_at = td::Timestamp::now(); + } + } +} + +void OverlayImpl::update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) { + auto in_peer = peer_list_.peers_.get(peer_id); + if (in_peer) { + in_peer->throughput_in_bytes_ctr += msg_size; + in_peer->throughput_in_packets_ctr++; + + if (is_query) { + in_peer->last_in_query_at = td::Timestamp::now(); + } + } +} + +void OverlayImpl::update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) { + auto fpeer = peer_list_.peers_.get(peer_id); + if (fpeer) { + fpeer->ip_addr_str = ip_str; + } +} + +bool OverlayImpl::has_good_peers() const { + return peer_list_.peers_.size() > peer_list_.bad_peers_.size(); +} + +bool OverlayImpl::is_root_public_key(const PublicKeyHash &key) const { + return peer_list_.root_public_keys_.count(key) > 0; +} + +std::vector OverlayImpl::get_neighbours(td::uint32 max_size) const { + if (max_size == 0 || max_size >= peer_list_.neighbours_.size()) { + return peer_list_.neighbours_; + } else { + std::vector vec; + for (td::uint32 i = 0; i < max_size; i++) { + vec.push_back( + peer_list_.neighbours_[td::Random::fast(0, static_cast(peer_list_.neighbours_.size()) - 1)]); + } + return vec; + } +} + +void OverlayImpl::send_message_to_neighbours(td::BufferSlice data) { + wait_neighbours_not_empty([this, data = std::move(data)](td::Result R) { + if (R.is_error()) { + return; + } + for (auto &n : peer_list_.neighbours_) { + td::actor::send_closure(manager_, &OverlayManager::send_message, n, local_id_, overlay_id_, data.clone()); + } + }); +} + +size_t OverlayImpl::neighbours_cnt() const { + return peer_list_.neighbours_.size(); +} + +void OverlayImpl::update_root_member_list(std::vector ids, + std::vector root_public_keys, OverlayMemberCertificate cert) { + td::uint32 expectd_size = + (td::uint32)(ids.size() + root_public_keys.size() * opts_.max_slaves_in_semiprivate_overlay_); + if (expectd_size > opts_.max_peers_) { + opts_.max_peers_ = expectd_size; + } + if (expectd_size > opts_.max_neighbours_) { + opts_.max_neighbours_ = expectd_size; + } + std::sort(ids.begin(), ids.end()); + auto old_root_public_keys = std::move(peer_list_.root_public_keys_); + for (const auto &pub_key : root_public_keys) { + auto it = old_root_public_keys.find(pub_key); + if (it != old_root_public_keys.end()) { + peer_list_.root_public_keys_.emplace(it->first, std::move(it->second)); + } else { + peer_list_.root_public_keys_.emplace(pub_key, PeerList::SlaveKeys{}); + } + } + std::vector to_del; + peer_list_.peers_.iterate([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) { + peer.set_permanent(std::binary_search(ids.begin(), ids.end(), key)); + if (peer.is_permanent_member()) { + peer.clear_certificate(); + } else { + auto S = validate_peer_certificate(peer.get_id(), peer.certificate()); + if (S.is_error()) { + to_del.push_back(peer.get_id()); + } + } + }); + for (const auto &id : to_del) { + del_peer(id); + } + for (const auto &id : ids) { + if (!peer_list_.peers_.exists(id)) { + OverlayNode node(id, overlay_id_, opts_.default_permanent_members_flags_); + OverlayPeer peer(std::move(node)); + peer.set_permanent(true); + CHECK(peer.is_permanent_member()); + peer_list_.peers_.insert(std::move(id), std::move(peer)); + } + } + + update_member_certificate(std::move(cert)); + update_neighbours(0); +} + +void OverlayImpl::update_member_certificate(OverlayMemberCertificate cert) { + peer_list_.cert_ = std::move(cert); + + if (is_persistent_node(local_id_)) { + peer_list_.local_cert_is_valid_until_ = td::Timestamp::in(86400.0 * 365 * 100); /* 100 years */ + } else { + auto R = validate_peer_certificate(local_id_, &peer_list_.cert_); + if (R.is_ok()) { + peer_list_.local_cert_is_valid_until_ = td::Timestamp::at_unix(cert.expire_at()); + } else { + peer_list_.local_cert_is_valid_until_ = td::Timestamp::never(); + } + } +} + +bool OverlayImpl::has_valid_membership_certificate() { + if (overlay_type_ != OverlayType::CertificatedMembers) { + return true; + } + + if (!peer_list_.local_cert_is_valid_until_) { + return false; + } + + return !peer_list_.local_cert_is_valid_until_.is_in_past(); +} + } // namespace overlay } // namespace ton diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 5b391441bf..d05f5ae287 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -27,41 +27,61 @@ #include "auto/tl/ton_api.hpp" #include "keys/encryptor.h" +#include "td/utils/Status.h" #include "td/utils/StringBuilder.h" +#include "td/utils/port/signals.h" +#include namespace ton { namespace overlay { -td::actor::ActorOwn Overlay::create(td::actor::ActorId keyring, - td::actor::ActorId adnl, - td::actor::ActorId manager, - td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, - OverlayIdFull overlay_id, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope, OverlayOptions opts) { - auto R = td::actor::create_actor("overlay", keyring, adnl, manager, dht_node, local_id, - std::move(overlay_id), true, std::vector(), - std::move(callback), std::move(rules), scope, opts); +const OverlayMemberCertificate OverlayNode::empty_certificate_{}; + +td::actor::ActorOwn Overlay::create_public(td::actor::ActorId keyring, + td::actor::ActorId adnl, + td::actor::ActorId manager, + td::actor::ActorId dht_node, + adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::unique_ptr callback, + OverlayPrivacyRules rules, td::string scope, OverlayOptions opts) { + auto R = td::actor::create_actor( + "overlay", keyring, adnl, manager, dht_node, local_id, std::move(overlay_id), OverlayType::Public, + std::vector(), std::vector(), OverlayMemberCertificate{}, std::move(callback), + std::move(rules), std::move(scope), std::move(opts)); return td::actor::ActorOwn(std::move(R)); } -td::actor::ActorOwn Overlay::create(td::actor::ActorId keyring, - td::actor::ActorId adnl, - td::actor::ActorId manager, - td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, - OverlayIdFull overlay_id, std::vector nodes, - std::unique_ptr callback, OverlayPrivacyRules rules, - std::string scope) { +td::actor::ActorOwn Overlay::create_private( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, + OverlayPrivacyRules rules, std::string scope, OverlayOptions opts) { auto R = td::actor::create_actor("overlay", keyring, adnl, manager, dht_node, local_id, - std::move(overlay_id), false, std::move(nodes), std::move(callback), - std::move(rules), std::move(scope)); + std::move(overlay_id), OverlayType::FixedMemberList, std::move(nodes), + std::vector(), OverlayMemberCertificate{}, + std::move(callback), std::move(rules), std::move(scope)); + return td::actor::ActorOwn(std::move(R)); +} + +td::actor::ActorOwn Overlay::create_semiprivate( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate cert, std::unique_ptr callback, OverlayPrivacyRules rules, + std::string scope, OverlayOptions opts) { + auto R = td::actor::create_actor( + "overlay", keyring, adnl, manager, dht_node, local_id, std::move(overlay_id), OverlayType::CertificatedMembers, + std::move(nodes), std::move(root_public_keys), std::move(cert), std::move(callback), std::move(rules), + std::move(scope), std::move(opts)); return td::actor::ActorOwn(std::move(R)); } OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId manager, td::actor::ActorId dht_node, - adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub, - std::vector nodes, std::unique_ptr callback, + adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, OverlayType overlay_type, + std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate cert, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, OverlayOptions opts) : keyring_(keyring) , adnl_(adnl) @@ -70,37 +90,28 @@ OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor , local_id_(local_id) , id_full_(std::move(overlay_id)) , callback_(std::move(callback)) - , public_(pub) + , overlay_type_(overlay_type) , rules_(std::move(rules)) , scope_(scope) , announce_self_(opts.announce_self_) - , frequent_dht_lookup_(opts.frequent_dht_lookup_) { + , opts_(std::move(opts)) { overlay_id_ = id_full_.compute_short_id(); + frequent_dht_lookup_ = opts_.frequent_dht_lookup_; + peer_list_.local_member_flags_ = opts_.local_overlay_member_flags_; - VLOG(OVERLAY_INFO) << this << ": creating " << (public_ ? "public" : "private"); + VLOG(OVERLAY_INFO) << this << ": creating"; - for (auto &node : nodes) { - CHECK(!public_); - auto X = OverlayNode{node, overlay_id_}; - do_add_peer(std::move(X)); - } + update_root_member_list(std::move(nodes), std::move(root_public_keys), std::move(cert)); update_neighbours(static_cast(nodes.size())); } void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeers &query, td::Promise promise) { - if (public_) { + if (overlay_type_ != OverlayType::FixedMemberList) { VLOG(OVERLAY_DEBUG) << this << ": received " << query.peers_->nodes_.size() << " nodes from " << src << " in getRandomPeers query"; - std::vector nodes; - for (auto &n : query.peers_->nodes_) { - auto N = OverlayNode::create(n); - if (N.is_ok()) { - nodes.emplace_back(N.move_as_ok()); - } - } - add_peers(std::move(nodes)); + add_peers(query.peers_); send_random_peers(src, std::move(promise)); } else { VLOG(OVERLAY_WARNING) << this << ": DROPPING getRandomPeers query from " << src << " in private overlay"; @@ -108,6 +119,19 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getR } } +void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeersV2 &query, + td::Promise promise) { + if (overlay_type_ != OverlayType::FixedMemberList) { + VLOG(OVERLAY_DEBUG) << this << ": received " << query.peers_->nodes_.size() << " nodes from " << src + << " in getRandomPeers query"; + add_peers(query.peers_); + send_random_peers_v2(src, std::move(promise)); + } else { + VLOG(OVERLAY_WARNING) << this << ": DROPPING getRandomPeers query from " << src << " in private overlay"; + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "overlay is private")); + } +} + void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, td::Promise promise) { auto it = broadcasts_.find(query.hash_); @@ -135,17 +159,19 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getB promise.set_error(td::Status::Error(ErrorCode::protoviolation, "dropping get broadcast list query")); } -void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) { - if (!public_) { - auto P = peers_.get(src); - if (P == nullptr) { - VLOG(OVERLAY_WARNING) << this << ": received query in private overlay from unknown source " << src; - promise.set_error(td::Status::Error(ErrorCode::protoviolation, "overlay is private")); - return; - } - } else { - on_ping_result(src, true); +/*void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, adnl::AdnlQueryId query_id, ton_api::overlay_customQuery &query) { + callback_->receive_query(src, query_id, id_, std::move(query.data_)); +} +*/ + +void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data, td::Promise promise) { + if (!is_valid_peer(src, extra ? extra->certificate_.get() : nullptr)) { + VLOG(OVERLAY_WARNING) << this << ": received query in private overlay from unknown source " << src; + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "overlay is not public")); + return; } + auto R = fetch_tl_object(data.clone(), true); if (R.is_error()) { @@ -163,16 +189,25 @@ void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr bcast) { + if (peer_list_.local_member_flags_ & OverlayMemberFlags::DoNotReceiveBroadcasts) { + return td::Status::OK(); + } return BroadcastSimple::create(this, message_from, std::move(bcast)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr b) { + if (peer_list_.local_member_flags_ & OverlayMemberFlags::DoNotReceiveBroadcasts) { + return td::Status::OK(); + } return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr b) { + if (peer_list_.local_member_flags_ & OverlayMemberFlags::DoNotReceiveBroadcasts) { + return td::Status::OK(); + } return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); } @@ -184,6 +219,7 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { + return td::Status::OK(); // disable this logic for now auto it = fec_broadcasts_.find(msg->hash_); if (it != fec_broadcasts_.end()) { VLOG(OVERLAY_DEBUG) << this << ": received fec opt-out message from " << message_from << " for broadcast " @@ -198,6 +234,7 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { + return td::Status::OK(); // disable this logic for now auto it = fec_broadcasts_.find(msg->hash_); if (it != fec_broadcasts_.end()) { VLOG(OVERLAY_DEBUG) << this << ": received fec completed message from " << message_from << " for broadcast " @@ -217,15 +254,13 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, return td::Status::OK(); } -void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) { - if (!public_) { - if (peers_.get(src) == nullptr) { - VLOG(OVERLAY_WARNING) << this << ": received query in private overlay from unknown source " << src; - return; - } - } else { - on_ping_result(src, true); +void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data) { + if (!is_valid_peer(src, extra ? extra->certificate_.get() : nullptr)) { + VLOG(OVERLAY_WARNING) << this << ": received message in private overlay from unknown source " << src; + return; } + auto X = fetch_tl_object(data.clone(), true); if (X.is_error()) { VLOG(OVERLAY_DEBUG) << this << ": received custom message"; @@ -240,44 +275,51 @@ void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice dat void OverlayImpl::alarm() { bcast_gc(); - - if(update_throughput_at_.is_in_past()) { + + if (update_throughput_at_.is_in_past()) { double t_elapsed = td::Time::now() - last_throughput_update_.at(); auto SelfId = actor_id(this); - peers_.iterate([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) { + iterate_all_peers([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) { peer.throughput_out_bytes = static_cast(peer.throughput_out_bytes_ctr / t_elapsed); peer.throughput_in_bytes = static_cast(peer.throughput_in_bytes_ctr / t_elapsed); - + peer.throughput_out_packets = static_cast(peer.throughput_out_packets_ctr / t_elapsed); peer.throughput_in_packets = static_cast(peer.throughput_in_packets_ctr / t_elapsed); - + peer.throughput_out_bytes_ctr = 0; peer.throughput_in_bytes_ctr = 0; - + peer.throughput_out_packets_ctr = 0; peer.throughput_in_packets_ctr = 0; - + auto P = td::PromiseCreator::lambda([SelfId, peer_id = key](td::Result result) { result.ensure(); td::actor::send_closure(SelfId, &Overlay::update_peer_ip_str, peer_id, result.move_as_ok()); }); - + td::actor::send_closure(adnl_, &adnl::AdnlSenderInterface::get_conn_ip_str, local_id_, key, std::move(P)); }); - + update_throughput_at_ = td::Timestamp::in(50.0); last_throughput_update_ = td::Timestamp::now(); } - - if (public_) { - if (peers_.size() > 0) { + + if (overlay_type_ != OverlayType::FixedMemberList) { + if (has_valid_membership_certificate()) { auto P = get_random_peer(); if (P) { - send_random_peers(P->get_id(), {}); + if (overlay_type_ == OverlayType::Public) { + send_random_peers(P->get_id(), {}); + } else { + send_random_peers_v2(P->get_id(), {}); + } } + } else { + VLOG(OVERLAY_WARNING) << "meber certificate ist invalid, valid_until=" + << peer_list_.local_cert_is_valid_until_.at_unix(); } - if (next_dht_query_ && next_dht_query_.is_in_past()) { + if (next_dht_query_ && next_dht_query_.is_in_past() && overlay_type_ == OverlayType::Public) { next_dht_query_ = td::Timestamp::never(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result res) { td::actor::send_closure(SelfId, &OverlayImpl::receive_dht_nodes, std::move(res), true); @@ -285,21 +327,22 @@ void OverlayImpl::alarm() { td::actor::send_closure(dht_node_, &dht::Dht::get_value, dht::DhtKey{overlay_id_.pubkey_hash(), "nodes", 0}, std::move(P)); } - if (update_db_at_.is_in_past()) { - if (peers_.size() > 0) { - std::vector vec; - for (td::uint32 i = 0; i < 20; i++) { - auto P = get_random_peer(); - if (!P) { - break; - } - vec.push_back(P->get()); + if (update_db_at_.is_in_past() && overlay_type_ == OverlayType::Public) { + std::vector vec; + for (td::uint32 i = 0; i < 20; i++) { + auto P = get_random_peer(); + if (!P) { + break; } + vec.push_back(P->get_node()->clone()); + } + if (vec.size() > 0) { td::actor::send_closure(manager_, &OverlayManager::save_to_db, local_id_, overlay_id_, std::move(vec)); } update_db_at_ = td::Timestamp::in(60.0); } + update_neighbours(0); alarm_timestamp() = td::Timestamp::in(1.0); } else { update_neighbours(0); @@ -308,7 +351,7 @@ void OverlayImpl::alarm() { } void OverlayImpl::receive_dht_nodes(td::Result res, bool dummy) { - CHECK(public_); + CHECK(overlay_type_ == OverlayType::Public); if (res.is_ok()) { auto v = res.move_as_ok(); auto R = fetch_tl_object(v.value().clone(), true); @@ -354,7 +397,7 @@ void OverlayImpl::receive_dht_nodes(td::Result res, bool dummy) { } void OverlayImpl::update_dht_nodes(OverlayNode node) { - if (!public_) { + if (overlay_type_ != OverlayType::Public) { return; } @@ -412,7 +455,7 @@ void OverlayImpl::bcast_gc() { } void OverlayImpl::wait_neighbours_not_empty(td::Promise promise, int max_retries) { - if (!neighbours_.empty()) { + if (!peer_list_.neighbours_.empty()) { promise.set_result(td::Unit()); } else if (max_retries > 0) { delay_action( @@ -425,18 +468,16 @@ void OverlayImpl::wait_neighbours_not_empty(td::Promise promise, int m } } -void OverlayImpl::send_message_to_neighbours(td::BufferSlice data) { - wait_neighbours_not_empty([this, data = std::move(data)](td::Result R) { - if (R.is_error()) { - return; - } - for (auto &n : neighbours_) { - td::actor::send_closure(manager_, &OverlayManager::send_message, n, local_id_, overlay_id_, data.clone()); - } - }); -} - void OverlayImpl::send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { + if (!has_valid_membership_certificate()) { + VLOG(OVERLAY_WARNING) << "member certificate is invalid, valid_until=" + << peer_list_.local_cert_is_valid_until_.at_unix(); + return; + } + if (!has_valid_broadcast_certificate(send_as, data.size(), false)) { + VLOG(OVERLAY_WARNING) << "broadcast source certificate is invalid"; + return; + } wait_neighbours_not_empty([this, send_as, flags, data = std::move(data)](td::Result R) mutable { if (R.is_error()) { return; @@ -449,6 +490,15 @@ void OverlayImpl::send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::Bu } void OverlayImpl::send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { + if (!has_valid_membership_certificate()) { + VLOG(OVERLAY_WARNING) << "meber certificate ist invalid, valid_until=" + << peer_list_.local_cert_is_valid_until_.at_unix(); + return; + } + if (!has_valid_broadcast_certificate(send_as, data.size(), true)) { + VLOG(OVERLAY_WARNING) << "broadcast source certificate is invalid"; + return; + } wait_neighbours_not_empty([this, send_as, flags, data = std::move(data)](td::Result R) mutable { if (R.is_error()) { return; @@ -472,6 +522,22 @@ td::Status OverlayImpl::check_date(td::uint32 date) { return td::Status::OK(); } +BroadcastCheckResult OverlayImpl::check_source_eligible(const PublicKeyHash &source, const Certificate *cert, + td::uint32 size, bool is_fec) { + if (size == 0) { + return BroadcastCheckResult::Forbidden; + } + + auto r = rules_.check_rules(source, size, is_fec); + if (!cert || r == BroadcastCheckResult::Allowed) { + return r; + } + + auto r2 = cert->check(source, overlay_id_, static_cast(td::Clocks::system()), size, is_fec); + r2 = broadcast_check_result_min(r2, rules_.check_rules(cert->issuer_hash(), size, is_fec)); + return broadcast_check_result_max(r, r2); +} + BroadcastCheckResult OverlayImpl::check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, bool is_fec) { if (size == 0) { @@ -479,7 +545,7 @@ BroadcastCheckResult OverlayImpl::check_source_eligible(PublicKey source, const } auto short_id = source.compute_short_id(); - auto r = rules_.check_rules(source.compute_short_id(), size, is_fec); + auto r = rules_.check_rules(short_id, size, is_fec); if (!cert || r == BroadcastCheckResult::Allowed) { return r; } @@ -514,21 +580,23 @@ void OverlayImpl::register_fec_broadcast(std::unique_ptr bcast) { } void OverlayImpl::get_self_node(td::Promise promise) { - OverlayNode s{local_id_, overlay_id_}; + OverlayNode s{local_id_, overlay_id_, peer_list_.local_member_flags_}; auto to_sign = s.to_sign(); - auto P = td::PromiseCreator::lambda([oid = print_id(), s = std::move(s), promise = std::move(promise)]( - td::Result> R) mutable { - if (R.is_error()) { - auto S = R.move_as_error(); - LOG(ERROR) << oid << ": failed to get self node: " << S; - promise.set_error(std::move(S)); - return; - } - auto V = R.move_as_ok(); - s.update_signature(std::move(V.first)); - s.update_adnl_id(adnl::AdnlNodeIdFull{V.second}); - promise.set_value(std::move(s)); - }); + auto P = td::PromiseCreator::lambda( + [oid = print_id(), s = std::move(s), cert = peer_list_.cert_, + promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + auto S = R.move_as_error(); + LOG(ERROR) << oid << ": failed to get self node: " << S; + promise.set_error(std::move(S)); + return; + } + auto V = R.move_as_ok(); + s.update_signature(std::move(V.first)); + s.update_adnl_id(adnl::AdnlNodeIdFull{V.second}); + s.update_certificate(std::move(cert)); + promise.set_value(std::move(s)); + }); td::actor::send_closure(keyring_, &keyring::Keyring::sign_add_get_public_key, local_id_.pubkey_hash(), std::move(to_sign), std::move(P)); @@ -620,17 +688,6 @@ void OverlayImpl::check_broadcast(PublicKeyHash src, td::BufferSlice data, td::P callback_->check_broadcast(src, overlay_id_, std::move(data), std::move(promise)); } -void OverlayImpl::update_peer_err_ctr(adnl::AdnlNodeIdShort peer_id, bool is_fec) { - auto src_peer = peers_.get(peer_id); - if(src_peer) { - if(is_fec) { - src_peer->fec_broadcast_errors++; - } else { - src_peer->broadcast_errors++; - } - } -} - void OverlayImpl::broadcast_checked(Overlay::BroadcastHash hash, td::Result R) { { auto it = broadcasts_.find(hash); @@ -652,28 +709,28 @@ void OverlayImpl::get_stats(td::Promiseoverlay_id_ = overlay_id_.bits256_value(); res->overlay_id_full_ = id_full_.pubkey().tl(); res->scope_ = scope_; - peers_.iterate([&](const adnl::AdnlNodeIdShort &key, const OverlayPeer &peer) { + iterate_all_peers([&](const adnl::AdnlNodeIdShort &key, const OverlayPeer &peer) { auto node_obj = create_tl_object(); node_obj->adnl_id_ = key.bits256_value(); node_obj->t_out_bytes_ = peer.throughput_out_bytes; node_obj->t_in_bytes_ = peer.throughput_in_bytes; - + node_obj->t_out_pckts_ = peer.throughput_out_packets; node_obj->t_in_pckts_ = peer.throughput_in_packets; - + node_obj->ip_addr_ = peer.ip_addr_str; - + node_obj->last_in_query_ = static_cast(peer.last_in_query_at.at_unix()); node_obj->last_out_query_ = static_cast(peer.last_out_query_at.at_unix()); - + node_obj->bdcst_errors_ = peer.broadcast_errors; node_obj->fec_bdcst_errors_ = peer.fec_broadcast_errors; - + res->nodes_.push_back(std::move(node_obj)); }); res->stats_.push_back( - create_tl_object("neighbours_cnt", PSTRING() << neighbours_.size())); + create_tl_object("neighbours_cnt", PSTRING() << neighbours_cnt())); callback_->get_stats_extra([promise = std::move(promise), res = std::move(res)](td::Result R) mutable { if (R.is_ok()) { @@ -683,6 +740,14 @@ void OverlayImpl::get_stats(td::Promise std::numeric_limits::max()) { + return false; + } + auto it = certs_.find(source); + return check_source_eligible(source, it == certs_.end() ? nullptr : it->second.get(), (td::uint32)size, is_fec); +} + } // namespace overlay } // namespace ton diff --git a/overlay/overlay.h b/overlay/overlay.h index a6d5cd6b57..f25db206a5 100644 --- a/overlay/overlay.h +++ b/overlay/overlay.h @@ -18,6 +18,7 @@ */ #pragma once +#include "auto/tl/ton_api.h" #include "td/utils/buffer.h" #include "td/utils/int_types.h" @@ -37,24 +38,29 @@ class Overlay : public td::actor::Actor { using BroadcastDataHash = td::Bits256; using BroadcastPartHash = td::Bits256; - static td::actor::ActorOwn create(td::actor::ActorId keyring, - td::actor::ActorId adnl, - td::actor::ActorId manager, - td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, - OverlayIdFull overlay_id, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope, OverlayOptions opts = {}); - static td::actor::ActorOwn create(td::actor::ActorId keyring, - td::actor::ActorId adnl, - td::actor::ActorId manager, - td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, - OverlayIdFull overlay_id, std::vector nodes, - std::unique_ptr callback, OverlayPrivacyRules rules, - std::string scope); + static td::actor::ActorOwn create_public( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope, OverlayOptions opts = {}); + static td::actor::ActorOwn create_private( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, + OverlayPrivacyRules rules, std::string scope, OverlayOptions opts = {}); + static td::actor::ActorOwn create_semiprivate( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate cert, std::unique_ptr callback, OverlayPrivacyRules rules, + std::string scope, OverlayOptions opts = {}); virtual void update_dht_node(td::actor::ActorId dht) = 0; - virtual void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) = 0; - virtual void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) = 0; + virtual void receive_message(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data) = 0; + virtual void receive_query(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data, td::Promise promise) = 0; virtual void send_message_to_neighbours(td::BufferSlice data) = 0; virtual void send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) = 0; virtual void send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) = 0; @@ -64,10 +70,14 @@ class Overlay : public td::actor::Actor { virtual void add_certificate(PublicKeyHash key, std::shared_ptr) = 0; virtual void set_privacy_rules(OverlayPrivacyRules rules) = 0; virtual void receive_nodes_from_db(tl_object_ptr nodes) = 0; + virtual void receive_nodes_from_db_v2(tl_object_ptr nodes) = 0; virtual void get_stats(td::Promise> promise) = 0; virtual void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) = 0; virtual void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) = 0; virtual void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) = 0; + virtual void update_member_certificate(OverlayMemberCertificate cert) = 0; + virtual void update_root_member_list(std::vector nodes, + std::vector root_public_keys, OverlayMemberCertificate cert) = 0; //virtual void receive_broadcast(td::BufferSlice data) = 0; //virtual void subscribe(std::unique_ptr callback) = 0; virtual void forget_peer(adnl::AdnlNodeIdShort peer_id) = 0; diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index f3ef3ab41d..8cf9e9b0bd 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -18,11 +18,15 @@ */ #pragma once +#include +#include #include #include #include +#include #include +#include "adnl/adnl-node-id.hpp" #include "overlay.h" #include "overlay-manager.h" #include "overlay-fec.hpp" @@ -32,6 +36,9 @@ #include "td/utils/DecTree.h" #include "td/utils/List.h" +#include "td/utils/Status.h" +#include "td/utils/Time.h" +#include "td/utils/buffer.h" #include "td/utils/overloaded.h" #include "fec/fec.h" @@ -40,6 +47,8 @@ #include "auto/tl/ton_api.h" #include "auto/tl/ton_api.hpp" +#include "td/utils/port/signals.h" +#include "tl-utils/common-utils.hpp" namespace ton { @@ -58,15 +67,17 @@ class OverlayPeer { adnl::AdnlNodeIdFull get_full_id() const { return node_.adnl_id_full(); } - OverlayNode get() const { - return node_.clone(); + const OverlayNode *get_node() const { + return &node_; } void update(OverlayNode node) { CHECK(get_id() == node.adnl_id_short()); - if (node.version() > node_.version()) { - node_ = std::move(node); - } + node_.update(std::move(node)); + } + void update_certificate(OverlayMemberCertificate cert) { + node_.update_certificate(std::move(cert)); } + OverlayPeer(OverlayNode node) : node_(std::move(node)) { id_ = node_.adnl_id_short(); } @@ -95,24 +106,44 @@ class OverlayPeer { return is_alive_; } + bool is_permanent_member() const { + return is_permanent_member_; + } + + void set_permanent(bool value) { + is_permanent_member_ = value; + } + + void clear_certificate() { + node_.clear_certificate(); + } + + auto certificate() const { + return node_.certificate(); + } + + bool has_full_id() const { + return node_.has_full_id(); + } + td::uint32 throughput_out_bytes = 0; td::uint32 throughput_in_bytes = 0; - + td::uint32 throughput_out_packets = 0; td::uint32 throughput_in_packets = 0; - + td::uint32 throughput_out_bytes_ctr = 0; td::uint32 throughput_in_bytes_ctr = 0; - + td::uint32 throughput_out_packets_ctr = 0; td::uint32 throughput_in_packets_ctr = 0; - + td::uint32 broadcast_errors = 0; td::uint32 fec_broadcast_errors = 0; - + td::Timestamp last_in_query_at = td::Timestamp::now(); td::Timestamp last_out_query_at = td::Timestamp::now(); - + td::string ip_addr_str = "undefined"; private: @@ -122,6 +153,7 @@ class OverlayPeer { bool is_neighbour_ = false; size_t missed_pings_ = 0; bool is_alive_ = true; + bool is_permanent_member_ = false; td::Timestamp last_ping_at_ = td::Timestamp::now(); }; @@ -129,19 +161,23 @@ class OverlayImpl : public Overlay { public: OverlayImpl(td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId manager, td::actor::ActorId dht_node, - adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub, - std::vector nodes, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope = "{ \"type\": \"undefined\" }", OverlayOptions opts = {}); + adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, OverlayType overlay_type, + std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate cert, std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope = "{ \"type\": \"undefined\" }", OverlayOptions opts = {}); void update_dht_node(td::actor::ActorId dht) override { dht_node_ = dht; } - void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) override; - void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) override; + void receive_message(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data) override; + void receive_query(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data, td::Promise promise) override; void send_message_to_neighbours(td::BufferSlice data) override; void send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) override; void send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) override; void receive_nodes_from_db(tl_object_ptr nodes) override; + void receive_nodes_from_db_v2(tl_object_ptr nodes) override; void get_self_node(td::Promise promise); @@ -149,8 +185,8 @@ class OverlayImpl : public Overlay { void start_up() override { update_throughput_at_ = td::Timestamp::in(50.0); last_throughput_update_ = td::Timestamp::now(); - - if (public_) { + + if (overlay_type_ == OverlayType::Public) { update_db_at_ = td::Timestamp::in(60.0); } alarm_timestamp() = td::Timestamp::in(1); @@ -158,13 +194,17 @@ class OverlayImpl : public Overlay { void on_ping_result(adnl::AdnlNodeIdShort peer, bool success); void receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R); + void receive_random_peers_v2(adnl::AdnlNodeIdShort src, td::Result R); void send_random_peers(adnl::AdnlNodeIdShort dst, td::Promise promise); + void send_random_peers_v2(adnl::AdnlNodeIdShort dst, td::Promise promise); void send_random_peers_cont(adnl::AdnlNodeIdShort dst, OverlayNode node, td::Promise promise); + void send_random_peers_v2_cont(adnl::AdnlNodeIdShort dst, OverlayNode node, td::Promise promise); void get_overlay_random_peers(td::uint32 max_peers, td::Promise> promise) override; void set_privacy_rules(OverlayPrivacyRules rules) override; void add_certificate(PublicKeyHash key, std::shared_ptr cert) override { certs_[key] = std::move(cert); } + void update_member_certificate(OverlayMemberCertificate cert) override; void receive_dht_nodes(td::Result res, bool dummy); void update_dht_nodes(OverlayNode node); @@ -187,6 +227,8 @@ class OverlayImpl : public Overlay { td::Status check_date(td::uint32 date); BroadcastCheckResult check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, bool is_fec); + BroadcastCheckResult check_source_eligible(const PublicKeyHash &source, const Certificate *cert, td::uint32 size, + bool is_fec); td::Status check_delivered(BroadcastHash hash); void broadcast_checked(Overlay::BroadcastHash hash, td::Result R); @@ -205,17 +247,7 @@ class OverlayImpl : public Overlay { void send_new_fec_broadcast_part(PublicKeyHash local_id, Overlay::BroadcastDataHash data_hash, td::uint32 size, td::uint32 flags, td::BufferSlice part, td::uint32 seqno, fec::FecType fec_type, td::uint32 date); - std::vector get_neighbours(td::uint32 max_size = 0) const { - if (max_size == 0 || max_size >= neighbours_.size()) { - return neighbours_; - } else { - std::vector vec; - for (td::uint32 i = 0; i < max_size; i++) { - vec.push_back(neighbours_[td::Random::fast(0, static_cast(neighbours_.size()) - 1)]); - } - return vec; - } - } + std::vector get_neighbours(td::uint32 max_size = 0) const; td::actor::ActorId overlay_manager() const { return manager_; } @@ -235,40 +267,54 @@ class OverlayImpl : public Overlay { td::Result get_encryptor(PublicKey source); void get_stats(td::Promise> promise) override; - - void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override { - auto out_peer = peers_.get(peer_id); - if(out_peer) { - out_peer->throughput_out_bytes_ctr += msg_size; - out_peer->throughput_out_packets_ctr++; - - if(is_query) - { - out_peer->last_out_query_at = td::Timestamp::now(); - } - } + + void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override; + + void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override; + + void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) override; + + void update_root_member_list(std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate cert) override; + + bool is_valid_peer(const adnl::AdnlNodeIdShort &id, const ton_api::overlay_MemberCertificate *certificate); + bool is_persistent_node(const adnl::AdnlNodeIdShort &id); + + td::uint32 max_data_bcasts() const { + return 100; } - - void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override { - auto in_peer = peers_.get(peer_id); - if(in_peer) { - in_peer->throughput_in_bytes_ctr += msg_size; - in_peer->throughput_in_packets_ctr++; - - if(is_query) - { - in_peer->last_in_query_at = td::Timestamp::now(); - } - } + td::uint32 max_bcasts() const { + return 1000; } - - void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) override { - auto fpeer = peers_.get(peer_id); - if(fpeer) { - fpeer->ip_addr_str = ip_str; - } + td::uint32 max_fec_bcasts() const { + return 20; + } + td::uint32 max_sources() const { + return 10; + } + td::uint32 max_encryptors() const { + return 16; + } + + td::uint32 max_neighbours() const { + return opts_.max_neighbours_; } + td::uint32 max_peers() const { + return opts_.max_peers_; + } + + td::uint32 nodes_to_send() const { + return opts_.nodes_to_send_; + } + + td::uint32 propagate_broadcast_to() const { + return opts_.propagate_broadcast_to_; + } + + bool has_valid_membership_certificate(); + bool has_valid_broadcast_certificate(const PublicKeyHash &source, size_t size, bool is_fec); + void forget_peer(adnl::AdnlNodeIdShort peer_id) override { del_peer(peer_id); } @@ -283,6 +329,8 @@ class OverlayImpl : public Overlay { void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeers &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeersV2 &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcastList &query, @@ -299,20 +347,28 @@ class OverlayImpl : public Overlay { td::Status process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg); td::Status process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg); - void do_add_peer(OverlayNode node); - void add_peer_in_cont(OverlayNode node); - void add_peer_in(OverlayNode node); + td::Status validate_peer_certificate(const adnl::AdnlNodeIdShort &node, const OverlayMemberCertificate &cert); + td::Status validate_peer_certificate(const adnl::AdnlNodeIdShort &node, const OverlayMemberCertificate *cert); + td::Status validate_peer_certificate(const adnl::AdnlNodeIdShort &node, ton_api::overlay_MemberCertificate *cert); void add_peer(OverlayNode node); void add_peers(std::vector nodes); + void add_peers(const tl_object_ptr &nodes); + void add_peers(const tl_object_ptr &nodes); void del_some_peers(); - void del_peer(adnl::AdnlNodeIdShort id); + void del_peer(const adnl::AdnlNodeIdShort &id); + void del_from_neighbour_list(OverlayPeer *P); + void del_from_neighbour_list(const adnl::AdnlNodeIdShort &id); + void iterate_all_peers(std::function cb); OverlayPeer *get_random_peer(bool only_alive = false); + bool is_root_public_key(const PublicKeyHash &key) const; + bool has_good_peers() const; + size_t neighbours_cnt() const; void finish_dht_query() { if (!next_dht_store_query_) { next_dht_store_query_ = td::Timestamp::in(td::Random::fast(60.0, 100.0)); } - if (frequent_dht_lookup_ && peers_.size() == bad_peers_.size()) { + if (frequent_dht_lookup_ && !has_good_peers()) { next_dht_query_ = td::Timestamp::in(td::Random::fast(6.0, 10.0)); } else { next_dht_query_ = next_dht_store_query_; @@ -327,14 +383,11 @@ class OverlayImpl : public Overlay { OverlayIdFull id_full_; OverlayIdShort overlay_id_; - td::DecTree peers_; td::Timestamp next_dht_query_ = td::Timestamp::in(1.0); td::Timestamp next_dht_store_query_ = td::Timestamp::in(1.0); td::Timestamp update_db_at_; td::Timestamp update_throughput_at_; td::Timestamp last_throughput_update_; - std::set bad_peers_; - adnl::AdnlNodeIdShort next_bad_peer_ = adnl::AdnlNodeIdShort::zero(); std::unique_ptr callback_; @@ -342,7 +395,6 @@ class OverlayImpl : public Overlay { std::map> fec_broadcasts_; std::set delivered_broadcasts_; - std::vector neighbours_; td::ListNode bcast_data_lru_; td::ListNode bcast_fec_lru_; std::queue bcast_lru_; @@ -351,33 +403,6 @@ class OverlayImpl : public Overlay { void bcast_gc(); - static td::uint32 max_data_bcasts() { - return 100; - } - static td::uint32 max_bcasts() { - return 1000; - } - static td::uint32 max_fec_bcasts() { - return 20; - } - static td::uint32 max_sources() { - return 10; - } - static td::uint32 max_neighbours() { - return 5; - } - static td::uint32 max_encryptors() { - return 16; - } - - static td::uint32 max_peers() { - return 20; - } - - static td::uint32 nodes_to_send() { - return 4; - } - static BroadcastHash get_broadcast_hash(adnl::AdnlNodeIdShort &src, td::Bits256 &data_hash) { td::uint8 buf[64]; td::MutableSlice m{buf, 64}; @@ -387,8 +412,7 @@ class OverlayImpl : public Overlay { return td::sha256_bits256(td::Slice(buf, 64)); } - bool public_; - bool semi_public_ = false; + OverlayType overlay_type_; OverlayPrivacyRules rules_; td::string scope_; bool announce_self_ = true; @@ -417,6 +441,25 @@ class OverlayImpl : public Overlay { td::ListNode encryptor_lru_; std::map> encryptor_map_; + + struct PeerList { + struct SlaveKey { + td::int32 expire_at{0}; + adnl::AdnlNodeIdShort node{}; + }; + using SlaveKeys = std::vector; + std::map root_public_keys_; + OverlayMemberCertificate cert_; + std::set bad_peers_; + adnl::AdnlNodeIdShort next_bad_peer_ = adnl::AdnlNodeIdShort::zero(); + td::DecTree peers_; + std::vector neighbours_; + + td::Timestamp local_cert_is_valid_until_; + td::uint32 local_member_flags_{0}; + } peer_list_; + + OverlayOptions opts_; }; } // namespace overlay diff --git a/overlay/overlays.h b/overlay/overlays.h index a7d76fe009..cc569343d5 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -18,7 +18,9 @@ */ #pragma once +#include "adnl/adnl-node-id.hpp" #include "adnl/adnl.h" +#include "auto/tl/ton_api.h" #include "dht/dht.h" #include "td/actor/PromiseFuture.h" @@ -33,6 +35,8 @@ namespace ton { namespace overlay { +enum class OverlayType { Public, FixedMemberList, CertificatedMembers }; + class OverlayIdShort { public: OverlayIdShort() { @@ -88,6 +92,10 @@ struct CertificateFlags { enum Values : td::uint32 { AllowFec = 1, Trusted = 2 }; }; +struct OverlayMemberFlags { + enum Values : td::uint32 { DoNotReceiveBroadcasts = 1 }; +}; + enum BroadcastCheckResult { Forbidden = 1, NeedCheck = 2, Allowed = 3 }; inline BroadcastCheckResult broadcast_check_result_max(BroadcastCheckResult l, BroadcastCheckResult r) { @@ -108,7 +116,6 @@ class OverlayPrivacyRules { } BroadcastCheckResult check_rules(PublicKeyHash hash, td::uint32 size, bool is_fec) { - auto it = authorized_keys_.find(hash); if (it == authorized_keys_.end()) { if (size > max_unath_size_) { @@ -158,9 +165,107 @@ class Certificate { td::SharedSlice signature_; }; +class OverlayMemberCertificate { + public: + OverlayMemberCertificate() { + expire_at_ = std::numeric_limits::max(); + } + OverlayMemberCertificate(PublicKey signed_by, td::uint32 flags, td::int32 slot, td::int32 expire_at, + td::BufferSlice signature) + : signed_by_(std::move(signed_by)) + , flags_(flags) + , slot_(slot) + , expire_at_(expire_at) + , signature_(std::move(signature)) { + } + OverlayMemberCertificate(const OverlayMemberCertificate &other) + : signed_by_(other.signed_by_) + , flags_(other.flags_) + , slot_(other.slot_) + , expire_at_(other.expire_at_) + , signature_(other.signature_.clone()) { + } + OverlayMemberCertificate(OverlayMemberCertificate &&) = default; + OverlayMemberCertificate &operator=(OverlayMemberCertificate &&) = default; + OverlayMemberCertificate &operator=(const OverlayMemberCertificate &other) { + signed_by_ = other.signed_by_; + flags_ = other.flags_; + slot_ = other.slot_; + expire_at_ = other.expire_at_; + signature_ = other.signature_.clone(); + return *this; + } + explicit OverlayMemberCertificate(const ton_api::overlay_MemberCertificate *cert); + td::Status check_signature(const adnl::AdnlNodeIdShort &node); + + bool is_expired() const { + return expire_at_ < td::Clocks::system() - 3; + } + + bool is_expired(double cur_time) const { + return expire_at_ < cur_time - 3; + } + + auto tl() const { + return create_tl_object(signed_by_.tl(), flags_, slot_, expire_at_, + signature_.clone_as_buffer_slice()); + } + + const auto &issued_by() const { + return signed_by_; + } + + td::Slice signature() const { + return signature_.as_slice(); + } + + td::BufferSlice to_sign_data(const adnl::AdnlNodeIdShort &node) const { + return ton::create_serialize_tl_object(node.tl(), flags_, slot_, + expire_at_); + } + + bool empty() const { + return signed_by_.empty(); + } + + bool is_newer(const OverlayMemberCertificate &other) const { + return !empty() && expire_at_ > other.expire_at_; + } + + auto slot() const { + return slot_; + } + + auto expire_at() const { + return expire_at_; + } + + void set_signature(td::Slice signature) { + signature_ = td::SharedSlice(signature); + } + void set_signature(td::SharedSlice signature) { + signature_ = std::move(signature); + } + + private: + PublicKey signed_by_; + td::uint32 flags_; + td::int32 slot_; + td::int32 expire_at_ = std::numeric_limits::max(); + td::SharedSlice signature_; +}; + + struct OverlayOptions { bool announce_self_ = true; bool frequent_dht_lookup_ = false; + td::uint32 local_overlay_member_flags_ = 0; + td::int32 max_slaves_in_semiprivate_overlay_ = 5; + td::uint32 max_peers_ = 20; + td::uint32 max_neighbours_ = 5; + td::uint32 nodes_to_send_ = 4; + td::uint32 propagate_broadcast_to_ = 5; + td::uint32 default_permanent_members_flags_ = 0; }; class Overlays : public td::actor::Actor { @@ -208,11 +313,20 @@ class Overlays : public td::actor::Actor { std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) = 0; virtual void create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - std::unique_ptr callback, OverlayPrivacyRules rules, - td::string scope, OverlayOptions opts) = 0; + std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, + OverlayOptions opts) = 0; + virtual void create_semiprivate_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, + std::vector root_public_keys, + OverlayMemberCertificate certificate, + std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope, OverlayOptions opts) = 0; virtual void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules, std::string scope) = 0; + virtual void create_private_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, std::unique_ptr callback, + OverlayPrivacyRules rules, std::string scope, OverlayOptions opts) = 0; virtual void delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id) = 0; virtual void send_query(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, @@ -246,6 +360,13 @@ class Overlays : public td::actor::Actor { virtual void update_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, PublicKeyHash key, std::shared_ptr cert) = 0; + virtual void update_member_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + OverlayMemberCertificate certificate) = 0; + virtual void update_root_member_list(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + std::vector nodes, + std::vector root_public_keys, + OverlayMemberCertificate certificate) = 0; + virtual void get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, td::uint32 max_peers, td::Promise> promise) = 0; virtual void get_stats(td::Promise> promise) = 0; diff --git a/test/test-overlay.cpp b/test/test-overlay.cpp new file mode 100644 index 0000000000..31db3e780f --- /dev/null +++ b/test/test-overlay.cpp @@ -0,0 +1,450 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. + + Copyright 2017-2020 Telegram Systems LLP +*/ +#include "adnl/adnl-node-id.hpp" +#include "adnl/adnl.h" +#include "adnl/utils.hpp" +#include "adnl/adnl-test-loopback-implementation.h" +#include "auto/tl/ton_api.h" +#include "checksum.h" +#include "common/bitstring.h" +#include "dht/dht.h" +#include "keys/keys.hpp" +#include "overlay-manager.h" +#include "overlay.h" +#include "overlay-id.hpp" +#include "overlay/overlays.h" +#include "td/actor/actor.h" +#include "td/utils/OptionParser.h" +#include "td/utils/Status.h" +#include "td/utils/Time.h" +#include "td/utils/UInt.h" +#include "td/utils/buffer.h" +#include "td/utils/crypto.h" +#include "td/utils/filesystem.h" +#include "td/utils/format.h" +#include "td/utils/port/path.h" +#include "td/utils/Random.h" +#include "td/utils/port/signals.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/overloaded.h" +#include "common/errorlog.h" +#include "tl-utils/common-utils.hpp" +#include "tl/TlObject.h" +#include +#include + +#if TD_DARWIN || TD_LINUX +#include +#endif +#include +#include + +#include + +struct Node { + ton::PrivateKey pk; + ton::PublicKeyHash id; + ton::PublicKey id_full; + ton::adnl::AdnlNodeIdShort adnl_id; + ton::adnl::AdnlNodeIdFull adnl_id_full; + bool can_receive; +}; + +static std::vector root_nodes; +static std::vector slave_nodes; +static std::vector all_nodes; +static td::uint32 total_nodes = 4; +static td::int32 node_slaves_cnt = 3; +static size_t remaining = 0; +static td::Bits256 bcast_hash; + +class Callback : public ton::overlay::Overlays::Callback { + public: + Callback(bool can_receive) : can_receive_(can_receive) { + } + void receive_message(ton::adnl::AdnlNodeIdShort src, ton::overlay::OverlayIdShort overlay_id, + td::BufferSlice data) override { + UNREACHABLE(); + } + void receive_query(ton::adnl::AdnlNodeIdShort src, ton::overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + UNREACHABLE(); + } + void receive_broadcast(ton::PublicKeyHash src, ton::overlay::OverlayIdShort overlay_id, + td::BufferSlice data) override { + CHECK(can_receive_); + CHECK(td::sha256_bits256(data.as_slice()) == bcast_hash); + CHECK(remaining > 0); + remaining--; + } + + private: + bool can_receive_; +}; + +int main(int argc, char *argv[]) { + SET_VERBOSITY_LEVEL(verbosity_INFO); + td::set_default_failure_signal_handler().ensure(); + + std::string db_root_ = "tmp-dir-test-catchain"; + td::rmrf(db_root_).ignore(); + td::mkdir(db_root_).ensure(); + + td::set_default_failure_signal_handler().ensure(); + + td::actor::ActorOwn keyring; + td::actor::ActorOwn network_manager; + td::actor::ActorOwn adnl; + td::actor::ActorOwn overlay_manager; + + td::actor::Scheduler scheduler({7}); + scheduler.run_in_context([&] { + ton::errorlog::ErrorLog::create(db_root_); + keyring = ton::keyring::Keyring::create(db_root_); + network_manager = td::actor::create_actor("test net"); + adnl = ton::adnl::Adnl::create(db_root_, keyring.get()); + overlay_manager = + ton::overlay::Overlays::create(db_root_, keyring.get(), adnl.get(), td::actor::ActorId{}); + td::actor::send_closure(adnl, &ton::adnl::Adnl::register_network_manager, network_manager.get()); + }); + + td::uint32 att = 0; + for (td::uint32 start = att; att < start + 5; att++) { + LOG(WARNING) << "Test #" << att; + root_nodes.resize(total_nodes); + slave_nodes.resize(total_nodes * node_slaves_cnt); + + auto overlay_id_full = + ton::create_serialize_tl_object(td::BufferSlice(PSTRING() << "TEST" << att)); + ton::overlay::OverlayIdFull overlay_id(overlay_id_full.clone()); + auto overlay_id_short = overlay_id.compute_short_id(); + + ton::overlay::OverlayOptions opts; + opts.max_slaves_in_semiprivate_overlay_ = node_slaves_cnt; + opts.default_permanent_members_flags_ = ton::overlay::OverlayMemberFlags::DoNotReceiveBroadcasts; + + ton::overlay::OverlayPrivacyRules rules( + 20 << 20, ton::overlay::CertificateFlags::AllowFec | ton::overlay::CertificateFlags::Trusted, {}); + + std::vector root_keys; + std::vector root_adnl; + + size_t real_members = 0; + + scheduler.run_in_context([&] { + auto addr = ton::adnl::TestLoopbackNetworkManager::generate_dummy_addr_list(); + + for (auto &n : root_nodes) { + bool receive_bcasts = (real_members == 0) ? true : (td::Random::fast_uint32() & 1); + if (receive_bcasts) { + real_members++; + } + n.can_receive = receive_bcasts; + + auto pk1 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub1 = pk1.compute_public_key(); + n.adnl_id_full = ton::adnl::AdnlNodeIdFull{pub1}; + n.adnl_id = ton::adnl::AdnlNodeIdShort{pub1.compute_short_id()}; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk1), true, [](td::Unit) {}); + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_id, ton::adnl::AdnlNodeIdFull{pub1}, addr, + static_cast(0)); + td::actor::send_closure(network_manager, &ton::adnl::TestLoopbackNetworkManager::add_node_id, n.adnl_id, true, + true); + + auto pk2 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub2 = pk2.compute_public_key(); + n.id_full = pub2; + n.id = pub2.compute_short_id(); + n.pk = pk2; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk2), true, [](td::Unit) {}); + + LOG(DEBUG) << "created node " << n.adnl_id << " " << n.id; + + all_nodes.push_back(&n); + root_keys.push_back(n.id); + root_adnl.push_back(n.adnl_id); + } + + for (auto &n : slave_nodes) { + bool receive_bcasts = (real_members == 0) ? true : (td::Random::fast_uint32() & 1); + if (receive_bcasts) { + real_members++; + } + n.can_receive = receive_bcasts; + + auto pk1 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub1 = pk1.compute_public_key(); + n.adnl_id_full = ton::adnl::AdnlNodeIdFull{pub1}; + n.adnl_id = ton::adnl::AdnlNodeIdShort{pub1.compute_short_id()}; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk1), true, [](td::Unit) {}); + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_id, ton::adnl::AdnlNodeIdFull{pub1}, addr, + static_cast(0)); + td::actor::send_closure(network_manager, &ton::adnl::TestLoopbackNetworkManager::add_node_id, n.adnl_id, true, + true); + + auto pk2 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub2 = pk2.compute_public_key(); + n.id_full = pub2; + n.id = pub2.compute_short_id(); + n.pk = pk2; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk2), true, [](td::Unit) {}); + + LOG(DEBUG) << "created node " << n.adnl_id << " " << n.id; + all_nodes.push_back(&n); + } + + for (auto &n1 : all_nodes) { + for (auto &n2 : all_nodes) { + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_peer, n1->adnl_id, n2->adnl_id_full, addr); + } + } + + for (auto &n1 : root_nodes) { + opts.local_overlay_member_flags_ = + (n1.can_receive ? 0 : ton::overlay::OverlayMemberFlags::DoNotReceiveBroadcasts); + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::create_semiprivate_overlay, n1.adnl_id, + ton::overlay::OverlayIdFull(overlay_id_full.clone()), root_adnl, root_keys, + ton::overlay::OverlayMemberCertificate{}, std::make_unique(n1.can_receive), + rules, "", opts); + } + for (size_t i = 0; i < slave_nodes.size(); i++) { + auto &n1 = slave_nodes[i]; + opts.local_overlay_member_flags_ = + (n1.can_receive ? 0 : ton::overlay::OverlayMemberFlags::DoNotReceiveBroadcasts); + + ton::overlay::OverlayMemberCertificate cert(root_nodes[i / node_slaves_cnt].id_full, 0, i % node_slaves_cnt, + 2000000000, td::BufferSlice()); + auto buf = cert.to_sign_data(n1.adnl_id); + auto dec = root_nodes[i / node_slaves_cnt].pk.create_decryptor().move_as_ok(); + auto signature = dec->sign(buf.as_slice()).move_as_ok(); + cert.set_signature(signature.as_slice()); + auto enc = root_nodes[i / node_slaves_cnt].id_full.create_encryptor().move_as_ok(); + enc->check_signature(cert.to_sign_data(n1.adnl_id), cert.signature()).ensure(); + + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::create_semiprivate_overlay, n1.adnl_id, + ton::overlay::OverlayIdFull(overlay_id_full.clone()), root_adnl, root_keys, cert, + std::make_unique(n1.can_receive), rules, "", opts); + } + }); + + td::BufferSlice broadcast(1 << 20); + td::Random::secure_bytes(broadcast.as_slice()); + remaining = real_members; + bcast_hash = td::sha256_bits256(broadcast.as_slice()); + + auto t = td::Timestamp::in(20.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + } + + scheduler.run_in_context([&] { + /*td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::get_stats, + [&](td::Result> R) { + if (R.is_ok()) { + auto res = R.move_as_ok(); + for (auto &o : res->overlays_) { + if (o->overlay_id_ == overlay_id_short.bits256_value()) { + LOG(ERROR) << "NODE " << o->adnl_id_ << " nodes=" << o->nodes_.size(); + for (auto &x : o->stats_) { + LOG(ERROR) << "\t" << x->key_ << " " << x->value_; + } + for (auto &x : o->nodes_) { + LOG(ERROR) << "\t\t" << x->adnl_id_; + } + } + } + } + });*/ + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::send_broadcast_fec_ex, root_nodes[0].adnl_id, + overlay_id_short, root_nodes[0].id, 0, std::move(broadcast)); + }); + + t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + if (!remaining) { + break; + } + } + + LOG_CHECK(!remaining) << "remaining=" << remaining << " all=" << real_members; + + broadcast = td::BufferSlice(700); + td::Random::secure_bytes(broadcast.as_slice()); + remaining = real_members; + bcast_hash = td::sha256_bits256(broadcast.as_slice()); + scheduler.run_in_context([&] { + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::send_broadcast_ex, root_nodes[0].adnl_id, + overlay_id_short, root_nodes[0].id, 0, std::move(broadcast)); + }); + + t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + if (!remaining) { + break; + } + } + + LOG_CHECK(!remaining) << "remaining=" << remaining; + + scheduler.run_in_context([&] { + root_nodes.clear(); + slave_nodes.clear(); + all_nodes.clear(); + }); + } + + for (td::uint32 start = att; att < start + 5; att++) { + LOG(WARNING) << "Test #" << att; + root_nodes.resize(total_nodes); + + auto overlay_id_full = + ton::create_serialize_tl_object(td::BufferSlice(PSTRING() << "TEST" << att)); + ton::overlay::OverlayIdFull overlay_id(overlay_id_full.clone()); + auto overlay_id_short = overlay_id.compute_short_id(); + + ton::overlay::OverlayOptions opts; + + ton::overlay::OverlayPrivacyRules rules( + 20 << 20, ton::overlay::CertificateFlags::AllowFec | ton::overlay::CertificateFlags::Trusted, {}); + + std::vector root_keys; + std::vector root_adnl; + + size_t real_members = 0; + + scheduler.run_in_context([&] { + auto addr = ton::adnl::TestLoopbackNetworkManager::generate_dummy_addr_list(); + + for (auto &n : root_nodes) { + real_members++; + auto pk1 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub1 = pk1.compute_public_key(); + n.adnl_id_full = ton::adnl::AdnlNodeIdFull{pub1}; + n.adnl_id = ton::adnl::AdnlNodeIdShort{pub1.compute_short_id()}; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk1), true, [](td::Unit) {}); + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_id, ton::adnl::AdnlNodeIdFull{pub1}, addr, + static_cast(0)); + td::actor::send_closure(network_manager, &ton::adnl::TestLoopbackNetworkManager::add_node_id, n.adnl_id, true, + true); + + auto pk2 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub2 = pk2.compute_public_key(); + n.id_full = pub2; + n.id = pub2.compute_short_id(); + n.pk = pk2; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk2), true, [](td::Unit) {}); + + LOG(DEBUG) << "created node " << n.adnl_id << " " << n.id; + + all_nodes.push_back(&n); + root_keys.push_back(n.id); + root_adnl.push_back(n.adnl_id); + } + + for (auto &n1 : all_nodes) { + for (auto &n2 : all_nodes) { + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_peer, n1->adnl_id, n2->adnl_id_full, addr); + } + } + + for (auto &n1 : root_nodes) { + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::create_private_overlay_ex, n1.adnl_id, + ton::overlay::OverlayIdFull(overlay_id_full.clone()), root_adnl, + std::make_unique(true), rules, "", opts); + } + }); + + auto t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + } + + td::BufferSlice broadcast(1 << 20); + td::Random::secure_bytes(broadcast.as_slice()); + remaining = real_members; + bcast_hash = td::sha256_bits256(broadcast.as_slice()); + + scheduler.run_in_context([&] { + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::send_broadcast_fec_ex, root_nodes[0].adnl_id, + overlay_id_short, root_nodes[0].id, 0, std::move(broadcast)); + }); + + t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + if (!remaining) { + break; + } + } + + LOG_CHECK(!remaining) << "remaining=" << remaining; + + broadcast = td::BufferSlice(700); + td::Random::secure_bytes(broadcast.as_slice()); + remaining = real_members; + bcast_hash = td::sha256_bits256(broadcast.as_slice()); + scheduler.run_in_context([&] { + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::send_broadcast_ex, root_nodes[0].adnl_id, + overlay_id_short, root_nodes[0].id, 0, std::move(broadcast)); + }); + + t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + if (!remaining) { + break; + } + } + + LOG_CHECK(!remaining) << "remaining=" << remaining; + + scheduler.run_in_context([&] { + root_nodes.clear(); + slave_nodes.clear(); + all_nodes.clear(); + }); + } + + td::rmrf(db_root_).ensure(); + std::_Exit(0); + return 0; +} diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 6deb369ade..d3bdf69247 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -323,10 +323,6 @@ class TestNode : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void update_shard_configuration(td::Ref state, - std::set shards_to_monitor, - std::set temporary_shards) override { - } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } void send_ext_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 89d211a4ee..0b7c0173be 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -209,10 +209,15 @@ dht.query node:dht.node = True; ---types--- overlay.node.toSign id:adnl.id.short overlay:int256 version:int = overlay.node.ToSign; +overlay.node.toSignEx id:adnl.id.short overlay:int256 flags:int version:int = overlay.node.ToSign; overlay.node id:PublicKey overlay:int256 version:int signature:bytes = overlay.Node; +overlay.nodeV2 id:PublicKey overlay:int256 flags:int version:int signature:bytes certificate:overlay.MemberCertificate = overlay.NodeV2; overlay.nodes nodes:(vector overlay.node) = overlay.Nodes; +overlay.nodesV2 nodes:(vector overlay.NodeV2) = overlay.NodesV2; +overlay.messageExtra flags:# certificate:flags.0?overlay.MemberCertificate = overlay.MessageExtra; overlay.message overlay:int256 = overlay.Message; +overlay.messageWithExtra overlay:int256 extra:overlay.messageExtra = overlay.Message; //overlay.randomPeers peers:(vector adnl.node) = overlay.RandomPeers; overlay.broadcastList hashes:(vector int256) = overlay.BroadcastList; @@ -225,6 +230,9 @@ overlay.broadcastFec.partId broadcast_hash:int256 data_hash:int256 seqno:int = o overlay.broadcast.toSign hash:int256 date:int = overlay.broadcast.ToSign; +overlay.memberCertificateId node:adnl.id.short flags:int slot:int expire_at:int = overlay.MemberCertificateId; +overlay.memberCertificate issued_by:PublicKey flags:int slot:int expire_at:int signature:bytes = overlay.MemberCertificate; +overlay.emptyMemberCertificate = overlay.MemberCertificate; overlay.certificate issued_by:PublicKey expire_at:int max_size:int signature:bytes = overlay.Certificate; overlay.certificateV2 issued_by:PublicKey expire_at:int max_size:int flags:int signature:bytes = overlay.Certificate; overlay.emptyCertificate = overlay.Certificate; @@ -242,13 +250,16 @@ overlay.broadcastNotFound = overlay.Broadcast; ---functions--- overlay.getRandomPeers peers:overlay.nodes = overlay.Nodes; +overlay.getRandomPeersV2 peers:overlay.NodesV2 = overlay.NodesV2; overlay.query overlay:int256 = True; +overlay.queryWithExtra overlay:int256 extra:overlay.messageExtra = True; overlay.getBroadcast hash:int256 = overlay.Broadcast; overlay.getBroadcastList list:overlay.broadcastList = overlay.BroadcastList; ---types--- +overlay.db.nodesV2 nodes:overlay.nodesV2 = overlay.db.Nodes; overlay.db.nodes nodes:overlay.nodes = overlay.db.Nodes; overlay.db.key.nodes local_id:int256 overlay:int256 = overlay.db.Key; @@ -407,7 +418,7 @@ tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int25 tonNode.privateBlockOverlayId zero_state_file_hash:int256 nodes:(vector int256) = tonNode.PrivateBlockOverlayId; tonNode.customOverlayId zero_state_file_hash:int256 name:string nodes:(vector int256) = tonNode.CustomOverlayId; -tonNode.privateBlockOverlayIdV2 zero_state_file_hash:int256 workchain:int shard:long nodes:(vector int256) senders:(vector int256) = tonNode.PrivateBlockOverlayIdV2; +tonNode.fastSyncOverlayId zero_state_file_hash:int256 shard:tonNode.shardId = tonNode.FastSyncOverlayId; tonNode.keyBlocks blocks:(vector tonNode.blockIdExt) incomplete:Bool error:Bool = tonNode.KeyBlocks; @@ -614,7 +625,9 @@ engine.dht.config dht:(vector engine.dht) gc:engine.gc = engine.dht.Config; engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNodeMaster; engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave; engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig; -engine.validator.extraConfig state_serializer_enabled:Bool = engine.validator.ExtraConfig; +engine.validator.fastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.FastSyncMemberCertificate; +engine.validator.extraConfig state_serializer_enabled:Bool fast_sync_member_certificates:(vector engine.validator.fastSyncMemberCertificate) + = engine.validator.ExtraConfig; engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) validators:(vector engine.validator) collators:(vector engine.collator) @@ -688,7 +701,8 @@ engine.validator.shardOverlayStats.neighbour id:string verison_major:int version roundtrip:double unreliability:double has_state:string = engine.validator.shardOverlayStats.Neighbour; engine.validator.shardOverlayStats shard:string mode:string neighbours:(vector engine.validator.shardOverlayStats.neighbour) = engine.validator.ShardOverlayStats; -engine.validator.privateBlockOverlayV2Stats shard:string nodes:(vector int256) senders:(vector int256) created_at:int = engine.validator.PrivateBlockOverlayV2Stats; +engine.validator.fastSyncOverlayStats shard:string validators_adnl:(vector int256) root_public_keys:(vector int256) + member_certificate:overlay.MemberCertificate = engine.validator.FastSyncOverlayStats; engine.validator.onePerfTimerStat time:int min:double avg:double max:double = engine.validator.OnePerfTimerStat; engine.validator.perfTimerStatsByName name:string stats:(vector engine.validator.OnePerfTimerStat) = engine.validator.PerfTimerStatsByName; @@ -772,6 +786,9 @@ engine.validator.delShard shard:tonNode.shardId = engine.validator.Success; engine.validator.setCollatorsList list:engine.validator.collatorsList = engine.validator.Success; engine.validator.showCollatorsList = engine.validator.CollatorsList; +engine.validator.signOverlayMemberCertificate sign_by:int256 adnl_id:int256 slot:int expire_at:int = overlay.MemberCertificate; +engine.validator.importFastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.Success; + ---types--- storage.pong = storage.Pong; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 549cc00f526752b96b46747475e1a7a67f35b12e..f574600b1a30e44839e4e7837f8240ee3de48fa7 100644 GIT binary patch delta 1958 zcmZ`(|4$or7`_j?gQ5&OZ0U6^D_=?khh0;qgC*0>eaT30NFcb0%k7{y)=*k#Zwp;! z2709EJ}D{(PXv2KSxS6muOIuy^`v&glD5-QG$h75WKo7Hn?t=~ zGjCE$9H~Obw+rq9d&CHPa*^R$V?~wkkMj+V_2XLSmH>gd0-L3x$oBg)t68y_*e50Y zy#4dPZ+k3x?+1qEs0jDU9Z*u;V4aw{^j#^rOr<7@m9MW z31dly*EWL%7%g(DhkQ#!?cBMaBQxqJ%@Zd%)P}&l{zi3p+{+nKO-@u6yyYmWO3`Qs%%E@BVf~Gv9ahM|jfpV~=HkC(VeX6#MyJh6tCyHe zk!Va11K|)1_SS*_F=viOBC;@eswWie75b%k48*%GNPk@d>2`K!*;+dEdGPPAH!wC; z`7i2@7zQD5Mh@QoWjlm^s$yfIhzyBtCoC+~fcHY(JK0^3 zg5|yL*==-Bq=7*>p6dW@!!xirDy1?XC&wgfMr#RRTpO)G<8SCU8yrcQ^XGHHsNck5 z9+(y@S@e^#^eS;GX@OXp@n^Pldun{+Je_+$((U#}P(G1JuxhbfEkolm)0yglGggXTVbA8VF%5!W>fMGDM ztECJ*O?D|)TZx;+x8&JUGQuS*tz@v4e2`4+CJrYz{n>S^SDEf2Hv7Si&Wm!xPa07P zPFF%n_=zhy*+s@!ZuyG5_XVZBn^=8q#sOXHp^S-GjS$EEyY)pe?HUU`aTuAg!<~yR zF59by7qC&%-%Ua+XZDU+d8J(-<;B|GFbwLagal&9Lpjb?R~ZvXp!UJ{Pd+NAj1QfX dqVZLeG~fL9l^lz(xU9;dmq}%kJx+eD{vV$q!Gr(+ delta 591 zcmaDegY7~q8}Fmp`c@23Ait3}Tb#vB%ph=bjkv{T2MGaY(W^@j_GqW(rDx`)>Xjwt zWTqsRS&g;>7e+#>oen#3z4I z(wXd|th0HEG7F=q)VYqU0x)GMNqT znM1XKZ}S1;3yiEs3uds%PmVN|Wezif$W1VHVP#yg`Ks0O46xHTH!MzJg?Qq{3I$G( zZ*CqF@?o4TxYB&Gfei;lQNU^qPD`wQ(<>;-EK4j&b;`+4&i03RDac^j7EnM<4!E5IW^AsxbHj4`3o*tLR$Xw&l;r1u!W5eyJX6AqKx#l9V4W@~ z$7nn`A&qsqr4pkvPH%{?fOKpRkYTJ*W$fI3){#+KW_w2*;{ulLEGdi$EYnY9Fh*=& JkjZ#P2>@6x-t+(f diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 450dbe4964..7691be0f3e 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1432,3 +1432,51 @@ td::Status ShowCollatorsListQuery::receive(td::BufferSlice data) { } return td::Status::OK(); } + +td::Status SignOverlayMemberCertificateQuery::run() { + TRY_RESULT_ASSIGN(key_hash_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(slot_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status SignOverlayMemberCertificateQuery::send() { + auto b = ton::create_serialize_tl_object( + key_hash_, adnl_id_, slot_, expire_at_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SignOverlayMemberCertificateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + TRY_STATUS(td::write_file(file_name_, data)); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status ImportFastSyncMemberCertificateQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status ImportFastSyncMemberCertificateQuery::send() { + TRY_RESULT(data, td::read_file(file_name_)); + TRY_RESULT(certificate, ton::fetch_tl_object(data, true)); + auto b = ton::create_serialize_tl_object( + adnl_id_, std::move(certificate)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ImportFastSyncMemberCertificateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 06514d0915..3ca962142f 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1443,3 +1443,55 @@ class ShowCollatorsListQuery : public Query { return get_name(); } }; + +class SignOverlayMemberCertificateQuery : public Query { + public: + SignOverlayMemberCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "signoverlaymembercertificate"; + } + static std::string get_help() { + return "signoverlaymembercertificate \tsign overlay member " + "certificate for (hex) with (hex) in slot , valid until , " + "save to "; + } + std::string name() const override { + return get_name(); + } + + private: + td::Bits256 key_hash_; + td::Bits256 adnl_id_; + int slot_; + ton::UnixTime expire_at_; + std::string file_name_; +}; + +class ImportFastSyncMemberCertificateQuery : public Query { + public: + ImportFastSyncMemberCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "importfastsyncmembercertificate"; + } + static std::string get_help() { + return "importfastsyncmembercertificate \timport member certificate for fast sync overlay " + "for (hex) from "; + } + std::string name() const override { + return get_name(); + } + + private: + td::Bits256 adnl_id_; + std::string file_name_; +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index f90be5a589..b13591b2fb 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -157,6 +157,8 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 4fe74b9440..e9779c2e70 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -163,6 +163,13 @@ Config::Config(const ton::ton_api::engine_validator_config &config) { } if (config.extraconfig_) { state_serializer_enabled = config.extraconfig_->state_serializer_enabled_; + for (auto &f : config.extraconfig_->fast_sync_member_certificates_) { + ton::adnl::AdnlNodeIdShort adnl_id{f->adnl_id_}; + ton::overlay::OverlayMemberCertificate certificate{f->certificate_.get()}; + if (!certificate.empty() && !certificate.is_expired()) { + fast_sync_member_certificates.emplace_back(adnl_id, std::move(certificate)); + } + } } else { state_serializer_enabled = true; } @@ -252,9 +259,15 @@ ton::tl_object_ptr Config::tl() const { } ton::tl_object_ptr extra_config_obj = {}; - if (!state_serializer_enabled) { + if (!state_serializer_enabled || !fast_sync_member_certificates.empty()) { // Non-default values - extra_config_obj = ton::create_tl_object(state_serializer_enabled); + extra_config_obj = ton::create_tl_object(); + extra_config_obj->state_serializer_enabled_ = state_serializer_enabled; + for (const auto &[adnl_id, certificate] : fast_sync_member_certificates) { + extra_config_obj->fast_sync_member_certificates_.push_back( + ton::create_tl_object(adnl_id.bits256_value(), + certificate.tl())); + } } std::vector> liteserver_vec; @@ -1995,6 +2008,10 @@ void ValidatorEngine::start_full_node() { td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_collator_adnl_id, ton::adnl::AdnlNodeIdShort(c.adnl_id)); } + for (auto &x : config_.fast_sync_member_certificates) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, + x.first, x.second); + } load_custom_overlays_config(); } else { started_full_node(); @@ -4140,6 +4157,95 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delShard }); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signOverlayMemberCertificate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + ton::PublicKeyHash public_key_hash{query.sign_by_}; + ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; + int slot = query.slot_; + int expire_at = query.expire_at_; + td::actor::send_closure( + keyring_, &ton::keyring::Keyring::get_public_key, public_key_hash, + [=, promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + ton::overlay::OverlayMemberCertificate certificate{R.move_as_ok(), 0, slot, expire_at, td::BufferSlice{}}; + if (certificate.is_expired()) { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::error, "certificate is expired"))); + return; + } + td::BufferSlice to_sign = certificate.to_sign_data(adnl_id); + td::actor::send_closure(keyring_, &ton::keyring::Keyring::sign_message, public_key_hash, std::move(to_sign), + [certificate = std::move(certificate), + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + certificate.set_signature(R.move_as_ok()); + promise.set_value(ton::serialize_tl_object(certificate.tl(), true)); + }); + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importFastSyncMemberCertificate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; + ton::overlay::OverlayMemberCertificate certificate{query.certificate_.get()}; + if (certificate.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "certificate is empty"))); + return; + } + td::Status S = certificate.check_signature(adnl_id); + if (S.is_error()) { + promise.set_value(create_control_query_error(std::move(S))); + return; + } + for (auto &old_cert : config_.fast_sync_member_certificates) { + if (old_cert.first == adnl_id && old_cert.second.issued_by() == certificate.issued_by() && + old_cert.second.expire_at() == certificate.expire_at() && old_cert.second.slot() == certificate.slot()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "duplicate certificate"))); + return; + } + } + + config_.fast_sync_member_certificates.emplace_back(adnl_id, certificate); + write_config([=, promise = std::move(promise), full_node = full_node_.get()](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + if (!full_node.empty()) { + td::actor::send_closure(full_node, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, + adnl_id, std::move(certificate)); + } + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index c4aeafbbc2..08ee2ff6f6 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -101,6 +101,8 @@ struct Config { std::vector shards_to_monitor; bool state_serializer_enabled = true; + std::vector> + fast_sync_member_certificates; void decref(ton::PublicKeyHash key); void incref(ton::PublicKeyHash key) { @@ -524,6 +526,10 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_showCollatorsList &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_signOverlayMemberCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_importFastSyncMemberCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index b9e0024a66..2d950b4643 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -153,8 +153,8 @@ set(FULL_NODE_SOURCE full-node-private-overlay.cpp full-node-serializer.hpp full-node-serializer.cpp - full-node-private-overlay-v2.hpp - full-node-private-overlay-v2.cpp + full-node-fast-sync-overlays.hpp + full-node-fast-sync-overlays.cpp net/download-block.hpp net/download-block.cpp diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp new file mode 100644 index 0000000000..7a1c4209cd --- /dev/null +++ b/validator/full-node-fast-sync-overlays.cpp @@ -0,0 +1,416 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#include "full-node-fast-sync-overlays.hpp" + +#include "checksum.h" +#include "ton/ton-tl.hpp" +#include "common/delay.h" +#include "td/utils/JsonBuilder.h" +#include "tl/tl_json.h" +#include "auto/tl/ton_api_json.h" +#include "full-node-serializer.hpp" + +namespace ton::validator::fullnode { + +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { + process_block_broadcast(src, query); +} + +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) { + process_block_broadcast(src, query); +} + +void FullNodeFastSyncOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); + if (B.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Received block broadcast in fast sync overlay from " << src << ": " + << B.ok().block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); +} + +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { + BlockIdExt block_id = create_block_id(query.block_->block_); + VLOG(FULL_NODE_DEBUG) << "Received newShardBlockBroadcast in fast sync overlay from " << src << ": " + << block_id.to_str(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, block_id, + query.block_->cc_seqno_, std::move(query.block_->data_)); +} + +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeFastSyncOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + BlockIdExt block_id; + CatchainSeqno cc_seqno; + td::uint32 validator_set_hash; + td::BufferSlice data; + auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, + overlay::Overlays::max_fec_broadcast_size()); + if (S.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << S; + return; + } + if (data.size() > FullNode::max_block_size()) { + VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; + return; + } + if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { + VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; + return; + } + VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in fast sync overlay from " << src << ": " << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, + validator_set_hash, std::move(data)); +} + +void FullNodeFastSyncOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { + auto B = fetch_tl_object(std::move(broadcast), true); + if (B.is_error()) { + return; + } + + ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); }); +} + +void FullNodeFastSyncOverlay::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { + if (!inited_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newShardBlockBroadcast in fast sync overlay: " << block_id.to_str(); + auto B = create_serialize_tl_object( + create_tl_object(create_tl_block_id(block_id), cc_seqno, std::move(data))); + if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(B)); + } else { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); + } +} + +void FullNodeFastSyncOverlay::send_broadcast(BlockBroadcast broadcast) { + if (!inited_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending block broadcast in fast sync overlay (with compression): " + << broadcast.block_id.to_str(); + auto B = serialize_block_broadcast(broadcast, true); // compression_enabled = true + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); + return; + } + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); +} + +void FullNodeFastSyncOverlay::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + if (!inited_) { + return; + } + auto B = + serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in fast sync overlay (with compression): " << block_id.to_str(); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); +} + +void FullNodeFastSyncOverlay::start_up() { + auto X = create_hash_tl_object(zero_state_file_hash_, create_tl_shard_id(shard_)); + td::BufferSlice b{32}; + b.as_slice().copy_from(as_slice(X)); + overlay_id_full_ = overlay::OverlayIdFull{std::move(b)}; + overlay_id_ = overlay_id_full_.compute_short_id(); + + try_init(); +} + +void FullNodeFastSyncOverlay::try_init() { + // Sometimes adnl id is added to validator engine later (or not at all) + td::actor::send_closure( + adnl_, &adnl::Adnl::check_id_exists, local_id_, [SelfId = actor_id(this)](td::Result R) { + if (R.is_ok() && R.ok()) { + td::actor::send_closure(SelfId, &FullNodeFastSyncOverlay::init); + } else { + delay_action([SelfId]() { td::actor::send_closure(SelfId, &FullNodeFastSyncOverlay::try_init); }, + td::Timestamp::in(30.0)); + } + }); +} + +void FullNodeFastSyncOverlay::init() { + LOG(INFO) << "Creating fast sync overlay for shard " << shard_.to_str() << ", adnl_id=" << local_id_; + class Callback : public overlay::Overlays::Callback { + public: + void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + } + void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + td::actor::send_closure(node_, &FullNodeFastSyncOverlay::receive_broadcast, src, std::move(data)); + } + void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + } + void get_stats_extra(td::Promise promise) override { + td::actor::send_closure(node_, &FullNodeFastSyncOverlay::get_stats_extra, std::move(promise)); + } + explicit Callback(td::actor::ActorId node) : node_(node) { + } + + private: + td::actor::ActorId node_; + }; + + overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), + overlay::CertificateFlags::AllowFec | overlay::CertificateFlags::Trusted, + {}}; + std::string scope = PSTRING() << R"({ "type": "fast-sync", "shard_id": )" << shard_.shard + << ", \"workchain_id\": " << shard_.workchain << " }"; + overlay::OverlayOptions options; + bool is_validator = std::find(current_validators_adnl_.begin(), current_validators_adnl_.end(), local_id_) != + current_validators_adnl_.end(); + if (!shard_.is_masterchain()) { + options.default_permanent_members_flags_ = overlay::OverlayMemberFlags::DoNotReceiveBroadcasts; + options.local_overlay_member_flags_ = is_validator ? overlay::OverlayMemberFlags::DoNotReceiveBroadcasts : 0; + } + options.max_slaves_in_semiprivate_overlay_ = 100000; // TODO: set lower limit (high limit for testing) + td::actor::send_closure(overlays_, &overlay::Overlays::create_semiprivate_overlay, local_id_, + overlay_id_full_.clone(), current_validators_adnl_, root_public_keys_, member_certificate_, + std::make_unique(actor_id(this)), rules, std::move(scope), options); + + inited_ = true; +} + +void FullNodeFastSyncOverlay::tear_down() { + if (inited_) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_); + } +} + +void FullNodeFastSyncOverlay::set_validators(std::vector root_public_keys, + std::vector current_validators_adnl) { + root_public_keys_ = std::move(root_public_keys); + current_validators_adnl_ = std::move(current_validators_adnl); + if (inited_) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_); + init(); + } +} + +void FullNodeFastSyncOverlay::set_member_certificate(overlay::OverlayMemberCertificate member_certificate) { + member_certificate_ = std::move(member_certificate); + if (inited_) { + td::actor::send_closure(overlays_, &overlay::Overlays::update_member_certificate, local_id_, overlay_id_, + member_certificate_); + } +} + +void FullNodeFastSyncOverlay::get_stats_extra(td::Promise promise) { + auto res = create_tl_object(); + res->shard_ = shard_.to_str(); + for (const auto &x : current_validators_adnl_) { + res->validators_adnl_.push_back(x.bits256_value()); + } + for (const auto &x : root_public_keys_) { + res->root_public_keys_.push_back(x.bits256_value()); + } + res->member_certificate_ = member_certificate_.tl(); + promise.set_result(td::json_encode(td::ToJson(*res), true)); +} + +td::actor::ActorId FullNodeFastSyncOverlays::choose_overlay(ShardIdFull shard) { + for (auto &p : id_to_overlays_) { + auto &overlays = p.second.overlays_; + ShardIdFull cur_shard = shard; + while (true) { + auto it = overlays.find(cur_shard); + if (it != overlays.end()) { + return it->second.get(); + } + if (cur_shard.pfx_len() == 0) { + break; + } + cur_shard = shard_parent(cur_shard); + } + } + return {}; +} + +void FullNodeFastSyncOverlays::update_overlays(td::Ref state, + std::set my_adnl_ids, + std::set monitoring_shards, + const FileHash &zero_state_file_hash, + const td::actor::ActorId &keyring, + const td::actor::ActorId &adnl, + const td::actor::ActorId &overlays, + const td::actor::ActorId &validator_manager, + const td::actor::ActorId &full_node) { + monitoring_shards.insert(ShardIdFull{masterchainId}); + std::set all_shards; + all_shards.insert(ShardIdFull{masterchainId}); + td::uint32 monitor_min_split = state->monitor_min_split_depth(basechainId); + for (td::uint64 i = 0; i < (1ULL << monitor_min_split); ++i) { + all_shards.insert(ShardIdFull{basechainId, (i * 2 + 1) << (63 - monitor_min_split)}); + } + + // Remove overlays for removed adnl ids and shards + for (auto it = id_to_overlays_.begin(); it != id_to_overlays_.end();) { + if (my_adnl_ids.count(it->first)) { + auto &overlays_info = it->second; + ; + auto ¤t_shards = overlays_info.is_validator_ ? all_shards : monitoring_shards; + for (auto it2 = overlays_info.overlays_.begin(); it2 != overlays_info.overlays_.end();) { + if (current_shards.count(it2->first)) { + ++it2; + } else { + it2 = overlays_info.overlays_.erase(it2); + } + } + ++it; + } else { + it = id_to_overlays_.erase(it); + } + } + + // On new keyblock - update validator set + bool updated_validators = false; + if (!last_key_block_seqno_ || last_key_block_seqno_.value() != state->last_key_block_id().seqno()) { + updated_validators = true; + last_key_block_seqno_ = state->last_key_block_id().seqno(); + root_public_keys_.clear(); + current_validators_adnl_.clear(); + // Previous, current and next validator sets + for (int i = -1; i <= 1; ++i) { + auto val_set = state->get_total_validator_set(i); + if (val_set.is_null()) { + continue; + } + for (const ValidatorDescr &val : val_set->export_vector()) { + PublicKeyHash public_key_hash = ValidatorFullId{val.key}.compute_short_id(); + root_public_keys_.push_back(public_key_hash); + if (i == 0) { + current_validators_adnl_.emplace_back(val.addr.is_zero() ? public_key_hash.bits256_value() : val.addr); + } + } + } + std::sort(root_public_keys_.begin(), root_public_keys_.end()); + root_public_keys_.erase(std::unique(root_public_keys_.begin(), root_public_keys_.end()), root_public_keys_.end()); + std::sort(current_validators_adnl_.begin(), current_validators_adnl_.end()); + + for (auto &[local_id, overlays_info] : id_to_overlays_) { + overlays_info.is_validator_ = + std::binary_search(current_validators_adnl_.begin(), current_validators_adnl_.end(), local_id); + for (auto &[_, overlay] : overlays_info.overlays_) { + td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::set_validators, root_public_keys_, + current_validators_adnl_); + } + } + } + + // Cleanup outdated certificates + double now = td::Clocks::system(); + for (auto &[_, certificates] : member_certificates_) { + certificates.erase(std::remove_if(certificates.begin(), certificates.end(), + [&](const overlay::OverlayMemberCertificate &certificate) { + return certificate.is_expired(now); + }), + certificates.end()); + } + + for (adnl::AdnlNodeIdShort local_id : my_adnl_ids) { + bool is_new = !id_to_overlays_.count(local_id); + auto &overlays_info = id_to_overlays_[local_id]; + // Update is_validator and current_certificate + if (is_new) { + overlays_info.is_validator_ = + std::binary_search(current_validators_adnl_.begin(), current_validators_adnl_.end(), local_id); + } + bool changed_certificate = false; + // Check if certificate is outdated or no longer authorized by current root keys + if (!overlays_info.current_certificate_.empty() && overlays_info.current_certificate_.is_expired(now)) { + changed_certificate = true; + overlays_info.current_certificate_ = {}; + } + if (!overlays_info.current_certificate_.empty() && updated_validators && + !std::binary_search(root_public_keys_.begin(), root_public_keys_.end(), + overlays_info.current_certificate_.issued_by().compute_short_id())) { + changed_certificate = true; + overlays_info.current_certificate_ = {}; + } + if (overlays_info.current_certificate_.empty()) { + auto it = member_certificates_.find(local_id); + if (it != member_certificates_.end()) { + for (const overlay::OverlayMemberCertificate &certificate : it->second) { + if (std::binary_search(root_public_keys_.begin(), root_public_keys_.end(), + certificate.issued_by().compute_short_id())) { + changed_certificate = true; + overlays_info.current_certificate_ = it->second.front(); + break; + } + } + } + } + + // Remove if it is not authorized + if (!overlays_info.is_validator_ && overlays_info.current_certificate_.empty()) { + id_to_overlays_.erase(local_id); + continue; + } + + // Update shard overlays + auto ¤t_shards = overlays_info.is_validator_ ? all_shards : monitoring_shards; + for (ShardIdFull shard_id : current_shards) { + auto &overlay = overlays_info.overlays_[shard_id]; + if (overlay.empty()) { + overlay = td::actor::create_actor( + PSTRING() << "FastSyncOv" << shard_id.to_str(), local_id, shard_id, zero_state_file_hash, + root_public_keys_, current_validators_adnl_, overlays_info.current_certificate_, keyring, adnl, overlays, + validator_manager, full_node); + } else if (changed_certificate) { + td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::set_member_certificate, + overlays_info.current_certificate_); + } + } + } +} + +void FullNodeFastSyncOverlays::add_member_certificate(adnl::AdnlNodeIdShort local_id, + overlay::OverlayMemberCertificate member_certificate) { + if (member_certificate.empty() || member_certificate.is_expired()) { + return; + } + member_certificates_[local_id].push_back(std::move(member_certificate)); + // Overlays will be updated in the next update_overlays +} + +} // namespace ton::validator::fullnode diff --git a/validator/full-node-private-overlay-v2.hpp b/validator/full-node-fast-sync-overlays.hpp similarity index 59% rename from validator/full-node-private-overlay-v2.hpp rename to validator/full-node-fast-sync-overlays.hpp index abb88fb439..30cb5dde81 100644 --- a/validator/full-node-private-overlay-v2.hpp +++ b/validator/full-node-fast-sync-overlays.hpp @@ -20,7 +20,7 @@ namespace ton::validator::fullnode { -class FullNodePrivateOverlayV2 : public td::actor::Actor { +class FullNodeFastSyncOverlay : public td::actor::Actor { public: void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast& query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed& query); @@ -28,9 +28,9 @@ class FullNodePrivateOverlayV2 : public td::actor::Actor { void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast& query); - void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); - void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); - void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast& query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed& query); + void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast& query); template void process_broadcast(PublicKeyHash, T&) { @@ -46,26 +46,26 @@ class FullNodePrivateOverlayV2 : public td::actor::Actor { void start_up() override; void tear_down() override; - void destroy() { - stop(); - } - - FullNodePrivateOverlayV2(adnl::AdnlNodeIdShort local_id, ShardIdFull shard, std::vector nodes, - std::vector senders, FileHash zero_state_file_hash, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId rldp2, - td::actor::ActorId overlays, - td::actor::ActorId validator_manager, - td::actor::ActorId full_node) + void set_validators(std::vector root_public_keys, + std::vector current_validators_adnl); + void set_member_certificate(overlay::OverlayMemberCertificate member_certificate); + + FullNodeFastSyncOverlay(adnl::AdnlNodeIdShort local_id, ShardIdFull shard, FileHash zero_state_file_hash, + std::vector root_public_keys, + std::vector current_validators_adnl, + overlay::OverlayMemberCertificate member_certificate, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId overlays, + td::actor::ActorId validator_manager, + td::actor::ActorId full_node) : local_id_(local_id) , shard_(shard) - , nodes_(std::move(nodes)) - , senders_(std::move(senders)) + , root_public_keys_(std::move(root_public_keys)) + , current_validators_adnl_(std::move(current_validators_adnl)) + , member_certificate_(std::move(member_certificate)) , zero_state_file_hash_(zero_state_file_hash) , keyring_(keyring) , adnl_(adnl) - , rldp_(rldp) - , rldp2_(rldp2) , overlays_(overlays) , validator_manager_(validator_manager) , full_node_(full_node) { @@ -74,14 +74,13 @@ class FullNodePrivateOverlayV2 : public td::actor::Actor { private: adnl::AdnlNodeIdShort local_id_; ShardIdFull shard_; - std::vector nodes_; - std::vector senders_; + std::vector root_public_keys_; + std::vector current_validators_adnl_; + overlay::OverlayMemberCertificate member_certificate_; FileHash zero_state_file_hash_; td::actor::ActorId keyring_; td::actor::ActorId adnl_; - td::actor::ActorId rldp_; - td::actor::ActorId rldp2_; td::actor::ActorId overlays_; td::actor::ActorId validator_manager_; td::actor::ActorId full_node_; @@ -96,29 +95,30 @@ class FullNodePrivateOverlayV2 : public td::actor::Actor { void get_stats_extra(td::Promise promise); }; -class FullNodePrivateBlockOverlaysV2 { +class FullNodeFastSyncOverlays { public: - td::actor::ActorId choose_overlay(ShardIdFull shard); + td::actor::ActorId choose_overlay(ShardIdFull shard); void update_overlays(td::Ref state, std::set my_adnl_ids, - const FileHash& zero_state_file_hash, const td::actor::ActorId& keyring, - const td::actor::ActorId& adnl, const td::actor::ActorId& rldp, - const td::actor::ActorId& rldp2, + std::set monitoring_shards, const FileHash& zero_state_file_hash, + const td::actor::ActorId& keyring, const td::actor::ActorId& adnl, const td::actor::ActorId& overlays, const td::actor::ActorId& validator_manager, const td::actor::ActorId& full_node); - void destroy_overlays(); + void add_member_certificate(adnl::AdnlNodeIdShort local_id, overlay::OverlayMemberCertificate member_certificate); private: struct Overlays { - struct ShardOverlay { - td::actor::ActorOwn overlay_; - std::vector nodes_, senders_; - bool is_sender_ = false; - }; - std::map overlays_; + std::map> overlays_; + overlay::OverlayMemberCertificate current_certificate_; + bool is_validator_{false}; }; std::map id_to_overlays_; // local_id -> overlays + std::map> member_certificates_; + + td::optional last_key_block_seqno_; + std::vector root_public_keys_; + std::vector current_validators_adnl_; }; } // namespace ton::validator::fullnode diff --git a/validator/full-node-private-overlay-v2.cpp b/validator/full-node-private-overlay-v2.cpp deleted file mode 100644 index 20e9cc55fe..0000000000 --- a/validator/full-node-private-overlay-v2.cpp +++ /dev/null @@ -1,384 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . -*/ - -#include "full-node-private-overlay-v2.hpp" - -#include "checksum.h" -#include "ton/ton-tl.hpp" -#include "common/delay.h" -#include "td/utils/JsonBuilder.h" -#include "tl/tl_json.h" -#include "auto/tl/ton_api_json.h" -#include "full-node-serializer.hpp" - -namespace ton::validator::fullnode { - -void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { - process_block_broadcast(src, query); -} - -void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) { - process_block_broadcast(src, query); -} - -void FullNodePrivateOverlayV2::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { - auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); - if (B.is_error()) { - LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); - return; - } - VLOG(FULL_NODE_DEBUG) << "Received block broadcast in private overlay from " << src << ": " - << B.ok().block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); -} - -void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { - BlockIdExt block_id = create_block_id(query.block_->block_); - VLOG(FULL_NODE_DEBUG) << "Received newShardBlockBroadcast in private overlay from " << src << ": " - << block_id.to_str(); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, block_id, - query.block_->cc_seqno_, std::move(query.block_->data_)); -} - -void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash src, - ton_api::tonNode_newBlockCandidateBroadcast &query) { - process_block_candidate_broadcast(src, query); -} - -void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash src, - ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { - process_block_candidate_broadcast(src, query); -} - -void FullNodePrivateOverlayV2::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { - BlockIdExt block_id; - CatchainSeqno cc_seqno; - td::uint32 validator_set_hash; - td::BufferSlice data; - auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, - overlay::Overlays::max_fec_broadcast_size()); - if (S.is_error()) { - LOG(DEBUG) << "dropped broadcast: " << S; - return; - } - if (data.size() > FullNode::max_block_size()) { - VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; - return; - } - if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { - VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; - return; - } - VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in private overlay from " << src << ": " << block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, - validator_set_hash, std::move(data)); -} - -void FullNodePrivateOverlayV2::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { - auto B = fetch_tl_object(std::move(broadcast), true); - if (B.is_error()) { - return; - } - - ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); }); -} - -void FullNodePrivateOverlayV2::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, - td::BufferSlice data) { - if (!inited_) { - return; - } - VLOG(FULL_NODE_DEBUG) << "Sending newShardBlockBroadcast in private overlay: " << block_id.to_str(); - auto B = create_serialize_tl_object( - create_tl_object(create_tl_block_id(block_id), cc_seqno, std::move(data))); - if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) { - td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), 0, std::move(B)); - } else { - td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); - } -} - -void FullNodePrivateOverlayV2::send_broadcast(BlockBroadcast broadcast) { - if (!inited_) { - return; - } - VLOG(FULL_NODE_DEBUG) << "Sending block broadcast in private overlay (with compression): " - << broadcast.block_id.to_str(); - auto B = serialize_block_broadcast(broadcast, true); // compression_enabled = true - if (B.is_error()) { - VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); - return; - } - td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); -} - -void FullNodePrivateOverlayV2::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, - td::uint32 validator_set_hash, td::BufferSlice data) { - if (!inited_) { - return; - } - auto B = - serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled - if (B.is_error()) { - VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); - return; - } - VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in private overlay (with compression): " << block_id.to_str(); - td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); -} - -void FullNodePrivateOverlayV2::start_up() { - std::sort(nodes_.begin(), nodes_.end()); - nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end()); - - std::vector nodes, senders; - for (const adnl::AdnlNodeIdShort &id : nodes_) { - nodes.push_back(id.bits256_value()); - } - for (const adnl::AdnlNodeIdShort &id : senders_) { - senders.push_back(id.bits256_value()); - } - auto X = create_hash_tl_object( - zero_state_file_hash_, shard_.workchain, shard_.shard, std::move(nodes), std::move(senders)); - td::BufferSlice b{32}; - b.as_slice().copy_from(as_slice(X)); - overlay_id_full_ = overlay::OverlayIdFull{std::move(b)}; - overlay_id_ = overlay_id_full_.compute_short_id(); - - try_init(); -} - -void FullNodePrivateOverlayV2::try_init() { - // Sometimes adnl id is added to validator engine later (or not at all) - td::actor::send_closure( - adnl_, &adnl::Adnl::check_id_exists, local_id_, [SelfId = actor_id(this)](td::Result R) { - if (R.is_ok() && R.ok()) { - td::actor::send_closure(SelfId, &FullNodePrivateOverlayV2::init); - } else { - delay_action([SelfId]() { td::actor::send_closure(SelfId, &FullNodePrivateOverlayV2::try_init); }, - td::Timestamp::in(30.0)); - } - }); -} - -void FullNodePrivateOverlayV2::init() { - LOG(FULL_NODE_INFO) << "Creating private block overlay for shard " << shard_.to_str() << ", adnl_id=" << local_id_ - << " : " << nodes_.size() << " nodes"; - class Callback : public overlay::Overlays::Callback { - public: - void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { - } - void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, - td::Promise promise) override { - } - void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { - td::actor::send_closure(node_, &FullNodePrivateOverlayV2::receive_broadcast, src, std::move(data)); - } - void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, - td::Promise promise) override { - } - void get_stats_extra(td::Promise promise) override { - td::actor::send_closure(node_, &FullNodePrivateOverlayV2::get_stats_extra, std::move(promise)); - } - Callback(td::actor::ActorId node) : node_(node) { - } - - private: - td::actor::ActorId node_; - }; - - std::map authorized_keys; - for (const adnl::AdnlNodeIdShort &sender : senders_) { - authorized_keys[sender.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size(); - } - overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), 0, std::move(authorized_keys)}; - std::string scope = PSTRING() << R"({ "type": "private-blocks-v2", "shard_id": )" << shard_.shard - << ", \"workchain_id\": " << shard_.workchain << " }"; - td::actor::send_closure(overlays_, &overlay::Overlays::create_private_overlay, local_id_, overlay_id_full_.clone(), - nodes_, std::make_unique(actor_id(this)), rules, std::move(scope)); - - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); - td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); - inited_ = true; -} - -void FullNodePrivateOverlayV2::tear_down() { - if (inited_) { - td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_); - } -} - -void FullNodePrivateOverlayV2::get_stats_extra(td::Promise promise) { - auto res = create_tl_object(); - res->shard_ = shard_.to_str(); - for (const auto &x : nodes_) { - res->nodes_.push_back(x.bits256_value()); - } - for (const auto &x : senders_) { - res->senders_.push_back(x.bits256_value()); - } - res->created_at_ = created_at_; - promise.set_result(td::json_encode(td::ToJson(*res), true)); -} - -td::actor::ActorId FullNodePrivateBlockOverlaysV2::choose_overlay(ShardIdFull shard) { - for (auto &p : id_to_overlays_) { - auto &overlays = p.second.overlays_; - ShardIdFull cur_shard = shard; - while (true) { - auto it = overlays.find(cur_shard); - if (it != overlays.end() && it->second.is_sender_) { - return it->second.overlay_.get(); - } - if (cur_shard.pfx_len() == 0) { - break; - } - cur_shard = shard_parent(cur_shard); - } - } - return {}; -} - -void FullNodePrivateBlockOverlaysV2::update_overlays( - td::Ref state, std::set my_adnl_ids, const FileHash &zero_state_file_hash, - const td::actor::ActorId &keyring, const td::actor::ActorId &adnl, - const td::actor::ActorId &rldp, const td::actor::ActorId &rldp2, - const td::actor::ActorId &overlays, - const td::actor::ActorId &validator_manager, - const td::actor::ActorId &full_node) { - if (my_adnl_ids.empty()) { - id_to_overlays_.clear(); - return; - } - auto collators = state->get_collator_config(true); - auto all_validators = state->get_total_validator_set(0); - - struct OverlayInfo { - std::vector nodes, senders; - }; - std::map overlay_infos; - - // Masterchain overlay: all validators + collators - OverlayInfo &mc_overlay = overlay_infos[ShardIdFull(masterchainId)]; - for (const auto &x : all_validators->export_vector()) { - td::Bits256 addr = x.addr.is_zero() ? ValidatorFullId(x.key).compute_short_id().bits256_value() : x.addr; - mc_overlay.nodes.emplace_back(addr); - mc_overlay.senders.emplace_back(addr); - } - for (const auto &x : collators.collator_nodes) { - mc_overlay.nodes.emplace_back(x.adnl_id); - } - - // Shard overlays: validators of the shard + collators of the shard - // See ValidatorManagerImpl::update_shards - std::set new_shards; - for (auto &v : state->get_shards()) { - ShardIdFull shard = v->shard(); - if (shard.is_masterchain()) { - continue; - } - if (v->before_split()) { - ShardIdFull l_shard{shard.workchain, shard_child(shard.shard, true)}; - ShardIdFull r_shard{shard.workchain, shard_child(shard.shard, false)}; - new_shards.insert(l_shard); - new_shards.insert(r_shard); - } else if (v->before_merge()) { - ShardIdFull p_shard{shard.workchain, shard_parent(shard.shard)}; - new_shards.insert(p_shard); - } else { - new_shards.insert(shard); - } - } - for (ShardIdFull shard : new_shards) { - auto val_set = state->get_validator_set(shard); - td::uint32 min_split = state->monitor_min_split_depth(shard.workchain); - OverlayInfo &overlay = - overlay_infos[shard_prefix_length(shard) <= min_split ? shard : shard_prefix(shard, min_split)]; - for (const auto &x : val_set->export_vector()) { - td::Bits256 addr = x.addr.is_zero() ? ValidatorFullId(x.key).compute_short_id().bits256_value() : x.addr; - overlay.nodes.emplace_back(addr); - overlay.senders.emplace_back(addr); - } - } - for (auto &p : overlay_infos) { - ShardIdFull shard = p.first; - OverlayInfo &overlay = p.second; - if (!shard.is_masterchain()) { - for (const auto &collator : collators.collator_nodes) { - if (shard_intersects(collator.shard, shard)) { - overlay.nodes.emplace_back(collator.adnl_id); - } - } - } - - std::sort(overlay.nodes.begin(), overlay.nodes.end()); - overlay.nodes.erase(std::unique(overlay.nodes.begin(), overlay.nodes.end()), overlay.nodes.end()); - std::sort(overlay.senders.begin(), overlay.senders.end()); - overlay.senders.erase(std::unique(overlay.senders.begin(), overlay.senders.end()), overlay.senders.end()); - } - - std::map old_private_block_overlays = std::move(id_to_overlays_); - id_to_overlays_.clear(); - - for (const auto &p : overlay_infos) { - ShardIdFull shard = p.first; - const OverlayInfo &new_overlay_info = p.second; - for (adnl::AdnlNodeIdShort local_id : new_overlay_info.nodes) { - if (!my_adnl_ids.count(local_id)) { - continue; - } - Overlays::ShardOverlay &new_overlay = id_to_overlays_[local_id].overlays_[shard]; - Overlays::ShardOverlay &old_overlay = old_private_block_overlays[local_id].overlays_[shard]; - if (!old_overlay.overlay_.empty() && old_overlay.nodes_ == new_overlay_info.nodes && - old_overlay.senders_ == new_overlay_info.senders) { - new_overlay = std::move(old_overlay); - old_overlay = {}; - } else { - new_overlay.nodes_ = new_overlay_info.nodes; - new_overlay.senders_ = new_overlay_info.senders; - new_overlay.is_sender_ = std::binary_search(new_overlay.senders_.begin(), new_overlay.senders_.end(), local_id); - new_overlay.overlay_ = td::actor::create_actor( - PSTRING() << "BlocksPrivateOverlay" << shard.to_str(), local_id, shard, new_overlay.nodes_, - new_overlay.senders_, zero_state_file_hash, keyring, adnl, rldp, rldp2, overlays, validator_manager, - full_node); - } - } - } - - // Delete old overlays, but not immediately - for (auto &p : old_private_block_overlays) { - for (auto &x : p.second.overlays_) { - if (x.second.overlay_.empty()) { - continue; - } - td::actor::ActorId id = x.second.overlay_.release(); - delay_action([id = std::move(id)]() { td::actor::send_closure(id, &FullNodePrivateOverlayV2::destroy); }, - td::Timestamp::in(30.0)); - } - } -} - -void FullNodePrivateBlockOverlaysV2::destroy_overlays() { - id_to_overlays_.clear(); -} - - -} // namespace ton::validator::fullnode diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index d0dbf1a64b..5aa69fe995 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -1419,9 +1419,9 @@ td::actor::ActorOwn FullNodeShard::create( td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, td::actor::ActorId full_node, FullNodeShardMode mode) { - return td::actor::create_actor("tonnode", shard, local_id, adnl_id, zero_state_file_hash, config, - keyring, adnl, rldp, rldp2, overlays, validator_manager, client, - full_node, mode); + return td::actor::create_actor(PSTRING() << "tonnode" << shard.to_str(), shard, local_id, adnl_id, + zero_state_file_hash, config, keyring, adnl, rldp, rldp2, overlays, + validator_manager, client, full_node, mode); } } // namespace fullnode diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 335578cfb7..8e84a4d456 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -198,8 +198,8 @@ void FullNodeImpl::initial_read_complete(BlockHandle top_handle) { td::actor::send_closure(it->second.actor, &FullNodeShard::set_handle, top_handle, std::move(P)); } -void FullNodeImpl::update_shard_configuration(td::Ref state, std::set shards_to_monitor, - std::set temporary_shards) { +void FullNodeImpl::on_new_masterchain_block(td::Ref state, std::set shards_to_monitor, + std::set temporary_shards) { CHECK(shards_to_monitor.count(ShardIdFull(masterchainId))); std::set new_shards; std::map new_active; @@ -232,7 +232,7 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s set_active(cut_shard(shard), FullNodeShardMode::active_temp); } - auto info_set_mode = [&](ShardIdFull shard, ShardInfo& info, FullNodeShardMode mode) { + auto info_set_mode = [&](ShardIdFull shard, ShardInfo &info, FullNodeShardMode mode) { if (info.mode == mode) { return; } @@ -251,7 +251,7 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s info.exists = true; } - for (auto& p : shards_) { + for (auto &p : shards_) { ShardIdFull shard = p.first; ShardInfo &info = p.second; info.exists = new_shards.count(shard); @@ -259,7 +259,7 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s info_set_mode(shard, info, it == new_active.end() ? FullNodeShardMode::inactive : it->second); } - for (const auto& s : new_active) { + for (const auto &s : new_active) { info_set_mode(s.first, shards_[s.first], s.second); } @@ -276,21 +276,25 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s } } - std::set my_adnl_ids; - for (const auto &p : local_collator_nodes_) { - my_adnl_ids.insert(p.first); - } - for (auto key : local_keys_) { - auto it = current_validators_.find(key); - if (it != current_validators_.end()) { - my_adnl_ids.insert(it->second); + if (!use_old_private_overlays_) { + std::set my_adnl_ids; + my_adnl_ids.insert(adnl_id_); + for (const auto &p : local_collator_nodes_) { + my_adnl_ids.insert(p.first); + } + for (auto key : local_keys_) { + auto it = current_validators_.find(key); + if (it != current_validators_.end()) { + my_adnl_ids.insert(it->second); + } } - } - if (use_old_private_overlays_) { - private_block_overlays_v2_.destroy_overlays(); - } else { - private_block_overlays_v2_.update_overlays(state, std::move(my_adnl_ids), zero_state_file_hash_, keyring_, adnl_, - rldp_, rldp2_, overlays_, validator_manager_, actor_id(this)); + std::set monitoring_shards; + for (ShardIdFull shard : shards_to_monitor) { + monitoring_shards.insert(cut_shard(shard)); + } + fast_sync_overlays_.update_overlays(state, std::move(my_adnl_ids), std::move(monitoring_shards), + zero_state_file_hash_, keyring_, adnl_, overlays_, validator_manager_, + actor_id(this)); } } @@ -348,9 +352,9 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_shard_block_info, block_id, cc_seqno, data.clone()); } - auto private_overlay = private_block_overlays_v2_.choose_overlay(ShardIdFull(masterchainId)); + auto private_overlay = fast_sync_overlays_.choose_overlay(ShardIdFull(masterchainId)); if (!private_overlay.empty()) { - td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_shard_block_info, block_id, cc_seqno, + td::actor::send_closure(private_overlay, &FullNodeFastSyncOverlay::send_shard_block_info, block_id, cc_seqno, data.clone()); } td::actor::send_closure(shard, &FullNodeShard::send_shard_block_info, block_id, cc_seqno, std::move(data)); @@ -368,9 +372,9 @@ void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_se td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_block_candidate, block_id, cc_seqno, validator_set_hash, data.clone()); } - auto private_overlay = private_block_overlays_v2_.choose_overlay(block_id.shard_full()); + auto private_overlay = fast_sync_overlays_.choose_overlay(block_id.shard_full()); if (!private_overlay.empty()) { - td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_block_candidate, block_id, cc_seqno, + td::actor::send_closure(private_overlay, &FullNodeFastSyncOverlay::send_block_candidate, block_id, cc_seqno, validator_set_hash, data.clone()); } if (broadcast_block_candidates_in_public_overlay_) { @@ -393,9 +397,9 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, bool custom_overlays td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, broadcast.clone()); } - auto private_overlay = private_block_overlays_v2_.choose_overlay(broadcast.block_id.shard_full()); + auto private_overlay = fast_sync_overlays_.choose_overlay(broadcast.block_id.shard_full()); if (!private_overlay.empty()) { - td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_broadcast, broadcast.clone()); + td::actor::send_closure(private_overlay, &FullNodeFastSyncOverlay::send_broadcast, broadcast.clone()); } td::actor::send_closure(shard, &FullNodeShard::send_broadcast, std::move(broadcast)); } @@ -645,9 +649,9 @@ void FullNodeImpl::start_up() { void initial_read_complete(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::initial_read_complete, handle); } - void update_shard_configuration(td::Ref state, std::set shards_to_monitor, - std::set temporary_shards) override { - td::actor::send_closure(id_, &FullNodeImpl::update_shard_configuration, std::move(state), + void on_new_masterchain_block(td::Ref state, std::set shards_to_monitor, + std::set temporary_shards) override { + td::actor::send_closure(id_, &FullNodeImpl::on_new_masterchain_block, std::move(state), std::move(shards_to_monitor), std::move(temporary_shards)); } void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) override { diff --git a/validator/full-node.h b/validator/full-node.h index 0545dfb98d..fa57e4f1c2 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -92,6 +92,9 @@ class FullNode : public td::actor::Actor { virtual void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data) = 0; + virtual void import_fast_sync_member_certificate(adnl::AdnlNodeIdShort local_id, + overlay::OverlayMemberCertificate cert) = 0; + static constexpr td::uint32 max_block_size() { return 4 << 20; } diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 33ccf2cd14..a1f20b9d2f 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -24,7 +24,7 @@ #include "interfaces/proof.h" #include "interfaces/shard.h" #include "full-node-private-overlay.hpp" -#include "full-node-private-overlay-v2.hpp" +#include "full-node-fast-sync-overlays.hpp" #include #include @@ -47,9 +47,8 @@ class FullNodeImpl : public FullNode { void add_collator_adnl_id(adnl::AdnlNodeIdShort id) override; void del_collator_adnl_id(adnl::AdnlNodeIdShort id) override; - void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, - td::uint32 expiry_at, td::uint32 max_size, - td::Promise promise) override; + void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, + td::uint32 max_size, td::Promise promise) override; void import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) override; @@ -60,7 +59,7 @@ class FullNodeImpl : public FullNode { void add_custom_overlay(CustomOverlayParams params, td::Promise promise) override; void del_custom_overlay(std::string name, td::Promise promise) override; - void update_shard_configuration(td::Ref state, std::set shards_to_monitor, + void on_new_masterchain_block(td::Ref state, std::set shards_to_monitor, std::set temporary_shards); void sync_completed(); @@ -95,6 +94,11 @@ class FullNodeImpl : public FullNode { void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data) override; + void import_fast_sync_member_certificate(adnl::AdnlNodeIdShort local_id, + overlay::OverlayMemberCertificate cert) override { + fast_sync_overlays_.add_member_certificate(local_id, std::move(cert)); + } + void start_up() override; FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, @@ -145,12 +149,13 @@ class FullNodeImpl : public FullNode { FullNodeConfig config_; // Private overlays: - // Old overlays - one overlay for all validators - // New overlays (v2) - overlay per shard (monitor_min_split depth). - bool use_old_private_overlays_ = false; // TODO: set from config + // Old overlays - one private overlay for all validators + // New overlays (fast sync overlays) - semiprivate overlay per shard (monitor_min_split depth) + // for validators and authorized nodes + bool use_old_private_overlays_ = false; // TODO: set from config or something std::map> private_block_overlays_; bool broadcast_block_candidates_in_public_overlay_ = false; - FullNodePrivateBlockOverlaysV2 private_block_overlays_v2_; + FullNodeFastSyncOverlays fast_sync_overlays_; struct CustomOverlayInfo { CustomOverlayParams params_; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 3b44e894fe..abe6d5202c 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -57,6 +57,8 @@ using ValidateCandidateResult = td::Variant; class ValidatorManager : public ValidatorManagerInterface { public: + virtual void init_last_masterchain_state(td::Ref state) { + } virtual void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; @@ -140,7 +142,6 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; - virtual void update_shard_configuration(td::Ref state, std::set shards_to_monitor) = 0; virtual void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) = 0; virtual void get_async_serializer_state(td::Promise promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 5153896c0b..dce07eac74 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -272,8 +272,6 @@ class ValidatorManagerImpl : public ValidatorManager { void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; - void update_shard_configuration(td::Ref state, std::set shards_to_monitor) override { - } void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override { UNREACHABLE(); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 428e2c1a9d..d192494b59 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -344,9 +344,6 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_client_state(bool from_db, td::Promise promise) override { UNREACHABLE(); } - void update_shard_configuration(td::Ref state, std::set shards_to_monitor) override { - UNREACHABLE(); - } void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override { UNREACHABLE(); diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index 3c40f1f676..39205f80c3 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -268,6 +268,7 @@ void ValidatorManagerMasterchainReiniter::downloaded_masterchain_state(td::Refreceived_state()); CHECK(handle_->is_applied()); LOG(INFO) << "downloaded masterchain state"; + td::actor::send_closure(manager_, &ValidatorManager::init_last_masterchain_state, state_); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::downloaded_all_shards); diff --git a/validator/manager.cpp b/validator/manager.cpp index 499ad7491b..ac24c8ed5b 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1740,6 +1740,7 @@ void ValidatorManagerImpl::send_top_shard_block_description(td::Refblock_id().shard_full(), desc->catchain_seqno()}] = desc; callback_->send_shard_block_info(desc->block_id(), desc->catchain_seqno(), desc->serialize()); + add_shard_block_description(desc); } } @@ -1826,6 +1827,14 @@ void ValidatorManagerImpl::start_up() { alarm_timestamp().relax(check_waiters_at_); } +void ValidatorManagerImpl::init_last_masterchain_state(td::Ref state) { + if (last_masterchain_state_.not_null()) { + return; + } + last_masterchain_state_ = std::move(state); + update_shard_overlays(); +} + void ValidatorManagerImpl::started(ValidatorManagerInitResult R) { CHECK(R.handle); CHECK(R.state.not_null()); @@ -1873,6 +1882,7 @@ void ValidatorManagerImpl::started(ValidatorManagerInitResult R) { } }); td::actor::send_closure(db_, &Db::get_persistent_state_descriptions, std::move(Q)); + update_shard_overlays(); } void ValidatorManagerImpl::read_gc_list(std::vector list) { @@ -2049,6 +2059,7 @@ void ValidatorManagerImpl::new_masterchain_block() { } } + update_shard_overlays(); update_shards(); update_shard_blocks(); @@ -2081,6 +2092,26 @@ void ValidatorManagerImpl::new_masterchain_block() { } } +void ValidatorManagerImpl::update_shard_overlays() { + CHECK(last_masterchain_state_.not_null()); + std::set shards_to_monitor; + shards_to_monitor.insert(ShardIdFull{masterchainId}); + std::set workchains; + for (const auto& shard : last_masterchain_state_->get_shards()) { + workchains.insert(shard->shard().workchain); + if (opts_->need_monitor(shard->shard(),last_masterchain_state_)) { + shards_to_monitor.insert(shard->shard()); + } + } + for (const auto &[wc, desc] : last_masterchain_state_->get_workchain_list()) { + if (!workchains.count(wc) && desc->active && + opts_->need_monitor(ShardIdFull{wc, shardIdAll}, last_masterchain_state_)) { + shards_to_monitor.insert(ShardIdFull{wc, shardIdAll}); + } + } + callback_->on_new_masterchain_block(last_masterchain_state_, std::move(shards_to_monitor), extra_active_shards_); +} + void ValidatorManagerImpl::update_shards() { if ((last_masterchain_state_->rotated_all_shards() || last_masterchain_seqno_ == 0) && opts_->get_last_fork_masterchain_seqno() <= last_masterchain_seqno_) { @@ -2836,11 +2867,6 @@ void ValidatorManagerImpl::get_shard_client_state(bool from_db, td::Promise state, - std::set shards_to_monitor) { - callback_->update_shard_configuration(std::move(state), std::move(shards_to_monitor), extra_active_shards_); -} - void ValidatorManagerImpl::update_async_serializer_state(AsyncSerializerState state, td::Promise promise) { td::actor::send_closure(db_, &Db::update_async_serializer_state, std::move(state), std::move(promise)); } diff --git a/validator/manager.hpp b/validator/manager.hpp index 2c15eb1125..bb4a1513bf 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -319,6 +319,7 @@ class ValidatorManagerImpl : public ValidatorManager { std::vector perf_timer_stats; void new_masterchain_block(); + void update_shard_overlays(); void update_shards(); void update_shard_blocks(); void written_destroyed_validator_sessions(std::vector> groups); @@ -521,7 +522,6 @@ class ValidatorManagerImpl : public ValidatorManager { void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; - void update_shard_configuration(td::Ref state, std::set shards_to_monitor) override; void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override; void get_async_serializer_state(td::Promise promise) override; @@ -557,6 +557,7 @@ class ValidatorManagerImpl : public ValidatorManager { void finished_wait_data(BlockHandle handle, td::Result> R); void start_up() override; + void init_last_masterchain_state(td::Ref state) override; void started(ValidatorManagerInitResult result); void read_gc_list(std::vector list); diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index 303e6333a6..bb9da158f5 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -70,19 +70,13 @@ void ShardClient::got_init_handle_from_db(BlockHandle handle) { } void ShardClient::got_init_state_from_db(td::Ref state) { - masterchain_state_ = std::move(state); - build_shard_overlays(); - masterchain_state_.clear(); - saved_to_db(); } void ShardClient::start_up_init_mode() { - build_shard_overlays(); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); - td::actor::send_closure(SelfId, &ShardClient::applied_all_shards); + td::actor::send_closure(SelfId, &ShardClient::apply_all_shards); }); td::MultiPromise mp; @@ -166,7 +160,6 @@ void ShardClient::download_masterchain_state() { void ShardClient::got_masterchain_block_state(td::Ref state) { masterchain_state_ = std::move(state); - build_shard_overlays(); if (started_) { apply_all_shards(); } @@ -189,7 +182,9 @@ void ShardClient::apply_all_shards() { ig.add_promise(std::move(P)); auto vec = masterchain_state_->get_shards(); + std::set workchains; for (auto &shard : vec) { + workchains.insert(shard->shard().workchain); if (opts_->need_monitor(shard->shard(), masterchain_state_)) { auto Q = td::PromiseCreator::lambda([SelfId = actor_id(this), promise = ig.get_promise(), shard = shard->shard()](td::Result> R) mutable { @@ -203,6 +198,21 @@ void ShardClient::apply_all_shards() { shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); } } + for (const auto &[wc, desc] : masterchain_state_->get_workchain_list()) { + if (!workchains.count(wc) && desc->active && opts_->need_monitor(ShardIdFull{wc, shardIdAll}, masterchain_state_)) { + auto Q = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = ig.get_promise(), wc](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix(PSTRING() << "workchain " << wc << ": ")); + } else { + td::actor::send_closure(SelfId, &ShardClient::downloaded_shard_state, R.move_as_ok(), std::move(promise)); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, + BlockIdExt{wc, shardIdAll, 0, desc->zerostate_root_hash, desc->zerostate_file_hash}, + shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); + } + } } void ShardClient::downloaded_shard_state(td::Ref state, td::Promise promise) { @@ -223,7 +233,6 @@ void ShardClient::new_masterchain_block_notification(BlockHandle handle, td::Ref masterchain_block_handle_ = std::move(handle); masterchain_state_ = std::move(state); waiting_ = false; - build_shard_overlays(); apply_all_shards(); } @@ -244,42 +253,6 @@ void ShardClient::get_processed_masterchain_block_id(td::Promise pro } } -void ShardClient::build_shard_overlays() { - std::set new_shards_to_monitor; - std::set workchains; - auto cur_time = masterchain_state_->get_unix_time(); - new_shards_to_monitor.insert(ShardIdFull(masterchainId)); - for (const auto &info : masterchain_state_->get_shards()) { - auto shard = info->shard(); - workchains.insert(shard.workchain); - if (opts_->need_monitor(shard, masterchain_state_)) { - new_shards_to_monitor.insert(shard); - } - } - - std::vector new_workchains; - for (const auto &wpair : masterchain_state_->get_workchain_list()) { - ton::WorkchainId wc = wpair.first; - const block::WorkchainInfo *winfo = wpair.second.get(); - auto shard = ShardIdFull(wc); - if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= cur_time && - opts_->need_monitor(shard, masterchain_state_)) { - new_shards_to_monitor.insert(shard); - if (shards_to_monitor_.count(shard) == 0) { - new_workchains.push_back(BlockIdExt(wc, shardIdAll, 0, winfo->zerostate_root_hash, winfo->zerostate_file_hash)); - } - } - } - - td::actor::send_closure(manager_, &ValidatorManager::update_shard_configuration, masterchain_state_, - new_shards_to_monitor); - shards_to_monitor_ = std::move(new_shards_to_monitor); - for (BlockIdExt block_id : new_workchains) { - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, block_id, shard_client_priority(), - td::Timestamp::in(60.0), [](td::Result>) {}); - } -} - void ShardClient::force_update_shard_client(BlockHandle handle, td::Promise promise) { CHECK(!init_mode_); CHECK(!started_); @@ -310,7 +283,6 @@ void ShardClient::force_update_shard_client_ex(BlockHandle handle, td::Ref promise_; - std::set shards_to_monitor_ = {ShardIdFull(masterchainId)}; - public: ShardClient(td::Ref opts, BlockHandle masterchain_block_handle, td::Ref masterchain_state, td::actor::ActorId manager, @@ -64,8 +62,6 @@ class ShardClient : public td::actor::Actor { return 2; } - void build_shard_overlays(); - void start_up() override; void start_up_init_mode(); void start_up_init_mode_finished(); diff --git a/validator/validator.h b/validator/validator.h index 819a46ea95..8cf9368b36 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -169,9 +169,9 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual ~Callback() = default; virtual void initial_read_complete(BlockHandle top_masterchain_blocks) = 0; - virtual void update_shard_configuration(td::Ref state, - std::set shards_to_monitor, - std::set temporary_shards) = 0; + virtual void on_new_masterchain_block(td::Ref state, + std::set shards_to_monitor, + std::set temporary_shards) = 0; virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; From 5f9efc7adde9498268fda85db39a9543563d08de Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 15 Jul 2024 12:07:46 +0300 Subject: [PATCH 100/524] Fix compilation error --- create-hardfork/create-hardfork.cpp | 4 ++++ test/test-ton-collator.cpp | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 32aa01c324..e2ec3f6265 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -236,6 +236,10 @@ class HardforkCreator : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } + void on_new_masterchain_block(td::Ref state, + std::set shards_to_monitor, + std::set temporary_shards) override { + } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } void send_ext_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index d3bdf69247..afebe11fe3 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -323,6 +323,10 @@ class TestNode : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } + void on_new_masterchain_block(td::Ref state, + std::set shards_to_monitor, + std::set temporary_shards) override { + } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } void send_ext_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { From f518c104446e4ebc5c77d8ed68e61ed51cc81149 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 15 Jul 2024 16:21:01 +0300 Subject: [PATCH 101/524] Improve selection neighbours in overlay --- overlay/overlay-peers.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index c61e49652e..a217686132 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -216,7 +216,7 @@ void OverlayImpl::add_peer(OverlayNode node) { CHECK(X); if (peer_list_.neighbours_.size() < max_neighbours() && - !(X->get_node()->flags() & OverlayMemberFlags::DoNotReceiveBroadcasts)) { + !(X->get_node()->flags() & OverlayMemberFlags::DoNotReceiveBroadcasts) && X->get_id() != local_id_) { peer_list_.neighbours_.push_back(X->get_id()); X->set_neighbour(true); } @@ -622,9 +622,17 @@ std::vector OverlayImpl::get_neighbours(td::uint32 max_si return peer_list_.neighbours_; } else { std::vector vec; + std::vector ul; for (td::uint32 i = 0; i < max_size; i++) { - vec.push_back( - peer_list_.neighbours_[td::Random::fast(0, static_cast(peer_list_.neighbours_.size()) - 1)]); + td::uint32 t = td::Random::fast(0, static_cast(peer_list_.neighbours_.size()) - 1 - i); + td::uint32 j; + for (j = 0; j < i; j++) { + if (ul[j] <= t) { + t++; + } + } + ul.emplace(ul.begin() + j, t); + vec.push_back(peer_list_.neighbours_[t]); } return vec; } From d81b69e38bce8e3c9fd57a663be1f8509a91c1b8 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 17 Jul 2024 15:34:45 +0300 Subject: [PATCH 102/524] Use previous and next validators in fast sync overlays --- validator/full-node-fast-sync-overlays.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index 7a1c4209cd..b88a55446d 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -318,14 +318,14 @@ void FullNodeFastSyncOverlays::update_overlays(td::Ref state, for (const ValidatorDescr &val : val_set->export_vector()) { PublicKeyHash public_key_hash = ValidatorFullId{val.key}.compute_short_id(); root_public_keys_.push_back(public_key_hash); - if (i == 0) { - current_validators_adnl_.emplace_back(val.addr.is_zero() ? public_key_hash.bits256_value() : val.addr); - } + current_validators_adnl_.emplace_back(val.addr.is_zero() ? public_key_hash.bits256_value() : val.addr); } } std::sort(root_public_keys_.begin(), root_public_keys_.end()); root_public_keys_.erase(std::unique(root_public_keys_.begin(), root_public_keys_.end()), root_public_keys_.end()); std::sort(current_validators_adnl_.begin(), current_validators_adnl_.end()); + current_validators_adnl_.erase(std::unique(current_validators_adnl_.begin(), current_validators_adnl_.end()), + current_validators_adnl_.end()); for (auto &[local_id, overlays_info] : id_to_overlays_) { overlays_info.is_validator_ = From 36b98e8aa022b0b7a3cb59ce169c5264aad338c1 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 17 Jul 2024 16:52:56 +0300 Subject: [PATCH 103/524] Fix compilation error in shard-client.cpp --- validator/shard-client.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index bb9da158f5..d9cdcf7e36 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -200,14 +200,14 @@ void ShardClient::apply_all_shards() { } for (const auto &[wc, desc] : masterchain_state_->get_workchain_list()) { if (!workchains.count(wc) && desc->active && opts_->need_monitor(ShardIdFull{wc, shardIdAll}, masterchain_state_)) { - auto Q = td::PromiseCreator::lambda( - [SelfId = actor_id(this), promise = ig.get_promise(), wc](td::Result> R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error_prefix(PSTRING() << "workchain " << wc << ": ")); - } else { - td::actor::send_closure(SelfId, &ShardClient::downloaded_shard_state, R.move_as_ok(), std::move(promise)); - } - }); + auto Q = td::PromiseCreator::lambda([SelfId = actor_id(this), promise = ig.get_promise(), + workchain = wc](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix(PSTRING() << "workchain " << workchain << ": ")); + } else { + td::actor::send_closure(SelfId, &ShardClient::downloaded_shard_state, R.move_as_ok(), std::move(promise)); + } + }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, BlockIdExt{wc, shardIdAll, 0, desc->zerostate_root_hash, desc->zerostate_file_hash}, shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); From 5743dbf335816483cc796623ce267318795a7125 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 18 Jul 2024 12:43:55 +0300 Subject: [PATCH 104/524] Changes in fast sync overlays Join all overlays (not only monitoring shards) in order to be able to send broadcasts to other overlays --- validator/full-node-fast-sync-overlays.cpp | 48 ++++++++++++++-------- validator/full-node-fast-sync-overlays.hpp | 5 ++- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index b88a55446d..d641215f24 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -201,8 +201,8 @@ void FullNodeFastSyncOverlay::init() { current_validators_adnl_.end(); if (!shard_.is_masterchain()) { options.default_permanent_members_flags_ = overlay::OverlayMemberFlags::DoNotReceiveBroadcasts; - options.local_overlay_member_flags_ = is_validator ? overlay::OverlayMemberFlags::DoNotReceiveBroadcasts : 0; } + options.local_overlay_member_flags_ = receive_broadcasts_ ? 0 : overlay::OverlayMemberFlags::DoNotReceiveBroadcasts; options.max_slaves_in_semiprivate_overlay_ = 100000; // TODO: set lower limit (high limit for testing) td::actor::send_closure(overlays_, &overlay::Overlays::create_semiprivate_overlay, local_id_, overlay_id_full_.clone(), current_validators_adnl_, root_public_keys_, member_certificate_, @@ -235,6 +235,17 @@ void FullNodeFastSyncOverlay::set_member_certificate(overlay::OverlayMemberCerti } } +void FullNodeFastSyncOverlay::set_receive_broadcasts(bool value) { + if (value == receive_broadcasts_) { + return; + } + receive_broadcasts_ = value; + if (inited_) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_); + init(); + } +} + void FullNodeFastSyncOverlay::get_stats_extra(td::Promise promise) { auto res = create_tl_object(); res->shard_ = shard_.to_str(); @@ -278,19 +289,21 @@ void FullNodeFastSyncOverlays::update_overlays(td::Ref state, monitoring_shards.insert(ShardIdFull{masterchainId}); std::set all_shards; all_shards.insert(ShardIdFull{masterchainId}); - td::uint32 monitor_min_split = state->monitor_min_split_depth(basechainId); - for (td::uint64 i = 0; i < (1ULL << monitor_min_split); ++i) { - all_shards.insert(ShardIdFull{basechainId, (i * 2 + 1) << (63 - monitor_min_split)}); + for (const auto& desc : state->get_shards()) { + ShardIdFull shard = desc->shard(); + td::uint32 monitor_min_split = state->monitor_min_split_depth(shard.workchain); + if (shard.pfx_len() > monitor_min_split) { + shard = shard_prefix(shard, monitor_min_split); + } + all_shards.insert(shard); } // Remove overlays for removed adnl ids and shards for (auto it = id_to_overlays_.begin(); it != id_to_overlays_.end();) { if (my_adnl_ids.count(it->first)) { auto &overlays_info = it->second; - ; - auto ¤t_shards = overlays_info.is_validator_ ? all_shards : monitoring_shards; for (auto it2 = overlays_info.overlays_.begin(); it2 != overlays_info.overlays_.end();) { - if (current_shards.count(it2->first)) { + if (all_shards.count(it2->first)) { ++it2; } else { it2 = overlays_info.overlays_.erase(it2); @@ -330,7 +343,7 @@ void FullNodeFastSyncOverlays::update_overlays(td::Ref state, for (auto &[local_id, overlays_info] : id_to_overlays_) { overlays_info.is_validator_ = std::binary_search(current_validators_adnl_.begin(), current_validators_adnl_.end(), local_id); - for (auto &[_, overlay] : overlays_info.overlays_) { + for (auto &[shard, overlay] : overlays_info.overlays_) { td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::set_validators, root_public_keys_, current_validators_adnl_); } @@ -388,17 +401,20 @@ void FullNodeFastSyncOverlays::update_overlays(td::Ref state, } // Update shard overlays - auto ¤t_shards = overlays_info.is_validator_ ? all_shards : monitoring_shards; - for (ShardIdFull shard_id : current_shards) { - auto &overlay = overlays_info.overlays_[shard_id]; + for (ShardIdFull shard : all_shards) { + bool receive_broadcasts = overlays_info.is_validator_ ? shard.is_masterchain() : monitoring_shards.count(shard); + auto &overlay = overlays_info.overlays_[shard]; if (overlay.empty()) { overlay = td::actor::create_actor( - PSTRING() << "FastSyncOv" << shard_id.to_str(), local_id, shard_id, zero_state_file_hash, - root_public_keys_, current_validators_adnl_, overlays_info.current_certificate_, keyring, adnl, overlays, + PSTRING() << "FastSyncOv" << shard.to_str(), local_id, shard, zero_state_file_hash, root_public_keys_, + current_validators_adnl_, overlays_info.current_certificate_, receive_broadcasts, keyring, adnl, overlays, validator_manager, full_node); - } else if (changed_certificate) { - td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::set_member_certificate, - overlays_info.current_certificate_); + } else { + td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::set_receive_broadcasts, receive_broadcasts); + if (changed_certificate) { + td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::set_member_certificate, + overlays_info.current_certificate_); + } } } } diff --git a/validator/full-node-fast-sync-overlays.hpp b/validator/full-node-fast-sync-overlays.hpp index 30cb5dde81..b89f880245 100644 --- a/validator/full-node-fast-sync-overlays.hpp +++ b/validator/full-node-fast-sync-overlays.hpp @@ -49,11 +49,12 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { void set_validators(std::vector root_public_keys, std::vector current_validators_adnl); void set_member_certificate(overlay::OverlayMemberCertificate member_certificate); + void set_receive_broadcasts(bool value); FullNodeFastSyncOverlay(adnl::AdnlNodeIdShort local_id, ShardIdFull shard, FileHash zero_state_file_hash, std::vector root_public_keys, std::vector current_validators_adnl, - overlay::OverlayMemberCertificate member_certificate, + overlay::OverlayMemberCertificate member_certificate, bool receive_broadcasts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId overlays, td::actor::ActorId validator_manager, @@ -63,6 +64,7 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { , root_public_keys_(std::move(root_public_keys)) , current_validators_adnl_(std::move(current_validators_adnl)) , member_certificate_(std::move(member_certificate)) + , receive_broadcasts_(receive_broadcasts) , zero_state_file_hash_(zero_state_file_hash) , keyring_(keyring) , adnl_(adnl) @@ -77,6 +79,7 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { std::vector root_public_keys_; std::vector current_validators_adnl_; overlay::OverlayMemberCertificate member_certificate_; + bool receive_broadcasts_; FileHash zero_state_file_hash_; td::actor::ActorId keyring_; From 4c8d25ac1bc0d91d187f98c5c36cb11c55d842c3 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 24 Jul 2024 18:02:59 +0300 Subject: [PATCH 105/524] Save and send block candidates in collator nodes --- validator/collator-node.cpp | 23 ++++++++++++++++++++--- validator/full-node.cpp | 28 +++++++++++++++------------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 2da56e802a..24ecda2a2a 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -256,7 +256,8 @@ static td::BufferSlice serialize_error(td::Status error) { return create_serialize_tl_object(error.code(), error.message().c_str()); } -static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey creator) { +static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey creator, CatchainSeqno* cc_seqno = nullptr, + td::uint32* val_set_hash = nullptr) { CHECK(!block.id.is_masterchain()); if (block.pubkey == creator) { return block; @@ -264,8 +265,10 @@ static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey cre auto root = vm::std_boc_deserialize(block.data).move_as_ok(); block::gen::Block::Record blk; block::gen::BlockExtra::Record extra; + block::gen::BlockInfo::Record info; CHECK(tlb::unpack_cell(root, blk)); CHECK(tlb::unpack_cell(blk.extra, extra)); + CHECK(tlb::unpack_cell(blk.info, info)); extra.created_by = creator.as_bits256(); CHECK(tlb::pack_cell(blk.extra, extra)); CHECK(tlb::pack_cell(root, blk)); @@ -274,6 +277,13 @@ static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey cre block.id.root_hash = root->get_hash().bits(); block.id.file_hash = block::compute_file_hash(block.data.as_slice()); block.pubkey = creator; + + if (cc_seqno) { + *cc_seqno = info.gen_catchain_seqno; + } + if (val_set_hash) { + *val_set_hash = info.gen_validator_list_hash_short; + } return block; } @@ -304,9 +314,16 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data prev_blocks.push_back(create_block_id(b)); } Ed25519_PublicKey creator(f->creator_); - new_promise = [new_promise = std::move(new_promise), creator](td::Result R) mutable { + new_promise = [new_promise = std::move(new_promise), creator, + manager = manager_](td::Result R) mutable { TRY_RESULT_PROMISE(new_promise, block, std::move(R)); - new_promise.set_value(change_creator(std::move(block), creator)); + CatchainSeqno cc_seqno; + td::uint32 val_set_hash; + block = change_creator(std::move(block), creator, &cc_seqno, &val_set_hash); + td::Promise P = + new_promise.wrap([block = block.clone()](td::Unit&&) mutable -> BlockCandidate { return std::move(block); }); + td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, block.id, std::move(block), cc_seqno, + val_set_hash, std::move(P)); }; if (!shard.is_valid_ext()) { new_promise.set_error(td::Status::Error(PSTRING() << "invalid shard " << shard.to_str())); diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 8e84a4d456..d447943f6c 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -352,9 +352,9 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_shard_block_info, block_id, cc_seqno, data.clone()); } - auto private_overlay = fast_sync_overlays_.choose_overlay(ShardIdFull(masterchainId)); - if (!private_overlay.empty()) { - td::actor::send_closure(private_overlay, &FullNodeFastSyncOverlay::send_shard_block_info, block_id, cc_seqno, + auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(ShardIdFull(masterchainId)); + if (!fast_sync_overlay.empty()) { + td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_shard_block_info, block_id, cc_seqno, data.clone()); } td::actor::send_closure(shard, &FullNodeShard::send_shard_block_info, block_id, cc_seqno, std::move(data)); @@ -372,9 +372,9 @@ void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_se td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_block_candidate, block_id, cc_seqno, validator_set_hash, data.clone()); } - auto private_overlay = fast_sync_overlays_.choose_overlay(block_id.shard_full()); - if (!private_overlay.empty()) { - td::actor::send_closure(private_overlay, &FullNodeFastSyncOverlay::send_block_candidate, block_id, cc_seqno, + auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(block_id.shard_full()); + if (!fast_sync_overlay.empty()) { + td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_block_candidate, block_id, cc_seqno, validator_set_hash, data.clone()); } if (broadcast_block_candidates_in_public_overlay_) { @@ -393,13 +393,15 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, bool custom_overlays VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; return; } - if (broadcast.block_id.is_masterchain() && !private_block_overlays_.empty()) { - td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, - broadcast.clone()); - } - auto private_overlay = fast_sync_overlays_.choose_overlay(broadcast.block_id.shard_full()); - if (!private_overlay.empty()) { - td::actor::send_closure(private_overlay, &FullNodeFastSyncOverlay::send_broadcast, broadcast.clone()); + if (broadcast.block_id.is_masterchain()) { + if (!private_block_overlays_.empty()) { + td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, + broadcast.clone()); + } + auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(broadcast.block_id.shard_full()); + if (!fast_sync_overlay.empty()) { + td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_broadcast, broadcast.clone()); + } } td::actor::send_closure(shard, &FullNodeShard::send_broadcast, std::move(broadcast)); } From 0e7374610d454f5671042a58f68ce4cf40b890cb Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 25 Jul 2024 16:33:19 +0300 Subject: [PATCH 106/524] Improve block broadcasts --- create-hardfork/create-hardfork.cpp | 3 +- overlay/overlays.h | 5 +- test/test-ton-collator.cpp | 3 +- tl/generate/scheme/ton_api.tl | 4 +- tl/generate/scheme/ton_api.tlo | Bin 101564 -> 101528 bytes validator/collator-node.cpp | 17 ++- validator/full-node-fast-sync-overlays.cpp | 4 +- validator/full-node-shard.cpp | 111 ++++++------------ validator/full-node-shard.h | 11 +- validator/full-node-shard.hpp | 38 ++---- validator/full-node.cpp | 128 +++++++-------------- validator/full-node.hpp | 8 +- validator/interfaces/validator-manager.h | 2 - validator/manager-disk.hpp | 2 - validator/manager-hardfork.hpp | 2 - validator/manager.cpp | 31 +---- validator/manager.hpp | 10 -- validator/validator-group.cpp | 5 +- validator/validator-group.hpp | 10 +- validator/validator.h | 3 +- 20 files changed, 122 insertions(+), 275 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index e2ec3f6265..8956771d63 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -237,8 +237,7 @@ class HardforkCreator : public td::actor::Actor { td::PromiseCreator::lambda([](td::Unit) {})); } void on_new_masterchain_block(td::Ref state, - std::set shards_to_monitor, - std::set temporary_shards) override { + std::set shards_to_monitor) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/overlay/overlays.h b/overlay/overlays.h index cc569343d5..cc112bc430 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -206,7 +206,10 @@ class OverlayMemberCertificate { return expire_at_ < cur_time - 3; } - auto tl() const { + tl_object_ptr tl() const { + if (empty()) { + return create_tl_object(); + } return create_tl_object(signed_by_.tl(), flags_, slot_, expire_at_, signature_.clone_as_buffer_slice()); } diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index afebe11fe3..c6aee54b32 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -324,8 +324,7 @@ class TestNode : public td::actor::Actor { td::PromiseCreator::lambda([](td::Unit) {})); } void on_new_masterchain_block(td::Ref state, - std::set shards_to_monitor, - std::set temporary_shards) override { + std::set shards_to_monitor) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 7238574419..df9a54fd0d 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -698,8 +698,8 @@ engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_i engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; engine.validator.shardOverlayStats.neighbour id:string verison_major:int version_minor:int flags:# - roundtrip:double unreliability:double has_state:string = engine.validator.shardOverlayStats.Neighbour; -engine.validator.shardOverlayStats shard:string mode:string + roundtrip:double unreliability:double = engine.validator.shardOverlayStats.Neighbour; +engine.validator.shardOverlayStats shard:string active:Bool neighbours:(vector engine.validator.shardOverlayStats.neighbour) = engine.validator.ShardOverlayStats; engine.validator.fastSyncOverlayStats shard:string validators_adnl:(vector int256) root_public_keys:(vector int256) member_certificate:overlay.MemberCertificate = engine.validator.FastSyncOverlayStats; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 4e080adea6755f3f0146586f56616042b9c7a3b8..532a1d3a334f6cc686ec78a758c2a51a4b536608 100644 GIT binary patch delta 140 zcmdlplWoRKwhbaOEFYvwoi|I!s4%fi`ffXOvxD58>0q%*+f+C~ayO3&`7lquSSvnx z!YYC3GUbdi(*v>@StplnQWwOR@9x!Km Xf*_;B0q%*+f+C~ayO3&`7lquSSvnx z!YTn4mfZZ5$seA`OfK1`Apz3B$N&LF&TKjywrvN&#%C3f(rokR6jES diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 24ecda2a2a..a817d92aac 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -256,8 +256,8 @@ static td::BufferSlice serialize_error(td::Status error) { return create_serialize_tl_object(error.code(), error.message().c_str()); } -static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey creator, CatchainSeqno* cc_seqno = nullptr, - td::uint32* val_set_hash = nullptr) { +static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey creator, CatchainSeqno& cc_seqno, + td::uint32& val_set_hash) { CHECK(!block.id.is_masterchain()); if (block.pubkey == creator) { return block; @@ -278,12 +278,8 @@ static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey cre block.id.file_hash = block::compute_file_hash(block.data.as_slice()); block.pubkey = creator; - if (cc_seqno) { - *cc_seqno = info.gen_catchain_seqno; - } - if (val_set_hash) { - *val_set_hash = info.gen_validator_list_hash_short; - } + cc_seqno = info.gen_catchain_seqno; + val_set_hash = info.gen_validator_list_hash_short; return block; } @@ -305,8 +301,9 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data }; if (!validator_adnl_ids_.count(src)) { new_promise.set_error(td::Status::Error("src is not a validator")); + return; } - TRY_RESULT_PROMISE(new_promise, f, fetch_tl_object(std::move(data), true)); + TRY_RESULT_PROMISE(new_promise, f, fetch_tl_object(data, true)); ShardIdFull shard = create_shard_id(f->shard_); CatchainSeqno cc_seqno = f->cc_seqno_; std::vector prev_blocks; @@ -319,7 +316,7 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data TRY_RESULT_PROMISE(new_promise, block, std::move(R)); CatchainSeqno cc_seqno; td::uint32 val_set_hash; - block = change_creator(std::move(block), creator, &cc_seqno, &val_set_hash); + block = change_creator(std::move(block), creator, cc_seqno, val_set_hash); td::Promise P = new_promise.wrap([block = block.clone()](td::Unit&&) mutable -> BlockCandidate { return std::move(block); }); td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, block.id, std::move(block), cc_seqno, diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index d641215f24..a5789771bd 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -255,7 +255,9 @@ void FullNodeFastSyncOverlay::get_stats_extra(td::Promise promise) for (const auto &x : root_public_keys_) { res->root_public_keys_.push_back(x.bits256_value()); } - res->member_certificate_ = member_certificate_.tl(); + if (!member_certificate_.empty()) { + res->member_certificate_ = member_certificate_.tl(); + } promise.set_result(td::json_encode(td::ToJson(*res), true)); } diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 00f3c6a2a1..29950363de 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -104,7 +104,7 @@ void FullNodeShardImpl::create_overlay() { td::actor::ActorId node_; }; overlay::OverlayOptions opts; - opts.announce_self_ = is_active(); + opts.announce_self_ = active_; td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay_ex, adnl_id_, overlay_id_full_.clone(), std::make_unique(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() @@ -119,7 +119,7 @@ void FullNodeShardImpl::create_overlay() { } void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broadcast, td::Promise promise) { - if (mode_ != FullNodeShardMode::active) { + if (!active_) { return promise.set_error(td::Status::Error("cannot check broadcast: shard is not active")); } auto B = fetch_tl_object(std::move(broadcast), true); @@ -161,16 +161,16 @@ void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promis create_overlay(); } -void FullNodeShardImpl::set_mode(FullNodeShardMode mode) { +void FullNodeShardImpl::set_active(bool active) { if (shard_.is_masterchain()) { return; } - bool was_active = is_active(); - mode_ = mode; - if (was_active != is_active()) { - td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); - create_overlay(); + if (active_ == active) { + return; } + active_ = active; + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); + create_overlay(); } void FullNodeShardImpl::try_get_next_block(td::Timestamp timeout, td::Promise promise) { @@ -219,7 +219,6 @@ void FullNodeShardImpl::got_next_block(td::Result R) { } void FullNodeShardImpl::get_next_block() { - //return; attempt_++; auto P = td::PromiseCreator::lambda([validator_manager = validator_manager_, attempt = attempt_, block_id = handle_->id(), SelfId = actor_id(this)](td::Result R) { @@ -623,12 +622,8 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise) { VLOG(FULL_NODE_DEBUG) << "Got query getCapabilities from " << src; - td::uint32 flags = 0; - if (mode_ != FullNodeShardMode::active) { - flags |= Neighbour::FLAG_NO_STATE; - } promise.set_value( - create_serialize_tl_object(proto_version_major(), proto_version_minor(), flags)); + create_serialize_tl_object(proto_version_major(), proto_version_minor(), 0)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, @@ -717,7 +712,7 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { - if (!is_active()) { + if (!active_) { td::actor::send_closure(overlays_, &overlay::Overlays::send_message, src, adnl_id_, overlay_id_, create_serialize_tl_object()); promise.set_error(td::Status::Error("shard is inactive")); @@ -811,7 +806,7 @@ void FullNodeShardImpl::process_block_broadcast(PublicKeyHash src, ton_api::tonN } void FullNodeShardImpl::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { - if (!is_active()) { + if (!active_) { return; } auto B = fetch_tl_object(std::move(broadcast), true); @@ -983,7 +978,7 @@ void FullNodeShardImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp t void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) { - auto &b = choose_neighbour(true); + auto &b = choose_neighbour(); td::actor::create_actor( "archive", masterchain_seqno, shard_prefix, std::move(tmp_dir), adnl_id_, overlay_id_, b.adnl_id, timeout, validator_manager_, rldp2_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) @@ -994,7 +989,7 @@ void FullNodeShardImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std: block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise>> promise) { // TODO: maybe more complex download (like other requests here) - auto &b = choose_neighbour(true); + auto &b = choose_neighbour(); if (b.adnl_id == adnl::AdnlNodeIdShort::zero()) { promise.set_error(td::Status::Error(ErrorCode::notready, "no nodes")); return; @@ -1213,22 +1208,11 @@ void FullNodeShardImpl::got_neighbours(std::vector vec) { continue; } if (neighbours_.size() == max_neighbours()) { - td::uint32 neighbours_with_state = 0; - for (const auto &n : neighbours_) { - if (n.second.has_state_known() && n.second.has_state()) { - ++neighbours_with_state; - } - } - adnl::AdnlNodeIdShort a = adnl::AdnlNodeIdShort::zero(); adnl::AdnlNodeIdShort b = adnl::AdnlNodeIdShort::zero(); td::uint32 cnt = 0; double u = 0; for (auto &n : neighbours_) { - if (neighbours_with_state <= min_neighbours_with_state() && n.second.has_state_known() && - n.second.has_state()) { - continue; - } if (n.second.unreliability > u) { u = n.second.unreliability; a = n.first; @@ -1252,7 +1236,7 @@ void FullNodeShardImpl::got_neighbours(std::vector vec) { } } -const Neighbour &FullNodeShardImpl::choose_neighbour(bool require_state) const { +const Neighbour &FullNodeShardImpl::choose_neighbour() const { if (neighbours_.size() == 0) { return Neighbour::zero; } @@ -1261,40 +1245,30 @@ const Neighbour &FullNodeShardImpl::choose_neighbour(bool require_state) const { for (auto &x : neighbours_) { min_unreliability = std::min(min_unreliability, x.second.unreliability); } - for (int attempt = 0; attempt < (require_state ? 2 : 1); ++attempt) { - const Neighbour *best = nullptr; - td::uint32 sum = 0; + const Neighbour *best = nullptr; + td::uint32 sum = 0; - for (auto &x : neighbours_) { - if (require_state) { - if (attempt == 0 && !(x.second.has_state_known() && x.second.has_state())) { - continue; - } - if (attempt == 1 && x.second.has_state_known()) { - continue; - } - } - auto unr = static_cast(x.second.unreliability - min_unreliability); + for (auto &x : neighbours_) { + auto unr = static_cast(x.second.unreliability - min_unreliability); - if (x.second.version_major < proto_version_major()) { - unr += 4; - } else if (x.second.version_major == proto_version_major() && x.second.version_minor < proto_version_minor()) { - unr += 2; - } + if (x.second.version_major < proto_version_major()) { + unr += 4; + } else if (x.second.version_major == proto_version_major() && x.second.version_minor < proto_version_minor()) { + unr += 2; + } - auto f = static_cast(fail_unreliability()); + auto f = static_cast(fail_unreliability()); - if (unr <= f) { - auto w = 1 << (f - unr); - sum += w; - if (td::Random::fast(0, sum - 1) <= w - 1) { - best = &x.second; - } + if (unr <= f) { + auto w = 1 << (f - unr); + sum += w; + if (td::Random::fast(0, sum - 1) <= w - 1) { + best = &x.second; } } - if (best) { - return *best; - } + } + if (best) { + return *best; } return Neighbour::zero; } @@ -1361,17 +1335,7 @@ void FullNodeShardImpl::ping_neighbours() { void FullNodeShardImpl::get_stats_extra(td::Promise promise) { auto res = create_tl_object(); res->shard_ = shard_.to_str(); - switch (mode_) { - case active: - res->mode_ = "active"; - break; - case active_temp: - res->mode_ = "active_temp"; - break; - case inactive: - res->mode_ = "inactive"; - break; - } + res->active_ = active_; for (const auto &p : neighbours_) { const auto &n = p.second; auto f = create_tl_object(); @@ -1381,7 +1345,6 @@ void FullNodeShardImpl::get_stats_extra(td::Promise promise) { f->flags_ = n.flags; f->roundtrip_ = n.roundtrip; f->unreliability_ = n.unreliability; - f->has_state_ = (n.has_state_known() ? (n.has_state() ? "true" : "false") : "undefined"); res->neighbours_.push_back(std::move(f)); } promise.set_result(td::json_encode(td::ToJson(*res), true)); @@ -1394,7 +1357,7 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, - td::actor::ActorId full_node, FullNodeShardMode mode) + td::actor::ActorId full_node, bool active) : shard_(shard) , local_id_(local_id) , adnl_id_(adnl_id) @@ -1407,7 +1370,7 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, , validator_manager_(validator_manager) , client_(client) , full_node_(full_node) - , mode_(mode) + , active_(active) , config_(config) { } @@ -1416,10 +1379,10 @@ td::actor::ActorOwn FullNodeShard::create( FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, td::actor::ActorId full_node, FullNodeShardMode mode) { + td::actor::ActorId client, td::actor::ActorId full_node, bool active) { return td::actor::create_actor(PSTRING() << "tonnode" << shard.to_str(), shard, local_id, adnl_id, zero_state_file_hash, config, keyring, adnl, rldp, rldp2, overlays, - validator_manager, client, full_node, mode); + validator_manager, client, full_node, active); } } // namespace fullnode diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 235e30514c..169453257b 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -28,12 +28,6 @@ namespace validator { namespace fullnode { -enum FullNodeShardMode { - active, // Node can answer queries about the shard - active_temp, // Like 'active', but queries about shard state are not allowed (only blocks) - inactive // Node is not a part of the overlay -}; - class FullNodeShard : public td::actor::Actor { public: virtual ~FullNodeShard() = default; @@ -42,7 +36,7 @@ class FullNodeShard : public td::actor::Actor { virtual ShardIdFull get_shard_full() const = 0; virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; - virtual void set_mode(FullNodeShardMode mode) = 0; + virtual void set_active(bool active) = 0; virtual void set_config(FullNodeConfig config) = 0; virtual void send_ihr_message(td::BufferSlice data) = 0; @@ -85,8 +79,7 @@ class FullNodeShard : public td::actor::Actor { FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, td::actor::ActorId full_node, - FullNodeShardMode mode = FullNodeShardMode::active); + td::actor::ActorId client, td::actor::ActorId full_node, bool active); }; } // namespace fullnode diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 03e5fc0d77..8ac5185d91 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -40,22 +40,14 @@ struct Neighbour { double roundtrip_weight = 0; double unreliability = 0; - Neighbour(adnl::AdnlNodeIdShort adnl_id) : adnl_id(std::move(adnl_id)) { + explicit Neighbour(adnl::AdnlNodeIdShort adnl_id) : adnl_id(std::move(adnl_id)) { } void update_proto_version(ton_api::tonNode_capabilities &q); void query_success(double t); void query_failed(); void update_roundtrip(double t); - bool has_state() const { - return !(flags & FLAG_NO_STATE); - } - bool has_state_known() const { - return version_major != 0; - } - static Neighbour zero; - static constexpr td::uint32 FLAG_NO_STATE = 1; }; class FullNodeShardImpl : public FullNodeShard { @@ -82,9 +74,6 @@ class FullNodeShardImpl : public FullNodeShard { static constexpr td::uint32 max_neighbours() { return 16; } - static constexpr td::uint32 min_neighbours_with_state() { - return 10; - } static constexpr double stop_unreliability() { return 5.0; } @@ -94,15 +83,12 @@ class FullNodeShardImpl : public FullNodeShard { void create_overlay(); void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; - void set_mode(FullNodeShardMode mode) override; + void set_active(bool active) override; void set_config(FullNodeConfig config) override { config_ = config; } - //td::Result fetch_block(td::BufferSlice data); - void prevalidate_block(BlockIdExt block_id, td::BufferSlice data, td::BufferSlice proof, - td::Promise promise); void try_get_next_block(td::Timestamp timestamp, td::Promise promise); void got_next_block(td::Result block); void get_next_block(); @@ -155,8 +141,6 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getOutMsgQueueProof &query, td::Promise promise); - // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query, - // td::Promise promise); void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise); void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data); @@ -222,16 +206,14 @@ class FullNodeShardImpl : public FullNodeShard { void got_neighbours(std::vector res); void update_neighbour_stats(adnl::AdnlNodeIdShort adnl_id, double t, bool success); void got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id, double t, td::BufferSlice data); - const Neighbour &choose_neighbour(bool require_state = false) const; + const Neighbour &choose_neighbour() const; template td::Promise create_neighbour_promise(const Neighbour &x, td::Promise p, bool require_state = false) { - return td::PromiseCreator::lambda([id = x.adnl_id, SelfId = actor_id(this), p = std::move(p), ts = td::Time::now(), - ignore_error = require_state && !x.has_state_known()](td::Result R) mutable { + return td::PromiseCreator::lambda([id = x.adnl_id, SelfId = actor_id(this), p = std::move(p), + ts = td::Time::now()](td::Result R) mutable { if (R.is_error() && R.error().code() != ErrorCode::notready && R.error().code() != ErrorCode::cancelled) { - if (!ignore_error) { - td::actor::send_closure(SelfId, &FullNodeShardImpl::update_neighbour_stats, id, td::Time::now() - ts, false); - } + td::actor::send_closure(SelfId, &FullNodeShardImpl::update_neighbour_stats, id, td::Time::now() - ts, false); } else { td::actor::send_closure(SelfId, &FullNodeShardImpl::update_neighbour_stats, id, td::Time::now() - ts, true); } @@ -245,17 +227,13 @@ class FullNodeShardImpl : public FullNodeShard { td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, td::actor::ActorId full_node, - FullNodeShardMode mode = FullNodeShardMode::active); + bool active); private: bool use_new_download() const { return false; } - bool is_active() const { - return mode_ != FullNodeShardMode::inactive; - } - ShardIdFull shard_; BlockHandle handle_; td::Promise promise_; @@ -289,7 +267,7 @@ class FullNodeShardImpl : public FullNodeShard { td::Timestamp ping_neighbours_at_; adnl::AdnlNodeIdShort last_pinged_neighbour_ = adnl::AdnlNodeIdShort::zero(); - FullNodeShardMode mode_; + bool active_; FullNodeConfig config_; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index d447943f6c..651c014945 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -198,81 +198,56 @@ void FullNodeImpl::initial_read_complete(BlockHandle top_handle) { td::actor::send_closure(it->second.actor, &FullNodeShard::set_handle, top_handle, std::move(P)); } -void FullNodeImpl::on_new_masterchain_block(td::Ref state, std::set shards_to_monitor, - std::set temporary_shards) { +void FullNodeImpl::on_new_masterchain_block(td::Ref state, std::set shards_to_monitor) { CHECK(shards_to_monitor.count(ShardIdFull(masterchainId))); - std::set new_shards; - std::map new_active; - new_shards.insert(ShardIdFull(masterchainId)); + bool join_all_overlays = !sign_cert_by_.is_zero(); + std::set all_shards; + std::set new_active; + all_shards.insert(ShardIdFull(masterchainId)); std::set workchains; auto cut_shard = [&](ShardIdFull shard) -> ShardIdFull { int min_split = state->monitor_min_split_depth(shard.workchain); return min_split < shard.pfx_len() ? shard_prefix(shard, min_split) : shard; }; - auto set_active = [&](ShardIdFull shard, FullNodeShardMode mode) { - while (new_active.emplace(shard, mode).second && shard.pfx_len() > 0) { + auto set_active = [&](ShardIdFull shard) { + while (new_active.emplace(shard).second && shard.pfx_len() > 0) { shard = shard_parent(shard); } }; for (auto &info : state->get_shards()) { workchains.insert(info->shard().workchain); - new_shards.insert(cut_shard(info->shard())); + all_shards.insert(cut_shard(info->shard())); } for (const auto &wpair : state->get_workchain_list()) { ton::WorkchainId wc = wpair.first; const block::WorkchainInfo *winfo = wpair.second.get(); if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= state->get_unix_time()) { - new_shards.insert(ShardIdFull(wc)); + all_shards.insert(ShardIdFull(wc)); } } for (ShardIdFull shard : shards_to_monitor) { - set_active(cut_shard(shard), FullNodeShardMode::active); - } - for (ShardIdFull shard : temporary_shards) { - set_active(cut_shard(shard), FullNodeShardMode::active_temp); + set_active(cut_shard(shard)); } - auto info_set_mode = [&](ShardIdFull shard, ShardInfo &info, FullNodeShardMode mode) { - if (info.mode == mode) { - return; - } - if (info.actor.empty()) { - add_shard_actor(shard, mode); - return; + for (auto it = shards_.begin(); it != shards_.end(); ) { + if (all_shards.count(it->first)) { + ++it; + } else { + it = shards_.erase(it); } - info.mode = mode; - td::actor::send_closure(info.actor, &FullNodeShard::set_mode, mode); - info.delete_at = - mode != FullNodeShardMode::inactive ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); - }; - - for (auto shard : new_shards) { - auto &info = shards_[shard]; - info.exists = true; - } - - for (auto &p : shards_) { - ShardIdFull shard = p.first; - ShardInfo &info = p.second; - info.exists = new_shards.count(shard); - auto it = new_active.find(shard); - info_set_mode(shard, info, it == new_active.end() ? FullNodeShardMode::inactive : it->second); } - - for (const auto &s : new_active) { - info_set_mode(s.first, shards_[s.first], s.second); + for (ShardIdFull shard : all_shards) { + bool active = new_active.count(shard); + bool overlay_exists = !shards_[shard].actor.empty(); + if (active || join_all_overlays || overlay_exists) { + update_shard_actor(shard, active); + } } - auto it = shards_.begin(); - while (it != shards_.end()) { - if (it->second.mode == FullNodeShardMode::inactive && it->second.delete_at && it->second.delete_at.is_in_past()) { - it->second.actor.reset(); - it->second.delete_at = td::Timestamp::never(); - } - if (!it->second.exists && it->second.actor.empty()) { - it = shards_.erase(it); - } else { - ++it; + for (auto &[_, shard_info] : shards_) { + if (!shard_info.active && shard_info.delete_at && shard_info.delete_at.is_in_past() && !join_all_overlays) { + shard_info.actor = {}; + shard_info.delete_at = td::Timestamp::never(); } } @@ -298,18 +273,19 @@ void FullNodeImpl::on_new_masterchain_block(td::Ref state, std } } -void FullNodeImpl::add_shard_actor(ShardIdFull shard, FullNodeShardMode mode) { +void FullNodeImpl::update_shard_actor(ShardIdFull shard, bool active) { ShardInfo &info = shards_[shard]; - if (!info.actor.empty()) { - return; - } - info.actor = FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, config_, keyring_, adnl_, rldp_, - rldp2_, overlays_, validator_manager_, client_, actor_id(this), mode); - info.mode = mode; - info.delete_at = mode != FullNodeShardMode::inactive ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); - if (all_validators_.size() > 0) { - td::actor::send_closure(info.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + if (info.actor.empty()) { + info.actor = FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, config_, keyring_, adnl_, rldp_, + rldp2_, overlays_, validator_manager_, client_, actor_id(this), active); + if (!all_validators_.empty()) { + td::actor::send_closure(info.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } + } else if (info.active != active) { + td::actor::send_closure(info.actor, &FullNodeShard::set_active, active); } + info.active = active; + info.delete_at = active ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); } void FullNodeImpl::sync_completed() { @@ -506,15 +482,11 @@ void FullNodeImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vect } td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { - ShardIdFull shard0 = shard; while (true) { auto it = shards_.find(shard); - if (it != shards_.end() && it->second.exists) { + if (it != shards_.end()) { if (it->second.actor.empty()) { - add_shard_actor(shard, FullNodeShardMode::inactive); - } - if (it->second.mode == FullNodeShardMode::inactive) { - it->second.delete_at = td::Timestamp::in(INACTIVE_SHARD_TTL); + update_shard_actor(shard, false); } return it->second.actor.get(); } @@ -523,19 +495,8 @@ td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { } shard = shard_parent(shard); } - shard = shard0; - auto it = shards_.find(shard); - if (it == shards_.end()) { - it = shards_.emplace(shard = ShardIdFull(shard.workchain), ShardInfo{}).first; - } - auto &info = it->second; - if (info.actor.empty()) { - add_shard_actor(shard, FullNodeShardMode::inactive); - } - if (info.mode == FullNodeShardMode::inactive) { - info.delete_at = td::Timestamp::in(INACTIVE_SHARD_TTL); - } - return info.actor.get(); + update_shard_actor(shard, false); + return shards_[shard].actor.get(); } td::actor::ActorId FullNodeImpl::get_shard(AccountIdPrefixFull dst) { @@ -635,7 +596,7 @@ void FullNodeImpl::process_block_candidate_broadcast(BlockIdExt block_id, Catcha } void FullNodeImpl::start_up() { - add_shard_actor(ShardIdFull{masterchainId}, FullNodeShardMode::active); + update_shard_actor(ShardIdFull{masterchainId}, true); if (local_id_.is_zero()) { if (adnl_id_.is_zero()) { auto pk = ton::PrivateKey{ton::privkeys::Ed25519::random()}; @@ -651,10 +612,9 @@ void FullNodeImpl::start_up() { void initial_read_complete(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::initial_read_complete, handle); } - void on_new_masterchain_block(td::Ref state, std::set shards_to_monitor, - std::set temporary_shards) override { + void on_new_masterchain_block(td::Ref state, std::set shards_to_monitor) override { td::actor::send_closure(id_, &FullNodeImpl::on_new_masterchain_block, std::move(state), - std::move(shards_to_monitor), std::move(temporary_shards)); + std::move(shards_to_monitor)); } void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) override { td::actor::send_closure(id_, &FullNodeImpl::send_ihr_message, dst, std::move(data)); @@ -716,7 +676,7 @@ void FullNodeImpl::start_up() { td::actor::send_closure(id_, &FullNodeImpl::new_key_block, std::move(handle)); } - Callback(td::actor::ActorId id) : id_(id) { + explicit Callback(td::actor::ActorId id) : id_(id) { } private: diff --git a/validator/full-node.hpp b/validator/full-node.hpp index a1f20b9d2f..442a4db2db 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -59,8 +59,7 @@ class FullNodeImpl : public FullNode { void add_custom_overlay(CustomOverlayParams params, td::Promise promise) override; void del_custom_overlay(std::string name, td::Promise promise) override; - void on_new_masterchain_block(td::Ref state, std::set shards_to_monitor, - std::set temporary_shards); + void on_new_masterchain_block(td::Ref state, std::set shards_to_monitor); void sync_completed(); @@ -111,13 +110,12 @@ class FullNodeImpl : public FullNode { private: struct ShardInfo { - bool exists = false; td::actor::ActorOwn actor; - FullNodeShardMode mode = FullNodeShardMode::inactive; + bool active = false; td::Timestamp delete_at = td::Timestamp::never(); }; - void add_shard_actor(ShardIdFull shard, FullNodeShardMode mode); + void update_shard_actor(ShardIdFull shard, bool active); PublicKeyHash local_id_; adnl::AdnlNodeIdShort adnl_id_; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index abe6d5202c..1ee02c482d 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -195,8 +195,6 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void add_lite_query_stats(int lite_query_id) { } - virtual void validated_new_block(BlockIdExt block_id) = 0; - virtual void add_persistent_state_description(td::Ref desc) = 0; static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 69a0a36875..0dcf438793 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -437,8 +437,6 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { promise.set_result(td::Status::Error("not implemented")); } - void validated_new_block(BlockIdExt block_id) override { - } void add_persistent_state_description(td::Ref desc) override { } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index edc675c454..48904bdb75 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -503,8 +503,6 @@ class ValidatorManagerImpl : public ValidatorManager { void update_options(td::Ref opts) override { opts_ = std::move(opts); } - void validated_new_block(BlockIdExt block_id) override { - } void add_persistent_state_description(td::Ref desc) override { } void get_validator_sessions_info( diff --git a/validator/manager.cpp b/validator/manager.cpp index 3aa222c430..abcc9e91fe 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -492,7 +492,7 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - if (!is_validator() && !is_shard_collator(block_id.shard_full())) { + if (!is_validator() && !is_shard_collator(block_id.shard_full()) && !cached_block_candidates_.count(block_id)) { return; } if (!last_masterchain_block_handle_) { @@ -2120,7 +2120,7 @@ void ValidatorManagerImpl::update_shard_overlays() { shards_to_monitor.insert(ShardIdFull{wc, shardIdAll}); } } - callback_->on_new_masterchain_block(last_masterchain_state_, std::move(shards_to_monitor), extra_active_shards_); + callback_->on_new_masterchain_block(last_masterchain_state_, std::move(shards_to_monitor)); } void ValidatorManagerImpl::update_shards() { @@ -2138,7 +2138,6 @@ void ValidatorManagerImpl::update_shards() { opts.proto_version = std::max(opts.proto_version, 1); } auto opts_hash = opts.get_hash(); - extra_active_shards_.clear(); std::map> new_shards; std::set future_shards; @@ -2193,11 +2192,6 @@ void ValidatorManagerImpl::update_shards() { default: LOG(FATAL) << "state=" << static_cast(v->fsm_state()); } - cleanup_last_validated_blocks(v->top_block_id().id); - } - - for (const auto& s : last_validated_blocks_) { - extra_active_shards_.insert(s.first); } new_shards.emplace(ShardIdFull{masterchainId, shardIdAll}, std::vector{last_masterchain_block_id_}); @@ -2270,7 +2264,6 @@ void ValidatorManagerImpl::update_shards() { auto validator_id = get_validator(shard, val_set); if (!validator_id.is_zero()) { - extra_active_shards_.insert(shard); auto val_group_id = get_validator_set_id(shard, val_set, opts_hash, key_seqno, opts); if (force_recover) { @@ -2366,24 +2359,6 @@ void ValidatorManagerImpl::update_shards() { } } -void ValidatorManagerImpl::cleanup_last_validated_blocks(BlockId new_block) { - auto process_shard = [&, this](ShardIdFull shard) { - auto it = last_validated_blocks_.find(shard); - if (it != last_validated_blocks_.end() && it->second < new_block.seqno) { - last_validated_blocks_.erase(it); - } - }; - ShardIdFull shard = new_block.shard_full(); - process_shard(shard); - if (shard.pfx_len() > 0) { - process_shard(shard_parent(shard)); - } - if (shard.pfx_len() < max_shard_pfx_len) { - process_shard(shard_child(shard, true)); - process_shard(shard_child(shard, false)); - } -} - void ValidatorManagerImpl::written_destroyed_validator_sessions(std::vector> list) { for (auto &v : list) { td::actor::send_closure(v, &ValidatorGroup::destroy); @@ -2463,7 +2438,7 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto validator_id = get_validator(shard, validator_set); CHECK(!validator_id.is_zero()); auto G = td::actor::create_actor( - "validatorgroup", shard, validator_id, session_id, validator_set, + PSTRING() << "valgroup" << shard.to_str(), shard, validator_id, session_id, validator_set, last_masterchain_state_->get_collator_config(true), opts, keyring_, adnl_, rldp_, overlays_, db_root_, actor_id(this), init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_, opts_->need_monitor(shard, last_masterchain_state_)); diff --git a/validator/manager.hpp b/validator/manager.hpp index 2a1638bd22..2654b2143a 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -624,11 +624,6 @@ class ValidatorManagerImpl : public ValidatorManager { void get_validator_sessions_info( td::Promise> promise) override; - void validated_new_block(BlockIdExt block_id) override { - BlockSeqno &last = last_validated_blocks_[block_id.shard_full()]; - last = std::max(last, block_id.seqno()); - } - void add_persistent_state_description(td::Ref desc) override; void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; @@ -745,8 +740,6 @@ class ValidatorManagerImpl : public ValidatorManager { return 3 * 10; } - void cleanup_last_validated_blocks(BlockId new_block); - void got_persistent_state_descriptions(std::vector> descs); void add_persistent_state_description_impl(td::Ref desc); td::Ref get_block_persistent_state(BlockIdExt block_id); @@ -773,9 +766,6 @@ class ValidatorManagerImpl : public ValidatorManager { }; std::map collator_nodes_; - std::set extra_active_shards_; - std::map last_validated_blocks_; - std::map> persistent_state_descriptions_; std::map> persistent_state_blocks_; }; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 4e23b6130a..b708e6ace0 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -177,9 +177,6 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, td::Ref sig_set, td::Ref approve_sig_set, bool send_broadcast, td::Promise promise, bool is_retry) { - if (!is_retry) { - td::actor::send_closure(manager_, &ValidatorManager::validated_new_block, block_id); - } auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -197,7 +194,7 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref }); run_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, std::move(sig_set), - std::move(approve_sig_set), send_broadcast, apply_blocks_, manager_, std::move(P)); + std::move(approve_sig_set), send_broadcast, monitoring_shard_, manager_, std::move(P)); } void ValidatorGroup::skip_round(td::uint32 round_id) { diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 6275734270..debb7d14aa 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -68,7 +68,7 @@ class ValidatorGroup : public td::actor::Actor { void update_options(td::Ref opts, bool apply_blocks) { opts_ = std::move(opts); - apply_blocks_ = apply_blocks; + monitoring_shard_ = apply_blocks; } ValidatorGroup(ShardIdFull shard, PublicKeyHash local_id, ValidatorSessionId session_id, @@ -77,7 +77,7 @@ class ValidatorGroup : public td::actor::Actor { td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, td::actor::ActorId validator_manager, bool create_session, - bool allow_unsafe_self_blocks_resync, td::Ref opts, bool apply_blocks) + bool allow_unsafe_self_blocks_resync, td::Ref opts, bool monitoring_shard) : shard_(shard) , local_id_(std::move(local_id)) , session_id_(session_id) @@ -93,7 +93,7 @@ class ValidatorGroup : public td::actor::Actor { , init_(create_session) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) , opts_(std::move(opts)) - , apply_blocks_(apply_blocks) { + , monitoring_shard_(monitoring_shard) { } private: @@ -141,7 +141,7 @@ class ValidatorGroup : public td::actor::Actor { bool allow_unsafe_self_blocks_resync_; td::Ref opts_; td::uint32 last_known_round_id_ = 0; - bool apply_blocks_ = true; + bool monitoring_shard_ = true; struct CachedCollatedBlock { td::optional result; @@ -151,7 +151,7 @@ class ValidatorGroup : public td::actor::Actor { void generated_block_candidate(std::shared_ptr cache, td::Result R); - typedef std::tuple CacheKey; + using CacheKey = std::tuple; std::map approved_candidates_cache_; void update_approve_cache(CacheKey key, UnixTime value); diff --git a/validator/validator.h b/validator/validator.h index 9428b844d8..a5e34bfeaf 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -170,8 +170,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void initial_read_complete(BlockHandle top_masterchain_blocks) = 0; virtual void on_new_masterchain_block(td::Ref state, - std::set shards_to_monitor, - std::set temporary_shards) = 0; + std::set shards_to_monitor) = 0; virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; From 70322f58c196e326bd41f6aabe68df74c7fbabb2 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 1 Aug 2024 13:57:32 +0300 Subject: [PATCH 107/524] Use candidates from DB in wait block data --- tl/generate/scheme/ton_api.tl | 1 + tl/generate/scheme/ton_api.tlo | Bin 101528 -> 101624 bytes validator/collator-node.cpp | 2 +- validator/db/fileref.cpp | 25 +++++++++++++++ validator/db/fileref.hpp | 37 ++++++++++++++++++++-- validator/db/rootdb.cpp | 37 ++++++++++++++-------- validator/db/rootdb.hpp | 1 + validator/downloaders/wait-block-data.cpp | 19 ++++++++--- validator/downloaders/wait-block-data.hpp | 12 ++++--- validator/interfaces/db.h | 1 + validator/manager-disk.cpp | 5 +++ validator/manager-disk.hpp | 1 + validator/manager-hardfork.cpp | 5 +++ validator/manager-hardfork.hpp | 1 + validator/manager.cpp | 21 +++++++++--- validator/manager.hpp | 1 + validator/validator.h | 1 + 17 files changed, 140 insertions(+), 30 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index df9a54fd0d..f7d0ad249f 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -524,6 +524,7 @@ db.filedb.key.proof block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.proofLink block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.signatures block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.candidate id:db.candidate.id = db.filedb.Key; +db.filedb.key.candidateRef id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.blockInfo block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.value key:db.filedb.Key prev:int256 next:int256 file_hash:int256 = db.filedb.Value; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 532a1d3a334f6cc686ec78a758c2a51a4b536608..ad6f2ce30e94ea1ba5020ba0a93c1e27918fb3cd 100644 GIT binary patch delta 276 zcmbO+lkLY$whawjqBFH#DD$Qy>7`}nqyj1L)Jg^hu8EG~o7Zq@7%*<$eAVi|u9 z&0|773_wui%%;O(+jbBnHu*%U1`C*Rpxgw)&?(bF6PZ#5l{!9poYqxGKjLv(L%*nbhRM3UQ90)HDXLvkZ!$rtE=fhC5WD R(gy5^=7h@a36+c%!T>-cXH5VA delta 237 zcmew{lWoRKwhawjqUsq-kMgD@>7`}nqyj1L)Jg^h&WVoVo7Zq@7%*|u9 z&0|773_wui%%;O(+jbBnHu*%U1`C*Rpxgw)&?(bF6PZ#5l{! } } - for (auto it = future_validator_groups_.begin(); it != future_validator_groups_.end(); ++it) { + for (auto it = future_validator_groups_.begin(); it != future_validator_groups_.end();) { if (get_future_validator_group(it->first.first, it->first.second).is_ok()) { ++it; } else { diff --git a/validator/db/fileref.cpp b/validator/db/fileref.cpp index 2e2a3b6b64..378e382c9e 100644 --- a/validator/db/fileref.cpp +++ b/validator/db/fileref.cpp @@ -213,6 +213,28 @@ std::string CandidateShort::filename_short() const { return PSTRING() << "candidate_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); } +CandidateRefShort CandidateRef::shortref() const { + return CandidateRefShort{block_id.id, hash()}; +} + +std::string CandidateRef::filename() const { + return PSTRING() << "candidateref_" << block_id.to_str(); +} + +std::string CandidateRef::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(block_id.id.shard)); + return PSTRING() << "candidateref_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" + << hash().to_hex(); +} + +std::string CandidateRefShort::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(block_id.shard)); + return PSTRING() << "candidateref_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" + << hash().to_hex(); +} + BlockInfoShort BlockInfo::shortref() const { return BlockInfoShort{block_id.id, hash()}; } @@ -259,6 +281,9 @@ FileReference::FileReference(tl_object_ptr key) { ref_ = fileref::Candidate{PublicKey{key.id_->source_}, create_block_id(key.id_->id_), key.id_->collated_data_file_hash_}; }, + [&](const ton_api::db_filedb_key_candidateRef& key) { + ref_ = fileref::CandidateRef{create_block_id(key.id_)}; + }, [&](const ton_api::db_filedb_key_blockInfo& key) { ref_ = fileref::BlockInfo{create_block_id(key.block_id_)}; })); diff --git a/validator/db/fileref.hpp b/validator/db/fileref.hpp index 14424a6531..6710cc0638 100644 --- a/validator/db/fileref.hpp +++ b/validator/db/fileref.hpp @@ -278,6 +278,38 @@ class Candidate { FileHash collated_data_file_hash; }; +class CandidateRefShort { + public: + FileHash hash() const { + return hashv; + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + std::string filename_short() const; + + BlockId block_id; + FileHash hashv; +}; + +class CandidateRef { + public: + tl_object_ptr tl() const { + return create_tl_object(create_tl_block_id(block_id)); + } + FileHash hash() const { + return create_hash_tl_object(create_tl_block_id(block_id)); + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + CandidateRefShort shortref() const; + std::string filename() const; + std::string filename_short() const; + + BlockIdExt block_id; +}; + class BlockInfoShort { public: FileHash hash() const { @@ -316,7 +348,7 @@ class FileReferenceShort { private: td::Variant + fileref::CandidateShort, fileref::CandidateRefShort, fileref::BlockInfoShort> ref_; public: @@ -340,7 +372,8 @@ class FileReferenceShort { class FileReference { private: td::Variant + fileref::Proof, fileref::ProofLink, fileref::Signatures, fileref::Candidate, fileref::CandidateRef, + fileref::BlockInfo> ref_; public: diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 10545e46f1..ccd5cb290c 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -177,21 +177,21 @@ void RootDb::get_block_proof_link(ConstBlockHandle handle, td::Promise promise) { + auto source = PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}; auto obj = create_serialize_tl_object( - PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}.tl(), create_tl_block_id(candidate.id), - std::move(candidate.data), std::move(candidate.collated_data)); - - auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - promise.set_value(td::Unit()); - } - }); + source.tl(), create_tl_block_id(candidate.id), std::move(candidate.data), std::move(candidate.collated_data)); + auto P = td::PromiseCreator::lambda( + [archive_db = archive_db_.get(), promise = std::move(promise), block_id = candidate.id, source, + collated_file_hash = candidate.collated_file_hash](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, _, std::move(R)); + td::actor::send_closure(archive_db, &ArchiveManager::add_temp_file_short, fileref::CandidateRef{block_id}, + create_serialize_tl_object( + source.tl(), create_tl_block_id(block_id), collated_file_hash), + std::move(promise)); + }); td::actor::send_closure(archive_db_, &ArchiveManager::add_temp_file_short, - fileref::Candidate{PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}, candidate.id, - candidate.collated_file_hash}, - std::move(obj), std::move(P)); + fileref::Candidate{source, candidate.id, candidate.collated_file_hash}, std::move(obj), + std::move(P)); } void RootDb::get_block_candidate(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, @@ -215,6 +215,17 @@ void RootDb::get_block_candidate(PublicKey source, BlockIdExt id, FileHash colla fileref::Candidate{source, id, collated_data_file_hash}, std::move(P)); } +void RootDb::get_block_candidate_by_block_id(BlockIdExt id, td::Promise promise) { + td::actor::send_closure( + archive_db_, &ArchiveManager::get_temp_file_short, fileref::CandidateRef{id}, + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, data, std::move(R)); + TRY_RESULT_PROMISE(promise, f, fetch_tl_object(data, true)); + td::actor::send_closure(SelfId, &RootDb::get_block_candidate, PublicKey{f->source_}, create_block_id(f->id_), + f->collated_data_file_hash_, std::move(promise)); + }); +} + void RootDb::store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) { if (handle->moved_to_archive()) { diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 74c46e43cf..52f6098e4d 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -58,6 +58,7 @@ class RootDb : public Db { void store_block_candidate(BlockCandidate candidate, td::Promise promise) override; void get_block_candidate(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) override; + void get_block_candidate_by_block_id(BlockIdExt id, td::Promise promise) override; void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; diff --git a/validator/downloaders/wait-block-data.cpp b/validator/downloaders/wait-block-data.cpp index 220a8a2cf6..53a3d351bc 100644 --- a/validator/downloaders/wait-block-data.cpp +++ b/validator/downloaders/wait-block-data.cpp @@ -106,13 +106,24 @@ void WaitBlockData::start() { }); td::actor::send_closure(manager_, &ValidatorManager::try_get_static_file, handle_->id().file_hash, std::move(P)); + } else if (try_get_candidate_) { + try_get_candidate_ = false; + td::actor::send_closure( + manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, handle_->id(), + [SelfId = actor_id(this), id = handle_->id()](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &WaitBlockData::start); + } else { + td::actor::send_closure(SelfId, &WaitBlockData::loaded_data, ReceivedBlock{id, R.move_as_ok()}); + } + }); } else { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &WaitBlockData::failed_to_get_block_data_from_net, R.move_as_error_prefix("net error: ")); } else { - td::actor::send_closure(SelfId, &WaitBlockData::got_data_from_net, R.move_as_ok()); + td::actor::send_closure(SelfId, &WaitBlockData::loaded_data, R.move_as_ok()); } }); @@ -137,16 +148,16 @@ void WaitBlockData::failed_to_get_block_data_from_net(td::Status reason) { td::Timestamp::in(0.1)); } -void WaitBlockData::got_data_from_net(ReceivedBlock block) { +void WaitBlockData::loaded_data(ReceivedBlock block) { auto X = create_block(std::move(block)); if (X.is_error()) { failed_to_get_block_data_from_net(X.move_as_error_prefix("bad block from net: ")); return; } - got_block_data_from_net(X.move_as_ok()); + loaded_block_data(X.move_as_ok()); } -void WaitBlockData::got_block_data_from_net(td::Ref block) { +void WaitBlockData::loaded_block_data(td::Ref block) { if (data_.not_null()) { return; } diff --git a/validator/downloaders/wait-block-data.hpp b/validator/downloaders/wait-block-data.hpp index 229b4bfc85..f3b367d510 100644 --- a/validator/downloaders/wait-block-data.hpp +++ b/validator/downloaders/wait-block-data.hpp @@ -30,15 +30,16 @@ class ValidatorManager; class WaitBlockData : public td::actor::Actor { public: WaitBlockData(BlockHandle handle, td::uint32 priority, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise> promise) + td::Timestamp timeout, bool try_get_candidate, td::Promise> promise) : handle_(std::move(handle)) , priority_(priority) , manager_(manager) , timeout_(timeout) + , try_get_candidate_(try_get_candidate) , promise_(std::move(promise)) , perf_timer_("waitdata", 1.0, [manager](double duration) { - send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitdata", duration); - }) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitdata", duration); + }) { } void update_timeout(td::Timestamp timeout, td::uint32 priority) { @@ -57,8 +58,8 @@ class WaitBlockData : public td::actor::Actor { void set_is_hardfork(bool value); void start(); void got_block_data_from_db(td::Ref data); - void got_data_from_net(ReceivedBlock data); - void got_block_data_from_net(td::Ref block); + void loaded_data(ReceivedBlock data); + void loaded_block_data(td::Ref block); void checked_proof_link(); void failed_to_get_block_data_from_net(td::Status reason); @@ -73,6 +74,7 @@ class WaitBlockData : public td::actor::Actor { td::actor::ActorId manager_; td::Timestamp timeout_; + bool try_get_candidate_; td::Promise> promise_; td::Ref data_; diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index f915b25c78..29ef715b36 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -46,6 +46,7 @@ class Db : public td::actor::Actor { virtual void store_block_candidate(BlockCandidate candidate, td::Promise promise) = 0; virtual void get_block_candidate(ton::PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) = 0; + virtual void get_block_candidate_by_block_id(BlockIdExt id, td::Promise promise) = 0; virtual void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 1795ee34fa..3634461076 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -589,6 +589,11 @@ void ValidatorManagerImpl::get_block_candidate_from_db(PublicKey source, BlockId td::actor::send_closure(db_, &Db::get_block_candidate, source, id, collated_data_file_hash, std::move(promise)); } +void ValidatorManagerImpl::get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_block_candidate_by_block_id, id, + promise.wrap([](BlockCandidate &&b) { return std::move(b.data); })); +} + void ValidatorManagerImpl::get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) { td::actor::send_closure(db_, &Db::get_block_proof, std::move(handle), std::move(promise)); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 0dcf438793..01a0c60eba 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -217,6 +217,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) override; void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) override; + void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) override; void get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) override; void get_block_proof_from_db_short(BlockIdExt id, td::Promise> promise) override; void get_block_proof_link_from_db(ConstBlockHandle handle, td::Promise> promise) override; diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index 1ba8c68d1b..e091983db3 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -415,6 +415,11 @@ void ValidatorManagerImpl::get_block_candidate_from_db(PublicKey source, BlockId td::actor::send_closure(db_, &Db::get_block_candidate, source, id, collated_data_file_hash, std::move(promise)); } +void ValidatorManagerImpl::get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_block_candidate_by_block_id, id, + promise.wrap([](BlockCandidate &&b) { return std::move(b.data); })); +} + void ValidatorManagerImpl::get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) { td::actor::send_closure(db_, &Db::get_block_proof, std::move(handle), std::move(promise)); } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 48904bdb75..28c521f430 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -263,6 +263,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) override; void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) override; + void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) override; void get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) override; void get_block_proof_from_db_short(BlockIdExt id, td::Promise> promise) override; void get_block_proof_link_from_db(ConstBlockHandle handle, td::Promise> promise) override; diff --git a/validator/manager.cpp b/validator/manager.cpp index abcc9e91fe..0c4a1a1bc0 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -622,7 +622,7 @@ void ValidatorManagerImpl::add_cached_block_candidate(ReceivedBlock block) { if (it != wait_block_data_.end()) { auto r_block = create_block(cached_block_candidates_[id].clone()); if (r_block.is_ok()) { - td::actor::send_closure(it->second.actor_, &WaitBlockData::got_block_data_from_net, r_block.move_as_ok()); + td::actor::send_closure(it->second.actor_, &WaitBlockData::loaded_block_data, r_block.move_as_ok()); } } } @@ -852,7 +852,8 @@ void ValidatorManagerImpl::wait_block_data(BlockHandle handle, td::uint32 priori td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_data, handle, std::move(R)); }); auto id = td::actor::create_actor("waitdata", handle, priority, actor_id(this), - td::Timestamp::at(timeout.at() + 10.0), std::move(P)) + td::Timestamp::at(timeout.at() + 10.0), + is_shard_collator(handle->id().shard_full()), std::move(P)) .release(); wait_block_data_[handle->id()].actor_ = id; it = wait_block_data_.find(handle->id()); @@ -1177,6 +1178,16 @@ void ValidatorManagerImpl::get_block_candidate_from_db(PublicKey source, BlockId td::actor::send_closure(db_, &Db::get_block_candidate, source, id, collated_data_file_hash, std::move(promise)); } +void ValidatorManagerImpl::get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) { + auto it = cached_block_candidates_.find(id); + if (it != cached_block_candidates_.end()) { + promise.set_result(it->second.data.clone()); + return; + } + td::actor::send_closure(db_, &Db::get_block_candidate_by_block_id, id, + promise.wrap([](BlockCandidate &&b) { return std::move(b.data); })); +} + void ValidatorManagerImpl::get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) { td::actor::send_closure(db_, &Db::get_block_proof, std::move(handle), std::move(promise)); } @@ -1289,9 +1300,9 @@ void ValidatorManagerImpl::finished_wait_data(BlockHandle handle, td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_data, handle, std::move(R)); }); - auto id = - td::actor::create_actor("waitdata", handle, X.second, actor_id(this), X.first, std::move(P)) - .release(); + auto id = td::actor::create_actor("waitdata", handle, X.second, actor_id(this), X.first, + is_shard_collator(handle->id().shard_full()), std::move(P)) + .release(); it->second.actor_ = id; return; } diff --git a/validator/manager.hpp b/validator/manager.hpp index 2654b2143a..007bce35f9 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -479,6 +479,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) override; void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) override; + void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) override; void get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) override; void get_block_proof_from_db_short(BlockIdExt id, td::Promise> promise) override; void get_block_proof_link_from_db(ConstBlockHandle handle, td::Promise> promise) override; diff --git a/validator/validator.h b/validator/validator.h index a5e34bfeaf..8bda3e871d 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -262,6 +262,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_data_from_db_short(BlockIdExt block_id, td::Promise> promise) = 0; virtual void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) = 0; + virtual void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) = 0; virtual void get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise) = 0; virtual void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) = 0; virtual void get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) = 0; From a14490637a861fced73dfd05201ea0b2eab71e59 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 1 Aug 2024 18:39:22 +0300 Subject: [PATCH 108/524] More overlay stats --- overlay/overlay.cpp | 4 +++ tl/generate/scheme/ton_api.tl | 3 +- tl/generate/scheme/ton_api.tlo | Bin 101624 -> 101736 bytes .../validator-engine-console-query.cpp | 26 ++++++++++++++---- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index d05f5ae287..e93d698a6b 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -726,6 +726,10 @@ void OverlayImpl::get_stats(td::Promisebdcst_errors_ = peer.broadcast_errors; node_obj->fec_bdcst_errors_ = peer.fec_broadcast_errors; + node_obj->is_neighbour_ = peer.is_neighbour(); + node_obj->is_alive_ = peer.is_alive(); + node_obj->node_flags_ = peer.get_node()->flags(); + res->nodes_.push_back(std::move(node_obj)); }); diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index f7d0ad249f..70cb790b71 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -693,7 +693,8 @@ engine.validator.proposalVote perm_key:int256 to_send:bytes = engine.validator.P engine.validator.dhtServerStatus id:int256 status:int = engine.validator.DhtServerStatus; engine.validator.dhtServersStatus servers:(vector engine.validator.dhtServerStatus) = engine.validator.DhtServersStatus; -engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string bdcst_errors:int fec_bdcst_errors:int last_in_query:int last_out_query:int t_out_bytes:int t_in_bytes:int t_out_pckts:int t_in_pckts:int = engine.validator.OverlayStatsNode; +engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string is_neighbour:Bool is_alive:Bool node_flags:int + bdcst_errors:int fec_bdcst_errors:int last_in_query:int last_out_query:int t_out_bytes:int t_in_bytes:int t_out_pckts:int t_in_pckts:int = engine.validator.OverlayStatsNode; engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) extra:string = engine.validator.OverlayStats; engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index ad6f2ce30e94ea1ba5020ba0a93c1e27918fb3cd..eae2a5590c1411adc052dbea3dbd6b5832abc004 100644 GIT binary patch delta 138 zcmew{lkLSUwhbXttOAVIHHwpErDQjkNp(%*1aTP|fMD|4?dFpmHd;UxC~lDD_sdU7 zWnciQxOq&-hj+5#GWE$9HY)OD7RTqMW~OH(<(C#sR$M5}!2uLW%*ia9tgujqhbs@L eEj}$LF};{!a>FvI&7U@!F|vS6*{rkq%uN6XZZ9eT delta 91 zcmaDci|xluwhbXttTz>kekx9um6F|DCe<~K6U1d?0D{S~JIp6LY_xzXP~0HP@0XvF k%D@0par2mv57%VHW$KeJY*gHwxXGN61th$A&gL^W0cPbTsQ>@~ diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index f88d2c601a..6266ac2ac9 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -910,8 +910,22 @@ td::Status GetOverlaysStatsJsonQuery::receive(td::BufferSlice data) { } else { tail = true; } - - sb << " {\n \"adnl_id\": \"" << n->adnl_id_ << "\",\n \"ip_addr\": \"" << n->ip_addr_ << "\",\n \"broadcast_errors\": " << n->bdcst_errors_ << ",\n \"fec_broadcast_errors\": " << n->fec_bdcst_errors_ << ",\n \"last_in_query_unix\": " << n->last_in_query_ << ",\n \"last_in_query_human\": \"" << time_to_human(n->last_in_query_) << "\",\n" << " \"last_out_query_unix\": " << n->last_out_query_ << ",\n \"last_out_query_human\": \"" << time_to_human(n->last_out_query_) << "\",\n" << "\n \"throughput\": { \"out_bytes_sec\": " << n->t_out_bytes_ << ", \"out_pckts_sec\": " << n->t_out_pckts_ << ", \"in_bytes_sec\": " << n->t_in_bytes_ << ", \"in_pckts_sec\": " << n->t_in_pckts_ << " }\n }"; + + sb << " {\n"; + sb << R"( "adnl_id": ")" << n->adnl_id_ << "\",\n"; + sb << R"( "ip_addr": ")" << n->ip_addr_ << "\",\n"; + sb << R"( "is_neighbour": )" << (n->is_neighbour_ ? "true" : "false") << ",\n"; + sb << R"( "is_alive": )" << (n->is_alive_ ? "true" : "false") << ",\n"; + sb << R"( "node_flags": )" << n->node_flags_ << ",\n"; + sb << R"( "broadcast_errors": )" << n->bdcst_errors_ << ",\n"; + sb << R"( "fec_broadcast_errors": )" << n->fec_bdcst_errors_ << ",\n"; + sb << R"( "last_in_query_unix": )" << n->last_in_query_ << ",\n"; + sb << R"( "last_in_query_human": ")" << time_to_human(n->last_in_query_) << "\",\n"; + sb << R"( "last_out_query_unix": )" << n->last_out_query_ << ",\n"; + sb << R"( "last_out_query_human": ")" << time_to_human(n->last_out_query_) << "\",\n\n"; + sb << R"( "throughput": { "out_bytes_sec": )" << n->t_out_bytes_ << R"(, "out_pckts_sec": )" << n->t_out_pckts_ + << R"(, "in_bytes_sec": )" << n->t_in_bytes_ << R"(, "in_pckts_sec": )" << n->t_in_pckts_ << " }\n"; + sb << " }"; overlay_t_out_bytes += n->t_out_bytes_; overlay_t_out_pckts += n->t_out_pckts_; @@ -920,8 +934,10 @@ td::Status GetOverlaysStatsJsonQuery::receive(td::BufferSlice data) { overlay_t_in_pckts += n->t_in_pckts_; } sb << " ],\n"; - - sb << " \"total_throughput\": { \"out_bytes_sec\": " << overlay_t_out_bytes << ", \"out_pckts_sec\": " << overlay_t_out_pckts << ", \"in_bytes_sec\": " << overlay_t_in_bytes << ", \"in_pckts_sec\": " << overlay_t_in_pckts << " },\n"; + + sb << " \"total_throughput\": { \"out_bytes_sec\": " << overlay_t_out_bytes + << ", \"out_pckts_sec\": " << overlay_t_out_pckts << ", \"in_bytes_sec\": " << overlay_t_in_bytes + << ", \"in_pckts_sec\": " << overlay_t_in_pckts << " },\n"; sb << " \"stats\": {\n"; From 15099f26645d382dba493eb74e645411075c5a09 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 6 Aug 2024 22:11:11 +0300 Subject: [PATCH 109/524] Fix setting bad_peers_ in overlay --- overlay/overlay-peers.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index a217686132..f1621eb444 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -255,12 +255,10 @@ void OverlayImpl::on_ping_result(adnl::AdnlNodeIdShort peer, bool success) { } if (OverlayPeer *p = peer_list_.peers_.get(peer)) { p->on_ping_result(success); - if (!p->is_permanent_member()) { - if (p->is_alive()) { - peer_list_.bad_peers_.erase(peer); - } else { - peer_list_.bad_peers_.insert(peer); - } + if (p->is_alive()) { + peer_list_.bad_peers_.erase(peer); + } else { + peer_list_.bad_peers_.insert(peer); } } } From dcb409462a0f601ac6485428c2056596b5b982cd Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 9 Aug 2024 09:20:42 +0300 Subject: [PATCH 110/524] Cleanup mempool on collators --- validator/manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index 0c4a1a1bc0..11454c8e9c 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2841,7 +2841,7 @@ void ValidatorManagerImpl::alarm() { } alarm_timestamp().relax(log_ls_stats_at_); if (cleanup_mempool_at_.is_in_past()) { - if (is_validator()) { + if (is_validator() || !collator_nodes_.empty()) { get_external_messages(ShardIdFull{masterchainId, shardIdAll}, [](td::Result, int>>>) {}); get_external_messages(ShardIdFull{basechainId, shardIdAll}, From 7ecc31bc969b792cb4abc22bfe52eb1db7511111 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 9 Aug 2024 15:49:47 +0300 Subject: [PATCH 111/524] Use block data from block candidate in AcceptBlockQuery --- validator/impl/accept-block.cpp | 29 +++++++++++++++++++++++++++++ validator/impl/accept-block.hpp | 2 ++ validator/impl/fabric.cpp | 6 +++--- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index befc17bbf2..bada8cca5f 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -416,8 +416,37 @@ void AcceptBlockQuery::got_block_handle(BlockHandle handle) { (is_masterchain() ? handle_->inited_proof() && handle_->is_applied() && handle_->inited_is_key_block() : handle_->inited_proof_link())) { finish_query(); + return; + } + if (data_.is_null()) { + td::actor::send_closure(manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, id_, [SelfId = actor_id(this)](td::Result R) { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &AcceptBlockQuery::got_block_candidate_data, R.move_as_ok()); + } else { + td::actor::send_closure(SelfId, &AcceptBlockQuery::got_block_handle_cont); + } + }); + } else { + got_block_handle_cont(); + } +} + +void AcceptBlockQuery::got_block_candidate_data(td::BufferSlice data) { + auto r_block = create_block(id_, std::move(data)); + if (r_block.is_error()) { + fatal_error("invalid block candidate data in db: " + r_block.error().to_string()); return; } + data_ = r_block.move_as_ok(); + VLOG(VALIDATOR_DEBUG) << "got block candidate data from db"; + if (data_.not_null() && !precheck_header()) { + fatal_error("invalid block header in AcceptBlock"); + return; + } + got_block_handle_cont(); +} + +void AcceptBlockQuery::got_block_handle_cont() { if (data_.not_null() && !handle_->received()) { td::actor::send_closure( manager_, &ValidatorManager::set_block_data, handle_, data_, [SelfId = actor_id(this)](td::Result R) { diff --git a/validator/impl/accept-block.hpp b/validator/impl/accept-block.hpp index dd33ea928f..4d547e39b9 100644 --- a/validator/impl/accept-block.hpp +++ b/validator/impl/accept-block.hpp @@ -71,6 +71,8 @@ class AcceptBlockQuery : public td::actor::Actor { void written_block_data(); void written_block_signatures(); void got_block_handle(BlockHandle handle); + void got_block_candidate_data(td::BufferSlice data); + void got_block_handle_cont(); void written_block_info(); void got_block_data(td::Ref data); void got_prev_state(td::Ref state); diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 1b08f76299..8d3fa429b7 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -133,9 +133,9 @@ void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector< td::Ref validator_set, td::Ref signatures, td::Ref approve_signatures, bool send_broadcast, bool apply, td::actor::ActorId manager, td::Promise promise) { - td::actor::create_actor("accept", id, std::move(data), prev, std::move(validator_set), - std::move(signatures), std::move(approve_signatures), send_broadcast, apply, - manager, std::move(promise)) + td::actor::create_actor( + PSTRING() << "accept" << id.id.to_str(), id, std::move(data), prev, std::move(validator_set), + std::move(signatures), std::move(approve_signatures), send_broadcast, apply, manager, std::move(promise)) .release(); } From e7da7a558058dcf191ce94be6da673bad18ba801 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 12 Aug 2024 10:07:15 +0300 Subject: [PATCH 112/524] OverlayImpl::get_neighbours bugfix --- overlay/overlay-peers.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index f1621eb444..d370024443 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -624,10 +624,8 @@ std::vector OverlayImpl::get_neighbours(td::uint32 max_si for (td::uint32 i = 0; i < max_size; i++) { td::uint32 t = td::Random::fast(0, static_cast(peer_list_.neighbours_.size()) - 1 - i); td::uint32 j; - for (j = 0; j < i; j++) { - if (ul[j] <= t) { - t++; - } + for (j = 0; j < i && ul[j] <= t; j++) { + t++; } ul.emplace(ul.begin() + j, t); vec.push_back(peer_list_.neighbours_[t]); From 2052b1574fe99503fae6f0564a5e86c49226f831 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 12 Aug 2024 13:56:29 +0300 Subject: [PATCH 113/524] --add-shard parameter --- validator-engine/validator-engine.cpp | 39 +++++++++++++++++++++++++-- validator-engine/validator-engine.hpp | 4 +++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 8191eede59..a6915f4098 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -68,6 +68,7 @@ #include #include #include +#include #include "git.h" #include "block-auto.h" #include "block-parse.h" @@ -592,8 +593,11 @@ td::Result Config::config_add_control_process(ton::PublicKeyHash key, td:: } td::Result Config::config_add_shard(ton::ShardIdFull shard) { - if (!shard.is_valid_ext() || shard.is_masterchain()) { - return td::Status::Error(PSTRING() << "invalid shard to collate " << shard.to_str()); + if (shard.is_masterchain()) { + return td::Status::Error("masterchain is monitored by default"); + } + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << shard.to_str()); } if (std::find(shards_to_monitor.begin(), shards_to_monitor.end(), shard) != shards_to_monitor.end()) { return false; @@ -1647,6 +1651,14 @@ void ValidatorEngine::load_empty_local_config(td::Promise promise) { } void ValidatorEngine::load_local_config(td::Promise promise) { + for (ton::ShardIdFull shard : add_shard_cmds_) { + auto R = config_.config_add_shard(shard); + if (R.is_error()) { + LOG(WARNING) << "Cannot add shard " << shard.to_str() << " : " << R.move_as_error(); + } else if (R.ok()) { + LOG(WARNING) << "Adding shard to monitor " << shard.to_str(); + } + } if (local_config_.size() == 0) { load_empty_local_config(std::move(promise)); return; @@ -1862,6 +1874,15 @@ void ValidatorEngine::load_config(td::Promise promise) { td::actor::send_closure(keyring_, &ton::keyring::Keyring::add_key_short, key.first, get_key_promise(ig)); } + for (ton::ShardIdFull shard : add_shard_cmds_) { + auto R = config_.config_add_shard(shard); + if (R.is_error()) { + LOG(WARNING) << "Cannot add shard " << shard.to_str() << " : " << R.move_as_error(); + } else if (R.ok()) { + LOG(WARNING) << "Adding shard to monitor " << shard.to_str(); + } + } + write_config(ig.get_promise()); } @@ -4629,6 +4650,20 @@ int main(int argc, char *argv[]) { p.add_option('M', "not-all-shards", "monitor only a necessary set of shards instead of all", [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_not_all_shards); }); }); + p.add_checked_option( + '\0', "add-shard", "add shard to monitor (same as addshard in validator console), format: 0:8000000000000000", + [&](td::Slice arg) -> td::Status { + std::string str = arg.str(); + int wc; + unsigned long long shard; + if (sscanf(str.c_str(), "%d:%016llx", &wc, &shard) != 2) { + return td::Status::Error(PSTRING() << "invalid shard " << str); + } + acts.push_back([=, &x]() { + td::actor::send_closure(x, &ValidatorEngine::add_shard_cmd, ton::ShardIdFull{wc, (ton::ShardId)shard}); + }); + return td::Status::OK(); + }); td::uint32 threads = 7; p.add_checked_option( 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 60986b1404..6a7680f0ec 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -239,6 +239,7 @@ class ValidatorEngine : public td::actor::Actor { std::string session_logs_file_; bool fast_state_serializer_enabled_ = false; bool not_all_shards_ = false; + std::vector add_shard_cmds_; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -324,6 +325,9 @@ class ValidatorEngine : public td::actor::Actor { void set_not_all_shards() { not_all_shards_ = true; } + void add_shard_cmd(ton::ShardIdFull shard) { + add_shard_cmds_.push_back(shard); + } void start_up() override; ValidatorEngine() { From 0ca022cc724093877d3c6a58b9575891297e117d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 19 Aug 2024 18:35:50 +0300 Subject: [PATCH 114/524] Improve validator session stats * Collator stats: block limits, number of processed external messages * Collator and validator work time * Last key block seqno * Approvers and signers --- tdutils/td/utils/Timer.cpp | 43 +++++++++++ tdutils/td/utils/Timer.h | 18 +++++ tl/generate/scheme/ton_api.tl | 17 +++-- tl/generate/scheme/ton_api.tlo | Bin 91004 -> 91968 bytes validator-session/validator-session-types.h | 3 + validator-session/validator-session.cpp | 6 ++ validator/impl/collator-impl.h | 5 ++ validator/impl/collator.cpp | 61 ++++++++++++++++ validator/impl/validate-query.cpp | 21 ++++++ validator/impl/validate-query.hpp | 4 ++ validator/interfaces/validator-manager.h | 16 +++++ validator/manager.cpp | 76 ++++++++++++++++---- validator/manager.hpp | 18 ++++- validator/validator-group.cpp | 1 + validator/validator-group.hpp | 11 +-- 15 files changed, 276 insertions(+), 24 deletions(-) diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index 1f72fba96c..24de099aa0 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -91,4 +91,47 @@ double PerfWarningTimer::elapsed() const { return Time::now() - start_at_; } +static double thread_cpu_clock() { +#if defined(CLOCK_THREAD_CPUTIME_ID) + timespec ts; + int result = clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts); + CHECK(result == 0); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1e-9; +#else + return 0.0; // TODO: MacOS and Windows support (currently cpu timer is used only in validators) +#endif +} + +ThreadCpuTimer::ThreadCpuTimer(bool is_paused) : is_paused_(is_paused) { + if (is_paused_) { + start_time_ = 0; + } else { + start_time_ = thread_cpu_clock(); + } +} + +void ThreadCpuTimer::pause() { + if (is_paused_) { + return; + } + elapsed_ += thread_cpu_clock() - start_time_; + is_paused_ = true; +} + +void ThreadCpuTimer::resume() { + if (!is_paused_) { + return; + } + start_time_ = thread_cpu_clock(); + is_paused_ = false; +} + +double ThreadCpuTimer::elapsed() const { + double res = elapsed_; + if (!is_paused_) { + res += thread_cpu_clock() - start_time_; + } + return res; +} + } // namespace td diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index 3e0cafbf59..a27cac8a7e 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -62,4 +62,22 @@ class PerfWarningTimer { std::function callback_; }; +class ThreadCpuTimer { + public: + ThreadCpuTimer() : ThreadCpuTimer(false) { + } + explicit ThreadCpuTimer(bool is_paused); + ThreadCpuTimer(const ThreadCpuTimer &other) = default; + ThreadCpuTimer &operator=(const ThreadCpuTimer &other) = default; + + double elapsed() const; + void pause(); + void resume(); + + private: + double elapsed_{0}; + double start_time_; + bool is_paused_{false}; +}; + } // namespace td diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index b33ca54253..6c5126a114 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -769,13 +769,19 @@ http.server.config dhs:(vector http.server.dnsEntry) local_hosts:(vector http.se ---types--- -validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int comment:string - block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double +validatorSession.collationStats bytes:int gas:int lt_delta:int cat_bytes:int cat_gas:int cat_lt_delta:int + limits_log:string ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validadorSession.CollationStats; + +validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int root_hash:int256 file_hash:int256 + comment:string block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double collation_time:double collated_at:double collation_cached:Bool + collation_work_time:double collation_cpu_work_time:double + collation_stats:validatorSession.collationStats validation_time:double validated_at:double validation_cached:Bool + validation_work_time:double validation_cpu_work_time:double gen_utime:double - approved_weight:long approved_33pct_at:double approved_66pct_at:double - signed_weight:long signed_33pct_at:double signed_66pct_at:double + approved_weight:long approved_33pct_at:double approved_66pct_at:double approvers:string + signed_weight:long signed_33pct_at:double signed_66pct_at:double signers:string serialize_time:double deserialize_time:double serialized_size:int = validatorSession.StatsProducer; validatorSession.statsRound timestamp:double producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound; @@ -786,7 +792,8 @@ validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:double self: first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats; validatorSession.newValidatorGroupStats.node id:int256 weight:long = validatorSession.newValidatorGroupStats.Node; -validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int timestamp:double +validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int + last_key_block_seqno:int timestamp:double self_idx:int nodes:(vector validatorSession.newValidatorGroupStats.node) = validatorSession.NewValidatorGroupStats; ---functions--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index da1aa331d2048803f582b99ede705270f28a22ea..509655394679a5ec39644c3c1973334603d33cf3 100644 GIT binary patch delta 610 zcmex!jP<}dR^CUm^{p77fMX-?d3}|4min3UWr;bNDTyihMZu}X#hLkedd~SdIf*4e zR&YsTN%6!D5}WVne_&);WW;%J@&&^cES-~cgf`b0WiT>!Z+>cQe~J}s%=C*ajKYe@ zIzhrWj|us31MQT4c3+ba^nVR7UuNC;>ijVBsp?Q;!{#{N)jgJh^a^yr3YA z7oV1yQ<7SgI^B?kQ5s1&F*!N4paes>C^aiJ86qq`Il+R11?2wC68{zmvVy#>KKY}O zIwyz)@)O7OxBQHgr`};SnBE}3*Z>Zk?JRvC1v>$lEt8?0mUVUZQDVR zLm8)U6=iG}6@kPiDE5#vH=dW4lt7X}H5w>8-A0UY2^VK#K|xV|S!z-7k(icE86l$stT&R7TW-yLzr4lrwbh6JOCHvmQw-UR>v delta 190 zcmX?bj`hzmR^CUm^{p77fPEwHdHu~21|Jw%_MewHGnvO|1&gOTyZYuGMj4EZU7Hn6 z>`!fVVVok!3X)cz9w@`8&Iw}PJSODBKK)<>qxAF!5k`UO7D9{}(=P}yHh|699wN;6 zfr%AllHBx-(u`u$L^cE?`is>@a WKr=uJr!SCZ>;SX2>&P&Qcmn`#Cp$&} diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index e13c36d24d..ba166be866 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -77,6 +77,8 @@ struct ValidatorSessionStats { ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero(); int block_status = status_none; double block_timestamp = -1.0; + td::Bits256 root_hash = td::Bits256::zero(); + td::Bits256 file_hash = td::Bits256::zero(); std::string comment; bool is_accepted = false; @@ -159,6 +161,7 @@ struct NewValidatorGroupStats { ValidatorSessionId session_id = ValidatorSessionId::zero(); ShardIdFull shard{masterchainId}; CatchainSeqno cc_seqno = 0; + BlockSeqno last_key_block_seqno = 0; double timestamp = -1.0; td::uint32 self_idx = 0; std::vector nodes; diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 46dd444033..bcd059d366 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -270,6 +270,8 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice } stat->deserialize_time = deserialize_time; stat->serialized_size = data.size(); + stat->root_hash = candidate->root_hash_; + stat->file_hash = file_hash; } if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || @@ -468,6 +470,8 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCan stat->collated_at = td::Clocks::system(); stat->block_timestamp = td::Clocks::system(); stat->collation_cached = collation_cached; + stat->root_hash = root_hash; + stat->file_hash = file_hash; } if (round != cur_round_) { return; @@ -602,6 +606,8 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { if (stat->block_timestamp <= 0.0) { stat->block_timestamp = td::Clocks::system(); } + stat->root_hash = B->root_hash_; + stat->file_hash = td::sha256_bits256(B->data_); } auto P = td::PromiseCreator::lambda([round = cur_round_, hash = block_id, root_hash = block->get_root_hash(), diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 913a0ed878..b8d9e56d3d 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -356,6 +356,11 @@ class Collator final : public td::actor::Actor { public: static td::uint32 get_skip_externals_queue_size(); + + private: + td::Timer work_timer_{true}; + td::ThreadCpuTimer cpu_work_timer_{true}; + CollationStats stats_; }; } // namespace validator diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index c6dd7caf2d..2dd0f55f65 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1772,6 +1772,12 @@ bool Collator::register_shard_block_creators(std::vector creator_li * @returns True if collation is successful, false otherwise. */ bool Collator::try_collate() { + work_timer_.resume(); + cpu_work_timer_.resume(); + SCOPE_EXIT { + work_timer_.pause(); + cpu_work_timer_.pause(); + }; if (!preinit_complete) { LOG(WARNING) << "running do_preinit()"; if (!do_preinit()) { @@ -3481,6 +3487,29 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT return true; } +/** + * Creates a string that explains which limit is exceeded. Used for collator stats. + * + * @param block_limit_status Status of block limits. + * @param cls Which limit class is exceeded. + * + * @returns String for collator stats. + */ +static std::string block_full_comment(const block::BlockLimitStatus& block_limit_status, unsigned cls) { + auto bytes = block_limit_status.estimate_block_size(); + if (!block_limit_status.limits.bytes.fits(cls, bytes)) { + return PSTRING() << "block_full bytes " << bytes; + } + if (!block_limit_status.limits.gas.fits(cls, block_limit_status.gas_used)) { + return PSTRING() << "block_full gas " << block_limit_status.gas_used; + } + auto lt_delta = block_limit_status.cur_lt - block_limit_status.limits.start_lt; + if (!block_limit_status.limits.lt_delta.fits(cls, lt_delta)) { + return PSTRING() << "block_full lt_delta " << lt_delta; + } + return ""; +} + /** * Processes inbound internal messages from message queues of the neighbors. * Messages are processed until the normal limit is reached, soft timeout is reached or there are no more messages. @@ -3495,11 +3524,14 @@ bool Collator::process_inbound_internal_messages() { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); if (block_full_) { LOG(INFO) << "BLOCK FULL, stop processing inbound internal messages"; + stats_.limits_log += PSTRING() << "INBOUND_INT_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) << "\n"; break; } if (soft_timeout_.is_in_past(td::Timestamp::now())) { block_full_ = true; LOG(WARNING) << "soft timeout reached, stop processing inbound internal messages"; + stats_.limits_log += PSTRING() << "INBOUND_INT_MESSAGES: timeout\n"; break; } auto kv = nb_out_msgs_->extract_cur(); @@ -3547,15 +3579,23 @@ bool Collator::process_inbound_external_messages() { } if (full) { LOG(INFO) << "BLOCK FULL, stop processing external messages"; + stats_.limits_log += PSTRING() << "INBOUND_EXT_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_soft) << "\n"; break; } if (medium_timeout_.is_in_past(td::Timestamp::now())) { LOG(WARNING) << "medium timeout reached, stop processing inbound external messages"; + stats_.limits_log += PSTRING() << "INBOUND_EXT_MESSAGES: timeout\n"; break; } auto ext_msg = ext_msg_struct.cell; ton::Bits256 hash{ext_msg->get_hash().bits()}; int r = process_external_message(std::move(ext_msg)); + if (r > 0) { + ++stats_.ext_msgs_accepted; + } else { + ++stats_.ext_msgs_rejected; + } if (r < 0) { bad_ext_msgs_.emplace_back(ext_msg_struct.hash); return false; @@ -3661,11 +3701,15 @@ bool Collator::process_dispatch_queue() { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); if (block_full_) { LOG(INFO) << "BLOCK FULL, stop processing dispatch queue"; + stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) + << "\n"; return true; } if (soft_timeout_.is_in_past(td::Timestamp::now())) { block_full_ = true; LOG(WARNING) << "soft timeout reached, stop processing dispatch queue"; + stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": timeout\n"; return true; } StdSmcAddress src_addr; @@ -3715,6 +3759,7 @@ bool Collator::process_dispatch_queue() { ++total_count; if (total_count >= max_total_count[iter]) { dispatch_queue_total_limit_reached_ = true; + stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": total limit reached\n"; break; } } @@ -4064,6 +4109,8 @@ bool Collator::process_new_messages(bool enqueue_only) { if ((block_full_ || have_unprocessed_account_dispatch_queue_) && !enqueue_only) { LOG(INFO) << "BLOCK FULL, enqueue all remaining new messages"; enqueue_only = true; + stats_.limits_log += PSTRING() << "NEW_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) << "\n"; } LOG(DEBUG) << "have message with lt=" << msg.lt; int res = process_one_new_message(std::move(msg), enqueue_only); @@ -5435,6 +5482,18 @@ bool Collator::create_block_candidate() { td::actor::send_closure_later(manager, &ValidatorManager::complete_external_messages, std::move(delay_ext_msgs_), std::move(bad_ext_msgs_)); } + + double work_time = work_timer_.elapsed(); + double cpu_work_time = cpu_work_timer_.elapsed(); + LOG(WARNING) << "Collate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; + stats_.bytes = block_limit_status_->estimate_block_size(); + stats_.gas = block_limit_status_->gas_used; + stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; + stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.bytes); + stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas); + stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); + td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, block_candidate->id, work_time, + cpu_work_time, std::move(stats_)); return true; } @@ -5539,6 +5598,7 @@ void Collator::after_get_external_messages(td::Result ext_msg_cell = ext_msg->root_cell(); @@ -5550,6 +5610,7 @@ void Collator::after_get_external_messages(td::Resulthash()); } } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 8c39a1ab4a..003b7f9f71 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -118,6 +118,7 @@ bool ValidateQuery::reject_query(std::string error, td::BufferSlice reason) { error = error_ctx() + error; LOG(ERROR) << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { + record_stats(); errorlog::ErrorLog::log(PSTRING() << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); @@ -155,6 +156,7 @@ bool ValidateQuery::soft_reject_query(std::string error, td::BufferSlice reason) error = error_ctx() + error; LOG(ERROR) << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { + record_stats(); errorlog::ErrorLog::log(PSTRING() << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); @@ -177,6 +179,7 @@ bool ValidateQuery::fatal_error(td::Status error) { error.ensure_error(); LOG(ERROR) << "aborting validation of block candidate for " << shard_.to_str() << " : " << error.to_string(); if (main_promise) { + record_stats(); auto c = error.code(); if (c <= -667 && c >= -670) { errorlog::ErrorLog::log(PSTRING() << "FATAL ERROR: aborting validation of block candidate for " << shard_.to_str() @@ -234,6 +237,7 @@ bool ValidateQuery::fatal_error(std::string err_msg, int err_code) { */ void ValidateQuery::finish_query() { if (main_promise) { + record_stats(); LOG(WARNING) << "validate query done"; main_promise.set_result(now_); } @@ -6764,6 +6768,12 @@ bool ValidateQuery::try_validate() { if (pending) { return true; } + work_timer_.resume(); + cpu_work_timer_.resume(); + SCOPE_EXIT { + work_timer_.pause(); + cpu_work_timer_.pause(); + }; try { if (!stage_) { LOG(WARNING) << "try_validate stage 0"; @@ -6903,6 +6913,17 @@ void ValidateQuery::written_candidate() { finish_query(); } +/** + * Sends validation work time to manager. + */ +void ValidateQuery::record_stats() { + double work_time = work_timer_.elapsed(); + double cpu_work_time = cpu_work_timer_.elapsed(); + LOG(WARNING) << "Validate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; + td::actor::send_closure(manager, &ValidatorManager::record_validate_query_stats, block_candidate.id, work_time, + cpu_work_time); +} + } // namespace validator } // namespace ton diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 824afb49d9..1049509380 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -398,6 +398,10 @@ class ValidateQuery : public td::actor::Actor { } return true; } + + td::Timer work_timer_{true}; + td::ThreadCpuTimer cpu_work_timer_{true}; + void record_stats(); }; } // namespace validator diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 0e9fab73b9..5ea074ddf2 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -52,6 +52,16 @@ struct AsyncSerializerState { UnixTime last_written_block_ts; }; +struct CollationStats { + td::uint32 bytes, gas, lt_delta; + int cat_bytes, cat_gas, cat_lt_delta; + std::string limits_log; + td::uint32 ext_msgs_total = 0; + td::uint32 ext_msgs_filtered = 0; + td::uint32 ext_msgs_accepted = 0; + td::uint32 ext_msgs_rejected = 0; +}; + using ValidateCandidateResult = td::Variant; class ValidatorManager : public ValidatorManagerInterface { @@ -192,6 +202,12 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void add_lite_query_stats(int lite_query_id) { } + virtual void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, + CollationStats stats) { + } + virtual void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { + } + static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/manager.cpp b/validator/manager.cpp index eb082d91ef..ec3dbce8da 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -44,6 +44,7 @@ #include "td/utils/JsonBuilder.h" #include "common/delay.h" +#include "td/utils/filesystem.h" #include "validator/stats-merger.h" @@ -2044,7 +2045,7 @@ void ValidatorManagerImpl::update_shards() { } new_validator_groups_.emplace(val_group_id, std::move(it2->second)); } else { - auto G = create_validator_group(val_group_id, shard, val_set, opts, started_); + auto G = create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_); if (!G.empty()) { td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_, last_masterchain_state_->get_unix_time()); @@ -2100,7 +2101,7 @@ void ValidatorManagerImpl::update_shards() { } new_validator_groups_.emplace(val_group_id, std::move(it2->second)); } else { - auto G = create_validator_group(val_group_id, shard, val_set, opts, started_); + auto G = create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_); if (!G.empty()) { td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_, last_masterchain_state_->get_unix_time()); @@ -2127,7 +2128,7 @@ void ValidatorManagerImpl::update_shards() { } else { new_next_validator_groups_.emplace( val_group_id, - ValidatorGroupEntry{create_validator_group(val_group_id, shard, val_set, opts, started_), shard}); + ValidatorGroupEntry{create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_), shard}); } } } @@ -2230,7 +2231,7 @@ ValidatorSessionId ValidatorManagerImpl::get_validator_set_id(ShardIdFull shard, } td::actor::ActorOwn ValidatorManagerImpl::create_validator_group( - ValidatorSessionId session_id, ShardIdFull shard, td::Ref validator_set, + ValidatorSessionId session_id, ShardIdFull shard, td::Ref validator_set, BlockSeqno key_seqno, validatorsession::ValidatorSessionOptions opts, bool init_session) { if (check_gc_list_.count(session_id) == 1) { return td::actor::ActorOwn{}; @@ -2241,8 +2242,8 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto validator_id = get_validator(shard, validator_set); CHECK(!validator_id.is_zero()); auto G = td::actor::create_actor( - "validatorgroup", shard, validator_id, session_id, validator_set, opts, keyring_, adnl_, rldp_, overlays_, - db_root_, actor_id(this), init_session, + "validatorgroup", shard, validator_id, session_id, validator_set, key_seqno, opts, keyring_, adnl_, rldp_, + overlays_, db_root_, actor_id(this), init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_); return G; } @@ -2831,13 +2832,35 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, for (const auto &round : stats.rounds) { std::vector> producers; for (const auto &producer : round.producers) { + BlockIdExt cur_block_id{block_id.id, producer.root_hash, producer.file_hash}; + auto it = recorded_block_stats_.find(cur_block_id); + tl_object_ptr collation_stats; + if (it != recorded_block_stats_.end() && it->second.collator_stats_) { + auto &stats = it->second.collator_stats_.value(); + collation_stats = create_tl_object( + stats.bytes, stats.gas, stats.lt_delta, stats.cat_bytes, stats.cat_gas, stats.cat_lt_delta, + stats.limits_log, stats.ext_msgs_total, stats.ext_msgs_filtered, stats.ext_msgs_accepted, + stats.ext_msgs_rejected); + } + std::string approvers, signers; + for (bool x : producer.approvers) { + approvers += (x ? '1' : '0'); + } + for (bool x : producer.signers) { + signers += (x ? '1' : '0'); + } producers.push_back(create_tl_object( - producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.comment, - producer.block_timestamp, producer.is_accepted, producer.is_ours, producer.got_submit_at, - producer.collation_time, producer.collated_at, producer.collation_cached, producer.validation_time, - producer.validated_at, producer.validation_cached, producer.gen_utime, producer.approved_weight, - producer.approved_33pct_at, producer.approved_66pct_at, producer.signed_weight, producer.signed_33pct_at, - producer.signed_66pct_at, producer.serialize_time, producer.deserialize_time, producer.serialized_size)); + producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.root_hash, + producer.file_hash, producer.comment, producer.block_timestamp, producer.is_accepted, producer.is_ours, + producer.got_submit_at, producer.collation_time, producer.collated_at, producer.collation_cached, + it == recorded_block_stats_.end() ? -1.0 : it->second.collator_work_time_, + it == recorded_block_stats_.end() ? -1.0 : it->second.collator_cpu_work_time_, std::move(collation_stats), + producer.validation_time, producer.validated_at, producer.validation_cached, + it == recorded_block_stats_.end() ? -1.0 : it->second.validator_work_time_, + it == recorded_block_stats_.end() ? -1.0 : it->second.validator_cpu_work_time_, producer.gen_utime, + producer.approved_weight, producer.approved_33pct_at, producer.approved_66pct_at, std::move(approvers), + producer.signed_weight, producer.signed_33pct_at, producer.signed_66pct_at, std::move(signers), + producer.serialize_time, producer.deserialize_time, producer.serialized_size)); } rounds.push_back(create_tl_object(round.timestamp, std::move(producers))); } @@ -2869,8 +2892,8 @@ void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewVa create_tl_object(node.id.bits256_value(), node.weight)); } auto obj = create_tl_object( - stats.session_id, stats.shard.workchain, stats.shard.shard, stats.cc_seqno, stats.timestamp, stats.self_idx, - std::move(nodes)); + stats.session_id, stats.shard.workchain, stats.shard.shard, stats.cc_seqno, stats.last_key_block_seqno, + stats.timestamp, stats.self_idx, std::move(nodes)); auto s = td::json_encode(td::ToJson(*obj.get()), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); @@ -3165,6 +3188,31 @@ td::actor::ActorOwn ValidatorManagerFactory::create( rldp, overlays); } +void ValidatorManagerImpl::record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, + CollationStats stats) { + auto &record = new_block_stats_record(block_id); + record.collator_work_time_ = work_time; + record.collator_cpu_work_time_ = cpu_work_time; + record.collator_stats_ = std::move(stats); +} + +void ValidatorManagerImpl::record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { + auto &record = new_block_stats_record(block_id); + record.validator_work_time_ = work_time; + record.validator_cpu_work_time_ = cpu_work_time; +} + +ValidatorManagerImpl::RecordedBlockStats &ValidatorManagerImpl::new_block_stats_record(BlockIdExt block_id) { + if (!recorded_block_stats_.count(block_id)) { + recorded_block_stats_lru_.push(block_id); + if (recorded_block_stats_lru_.size() > 4096) { + recorded_block_stats_.erase(recorded_block_stats_lru_.front()); + recorded_block_stats_lru_.pop(); + } + } + return recorded_block_stats_[block_id]; +} + size_t ValidatorManagerImpl::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) { before_query(); auto it1 = counter_cur_.find({wc, addr}); diff --git a/validator/manager.hpp b/validator/manager.hpp index 12354c6349..50fa79ccaf 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -38,6 +38,7 @@ #include #include #include +#include namespace ton { @@ -261,7 +262,7 @@ class ValidatorManagerImpl : public ValidatorManager { BlockSeqno last_key_block_seqno, const validatorsession::ValidatorSessionOptions &opts); td::actor::ActorOwn create_validator_group(ValidatorSessionId session_id, ShardIdFull shard, - td::Ref validator_set, + td::Ref validator_set, BlockSeqno key_seqno, validatorsession::ValidatorSessionOptions opts, bool create_catchain); struct ValidatorGroupEntry { @@ -708,6 +709,21 @@ class ValidatorManagerImpl : public ValidatorManager { td::uint32 ls_stats_check_ext_messages_{0}; td::actor::ActorOwn candidates_buffer_; + + struct RecordedBlockStats { + double collator_work_time_ = -1.0; + double collator_cpu_work_time_ = -1.0; + td::optional collator_stats_; + double validator_work_time_ = -1.0; + double validator_cpu_work_time_ = -1.0; + }; + std::map recorded_block_stats_; + std::queue recorded_block_stats_lru_; + + void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, + CollationStats stats) override; + void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) override; + RecordedBlockStats &new_block_stats_record(BlockIdExt block_id); }; } // namespace validator diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index fc3ebe5415..60573581b0 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -386,6 +386,7 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch stats.session_id = session_id_; stats.shard = shard_; stats.cc_seqno = validator_set_->get_catchain_seqno(); + stats.last_key_block_seqno = last_key_block_seqno_; stats.timestamp = td::Clocks::system(); td::uint32 idx = 0; for (const auto& node : validator_set_->export_vector()) { diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 3499da9d7e..936d2fdc71 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -69,15 +69,17 @@ class ValidatorGroup : public td::actor::Actor { } ValidatorGroup(ShardIdFull shard, PublicKeyHash local_id, ValidatorSessionId session_id, - td::Ref validator_set, validatorsession::ValidatorSessionOptions config, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, - std::string db_root, td::actor::ActorId validator_manager, bool create_session, + td::Ref validator_set, BlockSeqno last_key_block_seqno, + validatorsession::ValidatorSessionOptions config, td::actor::ActorId keyring, + td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId overlays, std::string db_root, + td::actor::ActorId validator_manager, bool create_session, bool allow_unsafe_self_blocks_resync, td::Ref opts) : shard_(shard) , local_id_(std::move(local_id)) , session_id_(session_id) , validator_set_(std::move(validator_set)) + , last_key_block_seqno_(last_key_block_seqno) , config_(std::move(config)) , keyring_(keyring) , adnl_(adnl) @@ -115,6 +117,7 @@ class ValidatorGroup : public td::actor::Actor { UnixTime min_ts_; td::Ref validator_set_; + BlockSeqno last_key_block_seqno_; validatorsession::ValidatorSessionOptions config_; td::actor::ActorId keyring_; From 1b4fb42859c84730d79b88372160f892c918d8b1 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 19 Aug 2024 19:55:11 +0300 Subject: [PATCH 115/524] End validator session stats --- catchain/catchain.h | 1 + catchain/catchain.hpp | 9 +++++++ tl/generate/scheme/ton_api.tl | 8 ++++-- tl/generate/scheme/ton_api.tlo | Bin 91968 -> 92472 bytes validator-session/validator-session-types.h | 11 +++++++++ validator-session/validator-session.cpp | 23 +++++++++++++++++ validator-session/validator-session.h | 1 + validator-session/validator-session.hpp | 1 + validator/impl/collator.cpp | 2 ++ validator/interfaces/validator-manager.h | 1 + validator/manager-disk.hpp | 3 +++ validator/manager-hardfork.hpp | 3 +++ validator/manager.cpp | 26 +++++++++++++++++++- validator/manager.hpp | 1 + validator/validator-group.cpp | 10 ++++++++ 15 files changed, 97 insertions(+), 3 deletions(-) diff --git a/catchain/catchain.h b/catchain/catchain.h index 912957e56a..c5c8af28d1 100644 --- a/catchain/catchain.h +++ b/catchain/catchain.h @@ -96,6 +96,7 @@ class CatChain : public td::actor::Actor { virtual void send_query_via(const PublicKeyHash &dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice query, td::uint64 max_answer_size, td::actor::ActorId via) = 0; + virtual void get_source_heights(td::Promise> promise) = 0; virtual void destroy() = 0; static td::actor::ActorOwn create(std::unique_ptr callback, const CatChainOptions &opts, diff --git a/catchain/catchain.hpp b/catchain/catchain.hpp index 8c8bb99ae9..586cf4744d 100644 --- a/catchain/catchain.hpp +++ b/catchain/catchain.hpp @@ -115,6 +115,15 @@ class CatChainImpl : public CatChain { td::actor::send_closure(receiver_, &CatChainReceiverInterface::send_custom_query_data_via, dst, name, std::move(promise), timeout, std::move(query), max_answer_size, via); } + void get_source_heights(td::Promise> promise) override { + std::vector heights(top_source_blocks_.size(), 0); + for (size_t i = 0; i < top_source_blocks_.size(); ++i) { + if (top_source_blocks_[i]) { + heights[i] = top_source_blocks_[i]->height(); + } + } + promise.set_result(std::move(heights)); + } void destroy() override; CatChainImpl(std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 6c5126a114..bf919b0fd0 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -388,8 +388,8 @@ tonNode.newShardBlock block:tonNode.blockIdExt cc_seqno:int data:bytes = tonNode tonNode.blockBroadcastCompressed.data signatures:(vector tonNode.blockSignature) proof_data:bytes = tonNode.blockBroadcaseCompressed.Data; -tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int - signatures:(vector tonNode.blockSignature) +tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + signatures:(vector tonNode.blockSignature) proof:bytes data:bytes = tonNode.Broadcast; tonNode.blockBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int flags:# compressed:bytes = tonNode.Broadcast; @@ -796,6 +796,10 @@ validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:lo last_key_block_seqno:int timestamp:double self_idx:int nodes:(vector validatorSession.newValidatorGroupStats.node) = validatorSession.NewValidatorGroupStats; +validatorSession.endValidatorGroupStats.node id:int256 catchain_blocks:int = validatorSession.endValidatorGroupStats.Node; +validatorSession.endValidatorGroupStats session_id:int256 timestamp:double + nodes:(vector validatorSession.endValidatorGroupStats.node) = validatorSession.EndValidatorGroupStats; + ---functions--- ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 509655394679a5ec39644c3c1973334603d33cf3..337dd071e3f746cd822236e2dcb63c194b379de4 100644 GIT binary patch delta 267 zcmX?bj&;W+R^CUm^{p77fNLY~VMEDjuZ-f=%Mx=kQxZ$^i-J>&i!<}{^jz~&CMz0g zZ@yp{!6@0H?XyS+LrE%7iKDUcWDVmVjJ=x`O~miwFbkwm(MXsRWYf)KLO#sX4@NLb zPj3)m6qv3d!l*L2Cs9QLWCtVEVMWetIvlob2SFOP>j*L`aN~7?2zOq7N-9t<$c)Jk kx- nodes; }; +struct EndValidatorGroupStats { + struct Node { + PublicKeyHash id = PublicKeyHash::zero(); + td::uint32 catchain_blocks = 0; + }; + + ValidatorSessionId session_id = ValidatorSessionId::zero(); + double timestamp = -1.0; + std::vector nodes; +}; + } // namespace validatorsession } // namespace ton diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index bcd059d366..be54437859 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -1003,6 +1003,29 @@ void ValidatorSessionImpl::get_current_stats(td::Promise promise.set_result(cur_stats_); } +void ValidatorSessionImpl::get_end_stats(td::Promise promise) { + if (!started_) { + promise.set_error(td::Status::Error(ErrorCode::notready, "not started")); + return; + } + EndValidatorGroupStats stats; + stats.session_id = unique_hash_; + stats.timestamp = td::Clocks::system(); + stats.nodes.resize(description().get_total_nodes()); + for (size_t i = 0; i < stats.nodes.size(); ++i) { + stats.nodes[i].id = description().get_source_id(i); + } + td::actor::send_closure(catchain_, &catchain::CatChain::get_source_heights, + [promise = std::move(promise), + stats = std::move(stats)](td::Result> R) mutable { + TRY_RESULT_PROMISE(promise, heights, std::move(R)); + for (size_t i = 0; i < std::min(heights.size(), stats.nodes.size()); ++i) { + stats.nodes[i].catchain_blocks = heights[i]; + } + promise.set_result(std::move(stats)); + }); +} + void ValidatorSessionImpl::get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) { diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index 0870f67184..2e1ed9b133 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -105,6 +105,7 @@ class ValidatorSession : public td::actor::Actor { virtual void start() = 0; virtual void destroy() = 0; virtual void get_current_stats(td::Promise promise) = 0; + virtual void get_end_stats(td::Promise promise) = 0; virtual void get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) = 0; diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 5805828246..2ee4885b98 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -187,6 +187,7 @@ class ValidatorSessionImpl : public ValidatorSession { void start() override; void destroy() override; void get_current_stats(td::Promise promise) override; + void get_end_stats(td::Promise promise) override; void get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) override; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 2dd0f55f65..f465c0f553 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -4119,6 +4119,8 @@ bool Collator::process_new_messages(bool enqueue_only) { } else if (res == 3) { LOG(INFO) << "All remaining new messages must be enqueued (BLOCK FULL)"; enqueue_only = true; + stats_.limits_log += PSTRING() << "NEW_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) << "\n"; } } return true; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 5ea074ddf2..b6016bc2bd 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -183,6 +183,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0; virtual void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) = 0; + virtual void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) = 0; virtual void get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) = 0; virtual void get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index a77be2725d..3a77f2301e 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -388,6 +388,9 @@ class ValidatorManagerImpl : public ValidatorManager { void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { UNREACHABLE(); } + void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override { + UNREACHABLE(); + } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index e7175b77bb..cf4d3799fa 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -450,6 +450,9 @@ class ValidatorManagerImpl : public ValidatorManager { void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { UNREACHABLE(); } + void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override { + UNREACHABLE(); + } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = diff --git a/validator/manager.cpp b/validator/manager.cpp index ec3dbce8da..fa592a788b 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2902,7 +2902,31 @@ void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewVa file << s << "\n"; file.close(); - LOG(INFO) << "Writing new validator group stats for " << stats.shard.to_str(); + LOG(INFO) << "Writing new validator group stats for " << stats.session_id << " shard=" << stats.shard.to_str() + << " cc_seqno=" << stats.cc_seqno; +} + +void ValidatorManagerImpl::log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) { + std::string fname = opts_->get_session_logs_file(); + if (fname.empty()) { + return; + } + std::vector> nodes; + for (const auto &node : stats.nodes) { + nodes.push_back(create_tl_object(node.id.bits256_value(), + node.catchain_blocks)); + } + auto obj = create_tl_object(stats.session_id, stats.timestamp, + std::move(nodes)); + auto s = td::json_encode(td::ToJson(*obj.get()), false); + s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); + + std::ofstream file; + file.open(fname, std::ios_base::app); + file << s << "\n"; + file.close(); + + LOG(INFO) << "Writing end validator group stats for " << stats.session_id; } void ValidatorManagerImpl::get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 50fa79ccaf..99aa4e0e11 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -590,6 +590,7 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override; void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override; + void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override; void update_options(td::Ref opts) override; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 60573581b0..4b61c07cd1 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -418,6 +418,16 @@ void ValidatorGroup::destroy() { td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats, block_id, std::move(stats)); }); + td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_end_stats, + [manager = manager_](td::Result R) { + if (R.is_error()) { + LOG(DEBUG) << "Failed to get validator session end stats: " << R.move_as_error(); + return; + } + auto stats = R.move_as_ok(); + td::actor::send_closure(manager, &ValidatorManager::log_end_validator_group_stats, + std::move(stats)); + }); auto ses = session_.release(); delay_action([ses]() mutable { td::actor::send_closure(ses, &validatorsession::ValidatorSession::destroy); }, td::Timestamp::in(10.0)); From dbe51d6d138c9f8ec1acd928916158ce09283718 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 16 Aug 2024 10:14:54 +0300 Subject: [PATCH 116/524] Status page in blockchain-explorer --- .../blockchain-explorer-query.cpp | 2 +- blockchain-explorer/blockchain-explorer.cpp | 66 +++++++++---- lite-client/ext-client.cpp | 94 +++++++++++++++---- lite-client/ext-client.h | 9 +- 4 files changed, 132 insertions(+), 39 deletions(-) diff --git a/blockchain-explorer/blockchain-explorer-query.cpp b/blockchain-explorer/blockchain-explorer-query.cpp index 1808a3c462..26a6787e1e 100644 --- a/blockchain-explorer/blockchain-explorer-query.cpp +++ b/blockchain-explorer/blockchain-explorer-query.cpp @@ -1432,7 +1432,7 @@ void HttpQueryStatus::finish_query() { for (td::uint32 i = 0; i < results_.ips.size(); i++) { A << ""; if (results_.ips[i].is_valid()) { - A << "" << results_.ips[i] << ""; + A << "" << results_.ips[i].get_ip_str() << ":" << results_.ips[i].get_port() << ""; } else { A << "hidden"; } diff --git a/blockchain-explorer/blockchain-explorer.cpp b/blockchain-explorer/blockchain-explorer.cpp index 029c718d62..ca50d52665 100644 --- a/blockchain-explorer/blockchain-explorer.cpp +++ b/blockchain-explorer/blockchain-explorer.cpp @@ -155,8 +155,12 @@ class CoreActor : public CoreActorInterface { td::int32 attempt_ = 0; td::int32 waiting_ = 0; - //void run_queries(); - void got_result(td::uint32 idx, td::int32 attempt, td::Result data); + size_t n_servers_ = 0; + + void run_queries(); + void got_servers_ready(td::int32 attempt, std::vector ready); + void send_ping(td::uint32 idx); + void got_ping_result(td::uint32 idx, td::int32 attempt, td::Result data); void add_result() { if (new_result_) { @@ -175,7 +179,7 @@ class CoreActor : public CoreActorInterface { add_result(); } attempt_ = t; - //run_queries(); + run_queries(); alarm_timestamp() = td::Timestamp::at_unix((attempt_ + 1) * 60); } @@ -450,7 +454,8 @@ class CoreActor : public CoreActorInterface { addrs_.push_back(remote_addr_); servers.push_back(liteclient::LiteServerConfig{ton::adnl::AdnlNodeIdFull{remote_public_key_}, remote_addr_}); } - client_ = liteclient::ExtClient::create(std::move(servers), make_callback()); + n_servers_ = servers.size(); + client_ = liteclient::ExtClient::create(std::move(servers), make_callback(), true); daemon_ = MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, static_cast(http_port_), nullptr, nullptr, &process_http_request, nullptr, MHD_OPTION_NOTIFY_COMPLETED, request_completed, nullptr, MHD_OPTION_THREAD_POOL_SIZE, 16, MHD_OPTION_END); @@ -458,7 +463,46 @@ class CoreActor : public CoreActorInterface { } }; -void CoreActor::got_result(td::uint32 idx, td::int32 attempt, td::Result R) { +void CoreActor::run_queries() { + waiting_ = 0; + new_result_ = std::make_shared(n_servers_, td::Timestamp::at_unix(attempt_ * 60)); + td::actor::send_closure(client_, &liteclient::ExtClient::get_servers_status, + [SelfId = actor_id(this), attempt = attempt_](td::Result> R) { + R.ensure(); + td::actor::send_closure(SelfId, &CoreActor::got_servers_ready, attempt, R.move_as_ok()); + }); +} + +void CoreActor::got_servers_ready(td::int32 attempt, std::vector ready) { + if (attempt != attempt_) { + return; + } + CHECK(ready.size() == n_servers_); + for (td::uint32 i = 0; i < n_servers_; i++) { + if (ready[i]) { + send_ping(i); + } + } + CHECK(waiting_ >= 0); + if (waiting_ == 0) { + add_result(); + } +} + +void CoreActor::send_ping(td::uint32 idx) { + waiting_++; + auto query = ton::create_tl_object(); + auto q = ton::create_tl_object(serialize_tl_object(query, true)); + + auto P = + td::PromiseCreator::lambda([SelfId = actor_id(this), idx, attempt = attempt_](td::Result R) { + td::actor::send_closure(SelfId, &CoreActor::got_ping_result, idx, attempt, std::move(R)); + }); + td::actor::send_closure(client_, &liteclient::ExtClient::send_query_to_server, "query", serialize_tl_object(q, true), + idx, td::Timestamp::in(10.0), std::move(P)); +} + +void CoreActor::got_ping_result(td::uint32 idx, td::int32 attempt, td::Result R) { if (attempt != attempt_) { return; } @@ -499,18 +543,6 @@ void CoreActor::got_result(td::uint32 idx, td::int32 attempt, td::Result(ready_.size(), td::Timestamp::at_unix(attempt_ * 60)); - for (td::uint32 i = 0; i < ready_.size(); i++) { - send_query(i); - } - CHECK(waiting_ >= 0); - if (waiting_ == 0) { - add_result(); - } -}*/ - void CoreActor::send_lite_query(td::BufferSlice query, td::Promise promise) { auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { diff --git a/lite-client/ext-client.cpp b/lite-client/ext-client.cpp index 0232c28f45..a0e48e64a4 100644 --- a/lite-client/ext-client.cpp +++ b/lite-client/ext-client.cpp @@ -22,8 +22,8 @@ namespace liteclient { class ExtClientImpl : public ExtClient { public: - ExtClientImpl(std::vector liteservers, td::unique_ptr callback) - : callback_(std::move(callback)) { + ExtClientImpl(std::vector liteservers, td::unique_ptr callback, bool connect_to_all) + : callback_(std::move(callback)), connect_to_all_(connect_to_all) { CHECK(!liteservers.empty()); servers_.resize(liteservers.size()); for (size_t i = 0; i < servers_.size(); ++i) { @@ -36,15 +36,65 @@ class ExtClientImpl : public ExtClient { LOG(INFO) << "Started ext client, " << servers_.size() << " liteservers"; td::Random::Fast rnd; td::random_shuffle(td::as_mutable_span(servers_), rnd); + server_indices_.resize(servers_.size()); + for (size_t i = 0; i < servers_.size(); ++i) { + server_indices_[servers_[i].idx] = i; + } + + if (connect_to_all_) { + for (size_t i = 0; i < servers_.size(); ++i) { + prepare_server(i, nullptr); + } + } } void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, td::Promise promise) override { QueryInfo query_info = get_query_info(data); TRY_RESULT_PROMISE(promise, server_idx, select_server(query_info)); + send_query_internal(std::move(name), std::move(data), std::move(query_info), server_idx, timeout, + std::move(promise)); + } + + void send_query_to_server(std::string name, td::BufferSlice data, size_t server_idx, td::Timestamp timeout, + td::Promise promise) override { + if (server_idx >= servers_.size()) { + promise.set_error(td::Status::Error(PSTRING() << "server idx " << server_idx << " is too big")); + return; + } + server_idx = server_indices_[server_idx]; + QueryInfo query_info = get_query_info(data); + prepare_server(server_idx, &query_info); + send_query_internal(std::move(name), std::move(data), std::move(query_info), server_idx, timeout, + std::move(promise)); + } + + void get_servers_status(td::Promise> promise) override { + std::vector status(servers_.size()); + for (const Server& s : servers_) { + status[s.idx] = s.alive; + } + promise.set_result(std::move(status)); + } + + void reset_servers() override { + LOG(INFO) << "Force resetting all liteservers"; + for (Server& server : servers_) { + server.alive = false; + server.timeout = {}; + server.ignore_until = {}; + server.client.reset(); + } + } + + private: + void send_query_internal(std::string name, td::BufferSlice data, QueryInfo query_info, size_t server_idx, + td::Timestamp timeout, td::Promise promise) { auto& server = servers_[server_idx]; CHECK(!server.client.empty()); - alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); + if (!connect_to_all_) { + alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); + } td::Promise P = [SelfId = actor_id(this), server_idx, promise = std::move(promise)](td::Result R) mutable { if (R.is_error() && @@ -59,17 +109,6 @@ class ExtClientImpl : public ExtClient { std::move(P)); } - void reset_servers() override { - LOG(INFO) << "Force resetting all liteservers"; - for (Server& server : servers_) { - server.alive = false; - server.timeout = {}; - server.ignore_until = {}; - server.client.reset(); - } - } - - private: td::Result select_server(const QueryInfo& query_info) { for (size_t i = 0; i < servers_.size(); ++i) { if (servers_[i].alive && servers_[i].config.accepts_query(query_info)) { @@ -101,12 +140,22 @@ class ExtClientImpl : public ExtClient { if (server_idx == servers_.size()) { return td::Status::Error(PSTRING() << "no liteserver for query " << query_info.to_str()); } + prepare_server(server_idx, &query_info); + return server_idx; + } + + void prepare_server(size_t server_idx, const QueryInfo* query_info) { Server& server = servers_[server_idx]; + if (server.alive) { + return; + } server.alive = true; server.ignore_until = {}; - alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); + if (!connect_to_all_) { + alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); + } if (!server.client.empty()) { - return server_idx; + return; } class Callback : public ton::adnl::AdnlExtClient::Callback { @@ -124,10 +173,9 @@ class ExtClientImpl : public ExtClient { size_t idx_; }; LOG(INFO) << "Connecting to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() << ":" - << server.config.addr.get_port() << ") for query " << query_info.to_str(); + << server.config.addr.get_port() << ") for query " << (query_info ? query_info->to_str() : "[none]"); server.client = ton::adnl::AdnlExtClient::create(server.config.adnl_id, server.config.addr, std::make_unique(actor_id(this), server_idx)); - return server_idx; } struct Server { @@ -139,12 +187,17 @@ class ExtClientImpl : public ExtClient { td::Timestamp ignore_until = td::Timestamp::never(); }; std::vector servers_; + std::vector server_indices_; td::unique_ptr callback_; + bool connect_to_all_ = false; static constexpr double MAX_NO_QUERIES_TIMEOUT = 100.0; static constexpr double BAD_SERVER_TIMEOUT = 30.0; void alarm() override { + if (connect_to_all_) { + return; + } for (Server& server : servers_) { if (server.timeout && server.timeout.is_in_past()) { LOG(INFO) << "Closing connection to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() @@ -168,7 +221,8 @@ td::actor::ActorOwn ExtClient::create(ton::adnl::AdnlNodeIdFull dst, } td::actor::ActorOwn ExtClient::create(std::vector liteservers, - td::unique_ptr callback) { - return td::actor::create_actor("ExtClient", std::move(liteservers), std::move(callback)); + td::unique_ptr callback, bool connect_to_all) { + return td::actor::create_actor("ExtClient", std::move(liteservers), std::move(callback), + connect_to_all); } } // namespace liteclient diff --git a/lite-client/ext-client.h b/lite-client/ext-client.h index adcff9bed1..ef4523fd69 100644 --- a/lite-client/ext-client.h +++ b/lite-client/ext-client.h @@ -30,12 +30,19 @@ class ExtClient : public td::actor::Actor { virtual void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, td::Promise promise) = 0; + virtual void send_query_to_server(std::string name, td::BufferSlice data, size_t server_idx, td::Timestamp timeout, + td::Promise promise) { + promise.set_error(td::Status::Error("not supported")); + } + virtual void get_servers_status(td::Promise> promise) { + promise.set_error(td::Status::Error("not supported")); + } virtual void reset_servers() { } static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, td::unique_ptr callback); static td::actor::ActorOwn create(std::vector liteservers, - td::unique_ptr callback); + td::unique_ptr callback, bool connect_to_all = false); }; } // namespace liteclient From faedb4635cf9c8328777a69a6c9e728a22f38dba Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 19 Aug 2024 20:31:33 +0300 Subject: [PATCH 117/524] Remove unused get validator session stats query --- tl/generate/scheme/ton_api.tl | 6 ---- tl/generate/scheme/ton_api.tlo | Bin 101736 -> 100916 bytes .../validator-engine-console-query.cpp | 20 ----------- .../validator-engine-console-query.h | 19 ---------- .../validator-engine-console.cpp | 1 - validator-engine/validator-engine.cpp | 27 -------------- validator-engine/validator-engine.hpp | 2 -- validator-session/validator-session.cpp | 12 ------- validator-session/validator-session.h | 2 -- validator-session/validator-session.hpp | 2 -- validator/manager-disk.hpp | 5 --- validator/manager-hardfork.hpp | 4 --- validator/manager.cpp | 33 ------------------ validator/manager.hpp | 3 -- validator/validator-group.cpp | 19 ---------- validator/validator-group.hpp | 2 -- validator/validator.h | 3 -- 17 files changed, 160 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 70cb790b71..cc2cc67e12 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -712,10 +712,6 @@ engine.validator.perfTimerStats stats:(vector engine.validator.PerfTimerStatsByN engine.validator.shardOutQueueSize size:long = engine.validator.ShardOutQueueSize; -engine.validator.validatorSessionInfo current_block:tonNode.blockId self:int256 current_round:int next_producers:(vector int256) = engine.validator.ValidatorSessionInfo; -engine.validator.validatorSessionsInfo sessions:(vector engine.validator.validatorSessionInfo) = engine.validator.ValidatorSessionsInfo; - -engine.validator.requiredBlockCandidates block_ids:(vector tonNode.blockId) = engine.validator.RequiredBlockCandidates; ---functions--- @@ -779,8 +775,6 @@ engine.validator.setStateSerializerEnabled enabled:Bool = engine.validator.Succe engine.validator.setCollatorOptionsJson json:string = engine.validator.Success; engine.validator.getCollatorOptionsJson = engine.validator.JsonConfig; -engine.validator.getValidatorSessionsInfo = engine.validator.ValidatorSessionsInfo; - engine.validator.addCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; engine.validator.addShard shard:tonNode.shardId = engine.validator.Success; engine.validator.delCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index eae2a5590c1411adc052dbea3dbd6b5832abc004..b788ae4f2aa565cb7262ae3319329ff2c9930420 100644 GIT binary patch delta 65 zcmaDci*3skHr_|G^{p77Kxrdyz4YcQ(h-cCx5z5UGH%#>)#`uM=7en>(#)dN z6sMg0CdPh=Ei88>fcwfbM>hTVxp*qsRU97z4mV?sWRK(_R=`LsCZ|sRxK)0#&1n$_ zkZE8?0g0<}Trt=K4BaJ=00XI=T+m`70Wt*Lb0EphE4F`VXYAWP)1J{@nss)w^fS%r g8zUG+^wFaWVjcrX56EjE2f+dfB)0ub6k~=y0Q&6R)c^nh diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 6266ac2ac9..5de0fe1f9d 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1291,26 +1291,6 @@ td::Status GetCollatorOptionsJsonQuery::receive(td::BufferSlice data) { return td::Status::OK(); } -td::Status GetValidatorSessionsInfoQuery::run() { - TRY_STATUS(tokenizer_.check_endl()); - return td::Status::OK(); -} - -td::Status GetValidatorSessionsInfoQuery::send() { - auto b = ton::create_serialize_tl_object(); - td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); - return td::Status::OK(); -} - -td::Status GetValidatorSessionsInfoQuery::receive(td::BufferSlice data) { - TRY_RESULT_PREFIX( - f, ton::fetch_tl_object(data.as_slice(), true), - "received incorrect answer: "); - std::string s = td::json_encode(td::ToJson(*f), true); - td::TerminalIO::out() << "---------\n" << s << "--------\n"; - return td::Status::OK(); -} - td::Status AddCollatorQuery::run() { TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 73b2ae835d..3cddb80c8a 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1293,25 +1293,6 @@ class GetCollatorOptionsJsonQuery : public Query { std::string file_name_; }; -class GetValidatorSessionsInfoQuery : public Query { - public: - GetValidatorSessionsInfoQuery(td::actor::ActorId console, Tokenizer tokenizer) - : Query(console, std::move(tokenizer)) { - } - td::Status run() override; - td::Status send() override; - td::Status receive(td::BufferSlice data) override; - static std::string get_name() { - return "getvalidatorsessions"; - } - static std::string get_help() { - return "getvalidatorsessions\tprint info about validator session"; - } - std::string name() const override { - return get_name(); - } -}; - class AddCollatorQuery : public Query { public: AddCollatorQuery(td::actor::ActorId console, Tokenizer tokenizer) diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index c37d2771b2..eae819613d 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -150,7 +150,6 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); - add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index a6915f4098..705439bcec 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -4081,33 +4081,6 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showColla } } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getValidatorSessionsInfo &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { - if (!(perm & ValidatorEnginePermissions::vep_default)) { - promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); - return; - } - - if (validator_manager_.empty()) { - promise.set_value( - create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); - return; - } - - auto P = td::PromiseCreator::lambda( - [promise = std::move(promise)]( - td::Result> R) mutable { - if (R.is_error()) { - promise.set_value(create_control_query_error(R.move_as_error())); - } else { - promise.set_value(ton::serialize_tl_object(R.move_as_ok(), true)); - } - }); - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::get_validator_sessions_info, - std::move(P)); -} - void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollator &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 6a7680f0ec..51e0ca9b39 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -504,8 +504,6 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); - void run_control_query(ton::ton_api::engine_validator_getValidatorSessionsInfo &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_addCollator &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_addShard &query, td::BufferSlice data, diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index d9adf177ae..186d6600eb 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -1206,18 +1206,6 @@ void ValidatorSessionImpl::stats_process_action(td::uint32 node_id, ton_api::val [](const auto &) {})); } -void ValidatorSessionImpl::get_session_info( - td::Promise> promise) { - std::vector next_producers; - for (td::uint32 round = cur_round_; round < cur_round_ + 20; ++round) { - td::uint32 node = description().get_node_by_priority(round, 0); - next_producers.push_back(description().get_source_id(node).bits256_value()); - } - promise.set_result(create_tl_object( - create_tl_block_id_simple(BlockId{}), description().get_source_id(local_idx()).bits256_value(), - cur_round_, std::move(next_producers))); -} - td::actor::ActorOwn ValidatorSession::create( catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index d59c4e3818..0870f67184 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -110,8 +110,6 @@ class ValidatorSession : public td::actor::Actor { td::Promise>> promise) = 0; virtual void set_catchain_max_block_delay(double value) = 0; - virtual void get_session_info(td::Promise> promise) = 0; - static td::actor::ActorOwn create( catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 6cb476747d..5805828246 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -175,8 +175,6 @@ class ValidatorSessionImpl : public ValidatorSession { ValidatorSessionCandidateId candidate_id); void stats_process_action(td::uint32 node_id, ton_api::validatorSession_round_Message &action); - void get_session_info(td::Promise> promise) override; - public: ValidatorSessionImpl(catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 01a0c60eba..f4f68468e7 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -441,11 +441,6 @@ class ValidatorManagerImpl : public ValidatorManager { void add_persistent_state_description(td::Ref desc) override { } - void get_validator_sessions_info( - td::Promise> promise) override { - UNREACHABLE(); - } - void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { UNREACHABLE(); } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 28c521f430..371c757064 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -506,10 +506,6 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_persistent_state_description(td::Ref desc) override { } - void get_validator_sessions_info( - td::Promise> promise) override { - UNREACHABLE(); - } void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { UNREACHABLE(); diff --git a/validator/manager.cpp b/validator/manager.cpp index 11454c8e9c..d3003a832f 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3394,39 +3394,6 @@ void ValidatorManagerImpl::update_options(td::Ref opts) opts_ = std::move(opts); } -void ValidatorManagerImpl::get_validator_sessions_info( - td::Promise> promise) { - std::vector> groups; - for (const auto& g : validator_groups_) { - groups.push_back(g.second.actor.get()); - } - struct IntermediateData { - std::vector> groups; - std::vector> result; - td::Promise> promise; - - static void step(IntermediateData data) { - if (data.groups.empty()) { - data.promise.set_result( - create_tl_object(std::move(data.result))); - return; - } - auto group = std::move(data.groups.back()); - data.groups.pop_back(); - auto P = td::PromiseCreator::lambda( - [data = - std::move(data)](td::Result> R) mutable { - if (R.is_ok()) { - data.result.push_back(R.move_as_ok()); - } - step(std::move(data)); - }); - td::actor::send_closure(group, &ValidatorGroup::get_session_info, std::move(P)); - } - }; - IntermediateData::step({std::move(groups), {}, std::move(promise)}); -} - void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) { if (shard.is_masterchain() || !shard.is_valid_ext()) { LOG(WARNING) << "cannot collate shard " << shard.to_str(); diff --git a/validator/manager.hpp b/validator/manager.hpp index 007bce35f9..8c881f6058 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -622,9 +622,6 @@ class ValidatorManagerImpl : public ValidatorManager { void update_options(td::Ref opts) override; - void get_validator_sessions_info( - td::Promise> promise) override; - void add_persistent_state_description(td::Ref desc) override; void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index b708e6ace0..83e2735883 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -448,25 +448,6 @@ void ValidatorGroup::get_validator_group_info_for_litequery_cont( promise.set_result(std::move(result)); } -void ValidatorGroup::get_session_info( - td::Promise> promise) { - if (session_.empty() || !started_) { - promise.set_error(td::Status::Error(ErrorCode::notready, "session not started")); - } - auto P = td::PromiseCreator::lambda( - [promise = std::move(promise), block_id = create_next_block_id_simple()]( - td::Result> R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - return; - } - auto info = R.move_as_ok(); - info->current_block_ = create_tl_block_id_simple(block_id); - promise.set_result(std::move(info)); - }); - td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_session_info, std::move(P)); -} - void ValidatorGroup::collate_block(td::uint32 round_id, td::Timestamp timeout, td::Promise promise, unsigned max_retries) { if (round_id < last_known_round_id_) { diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index debb7d14aa..f44ce868bc 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -64,8 +64,6 @@ class ValidatorGroup : public td::actor::Actor { void get_validator_group_info_for_litequery( td::Promise> promise); - void get_session_info(td::Promise> promise); - void update_options(td::Ref opts, bool apply_blocks) { opts_ = std::move(opts); monitoring_shard_ = apply_blocks; diff --git a/validator/validator.h b/validator/validator.h index 03aa5827d1..927f21d7c2 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -302,9 +302,6 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void update_options(td::Ref opts) = 0; - virtual void get_validator_sessions_info( - td::Promise> promise) = 0; - virtual void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; virtual void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; }; From 9b4a3a8263e90a046e82457ca439e0565845f5c8 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 10 Sep 2024 20:32:05 +0300 Subject: [PATCH 118/524] Synchronize last mc seqno between servers in proxy-liteserver --- utils/proxy-liteserver.cpp | 91 ++++++++++++++++++++++++++++++++++---- 1 file changed, 83 insertions(+), 8 deletions(-) diff --git a/utils/proxy-liteserver.cpp b/utils/proxy-liteserver.cpp index 66161fc450..a9baa75956 100644 --- a/utils/proxy-liteserver.cpp +++ b/utils/proxy-liteserver.cpp @@ -43,8 +43,12 @@ #if TD_DARWIN || TD_LINUX #include #endif +#include "td/utils/overloaded.h" + #include #include +#include +#include "td/utils/tl_storers.h" using namespace ton; @@ -252,26 +256,93 @@ class ProxyLiteserver : public td::actor::Actor { } void receive_query(td::BufferSlice data, td::Promise promise) { + // Like in ValidatorManagerImpl::run_ext_query + auto F = fetch_tl_object(data, true); + if (F.is_ok()) { + data = std::move(F.move_as_ok()->data_); + } else { + auto G = fetch_tl_prefix(data, true); + if (G.is_error()) { + promise.set_error(G.move_as_error()); + return; + } + } + + tl_object_ptr wait_mc_seqno_obj; + auto E = fetch_tl_prefix(data, true); + if (E.is_ok()) { + wait_mc_seqno_obj = E.move_as_ok(); + } liteclient::QueryInfo query_info = liteclient::get_query_info(data); ++ls_stats_[query_info.query_id]; - promise = [promise = std::move(promise), query_info, timer = td::Timer()](td::Result R) mutable { + promise = [promise = std::move(promise), query_info, timer = td::Timer(), + wait_mc_seqno = + (wait_mc_seqno_obj ? wait_mc_seqno_obj->seqno_ : 0)](td::Result R) mutable { if (R.is_ok()) { - LOG(INFO) << "Query " << query_info.to_str() << ": OK, time=" << timer.elapsed() - << ", response_size=" << R.ok().size(); + LOG(INFO) << "Query " << query_info.to_str() + << (wait_mc_seqno ? PSTRING() << " (wait seqno " << wait_mc_seqno << ")" : "") + << ": OK, time=" << timer.elapsed() << ", response_size=" << R.ok().size(); promise.set_value(R.move_as_ok()); return; } - LOG(INFO) << "Query " << query_info.to_str() << ": " << R.error(); + LOG(INFO) << "Query " << query_info.to_str() + << (wait_mc_seqno ? PSTRING() << " (wait seqno " << wait_mc_seqno << ")" : "") << ": " << R.error(); promise.set_value(create_serialize_tl_object( R.error().code(), "Gateway error: " + R.error().message().str())); }; - TRY_RESULT_PROMISE(promise, server_idx, select_server(query_info)); + TRY_RESULT_PROMISE(promise, server_idx, select_server(query_info)); Server& server = servers_[server_idx]; - LOG(INFO) << "Sending query " << query_info.to_str() << ", size=" << data.size() << ", to server #" << server_idx - << " (" << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; + LOG(INFO) << "Sending query " << query_info.to_str() + << (wait_mc_seqno_obj ? PSTRING() << " (wait seqno " << wait_mc_seqno_obj->seqno_ << ")" : "") + << ", size=" << data.size() << ", to server #" << server_idx << " (" << server.config.addr.get_ip_str() + << ":" << server.config.addr.get_port() << ")"; + + BlockSeqno wait_mc_seqno = wait_mc_seqno_obj ? wait_mc_seqno_obj->seqno_ : 0; + wait_mc_seqno = std::max(wait_mc_seqno, last_known_masterchain_seqno_); + if (server.last_known_masterchain_seqno < wait_mc_seqno) { + int timeout_ms = wait_mc_seqno_obj ? wait_mc_seqno_obj->timeout_ms_ : 8000; + data = serialize_tl_object(create_tl_object(wait_mc_seqno, timeout_ms), + true, std::move(data)); + } + data = create_serialize_tl_object(std::move(data)); td::actor::send_closure(server.client, &adnl::AdnlExtClient::send_query, "q", std::move(data), - td::Timestamp::in(8.0), std::move(promise)); + td::Timestamp::in(8.0), + [SelfId = actor_id(this), promise = std::move(promise), server_idx, + wait_mc_seqno](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ProxyLiteserver::process_query_response, + R.ok().clone(), server_idx, wait_mc_seqno); + } + promise.set_result(std::move(R)); + }); + } + + void process_query_response(td::BufferSlice data, size_t server_idx, BlockSeqno wait_mc_seqno) { + auto F = fetch_tl_object(data, true); + if (F.is_error() || F.ok()->get_id() == lite_api::liteServer_error::ID) { + return; + } + BlockSeqno new_seqno = wait_mc_seqno; + lite_api::downcast_call(*F.ok_ref(), td::overloaded( + [&](lite_api::liteServer_masterchainInfo& f) { + new_seqno = std::max(new_seqno, f.last_->seqno_); + }, + [&](lite_api::liteServer_masterchainInfoExt& f) { + new_seqno = std::max(new_seqno, f.last_->seqno_); + }, + [&](lite_api::liteServer_accountState& f) { + if (f.id_->workchain_ == masterchainId) { + new_seqno = std::max(new_seqno, f.id_->seqno_); + } + }, + [&](auto& obj) {})); + servers_[server_idx].last_known_masterchain_seqno = + std::max(servers_[server_idx].last_known_masterchain_seqno, new_seqno); + if (new_seqno > last_known_masterchain_seqno_) { + last_known_masterchain_seqno_ = new_seqno; + LOG(INFO) << "Last known masterchain seqno = " << new_seqno; + } } void alarm() override { @@ -307,11 +378,15 @@ class ProxyLiteserver : public td::actor::Actor { liteclient::LiteServerConfig config; td::actor::ActorOwn client; bool alive = false; + BlockSeqno last_known_masterchain_seqno = 0; }; std::vector servers_; std::map ls_stats_; // lite_api ID -> count, 0 for unknown + BlockSeqno last_known_masterchain_seqno_ = 0; + tl_object_ptr last_masterchain_info_; + std::string config_file() const { return db_root_ + "/config.json"; } From 8abdbf422fc7cfb7d9c9e6ba62455156328b5e1c Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 19 Sep 2024 12:14:32 +0300 Subject: [PATCH 119/524] Update query-utils.cpp --- lite-client/query-utils.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lite-client/query-utils.cpp b/lite-client/query-utils.cpp index f48e3e8dd7..a3a663be05 100644 --- a/lite-client/query-utils.cpp +++ b/lite-client/query-utils.cpp @@ -216,6 +216,8 @@ QueryInfo get_query_info(const lite_api::Function& f) { /* t_simple */ }, [&](const lite_api::liteServer_getBlockOutMsgQueueSize& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getDispatchQueueInfo& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getDispatchQueueMessages& q) { from_block_id(q.id_); }, [&](const auto&) { /* t_simple */ })); if (info.shard_id.workchain == masterchainId) { info.shard_id.shard = shardIdAll; From 9dfa3fa11a24f072043ebff8ab000677a2dc2059 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 26 Sep 2024 19:24:03 +0300 Subject: [PATCH 120/524] Configure shards for custom overlays --- tl/generate/scheme/ton_api.tl | 3 +- tl/generate/scheme/ton_api.tlo | Bin 104992 -> 105052 bytes .../validator-engine-console-query.cpp | 6 ++ validator/full-node.cpp | 70 ++++++++++-------- validator/full-node.h | 2 + 5 files changed, 48 insertions(+), 33 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 09363b55a5..aeb70ca4e7 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -660,7 +660,8 @@ engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector eng gc:engine.gc = engine.validator.Config; engine.validator.customOverlayNode adnl_id:int256 msg_sender:Bool msg_sender_priority:int block_sender:Bool = engine.validator.CustomOverlayNode; -engine.validator.customOverlay name:string nodes:(vector engine.validator.customOverlayNode) = engine.validator.CustomOverlay; +engine.validator.customOverlay name:string nodes:(vector engine.validator.customOverlayNode) sender_shards:(vector tonNode.shardId) + = engine.validator.CustomOverlay; engine.validator.customOverlaysConfig overlays:(vector engine.validator.customOverlay) = engine.validator.CustomOverlaysConfig; engine.validator.collatorOptions diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 1132032de0889fc8729f9ac6d49176038d0d3c8a..d2216fbcb73b3697993b70c607ebff9df7003c1f 100644 GIT binary patch delta 108 zcmZ3mh3(E3whap6tY>RdmdH&`6qnoVEbcm;1;m})vrd5%#JG7($cK6QLTN_tN$Yt8 zrJvo`AtWwhap6tg77i*UC*!6qnoVEbcm;1;m})vrYlRICD&Q^4)d%JVnlIIvlob h2SKu%L)M2df~7aF*>GnWm^ED>k5O^^+Hl78P5??=BoF`q diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 761de84029..e81e849398 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1229,6 +1229,12 @@ td::Status ShowCustomOverlaysQuery::receive(td::BufferSlice data) { : "") << (node->block_sender_ ? " (block sender)" : "") << "\n"; } + if (!overlay->sender_shards_.empty()) { + td::TerminalIO::out() << "Sender shards:\n"; + for (const auto &shard : overlay->sender_shards_) { + td::TerminalIO::out() << " " << ton::create_shard_id(shard).to_str() << "\n"; + } + } td::TerminalIO::out() << "\n"; } return td::Status::OK(); diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 651c014945..2cd578ab6e 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -21,6 +21,7 @@ #include "td/actor/MultiPromise.h" #include "full-node.h" #include "common/delay.h" +#include "ton/ton-tl.hpp" namespace ton { @@ -209,35 +210,28 @@ void FullNodeImpl::on_new_masterchain_block(td::Ref state, std int min_split = state->monitor_min_split_depth(shard.workchain); return min_split < shard.pfx_len() ? shard_prefix(shard, min_split) : shard; }; - auto set_active = [&](ShardIdFull shard) { - while (new_active.emplace(shard).second && shard.pfx_len() > 0) { - shard = shard_parent(shard); - } - }; for (auto &info : state->get_shards()) { workchains.insert(info->shard().workchain); all_shards.insert(cut_shard(info->shard())); } - for (const auto &wpair : state->get_workchain_list()) { - ton::WorkchainId wc = wpair.first; - const block::WorkchainInfo *winfo = wpair.second.get(); - if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= state->get_unix_time()) { + for (const auto &[wc, winfo] : state->get_workchain_list()) { + if (!workchains.contains(wc) && winfo->active && winfo->enabled_since <= state->get_unix_time()) { all_shards.insert(ShardIdFull(wc)); } } for (ShardIdFull shard : shards_to_monitor) { - set_active(cut_shard(shard)); + new_active.insert(cut_shard(shard)); } for (auto it = shards_.begin(); it != shards_.end(); ) { - if (all_shards.count(it->first)) { + if (all_shards.contains(it->first)) { ++it; } else { it = shards_.erase(it); } } for (ShardIdFull shard : all_shards) { - bool active = new_active.count(shard); + bool active = new_active.contains(shard); bool overlay_exists = !shards_[shard].actor.empty(); if (active || join_all_overlays || overlay_exists) { update_shard_actor(shard, active); @@ -254,8 +248,8 @@ void FullNodeImpl::on_new_masterchain_block(td::Ref state, std if (!use_old_private_overlays_) { std::set my_adnl_ids; my_adnl_ids.insert(adnl_id_); - for (const auto &p : local_collator_nodes_) { - my_adnl_ids.insert(p.first); + for (const auto &[adnl_id, _] : local_collator_nodes_) { + my_adnl_ids.insert(adnl_id); } for (auto key : local_keys_) { auto it = current_validators_.find(key); @@ -307,11 +301,12 @@ void FullNodeImpl::send_ext_message(AccountIdPrefixFull dst, td::BufferSlice dat VLOG(FULL_NODE_WARNING) << "dropping OUT ext message to unknown shard"; return; } - for (auto &private_overlay : custom_overlays_) { - for (auto &actor : private_overlay.second.actors_) { - auto local_id = actor.first; - if (private_overlay.second.params_.msg_senders_.count(local_id)) { - td::actor::send_closure(actor.second, &FullNodeCustomOverlay::send_external_message, data.clone()); + for (auto &[_, private_overlay] : custom_overlays_) { + if (private_overlay.params_.send_shard(dst.as_leaf_shard())) { + for (auto &[local_id, actor] : private_overlay.actors_) { + if (private_overlay.params_.msg_senders_.contains(local_id)) { + td::actor::send_closure(actor, &FullNodeCustomOverlay::send_external_message, data.clone()); + } } } } @@ -743,7 +738,7 @@ void FullNodeImpl::update_custom_overlay(CustomOverlayInfo &overlay) { } } -void FullNodeImpl::send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast) { +void FullNodeImpl::send_block_broadcast_to_custom_overlays(const BlockBroadcast &broadcast) { if (!custom_overlays_sent_broadcasts_.insert(broadcast.block_id).second) { return; } @@ -752,11 +747,12 @@ void FullNodeImpl::send_block_broadcast_to_custom_overlays(const BlockBroadcast& custom_overlays_sent_broadcasts_.erase(custom_overlays_sent_broadcasts_lru_.front()); custom_overlays_sent_broadcasts_lru_.pop(); } - for (auto &private_overlay : custom_overlays_) { - for (auto &actor : private_overlay.second.actors_) { - auto local_id = actor.first; - if (private_overlay.second.params_.block_senders_.count(local_id)) { - td::actor::send_closure(actor.second, &FullNodeCustomOverlay::send_broadcast, broadcast.clone()); + for (auto &[_, private_overlay] : custom_overlays_) { + if (private_overlay.params_.send_shard(broadcast.block_id.shard_full())) { + for (auto &[local_id, actor] : private_overlay.actors_) { + if (private_overlay.params_.block_senders_.contains(local_id)) { + td::actor::send_closure(actor, &FullNodeCustomOverlay::send_broadcast, broadcast.clone()); + } } } } @@ -774,12 +770,13 @@ void FullNodeImpl::send_block_candidate_broadcast_to_custom_overlays(const Block custom_overlays_sent_broadcasts_.erase(custom_overlays_sent_broadcasts_lru_.front()); custom_overlays_sent_broadcasts_lru_.pop(); } - for (auto &private_overlay : custom_overlays_) { - for (auto &actor : private_overlay.second.actors_) { - auto local_id = actor.first; - if (private_overlay.second.params_.block_senders_.count(local_id)) { - td::actor::send_closure(actor.second, &FullNodeCustomOverlay::send_block_candidate, block_id, cc_seqno, - validator_set_hash, data.clone()); + for (auto &[_, private_overlay] : custom_overlays_) { + if (private_overlay.params_.send_shard(block_id.shard_full())) { + for (auto &[local_id, actor] : private_overlay.actors_) { + if (private_overlay.params_.block_senders_.contains(local_id)) { + td::actor::send_closure(actor, &FullNodeCustomOverlay::send_block_candidate, block_id, cc_seqno, + validator_set_hash, data.clone()); + } } } } @@ -834,18 +831,27 @@ bool FullNodeConfig::operator!=(const FullNodeConfig &rhs) const { return !(*this == rhs); } +bool CustomOverlayParams::send_shard(const ShardIdFull &shard) const { + return sender_shards_.empty() || std::ranges::any_of(sender_shards_, [&](const ShardIdFull &our_shard) { + return shard_intersects(shard, our_shard); + }); +} + CustomOverlayParams CustomOverlayParams::fetch(const ton_api::engine_validator_customOverlay& f) { CustomOverlayParams c; c.name_ = f.name_; for (const auto &node : f.nodes_) { c.nodes_.emplace_back(node->adnl_id_); if (node->msg_sender_) { - c.msg_senders_[ton::adnl::AdnlNodeIdShort{node->adnl_id_}] = node->msg_sender_priority_; + c.msg_senders_[adnl::AdnlNodeIdShort{node->adnl_id_}] = node->msg_sender_priority_; } if (node->block_sender_) { c.block_senders_.emplace(node->adnl_id_); } } + for (const auto &shard : f.sender_shards_) { + c.sender_shards_.push_back(create_shard_id(shard)); + } return c; } diff --git a/validator/full-node.h b/validator/full-node.h index fa57e4f1c2..8b47b94791 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -60,7 +60,9 @@ struct CustomOverlayParams { std::vector nodes_; std::map msg_senders_; std::set block_senders_; + std::vector sender_shards_; + bool send_shard(const ShardIdFull& shard) const; static CustomOverlayParams fetch(const ton_api::engine_validator_customOverlay& f); }; From 7d2110c8b0da73d25a77ffc34c30250406fb2825 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 18 Nov 2024 17:14:13 +0300 Subject: [PATCH 121/524] Fix shard overlays --- validator/full-node.cpp | 54 +++++++++++++++++++++++++++-------------- validator/full-node.hpp | 1 + 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 0cd769e4d1..bb806e281e 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -206,13 +206,20 @@ void FullNodeImpl::on_new_masterchain_block(td::Ref state, std std::set new_active; all_shards.insert(ShardIdFull(masterchainId)); std::set workchains; + wc_monitor_min_split_ = state->monitor_min_split_depth(basechainId); auto cut_shard = [&](ShardIdFull shard) -> ShardIdFull { - int min_split = state->monitor_min_split_depth(shard.workchain); - return min_split < shard.pfx_len() ? shard_prefix(shard, min_split) : shard; + return wc_monitor_min_split_ < shard.pfx_len() ? shard_prefix(shard, wc_monitor_min_split_) : shard; }; for (auto &info : state->get_shards()) { workchains.insert(info->shard().workchain); - all_shards.insert(cut_shard(info->shard())); + ShardIdFull shard = cut_shard(info->shard()); + while (true) { + all_shards.insert(shard); + if (shard.pfx_len() == 0) { + break; + } + shard = shard_parent(shard); + } } for (const auto &[wc, winfo] : state->get_workchain_list()) { if (!workchains.contains(wc) && winfo->active && winfo->enabled_since <= state->get_unix_time()) { @@ -220,7 +227,14 @@ void FullNodeImpl::on_new_masterchain_block(td::Ref state, std } } for (ShardIdFull shard : shards_to_monitor) { - new_active.insert(cut_shard(shard)); + shard = cut_shard(shard); + while (true) { + new_active.insert(shard); + if (shard.pfx_len() == 0) { + break; + } + shard = shard_parent(shard); + } } for (auto it = shards_.begin(); it != shards_.end(); ) { @@ -478,21 +492,25 @@ void FullNodeImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vect } td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { - while (true) { - auto it = shards_.find(shard); - if (it != shards_.end()) { - if (it->second.actor.empty()) { - update_shard_actor(shard, false); - } - return it->second.actor.get(); - } - if (shard.pfx_len() == 0) { - break; - } - shard = shard_parent(shard); + if (shard.is_masterchain()) { + return shards_[ShardIdFull{masterchainId}].actor.get(); + } + if (shard.workchain != basechainId) { + return {}; + } + int pfx_len = shard.pfx_len(); + if (pfx_len > wc_monitor_min_split_) { + shard = shard_prefix(shard, wc_monitor_min_split_); } - update_shard_actor(shard, false); - return shards_[shard].actor.get(); + auto it = shards_.find(shard); + if (it != shards_.end()) { + update_shard_actor(shard, it->second.active); + return it->second.actor.get(); + } + + // Special case if shards_ was not yet initialized. + // This can happen briefly on node startup. + return shards_[ShardIdFull{masterchainId}].actor.get(); } td::actor::ActorId FullNodeImpl::get_shard(AccountIdPrefixFull dst) { diff --git a/validator/full-node.hpp b/validator/full-node.hpp index f6047ed5ff..d2de18201c 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -124,6 +124,7 @@ class FullNodeImpl : public FullNode { td::actor::ActorId get_shard(AccountIdPrefixFull dst); td::actor::ActorId get_shard(ShardIdFull shard); std::map shards_; + int wc_monitor_min_split_ = 0; td::actor::ActorId keyring_; td::actor::ActorId adnl_; From b3bea413e3e0e6db182a633f7e7cce6345c46d83 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 21 Nov 2024 11:47:39 +0300 Subject: [PATCH 122/524] Rework validator-collator interaction 1) Remove config 41, move "full collated data" to capabilities 2) Whitelist on collator nodes 3) "Ping" request for collator nodes 4) More customizable collators list for validators 5) CollationManager --- crypto/block/block.tlb | 5 - crypto/block/mc-config.cpp | 24 -- crypto/block/mc-config.h | 6 - tl/generate/scheme/ton_api.tl | 28 +- tl/generate/scheme/ton_api.tlo | Bin 105392 -> 107428 bytes ton/ton-types.h | 3 +- .../validator-engine-console-query.cpp | 144 ++++++- .../validator-engine-console-query.h | 104 +++++ .../validator-engine-console.cpp | 5 + validator-engine/validator-engine.cpp | 252 +++++++++--- validator-engine/validator-engine.hpp | 27 +- validator/CMakeLists.txt | 2 + validator/collation-manager.cpp | 373 ++++++++++++++++++ validator/collation-manager.hpp | 87 ++++ validator/collator-node.cpp | 56 ++- validator/collator-node.hpp | 1 + validator/full-node.cpp | 14 +- validator/impl/collator-impl.h | 2 +- validator/impl/collator.cpp | 7 +- validator/impl/shard.hpp | 3 - validator/impl/validate-query.cpp | 9 + validator/impl/validate-query.hpp | 2 +- validator/impl/validator-set.hpp | 3 +- validator/interfaces/shard.h | 1 - validator/interfaces/validator-set.h | 1 + validator/manager-disk.hpp | 5 + validator/manager-hardfork.hpp | 5 + validator/manager.cpp | 77 +++- validator/manager.hpp | 6 + validator/validator-group.cpp | 158 +------- validator/validator-group.hpp | 22 +- validator/validator-options.cpp | 53 ++- validator/validator-options.hpp | 17 +- validator/validator.h | 21 +- 34 files changed, 1204 insertions(+), 319 deletions(-) create mode 100644 validator/collation-manager.cpp create mode 100644 validator/collation-manager.hpp diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 5eddbde38e..aa7c0d3f95 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -801,11 +801,6 @@ misbehaviour_punishment_config_v1#01 = MisbehaviourPunishmentConfig; _ MisbehaviourPunishmentConfig = ConfigParam 40; -// collator_nodes: each collator is (workchain:int32 shard:uint64 adnl_id:uint256) -collator_info#00 = CollatorInfo; -colator_config#a0 full_collated_data:Bool collator_nodes:(HashmapE 352 CollatorInfo) = CollatorConfig; -_ CollatorConfig = ConfigParam 41; - size_limits_config#01 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 max_ext_msg_size:uint32 max_ext_msg_depth:uint16 = SizeLimitsConfig; size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 0a8c0c7b1d..56ee85ae3b 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2339,28 +2339,4 @@ td::optional PrecompiledContractsConfig::g return c; } -CollatorConfig Config::get_collator_config(bool need_collator_nodes) const { - CollatorConfig collator_config; - gen::CollatorConfig::Record rec; - auto cell = get_config_param(41, -41); - if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { - return collator_config; - } - collator_config.full_collated_data = rec.full_collated_data; - if (need_collator_nodes) { - vm::Dictionary dict{rec.collator_nodes->prefetch_ref(), 32 + 64 + 256}; - dict.check_for_each([&](Ref value, td::ConstBitPtr key, int n) { - CHECK(n == 32 + 64 + 256); - auto workchain = (td::int32)key.get_int(32); - key.advance(32); - td::uint64 shard = key.get_uint(64); - key.advance(64); - td::Bits256 adnl_id(key); - collator_config.collator_nodes.push_back({ton::ShardIdFull(workchain, shard), adnl_id}); - return true; - }); - } - return collator_config; -} - } // namespace block diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 94de329bfe..3c4421da50 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -543,11 +543,6 @@ struct CollatorNodeDescr { ton::NodeIdShort adnl_id; }; -struct CollatorConfig { - bool full_collated_data = false; - std::vector collator_nodes; -}; - class Config { enum { default_mc_catchain_lifetime = 200, @@ -664,7 +659,6 @@ class Config { std::vector compute_validator_set(ton::ShardIdFull shard, ton::UnixTime time, ton::CatchainSeqno cc_seqno) const; std::vector compute_total_validator_set(int next) const; - CollatorConfig get_collator_config(bool need_collator_nodes) const; td::Result get_size_limits_config() const; static td::Result do_get_size_limits_config(td::Ref cs); std::unique_ptr get_suspended_addresses(ton::UnixTime now) const; diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 05ea249ce2..0839f727d0 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -646,8 +646,9 @@ engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNode engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave; engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig; engine.validator.fastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.FastSyncMemberCertificate; +engine.validator.collatorNodeWhitelist enabled:Bool adnl_ids:(vector int256) = engine.validator.CollatorNodeWhitelist; engine.validator.extraConfig state_serializer_enabled:Bool fast_sync_member_certificates:(vector engine.validator.fastSyncMemberCertificate) - = engine.validator.ExtraConfig; + collator_node_whitelist:engine.validator.collatorNodeWhitelist = engine.validator.ExtraConfig; engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) validators:(vector engine.validator) collators:(vector engine.collator) @@ -670,11 +671,10 @@ engine.validator.collatorOptions dispatch_phase_2_max_per_initiator:int dispatch_phase_3_max_per_initiator:int whitelist:(vector string) prioritylist:(vector string) = engine.validator.CollatorOptions; -engine.validator.collatorsList.collator adnl_id:int256 trusted:Bool = engine.validator.collatorsList.Collator; +engine.validator.collatorsList.collator adnl_id:int256 = engine.validator.collatorsList.Collator; engine.validator.collatorsList.shard shard_id:tonNode.shardId collators:(vector engine.validator.collatorsList.collator) - = engine.validator.collatorsList.Shard; -engine.validator.collatorsList self_collate:Bool use_config_41:Bool shards:(vector engine.validator.collatorsList.shard) - = engine.validator.CollatorsList; + self_collate:Bool select_mode:string = engine.validator.collatorsList.Shard; +engine.validator.collatorsList shards:(vector engine.validator.collatorsList.shard) = engine.validator.CollatorsList; ---functions--- ---types--- @@ -740,6 +740,11 @@ engine.validator.perfTimerStats stats:(vector engine.validator.PerfTimerStatsByN engine.validator.shardOutQueueSize size:long = engine.validator.ShardOutQueueSize; +engine.validator.collationManagerStats.shard shard_id:tonNode.shardId self_collate:Bool select_mode:string active:Bool collators:(vector int256) = engine.validator.collationManagerStats.Shard; +engine.validator.collationManagerStats.collator adnl_id:int256 active:Bool alive:Bool ping_in:double = engine.validator.collationManagerStats.Collator; +engine.validator.collationManagerStats.localId adnl_id:int256 shards:(vector engine.validator.collationManagerStats.shard) + collators:(vector engine.validator.collationManagerStats.collator) = engine.validator.collationManagerStats.LocalId; +engine.validator.collationManagerStats local_ids:(vector engine.validator.collationManagerStats.localId) = engine.validator.CollationManagerStats; ---functions--- @@ -811,8 +816,14 @@ engine.validator.addShard shard:tonNode.shardId = engine.validator.Success; engine.validator.delCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; engine.validator.delShard shard:tonNode.shardId = engine.validator.Success; +engine.validator.collatorNodeSetWhitelistedValidator adnl_id:int256 add:Bool = engine.validator.Success; +engine.validator.collatorNodeSetWhitelistEnabled enabled:Bool = engine.validator.Success; +engine.validator.showCollatorNodeWhitelist = engine.validator.CollatorNodeWhitelist; + engine.validator.setCollatorsList list:engine.validator.collatorsList = engine.validator.Success; +engine.validator.clearCollatorsList = engine.validator.Success; engine.validator.showCollatorsList = engine.validator.CollatorsList; +engine.validator.getCollationManagerStats = engine.validator.CollationManagerStats; engine.validator.signOverlayMemberCertificate sign_by:int256 adnl_id:int256 slot:int expire_at:int = overlay.MemberCertificate; engine.validator.importFastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.Success; @@ -887,8 +898,8 @@ validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:double self: collatorNode.candidate source:PublicKey id:tonNode.blockIdExt data:bytes collated_data:bytes = collatorNode.Candidate; collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt decompressed_size:int data:bytes = collatorNode.Candidate; -collatorNode.generateBlockSuccess candidate:collatorNode.Candidate = collatorNode.GenerateBlockResult; -collatorNode.generateBlockError code:int message:string = collatorNode.GenerateBlockResult; +collatorNode.pong flags:# = collatorNode.Pong; +collatorNode.error code:int message:string = collatorNode.Error; validatorSession.newValidatorGroupStats.node id:int256 weight:long = validatorSession.newValidatorGroupStats.Node; validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int @@ -901,7 +912,8 @@ validatorSession.endValidatorGroupStats session_id:int256 timestamp:double ---functions--- collatorNode.generateBlock shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt) - creator:int256 = collatorNode.GenerateBlockResult; + creator:int256 = collatorNode.Candidate; +collatorNode.ping flags:# = collatorNode.Pong; ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index eaa9c416156da6d95b8fbd44bfa6b2ef83d1d1d9..dd2af2ec9de6dbde7d6644ac55b85bf42bba5df1 100644 GIT binary patch delta 1507 zcmZuxVQ3R)7``vw#bj+++oj0`uj!T6HLYDz*ABw+oNe2jUuq( z3AJ=G=ig~@??MyZVV3-bMrQ9RihU6lgDyiS+AO{1%-OT#^(1Qr3FpmcV9y(gN^s@OqdzWGQ9>ceBSc`<%|K!#4vEhwQ zKP$1!t=PyHF+?@EP;kV%oe_Tg;b}kaw7GV4!8$*ASkOdRf<|cQDgF2Ua-Kidz*l#fP)$;~A^G!6p}&GXg8a%F%;QtB%s%4AO^JPb~Y zKmMF?743qHItgNP!vI>QF14iN#;%mbL zZu%S{b=H?qGAThB6)8kiFO{3=fgAJPG)fF!T^Q=cWcA%6++D!Y9FLaR$ZnEbLz4VPqj{Otq6h8{*v+PS)h?xD_Lqkw9*y3uhg` zi2DrA7cNKmMyCl3JiRgih1IAZ(e}OI%d4}Wk}u2BIO!qulUM${ zV-)ujjvi~c^tDmiPZpq}HFsV-`tW_HDf@xJdHk<2%2N|$yeYbG>5mItlA4xPDK>3Q zQpBezmQo}^tJyz4X6Xm!e(h%79=-k7faT+GE9Rb*bVvPrRD;hF)+lAk5HspsYa2qC z?GQIi5)WVvW99(aMQ;&A@hczUjN(DkTrMTaT|cw_#>Bl|4;>!Lvn>{3-u(IT~Ngka1;e^6u z69omE%63}lq{@!>u-6XxO~XEq%U+qQ!sB`~q8a$GU9o-Y0^kK1V1qN4mFkeNIp^WN`EVnIF`ld^SGs()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status CollatorNodeAddWhitelistedValidatorQuery::send() { + auto b = ton::create_serialize_tl_object( + adnl_id_.bits256_value(), true); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status CollatorNodeAddWhitelistedValidatorQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status CollatorNodeDelWhitelistedValidatorQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status CollatorNodeDelWhitelistedValidatorQuery::send() { + auto b = ton::create_serialize_tl_object( + adnl_id_.bits256_value(), false); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status CollatorNodeDelWhitelistedValidatorQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status CollatorNodeEnableWhitelistQuery::run() { + TRY_RESULT(value, tokenizer_.get_token()); + if (value != 0 && value != 1) { + return td::Status::Error("expected 0 or 1"); + } + TRY_STATUS(tokenizer_.check_endl()); + enabled_ = value; + return td::Status::OK(); +} + +td::Status CollatorNodeEnableWhitelistQuery::send() { + auto b = ton::create_serialize_tl_object(enabled_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status CollatorNodeEnableWhitelistQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status CollatorNodeShowWhitelistQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status CollatorNodeShowWhitelistQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status CollatorNodeShowWhitelistQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, + ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "Collator node whitelist: " << (f->enabled_ ? "ENABLED" : "DISABLED") << "\n"; + td::TerminalIO::out() << f->adnl_ids_.size() << " validator adnl ids\n"; + for (const auto &id : f->adnl_ids_) { + td::TerminalIO::out() << id.to_hex() << "\n"; + } + return td::Status::OK(); +} + td::Status SetCollatorsListQuery::run() { TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); TRY_STATUS(tokenizer_.check_endl()); @@ -1611,9 +1697,7 @@ td::Status ClearCollatorsListQuery::run() { } td::Status ClearCollatorsListQuery::send() { - auto list = ton::create_tl_object(); - list->self_collate_ = true; - auto b = ton::create_serialize_tl_object(std::move(list)); + auto b = ton::create_serialize_tl_object(); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1640,20 +1724,60 @@ td::Status ShowCollatorsListQuery::receive(td::BufferSlice data) { TRY_RESULT_PREFIX(list, ton::fetch_tl_object(data.as_slice(), true), "received incorrect answer: "); td::TerminalIO::out() << "Collators list:\n"; - if (list->self_collate_) { - td::TerminalIO::out() << "self_collate = true\n"; - } - if (list->use_config_41_) { - td::TerminalIO::out() << "use_config_41 = true\n"; - } if (list->shards_.empty()) { td::TerminalIO::out() << "Shard list is empty\n"; return td::Status::OK(); } for (const auto &shard : list->shards_) { td::TerminalIO::out() << "Shard " << create_shard_id(shard->shard_id_).to_str() << "\n"; + td::TerminalIO::out() << " Self collate = " << shard->self_collate_ << "\n"; + td::TerminalIO::out() << " Select mode = " << shard->select_mode_ << "\n"; for (const auto &collator : shard->collators_) { - td::TerminalIO::out() << " Collator " << collator->adnl_id_ << (collator->trusted_ ? " (trusted)" : "") << "\n"; + td::TerminalIO::out() << " Collator " << collator->adnl_id_ << "\n"; + } + } + return td::Status::OK(); +} + +td::Status GetCollationManagerStatsQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status GetCollationManagerStatsQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetCollationManagerStatsQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(list, + ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + if (list->local_ids_.empty()) { + td::TerminalIO::out() << "No stats\n"; + return td::Status::OK();; + } + for (auto &stats : list->local_ids_) { + td::TerminalIO::out() << "VALIDATOR ADNL ID = " << stats->adnl_id_ << "\n"; + std::map collators; + for (auto &collator: stats->collators_) { + collators[collator->adnl_id_] = collator.get(); + } + for (auto &shard : stats->shards_) { + td::TerminalIO::out() << " Shard " << create_shard_id(shard->shard_id_).to_str() << "\n"; + td::TerminalIO::out() << " Self collate = " << shard->self_collate_ << "\n"; + td::TerminalIO::out() << " Select mode = " << shard->select_mode_ << "\n"; + td::TerminalIO::out() << " Active = " << shard->active_ << "\n"; + td::TerminalIO::out() << " Collators: " << shard->collators_.size() << "\n"; + for (auto &id : shard->collators_) { + auto collator = collators[id]; + if (collator == nullptr) { + return td::Status::Error("collator not found"); + } + td::TerminalIO::out() << " " << id << " alive=" << (int)collator->alive_ + << " ping_in=" << collator->ping_in_ << "\n"; + } } } return td::Status::OK(); diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 5708fecda2..2d59c29b71 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1457,6 +1457,91 @@ class DelShardQuery : public Query { td::int64 shard_; }; +class CollatorNodeAddWhitelistedValidatorQuery : public Query { + public: + CollatorNodeAddWhitelistedValidatorQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "collatorwhitelistadd"; + } + static std::string get_help() { + return "collatorwhitelistadd \tadd validator adnl id to collator node whitelist"; + } + std::string name() const override { + return get_name(); + } + + private: + ton::PublicKeyHash adnl_id_; +}; + +class CollatorNodeDelWhitelistedValidatorQuery : public Query { + public: + CollatorNodeDelWhitelistedValidatorQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "collatorwhitelistdel"; + } + static std::string get_help() { + return "collatorwhitelistdel \tremove validator adnl id from collator node whitelist"; + } + std::string name() const override { + return get_name(); + } + + private: + ton::PublicKeyHash adnl_id_; +}; + +class CollatorNodeEnableWhitelistQuery : public Query { + public: + CollatorNodeEnableWhitelistQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "collatorwhitelistenable"; + } + static std::string get_help() { + return "collatorwhitelistenable \tenable or disable collator node whiltelist (value is 0 or 1)"; + } + std::string name() const override { + return get_name(); + } + + private: + bool enabled_; +}; + +class CollatorNodeShowWhitelistQuery : public Query { + public: + CollatorNodeShowWhitelistQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "collatorwhitelistshow"; + } + static std::string get_help() { + return "collatorwhitelistshow\tshow collator node whitelist"; + } + std::string name() const override { + return get_name(); + } +}; + class SetCollatorsListQuery : public Query { public: SetCollatorsListQuery(td::actor::ActorId console, Tokenizer tokenizer) @@ -1517,6 +1602,25 @@ class ShowCollatorsListQuery : public Query { } }; +class GetCollationManagerStatsQuery : public Query { + public: + GetCollationManagerStatsQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "collationmanagerstats"; + } + static std::string get_help() { + return "collationmanagerstats\tshow stats of collation manager"; + } + std::string name() const override { + return get_name(); + } +}; + class SignOverlayMemberCertificateQuery : public Query { public: SignOverlayMemberCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 4c4922494d..67e932115c 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -157,9 +157,14 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); } diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 3722dcd282..8d2ce0bef9 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -75,6 +75,7 @@ #include "common/delay.h" #include "block/precompiled-smc/PrecompiledSmartContract.h" #include "interfaces/validator-manager.h" +#include "tl-utils/lite-utils.hpp" #if TON_USE_JEMALLOC #include @@ -147,9 +148,9 @@ Config::Config(const ton::ton_api::engine_validator_config &config) { } } for (auto &col : config.collators_) { - auto key = ton::PublicKeyHash{col->adnl_id_}; + auto id = ton::adnl::AdnlNodeIdShort{col->adnl_id_}; ton::ShardIdFull shard = ton::create_shard_id(col->shard_); - config_add_collator(key, shard).ensure(); + config_add_collator(id, shard).ensure(); } config_add_full_node_adnl_id(ton::PublicKeyHash{config.fullnode_}).ensure(); @@ -175,6 +176,12 @@ Config::Config(const ton::ton_api::engine_validator_config &config) { fast_sync_member_certificates.emplace_back(adnl_id, std::move(certificate)); } } + if (config.extraconfig_->collator_node_whitelist_) { + collator_node_whiltelist_enabled = config.extraconfig_->collator_node_whitelist_->enabled_; + for (const auto& id : config.extraconfig_->collator_node_whitelist_->adnl_ids_) { + collator_node_whitelist.emplace(id); + } + } } else { state_serializer_enabled = true; } @@ -242,9 +249,11 @@ ton::tl_object_ptr Config::tl() const { val.first.tl(), std::move(temp_vec), std::move(adnl_val_vec), val.second.election_date, val.second.expire_at)); } std::vector> col_vec; - for (auto &col : collators) { - col_vec.push_back( - ton::create_tl_object(col.adnl_id.tl(), ton::create_tl_shard_id(col.shard))); + for (auto &[col, shards] : collators) { + for (auto &shard : shards) { + col_vec.push_back( + ton::create_tl_object(col.bits256_value(), ton::create_tl_shard_id(shard))); + } } std::vector> full_node_slaves_vec; @@ -263,8 +272,17 @@ ton::tl_object_ptr Config::tl() const { full_node_config_obj = full_node_config.tl(); } + ton::tl_object_ptr collator_node_whitelist_obj = {}; + if (collator_node_whiltelist_enabled || !collator_node_whitelist.empty()) { + collator_node_whitelist_obj = ton::create_tl_object(); + collator_node_whitelist_obj->enabled_ = collator_node_whiltelist_enabled; + for (const auto& id : collator_node_whitelist) { + collator_node_whitelist_obj->adnl_ids_.push_back(id.bits256_value()); + } + } + ton::tl_object_ptr extra_config_obj = {}; - if (!state_serializer_enabled || !fast_sync_member_certificates.empty()) { + if (!state_serializer_enabled || !fast_sync_member_certificates.empty() || collator_node_whitelist_obj) { // Non-default values extra_config_obj = ton::create_tl_object(); extra_config_obj->state_serializer_enabled_ = state_serializer_enabled; @@ -273,6 +291,7 @@ ton::tl_object_ptr Config::tl() const { ton::create_tl_object(adnl_id.bits256_value(), certificate.tl())); } + extra_config_obj->collator_node_whitelist_ = std::move(collator_node_whitelist_obj); } std::vector> liteserver_vec; @@ -448,28 +467,28 @@ td::Result Config::config_add_validator_adnl_id(ton::PublicKeyHash perm_ke } } -td::Result Config::config_add_collator(ton::PublicKeyHash addr, ton::ShardIdFull shard) { +td::Result Config::config_add_collator(ton::adnl::AdnlNodeIdShort addr, ton::ShardIdFull shard) { if (!shard.is_valid_ext()) { return td::Status::Error(PSTRING() << "invalid shard: " << shard.to_str()); } - Collator c{addr, shard}; - if (std::find(collators.begin(), collators.end(), c) != collators.end()) { + auto& shards = collators[addr]; + if (std::ranges::find(shards, shard) != collators[addr].end()) { return false; } - collators.push_back(c); + shards.push_back(shard); return true; } -td::Result Config::config_del_collator(ton::PublicKeyHash addr, ton::ShardIdFull shard) { +td::Result Config::config_del_collator(ton::adnl::AdnlNodeIdShort addr, ton::ShardIdFull shard) { if (!shard.is_valid_ext()) { return td::Status::Error(PSTRING() << "invalid shard: " << shard.to_str()); } - Collator c{addr, shard}; - auto it = std::find(collators.begin(), collators.end(), c); - if (it == collators.end()) { + auto& shards = collators[addr]; + auto it = std::ranges::find(shards, shard); + if (it == shards.end()) { return false; } - collators.erase(it); + shards.erase(it); return true; } @@ -1564,6 +1583,11 @@ td::Status ValidatorEngine::load_global_config() { } validator_options_.write().set_fast_state_serializer_enabled(fast_state_serializer_enabled_); + for (auto& id : config_.collator_node_whitelist) { + validator_options_.write().set_collator_node_whitelisted_validator(id, true); + } + validator_options_.write().set_collator_node_whitelist_enabled(config_.collator_node_whiltelist_enabled); + return td::Status::OK(); } @@ -1572,14 +1596,16 @@ void ValidatorEngine::set_shard_check_function() { validator_options_.write().set_shard_check_function([](ton::ShardIdFull shard) -> bool { return true; }); } else { std::vector shards = {ton::ShardIdFull(ton::masterchainId)}; - for (const auto& c : config_.collators) { - shards.push_back(c.shard); + for (const auto& [_, collator_shards] : config_.collators) { + for (const auto& shard : collator_shards) { + shards.push_back(shard); + } } for (const auto& s : config_.shards_to_monitor) { shards.push_back(s); } std::sort(shards.begin(), shards.end()); - shards.erase(std::unique(shards.begin(), shards.end()), shards.end()); + shards.erase(std::ranges::unique(shards).begin(), shards.end()); validator_options_.write().set_shard_check_function( [shards = std::move(shards)](ton::ShardIdFull shard) -> bool { for (auto s : shards) { @@ -1613,8 +1639,13 @@ void ValidatorEngine::load_collators_list() { return; } td::Ref list{true}; - list.write().unpack(*collators_list_); - validator_options_.write().set_collators_list(std::move(list)); + S = list.write().unpack(*collators_list_); + if (S.is_ok()) { + validator_options_.write().set_collators_list(std::move(list)); + } else { + LOG(ERROR) << "Invalid collators list: " << S.move_as_error(); + collators_list_ = {}; + } } void ValidatorEngine::load_empty_local_config(td::Promise promise) { @@ -2100,9 +2131,8 @@ void ValidatorEngine::start_full_node() { td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_permanent_key, v.first, [](td::Unit) {}); } - for (auto &c : config_.collators) { - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_collator_adnl_id, - ton::adnl::AdnlNodeIdShort(c.adnl_id)); + for (auto &[c, _] : config_.collators) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_collator_adnl_id, c); } for (auto &x : config_.fast_sync_member_certificates) { td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, @@ -2139,9 +2169,10 @@ void ValidatorEngine::started_lite_server() { } void ValidatorEngine::start_collator() { - for (auto &c : config_.collators) { - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, - ton::adnl::AdnlNodeIdShort(c.adnl_id), c.shard); + for (auto& [id, shards] : config_.collators) { + for (auto& shard : shards) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, id, shard); + } } started_collator(); @@ -4054,6 +4085,92 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setCollat promise.set_value(ton::create_serialize_tl_object()); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_collatorNodeSetWhitelistedValidator &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; + if (query.add_) { + if (!config_.collator_node_whitelist.insert(adnl_id).second) { + promise.set_value(ton::create_serialize_tl_object()); + return; + } + } else { + if (config_.collator_node_whitelist.erase(adnl_id) == 0) { + promise.set_value(ton::create_serialize_tl_object()); + return; + } + } + + validator_options_.write().set_collator_node_whitelisted_validator(adnl_id, query.add_); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_collatorNodeSetWhitelistEnabled &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + if (config_.collator_node_whiltelist_enabled == query.enabled_) { + promise.set_value(ton::create_serialize_tl_object()); + return; + } + config_.collator_node_whiltelist_enabled = query.enabled_; + validator_options_.write().set_collator_node_whitelist_enabled(query.enabled_); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCollatorNodeWhitelist &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + ton::tl_object_ptr result = {}; + result = ton::create_tl_object(); + result->enabled_ = config_.collator_node_whiltelist_enabled; + for (const auto &id : config_.collator_node_whitelist) { + result->adnl_ids_.push_back(id.bits256_value()); + } + promise.set_value(ton::serialize_tl_object(result, true)); +} + void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getCollatorOptionsJson &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { @@ -4107,16 +4224,44 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setCollat return; } + td::Ref list{true}; + auto S = list.write().unpack(*query.list_); + if (S.is_error()) { + promise.set_value(create_control_query_error(S.move_as_error_prefix("Invalid collators list: "))); + return; + } auto s = td::json_encode(td::ToJson(*query.list_), true); - auto S = td::write_file(collators_list_file(), s); + S = td::write_file(collators_list_file(), s); if (S.is_error()) { promise.set_value(create_control_query_error(std::move(S))); return; } - collators_list_ = std::move(query.list_); - td::Ref list{true}; - list.write().unpack(*collators_list_); + validator_options_.write().set_collators_list(std::move(list)); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_clearCollatorsList &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto S = td::unlink(collators_list_file()); + if (S.is_error()) { + promise.set_value(create_control_query_error(std::move(S))); + return; + } + + td::Ref list{true, ton::validator::CollatorsList::default_list()}; + collators_list_ = {}; validator_options_.write().set_collators_list(std::move(list)); td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, validator_options_); @@ -4136,12 +4281,33 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showColla if (collators_list_) { promise.set_value(ton::serialize_tl_object(collators_list_, true)); } else { - auto list = ton::create_tl_object(); - list->self_collate_ = true; - promise.set_value(ton::serialize_tl_object(list, true)); + promise.set_value(create_control_query_error(td::Status::Error("collators list is empty"))); } } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getCollationManagerStats &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + td::actor::send_closure( + validator_manager_, &ton::validator::ValidatorManagerInterface::get_collation_manager_stats, + [promise = std::move(promise)]( + td::Result> R) mutable { + if (R.is_ok()) { + promise.set_value(ton::serialize_tl_object(R.move_as_ok(), true)); + } else { + promise.set_value(create_control_query_error(R.move_as_error())); + } + }); +} + void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollator &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { @@ -4154,7 +4320,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollat return; } - auto id = ton::PublicKeyHash{query.adnl_id_}; + auto id = ton::adnl::AdnlNodeIdShort{query.adnl_id_}; auto shard = ton::create_shard_id(query.shard_); auto R = config_.config_add_collator(id, shard); if (R.is_error()) { @@ -4169,12 +4335,10 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollat if (!validator_manager_.empty()) { td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, validator_options_); - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, - ton::adnl::AdnlNodeIdShort(id), shard); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, id, shard); } if (!full_node_.empty()) { - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_collator_adnl_id, - ton::adnl::AdnlNodeIdShort(id)); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_collator_adnl_id, id); } write_config([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -4229,7 +4393,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCollat return; } - auto id = ton::PublicKeyHash{query.adnl_id_}; + auto id = ton::adnl::AdnlNodeIdShort{query.adnl_id_}; auto shard = ton::create_shard_id(query.shard_); auto R = config_.config_del_collator(id, shard); if (R.is_error()) { @@ -4248,12 +4412,10 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCollat if (!validator_manager_.empty()) { td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, validator_options_); - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::del_collator, - ton::adnl::AdnlNodeIdShort(id), shard); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::del_collator, id, shard); } if (!full_node_.empty()) { - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::del_collator_adnl_id, - ton::adnl::AdnlNodeIdShort(id)); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::del_collator_adnl_id, id); } write_config([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -4318,7 +4480,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signOverl int expire_at = query.expire_at_; td::actor::send_closure( keyring_, &ton::keyring::Keyring::get_public_key, public_key_hash, - [=, promise = std::move(promise)](td::Result R) mutable { + [=, keyring = keyring_.get(), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); return; @@ -4330,7 +4492,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signOverl return; } td::BufferSlice to_sign = certificate.to_sign_data(adnl_id); - td::actor::send_closure(keyring_, &ton::keyring::Keyring::sign_message, public_key_hash, std::move(to_sign), + td::actor::send_closure(keyring, &ton::keyring::Keyring::sign_message, public_key_hash, std::move(to_sign), [certificate = std::move(certificate), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 4a8e9bdb48..7daefb57a5 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -67,14 +67,6 @@ struct Config { ton::UnixTime election_date; ton::UnixTime expire_at; }; - struct Collator { - ton::PublicKeyHash adnl_id; - ton::ShardIdFull shard; - - bool operator==(const Collator& b) const { - return adnl_id == b.adnl_id && shard == b.shard; - } - }; struct Control { ton::PublicKeyHash key; std::map clients; @@ -90,7 +82,9 @@ struct Config { std::map adnl_ids; std::set dht_ids; std::map validators; - std::vector collators; + std::map> collators; + bool collator_node_whiltelist_enabled = false; + std::set collator_node_whitelist; ton::PublicKeyHash full_node = ton::PublicKeyHash::zero(); std::vector full_node_slaves; std::map full_node_masters; @@ -120,8 +114,8 @@ struct Config { ton::UnixTime expire_at); td::Result config_add_validator_adnl_id(ton::PublicKeyHash perm_key, ton::PublicKeyHash adnl_id, ton::UnixTime expire_at); - td::Result config_add_collator(ton::PublicKeyHash addr, ton::ShardIdFull shard); - td::Result config_del_collator(ton::PublicKeyHash addr, ton::ShardIdFull shard); + td::Result config_add_collator(ton::adnl::AdnlNodeIdShort addr, ton::ShardIdFull shard); + td::Result config_del_collator(ton::adnl::AdnlNodeIdShort addr, ton::ShardIdFull shard); td::Result config_add_full_node_adnl_id(ton::PublicKeyHash id); td::Result config_add_full_node_slave(td::IPAddress addr, ton::PublicKey id); td::Result config_add_full_node_master(td::int32 port, ton::PublicKeyHash id); @@ -537,10 +531,21 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_setCollatorOptionsJson &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_collatorNodeSetWhitelistedValidator &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_collatorNodeSetWhitelistEnabled &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_showCollatorNodeWhitelist &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_setCollatorsList &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_clearCollatorsList &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_showCollatorsList &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getCollationManagerStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_signOverlayMemberCertificate &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_importFastSyncMemberCertificate &query, td::BufferSlice data, diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 60f66af372..abc602674d 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -58,6 +58,7 @@ set(VALIDATOR_HEADERS import-db-slice.hpp queue-size-counter.hpp + collation-manager.hpp collator-node.hpp manager-disk.h manager-disk.hpp @@ -74,6 +75,7 @@ set(VALIDATOR_HEADERS set(VALIDATOR_SOURCE apply-block.cpp block-handle.cpp + collation-manager.cpp collator-node.cpp get-next-key-blocks.cpp import-db-slice.cpp diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp new file mode 100644 index 0000000000..2ca3a5f10f --- /dev/null +++ b/validator/collation-manager.cpp @@ -0,0 +1,373 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "collation-manager.hpp" + +#include "collator-node.hpp" +#include "fabric.h" +#include "td/utils/Random.h" + +#include +#include +#include + +namespace ton::validator { + +void CollationManager::start_up() { + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); + update_collators_list(*opts_->get_collators_list()); +} + +void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, + std::vector prev, Ed25519_PublicKey creator, + td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, td::Promise promise) { + if (shard.is_masterchain()) { + run_collate_query(shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), + opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), std::move(promise), + std::move(cancellation_token), 0); + return; + } + collate_shard_block(shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), + max_answer_size, std::move(cancellation_token), std::move(promise), td::Timestamp::in(10.0)); +} + +void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, + std::vector prev, Ed25519_PublicKey creator, + td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, + td::Promise promise, td::Timestamp timeout) { + TRY_STATUS_PROMISE(promise, cancellation_token.check()); + ShardInfo* s = select_shard_info(shard); + if (s == nullptr) { + promise.set_error( + td::Status::Error(PSTRING() << "shard " << shard.to_str() << " is not configured in collators list")); + return; + } + + adnl::AdnlNodeIdShort selected_collator = adnl::AdnlNodeIdShort::zero(); + size_t selected_idx = 0; + switch (s->select_mode) { + case CollatorsList::mode_random: { + int cnt = 0; + for (size_t i = 0; i < s->collators.size(); ++i) { + adnl::AdnlNodeIdShort collator = s->collators[i]; + if (collators_[collator].alive) { + ++cnt; + if (td::Random::fast(1, cnt) == 1) { + selected_collator = collator; + selected_idx = i; + } + } + } + break; + } + case CollatorsList::mode_ordered: { + for (size_t i = 0; i < s->collators.size(); ++i) { + adnl::AdnlNodeIdShort collator = s->collators[i]; + if (collators_[collator].alive) { + selected_collator = collator; + selected_idx = i; + break; + } + } + break; + } + case CollatorsList::mode_round_robin: { + size_t iters = 0; + for (size_t i = s->cur_idx; iters < s->collators.size(); (++i) %= s->collators.size()) { + adnl::AdnlNodeIdShort& collator = s->collators[i]; + if (collators_[collator].alive) { + selected_collator = collator; + selected_idx = i; + s->cur_idx = (i + 1) % s->collators.size(); + break; + } + } + break; + } + } + + if (selected_collator.is_zero() && s->self_collate) { + run_collate_query(shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), + opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), std::move(promise), + std::move(cancellation_token), 0); + return; + } + + std::vector> prev_blocks; + BlockId next_block_id{shard, 0}; + for (const BlockIdExt& p : prev) { + prev_blocks.push_back(create_tl_block_id(p)); + next_block_id.seqno = std::max(next_block_id.seqno, p.seqno() + 1); + } + + promise = [=, SelfId = actor_id(this), promise = std::move(promise), + retry_at = td::Timestamp::in(0.5)](td::Result R) mutable { + if (R.is_ok()) { + promise.set_value(R.move_as_ok()); + return; + } + if (!selected_collator.is_zero()) { + td::actor::send_closure(SelfId, &CollationManager::on_collate_query_error, selected_collator); + } + LOG(INFO) << "ERROR: collate query for " << next_block_id.to_str() << " to #" << selected_idx << " (" + << selected_collator << "): " << R.error(); + if (timeout < retry_at) { + promise.set_error(R.move_as_error()); + return; + } + delay_action( + [=, promise = std::move(promise)]() mutable { + td::actor::send_closure(SelfId, &CollationManager::collate_shard_block, shard, min_masterchain_block_id, prev, + creator, validator_set, max_answer_size, cancellation_token, std::move(promise), + timeout); + }, + retry_at); + }; + + if (selected_collator.is_zero()) { + promise.set_error(td::Status::Error(PSTRING() << "shard " << shard.to_str() << " has no alive collator node")); + return; + } + + td::BufferSlice query = create_serialize_tl_object( + create_tl_shard_id(shard), validator_set->get_catchain_seqno(), std::move(prev_blocks), creator.as_bits256()); + LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to #" << selected_idx << "(" + << selected_collator << ")"; + + td::Promise P = [=, SelfId = actor_id(this), promise = std::move(promise), + timer = td::Timer()](td::Result R) mutable { + TRY_RESULT_PROMISE_PREFIX(promise, data, std::move(R), "rldp query failed: "); + auto r_error = fetch_tl_object(data, true); + if (r_error.is_ok()) { + auto error = r_error.move_as_ok(); + promise.set_error(td::Status::Error(error->code_, error->message_)); + return; + } + TRY_RESULT_PROMISE(promise, f, fetch_tl_object(data, true)); + TRY_RESULT_PROMISE(promise, candidate, + CollatorNode::deserialize_candidate(std::move(f), td::narrow_cast(max_answer_size))); + if (candidate.pubkey.as_bits256() != creator.as_bits256()) { + promise.set_error(td::Status::Error("collate query: block candidate source mismatch")); + return; + } + if (candidate.id.id != next_block_id) { + promise.set_error(td::Status::Error("collate query: block id mismatch")); + return; + } + LOG(INFO) << "got collated block " << next_block_id.to_str() << " from #" << selected_idx << " (" + << selected_collator << ") in " << timer.elapsed() << "s"; + promise.set_result(std::move(candidate)); + }; + td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_id_, selected_collator, "collatequery", std::move(P), + timeout, std::move(query), max_answer_size); +} + +void CollationManager::update_options(td::Ref opts) { + auto old_list = opts_->get_collators_list(); + opts_ = std::move(opts); + auto list = opts_->get_collators_list(); + if (old_list != list) { + update_collators_list(*list); + } +} + +void CollationManager::validator_group_started(ShardIdFull shard) { + if (active_validator_groups_[shard]++ != 0) { + return; + } + ShardInfo* s = select_shard_info(shard); + if (s == nullptr) { + return; + } + if (s->active_cnt++ != 0) { + return; + } + for (adnl::AdnlNodeIdShort id : s->collators) { + CollatorInfo& collator = collators_[id]; + collator.active_cnt++; + } + alarm(); +} + +void CollationManager::validator_group_finished(ShardIdFull shard) { + if (--active_validator_groups_[shard] != 0) { + return; + } + active_validator_groups_.erase(shard); + ShardInfo* s = select_shard_info(shard); + if (s == nullptr) { + return; + } + if (--s->active_cnt != 0) { + return; + } + for (adnl::AdnlNodeIdShort id : s->collators) { + CollatorInfo& collator = collators_[id]; + --collator.active_cnt; + } + alarm(); +} + +void CollationManager::get_stats( + td::Promise> promise) { + auto stats = create_tl_object(); + stats->adnl_id_ = local_id_.bits256_value(); + for (ShardInfo& s : shards_) { + auto obj = create_tl_object(); + obj->shard_id_ = create_tl_shard_id(s.shard_id); + obj->active_ = s.active_cnt; + obj->self_collate_ = s.self_collate; + switch (s.select_mode) { + case CollatorsList::mode_random: + obj->select_mode_ = "random"; + break; + case CollatorsList::mode_ordered: + obj->select_mode_ = "ordered"; + break; + case CollatorsList::mode_round_robin: + obj->select_mode_ = "round_robin"; + break; + } + for (adnl::AdnlNodeIdShort& id : s.collators) { + obj->collators_.push_back(id.bits256_value()); + } + stats->shards_.push_back(std::move(obj)); + } + for (auto& [id, collator] : collators_) { + auto obj = create_tl_object(); + obj->adnl_id_ = id.bits256_value(); + obj->active_ = collator.active_cnt; + obj->alive_ = collator.alive; + if (collator.active_cnt && !collator.sent_ping) { + obj->ping_in_ = collator.ping_at.in(); + } else { + obj->ping_in_ = -1.0; + } + stats->collators_.push_back(std::move(obj)); + } + promise.set_value(std::move(stats)); +} + +void CollationManager::update_collators_list(const CollatorsList& collators_list) { + shards_.clear(); + for (auto& [_, collator] : collators_) { + collator.active_cnt = 0; + } + auto old_collators = std::move(collators_); + collators_.clear(); + for (const auto& shard : collators_list.shards) { + shards_.push_back({.shard_id = shard.shard_id, .select_mode = shard.select_mode, .collators = shard.collators}); + for (auto id : shard.collators) { + auto it = old_collators.find(id); + if (it == old_collators.end()) { + collators_[id]; + } else { + collators_[id] = std::move(it->second); + old_collators.erase(it); + } + } + } + for (auto& [shard, _] : active_validator_groups_) { + ShardInfo* s = select_shard_info(shard); + if (s == nullptr) { + continue; + } + if (s->active_cnt++ != 0) { + continue; + } + for (adnl::AdnlNodeIdShort id : s->collators) { + CollatorInfo& collator = collators_[id]; + collator.active_cnt++; + } + } + alarm(); +} + +CollationManager::ShardInfo* CollationManager::select_shard_info(ShardIdFull shard) { + for (auto& s : shards_) { + if (shard_intersects(shard, s.shard_id)) { + return &s; + } + } + return nullptr; +} + +void CollationManager::alarm() { + alarm_timestamp() = td::Timestamp::never(); + for (auto& [id, collator] : collators_) { + if (collator.active_cnt == 0 || collator.sent_ping) { + continue; + } + if (collator.ping_at.is_in_past()) { + collator.sent_ping = true; + td::BufferSlice query = create_serialize_tl_object(0); + td::Promise P = [=, SelfId = actor_id(this)](td::Result R) mutable { + td::actor::send_closure(SelfId, &CollationManager::got_pong, id, std::move(R)); + }; + LOG(DEBUG) << "sending ping to " << id; + td::actor::send_closure(rldp_, &rldp::Rldp::send_query, local_id_, id, "collatorping", std::move(P), + td::Timestamp::in(2.0), std::move(query)); + } else { + alarm_timestamp().relax(collator.ping_at); + } + } +} + +void CollationManager::got_pong(adnl::AdnlNodeIdShort id, td::Result R) { + auto it = collators_.find(id); + if (it == collators_.end()) { + return; + } + CollatorInfo& collator = it->second; + collator.sent_ping = false; + + auto r_pong = [&]() -> td::Result> { + TRY_RESULT_PREFIX(data, std::move(R), "rldp query error: "); + auto r_error = fetch_tl_object(data, true); + if (r_error.is_ok()) { + auto error = r_error.move_as_ok(); + return td::Status::Error(error->code_, error->message_); + } + return fetch_tl_object(data, true); + }(); + if (r_pong.is_error()) { + LOG(DEBUG) << "pong from " << id << " : " << r_pong.move_as_error(); + collator.alive = false; + } else { + LOG(DEBUG) << "pong from " << id << " : OK"; + collator.alive = true; + } + collator.ping_at = td::Timestamp::in(td::Random::fast(10.0, 20.0)); + if (collator.active_cnt && !collator.sent_ping) { + alarm_timestamp().relax(collator.ping_at); + } +} + +void CollationManager::on_collate_query_error(adnl::AdnlNodeIdShort id) { + auto it = collators_.find(id); + if (it == collators_.end()) { + return; + } + CollatorInfo& collator = it->second; + collator.ping_at = td::Timestamp::now(); + if (collator.active_cnt && !collator.sent_ping) { + alarm_timestamp().relax(collator.ping_at); + } +} + +} // namespace ton::validator diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp new file mode 100644 index 0000000000..7ceea1e6b6 --- /dev/null +++ b/validator/collation-manager.hpp @@ -0,0 +1,87 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "interfaces/validator-manager.h" +#include "rldp/rldp.h" +#include + +namespace ton::validator { + +class ValidatorManager; + +class CollationManager : public td::actor::Actor { + public: + CollationManager(adnl::AdnlNodeIdShort local_id, td::Ref opts, + td::actor::ActorId manager, td::actor::ActorId rldp) + : local_id_(local_id), opts_(opts), manager_(manager), rldp_(rldp) { + } + + void start_up() override; + void alarm() override; + + void collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, + Ed25519_PublicKey creator, td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, td::Promise promise); + + void update_options(td::Ref opts); + + void validator_group_started(ShardIdFull shard); + void validator_group_finished(ShardIdFull shard); + + void get_stats(td::Promise> promise); + + private: + adnl::AdnlNodeIdShort local_id_; + td::Ref opts_; + td::actor::ActorId manager_; + td::actor::ActorId rldp_; + + void collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, + Ed25519_PublicKey creator, td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, td::Promise promise, + td::Timestamp timeout); + + void update_collators_list(const CollatorsList& collators_list); + + struct CollatorInfo { + bool alive = false; + td::Timestamp ping_at = td::Timestamp::now(); + bool sent_ping = false; + size_t active_cnt = 0; + }; + std::map collators_; + + struct ShardInfo { + ShardIdFull shard_id; + CollatorsList::SelectMode select_mode; + std::vector collators; + bool self_collate = false; + size_t cur_idx = 0; + + size_t active_cnt = 0; + }; + std::vector shards_; + + std::map active_validator_groups_; + + ShardInfo* select_shard_info(ShardIdFull shard); + void got_pong(adnl::AdnlNodeIdShort id, td::Result R); + void on_collate_query_error(adnl::AdnlNodeIdShort id); +}; + +} // namespace ton::validator diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 0b4204d86b..776fbc9e92 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -54,17 +54,22 @@ void CollatorNode::start_up() { td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlock::ID), std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_ping::ID), + std::make_unique(actor_id(this))); td::actor::send_closure(rldp_, &rldp::Rldp::add_id, adnl::AdnlNodeIdShort(local_id_)); } void CollatorNode::tear_down() { td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlock::ID)); + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_ping::ID)); } void CollatorNode::add_shard(ShardIdFull shard) { CHECK(shard.is_valid_ext() && !shard.is_masterchain()); - if (std::find(collating_shards_.begin(), collating_shards_.end(), shard) != collating_shards_.end()) { + if (std::ranges::find(collating_shards_, shard) != collating_shards_.end()) { return; } LOG(INFO) << "Collator node: local_id=" << local_id_ << " , shard=" << shard.to_str(); @@ -72,7 +77,7 @@ void CollatorNode::add_shard(ShardIdFull shard) { } void CollatorNode::del_shard(ShardIdFull shard) { - auto it = std::find(collating_shards_.begin(), collating_shards_.end(), shard); + auto it = std::ranges::find(collating_shards_, shard); if (it != collating_shards_.end()) { collating_shards_.erase(it); } @@ -127,7 +132,7 @@ void CollatorNode::new_masterchain_block_notification(td::Ref } } for (auto it = validator_groups_.begin(); it != validator_groups_.end();) { - if (new_shards.count(it->first)) { + if (new_shards.contains(it->first)) { ++it; } else { it->second.cleanup(); @@ -253,7 +258,7 @@ void CollatorNode::CacheEntry::cancel(td::Status reason) { } static td::BufferSlice serialize_error(td::Status error) { - return create_serialize_tl_object(error.code(), error.message().c_str()); + return create_serialize_tl_object(error.code(), error.message().c_str()); } static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey creator, CatchainSeqno& cc_seqno, @@ -285,25 +290,32 @@ static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey cre void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) { - td::Promise new_promise = [promise = std::move(promise), src](td::Result R) mutable { + promise = [promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { - LOG(INFO) << "adnl query from " << src << ", error: " << R.error(); if (R.error().code() == ErrorCode::timeout) { promise.set_error(R.move_as_error()); } else { promise.set_result(serialize_error(R.move_as_error())); } } else { - LOG(INFO) << "adnl query from " << src << ", success"; - promise.set_result(create_serialize_tl_object( - serialize_candidate(R.move_as_ok(), true))); + promise.set_result(R.move_as_ok()); } }; - if (!validator_adnl_ids_.count(src)) { - new_promise.set_error(td::Status::Error("src is not a validator")); + if (!opts_->check_collator_node_whitelist(src)) { + promise.set_error(td::Status::Error("not authorized")); + return; + } + if (!validator_adnl_ids_.contains(src)) { + promise.set_error(td::Status::Error("src is not a validator")); + return; + } + auto r_ping = fetch_tl_object(data, true); + if (r_ping.is_ok()) { + process_ping(src, *r_ping.ok_ref(), std::move(promise)); return; } - TRY_RESULT_PROMISE(new_promise, f, fetch_tl_object(data, true)); + + TRY_RESULT_PROMISE(promise, f, fetch_tl_object(data, true)); ShardIdFull shard = create_shard_id(f->shard_); CatchainSeqno cc_seqno = f->cc_seqno_; std::vector prev_blocks; @@ -311,6 +323,16 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data prev_blocks.push_back(create_block_id(b)); } Ed25519_PublicKey creator(f->creator_); + td::Promise new_promise = [promise = std::move(promise), src, + shard](td::Result R) mutable { + if (R.is_error()) { + LOG(INFO) << "collate query from " << src << ", shard=" << shard.to_str() << ": error: " << R.error(); + promise.set_error(R.move_as_error()); + } else { + LOG(INFO) << "collate query from " << src << ", shard=" << shard.to_str() << ": success"; + promise.set_result(serialize_tl_object(serialize_candidate(R.move_as_ok(), true), true)); + } + }; new_promise = [new_promise = std::move(new_promise), creator, manager = manager_](td::Result R) mutable { TRY_RESULT_PROMISE(new_promise, block, std::move(R)); @@ -427,9 +449,15 @@ void CollatorNode::process_result(std::shared_ptr cache_entry, td::R cache_entry->promises.clear(); } +void CollatorNode::process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode_ping& ping, + td::Promise promise) { + LOG(DEBUG) << "got ping from " << src; + promise.set_result(create_serialize_tl_object(0)); +} + bool CollatorNode::can_collate_shard(ShardIdFull shard) const { - return std::any_of(collating_shards_.begin(), collating_shards_.end(), - [&](const ShardIdFull& our_shard) { return shard_intersects(shard, our_shard); }); + return std::ranges::any_of(collating_shards_, + [&](const ShardIdFull& our_shard) { return shard_intersects(shard, our_shard); }); } tl_object_ptr CollatorNode::serialize_candidate(const BlockCandidate& block, diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index 1a71f08a4c..a361ceb309 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -43,6 +43,7 @@ class CollatorNode : public td::actor::Actor { private: void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); + void process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode_ping& ping, td::Promise promise); bool can_collate_shard(ShardIdFull shard) const; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index bb806e281e..4488b77dff 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -502,10 +502,16 @@ td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { if (pfx_len > wc_monitor_min_split_) { shard = shard_prefix(shard, wc_monitor_min_split_); } - auto it = shards_.find(shard); - if (it != shards_.end()) { - update_shard_actor(shard, it->second.active); - return it->second.actor.get(); + while (true) { + auto it = shards_.find(shard); + if (it != shards_.end()) { + update_shard_actor(shard, it->second.active); + return it->second.actor.get(); + } + if (shard.pfx_len() == 0) { + break; + } + shard = shard_parent(shard); } // Special case if shards_ was not yet initialized. diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index d91ee1bf69..72154f8611 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -45,7 +45,7 @@ class Collator final : public td::actor::Actor { } static constexpr long long supported_capabilities() { return ton::capCreateStatsEnabled | ton::capBounceMsgBody | ton::capReportVersion | ton::capShortDequeue | - ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages; + ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages | ton::capFullCollatedData; } using LtCellRef = block::LtCellRef; using NewOutMsg = block::NewOutMsg; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 5b82d481ae..677b73ae0b 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -730,6 +730,8 @@ bool Collator::unpack_last_mc_state() { store_out_msg_queue_size_ = config_->has_capability(ton::capStoreOutMsgQueueSize); msg_metadata_enabled_ = config_->has_capability(ton::capMsgMetadata); deferring_messages_enabled_ = config_->has_capability(ton::capDeferMessages); + full_collated_data_ = config_->has_capability(capFullCollatedData); + LOG(DEBUG) << "full_collated_data is " << full_collated_data_; shard_conf_ = std::make_unique(*config_); prev_key_block_exists_ = config_->get_last_key_block(prev_key_block_, prev_key_block_lt_); if (prev_key_block_exists_) { @@ -768,8 +770,6 @@ bool Collator::unpack_last_mc_state() { << " have been enabled in global configuration, but we support only " << supported_version() << " (upgrade validator software?)"; } - full_collated_data_ = config_->get_collator_config(false).full_collated_data; - LOG(DEBUG) << "full_collated_data is " << full_collated_data_; // TODO: extract start_lt and end_lt from prev_mc_block as well // std::cerr << " block::gen::ShardState::print_ref(mc_state_root) = "; // block::gen::t_ShardState.print_ref(std::cerr, mc_state_root, 2); @@ -817,6 +817,9 @@ bool Collator::request_neighbor_msg_queues() { auto neighbor_list = shard_conf_->get_neighbor_shard_hash_ids(shard_); LOG(DEBUG) << "got a preliminary list of " << neighbor_list.size() << " neighbors for " << shard_.to_str(); for (ton::BlockId blk_id : neighbor_list) { + if (blk_id.seqno == 0 && blk_id.shard_full() != shard_) { + continue; + } auto shard_ptr = shard_conf_->get_shard_hash(ton::ShardIdFull(blk_id)); if (shard_ptr.is_null()) { return fatal_error(-667, "cannot obtain shard hash for neighbor "s + blk_id.to_str()); diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index c8c7aca20e..b13be02195 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -170,9 +170,6 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { block::WorkchainSet get_workchain_list() const override { return config_ ? config_->get_workchain_list() : block::WorkchainSet(); } - block::CollatorConfig get_collator_config(bool need_collator_nodes) const override { - return config_ ? config_->get_collator_config(need_collator_nodes) : block::CollatorConfig(); - } private: ZeroStateIdExt zerostate_id_; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index ca00e00e9b..4c6264025c 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1536,6 +1536,9 @@ bool ValidateQuery::request_neighbor_queues() { auto neighbor_list = new_shard_conf_->get_neighbor_shard_hash_ids(shard_); LOG(DEBUG) << "got a preliminary list of " << neighbor_list.size() << " neighbors for " << shard_.to_str(); for (ton::BlockId blk_id : neighbor_list) { + if (blk_id.seqno == 0 && blk_id.shard_full() != shard_) { + continue; + } auto shard_ptr = new_shard_conf_->get_shard_hash(ton::ShardIdFull(blk_id)); if (shard_ptr.is_null()) { return reject_query("cannot obtain shard hash for neighbor "s + blk_id.to_str()); @@ -2305,6 +2308,12 @@ bool ValidateQuery::prepare_out_msg_queue_size() { have_out_msg_queue_size_in_state_ = true; return true; } + if (ps_.out_msg_queue_->is_empty()) { + old_out_msg_queue_size_ = 0; + out_msg_queue_size_known_ = true; + have_out_msg_queue_size_in_state_ = true; + return true; + } if (!store_out_msg_queue_size_) { // Don't need it return true; } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index a21a56e90a..585d553c5e 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -113,7 +113,7 @@ class ValidateQuery : public td::actor::Actor { } static constexpr long long supported_capabilities() { return ton::capCreateStatsEnabled | ton::capBounceMsgBody | ton::capReportVersion | ton::capShortDequeue | - ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages; + ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages | ton::capFullCollatedData; } public: diff --git a/validator/impl/validator-set.hpp b/validator/impl/validator-set.hpp index 3141f36c54..8b833e0478 100644 --- a/validator/impl/validator-set.hpp +++ b/validator/impl/validator-set.hpp @@ -50,6 +50,7 @@ class ValidatorSetQ : public ValidatorSet { td::Ref signatures) const override; td::Result check_approve_signatures(RootHash root_hash, FileHash file_hash, td::Ref signatures) const override; + const ValidatorDescr* find_validator(const NodeIdShort& id) const override; ValidatorSetQ* make_copy() const override; @@ -62,8 +63,6 @@ class ValidatorSetQ : public ValidatorSet { ValidatorWeight total_weight_; std::vector ids_; std::vector> ids_map_; - - const ValidatorDescr* find_validator(const NodeIdShort& id) const; }; class ValidatorSetCompute { diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 3383f6fc21..3546f0a39b 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -85,7 +85,6 @@ class MasterchainState : virtual public ShardState { virtual bool check_old_mc_block_id(const ton::BlockIdExt& blkid, bool strict = false) const = 0; virtual td::Result> get_config_holder() const = 0; virtual block::WorkchainSet get_workchain_list() const = 0; - virtual block::CollatorConfig get_collator_config(bool need_collator_nodes) const = 0; virtual td::Status prepare() { return td::Status::OK(); } diff --git a/validator/interfaces/validator-set.h b/validator/interfaces/validator-set.h index b71c0bfeae..20569df75a 100644 --- a/validator/interfaces/validator-set.h +++ b/validator/interfaces/validator-set.h @@ -35,6 +35,7 @@ class ValidatorSet : public td::CntObject { virtual td::uint32 get_validator_set_hash() const = 0; virtual ShardId get_validator_set_from() const = 0; virtual std::vector export_vector() const = 0; + virtual const ValidatorDescr* find_validator(const NodeIdShort& id) const = 0; virtual td::Result check_signatures(RootHash root_hash, FileHash file_hash, td::Ref signatures) const = 0; virtual td::Result check_approve_signatures(RootHash root_hash, FileHash file_hash, diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index c54ccde55f..c6d7337f91 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -457,6 +457,11 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void get_collation_manager_stats( + td::Promise> promise) override { + UNREACHABLE(); + } + void update_options(td::Ref opts) override { opts_ = std::move(opts); } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index daf83a197e..860243fb9f 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -525,6 +525,11 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void get_collation_manager_stats( + td::Promise> promise) override { + UNREACHABLE(); + } + private: td::Ref opts_; diff --git a/validator/manager.cpp b/validator/manager.cpp index f972635704..3f59d784d4 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -799,7 +799,13 @@ void ValidatorManagerImpl::wait_neighbor_msg_queue_proofs( public: Worker(size_t pending, td::Promise>> promise) : pending_(pending), promise_(std::move(promise)) { - CHECK(pending_ > 0); + } + + void start_up() override { + if (pending_ == 0) { + promise_.set_result(std::move(result_)); + stop(); + } } void on_result(td::Ref res) { @@ -2462,15 +2468,27 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto validator_id = get_validator(shard, validator_set); CHECK(!validator_id.is_zero()); + auto descr = validator_set->find_validator(validator_id.bits256_value()); + CHECK(descr); + auto adnl_id = adnl::AdnlNodeIdShort{ + descr->addr.is_zero() ? ValidatorFullId{descr->key}.compute_short_id().bits256_value() : descr->addr}; auto G = td::actor::create_actor( - PSTRING() << "valgroup" << shard.to_str(), shard, validator_id, session_id, validator_set, key_seqno, - last_masterchain_state_->get_collator_config(true), opts, keyring_, adnl_, rldp_, overlays_, db_root_, - actor_id(this), init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_, + PSTRING() << "valgroup" << shard.to_str(), shard, validator_id, session_id, validator_set, key_seqno, opts, + keyring_, adnl_, rldp_, overlays_, db_root_, actor_id(this), get_collation_manager(adnl_id), init_session, + opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_, opts_->need_monitor(shard, last_masterchain_state_)); return G; } } +td::actor::ActorId ValidatorManagerImpl::get_collation_manager(adnl::AdnlNodeIdShort adnl_id) { + auto &actor = collation_managers_[adnl_id]; + if (actor.empty()) { + actor = td::actor::create_actor("collation", adnl_id, opts_, actor_id(this), rldp_); + } + return actor.get(); +} + void ValidatorManagerImpl::add_handle_to_lru(BlockHandle handle) { auto it = handle_lru_map_.find(handle->id()); if (it != handle_lru_map_.end()) { @@ -3458,6 +3476,9 @@ void ValidatorManagerImpl::update_options(td::Ref opts) for (auto &collator : collator_nodes_) { td::actor::send_closure(collator.second.actor, &CollatorNode::update_options, opts); } + for (auto &[_, c] : collation_managers_) { + td::actor::send_closure(c, &CollationManager::update_options, opts); + } opts_ = std::move(opts); } @@ -3492,6 +3513,54 @@ void ValidatorManagerImpl::del_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh } } +void ValidatorManagerImpl::get_collation_manager_stats( + td::Promise> promise) { + class Cb : public td::actor::Actor { + public: + explicit Cb(td::Promise> promise) + : promise_(std::move(promise)) { + } + + void got_stats(tl_object_ptr s) { + result_.push_back(std::move(s)); + dec_pending(); + } + + void inc_pending() { + ++pending_; + } + + void dec_pending() { + CHECK(pending_ > 0); + --pending_; + if (pending_ == 0) { + promise_.set_result(create_tl_object(std::move(result_))); + stop(); + } + } + + private: + td::Promise> promise_; + size_t pending_ = 1; + std::vector> result_; + }; + auto callback = td::actor::create_actor("stats", std::move(promise)).release(); + + for (auto &[_, actor] : collation_managers_) { + td::actor::send_closure(callback, &Cb::inc_pending); + td::actor::send_closure( + actor, &CollationManager::get_stats, + [callback](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(callback, &Cb::dec_pending); + } else { + td::actor::send_closure(callback, &Cb::got_stats, R.move_as_ok()); + } + }); + } + td::actor::send_closure(callback, &Cb::dec_pending); +} + void ValidatorManagerImpl::add_persistent_state_description(td::Ref desc) { auto now = (UnixTime)td::Clocks::system(); if (desc->end_time <= now) { diff --git a/validator/manager.hpp b/validator/manager.hpp index b7e457e27e..055adf304a 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -283,12 +283,15 @@ class ValidatorManagerImpl : public ValidatorManager { td::Ref validator_set, BlockSeqno key_seqno, validatorsession::ValidatorSessionOptions opts, bool create_catchain); + td::actor::ActorId get_collation_manager(adnl::AdnlNodeIdShort adnl_id); + struct ValidatorGroupEntry { td::actor::ActorOwn actor; ShardIdFull shard; }; std::map validator_groups_; std::map next_validator_groups_; + std::map> collation_managers_; std::set check_gc_list_; std::vector gc_list_; @@ -634,6 +637,9 @@ class ValidatorManagerImpl : public ValidatorManager { void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; + void get_collation_manager_stats( + td::Promise> promise) override; + void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { if (last_masterchain_state_.is_null()) { diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 55798a4828..f95266b33b 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -65,7 +65,10 @@ void ValidatorGroup::generate_block_candidate( td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, source_info, std::move(cache), std::move(R)); }; - collate_block(source_info, td::Timestamp::in(10.0), std::move(P)); + td::uint64 max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; + td::actor::send_closure(collation_manager_, &CollationManager::collate_block, shard_, min_masterchain_block_id_, + prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_, + max_answer_size, cancellation_token_source_.get_cancellation_token(), std::move(P)); } void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo source_info, @@ -218,9 +221,9 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref return; } LOG_CHECK(R.error().code() == ErrorCode::timeout || R.error().code() == ErrorCode::notready) << R.move_as_error(); - td::actor::send_closure(SelfId, &ValidatorGroup::accept_block_query, block_id, std::move(block), - std::move(prev), std::move(sig_set), std::move(approve_sig_set), send_broadcast_mode, - std::move(promise), true); + td::actor::send_closure(SelfId, &ValidatorGroup::accept_block_query, block_id, std::move(block), std::move(prev), + std::move(sig_set), std::move(approve_sig_set), send_broadcast_mode, std::move(promise), + true); } else { promise.set_value(R.move_as_ok()); } @@ -292,7 +295,8 @@ std::unique_ptr ValidatorGroup::ma td::actor::send_closure(id_, &ValidatorGroup::generate_block_candidate, std::move(source_info), std::move(promise)); } - void on_block_committed(validatorsession::BlockSourceInfo source_info, validatorsession::ValidatorSessionRootHash root_hash, + void on_block_committed(validatorsession::BlockSourceInfo source_info, + validatorsession::ValidatorSessionRootHash root_hash, validatorsession::ValidatorSessionFileHash file_hash, td::BufferSlice data, std::vector> signatures, std::vector> approve_signatures, @@ -409,7 +413,7 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch stats.last_key_block_seqno = last_key_block_seqno_; stats.timestamp = td::Clocks::system(); td::uint32 idx = 0; - for (const auto& node : validator_set_->export_vector()) { + for (const auto &node : validator_set_->export_vector()) { PublicKeyHash id = ValidatorFullId{node.key}.compute_short_id(); if (id == local_id_) { stats.self_idx = idx; @@ -489,7 +493,7 @@ void ValidatorGroup::get_validator_group_info_for_litequery_cont( auto result = create_tl_object(); result->next_block_id_ = create_tl_lite_block_id_simple(next_block_id); - for (const BlockIdExt& prev : prev_block_ids_) { + for (const BlockIdExt &prev : prev_block_ids_) { result->prev_.push_back(create_tl_lite_block_id(prev)); } result->cc_seqno_ = validator_set_->get_catchain_seqno(); @@ -497,146 +501,6 @@ void ValidatorGroup::get_validator_group_info_for_litequery_cont( promise.set_result(std::move(result)); } -void ValidatorGroup::collate_block(validatorsession::BlockSourceInfo source_info, td::Timestamp timeout, - td::Promise promise, unsigned max_retries) { - if (source_info.round < last_known_round_id_) { - promise.set_error(td::Status::Error("too old")); - return; - } - BlockId next_block_id = create_next_block_id_simple(); - adnl::AdnlNodeIdShort collator_adnl_id = adnl::AdnlNodeIdShort::zero(); - bool self_collate = false; - bool trusted_collator = false; - - if (shard_.is_masterchain()) { - self_collate = true; - } else { - for (const auto &s : opts_->get_collators_list()->shards) { - if (!shard_intersects(s.shard_id, shard_)) { - continue; - } - if (!s.collators.empty()) { - const CollatorsList::Collator &col = s.collators[td::Random::fast(0, s.collators.size() - 1)]; - collator_adnl_id = col.adnl_id; - trusted_collator = col.trusted; - break; - } - } - if (collator_adnl_id.is_zero()) { - if (opts_->get_collators_list()->self_collate) { - self_collate = true; - } else if (opts_->get_collators_list()->use_config_41) { - // TODO: some way to choose node (similar to "unreliability" in full-node) - int cnt = 0; - for (const block::CollatorNodeDescr &c : collator_config_.collator_nodes) { - if (shard_intersects(shard_, c.shard)) { - if (td::Random::fast(0, cnt) == 0) { - collator_adnl_id = adnl::AdnlNodeIdShort(c.adnl_id); - } - ++cnt; - } - } - } - } - } - if (self_collate) { - run_collate_query(shard_, min_masterchain_block_id_, prev_block_ids_, - Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_, - opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), std::move(promise), - cancellation_token_source_.get_cancellation_token(), 0); - return; - } - if (collator_adnl_id.is_zero()) { - promise.set_error(td::Status::Error(PSTRING() << "no collator for shard " << shard_.to_str())); - return; - } - - promise = td::PromiseCreator::lambda([=, SelfId = actor_id(this), promise = std::move(promise), - timer = td::Timer()](td::Result R) mutable { - if (R.is_ok()) { - LOG(INFO) << "collate query for " << next_block_id.to_str() << ": success, time=" << timer.elapsed() << "s"; - promise.set_result(R.move_as_ok()); - return; - } - bool retry = (!timeout || !timeout.is_in_past()) && max_retries > 0; - LOG(WARNING) << "collate query for " << next_block_id.to_str() << ": " << R.error() << ", time=" << timer.elapsed() - << "s, " << (retry ? "retrying" : "giving up"); - if (retry) { - td::actor::send_closure(SelfId, &ValidatorGroup::collate_block, source_info, timeout, std::move(promise), - max_retries - 1); - } else { - promise.set_result(td::Status::Error(ErrorCode::timeout, "timeout")); - } - }); - - std::vector> prev_blocks; - for (const BlockIdExt &p : prev_block_ids_) { - prev_blocks.push_back(create_tl_block_id(p)); - } - td::BufferSlice query = create_serialize_tl_object( - create_tl_shard_id(shard_), validator_set_->get_catchain_seqno(), std::move(prev_blocks), - local_id_full_.ed25519_value().raw()); - - auto P = td::PromiseCreator::lambda( - [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error_prefix("rldp query failed: ")); - return; - } - td::actor::send_closure(SelfId, &ValidatorGroup::receive_collate_query_response, source_info, R.move_as_ok(), - trusted_collator, std::move(promise)); - }); - LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to " << collator_adnl_id; - size_t max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; - td::Timestamp query_timeout = td::Timestamp::in(10.0); - query_timeout.relax(timeout); - td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_adnl_id_, collator_adnl_id, "collatequery", - std::move(P), timeout, std::move(query), max_answer_size); -} - -void ValidatorGroup::receive_collate_query_response(validatorsession::BlockSourceInfo source_info, td::BufferSlice data, - bool trusted_collator, td::Promise promise) { - if (source_info.round < last_known_round_id_) { - promise.set_error(td::Status::Error("too old")); - return; - } - TRY_RESULT_PROMISE(promise, f, fetch_tl_object(data, true)); - td::Result res; - ton_api::downcast_call(*f, td::overloaded( - [&](ton_api::collatorNode_generateBlockError &r) { - td::Status error = td::Status::Error(r.code_, r.message_); - res = error.move_as_error_prefix("collate query: "); - }, - [&](ton_api::collatorNode_generateBlockSuccess &r) { - res = CollatorNode::deserialize_candidate( - std::move(r.candidate_), - config_.max_block_size + config_.max_collated_data_size + 1024); - })); - TRY_RESULT_PROMISE(promise, candidate, std::move(res)); - if (candidate.pubkey.as_bits256() != local_id_full_.ed25519_value().raw()) { - promise.set_error(td::Status::Error("collate query: block candidate source mismatch")); - return; - } - if (candidate.id.shard_full() != shard_) { - promise.set_error(td::Status::Error("collate query: shard mismatch")); - return; - } - - if (trusted_collator) { - promise.set_result(std::move(candidate)); - return; - } - auto P = td::PromiseCreator::lambda( - [candidate = candidate.clone(), promise = std::move(promise)](td::Result> R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error_prefix("validate received block error: ")); - return; - } - promise.set_result(std::move(candidate)); - }); - validate_block_candidate(source_info, std::move(candidate), std::move(P)); -} - } // namespace validator } // namespace ton diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 38fc9fb447..fb7a4dcfb8 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -18,6 +18,7 @@ */ #pragma once +#include "collation-manager.hpp" #include "interfaces/validator-manager.h" #include "validator-session/validator-session.h" @@ -59,6 +60,10 @@ class ValidatorGroup : public td::actor::Actor { init_ = false; create_session(); } + td::actor::send_closure(collation_manager_, &CollationManager::validator_group_started, shard_); + } + void tear_down() override { + td::actor::send_closure(collation_manager_, &CollationManager::validator_group_finished, shard_); } void get_validator_group_info_for_litequery( @@ -71,17 +76,17 @@ class ValidatorGroup : public td::actor::Actor { ValidatorGroup(ShardIdFull shard, PublicKeyHash local_id, ValidatorSessionId session_id, td::Ref validator_set, BlockSeqno last_key_block_seqno, - block::CollatorConfig collator_config, validatorsession::ValidatorSessionOptions config, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, - std::string db_root, td::actor::ActorId validator_manager, bool create_session, + validatorsession::ValidatorSessionOptions config, td::actor::ActorId keyring, + td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId overlays, std::string db_root, + td::actor::ActorId validator_manager, + td::actor::ActorId collation_manager, bool create_session, bool allow_unsafe_self_blocks_resync, td::Ref opts, bool monitoring_shard) : shard_(shard) , local_id_(std::move(local_id)) , session_id_(session_id) , validator_set_(std::move(validator_set)) , last_key_block_seqno_(last_key_block_seqno) - , collator_config_(std::move(collator_config)) , config_(std::move(config)) , keyring_(keyring) , adnl_(adnl) @@ -89,6 +94,7 @@ class ValidatorGroup : public td::actor::Actor { , overlays_(overlays) , db_root_(std::move(db_root)) , manager_(validator_manager) + , collation_manager_(collation_manager) , init_(create_session) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) , opts_(std::move(opts)) @@ -97,10 +103,6 @@ class ValidatorGroup : public td::actor::Actor { private: std::unique_ptr make_validator_session_callback(); - void collate_block(validatorsession::BlockSourceInfo source_info, td::Timestamp timeout, - td::Promise promise, unsigned max_retries = 4); - void receive_collate_query_response(validatorsession::BlockSourceInfo source_info, td::BufferSlice data, - bool trusted_collator, td::Promise promise); struct PostponedAccept { RootHash root_hash; @@ -124,7 +126,6 @@ class ValidatorGroup : public td::actor::Actor { td::Ref validator_set_; BlockSeqno last_key_block_seqno_; - block::CollatorConfig collator_config_; validatorsession::ValidatorSessionOptions config_; td::actor::ActorId keyring_; @@ -133,6 +134,7 @@ class ValidatorGroup : public td::actor::Actor { td::actor::ActorId overlays_; std::string db_root_; td::actor::ActorId manager_; + td::actor::ActorId collation_manager_; td::actor::ActorOwn session_; adnl::AdnlNodeIdShort local_adnl_id_; diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index 4f9b8c538a..230a5df469 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -26,29 +26,58 @@ namespace ton { namespace validator { -void CollatorsList::unpack(const ton_api::engine_validator_collatorsList& obj) { +td::Status CollatorsList::unpack(const ton_api::engine_validator_collatorsList& obj) { shards.clear(); - self_collate = obj.self_collate_; - use_config_41 = obj.use_config_41_; + self_collate = false; for (const auto& shard_obj : obj.shards_) { + ShardIdFull shard_id = create_shard_id(shard_obj->shard_id_); + if (shard_id.is_masterchain()) { + return td::Status::Error("masterchain shard in collators list"); + } + if (!shard_id.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << shard_id.to_str()); + } shards.emplace_back(); Shard& shard = shards.back(); - shard.shard_id = create_shard_id(shard_obj->shard_id_); + shard.shard_id = shard_id; + shard.self_collate = shard_obj->self_collate_; + if (shard.self_collate) { + self_collate = true; + } + if (shard_obj->select_mode_.empty() || shard_obj->select_mode_ == "random") { + shard.select_mode = mode_random; + } else if (shard_obj->select_mode_ == "ordered") { + shard.select_mode = mode_ordered; + } else if (shard_obj->select_mode_ == "round_robin") { + shard.select_mode = mode_round_robin; + } else { + return td::Status::Error(PSTRING() << "invalid select mode '" << shard_obj->select_mode_ + << "' (allowed: 'random', 'ordered', 'round_robin')"); + } for (const auto& collator : shard_obj->collators_) { - shard.collators.push_back({adnl::AdnlNodeIdShort{collator->adnl_id_}, collator->trusted_}); + shard.collators.push_back(adnl::AdnlNodeIdShort{collator->adnl_id_}); } } + return td::Status::OK(); +} + +CollatorsList CollatorsList::default_list() { + CollatorsList list; + list.shards.push_back( + {.shard_id = ShardIdFull{basechainId, shardIdAll}, .select_mode = mode_random, .self_collate = true}); + list.self_collate = true; + return list; } -td::Ref ValidatorManagerOptions::create( - BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, bool allow_blockchain_init, - double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, - double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) { +td::Ref ValidatorManagerOptions::create(BlockIdExt zero_block_id, BlockIdExt init_block_id, + std::function check_shard, + bool allow_blockchain_init, double sync_blocks_before, + double block_ttl, double state_ttl, + double max_mempool_num, double archive_ttl, + double key_proof_ttl, bool initial_sync_disabled) { return td::make_ref(zero_block_id, init_block_id, std::move(check_shard), allow_blockchain_init, sync_blocks_before, block_ttl, state_ttl, - max_mempool_num, - archive_ttl, key_proof_ttl, initial_sync_disabled); + max_mempool_num, archive_ttl, key_proof_ttl, initial_sync_disabled); } } // namespace validator diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 8e876625d0..05cc73d312 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -157,6 +157,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { td::Ref get_collators_list() const override { return collators_list_; } + bool check_collator_node_whitelist(adnl::AdnlNodeIdShort id) const override { + return !collator_node_whitelist_enabled_ || collator_node_whitelist_.contains(id); + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -255,6 +258,16 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_collators_list(td::Ref list) override { collators_list_ = std::move(list); } + void set_collator_node_whitelisted_validator(adnl::AdnlNodeIdShort id, bool add) override { + if (add) { + collator_node_whitelist_.insert(id); + } else { + collator_node_whitelist_.erase(id); + } + } + void set_collator_node_whitelist_enabled(bool enabled) override { + collator_node_whitelist_enabled_ = enabled; + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); @@ -308,7 +321,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool state_serializer_enabled_ = true; td::Ref collator_options_{true}; bool fast_state_serializer_enabled_ = false; - td::Ref collators_list_{true, CollatorsList{}}; + td::Ref collators_list_{true, CollatorsList::default_list()}; + std::set collator_node_whitelist_; + bool collator_node_whitelist_enabled_ = false; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 4580767684..3eaaaddf9b 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -73,19 +73,20 @@ struct CollatorOptions : public td::CntObject { }; struct CollatorsList : public td::CntObject { - struct Collator { - adnl::AdnlNodeIdShort adnl_id; - bool trusted; + enum SelectMode { + mode_random, mode_ordered, mode_round_robin }; struct Shard { ShardIdFull shard_id; - std::vector collators; + SelectMode select_mode = mode_random; + std::vector collators; + bool self_collate = false; }; - bool self_collate = true; - bool use_config_41 = false; std::vector shards; + bool self_collate = false; - void unpack(const ton_api::engine_validator_collatorsList& obj); + td::Status unpack(const ton_api::engine_validator_collatorsList& obj); + static CollatorsList default_list(); }; struct ValidatorManagerOptions : public td::CntObject { @@ -130,6 +131,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual td::Ref get_collator_options() const = 0; virtual bool get_fast_state_serializer_enabled() const = 0; virtual td::Ref get_collators_list() const = 0; + virtual bool check_collator_node_whitelist(adnl::AdnlNodeIdShort id) const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; @@ -163,6 +165,8 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_collator_options(td::Ref value) = 0; virtual void set_fast_state_serializer_enabled(bool value) = 0; virtual void set_collators_list(td::Ref list) = 0; + virtual void set_collator_node_whitelisted_validator(adnl::AdnlNodeIdShort id, bool add) = 0; + virtual void set_collator_node_whitelist_enabled(bool enabled) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, @@ -314,6 +318,9 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; virtual void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; + + virtual void get_collation_manager_stats( + td::Promise> promise) = 0; }; } // namespace validator From b60f6ee72f41cd5003860482309632c81c80d3ab Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 26 Nov 2024 08:54:19 +0300 Subject: [PATCH 123/524] Write collation stats to session stats, add collator options for collated data --- tl/generate/scheme/ton_api.tl | 13 +++++-- tl/generate/scheme/ton_api.tlo | Bin 107428 -> 108128 bytes validator-engine/validator-engine.cpp | 4 ++ validator/collator-node.cpp | 3 +- validator/fabric.h | 2 +- validator/impl/collator.cpp | 46 +++++++++++++++++++---- validator/impl/validate-query.cpp | 2 +- validator/interfaces/validator-manager.h | 18 +++++++-- validator/manager.cpp | 29 ++++++++++---- validator/manager.hpp | 3 +- validator/validator.h | 5 +++ 11 files changed, 98 insertions(+), 27 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 0839f727d0..f7b23d1b8c 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -669,7 +669,8 @@ engine.validator.collatorOptions deferring_enabled:Bool defer_messages_after:int defer_out_queue_size_limit:long dispatch_phase_2_max_total:int dispatch_phase_3_max_total:int dispatch_phase_2_max_per_initiator:int dispatch_phase_3_max_per_initiator:int - whitelist:(vector string) prioritylist:(vector string) = engine.validator.CollatorOptions; + whitelist:(vector string) prioritylist:(vector string) + force_full_collated_data:Bool ignore_collated_data_limits:Bool = engine.validator.CollatorOptions; engine.validator.collatorsList.collator adnl_id:int256 = engine.validator.collatorsList.Collator; engine.validator.collatorsList.shard shard_id:tonNode.shardId collators:(vector engine.validator.collatorsList.collator) @@ -874,8 +875,12 @@ http.server.config dhs:(vector http.server.dnsEntry) local_hosts:(vector http.se ---types--- -validatorSession.collationStats bytes:int gas:int lt_delta:int cat_bytes:int cat_gas:int cat_lt_delta:int - limits_log:string ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validadorSession.CollationStats; +validatorSession.collationStats actual_bytes:int actual_collated_data_bytes:int + bytes:int gas:int lt_delta:int collated_data_bytes:int + cat_bytes:int cat_gas:int cat_lt_delta:int cat_collated_data_bytes:int + limits_log:string ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int + work_time:double cpu_work_time:double + serialized_size:int serialized_size_no_collated_data:int = validadorSession.CollationStats; validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int root_hash:int256 file_hash:int256 comment:string block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double @@ -896,6 +901,8 @@ validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:double self: signatures:int signatures_weight:long approve_signatures:int approve_signatures_weight:long first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats; +validatorSession.statsCollatedBlock timestamp:double id:tonNode.blockIdExt stats:validatorSession.collationStats = validatorSession.StatsCollatedBlock; + collatorNode.candidate source:PublicKey id:tonNode.blockIdExt data:bytes collated_data:bytes = collatorNode.Candidate; collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt decompressed_size:int data:bytes = collatorNode.Candidate; collatorNode.pong flags:# = collatorNode.Pong; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index dd2af2ec9de6dbde7d6644ac55b85bf42bba5df1..2cff142655e991800ee801a06bd7a6e6a29e0657 100644 GIT binary patch delta 536 zcmZ2-obACGHr_|G^{p77z+@xuX)%^Y4Lg?2cf@>@SXTcz^nJ6Ar6jY4@q**ZWr;bN zDTyWdMZu}X#hLkedch@$CB@G9IXQ_XsVPo5`N`RglMgb9Z+5ZS!N_=U^IzN3IjkVV z6efQ>At~%%Py*Cc473U)cJr8!5BKJutDZAXHrUQ0DUp_6l$;u$R+^I&pA0oI9%xzO zWXBWIGSZppdHF@Ds50?6nYo!I#ghxp$v~X^W{p7;*vZos^cfXoL5>I6A`G-q`q_O= zp2Xym(!`wjq{@=i;>nJyWh9^iDE5J*rwh6>N>2~aXG{Jd88Dujyup=~vpl~jJH8|{H+3>!x)dL8azSZ4Txj||A4a>$0*)+- z3dN~KnZO{fN==C`29oi4`KU%uW(<-C`)~Uf1IC+TETA}GlKej8m8(=0 delta 183 zcmaEGhHc4lHr_|G^{p77z<49?X)%@;Q(jNld`HYjiRGQ8e&%K!OG)O zwL6`|3X)ftu9(RvDePZRl9``Z4Ac%1ym?H>hjX*#>gSA;AMD|Pm@{RqK@%&;9Qo-R z?HGlp-_d8};RNwPin*uz8Zf3$e_+6vGrh);@yz58_8eeKx3d^C-W1#3>BZR21QwWl NVY|q74PQo!NB~(wMzsI{ diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 8d2ce0bef9..cf3ca3ff56 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -2707,6 +2707,8 @@ static td::Result> parse_collator_optio f.dispatch_phase_2_max_per_initiator_ = opts.dispatch_phase_2_max_per_initiator; f.dispatch_phase_3_max_per_initiator_ = opts.dispatch_phase_3_max_per_initiator ? opts.dispatch_phase_3_max_per_initiator.value() : -1; + f.force_full_collated_data_ = false; + f.ignore_collated_data_limits_ = false; TRY_RESULT_PREFIX(json, td::json_decode(json_str), "failed to parse json: "); TRY_STATUS_PREFIX(ton::ton_api::from_json(f, json.get_object()), "json does not fit TL scheme: "); @@ -2746,6 +2748,8 @@ static td::Result> parse_collator_optio TRY_RESULT(addr, block::StdAddress::parse(s)); opts.prioritylist.emplace(addr.workchain, addr.addr); } + opts.force_full_collated_data = f.force_full_collated_data_; + opts.ignore_collated_data_limits = f.ignore_collated_data_limits_; return ref; } diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 776fbc9e92..cf2770d934 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -431,7 +431,8 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std << ", time=" << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); }, - cache_entry->cancellation_token_source.get_cancellation_token(), CollateMode::skip_store_candidate); + cache_entry->cancellation_token_source.get_cancellation_token(), + CollateMode::skip_store_candidate | CollateMode::from_collator_node); } void CollatorNode::process_result(std::shared_ptr cache_entry, td::Result R) { diff --git a/validator/fabric.h b/validator/fabric.h index 629065176b..20358822e1 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -27,7 +27,7 @@ namespace ton { namespace validator { enum ValidateMode { fake = 1 }; -enum CollateMode { skip_store_candidate = 1 }; +enum CollateMode { skip_store_candidate = 1, from_collator_node = 2 }; td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_, td::Ref opts); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 677b73ae0b..0054ea98aa 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -16,6 +16,7 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "candidate-serializer.h" #include "collator-impl.h" #include "vm/boc.h" #include "td/db/utils/BlobView.h" @@ -77,7 +78,7 @@ static inline bool dbg(int c) { * @param timeout The timeout for the collator. * @param promise The promise to return the result. * @param cancellation_token Token to cancel collation. - * @param mode +1 - skip storing candidate to disk. + * @param mode +1 - skip storing candidate to disk, +2 - called from CollatorNode. * @param attempt_idx The index of the attempt, starting from 0. On later attempts collator decreases block limits and skips some steps. */ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, @@ -730,7 +731,7 @@ bool Collator::unpack_last_mc_state() { store_out_msg_queue_size_ = config_->has_capability(ton::capStoreOutMsgQueueSize); msg_metadata_enabled_ = config_->has_capability(ton::capMsgMetadata); deferring_messages_enabled_ = config_->has_capability(ton::capDeferMessages); - full_collated_data_ = config_->has_capability(capFullCollatedData); + full_collated_data_ = config_->has_capability(capFullCollatedData) || collator_opts_->force_full_collated_data; LOG(DEBUG) << "full_collated_data is " << full_collated_data_; shard_conf_ = std::make_unique(*config_); prev_key_block_exists_ = config_->get_last_key_block(prev_key_block_, prev_key_block_lt_); @@ -751,15 +752,24 @@ bool Collator::unpack_last_mc_state() { LOG(INFO) << "Attempt #3: bytes, gas limits /= 2"; block_limits_->bytes.multiply_by(0.5); block_limits_->gas.multiply_by(0.5); + block_limits_->collated_data.multiply_by(0.5); } else if (attempt_idx_ == 4) { LOG(INFO) << "Attempt #4: bytes, gas limits /= 4"; block_limits_->bytes.multiply_by(0.25); block_limits_->gas.multiply_by(0.25); + block_limits_->collated_data.multiply_by(0.25); + } + if (collator_opts_->ignore_collated_data_limits) { + block_limits_->collated_data = block::ParamLimits{1 << 30, 1 << 30, 1 << 30}; } LOG(DEBUG) << "block limits: bytes [" << block_limits_->bytes.underload() << ", " << block_limits_->bytes.soft() << ", " << block_limits_->bytes.hard() << "]"; LOG(DEBUG) << "block limits: gas [" << block_limits_->gas.underload() << ", " << block_limits_->gas.soft() << ", " << block_limits_->gas.hard() << "]"; + LOG(DEBUG) << "block limits: lt_delta [" << block_limits_->lt_delta.underload() << ", " + << block_limits_->lt_delta.soft() << ", " << block_limits_->lt_delta.hard() << "]"; + LOG(DEBUG) << "block limits: collated_data_bytes [" << block_limits_->collated_data.underload() << ", " + << block_limits_->collated_data.soft() << ", " << block_limits_->collated_data.hard() << "]"; if (config_->has_capabilities() && (config_->get_capabilities() & ~supported_capabilities())) { LOG(ERROR) << "block generation capabilities " << config_->get_capabilities() << " have been enabled in global configuration, but we support only " << supported_capabilities() @@ -3678,6 +3688,10 @@ static std::string block_full_comment(const block::BlockLimitStatus& block_limit if (!block_limit_status.limits.lt_delta.fits(cls, lt_delta)) { return PSTRING() << "block_full lt_delta " << lt_delta; } + auto collated_data_bytes = block_limit_status.collated_data_stat.estimate_proof_size(); + if (!block_limit_status.limits.collated_data.fits(cls, collated_data_bytes)) { + return PSTRING() << "block_full collated_data " << collated_data_bytes; + } return ""; } @@ -5810,7 +5824,8 @@ bool Collator::create_block_candidate() { << ") exceeds the limit in consensus config (" << consensus_config.max_block_size << ")"); } - if (block_candidate->collated_data.size() > consensus_config.max_collated_data_size) { + if (block_candidate->collated_data.size() > consensus_config.max_collated_data_size && + !collator_opts_->ignore_collated_data_limits) { return fatal_error(PSTRING() << "collated data size (" << block_candidate->collated_data.size() << ") exceeds the limit in consensus config (" << consensus_config.max_collated_data_size << ")"); @@ -5838,14 +5853,31 @@ bool Collator::create_block_candidate() { double work_time = work_timer_.elapsed(); double cpu_work_time = cpu_work_timer_.elapsed(); LOG(WARNING) << "Collate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; - stats_.bytes = block_limit_status_->estimate_block_size(); + stats_.actual_bytes = block_candidate->data.size(); + stats_.actual_collated_data_bytes = block_candidate->collated_data.size(); + stats_.estimated_bytes = block_limit_status_->estimate_block_size(); stats_.gas = block_limit_status_->gas_used; stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; - stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.bytes); + stats_.estimated_collated_data_bytes = block_limit_status_->collated_data_stat.estimate_proof_size(); + stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.estimated_bytes); stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas); stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); - td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, block_candidate->id, work_time, - cpu_work_time, std::move(stats_)); + stats_.cat_collated_data_bytes = + block_limit_status_->limits.classify_collated_data_size(stats_.estimated_collated_data_bytes); + stats_.work_time = work_time; + stats_.cpu_work_time = cpu_work_time; + + // TODO: remove this later (currently needed to collect stats) + if (mode_ & CollateMode::from_collator_node) { + size_t d; + stats_.serialized_size = + validatorsession::compress_candidate_data(block_candidate->data, block_candidate->collated_data, d).ok().size(); + stats_.serialized_size_no_collated_data = + validatorsession::compress_candidate_data(block_candidate->data, td::Slice{}, d).ok().size(); + } + + td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, block_candidate->id, + std::move(stats_)); return true; } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 4c6264025c..34c164eee6 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -65,7 +65,7 @@ std::string ErrorCtx::as_string() const { * @param manager The ActorId of the ValidatorManager. * @param timeout The timeout for the validation. * @param promise The Promise to return the ValidateCandidateResult to. - * @param is_fake A boolean indicating if the validation is fake (performed when creating a hardfork). + * @param mode +1 - fake mode */ ValidateQuery::ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, Ref validator_set, diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index a728c93b96..b52aab76d8 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -54,13 +54,24 @@ struct AsyncSerializerState { }; struct CollationStats { - td::uint32 bytes, gas, lt_delta; - int cat_bytes, cat_gas, cat_lt_delta; + td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0; + td::uint32 estimated_bytes = 0, gas = 0, lt_delta = 0, estimated_collated_data_bytes = 0; + int cat_bytes = 0, cat_gas = 0, cat_lt_delta = 0, cat_collated_data_bytes = 0; std::string limits_log; td::uint32 ext_msgs_total = 0; td::uint32 ext_msgs_filtered = 0; td::uint32 ext_msgs_accepted = 0; td::uint32 ext_msgs_rejected = 0; + double work_time = 0.0, cpu_work_time = 0.0; + td::uint32 serialized_size = 0, serialized_size_no_collated_data = 0; + + tl_object_ptr tl() const { + return create_tl_object( + actual_bytes, actual_collated_data_bytes, estimated_bytes, gas, lt_delta, estimated_collated_data_bytes, + cat_bytes, cat_gas, cat_lt_delta, cat_collated_data_bytes, limits_log, ext_msgs_total, ext_msgs_filtered, + ext_msgs_accepted, ext_msgs_rejected, work_time, cpu_work_time, serialized_size, + serialized_size_no_collated_data); + } }; using ValidateCandidateResult = td::Variant; @@ -208,8 +219,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void add_lite_query_stats(int lite_query_id) { } - virtual void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, - CollationStats stats) { + virtual void record_collate_query_stats(BlockIdExt block_id, CollationStats stats) { } virtual void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { } diff --git a/validator/manager.cpp b/validator/manager.cpp index 3f59d784d4..caaa3a7962 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3099,10 +3099,7 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, tl_object_ptr collation_stats; if (it != recorded_block_stats_.end() && it->second.collator_stats_) { auto &stats = it->second.collator_stats_.value(); - collation_stats = create_tl_object( - stats.bytes, stats.gas, stats.lt_delta, stats.cat_bytes, stats.cat_gas, stats.cat_lt_delta, - stats.limits_log, stats.ext_msgs_total, stats.ext_msgs_filtered, stats.ext_msgs_accepted, - stats.ext_msgs_rejected); + collation_stats = stats.tl(); } std::string approvers, signers; for (bool x : producer.approvers) { @@ -3616,12 +3613,28 @@ td::actor::ActorOwn ValidatorManagerFactory::create( rldp, overlays); } -void ValidatorManagerImpl::record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, - CollationStats stats) { +void ValidatorManagerImpl::record_collate_query_stats(BlockIdExt block_id, CollationStats stats) { auto &record = new_block_stats_record(block_id); - record.collator_work_time_ = work_time; - record.collator_cpu_work_time_ = cpu_work_time; + record.collator_work_time_ = stats.work_time; + record.collator_cpu_work_time_ = stats.cpu_work_time; record.collator_stats_ = std::move(stats); + + std::string fname = opts_->get_session_logs_file(); + if (fname.empty()) { + return; + } + + auto obj = create_tl_object(td::Clocks::system(), + create_tl_block_id(block_id), stats.tl()); + auto s = td::json_encode(td::ToJson(*obj.get()), false); + s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); + + std::ofstream file; + file.open(fname, std::ios_base::app); + file << s << "\n"; + file.close(); + + LOG(DEBUG) << "Writing collation stats stats for " << block_id.id.to_str(); } void ValidatorManagerImpl::record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 055adf304a..e123f27e96 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -780,8 +780,7 @@ class ValidatorManagerImpl : public ValidatorManager { std::map recorded_block_stats_; std::queue recorded_block_stats_lru_; - void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, - CollationStats stats) override; + void record_collate_query_stats(BlockIdExt block_id, CollationStats stats) override; void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) override; RecordedBlockStats &new_block_stats_record(BlockIdExt block_id); diff --git a/validator/validator.h b/validator/validator.h index 3eaaaddf9b..393dc69f42 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -70,6 +70,11 @@ struct CollatorOptions : public td::CntObject { std::set> whitelist; // Prioritize these accounts on each phase of process_dispatch_queue std::set> prioritylist; + + // Always enable full_collated_data + bool force_full_collated_data = false; + // Ignore collated data size limits from block limits and catchain config + bool ignore_collated_data_limits = false; }; struct CollatorsList : public td::CntObject { From d9aeab07db471d314b6f9a5ce7641daeb91fd4eb Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 26 Nov 2024 10:53:55 +0300 Subject: [PATCH 124/524] Send telemetry broadcasts to fast sync overlays --- validator-engine/validator-engine.cpp | 2 +- validator/full-node-fast-sync-overlays.cpp | 79 ++++++++++++++++++++-- validator/full-node-fast-sync-overlays.hpp | 13 +++- validator/full-node.cpp | 68 ++++++++++++++----- validator/full-node.hpp | 2 +- validator/impl/validator-set.hpp | 1 - validator/interfaces/validator-set.h | 1 - validator/manager.cpp | 2 +- 8 files changed, 141 insertions(+), 27 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 6b91d3ebce..16aabde5f1 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -4985,7 +4985,7 @@ int main(int argc, char *argv[]) { }); p.add_option( '\0', "collect-validator-telemetry", - "store validator telemetry from private block overlay to a given file (json format)", + "store validator telemetry from fast sync overlay to a given file (json format)", [&](td::Slice s) { acts.push_back( [&x, s = s.str()]() { diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index a5789771bd..270c53f07b 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -87,9 +87,42 @@ void FullNodeFastSyncOverlay::process_block_candidate_broadcast(PublicKeyHash sr validator_set_hash, std::move(data)); } +void FullNodeFastSyncOverlay::process_telemetry_broadcast( + adnl::AdnlNodeIdShort src, const tl_object_ptr &telemetry) { + if (telemetry->adnl_id_ != src.bits256_value()) { + VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": adnl_id mismatch"; + return; + } + auto now = (td::int32)td::Clocks::system(); + if (telemetry->timestamp_ < now - 60) { + VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": too old (" + << now - telemetry->timestamp_ << "s ago)"; + return; + } + if (telemetry->timestamp_ > now + 60) { + VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": too new (" + << telemetry->timestamp_ - now << "s in the future)"; + return; + } + VLOG(FULL_NODE_DEBUG) << "Got telemetry broadcast from " << src; + auto s = td::json_encode(td::ToJson(*telemetry), false); + std::erase_if(s, [](char c) { return c == '\n' || c == '\r'; }); + telemetry_file_ << s << "\n"; + telemetry_file_.flush(); + if (telemetry_file_.fail()) { + VLOG(FULL_NODE_WARNING) << "Failed to write telemetry to file"; + } +} + void FullNodeFastSyncOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { auto B = fetch_tl_object(std::move(broadcast), true); if (B.is_error()) { + if (collect_telemetry_ && src != local_id_.pubkey_hash()) { + auto R = fetch_tl_prefix(broadcast, true); + if (R.is_ok()) { + process_telemetry_broadcast(adnl::AdnlNodeIdShort{src}, R.ok()); + } + } return; } @@ -143,6 +176,30 @@ void FullNodeFastSyncOverlay::send_block_candidate(BlockIdExt block_id, Catchain local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); } +void FullNodeFastSyncOverlay::send_validator_telemetry(tl_object_ptr telemetry) { + process_telemetry_broadcast(local_id_, telemetry); + auto data = serialize_tl_object(telemetry, true); + if (data.size() <= overlay::Overlays::max_simple_broadcast_size()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(data)); + } else { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(data)); + } +} + +void FullNodeFastSyncOverlay::collect_validator_telemetry(std::string filename) { + if (collect_telemetry_) { + telemetry_file_.close(); + } + collect_telemetry_ = true; + LOG(FULL_NODE_WARNING) << "Collecting validator telemetry to " << filename << " (local id: " << local_id_ << ")"; + telemetry_file_.open(filename, std::ios_base::app); + if (!telemetry_file_.is_open()) { + LOG(WARNING) << "Cannot open file " << filename << " for validator telemetry"; + } +} + void FullNodeFastSyncOverlay::start_up() { auto X = create_hash_tl_object(zero_state_file_hash_, create_tl_shard_id(shard_)); td::BufferSlice b{32}; @@ -261,14 +318,15 @@ void FullNodeFastSyncOverlay::get_stats_extra(td::Promise promise) promise.set_result(td::json_encode(td::ToJson(*res), true)); } -td::actor::ActorId FullNodeFastSyncOverlays::choose_overlay(ShardIdFull shard) { +std::pair, adnl::AdnlNodeIdShort> FullNodeFastSyncOverlays::choose_overlay( + ShardIdFull shard) { for (auto &p : id_to_overlays_) { auto &overlays = p.second.overlays_; ShardIdFull cur_shard = shard; while (true) { auto it = overlays.find(cur_shard); if (it != overlays.end()) { - return it->second.get(); + return {it->second.get(), p.first}; } if (cur_shard.pfx_len() == 0) { break; @@ -276,7 +334,20 @@ td::actor::ActorId FullNodeFastSyncOverlays::choose_ove cur_shard = shard_parent(cur_shard); } } - return {}; + return {td::actor::ActorId{}, adnl::AdnlNodeIdShort::zero()}; +} + +td::actor::ActorId FullNodeFastSyncOverlays::get_masterchain_overlay_for( + adnl::AdnlNodeIdShort adnl_id) { + auto it = id_to_overlays_.find(adnl_id); + if (it == id_to_overlays_.end()) { + return {}; + } + auto it2 = it->second.overlays_.find(ShardIdFull{masterchainId}); + if (it2 == it->second.overlays_.end()) { + return {}; + } + return it2->second.get(); } void FullNodeFastSyncOverlays::update_overlays(td::Ref state, @@ -291,7 +362,7 @@ void FullNodeFastSyncOverlays::update_overlays(td::Ref state, monitoring_shards.insert(ShardIdFull{masterchainId}); std::set all_shards; all_shards.insert(ShardIdFull{masterchainId}); - for (const auto& desc : state->get_shards()) { + for (const auto &desc : state->get_shards()) { ShardIdFull shard = desc->shard(); td::uint32 monitor_min_split = state->monitor_min_split_depth(shard.workchain); if (shard.pfx_len() > monitor_min_split) { diff --git a/validator/full-node-fast-sync-overlays.hpp b/validator/full-node-fast-sync-overlays.hpp index b89f880245..05d83071d7 100644 --- a/validator/full-node-fast-sync-overlays.hpp +++ b/validator/full-node-fast-sync-overlays.hpp @@ -17,6 +17,7 @@ #pragma once #include "full-node.h" +#include namespace ton::validator::fullnode { @@ -32,6 +33,9 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed& query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast& query); + void process_telemetry_broadcast(adnl::AdnlNodeIdShort src, + const tl_object_ptr& telemetry); + template void process_broadcast(PublicKeyHash, T&) { VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; @@ -42,6 +46,9 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { void send_broadcast(BlockBroadcast broadcast); void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data); + void send_validator_telemetry(tl_object_ptr telemetry); + + void collect_validator_telemetry(std::string filename); void start_up() override; void tear_down() override; @@ -96,11 +103,15 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { void try_init(); void init(); void get_stats_extra(td::Promise promise); + + bool collect_telemetry_ = false; + std::ofstream telemetry_file_; }; class FullNodeFastSyncOverlays { public: - td::actor::ActorId choose_overlay(ShardIdFull shard); + std::pair, adnl::AdnlNodeIdShort> choose_overlay(ShardIdFull shard); + td::actor::ActorId get_masterchain_overlay_for(adnl::AdnlNodeIdShort adnl_id); void update_overlays(td::Ref state, std::set my_adnl_ids, std::set monitoring_shards, const FileHash& zero_state_file_hash, const td::actor::ActorId& keyring, const td::actor::ActorId& adnl, diff --git a/validator/full-node.cpp b/validator/full-node.cpp index ed3b5bcb96..d927eef2a1 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -279,6 +279,7 @@ void FullNodeImpl::on_new_masterchain_block(td::Ref state, std fast_sync_overlays_.update_overlays(state, std::move(my_adnl_ids), std::move(monitoring_shards), zero_state_file_hash_, keyring_, adnl_, overlays_, validator_manager_, actor_id(this)); + update_validator_telemetry_collector(); } } @@ -338,7 +339,7 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_shard_block_info, block_id, cc_seqno, data.clone()); } - auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(ShardIdFull(masterchainId)); + auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(ShardIdFull(masterchainId)).first; if (!fast_sync_overlay.empty()) { td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_shard_block_info, block_id, cc_seqno, data.clone()); @@ -358,7 +359,7 @@ void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_se td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_block_candidate, block_id, cc_seqno, validator_set_hash, data.clone()); } - auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(block_id.shard_full()); + auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(block_id.shard_full()).first; if (!fast_sync_overlay.empty()) { td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_block_candidate, block_id, cc_seqno, validator_set_hash, data.clone()); @@ -383,7 +384,7 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, int mode) { td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, broadcast.clone()); } - auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(broadcast.block_id.shard_full()); + auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(broadcast.block_id.shard_full()).first; if (!fast_sync_overlay.empty()) { td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_broadcast, broadcast.clone()); } @@ -595,12 +596,21 @@ void FullNodeImpl::new_key_block(BlockHandle handle) { } void FullNodeImpl::send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) { - auto it = private_block_overlays_.find(key); - if (it == private_block_overlays_.end()) { - VLOG(FULL_NODE_INFO) << "Cannot send validator telemetry for " << key << " : no private block overlay"; - return; + if (use_old_private_overlays_) { + auto it = private_block_overlays_.find(key); + if (it == private_block_overlays_.end()) { + VLOG(FULL_NODE_INFO) << "Cannot send validator telemetry for " << key << " : no private block overlay"; + return; + } + td::actor::send_closure(it->second, &FullNodePrivateBlockOverlay::send_validator_telemetry, std::move(telemetry)); + } else { + auto overlay = fast_sync_overlays_.get_masterchain_overlay_for(adnl::AdnlNodeIdShort{telemetry->adnl_id_}); + if (overlay.empty()) { + VLOG(FULL_NODE_INFO) << "Cannot send validator telemetry for adnl id " << key << " : no fast sync overlay"; + return; + } + td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::send_validator_telemetry, std::move(telemetry)); } - td::actor::send_closure(it->second, &FullNodePrivateBlockOverlay::send_validator_telemetry, std::move(telemetry)); } void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast) { @@ -631,19 +641,43 @@ void FullNodeImpl::set_validator_telemetry_filename(std::string value) { } void FullNodeImpl::update_validator_telemetry_collector() { - if (validator_telemetry_filename_.empty() || private_block_overlays_.empty()) { - validator_telemetry_collector_key_ = PublicKeyHash::zero(); - return; - } - if (!private_block_overlays_.contains(validator_telemetry_collector_key_)) { - auto it = private_block_overlays_.begin(); - validator_telemetry_collector_key_ = it->first; - td::actor::send_closure(it->second, &FullNodePrivateBlockOverlay::collect_validator_telemetry, - validator_telemetry_filename_); + if (use_old_private_overlays_) { + if (validator_telemetry_filename_.empty() || private_block_overlays_.empty()) { + validator_telemetry_collector_key_ = PublicKeyHash::zero(); + return; + } + if (!private_block_overlays_.contains(validator_telemetry_collector_key_)) { + auto it = private_block_overlays_.begin(); + validator_telemetry_collector_key_ = it->first; + td::actor::send_closure(it->second, &FullNodePrivateBlockOverlay::collect_validator_telemetry, + validator_telemetry_filename_); + } + } else { + if (validator_telemetry_filename_.empty()) { + validator_telemetry_collector_key_ = PublicKeyHash::zero(); + return; + } + if (fast_sync_overlays_.get_masterchain_overlay_for(adnl::AdnlNodeIdShort{validator_telemetry_collector_key_}) + .empty()) { + auto [actor, adnl_id] = fast_sync_overlays_.choose_overlay(ShardIdFull{masterchainId}); + validator_telemetry_collector_key_ = adnl_id.pubkey_hash(); + if (!actor.empty()) { + td::actor::send_closure(actor, &FullNodeFastSyncOverlay::collect_validator_telemetry, + validator_telemetry_filename_); + } + } } } void FullNodeImpl::start_up() { + // TODO: enable fast sync overlays by other means (e.g. some config param) + // TODO: in the future - remove the old private overlay entirely + // This env var is for testing + auto fast_sync_env = getenv("TON_FAST_SYNC_OVERLAYS"); + if (fast_sync_env && !strcmp(fast_sync_env, "1")) { + use_old_private_overlays_ = false; + } + update_shard_actor(ShardIdFull{masterchainId}, true); if (local_id_.is_zero()) { if (adnl_id_.is_zero()) { diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 4194407d66..9e254d7d4b 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -154,7 +154,7 @@ class FullNodeImpl : public FullNode { // Old overlays - one private overlay for all validators // New overlays (fast sync overlays) - semiprivate overlay per shard (monitor_min_split depth) // for validators and authorized nodes - bool use_old_private_overlays_ = false; // TODO: set from config or something + bool use_old_private_overlays_ = true; std::map> private_block_overlays_; bool broadcast_block_candidates_in_public_overlay_ = false; FullNodeFastSyncOverlays fast_sync_overlays_; diff --git a/validator/impl/validator-set.hpp b/validator/impl/validator-set.hpp index 67fd9cd209..951ca4b715 100644 --- a/validator/impl/validator-set.hpp +++ b/validator/impl/validator-set.hpp @@ -51,7 +51,6 @@ class ValidatorSetQ : public ValidatorSet { td::Ref signatures) const override; td::Result check_approve_signatures(RootHash root_hash, FileHash file_hash, td::Ref signatures) const override; - const ValidatorDescr* find_validator(const NodeIdShort& id) const override; ValidatorSetQ* make_copy() const override; diff --git a/validator/interfaces/validator-set.h b/validator/interfaces/validator-set.h index d690937e07..ad7fb9b552 100644 --- a/validator/interfaces/validator-set.h +++ b/validator/interfaces/validator-set.h @@ -36,7 +36,6 @@ class ValidatorSet : public td::CntObject { virtual td::uint32 get_validator_set_hash() const = 0; virtual ShardId get_validator_set_from() const = 0; virtual std::vector export_vector() const = 0; - virtual const ValidatorDescr* find_validator(const NodeIdShort& id) const = 0; virtual td::Result check_signatures(RootHash root_hash, FileHash file_hash, td::Ref signatures) const = 0; virtual td::Result check_approve_signatures(RootHash root_hash, FileHash file_hash, diff --git a/validator/manager.cpp b/validator/manager.cpp index c928090f4d..4c8e6ecc89 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2475,7 +2475,7 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto validator_id = get_validator(shard, validator_set); CHECK(!validator_id.is_zero()); - auto descr = validator_set->find_validator(validator_id.bits256_value()); + auto descr = validator_set->get_validator(validator_id.bits256_value()); CHECK(descr); auto adnl_id = adnl::AdnlNodeIdShort{ descr->addr.is_zero() ? ValidatorFullId{descr->key}.compute_short_id().bits256_value() : descr->addr}; From bf572f9599f92841581de08092db46eb911651e7 Mon Sep 17 00:00:00 2001 From: birydrad <> Date: Tue, 26 Nov 2024 14:01:20 +0400 Subject: [PATCH 125/524] optimistic out-msg-queue broadcast --- tdutils/td/utils/StringBuilder.h | 15 ++ tdutils/td/utils/Timer.cpp | 39 +++- tdutils/td/utils/Timer.h | 43 ++++- tdutils/td/utils/logging.h | 10 +- tl/generate/scheme/ton_api.tl | 6 +- tl/generate/scheme/ton_api.tlo | Bin 108568 -> 108888 bytes ton/ton-types.h | 49 ++++- validator/collation-manager.cpp | 11 +- validator/collation-manager.hpp | 6 +- validator/collator-node.cpp | 119 ++++++++++-- validator/collator-node.hpp | 7 +- validator/full-node-fast-sync-overlays.cpp | 43 +++++ validator/full-node-fast-sync-overlays.hpp | 2 + validator/full-node-shard.hpp | 3 + validator/full-node.cpp | 11 ++ validator/full-node.hpp | 1 + validator/impl/collator-impl.h | 15 +- validator/impl/collator.cpp | 205 ++++++++++++++------- validator/impl/out-msg-queue-proof.cpp | 113 ++++++++++-- validator/impl/out-msg-queue-proof.hpp | 18 +- validator/manager.cpp | 17 +- validator/manager.hpp | 1 + validator/validator-group.cpp | 8 +- validator/validator.h | 7 + 24 files changed, 623 insertions(+), 126 deletions(-) diff --git a/tdutils/td/utils/StringBuilder.h b/tdutils/td/utils/StringBuilder.h index 99e9d51727..685416fe35 100644 --- a/tdutils/td/utils/StringBuilder.h +++ b/tdutils/td/utils/StringBuilder.h @@ -149,4 +149,19 @@ std::enable_if_t::value, string> to_string(const T &x) { return sb.as_cslice().str(); } +template +struct LambdaPrintHelper { + SB& sb; +}; +template +SB& operator<<(const LambdaPrintHelper& helper, F&& f) { + f(helper.sb); + return helper.sb; +} +struct LambdaPrint {}; + +inline LambdaPrintHelper operator<<(td::StringBuilder& sb, const LambdaPrint&) { + return LambdaPrintHelper{sb}; +} + } // namespace td diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index 24de099aa0..c2c6789555 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -22,6 +22,8 @@ #include "td/utils/logging.h" #include "td/utils/Time.h" +#include + namespace td { Timer::Timer(bool is_paused) : is_paused_(is_paused) { @@ -60,12 +62,15 @@ StringBuilder &operator<<(StringBuilder &string_builder, const Timer &timer) { return string_builder << format::as_time(timer.elapsed()); } -PerfWarningTimer::PerfWarningTimer(string name, double max_duration, std::function&& callback) +PerfWarningTimer::PerfWarningTimer(string name, double max_duration, std::function &&callback) : name_(std::move(name)), start_at_(Time::now()), max_duration_(max_duration), callback_(std::move(callback)) { } PerfWarningTimer::PerfWarningTimer(PerfWarningTimer &&other) - : name_(std::move(other.name_)), start_at_(other.start_at_), max_duration_(other.max_duration_), callback_(std::move(other.callback_)) { + : name_(std::move(other.name_)) + , start_at_(other.start_at_) + , max_duration_(other.max_duration_) + , callback_(std::move(other.callback_)) { other.start_at_ = 0; } @@ -134,4 +139,34 @@ double ThreadCpuTimer::elapsed() const { return res; } +PerfLogAction PerfLog::start_action(std::string name) { + auto i = entries_.size(); + entries_.push_back({.name = std::move(name), .begin = td::Timestamp::now().at()}); + return PerfLogAction{i, std::unique_ptr(this)}; +} +td::StringBuilder &operator<<(StringBuilder &sb, const PerfLog &log) { + sb << "{"; + std::vector ids(log.entries_.size()); + std::iota(ids.begin(), ids.end(), 0); + std::sort(ids.begin(), ids.end(), [&](auto a, auto b) { + return log.entries_[a].end - log.entries_[a].begin > log.entries_[b].end - log.entries_[b].begin; + }); + sb << "{"; + for (size_t i = 0; i < log.entries_.size(); i++) { + sb << "\n\t"; + auto &entry = log.entries_[ids[i]]; + sb << "{" << entry.name << ":" << entry.begin << "->" << entry.end << "(" << entry.end - entry.begin << ")" + << td::format::cond(entry.status.is_error(), entry.status, "") << "}"; + } + sb << "\n}"; + return sb; +} + +double PerfLog::finish_action(size_t i, td::Status status) { + auto &entry = entries_[i]; + CHECK(entry.end == 0); + entry.end = td::Timestamp::now().at(); + entry.status = std::move(status); + return entry.end - entry.begin; +} } // namespace td diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index a27cac8a7e..d787f1ca22 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -19,6 +19,7 @@ #pragma once #include "td/utils/StringBuilder.h" +#include "td/utils/Status.h" #include @@ -46,7 +47,7 @@ class Timer { class PerfWarningTimer { public: - explicit PerfWarningTimer(string name, double max_duration = 0.1, std::function&& callback = {}); + explicit PerfWarningTimer(string name, double max_duration = 0.1, std::function &&callback = {}); PerfWarningTimer(const PerfWarningTimer &) = delete; PerfWarningTimer &operator=(const PerfWarningTimer &) = delete; PerfWarningTimer(PerfWarningTimer &&other); @@ -80,4 +81,44 @@ class ThreadCpuTimer { bool is_paused_{false}; }; +class PerfLog; +struct EmptyDeleter { + template + void operator()(T *) { + } +}; +class PerfLogAction { + public: + template + double finish(const T &result); + size_t i_{0}; + std::unique_ptr perf_log_; +}; + +class PerfLog { + public: + PerfLogAction start_action(std::string name); + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const PerfLog &log); + + private: + struct Entry { + std::string name; + double begin{}; + double end{}; + td::Status status; + }; + std::vector entries_; + friend class PerfLogAction; + + double finish_action(size_t i, td::Status status); +}; +template +double PerfLogAction::finish(const T &result) { + if (result.is_ok()) { + return perf_log_->finish_action(i_, td::Status::OK()); + } else { + return perf_log_->finish_action(i_, result.error().clone()); + } +} + } // namespace td diff --git a/tdutils/td/utils/logging.h b/tdutils/td/utils/logging.h index d00fba1548..5c9a0621f1 100644 --- a/tdutils/td/utils/logging.h +++ b/tdutils/td/utils/logging.h @@ -74,6 +74,7 @@ #define LOG(level) LOG_IMPL(level, level, true, ::td::Slice()) #define LOG_IF(level, condition) LOG_IMPL(level, level, condition, #condition) +#define FLOG(level) LOG_IMPL(level, level, true, ::td::Slice()) << td::LambdaPrint{} << [&](auto &sb) #define VLOG(level) LOG_IMPL(DEBUG, level, true, TD_DEFINE_STR(level)) #define VLOG_IF(level, condition) LOG_IMPL(DEBUG, level, condition, TD_DEFINE_STR(level) " " #condition) @@ -95,13 +96,13 @@ inline bool no_return_func() { #define DUMMY_LOG_CHECK(condition) LOG_IF(NEVER, !(condition)) #ifdef TD_DEBUG - #if TD_MSVC +#if TD_MSVC #define LOG_CHECK(condition) \ __analysis_assume(!!(condition)); \ LOG_IMPL(FATAL, FATAL, !(condition), #condition) - #else +#else #define LOG_CHECK(condition) LOG_IMPL(FATAL, FATAL, !(condition) && no_return_func(), #condition) - #endif +#endif #else #define LOG_CHECK DUMMY_LOG_CHECK #endif @@ -263,6 +264,9 @@ class Logger { sb_ << other; return *this; } + LambdaPrintHelper operator<<(const LambdaPrint &) { + return LambdaPrintHelper{*this}; + } MutableCSlice as_cslice() { return sb_.as_cslice(); diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 5b272f268e..720f8d4964 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -433,6 +433,10 @@ tonNode.newBlockCandidateBroadcast id:tonNode.blockIdExt catchain_seqno:int vali tonNode.newBlockCandidateBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int collator_signature:tonNode.blockSignature flags:# compressed:bytes = tonNode.Broadcast; +// optimistic broadcast of response to tonNode.getOutMsgQueueProof with dst_shard, block and limits arguments +tonNode.outMsgQueueProofBroadcast dst_shard:tonNode.shardId block:tonNode.blockIdExt + limits:ImportedMsgQueueLimits proof:tonNode.OutMsgQueueProof = tonNode.Broadcast; + tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int256 = tonNode.ShardPublicOverlayId; tonNode.privateBlockOverlayId zero_state_file_hash:int256 nodes:(vector int256) = tonNode.PrivateBlockOverlayId; @@ -924,7 +928,7 @@ validatorSession.endValidatorGroupStats session_id:int256 timestamp:double ---functions--- collatorNode.generateBlock shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt) - creator:int256 = collatorNode.Candidate; + creator:int256 round:int first_block_round:int priority:int = collatorNode.Candidate; collatorNode.ping flags:# = collatorNode.Pong; ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index c984fba0b30b868520b6ea2239c9d376838d2378..19a8d13138d85dd2bea8b7e28793cbc85e441aa9 100644 GIT binary patch delta 392 zcmbPnf$hd6whensSWchfai4s^M1+xJqNDicHzp07jE6Vp+7;h}$V_-DG2K9pF=cv! zEMpB=X!-+LMiWkuwwuRFZQDVRw(SW@j2f0KCPv{;ruRfM zsxY!o-x$p}fAWHS*6DAp8RdBe(=v;SOX8Dq@{_aUr(d*Ul%L)+nNef%nG%8R88M6- FyaE5biE#h` delta 264 zcmca{iEYLOwhensSgKTm3MU^h5n*JX=qSGVjY$J1Ofcx@{BUm4dfX`AY2c5Mu=t8 zff5D~u`O`cf>}VtEDA8iG71=0Oi*NOnC$ab1)@_$k6uUgvx diff --git a/ton/ton-types.h b/ton/ton-types.h index 11741c5ec1..aeb0595ade 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -425,14 +425,47 @@ struct Ed25519_PublicKey { }; // represents (the contents of) a block + +struct OutMsgQueueProofBroadcast : public td::CntObject { + OutMsgQueueProofBroadcast(ShardIdFull dst_shard, BlockIdExt block_id, td::int32 max_bytes, td::int32 max_msgs, + td::BufferSlice queue_proofs, td::BufferSlice block_state_proofs, + std::vector msg_counts) + : dst_shard(std::move(dst_shard)) + , block_id(block_id) + , max_bytes(max_bytes) + , max_msgs(max_msgs) + , queue_proofs(std::move(queue_proofs)) + , block_state_proofs(std::move(block_state_proofs)) + , msg_counts(std::move(msg_counts)) { + } + ShardIdFull dst_shard; + BlockIdExt block_id; + + // importedMsgQueueLimits + td::uint32 max_bytes; + td::uint32 max_msgs; + + // outMsgQueueProof + td::BufferSlice queue_proofs; + td::BufferSlice block_state_proofs; + std::vector msg_counts; + + virtual OutMsgQueueProofBroadcast* make_copy() const { + return new OutMsgQueueProofBroadcast(dst_shard, block_id, max_bytes, max_msgs, queue_proofs.clone(), + block_state_proofs.clone(), msg_counts); + } +}; + struct BlockCandidate { BlockCandidate(Ed25519_PublicKey pubkey, BlockIdExt id, FileHash collated_file_hash, td::BufferSlice data, - td::BufferSlice collated_data) + td::BufferSlice collated_data, + std::vector> out_msg_queue_broadcasts = {}) : pubkey(pubkey) , id(id) , collated_file_hash(collated_file_hash) , data(std::move(data)) - , collated_data(std::move(collated_data)) { + , collated_data(std::move(collated_data)) + , out_msg_queue_proof_broadcasts(std::move(out_msg_queue_broadcasts)) { } Ed25519_PublicKey pubkey; BlockIdExt id; @@ -440,11 +473,21 @@ struct BlockCandidate { td::BufferSlice data; td::BufferSlice collated_data; + // used only locally + std::vector> out_msg_queue_proof_broadcasts; + BlockCandidate clone() const { - return BlockCandidate{pubkey, id, collated_file_hash, data.clone(), collated_data.clone()}; + return BlockCandidate{ + pubkey, id, collated_file_hash, data.clone(), collated_data.clone(), out_msg_queue_proof_broadcasts}; } }; +struct BlockCandidatePriority { + td::uint32 round{}; + td::uint32 first_block_round{}; + td::int32 priority{}; +}; + struct ValidatorDescr { /* ton::validator::ValidatorFullId */ Ed25519_PublicKey key; ValidatorWeight weight; diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index 2ca3a5f10f..e844705958 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -33,6 +33,7 @@ void CollationManager::start_up() { void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, + BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise) { if (shard.is_masterchain()) { @@ -41,12 +42,13 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha std::move(cancellation_token), 0); return; } - collate_shard_block(shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), + collate_shard_block(shard, min_masterchain_block_id, std::move(prev), creator, priority, std::move(validator_set), max_answer_size, std::move(cancellation_token), std::move(promise), td::Timestamp::in(10.0)); } void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, + BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise, td::Timestamp timeout) { @@ -133,8 +135,8 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas delay_action( [=, promise = std::move(promise)]() mutable { td::actor::send_closure(SelfId, &CollationManager::collate_shard_block, shard, min_masterchain_block_id, prev, - creator, validator_set, max_answer_size, cancellation_token, std::move(promise), - timeout); + creator, priority, validator_set, max_answer_size, cancellation_token, + std::move(promise), timeout); }, retry_at); }; @@ -145,7 +147,8 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas } td::BufferSlice query = create_serialize_tl_object( - create_tl_shard_id(shard), validator_set->get_catchain_seqno(), std::move(prev_blocks), creator.as_bits256()); + create_tl_shard_id(shard), validator_set->get_catchain_seqno(), std::move(prev_blocks), creator.as_bits256(), + priority.round, priority.first_block_round, priority.priority); LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to #" << selected_idx << "(" << selected_collator << ")"; diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp index 7ceea1e6b6..9ca69814b7 100644 --- a/validator/collation-manager.hpp +++ b/validator/collation-manager.hpp @@ -35,7 +35,8 @@ class CollationManager : public td::actor::Actor { void alarm() override; void collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - Ed25519_PublicKey creator, td::Ref validator_set, td::uint64 max_answer_size, + Ed25519_PublicKey creator, BlockCandidatePriority priority, + td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise); void update_options(td::Ref opts); @@ -52,7 +53,8 @@ class CollationManager : public td::actor::Actor { td::actor::ActorId rldp_; void collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - Ed25519_PublicKey creator, td::Ref validator_set, td::uint64 max_answer_size, + Ed25519_PublicKey creator, BlockCandidatePriority priority, + td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise, td::Timestamp timeout); diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index cf2770d934..1b8eb79e88 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -191,17 +191,41 @@ void CollatorNode::update_validator_group_info(ShardIdFull shard, std::vectorblock_seqno < info.next_block_seqno) { cache_entry->cancel(td::Status::Error(PSTRING() << "next block seqno " << cache_entry->block_seqno << " is too small, expected " << info.next_block_seqno)); + if (!cache_entry->has_external_query_at && cache_entry->has_internal_query_at) { + LOG(INFO) << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno + << ", next_block_seqno=" << cache_entry->block_seqno + << ": nobody asked for block, but we tried to generate it"; + } + if (cache_entry->has_external_query_at && !cache_entry->has_internal_query_at) { + LOG(INFO) << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno + << ", next_block_seqno=" << cache_entry->block_seqno + << ": somebody asked for block we didn't even tried to generate"; + } cache_it = info.cache.erase(cache_it); continue; } if (cache_entry->block_seqno == info.next_block_seqno && cached_prev != info.prev) { cache_entry->cancel(td::Status::Error("invalid prev blocks")); + if (!cache_entry->has_external_query_at && cache_entry->has_internal_query_at) { + LOG(INFO) << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno + << ", next_block_seqno=" << cache_entry->block_seqno + << ": nobody asked for block, but we tried to generate it"; + } + if (cache_entry->has_external_query_at && !cache_entry->has_internal_query_at) { + LOG(INFO) << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno + << ", next_block_seqno=" << cache_entry->block_seqno + << ": somebody asked for block we didn't even tried to generate"; + } cache_it = info.cache.erase(cache_it); continue; } ++cache_it; } - generate_block(shard, cc_seqno, info.prev, td::Timestamp::in(10.0), [](td::Result) {}); + generate_block(shard, cc_seqno, info.prev, {}, td::Timestamp::in(10.0), [](td::Result) {}); } return; } @@ -285,6 +309,14 @@ static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey cre cc_seqno = info.gen_catchain_seqno; val_set_hash = info.gen_validator_list_hash_short; + + for (auto& broadcast_ref : block.out_msg_queue_proof_broadcasts) { + auto block_state_proof = create_block_state_proof(root).move_as_ok(); + + auto &broadcast = broadcast_ref.write(); + broadcast.block_id = block.id; + broadcast.block_state_proofs = vm::std_boc_serialize(std::move(block_state_proof), 31).move_as_ok(); + } return block; } @@ -322,6 +354,11 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data for (const auto& b : f->prev_blocks_) { prev_blocks.push_back(create_block_id(b)); } + auto priority = BlockCandidatePriority { + .round = static_cast(f->round_), + .first_block_round = static_cast(f->first_block_round_), + .priority = f->priority_ + }; Ed25519_PublicKey creator(f->creator_); td::Promise new_promise = [promise = std::move(promise), src, shard](td::Result R) mutable { @@ -353,11 +390,13 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data return; } LOG(INFO) << "got adnl query from " << src << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno; - generate_block(shard, cc_seqno, std::move(prev_blocks), td::Timestamp::in(10.0), std::move(new_promise)); + generate_block(shard, cc_seqno, std::move(prev_blocks), priority, td::Timestamp::in(10.0), std::move(new_promise)); } void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, - td::Timestamp timeout, td::Promise promise) { + std::optional o_priority, td::Timestamp timeout, + td::Promise promise) { + bool is_external = !o_priority; if (last_masterchain_state_.is_null()) { promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); return; @@ -379,8 +418,8 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std promise.set_error(td::Status::Error(ErrorCode::timeout)); return; } - td::actor::send_closure(SelfId, &CollatorNode::generate_block, shard, cc_seqno, std::move(prev_blocks), timeout, - std::move(promise)); + td::actor::send_closure(SelfId, &CollatorNode::generate_block, shard, cc_seqno, std::move(prev_blocks), + std::move(o_priority), timeout, std::move(promise)); }); return; } @@ -399,36 +438,81 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std return; } + static auto prefix_inner = [] (auto &sb, auto &shard, auto cc_seqno, auto block_seqno, + const std::optional &o_priority) { + sb << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno; + if (o_priority) { + sb << " external{"; + sb << "round_offset=" << o_priority->round - o_priority->first_block_round << ",priority=" << o_priority->priority; + sb << ",first_block_round=" << o_priority->first_block_round; + sb << "}"; + } else { + sb << " internal" ; + } + }; + auto prefix = [&] (auto &sb) { + prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); + }; + auto cache_entry = validator_group_info.cache[prev_blocks]; if (cache_entry == nullptr) { cache_entry = validator_group_info.cache[prev_blocks] = std::make_shared(); } + if (is_external && !cache_entry->has_external_query_at) { + cache_entry->has_external_query_at = td::Timestamp::now(); + if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { + FLOG(INFO) { + prefix(sb); + sb << ": got external query " << cache_entry->has_external_query_at.at() - cache_entry->has_internal_query_at.at() + << "s after internal query [WON]"; + }; + } + } + if (!is_external && !cache_entry->has_internal_query_at) { + cache_entry->has_internal_query_at = td::Timestamp::now(); + if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { + FLOG(INFO) { + prefix(sb); + sb << ": got internal query " << cache_entry->has_internal_query_at.at() - cache_entry->has_external_query_at.at() + << "s after external query [LOST]"; + }; + } + } if (cache_entry->result) { - LOG(INFO) << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno - << ": using cached result"; + auto has_result_ago = td::Timestamp::now().at() - cache_entry->has_result_at.at(); + FLOG(INFO) { + prefix(sb); + sb << ": using cached result " << " generated " << has_result_ago << "s ago"; + sb << (is_external ? " for external query [WON]" : " for internal query "); + }; + promise.set_result(cache_entry->result.value().clone()); return; } cache_entry->promises.push_back(std::move(promise)); + if (cache_entry->started) { - LOG(INFO) << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno - << ": collation in progress, waiting"; + FLOG(INFO) { + prefix(sb); + sb << ": collation in progress, waiting"; + }; return; } - LOG(INFO) << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno - << ": starting collation"; + FLOG(INFO) { + prefix(sb); + sb << ": starting collation"; + }; cache_entry->started = true; cache_entry->block_seqno = block_seqno; run_collate_query( shard, last_masterchain_state_->get_block_id(), std::move(prev_blocks), Ed25519_PublicKey{td::Bits256::zero()}, last_masterchain_state_->get_validator_set(shard), opts_->get_collator_options(), manager_, timeout, [=, SelfId = actor_id(this), timer = td::Timer{}](td::Result R) { - LOG(INFO) << "generate block result" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno - << ", time=" << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); + FLOG(INFO) { + prefix_inner(sb, shard, cc_seqno, block_seqno,o_priority); + sb << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); + }; td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); }, cache_entry->cancellation_token_source.get_cancellation_token(), @@ -443,6 +527,7 @@ void CollatorNode::process_result(std::shared_ptr cache_entry, td::R } } else { cache_entry->result = R.move_as_ok(); + cache_entry->has_result_at = td::Timestamp::now(); for (auto& p : cache_entry->promises) { p.set_result(cache_entry->result.value().clone()); } diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index a361ceb309..54876c35fd 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -19,6 +19,7 @@ #include "interfaces/validator-manager.h" #include "rldp/rldp.h" #include +#include namespace ton::validator { @@ -57,6 +58,9 @@ class CollatorNode : public td::actor::Actor { struct CacheEntry { bool started = false; + td::Timestamp has_internal_query_at; + td::Timestamp has_external_query_at; + td::Timestamp has_result_at; BlockSeqno block_seqno = 0; td::optional result; td::CancellationTokenSource cancellation_token_source; @@ -84,7 +88,8 @@ class CollatorNode : public td::actor::Actor { td::Result get_future_validator_group(ShardIdFull shard, CatchainSeqno cc_seqno); void generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, - td::Timestamp timeout, td::Promise promise); + std::optional o_priority, td::Timestamp timeout, + td::Promise promise); void process_result(std::shared_ptr cache_entry, td::Result R); public: diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index 270c53f07b..4e0d11e489 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -46,6 +46,33 @@ void FullNodeFastSyncOverlay::process_block_broadcast(PublicKeyHash src, ton_api td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); } +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_outMsgQueueProofBroadcast &query) { + BlockIdExt block_id = create_block_id(query.block_); + ShardIdFull shard_id = create_shard_id(query.dst_shard_); + if (query.proof_->get_id() != ton_api::tonNode_outMsgQueueProof::ID) { + LOG(ERROR) << "got tonNode.outMsgQueueProofBroadcast with proof not tonNode.outMsgQueueProof"; + return; + } + auto tl_proof = move_tl_object_as(query.proof_); + auto R = OutMsgQueueProof::fetch(shard_id, {block_id}, + block::ImportedMsgQueueLimits{.max_bytes = td::uint32(query.limits_->max_bytes_), + .max_msgs = td::uint32(query.limits_->max_msgs_)}, + *tl_proof); + if (R.is_error()) { + LOG(ERROR) << "got tonNode.outMsgQueueProofBroadcast with invalid proof: " << R.error(); + return; + } + if (R.ok().size() != 1) { + LOG(ERROR) << "got tonNode.outMsgQueueProofBroadcast with invalid proofs count=" << R.ok().size(); + return; + } + auto proof = std::move(R.move_as_ok()[0]); + + LOG(INFO) << "got tonNode.outMsgQueueProofBroadcast " << shard_id.to_str() << " " << block_id.to_str(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::add_out_msg_queue_proof, shard_id, + std::move(proof)); +} + void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { BlockIdExt block_id = create_block_id(query.block_->block_); VLOG(FULL_NODE_DEBUG) << "Received newShardBlockBroadcast in fast sync overlay from " << src << ": " @@ -200,6 +227,22 @@ void FullNodeFastSyncOverlay::collect_validator_telemetry(std::string filename) } } +void FullNodeFastSyncOverlay::send_out_msg_queue_proof_broadcast(td::Ref broadcast) { + if (!inited_) { + return; + } + auto B = create_serialize_tl_object( + create_tl_shard_id(broadcast->dst_shard), create_tl_block_id(broadcast->block_id), + create_tl_object(broadcast->max_bytes, broadcast->max_msgs), + create_tl_object(broadcast->queue_proofs.clone(), + broadcast->block_state_proofs.clone(), + std::vector(broadcast->msg_counts))); + VLOG(FULL_NODE_DEBUG) << "Sending outMsgQueueProof in fast sync overlay: " << broadcast->dst_shard.to_str() << " " + << broadcast->block_id.to_str(); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); +} + void FullNodeFastSyncOverlay::start_up() { auto X = create_hash_tl_object(zero_state_file_hash_, create_tl_shard_id(shard_)); td::BufferSlice b{32}; diff --git a/validator/full-node-fast-sync-overlays.hpp b/validator/full-node-fast-sync-overlays.hpp index 05d83071d7..e0db87b7fe 100644 --- a/validator/full-node-fast-sync-overlays.hpp +++ b/validator/full-node-fast-sync-overlays.hpp @@ -25,6 +25,7 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { public: void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast& query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed& query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_outMsgQueueProofBroadcast& query); void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast& query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast& query); @@ -46,6 +47,7 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { void send_broadcast(BlockBroadcast broadcast); void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data); + void send_out_msg_queue_proof_broadcast(td::Ref broadcast); void send_validator_telemetry(tl_object_ptr telemetry); void collect_validator_telemetry(std::string filename); diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 8ac5185d91..fd1ef943e8 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -151,6 +151,9 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_outMsgQueueProofBroadcast &query) { + LOG(ERROR) << "Ignore outMsgQueueProofBroadcast"; + } void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 0278c9ae47..e7527d7685 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -371,6 +371,14 @@ void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_se } } +void FullNodeImpl::send_out_msg_queue_proof_broadcast(td::Ref broadcast) { + auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(broadcast->dst_shard).first; + if (!fast_sync_overlay.empty()) { + td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_out_msg_queue_proof_broadcast, + std::move(broadcast)); + } +} + void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, int mode) { if (mode & broadcast_mode_custom) { send_block_broadcast_to_custom_overlays(broadcast); @@ -713,6 +721,9 @@ void FullNodeImpl::start_up() { td::actor::send_closure(id_, &FullNodeImpl::send_block_candidate, block_id, cc_seqno, validator_set_hash, std::move(data)); } + void send_out_msg_queue_proof_broadcast(td::Ref broadcast) override { + td::actor::send_closure(id_, &FullNodeImpl::send_out_msg_queue_proof_broadcast, std::move(broadcast)); + } void send_broadcast(BlockBroadcast broadcast, int mode) override { td::actor::send_closure(id_, &FullNodeImpl::send_broadcast, std::move(broadcast), mode); } diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 9e254d7d4b..5533b1f431 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -70,6 +70,7 @@ class FullNodeImpl : public FullNode { void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data); void send_broadcast(BlockBroadcast broadcast, int mode); + void send_out_msg_queue_proof_broadcast(td::Ref broadcats); void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 72154f8611..0090e95dec 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -237,6 +237,7 @@ class Collator final : public td::actor::Actor { bool store_out_msg_queue_size_ = false; td::PerfWarningTimer perf_timer_; + td::PerfLog perf_log_; // block::Account* lookup_account(td::ConstBitPtr addr) const; std::unique_ptr make_account_from(td::ConstBitPtr addr, Ref account, @@ -252,18 +253,18 @@ class Collator final : public td::actor::Actor { bool fatal_error(int err_code, std::string err_msg); bool fatal_error(std::string err_msg, int err_code = -666); void check_pending(); - void after_get_mc_state(td::Result, BlockIdExt>> res); - void after_get_shard_state(int idx, td::Result> res); - void after_get_block_data(int idx, td::Result> res); - void after_get_shard_blocks(td::Result>> res); + void after_get_mc_state(td::Result, BlockIdExt>> res, td::PerfLogAction token); + void after_get_shard_state(int idx, td::Result> res, td::PerfLogAction token); + void after_get_block_data(int idx, td::Result> res, td::PerfLogAction token); + void after_get_shard_blocks(td::Result>> res, td::PerfLogAction token); bool preprocess_prev_mc_state(); bool register_mc_state(Ref other_mc_state); bool request_aux_mc_state(BlockSeqno seqno, Ref& state); Ref get_aux_mc_state(BlockSeqno seqno) const; - void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res); + void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res, td::PerfLogAction token); bool fix_one_processed_upto(block::MsgProcessedUpto& proc, const ton::ShardIdFull& owner); bool fix_processed_upto(block::MsgProcessedUptoCollection& upto); - void got_neighbor_msg_queues(td::Result>> R); + void got_neighbor_msg_queues(td::Result>> R, td::PerfLogAction token); void got_neighbor_msg_queue(unsigned i, Ref res); void got_out_queue_size(size_t i, td::Result res); bool adjust_shard_config(); @@ -309,7 +310,7 @@ class Collator final : public td::actor::Actor { bool is_our_address(Ref addr_ref) const; bool is_our_address(ton::AccountIdPrefixFull addr_prefix) const; bool is_our_address(const ton::StdSmcAddress& addr) const; - void after_get_external_messages(td::Result, int>>> res); + void after_get_external_messages(td::Result, int>>> res, td::PerfLogAction token); td::Result register_external_message_cell(Ref ext_msg, const ExtMessage::Hash& ext_hash, int priority); // td::Result register_external_message(td::Slice ext_msg_boc); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 0054ea98aa..cd97e3fcc6 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -217,25 +217,30 @@ void Collator::start_up() { // 2. learn latest masterchain state and block id LOG(DEBUG) << "sending get_top_masterchain_state_block() to Manager"; ++pending; + auto token = perf_log_.start_action("get_top_masterchain_state_block"); if (!is_hardfork_) { td::actor::send_closure_later(manager, &ValidatorManager::get_top_masterchain_state_block, - [self = get_self()](td::Result, BlockIdExt>> res) { + [self = get_self(), token = std::move(token)]( + td::Result, BlockIdExt>> res) mutable { LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; td::actor::send_closure_later(std::move(self), &Collator::after_get_mc_state, - std::move(res)); + std::move(res), std::move(token)); }); } else { - td::actor::send_closure_later( - manager, &ValidatorManager::get_shard_state_from_db_short, min_mc_block_id, - [self = get_self(), block_id = min_mc_block_id](td::Result> res) { - LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; - if (res.is_error()) { - td::actor::send_closure_later(std::move(self), &Collator::after_get_mc_state, res.move_as_error()); - } else { - td::actor::send_closure_later(std::move(self), &Collator::after_get_mc_state, - std::make_pair(Ref(res.move_as_ok()), block_id)); - } - }); + td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db_short, min_mc_block_id, + [self = get_self(), block_id = min_mc_block_id, + token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; + if (res.is_error()) { + td::actor::send_closure_later(std::move(self), &Collator::after_get_mc_state, + res.move_as_error(), std::move(token)); + } else { + td::actor::send_closure_later( + std::move(self), &Collator::after_get_mc_state, + std::make_pair(Ref(res.move_as_ok()), block_id), + std::move(token)); + } + }); } } // 3. load previous block(s) and corresponding state(s) @@ -245,23 +250,27 @@ void Collator::start_up() { // 3.1. load state LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), - timeout, [self = get_self(), i](td::Result> res) { - LOG(DEBUG) << "got answer to wait_block_state query #" << i; - td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state, i, - std::move(res)); - }); + auto token = perf_log_.start_action(PSTRING() << "wait_block_state #" << i); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), timeout, + [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query #" << i; + td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state, i, std::move(res), + std::move(token)); + }); if (prev_blocks[i].seqno()) { // 3.2. load block // NB: we need the block itself only for extracting start_lt and end_lt to create correct prev_blk:ExtBlkRef and related Merkle proofs LOG(DEBUG) << "sending wait_block_data() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_data_short, prev_blocks[i], priority(), - timeout, [self = get_self(), i](td::Result> res) { - LOG(DEBUG) << "got answer to wait_block_data query #" << i; - td::actor::send_closure_later(std::move(self), &Collator::after_get_block_data, i, - std::move(res)); - }); + auto token = perf_log_.start_action(PSTRING() << "wait_block_data #" << i); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_data_short, prev_blocks[i], priority(), timeout, + [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_data query #" << i; + td::actor::send_closure_later(std::move(self), &Collator::after_get_block_data, i, std::move(res), + std::move(token)); + }); } } if (is_hardfork_) { @@ -271,22 +280,28 @@ void Collator::start_up() { if (!is_hardfork_) { LOG(DEBUG) << "sending get_external_messages() query to Manager"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::get_external_messages, shard_, - [self = get_self()](td::Result, int>>> res) -> void { + auto token = perf_log_.start_action("get_external_messages"); + td::actor::send_closure_later( + manager, &ValidatorManager::get_external_messages, shard_, + [self = get_self(), + token = std::move(token)](td::Result, int>>> res) mutable -> void { LOG(DEBUG) << "got answer to get_external_messages() query"; - td::actor::send_closure_later(std::move(self), &Collator::after_get_external_messages, std::move(res)); + td::actor::send_closure_later(std::move(self), &Collator::after_get_external_messages, std::move(res), + std::move(token)); }); } if (is_masterchain() && !is_hardfork_) { // 5. load shard block info messages LOG(DEBUG) << "sending get_shard_blocks_for_collator() query to Manager"; ++pending; - td::actor::send_closure_later( - manager, &ValidatorManager::get_shard_blocks_for_collator, prev_blocks[0], - [self = get_self()](td::Result>> res) -> void { - LOG(DEBUG) << "got answer to get_shard_blocks_for_collator() query"; - td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_blocks, std::move(res)); - }); + auto token = perf_log_.start_action("get_shard_blocks_for_collator"); + td::actor::send_closure_later(manager, &ValidatorManager::get_shard_blocks_for_collator, prev_blocks[0], + [self = get_self(), token = std::move(token)]( + td::Result>> res) mutable -> void { + LOG(DEBUG) << "got answer to get_shard_blocks_for_collator() query"; + td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_blocks, + std::move(res), std::move(token)); + }); } // 6. set timeout alarm_timestamp() = timeout; @@ -362,6 +377,8 @@ bool Collator::fatal_error(td::Status error) { td::Timestamp::in(10.0), std::move(main_promise), std::move(cancellation_token_), mode_, attempt_idx_ + 1); } else { + LOG(INFO) << "collation failed in " << perf_timer_.elapsed() << " s " << error; + LOG(INFO) << perf_log_; main_promise(std::move(error)); } busy_ = false; @@ -482,12 +499,14 @@ bool Collator::request_aux_mc_state(BlockSeqno seqno, Ref& st CHECK(blkid.is_valid_ext() && blkid.is_masterchain()); LOG(DEBUG) << "sending auxiliary wait_block_state() query for " << blkid.to_str() << " to Manager"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout, - [self = get_self(), blkid](td::Result> res) { - LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str(); - td::actor::send_closure_later(std::move(self), &Collator::after_get_aux_shard_state, - blkid, std::move(res)); - }); + auto token = perf_log_.start_action(PSTRING() << "auxiliary wait_block_state " << blkid.to_str()); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout, + [self = get_self(), blkid, token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str(); + td::actor::send_closure_later(std::move(self), &Collator::after_get_aux_shard_state, blkid, std::move(res), + std::move(token)); + }); state.clear(); return true; } @@ -515,9 +534,11 @@ Ref Collator::get_aux_mc_state(BlockSeqno seqno) const { * @param blkid The BlockIdExt of the shard state. * @param res The result of retrieving the shard state. */ -void Collator::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res) { +void Collator::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res, + td::PerfLogAction token) { LOG(DEBUG) << "in Collator::after_get_aux_shard_state(" << blkid.to_str() << ")"; --pending; + token.finish(res); if (res.is_error()) { fatal_error("cannot load auxiliary masterchain state for "s + blkid.to_str() + " : " + res.move_as_error().to_string()); @@ -579,9 +600,11 @@ bool Collator::preprocess_prev_mc_state() { * * @param res The retrieved masterchain state. */ -void Collator::after_get_mc_state(td::Result, BlockIdExt>> res) { +void Collator::after_get_mc_state(td::Result, BlockIdExt>> res, + td::PerfLogAction token) { LOG(WARNING) << "in Collator::after_get_mc_state()"; --pending; + token.finish(res); if (res.is_error()) { fatal_error(res.move_as_error()); return; @@ -598,12 +621,14 @@ void Collator::after_get_mc_state(td::Result, Bl // NB. it is needed only for creating a correct ExtBlkRef reference to it, which requires start_lt and end_lt LOG(DEBUG) << "sending wait_block_data() query #-1 for " << mc_block_id_.to_str() << " to Manager"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_data_short, mc_block_id_, priority(), timeout, - [self = get_self()](td::Result> res) { - LOG(DEBUG) << "got answer to wait_block_data query #-1"; - td::actor::send_closure_later(std::move(self), &Collator::after_get_block_data, -1, - std::move(res)); - }); + auto token = perf_log_.start_action("wait_block_data #-1"); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_data_short, mc_block_id_, priority(), timeout, + [self = get_self(), token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_data query #-1"; + td::actor::send_closure_later(std::move(self), &Collator::after_get_block_data, -1, std::move(res), + std::move(token)); + }); } check_pending(); } @@ -614,9 +639,10 @@ void Collator::after_get_mc_state(td::Result, Bl * @param idx The index of the previous shard block (0 or 1). * @param res The retrieved shard state. */ -void Collator::after_get_shard_state(int idx, td::Result> res) { +void Collator::after_get_shard_state(int idx, td::Result> res, td::PerfLogAction token) { LOG(WARNING) << "in Collator::after_get_shard_state(" << idx << ")"; --pending; + token.finish(res); if (res.is_error()) { fatal_error(res.move_as_error()); return; @@ -647,9 +673,10 @@ void Collator::after_get_shard_state(int idx, td::Result> res) { * @param idx The index of the previous block (0 or 1). * @param res The retreved block data. */ -void Collator::after_get_block_data(int idx, td::Result> res) { +void Collator::after_get_block_data(int idx, td::Result> res, td::PerfLogAction token) { LOG(DEBUG) << "in Collator::after_get_block_data(" << idx << ")"; --pending; + token.finish(res); if (res.is_error()) { fatal_error(res.move_as_error()); return; @@ -691,8 +718,10 @@ void Collator::after_get_block_data(int idx, td::Result> res) { * * @param res The retrieved shard block descriptions. */ -void Collator::after_get_shard_blocks(td::Result>> res) { +void Collator::after_get_shard_blocks(td::Result>> res, + td::PerfLogAction token) { --pending; + token.finish(res); if (res.is_error()) { fatal_error(res.move_as_error()); return; @@ -848,10 +877,13 @@ bool Collator::request_neighbor_msg_queues() { ++i; } ++pending; + auto token = perf_log_.start_action("neighbor_msg_queues"); td::actor::send_closure_later( manager, &ValidatorManager::wait_neighbor_msg_queue_proofs, shard_, std::move(top_blocks), timeout, - [self = get_self()](td::Result>> res) { - td::actor::send_closure_later(std::move(self), &Collator::got_neighbor_msg_queues, std::move(res)); + [self = get_self(), + token = std::move(token)](td::Result>> res) mutable { + td::actor::send_closure_later(std::move(self), &Collator::got_neighbor_msg_queues, std::move(res), + std::move(token)); }); return true; } @@ -883,13 +915,15 @@ bool Collator::request_out_msg_queue_size() { * @param i The index of the neighbor. * @param res The obtained outbound queue. */ -void Collator::got_neighbor_msg_queues(td::Result>> R) { +void Collator::got_neighbor_msg_queues(td::Result>> R, + td::PerfLogAction token) { --pending; + double duration = token.finish(R); if (R.is_error()) { fatal_error(R.move_as_error_prefix("failed to get neighbor msg queues: ")); return; } - LOG(INFO) << "neighbor output queues fetched"; + LOG(INFO) << "neighbor output queues fetched, took " << duration << "s"; auto res = R.move_as_ok(); unsigned i = 0; for (block::McShardDescr& descr : neighbors_) { @@ -2091,12 +2125,9 @@ bool Collator::init_lt() { * @returns True if the configuration parameters were successfully fetched and initialized, false otherwise. */ bool Collator::fetch_config_params() { - auto res = block::FetchConfigParams::fetch_config_params(*config_, - &old_mparams_, &storage_prices_, &storage_phase_cfg_, - &rand_seed_, &compute_phase_cfg_, &action_phase_cfg_, - &masterchain_create_fee_, &basechain_create_fee_, - workchain(), now_ - ); + auto res = block::FetchConfigParams::fetch_config_params( + *config_, &old_mparams_, &storage_prices_, &storage_phase_cfg_, &rand_seed_, &compute_phase_cfg_, + &action_phase_cfg_, &masterchain_create_fee_, &basechain_create_fee_, workchain(), now_); if (res.is_error()) { return fatal_error(res.move_as_error()); } @@ -2217,6 +2248,11 @@ bool Collator::init_value_create() { bool Collator::do_collate() { // After do_collate started it will not be interrupted by timeout alarm_timestamp() = td::Timestamp::never(); + auto token = perf_log_.start_action("do_collate"); + td::Status status = td::Status::Error("some error"); + SCOPE_EXIT { + token.finish(status); + }; LOG(WARNING) << "do_collate() : start"; if (!fetch_config_params()) { @@ -2342,6 +2378,7 @@ bool Collator::do_collate() { if (!create_block_candidate()) { return fatal_error("cannot serialize a new Block candidate"); } + status = td::Status::OK(); return true; } @@ -5811,12 +5848,43 @@ bool Collator::create_block_candidate() { << block_limit_status_->transactions; LOG(INFO) << "serialized collated data size " << cdata_slice.size() << " bytes (preliminary estimate was " << block_limit_status_->collated_data_stat.estimate_proof_size() << ")"; + auto new_block_id_ext = ton::BlockIdExt{ton::BlockId{shard_, new_block_seqno}, new_block->get_hash().bits(), + block::compute_file_hash(blk_slice.as_slice())}; // 3. create a BlockCandidate - block_candidate = std::make_unique( - created_by_, - ton::BlockIdExt{ton::BlockId{shard_, new_block_seqno}, new_block->get_hash().bits(), - block::compute_file_hash(blk_slice.as_slice())}, - block::compute_file_hash(cdata_slice.as_slice()), blk_slice.clone(), cdata_slice.clone()); + block_candidate = + std::make_unique(created_by_, new_block_id_ext, block::compute_file_hash(cdata_slice.as_slice()), + blk_slice.clone(), cdata_slice.clone()); + const bool need_out_msg_queue_broadcasts = true; + if (need_out_msg_queue_broadcasts) { + // we can't generate two proofs at the same time for the same root (it is not currently supported by cells) + // so we have can't reuse new state and have to regenerate it with merkle update + auto new_state = vm::MerkleUpdate::apply(prev_state_root_pure_, state_update); + CHECK(new_state.not_null()); + CHECK(new_state->get_hash() == state_root->get_hash()); + assert(config_ && shard_conf_); + auto neighbor_list = shard_conf_->get_neighbor_shard_hash_ids(shard_); + LOG(INFO) << "Build OutMsgQueueProofs for " << neighbor_list.size() << " neighbours"; + for (ton::BlockId blk_id : neighbor_list) { + auto prefix = blk_id.shard_full(); + auto limits = mc_state_->get_imported_msg_queue_limits(blk_id.workchain); + + // one could use monitor_min_split_depth here, to decrease number of broadcasts + // but current implementation OutMsgQueueImporter doesn't support it + + auto r_proof = OutMsgQueueProof::build( + prefix, {OutMsgQueueProof::OneBlock{.id = new_block_id_ext, .state_root = new_state, .block_root = new_block}}, limits); + if (r_proof.is_ok()) { + auto proof = r_proof.move_as_ok(); + block_candidate->out_msg_queue_proof_broadcasts.push_back(td::Ref( + true, OutMsgQueueProofBroadcast(prefix, new_block_id_ext, limits.max_bytes, limits.max_msgs, + std::move(proof->queue_proofs_), std::move(proof->block_state_proofs_), + std::move(proof->msg_counts_)))); + } else { + LOG(ERROR) << "Failed to build OutMsgQueueProof: " << r_proof.error(); + } + } + } + // 3.1 check block and collated data size auto consensus_config = config_->get_consensus_config(); if (block_candidate->data.size() > consensus_config.max_block_size) { @@ -5896,6 +5964,7 @@ void Collator::return_block_candidate(td::Result saved) { CHECK(block_candidate); LOG(WARNING) << "sending new BlockCandidate to Promise"; LOG(WARNING) << "collation took " << perf_timer_.elapsed() << " s"; + LOG(WARNING) << perf_log_; main_promise(block_candidate->clone()); busy_ = false; stop(); @@ -5974,9 +6043,11 @@ td::Result Collator::register_external_message_cell(Ref ext_msg, * * @param res The result of the external message retrieval operation. */ -void Collator::after_get_external_messages(td::Result, int>>> res) { +void Collator::after_get_external_messages(td::Result, int>>> res, + td::PerfLogAction token) { // res: pair {ext msg, priority} --pending; + token.finish(res); if (res.is_error()) { fatal_error(res.move_as_error()); return; diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 444d5846f9..4e8802c7e5 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -320,13 +320,23 @@ void OutMsgQueueImporter::get_neighbor_msg_queue_proofs( if (prefix.pfx_len() > min_split) { prefix = shard_prefix(prefix, min_split); } - new_queries[prefix].push_back(block); + + LOG(INFO) << "search for out msg queue proof " << prefix.to_str() << block.to_str(); + auto& small_entry = small_cache_[std::make_pair(dst_shard, block)]; + if (!small_entry.result.is_null()) { + entry->result[block] = small_entry.result; + entry->from_small_cache++; + alarm_timestamp().relax(small_entry.timeout = td::Timestamp::in(CACHE_TTL)); + } else { + small_entry.pending_entries.push_back(entry); + ++entry->pending; + new_queries[prefix].push_back(block); + } } }; auto limits = last_masterchain_state_->get_imported_msg_queue_limits(dst_shard.workchain); for (auto& p : new_queries) { for (size_t i = 0; i < p.second.size(); i += 16) { - ++entry->pending; size_t j = std::min(i + 16, p.second.size()); get_proof_import(entry, std::vector(p.second.begin() + i, p.second.begin() + j), limits * (td::uint32)(j - i)); @@ -355,7 +365,7 @@ void OutMsgQueueImporter::get_proof_local(std::shared_ptr entry, Blo if (block.seqno() == 0) { std::vector> proof = { td::Ref(true, block, state->root_cell(), td::Ref{})}; - td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, std::move(proof)); + td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, std::move(proof), ProofSource::Local); return; } td::actor::send_closure( @@ -371,7 +381,8 @@ void OutMsgQueueImporter::get_proof_local(std::shared_ptr entry, Blo Ref block_state_proof = create_block_state_proof(R.ok()->root_cell()).move_as_ok(); std::vector> proof = { td::Ref(true, block, state->root_cell(), std::move(block_state_proof))}; - td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, std::move(proof)); + td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, std::move(proof), + ProofSource::Local); }); }); } @@ -395,27 +406,70 @@ void OutMsgQueueImporter::get_proof_import(std::shared_ptr entry, st retry_after); return; } - td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, R.move_as_ok()); + td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, R.move_as_ok(), ProofSource::Query); }); } -void OutMsgQueueImporter::got_proof(std::shared_ptr entry, std::vector> proofs) { +void OutMsgQueueImporter::got_proof(std::shared_ptr entry, std::vector> proofs, + ProofSource proof_source) { if (!check_timeout(entry)) { return; } + // TODO: maybe save proof to small cache? It would allow other queries to reuse this result for (auto& p : proofs) { - entry->result[p->block_id_] = std::move(p); - } - CHECK(entry->pending > 0); - if (--entry->pending == 0) { - finish_query(entry); + auto block_id = p->block_id_; + if (entry->result.emplace(block_id, std::move(p)).second) { + CHECK(entry->pending > 0); + switch (proof_source) { + case ProofSource::SmallCache: + entry->from_small_cache++; + break; + case ProofSource::Broadcast: + entry->from_broadcast++; + break; + case ProofSource::Query: + entry->from_query++; + break; + case ProofSource::Local: + entry->from_local++; + break; + } + if (--entry->pending == 0) { + finish_query(entry); + } + } } } void OutMsgQueueImporter::finish_query(std::shared_ptr entry) { - LOG(DEBUG) << "Done importing neighbor msg queues for shard " << entry->dst_shard.to_str() << ", " - << entry->blocks.size() << " blocks in " << entry->timer.elapsed() << "s"; + FLOG(INFO) { + sb << "Done importing neighbor msg queues for shard " << entry->dst_shard.to_str() << ", " << entry->blocks.size() + << " blocks in " << entry->timer.elapsed() << "s"; + sb << " sources{"; + if (entry->from_broadcast) { + sb << " broadcast=" << entry->from_broadcast; + } + if (entry->from_small_cache) { + sb << " small_cache=" << entry->from_small_cache; + } + if (entry->from_local) { + sb << " local=" << entry->from_local; + } + if (entry->from_query) { + sb << " query=" << entry->from_query; + } + sb << "}"; + + if (!small_cache_.empty()) { + sb << " small_cache_size=" << small_cache_.size(); + } + if (!cache_.empty()) { + sb << " cache_size=" << cache_.size(); + } + }; + entry->done = true; + CHECK(entry->blocks.size() == entry->result.size()); alarm_timestamp().relax(entry->timeout = td::Timestamp::in(CACHE_TTL)); for (auto& p : entry->promises) { p.first.set_result(entry->result); @@ -441,8 +495,7 @@ bool OutMsgQueueImporter::check_timeout(std::shared_ptr entry) { } void OutMsgQueueImporter::alarm() { - auto it = cache_.begin(); - while (it != cache_.end()) { + for (auto it = cache_.begin(); it != cache_.end();) { auto& promises = it->second->promises; if (it->second->timeout.is_in_past()) { if (!it->second->done) { @@ -469,6 +522,36 @@ void OutMsgQueueImporter::alarm() { promises.resize(j); ++it; } + + for (auto it = small_cache_.begin(); it != small_cache_.end();) { + td::remove_if(it->second.pending_entries, + [](const std::shared_ptr& entry) { return entry->done || entry->promises.empty(); }); + if (it->second.timeout.is_in_past()) { + if (it->second.pending_entries.empty()) { + it = small_cache_.erase(it); + } else { + ++it; + } + } else { + alarm_timestamp().relax(it->second.timeout); + ++it; + } + } +} + +void OutMsgQueueImporter::add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref proof) { + LOG(INFO) << "add out msg queue proof " << dst_shard.to_str() << proof->block_id_.to_str(); + auto& small_entry = small_cache_[std::make_pair(dst_shard, proof->block_id_)]; + if (!small_entry.result.is_null()) { + return; + } + alarm_timestamp().relax(small_entry.timeout = td::Timestamp::in(CACHE_TTL)); + small_entry.result = proof; + CHECK(proof.not_null()); + auto pending_entries = std::move(small_entry.pending_entries); + for (auto& entry : pending_entries) { + got_proof(entry, {proof}, ProofSource::Broadcast); + } } void BuildOutMsgQueueProof::abort_query(td::Status reason) { diff --git a/validator/impl/out-msg-queue-proof.hpp b/validator/impl/out-msg-queue-proof.hpp index c115a27630..6f4c08d1de 100644 --- a/validator/impl/out-msg-queue-proof.hpp +++ b/validator/impl/out-msg-queue-proof.hpp @@ -41,6 +41,7 @@ class OutMsgQueueImporter : public td::actor::Actor { void new_masterchain_block_notification(td::Ref state, std::set collating_shards); void get_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, td::Promise>> promise); + void add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref proof); void update_options(td::Ref opts) { opts_ = std::move(opts); @@ -62,13 +63,28 @@ class OutMsgQueueImporter : public td::actor::Actor { td::Timestamp timeout = td::Timestamp::never(); td::Timer timer; size_t pending = 0; + size_t from_small_cache = 0; + size_t from_broadcast = 0; + size_t from_query = 0; + size_t from_local = 0; }; std::map>, std::shared_ptr> cache_; + // This cache has smaller granularity, proof is stored for each block separately + struct SmallCacheEntry { + td::Ref result; + std::vector> pending_entries; + td::Timestamp timeout = td::Timestamp::never(); + }; + std::map, SmallCacheEntry> small_cache_; + void get_proof_local(std::shared_ptr entry, BlockIdExt block); void get_proof_import(std::shared_ptr entry, std::vector blocks, block::ImportedMsgQueueLimits limits); - void got_proof(std::shared_ptr entry, std::vector> proofs); + enum class ProofSource { + SmallCache, Broadcast, Query, Local + }; + void got_proof(std::shared_ptr entry, std::vector> proofs, ProofSource proof_source); void finish_query(std::shared_ptr entry); bool check_timeout(std::shared_ptr entry); diff --git a/validator/manager.cpp b/validator/manager.cpp index d4ea681011..3ff33d234a 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1439,6 +1439,11 @@ void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate can if (!id.is_masterchain()) { add_cached_block_candidate(ReceivedBlock{id, candidate.data.clone()}); } + LOG(INFO) << "Got candidate " << id.to_str() << " with " << candidate.out_msg_queue_proof_broadcasts.size() + << " out msg queue proof broadcasts"; + for (auto broadcast : candidate.out_msg_queue_proof_broadcasts) { + callback_->send_out_msg_queue_proof_broadcast(broadcast); + } td::actor::send_closure(db_, &Db::store_block_candidate, std::move(candidate), std::move(promise)); } @@ -3519,7 +3524,7 @@ void ValidatorManagerImpl::del_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh } else { td::actor::send_closure(it->second.actor, &CollatorNode::del_shard, shard); } -} +}; void ValidatorManagerImpl::get_collation_manager_stats( td::Promise> promise) { @@ -3569,6 +3574,16 @@ void ValidatorManagerImpl::get_collation_manager_stats( td::actor::send_closure(callback, &Cb::dec_pending); } +void ValidatorManagerImpl::add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref proof) { + if (!collator_nodes_.empty()) { + if (out_msg_queue_importer_.empty()) { + out_msg_queue_importer_ = td::actor::create_actor("outmsgqueueimporter", actor_id(this), + opts_, last_masterchain_state_); + } + td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::add_out_msg_queue_proof, dst_shard, + std::move(proof)); + } +} void ValidatorManagerImpl::add_persistent_state_description(td::Ref desc) { auto now = (UnixTime)td::Clocks::system(); if (desc->end_time <= now) { diff --git a/validator/manager.hpp b/validator/manager.hpp index c0d74456f1..649d51014c 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -640,6 +640,7 @@ class ValidatorManagerImpl : public ValidatorManager { void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; + void add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref proof) override; void get_collation_manager_stats( td::Promise> promise) override; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index f95266b33b..344f5c1c33 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -66,8 +66,14 @@ void ValidatorGroup::generate_block_candidate( std::move(R)); }; td::uint64 max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; + auto block_candidate_priority = BlockCandidatePriority{ + .round = source_info.round, + .first_block_round = source_info.first_block_round, + .priority = source_info.source_priority + }; td::actor::send_closure(collation_manager_, &CollationManager::collate_block, shard_, min_masterchain_block_id_, - prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_, + prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, + block_candidate_priority, validator_set_, max_answer_size, cancellation_token_source_.get_cancellation_token(), std::move(P)); } diff --git a/validator/validator.h b/validator/validator.h index 2cd7710211..c4a779cfde 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -198,6 +198,9 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data) = 0; virtual void send_broadcast(BlockBroadcast broadcast, int mode) = 0; + virtual void send_out_msg_queue_proof_broadcast(td::Ref broadcats) { + LOG(ERROR) << "Unimplemented send_out_msg_queue_proof_broadcast - ignore broadcast"; + } virtual void download_block(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; virtual void download_zero_state(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, @@ -325,6 +328,10 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; virtual void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; + virtual void add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref proof) { + LOG(ERROR) << "Unimplemented add_out_msg_queu_proof - ignore broadcast"; + } + virtual void get_collation_manager_stats( td::Promise> promise) = 0; }; From 5d79855c94711ad161b41b75c18edff5396be842 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 27 Nov 2024 18:12:23 +0300 Subject: [PATCH 126/524] Out msg queues: improve logs, various small changes --- ton/ton-types.h | 15 ++--- validator-session/validator-session-types.h | 3 +- validator-session/validator-session.cpp | 18 ++--- validator/full-node-fast-sync-overlays.cpp | 13 ++-- validator/full-node-shard.cpp | 9 ++- validator/impl/collator.cpp | 18 +++-- validator/impl/out-msg-queue-proof.cpp | 75 +++++++++++---------- validator/manager.cpp | 16 +++-- validator/validator-group.cpp | 18 ++--- 9 files changed, 106 insertions(+), 79 deletions(-) diff --git a/ton/ton-types.h b/ton/ton-types.h index aeb0595ade..f8eb49df1c 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -428,15 +428,14 @@ struct Ed25519_PublicKey { struct OutMsgQueueProofBroadcast : public td::CntObject { OutMsgQueueProofBroadcast(ShardIdFull dst_shard, BlockIdExt block_id, td::int32 max_bytes, td::int32 max_msgs, - td::BufferSlice queue_proofs, td::BufferSlice block_state_proofs, - std::vector msg_counts) + td::BufferSlice queue_proof, td::BufferSlice block_state_proof, int msg_count) : dst_shard(std::move(dst_shard)) , block_id(block_id) , max_bytes(max_bytes) , max_msgs(max_msgs) - , queue_proofs(std::move(queue_proofs)) - , block_state_proofs(std::move(block_state_proofs)) - , msg_counts(std::move(msg_counts)) { + , queue_proofs(std::move(queue_proof)) + , block_state_proofs(std::move(block_state_proof)) + , msg_count(std::move(msg_count)) { } ShardIdFull dst_shard; BlockIdExt block_id; @@ -448,11 +447,11 @@ struct OutMsgQueueProofBroadcast : public td::CntObject { // outMsgQueueProof td::BufferSlice queue_proofs; td::BufferSlice block_state_proofs; - std::vector msg_counts; + int msg_count; - virtual OutMsgQueueProofBroadcast* make_copy() const { + OutMsgQueueProofBroadcast* make_copy() const override { return new OutMsgQueueProofBroadcast(dst_shard, block_id, max_bytes, max_msgs, queue_proofs.clone(), - block_state_proofs.clone(), msg_counts); + block_state_proofs.clone(), msg_count); } }; diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index 7147bf2d36..2edc42bbdf 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -179,9 +179,8 @@ struct EndValidatorGroupStats { }; struct BlockSourceInfo { - td::uint32 round, first_block_round; PublicKey source; - td::int32 source_priority; + BlockCandidatePriority priority; }; } // namespace validatorsession diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 3a913990cb..99ee61f23a 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -553,9 +553,9 @@ void ValidatorSessionImpl::check_generate_slot() { LOG(WARNING) << print_id << ": failed to generate block candidate: " << R.move_as_error(); } }); - callback_->on_generate_slot( - BlockSourceInfo{cur_round_, first_block_round_, description().get_source_public_key(local_idx()), priority}, - std::move(P)); + callback_->on_generate_slot(BlockSourceInfo{description().get_source_public_key(local_idx()), + BlockCandidatePriority{cur_round_, first_block_round_, priority}}, + std::move(P)); } else { alarm_timestamp().relax(t); } @@ -634,8 +634,9 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { pending_approve_.insert(block_id); callback_->on_candidate( - BlockSourceInfo{cur_round_, first_block_round_, description().get_source_public_key(block->get_src_idx()), - description().get_node_priority(block->get_src_idx(), cur_round_)}, + BlockSourceInfo{description().get_source_public_key(block->get_src_idx()), + BlockCandidatePriority{cur_round_, first_block_round_, + description().get_node_priority(block->get_src_idx(), cur_round_)}}, B->root_hash_, B->data_.clone(), B->collated_data_.clone(), std::move(P)); } else if (T.is_in_past()) { if (!active_requests_.count(block_id)) { @@ -909,9 +910,10 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { stats.rounds.pop_back(); } - BlockSourceInfo source_info{cur_round_, first_block_round_, - description().get_source_public_key(block->get_src_idx()), - description().get_node_priority(block->get_src_idx(), cur_round_)}; + BlockSourceInfo source_info{ + description().get_source_public_key(block->get_src_idx()), + BlockCandidatePriority{cur_round_, first_block_round_, + description().get_node_priority(block->get_src_idx(), cur_round_)}}; if (it == blocks_.end()) { callback_->on_block_committed(std::move(source_info), block->get_root_hash(), block->get_file_hash(), td::BufferSlice(), std::move(export_sigs), std::move(export_approve_sigs), diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index 4e0d11e489..947f9dee8c 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -47,6 +47,9 @@ void FullNodeFastSyncOverlay::process_block_broadcast(PublicKeyHash src, ton_api } void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_outMsgQueueProofBroadcast &query) { + if (src == local_id_.pubkey_hash()) { + return; // dropping broadcast from self + } BlockIdExt block_id = create_block_id(query.block_); ShardIdFull shard_id = create_shard_id(query.dst_shard_); if (query.proof_->get_id() != ton_api::tonNode_outMsgQueueProof::ID) { @@ -68,7 +71,8 @@ void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonN } auto proof = std::move(R.move_as_ok()[0]); - LOG(INFO) << "got tonNode.outMsgQueueProofBroadcast " << shard_id.to_str() << " " << block_id.to_str(); + LOG(INFO) << "got tonNode.outMsgQueueProofBroadcast to " << shard_id.to_str() << " from " << block_id.to_str() + << ", msgs=" << proof->msg_count_ << ", size=" << tl_proof->queue_proofs_.size(); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::add_out_msg_queue_proof, shard_id, std::move(proof)); } @@ -236,9 +240,10 @@ void FullNodeFastSyncOverlay::send_out_msg_queue_proof_broadcast(td::Ref(broadcast->max_bytes, broadcast->max_msgs), create_tl_object(broadcast->queue_proofs.clone(), broadcast->block_state_proofs.clone(), - std::vector(broadcast->msg_counts))); - VLOG(FULL_NODE_DEBUG) << "Sending outMsgQueueProof in fast sync overlay: " << broadcast->dst_shard.to_str() << " " - << broadcast->block_id.to_str(); + std::vector(1, broadcast->msg_count))); + VLOG(FULL_NODE_DEBUG) << "Sending outMsgQueueProof in fast sync overlay to " << broadcast->dst_shard.to_str() + << " from " << broadcast->block_id.to_str() << ", msgs=" << broadcast->msg_count + << " bytes=" << broadcast->queue_proofs.size(); td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); } diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 29950363de..c351653f4e 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -703,8 +703,13 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_result(serialize_tl_object(R.move_as_ok(), true)); } }); - VLOG(FULL_NODE_DEBUG) << "Got query getOutMsgQueueProof (" << blocks.size() << " blocks) to shard " - << dst_shard.to_str() << " from " << src; + FLOG(DEBUG) { + sb << "Got query getOutMsgQueueProof to shard " << dst_shard.to_str() << " from blocks"; + for (const BlockIdExt &id : blocks) { + sb << " " << id.id.to_str(); + } + sb << " from " << src; + }; td::actor::create_actor("buildqueueproof", dst_shard, std::move(blocks), limits, validator_manager_, std::move(P)) .release(); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index cd97e3fcc6..dd8d551692 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -5854,33 +5854,39 @@ bool Collator::create_block_candidate() { block_candidate = std::make_unique(created_by_, new_block_id_ext, block::compute_file_hash(cdata_slice.as_slice()), blk_slice.clone(), cdata_slice.clone()); - const bool need_out_msg_queue_broadcasts = true; + const bool need_out_msg_queue_broadcasts = !is_masterchain(); if (need_out_msg_queue_broadcasts) { // we can't generate two proofs at the same time for the same root (it is not currently supported by cells) // so we have can't reuse new state and have to regenerate it with merkle update auto new_state = vm::MerkleUpdate::apply(prev_state_root_pure_, state_update); CHECK(new_state.not_null()); CHECK(new_state->get_hash() == state_root->get_hash()); - assert(config_ && shard_conf_); + CHECK(shard_conf_); auto neighbor_list = shard_conf_->get_neighbor_shard_hash_ids(shard_); LOG(INFO) << "Build OutMsgQueueProofs for " << neighbor_list.size() << " neighbours"; - for (ton::BlockId blk_id : neighbor_list) { + for (BlockId blk_id : neighbor_list) { auto prefix = blk_id.shard_full(); + if (shard_intersects(prefix, shard_)) { + continue; + } auto limits = mc_state_->get_imported_msg_queue_limits(blk_id.workchain); // one could use monitor_min_split_depth here, to decrease number of broadcasts // but current implementation OutMsgQueueImporter doesn't support it auto r_proof = OutMsgQueueProof::build( - prefix, {OutMsgQueueProof::OneBlock{.id = new_block_id_ext, .state_root = new_state, .block_root = new_block}}, limits); + prefix, + {OutMsgQueueProof::OneBlock{.id = new_block_id_ext, .state_root = new_state, .block_root = new_block}}, + limits); if (r_proof.is_ok()) { auto proof = r_proof.move_as_ok(); + CHECK(proof->msg_counts_.size() == 1); block_candidate->out_msg_queue_proof_broadcasts.push_back(td::Ref( true, OutMsgQueueProofBroadcast(prefix, new_block_id_ext, limits.max_bytes, limits.max_msgs, std::move(proof->queue_proofs_), std::move(proof->block_state_proofs_), - std::move(proof->msg_counts_)))); + proof->msg_counts_[0]))); } else { - LOG(ERROR) << "Failed to build OutMsgQueueProof: " << r_proof.error(); + LOG(ERROR) << "Failed to build OutMsgQueueProof to " << prefix.to_str() << ": " << r_proof.error(); } } } diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 4e8802c7e5..edd20a8d21 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -92,29 +92,6 @@ static td::Result> process_queue( ++msg_count[kv->source]; ++msg_count_total; - // TODO: Get processed_upto from destination shard (in request?) - /* - // Parse message to check if it was processed (as in Collator::process_inbound_message) - ton::LogicalTime enqueued_lt = kv->msg->prefetch_ulong(64); - auto msg_env = kv->msg->prefetch_ref(); - block::tlb::MsgEnvelope::Record_std env; - if (!tlb::unpack_cell(msg_env, env)) { - return td::Status::Error("cannot unpack MsgEnvelope of an internal message"); - } - vm::CellSlice cs{vm::NoVmOrd{}, env.msg}; - block::gen::CommonMsgInfo::Record_int_msg_info info; - if (!tlb::unpack(cs, info)) { - return td::Status::Error("cannot unpack CommonMsgInfo of an internal message"); - } - auto src_prefix = block::tlb::MsgAddressInt::get_prefix(info.src); - auto dest_prefix = block::tlb::MsgAddressInt::get_prefix(info.dest); - auto cur_prefix = block::interpolate_addr(src_prefix, dest_prefix, env.cur_addr); - auto next_prefix = block::interpolate_addr(src_prefix, dest_prefix, env.next_addr); - block::EnqueuedMsgDescr descr{cur_prefix, next_prefix, kv->lt, enqueued_lt, env.msg->get_hash().bits()}; - if (dst_processed_upto->already_processed(descr)) { - } else { - }*/ - dfs_cs(*kv->msg); TRY_STATUS_PREFIX(check_no_prunned(*kv->msg), "invalid message proof: ") if (estimated_proof_size >= limits.max_bytes || msg_count_total >= (long long)limits.max_msgs) { @@ -301,7 +278,12 @@ void OutMsgQueueImporter::get_neighbor_msg_queue_proofs( return; } - LOG(DEBUG) << "Importing neighbor msg queues for shard " << dst_shard.to_str() << ", " << blocks.size() << " blocks"; + FLOG(DEBUG) { + sb << "Importing neighbor msg queues for shard " << dst_shard.to_str() << ", " << blocks.size() << " blocks:"; + for (const BlockIdExt& block : blocks) { + sb << " " << block.id.to_str(); + } + }; cache_[{dst_shard, blocks}] = entry = std::make_shared(); entry->dst_shard = dst_shard; @@ -321,7 +303,7 @@ void OutMsgQueueImporter::get_neighbor_msg_queue_proofs( prefix = shard_prefix(prefix, min_split); } - LOG(INFO) << "search for out msg queue proof " << prefix.to_str() << block.to_str(); + LOG(DEBUG) << "search for out msg queue proof " << prefix.to_str() << " " << block.to_str(); auto& small_entry = small_cache_[std::make_pair(dst_shard, block)]; if (!small_entry.result.is_null()) { entry->result[block] = small_entry.result; @@ -397,7 +379,13 @@ void OutMsgQueueImporter::get_proof_import(std::shared_ptr entry, st [=, SelfId = actor_id(this), retry_after = td::Timestamp::in(0.1), dst_shard = entry->dst_shard](td::Result>> R) { if (R.is_error()) { - LOG(DEBUG) << "Failed to get out msg queue for " << dst_shard.to_str() << ": " << R.move_as_error(); + FLOG(DEBUG) { + sb << "Failed to get out msg queue for " << dst_shard.to_str() << " from"; + for (const BlockIdExt &block : blocks) { + sb << " " << block.id.to_str(); + } + sb << ": " << R.move_as_error(); + }; delay_action( [=]() { td::actor::send_closure(SelfId, &OutMsgQueueImporter::get_proof_import, entry, std::move(blocks), @@ -443,8 +431,11 @@ void OutMsgQueueImporter::got_proof(std::shared_ptr entry, std::vect void OutMsgQueueImporter::finish_query(std::shared_ptr entry) { FLOG(INFO) { - sb << "Done importing neighbor msg queues for shard " << entry->dst_shard.to_str() << ", " << entry->blocks.size() - << " blocks in " << entry->timer.elapsed() << "s"; + sb << "Done importing neighbor msg queues for shard " << entry->dst_shard.to_str() << " from"; + for (const BlockIdExt &block : entry->blocks) { + sb << " " << block.id.to_str(); + } + sb << " in " << entry->timer.elapsed() << "s"; sb << " sources{"; if (entry->from_broadcast) { sb << " broadcast=" << entry->from_broadcast; @@ -479,8 +470,13 @@ void OutMsgQueueImporter::finish_query(std::shared_ptr entry) { bool OutMsgQueueImporter::check_timeout(std::shared_ptr entry) { if (entry->timeout.is_in_past()) { - LOG(DEBUG) << "Aborting importing neighbor msg queues for shard " << entry->dst_shard.to_str() << ", " - << entry->blocks.size() << " blocks: timeout"; + FLOG(DEBUG) { + sb << "Aborting importing neighbor msg queues for shard " << entry->dst_shard.to_str() << " from"; + for (const BlockIdExt &block : entry->blocks) { + sb << " " << block.id.to_str(); + } + sb << ": timeout"; + }; for (auto& p : entry->promises) { p.first.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); } @@ -499,8 +495,13 @@ void OutMsgQueueImporter::alarm() { auto& promises = it->second->promises; if (it->second->timeout.is_in_past()) { if (!it->second->done) { - LOG(DEBUG) << "Aborting importing neighbor msg queues for shard " << it->second->dst_shard.to_str() << ", " - << it->second->blocks.size() << " blocks: timeout"; + FLOG(DEBUG) { + sb << "Aborting importing neighbor msg queues for shard " << it->second->dst_shard.to_str() << " from"; + for (const BlockIdExt &block : it->second->blocks) { + sb << " " << block.id.to_str(); + } + sb << ": timeout"; + }; for (auto& p : promises) { p.first.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); } @@ -540,7 +541,7 @@ void OutMsgQueueImporter::alarm() { } void OutMsgQueueImporter::add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref proof) { - LOG(INFO) << "add out msg queue proof " << dst_shard.to_str() << proof->block_id_.to_str(); + LOG(INFO) << "add out msg queue proof " << dst_shard.to_str() << " " << proof->block_id_.to_str(); auto& small_entry = small_cache_[std::make_pair(dst_shard, proof->block_id_)]; if (!small_entry.result.is_null()) { return; @@ -556,7 +557,13 @@ void OutMsgQueueImporter::add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref void BuildOutMsgQueueProof::abort_query(td::Status reason) { if (promise_) { - LOG(DEBUG) << "failed to build msg queue proof to " << dst_shard_.to_str() << ": " << reason; + FLOG(DEBUG) { + sb << "failed to build msg queue proof to " << dst_shard_.to_str() << " from"; + for (const auto& block : blocks_) { + sb << " " << block.id.id.to_str(); + } + sb << ": " << reason; + }; promise_.set_error( reason.move_as_error_prefix(PSTRING() << "failed to build msg queue proof to " << dst_shard_.to_str() << ": ")); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 3ff33d234a..903e315546 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -798,7 +798,9 @@ void ValidatorManagerImpl::wait_neighbor_msg_queue_proofs( if (dst_shard.is_masterchain()) { // We spit queries for masterchain {dst_shard, {block_1, ..., block_n}} into separate queries // {dst_shard, {block_1}}, ..., {dst_shard, {block_n}} - // Also, use cache + // Also, use cache. + // This is performed here and not in OutMsgQueueImporter because it's important to use + // cached_msg_queue_to_masterchain_, which is related to the current list of shard block descriptions class Worker : public td::actor::Actor { public: Worker(size_t pending, td::Promise>> promise) @@ -2958,12 +2960,15 @@ PublicKeyHash ValidatorManagerImpl::get_validator(ShardIdFull shard, td::Refget_collators_list()->self_collate; } bool ValidatorManagerImpl::Collator::can_collate_shard(ShardIdFull shard) const { @@ -3524,7 +3529,7 @@ void ValidatorManagerImpl::del_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh } else { td::actor::send_closure(it->second.actor, &CollatorNode::del_shard, shard); } -}; +} void ValidatorManagerImpl::get_collation_manager_stats( td::Promise> promise) { @@ -3575,15 +3580,18 @@ void ValidatorManagerImpl::get_collation_manager_stats( } void ValidatorManagerImpl::add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref proof) { - if (!collator_nodes_.empty()) { + if (is_shard_collator(dst_shard)) { if (out_msg_queue_importer_.empty()) { out_msg_queue_importer_ = td::actor::create_actor("outmsgqueueimporter", actor_id(this), opts_, last_masterchain_state_); } td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::add_out_msg_queue_proof, dst_shard, std::move(proof)); + } else { + VLOG(VALIDATOR_DEBUG) << "Dropping unneeded out msg queue proof to shard " << dst_shard.to_str(); } } + void ValidatorManagerImpl::add_persistent_state_description(td::Ref desc) { auto now = (UnixTime)td::Clocks::system(); if (desc->end_time <= now) { diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 344f5c1c33..97f34ea7cd 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -32,13 +32,14 @@ namespace ton { namespace validator { static bool need_send_candidate_broadcast(const validatorsession::BlockSourceInfo &source_info, bool is_masterchain) { - return source_info.first_block_round == source_info.round && source_info.source_priority == 0 && !is_masterchain; + return source_info.priority.first_block_round == source_info.priority.round && source_info.priority.priority == 0 && + !is_masterchain; } void ValidatorGroup::generate_block_candidate( validatorsession::BlockSourceInfo source_info, td::Promise promise) { - td::uint32 round_id = source_info.round; + td::uint32 round_id = source_info.priority.round; if (round_id > last_known_round_id_) { last_known_round_id_ = round_id; } @@ -66,15 +67,10 @@ void ValidatorGroup::generate_block_candidate( std::move(R)); }; td::uint64 max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; - auto block_candidate_priority = BlockCandidatePriority{ - .round = source_info.round, - .first_block_round = source_info.first_block_round, - .priority = source_info.source_priority - }; td::actor::send_closure(collation_manager_, &CollationManager::collate_block, shard_, min_masterchain_block_id_, prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, - block_candidate_priority, validator_set_, - max_answer_size, cancellation_token_source_.get_cancellation_token(), std::move(P)); + source_info.priority, validator_set_, max_answer_size, + cancellation_token_source_.get_cancellation_token(), std::move(P)); } void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo source_info, @@ -103,7 +99,7 @@ void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block, td::Promise> promise) { - td::uint32 round_id = source_info.round; + td::uint32 round_id = source_info.priority.round; if (round_id > last_known_round_id_) { last_known_round_id_ = round_id; } @@ -174,7 +170,7 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so validatorsession::ValidatorSessionStats stats, td::Promise promise) { stats.cc_seqno = validator_set_->get_catchain_seqno(); - td::uint32 round_id = source_info.round; + td::uint32 round_id = source_info.priority.round; if (round_id >= last_known_round_id_) { last_known_round_id_ = round_id + 1; } From 5fae8db7a03d9d16b8bbaedd16a3d70202d22c28 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 28 Nov 2024 13:09:40 +0300 Subject: [PATCH 127/524] Adapt "get msg queue sizes" in lite-client and tonlib to non-full liteservers --- lite-client/lite-client.cpp | 90 +++++++++++++++--- lite-client/lite-client.h | 3 +- tonlib/tonlib/TonlibClient.cpp | 167 ++++++++++++++++++++++++++++++--- tonlib/tonlib/tonlib-cli.cpp | 19 ++++ 4 files changed, 253 insertions(+), 26 deletions(-) diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index dc09ae52b9..ce3dd4b20a 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -1627,27 +1627,93 @@ void TestNode::send_compute_complaint_price_query(ton::StdSmcAddress elector_add } bool TestNode::get_msg_queue_sizes() { - auto q = ton::serialize_tl_object(ton::create_tl_object(0, 0, 0), true); - return envelope_send_query(std::move(q), [Self = actor_id(this)](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "liteServer.getOutMsgQueueSizes error: " << res.move_as_error(); + ton::BlockIdExt blkid = mc_last_id_; + if (!blkid.is_valid_full()) { + return set_error("must obtain last block information before making other queries"); + } + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } + auto b = + ton::create_serialize_tl_object(ton::create_tl_lite_block_id(blkid)); + LOG(INFO) << "requesting recent shard configuration"; + return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result R) -> void { + if (R.is_error()) { return; } - auto F = ton::fetch_tl_object(res.move_as_ok(), true); + auto F = ton::fetch_tl_object(R.move_as_ok(), true); if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getOutMsgQueueSizes"; - return; + LOG(ERROR) << "cannot parse answer to liteServer.getAllShardsInfo"; + } else { + auto f = F.move_as_ok(); + td::actor::send_closure_later(Self, &TestNode::get_msg_queue_sizes_cont, blkid, std::move(f->data_)); } - td::actor::send_closure_later(Self, &TestNode::got_msg_queue_sizes, F.move_as_ok()); }); } -void TestNode::got_msg_queue_sizes(ton::tl_object_ptr f) { +void TestNode::get_msg_queue_sizes_cont(ton::BlockIdExt mc_blkid, td::BufferSlice data) { + LOG(INFO) << "got shard configuration with respect to block " << mc_blkid.to_str(); + std::vector blocks; + blocks.push_back(mc_blkid); + auto R = vm::std_boc_deserialize(data.clone()); + if (R.is_error()) { + set_error(R.move_as_error_prefix("cannot deserialize shard configuration: ")); + return; + } + auto root = R.move_as_ok(); + block::ShardConfig sh_conf; + if (!sh_conf.unpack(vm::load_cell_slice_ref(root))) { + set_error("cannot extract shard block list from shard configuration"); + return; + } + auto ids = sh_conf.get_shard_hash_ids(true); + for (auto id : ids) { + auto ref = sh_conf.get_shard_hash(ton::ShardIdFull(id)); + if (ref.not_null()) { + blocks.push_back(ref->top_block_id()); + } + } + + struct QueryInfo { + std::vector blocks; + std::vector sizes; + size_t pending; + }; + auto info = std::make_shared(); + info->blocks = std::move(blocks); + info->sizes.resize(info->blocks.size(), 0); + info->pending = info->blocks.size(); + + for (size_t i = 0; i < info->blocks.size(); ++i) { + ton::BlockIdExt block_id = info->blocks[i]; + auto b = ton::create_serialize_tl_object( + 0, ton::create_tl_lite_block_id(block_id), false); + LOG(DEBUG) << "requesting queue size for block " << block_id.to_str(); + envelope_send_query(std::move(b), [=, this](td::Result R) -> void { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + set_error(F.move_as_error_prefix("failed to get queue size: ")); + return; + } + auto f = F.move_as_ok(); + LOG(DEBUG) << "got queue size for block " << block_id.to_str() << " : " << f->size_; + info->sizes[i] = f->size_; + if (--info->pending == 0) { + get_msg_queue_sizes_finish(std::move(info->blocks), std::move(info->sizes)); + } + }); + } +} + +void TestNode::get_msg_queue_sizes_finish(std::vector blocks, std::vector sizes) { + CHECK(blocks.size() == sizes.size()); td::TerminalIO::out() << "Outbound message queue sizes:" << std::endl; - for (auto &x : f->shards_) { - td::TerminalIO::out() << ton::create_block_id(x->id_).id.to_str() << " " << x->size_ << std::endl; + for (size_t i = 0; i < blocks.size(); ++i) { + td::TerminalIO::out() << blocks[i].id.to_str() << " " << sizes[i] << std::endl; } - td::TerminalIO::out() << "External message queue size limit: " << f->ext_msg_queue_size_limit_ << std::endl; } bool TestNode::get_dispatch_queue_info(ton::BlockIdExt block_id) { diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 721d2b20d6..57804418f7 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -324,7 +324,8 @@ class TestNode : public td::actor::Actor { void send_compute_complaint_price_query(ton::StdSmcAddress elector_addr, unsigned expires_in, unsigned bits, unsigned refs, td::Bits256 chash, std::string filename); bool get_msg_queue_sizes(); - void got_msg_queue_sizes(ton::tl_object_ptr f); + void get_msg_queue_sizes_cont(ton::BlockIdExt mc_blkid, td::BufferSlice data); + void get_msg_queue_sizes_finish(std::vector blocks, std::vector sizes); bool get_dispatch_queue_info(ton::BlockIdExt block_id); bool get_dispatch_queue_info_cont(ton::BlockIdExt block_id, bool first, td::Bits256 after_addr); void got_dispatch_queue_info(ton::BlockIdExt block_id, diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index b9ff4899cc..73cb2a16a8 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -1800,6 +1800,132 @@ class GetShardBlockProof : public td::actor::Actor { std::vector> links_; }; +class GetOutMsgQueueSizes : public td::actor::Actor { + public: + GetOutMsgQueueSizes(ExtClientRef ext_client_ref, std::vector blocks, + td::actor::ActorShared<> parent, + td::Promise>&& promise) + : blocks_(std::move(blocks)), parent_(std::move(parent)), promise_(std::move(promise)) { + client_.set_client(ext_client_ref); + } + + void start_up() override { + sizes_.resize(blocks_.size()); + pending_ = blocks_.size() + 1; + + for (size_t i = 0; i < blocks_.size(); ++i) { + client_.send_query( + ton::lite_api::liteServer_getBlockOutMsgQueueSize(1, ton::create_tl_lite_block_id(blocks_[i]), true), + [SelfId = actor_id(this), i](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetOutMsgQueueSizes::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetOutMsgQueueSizes::got_block_queue_size, i, R.move_as_ok()); + } + }); + } + + client_.send_query( + ton::lite_api::liteServer_getOutMsgQueueSizes(1, ton::masterchainId, ton::shardIdAll), + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetOutMsgQueueSizes::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetOutMsgQueueSizes::got_ext_msg_queue_size_limit, + R.ok()->ext_msg_queue_size_limit_); + } + }); + } + + void got_block_queue_size(size_t i, lite_api_ptr f) { + try { + auto S = [&, this]() -> td::Status { + TRY_RESULT_PREFIX(roots, vm::std_boc_deserialize_multi(f->proof_), "cannot deserialize proof: "); + if (roots.size() != 2) { + return td::Status::Error("expected 2 roots in proof"); + } + auto state_root = vm::MerkleProof::virtualize(std::move(roots[1]), 1); + if (state_root.is_null()) { + return td::Status::Error("state proof is invalid"); + } + ton::Bits256 state_hash = state_root->get_hash().bits(); + TRY_STATUS_PREFIX(block::check_block_header_proof(vm::MerkleProof::virtualize(std::move(roots[0]), 1), + blocks_[i], &state_hash, true, nullptr, nullptr), + "error in block header proof: "); + + block::gen::ShardStateUnsplit::Record sstate; + block::gen::OutMsgQueueInfo::Record out_msg_queue_info; + if (!tlb::unpack_cell(state_root, sstate) || !tlb::unpack_cell(sstate.out_msg_queue_info, out_msg_queue_info)) { + return td::Status::Error("cannot unpack shard state"); + } + vm::CellSlice& extra_slice = out_msg_queue_info.extra.write(); + if (extra_slice.fetch_long(1) == 0) { + return td::Status::Error("no out_msg_queue_size in shard state"); + } + block::gen::OutMsgQueueExtra::Record out_msg_queue_extra; + if (!tlb::unpack(extra_slice, out_msg_queue_extra)) { + return td::Status::Error("cannot unpack OutMsgQueueExtra"); + } + vm::CellSlice& size_slice = out_msg_queue_extra.out_queue_size.write(); + if (size_slice.fetch_long(1) == 0) { + return td::Status::Error("no out_msg_queue_size in shard state"); + } + td::uint64 size = size_slice.prefetch_ulong(48); + if (size != f->size_) { + return td::Status::Error("queue size mismatch"); + } + return td::Status::OK(); + }(); + if (S.is_error()) { + abort(std::move(S)); + return; + } + } catch (vm::VmError& err) { + abort(err.as_status()); + return; + } catch (vm::VmVirtError& err) { + abort(err.as_status()); + return; + } + + sizes_[i] = f->size_; + dec_pending(); + } + + void got_ext_msg_queue_size_limit(td::uint32 value) { + ext_msg_queue_size_limit_ = value; + dec_pending(); + } + + void dec_pending() { + if (--pending_ == 0) { + std::vector> shards; + for (size_t i = 0; i < blocks_.size(); ++i) { + shards.push_back( + tonlib_api::make_object(to_tonlib_api(blocks_[i]), sizes_[i])); + } + promise_.set_result( + tonlib_api::make_object(std::move(shards), ext_msg_queue_size_limit_)); + stop(); + } + } + + void abort(td::Status error) { + promise_.set_error(std::move(error)); + stop(); + } + + private: + std::vector blocks_; + td::actor::ActorShared<> parent_; + td::Promise> promise_; + ExtClient client_; + + std::vector sizes_; + td::uint32 ext_msg_queue_size_limit_ = 0; + size_t pending_ = 0; +}; + auto to_lite_api(const tonlib_api::ton_blockIdExt& blk) -> td::Result>; auto to_tonlib_api(const ton::lite_api::liteServer_transactionId& txid) -> tonlib_api_ptr; @@ -6129,19 +6255,34 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getShardBlockProof& td::Status TonlibClient::do_request(const tonlib_api::blocks_getOutMsgQueueSizes& request, td::Promise>&& promise) { - client_.send_query(ton::lite_api::liteServer_getOutMsgQueueSizes(request.mode_, request.wc_, request.shard_), - promise.wrap([](lite_api_ptr&& queue_sizes) { - tonlib_api::blocks_outMsgQueueSizes result; - result.ext_msg_queue_size_limit_ = queue_sizes->ext_msg_queue_size_limit_; - for (auto &x : queue_sizes->shards_) { - tonlib_api::blocks_outMsgQueueSize shard; - shard.id_ = to_tonlib_api(*x->id_); - shard.size_ = x->size_; - result.shards_.push_back(tonlib_api::make_object(std::move(shard))); - } - return tonlib_api::make_object(std::move(result)); - })); - + auto req_mode = request.mode_; + auto req_shard = ton::ShardIdFull{request.wc_, (ton::ShardId)request.shard_}; + if ((req_mode & 1) && !req_shard.is_valid_ext()) { + return td::Status::Error("invalid shard"); + } + client_.with_last_block( + [=, self = this, promise = std::move(promise)](td::Result r_last_block) mutable { + TRY_RESULT_PROMISE_PREFIX(promise, last_block, std::move(r_last_block), "get last block failed: "); + do_request(tonlib_api::blocks_getShards(to_tonlib_api(last_block.last_block_id)), + [=, mc_blkid = last_block.last_block_id, + promise = std::move(promise)](td::Result> R) mutable { + TRY_RESULT_PROMISE_PREFIX(promise, shards, std::move(R), "get shards failed: "); + std::vector blocks; + if (!(req_mode & 1) || ton::shard_intersects(mc_blkid.shard_full(), req_shard)) { + blocks.push_back(mc_blkid); + } + for (const auto& shard : shards->shards_) { + TRY_RESULT_PROMISE(promise, block_id, to_block_id(*shard)); + if (!(req_mode & 1) || ton::shard_intersects(block_id.shard_full(), req_shard)) { + blocks.push_back(block_id); + } + } + auto actor_id = self->actor_id_++; + self->actors_[actor_id] = td::actor::create_actor( + "GetOutMsgQueueSizes", self->client_.get_client(), std::move(blocks), + actor_shared(this, actor_id), std::move(promise)); + }); + }); return td::Status::OK(); } diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 2c7100f24b..364b8f6636 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -430,6 +430,7 @@ class TonlibCli : public td::actor::Actor { << "\t 'k' modifier - use fake key\n" << "\t 'c' modifier - just esmitate fees\n"; td::TerminalIO::out() << "getmasterchainsignatures - get sigratures of masterchain block \n"; + td::TerminalIO::out() << "msgqueuesizes - get out msg queue sizes in the latest shard states\n"; } else if (cmd == "genkey") { generate_key(); } else if (cmd == "exit" || cmd == "quit") { @@ -517,6 +518,8 @@ class TonlibCli : public td::actor::Actor { } else if (cmd == "getmasterchainsignatures") { auto seqno = parser.read_word(); run_get_masterchain_block_signatures(seqno, std::move(cmd_promise)); + } else if (cmd == "msgqueuesizes") { + run_get_out_msg_queue_sizes(std::move(cmd_promise)); } else if (cmd == "showtransactions") { run_show_transactions(parser, std::move(cmd_promise)); } else { @@ -2161,6 +2164,22 @@ class TonlibCli : public td::actor::Actor { })); } + void run_get_out_msg_queue_sizes(td::Promise promise) { + send_query(make_object(0, 0, 0), + promise.wrap([](tonlib_api::object_ptr&& f) { + td::TerminalIO::out() << "Outbound message queue sizes:" << std::endl; + for (const auto& shard : f->shards_) { + td::TerminalIO::out() << ton::BlockId{shard->id_->workchain_, (ton::ShardId)shard->id_->shard_, + (ton::BlockSeqno)shard->id_->seqno_} + .to_str() + << " " << shard->size_ << std::endl; + } + td::TerminalIO::out() << "External message queue size limit: " << f->ext_msg_queue_size_limit_ + << std::endl; + return td::Unit(); + })); + } + void run_show_transactions(td::ConstParser& parser, td::Promise promise) { TRY_RESULT_PROMISE(promise, address, to_account_address(parser.read_word(), false)); TRY_RESULT_PROMISE(promise, lt, td::to_integer_safe(parser.read_word())); From 923f1cd69b6d31f1e131764f0f9d2a208cfd4d63 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 28 Nov 2024 14:13:49 +0300 Subject: [PATCH 128/524] Improve collator node pings and collation manager stats --- tl/generate/scheme/ton_api.tl | 2 +- tl/generate/scheme/ton_api.tlo | Bin 108888 -> 108972 bytes .../validator-engine-console-query.cpp | 20 +++++++++++++-- validator/collation-manager.cpp | 11 +++++--- validator/collation-manager.hpp | 2 ++ validator/collator-node.cpp | 24 +++++++++++++++++- validator/collator-node.hpp | 4 +++ validator/manager.cpp | 10 ++++++++ 8 files changed, 66 insertions(+), 7 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 720f8d4964..4bfd578838 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -750,7 +750,7 @@ engine.validator.perfTimerStats stats:(vector engine.validator.PerfTimerStatsByN engine.validator.shardOutQueueSize size:long = engine.validator.ShardOutQueueSize; engine.validator.collationManagerStats.shard shard_id:tonNode.shardId self_collate:Bool select_mode:string active:Bool collators:(vector int256) = engine.validator.collationManagerStats.Shard; -engine.validator.collationManagerStats.collator adnl_id:int256 active:Bool alive:Bool ping_in:double = engine.validator.collationManagerStats.Collator; +engine.validator.collationManagerStats.collator adnl_id:int256 active:Bool alive:Bool ping_in:double last_ping_ago:double last_ping_status:string = engine.validator.collationManagerStats.Collator; engine.validator.collationManagerStats.localId adnl_id:int256 shards:(vector engine.validator.collationManagerStats.shard) collators:(vector engine.validator.collationManagerStats.collator) = engine.validator.collationManagerStats.LocalId; engine.validator.collationManagerStats local_ids:(vector engine.validator.collationManagerStats.localId) = engine.validator.CollationManagerStats; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 19a8d13138d85dd2bea8b7e28793cbc85e441aa9..d5ebffaa061ab442de15cdea2f087fbaf6254856 100644 GIT binary patch delta 126 zcmca{iEYhgwhcP+ERLK?;+qZRcPt09CL73W2qx#}UB)%XsFFignJ%4gtwzL33q_`xpq_lW);BJTwleZi{ L1F@>*#FYmC=kqRX delta 77 zcmZ2;neE0UwhcP+ELtBY&)IAszhgO=HQ7L3Lohi%CnvEazX&9C^O%qi%jCcl8k2pG WM^B!xhXbOFlast_ping_status_; + std::erase_if(status, [](char c) { return c < (char)32; }); + if (status.size() > 128) { + status.resize(128); + } + sb << td::StringBuilder::FixedDouble(collator->last_ping_ago_, 3) << ": " << status; + } + td::TerminalIO::out() << sb.as_cslice() << "\n"; } } } diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index e844705958..879b8d8953 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -261,6 +261,8 @@ void CollationManager::get_stats( } else { obj->ping_in_ = -1.0; } + obj->last_ping_ago_ = collator.last_ping_at ? td::Time::now() - collator.last_ping_at.at() : -1.0; + obj->last_ping_status_ = collator.last_ping_status.is_ok() ? "OK" : collator.last_ping_status.message().str(); stats->collators_.push_back(std::move(obj)); } promise.set_value(std::move(stats)); @@ -323,7 +325,7 @@ void CollationManager::alarm() { td::actor::send_closure(SelfId, &CollationManager::got_pong, id, std::move(R)); }; LOG(DEBUG) << "sending ping to " << id; - td::actor::send_closure(rldp_, &rldp::Rldp::send_query, local_id_, id, "collatorping", std::move(P), + td::actor::send_closure(rldp_, &rldp::Rldp::send_query, local_id_, id, "ping", std::move(P), td::Timestamp::in(2.0), std::move(query)); } else { alarm_timestamp().relax(collator.ping_at); @@ -340,7 +342,7 @@ void CollationManager::got_pong(adnl::AdnlNodeIdShort id, td::Result td::Result> { - TRY_RESULT_PREFIX(data, std::move(R), "rldp query error: "); + TRY_RESULT(data, std::move(R)); auto r_error = fetch_tl_object(data, true); if (r_error.is_ok()) { auto error = r_error.move_as_ok(); @@ -348,12 +350,15 @@ void CollationManager::got_pong(adnl::AdnlNodeIdShort id, td::Result(data, true); }(); + collator.last_ping_at = td::Timestamp::now(); if (r_pong.is_error()) { - LOG(DEBUG) << "pong from " << id << " : " << r_pong.move_as_error(); + LOG(DEBUG) << "pong from " << id << " : " << r_pong.error(); collator.alive = false; + collator.last_ping_status = r_pong.move_as_error(); } else { LOG(DEBUG) << "pong from " << id << " : OK"; collator.alive = true; + collator.last_ping_status = td::Status::OK(); } collator.ping_at = td::Timestamp::in(td::Random::fast(10.0, 20.0)); if (collator.active_cnt && !collator.sent_ping) { diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp index 9ca69814b7..0ca4617d0a 100644 --- a/validator/collation-manager.hpp +++ b/validator/collation-manager.hpp @@ -65,6 +65,8 @@ class CollationManager : public td::actor::Actor { td::Timestamp ping_at = td::Timestamp::now(); bool sent_ping = false; size_t active_cnt = 0; + td::Timestamp last_ping_at = td::Timestamp::never(); + td::Status last_ping_status = td::Status::Error("not pinged"); }; std::map collators_; diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 1b8eb79e88..968835c22e 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -167,6 +167,10 @@ void CollatorNode::new_masterchain_block_notification(td::Ref } } +void CollatorNode::update_shard_client_handle(BlockHandle shard_client_handle) { + shard_client_handle_ = shard_client_handle; +} + void CollatorNode::update_validator_group_info(ShardIdFull shard, std::vector prev, CatchainSeqno cc_seqno) { if (!can_collate_shard(shard)) { @@ -225,7 +229,12 @@ void CollatorNode::update_validator_group_info(ShardIdFull shard, std::vector) {}); + auto S = check_out_of_sync(); + if (S.is_ok()) { + generate_block(shard, cc_seqno, info.prev, {}, td::Timestamp::in(10.0), [](td::Result) {}); + } else { + LOG(DEBUG) << "not generating block automatically: " << S; + } } return; } @@ -535,9 +544,22 @@ void CollatorNode::process_result(std::shared_ptr cache_entry, td::R cache_entry->promises.clear(); } +td::Status CollatorNode::check_out_of_sync() { + if (last_masterchain_state_.is_null() || !shard_client_handle_) { + return td::Status::Error("not inited"); + } + auto now = (UnixTime)td::Clocks::system(); + if (last_masterchain_state_->get_unix_time() < now - 60 || shard_client_handle_->unix_time() < now - 60) { + return td::Status::Error(PSTRING() << "out of sync: mc " << now - last_masterchain_state_->get_unix_time() + << "s ago, shardclient " << now - shard_client_handle_->unix_time() << "s ago"); + } + return td::Status::OK(); +} + void CollatorNode::process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode_ping& ping, td::Promise promise) { LOG(DEBUG) << "got ping from " << src; + TRY_STATUS_PROMISE(promise, check_out_of_sync()); promise.set_result(create_serialize_tl_object(0)); } diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index 54876c35fd..cca77d26c6 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -36,6 +36,7 @@ class CollatorNode : public td::actor::Actor { void del_shard(ShardIdFull shard); void new_masterchain_block_notification(td::Ref state); + void update_shard_client_handle(BlockHandle shard_client_handle); void update_validator_group_info(ShardIdFull shard, std::vector prev, CatchainSeqno cc_seqno); void update_options(td::Ref opts) { @@ -84,6 +85,7 @@ class CollatorNode : public td::actor::Actor { std::map, FutureValidatorGroup> future_validator_groups_; td::Ref last_masterchain_state_; + BlockHandle shard_client_handle_; td::Result get_future_validator_group(ShardIdFull shard, CatchainSeqno cc_seqno); @@ -92,6 +94,8 @@ class CollatorNode : public td::actor::Actor { td::Promise promise); void process_result(std::shared_ptr cache_entry, td::Result R); + td::Status check_out_of_sync(); + public: static tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress); static td::Result deserialize_candidate(tl_object_ptr f, diff --git a/validator/manager.cpp b/validator/manager.cpp index 903e315546..b8394bdc58 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2754,6 +2754,9 @@ void ValidatorManagerImpl::update_shard_client_block_handle(BlockHandle handle, last_liteserver_state_ = std::move(state); } } + for (auto &c : collator_nodes_) { + td::actor::send_closure(c.second.actor, &CollatorNode::update_shard_client_handle, shard_client_handle_); + } shard_client_update(seqno); promise.set_value(td::Unit()); } @@ -3509,6 +3512,13 @@ void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh if (it == collator_nodes_.end()) { it = collator_nodes_.emplace(id, Collator()).first; it->second.actor = td::actor::create_actor("collatornode", id, opts_, actor_id(this), adnl_, rldp_); + if (last_masterchain_state_.not_null()) { + td::actor::send_closure(it->second.actor, &CollatorNode::new_masterchain_block_notification, + last_masterchain_state_); + } + if (shard_client_handle_) { + td::actor::send_closure(it->second.actor, &CollatorNode::update_shard_client_handle, shard_client_handle_); + } } if (!it->second.shards.insert(shard).second) { return; From 0280a288c6c038f2cfcce001cd71e828f76aee58 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 29 Nov 2024 10:34:01 +0300 Subject: [PATCH 129/524] Check supported version in collator-node --- validator/collator-node.cpp | 88 +++++++++++++++++++++++++--------- validator/collator-node.hpp | 4 ++ validator/impl/collator-impl.h | 3 ++ 3 files changed, 73 insertions(+), 22 deletions(-) diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 968835c22e..869ac9ba0f 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -21,6 +21,7 @@ #include "block-db.h" #include "td/utils/lz4.h" #include "checksum.h" +#include "impl/collator-impl.h" #include "impl/shard.hpp" #include "validator-session/candidate-serializer.h" @@ -85,6 +86,15 @@ void CollatorNode::del_shard(ShardIdFull shard) { void CollatorNode::new_masterchain_block_notification(td::Ref state) { last_masterchain_state_ = state; + + if (state->last_key_block_id().seqno() != last_key_block_seqno_) { + last_key_block_seqno_ = state->last_key_block_id().seqno(); + mc_config_status_ = check_mc_config(); + if (mc_config_status_.is_error()) { + LOG(ERROR) << "Cannot validate masterchain config (possibly outdated software):" << mc_config_status_; + } + } + if (validator_adnl_ids_.empty() || state->is_key_state()) { validator_adnl_ids_.clear(); for (int next : {-1, 0, 1}) { @@ -216,7 +226,7 @@ void CollatorNode::update_validator_group_info(ShardIdFull shard, std::vectorhas_external_query_at && !cache_entry->has_internal_query_at) { LOG(INFO) << "generate block query" @@ -230,11 +240,15 @@ void CollatorNode::update_validator_group_info(ShardIdFull shard, std::vector) {}); - } else { + if (S.is_error()) { LOG(DEBUG) << "not generating block automatically: " << S; + return; + } + if (mc_config_status_.is_error()) { + LOG(DEBUG) << "not generating block automatically: unsupported mc config: " << mc_config_status_; + return; } + generate_block(shard, cc_seqno, info.prev, {}, td::Timestamp::in(10.0), [](td::Result) {}); } return; } @@ -322,7 +336,7 @@ static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey cre for (auto& broadcast_ref : block.out_msg_queue_proof_broadcasts) { auto block_state_proof = create_block_state_proof(root).move_as_ok(); - auto &broadcast = broadcast_ref.write(); + auto& broadcast = broadcast_ref.write(); broadcast.block_id = block.id; broadcast.block_state_proofs = vm::std_boc_serialize(std::move(block_state_proof), 31).move_as_ok(); } @@ -363,11 +377,9 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data for (const auto& b : f->prev_blocks_) { prev_blocks.push_back(create_block_id(b)); } - auto priority = BlockCandidatePriority { - .round = static_cast(f->round_), - .first_block_round = static_cast(f->first_block_round_), - .priority = f->priority_ - }; + auto priority = BlockCandidatePriority{.round = static_cast(f->round_), + .first_block_round = static_cast(f->first_block_round_), + .priority = f->priority_}; Ed25519_PublicKey creator(f->creator_); td::Promise new_promise = [promise = std::move(promise), src, shard](td::Result R) mutable { @@ -447,22 +459,21 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std return; } - static auto prefix_inner = [] (auto &sb, auto &shard, auto cc_seqno, auto block_seqno, - const std::optional &o_priority) { + static auto prefix_inner = [](auto& sb, auto& shard, auto cc_seqno, auto block_seqno, + const std::optional& o_priority) { sb << "generate block query" << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno; if (o_priority) { sb << " external{"; - sb << "round_offset=" << o_priority->round - o_priority->first_block_round << ",priority=" << o_priority->priority; + sb << "round_offset=" << o_priority->round - o_priority->first_block_round + << ",priority=" << o_priority->priority; sb << ",first_block_round=" << o_priority->first_block_round; sb << "}"; } else { - sb << " internal" ; + sb << " internal"; } }; - auto prefix = [&] (auto &sb) { - prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); - }; + auto prefix = [&](auto& sb) { prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); }; auto cache_entry = validator_group_info.cache[prev_blocks]; if (cache_entry == nullptr) { @@ -473,7 +484,8 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { FLOG(INFO) { prefix(sb); - sb << ": got external query " << cache_entry->has_external_query_at.at() - cache_entry->has_internal_query_at.at() + sb << ": got external query " + << cache_entry->has_external_query_at.at() - cache_entry->has_internal_query_at.at() << "s after internal query [WON]"; }; } @@ -483,7 +495,8 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { FLOG(INFO) { prefix(sb); - sb << ": got internal query " << cache_entry->has_internal_query_at.at() - cache_entry->has_external_query_at.at() + sb << ": got internal query " + << cache_entry->has_internal_query_at.at() - cache_entry->has_external_query_at.at() << "s after external query [LOST]"; }; } @@ -503,8 +516,8 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std if (cache_entry->started) { FLOG(INFO) { - prefix(sb); - sb << ": collation in progress, waiting"; + prefix(sb); + sb << ": collation in progress, waiting"; }; return; } @@ -519,7 +532,7 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std last_masterchain_state_->get_validator_set(shard), opts_->get_collator_options(), manager_, timeout, [=, SelfId = actor_id(this), timer = td::Timer{}](td::Result R) { FLOG(INFO) { - prefix_inner(sb, shard, cc_seqno, block_seqno,o_priority); + prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); sb << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); }; td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); @@ -556,10 +569,41 @@ td::Status CollatorNode::check_out_of_sync() { return td::Status::OK(); } +td::Status CollatorNode::check_mc_config() { + if (last_masterchain_state_.is_null()) { + return td::Status::Error("not inited"); + } + TRY_RESULT_PREFIX( + config, + block::ConfigInfo::extract_config(last_masterchain_state_->root_cell(), block::ConfigInfo::needCapabilities), + "cannot unpack masterchain config"); + if (config->get_global_version() > Collator::supported_version()) { + return td::Status::Error(PSTRING() << "unsupported global version " << config->get_global_version() + << " (supported: " << Collator::supported_version() << ")"); + } + if (config->get_capabilities() & ~Collator::supported_capabilities()) { + return td::Status::Error(PSTRING() << "unsupported capabilities " << config->get_capabilities() + << " (supported: " << Collator::supported_capabilities() << ")"); + } + td::Status S = td::Status::OK(); + config->foreach_config_param([&](int idx, td::Ref param) { + if (idx < 0) { + return true; + } + if (!block::gen::ConfigParam{idx}.validate_ref(1024, std::move(param))) { + S = td::Status::Error(PSTRING() << "unknown ConfigParam " << idx); + return false; + } + return true; + }); + return S; +} + void CollatorNode::process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode_ping& ping, td::Promise promise) { LOG(DEBUG) << "got ping from " << src; TRY_STATUS_PROMISE(promise, check_out_of_sync()); + TRY_STATUS_PROMISE_PREFIX(promise, mc_config_status_.clone(), "unsupported mc config: "); promise.set_result(create_serialize_tl_object(0)); } diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index cca77d26c6..5aab246356 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -87,6 +87,9 @@ class CollatorNode : public td::actor::Actor { td::Ref last_masterchain_state_; BlockHandle shard_client_handle_; + td::Status mc_config_status_ = td::Status::Error("not inited"); + BlockSeqno last_key_block_seqno_ = (BlockSeqno)-1; + td::Result get_future_validator_group(ShardIdFull shard, CatchainSeqno cc_seqno); void generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, @@ -95,6 +98,7 @@ class CollatorNode : public td::actor::Actor { void process_result(std::shared_ptr cache_entry, td::Result R); td::Status check_out_of_sync(); + td::Status check_mc_config(); public: static tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 0090e95dec..b207242fb1 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -40,6 +40,7 @@ namespace validator { using td::Ref; class Collator final : public td::actor::Actor { + public: static constexpr int supported_version() { return SUPPORTED_VERSION; } @@ -47,6 +48,8 @@ class Collator final : public td::actor::Actor { return ton::capCreateStatsEnabled | ton::capBounceMsgBody | ton::capReportVersion | ton::capShortDequeue | ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages | ton::capFullCollatedData; } + + private: using LtCellRef = block::LtCellRef; using NewOutMsg = block::NewOutMsg; const ShardIdFull shard_; From 4704de76c6e26a49ee85693521ecc4f8dc49cb9d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 29 Nov 2024 13:49:24 +0300 Subject: [PATCH 130/524] Fix compilation errors --- tdutils/td/utils/Timer.cpp | 1 + validator-engine/validator-engine.cpp | 6 +++--- validator/collation-manager.cpp | 2 +- validator/collator-node.cpp | 8 ++++---- validator/full-node.cpp | 6 +++--- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index c2c6789555..abb69b1000 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -23,6 +23,7 @@ #include "td/utils/Time.h" #include +#include namespace td { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 3b9715b356..2e03ce458a 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -472,7 +472,7 @@ td::Result Config::config_add_collator(ton::adnl::AdnlNodeIdShort addr, to return td::Status::Error(PSTRING() << "invalid shard: " << shard.to_str()); } auto& shards = collators[addr]; - if (std::ranges::find(shards, shard) != collators[addr].end()) { + if (std::find(shards.begin(), shards.end(), shard) != shards.end()) { return false; } shards.push_back(shard); @@ -484,7 +484,7 @@ td::Result Config::config_del_collator(ton::adnl::AdnlNodeIdShort addr, to return td::Status::Error(PSTRING() << "invalid shard: " << shard.to_str()); } auto& shards = collators[addr]; - auto it = std::ranges::find(shards, shard); + auto it = std::find(shards.begin(), shards.end(), shard); if (it == shards.end()) { return false; } @@ -1605,7 +1605,7 @@ void ValidatorEngine::set_shard_check_function() { shards.push_back(s); } std::sort(shards.begin(), shards.end()); - shards.erase(std::ranges::unique(shards).begin(), shards.end()); + shards.erase(std::unique(shards.begin(), shards.end()), shards.end()); validator_options_.write().set_shard_check_function( [shards = std::move(shards)](ton::ShardIdFull shard) -> bool { for (auto s : shards) { diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index 879b8d8953..eb9aa626d7 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -321,7 +321,7 @@ void CollationManager::alarm() { if (collator.ping_at.is_in_past()) { collator.sent_ping = true; td::BufferSlice query = create_serialize_tl_object(0); - td::Promise P = [=, SelfId = actor_id(this)](td::Result R) mutable { + td::Promise P = [=, id = id, SelfId = actor_id(this)](td::Result R) mutable { td::actor::send_closure(SelfId, &CollationManager::got_pong, id, std::move(R)); }; LOG(DEBUG) << "sending ping to " << id; diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 869ac9ba0f..94bb18a1bd 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -70,7 +70,7 @@ void CollatorNode::tear_down() { void CollatorNode::add_shard(ShardIdFull shard) { CHECK(shard.is_valid_ext() && !shard.is_masterchain()); - if (std::ranges::find(collating_shards_, shard) != collating_shards_.end()) { + if (std::find(collating_shards_.begin(), collating_shards_.end(), shard) != collating_shards_.end()) { return; } LOG(INFO) << "Collator node: local_id=" << local_id_ << " , shard=" << shard.to_str(); @@ -78,7 +78,7 @@ void CollatorNode::add_shard(ShardIdFull shard) { } void CollatorNode::del_shard(ShardIdFull shard) { - auto it = std::ranges::find(collating_shards_, shard); + auto it = std::find(collating_shards_.begin(), collating_shards_.end(), shard); if (it != collating_shards_.end()) { collating_shards_.erase(it); } @@ -608,8 +608,8 @@ void CollatorNode::process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode } bool CollatorNode::can_collate_shard(ShardIdFull shard) const { - return std::ranges::any_of(collating_shards_, - [&](const ShardIdFull& our_shard) { return shard_intersects(shard, our_shard); }); + return std::any_of(collating_shards_.begin(), collating_shards_.end(), + [&](const ShardIdFull& our_shard) { return shard_intersects(shard, our_shard); }); } tl_object_ptr CollatorNode::serialize_candidate(const BlockCandidate& block, diff --git a/validator/full-node.cpp b/validator/full-node.cpp index e7527d7685..3743971395 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -936,9 +936,9 @@ bool FullNodeConfig::operator!=(const FullNodeConfig &rhs) const { } bool CustomOverlayParams::send_shard(const ShardIdFull &shard) const { - return sender_shards_.empty() || std::ranges::any_of(sender_shards_, [&](const ShardIdFull &our_shard) { - return shard_intersects(shard, our_shard); - }); + return sender_shards_.empty() || + std::any_of(sender_shards_.begin(), sender_shards_.end(), + [&](const ShardIdFull &our_shard) { return shard_intersects(shard, our_shard); }); } CustomOverlayParams CustomOverlayParams::fetch(const ton_api::engine_validator_customOverlay& f) { From 378b5e94c8051cd0fc872a05e92128987328fedf Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 4 Dec 2024 18:06:19 +0300 Subject: [PATCH 131/524] Automatically issue and import fast sync overlay certificates --- adnl/adnl-node-id.hpp | 4 +- dht-server/dht-server.cpp | 7 +- tl/generate/scheme/ton_api.tl | 10 +- tl/generate/scheme/ton_api.tlo | Bin 109280 -> 110212 bytes .../validator-engine-console-query.cpp | 103 +-- .../validator-engine-console-query.h | 71 ++- .../validator-engine-console.cpp | 2 + validator-engine/validator-engine.cpp | 594 ++++++++++++++---- validator-engine/validator-engine.hpp | 40 +- validator/full-node-fast-sync-overlays.cpp | 2 +- validator/full-node.h | 2 + validator/full-node.hpp | 5 +- 12 files changed, 659 insertions(+), 181 deletions(-) diff --git a/adnl/adnl-node-id.hpp b/adnl/adnl-node-id.hpp index 2d3ade1644..43e3629f4c 100644 --- a/adnl/adnl-node-id.hpp +++ b/adnl/adnl-node-id.hpp @@ -37,7 +37,9 @@ class AdnlNodeIdShort { } explicit AdnlNodeIdShort(td::Bits256 value) : hash_(value) { } - explicit AdnlNodeIdShort(tl_object_ptr obj) : hash_(obj->id_) { + explicit AdnlNodeIdShort(tl_object_ptr &&obj) : hash_(obj->id_) { + } + explicit AdnlNodeIdShort(const tl_object_ptr &obj) : hash_(obj->id_) { } const auto &pubkey_hash() const { diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index 006e740820..46a011352c 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -171,8 +171,8 @@ ton::tl_object_ptr Config::tl() const { } return ton::create_tl_object( out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), - ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), - nullptr, nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec)); + ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), nullptr, + nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, @@ -1195,7 +1195,8 @@ int main(int argc, char *argv[]) { SET_VERBOSITY_LEVEL(v); }); p.add_option('V', "version", "shows dht-server build information", [&]() { - std::cout << "dht-server build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "dht-server build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_option('h', "help", "prints_help", [&]() { diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index a4d34428e7..396c06bc1a 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -464,6 +464,8 @@ tonNode.outMsgQueueProof queue_proofs:bytes block_state_proofs:bytes msg_counts: tonNode.outMsgQueueProofEmpty = tonNode.OutMsgQueueProof; tonNode.forgetPeer = tonNode.ForgetPeer; +tonNode.newFastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = tonNode.NewFastSyncMemberCertificate; + ---functions--- @@ -501,6 +503,8 @@ tonNode.slave.sendExtMessage message:tonNode.externalMessage = tonNode.Success; tonNode.query = Object; +tonNode.requestFastSyncOverlayMemberCertificate sign_by:int256 adnl_id:int256 slot:int = overlay.MemberCertificate; + ---types--- // bit 0 - started @@ -654,9 +658,11 @@ engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNode engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave; engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig; engine.validator.fastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.FastSyncMemberCertificate; +engine.validator.fastSyncOverlayClient adnl_id:int256 slot:int = engine.validator.FastSyncOverlayClient; engine.validator.collatorNodeWhitelist enabled:Bool adnl_ids:(vector int256) = engine.validator.CollatorNodeWhitelist; engine.validator.extraConfig state_serializer_enabled:Bool fast_sync_member_certificates:(vector engine.validator.fastSyncMemberCertificate) - collator_node_whitelist:engine.validator.collatorNodeWhitelist = engine.validator.ExtraConfig; + collator_node_whitelist:engine.validator.collatorNodeWhitelist + fast_sync_overlay_clients:(vector engine.validator.fastSyncOverlayClient) = engine.validator.ExtraConfig; engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) validators:(vector engine.validator) collators:(vector engine.collator) @@ -840,6 +846,8 @@ engine.validator.getCollationManagerStats = engine.validator.CollationManagerSta engine.validator.signOverlayMemberCertificate sign_by:int256 adnl_id:int256 slot:int expire_at:int = overlay.MemberCertificate; engine.validator.importFastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.Success; +engine.validator.addFastSyncClient adnl_id:int256 slot:int = engine.validator.Success; +engine.validator.delFastSyncClient adnl_id:int256 = engine.validator.Success; ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index cfb78d6fdf63bab403a3e0491b987293043a6759..faef1564525053095bedcd3d1a7a3d48fe583ec0 100644 GIT binary patch delta 449 zcmaEGm96D28}Fmp`c@23V6l<6T9WzIsrskDQTratq zN9u`;!1}NAi&aYU^ZfEtQuX{&%O~HGl$xx>Ew%ZB*#UOOW1BbH@fU$i*?f3|BpcYc z``Z*bfdWQ1j|us(Oy=CaSyuYleND-<#Nv|p;>x__`24ceqMXFa_++q6lLKeUi-W9Z zggUIqnN5eow(TIuX~(xOg*vTphXOamX$&C6lP4_JnXb^pC^I?0iGu|svN>VLoG8RYhG2`PKS*Pg*#0AxaRWDN;tlq9O49=i7)8*Pug_+5-M*rb@r}9d(og20`mm5I zN-Zo+EiQo>?hgq9-_+cs)FS89qLR$C%;dz9)ZJ%Qm_XsiJpE=PWAgNbCZLs5RxdGFUXj7LfqVOu62>>?+bz~I`Y-|jRq8fS diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 2d78b412e1..c0170a0671 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -807,21 +807,21 @@ td::Status SignCertificateQuery::send() { auto sign = ton::create_serialize_tl_object(signer_.tl(), std::move(cid)); auto pub = ton::create_serialize_tl_object(signer_.tl()); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(pub), - td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &SignCertificateQuery::receive_pubkey, R.move_as_ok()); - } - })); + td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &SignCertificateQuery::receive_pubkey, R.move_as_ok()); + } + })); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(sign), - td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &SignCertificateQuery::receive_signature, R.move_as_ok()); - } - })); + td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &SignCertificateQuery::receive_signature, R.move_as_ok()); + } + })); return td::Status::OK(); } @@ -833,33 +833,32 @@ void SignCertificateQuery::receive_pubkey(td::BufferSlice R) { } pubkey_ = f.move_as_ok(); has_pubkey_ = true; - if(has_signature_) { + if (has_signature_) { save_certificate(); } } - td::Status SignCertificateQuery::receive(td::BufferSlice data) { UNREACHABLE(); } void SignCertificateQuery::receive_signature(td::BufferSlice R) { auto f = ton::fetch_tl_object(R.as_slice(), true); - if(f.is_error()){ + if (f.is_error()) { handle_error(f.move_as_error_prefix("Failed to get signature: ")); return; } signature_ = std::move(f.move_as_ok()->signature_); - if(has_pubkey_) { + if (has_pubkey_) { save_certificate(); } } void SignCertificateQuery::save_certificate() { - auto c = ton::create_serialize_tl_object( - std::move(pubkey_), expire_at_, max_size_, std::move(signature_)); + auto c = ton::create_serialize_tl_object(std::move(pubkey_), expire_at_, max_size_, + std::move(signature_)); auto w = td::write_file(out_file_, c.as_slice()); - if(w.is_error()) { + if (w.is_error()) { handle_error(w.move_as_error_prefix("Failed to write certificate to file: ")); return; } @@ -880,11 +879,8 @@ td::Status ImportCertificateQuery::send() { TRY_RESULT_PREFIX(cert, ton::fetch_tl_object(data.as_slice(), true), "incorrect certificate"); auto b = ton::create_serialize_tl_object( - overlay_, - ton::create_tl_object(id_), - ton::create_tl_object(kh_.tl()), - std::move(cert) - ); + overlay_, ton::create_tl_object(id_), + ton::create_tl_object(kh_.tl()), std::move(cert)); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1031,7 +1027,6 @@ td::Status GetOverlaysStatsJsonQuery::receive(td::BufferSlice data) { return td::Status::OK(); } - td::Status ImportCertificateQuery::receive(td::BufferSlice data) { TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), "received incorrect answer: "); @@ -1039,10 +1034,9 @@ td::Status ImportCertificateQuery::receive(td::BufferSlice data) { return td::Status::OK(); } - td::Status SignShardOverlayCertificateQuery::run() { TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(max_size_, tokenizer_.get_token()); @@ -1052,8 +1046,8 @@ td::Status SignShardOverlayCertificateQuery::run() { } td::Status SignShardOverlayCertificateQuery::send() { - auto b = ton::create_serialize_tl_object - (wc_, shard_, ton::create_tl_object(key_.tl()), expire_at_, max_size_); + auto b = ton::create_serialize_tl_object( + wc_, shard_, ton::create_tl_object(key_.tl()), expire_at_, max_size_); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1062,7 +1056,7 @@ td::Status SignShardOverlayCertificateQuery::receive(td::BufferSlice data) { TRY_RESULT_PREFIX(c, ton::fetch_tl_object(data.as_slice(), true), "received incorrect cert: "); auto w = td::write_file(out_file_, data.as_slice()); - if(w.is_error()) { + if (w.is_error()) { return w.move_as_error_prefix("Failed to write certificate to file: "); } td::TerminalIO::out() << "saved certificate\n"; @@ -1072,7 +1066,7 @@ td::Status SignShardOverlayCertificateQuery::receive(td::BufferSlice data) { td::Status ImportShardOverlayCertificateQuery::run() { TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(in_file_, tokenizer_.get_token()); @@ -1083,8 +1077,8 @@ td::Status ImportShardOverlayCertificateQuery::send() { TRY_RESULT(data, td::read_file(in_file_)); TRY_RESULT_PREFIX(cert, ton::fetch_tl_object(data.as_slice(), true), "incorrect certificate"); - auto b = ton::create_serialize_tl_object - (wc_, shard_, ton::create_tl_object(key_.tl()), std::move(cert)); + auto b = ton::create_serialize_tl_object( + wc_, shard_, ton::create_tl_object(key_.tl()), std::move(cert)); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1908,3 +1902,42 @@ td::Status ImportFastSyncMemberCertificateQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "success\n"; return td::Status::OK(); } + +td::Status AddFastSyncOverlayClientQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(slot_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status AddFastSyncOverlayClientQuery::send() { + auto b = ton::create_serialize_tl_object(adnl_id_, slot_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status AddFastSyncOverlayClientQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status DelFastSyncOverlayClientQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status DelFastSyncOverlayClientQuery::send() { + auto b = ton::create_serialize_tl_object(adnl_id_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status DelFastSyncOverlayClientQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index c34f4f5b9c..60e05d59c1 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -979,9 +979,9 @@ class GetOverlaysStatsJsonQuery : public Query { std::string name() const override { return get_name(); } - -private: - std::string file_name_; + + private: + std::string file_name_; }; class SignCertificateQuery : public Query { @@ -996,7 +996,8 @@ class SignCertificateQuery : public Query { return "signcert"; } static std::string get_help() { - return "signcert \tsign overlay certificate by key"; + return "signcert \tsign overlay certificate by " + " key"; } std::string name() const override { return get_name(); @@ -1004,9 +1005,8 @@ class SignCertificateQuery : public Query { void receive_pubkey(td::BufferSlice R); void receive_signature(td::BufferSlice R); - private: - void save_certificate(); + void save_certificate(); td::Bits256 overlay_; td::Bits256 id_; @@ -1057,14 +1057,14 @@ class SignShardOverlayCertificateQuery : public Query { return "signshardoverlaycert"; } static std::string get_help() { - return "signshardoverlaycert \tsign certificate for in currently active shard overlay"; + return "signshardoverlaycert \tsign certificate for " + " in currently active shard overlay"; } std::string name() const override { return get_name(); } private: - td::int32 wc_; td::int64 shard_; td::int32 expire_at_; @@ -1073,7 +1073,6 @@ class SignShardOverlayCertificateQuery : public Query { std::string out_file_; }; - class ImportShardOverlayCertificateQuery : public Query { public: ImportShardOverlayCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) @@ -1086,14 +1085,14 @@ class ImportShardOverlayCertificateQuery : public Query { return "importshardoverlaycert"; } static std::string get_help() { - return "importshardoverlaycert \timport certificate for in currently active shard overlay"; + return "importshardoverlaycert \timport certificate for in " + "currently active shard overlay"; } std::string name() const override { return get_name(); } private: - td::int32 wc_; td::int64 shard_; ton::PublicKeyHash key_; @@ -1134,7 +1133,8 @@ class GetPerfTimerStatsJsonQuery : public Query { return "getperftimerstatsjson"; } static std::string get_help() { - return "getperftimerstatsjson \tgets min, average and max event processing time for last 60, 300 and 3600 seconds and writes to json file"; + return "getperftimerstatsjson \tgets min, average and max event processing time for last 60, 300 and 3600 " + "seconds and writes to json file"; } std::string name() const override { return get_name(); @@ -1696,3 +1696,50 @@ class ImportFastSyncMemberCertificateQuery : public Query { td::Bits256 adnl_id_; std::string file_name_; }; + +class AddFastSyncOverlayClientQuery : public Query { + public: + AddFastSyncOverlayClientQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "addfastsyncoverlayclient"; + } + static std::string get_help() { + return "addfastsyncoverlayclient \tstarts issuing member certificates " + "to (hex) on slot (int)"; + } + std::string name() const override { + return get_name(); + } + + private: + td::Bits256 adnl_id_; + td::int32 slot_; +}; + +class DelFastSyncOverlayClientQuery : public Query { + public: + DelFastSyncOverlayClientQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "delfastsyncoverlayclient"; + } + static std::string get_help() { + return "delfastsyncoverlayclient \tstops issuing member certificates " + "to (hex)"; + } + std::string name() const override { + return get_name(); + } + + private: + td::Bits256 adnl_id_; +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 9e85b567e8..d803168120 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -168,6 +168,8 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 5f4c7c5cce..abba70c575 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -27,9 +27,17 @@ */ #include "validator-engine.hpp" +#include "adnl/adnl-node-id.hpp" #include "auto/tl/ton_api.h" +#include "errorcode.h" +#include "keys/keys.hpp" #include "overlay-manager.h" +#include "overlays.h" +#include "td/actor/PromiseFuture.h" #include "td/actor/actor.h" +#include "td/utils/Status.h" +#include "td/utils/Time.h" +#include "td/utils/buffer.h" #include "tl-utils/tl-utils.hpp" #include "tl/TlObject.h" #include "ton/ton-types.h" @@ -59,6 +67,8 @@ #include "memprof/memprof.h" #include "dht/dht.hpp" +#include +#include #if TD_DARWIN || TD_LINUX #include @@ -182,6 +192,10 @@ Config::Config(const ton::ton_api::engine_validator_config &config) { collator_node_whitelist.emplace(id); } } + for (auto &client : config.extraconfig_->fast_sync_overlay_clients_) { + auto key = ton::adnl::AdnlNodeIdShort{client->adnl_id_}; + fast_sync_overlay_clients.emplace_back(std::move(key), client->slot_); + } } else { state_serializer_enabled = true; } @@ -282,7 +296,8 @@ ton::tl_object_ptr Config::tl() const { } ton::tl_object_ptr extra_config_obj = {}; - if (!state_serializer_enabled || !fast_sync_member_certificates.empty() || collator_node_whitelist_obj) { + if (!state_serializer_enabled || !fast_sync_member_certificates.empty() || collator_node_whitelist_obj || + !fast_sync_overlay_clients.empty()) { // Non-default values extra_config_obj = ton::create_tl_object(); extra_config_obj->state_serializer_enabled_ = state_serializer_enabled; @@ -292,6 +307,11 @@ ton::tl_object_ptr Config::tl() const { certificate.tl())); } extra_config_obj->collator_node_whitelist_ = std::move(collator_node_whitelist_obj); + for (const auto &client : fast_sync_overlay_clients) { + extra_config_obj->fast_sync_overlay_clients_.push_back( + ton::create_tl_object(client.id.bits256_value(), + client.slot)); + } } std::vector> liteserver_vec; @@ -1352,10 +1372,12 @@ void ValidatorEngine::schedule_shutdown(double at) { LOG(DEBUG) << "Scheduled shutdown is in past (" << at << ")"; } else { LOG(INFO) << "Schedule shutdown for " << at << " (in " << ts.in() << "s)"; - ton::delay_action([]() { - LOG(WARNING) << "Shutting down as scheduled"; - std::_Exit(0); - }, ts); + ton::delay_action( + []() { + LOG(WARNING) << "Shutting down as scheduled"; + std::_Exit(0); + }, + ts); } } void ValidatorEngine::start_up() { @@ -1388,12 +1410,14 @@ void ValidatorEngine::alarm() { auto cur_t = config->get_validator_set_start_stop(0); CHECK(cur_t.first > 0); - auto val_set = state_->get_total_validator_set(0); - auto e = val_set->export_vector(); std::set to_del; for (auto &val : config_.validators) { bool is_validator = false; - if (val_set->is_validator(ton::NodeIdShort{val.first.bits256_value()})) { + if (validator_set_next_.not_null() && + validator_set_next_->is_validator(ton::NodeIdShort{val.first.bits256_value()})) { + is_validator = true; + } + if (validator_set_.not_null() && validator_set_->is_validator(ton::NodeIdShort{val.first.bits256_value()})) { is_validator = true; } if (!is_validator && val.second.election_date < cur_t.first && cur_t.first + 600 < state_->get_unix_time()) { @@ -1414,9 +1438,41 @@ void ValidatorEngine::alarm() { need_write = true; } + { + std::set fs_to_del; + for (auto &x : config_.fast_sync_member_certificates) { + if (x.second.is_expired()) { + fs_to_del.insert(x.first); + continue; + } + auto issued_by = x.second.issued_by().compute_short_id().bits256_value(); + if (validator_set_.not_null() && validator_set_->is_validator(issued_by)) { + continue; + } + if (validator_set_prev_.not_null() && validator_set_prev_->is_validator(issued_by)) { + continue; + } + if (validator_set_next_.not_null() && validator_set_next_->is_validator(issued_by)) { + continue; + } + fs_to_del.insert(x.first); + } + if (!fs_to_del.empty()) { + need_write = true; + std::erase_if(config_.fast_sync_member_certificates, + [&](const std::pair &e) { + return !fs_to_del.contains(e.first); + }); + } + } + if (need_write) { write_config([](td::Unit) {}); } + if (issue_fast_sync_overlay_certificates_at_.is_in_past()) { + issue_fast_sync_overlay_certificates_at_ = td::Timestamp::in(60.0); + issue_fast_sync_overlay_certificates(); + } } for (auto &x : config_.gc) { if (running_gc_.count(x) == 0) { @@ -1443,6 +1499,16 @@ void ValidatorEngine::deleted_key(ton::PublicKeyHash x) { } } +void ValidatorEngine::got_state(td::Ref state) { + if (state_.not_null() && state_->get_block_id() == state->get_block_id()) { + return; + } + state_ = std::move(state); + validator_set_ = state_->get_total_validator_set(0); + validator_set_next_ = state_->get_total_validator_set(1); + validator_set_prev_ = state_->get_total_validator_set(-1); +} + td::Status ValidatorEngine::load_global_config() { TRY_RESULT_PREFIX(conf_data, td::read_file(global_config_), "failed to read: "); TRY_RESULT_PREFIX(conf_json, td::json_decode(conf_data.as_slice()), "failed to parse json: "); @@ -1596,25 +1662,24 @@ void ValidatorEngine::set_shard_check_function() { validator_options_.write().set_shard_check_function([](ton::ShardIdFull shard) -> bool { return true; }); } else { std::vector shards = {ton::ShardIdFull(ton::masterchainId)}; - for (const auto& [_, collator_shards] : config_.collators) { - for (const auto& shard : collator_shards) { + for (const auto &[_, collator_shards] : config_.collators) { + for (const auto &shard : collator_shards) { shards.push_back(shard); } } - for (const auto& s : config_.shards_to_monitor) { + for (const auto &s : config_.shards_to_monitor) { shards.push_back(s); } std::sort(shards.begin(), shards.end()); shards.erase(std::unique(shards.begin(), shards.end()), shards.end()); - validator_options_.write().set_shard_check_function( - [shards = std::move(shards)](ton::ShardIdFull shard) -> bool { - for (auto s : shards) { - if (shard_intersects(shard, s)) { - return true; - } - } - return false; - }); + validator_options_.write().set_shard_check_function([shards = std::move(shards)](ton::ShardIdFull shard) -> bool { + for (auto s : shards) { + if (shard_intersects(shard, s)) { + return true; + } + } + return false; + }); } } @@ -2100,7 +2165,8 @@ void ValidatorEngine::started_validator() { } void ValidatorEngine::start_full_node() { - if (!config_.full_node.is_zero() || config_.full_node_slaves.size() > 0) { + if (!config_.full_node.is_zero() || !config_.full_node_slaves.empty()) { + full_node_id_ = ton::adnl::AdnlNodeIdShort{config_.full_node}; auto pk = ton::PrivateKey{ton::privkeys::Ed25519::random()}; auto short_id = pk.compute_short_id(); td::actor::send_closure(keyring_, &ton::keyring::Keyring::add_key, std::move(pk), true, [](td::Unit) {}); @@ -2123,8 +2189,8 @@ void ValidatorEngine::start_full_node() { td::actor::send_closure(SelfId, &ValidatorEngine::started_full_node); }); full_node_ = ton::validator::fullnode::FullNode::create( - short_id, ton::adnl::AdnlNodeIdShort{config_.full_node}, validator_options_->zero_block_id().file_hash, - config_.full_node_config, keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(), + short_id, full_node_id_, validator_options_->zero_block_id().file_hash, config_.full_node_config, + keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(), default_dht_node_.is_zero() ? td::actor::ActorId{} : dht_nodes_[default_dht_node_].get(), overlay_manager_.get(), validator_manager_.get(), full_node_client_.get(), db_root_, std::move(P)); for (auto &v : config_.validators) { @@ -2143,6 +2209,7 @@ void ValidatorEngine::start_full_node() { validator_telemetry_filename_); } load_custom_overlays_config(); + register_fast_sync_certificate_callback(); } else { started_full_node(); } @@ -2382,9 +2449,17 @@ void ValidatorEngine::try_add_full_node_adnl_addr(ton::PublicKeyHash id, td::Pro return; } - if (!full_node_.empty()) { - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::update_adnl_id, - ton::adnl::AdnlNodeIdShort{id}, [](td::Unit) {}); + if (!full_node_.empty() && id != full_node_id_.pubkey_hash()) { + td::actor::send_closure( + adnl_.get(), &ton::adnl::Adnl::unsubscribe, full_node_id_, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_newFastSyncMemberCertificate::ID)); + td::actor::send_closure( + adnl_.get(), &ton::adnl::Adnl::unsubscribe, full_node_id_, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_requestFastSyncOverlayMemberCertificate::ID)); + full_node_id_ = ton::adnl::AdnlNodeIdShort{id}; + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::update_adnl_id, full_node_id_, + [](td::Unit) {}); + register_fast_sync_certificate_callback(); } write_config(std::move(promise)); @@ -2644,9 +2719,225 @@ void ValidatorEngine::try_del_proxy(td::uint32 ip, td::int32 port, std::vector validator_engine) + : validator_engine_(std::move(validator_engine)) { + } + void receive_message(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, + td::BufferSlice data) override { + auto R = ton::fetch_tl_object( + std::move(data), true); + if (R.is_error()) { + return; + } + auto res = R.move_as_ok(); + auto cert = ton::overlay::OverlayMemberCertificate(res->certificate_.get()); + if (cert.empty()) { + return; + } + LOG(DEBUG) << "Received tonNode.newFastSyncMemberCertificate from " << src; + td::actor::send_closure(validator_engine_, &ValidatorEngine::try_import_fast_sync_member_certificate, dst, + std::move(cert), td::PromiseCreator::lambda([](td::Result R) { + if (R.is_error()) { + LOG(WARNING) << "failed to import overlay member certificate: " << R.move_as_error(); + } + })); + } + void receive_query(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + auto R = + ton::fetch_tl_object(std::move(data), true); + if (R.is_error()) { + return; + } + auto q = R.move_as_ok(); + td::actor::send_closure( + validator_engine_, &ValidatorEngine::process_fast_sync_overlay_certificate_request, + ton::PublicKeyHash{q->sign_by_}, ton::adnl::AdnlNodeIdShort{q->adnl_id_}, 0, q->slot_, + (td::int32)td::Clocks::system() + 3600, + td::PromiseCreator::lambda( + [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + auto cert = R.move_as_ok(); + promise.set_value(ton::serialize_tl_object(cert.tl(), true)); + })); + } + + private: + td::actor::ActorId validator_engine_; + }; + td::actor::send_closure( + adnl_.get(), &ton::adnl::Adnl::subscribe, full_node_id_, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_newFastSyncMemberCertificate::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure( + adnl_.get(), &ton::adnl::Adnl::subscribe, full_node_id_, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_requestFastSyncOverlayMemberCertificate::ID), + std::make_unique(actor_id(this))); +} + +void ValidatorEngine::try_import_fast_sync_member_certificate(ton::adnl::AdnlNodeIdShort id, + ton::overlay::OverlayMemberCertificate certificate, + td::Promise promise) { + if (!started_ || state_.is_null()) { + return promise.set_error(td::Status::Error("not started")); + } + if (certificate.slot() < 0 || + certificate.slot() >= ton::validator::fullnode::FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS) { + return promise.set_error(td::Status::Error( + PSTRING() << "invalid slot (max " << ton::validator::fullnode::FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS + << " clients)")); + } + if (certificate.expire_at() < td::Clocks::system() + 60) { + return promise.set_error(td::Status::Error("certificate expires too soon")); + } + TRY_STATUS_PROMISE_PREFIX(promise, certificate.check_signature(id), "invalid certificate: "); + + auto cert_score = [this](ton::overlay::OverlayMemberCertificate &cert) -> td::int64 { + auto issued_by = cert.issued_by().compute_short_id().bits256_value(); + if (validator_set_next_.not_null() && validator_set_next_->is_validator(issued_by)) { + return cert.expire_at() + (1ll << 32); + } + if (validator_set_.not_null() && validator_set_->is_validator(issued_by)) { + return cert.expire_at() + (1ll << 32); + } + if (validator_set_prev_.not_null() && validator_set_prev_->is_validator(issued_by)) { + return cert.expire_at() + (0ll << 32); + } + return -1; + }; + for (auto &x : config_.fast_sync_member_certificates) { + if (x.first == id) { + // fast check + if (x.second.issued_by() == certificate.issued_by()) { + if (x.second.expire_at() < certificate.expire_at()) { + x.second = std::move(certificate); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, + x.first, x.second); + write_config(std::move(promise)); + return; + } + LOG(DEBUG) << "Not importing certificate: certificate from the same issuer exists with bigger ttl"; + promise.set_value(td::Unit()); + return; + } + auto new_score = cert_score(certificate); + auto old_score = cert_score(x.second); + if (new_score > old_score) { + x.second = std::move(certificate); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, + x.first, x.second); + write_config(std::move(promise)); + return; + } + LOG(DEBUG) << "Not importing certificate: certificate with better score exists"; + promise.set_value(td::Unit()); + return; + } + } + + auto new_score = cert_score(certificate); + if (new_score < 0) { + LOG(DEBUG) << "Not importing certificate: issuer is not a validator"; + promise.set_value(td::Unit()); + return; + } + + auto &x = config_.fast_sync_member_certificates.emplace_back(std::move(id), std::move(certificate)); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, x.first, + x.second); + write_config(std::move(promise)); +} + +void ValidatorEngine::issue_fast_sync_overlay_certificates() { + if (state_.is_null() || config_.fast_sync_overlay_clients.empty() || full_node_id_.is_zero()) { + return; + } + auto issue_by = find_local_validator_for_cert_issuing(); + if (issue_by.is_zero()) { + return; + } + auto src = full_node_id_; + td::int32 expire_at = static_cast(td::Clocks::system()) + 3600; + for (auto &client : config_.fast_sync_overlay_clients) { + issue_fast_sync_overlay_certificate( + issue_by, client.id, 0, client.slot, expire_at, + [src, dst = client.id, adnl = adnl_.get()](td::Result R) { + if (R.is_error()) { + LOG(WARNING) << "cannot issue fast sync overlay certificate for " << dst << ": " << R.move_as_error(); + return; + } + auto cert = R.move_as_ok(); + LOG(INFO) << "Sending fast sync overlay certificate issued by " << cert.issued_by().compute_short_id() + << " to " << dst << " slot " << cert.slot(); + td::actor::send_closure(adnl, &ton::adnl::Adnl::send_message, src, dst, + ton::create_serialize_tl_object( + dst.bits256_value(), cert.tl())); + }); + } +} + +void ValidatorEngine::issue_fast_sync_overlay_certificate(ton::PublicKeyHash issue_by, + ton::adnl::AdnlNodeIdShort issue_to, td::uint32 flags, + td::int32 slot, td::int32 expire_at, + td::Promise promise) { + if (issue_by.is_zero()) { + issue_by = find_local_validator_for_cert_issuing(); + if (issue_by.is_zero()) { + return promise.set_error(td::Status::Error(ton::ErrorCode::notready, "cannot find a local validator")); + } + } + if (expire_at < td::Clocks::system() + 10) { + return promise.set_error(td::Status::Error(ton::ErrorCode::error, "expire at is in too near future")); + } + ton::overlay::OverlayMemberCertificate cert(ton::PublicKey(), flags, slot, expire_at, td::BufferSlice()); + auto to_sign = cert.to_sign_data(issue_to); + td::actor::send_closure(keyring_, &ton::keyring::Keyring::sign_add_get_public_key, issue_by, std::move(to_sign), + [slot, flags, expire_at, promise = std::move(promise)]( + td::Result> R) mutable { + TRY_RESULT_PROMISE_PREFIX(promise, res, std::move(R), "failed to sign certificate: "); + ton::overlay::OverlayMemberCertificate cert(std::move(res.second), flags, slot, expire_at, + std::move(res.first)); + promise.set_value(std::move(cert)); + }); +} + +void ValidatorEngine::process_fast_sync_overlay_certificate_request( + ton::PublicKeyHash issue_by, ton::adnl::AdnlNodeIdShort issue_to, td::uint32 flags, td::int32 slot, + td::int32 expire_at, td::Promise promise) { + for (auto &client : config_.fast_sync_overlay_clients) { + if (client.id == issue_to && (slot < 0 || slot == client.slot)) { + return issue_fast_sync_overlay_certificate(std::move(issue_by), std::move(issue_to), flags, client.slot, + expire_at, std::move(promise)); + } + } + promise.set_error(td::Status::Error(ton::ErrorCode::error, "cannot issue certificate to unknown adnl id")); +} + +ton::PublicKeyHash ValidatorEngine::find_local_validator_for_cert_issuing() { + if (state_.is_null()) { + return ton::PublicKeyHash{}; + } + for (auto& val_set : {validator_set_, validator_set_next_, validator_set_prev_}) { + if (val_set.is_null()) { + continue; + } + for (auto &[val_id, _]: config_.validators) { + if (val_set->is_validator(ton::NodeIdShort{val_id.bits256_value()})) { + return val_id; + } + } + } + return ton::PublicKeyHash::zero(); +} + void ValidatorEngine::load_custom_overlays_config() { - custom_overlays_config_ = - ton::create_tl_object(); + custom_overlays_config_ = ton::create_tl_object(); auto data_R = td::read_file(custom_overlays_config_file()); if (data_R.is_error()) { return; @@ -2699,7 +2990,7 @@ void ValidatorEngine::del_custom_overlay_from_config(std::string name, td::Promi static td::Result> parse_collator_options(td::MutableSlice json_str) { td::Ref ref{true}; - ton::validator::CollatorOptions& opts = ref.write(); + ton::validator::CollatorOptions &opts = ref.write(); // Set default values (from_json leaves missing fields as is) ton::ton_api::engine_validator_collatorOptions f; @@ -3706,7 +3997,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importCer return; } auto r = ton::overlay::Certificate::create(std::move(query.cert_)); - if(r.is_error()) { + if (r.is_error()) { promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: "))); } //TODO force Overlays::update_certificate to return result @@ -3721,17 +4012,15 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importCer }); */ td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::update_certificate, - ton::adnl::AdnlNodeIdShort{query.local_id_->id_}, - ton::overlay::OverlayIdShort{query.overlay_id_}, - ton::PublicKeyHash{query.signed_key_->key_hash_}, - r.move_as_ok()); - promise.set_value( - ton::serialize_tl_object(ton::create_tl_object(), true) - ); + ton::adnl::AdnlNodeIdShort{query.local_id_->id_}, + ton::overlay::OverlayIdShort{query.overlay_id_}, + ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok()); + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -3746,7 +4035,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importSha return; } auto r = ton::overlay::Certificate::create(std::move(query.cert_)); - if(r.is_error()) { + if (r.is_error()) { promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: "))); } auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { @@ -3758,12 +4047,13 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importSha } }); ton::ShardIdFull shard_id{ton::WorkchainId{query.workchain_}, static_cast(query.shard_)}; - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_shard_overlay_certificate, - shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok(), std::move(P)); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_shard_overlay_certificate, shard_id, + ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok(), std::move(P)); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -3785,11 +4075,11 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShard promise.set_value(R.move_as_ok()); } }); - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::sign_shard_overlay_certificate, - shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, query.expire_at_, query.max_size_, std::move(P)); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::sign_shard_overlay_certificate, shard_id, + ton::PublicKeyHash{query.signed_key_->key_hash_}, query.expire_at_, query.max_size_, + std::move(P)); } - void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_default)) { @@ -3857,42 +4147,46 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getPerfTi return; } - auto P = td::PromiseCreator::lambda( - [promise = std::move(promise), query = std::move(query)](td::Result> R) mutable { - const std::vector times{60, 300, 3600}; - double now = td::Time::now(); - if (R.is_error()) { - promise.set_value(create_control_query_error(R.move_as_error())); - } else { - auto r = R.move_as_ok(); - std::vector> by_name; - for (const auto &stats : r) { - if (stats.name == query.name_ || query.name_.empty()) { - std::vector> by_time; - for (const auto &t : times) { - double min = std::numeric_limits::lowest(); - double max = std::numeric_limits::max(); - double sum = 0; - int cnt = 0; - for (const auto &stat : stats.stats) { - double time = stat.first; - double duration = stat.second; - if (now - time <= static_cast(t)) { - min = td::min(min, duration); - max = td::max(max, duration); - sum += duration; - ++cnt; - } - } - by_time.push_back(ton::create_tl_object(t, min, sum / static_cast(cnt), max)); + auto P = td::PromiseCreator::lambda([promise = std::move(promise), query = std::move(query)]( + td::Result> R) mutable { + const std::vector times{60, 300, 3600}; + double now = td::Time::now(); + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + auto r = R.move_as_ok(); + std::vector> by_name; + for (const auto &stats : r) { + if (stats.name == query.name_ || query.name_.empty()) { + std::vector> by_time; + for (const auto &t : times) { + double min = std::numeric_limits::lowest(); + double max = std::numeric_limits::max(); + double sum = 0; + int cnt = 0; + for (const auto &stat : stats.stats) { + double time = stat.first; + double duration = stat.second; + if (now - time <= static_cast(t)) { + min = td::min(min, duration); + max = td::max(max, duration); + sum += duration; + ++cnt; } - by_name.push_back(ton::create_tl_object(stats.name, std::move(by_time))); } + by_time.push_back(ton::create_tl_object( + t, min, sum / static_cast(cnt), max)); } - promise.set_value(ton::create_serialize_tl_object(std::move(by_name))); + by_name.push_back(ton::create_tl_object( + stats.name, std::move(by_time))); } - }); - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::prepare_perf_timer_stats, std::move(P)); + } + promise.set_value( + ton::create_serialize_tl_object(std::move(by_name))); + } + }); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::prepare_perf_timer_stats, + std::move(P)); } void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getShardOutQueueSize &query, @@ -4015,9 +4309,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setExtMes }); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCustomOverlay &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCustomOverlay &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -4086,9 +4379,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCustom }); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_default)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -4312,7 +4604,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addShard if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); } else { - promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); } }); } @@ -4348,7 +4641,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delShard if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); } else { - promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); } }); } @@ -4484,7 +4778,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollat if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); } else { - promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); } }); } @@ -4529,7 +4824,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCollat if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); } else { - promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); } }); } @@ -4550,30 +4846,16 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signOverl ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; int slot = query.slot_; int expire_at = query.expire_at_; - td::actor::send_closure( - keyring_, &ton::keyring::Keyring::get_public_key, public_key_hash, - [=, keyring = keyring_.get(), promise = std::move(promise)](td::Result R) mutable { + + issue_fast_sync_overlay_certificate( + public_key_hash, adnl_id, 0, slot, expire_at, + [promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); return; } - ton::overlay::OverlayMemberCertificate certificate{R.move_as_ok(), 0, slot, expire_at, td::BufferSlice{}}; - if (certificate.is_expired()) { - promise.set_value( - create_control_query_error(td::Status::Error(ton::ErrorCode::error, "certificate is expired"))); - return; - } - td::BufferSlice to_sign = certificate.to_sign_data(adnl_id); - td::actor::send_closure(keyring, &ton::keyring::Keyring::sign_message, public_key_hash, std::move(to_sign), - [certificate = std::move(certificate), - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_value(create_control_query_error(R.move_as_error())); - return; - } - certificate.set_signature(R.move_as_ok()); - promise.set_value(ton::serialize_tl_object(certificate.tl(), true)); - }); + auto cert = R.move_as_ok(); + promise.set_value(ton::serialize_tl_object(cert.tl(), true)); }); } @@ -4600,23 +4882,92 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importFas promise.set_value(create_control_query_error(std::move(S))); return; } - for (auto &old_cert : config_.fast_sync_member_certificates) { - if (old_cert.first == adnl_id && old_cert.second.issued_by() == certificate.issued_by() && - old_cert.second.expire_at() == certificate.expire_at() && old_cert.second.slot() == certificate.slot()) { - promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "duplicate certificate"))); + + try_import_fast_sync_member_certificate( + std::move(adnl_id), std::move(certificate), [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addFastSyncClient &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; + td::int32 slot = query.slot_; + if (slot < 0 || slot >= ton::validator::fullnode::FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS) { + promise.set_value(create_control_query_error(td::Status::Error( + PSTRING() << "invalid slot (max " << ton::validator::fullnode::FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS + << " clients)"))); + return; + } + + bool found = false; + for (auto &c : config_.fast_sync_overlay_clients) { + if (c.slot == slot) { + if (c.id == adnl_id) { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + issue_fast_sync_overlay_certificates(); + } else { + promise.set_value(create_control_query_error(td::Status::Error("duplicate slot"))); + } return; } + if (c.id == adnl_id) { + found = true; + c.slot = slot; + } } + if (!found) { + config_.fast_sync_overlay_clients.emplace_back(adnl_id, slot); + } + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); + issue_fast_sync_overlay_certificates(); +} - config_.fast_sync_member_certificates.emplace_back(adnl_id, certificate); - write_config([=, promise = std::move(promise), full_node = full_node_.get()](td::Result R) mutable { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delFastSyncClient &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; + for (auto &c : config_.fast_sync_overlay_clients) { + if (c.id == adnl_id) { + std::swap(c, config_.fast_sync_overlay_clients.back()); + config_.fast_sync_overlay_clients.pop_back(); + break; + } + } + write_config([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); } else { - if (!full_node.empty()) { - td::actor::send_closure(full_node, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, - adnl_id, std::move(certificate)); - } promise.set_value( ton::serialize_tl_object(ton::create_tl_object(), true)); } @@ -4692,9 +5043,8 @@ void ValidatorEngine::get_current_validator_perm_key(td::Promiseget_total_validator_set(0); - CHECK(val_set.not_null()); - auto vec = val_set->export_vector(); + CHECK(validator_set_.not_null()); + auto vec = validator_set_->export_vector(); for (size_t idx = 0; idx < vec.size(); idx++) { auto &el = vec[idx]; ton::PublicKey pub{ton::pubkeys::Ed25519{el.key.as_bits256()}}; diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 89dd22cc4f..eac31f46aa 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -27,11 +27,14 @@ */ #pragma once +#include "adnl/adnl-node-id.hpp" #include "adnl/adnl.h" #include "auto/tl/ton_api.h" +#include "overlays.h" #include "rldp/rldp.h" #include "rldp2/rldp.h" #include "dht/dht.h" +#include "td/actor/PromiseFuture.h" #include "validator/manager.h" #include "validator/validator.h" #include "validator/full-node.h" @@ -75,6 +78,13 @@ struct Config { ton::PublicKey key; td::IPAddress addr; }; + struct FastSyncOverlayClient { + FastSyncOverlayClient() = default; + FastSyncOverlayClient(ton::adnl::AdnlNodeIdShort id, td::int32 slot) : id(id), slot(slot) { + } + ton::adnl::AdnlNodeIdShort id; + td::int32 slot; + }; std::map keys_refcnt; td::uint16 out_port; @@ -93,6 +103,7 @@ struct Config { std::map controls; std::set gc; std::vector shards_to_monitor; + std::vector fast_sync_overlay_clients; bool state_serializer_enabled = true; std::vector> @@ -158,6 +169,7 @@ class ValidatorEngine : public td::actor::Actor { td::actor::ActorOwn validator_manager_; td::actor::ActorOwn full_node_client_; td::actor::ActorOwn full_node_; + ton::adnl::AdnlNodeIdShort full_node_id_ = ton::adnl::AdnlNodeIdShort::zero(); std::map> full_node_masters_; td::actor::ActorOwn control_ext_server_; @@ -187,13 +199,13 @@ class ValidatorEngine : public td::actor::Actor { std::map keys_; td::Ref state_; + td::Ref validator_set_, validator_set_prev_, validator_set_next_; + td::Timestamp issue_fast_sync_overlay_certificates_at_ = td::Timestamp::now(); td::Promise get_key_promise(td::MultiPromise::InitGuard &ig); void got_key(ton::PublicKey key); void deleted_key(ton::PublicKeyHash key); - void got_state(td::Ref state) { - state_ = std::move(state); - } + void got_state(td::Ref state); void write_config(td::Promise promise); @@ -420,6 +432,20 @@ class ValidatorEngine : public td::actor::Actor { void try_del_proxy(td::uint32 ip, td::int32 port, std::vector cats, std::vector prio_cats, td::Promise promise); + void register_fast_sync_certificate_callback(); + void try_import_fast_sync_member_certificate(ton::adnl::AdnlNodeIdShort id, + ton::overlay::OverlayMemberCertificate certificate, + td::Promise promise); + + void issue_fast_sync_overlay_certificates(); + void issue_fast_sync_overlay_certificate(ton::PublicKeyHash issue_by, ton::adnl::AdnlNodeIdShort issue_to, + td::uint32 flags, td::int32 slot, td::int32 expire_at, + td::Promise promise); + void process_fast_sync_overlay_certificate_request(ton::PublicKeyHash issue_by, ton::adnl::AdnlNodeIdShort issue_to, + td::uint32 flags, td::int32 slot, td::int32 expire_at, + td::Promise promise); + ton::PublicKeyHash find_local_validator_for_cert_issuing(); + std::string custom_overlays_config_file() const { return db_root_ + "/custom-overlays.json"; } @@ -432,8 +458,8 @@ class ValidatorEngine : public td::actor::Actor { void load_custom_overlays_config(); td::Status write_custom_overlays_config(); - void add_custom_overlay_to_config( - ton::tl_object_ptr overlay, td::Promise promise); + void add_custom_overlay_to_config(ton::tl_object_ptr overlay, + td::Promise promise); void del_custom_overlay_from_config(std::string name, td::Promise promise); void load_collator_options(); @@ -560,6 +586,10 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getAdnlStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_addFastSyncClient &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_delFastSyncClient &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index 947f9dee8c..6554c51801 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -308,7 +308,7 @@ void FullNodeFastSyncOverlay::init() { options.default_permanent_members_flags_ = overlay::OverlayMemberFlags::DoNotReceiveBroadcasts; } options.local_overlay_member_flags_ = receive_broadcasts_ ? 0 : overlay::OverlayMemberFlags::DoNotReceiveBroadcasts; - options.max_slaves_in_semiprivate_overlay_ = 100000; // TODO: set lower limit (high limit for testing) + options.max_slaves_in_semiprivate_overlay_ = FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS; td::actor::send_closure(overlays_, &overlay::Overlays::create_semiprivate_overlay, local_id_, overlay_id_full_.clone(), current_validators_adnl_, root_public_keys_, member_certificate_, std::make_unique(actor_id(this)), rules, std::move(scope), options); diff --git a/validator/full-node.h b/validator/full-node.h index f8fc123bee..c9a49472d8 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -111,6 +111,8 @@ class FullNode : public td::actor::Actor { } enum { broadcast_mode_public = 1, broadcast_mode_private_block = 2, broadcast_mode_custom = 4 }; + static constexpr td::int32 MAX_FAST_SYNC_OVERLAY_CLIENTS = 5000; // TODO: set lower limit (high limit for testing) + static td::actor::ActorOwn create( ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 414b3170a8..b625e4c2d9 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -100,7 +100,10 @@ class FullNodeImpl : public FullNode { void set_validator_telemetry_filename(std::string value) override; void import_fast_sync_member_certificate(adnl::AdnlNodeIdShort local_id, - overlay::OverlayMemberCertificate cert) override { + overlay::OverlayMemberCertificate cert) override { + VLOG(FULL_NODE_DEBUG) << "Importing fast sync overlay certificate for " << local_id << " issued by " + << cert.issued_by().compute_short_id() << " expires in " + << (double)cert.expire_at() - td::Clocks::system(); fast_sync_overlays_.add_member_certificate(local_id, std::move(cert)); } From 20c20e236b4a9341174dd648e98c9d5585f1fcc7 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 10 Dec 2024 15:46:28 +0300 Subject: [PATCH 132/524] Check peer version before getOutMsgQueueProof --- validator/full-node-shard.cpp | 30 +++++++++++++++++++++--------- validator/full-node-shard.hpp | 2 +- validator/manager.cpp | 2 +- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 7d33a1950a..e62bfe42b1 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -1005,7 +1005,7 @@ void FullNodeShardImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std: block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise>> promise) { // TODO: maybe more complex download (like other requests here) - auto &b = choose_neighbour(); + auto &b = choose_neighbour(3, 0); // Required version: 3.0 if (b.adnl_id == adnl::AdnlNodeIdShort::zero()) { promise.set_error(td::Status::Error(ErrorCode::notready, "no nodes")); return; @@ -1252,24 +1252,36 @@ void FullNodeShardImpl::got_neighbours(std::vector vec) { } } -const Neighbour &FullNodeShardImpl::choose_neighbour() const { +const Neighbour &FullNodeShardImpl::choose_neighbour(td::uint32 required_version_major, + td::uint32 required_version_minor) const { if (neighbours_.size() == 0) { return Neighbour::zero; } + auto is_eligible = + [&](const Neighbour &n) { + return n.version_major > required_version_major || + (n.version_major == required_version_major && n.version_minor >= required_version_minor); + }; double min_unreliability = 1e9; - for (auto &x : neighbours_) { - min_unreliability = std::min(min_unreliability, x.second.unreliability); + for (auto &[_, x] : neighbours_) { + if (!is_eligible(x)) { + continue; + } + min_unreliability = std::min(min_unreliability, x.unreliability); } const Neighbour *best = nullptr; td::uint32 sum = 0; - for (auto &x : neighbours_) { - auto unr = static_cast(x.second.unreliability - min_unreliability); + for (auto &[_, x] : neighbours_) { + if (!is_eligible(x)) { + continue; + } + auto unr = static_cast(x.unreliability - min_unreliability); - if (x.second.version_major < proto_version_major()) { + if (x.version_major < proto_version_major()) { unr += 4; - } else if (x.second.version_major == proto_version_major() && x.second.version_minor < proto_version_minor()) { + } else if (x.version_major == proto_version_major() && x.version_minor < proto_version_minor()) { unr += 2; } @@ -1279,7 +1291,7 @@ const Neighbour &FullNodeShardImpl::choose_neighbour() const { auto w = 1 << (f - unr); sum += w; if (td::Random::fast(0, sum - 1) <= w - 1) { - best = &x.second; + best = &x; } } } diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index fd1ef943e8..59f671c9ae 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -209,7 +209,7 @@ class FullNodeShardImpl : public FullNodeShard { void got_neighbours(std::vector res); void update_neighbour_stats(adnl::AdnlNodeIdShort adnl_id, double t, bool success); void got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id, double t, td::BufferSlice data); - const Neighbour &choose_neighbour() const; + const Neighbour &choose_neighbour(td::uint32 required_version_major = 0, td::uint32 required_version_minor = 0) const; template td::Promise create_neighbour_promise(const Neighbour &x, td::Promise p, bool require_state = false) { diff --git a/validator/manager.cpp b/validator/manager.cpp index b8394bdc58..31278788cc 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1794,7 +1794,7 @@ void ValidatorManagerImpl::send_validator_telemetry(PublicKeyHash key, void ValidatorManagerImpl::send_get_out_msg_queue_proof_request( ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) { - callback_->download_out_msg_queue_proof(dst_shard, std::move(blocks), limits, td::Timestamp::in(10.0), + callback_->download_out_msg_queue_proof(dst_shard, std::move(blocks), limits, td::Timestamp::in(5.0), std::move(promise)); } From 160b539eaad7bc97b7e238168756cca676a5f3be Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 11 Dec 2024 16:07:48 +0300 Subject: [PATCH 133/524] Add logs add-collator --- .../validator-engine-console-query.cpp | 8 ++++++-- validator-engine-console/validator-engine-console-query.h | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index d229b93291..1822340677 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1607,7 +1607,9 @@ td::Status AddCollatorQuery::send() { td::Status AddCollatorQuery::receive(td::BufferSlice data) { TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), "received incorrect answer: "); - td::TerminalIO::out() << "successfully added collator\n"; + td::TerminalIO::out() << "successfully added collator for shard " << shard_.to_str() << "\n"; + td::TerminalIO::out() << "ADNL ID = " << adnl_id_.bits256_value().to_hex() << " (" << adnl_id_.bits256_value() + << ")\n"; return td::Status::OK(); } @@ -1627,7 +1629,9 @@ td::Status DelCollatorQuery::send() { td::Status DelCollatorQuery::receive(td::BufferSlice data) { TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), "received incorrect answer: "); - td::TerminalIO::out() << "successfully removed collator\n"; + td::TerminalIO::out() << "successfully removed collator for shard " << shard_.to_str() << "\n"; + td::TerminalIO::out() << "ADNL ID = " << adnl_id_.bits256_value().to_hex() << " (" << adnl_id_.bits256_value() + << ")\n"; return td::Status::OK(); } diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 4173745185..ad9f7c3b3d 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -151,8 +151,8 @@ inline td::Result Tokenizer::get_token() { auto r_wc = td::to_integer_safe(word); if (r_wc.is_ok()) { TRY_RESULT_ASSIGN(word, get_raw_token()); - TRY_RESULT(shard, td::to_integer_safe(word)); - return ton::ShardIdFull{r_wc.move_as_ok(), shard}; + TRY_RESULT(shard, td::to_integer_safe(word)); + return ton::ShardIdFull{r_wc.move_as_ok(), (ton::ShardId)shard}; } return ton::ShardIdFull::parse(word); } From c955a5333d7c8643b67d76ae42950af7e97796d1 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 24 Dec 2024 11:58:51 +0300 Subject: [PATCH 134/524] Fix loading block candidates in WaitBlockData --- validator/downloaders/wait-block-data.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/downloaders/wait-block-data.cpp b/validator/downloaders/wait-block-data.cpp index 53a3d351bc..86e8f4b8c1 100644 --- a/validator/downloaders/wait-block-data.cpp +++ b/validator/downloaders/wait-block-data.cpp @@ -106,7 +106,7 @@ void WaitBlockData::start() { }); td::actor::send_closure(manager_, &ValidatorManager::try_get_static_file, handle_->id().file_hash, std::move(P)); - } else if (try_get_candidate_) { + } else if (try_get_candidate_ && !handle_->id().is_masterchain()) { try_get_candidate_ = false; td::actor::send_closure( manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, handle_->id(), From 392cf64758a0698c4b237c3e233a9727b11685aa Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 26 Dec 2024 14:03:00 +0300 Subject: [PATCH 135/524] Support optional fields in tl json generator --- tdtl/td/tl/tl_simple.h | 13 +++++++++++++ tl/generate/tl_json_converter.cpp | 29 +++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/tdtl/td/tl/tl_simple.h b/tdtl/td/tl/tl_simple.h index dd5a1e53a4..06d6ea133d 100644 --- a/tdtl/td/tl/tl_simple.h +++ b/tdtl/td/tl/tl_simple.h @@ -81,12 +81,16 @@ struct Type { struct Arg { const Type *type; std::string name; + int var_num = -1; + int exist_var_num = -1; + int exist_var_bit = -1; }; struct Constructor { std::string name; std::int32_t id; std::vector args; + int var_count = 0; const CustomType *type; }; @@ -100,6 +104,7 @@ struct CustomType { struct Function { std::string name; + int var_count = 0; std::int32_t id; std::vector args; const Type *type; @@ -248,11 +253,15 @@ class Schema { constructor = constructors_.back().get(); constructor->id = from->id; constructor->name = from->name; + constructor->var_count = from->var_count; constructor->type = get_custom_type(config_->get_type(from->type_id)); for (auto &from_arg : from->args) { Arg arg; arg.name = from_arg.name; arg.type = get_type(from_arg.type); + arg.var_num = from_arg.var_num; + arg.exist_var_num = from_arg.exist_var_num; + arg.exist_var_bit = from_arg.exist_var_bit; constructor->args.push_back(std::move(arg)); } } @@ -266,11 +275,15 @@ class Schema { function = functions_.back().get(); function->id = from->id; function->name = from->name; + function->var_count = from->var_count; function->type = get_type(config_->get_type(from->type_id)); for (auto &from_arg : from->args) { Arg arg; arg.name = from_arg.name; arg.type = get_type(from_arg.type); + arg.var_num = from_arg.var_num; + arg.exist_var_num = from_arg.exist_var_num; + arg.exist_var_bit = from_arg.exist_var_bit; function->args.push_back(std::move(arg)); } } diff --git a/tl/generate/tl_json_converter.cpp b/tl/generate/tl_json_converter.cpp index a0213ef512..d1a0488fb8 100644 --- a/tl/generate/tl_json_converter.cpp +++ b/tl/generate/tl_json_converter.cpp @@ -48,13 +48,30 @@ void gen_to_json_constructor(StringBuilder &sb, const T *constructor, bool is_he sb << " {\n"; sb << " auto jo = jv.enter_object();\n"; sb << " jo(\"@type\", \"" << constructor->name << "\");\n"; + std::vector var_names(constructor->var_count); + for (auto &arg : constructor->args) { + if (arg.var_num >= 0) { + CHECK(arg.var_num < (int)var_names.size()); + var_names[arg.var_num] = tl::simple::gen_cpp_field_name(arg.name); + } + } for (auto &arg : constructor->args) { auto field_name = tl::simple::gen_cpp_field_name(arg.name); - // TODO: or as null - bool is_custom = arg.type->type == tl::simple::Type::Custom; - - if (is_custom) { - sb << " if (object." << field_name << ") {\n "; + bool is_optional = arg.type->type == tl::simple::Type::Custom || arg.exist_var_num >= 0; + + if (is_optional) { + sb << " if ("; + if (arg.type->type == tl::simple::Type::Custom) { + sb << "object." << field_name; + if (arg.exist_var_num >= 0) { + sb << " && "; + } + } + if (arg.exist_var_num >= 0) { + CHECK(arg.exist_var_num < (int)var_names.size()); + sb << "(object." << var_names[arg.exist_var_num] << " & " << (1 << arg.exist_var_bit) << ")"; + } + sb << ") {\n "; } auto object = PSTRING() << "object." << tl::simple::gen_cpp_field_name(arg.name); if (arg.type->type == tl::simple::Type::Bytes || arg.type->type == tl::simple::Type::SecureBytes) { @@ -72,7 +89,7 @@ void gen_to_json_constructor(StringBuilder &sb, const T *constructor, bool is_he object = PSTRING() << "JsonVectorInt64{" << object << "}"; } sb << " jo(\"" << arg.name << "\", ToJson(" << object << "));\n"; - if (is_custom) { + if (is_optional) { sb << " }\n"; } } From 71342bdcd4ba387f0629f797bf9de722c05752cd Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 26 Dec 2024 14:03:12 +0300 Subject: [PATCH 136/524] Rework session stats --- tl/generate/scheme/ton_api.tl | 90 +++++++++------ tl/generate/scheme/ton_api.tlo | Bin 110212 -> 111624 bytes ton/ton-types.h | 11 ++ validator-session/validator-session-types.h | 116 ++++++++++++++++--- validator-session/validator-session.cpp | 70 +++++------ validator-session/validator-session.h | 5 - validator-session/validator-session.hpp | 3 +- validator/collation-manager.cpp | 64 +++++----- validator/collation-manager.hpp | 4 +- validator/collator-node.cpp | 16 ++- validator/fabric.h | 3 +- validator/impl/collator-impl.h | 4 +- validator/impl/collator.cpp | 31 ++--- validator/impl/fabric.cpp | 13 ++- validator/impl/validate-query.cpp | 36 +++--- validator/impl/validate-query.hpp | 2 +- validator/interfaces/validator-manager.h | 78 ++++++++++--- validator/manager-disk.cpp | 3 +- validator/manager-disk.hpp | 9 -- validator/manager-hardfork.hpp | 9 -- validator/manager.cpp | 122 ++++++-------------- validator/manager.hpp | 18 +-- validator/validator-group.cpp | 55 ++++----- validator/validator-group.hpp | 12 +- 24 files changed, 438 insertions(+), 336 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 396c06bc1a..bdcf0a296a 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -895,48 +895,66 @@ http.server.config dhs:(vector http.server.dnsEntry) local_hosts:(vector http.se ---types--- -validatorSession.collationStats actual_bytes:int actual_collated_data_bytes:int - bytes:int gas:int lt_delta:int collated_data_bytes:int - cat_bytes:int cat_gas:int cat_lt_delta:int cat_collated_data_bytes:int - limits_log:string ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int - work_time:double cpu_work_time:double - serialized_size:int serialized_size_no_collated_data:int = validadorSession.CollationStats; - -validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int root_hash:int256 file_hash:int256 - comment:string block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double - collation_time:double collated_at:double collation_cached:Bool - collation_work_time:double collation_cpu_work_time:double - collation_stats:validatorSession.collationStats - validation_time:double validated_at:double validation_cached:Bool - validation_work_time:double validation_cpu_work_time:double - gen_utime:double - approved_weight:long approved_33pct_at:double approved_66pct_at:double approvers:string - signed_weight:long signed_33pct_at:double signed_66pct_at:double signers:string - serialize_time:double deserialize_time:double serialized_size:int = validatorSession.StatsProducer; - -validatorSession.statsRound timestamp:double producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound; - -validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:double self:int256 session_id:int256 cc_seqno:int - creator:int256 total_validators:int total_weight:long - signatures:int signatures_weight:long approve_signatures:int approve_signatures_weight:long - first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats; - -validatorSession.statsCollatedBlock timestamp:double id:tonNode.blockIdExt stats:validatorSession.collationStats = validatorSession.StatsCollatedBlock; +validatorStats.stats.producer flags:# + validator_id:int256 block_status:int + candidate_id:flags.0?int256 block_id:flags.0?tonNode.blockIdExt collated_data_hash:flags.0?int256 + is_accepted:flags.0?Bool is_ours:flags.0?Bool + got_block_at:flags.0?double got_submit_at:flags.0?double gen_utime:flags.0?int comment:flags.0?string + collation_time:flags.1?double collated_at:flags.1?double collation_cached:flags.1?Bool self_collated:flags.1?Bool collator_node_id:flags.2?int256 + validation_time:flags.3?double validated_at:flags.3?double validation_cached:flags.3?Bool + approved_weight:flags.0?long approved_33pct_at:flags.0?double approved_66pct_at:flags.0?double approvers:flags.0?string + signed_weight:flags.0?long signed_33pct_at:flags.0?double signed_66pct_at:flags.0?double signers:flags.0?string + serialize_time:flags.4?double deserialize_time:flags.4?double serialized_size:flags.4?int = validatorStats.stats.Producer; + +validatorStats.stats.round started_at:double producers:(vector validatorStats.stats.producer) = validatorStats.stats.Round; + +validatorStats.stats flags:# + session_id:int256 self:int256 block_id:tonNode.blockIdExt cc_seqno:int success:Bool timestamp:double + creator:flags.0?int256 + total_validators:int total_weight:long + signatures:flags.0?int signatures_weight:flags.0?long + approve_signatures:flags.0?int approve_signatures_weight:flags.0?long + first_round:int rounds:(vector validatorStats.stats.round) = validatorStats.Stats; + +validatorStats.blockLimitsStatus + bytes:int gas:int lt_delta:int collated_data_bytes:int + cat_bytes:int cat_gas:int cat_lt_delta:int cat_collated_data_bytes:int + limits_log:string = validatorStats.BlockLimitsStatus; +validatorStats.extMsgsStats + ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validatorStats.ExtMsgsStats; + +validatorStats.collatedBlock flags:# + block_id:tonNode.blockIdExt collated_data_hash:int256 cc_seqno:int collated_at:double + bytes:int collated_data_bytes:int attempt:int + collator_node_id:flags.0?int256 validator_id:flags.1?int256 + total_time:double work_time:double cpu_work_time:double time_stats:string + block_limits:validatorStats.blockLimitsStatus + ext_msgs_stats:validatorStats.extMsgsStats = validatorSession.stats.CollatedBlock; + +validatorStats.validatedBlock + block_id:tonNode.blockIdExt collated_data_hash:int256 validated_at:double + valid:Bool comment:string + bytes:int collated_data_bytes:int + total_time:double work_time:double cpu_work_time:double = validatorStats.ValidatedBlock; + +validatorStats.newValidatorGroup.node id:int256 weight:long = validatorStats.newValidatorGroup.Node; +validatorStats.newValidatorGroup session_id:int256 shard:tonNode.shardId cc_seqno:int + last_key_block_seqno:int started_at:double + self_idx:int nodes:(vector validatorStats.newValidatorGroup.node) = validatorStats.NewValidatorGroup; + +validatorStats.endValidatorGroup.node id:int256 catchain_blocks:int = validatorStats.endValidatorGroup.Node; +validatorStats.endValidatorGroup session_id:int256 timestamp:double + nodes:(vector validatorStats.endValidatorGroup.node) = validatorStats.EndValidatorGroup; + +validatorStats.collatorNodeResponse collator_node_id:int256 validator_id:int256 timestamp:double + block_id:tonNode.blockIdExt original_block_id:tonNode.blockIdExt collated_data_hash:int256 + = validatorStats.CollatorNodeResponse; collatorNode.candidate source:PublicKey id:tonNode.blockIdExt data:bytes collated_data:bytes = collatorNode.Candidate; collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt decompressed_size:int data:bytes = collatorNode.Candidate; collatorNode.pong flags:# = collatorNode.Pong; collatorNode.error code:int message:string = collatorNode.Error; -validatorSession.newValidatorGroupStats.node id:int256 weight:long = validatorSession.newValidatorGroupStats.Node; -validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int - last_key_block_seqno:int timestamp:double - self_idx:int nodes:(vector validatorSession.newValidatorGroupStats.node) = validatorSession.NewValidatorGroupStats; - -validatorSession.endValidatorGroupStats.node id:int256 catchain_blocks:int = validatorSession.endValidatorGroupStats.Node; -validatorSession.endValidatorGroupStats session_id:int256 timestamp:double - nodes:(vector validatorSession.endValidatorGroupStats.node) = validatorSession.EndValidatorGroupStats; - ---functions--- collatorNode.generateBlock shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt) creator:int256 round:int first_block_round:int priority:int = collatorNode.Candidate; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index faef1564525053095bedcd3d1a7a3d48fe583ec0..f091213cf90e633cb8909ffc33ba05310ad4d1f0 100644 GIT binary patch delta 2515 zcma)8Urbw781J{W1zP^JP^8$h7HJg%B2$pTn1IN}q(CWuB#Pd4?G-AexAtB(GFc+H z8BE+99?f|WP08Yum^t@=F>!GZ8cCKojV6+8I*rlA_~3(ydyvfY-CnxOrAy?Y;e6+J ze&2V#^ZS17$?L|&o5q4)-g&51r~K`|Q1E-p#fH|KK*rnlkssO`$?K0vhF?kHfQQbH zz&(ZaT|z7tlK9BuF(EufzInt+puKsiLGS&8n@G8jhGNl3ND`7mQb-c(WsomA9hF23 z%!s7Z8lFHB~kTwQT+TF%8?w{Z9M0|2$#SkjaQs|(p_Nme5TDT%FXsCg~&{p+$eIo z;v<`$do_vEpnxzFk1U-ph4qUq?Qi_OW^0DIVmkgLr07W>1lN2|LD2XhKDbkJKCFIt zrWeF)d|B+X&`K-Rk=GWg3X8We9MeFEo|y(0{g49>-57)_I@Sno`bi_y(bzNKwph|& zA<4zrn{!cWSq2-O^Mlo%$;?F~EYmJUoj@*}D;m8G*4Wya_WB+d}b3aIr)C1 z=+pAC{3EDME+*?vIh%Xn)yt?*P}Yv$eyS-a#DdXbYF@GS*=XZ%b3p z!dp$>?=g*M)4H5gt#!tSIP6j0+F&CL0J+hgJAyH#2pw`|n})5{sHXvO$Yo@QvHETO zqtkwH=d1fhR!%wT^;W1NiwmAotB{ONMB||tm+p<+S2F)nSzhMN$i|N@a|3_udAo;HonX%?~Ge^h;UeWS(=-TYV5Dsj!aw*$fI2Qq=>BD> zcsmFNbYwGaTSgVda_H?ru;$53tTjNax9oP2k3G+A`MKJIS3W4K*ycUVyVWke3tgJ7 z2EdYG+)XNDWvHO-Av@g+K#2;)s4YdI1Z(_JHjKyk=)|OymipNjSjj_BqFndq0Gk2B zaAwV}@3bfflcG{BHh>NfvvZK#zlM=z>It+cy;DYIWN@rVxhQDyiJXz(gmlE3S&jzF~?NwuSn2iNMJgE9Yq DLoy>h delta 1670 zcmb7EUrbwN6z@@5I_y|S3*1@)C19Xghy2k9%5-dXkZfaRAb%X}_O|yX>(=$M-di>o z6ZbF;7!zS%+%eIIy3A-I%TOL_j1P1n^JR&?m<&^5VivcUJ=BE6J?z2n+``(GzU)5S zeBb%=JLfy+ch2ks%jI>;&cD8R`E}Fg*Zuv@`#sIyJ-+o{!;BP-$datahZRkWsIj(# zYBVb8{4}gfy0)++EN}EY0N>SrI>nZpv7R@6J0_1B09@7|R}<5Q6I2h!sGKk5lx%mg>fWyE_Ef`%HN}xwqq)I5*OmOGsaVrygv@2F$Ooco_zbZ0bG1!# zELv4x+By(dGl@-=k;qG|U zmD#)9D^~LLPrTSbuYP4;H{4!cf9ul?v(Y2`suvvI!nTlj?RM1>2XCnR??1gZDXb@? zs2F-rS2Wgmqt22O;L33jSigTa%N{Ue&oO97-5Y{d!08Z}aq1oThRyezac=+)By6Lz_7Si?5buOzE1DEfv9vM@L}~JN2<2KOcn=yf6;UXdiBEVE+w zQss*g$@D2i;yfW%ZBg7rV?bz)tnce4ygS94EPo*2(iF6!?M)a!&k69@cZFzhL_Km= z(RAtT^k-RcVdNydz>;%M<#T4L!#|z&j#1 zS@SKy>3Cuyo>${JeDZbG<`v=TL|$pabO;1@Udyx5HF_fPgi63xesW?o2v%GIp5xlP z(1NQ$9&cmO<=mW6zP<90*yjhgO(;;|PyifQEIBbY-4!2c8Q<%On{WcbzN>0iJ0+#rS&S&g6sp qCt+lOPK9XCw6-?U$cHU`+lr4Tp}CB1sSF=6%he&6P5l^#K;VDclU@)2 diff --git a/ton/ton-types.h b/ton/ton-types.h index 220af6e02c..51d9e2d2e9 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -501,6 +501,17 @@ struct BlockCandidate { } }; +struct GeneratedCandidate { + BlockCandidate candidate; + bool is_cached = false; + bool self_collated = false; + td::Bits256 collator_node_id = td::Bits256::zero(); + + GeneratedCandidate clone() const { + return {candidate.clone(), is_cached, self_collated, collator_node_id}; + } +}; + struct BlockCandidatePriority { td::uint32 round{}; td::uint32 first_block_round{}; diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index 2edc42bbdf..59b2141aad 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -23,6 +23,8 @@ #include "adnl/adnl-node-id.hpp" #include "ton/ton-types.h" +#include + namespace ton { namespace validatorsession { @@ -73,24 +75,25 @@ struct ValidatorSessionStats { enum { status_none = 0, status_received = 1, status_rejected = 2, status_approved = 3 }; struct Producer { - PublicKeyHash id = PublicKeyHash::zero(); - ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero(); int block_status = status_none; - double block_timestamp = -1.0; - td::Bits256 root_hash = td::Bits256::zero(); - td::Bits256 file_hash = td::Bits256::zero(); - std::string comment; - + PublicKeyHash validator_id = PublicKeyHash::zero(); + ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero(); + BlockIdExt block_id{workchainIdNotYet, 0, 0, td::Bits256::zero(), td::Bits256::zero()}; + td::Bits256 collated_data_hash = td::Bits256::zero(); bool is_accepted = false; bool is_ours = false; + double got_block_at = -1.0; double got_submit_at = -1.0; - double collation_time = -1.0; - double validation_time = -1.0; - double collated_at = -1.0; - double validated_at = -1.0; + td::int32 gen_utime = -1; + std::string comment; + + double collation_time = -1.0, collated_at = -1.0; bool collation_cached = false; + bool self_collated = false; + td::Bits256 collator_node_id = td::Bits256::zero(); + + double validation_time = -1.0, validated_at = -1.0; bool validation_cached = false; - double gen_utime = -1.0; std::vector approvers, signers; ValidatorWeight approved_weight = 0; @@ -129,20 +132,52 @@ struct ValidatorSessionStats { } } } + + tl_object_ptr tl() const { + std::string approvers_str(approvers.size(), '0'); + for (size_t i = 0; i < approvers.size(); ++i) { + approvers_str[i] = '0' + approvers[i]; + } + std::string signers_str(signers.size(), '0'); + for (size_t i = 0; i < signers.size(); ++i) { + signers_str[i] = '0' + signers[i]; + } + int flags = + (block_status != status_none ? ton_api::validatorStats_stats_producer::Flags::BLOCK_ID_MASK : 0) | + (collated_at >= 0.0 ? ton_api::validatorStats_stats_producer::Flags::COLLATED_AT_MASK : 0) | + (!collator_node_id.is_zero() ? ton_api::validatorStats_stats_producer::Flags::COLLATOR_NODE_ID_MASK : 0) | + (validated_at >= 0.0 ? ton_api::validatorStats_stats_producer::Flags::VALIDATED_AT_MASK : 0) | + (serialize_time >= 0.0 || deserialize_time >= 0.0 || serialized_size >= 0 + ? ton_api::validatorStats_stats_producer::Flags::SERIALIZE_TIME_MASK + : 0); + return create_tl_object( + flags, validator_id.bits256_value(), block_status, candidate_id, create_tl_block_id(block_id), + collated_data_hash, is_accepted, is_ours, got_block_at, got_submit_at, gen_utime, comment, collation_time, + collated_at, collation_cached, self_collated, collator_node_id, validation_time, validated_at, + validation_cached, approved_weight, approved_33pct_at, approved_66pct_at, std::move(approvers_str), + signed_weight, signed_33pct_at, signed_66pct_at, std::move(signers_str), serialize_time, deserialize_time, + serialized_size); + } }; struct Round { - double timestamp = -1.0; + double started_at = -1.0; std::vector producers; - }; - td::uint32 first_round; - std::vector rounds; + tl_object_ptr tl() const { + std::vector> producers_tl; + for (const auto &producer : producers) { + producers_tl.push_back(producer.tl()); + } + return create_tl_object(started_at, std::move(producers_tl)); + } + }; - bool success = false; ValidatorSessionId session_id = ValidatorSessionId::zero(); + PublicKeyHash self = PublicKeyHash::zero(); + BlockIdExt block_id{workchainIdNotYet, 0, 0, td::Bits256::zero(), td::Bits256::zero()}; CatchainSeqno cc_seqno = 0; + bool success = false; double timestamp = -1.0; - PublicKeyHash self = PublicKeyHash::zero(); PublicKeyHash creator = PublicKeyHash::zero(); td::uint32 total_validators = 0; ValidatorWeight total_weight = 0; @@ -150,6 +185,29 @@ struct ValidatorSessionStats { ValidatorWeight signatures_weight = 0; td::uint32 approve_signatures = 0; ValidatorWeight approve_signatures_weight = 0; + + td::uint32 first_round = 0; + std::vector rounds; + + void fix_block_ids() { + for (auto &round : rounds) { + for (auto &producer : round.producers) { + producer.block_id.id = block_id.id; + } + } + } + + tl_object_ptr tl() const { + std::vector> rounds_tl; + for (const auto &round : rounds) { + rounds_tl.push_back(round.tl()); + } + int flags = success ? ton_api::validatorStats_stats::Flags::CREATOR_MASK : 0; + return create_tl_object( + flags, session_id, self.bits256_value(), create_tl_block_id(block_id), cc_seqno, success, timestamp, + creator.bits256_value(), total_validators, total_weight, signatures, signatures_weight, approve_signatures, + approve_signatures_weight, first_round, std::move(rounds_tl)); + } }; struct NewValidatorGroupStats { @@ -162,9 +220,20 @@ struct NewValidatorGroupStats { ShardIdFull shard{masterchainId}; CatchainSeqno cc_seqno = 0; BlockSeqno last_key_block_seqno = 0; - double timestamp = -1.0; + double started_at = -1.0; td::uint32 self_idx = 0; std::vector nodes; + + tl_object_ptr tl() const { + std::vector> nodes_arr; + for (const auto &node : nodes) { + nodes_arr.push_back( + create_tl_object(node.id.bits256_value(), node.weight)); + } + return create_tl_object(session_id, create_tl_shard_id(shard), cc_seqno, + last_key_block_seqno, started_at, self_idx, + std::move(nodes_arr)); + } }; struct EndValidatorGroupStats { @@ -176,6 +245,15 @@ struct EndValidatorGroupStats { ValidatorSessionId session_id = ValidatorSessionId::zero(); double timestamp = -1.0; std::vector nodes; + + tl_object_ptr tl() const { + std::vector> nodes_arr; + for (const auto &node : nodes) { + nodes_arr.push_back(create_tl_object(node.id.bits256_value(), + node.catchain_blocks)); + } + return create_tl_object(session_id, timestamp, std::move(nodes_arr)); + } }; struct BlockSourceInfo { diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 99ee61f23a..5cc31a7ba2 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -266,13 +266,14 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice if (stat->block_status == ValidatorSessionStats::status_none) { stat->block_status = ValidatorSessionStats::status_received; } - if (stat->block_timestamp <= 0.0) { - stat->block_timestamp = td::Clocks::system(); + if (stat->got_block_at <= 0.0) { + stat->got_block_at = td::Clocks::system(); } stat->deserialize_time = deserialize_time; stat->serialized_size = data.size(); - stat->root_hash = candidate->root_hash_; - stat->file_hash = file_hash; + stat->block_id.root_hash = candidate->root_hash_; + stat->block_id.file_hash = file_hash; + stat->collated_data_hash = collated_data_file_hash; } if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || @@ -451,35 +452,36 @@ void ValidatorSessionImpl::candidate_approved_signed(td::uint32 round, Validator } } -void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCandidateId root_hash, - td::BufferSlice data, td::BufferSlice collated_data, double collation_time, - bool collation_cached) { - if (data.size() > description().opts().max_block_size || - collated_data.size() > description().opts().max_collated_data_size) { - LOG(ERROR) << this << ": generated candidate is too big. Dropping. size=" << data.size() << " " - << collated_data.size(); +void ValidatorSessionImpl::generated_block(td::uint32 round, GeneratedCandidate c, double collation_time) { + if (c.candidate.data.size() > description().opts().max_block_size || + c.candidate.collated_data.size() > description().opts().max_collated_data_size) { + LOG(ERROR) << this << ": generated candidate is too big. Dropping. size=" << c.candidate.data.size() << " " + << c.candidate.collated_data.size(); return; } - auto file_hash = sha256_bits256(data.as_slice()); - auto collated_data_file_hash = sha256_bits256(collated_data.as_slice()); - auto block_id = description().candidate_id(local_idx(), root_hash, file_hash, collated_data_file_hash); + auto file_hash = sha256_bits256(c.candidate.data.as_slice()); + auto collated_data_file_hash = sha256_bits256(c.candidate.collated_data.as_slice()); + auto block_id = description().candidate_id(local_idx(), c.candidate.id.root_hash, file_hash, collated_data_file_hash); auto stat = stats_get_candidate_stat(round, local_id(), block_id); if (stat) { stat->block_status = ValidatorSessionStats::status_received; stat->collation_time = collation_time; stat->collated_at = td::Clocks::system(); - stat->block_timestamp = td::Clocks::system(); - stat->collation_cached = collation_cached; - stat->root_hash = root_hash; - stat->file_hash = file_hash; + stat->got_block_at = td::Clocks::system(); + stat->collation_cached = c.is_cached; + stat->self_collated = c.self_collated; + stat->collator_node_id = c.collator_node_id; + stat->block_id = c.candidate.id; + stat->collated_data_hash = collated_data_file_hash; } if (round != cur_round_) { return; } td::Timer serialize_timer; - auto b = create_tl_object(local_id().tl(), round, root_hash, std::move(data), - std::move(collated_data)); + auto b = create_tl_object(local_id().tl(), round, c.candidate.id.root_hash, + std::move(c.candidate.data), + std::move(c.candidate.collated_data)); auto B = serialize_candidate(b, compress_block_candidates_).move_as_ok(); if (stat) { stat->serialize_time = serialize_timer.elapsed(); @@ -546,9 +548,8 @@ void ValidatorSessionImpl::check_generate_slot() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), print_id = print_id(), timer = std::move(timer), round = cur_round_](td::Result R) { if (R.is_ok()) { - auto c = std::move(R.ok_ref().candidate); - td::actor::send_closure(SelfId, &ValidatorSessionImpl::generated_block, round, c.id.root_hash, - c.data.clone(), c.collated_data.clone(), timer.elapsed(), R.ok().is_cached); + td::actor::send_closure(SelfId, &ValidatorSessionImpl::generated_block, round, R.move_as_ok(), + timer.elapsed()); } else { LOG(WARNING) << print_id << ": failed to generate block candidate: " << R.move_as_error(); } @@ -606,11 +607,12 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { if (stat->block_status == ValidatorSessionStats::status_none) { stat->block_status = ValidatorSessionStats::status_received; } - if (stat->block_timestamp <= 0.0) { - stat->block_timestamp = td::Clocks::system(); + if (stat->got_block_at <= 0.0) { + stat->got_block_at = td::Clocks::system(); } - stat->root_hash = B->root_hash_; - stat->file_hash = td::sha256_bits256(B->data_); + stat->block_id.root_hash = B->root_hash_; + stat->block_id.file_hash = td::sha256_bits256(B->data_); + stat->collated_data_hash = td::sha256_bits256(B->collated_data_); } auto P = td::PromiseCreator::lambda([round = cur_round_, hash = block_id, root_hash = block->get_root_hash(), @@ -933,7 +935,7 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { while (round_idx >= cur_stats_.rounds.size()) { stats_add_round(); } - cur_stats_.rounds[round_idx].timestamp = td::Clocks::system(); + cur_stats_.rounds[round_idx].started_at = td::Clocks::system(); } auto it2 = blocks_.begin(); while (it2 != blocks_.end()) { @@ -1034,9 +1036,7 @@ void ValidatorSessionImpl::get_end_stats(td::Promise pro promise.set_error(td::Status::Error(ErrorCode::notready, "not started")); return; } - EndValidatorGroupStats stats; - stats.session_id = unique_hash_; - stats.timestamp = td::Clocks::system(); + EndValidatorGroupStats stats{.session_id = unique_hash_, .timestamp = td::Clocks::system()}; stats.nodes.resize(description().get_total_nodes()); for (size_t i = 0; i < stats.nodes.size(); ++i) { stats.nodes[i].id = description().get_source_id(i); @@ -1145,7 +1145,7 @@ void ValidatorSessionImpl::stats_init() { if (cur_stats_.rounds.empty()) { stats_add_round(); } - cur_stats_.rounds[0].timestamp = td::Clocks::system(); + cur_stats_.rounds[0].started_at = td::Clocks::system(); stats_inited_ = true; } @@ -1158,13 +1158,13 @@ void ValidatorSessionImpl::stats_add_round() { td::int32 priority = description().get_node_priority(i, round); if (priority >= 0) { CHECK((size_t)priority < stat.producers.size()); - stat.producers[priority].id = description().get_source_id(i); + stat.producers[priority].validator_id = description().get_source_id(i); stat.producers[priority].is_ours = (local_idx() == i); stat.producers[priority].approvers.resize(description().get_total_nodes(), false); stat.producers[priority].signers.resize(description().get_total_nodes(), false); } } - while (!stat.producers.empty() && stat.producers.back().id.is_zero()) { + while (!stat.producers.empty() && stat.producers.back().validator_id.is_zero()) { stat.producers.pop_back(); } } @@ -1179,7 +1179,7 @@ ValidatorSessionStats::Producer *ValidatorSessionImpl::stats_get_candidate_stat( } auto &stats_round = cur_stats_.rounds[round - cur_stats_.first_round]; auto it = std::find_if(stats_round.producers.begin(), stats_round.producers.end(), - [&](const ValidatorSessionStats::Producer &p) { return p.id == src; }); + [&](const ValidatorSessionStats::Producer &p) { return p.validator_id == src; }); if (it == stats_round.producers.end()) { return nullptr; } diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index b099d65ecf..641fb4866c 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -78,11 +78,6 @@ class ValidatorSession : public td::actor::Actor { bool is_cached_ = false; }; - struct GeneratedCandidate { - BlockCandidate candidate; - bool is_cached = false; - }; - class Callback { public: virtual void on_candidate(BlockSourceInfo source_info, ValidatorSessionRootHash root_hash, td::BufferSlice data, diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 690346f70a..2d996101ad 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -217,8 +217,7 @@ class ValidatorSessionImpl : public ValidatorSession { void candidate_approved_signed(td::uint32 round, ValidatorSessionCandidateId hash, td::uint32 ok_from, td::BufferSlice signature); - void generated_block(td::uint32 round, ValidatorSessionRootHash root_hash, td::BufferSlice data, - td::BufferSlice collated, double collation_time, bool collation_cached); + void generated_block(td::uint32 round, GeneratedCandidate c, double collation_time); void signed_block(td::uint32 round, ValidatorSessionCandidateId hash, td::BufferSlice signature); void end_request(td::uint32 round, ValidatorSessionCandidateId block_id) { diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index eb9aa626d7..8fabfcaeec 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -33,13 +33,16 @@ void CollationManager::start_up() { void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, - BlockCandidatePriority priority, - td::Ref validator_set, td::uint64 max_answer_size, - td::CancellationToken cancellation_token, td::Promise promise) { + BlockCandidatePriority priority, td::Ref validator_set, + td::uint64 max_answer_size, td::CancellationToken cancellation_token, + td::Promise promise) { if (shard.is_masterchain()) { - run_collate_query(shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), - opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), std::move(promise), - std::move(cancellation_token), 0); + run_collate_query( + shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), + opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), promise.wrap([](BlockCandidate&& candidate) { + return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; + }), + adnl::AdnlNodeIdShort::zero(), std::move(cancellation_token), 0); return; } collate_shard_block(shard, min_masterchain_block_id, std::move(prev), creator, priority, std::move(validator_set), @@ -48,10 +51,9 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, - BlockCandidatePriority priority, - td::Ref validator_set, td::uint64 max_answer_size, - td::CancellationToken cancellation_token, - td::Promise promise, td::Timestamp timeout) { + BlockCandidatePriority priority, td::Ref validator_set, + td::uint64 max_answer_size, td::CancellationToken cancellation_token, + td::Promise promise, td::Timestamp timeout) { TRY_STATUS_PROMISE(promise, cancellation_token.check()); ShardInfo* s = select_shard_info(shard); if (s == nullptr) { @@ -104,9 +106,12 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas } if (selected_collator.is_zero() && s->self_collate) { - run_collate_query(shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), - opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), std::move(promise), - std::move(cancellation_token), 0); + run_collate_query( + shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), + opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), promise.wrap([](BlockCandidate&& candidate) { + return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; + }), + adnl::AdnlNodeIdShort::zero(), std::move(cancellation_token), 0); return; } @@ -117,10 +122,13 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas next_block_id.seqno = std::max(next_block_id.seqno, p.seqno() + 1); } - promise = [=, SelfId = actor_id(this), promise = std::move(promise), - retry_at = td::Timestamp::in(0.5)](td::Result R) mutable { + td::Promise P = [=, SelfId = actor_id(this), promise = std::move(promise), + retry_at = td::Timestamp::in(0.5)](td::Result R) mutable { if (R.is_ok()) { - promise.set_value(R.move_as_ok()); + promise.set_value(GeneratedCandidate{.candidate = R.move_as_ok(), + .is_cached = false, + .self_collated = false, + .collator_node_id = selected_collator.bits256_value()}); return; } if (!selected_collator.is_zero()) { @@ -142,7 +150,7 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas }; if (selected_collator.is_zero()) { - promise.set_error(td::Status::Error(PSTRING() << "shard " << shard.to_str() << " has no alive collator node")); + P.set_error(td::Status::Error(PSTRING() << "shard " << shard.to_str() << " has no alive collator node")); return; } @@ -152,32 +160,32 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to #" << selected_idx << "(" << selected_collator << ")"; - td::Promise P = [=, SelfId = actor_id(this), promise = std::move(promise), - timer = td::Timer()](td::Result R) mutable { - TRY_RESULT_PROMISE_PREFIX(promise, data, std::move(R), "rldp query failed: "); + td::Promise P2 = [=, SelfId = actor_id(this), P = std::move(P), + timer = td::Timer()](td::Result R) mutable { + TRY_RESULT_PROMISE_PREFIX(P, data, std::move(R), "rldp query failed: "); auto r_error = fetch_tl_object(data, true); if (r_error.is_ok()) { auto error = r_error.move_as_ok(); - promise.set_error(td::Status::Error(error->code_, error->message_)); + P.set_error(td::Status::Error(error->code_, error->message_)); return; } - TRY_RESULT_PROMISE(promise, f, fetch_tl_object(data, true)); - TRY_RESULT_PROMISE(promise, candidate, + TRY_RESULT_PROMISE(P, f, fetch_tl_object(data, true)); + TRY_RESULT_PROMISE(P, candidate, CollatorNode::deserialize_candidate(std::move(f), td::narrow_cast(max_answer_size))); if (candidate.pubkey.as_bits256() != creator.as_bits256()) { - promise.set_error(td::Status::Error("collate query: block candidate source mismatch")); + P.set_error(td::Status::Error("collate query: block candidate source mismatch")); return; } if (candidate.id.id != next_block_id) { - promise.set_error(td::Status::Error("collate query: block id mismatch")); + P.set_error(td::Status::Error("collate query: block id mismatch")); return; } LOG(INFO) << "got collated block " << next_block_id.to_str() << " from #" << selected_idx << " (" << selected_collator << ") in " << timer.elapsed() << "s"; - promise.set_result(std::move(candidate)); + P.set_result(std::move(candidate)); }; - td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_id_, selected_collator, "collatequery", std::move(P), - timeout, std::move(query), max_answer_size); + td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_id_, selected_collator, "collatequery", + std::move(P2), timeout, std::move(query), max_answer_size); } void CollationManager::update_options(td::Ref opts) { diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp index 0ca4617d0a..72cc02e5a2 100644 --- a/validator/collation-manager.hpp +++ b/validator/collation-manager.hpp @@ -37,7 +37,7 @@ class CollationManager : public td::actor::Actor { void collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, - td::CancellationToken cancellation_token, td::Promise promise); + td::CancellationToken cancellation_token, td::Promise promise); void update_options(td::Ref opts); @@ -55,7 +55,7 @@ class CollationManager : public td::actor::Actor { void collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, - td::CancellationToken cancellation_token, td::Promise promise, + td::CancellationToken cancellation_token, td::Promise promise, td::Timestamp timeout); void update_collators_list(const CollatorsList& collators_list); diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 94bb18a1bd..cbab8b7950 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -391,12 +391,24 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data promise.set_result(serialize_tl_object(serialize_candidate(R.move_as_ok(), true), true)); } }; - new_promise = [new_promise = std::move(new_promise), creator, + new_promise = [new_promise = std::move(new_promise), creator, local_id = local_id_, manager = manager_](td::Result R) mutable { TRY_RESULT_PROMISE(new_promise, block, std::move(R)); + + CollatorNodeResponseStats stats; + stats.collator_node_id = local_id.bits256_value(); + stats.validator_id = creator.as_bits256(); + stats.original_block_id = block.id; + stats.collated_data_hash = block.collated_file_hash; + CatchainSeqno cc_seqno; td::uint32 val_set_hash; block = change_creator(std::move(block), creator, cc_seqno, val_set_hash); + + stats.block_id = block.id; + stats.timestamp = td::Clocks::system(); + td::actor::send_closure(manager, &ValidatorManager::log_collator_node_response_stats, std::move(stats)); + td::Promise P = new_promise.wrap([block = block.clone()](td::Unit&&) mutable -> BlockCandidate { return std::move(block); }); td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, block.id, std::move(block), cc_seqno, @@ -537,7 +549,7 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std }; td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); }, - cache_entry->cancellation_token_source.get_cancellation_token(), + local_id_, cache_entry->cancellation_token_source.get_cancellation_token(), CollateMode::skip_store_candidate | CollateMode::from_collator_node); } diff --git a/validator/fabric.h b/validator/fabric.h index 20358822e1..d2e9b47356 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -89,7 +89,8 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc Ed25519_PublicKey creator, td::Ref validator_set, td::Ref collator_opts, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, - td::CancellationToken cancellation_token, unsigned mode, int attempt_idx = 0); + adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode, + int attempt_idx = 0); void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index b207242fb1..3572109d90 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -79,6 +79,7 @@ class Collator final : public td::actor::Actor { td::Timestamp timeout; td::Timestamp queue_cleanup_timeout_, soft_timeout_, medium_timeout_; td::Promise main_promise; + adnl::AdnlNodeIdShort collator_node_id_ = adnl::AdnlNodeIdShort::zero(); unsigned mode_ = 0; int attempt_idx_; bool allow_repeat_collation_ = false; @@ -97,7 +98,8 @@ class Collator final : public td::actor::Actor { Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, Ref validator_set, Ed25519_PublicKey collator_id, Ref collator_opts, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, - td::CancellationToken cancellation_token, unsigned mode, int attempt_idx); + adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode, + int attempt_idx); ~Collator() override = default; bool is_busy() const { return busy_; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index dd8d551692..94d98ac598 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -77,6 +77,7 @@ static inline bool dbg(int c) { * @param manager The ActorId of the ValidatorManager. * @param timeout The timeout for the collator. * @param promise The promise to return the result. + * @param collator_node_id ADNL id of the collator node that generates the block (zero if it's not a collator node) * @param cancellation_token Token to cancel collation. * @param mode +1 - skip storing candidate to disk, +2 - called from CollatorNode. * @param attempt_idx The index of the attempt, starting from 0. On later attempts collator decreases block limits and skips some steps. @@ -84,8 +85,8 @@ static inline bool dbg(int c) { Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, td::Ref validator_set, Ed25519_PublicKey collator_id, Ref collator_opts, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise, td::CancellationToken cancellation_token, - unsigned mode, int attempt_idx) + td::Timestamp timeout, td::Promise promise, adnl::AdnlNodeIdShort collator_node_id, + td::CancellationToken cancellation_token, unsigned mode, int attempt_idx) : shard_(shard) , is_hardfork_(is_hardfork) , min_mc_block_id{min_masterchain_block_id} @@ -100,6 +101,7 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha , soft_timeout_(td::Timestamp::at(timeout.at() - 3.0)) , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) , main_promise(std::move(promise)) + , collator_node_id_(collator_node_id) , mode_(mode) , attempt_idx_(attempt_idx) , perf_timer_("collate", 0.1, @@ -374,8 +376,8 @@ bool Collator::fatal_error(td::Status error) { !is_hardfork_ && !timeout.is_in_past()) { LOG(WARNING) << "Repeating collation (attempt #" << attempt_idx_ + 1 << ")"; run_collate_query(shard_, min_mc_block_id, prev_blocks, created_by_, validator_set_, collator_opts_, manager, - td::Timestamp::in(10.0), std::move(main_promise), std::move(cancellation_token_), mode_, - attempt_idx_ + 1); + td::Timestamp::in(10.0), std::move(main_promise), collator_node_id_, + std::move(cancellation_token_), mode_, attempt_idx_ + 1); } else { LOG(INFO) << "collation failed in " << perf_timer_.elapsed() << " s " << error; LOG(INFO) << perf_log_; @@ -5927,8 +5929,15 @@ bool Collator::create_block_candidate() { double work_time = work_timer_.elapsed(); double cpu_work_time = cpu_work_timer_.elapsed(); LOG(WARNING) << "Collate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; + stats_.block_id = block_candidate->id; + stats_.collated_data_hash = block_candidate->collated_file_hash; + stats_.cc_seqno = validator_set_->get_catchain_seqno(); + stats_.collated_at = td::Clocks::system(); stats_.actual_bytes = block_candidate->data.size(); stats_.actual_collated_data_bytes = block_candidate->collated_data.size(); + stats_.attempt = attempt_idx_; + stats_.collator_node_id = collator_node_id_.bits256_value(); + stats_.validator_id = created_by_.as_bits256(); stats_.estimated_bytes = block_limit_status_->estimate_block_size(); stats_.gas = block_limit_status_->gas_used; stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; @@ -5938,20 +5947,12 @@ bool Collator::create_block_candidate() { stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); stats_.cat_collated_data_bytes = block_limit_status_->limits.classify_collated_data_size(stats_.estimated_collated_data_bytes); + stats_.total_time = perf_timer_.elapsed(); stats_.work_time = work_time; stats_.cpu_work_time = cpu_work_time; + stats_.time_stats = (PSTRING() << perf_log_); - // TODO: remove this later (currently needed to collect stats) - if (mode_ & CollateMode::from_collator_node) { - size_t d; - stats_.serialized_size = - validatorsession::compress_candidate_data(block_candidate->data, block_candidate->collated_data, d).ok().size(); - stats_.serialized_size_no_collated_data = - validatorsession::compress_candidate_data(block_candidate->data, td::Slice{}, d).ok().size(); - } - - td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, block_candidate->id, - std::move(stats_)); + td::actor::send_closure(manager, &ValidatorManager::log_collate_query_stats, std::move(stats_)); return true; } diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index db15b1edbc..d1555edc5a 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -216,7 +216,8 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc Ed25519_PublicKey creator, td::Ref validator_set, td::Ref collator_opts, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, - td::CancellationToken cancellation_token, unsigned mode, int attempt_idx) { + adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode, + int attempt_idx) { BlockSeqno seqno = 0; for (auto& p : prev) { if (p.seqno() > seqno) { @@ -227,7 +228,7 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc << (attempt_idx ? "_" + td::to_string(attempt_idx) : ""), shard, false, min_masterchain_block_id, std::move(prev), std::move(validator_set), creator, std::move(collator_opts), std::move(manager), timeout, std::move(promise), - std::move(cancellation_token), mode, attempt_idx) + collator_node_id, std::move(cancellation_token), mode, attempt_idx) .release(); } @@ -240,10 +241,10 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b seqno = p.seqno(); } } - td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, - min_masterchain_block_id, std::move(prev), td::Ref{}, - Ed25519_PublicKey{Bits256::zero()}, td::Ref{true}, - std::move(manager), timeout, std::move(promise), td::CancellationToken{}, 0, 0) + td::actor::create_actor( + PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, min_masterchain_block_id, + std::move(prev), td::Ref{}, Ed25519_PublicKey{Bits256::zero()}, td::Ref{true}, + std::move(manager), timeout, std::move(promise), adnl::AdnlNodeIdShort::zero(), td::CancellationToken{}, 0, 0) .release(); } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 34c164eee6..92b32b75c6 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -116,13 +116,12 @@ bool ValidateQuery::reject_query(std::string error, td::BufferSlice reason) { error = error_ctx() + error; LOG(ERROR) << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { - record_stats(); + record_stats(false, error); errorlog::ErrorLog::log(PSTRING() << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); errorlog::ErrorLog::log_file(block_candidate.data.clone()); errorlog::ErrorLog::log_file(block_candidate.collated_data.clone()); - LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s"; main_promise.set_result(CandidateReject{std::move(error), std::move(reason)}); } stop(); @@ -155,13 +154,12 @@ bool ValidateQuery::soft_reject_query(std::string error, td::BufferSlice reason) error = error_ctx() + error; LOG(ERROR) << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { - record_stats(); + record_stats(false, error); errorlog::ErrorLog::log(PSTRING() << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); errorlog::ErrorLog::log_file(block_candidate.data.clone()); errorlog::ErrorLog::log_file(block_candidate.collated_data.clone()); - LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s"; main_promise.set_result(CandidateReject{std::move(error), std::move(reason)}); } stop(); @@ -179,7 +177,7 @@ bool ValidateQuery::fatal_error(td::Status error) { error.ensure_error(); LOG(ERROR) << "aborting validation of block candidate for " << shard_.to_str() << " : " << error.to_string(); if (main_promise) { - record_stats(); + record_stats(false, error.message().str()); auto c = error.code(); if (c <= -667 && c >= -670) { errorlog::ErrorLog::log(PSTRING() << "FATAL ERROR: aborting validation of block candidate for " << shard_.to_str() @@ -188,7 +186,6 @@ bool ValidateQuery::fatal_error(td::Status error) { errorlog::ErrorLog::log_file(block_candidate.data.clone()); errorlog::ErrorLog::log_file(block_candidate.collated_data.clone()); } - LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s"; main_promise(std::move(error)); } stop(); @@ -238,9 +235,8 @@ bool ValidateQuery::fatal_error(std::string err_msg, int err_code) { */ void ValidateQuery::finish_query() { if (main_promise) { - record_stats(); + record_stats(true); LOG(WARNING) << "validate query done"; - LOG(WARNING) << "validation took " << perf_timer_.elapsed() << " s"; main_promise.set_result(now_); } stop(); @@ -7061,13 +7057,25 @@ void ValidateQuery::written_candidate() { /** * Sends validation work time to manager. */ -void ValidateQuery::record_stats() { - double work_time = work_timer_.elapsed(); - double cpu_work_time = cpu_work_timer_.elapsed(); +void ValidateQuery::record_stats(bool valid, std::string error_message) { + ValidationStats stats; + stats.block_id = id_; + stats.collated_data_hash = block_candidate.collated_file_hash; + stats.validated_at = td::Clocks::system(); + stats.valid = valid; + if (valid) { + stats.comment = (PSTRING() << "OK ts=" << now_); + } else { + stats.comment = std::move(error_message); + } + stats.actual_bytes = block_candidate.data.size(); + stats.actual_collated_data_bytes = block_candidate.collated_data.size(); + stats.total_time = perf_timer_.elapsed(); + stats.work_time = work_timer_.elapsed(); + stats.cpu_work_time = cpu_work_timer_.elapsed(); LOG(WARNING) << "validation took " << perf_timer_.elapsed() << "s"; - LOG(WARNING) << "Validate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; - td::actor::send_closure(manager, &ValidatorManager::record_validate_query_stats, block_candidate.id, work_time, - cpu_work_time); + LOG(WARNING) << "Validate query work time = " << stats.work_time << "s, cpu time = " << stats.cpu_work_time << "s"; + td::actor::send_closure(manager, &ValidatorManager::log_validate_query_stats, std::move(stats)); } } // namespace validator diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 585d553c5e..328a9e1ebc 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -407,7 +407,7 @@ class ValidateQuery : public td::actor::Actor { td::Timer work_timer_{true}; td::ThreadCpuTimer cpu_work_timer_{true}; - void record_stats(); + void record_stats(bool valid, std::string error_message = ""); }; } // namespace validator diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 14d10eccce..59f114b335 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -32,6 +32,8 @@ #include "auto/tl/lite_api.h" #include "impl/out-msg-queue-proof.hpp" +#include + namespace ton { namespace validator { @@ -54,7 +56,14 @@ struct AsyncSerializerState { }; struct CollationStats { + BlockIdExt block_id; + td::Bits256 collated_data_hash = td::Bits256::zero(); + CatchainSeqno cc_seqno = 0; + double collated_at = -1.0; td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0; + int attempt = 0; + td::Bits256 collator_node_id = td::Bits256::zero(); + td::Bits256 validator_id = td::Bits256::zero(); td::uint32 estimated_bytes = 0, gas = 0, lt_delta = 0, estimated_collated_data_bytes = 0; int cat_bytes = 0, cat_gas = 0, cat_lt_delta = 0, cat_collated_data_bytes = 0; std::string limits_log; @@ -62,15 +71,51 @@ struct CollationStats { td::uint32 ext_msgs_filtered = 0; td::uint32 ext_msgs_accepted = 0; td::uint32 ext_msgs_rejected = 0; - double work_time = 0.0, cpu_work_time = 0.0; - td::uint32 serialized_size = 0, serialized_size_no_collated_data = 0; - - tl_object_ptr tl() const { - return create_tl_object( - actual_bytes, actual_collated_data_bytes, estimated_bytes, gas, lt_delta, estimated_collated_data_bytes, - cat_bytes, cat_gas, cat_lt_delta, cat_collated_data_bytes, limits_log, ext_msgs_total, ext_msgs_filtered, - ext_msgs_accepted, ext_msgs_rejected, work_time, cpu_work_time, serialized_size, - serialized_size_no_collated_data); + double total_time = 0.0, work_time = 0.0, cpu_work_time = 0.0; + std::string time_stats; + + tl_object_ptr tl() const { + int flags = (collator_node_id.is_zero() ? 0 : ton_api::validatorStats_collatedBlock::Flags::COLLATOR_NODE_ID_MASK) | + (validator_id.is_zero() ? 0 : ton_api::validatorStats_collatedBlock::Flags::VALIDATOR_ID_MASK); + return create_tl_object( + flags, create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes, + actual_collated_data_bytes, attempt, collator_node_id, validator_id, total_time, work_time, cpu_work_time, + time_stats, + create_tl_object(estimated_bytes, gas, lt_delta, + estimated_collated_data_bytes, cat_bytes, cat_gas, + cat_lt_delta, cat_collated_data_bytes, limits_log), + create_tl_object(ext_msgs_total, ext_msgs_filtered, ext_msgs_accepted, + ext_msgs_rejected)); + } +}; + +struct ValidationStats { + BlockIdExt block_id; + td::Bits256 collated_data_hash = td::Bits256::zero(); + double validated_at = -1.0; + bool valid = false; + std::string comment; + td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0; + double total_time = 0.0, work_time = 0.0, cpu_work_time = 0.0; + + tl_object_ptr tl() const { + return create_tl_object( + create_tl_block_id(block_id), collated_data_hash, validated_at, valid, comment, actual_bytes, + actual_collated_data_bytes, total_time, work_time, cpu_work_time); + } +}; + +struct CollatorNodeResponseStats { + td::Bits256 collator_node_id = td::Bits256::zero(); + td::Bits256 validator_id = td::Bits256::zero(); + double timestamp = -1.0; + BlockIdExt block_id, original_block_id; + td::Bits256 collated_data_hash = td::Bits256::zero(); + + tl_object_ptr tl() const { + return create_tl_object( + collator_node_id, validator_id, timestamp, create_tl_block_id(block_id), create_tl_block_id(original_block_id), + collated_data_hash);; } }; @@ -198,9 +243,12 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) = 0; - virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0; - virtual void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) = 0; - virtual void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) = 0; + virtual void log_validator_session_stats(validatorsession::ValidatorSessionStats stats) { + } + virtual void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) { + } + virtual void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) { + } virtual void get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) = 0; virtual void get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) = 0; @@ -220,9 +268,11 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void add_lite_query_stats(int lite_query_id) { } - virtual void record_collate_query_stats(BlockIdExt block_id, CollationStats stats) { + virtual void log_collate_query_stats(CollationStats stats) { + } + virtual void log_validate_query_stats(ValidationStats stats) { } - virtual void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { + virtual void log_collator_node_response_stats(CollatorNodeResponseStats stats) { } virtual void add_persistent_state_description(td::Ref desc) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 542382520b..5ebe546782 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -129,7 +129,8 @@ void ValidatorManagerImpl::sync_complete(td::Promise promise) { Ed25519_PublicKey created_by{td::Bits256::zero()}; td::as(created_by.as_bits256().data() + 32 - 4) = ((unsigned)std::time(nullptr) >> 8); run_collate_query(shard_id, last_masterchain_block_id_, prev, created_by, val_set, td::Ref{true}, - actor_id(this), td::Timestamp::in(10.0), std::move(P), td::CancellationToken{}, 0); + actor_id(this), td::Timestamp::in(10.0), std::move(P), adnl::AdnlNodeIdShort::zero(), + td::CancellationToken{}, 0); } void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector prev, BlockIdExt last, diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 30bc7756b9..24e9e30a81 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -403,15 +403,6 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override { UNREACHABLE(); } - void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { - UNREACHABLE(); - } - void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { - UNREACHABLE(); - } - void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override { - UNREACHABLE(); - } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = td::actor::create_actor("queuesizecounter", td::Ref{}, diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 804ef0c732..3191adf0fd 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -468,15 +468,6 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override { UNREACHABLE(); } - void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { - UNREACHABLE(); - } - void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { - UNREACHABLE(); - } - void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override { - UNREACHABLE(); - } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = td::actor::create_actor("queuesizecounter", td::Ref{}, diff --git a/validator/manager.cpp b/validator/manager.cpp index 31278788cc..ff142c5707 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3107,61 +3107,20 @@ void ValidatorManagerImpl::wait_shard_client_state(BlockSeqno seqno, td::Timesta shard_client_waiters_[seqno].waiting_.emplace_back(timeout, 0, std::move(promise)); } -void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, - validatorsession::ValidatorSessionStats stats) { +void ValidatorManagerImpl::log_validator_session_stats(validatorsession::ValidatorSessionStats stats) { std::string fname = opts_->get_session_logs_file(); if (fname.empty()) { return; } - - std::vector> rounds; - for (const auto &round : stats.rounds) { - std::vector> producers; - for (const auto &producer : round.producers) { - BlockIdExt cur_block_id{block_id.id, producer.root_hash, producer.file_hash}; - auto it = recorded_block_stats_.find(cur_block_id); - tl_object_ptr collation_stats; - if (it != recorded_block_stats_.end() && it->second.collator_stats_) { - auto &stats = it->second.collator_stats_.value(); - collation_stats = stats.tl(); - } - std::string approvers, signers; - for (bool x : producer.approvers) { - approvers += (x ? '1' : '0'); - } - for (bool x : producer.signers) { - signers += (x ? '1' : '0'); - } - producers.push_back(create_tl_object( - producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.root_hash, - producer.file_hash, producer.comment, producer.block_timestamp, producer.is_accepted, producer.is_ours, - producer.got_submit_at, producer.collation_time, producer.collated_at, producer.collation_cached, - it == recorded_block_stats_.end() ? -1.0 : it->second.collator_work_time_, - it == recorded_block_stats_.end() ? -1.0 : it->second.collator_cpu_work_time_, std::move(collation_stats), - producer.validation_time, producer.validated_at, producer.validation_cached, - it == recorded_block_stats_.end() ? -1.0 : it->second.validator_work_time_, - it == recorded_block_stats_.end() ? -1.0 : it->second.validator_cpu_work_time_, producer.gen_utime, - producer.approved_weight, producer.approved_33pct_at, producer.approved_66pct_at, std::move(approvers), - producer.signed_weight, producer.signed_33pct_at, producer.signed_66pct_at, std::move(signers), - producer.serialize_time, producer.deserialize_time, producer.serialized_size)); - } - rounds.push_back(create_tl_object(round.timestamp, std::move(producers))); - } - - auto obj = create_tl_object( - stats.success, create_tl_block_id(block_id), stats.timestamp, stats.self.bits256_value(), stats.session_id, - stats.cc_seqno, stats.creator.bits256_value(), stats.total_validators, stats.total_weight, stats.signatures, - stats.signatures_weight, stats.approve_signatures, stats.approve_signatures_weight, stats.first_round, - std::move(rounds)); + stats.fix_block_ids(); + auto obj = stats.tl(); auto s = td::json_encode(td::ToJson(*obj.get()), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; file.open(fname, std::ios_base::app); file << s << "\n"; file.close(); - - LOG(INFO) << "Writing validator session stats for " << block_id.id.to_str(); + LOG(INFO) << "Writing validator session stats for " << stats.block_id.to_str(); } void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) { @@ -3169,22 +3128,13 @@ void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewVa if (fname.empty()) { return; } - std::vector> nodes; - for (const auto &node : stats.nodes) { - nodes.push_back( - create_tl_object(node.id.bits256_value(), node.weight)); - } - auto obj = create_tl_object( - stats.session_id, stats.shard.workchain, stats.shard.shard, stats.cc_seqno, stats.last_key_block_seqno, - stats.timestamp, stats.self_idx, std::move(nodes)); + auto obj = stats.tl(); auto s = td::json_encode(td::ToJson(*obj.get()), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; file.open(fname, std::ios_base::app); file << s << "\n"; file.close(); - LOG(INFO) << "Writing new validator group stats for " << stats.session_id << " shard=" << stats.shard.to_str() << " cc_seqno=" << stats.cc_seqno; } @@ -3194,21 +3144,13 @@ void ValidatorManagerImpl::log_end_validator_group_stats(validatorsession::EndVa if (fname.empty()) { return; } - std::vector> nodes; - for (const auto &node : stats.nodes) { - nodes.push_back(create_tl_object(node.id.bits256_value(), - node.catchain_blocks)); - } - auto obj = create_tl_object(stats.session_id, stats.timestamp, - std::move(nodes)); + auto obj = stats.tl(); auto s = td::json_encode(td::ToJson(*obj.get()), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; file.open(fname, std::ios_base::app); file << s << "\n"; file.close(); - LOG(INFO) << "Writing end validator group stats for " << stats.session_id; } @@ -3657,45 +3599,49 @@ td::actor::ActorOwn ValidatorManagerFactory::create( rldp, overlays); } -void ValidatorManagerImpl::record_collate_query_stats(BlockIdExt block_id, CollationStats stats) { - auto &record = new_block_stats_record(block_id); - record.collator_work_time_ = stats.work_time; - record.collator_cpu_work_time_ = stats.cpu_work_time; - record.collator_stats_ = std::move(stats); - +void ValidatorManagerImpl::log_collate_query_stats(CollationStats stats) { std::string fname = opts_->get_session_logs_file(); if (fname.empty()) { return; } - - auto obj = create_tl_object(td::Clocks::system(), - create_tl_block_id(block_id), stats.tl()); + auto obj = stats.tl(); auto s = td::json_encode(td::ToJson(*obj.get()), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; file.open(fname, std::ios_base::app); file << s << "\n"; file.close(); - - LOG(DEBUG) << "Writing collation stats stats for " << block_id.id.to_str(); + LOG(DEBUG) << "Writing collation stats stats for " << stats.block_id.to_str(); } -void ValidatorManagerImpl::record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { - auto &record = new_block_stats_record(block_id); - record.validator_work_time_ = work_time; - record.validator_cpu_work_time_ = cpu_work_time; +void ValidatorManagerImpl::log_validate_query_stats(ValidationStats stats) { + std::string fname = opts_->get_session_logs_file(); + if (fname.empty()) { + return; + } + auto obj = stats.tl(); + auto s = td::json_encode(td::ToJson(*obj.get()), false); + s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); + std::ofstream file; + file.open(fname, std::ios_base::app); + file << s << "\n"; + file.close(); + LOG(DEBUG) << "Writing validation stats stats for " << stats.block_id.to_str(); } -ValidatorManagerImpl::RecordedBlockStats &ValidatorManagerImpl::new_block_stats_record(BlockIdExt block_id) { - if (!recorded_block_stats_.count(block_id)) { - recorded_block_stats_lru_.push(block_id); - if (recorded_block_stats_lru_.size() > 4096) { - recorded_block_stats_.erase(recorded_block_stats_lru_.front()); - recorded_block_stats_lru_.pop(); - } +void ValidatorManagerImpl::log_collator_node_response_stats(CollatorNodeResponseStats stats) { + std::string fname = opts_->get_session_logs_file(); + if (fname.empty()) { + return; } - return recorded_block_stats_[block_id]; + auto obj = stats.tl(); + auto s = td::json_encode(td::ToJson(*obj.get()), false); + s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); + std::ofstream file; + file.open(fname, std::ios_base::app); + file << s << "\n"; + file.close(); + LOG(DEBUG) << "Writing collator node response stats stats for " << stats.block_id.to_str(); } size_t ValidatorManagerImpl::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 3353989fd8..febed73df4 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -630,7 +630,7 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override; - void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override; + void log_validator_session_stats(validatorsession::ValidatorSessionStats stats) override; void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override; void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override; @@ -775,19 +775,9 @@ class ValidatorManagerImpl : public ValidatorManager { td::actor::ActorOwn candidates_buffer_; - struct RecordedBlockStats { - double collator_work_time_ = -1.0; - double collator_cpu_work_time_ = -1.0; - td::optional collator_stats_; - double validator_work_time_ = -1.0; - double validator_cpu_work_time_ = -1.0; - }; - std::map recorded_block_stats_; - std::queue recorded_block_stats_lru_; - - void record_collate_query_stats(BlockIdExt block_id, CollationStats stats) override; - void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) override; - RecordedBlockStats &new_block_stats_record(BlockIdExt block_id); + void log_collate_query_stats(CollationStats stats) override; + void log_validate_query_stats(ValidationStats stats) override; + void log_collator_node_response_stats(CollatorNodeResponseStats stats) override; std::map> validator_telemetry_; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 97f34ea7cd..dad2988419 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -36,9 +36,8 @@ static bool need_send_candidate_broadcast(const validatorsession::BlockSourceInf !is_masterchain; } -void ValidatorGroup::generate_block_candidate( - validatorsession::BlockSourceInfo source_info, - td::Promise promise) { +void ValidatorGroup::generate_block_candidate(validatorsession::BlockSourceInfo source_info, + td::Promise promise) { td::uint32 round_id = source_info.priority.round; if (round_id > last_known_round_id_) { last_known_round_id_ = round_id; @@ -49,20 +48,21 @@ void ValidatorGroup::generate_block_candidate( } if (cached_collated_block_) { if (cached_collated_block_->result) { - promise.set_value({cached_collated_block_->result.value().clone(), true}); + auto res = cached_collated_block_->result.value().clone(); + res.is_cached = true; + promise.set_value(std::move(res)); } else { - cached_collated_block_->promises.push_back(promise.wrap([](BlockCandidate &&res) { - return validatorsession::ValidatorSession::GeneratedCandidate{std::move(res), true}; + cached_collated_block_->promises.push_back(promise.wrap([](GeneratedCandidate &&res) { + res.is_cached = true; + return std::move(res); })); } return; } cached_collated_block_ = std::make_shared(); - cached_collated_block_->promises.push_back(promise.wrap([](BlockCandidate &&res) { - return validatorsession::ValidatorSession::GeneratedCandidate{std::move(res), false}; - })); - td::Promise P = [SelfId = actor_id(this), cache = cached_collated_block_, - source_info](td::Result R) { + cached_collated_block_->promises.push_back(std::move(promise)); + td::Promise P = [SelfId = actor_id(this), cache = cached_collated_block_, + source_info](td::Result R) { td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, source_info, std::move(cache), std::move(R)); }; @@ -75,7 +75,7 @@ void ValidatorGroup::generate_block_candidate( void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo source_info, std::shared_ptr cache, - td::Result R) { + td::Result R) { if (R.is_error()) { for (auto &p : cache->promises) { p.set_error(R.error().clone()); @@ -84,12 +84,12 @@ void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo cached_collated_block_ = nullptr; } } else { - auto candidate = R.move_as_ok(); - add_available_block_candidate(candidate.pubkey.as_bits256(), candidate.id, candidate.collated_file_hash); + auto c = R.move_as_ok(); + add_available_block_candidate(c.candidate.pubkey.as_bits256(), c.candidate.id, c.candidate.collated_file_hash); if (need_send_candidate_broadcast(source_info, shard_.is_masterchain())) { - send_block_candidate_broadcast(candidate.id, candidate.data.clone()); + send_block_candidate_broadcast(c.candidate.id, c.candidate.data.clone()); } - cache->result = std::move(candidate); + cache->result = std::move(c); for (auto &p : cache->promises) { p.set_value(cache->result.value().clone()); } @@ -186,7 +186,8 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so } auto next_block_id = create_next_block_id(root_hash, file_hash); LOG(WARNING) << "Accepted block " << next_block_id; - td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, next_block_id, std::move(stats)); + stats.block_id = next_block_id; + td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, std::move(stats)); auto block = block_data.size() > 0 ? create_block(next_block_id, std::move(block_data)).move_as_ok() : td::Ref{}; @@ -293,7 +294,7 @@ std::unique_ptr ValidatorGroup::ma std::move(candidate), std::move(P)); } void on_generate_slot(validatorsession::BlockSourceInfo source_info, - td::Promise promise) override { + td::Promise promise) override { td::actor::send_closure(id_, &ValidatorGroup::generate_block_candidate, std::move(source_info), std::move(promise)); } @@ -397,8 +398,8 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch for (auto &p : postponed_accept_) { auto next_block_id = create_next_block_id(p.root_hash, p.file_hash); - td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, next_block_id, - std::move(p.stats)); + p.stats.block_id = next_block_id; + td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, std::move(p.stats)); auto block = p.block.size() > 0 ? create_block(next_block_id, std::move(p.block)).move_as_ok() : td::Ref{}; @@ -408,12 +409,11 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch } postponed_accept_.clear(); - validatorsession::NewValidatorGroupStats stats; - stats.session_id = session_id_; - stats.shard = shard_; - stats.cc_seqno = validator_set_->get_catchain_seqno(); - stats.last_key_block_seqno = last_key_block_seqno_; - stats.timestamp = td::Clocks::system(); + validatorsession::NewValidatorGroupStats stats{.session_id = session_id_, + .shard = shard_, + .cc_seqno = validator_set_->get_catchain_seqno(), + .last_key_block_seqno = last_key_block_seqno_, + .started_at = td::Clocks::system()}; td::uint32 idx = 0; for (const auto &node : validator_set_->export_vector()) { PublicKeyHash id = ValidatorFullId{node.key}.compute_short_id(); @@ -441,7 +441,8 @@ void ValidatorGroup::destroy() { return; } stats.cc_seqno = cc_seqno; - td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats, block_id, + stats.block_id = block_id; + td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats, std::move(stats)); }); td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_end_stats, diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index fb7a4dcfb8..675885fcc3 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -35,8 +35,7 @@ class ValidatorManager; class ValidatorGroup : public td::actor::Actor { public: - void generate_block_candidate(validatorsession::BlockSourceInfo source_info, - td::Promise promise); + void generate_block_candidate(validatorsession::BlockSourceInfo source_info, td::Promise promise); void validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block, td::Promise> promise); void accept_block_candidate(validatorsession::BlockSourceInfo source_info, td::BufferSlice block, RootHash root_hash, @@ -146,14 +145,14 @@ class ValidatorGroup : public td::actor::Actor { bool monitoring_shard_ = true; struct CachedCollatedBlock { - td::optional result; - std::vector> promises; + td::optional result; + std::vector> promises; }; std::shared_ptr cached_collated_block_; td::CancellationTokenSource cancellation_token_source_; void generated_block_candidate(validatorsession::BlockSourceInfo source_info, - std::shared_ptr cache, td::Result R); + std::shared_ptr cache, td::Result R); using CacheKey = std::tuple; std::map approved_candidates_cache_; @@ -165,8 +164,7 @@ class ValidatorGroup : public td::actor::Actor { } void get_validator_group_info_for_litequery_cont( - td::uint32 expected_round, - std::vector> candidates, + td::uint32 expected_round, std::vector> candidates, td::Promise> promise); std::set> available_block_candidates_; // source, id, collated hash From e6984e4799519cd6d1751e3aa06853a01333a560 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 27 Dec 2024 12:20:45 +0300 Subject: [PATCH 137/524] Add "self" to all session stats records --- tl/generate/scheme/ton_api.tl | 11 +++++---- tl/generate/scheme/ton_api.tlo | Bin 111624 -> 111644 bytes validator-session/validator-session-types.h | 7 ++++-- validator-session/validator-session.cpp | 2 +- validator/collator-node.cpp | 4 ++-- validator/fabric.h | 2 +- validator/impl/collator.cpp | 5 ++-- validator/impl/fabric.cpp | 8 +++---- validator/impl/validate-query.cpp | 10 ++++---- validator/impl/validate-query.hpp | 3 ++- validator/interfaces/validator-manager.h | 24 ++++++++++---------- validator/manager-disk.cpp | 2 +- validator/validator-group.cpp | 7 +++--- 13 files changed, 47 insertions(+), 38 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index bdcf0a296a..3ddb69490b 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -923,16 +923,17 @@ validatorStats.blockLimitsStatus validatorStats.extMsgsStats ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validatorStats.ExtMsgsStats; -validatorStats.collatedBlock flags:# +validatorStats.collatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 cc_seqno:int collated_at:double bytes:int collated_data_bytes:int attempt:int - collator_node_id:flags.0?int256 validator_id:flags.1?int256 + self:int256 is_validator:Bool total_time:double work_time:double cpu_work_time:double time_stats:string block_limits:validatorStats.blockLimitsStatus ext_msgs_stats:validatorStats.extMsgsStats = validatorSession.stats.CollatedBlock; validatorStats.validatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 validated_at:double + self:int256 valid:Bool comment:string bytes:int collated_data_bytes:int total_time:double work_time:double cpu_work_time:double = validatorStats.ValidatedBlock; @@ -940,13 +941,13 @@ validatorStats.validatedBlock validatorStats.newValidatorGroup.node id:int256 weight:long = validatorStats.newValidatorGroup.Node; validatorStats.newValidatorGroup session_id:int256 shard:tonNode.shardId cc_seqno:int last_key_block_seqno:int started_at:double - self_idx:int nodes:(vector validatorStats.newValidatorGroup.node) = validatorStats.NewValidatorGroup; + self_idx:int self:int256 nodes:(vector validatorStats.newValidatorGroup.node) = validatorStats.NewValidatorGroup; validatorStats.endValidatorGroup.node id:int256 catchain_blocks:int = validatorStats.endValidatorGroup.Node; -validatorStats.endValidatorGroup session_id:int256 timestamp:double +validatorStats.endValidatorGroup session_id:int256 timestamp:double self:int256 nodes:(vector validatorStats.endValidatorGroup.node) = validatorStats.EndValidatorGroup; -validatorStats.collatorNodeResponse collator_node_id:int256 validator_id:int256 timestamp:double +validatorStats.collatorNodeResponse self:int256 validator_id:int256 timestamp:double block_id:tonNode.blockIdExt original_block_id:tonNode.blockIdExt collated_data_hash:int256 = validatorStats.CollatorNodeResponse; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index f091213cf90e633cb8909ffc33ba05310ad4d1f0..9522bf9a607a2cf5097dbf4d58cfb0421d3a5d8f 100644 GIT binary patch delta 333 zcmeBp!8Ydw+lD*=u zxMgtj8oTXl>=@_pgLO>z@nclr1nIqbOvs0SdSeu0`1B8cj9aFE2xnB_<;g6LFH6kH zOi3)sFPhFcnNenXZ!}}}WQ8o&>6HPDspShy6;6DC{O6|ksm zcR1O+#%}u>JH|QuU>(za{1_D=tO? nodes; tl_object_ptr tl() const { @@ -232,7 +233,7 @@ struct NewValidatorGroupStats { } return create_tl_object(session_id, create_tl_shard_id(shard), cc_seqno, last_key_block_seqno, started_at, self_idx, - std::move(nodes_arr)); + self.bits256_value(), std::move(nodes_arr)); } }; @@ -244,6 +245,7 @@ struct EndValidatorGroupStats { ValidatorSessionId session_id = ValidatorSessionId::zero(); double timestamp = -1.0; + PublicKeyHash self = PublicKeyHash::zero(); std::vector nodes; tl_object_ptr tl() const { @@ -252,7 +254,8 @@ struct EndValidatorGroupStats { nodes_arr.push_back(create_tl_object(node.id.bits256_value(), node.catchain_blocks)); } - return create_tl_object(session_id, timestamp, std::move(nodes_arr)); + return create_tl_object(session_id, timestamp, self.bits256_value(), + std::move(nodes_arr)); } }; diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 5cc31a7ba2..e18dd1eb56 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -1036,7 +1036,7 @@ void ValidatorSessionImpl::get_end_stats(td::Promise pro promise.set_error(td::Status::Error(ErrorCode::notready, "not started")); return; } - EndValidatorGroupStats stats{.session_id = unique_hash_, .timestamp = td::Clocks::system()}; + EndValidatorGroupStats stats{.session_id = unique_hash_, .timestamp = td::Clocks::system(), .self = local_id()}; stats.nodes.resize(description().get_total_nodes()); for (size_t i = 0; i < stats.nodes.size(); ++i) { stats.nodes[i].id = description().get_source_id(i); diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index cbab8b7950..ff84456bd5 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -396,8 +396,8 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data TRY_RESULT_PROMISE(new_promise, block, std::move(R)); CollatorNodeResponseStats stats; - stats.collator_node_id = local_id.bits256_value(); - stats.validator_id = creator.as_bits256(); + stats.self = local_id.pubkey_hash(); + stats.validator_id = PublicKey(pubkeys::Ed25519(creator)).compute_short_id(); stats.original_block_id = block.id; stats.collated_data_hash = block.collated_file_hash; diff --git a/validator/fabric.h b/validator/fabric.h index d2e9b47356..86868f766a 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -82,7 +82,7 @@ void run_check_proof_query(BlockIdExt id, td::Ref proof, td::actor::Actor void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - BlockCandidate candidate, td::Ref validator_set, + BlockCandidate candidate, td::Ref validator_set, PublicKeyHash local_validator_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, unsigned mode = 0); void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 94d98ac598..bc8e0ef539 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -5936,8 +5936,9 @@ bool Collator::create_block_candidate() { stats_.actual_bytes = block_candidate->data.size(); stats_.actual_collated_data_bytes = block_candidate->collated_data.size(); stats_.attempt = attempt_idx_; - stats_.collator_node_id = collator_node_id_.bits256_value(); - stats_.validator_id = created_by_.as_bits256(); + stats_.is_validator = !(mode_ & CollateMode::from_collator_node); + stats_.self = stats_.is_validator ? PublicKey(pubkeys::Ed25519(created_by_)).compute_short_id() + : collator_node_id_.pubkey_hash(); stats_.estimated_bytes = block_limit_status_->estimate_block_size(); stats_.gas = block_limit_status_->gas_used; stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index d1555edc5a..ae341a5cbd 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -192,8 +192,8 @@ void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::act .release(); } -void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, - std::vector prev, BlockCandidate candidate, td::Ref validator_set, +void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, + BlockCandidate candidate, td::Ref validator_set, PublicKeyHash local_validator_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, unsigned mode) { BlockSeqno seqno = 0; @@ -207,8 +207,8 @@ void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, td::actor::create_actor(PSTRING() << (is_fake ? "fakevalidate" : "validateblock") << shard.to_str() << ":" << (seqno + 1) << "#" << idx.fetch_add(1), shard, min_masterchain_block_id, std::move(prev), std::move(candidate), - std::move(validator_set), std::move(manager), timeout, std::move(promise), - mode) + std::move(validator_set), local_validator_id, std::move(manager), timeout, + std::move(promise), mode) .release(); } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 92b32b75c6..b38d2f37d6 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -67,16 +67,17 @@ std::string ErrorCtx::as_string() const { * @param promise The Promise to return the ValidateCandidateResult to. * @param mode +1 - fake mode */ -ValidateQuery::ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, - std::vector prev, BlockCandidate candidate, Ref validator_set, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, unsigned mode) +ValidateQuery::ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, + BlockCandidate candidate, Ref validator_set, + PublicKeyHash local_validator_id, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise, unsigned mode) : shard_(shard) , id_(candidate.id) , min_mc_block_id(min_masterchain_block_id) , prev_blocks(std::move(prev)) , block_candidate(std::move(candidate)) , validator_set_(std::move(validator_set)) + , local_validator_id_(local_validator_id) , manager(manager) , timeout(timeout) , main_promise(std::move(promise)) @@ -7062,6 +7063,7 @@ void ValidateQuery::record_stats(bool valid, std::string error_message) { stats.block_id = id_; stats.collated_data_hash = block_candidate.collated_file_hash; stats.validated_at = td::Clocks::system(); + stats.self = local_validator_id_; stats.valid = valid; if (valid) { stats.comment = (PSTRING() << "OK ts=" << now_); diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 328a9e1ebc..05a397c859 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -118,7 +118,7 @@ class ValidateQuery : public td::actor::Actor { public: ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - BlockCandidate candidate, td::Ref validator_set, + BlockCandidate candidate, td::Ref validator_set, PublicKeyHash local_validator_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, unsigned mode = 0); @@ -132,6 +132,7 @@ class ValidateQuery : public td::actor::Actor { std::vector> prev_states; BlockCandidate block_candidate; td::Ref validator_set_; + PublicKeyHash local_validator_id_ = PublicKeyHash::zero(); td::actor::ActorId manager; td::Timestamp timeout; td::Promise main_promise; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 59f114b335..6c31fe22f1 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -62,8 +62,8 @@ struct CollationStats { double collated_at = -1.0; td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0; int attempt = 0; - td::Bits256 collator_node_id = td::Bits256::zero(); - td::Bits256 validator_id = td::Bits256::zero(); + PublicKeyHash self = PublicKeyHash::zero(); + bool is_validator = false; td::uint32 estimated_bytes = 0, gas = 0, lt_delta = 0, estimated_collated_data_bytes = 0; int cat_bytes = 0, cat_gas = 0, cat_lt_delta = 0, cat_collated_data_bytes = 0; std::string limits_log; @@ -75,11 +75,9 @@ struct CollationStats { std::string time_stats; tl_object_ptr tl() const { - int flags = (collator_node_id.is_zero() ? 0 : ton_api::validatorStats_collatedBlock::Flags::COLLATOR_NODE_ID_MASK) | - (validator_id.is_zero() ? 0 : ton_api::validatorStats_collatedBlock::Flags::VALIDATOR_ID_MASK); return create_tl_object( - flags, create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes, - actual_collated_data_bytes, attempt, collator_node_id, validator_id, total_time, work_time, cpu_work_time, + create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes, + actual_collated_data_bytes, attempt, self.bits256_value(), is_validator, total_time, work_time, cpu_work_time, time_stats, create_tl_object(estimated_bytes, gas, lt_delta, estimated_collated_data_bytes, cat_bytes, cat_gas, @@ -93,6 +91,7 @@ struct ValidationStats { BlockIdExt block_id; td::Bits256 collated_data_hash = td::Bits256::zero(); double validated_at = -1.0; + PublicKeyHash self = PublicKeyHash::zero(); bool valid = false; std::string comment; td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0; @@ -100,22 +99,23 @@ struct ValidationStats { tl_object_ptr tl() const { return create_tl_object( - create_tl_block_id(block_id), collated_data_hash, validated_at, valid, comment, actual_bytes, - actual_collated_data_bytes, total_time, work_time, cpu_work_time); + create_tl_block_id(block_id), collated_data_hash, validated_at, self.bits256_value(), valid, comment, + actual_bytes, actual_collated_data_bytes, total_time, work_time, cpu_work_time); } }; struct CollatorNodeResponseStats { - td::Bits256 collator_node_id = td::Bits256::zero(); - td::Bits256 validator_id = td::Bits256::zero(); + PublicKeyHash self = PublicKeyHash::zero(); + PublicKeyHash validator_id = PublicKeyHash::zero(); double timestamp = -1.0; BlockIdExt block_id, original_block_id; td::Bits256 collated_data_hash = td::Bits256::zero(); tl_object_ptr tl() const { return create_tl_object( - collator_node_id, validator_id, timestamp, create_tl_block_id(block_id), create_tl_block_id(original_block_id), - collated_data_hash);; + self.bits256_value(), validator_id.bits256_value(), timestamp, create_tl_block_id(block_id), + create_tl_block_id(original_block_id), collated_data_hash); + ; } }; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 5ebe546782..4bbd48540b 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -153,7 +153,7 @@ void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector prev, BlockIdExt min_masterch .shard = shard_, .cc_seqno = validator_set_->get_catchain_seqno(), .last_key_block_seqno = last_key_block_seqno_, - .started_at = td::Clocks::system()}; + .started_at = td::Clocks::system(), + .self = local_id_}; td::uint32 idx = 0; for (const auto &node : validator_set_->export_vector()) { PublicKeyHash id = ValidatorFullId{node.key}.compute_short_id(); From ca1093d9bad844acf6a29280292c8f64629a452c Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 27 Dec 2024 13:00:00 +0300 Subject: [PATCH 138/524] Improve handling sessino stats file --- validator/manager.cpp | 128 ++++++++++++++++++------------------------ validator/manager.hpp | 8 +++ 2 files changed, 63 insertions(+), 73 deletions(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index ff142c5707..545f0ec736 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1866,6 +1866,7 @@ void ValidatorManagerImpl::start_up() { } validator_manager_init(opts_, actor_id(this), db_.get(), std::move(P)); + init_session_stats(); check_waiters_at_ = td::Timestamp::in(1.0); alarm_timestamp().relax(check_waiters_at_); @@ -3108,50 +3109,16 @@ void ValidatorManagerImpl::wait_shard_client_state(BlockSeqno seqno, td::Timesta } void ValidatorManagerImpl::log_validator_session_stats(validatorsession::ValidatorSessionStats stats) { - std::string fname = opts_->get_session_logs_file(); - if (fname.empty()) { - return; - } stats.fix_block_ids(); - auto obj = stats.tl(); - auto s = td::json_encode(td::ToJson(*obj.get()), false); - s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; - file.open(fname, std::ios_base::app); - file << s << "\n"; - file.close(); - LOG(INFO) << "Writing validator session stats for " << stats.block_id.to_str(); + write_session_stats(stats); } void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) { - std::string fname = opts_->get_session_logs_file(); - if (fname.empty()) { - return; - } - auto obj = stats.tl(); - auto s = td::json_encode(td::ToJson(*obj.get()), false); - s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; - file.open(fname, std::ios_base::app); - file << s << "\n"; - file.close(); - LOG(INFO) << "Writing new validator group stats for " << stats.session_id << " shard=" << stats.shard.to_str() - << " cc_seqno=" << stats.cc_seqno; + write_session_stats(stats); } void ValidatorManagerImpl::log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) { - std::string fname = opts_->get_session_logs_file(); - if (fname.empty()) { - return; - } - auto obj = stats.tl(); - auto s = td::json_encode(td::ToJson(*obj.get()), false); - s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; - file.open(fname, std::ios_base::app); - file << s << "\n"; - file.close(); - LOG(INFO) << "Writing end validator group stats for " << stats.session_id; + write_session_stats(stats); } void ValidatorManagerImpl::get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) { @@ -3443,6 +3410,7 @@ void ValidatorManagerImpl::update_options(td::Ref opts) td::actor::send_closure(c, &CollationManager::update_options, opts); } opts_ = std::move(opts); + init_session_stats(); } void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) { @@ -3600,48 +3568,15 @@ td::actor::ActorOwn ValidatorManagerFactory::create( } void ValidatorManagerImpl::log_collate_query_stats(CollationStats stats) { - std::string fname = opts_->get_session_logs_file(); - if (fname.empty()) { - return; - } - auto obj = stats.tl(); - auto s = td::json_encode(td::ToJson(*obj.get()), false); - s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; - file.open(fname, std::ios_base::app); - file << s << "\n"; - file.close(); - LOG(DEBUG) << "Writing collation stats stats for " << stats.block_id.to_str(); + write_session_stats(stats); } void ValidatorManagerImpl::log_validate_query_stats(ValidationStats stats) { - std::string fname = opts_->get_session_logs_file(); - if (fname.empty()) { - return; - } - auto obj = stats.tl(); - auto s = td::json_encode(td::ToJson(*obj.get()), false); - s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; - file.open(fname, std::ios_base::app); - file << s << "\n"; - file.close(); - LOG(DEBUG) << "Writing validation stats stats for " << stats.block_id.to_str(); + write_session_stats(stats); } void ValidatorManagerImpl::log_collator_node_response_stats(CollatorNodeResponseStats stats) { - std::string fname = opts_->get_session_logs_file(); - if (fname.empty()) { - return; - } - auto obj = stats.tl(); - auto s = td::json_encode(td::ToJson(*obj.get()), false); - s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; - file.open(fname, std::ios_base::app); - file << s << "\n"; - file.close(); - LOG(DEBUG) << "Writing collator node response stats stats for " << stats.block_id.to_str(); + write_session_stats(stats); } size_t ValidatorManagerImpl::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) { @@ -3702,6 +3637,53 @@ void ValidatorManagerImpl::init_validator_telemetry() { } } +void ValidatorManagerImpl::init_session_stats() { + if (opts_->get_session_logs_file() == session_stats_filename_) { + return; + } + session_stats_filename_ = opts_->get_session_logs_file(); + if (session_stats_filename_.empty()) { + session_stats_enabled_ = false; + session_stats_fd_.close(); + return; + } + auto r_fd = td::FileFd::open(session_stats_filename_, + td::FileFd::Flags::Write | td::FileFd::Flags::Append | td::FileFd::Create); + if (r_fd.is_error()) { + LOG(ERROR) << "Failed to open session stats file for writing: " << r_fd.move_as_error(); + session_stats_filename_.clear(); + session_stats_enabled_ = false; + return; + } + session_stats_fd_ = r_fd.move_as_ok(); + session_stats_enabled_ = true; +} + +template +void ValidatorManagerImpl::write_session_stats(const T &obj) { + if (!session_stats_enabled_) { + return; + } + auto s = td::json_encode(td::ToJson(*obj.tl()), false); + s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); + s += '\n'; + td::Slice slice{s}; + while (!slice.empty()) { + auto R = session_stats_fd_.write(slice); + if (R.is_error()) { + LOG(WARNING) << "Failed to write to session stats: " << R.move_as_error(); + } + if (R.ok() == 0) { + LOG(WARNING) << "Failed to write to session stats"; + } + slice.remove_prefix(R.ok()); + } + auto S = session_stats_fd_.sync(); + if (S.is_error()) { + LOG(WARNING) << "Failed to write to session stats: " << S; + } +} + } // namespace validator } // namespace ton diff --git a/validator/manager.hpp b/validator/manager.hpp index febed73df4..d8c30fb5b9 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -793,6 +793,14 @@ class ValidatorManagerImpl : public ValidatorManager { std::map> persistent_state_descriptions_; std::map> persistent_state_blocks_; + + bool session_stats_enabled_ = false; + std::string session_stats_filename_; + td::FileFd session_stats_fd_; + + void init_session_stats(); + template + void write_session_stats(const T &obj); }; } // namespace validator From 4ec34edc0c8e963412c62da67ef47d7c958dcc2d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 27 Dec 2024 13:01:00 +0300 Subject: [PATCH 139/524] Fix telementry collection --- validator/full-node-fast-sync-overlays.cpp | 4 +++- validator/full-node-private-overlay.cpp | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index 6554c51801..90bec5933b 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -208,7 +208,9 @@ void FullNodeFastSyncOverlay::send_block_candidate(BlockIdExt block_id, Catchain } void FullNodeFastSyncOverlay::send_validator_telemetry(tl_object_ptr telemetry) { - process_telemetry_broadcast(local_id_, telemetry); + if (collect_telemetry_) { + process_telemetry_broadcast(local_id_, telemetry); + } auto data = serialize_tl_object(telemetry, true); if (data.size() <= overlay::Overlays::max_simple_broadcast_size()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index 1acfbd4ed1..67e5f451a3 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -185,7 +185,9 @@ void FullNodePrivateBlockOverlay::send_broadcast(BlockBroadcast broadcast) { } void FullNodePrivateBlockOverlay::send_validator_telemetry(tl_object_ptr telemetry) { - process_telemetry_broadcast(local_id_.pubkey_hash(), telemetry); + if (collect_telemetry_) { + process_telemetry_broadcast(local_id_.pubkey_hash(), telemetry); + } auto data = serialize_tl_object(telemetry, true); if (data.size() <= overlay::Overlays::max_simple_broadcast_size()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, From 5ce9d0bcdbeab6be5105c698583bab4901c911ab Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 27 Dec 2024 15:23:31 +0300 Subject: [PATCH 140/524] Add more information to validatorStats.newValidatorGroup.node --- tl/generate/scheme/ton_api.tl | 2 +- tl/generate/scheme/ton_api.tlo | Bin 111644 -> 111708 bytes validator-session/validator-session-types.h | 6 ++++-- validator/validator-group.cpp | 10 +++++++--- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 3ddb69490b..bae1077c60 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -938,7 +938,7 @@ validatorStats.validatedBlock bytes:int collated_data_bytes:int total_time:double work_time:double cpu_work_time:double = validatorStats.ValidatedBlock; -validatorStats.newValidatorGroup.node id:int256 weight:long = validatorStats.newValidatorGroup.Node; +validatorStats.newValidatorGroup.node id:int256 pubkey:PublicKey adnl_id:int256 weight:long = validatorStats.newValidatorGroup.Node; validatorStats.newValidatorGroup session_id:int256 shard:tonNode.shardId cc_seqno:int last_key_block_seqno:int started_at:double self_idx:int self:int256 nodes:(vector validatorStats.newValidatorGroup.node) = validatorStats.NewValidatorGroup; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 9522bf9a607a2cf5097dbf4d58cfb0421d3a5d8f..98305606b167e16c79843b3c684fe4c366a0bfbe 100644 GIT binary patch delta 87 zcmbR9g6+-=whbEgEG*eFdzy9Zx9iw5DoB8N+XKQGGg!gQ>3d=rH8?>^ZXOfzVVUf> gRC#hph|FYx7p&83H!;dj7l>t4foQUbWqc9_00yNT?*IS* delta 69 zcmccff^E(VwhbEgEc>!G-J5mnx9iw5DoB8N+XKQGGg!gQ>3d=rHNdRN8&+sc4~S({ MfhehnWqc9_0BZ3VtpET3 diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index a2a3ebc1fb..e613b08a5c 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -213,6 +213,8 @@ struct ValidatorSessionStats { struct NewValidatorGroupStats { struct Node { PublicKeyHash id = PublicKeyHash::zero(); + PublicKey pubkey; + adnl::AdnlNodeIdShort adnl_id = adnl::AdnlNodeIdShort::zero(); ValidatorWeight weight = 0; }; @@ -228,8 +230,8 @@ struct NewValidatorGroupStats { tl_object_ptr tl() const { std::vector> nodes_arr; for (const auto &node : nodes) { - nodes_arr.push_back( - create_tl_object(node.id.bits256_value(), node.weight)); + nodes_arr.push_back(create_tl_object( + node.id.bits256_value(), node.pubkey.tl(), node.adnl_id.bits256_value(), node.weight)); } return create_tl_object(session_id, create_tl_shard_id(shard), cc_seqno, last_key_block_seqno, started_at, self_idx, diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index efa2a428c4..d3819f07de 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -155,8 +155,8 @@ void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo return; } VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id; - run_validate_query(shard_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, - local_id_, manager_, td::Timestamp::in(15.0), std::move(P)); + run_validate_query(shard_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, local_id_, + manager_, td::Timestamp::in(15.0), std::move(P)); } void ValidatorGroup::update_approve_cache(CacheKey key, UnixTime value) { @@ -421,7 +421,11 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch if (id == local_id_) { stats.self_idx = idx; } - stats.nodes.push_back(validatorsession::NewValidatorGroupStats::Node{id, node.weight}); + stats.nodes.push_back(validatorsession::NewValidatorGroupStats::Node{ + .id = id, + .pubkey = PublicKey(pubkeys::Ed25519(node.key)), + .adnl_id = (node.addr.is_zero() ? adnl::AdnlNodeIdShort{id} : adnl::AdnlNodeIdShort{node.addr}), + .weight = node.weight}); ++idx; } td::actor::send_closure(manager_, &ValidatorManager::log_new_validator_group_stats, std::move(stats)); From e8e7883329ab97883ee03cb7bb919213f4951524 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 7 Jan 2025 10:54:09 +0300 Subject: [PATCH 141/524] Add more required cells to the proof in collated data --- validator/impl/collator-impl.h | 2 +- validator/impl/collator.cpp | 33 ++++++++++++++++++++++++++++++- validator/impl/validate-query.cpp | 6 +++--- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 3572109d90..9d73746b70 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -224,7 +224,7 @@ class Collator final : public td::actor::Actor { std::unique_ptr block_candidate; - std::unique_ptr dispatch_queue_; + std::unique_ptr dispatch_queue_, old_dispatch_queue_; std::map sender_generated_messages_count_; unsigned dispatch_queue_ops_{0}; std::map last_dispatch_queue_emitted_lt_; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index bc8e0ef539..e9e286c96a 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1215,6 +1215,7 @@ bool Collator::import_shard_state_data(block::ShardState& ss) { processed_upto_ = std::move(ss.processed_upto_); ihr_pending = std::move(ss.ihr_pending_); dispatch_queue_ = std::move(ss.dispatch_queue_); + old_dispatch_queue_ = std::make_unique(*dispatch_queue_); block_create_stats_ = std::move(ss.block_create_stats_); if (ss.out_msg_queue_size_) { have_out_msg_queue_size_in_state_ = true; @@ -5708,6 +5709,11 @@ Ref Collator::collate_shard_block_descr_set() { return cell; } +/** + * Visits certain cells in out msg queue and dispatch queue to add them to the proof + * + * @returns True on success, Falise if error occurred + */ bool Collator::prepare_msg_queue_proof() { auto res = old_out_msg_queue_->scan_diff( *out_msg_queue_, @@ -5732,7 +5738,32 @@ bool Collator::prepare_msg_queue_proof() { } return true; }, - 3); + 2); + if (!res) { + return false; + } + res = old_dispatch_queue_->scan_diff( + *dispatch_queue_, + [this](td::ConstBitPtr, int, Ref old_value, Ref new_value) { + if (old_value.not_null()) { + old_value = old_dispatch_queue_->extract_value(std::move(old_value)); + vm::Dictionary dispatch_dict{64}; + td::uint64 dispatch_dict_size; + CHECK(block::unpack_account_dispatch_queue(old_value, dispatch_dict, dispatch_dict_size)); + td::BitArray<64> max_lt; + CHECK(dispatch_dict.get_minmax_key(max_lt, true).not_null()); + } + if (new_value.not_null()) { + new_value = dispatch_queue_->extract_value(std::move(new_value)); + vm::Dictionary dispatch_dict{64}; + td::uint64 dispatch_dict_size; + CHECK(block::unpack_account_dispatch_queue(new_value, dispatch_dict, dispatch_dict_size)); + td::BitArray<64> min_lt; + CHECK(dispatch_dict.get_minmax_key(min_lt, false).not_null()); + } + return true; + }, + 2); return res; } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index b38d2f37d6..2d1e95a5ba 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -2923,7 +2923,7 @@ bool ValidateQuery::precheck_account_updates() { CHECK(key_len == 256); return precheck_one_account_update(key, std::move(old_val_extra), std::move(new_val_extra)); }, - 3 /* check augmentation of changed nodes */)) { + 2 /* check augmentation of changed nodes in the new dict */)) { return reject_query("invalid ShardAccounts dictionary in the new state"); } } catch (vm::VmError& err) { @@ -3372,7 +3372,7 @@ bool ValidateQuery::precheck_message_queue_update() { CHECK(key_len == 352); return precheck_one_message_queue_update(key, std::move(old_val_extra), std::move(new_val_extra)); }, - 3 /* check augmentation of changed nodes */)) { + 2 /* check augmentation of changed nodes in the new dict */)) { return reject_query("invalid OutMsgQueue dictionary in the new state"); } } catch (vm::VmError& err) { @@ -3533,7 +3533,7 @@ bool ValidateQuery::unpack_dispatch_queue_update() { return check_account_dispatch_queue_update(key, ps_.dispatch_queue_->extract_value(std::move(old_val_extra)), ns_.dispatch_queue_->extract_value(std::move(new_val_extra))); }, - 3 /* check augmentation of changed nodes */); + 2 /* check augmentation of changed nodes in the new dict */); if (!res) { return reject_query("invalid DispatchQueue dictionary in the new state"); } From 1ee3e5d6a72d688f2f48f4c08e9f230137ce1bbd Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 7 Jan 2025 12:56:29 +0300 Subject: [PATCH 142/524] Fix self_collate --- validator/collation-manager.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index 8fabfcaeec..2b06ad482a 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -284,7 +284,10 @@ void CollationManager::update_collators_list(const CollatorsList& collators_list auto old_collators = std::move(collators_); collators_.clear(); for (const auto& shard : collators_list.shards) { - shards_.push_back({.shard_id = shard.shard_id, .select_mode = shard.select_mode, .collators = shard.collators}); + shards_.push_back({.shard_id = shard.shard_id, + .select_mode = shard.select_mode, + .collators = shard.collators, + .self_collate = shard.self_collate}); for (auto id : shard.collators) { auto it = old_collators.find(id); if (it == old_collators.end()) { From 0763691f7099d54a552969a50364dce9c4cddf08 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 27 Jan 2025 11:49:18 +0300 Subject: [PATCH 143/524] Simplify loading virtual states from collated data, fix message queue cleanup --- validator/fabric.h | 2 +- validator/impl/collator.cpp | 23 +++++--- validator/impl/fabric.cpp | 2 +- validator/impl/validate-query.cpp | 88 +++++++++---------------------- validator/impl/validate-query.hpp | 3 +- 5 files changed, 44 insertions(+), 74 deletions(-) diff --git a/validator/fabric.h b/validator/fabric.h index 86868f766a..19319f36e2 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -40,7 +40,7 @@ td::Result> create_proof(BlockIdExt masterchain_block_id, td::Buf td::Result> create_proof_link(BlockIdExt block_id, td::BufferSlice proof); td::Result> create_signature_set(td::BufferSlice sig_set); td::Result> create_shard_state(BlockIdExt block_id, td::BufferSlice data); -td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell); +td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell); td::Result create_block_handle(td::BufferSlice data); td::Result create_block_handle(td::Slice data); td::Result create_temp_block_handle(td::BufferSlice data); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index e9e286c96a..cf4c1fad83 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -2450,8 +2450,7 @@ bool Collator::out_msg_queue_cleanup() { auto pure_out_msg_queue = std::make_unique(r_cell.move_as_ok().data_cell, 352, block::tlb::aug_OutMsgQueue); td::uint32 deleted = 0; - bool fail = false; - pure_out_msg_queue->check_for_each([&](Ref value, td::ConstBitPtr key, int n) -> bool { + bool ok = pure_out_msg_queue->check_for_each([&](Ref value, td::ConstBitPtr key, int n) -> bool { vm::CellSlice& cs = value.write(); assert(n == 352); block::EnqueuedMsgDescr enq_msg_descr; @@ -2461,7 +2460,6 @@ bool Collator::out_msg_queue_cleanup() { && enq_msg_descr.check_key(key) // check key && enq_msg_descr.lt_ == created_lt)) { LOG(ERROR) << "cannot unpack EnqueuedMsg with key " << key.to_hex(n); - fail = true; return false; } LOG(DEBUG) << "scanning outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," @@ -2495,7 +2493,6 @@ bool Collator::out_msg_queue_cleanup() { if (!dequeue_message(std::move(enq_msg_descr.msg_env_), deliver_lt)) { fatal_error(PSTRING() << "cannot dequeue outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," << enq_msg_descr.hash_.to_hex() << ") by inserting a msg_export_deq record"); - fail = true; return false; } register_out_msg_queue_op(); @@ -2504,11 +2501,11 @@ bool Collator::out_msg_queue_cleanup() { block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); } } - return !delivered; + return true; }); LOG(WARNING) << "deleted " << deleted << " messages from out_msg_queue after merge, remaining queue size is " << out_msg_queue_size_; - if (fail) { + if (!ok) { return fatal_error("error scanning/updating OutMsgQueue"); } } else { @@ -5801,7 +5798,19 @@ bool Collator::create_collated_data() { if (state_proof.is_null()) { return fatal_error("cannot generate Merkle proof for previous state"); } - proofs[prev_state_root_->get_hash().bits()] = std::move(state_proof); + if (after_merge_) { + bool special; + auto cs = vm::load_cell_slice_special(state_proof, special); + CHECK(cs.special_type() == vm::CellTraits::SpecialType::MerkleProof); + cs = vm::load_cell_slice(cs.prefetch_ref(0)); + CHECK(cs.size_refs() == 2); + CHECK(cs.size() == 32); + CHECK(cs.prefetch_ulong(32) == 0x5f327da5U); + proofs[cs.prefetch_ref(0)->get_hash(0).bits()] = vm::CellBuilder::create_merkle_proof(cs.prefetch_ref(0)); + proofs[cs.prefetch_ref(1)->get_hash(0).bits()] = vm::CellBuilder::create_merkle_proof(cs.prefetch_ref(1)); + } else { + proofs[prev_state_root_->get_hash().bits()] = std::move(state_proof); + } } // 4. Proofs for message queues for (vm::MerkleProofBuilder &mpb : neighbor_proof_builders_) { diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index ae341a5cbd..2f6ddf1ebf 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -84,7 +84,7 @@ td::Result> create_shard_state(BlockIdExt block_id, td::Buff } } -td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell) { +td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell) { auto res = ShardStateQ::fetch(block_id, {}, std::move(root_cell)); if (res.is_error()) { return res.move_as_error(); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 3b3d766c4c..18cc55d0e2 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1243,10 +1243,21 @@ bool ValidateQuery::check_this_shard_mc_info() { * @returns True if the previous state is computed successfully, false otherwise. */ bool ValidateQuery::compute_prev_state() { + CHECK(prev_states.size() == 1u + after_merge_); + CHECK(prev_states.size() == prev_blocks.size()); if (!is_masterchain() && full_collated_data_) { - return compute_prev_state_from_collated_data(); + for (size_t i = 0; i < prev_states.size(); i++) { + Ref root = get_virt_state_root(prev_blocks[i]); + if (root.is_null()) { + return reject_query(PSTRING() << "cannot get previous state from collated data: " << prev_blocks[i].to_str()); + } + auto r_state = create_shard_state(prev_blocks[i], std::move(root)); + if (r_state.is_error()) { + return reject_query("failed to parse previous state from collated data", r_state.move_as_error()); + } + prev_states[i] = r_state.move_as_ok(); + } } - CHECK(prev_states.size() == 1u + after_merge_); // Extend validator timeout if previous block is too old UnixTime prev_ts = prev_states[0]->get_unix_time(); if (after_merge_) { @@ -1274,46 +1285,6 @@ bool ValidateQuery::compute_prev_state() { return true; } -bool ValidateQuery::compute_prev_state_from_collated_data() { - td::Bits256 state_hash; - if (id_.seqno() == 1) { - if (prev_blocks.size() != 1) { - return reject_query("seqno is 1, but number of previous blocks is not 1"); - } - state_hash = prev_blocks[0].root_hash; - } else { - std::vector> prev_state_roots(prev_blocks.size()); - for (size_t i = 0; i < prev_blocks.size(); ++i) { - prev_state_roots[i] = get_virt_state_root(prev_blocks[i].root_hash); - if (prev_state_roots[i].is_null()) { - return reject_query(PSTRING() << "cannot get hash of previous state root: " << prev_blocks[i]); - } - } - - if (prev_state_roots.size() == 1) { - state_hash = prev_state_roots[0]->get_hash().bits(); - } else { - CHECK(prev_state_roots.size() == 2); - Ref merged; - if (!block::gen::t_ShardState.cell_pack_split_state(merged, prev_state_roots[0], prev_state_roots[1])) { - return fatal_error(-667, "cannot construct mechanically merged previously state"); - } - state_hash = merged->get_hash().bits(); - } - } - if (state_hash != prev_state_hash_) { - return reject_query("previous state hash mismatch for block "s + id_.to_str() + " : block header declares " + - prev_state_hash_.to_hex() + " , actual " + state_hash.to_hex()); - } - auto it = virt_roots_.find(state_hash); - if (it == virt_roots_.end()) { - return reject_query(PSTRING() << "no state root for previous block in collated data (hash = " - << state_hash.to_hex() << ")"); - } - prev_state_root_ = it->second; - return true; -} - /** * Computes the next shard state using the previous state and the block's Merkle update. */ @@ -1574,21 +1545,11 @@ bool ValidateQuery::request_neighbor_queues() { ++i; continue; } - td::Bits256 state_root_hash; - if (descr.blk_.seqno() == 0) { - state_root_hash = descr.blk_.root_hash; - } else { - Ref state_root = get_virt_state_root(descr.blk_.root_hash); - if (state_root.is_null()) { - return reject_query(PSTRING() << "cannot get hash of state root: " << descr.blk_); - } - state_root_hash = state_root->get_hash().bits(); - } - auto it = virt_roots_.find(state_root_hash); - if (it == virt_roots_.end()) { - return reject_query(PSTRING() << "cannot get state root form collated data: " << descr.blk_); + auto state_root = get_virt_state_root(descr.blk_); + if (state_root.is_null()) { + return reject_query(PSTRING() << "cannot get state root form collated data: " << descr.blk_.to_str()); } - auto state = ShardStateQ::fetch(descr.blk_, {}, it->second); + auto state = ShardStateQ::fetch(descr.blk_, {}, std::move(state_root)); if (state.is_error()) { return reject_query("cannot fetch shard state from collated data", state.move_as_error()); } @@ -6898,22 +6859,23 @@ bool ValidateQuery::postcheck_value_flow() { return true; } -Ref ValidateQuery::get_virt_state_root(td::Bits256 block_root_hash) { - auto it = virt_roots_.find(block_root_hash); +Ref ValidateQuery::get_virt_state_root(const BlockIdExt& block_id) { + auto it = virt_roots_.find(block_id.root_hash); if (it == virt_roots_.end()) { return {}; } Ref root = it->second; + if (block_id.seqno() == 0) { + return root; + } block::gen::Block::Record block; if (!tlb::unpack_cell(root, block)) { return {}; } vm::CellSlice upd_cs{vm::NoVmSpec(), block.state_update}; - if (!(upd_cs.is_special() && upd_cs.prefetch_long(8) == 4 // merkle update - && upd_cs.size_ext() == 0x20228)) { - return {}; - } - return vm::MerkleProof::virtualize_raw(upd_cs.prefetch_ref(1), {0, 1}); + td::Bits256 state_root_hash = upd_cs.prefetch_ref(1)->get_hash(0).bits(); + it = virt_roots_.find(state_root_hash); + return it == virt_roots_.end() ? Ref{} : it->second; } /** diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 289319d139..32d8e9f461 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -306,7 +306,6 @@ class ValidateQuery : public td::actor::Actor { bool extract_collated_data(); bool try_validate(); bool compute_prev_state(); - bool compute_prev_state_from_collated_data(); bool compute_next_state(); bool unpack_merge_prev_state(); bool unpack_prev_state(); @@ -397,7 +396,7 @@ class ValidateQuery : public td::actor::Actor { const block::CurrencyCollection& create); bool check_mc_block_extra(); - Ref get_virt_state_root(td::Bits256 block_root_hash); + Ref get_virt_state_root(const BlockIdExt& block_id); bool check_timeout() { if (timeout && timeout.is_in_past()) { From c863c42ed18be62d43f0055b712d5e9a5c2451ac Mon Sep 17 00:00:00 2001 From: birydrad <> Date: Wed, 11 Dec 2024 14:48:48 +0300 Subject: [PATCH 144/524] celldb: version 2 - thread safe cache - parallel commit - multiple optimizations - support of key-value merge operations - improved tests and benchmarks - in-memory version won't read from key value after start - uses vector in-memory table now - use rocksdb::WALRecoveryMode::kTolerateCorruptedTailRecords - do not silently ignore errors during recovery --- CMakeLists.txt | 1 + crypto/CMakeLists.txt | 3 +- crypto/test/test-db.cpp | 949 ++++++++++++--- crypto/vm/cells/Cell.h | 1 + crypto/vm/cells/DataCell.cpp | 5 +- crypto/vm/cells/DataCell.h | 8 +- crypto/vm/cells/ExtCell.h | 20 + crypto/vm/cells/PrunnedCell.h | 4 + crypto/vm/cells/UsageCell.h | 3 + crypto/vm/cells/VirtualCell.h | 3 + crypto/vm/db/CellHashTable.h | 13 +- crypto/vm/db/CellStorage.cpp | 85 ++ crypto/vm/db/CellStorage.h | 23 + crypto/vm/db/DynamicBagOfCellsDb.cpp | 79 +- crypto/vm/db/DynamicBagOfCellsDb.h | 71 +- crypto/vm/db/DynamicBagOfCellsDbV2.cpp | 1505 ++++++++++++++++++++++++ crypto/vm/db/InMemoryBagOfCellsDb.cpp | 173 ++- crypto/vm/db/StaticBagOfCellsDb.cpp | 18 +- tddb/td/db/KeyValue.h | 53 +- tddb/td/db/MemoryKeyValue.cpp | 82 +- tddb/td/db/MemoryKeyValue.h | 36 +- tddb/td/db/RocksDb.cpp | 123 +- tddb/td/db/RocksDb.h | 26 +- tdutils/td/utils/MpmcQueue.h | 4 +- tdutils/td/utils/Status.h | 7 + tdutils/td/utils/ThreadSafeCounter.h | 60 +- validator-engine/validator-engine.cpp | 12 +- validator-engine/validator-engine.hpp | 4 + validator/db/celldb.cpp | 202 +++- validator/db/celldb.hpp | 8 +- validator/validator-options.hpp | 7 + validator/validator.h | 2 + 32 files changed, 3274 insertions(+), 316 deletions(-) create mode 100644 crypto/vm/db/DynamicBagOfCellsDbV2.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cea3fc7ec4..dfbbd60ace 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -156,6 +156,7 @@ if (TON_USE_ROCKSDB) set(WITH_GFLAGS OFF CACHE BOOL "build with GFlags") set(WITH_TESTS OFF CACHE BOOL "build with tests") set(WITH_TOOLS OFF CACHE BOOL "build with tools") + set(USE_RTTI ON CACHE BOOL "use rtti") set(FAIL_ON_WARNINGS OFF CACHE BOOL "fail on warnings") message("Add rocksdb") add_subdirectory(third-party/rocksdb EXCLUDE_FROM_ALL) diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 069083381c..530dc7ba54 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -144,6 +144,7 @@ set(TON_CRYPTO_SOURCE set(TON_DB_SOURCE vm/db/DynamicBagOfCellsDb.cpp + vm/db/DynamicBagOfCellsDbV2.cpp vm/db/CellStorage.cpp vm/db/TonDb.cpp @@ -541,7 +542,7 @@ target_include_directories(create-state PUBLIC $ #include #include @@ -59,84 +54,87 @@ #include #include "openssl/digest.hpp" +#include "storage/db.h" +#include "td/utils/VectorQueue.h" #include "vm/dict.h" -#include #include #include #include -#include +#include -namespace vm { -class ThreadExecutor : public DynamicBagOfCellsDb::AsyncExecutor { +#include +#include +#include + +#include "td/actor/actor.h" +#include "td/utils/overloaded.h" + + +class ActorExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { public: - explicit ThreadExecutor(size_t threads_n) { - for (size_t i = 0; i < threads_n; ++i) { - threads_.emplace_back([this]() { - while (true) { - auto task = pop_task(); - if (!task) { - break; - } - CHECK(generation_.load() % 2 == 1); - task(); - } - }); - } + ActorExecutor(size_t tn) : tn_(tn) { + scheduler_.run_in_context([&] { worker_ = td::actor::create_actor("Worker"); }); + thread_ = td::thread([this]() { scheduler_.run(); }); } - - ~ThreadExecutor() override { - for (size_t i = 0; i < threads_.size(); ++i) { - push_task({}); + ~ActorExecutor() { + scheduler_.run_in_context_external([&] { send_closure(worker_, &Worker::close); }); + thread_.join(); + } + std::string describe() const override { + return PSTRING() << "ActorExecutor(tn=" << tn_ << ")"; + } + class Worker : public td::actor::Actor { + public: + void close() { + td::actor::core::SchedulerContext::get()->stop(); + stop(); } - for (auto &t : threads_) { - t.join(); + void execute_sync(std::function f) { + f(); } - } + }; void execute_async(std::function f) override { - push_task(std::move(f)); + class Runner : public td::actor::Actor { + public: + explicit Runner(std::function f) : f_(std::move(f)) { + } + void start_up() override { + f_(); + stop(); + } + + private: + std::function f_; + }; + auto context = td::actor::SchedulerContext::get(); + if (context) { + td::actor::create_actor("executeasync", std::move(f)).release(); + } else { + scheduler_.run_in_context_external( + [&] { td::actor::create_actor("executeasync", std::move(f)).release(); }); + } } void execute_sync(std::function f) override { - auto x = generation_.load(); - std::scoped_lock lock(sync_mutex_); - CHECK(x == generation_); - CHECK(generation_.load() % 2 == 1); - f(); - CHECK(generation_.load() % 2 == 1); - } - void inc_generation() { - generation_.fetch_add(1); + auto context = td::actor::SchedulerContext::get(); + if (context) { + td::actor::send_closure(worker_, &Worker::execute_sync, std::move(f)); + } else { + scheduler_.run_in_context_external( + [&] { td::actor::send_closure(worker_, &Worker::execute_sync, std::move(f)); }); + } } private: - std::atomic generation_{0}; - std::queue, size_t>> queue_; - std::mutex queue_mutex_; - std::condition_variable cv_; - std::mutex sync_mutex_; - std::vector threads_; - - std::function pop_task() { - std::unique_lock lock(queue_mutex_); - cv_.wait(lock, [&] { return !queue_.empty(); }); - CHECK(!queue_.empty()); - auto task = std::move(queue_.front()); - queue_.pop(); - CHECK(task.second == generation_); - return task.first; - } - - void push_task(std::function task) { - { - std::scoped_lock lock(queue_mutex_); - queue_.emplace(std::move(task), generation_.load()); - } - cv_.notify_one(); - } + size_t tn_; + td::actor::Scheduler scheduler_{{tn_}, false, td::actor::Scheduler::Paused}; + td::actor::ActorOwn worker_; + td::thread thread_; }; +namespace vm { std::vector do_get_serialization_modes() { std::vector res; for (int i = 0; i < 32; i++) { @@ -324,6 +322,34 @@ TEST(Cell, sha_benchmark_threaded) { bench_threaded([n]() { return BenchSha256(n); }); } } +class BenchTasks : public td::Benchmark { + public: + explicit BenchTasks(size_t tn) : tn_(tn) { + } + + std::string get_description() const override { + return PSTRING() << "bench_tasks(threads_n=" << tn_ << ")"; + } + + void run(int n) override { + ActorExecutor executor(tn_); + for (int i = 0; i < n; i++) { + std::latch latch(tn_); + for (size_t j = 0; j < tn_; j++) { + executor.execute_async([&]() { latch.count_down(); }); + } + latch.wait(); + } + } + + private: + size_t tn_{}; +}; +TEST(Bench, Tasks) { + for (size_t tn : {1, 4, 16}) { + bench(BenchTasks(tn)); + } +} std::string serialize_boc(Ref cell, int mode = 31) { CHECK(cell.not_null()); @@ -437,6 +463,8 @@ class CellExplorer { cs_ = {}; break; } + default: + UNREACHABLE(); } } @@ -474,6 +502,8 @@ class CellExplorer { case op.ReadCellSlice: log_ << "read slice " << op.children_mask << "\n"; break; + default: + UNREACHABLE(); } } void log_cell(const Ref &cell) { @@ -627,7 +657,9 @@ TEST(Cell, MerkleProof) { auto exploration2 = CellExplorer::explore(usage_cell, exploration.ops); ASSERT_EQ(exploration.log, exploration2.log); - auto is_prunned = [&](const Ref &cell) { return exploration.visited.count(cell->get_hash()) == 0; }; + auto is_prunned = [&](const Ref &cell_to_check) { + return exploration.visited.count(cell_to_check->get_hash()) == 0; + }; auto proof = MerkleProof::generate(cell, is_prunned); // CellBuilder::virtualize(proof, 1); //ASSERT_EQ(1u, proof->get_level()); @@ -706,7 +738,7 @@ TEST(Cell, MerkleProofCombine) { check(proof_union_fast); } { - auto cell = MerkleProof::virtualize(proof12, 1); + cell = MerkleProof::virtualize(proof12, 1); auto usage_tree = std::make_shared(); auto usage_cell = UsageCell::create(cell, usage_tree->root_ptr()); @@ -927,7 +959,6 @@ TEST(TonDb, InMemoryDynamicBocSimple) { auto before = counter(); SCOPE_EXIT { LOG_CHECK(before == counter()) << before << " vs " << counter(); - ; }; td::Random::Xorshift128plus rnd{123}; auto kv = std::make_shared(); @@ -963,26 +994,193 @@ TEST(TonDb, InMemoryDynamicBocSimple) { int VERBOSITY_NAME(boc) = VERBOSITY_NAME(DEBUG) + 10; +struct CellMerger : td::Merger { + void merge_value_and_update(std::string &value, td::Slice update) override { + return CellStorer::merge_value_and_refcnt_diff(value, update); + } + void merge_update_and_update(std::string &left_update, td::Slice right_update) override { + LOG(ERROR) << "update_and_update"; + UNREACHABLE(); + return CellStorer::merge_refcnt_diffs(left_update, right_update); + } +}; +struct CompactionFilterEraseEmptyValues : public rocksdb::CompactionFilter { + bool Filter(int level, const rocksdb::Slice & /*key*/, const rocksdb::Slice &existing_value, std::string *new_value, + bool *value_changed) const override { + return existing_value.empty(); + } + bool FilterMergeOperand(int, const rocksdb::Slice & /*key*/, const rocksdb::Slice &operand) const override { + return operand.empty(); + } + + // Name of the compaction filter + const char *Name() const override { + return "CompactionFilterEraseEmptyValues"; + } +}; +auto to_td(rocksdb::Slice value) -> td::Slice { + return td::Slice(value.data(), value.size()); +} + +struct MergeOperatorAddCellRefcnt : public rocksdb::MergeOperator { + const char *Name() const override { + return "MergeOperatorAddCellRefcnt"; + } + bool FullMergeV2(const MergeOperationInput &merge_in, MergeOperationOutput *merge_out) const override { + CHECK(merge_in.existing_value); + auto &value = *merge_in.existing_value; + CHECK(merge_in.operand_list.size() >= 1); + td::Slice diff; + std::string diff_buf; + if (merge_in.operand_list.size() == 1) { + diff = to_td(merge_in.operand_list[0]); + } else { + diff_buf = merge_in.operand_list[0].ToString(); + for (size_t i = 1; i < merge_in.operand_list.size(); ++i) { + CellStorer::merge_refcnt_diffs(diff_buf, to_td(merge_in.operand_list[i])); + } + diff = diff_buf; + } + + merge_out->new_value = value.ToString(); + CellStorer::merge_value_and_refcnt_diff(merge_out->new_value, diff); + return true; + } + bool PartialMerge(const rocksdb::Slice & /*key*/, const rocksdb::Slice &left, const rocksdb::Slice &right, + std::string *new_value, rocksdb::Logger *logger) const override { + *new_value = left.ToString(); + CellStorer::merge_refcnt_diffs(*new_value, to_td(right)); + return true; + } +}; + +struct DB { + std::unique_ptr dboc; + std::shared_ptr kv; + void reset_loader() { + dboc->set_loader(std::make_unique(kv->snapshot())); + } +}; struct BocOptions { - std::shared_ptr async_executor; - std::optional o_in_memory; + using AsyncExecutor = DynamicBagOfCellsDb::AsyncExecutor; + + using CreateInMemoryOptions = DynamicBagOfCellsDb::CreateInMemoryOptions; + using CreateV1Options = DynamicBagOfCellsDb::CreateV1Options; + using CreateV2Options = DynamicBagOfCellsDb::CreateV2Options; + + std::shared_ptr async_executor; + struct KvOptions { + enum KvType { InMemory, RocksDb } kv_type{InMemory}; + bool experimental{false}; + bool no_transactions{false}; + size_t cache_size{0}; + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const KvOptions &kv_options) { + if (kv_options.kv_type == KvType::InMemory) { + return sb << "InMemory{}"; + } + return sb << "RockDb{cache_size=" << kv_options.cache_size << ", no_transactions=" << kv_options.no_transactions + << ", experimental=" << kv_options.experimental << "}"; + } + }; + KvOptions kv_options; + std::variant options; td::uint64 seed{123}; - auto create_dboc(td::KeyValueReader *kv, std::optional o_root_n) { - if (o_in_memory) { - auto res = DynamicBagOfCellsDb::create_in_memory(kv, *o_in_memory); - auto stats = res->get_stats().move_as_ok(); - if (o_root_n) { - ASSERT_EQ(*o_root_n, stats.roots_total_count); + std::shared_ptr create_kv(std::shared_ptr old_key_value, bool no_reads = false) { + if (kv_options.kv_type == KvOptions::InMemory) { + if (old_key_value) { + return old_key_value; } - VLOG(boc) << "reset roots_n=" << stats.roots_total_count << " cells_n=" << stats.cells_total_count; - return res; + return std::make_shared(std::make_shared()); + } else if (kv_options.kv_type == KvOptions::RocksDb) { + auto merge_operator = std::make_shared(); + static const CompactionFilterEraseEmptyValues compaction_filter; + CHECK(!old_key_value || old_key_value.use_count() == 1); + std::string db_path = "test_celldb"; + if (old_key_value) { + //LOG(ERROR) << "Reload rocksdb"; + old_key_value.reset(); + } else { + //LOG(ERROR) << "New rocksdb"; + td::RocksDb::destroy(db_path).ensure(); + } + auto db_options = td::RocksDbOptions{ + .block_cache = {}, + .merge_operator = merge_operator, + .compaction_filter = &compaction_filter, + .experimental = kv_options.experimental, + .no_reads = no_reads, + .no_transactions = kv_options.no_transactions, + .use_direct_reads = true, + .no_block_cache = true, + }; + if (kv_options.cache_size != 0) { + db_options.no_block_cache = false; + db_options.block_cache = rocksdb::NewLRUCache(kv_options.cache_size); + } + return std::make_shared(td::RocksDb::open(db_path, std::move(db_options)).move_as_ok()); + } else { + UNREACHABLE(); + } + } + void check_kv_is_empty(KeyValue &kv) { + if (kv_options.kv_type == KvOptions::InMemory) { + ASSERT_EQ(0u, kv.count("").move_as_ok()); + return; } - return DynamicBagOfCellsDb::create(); + + size_t non_empty_values = 0; + kv.for_each([&](auto key, auto value) { + non_empty_values += !value.empty(); + return td::Status::OK(); + }); + if (non_empty_values != 0) { + kv.for_each([&](auto key, auto value) { + LOG(ERROR) << "Key: " << td::hex_encode(key) << " Value: " << td::hex_encode(value); + std::string x; + LOG(ERROR) << int(kv.get(key, x).move_as_ok()); + return td::Status::OK(); + }); + } + ASSERT_EQ(0u, non_empty_values); + } + + [[nodiscard]] auto create_db(DB db, std::optional o_root_n) { + auto old_boc = std::move(db.dboc); + auto old_kv = std::move(db.kv); + old_boc.reset(); + using ResT = DB; + return std::visit(td::overloaded( + [&](CreateV1Options &) -> ResT { + auto new_kv = create_kv(std::move(old_kv)); + auto res = DynamicBagOfCellsDb::create(); + res->set_loader(std::make_unique(new_kv->snapshot())); + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }, + [&](CreateV2Options &options) -> ResT { + auto new_kv = create_kv(std::move(old_kv)); + auto res = DynamicBagOfCellsDb::create_v2(options); + res->set_loader(std::make_unique(new_kv->snapshot())); + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }, + [&](CreateInMemoryOptions &options) -> ResT { + auto read_kv = create_kv(std::move(old_kv), false); + auto res = DynamicBagOfCellsDb::create_in_memory(read_kv.get(), options); + auto new_kv = create_kv(std::move(read_kv), true); + res->set_loader(std::make_unique(new_kv->snapshot())); + auto stats = res->get_stats().move_as_ok(); + if (o_root_n) { + ASSERT_EQ(*o_root_n, stats.roots_total_count); + } + VLOG(boc) << "reset roots_n=" << stats.roots_total_count + << " cells_n=" << stats.cells_total_count; + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }), + options); }; void prepare_commit(DynamicBagOfCellsDb &dboc) { + td::PerfWarningTimer warning_timer("test_db_prepare_commit"); if (async_executor) { - async_executor->inc_generation(); std::latch latch(1); td::Result res; async_executor->execute_sync([&] { @@ -993,70 +1191,172 @@ struct BocOptions { }); latch.wait(); async_executor->execute_sync([&] {}); - async_executor->inc_generation(); + res.ensure(); } else { dboc.prepare_commit(); } } + enum CacheAction { ResetCache, KeepCache }; + void write_commit(DynamicBagOfCellsDb &dboc, std::shared_ptr kv, CacheAction action) { + td::PerfWarningTimer warning_timer("test_db_write_commit"); + kv->begin_write_batch().ensure(); + CellStorer cell_storer(*kv); + { + td::PerfWarningTimer timer("test_db_commit"); + dboc.commit(cell_storer).ensure(); + } + { + td::PerfWarningTimer timer("test_db_commit_write_batch"); + kv->commit_write_batch().ensure(); + } + switch (action) { + case ResetCache: { + td::PerfWarningTimer timer("test_db_reset_cache"); + dboc.set_loader(std::make_unique(kv->snapshot())); + break; + } + case KeepCache: + break; + } + } + + void commit(DB &db, CacheAction action = ResetCache) { + prepare_commit(*db.dboc); + write_commit(*db.dboc, db.kv, action); + } + + std::string description() const { + td::StringBuilder sb; + + sb << "DBOC(type="; + std::visit(td::overloaded([&](const CreateV1Options &) { sb << "V1"; }, + [&](const CreateV2Options &options) { + sb << "V2(concurrency=" << options.extra_threads + 1; + if (options.executor) { + sb << ", executor=" << options.executor->describe(); + } else { + sb << ", executor=threads"; + } + sb << ")"; + }, + [&](const CreateInMemoryOptions &options) { + sb << "InMemory(use_arena=" << options.use_arena + << ", less_memory=" << options.use_less_memory_during_creation << ")"; + }), + options); + sb << kv_options; + if (async_executor) { + sb << ", executor=" << async_executor->describe(); + } + sb << ")"; + + return sb.as_cslice().str(); + } }; template -void with_all_boc_options(F &&f, size_t tests_n = 500) { +void with_all_boc_options(F &&f, size_t tests_n, bool single_thread = false) { LOG(INFO) << "Test dynamic boc"; auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; + std::map>> benches; auto run = [&](BocOptions options) { - LOG(INFO) << "\t" << (options.o_in_memory ? "in memory" : "on disk") << (options.async_executor ? " async" : ""); - if (options.o_in_memory) { - LOG(INFO) << "\t\tuse_arena=" << options.o_in_memory->use_arena - << " less_memory=" << options.o_in_memory->use_less_memory_during_creation; - } + auto description = options.description(); + LOG(INFO) << "Running " << description; + auto start = td::Timestamp::now(); + DynamicBagOfCellsDb::Stats stats; + auto o_in_memory = std::get_if(&options.options); for (td::uint32 i = 0; i < tests_n; i++) { auto before = counter(); + options.seed = i == 0 ? 123 : i; - f(options); + auto stats_diff = f(options); + stats.apply_diff(stats_diff); + auto after = counter(); - LOG_CHECK((options.o_in_memory && options.o_in_memory->use_arena) || before == after) - << before << " vs " << after; + LOG_CHECK((o_in_memory && o_in_memory->use_arena) || before == after) << before << " vs " << after; + } + LOG(INFO) << "\ttook " << td::Timestamp::now().at() - start.at() << " seconds"; + LOG(INFO) << stats; + for (auto &[key, value] : stats.named_stats.stats_int) { + if (td::begins_with(key, "bench_")) { + benches[key].emplace_back(value, description); + } } }; - run({.async_executor = std::make_shared(4)}); - run({}); - for (auto use_arena : {false, true}) { - for (auto less_memory : {false, true}) { - run({.o_in_memory = - DynamicBagOfCellsDb::CreateInMemoryOptions{.extra_threads = std::thread::hardware_concurrency(), - .verbose = false, - .use_arena = use_arena, - .use_less_memory_during_creation = less_memory}}); + + // NB: use .experimental to play with different RocksDb parameters + // Note, that new benchmark are necessary to fully understand the effect of different RocksDb options + std::vector kv_options_list = { + // BocOptions::KvOptions{.kv_type = BocOptions::KvOptions::InMemory}, + // BocOptions::KvOptions{.kv_type = BocOptions::KvOptions::RocksDb, .experimental = false, .cache_size = 0}, + BocOptions::KvOptions{ + .kv_type = BocOptions::KvOptions::RocksDb, .experimental = false, .cache_size = size_t{128 << 20}}, + }; + std::vector has_executor_options = {false, true}; + for (auto kv_options : kv_options_list) { + for (bool has_executor : has_executor_options) { + std::shared_ptr executor; + if (has_executor) { + executor = std::make_shared( + 4); // 4 - to compare V1 and V2, because V1 has parallel_load = 4 by default + } + // V2 - 4 threads + run({.async_executor = executor, + .kv_options = kv_options, + .options = DynamicBagOfCellsDb::CreateV2Options{ + .extra_threads = 3, .executor = executor, .cache_ttl_max = 5}}); + + // V1 + run({.async_executor = executor, .kv_options = kv_options, .options = DynamicBagOfCellsDb::CreateV1Options{}}); + + // V2 - one thread + run({.async_executor = executor, + .kv_options = kv_options, + .options = DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 0, .executor = executor, .cache_ttl_max = 5}}); + + // InMemory + for (auto use_arena : {false, true}) { + for (auto less_memory : {false, true}) { + run({.async_executor = executor, + .kv_options = kv_options, + .options = + DynamicBagOfCellsDb::CreateInMemoryOptions{.extra_threads = std::thread::hardware_concurrency(), + .verbose = false, + .use_arena = use_arena, + .use_less_memory_during_creation = less_memory}}); + } + } + } + } + + for (auto &[name, v] : benches) { + std::sort(v.begin(), v.end()); + LOG(INFO) << "Bench " << name; + for (auto &[t, name] : v) { + LOG(INFO) << "\t" << name << " " << double(t) / 1000 << "s"; } } } -void test_dynamic_boc(BocOptions options) { - auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; - auto before = counter(); - SCOPE_EXIT { - LOG_CHECK((options.o_in_memory && options.o_in_memory->use_arena) || before == counter()) - << before << " vs " << counter(); - }; +DynamicBagOfCellsDb::Stats test_dynamic_boc(BocOptions options) { + DynamicBagOfCellsDb::Stats stats; td::Random::Xorshift128plus rnd{options.seed}; std::string old_root_hash; std::string old_root_serialization; - auto kv = std::make_shared(); - auto create_dboc = [&]() { + DB db; + auto reload_db = [&]() { auto roots_n = old_root_hash.empty() ? 0 : 1; - return options.create_dboc(kv.get(), roots_n); + db = options.create_db(std::move(db), roots_n); }; - auto dboc = create_dboc(); - dboc->set_loader(std::make_unique(kv)); + reload_db(); for (int t = 1000; t >= 0; t--) { if (rnd() % 10 == 0) { - dboc = create_dboc(); + reload_db(); } - dboc->set_loader(std::make_unique(kv)); + db.reset_loader(); Ref old_root; if (!old_root_hash.empty()) { - old_root = dboc->load_cell(old_root_hash).move_as_ok(); + old_root = db.dboc->load_cell(old_root_hash).move_as_ok(); auto serialization = serialize_boc(old_root); ASSERT_EQ(old_root_serialization, serialization); } @@ -1071,47 +1371,61 @@ void test_dynamic_boc(BocOptions options) { ->get_root_cell(0) .move_as_ok(); - dboc->dec(old_root); + db.dboc->dec(old_root); if (t != 0) { - dboc->inc(cell); - } - dboc->prepare_commit().ensure(); - { - CellStorer cell_storer(*kv); - dboc->commit(cell_storer).ensure(); + db.dboc->inc(cell); } + options.commit(db, BocOptions::ResetCache); } - ASSERT_EQ(0u, kv->count("").ok()); + options.check_kv_is_empty(*db.kv); + + stats.named_stats.apply_diff(db.kv->get_usage_stats().to_named_stats()); + return stats; } TEST(TonDb, DynamicBoc) { with_all_boc_options(test_dynamic_boc, 1); }; -void test_dynamic_boc2(BocOptions options) { +DynamicBagOfCellsDb::Stats test_dynamic_boc2(BocOptions options) { td::Random::Xorshift128plus rnd{options.seed}; + DynamicBagOfCellsDb::Stats stats; - int total_roots = rnd.fast(1, !rnd.fast(0, 10) * 100 + 10); + int total_roots = rnd.fast(1, !rnd.fast(0, 30) * 100 + 10); int max_roots = rnd.fast(1, 20); + int max_cells = 20; + + // VERBOSITY_NAME(boc) = 1; + // LOG(WARNING) << "====================================================\n\n"; + // max_roots = 2; + // total_roots = 2; + // max_cells = 2; + + auto meta_key = [](size_t i) { return PSTRING() << "meta." << i; }; + std::array meta; + int last_commit_at = 0; int first_root_id = 0; int last_root_id = 0; - auto kv = std::make_shared(); - auto create_dboc = [&](td::int64 root_n) { return options.create_dboc(kv.get(), root_n); }; - auto dboc = create_dboc(0); - dboc->set_loader(std::make_unique(kv)); + DB db; + auto reload_db = [&](td::int64 root_n) { db = options.create_db(std::move(db), root_n); }; + reload_db(0); auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; auto before = counter(); - SCOPE_EXIT{ - // LOG_CHECK((options.o_in_memory && options.o_in_memory->use_arena) || before == counter()) - // << before << " vs " << counter(); + SCOPE_EXIT { + bool skip_check = false; + if (std::holds_alternative(options.options) && + std::get(options.options).use_arena) { + skip_check = true; + } + LOG_IF(FATAL, !(skip_check || before == counter())) << before << " vs " << counter(); }; std::vector> roots(max_roots); std::vector root_hashes(max_roots); auto add_root = [&](Ref root) { - dboc->inc(root); + db.dboc->inc(root); root_hashes[last_root_id % max_roots] = (root->get_hash().as_slice().str()); roots[last_root_id % max_roots] = root; last_root_id++; @@ -1124,9 +1438,9 @@ void test_dynamic_boc2(BocOptions options) { VLOG(boc) << " from db"; auto from_root_hash = root_hashes[root_id % max_roots]; if (rnd() % 2 == 0) { - from_root = dboc->load_root(from_root_hash).move_as_ok(); + from_root = db.dboc->load_root(from_root_hash).move_as_ok(); } else { - from_root = dboc->load_cell(from_root_hash).move_as_ok(); + from_root = db.dboc->load_cell(from_root_hash).move_as_ok(); } } else { VLOG(boc) << "FROM MEMORY"; @@ -1147,31 +1461,69 @@ void test_dynamic_boc2(BocOptions options) { from_root = get_root(rnd.fast(first_root_id, last_root_id - 1)); } VLOG(boc) << " ..."; - auto new_root = gen_random_cell(rnd.fast(1, 20), from_root, rnd); - root_cnt[new_root->get_hash()]++; - add_root(std::move(new_root)); + auto new_root_cell = gen_random_cell(rnd.fast(1, max_cells), from_root, rnd); + root_cnt[new_root_cell->get_hash()]++; + add_root(std::move(new_root_cell)); VLOG(boc) << " OK"; }; - auto commit = [&] { - VLOG(boc) << "commit"; - //rnd.fast(0, 1); - options.prepare_commit(*dboc); - { - CellStorer cell_storer(*kv); - dboc->commit(cell_storer); + td::UsageStats commit_stats{}; + auto commit = [&](bool finish = false) { + for (size_t i = 0; i < meta.size(); i++) { + std::string value; + auto status = db.dboc->meta_get(meta_key(i), value).move_as_ok(); + if (status == KeyValue::GetStatus::Ok) { + ASSERT_EQ(value, meta[i]); + ASSERT_TRUE(!meta[i].empty()); + } else { + ASSERT_TRUE(meta[i].empty()); + } + + if (meta[i].empty()) { + if (!finish && rnd() % 2 == 0) { + meta[i] = td::to_string(rnd()); + db.dboc->meta_set(meta_key(i), meta[i]); + VLOG(boc) << "meta set " << meta_key(i) << " " << meta[i]; + } + } else { + auto f = finish ? 1 : rnd() % 3; + if (f == 0) { + meta[i] = td::to_string(rnd()); + db.dboc->meta_set(meta_key(i), meta[i]); + VLOG(boc) << "meta set " << meta_key(i) << " " << meta[i]; + } else if (f == 1) { + meta[i] = ""; + db.dboc->meta_erase(meta_key(i)); + VLOG(boc) << "meta erase " << meta_key(i); + } + } } - dboc->set_loader(std::make_unique(kv)); + + VLOG(boc) << "before commit cells_in_db=" << db.kv->count(""); + //rnd.fast(0, 1); + auto stats_before = db.kv->get_usage_stats(); + options.commit(db, BocOptions::ResetCache); + auto stats_after = db.kv->get_usage_stats(); + commit_stats = commit_stats + stats_after - stats_before; + VLOG(boc) << "after commit cells_in_db=" << db.kv->count(""); + + // db.reset_loader(); for (int i = last_commit_at; i < last_root_id; i++) { roots[i % max_roots].clear(); } last_commit_at = last_root_id; }; - auto reset = [&] { + auto reset = [&](bool force_full = false) { VLOG(boc) << "reset"; commit(); - dboc = create_dboc(td::int64(root_cnt.size())); - dboc->set_loader(std::make_unique(kv)); + if (rnd() % 3 == 0 || force_full) { + // very slow for rocksdb + auto r_stats = db.dboc->get_stats(); + if (r_stats.is_ok()) { + stats.apply_diff(r_stats.ok()); + } + reload_db(root_cnt.size()); + } }; auto delete_root = [&] { @@ -1187,22 +1539,28 @@ void test_dynamic_boc2(BocOptions options) { root_cnt.erase(it); } - dboc->dec(std::move(old_root)); + db.dboc->dec(std::move(old_root)); first_root_id++; VLOG(boc) << " OK"; }; td::RandomSteps steps({{new_root, 10}, {delete_root, 9}, {commit, 2}, {reset, 1}}); while (first_root_id != total_roots) { - VLOG(boc) << first_root_id << " " << last_root_id << " " << kv->count("").ok(); + VLOG(boc) << first_root_id << " " << last_root_id; // << " " << db.kv->count("").ok(); steps.step(rnd); } - commit(); - ASSERT_EQ(0u, kv->count("").ok()); + commit(true); + options.check_kv_is_empty(*db.kv); + + // auto stats = kv->get_usage_stats(); + // LOG(ERROR) << "total: " << stats; + reset(true); + stats.named_stats.apply_diff(db.kv->get_usage_stats().to_named_stats()); + return stats; } TEST(TonDb, DynamicBoc2) { - with_all_boc_options(test_dynamic_boc2); + with_all_boc_options(test_dynamic_boc2, 50); } template @@ -1341,6 +1699,10 @@ class CompactArray { size_t size() const { return size_; } + void reset() { + size_ = 0; + root_ = {}; + } Ref merkle_proof(std::vector keys) { std::set hashes; @@ -1435,6 +1797,283 @@ class FastCompactArray { std::vector v_; }; +struct BocTestHelper { + public: + BocTestHelper() = default; + BocTestHelper(td::int64 seed) : rnd_(seed) { + } + + CompactArray create_array(size_t size, td::uint64 max_value) { + std::vector v(size); + td::Random::Xorshift128plus rnd{123}; + for (auto &x : v) { + x = rnd() % max_value; + } + return CompactArray(v); + } + + private: + td::Random::Xorshift128plus rnd_{123}; +}; + +DynamicBagOfCellsDb::Stats bench_dboc_get_and_set(BocOptions options) { + BocTestHelper helper(options.seed); + size_t n = 1 << 20; + size_t max_value = 1 << 26; + auto arr = helper.create_array(n, max_value); + + // auto kv = std::make_shared(); + td::Slice db_path = "compact_array_db"; + td::RocksDb::destroy(db_path).ensure(); + + DB db = options.create_db({}, {}); + DynamicBagOfCellsDb::Stats stats; + + td::Timer total_timer; + + auto bench = [&](td::Slice desc, auto &&f) { + auto before = db.dboc->get_stats().move_as_ok(); + td::Timer timer; + LOG(ERROR) << "Benchmarking " << desc; + f(); + stats.named_stats.stats_int[desc.str()] = td::int64(timer.elapsed() * 1000); + LOG(ERROR) << "Benchmarking " << desc << " done: " << timer.elapsed() << "s\n"; + auto after = db.dboc->get_stats().move_as_ok(); + after.named_stats.subtract_diff(before.named_stats); + LOG(ERROR) << after; + }; + + td::VectorQueue roots; + // Save array in db + bench(PSLICE() << "bench_inc_large_db(n=" << n << ")", [&] { + db.dboc->inc(arr.root()); + roots.push(arr.root()->get_hash()); + options.commit(db, BocOptions::ResetCache); + }); + bench("bench_compactify", [&] { + auto status = dynamic_cast(*db.kv).raw_db()->CompactRange({}, nullptr, nullptr); + LOG_IF(FATAL, !status.ok()) << status.ToString(); + }); + db = options.create_db(std::move(db), {}); + + bench(PSLICE() << "bench_inc_large_existed_db(n=" << n << ")", [&] { + db.dboc->inc(arr.root()); + roots.push(arr.root()->get_hash()); + options.commit(db, BocOptions::ResetCache); + }); + + td::Random::Xorshift128plus rnd{123}; + while (false) { + auto hash = arr.root()->get_hash(); + arr = CompactArray{n, db.dboc->load_root(hash.as_slice()).move_as_ok()}; + td::Timer timer; + for (size_t i = 0; i < 10000; i++) { + auto pos = rnd() % n; + arr.get(pos); + } + LOG(ERROR) << timer.elapsed() << "s\n"; + db.reset_loader(); + } + + for (auto p : + std::vector>{{10000, 0}, {10000, 5}, {5000, 5000}, {5, 10000}, {0, 10000}}) { + auto get_n = p.first; + auto set_n = p.second; + auto hash = arr.root()->get_hash(); + arr = CompactArray{n, db.dboc->load_root(hash.as_slice()).move_as_ok()}; + bench(PSTRING() << "bench_changes(get_n=" << get_n << ", set_n=" << set_n << ")", [&] { + for (size_t i = 0; i < get_n; i++) { + auto pos = rnd() % n; + arr.get(pos); + } + for (size_t i = 0; i < set_n; i++) { + auto pos = rnd() % n; + auto value = rnd() % max_value; + arr.set(pos, value); + } + }); + bench(PSTRING() << "bench_commit(get_n=" << get_n << ", set_n=" << set_n << ")", [&] { + db.dboc->inc(arr.root()); + roots.push(arr.root()->get_hash()); + options.commit(db, BocOptions::ResetCache); + }); + } + arr.reset(); + + bench(PSLICE() << "bench_dec_some_roots()", [&] { + while (roots.size() > 1) { + auto hash = roots.pop(); + auto cell = db.dboc->load_cell(hash.as_slice()).move_as_ok(); + db.dboc->dec(cell); + } + options.commit(db, BocOptions::ResetCache); + }); + + db = options.create_db(std::move(db), {}); + + bench(PSLICE() << "bench_dec_large_root(n=" << n << ")", [&] { + while (!roots.empty()) { + auto hash = roots.pop(); + auto cell = db.dboc->load_cell(hash.as_slice()).move_as_ok(); + db.dboc->dec(cell); + + /* + do { + auto cell = db.dboc->load_cell(hash.as_slice()).move_as_ok(); + db.dboc->dec(cell); + cell = {}; + options.prepare_commit(*db.dboc); + //db.dboc->prepare_commit().ensure(); + db.reset_loader(); + db = options.create_db(std::move(db), {}); + } while (true); + */ + } + options.commit(db, BocOptions::ResetCache); + }); + stats.named_stats.stats_int["bench_total"] = td::int64(total_timer.elapsed() * 1000); + + return stats; +} + +TEST(TonDb, BenchDynamicBocGetAndSet) { + with_all_boc_options(bench_dboc_get_and_set, 1); +} + +TEST(TonDb, DynamicBocIncSimple) { + auto kv = std::make_shared(std::make_shared()); + auto db = DynamicBagOfCellsDb::create_v2({.extra_threads = 0}); + db->set_loader(std::make_unique(kv)); + + td::Random::Xorshift128plus rnd(123); + size_t size = 4; + std::vector values(size); + for (auto &v : values) { + //v = rnd() % 2; + v = rnd(); + } + // 1. Create large dictionary and store it in db + auto arr_ptr = std::make_unique(values); + auto &arr = *arr_ptr; + td::VectorQueue queue; + auto push = [&]() { + //LOG(ERROR) << "PUSH ROOT"; + auto begin_stats = kv->get_usage_stats(); + db->inc(arr.root()); + queue.push(arr.root()->get_hash()); + vm::CellStorer cell_storer(*kv); + db->commit(cell_storer); + auto end_stats = kv->get_usage_stats(); + LOG(ERROR) << end_stats - begin_stats; + db->set_loader(std::make_unique(kv)); + auto hash = arr.root()->get_hash(); + arr = CompactArray{size, db->load_root(hash.as_slice()).move_as_ok()}; + //LOG(ERROR) << "CELLS IN DB: " << kv->count("").move_as_ok(); + }; + auto pop = [&]() { + if (queue.empty()) { + return; + } + //LOG(ERROR) << "POP ROOT"; + auto begin_stats = kv->get_usage_stats(); + auto cell = db->load_cell(queue.pop().as_slice()).move_as_ok(); + db->dec(cell); + vm::CellStorer cell_storer(*kv); + db->commit(cell_storer); + auto end_stats = kv->get_usage_stats(); + db->set_loader(std::make_unique(kv)); + //LOG(ERROR) << end_stats - begin_stats; + //LOG(ERROR) << "CELLS IN DB: " << kv->count("").move_as_ok(); + }; + auto upd = [&] { + for (int i = 0; i < 20; i++) { + auto pos = rnd.fast(0, td::narrow_cast(size) - 1); + if (rnd() % 2) { + auto value = rnd() % 2; + arr.set(pos, value); + } else { + arr.get(pos); + } + } + }; + + //LOG(ERROR) << "Created compact array"; + push(); + pop(); + //CHECK(kv->count("").move_as_ok() == 0); + + // 2. Lets change first 20 keys and read last 20 keys + /* + for (size_t i = 0; i < 20 && i < size; i++) { + arr.set(i, rnd()); + } + */ + //arr.set(0, rnd()); + arr.set(size - 1, rnd()); + for (size_t i = 0; i < 20 && i < size; i++) { + arr.get(size - i - 1); + } + + // 3. And now commit diff with stats + push(); + push(); + upd(); + upd(); + push(); + push(); + upd(); + pop(); + pop(); + upd(); + push(); + push(); + while (!queue.empty()) { + pop(); + } + LOG(ERROR) << "CELLS IN DB: " << kv->count("").move_as_ok(); +} + +class BenchCellStorerMergeRefcntDiffs : public td::Benchmark { + public: + std::string get_description() const override { + return PSTRING() << "bench_cells_storer_merge_refcnt_diffs"; + } + + void run(int n) override { + auto cell = vm::CellBuilder().store_bytes(std::string(32, 'A')).finalize(); + auto left_update = CellStorer::serialize_refcnt_diffs(1); + auto right_update = CellStorer::serialize_refcnt_diffs(1); + for (int i = 0; i < n; i++) { + CellStorer::merge_refcnt_diffs(left_update, right_update); + } + } + + private: + size_t tn_{}; +}; +class BenchCellStorerMergeValueAndRefcntDiff : public td::Benchmark { + public: + std::string get_description() const override { + return PSTRING() << "bench_cells_storer_merge_value_and_refcnt_diffs"; + } + + void run(int n) override { + auto cell = vm::CellBuilder().store_bytes(std::string(32, 'A')).finalize(); + auto value = CellStorer::serialize_value(10, cell, false); + auto update = CellStorer::serialize_refcnt_diffs(1); + for (int i = 0; i < n; i++) { + CellStorer::merge_value_and_refcnt_diff(value, update); + } + } + + private: + size_t tn_{}; +}; +TEST(Bench, CellStorerMerge) { + bench(BenchCellStorerMergeRefcntDiffs()); + bench(BenchCellStorerMergeValueAndRefcntDiff()); +} + TEST(Cell, BocHands) { serialize_boc(CellBuilder{}.store_bytes("AAAAAAAA").finalize()); auto a = CellBuilder{}.store_bytes("abcd").store_ref(CellBuilder{}.store_bytes("???").finalize()).finalize(); @@ -2262,7 +2901,8 @@ TEST(TonDb, BocRespectsUsageCell) { ASSERT_STREQ(serialization, serialization_of_virtualized_cell); } -void test_dynamic_boc_respectes_usage_cell(vm::BocOptions options) { +/* +vm::DynamicBagOfCellsDb::Stats test_dynamic_boc_respects_usage_cell(vm::BocOptions options) { td::Random::Xorshift128plus rnd(options.seed); auto cell = vm::gen_random_cell(20, rnd, true); auto usage_tree = std::make_shared(); @@ -2283,11 +2923,14 @@ void test_dynamic_boc_respectes_usage_cell(vm::BocOptions options) { auto serialization_of_virtualized_cell = serialize_boc(virtualized_proof); auto serialization = serialize_boc(cell); ASSERT_STREQ(serialization, serialization_of_virtualized_cell); + vm::DynamicBagOfCellsDb::Stats stats; + return stats; } TEST(TonDb, DynamicBocRespectsUsageCell) { - vm::with_all_boc_options(test_dynamic_boc_respectes_usage_cell, 20); + vm::with_all_boc_options(test_dynamic_boc_respects_usage_cell, 20, true); } +*/ TEST(TonDb, LargeBocSerializer) { td::Random::Xorshift128plus rnd{123}; diff --git a/crypto/vm/cells/Cell.h b/crypto/vm/cells/Cell.h index a75371dbb7..e2b47ffc01 100644 --- a/crypto/vm/cells/Cell.h +++ b/crypto/vm/cells/Cell.h @@ -55,6 +55,7 @@ class Cell : public CellTraits { } // load interface + virtual td::Status set_data_cell(Ref &&data_cell) const = 0; virtual td::Result load_cell() const = 0; virtual Ref virtualize(VirtualizationParameters virt) const; virtual td::uint32 get_virtualization() const = 0; diff --git a/crypto/vm/cells/DataCell.cpp b/crypto/vm/cells/DataCell.cpp index 4dd301616b..73e86517a2 100644 --- a/crypto/vm/cells/DataCell.cpp +++ b/crypto/vm/cells/DataCell.cpp @@ -36,7 +36,8 @@ struct ArenaAllocator { T* obj = new (ptr) T(std::forward(args)...); return std::unique_ptr(obj); } -private: + + private: td::MutableSlice alloc_batch() { size_t batch_size = 1 << 20; auto batch = std::make_unique(batch_size); @@ -53,7 +54,7 @@ struct ArenaAllocator { return res; } }; -} +} // namespace std::unique_ptr DataCell::create_empty_data_cell(Info info) { if (use_arena) { ArenaAllocator allocator; diff --git a/crypto/vm/cells/DataCell.h b/crypto/vm/cells/DataCell.h index 6d3c845fc4..b39ee1d4b2 100644 --- a/crypto/vm/cells/DataCell.h +++ b/crypto/vm/cells/DataCell.h @@ -31,6 +31,9 @@ class DataCell : public Cell { static thread_local bool use_arena; DataCell(const DataCell& other) = delete; + DataCell(DataCell&& other) = delete; + DataCell& operator=(const DataCell& other) = delete; + DataCell& operator=(DataCell&& other) = delete; ~DataCell() override; static void store_depth(td::uint8* dest, td::uint16 depth) { @@ -126,6 +129,10 @@ class DataCell : public Cell { explicit DataCell(Info info); public: + td::Status set_data_cell(Ref&& data_cell) const override { + CHECK(get_hash() == data_cell->get_hash()); + return td::Status::OK(); + } td::Result load_cell() const override { return LoadedCell{Ref{this}, {}, {}}; } @@ -228,4 +235,3 @@ inline CellHash as_cell_hash(const Ref& cell) { } } // namespace vm - diff --git a/crypto/vm/cells/ExtCell.h b/crypto/vm/cells/ExtCell.h index 401bb04831..dbbd8575b7 100644 --- a/crypto/vm/cells/ExtCell.h +++ b/crypto/vm/cells/ExtCell.h @@ -65,6 +65,9 @@ class ExtCell : public Cell { bool is_loaded() const override { return CellView(this)->is_loaded(); } + Ref> get_prunned_cell() const { + return prunned_cell_.load(); + } private: mutable td::AtomicRef data_cell_; @@ -112,6 +115,23 @@ class ExtCell : public Cell { return CellView(this)->get_depth(level); } + td::Status set_data_cell(Ref&& new_data_cell) const override { + auto prunned_cell = prunned_cell_.load(); + if (prunned_cell.is_null()) { + auto old_data_cell = data_cell_.get_unsafe(); + DCHECK(old_data_cell); + TRY_STATUS(old_data_cell->check_equals_unloaded(new_data_cell)); + return td::Status::OK(); + } + + TRY_STATUS(prunned_cell->check_equals_unloaded(new_data_cell)); + if (data_cell_.store_if_empty(new_data_cell)) { + prunned_cell_.store({}); + get_thread_safe_counter_unloaded().add(-1); + } + return td::Status::OK(); + } + td::Result> load_data_cell() const { auto data_cell = data_cell_.get_unsafe(); if (data_cell) { diff --git a/crypto/vm/cells/PrunnedCell.h b/crypto/vm/cells/PrunnedCell.h index a58b245ccf..6e8b77093d 100644 --- a/crypto/vm/cells/PrunnedCell.h +++ b/crypto/vm/cells/PrunnedCell.h @@ -142,6 +142,10 @@ class PrunnedCell : public Cell { return info_.get_depth(get_storage())[get_level_mask().apply(level).get_hash_i()]; } + td::Status set_data_cell(Ref &&data_cell) const override { + return td::Status::OK(); + } + td::Result load_cell() const override { return td::Status::Error("Can't load prunned branch"); } diff --git a/crypto/vm/cells/UsageCell.h b/crypto/vm/cells/UsageCell.h index 3e6e88982a..978b91f763 100644 --- a/crypto/vm/cells/UsageCell.h +++ b/crypto/vm/cells/UsageCell.h @@ -36,6 +36,9 @@ class UsageCell : public Cell { return Ref{true, std::move(cell), std::move(tree_node), PrivateTag{}}; } + td::Status set_data_cell(Ref &&data_cell) const override { + return cell_->set_data_cell(std::move(data_cell)); + } // load interface td::Result load_cell() const override { TRY_RESULT(loaded_cell, cell_->load_cell()); diff --git a/crypto/vm/cells/VirtualCell.h b/crypto/vm/cells/VirtualCell.h index 02abc1c881..a75bdf9de3 100644 --- a/crypto/vm/cells/VirtualCell.h +++ b/crypto/vm/cells/VirtualCell.h @@ -37,6 +37,9 @@ class VirtualCell : public Cell { } // load interface + td::Status set_data_cell(Ref &&data_cell) const override { + return cell_->set_data_cell(std::move(data_cell)); + } td::Result load_cell() const override { TRY_RESULT(loaded_cell, cell_->load_cell()); loaded_cell.virt = loaded_cell.virt.apply(virt_); diff --git a/crypto/vm/db/CellHashTable.h b/crypto/vm/db/CellHashTable.h index 522c987be6..a389806380 100644 --- a/crypto/vm/db/CellHashTable.h +++ b/crypto/vm/db/CellHashTable.h @@ -40,6 +40,17 @@ class CellHashTable { return res; } + template + std::pair emplace(td::Slice hash, ArgsT &&...args) { + auto it = set_.find(hash); + if (it != set_.end()) { + return std::pair(const_cast(*it), false); + } + auto res = set_.emplace(std::forward(args)...); + CHECK(res.second); + return std::pair(const_cast(*res.first), res.second); + } + template void for_each(F &&f) { for (auto &info : set_) { @@ -64,7 +75,7 @@ class CellHashTable { size_t size() const { return set_.size(); } - InfoT* get_if_exists(td::Slice hash) { + InfoT *get_if_exists(td::Slice hash) { auto it = set_.find(hash); if (it != set_.end()) { return &const_cast(*it); diff --git a/crypto/vm/db/CellStorage.cpp b/crypto/vm/db/CellStorage.cpp index 06df461ef6..a07d85e870 100644 --- a/crypto/vm/db/CellStorage.cpp +++ b/crypto/vm/db/CellStorage.cpp @@ -17,14 +17,19 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "vm/db/CellStorage.h" + +#include "td/utils/Parser.h" #include "vm/db/DynamicBagOfCellsDb.h" #include "vm/boc.h" #include "td/utils/base64.h" #include "td/utils/tl_parsers.h" #include "td/utils/tl_helpers.h" +#include + namespace vm { namespace { + class RefcntCellStorer { public: RefcntCellStorer(td::int32 refcnt, const td::Ref &cell, bool as_boc) @@ -43,7 +48,9 @@ class RefcntCellStorer { storer.store_slice(data); return; } + CHECK(refcnt_ > 0); store(refcnt_, storer); + CHECK(cell_.not_null()) store(*cell_, storer); for (unsigned i = 0; i < cell_->size_refs(); i++) { auto cell = cell_->get_ref(i); @@ -91,6 +98,7 @@ class RefcntCellParser { stored_boc_ = true; parse(refcnt, parser); } + CHECK(refcnt > 0); if (!need_data_) { return; } @@ -159,6 +167,9 @@ td::Result CellLoader::load(td::Slice hash, bool need_da DCHECK(get_status == KeyValue::GetStatus::NotFound); return LoadResult{}; } + if (serialized.empty()) { + return LoadResult{}; + } TRY_RESULT(res, load(hash, serialized, need_data, ext_cell_creator)); if (on_load_callback_) { on_load_callback_(res); @@ -198,6 +209,7 @@ td::Result CellLoader::load_refcnt(td::Slice hash) { if (res.refcnt_ == -1) { parse(res.refcnt_, parser); } + CHECK(res.refcnt_ > 0); TRY_STATUS(parser.get_status()); return res; } @@ -216,4 +228,77 @@ std::string CellStorer::serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc) { return kv_.set(cell->get_hash().as_slice(), serialize_value(refcnt, cell, as_boc)); } + +td::Status CellStorer::merge(td::Slice hash, td::int32 refcnt_diff) { + return kv_.merge(hash, serialize_refcnt_diffs(refcnt_diff)); +} + +void CellStorer::merge_value_and_refcnt_diff(std::string &left, td::Slice right) { + if (right.empty()) { + return; + } + CHECK(left.size() > 4); + CHECK(right.size() == 4); + + td::int32 left_refcnt = td::as(left.data()); + size_t shift = 0; + if (left_refcnt == -1) { + CHECK(left.size() >= 8); + left_refcnt = td::as(left.data() + 4); + shift = 4; + } + td::int32 right_refcnt_diff = td::as(right.data()); + td::int32 new_refcnt = left_refcnt + right_refcnt_diff; + CHECK(new_refcnt > 0); + td::as(left.data() + shift) = new_refcnt; +} +void CellStorer::merge_refcnt_diffs(std::string &left, td::Slice right) { + if (right.empty()) { + return; + } + if (left.empty()) { + left = right.str(); + return; + } + CHECK(left.size() == 4); + CHECK(right.size() == 4); + td::int32 left_refcnt_diff = td::as(left.data()); + td::int32 right_refcnt_diff = td::as(right.data()); + td::int32 total_refcnt_diff = left_refcnt_diff + right_refcnt_diff; + td::as(left.data()) = total_refcnt_diff; +} + +std::string CellStorer::serialize_refcnt_diffs(td::int32 refcnt_diff) { + TD_PERF_COUNTER(cell_store_refcnt_diff); + std::string s(4, 0); + td::as(s.data()) = refcnt_diff; + return s; +} + +td::Status CellStorer::apply_diff(const Diff &diff) { + switch (diff.type) { + case Diff::Set: + return kv_.set(diff.key.as_slice(), diff.value); + case Diff::Erase: + return kv_.erase(diff.key.as_slice()); + case Diff::Merge: + return kv_.merge(diff.key.as_slice(), diff.value); + default: + UNREACHABLE(); + } +} +td::Status CellStorer::apply_meta_diff(const MetaDiff &diff) { + switch (diff.type) { + case MetaDiff::Set: + CHECK(diff.key.size() != CellTraits::hash_bytes); + CHECK(!diff.value.empty()); + return kv_.set(diff.key, diff.value); + case MetaDiff::Erase: + CHECK(diff.key.size() != CellTraits::hash_bytes); + CHECK(diff.value.empty()); + return kv_.erase(diff.key); + default: + UNREACHABLE(); + } +} } // namespace vm diff --git a/crypto/vm/db/CellStorage.h b/crypto/vm/db/CellStorage.h index cabd7fdcbd..ca32a8007d 100644 --- a/crypto/vm/db/CellStorage.h +++ b/crypto/vm/db/CellStorage.h @@ -51,6 +51,9 @@ class CellLoader { td::Result load(td::Slice hash, bool need_data, ExtCellCreator &ext_cell_creator); static td::Result load(td::Slice hash, td::Slice value, bool need_data, ExtCellCreator &ext_cell_creator); td::Result load_refcnt(td::Slice hash); // This only loads refcnt_, cell_ == null + KeyValueReader &key_value_reader() const { + return *reader_; + } private: std::shared_ptr reader_; @@ -62,8 +65,28 @@ class CellStorer { CellStorer(KeyValue &kv); td::Status erase(td::Slice hash); td::Status set(td::int32 refcnt, const td::Ref &cell, bool as_boc); + td::Status merge(td::Slice hash, td::int32 refcnt_diff); + + static void merge_value_and_refcnt_diff(std::string &value, td::Slice right); + static void merge_refcnt_diffs(std::string &left, td::Slice right); + static std::string serialize_refcnt_diffs(td::int32 refcnt_diff); + static std::string serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc); + struct Diff { + enum Type { Set, Erase, Merge } type{Set}; + CellHash key; + std::string value{}; + }; + td::Status apply_diff(const Diff &diff); + + struct MetaDiff { + enum Type { Set, Erase } type{Set}; + std::string key; + std::string value{}; + }; + td::Status apply_meta_diff(const MetaDiff &diff); + private: KeyValue &kv_; }; diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index 093037583b..bd23733e03 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -66,19 +66,27 @@ struct CellInfo { struct Eq { using is_transparent = void; // Pred to use - bool operator()(const CellInfo &info, const CellInfo &other_info) const { return info.key() == other_info.key();} - bool operator()(const CellInfo &info, td::Slice hash) const { return info.key().as_slice() == hash;} - bool operator()(td::Slice hash, const CellInfo &info) const { return info.key().as_slice() == hash;} - + bool operator()(const CellInfo &info, const CellInfo &other_info) const { + return info.key() == other_info.key(); + } + bool operator()(const CellInfo &info, td::Slice hash) const { + return info.key().as_slice() == hash; + } + bool operator()(td::Slice hash, const CellInfo &info) const { + return info.key().as_slice() == hash; + } }; struct Hash { using is_transparent = void; // Pred to use using transparent_key_equal = Eq; - size_t operator()(td::Slice hash) const { return cell_hash_slice_hash(hash); } - size_t operator()(const CellInfo &info) const { return cell_hash_slice_hash(info.key().as_slice());} + size_t operator()(td::Slice hash) const { + return cell_hash_slice_hash(hash); + } + size_t operator()(const CellInfo &info) const { + return cell_hash_slice_hash(info.key().as_slice()); + } }; }; - bool operator<(const CellInfo &a, td::Slice b) { return a.key().as_slice() < b; } @@ -99,6 +107,30 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { return get_cell_info_lazy(level_mask, hash, depth).cell; } + td::Result>> meta_get_all() const override { + std::vector> result; + auto s = loader_->key_value_reader().for_each_in_range("desc", "desd", + [&](const td::Slice &key, const td::Slice &value) { + if (td::begins_with(key, "desc") && key.size() != 32) { + result.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + TRY_STATUS(std::move(s)); + return result; + } + td::Result meta_get(td::Slice key, std::string &value) override { + return loader_->key_value_reader().get(key, value); + } + td::Status meta_set(td::Slice key, td::Slice value) override { + meta_diffs_.push_back( + CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Set, .key = key.str(), .value = value.str()}); + return td::Status::OK(); + } + td::Status meta_erase(td::Slice key) override { + meta_diffs_.push_back(CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Erase, .key = key.str()}); + return td::Status::OK(); + } td::Result> load_cell(td::Slice hash) override { auto info = hash_table_.get_if_exists(hash); if (info && info->sync_with_db) { @@ -198,21 +230,29 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat if (is_prepared_for_commit()) { return td::Status::OK(); } + td::PerfWarningTimer timer_dfs_new_cells_in_db("dfs_new_cells_in_db"); for (auto &new_cell : to_inc_) { auto &new_cell_info = get_cell_info(new_cell); dfs_new_cells_in_db(new_cell_info); } + timer_dfs_new_cells_in_db.reset(); + td::PerfWarningTimer timer_dfs_new_cells("dfs_new_cells"); for (auto &new_cell : to_inc_) { auto &new_cell_info = get_cell_info(new_cell); dfs_new_cells(new_cell_info); } + timer_dfs_new_cells.reset(); + td::PerfWarningTimer timer_dfs_old_cells("dfs_old_cells"); for (auto &old_cell : to_dec_) { auto &old_cell_info = get_cell_info(old_cell); dfs_old_cells(old_cell_info); } + timer_dfs_old_cells.reset(); + td::PerfWarningTimer timer_save_diff_prepare("save_diff_prepare"); save_diff_prepare(); + timer_save_diff_prepare.reset(); to_inc_.clear(); to_dec_.clear(); @@ -222,6 +262,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat td::Status commit(CellStorer &storer) override { prepare_commit(); + td::PerfWarningTimer times_save_diff("save diff", 0.01); save_diff(storer); // Some elements are erased from hash table, to keep it small. // Hash table is no longer represents the difference between the loader and @@ -249,7 +290,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat celldb_compress_depth_ = value; } - vm::ExtCellCreator& as_ext_cell_creator() override { + vm::ExtCellCreator &as_ext_cell_creator() override { return *this; } @@ -259,6 +300,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat std::vector> to_dec_; CellHashTable hash_table_; std::vector visited_; + std::vector meta_diffs_; Stats stats_diff_; td::uint32 celldb_compress_depth_{0}; @@ -269,8 +311,9 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat class SimpleExtCellCreator : public ExtCellCreator { public: - explicit SimpleExtCellCreator(std::shared_ptr cell_db_reader) : - cell_db_reader_(std::move(cell_db_reader)) {} + explicit SimpleExtCellCreator(std::shared_ptr cell_db_reader) + : cell_db_reader_(std::move(cell_db_reader)) { + } td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { TRY_RESULT(ext_cell, DynamicBocExtCell::create(PrunnedCellInfo{level_mask, hash, depth}, @@ -279,7 +322,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return std::move(ext_cell); } - std::vector>& get_created_cells() { + std::vector> &get_created_cells() { return created_cells_; } @@ -382,8 +425,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat } bool not_in_db = false; - for_each( - info, [¬_in_db, this](auto &child_info) { not_in_db |= !dfs_new_cells_in_db(child_info); }, false); + for_each(info, [¬_in_db, this](auto &child_info) { not_in_db |= !dfs_new_cells_in_db(child_info); }, false); if (not_in_db) { CHECK(!info.in_db); @@ -441,6 +483,10 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat for (auto info_ptr : visited_) { save_cell(*info_ptr, storer); } + for (auto meta_diff : meta_diffs_) { + storer.apply_meta_diff(meta_diff); + } + meta_diffs_.clear(); visited_.clear(); } @@ -558,6 +604,8 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat } auto res = r_res.move_as_ok(); if (res.status != CellLoader::LoadResult::Ok) { + LOG_CHECK(info.cell.not_null()) << "Trying to load nonexistent cell from db " + << CellHash::from_slice(hash).to_hex(); break; } info.cell = std::move(res.cell()); @@ -651,7 +699,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat CellHashTable cells_; - std::queue load_queue_; + std::queue load_queue_; td::uint32 active_load_ = 0; td::uint32 max_parallel_load_ = 4; }; @@ -814,11 +862,10 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat pca_state_->promise_.set_result(td::Unit()); pca_state_ = {}; } - }; } // namespace -std::unique_ptr DynamicBagOfCellsDb::create() { +std::unique_ptr DynamicBagOfCellsDb::create(CreateV1Options) { return std::make_unique(); } } // namespace vm diff --git a/crypto/vm/db/DynamicBagOfCellsDb.h b/crypto/vm/db/DynamicBagOfCellsDb.h index 62864ad979..ca23d72f21 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.h +++ b/crypto/vm/db/DynamicBagOfCellsDb.h @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "td/db/KeyValue.h" #include "vm/cells.h" #include "td/utils/Slice.h" @@ -49,13 +50,23 @@ class CellDbReader { class DynamicBagOfCellsDb { public: virtual ~DynamicBagOfCellsDb() = default; + + virtual td::Result>> meta_get_all() const = 0; + virtual td::Result meta_get(td::Slice key, std::string &value) = 0; + virtual td::Status meta_set(td::Slice key, td::Slice value) = 0; + virtual td::Status meta_erase(td::Slice key) = 0; + virtual td::Result> load_cell(td::Slice hash) = 0; virtual td::Result> load_root(td::Slice hash) = 0; virtual td::Result> load_root_thread_safe(td::Slice hash) const = 0; + virtual td::Result>> load_known_roots() const { + return std::vector>(); + } struct Stats { td::int64 roots_total_count{0}; td::int64 cells_total_count{0}; td::int64 cells_total_size{0}; + td::NamedStats named_stats; std::vector> custom_stats; void apply_diff(const Stats &diff) { roots_total_count += diff.roots_total_count; @@ -64,6 +75,20 @@ class DynamicBagOfCellsDb { CHECK(roots_total_count >= 0); CHECK(cells_total_count >= 0); CHECK(cells_total_size >= 0); + named_stats.apply_diff(diff.named_stats); + } + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const Stats &stats) { + sb << "STATS\n"; + for (auto &p : stats.custom_stats) { + sb << "\t" << p.first << "\t" << p.second << "\n"; + } + for (auto &p : stats.named_stats.stats_int) { + sb << "\t" << p.first << "\t" << p.second << "\n"; + } + for (auto &p : stats.named_stats.stats_str) { + sb << "\t" << p.first << "\t" << p.second << "\n"; + } + return sb; } }; virtual void inc(const Ref &old_root) = 0; @@ -72,7 +97,7 @@ class DynamicBagOfCellsDb { virtual td::Status prepare_commit() = 0; virtual Stats get_stats_diff() = 0; virtual td::Result get_stats() { - return td::Status::Error("Not implemented"); + return Stats{}; } virtual td::Status commit(CellStorer &) = 0; virtual std::shared_ptr get_cell_db_reader() = 0; @@ -83,25 +108,49 @@ class DynamicBagOfCellsDb { virtual void set_celldb_compress_depth(td::uint32 value) = 0; virtual vm::ExtCellCreator &as_ext_cell_creator() = 0; - static std::unique_ptr create(); + class AsyncExecutor { + public: + virtual ~AsyncExecutor() { + } + virtual void execute_async(std::function f) = 0; + virtual void execute_sync(std::function f) = 0; + virtual std::string describe() const { + return "AsyncExecutor"; + } + }; + + struct CreateV1Options { + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const CreateV1Options &options) { + return sb << "V1{}"; + } + }; + static std::unique_ptr create(CreateV1Options = {}); + + struct CreateV2Options { + size_t extra_threads{std::thread::hardware_concurrency()}; + std::shared_ptr executor{}; + size_t cache_ttl_max{2000}; + size_t cache_size_max{1000000}; + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const CreateV2Options &options) { + return sb << "V2{extra_threads=" << options.extra_threads << ", cache_ttl_max=" << options.cache_ttl_max + << ", cache_size_max=" << options.cache_size_max << "}"; + } + }; + static std::unique_ptr create_v2(CreateV2Options options); struct CreateInMemoryOptions { size_t extra_threads{std::thread::hardware_concurrency()}; bool verbose{true}; - // Allocated DataCels will never be deleted + // Allocated DataCells will never be deleted bool use_arena{false}; // Almost no overhead in memory during creation, but will scan database twice bool use_less_memory_during_creation{true}; - }; - static std::unique_ptr create_in_memory(td::KeyValueReader *kv, CreateInMemoryOptions options); - - class AsyncExecutor { - public: - virtual ~AsyncExecutor() { + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const CreateInMemoryOptions &options) { + return sb << "InMemory{extra_threads=" << options.extra_threads << ", use_arena=" << options.use_arena + << ", use_less_memory_during_creation=" << options.use_less_memory_during_creation << "}"; } - virtual void execute_async(std::function f) = 0; - virtual void execute_sync(std::function f) = 0; }; + static std::unique_ptr create_in_memory(td::KeyValueReader *kv, CreateInMemoryOptions options); virtual void load_cell_async(td::Slice hash, std::shared_ptr executor, td::Promise> promise) = 0; diff --git a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp new file mode 100644 index 0000000000..45fdffee89 --- /dev/null +++ b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp @@ -0,0 +1,1505 @@ +#include "vm/db/DynamicBagOfCellsDb.h" +#include "vm/db/CellStorage.h" +#include "vm/db/CellHashTable.h" + +#include "vm/cells/ExtCell.h" + +#include "td/utils/base64.h" +#include "td/utils/format.h" +#include "td/utils/ThreadSafeCounter.h" +#include "td/utils/misc.h" +#include "validator/validator.h" + +#include "vm/cellslice.h" + +#include + +namespace vm { +namespace { + +// Very stupid Vector/MpmcQueue +template +struct TsVector { + TsVector() { + first_block_size_ = 64; + blocks_[0].data.resize(first_block_size_); + blocks_[0].is_ready = true; + } + TsVector(std::vector base) { + first_block_size_ = base.size(); + blocks_[0].data = std::move(base); + blocks_[0].is_ready = true; + } + struct Block { + std::mutex mutex; + std::atomic is_ready{false}; + std::vector data; + }; + T &at(size_t i) { + td::uint64 j = i / first_block_size_; + td::int32 hb = 63 - td::count_leading_zeroes64(j); // hb = -1 if j=0, else hb>=0 + + // If j=0, hb<0, so hb>>31 = -1 => mask=0 + // If j>0, hb>=0, so hb>>31=0 => mask=~0 (all ones) + td::uint64 mask = ~(td::uint64)(hb >> 31); + + size_t block_i = hb + 1; + uint64_t shift = hb & 63ULL; + uint64_t start = ((1ULL << shift) * first_block_size_) & mask; + size_t pos_in_block = i - start; + auto &block = blocks_[block_i]; + if (block.is_ready.load(std::memory_order_acquire)) { + return block.data.at(pos_in_block); + } + + std::unique_lock lock(block.mutex); + if (block.is_ready.load(std::memory_order_acquire)) { + return block.data.at(pos_in_block); + } + block.resize(start); + block.is_ready.store(true, std::memory_order_release); + return block.data.at(pos_in_block); + } + template + void push_back(S &&value) { + at(end_.fetch_add(1, std::memory_order_relaxed)) = std::forward(value); + } + T pop_front() { + auto pos = begin_.fetch_add(1, std::memory_order_relaxed); + while (pos >= end_.load(std::memory_order_acquire)) { + // This may (or may not) use too much CPU + td::this_thread::yield(); + } + return std::move(at(pos)); + } + size_t size() const { + return end_.load(); + } + + std::array blocks_; + size_t first_block_size_{0}; + std::atomic begin_{0}; + std::atomic end_{0}; +}; +struct CellInfo; + +class CellDbReaderExt; +struct DynamicBocExtCellExtra { + std::shared_ptr reader; +}; + +class DynamicBocCellLoader { + public: + static td::Result> load_data_cell(const ExtCell &cell, + const DynamicBocExtCellExtra &extra); +}; +using DynamicBocExtCell = ExtCell; + +class CellDbReaderExt : public CellDbReader { + public: + virtual td::Result> load_ext_cell(Ref cell) = 0; +}; + +td::Result> DynamicBocCellLoader::load_data_cell(const DynamicBocExtCell &cell, + const DynamicBocExtCellExtra &extra) { + return extra.reader->load_ext_cell(Ref(&cell)); +} + +#define S(x) \ + td::NamedThreadSafeCounter::CounterRef x { \ + nc.get_counter(#x) \ + } + +struct CacheStats { + td::NamedThreadSafeCounter nc; + S(load_cell_ext); + S(load_cell_ext_cache_hits); + S(load_cell_sync); + S(load_cell_sync_cache_hits); + S(load_cell_async); + S(load_cell_async_cache_hits); + S(ext_cells); + S(ext_cells_load); + S(ext_cells_load_cache_hits); + + S(kv_read_found); + S(kv_read_not_found); + + S(sync_with_db); + S(sync_with_db_only_ref); + S(load_cell_no_cache); +}; + +struct CommitStats { + td::NamedThreadSafeCounter nc; + + S(to_inc); + S(to_dec); + + S(gather_new_cells_calls); + S(gather_new_cells_calls_it); + S(update_parents_calls); + S(update_parents_calls_it); + S(dec_calls); + S(dec_calls_it); + + S(new_cells); + S(new_cells_leaves); + + S(new_cells_loaded_not_in_db); + S(new_cells_loaded_in_db); + S(new_cells_not_in_db_fast); + + S(dec_loaded); + S(dec_to_zero); + + S(changes_loaded); + + // new diff logic + S(diff_zero); + S(diff_full); + S(diff_erase); + S(diff_ref_cnt); + + // old full data logic + S(inc_save); + S(inc_save_full); + S(inc_save_only_ref_cnt); + S(inc_new_cell); + S(inc_just_ref_cnt); + + S(dec_save); + S(dec_save_full); + S(dec_save_only_refcnt); + S(dec_save_erase); + S(dec_erase_cell); + S(dec_just_ref_cnt); +}; + +template +struct AtomicPod { + T load() const { + while (true) { + if (auto res = try_read_stable()) { + return res->second; + } + } + } + + template + std::pair update(F &&f) { + while (true) { + auto res = try_read_stable(); + if (!res) { + continue; + } + auto [before, old_data] = *res; + + auto o_new_data = f(old_data); + if (!o_new_data) { + return {old_data, false}; + } + + if (!lock_.compare_exchange_weak(before, before + 1, std::memory_order_acq_rel, std::memory_order_relaxed)) { + continue; + } + + data_ = *o_new_data; // relaxed store inside lock + lock_.fetch_add(1, std::memory_order_release); + return {*o_new_data, true}; + } + } + + private: + mutable std::atomic lock_{0}; + T data_{}; + + std::optional> try_read_stable() const { + auto before = lock_.load(std::memory_order_acquire); + if (before % 2 == 1) { + return std::nullopt; + } + T temp = data_; // relaxed read is ok, checked by versioning + auto after = lock_.load(std::memory_order_acquire); + if (after != before) { + return std::nullopt; + } + return std::make_pair(before, temp); + } +}; + +struct InDbInfo { + std::vector parents; + std::atomic pending_children{0}; + std::atomic maybe_in_db{true}; + std::atomic visited_in_gather_new_cells{false}; +}; +td::StringBuilder &operator<<(td::StringBuilder &sb, const InDbInfo &info) { + sb << "mb_in_db:" << info.maybe_in_db.load() << " chld_n:" << info.pending_children + << " prnt_n:" << info.parents.size(); + return sb; +} +struct CellInfo { + struct State { + // db_ref_cnt and in_db are correct + bool sync_with_db{false}; + + // ignore if sync_with_db is false + td::int32 db_ref_cnt{0}; + td::int32 db_refcnt_fixup{0}; + + // if true - cell is definitely in db + // if false - we know that cell is not in db only is sync_with_db=true + bool in_db{false}; + + // diff to be applied + }; + AtomicPod state; + std::atomic ref_cnt_diff{0}; + + std::atomic visited{false}; + td::unique_ptr in_db_info_ptr; + std::mutex mutex; + + // Could be AtomicRef, but is am not sure that it is worth it + const Ref cell; + + explicit CellInfo(Ref cell) : cell(std::move(cell)) { + } + + InDbInfo &in_db_info() { + return *in_db_info_ptr; + } + const InDbInfo &in_db_info() const { + return *in_db_info_ptr; + } + InDbInfo &in_db_info_create() { // NOT thread safe + if (!in_db_info_ptr) { + in_db_info_ptr = td::make_unique(); + } + return in_db_info(); + } + InDbInfo &in_db_info_create(CellInfo *parent) { // Thread Safe + std::unique_lock lock(mutex); + if (!in_db_info_ptr) { + in_db_info_ptr = td::make_unique(); + } + auto &res = *in_db_info_ptr; + if (parent != nullptr) { + res.parents.emplace_back(parent); + } + lock.unlock(); + return res; + } + void in_db_info_destroy() { + in_db_info_ptr = nullptr; + } + td::int32 inc_ref_cnt() { + return ref_cnt_diff.fetch_add(1, std::memory_order_relaxed) + 1; + } + td::int32 dec_ref_cnt() { + return ref_cnt_diff.fetch_sub(1, std::memory_order_relaxed) - 1; + } + td::int32 get_ref_cnt_diff() const { + return ref_cnt_diff.load(std::memory_order_relaxed); + } + + void set_not_in_db() { + state.update([&](State state) -> std::optional { + if (state.sync_with_db) { + CHECK(state.db_ref_cnt == 0); + CHECK(!state.in_db); + return {}; + } + state.sync_with_db = true; + state.in_db = false; + state.db_ref_cnt = 0; + return state; + }); + } + void set_in_db() { + state.update([&](State state) -> std::optional { + if (state.sync_with_db) { + //LOG_CHECK(state.in_db) << *this; + return {}; + } + state.in_db = true; + return state; + }); + } + void synced_with_db(td::int32 db_ref_cnt) { + state.update([&](State state) -> std::optional { + if (state.sync_with_db) { + CHECK(state.in_db); + CHECK(state.db_ref_cnt == db_ref_cnt); + return {}; + } + state.in_db = true; + state.db_ref_cnt = db_ref_cnt; + return state; + }); + } + bool visit() { + return !visited.exchange(true); + } + void on_written_to_db() { + auto diff = ref_cnt_diff.exchange(0); + state.update([&](State state) -> std::optional { + if (diff == 0) { + return {}; + } + if (state.sync_with_db) { + state.db_ref_cnt += diff; + CHECK(state.db_ref_cnt >= 0); + state.in_db = state.db_ref_cnt > 0; + } else { + CHECK(diff > 0); + state.in_db = true; + state.db_refcnt_fixup += diff; + } + return state; + }); + } + + td::Result> get_data_cell() { + TRY_RESULT(loaded_cell, cell->load_cell()); + return loaded_cell.data_cell; + } + Cell::Hash key() const { + return cell->get_hash(); + } + bool operator<(const CellInfo &other) const { + return key() < other.key(); + } + + struct Eq { + using is_transparent = void; // Pred to use + bool operator()(const CellInfo &info, const CellInfo &other_info) const { + return info.key() == other_info.key(); + } + bool operator()(const CellInfo &info, td::Slice hash) const { + return info.key().as_slice() == hash; + } + bool operator()(td::Slice hash, const CellInfo &info) const { + return info.key().as_slice() == hash; + } + }; + struct Hash { + using is_transparent = void; // Pred to use + using transparent_key_equal = Eq; + size_t operator()(td::Slice hash) const { + return cell_hash_slice_hash(hash); + } + size_t operator()(const CellInfo &info) const { + return cell_hash_slice_hash(info.key().as_slice()); + } + }; +}; +td::StringBuilder &operator<<(td::StringBuilder &sb, const CellInfo &info) { + if (info.cell->is_loaded()) { + auto data_cell = info.cell->load_cell().move_as_ok().data_cell; + vm::CellSlice cs(vm::NoVm{}, data_cell); + sb << data_cell->get_hash().to_hex().substr(0, 8) << " refs:" << data_cell->size_refs() + << " data:" << cs.data_bits().to_hex(cs.size()) << " data_ptr=" << data_cell.get() << " data_ref_cnt(" + << data_cell->get_refcnt() << ")"; + } else { + sb << info.cell->get_hash().to_hex().substr(0, 8); + } + auto state = info.state.load(); + sb << " " << &info; + sb << "\n\tin_db=" << state.in_db << " sync_with_db=" << state.sync_with_db + << " ref_cnt_diff=" << info.get_ref_cnt_diff() << " db_ref_cnt=" << state.db_ref_cnt + << " db_ref_cnt_fixup=" << state.db_refcnt_fixup; + if (state.sync_with_db) { + sb << " REFS(" << info.get_ref_cnt_diff() + state.db_ref_cnt << ")"; + } + if (info.in_db_info_ptr) { + sb << " " << info.in_db_info(); + } + sb << " visited=" << info.visited.load(); + return sb; +} + +struct ExecutorOptions { + size_t extra_threads_n{0}; + std::shared_ptr async_executor; +}; +template +class ExecutorImpl { + public: + ExecutorImpl(ExecutorOptions options) : options_(options) { + } + ExecutorOptions options_; + using InputData = std::vector>; + using OutputData = std::vector>; + struct InputChunk { + td::Span infos; + size_t begin{}; + size_t end{}; + }; + + template + OutputData process(const InputData &data, const F &process_task_f) { + if (options_.extra_threads_n > 0) { + return process_parallel(data, process_task_f); + } else { + return process_sequential(data, process_task_f); + } + } + template + struct SingleThreadWorker { + const F &process_task_f; + mutable std::vector results{}; + void add_task(InputT input) const { + process_task_f(input, *this); + } + void add_result(OutputT output) const { + results.push_back(output); + } + }; + template + OutputData process_sequential(const InputData &data, const F &process_task_f) { + auto w = SingleThreadWorker{process_task_f}; + for (auto &chunk : data) { + for (auto &info : chunk) { + process_task_f(info, w); + } + } + + return {std::move(w.results)}; + } + + template + struct Shared; + + template + struct Worker { + size_t worker_i{}; + std::shared_ptr> shared; + + void add_task(InputT input) const { + shared->delay_or_process_task(input, *this); + } + void add_result(OutputT value) const { + shared->add_result(value, worker_i); + } + void loop() const { + shared->loop(*this); + } + }; + + template + struct Shared { + Shared(size_t workers_n, const InputData &input_data, const ProcessTaskF &process_task_f) + : input_chunks(prepare_input_chunks(input_data)) + , workers_n(workers_n) + , input_size(input_chunks.empty() ? 0 : input_chunks.back().end) + , batch_size(std::clamp(input_size / workers_n / 4, size_t(1), size_t(128))) + , process_task_f(process_task_f) { + } + + const std::vector input_chunks; + + const size_t workers_n{0}; + const size_t input_size{0}; + const size_t batch_size{128}; + + const ProcessTaskF &process_task_f; + + // Position in input + std::atomic next_input_i{0}; + + // Shared queue + // Probably a simpler queue would also work fine + td::MpmcQueue mpmc_queue{workers_n}; + using Waiter = td::MpmcSleepyWaiter; + Waiter waiter; + std::atomic mpmc_queue_size{workers_n}; // guard + + // Output vectors + struct ThreadData { + std::vector output; + char pad[TD_CONCURRENCY_PAD - sizeof(output)]; + }; + std::vector thread_data{workers_n}; + + auto prepare_input_chunks(const InputData &input_data) { + std::vector chunks; + for (auto &chunk : input_data) { + size_t prev_end = chunks.empty() ? 0 : chunks.back().end; + chunks.push_back({.infos = td::as_span(chunk), .begin = prev_end, .end = prev_end + chunk.size()}); + } + return chunks; + } + + void delay_or_process_task(InputT input, const Worker &worker) { + // if there is enough tasks in queue, we continue recursion + if (mpmc_queue_size.load(std::memory_order_acquire) > 256) { + process_task_f(input, worker); + } else { + mpmc_queue_size.fetch_add(1, std::memory_order_acq_rel); + mpmc_queue.push(input, worker.worker_i); + waiter.notify(); + } + } + + void add_result(OutputT result, size_t worker_i) { + thread_data[worker_i].output.push_back(std::move(result)); + } + + void process_initial_input(const Worker &worker) { + size_t input_chunk_i = 0; + while (true) { + auto begin_i = next_input_i.fetch_add(batch_size, std::memory_order_relaxed); + auto end_i = begin_i + batch_size; + if (begin_i >= input_size) { + break; + } + for (size_t i = begin_i; i < end_i && i < input_size; i++) { + while (input_chunks[input_chunk_i].end <= i) { + input_chunk_i++; + } + auto offset = i - input_chunks[input_chunk_i].begin; + auto task = input_chunks[input_chunk_i].infos[offset]; + process_task_f(task, worker); + } + } + } + + void on_processed_task_from_queue(size_t worker_i) { + if (mpmc_queue_size.fetch_add(-1, std::memory_order_acq_rel) == 1) { + for (size_t i = 0; i < workers_n; i++) { + mpmc_queue.push(nullptr, worker_i); + waiter.notify(); + } + } + } + + void process_queue(const Worker &worker) { + on_processed_task_from_queue(worker.worker_i); + + Waiter::Slot slot; + waiter.init_slot(slot, td::narrow_cast(worker.worker_i)); + + while (true) { + InputT input{}; + if (mpmc_queue.try_pop(input, worker.worker_i)) { + waiter.stop_wait(slot); + if (!input) { + break; + } + process_task_f(input, worker); + on_processed_task_from_queue(worker.worker_i); + } else { + waiter.wait(slot); + } + } + } + void loop(const Worker &worker) { + process_initial_input(worker); + process_queue(worker); + } + void finish() const { + CHECK(mpmc_queue_size == 0); + } + }; + + template + OutputData process_parallel(const InputData &input_data, const F &process_task_f) { + const size_t workers_n = options_.extra_threads_n + 1; + auto shared = std::make_shared>(workers_n, input_data, process_task_f); + + CHECK(workers_n >= 1); + for (size_t i = 0; i < workers_n; i++) { + auto to_run = [worker = Worker{.worker_i = i, .shared = shared}] { worker.loop(); }; + + if (i + 1 == workers_n) { + to_run(); + } else if (options_.async_executor) { + options_.async_executor->execute_async(std::move(to_run)); + } else { + // NB: td::thread, NOT std::thread + td::thread(std::move(to_run)).detach(); + } + } + shared->finish(); + return td::transform(shared->thread_data, [](auto &&x) { return std::move(x.output); }); + } +}; +struct Executor { + Executor(ExecutorOptions options = {}) : options_(options) { + } + + template + auto operator()(const std::vector> &data, const F &process_task_f) { + return ExecutorImpl(options_).process(data, process_task_f); + } + + private: + ExecutorOptions options_; +}; + +// Thread safe storage for CellInfo +// Will be used by everybody as shared cache. Yes there is some overhead, but it don't want to create other hash table +struct CellInfoStorage { + public: + // All methods are thead safe + // All CellInfo pointers lives as long as CellInfoStorage + + // returns CellInfo, only if it is already exists + CellInfo *get_cell_info(td::Slice hash) { + return lock(hash)->hash_table.get_if_exists(hash); + } + + CellInfo &create_cell_info_from_db(Ref data_cell, td::int32 ref_cnt) { + auto &info = create_cell_info_from_data_cell(std::move(data_cell)); + info.synced_with_db(ref_cnt); + return info; + } + + // Creates CellInfo from data_cell, or updates existing CellInfo if is not yet loaded + CellInfo &create_cell_info_from_data_cell(Ref cell) { + CHECK(cell.not_null()); + CHECK(cell->is_loaded()); + + auto hash = cell->get_hash(); + auto [info, created] = lock(hash.as_slice())->hash_table.emplace(hash.as_slice(), std::move(cell)); + + if (!created) { + info.cell->set_data_cell(std::move(cell)); + } + return info; + } + + // Creates CellInfo from cell. If cell is loaded, it will be used to rewrite or udpate current cell + CellInfo &create_cell_info(Ref cell, CellDbReaderExt *from_reader, CacheStats &stats) { + if (cell->is_loaded()) { + return create_cell_info_from_data_cell(cell->load_cell().move_as_ok().data_cell); + } + + bool our_ext_cell = false; + auto ext_cell = dynamic_cast(cell.get()); + if (ext_cell) { + auto prunned_cell = ext_cell->get_prunned_cell(); + if (prunned_cell.not_null()) { + our_ext_cell = prunned_cell->get_extra().reader.get() == from_reader; + } + our_ext_cell = true; + } else if (!cell->is_loaded()) { + // if we cached cell from OTHER db is good idea to drop it ASAP + force_drop_cache_.store(true, std::memory_order_relaxed); + } + + auto hash = cell->get_hash(); + auto [info, created] = lock(hash.as_slice())->hash_table.emplace(hash.as_slice(), std::move(cell)); + if (our_ext_cell) { + stats.ext_cells_load.inc(); + if (info.cell->is_loaded()) { + stats.ext_cells_load_cache_hits.inc(); + } + info.set_in_db(); + } + return info; + } + + void dump() { + LOG(ERROR) << "===========BEGIN DUMP==========="; + for (auto &bucket : buckets_) { + std::lock_guard guard(bucket.mutex); + bucket.hash_table.for_each([&](auto &info) { LOG(INFO) << info; }); + } + LOG(ERROR) << "===========END DUMP==========="; + } + + size_t cache_size() { + size_t res = 0; + for (auto &bucket : buckets_) { + std::lock_guard guard(bucket.mutex); + res += bucket.hash_table.size(); + } + return res; + } + bool force_drop_cache() { + return force_drop_cache_.load(std::memory_order_relaxed); + } + + private: + struct Bucket { + std::mutex mutex; + CellHashTable hash_table; + }; + constexpr static size_t buckets_n = 8192; + std::array bucket_; + + struct Unlock { + void operator()(Bucket *bucket) const { + bucket->mutex.unlock(); + } + }; + std::array buckets_{}; + std::atomic force_drop_cache_{false}; + + std::unique_ptr lock(Bucket &bucket) { + bucket.mutex.lock(); + return std::unique_ptr(&bucket); + } + std::unique_ptr lock(td::Slice key) { + auto hash = td::as(key.substr(16, 8).ubegin()); + auto bucket_i = hash % buckets_n; + return lock(buckets_[bucket_i]); + } +}; + +class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { + public: + explicit DynamicBagOfCellsDbImplV2(CreateV2Options options) : options_(options) { + get_thread_safe_counter().inc(); + // LOG(ERROR) << "Constructor called for DynamicBagOfCellsDbImplV2"; + } + ~DynamicBagOfCellsDbImplV2() { + // LOG(ERROR) << "Destructor called for DynamicBagOfCellsDbImplV2"; + get_thread_safe_counter().add(-1); + + if (cell_db_reader_) { + cell_db_reader_->drop_cache(); + } + } + + td::Result>> meta_get_all() const override { + CHECK(meta_db_fixup_.empty()); + std::vector> result; + auto s = cell_db_reader_->key_value_reader().for_each_in_range( + "desc", "desd", [&](const td::Slice &key, const td::Slice &value) { + if (td::begins_with(key, "desc") && key.size() != 32) { + result.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + TRY_STATUS(std::move(s)); + return result; + } + td::Result meta_get(td::Slice key, std::string &value) override { + auto it = meta_db_fixup_.find(key); + if (it != meta_db_fixup_.end()) { + if (it->second.empty()) { + return KeyValue::GetStatus::NotFound; + } + value = it->second; + return KeyValue::GetStatus::Ok; + } + return cell_db_reader_->key_value_reader().get(key, value); + } + td::Status meta_set(td::Slice key, td::Slice value) override { + meta_diffs_.push_back( + CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Set, .key = key.str(), .value = value.str()}); + return td::Status::OK(); + } + td::Status meta_erase(td::Slice key) override { + meta_diffs_.push_back(CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Erase, .key = key.str()}); + return td::Status::OK(); + } + td::Result> load_cell(td::Slice hash) override { + CHECK(cell_db_reader_); + return cell_db_reader_->load_cell(hash); + } + td::Result> load_root(td::Slice hash) override { + return load_cell(hash); + } + td::Result> load_root_thread_safe(td::Slice hash) const override { + // TODO: it is better to use AtomicRef, or atomic shared pointer + // But to use AtomicRef we need a little refactoring + // And std::atomic> is still unsupported by clang + std::unique_lock lock(atomic_cell_db_reader_mutex_); + auto reader = atomic_cell_db_reader_; + lock.unlock(); + if (!reader) { + return td::Status::Error("Empty reader"); + } + return reader->load_cell(hash); + } + void load_cell_async(td::Slice hash, std::shared_ptr executor, + td::Promise> promise) override { + CHECK(cell_db_reader_); + return cell_db_reader_->load_cell_async(hash, std::move(executor), std::move(promise)); + } + void prepare_commit_async(std::shared_ptr executor, td::Promise promise) override { + auto promise_ptr = std::make_shared>(std::move(promise)); + executor->execute_async([this, promise_ptr = std::move(promise_ptr)] { + prepare_commit(); + promise_ptr->set_value(td::Unit()); + }); + } + + void inc(const Ref &cell) override { + if (cell.is_null()) { + return; + } + if (cell->get_virtualization() != 0) { + return; + } + to_inc_.push_back(cell); + } + void dec(const Ref &cell) override { + if (cell.is_null()) { + return; + } + if (cell->get_virtualization() != 0) { + return; + } + to_dec_.push_back(cell); + } + + bool is_prepared_for_commit() { + return to_inc_.empty() && to_dec_.empty(); + } + + Stats get_stats_diff() override { + return {}; + } + + td::Status prepare_commit() override { + if (is_prepared_for_commit()) { + return td::Status::OK(); + } + // NB: we don't use options.executor, because it is prone to deadlocks. We need extra_threads_n threads + // available for blocking + Executor executor{{.extra_threads_n = options_.extra_threads, .async_executor = {}}}; + // calculate in_db for all vertices reachable from to_inc_ roots + // - for ext cells we already know they are in db + // - calculate in_db up from leaves + // - if at least one child is not in db, then the cell is definitely not in db + // - so in best case only leaves will be loaded from db + // - this is optional step. All other logic must work in any case + // - only already loaded cells are loaded from db + + stats_.to_inc.add(to_inc_.size()); + stats_.to_dec.add(to_dec_.size()); + + std::vector> visited_cells; + auto add_visited_cells = [&](std::vector> new_visited_cells) { + for (auto &x : new_visited_cells) { + visited_cells.push_back(std::move(x)); + } + }; + + std::vector> new_cells_leaves; + { + td::PerfWarningTimer timer("celldb_v2: gather_new_cells"); + std::vector prepared_to_inc; + std::vector visited_roots; + for (auto &cell : to_inc_) { + auto &info = cell_db_reader_->cell_info(cell); + if (info.inc_ref_cnt() == 1 && info.visit()) { + visited_roots.push_back(&info); + } + if (info.state.load().in_db) { + continue; + } + auto &in_db_info = info.in_db_info_create(nullptr); + if (!in_db_info.visited_in_gather_new_cells.exchange(true)) { + prepared_to_inc.push_back(&info); + } + } + new_cells_leaves = + executor({std::move(prepared_to_inc)}, [&](CellInfo *info, auto &worker) { gather_new_cells(info, worker); }); + visited_cells.push_back(std::move(visited_roots)); + } + + // LOG(WARNING) << "new_cells_leaves: " << new_cells_leaves.size(); + { + td::PerfWarningTimer timer("celldb_v2: update_parents"); + add_visited_cells( + executor({std::move(new_cells_leaves)}, [&](CellInfo *info, auto &worker) { update_parents(info, worker); })); + } + { + td::PerfWarningTimer timer("dec"); + std::vector prepared_to_dec; + for (auto &cell : to_dec_) { + auto &info = cell_db_reader_->cell_info(cell); + prepared_to_dec.push_back(&info); + } + add_visited_cells( + executor({std::move(prepared_to_dec)}, [&](CellInfo *info, auto &worker) { dec_cell(info, worker); })); + } + + td::PerfWarningTimer timer_serialize("celldb_v2: save_diff_serialize", 0.01); + // LOG(INFO) << "threads_n = " << options_.extra_threads + 1; + diff_chunks_ = executor.operator()( + visited_cells, [&](CellInfo *info, auto &worker) { serialize_diff(info, worker); }); + timer_serialize.reset(); + + { + td::PerfWarningTimer timer("celldb_v2: clear"); + to_inc_.clear(); + to_dec_.clear(); + } + + //cell_db_reader_->dump(); + return td::Status::OK(); + } + + td::Status commit(CellStorer &storer) override { + prepare_commit(); + save_diff(storer); + // We DON'T delete entries from cache, so cache actually represents diff with snapshot in reader + // But we don't want took keep old snapshot forever + LOG_IF(ERROR, dbg) << "clear cell_db_reader"; + //cell_db_reader_->dump(); + //TODO: call drop_cache reliably via rtti + + constexpr bool always_drop_cache = false; + if (always_drop_cache) { + td::PerfWarningTimer timer("celldb_v2: reset reader"); + cell_db_reader_->drop_cache(); + cache_stats_.apply_diff(cell_db_reader_->get_stats()); + cache_stats_.stats_int["commits"] += 1; + cell_db_reader_ = {}; + // keep atomic reader, to it will be reused + } + return td::Status::OK(); + } + + std::shared_ptr get_cell_db_reader() override { + CHECK(cell_db_reader_); + return cell_db_reader_; + } + + td::Status set_loader(std::unique_ptr loader) override { + if (cell_db_reader_) { + auto cache_size = cell_db_reader_->cache_size(); + bool force_drop_cache = cell_db_reader_->force_drop_cache(); + if (loader && cache_size < options_.cache_size_max && cell_db_reader_ttl_ < options_.cache_ttl_max && + !force_drop_cache) { + // keep cache + cell_db_reader_ttl_++; + return td::Status::OK(); + } + + td::PerfWarningTimer timer(PSTRING() << "celldb_v2: reset reader, TTL=" << cell_db_reader_ttl_ << "/" + << options_.cache_ttl_max << ", cache_size=" << cache_size + << ", force_drop_cache=" << force_drop_cache); + cache_stats_.apply_diff(cell_db_reader_->get_stats()); + cell_db_reader_->drop_cache(); + cell_db_reader_ = {}; + meta_db_fixup_ = {}; + cell_db_reader_ttl_ = 0; + } + + if (loader) { + cell_db_reader_ = std::make_shared(std::move(loader)); + cell_db_reader_ttl_ = 0; + } + + { + std::lock_guard guard(atomic_cell_db_reader_mutex_); + atomic_cell_db_reader_ = cell_db_reader_; + } + return td::Status::OK(); + } + + void set_celldb_compress_depth(td::uint32 value) override { + CHECK(value == 0); + } + + vm::ExtCellCreator &as_ext_cell_creator() override { + CHECK(cell_db_reader_); + return *cell_db_reader_; + } + td::Result get_stats() override { + auto ps = stats_.nc.get_stats().with_prefix("storage_"); + ps.apply_diff(cache_stats_.with_prefix("cache_cum_")); + if (cell_db_reader_) { + ps.apply_diff(cell_db_reader_->get_stats().with_prefix("cache_now_")); + ps.apply_diff(cell_db_reader_->get_stats().with_prefix("cache_cum_")); + } + Stats res; + res.named_stats = std::move(ps); + res.named_stats.stats_int["cache.size"] = cell_db_reader_ ? cell_db_reader_->cache_size() : 0; + res.named_stats.stats_int["cache.size_max"] = options_.cache_size_max; + res.named_stats.stats_int["cache.ttl"] = cell_db_reader_ttl_; + res.named_stats.stats_int["cache.ttl_max"] = options_.cache_ttl_max; + return res; + } + + private: + static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() { + static auto res = td::NamedThreadSafeCounter::get_default().get_counter("DynamicBagOfCellsDb"); + return res; + } + + class CellDbReaderImpl : public CellDbReaderExt, + public ExtCellCreator, + public std::enable_shared_from_this { + public: + explicit CellDbReaderImpl(std::unique_ptr cell_loader) : cell_loader_(std::move(cell_loader)) { + } + + size_t cache_size() const { + // NOT thread safe + if (internal_storage_) { + return internal_storage_->cache_size(); + } + return 0; + } + bool force_drop_cache() const { + // NOT thread safe + if (internal_storage_) { + return internal_storage_->force_drop_cache(); + } + return false; + } + void drop_cache() { + // NOT thread safe + internal_storage_.reset(); + } + + td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { + // thread safe function + stats_.ext_cells.inc(); + TRY_RESULT(ext_cell, DynamicBocExtCell::create(PrunnedCellInfo{level_mask, hash, depth}, + DynamicBocExtCellExtra{shared_from_this()})); + + return ext_cell; + } + CellInfo *register_ext_cell_inner(Ref ext_cell, CellInfoStorage &storage) { + auto &info = storage.create_cell_info(std::move(ext_cell), this, stats_); + return &info; + } + + void load_cell_async(td::Slice hash, std::shared_ptr executor, td::Promise> promise) { + // thread safe function + stats_.load_cell_async.inc(); + auto maybe_cell = load_cell_fast_path(hash, false, nullptr); + if (maybe_cell.not_null()) { + stats_.load_cell_async_cache_hits.inc(); + return promise.set_value(std::move(maybe_cell)); + } + auto promise_ptr = std::make_shared>>(std::move(promise)); + + executor->execute_async( + [self = shared_from_this(), promise_ptr = std::move(promise_ptr), hash = CellHash::from_slice(hash)]() { + promise_ptr->set_result(self->load_cell(hash.as_slice())); + }); + } + + td::Result> load_cell(td::Slice hash) override { + // thread safe function + stats_.load_cell_sync.inc(); + bool loaded{false}; + auto maybe_cell = load_cell_fast_path(hash, true, &loaded); + if (maybe_cell.not_null()) { + if (!loaded) { + stats_.load_cell_sync_cache_hits.inc(); + } + return maybe_cell; + } + return load_cell_slow_path(hash); + } + + td::Result> load_ext_cell(Ref ext_cell) override { + // thread safe function. + // Called by external cell + stats_.load_cell_ext.inc(); + auto storage = weak_storage_.lock(); + if (!storage) { + TRY_RESULT(load_result, load_cell_no_cache(ext_cell->get_hash().as_slice())); + return load_result.cell_; + } + // we delayed registering ext cell till this moment + auto cell_info = register_ext_cell_inner(std::move(ext_cell), *storage); + + CHECK(cell_info != nullptr); // currently all ext_cells are registered in cache + if (!cell_info->cell->is_loaded()) { + sync_with_db(*cell_info, true); + CHECK(cell_info->cell->is_loaded()); // critical, better to fail + } else { + stats_.load_cell_ext_cache_hits.inc(); + } + return cell_info->cell->load_cell().move_as_ok().data_cell; + } + + CellInfo &cell_info(Ref cell) { + // thread safe function, but called only by DB + CHECK(internal_storage_) + return internal_storage_->create_cell_info(std::move(cell), this, stats_); + } + + std::pair sync_with_db(CellInfo &info, bool need_data) { + // thread safe function, but called only by DB + auto effective_need_data = need_data; + if (info.cell->is_loaded()) { + effective_need_data = false; + } + return info.state.update([&](CellInfo::State state) -> std::optional { + if (state.sync_with_db) { + return {}; + } + stats_.sync_with_db.inc(); + if (!effective_need_data) { + stats_.sync_with_db_only_ref.inc(); + } + auto load_result = + cell_loader_->load(info.cell->get_hash().as_slice(), effective_need_data, *this).move_as_ok(); + + state.sync_with_db = true; + if (load_result.status == CellLoader::LoadResult::NotFound) { + CHECK(state.in_db == false); + CHECK(state.db_ref_cnt == 0); + stats_.kv_read_not_found.inc(); + return state; + } + stats_.kv_read_found.inc(); + + state.in_db = true; + state.db_ref_cnt = load_result.refcnt() + state.db_refcnt_fixup; + if (load_result.cell().not_null()) { + info.cell->set_data_cell(std::move(load_result.cell())); + } + CHECK(!need_data || info.cell->is_loaded()); + return state; + }); + } + + void dump() { + internal_storage_->dump(); + } + + td::NamedStats get_stats() const { + return stats_.nc.get_stats(); + } + td::KeyValueReader &key_value_reader() { + return cell_loader_->key_value_reader(); + } + + private: + static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() { + static auto res = td::NamedThreadSafeCounter::get_default().get_counter("DynamicBagOfCellsDbLoader"); + return res; + } + std::shared_ptr internal_storage_{std::make_shared()}; + std::weak_ptr weak_storage_{internal_storage_}; + std::unique_ptr cell_loader_; + CacheStats stats_; + + Ref load_cell_fast_path(td::Slice hash, bool may_block, bool *loaded) { + auto storage = weak_storage_.lock(); + if (!storage) { + return {}; + } + auto cell_info = storage->get_cell_info(hash); + if (cell_info != nullptr) { + if (!cell_info->cell->is_loaded()) { + if (may_block) { + if (loaded) { + *loaded = true; + } + CHECK(cell_info->state.load().in_db); + sync_with_db(*cell_info, true); + CHECK(cell_info->cell->is_loaded()); + } else { + return {}; + } + } + return cell_info->cell->load_cell().move_as_ok().data_cell; + } + return {}; + } + td::Result load_cell_no_cache(td::Slice hash) { + stats_.load_cell_no_cache.inc(); + TRY_RESULT(load_result, cell_loader_->load(hash, true, *this)); + if (load_result.status == CellLoader::LoadResult::NotFound) { + stats_.kv_read_not_found.inc(); + return td::Status::Error("Cell load failed: not in db"); + } + stats_.kv_read_found.inc(); + return load_result; + } + td::Result> load_cell_slow_path(td::Slice hash) { + TRY_RESULT(load_result, load_cell_no_cache(hash)); + auto storage = weak_storage_.lock(); + if (!storage) { + return load_result.cell_; + } + auto &cell_info = storage->create_cell_info_from_db(std::move(load_result.cell()), load_result.refcnt()); + return cell_info.cell->load_cell().move_as_ok().data_cell; + } + }; + + CreateV2Options options_; + std::vector> to_inc_; + std::vector> to_dec_; + std::vector> diff_chunks_; + std::vector meta_diffs_; + std::map> meta_db_fixup_; + + mutable std::mutex atomic_cell_db_reader_mutex_; + std::shared_ptr atomic_cell_db_reader_; + + std::shared_ptr cell_db_reader_; + size_t cell_db_reader_ttl_{0}; + td::NamedStats cache_stats_; + CommitStats stats_; + bool dbg{false}; + + template + void gather_new_cells(CellInfo *info, WorkerT &worker) { + stats_.gather_new_cells_calls.inc(); + do { + // invariant: info is not in DB; with created in_db_info + // we enter into each root only once + stats_.gather_new_cells_calls_it.inc(); + stats_.new_cells.inc(); + auto &in_db_info = info->in_db_info(); + + CellSlice cs(vm::NoVm{}, info->cell); // ensure cell is loaded + CellInfo *prev_child_info = nullptr; + while (cs.have_refs()) { + auto *child_info = &cell_db_reader_->cell_info(cs.fetch_ref()); + auto child_state = child_info->state.load(); + + if (child_state.in_db) { + LOG_IF(INFO, dbg) << "gather_new_cells: IN DB\n\tchld: " << *child_info; + continue; + } + + auto &child_in_db_info = child_info->in_db_info_create(info); + in_db_info.pending_children.fetch_add(1, std::memory_order_relaxed); + + if (child_in_db_info.visited_in_gather_new_cells.exchange(true)) { + continue; + } + + if (prev_child_info != nullptr) { + worker.add_task(prev_child_info); + } + prev_child_info = child_info; + } + LOG_IF(INFO, dbg) << "gather_new_cells: NOT IN DB\n\t" << *info; + if (in_db_info.pending_children.load(std::memory_order_relaxed) == 0) { + worker.add_result(info); + stats_.new_cells_leaves.inc(); + LOG_IF(WARNING, dbg) << "gather_new_cells: ADD LEAVE\n\t" << *info; + } + info = prev_child_info; + } while (info != nullptr); + } + + template + void update_parents(CellInfo *info, const WorkerT &worker) { + stats_.update_parents_calls.inc(); + size_t it = 0; + do { + stats_.update_parents_calls_it.inc(); + it++; + //LOG(INFO) << "update_parents: it=" << it << "\n\t"; + auto &in_db_info = info->in_db_info(); + bool in_db = false; + if (in_db_info.maybe_in_db.load(std::memory_order_relaxed)) { + auto [state, loaded] = cell_db_reader_->sync_with_db(*info, false); + in_db = state.in_db; + if (in_db) { + stats_.new_cells_loaded_in_db.inc(); + } else { + stats_.new_cells_loaded_not_in_db.inc(); + } + } else { + stats_.new_cells_not_in_db_fast.inc(); + info->set_not_in_db(); + } + LOG_IF(INFO, dbg) << "update_parents: it=" << it << "\n\t" << *info; + + CellInfo *prev_parent{nullptr}; + for (auto &parent : in_db_info.parents) { + auto &parent_in_db_info = parent->in_db_info(); + if (!in_db) { + parent_in_db_info.maybe_in_db.store(false, std::memory_order_relaxed); + } + if (parent_in_db_info.pending_children.fetch_sub(1, std::memory_order_release) == 1) { + if (prev_parent) { + worker.add_task(prev_parent); + } + prev_parent = parent; + } + } + if (!in_db) { + CellSlice cs(vm::NoVm{}, info->cell); + while (cs.have_refs()) { + auto child = cs.fetch_ref(); + auto &child_info = cell_db_reader_->cell_info(std::move(child)); + if (child_info.inc_ref_cnt() == 1 && child_info.visit()) { + worker.add_result(&child_info); + } + } + } + info->in_db_info_destroy(); + info = prev_parent; + } while (info); + } + + template + void dec_cell(CellInfo *info, WorkerT &worker) { + stats_.dec_calls.inc(); + + while (true) { + stats_.dec_calls_it.inc(); + if (info->visit()) { + worker.add_result(info); + } + auto ref_cnt_diff = info->dec_ref_cnt(); + if (ref_cnt_diff > 0) { + LOG_IF(INFO, dbg) << "NOT DEC" + << "\n\t" << info; + break; + } + auto state = info->state.load(); + if (ref_cnt_diff == 0 && state.in_db) { + LOG_IF(INFO, dbg) << "NOT DEC (in_db) " + << "\n\t" << info; + break; + } + if (!state.sync_with_db) { + state = cell_db_reader_->sync_with_db(*info, true).first; + stats_.dec_loaded.inc(); + CHECK(ref_cnt_diff == 0 || state.in_db); + } + auto ref_cnt = state.db_ref_cnt + ref_cnt_diff; + if (ref_cnt > 0) { + LOG_IF(INFO, dbg) << "DEC " << ref_cnt << "\n\t" << info; + } else { + LOG_IF(ERROR, dbg) << "DEC " << ref_cnt << "\n\t" << info; + } + CHECK(ref_cnt >= 0); + if (ref_cnt > 0) { + break; + } + stats_.dec_to_zero.inc(); + CellSlice cs(vm::NoVm{}, info->cell); + if (!cs.have_refs()) { + break; + } + while (cs.size_refs() > 1) { + worker.add_task(&cell_db_reader_->cell_info(cs.fetch_ref())); + } + info = &cell_db_reader_->cell_info(cs.fetch_ref()); + } + } + + template + void serialize_diff(CellInfo *info, Worker &worker) { + info->visited.store(false, std::memory_order_relaxed); + auto ref_cnt_diff = info->get_ref_cnt_diff(); + if (ref_cnt_diff == 0) { + stats_.diff_zero.inc(); + return; + } + + bool merge_supported = true; + if (merge_supported) { + auto state = info->state.load(); + if (ref_cnt_diff < 0) { + CHECK(state.sync_with_db); + /* + if (state.db_ref_cnt + ref_cnt_diff == 0) { + LOG(ERROR) << "DEC ERASE " << info->cell->get_hash().to_hex(); + } else { + LOG(ERROR) << "DEC MERGE " << info->cell->get_hash().to_hex() << *info; + } + */ + } + if (ref_cnt_diff < 0 && state.sync_with_db && state.db_ref_cnt + ref_cnt_diff == 0) { + // Erase is better than Merge+CompactionFilter + // So I see no reason for CompactionFilter at all + worker.add_result({.type = CellStorer::Diff::Erase, .key = info->cell->get_hash()}); + stats_.diff_erase.inc(); + } else { + bool with_data = ref_cnt_diff > 0 && !state.in_db; + if (with_data) { + CHECK(state.sync_with_db); + auto data_cell = info->cell->load_cell().move_as_ok().data_cell; + stats_.diff_full.inc(); + worker.add_result({.type = CellStorer::Diff::Set, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_value(ref_cnt_diff + state.db_ref_cnt, data_cell, false)}); + } else { + stats_.diff_ref_cnt.inc(); + worker.add_result({.type = CellStorer::Diff::Merge, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_refcnt_diffs(ref_cnt_diff)}); + } + } + info->on_written_to_db(); + return; + } + + auto state = info->state.load(); + if (!state.sync_with_db) { + stats_.changes_loaded.inc(); + state = cell_db_reader_->sync_with_db(*info, true).first; + } + CHECK(state.sync_with_db); + auto new_ref_cnt = ref_cnt_diff + state.db_ref_cnt; + + if (ref_cnt_diff < 0) { + stats_.dec_save.inc(); + if (new_ref_cnt == 0) { + stats_.dec_erase_cell.inc(); + + LOG_IF(ERROR, dbg) << "DEC ERASE " << *info; + worker.add_result({.type = CellStorer::Diff::Erase, .key = info->cell->get_hash()}); + stats_.dec_save_erase.inc(); + } else { + stats_.dec_just_ref_cnt.inc(); + + LOG_IF(ERROR, dbg) << "DEC REFCNT " << *info; + CHECK(info->cell->is_loaded()); + worker.add_result( + {.type = CellStorer::Diff::Set, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, false)}); + stats_.dec_save_full.inc(); + } + } else { + stats_.inc_save.inc(); + CHECK(info->cell->is_loaded()); + if (state.db_ref_cnt == 0) { + stats_.inc_new_cell.inc(); + LOG_IF(ERROR, dbg) << "INC CREATE " << *info; + } else { + stats_.inc_just_ref_cnt.inc(); + LOG_IF(ERROR, dbg) << "INC REFCNT " << *info; + } + + worker.add_result( + {.type = CellStorer::Diff::Set, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, false)}); + stats_.inc_save_full.inc(); + } + } + + void save_diff(CellStorer &storer) { + td::PerfWarningTimer timer("celldb_v2: save_diff"); + td::PerfWarningTimer timer_store_to_db("celldb_v2: save_diff_store_to_db", 0.01); + // Have no idea hot to parallelize this in case of rocksdb + for (auto &diffs : diff_chunks_) { + for (auto &diff : diffs) { + storer.apply_diff(diff).ensure(); + } + } + for (auto &meta_diff : meta_diffs_) { + meta_db_fixup_[meta_diff.key] = meta_diff.value; + storer.apply_meta_diff(meta_diff).ensure(); + } + timer_store_to_db.reset(); + td::PerfWarningTimer timer_clear("celldb_v2: save_diff_clear"); + diff_chunks_.clear(); + meta_diffs_.clear(); + timer_clear.reset(); + } +}; +} // namespace + +std::unique_ptr DynamicBagOfCellsDb::create_v2(CreateV2Options options) { + return std::make_unique(options); +} +} // namespace vm diff --git a/crypto/vm/db/InMemoryBagOfCellsDb.cpp b/crypto/vm/db/InMemoryBagOfCellsDb.cpp index 03cad0934e..b0a30cfb2d 100644 --- a/crypto/vm/db/InMemoryBagOfCellsDb.cpp +++ b/crypto/vm/db/InMemoryBagOfCellsDb.cpp @@ -413,6 +413,7 @@ class CellStorage { size_t dense_ht_size = 0; size_t new_ht_size = 0; for_each_bucket(0, [&](auto bucket_id, CellBucket &bucket) { + // TODO: this leads to CE when use_dense_hash_map == false dense_ht_capacity += bucket.infos_.dense_ht_values_.size(); dense_ht_size += bucket.infos_.dense_ht_size_; new_ht_capacity += bucket.infos_.new_ht_.bucket_count(); @@ -468,6 +469,14 @@ class CellStorage { } return td::Status::Error("not found"); } + td::Result>> load_known_roots_local() const { + auto lock = local_access_.lock(); + std::vector> result; + for (auto &root : roots_) { + result.emplace_back(root); + } + return result; + } td::Result> load_root_shared(const CellHash &hash) const { std::lock_guard lock(root_mutex_); if (auto it = roots_.find(hash); it != roots_.end()) { @@ -620,7 +629,7 @@ class CellStorage { sb << "\n\t" << key << "=" << value; } LOG_IF(ERROR, desc_count != 0 && desc_count != stats.roots_total_count + 1) - << "desc<> keys count is " << desc_count << " wich is different from roots count " << stats.roots_total_count; + << "desc<> keys count is " << desc_count << " which is different from roots count " << stats.roots_total_count; LOG_IF(WARNING, verbose) << P << "done in " << full_timer.elapsed() << "\n\troots_count=" << stats.roots_total_count << "\n\t" << desc_count << "\n\tcells_count=" << stats.cells_total_count @@ -757,15 +766,77 @@ class CellStorage { } }; +class MetaStorage { + public: + explicit MetaStorage(std::vector> values) + : meta_(std::move_iterator(values.begin()), std::move_iterator(values.end())) { + for (auto &p : meta_) { + CHECK(p.first.size() != CellTraits::hash_bytes); + } + } + std::vector> meta_get_all() const { + return td::transform(meta_, [](const auto &p) { return std::make_pair(p.first, p.second); }); + } + KeyValue::GetStatus meta_get(td::Slice key, std::string &value) const { + auto lock = local_access_.lock(); + auto it = meta_.find(key.str()); + if (it == meta_.end()) { + return KeyValue::GetStatus::NotFound; + } + value = it->second; + return KeyValue::GetStatus::Ok; + } + void meta_set(td::Slice key, td::Slice value) { + auto lock = local_access_.lock(); + meta_[key.str()] = value.str(); + meta_diffs_.push_back( + CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Set, .key = key.str(), .value = value.str()}); + } + void meta_erase(td::Slice key) { + auto lock = local_access_.lock(); + meta_.erase(key.str()); + meta_diffs_.push_back(CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Erase, .key = key.str()}); + } + std::vector extract_diffs() { + auto lock = local_access_.lock(); + return std::move(meta_diffs_); + } + + private: + mutable UniqueAccess local_access_; + std::unordered_map meta_; + std::vector meta_diffs_; +}; + class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { public: - explicit InMemoryBagOfCellsDb(td::unique_ptr storage) : storage_(std::move(storage)) { + explicit InMemoryBagOfCellsDb(td::unique_ptr storage, td::unique_ptr meta_storage) + : storage_(std::move(storage)), meta_storage_(std::move(meta_storage)) { + } + + td::Result>> meta_get_all() const override { + return meta_storage_->meta_get_all(); + } + td::Result meta_get(td::Slice key, std::string &value) override { + CHECK(key.size() != CellTraits::hash_bytes); + return meta_storage_->meta_get(key, value); + } + td::Status meta_set(td::Slice key, td::Slice value) override { + meta_storage_->meta_set(key, value); + return td::Status::OK(); + } + td::Status meta_erase(td::Slice key) override { + meta_storage_->meta_erase(key); + return td::Status::OK(); } td::Result> load_cell(td::Slice hash) override { return storage_->load_cell(CellHash::from_slice(hash)); } + td::Result>> load_known_roots() const override { + return storage_->load_known_roots_local(); + } td::Result> load_root(td::Slice hash) override { return storage_->load_root_local(CellHash::from_slice(hash)); } @@ -798,29 +869,37 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { TRY_STATUS(prepare_commit()); } + td::PerfWarningTimer times_save_diff("save diff"); Stats diff; CHECK(to_dec_.empty()); - for (auto &it : info_) { - auto &info = it.second; + for (auto &info : info_) { if (info.diff_refcnt == 0) { continue; } auto refcnt = td::narrow_cast(static_cast(info.db_refcnt) + info.diff_refcnt); - CHECK(refcnt >= 0); + LOG_CHECK(refcnt >= 0) << info.db_refcnt << " + " << info.diff_refcnt; if (refcnt > 0) { - cell_storer.set(refcnt, info.cell, false); + if (info.db_refcnt == 0) { + TRY_STATUS(cell_storer.set(refcnt, info.cell, false)); + } else { + TRY_STATUS(cell_storer.merge(info.cell->get_hash().as_slice(), info.diff_refcnt)); + } storage_->set(refcnt, info.cell); if (info.db_refcnt == 0) { diff.cells_total_count++; diff.cells_total_size += static_cast(info.cell->get_storage_size()); } } else { - cell_storer.erase(info.cell->get_hash().as_slice()); + TRY_STATUS(cell_storer.erase(info.cell->get_hash().as_slice())); storage_->erase(info.cell->get_hash()); diff.cells_total_count--; diff.cells_total_size -= static_cast(info.cell->get_storage_size()); } } + auto meta_diffs = meta_storage_->extract_diffs(); + for (const auto &meta_diff : meta_diffs) { + TRY_STATUS(cell_storer.apply_meta_diff(meta_diff)); + } storage_->apply_stats_diff(diff); info_ = {}; return td::Status::OK(); @@ -872,13 +951,39 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { private: td::unique_ptr storage_; + td::unique_ptr meta_storage_; struct Info { - td::int32 db_refcnt{0}; - td::int32 diff_refcnt{0}; + mutable td::int32 db_refcnt{0}; + mutable td::int32 diff_refcnt{0}; Ref cell; + vm::CellHash key() const { + return cell->get_hash(); + } + struct Eq { + using is_transparent = void; // Pred to use + bool operator()(const Info &info, const Info &other_info) const { + return info.key() == other_info.key(); + } + bool operator()(const Info &info, td::Slice hash) const { + return info.key().as_slice() == hash; + } + bool operator()(td::Slice hash, const Info &info) const { + return info.key().as_slice() == hash; + } + }; + struct Hash { + using is_transparent = void; // Pred to use + using transparent_key_equal = Eq; + size_t operator()(td::Slice hash) const { + return cell_hash_slice_hash(hash); + } + size_t operator()(const Info &info) const { + return cell_hash_slice_hash(info.key().as_slice()); + } + }; }; - td::HashMap info_; + td::HashSet info_; std::unique_ptr loader_; std::vector> to_inc_; @@ -886,13 +991,13 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { Ref do_inc(Ref cell) { auto cell_hash = cell->get_hash(); - if (auto it = info_.find(cell_hash); it != info_.end()) { - CHECK(it->second.diff_refcnt != std::numeric_limits::max()); - it->second.diff_refcnt++; - return it->second.cell; + if (auto it = info_.find(cell_hash.as_slice()); it != info_.end()) { + CHECK(it->diff_refcnt != std::numeric_limits::max()); + it->diff_refcnt++; + return it->cell; } if (auto o_info = storage_->get_info(cell_hash)) { - info_.emplace(cell_hash, Info{.db_refcnt = o_info->db_refcnt, .diff_refcnt = 1, .cell = o_info->cell}); + info_.emplace(Info{.db_refcnt = o_info->db_refcnt, .diff_refcnt = 1, .cell = o_info->cell}); return std::move(o_info->cell); } @@ -905,21 +1010,21 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { } auto res = cb.finalize(cs.is_special()); CHECK(res->get_hash() == cell_hash); - info_.emplace(cell_hash, Info{.db_refcnt = 0, .diff_refcnt = 1, .cell = res}); + info_.emplace(Info{.db_refcnt = 0, .diff_refcnt = 1, .cell = res}); return res; } void do_dec(Ref cell) { auto cell_hash = cell->get_hash(); - auto it = info_.find(cell_hash); + auto it = info_.find(cell_hash.as_slice()); if (it != info_.end()) { - CHECK(it->second.diff_refcnt != std::numeric_limits::min()); - --it->second.diff_refcnt; + CHECK(it->diff_refcnt != std::numeric_limits::min()); + --it->diff_refcnt; } else { auto info = *storage_->get_info(cell_hash); - it = info_.emplace(cell_hash, Info{.db_refcnt = info.db_refcnt, .diff_refcnt = -1, .cell = info.cell}).first; + it = info_.emplace(Info{.db_refcnt = info.db_refcnt, .diff_refcnt = -1, .cell = info.cell}).first; } - if (it->second.diff_refcnt + it->second.db_refcnt != 0) { + if (it->diff_refcnt + it->db_refcnt != 0) { return; } CellSlice cs(NoVm{}, std::move(cell)); @@ -936,7 +1041,8 @@ std::unique_ptr DynamicBagOfCellsDb::create_in_memory(td::K if (kv == nullptr) { LOG_IF(WARNING, options.verbose) << "Create empty in-memory cells database (no key value is given)"; auto storage = CellStorage::build(options, [](auto, auto, auto) { return std::make_pair(0, 0); }); - return std::make_unique(std::move(storage)); + auto meta_storage = td::make_unique(std::vector>{}); + return std::make_unique(std::move(storage), std::move(meta_storage)); } std::vector keys; @@ -962,6 +1068,9 @@ std::unique_ptr DynamicBagOfCellsDb::create_in_memory(td::K local_desc_count++; return td::Status::OK(); } + if (key.size() != 32) { + return td::Status::OK(); + } auto r_res = CellLoader::load(key, value.str(), true, pc_creator); if (r_res.is_error()) { LOG(ERROR) << r_res.error() << " at " << td::format::escaped(key); @@ -983,6 +1092,24 @@ std::unique_ptr DynamicBagOfCellsDb::create_in_memory(td::K }; auto storage = CellStorage::build(options, parallel_scan_cells); - return std::make_unique(std::move(storage)); + + std::vector> meta; + // NB: it scans 1/(2^32) of the database which is not much + kv->for_each_in_range("desc", "desd", [&meta](td::Slice key, td::Slice value) { + if (key.size() != 32) { + meta.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + // this is for tests mostly. desc* keys are expected to correspond to roots + kv->for_each_in_range("meta", "metb", [&meta](td::Slice key, td::Slice value) { + if (key.size() != 32) { + meta.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + auto meta_storage = td::make_unique(std::move(meta)); + + return std::make_unique(std::move(storage), std::move(meta_storage)); } } // namespace vm diff --git a/crypto/vm/db/StaticBagOfCellsDb.cpp b/crypto/vm/db/StaticBagOfCellsDb.cpp index 80dbfbf0b2..c65d2624ff 100644 --- a/crypto/vm/db/StaticBagOfCellsDb.cpp +++ b/crypto/vm/db/StaticBagOfCellsDb.cpp @@ -40,6 +40,9 @@ class RootCell : public Cell { struct PrivateTag {}; public: + td::Status set_data_cell(Ref &&data_cell) const override { + return cell_->set_data_cell(std::move(data_cell)); + } td::Result load_cell() const override { return cell_->load_cell(); } @@ -94,11 +97,11 @@ class DataCellCacheNoop { class DataCellCacheMutex { public: Ref store(int idx, Ref cell) { - auto lock = cells_rw_mutex_.lock_write(); + std::lock_guard lock(mutex_); return cells_.emplace(idx, std::move(cell)).first->second; } Ref load(int idx) { - auto lock = cells_rw_mutex_.lock_read(); + std::lock_guard lock(mutex_); auto it = cells_.find(idx); if (it != cells_.end()) { return it->second; @@ -106,12 +109,13 @@ class DataCellCacheMutex { return {}; } void clear() { - auto guard = cells_rw_mutex_.lock_write(); + std::lock_guard lock(mutex_); cells_.clear(); } private: - td::RwMutex cells_rw_mutex_; + std::mutex mutex_; + // NB: in case of high contention, one should use multiple buckets with per bucket mutexes td::HashMap> cells_; }; @@ -246,7 +250,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { BagOfCells::Info info_; std::mutex index_i_mutex_; - td::RwMutex index_data_rw_mutex_; + std::mutex index_mutex_; std::string index_data_; std::atomic index_i_{0}; size_t index_offset_{0}; @@ -319,7 +323,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { info_.index_offset + (td::int64)idx * info_.offset_byte_size)); offset_view = new_offset_view; } else { - guard = index_data_rw_mutex_.lock_read().move_as_ok(); + std::lock_guard guard(index_mutex_); offset_view = td::Slice(index_data_).substr((td::int64)idx * info_.offset_byte_size, info_.offset_byte_size); } @@ -432,7 +436,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { } td::uint8 tmp[8]; info_.write_offset(tmp, index_offset_); - auto guard = index_data_rw_mutex_.lock_write(); + std::lock_guard guard(index_mutex_); index_data_.append(reinterpret_cast(tmp), info_.offset_byte_size); } return td::Status::OK(); diff --git a/tddb/td/db/KeyValue.h b/tddb/td/db/KeyValue.h index 12c3a4f8d7..c3f83919bf 100644 --- a/tddb/td/db/KeyValue.h +++ b/tddb/td/db/KeyValue.h @@ -20,19 +20,51 @@ #include "td/utils/Status.h" #include "td/utils/Time.h" #include "td/utils/logging.h" +#include "td/utils/ThreadSafeCounter.h" #include namespace td { +struct UsageStats { + size_t get_count{}; + size_t get_found_count{}; + size_t get_not_found_count{}; + size_t set_count{}; + UsageStats operator+(const UsageStats& other) const { + return UsageStats{.get_count = get_count + other.get_count, + .get_found_count = get_found_count + other.get_found_count, + .get_not_found_count = get_not_found_count + other.get_not_found_count, + .set_count = set_count + other.set_count}; + } + UsageStats operator-(const UsageStats& other) const { + return UsageStats{.get_count = get_count - other.get_count, + .get_found_count = get_found_count - other.get_found_count, + .get_not_found_count = get_not_found_count - other.get_not_found_count, + .set_count = set_count - other.set_count}; + } + NamedStats to_named_stats() const { + NamedStats ns; + ns.stats_int["usage_get_count"] += get_count; + ns.stats_int["usage_get_found_count"] += get_found_count; + ns.stats_int["usage_get_not_found_count"] += get_not_found_count; + ns.stats_int["usage_set_count"] += set_count; + return ns; + } +}; +inline td::StringBuilder& operator<<(td::StringBuilder& sb, const UsageStats& stats) { + sb << "get: " << stats.get_count << ", +" << stats.get_found_count << ", -" << stats.get_not_found_count; + return sb; +} + class KeyValueReader { public: virtual ~KeyValueReader() = default; enum class GetStatus : int32 { Ok, NotFound }; - virtual Result get(Slice key, std::string &value) = 0; + virtual Result get(Slice key, std::string& value) = 0; virtual Result count(Slice prefix) = 0; virtual Status for_each(std::function f) { return Status::Error("for_each is not supported"); } - virtual Status for_each_in_range (Slice begin, Slice end, std::function f) { + virtual Status for_each_in_range(Slice begin, Slice end, std::function f) { return td::Status::Error("foreach_range is not supported"); } }; @@ -42,7 +74,7 @@ class PrefixedKeyValueReader : public KeyValueReader { PrefixedKeyValueReader(std::shared_ptr reader, Slice prefix) : reader_(std::move(reader)), prefix_(prefix.str()) { } - Result get(Slice key, std::string &value) override { + Result get(Slice key, std::string& value) override { return reader_->get(PSLICE() << prefix_ << key, value); } Result count(Slice prefix) override { @@ -54,14 +86,16 @@ class PrefixedKeyValueReader : public KeyValueReader { std::string prefix_; }; -class KeyValueUtils { - public: -}; - class KeyValue : public KeyValueReader { public: virtual Status set(Slice key, Slice value) = 0; virtual Status erase(Slice key) = 0; + virtual Status merge(Slice key, Slice value) { + return Status::Error("merge is not supported"); + } + virtual Status run_gc() { + return Status::OK(); + } virtual Status begin_write_batch() = 0; virtual Status commit_write_batch() = 0; @@ -80,12 +114,15 @@ class KeyValue : public KeyValueReader { virtual Status flush() { return Status::OK(); } + virtual UsageStats get_usage_stats() { + return {}; + } }; class PrefixedKeyValue : public KeyValue { public: PrefixedKeyValue(std::shared_ptr kv, Slice prefix) : kv_(std::move(kv)), prefix_(prefix.str()) { } - Result get(Slice key, std::string &value) override { + Result get(Slice key, std::string& value) override { return kv_->get(PSLICE() << prefix_ << key, value); } Result count(Slice prefix) override { diff --git a/tddb/td/db/MemoryKeyValue.cpp b/tddb/td/db/MemoryKeyValue.cpp index 0801336024..7105f72b94 100644 --- a/tddb/td/db/MemoryKeyValue.cpp +++ b/tddb/td/db/MemoryKeyValue.cpp @@ -22,57 +22,99 @@ namespace td { Result MemoryKeyValue::get(Slice key, std::string &value) { - auto it = map_.find(key); - if (it == map_.end()) { + auto bucket = lock(key); + auto &map = bucket->map; + + usage_stats_.get_count++; + auto it = map.find(key); + if (it == map.end()) { + usage_stats_.get_not_found_count++; return GetStatus::NotFound; } value = it->second; + usage_stats_.get_found_count++; return GetStatus::Ok; } +std::unique_ptr MemoryKeyValue::lock(td::Slice key) { + auto bucket_id = std::hash()(std::string_view(key.data(), key.size())) % buckets_.size(); + return lock(buckets_[bucket_id]); +} + Status MemoryKeyValue::for_each(std::function f) { - for (auto &it : map_) { - TRY_STATUS(f(it.first, it.second)); + for (auto &unlocked_bucket : buckets_) { + auto bucket = lock(unlocked_bucket); + for (auto &it : bucket->map) { + TRY_STATUS(f(it.first, it.second)); + } } return Status::OK(); } Status MemoryKeyValue::for_each_in_range(Slice begin, Slice end, std::function f) { - for (auto it = map_.lower_bound(begin); it != map_.end(); it++) { - if (it->first < end) { - TRY_STATUS(f(it->first, it->second)); - } else { - break; + for (auto &unlocked_bucket : buckets_) { + auto bucket = lock(unlocked_bucket); + auto &map = bucket->map; + for (auto it = map.lower_bound(begin); it != map.end(); it++) { + if (it->first < end) { + TRY_STATUS(f(it->first, it->second)); + } else { + break; + } } } return Status::OK(); } Status MemoryKeyValue::set(Slice key, Slice value) { - map_[key.str()] = value.str(); + auto bucket = lock(key); + auto &map = bucket->map; + + usage_stats_.set_count++; + map[key.str()] = value.str(); return Status::OK(); } +Status MemoryKeyValue::merge(Slice key, Slice update) { + CHECK(merger_); + auto bucket = lock(key); + auto &map = bucket->map; + auto &value = map[key.str()]; + merger_->merge_value_and_update(value, update); + if (value.empty()) { + map.erase(key.str()); + } + return td::Status::OK(); +} Status MemoryKeyValue::erase(Slice key) { - auto it = map_.find(key); - if (it != map_.end()) { - map_.erase(it); + auto bucket = lock(key); + auto &map = bucket->map; + auto it = map.find(key); + if (it != map.end()) { + map.erase(it); } return Status::OK(); } Result MemoryKeyValue::count(Slice prefix) { size_t res = 0; - for (auto it = map_.lower_bound(prefix); it != map_.end(); it++) { - if (Slice(it->first).truncate(prefix.size()) != prefix) { - break; + for (auto &unlocked_bucket : buckets_) { + auto bucket = lock(unlocked_bucket); + auto &map = bucket->map; + for (auto it = map.lower_bound(prefix); it != map.end(); it++) { + if (Slice(it->first).truncate(prefix.size()) != prefix) { + break; + } + res++; } - res++; } return res; } std::unique_ptr MemoryKeyValue::snapshot() { auto res = std::make_unique(); - res->map_ = map_; + for (size_t i = 0; i < buckets_.size(); i++) { + auto bucket = lock(buckets_[i]); + res->buckets_[i].map = bucket->map; + } return std::move(res); } @@ -80,10 +122,10 @@ std::string MemoryKeyValue::stats() const { return PSTRING() << "MemoryKeyValueStats{" << tag("get_count", get_count_) << "}"; } Status MemoryKeyValue::begin_write_batch() { - UNREACHABLE(); + return Status::OK(); } Status MemoryKeyValue::commit_write_batch() { - UNREACHABLE(); + return Status::OK(); } Status MemoryKeyValue::abort_write_batch() { UNREACHABLE(); diff --git a/tddb/td/db/MemoryKeyValue.h b/tddb/td/db/MemoryKeyValue.h index f0b5faa08b..cf896095d8 100644 --- a/tddb/td/db/MemoryKeyValue.h +++ b/tddb/td/db/MemoryKeyValue.h @@ -22,12 +22,22 @@ #include namespace td { + +struct Merger { + virtual ~Merger() = default; + virtual void merge_value_and_update(std::string &value, Slice update) = 0; + virtual void merge_update_and_update(std::string &left_update, Slice right_update) = 0; +}; class MemoryKeyValue : public KeyValue { public: - Result get(Slice key, std::string &value) override; + MemoryKeyValue() = default; + MemoryKeyValue(std::shared_ptr merger) : merger_(std::move(merger)) { + } + Result get(Slice key, std::string& value) override; Status for_each(std::function f) override; Status for_each_in_range(Slice begin, Slice end, std::function f) override; Status set(Slice key, Slice value) override; + Status merge(Slice key, Slice value) override; Status erase(Slice key) override; Result count(Slice prefix) override; @@ -43,8 +53,30 @@ class MemoryKeyValue : public KeyValue { std::string stats() const override; + UsageStats get_usage_stats() override { + return usage_stats_; + } + private: - std::map> map_; + static constexpr size_t buckets_n = 64; + struct Bucket { + std::mutex mutex; + std::map> map; + }; + struct Unlock { + void operator()(Bucket* bucket) const { + bucket->mutex.unlock(); + } + }; + std::array buckets_{}; int64 get_count_{0}; + UsageStats usage_stats_{}; + std::shared_ptr merger_; + + std::unique_ptr lock(Bucket& bucket) { + bucket.mutex.lock(); + return std::unique_ptr(&bucket); + } + std::unique_ptr lock(td::Slice key); }; } // namespace td diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index f1aa64a5d7..b56f3b145f 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -24,10 +24,13 @@ #include "rocksdb/write_batch.h" #include "rocksdb/utilities/optimistic_transaction_db.h" #include "rocksdb/utilities/transaction.h" +#include "td/utils/misc.h" + +#include namespace td { namespace { -static Status from_rocksdb(rocksdb::Status status) { +static Status from_rocksdb(const rocksdb::Status &status) { if (status.ok()) { return Status::OK(); } @@ -56,62 +59,83 @@ RocksDb::~RocksDb() { } RocksDb RocksDb::clone() const { + if (transaction_db_) { + return RocksDb{transaction_db_, options_}; + } return RocksDb{db_, options_}; } Result RocksDb::open(std::string path, RocksDbOptions options) { - rocksdb::OptimisticTransactionDB *db; - { - rocksdb::Options db_options; + rocksdb::Options db_options; + db_options.merge_operator = options.merge_operator; + db_options.compaction_filter = options.compaction_filter; - static auto default_cache = rocksdb::NewLRUCache(1 << 30); - if (!options.no_block_cache && options.block_cache == nullptr) { - options.block_cache = default_cache; - } + static auto default_cache = rocksdb::NewLRUCache(1 << 30); + if (!options.no_block_cache && options.block_cache == nullptr) { + options.block_cache = default_cache; + } - rocksdb::BlockBasedTableOptions table_options; - if (options.no_block_cache) { - table_options.no_block_cache = true; - } else { - table_options.block_cache = options.block_cache; - } - db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); - - db_options.use_direct_reads = options.use_direct_reads; - db_options.manual_wal_flush = true; - db_options.create_if_missing = true; - db_options.max_background_compactions = 4; - db_options.max_background_flushes = 2; - db_options.bytes_per_sync = 1 << 20; - db_options.writable_file_max_buffer_size = 2 << 14; - db_options.statistics = options.statistics; - db_options.max_log_file_size = 100 << 20; - db_options.keep_log_file_num = 1; - rocksdb::OptimisticTransactionDBOptions occ_options; - occ_options.validate_policy = rocksdb::OccValidationPolicy::kValidateSerial; + rocksdb::BlockBasedTableOptions table_options; + if (options.no_block_cache) { + table_options.no_block_cache = true; + } else { + table_options.block_cache = options.block_cache; + } + db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); + + // table_options.block_align = true; + if (options.no_reads) { + db_options.memtable_factory.reset(new rocksdb::VectorRepFactory()); + db_options.allow_concurrent_memtable_write = false; + } + + db_options.wal_recovery_mode = rocksdb::WALRecoveryMode::kTolerateCorruptedTailRecords; + db_options.use_direct_reads = options.use_direct_reads; + db_options.manual_wal_flush = true; + db_options.create_if_missing = true; + db_options.max_background_compactions = 4; + db_options.max_background_flushes = 2; + db_options.bytes_per_sync = 1 << 20; + db_options.writable_file_max_buffer_size = 2 << 14; + db_options.statistics = options.statistics; + db_options.max_log_file_size = 100 << 20; + db_options.keep_log_file_num = 1; + + if (options.experimental) { + // Place your experimental options here + } + + if (options.no_transactions) { + rocksdb::DB *db{nullptr}; + TRY_STATUS(from_rocksdb(rocksdb::DB::Open(db_options, std::move(path), &db))); + return RocksDb(std::shared_ptr(db), std::move(options)); + } else { + rocksdb::OptimisticTransactionDB *db{nullptr}; rocksdb::ColumnFamilyOptions cf_options(db_options); std::vector column_families; column_families.push_back(rocksdb::ColumnFamilyDescriptor(rocksdb::kDefaultColumnFamilyName, cf_options)); std::vector handles; + rocksdb::OptimisticTransactionDBOptions occ_options; + occ_options.validate_policy = rocksdb::OccValidationPolicy::kValidateSerial; TRY_STATUS(from_rocksdb(rocksdb::OptimisticTransactionDB::Open(db_options, occ_options, std::move(path), column_families, &handles, &db))); CHECK(handles.size() == 1); // i can delete the handle since DBImpl is always holding a reference to // default column family delete handles[0]; + return RocksDb(std::shared_ptr(db), std::move(options)); } - return RocksDb(std::shared_ptr(db), std::move(options)); } std::shared_ptr RocksDb::create_statistics() { return rocksdb::CreateDBStatistics(); } -std::string RocksDb::statistics_to_string(const std::shared_ptr statistics) { +std::string RocksDb::statistics_to_string(const std::shared_ptr &statistics) { return statistics->ToString(); } -void RocksDb::reset_statistics(const std::shared_ptr statistics) { +void RocksDb::reset_statistics(const std::shared_ptr &statistics) { statistics->Reset(); } @@ -133,7 +157,9 @@ std::string RocksDb::stats() const { } Result RocksDb::get(Slice key, std::string &value) { - //LOG(ERROR) << "GET"; + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::Status status; if (snapshot_) { rocksdb::ReadOptions options; @@ -162,6 +188,18 @@ Status RocksDb::set(Slice key, Slice value) { } return from_rocksdb(db_->Put({}, to_rocksdb(key), to_rocksdb(value))); } +Status RocksDb::merge(Slice key, Slice value) { + if (write_batch_) { + return from_rocksdb(write_batch_->Merge(to_rocksdb(key), to_rocksdb(value))); + } + if (transaction_) { + return from_rocksdb(transaction_->Merge(to_rocksdb(key), to_rocksdb(value))); + } + return from_rocksdb(db_->Merge({}, to_rocksdb(key), to_rocksdb(value))); +} +Status RocksDb::run_gc() { + return from_rocksdb(db_->CompactRange({}, nullptr, nullptr)); +} Status RocksDb::erase(Slice key) { if (write_batch_) { @@ -174,7 +212,11 @@ Status RocksDb::erase(Slice key) { } Result RocksDb::count(Slice prefix) { + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::ReadOptions options; + options.auto_prefix_mode = true; options.snapshot = snapshot_.get(); std::unique_ptr iterator; if (snapshot_ || !transaction_) { @@ -197,7 +239,11 @@ Result RocksDb::count(Slice prefix) { } Status RocksDb::for_each(std::function f) { + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::ReadOptions options; + options.auto_prefix_mode = true; options.snapshot = snapshot_.get(); std::unique_ptr iterator; if (snapshot_ || !transaction_) { @@ -219,7 +265,11 @@ Status RocksDb::for_each(std::function f) { } Status RocksDb::for_each_in_range(Slice begin, Slice end, std::function f) { + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::ReadOptions options; + options.auto_prefix_mode = true; options.snapshot = snapshot_.get(); std::unique_ptr iterator; if (snapshot_ || !transaction_) { @@ -252,9 +302,10 @@ Status RocksDb::begin_write_batch() { Status RocksDb::begin_transaction() { CHECK(!write_batch_); + CHECK(transaction_db_); rocksdb::WriteOptions options; options.sync = true; - transaction_.reset(db_->BeginTransaction(options, {})); + transaction_.reset(transaction_db_->BeginTransaction(options, {})); return Status::OK(); } @@ -307,7 +358,11 @@ Status RocksDb::end_snapshot() { } RocksDb::RocksDb(std::shared_ptr db, RocksDbOptions options) - : db_(std::move(db)), options_(options) { + : transaction_db_{db}, db_(std::move(db)), options_(std::move(options)) { +} + +RocksDb::RocksDb(std::shared_ptr db, RocksDbOptions options) + : db_(std::move(db)), options_(std::move(options)) { } void RocksDbSnapshotStatistics::begin_snapshot(const rocksdb::Snapshot *snapshot) { diff --git a/tddb/td/db/RocksDb.h b/tddb/td/db/RocksDb.h index 499a332810..d24a20dd7b 100644 --- a/tddb/td/db/RocksDb.h +++ b/tddb/td/db/RocksDb.h @@ -36,12 +36,16 @@ #include namespace rocksdb { +class DB; +class Comparator; class Cache; class OptimisticTransactionDB; class Transaction; class WriteBatch; class Snapshot; class Statistics; +class MergeOperator; +class CompactionFilter; } // namespace rocksdb namespace td { @@ -61,6 +65,14 @@ struct RocksDbOptions { std::shared_ptr statistics = nullptr; std::shared_ptr block_cache; // Default - one 1G cache for all RocksDb std::shared_ptr snapshot_statistics = nullptr; + + std::shared_ptr merge_operator = nullptr; + const rocksdb::CompactionFilter *compaction_filter = nullptr; + + bool experimental = false; + bool no_reads = false; + bool no_transactions = false; + bool use_direct_reads = false; bool no_block_cache = false; }; @@ -73,10 +85,12 @@ class RocksDb : public KeyValue { Result get(Slice key, std::string &value) override; Status set(Slice key, Slice value) override; + Status merge(Slice key, Slice value) override; Status erase(Slice key) override; + Status run_gc() override; Result count(Slice prefix) override; Status for_each(std::function f) override; - Status for_each_in_range (Slice begin, Slice end, std::function f) override; + Status for_each_in_range(Slice begin, Slice end, std::function f) override; Status begin_write_batch() override; Status commit_write_batch() override; @@ -94,8 +108,8 @@ class RocksDb : public KeyValue { std::string stats() const override; static std::shared_ptr create_statistics(); - static std::string statistics_to_string(const std::shared_ptr statistics); - static void reset_statistics(const std::shared_ptr statistics); + static std::string statistics_to_string(const std::shared_ptr &statistics); + static void reset_statistics(const std::shared_ptr &statistics); static std::shared_ptr create_cache(size_t capacity); @@ -103,12 +117,13 @@ class RocksDb : public KeyValue { RocksDb &operator=(RocksDb &&); ~RocksDb(); - std::shared_ptr raw_db() const { + std::shared_ptr raw_db() const { return db_; }; private: - std::shared_ptr db_; + std::shared_ptr transaction_db_; + std::shared_ptr db_; RocksDbOptions options_; std::unique_ptr transaction_; @@ -123,5 +138,6 @@ class RocksDb : public KeyValue { std::unique_ptr snapshot_; explicit RocksDb(std::shared_ptr db, RocksDbOptions options); + explicit RocksDb(std::shared_ptr db, RocksDbOptions options); }; } // namespace td diff --git a/tdutils/td/utils/MpmcQueue.h b/tdutils/td/utils/MpmcQueue.h index e6504e358c..1a5f8fa366 100644 --- a/tdutils/td/utils/MpmcQueue.h +++ b/tdutils/td/utils/MpmcQueue.h @@ -414,7 +414,9 @@ class MpmcQueue { while (true) { auto node = hazard_pointers_.protect(thread_id, 0, read_pos_); auto &block = node->block; - if (block.write_pos <= block.read_pos && node->next.load(std::memory_order_relaxed) == nullptr) { + auto read_pos = block.read_pos.load(); + auto write_pos = block.write_pos.load(); + if (write_pos <= read_pos && node->next.load(std::memory_order_relaxed) == nullptr) { return false; } auto pos = block.read_pos++; diff --git a/tdutils/td/utils/Status.h b/tdutils/td/utils/Status.h index cff8081437..f75de466a9 100644 --- a/tdutils/td/utils/Status.h +++ b/tdutils/td/utils/Status.h @@ -619,6 +619,13 @@ inline Result::Result(Status &&status) : status_(std::move(status)) { inline StringBuilder &operator<<(StringBuilder &string_builder, const Status &status) { return status.print(string_builder); } +template +StringBuilder &operator<<(StringBuilder &sb, const Result &result) { + if (result.is_ok()) { + return sb << "Ok{" << result.ok() << "}"; + } + return sb << result.error(); +} namespace detail { diff --git a/tdutils/td/utils/ThreadSafeCounter.h b/tdutils/td/utils/ThreadSafeCounter.h index aa976b2fbb..46dc16bf70 100644 --- a/tdutils/td/utils/ThreadSafeCounter.h +++ b/tdutils/td/utils/ThreadSafeCounter.h @@ -19,6 +19,7 @@ #pragma once +#include "port/thread.h" #include "td/utils/common.h" #include "td/utils/Slice.h" #include "td/utils/StringBuilder.h" @@ -26,6 +27,7 @@ #include #include +#include #include namespace td { @@ -69,6 +71,50 @@ class ThreadSafeCounter { ThreadSafeMultiCounter<1> counter_; }; +struct NamedStats { + std::map stats_int; + std::map stats_str; + + NamedStats with_suffix(const std::string &suffix) const { + NamedStats res; + for (auto &p : stats_int) { + res.stats_int[p.first + suffix] = p.second; + } + for (auto &p : stats_str) { + res.stats_str[p.first + suffix] = p.second; + } + return res; + } + NamedStats with_prefix(const std::string &prefix) const { + NamedStats res; + for (auto &p : stats_int) { + res.stats_int[prefix + p.first] = p.second; + } + for (auto &p : stats_str) { + res.stats_str[prefix + p.first] = p.second; + } + return res; + } + void apply_diff(const NamedStats &other) { + for (auto &p : other.stats_int) { + stats_int[p.first] += p.second; + } + for (auto &p : other.stats_str) { + stats_str[p.first] = p.second; + } + } + void subtract_diff(const NamedStats &other) { + for (auto &p : other.stats_int) { + stats_int[p.first] -= p.second; + } + } + NamedStats combine_with(const NamedStats &other) const { + NamedStats res = *this; + res.apply_diff(other); + return res; + } +}; + class NamedThreadSafeCounter { static constexpr int N = 128; using Counter = ThreadSafeMultiCounter; @@ -79,6 +125,9 @@ class NamedThreadSafeCounter { CounterRef() = default; CounterRef(size_t index, Counter *counter) : index_(index), counter_(counter) { } + void inc() { + add(1); + } void add(int64 diff) { counter_->add(index_, diff); } @@ -119,6 +168,11 @@ class NamedThreadSafeCounter { f(names_[i], counter_.sum(i)); } } + NamedStats get_stats() const { + NamedStats res; + for_each([&](Slice name, int64 cnt) { res.stats_int.emplace(name.str(), cnt); }); + return res; + } void clear() { std::unique_lock guard(mutex_); @@ -181,11 +235,11 @@ struct NamedPerfCounter { } // namespace td -#define TD_PERF_COUNTER(name) \ +#define TD_PERF_COUNTER(name) \ static auto perf_##name = td::NamedPerfCounter::get_default().get_counter(td::Slice(#name)); \ auto scoped_perf_##name = td::NamedPerfCounter::ScopedPerfCounterRef{.perf_counter = perf_##name}; -#define TD_PERF_COUNTER_SINCE(name, since) \ +#define TD_PERF_COUNTER_SINCE(name, since) \ static auto perf_##name = td::NamedPerfCounter::get_default().get_counter(td::Slice(#name)); \ - auto scoped_perf_##name = \ + auto scoped_perf_##name = \ td::NamedPerfCounter::ScopedPerfCounterRef{.perf_counter = perf_##name, .started_at_ticks = since}; diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 2ea04e1833..cbcc3ab1f0 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1414,6 +1414,9 @@ td::Status ValidatorEngine::load_global_config() { if (zero_state.root_hash.is_zero() || zero_state.file_hash.is_zero()) { return td::Status::Error(ton::ErrorCode::error, "[validator] section contains incomplete [zero_state]"); } + if (celldb_in_memory_ && celldb_v2_) { + return td::Status::Error(ton::ErrorCode::error, "at most one of --celldb-in-memory --celldb-v2 could be used"); + } ton::BlockIdExt init_block; if (!conf.validator_->init_block_) { @@ -1461,11 +1464,12 @@ td::Status ValidatorEngine::load_global_config() { if (!session_logs_file_.empty()) { validator_options_.write().set_session_logs_file(session_logs_file_); } - if (celldb_in_memory_) { + if (celldb_in_memory_ || celldb_v2_) { celldb_compress_depth_ = 0; } validator_options_.write().set_celldb_compress_depth(celldb_compress_depth_); validator_options_.write().set_celldb_in_memory(celldb_in_memory_); + validator_options_.write().set_celldb_v2(celldb_v2_); validator_options_.write().set_max_open_archive_files(max_open_archive_files_); validator_options_.write().set_archive_preload_period(archive_preload_period_); validator_options_.write().set_disable_rocksdb_stats(disable_rocksdb_stats_); @@ -4526,6 +4530,12 @@ int main(int argc, char *argv[]) { [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_in_memory, true); }); }); + p.add_option( + '\0', "celldb-v2", + "use new version off celldb", + [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_v2, true); }); + }); p.add_checked_option( '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.4)", [&](td::Slice s) -> td::Status { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index e0dc91f137..5a1db7f3f5 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -218,6 +218,7 @@ class ValidatorEngine : public td::actor::Actor { bool celldb_direct_io_ = false; bool celldb_preload_all_ = false; bool celldb_in_memory_ = false; + bool celldb_v2_ = false; td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; bool read_config_ = false; bool started_keyring_ = false; @@ -311,6 +312,9 @@ class ValidatorEngine : public td::actor::Actor { void set_celldb_in_memory(bool value) { celldb_in_memory_ = value; } + void set_celldb_v2(bool value) { + celldb_v2_ = value; + } void set_catchain_max_block_delay(double value) { catchain_max_block_delay_ = value; } diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index e86a373d1d..5fc5a83a4e 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -28,6 +28,9 @@ #include "ton/ton-io.hpp" #include "common/delay.h" +#include +#include + namespace ton { namespace validator { @@ -73,6 +76,41 @@ CellDbIn::CellDbIn(td::actor::ActorId root_db, td::actor::ActorId td::Slice { + return td::Slice(value.data(), value.size()); + } + bool FullMergeV2(const MergeOperationInput& merge_in, MergeOperationOutput* merge_out) const override { + CHECK(merge_in.existing_value); + auto& value = *merge_in.existing_value; + CHECK(merge_in.operand_list.size() >= 1); + td::Slice diff; + std::string diff_buf; + if (merge_in.operand_list.size() == 1) { + diff = to_td(merge_in.operand_list[0]); + } else { + diff_buf = merge_in.operand_list[0].ToString(); + for (size_t i = 1; i < merge_in.operand_list.size(); ++i) { + vm::CellStorer::merge_refcnt_diffs(diff_buf, to_td(merge_in.operand_list[i])); + } + diff = diff_buf; + } + + merge_out->new_value = value.ToString(); + vm::CellStorer::merge_value_and_refcnt_diff(merge_out->new_value, diff); + return true; + } + bool PartialMerge(const rocksdb::Slice& /*key*/, const rocksdb::Slice& left, const rocksdb::Slice& right, + std::string* new_value, rocksdb::Logger* logger) const override { + *new_value = left.ToString(); + vm::CellStorer::merge_refcnt_diffs(*new_value, to_td(right)); + return true; + } +}; + void CellDbIn::start_up() { on_load_callback_ = [actor = std::make_shared>( td::actor::create_actor("celldbmigration", actor_id(this))), @@ -96,46 +134,147 @@ void CellDbIn::start_up() { db_options.snapshot_statistics = snapshot_statistics_; } db_options.statistics = statistics_; - if (opts_->get_celldb_cache_size()) { - db_options.block_cache = td::RocksDb::create_cache(opts_->get_celldb_cache_size().value()); - LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(opts_->get_celldb_cache_size().value()); + auto o_celldb_cache_size = opts_->get_celldb_cache_size(); + + std::optional boc_in_memory_options; + std::optional boc_v1_options; + std::optional boc_v2_options; + + if (opts_->get_celldb_v2()) { + boc_v2_options = + vm::DynamicBagOfCellsDb::CreateV2Options{.extra_threads = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 8u), + .executor = {}, + .cache_ttl_max = 2000, + .cache_size_max = 1000000}; + size_t min_rocksdb_cache = std::max(size_t{1} << 30, boc_v2_options->cache_size_max * 5000); + if (!o_celldb_cache_size || o_celldb_cache_size.value() < min_rocksdb_cache) { + LOG(WARNING) << "Increase CellDb block cache size to " << td::format::as_size(min_rocksdb_cache) << " from " + << td::format::as_size(o_celldb_cache_size.value()); + o_celldb_cache_size = min_rocksdb_cache; + } + LOG(WARNING) << "Using V2 DynamicBagOfCells with options " << *boc_v2_options; + } else if (opts_->get_celldb_in_memory()) { + // default options + boc_in_memory_options = vm::DynamicBagOfCellsDb::CreateInMemoryOptions{ + .extra_threads = std::thread::hardware_concurrency(), + .verbose = true, + .use_arena = false, + .use_less_memory_during_creation = true, + }; + LOG(WARNING) << "Using InMemory DynamicBagOfCells with options " << *boc_v2_options; + } else { + boc_v1_options = vm::DynamicBagOfCellsDb::CreateV1Options{}; + LOG(WARNING) << "Using V1 DynamicBagOfCells with options " << *boc_v1_options; + } + + if (o_celldb_cache_size) { + db_options.block_cache = td::RocksDb::create_cache(o_celldb_cache_size.value()); + LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(o_celldb_cache_size.value()); } db_options.use_direct_reads = opts_->get_celldb_direct_io(); + // NB: from now on we MUST use this merge operator + // Only V2 and InMemory BoC actually use them, but it still should be kept for V1, + // to handle updates written by V2 or InMemory BoCs + db_options.merge_operator = std::make_shared(); + if (opts_->get_celldb_in_memory()) { td::RocksDbOptions read_db_options; read_db_options.use_direct_reads = true; read_db_options.no_block_cache = true; read_db_options.block_cache = {}; + read_db_options.merge_operator = std::make_shared(); LOG(WARNING) << "Loading all cells in memory (because of --celldb-in-memory)"; td::Timer timer; auto read_cell_db = std::make_shared(td::RocksDb::open(path_, std::move(read_db_options)).move_as_ok()); - boc_ = vm::DynamicBagOfCellsDb::create_in_memory(read_cell_db.get(), {}); + boc_ = vm::DynamicBagOfCellsDb::create_in_memory(read_cell_db.get(), *boc_in_memory_options); in_memory_load_time_ = timer.elapsed(); - td::actor::send_closure(parent_, &CellDb::set_in_memory_boc, boc_); + + // no reads will be allowed from rocksdb, only writes + db_options.no_reads = true; } auto rocks_db = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); rocks_db_ = rocks_db->raw_db(); cell_db_ = std::move(rocks_db); if (!opts_->get_celldb_in_memory()) { - boc_ = vm::DynamicBagOfCellsDb::create(); + if (opts_->get_celldb_v2()) { + boc_ = vm::DynamicBagOfCellsDb::create_v2(*boc_v2_options); + } else { + boc_ = vm::DynamicBagOfCellsDb::create(*boc_v1_options); + } boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth()); boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); - td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); } + auto meta = boc_->meta_get_all().move_as_ok(); + size_t missing_roots = 0; + size_t unknown_roots = 0; + std::set root_hashes; + for (auto [k, v] : meta) { + if (k == "desczero") { + continue; + } + auto obj = fetch_tl_object(td::BufferSlice{v}, true); + obj.ensure(); + auto entry = DbEntry{obj.move_as_ok()}; + root_hashes.insert(vm::CellHash::from_slice(entry.root_hash.as_slice())); + auto cell = boc_->load_cell(entry.root_hash.as_slice()); + missing_roots += cell.is_error(); + LOG_IF(ERROR, cell.is_error()) << "Cannot load root from meta: " << entry.block_id.to_str() << " " << cell.error(); + } + auto known_roots = boc_->load_known_roots().move_as_ok(); + for (auto& root : known_roots) { + block::gen::ShardStateUnsplit::Record info; + block::gen::OutMsgQueueInfo::Record qinfo; + block::ShardId shard; + if (!(tlb::unpack_cell(root, info) && shard.deserialize(info.shard_id.write()) && + tlb::unpack_cell(info.out_msg_queue_info, qinfo))) { + LOG(FATAL) << "cannot create ShardDescr from a root in celldb"; + } + if (!root_hashes.contains(root->get_hash())) { + unknown_roots++; + LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no; + constexpr bool delete_unknown_roots = false; + if (delete_unknown_roots) { + vm::CellStorer stor{*cell_db_}; + cell_db_->begin_write_batch().ensure(); + boc_->dec(root); + boc_->commit(stor).ensure(); + cell_db_->commit_write_batch().ensure(); + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + } + LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no << " REMOVED"; + } + } + } + + LOG_IF(ERROR, missing_roots != 1) << "Missing root hashes: " << missing_roots; + LOG_IF(ERROR, unknown_roots != 0) << "Unknown roots: " << unknown_roots; + + LOG_IF(FATAL, missing_roots > 1) << "Missing root hashes: " << missing_roots; + LOG_IF(FATAL, unknown_roots != 0) << "Unknown roots: " << unknown_roots; + alarm_timestamp() = td::Timestamp::in(10.0); auto empty = get_empty_key_hash(); if (get_block(empty).is_error()) { DbEntry e{get_empty_key(), empty, empty, RootHash::zero()}; + vm::CellStorer stor{*cell_db_}; cell_db_->begin_write_batch().ensure(); set_block(empty, std::move(e)); + boc_->commit(stor); cell_db_->commit_write_batch().ensure(); } + if (opts_->get_celldb_v2() || opts_->get_celldb_in_memory()) { + send_closure(parent_, &CellDb::set_thread_safe_boc, boc_); + } else { + send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + } + if (opts_->get_celldb_preload_all()) { // Iterate whole DB in a separate thread delay_action( @@ -161,7 +300,7 @@ void CellDbIn::start_up() { { std::string key = "stats.last_deleted_mc_seqno", value; - auto R = cell_db_->get(td::as_slice(key), value); + auto R = boc_->meta_get(td::as_slice(key), value); R.ensure(); if (R.ok() == td::KeyValue::GetStatus::Ok) { auto r_value = td::to_integer_safe(value); @@ -240,10 +379,10 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi td::Timer timer_write; vm::CellStorer stor{*cell_db_}; cell_db_->begin_write_batch().ensure(); - boc_->commit(stor).ensure(); set_block(get_empty_key_hash(), std::move(E)); set_block(D.prev, std::move(P)); set_block(key_hash, std::move(D)); + boc_->commit(stor).ensure(); cell_db_->commit_write_batch().ensure(); timer_write.pause(); @@ -266,11 +405,10 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi void CellDbIn::get_cell_db_reader(td::Promise> promise) { if (db_busy_) { - action_queue_.push( - [self = this, promise = std::move(promise)](td::Result R) mutable { - R.ensure(); - self->get_cell_db_reader(std::move(promise)); - }); + action_queue_.push([self = this, promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + self->get_cell_db_reader(std::move(promise)); + }); return; } promise.set_result(boc_->get_cell_db_reader()); @@ -440,9 +578,16 @@ void CellDbIn::gc_cont2(BlockHandle handle) { timer_get_keys.reset(); td::PerfWarningTimer timer_boc{"gccell_boc", 0.05}; - auto cell = boc_->load_cell(F.root_hash.as_slice()).move_as_ok(); + auto r_cell = boc_->load_cell(F.root_hash.as_slice()); + td::Ref cell; + if (r_cell.is_ok()) { + cell = r_cell.move_as_ok(); + boc_->dec(cell); + LOG(ERROR) << "GC of " << handle->id().to_str(); + } else { + LOG(ERROR) << "GC of UNKNOWN root: " << handle->id().to_str(); + } - boc_->dec(cell); db_busy_ = true; boc_->prepare_commit_async( async_executor, [this, SelfId = actor_id(this), timer_boc = std::move(timer_boc), F = std::move(F), key_hash, @@ -458,17 +603,19 @@ void CellDbIn::gc_cont2(BlockHandle handle) { td::PerfWarningTimer timer_write_batch{"gccell_write_batch", 0.05}; cell_db_->begin_write_batch().ensure(); - boc_->commit(stor).ensure(); - cell_db_->erase(get_key(key_hash)).ensure(); + boc_->meta_erase(get_key(key_hash)).ensure(); set_block(F.prev, std::move(P)); set_block(F.next, std::move(N)); if (handle->id().is_masterchain()) { last_deleted_mc_state_ = handle->id().seqno(); std::string key = "stats.last_deleted_mc_seqno", value = td::to_string(last_deleted_mc_state_); - cell_db_->set(td::as_slice(key), td::as_slice(value)); + boc_->meta_set(td::as_slice(key), td::as_slice(value)); } + + boc_->commit(stor).ensure(); cell_db_->commit_write_batch().ensure(); + alarm_timestamp() = td::Timestamp::now(); timer_write_batch.reset(); @@ -530,7 +677,7 @@ CellDbIn::KeyHash CellDbIn::get_empty_key_hash() { td::Result CellDbIn::get_block(KeyHash key_hash) { const auto key = get_key(key_hash); std::string value; - auto R = cell_db_->get(td::as_slice(key), value); + auto R = boc_->meta_get(td::as_slice(key), value); R.ensure(); auto S = R.move_as_ok(); if (S == td::KeyValue::GetStatus::NotFound) { @@ -543,7 +690,7 @@ td::Result CellDbIn::get_block(KeyHash key_hash) { void CellDbIn::set_block(KeyHash key_hash, DbEntry e) { const auto key = get_key(key_hash); - cell_db_->set(td::as_slice(key), e.release()).ensure(); + boc_->meta_set(td::as_slice(key), e.release()); } void CellDbIn::migrate_cell(td::Bits256 hash) { @@ -631,12 +778,14 @@ void CellDb::alarm() { } void CellDb::load_cell(RootHash hash, td::Promise> promise) { - if (in_memory_boc_) { - auto result = in_memory_boc_->load_root_thread_safe(hash.as_slice()); + if (thread_safe_boc_) { + auto result = thread_safe_boc_->load_root_thread_safe(hash.as_slice()); if (result.is_ok()) { return async_apply("load_cell_result", std::move(promise), std::move(result)); } else { LOG(ERROR) << "load_root_thread_safe failed - this is suspicious"; + send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); + return; } } if (!started_) { @@ -710,6 +859,13 @@ std::vector> CellDbIn::CellDbStatistics::pre for (auto& [key, value] : boc_stats_->custom_stats) { stats.emplace_back(key, value); } + + for (auto& [key, value] : boc_stats_->named_stats.stats_str) { + stats.emplace_back(key, value); + } + for (auto& [key, value] : boc_stats_->named_stats.stats_int) { + stats.emplace_back(key, td::to_string(value)); + } } return stats; } diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 5639b9748b..8d97e13c7e 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -195,13 +195,13 @@ class CellDb : public CellDbBase { started_ = true; boc_->set_loader(std::make_unique(std::move(snapshot), on_load_callback_)).ensure(); } - void set_in_memory_boc(std::shared_ptr in_memory_boc) { - CHECK(opts_->get_celldb_in_memory()); + void set_thread_safe_boc(std::shared_ptr thread_safe_boc) { + CHECK(opts_->get_celldb_in_memory() || opts_->get_celldb_v2()); if (!started_) { alarm(); } started_ = true; - in_memory_boc_ = std::move(in_memory_boc); + thread_safe_boc_ = std::move(thread_safe_boc); } void get_cell_db_reader(td::Promise> promise); @@ -219,7 +219,7 @@ class CellDb : public CellDbBase { td::actor::ActorOwn cell_db_; std::unique_ptr boc_; - std::shared_ptr in_memory_boc_; + std::shared_ptr thread_safe_boc_; bool started_ = false; std::vector> prepared_stats_{{"started", "false"}}; diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index ace6b1066f..45b8d7ec20 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -139,6 +139,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool get_celldb_in_memory() const override { return celldb_in_memory_; } + bool get_celldb_v2() const override { + return celldb_v2_; + } td::optional get_catchain_max_block_delay() const override { return catchain_max_block_delay_; } @@ -237,6 +240,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_celldb_in_memory(bool value) override { celldb_in_memory_ = value; } + void set_celldb_v2(bool value) override { + celldb_v2_ = value; + } void set_catchain_max_block_delay(double value) override { catchain_max_block_delay_ = value; } @@ -304,6 +310,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool celldb_direct_io_ = false; bool celldb_preload_all_ = false; bool celldb_in_memory_ = false; + bool celldb_v2_ = false; td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; bool state_serializer_enabled_ = true; td::Ref collator_options_{true}; diff --git a/validator/validator.h b/validator/validator.h index 5d6c0173c9..66795ceceb 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -104,6 +104,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual std::string get_session_logs_file() const = 0; virtual td::uint32 get_celldb_compress_depth() const = 0; virtual bool get_celldb_in_memory() const = 0; + virtual bool get_celldb_v2() const = 0; virtual size_t get_max_open_archive_files() const = 0; virtual double get_archive_preload_period() const = 0; virtual bool get_disable_rocksdb_stats() const = 0; @@ -144,6 +145,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_celldb_direct_io(bool value) = 0; virtual void set_celldb_preload_all(bool value) = 0; virtual void set_celldb_in_memory(bool value) = 0; + virtual void set_celldb_v2(bool value) = 0; virtual void set_catchain_max_block_delay(double value) = 0; virtual void set_catchain_max_block_delay_slow(double value) = 0; virtual void set_state_serializer_enabled(bool value) = 0; From 597f3b7020c4671c2007184b45d4fd3798a4d802 Mon Sep 17 00:00:00 2001 From: birydrad <> Date: Wed, 15 Jan 2025 14:25:25 +0000 Subject: [PATCH 145/524] celldb: add test for load nonexisting cell, test thread safeness of CellUsageTree, fixes --- crypto/test/test-db.cpp | 35 ++++++- crypto/vm/db/DynamicBagOfCellsDb.cpp | 8 +- crypto/vm/db/DynamicBagOfCellsDb.h | 2 +- crypto/vm/db/DynamicBagOfCellsDbV2.cpp | 8 +- crypto/vm/db/InMemoryBagOfCellsDb.cpp | 15 ++- validator/db/celldb.cpp | 122 ++++++++++++++----------- validator/db/celldb.hpp | 1 + 7 files changed, 130 insertions(+), 61 deletions(-) diff --git a/crypto/test/test-db.cpp b/crypto/test/test-db.cpp index bc4dac606e..2b77e71182 100644 --- a/crypto/test/test-db.cpp +++ b/crypto/test/test-db.cpp @@ -50,6 +50,7 @@ #include #include #include +#include #include @@ -1312,7 +1313,8 @@ void with_all_boc_options(F &&f, size_t tests_n, bool single_thread = false) { // V2 - one thread run({.async_executor = executor, .kv_options = kv_options, - .options = DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 0, .executor = executor, .cache_ttl_max = 5}}); + .options = + DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 0, .executor = executor, .cache_ttl_max = 5}}); // InMemory for (auto use_arena : {false, true}) { @@ -1353,6 +1355,8 @@ DynamicBagOfCellsDb::Stats test_dynamic_boc(BocOptions options) { if (rnd() % 10 == 0) { reload_db(); } + db.dboc->load_cell(vm::CellHash{}.as_slice()).ensure_error(); + db.reset_loader(); Ref old_root; if (!old_root_hash.empty()) { @@ -2901,6 +2905,35 @@ TEST(TonDb, BocRespectsUsageCell) { ASSERT_STREQ(serialization, serialization_of_virtualized_cell); } +TEST(UsageTree, ThreadSafe) { + size_t test_n = 100; + td::Random::Xorshift128plus rnd(123); + for (size_t test_i = 0; test_i < test_n; test_i++) { + auto cell = vm::gen_random_cell(rnd.fast(2, 100), rnd, false); + auto usage_tree = std::make_shared(); + auto usage_cell = vm::UsageCell::create(cell, usage_tree->root_ptr()); + std::ptrdiff_t threads_n = 1; // TODO: when CellUsageTree is thread safe, change it to 4 + auto barrier = std::barrier{threads_n}; + std::vector threads; + std::vector explorations(threads_n); + for (std::ptrdiff_t i = 0; i < threads_n; i++) { + threads.emplace_back([&, i = i]() { + barrier.arrive_and_wait(); + explorations[i] = vm::CellExplorer::random_explore(usage_cell, rnd); + }); + } + for (auto &thread : threads) { + thread.join(); + } + auto proof = vm::MerkleProof::generate(cell, usage_tree.get()); + auto virtualized_proof = vm::MerkleProof::virtualize(proof, 1); + for (auto &exploration : explorations) { + auto new_exploration = vm::CellExplorer::explore(virtualized_proof, exploration.ops); + ASSERT_EQ(exploration.log, new_exploration.log); + } + } +} + /* vm::DynamicBagOfCellsDb::Stats test_dynamic_boc_respects_usage_cell(vm::BocOptions options) { td::Random::Xorshift128plus rnd(options.seed); diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index bd23733e03..d6731b0390 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -107,15 +107,21 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { return get_cell_info_lazy(level_mask, hash, depth).cell; } - td::Result>> meta_get_all() const override { + td::Result>> meta_get_all(size_t max_count) const override { std::vector> result; auto s = loader_->key_value_reader().for_each_in_range("desc", "desd", [&](const td::Slice &key, const td::Slice &value) { + if (result.size() >= max_count) { + return td::Status::Error("COUNT_LIMIT"); + } if (td::begins_with(key, "desc") && key.size() != 32) { result.emplace_back(key.str(), value.str()); } return td::Status::OK(); }); + if (s.message() == "COUNT_LIMIT") { + s = td::Status::OK(); + } TRY_STATUS(std::move(s)); return result; } diff --git a/crypto/vm/db/DynamicBagOfCellsDb.h b/crypto/vm/db/DynamicBagOfCellsDb.h index ca23d72f21..82028f3fe4 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.h +++ b/crypto/vm/db/DynamicBagOfCellsDb.h @@ -51,7 +51,7 @@ class DynamicBagOfCellsDb { public: virtual ~DynamicBagOfCellsDb() = default; - virtual td::Result>> meta_get_all() const = 0; + virtual td::Result>> meta_get_all(size_t max_count) const = 0; virtual td::Result meta_get(td::Slice key, std::string &value) = 0; virtual td::Status meta_set(td::Slice key, td::Slice value) = 0; virtual td::Status meta_erase(td::Slice key) = 0; diff --git a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp index 45fdffee89..eff74e214d 100644 --- a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp @@ -765,16 +765,22 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { } } - td::Result>> meta_get_all() const override { + td::Result>> meta_get_all(size_t max_count) const override { CHECK(meta_db_fixup_.empty()); std::vector> result; auto s = cell_db_reader_->key_value_reader().for_each_in_range( "desc", "desd", [&](const td::Slice &key, const td::Slice &value) { + if (result.size() >= max_count) { + return td::Status::Error("COUNT_LIMIT"); + } if (td::begins_with(key, "desc") && key.size() != 32) { result.emplace_back(key.str(), value.str()); } return td::Status::OK(); }); + if (s.message() == "COUNT_LIMIT") { + s = td::Status::OK(); + } TRY_STATUS(std::move(s)); return result; } diff --git a/crypto/vm/db/InMemoryBagOfCellsDb.cpp b/crypto/vm/db/InMemoryBagOfCellsDb.cpp index b0a30cfb2d..e43cfde4e0 100644 --- a/crypto/vm/db/InMemoryBagOfCellsDb.cpp +++ b/crypto/vm/db/InMemoryBagOfCellsDb.cpp @@ -774,8 +774,15 @@ class MetaStorage { CHECK(p.first.size() != CellTraits::hash_bytes); } } - std::vector> meta_get_all() const { - return td::transform(meta_, [](const auto &p) { return std::make_pair(p.first, p.second); }); + std::vector> meta_get_all(size_t max_count) const { + std::vector> res; + for (const auto &[k, v] : meta_) { + if (res.size() >= max_count) { + break; + } + res.emplace_back(k, v); + } + return res; } KeyValue::GetStatus meta_get(td::Slice key, std::string &value) const { auto lock = local_access_.lock(); @@ -814,8 +821,8 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { : storage_(std::move(storage)), meta_storage_(std::move(meta_storage)) { } - td::Result>> meta_get_all() const override { - return meta_storage_->meta_get_all(); + td::Result>> meta_get_all(size_t max_count) const override { + return meta_storage_->meta_get_all(max_count); } td::Result meta_get(td::Slice key, std::string &value) override { CHECK(key.size() != CellTraits::hash_bytes); diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 5fc5a83a4e..90c659cc46 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -111,6 +111,66 @@ struct MergeOperatorAddCellRefcnt : public rocksdb::MergeOperator { } }; +void CellDbIn::validate_meta() { + LOG(INFO) << "Validating metadata\n"; + size_t max_meta_keys_loaded = opts_->get_celldb_in_memory() ? std::numeric_limits::max() : 10000; + auto meta = boc_->meta_get_all(max_meta_keys_loaded).move_as_ok(); + bool partial_check = meta.size() == max_meta_keys_loaded; + if (partial_check) { + LOG(ERROR) << "Too much metadata in the database, do only partial check"; + } + size_t missing_roots = 0; + size_t unknown_roots = 0; + std::set root_hashes; + for (auto [k, v] : meta) { + if (k == "desczero") { + continue; + } + auto obj = fetch_tl_object(td::BufferSlice{v}, true); + obj.ensure(); + auto entry = DbEntry{obj.move_as_ok()}; + root_hashes.insert(vm::CellHash::from_slice(entry.root_hash.as_slice())); + auto cell = boc_->load_cell(entry.root_hash.as_slice()); + missing_roots += cell.is_error(); + LOG_IF(ERROR, cell.is_error()) << "Cannot load root from meta: " << entry.block_id.to_str() << " " << cell.error(); + } + + // load_known_roots is only supported by InMemory database, so it is ok to check all known roots here + auto known_roots = boc_->load_known_roots().move_as_ok(); + for (auto& root : known_roots) { + block::gen::ShardStateUnsplit::Record info; + block::gen::OutMsgQueueInfo::Record qinfo; + block::ShardId shard; + if (!(tlb::unpack_cell(root, info) && shard.deserialize(info.shard_id.write()) && + tlb::unpack_cell(info.out_msg_queue_info, qinfo))) { + LOG(FATAL) << "cannot create ShardDescr from a root in celldb"; + } + if (!partial_check && !root_hashes.contains(root->get_hash())) { + unknown_roots++; + LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no; + constexpr bool delete_unknown_roots = false; + if (delete_unknown_roots) { + vm::CellStorer stor{*cell_db_}; + cell_db_->begin_write_batch().ensure(); + boc_->dec(root); + boc_->commit(stor).ensure(); + cell_db_->commit_write_batch().ensure(); + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + } + LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no << " REMOVED"; + } + } + } + + LOG_IF(ERROR, missing_roots != 0) << "Missing root hashes: " << missing_roots; + LOG_IF(ERROR, unknown_roots != 0) << "Unknown roots: " << unknown_roots; + + LOG_IF(FATAL, missing_roots != 0) << "Missing root hashes: " << missing_roots; + LOG_IF(FATAL, unknown_roots != 0) << "Unknown roots: " << unknown_roots; + LOG(INFO) << "Validating metadata: OK\n"; +} + void CellDbIn::start_up() { on_load_callback_ = [actor = std::make_shared>( td::actor::create_actor("celldbmigration", actor_id(this))), @@ -141,11 +201,11 @@ void CellDbIn::start_up() { std::optional boc_v2_options; if (opts_->get_celldb_v2()) { - boc_v2_options = - vm::DynamicBagOfCellsDb::CreateV2Options{.extra_threads = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 8u), - .executor = {}, - .cache_ttl_max = 2000, - .cache_size_max = 1000000}; + boc_v2_options = vm::DynamicBagOfCellsDb::CreateV2Options{ + .extra_threads = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 8u), + .executor = {}, + .cache_ttl_max = 2000, + .cache_size_max = 1000000}; size_t min_rocksdb_cache = std::max(size_t{1} << 30, boc_v2_options->cache_size_max * 5000); if (!o_celldb_cache_size || o_celldb_cache_size.value() < min_rocksdb_cache) { LOG(WARNING) << "Increase CellDb block cache size to " << td::format::as_size(min_rocksdb_cache) << " from " @@ -208,54 +268,7 @@ void CellDbIn::start_up() { boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); } - auto meta = boc_->meta_get_all().move_as_ok(); - size_t missing_roots = 0; - size_t unknown_roots = 0; - std::set root_hashes; - for (auto [k, v] : meta) { - if (k == "desczero") { - continue; - } - auto obj = fetch_tl_object(td::BufferSlice{v}, true); - obj.ensure(); - auto entry = DbEntry{obj.move_as_ok()}; - root_hashes.insert(vm::CellHash::from_slice(entry.root_hash.as_slice())); - auto cell = boc_->load_cell(entry.root_hash.as_slice()); - missing_roots += cell.is_error(); - LOG_IF(ERROR, cell.is_error()) << "Cannot load root from meta: " << entry.block_id.to_str() << " " << cell.error(); - } - auto known_roots = boc_->load_known_roots().move_as_ok(); - for (auto& root : known_roots) { - block::gen::ShardStateUnsplit::Record info; - block::gen::OutMsgQueueInfo::Record qinfo; - block::ShardId shard; - if (!(tlb::unpack_cell(root, info) && shard.deserialize(info.shard_id.write()) && - tlb::unpack_cell(info.out_msg_queue_info, qinfo))) { - LOG(FATAL) << "cannot create ShardDescr from a root in celldb"; - } - if (!root_hashes.contains(root->get_hash())) { - unknown_roots++; - LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no; - constexpr bool delete_unknown_roots = false; - if (delete_unknown_roots) { - vm::CellStorer stor{*cell_db_}; - cell_db_->begin_write_batch().ensure(); - boc_->dec(root); - boc_->commit(stor).ensure(); - cell_db_->commit_write_batch().ensure(); - if (!opts_->get_celldb_in_memory()) { - boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); - } - LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no << " REMOVED"; - } - } - } - - LOG_IF(ERROR, missing_roots != 1) << "Missing root hashes: " << missing_roots; - LOG_IF(ERROR, unknown_roots != 0) << "Unknown roots: " << unknown_roots; - - LOG_IF(FATAL, missing_roots > 1) << "Missing root hashes: " << missing_roots; - LOG_IF(FATAL, unknown_roots != 0) << "Unknown roots: " << unknown_roots; + validate_meta(); alarm_timestamp() = td::Timestamp::in(10.0); @@ -267,6 +280,9 @@ void CellDbIn::start_up() { set_block(empty, std::move(e)); boc_->commit(stor); cell_db_->commit_write_batch().ensure(); + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + } } if (opts_->get_celldb_v2() || opts_->get_celldb_in_memory()) { diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 8d97e13c7e..1e1ccddabf 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -74,6 +74,7 @@ class CellDbIn : public CellDbBase { CellDbIn(td::actor::ActorId root_db, td::actor::ActorId parent, std::string path, td::Ref opts); + void validate_meta(); void start_up() override; void alarm() override; From 57d7c2a8958de3a30a319887b18458545378d964 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 28 Feb 2025 18:00:36 +0300 Subject: [PATCH 146/524] New account storage stat --- common/global-version.h | 2 +- crypto/CMakeLists.txt | 2 + crypto/block/account-storage-stat.cpp | 172 ++++++++++++++++++++ crypto/block/account-storage-stat.h | 119 ++++++++++++++ crypto/block/block-parse.cpp | 45 +++--- crypto/block/block-parse.h | 7 - crypto/block/block.tlb | 17 +- crypto/block/create-state.cpp | 7 +- crypto/block/mc-config.h | 6 +- crypto/block/transaction.cpp | 225 +++++++++++++------------- crypto/block/transaction.h | 12 +- crypto/vm/boc.h | 8 +- crypto/vm/cells/CellSlice.cpp | 8 + crypto/vm/cells/CellSlice.h | 1 + doc/GlobalVersions.md | 7 + tonlib/tonlib/TonlibClient.cpp | 15 +- validator/impl/validate-query.cpp | 1 + 17 files changed, 478 insertions(+), 176 deletions(-) create mode 100644 crypto/block/account-storage-stat.cpp create mode 100644 crypto/block/account-storage-stat.h diff --git a/common/global-version.h b/common/global-version.h index 2308ce3e9c..b54a3bdc52 100644 --- a/common/global-version.h +++ b/common/global-version.h @@ -19,6 +19,6 @@ namespace ton { // See doc/GlobalVersions.md -constexpr int SUPPORTED_VERSION = 10; +constexpr int SUPPORTED_VERSION = 11; } diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 069083381c..59b43d13ab 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -214,6 +214,8 @@ set(BLOCK_SOURCE block/mc-config.cpp block/output-queue-merger.cpp block/transaction.cpp + block/account-storage-stat.h + block/account-storage-stat.cpp block/precompiled-smc/PrecompiledSmartContract.cpp ${TLB_BLOCK_AUTO} diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp new file mode 100644 index 0000000000..8e8b6f7e16 --- /dev/null +++ b/crypto/block/account-storage-stat.cpp @@ -0,0 +1,172 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "account-storage-stat.h" + +namespace block { + +AccountStorageStat::AccountStorageStat() : AccountStorageStat({}, {}, 0, 0) { +} + +AccountStorageStat::AccountStorageStat(Ref dict_root, std::vector> roots, + td::uint64 total_cells, td::uint64 total_bits) + : dict_(std::move(dict_root), 256), roots_(std::move(roots)), total_cells_(total_cells), total_bits_(total_bits) { +} + +AccountStorageStat::AccountStorageStat(const AccountStorageStat& other) + : AccountStorageStat(other.dict_.get_root_cell(), other.roots_, other.total_cells_, other.total_bits_) { +} + +AccountStorageStat::AccountStorageStat(AccountStorageStat&& other) + : AccountStorageStat(other.dict_.get_root_cell(), std::move(other.roots_), other.total_cells_, other.total_bits_) { + cache_ = std::move(other.cache_); +} + +AccountStorageStat& AccountStorageStat::operator=(const AccountStorageStat& other) { + dict_ = other.dict_; + total_cells_ = other.total_cells_; + total_bits_ = other.total_bits_; + roots_ = other.roots_; + cache_ = {}; + return *this; +} + +AccountStorageStat& AccountStorageStat::operator=(AccountStorageStat&& other) { + dict_ = std::move(other.dict_); + total_cells_ = other.total_cells_; + total_bits_ = other.total_bits_; + roots_ = std::move(other.roots_); + cache_ = std::move(other.cache_); + return *this; +} + +td::Result AccountStorageStat::add_root(const Ref& cell) { + roots_.push_back(cell); + return add_cell(cell); +} + +td::Status AccountStorageStat::remove_root(const Ref& cell) { + auto it = std::find_if(roots_.begin(), roots_.end(), + [&](const Ref& c) { return c->get_hash() == cell->get_hash(); }); + if (it == roots_.end()) { + return td::Status::Error(PSTRING() << "no such root " << cell->get_hash().to_hex()); + } + roots_.erase(it); + return remove_cell(cell); +} + +td::Result AccountStorageStat::replace_roots(std::vector> new_roots) { + std::vector> old_roots = roots_; + td::uint32 max_merkle_depth = 0; + for (const Ref& root : new_roots) { + if (root.is_null()) { + continue; + } + TRY_RESULT(info, add_root(root)); + max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); + } + for (const Ref& root : old_roots) { + TRY_STATUS(remove_root(root)); + } + return CellInfo{max_merkle_depth}; +} + +td::Result AccountStorageStat::add_cell(const Ref& cell) { + Entry& e = get_entry(cell); + ++e.refcnt; + if (e.refcnt == 0) { + return td::Status::Error(PSTRING() << "cell " << cell->get_hash().to_hex() << ": refcnt overflow"); + } + if (e.refcnt != 1) { + update_dict(e); + return CellInfo{e.max_merkle_depth}; + } + td::uint32 max_merkle_depth = 0; + vm::CellSlice cs{vm::NoVm{}, cell}; + ++total_cells_; + total_bits_ += cs.size(); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + TRY_RESULT(info, add_cell(cs.prefetch_ref(i))); + max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); + } + if (cs.special_type() == vm::CellTraits::SpecialType::MerkleProof || + cs.special_type() == vm::CellTraits::SpecialType::MerkleUpdate) { + ++max_merkle_depth; + } + max_merkle_depth = std::min(max_merkle_depth, MERKLE_DEPTH_LIMIT); + Entry& e2 = get_entry(cell); + e2.max_merkle_depth = max_merkle_depth; + update_dict(e2); + return CellInfo{max_merkle_depth}; +} + +td::Status AccountStorageStat::remove_cell(const Ref& cell) { + Entry& e = get_entry(cell); + if (e.refcnt == 0) { + return td::Status::Error(PSTRING() << "cell " << cell->get_hash().to_hex() << " is not in the dict"); + } + --e.refcnt; + update_dict(e); + if (e.refcnt != 0) { + return td::Status::OK(); + } + vm::CellSlice cs{vm::NoVm{}, std::move(cell)}; + if (total_cells_ == 0 || total_bits_ < cs.size()) { + return td::Status::Error("total_cell/total_bits becomes negative"); + } + --total_cells_; + total_bits_ -= cs.size(); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + TRY_STATUS(remove_cell(cs.prefetch_ref(i))); + } + return td::Status::OK(); +} + +bool AccountStorageStat::Entry::serialize(vm::CellBuilder& cb) const { + return cb.store_long_bool(refcnt, 32) && cb.store_long_bool(max_merkle_depth, 2); +} + +void AccountStorageStat::Entry::fetch(Ref cs) { + if (cs.is_null()) { + refcnt = max_merkle_depth = 0; + } else { + refcnt = (td::uint32)cs.write().fetch_ulong(32); + max_merkle_depth = (td::uint32)cs.write().fetch_ulong(2); + } +} + +AccountStorageStat::Entry& AccountStorageStat::get_entry(const Ref& cell) { + return cache_.apply(cell->get_hash().as_slice(), [&](Entry& e) { + if (e.inited) { + return; + } + e.inited = true; + e.hash = cell->get_hash(); + e.fetch(dict_.lookup(e.hash.as_bitslice())); + }); +} + +void AccountStorageStat::update_dict(const Entry& e) { + if (e.refcnt == 0) { + dict_.lookup_delete(e.hash.as_bitslice()); + } else { + vm::CellBuilder cb; + CHECK(e.serialize(cb)); + dict_.set_builder(e.hash.as_bitslice(), cb); + } +} + +} // namespace block \ No newline at end of file diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h new file mode 100644 index 0000000000..894192d4f9 --- /dev/null +++ b/crypto/block/account-storage-stat.h @@ -0,0 +1,119 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#pragma once +#include "common/refcnt.hpp" +#include "vm/dict.h" +#include "ton/ton-types.h" +#include "ton/ton-shard.h" +#include "common/bitstring.h" +#include "block.h" +#include "vm/db/CellHashTable.h" + +namespace block { +using td::Ref; + +class AccountStorageStat { + public: + struct CellInfo { + td::uint32 max_merkle_depth = 0; + }; + + AccountStorageStat(); + AccountStorageStat(Ref dict_root, std::vector> roots, td::uint64 total_cells, + td::uint64 total_bits); + AccountStorageStat(const AccountStorageStat &other); + AccountStorageStat(AccountStorageStat &&other); + ~AccountStorageStat() = default; + + AccountStorageStat &operator=(const AccountStorageStat &other); + AccountStorageStat &operator=(AccountStorageStat &&other); + + td::uint64 get_total_cells() const { + return total_cells_; + } + + td::uint64 get_total_bits() const { + return total_bits_; + } + + Ref get_dict_root() const { + return dict_.get_root_cell(); + } + + td::Bits256 get_dict_hash() const { + return dict_.is_empty() ? td::Bits256::zero() : td::Bits256{dict_.get_root_cell()->get_hash().bits()}; + } + + td::Result add_root(const Ref &cell); + td::Status remove_root(const Ref &cell); + td::Result replace_roots(std::vector> new_roots); + + private: + vm::Dictionary dict_; + td::uint64 total_cells_, total_bits_; + std::vector> roots_; + + td::Result add_cell(const Ref &cell); + td::Status remove_cell(const Ref &cell); + + struct Entry { + bool inited = false; + vm::Cell::Hash hash; + td::uint32 refcnt = 0; + td::uint32 max_merkle_depth = 0; + + void fetch(Ref cs); + bool serialize(vm::CellBuilder &cb) const; + + vm::Cell::Hash key() const { + return hash; + } + bool operator<(const Entry &other) const { + return key() < other.key(); + } + struct Eq { + using is_transparent = void; // Pred to use + bool operator()(const Entry &info, const Entry &other_info) const { + return info.key() == other_info.key(); + } + bool operator()(const Entry &info, td::Slice hash) const { + return info.key().as_slice() == hash; + } + bool operator()(td::Slice hash, const Entry &info) const { + return info.key().as_slice() == hash; + } + }; + struct Hash { + using is_transparent = void; // Pred to use + using transparent_key_equal = Eq; + size_t operator()(td::Slice hash) const { + return cell_hash_slice_hash(hash); + } + size_t operator()(const Entry &info) const { + return cell_hash_slice_hash(info.key().as_slice()); + } + }; + }; + vm::CellHashTable cache_; + + Entry &get_entry(const Ref &cell); + void update_dict(const Entry &e); + + static constexpr td::uint32 MERKLE_DEPTH_LIMIT = 3; +}; + +} // namespace block diff --git a/crypto/block/block-parse.cpp b/crypto/block/block-parse.cpp index 50851c7956..6d645ac79d 100644 --- a/crypto/block/block-parse.cpp +++ b/crypto/block/block-parse.cpp @@ -960,41 +960,34 @@ const MsgEnvelope t_MsgEnvelope; const RefTo t_Ref_MsgEnvelope; bool StorageUsed::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { - return t_VarUInteger_7.validate_skip(ops, cs, weak) // cells:(VarUInteger 7) - && t_VarUInteger_7.validate_skip(ops, cs, weak) // bits:(VarUInteger 7) - && t_VarUInteger_7.validate_skip(ops, cs, weak); // public_cells:(VarUInteger 7) -} - -bool StorageUsed::skip(vm::CellSlice& cs) const { - return t_VarUInteger_7.skip(cs) // cells:(VarUInteger 7) - && t_VarUInteger_7.skip(cs) // bits:(VarUInteger 7) - && t_VarUInteger_7.skip(cs); // public_cells:(VarUInteger 7) -} - -const StorageUsed t_StorageUsed; - -bool StorageUsedShort::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return t_VarUInteger_7.validate_skip(ops, cs, weak) // cells:(VarUInteger 7) && t_VarUInteger_7.validate_skip(ops, cs, weak); // bits:(VarUInteger 7) } -bool StorageUsedShort::skip(vm::CellSlice& cs) const { +bool StorageUsed::skip(vm::CellSlice& cs) const { return t_VarUInteger_7.skip(cs) // cells:(VarUInteger 7) && t_VarUInteger_7.skip(cs); // bits:(VarUInteger 7) } -const StorageUsedShort t_StorageUsedShort; +const StorageUsed t_StorageUsed; const Maybe t_Maybe_Grams; bool StorageInfo::skip(vm::CellSlice& cs) const { - return t_StorageUsed.skip(cs) // used:StorageUsed - && cs.advance(32) // last_paid:uint32 - && t_Maybe_Grams.skip(cs); // due_payment:(Maybe Grams) + int extra_tag = 0; + return t_StorageUsed.skip(cs) // used:StorageUsed + && cs.fetch_uint_to(3, extra_tag) // storage_extra:StorageExtraInfo + && (extra_tag == 0 || cs.advance(256)) // storage_extra_info$001 dict_hash:uint256 = StorageExtraInfo; + && cs.advance(32) // last_paid:uint32 + && t_Maybe_Grams.skip(cs); // due_payment:(Maybe Grams) } bool StorageInfo::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { - return t_StorageUsed.validate_skip(ops, cs, weak) // used:StorageUsed + int extra_tag = 0; + return t_StorageUsed.validate_skip(ops, cs, weak) // used:StorageUsed + && cs.fetch_uint_to(3, extra_tag) // storage_extra:StorageExtraInfo + && (extra_tag == 0 || + (extra_tag == 1 && cs.advance(256))) // storage_extra_info$001 dict_hash:uint256 = StorageExtraInfo; && cs.advance(32) // last_paid:uint32 && t_Maybe_Grams.validate_skip(ops, cs, weak); // due_payment:(Maybe Grams) } @@ -1368,7 +1361,7 @@ bool TrActionPhase::skip(vm::CellSlice& cs) const { && cs.advance(16 * 4 + 256) // tot_actions:uint16 spec_actions:uint16 // skipped_actions:uint16 msgs_created:uint16 // action_list_hash:uint256 - && t_StorageUsedShort.skip(cs); // tot_msg_size:StorageUsedShort + && t_StorageUsed.skip(cs); // tot_msg_size:StorageUsed } bool TrActionPhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { @@ -1381,7 +1374,7 @@ bool TrActionPhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const && cs.advance(16 * 4 + 256) // tot_actions:uint16 spec_actions:uint16 // skipped_actions:uint16 msgs_created:uint16 // action_list_hash:uint256 - && t_StorageUsedShort.validate_skip(ops, cs, weak); // tot_msg_size:StorageUsed + && t_StorageUsed.validate_skip(ops, cs, weak); // tot_msg_size:StorageUsed } const TrActionPhase t_TrActionPhase; @@ -1392,11 +1385,11 @@ bool TrBouncePhase::skip(vm::CellSlice& cs) const { return cs.advance(2); // tr_phase_bounce_negfunds$00 case tr_phase_bounce_nofunds: return cs.advance(2) // tr_phase_bounce_nofunds$01 - && t_StorageUsedShort.skip(cs) // msg_size:StorageUsedShort + && t_StorageUsed.skip(cs) // msg_size:StorageUsed && t_Grams.skip(cs); // req_fwd_fees:Grams case tr_phase_bounce_ok: return cs.advance(1) // tr_phase_bounce_ok$1 - && t_StorageUsedShort.skip(cs) // msg_size:StorageUsedShort + && t_StorageUsed.skip(cs) // msg_size:StorageUsed && t_Grams.skip(cs) // msg_fees:Grams && t_Grams.skip(cs); // fwd_fees:Grams } @@ -1409,11 +1402,11 @@ bool TrBouncePhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const return cs.advance(2); // tr_phase_bounce_negfunds$00 case tr_phase_bounce_nofunds: return cs.advance(2) // tr_phase_bounce_nofunds$01 - && t_StorageUsedShort.validate_skip(ops, cs, weak) // msg_size:StorageUsedShort + && t_StorageUsed.validate_skip(ops, cs, weak) // msg_size:StorageUsed && t_Grams.validate_skip(ops, cs, weak); // req_fwd_fees:Grams case tr_phase_bounce_ok: return cs.advance(1) // tr_phase_bounce_ok$1 - && t_StorageUsedShort.validate_skip(ops, cs, weak) // msg_size:StorageUsedShort + && t_StorageUsed.validate_skip(ops, cs, weak) // msg_size:StorageUsed && t_Grams.validate_skip(ops, cs, weak) // msg_fees:Grams && t_Grams.validate_skip(ops, cs, weak); // fwd_fees:Grams } diff --git a/crypto/block/block-parse.h b/crypto/block/block-parse.h index 65f8b91fe2..fd17c6579f 100644 --- a/crypto/block/block-parse.h +++ b/crypto/block/block-parse.h @@ -493,13 +493,6 @@ struct StorageUsed final : TLB_Complex { extern const StorageUsed t_StorageUsed; -struct StorageUsedShort final : TLB_Complex { - bool skip(vm::CellSlice& cs) const override; - bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override; -}; - -extern const StorageUsedShort t_StorageUsedShort; - struct StorageInfo final : TLB_Complex { bool skip(vm::CellSlice& cs) const override; bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override; diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 4a8bbc065d..5cba3c69f1 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -246,14 +246,13 @@ out_msg_queue_extra#0 dispatch_queue:DispatchQueue out_queue_size:(Maybe uint48) _ out_queue:OutMsgQueue proc_info:ProcessedInfo extra:(Maybe OutMsgQueueExtra) = OutMsgQueueInfo; -// -storage_used$_ cells:(VarUInteger 7) bits:(VarUInteger 7) - public_cells:(VarUInteger 7) = StorageUsed; -storage_used_short$_ cells:(VarUInteger 7) - bits:(VarUInteger 7) = StorageUsedShort; +storage_extra_none$000 = StorageExtraInfo; +storage_extra_info$001 dict_hash:uint256 = StorageExtraInfo; + +storage_used$_ cells:(VarUInteger 7) bits:(VarUInteger 7) = StorageUsed; -storage_info$_ used:StorageUsed last_paid:uint32 +storage_info$_ used:StorageUsed storage_extra:StorageExtraInfo last_paid:uint32 due_payment:(Maybe Grams) = StorageInfo; account_none$0 = Account; @@ -341,13 +340,13 @@ tr_phase_action$_ success:Bool valid:Bool no_funds:Bool total_fwd_fees:(Maybe Grams) total_action_fees:(Maybe Grams) result_code:int32 result_arg:(Maybe int32) tot_actions:uint16 spec_actions:uint16 skipped_actions:uint16 msgs_created:uint16 - action_list_hash:bits256 tot_msg_size:StorageUsedShort + action_list_hash:bits256 tot_msg_size:StorageUsed = TrActionPhase; tr_phase_bounce_negfunds$00 = TrBouncePhase; -tr_phase_bounce_nofunds$01 msg_size:StorageUsedShort +tr_phase_bounce_nofunds$01 msg_size:StorageUsed req_fwd_fees:Grams = TrBouncePhase; -tr_phase_bounce_ok$1 msg_size:StorageUsedShort +tr_phase_bounce_ok$1 msg_size:StorageUsed msg_fees:Grams fwd_fees:Grams = TrBouncePhase; // trans_ord$0000 credit_first:Bool diff --git a/crypto/block/create-state.cpp b/crypto/block/create-state.cpp index c8c8b970da..4a74ac0f60 100644 --- a/crypto/block/create-state.cpp +++ b/crypto/block/create-state.cpp @@ -338,10 +338,11 @@ td::RefInt256 create_smartcontract(td::RefInt256 smc_addr, Ref code, R PDO(cb.store_long_rchk_bool(workchain_id, ctor == 2 ? 8 : 32) && cb.store_bits_bool(addr.cbits(), 256)); THRERR("Cannot serialize addr:MsgAddressInt of the new smart contract"); // storage_stat:StorageInfo -> storage_stat.used:StorageUsed - PDO(block::store_UInt7(cb, stats.cells) // cells:(VarUInteger 7) - && block::store_UInt7(cb, stats.bits) // bits:(VarUInteger 7) - && block::store_UInt7(cb, stats.public_cells)); // public_cells:(VarUInteger 7) + PDO(block::store_UInt7(cb, stats.cells) // cells:(VarUInteger 7) + && block::store_UInt7(cb, stats.bits)) // bits:(VarUInteger 7) THRERR("Cannot serialize used:StorageUsed of the new smart contract"); + PDO(cb.store_zeroes_bool(3)); // extra:StorageExtraInfo + THRERR("Cannot serialize storage_extra:StorageExtraInfo of the new smart contract"); PDO(cb.store_long_bool(0, 33)); // last_paid:uint32 due_payment:(Maybe Grams) PDO(cb.append_data_cell_bool(storage)); // storage:AccountStorage THRERR("Cannot create Account of the new smart contract"); diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 98e6a26df1..bb4825eadb 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -156,6 +156,10 @@ class McShardHashI : public td::CntObject { virtual bool before_merge() const = 0; }; +struct StorageUsed { + td::uint64 cells = 0, bits = 0; +}; + struct McShardHash : public McShardHashI { ton::BlockIdExt blk_; ton::LogicalTime start_lt_, end_lt_; @@ -336,7 +340,7 @@ struct StoragePrices { , mc_cell_price(_mc_cprice) { } static td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector& pricing, - const vm::CellStorageStat& storage_stat, ton::UnixTime last_paid, + const StorageUsed& storage_used, ton::UnixTime last_paid, bool is_special, bool is_masterchain); }; diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 34d2351144..4e92b8f6cc 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -257,6 +257,11 @@ bool Account::unpack_storage_info(vm::CellSlice& cs) { return false; } last_paid = info.last_paid; + if (info.storage_extra.write().fetch_long(3) == 1) { + info.storage_extra->prefetch_bits_to(storage_dict_hash.value_force()); + } else { + storage_dict_hash = {}; + } if (info.due_payment->prefetch_ulong(1) == 1) { vm::CellSlice& cs2 = info.due_payment.write(); cs2.advance(1); @@ -268,11 +273,9 @@ bool Account::unpack_storage_info(vm::CellSlice& cs) { due_payment = td::zero_refint(); } unsigned long long u = 0; - u |= storage_stat.cells = block::tlb::t_VarUInteger_7.as_uint(*used.cells); - u |= storage_stat.bits = block::tlb::t_VarUInteger_7.as_uint(*used.bits); - u |= storage_stat.public_cells = block::tlb::t_VarUInteger_7.as_uint(*used.public_cells); - LOG(DEBUG) << "last_paid=" << last_paid << "; cells=" << storage_stat.cells << " bits=" << storage_stat.bits - << " public_cells=" << storage_stat.public_cells; + u |= storage_used.cells = block::tlb::t_VarUInteger_7.as_uint(*used.cells); + u |= storage_used.bits = block::tlb::t_VarUInteger_7.as_uint(*used.bits); + LOG(DEBUG) << "last_paid=" << last_paid << "; cells=" << storage_used.cells << " bits=" << storage_used.bits; return (u != std::numeric_limits::max()); } @@ -527,7 +530,8 @@ bool Account::init_new(ton::UnixTime now) { last_trans_hash_.set_zero(); now_ = now; last_paid = 0; - storage_stat.clear(); + storage_used = {}; + storage_dict_hash = {}; due_payment = td::zero_refint(); balance.set_zero(); if (my_addr_exact.is_null()) { @@ -617,12 +621,12 @@ bool Account::belongs_to_shard(ton::ShardIdFull shard) const { * @param payment The total sum to be updated. * @param delta The time delta for which the payment is calculated. * @param prices The storage prices. - * @param storage Account storage statistics. + * @param storage_used Account storage statistics. * @param is_mc A flag indicating whether the account is in the masterchain. */ void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, const block::StoragePrices& prices, - const vm::CellStorageStat& storage, bool is_mc) { - td::BigInt256 c{(long long)storage.cells}, b{(long long)storage.bits}; + const StorageUsed& storage_used, bool is_mc) { + td::BigInt256 c{(long long)storage_used.cells}, b{(long long)storage_used.bits}; if (is_mc) { // storage.cells * prices.mc_cell_price + storage.bits * prices.mc_bit_price; c.mul_short(prices.mc_cell_price); @@ -643,7 +647,7 @@ void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, co * * @param now The current Unix time. * @param pricing The vector of storage prices. - * @param storage_stat Account storage statistics. + * @param storage_used Account storage statistics. * @param last_paid The Unix time when the last payment was made. * @param is_special A flag indicating if the account is special. * @param is_masterchain A flag indicating if the account is in the masterchain. @@ -651,7 +655,7 @@ void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, co * @returns The computed storage fees as RefInt256. */ td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std::vector& pricing, - const vm::CellStorageStat& storage_stat, ton::UnixTime last_paid, + const StorageUsed& storage_used, ton::UnixTime last_paid, bool is_special, bool is_masterchain) { if (now <= last_paid || !last_paid || is_special || pricing.empty() || now <= pricing[0].valid_since) { return td::zero_refint(); @@ -669,7 +673,7 @@ td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std:: ton::UnixTime valid_until = (i < n - 1 ? std::min(now, pricing[i + 1].valid_since) : now); if (upto < valid_until) { assert(upto >= pricing[i].valid_since); - add_partial_storage_payment(total.unique_write(), valid_until - upto, pricing[i], storage_stat, is_masterchain); + add_partial_storage_payment(total.unique_write(), valid_until - upto, pricing[i], storage_used, is_masterchain); } upto = valid_until; } @@ -685,7 +689,7 @@ td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std:: * @returns The computed storage fees as RefInt256. */ td::RefInt256 Account::compute_storage_fees(ton::UnixTime now, const std::vector& pricing) const { - return StoragePrices::compute_storage_fees(now, pricing, storage_stat, last_paid, is_special, is_masterchain()); + return StoragePrices::compute_storage_fees(now, pricing, storage_used, last_paid, is_special, is_masterchain()); } namespace transaction { @@ -1848,7 +1852,7 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { if (S.is_error()) { // Rollback changes to state, fail action phase LOG(INFO) << "Account state size exceeded limits: " << S.move_as_error(); - new_storage_stat.clear(); + new_account_storage_stat = {}; new_code = old_code; new_data = old_data; new_library = old_library; @@ -2104,7 +2108,7 @@ int Transaction::try_action_change_library(vm::CellSlice& cs, ActionPhase& ap, c LOG(DEBUG) << "added " << ((rec.mode >> 1) ? "public" : "private") << " library with hash " << hash.to_hex(); } new_library = std::move(dict).extract_root_cell(); - } catch (vm::VmError& vme) { + } catch (vm::VmError&) { return 42; } ap.spec_actions++; @@ -2931,7 +2935,7 @@ static td::uint32 get_public_libraries_diff_count(const td::Ref& old_l * This function is not called for special accounts. * * @param size_limits The size limits configuration. - * @param update_storage_stat Store storage stat in the Transaction's CellStorageStat. + * @param update_storage_stat Store storage stat in the Transaction's AccountStorageStat. * * @returns A `td::Status` indicating the result of the check. * - If the state limits are within the allowed range, returns OK. @@ -2939,60 +2943,47 @@ static td::uint32 get_public_libraries_diff_count(const td::Ref& old_l */ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, bool update_storage_stat) { auto cell_equal = [](const td::Ref& a, const td::Ref& b) -> bool { - if (a.is_null()) { - return b.is_null(); - } - if (b.is_null()) { - return false; - } - return a->get_hash() == b->get_hash(); + return a.is_null() || b.is_null() ? a.is_null() == b.is_null() : a->get_hash() == b->get_hash(); }; if (cell_equal(account.code, new_code) && cell_equal(account.data, new_data) && cell_equal(account.library, new_library)) { return td::Status::OK(); } - vm::CellStorageStat storage_stat; - storage_stat.limit_cells = size_limits.max_acc_state_cells; - storage_stat.limit_bits = size_limits.max_acc_state_bits; + AccountStorageStat storage_stat; + if (update_storage_stat && account.account_storage_stat) { + storage_stat = account.account_storage_stat.value(); + } { TD_PERF_COUNTER(transaction_storage_stat_a); td::Timer timer; - auto add_used_storage = [&](const td::Ref& cell) -> td::Status { - if (cell.not_null()) { - TRY_RESULT(res, storage_stat.add_used_storage(cell)); - if (res.max_merkle_depth > max_allowed_merkle_depth) { - return td::Status::Error("too big merkle depth"); - } - } - return td::Status::OK(); - }; - TRY_STATUS(add_used_storage(new_code)); - TRY_STATUS(add_used_storage(new_data)); - TRY_STATUS(add_used_storage(new_library)); + TRY_RESULT(info, storage_stat.replace_roots({new_code, new_data, new_library})); + if (info.max_merkle_depth > max_allowed_merkle_depth) { + return td::Status::Error("too big Merkle depth"); + } if (timer.elapsed() > 0.1) { - LOG(INFO) << "Compute used storage took " << timer.elapsed() << "s"; + LOG(INFO) << "Compute used storage (1) took " << timer.elapsed() << "s"; } } - if (acc_status == Account::acc_active) { - storage_stat.clear_limit(); - } else { - storage_stat.clear(); - } - td::Status res; - if (storage_stat.cells > size_limits.max_acc_state_cells || storage_stat.bits > size_limits.max_acc_state_bits) { - res = td::Status::Error(PSTRING() << "account state is too big"); - } else if (account.is_masterchain() && !cell_equal(account.library, new_library) && - get_public_libraries_count(new_library) > size_limits.max_acc_public_libraries) { - res = td::Status::Error("too many public libraries"); - } else { - res = td::Status::OK(); + if (storage_stat.get_total_cells() > size_limits.max_acc_state_cells || + storage_stat.get_total_bits() > size_limits.max_acc_state_bits) { + return td::Status::Error(PSTRING() << "account state is too big: cells=" << storage_stat.get_total_cells() + << ", bits=" << storage_stat.get_total_bits() + << " (max cells=" << size_limits.max_acc_state_cells + << ", max bits=" << size_limits.max_acc_state_bits << ")"); + } + if (account.is_masterchain() && !cell_equal(account.library, new_library)) { + auto libraries_count = get_public_libraries_count(new_library); + if (libraries_count > size_limits.max_acc_public_libraries) { + return td::Status::Error(PSTRING() << "too many public libraries: " << libraries_count << " (max " + << size_limits.max_acc_public_libraries << ")"); + } } if (update_storage_stat) { // storage_stat will be reused in compute_state() - new_storage_stat = std::move(storage_stat); + new_account_storage_stat.value_force() = std::move(storage_stat); } - return res; + return td::Status::OK(); } /** @@ -3140,44 +3131,6 @@ bool Account::store_acc_status(vm::CellBuilder& cb, int acc_status) const { return cb.store_long_bool(v, 2); } -/** - * Tries to update the storage statistics based on the old storage statistics and old account state without fully recomputing it. - * - * It succeeds if only root cell of AccountStorage is changed. - * old_cs and new_cell are AccountStorage without extra currencies (if global_version >= 10). - * - * @param old_stat The old storage statistics. - * @param old_cs The old AccountStorage. - * @param new_cell The new AccountStorage. - * - * @returns An optional value of type vm::CellStorageStat. If the update is successful, it returns the new storage statistics. Otherwise, it returns an empty optional. - */ -static td::optional try_update_storage_stat(const vm::CellStorageStat& old_stat, - td::Ref old_cs, - td::Ref new_cell) { - if (old_stat.cells == 0 || old_cs.is_null()) { - return {}; - } - vm::CellSlice new_cs = vm::CellSlice(vm::NoVm(), new_cell); - if (old_cs->size_refs() != new_cs.size_refs()) { - return {}; - } - for (unsigned i = 0; i < old_cs->size_refs(); ++i) { - if (old_cs->prefetch_ref(i)->get_hash() != new_cs.prefetch_ref(i)->get_hash()) { - return {}; - } - } - if (old_stat.bits < old_cs->size()) { - return {}; - } - - vm::CellStorageStat new_stat; - new_stat.cells = old_stat.cells; - new_stat.bits = old_stat.bits - old_cs->size() + new_cs.size(); - new_stat.public_cells = old_stat.public_cells; - return new_stat; -} - /** * Removes extra currencies dict from AccountStorage. * @@ -3185,9 +3138,9 @@ static td::optional try_update_storage_stat(const vm::CellS * * @param storage_cs AccountStorage as CellSlice. * - * @returns AccountStorage without extra currencies as Cell. + * @returns AccountStorage without extra currencies as CellSlice. */ -static td::Ref storage_without_extra_currencies(td::Ref storage_cs) { +static td::Ref storage_without_extra_currencies(td::Ref storage_cs) { block::gen::AccountStorage::Record rec; if (!block::gen::csr_unpack(storage_cs, rec)) { LOG(ERROR) << "failed to unpack AccountStorage"; @@ -3205,18 +3158,20 @@ static td::Ref storage_without_extra_currencies(td::Ref return {}; } } - td::Ref cell; - if (!block::gen::pack_cell(cell, rec)) { + td::Ref result; + if (!block::gen::csr_pack(result, rec)) { LOG(ERROR) << "failed to pack AccountStorage"; return {}; } - return cell; + return result; } namespace transaction { /** * Computes the new state of the account. * + * @param cfg The configuration for the serialization phase. + * * @returns True if the state computation is successful, false otherwise. */ bool Transaction::compute_state(const SerializeConfig& cfg) { @@ -3290,45 +3245,82 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } else { new_inner_state.clear(); } - vm::CellStorageStat& stats = new_storage_stat; + td::Ref old_storage_for_stat = account.storage; - td::Ref new_storage_for_stat = storage; + td::Ref new_storage_for_stat = new_storage; if (cfg.extra_currency_v2) { new_storage_for_stat = storage_without_extra_currencies(new_storage); if (new_storage_for_stat.is_null()) { return false; } if (old_storage_for_stat.not_null()) { - old_storage_for_stat = vm::load_cell_slice_ref(storage_without_extra_currencies(old_storage_for_stat)); + old_storage_for_stat = storage_without_extra_currencies(old_storage_for_stat); if (old_storage_for_stat.is_null()) { return false; } } + } else if (cfg.store_storage_dict_hash) { + LOG(ERROR) << "unsupported store_storage_dict_hash=true, extra_currency_v2=false"; + return false; } - auto new_stats = try_update_storage_stat(account.storage_stat, old_storage_for_stat, storage); - if (new_stats) { - stats = new_stats.unwrap(); + + bool storage_refs_changed = false; + if (old_storage_for_stat.is_null() || new_storage_for_stat->size_refs() != old_storage_for_stat->size_refs()) { + storage_refs_changed = true; } else { + for (unsigned i = 0; i < new_storage_for_stat->size_refs(); i++) { + if (new_storage_for_stat->prefetch_ref(i)->get_hash() != old_storage_for_stat->prefetch_ref(i)->get_hash()) { + storage_refs_changed = true; + break; + } + } + } + + if (storage_refs_changed || (cfg.store_storage_dict_hash && !account.storage_dict_hash)) { TD_PERF_COUNTER(transaction_storage_stat_b); td::Timer timer; - stats.add_used_storage(new_storage_for_stat).ensure(); + if (!new_account_storage_stat && account.account_storage_stat) { + new_account_storage_stat = account.account_storage_stat; + } + AccountStorageStat& stats = new_account_storage_stat.value_force(); + // Don't check Merkle depth and size here - they were checked in check_state_limits + auto S = stats.replace_roots(new_storage->prefetch_all_refs()).move_as_status(); + if (S.is_error()) { + LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error(); + return false; + } + new_storage_dict_hash = stats.get_dict_hash(); + // Root of AccountStorage is not counted in AccountStorageStat + new_storage_used.cells = stats.get_total_cells() + 1; + new_storage_used.bits = stats.get_total_bits() + new_storage->size(); if (timer.elapsed() > 0.1) { - LOG(INFO) << "Compute used storage took " << timer.elapsed() << "s"; + LOG(INFO) << "Compute used storage (2) took " << timer.elapsed() << "s"; } - } - CHECK(cb.store_long_bool(1, 1) // account$1 - && cb.append_cellslice_bool(account.my_addr) // addr:MsgAddressInt - && block::store_UInt7(cb, stats.cells) // storage_used$_ cells:(VarUInteger 7) - && block::store_UInt7(cb, stats.bits) // bits:(VarUInteger 7) - && block::store_UInt7(cb, stats.public_cells) // public_cells:(VarUInteger 7) - && cb.store_long_bool(last_paid, 32)); // last_paid:uint32 + } else { + new_storage_used = account.storage_used; + new_storage_used.bits -= old_storage_for_stat->size(); + new_storage_used.bits += new_storage_for_stat->size(); + new_storage_dict_hash = account.storage_dict_hash; + new_account_storage_stat = account.account_storage_stat; + } + if (!cfg.store_storage_dict_hash) { + new_storage_dict_hash = {}; + } + + CHECK(cb.store_long_bool(1, 1) // account$1 + && cb.append_cellslice_bool(account.my_addr) // addr:MsgAddressInt + && block::store_UInt7(cb, new_storage_used.cells) // storage_used$_ cells:(VarUInteger 7) + && block::store_UInt7(cb, new_storage_used.bits) // bits:(VarUInteger 7) + && cb.store_long_bool(new_storage_dict_hash ? 1 : 0, 3) // extra:StorageExtraInfo + && (!new_storage_dict_hash || cb.store_bits_bool(new_storage_dict_hash.value())) // dict_hash:uint256 + && cb.store_long_bool(last_paid, 32)); // last_paid:uint32 if (due_payment.not_null() && td::sgn(due_payment) != 0) { CHECK(cb.store_long_bool(1, 1) && block::tlb::t_Grams.store_integer_ref(cb, due_payment)); // due_payment:(Maybe Grams) } else { CHECK(cb.store_long_bool(0, 1)); } - CHECK(cb.append_data_cell_bool(std::move(storage))); + CHECK(cb.append_cellslice_bool(new_storage)); new_total_state = cb.finalize(); if (verbosity > 2) { FLOG(INFO) { @@ -3345,6 +3337,8 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { * * Updates root. * + * @param cfg The configuration for the serialization. + * * @returns True if the serialization is successful, False otherwise. */ bool Transaction::serialize(const SerializeConfig& cfg) { @@ -3688,7 +3682,9 @@ Ref Transaction::commit(Account& acc) { acc.last_trans_end_lt_ = end_lt; acc.last_trans_hash_ = root->get_hash().bits(); acc.last_paid = last_paid; - acc.storage_stat = new_storage_stat; + acc.storage_used = new_storage_used; + acc.account_storage_stat = std::move(new_account_storage_stat); + acc.storage_dict_hash = new_storage_dict_hash; acc.storage = new_storage; acc.balance = std::move(balance); acc.due_payment = std::move(due_payment); @@ -3936,6 +3932,7 @@ td::Status FetchConfigParams::fetch_config_params( } { serialize_cfg->extra_currency_v2 = config.get_global_version() >= 10; + serialize_cfg->store_storage_dict_hash = config.get_global_version() >= 11; } { // fetch block_grams_created diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 8e612e6a5b..aa08719a8f 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "account-storage-stat.h" #include "common/refcnt.hpp" #include "common/refint.h" #include "vm/cells.h" @@ -179,6 +180,7 @@ struct ActionPhaseConfig { struct SerializeConfig { bool extra_currency_v2{false}; + bool store_storage_dict_hash{false}; }; struct CreditPhase { @@ -266,8 +268,12 @@ struct Account { ton::LogicalTime last_trans_lt_; ton::Bits256 last_trans_hash_; ton::LogicalTime block_lt; + ton::UnixTime last_paid; - vm::CellStorageStat storage_stat; + StorageUsed storage_used; + td::optional storage_dict_hash; + td::optional account_storage_stat; + block::CurrencyCollection balance; td::RefInt256 due_payment; Ref orig_total_state; // ^Account @@ -377,7 +383,9 @@ struct Transaction { std::unique_ptr compute_phase; std::unique_ptr action_phase; std::unique_ptr bounce_phase; - vm::CellStorageStat new_storage_stat; + StorageUsed new_storage_used; + td::optional new_account_storage_stat; + td::optional new_storage_dict_hash; bool gas_limit_overridden{false}; Transaction(const Account& _account, int ttype, ton::LogicalTime req_start_lt, ton::UnixTime _now, Ref _inmsg = {}); diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 17e7eb69de..3a7ddc48d7 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -113,21 +113,19 @@ class NewCellStorageStat { struct CellStorageStat { unsigned long long cells; unsigned long long bits; - unsigned long long public_cells; struct CellInfo { td::uint32 max_merkle_depth = 0; }; td::HashMap seen; - CellStorageStat() : cells(0), bits(0), public_cells(0) { + CellStorageStat() : cells(0), bits(0) { } - explicit CellStorageStat(unsigned long long limit_cells) - : cells(0), bits(0), public_cells(0), limit_cells(limit_cells) { + explicit CellStorageStat(unsigned long long limit_cells) : cells(0), bits(0), limit_cells(limit_cells) { } void clear_seen() { seen.clear(); } void clear() { - cells = bits = public_cells = 0; + cells = bits = 0; clear_limit(); clear_seen(); } diff --git a/crypto/vm/cells/CellSlice.cpp b/crypto/vm/cells/CellSlice.cpp index 9cd3e931aa..bea20f95d1 100644 --- a/crypto/vm/cells/CellSlice.cpp +++ b/crypto/vm/cells/CellSlice.cpp @@ -773,6 +773,14 @@ bool CellSlice::prefetch_maybe_ref(Ref& res) const { } } +std::vector> CellSlice::prefetch_all_refs() const { + std::vector> res(size_refs()); + for (unsigned i = 0; i < size_refs(); ++i) { + res[i] = prefetch_ref(i); + } + return res; +} + bool CellSlice::fetch_maybe_ref(Ref& res) { auto z = prefetch_ulong(1); if (!z) { diff --git a/crypto/vm/cells/CellSlice.h b/crypto/vm/cells/CellSlice.h index ecce30f5c5..7525272b59 100644 --- a/crypto/vm/cells/CellSlice.h +++ b/crypto/vm/cells/CellSlice.h @@ -190,6 +190,7 @@ class CellSlice : public td::CntObject { } bool fetch_maybe_ref(Ref& ref); bool prefetch_maybe_ref(Ref& ref) const; + std::vector> prefetch_all_refs() const; td::BitSlice fetch_bits(unsigned bits); td::BitSlice prefetch_bits(unsigned bits) const; td::Ref fetch_subslice(unsigned bits, unsigned refs = 0); diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 77963e9594..c0be0b108a 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -156,3 +156,10 @@ Example: if the last masterchain block seqno is `19071` then the list contains b ### TVM changes - `SENDMSG` calculates messages size and fees without extra currencies, uses new +64 and +128 mode behavior. - `SENDMSG` does not check the number of extra currencies. + +## Version 11 +### New account storage stat +Along with the storage stat (cells and bits count), each account now stores the hash of the **storage dict**. + +**Storage dict** is the dictionary that stores refcnt for each cell in the account state. +This is required to help computing storage stats in the future, after collator-validator separation. \ No newline at end of file diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index d73e715c93..f91f2bf14d 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -180,7 +180,7 @@ struct RawAccountState { td::Ref extra_currencies; ton::UnixTime storage_last_paid{0}; - vm::CellStorageStat storage_stat; + block::StorageUsed storage_used; td::Ref code; td::Ref data; @@ -1036,7 +1036,7 @@ class Query { TRY_RESULT(basechain_msg_prices, cfg->get_msg_prices(false)); block::MsgPrices* msg_prices[2] = {&basechain_msg_prices, &masterchain_msg_prices}; auto storage_fee_256 = block::StoragePrices::compute_storage_fees( - raw_.source->get_sync_time(), storage_prices, raw_.source->raw().storage_stat, + raw_.source->get_sync_time(), storage_prices, raw_.source->raw().storage_used, raw_.source->raw().storage_last_paid, false, is_masterchain); auto storage_fee = storage_fee_256.is_null() ? 0 : storage_fee_256->to_long(); @@ -1085,7 +1085,7 @@ class Query { TRY_RESULT(dest_gas_limits_prices, cfg->get_gas_limits_prices(dest_is_masterchain)); auto dest_storage_fee_256 = destination ? block::StoragePrices::compute_storage_fees( - destination->get_sync_time(), storage_prices, destination->raw().storage_stat, + destination->get_sync_time(), storage_prices, destination->raw().storage_used, destination->raw().storage_last_paid, false, is_masterchain) : td::make_refint(0); Fee dst_fee; @@ -1399,17 +1399,16 @@ class GetRawAccountState : public td::actor::Actor { return td::Status::Error("Failed to unpack StorageInfo"); } unsigned long long u = 0; - vm::CellStorageStat storage_stat; + block::StorageUsed storage_stat; u |= storage_stat.cells = block::tlb::t_VarUInteger_7.as_uint(*storage_used.cells); u |= storage_stat.bits = block::tlb::t_VarUInteger_7.as_uint(*storage_used.bits); - u |= storage_stat.public_cells = block::tlb::t_VarUInteger_7.as_uint(*storage_used.public_cells); //LOG(DEBUG) << "last_paid=" << res.storage_last_paid << "; cells=" << storage_stat.cells - //<< " bits=" << storage_stat.bits << " public_cells=" << storage_stat.public_cells; + //<< " bits=" << storage_stat.bits; if (u == std::numeric_limits::max()) { return td::Status::Error("Failed to unpack StorageStat"); } - res.storage_stat = storage_stat; + res.storage_used = storage_stat; } block::gen::AccountStorage::Record storage; @@ -2089,7 +2088,7 @@ class RunEmulator : public TonlibQueryActor { raw.balance = balance.grams->to_long(); raw.extra_currencies = balance.extra; raw.storage_last_paid = std::move(account.last_paid); - raw.storage_stat = std::move(account.storage_stat); + raw.storage_used = account.storage_used; raw.code = std::move(account.code); raw.data = std::move(account.data); raw.state = std::move(account.total_state); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 90966d8203..5f2ba66151 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1008,6 +1008,7 @@ bool ValidateQuery::fetch_config_params() { } { serialize_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; + serialize_cfg_.store_storage_dict_hash = config_->get_global_version() >= 11; } { // fetch block_grams_created From f92cdf327c1b26d3bcb0379ff2f3451505915afb Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 5 Mar 2025 16:10:50 +0300 Subject: [PATCH 147/524] Fix nullptr error in Collator::finalize_stats --- validator/impl/collator.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 400d17a636..fee6f21d1f 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -6178,21 +6178,23 @@ void Collator::finalize_stats() { } else { stats_.block_id.id = new_id; } - stats_.cc_seqno = validator_set_->get_catchain_seqno(); + stats_.cc_seqno = validator_set_.not_null() ? validator_set_->get_catchain_seqno() : 0; stats_.collated_at = td::Clocks::system(); stats_.attempt = attempt_idx_; stats_.is_validator = !(mode_ & CollateMode::from_collator_node); stats_.self = stats_.is_validator ? PublicKey(pubkeys::Ed25519(created_by_)).compute_short_id() : collator_node_id_.pubkey_hash(); - stats_.estimated_bytes = block_limit_status_->estimate_block_size(); - stats_.gas = block_limit_status_->gas_used; - stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; - stats_.estimated_collated_data_bytes = block_limit_status_->collated_data_stat.estimate_proof_size(); - stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.estimated_bytes); - stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas); - stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); - stats_.cat_collated_data_bytes = - block_limit_status_->limits.classify_collated_data_size(stats_.estimated_collated_data_bytes); + if (block_limit_status_) { + stats_.estimated_bytes = block_limit_status_->estimate_block_size(); + stats_.gas = block_limit_status_->gas_used; + stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; + stats_.estimated_collated_data_bytes = block_limit_status_->collated_data_stat.estimate_proof_size(); + stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.estimated_bytes); + stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas); + stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); + stats_.cat_collated_data_bytes = + block_limit_status_->limits.classify_collated_data_size(stats_.estimated_collated_data_bytes); + } stats_.total_time = perf_timer_.elapsed(); stats_.work_time = work_time; stats_.cpu_work_time = cpu_work_time; From 928f02e6a17a4f8582678c83cf3aee348d7db29d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 5 Mar 2025 17:02:16 +0300 Subject: [PATCH 148/524] New account storage stat for accelerator --- crypto/block/account-storage-stat.cpp | 195 +++++++++++++++++--------- crypto/block/account-storage-stat.h | 21 ++- crypto/block/block.tlb | 1 + crypto/block/transaction.cpp | 176 ++++++++++++++++++----- crypto/block/transaction.h | 7 +- crypto/vm/boc.cpp | 21 +-- crypto/vm/boc.h | 10 +- crypto/vm/cells/CellUsageTree.cpp | 2 +- crypto/vm/cells/CellUsageTree.h | 4 + crypto/vm/vm.h | 5 + validator/impl/collator-impl.h | 11 ++ validator/impl/collator.cpp | 154 ++++++++++++++++++++ validator/impl/validate-query.cpp | 29 ++++ validator/impl/validate-query.hpp | 1 + 14 files changed, 512 insertions(+), 125 deletions(-) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index 8e8b6f7e16..f5545f8b2e 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -53,51 +53,83 @@ AccountStorageStat& AccountStorageStat::operator=(AccountStorageStat&& other) { return *this; } -td::Result AccountStorageStat::add_root(const Ref& cell) { - roots_.push_back(cell); - return add_cell(cell); -} - -td::Status AccountStorageStat::remove_root(const Ref& cell) { - auto it = std::find_if(roots_.begin(), roots_.end(), - [&](const Ref& c) { return c->get_hash() == cell->get_hash(); }); - if (it == roots_.end()) { - return td::Status::Error(PSTRING() << "no such root " << cell->get_hash().to_hex()); - } - roots_.erase(it); - return remove_cell(cell); -} - td::Result AccountStorageStat::replace_roots(std::vector> new_roots) { - std::vector> old_roots = roots_; + std::erase_if(new_roots, [](const Ref& c) { return c.is_null(); }); + auto cmp = [](const Ref& c1, const Ref& c2) { return c1->get_hash() < c2->get_hash(); }; + std::sort(new_roots.begin(), new_roots.end(), cmp); + std::sort(roots_.begin(), roots_.end(), cmp); + std::vector> to_add, to_del; + std::set_difference(new_roots.begin(), new_roots.end(), roots_.begin(), roots_.end(), std::back_inserter(to_add), + cmp); + std::set_difference(roots_.begin(), roots_.end(), new_roots.begin(), new_roots.end(), std::back_inserter(to_del), + cmp); + td::uint32 max_merkle_depth = 0; - for (const Ref& root : new_roots) { - if (root.is_null()) { - continue; - } - TRY_RESULT(info, add_root(root)); + for (const Ref& root : to_add) { + TRY_RESULT(info, add_cell(root)); max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); } - for (const Ref& root : old_roots) { - TRY_STATUS(remove_root(root)); + for (const Ref& root : to_del) { + TRY_STATUS(remove_cell(root)); } + + roots_ = std::move(new_roots); + td::Status S = td::Status::OK(); + cache_.for_each([&](Entry& e) { + if (S.is_ok()) { + S = commit_entry(e); + } + }); + TRY_STATUS(std::move(S)); return CellInfo{max_merkle_depth}; } +void AccountStorageStat::add_hint(const td::HashSet& hint) { + td::HashSet visited; + std::function&, bool)> dfs = [&](const Ref& cell, bool is_root) { + if (!visited.insert(cell->get_hash()).second) { + return; + } + Entry& e = get_entry(cell); + e.exists = e.exists_known = true; + if (is_root) { + fetch_entry(e).ignore(); + if (e.max_merkle_depth && e.max_merkle_depth.value() != 0) { + return; + } + } + if (hint.contains(cell->get_hash())) { + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + dfs(cs.prefetch_ref(i), false); + } + } + }; + for (const Ref& root : roots_) { + dfs(root, true); + } +} + td::Result AccountStorageStat::add_cell(const Ref& cell) { Entry& e = get_entry(cell); - ++e.refcnt; - if (e.refcnt == 0) { - return td::Status::Error(PSTRING() << "cell " << cell->get_hash().to_hex() << ": refcnt overflow"); + if (!e.exists_known || e.refcnt_diff < 0) { + TRY_STATUS(fetch_entry(e)); } - if (e.refcnt != 1) { - update_dict(e); - return CellInfo{e.max_merkle_depth}; + ++e.refcnt_diff; + if (e.exists || e.refcnt_diff > 1 || (e.refcnt && e.refcnt.value() + e.refcnt_diff != 1)) { + if (!e.max_merkle_depth) { + TRY_STATUS(fetch_entry(e)); + if (!e.max_merkle_depth) { + return td::Status::Error(PSTRING() << "unexpected unknown Merkle depth of cell " << cell->get_hash()); + } + } + return CellInfo{e.max_merkle_depth.value()}; } + td::uint32 max_merkle_depth = 0; - vm::CellSlice cs{vm::NoVm{}, cell}; - ++total_cells_; - total_bits_ += cs.size(); + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); for (unsigned i = 0; i < cs.size_refs(); ++i) { TRY_RESULT(info, add_cell(cs.prefetch_ref(i))); max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); @@ -109,64 +141,99 @@ td::Result AccountStorageStat::add_cell(const Ref< max_merkle_depth = std::min(max_merkle_depth, MERKLE_DEPTH_LIMIT); Entry& e2 = get_entry(cell); e2.max_merkle_depth = max_merkle_depth; - update_dict(e2); return CellInfo{max_merkle_depth}; } td::Status AccountStorageStat::remove_cell(const Ref& cell) { Entry& e = get_entry(cell); - if (e.refcnt == 0) { - return td::Status::Error(PSTRING() << "cell " << cell->get_hash().to_hex() << " is not in the dict"); + if (!e.exists_known) { + TRY_STATUS(fetch_entry(e)); } - --e.refcnt; - update_dict(e); - if (e.refcnt != 0) { - return td::Status::OK(); + if (!e.exists) { + return td::Status::Error(PSTRING() << "Failed to remove cell " << cell->get_hash().to_hex() + << " : does not exist in the dict"); } - vm::CellSlice cs{vm::NoVm{}, std::move(cell)}; - if (total_cells_ == 0 || total_bits_ < cs.size()) { - return td::Status::Error("total_cell/total_bits becomes negative"); + --e.refcnt_diff; + if (e.refcnt_diff < 0 && !e.refcnt) { + TRY_STATUS(fetch_entry(e)); } - --total_cells_; - total_bits_ -= cs.size(); + if (e.refcnt.value() + e.refcnt_diff != 0) { + return td::Status::OK(); + } + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); for (unsigned i = 0; i < cs.size_refs(); ++i) { TRY_STATUS(remove_cell(cs.prefetch_ref(i))); } return td::Status::OK(); } -bool AccountStorageStat::Entry::serialize(vm::CellBuilder& cb) const { - return cb.store_long_bool(refcnt, 32) && cb.store_long_bool(max_merkle_depth, 2); -} - -void AccountStorageStat::Entry::fetch(Ref cs) { - if (cs.is_null()) { - refcnt = max_merkle_depth = 0; - } else { - refcnt = (td::uint32)cs.write().fetch_ulong(32); - max_merkle_depth = (td::uint32)cs.write().fetch_ulong(2); - } -} - AccountStorageStat::Entry& AccountStorageStat::get_entry(const Ref& cell) { return cache_.apply(cell->get_hash().as_slice(), [&](Entry& e) { if (e.inited) { return; } e.inited = true; - e.hash = cell->get_hash(); - e.fetch(dict_.lookup(e.hash.as_bitslice())); + e.cell = cell; }); } -void AccountStorageStat::update_dict(const Entry& e) { - if (e.refcnt == 0) { - dict_.lookup_delete(e.hash.as_bitslice()); +td::Status AccountStorageStat::fetch_entry(Entry& e) { + if (e.exists_known && e.refcnt && (!e.exists || e.max_merkle_depth)) { + return td::Status::OK(); + } + auto cs = dict_.lookup(e.cell->get_hash().as_bitslice()); + if (cs.is_null()) { + e.exists = false; + e.refcnt = 0; } else { + if (cs->size_ext() != 32 + 2) { + return td::Status::Error(PSTRING() << "invalid record for cell " << e.cell->get_hash().to_hex()); + } + e.exists = true; + e.refcnt = (td::uint32)cs.write().fetch_ulong(32); + e.max_merkle_depth = (td::uint32)cs.write().fetch_ulong(2); + if (e.refcnt.value() == 0) { + return td::Status::Error(PSTRING() << "invalid refcnt=0 for cell " << e.cell->get_hash().to_hex()); + } + } + e.exists_known = true; + return td::Status::OK(); +} + +td::Status AccountStorageStat::commit_entry(Entry& e) { + if (e.refcnt_diff == 0) { + return td::Status::OK(); + } + TRY_STATUS(fetch_entry(e)); + e.refcnt.value() += e.refcnt_diff; + e.refcnt_diff = 0; + bool spec; + if (e.refcnt.value() == 0) { + --total_cells_; + total_bits_ -= vm::load_cell_slice_special(e.cell, spec).size(); + e.exists = false; + if (dict_.lookup_delete(e.cell->get_hash().as_bitslice()).is_null()) { + return td::Status::Error(PSTRING() << "Failed to delete entry " << e.cell->get_hash().to_hex()); + } + } else { + if (!e.exists) { + ++total_cells_; + total_bits_ += vm::load_cell_slice_special(e.cell, spec).size(); + } + e.exists = true; + if (!e.max_merkle_depth) { + return td::Status::Error(PSTRING() << "Failed to store entry " << e.cell->get_hash().to_hex() + << " : unknown merkle depth"); + } vm::CellBuilder cb; - CHECK(e.serialize(cb)); - dict_.set_builder(e.hash.as_bitslice(), cb); + dict_.set_builder(e.cell->get_hash().as_bitslice(), cb); + CHECK(cb.store_long_bool(e.refcnt.value(), 32) && cb.store_long_bool(e.max_merkle_depth.value(), 2)); + if (!dict_.set_builder(e.cell->get_hash().as_bitslice(), cb)) { + return td::Status::Error(PSTRING() << "Failed to store entry " << e.cell->get_hash().to_hex()); + } } + return td::Status::OK(); } } // namespace block \ No newline at end of file diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h index 894192d4f9..69ac218fe0 100644 --- a/crypto/block/account-storage-stat.h +++ b/crypto/block/account-storage-stat.h @@ -58,9 +58,8 @@ class AccountStorageStat { return dict_.is_empty() ? td::Bits256::zero() : td::Bits256{dict_.get_root_cell()->get_hash().bits()}; } - td::Result add_root(const Ref &cell); - td::Status remove_root(const Ref &cell); - td::Result replace_roots(std::vector> new_roots); + td::Result replace_roots(std::vector> hint); + void add_hint(const td::HashSet &visited); private: vm::Dictionary dict_; @@ -72,15 +71,14 @@ class AccountStorageStat { struct Entry { bool inited = false; - vm::Cell::Hash hash; - td::uint32 refcnt = 0; - td::uint32 max_merkle_depth = 0; - - void fetch(Ref cs); - bool serialize(vm::CellBuilder &cb) const; + Ref cell; + bool exists_known = false; + bool exists = false; + td::optional refcnt, max_merkle_depth; + td::int32 refcnt_diff = 0; vm::Cell::Hash key() const { - return hash; + return cell->get_hash(); } bool operator<(const Entry &other) const { return key() < other.key(); @@ -111,7 +109,8 @@ class AccountStorageStat { vm::CellHashTable cache_; Entry &get_entry(const Ref &cell); - void update_dict(const Entry &e); + td::Status fetch_entry(Entry &e); + td::Status commit_entry(Entry &e); static constexpr td::uint32 MERKLE_DEPTH_LIMIT = 3; }; diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index dd3b50a73c..5d7f1db945 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -852,6 +852,7 @@ top_block_descr#d5 proof_for:BlockIdExt signatures:(Maybe ^BlockSignatures) // COLLATED DATA // top_block_descr_set#4ac789f3 collection:(HashmapE 96 ^TopBlockDescr) = TopBlockDescrSet; +account_storage_dict_proof#37c1e3fc proof:^Cell = AccountStorageDictProof; // // VALIDATOR MISBEHAVIOR COMPLAINTS diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 4e92b8f6cc..bab126d1fa 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -262,6 +262,7 @@ bool Account::unpack_storage_info(vm::CellSlice& cs) { } else { storage_dict_hash = {}; } + orig_storage_dict_hash = storage_dict_hash; if (info.due_payment->prefetch_ulong(1) == 1) { vm::CellSlice& cs2 = info.due_payment.write(); cs2.advance(1); @@ -310,8 +311,8 @@ bool Account::unpack_state(vm::CellSlice& cs) { tock = z & 1; LOG(DEBUG) << "tick=" << tick << ", tock=" << tock; } - code = state.code->prefetch_ref(); - data = state.data->prefetch_ref(); + code = orig_code = state.code->prefetch_ref(); + data = orig_data = state.data->prefetch_ref(); library = orig_library = state.library->prefetch_ref(); return true; } @@ -531,7 +532,7 @@ bool Account::init_new(ton::UnixTime now) { now_ = now; last_paid = 0; storage_used = {}; - storage_dict_hash = {}; + orig_storage_dict_hash = storage_dict_hash = {}; due_payment = td::zero_refint(); balance.set_zero(); if (my_addr_exact.is_null()) { @@ -562,6 +563,113 @@ bool Account::init_new(ton::UnixTime now) { return true; } +/** + * Removes extra currencies dict from AccountStorage. + * + * This is used for computing account storage stats. + * + * @param storage_cs AccountStorage as CellSlice. + * + * @returns AccountStorage without extra currencies as CellSlice. + */ +static td::Ref storage_without_extra_currencies(td::Ref storage_cs) { + block::gen::AccountStorage::Record rec; + if (!block::gen::csr_unpack(storage_cs, rec)) { + LOG(ERROR) << "failed to unpack AccountStorage"; + return {}; + } + if (rec.balance->size_refs() > 0) { + block::gen::CurrencyCollection::Record balance; + if (!block::gen::csr_unpack(rec.balance, balance)) { + LOG(ERROR) << "failed to unpack AccountStorage"; + return {}; + } + balance.other = vm::CellBuilder{}.store_zeroes(1).as_cellslice_ref(); + if (!block::gen::csr_pack(rec.balance, balance)) { + LOG(ERROR) << "failed to pack AccountStorage"; + return {}; + } + } + td::Ref result; + if (!block::gen::csr_pack(result, rec)) { + LOG(ERROR) << "failed to pack AccountStorage"; + return {}; + } + return result; +} + +/** + * Computes storage dict of the account from scratch. + * This requires storage_dict_hash to be set, as it guarantees that the stored storage_used was computed recently + * (in older versions it included extra currency balance, in newer versions it does not). + * + * @returns Root of the dictionary, or Error + */ +td::Result> Account::compute_account_storage_dict() const { + if (storage.is_null()) { + return td::Status::Error("cannot compute storage dict: empty storage"); + } + if (!storage_dict_hash) { + return td::Status::Error("cannot compute storage dict: storage_dict_hash is not set"); + } + AccountStorageStat stat; + auto storage_for_stat = storage_without_extra_currencies(storage); + if (storage_for_stat.is_null()) { + return td::Status::Error("cannot compute storage dict: invalid storage"); + } + TRY_STATUS(stat.replace_roots(storage_for_stat->prefetch_all_refs()).move_as_status()); + // Root of AccountStorage is not counted in AccountStorageStat + td::uint64 expected_cells = stat.get_total_cells() + 1; + td::uint64 expected_bits = stat.get_total_bits() + storage->size(); + if (expected_cells != storage_used.cells || expected_bits != storage_used.bits) { + return td::Status::Error(PSTRING() << "invalid storage_used: computed cells=" << expected_cells + << " bits=" << expected_bits << ", found cells" << storage_used.cells + << " bits=" << storage_used.bits); + } + if (storage_dict_hash.value() != stat.get_dict_hash()) { + return td::Status::Error(PSTRING() << "invalid storage dict hash: computed " << stat.get_dict_hash().to_hex() + << ", found " << storage_dict_hash.value().to_hex()); + } + return stat.get_dict_root(); +} + +/** + * Initializes account_storage_stat of the account using the existing dict_root. + * This is not strictly necessary, as the storage stat is recomputed in Transaction. + * However, it can be used to optimize cell usage. + * This requires storage_dict_hash to be set, as it guarantees that the stored storage_used was computed recently + * (in older versions it included extra currency balance, in newer versions it does not). + * + * @param dict_root Root of the storage dictionary. + * + * @returns Status of the operation. + */ +td::Status Account::init_account_storage_stat(Ref dict_root) { + if (storage.is_null()) { + if (dict_root.not_null()) { + return td::Status::Error("storage is null, but dict_root is not null"); + } + account_storage_stat = {}; + return td::Status::OK(); + } + if (!storage_dict_hash) { + return td::Status::Error("cannot init storage dict: storage_dict_hash is not set"); + } + // Root of AccountStorage is not counted in AccountStorageStat + if (storage_used.cells < 1 || storage_used.bits < storage->size()) { + return td::Status::Error(PSTRING() << "storage_used is too small: cells=" << storage_used.cells + << " bits=" << storage_used.bits << " storage_root_bits=" << storage->size()); + } + AccountStorageStat new_stat(std::move(dict_root), storage->prefetch_all_refs(), storage_used.cells - 1, + storage_used.bits - storage->size()); + if (storage_dict_hash.value() != new_stat.get_dict_hash()) { + return td::Status::Error(PSTRING() << "invalid storage dict hash: computed " << new_stat.get_dict_hash().to_hex() + << ", found " << storage_dict_hash.value().to_hex()); + } + account_storage_stat = std::move(new_stat); + return td::Status::OK(); +} + /** * Resets the split depth of the account. * @@ -1814,6 +1922,7 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { << cfg.flat_gas_limit << "]; remaining balance=" << balance.to_str(); CHECK(td::sgn(balance.grams) >= 0); } + cp.vm_loaded_cells = vm.extract_loaded_cells(); return true; } @@ -2956,6 +3065,9 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, { TD_PERF_COUNTER(transaction_storage_stat_a); td::Timer timer; + if (update_storage_stat && compute_phase) { + storage_stat.add_hint(compute_phase->vm_loaded_cells); + } TRY_RESULT(info, storage_stat.replace_roots({new_code, new_data, new_library})); if (info.max_merkle_depth > max_allowed_merkle_depth) { return td::Status::Error("too big Merkle depth"); @@ -3131,41 +3243,6 @@ bool Account::store_acc_status(vm::CellBuilder& cb, int acc_status) const { return cb.store_long_bool(v, 2); } -/** - * Removes extra currencies dict from AccountStorage. - * - * This is used for computing account storage stats. - * - * @param storage_cs AccountStorage as CellSlice. - * - * @returns AccountStorage without extra currencies as CellSlice. - */ -static td::Ref storage_without_extra_currencies(td::Ref storage_cs) { - block::gen::AccountStorage::Record rec; - if (!block::gen::csr_unpack(storage_cs, rec)) { - LOG(ERROR) << "failed to unpack AccountStorage"; - return {}; - } - if (rec.balance->size_refs() > 0) { - block::gen::CurrencyCollection::Record balance; - if (!block::gen::csr_unpack(rec.balance, balance)) { - LOG(ERROR) << "failed to unpack AccountStorage"; - return {}; - } - balance.other = vm::CellBuilder{}.store_zeroes(1).as_cellslice_ref(); - if (!block::gen::csr_pack(rec.balance, balance)) { - LOG(ERROR) << "failed to pack AccountStorage"; - return {}; - } - } - td::Ref result; - if (!block::gen::csr_pack(result, rec)) { - LOG(ERROR) << "failed to pack AccountStorage"; - return {}; - } - return result; -} - namespace transaction { /** * Computes the new state of the account. @@ -3284,6 +3361,9 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } AccountStorageStat& stats = new_account_storage_stat.value_force(); // Don't check Merkle depth and size here - they were checked in check_state_limits + if (compute_phase) { + stats.add_hint(compute_phase->vm_loaded_cells); + } auto S = stats.replace_roots(new_storage->prefetch_all_refs()).move_as_status(); if (S.is_error()) { LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error(); @@ -3306,6 +3386,26 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { if (!cfg.store_storage_dict_hash) { new_storage_dict_hash = {}; } + if (false) { + vm::CellStorageStat control_stats; + control_stats.add_used_storage(new_storage); + if (control_stats.bits != new_storage_used.bits || control_stats.cells != new_storage_used.cells) { + LOG(ERROR) << " [ QQQQQQ 1 Wrong storage stat " << account.workchain << ":" << account.addr.to_hex() << " " + << start_lt << " : " << new_storage_used.cells << "," << new_storage_used.bits + << " != " << control_stats.cells << "," << control_stats.bits << " ] "; + return false; + } + AccountStorageStat control_stats_2; + control_stats_2.replace_roots(new_storage->prefetch_all_refs()); + if (control_stats_2.get_total_bits() + new_storage->size() != new_storage_used.bits || + control_stats_2.get_total_cells() + 1 != new_storage_used.cells) { + LOG(ERROR) << " [ QQQQQQ 2 Wrong storage stat " << account.workchain << ":" << account.addr.to_hex() << " " + << start_lt << " : " << new_storage_used.cells << "," << new_storage_used.bits + << " != " << control_stats_2.get_total_cells() + 1 << "," + << control_stats_2.get_total_bits() + new_storage->size() << " ] "; + return false; + } + } CHECK(cb.store_long_bool(1, 1) // account$1 && cb.append_cellslice_bool(account.my_addr) // addr:MsgAddressInt diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index aa08719a8f..b2f41f375a 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -208,6 +208,7 @@ struct ComputePhase { Ref actions; std::string vm_log; td::optional precompiled_gas_usage; + td::HashSet vm_loaded_cells; }; struct ActionPhase { @@ -272,6 +273,7 @@ struct Account { ton::UnixTime last_paid; StorageUsed storage_used; td::optional storage_dict_hash; + td::optional orig_storage_dict_hash; td::optional account_storage_stat; block::CurrencyCollection balance; @@ -281,7 +283,8 @@ struct Account { Ref storage; // AccountStorage Ref inner_state; // StateInit ton::Bits256 state_hash; // hash of StateInit for frozen accounts - Ref code, data, library, orig_library; + Ref code, data, library; + Ref orig_code, orig_data, orig_library; std::vector transactions; Account() = default; Account(ton::WorkchainId wc, td::ConstBitPtr _addr) : workchain(wc), addr(_addr) { @@ -295,6 +298,8 @@ struct Account { bool set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr); bool unpack(Ref account, ton::UnixTime now, bool special); bool init_new(ton::UnixTime now); + td::Result> compute_account_storage_dict() const; + td::Status init_account_storage_stat(Ref dict_root); bool deactivate(); bool recompute_tmp_addr(Ref& tmp_addr, int split_depth, td::ConstBitPtr orig_addr_rewrite) const; td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector& pricing) const; diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 72afb99885..67a443f062 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -1259,14 +1259,6 @@ bool VmStorageStat::add_storage(const CellSlice& cs) { return true; } -static td::uint64 estimate_prunned_size() { - return 41; -} - -static td::uint64 estimate_serialized_size(const Ref& cell) { - return cell->get_serialized_size() + cell->size_refs() * 3 + 3; -} - void ProofStorageStat::add_cell(const Ref& cell) { auto& status = cells_[cell->get_hash()]; if (status == c_loaded) { @@ -1290,4 +1282,17 @@ td::uint64 ProofStorageStat::estimate_proof_size() const { return proof_size_; } +ProofStorageStat::CellStatus ProofStorageStat::get_cell_status(const Cell::Hash& hash) const { + auto it = cells_.find(hash); + return it == cells_.end() ? c_none : it->second; +} + +td::uint64 ProofStorageStat::estimate_prunned_size() { + return 41; +} + +td::uint64 ProofStorageStat::estimate_serialized_size(const Ref& cell) { + return cell->get_serialized_size() + cell->size_refs() * 3 + 3; +} + } // namespace vm diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 3a7ddc48d7..64cdbc928e 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -167,11 +167,17 @@ class ProofStorageStat { public: void add_cell(const Ref& cell); td::uint64 estimate_proof_size() const; - private: + enum CellStatus { c_none = 0, c_prunned = 1, c_loaded = 2 }; - td::HashMap cells_; + CellStatus get_cell_status(const Cell::Hash& hash) const; + + static td::uint64 estimate_prunned_size(); + static td::uint64 estimate_serialized_size(const Ref& cell); + + private: + td::HashMap cells_; td::uint64 proof_size_ = 0; }; diff --git a/crypto/vm/cells/CellUsageTree.cpp b/crypto/vm/cells/CellUsageTree.cpp index 410b3fcd64..3874998c2c 100644 --- a/crypto/vm/cells/CellUsageTree.cpp +++ b/crypto/vm/cells/CellUsageTree.cpp @@ -112,7 +112,7 @@ void CellUsageTree::set_use_mark_for_is_loaded(bool use_mark) { } void CellUsageTree::on_load(NodeId node_id, const td::Ref& cell) { - if (nodes_[node_id].is_loaded) { + if (ignore_loads_ || nodes_[node_id].is_loaded) { return; } nodes_[node_id].is_loaded = true; diff --git a/crypto/vm/cells/CellUsageTree.h b/crypto/vm/cells/CellUsageTree.h index af0f21f534..c37b2c2d0b 100644 --- a/crypto/vm/cells/CellUsageTree.h +++ b/crypto/vm/cells/CellUsageTree.h @@ -66,6 +66,9 @@ class CellUsageTree : public std::enable_shared_from_this { void set_cell_load_callback(std::function&)> f) { cell_load_callback_ = std::move(f); } + void set_ignore_loads(bool value) { + ignore_loads_ = value; + } private: struct Node { @@ -80,5 +83,6 @@ class CellUsageTree : public std::enable_shared_from_this { void on_load(NodeId node_id, const td::Ref& cell); NodeId create_node(NodeId parent); + bool ignore_loads_ = false; }; } // namespace vm diff --git a/crypto/vm/vm.h b/crypto/vm/vm.h index a171ef27e6..679a72defe 100644 --- a/crypto/vm/vm.h +++ b/crypto/vm/vm.h @@ -19,6 +19,7 @@ #pragma once #include "common/refcnt.hpp" +#include "td/utils/HashMap.h" #include "vm/cellslice.h" #include "vm/stack.hpp" #include "vm/vmstate.h" @@ -424,6 +425,10 @@ class VmState final : public VmStateInterface { } } + td::HashSet extract_loaded_cells() { + return std::move(loaded_cells); + } + private: void init_cregs(bool same_c3 = false, bool push_0 = true); int run_inner(); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index ffccc4ae34..9d40c4f3c5 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -220,6 +220,15 @@ class Collator final : public td::actor::Actor { std::vector neighbor_proof_builders_; std::vector> collated_roots_; + struct AccountStorageDict { + bool inited = false; + vm::MerkleProofBuilder mpb; + Ref proof_root; + size_t proof_size_estimate = 0; + bool add_to_collated_data = false; + }; + std::map account_storage_dicts_; + std::unique_ptr block_candidate; std::unique_ptr dispatch_queue_, old_dispatch_queue_; @@ -245,6 +254,7 @@ class Collator final : public td::actor::Actor { block::Account* lookup_account(td::ConstBitPtr addr) const; std::unique_ptr make_account_from(td::ConstBitPtr addr, Ref account, bool force_create); + bool init_account_storage_dict(block::Account& account); td::Result make_account(td::ConstBitPtr addr, bool force_create = false); td::actor::ActorId get_self() { return actor_id(this); @@ -344,6 +354,7 @@ class Collator final : public td::actor::Actor { bool register_dispatch_queue_op(bool force = false); bool update_account_dict_estimation(const block::transaction::Transaction& trans); bool update_min_mc_seqno(ton::BlockSeqno some_mc_seqno); + bool process_account_storage_dict(const block::Account& account); bool combine_account_transactions(); bool update_public_libraries(); bool update_account_public_libraries(Ref orig_libs, Ref final_libs, const td::Bits256& addr); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index fee6f21d1f..6b7d934d3b 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -2614,9 +2614,60 @@ std::unique_ptr Collator::make_account_from(td::ConstBitPtr addr return nullptr; } ptr->block_lt = start_lt; + if (!init_account_storage_dict(*ptr)) { + return nullptr; + } return ptr; } +/** + * If full collated data is enabled, initialize account storage dict and prepare MerkleProofBuilder for it + * + * @param account Account to initialize storage dict for + * @return True on success, False on failure + */ +bool Collator::init_account_storage_dict(block::Account& account) { + if (!full_collated_data_ || is_masterchain() || !account.storage_dict_hash || account.storage.is_null()) { + return true; + } + if (account.storage_used.cells < 10) { // TODO: some other threshold? + return true; + } + td::Bits256 storage_dict_hash = account.storage_dict_hash.value(); + if (storage_dict_hash.is_zero()) { + return true; + } + AccountStorageDict& dict = account_storage_dicts_[storage_dict_hash]; + if (!dict.inited) { + dict.inited = true; + // don't mark cells in account state as loaded during compute_account_storage_dict + state_usage_tree_->set_ignore_loads(true); + auto res = account.compute_account_storage_dict(); + state_usage_tree_->set_ignore_loads(false); + if (res.is_error()) { + return fatal_error(res.move_as_error_prefix(PSTRING() << "Failed to init account storage dict for " + << account.addr.to_hex() << ": ")); + } + if (res.ok().is_null()) { // Impossible if storage_dict_hash is not zero + return fatal_error(PSTRING() << "Failed to init account storage dict for " << account.addr.to_hex() + << ": dict is empty"); + } + dict.mpb = vm::MerkleProofBuilder(res.move_as_ok()); + dict.mpb.set_cell_load_callback([&](const td::Ref& cell) { + if (block_limit_status_) { + block_limit_status_->collated_data_stat.add_cell(cell); + } + }); + } + auto S = account.init_account_storage_stat(dict.mpb.root()); + if (S.is_error()) { + return fatal_error(S.move_as_error_prefix(PSTRING() << "Failed to init account storage dict for " + << account.addr.to_hex() << ": ")); + } + return true; +} + + /** * Looks up an account in the Collator's account map. * @@ -2666,12 +2717,99 @@ td::Result Collator::make_account(td::ConstBitPtr addr, bool fo return ins.first->second.get(); } +/** + * Decides whether to include storage dict proof to collated data for this account or not. + * + * @param account Account object + * + * @returns True if the operation is successful, false otherwise. + */ +bool Collator::process_account_storage_dict(const block::Account& account) { + if (!account.orig_storage_dict_hash) { + return true; + } + td::Bits256 storage_dict_hash = account.orig_storage_dict_hash.value(); + auto it = account_storage_dicts_.find(storage_dict_hash); + if (it == account_storage_dicts_.end()) { + return true; + } + CHECK(full_collated_data_ && !is_masterchain()); + AccountStorageDict& dict = it->second; + if (dict.add_to_collated_data) { + LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() << " : already included"; + return true; + } + + td::HashSet visited; + bool calculate_proof_size_diff = true; + td::int64 proof_size_diff = 0; + std::function&)> dfs = [&](const Ref& cell) { + if (cell.is_null() || !visited.emplace(cell->get_hash()).second) { + return; + } + auto loaded_cell = cell->load_cell().move_as_ok(); + if (calculate_proof_size_diff) { + switch (block_limit_status_->collated_data_stat.get_cell_status(cell->get_hash())) { + case vm::ProofStorageStat::c_none: + proof_size_diff += vm::ProofStorageStat::estimate_serialized_size(loaded_cell.data_cell); + break; + case vm::ProofStorageStat::c_prunned: + proof_size_diff -= vm::ProofStorageStat::estimate_prunned_size(); + proof_size_diff += vm::ProofStorageStat::estimate_serialized_size(loaded_cell.data_cell); + break; + case vm::ProofStorageStat::c_loaded: + break; + } + } + vm::CellSlice cs{std::move(loaded_cell)}; + for (unsigned i = 0; i < cs.size_refs(); ++i) { + dfs(cs.prefetch_ref(i)); + } + }; + + // Visit all cells in the original account storage to calculate collated data increase + state_usage_tree_->set_ignore_loads(true); + dfs(account.orig_code); + dfs(account.orig_data); + dfs(account.orig_library); + state_usage_tree_->set_ignore_loads(false); + + if (proof_size_diff > (td::int64)dict.proof_size_estimate) { + LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() + << " : account_proof_size=" << proof_size_diff << ", dict_proof_size=" << dict.proof_size_estimate + << ", include dict in collated data"; + dict.add_to_collated_data = true; + } else { + LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() + << " : account_proof_size=" << proof_size_diff << ", dict_proof_size=" << dict.proof_size_estimate + << ", DO NOT include dict in collated data"; + // Include account storage in collated data + calculate_proof_size_diff = false; + visited.clear(); + dfs(account.orig_code); + dfs(account.orig_data); + dfs(account.orig_library); + } + + return true; +} + /** * Combines account transactions and updates the ShardAccountBlocks and ShardAccounts. * * @returns True if the operation is successful, false otherwise. */ bool Collator::combine_account_transactions() { + for (auto& [hash, dict] : account_storage_dicts_) { + auto res = dict.mpb.extract_proof(); + if (res.is_error()) { + return fatal_error(res.move_as_error_prefix(PSTRING() << "Failed to generate proof for account storage dict " + << hash.to_hex() << ": ")); + } + dict.proof_root = res.move_as_ok(); + dict.proof_size_estimate = vm::std_boc_serialize(dict.proof_root, 31).move_as_ok().size(); + } + vm::AugmentedDictionary dict{256, block::tlb::aug_ShardAccountBlocks}; for (auto& z : accounts) { block::Account& acc = *(z.second); @@ -2755,6 +2893,9 @@ bool Collator::combine_account_transactions() { } } } + if (!process_account_storage_dict(acc)) { + return false; + } } else { if (acc.total_state->get_hash() != acc.orig_total_state->get_hash()) { return fatal_error(std::string{"total state of account "} + z.first.to_hex() + @@ -5885,6 +6026,19 @@ bool Collator::create_collated_data() { for (auto& p : proofs) { collated_roots_.push_back(std::move(p.second)); } + + // 5. Proofs for account storage dicts + for (auto& [_, dict] : account_storage_dicts_) { + if (!dict.add_to_collated_data) { + continue; + } + CHECK(dict.proof_root.not_null()); + // account_storage_dict_proof#37c1e3fc proof:^Cell = AccountStorageDictProof; + collated_roots_.push_back(vm::CellBuilder() + .store_long(block::gen::AccountStorageDictProof::cons_tag[0], 32) + .store_ref(dict.proof_root) + .finalize_novm()); + } return true; } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 5eab8a14d3..a07ef157e8 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -633,6 +633,23 @@ bool ValidateQuery::extract_collated_data_from(Ref croot, int idx) { top_shard_descr_dict_ = std::make_unique(cs.prefetch_ref(), 96); return true; } + if (block::gen::t_AccountStorageDictProof.has_valid_tag(cs)) { + if (!block::gen::t_AccountStorageDictProof.validate_upto(10000, cs)) { + return reject_query("invalid AccountStorageDictProof"); + } + // account_storage_dict_proof#37c1e3fc proof:^Cell = AccountStorageDictProof; + Ref proof = cs.prefetch_ref(); + auto virt_root = vm::MerkleProof::virtualize(proof, 1); + if (virt_root.is_null()) { + return reject_query("invalid Merkle proof in AccountStorageDictProof"); + } + LOG(DEBUG) << "collated datum # " << idx << " is an AccountStorageDictProof with hash " + << virt_root->get_hash().to_hex(); + if (!virt_account_storage_dicts_.emplace(virt_root->get_hash().bits(), virt_root).second) { + return reject_query("duplicate AccountStorageDictProof"); + } + return true; + } LOG(WARNING) << "collated datum # " << idx << " has unknown type (magic " << cs.prefetch_ulong(32) << "), ignoring"; return true; } @@ -5253,6 +5270,18 @@ std::unique_ptr ValidateQuery::unpack_account(td::ConstBitPtr ad << " does not really belong to current shard"); return {}; } + if (new_acc->storage_dict_hash) { + auto it = virt_account_storage_dicts_.find(new_acc->storage_dict_hash.value()); + if (it != virt_account_storage_dicts_.end()) { + LOG(DEBUG) << "Using account storage dict proof for account " << addr.to_hex(256) + << ", hash=" << it->second->get_hash().to_hex(); + auto S = new_acc->init_account_storage_stat(it->second); + if (S.is_error()) { + reject_query(PSTRING() << "Failed to init account storage stat for account " << addr.to_hex(256), std::move(S)); + return {}; + } + } + } return new_acc; } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 2a93c7188f..e42f885487 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -170,6 +170,7 @@ class ValidateQuery : public td::actor::Actor { std::vector> collated_roots_; std::map> virt_roots_; std::unique_ptr top_shard_descr_dict_; + std::map> virt_account_storage_dicts_; Ref shard_hashes_; // from McBlockExtra Ref blk_config_params_; // from McBlockExtra From 61ef357941668b51ba7319627c91b180e317d387 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 6 Mar 2025 17:15:27 +0300 Subject: [PATCH 149/524] Request persistent state size, add missing queries to full-node-master --- tl/generate/scheme/ton_api.tl | 2 ++ tl/generate/scheme/ton_api.tlo | Bin 102484 -> 102788 bytes validator/db/archive-manager.cpp | 50 +++++++++++++++-------------- validator/db/archive-manager.hpp | 9 ++++-- validator/db/rootdb.cpp | 6 ++-- validator/db/rootdb.hpp | 4 +-- validator/full-node-master.cpp | 37 +++++++++++++++++++--- validator/full-node-master.hpp | 4 +++ validator/full-node-shard.cpp | 22 ++++++++++--- validator/full-node-shard.hpp | 2 ++ validator/interfaces/db.h | 4 +-- validator/manager-disk.cpp | 6 ++-- validator/manager-disk.hpp | 4 +-- validator/manager-hardfork.hpp | 4 +-- validator/manager.cpp | 6 ++-- validator/manager.hpp | 4 +-- validator/net/download-state.cpp | 52 ++++++++++++++++++++++++++++--- validator/net/download-state.hpp | 3 ++ validator/validator.h | 4 +-- 19 files changed, 163 insertions(+), 60 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index cfc9f3a198..14e0121fd6 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -406,6 +406,7 @@ tonNode.preparedProof = tonNode.PreparedProof; tonNode.preparedProofLink = tonNode.PreparedProof; tonNode.preparedState = tonNode.PreparedState; tonNode.notFoundState = tonNode.PreparedState; +tonNode.persistentStateSize size:long = tonNode.PersistentStateSize; tonNode.prepared = tonNode.Prepared; tonNode.notFound = tonNode.Prepared; tonNode.data data:bytes = tonNode.Data; @@ -472,6 +473,7 @@ tonNode.prepareKeyBlockProofs blocks:(vector tonNode.blockIdExt) allow_partial:B tonNode.prepareBlock block:tonNode.blockIdExt = tonNode.Prepared; tonNode.prepareBlocks blocks:(vector tonNode.blockIdExt) = tonNode.Prepared; tonNode.preparePersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PreparedState; +tonNode.getPersistentStateSize block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PersistentStateSize; tonNode.prepareZeroState block:tonNode.blockIdExt = tonNode.PreparedState; tonNode.getNextKeyBlockIds block:tonNode.blockIdExt max_size:int = tonNode.KeyBlocks; tonNode.downloadNextBlockFull prev_block:tonNode.blockIdExt = tonNode.DataFull; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 96ecb7751b1ad3a31830cba56037d0037988cde3..5d9ccc2d8586fcb9da5e367372b01e3394ff8b62 100644 GIT binary patch delta 191 zcmcbzfURXQ8}Fmp`c@23puUlJwz1-+%POCwOY-yl@>5dv0#b{LGmA@7^GbqC5=&Bp zGpkY=Cm&=I-@L+Dgq?Bo=Dk*n@2jBbD8SSKlAr9b(|`rU*d8Lqn4rhlwSB7x<9r7V g-9PDWaxmM{Q%kT|$NK&7T6}-|A1THVJ;u)Mfu4->9hSFr HF|q>y03{O* diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 8c7cde1707..9b1ebf6c2f 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -314,7 +314,15 @@ void ArchiveManager::register_perm_state(FileReferenceShort id) { BlockSeqno masterchain_seqno = 0; id.ref().visit(td::overloaded( [&](const fileref::PersistentStateShort &x) { masterchain_seqno = x.masterchain_seqno; }, [&](const auto &) {})); - perm_states_[{masterchain_seqno, id.hash()}] = id; + td::uint64 size; + auto r_stat = td::stat(db_root_ + "/archive/states/" + id.filename_short()); + if (r_stat.is_error()) { + LOG(WARNING) << "Cannot stat persistent state file " << id.filename_short() << " : " << r_stat.move_as_error(); + size = 0; + } else { + size = r_stat.ok().size_; + } + perm_states_[{masterchain_seqno, id.hash()}] = {.id = id, .size = size}; } void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise promise) { @@ -417,7 +425,7 @@ void ArchiveManager::get_previous_persistent_state_files( BlockSeqno mc_seqno = it->first.first; std::vector> files; while (it->first.first == mc_seqno) { - files.emplace_back(db_root_ + "/archive/states/" + it->second.filename_short(), it->second.shard()); + files.emplace_back(db_root_ + "/archive/states/" + it->second.id.filename_short(), it->second.id.shard()); if (it == perm_states_.begin()) { break; } @@ -452,15 +460,16 @@ void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt td::actor::create_actor("readfile", path, offset, max_size, 0, std::move(promise)).release(); } -void ArchiveManager::check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { +void ArchiveManager::get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; auto hash = id.hash(); - if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { - promise.set_result(false); + auto it = perm_states_.find({masterchain_block_id.seqno(), hash}); + if (it == perm_states_.end()) { + promise.set_error(td::Status::Error(ErrorCode::notready)); return; } - promise.set_result(true); + promise.set_result(it->second.size); } void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, @@ -1023,15 +1032,15 @@ void ArchiveManager::persistent_state_gc(std::pair last) { int res = 0; BlockSeqno seqno = 0; - F.ref().visit(td::overloaded([&](const fileref::ZeroStateShort &) { res = 1; }, - [&](const fileref::PersistentStateShort &x) { - res = 0; - seqno = x.masterchain_seqno; - }, - [&](const auto &obj) { res = -1; })); + F.id.ref().visit(td::overloaded([&](const fileref::ZeroStateShort &) { res = 1; }, + [&](const fileref::PersistentStateShort &x) { + res = 0; + seqno = x.masterchain_seqno; + }, + [&](const auto &obj) { res = -1; })); if (res == -1) { - td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore(); + td::unlink(db_root_ + "/archive/states/" + F.id.filename_short()).ignore(); perm_states_.erase(it); } if (res != 0) { @@ -1081,7 +1090,7 @@ void ArchiveManager::got_gc_masterchain_handle(ConstBlockHandle handle, std::pai CHECK(it != perm_states_.end()); auto &F = it->second; if (to_del) { - td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore(); + td::unlink(db_root_ + "/archive/states/" + F.id.filename_short()).ignore(); perm_states_.erase(it); } delay_action( @@ -1202,12 +1211,7 @@ void ArchiveManager::prepare_stats(td::Promise states; for (auto &[key, file] : perm_states_) { BlockSeqno seqno = key.first; - auto r_stat = td::stat(db_root_ + "/archive/states/" + file.filename_short()); - if (r_stat.is_error()) { - LOG(WARNING) << "Cannot stat persistent state file " << file.filename_short() << " : " << r_stat.move_as_error(); - } else { - states[seqno] += r_stat.move_as_ok().size_; - } + states[seqno] += file.size; } td::StringBuilder sb; for (auto &[seqno, size] : states) { @@ -1308,7 +1312,7 @@ void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle han auto it = perm_states_.begin(); while (it != perm_states_.end()) { int res = 0; - it->second.ref().visit(td::overloaded( + it->second.id.ref().visit(td::overloaded( [&](const fileref::ZeroStateShort &x) { res = -1; }, [&](const fileref::PersistentStateShort &x) { res = x.masterchain_seqno <= masterchain_seqno ? -1 : 1; }, [&](const auto &obj) { res = 1; })); @@ -1317,7 +1321,7 @@ void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle han } else { auto it2 = it; it++; - td::unlink(db_root_ + "/archive/states/" + it2->second.filename_short()).ignore(); + td::unlink(db_root_ + "/archive/states/" + it2->second.id.filename_short()).ignore(); perm_states_.erase(it2); } } diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index d919e32eee..cd79ccc6e8 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -52,7 +52,8 @@ class ArchiveManager : public td::actor::Actor { void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, td::int64 max_size, td::Promise promise); - void check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); + void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise); void check_zero_state(BlockIdExt block_id, td::Promise promise); void get_previous_persistent_state_files(BlockSeqno cur_mc_seqno, td::Promise>> promise); @@ -189,7 +190,11 @@ class ArchiveManager : public td::actor::Actor { return p.key ? key_files_ : p.temp ? temp_files_ : files_; } - std::map, FileReferenceShort> perm_states_; // Mc block seqno, hash -> state + struct PermState { + FileReferenceShort id; + td::uint64 size; + }; + std::map, PermState> perm_states_; // Mc block seqno, hash -> state void load_package(PackageId seqno); void delete_package(PackageId seqno, td::Promise promise); diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 8d83e7a7d1..2b370eb065 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -310,9 +310,9 @@ void RootDb::get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt mas offset, max_size, std::move(promise)); } -void RootDb::check_persistent_state_file_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - td::actor::send_closure(archive_db_, &ArchiveManager::check_persistent_state, block_id, masterchain_block_id, +void RootDb::get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state_file_size, block_id, masterchain_block_id, std::move(promise)); } diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 52f6098e4d..9e5b0b302e 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -80,8 +80,8 @@ class RootDb : public Db { td::Promise promise) override; void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, td::int64 max_length, td::Promise promise) override; - void check_persistent_state_file_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override; + void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; void get_zero_state_file(BlockIdExt block_id, td::Promise promise) override; void check_zero_state_file_exists(BlockIdExt block_id, td::Promise promise) override; diff --git a/validator/full-node-master.cpp b/validator/full-node-master.cpp index da49f0e2ea..8dc2a6815b 100644 --- a/validator/full-node-master.cpp +++ b/validator/full-node-master.cpp @@ -275,20 +275,32 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise) { - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error() || !R.move_as_ok()) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); return; } - auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); }); auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_persistent_state_exists, block_id, + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, + masterchain_block_id, std::move(P)); +} + +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, size, std::move(R)); + promise.set_value(create_serialize_tl_object(size)); + }); + auto block_id = create_block_id(query.block_); + auto masterchain_block_id = create_block_id(query.masterchain_block_); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, masterchain_block_id, std::move(P)); } @@ -389,6 +401,21 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo ShardIdFull{masterchainId}, std::move(P)); } +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getShardArchiveInfo &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_serialize_tl_object()); + } else { + promise.set_value(create_serialize_tl_object(R.move_as_ok())); + } + }); + ShardIdFull shard_prefix = create_shard_id(query.shard_prefix_); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_, + shard_prefix, std::move(P)); +} + void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise) { td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_slice, query.archive_id_, diff --git a/validator/full-node-master.hpp b/validator/full-node-master.hpp index ce0aedd35c..d6160a94ee 100644 --- a/validator/full-node-master.hpp +++ b/validator/full-node-master.hpp @@ -66,6 +66,8 @@ class FullNodeMasterImpl : public FullNodeMaster { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadZeroState &query, @@ -80,6 +82,8 @@ class FullNodeMasterImpl : public FullNodeMaster { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getShardArchiveInfo &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise); // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query, diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index ac0eb76882..a8ddb338ad 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -509,13 +509,12 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise) { auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error() || !R.move_as_ok()) { + td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); return; } - auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); }); @@ -523,7 +522,22 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod auto masterchain_block_id = create_block_id(query.masterchain_block_); VLOG(FULL_NODE_DEBUG) << "Got query preparePersistentState " << block_id.to_str() << " " << masterchain_block_id.to_str() << " from " << src; - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_persistent_state_exists, block_id, + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, + masterchain_block_id, std::move(P)); +} + +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, size, std::move(R)); + promise.set_value(create_serialize_tl_object(size)); + }); + auto block_id = create_block_id(query.block_); + auto masterchain_block_id = create_block_id(query.masterchain_block_); + VLOG(FULL_NODE_DEBUG) << "Got query getPersistentStateSize " << block_id.to_str() << " " + << masterchain_block_id.to_str() << " from " << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, masterchain_block_id, std::move(P)); } diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index fb3eef7697..8cbd614aa1 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -123,6 +123,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadZeroState &query, diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 29ef715b36..9a2eea56e0 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -62,8 +62,8 @@ class Db : public td::actor::Actor { td::Promise promise) = 0; virtual void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, td::int64 max_length, td::Promise promise) = 0; - virtual void check_persistent_state_file_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) = 0; + virtual void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) = 0; virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) = 0; virtual void get_zero_state_file(BlockIdExt block_id, td::Promise promise) = 0; virtual void check_zero_state_file_exists(BlockIdExt block_id, td::Promise promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 62fdc4b435..8e4a4d0868 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -200,9 +200,9 @@ void ValidatorManagerImpl::get_zero_state(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(db_, &Db::check_persistent_state_file_exists, block_id, masterchain_block_id, +void ValidatorManagerImpl::get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, std::move(promise)); } void ValidatorManagerImpl::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index cd06bf555d..62f54b2e95 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -108,8 +108,8 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_data(BlockHandle handle, td::Promise promise) override; void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override; void get_zero_state(BlockIdExt block_id, td::Promise promise) override; - void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override; + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override; void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) override; void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 0b8b9e736d..486c185f7a 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -128,8 +128,8 @@ class ValidatorManagerImpl : public ValidatorManager { void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override { UNREACHABLE(); } - void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override { + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override { UNREACHABLE(); } void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, diff --git a/validator/manager.cpp b/validator/manager.cpp index b0ac54092d..b1dc4cc9a9 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -304,9 +304,9 @@ void ValidatorManagerImpl::get_zero_state(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(db_, &Db::check_persistent_state_file_exists, block_id, masterchain_block_id, +void ValidatorManagerImpl::get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, std::move(promise)); } void ValidatorManagerImpl::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, diff --git a/validator/manager.hpp b/validator/manager.hpp index 418deb3505..347b8f7ef9 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -377,8 +377,8 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_data(BlockHandle handle, td::Promise promise) override; void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override; void get_zero_state(BlockIdExt block_id, td::Promise promise) override; - void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override; + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override; void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) override; void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index 6735a2b5fd..c481adbbfb 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -168,6 +168,7 @@ void DownloadState::got_block_state_description(td::BufferSlice data) { }, [&, self = this](ton_api::tonNode_preparedState &f) { if (masterchain_block_id_.is_valid()) { + request_total_size(); got_block_state_part(td::BufferSlice{}, 0); return; } @@ -190,8 +191,37 @@ void DownloadState::got_block_state_description(td::BufferSlice data) { create_serialize_tl_object_suffix(std::move(query)), td::Timestamp::in(3.0), std::move(P)); } + status_.set_status(PSTRING() << block_id_.id.to_str() << " : download started"); })); - status_.set_status(PSTRING() << block_id_.id.to_str() << " : 0 bytes, 0B/s"); +} + +void DownloadState::request_total_size() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + return; + } + auto res = fetch_tl_object(R.move_as_ok(), true); + if (res.is_error()) { + return; + } + td::actor::send_closure(SelfId, &DownloadState::got_total_size, res.ok()->size_); + }); + + td::BufferSlice query = create_serialize_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_)); + if (client_.empty()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, + "get size", std::move(P), td::Timestamp::in(3.0), std::move(query), + FullNode::max_state_size(), rldp_); + } else { + td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get size", + create_serialize_tl_object_suffix(std::move(query)), + td::Timestamp::in(3.0), std::move(P)); + } +} + +void DownloadState::got_total_size(td::uint64 size) { + total_size_ = size; } void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 requested_size) { @@ -203,10 +233,22 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques if (elapsed > 5.0) { prev_logged_timer_ = td::Timer(); auto speed = (td::uint64)((double)(sum_ - prev_logged_sum_) / elapsed); - LOG(WARNING) << "downloading state " << block_id_.to_str() << ": " << td::format::as_size(sum_) << " (" - << td::format::as_size(speed) << "/s)"; - status_.set_status(PSTRING() << block_id_.id.to_str() << " : " << sum_ << " bytes, " << td::format::as_size(speed) - << "/s"); + td::StringBuilder sb; + sb << td::format::as_size(sum_); + if (total_size_) { + sb << "/" << td::format::as_size(total_size_); + } + sb << " (" << td::format::as_size(speed) << "/s"; + if (total_size_) { + sb << ", " << td::StringBuilder::FixedDouble((double)sum_ / (double)total_size_ * 100.0, 2) << "%"; + if (speed > 0 && total_size_ >= sum_) { + td::uint64 rem = (total_size_ - sum_) / speed; + sb << ", " << rem << "s remaining"; + } + } + sb << ")"; + LOG(WARNING) << "downloading state " << block_id_.to_str() << " : " << sb.as_cslice(); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : " << sb.as_cslice()); prev_logged_sum_ = sum_; } diff --git a/validator/net/download-state.hpp b/validator/net/download-state.hpp index 470c543181..29854ce273 100644 --- a/validator/net/download-state.hpp +++ b/validator/net/download-state.hpp @@ -49,6 +49,8 @@ class DownloadState : public td::actor::Actor { void got_block_handle(BlockHandle handle); void got_node_to_download(adnl::AdnlNodeIdShort node); void got_block_state_description(td::BufferSlice data_description); + void request_total_size(); + void got_total_size(td::uint64 size); void got_block_state_part(td::BufferSlice data, td::uint32 requested_size); void got_block_state(td::BufferSlice data); @@ -77,6 +79,7 @@ class DownloadState : public td::actor::Actor { td::uint64 prev_logged_sum_ = 0; td::Timer prev_logged_timer_; + td::uint64 total_size_ = 0; ProcessStatus status_; }; diff --git a/validator/validator.h b/validator/validator.h index 5d6c0173c9..eecb30c078 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -227,8 +227,8 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_data(BlockHandle handle, td::Promise promise) = 0; virtual void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) = 0; virtual void get_zero_state(BlockIdExt block_id, td::Promise promise) = 0; - virtual void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) = 0; + virtual void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) = 0; virtual void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, From e0aa7f0f64a044692fb8c53a68a28cb6ff5cd9a0 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 11 Mar 2025 15:59:25 +0300 Subject: [PATCH 150/524] Improve estimating account proof size --- crypto/block/transaction.cpp | 17 +++++++----- crypto/block/transaction.h | 3 ++- validator/impl/collator-impl.h | 2 ++ validator/impl/collator.cpp | 49 +++++++++++++++++++++++++++------- 4 files changed, 54 insertions(+), 17 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index bab126d1fa..8cdf261aaa 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -3044,13 +3044,13 @@ static td::uint32 get_public_libraries_diff_count(const td::Ref& old_l * This function is not called for special accounts. * * @param size_limits The size limits configuration. - * @param update_storage_stat Store storage stat in the Transaction's AccountStorageStat. + * @param is_account_stat Store storage stat in the Transaction's AccountStorageStat. * * @returns A `td::Status` indicating the result of the check. * - If the state limits are within the allowed range, returns OK. * - If the state limits exceed the maximum allowed range, returns an error. */ -td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, bool update_storage_stat) { +td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, bool is_account_stat) { auto cell_equal = [](const td::Ref& a, const td::Ref& b) -> bool { return a.is_null() || b.is_null() ? a.is_null() == b.is_null() : a->get_hash() == b->get_hash(); }; @@ -3059,13 +3059,13 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, return td::Status::OK(); } AccountStorageStat storage_stat; - if (update_storage_stat && account.account_storage_stat) { + if (is_account_stat && account.account_storage_stat) { storage_stat = account.account_storage_stat.value(); } { TD_PERF_COUNTER(transaction_storage_stat_a); td::Timer timer; - if (update_storage_stat && compute_phase) { + if (is_account_stat && compute_phase) { storage_stat.add_hint(compute_phase->vm_loaded_cells); } TRY_RESULT(info, storage_stat.replace_roots({new_code, new_data, new_library})); @@ -3091,9 +3091,12 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, << size_limits.max_acc_public_libraries << ")"); } } - if (update_storage_stat) { + if (is_account_stat) { // storage_stat will be reused in compute_state() new_account_storage_stat.value_force() = std::move(storage_stat); + storage_stats_updates.push_back(new_code); + storage_stats_updates.push_back(new_data); + storage_stats_updates.push_back(new_library); } return td::Status::OK(); } @@ -3364,7 +3367,9 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { if (compute_phase) { stats.add_hint(compute_phase->vm_loaded_cells); } - auto S = stats.replace_roots(new_storage->prefetch_all_refs()).move_as_status(); + auto roots = new_storage->prefetch_all_refs(); + storage_stats_updates.insert(storage_stats_updates.end(), roots.begin(), roots.end()); + auto S = stats.replace_roots(std::move(roots)).move_as_status(); if (S.is_error()) { LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error(); return false; diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index b2f41f375a..30b2604db4 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -392,6 +392,7 @@ struct Transaction { td::optional new_account_storage_stat; td::optional new_storage_dict_hash; bool gas_limit_overridden{false}; + std::vector> storage_stats_updates; Transaction(const Account& _account, int ttype, ton::LogicalTime req_start_lt, ton::UnixTime _now, Ref _inmsg = {}); bool unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* cfg); @@ -405,7 +406,7 @@ struct Transaction { bool run_precompiled_contract(const ComputePhaseConfig& cfg, precompiled::PrecompiledSmartContract& precompiled); bool prepare_compute_phase(const ComputePhaseConfig& cfg); bool prepare_action_phase(const ActionPhaseConfig& cfg); - td::Status check_state_limits(const SizeLimitsConfig& size_limits, bool update_storage_stat = true); + td::Status check_state_limits(const SizeLimitsConfig& size_limits, bool is_account_stat = true); bool prepare_bounce_phase(const ActionPhaseConfig& cfg); bool compute_state(const SerializeConfig& cfg); bool serialize(const SerializeConfig& cfg); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 9d40c4f3c5..b02fbedc33 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -226,6 +226,7 @@ class Collator final : public td::actor::Actor { Ref proof_root; size_t proof_size_estimate = 0; bool add_to_collated_data = false; + std::vector> storage_stat_updates; }; std::map account_storage_dicts_; @@ -353,6 +354,7 @@ class Collator final : public td::actor::Actor { bool register_out_msg_queue_op(bool force = false); bool register_dispatch_queue_op(bool force = false); bool update_account_dict_estimation(const block::transaction::Transaction& trans); + void update_account_storage_dict_info(const block::transaction::Transaction& trans); bool update_min_mc_seqno(ton::BlockSeqno some_mc_seqno); bool process_account_storage_dict(const block::Account& account); bool combine_account_transactions(); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 6b7d934d3b..00699997cb 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -2739,16 +2739,23 @@ bool Collator::process_account_storage_dict(const block::Account& account) { LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() << " : already included"; return true; } + if (dict.storage_stat_updates.empty()) { + LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() << " : not required (no storage updates)"; + return true; + } td::HashSet visited; bool calculate_proof_size_diff = true; td::int64 proof_size_diff = 0; - std::function&)> dfs = [&](const Ref& cell) { + std::function&, bool)> dfs = [&](const Ref& cell, bool from_old_state) { if (cell.is_null() || !visited.emplace(cell->get_hash()).second) { return; } auto loaded_cell = cell->load_cell().move_as_ok(); - if (calculate_proof_size_diff) { + if (!loaded_cell.tree_node.empty()) { + from_old_state = true; + } + if (calculate_proof_size_diff && from_old_state) { switch (block_limit_status_->collated_data_stat.get_cell_status(cell->get_hash())) { case vm::ProofStorageStat::c_none: proof_size_diff += vm::ProofStorageStat::estimate_serialized_size(loaded_cell.data_cell); @@ -2763,15 +2770,15 @@ bool Collator::process_account_storage_dict(const block::Account& account) { } vm::CellSlice cs{std::move(loaded_cell)}; for (unsigned i = 0; i < cs.size_refs(); ++i) { - dfs(cs.prefetch_ref(i)); + dfs(cs.prefetch_ref(i), from_old_state); } }; - // Visit all cells in the original account storage to calculate collated data increase + // Visit cells that were used in storage stat computation to calculate collated data increase state_usage_tree_->set_ignore_loads(true); - dfs(account.orig_code); - dfs(account.orig_data); - dfs(account.orig_library); + for (const auto& cell : dict.storage_stat_updates) { + dfs(cell, false); + } state_usage_tree_->set_ignore_loads(false); if (proof_size_diff > (td::int64)dict.proof_size_estimate) { @@ -2786,9 +2793,9 @@ bool Collator::process_account_storage_dict(const block::Account& account) { // Include account storage in collated data calculate_proof_size_diff = false; visited.clear(); - dfs(account.orig_code); - dfs(account.orig_data); - dfs(account.orig_library); + for (const auto& cell : dict.storage_stat_updates) { + dfs(cell, false); + } } return true; @@ -3068,6 +3075,7 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t if (!update_account_dict_estimation(*trans)) { return fatal_error(-666, "cannot update account dict size estimation"); } + update_account_storage_dict_info(*trans); update_max_lt(acc->last_trans_end_lt_); block::MsgMetadata new_msg_metadata{0, acc->workchain, acc->addr, trans->start_lt}; register_new_msgs(*trans, std::move(new_msg_metadata)); @@ -3165,6 +3173,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, fatal_error("cannot update account dict size estimation"); return {}; } + update_account_storage_dict_info(*trans); td::optional new_msg_metadata; if (external || is_special_tx) { @@ -5456,6 +5465,26 @@ bool Collator::update_account_dict_estimation(const block::transaction::Transact return true; } +/** + * Update `storage_stat_updates` for AccountStorageDict for the new transaction. + * + * @param trans Newly-created transaction. + */ +void Collator::update_account_storage_dict_info(const block::transaction::Transaction& trans) { + if (!trans.account.orig_storage_dict_hash) { + return; + } + auto it = account_storage_dicts_.find(trans.account.orig_storage_dict_hash.value()); + if (it == account_storage_dicts_.end()) { + return; + } + for (const Ref& cell : trans.storage_stats_updates) { + if (cell.not_null()) { + it->second.storage_stat_updates.push_back(cell); + } + } +} + /** * Creates a new shard state and the Merkle update. * From ad123c4879ef2ebf25eeb1fc0a058d450c9c8f36 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 13 Mar 2025 11:42:07 +0300 Subject: [PATCH 151/524] Rework estimating and generating collated data --- crypto/block/account-storage-stat.h | 12 +++ crypto/block/block.cpp | 7 +- crypto/block/block.h | 4 +- crypto/block/transaction.cpp | 35 +++------ crypto/vm/boc.cpp | 30 +++++--- crypto/vm/boc.h | 12 +-- crypto/vm/cells/CellUsageTree.h | 8 +- crypto/vm/cells/MerkleProof.h | 3 + validator/impl/collator-impl.h | 9 ++- validator/impl/collator.cpp | 113 +++++++++++++++++----------- 10 files changed, 142 insertions(+), 91 deletions(-) diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h index 69ac218fe0..ed3f58d82f 100644 --- a/crypto/block/account-storage-stat.h +++ b/crypto/block/account-storage-stat.h @@ -115,4 +115,16 @@ class AccountStorageStat { static constexpr td::uint32 MERKLE_DEPTH_LIMIT = 3; }; +class StorageStatCalculationContext : public td::Context { + public: + explicit StorageStatCalculationContext(bool active) : active_(active) { + } + bool calculating_storage_stat() const { + return active_; + } + + private: + bool active_ = false; +}; + } // namespace block diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index eb5640757b..d06e7a40cb 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -751,14 +751,13 @@ td::uint64 BlockLimitStatus::estimate_block_size(const vm::NewCellStorageStat::S } int BlockLimitStatus::classify() const { - return limits.classify(estimate_block_size(), gas_used, cur_lt, collated_data_stat.estimate_proof_size()); + return limits.classify(estimate_block_size(), gas_used, cur_lt, collated_data_size_estimate); } bool BlockLimitStatus::fits(unsigned cls) const { return cls >= ParamLimits::limits_cnt || (limits.gas.fits(cls, gas_used) && limits.lt_delta.fits(cls, cur_lt - limits.start_lt) && - limits.bytes.fits(cls, estimate_block_size()) && - limits.collated_data.fits(cls, collated_data_stat.estimate_proof_size())); + limits.bytes.fits(cls, estimate_block_size()) && limits.collated_data.fits(cls, collated_data_size_estimate)); } bool BlockLimitStatus::would_fit(unsigned cls, ton::LogicalTime end_lt, td::uint64 more_gas, @@ -766,7 +765,7 @@ bool BlockLimitStatus::would_fit(unsigned cls, ton::LogicalTime end_lt, td::uint return cls >= ParamLimits::limits_cnt || (limits.gas.fits(cls, gas_used + more_gas) && limits.lt_delta.fits(cls, std::max(cur_lt, end_lt) - limits.start_lt) && limits.bytes.fits(cls, estimate_block_size(extra)) && - limits.collated_data.fits(cls, collated_data_stat.estimate_proof_size())); + limits.collated_data.fits(cls, collated_data_size_estimate)); } // SETS: account_dict, shard_libraries_, mc_state_extra diff --git a/crypto/block/block.h b/crypto/block/block.h index c6c988cdd4..65834d79e9 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -280,7 +280,7 @@ struct BlockLimitStatus { td::uint64 gas_used{}; vm::NewCellStorageStat st_stat; unsigned accounts{}, transactions{}, extra_out_msgs{}; - vm::ProofStorageStat collated_data_stat; + td::uint64 collated_data_size_estimate = 0; unsigned public_library_diff{}; BlockLimitStatus(const BlockLimits& limits_, ton::LogicalTime lt = 0) : limits(limits_), cur_lt(std::max(limits_.start_lt, lt)) { @@ -292,7 +292,7 @@ struct BlockLimitStatus { gas_used = 0; extra_out_msgs = 0; public_library_diff = 0; - collated_data_stat = {}; + collated_data_size_estimate = 0; } td::uint64 estimate_block_size(const vm::NewCellStorageStat::Stat* extra = nullptr) const; int classify() const; diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 8cdf261aaa..c5310933ba 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -3068,6 +3068,8 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, if (is_account_stat && compute_phase) { storage_stat.add_hint(compute_phase->vm_loaded_cells); } + StorageStatCalculationContext context{is_account_stat}; + StorageStatCalculationContext::Guard guard{&context}; TRY_RESULT(info, storage_stat.replace_roots({new_code, new_data, new_library})); if (info.max_merkle_depth > max_allowed_merkle_depth) { return td::Status::Error("too big Merkle depth"); @@ -3369,10 +3371,15 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } auto roots = new_storage->prefetch_all_refs(); storage_stats_updates.insert(storage_stats_updates.end(), roots.begin(), roots.end()); - auto S = stats.replace_roots(std::move(roots)).move_as_status(); - if (S.is_error()) { - LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error(); - return false; + { + StorageStatCalculationContext context{true}; + StorageStatCalculationContext::Guard guard{&context}; + auto S = stats.replace_roots(std::move(roots)).move_as_status(); + if (S.is_error()) { + LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " + << S.move_as_error(); + return false; + } } new_storage_dict_hash = stats.get_dict_hash(); // Root of AccountStorage is not counted in AccountStorageStat @@ -3391,26 +3398,6 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { if (!cfg.store_storage_dict_hash) { new_storage_dict_hash = {}; } - if (false) { - vm::CellStorageStat control_stats; - control_stats.add_used_storage(new_storage); - if (control_stats.bits != new_storage_used.bits || control_stats.cells != new_storage_used.cells) { - LOG(ERROR) << " [ QQQQQQ 1 Wrong storage stat " << account.workchain << ":" << account.addr.to_hex() << " " - << start_lt << " : " << new_storage_used.cells << "," << new_storage_used.bits - << " != " << control_stats.cells << "," << control_stats.bits << " ] "; - return false; - } - AccountStorageStat control_stats_2; - control_stats_2.replace_roots(new_storage->prefetch_all_refs()); - if (control_stats_2.get_total_bits() + new_storage->size() != new_storage_used.bits || - control_stats_2.get_total_cells() + 1 != new_storage_used.cells) { - LOG(ERROR) << " [ QQQQQQ 2 Wrong storage stat " << account.workchain << ":" << account.addr.to_hex() << " " - << start_lt << " : " << new_storage_used.cells << "," << new_storage_used.bits - << " != " << control_stats_2.get_total_cells() + 1 << "," - << control_stats_2.get_total_bits() + new_storage->size() << " ] "; - return false; - } - } CHECK(cb.store_long_bool(1, 1) // account$1 && cb.append_cellslice_bool(account.my_addr) // addr:MsgAddressInt diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 67a443f062..073f264665 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -1259,32 +1259,44 @@ bool VmStorageStat::add_storage(const CellSlice& cs) { return true; } -void ProofStorageStat::add_cell(const Ref& cell) { - auto& status = cells_[cell->get_hash()]; +void ProofStorageStat::add_loaded_cell(const Ref& cell) { + auto& [status, size] = cells_[cell->get_hash()]; if (status == c_loaded) { return; } - if (status == c_prunned) { - proof_size_ -= estimate_prunned_size(); - } + proof_size_ -= size; status = c_loaded; - proof_size_ += estimate_serialized_size(cell); + proof_size_ += size = estimate_serialized_size(cell); for (unsigned i = 0; i < cell->size_refs(); ++i) { - auto& child_status = cells_[cell->get_ref(i)->get_hash()]; + auto& [child_status, child_size] = cells_[cell->get_ref(i)->get_hash()]; if (child_status == c_none) { child_status = c_prunned; - proof_size_ += estimate_prunned_size(); + proof_size_ += child_size = estimate_prunned_size(); + } + } +} + +void ProofStorageStat::add_loaded_cells(const ProofStorageStat& other) { + for (const auto& [hash, x] : other.cells_) { + const auto& [new_status, new_size] = x; + auto& [old_status, old_size] = cells_[hash]; + if (old_status >= new_status) { + continue; } + proof_size_ -= old_size; + old_status = new_status; + proof_size_ += old_size = new_size; } } + td::uint64 ProofStorageStat::estimate_proof_size() const { return proof_size_; } ProofStorageStat::CellStatus ProofStorageStat::get_cell_status(const Cell::Hash& hash) const { auto it = cells_.find(hash); - return it == cells_.end() ? c_none : it->second; + return it == cells_.end() ? c_none : it->second.first; } td::uint64 ProofStorageStat::estimate_prunned_size() { diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 64cdbc928e..59a984c1c2 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -165,19 +165,21 @@ struct VmStorageStat { class ProofStorageStat { public: - void add_cell(const Ref& cell); + void add_loaded_cell(const Ref& cell); + void add_loaded_cells(const ProofStorageStat& other); td::uint64 estimate_proof_size() const; - enum CellStatus { - c_none = 0, c_prunned = 1, c_loaded = 2 - }; + enum CellStatus { c_none = 0, c_prunned = 1, c_loaded = 2 }; CellStatus get_cell_status(const Cell::Hash& hash) const; + bool is_loaded(const Cell::Hash& hash) const { + return get_cell_status(hash) == c_loaded; + } static td::uint64 estimate_prunned_size(); static td::uint64 estimate_serialized_size(const Ref& cell); private: - td::HashMap cells_; + td::HashMap> cells_; td::uint64 proof_size_ = 0; }; diff --git a/crypto/vm/cells/CellUsageTree.h b/crypto/vm/cells/CellUsageTree.h index c37b2c2d0b..d845be10d7 100644 --- a/crypto/vm/cells/CellUsageTree.h +++ b/crypto/vm/cells/CellUsageTree.h @@ -67,7 +67,11 @@ class CellUsageTree : public std::enable_shared_from_this { cell_load_callback_ = std::move(f); } void set_ignore_loads(bool value) { - ignore_loads_ = value; + if (value) { + ++ignore_loads_; + } else { + --ignore_loads_; + } } private: @@ -83,6 +87,6 @@ class CellUsageTree : public std::enable_shared_from_this { void on_load(NodeId node_id, const td::Ref& cell); NodeId create_node(NodeId parent); - bool ignore_loads_ = false; + int ignore_loads_ = 0; }; } // namespace vm diff --git a/crypto/vm/cells/MerkleProof.h b/crypto/vm/cells/MerkleProof.h index fc2cb6ebde..96a31d64e7 100644 --- a/crypto/vm/cells/MerkleProof.h +++ b/crypto/vm/cells/MerkleProof.h @@ -63,6 +63,9 @@ class MerkleProofBuilder { Ref root() const { return usage_root; } + Ref original_root() const { + return orig_root; + } td::Result> extract_proof() const; bool extract_proof_to(Ref &proof_root) const; td::Result extract_proof_boc() const; diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index b02fbedc33..6e5f876eeb 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -185,6 +185,7 @@ class Collator final : public td::actor::Actor { td::RefInt256 masterchain_create_fee_, basechain_create_fee_; std::unique_ptr block_limits_; std::unique_ptr block_limit_status_; + vm::ProofStorageStat collated_data_stat; int block_limit_class_ = 0; ton::LogicalTime min_new_msg_lt{std::numeric_limits::max()}; block::CurrencyCollection total_balance_, old_total_balance_, total_validator_fees_; @@ -223,8 +224,7 @@ class Collator final : public td::actor::Actor { struct AccountStorageDict { bool inited = false; vm::MerkleProofBuilder mpb; - Ref proof_root; - size_t proof_size_estimate = 0; + vm::ProofStorageStat proof_stat; bool add_to_collated_data = false; std::vector> storage_stat_updates; }; @@ -405,6 +405,11 @@ class Collator final : public td::actor::Actor { CollationStats stats_; void finalize_stats(); + + AccountStorageDict* current_tx_storage_dict_ = nullptr; + + void on_cell_loaded(const Ref& cell); + void set_current_tx_storage_dict(const block::Account& account); }; } // namespace validator diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 00699997cb..29151a02d8 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -944,10 +944,8 @@ void Collator::got_neighbor_msg_queue(unsigned i, Ref res) { neighbor_proof_builders_.push_back(vm::MerkleProofBuilder{res->state_root_}); state_root = neighbor_proof_builders_.back().root(); if (full_collated_data_ && !block_id.is_masterchain()) { - neighbor_proof_builders_.back().set_cell_load_callback([&](const td::Ref& cell) { - if (block_limit_status_) { - block_limit_status_->collated_data_stat.add_cell(cell); - } + neighbor_proof_builders_.back().set_cell_load_callback([&](const Ref& cell) { + on_cell_loaded(cell); }); } } @@ -1056,10 +1054,8 @@ bool Collator::unpack_merge_last_state() { // 1. prepare for creating a MerkleUpdate based on previous state state_usage_tree_ = std::make_shared(); if (full_collated_data_ && !is_masterchain()) { - state_usage_tree_->set_cell_load_callback([&](const td::Ref& cell) { - if (block_limit_status_) { - block_limit_status_->collated_data_stat.add_cell(cell); - } + state_usage_tree_->set_cell_load_callback([&](const Ref& cell) { + on_cell_loaded(cell); }); } prev_state_root_ = vm::UsageCell::create(prev_state_root_pure_, state_usage_tree_->root_ptr()); @@ -1107,10 +1103,8 @@ bool Collator::unpack_last_state() { // prepare for creating a MerkleUpdate based on previous state state_usage_tree_ = std::make_shared(); if (full_collated_data_ && !is_masterchain()) { - state_usage_tree_->set_cell_load_callback([&](const td::Ref& cell) { - if (block_limit_status_) { - block_limit_status_->collated_data_stat.add_cell(cell); - } + state_usage_tree_->set_cell_load_callback([&](const Ref& cell) { + on_cell_loaded(cell); }); } prev_state_root_ = vm::UsageCell::create(prev_state_root_pure_, state_usage_tree_->root_ptr()); @@ -1596,6 +1590,7 @@ bool Collator::init_block_limits() { } block_limits_->usage_tree = state_usage_tree_.get(); block_limit_status_ = std::make_unique(*block_limits_); + block_limit_status_->collated_data_size_estimate = collated_data_stat.estimate_proof_size(); return true; } @@ -2653,10 +2648,8 @@ bool Collator::init_account_storage_dict(block::Account& account) { << ": dict is empty"); } dict.mpb = vm::MerkleProofBuilder(res.move_as_ok()); - dict.mpb.set_cell_load_callback([&](const td::Ref& cell) { - if (block_limit_status_) { - block_limit_status_->collated_data_stat.add_cell(cell); - } + dict.mpb.set_cell_load_callback([&](const Ref& cell) { + on_cell_loaded(cell); }); } auto S = account.init_account_storage_stat(dict.mpb.root()); @@ -2756,7 +2749,7 @@ bool Collator::process_account_storage_dict(const block::Account& account) { from_old_state = true; } if (calculate_proof_size_diff && from_old_state) { - switch (block_limit_status_->collated_data_stat.get_cell_status(cell->get_hash())) { + switch (collated_data_stat.get_cell_status(cell->get_hash())) { case vm::ProofStorageStat::c_none: proof_size_diff += vm::ProofStorageStat::estimate_serialized_size(loaded_cell.data_cell); break; @@ -2781,14 +2774,16 @@ bool Collator::process_account_storage_dict(const block::Account& account) { } state_usage_tree_->set_ignore_loads(false); - if (proof_size_diff > (td::int64)dict.proof_size_estimate) { + if (proof_size_diff > (td::int64)dict.proof_stat.estimate_proof_size()) { LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() - << " : account_proof_size=" << proof_size_diff << ", dict_proof_size=" << dict.proof_size_estimate - << ", include dict in collated data"; + << " : account_proof_size=" << proof_size_diff + << ", dict_proof_size=" << dict.proof_stat.estimate_proof_size() << ", include dict in collated data"; dict.add_to_collated_data = true; + collated_data_stat.add_loaded_cells(dict.proof_stat); } else { LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() - << " : account_proof_size=" << proof_size_diff << ", dict_proof_size=" << dict.proof_size_estimate + << " : account_proof_size=" << proof_size_diff + << ", dict_proof_size=" << dict.proof_stat.estimate_proof_size() << ", DO NOT include dict in collated data"; // Include account storage in collated data calculate_proof_size_diff = false; @@ -2807,16 +2802,6 @@ bool Collator::process_account_storage_dict(const block::Account& account) { * @returns True if the operation is successful, false otherwise. */ bool Collator::combine_account_transactions() { - for (auto& [hash, dict] : account_storage_dicts_) { - auto res = dict.mpb.extract_proof(); - if (res.is_error()) { - return fatal_error(res.move_as_error_prefix(PSTRING() << "Failed to generate proof for account storage dict " - << hash.to_hex() << ": ")); - } - dict.proof_root = res.move_as_ok(); - dict.proof_size_estimate = vm::std_boc_serialize(dict.proof_root, 31).move_as_ok().size(); - } - vm::AugmentedDictionary dict{256, block::tlb::aug_ShardAccountBlocks}; for (auto& z : accounts) { block::Account& acc = *(z.second); @@ -3041,6 +3026,7 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t << "last transaction time in the state of account " << workchain() << ":" << smc_addr.to_hex() << " is too large")); } + set_current_tx_storage_dict(*acc); std::unique_ptr trans = std::make_unique( *acc, mask == 2 ? block::transaction::Transaction::tr_tick : block::transaction::Transaction::tr_tock, req_start_lt, now_); @@ -3145,6 +3131,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, if (it != last_dispatch_queue_emitted_lt_.end()) { after_lt = std::max(after_lt, it->second); } + set_current_tx_storage_dict(*acc); auto res = impl_create_ordinary_transaction(msg_root, acc, now_, start_lt, &storage_phase_cfg_, &compute_phase_cfg_, &action_phase_cfg_, &serialize_cfg_, external, after_lt); if (res.is_error()) { @@ -3900,7 +3887,7 @@ static std::string block_full_comment(const block::BlockLimitStatus& block_limit if (!block_limit_status.limits.lt_delta.fits(cls, lt_delta)) { return PSTRING() << "block_full lt_delta " << lt_delta; } - auto collated_data_bytes = block_limit_status.collated_data_stat.estimate_proof_size(); + auto collated_data_bytes = block_limit_status.collated_data_size_estimate; if (!block_limit_status.limits.collated_data.fits(cls, collated_data_bytes)) { return PSTRING() << "block_full collated_data " << collated_data_bytes; } @@ -5170,7 +5157,7 @@ bool Collator::check_block_overload() { LOG(INFO) << "block load statistics: gas=" << block_limit_status_->gas_used << " lt_delta=" << block_limit_status_->cur_lt - block_limit_status_->limits.start_lt << " size_estimate=" << block_size_estimate_ - << " collated_size_estimate=" << block_limit_status_->collated_data_stat.estimate_proof_size(); + << " collated_size_estimate=" << block_limit_status_->collated_data_size_estimate; block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); if (block_limit_class_ >= block::ParamLimits::cl_soft || dispatch_queue_total_limit_reached_) { std::string message = "block is overloaded "; @@ -6015,7 +6002,9 @@ bool Collator::create_collated_data() { } state_usage_tree_->set_use_mark_for_is_loaded(false); - Ref state_proof = vm::MerkleProof::generate(prev_state_root_, state_usage_tree_.get()); + Ref state_proof = vm::MerkleProof::generate(prev_state_root_, [&](const Ref& c) { + return !collated_data_stat.is_loaded(c->get_hash()); + }); if (state_proof.is_null()) { return fatal_error("cannot generate Merkle proof for previous state"); } @@ -6035,11 +6024,9 @@ bool Collator::create_collated_data() { } // 4. Proofs for message queues for (vm::MerkleProofBuilder &mpb : neighbor_proof_builders_) { - auto r_proof = mpb.extract_proof(); - if (r_proof.is_error()) { - return fatal_error(r_proof.move_as_error_prefix("cannot generate Merkle proof for neighbor: ")); - } - Ref proof = r_proof.move_as_ok(); + Ref proof = vm::MerkleProof::generate(mpb.original_root(), [&](const Ref& c) { + return !collated_data_stat.is_loaded(c->get_hash()); + }); if (proof.is_null()) { return fatal_error("cannot generate Merkle proof for neighbor"); } @@ -6061,11 +6048,15 @@ bool Collator::create_collated_data() { if (!dict.add_to_collated_data) { continue; } - CHECK(dict.proof_root.not_null()); + Ref proof = vm::MerkleProof::generate( + dict.mpb.original_root(), [&](const Ref& c) { return !collated_data_stat.is_loaded(c->get_hash()); }); + if (proof.is_null()) { + return fatal_error("cannot generate Merkle proof for neighbor"); + } // account_storage_dict_proof#37c1e3fc proof:^Cell = AccountStorageDictProof; collated_roots_.push_back(vm::CellBuilder() .store_long(block::gen::AccountStorageDictProof::cons_tag[0], 32) - .store_ref(dict.proof_root) + .store_ref(proof) .finalize_novm()); } return true; @@ -6123,7 +6114,7 @@ bool Collator::create_block_candidate() { << st.internal_refs << " " << st.external_refs << " " << block_limit_status_->accounts << " " << block_limit_status_->transactions; LOG(INFO) << "serialized collated data size " << cdata_slice.size() << " bytes (preliminary estimate was " - << block_limit_status_->collated_data_stat.estimate_proof_size() << ")"; + << block_limit_status_->collated_data_size_estimate << ")"; auto new_block_id_ext = ton::BlockIdExt{ton::BlockId{shard_, new_block_seqno}, new_block->get_hash().bits(), block::compute_file_hash(blk_slice.as_slice())}; // 3. create a BlockCandidate @@ -6371,7 +6362,7 @@ void Collator::finalize_stats() { stats_.estimated_bytes = block_limit_status_->estimate_block_size(); stats_.gas = block_limit_status_->gas_used; stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; - stats_.estimated_collated_data_bytes = block_limit_status_->collated_data_stat.estimate_proof_size(); + stats_.estimated_collated_data_bytes = block_limit_status_->collated_data_size_estimate; stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.estimated_bytes); stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas); stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); @@ -6384,6 +6375,42 @@ void Collator::finalize_stats() { stats_.time_stats = (PSTRING() << perf_log_); } +/** + * This method is called when cell from shard state is loaded. + * Only if full_collated_data is set. + * When storage stat is calculated, StorageStatCalculationContext::calculating_storage_stat() returns true. + * In this case, loaded cell is stored in a separate StorageStat for the storage dict of the account (if it exists). + * + * @param cell Loaded cell + */ +void Collator::on_cell_loaded(const Ref& cell) { + auto context = block::StorageStatCalculationContext::get(); + vm::ProofStorageStat* stat = (context && context->calculating_storage_stat() && current_tx_storage_dict_ + ? ¤t_tx_storage_dict_->proof_stat + : &collated_data_stat); + if (block_limit_status_) { + block_limit_status_->collated_data_size_estimate -= stat->estimate_proof_size(); + } + stat->add_loaded_cell(cell); + if (block_limit_status_) { + block_limit_status_->collated_data_size_estimate += stat->estimate_proof_size(); + } +} + +/** + * Initializes current_tx_storage_dict_ to be used in on_cell_loaded. + * + * @param account Account of the current transaction + */ +void Collator::set_current_tx_storage_dict(const block::Account& account) { + if (!account.orig_storage_dict_hash) { + current_tx_storage_dict_ = nullptr; + return; + } + auto it = account_storage_dicts_.find(account.orig_storage_dict_hash.value()); + current_tx_storage_dict_ = it == account_storage_dicts_.end() ? nullptr : &it->second; +} + } // namespace validator } // namespace ton From 5f9d8af6c755355e43dbb01c44d7ccf3af0a0a6f Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Thu, 13 Mar 2025 08:17:40 -0400 Subject: [PATCH 152/524] Cmake refactoring (#1551) * Ignore clangd .cache directory With the in-tree build, clangd likes to put its cache files inside of /.cache. Thus, put the directory to .gitignore. * Remove redundant cmake_minimum_required calls There is no need to repeat the same information in all subprojects, the call in the root CMakeLists.txt is more than enough. * Bump minimum CMake version to 3.16 This avoids deprecation warnings on newer CMake versions. 3.16 was chosen as a version easily available on Ubuntu 20.04 -- the oldest distributive we support. * Do not automatically add -stdlib=libc++ when sanitizers are requested In my experience, adding -stdlib=libc++ during compilation of a project with C++ dependencies which are not consistently using libc++ never works. In this particular case, modern enough stdlibc++ works just fine when compiled with Clang and sanitizers, while using libc++ results in obscure linker errors. * Do not fall back to -O2 if CMAKE_BUILD_TYPE == RelWithDebInfo * Use CMAKE_{C,CXX}_COMPILER_LAUNCHER for enabling ccache Additionally, if user has already specified either of these two variables, don't override their choices. As per documentation, RULE_LAUNCH_{COMPILE,LINK} are only supposed to be used in internal CMake code. More importantly, the previous approach appended ccache two times if CMAKE_CXX_COMPILER_LAUNCHER was provided in command line or via an environment variable. * Remove unused crypto/openssl/digest.h --- .gitignore | 1 + CMakeLists.txt | 31 ++--- adnl/CMakeLists.txt | 2 - blockchain-explorer/CMakeLists.txt | 2 - catchain/CMakeLists.txt | 2 - common/CMakeLists.txt | 2 - create-hardfork/CMakeLists.txt | 2 - crypto/CMakeLists.txt | 2 - crypto/openssl/digest.h | 151 ------------------------ dht-server/CMakeLists.txt | 2 - dht/CMakeLists.txt | 2 - emulator/CMakeLists.txt | 2 - example/android/CMakeLists.txt | 2 +- fec/CMakeLists.txt | 2 - http/CMakeLists.txt | 2 - keyring/CMakeLists.txt | 2 - keys/CMakeLists.txt | 2 - lite-client/CMakeLists.txt | 2 - memprof/CMakeLists.txt | 2 - overlay/CMakeLists.txt | 2 - rldp-http-proxy/CMakeLists.txt | 2 - rldp/CMakeLists.txt | 2 - rldp2/CMakeLists.txt | 2 - storage/CMakeLists.txt | 2 - storage/storage-daemon/CMakeLists.txt | 2 - tdactor/CMakeLists.txt | 2 - tdactor/benchmark/CMakeLists.txt | 2 - tddb/CMakeLists.txt | 2 - tdfec/CMakeLists.txt | 2 - tdfec/benchmark/CMakeLists.txt | 2 - tdnet/CMakeLists.txt | 2 - tdtl/CMakeLists.txt | 2 - tdutils/CMakeLists.txt | 2 - tdutils/generate/CMakeLists.txt | 2 - terminal/CMakeLists.txt | 2 - tl-utils/CMakeLists.txt | 2 - tl/CMakeLists.txt | 2 - tl/generate/CMakeLists.txt | 2 - tolk/CMakeLists.txt | 2 - tonlib/CMakeLists.txt | 2 - utils/CMakeLists.txt | 2 - validator-engine-console/CMakeLists.txt | 2 - validator-engine/CMakeLists.txt | 2 - validator-session/CMakeLists.txt | 2 - validator/CMakeLists.txt | 2 - validator/impl/CMakeLists.txt | 2 - 46 files changed, 19 insertions(+), 250 deletions(-) delete mode 100644 crypto/openssl/digest.h diff --git a/.gitignore b/.gitignore index e5bb366cee..ffcf936986 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ crypto/smartcont/auto/ test/regression-tests.cache/ *.swp **/*build*/ +.cache .idea .vscode .DS_Store diff --git a/CMakeLists.txt b/CMakeLists.txt index cea3fc7ec4..59f7c5e082 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) project(TON VERSION 0.5 LANGUAGES C CXX) set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -159,6 +159,9 @@ if (TON_USE_ROCKSDB) set(FAIL_ON_WARNINGS OFF CACHE BOOL "fail on warnings") message("Add rocksdb") add_subdirectory(third-party/rocksdb EXCLUDE_FROM_ALL) + # Broken CMake in rocksdb alters properties it has no business changing. + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE) + set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK) endif() option(USE_COROUTINES "experimental support of coroutines" OFF) @@ -188,11 +191,12 @@ include(BuildSECP256K1) # Configure CCache if available find_program(CCACHE_FOUND ccache) -#set(CCACHE_FOUND 0) if (CCACHE_FOUND) - message(STATUS "Found ccache") - set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) - set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache) + if (NOT DEFINED CMAKE_C_COMPILER_LAUNCHER AND NOT DEFINED CMAKE_CXX_COMPILER_LAUNCHER) + message(STATUS "Using ccache") + set(CMAKE_C_COMPILER_LAUNCHER ccache) + set(CMAKE_CXX_COMPILER_LAUNCHER ccache) + endif() else() message(STATUS "Could NOT find ccache") endif() @@ -341,6 +345,14 @@ add_cxx_compiler_flag("-Qunused-arguments") add_cxx_compiler_flag("-Wno-unused-private-field") add_cxx_compiler_flag("-Wno-redundant-move") +if (GCC OR CLANG) + if (CMAKE_BUILD_TYPE MATCHES "RelWithDebInfo") + # For historical reasons, CMake falls back to -O2 optimization level when CMAKE_BUILD_TYPE is + # set to RelWithDebInfo. + add_compile_options(-O3) + endif() +endif() + #add_cxx_compiler_flag("-Wno-unused-function") #add_cxx_compiler_flag("-Wno-unused-variable") #add_cxx_compiler_flag("-Wno-shorten-64-to-32") @@ -351,22 +363,13 @@ if (CLANG) #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") endif() if (TON_USE_ASAN) - if (CLANG) - add_cxx_compiler_flag("-stdlib=libc++") - endif() add_cxx_compiler_flag("-fsanitize=address") add_definitions(-DTD_USE_ASAN=1) endif() if (TON_USE_TSAN) - if (CLANG) - add_cxx_compiler_flag("-stdlib=libc++") - endif() add_cxx_compiler_flag("-fsanitize=thread") endif() if (TON_USE_UBSAN) - if (CLANG) - add_cxx_compiler_flag("-stdlib=libc++") - endif() add_cxx_compiler_flag("-fsanitize=undefined") endif() #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread") diff --git a/adnl/CMakeLists.txt b/adnl/CMakeLists.txt index 111c4c5003..f1d1bfbfb4 100644 --- a/adnl/CMakeLists.txt +++ b/adnl/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - #BEGIN internal if (NOT TON_ONLY_TONLIB) set(ADNL_HEADERS diff --git a/blockchain-explorer/CMakeLists.txt b/blockchain-explorer/CMakeLists.txt index 86cb3e7404..afd3041770 100644 --- a/blockchain-explorer/CMakeLists.txt +++ b/blockchain-explorer/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - option(NIX "Use \"ON\" for a static build." OFF) set(BLOCHAIN_EXPLORER_SOURCE diff --git a/catchain/CMakeLists.txt b/catchain/CMakeLists.txt index 3f766688eb..6175cb86b6 100644 --- a/catchain/CMakeLists.txt +++ b/catchain/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 88a3671b34..9252178b50 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(COMMON_SOURCE checksum.h errorcode.h diff --git a/create-hardfork/CMakeLists.txt b/create-hardfork/CMakeLists.txt index 3024603c2e..daa7f2e730 100644 --- a/create-hardfork/CMakeLists.txt +++ b/create-hardfork/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 069083381c..92ebf80448 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/crypto/openssl/digest.h b/crypto/openssl/digest.h deleted file mode 100644 index d0dfc4c086..0000000000 --- a/crypto/openssl/digest.h +++ /dev/null @@ -1,151 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2019 Telegram Systems LLP -*/ -#pragma once -#include - -#include -#include - -#include "td/utils/Slice.h" - -namespace digest { -struct OpensslEVP_SHA1 { - enum { digest_bytes = 20 }; - static const EVP_MD *get_evp() { - return EVP_sha1(); - } -}; - -struct OpensslEVP_SHA256 { - enum { digest_bytes = 32 }; - static const EVP_MD *get_evp() { - return EVP_sha256(); - } -}; - -struct OpensslEVP_SHA512 { - enum { digest_bytes = 64 }; - static const EVP_MD *get_evp() { - return EVP_sha512(); - } -}; - -template -class HashCtx { - EVP_MD_CTX *ctx{nullptr}; - void init(); - void clear(); - - public: - enum { digest_bytes = H::digest_bytes }; - HashCtx() { - init(); - } - HashCtx(const void *data, std::size_t len) { - init(); - feed(data, len); - } - ~HashCtx() { - clear(); - } - void reset(); - void feed(const void *data, std::size_t len); - void feed(td::Slice slice) { - feed(slice.data(), slice.size()); - } - std::size_t extract(unsigned char buffer[digest_bytes]); - std::size_t extract(td::MutableSlice slice); - std::string extract(); -}; - -template -void HashCtx::init() { - ctx = EVP_MD_CTX_create(); - reset(); -} - -template -void HashCtx::reset() { - EVP_DigestInit_ex(ctx, H::get_evp(), 0); -} - -template -void HashCtx::clear() { - EVP_MD_CTX_destroy(ctx); - ctx = nullptr; -} - -template -void HashCtx::feed(const void *data, std::size_t len) { - EVP_DigestUpdate(ctx, data, len); -} - -template -std::size_t HashCtx::extract(unsigned char buffer[digest_bytes]) { - unsigned olen = 0; - EVP_DigestFinal_ex(ctx, buffer, &olen); - assert(olen == digest_bytes); - return olen; -} - -template -std::size_t HashCtx::extract(td::MutableSlice slice) { - return extract(slice.ubegin()); -} - -template -std::string HashCtx::extract() { - unsigned char buffer[digest_bytes]; - unsigned olen = 0; - EVP_DigestFinal_ex(ctx, buffer, &olen); - assert(olen == digest_bytes); - return std::string((char *)buffer, olen); -} - -typedef HashCtx SHA1; -typedef HashCtx SHA256; -typedef HashCtx SHA512; - -template -std::size_t hash_str(unsigned char buffer[T::digest_bytes], const void *data, std::size_t size) { - T hasher(data, size); - return hasher.extract(buffer); -} - -template -std::size_t hash_two_str(unsigned char buffer[T::digest_bytes], const void *data1, std::size_t size1, const void *data2, - std::size_t size2) { - T hasher(data1, size1); - hasher.feed(data2, size2); - return hasher.extract(buffer); -} - -template -std::string hash_str(const void *data, std::size_t size) { - T hasher(data, size); - return hasher.extract(); -} - -template -std::string hash_two_str(const void *data1, std::size_t size1, const void *data2, std::size_t size2) { - T hasher(data1, size1); - hasher.feed(data2, size2); - return hasher.extract(); -} -} // namespace digest diff --git a/dht-server/CMakeLists.txt b/dht-server/CMakeLists.txt index 6daac03341..07c93f2473 100644 --- a/dht-server/CMakeLists.txt +++ b/dht-server/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/dht/CMakeLists.txt b/dht/CMakeLists.txt index 95ee706910..f8d49a2d82 100644 --- a/dht/CMakeLists.txt +++ b/dht/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/emulator/CMakeLists.txt b/emulator/CMakeLists.txt index 663c8fd26d..e1bc38f520 100644 --- a/emulator/CMakeLists.txt +++ b/emulator/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - option(EMULATOR_STATIC "Build emulator as static library" OFF) set(EMULATOR_STATIC_SOURCE diff --git a/example/android/CMakeLists.txt b/example/android/CMakeLists.txt index 0101ab6418..b0f998a8b3 100644 --- a/example/android/CMakeLists.txt +++ b/example/android/CMakeLists.txt @@ -3,7 +3,7 @@ # Sets the minimum version of CMake required to build the native library. -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) project(TON_ANDROID VERSION 0.5 LANGUAGES C CXX) diff --git a/fec/CMakeLists.txt b/fec/CMakeLists.txt index 2a3056071c..8549dc0e55 100644 --- a/fec/CMakeLists.txt +++ b/fec/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/http/CMakeLists.txt b/http/CMakeLists.txt index 4a3fccf827..8915254463 100644 --- a/http/CMakeLists.txt +++ b/http/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(HTTP_SOURCE http.h http.cpp diff --git a/keyring/CMakeLists.txt b/keyring/CMakeLists.txt index f8f610f2f8..6a5599e98d 100644 --- a/keyring/CMakeLists.txt +++ b/keyring/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(KEYRING_SOURCE keyring.h keyring.hpp diff --git a/keys/CMakeLists.txt b/keys/CMakeLists.txt index e80436b7b2..1b50236dd1 100644 --- a/keys/CMakeLists.txt +++ b/keys/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(KEYS_SOURCE keys.cpp encryptor.cpp diff --git a/lite-client/CMakeLists.txt b/lite-client/CMakeLists.txt index b28a14e9a5..03d0500957 100644 --- a/lite-client/CMakeLists.txt +++ b/lite-client/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - add_library(lite-client-common STATIC lite-client-common.cpp lite-client-common.h ext-client.cpp ext-client.h query-utils.hpp query-utils.cpp) target_link_libraries(lite-client-common PUBLIC tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto) diff --git a/memprof/CMakeLists.txt b/memprof/CMakeLists.txt index 2ccf11dfd6..677f305115 100644 --- a/memprof/CMakeLists.txt +++ b/memprof/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(MEMPROF_SOURCE memprof/memprof.cpp memprof/memprof.h diff --git a/overlay/CMakeLists.txt b/overlay/CMakeLists.txt index ab9722a60c..c2c9dc6173 100644 --- a/overlay/CMakeLists.txt +++ b/overlay/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/rldp-http-proxy/CMakeLists.txt b/rldp-http-proxy/CMakeLists.txt index f7e30c802d..f7a9d73d98 100644 --- a/rldp-http-proxy/CMakeLists.txt +++ b/rldp-http-proxy/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - add_executable(rldp-http-proxy rldp-http-proxy.cpp DNSResolver.h DNSResolver.cpp) target_include_directories(rldp-http-proxy PUBLIC $) target_link_libraries(rldp-http-proxy PRIVATE tonhttp rldp rldp2 dht tonlib git) diff --git a/rldp/CMakeLists.txt b/rldp/CMakeLists.txt index 39e0d3ca83..6884763282 100644 --- a/rldp/CMakeLists.txt +++ b/rldp/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/rldp2/CMakeLists.txt b/rldp2/CMakeLists.txt index c144ec01d5..984e7bb1a0 100644 --- a/rldp2/CMakeLists.txt +++ b/rldp2/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index 30403f5e21..5a53e73988 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/storage/storage-daemon/CMakeLists.txt b/storage/storage-daemon/CMakeLists.txt index cabc614393..5392d4ab90 100644 --- a/storage/storage-daemon/CMakeLists.txt +++ b/storage/storage-daemon/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - add_executable(embed-provider-code smartcont/embed-provider-code.cpp) add_custom_command( diff --git a/tdactor/CMakeLists.txt b/tdactor/CMakeLists.txt index 98b900a1e9..44580dd597 100644 --- a/tdactor/CMakeLists.txt +++ b/tdactor/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - #SOURCE SETS set(TDACTOR_SOURCE td/actor/core/ActorExecutor.cpp diff --git a/tdactor/benchmark/CMakeLists.txt b/tdactor/benchmark/CMakeLists.txt index c4ff79a1b2..662ae20c48 100644 --- a/tdactor/benchmark/CMakeLists.txt +++ b/tdactor/benchmark/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(BENCHMARK_SOURCE benchmark.cpp third_party/mp-queue.c diff --git a/tddb/CMakeLists.txt b/tddb/CMakeLists.txt index 55476e6488..57e69403b4 100644 --- a/tddb/CMakeLists.txt +++ b/tddb/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - #SOURCE SETS set(TDDB_UTILS_SOURCE td/db/utils/BlobView.cpp diff --git a/tdfec/CMakeLists.txt b/tdfec/CMakeLists.txt index 828ff90d50..97e4eb6918 100644 --- a/tdfec/CMakeLists.txt +++ b/tdfec/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(TDFEC_SOURCE td/fec/raptorq/Rfc.cpp td/fec/raptorq/Rfc.h diff --git a/tdfec/benchmark/CMakeLists.txt b/tdfec/benchmark/CMakeLists.txt index ee8f72cbf8..66224a6d41 100644 --- a/tdfec/benchmark/CMakeLists.txt +++ b/tdfec/benchmark/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - add_executable(benchmark-fec benchmark.cpp ) target_include_directories(benchmark-fec PUBLIC $) target_link_libraries(benchmark-fec PRIVATE tdfec) diff --git a/tdnet/CMakeLists.txt b/tdnet/CMakeLists.txt index bc00a67692..ff4ce95d05 100644 --- a/tdnet/CMakeLists.txt +++ b/tdnet/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(TDNET_SOURCE td/net/FdListener.cpp td/net/TcpListener.cpp diff --git a/tdtl/CMakeLists.txt b/tdtl/CMakeLists.txt index 482bd0f7bc..0601b92203 100644 --- a/tdtl/CMakeLists.txt +++ b/tdtl/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - #SOURCE SETS set(TDTL_SOURCE td/tl/tl_config.cpp diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index f8c191a144..716ec322a2 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - option(TDUTILS_MIME_TYPE "Generate mime types conversion (gperf is required)" ON) if (WIN32) diff --git a/tdutils/generate/CMakeLists.txt b/tdutils/generate/CMakeLists.txt index 194fda3911..3912860660 100644 --- a/tdutils/generate/CMakeLists.txt +++ b/tdutils/generate/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - # Generates files for MIME type <-> extension conversions # DEPENDS ON: gperf grep bash/powershell diff --git a/terminal/CMakeLists.txt b/terminal/CMakeLists.txt index af51153f32..2d0c503045 100644 --- a/terminal/CMakeLists.txt +++ b/terminal/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/tl-utils/CMakeLists.txt b/tl-utils/CMakeLists.txt index d5c52d48ac..709bc7cb5b 100644 --- a/tl-utils/CMakeLists.txt +++ b/tl-utils/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(TL_UTILS_SOURCE common-utils.hpp tl-utils.hpp diff --git a/tl/CMakeLists.txt b/tl/CMakeLists.txt index d0760a3492..168a5d73ef 100644 --- a/tl/CMakeLists.txt +++ b/tl/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - add_subdirectory(generate) set_source_files_properties(${TL_TON_API} PROPERTIES GENERATED TRUE) add_library(tl_api STATIC diff --git a/tl/generate/CMakeLists.txt b/tl/generate/CMakeLists.txt index 083d397370..8ff7f79a47 100644 --- a/tl/generate/CMakeLists.txt +++ b/tl/generate/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - file(MAKE_DIRECTORY auto/tl) set(TL_TON_API diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index de40811577..c4abfec04a 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(TOLK_SOURCE src-file.cpp lexer.cpp diff --git a/tonlib/CMakeLists.txt b/tonlib/CMakeLists.txt index 3dbd628d31..2e3c583d43 100644 --- a/tonlib/CMakeLists.txt +++ b/tonlib/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - option(TONLIBJSON_STATIC "Build tonlibjson as static library" OFF) if (NOT OPENSSL_FOUND) diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index e9ffd2a27b..99a4e92df5 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/validator-engine-console/CMakeLists.txt b/validator-engine-console/CMakeLists.txt index d87d8a6f3d..73f3a58775 100644 --- a/validator-engine-console/CMakeLists.txt +++ b/validator-engine-console/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - add_executable (validator-engine-console validator-engine-console.cpp validator-engine-console.h validator-engine-console-query.cpp validator-engine-console-query.h ) diff --git a/validator-engine/CMakeLists.txt b/validator-engine/CMakeLists.txt index 73949d8080..693becdc85 100644 --- a/validator-engine/CMakeLists.txt +++ b/validator-engine/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/validator-session/CMakeLists.txt b/validator-session/CMakeLists.txt index 6b4e8f6875..94363c79cf 100644 --- a/validator-session/CMakeLists.txt +++ b/validator-session/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 5f5544b226..f7a4ce1d11 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/validator/impl/CMakeLists.txt b/validator/impl/CMakeLists.txt index 978cf859a4..9ab32c47be 100644 --- a/validator/impl/CMakeLists.txt +++ b/validator/impl/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() From 8194d1f09d9165fc7a03492dd737e75e9ab697a9 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 27 Feb 2025 19:12:12 +0300 Subject: [PATCH 153/524] [Tolk] Support binary number literals 0b1100 Along with hex 0x, 0b is also parsed. Example: 0b1010101010101010 == 43690. --- tolk-tester/tests/numbers-tests.tolk | 35 +++++++++++++++++++++++ tolk/ast-from-tokens.cpp | 21 +++++++++++++- tolk/lexer.cpp | 42 +++++++++++++++++++--------- 3 files changed, 84 insertions(+), 14 deletions(-) create mode 100644 tolk-tester/tests/numbers-tests.tolk diff --git a/tolk-tester/tests/numbers-tests.tolk b/tolk-tester/tests/numbers-tests.tolk new file mode 100644 index 0000000000..2dfdee8fdc --- /dev/null +++ b/tolk-tester/tests/numbers-tests.tolk @@ -0,0 +1,35 @@ +const x1 = 0xFF; +const x2 = 0xAbcdEF01234561AC; +const x3 = 0x00BEC; +const x4 = -0xD70B; + +const d1 = 255; +const d2 = 12379813738877116844; +const d3 = 003052; +const d4 = -55051; + +const b1 = 0b11111111; +const b2 = 0b1010101111001101111011110000000100100011010001010110000110101100; +const b3 = 0b00101111101100; +const b4 = -0b1101011100001011; + +fun main() { + assert(x1 == d1 && x1 == b1, 100); + assert(x2 == d2 && x2 == b2, 100); + assert(x3 == d3 && x3 == b3, 100); + assert(x4 == d4 && x4 == b4, 100); + assert(x1 + x2 + x3 == b1 + b2 + b3, 100); + assert(-x2 == -d2 && -x2 == -b2, 100); + + return ( + 0b1010101010101010+0b00 == 43690, + 0b0 == 0, + -0b1010100001010101010-0b1+0b1 == -344746, + 0b00001 == 1, + --0b00101111101100 == 3052 + ); +} + +/** +@testcase | 0 | | -1 -1 -1 -1 -1 + */ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index fcaa115773..66e47d2189 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -123,6 +123,25 @@ static AnyExprV maybe_replace_eq_null_with_isNull_check(V v return createV(v->loc, v_nullable, v->tok == tok_neq); } +// parse `123` / `0xFF` / `0b10001` to td::RefInt256 +static td::RefInt256 parse_tok_int_const(std::string_view text) { + bool bin = text[0] == '0' && text[1] == 'b'; + if (!bin) { + // this function parses decimal and hex numbers + return td::string_to_int256(static_cast(text)); + } + // parse a binary number; to make it simpler, don't allow too long numbers, it's impractical + if (text.size() > 64 + 2) { + return {}; + } + uint64_t result = 0; + for (char c : text.substr(2)) { // skip "0b" + result = (result << 1) | static_cast(c - '0'); + } + return td::make_refint(result); +} + + /* * @@ -313,7 +332,7 @@ static AnyExprV parse_expr100(Lexer& lex) { } case tok_int_const: { std::string_view orig_str = lex.cur_str(); - td::RefInt256 intval = td::string_to_int256(static_cast(orig_str)); + td::RefInt256 intval = parse_tok_int_const(orig_str); if (intval.is_null() || !intval->signed_fits_bits(257)) { lex.error("invalid integer constant"); } diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index 06913a5fc4..86ae306a4b 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -238,27 +238,43 @@ struct ChunkAnnotation final : ChunkLexerBase { // A number, may be a hex one. struct ChunkNumber final : ChunkLexerBase { - bool parse(Lexer* lex) const override { + static bool parse_hex_or_bin(Lexer* lex, bool bin) { const char* str_begin = lex->c_str(); - bool hex = false; - if (lex->char_at() == '0' && lex->char_at(1) == 'x') { - lex->skip_chars(2); - hex = true; - } + lex->skip_chars(2); // 0x / 0b if (lex->is_eof()) { return false; } + while (!lex->is_eof()) { char c = lex->char_at(); - if (c >= '0' && c <= '9') { - lex->skip_chars(1); - continue; - } - if (!hex) { + bool ok = bin + ? c == '0' || c == '1' + : (c >= '0' && c <= '9') || ((c | 0x20) >= 'a' && (c | 0x20) <= 'f'); + if (!ok) { break; } - c |= 0x20; - if (c < 'a' || c > 'f') { + lex->skip_chars(1); + } + + std::string_view str_val(str_begin, lex->c_str() - str_begin); + lex->add_token(tok_int_const, str_val); + return true; + } + + bool parse(Lexer* lex) const override { + if (lex->char_at() == '0') { + if (lex->char_at(1) == 'x') { + return parse_hex_or_bin(lex, false); + } + if (lex->char_at(1) == 'b') { + return parse_hex_or_bin(lex, true); + } + } + + const char* str_begin = lex->c_str(); + while (!lex->is_eof()) { + char c = lex->char_at(); + if (c < '0' || c > '9') { break; } lex->skip_chars(1); From d54e708b5719349180dea2d268feb997d8dd6c76 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 28 Feb 2025 13:02:17 +0300 Subject: [PATCH 154/524] [Tolk] Support trailing comma in various places Trailing comma is now allowed in: * tensors * tuples * function calls * function parameters Note, that `(5)` is not a tensor, it's just integer `5`. With a trailing comma `(5,)` it's still `(5)`. --- tolk-tester/tests/a6.tolk | 8 ++++---- tolk-tester/tests/allow_post_modification.tolk | 2 +- tolk-tester/tests/assignment-tests.tolk | 18 ++++++++++++++---- tolk-tester/tests/inference-tests.tolk | 9 +++++++++ tolk-tester/tests/no-spaces.tolk | 10 +++++----- tolk-tester/tests/nullable-tensors.tolk | 9 +++++++++ tolk/ast-from-tokens.cpp | 18 ++++++++++++++++++ 7 files changed, 60 insertions(+), 14 deletions(-) diff --git a/tolk-tester/tests/a6.tolk b/tolk-tester/tests/a6.tolk index 9449454648..ca1737a1ea 100644 --- a/tolk-tester/tests/a6.tolk +++ b/tolk-tester/tests/a6.tolk @@ -4,7 +4,7 @@ fun f(a: int, b: int, c: int, d: int, e: int, f: int): (int, int) { __expect_type(D, "int"); __expect_type(D*D, "int"); __expect_type(calc_phi, "() -> int"); - return (Dx/D,Dy/D); + return (Dx/D,Dy/D,); };;;; fun calc_phi(): int { @@ -16,7 +16,7 @@ fun calc_phi(): int { do { (p,q)=(q,p+q); } while (q <= n); //;; - return mulDivRound(p, n, q); + return mulDivRound(p, n, q,); } fun calc_sqrt2(): int { @@ -31,10 +31,10 @@ fun calc_sqrt2(): int { return mulDivRound(p, n, q); } -fun calc_root(m: int) { +fun calc_root(m: int,) { var base: int=1; repeat(70) { base *= 10; } - var (a, b, c) = (1,0,-m); + var (a, b, c) = (1,0,-m,); var (p1, q1, p2, q2) = (1, 0, 0, 1); do { var k: int=-1; diff --git a/tolk-tester/tests/allow_post_modification.tolk b/tolk-tester/tests/allow_post_modification.tolk index e20e8218ef..180654e327 100644 --- a/tolk-tester/tests/allow_post_modification.tolk +++ b/tolk-tester/tests/allow_post_modification.tolk @@ -47,7 +47,7 @@ fun test_call_1(x: int): (int, int, int, int, int, int, int) { return foo1(x, x.`~inc`(x / 20), x, x = x * 2, x, x += 1, x); } -fun foo2(x1: int, x2: int, x3456: (int, int, int, int), x7: int): (int, int, int, int, int, int, int) { +fun foo2(x1: int, x2: int, x3456: (int, int, int, int), x7: int, ): (int, int, int, int, int, int, int) { var (x3: int, x4: int, x5: int, x6: int) = x3456; return (x1, x2, x3, x4, x5, x6, x7); } diff --git a/tolk-tester/tests/assignment-tests.tolk b/tolk-tester/tests/assignment-tests.tolk index bb6476521f..f4d2c0d78e 100644 --- a/tolk-tester/tests/assignment-tests.tolk +++ b/tolk-tester/tests/assignment-tests.tolk @@ -5,7 +5,7 @@ fun extractFromTypedTuple(params: [int]) { @method_id(101) fun test101(x: int) { - var params = [x]; + var params = [x, ]; return extractFromTypedTuple(params); } @@ -38,7 +38,7 @@ fun getTensor_1X(x: int) { } fun getTuple_12() { callOrder.tuplePush(110); - return [1, 2]; + return [1, 2, ]; } fun getTuple_1X(x: int) { callOrder.tuplePush(111); @@ -133,7 +133,7 @@ fun test108() { fun test109() { callOrder = createEmptyTuple(); var x = 0; - [getTuple_12().0, getTuple_1X(x = getIntValue5()).1, getTuple_1X(x += 10).0] = [getIntValue5(), getIntValue5(), getIntValueX(x)]; + [getTuple_12().0, getTuple_1X(x = getIntValue5()).1, getTuple_1X(x += 10).0] = [getIntValue5(), getIntValue5(), getIntValueX(x), ]; return (callOrder, x); } @@ -203,9 +203,18 @@ fun test116() { return (a, b, c, d, rhs2); } +@method_id(117) +fun test117() { + var c = [((1, ), ), ]; + __expect_type(c, "[int]"); + c.0 = 10; + c.0 = (20, ); + return c; +} + -fun main(value: int) { +fun main(value: int, ) { var (x: int?, y) = (autoInferIntNull(value), autoInferIntNull(value * 2)); if (x == null && y == null) { return null; } return x == null || y == null ? -1 : x! + y!; @@ -231,6 +240,7 @@ fun main(value: int) { @testcase | 114 | | 13 [ 1 14 ] 1 3 @testcase | 115 | | [ 101 111 ] 9 9 @testcase | 116 | | 1 2 3 4 [ 1 2 3 4 ] +@testcase | 117 | | [ 20 ] @fif_codegen diff --git a/tolk-tester/tests/inference-tests.tolk b/tolk-tester/tests/inference-tests.tolk index 5020d0ddc8..e785ac5aa4 100644 --- a/tolk-tester/tests/inference-tests.tolk +++ b/tolk-tester/tests/inference-tests.tolk @@ -30,6 +30,15 @@ fun test1(x: int, y: int) { __expect_type(beginCell(), "builder"); __expect_type(beginCell().endCell(), "cell"); } + __expect_type((1, 2), "(int, int)"); + __expect_type((1, 2, ), "(int, int)"); + __expect_type((1, ), "int"); + __expect_type((((1,)),), "int"); + __expect_type([1, 2], "[int, int]"); + __expect_type([1, 2, ], "[int, int]"); + __expect_type([1, ], "[int]"); + __expect_type([((1, ), ), ], "[int]"); + __expect_type([[[1, ], ], ], "[[[int]]]"); } fun test2(x: int, y: bool) { diff --git a/tolk-tester/tests/no-spaces.tolk b/tolk-tester/tests/no-spaces.tolk index 409bb342ca..d63ca14110 100644 --- a/tolk-tester/tests/no-spaces.tolk +++ b/tolk-tester/tests/no-spaces.tolk @@ -3,11 +3,11 @@ const int10:int=10; fun just10(): int { return int10; } fun eq(v: int): int { return`v`; } -@method_id(101) fun `get_-1` (): int {return-1;} -@method_id(102) fun `get_--1` (): int {return--1;} -@method_id(103) fun `get_---1`(): int {return---1;} -@method_id(104) fun `get_+++1`(): int {return+++1;} -@method_id(105) fun `get_+-+1`(): int {return+-+1;} +@method_id(101,) fun `get_-1` (): int {return-1;} +@method_id(102,) fun `get_--1` (): int {return--1;} +@method_id(103,) fun `get_---1`(): int {return---1;} +@method_id(104,) fun `get_+++1`(): int {return+++1;} +@method_id(105,) fun `get_+-+1`(): int {return+-+1;} global `some()var`:int; diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index d0720273aa..6de907d028 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -415,6 +415,13 @@ fun test135() { t1!.0 == null, t2!.0 == null, e1!.1.0 == null, e1!.1.1 == null, e2!.1.0 == null, e2!.1.1 == null); } +@method_id(136, ) +fun test136(x: int?, ) { + var t1 = (x, ); // it's not a tensor with 1 element + __expect_type(t1, "int?", ); + return ((t1, t1 == null, ), ); // testing trailing comma everywhere :) +} + fun main(){} @@ -466,6 +473,8 @@ fun main(){} @testcase | 133 | | 60 @testcase | 134 | | 11 21 -1 @testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 -1 (null) -1 (null) 0 777 10 -1 (null) -1 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 +@testcase | 136 | 9 | 9 0 +@testcase | 136 | null | (null) -1 @fif_codegen """ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 66e47d2189..22974b1f56 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -236,6 +236,9 @@ static V parse_parameter_list(Lexer& lex) { params.push_back(parse_parameter(lex, true)); while (lex.tok() == tok_comma) { lex.next(); + if (lex.tok() == tok_clpar) { // trailing comma + break; + } params.push_back(parse_parameter(lex, false)); } } @@ -266,6 +269,9 @@ static V parse_argument_list(Lexer& lex) { args.push_back(parse_argument(lex)); while (lex.tok() == tok_comma) { lex.next(); + if (lex.tok() == tok_clpar) { // trailing comma + break; + } args.push_back(parse_argument(lex)); } } @@ -311,9 +317,15 @@ static AnyExprV parse_expr100(Lexer& lex) { std::vector items(1, first); while (lex.tok() == tok_comma) { lex.next(); + if (lex.tok() == tok_clpar) { // trailing comma + break; + } items.emplace_back(parse_expr(lex)); } lex.expect(tok_clpar, "`)`"); + if (items.size() == 1) { // we can reach here for 1 element with trailing comma: `(item, )` + return items[0]; // then just return item, not a 1-element tensor, + } // since 1-element tensors won't be type compatible with item's type return createV(loc, std::move(items)); } case tok_opbracket: { @@ -325,6 +337,9 @@ static AnyExprV parse_expr100(Lexer& lex) { std::vector items(1, parse_expr(lex)); while (lex.tok() == tok_comma) { lex.next(); + if (lex.tok() == tok_clbracket) { // trailing comma + break; + } items.emplace_back(parse_expr(lex)); } lex.expect(tok_clbracket, "`]`"); @@ -952,6 +967,9 @@ static V parse_annotation(Lexer& lex) { args.push_back(parse_expr(lex)); while (lex.tok() == tok_comma) { lex.next(); + if (lex.tok() == tok_clpar) { // trailing comma + break; + } args.push_back(parse_expr(lex)); } lex.expect(tok_clpar, "`)`"); From 49cc6d0e06b6510ce4316d2ff763b02ec00c5060 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 13 Mar 2025 18:00:34 +0400 Subject: [PATCH 155/524] [Tolk] Types `int32`, `uint64`, and fixed ints in general MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces fixed-size integer types: * intN for signed integers (0 < N ≤ 256) * uintN for unsigned integers (0 < N ≤ 257) The generic `int` type remains unchanged and can be implicitly cast to and from any `intN`. * arithmetic operations on `intN` degrade to `int` * numeric literals (like 0, 100) are just `int` * direct assignment between `intN` and `intM` is disallowed (as a probable error) There is no runtime overflow. A variable is declared as `int8` can hold any 257-bit value during execution. Overflow will occur when packing to a cell (in the future). --- tolk-tester/tests/intN-tests.tolk | 178 +++++++++++++++++++++++ tolk-tester/tests/invalid-typing-31.tolk | 10 ++ tolk-tester/tests/invalid-typing-32.tolk | 8 + tolk-tester/tests/invalid-typing-33.tolk | 10 ++ tolk-tester/tests/invalid-typing-34.tolk | 16 ++ tolk-tester/tests/nullable-tensors.tolk | 7 + tolk/pipe-ast-to-legacy.cpp | 25 +++- tolk/pipe-check-inferred-types.cpp | 8 +- tolk/pipe-infer-types-and-calls.cpp | 11 +- tolk/smart-casts-cfg.cpp | 4 + tolk/type-system.cpp | 78 +++++++++- tolk/type-system.h | 25 ++++ 12 files changed, 370 insertions(+), 10 deletions(-) create mode 100644 tolk-tester/tests/intN-tests.tolk create mode 100644 tolk-tester/tests/invalid-typing-31.tolk create mode 100644 tolk-tester/tests/invalid-typing-32.tolk create mode 100644 tolk-tester/tests/invalid-typing-33.tolk create mode 100644 tolk-tester/tests/invalid-typing-34.tolk diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk new file mode 100644 index 0000000000..3f4cdc015c --- /dev/null +++ b/tolk-tester/tests/intN-tests.tolk @@ -0,0 +1,178 @@ +fun takeAnyInt(a: int) { } +fun getAnyInt(): int { return 0; } +fun getNullableInt32(): int32? { return 32; } +fun getNullableVarInt32(): varint32? { return 32; } + + +fun autoInferInt8(x: int8) { + if (random()) { return x; } + else if (random()) { return 0 as int8; } + else if (random()) { return x!; } + else { return x += x; } +} + +fun autoInferUInt16Opt(x: uint16) { + if (random()) { return autoInferInt8(x as int8) as uint16; } + return null; +} + +fun autoInferInt_v1() { + if (random()) { return 0; } + else if (random()) { return 0 as uint16; } + else { return 0 as int8; } +} + +fun autoInferInt_v2() { + if (random()) { return 0 as uint1; } + else if (random()) { return 0 as int123; } + else { return 0 as int8; } +} + +@method_id(101) +fun test1(x: int8) { + var y: int8 = x; + y += x; + x *= y; + x = 1000000; + y = x; + __expect_type(x / y, "int"); + __expect_type(x & y, "int"); + __expect_type(x != y, "bool"); + if (x != y) { return -1; } + if (!(y == x)) { return -1; } + + var a1: uint8 = 0; + var a2: uint8? = 0; + var a3: uint8? = 0 as int?; + __expect_type(a1, "uint8"); + __expect_type(a2, "uint8"); + __expect_type(a3, "uint8?"); + + var z = x + y; + __expect_type(z, "int"); + return x / y; +} + +fun test2(): (uint8, uint8, uint8) { + var x: uint1 = 1; + return (1, x as uint8, x as int); +} + +fun test3(op: int32, qid: uint64) { + op = qid as int32; + + op + qid; + op = op + qid; + op = op & qid; + op += qid; + op &= qid; + if (op == qid) {} + if ((op as int32?)! == qid) {} + if (op == (qid as uint64)) {} + if ((op as uint257?)! != (qid as int256?)!) {} + __expect_type(op << qid, "int"); + + takeAnyInt(op); + op = getAnyInt(); + + __expect_type(autoInferInt8, "(int8) -> int8"); + __expect_type(autoInferUInt16Opt(0), "uint16?"); + __expect_type(autoInferInt_v1(), "int"); + __expect_type(autoInferInt_v2(), "int"); + + var amount: uint100 = 1000; + var percent: uint8 = 50; + var new = amount * percent / 100; + amount = new; +} + +@method_id(104) +fun test4(): (int32, int32?, bool, bool) { + var x = getNullableInt32(); + __expect_type(x, "int32?"); + if (x! != x! || x! != 32) { + return (0, null, false, false); + } + if (x == null) { + return (-1, null, false, false); + } + x += x; + return (x, x, x == x, x == getAnyInt()); +} + +@method_id(105) +fun test5() { + var (x: int8, y: uint16) = (1, 2); + var cell = beginCell().storeInt(x, 32).storeInt(y, 32 as int32).endCell(); + var slice = cell.beginParse(); + x = slice.loadInt(32); + y = slice.loadInt(32); + __expect_type(x & y, "int"); + return (x + y, x && y, x & y); +} + +fun test6() { + var x: int11 = ~(0 as int22); + while (x) {} + return x ? x : 0; +} + +fun test7() { + var n = getNullableVarInt32(); + __expect_type(n!, "varint32"); + __expect_type(n, "varint32?"); + if (n != null) { + __expect_type(n, "varint32"); + __expect_type(n += n, "varint32"); + __expect_type(n += n as int8, "varint32"); + } else { + n = 0; + n = 0 as varint32?; + n = 0 as varint32; + } + __expect_type(n, "varint32"); + __expect_type(n as uint32, "uint32"); + __expect_type(n as varint16, "varint16"); + __expect_type((n as varint16) as int, "int"); +} + +fun test13(x1: int?, x2: int8?, x3: int, x4: int8): (int8?, int8?, int8?, int8?, int32?, int32?) { + return (x1, x2, x3, x4, x4 as int32, x4 as int32?); +} + +@method_id(114) +fun test14(firstComponent: int8?): (int, int16)? { + return firstComponent! < 10 ? null : (firstComponent!, 2); +} + +fun assign0(mutate v: T) { v = 0; } + +fun main() { + var t = createEmptyTuple(); + t.tuplePush(1); + t.tuplePush(2); + t.tuplePush(3); + assign0(mutate t.0 as int8); + assign0(mutate t.1 as uint16); + assign0(mutate t.2 as int); + __expect_type(t.0 as int1, "int1"); + __expect_type(t.0 as uint257, "uint257"); + __expect_type(0 as int32, "int32"); + __expect_type((0 as int32) as uint64?, "uint64?"); + __expect_type(null as (int8, [uint16?])?, "(int8, [uint16?])?"); + __expect_type(10>3 ? 0 as uint8 : 0 as uint16, "int"); + return t; +} + +/** +@testcase | 0 | | [ 0 0 0 ] +@testcase | 101 | 0 | 1 +@testcase | 104 | | 64 64 -1 0 +@testcase | 105 | | 3 -1 0 +@testcase | 114 | 5 | (null) (null) 0 +@testcase | 114 | 15 | 15 2 -1 + +@fif_codegen DECLPROC assign0 +@fif_codegen DECLPROC assign0 +@fif_codegen DECLPROC assign0 +*/ diff --git a/tolk-tester/tests/invalid-typing-31.tolk b/tolk-tester/tests/invalid-typing-31.tolk new file mode 100644 index 0000000000..240d5b398a --- /dev/null +++ b/tolk-tester/tests/invalid-typing-31.tolk @@ -0,0 +1,10 @@ +fun testCantCastBoolToUIntN(x: int) { + var eq = x == 0; + eq as int8; // ok + eq as uint8; +} + +/** +@compilation_should_fail +@stderr type `bool` can not be cast to `uint8` + */ diff --git a/tolk-tester/tests/invalid-typing-32.tolk b/tolk-tester/tests/invalid-typing-32.tolk new file mode 100644 index 0000000000..134a8c4852 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-32.tolk @@ -0,0 +1,8 @@ +fun testUInt258DoesntExist() { + var c = 0 as uint258; +} + +/** +@compilation_should_fail +@stderr unknown type name `uint258` + */ diff --git a/tolk-tester/tests/invalid-typing-33.tolk b/tolk-tester/tests/invalid-typing-33.tolk new file mode 100644 index 0000000000..d112b36068 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-33.tolk @@ -0,0 +1,10 @@ +fun testAssignBetweenDifferentIntN(op: int32, qid: uint64) { + op = qid as int32; // ok + op = qid; +} + +/** +@compilation_should_fail +@stderr can not assign `uint64` to variable of type `int32` +@stderr op = qid; + */ diff --git a/tolk-tester/tests/invalid-typing-34.tolk b/tolk-tester/tests/invalid-typing-34.tolk new file mode 100644 index 0000000000..ed40fc48d5 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-34.tolk @@ -0,0 +1,16 @@ +fun assign0(mutate x: uint8) { + x = 0; +} + +fun testCantPassDifferentIntN(x: int8) { + assign0(mutate (x as int)); // ok + assign0(mutate (x as uint8)); // ok + assign0(mutate ((x as int64) as int)); // ok + assign0(mutate x); +} + +/** +@compilation_should_fail +@stderr can not pass `int8` to `uint8` +@stderr assign0(mutate x); + */ diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index 6de907d028..6b53306b68 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -422,6 +422,12 @@ fun test136(x: int?, ) { return ((t1, t1 == null, ), ); // testing trailing comma everywhere :) } +@method_id(140) +fun test140(x: int8, y: int16) { + var t1: (int32, int)? = (x, y) as (int32, int64)?; + var t2: (int32, int)? = null as (int32, int64)?; + return (t1, t2); +} fun main(){} @@ -475,6 +481,7 @@ fun main(){} @testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 -1 (null) -1 (null) 0 777 10 -1 (null) -1 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 @testcase | 136 | 9 | 9 0 @testcase | 136 | null | (null) -1 +@testcase | 140 | 8 9 | 8 9 -1 (null) (null) 0 @fif_codegen """ diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 1561aa4037..4347de3e82 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -535,7 +535,30 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as()) { + return rvect; + } + // pass `int8` to `int` + // it comes from auto cast when an integer (even a literal) is assigned to intN + // to changes in rvect, intN is int at TVM level + if (target_type == TypeDataInt::create() && original_type->try_as()) { + return rvect; + } + // pass `int` to `int8` + // in code, it's probably done with `as` operator + // no changes in rvect + if (original_type == TypeDataInt::create() && target_type->try_as()) { + return rvect; + } + // pass `int8` to `int16` / `int8` to `uint8` + // in code, it's probably done with `as` operator + // no changes in rvect + if (original_type->try_as() && target_type->try_as()) { return rvect; } // pass something to `unknown` diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index bae67c5fd6..5f78b9d25d 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -126,7 +126,7 @@ static void handle_possible_compiler_internal_call(FunctionPtr cur_f, Vinferred_type == TypeDataInt::create(); + return v_inferred->inferred_type == TypeDataInt::create() || v_inferred->inferred_type->try_as(); } static bool expect_boolean(AnyExprV v_inferred) { @@ -145,6 +145,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { parent::visit(rhs); // all operators (+=, etc.) can work for integers (if both sides are integers) + // for intN, they are also allowed (int16 |= int8 is ok, since int16 | int8 is ok, all arithmetic is int) bool types_ok = expect_integer(lhs) && expect_integer(rhs); // bitwise operators &= |= ^= are "overloaded" for booleans also (if both sides are booleans) if (!types_ok && (v->tok == tok_set_bitwise_and || v->tok == tok_set_bitwise_or || v->tok == tok_set_bitwise_xor)) { @@ -181,8 +182,9 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { switch (v->tok) { // == != can compare both integers and booleans, (int == bool) is NOT allowed + // for intN, it also works: (int8 == int16) is ok, (int == uint32) is ok // note, that `int?` and `int?` can't be compared, since Fift `EQUAL` works with integers only - // (if to allow `int?` in the future, `==` must be expressed in a complicated Fift code considering TVM NULL) + // (if to allow `int?`/`int8?` in the future, `==` must be expressed in a complicated Fift code considering TVM NULL) case tok_eq: case tok_neq: { bool both_int = expect_integer(lhs) && expect_integer(rhs); @@ -207,6 +209,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } break; // & | ^ are "overloaded" both for integers and booleans, (int & bool) is NOT allowed + // they are allowed for intN (int16 & int32 is ok) and always "fall back" to general int case tok_bitwise_and: case tok_bitwise_or: case tok_bitwise_xor: { @@ -228,6 +231,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { break; } // others are mathematical: + * ... + // they are allowed for intN (int16 + int32 is ok) and always "fall back" to general int default: if (!expect_integer(lhs) || !expect_integer(rhs)) { fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 5fb12059fd..ad1c59aebb 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -457,14 +457,13 @@ class InferTypesAndCallsAndFieldsVisitor final { FlowContext rhs_flow = std::move(after_lhs.out_flow); ExprFlow after_rhs = infer_any_expr(rhs, std::move(rhs_flow), false, lhs->inferred_type); - // almost all operators implementation is hardcoded by built-in functions `_+_` and similar + // all operators implementation is hardcoded by built-in functions `_+_` and similar std::string_view builtin_func = v->operator_name; // "+" for operator += assign_inferred_type(v, lhs); - if (!builtin_func.empty()) { - FunctionPtr builtin_sym = lookup_global_symbol("_" + static_cast(builtin_func) + "_")->try_as(); - v->mutate()->assign_fun_ref(builtin_sym); - } + + FunctionPtr builtin_sym = lookup_global_symbol("_" + static_cast(builtin_func) + "_")->try_as(); + v->mutate()->assign_fun_ref(builtin_sym); return ExprFlow(std::move(after_rhs.out_flow), used_as_condition); } @@ -530,7 +529,6 @@ class InferTypesAndCallsAndFieldsVisitor final { } else { assign_inferred_type(v, TypeDataInt::create()); } - assign_inferred_type(v, rhs); // (int & int) is int, (bool & bool) is bool break; // && || result in booleans, but building flow facts is tricky due to short-circuit case tok_logical_and: { @@ -560,6 +558,7 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(out_flow), std::move(true_flow), std::move(false_flow)); } // others are mathematical: + * ... + // they are allowed for intN (int16 + int32 is ok) and always "fall back" to general int default: flow = infer_any_expr(lhs, std::move(flow), false).out_flow; flow = infer_any_expr(rhs, std::move(flow), false).out_flow; diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index 7b86f519f0..9a2f7ef4df 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -192,6 +192,10 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { return TypeDataTypedTuple::create(std::move(types_lca)); } + if (a->try_as() && b->try_as()) { // cond ? int32 : int64 + return TypeDataInt::create(); + } + return nullptr; } diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index d73625c200..c88acb7195 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -18,6 +18,7 @@ #include "lexer.h" #include "platform-utils.h" #include "compiler-state.h" +#include #include namespace tolk { @@ -178,6 +179,18 @@ TypePtr TypeDataTypedTuple::create(std::vector&& items) { return hash.register_unique(new TypeDataTypedTuple(hash.type_id(), hash.children_flags(), std::move(items))); } +TypePtr TypeDataIntN::create(bool is_unsigned, bool is_variadic, int n_bits) { + TypeDataTypeIdCalculation hash(1678330938771108027ULL); + hash.feed_hash(is_unsigned); + hash.feed_hash(is_variadic); + hash.feed_hash(n_bits); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + return hash.register_unique(new TypeDataIntN(hash.type_id(), is_unsigned, is_variadic, n_bits)); +} + TypePtr TypeDataUnresolved::create(std::string&& text, SrcLocation loc) { TypeDataTypeIdCalculation hash(3680147223540048162ULL); hash.feed_string(text); @@ -240,6 +253,13 @@ std::string TypeDataTypedTuple::as_human_readable() const { return result; } +std::string TypeDataIntN::as_human_readable() const { + std::string s_int = is_variadic + ? is_unsigned ? "varuint" : "varint" + : is_unsigned ? "uint" : "int"; + return s_int + std::to_string(n_bits); +} + // -------------------------------------------- // traverse() @@ -327,6 +347,9 @@ bool TypeDataInt::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (rhs->try_as()) { + return true; + } return rhs == TypeDataNever::create(); } @@ -431,6 +454,16 @@ bool TypeDataTypedTuple::can_rhs_be_assigned(TypePtr rhs) const { return rhs == TypeDataNever::create(); } +bool TypeDataIntN::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + if (rhs == TypeDataInt::create()) { + return true; + } + return rhs == TypeDataNever::create(); // `int8` is NOT assignable to `int32` without `as` +} + bool TypeDataUnknown::can_rhs_be_assigned(TypePtr rhs) const { return true; } @@ -463,14 +496,23 @@ bool TypeDataInt::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { // `int` as `int?` return can_be_casted_with_as_operator(to_nullable->inner); } + if (cast_to->try_as()) { // `int` as `int8` / `int` as `uint2` + return true; + } return cast_to == this; } bool TypeDataBool::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to == TypeDataInt::create()) { + return true; + } if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } - return cast_to == this || cast_to == TypeDataInt::create(); + if (const auto* to_intN = cast_to->try_as()) { + return !to_intN->is_unsigned; // `bool` as `int8` ok, `bool` as `uintN` not (true is -1) + } + return cast_to == this; } bool TypeDataCell::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -560,6 +602,16 @@ bool TypeDataTypedTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { return false; } +bool TypeDataIntN::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to->try_as()) { // `int8` as `int32`, `int256` as `uint5`, anything + return true; + } + if (const auto* to_nullable = cast_to->try_as()) { // `int8` as `int32?` + return can_be_casted_with_as_operator(to_nullable->inner); + } + return cast_to == TypeDataInt::create(); +} + bool TypeDataUnknown::can_be_casted_with_as_operator(TypePtr cast_to) const { // 'unknown' can be cast to any TVM value return cast_to->get_width_on_stack() == 1; @@ -657,6 +709,16 @@ std::vector parse_nested_type_list_in_parenthesis(Lexer& lex) { return parse_nested_type_list(lex, tok_oppar, "`(`", tok_clpar, "`)` or `,`"); } +static TypePtr parse_intN(std::string_view strN, bool is_unsigned) { + int n; + auto result = std::from_chars(strN.data() + 3 + static_cast(is_unsigned), strN.data() + strN.size(), n); + bool parsed = result.ec == std::errc() && result.ptr == strN.data() + strN.size(); + if (!parsed || n <= 0 || n > 256 + static_cast(is_unsigned)) { + return nullptr; // `int1000`, maybe it's user-defined alias, let it be unresolved + } + return TypeDataIntN::create(is_unsigned, false, n); +} + static TypePtr parse_simple_type(Lexer& lex) { switch (lex.tok()) { case tok_self: @@ -681,12 +743,26 @@ static TypePtr parse_simple_type(Lexer& lex) { case 7: if (str == "builder") return TypeDataBuilder::create(); break; + case 8: + if (str == "varint16") return TypeDataIntN::create(false, true, 16); + if (str == "varint32") return TypeDataIntN::create(false, true, 32); + break; case 12: if (str == "continuation") return TypeDataContinuation::create(); break; default: break; } + if (str.starts_with("int")) { + if (TypePtr intN = parse_intN(str, false)) { + return intN; + } + } + if (str.size() > 4 && str.starts_with("uint")) { + if (TypePtr uintN = parse_intN(str, true)) { + return uintN; + } + } return TypeDataUnresolved::create(std::string(str), loc); } case tok_null: diff --git a/tolk/type-system.h b/tolk/type-system.h index 4b671e30d8..b64fcfca12 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -366,6 +366,31 @@ class TypeDataTypedTuple final : public TypeData { TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; }; +/* + * `int8`, `int32`, `uint1`, `uint257`, `varint16` are TypeDataIntN. At TVM level, it's just int. + * The purpose of intN is to be used in struct fields, describing the way of serialization (n bits). + * A field `value: int32` has the TYPE `int32`, so being assigned to a variable, that variable is also `int32`. + * intN is smoothly cast from/to plain int, mathematical operators on intN also "fall back" to general int. + */ +class TypeDataIntN final : public TypeData { + TypeDataIntN(uint64_t type_id, bool is_unsigned, bool is_variadic, int n_bits) + : TypeData(type_id, 0, 1) + , is_unsigned(is_unsigned) + , is_variadic(is_variadic) + , n_bits(n_bits) {} + +public: + const bool is_unsigned; + const bool is_variadic; + const int n_bits; + + static TypePtr create(bool is_unsigned, bool is_variadic, int n_bits); + + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + /* * `unknown` is a special type, which can appear in corner cases. * The type of exception argument (which can hold any TVM value at runtime) is unknown. From 620a12d373a6ff0ef95c4d035f0b7acb4152e7ea Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 13 Mar 2025 18:01:10 +0400 Subject: [PATCH 156/524] [Tolk] Support intN and bool constants Constant evaluator has been rewritten to utilize type inferring mechanism. Now, > const a = 1 as int8; is valid, and constants can not only be int/slice, but also intN/uintN and bool. --- tolk-tester/tests/co1.tolk | 32 ++++++- tolk-tester/tests/intN-tests.tolk | 9 +- tolk-tester/tests/invalid-cyclic-1.tolk | 3 +- tolk-tester/tests/invalid-declaration-13.tolk | 2 +- tolk-tester/tests/invalid-declaration-14.tolk | 7 ++ tolk-tester/tests/invalid-declaration-15.tolk | 6 ++ tolk-tester/tests/invalid-declaration-16.tolk | 9 ++ tolk-tester/tests/invalid-typing-35.tolk | 6 ++ tolk/ast-replacer.h | 1 + tolk/ast-visitor.h | 1 + tolk/compiler-state.cpp | 4 + tolk/constant-evaluator.cpp | 94 ++++++++++--------- tolk/constant-evaluator.h | 30 +++--- tolk/pipe-ast-to-legacy.cpp | 8 +- tolk/pipe-check-inferred-types.cpp | 20 ++++ tolk/pipe-constant-folding.cpp | 9 ++ tolk/pipe-infer-types-and-calls.cpp | 41 +++++++- tolk/pipe-register-symbols.cpp | 14 +-- tolk/pipe-resolve-identifiers.cpp | 6 ++ tolk/symtable.cpp | 8 ++ tolk/symtable.h | 16 ++-- 21 files changed, 238 insertions(+), 88 deletions(-) create mode 100644 tolk-tester/tests/invalid-declaration-14.tolk create mode 100644 tolk-tester/tests/invalid-declaration-15.tolk create mode 100644 tolk-tester/tests/invalid-declaration-16.tolk create mode 100644 tolk-tester/tests/invalid-typing-35.tolk diff --git a/tolk-tester/tests/co1.tolk b/tolk-tester/tests/co1.tolk index f124e1de85..66b89190aa 100644 --- a/tolk-tester/tests/co1.tolk +++ b/tolk-tester/tests/co1.tolk @@ -16,6 +16,16 @@ const str2int = 0xAABBCC; const nibbles: int = 4; +const strange_zero = (!10 as int); +const strange_minus_1: int = (!0 as int); + +const true1 = true; +const true2 = !!true; +const true3 = true1 && true2; + +const false1 = !true; +const false2 = false1 || false; + fun iget1(): int { return int1; } fun iget2(): int { return int2; } fun iget3(): int { return int1+int2; } @@ -43,6 +53,21 @@ asm "SDEQ"; fun stslicer(b: builder, s: slice): builder asm "STSLICER"; +@method_id(101) +fun test1() { + return (strange_zero, strange_minus_1); +} + +@method_id(102) +fun test2() { + return (true1, true2, true3); +} + +@method_id(103) +fun test3() { + return (false1, false2); +} + fun main() { var i1: int = iget1(); var i2: int = iget2(); @@ -66,7 +91,10 @@ fun main() { } /** -@testcase | 0 | | 0 +@testcase | 0 | | 0 +@testcase | 101 | | 0 -1 +@testcase | 102 | | -1 -1 -1 +@testcase | 103 | | 0 0 -@code_hash 61273295789179921867241079778489100375537711211918844448475493726205774530743 +@code_hash 28102194299745406750019953961984060488024870092664444642078578246708959881688 */ diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index 3f4cdc015c..16aa671680 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -3,6 +3,9 @@ fun getAnyInt(): int { return 0; } fun getNullableInt32(): int32? { return 32; } fun getNullableVarInt32(): varint32? { return 32; } +const someN_u128: uint128 = 128; +const someN_i32: int32 = 20 + (12 as int32) * (1 as uint8); + fun autoInferInt8(x: int8) { if (random()) { return x; } @@ -50,7 +53,8 @@ fun test1(x: int8) { var z = x + y; __expect_type(z, "int"); - return x / y; + __expect_type(someN_u128, "uint128"); + return x / y + someN_u128; } fun test2(): (uint8, uint8, uint8) { @@ -60,6 +64,7 @@ fun test2(): (uint8, uint8, uint8) { fun test3(op: int32, qid: uint64) { op = qid as int32; + op = someN_i32; op + qid; op = op + qid; @@ -166,7 +171,7 @@ fun main() { /** @testcase | 0 | | [ 0 0 0 ] -@testcase | 101 | 0 | 1 +@testcase | 101 | 0 | 129 @testcase | 104 | | 64 64 -1 0 @testcase | 105 | | 3 -1 0 @testcase | 114 | 5 | (null) (null) 0 diff --git a/tolk-tester/tests/invalid-cyclic-1.tolk b/tolk-tester/tests/invalid-cyclic-1.tolk index c46b1640e0..0f259fa68a 100644 --- a/tolk-tester/tests/invalid-cyclic-1.tolk +++ b/tolk-tester/tests/invalid-cyclic-1.tolk @@ -3,6 +3,5 @@ const TWO = ONE + 1; /** @compilation_should_fail -@stderr const ONE -@stderr undefined symbol `TWO` +@stderr const `TWO` appears, directly or indirectly, in its own initializer */ diff --git a/tolk-tester/tests/invalid-declaration-13.tolk b/tolk-tester/tests/invalid-declaration-13.tolk index 758a4f21da..ada0ef5f41 100644 --- a/tolk-tester/tests/invalid-declaration-13.tolk +++ b/tolk-tester/tests/invalid-declaration-13.tolk @@ -2,6 +2,6 @@ const c: slice = 123 + 456; /** @compilation_should_fail -@stderr expression type does not match declared type +@stderr can not assign `int` to `slice` @stderr const c */ diff --git a/tolk-tester/tests/invalid-declaration-14.tolk b/tolk-tester/tests/invalid-declaration-14.tolk new file mode 100644 index 0000000000..339addfb57 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-14.tolk @@ -0,0 +1,7 @@ +const ttt = 1; +const asdf = ttt; + +/** +@compilation_should_fail +@stderr generic T not expected here + */ diff --git a/tolk-tester/tests/invalid-declaration-15.tolk b/tolk-tester/tests/invalid-declaration-15.tolk new file mode 100644 index 0000000000..72f393eff1 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-15.tolk @@ -0,0 +1,6 @@ +const asdf = ttt; + +/** +@compilation_should_fail +@stderr undefined symbol `ttt` + */ diff --git a/tolk-tester/tests/invalid-declaration-16.tolk b/tolk-tester/tests/invalid-declaration-16.tolk new file mode 100644 index 0000000000..1dae9c9b2e --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-16.tolk @@ -0,0 +1,9 @@ +fun foo() { return 1; } + +const asdf = 1 + foo(); + +/** +@compilation_should_fail +@stderr not a constant expression +@stderr const asdf + */ diff --git a/tolk-tester/tests/invalid-typing-35.tolk b/tolk-tester/tests/invalid-typing-35.tolk new file mode 100644 index 0000000000..b43aaef55f --- /dev/null +++ b/tolk-tester/tests/invalid-typing-35.tolk @@ -0,0 +1,6 @@ +const asdf = 1 + ""; + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int` and `slice` + */ diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 5103cc9238..d1ab4c4507 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -187,6 +187,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { const std::vector& get_all_not_builtin_functions(); +const std::vector& get_all_declared_constants(); template void replace_ast_of_all_functions() { diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index d697aa82bd..737646bb5b 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -180,6 +180,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { const std::vector& get_all_not_builtin_functions(); +const std::vector& get_all_declared_constants(); template void visit_ast_of_all_functions() { diff --git a/tolk/compiler-state.cpp b/tolk/compiler-state.cpp index 95a7e6a592..e02d0220a5 100644 --- a/tolk/compiler-state.cpp +++ b/tolk/compiler-state.cpp @@ -70,4 +70,8 @@ const std::vector& get_all_not_builtin_functions() { return G.all_functions; } +const std::vector& get_all_declared_constants() { + return G.all_constants; +} + } // namespace tolk diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index 4d11b9223e..214fb729d4 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -147,6 +147,12 @@ static td::RefInt256 parse_vertex_string_const_as_int(V v) { } } +static ConstantValue parse_vertex_string_const(V v) { + return v->is_bitslice() + ? ConstantValue(parse_vertex_string_const_as_slice(v)) + : ConstantValue(parse_vertex_string_const_as_int(v)); +} + struct ConstantEvaluator { static bool is_overflow(const td::RefInt256& intval) { @@ -154,10 +160,8 @@ struct ConstantEvaluator { } static ConstantValue handle_unary_operator(V v, const ConstantValue& rhs) { - if (!rhs.is_int()) { - v->error("invalid operator, expecting integer"); - } - td::RefInt256 intval = std::get(rhs.value); + tolk_assert(rhs.is_int()); // type inferring has passed before, so it's int/bool + td::RefInt256 intval = rhs.as_int(); switch (v->tok) { case tok_minus: @@ -167,7 +171,7 @@ struct ConstantEvaluator { break; case tok_bitwise_not: intval = ~intval; - break; + break; case tok_logical_not: intval = td::make_refint(intval == 0 ? -1 : 0); break; @@ -178,15 +182,13 @@ struct ConstantEvaluator { if (is_overflow(intval)) { v->error("integer overflow"); } - return ConstantValue::from_int(std::move(intval)); + return ConstantValue(std::move(intval)); } static ConstantValue handle_binary_operator(V v, const ConstantValue& lhs, const ConstantValue& rhs) { - if (!lhs.is_int() || !rhs.is_int()) { - v->error("invalid operator, expecting integer"); - } - td::RefInt256 lhs_intval = std::get(lhs.value); - td::RefInt256 rhs_intval = std::get(rhs.value); + tolk_assert(lhs.is_int() && rhs.is_int()); // type inferring has passed before, so they are int/bool + td::RefInt256 lhs_intval = lhs.as_int(); + td::RefInt256 rhs_intval = rhs.as_int(); td::RefInt256 intval; switch (v->tok) { @@ -238,6 +240,12 @@ struct ConstantEvaluator { case tok_neq: intval = td::make_refint(lhs_intval != rhs_intval ? -1 : 0); break; + case tok_logical_and: + intval = td::make_refint(lhs_intval != 0 && rhs_intval != 0 ? -1 : 0); + break; + case tok_logical_or: + intval = td::make_refint(lhs_intval != 0 || rhs_intval != 0 ? -1 : 0); + break; default: v->error("unsupported binary operator in constant expression"); } @@ -245,32 +253,28 @@ struct ConstantEvaluator { if (is_overflow(intval)) { v->error("integer overflow"); } - return ConstantValue::from_int(std::move(intval)); + return ConstantValue(std::move(intval)); } + // `const a = 1 + b`, we met `b` static ConstantValue handle_reference(V v) { - // todo better handle "appears, directly or indirectly, in its own initializer" - std::string_view name = v->get_name(); - const Symbol* sym = lookup_global_symbol(name); - if (!sym) { - v->error("undefined symbol `" + static_cast(name) + "`"); - } - GlobalConstPtr const_ref = sym->try_as(); + GlobalConstPtr const_ref = v->sym->try_as(); if (!const_ref) { - v->error("symbol `" + static_cast(name) + "` is not a constant"); + v->error("symbol `" + static_cast(v->get_name()) + "` is not a constant"); } - if (v->has_instantiationTs()) { // SOME_CONST - v->error("constant is not a generic"); + + if (!const_ref->value.initialized()) { // maybe, `b` was already calculated + eval_and_assign_const_init_value(const_ref); // if not, dig recursively into `b` } - return {const_ref->value}; + return const_ref->value; } static ConstantValue visit(AnyExprV v) { if (auto v_int = v->try_as()) { - return ConstantValue::from_int(v_int->intval); + return ConstantValue(v_int->intval); } if (auto v_bool = v->try_as()) { - return ConstantValue::from_int(v_bool->bool_val ? -1 : 0); + return ConstantValue(v_bool->bool_val ? -1 : 0); } if (auto v_unop = v->try_as()) { return handle_unary_operator(v_unop, visit(v_unop->get_rhs())); @@ -284,34 +288,32 @@ struct ConstantEvaluator { if (auto v_par = v->try_as()) { return visit(v_par->get_expr()); } - if (v->try_as()) { - return eval_const_init_value(v); + if (auto v_cast_to = v->try_as()) { + return visit(v_cast_to->get_expr()); + } + if (auto v_string = v->try_as()) { + return parse_vertex_string_const(v_string); } v->error("not a constant expression"); } - static ConstantValue eval_const_init_value(AnyExprV init_value) { - // it init_value is incorrect, an exception is thrown - return visit(init_value); + // evaluate `const a = 2 + 3` into 5 + // type inferring has already passed, to types are correct, `const a = 1 + ""` can not occur + // recursive initializers `const a = b; const b = a` also 100% don't exist, checked on type inferring + // if init_value is not a constant expression like `const a = foo()`, an exception is thrown + static ConstantValue eval_const_init_value(GlobalConstPtr const_ref) { + return visit(const_ref->init_value); } }; -ConstantValue eval_const_init_value(AnyExprV init_value) { - // at first, handle most simple cases, not to launch heavy computation algorithm: just a number, just a string - // just `c = 1` or `c = 0xFF` - if (auto v_int = init_value->try_as()) { - return {v_int->intval}; - } - // just `c = "strval"`, probably with modifier (address, etc.) - if (auto v_string = init_value->try_as()) { - if (v_string->is_bitslice()) { - return {parse_vertex_string_const_as_slice(v_string)}; - } else { - return {parse_vertex_string_const_as_int(v_string)}; - } - } - // something more complex, like `c = anotherC` or `c = 1 << 8` - return ConstantEvaluator::eval_const_init_value(init_value); +ConstantValue eval_string_const_considering_modifier(AnyExprV v_string) { + tolk_assert(v_string->type == ast_string_const); + return parse_vertex_string_const(v_string->as()); +} + +void eval_and_assign_const_init_value(GlobalConstPtr const_ref) { + ConstantValue init_value = ConstantEvaluator::eval_const_init_value(const_ref); + const_ref->mutate()->assign_const_value(std::move(init_value)); } } // namespace tolk diff --git a/tolk/constant-evaluator.h b/tolk/constant-evaluator.h index 0f99867d8e..218bd8c4bf 100644 --- a/tolk/constant-evaluator.h +++ b/tolk/constant-evaluator.h @@ -22,24 +22,32 @@ namespace tolk { -struct ConstantValue { - std::variant value; +class ConstantValue { + std::variant< + td::RefInt256, // is set for int, intN, coins, bool + std::string // is set for slice, bytesN + > value; + +public: + ConstantValue() = default; // by default, not initialized + + explicit ConstantValue(int value) + : value(td::make_refint(value)) {} + explicit ConstantValue(td::RefInt256 value) + : value(std::move(value)) {} + explicit ConstantValue(std::string value) + : value(std::move(value)) {} + + bool initialized() const { return is_slice() || std::get(value).not_null(); } bool is_int() const { return std::holds_alternative(value); } bool is_slice() const { return std::holds_alternative(value); } td::RefInt256 as_int() const { return std::get(value); } const std::string& as_slice() const { return std::get(value); } - - static ConstantValue from_int(int value) { - return {td::make_refint(value)}; - } - - static ConstantValue from_int(td::RefInt256 value) { - return {std::move(value)}; - } }; -ConstantValue eval_const_init_value(AnyExprV init_value); +ConstantValue eval_string_const_considering_modifier(AnyExprV v_string); +void eval_and_assign_const_init_value(GlobalConstPtr const_ref); } // namespace tolk diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 4347de3e82..476559d35b 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -644,13 +644,13 @@ std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, Co return local_ir_idx; } if (GlobalConstPtr const_ref = sym->try_as()) { - if (const_ref->is_int_const()) { + if (const_ref->value.is_int()) { std::vector rvect = code.create_tmp_var(TypeDataInt::create(), loc, "(glob-const)"); - code.emplace_back(loc, Op::_IntConst, rvect, const_ref->as_int_const()); + code.emplace_back(loc, Op::_IntConst, rvect, const_ref->value.as_int()); return rvect; } else { std::vector rvect = code.create_tmp_var(TypeDataSlice::create(), loc, "(glob-const)"); - code.emplace_back(loc, Op::_SliceConst, rvect, const_ref->as_slice_const()); + code.emplace_back(loc, Op::_SliceConst, rvect, const_ref->value.as_slice()); return rvect; } } @@ -1013,7 +1013,7 @@ static std::vector process_int_const(V v, CodeBlob& co } static std::vector process_string_const(V v, CodeBlob& code, TypePtr target_type) { - ConstantValue value = eval_const_init_value(v); + ConstantValue value = eval_string_const_considering_modifier(v); std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(str-const)"); if (value.is_int()) { code.emplace_back(v->loc, Op::_IntConst, rvect, value.as_int()); diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 5f78b9d25d..97cabd1fe9 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -581,10 +581,30 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } } + + // given `const a = 2 + 3` check types within its init_value + // so, `const a = 1 + some_slice` will fire a reasonable error + void start_visiting_constant(GlobalConstPtr const_ref) { + parent::visit(const_ref->init_value); + + // if no errors occurred, init value has correct type + // (though it may not be a valid constant expression, this would be checked after type inferring) + if (const_ref->declared_type) { // `const a: int = ...` + TypePtr inferred_type = const_ref->init_value->inferred_type; + if (!const_ref->declared_type->can_rhs_be_assigned(inferred_type)) { + throw ParseError(const_ref->loc, "can not assign " + to_string(inferred_type) + " to " + to_string(const_ref->declared_type)); + } + } + } }; void pipeline_check_inferred_types() { visit_ast_of_all_functions(); + + CheckInferredTypesVisitor visitor; + for (GlobalConstPtr const_ref : get_all_declared_constants()) { + visitor.start_visiting_constant(const_ref); + } } } // namespace tolk diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index 9c27029b5b..4b0fe9723f 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -106,6 +106,15 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { }; void pipeline_constant_folding() { + // here (after type inferring) evaluate `const a = 2 + 3` into `5` + // non-constant expressions like `const a = foo()` fire an error here + for (GlobalConstPtr const_ref : get_all_declared_constants()) { + // for `const a = b`, `b` could be already calculated while calculating `a` + if (!const_ref->value.initialized()) { + eval_and_assign_const_init_value(const_ref); + } + } + replace_ast_of_all_functions(); } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index ad1c59aebb..58d28d421b 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -96,6 +96,7 @@ namespace tolk { static void infer_and_save_return_type_of_function(FunctionPtr fun_ref); +static void infer_and_save_type_of_constant(GlobalConstPtr const_ref); static TypePtr get_or_infer_return_type(FunctionPtr fun_ref) { if (!fun_ref->inferred_return_type) { @@ -677,7 +678,10 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, declared_or_smart_casted); } else if (GlobalConstPtr const_ref = v->sym->try_as()) { - assign_inferred_type(v, const_ref->is_int_const() ? TypeDataInt::create() : TypeDataSlice::create()); + if (!const_ref->inferred_type) { + infer_and_save_type_of_constant(const_ref); + } + assign_inferred_type(v, const_ref->inferred_type); } else if (GlobalVarPtr glob_ref = v->sym->try_as()) { // there are no smart casts for globals, it's a way of preventing reading one global multiple times, it costs gas @@ -1245,6 +1249,14 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_fun_full_type(fun_ref, inferred_return_type); fun_ref->mutate()->assign_is_type_inferring_done(); } + + // given `const a = 2 + 3` infer that it's int + // for `const a: int = ...` still infer all sub expressions (to be checked in a later pipe) + void start_visiting_constant(GlobalConstPtr const_ref) { + FlowContext const_flow; + infer_any_expr(const_ref->init_value, std::move(const_flow), false, const_ref->declared_type); + const_ref->mutate()->assign_inferred_type(const_ref->declared_type == nullptr ? const_ref->init_value->inferred_type : const_ref->declared_type); + } }; class LaunchInferTypesAndMethodsOnce final { @@ -1288,8 +1300,35 @@ static void infer_and_save_return_type_of_function(FunctionPtr fun_ref) { called_stack.pop_back(); } +// infer constant type "on demand" +// example: `const a = 1 + b;` +// when analyzing `a`, we need to infer what type const_ref=b has +static void infer_and_save_type_of_constant(GlobalConstPtr const_ref) { + static std::vector called_stack; + + // prevent recursion like `const a = b; const b = a` + bool contains = std::find(called_stack.begin(), called_stack.end(), const_ref) != called_stack.end(); + if (contains) { + throw ParseError(const_ref->loc, "const `" + const_ref->name + "` appears, directly or indirectly, in its own initializer"); + } + + called_stack.push_back(const_ref); + InferTypesAndCallsAndFieldsVisitor visitor; + visitor.start_visiting_constant(const_ref); + called_stack.pop_back(); +} + void pipeline_infer_types_and_calls_and_fields() { visit_ast_of_all_functions(); + + // analyze constants that weren't referenced by any function + InferTypesAndCallsAndFieldsVisitor visitor; + for (GlobalConstPtr const_ref : get_all_declared_constants()) { + if (!const_ref->inferred_type) { + visitor.start_visiting_constant(const_ref); + } + } + // (later, at constant folding, `const a = 2 + 3` will be evaluated to 5) } void pipeline_infer_types_and_calls_and_fields(FunctionPtr fun_ref) { diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 45246d6b93..ea4cc8110b 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -115,16 +115,10 @@ static const GenericsDeclaration* construct_genericTs(V v_li } static void register_constant(V v) { - ConstantValue init_value = eval_const_init_value(v->get_init_value()); - GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->declared_type, std::move(init_value)); - - if (v->declared_type) { - bool ok = (c_sym->is_int_const() && (v->declared_type == TypeDataInt::create())) - || (c_sym->is_slice_const() && (v->declared_type == TypeDataSlice::create())); - if (!ok) { - v->error("expression type does not match declared type"); - } - } + GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->declared_type, v->get_init_value()); + // init value of constant is not evaluated here + // at first, it will be type checked (in type inference pipe) + // then, at constant folding pipe, `const a = 2 + 3` will be evaluated to 5 G.symtable.add_global_const(c_sym); G.all_constants.push_back(c_sym); diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 5a735885ed..2ace2e7193 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -313,6 +313,11 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { current_genericTs = nullptr; cur_f = nullptr; } + + void start_visiting_constant(V v) { + // `const a = b`, resolve `b` + parent::visit(v->get_init_value()); + } }; NameAndScopeResolver AssignSymInsideFunctionVisitor::current_scope; @@ -333,6 +338,7 @@ void pipeline_resolve_identifiers_and_assign_symbols() { v_global->var_ref->mutate()->assign_resolved_type(declared_type); } else if (auto v_const = v->try_as()) { + visitor.start_visiting_constant(v_const); if (v_const->declared_type) { TypePtr declared_type = finalize_type_data(nullptr, v_const->const_ref->declared_type, nullptr); v_const->mutate()->assign_resolved_type(declared_type); diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 51dc344010..ea009957a5 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -93,6 +93,14 @@ void GlobalConstData::assign_resolved_type(TypePtr declared_type) { this->declared_type = declared_type; } +void GlobalConstData::assign_inferred_type(TypePtr inferred_type) { + this->inferred_type = inferred_type; +} + +void GlobalConstData::assign_const_value(ConstantValue&& value) { + this->value = std::move(value); +} + void LocalVarData::assign_ir_idx(std::vector&& ir_idx) { this->ir_idx = std::move(ir_idx); } diff --git a/tolk/symtable.h b/tolk/symtable.h index 9419afce6a..e65327fdba 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -197,23 +197,21 @@ struct GlobalVarData final : Symbol { }; struct GlobalConstData final : Symbol { + AnyExprV init_value; ConstantValue value; - TypePtr declared_type; // may be nullptr + TypePtr declared_type; // `const a: int = ...`; nullptr for `const a = ...` + TypePtr inferred_type = nullptr; // filled at type inferring pass - GlobalConstData(std::string name, SrcLocation loc, TypePtr declared_type, ConstantValue&& value) + GlobalConstData(std::string name, SrcLocation loc, TypePtr declared_type, AnyExprV init_value) : Symbol(std::move(name), loc) - , value(std::move(value)) + , init_value(init_value) , declared_type(declared_type) { } - bool is_int_const() const { return value.is_int(); } - bool is_slice_const() const { return value.is_slice(); } - - td::RefInt256 as_int_const() const { return value.as_int(); } - const std::string& as_slice_const() const { return value.as_slice(); } - GlobalConstData* mutate() const { return const_cast(this); } void assign_resolved_type(TypePtr declared_type); + void assign_inferred_type(TypePtr inferred_type); + void assign_const_value(ConstantValue&& value); }; class GlobalSymbolTable { From b086bc7ddeb92ef2e9c9ac5571b44bf5aa6d8139 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 13 Mar 2025 18:08:49 +0400 Subject: [PATCH 157/524] [Tolk] Type `coins`, function `ton("0.05")` A new compile-time function is introduced. It converts a constant floating-point string to nanotoncoins. Example: `ton("0.05")` is equal to 50000000. It requires a constant string; `ton(some_var)` is an error. Type of its return value is `coins`, not `int`. Like `intN`, it's backed by a general int at TVM level, all arithmetics (except addition) degrade to int. Type `coins` will be serialized as varint16 in the future. --- crypto/smartcont/tolk-stdlib/common.tolk | 11 +++- tolk-tester/tests/intN-tests.tolk | 78 ++++++++++++++++++++++++ tolk-tester/tests/invalid-call-12.tolk | 8 +++ tolk-tester/tests/invalid-call-13.tolk | 8 +++ tolk-tester/tests/invalid-call-14.tolk | 6 ++ tolk-tester/tests/invalid-typing-36.tolk | 12 ++++ tolk-tester/tests/invalid-typing-37.tolk | 11 ++++ tolk-tester/tests/invalid-typing-38.tolk | 10 +++ tolk/builtins.cpp | 10 +++ tolk/constant-evaluator.cpp | 77 +++++++++++++++++++++++ tolk/constant-evaluator.h | 1 + tolk/pipe-ast-to-legacy.cpp | 20 ++++++ tolk/pipe-check-inferred-types.cpp | 2 +- tolk/pipe-constant-folding.cpp | 13 ++++ tolk/pipe-infer-types-and-calls.cpp | 6 +- tolk/type-system.cpp | 34 ++++++++++- tolk/type-system.h | 18 ++++++ 17 files changed, 320 insertions(+), 5 deletions(-) create mode 100644 tolk-tester/tests/invalid-call-12.tolk create mode 100644 tolk-tester/tests/invalid-call-13.tolk create mode 100644 tolk-tester/tests/invalid-call-14.tolk create mode 100644 tolk-tester/tests/invalid-typing-36.tolk create mode 100644 tolk-tester/tests/invalid-typing-37.tolk create mode 100644 tolk-tester/tests/invalid-typing-38.tolk diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 82757c22bd..9647243f95 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -54,6 +54,13 @@ fun tupleLast(self: tuple): T Mathematical primitives. */ +/// Converts a constant floating-point string to nanotoncoins. +/// Example: `ton("0.05")` is equal to 50000000. +/// Note, that `ton()` requires a constant string; `ton(some_var)` is an error +@pure +fun ton(floatString: slice): coins + builtin; + /// Computes the minimum of two integers. @pure fun min(x: int, y: int): int @@ -386,7 +393,7 @@ fun preloadBits(self: slice, len: int): slice /// Loads serialized amount of Toncoins (any unsigned integer up to `2^120 - 1`). @pure -fun loadCoins(mutate self: slice): int +fun loadCoins(mutate self: slice): coins asm( -> 1 0) "LDGRAMS"; /// Loads bool (-1 or 0) from a slice @@ -485,7 +492,7 @@ fun storeSlice(mutate self: builder, s: slice): self /// Stores amount of Toncoins into a builder. @pure -fun storeCoins(mutate self: builder, x: int): self +fun storeCoins(mutate self: builder, x: coins): self asm "STGRAMS"; /// Stores bool (-1 or 0) into a builder. diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index 16aa671680..abafdf1d72 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -5,6 +5,10 @@ fun getNullableVarInt32(): varint32? { return 32; } const someN_u128: uint128 = 128; const someN_i32: int32 = 20 + (12 as int32) * (1 as uint8); +const someN_coins = 10 as coins; + +const cost1 = ton("1.234"); +const cost2 = ton("0.05") + ton("0.001"); fun autoInferInt8(x: int8) { @@ -54,6 +58,7 @@ fun test1(x: int8) { var z = x + y; __expect_type(z, "int"); __expect_type(someN_u128, "uint128"); + __expect_type(someN_coins, "coins"); return x / y + someN_u128; } @@ -141,6 +146,65 @@ fun test7() { __expect_type((n as varint16) as int, "int"); } +fun test8(amount1: coins, amount2: coins) { + __expect_type(amount1 + 0, "coins"); + __expect_type(amount1 + amount2, "coins"); + __expect_type(amount1 += amount2, "coins"); + __expect_type(amount1 - 10, "coins"); + __expect_type(amount1 &= 1, "coins"); + __expect_type(amount1 &= (1 as coins), "coins"); + + __expect_type(amount1 & 1, "int"); + __expect_type(amount1 * 10, "int"); + __expect_type(amount1 << amount2, "int"); +} + +@method_id(109) +fun test9(c: coins, i: int8) { + __expect_type(c as int8, "int8"); + __expect_type(i as coins, "coins"); + __expect_type(c + i, "coins"); + + var amount: coins = 1000; + var percent: uint8 = 50; + var new = amount * percent / 100; + amount = new; + __expect_type(amount, "coins"); + if (!amount) { while (amount) { amount -= 1; } } + + takeAnyInt(amount); + takeAnyInt(percent); + takeAnyInt(true as int3); + takeAnyInt(amount as int3); + + return (amount, (true as int8) as coins); +} + +@method_id(110) +fun test10() { + return (ton("0.05"), ton("0.05") + 100, cost1, cost2); +} + +@method_id(111) +fun test11() { + __expect_type(ton("0"), "coins"); + __expect_type(ton("0") + ton("0"), "coins"); + __expect_type(ton("0") * 20, "int"); + return [ + ton("1"), + ton("1.0"), + ton("1.00000"), + ton("-321.123456789"), + ton("+321.123456789876"), + ton("0001.1000") + ]; +} + +fun test12() { + var a = ton("1") + ton("2"); // check absence in fif codegen + return ton("0.1"); +} + fun test13(x1: int?, x2: int8?, x3: int, x4: int8): (int8?, int8?, int8?, int8?, int32?, int32?) { return (x1, x2, x3, x4, x4 as int32, x4 as int32?); } @@ -165,6 +229,9 @@ fun main() { __expect_type(0 as int32, "int32"); __expect_type((0 as int32) as uint64?, "uint64?"); __expect_type(null as (int8, [uint16?])?, "(int8, [uint16?])?"); + __expect_type(0 as coins, "coins"); + __expect_type(someN_coins as coins?, "coins?"); + __expect_type(someN_coins as int8?, "int8?"); __expect_type(10>3 ? 0 as uint8 : 0 as uint16, "int"); return t; } @@ -174,10 +241,21 @@ fun main() { @testcase | 101 | 0 | 129 @testcase | 104 | | 64 64 -1 0 @testcase | 105 | | 3 -1 0 +@testcase | 109 | 4 4 | 500 -1 +@testcase | 110 | | 50000000 50000100 1234000000 51000000 +@testcase | 111 | | [ 1000000000 1000000000 1000000000 -321123456789 321123456789 1100000000 ] @testcase | 114 | 5 | (null) (null) 0 @testcase | 114 | 15 | 15 2 -1 @fif_codegen DECLPROC assign0 @fif_codegen DECLPROC assign0 @fif_codegen DECLPROC assign0 + +@fif_codegen +""" + test12 PROC:<{ + // + 100000000 PUSHINT // '4=100000000 + }> +""" */ diff --git a/tolk-tester/tests/invalid-call-12.tolk b/tolk-tester/tests/invalid-call-12.tolk new file mode 100644 index 0000000000..a0f9bea135 --- /dev/null +++ b/tolk-tester/tests/invalid-call-12.tolk @@ -0,0 +1,8 @@ +fun main(am: slice) { + return ton(am); +} + +/** +@compilation_should_fail +@stderr function `ton` requires a constant string, like `ton("0.05")` + */ diff --git a/tolk-tester/tests/invalid-call-13.tolk b/tolk-tester/tests/invalid-call-13.tolk new file mode 100644 index 0000000000..11cce60350 --- /dev/null +++ b/tolk-tester/tests/invalid-call-13.tolk @@ -0,0 +1,8 @@ +fun main() { + return ton(123); +} + +/** +@compilation_should_fail +@stderr function `ton` requires a constant string, like `ton("0.05")` + */ diff --git a/tolk-tester/tests/invalid-call-14.tolk b/tolk-tester/tests/invalid-call-14.tolk new file mode 100644 index 0000000000..2d15d9fb13 --- /dev/null +++ b/tolk-tester/tests/invalid-call-14.tolk @@ -0,0 +1,6 @@ +const ss = ton("0.0.0"); + +/** +@compilation_should_fail +@stderr argument is not a valid number like "0.05" + */ diff --git a/tolk-tester/tests/invalid-typing-36.tolk b/tolk-tester/tests/invalid-typing-36.tolk new file mode 100644 index 0000000000..d21099eb81 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-36.tolk @@ -0,0 +1,12 @@ +fun cantCastBoolToCoins() { + var k1 = true as int8; // ok + var k2 = true as int1; // ok + var k3 = (true as int) as coins; // ok + var k4 = true as coins; +} + +/** +@compilation_should_fail +@stderr type `bool` can not be cast to `coins` +@stderr true as coins + */ diff --git a/tolk-tester/tests/invalid-typing-37.tolk b/tolk-tester/tests/invalid-typing-37.tolk new file mode 100644 index 0000000000..f55c6dd484 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-37.tolk @@ -0,0 +1,11 @@ +fun cantAssignIntNToCoins(n: int8, c: coins) { + n = c as int8; // ok + n = c as int; // ok + n = c; +} + +/** +@compilation_should_fail +@stderr can not assign `coins` to variable of type `int8` +@stderr n = c; + */ diff --git a/tolk-tester/tests/invalid-typing-38.tolk b/tolk-tester/tests/invalid-typing-38.tolk new file mode 100644 index 0000000000..58208caab3 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-38.tolk @@ -0,0 +1,10 @@ +fun cantUnifyCoinsAndUInt8(n: int8, c: coins) { + __expect_type(random() ? n : c as int8, "int8"); + __expect_type(random() ? n : c as int16, "int"); + random() ? n : c; +} + +/** +@compilation_should_fail +@stderr types of ternary branches are incompatible: `int8` and `coins` + */ diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index cb89c984c3..9ef4527aee 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1071,6 +1071,13 @@ AsmOp compile_tuple_set_at(std::vector& res, std::vector& ar return exec_op("SETINDEXVAR", 2, 1); } +// fun ton(amount: slice): coins; ton("0.05") replaced by 50000000 at compile-time +AsmOp compile_int_to_ton_coins(std::vector&, std::vector&, SrcLocation) { + // all ton() invocations are constants, replaced by integers; no dynamic values allowed, no work at runtime + tolk_assert(false); + return AsmOp::Nop(); +} + // fun __isNull(X arg): bool AsmOp compile_is_null(std::vector& res, std::vector& args, SrcLocation) { tolk_assert(args.size() == 1 && res.size() == 1); @@ -1214,6 +1221,9 @@ void define_builtins() { // functions from stdlib marked as `builtin`, implemented at compiler level for optimizations // (for example, `loadInt(1)` is `1 LDI`, but `loadInt(n)` for non-constant requires it be on a stack and `LDIX`) + define_builtin_func("ton", {TypeDataUnknown::create()}, TypeDataCoins::create(), nullptr, // `unknown` to pass inferring for `ton(1)` (to fire a better error later) + compile_int_to_ton_coins, + FunctionData::flagMarkedAsPure); define_builtin_func("mulDivFloor", ParamsInt3, Int, nullptr, std::bind(compile_muldiv, _1, _2, _3, -1), FunctionData::flagMarkedAsPure); diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index 214fb729d4..5c3b3b19a0 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -153,6 +153,67 @@ static ConstantValue parse_vertex_string_const(V v) { : ConstantValue(parse_vertex_string_const_as_int(v)); } +// given `ton("0.05")` evaluate it to 50000000 +static ConstantValue parse_vertex_call_to_ton_function(V v) { + tolk_assert(v->get_num_args() == 1); // checked by type inferring + AnyExprV v_arg = v->get_arg(0)->get_expr(); + + std::string_view str; + if (auto as_string = v_arg->try_as(); as_string && !as_string->modifier) { + str = as_string->str_val; + } else { + // ton(SOME_CONST) is not supported + // ton(0.05) is not supported (it can't be represented in AST even) + } + if (str.empty()) { + v->error("function `ton` requires a constant string, like `ton(\"0.05\")`"); + } + + bool is_negative = false; + size_t i = 0; + + // handle optional leading sign + if (str[0] == '-') { + is_negative = true; + i++; + } else if (str[0] == '+') { + i++; + } + + // parse "0.05" into integer part (before dot) and fractional (after) + int64_t integer_part = 0; + int64_t fractional_part = 0; + int fractional_digits = 0; + bool seen_dot = false; + + for (; i < str.size(); ++i) { + char c = str[i]; + if (c == '.') { + if (seen_dot) { + v_arg->error("argument is not a valid number like \"0.05\""); + } + seen_dot = true; + } else if (c >= '0' && c <= '9') { + if (!seen_dot) { + integer_part = integer_part * 10 + (c - '0'); + } else if (fractional_digits < 9) { + fractional_part = fractional_part * 10 + (c - '0'); + fractional_digits++; + } + } else { + v_arg->error("argument is not a valid number like \"0.05\""); + } + } + + while (fractional_digits < 9) { // after "0.05" fractional_digits is 2, scale up to 9 + fractional_part *= 10; + fractional_digits++; + } + + int64_t result = integer_part * 1000000000LL + fractional_part; + return ConstantValue(td::make_refint(is_negative ? -result : result)); +} + struct ConstantEvaluator { static bool is_overflow(const td::RefInt256& intval) { @@ -269,6 +330,14 @@ struct ConstantEvaluator { return const_ref->value; } + // `const a = ton("0.05")`, we met `ton("0.05")` + static ConstantValue handle_function_call(V v) { + if (v->fun_maybe && v->fun_maybe->is_builtin_function() && v->fun_maybe->name == "ton") { + return parse_vertex_call_to_ton_function(v); + } + v->error("not a constant expression"); + } + static ConstantValue visit(AnyExprV v) { if (auto v_int = v->try_as()) { return ConstantValue(v_int->intval); @@ -285,6 +354,9 @@ struct ConstantEvaluator { if (auto v_ref = v->try_as()) { return handle_reference(v_ref); } + if (auto v_call = v->try_as()) { + return handle_function_call(v_call); + } if (auto v_par = v->try_as()) { return visit(v_par->get_expr()); } @@ -311,6 +383,11 @@ ConstantValue eval_string_const_considering_modifier(AnyExprV v_string) { return parse_vertex_string_const(v_string->as()); } +ConstantValue eval_call_to_ton_function(AnyExprV v_call) { + tolk_assert(v_call->type == ast_function_call && v_call->as()->fun_maybe->is_builtin_function()); + return parse_vertex_call_to_ton_function(v_call->as()); +} + void eval_and_assign_const_init_value(GlobalConstPtr const_ref) { ConstantValue init_value = ConstantEvaluator::eval_const_init_value(const_ref); const_ref->mutate()->assign_const_value(std::move(init_value)); diff --git a/tolk/constant-evaluator.h b/tolk/constant-evaluator.h index 218bd8c4bf..4737352ea4 100644 --- a/tolk/constant-evaluator.h +++ b/tolk/constant-evaluator.h @@ -48,6 +48,7 @@ class ConstantValue { }; ConstantValue eval_string_const_considering_modifier(AnyExprV v_string); +ConstantValue eval_call_to_ton_function(AnyExprV v_call); void eval_and_assign_const_init_value(GlobalConstPtr const_ref); } // namespace tolk diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 476559d35b..3d0f5dfeea 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -549,18 +549,38 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as()) { return rvect; } + // pass `coins` to `int` + // same as above + if (target_type == TypeDataInt::create() && original_type == TypeDataCoins::create()) { + return rvect; + } // pass `int` to `int8` // in code, it's probably done with `as` operator // no changes in rvect if (original_type == TypeDataInt::create() && target_type->try_as()) { return rvect; } + // pass `int` to `coins` + // same as above + if (original_type == TypeDataInt::create() && target_type == TypeDataCoins::create()) { + return rvect; + } // pass `int8` to `int16` / `int8` to `uint8` // in code, it's probably done with `as` operator // no changes in rvect if (original_type->try_as() && target_type->try_as()) { return rvect; } + // pass `int8` to `coins` + // same as above + if (target_type == TypeDataCoins::create() && original_type->try_as()) { + return rvect; + } + // pass `coins` to `int8` + // same as above + if (original_type == TypeDataCoins::create() && target_type->try_as()) { + return rvect; + } // pass something to `unknown` // probably, it comes from `_ = rhs`, type of `_` is unknown, it's target_type of rhs // no changes in rvect diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 97cabd1fe9..4b5abbc8bc 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -126,7 +126,7 @@ static void handle_possible_compiler_internal_call(FunctionPtr cur_f, Vinferred_type == TypeDataInt::create() || v_inferred->inferred_type->try_as(); + return v_inferred->inferred_type == TypeDataInt::create() || v_inferred->inferred_type->try_as() || v_inferred->inferred_type == TypeDataCoins::create(); } static bool expect_boolean(AnyExprV v_inferred) { diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index 4b0fe9723f..bdb1763aec 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -99,6 +99,19 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { return v; } + AnyExprV replace(V v) override { + parent::replace(v); + + // replace `ton("0.05")` with 50000000 + if (v->fun_maybe && v->fun_maybe->is_builtin_function() && v->fun_maybe->name == "ton") { + ConstantValue value = eval_call_to_ton_function(v); + tolk_assert(value.is_int()); + return create_int_const(v->loc, value.as_int()); + } + + return v; + } + public: bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 58d28d421b..aa8fe64d13 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -563,7 +563,11 @@ class InferTypesAndCallsAndFieldsVisitor final { default: flow = infer_any_expr(lhs, std::move(flow), false).out_flow; flow = infer_any_expr(rhs, std::move(flow), false).out_flow; - assign_inferred_type(v, TypeDataInt::create()); + if ((v->tok == tok_plus || v->tok == tok_minus) && lhs->inferred_type == TypeDataCoins::create()) { + assign_inferred_type(v, TypeDataCoins::create()); // coins + coins = coins + } else { + assign_inferred_type(v, TypeDataInt::create()); // int8 + int8 = int, as well as other operators/types + } } if (!builtin_func.empty()) { diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index c88acb7195..d8d35d8903 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -84,6 +84,7 @@ TypePtr TypeDataBuilder::singleton; TypePtr TypeDataTuple::singleton; TypePtr TypeDataContinuation::singleton; TypePtr TypeDataNullLiteral::singleton; +TypePtr TypeDataCoins::singleton; TypePtr TypeDataUnknown::singleton; TypePtr TypeDataNever::singleton; TypePtr TypeDataVoid::singleton; @@ -97,6 +98,7 @@ void type_system_init() { TypeDataTuple::singleton = new TypeDataTuple; TypeDataContinuation::singleton = new TypeDataContinuation; TypeDataNullLiteral::singleton = new TypeDataNullLiteral; + TypeDataCoins::singleton = new TypeDataCoins; TypeDataUnknown::singleton = new TypeDataUnknown; TypeDataNever::singleton = new TypeDataNever; TypeDataVoid::singleton = new TypeDataVoid; @@ -350,6 +352,9 @@ bool TypeDataInt::can_rhs_be_assigned(TypePtr rhs) const { if (rhs->try_as()) { return true; } + if (rhs == TypeDataCoins::create()) { + return true; + } return rhs == TypeDataNever::create(); } @@ -464,6 +469,16 @@ bool TypeDataIntN::can_rhs_be_assigned(TypePtr rhs) const { return rhs == TypeDataNever::create(); // `int8` is NOT assignable to `int32` without `as` } +bool TypeDataCoins::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + if (rhs == TypeDataInt::create()) { + return true; + } + return rhs == TypeDataNever::create(); +} + bool TypeDataUnknown::can_rhs_be_assigned(TypePtr rhs) const { return true; } @@ -499,6 +514,9 @@ bool TypeDataInt::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to->try_as()) { // `int` as `int8` / `int` as `uint2` return true; } + if (cast_to == TypeDataCoins::create()) { // `int` as `coins` + return true; + } return cast_to == this; } @@ -609,7 +627,20 @@ bool TypeDataIntN::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { // `int8` as `int32?` return can_be_casted_with_as_operator(to_nullable->inner); } - return cast_to == TypeDataInt::create(); + return cast_to == TypeDataInt::create() || cast_to == TypeDataCoins::create(); +} + +bool TypeDataCoins::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to->try_as()) { // `coins` as `int8` + return true; + } + if (const auto* to_nullable = cast_to->try_as()) { // `coins` as `coins?` / `coins` as `int?` + return can_be_casted_with_as_operator(to_nullable->inner); + } + if (cast_to == TypeDataInt::create()) { + return true; + } + return cast_to == this; } bool TypeDataUnknown::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -738,6 +769,7 @@ static TypePtr parse_simple_type(Lexer& lex) { case 5: if (str == "slice") return TypeDataSlice::create(); if (str == "tuple") return TypeDataTuple::create(); + if (str == "coins") return TypeDataCoins::create(); if (str == "never") return TypeDataNever::create(); break; case 7: diff --git a/tolk/type-system.h b/tolk/type-system.h index b64fcfca12..f72cd214b2 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -391,6 +391,24 @@ class TypeDataIntN final : public TypeData { bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; +/* + * `coins` is just integer at TVM level, but encoded as varint when serializing structures. + * Example: `var cost = ton("0.05")` has type `coins`. + */ +class TypeDataCoins final : public TypeData { + TypeDataCoins() : TypeData(17ULL, 0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "coins"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + /* * `unknown` is a special type, which can appear in corner cases. * The type of exception argument (which can hold any TVM value at runtime) is unknown. From 78c05fd52df3923c97b994cd8f69c7a2bd9f6fe3 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 13 Mar 2025 18:09:41 +0400 Subject: [PATCH 158/524] [Tolk] Types `bytesN` and `bitsN` backed by TVM slice Their purpose is exactly like `intN`: future serialization, to specify fixed-size binary fields (like TL/B bits256). Note, that unlike intN, explicit casting is required. > fun calcHash(raw: bits512) { } > calcHash(someSlice); // error > calcHash(someSlice as bits512); // ok --- tolk-tester/tests/bytesN-tests.tolk | 48 +++++++++++++++++++ tolk-tester/tests/invalid-typing-39.tolk | 9 ++++ tolk-tester/tests/invalid-typing-40.tolk | 10 ++++ tolk-tester/tests/invalid-typing-41.tolk | 12 +++++ tolk-tester/tests/invalid-typing-42.tolk | 12 +++++ tolk-tester/tests/invalid-typing-43.tolk | 9 ++++ tolk/pipe-ast-to-legacy.cpp | 16 +++++++ tolk/type-system.cpp | 60 +++++++++++++++++++++++- tolk/type-system.h | 23 +++++++++ 9 files changed, 198 insertions(+), 1 deletion(-) create mode 100644 tolk-tester/tests/bytesN-tests.tolk create mode 100644 tolk-tester/tests/invalid-typing-39.tolk create mode 100644 tolk-tester/tests/invalid-typing-40.tolk create mode 100644 tolk-tester/tests/invalid-typing-41.tolk create mode 100644 tolk-tester/tests/invalid-typing-42.tolk create mode 100644 tolk-tester/tests/invalid-typing-43.tolk diff --git a/tolk-tester/tests/bytesN-tests.tolk b/tolk-tester/tests/bytesN-tests.tolk new file mode 100644 index 0000000000..32b975bf78 --- /dev/null +++ b/tolk-tester/tests/bytesN-tests.tolk @@ -0,0 +1,48 @@ +fun takeAnySlice(slice: slice) {} +fun getSomeSlice() { return beginCell().endCell().beginParse(); } +fun getNullableBytes16(): bytes16? { return getSomeSlice() as bytes16; } + +const someBytes = "asdf" as bytes16; +const anotherBytes: bytes16 = "asdf" as bytes16; + +fun autoInferBytes16() { + if (random()) { return someBytes; } + else if (random()) { return true ? "" as bytes16 : anotherBytes; } + else { return someBytes!; } +} + +@method_id(101) +fun test1() { + var ss = beginCell().storeInt(1, 32).storeInt(2, 32).endCell().beginParse() as bits32; + __expect_type(ss, "bits32"); + __expect_type(ss as slice, "slice"); + __expect_type(ss as slice?, "slice?"); + __expect_type(ss as bytes128, "bytes128"); + __expect_type(someBytes, "bytes16"); + __expect_type(10>3 ? (null, ss as bits1) : (ss as bytes1, null), "(bytes1?, bits1?)"); + return (getRemainingBitsCount(ss as slice), loadInt(mutate ss as slice, 32), (ss as slice).loadInt(32)); +} + +fun test2(a: bytes8, b: bits64) { + a = b as bytes8; + b = a as bits64; + (a as slice).loadRef(); + (b as slice?)!.loadRef(); +} + +@method_id(103) +fun test3() { + var slice = beginCell().storeInt(1, 32).storeInt(2, 32).endCell().beginParse(); + var bq = slice as bits16?; + if (bq != null) { + return (bq as slice).loadInt(32 as uint8) + (slice = bq as slice).getRemainingBitsCount(); + } + return -1; +} + +fun main() {} + +/** +@testcase | 101 | | 64 1 2 +@testcase | 103 | | 33 + */ diff --git a/tolk-tester/tests/invalid-typing-39.tolk b/tolk-tester/tests/invalid-typing-39.tolk new file mode 100644 index 0000000000..47ebb1a2fb --- /dev/null +++ b/tolk-tester/tests/invalid-typing-39.tolk @@ -0,0 +1,9 @@ +fun cantUnifyBytesNAndBytesM(n: bytes8, c: bytes16) { + __expect_type(random() ? n : c as bytes8, "bytes8"); + random() ? n : c; +} + +/** +@compilation_should_fail +@stderr types of ternary branches are incompatible: `bytes8` and `bytes16` + */ diff --git a/tolk-tester/tests/invalid-typing-40.tolk b/tolk-tester/tests/invalid-typing-40.tolk new file mode 100644 index 0000000000..274f51b4c2 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-40.tolk @@ -0,0 +1,10 @@ +fun cantUnifyBytesNAndSlice(n: slice, c: bytes16) { + __expect_type(random() ? n : c as slice, "slice"); + __expect_type(random() ? n : c as slice?, "slice?"); + random() ? n : c; +} + +/** +@compilation_should_fail +@stderr types of ternary branches are incompatible: `slice` and `bytes16` + */ diff --git a/tolk-tester/tests/invalid-typing-41.tolk b/tolk-tester/tests/invalid-typing-41.tolk new file mode 100644 index 0000000000..b04e8d974f --- /dev/null +++ b/tolk-tester/tests/invalid-typing-41.tolk @@ -0,0 +1,12 @@ +fun takeAnySlice(s: slice) {} + +fun cantPassBytesNToSlice(c: bits16) { + takeAnySlice(c as slice); // ok + takeAnySlice(c); +} + +/** +@compilation_should_fail +@stderr can not pass `bits16` to `slice` +@stderr takeAnySlice(c); + */ diff --git a/tolk-tester/tests/invalid-typing-42.tolk b/tolk-tester/tests/invalid-typing-42.tolk new file mode 100644 index 0000000000..d380279a94 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-42.tolk @@ -0,0 +1,12 @@ +fun getAnySlice() { return beginCell().endCell().beginParse(); } + +fun cantAssignSliceToBytesN() { + var c1: bytes16 = getAnySlice() as bytes16; // ok + var c2: bytes16 = getAnySlice(); +} + +/** +@compilation_should_fail +@stderr can not assign `slice` to variable of type `bytes16` +@stderr var c2: bytes16 = getAnySlice(); + */ diff --git a/tolk-tester/tests/invalid-typing-43.tolk b/tolk-tester/tests/invalid-typing-43.tolk new file mode 100644 index 0000000000..463ba4871e --- /dev/null +++ b/tolk-tester/tests/invalid-typing-43.tolk @@ -0,0 +1,9 @@ +fun cantAutoCastBytesNToSlice() { + var b = beginCell().storeInt(1, 32).endCell().beginParse() as bits32; + return b.loadInt(32); +} + +/** +@compilation_should_fail +@stderr can not call method for `slice` with object of type `bits32` + */ diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 3d0f5dfeea..d0c252847c 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -581,6 +581,22 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as()) { return rvect; } + // pass `bytes32` to `slice` + // in code, it's probably done with `as` operator + // no changes in rvect, since bytesN is slice at TVM level + if (target_type == TypeDataSlice::create() && original_type->try_as()) { + return rvect; + } + // pass `slice` to `bytes32` + // same as above + if (original_type == TypeDataSlice::create() && target_type->try_as()) { + return rvect; + } + // pass `bytes32` to `bytes64` / `bits128` to `bytes16` + // no changes in rvect + if (original_type->try_as() && target_type->try_as()) { + return rvect; + } // pass something to `unknown` // probably, it comes from `_ = rhs`, type of `_` is unknown, it's target_type of rhs // no changes in rvect diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index d8d35d8903..ebcfc6fe6f 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -193,6 +193,17 @@ TypePtr TypeDataIntN::create(bool is_unsigned, bool is_variadic, int n_bits) { return hash.register_unique(new TypeDataIntN(hash.type_id(), is_unsigned, is_variadic, n_bits)); } +TypePtr TypeDataBytesN::create(bool is_bits, int n_width) { + TypeDataTypeIdCalculation hash(7810988137199333041ULL); + hash.feed_hash(is_bits); + hash.feed_hash(n_width); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + return hash.register_unique(new TypeDataBytesN(hash.type_id(), is_bits, n_width)); +} + TypePtr TypeDataUnresolved::create(std::string&& text, SrcLocation loc) { TypeDataTypeIdCalculation hash(3680147223540048162ULL); hash.feed_string(text); @@ -262,6 +273,11 @@ std::string TypeDataIntN::as_human_readable() const { return s_int + std::to_string(n_bits); } +std::string TypeDataBytesN::as_human_readable() const { + std::string s_bytes = is_bits ? "bits" : "bytes"; + return s_bytes + std::to_string(n_width); +} + // -------------------------------------------- // traverse() @@ -376,7 +392,7 @@ bool TypeDataSlice::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } - return rhs == TypeDataNever::create(); + return rhs == TypeDataNever::create(); // note, that bytesN is NOT automatically cast to slice without `as` operator } bool TypeDataBuilder::can_rhs_be_assigned(TypePtr rhs) const { @@ -469,6 +485,15 @@ bool TypeDataIntN::can_rhs_be_assigned(TypePtr rhs) const { return rhs == TypeDataNever::create(); // `int8` is NOT assignable to `int32` without `as` } +bool TypeDataBytesN::can_rhs_be_assigned(TypePtr rhs) const { + // `slice` is NOT assignable to bytesN without `as` + // `bytes32` is NOT assignable to `bytes256` and even to `bits256` without `as` + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + bool TypeDataCoins::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; @@ -541,6 +566,9 @@ bool TypeDataCell::can_be_casted_with_as_operator(TypePtr cast_to) const { } bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to->try_as()) { // `slice` to `bytes32` / `slice` to `bits8` + return true; + } if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } @@ -630,6 +658,16 @@ bool TypeDataIntN::can_be_casted_with_as_operator(TypePtr cast_to) const { return cast_to == TypeDataInt::create() || cast_to == TypeDataCoins::create(); } +bool TypeDataBytesN::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to->try_as()) { // `bytes256` as `bytes512`, `bits1` as `bytes8` + return true; + } + if (const auto* to_nullable = cast_to->try_as()) { // `bytes8` as `slice?` + return can_be_casted_with_as_operator(to_nullable->inner); + } + return cast_to == TypeDataSlice::create(); +} + bool TypeDataCoins::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to->try_as()) { // `coins` as `int8` return true; @@ -750,6 +788,16 @@ static TypePtr parse_intN(std::string_view strN, bool is_unsigned) { return TypeDataIntN::create(is_unsigned, false, n); } +static TypePtr parse_bytesN(std::string_view strN, bool is_bits) { + int n; + auto result = std::from_chars(strN.data() + 5 - static_cast(is_bits), strN.data() + strN.size(), n); + bool parsed = result.ec == std::errc() && result.ptr == strN.data() + strN.size(); + if (!parsed || n <= 0 || n > 1024) { + return nullptr; // `bytes9999`, maybe it's user-defined alias, let it be unresolved + } + return TypeDataBytesN::create(is_bits, n); +} + static TypePtr parse_simple_type(Lexer& lex) { switch (lex.tok()) { case tok_self: @@ -795,6 +843,16 @@ static TypePtr parse_simple_type(Lexer& lex) { return uintN; } } + if (str.size() > 4 && str.starts_with("bits")) { + if (TypePtr bitsN = parse_bytesN(str, true)) { + return bitsN; + } + } + if (str.size() > 5 && str.starts_with("bytes")) { + if (TypePtr bytesN = parse_bytesN(str, false)) { + return bytesN; + } + } return TypeDataUnresolved::create(std::string(str), loc); } case tok_null: diff --git a/tolk/type-system.h b/tolk/type-system.h index f72cd214b2..07a2496ca0 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -409,6 +409,29 @@ class TypeDataCoins final : public TypeData { bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; +/* + * `bytes256`, `bits512`, `bytes8` are TypeDataBytesN. At TVM level, it's just slice. + * The purpose of bytesN is to be used in struct fields, describing the way of serialization (n bytes / n bits). + * In this essence, bytesN is very similar to intN. + * Note, that unlike intN automatically cast to/from int, bytesN does NOT auto cast to slice (without `as`). + */ +class TypeDataBytesN final : public TypeData { + TypeDataBytesN(uint64_t type_id, bool is_bits, int n_width) + : TypeData(type_id, 0, 1) + , is_bits(is_bits) + , n_width(n_width) {} + +public: + const bool is_bits; + const int n_width; + + static TypePtr create(bool is_bits, int n_width); + + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + /* * `unknown` is a special type, which can appear in corner cases. * The type of exception argument (which can hold any TVM value at runtime) is unknown. From 35f45b030cf79ac13822a78e1ef803c1c73bb04b Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 13 Mar 2025 18:11:57 +0400 Subject: [PATCH 159/524] [Tolk] Tidy up the tests, separate into folders This commit reorganizes tests for the compiler. Before, they all were places in a single folder. Now, they negative tests are separated. Negative tests are named just `err-XXXX`, for quick file search in IDE. Tolk-tester can now execute tests by "contains" pattern: > tolk-tester tests catch will find "tests/**/*catch*.tolk" --- tolk-tester/tests/{a6.tolk => a-tests.tolk} | 2 +- tolk-tester/tests/a6_1.tolk | 22 --- tolk-tester/tests/a6_5.tolk | 26 ---- tolk-tester/tests/a7.tolk | 24 --- ...tion.tolk => allow-post-modification.tolk} | 0 ...{asm_arg_order.tolk => asm-arg-order.tolk} | 0 tolk-tester/tests/c2.tolk | 28 ---- tolk-tester/tests/c2_1.tolk | 14 -- tolk-tester/tests/code_after_ifelse.tolk | 41 ----- ...heck_demo.tolk => codegen-check-demo.tolk} | 0 .../tests/{co1.tolk => constants-tests.tolk} | 0 .../{if_stmt.tolk => if-else-tests.tolk} | 48 ++++++ tolk-tester/tests/inference-tests.tolk | 12 +- tolk-tester/tests/inline-tests.tolk | 140 ++++++++++++++++++ tolk-tester/tests/inline_big.tolk | 62 -------- tolk-tester/tests/inline_if.tolk | 28 ---- tolk-tester/tests/inline_loops.tolk | 48 ------ .../err-1084.tolk} | 0 .../err-1153.tolk} | 0 .../err-1209.tolk} | 0 .../err-1361.tolk} | 0 .../err-1449.tolk} | 0 .../err-1505.tolk} | 0 .../err-1601.tolk} | 0 .../err-1683.tolk} | 0 .../err-1701.tolk} | 0 .../err-1790.tolk} | 0 .../err-1794.tolk} | 0 .../err-1805.tolk} | 0 .../err-1842.tolk} | 0 .../err-1918.tolk} | 2 +- .../err-1940.tolk} | 0 tolk-tester/tests/invalid-no-import-1.tolk | 8 - .../err-4002.tolk} | 0 .../err-4061.tolk} | 0 .../err-4064.tolk} | 0 .../err-4080.tolk} | 0 .../err-4083.tolk} | 0 .../err-4104.tolk} | 0 .../err-4127.tolk} | 0 .../err-4195.tolk} | 0 .../err-4266.tolk} | 0 .../err-4377.tolk} | 0 .../err-4381.tolk} | 0 .../err-4385.tolk} | 0 .../err-4413.tolk} | 0 .../err-4416.tolk} | 0 .../err-4473.tolk} | 0 .../err-4506.tolk} | 0 .../err-4510.tolk} | 0 .../err-4520.tolk} | 0 .../err-4542.tolk} | 0 .../err-4553.tolk} | 0 .../err-4571.tolk} | 0 .../err-4604.tolk} | 0 .../err-4639.tolk} | 0 .../err-4705.tolk} | 0 .../err-4728.tolk} | 0 .../err-4751.tolk} | 0 .../err-4804.tolk} | 0 .../err-4819.tolk} | 0 .../err-4828.tolk} | 0 .../err-4838.tolk} | 0 .../err-4841.tolk} | 0 .../err-4851.tolk} | 0 .../err-4880.tolk} | 0 .../err-4899.tolk} | 0 .../err-4912.tolk} | 0 .../err-4941.tolk} | 0 .../err-4951.tolk} | 0 .../err-4956.tolk} | 0 .../err-2055.tolk} | 0 .../err-2188.tolk} | 0 .../tests/invalid-symbol/err-2205.tolk | 8 + .../err-2255.tolk} | 0 .../err-2374.tolk} | 0 .../err-2463.tolk} | 0 .../err-2501.tolk} | 0 .../err-2520.tolk} | 0 .../err-2539.tolk} | 4 +- .../err-2649.tolk} | 0 .../err-2673.tolk} | 0 .../err-2715.tolk} | 0 .../err-2750.tolk} | 0 .../err-2804.tolk} | 0 .../err-2853.tolk} | 0 .../err-2914.tolk} | 0 .../err-2980.tolk} | 4 +- .../err-3001.tolk} | 0 .../err-3015.tolk} | 0 .../err-3035.tolk} | 0 .../err-3041.tolk} | 0 .../err-3044.tolk} | 0 .../err-3059.tolk} | 0 .../err-3102.tolk} | 0 .../err-3164.tolk} | 0 .../err-3183.tolk} | 0 .../err-3197.tolk} | 0 .../err-3205.tolk} | 0 .../err-3274.tolk} | 0 .../err-3295.tolk} | 0 .../err-3390.tolk} | 0 .../err-3407.tolk} | 0 .../err-3472.tolk} | 0 .../err-3527.tolk} | 0 .../err-3551.tolk} | 0 .../err-3586.tolk} | 0 .../err-3588.tolk} | 0 .../err-3603.tolk} | 0 .../tests/invalid-syntax/err-3618.tolk | 6 + .../err-3714.tolk} | 0 .../err-3719.tolk} | 0 .../err-3781.tolk} | 0 .../err-3819.tolk} | 0 .../err-3851.tolk} | 0 .../err-3853.tolk} | 0 .../err-3898.tolk} | 0 .../err-3910.tolk} | 0 .../err-3967.tolk} | 0 .../err-3992.tolk} | 0 .../err-6039.tolk} | 0 .../err-6040.tolk} | 0 .../err-6041.tolk} | 0 .../err-6080.tolk} | 0 .../err-6085.tolk} | 0 .../err-6087.tolk} | 0 .../err-6095.tolk} | 0 .../err-6114.tolk} | 0 .../err-6130.tolk} | 0 .../err-6134.tolk} | 0 .../err-6135.tolk} | 0 .../err-6148.tolk} | 0 .../err-6188.tolk} | 0 .../err-6200.tolk} | 0 .../err-6204.tolk} | 0 .../err-6220.tolk} | 0 .../err-6249.tolk} | 0 .../err-6257.tolk} | 0 .../err-6288.tolk} | 0 .../err-6337.tolk} | 0 .../err-6368.tolk} | 0 .../err-6369.tolk} | 0 .../err-6370.tolk} | 0 .../err-6386.tolk} | 0 .../err-6391.tolk} | 0 .../err-6393.tolk} | 0 .../err-6403.tolk} | 0 .../err-6407.tolk} | 0 .../err-6450.tolk} | 0 .../err-6496.tolk} | 0 .../err-6499.tolk} | 0 .../err-6509.tolk} | 0 .../err-6519.tolk} | 0 .../err-6533.tolk} | 0 .../err-6553.tolk} | 0 .../err-6580.tolk} | 0 .../err-6600.tolk} | 0 .../err-6613.tolk} | 0 .../err-6619.tolk} | 0 .../err-6629.tolk} | 0 .../err-6637.tolk} | 0 .../err-6711.tolk} | 0 .../err-6713.tolk} | 0 .../err-6731.tolk} | 0 .../err-6734.tolk} | 0 .../err-6737.tolk} | 0 .../err-6743.tolk} | 0 .../err-6803.tolk} | 0 .../err-6810.tolk} | 0 .../err-6820.tolk} | 0 .../err-6839.tolk} | 0 .../err-6840.tolk} | 0 .../err-6841.tolk} | 0 .../err-6844.tolk} | 0 .../err-6881.tolk} | 0 .../err-6901.tolk} | 0 .../err-6903.tolk} | 0 .../err-6921.tolk} | 0 .../err-6925.tolk} | 0 .../err-6937.tolk} | 0 .../err-6981.tolk} | 0 .../err-6992.tolk} | 0 tolk-tester/tests/method_id.tolk | 17 --- tolk-tester/tests/some-tests-1.tolk | 127 ++++++++++++++++ .../tests/{a10.tolk => some-tests-2.tolk} | 0 .../tests/{w2.tolk => some-tests-3.tolk} | 0 .../tests/{s1.tolk => strings-tests.tolk} | 0 .../{try-func.tolk => try-catch-tests.tolk} | 0 ...t_loops.tolk => unbalanced-ret-loops.tolk} | 0 tolk-tester/tests/unbalanced-ret.tolk | 73 +++++++++ tolk-tester/tests/unbalanced_ret.tolk | 17 --- tolk-tester/tests/unbalanced_ret_inline.tolk | 19 --- tolk-tester/tests/unbalanced_ret_nested.tolk | 37 ----- tolk-tester/tests/use-before-declare.tolk | 15 ++ .../{var-apply.tolk => var-apply-tests.tolk} | 30 +++- tolk-tester/tests/w1.tolk | 14 -- tolk-tester/tests/w6.tolk | 19 --- tolk-tester/tests/w7.tolk | 26 ---- tolk-tester/tests/w9.tolk | 14 -- .../unreachable-1.tolk | 0 .../unreachable-2.tolk | 0 .../unreachable-3.tolk | 0 .../unreachable-4.tolk | 0 .../{ => warnings-not-errors}/warnings-1.tolk | 0 .../{ => warnings-not-errors}/warnings-2.tolk | 0 tolk-tester/tolk-tester.js | 46 +++--- tolk-tester/tolk-tester.py | 63 ++++---- 207 files changed, 527 insertions(+), 517 deletions(-) rename tolk-tester/tests/{a6.tolk => a-tests.tolk} (98%) delete mode 100644 tolk-tester/tests/a6_1.tolk delete mode 100644 tolk-tester/tests/a6_5.tolk delete mode 100644 tolk-tester/tests/a7.tolk rename tolk-tester/tests/{allow_post_modification.tolk => allow-post-modification.tolk} (100%) rename tolk-tester/tests/{asm_arg_order.tolk => asm-arg-order.tolk} (100%) delete mode 100644 tolk-tester/tests/c2.tolk delete mode 100644 tolk-tester/tests/c2_1.tolk delete mode 100644 tolk-tester/tests/code_after_ifelse.tolk rename tolk-tester/tests/{codegen_check_demo.tolk => codegen-check-demo.tolk} (100%) rename tolk-tester/tests/{co1.tolk => constants-tests.tolk} (100%) rename tolk-tester/tests/{if_stmt.tolk => if-else-tests.tolk} (59%) create mode 100644 tolk-tester/tests/inline-tests.tolk delete mode 100644 tolk-tester/tests/inline_big.tolk delete mode 100644 tolk-tester/tests/inline_if.tolk delete mode 100644 tolk-tester/tests/inline_loops.tolk rename tolk-tester/tests/{invalid-declaration-8.tolk => invalid-declaration/err-1084.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-16.tolk => invalid-declaration/err-1153.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-1.tolk => invalid-declaration/err-1209.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-10.tolk => invalid-declaration/err-1361.tolk} (100%) rename tolk-tester/tests/{invalid-get-method-2.tolk => invalid-declaration/err-1449.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-13.tolk => invalid-declaration/err-1505.tolk} (100%) rename tolk-tester/tests/{invalid-self-6.tolk => invalid-declaration/err-1601.tolk} (100%) rename tolk-tester/tests/{invalid-redefinition-1.tolk => invalid-declaration/err-1683.tolk} (100%) rename tolk-tester/tests/{invalid-builtin-1.tolk => invalid-declaration/err-1701.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-11.tolk => invalid-declaration/err-1790.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-6.tolk => invalid-declaration/err-1794.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-9.tolk => invalid-declaration/err-1805.tolk} (100%) rename tolk-tester/tests/{invalid-cyclic-1.tolk => invalid-declaration/err-1842.tolk} (100%) rename tolk-tester/tests/{invalid-redefinition-2.tolk => invalid-declaration/err-1918.tolk} (78%) rename tolk-tester/tests/{invalid-get-method-1.tolk => invalid-declaration/err-1940.tolk} (100%) delete mode 100644 tolk-tester/tests/invalid-no-import-1.tolk rename tolk-tester/tests/{invalid-call-5.tolk => invalid-semantics/err-4002.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-5.tolk => invalid-semantics/err-4061.tolk} (100%) rename tolk-tester/tests/{invalid-pure-1.tolk => invalid-semantics/err-4064.tolk} (100%) rename tolk-tester/tests/{invalid-call-4.tolk => invalid-semantics/err-4080.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-16.tolk => invalid-semantics/err-4083.tolk} (100%) rename tolk-tester/tests/{invalid-generics-6.tolk => invalid-semantics/err-4104.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-14.tolk => invalid-semantics/err-4127.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-18.tolk => invalid-semantics/err-4195.tolk} (100%) rename tolk-tester/tests/{invalid-generics-10.tolk => invalid-semantics/err-4266.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-7.tolk => invalid-semantics/err-4377.tolk} (100%) rename tolk-tester/tests/{invalid-typing-13.tolk => invalid-semantics/err-4381.tolk} (100%) rename tolk-tester/tests/{invalid-pure-3.tolk => invalid-semantics/err-4385.tolk} (100%) rename tolk-tester/tests/{invalid-pure-2.tolk => invalid-semantics/err-4413.tolk} (100%) rename tolk-tester/tests/{invalid-generics-5.tolk => invalid-semantics/err-4416.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-11.tolk => invalid-semantics/err-4473.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-10.tolk => invalid-semantics/err-4506.tolk} (100%) rename tolk-tester/tests/{invalid-generics-11.tolk => invalid-semantics/err-4510.tolk} (100%) rename tolk-tester/tests/{invalid-call-8.tolk => invalid-semantics/err-4520.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-17.tolk => invalid-semantics/err-4542.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-4.tolk => invalid-semantics/err-4553.tolk} (100%) rename tolk-tester/tests/{invalid-syntax-3.tolk => invalid-semantics/err-4571.tolk} (100%) rename tolk-tester/tests/{invalid-generics-9.tolk => invalid-semantics/err-4604.tolk} (100%) rename tolk-tester/tests/{invalid-syntax-6.tolk => invalid-semantics/err-4639.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-12.tolk => invalid-semantics/err-4705.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-1.tolk => invalid-semantics/err-4728.tolk} (100%) rename tolk-tester/tests/{invalid-call-3.tolk => invalid-semantics/err-4751.tolk} (100%) rename tolk-tester/tests/{invalid-call-11.tolk => invalid-semantics/err-4804.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-3.tolk => invalid-semantics/err-4819.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-15.tolk => invalid-semantics/err-4828.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-6.tolk => invalid-semantics/err-4838.tolk} (100%) rename tolk-tester/tests/{invalid-call-6.tolk => invalid-semantics/err-4841.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-9.tolk => invalid-semantics/err-4851.tolk} (100%) rename tolk-tester/tests/{invalid-generics-1.tolk => invalid-semantics/err-4880.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-14.tolk => invalid-semantics/err-4899.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-2.tolk => invalid-semantics/err-4912.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-19.tolk => invalid-semantics/err-4941.tolk} (100%) rename tolk-tester/tests/{invalid-call-9.tolk => invalid-semantics/err-4951.tolk} (100%) rename tolk-tester/tests/{invalid-call-2.tolk => invalid-semantics/err-4956.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-8.tolk => invalid-symbol/err-2055.tolk} (100%) rename tolk-tester/tests/{invalid-call-7.tolk => invalid-symbol/err-2188.tolk} (100%) create mode 100644 tolk-tester/tests/invalid-symbol/err-2205.tolk rename tolk-tester/tests/{invalid-typing-32.tolk => invalid-symbol/err-2255.tolk} (100%) rename tolk-tester/tests/{invalid-symbol-1.tolk => invalid-symbol/err-2374.tolk} (100%) rename tolk-tester/tests/{invalid-symbol-2.tolk => invalid-symbol/err-2463.tolk} (100%) rename tolk-tester/tests/{invalid-redefinition-6.tolk => invalid-symbol/err-2501.tolk} (100%) rename tolk-tester/tests/{invalid-self-7.tolk => invalid-symbol/err-2520.tolk} (100%) rename tolk-tester/tests/{invalid-no-import-2.tolk => invalid-symbol/err-2539.tolk} (66%) rename tolk-tester/tests/{invalid-redefinition-3.tolk => invalid-symbol/err-2649.tolk} (100%) rename tolk-tester/tests/{invalid-self-3.tolk => invalid-symbol/err-2673.tolk} (100%) rename tolk-tester/tests/{invalid-redefinition-5.tolk => invalid-symbol/err-2715.tolk} (100%) rename tolk-tester/tests/{invalid-typing-1.tolk => invalid-symbol/err-2750.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-15.tolk => invalid-symbol/err-2804.tolk} (100%) rename tolk-tester/tests/{invalid-redefinition-4.tolk => invalid-symbol/err-2853.tolk} (100%) rename tolk-tester/tests/{invalid-call-1.tolk => invalid-symbol/err-2914.tolk} (100%) rename tolk-tester/tests/{invalid-import.tolk => invalid-symbol/err-2980.tolk} (73%) rename tolk-tester/tests/{invalid-tolk-version.tolk => invalid-syntax/err-3001.tolk} (100%) rename tolk-tester/tests/{invalid-call-13.tolk => invalid-syntax/err-3015.tolk} (100%) rename tolk-tester/tests/{invalid-bitwise-4.tolk => invalid-syntax/err-3035.tolk} (100%) rename tolk-tester/tests/{invalid-nopar-1.tolk => invalid-syntax/err-3041.tolk} (100%) rename tolk-tester/tests/{invalid-shift-1.tolk => invalid-syntax/err-3044.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-4.tolk => invalid-syntax/err-3059.tolk} (100%) rename tolk-tester/tests/{invalid-bitwise-3.tolk => invalid-syntax/err-3102.tolk} (100%) rename tolk-tester/tests/{invalid-catch-1.tolk => invalid-syntax/err-3164.tolk} (100%) rename tolk-tester/tests/{invalid-nopar-2.tolk => invalid-syntax/err-3183.tolk} (100%) rename tolk-tester/tests/{invalid-nopar-4.tolk => invalid-syntax/err-3197.tolk} (100%) rename tolk-tester/tests/{invalid-syntax-7.tolk => invalid-syntax/err-3205.tolk} (100%) rename tolk-tester/tests/{invalid-nopar-3.tolk => invalid-syntax/err-3274.tolk} (100%) rename tolk-tester/tests/{invalid-const-1.tolk => invalid-syntax/err-3295.tolk} (100%) rename tolk-tester/tests/{invalid-cmt-old.tolk => invalid-syntax/err-3390.tolk} (100%) rename tolk-tester/tests/{invalid-cmt-nested.tolk => invalid-syntax/err-3407.tolk} (100%) rename tolk-tester/tests/{invalid-call-14.tolk => invalid-syntax/err-3472.tolk} (100%) rename tolk-tester/tests/{invalid-syntax-5.tolk => invalid-syntax/err-3527.tolk} (100%) rename tolk-tester/tests/{invalid-syntax-1.tolk => invalid-syntax/err-3551.tolk} (100%) rename tolk-tester/tests/{invalid-catch-2.tolk => invalid-syntax/err-3586.tolk} (100%) rename tolk-tester/tests/{invalid-bitwise-5.tolk => invalid-syntax/err-3588.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-7.tolk => invalid-syntax/err-3603.tolk} (100%) create mode 100644 tolk-tester/tests/invalid-syntax/err-3618.tolk rename tolk-tester/tests/{invalid-call-12.tolk => invalid-syntax/err-3714.tolk} (100%) rename tolk-tester/tests/{invalid-syntax-4.tolk => invalid-syntax/err-3719.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-5.tolk => invalid-syntax/err-3781.tolk} (100%) rename tolk-tester/tests/{invalid-bitwise-1.tolk => invalid-syntax/err-3819.tolk} (100%) rename tolk-tester/tests/{invalid-bitwise-7.tolk => invalid-syntax/err-3851.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-2.tolk => invalid-syntax/err-3853.tolk} (100%) rename tolk-tester/tests/{invalid-syntax-2.tolk => invalid-syntax/err-3898.tolk} (100%) rename tolk-tester/tests/{invalid-bitwise-2.tolk => invalid-syntax/err-3910.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-3.tolk => invalid-syntax/err-3967.tolk} (100%) rename tolk-tester/tests/{invalid-bitwise-6.tolk => invalid-syntax/err-3992.tolk} (100%) rename tolk-tester/tests/{invalid-assign-1.tolk => invalid-typing/err-6039.tolk} (100%) rename tolk-tester/tests/{invalid-assign-3.tolk => invalid-typing/err-6040.tolk} (100%) rename tolk-tester/tests/{invalid-call-10.tolk => invalid-typing/err-6041.tolk} (100%) rename tolk-tester/tests/{invalid-typing-18.tolk => invalid-typing/err-6080.tolk} (100%) rename tolk-tester/tests/{invalid-generics-14.tolk => invalid-typing/err-6085.tolk} (100%) rename tolk-tester/tests/{invalid-typing-43.tolk => invalid-typing/err-6087.tolk} (100%) rename tolk-tester/tests/{invalid-generics-4.tolk => invalid-typing/err-6095.tolk} (100%) rename tolk-tester/tests/{invalid-typing-20.tolk => invalid-typing/err-6114.tolk} (100%) rename tolk-tester/tests/{invalid-typing-44.tolk => invalid-typing/err-6130.tolk} (100%) rename tolk-tester/tests/{invalid-typing-4.tolk => invalid-typing/err-6134.tolk} (100%) rename tolk-tester/tests/{invalid-typing-37.tolk => invalid-typing/err-6135.tolk} (100%) rename tolk-tester/tests/{invalid-typing-25.tolk => invalid-typing/err-6148.tolk} (100%) rename tolk-tester/tests/{invalid-typing-10.tolk => invalid-typing/err-6188.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-13.tolk => invalid-typing/err-6200.tolk} (100%) rename tolk-tester/tests/{invalid-typing-38.tolk => invalid-typing/err-6204.tolk} (100%) rename tolk-tester/tests/{invalid-typing-24.tolk => invalid-typing/err-6220.tolk} (100%) rename tolk-tester/tests/{invalid-typing-40.tolk => invalid-typing/err-6249.tolk} (100%) rename tolk-tester/tests/{invalid-self-5.tolk => invalid-typing/err-6257.tolk} (100%) rename tolk-tester/tests/{invalid-self-1.tolk => invalid-typing/err-6288.tolk} (100%) rename tolk-tester/tests/{invalid-typing-16.tolk => invalid-typing/err-6337.tolk} (100%) rename tolk-tester/tests/{invalid-typing-45.tolk => invalid-typing/err-6368.tolk} (100%) rename tolk-tester/tests/{invalid-generics-3.tolk => invalid-typing/err-6369.tolk} (100%) rename tolk-tester/tests/{invalid-typing-23.tolk => invalid-typing/err-6370.tolk} (100%) rename tolk-tester/tests/{invalid-generics-2.tolk => invalid-typing/err-6386.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-20.tolk => invalid-typing/err-6391.tolk} (100%) rename tolk-tester/tests/{invalid-typing-9.tolk => invalid-typing/err-6393.tolk} (100%) rename tolk-tester/tests/{invalid-typing-34.tolk => invalid-typing/err-6403.tolk} (100%) rename tolk-tester/tests/{invalid-never-1.tolk => invalid-typing/err-6407.tolk} (100%) rename tolk-tester/tests/{invalid-typing-2.tolk => invalid-typing/err-6450.tolk} (100%) rename tolk-tester/tests/{invalid-typing-29.tolk => invalid-typing/err-6496.tolk} (100%) rename tolk-tester/tests/{invalid-typing-22.tolk => invalid-typing/err-6499.tolk} (100%) rename tolk-tester/tests/{invalid-generics-13.tolk => invalid-typing/err-6509.tolk} (100%) rename tolk-tester/tests/{invalid-typing-5.tolk => invalid-typing/err-6519.tolk} (100%) rename tolk-tester/tests/{invalid-self-2.tolk => invalid-typing/err-6533.tolk} (100%) rename tolk-tester/tests/{invalid-typing-26.tolk => invalid-typing/err-6553.tolk} (100%) rename tolk-tester/tests/{invalid-typing-21.tolk => invalid-typing/err-6580.tolk} (100%) rename tolk-tester/tests/{invalid-typing-8.tolk => invalid-typing/err-6600.tolk} (100%) rename tolk-tester/tests/{invalid-typing-31.tolk => invalid-typing/err-6613.tolk} (100%) rename tolk-tester/tests/{invalid-typing-35.tolk => invalid-typing/err-6619.tolk} (100%) rename tolk-tester/tests/{invalid-generics-8.tolk => invalid-typing/err-6629.tolk} (100%) rename tolk-tester/tests/{invalid-typing-14.tolk => invalid-typing/err-6637.tolk} (100%) rename tolk-tester/tests/{invalid-typing-11.tolk => invalid-typing/err-6711.tolk} (100%) rename tolk-tester/tests/{invalid-typing-27.tolk => invalid-typing/err-6713.tolk} (100%) rename tolk-tester/tests/{invalid-assign-2.tolk => invalid-typing/err-6731.tolk} (100%) rename tolk-tester/tests/{invalid-typing-12.tolk => invalid-typing/err-6734.tolk} (100%) rename tolk-tester/tests/{invalid-typing-33.tolk => invalid-typing/err-6737.tolk} (100%) rename tolk-tester/tests/{invalid-typing-6.tolk => invalid-typing/err-6743.tolk} (100%) rename tolk-tester/tests/{invalid-typing-41.tolk => invalid-typing/err-6803.tolk} (100%) rename tolk-tester/tests/{invalid-typing-36.tolk => invalid-typing/err-6810.tolk} (100%) rename tolk-tester/tests/{invalid-typing-39.tolk => invalid-typing/err-6820.tolk} (100%) rename tolk-tester/tests/{invalid-typing-3.tolk => invalid-typing/err-6839.tolk} (100%) rename tolk-tester/tests/{invalid-typing-15.tolk => invalid-typing/err-6840.tolk} (100%) rename tolk-tester/tests/{invalid-typing-28.tolk => invalid-typing/err-6841.tolk} (100%) rename tolk-tester/tests/{invalid-self-4.tolk => invalid-typing/err-6844.tolk} (100%) rename tolk-tester/tests/{invalid-typing-17.tolk => invalid-typing/err-6881.tolk} (100%) rename tolk-tester/tests/{invalid-typing-7.tolk => invalid-typing/err-6901.tolk} (100%) rename tolk-tester/tests/{invalid-typing-19.tolk => invalid-typing/err-6903.tolk} (100%) rename tolk-tester/tests/{invalid-generics-7.tolk => invalid-typing/err-6921.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-12.tolk => invalid-typing/err-6925.tolk} (100%) rename tolk-tester/tests/{invalid-typing-42.tolk => invalid-typing/err-6937.tolk} (100%) rename tolk-tester/tests/{invalid-generics-12.tolk => invalid-typing/err-6981.tolk} (100%) rename tolk-tester/tests/{invalid-typing-30.tolk => invalid-typing/err-6992.tolk} (100%) delete mode 100644 tolk-tester/tests/method_id.tolk create mode 100644 tolk-tester/tests/some-tests-1.tolk rename tolk-tester/tests/{a10.tolk => some-tests-2.tolk} (100%) rename tolk-tester/tests/{w2.tolk => some-tests-3.tolk} (100%) rename tolk-tester/tests/{s1.tolk => strings-tests.tolk} (100%) rename tolk-tester/tests/{try-func.tolk => try-catch-tests.tolk} (100%) rename tolk-tester/tests/{unbalanced_ret_loops.tolk => unbalanced-ret-loops.tolk} (100%) create mode 100644 tolk-tester/tests/unbalanced-ret.tolk delete mode 100644 tolk-tester/tests/unbalanced_ret.tolk delete mode 100644 tolk-tester/tests/unbalanced_ret_inline.tolk delete mode 100644 tolk-tester/tests/unbalanced_ret_nested.tolk rename tolk-tester/tests/{var-apply.tolk => var-apply-tests.tolk} (85%) delete mode 100644 tolk-tester/tests/w1.tolk delete mode 100644 tolk-tester/tests/w6.tolk delete mode 100644 tolk-tester/tests/w7.tolk delete mode 100644 tolk-tester/tests/w9.tolk rename tolk-tester/tests/{ => warnings-not-errors}/unreachable-1.tolk (100%) rename tolk-tester/tests/{ => warnings-not-errors}/unreachable-2.tolk (100%) rename tolk-tester/tests/{ => warnings-not-errors}/unreachable-3.tolk (100%) rename tolk-tester/tests/{ => warnings-not-errors}/unreachable-4.tolk (100%) rename tolk-tester/tests/{ => warnings-not-errors}/warnings-1.tolk (100%) rename tolk-tester/tests/{ => warnings-not-errors}/warnings-2.tolk (100%) diff --git a/tolk-tester/tests/a6.tolk b/tolk-tester/tests/a-tests.tolk similarity index 98% rename from tolk-tester/tests/a6.tolk rename to tolk-tester/tests/a-tests.tolk index ca1737a1ea..65bff5ad11 100644 --- a/tolk-tester/tests/a6.tolk +++ b/tolk-tester/tests/a-tests.tolk @@ -76,7 +76,7 @@ fun main(): int { } /** - method_id | in | out + method_id | in | out @testcase | 0 | | 31415926535897932384626433832795028841971693993751058209749445923078164 @code_hash 84337043972311674339187056298873613816389434478842780265748859098303774481976 diff --git a/tolk-tester/tests/a6_1.tolk b/tolk-tester/tests/a6_1.tolk deleted file mode 100644 index 8079972b14..0000000000 --- a/tolk-tester/tests/a6_1.tolk +++ /dev/null @@ -1,22 +0,0 @@ -fun main(a: int, b: int, c: int, d: int, e: int, f: int): (int, int) { - var D: int = a * d - b * c; - var Dx: int = e * d - b * f; - var Dy: int = a * f - e * c; - return (Dx / D, Dy / D); -} - -@method_id(101) -fun testDivMod(x: int, y: int) { - return (divMod(x, y), modDiv(x, y), mulDivMod(x, y, 10)); -} - -/** - method_id | in | out -@testcase | 0 | 1 1 1 -1 10 6 | 8 2 -@testcase | 0 | 817 -31 624 -241 132272 272276 | 132 -788 -@testcase | 0 | -886 562 498 -212 -36452 -68958 | -505 -861 -@testcase | 0 | 448 -433 -444 792 150012 -356232 | -218 -572 -@testcase | 0 | -40 -821 433 -734 -721629 -741724 | -206 889 -@testcase | 0 | -261 -98 -494 868 -166153 733738 | 263 995 -@testcase | 101 | 112 3 | 37 1 1 37 33 6 -*/ diff --git a/tolk-tester/tests/a6_5.tolk b/tolk-tester/tests/a6_5.tolk deleted file mode 100644 index 43fd59c5a0..0000000000 --- a/tolk-tester/tests/a6_5.tolk +++ /dev/null @@ -1,26 +0,0 @@ -@deprecated -fun twice(f: int -> int, x: int) { - return f (f (x)); -} - -fun sqr(x: int) { - return x * x; -} - -fun main(x: int): int { - var f = sqr; - return twice(f, x) * f(x); -} - -@method_id(4) -fun pow6(x: int): int { - return twice(sqr, x) * sqr(x); -} - -/** - method_id | in | out -@testcase | 0 | 3 | 729 -@testcase | 0 | 10 | 1000000 -@testcase | 4 | 3 | 729 -@testcase | 4 | 10 | 1000000 -*/ diff --git a/tolk-tester/tests/a7.tolk b/tolk-tester/tests/a7.tolk deleted file mode 100644 index 1c0ae2eb36..0000000000 --- a/tolk-tester/tests/a7.tolk +++ /dev/null @@ -1,24 +0,0 @@ -fun main() { } -@method_id(1) -fun steps(x: int): int { - var n = 0; - while (x > 1) { - n += 1; - if (x & 1) { - x = 3 * x + 1; - } else { - x >>= 1; - } - } - return n; -} - -/** - method_id | in | out -@testcase | 1 | 1 | 0 -@testcase | 1 | 2 | 1 -@testcase | 1 | 5 | 5 -@testcase | 1 | 19 | 20 -@testcase | 1 | 27 | 111 -@testcase | 1 | 100 | 25 -*/ diff --git a/tolk-tester/tests/allow_post_modification.tolk b/tolk-tester/tests/allow-post-modification.tolk similarity index 100% rename from tolk-tester/tests/allow_post_modification.tolk rename to tolk-tester/tests/allow-post-modification.tolk diff --git a/tolk-tester/tests/asm_arg_order.tolk b/tolk-tester/tests/asm-arg-order.tolk similarity index 100% rename from tolk-tester/tests/asm_arg_order.tolk rename to tolk-tester/tests/asm-arg-order.tolk diff --git a/tolk-tester/tests/c2.tolk b/tolk-tester/tests/c2.tolk deleted file mode 100644 index bcbc6c9386..0000000000 --- a/tolk-tester/tests/c2.tolk +++ /dev/null @@ -1,28 +0,0 @@ -global op: (int, int) -> int; - -fun check_assoc(a: int, b: int, c: int): bool { - return op(op(a, b), c) == op(a, op(b, c)); -} - -fun unnamed_args(_: int, _: slice, _: int) { - return true; -} - -fun main(x: int, y: int, z: int): bool? { - op = `_+_`; - if (0) { return null; } - return check_assoc(x, y, z); -} - -@method_id(101) -fun test101(x: int, z: int) { - return unnamed_args(x, "asdf", z); -} - -/** - method_id | in | out -@testcase | 0 | 2 3 9 | -1 -@testcase | 0 | 11 22 44 | -1 -@testcase | 0 | -1 -10 -20 | -1 -@testcase | 101 | 1 10 | -1 -*/ diff --git a/tolk-tester/tests/c2_1.tolk b/tolk-tester/tests/c2_1.tolk deleted file mode 100644 index ef1e589ad7..0000000000 --- a/tolk-tester/tests/c2_1.tolk +++ /dev/null @@ -1,14 +0,0 @@ -fun check_assoc(op: (int, int) -> int, a: int, b: int, c: int) { - return op(op(a, b), c) == op(a, op(b, c)); -} - -fun main(x: int, y: int, z: int): bool { - return check_assoc(`_+_`, x, y, z); -} - -/** - method_id | in | out -@testcase | 0 | 2 3 9 | -1 -@testcase | 0 | 11 22 44 | -1 -@testcase | 0 | -1 -10 -20 | -1 -*/ diff --git a/tolk-tester/tests/code_after_ifelse.tolk b/tolk-tester/tests/code_after_ifelse.tolk deleted file mode 100644 index 6a16262f85..0000000000 --- a/tolk-tester/tests/code_after_ifelse.tolk +++ /dev/null @@ -1,41 +0,0 @@ -fun elseif(cond: int) { - if (cond > 0) { - throw(cond); - } -} - -@inline -@method_id(101) -fun foo(x: int): int { - if (x==1) { - return 111; - } else { - x *= 2; - } - return x + 1; -} - -fun main(x: int): (int, int) { - return (foo(x), 222); -} - -@method_id(102) -fun test2(x: int) { - try { - if (x < 0) { return -1; } - elseif (x); - } catch(excNo) { - return excNo * 1000; - } - return 0; -} - -/** - method_id | in | out -@testcase | 0 | 1 | 111 222 -@testcase | 0 | 3 | 7 222 -@testcase | 101 | 1 | 111 -@testcase | 101 | 3 | 7 -@testcase | 102 | -5 | -1 -@testcase | 102 | 5 | 5000 -*/ diff --git a/tolk-tester/tests/codegen_check_demo.tolk b/tolk-tester/tests/codegen-check-demo.tolk similarity index 100% rename from tolk-tester/tests/codegen_check_demo.tolk rename to tolk-tester/tests/codegen-check-demo.tolk diff --git a/tolk-tester/tests/co1.tolk b/tolk-tester/tests/constants-tests.tolk similarity index 100% rename from tolk-tester/tests/co1.tolk rename to tolk-tester/tests/constants-tests.tolk diff --git a/tolk-tester/tests/if_stmt.tolk b/tolk-tester/tests/if-else-tests.tolk similarity index 59% rename from tolk-tester/tests/if_stmt.tolk rename to tolk-tester/tests/if-else-tests.tolk index 0f78a51629..0f1d31f942 100644 --- a/tolk-tester/tests/if_stmt.tolk +++ b/tolk-tester/tests/if-else-tests.tolk @@ -31,6 +31,48 @@ fun test3(x: int) { return -1; } +fun elseif(cond: int) { + if (cond > 0) { + throw(cond); + } +} + +@inline +@method_id(104) +fun test4(x: int): int { + if (x==1) { + return 111; + } else { + x *= 2; + } + return x + 1; +} + +@method_id(105) +fun test5(x: int): (int, int) { + return (test4(x), 222); +} + +@method_id(106) +fun test6(x: int) { + try { + if (x < 0) { return -1; } + elseif (x); + } catch(excNo) { + return excNo * 1000; + } + return 0; +} + +fun doNothing(): void {} + +@method_id(107) +fun test7() { + if (random()) { return doNothing(); } + // here we test that no "missing return" error +} + + fun main() { } @@ -48,6 +90,12 @@ fun main() { @testcase | 103 | 20 | 20 @testcase | 103 | 40 | 50 @testcase | 103 | 50 | -1 +@testcase | 104 | 1 | 111 +@testcase | 104 | 3 | 7 +@testcase | 105 | 1 | 111 222 +@testcase | 105 | 3 | 7 222 +@testcase | 106 | -5 | -1 +@testcase | 106 | 5 | 5000 @fif_codegen """ diff --git a/tolk-tester/tests/inference-tests.tolk b/tolk-tester/tests/inference-tests.tolk index e785ac5aa4..463d4a7b7e 100644 --- a/tolk-tester/tests/inference-tests.tolk +++ b/tolk-tester/tests/inference-tests.tolk @@ -95,6 +95,15 @@ fun test7() { // __expect_type(eq<(int, slice)>, "(int, slice) -> (int, slice)"); } +fun unnamed_args(_: int, _: slice, _: int) { + return true; +} + +@method_id(108) +fun test8(x: int, z: int) { + return unnamed_args(x, "asdf", z); +} + fun alwaysThrows(): never { throw 123; } fun alwaysThrowsNotAnnotated() { throw 123; } fun alwaysThrowsNotAnnotated2() { alwaysThrows(); } @@ -112,5 +121,6 @@ fun main() { } /** -@testcase | 0 | | 0 +@testcase | 0 | | 0 +@testcase | 108 | 1 10 | -1 */ diff --git a/tolk-tester/tests/inline-tests.tolk b/tolk-tester/tests/inline-tests.tolk new file mode 100644 index 0000000000..13add753e3 --- /dev/null +++ b/tolk-tester/tests/inline-tests.tolk @@ -0,0 +1,140 @@ +fun foo1(x: int): int { + if (x == 1) { + return 1; + } + return 2; +} + +@inline +fun foo2(x: int): int { + if (x == 1) { + return 11; + } + return 22; +} + +@inline_ref +fun foo3(x: int): int { + if (x == 1) { + return 111; + } + return 222; +} + +@method_id(101) +fun test1(x: int): (int, int, int) { + return (foo1(x)+1, foo2(x)+1, foo3(x)+1); +} + +global g: int; + +@inline +fun foo_repeat() { + g = 1; + repeat(5) { + g *= 2; + } +} + +@inline +fun foo_until(): int { + g = 1; + var i: int = 0; + do { + g *= 2; + i += 1; + } while (i < 8); + return i; +} + +@inline +fun foo_while(): int { + g = 1; + var i: int = 0; + while (i < 10) { + g *= 2; + i += 1; + } + return i; +} + +@method_id(102) +fun test2() { + foo_repeat(); + var x: int = g; + foo_until(); + var y: int = g; + foo_while(); + var z: int = g; + return (x, y, z); +} + +@inline +fun foo_big(x: int): int { + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + return x; +} + +@method_id(103) +fun test3(x: int): int { + return foo_big(x) * 10 + 5; +} + +fun main() {} + +/** + method_id | in | out +@testcase | 101 | 1 | 2 12 112 +@testcase | 101 | 2 | 3 23 223 +@testcase | 102 | | 32 256 1024 +@testcase | 103 | 9 | 9111111111111111111111111111111111111111111111111115 +*/ diff --git a/tolk-tester/tests/inline_big.tolk b/tolk-tester/tests/inline_big.tolk deleted file mode 100644 index be014eb5d5..0000000000 --- a/tolk-tester/tests/inline_big.tolk +++ /dev/null @@ -1,62 +0,0 @@ -@inline -fun foo(x: int): int { - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - return x; -} - -fun main(x: int): int { - return foo(x) * 10 + 5; -} -/** - method_id | in | out -@testcase | 0 | 9 | 9111111111111111111111111111111111111111111111111115 -*/ diff --git a/tolk-tester/tests/inline_if.tolk b/tolk-tester/tests/inline_if.tolk deleted file mode 100644 index 9f1fa8c12d..0000000000 --- a/tolk-tester/tests/inline_if.tolk +++ /dev/null @@ -1,28 +0,0 @@ -fun foo1(x: int): int { - if (x == 1) { - return 1; - } - return 2; -} -@inline -fun foo2(x: int): int { - if (x == 1) { - return 11; - } - return 22; -} -@inline_ref -fun foo3(x: int): int { - if (x == 1) { - return 111; - } - return 222; -} -fun main(x: int): (int, int, int) { - return (foo1(x)+1, foo2(x)+1, foo3(x)+1); -} -/** - method_id | in | out -@testcase | 0 | 1 | 2 12 112 -@testcase | 0 | 2 | 3 23 223 -*/ diff --git a/tolk-tester/tests/inline_loops.tolk b/tolk-tester/tests/inline_loops.tolk deleted file mode 100644 index eba595a5e6..0000000000 --- a/tolk-tester/tests/inline_loops.tolk +++ /dev/null @@ -1,48 +0,0 @@ -global g: int; - -@inline -fun foo_repeat() { - g = 1; - repeat(5) { - g *= 2; - } -} - -@inline -fun foo_until(): int { - g = 1; - var i: int = 0; - do { - g *= 2; - i += 1; - } while (i < 8); - return i; -} - -@inline -fun foo_while(): int { - g = 1; - var i: int = 0; - while (i < 10) { - g *= 2; - i += 1; - } - return i; -} - -fun main() { - foo_repeat(); - var x: int = g; - foo_until(); - var y: int = g; - foo_while(); - var z: int = g; - return (x, y, z); -} - -/** - method_id | in | out -@testcase | 0 | | 32 256 1024 - -@code_hash 102749806552989901976653997041637095139193406161777448419603700344770997608788 -*/ diff --git a/tolk-tester/tests/invalid-declaration-8.tolk b/tolk-tester/tests/invalid-declaration/err-1084.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-8.tolk rename to tolk-tester/tests/invalid-declaration/err-1084.tolk diff --git a/tolk-tester/tests/invalid-declaration-16.tolk b/tolk-tester/tests/invalid-declaration/err-1153.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-16.tolk rename to tolk-tester/tests/invalid-declaration/err-1153.tolk diff --git a/tolk-tester/tests/invalid-declaration-1.tolk b/tolk-tester/tests/invalid-declaration/err-1209.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-1.tolk rename to tolk-tester/tests/invalid-declaration/err-1209.tolk diff --git a/tolk-tester/tests/invalid-declaration-10.tolk b/tolk-tester/tests/invalid-declaration/err-1361.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-10.tolk rename to tolk-tester/tests/invalid-declaration/err-1361.tolk diff --git a/tolk-tester/tests/invalid-get-method-2.tolk b/tolk-tester/tests/invalid-declaration/err-1449.tolk similarity index 100% rename from tolk-tester/tests/invalid-get-method-2.tolk rename to tolk-tester/tests/invalid-declaration/err-1449.tolk diff --git a/tolk-tester/tests/invalid-mutate-13.tolk b/tolk-tester/tests/invalid-declaration/err-1505.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-13.tolk rename to tolk-tester/tests/invalid-declaration/err-1505.tolk diff --git a/tolk-tester/tests/invalid-self-6.tolk b/tolk-tester/tests/invalid-declaration/err-1601.tolk similarity index 100% rename from tolk-tester/tests/invalid-self-6.tolk rename to tolk-tester/tests/invalid-declaration/err-1601.tolk diff --git a/tolk-tester/tests/invalid-redefinition-1.tolk b/tolk-tester/tests/invalid-declaration/err-1683.tolk similarity index 100% rename from tolk-tester/tests/invalid-redefinition-1.tolk rename to tolk-tester/tests/invalid-declaration/err-1683.tolk diff --git a/tolk-tester/tests/invalid-builtin-1.tolk b/tolk-tester/tests/invalid-declaration/err-1701.tolk similarity index 100% rename from tolk-tester/tests/invalid-builtin-1.tolk rename to tolk-tester/tests/invalid-declaration/err-1701.tolk diff --git a/tolk-tester/tests/invalid-declaration-11.tolk b/tolk-tester/tests/invalid-declaration/err-1790.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-11.tolk rename to tolk-tester/tests/invalid-declaration/err-1790.tolk diff --git a/tolk-tester/tests/invalid-declaration-6.tolk b/tolk-tester/tests/invalid-declaration/err-1794.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-6.tolk rename to tolk-tester/tests/invalid-declaration/err-1794.tolk diff --git a/tolk-tester/tests/invalid-declaration-9.tolk b/tolk-tester/tests/invalid-declaration/err-1805.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-9.tolk rename to tolk-tester/tests/invalid-declaration/err-1805.tolk diff --git a/tolk-tester/tests/invalid-cyclic-1.tolk b/tolk-tester/tests/invalid-declaration/err-1842.tolk similarity index 100% rename from tolk-tester/tests/invalid-cyclic-1.tolk rename to tolk-tester/tests/invalid-declaration/err-1842.tolk diff --git a/tolk-tester/tests/invalid-redefinition-2.tolk b/tolk-tester/tests/invalid-declaration/err-1918.tolk similarity index 78% rename from tolk-tester/tests/invalid-redefinition-2.tolk rename to tolk-tester/tests/invalid-declaration/err-1918.tolk index 3a300dc2eb..ae801de265 100644 --- a/tolk-tester/tests/invalid-redefinition-2.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1918.tolk @@ -8,5 +8,5 @@ fun hello(): int { @compilation_should_fail @stderr fun hello() @stderr redefinition of symbol, previous was at -@stderr invalid-redefinition-2.tolk:1:1 +@stderr err-1918.tolk:1:1 */ diff --git a/tolk-tester/tests/invalid-get-method-1.tolk b/tolk-tester/tests/invalid-declaration/err-1940.tolk similarity index 100% rename from tolk-tester/tests/invalid-get-method-1.tolk rename to tolk-tester/tests/invalid-declaration/err-1940.tolk diff --git a/tolk-tester/tests/invalid-no-import-1.tolk b/tolk-tester/tests/invalid-no-import-1.tolk deleted file mode 100644 index 89f879a36c..0000000000 --- a/tolk-tester/tests/invalid-no-import-1.tolk +++ /dev/null @@ -1,8 +0,0 @@ -import "imports/some-math.tolk"; -import "imports/invalid-no-import.tolk"; - -/** -@compilation_should_fail -@stderr imports/invalid-no-import.tolk:2:13 -@stderr Using a non-imported symbol `someAdd` - */ diff --git a/tolk-tester/tests/invalid-call-5.tolk b/tolk-tester/tests/invalid-semantics/err-4002.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-5.tolk rename to tolk-tester/tests/invalid-semantics/err-4002.tolk diff --git a/tolk-tester/tests/invalid-mutate-5.tolk b/tolk-tester/tests/invalid-semantics/err-4061.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-5.tolk rename to tolk-tester/tests/invalid-semantics/err-4061.tolk diff --git a/tolk-tester/tests/invalid-pure-1.tolk b/tolk-tester/tests/invalid-semantics/err-4064.tolk similarity index 100% rename from tolk-tester/tests/invalid-pure-1.tolk rename to tolk-tester/tests/invalid-semantics/err-4064.tolk diff --git a/tolk-tester/tests/invalid-call-4.tolk b/tolk-tester/tests/invalid-semantics/err-4080.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-4.tolk rename to tolk-tester/tests/invalid-semantics/err-4080.tolk diff --git a/tolk-tester/tests/invalid-mutate-16.tolk b/tolk-tester/tests/invalid-semantics/err-4083.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-16.tolk rename to tolk-tester/tests/invalid-semantics/err-4083.tolk diff --git a/tolk-tester/tests/invalid-generics-6.tolk b/tolk-tester/tests/invalid-semantics/err-4104.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-6.tolk rename to tolk-tester/tests/invalid-semantics/err-4104.tolk diff --git a/tolk-tester/tests/invalid-declaration-14.tolk b/tolk-tester/tests/invalid-semantics/err-4127.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-14.tolk rename to tolk-tester/tests/invalid-semantics/err-4127.tolk diff --git a/tolk-tester/tests/invalid-mutate-18.tolk b/tolk-tester/tests/invalid-semantics/err-4195.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-18.tolk rename to tolk-tester/tests/invalid-semantics/err-4195.tolk diff --git a/tolk-tester/tests/invalid-generics-10.tolk b/tolk-tester/tests/invalid-semantics/err-4266.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-10.tolk rename to tolk-tester/tests/invalid-semantics/err-4266.tolk diff --git a/tolk-tester/tests/invalid-mutate-7.tolk b/tolk-tester/tests/invalid-semantics/err-4377.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-7.tolk rename to tolk-tester/tests/invalid-semantics/err-4377.tolk diff --git a/tolk-tester/tests/invalid-typing-13.tolk b/tolk-tester/tests/invalid-semantics/err-4381.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-13.tolk rename to tolk-tester/tests/invalid-semantics/err-4381.tolk diff --git a/tolk-tester/tests/invalid-pure-3.tolk b/tolk-tester/tests/invalid-semantics/err-4385.tolk similarity index 100% rename from tolk-tester/tests/invalid-pure-3.tolk rename to tolk-tester/tests/invalid-semantics/err-4385.tolk diff --git a/tolk-tester/tests/invalid-pure-2.tolk b/tolk-tester/tests/invalid-semantics/err-4413.tolk similarity index 100% rename from tolk-tester/tests/invalid-pure-2.tolk rename to tolk-tester/tests/invalid-semantics/err-4413.tolk diff --git a/tolk-tester/tests/invalid-generics-5.tolk b/tolk-tester/tests/invalid-semantics/err-4416.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-5.tolk rename to tolk-tester/tests/invalid-semantics/err-4416.tolk diff --git a/tolk-tester/tests/invalid-mutate-11.tolk b/tolk-tester/tests/invalid-semantics/err-4473.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-11.tolk rename to tolk-tester/tests/invalid-semantics/err-4473.tolk diff --git a/tolk-tester/tests/invalid-mutate-10.tolk b/tolk-tester/tests/invalid-semantics/err-4506.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-10.tolk rename to tolk-tester/tests/invalid-semantics/err-4506.tolk diff --git a/tolk-tester/tests/invalid-generics-11.tolk b/tolk-tester/tests/invalid-semantics/err-4510.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-11.tolk rename to tolk-tester/tests/invalid-semantics/err-4510.tolk diff --git a/tolk-tester/tests/invalid-call-8.tolk b/tolk-tester/tests/invalid-semantics/err-4520.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-8.tolk rename to tolk-tester/tests/invalid-semantics/err-4520.tolk diff --git a/tolk-tester/tests/invalid-mutate-17.tolk b/tolk-tester/tests/invalid-semantics/err-4542.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-17.tolk rename to tolk-tester/tests/invalid-semantics/err-4542.tolk diff --git a/tolk-tester/tests/invalid-mutate-4.tolk b/tolk-tester/tests/invalid-semantics/err-4553.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-4.tolk rename to tolk-tester/tests/invalid-semantics/err-4553.tolk diff --git a/tolk-tester/tests/invalid-syntax-3.tolk b/tolk-tester/tests/invalid-semantics/err-4571.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-3.tolk rename to tolk-tester/tests/invalid-semantics/err-4571.tolk diff --git a/tolk-tester/tests/invalid-generics-9.tolk b/tolk-tester/tests/invalid-semantics/err-4604.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-9.tolk rename to tolk-tester/tests/invalid-semantics/err-4604.tolk diff --git a/tolk-tester/tests/invalid-syntax-6.tolk b/tolk-tester/tests/invalid-semantics/err-4639.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-6.tolk rename to tolk-tester/tests/invalid-semantics/err-4639.tolk diff --git a/tolk-tester/tests/invalid-mutate-12.tolk b/tolk-tester/tests/invalid-semantics/err-4705.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-12.tolk rename to tolk-tester/tests/invalid-semantics/err-4705.tolk diff --git a/tolk-tester/tests/invalid-mutate-1.tolk b/tolk-tester/tests/invalid-semantics/err-4728.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-1.tolk rename to tolk-tester/tests/invalid-semantics/err-4728.tolk diff --git a/tolk-tester/tests/invalid-call-3.tolk b/tolk-tester/tests/invalid-semantics/err-4751.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-3.tolk rename to tolk-tester/tests/invalid-semantics/err-4751.tolk diff --git a/tolk-tester/tests/invalid-call-11.tolk b/tolk-tester/tests/invalid-semantics/err-4804.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-11.tolk rename to tolk-tester/tests/invalid-semantics/err-4804.tolk diff --git a/tolk-tester/tests/invalid-mutate-3.tolk b/tolk-tester/tests/invalid-semantics/err-4819.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-3.tolk rename to tolk-tester/tests/invalid-semantics/err-4819.tolk diff --git a/tolk-tester/tests/invalid-mutate-15.tolk b/tolk-tester/tests/invalid-semantics/err-4828.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-15.tolk rename to tolk-tester/tests/invalid-semantics/err-4828.tolk diff --git a/tolk-tester/tests/invalid-mutate-6.tolk b/tolk-tester/tests/invalid-semantics/err-4838.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-6.tolk rename to tolk-tester/tests/invalid-semantics/err-4838.tolk diff --git a/tolk-tester/tests/invalid-call-6.tolk b/tolk-tester/tests/invalid-semantics/err-4841.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-6.tolk rename to tolk-tester/tests/invalid-semantics/err-4841.tolk diff --git a/tolk-tester/tests/invalid-mutate-9.tolk b/tolk-tester/tests/invalid-semantics/err-4851.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-9.tolk rename to tolk-tester/tests/invalid-semantics/err-4851.tolk diff --git a/tolk-tester/tests/invalid-generics-1.tolk b/tolk-tester/tests/invalid-semantics/err-4880.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-1.tolk rename to tolk-tester/tests/invalid-semantics/err-4880.tolk diff --git a/tolk-tester/tests/invalid-mutate-14.tolk b/tolk-tester/tests/invalid-semantics/err-4899.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-14.tolk rename to tolk-tester/tests/invalid-semantics/err-4899.tolk diff --git a/tolk-tester/tests/invalid-mutate-2.tolk b/tolk-tester/tests/invalid-semantics/err-4912.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-2.tolk rename to tolk-tester/tests/invalid-semantics/err-4912.tolk diff --git a/tolk-tester/tests/invalid-mutate-19.tolk b/tolk-tester/tests/invalid-semantics/err-4941.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-19.tolk rename to tolk-tester/tests/invalid-semantics/err-4941.tolk diff --git a/tolk-tester/tests/invalid-call-9.tolk b/tolk-tester/tests/invalid-semantics/err-4951.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-9.tolk rename to tolk-tester/tests/invalid-semantics/err-4951.tolk diff --git a/tolk-tester/tests/invalid-call-2.tolk b/tolk-tester/tests/invalid-semantics/err-4956.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-2.tolk rename to tolk-tester/tests/invalid-semantics/err-4956.tolk diff --git a/tolk-tester/tests/invalid-mutate-8.tolk b/tolk-tester/tests/invalid-symbol/err-2055.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-8.tolk rename to tolk-tester/tests/invalid-symbol/err-2055.tolk diff --git a/tolk-tester/tests/invalid-call-7.tolk b/tolk-tester/tests/invalid-symbol/err-2188.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-7.tolk rename to tolk-tester/tests/invalid-symbol/err-2188.tolk diff --git a/tolk-tester/tests/invalid-symbol/err-2205.tolk b/tolk-tester/tests/invalid-symbol/err-2205.tolk new file mode 100644 index 0000000000..3c9d72204f --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2205.tolk @@ -0,0 +1,8 @@ +import "../imports/some-math.tolk"; +import "../imports/invalid-no-import.tolk"; + +/** +@compilation_should_fail +@stderr ../imports/invalid-no-import.tolk:2:13 +@stderr Using a non-imported symbol `someAdd` + */ diff --git a/tolk-tester/tests/invalid-typing-32.tolk b/tolk-tester/tests/invalid-symbol/err-2255.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-32.tolk rename to tolk-tester/tests/invalid-symbol/err-2255.tolk diff --git a/tolk-tester/tests/invalid-symbol-1.tolk b/tolk-tester/tests/invalid-symbol/err-2374.tolk similarity index 100% rename from tolk-tester/tests/invalid-symbol-1.tolk rename to tolk-tester/tests/invalid-symbol/err-2374.tolk diff --git a/tolk-tester/tests/invalid-symbol-2.tolk b/tolk-tester/tests/invalid-symbol/err-2463.tolk similarity index 100% rename from tolk-tester/tests/invalid-symbol-2.tolk rename to tolk-tester/tests/invalid-symbol/err-2463.tolk diff --git a/tolk-tester/tests/invalid-redefinition-6.tolk b/tolk-tester/tests/invalid-symbol/err-2501.tolk similarity index 100% rename from tolk-tester/tests/invalid-redefinition-6.tolk rename to tolk-tester/tests/invalid-symbol/err-2501.tolk diff --git a/tolk-tester/tests/invalid-self-7.tolk b/tolk-tester/tests/invalid-symbol/err-2520.tolk similarity index 100% rename from tolk-tester/tests/invalid-self-7.tolk rename to tolk-tester/tests/invalid-symbol/err-2520.tolk diff --git a/tolk-tester/tests/invalid-no-import-2.tolk b/tolk-tester/tests/invalid-symbol/err-2539.tolk similarity index 66% rename from tolk-tester/tests/invalid-no-import-2.tolk rename to tolk-tester/tests/invalid-symbol/err-2539.tolk index d78346b905..f7c66c2582 100644 --- a/tolk-tester/tests/invalid-no-import-2.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2539.tolk @@ -1,9 +1,9 @@ import "@stdlib/tvm-dicts" -import "imports/use-dicts-err.tolk" +import "../imports/use-dicts-err.tolk" /** @compilation_should_fail -@stderr imports/use-dicts-err.tolk:2:22 +@stderr ../imports/use-dicts-err.tolk:2:22 @stderr Using a non-imported symbol `createEmptyDict` @stderr Forgot to import "@stdlib/tvm-dicts"? */ diff --git a/tolk-tester/tests/invalid-redefinition-3.tolk b/tolk-tester/tests/invalid-symbol/err-2649.tolk similarity index 100% rename from tolk-tester/tests/invalid-redefinition-3.tolk rename to tolk-tester/tests/invalid-symbol/err-2649.tolk diff --git a/tolk-tester/tests/invalid-self-3.tolk b/tolk-tester/tests/invalid-symbol/err-2673.tolk similarity index 100% rename from tolk-tester/tests/invalid-self-3.tolk rename to tolk-tester/tests/invalid-symbol/err-2673.tolk diff --git a/tolk-tester/tests/invalid-redefinition-5.tolk b/tolk-tester/tests/invalid-symbol/err-2715.tolk similarity index 100% rename from tolk-tester/tests/invalid-redefinition-5.tolk rename to tolk-tester/tests/invalid-symbol/err-2715.tolk diff --git a/tolk-tester/tests/invalid-typing-1.tolk b/tolk-tester/tests/invalid-symbol/err-2750.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-1.tolk rename to tolk-tester/tests/invalid-symbol/err-2750.tolk diff --git a/tolk-tester/tests/invalid-declaration-15.tolk b/tolk-tester/tests/invalid-symbol/err-2804.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-15.tolk rename to tolk-tester/tests/invalid-symbol/err-2804.tolk diff --git a/tolk-tester/tests/invalid-redefinition-4.tolk b/tolk-tester/tests/invalid-symbol/err-2853.tolk similarity index 100% rename from tolk-tester/tests/invalid-redefinition-4.tolk rename to tolk-tester/tests/invalid-symbol/err-2853.tolk diff --git a/tolk-tester/tests/invalid-call-1.tolk b/tolk-tester/tests/invalid-symbol/err-2914.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-1.tolk rename to tolk-tester/tests/invalid-symbol/err-2914.tolk diff --git a/tolk-tester/tests/invalid-import.tolk b/tolk-tester/tests/invalid-symbol/err-2980.tolk similarity index 73% rename from tolk-tester/tests/invalid-import.tolk rename to tolk-tester/tests/invalid-symbol/err-2980.tolk index 416764b62d..de29cc6578 100644 --- a/tolk-tester/tests/invalid-import.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2980.tolk @@ -4,8 +4,8 @@ /** @compilation_should_fail -On Linux/Mac, `realpath()` returns an error, and the error message is `cannot find file` +On Linux/Mac, `realpath()` returns an error, and the error message is "cannot find file" On Windows, it fails after, on reading, with a message "cannot open file" -@stderr invalid-import.tolk:2:7: error: Failed to import: cannot +@stderr err-2980.tolk:2:7: error: Failed to import: cannot @stderr import "unexisting.tolk"; */ diff --git a/tolk-tester/tests/invalid-tolk-version.tolk b/tolk-tester/tests/invalid-syntax/err-3001.tolk similarity index 100% rename from tolk-tester/tests/invalid-tolk-version.tolk rename to tolk-tester/tests/invalid-syntax/err-3001.tolk diff --git a/tolk-tester/tests/invalid-call-13.tolk b/tolk-tester/tests/invalid-syntax/err-3015.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-13.tolk rename to tolk-tester/tests/invalid-syntax/err-3015.tolk diff --git a/tolk-tester/tests/invalid-bitwise-4.tolk b/tolk-tester/tests/invalid-syntax/err-3035.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-4.tolk rename to tolk-tester/tests/invalid-syntax/err-3035.tolk diff --git a/tolk-tester/tests/invalid-nopar-1.tolk b/tolk-tester/tests/invalid-syntax/err-3041.tolk similarity index 100% rename from tolk-tester/tests/invalid-nopar-1.tolk rename to tolk-tester/tests/invalid-syntax/err-3041.tolk diff --git a/tolk-tester/tests/invalid-shift-1.tolk b/tolk-tester/tests/invalid-syntax/err-3044.tolk similarity index 100% rename from tolk-tester/tests/invalid-shift-1.tolk rename to tolk-tester/tests/invalid-syntax/err-3044.tolk diff --git a/tolk-tester/tests/invalid-declaration-4.tolk b/tolk-tester/tests/invalid-syntax/err-3059.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-4.tolk rename to tolk-tester/tests/invalid-syntax/err-3059.tolk diff --git a/tolk-tester/tests/invalid-bitwise-3.tolk b/tolk-tester/tests/invalid-syntax/err-3102.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-3.tolk rename to tolk-tester/tests/invalid-syntax/err-3102.tolk diff --git a/tolk-tester/tests/invalid-catch-1.tolk b/tolk-tester/tests/invalid-syntax/err-3164.tolk similarity index 100% rename from tolk-tester/tests/invalid-catch-1.tolk rename to tolk-tester/tests/invalid-syntax/err-3164.tolk diff --git a/tolk-tester/tests/invalid-nopar-2.tolk b/tolk-tester/tests/invalid-syntax/err-3183.tolk similarity index 100% rename from tolk-tester/tests/invalid-nopar-2.tolk rename to tolk-tester/tests/invalid-syntax/err-3183.tolk diff --git a/tolk-tester/tests/invalid-nopar-4.tolk b/tolk-tester/tests/invalid-syntax/err-3197.tolk similarity index 100% rename from tolk-tester/tests/invalid-nopar-4.tolk rename to tolk-tester/tests/invalid-syntax/err-3197.tolk diff --git a/tolk-tester/tests/invalid-syntax-7.tolk b/tolk-tester/tests/invalid-syntax/err-3205.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-7.tolk rename to tolk-tester/tests/invalid-syntax/err-3205.tolk diff --git a/tolk-tester/tests/invalid-nopar-3.tolk b/tolk-tester/tests/invalid-syntax/err-3274.tolk similarity index 100% rename from tolk-tester/tests/invalid-nopar-3.tolk rename to tolk-tester/tests/invalid-syntax/err-3274.tolk diff --git a/tolk-tester/tests/invalid-const-1.tolk b/tolk-tester/tests/invalid-syntax/err-3295.tolk similarity index 100% rename from tolk-tester/tests/invalid-const-1.tolk rename to tolk-tester/tests/invalid-syntax/err-3295.tolk diff --git a/tolk-tester/tests/invalid-cmt-old.tolk b/tolk-tester/tests/invalid-syntax/err-3390.tolk similarity index 100% rename from tolk-tester/tests/invalid-cmt-old.tolk rename to tolk-tester/tests/invalid-syntax/err-3390.tolk diff --git a/tolk-tester/tests/invalid-cmt-nested.tolk b/tolk-tester/tests/invalid-syntax/err-3407.tolk similarity index 100% rename from tolk-tester/tests/invalid-cmt-nested.tolk rename to tolk-tester/tests/invalid-syntax/err-3407.tolk diff --git a/tolk-tester/tests/invalid-call-14.tolk b/tolk-tester/tests/invalid-syntax/err-3472.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-14.tolk rename to tolk-tester/tests/invalid-syntax/err-3472.tolk diff --git a/tolk-tester/tests/invalid-syntax-5.tolk b/tolk-tester/tests/invalid-syntax/err-3527.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-5.tolk rename to tolk-tester/tests/invalid-syntax/err-3527.tolk diff --git a/tolk-tester/tests/invalid-syntax-1.tolk b/tolk-tester/tests/invalid-syntax/err-3551.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-1.tolk rename to tolk-tester/tests/invalid-syntax/err-3551.tolk diff --git a/tolk-tester/tests/invalid-catch-2.tolk b/tolk-tester/tests/invalid-syntax/err-3586.tolk similarity index 100% rename from tolk-tester/tests/invalid-catch-2.tolk rename to tolk-tester/tests/invalid-syntax/err-3586.tolk diff --git a/tolk-tester/tests/invalid-bitwise-5.tolk b/tolk-tester/tests/invalid-syntax/err-3588.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-5.tolk rename to tolk-tester/tests/invalid-syntax/err-3588.tolk diff --git a/tolk-tester/tests/invalid-declaration-7.tolk b/tolk-tester/tests/invalid-syntax/err-3603.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-7.tolk rename to tolk-tester/tests/invalid-syntax/err-3603.tolk diff --git a/tolk-tester/tests/invalid-syntax/err-3618.tolk b/tolk-tester/tests/invalid-syntax/err-3618.tolk new file mode 100644 index 0000000000..7751c4a95f --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3618.tolk @@ -0,0 +1,6 @@ +const x = 0b0012; + +/** +@compilation_should_fail +@stderr expected `;`, got `2` + */ diff --git a/tolk-tester/tests/invalid-call-12.tolk b/tolk-tester/tests/invalid-syntax/err-3714.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-12.tolk rename to tolk-tester/tests/invalid-syntax/err-3714.tolk diff --git a/tolk-tester/tests/invalid-syntax-4.tolk b/tolk-tester/tests/invalid-syntax/err-3719.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-4.tolk rename to tolk-tester/tests/invalid-syntax/err-3719.tolk diff --git a/tolk-tester/tests/invalid-declaration-5.tolk b/tolk-tester/tests/invalid-syntax/err-3781.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-5.tolk rename to tolk-tester/tests/invalid-syntax/err-3781.tolk diff --git a/tolk-tester/tests/invalid-bitwise-1.tolk b/tolk-tester/tests/invalid-syntax/err-3819.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-1.tolk rename to tolk-tester/tests/invalid-syntax/err-3819.tolk diff --git a/tolk-tester/tests/invalid-bitwise-7.tolk b/tolk-tester/tests/invalid-syntax/err-3851.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-7.tolk rename to tolk-tester/tests/invalid-syntax/err-3851.tolk diff --git a/tolk-tester/tests/invalid-declaration-2.tolk b/tolk-tester/tests/invalid-syntax/err-3853.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-2.tolk rename to tolk-tester/tests/invalid-syntax/err-3853.tolk diff --git a/tolk-tester/tests/invalid-syntax-2.tolk b/tolk-tester/tests/invalid-syntax/err-3898.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-2.tolk rename to tolk-tester/tests/invalid-syntax/err-3898.tolk diff --git a/tolk-tester/tests/invalid-bitwise-2.tolk b/tolk-tester/tests/invalid-syntax/err-3910.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-2.tolk rename to tolk-tester/tests/invalid-syntax/err-3910.tolk diff --git a/tolk-tester/tests/invalid-declaration-3.tolk b/tolk-tester/tests/invalid-syntax/err-3967.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-3.tolk rename to tolk-tester/tests/invalid-syntax/err-3967.tolk diff --git a/tolk-tester/tests/invalid-bitwise-6.tolk b/tolk-tester/tests/invalid-syntax/err-3992.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-6.tolk rename to tolk-tester/tests/invalid-syntax/err-3992.tolk diff --git a/tolk-tester/tests/invalid-assign-1.tolk b/tolk-tester/tests/invalid-typing/err-6039.tolk similarity index 100% rename from tolk-tester/tests/invalid-assign-1.tolk rename to tolk-tester/tests/invalid-typing/err-6039.tolk diff --git a/tolk-tester/tests/invalid-assign-3.tolk b/tolk-tester/tests/invalid-typing/err-6040.tolk similarity index 100% rename from tolk-tester/tests/invalid-assign-3.tolk rename to tolk-tester/tests/invalid-typing/err-6040.tolk diff --git a/tolk-tester/tests/invalid-call-10.tolk b/tolk-tester/tests/invalid-typing/err-6041.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-10.tolk rename to tolk-tester/tests/invalid-typing/err-6041.tolk diff --git a/tolk-tester/tests/invalid-typing-18.tolk b/tolk-tester/tests/invalid-typing/err-6080.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-18.tolk rename to tolk-tester/tests/invalid-typing/err-6080.tolk diff --git a/tolk-tester/tests/invalid-generics-14.tolk b/tolk-tester/tests/invalid-typing/err-6085.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-14.tolk rename to tolk-tester/tests/invalid-typing/err-6085.tolk diff --git a/tolk-tester/tests/invalid-typing-43.tolk b/tolk-tester/tests/invalid-typing/err-6087.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-43.tolk rename to tolk-tester/tests/invalid-typing/err-6087.tolk diff --git a/tolk-tester/tests/invalid-generics-4.tolk b/tolk-tester/tests/invalid-typing/err-6095.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-4.tolk rename to tolk-tester/tests/invalid-typing/err-6095.tolk diff --git a/tolk-tester/tests/invalid-typing-20.tolk b/tolk-tester/tests/invalid-typing/err-6114.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-20.tolk rename to tolk-tester/tests/invalid-typing/err-6114.tolk diff --git a/tolk-tester/tests/invalid-typing-44.tolk b/tolk-tester/tests/invalid-typing/err-6130.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-44.tolk rename to tolk-tester/tests/invalid-typing/err-6130.tolk diff --git a/tolk-tester/tests/invalid-typing-4.tolk b/tolk-tester/tests/invalid-typing/err-6134.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-4.tolk rename to tolk-tester/tests/invalid-typing/err-6134.tolk diff --git a/tolk-tester/tests/invalid-typing-37.tolk b/tolk-tester/tests/invalid-typing/err-6135.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-37.tolk rename to tolk-tester/tests/invalid-typing/err-6135.tolk diff --git a/tolk-tester/tests/invalid-typing-25.tolk b/tolk-tester/tests/invalid-typing/err-6148.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-25.tolk rename to tolk-tester/tests/invalid-typing/err-6148.tolk diff --git a/tolk-tester/tests/invalid-typing-10.tolk b/tolk-tester/tests/invalid-typing/err-6188.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-10.tolk rename to tolk-tester/tests/invalid-typing/err-6188.tolk diff --git a/tolk-tester/tests/invalid-declaration-13.tolk b/tolk-tester/tests/invalid-typing/err-6200.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-13.tolk rename to tolk-tester/tests/invalid-typing/err-6200.tolk diff --git a/tolk-tester/tests/invalid-typing-38.tolk b/tolk-tester/tests/invalid-typing/err-6204.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-38.tolk rename to tolk-tester/tests/invalid-typing/err-6204.tolk diff --git a/tolk-tester/tests/invalid-typing-24.tolk b/tolk-tester/tests/invalid-typing/err-6220.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-24.tolk rename to tolk-tester/tests/invalid-typing/err-6220.tolk diff --git a/tolk-tester/tests/invalid-typing-40.tolk b/tolk-tester/tests/invalid-typing/err-6249.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-40.tolk rename to tolk-tester/tests/invalid-typing/err-6249.tolk diff --git a/tolk-tester/tests/invalid-self-5.tolk b/tolk-tester/tests/invalid-typing/err-6257.tolk similarity index 100% rename from tolk-tester/tests/invalid-self-5.tolk rename to tolk-tester/tests/invalid-typing/err-6257.tolk diff --git a/tolk-tester/tests/invalid-self-1.tolk b/tolk-tester/tests/invalid-typing/err-6288.tolk similarity index 100% rename from tolk-tester/tests/invalid-self-1.tolk rename to tolk-tester/tests/invalid-typing/err-6288.tolk diff --git a/tolk-tester/tests/invalid-typing-16.tolk b/tolk-tester/tests/invalid-typing/err-6337.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-16.tolk rename to tolk-tester/tests/invalid-typing/err-6337.tolk diff --git a/tolk-tester/tests/invalid-typing-45.tolk b/tolk-tester/tests/invalid-typing/err-6368.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-45.tolk rename to tolk-tester/tests/invalid-typing/err-6368.tolk diff --git a/tolk-tester/tests/invalid-generics-3.tolk b/tolk-tester/tests/invalid-typing/err-6369.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-3.tolk rename to tolk-tester/tests/invalid-typing/err-6369.tolk diff --git a/tolk-tester/tests/invalid-typing-23.tolk b/tolk-tester/tests/invalid-typing/err-6370.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-23.tolk rename to tolk-tester/tests/invalid-typing/err-6370.tolk diff --git a/tolk-tester/tests/invalid-generics-2.tolk b/tolk-tester/tests/invalid-typing/err-6386.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-2.tolk rename to tolk-tester/tests/invalid-typing/err-6386.tolk diff --git a/tolk-tester/tests/invalid-mutate-20.tolk b/tolk-tester/tests/invalid-typing/err-6391.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-20.tolk rename to tolk-tester/tests/invalid-typing/err-6391.tolk diff --git a/tolk-tester/tests/invalid-typing-9.tolk b/tolk-tester/tests/invalid-typing/err-6393.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-9.tolk rename to tolk-tester/tests/invalid-typing/err-6393.tolk diff --git a/tolk-tester/tests/invalid-typing-34.tolk b/tolk-tester/tests/invalid-typing/err-6403.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-34.tolk rename to tolk-tester/tests/invalid-typing/err-6403.tolk diff --git a/tolk-tester/tests/invalid-never-1.tolk b/tolk-tester/tests/invalid-typing/err-6407.tolk similarity index 100% rename from tolk-tester/tests/invalid-never-1.tolk rename to tolk-tester/tests/invalid-typing/err-6407.tolk diff --git a/tolk-tester/tests/invalid-typing-2.tolk b/tolk-tester/tests/invalid-typing/err-6450.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-2.tolk rename to tolk-tester/tests/invalid-typing/err-6450.tolk diff --git a/tolk-tester/tests/invalid-typing-29.tolk b/tolk-tester/tests/invalid-typing/err-6496.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-29.tolk rename to tolk-tester/tests/invalid-typing/err-6496.tolk diff --git a/tolk-tester/tests/invalid-typing-22.tolk b/tolk-tester/tests/invalid-typing/err-6499.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-22.tolk rename to tolk-tester/tests/invalid-typing/err-6499.tolk diff --git a/tolk-tester/tests/invalid-generics-13.tolk b/tolk-tester/tests/invalid-typing/err-6509.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-13.tolk rename to tolk-tester/tests/invalid-typing/err-6509.tolk diff --git a/tolk-tester/tests/invalid-typing-5.tolk b/tolk-tester/tests/invalid-typing/err-6519.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-5.tolk rename to tolk-tester/tests/invalid-typing/err-6519.tolk diff --git a/tolk-tester/tests/invalid-self-2.tolk b/tolk-tester/tests/invalid-typing/err-6533.tolk similarity index 100% rename from tolk-tester/tests/invalid-self-2.tolk rename to tolk-tester/tests/invalid-typing/err-6533.tolk diff --git a/tolk-tester/tests/invalid-typing-26.tolk b/tolk-tester/tests/invalid-typing/err-6553.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-26.tolk rename to tolk-tester/tests/invalid-typing/err-6553.tolk diff --git a/tolk-tester/tests/invalid-typing-21.tolk b/tolk-tester/tests/invalid-typing/err-6580.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-21.tolk rename to tolk-tester/tests/invalid-typing/err-6580.tolk diff --git a/tolk-tester/tests/invalid-typing-8.tolk b/tolk-tester/tests/invalid-typing/err-6600.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-8.tolk rename to tolk-tester/tests/invalid-typing/err-6600.tolk diff --git a/tolk-tester/tests/invalid-typing-31.tolk b/tolk-tester/tests/invalid-typing/err-6613.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-31.tolk rename to tolk-tester/tests/invalid-typing/err-6613.tolk diff --git a/tolk-tester/tests/invalid-typing-35.tolk b/tolk-tester/tests/invalid-typing/err-6619.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-35.tolk rename to tolk-tester/tests/invalid-typing/err-6619.tolk diff --git a/tolk-tester/tests/invalid-generics-8.tolk b/tolk-tester/tests/invalid-typing/err-6629.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-8.tolk rename to tolk-tester/tests/invalid-typing/err-6629.tolk diff --git a/tolk-tester/tests/invalid-typing-14.tolk b/tolk-tester/tests/invalid-typing/err-6637.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-14.tolk rename to tolk-tester/tests/invalid-typing/err-6637.tolk diff --git a/tolk-tester/tests/invalid-typing-11.tolk b/tolk-tester/tests/invalid-typing/err-6711.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-11.tolk rename to tolk-tester/tests/invalid-typing/err-6711.tolk diff --git a/tolk-tester/tests/invalid-typing-27.tolk b/tolk-tester/tests/invalid-typing/err-6713.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-27.tolk rename to tolk-tester/tests/invalid-typing/err-6713.tolk diff --git a/tolk-tester/tests/invalid-assign-2.tolk b/tolk-tester/tests/invalid-typing/err-6731.tolk similarity index 100% rename from tolk-tester/tests/invalid-assign-2.tolk rename to tolk-tester/tests/invalid-typing/err-6731.tolk diff --git a/tolk-tester/tests/invalid-typing-12.tolk b/tolk-tester/tests/invalid-typing/err-6734.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-12.tolk rename to tolk-tester/tests/invalid-typing/err-6734.tolk diff --git a/tolk-tester/tests/invalid-typing-33.tolk b/tolk-tester/tests/invalid-typing/err-6737.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-33.tolk rename to tolk-tester/tests/invalid-typing/err-6737.tolk diff --git a/tolk-tester/tests/invalid-typing-6.tolk b/tolk-tester/tests/invalid-typing/err-6743.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-6.tolk rename to tolk-tester/tests/invalid-typing/err-6743.tolk diff --git a/tolk-tester/tests/invalid-typing-41.tolk b/tolk-tester/tests/invalid-typing/err-6803.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-41.tolk rename to tolk-tester/tests/invalid-typing/err-6803.tolk diff --git a/tolk-tester/tests/invalid-typing-36.tolk b/tolk-tester/tests/invalid-typing/err-6810.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-36.tolk rename to tolk-tester/tests/invalid-typing/err-6810.tolk diff --git a/tolk-tester/tests/invalid-typing-39.tolk b/tolk-tester/tests/invalid-typing/err-6820.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-39.tolk rename to tolk-tester/tests/invalid-typing/err-6820.tolk diff --git a/tolk-tester/tests/invalid-typing-3.tolk b/tolk-tester/tests/invalid-typing/err-6839.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-3.tolk rename to tolk-tester/tests/invalid-typing/err-6839.tolk diff --git a/tolk-tester/tests/invalid-typing-15.tolk b/tolk-tester/tests/invalid-typing/err-6840.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-15.tolk rename to tolk-tester/tests/invalid-typing/err-6840.tolk diff --git a/tolk-tester/tests/invalid-typing-28.tolk b/tolk-tester/tests/invalid-typing/err-6841.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-28.tolk rename to tolk-tester/tests/invalid-typing/err-6841.tolk diff --git a/tolk-tester/tests/invalid-self-4.tolk b/tolk-tester/tests/invalid-typing/err-6844.tolk similarity index 100% rename from tolk-tester/tests/invalid-self-4.tolk rename to tolk-tester/tests/invalid-typing/err-6844.tolk diff --git a/tolk-tester/tests/invalid-typing-17.tolk b/tolk-tester/tests/invalid-typing/err-6881.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-17.tolk rename to tolk-tester/tests/invalid-typing/err-6881.tolk diff --git a/tolk-tester/tests/invalid-typing-7.tolk b/tolk-tester/tests/invalid-typing/err-6901.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-7.tolk rename to tolk-tester/tests/invalid-typing/err-6901.tolk diff --git a/tolk-tester/tests/invalid-typing-19.tolk b/tolk-tester/tests/invalid-typing/err-6903.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-19.tolk rename to tolk-tester/tests/invalid-typing/err-6903.tolk diff --git a/tolk-tester/tests/invalid-generics-7.tolk b/tolk-tester/tests/invalid-typing/err-6921.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-7.tolk rename to tolk-tester/tests/invalid-typing/err-6921.tolk diff --git a/tolk-tester/tests/invalid-declaration-12.tolk b/tolk-tester/tests/invalid-typing/err-6925.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-12.tolk rename to tolk-tester/tests/invalid-typing/err-6925.tolk diff --git a/tolk-tester/tests/invalid-typing-42.tolk b/tolk-tester/tests/invalid-typing/err-6937.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-42.tolk rename to tolk-tester/tests/invalid-typing/err-6937.tolk diff --git a/tolk-tester/tests/invalid-generics-12.tolk b/tolk-tester/tests/invalid-typing/err-6981.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-12.tolk rename to tolk-tester/tests/invalid-typing/err-6981.tolk diff --git a/tolk-tester/tests/invalid-typing-30.tolk b/tolk-tester/tests/invalid-typing/err-6992.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-30.tolk rename to tolk-tester/tests/invalid-typing/err-6992.tolk diff --git a/tolk-tester/tests/method_id.tolk b/tolk-tester/tests/method_id.tolk deleted file mode 100644 index e7e70d245d..0000000000 --- a/tolk-tester/tests/method_id.tolk +++ /dev/null @@ -1,17 +0,0 @@ -@method_id(1) -fun foo1(): int { return 111; } -@method_id(3) -fun foo2(): int { return 222; } -@method_id(10) -fun foo3(): int { return 333; } -@method_id(11) -fun slice(slice: slice): slice { return slice; } -fun main(): int { return 999; } - -/** - method_id | in | out -@testcase | 1 | | 111 -@testcase | 3 | | 222 -@testcase | 10 | | 333 -@testcase | 0 | | 999 -*/ diff --git a/tolk-tester/tests/some-tests-1.tolk b/tolk-tester/tests/some-tests-1.tolk new file mode 100644 index 0000000000..42bdc773a1 --- /dev/null +++ b/tolk-tester/tests/some-tests-1.tolk @@ -0,0 +1,127 @@ +fun main(a: int, b: int, c: int, d: int, e: int, f: int): (int, int) { + var D: int = a * d - b * c; + var Dx: int = e * d - b * f; + var Dy: int = a * f - e * c; + return (Dx / D, Dy / D); +} + +@method_id(101) +fun testDivMod(x: int, y: int) { + return (divMod(x, y), modDiv(x, y), mulDivMod(x, y, 10)); +} + +@deprecated +fun twice(f: (int) -> int, x: int) { + return f (f (x)); +} + +fun sqr(x: int) { + return x * x; +} + +@method_id(102) +fun testCallVar(x: int): int { + var f = sqr; + return twice(f, x) * f(x); +} + +@method_id(103) +fun testCallDirect(x: int): int { + return twice(sqr, x) * sqr(x); +} + +@method_id(104) +fun testLoop(x: int): int { + var n = 0; + while (x > 1) { + n += 1; + if (x & 1) { + x = 3 * x + 1; + } else { + x >>= 1; + } + } + return n; +} + +@method_id(105) +fun test5(id: int): (int, int) { + if (id > 0) { + if (id > 10) { + return (2 * id, 3 * id); + } + } + return (5, 6); +} + +@method_id(106) +fun test6(x: int): int { + var i: int = 0; + do { + i = i + 1; + if (i > 5) { + return 1; + } + var f: bool = (i * i == 64); + } while (!f); + return -1; +} + +@method_id(107) +fun test7(y: int): int { + var x: int = 1; + if (y > 0) { + return 1; + } + return x > 0 ? -1 : 0; +} + +@method_id(108) +fun test8(y: int): int { + if (y > 0) { + return 1; + } + return 2; +} + +@method_id(109) +fun test9(s: int) { + var (z, t) = (17, s); + while (z > 0) { + t = s; + z -= 1; + } + return ~ t; +} + + +/** + method_id | in | out +@testcase | 0 | 1 1 1 -1 10 6 | 8 2 +@testcase | 0 | 817 -31 624 -241 132272 272276 | 132 -788 +@testcase | 0 | -886 562 498 -212 -36452 -68958 | -505 -861 +@testcase | 0 | 448 -433 -444 792 150012 -356232 | -218 -572 +@testcase | 0 | -40 -821 433 -734 -721629 -741724 | -206 889 +@testcase | 0 | -261 -98 -494 868 -166153 733738 | 263 995 +@testcase | 101 | 112 3 | 37 1 1 37 33 6 +@testcase | 102 | 3 | 729 +@testcase | 102 | 10 | 1000000 +@testcase | 103 | 3 | 729 +@testcase | 103 | 10 | 1000000 +@testcase | 104 | 1 | 0 +@testcase | 104 | 2 | 1 +@testcase | 104 | 5 | 5 +@testcase | 104 | 19 | 20 +@testcase | 104 | 27 | 111 +@testcase | 104 | 100 | 25 +@testcase | 105 | 0 | 5 6 +@testcase | 105 | 4 | 5 6 +@testcase | 105 | 11 | 22 33 +@testcase | 106 | 0 | 1 +@testcase | 107 | 10 | 1 +@testcase | 107 | -5 | -1 +@testcase | 108 | 10 | 1 +@testcase | 108 | -5 | 2 +@testcase | 109 | 1 | -2 +@testcase | 109 | 5 | -6 +*/ diff --git a/tolk-tester/tests/a10.tolk b/tolk-tester/tests/some-tests-2.tolk similarity index 100% rename from tolk-tester/tests/a10.tolk rename to tolk-tester/tests/some-tests-2.tolk diff --git a/tolk-tester/tests/w2.tolk b/tolk-tester/tests/some-tests-3.tolk similarity index 100% rename from tolk-tester/tests/w2.tolk rename to tolk-tester/tests/some-tests-3.tolk diff --git a/tolk-tester/tests/s1.tolk b/tolk-tester/tests/strings-tests.tolk similarity index 100% rename from tolk-tester/tests/s1.tolk rename to tolk-tester/tests/strings-tests.tolk diff --git a/tolk-tester/tests/try-func.tolk b/tolk-tester/tests/try-catch-tests.tolk similarity index 100% rename from tolk-tester/tests/try-func.tolk rename to tolk-tester/tests/try-catch-tests.tolk diff --git a/tolk-tester/tests/unbalanced_ret_loops.tolk b/tolk-tester/tests/unbalanced-ret-loops.tolk similarity index 100% rename from tolk-tester/tests/unbalanced_ret_loops.tolk rename to tolk-tester/tests/unbalanced-ret-loops.tolk diff --git a/tolk-tester/tests/unbalanced-ret.tolk b/tolk-tester/tests/unbalanced-ret.tolk new file mode 100644 index 0000000000..eca5dd2a58 --- /dev/null +++ b/tolk-tester/tests/unbalanced-ret.tolk @@ -0,0 +1,73 @@ +fun main(x: int): (int, int) { + var y: int = 5; + if (x < 0) { + x *= 2; + y += 1; + if (x == -10) { + return (111, 0); + } + } + return (x + 1, y); +} + +@inline +fun foo1(x: int): int { + if (x < 0) { + x *= 2; + if (x == -10) { + return 111; + } + } + return x + 1; +} + +@method_id(101) +fun test1(x: int): int { + return foo1(x) * 10; +} + +fun foo2(y: int): int { + if (y < 0) { + y *= 2; + if (y == -10) { + return 111; + } + } + return y + 1; +} + +fun bar2(x: int, y: int): (int, int) { + if (x < 0) { + y = foo2(y); + x *= 2; + if (x == -10) { + return (111, y); + } + } + return (x + 1, y); +} + +@method_id(102) +fun test2(x: int, y: int): (int, int) { + (x, y) = bar2(x, y); + return (x, y * 10); +} + +/** + method_id | in | out +@testcase | 0 | 10 | 11 5 +@testcase | 0 | -5 | 111 0 +@testcase | 0 | -4 | -7 6 +@testcase | 101 | 10 | 110 +@testcase | 101 | -5 | 1110 +@testcase | 101 | -4 | -70 +@testcase | 102 | 3 3 | 4 30 +@testcase | 102 | 3 -5 | 4 -50 +@testcase | 102 | 3 -4 | 4 -40 +@testcase | 102 | -5 3 | 111 40 +@testcase | 102 | -5 -5 | 111 1110 +@testcase | 102 | -5 -4 | 111 -70 +@testcase | 102 | -4 3 | -7 40 +@testcase | 102 | -4 -5 | -7 1110 +@testcase | 102 | -4 -4 | -7 -70 +*/ diff --git a/tolk-tester/tests/unbalanced_ret.tolk b/tolk-tester/tests/unbalanced_ret.tolk deleted file mode 100644 index 6cf42643a1..0000000000 --- a/tolk-tester/tests/unbalanced_ret.tolk +++ /dev/null @@ -1,17 +0,0 @@ -fun main(x: int): (int, int) { - var y: int = 5; - if (x < 0) { - x *= 2; - y += 1; - if (x == -10) { - return (111, 0); - } - } - return (x + 1, y); -} -/** - method_id | in | out -@testcase | 0 | 10 | 11 5 -@testcase | 0 | -5 | 111 0 -@testcase | 0 | -4 | -7 6 -*/ diff --git a/tolk-tester/tests/unbalanced_ret_inline.tolk b/tolk-tester/tests/unbalanced_ret_inline.tolk deleted file mode 100644 index 4e24fbd8f6..0000000000 --- a/tolk-tester/tests/unbalanced_ret_inline.tolk +++ /dev/null @@ -1,19 +0,0 @@ -@inline -fun foo(x: int): int { - if (x < 0) { - x *= 2; - if (x == -10) { - return 111; - } - } - return x + 1; -} -fun main(x: int): int { - return foo(x) * 10; -} -/** - method_id | in | out -@testcase | 0 | 10 | 110 -@testcase | 0 | -5 | 1110 -@testcase | 0 | -4 | -70 -*/ diff --git a/tolk-tester/tests/unbalanced_ret_nested.tolk b/tolk-tester/tests/unbalanced_ret_nested.tolk deleted file mode 100644 index 05e6092400..0000000000 --- a/tolk-tester/tests/unbalanced_ret_nested.tolk +++ /dev/null @@ -1,37 +0,0 @@ -fun foo(y: int): int { - if (y < 0) { - y *= 2; - if (y == -10) { - return 111; - } - } - return y + 1; -} -fun bar(x: int, y: int): (int, int) { - if (x < 0) { - y = foo(y); - x *= 2; - if (x == -10) { - return (111, y); - } - } - return (x + 1, y); -} -fun main(x: int, y: int): (int, int) { - (x, y) = bar(x, y); - return (x, y * 10); -} -/** - method_id | in | out -@testcase | 0 | 3 3 | 4 30 -@testcase | 0 | 3 -5 | 4 -50 -@testcase | 0 | 3 -4 | 4 -40 -@testcase | 0 | -5 3 | 111 40 -@testcase | 0 | -5 -5 | 111 1110 -@testcase | 0 | -5 -4 | 111 -70 -@testcase | 0 | -4 3 | -7 40 -@testcase | 0 | -4 -5 | -7 1110 -@testcase | 0 | -4 -4 | -7 -70 - -@code_hash 68625253347714662162648433047986779710161195298061582217368558479961252943991 -*/ diff --git a/tolk-tester/tests/use-before-declare.tolk b/tolk-tester/tests/use-before-declare.tolk index 2a0e0e7ff5..9c5f063496 100644 --- a/tolk-tester/tests/use-before-declare.tolk +++ b/tolk-tester/tests/use-before-declare.tolk @@ -33,9 +33,16 @@ fun test1(): int { return demo_var + demo_slice; } +fun test2() { + return second; +} + global demo_slice: slice; const demo_20: int = 20; +const second = first + 1; +const first = 1; + /** @testcase | 0 | | 34 @@ -46,4 +53,12 @@ const demo_20: int = 20; 30 PUSHINT // '10 }> """ + +@fif_codegen +""" + test2 PROC:<{ + // + 2 PUSHINT // '0=2 + }> +""" */ diff --git a/tolk-tester/tests/var-apply.tolk b/tolk-tester/tests/var-apply-tests.tolk similarity index 85% rename from tolk-tester/tests/var-apply.tolk rename to tolk-tester/tests/var-apply-tests.tolk index d189430fbb..96b4d134de 100644 --- a/tolk-tester/tests/var-apply.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -175,6 +175,28 @@ fun testVarsModificationInsideVarCall(x: int) { return x > 3 ? cb(x, x += 5, eq(x *= x), x, eq(x)) : null; } +fun check_assoc_1(op: (int, int) -> int, a: int, b: int, c: int) { + return op(op(a, b), c) == op(a, op(b, c)); +} + +@method_id(111) +fun testApplyNativePlus(x: int, y: int, z: int): bool { + return check_assoc_1(`_+_`, x, y, z); +} + +global op: (int, int) -> int; + +fun check_assoc_2(a: int, b: int, c: int): bool { + return op(op(a, b), c) == op(a, op(b, c)); +} + +@method_id(112) +fun testApplyGlobalVar(x: int, y: int, z: int): bool? { + op = `_+_`; + if (0) { return null; } + return check_assoc_2(x, y, z); +} + fun main() {} /** @@ -189,4 +211,10 @@ fun main() {} @testcase | 109 | | 0 3 0 7 @testcase | 110 | 5 | 5 10 100 100 100 -1 @testcase | 110 | 0 | (null) (null) (null) (null) (null) 0 - */ +@testcase | 111 | 2 3 9 | -1 +@testcase | 111 | 11 22 44 | -1 +@testcase | 111 | -1 -10 -20 | -1 +@testcase | 112 | 2 3 9 | -1 +@testcase | 112 | 11 22 44 | -1 +@testcase | 112 | -1 -10 -20 | -1 +*/ diff --git a/tolk-tester/tests/w1.tolk b/tolk-tester/tests/w1.tolk deleted file mode 100644 index eb06bec67f..0000000000 --- a/tolk-tester/tests/w1.tolk +++ /dev/null @@ -1,14 +0,0 @@ -fun main(id: int): (int, int) { - if (id > 0) { - if (id > 10) { - return (2 * id, 3 * id); - } - } - return (5, 6); -} -/** - method_id | in | out -@testcase | 0 | 0 | 5 6 -@testcase | 0 | 4 | 5 6 -@testcase | 0 | 11 | 22 33 -*/ diff --git a/tolk-tester/tests/w6.tolk b/tolk-tester/tests/w6.tolk deleted file mode 100644 index 489ffa8cca..0000000000 --- a/tolk-tester/tests/w6.tolk +++ /dev/null @@ -1,19 +0,0 @@ -fun main(x: int): int { - var i: int = 0; - // int f = false; - do { - i = i + 1; - if (i > 5) { - return 1; - } - var f: bool = (i * i == 64); - } while (!f); - return -1; -} - -/** - method_id | in | out -@testcase | 0 | 0 | 1 - -@code_hash 36599880583276393028571473830850694081778552118303309411432666239740650614479 -*/ diff --git a/tolk-tester/tests/w7.tolk b/tolk-tester/tests/w7.tolk deleted file mode 100644 index 3d68c775ba..0000000000 --- a/tolk-tester/tests/w7.tolk +++ /dev/null @@ -1,26 +0,0 @@ -@method_id(1) -fun test(y: int): int { - var x: int = 1; - if (y > 0) { - return 1; - } - return x > 0 ? -1 : 0; -} - -@method_id(2) -fun f(y: int): int { - if (y > 0) { - return 1; - } - return 2; -} - -fun main() { } - -/** - method_id | in | out -@testcase | 1 | 10 | 1 -@testcase | 1 | -5 | -1 -@testcase | 2 | 10 | 1 -@testcase | 2 | -5 | 2 -*/ diff --git a/tolk-tester/tests/w9.tolk b/tolk-tester/tests/w9.tolk deleted file mode 100644 index b88dc736ef..0000000000 --- a/tolk-tester/tests/w9.tolk +++ /dev/null @@ -1,14 +0,0 @@ -fun main(s: int) { - var (z, t) = (17, s); - while (z > 0) { - t = s; - z -= 1; - } - return ~ t; -} - -/** - method_id | in | out -@testcase | 0 | 1 | -2 -@testcase | 0 | 5 | -6 -*/ diff --git a/tolk-tester/tests/unreachable-1.tolk b/tolk-tester/tests/warnings-not-errors/unreachable-1.tolk similarity index 100% rename from tolk-tester/tests/unreachable-1.tolk rename to tolk-tester/tests/warnings-not-errors/unreachable-1.tolk diff --git a/tolk-tester/tests/unreachable-2.tolk b/tolk-tester/tests/warnings-not-errors/unreachable-2.tolk similarity index 100% rename from tolk-tester/tests/unreachable-2.tolk rename to tolk-tester/tests/warnings-not-errors/unreachable-2.tolk diff --git a/tolk-tester/tests/unreachable-3.tolk b/tolk-tester/tests/warnings-not-errors/unreachable-3.tolk similarity index 100% rename from tolk-tester/tests/unreachable-3.tolk rename to tolk-tester/tests/warnings-not-errors/unreachable-3.tolk diff --git a/tolk-tester/tests/unreachable-4.tolk b/tolk-tester/tests/warnings-not-errors/unreachable-4.tolk similarity index 100% rename from tolk-tester/tests/unreachable-4.tolk rename to tolk-tester/tests/warnings-not-errors/unreachable-4.tolk diff --git a/tolk-tester/tests/warnings-1.tolk b/tolk-tester/tests/warnings-not-errors/warnings-1.tolk similarity index 100% rename from tolk-tester/tests/warnings-1.tolk rename to tolk-tester/tests/warnings-not-errors/warnings-1.tolk diff --git a/tolk-tester/tests/warnings-2.tolk b/tolk-tester/tests/warnings-not-errors/warnings-2.tolk similarity index 100% rename from tolk-tester/tests/warnings-2.tolk rename to tolk-tester/tests/warnings-not-errors/warnings-2.tolk diff --git a/tolk-tester/tolk-tester.js b/tolk-tester/tolk-tester.js index c7e7102140..0aaf0738f8 100644 --- a/tolk-tester/tolk-tester.js +++ b/tolk-tester/tolk-tester.js @@ -1,4 +1,4 @@ -// Usage: `node tolk-tester.js tests_dir` OR `node tolk-tester.js test_file.tolk` +// Usage: `node tolk-tester.js tests_dir` OR `node tolk-tester.js tests_dir file_pattern` // from current dir, providing some env (see getenv() calls). // This is a JS version of tolk-tester.py to test Tolk compiled to WASM. // Don't forget to keep it identical to Python version! @@ -32,32 +32,38 @@ const TMP_DIR = os.tmpdir() class CmdLineOptions { constructor(/**string[]*/ argv) { - if (argv.length !== 3) { - print("Usage: node tolk-tester.js tests_dir OR node tolk-tester.js test_file.tolk") + if (argv.length < 3) { + print("Usage: node tolk-tester.js tests_dir [file_pattern]") process.exit(1) } - if (!fs.existsSync(argv[2])) { - print(`Input '${argv[2]}' doesn't exist`) + if (!fs.existsSync(argv[2]) || !fs.lstatSync(argv[2]).isDirectory()) { + print(`Directory '${argv[2]}' doesn't exist`) process.exit(1) } - if (fs.lstatSync(argv[2]).isDirectory()) { - this.tests_dir = argv[2] - this.test_file = null - } else { - this.tests_dir = path.dirname(argv[2]) - this.test_file = argv[2] - } + this.tests_dir = argv[2] + this.file_pattern = argv[3] } /** @return {string[]} */ find_tests() { - if (this.test_file) // an option to run (debug) a single test - return [this.test_file] + let all_test_files = [] + let all_children_of_tests_dir = fs.readdirSync(this.tests_dir) + all_children_of_tests_dir.sort() + for (let f of all_children_of_tests_dir) + if (f.endsWith(".tolk")) + all_test_files.push(path.join(this.tests_dir, f)) + for (let f of all_children_of_tests_dir) + if (!f.endsWith(".tolk") && f !== "imports") { + let subdir = path.join(this.tests_dir, f) + let children_of_subdir = fs.readdirSync(subdir) + children_of_subdir.sort() + all_test_files.push(...children_of_subdir.map(f => path.join(subdir, f))) + } - let tests = fs.readdirSync(this.tests_dir).filter(f => f.endsWith('.tolk') || f.endsWith('.ton')) - tests.sort() - return tests.map(f => path.join(this.tests_dir, f)) + if (this.file_pattern) + all_test_files = all_test_files.filter(f => f.includes(this.file_pattern)) + return all_test_files } } @@ -126,7 +132,7 @@ class TolkTestCaseInputOutput { check(/**string[]*/ stdout_lines, /**number*/ line_idx) { if (stdout_lines[line_idx] !== this.expected_output) - throw new CompareOutputError(`error on case #${line_idx + 1} (${this.method_id} | ${this.input}): expected '${this.expected_output}', found '${stdout_lines[line_idx]}'`, stdout_lines.join("\n")) + throw new CompareOutputError(`error on case #${line_idx + 1} (${this.method_id} | ${this.input}):\n expect: ${this.expected_output}\n actual: ${stdout_lines[line_idx]}`, stdout_lines.join("\n")) } } @@ -400,7 +406,7 @@ class TolkTestFile { async function run_all_tests(/**string[]*/ tests) { for (let ti = 0; ti < tests.length; ++ti) { let tolk_filename = tests[ti] - print(`Running test ${ti + 1}/${tests.length}: ${tolk_filename}`) + print(`Running test ${ti + 1}/${tests.length}: ${path.basename(tolk_filename)}`) let artifacts_folder = path.join(TMP_DIR, tolk_filename) let testcase = new TolkTestFile(tolk_filename, artifacts_folder) @@ -413,7 +419,7 @@ async function run_all_tests(/**string[]*/ tests) { fs.rmSync(artifacts_folder, {recursive: true}) if (testcase.compilation_should_fail) - print(" OK, compilation failed as it should") + print(" OK, stderr match") else print(` OK, ${testcase.input_output.length} cases`) } catch (e) { diff --git a/tolk-tester/tolk-tester.py b/tolk-tester/tolk-tester.py index 0b3c774ca9..3b4fb5b7f6 100644 --- a/tolk-tester/tolk-tester.py +++ b/tolk-tester/tolk-tester.py @@ -1,4 +1,4 @@ -# Usage: `tolk-tester.py tests_dir` OR `tolk-tester.py test_file.tolk` +# Usage: `tolk-tester.py tests_dir` OR `tolk-tester.py tests_dir file_pattern` # from current dir, providing some env (see getenv() calls). # Every .tolk file should provide /* testcase description in a comment */, consider tests/ folder. # @@ -36,27 +36,33 @@ def getenv(name, default=None): class CmdLineOptions: def __init__(self, argv: List[str]): - if len(argv) != 2: - print("Usage: tolk-tester.py tests_dir OR tolk-tester.py test_file.tolk", file=sys.stderr) + if len(argv) < 2: + print("Usage: tolk-tester.py tests_dir [file_pattern]", file=sys.stderr) exit(1) - if not os.path.exists(argv[1]): - print("Input '%s' doesn't exist" % argv[1], file=sys.stderr) + if not os.path.isdir(argv[1]): + print("Directory '%s' doesn't exist" % argv[1], file=sys.stderr) exit(1) - if os.path.isdir(argv[1]): - self.tests_dir = argv[1] - self.test_file = None - else: - self.tests_dir = os.path.dirname(argv[1]) - self.test_file = argv[1] + self.tests_dir = argv[1] + self.file_pattern = argv[2] if len(argv) > 2 else None def find_tests(self) -> List[str]: - if self.test_file is not None: # an option to run (debug) a single test - return [self.test_file] - - tests = [f for f in os.listdir(self.tests_dir) if f.endswith(".tolk") or f.endswith(".ton")] - tests.sort() - return [os.path.join(self.tests_dir, f) for f in tests] + all_test_files: List[str] = [] + all_children_of_tests_dir = os.listdir(self.tests_dir) + all_children_of_tests_dir.sort() + for f in all_children_of_tests_dir: + if f.endswith(".tolk"): + all_test_files.append(os.path.join(self.tests_dir, f)) + for f in all_children_of_tests_dir: + if not f.endswith(".tolk") and f != "imports": + subdir = os.path.join(self.tests_dir, f) + children_of_subdir = os.listdir(subdir) + children_of_subdir.sort() + all_test_files += [os.path.join(subdir, f) for f in children_of_subdir] + + if self.file_pattern is not None: + all_test_files = [f for f in all_test_files if f.find(self.file_pattern) != -1] + return all_test_files class ParseInputError(Exception): @@ -101,6 +107,7 @@ class TolkTestCaseInputOutput: """ reJustNumber = re.compile(r"[-+]?\d+") reMathExpr = re.compile(r"[0x123456789()+\-*/<>]+") + reGasUsed = re.compile(r"gas:\sused=(\d+)") def __init__(self, method_id_str: str, input_str: str, output_str: str): processed_inputs = [] @@ -122,7 +129,7 @@ def __init__(self, method_id_str: str, input_str: str, output_str: str): def check(self, stdout_lines: List[str], line_idx: int): if stdout_lines[line_idx] != self.expected_output: - raise CompareOutputError("error on case #%d (%d | %s): expected '%s', found '%s'" % (line_idx + 1, self.method_id, self.input, self.expected_output, stdout_lines[line_idx]), "\n".join(stdout_lines)) + raise CompareOutputError("error on case #%d (%d | %s):\n expect: %s\n actual: %s" % (line_idx + 1, self.method_id, self.input, self.expected_output, stdout_lines[line_idx]), "\n".join(stdout_lines)) class TolkTestCaseStderr: @@ -331,7 +338,7 @@ def run_and_check(self): should_include.check(stderr) if exit_code != 0 and self.compilation_should_fail: - return + return 0 if exit_code != 0 and not self.compilation_should_fail: raise TolkCompilationFailedError("tolk exit_code = %d" % exit_code, stderr) @@ -350,6 +357,7 @@ def run_and_check(self): if exit_code != 0: raise FiftExecutionFailedError("fift exit_code = %d" % exit_code, stderr) + gas_used = sum(map(int, TolkTestCaseInputOutput.reGasUsed.findall(stderr))) stdout_lines = [x.strip() for x in stdout.split("\n")] stdout_lines = [x for x in stdout_lines if x != ""] fif_code_hash = None @@ -372,11 +380,14 @@ def run_and_check(self): if self.expected_hash is not None: self.expected_hash.check(fif_code_hash) + return gas_used + def run_all_tests(tests: List[str]): + total_gas_used = 0 for ti in range(len(tests)): tolk_filename = tests[ti] - print("Running test %d/%d: %s" % (ti + 1, len(tests), tolk_filename), file=sys.stderr) + print("Running test %d/%d: %s" % (ti + 1, len(tests), os.path.basename(tolk_filename)), file=sys.stderr) artifacts_folder = os.path.join(TMP_DIR, tolk_filename) testcase = TolkTestFile(tolk_filename, artifacts_folder) @@ -384,13 +395,14 @@ def run_all_tests(tests: List[str]): if not os.path.exists(artifacts_folder): os.makedirs(artifacts_folder) testcase.parse_input_from_tolk_file() - testcase.run_and_check() + gas_used = testcase.run_and_check() shutil.rmtree(artifacts_folder) + total_gas_used += gas_used if testcase.compilation_should_fail: - print(" OK, compilation failed as it should", file=sys.stderr) + print(" OK, stderr match", file=sys.stderr) else: - print(" OK, %d cases" % len(testcase.input_output), file=sys.stderr) + print(" OK, %d cases, gas %d" % (len(testcase.input_output), gas_used), file=sys.stderr) except ParseInputError as e: print(" Error parsing input (cur line #%d):" % (testcase.line_idx + 1), e, file=sys.stderr) exit(2) @@ -423,9 +435,10 @@ def run_all_tests(tests: List[str]): print(" Mismatch in code hash:", e, file=sys.stderr) print(" Was compiled to:", testcase.get_compiled_fif_filename(), file=sys.stderr) exit(2) + return total_gas_used tests = CmdLineOptions(sys.argv).find_tests() print("Found", len(tests), "tests", file=sys.stderr) -run_all_tests(tests) -print("Done, %d tests" % len(tests), file=sys.stderr) +total_gas_used = run_all_tests(tests) +print("Done, %d tests, gas %d" % (len(tests), total_gas_used), file=sys.stderr) From 888cb43319f4e54c656b85df6ff14f5b73c8bf68 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 14 Mar 2025 10:34:50 +0300 Subject: [PATCH 160/524] Fix computing storage stat --- crypto/block/transaction.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 4e92b8f6cc..33d0f73149 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -3284,7 +3284,7 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } AccountStorageStat& stats = new_account_storage_stat.value_force(); // Don't check Merkle depth and size here - they were checked in check_state_limits - auto S = stats.replace_roots(new_storage->prefetch_all_refs()).move_as_status(); + auto S = stats.replace_roots(new_storage_for_stat->prefetch_all_refs()).move_as_status(); if (S.is_error()) { LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error(); return false; @@ -3292,7 +3292,7 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { new_storage_dict_hash = stats.get_dict_hash(); // Root of AccountStorage is not counted in AccountStorageStat new_storage_used.cells = stats.get_total_cells() + 1; - new_storage_used.bits = stats.get_total_bits() + new_storage->size(); + new_storage_used.bits = stats.get_total_bits() + new_storage_for_stat->size(); if (timer.elapsed() > 0.1) { LOG(INFO) << "Compute used storage (2) took " << timer.elapsed() << "s"; } From f12b63f3b8d2977c4a062a8d2573a1b6e78a58f8 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 14 Mar 2025 10:36:02 +0300 Subject: [PATCH 161/524] Optimize AccountStorageStat cleanup --- crypto/block/account-storage-stat.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index f5545f8b2e..bdd68b1fdb 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -55,6 +55,14 @@ AccountStorageStat& AccountStorageStat::operator=(AccountStorageStat&& other) { td::Result AccountStorageStat::replace_roots(std::vector> new_roots) { std::erase_if(new_roots, [](const Ref& c) { return c.is_null(); }); + if (new_roots.empty()) { + roots_.clear(); + total_bits_ = total_cells_ = 0; + dict_ = vm::Dictionary{256}; + cache_ = {}; + return CellInfo{}; + } + auto cmp = [](const Ref& c1, const Ref& c2) { return c1->get_hash() < c2->get_hash(); }; std::sort(new_roots.begin(), new_roots.end(), cmp); std::sort(roots_.begin(), roots_.end(), cmp); From 9f8c3dabff295644c796dcd580ed4ff0a6c309b0 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 14 Mar 2025 11:59:35 +0300 Subject: [PATCH 162/524] Don't store zero dict hash --- crypto/block/transaction.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 33d0f73149..7dbc5249f7 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -3303,7 +3303,7 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { new_storage_dict_hash = account.storage_dict_hash; new_account_storage_stat = account.account_storage_stat; } - if (!cfg.store_storage_dict_hash) { + if (!cfg.store_storage_dict_hash || (new_storage_dict_hash && new_storage_dict_hash.value().is_zero())) { new_storage_dict_hash = {}; } From 21b8f2a1f6e04f095022ff19b45b7ff896c80e56 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 14 Mar 2025 13:30:20 +0300 Subject: [PATCH 163/524] Change collated data serialize mode to 2 --- validator-session/candidate-serializer.cpp | 11 +++++++---- validator-session/candidate-serializer.h | 6 ++++-- validator-session/validator-session.cpp | 3 ++- validator/collation-manager.cpp | 15 +++++++++------ validator/collation-manager.hpp | 8 ++++---- validator/collator-node.cpp | 6 +++--- validator/collator-node.hpp | 2 +- validator/impl/collator.cpp | 5 +++-- validator/validator-group.cpp | 2 +- 9 files changed, 34 insertions(+), 24 deletions(-) diff --git a/validator-session/candidate-serializer.cpp b/validator-session/candidate-serializer.cpp index 4442504edd..2fc7d7a5f5 100644 --- a/validator-session/candidate-serializer.cpp +++ b/validator-session/candidate-serializer.cpp @@ -35,7 +35,8 @@ td::Result serialize_candidate(const tl_object_ptr> deserialize_candidate(td::Slice data, bool compression_enabled, - int max_decompressed_data_size) { + int max_decompressed_data_size, + int proto_version) { if (!compression_enabled) { return fetch_tl_object(data, true); } @@ -43,7 +44,7 @@ td::Result> deserialize_candi if (f->decompressed_size_ > max_decompressed_data_size) { return td::Status::Error("decompressed size is too big"); } - TRY_RESULT(p, decompress_candidate_data(f->data_, f->decompressed_size_)); + TRY_RESULT(p, decompress_candidate_data(f->data_, f->decompressed_size_, proto_version)); return create_tl_object(f->src_, f->round_, f->root_hash_, std::move(p.first), std::move(p.second)); } @@ -68,7 +69,8 @@ td::Result compress_candidate_data(td::Slice block, td::Slice c } td::Result> decompress_candidate_data(td::Slice compressed, - int decompressed_size) { + int decompressed_size, + int proto_version) { TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size)); if (decompressed.size() != (size_t)decompressed_size) { return td::Status::Error("decompressed size mismatch"); @@ -79,7 +81,8 @@ td::Result> decompress_candidate_dat } TRY_RESULT(block_data, vm::std_boc_serialize(roots[0], 31)); roots.erase(roots.begin()); - TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), 31)); + int collated_data_mode = proto_version >= 5 ? 2 : 31; + TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), collated_data_mode)); LOG(DEBUG) << "Decompressing block candidate: " << compressed.size() << " -> " << block_data.size() + collated_data.size(); return std::make_pair(std::move(block_data), std::move(collated_data)); diff --git a/validator-session/candidate-serializer.h b/validator-session/candidate-serializer.h index e88376cd77..7cc77f0be3 100644 --- a/validator-session/candidate-serializer.h +++ b/validator-session/candidate-serializer.h @@ -24,11 +24,13 @@ td::Result serialize_candidate(const tl_object_ptr> deserialize_candidate(td::Slice data, bool compression_enabled, - int max_decompressed_data_size); + int max_decompressed_data_size, + int proto_version); td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, size_t& decompressed_size); td::Result> decompress_candidate_data(td::Slice compressed, - int decompressed_size); + int decompressed_size, + int proto_version); } // namespace ton::validatorsession diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index e18dd1eb56..79233d6990 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -229,7 +229,8 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice td::Timer deserialize_timer; auto R = deserialize_candidate(data, compress_block_candidates_, - description().opts().max_block_size + description().opts().max_collated_data_size + 1024); + description().opts().max_block_size + description().opts().max_collated_data_size + 1024, + description().opts().proto_version); double deserialize_time = deserialize_timer.elapsed(); if (R.is_error()) { VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index 2b06ad482a..27af220df2 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -35,7 +35,7 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha std::vector prev, Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, - td::Promise promise) { + td::Promise promise, int proto_version) { if (shard.is_masterchain()) { run_collate_query( shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), @@ -46,14 +46,16 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha return; } collate_shard_block(shard, min_masterchain_block_id, std::move(prev), creator, priority, std::move(validator_set), - max_answer_size, std::move(cancellation_token), std::move(promise), td::Timestamp::in(10.0)); + max_answer_size, std::move(cancellation_token), std::move(promise), td::Timestamp::in(10.0), + proto_version); } void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, - td::Promise promise, td::Timestamp timeout) { + td::Promise promise, td::Timestamp timeout, + int proto_version) { TRY_STATUS_PROMISE(promise, cancellation_token.check()); ShardInfo* s = select_shard_info(shard); if (s == nullptr) { @@ -144,7 +146,7 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas [=, promise = std::move(promise)]() mutable { td::actor::send_closure(SelfId, &CollationManager::collate_shard_block, shard, min_masterchain_block_id, prev, creator, priority, validator_set, max_answer_size, cancellation_token, - std::move(promise), timeout); + std::move(promise), timeout, proto_version); }, retry_at); }; @@ -170,8 +172,9 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas return; } TRY_RESULT_PROMISE(P, f, fetch_tl_object(data, true)); - TRY_RESULT_PROMISE(P, candidate, - CollatorNode::deserialize_candidate(std::move(f), td::narrow_cast(max_answer_size))); + TRY_RESULT_PROMISE( + P, candidate, + CollatorNode::deserialize_candidate(std::move(f), td::narrow_cast(max_answer_size), proto_version)); if (candidate.pubkey.as_bits256() != creator.as_bits256()) { P.set_error(td::Status::Error("collate query: block candidate source mismatch")); return; diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp index 72cc02e5a2..048d30812b 100644 --- a/validator/collation-manager.hpp +++ b/validator/collation-manager.hpp @@ -35,9 +35,9 @@ class CollationManager : public td::actor::Actor { void alarm() override; void collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - Ed25519_PublicKey creator, BlockCandidatePriority priority, - td::Ref validator_set, td::uint64 max_answer_size, - td::CancellationToken cancellation_token, td::Promise promise); + Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, + td::uint64 max_answer_size, td::CancellationToken cancellation_token, + td::Promise promise, int proto_version); void update_options(td::Ref opts); @@ -56,7 +56,7 @@ class CollationManager : public td::actor::Actor { Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise, - td::Timestamp timeout); + td::Timestamp timeout, int proto_version); void update_collators_list(const CollatorsList& collators_list); diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index ff84456bd5..238eb89218 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -640,7 +640,7 @@ tl_object_ptr CollatorNode::serialize_candidate } td::Result CollatorNode::deserialize_candidate(tl_object_ptr f, - int max_decompressed_data_size) { + int max_decompressed_data_size, int proto_version) { td::Result res; ton_api::downcast_call(*f, td::overloaded( [&](ton_api::collatorNode_candidate& c) { @@ -663,8 +663,8 @@ td::Result CollatorNode::deserialize_candidate(tl_object_ptr max_decompressed_data_size) { return td::Status::Error("decompressed size is too big"); } - TRY_RESULT( - p, validatorsession::decompress_candidate_data(c.data_, c.decompressed_size_)); + TRY_RESULT(p, validatorsession::decompress_candidate_data( + c.data_, c.decompressed_size_, proto_version)); auto collated_data_hash = td::sha256_bits256(p.second); auto key = ton::PublicKey{c.source_}; if (!key.is_ed25519()) { diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index 5aab246356..604338f0b6 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -103,7 +103,7 @@ class CollatorNode : public td::actor::Actor { public: static tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress); static td::Result deserialize_candidate(tl_object_ptr f, - int max_decompressed_data_size); + int max_decompressed_data_size, int proto_version); }; } // namespace ton::validator diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index fee6f21d1f..362af889cb 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -5901,6 +5901,7 @@ bool Collator::create_collated_data() { * @returns True if the block candidate was created successfully, false otherwise. */ bool Collator::create_block_candidate() { + auto consensus_config = config_->get_consensus_config(); // 1. serialize block LOG(INFO) << "serializing new Block"; vm::BagOfCells boc; @@ -5926,7 +5927,8 @@ bool Collator::create_block_candidate() { if (res.is_error()) { return fatal_error(res.move_as_error()); } - auto cdata_res = boc_collated.serialize_to_slice(31); + int cdata_serialize_mode = consensus_config.proto_version >= 5 ? 2 : 31; + auto cdata_res = boc_collated.serialize_to_slice(cdata_serialize_mode); if (cdata_res.is_error()) { LOG(ERROR) << "cannot serialize collated data"; return fatal_error(cdata_res.move_as_error()); @@ -5985,7 +5987,6 @@ bool Collator::create_block_candidate() { } // 3.1 check block and collated data size - auto consensus_config = config_->get_consensus_config(); if (block_candidate->data.size() > consensus_config.max_block_size) { return fatal_error(PSTRING() << "block size (" << block_candidate->data.size() << ") exceeds the limit in consensus config (" << consensus_config.max_block_size diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 88c0535f9e..ed063cb6dd 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -70,7 +70,7 @@ void ValidatorGroup::generate_block_candidate(validatorsession::BlockSourceInfo td::actor::send_closure(collation_manager_, &CollationManager::collate_block, shard_, min_masterchain_block_id_, prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, source_info.priority, validator_set_, max_answer_size, - cancellation_token_source_.get_cancellation_token(), std::move(P)); + cancellation_token_source_.get_cancellation_token(), std::move(P), config_.proto_version); } void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo source_info, From 1cb420163f825a8d00f63b7ba6a2036eedaefb72 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 17 Mar 2025 12:03:51 +0300 Subject: [PATCH 164/524] Ping neighbours in private overlays, show ping time in overlay stats --- catchain/catchain-receiver.cpp | 1 + overlay/overlay-peers.cpp | 48 +++++++++++++----- overlay/overlay.cpp | 27 ++++++++-- overlay/overlay.hpp | 23 ++++++--- overlay/overlays.h | 1 + tl/generate/scheme/ton_api.tl | 6 ++- tl/generate/scheme/ton_api.tlo | Bin 102788 -> 103024 bytes .../validator-engine-console-query.cpp | 12 ++++- validator/full-node-private-overlay.cpp | 1 + 9 files changed, 92 insertions(+), 27 deletions(-) diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp index a6160383cb..b663cfc06f 100644 --- a/catchain/catchain-receiver.cpp +++ b/catchain/catchain-receiver.cpp @@ -528,6 +528,7 @@ void CatChainReceiverImpl::start_up() { } overlay::OverlayOptions overlay_options; overlay_options.broadcast_speed_multiplier_ = opts_.broadcast_speed_multiplier; + overlay_options.private_ping_peers_ = true; td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay_ex, get_source(local_idx_)->get_adnl_id(), overlay_full_id_.clone(), std::move(ids), make_callback(), overlay::OverlayPrivacyRules{0, 0, std::move(root_keys)}, diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index 7def4a2d39..923f0fb127 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -247,12 +247,16 @@ void OverlayImpl::add_peers(const tl_object_ptr &nodes } } -void OverlayImpl::on_ping_result(adnl::AdnlNodeIdShort peer, bool success) { - if (overlay_type_ == OverlayType::FixedMemberList) { +void OverlayImpl::on_ping_result(adnl::AdnlNodeIdShort peer, bool success, double store_ping_time) { + if (overlay_type_ == OverlayType::FixedMemberList && (!success || store_ping_time < 0.0)) { return; } if (OverlayPeer *p = peer_list_.peers_.get(peer)) { p->on_ping_result(success); + if (store_ping_time >= 0.0 && success) { + p->last_ping_at = td::Timestamp::now(); + p->last_ping_time = store_ping_time; + } if (p->is_alive()) { peer_list_.bad_peers_.erase(peer); } else { @@ -261,9 +265,9 @@ void OverlayImpl::on_ping_result(adnl::AdnlNodeIdShort peer, bool success) { } } -void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R) { +void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R, double elapsed) { CHECK(overlay_type_ != OverlayType::FixedMemberList); - on_ping_result(src, R.is_ok()); + on_ping_result(src, R.is_ok(), elapsed); if (R.is_error()) { VLOG(OVERLAY_NOTICE) << this << ": failed getRandomPeers query: " << R.move_as_error(); return; @@ -278,9 +282,9 @@ void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R) { +void OverlayImpl::receive_random_peers_v2(adnl::AdnlNodeIdShort src, td::Result R, double elapsed) { CHECK(overlay_type_ != OverlayType::FixedMemberList); - on_ping_result(src, R.is_ok()); + on_ping_result(src, R.is_ok(), elapsed); if (R.is_error()) { VLOG(OVERLAY_NOTICE) << this << ": failed getRandomPeersV2 query: " << R.move_as_error(); return; @@ -318,9 +322,9 @@ void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode auto Q = create_tl_object(std::move(vec)); promise.set_value(serialize_tl_object(Q, true)); } else { - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), src, oid = print_id()](td::Result res) { - td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers, src, std::move(res)); + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), src, timer = td::Timer()](td::Result res) { + td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers, src, std::move(res), timer.elapsed()); }); auto Q = create_tl_object(create_tl_object(std::move(vec))); @@ -367,9 +371,9 @@ void OverlayImpl::send_random_peers_v2_cont(adnl::AdnlNodeIdShort src, OverlayNo auto Q = create_tl_object(std::move(vec)); promise.set_value(serialize_tl_object(Q, true)); } else { - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), src, oid = print_id()](td::Result res) { - td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers_v2, src, std::move(res)); + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), src, timer = td::Timer()](td::Result res) { + td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers_v2, src, std::move(res), timer.elapsed()); }); auto Q = create_tl_object(create_tl_object(std::move(vec))); @@ -392,6 +396,26 @@ void OverlayImpl::send_random_peers_v2(adnl::AdnlNodeIdShort src, td::Promise R) { + if (R.is_error()) { + VLOG(OVERLAY_INFO) << oid << " ping to " << peer << " failed : " << R.move_as_error(); + return; + } + td::actor::send_closure(SelfId, &OverlayImpl::receive_pong, peer, timer.elapsed()); + }); + td::actor::send_closure(manager_, &OverlayManager::send_query, peer, local_id_, overlay_id_, "overlay ping", + std::move(P), td::Timestamp::in(5.0), create_serialize_tl_object()); + } +} + +void OverlayImpl::receive_pong(adnl::AdnlNodeIdShort peer, double elapsed) { + on_ping_result(peer, true, elapsed); +} + void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) { if (peer_list_.peers_.size() == 0) { return; diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 30a40b1cf2..cad18bedcb 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -134,6 +134,11 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getR } } +void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_ping &query, + td::Promise promise) { + promise.set_value(create_serialize_tl_object()); +} + void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, td::Promise promise) { auto it = broadcasts_.find(query.hash_); @@ -319,7 +324,7 @@ void OverlayImpl::alarm() { } } } else { - VLOG(OVERLAY_WARNING) << "meber certificate ist invalid, valid_until=" + VLOG(OVERLAY_WARNING) << "member certificate ist invalid, valid_until=" << peer_list_.local_cert_is_valid_until_.at_unix(); } if (next_dht_query_ && next_dht_query_.is_in_past() && overlay_type_ == OverlayType::Public) { @@ -356,8 +361,19 @@ void OverlayImpl::alarm() { } alarm_timestamp() = td::Timestamp::in(1.0); } else { - update_neighbours(0); - alarm_timestamp() = td::Timestamp::in(60.0 + td::Random::fast(0, 100) * 0.6); + if (update_neighbours_at_.is_in_past()) { + update_neighbours(0); + update_neighbours_at_ = td::Timestamp::in(60.0 + td::Random::fast(0, 100) * 0.6); + } + if (opts_.private_ping_peers_) { + if (private_ping_peers_at_.is_in_past()) { + ping_random_peers(); + private_ping_peers_at_ = td::Timestamp::in(td::Random::fast(30.0, 50.0)); + } + alarm_timestamp().relax(private_ping_peers_at_); + } + alarm_timestamp().relax(update_neighbours_at_); + alarm_timestamp().relax(update_throughput_at_); } } @@ -483,7 +499,7 @@ void OverlayImpl::send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::Bu void OverlayImpl::send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { if (!has_valid_membership_certificate()) { - VLOG(OVERLAY_WARNING) << "meber certificate is invalid, valid_until=" + VLOG(OVERLAY_WARNING) << "member certificate is invalid, valid_until=" << peer_list_.local_cert_is_valid_until_.at_unix(); return; } @@ -721,6 +737,9 @@ void OverlayImpl::get_stats(td::Promiseis_alive_ = peer.is_alive(); node_obj->node_flags_ = peer.get_node()->flags(); + node_obj->last_ping_at_ = (peer.last_ping_at ? peer.last_ping_at.at_unix() : -1.0); + node_obj->last_ping_time_ = peer.last_ping_time; + res->nodes_.push_back(std::move(node_obj)); }); diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 41a04dec25..7727f5f1fb 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -105,11 +105,11 @@ class OverlayPeer { void on_ping_result(bool success) { if (success) { missed_pings_ = 0; - last_ping_at_ = td::Timestamp::now(); + last_receive_at_ = td::Timestamp::now(); is_alive_ = true; } else { ++missed_pings_; - if (missed_pings_ >= 3 && last_ping_at_.is_in_past(td::Timestamp::in(-15.0))) { + if (missed_pings_ >= 3 && last_receive_at_.is_in_past(td::Timestamp::in(-15.0))) { is_alive_ = false; } } @@ -149,6 +149,9 @@ class OverlayPeer { td::string ip_addr_str = "undefined"; + td::Timestamp last_ping_at = td::Timestamp::never(); + double last_ping_time = -1.0; + private: OverlayNode node_; adnl::AdnlNodeIdShort id_; @@ -157,7 +160,7 @@ class OverlayPeer { size_t missed_pings_ = 0; bool is_alive_ = true; bool is_permanent_member_ = false; - td::Timestamp last_ping_at_ = td::Timestamp::now(); + td::Timestamp last_receive_at_ = td::Timestamp::now(); }; class OverlayImpl : public Overlay { @@ -195,13 +198,15 @@ class OverlayImpl : public Overlay { alarm_timestamp() = td::Timestamp::in(1); } - void on_ping_result(adnl::AdnlNodeIdShort peer, bool success); - void receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R); - void receive_random_peers_v2(adnl::AdnlNodeIdShort src, td::Result R); + void on_ping_result(adnl::AdnlNodeIdShort peer, bool success, double store_ping_time = -1.0); + void receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R, double elapsed); + void receive_random_peers_v2(adnl::AdnlNodeIdShort src, td::Result R, double elapsed); void send_random_peers(adnl::AdnlNodeIdShort dst, td::Promise promise); void send_random_peers_v2(adnl::AdnlNodeIdShort dst, td::Promise promise); void send_random_peers_cont(adnl::AdnlNodeIdShort dst, OverlayNode node, td::Promise promise); void send_random_peers_v2_cont(adnl::AdnlNodeIdShort dst, OverlayNode node, td::Promise promise); + void ping_random_peers(); + void receive_pong(adnl::AdnlNodeIdShort peer, double elapsed); void get_overlay_random_peers(td::uint32 max_peers, td::Promise> promise) override; void set_privacy_rules(OverlayPrivacyRules rules) override; void add_certificate(PublicKeyHash key, std::shared_ptr cert) override { @@ -335,6 +340,7 @@ class OverlayImpl : public Overlay { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeersV2 &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_ping &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcastList &query, @@ -390,9 +396,10 @@ class OverlayImpl : public Overlay { td::Timestamp next_dht_query_ = td::Timestamp::in(1.0); td::Timestamp next_dht_store_query_ = td::Timestamp::in(1.0); td::Timestamp update_db_at_; - td::Timestamp update_throughput_at_; - td::Timestamp update_neighbours_at_; + td::Timestamp update_throughput_at_ = td::Timestamp::now(); + td::Timestamp update_neighbours_at_ = td::Timestamp::now(); td::Timestamp last_throughput_update_; + td::Timestamp private_ping_peers_at_ = td::Timestamp::now(); std::unique_ptr callback_; diff --git a/overlay/overlays.h b/overlay/overlays.h index 5eb63b13f8..972e48cce6 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -270,6 +270,7 @@ struct OverlayOptions { td::uint32 propagate_broadcast_to_ = 5; td::uint32 default_permanent_members_flags_ = 0; double broadcast_speed_multiplier_ = 1.0; + bool private_ping_peers_ = false; }; class Overlays : public td::actor::Actor { diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 14e0121fd6..580d74861c 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -234,6 +234,8 @@ overlay.nodeV2 id:PublicKey overlay:int256 flags:int version:int signature:bytes overlay.nodes nodes:(vector overlay.node) = overlay.Nodes; overlay.nodesV2 nodes:(vector overlay.NodeV2) = overlay.NodesV2; +overlay.pong = overlay.Pong; + overlay.messageExtra flags:# certificate:flags.0?overlay.MemberCertificate = overlay.MessageExtra; overlay.message overlay:int256 = overlay.Message; overlay.messageWithExtra overlay:int256 extra:overlay.messageExtra = overlay.Message; @@ -270,6 +272,7 @@ overlay.broadcastNotFound = overlay.Broadcast; overlay.getRandomPeers peers:overlay.nodes = overlay.Nodes; overlay.getRandomPeersV2 peers:overlay.NodesV2 = overlay.NodesV2; +overlay.ping = overlay.Pong; overlay.query overlay:int256 = True; overlay.queryWithExtra overlay:int256 extra:overlay.messageExtra = True; @@ -714,7 +717,8 @@ engine.validator.overlayStatsTraffic t_out_bytes:long t_in_bytes:long t_out_pckt engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string is_neighbour:Bool is_alive:Bool node_flags:int bdcst_errors:int fec_bdcst_errors:int last_in_query:int last_out_query:int - traffic:engine.validator.overlayStatsTraffic traffic_responses:engine.validator.overlayStatsTraffic = engine.validator.OverlayStatsNode; + traffic:engine.validator.overlayStatsTraffic traffic_responses:engine.validator.overlayStatsTraffic + last_ping_at:double last_ping_time:double = engine.validator.OverlayStatsNode; engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 5d9ccc2d8586fcb9da5e367372b01e3394ff8b62..ae1785ebb566de1406a4792679287455daaccfc3 100644 GIT binary patch delta 271 zcmZo!%=Tdk8}Fmp`c@23ps|tHU5a(_s--s-C(BC7ZZ4Aw;^1N7C`jkYFH0@TNvxbC zsId8jl8yx9md#(S`YTvKN+;*+Fqph!^Bf2#Vv7nVNc840As^n!6W^(CUbCf!Q9%0H zeNCR6#Nv|pg3P@1_{2%6GW>i9eo1C->SX^o(aAM?Il$&@HreK*q=Rg=UO|3dI?$aU zF_7UPCl@)h>2TP#9R!JOF1c~1fU$dfpeN&RWu3+?Jee>@>w&DqqjS4ME~ASI04f+_ ACIA2c delta 136 zcmeycgso*U8}Fmp`c@23puUmUU5fRR%khhflVzo3H%85lqcZXOfz;oiJ)%VI{JB4;)o4%@baAmPm$ bw)rS+Ho1ABfU#>ks~6*MY`ARoI diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index d111001948..ab214c3f7c 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -920,6 +920,10 @@ td::Status GetOverlaysStatsQuery::receive(td::BufferSlice data) { << "\n"; sb << " is_neighbour: " << n->is_neighbour_ << " is_alive: " << n->is_alive_ << " node_flags: " << n->node_flags_ << "\n"; + if (n->last_ping_time_ >= 0.0) { + sb << " last_ping_at: " << (td::uint32)n->last_ping_at_ << " (" << time_to_human((td::uint32)n->last_ping_at_) + << ") last_ping_time: " << n->last_ping_time_ << "\n"; + } print_traffic("throughput", " ", n->traffic_); print_traffic("throughput (responses only)", " ", n->traffic_responses_); } @@ -985,8 +989,12 @@ td::Status GetOverlaysStatsJsonQuery::receive(td::BufferSlice data) { << ",\n \"last_in_query_unix\": " << n->last_in_query_ << ",\n \"last_in_query_human\": \"" << time_to_human(n->last_in_query_) << "\",\n" << " \"last_out_query_unix\": " << n->last_out_query_ << ",\n \"last_out_query_human\": \"" - << time_to_human(n->last_out_query_) << "\",\n" - << "\n "; + << time_to_human(n->last_out_query_) << "\",\n"; + if (n->last_ping_time_ >= 0.0) { + sb << " \"last_ping_at\": " << (td::uint32)n->last_ping_at_ << ", \"last_ping_at_human\": \"" + << time_to_human((td::uint32)n->last_ping_at_) << "\", \"last_ping_time\": " << n->last_ping_time_ << ",\n"; + } + sb << "\n "; print_traffic("throughput", n->traffic_); sb << ",\n "; print_traffic("throughput_responses", n->traffic_responses_); diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index f86323fc01..1fb9c15079 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -266,6 +266,7 @@ void FullNodePrivateBlockOverlay::init() { {}}; overlay::OverlayOptions overlay_options; overlay_options.broadcast_speed_multiplier_ = opts_.private_broadcast_speed_multiplier_; + overlay_options.private_ping_peers_ = true; td::actor::send_closure(overlays_, &overlay::Overlays::create_private_overlay_ex, local_id_, overlay_id_full_.clone(), nodes_, std::make_unique(actor_id(this)), rules, R"({ "type": "private-blocks" })", overlay_options); From 3740efdb75c39aa83a0917e1080e6e0a3656063f Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 13 Mar 2025 18:25:08 +0400 Subject: [PATCH 165/524] [Tolk] Replace `"..."c` postfixes with `stringCrc32("...")` functions With the introduction function `ton("0.05")`, I decided to remove old FunC-style string postfixes "..."c, "..."H, etc. in favor of a clearer and more flexible approach. "..."c -> stringCrc32("...") no way -> stringCrc16("...") "..."H -> stringSha256("...") "..."h -> stringSha256_32("...") "..."a -> stringAddressToSlice("...") "..."s -> stringHexToSlice("...") "..."u -> stringToBase256("...") These functions are compile-time only. They can be used in constant initialization. --- crypto/smartcont/tolk-stdlib/common.tolk | 54 +++++- tolk-tester/tests/constants-tests.tolk | 2 +- tolk-tester/tests/inference-tests.tolk | 2 +- tolk-tester/tests/parse-address.tolk | 194 ++++++++++---------- tolk-tester/tests/strings-tests.tolk | 41 +++-- tolk/ast-from-tokens.cpp | 11 +- tolk/ast-replicator.h | 2 +- tolk/ast-stringifier.h | 6 +- tolk/ast.cpp | 4 + tolk/ast.h | 19 +- tolk/builtins.cpp | 34 +++- tolk/constant-evaluator.cpp | 217 ++++++++++++----------- tolk/constant-evaluator.h | 4 +- tolk/lexer.cpp | 11 +- tolk/lexer.h | 1 - tolk/pipe-ast-to-legacy.cpp | 8 +- tolk/pipe-constant-folding.cpp | 34 +++- tolk/pipe-infer-types-and-calls.cpp | 7 +- tolk/symtable.h | 2 + 19 files changed, 382 insertions(+), 271 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 9647243f95..d5290976a1 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -194,6 +194,42 @@ fun commitContractDataAndActions(): void Signature checks, hashing, cryptography. */ +/// Compile-time function that calculates crc32 of a constant string. +/// Example: `const op = stringCrc32("some_str")` = 4013618352 = 0xEF3AF4B0 +/// Note: stringCrc32(slice_var) does not work! It accepts a constant string and works at compile-time. +@pure +fun stringCrc32(constString: slice): int + builtin; + +/// Compile-time function that calculates crc16 (XMODEM) of a constant string. +/// Example: `const op = stringCrc16("some_str")` = 53407 = 0xD09F +/// Note: stringCrc16(slice_var) does not work! It accepts a constant string and works at compile-time. +@pure +fun stringCrc16(constString: slice): int + builtin; + +/// Compile-time function that calculates sha256 of a constant string and returns 256-bit integer. +/// Example: `const hash = stringSha256("some_crypto_key")` +/// Note: it's a compile-time function, `stringSha256(slice_var)` does not work. +/// Use `sliceBitsHash` or `sliceHash` (declared below) to hash a slice without/with its refs at runtime. +@pure +fun stringSha256(constString: slice): int + builtin; + +/// Compile-time function that calculates sha256 of a constant string and takes the first 32 bits. +/// Example: `const minihash = stringSha256_32("some_crypto_key")` +/// Note: stringSha256_32(slice_var) does not work! It accepts a constant string and works at compile-time. +@pure +fun stringSha256_32(constString: slice): int + builtin; + +/// Compile-time function that takes N-chars ascii string and interprets it as a number in base 256. +/// Example: `const value = stringToBase256("AB")` = 16706 (65*256 + 66) +/// Note: stringToBase256(slice_var) does not work! It accepts a constant string and works at compile-time. +@pure +fun stringToBase256(constString: slice): int + builtin; + /// Computes the representation hash of a `cell` [c] and returns it as a 256-bit unsigned integer `x`. /// Useful for signing and checking signatures of arbitrary entities represented by a tree of cells. @pure @@ -210,7 +246,7 @@ fun sliceHash(s: slice): int /// Computes sha256 of the data bits of `slice` [s]. If the bit length of `s` is not divisible by eight, /// throws a cell underflow exception. The hash value is returned as a 256-bit unsigned integer `x`. @pure -fun stringHash(s: slice): int +fun sliceBitsHash(s: slice): int asm "SHA256U"; /// Checks the Ed25519-`signature` of a `hash` (a 256-bit unsigned integer, usually computed as the hash of some data) @@ -340,6 +376,22 @@ fun debugDumpStack(): void When you _preload_ some data, you just get the result without mutating the slice. */ +/// Compile-time function that converts a constant string to TL-encoded MsgAddressInt (TVM slice). +/// Example: stringAddressToSlice("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5") +/// Example: stringAddressToSlice("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8") +/// Note: it's a compile-time function, `stringAddressToSlice(slice_var)` does not work. +/// Use `parseStandardAddress` to decode a slice at runtime into workchain and hash. +@pure +fun stringAddressToSlice(constStringAddress: slice): slice + builtin; + +/// Compile-time function that converts a constant hex-encoded string to N/2 bytes. +/// Example: `const v = stringHexToSlice("abcdef")` = slice with 3 bytes `[ 0xAB, 0xCD, 0xEF ]` +/// Note: stringHexToSlice(slice_var) does not work! It accepts a constant string and works at compile-time. +@pure +fun stringHexToSlice(constStringBytesHex: slice): slice + builtin; + /// Converts a `cell` [c] into a `slice`. Notice that [c] must be either an ordinary cell, /// or an exotic cell (see [TVM.pdf](https://ton-blockchain.github.io/docs/tvm.pdf), 3.1.2) /// which is automatically loaded to yield an ordinary cell `c'`, converted into a `slice` afterwards. diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index 66b89190aa..e145306eaa 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -7,7 +7,7 @@ const int111: int = 111; const int1r = int1; const str1 = "const1"; -const str2 = "aabbcc"s; +const str2 = stringHexToSlice("aabbcc"); const str2r: slice = str2; diff --git a/tolk-tester/tests/inference-tests.tolk b/tolk-tester/tests/inference-tests.tolk index 463d4a7b7e..4368483170 100644 --- a/tolk-tester/tests/inference-tests.tolk +++ b/tolk-tester/tests/inference-tests.tolk @@ -5,7 +5,7 @@ fun eq(value: X): X { return value; } fun test1(x: int, y: int) { __expect_type(0, "int"); - __expect_type("0"c, "int"); + __expect_type(stringCrc32("0"), "int"); __expect_type(x, "int"); __expect_type(x + y, "int"); __expect_type(x * y, "int"); diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index 385aa3b53f..64ca5d67e5 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -1,5 +1,5 @@ -const cc1 = "0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e"a; -const cc2 = "EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"a; +const cc1 = stringAddressToSlice("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e"); +const cc2 = stringAddressToSlice("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); fun verifyAddr(addr: slice, workchain: int, number: int) { assert (addr.getRemainingBitsCount() == 3 + 8 + 256) throw 112; @@ -9,101 +9,101 @@ fun verifyAddr(addr: slice, workchain: int, number: int) { } fun main() { - verifyAddr("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"a, 255, 23158417847463239084714197001737581570653996933128112807891516801582625927987); - verifyAddr("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"a, 0, 0); - verifyAddr("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"a, 0, 65607996509792174074532427555986248720836864382484024657400295821210434460432); - verifyAddr("UQCOgxbCOjOLH_cEuQdGgS23zBM5SrQQepMFedjK-oixYbis"a, 0, 64460038539088394980732229180523693489682583665805557562964506609821558550881); - verifyAddr("EQDa4VOnTYlLvDJ0gZjNYm5PXfSmmtL6Vs6A_CZEtXCNICq_"a, 0, 99002318936150612861744867526221033858534811876886359650897405270877291973920); - verifyAddr("Ef8BtXO9bcTMXjg9bgivKh4lhJmZWQPP6_rb9vfjlTP5FJtM"a, 255, 772910975127952880303441415761050161913031788763061162001556772893733681428); - verifyAddr("Ef89xh-uy860-mCcvS8zcAUs8bApmxLGygDLEKjUk5RL-311"a, 255, 27941138149036269893630478666581900122707382189183906805784676408403709676539); - verifyAddr("Ef_vA6yRfmt2P4UHnxlrQUZFcBnKux8mL2eMqBgpeMFPorr4"a, 255, 108109262375472472702582493362335418330829651067377177643099076957184687427490); - verifyAddr("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"a, 255, 18502444830824300068094395885436326119386947594392869497312068745716154912158); - verifyAddr("Ef_fvrd0hBoVJUxoi7wH173Zk8NPiyVvxh5IoYSjEYZbOhsu"a, 255, 101202732337223525952216789200341489000836292542250083765062769181728788863802); - verifyAddr("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"a, 255, 46952625717497919357580310066854892621799390294920450816077086267929711460872); - verifyAddr("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"a, 255, 48545777798729612074233611768739897492467685225150339217043102685589809464695); - verifyAddr("Ef9LynHHKgBxY6-l-W_dWN-CtGT2_ji5rN3EzOI-p9zWEfq6"a, 255, 34281152017620085319078796986198022632548048219136747083019177301186013091345); - verifyAddr("Ef9hMd78gzSiVsK0zz0AHtEja8x1UoB_NDZMjn-l86NQK_2Y"a, 255, 43962460814164090767878334494257755557842170134382045184921495822637115592747); - verifyAddr("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"a, 255, 23557057702048801338698514499604413540742716310574705490458593067566768087319); - verifyAddr("Ef_fdIbThooPs4_r2DE_Z6ZsWycJdHLnsuKAJHTcbaZaipez"a, 255, 101071650030310556115830521522496708686577365303530257137459798093298869361290); - verifyAddr("Ef_lva0qEiZhWrrZJl-IJxyCcTQmmTo71fIWyQ31HxJ8NurV"a, 255, 103914771557158282349484109182290824591675204108148026180964788916630125182006); - verifyAddr("Ef8sMGKypw006AeRYqimLjmY2Ufp-SHk8C0ZJBNgVBlzw_Nr"a, 255, 19987255184378161380023126214650814972824352533523055905552702178965809886147); - verifyAddr("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff+W72r5gqPrHF"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr("EQCaSCHVak-jIc9ANutTAfHpZNM3YdGky7yaDzsTrg0WhFlm"a, 0, 69783625181781015447914682554083924798054947959007050695795761257887453484676); - verifyAddr("EQBS9U3AfD15fGmOtRMXQAxcPVBwNuItfLcDni9fkbTyyNX0"a, 0, 37523067738561024305547433298623118197038688994386001017161816416175242146504); - verifyAddr("EQBiMNL9qNWMAkJHuM0BFneYcuHL17kzS4pswpaEO-NGWrFG"a, 0, 44412924025649114419413541526870954696667907029239618728289150652715284776538); - verifyAddr("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi"a, 0, 9407242825041766837311851458322335726136775042891143504070507665010681354284); - verifyAddr("EQD-nhrinjv0B4LTgr0dRHTHwH1MOsgGhKBXJZd7vESMZUf1"a, 0, 115166810931401616117484448645661180241548402534908005320733783571353775148133); - verifyAddr("EQAVD3Fni9I6j8XeSIl-wAGBEhqhame6OtAY0GScKT0D9X6f"a, 0, 9525855215156855607080079714361451576383963668563135377495902959388099150837); - verifyAddr("EQC6ACq3VANZjqfRBy7JMHkpLwqQ9qyYJsCIGx1mYbQgxaKw"a, 0, 84130484652351964071210477536969520113177637645401392541565606610268614566085); - verifyAddr("EQCIJLNFIko5CvpKn9oAkrDgLocDOoD4vwmHxNx_fsG_LkwW"a, 0, 61579391178099797614367237687950512448308156724136883899001108680249616482094); - verifyAddr("EQCe4AYIBce1pAk2qJJPSs1OzyZRlKjkfq8zuC8D7erv6DUP"a, 0, 71861245445432818728925844931259040612664802586395398157190478191760507596776); - verifyAddr("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"a, 0, 78559023162479717496981724991265882229440558807791659796411897368395464230649); - verifyAddr("EQBBlraAps0OZaB9Q8ePQn2wVAaL1G411A-dNppyWe3X3GIT"a, 0, 29666621803903557832193058147214384979915773445007872807927344851911086823388); - verifyAddr("EQBiASqUqaVizrozLRbszkWC2kETbkhpO2qniDVDPPg2_0W8"a, 0, 44328719889509369519441680467651025944540360433148852643949783408843779749631); - verifyAddr("EQBu2Q1EO8gIoNA1qoGWnHUudKfmqlKEDTQE-DxN-_4sdg14"a, 0, 50137910719490808065414827264266674858051167131188257457782826342827836714102); - verifyAddr("EQA5bvxWd5-q2vJUVqR9AlbEIfdFysLR0PXGgVlBf8x5hWuF"a, 0, 25977927117604457079092522008276392864656238504700352770597256138254994667909); - verifyAddr("EQBguMSHjFv5bfoOdshr3ruS9ymSZzhRKMovoNrxGxZXvmee"a, 0, 43748489720571123896506696370504498290006245978262404519821633796370658121662); - verifyAddr("EQAxL0oF1-zNgimPKthbDnYS4xj94rHtfNRN7_Pd1r2LNNv3"a, 0, 22246882279393590648219842750911786376759362211171398419754426796438233910068); - verifyAddr("EQANX1uRKGZfyPIwEaIXrR0ZOqadct5q10dvKxWIxx7SQqzW"a, 0, 6048549475100840191738010856156544571222758030966479209409932714701987172930); - verifyAddr("EQBitdFDoU5DWSjfKq7AsO29RIwAnBzzvcVVSn5ekQoB9Liv"a, 0, 44647902768175374073183447303109856895983123510911038181495138821771906122228); - verifyAddr("EQBgbux7VSjqJHP7ByRK1q4QuVZbpSCesNgvz5qad3lfXX_B"a, 0, 43618018778298854282398238948198420936771670943015013768514626213000552996701); - verifyAddr("EQDisBd8U7M3CEOZ8gcWCdetdmJi3AI31zIT5qBwOdmUbsxY"a, 0, 102533830955233207294921564956803510155400341370448300698800842506363763004526); - verifyAddr("EQAZpn_eynVlf7Ii2d6jP_p1URPrdF9F3S7DiudQyelkjzwE"a, 0, 11602000355550451044739442929923326898313570892134000961608306166632391730319); - verifyAddr("EQDE0HBgfkOiqHezLtExBGTvOs8eitthHQosBjW3BmDy1y2K"a, 0, 89021598108837008984355105304701054698583123510131754065320641619941010764503); - verifyAddr("EQDyT36zktBN9PVWvZ1joRxhIfEUgCPt4F2isa-enUA_d6CP"a, 0, 109600164736599393471831241268953938618560132398692391517933933264745646800759); - verifyAddr("EQDSMUGwt25IQd3_yHjI03F71G8Kp2GMaMEv2TiWoTKbsyRH"a, 0, 95072727086440754059372943502908629555499501854161516009430039520728770059187); - verifyAddr("EQAgK1EcrvEuL9sCtoj3cNhVNOuf3lo5GIPE2gn1fwZZYB3j"a, 0, 14550545393206146289454646242321274637527057595221202748348667645886114191712); - verifyAddr("EQCDKqL5w_6MD-Z7AOButu-uR-ZJTsgNU1fu464hn9grY81U"a, 0, 59328315557704100696483472039557119625141880163887490602190749720459366378339); - verifyAddr("EQB1aVMyFBhnlYXmQjsma0S63kvxKU7ccZKFNCFTwX7ASPv4"a, 0, 53106696421104300082516512931084483581353095629408473618166869610568148238408); - verifyAddr("EQBbjrXHoxDyh1ZYGBdBoQgLaScxW6pZR1hEhJC8BqF-5Kgq"a, 0, 41412616102566803060532874463898939692666425753852274254609049615175463829220); - verifyAddr("EQC-QeZ13QP0lszxNKt380fCWuaV94vwC/bfuqmrlg1/fJPA"a, 0, 86055876869280374285292827775555707420719385459150221433115419095878595346300); - verifyAddr("EQAiUwpF27vXCngqNhf_TQ5E_06ah0G4zuSrnfU7CLLaht5H"a, 0, 15525356059048115813946213102829493539706126913595626308144289257869196581510); - verifyAddr("EQBqiVjmhe2iVGmgOSDO1FGjSiz_AMtb1w7lLEiP4XIF_SFy"a, 0, 48187833566271418625754761625661652107159264793429628379411792200127405491709); - verifyAddr("EQDmwvaK2d_SbaPdpOM60effPWeKsksgDVwFPEyxuftM396K"a, 0, 104376425077737068747642645125299653296942252727305637929378053253273342397663); - verifyAddr("EQDWtPZZgF7wvIMUHZQojuD3utiuivsW7WslRJ33dgv-5yc8"a, 0, 97114682311034709685427168495629428400170984047839002197324103884924936519399); - verifyAddr("EQAA7z0JI0JKqbN-1uENKz9JrxIO5ZRY-ehMeg9fPncx50Ck"a, 0, 422697701361909095759185681783393186844038628935759044330165207027374567911); - verifyAddr("EQBVUHRoCq6coQYUwOAhGSoAmQ6Mpm7dFlDYon6HMgWV8Ftr"a, 0, 38588743302295548905191533977469452945717219128199196974980570837505276220912); - verifyAddr("EQCTdvDCf0bA5dOPI1-44tB2ZfNcMGiklzvg27TovgDEqM6E"a, 0, 66700138358140658950710678965721715920748906761125730971082529064117803730088); - verifyAddr("EQBDBKE5WGKIlnoi3OOzw7vkKKIX55eWjPvgxJWwek8AyL2J"a, 0, 30313140970524770883308749215942283658935592719811899513010665548955593408712); - verifyAddr("EQAvCSyLCo21GrqLAifdov4WkOxuGQCjMRxgF1cXSaNzLHZe"a, 0, 21274912932379789207153885262858116665851037273450532982121514600400844714796); - verifyAddr("EQCsLpDeHB2qpRbmsCb_0xmsYVNx1NgeYrvHGT1TDrHkDgL4"a, 0, 77880084760844670718511036557364450189323549135231036747480176919181282894862); - verifyAddr("EQCTQ8kPwyX92r48gCIL_pLN_RcQT9ghZygnmDTYkOkuW_j5"a, 0, 66609755171046741472463433430016629628609840960137226492652665879987546041947); - verifyAddr("EQCTrFRSHt-tfk7WxK9ZHQmqLcgxXxTK7wGfCEbqgY2W9Mcx"a, 0, 66794468397542182731534559853537484892417154018190804733043974345563210356468); - verifyAddr("EQCv28y49GdaLncoclv0ISdDlMUY_cxDPGNWFCPT8t4GuqUJ"a, 0, 79543100951881731989812212377176715376834409392116644269458867858071577560762); - verifyAddr("EQCVL-k6deDR56Z8pcb0Btg0lGfaivOGfdDCD1vvyRsyL9vS"a, 0, 67479265933941008511790471646564661743401752930295407567346938670637286896175); - verifyAddr("EQD6t2dXDjZxF1DqunKF-8dEWivJdliY_0FYiCXnthuqnDCa"a, 0, 113402258385556889021060606279033166272577193563727959698596277924908309916316); - verifyAddr("EQDE98XNzXiPq7VnbJJ2M4-Ht3tX_OWR0xUTTnDC8NObLmyU"a, 0, 89091094739778473356272490822716056624384395256388481953562551087642791090990); - verifyAddr("EQDfeRDE1TDhwt478CDR0Q7MDwqcTUhfjqyTT59mgoAaF6f7"a, 0, 101079669463449311486034260688909914923832300293253430457119371423825321269783); - verifyAddr("EQDijcEyUKa-QgCbeGlggQk1uBtt2ZRHyW4Y4gB4R6MN6RLW"a, 0, 102473162609487797404330889623966425536887610061087715571345738626121871855081); - verifyAddr("EQDOtFOt41skbjBkZF89oYXpoDECjlxIzD-ShWAOYyzuxqLA"a, 0, 93495056812773926196963707371665512785268729004579280701087533371037976424134); - verifyAddr("EQDuJKSFWU7AYqH6KLFfAbYvMuz346eWmJvG6_2NYE42_B4T"a, 0, 107715199938628393100813870735031557263256555616273999363057194279168054802172); - verifyAddr("EQDwGu4vFv1e3wn8min_iy7OPJXegOYTFQ5bZFZ5a5ZPiBpX"a, 0, 108602665568837301744601989570019709742180613578164394799026726718721456754568); - verifyAddr("EQC4G2ph6AS_mD_-cIv4aIYm1z5jAgCW_TTDEr72ygXOP2X-"a, 0, 83274003234732023403481554420859495155084746906198543572711543697320249249343); - verifyAddr("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"a, 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); - verifyAddr("EQDoIA20MF1qEcSPtROdCu5ukGx9dVjgMeJh1oQ4A4cf_Jif"a, 0, 104993214557977037193613824776415934089204193426692473563548548423424814817276); - verifyAddr("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"a, 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); - verifyAddr("EQClLO4EnZ_rTyV1GVpWy53pLgWJRki5c4ZzuM_1O_ClBkO9"a, 0, 74711004027159342540251007601464500186374346239921204216319145006974068892934); - verifyAddr("EQDmkj65Ab_m0aZaW8IpKw4kYqIgITw_HRstYEkVQ6NIYCyW"a, 0, 104290347741656803921830951060768893809692975574470790497562993373950614128736); - verifyAddr("EQCqNTwAYUNhPFS0RgqZoTLGJcQQxbAJ7csUo4YO3_TONLab"a, 0, 76987241268612358571638783428744566580605181728938801022059780105627411729972); - verifyAddr("EQCL3DmCynaRK7-vsfeNmd4Jj-UxAIHPvA4qS2xwaL6UpLbF"a, 0, 63260589232981964910240894899061676480139492286430589202252472895352724165796); - verifyAddr("EQDbU1SVEjBE73oUqgAoM9gDcShUkM5EC2PgoCjuwVUKo-Ee"a, 0, 99203745911752606845646497420891218522647962685916739950275357890977532807843); - verifyAddr("EQD02VdcF4TDbCKLLhZJ39NQTu6aWq2LnLjp0oXqbNu_BANK"a, 0, 110748343802097970709980079967961144373090790244250392237586606542170934198020); - verifyAddr("EQBynBO23ywHy_CgarY9NK9FTz0yDsG82PtcbSTQgGoXwiuA"a, 0, 51839428943991432793039248316067731096592274748149794482308513726460953499586); - verifyAddr("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"a, 0, 9183547432069678364603018431103042146626948674383548774683927217595824907333); - verifyAddr("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"a, 0, 45985353862647206060987594732861817093328871106941773337270673759241903247880); - verifyAddr("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr("kQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPgpP"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr("kf-Dfdg-YQXaR2Q97gZJ4fGBtmV1DHOU1y1RPyyZZtRy_Ikh"a, 255, 59475331506450494976393625198911249698879029820580340449086829444312920781564); - verifyAddr("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 0, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr("0:0000000000000000000000000000000000000000000000000000000000000000"a, 0, 0); - verifyAddr("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"a, 0, 115792089237316195423570985008687907853269984665640564039457584007913129639935); - verifyAddr("0:zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"a, 0, 23158417847463239084714197001737581570653996933128112807891516801582625927987); - verifyAddr("0:0000000000000000000000000000000000000000000000000000000000000000"a, 0, 0); - verifyAddr("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 1, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 9, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr("99:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 99, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr("-1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 255, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(stringAddressToSlice("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"), 255, 23158417847463239084714197001737581570653996933128112807891516801582625927987); + verifyAddr(stringAddressToSlice("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), 0, 0); + verifyAddr(stringAddressToSlice("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"), 0, 65607996509792174074532427555986248720836864382484024657400295821210434460432); + verifyAddr(stringAddressToSlice("UQCOgxbCOjOLH_cEuQdGgS23zBM5SrQQepMFedjK-oixYbis"), 0, 64460038539088394980732229180523693489682583665805557562964506609821558550881); + verifyAddr(stringAddressToSlice("EQDa4VOnTYlLvDJ0gZjNYm5PXfSmmtL6Vs6A_CZEtXCNICq_"), 0, 99002318936150612861744867526221033858534811876886359650897405270877291973920); + verifyAddr(stringAddressToSlice("Ef8BtXO9bcTMXjg9bgivKh4lhJmZWQPP6_rb9vfjlTP5FJtM"), 255, 772910975127952880303441415761050161913031788763061162001556772893733681428); + verifyAddr(stringAddressToSlice("Ef89xh-uy860-mCcvS8zcAUs8bApmxLGygDLEKjUk5RL-311"), 255, 27941138149036269893630478666581900122707382189183906805784676408403709676539); + verifyAddr(stringAddressToSlice("Ef_vA6yRfmt2P4UHnxlrQUZFcBnKux8mL2eMqBgpeMFPorr4"), 255, 108109262375472472702582493362335418330829651067377177643099076957184687427490); + verifyAddr(stringAddressToSlice("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), 255, 18502444830824300068094395885436326119386947594392869497312068745716154912158); + verifyAddr(stringAddressToSlice("Ef_fvrd0hBoVJUxoi7wH173Zk8NPiyVvxh5IoYSjEYZbOhsu"), 255, 101202732337223525952216789200341489000836292542250083765062769181728788863802); + verifyAddr(stringAddressToSlice("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"), 255, 46952625717497919357580310066854892621799390294920450816077086267929711460872); + verifyAddr(stringAddressToSlice("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"), 255, 48545777798729612074233611768739897492467685225150339217043102685589809464695); + verifyAddr(stringAddressToSlice("Ef9LynHHKgBxY6-l-W_dWN-CtGT2_ji5rN3EzOI-p9zWEfq6"), 255, 34281152017620085319078796986198022632548048219136747083019177301186013091345); + verifyAddr(stringAddressToSlice("Ef9hMd78gzSiVsK0zz0AHtEja8x1UoB_NDZMjn-l86NQK_2Y"), 255, 43962460814164090767878334494257755557842170134382045184921495822637115592747); + verifyAddr(stringAddressToSlice("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"), 255, 23557057702048801338698514499604413540742716310574705490458593067566768087319); + verifyAddr(stringAddressToSlice("Ef_fdIbThooPs4_r2DE_Z6ZsWycJdHLnsuKAJHTcbaZaipez"), 255, 101071650030310556115830521522496708686577365303530257137459798093298869361290); + verifyAddr(stringAddressToSlice("Ef_lva0qEiZhWrrZJl-IJxyCcTQmmTo71fIWyQ31HxJ8NurV"), 255, 103914771557158282349484109182290824591675204108148026180964788916630125182006); + verifyAddr(stringAddressToSlice("Ef8sMGKypw006AeRYqimLjmY2Ufp-SHk8C0ZJBNgVBlzw_Nr"), 255, 19987255184378161380023126214650814972824352533523055905552702178965809886147); + verifyAddr(stringAddressToSlice("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff+W72r5gqPrHF"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(stringAddressToSlice("EQCaSCHVak-jIc9ANutTAfHpZNM3YdGky7yaDzsTrg0WhFlm"), 0, 69783625181781015447914682554083924798054947959007050695795761257887453484676); + verifyAddr(stringAddressToSlice("EQBS9U3AfD15fGmOtRMXQAxcPVBwNuItfLcDni9fkbTyyNX0"), 0, 37523067738561024305547433298623118197038688994386001017161816416175242146504); + verifyAddr(stringAddressToSlice("EQBiMNL9qNWMAkJHuM0BFneYcuHL17kzS4pswpaEO-NGWrFG"), 0, 44412924025649114419413541526870954696667907029239618728289150652715284776538); + verifyAddr(stringAddressToSlice("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi"), 0, 9407242825041766837311851458322335726136775042891143504070507665010681354284); + verifyAddr(stringAddressToSlice("EQD-nhrinjv0B4LTgr0dRHTHwH1MOsgGhKBXJZd7vESMZUf1"), 0, 115166810931401616117484448645661180241548402534908005320733783571353775148133); + verifyAddr(stringAddressToSlice("EQAVD3Fni9I6j8XeSIl-wAGBEhqhame6OtAY0GScKT0D9X6f"), 0, 9525855215156855607080079714361451576383963668563135377495902959388099150837); + verifyAddr(stringAddressToSlice("EQC6ACq3VANZjqfRBy7JMHkpLwqQ9qyYJsCIGx1mYbQgxaKw"), 0, 84130484652351964071210477536969520113177637645401392541565606610268614566085); + verifyAddr(stringAddressToSlice("EQCIJLNFIko5CvpKn9oAkrDgLocDOoD4vwmHxNx_fsG_LkwW"), 0, 61579391178099797614367237687950512448308156724136883899001108680249616482094); + verifyAddr(stringAddressToSlice("EQCe4AYIBce1pAk2qJJPSs1OzyZRlKjkfq8zuC8D7erv6DUP"), 0, 71861245445432818728925844931259040612664802586395398157190478191760507596776); + verifyAddr(stringAddressToSlice("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"), 0, 78559023162479717496981724991265882229440558807791659796411897368395464230649); + verifyAddr(stringAddressToSlice("EQBBlraAps0OZaB9Q8ePQn2wVAaL1G411A-dNppyWe3X3GIT"), 0, 29666621803903557832193058147214384979915773445007872807927344851911086823388); + verifyAddr(stringAddressToSlice("EQBiASqUqaVizrozLRbszkWC2kETbkhpO2qniDVDPPg2_0W8"), 0, 44328719889509369519441680467651025944540360433148852643949783408843779749631); + verifyAddr(stringAddressToSlice("EQBu2Q1EO8gIoNA1qoGWnHUudKfmqlKEDTQE-DxN-_4sdg14"), 0, 50137910719490808065414827264266674858051167131188257457782826342827836714102); + verifyAddr(stringAddressToSlice("EQA5bvxWd5-q2vJUVqR9AlbEIfdFysLR0PXGgVlBf8x5hWuF"), 0, 25977927117604457079092522008276392864656238504700352770597256138254994667909); + verifyAddr(stringAddressToSlice("EQBguMSHjFv5bfoOdshr3ruS9ymSZzhRKMovoNrxGxZXvmee"), 0, 43748489720571123896506696370504498290006245978262404519821633796370658121662); + verifyAddr(stringAddressToSlice("EQAxL0oF1-zNgimPKthbDnYS4xj94rHtfNRN7_Pd1r2LNNv3"), 0, 22246882279393590648219842750911786376759362211171398419754426796438233910068); + verifyAddr(stringAddressToSlice("EQANX1uRKGZfyPIwEaIXrR0ZOqadct5q10dvKxWIxx7SQqzW"), 0, 6048549475100840191738010856156544571222758030966479209409932714701987172930); + verifyAddr(stringAddressToSlice("EQBitdFDoU5DWSjfKq7AsO29RIwAnBzzvcVVSn5ekQoB9Liv"), 0, 44647902768175374073183447303109856895983123510911038181495138821771906122228); + verifyAddr(stringAddressToSlice("EQBgbux7VSjqJHP7ByRK1q4QuVZbpSCesNgvz5qad3lfXX_B"), 0, 43618018778298854282398238948198420936771670943015013768514626213000552996701); + verifyAddr(stringAddressToSlice("EQDisBd8U7M3CEOZ8gcWCdetdmJi3AI31zIT5qBwOdmUbsxY"), 0, 102533830955233207294921564956803510155400341370448300698800842506363763004526); + verifyAddr(stringAddressToSlice("EQAZpn_eynVlf7Ii2d6jP_p1URPrdF9F3S7DiudQyelkjzwE"), 0, 11602000355550451044739442929923326898313570892134000961608306166632391730319); + verifyAddr(stringAddressToSlice("EQDE0HBgfkOiqHezLtExBGTvOs8eitthHQosBjW3BmDy1y2K"), 0, 89021598108837008984355105304701054698583123510131754065320641619941010764503); + verifyAddr(stringAddressToSlice("EQDyT36zktBN9PVWvZ1joRxhIfEUgCPt4F2isa-enUA_d6CP"), 0, 109600164736599393471831241268953938618560132398692391517933933264745646800759); + verifyAddr(stringAddressToSlice("EQDSMUGwt25IQd3_yHjI03F71G8Kp2GMaMEv2TiWoTKbsyRH"), 0, 95072727086440754059372943502908629555499501854161516009430039520728770059187); + verifyAddr(stringAddressToSlice("EQAgK1EcrvEuL9sCtoj3cNhVNOuf3lo5GIPE2gn1fwZZYB3j"), 0, 14550545393206146289454646242321274637527057595221202748348667645886114191712); + verifyAddr(stringAddressToSlice("EQCDKqL5w_6MD-Z7AOButu-uR-ZJTsgNU1fu464hn9grY81U"), 0, 59328315557704100696483472039557119625141880163887490602190749720459366378339); + verifyAddr(stringAddressToSlice("EQB1aVMyFBhnlYXmQjsma0S63kvxKU7ccZKFNCFTwX7ASPv4"), 0, 53106696421104300082516512931084483581353095629408473618166869610568148238408); + verifyAddr(stringAddressToSlice("EQBbjrXHoxDyh1ZYGBdBoQgLaScxW6pZR1hEhJC8BqF-5Kgq"), 0, 41412616102566803060532874463898939692666425753852274254609049615175463829220); + verifyAddr(stringAddressToSlice("EQC-QeZ13QP0lszxNKt380fCWuaV94vwC/bfuqmrlg1/fJPA"), 0, 86055876869280374285292827775555707420719385459150221433115419095878595346300); + verifyAddr(stringAddressToSlice("EQAiUwpF27vXCngqNhf_TQ5E_06ah0G4zuSrnfU7CLLaht5H"), 0, 15525356059048115813946213102829493539706126913595626308144289257869196581510); + verifyAddr(stringAddressToSlice("EQBqiVjmhe2iVGmgOSDO1FGjSiz_AMtb1w7lLEiP4XIF_SFy"), 0, 48187833566271418625754761625661652107159264793429628379411792200127405491709); + verifyAddr(stringAddressToSlice("EQDmwvaK2d_SbaPdpOM60effPWeKsksgDVwFPEyxuftM396K"), 0, 104376425077737068747642645125299653296942252727305637929378053253273342397663); + verifyAddr(stringAddressToSlice("EQDWtPZZgF7wvIMUHZQojuD3utiuivsW7WslRJ33dgv-5yc8"), 0, 97114682311034709685427168495629428400170984047839002197324103884924936519399); + verifyAddr(stringAddressToSlice("EQAA7z0JI0JKqbN-1uENKz9JrxIO5ZRY-ehMeg9fPncx50Ck"), 0, 422697701361909095759185681783393186844038628935759044330165207027374567911); + verifyAddr(stringAddressToSlice("EQBVUHRoCq6coQYUwOAhGSoAmQ6Mpm7dFlDYon6HMgWV8Ftr"), 0, 38588743302295548905191533977469452945717219128199196974980570837505276220912); + verifyAddr(stringAddressToSlice("EQCTdvDCf0bA5dOPI1-44tB2ZfNcMGiklzvg27TovgDEqM6E"), 0, 66700138358140658950710678965721715920748906761125730971082529064117803730088); + verifyAddr(stringAddressToSlice("EQBDBKE5WGKIlnoi3OOzw7vkKKIX55eWjPvgxJWwek8AyL2J"), 0, 30313140970524770883308749215942283658935592719811899513010665548955593408712); + verifyAddr(stringAddressToSlice("EQAvCSyLCo21GrqLAifdov4WkOxuGQCjMRxgF1cXSaNzLHZe"), 0, 21274912932379789207153885262858116665851037273450532982121514600400844714796); + verifyAddr(stringAddressToSlice("EQCsLpDeHB2qpRbmsCb_0xmsYVNx1NgeYrvHGT1TDrHkDgL4"), 0, 77880084760844670718511036557364450189323549135231036747480176919181282894862); + verifyAddr(stringAddressToSlice("EQCTQ8kPwyX92r48gCIL_pLN_RcQT9ghZygnmDTYkOkuW_j5"), 0, 66609755171046741472463433430016629628609840960137226492652665879987546041947); + verifyAddr(stringAddressToSlice("EQCTrFRSHt-tfk7WxK9ZHQmqLcgxXxTK7wGfCEbqgY2W9Mcx"), 0, 66794468397542182731534559853537484892417154018190804733043974345563210356468); + verifyAddr(stringAddressToSlice("EQCv28y49GdaLncoclv0ISdDlMUY_cxDPGNWFCPT8t4GuqUJ"), 0, 79543100951881731989812212377176715376834409392116644269458867858071577560762); + verifyAddr(stringAddressToSlice("EQCVL-k6deDR56Z8pcb0Btg0lGfaivOGfdDCD1vvyRsyL9vS"), 0, 67479265933941008511790471646564661743401752930295407567346938670637286896175); + verifyAddr(stringAddressToSlice("EQD6t2dXDjZxF1DqunKF-8dEWivJdliY_0FYiCXnthuqnDCa"), 0, 113402258385556889021060606279033166272577193563727959698596277924908309916316); + verifyAddr(stringAddressToSlice("EQDE98XNzXiPq7VnbJJ2M4-Ht3tX_OWR0xUTTnDC8NObLmyU"), 0, 89091094739778473356272490822716056624384395256388481953562551087642791090990); + verifyAddr(stringAddressToSlice("EQDfeRDE1TDhwt478CDR0Q7MDwqcTUhfjqyTT59mgoAaF6f7"), 0, 101079669463449311486034260688909914923832300293253430457119371423825321269783); + verifyAddr(stringAddressToSlice("EQDijcEyUKa-QgCbeGlggQk1uBtt2ZRHyW4Y4gB4R6MN6RLW"), 0, 102473162609487797404330889623966425536887610061087715571345738626121871855081); + verifyAddr(stringAddressToSlice("EQDOtFOt41skbjBkZF89oYXpoDECjlxIzD-ShWAOYyzuxqLA"), 0, 93495056812773926196963707371665512785268729004579280701087533371037976424134); + verifyAddr(stringAddressToSlice("EQDuJKSFWU7AYqH6KLFfAbYvMuz346eWmJvG6_2NYE42_B4T"), 0, 107715199938628393100813870735031557263256555616273999363057194279168054802172); + verifyAddr(stringAddressToSlice("EQDwGu4vFv1e3wn8min_iy7OPJXegOYTFQ5bZFZ5a5ZPiBpX"), 0, 108602665568837301744601989570019709742180613578164394799026726718721456754568); + verifyAddr(stringAddressToSlice("EQC4G2ph6AS_mD_-cIv4aIYm1z5jAgCW_TTDEr72ygXOP2X-"), 0, 83274003234732023403481554420859495155084746906198543572711543697320249249343); + verifyAddr(stringAddressToSlice("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"), 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); + verifyAddr(stringAddressToSlice("EQDoIA20MF1qEcSPtROdCu5ukGx9dVjgMeJh1oQ4A4cf_Jif"), 0, 104993214557977037193613824776415934089204193426692473563548548423424814817276); + verifyAddr(stringAddressToSlice("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"), 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); + verifyAddr(stringAddressToSlice("EQClLO4EnZ_rTyV1GVpWy53pLgWJRki5c4ZzuM_1O_ClBkO9"), 0, 74711004027159342540251007601464500186374346239921204216319145006974068892934); + verifyAddr(stringAddressToSlice("EQDmkj65Ab_m0aZaW8IpKw4kYqIgITw_HRstYEkVQ6NIYCyW"), 0, 104290347741656803921830951060768893809692975574470790497562993373950614128736); + verifyAddr(stringAddressToSlice("EQCqNTwAYUNhPFS0RgqZoTLGJcQQxbAJ7csUo4YO3_TONLab"), 0, 76987241268612358571638783428744566580605181728938801022059780105627411729972); + verifyAddr(stringAddressToSlice("EQCL3DmCynaRK7-vsfeNmd4Jj-UxAIHPvA4qS2xwaL6UpLbF"), 0, 63260589232981964910240894899061676480139492286430589202252472895352724165796); + verifyAddr(stringAddressToSlice("EQDbU1SVEjBE73oUqgAoM9gDcShUkM5EC2PgoCjuwVUKo-Ee"), 0, 99203745911752606845646497420891218522647962685916739950275357890977532807843); + verifyAddr(stringAddressToSlice("EQD02VdcF4TDbCKLLhZJ39NQTu6aWq2LnLjp0oXqbNu_BANK"), 0, 110748343802097970709980079967961144373090790244250392237586606542170934198020); + verifyAddr(stringAddressToSlice("EQBynBO23ywHy_CgarY9NK9FTz0yDsG82PtcbSTQgGoXwiuA"), 0, 51839428943991432793039248316067731096592274748149794482308513726460953499586); + verifyAddr(stringAddressToSlice("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(stringAddressToSlice("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"), 0, 9183547432069678364603018431103042146626948674383548774683927217595824907333); + verifyAddr(stringAddressToSlice("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"), 0, 45985353862647206060987594732861817093328871106941773337270673759241903247880); + verifyAddr(stringAddressToSlice("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(stringAddressToSlice("kQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPgpP"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(stringAddressToSlice("kf-Dfdg-YQXaR2Q97gZJ4fGBtmV1DHOU1y1RPyyZZtRy_Ikh"), 255, 59475331506450494976393625198911249698879029820580340449086829444312920781564); + verifyAddr(stringAddressToSlice("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 0, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(stringAddressToSlice("0:0000000000000000000000000000000000000000000000000000000000000000"), 0, 0); + verifyAddr(stringAddressToSlice("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"), 0, 115792089237316195423570985008687907853269984665640564039457584007913129639935); + verifyAddr(stringAddressToSlice("0:zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"), 0, 23158417847463239084714197001737581570653996933128112807891516801582625927987); + verifyAddr(stringAddressToSlice("0:0000000000000000000000000000000000000000000000000000000000000000"), 0, 0); + verifyAddr(stringAddressToSlice("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 1, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(stringAddressToSlice("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 9, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(stringAddressToSlice("99:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 99, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(stringAddressToSlice("-1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 255, 37304138005561100291416421295333982606153966175434134130332440738068913455320); return cc1.isSliceBitsEqual(cc2); } diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index c7c4f69465..af8c6edeb0 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -3,27 +3,31 @@ get ascii_slice(): slice { } get raw_slice(): slice { - return "abcdef"s; + return stringHexToSlice("abcdef"); } get addr_slice(): slice { - return "Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"a; + return stringAddressToSlice("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"); } get string_hex(): int { - return "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"u; + return stringToBase256("ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"); } get fun string_minihash(): int { // 'get' and 'get fun' both possible - return "transfer(slice, int)"h; + return stringSha256_32("transfer(slice, int)"); } get fun string_maxihash(): int { - return "transfer(slice, int)"H; + return stringSha256("transfer(slice, int)"); } get fun string_crc32(): int { - return "transfer(slice, int)"c; + return stringCrc32("transfer(slice, int)"); +} + +get fun string_crc16(): int { + return stringCrc16("transfer(slice, int)"); } @pure @@ -35,6 +39,15 @@ asm "ENDC" "CTOS"; fun sdeq(s1: slice, s2: slice): int asm "SDEQ"; +fun calcSliceBytes(slice: slice): tuple { + var t = createEmptyTuple(); + var n = 0; + while (n = slice.getRemainingBitsCount()) { + t.tuplePush(slice.loadUint(min(8, n))); + } + return t; +} + fun main() { var s_ascii: slice = ascii_slice(); var s_raw: slice = raw_slice(); @@ -42,7 +55,8 @@ fun main() { var i_hex: int = string_hex(); var i_mini: int = string_minihash(); var i_maxi: int = string_maxihash(); - var i_crc: int = string_crc32(); + var i_crc32: int = string_crc32(); + var i_crc16: int = string_crc16(); assert(sdeq(s_ascii, newc().storeUint(0x737472696E67, 12 * 4).endcs())) throw 101; assert(sdeq(s_raw, newc().storeUint(0xABCDEF, 6 * 4).endcs())) throw 102; assert(sdeq(s_addr, newc().storeUint(4, 3).storeInt(-1, 8) @@ -50,12 +64,19 @@ fun main() { assert(i_hex == 0x4142434445464748494A4B4C4D4E4F505152535455565758595A303132333435) throw 104; assert(i_mini == 0x7a62e8a8) throw 105; assert(i_maxi == 0x7a62e8a8ebac41bd6de16c65e7be363bc2d2cbc6a0873778dead4795c13db979) throw 106; - assert(i_crc == 2235694568) throw 107; + assert(i_crc32 == 2235694568 && i_crc32 == 0x8541fde8) throw 107; + assert(i_crc16 == 52550 && i_crc16 == 0xCD46) throw 108; return 0; } +@method_id(101) +fun test1() { + return calcSliceBytes("ABCD"); +} + /** -@testcase | 0 | | 0 +@testcase | 0 | | 0 +@testcase | 101 | | [ 65 66 67 68 ] -@code_hash 13830542019509784148027107880226447201604257839069192762244575629978154217223 +@code_hash 38184847030631877916087987911699475358017315230885358090110033079289166112584 */ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 22974b1f56..94ef848c17 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -357,12 +357,7 @@ static AnyExprV parse_expr100(Lexer& lex) { case tok_string_const: { std::string_view str_val = lex.cur_str(); lex.next(); - char modifier = 0; - if (lex.tok() == tok_string_modifier) { - modifier = lex.cur_str()[0]; - lex.next(); - } - return createV(loc, str_val, modifier); + return createV(loc, str_val); } case tok_underscore: { lex.next(); @@ -927,7 +922,7 @@ static AnyV parse_asm_func_body(Lexer& lex, V param_list) { lex.check(tok_string_const, "\"ASM COMMAND\""); while (lex.tok() == tok_string_const) { std::string_view asm_command = lex.cur_str(); - asm_commands.push_back(createV(lex.cur_location(), asm_command, 0)); + asm_commands.push_back(createV(lex.cur_location(), asm_command)); lex.next(); } lex.expect(tok_semicolon, "`;`"); @@ -1142,7 +1137,7 @@ static AnyV parse_import_directive(Lexer& lex) { if (rel_filename.empty()) { lex.error("imported file name is an empty string"); } - auto v_str = createV(lex.cur_location(), rel_filename, 0); + auto v_str = createV(lex.cur_location(), rel_filename); lex.next(); return createV(loc, v_str); // semicolon is not necessary } diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 16bbbeb8d1..bf48c2389b 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -80,7 +80,7 @@ class ASTReplicatorFunction : public ASTReplicator { return createV(v->loc, v->intval, v->orig_str); } virtual V clone(V v) { - return createV(v->loc, v->str_val, v->modifier); + return createV(v->loc, v->str_val); } virtual V clone(V v) { return createV(v->loc, v->bool_val); diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index a7f260de80..07f03745ca 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -136,11 +136,7 @@ class ASTStringifier final : public ASTVisitor { case ast_int_const: return static_cast(v->as()->orig_str); case ast_string_const: - if (char modifier = v->as()->modifier) { - return "\"" + static_cast(v->as()->str_val) + "\"" + std::string(1, modifier); - } else { - return "\"" + static_cast(v->as()->str_val) + "\""; - } + return "\"" + static_cast(v->as()->str_val) + "\""; case ast_bool_const: return v->as()->bool_val ? "true" : "false"; case ast_dot_access: { diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 26eaacd59c..ac93d21c30 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -126,6 +126,10 @@ void Vertex::assign_sym(const Symbol* sym) { this->sym = sym; } +void Vertex::assign_literal_value(ConstantValue&& literal_value) { + this->literal_value = std::move(literal_value); +} + void Vertex::assign_fun_ref(FunctionPtr fun_ref) { this->fun_maybe = fun_ref; } diff --git a/tolk/ast.h b/tolk/ast.h index 9b7c5d1a58..385e1ff895 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -458,25 +458,18 @@ struct Vertex final : ASTExprLeaf { template<> // ast_string_const is a string literal in double quotes or """ when multiline -// examples: "asdf" / "Ef8zMz..."a / "to_calc_crc32_from"c -// an optional modifier specifies how a string is parsed (probably, like an integer) +// examples: "asdf" / "LTIME" (in asm body) / stringCrc32("asdf") (as an argument) // note, that TVM doesn't have strings, it has only slices, so "hello" has type slice struct Vertex final : ASTExprLeaf { std::string_view str_val; - char modifier; + ConstantValue literal_value; // value of type `slice`, calculated after type inferring, at constants evaluation - bool is_bitslice() const { - char m = modifier; - return m == 0 || m == 's' || m == 'a'; - } - bool is_intval() const { - char m = modifier; - return m == 'u' || m == 'h' || m == 'H' || m == 'c'; - } + Vertex* mutate() const { return const_cast(this); } + void assign_literal_value(ConstantValue&& literal_value); - Vertex(SrcLocation loc, std::string_view str_val, char modifier) + Vertex(SrcLocation loc, std::string_view str_val) : ASTExprLeaf(ast_string_const, loc) - , str_val(str_val), modifier(modifier) {} + , str_val(str_val) {} }; template<> diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 9ef4527aee..75ead3458f 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1072,7 +1072,8 @@ AsmOp compile_tuple_set_at(std::vector& res, std::vector& ar } // fun ton(amount: slice): coins; ton("0.05") replaced by 50000000 at compile-time -AsmOp compile_int_to_ton_coins(std::vector&, std::vector&, SrcLocation) { +// same for stringCrc32(constString: slice) and others +AsmOp compile_time_only_function(std::vector&, std::vector&, SrcLocation) { // all ton() invocations are constants, replaced by integers; no dynamic values allowed, no work at runtime tolk_assert(false); return AsmOp::Nop(); @@ -1219,11 +1220,36 @@ void define_builtins() { compile_throw_if_unless, 0); + // compile-time only functions, evaluated essentially at compile-time, no runtime implementation + // they are placed in stdlib and marked as `builtin` + // note their parameter being `unknown`: in order to `ton(1)` pass type inferring but fire a more gentle error later + define_builtin_func("ton", {TypeDataUnknown::create()}, TypeDataCoins::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + define_builtin_func("stringCrc32", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + define_builtin_func("stringCrc16", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + define_builtin_func("stringSha256", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + define_builtin_func("stringSha256_32", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + define_builtin_func("stringToBase256", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + define_builtin_func("stringAddressToSlice", {TypeDataUnknown::create()}, TypeDataSlice::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + define_builtin_func("stringHexToSlice", {TypeDataUnknown::create()}, TypeDataSlice::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + // functions from stdlib marked as `builtin`, implemented at compiler level for optimizations // (for example, `loadInt(1)` is `1 LDI`, but `loadInt(n)` for non-constant requires it be on a stack and `LDIX`) - define_builtin_func("ton", {TypeDataUnknown::create()}, TypeDataCoins::create(), nullptr, // `unknown` to pass inferring for `ton(1)` (to fire a better error later) - compile_int_to_ton_coins, - FunctionData::flagMarkedAsPure); define_builtin_func("mulDivFloor", ParamsInt3, Int, nullptr, std::bind(compile_muldiv, _1, _2, _3, -1), FunctionData::flagMarkedAsPure); diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index 5c3b3b19a0..7daebc1da3 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -24,6 +24,14 @@ namespace tolk { +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_const_string_required(SrcLocation loc, std::string_view f_name, std::string_view example_arg) { + std::string name = static_cast(f_name); + std::string example = static_cast(example_arg); + throw ParseError(loc, "function `" + name + "` requires a constant string, like `" + name + "(\"" + example + "\")`"); +} + + // parse address like "EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5" // based on unpack_std_smc_addr() from block.cpp // (which is not included to avoid linking with ton_crypto) @@ -44,10 +52,10 @@ static bool parse_friendly_address(const char packed[48], ton::WorkchainId& work // parse address like "0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8" // based on StdAddress::parse_addr() from block.cpp // (which is not included to avoid linking with ton_crypto) -static bool parse_raw_address(const std::string& acc_string, int& workchain, ton::StdSmcAddress& addr) { +static bool parse_raw_address(std::string_view acc_string, int& workchain, ton::StdSmcAddress& addr) { size_t pos = acc_string.find(':'); if (pos != std::string::npos) { - td::Result r_wc = td::to_integer_safe(acc_string.substr(0, pos)); + td::Result r_wc = td::to_integer_safe(td::Slice(acc_string.data(), pos)); if (r_wc.is_error()) { return false; } @@ -82,93 +90,8 @@ static bool parse_raw_address(const std::string& acc_string, int& workchain, ton return true; } - -static std::string parse_vertex_string_const_as_slice(V v) { - std::string str = static_cast(v->str_val); - switch (v->modifier) { - case 0: { - return td::hex_encode(str); - } - case 's': { - unsigned char buff[128]; - long bits = td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.data(), str.data() + str.size()); - if (bits < 0) { - v->error("invalid hex bitstring constant '" + str + "'"); - } - return str; - } - case 'a': { // MsgAddress - ton::WorkchainId workchain; - ton::StdSmcAddress addr; - bool correct = (str.size() == 48 && parse_friendly_address(str.data(), workchain, addr)) || - (str.size() != 48 && parse_raw_address(str, workchain, addr)); - if (!correct) { - v->error("invalid standard address '" + str + "'"); - } - if (workchain < -128 || workchain >= 128) { - v->error("anycast addresses not supported"); - } - - unsigned char data[3 + 8 + 256]; // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; - td::bitstring::bits_store_long_top(data, 0, static_cast(4) << (64 - 3), 3); - td::bitstring::bits_store_long_top(data, 3, static_cast(workchain) << (64 - 8), 8); - td::bitstring::bits_memcpy(data, 3 + 8, addr.bits().ptr, 0, ton::StdSmcAddress::size()); - return td::BitSlice{data, sizeof(data)}.to_hex(); - } - default: - tolk_assert(false); - } -} - -static td::RefInt256 parse_vertex_string_const_as_int(V v) { - std::string str = static_cast(v->str_val); - switch (v->modifier) { - case 'u': { - td::RefInt256 intval = td::hex_string_to_int256(td::hex_encode(str)); - if (str.empty()) { - v->error("empty integer ascii-constant"); - } - if (intval.is_null()) { - v->error("too long integer ascii-constant"); - } - return intval; - } - case 'h': - case 'H': { - unsigned char hash[32]; - digest::hash_str(hash, str.data(), str.size()); - return td::bits_to_refint(hash, (v->modifier == 'h') ? 32 : 256, false); - } - case 'c': { - return td::make_refint(td::crc32(td::Slice{str})); - } - default: - tolk_assert(false); - } -} - -static ConstantValue parse_vertex_string_const(V v) { - return v->is_bitslice() - ? ConstantValue(parse_vertex_string_const_as_slice(v)) - : ConstantValue(parse_vertex_string_const_as_int(v)); -} - -// given `ton("0.05")` evaluate it to 50000000 -static ConstantValue parse_vertex_call_to_ton_function(V v) { - tolk_assert(v->get_num_args() == 1); // checked by type inferring - AnyExprV v_arg = v->get_arg(0)->get_expr(); - - std::string_view str; - if (auto as_string = v_arg->try_as(); as_string && !as_string->modifier) { - str = as_string->str_val; - } else { - // ton(SOME_CONST) is not supported - // ton(0.05) is not supported (it can't be represented in AST even) - } - if (str.empty()) { - v->error("function `ton` requires a constant string, like `ton(\"0.05\")`"); - } - +// internal helper: for `ton("0.05")`, parse string literal "0.05" to 50000000 +static td::RefInt256 parse_nanotons_as_floating_string(SrcLocation loc, std::string_view str) { bool is_negative = false; size_t i = 0; @@ -190,7 +113,7 @@ static ConstantValue parse_vertex_call_to_ton_function(V v) { char c = str[i]; if (c == '.') { if (seen_dot) { - v_arg->error("argument is not a valid number like \"0.05\""); + throw ParseError(loc, "argument is not a valid number like \"0.05\""); } seen_dot = true; } else if (c >= '0' && c <= '9') { @@ -201,7 +124,7 @@ static ConstantValue parse_vertex_call_to_ton_function(V v) { fractional_digits++; } } else { - v_arg->error("argument is not a valid number like \"0.05\""); + throw ParseError(loc, "argument is not a valid number like \"0.05\""); } } @@ -211,7 +134,99 @@ static ConstantValue parse_vertex_call_to_ton_function(V v) { } int64_t result = integer_part * 1000000000LL + fractional_part; - return ConstantValue(td::make_refint(is_negative ? -result : result)); + return td::make_refint(is_negative ? -result : result); +} + +static ConstantValue parse_vertex_string_const(V v) { + td::Slice str_slice = td::Slice(v->str_val.data(), v->str_val.size()); + return ConstantValue(td::hex_encode(str_slice)); +} + +// given `ton("0.05")` evaluate it to 50000000 +// given `stringCrc32("some_str")` evaluate it +// etc. +// currently, all compile-time functions accept 1 argument, a literal string +static ConstantValue parse_vertex_call_to_compile_time_function(V v, std::string_view f_name) { + tolk_assert(v->get_num_args() == 1); // checked by type inferring + AnyExprV v_arg = v->get_arg(0)->get_expr(); + + std::string_view str; + if (auto as_string = v_arg->try_as()) { + str = as_string->str_val; + } else { + // ton(SOME_CONST) is not supported + // ton(0.05) is not supported (it can't be represented in AST even) + // stringCrc32(SOME_CONST) / stringCrc32(some_var) also, it's compile-time literal-only + } + if (str.empty()) { + fire_error_const_string_required(v->loc, f_name, f_name == "ton" ? "0.05" : "some_str"); + } + + if (f_name == "ton") { + return ConstantValue(parse_nanotons_as_floating_string(v_arg->loc, str)); + } + + if (f_name == "stringCrc32") { // previously, postfix "..."c + return ConstantValue(td::make_refint(td::crc32(td::Slice{str.data(), str.size()}))); + } + + if (f_name == "stringCrc16") { // previously, there was no postfix in FunC, no way to calc at compile-time + return ConstantValue(td::make_refint(td::crc16(td::Slice{str.data(), str.size()}))); + } + + if (f_name == "stringSha256") { // previously, postfix "..."H + unsigned char hash[32]; + digest::hash_str(hash, str.data(), str.size()); + return ConstantValue(td::bits_to_refint(hash, 256, false)); + } + + if (f_name == "stringSha256_32") { // previously, postfix "..."h + unsigned char hash[32]; + digest::hash_str(hash, str.data(), str.size()); + return ConstantValue(td::bits_to_refint(hash, 32, false)); + } + + if (f_name == "stringAddressToSlice") { // previously, postfix "..."a + ton::WorkchainId workchain; + ton::StdSmcAddress addr; + bool correct = (str.size() == 48 && parse_friendly_address(str.data(), workchain, addr)) || + (str.size() != 48 && parse_raw_address(str, workchain, addr)); + if (!correct) { + v_arg->error("invalid standard address"); + } + if (workchain < -128 || workchain >= 128) { + v_arg->error("anycast addresses not supported"); + } + + unsigned char data[3 + 8 + 256]; // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; + td::bitstring::bits_store_long_top(data, 0, static_cast(4) << (64 - 3), 3); + td::bitstring::bits_store_long_top(data, 3, static_cast(workchain) << (64 - 8), 8); + td::bitstring::bits_memcpy(data, 3 + 8, addr.bits().ptr, 0, ton::StdSmcAddress::size()); + return ConstantValue(td::BitSlice{data, sizeof(data)}.to_hex()); + } + + if (f_name == "stringHexToSlice") { // previously, postfix "..."s + unsigned char buff[128]; + long bits = td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.data(), str.data() + str.size()); + if (bits < 0) { + v_arg->error("invalid hex bitstring constant"); + } + return ConstantValue(static_cast(str)); + } + + if (f_name == "stringToBase256") { // previously, postfix "..."u + td::RefInt256 intval = td::hex_string_to_int256(td::hex_encode(td::Slice(str.data(), str.size()))); + if (str.empty()) { + v_arg->error("empty integer ascii-constant"); + } + if (intval.is_null()) { + v_arg->error("too long integer ascii-constant"); + } + return ConstantValue(std::move(intval)); + } + + tolk_assert(false); + return ConstantValue(); } @@ -332,8 +347,8 @@ struct ConstantEvaluator { // `const a = ton("0.05")`, we met `ton("0.05")` static ConstantValue handle_function_call(V v) { - if (v->fun_maybe && v->fun_maybe->is_builtin_function() && v->fun_maybe->name == "ton") { - return parse_vertex_call_to_ton_function(v); + if (v->fun_maybe && v->fun_maybe->is_compile_time_only()) { + return parse_vertex_call_to_compile_time_function(v, v->fun_maybe->name); } v->error("not a constant expression"); } @@ -378,14 +393,16 @@ struct ConstantEvaluator { } }; -ConstantValue eval_string_const_considering_modifier(AnyExprV v_string) { - tolk_assert(v_string->type == ast_string_const); - return parse_vertex_string_const(v_string->as()); +ConstantValue eval_string_const_standalone(AnyExprV v_string) { + auto v = v_string->try_as(); + tolk_assert(v); + return parse_vertex_string_const(v); } -ConstantValue eval_call_to_ton_function(AnyExprV v_call) { - tolk_assert(v_call->type == ast_function_call && v_call->as()->fun_maybe->is_builtin_function()); - return parse_vertex_call_to_ton_function(v_call->as()); +ConstantValue eval_call_to_compile_time_function(AnyExprV v_call) { + auto v = v_call->try_as(); + tolk_assert(v && v->fun_maybe->is_compile_time_only()); + return parse_vertex_call_to_compile_time_function(v, v->fun_maybe->name); } void eval_and_assign_const_init_value(GlobalConstPtr const_ref) { diff --git a/tolk/constant-evaluator.h b/tolk/constant-evaluator.h index 4737352ea4..b065e6f49b 100644 --- a/tolk/constant-evaluator.h +++ b/tolk/constant-evaluator.h @@ -47,8 +47,8 @@ class ConstantValue { const std::string& as_slice() const { return std::get(value); } }; -ConstantValue eval_string_const_considering_modifier(AnyExprV v_string); -ConstantValue eval_call_to_ton_function(AnyExprV v_call); +ConstantValue eval_string_const_standalone(AnyExprV v_string); +ConstantValue eval_call_to_compile_time_function(AnyExprV v_call); void eval_and_assign_const_init_value(GlobalConstPtr const_ref); } // namespace tolk diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index 86ae306a4b..961be1674a 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -23,7 +23,6 @@ namespace tolk { // By 'chunk' in lexer I mean a token or a list of tokens parsed simultaneously. // E.g., when we meet "str", ChunkString is called, it emits tok_string. -// E.g., when we meet "str"x, ChunkString emits not only tok_string, but tok_string_modifier. // E.g., when we meet //, ChunkInlineComment is called, it emits nothing (just skips a line). // We store all valid chunks lexers in a prefix tree (LexingTrie), see below. struct ChunkLexerBase { @@ -170,8 +169,8 @@ struct ChunkMultilineComment final : ChunkLexerBase { // A string, starting from " // Note, that there are no escape symbols inside: the purpose of strings in Tolk just doesn't need it. -// After a closing quote, a string modifier may be present, like "Ef8zMzMzMzMzMzMzMzMzMzM0vF"a. -// If present, it emits a separate tok_string_modifier. +// In FunC, a string might have ended with a modifier like `"..."c` +// It's not valid in Tolk, valid is `stringCrc32("...")` struct ChunkString final : ChunkLexerBase { bool parse(Lexer* lex) const override { const char* str_begin = lex->c_str(); @@ -187,12 +186,6 @@ struct ChunkString final : ChunkLexerBase { lex->skip_chars(1); lex->add_token(tok_string_const, str_val); - if (std::isalpha(lex->char_at())) { - std::string_view modifier_val(lex->c_str(), 1); - lex->skip_chars(1); - lex->add_token(tok_string_modifier, modifier_val); - } - return true; } }; diff --git a/tolk/lexer.h b/tolk/lexer.h index 58bc3640af..34c792ebcc 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -48,7 +48,6 @@ enum TokenType { tok_int_const, tok_string_const, - tok_string_modifier, tok_true, tok_false, tok_null, diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index d0c252847c..b26c9d2c4d 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1049,13 +1049,9 @@ static std::vector process_int_const(V v, CodeBlob& co } static std::vector process_string_const(V v, CodeBlob& code, TypePtr target_type) { - ConstantValue value = eval_string_const_considering_modifier(v); + tolk_assert(v->literal_value.is_slice()); std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(str-const)"); - if (value.is_int()) { - code.emplace_back(v->loc, Op::_IntConst, rvect, value.as_int()); - } else { - code.emplace_back(v->loc, Op::_SliceConst, rvect, value.as_slice()); - } + code.emplace_back(v->loc, Op::_SliceConst, rvect, v->literal_value.as_slice()); return transition_to_target_type(std::move(rvect), code, target_type, v); } diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index bdb1763aec..13ecece516 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -57,6 +57,14 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { return inner; } + static V create_string_const(SrcLocation loc, ConstantValue&& literal_value) { + auto v_string = createV(loc, literal_value.as_slice()); + v_string->assign_inferred_type(TypeDataSlice::create()); + v_string->assign_literal_value(std::move(literal_value)); + v_string->assign_rvalue_true(); + return v_string; + } + AnyExprV replace(V v) override { parent::replace(v); @@ -102,17 +110,31 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { AnyExprV replace(V v) override { parent::replace(v); - // replace `ton("0.05")` with 50000000 - if (v->fun_maybe && v->fun_maybe->is_builtin_function() && v->fun_maybe->name == "ton") { - ConstantValue value = eval_call_to_ton_function(v); - tolk_assert(value.is_int()); - return create_int_const(v->loc, value.as_int()); + // replace `ton("0.05")` with 50000000 / `stringCrc32("some_str")` with calculated value / etc. + if (v->fun_maybe && v->fun_maybe->is_compile_time_only()) { + ConstantValue value = eval_call_to_compile_time_function(v); + if (value.is_int()) { + return create_int_const(v->loc, value.as_int()); + } + if (value.is_slice()) { + return create_string_const(v->loc, std::move(value)); + } + tolk_assert(false); } return v; } -public: + AnyExprV replace(V v) override { + // when "some_str" occurs as a standalone constant (not inside `stringCrc32("some_str")`, + // it's actually a slice + ConstantValue literal_value = eval_string_const_standalone(v); + v->mutate()->assign_literal_value(std::move(literal_value)); + + return v; + } + + public: bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index aa8fe64d13..9db82f5767 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -280,12 +280,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } static ExprFlow infer_string_const(V v, FlowContext&& flow, bool used_as_condition) { - if (v->is_bitslice()) { - assign_inferred_type(v, TypeDataSlice::create()); - } else { - assign_inferred_type(v, TypeDataInt::create()); - } - + assign_inferred_type(v, TypeDataSlice::create()); return ExprFlow(std::move(flow), used_as_condition); } diff --git a/tolk/symtable.h b/tolk/symtable.h index e65327fdba..e78eef6e81 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -103,6 +103,7 @@ struct FunctionData final : Symbol { flagAcceptsSelf = 512, // is a member function (has `self` first parameter) flagReturnsSelf = 1024, // return type is `self` (returns the mutated 1st argument), calls can be chainable flagReallyUsed = 2048, // calculated via dfs from used functions; declared but unused functions are not codegenerated + flagCompileTimeOnly = 4096, // calculated only at compile-time for constant arguments: `ton("0.05")`, `stringCrc32`, and others }; int method_id = EMPTY_METHOD_ID; @@ -163,6 +164,7 @@ struct FunctionData final : Symbol { bool does_return_self() const { return flags & flagReturnsSelf; } bool does_mutate_self() const { return (flags & flagAcceptsSelf) && parameters[0].is_mutate_parameter(); } bool is_really_used() const { return flags & flagReallyUsed; } + bool is_compile_time_only() const { return flags & flagCompileTimeOnly; } bool does_need_codegen() const; From 6ba2355ca0a5a00c5fbd71d76b0bd0836cc624b0 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 13 Mar 2025 20:20:16 +0400 Subject: [PATCH 166/524] [Tolk] Bump version to v0.10 --- crypto/smartcont/tolk-stdlib/common.tolk | 2 +- crypto/smartcont/tolk-stdlib/gas-payments.tolk | 2 +- crypto/smartcont/tolk-stdlib/lisp-lists.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk | 2 +- tolk/tolk-version.h | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index d5290976a1..0075764b66 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -1,7 +1,7 @@ // Standard library for Tolk (LGPL licence). // It contains common functions that are available out of the box, the user doesn't have to import anything. // More specific functions are required to be imported explicitly, like "@stdlib/tvm-dicts". -tolk 0.9 +tolk 0.10 /** Tuple manipulation primitives. diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 9873ca94d9..11e3db7ff1 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.9 +tolk 0.10 /** Gas and payment related primitives. diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk index e63438b513..fd6d2c8943 100644 --- a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.9 +tolk 0.10 /** Lisp-style lists are nested 2-elements tuples: `(1, (2, (3, null)))` represents list `[1, 2, 3]`. diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index ee20568742..d4babd1efb 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.9 +tolk 0.10 /** Dictionaries are represented as `cell` data type (cells can store anything, dicts in particular). diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index 136eaa4a19..48bacec6c4 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.9 +tolk 0.10 /// Usually `c3` has a continuation initialized by the whole code of the contract. It is used for function calls. /// The primitive returns the current value of `c3`. diff --git a/tolk/tolk-version.h b/tolk/tolk-version.h index bbea63ffdf..be161a393f 100644 --- a/tolk/tolk-version.h +++ b/tolk/tolk-version.h @@ -18,6 +18,6 @@ namespace tolk { -constexpr const char* TOLK_VERSION = "0.9.0"; +constexpr const char* TOLK_VERSION = "0.10.0"; } // namespace tolk From f3d67d57751688120ae77ef5b0b1e7a5a37b9e9f Mon Sep 17 00:00:00 2001 From: neodix42 Date: Wed, 19 Mar 2025 12:57:42 +0400 Subject: [PATCH 167/524] force clang-16 usage when compiling ubuntu portable shared libraries (#1560) --- assembly/native/build-ubuntu-portable-libs.sh | 4 ++-- assembly/native/build-ubuntu-portable.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/assembly/native/build-ubuntu-portable-libs.sh b/assembly/native/build-ubuntu-portable-libs.sh index 2f0a1ba4d1..b634ce0c41 100644 --- a/assembly/native/build-ubuntu-portable-libs.sh +++ b/assembly/native/build-ubuntu-portable-libs.sh @@ -21,8 +21,8 @@ else rm -rf .ninja* CMakeCache.txt fi -export CC=$(which clang) -export CXX=$(which clang++) +export CC=$(which clang-16) +export CXX=$(which clang++-16) export CCACHE_DISABLE=1 if [ ! -d "lz4" ]; then diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh index 16e77ac8d4..00b34f8320 100644 --- a/assembly/native/build-ubuntu-portable.sh +++ b/assembly/native/build-ubuntu-portable.sh @@ -24,8 +24,8 @@ else rm -rf .ninja* CMakeCache.txt fi -export CC=$(which clang) -export CXX=$(which clang++) +export CC=$(which clang-16) +export CXX=$(which clang++-16) export CCACHE_DISABLE=1 if [ ! -d "lz4" ]; then From e3a8a770abdc599833a8e8e4609689bb4d76443b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 19 Mar 2025 13:45:42 +0300 Subject: [PATCH 168/524] Fix null reference error in ValidatorManagerImpl::prepare_stats (#1561) --- validator/manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index b0ac54092d..00b923c236 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2891,7 +2891,7 @@ void ValidatorManagerImpl::prepare_stats(td::Promiseget_state_serializer_enabled(); - if (is_validator() && last_masterchain_state_->get_global_id() == -239) { + if (is_validator() && last_masterchain_state_.not_null() && last_masterchain_state_->get_global_id() == -239) { serializer_enabled = false; } vec.emplace_back("stateserializerenabled", serializer_enabled ? "true" : "false"); From 79aad724cf453db30717767375a0492a63f351b0 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 20 Mar 2025 19:10:57 +0300 Subject: [PATCH 169/524] Add missing stop() to ArchiveImporter --- validator/import-db-slice.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index 06573d347e..3e1392bef9 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -513,6 +513,7 @@ void ArchiveImporter::abort_query(td::Status error) { td::unlink(f).ignore(); } promise_.set_error(std::move(error)); + stop(); return; } LOG(INFO) << "Archive import: " << error; From 31b351c82accac0fd6f4213b2973559b71943a98 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Thu, 20 Mar 2025 19:37:27 -0400 Subject: [PATCH 170/524] Implement SHA256 hasher using the deprecated SHA256_* function family SHA256_* family avoids a (gigantic) overhead of OpenSSL's EVP interface. Locally, this gives 6% performance boost for block verification. --- crypto/openssl/digest.hpp | 57 +++++++++++++++++++++++++++++++++++- crypto/vm/cells/DataCell.cpp | 18 +++++------- 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/crypto/openssl/digest.hpp b/crypto/openssl/digest.hpp index 2adeef1d77..3e36f7e99c 100644 --- a/crypto/openssl/digest.hpp +++ b/crypto/openssl/digest.hpp @@ -21,6 +21,7 @@ #include #include +#include #include "td/utils/Slice.h" @@ -124,9 +125,63 @@ std::string HashCtx::extract() { } typedef HashCtx SHA1; -typedef HashCtx SHA256; typedef HashCtx SHA512; +struct SHA256Tag {}; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + +template <> +struct HashCtx { + public: + enum { digest_bytes = 32 }; + + HashCtx() { + SHA256_Init(&ctx_); + } + + HashCtx(const void *data, std::size_t len) : HashCtx() { + feed(data, len); + } + + void reset() { + SHA256_Init(&ctx_); + } + + void feed(const void *data, std::size_t len) { + SHA256_Update(&ctx_, data, len); + } + + void feed(td::Slice slice) { + feed(slice.data(), slice.size()); + } + + std::size_t extract(unsigned char buffer[digest_bytes]) { + SHA256_Final(buffer, &ctx_); + return digest_bytes; + } + + std::size_t extract(td::MutableSlice slice) { + CHECK(slice.size() == digest_bytes); + SHA256_Final(slice.ubegin(), &ctx_); + return digest_bytes; + } + + std::string extract() { + unsigned char buffer[digest_bytes]; + extract(buffer); + return std::string((char *)buffer, digest_bytes); + } + + private: + SHA256_CTX ctx_; +}; + +#pragma GCC diagnostic pop + +typedef HashCtx SHA256; + template std::size_t hash_str(unsigned char buffer[T::digest_bytes], const void *data, std::size_t size) { T hasher(data, size); diff --git a/crypto/vm/cells/DataCell.cpp b/crypto/vm/cells/DataCell.cpp index 4dd301616b..e5f21b0130 100644 --- a/crypto/vm/cells/DataCell.cpp +++ b/crypto/vm/cells/DataCell.cpp @@ -268,18 +268,16 @@ td::Result> DataCell::create(td::ConstBitPtr data, unsigned bits, tmp[0] = info.d1(level_mask.apply(level_i)); tmp[1] = info.d2(); - static TD_THREAD_LOCAL digest::SHA256* hasher; - td::init_thread_local(hasher); - hasher->reset(); + digest::SHA256 hasher; - hasher->feed(td::Slice(tmp, 2)); + hasher.feed(td::Slice(tmp, 2)); if (hash_i == hash_i_offset) { DCHECK(level_i == 0 || type == SpecialType::PrunnedBranch); - hasher->feed(td::Slice(data_ptr, (bits + 7) >> 3)); + hasher.feed(td::Slice(data_ptr, (bits + 7) >> 3)); } else { DCHECK(level_i != 0 && type != SpecialType::PrunnedBranch); - hasher->feed(hashes_ptr[hash_i - hash_i_offset - 1].as_slice()); + hasher.feed(hashes_ptr[hash_i - hash_i_offset - 1].as_slice()); } auto dest_i = hash_i - hash_i_offset; @@ -297,7 +295,7 @@ td::Result> DataCell::create(td::ConstBitPtr data, unsigned bits, // add depth into hash td::uint8 child_depth_buf[depth_bytes]; store_depth(child_depth_buf, child_depth); - hasher->feed(td::Slice(child_depth_buf, depth_bytes)); + hasher.feed(td::Slice(child_depth_buf, depth_bytes)); depth = std::max(depth, child_depth); } @@ -312,12 +310,12 @@ td::Result> DataCell::create(td::ConstBitPtr data, unsigned bits, // children hash for (int i = 0; i < info.refs_count_; i++) { if (type == SpecialType::MerkleProof || type == SpecialType::MerkleUpdate) { - hasher->feed(refs_ptr[i]->get_hash(level_i + 1).as_slice()); + hasher.feed(refs_ptr[i]->get_hash(level_i + 1).as_slice()); } else { - hasher->feed(refs_ptr[i]->get_hash(level_i).as_slice()); + hasher.feed(refs_ptr[i]->get_hash(level_i).as_slice()); } } - auto extracted_size = hasher->extract(hashes_ptr[dest_i].as_slice()); + auto extracted_size = hasher.extract(hashes_ptr[dest_i].as_slice()); DCHECK(extracted_size == hash_bytes); } From 513a60fec5374ef19f3018d4ef81d25ef149a48b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sat, 22 Mar 2025 13:51:15 +0300 Subject: [PATCH 171/524] Cache validator sets for create_shard_state --- crypto/block/mc-config.cpp | 71 ++++++++++++++++++++++++++++++-- crypto/block/mc-config.h | 4 +- lite-client/lite-client.h | 2 +- validator/impl/shard.cpp | 4 +- validator/impl/validator-set.hpp | 2 +- 5 files changed, 73 insertions(+), 10 deletions(-) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 0f019b068b..7b4f326f8d 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -38,6 +38,7 @@ #include "openssl/digest.hpp" #include #include +#include namespace block { using namespace std::literals::string_literals; @@ -253,7 +254,7 @@ td::Status Config::unpack() { } config_dict = std::make_unique(config_root, 32); if (mode & needValidatorSet) { - auto vset_res = unpack_validator_set(get_config_param(35, 34)); + auto vset_res = unpack_validator_set(get_config_param(35, 34), true); if (vset_res.is_error()) { return vset_res.move_as_error(); } @@ -486,15 +487,74 @@ td::Result Config::unpack_workchain_list(Ref root) { return std::move(pair.first); } -td::Result> Config::unpack_validator_set(Ref vset_root) { +class ValidatorSetCache { +public: + ValidatorSetCache() { + cache_.reserve(MAX_CACHE_SIZE + 1); + } + + std::shared_ptr get(const vm::CellHash& hash) { + std::lock_guard lock{mutex_}; + auto it = cache_.find(hash); + if (it == cache_.end()) { + return {}; + } + auto entry = it->second.get(); + entry->remove(); + lru_.put(entry); + return entry->value; + } + + void set(const vm::CellHash& hash, std::shared_ptr vset) { + std::lock_guard lock{mutex_}; + std::unique_ptr& entry = cache_[hash]; + if (entry == nullptr) { + entry = std::make_unique(hash, std::move(vset)); + } else { + entry->value = std::move(vset); + entry->remove(); + } + lru_.put(entry.get()); + if (cache_.size() > MAX_CACHE_SIZE) { + auto to_remove = (CacheEntry*)lru_.get(); + CHECK(to_remove); + to_remove->remove(); + cache_.erase(to_remove->key); + } + } + +private: + std::mutex mutex_; + + struct CacheEntry : td::ListNode { + explicit CacheEntry(vm::CellHash key, std::shared_ptr value) : key(key), value(std::move(value)) { + } + vm::CellHash key; + std::shared_ptr value; + }; + td::HashMap> cache_; + td::ListNode lru_; + + static constexpr size_t MAX_CACHE_SIZE = 100; +}; + +td::Result> Config::unpack_validator_set(Ref vset_root, bool use_cache) { if (vset_root.is_null()) { return td::Status::Error("validator set is absent"); } + static ValidatorSetCache cache; + if (use_cache) { + auto result = cache.get(vset_root->get_hash()); + if (result) { + return result; + } + } + gen::ValidatorSet::Record_validators_ext rec; Ref dict_root; if (!tlb::unpack_cell(vset_root, rec)) { gen::ValidatorSet::Record_validators rec0; - if (!tlb::unpack_cell(std::move(vset_root), rec0)) { + if (!tlb::unpack_cell(vset_root, rec0)) { return td::Status::Error("validator set is invalid"); } rec.utime_since = rec0.utime_since; @@ -515,7 +575,7 @@ td::Result> Config::unpack_validator_set(Ref(rec.utime_since, rec.utime_until, rec.total, rec.main); + auto ptr = std::make_shared(rec.utime_since, rec.utime_until, rec.total, rec.main); for (int i = 0; i < rec.total; i++) { key_buffer.store_ulong(i); auto descr_cs = dict.lookup(key_buffer.bits(), 16); @@ -548,6 +608,9 @@ td::Result> Config::unpack_validator_set(Reftotal_weight) { return td::Status::Error("validator set declares incorrect total weight"); } + if (use_cache) { + cache.set(vset_root->get_hash(), ptr); + } return std::move(ptr); } diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 98e6a26df1..a44522f11c 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -559,7 +559,7 @@ class Config { td::BitArray<256> config_addr; Ref config_root; std::unique_ptr config_dict; - std::unique_ptr cur_validators_; + std::shared_ptr cur_validators_; std::unique_ptr workchains_dict_; WorkchainSet workchains_; int version_{-1}; @@ -623,7 +623,7 @@ class Config { bool set_block_id_ext(const ton::BlockIdExt& block_id_ext); td::Result> get_special_smartcontracts(bool without_config = false) const; bool is_special_smartcontract(const ton::StdSmcAddress& addr) const; - static td::Result> unpack_validator_set(Ref valset_root); + static td::Result> unpack_validator_set(Ref valset_root, bool use_cache = false); td::Result> get_storage_prices() const; static td::Result do_get_one_storage_prices(vm::CellSlice cs); td::Result get_gas_limits_prices(bool is_masterchain = false) const; diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 721d2b20d6..3284f255d6 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -143,7 +143,7 @@ class TestNode : public td::actor::Actor { ton::LogicalTime end_lt{0}; ton::Bits256 vset_hash; Ref vset_root; - std::unique_ptr vset; + std::shared_ptr vset; std::map vset_map; int special_idx{-1}; std::pair created_total, created_special; diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 0ab216f712..d2d402dd66 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -390,12 +390,12 @@ td::Status MasterchainStateQ::mc_reinit() { auto cv_root = config_->get_config_param(35, 34); if (cv_root.not_null()) { - TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(cv_root))); + TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(cv_root), true)); cur_validators_ = std::move(validators); } auto nv_root = config_->get_config_param(37, 36); if (nv_root.not_null()) { - TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root))); + TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root), true)); next_validators_ = std::move(validators); } diff --git a/validator/impl/validator-set.hpp b/validator/impl/validator-set.hpp index 951ca4b715..81c0bd5183 100644 --- a/validator/impl/validator-set.hpp +++ b/validator/impl/validator-set.hpp @@ -74,7 +74,7 @@ class ValidatorSetCompute { private: const block::Config* config_{nullptr}; - std::unique_ptr cur_validators_, next_validators_; + std::shared_ptr cur_validators_, next_validators_; td::Ref compute_validator_set(ShardIdFull shard, const block::ValidatorSet& vset, UnixTime time, CatchainSeqno seqno) const; }; From b304371f538e489affdb580e53df50ce4b058079 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sat, 22 Mar 2025 13:56:52 +0300 Subject: [PATCH 172/524] Permanent celldb, improve importing archive slices, new flags for validator-engine --- crypto/vm/db/CellStorage.cpp | 33 +- crypto/vm/db/CellStorage.h | 3 +- validator-engine/validator-engine.cpp | 30 +- validator-engine/validator-engine.hpp | 12 + validator/CMakeLists.txt | 4 + validator/db/celldb.cpp | 204 +++++- validator/db/celldb.hpp | 8 + .../permanent-celldb-utils.cpp | 83 +++ .../permanent-celldb/permanent-celldb-utils.h | 37 + validator/db/rootdb.cpp | 35 + validator/db/rootdb.hpp | 4 + validator/downloaders/wait-block-state.cpp | 28 +- validator/downloaders/wait-block-state.hpp | 11 +- validator/import-db-slice-local.cpp | 635 ++++++++++++++++++ validator/import-db-slice-local.hpp | 103 +++ validator/import-db-slice.cpp | 2 +- validator/interfaces/db.h | 4 + validator/interfaces/validator-manager.h | 4 + validator/manager-disk.cpp | 14 +- validator/manager-disk.hpp | 4 + validator/manager-hardfork.cpp | 4 +- validator/manager-hardfork.hpp | 7 + validator/manager.cpp | 61 +- validator/manager.hpp | 3 + validator/validator-options.cpp | 15 +- validator/validator-options.hpp | 18 +- validator/validator.h | 6 +- 27 files changed, 1304 insertions(+), 68 deletions(-) create mode 100644 validator/db/permanent-celldb/permanent-celldb-utils.cpp create mode 100644 validator/db/permanent-celldb/permanent-celldb-utils.h create mode 100644 validator/import-db-slice-local.cpp create mode 100644 validator/import-db-slice-local.hpp diff --git a/crypto/vm/db/CellStorage.cpp b/crypto/vm/db/CellStorage.cpp index a07d85e870..0c8ba179a6 100644 --- a/crypto/vm/db/CellStorage.cpp +++ b/crypto/vm/db/CellStorage.cpp @@ -32,8 +32,9 @@ namespace { class RefcntCellStorer { public: - RefcntCellStorer(td::int32 refcnt, const td::Ref &cell, bool as_boc) - : refcnt_(refcnt), cell_(cell), as_boc_(as_boc) { + RefcntCellStorer(td::int32 refcnt, const td::Ref &cell, bool as_boc, int max_level = vm::Cell::max_level) + : refcnt_(refcnt), cell_(cell), as_boc_(as_boc), max_level_(max_level) { + CHECK(!as_boc_ || max_level_ == vm::Cell::max_level); } template @@ -51,10 +52,25 @@ class RefcntCellStorer { CHECK(refcnt_ > 0); store(refcnt_, storer); CHECK(cell_.not_null()) - store(*cell_, storer); + if (max_level_ == vm::Cell::max_level) { + store(*cell_, storer); + } else { + auto level_mask = cell_->get_level_mask().apply(max_level_); + storer.template store_binary( + static_cast(cell_->get_refs_cnt() + 8 * cell_->is_special() + 32 * level_mask.get_mask())); + auto d2 = static_cast((cell_->get_bits() / 8) * 2); + if ((cell_->get_bits() & 7) != 0) { + d2 = static_cast(d2 + 1); + } + storer.template store_binary(d2); + storer.store_slice(td::Slice(cell_->get_data(), (cell_->get_bits() + 7) / 8)); + } + for (unsigned i = 0; i < cell_->size_refs(); i++) { auto cell = cell_->get_ref(i); - auto level_mask = cell->get_level_mask(); + auto level_mask = + cell->get_level_mask().apply(max_level_ + (cell_->special_type() == CellTraits::SpecialType::MerkleProof || + cell_->special_type() == CellTraits::SpecialType::MerkleUpdate)); auto level = level_mask.get_level(); td::uint8 x = static_cast(level_mask.get_mask()); storer.store_slice(td::Slice(&x, 1)); @@ -79,6 +95,7 @@ class RefcntCellStorer { td::int32 refcnt_; td::Ref cell_; bool as_boc_; + int max_level_; }; class RefcntCellParser { @@ -120,13 +137,13 @@ class RefcntCellParser { Ref refs[Cell::max_refs]; for (int i = 0; i < info.refs_cnt; i++) { if (data.size() < 1) { - return td::Status::Error("Not enought data"); + return td::Status::Error("Not enough data"); } Cell::LevelMask level_mask(data[0]); auto n = level_mask.get_hashes_count(); auto end_offset = 1 + n * (Cell::hash_bytes + Cell::depth_bytes); if (data.size() < end_offset) { - return td::Status::Error("Not enought data"); + return td::Status::Error("Not enough data"); } TRY_RESULT(ext_cell, ext_cell_creator.ext_cell(level_mask, data.substr(1, n * Cell::hash_bytes), @@ -221,8 +238,8 @@ td::Status CellStorer::erase(td::Slice hash) { return kv_.erase(hash); } -std::string CellStorer::serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc) { - return td::serialize(RefcntCellStorer(refcnt, cell, as_boc)); +std::string CellStorer::serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc, int max_level) { + return td::serialize(RefcntCellStorer(refcnt, cell, as_boc, max_level)); } td::Status CellStorer::set(td::int32 refcnt, const td::Ref &cell, bool as_boc) { diff --git a/crypto/vm/db/CellStorage.h b/crypto/vm/db/CellStorage.h index ca32a8007d..fe1d6b250f 100644 --- a/crypto/vm/db/CellStorage.h +++ b/crypto/vm/db/CellStorage.h @@ -71,7 +71,8 @@ class CellStorer { static void merge_refcnt_diffs(std::string &left, td::Slice right); static std::string serialize_refcnt_diffs(td::int32 refcnt_diff); - static std::string serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc); + static std::string serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc, + int max_level = vm::Cell::max_level); struct Diff { enum Type { Set, Erase, Merge } type{Set}; diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index cbcc3ab1f0..273ae3ba3b 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1488,6 +1488,8 @@ td::Status ValidatorEngine::load_global_config() { if (catchain_max_block_delay_slow_) { validator_options_.write().set_catchain_max_block_delay_slow(catchain_max_block_delay_slow_.value()); } + validator_options_.write().set_permanent_celldb(permanent_celldb_); + validator_options_.write().set_initial_sync_disabled(skip_key_sync_); std::vector h; for (auto &x : conf.validator_->hardforks_) { @@ -1515,16 +1517,23 @@ td::Status ValidatorEngine::load_global_config() { void ValidatorEngine::set_shard_check_function() { if (!not_all_shards_) { - validator_options_.write().set_shard_check_function([](ton::ShardIdFull shard) -> bool { return true; }); + validator_options_.write().set_shard_check_function( + [sync_shards_upto = sync_shards_upto_](ton::ShardIdFull shard, ton::BlockSeqno mc_seqno) -> bool { + return shard.is_masterchain() || !sync_shards_upto || mc_seqno <= sync_shards_upto.value(); + }); } else { std::vector shards = {ton::ShardIdFull(ton::masterchainId)}; - for (const auto& s : config_.shards_to_monitor) { + for (const auto &s : config_.shards_to_monitor) { shards.push_back(s); } std::sort(shards.begin(), shards.end()); shards.erase(std::unique(shards.begin(), shards.end()), shards.end()); validator_options_.write().set_shard_check_function( - [shards = std::move(shards)](ton::ShardIdFull shard) -> bool { + [shards = std::move(shards), sync_shards_upto = sync_shards_upto_](ton::ShardIdFull shard, + ton::BlockSeqno mc_seqno) -> bool { + if (!shard.is_masterchain() && sync_shards_upto && mc_seqno > sync_shards_upto.value()) { + return false; + } for (auto s : shards) { if (shard_intersects(shard, s)) { return true; @@ -4613,6 +4622,21 @@ int main(int argc, char *argv[]) { [&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_broadcast_speed_multiplier_private, v); }); return td::Status::OK(); }); + p.add_option( + '\0', "permanent-celldb", + "disable garbage collection in CellDb. This improves performance on archival nodes (once enabled, this option " + "cannot be disabled)", + [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_permanent_celldb, true); }); }); + p.add_option('\0', "skip-key-sync", + "don't select the best persistent state on initial sync, start on init_block from global config", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_skip_key_sync, true); }); + }); + p.add_checked_option( + '\0', "sync-shards-upto", "stop syncing shards on this masterchain seqno", [&](td::Slice s) -> td::Status { + TRY_RESULT(v, td::to_integer_safe(s)); + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_sync_shards_upto, v); }); + return td::Status::OK(); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 5a1db7f3f5..dceb3503ab 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -233,6 +233,9 @@ class ValidatorEngine : public td::actor::Actor { double broadcast_speed_multiplier_catchain_ = 1.0; double broadcast_speed_multiplier_public_ = 1.0; double broadcast_speed_multiplier_private_ = 1.0; + bool permanent_celldb_ = false; + bool skip_key_sync_ = false; + td::optional sync_shards_upto_; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -345,6 +348,15 @@ class ValidatorEngine : public td::actor::Actor { void set_broadcast_speed_multiplier_private(double value) { broadcast_speed_multiplier_private_ = value; } + void set_permanent_celldb(bool value) { + permanent_celldb_ = value; + } + void set_skip_key_sync(bool value) { + skip_key_sync_ = value; + } + void set_sync_shards_upto(ton::BlockSeqno seqno) { + sync_shards_upto_ = seqno; + } void start_up() override; ValidatorEngine() { diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index f7a4ce1d11..9fb0d190fa 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -14,6 +14,8 @@ set(VALIDATOR_DB_SOURCE db/archive-slice.hpp db/celldb.cpp db/celldb.hpp + db/permanent-celldb/permanent-celldb-utils.cpp + db/permanent-celldb/permanent-celldb-utils.h db/files-async.hpp db/fileref.hpp db/fileref.cpp @@ -54,6 +56,7 @@ set(VALIDATOR_HEADERS invariants.hpp import-db-slice.hpp + import-db-slice-local.hpp queue-size-counter.hpp validator-telemetry.hpp @@ -74,6 +77,7 @@ set(VALIDATOR_SOURCE block-handle.cpp get-next-key-blocks.cpp import-db-slice.cpp + import-db-slice-local.cpp shard-client.cpp state-serializer.cpp token-manager.cpp diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 90c659cc46..65a60af3f3 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -27,6 +27,9 @@ #include "ton/ton-tl.hpp" #include "ton/ton-io.hpp" #include "common/delay.h" +#include "block/block-auto.h" +#include "permanent-celldb/permanent-celldb-utils.h" +#include "td/actor/MultiPromise.h" #include #include @@ -324,6 +327,30 @@ void CellDbIn::start_up() { last_deleted_mc_state_ = r_value.move_as_ok(); } } + { + std::string key = "opts.permanent_mode", value; + auto R = boc_->meta_get(td::as_slice(key), value); + R.ensure(); + bool stored_permanent_mode = R.ok() == td::KeyValue::GetStatus::Ok; + permanent_mode_ = stored_permanent_mode || opts_->get_permanent_celldb(); + if (permanent_mode_) { + LOG(WARNING) << "Celldb is in permanent mode"; + if (!stored_permanent_mode) { + cell_db_->begin_write_batch().ensure(); + value = "1"; + vm::CellStorer stor{*cell_db_}; + boc_->meta_set(td::as_slice(key), td::as_slice(value)); + boc_->commit(stor).ensure(); + cell_db_->commit_write_batch().ensure(); + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + } + } + } + cell_db_statistics_.permanent_mode_ = permanent_mode_; + LOG_IF(FATAL, permanent_mode_ && opts_->get_celldb_in_memory()) + << "celldb permanent_mode and in_memory_mode are not compatible"; + } } void CellDbIn::load_cell(RootHash hash, td::Promise> promise) { @@ -359,7 +386,9 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi auto R = get_block(key_hash); // duplicate if (R.is_ok()) { - promise.set_result(boc_->load_cell(cell->get_hash().as_slice())); + delay_action([cell = boc_->load_cell(cell->get_hash().as_slice()), + promise = std::move(promise)]() mutable { promise.set_result(std::move(cell)); }, + td::Timestamp::now()); return; } @@ -370,7 +399,7 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi cell = std::move(cell)](td::Result Res) mutable { Res.ensure(); timer_prepare.pause(); - td::actor::send_lambda( + td::actor::send_lambda_later( SelfId, [=, this, timer = std::move(timer), promise = std::move(promise), cell = std::move(cell)]() mutable { TD_PERF_COUNTER(celldb_store_cell); auto empty = get_empty_key_hash(); @@ -407,7 +436,9 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); } - promise.set_result(boc_->load_cell(cell->get_hash().as_slice())); + delay_action([cell = boc_->load_cell(cell->get_hash().as_slice()), + promise = std::move(promise)]() mutable { promise.set_result(std::move(cell)); }, + td::Timestamp::now()); if (!opts_->get_disable_rocksdb_stats()) { cell_db_statistics_.store_cell_time_.insert(timer.elapsed() * 1e6); cell_db_statistics_.store_cell_prepare_time_.insert(timer_prepare.elapsed() * 1e6); @@ -430,6 +461,149 @@ void CellDbIn::get_cell_db_reader(td::Promise> promise.set_result(boc_->get_cell_db_reader()); } +void CellDbIn::store_block_state_permanent(td::Ref block, td::Promise> promise) { + if (!permanent_mode_) { + promise.set_error(td::Status::Error("celldb is not in permanent mode")); + return; + } + if (db_busy_) { + action_queue_.push( + [self = this, block = std::move(block), promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + self->store_block_state_permanent(std::move(block), std::move(promise)); + }); + return; + } + auto key_hash = get_key_hash(block->block_id()); + auto R = get_block(key_hash); + // duplicate + if (R.is_ok()) { + delay_action([cell = boc_->load_cell(R.ok().root_hash.as_slice()), + promise = std::move(promise)]() mutable { promise.set_result(std::move(cell)); }, + td::Timestamp::now()); + return; + } + store_block_state_permanent_bulk( + {block}, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + block::gen::Block::Record rec; + if (!block::gen::unpack_cell(block->root_cell(), rec)) { + promise.set_error(td::Status::Error("cannot unpack Block record")); + return; + } + bool spec; + vm::CellSlice update_cs = vm::load_cell_slice_special(rec.state_update, spec); + if (update_cs.special_type() != vm::CellTraits::SpecialType::MerkleUpdate) { + promise.set_error(td::Status::Error("invalid Merkle update in block")); + return; + } + td::Ref new_state_root = update_cs.prefetch_ref(1); + RootHash state_root_hash = new_state_root->get_hash(0).bits(); + td::actor::send_closure(SelfId, &CellDbIn::load_cell, state_root_hash, std::move(promise)); + }); +} + +void CellDbIn::store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise) { + if (!permanent_mode_) { + promise.set_error(td::Status::Error("celldb is not in permanent mode")); + return; + } + if (db_busy_) { + action_queue_.push( + [self = this, blocks = std::move(blocks), promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + self->store_block_state_permanent_bulk(std::move(blocks), std::move(promise)); + }); + return; + } + td::PerfWarningTimer timer{"storecellbulk", 0.1}; + td::Timer timer_prepare; + std::map> new_blocks; + for (auto& block : blocks) { + BlockIdExt block_id = block->block_id(); + if (new_blocks.contains(block_id)) { + continue; + } + if (get_block(get_key_hash(block_id)).is_ok()) { + continue; + } + new_blocks[block_id] = std::move(block); + } + if (new_blocks.empty()) { + promise.set_value(td::Unit{}); + return; + } + for (auto& [block_id, block] : new_blocks) { + std::vector prev; + BlockIdExt mc_blkid; + bool after_split; + TRY_STATUS_PROMISE(promise, + block::unpack_block_prev_blk_try(block->root_cell(), block_id, prev, mc_blkid, after_split)); + for (const BlockIdExt& prev_id : prev) { + if (!new_blocks.contains(prev_id) && get_block(get_key_hash(prev_id)).is_error()) { + promise.set_error(td::Status::Error("cannot store block state: previous block is not in db")); + return; + } + } + } + db_busy_ = true; + calculate_permanent_celldb_update( + new_blocks, async_executor, + [this, SelfId = actor_id(this), timer = std::move(timer), timer_prepare = std::move(timer_prepare), + promise = std::move(promise)](td::Result> R) mutable { + TRY_RESULT_PROMISE(promise, updates, std::move(R)); + td::actor::send_lambda_later( + SelfId, [=, this, timer = std::move(timer), timer_prepare = std::move(timer_prepare), + updates = std::move(updates), promise = std::move(promise)]() mutable { + TD_PERF_COUNTER(celldb_store_cell_multi); + timer_prepare.pause(); + td::Timer timer_write; + vm::CellStorer stor{*cell_db_}; + cell_db_->begin_write_batch().ensure(); + + for (auto& update : updates) { + for (auto& [k, v] : update.to_store) { + cell_db_->set(k.as_slice(), v).ensure(); + } + } + + CHECK(!updates.empty()); + auto empty = get_empty_key_hash(); + auto E = get_block(empty).move_as_ok(); + for (size_t i = 0; i < updates.size(); ++i) { + KeyHash prev = i == 0 ? empty : get_key_hash(updates[i - 1].block_id); + KeyHash next = i + 1 == updates.size() ? E.next : get_key_hash(updates[i + 1].block_id); + DbEntry entry{updates[i].block_id, prev, next, updates[i].state_root_hash}; + set_block(get_key_hash(updates[i].block_id), std::move(entry)); + } + E.next = get_key_hash(updates[0].block_id); + if (E.prev == empty) { + E.prev = get_key_hash(updates.back().block_id); + } + set_block(empty, std::move(E)); + + boc_->commit(stor).ensure(); // Save meta + cell_db_->commit_write_batch().ensure(); + timer_write.pause(); + + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + } + + if (!opts_->get_disable_rocksdb_stats()) { + cell_db_statistics_.store_cell_time_.insert(timer.elapsed() * 1e6); + cell_db_statistics_.store_cell_prepare_time_.insert(timer_prepare.elapsed() * 1e6); + cell_db_statistics_.store_cell_write_time_.insert(timer_write.elapsed() * 1e6); + cell_db_statistics_.store_cell_bulk_queries_++; + cell_db_statistics_.store_cell_bulk_total_blocks_ += updates.size(); + } + release_db(); + promise.set_result(td::Unit()); + }); + }); +} + std::vector> CellDbIn::prepare_stats() { TD_PERF_COUNTER(celldb_prepare_stats); auto r_boc_stats = boc_->get_stats(); @@ -497,6 +671,7 @@ void CellDbIn::flush_db_stats() { } td::RocksDb::reset_statistics(statistics_); cell_db_statistics_.clear(); + cell_db_statistics_.permanent_mode_ = permanent_mode_; } void CellDbIn::alarm() { @@ -515,6 +690,10 @@ void CellDbIn::alarm() { << " queue_size=" << cells_to_migrate_.size(); migration_stats_ = {}; } + if (permanent_mode_) { + skip_gc(); + return; + } auto E = get_block(get_empty_key_hash()).move_as_ok(); auto N = get_block(E.next).move_as_ok(); if (N.is_empty()) { @@ -568,6 +747,7 @@ void CellDbIn::gc_cont2(BlockHandle handle) { }); return; } + CHECK(!permanent_mode_); td::PerfWarningTimer timer{"gccell", 0.1}; td::PerfWarningTimer timer_all{"gccell_all", 0.05}; @@ -610,7 +790,7 @@ void CellDbIn::gc_cont2(BlockHandle handle) { P = std::move(P), N = std::move(N), cell = std::move(cell), timer = std::move(timer), timer_all = std::move(timer_all), handle](td::Result R) mutable { R.ensure(); - td::actor::send_lambda(SelfId, [this, timer_boc = std::move(timer_boc), F = std::move(F), key_hash, + td::actor::send_lambda_later(SelfId, [this, timer_boc = std::move(timer_boc), F = std::move(F), key_hash, P = std::move(P), N = std::move(N), cell = std::move(cell), timer = std::move(timer), timer_all = std::move(timer_all), handle]() mutable { TD_PERF_COUNTER(celldb_gc_cell); @@ -710,6 +890,9 @@ void CellDbIn::set_block(KeyHash key_hash, DbEntry e) { } void CellDbIn::migrate_cell(td::Bits256 hash) { + if (permanent_mode_) { + return; + } cells_to_migrate_.insert(hash); if (!migration_active_) { migration_active_ = true; @@ -823,6 +1006,14 @@ void CellDb::store_cell(BlockIdExt block_id, td::Ref cell, td::Promise td::actor::send_closure(cell_db_, &CellDbIn::store_cell, block_id, std::move(cell), std::move(promise)); } +void CellDb::store_block_state_permanent(td::Ref block, td::Promise> promise) { + td::actor::send_closure(cell_db_, &CellDbIn::store_block_state_permanent, std::move(block), std::move(promise)); +} + +void CellDb::store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise) { + td::actor::send_closure(cell_db_, &CellDbIn::store_block_state_permanent_bulk, std::move(blocks), std::move(promise)); +} + void CellDb::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(cell_db_, &CellDbIn::get_cell_db_reader, std::move(promise)); } @@ -859,9 +1050,14 @@ td::BufferSlice CellDbIn::DbEntry::release() { std::vector> CellDbIn::CellDbStatistics::prepare_stats() { std::vector> stats; + stats.emplace_back("permanent_mode", PSTRING() << permanent_mode_); stats.emplace_back("store_cell.micros", PSTRING() << store_cell_time_.to_string()); stats.emplace_back("store_cell.prepare.micros", PSTRING() << store_cell_prepare_time_.to_string()); stats.emplace_back("store_cell.write.micros", PSTRING() << store_cell_write_time_.to_string()); + if (permanent_mode_) { + stats.emplace_back("store_cell.bulk.queries", PSTRING() << store_cell_bulk_queries_); + stats.emplace_back("store_cell.bulk.total_blocks", PSTRING() << store_cell_bulk_total_blocks_); + } stats.emplace_back("gc_cell.micros", PSTRING() << gc_cell_time_.to_string()); stats.emplace_back("total_time.micros", PSTRING() << (td::Timestamp::now().at() - stats_start_time_.at()) * 1e6); stats.emplace_back("in_memory", PSTRING() << bool(in_memory_load_time_)); diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 1e1ccddabf..f70f6aef82 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -66,6 +66,8 @@ class CellDbIn : public CellDbBase { void load_cell(RootHash hash, td::Promise> promise); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); void get_cell_db_reader(td::Promise> promise); + void store_block_state_permanent(td::Ref block, td::Promise> promise); + void store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise); void migrate_cell(td::Bits256 hash); @@ -137,9 +139,12 @@ class CellDbIn : public CellDbBase { std::unique_ptr migration_stats_; struct CellDbStatistics { + bool permanent_mode_; PercentileStats store_cell_time_; PercentileStats store_cell_prepare_time_; PercentileStats store_cell_write_time_; + size_t store_cell_bulk_queries_ = 0; + size_t store_cell_bulk_total_blocks_ = 0; PercentileStats gc_cell_time_; td::Timestamp stats_start_time_ = td::Timestamp::now(); std::optional in_memory_load_time_; @@ -156,6 +161,7 @@ class CellDbIn : public CellDbBase { CellDbStatistics cell_db_statistics_; td::Timestamp statistics_flush_at_ = td::Timestamp::never(); BlockSeqno last_deleted_mc_state_ = 0; + bool permanent_mode_ = false; bool db_busy_ = false; std::queue> action_queue_; @@ -188,6 +194,8 @@ class CellDb : public CellDbBase { void prepare_stats(td::Promise>> promise); void load_cell(RootHash hash, td::Promise> promise); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); + void store_block_state_permanent(td::Ref block, td::Promise> promise); + void store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise); void update_snapshot(std::unique_ptr snapshot) { CHECK(!opts_->get_celldb_in_memory()); if (!started_) { diff --git a/validator/db/permanent-celldb/permanent-celldb-utils.cpp b/validator/db/permanent-celldb/permanent-celldb-utils.cpp new file mode 100644 index 0000000000..b1d19ceb4b --- /dev/null +++ b/validator/db/permanent-celldb/permanent-celldb-utils.cpp @@ -0,0 +1,83 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "permanent-celldb-utils.h" +#include "block/block-auto.h" +#include "td/actor/MultiPromise.h" +#include "td/utils/HashMap.h" +#include "vm/db/CellStorage.h" + +namespace ton::validator { + +void calculate_permanent_celldb_update(const std::map>& blocks, + std::shared_ptr executor, + td::Promise> promise) { + td::MultiPromise mp; + auto ig = mp.init_guard(); + auto updates = std::make_shared>(); + updates->reserve(blocks.size()); + for (auto& [_, block] : blocks) { + executor->execute_async([block = block, updates, executor, + promise = std::make_shared>(ig.get_promise())]() mutable { + block::gen::Block::Record rec; + if (!block::gen::unpack_cell(block->root_cell(), rec)) { + promise->set_error(td::Status::Error("cannot unpack Block record")); + return; + } + bool spec; + vm::CellSlice update_cs = vm::load_cell_slice_special(rec.state_update, spec); + if (update_cs.special_type() != vm::CellTraits::SpecialType::MerkleUpdate) { + promise->set_error(td::Status::Error("invalid Merkle update in block")); + return; + } + td::Ref new_state_root = update_cs.prefetch_ref(1); + td::HashMap visited; + PermanentCellDbUpdate update{.block_id = block->block_id(), + .state_root_hash = new_state_root->get_hash(0).bits()}; + std::function&, int)> dfs = [&](const td::Ref& cell, int merkle_depth) { + int& vis = visited[cell->get_hash()]; + if (vis & (1 << merkle_depth)) { + return; + } + vis |= (1 << merkle_depth); + vm::CellSlice cs{vm::NoVm(), cell}; + if (cs.special_type() == vm::CellTraits::SpecialType::PrunnedBranch && cell->get_level() == merkle_depth + 1) { + return; + } + update.to_store.emplace_back( + cell->get_hash(merkle_depth), + vm::CellStorer::serialize_value(1 << 29, cell->load_cell().move_as_ok().data_cell, false, merkle_depth)); + merkle_depth = cs.child_merkle_depth(merkle_depth); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + dfs(cs.prefetch_ref(i), merkle_depth); + } + }; + dfs(new_state_root, 0); + executor->execute_sync( + [update = std::move(update), updates = std::move(updates), promise = std::move(promise)]() mutable { + updates->push_back(std::move(update)); + promise->set_result(td::Unit()); + }); + }); + } + + ig.add_promise([updates, promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + promise.set_value(std::move(*updates)); + }); +} + +} // namespace ton::validator diff --git a/validator/db/permanent-celldb/permanent-celldb-utils.h b/validator/db/permanent-celldb/permanent-celldb-utils.h new file mode 100644 index 0000000000..fad2ddfcdd --- /dev/null +++ b/validator/db/permanent-celldb/permanent-celldb-utils.h @@ -0,0 +1,37 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "vm/cells.h" +#include "ton/ton-types.h" +#include "interfaces/block.h" +#include "vm/db/DynamicBagOfCellsDb.h" + +#include +#include + +namespace ton::validator { + +struct PermanentCellDbUpdate { + BlockIdExt block_id; + RootHash state_root_hash; + std::vector> to_store; +}; +void calculate_permanent_celldb_update(const std::map>& blocks, + std::shared_ptr executor, + td::Promise> promise); + +} // namespace ton::validator diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 8d83e7a7d1..7213e8d9a4 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -259,6 +259,41 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref state, } } +void RootDb::store_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) { + if (handle->id() != block->block_id()) { + promise.set_error(td::Status::Error("block id mismatch")); + return; + } + if (handle->moved_to_archive() || handle->inited_state_boc()) { + get_block_state(handle, std::move(promise)); + return; + } + auto P = td::PromiseCreator::lambda( + [b = archive_db_.get(), handle, promise = std::move(promise)](td::Result> R) mutable { + TRY_RESULT_PROMISE(promise, root, std::move(R)); + handle->set_state_root_hash(root->get_hash().bits()); + handle->set_state_boc(); + + auto S = create_shard_state(handle->id(), std::move(root)); + S.ensure(); + + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise), state = S.move_as_ok()](td::Result R) mutable { + R.ensure(); + promise.set_value(std::move(state)); + }); + + td::actor::send_closure(b, &ArchiveManager::update_handle, std::move(handle), std::move(P)); + }); + td::actor::send_closure(cell_db_, &CellDb::store_block_state_permanent, std::move(block), std::move(P)); +} + +void RootDb::store_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) { + td::actor::send_closure(cell_db_, &CellDb::store_block_state_permanent_bulk, std::move(blocks), std::move(promise)); +} + void RootDb::get_block_state(ConstBlockHandle handle, td::Promise> promise) { if (handle->inited_state_boc()) { if (handle->deleted_state_boc()) { diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 52f6098e4d..5e59e1a956 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -62,6 +62,10 @@ class RootDb : public Db { void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; + void store_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override; + void store_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) override; void get_block_state(ConstBlockHandle handle, td::Promise> promise) override; void get_cell_db_reader(td::Promise> promise) override; diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index c80e7d8966..6d05e654e9 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -68,6 +68,8 @@ void WaitBlockState::start() { return; } bool inited_proof = handle_->id().is_masterchain() ? handle_->inited_proof() : handle_->inited_proof_link(); + bool allow_download = + last_masterchain_state_.is_null() || opts_->need_monitor(handle_->id().shard_full(), last_masterchain_state_); if (handle_->received_state() && inited_proof) { reading_from_db_ = true; @@ -81,8 +83,8 @@ void WaitBlockState::start() { td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle_, std::move(P)); } else if (handle_->id().id.seqno == 0 && next_static_file_attempt_.is_in_past()) { next_static_file_attempt_ = td::Timestamp::in(60.0); - // id.file_hash contrains correct file hash of zero state - // => if file with this sha256 is found it is garanteed to be correct + // id.file_hash contains correct file hash of zero state + // => if file with this sha256 is found it is guaranteed to be correct // => if it is not, this error is permanent auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id = handle_->id()](td::Result R) { if (R.is_error()) { @@ -108,7 +110,7 @@ void WaitBlockState::start() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_zero_state_request, handle_->id(), priority_, std::move(P)); - } else if (check_persistent_state_desc() && !handle_->received_state()) { + } else if (check_persistent_state_desc() && !handle_->received_state() && allow_download) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { LOG(WARNING) << "failed to get persistent state: " << R.move_as_error(); @@ -122,7 +124,11 @@ void WaitBlockState::start() { timeout_, std::move(P)) .release(); } else if (!handle_->inited_prev() || (!handle_->inited_proof() && !handle_->inited_proof_link())) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { + if (!allow_download) { + abort_query(td::Status::Error(PSTRING() << "not monitoring shard " << handle_->id().shard_full())); + return; + } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); }, td::Timestamp::in(0.1)); @@ -148,6 +154,10 @@ void WaitBlockState::start() { td::actor::send_closure(manager_, &ValidatorManager::wait_prev_block_state, handle_, priority_, timeout_, std::move(P)); } else if (handle_->id().is_masterchain() && !handle_->inited_proof()) { + if (!allow_download) { + abort_query(td::Status::Error(PSTRING() << "not monitoring shard " << handle_->id().shard_full())); + return; + } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { if (R.is_error()) { delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof); }, @@ -161,6 +171,10 @@ void WaitBlockState::start() { td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_request, handle_->id(), priority_, std::move(P)); } else if (block_.is_null()) { + if (!allow_download && !handle_->received()) { + abort_query(td::Status::Error(PSTRING() << "not monitoring shard " << handle_->id().shard_full())); + return; + } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &WaitBlockState::failed_to_get_block_data, @@ -245,6 +259,12 @@ void WaitBlockState::got_block_data(td::Ref data) { } void WaitBlockState::apply() { + if (opts_->get_permanent_celldb()) { + td::actor::send_closure(manager_, &ValidatorManager::set_block_state_from_data, handle_, block_, + std::move(promise_)); + stop(); + return; + } TD_PERF_COUNTER(apply_block_to_state); td::PerfWarningTimer t{"applyblocktostate", 0.1}; auto S = prev_state_.write().apply_block(handle_->id(), block_); diff --git a/validator/downloaders/wait-block-state.hpp b/validator/downloaders/wait-block-state.hpp index 6a14d909fc..a9317381c6 100644 --- a/validator/downloaders/wait-block-state.hpp +++ b/validator/downloaders/wait-block-state.hpp @@ -26,18 +26,21 @@ namespace validator { class WaitBlockState : public td::actor::Actor { public: - WaitBlockState(BlockHandle handle, td::uint32 priority, td::actor::ActorId manager, + WaitBlockState(BlockHandle handle, td::uint32 priority, td::Ref opts, + td::Ref last_masterchain_state, td::actor::ActorId manager, td::Timestamp timeout, td::Promise> promise, td::Ref persistent_state_desc = {}) : handle_(std::move(handle)) , priority_(priority) + , opts_(opts) + , last_masterchain_state_(last_masterchain_state) , manager_(manager) , timeout_(timeout) , promise_(std::move(promise)) , persistent_state_desc_(std::move(persistent_state_desc)) , perf_timer_("waitstate", 1.0, [manager](double duration) { - send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitstate", duration); - }) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitstate", duration); + }) { } void abort_query(td::Status reason); @@ -89,6 +92,8 @@ class WaitBlockState : public td::actor::Actor { td::uint32 priority_; + td::Ref opts_; + td::Ref last_masterchain_state_; td::actor::ActorId manager_; td::Timestamp timeout_; td::Promise> promise_; diff --git a/validator/import-db-slice-local.cpp b/validator/import-db-slice-local.cpp new file mode 100644 index 0000000000..a16a26c630 --- /dev/null +++ b/validator/import-db-slice-local.cpp @@ -0,0 +1,635 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "import-db-slice-local.hpp" + +#include "validator/db/fileref.hpp" +#include "td/utils/overloaded.h" +#include "validator/fabric.h" +#include "td/actor/MultiPromise.h" +#include "common/checksum.h" +#include "td/utils/port/path.h" +#include "ton/ton-io.hpp" +#include "downloaders/download-state.hpp" +#include "block/block-auto.h" + +#include + +namespace ton { + +namespace validator { + +ArchiveImporterLocal::ArchiveImporterLocal(std::string db_root, td::Ref state, + BlockSeqno shard_client_seqno, td::Ref opts, + td::actor::ActorId manager, + std::vector to_import_files, + td::Promise> promise) + : db_root_(std::move(db_root)) + , last_masterchain_state_(std::move(state)) + , shard_client_seqno_(shard_client_seqno) + , opts_(std::move(opts)) + , manager_(manager) + , to_import_files_(std::move(to_import_files)) + , promise_(std::move(promise)) { +} + +void ArchiveImporterLocal::start_up() { + LOG(WARNING) << "Importing archive for masterchain seqno #" << shard_client_seqno_ + 1 << " from disk"; + for (const std::string &path : to_import_files_) { + LOG(INFO) << "Importing file from disk " << path; + td::Status S = process_package(path); + if (S.is_error()) { + LOG(WARNING) << "Error processing package " << path << ": " << S; + } + } + + process_masterchain_blocks(); +} + +td::Status ArchiveImporterLocal::process_package(std::string path) { + LOG(DEBUG) << "Processing package " << path; + TRY_RESULT(p, Package::open(path, false, false)); + auto package = std::make_shared(std::move(p)); + + td::Status S = td::Status::OK(); + package->iterate([&](std::string filename, td::BufferSlice data, td::uint64) -> bool { + auto F = FileReference::create(filename); + if (F.is_error()) { + S = F.move_as_error(); + return false; + } + auto f = F.move_as_ok(); + + BlockIdExt block_id; + bool is_proof = false; + bool ignore = true; + + f.ref().visit(td::overloaded( + [&](const fileref::Proof &p) { + block_id = p.block_id; + ignore = !block_id.is_masterchain(); + is_proof = true; + }, + [&](const fileref::ProofLink &p) { + block_id = p.block_id; + ignore = block_id.is_masterchain(); + is_proof = true; + }, + [&](const fileref::Block &p) { + block_id = p.block_id; + ignore = false; + is_proof = false; + }, + [&](const auto &) { ignore = true; })); + + if (ignore || block_id.is_masterchain() && block_id.seqno() <= last_masterchain_state_->get_seqno()) { + return true; + } + + if (is_proof) { + if (block_id.is_masterchain()) { + auto R = create_proof(block_id, std::move(data)); + if (R.is_error()) { + S = R.move_as_error(); + return false; + } + blocks_[block_id].proof = R.move_as_ok(); + } else { + auto R = create_proof_link(block_id, std::move(data)); + if (R.is_error()) { + S = R.move_as_error(); + return false; + } + blocks_[block_id].proof_link = R.move_as_ok(); + } + } else { + if (td::sha256_bits256(data) != block_id.file_hash) { + S = td::Status::Error(ErrorCode::protoviolation, "bad block file hash"); + return false; + } + auto R = create_block(block_id, std::move(data)); + if (R.is_error()) { + S = R.move_as_error(); + return false; + } + blocks_[block_id].block = R.move_as_ok(); + } + if (block_id.is_masterchain()) { + masterchain_blocks_[block_id.seqno()] = block_id; + } + return true; + }); + return S; +} + +void ArchiveImporterLocal::process_masterchain_blocks() { + if (masterchain_blocks_.empty()) { + LOG(INFO) << "No masterchain blocks in the archive"; + checked_masterchain_proofs(); + return; + } + + if (masterchain_blocks_.begin()->first != last_masterchain_state_->get_seqno() + 1) { + abort_query(td::Status::Error(ErrorCode::notready, PSTRING() << "expected masterchain seqno " + << last_masterchain_state_->get_seqno() + 1 + << ", found " << masterchain_blocks_.begin()->first)); + return; + } + { + BlockSeqno expected_seqno = last_masterchain_state_->get_seqno() + 1; + for (auto &[seqno, _] : masterchain_blocks_) { + if (seqno != expected_seqno) { + abort_query( + td::Status::Error(ErrorCode::protoviolation, "non-consequential masterchain blocks in the archive")); + return; + } + ++expected_seqno; + } + } + BlockInfo &first_block = blocks_[masterchain_blocks_.begin()->second]; + if (first_block.proof.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block proof")); + return; + } + if (first_block.block.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block data")); + return; + } + block::gen::Block::Record rec; + block::gen::BlockInfo::Record info; + if (!(block::gen::unpack_cell(first_block.block->root_cell(), rec) && block::gen::unpack_cell(rec.info, info))) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "cannot unpack masterchain block info")); + return; + } + if (info.key_block) { + import_first_key_block(); + return; + } + + process_masterchain_blocks_cont(); +} + +void ArchiveImporterLocal::import_first_key_block() { + BlockIdExt block_id = masterchain_blocks_.begin()->second; + BlockInfo &first_block = blocks_[block_id]; + LOG(INFO) << "First block in archive is key block : " << block_id.id.to_str(); + + auto P = + td::PromiseCreator::lambda([SelfId = actor_id(this), prev_block_id = last_masterchain_state_->get_block_id()]( + td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + auto handle = R.move_as_ok(); + CHECK(!handle->merge_before()); + if (handle->one_prev(true) != prev_block_id) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + td::Status::Error(ErrorCode::protoviolation, "prev block mismatch")); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::checked_key_block_proof, std::move(handle)); + }); + + run_check_proof_query(block_id, first_block.proof, manager_, td::Timestamp::in(10.0), std::move(P), + last_masterchain_state_, opts_->is_hardfork(block_id)); +} + +void ArchiveImporterLocal::checked_key_block_proof(BlockHandle handle) { + BlockIdExt block_id = masterchain_blocks_.begin()->second; + CHECK(block_id == handle->id()); + BlockInfo &first_block = blocks_[block_id]; + run_apply_block_query( + handle->id(), first_block.block, handle->id(), manager_, td::Timestamp::in(600.0), + [SelfId = actor_id(this), manager = manager_, handle](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure( + manager, &ValidatorManager::get_shard_state_from_db, handle, [=](td::Result> R2) { + if (R2.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R2.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::applied_key_block, + td::Ref{R2.move_as_ok()}); + }); + }); +} + +void ArchiveImporterLocal::applied_key_block(td::Ref state) { + CHECK(state->get_block_id() == masterchain_blocks_.begin()->second); + last_masterchain_state_ = state; + imported_any_ = true; + masterchain_blocks_.erase(masterchain_blocks_.begin()); + blocks_.erase(state->get_block_id()); + LOG(INFO) << "Imported key block " << state->get_block_id().id.to_str(); + if (masterchain_blocks_.empty()) { + LOG(INFO) << "No more masterchain blocks in the archive"; + checked_masterchain_proofs(); + return; + } + process_masterchain_blocks_cont(); +} + +void ArchiveImporterLocal::process_masterchain_blocks_cont() { + LOG(INFO) << "Importing masterchain blocks from " << masterchain_blocks_.begin()->first << " to " + << masterchain_blocks_.rbegin()->first; + + td::MultiPromise mp; + auto ig = mp.init_guard(); + + BlockIdExt prev_block_id = last_masterchain_state_->get_block_id(); + for (auto &[_, block_id] : masterchain_blocks_) { + auto &info = blocks_[block_id]; + info.import = true; + if (info.proof.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block proof")); + return; + } + if (info.block.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block data")); + return; + } + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), prev_block_id, promise = ig.get_promise()](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + CHECK(!handle->merge_before()); + if (handle->one_prev(true) != prev_block_id) { + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "prev block mismatch")); + return; + } + promise.set_result(td::Unit()); + }); + run_check_proof_query(block_id, info.proof, manager_, td::Timestamp::in(10.0), std::move(P), + last_masterchain_state_, opts_->is_hardfork(block_id)); + prev_block_id = block_id; + } + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + } else { + LOG(INFO) << "Checked proofs for masterchain blocks"; + td::actor::send_closure(SelfId, &ArchiveImporterLocal::checked_masterchain_proofs); + } + }); +} + +void ArchiveImporterLocal::checked_masterchain_proofs() { + if (shard_client_seqno_ == last_masterchain_state_->get_seqno()) { + got_shard_client_state(last_masterchain_state_); + } else { + CHECK(shard_client_seqno_ < last_masterchain_state_->get_seqno()); + BlockIdExt block_id; + if (!last_masterchain_state_->get_old_mc_block_id(shard_client_seqno_, block_id)) { + abort_query(td::Status::Error("failed to get shard client block id")); + return; + } + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db_short, block_id, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + R.move_as_error_prefix("failed to get shard client state: ")); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::got_shard_client_state, + td::Ref{R.move_as_ok()}); + }); + } +} + +void ArchiveImporterLocal::got_shard_client_state(td::Ref state) { + CHECK(state->get_seqno() == shard_client_seqno_); + LOG(DEBUG) << "got_shard_client_state " << shard_client_seqno_; + shard_client_state_ = state; + new_shard_client_seqno_ = shard_client_seqno_; + current_shard_client_seqno_ = shard_client_seqno_; + for (auto &shard : state->get_shards()) { + visited_shard_blocks_.insert(shard->top_block_id()); + } + try_advance_shard_client_seqno(); +} + +void ArchiveImporterLocal::try_advance_shard_client_seqno() { + BlockSeqno seqno = new_shard_client_seqno_ + 1; + auto it = masterchain_blocks_.find(seqno); + if (it != masterchain_blocks_.end()) { + try_advance_shard_client_seqno_cont(blocks_[it->second].block); + return; + } + if (seqno > last_masterchain_state_->get_seqno()) { + processed_shard_blocks(); + return; + } + BlockIdExt block_id; + if (!last_masterchain_state_->get_old_mc_block_id(seqno, block_id)) { + abort_query(td::Status::Error("failed to get old mc block id")); + return; + } + td::actor::send_closure(manager_, &ValidatorManager::get_block_data_from_db_short, block_id, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + R.move_as_error_prefix("failed to get block data: ")); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::try_advance_shard_client_seqno_cont, + R.move_as_ok()); + }); +} + +void ArchiveImporterLocal::try_advance_shard_client_seqno_cont(td::Ref mc_block) { + CHECK(mc_block.not_null()); + CHECK(mc_block->block_id().seqno() == new_shard_client_seqno_ + 1); + LOG(DEBUG) << "try_advance_shard_client_seqno " << new_shard_client_seqno_ + 1; + + block::gen::Block::Record rec; + block::gen::BlockExtra::Record extra; + block::gen::McBlockExtra::Record mc_extra; + CHECK(block::gen::unpack_cell(mc_block->root_cell(), rec) && block::gen::unpack_cell(rec.extra, extra) && + block::gen::unpack_cell(extra.custom->prefetch_ref(), mc_extra)); + auto shard_config = std::make_unique(mc_extra.shard_hashes->prefetch_ref()); + + std::vector blocks_to_import; + std::function dfs = [&](const BlockIdExt &block_id) -> td::Status { + if (visited_shard_blocks_.contains(block_id)) { + return td::Status::OK(); + } + if (block_id.seqno() == 0) { + new_zerostates_.insert(block_id); + return td::Status::OK(); + } + visited_shard_blocks_.insert(block_id); + auto &info = blocks_[block_id]; + if (info.block.is_null()) { + return td::Status::Error(PSTRING() << "no shard block " << block_id.to_str()); + } + blocks_to_import.push_back(block_id); + + std::vector prev; + BlockIdExt mc_blkid; + bool after_split; + TRY_STATUS(block::unpack_block_prev_blk_try(info.block->root_cell(), block_id, prev, mc_blkid, after_split)); + for (const BlockIdExt &prev_block_id : prev) { + TRY_STATUS(dfs(prev_block_id)); + } + + return td::Status::OK(); + }; + td::Status S = td::Status::OK(); + std::vector top_shard_blocks; + shard_config->process_shard_hashes([&](block::McShardHash &shard) { + if (!opts_->need_monitor(shard.shard(), shard_client_state_)) { + return 0; + } + S = dfs(shard.top_block_id()); + top_shard_blocks.push_back(shard.top_block_id()); + if (S.is_error()) { + return -1; + } + return 0; + }); + if (S.is_error()) { + LOG(DEBUG) << "Cannot advance shard client seqno to " << new_shard_client_seqno_ + 1 << " : " << S; + processed_shard_blocks(); + return; + } + shard_configs_[mc_block->block_id().seqno()] = {mc_block->block_id(), std::move(top_shard_blocks)}; + ++new_shard_client_seqno_; + LOG(DEBUG) << "Advancing shard client seqno to " << new_shard_client_seqno_; + for (const BlockIdExt &block_id : blocks_to_import) { + blocks_[block_id].import = true; + } + td::actor::send_closure(actor_id(this), &ArchiveImporterLocal::try_advance_shard_client_seqno); +} + +void ArchiveImporterLocal::processed_shard_blocks() { + if (new_shard_client_seqno_ == shard_client_seqno_) { + LOG(INFO) << "No new shard blocks"; + } else { + LOG(INFO) << "New shard client seqno = " << new_shard_client_seqno_; + } + + td::MultiPromise mp; + auto ig = mp.init_guard(); + for (const BlockIdExt &block_id : new_zerostates_) { + LOG(INFO) << "Downloading zerostate " << block_id.to_str(); + td::actor::create_actor( + "downloadstate", block_id, shard_client_state_->get_block_id(), 2, manager_, td::Timestamp::in(3600), + ig.get_promise().wrap([](td::Ref &&) { return td::Unit(); })) + .release(); + } + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::store_data); + } + }); +} + +void ArchiveImporterLocal::store_data() { + td::MultiPromise mp; + auto ig = mp.init_guard(); + + if (opts_->get_permanent_celldb()) { + std::vector> blocks; + for (auto &[_, info] : blocks_) { + if (info.import) { + blocks.push_back(info.block); + } + } + td::actor::send_closure(manager_, &ValidatorManager::set_block_state_from_data_preliminary, std::move(blocks), + ig.get_promise()); + } + for (auto &[block_id, info] : blocks_) { + if (info.import) { + td::actor::send_closure( + manager_, &ValidatorManager::get_block_handle, block_id, true, + [promise = ig.get_promise(), block = info.block, manager = manager_](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + td::actor::send_closure(manager, &ValidatorManager::set_block_data, handle, std::move(block), + std::move(promise)); + }); + if (info.proof_link.not_null()) { + run_check_proof_link_query(block_id, info.proof_link, manager_, td::Timestamp::in(60.0), + ig.get_promise().wrap([](BlockHandle &&) { return td::Unit(); })); + } + } + } + + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::apply_next_masterchain_block); + } + }); +} + +void ArchiveImporterLocal::apply_next_masterchain_block() { + auto it = masterchain_blocks_.find(last_masterchain_state_->get_seqno() + 1); + if (it == masterchain_blocks_.end()) { + LOG(INFO) << "Applied masterchain blocks, last seqno = " << last_masterchain_state_->get_seqno(); + apply_shard_blocks(); + return; + } + BlockIdExt block_id = it->second; + LOG(DEBUG) << "Applying masterchain block " << block_id.to_str(); + BlockInfo &info = blocks_[block_id]; + run_apply_block_query(block_id, info.block, block_id, manager_, td::Timestamp::in(600.0), + [=, SelfId = actor_id(this), manager = manager_](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure( + manager, &ValidatorManager::get_shard_state_from_db_short, block_id, + [=](td::Result> R2) { + if (R2.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + R2.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::applied_next_masterchain_block, + td::Ref{R2.move_as_ok()}); + }); + }); +} + +void ArchiveImporterLocal::applied_next_masterchain_block(td::Ref state) { + last_masterchain_state_ = state; + imported_any_ = true; + LOG(DEBUG) << "Applied masterchain block " << state->get_block_id().to_str(); + apply_next_masterchain_block(); +} + +void ArchiveImporterLocal::apply_shard_blocks() { + if (current_shard_client_seqno_ == new_shard_client_seqno_) { + finish_query(); + return; + } + auto it = shard_configs_.find(current_shard_client_seqno_ + 1); + if (it == shard_configs_.end()) { + abort_query(td::Status::Error("no shard config for the next shard client seqno")); + return; + } + + td::MultiPromise mp; + auto ig = mp.init_guard(); + BlockIdExt mc_block_id = it->second.first; + LOG(DEBUG) << "Applying top shard blocks from " << current_shard_client_seqno_ + 1; + for (const BlockIdExt &block_id : it->second.second) { + apply_shard_block(block_id, mc_block_id, ig.get_promise()); + } + + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::applied_shard_blocks); + }); +} + +void ArchiveImporterLocal::applied_shard_blocks() { + LOG(DEBUG) << "Applied top shard blocks from " << current_shard_client_seqno_ + 1; + ++current_shard_client_seqno_; + imported_any_ = true; + apply_shard_blocks(); +} + +void ArchiveImporterLocal::apply_shard_block(BlockIdExt block_id, BlockIdExt mc_block_id, + td::Promise promise) { + td::actor::send_closure( + manager_, &ValidatorManager::get_block_handle, block_id, true, + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + td::actor::send_closure(SelfId, &ArchiveImporterLocal::apply_shard_block_cont1, R.move_as_ok(), mc_block_id, + std::move(promise)); + }); +} + +void ArchiveImporterLocal::apply_shard_block_cont1(BlockHandle handle, BlockIdExt mc_block_id, + td::Promise promise) { + if (handle->is_applied()) { + promise.set_value(td::Unit()); + return; + } + + promise = [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + td::actor::send_closure(SelfId, &ArchiveImporterLocal::apply_shard_block_cont2, handle, mc_block_id, + std::move(promise)); + }; + + if (!handle->merge_before() && handle->one_prev(true).shard_full() == handle->id().shard_full()) { + apply_shard_block(handle->one_prev(true), mc_block_id, std::move(promise)); + } else { + td::MultiPromise mp; + auto ig = mp.init_guard(); + ig.add_promise(std::move(promise)); + check_shard_block_applied(handle->one_prev(true), ig.get_promise()); + if (handle->merge_before()) { + check_shard_block_applied(handle->one_prev(false), ig.get_promise()); + } + } +} + +void ArchiveImporterLocal::apply_shard_block_cont2(BlockHandle handle, BlockIdExt mc_block_id, + td::Promise promise) { + td::Ref block = blocks_[handle->id()].block; + CHECK(block.not_null()); + LOG(DEBUG) << "Applying shard block " << handle->id().to_str(); + run_apply_block_query(handle->id(), std::move(block), mc_block_id, manager_, td::Timestamp::in(600.0), + std::move(promise)); +} + +void ArchiveImporterLocal::check_shard_block_applied(BlockIdExt block_id, td::Promise promise) { + td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, block_id, false, + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + if (!handle->is_applied()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "not applied")); + } else { + promise.set_value(td::Unit()); + } + }); +} + +void ArchiveImporterLocal::abort_query(td::Status error) { + if (!imported_any_) { + LOG(ERROR) << "Archive import: " << error; + promise_.set_error(std::move(error)); + stop(); + } else { + LOG(WARNING) << "Archive import: " << error; + finish_query(); + } +} + +void ArchiveImporterLocal::finish_query() { + LOG(WARNING) << "Imported archive in " << timer_.elapsed() << "s : mc_seqno=" << last_masterchain_state_->get_seqno() + << " shard_seqno=" << current_shard_client_seqno_; + promise_.set_value({last_masterchain_state_->get_seqno(), + std::min(last_masterchain_state_->get_seqno(), current_shard_client_seqno_)}); + stop(); +} + +} // namespace validator + +} // namespace ton diff --git a/validator/import-db-slice-local.hpp b/validator/import-db-slice-local.hpp new file mode 100644 index 0000000000..778e4eed60 --- /dev/null +++ b/validator/import-db-slice-local.hpp @@ -0,0 +1,103 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "td/actor/actor.h" +#include "td/utils/port/path.h" +#include "validator/interfaces/validator-manager.h" +#include "validator/db/package.hpp" + +namespace ton { + +namespace validator { + +class ArchiveImporterLocal : public td::actor::Actor { + public: + ArchiveImporterLocal(std::string db_root, td::Ref state, BlockSeqno shard_client_seqno, + td::Ref opts, td::actor::ActorId manager, + std::vector to_import_files, + td::Promise> promise); + void start_up() override; + + void abort_query(td::Status error); + void finish_query(); + + td::Status process_package(std::string path); + + void process_masterchain_blocks(); + void process_masterchain_blocks_cont(); + + void import_first_key_block(); + void checked_key_block_proof(BlockHandle handle); + void applied_key_block(td::Ref state); + + void checked_masterchain_proofs(); + void got_shard_client_state(td::Ref state); + + void try_advance_shard_client_seqno(); + void try_advance_shard_client_seqno_cont(td::Ref mc_block); + + void processed_shard_blocks(); + void store_data(); + void apply_next_masterchain_block(); + void applied_next_masterchain_block(td::Ref state); + + void apply_shard_blocks(); + void applied_shard_blocks(); + + void apply_shard_block(BlockIdExt block_id, BlockIdExt mc_block_id, td::Promise promise); + void apply_shard_block_cont1(BlockHandle handle, BlockIdExt mc_block_id, td::Promise promise); + void apply_shard_block_cont2(BlockHandle handle, BlockIdExt mc_block_id, td::Promise promise); + void check_shard_block_applied(BlockIdExt block_id, td::Promise promise); + + private: + std::string db_root_; + td::Ref last_masterchain_state_; + BlockSeqno shard_client_seqno_; + + td::Ref opts_; + + td::actor::ActorId manager_; + + std::vector to_import_files_; + td::Promise> promise_; + + struct BlockInfo { + td::Ref block; + td::Ref proof; + td::Ref proof_link; + bool import = false; + }; + std::map blocks_; + std::map masterchain_blocks_; + + td::Ref shard_client_state_; + BlockSeqno new_shard_client_seqno_; + BlockSeqno current_shard_client_seqno_; + std::set visited_shard_blocks_; + std::set new_zerostates_; + + std::map>> shard_configs_; + + bool imported_any_ = false; + + td::Timer timer_; +}; + +} // namespace validator + +} // namespace ton diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index 06573d347e..64861a8bb9 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -85,7 +85,7 @@ void ArchiveImporter::downloaded_mc_archive(std::string path) { void ArchiveImporter::processed_mc_archive() { if (masterchain_blocks_.empty()) { - LOG(DEBUG) << "No masterhchain blocks in archive"; + LOG(DEBUG) << "No masterchain blocks in archive"; last_masterchain_seqno_ = last_masterchain_state_->get_seqno(); checked_all_masterchain_blocks(); return; diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 29ef715b36..3a26871e1a 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -50,6 +50,10 @@ class Db : public td::actor::Actor { virtual void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; + virtual void store_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) = 0; + virtual void store_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) = 0; virtual void get_block_state(ConstBlockHandle handle, td::Promise> promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 00fb77e1eb..c311cc08e1 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -70,6 +70,10 @@ class ValidatorManager : public ValidatorManagerInterface { } virtual void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; + virtual void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) = 0; + virtual void set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 62fdc4b435..a487556c46 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -321,8 +321,8 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle->id(), std::move(R)); }); - auto id = td::actor::create_actor("waitstate", handle, 0, actor_id(this), td::Timestamp::in(10.0), - std::move(P)) + auto id = td::actor::create_actor("waitstate", handle, 0, opts_, last_masterchain_state_, + actor_id(this), td::Timestamp::in(10.0), std::move(P)) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); @@ -686,6 +686,16 @@ void ValidatorManagerImpl::set_block_state(BlockHandle handle, td::Ref block, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data, handle, block, std::move(promise)); +} + +void ValidatorManagerImpl::set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data_preliminary, std::move(blocks), std::move(promise)); +} + void ValidatorManagerImpl::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise)); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index cd06bf555d..3c516ef155 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -148,6 +148,10 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; + void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override; + void set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) override; void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) override; diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index 91c598aa2e..c3d6267bef 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -171,8 +171,8 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle->id(), std::move(R)); }); - auto id = td::actor::create_actor("waitstate", handle, 0, actor_id(this), td::Timestamp::in(10.0), - std::move(P)) + auto id = td::actor::create_actor("waitstate", handle, 0, opts_, last_masterchain_state_, + actor_id(this), td::Timestamp::in(10.0), std::move(P)) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 0b8b9e736d..b28b29685f 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -174,6 +174,13 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { UNREACHABLE(); } + void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override { + UNREACHABLE(); + } + void set_block_state_from_data_preliminary(std::vector> blocks, td::Promise promise) { + UNREACHABLE(); + } void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) override { diff --git a/validator/manager.cpp b/validator/manager.cpp index 00b923c236..1d75b523a6 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -31,6 +31,7 @@ #include "state-serializer.hpp" #include "get-next-key-blocks.h" #include "import-db-slice.hpp" +#include "import-db-slice-local.hpp" #include "auto/tl/lite_api.h" #include "tl-utils/lite-utils.hpp" @@ -685,10 +686,6 @@ void ValidatorManagerImpl::run_ext_query(td::BufferSlice data, td::Promise> promise) { - if (last_masterchain_state_.not_null() && !opts_->need_monitor(handle->id().shard_full(), last_masterchain_state_)) { - return promise.set_error( - td::Status::Error(PSTRING() << "not monitoring shard " << handle->id().shard_full().to_str())); - } auto it0 = block_state_cache_.find(handle->id()); if (it0 != block_state_cache_.end()) { it0->second.ttl_ = td::Timestamp::in(30.0); @@ -700,10 +697,11 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); }); - auto id = td::actor::create_actor("waitstate", handle, priority, actor_id(this), - td::Timestamp::at(timeout.at() + 10.0), std::move(P), - get_block_persistent_state_to_download(handle->id())) - .release(); + auto id = + td::actor::create_actor("waitstate", handle, priority, opts_, last_masterchain_state_, + actor_id(this), td::Timestamp::at(timeout.at() + 10.0), std::move(P), + get_block_persistent_state_to_download(handle->id())) + .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); } @@ -762,10 +760,6 @@ void ValidatorManagerImpl::wait_block_data_short(BlockIdExt block_id, td::uint32 void ValidatorManagerImpl::wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) { - if (last_masterchain_state_.not_null() && !opts_->need_monitor(left_id.shard_full(), last_masterchain_state_)) { - return promise.set_error( - td::Status::Error(PSTRING() << "not monitoring shard " << left_id.shard_full().to_str())); - } td::actor::create_actor("merge", left_id, right_id, priority, actor_id(this), timeout, std::move(promise)) .release(); @@ -1149,10 +1143,10 @@ void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); }); - auto id = - td::actor::create_actor("waitstate", handle, X.second, actor_id(this), X.first, - std::move(P), get_block_persistent_state_to_download(handle->id())) - .release(); + auto id = td::actor::create_actor("waitstate", handle, X.second, opts_, last_masterchain_state_, + actor_id(this), X.first, std::move(P), + get_block_persistent_state_to_download(handle->id())) + .release(); it->second.actor_ = id; return; } @@ -1211,6 +1205,16 @@ void ValidatorManagerImpl::set_block_state(BlockHandle handle, td::Ref block, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data, handle, block, std::move(promise)); +} + +void ValidatorManagerImpl::set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data_preliminary, std::move(blocks), std::move(promise)); +} + void ValidatorManagerImpl::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise)); } @@ -1688,6 +1692,7 @@ void ValidatorManagerImpl::start_up() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::started, R.move_as_ok()); }); + size_t to_import_files = 0; auto to_import_dir = db_root_ + "/import"; auto S = td::WalkPath::run(to_import_dir, [&](td::CSlice cfname, td::WalkPath::Type t) -> void { auto fname = td::Slice(cfname); @@ -1707,26 +1712,30 @@ void ValidatorManagerImpl::start_up() { } fname = fname.substr(8); - while (fname.size() > 1 && fname[0] == '0') { - fname.remove_prefix(1); - } auto i = fname.find('.'); if (i == td::Slice::npos) { return; } fname = fname.substr(0, i); + while (fname.size() > 1 && fname[0] == '0') { + fname.remove_prefix(1); + } auto v = td::to_integer_safe(fname); if (v.is_error()) { return; } auto seqno = v.move_as_ok(); - LOG(INFO) << "found archive slice '" << cfname << "' for seqno " << seqno; + LOG(DEBUG) << "found archive slice '" << cfname << "' for seqno " << seqno; to_import_[seqno].push_back(cfname.str()); + ++to_import_files; } }); if (S.is_error()) { LOG(INFO) << "failed to load blocks from import dir: " << S; } + if (to_import_files > 0) { + LOG(INFO) << "found " << to_import_files << " files to import"; + } validator_manager_init(opts_, actor_id(this), db_.get(), std::move(P)); @@ -1909,9 +1918,15 @@ void ValidatorManagerImpl::download_next_archive() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::checked_archive_slice, R.ok().first, R.ok().second); } }); - td::actor::create_actor("archiveimport", db_root_, last_masterchain_state_, seqno, opts_, - actor_id(this), std::move(to_import_files), std::move(P)) - .release(); + if (to_import_files.empty()) { + td::actor::create_actor("archiveimport", db_root_, last_masterchain_state_, seqno, opts_, + actor_id(this), std::move(to_import_files), std::move(P)) + .release(); + } else { + td::actor::create_actor("archiveimport", db_root_, last_masterchain_state_, seqno, opts_, + actor_id(this), std::move(to_import_files), std::move(P)) + .release(); + } } void ValidatorManagerImpl::checked_archive_slice(BlockSeqno new_last_mc_seqno, BlockSeqno new_shard_client_seqno) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 418deb3505..850baf47be 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -407,6 +407,9 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; + void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override; + void set_block_state_from_data_preliminary(std::vector> blocks, td::Promise promise) override; void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) override; diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index cb26fe44d6..a87562377f 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -24,14 +24,13 @@ namespace ton { namespace validator { -td::Ref ValidatorManagerOptions::create( - BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, bool allow_blockchain_init, - double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, - double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) { - return td::make_ref(zero_block_id, init_block_id, std::move(check_shard), - allow_blockchain_init, sync_blocks_before, block_ttl, state_ttl, - max_mempool_num, +td::Ref ValidatorManagerOptions::create(BlockIdExt zero_block_id, BlockIdExt init_block_id, + bool allow_blockchain_init, double sync_blocks_before, + double block_ttl, double state_ttl, + double max_mempool_num, double archive_ttl, + double key_proof_ttl, bool initial_sync_disabled) { + return td::make_ref(zero_block_id, init_block_id, allow_blockchain_init, + sync_blocks_before, block_ttl, state_ttl, max_mempool_num, archive_ttl, key_proof_ttl, initial_sync_disabled); } diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 45b8d7ec20..def897995d 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -34,7 +34,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { } bool need_monitor(ShardIdFull shard, const td::Ref& state) const override { td::uint32 min_split = state->monitor_min_split_depth(shard.workchain); - return check_shard_((td::uint32)shard.pfx_len() <= min_split ? shard : shard_prefix(shard, min_split)); + return check_shard_((td::uint32)shard.pfx_len() <= min_split ? shard : shard_prefix(shard, min_split), + state->get_seqno()); } bool allow_blockchain_init() const override { return allow_blockchain_init_; @@ -160,6 +161,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { double get_catchain_broadcast_speed_multiplier() const override { return catchain_broadcast_speed_multipliers_; } + bool get_permanent_celldb() const override { + return permanent_celldb_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -167,7 +171,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_init_block_id(BlockIdExt block_id) override { init_block_id_ = block_id; } - void set_shard_check_function(std::function check_shard) override { + void set_shard_check_function(std::function check_shard) override { check_shard_ = std::move(check_shard); } void set_allow_blockchain_init(bool value) override { @@ -261,18 +265,19 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_catchain_broadcast_speed_multiplier(double value) override { catchain_broadcast_speed_multipliers_ = value; } + void set_permanent_celldb(bool value) override { + permanent_celldb_ = value; + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); } - ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, bool allow_blockchain_init, + ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, bool allow_blockchain_init, double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) : zero_block_id_(zero_block_id) , init_block_id_(init_block_id) - , check_shard_(std::move(check_shard)) , allow_blockchain_init_(allow_blockchain_init) , sync_blocks_before_(sync_blocks_before) , block_ttl_(block_ttl) @@ -286,7 +291,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { private: BlockIdExt zero_block_id_; BlockIdExt init_block_id_; - std::function check_shard_; + std::function check_shard_ = [](ShardIdFull, BlockSeqno) { return true; }; bool allow_blockchain_init_; double sync_blocks_before_; double block_ttl_; @@ -316,6 +321,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { td::Ref collator_options_{true}; bool fast_state_serializer_enabled_ = false; double catchain_broadcast_speed_multipliers_; + bool permanent_celldb_ = false; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 66795ceceb..4631bf7adb 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -118,10 +118,11 @@ struct ValidatorManagerOptions : public td::CntObject { virtual td::Ref get_collator_options() const = 0; virtual bool get_fast_state_serializer_enabled() const = 0; virtual double get_catchain_broadcast_speed_multiplier() const = 0; + virtual bool get_permanent_celldb() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; - virtual void set_shard_check_function(std::function check_shard) = 0; + virtual void set_shard_check_function(std::function check_shard) = 0; virtual void set_allow_blockchain_init(bool value) = 0; virtual void set_sync_blocks_before(double value) = 0; virtual void set_block_ttl(double value) = 0; @@ -152,11 +153,10 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_collator_options(td::Ref value) = 0; virtual void set_fast_state_serializer_enabled(bool value) = 0; virtual void set_catchain_broadcast_speed_multiplier(double value) = 0; + virtual void set_permanent_celldb(bool value) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, - - std::function check_shard = [](ShardIdFull) { return true; }, bool allow_blockchain_init = false, double sync_blocks_before = 3600, double block_ttl = 86400, double state_ttl = 86400, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650, double max_mempool_num = 999999, bool initial_sync_disabled = false); From 949ae86038af8cca3dcbce845bf02daa3ede36ad Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sat, 22 Mar 2025 13:57:03 +0300 Subject: [PATCH 173/524] Fix persistent state lookup --- validator/net/download-state.cpp | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index 6735a2b5fd..550c8882e8 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -73,16 +73,20 @@ void DownloadState::start_up() { status_ = ProcessStatus(validator_manager_, "process.download_state_net"); alarm_timestamp() = timeout_; - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id_, - masterchain_block_id_, - [SelfId = actor_id(this), block_id = block_id_](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &DownloadState::get_block_handle); - } else { - LOG(WARNING) << "got block state from disk: " << block_id.to_str(); - td::actor::send_closure(SelfId, &DownloadState::got_block_state, R.move_as_ok()); - } - }); + td::Promise P = [SelfId = actor_id(this), block_id = block_id_](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadState::get_block_handle); + } else { + LOG(WARNING) << "got block state from disk: " << block_id.to_str(); + td::actor::send_closure(SelfId, &DownloadState::got_block_state, R.move_as_ok()); + } + }; + if (block_id_.seqno() == 0) { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_zero_state, block_id_, std::move(P)); + } else { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id_, + masterchain_block_id_, std::move(P)); + } } void DownloadState::get_block_handle() { From 71ddef2de02d7568127d33286861f48333a2d041 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sat, 22 Mar 2025 13:57:32 +0300 Subject: [PATCH 174/524] Tool for generating liteserver desc for global config --- utils/CMakeLists.txt | 3 + utils/prepare-ls-slice-config.cpp | 266 ++++++++++++++++++++++++++++++ 2 files changed, 269 insertions(+) create mode 100644 utils/prepare-ls-slice-config.cpp diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 99a4e92df5..0afdf7389f 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -22,4 +22,7 @@ target_include_directories(pack-viewer PUBLIC $. +*/ +#include "td/utils/filesystem.h" +#include "td/actor/actor.h" +#include "td/actor/MultiPromise.h" +#include "td/utils/OptionParser.h" +#include "td/utils/port/path.h" +#include "td/utils/port/signals.h" +#include "td/utils/port/IPAddress.h" +#include "td/utils/Random.h" +#include "td/utils/FileLog.h" +#include "git.h" +#include "auto/tl/ton_api.h" +#include "auto/tl/lite_api.h" +#include "tl-utils/lite-utils.hpp" +#include "auto/tl/ton_api_json.h" +#include "adnl/adnl.h" +#include "lite-client/ext-client.h" +#include "ton/lite-tl.hpp" + +#include "td/utils/overloaded.h" + +#include +#include "td/utils/tl_storers.h" +#include "vm/boc.h" +#include "vm/cells/MerkleProof.h" + +#include +#include "block/block-auto.h" +#include "block/mc-config.h" + +using namespace ton; + +std::string global_config_file; +td::optional start_mc_seqno, end_mc_seqno; +std::vector shards; + +class PrepareLsSliceConfig : public td::actor::Actor { + public: + void start_up() override { + if (start_mc_seqno && end_mc_seqno && start_mc_seqno.value() > end_mc_seqno.value()) { + LOG(FATAL) << "from-seqno is greater than to-seqno"; + } + + if (!start_mc_seqno && !end_mc_seqno) { + auto slice = create_tl_object(); + if (shards.empty()) { + slice->shards_.push_back(create_tl_shard_id(ShardIdFull{basechainId, shardIdAll})); + } else { + for (const ShardIdFull& shard : shards) { + if (!shard.is_masterchain()) { + slice->shards_.push_back(create_tl_shard_id(shard)); + } + } + } + print_result(*slice); + return; + } + + auto gc_s = td::read_file(global_config_file).move_as_ok(); + auto gc_j = td::json_decode(gc_s.as_slice()).move_as_ok(); + ton_api::liteclient_config_global gc; + ton_api::from_json(gc, gc_j.get_object()).ensure(); + auto r_servers = liteclient::LiteServerConfig::parse_global_config(gc); + r_servers.ensure(); + client_ = liteclient::ExtClient::create(r_servers.move_as_ok(), nullptr); + + slice_timed_ = create_tl_object(); + ++pending_; + request_shards_info(start_mc_seqno, true); + request_shards_info(end_mc_seqno, false); + dec_pending(); + } + + template + static td::BufferSlice create_query(Args&&... args) { + Type object(std::forward(args)...); + return create_serialize_tl_object(serialize_tl_object(&object, true)); + } + + template + static tl_object_ptr parse_response(const td::Result& R) { + R.ensure(); + auto err = fetch_tl_object(R.ok(), true); + if (err.is_ok()) { + LOG(FATAL) << "liteserver error: " << err.ok()->message_; + } + auto res = fetch_tl_object(R.ok(), true); + res.ensure(); + return res.move_as_ok(); + } + + void request_shards_info(td::optional seqno, bool is_start) { + if (!seqno) { + return; + } + ++pending_; + td::actor::send_closure( + client_, &liteclient::ExtClient::send_query, "q", + create_query( + 1, create_tl_object(masterchainId, shardIdAll, seqno.value()), 0, 0), + td::Timestamp::in(5.0), [=, client = client_.get(), SelfId = actor_id(this)](td::Result R) { + auto mc_header = parse_response(std::move(R)); + auto block_id = create_block_id(mc_header->id_); + td::actor::send_closure( + client, &liteclient::ExtClient::send_query, "q", + create_query(create_tl_lite_block_id(block_id)), + td::Timestamp::in(5.0), [=, mc_header = std::move(mc_header)](td::Result R) mutable { + auto shards_info = parse_response(std::move(R)); + td::actor::send_closure(SelfId, &PrepareLsSliceConfig::got_shards_info, std::move(mc_header), + std::move(shards_info), is_start); + }); + }); + } + + static tl_object_ptr parse_header(const lite_api::liteServer_blockHeader& obj, + bool is_start) { + auto res = create_tl_object(); + + BlockIdExt block_id = create_block_id(obj.id_); + res->shard_id_ = create_tl_shard_id(block_id.shard_full()); + res->seqno_ = block_id.seqno(); + + auto root = vm::std_boc_deserialize(obj.header_proof_).move_as_ok(); + root = vm::MerkleProof::virtualize(root, 1); + block::gen::Block::Record blk; + block::gen::BlockInfo::Record info; + CHECK(tlb::unpack_cell(root, blk) && tlb::unpack_cell(blk.info, info)); + res->utime_ = info.gen_utime; + res->lt_ = (is_start ? info.start_lt : info.end_lt); + + return res; + } + + void got_shards_info(tl_object_ptr mc_header, + tl_object_ptr shards_info, bool is_start) { + (is_start ? slice_timed_->shards_from_ : slice_timed_->shards_to_).push_back(parse_header(*mc_header, is_start)); + + auto root = vm::std_boc_deserialize(shards_info->data_).move_as_ok(); + block::ShardConfig sh_conf; + CHECK(sh_conf.unpack(vm::load_cell_slice_ref(root))); + auto ids = sh_conf.get_shard_hash_ids(true); + for (auto id : ids) { + BlockIdExt block_id = sh_conf.get_shard_hash(ton::ShardIdFull(id))->top_block_id(); + bool ok = shards.empty(); + for (const auto& our_shard : shards) { + if (shard_intersects(our_shard, block_id.shard_full())) { + ok = true; + break; + } + } + if (ok) { + ++pending_; + td::actor::send_closure( + client_, &liteclient::ExtClient::send_query, "q", + create_query(create_tl_lite_block_id(block_id), 0xffff), + td::Timestamp::in(5.0), [=, SelfId = actor_id(this)](td::Result R) mutable { + auto header = parse_response(std::move(R)); + td::actor::send_closure(SelfId, &PrepareLsSliceConfig::got_block_header, std::move(header), is_start); + }); + } + } + + dec_pending(); + } + + void got_block_header(tl_object_ptr header, bool is_start) { + (is_start ? slice_timed_->shards_from_ : slice_timed_->shards_to_).push_back(parse_header(*header, is_start)); + dec_pending(); + } + + void print_result(const ton_api::liteserver_descV2_Slice& result) { + auto s = td::json_encode(td::ToJson(result), true); + std::cout << s << "\n"; + std::cout.flush(); + exit(0); + } + + private: + td::actor::ActorOwn client_; + tl_object_ptr slice_timed_; + size_t pending_ = 0; + + void dec_pending() { + --pending_; + if (pending_ == 0) { + auto cmp = [](const tl_object_ptr& a, + const tl_object_ptr& b) { + return create_shard_id(a->shard_id_) < create_shard_id(b->shard_id_); + }; + std::sort(slice_timed_->shards_from_.begin(), slice_timed_->shards_from_.end(), cmp); + std::sort(slice_timed_->shards_to_.begin(), slice_timed_->shards_to_.end(), cmp); + print_result(*slice_timed_); + } + } +}; + +int main(int argc, char* argv[]) { + SET_VERBOSITY_LEVEL(verbosity_INFO); + + td::unique_ptr logger_; + SCOPE_EXIT { + td::log_interface = td::default_log_interface; + }; + + td::OptionParser p; + p.set_description( + "Generate liteserver.descV2.Slice for global-config.json from given shards and masterchain seqnos\n"); + p.add_option('v', "verbosity", "set verbosity level", [&](td::Slice arg) { + int v = VERBOSITY_NAME(FATAL) + (td::to_integer(arg)); + SET_VERBOSITY_LEVEL(v); + }); + p.add_option('V', "version", "show build information", [&]() { + std::cout << "prepare-ls-slice-config build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::exit(0); + }); + p.add_option('h', "help", "print help", [&]() { + char b[10240]; + td::StringBuilder sb(td::MutableSlice{b, 10000}); + sb << p; + std::cout << sb.as_cslice().c_str(); + std::exit(2); + }); + p.add_option('C', "global-config", "global TON configuration file (used to fetch shard configuration)", + [&](td::Slice arg) { global_config_file = arg.str(); }); + p.add_checked_option('f', "from-seqno", "starting masterchain seqno (default: none)", + [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(start_mc_seqno, td::to_integer_safe(arg)) + return td::Status::OK(); + }); + p.add_checked_option('t', "to-seqno", "ending masterchain seqno (default: none)", [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(end_mc_seqno, td::to_integer_safe(arg)) + return td::Status::OK(); + }); + p.add_checked_option('s', "shard", "shard in format 0:8000000000000000 (default: all shards)", + [&](td::Slice arg) -> td::Status { + TRY_RESULT(shard, ShardIdFull::parse(arg)); + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << arg); + } + shards.push_back(shard); + return td::Status::OK(); + }); + + p.run(argc, argv).ensure(); + td::actor::Scheduler scheduler({3}); + + scheduler.run_in_context([&] { td::actor::create_actor("main").release(); }); + while (scheduler.run(1)) { + } +} From ab4888702aa0f7844e90826f060f715ee7cc31ee Mon Sep 17 00:00:00 2001 From: k-dimentional tree <98183693+kdimentionaltree@users.noreply.github.com> Date: Mon, 24 Mar 2025 16:23:07 +0300 Subject: [PATCH 175/524] Added normalized hash of external-in message and increased stack depth for tonlib (#1557) * Update stack_limit * Added external-in normalized hash and increased stack depth for tonlib * Removed debug logs --- tl/generate/scheme/tonlib_api.tl | 2 +- tl/generate/scheme/tonlib_api.tlo | Bin 34344 -> 34380 bytes tonlib/tonlib/TonlibClient.cpp | 44 ++++++++++++++++++++++++++++-- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index 31ca6fd400..e7d8cb59c5 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -57,7 +57,7 @@ raw.message hash:bytes source:accountAddress destination:accountAddress value:in raw.transaction address:accountAddress utime:int53 data:bytes transaction_id:internal.transactionId fee:int64 storage_fee:int64 other_fee:int64 in_msg:raw.message out_msgs:vector = raw.Transaction; raw.transactions transactions:vector previous_transaction_id:internal.transactionId = raw.Transactions; -raw.extMessageInfo hash:bytes = raw.ExtMessageInfo; +raw.extMessageInfo hash:bytes hash_norm:bytes = raw.ExtMessageInfo; pchan.config alice_public_key:string alice_address:accountAddress bob_public_key:string bob_address:accountAddress init_timeout:int32 close_timeout:int32 channel_id:int64 = pchan.Config; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 10b9ed8db207f9ad7b39322937bafc107802d319..9b366407e367642cb707b63d0f93161ff34b1bf1 100644 GIT binary patch delta 106 zcmZ3{!*r&HX+sMy%VYiX8k;+K=QzlKSVBdK<$9?VCBCV}#fj;uo_T5c3?T8z6GCMs v8`$$qPEcgw;mk-Z&WO*;FUnwETN*ra=p}w65rI~;>7e+&%CsJ29Ws8V?sWR blO^r7A?j!NEinMICOgza7@JG#Z?FLXZZIFV diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index d73e715c93..ae3189ba8b 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -3312,9 +3312,47 @@ td::Status TonlibClient::do_request(const tonlib_api::raw_sendMessageReturnHash& td::Promise>&& promise) { TRY_RESULT_PREFIX(body, vm::std_boc_deserialize(request.body_), TonlibError::InvalidBagOfCells("body")); auto hash = body->get_hash().as_slice().str(); + + // compute hash normalized + block::gen::Message::Record message; + if (!tlb::type_unpack_cell(body, block::gen::t_Message_Any, message)) { + return td::Status::Error("Failed to unpack Message"); + } + auto tag = block::gen::CommonMsgInfo().get_tag(*message.info); + if (tag != block::gen::CommonMsgInfo::ext_in_msg_info) { + return td::Status::Error("CommonMsgInfo tag is not ext_in_msg_info"); + } + block::gen::CommonMsgInfo::Record_ext_in_msg_info msg_info; + if (!tlb::csr_unpack(message.info, msg_info)) { + return td::Status::Error("Failed to unpack CommonMsgInfo::ext_in_msg_info"); + } + + td::Ref body_norm; + auto body_cs = message.body.write(); + if (body_cs.fetch_long(1) == 1) { + body_norm = body_cs.fetch_ref(); + } else { + body_norm = vm::CellBuilder().append_cellslice(body_cs).finalize(); + } + + auto cb = vm::CellBuilder(); + bool status = + cb.store_long_bool(2, 2) && // message$_ -> info:CommonMsgInfo -> ext_in_msg_info$10 + cb.store_long_bool(0, 2) && // message$_ -> info:CommonMsgInfo -> src:MsgAddressExt -> addr_none$00 + cb.append_cellslice_bool(msg_info.dest) && // message$_ -> info:CommonMsgInfo -> dest:MsgAddressInt + cb.store_long_bool(0, 4) && // message$_ -> info:CommonMsgInfo -> import_fee:Grams -> 0 + cb.store_long_bool(0, 1) && // message$_ -> init:(Maybe (Either StateInit ^StateInit)) -> nothing$0 + cb.store_long_bool(1, 1) && // message$_ -> body:(Either X ^X) -> right$1 + cb.store_ref_bool(body_norm); + + if (!status) { + return td::Status::Error("Failed to build normalized message"); + } + auto hash_norm = cb.finalize()->get_hash().as_slice().str(); + make_request(int_api::SendMessage{std::move(body)}, - promise.wrap([hash = std::move(hash)](auto res) { - return tonlib_api::make_object(std::move(hash)); + promise.wrap([hash = std::move(hash), hash_norm = std::move(hash_norm)](auto res) { + return tonlib_api::make_object(std::move(hash), std::move(hash_norm)); })); return td::Status::OK(); } @@ -4510,7 +4548,7 @@ auto to_tonlib_api(const vm::StackEntry& entry, int& limit) -> td::Result& stack) -> td::Result>> { - int stack_limit = 1000; + int stack_limit = 8000; std::vector> tl_stack; for (auto& entry: stack->as_span()) { TRY_RESULT(tl_entry, to_tonlib_api(entry, --stack_limit)); From 48f5657d221a52866f4167fd49adf92a3d5affed Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 25 Mar 2025 12:48:23 +0300 Subject: [PATCH 176/524] Import proof for initial block --- validator/downloaders/download-state.cpp | 16 +++++- validator/downloaders/download-state.hpp | 1 + validator/interfaces/validator-manager.h | 5 ++ validator/manager-init.cpp | 27 +++++----- validator/manager.cpp | 63 ++++++++++++++++++++++++ validator/manager.hpp | 4 ++ 6 files changed, 103 insertions(+), 13 deletions(-) diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index 8473cb228b..a6de05893d 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -72,7 +72,22 @@ void DownloadShardState::download_state() { checked_proof_link(); return; } + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading proof"); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = block_id_](td::Result R) { + if (R.is_error()) { + LOG(DEBUG) << "Cannot get proof link from import: " << R.move_as_error(); + td::actor::send_closure(SelfId, &DownloadShardState::download_proof_link); + } else { + LOG(INFO) << "Got proof link for " << block_id.to_str() << " from import"; + td::actor::send_closure(SelfId, &DownloadShardState::downloaded_proof_link, R.move_as_ok()); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::get_block_proof_link_from_import, block_id_, + masterchain_block_id_, std::move(P)); +} + +void DownloadShardState::download_proof_link() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { fail_handler(SelfId, R.move_as_error()); @@ -82,7 +97,6 @@ void DownloadShardState::download_state() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, block_id_, priority_, std::move(P)); - status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading proof"); } void DownloadShardState::downloaded_proof_link(td::BufferSlice data) { diff --git a/validator/downloaders/download-state.hpp b/validator/downloaders/download-state.hpp index bde80aae15..9b3d7fa5e1 100644 --- a/validator/downloaders/download-state.hpp +++ b/validator/downloaders/download-state.hpp @@ -38,6 +38,7 @@ class DownloadShardState : public td::actor::Actor { void checked_proof_link(); void download_state(); + void download_proof_link(); void download_zero_state(); void downloaded_zero_state(td::BufferSlice data); diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index c311cc08e1..3cb2a2c457 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -156,6 +156,11 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; + virtual void get_block_proof_link_from_import(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + promise.set_error(td::Status::Error("not supported")); + } + virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index 6f304680ba..3b57eb9b8e 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -59,7 +59,7 @@ void ValidatorManagerMasterchainReiniter::got_masterchain_handle(BlockHandle han handle_ = std::move(handle); key_blocks_.push_back(handle_); - if (opts_->initial_sync_disabled()) { + if (opts_->initial_sync_disabled() && handle_->id().seqno() == 0) { status_.set_status(PSTRING() << "downloading masterchain state " << handle_->id().seqno()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); @@ -94,22 +94,21 @@ void ValidatorManagerMasterchainReiniter::download_proof_link() { td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::downloaded_proof_link, R.move_as_ok()); } }); - td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, handle_->id(), 2, + td::actor::send_closure(manager_, &ValidatorManager::get_block_proof_link_from_import, handle_->id(), handle_->id(), std::move(P)); } } -void ValidatorManagerMasterchainReiniter::downloaded_proof_link(td::BufferSlice proof) { - auto pp = create_proof_link(handle_->id(), std::move(proof)); - if (pp.is_error()) { - LOG(WARNING) << "bad proof link: " << pp.move_as_error(); +void ValidatorManagerMasterchainReiniter::downloaded_proof_link(td::BufferSlice data) { + auto r_proof = create_proof(handle_->id(), std::move(data)); + if (r_proof.is_error()) { + LOG(WARNING) << "bad proof link: " << r_proof.move_as_error(); download_proof_link(); return; } + auto proof = r_proof.move_as_ok(); - auto proof_link = pp.move_as_ok(); - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), db = db_, proof_link](td::Result R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), db = db_, proof](td::Result R) { if (R.is_error()) { LOG(WARNING) << "downloaded proof link failed: " << R.move_as_error(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link); @@ -118,11 +117,11 @@ void ValidatorManagerMasterchainReiniter::downloaded_proof_link(td::BufferSlice R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::try_download_key_blocks, false); }); - td::actor::send_closure(db, &Db::add_key_block_proof_link, proof_link, std::move(P)); + td::actor::send_closure(db, &Db::add_key_block_proof_link, proof, std::move(P)); } }); - - run_check_proof_link_query(handle_->id(), proof_link, manager_, td::Timestamp::in(60.0), std::move(P)); + run_check_proof_query(handle_->id(), proof, manager_, td::Timestamp::in(60.0), std::move(P), + /* skip_check_signatures = */ true); } void ValidatorManagerMasterchainReiniter::downloaded_zero_state() { @@ -130,6 +129,10 @@ void ValidatorManagerMasterchainReiniter::downloaded_zero_state() { } void ValidatorManagerMasterchainReiniter::try_download_key_blocks(bool try_start) { + if (opts_->initial_sync_disabled()) { + download_masterchain_state(); + return; + } if (!download_new_key_blocks_until_) { if (opts_->allow_blockchain_init()) { download_new_key_blocks_until_ = td::Timestamp::in(60.0); diff --git a/validator/manager.cpp b/validator/manager.cpp index 1d75b523a6..680f949eca 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -43,6 +43,7 @@ #include "td/utils/JsonBuilder.h" #include "common/delay.h" +#include "db/fileref.hpp" #include "td/utils/filesystem.h" #include "validator/stats-merger.h" @@ -1671,6 +1672,67 @@ void ValidatorManagerImpl::send_download_archive_request(BlockSeqno mc_seqno, Sh callback_->download_archive(mc_seqno, shard_prefix, std::move(tmp_dir), timeout, std::move(promise)); } +void ValidatorManagerImpl::get_block_proof_link_from_import(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + auto it = to_import_all_.upper_bound(masterchain_block_id.seqno() + 1); + while (true) { + if (it == to_import_all_.begin()) { + promise.set_error(td::Status::Error("proof not found")); + return; + } + --it; + bool stop = false; + for (const std::string &path : it->second) { + td::BufferSlice result; + auto r_package = Package::open(path, false, false); + if (r_package.is_error()) { + LOG(WARNING) << "Cannot open package " << path << " : " << r_package.move_as_error(); + continue; + } + auto package = r_package.move_as_ok(); + package.iterate([&](std::string filename, td::BufferSlice data, td::uint64) -> bool { + auto F = FileReference::create(filename); + if (F.is_error()) { + return true; + } + auto f = F.move_as_ok(); + BlockIdExt id; + bool is_proof = false; + f.ref().visit(td::overloaded( + [&](const fileref::Block &p) { + id = p.block_id; + is_proof = false; + }, + [&](const fileref::Proof &p) { + id = p.block_id; + is_proof = true; + }, + [&](const fileref::ProofLink &p) { + id = p.block_id; + is_proof = true; + }, + [&](const auto &) {})); + if (is_proof && id == block_id) { + result = std::move(data); + return false; + } + if (shard_intersects(id.shard_full(), block_id.shard_full()) && id.seqno() < block_id.seqno()) { + stop = true; + } + return true; + }); + if (!result.empty()) { + promise.set_result(std::move(result)); + return; + } + } + if (block_id.is_masterchain() || stop) { + promise.set_error(td::Status::Error("proof not found")); + return; + } + } +} + void ValidatorManagerImpl::start_up() { db_ = create_db_actor(actor_id(this), db_root_, opts_); actor_stats_ = td::actor::create_actor("actor_stats"); @@ -1730,6 +1792,7 @@ void ValidatorManagerImpl::start_up() { ++to_import_files; } }); + to_import_all_ = to_import_; if (S.is_error()) { LOG(INFO) << "failed to load blocks from import dir: " << S; } diff --git a/validator/manager.hpp b/validator/manager.hpp index 850baf47be..0e26e014ff 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -521,6 +521,9 @@ class ValidatorManagerImpl : public ValidatorManager { void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) override; + void get_block_proof_link_from_import(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override; + void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; @@ -706,6 +709,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::actor::ActorOwn serializer_; std::map> to_import_; + std::map> to_import_all_; private: std::unique_ptr callback_; From 0eeb565ad35c4b5ce1029b3d41c88a175c419ab0 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 27 Mar 2025 13:09:29 +0300 Subject: [PATCH 177/524] Perf timer for archive import --- validator/import-db-slice-local.cpp | 8 ++++++-- validator/import-db-slice-local.hpp | 2 +- validator/import-db-slice.cpp | 6 +++++- validator/import-db-slice.hpp | 2 ++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/validator/import-db-slice-local.cpp b/validator/import-db-slice-local.cpp index a16a26c630..38edbb10eb 100644 --- a/validator/import-db-slice-local.cpp +++ b/validator/import-db-slice-local.cpp @@ -43,7 +43,10 @@ ArchiveImporterLocal::ArchiveImporterLocal(std::string db_root, td::Refget_seqno(), std::min(last_masterchain_state_->get_seqno(), shard_client_seqno_)}); } diff --git a/validator/import-db-slice.hpp b/validator/import-db-slice.hpp index 04f22642dc..4cf6ce40ac 100644 --- a/validator/import-db-slice.hpp +++ b/validator/import-db-slice.hpp @@ -91,6 +91,8 @@ class ArchiveImporter : public td::actor::Actor { bool imported_any_ = false; bool have_shard_blocks_ = false; std::vector files_to_cleanup_; + + td::PerfWarningTimer perf_timer_; }; } // namespace validator From 9e2664095df129c15069f31b77f3f5b45ead8524 Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Thu, 27 Mar 2025 08:21:21 -0400 Subject: [PATCH 178/524] Remove unused check for SSE4.2 (#1578) --- CMakeLists.txt | 50 -------------------------------------------------- 1 file changed, 50 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 59f7c5e082..dd193d8aaf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,56 +29,6 @@ if (TON_REAL_BINARY_DIR STREQUAL TON_REAL_SOURCE_DIR) message(FATAL_ERROR "In-source build failed.") endif() -# HAVE_SSE42 for crc32c and rocksdb -include(CheckCXXSourceCompiles) -# Check for SSE4.2 support in the compiler. -set(OLD_CMAKE_REQURED_FLAGS ${CMAKE_REQUIRED_FLAGS}) -if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} /arch:AVX") -else(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -msse4.2") -endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") -check_cxx_source_compiles(" -#if defined(_MSC_VER) -#include -#else // !defined(_MSC_VER) -#include -#include -#endif // defined(_MSC_VER) - -int main() { - _mm_crc32_u8(0, 0); _mm_crc32_u32(0, 0); -#if defined(_M_X64) || defined(__x86_64__) - _mm_crc32_u64(0, 0); -#endif // defined(_M_X64) || defined(__x86_64__) - return 0; -} -" CRC32C_HAVE_SSE42) -set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQURED_FLAGS}) - -if(NOT MSVC) - set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul") -endif() -CHECK_CXX_SOURCE_COMPILES(" -#include -#include -#include -int main() { - volatile uint32_t x = _mm_crc32_u32(0, 0); - const auto a = _mm_set_epi64x(0, 0); - const auto b = _mm_set_epi64x(0, 0); - const auto c = _mm_clmulepi64_si128(a, b, 0x00); - auto d = _mm_cvtsi128_si64(c); -} -" ROCKSDB_HAVE_SSE42) -unset(CMAKE_REQUIRED_FLAGS) - -if (ROCKSDB_HAVE_SSE42 AND CRC32C_HAVE_SSE42) - set(HAVE_SSE42 TRUE) -else() - set(HAVE_SSE42 FALSE) -endif() - set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED TRUE) set(CMAKE_CXX_EXTENSIONS FALSE) From 1166a40857e9c720f87380fc875e947a8d906863 Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Thu, 27 Mar 2025 08:22:33 -0400 Subject: [PATCH 179/524] Add a CMake option to disable git watcher (#1579) Whenever git watcher decides that the git state has changed, it spends multiple seconds trying to retrieve information about this changed state. This is incredibly annoying as it slows down edit-compile-run cycle for no reason. --- git_watcher.cmake | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/git_watcher.cmake b/git_watcher.cmake index 78e57ba149..5993916c40 100644 --- a/git_watcher.cmake +++ b/git_watcher.cmake @@ -315,4 +315,14 @@ function(Main) endfunction() # And off we go... -Main() +option(ENABLE_GIT_WATCHER "Use git tree information in the version strings" ON) + +if (ENABLE_GIT_WATCHER) + Main() +else() + add_custom_target(check_git) + set(GIT_RETRIEVED_STATE false) + set(GIT_IS_DIRTY false) + set(GIT_COMMIT_BODY "\"\"") + configure_file("${PRE_CONFIGURE_FILE}" "${POST_CONFIGURE_FILE}" @ONLY) +endif() From 070e27d3c6f32837c9b2b8add64e427f9287695e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 28 Mar 2025 10:54:13 +0300 Subject: [PATCH 180/524] Dictionary::multiset; optimize account storage stat --- crypto/block/account-storage-stat.cpp | 202 +++++++++++++------- crypto/block/account-storage-stat.h | 32 +++- crypto/common/bitstring.cpp | 3 + crypto/vm/dict.cpp | 254 ++++++++++++++++++++++++++ crypto/vm/dict.h | 4 +- 5 files changed, 421 insertions(+), 74 deletions(-) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index 8e8b6f7e16..918d2ac005 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -53,51 +53,95 @@ AccountStorageStat& AccountStorageStat::operator=(AccountStorageStat&& other) { return *this; } -td::Result AccountStorageStat::add_root(const Ref& cell) { - roots_.push_back(cell); - return add_cell(cell); -} - -td::Status AccountStorageStat::remove_root(const Ref& cell) { - auto it = std::find_if(roots_.begin(), roots_.end(), - [&](const Ref& c) { return c->get_hash() == cell->get_hash(); }); - if (it == roots_.end()) { - return td::Status::Error(PSTRING() << "no such root " << cell->get_hash().to_hex()); +td::Result AccountStorageStat::replace_roots(std::vector> new_roots) { + std::erase_if(new_roots, [](const Ref& c) { return c.is_null(); }); + if (new_roots.empty()) { + roots_.clear(); + total_bits_ = total_cells_ = 0; + dict_ = vm::Dictionary{256}; + cache_ = {}; + return CellInfo{}; } - roots_.erase(it); - return remove_cell(cell); -} -td::Result AccountStorageStat::replace_roots(std::vector> new_roots) { - std::vector> old_roots = roots_; + auto cmp = [](const Ref& c1, const Ref& c2) { return c1->get_hash() < c2->get_hash(); }; + std::sort(new_roots.begin(), new_roots.end(), cmp); + std::sort(roots_.begin(), roots_.end(), cmp); + std::vector> to_add, to_del; + std::set_difference(new_roots.begin(), new_roots.end(), roots_.begin(), roots_.end(), std::back_inserter(to_add), + cmp); + std::set_difference(roots_.begin(), roots_.end(), new_roots.begin(), new_roots.end(), std::back_inserter(to_del), + cmp); + td::uint32 max_merkle_depth = 0; - for (const Ref& root : new_roots) { - if (root.is_null()) { - continue; - } - TRY_RESULT(info, add_root(root)); + for (const Ref& root : to_add) { + TRY_RESULT(info, add_cell(root)); max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); } - for (const Ref& root : old_roots) { - TRY_STATUS(remove_root(root)); + for (const Ref& root : to_del) { + TRY_STATUS(remove_cell(root)); + } + + roots_ = std::move(new_roots); + td::Status S = td::Status::OK(); + std::vector>> values; + cache_.for_each([&](Entry& e) { + if (S.is_ok()) { + S = commit_entry(e, values); + } + }); + TRY_STATUS(std::move(S)); + if (!dict_.multiset(values)) { + return td::Status::Error("failed to update dictionary"); } return CellInfo{max_merkle_depth}; } +void AccountStorageStat::add_hint(const td::HashSet& hint) { + td::HashSet visited; + std::function&, bool)> dfs = [&](const Ref& cell, bool is_root) { + if (!visited.insert(cell->get_hash()).second) { + return; + } + Entry& e = get_entry(cell); + e.exists = e.exists_known = true; + if (is_root) { + fetch_entry(e).ignore(); + if (e.max_merkle_depth && e.max_merkle_depth.value() != 0) { + return; + } + } + if (hint.contains(cell->get_hash())) { + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + dfs(cs.prefetch_ref(i), false); + } + } + }; + for (const Ref& root : roots_) { + dfs(root, true); + } +} + td::Result AccountStorageStat::add_cell(const Ref& cell) { Entry& e = get_entry(cell); - ++e.refcnt; - if (e.refcnt == 0) { - return td::Status::Error(PSTRING() << "cell " << cell->get_hash().to_hex() << ": refcnt overflow"); + if (!e.exists_known || e.refcnt_diff < 0) { + TRY_STATUS(fetch_entry(e)); } - if (e.refcnt != 1) { - update_dict(e); - return CellInfo{e.max_merkle_depth}; + ++e.refcnt_diff; + if (e.exists || e.refcnt_diff > 1 || (e.refcnt && e.refcnt.value() + e.refcnt_diff != 1)) { + if (!e.max_merkle_depth) { + TRY_STATUS(fetch_entry(e)); + if (!e.max_merkle_depth) { + return td::Status::Error(PSTRING() << "unexpected unknown Merkle depth of cell " << cell->get_hash()); + } + } + return CellInfo{e.max_merkle_depth.value()}; } + td::uint32 max_merkle_depth = 0; - vm::CellSlice cs{vm::NoVm{}, cell}; - ++total_cells_; - total_bits_ += cs.size(); + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); for (unsigned i = 0; i < cs.size_refs(); ++i) { TRY_RESULT(info, add_cell(cs.prefetch_ref(i))); max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); @@ -109,64 +153,96 @@ td::Result AccountStorageStat::add_cell(const Ref< max_merkle_depth = std::min(max_merkle_depth, MERKLE_DEPTH_LIMIT); Entry& e2 = get_entry(cell); e2.max_merkle_depth = max_merkle_depth; - update_dict(e2); return CellInfo{max_merkle_depth}; } td::Status AccountStorageStat::remove_cell(const Ref& cell) { Entry& e = get_entry(cell); - if (e.refcnt == 0) { - return td::Status::Error(PSTRING() << "cell " << cell->get_hash().to_hex() << " is not in the dict"); + if (!e.exists_known) { + TRY_STATUS(fetch_entry(e)); } - --e.refcnt; - update_dict(e); - if (e.refcnt != 0) { - return td::Status::OK(); + if (!e.exists) { + return td::Status::Error(PSTRING() << "Failed to remove cell " << cell->get_hash().to_hex() + << " : does not exist in the dict"); } - vm::CellSlice cs{vm::NoVm{}, std::move(cell)}; - if (total_cells_ == 0 || total_bits_ < cs.size()) { - return td::Status::Error("total_cell/total_bits becomes negative"); + --e.refcnt_diff; + if (e.refcnt_diff < 0 && !e.refcnt) { + TRY_STATUS(fetch_entry(e)); + } + if (e.refcnt.value() + e.refcnt_diff != 0) { + return td::Status::OK(); } - --total_cells_; - total_bits_ -= cs.size(); + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); for (unsigned i = 0; i < cs.size_refs(); ++i) { TRY_STATUS(remove_cell(cs.prefetch_ref(i))); } return td::Status::OK(); } -bool AccountStorageStat::Entry::serialize(vm::CellBuilder& cb) const { - return cb.store_long_bool(refcnt, 32) && cb.store_long_bool(max_merkle_depth, 2); -} - -void AccountStorageStat::Entry::fetch(Ref cs) { - if (cs.is_null()) { - refcnt = max_merkle_depth = 0; - } else { - refcnt = (td::uint32)cs.write().fetch_ulong(32); - max_merkle_depth = (td::uint32)cs.write().fetch_ulong(2); - } -} - AccountStorageStat::Entry& AccountStorageStat::get_entry(const Ref& cell) { return cache_.apply(cell->get_hash().as_slice(), [&](Entry& e) { if (e.inited) { return; } e.inited = true; + e.cell = cell; e.hash = cell->get_hash(); - e.fetch(dict_.lookup(e.hash.as_bitslice())); }); } -void AccountStorageStat::update_dict(const Entry& e) { - if (e.refcnt == 0) { - dict_.lookup_delete(e.hash.as_bitslice()); +td::Status AccountStorageStat::fetch_entry(Entry& e) { + if (e.exists_known && e.refcnt && (!e.exists || e.max_merkle_depth)) { + return td::Status::OK(); + } + auto cs = dict_.lookup(e.hash.as_bitslice()); + if (cs.is_null()) { + e.exists = false; + e.refcnt = 0; + } else { + if (cs->size_ext() != 32 + 2) { + return td::Status::Error(PSTRING() << "invalid record for cell " << e.hash.to_hex()); + } + e.exists = true; + e.refcnt = (td::uint32)cs.write().fetch_ulong(32); + e.max_merkle_depth = (td::uint32)cs.write().fetch_ulong(2); + if (e.refcnt.value() == 0) { + return td::Status::Error(PSTRING() << "invalid refcnt=0 for cell " << e.hash.to_hex()); + } + } + e.exists_known = true; + return td::Status::OK(); +} + +td::Status AccountStorageStat::commit_entry(Entry& e, + std::vector>>& values) { + if (e.refcnt_diff == 0) { + return td::Status::OK(); + } + TRY_STATUS(fetch_entry(e)); + e.refcnt.value() += e.refcnt_diff; + e.refcnt_diff = 0; + bool spec; + if (e.refcnt.value() == 0) { + --total_cells_; + total_bits_ -= vm::load_cell_slice_special(e.cell, spec).size(); + e.exists = false; + values.emplace_back(e.hash.bits(), td::Ref{}); } else { - vm::CellBuilder cb; - CHECK(e.serialize(cb)); - dict_.set_builder(e.hash.as_bitslice(), cb); + if (!e.exists) { + ++total_cells_; + total_bits_ += vm::load_cell_slice_special(e.cell, spec).size(); + } + e.exists = true; + if (!e.max_merkle_depth) { + return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown merkle depth"); + } + td::Ref cbr{true}; + auto& cb = cbr.write(); + CHECK(cb.store_long_bool(e.refcnt.value(), 32) && cb.store_long_bool(e.max_merkle_depth.value(), 2)); + values.emplace_back(e.hash.bits(), std::move(cbr)); } + return td::Status::OK(); } } // namespace block \ No newline at end of file diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h index 894192d4f9..83eb3af5ae 100644 --- a/crypto/block/account-storage-stat.h +++ b/crypto/block/account-storage-stat.h @@ -58,9 +58,8 @@ class AccountStorageStat { return dict_.is_empty() ? td::Bits256::zero() : td::Bits256{dict_.get_root_cell()->get_hash().bits()}; } - td::Result add_root(const Ref &cell); - td::Status remove_root(const Ref &cell); - td::Result replace_roots(std::vector> new_roots); + td::Result replace_roots(std::vector> hint); + void add_hint(const td::HashSet &visited); private: vm::Dictionary dict_; @@ -72,12 +71,12 @@ class AccountStorageStat { struct Entry { bool inited = false; - vm::Cell::Hash hash; - td::uint32 refcnt = 0; - td::uint32 max_merkle_depth = 0; - - void fetch(Ref cs); - bool serialize(vm::CellBuilder &cb) const; + vm::CellHash hash; + Ref cell; + bool exists_known = false; + bool exists = false; + td::optional refcnt, max_merkle_depth; + td::int32 refcnt_diff = 0; vm::Cell::Hash key() const { return hash; @@ -111,9 +110,22 @@ class AccountStorageStat { vm::CellHashTable cache_; Entry &get_entry(const Ref &cell); - void update_dict(const Entry &e); + td::Status fetch_entry(Entry &e); + td::Status commit_entry(Entry &e, std::vector>> &values); static constexpr td::uint32 MERKLE_DEPTH_LIMIT = 3; }; +class StorageStatCalculationContext : public td::Context { + public: + explicit StorageStatCalculationContext(bool active) : active_(active) { + } + bool calculating_storage_stat() const { + return active_; + } + + private: + bool active_ = false; +}; + } // namespace block diff --git a/crypto/common/bitstring.cpp b/crypto/common/bitstring.cpp index 52e57c9a8a..3a6f331191 100644 --- a/crypto/common/bitstring.cpp +++ b/crypto/common/bitstring.cpp @@ -347,6 +347,9 @@ std::size_t bits_memscan_rev(ConstBitPtr bs, std::size_t bit_count, bool cmp_to) int bits_memcmp(const unsigned char* bs1, int bs1_offs, const unsigned char* bs2, int bs2_offs, std::size_t bit_count, std::size_t* same_upto) { if (!bit_count) { + if (same_upto) { + *same_upto = 0; + } return 0; } bs1 += (bs1_offs >> 3); diff --git a/crypto/vm/dict.cpp b/crypto/vm/dict.cpp index 41f9c3396f..e1e683fe36 100644 --- a/crypto/vm/dict.cpp +++ b/crypto/vm/dict.cpp @@ -1772,6 +1772,260 @@ void Dictionary::map(const simple_map_func_t& simple_map_func) { map(map_func); } +bool Dictionary::multiset(td::MutableSpan>> new_values) { + force_validate(); + auto cmp = [&](const std::pair>& a, + const std::pair>& b) { + return td::bitstring::bits_memcmp(a.first, b.first, key_bits) < 0; + }; + if (!std::is_sorted(new_values.begin(), new_values.end(), cmp)) { + std::sort(new_values.begin(), new_values.end(), cmp); + } + for (size_t i = 0; i + 1 < new_values.size(); ++i) { + if (td::bitstring::bits_memcmp(new_values[i].first, new_values[i + 1].first, key_bits) == 0) { + return false; + } + } + unsigned char key_buffer[max_key_bytes]; + try { + Ref root = dict_multiset(get_root_cell(), new_values, key_buffer, key_bits, key_bits, 0); + set_root_cell(std::move(root)); + return true; + } catch (CombineError) { + return false; + } +} + +static Ref dict_build(td::Span>> values, int total_key_len, + int prefix_len) { + if (values.empty()) { + return {}; + } + if (values.size() == 1) { + if (values[0].second.is_null()) { + throw CombineError{}; + } + CellBuilder cb; + append_dict_label(cb, values[0].first + prefix_len, total_key_len - prefix_len, total_key_len - prefix_len); + if (!cb.append_builder_bool(values[0].second)) { + throw VmError{Excno::cell_ov, "cannot store new value into a dictionary cell"}; + } + return cb.finalize(); + } + size_t common_prefix_len; + td::bitstring::bits_memcmp(values.front().first + prefix_len, values.back().first + prefix_len, + total_key_len - prefix_len, &common_prefix_len); + CHECK(prefix_len + common_prefix_len < total_key_len); + size_t idx = 0; + while (values[idx].first[prefix_len + common_prefix_len] == 0) { + ++idx; + } + Ref left = dict_build(values.substr(0, idx), total_key_len, prefix_len + common_prefix_len + 1); + Ref right = dict_build(values.substr(idx), total_key_len, prefix_len + common_prefix_len + 1); + CellBuilder cb; + append_dict_label(cb, values[0].first + prefix_len, common_prefix_len, total_key_len - prefix_len); + if (!(cb.store_ref_bool(std::move(left)) && cb.store_ref_bool(std::move(right)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); +} + +// Based on DictionaryFixed::dict_combine_with, but dict2 is replaced with a list of values, mode is false +Ref Dictionary::dict_multiset(Ref dict1, td::Span>> values2, + td::BitPtr key_buffer, int n, int total_key_len, int skip1) { + int prefix_len = total_key_len - n; + for (auto& [k, _] : values2) { + CHECK(td::bitstring::bits_memcmp(k, key_buffer - prefix_len, prefix_len) == 0); + } + if (dict1.is_null()) { + return dict_build(values2, total_key_len, prefix_len); + } + if (values2.empty()) { + assert(!skip1); + return dict1; + } + size_t common_prefix_len; + td::bitstring::bits_memcmp(values2.front().first + prefix_len, values2.back().first + prefix_len, + total_key_len - prefix_len, &common_prefix_len); + assert(prefix_len + common_prefix_len < total_key_len || values2.size() == 1); + // both dictionaries non-empty + // skip1: remove that much first bits from all keys in dictionary dict1 (its keys are actually n + skip1 bits long) + // resulting dictionary will have n-bit keys + LabelParser label1{dict1, n + skip1, LabelParser::chk_all}; + int l1 = label1.l_bits - skip1, l2 = (int)common_prefix_len; + assert(l1 >= 0 && l2 >= 0); + assert(!skip1 || label1.common_prefix_len(key_buffer - skip1, skip1) == skip1); + int c = label1.common_prefix_len(values2[0].first + prefix_len - skip1, skip1 + l2) - skip1; + label1.extract_label_to(key_buffer - skip1); + assert(c >= 0 && c <= l1 && c <= l2); + if (c < l1 && c < l2) { + // the two dictionaries have disjoint keys + CellBuilder cb; + append_dict_label(cb, key_buffer + c + 1, l1 - c - 1, n - c - 1); + if (!cell_builder_add_slice_bool(cb, *label1.remainder)) { + throw VmError{Excno::cell_ov, "cannot prune label of an old dictionary cell while merging dictionaries"}; + } + label1.remainder.clear(); + dict1 = cb.finalize(); + // cb.reset(); // included into finalize(); + // now dict1 has been "pruned" -- first skip1+c+1 bits removed from its root egde label + Ref dict2 = dict_build(values2, total_key_len, prefix_len + c + 1); + if (!values2[0].first[prefix_len + c]) { + std::swap(dict1, dict2); + } + // put dict1 into the left tree (with smaller labels), dict2 into the right tree + append_dict_label(cb, key_buffer, c, n); + if (!(cb.store_ref_bool(std::move(dict1)) && cb.store_ref_bool(std::move(dict2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + + auto combine_func = [&](CellBuilder& cb, const Ref& cb2) -> bool { + if (cb2.is_null()) { + return false; + } + cb.append_builder(*cb2); + return true; + }; + size_t idx = 0; + while (prefix_len + common_prefix_len < total_key_len && idx < values2.size() && + values2[idx].first[prefix_len + common_prefix_len] == 0) { + ++idx; + } + auto values2_left = values2.substr(0, idx); + auto values2_right = values2.substr(idx); + + if (c == l1 && c == l2) { + // funny enough, the non-skipped parts of labels of l1 and l2 match + CellBuilder cb; + append_dict_label(cb, key_buffer, c, n); + if (c == n) { + // our two dictionaries are in fact leafs with matching edge labels (keys) + if (!combine_func(cb, values2[0].second)) { + // alas, the two values did not combine, this key will be absent from resulting dictionary + return {}; + } + return cb.finalize(); + } + assert(c < n); + key_buffer += c + 1; + key_buffer[-1] = 0; + // combine left subtrees + auto c1 = dict_multiset(label1.remainder->prefetch_ref(0), values2_left, key_buffer, n - c - 1, total_key_len, 0); + key_buffer[-1] = 1; + // combine right subtrees + auto c2 = dict_multiset(label1.remainder->prefetch_ref(1), values2_right, key_buffer, n - c - 1, total_key_len, 0); + label1.remainder.clear(); + // c1 and c2 are merged left and right children of dict1 and dict2 + if (!c1.is_null() && !c2.is_null()) { + // both children non-empty, simply put them into the new node + if (!(cb.store_ref_bool(std::move(c1)) && cb.store_ref_bool(std::move(c2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + if (c1.is_null() && c2.is_null()) { + return {}; // both children empty, resulting dictionary also empty + } + // exactly one of c1 and c2 is non-empty, have to merge labels + bool sw = c1.is_null(); + key_buffer[-1] = sw; + if (sw) { + c1 = std::move(c2); + } + LabelParser label3{std::move(c1), n - c - 1, LabelParser::chk_all}; + label3.extract_label_to(key_buffer); + key_buffer -= c + 1; + // store combined label for the new edge + cb.reset(); + append_dict_label(cb, key_buffer, c + 1 + label3.l_bits, n); + // store payload + if (!cell_builder_add_slice_bool(cb, *label3.remainder)) { + throw VmError{Excno::cell_ov, "cannot change label of an old dictionary cell while merging edges"}; + } + return cb.finalize(); + } + if (c == l1) { + assert(c < l2); + dict1.clear(); + // children of root node of dict1 + auto c1 = label1.remainder->prefetch_ref(0); + auto c2 = label1.remainder->prefetch_ref(1); + label1.remainder.clear(); + // have to merge dict2 with one of the children of dict1 + td::bitstring::bits_memcpy(key_buffer, values2[0].first + prefix_len, l2); // dict2 has longer label, extract it + bool sw = key_buffer[c]; + if (!sw) { + // merge c1 with dict2 + c1 = dict_multiset(std::move(c1), values2, key_buffer + c + 1, n - c - 1, total_key_len, 0); + } else { + // merge c2 with dict2 + c2 = dict_multiset(std::move(c2), values2, key_buffer + c + 1, n - c - 1, total_key_len, 0); + } + if (!c1.is_null() && !c2.is_null()) { + CellBuilder cb; + append_dict_label(cb, key_buffer, c, n); + if (!(cb.store_ref_bool(std::move(c1)) && cb.store_ref_bool(std::move(c2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + // one of children is empty, have to merge root edges + key_buffer[c] = !sw; + if (!sw) { + std::swap(c1, c2); + } + assert(!c1.is_null() && c2.is_null()); + LabelParser label3{std::move(c1), n - c - 1, LabelParser::chk_all}; + label3.extract_label_to(key_buffer + c + 1); + CellBuilder cb; + append_dict_label(cb, key_buffer, c + 1 + label3.l_bits, n); + // store payload + if (!cell_builder_add_slice_bool(cb, *label3.remainder)) { + throw VmError{Excno::cell_ov, "cannot change label of an old dictionary cell while merging edges"}; + } + return cb.finalize(); + } else { + assert(c == l2 && c < l1); + // have to merge dict1 with one of the children of dict2 + bool sw = key_buffer[c]; + Ref c1, c2; + if (!sw) { + // merge dict1 with c1 + c1 = dict_multiset(std::move(dict1), values2_left, key_buffer + c + 1, n - c - 1, total_key_len, skip1 + c + 1); + c2 = dict_build(values2_right, total_key_len, prefix_len + l2 + 1); + } else { + // merge dict1 with c2 + c2 = dict_multiset(std::move(dict1), values2_right, key_buffer + c + 1, n - c - 1, total_key_len, skip1 + c + 1); + c1 = dict_build(values2_left, total_key_len, prefix_len + l2 + 1); + } + if (!c1.is_null() && !c2.is_null()) { + CellBuilder cb; + append_dict_label(cb, key_buffer, c, n); + if (!(cb.store_ref_bool(std::move(c1)) && cb.store_ref_bool(std::move(c2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + // one of children is empty, have to merge root edges + key_buffer[c] = !sw; + if (!sw) { + std::swap(c1, c2); + } + assert(!c1.is_null() && c2.is_null()); + LabelParser label3{std::move(c1), n - c - 1, LabelParser::chk_all}; + label3.extract_label_to(key_buffer + c + 1); + CellBuilder cb; + append_dict_label(cb, key_buffer, c + 1 + label3.l_bits, n); + // store payload + if (!cell_builder_add_slice_bool(cb, *label3.remainder)) { + throw VmError{Excno::cell_ov, "cannot change label of an old dictionary cell while merging edges"}; + } + return cb.finalize(); + } +} + // mode: +1 = forbid empty dict1 with non-empty dict2 // +2 = forbid empty dict2 with non-empty dict1 Ref DictionaryFixed::dict_combine_with(Ref dict1, Ref dict2, td::BitPtr key_buffer, int n, diff --git a/crypto/vm/dict.h b/crypto/vm/dict.h index c4044963f5..e22b909465 100644 --- a/crypto/vm/dict.h +++ b/crypto/vm/dict.h @@ -527,13 +527,15 @@ class Dictionary final : public DictionaryFixed { auto range(bool rev = false, bool sgnd = false) { return dict_range(*this, rev, sgnd); } + bool multiset(td::MutableSpan>> new_values); private: bool check_fork(CellSlice& cs, Ref c1, Ref c2, int n) const override { return cs.empty_ext(); } static Ref extract_value_ref(Ref cs); - std::pair, int> dict_filter(Ref dict, td::BitPtr key, int n, const filter_func_t& check_leaf) const; + static Ref dict_multiset(Ref dict1, td::Span>> values2, + td::BitPtr key_buffer, int n, int total_key_len, int skip1); }; class PrefixDictionary final : public DictionaryBase { From ad3c05938dd7b5a1b77444e545ef039853bd3377 Mon Sep 17 00:00:00 2001 From: leopardracer <136604165+leopardracer@users.noreply.github.com> Date: Fri, 28 Mar 2025 11:19:45 +0200 Subject: [PATCH 181/524] fix: typos in comments (#1580) * Update collator.cpp * Update Stack.fif * Update parse-func.cpp --- crypto/fift/lib/Stack.fif | 2 +- crypto/func/parse-func.cpp | 2 +- validator/impl/collator.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crypto/fift/lib/Stack.fif b/crypto/fift/lib/Stack.fif index c7ef6f8e58..6efc0a060f 100644 --- a/crypto/fift/lib/Stack.fif +++ b/crypto/fift/lib/Stack.fif @@ -1,4 +1,4 @@ -library Stack // advanced stack manupulation library +library Stack // advanced stack manipulation library "Lists.fif" include // S(a b c - a c 2 a b) would compile to code performing the requested stack manipulation diff --git a/crypto/func/parse-func.cpp b/crypto/func/parse-func.cpp index 4fc1cece52..179cc9bb41 100644 --- a/crypto/func/parse-func.cpp +++ b/crypto/func/parse-func.cpp @@ -288,7 +288,7 @@ void parse_const_decl(Lexer& lex) { code.emplace_back(loc, Op::_Import, std::vector()); auto tmp_vars = x->pre_compile(code); code.emplace_back(loc, Op::_Return, std::move(tmp_vars)); - code.emplace_back(loc, Op::_Nop); // This is neccessary to prevent SIGSEGV! + code.emplace_back(loc, Op::_Nop); // This is necessary to prevent SIGSEGV! // It is REQUIRED to execute "optimizations" as in func.cpp code.simplify_var_types(); code.prune_unreachable_code(); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 2a6d7a2b2a..805240b3eb 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -5723,7 +5723,7 @@ void Collator::return_block_candidate(td::Result saved) { * @returns Result indicating the success or failure of the registration. * - If the external message is invalid, returns an error. * - If the external message has been previously rejected, returns an error - * - If the external message has been previuosly registered and accepted, returns false. + * - If the external message has been previously registered and accepted, returns false. * - Otherwise returns true. */ td::Result Collator::register_external_message_cell(Ref ext_msg, const ExtMessage::Hash& ext_hash, From 57772a9bfce29aa857b2d7393441f13b68049984 Mon Sep 17 00:00:00 2001 From: Stanislav-Povolotsky Date: Fri, 28 Mar 2025 10:30:39 +0100 Subject: [PATCH 182/524] Fix TonlibClient.cpp (#1534) --- tonlib/tonlib/TonlibClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index ae3189ba8b..b137ee7699 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -268,7 +268,7 @@ td::Result> add_extra_currencies(const td::Ref &e1, block::CurrencyCollection c1{td::zero_refint(), e1}; block::CurrencyCollection c2{td::zero_refint(), e2}; TRY_RESULT_ASSIGN(c1, TRY_VM(td::Result{c1 + c2})); - if (c1.is_valid()) { + if (!c1.is_valid()) { return td::Status::Error("Failed to add extra currencies"); } return c1.extra; From 353c621d015306081762b5f7acec44871a1f43bf Mon Sep 17 00:00:00 2001 From: neodix42 Date: Sat, 29 Mar 2025 17:53:54 +0400 Subject: [PATCH 183/524] Improve github actions utilizing github cache and ccache (#1582) * improve github actions by utilizing github cache and ccache. * download libmicrohttpd from ftp * minor fix * no cache * no cache 2 * no cache test 3 * with cache test 1 * with cache test 2 * test without gh cache * test with gh cache --- .../build-ton-linux-android-tonlib.yml | 8 ++ .../build-ton-linux-arm64-appimage.yml | 49 +++++++++- .../build-ton-linux-arm64-shared.yml | 38 ++++++-- .../build-ton-linux-x86-64-appimage.yml | 48 +++++++++- .../build-ton-linux-x86-64-shared.yml | 38 ++++++-- .../build-ton-macos-13-x86-64-portable.yml | 34 ++++++- .../build-ton-macos-14-arm64-portable.yml | 34 ++++++- .../build-ton-macos-15-arm64-shared.yml | 30 ++++-- .../build-ton-macos-arm64-shared.yml | 30 ++++-- .../build-ton-macos-x86-64-shared.yml | 30 ++++-- .../workflows/build-ton-wasm-emscripten.yml | 22 ++++- .github/workflows/ton-x86-64-windows.yml | 17 ++++ assembly/native/build-macos-portable.sh | 89 ++++++++++-------- assembly/native/build-macos-shared.sh | 25 +++-- assembly/native/build-ubuntu-appimages.sh | 28 ++++-- assembly/native/build-ubuntu-portable-libs.sh | 91 +++++++++++-------- assembly/native/build-ubuntu-portable.sh | 83 +++++++++-------- assembly/native/build-ubuntu-shared.sh | 37 ++++---- assembly/native/build-windows-2019.bat | 21 ++--- assembly/native/build-windows.bat | 21 ++--- assembly/wasm/fift-func-wasm-build-ubuntu.sh | 83 ++++++++--------- 21 files changed, 596 insertions(+), 260 deletions(-) diff --git a/.github/workflows/build-ton-linux-android-tonlib.yml b/.github/workflows/build-ton-linux-android-tonlib.yml index b1ef528182..5cd74e3639 100644 --- a/.github/workflows/build-ton-linux-android-tonlib.yml +++ b/.github/workflows/build-ton-linux-android-tonlib.yml @@ -18,6 +18,14 @@ jobs: sudo apt-get install -y build-essential git cmake ninja-build automake libtool texinfo autoconf libgflags-dev \ zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev \ libtool autoconf libsodium-dev libsecp256k1-dev liblz4-dev + + + - name: Cache Android NDK + id: cache-android-ndk + uses: actions/cache@v4 + with: + path: android-ndk-r25b + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-cache-android-ndk-${{ hashFiles('**/assembly/android/build-android-tonlib.sh') }} - name: Build TON run: | diff --git a/.github/workflows/build-ton-linux-arm64-appimage.yml b/.github/workflows/build-ton-linux-arm64-appimage.yml index d464d8a2ae..289cd2a7e2 100644 --- a/.github/workflows/build-ton-linux-arm64-appimage.yml +++ b/.github/workflows/build-ton-linux-arm64-appimage.yml @@ -12,17 +12,45 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + - name: Install system libraries run: | sudo apt update - sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev + sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache sudo apt remove libgsl-dev + mkdir ~/.ccache 3pp - name: Install clang-16 run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 16 all + sudo ./llvm.sh 16 clang + + - name: Cache 3pp + id: cache-3pp + uses: actions/cache@v4 + with: + path: 3pp + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-3pp-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + + - name: Cache OpenSSL + id: cache-openssl + uses: actions/cache@v4 + with: + path: openssl_3 + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + + - name: Restore cache TON + uses: actions/cache/restore@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-ccache - name: Build TON run: | @@ -30,7 +58,14 @@ jobs: git submodule update cp assembly/native/build-ubuntu-appimages.sh . chmod +x build-ubuntu-appimages.sh - ./build-ubuntu-appimages.sh -a + ./build-ubuntu-appimages.sh -a -c + ccache -sp + + - name: Save cache TON + uses: actions/cache/save@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} - name: Make AppImages run: | @@ -41,12 +76,18 @@ jobs: ./create-appimages.sh aarch64 rm -rf artifacts + - name: Save/Restore cache TON libs + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-libs-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-libs-ccache - name: Build TON libs run: | cp assembly/native/build-ubuntu-portable-libs.sh . chmod +x build-ubuntu-portable-libs.sh - ./build-ubuntu-portable-libs.sh -a + ./build-ubuntu-portable-libs.sh -a -c cp ./artifacts/libtonlibjson.so appimages/artifacts/ cp ./artifacts/libemulator.so appimages/artifacts/ diff --git a/.github/workflows/build-ton-linux-arm64-shared.yml b/.github/workflows/build-ton-linux-arm64-shared.yml index 6433df0b56..540fffece6 100644 --- a/.github/workflows/build-ton-linux-arm64-shared.yml +++ b/.github/workflows/build-ton-linux-arm64-shared.yml @@ -16,17 +16,39 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + - name: Install system libraries run: | sudo apt-get update - sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev + sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev ccache + mkdir ~/.ccache - if: matrix.os != 'ubuntu-24.04-arm' name: Install llvm-16 run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 16 all + sudo ./llvm.sh 16 clang + + - name: Cache OpenSSL + id: cache-openssl + uses: actions/cache@v4 + with: + path: openssl_3 + key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-shared.sh') }} + + - name: Cache TON test + id: cache-ton + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-shared-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-shared-ccache - name: Build TON run: | @@ -34,10 +56,10 @@ jobs: git submodule update cp assembly/native/build-ubuntu-shared.sh . chmod +x build-ubuntu-shared.sh - ./build-ubuntu-shared.sh -t -a + ./build-ubuntu-shared.sh -t -c + ccache -sp - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-binaries-${{ matrix.os }} - path: artifacts + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 diff --git a/.github/workflows/build-ton-linux-x86-64-appimage.yml b/.github/workflows/build-ton-linux-x86-64-appimage.yml index 4f78ece9da..0f5645a3ad 100644 --- a/.github/workflows/build-ton-linux-x86-64-appimage.yml +++ b/.github/workflows/build-ton-linux-x86-64-appimage.yml @@ -12,11 +12,18 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + - name: Install system libraries run: | sudo apt update - sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev + sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache sudo apt remove libgsl-dev + mkdir ~/.ccache 3pp - name: Install gcc-11 g++-11 run: | @@ -28,7 +35,27 @@ jobs: run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 16 all + sudo ./llvm.sh 16 clang + + - name: Cache 3pp + id: cache-3pp + uses: actions/cache@v4 + with: + path: 3pp + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-3pp-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + + - name: Cache OpenSSL + id: cache-openssl + uses: actions/cache@v4 + with: + path: openssl_3 + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + + - name: Restore cache TON + uses: actions/cache/restore@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-portable-ccache - name: Build TON run: | @@ -36,7 +63,14 @@ jobs: git submodule update cp assembly/native/build-ubuntu-appimages.sh . chmod +x build-ubuntu-appimages.sh - ./build-ubuntu-appimages.sh -a + ./build-ubuntu-appimages.sh -a -c + ccache -sp + + - name: Save cache TON + uses: actions/cache/save@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} - name: Make AppImages run: | @@ -47,12 +81,18 @@ jobs: ./create-appimages.sh x86_64 rm -rf artifacts + - name: Save/Restore cache TON libs + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-portable-libs-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-portable-libs-ccache - name: Build TON libs run: | cp assembly/native/build-ubuntu-portable-libs.sh . chmod +x build-ubuntu-portable-libs.sh - ./build-ubuntu-portable-libs.sh -a + ./build-ubuntu-portable-libs.sh -a -c cp ./artifacts/libtonlibjson.so appimages/artifacts/ cp ./artifacts/libemulator.so appimages/artifacts/ diff --git a/.github/workflows/build-ton-linux-x86-64-shared.yml b/.github/workflows/build-ton-linux-x86-64-shared.yml index bf78f7df15..37eddf3695 100644 --- a/.github/workflows/build-ton-linux-x86-64-shared.yml +++ b/.github/workflows/build-ton-linux-x86-64-shared.yml @@ -16,12 +16,20 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + - name: Install system libraries run: | sudo apt-get update - sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev + sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev ccache + mkdir ~/.ccache - if: matrix.os == 'ubuntu-20.04' + name: Install gcc-11 run: | sudo apt install -y manpages-dev software-properties-common sudo add-apt-repository ppa:ubuntu-toolchain-r/test @@ -31,7 +39,21 @@ jobs: run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 16 all + sudo ./llvm.sh 16 clang + + - name: Cache OpenSSL + id: cache-openssl + uses: actions/cache@v4 + with: + path: openssl_3 + key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-shared.sh') }} + + - name: Restore cache TON + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-shared-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-shared-ccache - name: Build TON run: | @@ -39,10 +61,10 @@ jobs: git submodule update cp assembly/native/build-ubuntu-shared.sh . chmod +x build-ubuntu-shared.sh - ./build-ubuntu-shared.sh -t -a + ./build-ubuntu-shared.sh -t -c + ccache -sp - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-binaries-${{ matrix.os }} - path: artifacts + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 diff --git a/.github/workflows/build-ton-macos-13-x86-64-portable.yml b/.github/workflows/build-ton-macos-13-x86-64-portable.yml index 5e50a46809..6a6a6a255f 100644 --- a/.github/workflows/build-ton-macos-13-x86-64-portable.yml +++ b/.github/workflows/build-ton-macos-13-x86-64-portable.yml @@ -12,13 +12,45 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + + - name: Create directories + run: | + mkdir -p ~/.ccache + mkdir -p 3pp + + - name: Cache 3pp + id: cache-3pp + uses: actions/cache@v4 + with: + path: 3pp + key: ${{ runner.os }}-${{ runner.arch }}-13-3pp-${{ hashFiles('**/assembly/native/build-macos-portable.sh') }} + + - name: Cache TON test + id: cache-ton + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-13-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-13-portable-ccache + - name: Build TON run: | git submodule sync --recursive git submodule update cp assembly/native/build-macos-portable.sh . chmod +x build-macos-portable.sh - ./build-macos-portable.sh -t -a + ./build-macos-portable.sh -t -a -c -o 13.0 + ccache -sp + + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 - name: Upload artifacts uses: actions/upload-artifact@master diff --git a/.github/workflows/build-ton-macos-14-arm64-portable.yml b/.github/workflows/build-ton-macos-14-arm64-portable.yml index 8eb3af70a2..995ff1c404 100644 --- a/.github/workflows/build-ton-macos-14-arm64-portable.yml +++ b/.github/workflows/build-ton-macos-14-arm64-portable.yml @@ -12,13 +12,45 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + + - name: Create directories + run: | + mkdir -p ~/.ccache + mkdir -p 3pp + + - name: Cache 3pp + id: cache-3pp + uses: actions/cache@v4 + with: + path: 3pp + key: ${{ runner.os }}-${{ runner.arch }}-14-3pp-${{ hashFiles('**/assembly/native/build-macos-portable.sh') }} + + - name: Cache TON + id: cache-ton + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-14-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-14-portable-ccache + - name: Build TON run: | git submodule sync --recursive git submodule update cp assembly/native/build-macos-portable.sh . chmod +x build-macos-portable.sh - ./build-macos-portable.sh -t -a + ./build-macos-portable.sh -t -a -c -o 14.0 + ccache -sp + + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 - name: Upload artifacts uses: actions/upload-artifact@master diff --git a/.github/workflows/build-ton-macos-15-arm64-shared.yml b/.github/workflows/build-ton-macos-15-arm64-shared.yml index 00d8f6392c..1423b4eb04 100644 --- a/.github/workflows/build-ton-macos-15-arm64-shared.yml +++ b/.github/workflows/build-ton-macos-15-arm64-shared.yml @@ -12,16 +12,34 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + + - name: Create ~/.ccache + run: | + mkdir -p ~/.ccache + + - name: Cache TON + id: cache-ton + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-15-shared-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-15-shared-ccache + - name: Build TON run: | git submodule sync --recursive git submodule update cp assembly/native/build-macos-shared.sh . chmod +x build-macos-shared.sh - ./build-macos-shared.sh -t -a + ./build-macos-shared.sh -t -c -o 15.0 + ccache -sp - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-binaries-macos-15 - path: artifacts + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 diff --git a/.github/workflows/build-ton-macos-arm64-shared.yml b/.github/workflows/build-ton-macos-arm64-shared.yml index 7481f9ff62..5ef9266af8 100644 --- a/.github/workflows/build-ton-macos-arm64-shared.yml +++ b/.github/workflows/build-ton-macos-arm64-shared.yml @@ -12,16 +12,34 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + + - name: Create ~/.ccache + run: | + mkdir -p ~/.ccache + + - name: Cache TON + id: cache-ton + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-14-shared-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-14-shared-ccache + - name: Build TON run: | git submodule sync --recursive git submodule update cp assembly/native/build-macos-shared.sh . chmod +x build-macos-shared.sh - ./build-macos-shared.sh -t -a + ./build-macos-shared.sh -t -c -o 14.0 + ccache -sp - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-binaries-macos-14 - path: artifacts + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 diff --git a/.github/workflows/build-ton-macos-x86-64-shared.yml b/.github/workflows/build-ton-macos-x86-64-shared.yml index 6a69b2e351..6200af6633 100644 --- a/.github/workflows/build-ton-macos-x86-64-shared.yml +++ b/.github/workflows/build-ton-macos-x86-64-shared.yml @@ -12,16 +12,34 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + + - name: Create ~/.ccache + run: | + mkdir -p ~/.ccache + + - name: Cache TON test + id: cache-ton + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-13-shared-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-13-shared-ccache + - name: Build TON run: | git submodule sync --recursive git submodule update cp assembly/native/build-macos-shared.sh . chmod +x build-macos-shared.sh - ./build-macos-shared.sh -t -a + ./build-macos-shared.sh -t -c + ccache -sp - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-binaries-macos-13 - path: artifacts + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 diff --git a/.github/workflows/build-ton-wasm-emscripten.yml b/.github/workflows/build-ton-wasm-emscripten.yml index bac0cf984e..f9f8611485 100644 --- a/.github/workflows/build-ton-wasm-emscripten.yml +++ b/.github/workflows/build-ton-wasm-emscripten.yml @@ -16,6 +16,27 @@ jobs: run: | sudo apt-get update sudo apt-get install -y build-essential git openssl cmake ninja-build zlib1g-dev libssl-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev + mkdir -p 3pp_emscripten + + - name: Install clang-16 + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 16 clang + + - name: Cache OpenSSL + id: cache-openssl + uses: actions/cache@v4 + with: + path: openssl_3 + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-openssl_3-${{ hashFiles('**/assembly/wasm/fift-func-wasm-build-ubuntu.sh') }} + + - name: Cache 3pp Emscripten + id: cache-3pp-emscripten + uses: actions/cache@v4 + with: + path: 3pp_emscripten + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-3pp_em-${{ hashFiles('**/assembly/wasm/fift-func-wasm-build-ubuntu.sh') }} - name: Build TON WASM artifacts run: | @@ -43,7 +64,6 @@ jobs: cp artifacts/funcfiftlib.js func-js/node_modules/@ton-community/func-js-bin/dist/funcfiftlib.js npx func-js stdlib.fc intrinsics.fc --fift ./output.f - - name: Upload artifacts uses: actions/upload-artifact@master with: diff --git a/.github/workflows/ton-x86-64-windows.yml b/.github/workflows/ton-x86-64-windows.yml index baaad778bb..b930d25a96 100644 --- a/.github/workflows/ton-x86-64-windows.yml +++ b/.github/workflows/ton-x86-64-windows.yml @@ -21,6 +21,17 @@ jobs: with: submodules: 'recursive' + - name: Create directories + run: | + mkdir third_libs + + - name: Cache 3pp + id: cache-3pp + uses: actions/cache@v4 + with: + path: third_libs + key: ${{ runner.os }}-${{ runner.arch }}-windows-2019-3pp-${{ hashFiles('**/assembly/native/build-windows-2019.bat') }} + - name: Build TON run: | git submodule sync --recursive @@ -28,6 +39,12 @@ jobs: copy assembly\native\build-windows-github-2019.bat . copy assembly\native\build-windows-2019.bat . build-windows-github-2019.bat Enterprise + ccache -sp + +# - name: Run Tests +# run: | +# cd build +# ctest -C Release --output-on-failure -E "test-bigint" --timeout 1800 - name: Upload artifacts uses: actions/upload-artifact@master diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index b785339e8d..81ac507b76 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -2,14 +2,16 @@ with_tests=false with_artifacts=false -OSX_TARGET=10.15 +with_ccache=false +OSX_TARGET=11.0 -while getopts 'tao:' flag; do +while getopts 'taco:' flag; do case "${flag}" in t) with_tests=true ;; a) with_artifacts=true ;; o) OSX_TARGET=${OPTARG} ;; + c) with_ccache=true ;; *) break ;; esac @@ -25,8 +27,19 @@ fi export NONINTERACTIVE=1 brew install ninja pkg-config automake libtool autoconf texinfo +export PATH=/usr/local/opt/ccache/libexec:$PATH brew install llvm@16 +if [ "$with_ccache" = true ]; then + brew install ccache + mkdir -p ~/.ccache + export CCACHE_DIR=~/.ccache + ccache -M 0 + test $? -eq 0 || { echo "ccache not installed"; exit 1; } +else + export CCACHE_DISABLE=1 +fi + if [ -f /opt/homebrew/opt/llvm@16/bin/clang ]; then export CC=/opt/homebrew/opt/llvm@16/bin/clang @@ -35,77 +48,79 @@ else export CC=/usr/local/opt/llvm@16/bin/clang export CXX=/usr/local/opt/llvm@16/bin/clang++ fi -export CCACHE_DISABLE=1 -if [ ! -d "lz4" ]; then -git clone https://github.com/lz4/lz4.git -cd lz4 +if [ ! -d "../3pp/lz4" ]; then +mkdir -p ../3pp +git clone https://github.com/lz4/lz4.git ../3pp/lz4 +cd ../3pp/lz4 lz4Path=`pwd` git checkout v1.9.4 -make -j12 +make -j$(nproc) test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } -cd .. +cd ../../build # ./lib/liblz4.a # ./lib else - lz4Path=$(pwd)/lz4 + lz4Path=$(pwd)/../3pp/lz4 echo "Using compiled lz4" fi -if [ ! -d "libsodium" ]; then +if [ ! -d "../3pp/libsodium" ]; then export LIBSODIUM_FULL_BUILD=1 - git clone https://github.com/jedisct1/libsodium.git - cd libsodium + git clone https://github.com/jedisct1/libsodium.git ../3pp/libsodium + cd ../3pp/libsodium sodiumPath=`pwd` git checkout 1.0.18 ./autogen.sh ./configure --with-pic --enable-static - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } - cd .. + cd ../../build else - sodiumPath=$(pwd)/libsodium + sodiumPath=$(pwd)/../3pp/libsodium echo "Using compiled libsodium" fi -if [ ! -d "openssl_3" ]; then - git clone https://github.com/openssl/openssl openssl_3 - cd openssl_3 +if [ ! -d "../3pp/openssl_3" ]; then + git clone https://github.com/openssl/openssl ../3pp/openssl_3 + cd ../3pp/openssl_3 opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make build_libs -j12 + make build_libs -j$(nproc) test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd .. + cd ../../build else - opensslPath=$(pwd)/openssl_3 + opensslPath=$(pwd)/../3pp/openssl_3 echo "Using compiled openssl_3" fi -if [ ! -d "zlib" ]; then - git clone https://github.com/madler/zlib.git - cd zlib +if [ ! -d "../3pp/zlib" ]; then + git clone https://github.com/madler/zlib.git ../3pp/zlib + cd ../3pp/zlib zlibPath=`pwd` ./configure --static - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } - cd .. + cd ../../build else - zlibPath=$(pwd)/zlib + zlibPath=$(pwd)/../3pp/zlib echo "Using compiled zlib" fi -if [ ! -d "libmicrohttpd" ]; then - git clone https://git.gnunet.org/libmicrohttpd.git - cd libmicrohttpd +if [ ! -d "../3pp/libmicrohttpd" ]; then + mkdir -p ../3pp/libmicrohttpd + wget -O ../3pp/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz + cd ../3pp/libmicrohttpd/ + tar xf libmicrohttpd-1.0.1.tar.gz + cd libmicrohttpd-1.0.1 libmicrohttpdPath=`pwd` - ./autogen.sh ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } - cd .. + cd ../../../build else - libmicrohttpdPath=$(pwd)/libmicrohttpd + libmicrohttpdPath=$(pwd)/../3pp/libmicrohttpd/libmicrohttpd-1.0.1 echo "Using compiled libmicrohttpd" fi @@ -181,9 +196,3 @@ if [ "$with_artifacts" = true ]; then rsync -r crypto/fift/lib artifacts/ chmod -R +x artifacts/* fi - -if [ "$with_tests" = true ]; then - cd build -# ctest --output-on-failure -E "test-catchain|test-actors" - ctest --output-on-failure -fi diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index 5c4a5fe07e..dbb4e7d29a 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -2,14 +2,16 @@ with_tests=false with_artifacts=false -OSX_TARGET=10.15 +with_ccache=false +OSX_TARGET=11.0 -while getopts 'tao:' flag; do +while getopts 'taco:' flag; do case "${flag}" in t) with_tests=true ;; a) with_artifacts=true ;; o) OSX_TARGET=${OPTARG} ;; + c) with_ccache=true ;; *) break ;; esac @@ -25,8 +27,19 @@ fi export NONINTERACTIVE=1 brew install ninja libsodium libmicrohttpd pkg-config automake libtool autoconf gnutls +export PATH=/usr/local/opt/ccache/libexec:$PATH brew install llvm@16 +if [ "$with_ccache" = true ]; then + brew install ccache + mkdir -p ~/.ccache + export CCACHE_DIR=~/.ccache + ccache -M 0 + test $? -eq 0 || { echo "ccache not installed"; exit 1; } +else + export CCACHE_DISABLE=1 +fi + if [ -f /opt/homebrew/opt/llvm@16/bin/clang ]; then export CC=/opt/homebrew/opt/llvm@16/bin/clang export CXX=/opt/homebrew/opt/llvm@16/bin/clang++ @@ -34,14 +47,13 @@ else export CC=/usr/local/opt/llvm@16/bin/clang export CXX=/usr/local/opt/llvm@16/bin/clang++ fi -export CCACHE_DISABLE=1 if [ ! -d "lz4" ]; then git clone https://github.com/lz4/lz4 cd lz4 lz4Path=`pwd` git checkout v1.9.4 - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } cd .. else @@ -110,8 +122,3 @@ if [ "$with_artifacts" = true ]; then chmod -R +x artifacts/* fi -if [ "$with_tests" = true ]; then - cd build -# ctest --output-on-failure -E "test-catchain|test-actors" - ctest --output-on-failure --timeout 1800 -fi diff --git a/assembly/native/build-ubuntu-appimages.sh b/assembly/native/build-ubuntu-appimages.sh index 4e63234d99..458ca9d0bf 100644 --- a/assembly/native/build-ubuntu-appimages.sh +++ b/assembly/native/build-ubuntu-appimages.sh @@ -2,17 +2,27 @@ with_tests=false with_artifacts=false +with_ccache=false - -while getopts 'ta' flag; do +while getopts 'tac' flag; do case "${flag}" in t) with_tests=true ;; a) with_artifacts=true ;; + c) with_ccache=true ;; *) break ;; esac done +if [ "$with_ccache" = true ]; then + mkdir -p ~/.ccache + export CCACHE_DIR=~/.ccache + ccache -M 0 + test $? -eq 0 || { echo "ccache not installed"; exit 1; } +else + export CCACHE_DISABLE=1 +fi + if [ ! -d "build" ]; then mkdir build cd build @@ -23,19 +33,18 @@ fi export CC=$(which clang-16) export CXX=$(which clang++-16) -export CCACHE_DISABLE=1 -if [ ! -d "openssl_3" ]; then - git clone https://github.com/openssl/openssl openssl_3 - cd openssl_3 +if [ ! -d "../openssl_3" ]; then + git clone https://github.com/openssl/openssl ../openssl_3 + cd ../openssl_3 opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make build_libs -j12 + make build_libs -j$(nproc) test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd .. + cd ../build else - opensslPath=$(pwd)/openssl_3 + opensslPath=$(pwd)/../openssl_3 echo "Using compiled openssl_3" fi @@ -104,6 +113,5 @@ fi if [ "$with_tests" = true ]; then cd build -# ctest --output-on-failure -E "test-catchain|test-actors|test-smartcont|test-adnl|test-validator-session-state|test-dht|test-rldp" ctest --output-on-failure --timeout 1800 fi diff --git a/assembly/native/build-ubuntu-portable-libs.sh b/assembly/native/build-ubuntu-portable-libs.sh index b634ce0c41..c7bffd40ee 100644 --- a/assembly/native/build-ubuntu-portable-libs.sh +++ b/assembly/native/build-ubuntu-portable-libs.sh @@ -1,18 +1,29 @@ #/bin/bash #sudo apt-get update -#sudo apt-get install -y build-essential git cmake ninja-build automake libtool texinfo autoconf libc++-dev libc++abi-dev +#sudo apt-get install -y build-essential git cmake ninja-build automake libtool texinfo autoconf libc++-dev libc++abi-dev ccache with_artifacts=false +with_ccache=false -while getopts 'ta' flag; do +while getopts 'tac' flag; do case "${flag}" in a) with_artifacts=true ;; + c) with_ccache=true ;; *) break ;; esac done +if [ "$with_ccache" = true ]; then + mkdir -p ~/.ccache + export CCACHE_DIR=~/.ccache + ccache -M 0 + test $? -eq 0 || { echo "ccache not installed"; exit 1; } +else + export CCACHE_DISABLE=1 +fi + if [ ! -d "build" ]; then mkdir build cd build @@ -23,77 +34,78 @@ fi export CC=$(which clang-16) export CXX=$(which clang++-16) -export CCACHE_DISABLE=1 -if [ ! -d "lz4" ]; then -git clone https://github.com/lz4/lz4.git -cd lz4 +if [ ! -d "../3pp/lz4" ]; then +mkdir -p ../3pp +git clone https://github.com/lz4/lz4.git ../3pp/lz4 +cd ../3pp/lz4 lz4Path=`pwd` git checkout v1.9.4 -CFLAGS="-fPIC" make -j12 +CFLAGS="-fPIC" make -j$(nproc) test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } -cd .. -# ./lib/liblz4.a -# ./lib +cd ../../build else - lz4Path=$(pwd)/lz4 + lz4Path=$(pwd)/../3pp/lz4 echo "Using compiled lz4" fi -if [ ! -d "libsodium" ]; then +if [ ! -d "../3pp/libsodium" ]; then export LIBSODIUM_FULL_BUILD=1 - git clone https://github.com/jedisct1/libsodium.git - cd libsodium + mkdir -p ../3pp/libsodium + wget -O ../3pp/libsodium/libsodium-1.0.18.tar.gz https://github.com/jedisct1/libsodium/releases/download/1.0.18-RELEASE/libsodium-1.0.18.tar.gz + cd ../3pp/libsodium + tar xf libsodium-1.0.18.tar.gz + cd libsodium-1.0.18 sodiumPath=`pwd` - git checkout 1.0.18 - ./autogen.sh ./configure --with-pic --enable-static - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } - cd .. + cd ../../../build else - sodiumPath=$(pwd)/libsodium + sodiumPath=$(pwd)/../3pp/libsodium/libsodium-1.0.18 echo "Using compiled libsodium" fi -if [ ! -d "openssl_3" ]; then - git clone https://github.com/openssl/openssl openssl_3 - cd openssl_3 +if [ ! -d "../3pp/openssl_3" ]; then + git clone https://github.com/openssl/openssl ../3pp/openssl_3 + cd ../3pp/openssl_3 opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make build_libs -j12 + make build_libs -j$(nproc) test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd .. + cd ../../build else - opensslPath=$(pwd)/openssl_3 + opensslPath=$(pwd)/../3pp/openssl_3 echo "Using compiled openssl_3" fi -if [ ! -d "zlib" ]; then - git clone https://github.com/madler/zlib.git - cd zlib +if [ ! -d "../3pp/zlib" ]; then + git clone https://github.com/madler/zlib.git ../3pp/zlib + cd ../3pp/zlib zlibPath=`pwd` ./configure --static - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } - cd .. + cd ../../build else - zlibPath=$(pwd)/zlib + zlibPath=$(pwd)/../3pp/zlib echo "Using compiled zlib" fi -if [ ! -d "libmicrohttpd" ]; then - git clone https://git.gnunet.org/libmicrohttpd.git - cd libmicrohttpd +if [ ! -d "../3pp/libmicrohttpd" ]; then + mkdir -p ../3pp/libmicrohttpd + wget -O ../3pp/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz + cd ../3pp/libmicrohttpd/ + tar xf libmicrohttpd-1.0.1.tar.gz + cd libmicrohttpd-1.0.1 libmicrohttpdPath=`pwd` - ./autogen.sh ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } - cd .. + cd ../../../build else - libmicrohttpdPath=$(pwd)/libmicrohttpd + libmicrohttpdPath=$(pwd)/../3pp/libmicrohttpd/libmicrohttpd-1.0.1 echo "Using compiled libmicrohttpd" fi @@ -117,11 +129,10 @@ cmake -GNinja .. \ -DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a - test $? -eq 0 || { echo "Can't configure ton"; exit 1; } ninja tonlibjson emulator -test $? -eq 0 || { echo "Can't compile ton"; exit 1; } +test $? -eq 0 || { echo "Can't compile tonlibjson and emulator"; exit 1; } cd .. diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh index 00b34f8320..f208148c9a 100644 --- a/assembly/native/build-ubuntu-portable.sh +++ b/assembly/native/build-ubuntu-portable.sh @@ -6,16 +6,26 @@ with_tests=false with_artifacts=false - -while getopts 'ta' flag; do +while getopts 'tac' flag; do case "${flag}" in t) with_tests=true ;; a) with_artifacts=true ;; + c) with_ccache=true ;; *) break ;; esac done +if [ "$with_ccache" = true ]; then + mkdir -p ~/.ccache + export CCACHE_DIR=~/.ccache + ccache -M 0 + test $? -eq 0 || { echo "ccache not installed"; exit 1; } +else + export CCACHE_DISABLE=1 + rm -rf ~/.ccache +fi + if [ ! -d "build" ]; then mkdir build cd build @@ -26,77 +36,77 @@ fi export CC=$(which clang-16) export CXX=$(which clang++-16) -export CCACHE_DISABLE=1 -if [ ! -d "lz4" ]; then -git clone https://github.com/lz4/lz4.git -cd lz4 +if [ ! -d "../3pp/lz4" ]; then +mkdir -p ../3pp +git clone https://github.com/lz4/lz4.git ../3pp/lz4 +cd ../3pp/lz4 lz4Path=`pwd` git checkout v1.9.4 -CFLAGS="-fPIC" make -j12 +CFLAGS="-fPIC" make -j$(nproc) test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } -cd .. -# ./lib/liblz4.a -# ./lib +cd ../../build else - lz4Path=$(pwd)/lz4 + lz4Path=$(pwd)/../3pp/lz4 echo "Using compiled lz4" fi -if [ ! -d "libsodium" ]; then +if [ ! -d "../3pp/libsodium" ]; then export LIBSODIUM_FULL_BUILD=1 - git clone https://github.com/jedisct1/libsodium.git - cd libsodium + git clone https://github.com/jedisct1/libsodium.git ../3pp/libsodium + cd ../3pp/libsodium sodiumPath=`pwd` git checkout 1.0.18 ./autogen.sh ./configure --with-pic --enable-static - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } - cd .. + cd ../../build else - sodiumPath=$(pwd)/libsodium + sodiumPath=$(pwd)/../3pp/libsodium echo "Using compiled libsodium" fi -if [ ! -d "openssl_3" ]; then - git clone https://github.com/openssl/openssl openssl_3 - cd openssl_3 +if [ ! -d "../3pp/openssl_3" ]; then + git clone https://github.com/openssl/openssl ../3pp/openssl_3 + cd ../3pp/openssl_3 opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make build_libs -j12 + make build_libs -j$(nproc) test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd .. + cd ../../build else - opensslPath=$(pwd)/openssl_3 + opensslPath=$(pwd)/../3pp/openssl_3 echo "Using compiled openssl_3" fi -if [ ! -d "zlib" ]; then - git clone https://github.com/madler/zlib.git - cd zlib +if [ ! -d "../3pp/zlib" ]; then + git clone https://github.com/madler/zlib.git ../3pp/zlib + cd ../3pp/zlib zlibPath=`pwd` ./configure --static - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } - cd .. + cd ../../build else - zlibPath=$(pwd)/zlib + zlibPath=$(pwd)/../3pp/zlib echo "Using compiled zlib" fi -if [ ! -d "libmicrohttpd" ]; then - git clone https://git.gnunet.org/libmicrohttpd.git - cd libmicrohttpd +if [ ! -d "../3pp/libmicrohttpd" ]; then + mkdir -p ../3pp/libmicrohttpd + wget -O ../3pp/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz + cd ../3pp/libmicrohttpd/ + tar xf libmicrohttpd-1.0.1.tar.gz + cd libmicrohttpd-1.0.1 libmicrohttpdPath=`pwd` - ./autogen.sh ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } - cd .. + cd ../../../build else - libmicrohttpdPath=$(pwd)/libmicrohttpd + libmicrohttpdPath=$(pwd)/../3pp/libmicrohttpd/libmicrohttpd-1.0.1 echo "Using compiled libmicrohttpd" fi @@ -120,7 +130,6 @@ cmake -GNinja .. \ -DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a - test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index 49cc8e1ea5..68ab6aba62 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -1,21 +1,31 @@ #/bin/bash #sudo apt-get update -#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev +#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev ccache with_tests=false with_artifacts=false +with_ccache=false - -while getopts 'ta' flag; do +while getopts 'tac' flag; do case "${flag}" in t) with_tests=true ;; a) with_artifacts=true ;; + c) with_ccache=true ;; *) break ;; esac done +if [ "$with_ccache" = true ]; then + mkdir -p ~/.ccache + export CCACHE_DIR=~/.ccache + ccache -M 0 + test $? -eq 0 || { echo "ccache not installed"; exit 1; } +else + export CCACHE_DISABLE=1 +fi + if [ ! -d "build" ]; then mkdir build cd build @@ -26,19 +36,19 @@ fi export CC=$(which clang-16) export CXX=$(which clang++-16) -export CCACHE_DISABLE=1 -if [ ! -d "openssl_3" ]; then - git clone https://github.com/openssl/openssl openssl_3 - cd openssl_3 + +if [ ! -d "../openssl_3" ]; then + git clone https://github.com/openssl/openssl ../openssl_3 + cd ../openssl_3 opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make build_libs -j12 + make build_libs -j$(nproc) test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd .. + cd ../build else - opensslPath=$(pwd)/openssl_3 + opensslPath=$(pwd)/../openssl_3 echo "Using compiled openssl_3" fi @@ -78,6 +88,7 @@ ldd ./validator-engine/validator-engine || exit 1 cd .. + if [ "$with_artifacts" = true ]; then rm -rf artifacts mkdir artifacts @@ -94,9 +105,3 @@ if [ "$with_artifacts" = true ]; then cp -R crypto/fift/lib artifacts chmod -R +x artifacts/* fi - -if [ "$with_tests" = true ]; then - cd build -# ctest --output-on-failure -E "test-catchain|test-actors|test-smartcont|test-adnl|test-validator-session-state|test-dht|test-rldp" - ctest --output-on-failure --timeout 1800 -fi diff --git a/assembly/native/build-windows-2019.bat b/assembly/native/build-windows-2019.bat index 844c09fcd6..0b96978ca7 100644 --- a/assembly/native/build-windows-2019.bat +++ b/assembly/native/build-windows-2019.bat @@ -26,6 +26,13 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) +echo Installing ccache... +choco install -y ccache +IF %errorlevel% NEQ 0 ( + echo Can't install ccache + exit /b %errorlevel% +) + echo Installing nasm... choco install -y nasm where nasm @@ -35,7 +42,9 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) -mkdir third_libs +if not exist "third_libs" ( + mkdir "third_libs" +) cd third_libs set third_libs=%cd% @@ -161,16 +170,6 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) -IF "%1"=="-t" ( - echo Running tests... -REM ctest -C Release --output-on-failure -E "test-catchain|test-actors|test-validator-session-state" - ctest -C Release --output-on-failure -E "test-bigint" --timeout 1800 - IF %errorlevel% NEQ 0 ( - echo Some tests failed - exit /b %errorlevel% - ) -) - echo Strip and copy artifacts cd .. echo where strip diff --git a/assembly/native/build-windows.bat b/assembly/native/build-windows.bat index 68f83c394d..c513140782 100644 --- a/assembly/native/build-windows.bat +++ b/assembly/native/build-windows.bat @@ -26,6 +26,13 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) +echo Installing ccache... +choco install -y ccache +IF %errorlevel% NEQ 0 ( + echo Can't install ccache + exit /b %errorlevel% +) + echo Installing nasm... choco install -y nasm where nasm @@ -35,7 +42,9 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) -mkdir third_libs +if not exist "third_libs" ( + mkdir "third_libs" +) cd third_libs set third_libs=%cd% @@ -161,16 +170,6 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) -IF "%1"=="-t" ( - echo Running tests... -REM ctest -C Release --output-on-failure -E "test-catchain|test-actors|test-validator-session-state" - ctest -C Release --output-on-failure -E "test-bigint" --timeout 1800 - IF %errorlevel% NEQ 0 ( - echo Some tests failed - exit /b %errorlevel% - ) -) - echo Strip and copy artifacts cd .. echo where strip diff --git a/assembly/wasm/fift-func-wasm-build-ubuntu.sh b/assembly/wasm/fift-func-wasm-build-ubuntu.sh index a463c02aac..a80430c1dd 100644 --- a/assembly/wasm/fift-func-wasm-build-ubuntu.sh +++ b/assembly/wasm/fift-func-wasm-build-ubuntu.sh @@ -2,7 +2,7 @@ # sudo apt update # sudo apt install -y build-essential git make cmake ninja-build clang libgflags-dev zlib1g-dev libssl-dev \ # libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip \ -# nodejs libsodium-dev automake libtool libjemalloc-dev +# nodejs libsodium-dev automake libtool libjemalloc-dev ccache # wget https://apt.llvm.org/llvm.sh # chmod +x llvm.sh @@ -30,19 +30,18 @@ if [ "$scratch_new" = true ]; then rm -rf openssl zlib lz4 emsdk libsodium build openssl_em fi - -if [ ! -d "openssl" ]; then - git clone https://github.com/openssl/openssl.git - cp -r openssl openssl_em - cd openssl +if [ ! -d "openssl_3" ]; then + git clone https://github.com/openssl/openssl openssl_3 + cd openssl_3 + opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make -j16 - OPENSSL_DIR=`pwd` + make build_libs -j$(nproc) + test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } cd .. else - OPENSSL_DIR=`pwd`/openssl - echo Using compiled openssl at $OPENSSL_DIR + opensslPath=$(pwd)/openssl_3 + echo "Using compiled openssl_3" fi if [ ! -d "build" ]; then @@ -50,9 +49,9 @@ if [ ! -d "build" ]; then cd build cmake -GNinja -DTON_USE_JEMALLOC=ON .. \ -DCMAKE_BUILD_TYPE=Release \ - -DOPENSSL_ROOT_DIR=$OPENSSL_DIR \ - -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \ - -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.so + -DOPENSSL_ROOT_DIR=$opensslPath \ + -DOPENSSL_INCLUDE_DIR=$opensslPath/include \ + -DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.so test $? -eq 0 || { echo "Can't configure TON build"; exit 1; } ninja fift smc-envelope @@ -82,61 +81,63 @@ export CCACHE_DISABLE=1 cd .. -if [ ! -f "openssl_em/openssl_em" ]; then - cd openssl_em +if [ ! -f "3pp_emscripten/openssl_em/openssl_em" ]; then + mkdir -p 3pp_emscripten + git clone https://github.com/openssl/openssl 3pp_emscripten/openssl_em + cd 3pp_emscripten/openssl_em emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test no-tests no-fuzz-afl no-fuzz-libfuzzer sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile sed -i 's/-ldl//g' Makefile sed -i 's/-O3/-Os/g' Makefile emmake make depend - emmake make -j16 + emmake make -j$(nproc) test $? -eq 0 || { echo "Can't compile OpenSSL with emmake "; exit 1; } - OPENSSL_DIR=`pwd` + opensslPath=`pwd` touch openssl_em - cd .. + cd ../.. else - OPENSSL_DIR=`pwd`/openssl_em - echo Using compiled with empscripten openssl at $OPENSSL_DIR + opensslPath=`pwd`/3pp_emscripten/openssl_em + echo Using compiled with empscripten openssl at $opensslPath fi -if [ ! -d "zlib" ]; then - git clone https://github.com/madler/zlib.git - cd zlib +if [ ! -d "3pp_emscripten/zlib" ]; then + git clone https://github.com/madler/zlib.git 3pp_emscripten/zlib + cd 3pp_emscripten/zlib git checkout v1.3.1 ZLIB_DIR=`pwd` emconfigure ./configure --static - emmake make -j16 + emmake make -j$(nproc) test $? -eq 0 || { echo "Can't compile zlib with emmake "; exit 1; } - cd .. + cd ../.. else - ZLIB_DIR=`pwd`/zlib + ZLIB_DIR=`pwd`/3pp_emscripten/zlib echo Using compiled zlib with emscripten at $ZLIB_DIR fi -if [ ! -d "lz4" ]; then - git clone https://github.com/lz4/lz4.git - cd lz4 +if [ ! -d "3pp_emscripten/lz4" ]; then + git clone https://github.com/lz4/lz4.git 3pp_emscripten/lz4 + cd 3pp_emscripten/lz4 git checkout v1.9.4 LZ4_DIR=`pwd` - emmake make -j16 + emmake make -j$(nproc) test $? -eq 0 || { echo "Can't compile lz4 with emmake "; exit 1; } - cd .. + cd ../.. else - LZ4_DIR=`pwd`/lz4 + LZ4_DIR=`pwd`/3pp_emscripten/lz4 echo Using compiled lz4 with emscripten at $LZ4_DIR fi -if [ ! -d "libsodium" ]; then - git clone https://github.com/jedisct1/libsodium - cd libsodium +if [ ! -d "3pp_emscripten/libsodium" ]; then + git clone https://github.com/jedisct1/libsodium 3pp_emscripten/libsodium + cd 3pp_emscripten/libsodium git checkout 1.0.18-RELEASE SODIUM_DIR=`pwd` emconfigure ./configure --disable-ssp - emmake make -j16 + emmake make -j$(nproc) test $? -eq 0 || { echo "Can't compile libsodium with emmake "; exit 1; } - cd .. + cd ../.. else - SODIUM_DIR=`pwd`/libsodium + SODIUM_DIR=`pwd`/3pp_emscripten/libsodium echo Using compiled libsodium with emscripten at $SODIUM_DIR fi @@ -150,8 +151,8 @@ emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAK -DLZ4_LIBRARIES=$LZ4_DIR/lib/liblz4.a \ -DLZ4_INCLUDE_DIRS=$LZ4_DIR/lib \ -DOPENSSL_FOUND=1 \ --DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \ --DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.a \ +-DOPENSSL_INCLUDE_DIR=$opensslPath/include \ +-DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a \ -DCMAKE_TOOLCHAIN_FILE=$EMSDK_DIR/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake \ -DCMAKE_CXX_FLAGS="-sUSE_ZLIB=1" \ -DSODIUM_FOUND=1 \ @@ -163,7 +164,7 @@ emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAK test $? -eq 0 || { echo "Can't configure TON with emmake "; exit 1; } cp -R ../crypto/smartcont ../crypto/fift/lib crypto -emmake make -j16 funcfiftlib func fift tlbc emulator-emscripten +emmake make -j$(nproc) funcfiftlib func fift tlbc emulator-emscripten test $? -eq 0 || { echo "Can't compile TON with emmake "; exit 1; } From 84082e79f7a8327d90ac23d7d594b8bbe661d05c Mon Sep 17 00:00:00 2001 From: birydrad Date: Sat, 29 Mar 2025 17:55:09 +0400 Subject: [PATCH 184/524] celldb: version 2 (#1463) * celldb: version 2 - thread safe cache - parallel commit - multiple optimizations - support of key-value merge operations - improved tests and benchmarks - in-memory version won't read from key value after start - uses vector in-memory table now - use rocksdb::WALRecoveryMode::kTolerateCorruptedTailRecords - do not silently ignore errors during recovery * celldb: add test for load nonexisting cell, test thread safeness of CellUsageTree, fixes --------- Co-authored-by: birydrad <> --- CMakeLists.txt | 1 + crypto/CMakeLists.txt | 3 +- crypto/test/test-db.cpp | 982 ++++++++++++--- crypto/vm/cells/Cell.h | 1 + crypto/vm/cells/DataCell.cpp | 5 +- crypto/vm/cells/DataCell.h | 8 +- crypto/vm/cells/ExtCell.h | 20 + crypto/vm/cells/PrunnedCell.h | 4 + crypto/vm/cells/UsageCell.h | 3 + crypto/vm/cells/VirtualCell.h | 3 + crypto/vm/db/CellHashTable.h | 13 +- crypto/vm/db/CellStorage.cpp | 85 ++ crypto/vm/db/CellStorage.h | 23 + crypto/vm/db/DynamicBagOfCellsDb.cpp | 85 +- crypto/vm/db/DynamicBagOfCellsDb.h | 71 +- crypto/vm/db/DynamicBagOfCellsDbV2.cpp | 1511 ++++++++++++++++++++++++ crypto/vm/db/InMemoryBagOfCellsDb.cpp | 180 ++- crypto/vm/db/StaticBagOfCellsDb.cpp | 18 +- tddb/td/db/KeyValue.h | 53 +- tddb/td/db/MemoryKeyValue.cpp | 82 +- tddb/td/db/MemoryKeyValue.h | 36 +- tddb/td/db/RocksDb.cpp | 123 +- tddb/td/db/RocksDb.h | 26 +- tdutils/td/utils/MpmcQueue.h | 4 +- tdutils/td/utils/Status.h | 7 + tdutils/td/utils/ThreadSafeCounter.h | 60 +- validator-engine/validator-engine.cpp | 12 +- validator-engine/validator-engine.hpp | 4 + validator/db/celldb.cpp | 218 +++- validator/db/celldb.hpp | 9 +- validator/validator-options.hpp | 7 + validator/validator.h | 2 + 32 files changed, 3343 insertions(+), 316 deletions(-) create mode 100644 crypto/vm/db/DynamicBagOfCellsDbV2.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index dd193d8aaf..2c65d66a95 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -106,6 +106,7 @@ if (TON_USE_ROCKSDB) set(WITH_GFLAGS OFF CACHE BOOL "build with GFlags") set(WITH_TESTS OFF CACHE BOOL "build with tests") set(WITH_TOOLS OFF CACHE BOOL "build with tools") + set(USE_RTTI ON CACHE BOOL "use rtti") set(FAIL_ON_WARNINGS OFF CACHE BOOL "fail on warnings") message("Add rocksdb") add_subdirectory(third-party/rocksdb EXCLUDE_FROM_ALL) diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 92ebf80448..2988f501ff 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -142,6 +142,7 @@ set(TON_CRYPTO_SOURCE set(TON_DB_SOURCE vm/db/DynamicBagOfCellsDb.cpp + vm/db/DynamicBagOfCellsDbV2.cpp vm/db/CellStorage.cpp vm/db/TonDb.cpp @@ -539,7 +540,7 @@ target_include_directories(create-state PUBLIC $ #include #include +#include #include #include "openssl/digest.hpp" +#include "storage/db.h" +#include "td/utils/VectorQueue.h" #include "vm/dict.h" -#include #include #include #include -#include +#include -namespace vm { -class ThreadExecutor : public DynamicBagOfCellsDb::AsyncExecutor { +#include +#include +#include + +#include "td/actor/actor.h" +#include "td/utils/overloaded.h" + + +class ActorExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { public: - explicit ThreadExecutor(size_t threads_n) { - for (size_t i = 0; i < threads_n; ++i) { - threads_.emplace_back([this]() { - while (true) { - auto task = pop_task(); - if (!task) { - break; - } - CHECK(generation_.load() % 2 == 1); - task(); - } - }); - } + ActorExecutor(size_t tn) : tn_(tn) { + scheduler_.run_in_context([&] { worker_ = td::actor::create_actor("Worker"); }); + thread_ = td::thread([this]() { scheduler_.run(); }); } - - ~ThreadExecutor() override { - for (size_t i = 0; i < threads_.size(); ++i) { - push_task({}); + ~ActorExecutor() { + scheduler_.run_in_context_external([&] { send_closure(worker_, &Worker::close); }); + thread_.join(); + } + std::string describe() const override { + return PSTRING() << "ActorExecutor(tn=" << tn_ << ")"; + } + class Worker : public td::actor::Actor { + public: + void close() { + td::actor::core::SchedulerContext::get()->stop(); + stop(); } - for (auto &t : threads_) { - t.join(); + void execute_sync(std::function f) { + f(); } - } + }; void execute_async(std::function f) override { - push_task(std::move(f)); + class Runner : public td::actor::Actor { + public: + explicit Runner(std::function f) : f_(std::move(f)) { + } + void start_up() override { + f_(); + stop(); + } + + private: + std::function f_; + }; + auto context = td::actor::SchedulerContext::get(); + if (context) { + td::actor::create_actor("executeasync", std::move(f)).release(); + } else { + scheduler_.run_in_context_external( + [&] { td::actor::create_actor("executeasync", std::move(f)).release(); }); + } } void execute_sync(std::function f) override { - auto x = generation_.load(); - std::scoped_lock lock(sync_mutex_); - CHECK(x == generation_); - CHECK(generation_.load() % 2 == 1); - f(); - CHECK(generation_.load() % 2 == 1); - } - void inc_generation() { - generation_.fetch_add(1); + auto context = td::actor::SchedulerContext::get(); + if (context) { + td::actor::send_closure(worker_, &Worker::execute_sync, std::move(f)); + } else { + scheduler_.run_in_context_external( + [&] { td::actor::send_closure(worker_, &Worker::execute_sync, std::move(f)); }); + } } private: - std::atomic generation_{0}; - std::queue, size_t>> queue_; - std::mutex queue_mutex_; - std::condition_variable cv_; - std::mutex sync_mutex_; - std::vector threads_; - - std::function pop_task() { - std::unique_lock lock(queue_mutex_); - cv_.wait(lock, [&] { return !queue_.empty(); }); - CHECK(!queue_.empty()); - auto task = std::move(queue_.front()); - queue_.pop(); - CHECK(task.second == generation_); - return task.first; - } - - void push_task(std::function task) { - { - std::scoped_lock lock(queue_mutex_); - queue_.emplace(std::move(task), generation_.load()); - } - cv_.notify_one(); - } + size_t tn_; + td::actor::Scheduler scheduler_{{tn_}, false, td::actor::Scheduler::Paused}; + td::actor::ActorOwn worker_; + td::thread thread_; }; +namespace vm { std::vector do_get_serialization_modes() { std::vector res; for (int i = 0; i < 32; i++) { @@ -324,6 +323,34 @@ TEST(Cell, sha_benchmark_threaded) { bench_threaded([n]() { return BenchSha256(n); }); } } +class BenchTasks : public td::Benchmark { + public: + explicit BenchTasks(size_t tn) : tn_(tn) { + } + + std::string get_description() const override { + return PSTRING() << "bench_tasks(threads_n=" << tn_ << ")"; + } + + void run(int n) override { + ActorExecutor executor(tn_); + for (int i = 0; i < n; i++) { + std::latch latch(tn_); + for (size_t j = 0; j < tn_; j++) { + executor.execute_async([&]() { latch.count_down(); }); + } + latch.wait(); + } + } + + private: + size_t tn_{}; +}; +TEST(Bench, Tasks) { + for (size_t tn : {1, 4, 16}) { + bench(BenchTasks(tn)); + } +} std::string serialize_boc(Ref cell, int mode = 31) { CHECK(cell.not_null()); @@ -437,6 +464,8 @@ class CellExplorer { cs_ = {}; break; } + default: + UNREACHABLE(); } } @@ -474,6 +503,8 @@ class CellExplorer { case op.ReadCellSlice: log_ << "read slice " << op.children_mask << "\n"; break; + default: + UNREACHABLE(); } } void log_cell(const Ref &cell) { @@ -627,7 +658,9 @@ TEST(Cell, MerkleProof) { auto exploration2 = CellExplorer::explore(usage_cell, exploration.ops); ASSERT_EQ(exploration.log, exploration2.log); - auto is_prunned = [&](const Ref &cell) { return exploration.visited.count(cell->get_hash()) == 0; }; + auto is_prunned = [&](const Ref &cell_to_check) { + return exploration.visited.count(cell_to_check->get_hash()) == 0; + }; auto proof = MerkleProof::generate(cell, is_prunned); // CellBuilder::virtualize(proof, 1); //ASSERT_EQ(1u, proof->get_level()); @@ -706,7 +739,7 @@ TEST(Cell, MerkleProofCombine) { check(proof_union_fast); } { - auto cell = MerkleProof::virtualize(proof12, 1); + cell = MerkleProof::virtualize(proof12, 1); auto usage_tree = std::make_shared(); auto usage_cell = UsageCell::create(cell, usage_tree->root_ptr()); @@ -927,7 +960,6 @@ TEST(TonDb, InMemoryDynamicBocSimple) { auto before = counter(); SCOPE_EXIT { LOG_CHECK(before == counter()) << before << " vs " << counter(); - ; }; td::Random::Xorshift128plus rnd{123}; auto kv = std::make_shared(); @@ -963,26 +995,193 @@ TEST(TonDb, InMemoryDynamicBocSimple) { int VERBOSITY_NAME(boc) = VERBOSITY_NAME(DEBUG) + 10; +struct CellMerger : td::Merger { + void merge_value_and_update(std::string &value, td::Slice update) override { + return CellStorer::merge_value_and_refcnt_diff(value, update); + } + void merge_update_and_update(std::string &left_update, td::Slice right_update) override { + LOG(ERROR) << "update_and_update"; + UNREACHABLE(); + return CellStorer::merge_refcnt_diffs(left_update, right_update); + } +}; +struct CompactionFilterEraseEmptyValues : public rocksdb::CompactionFilter { + bool Filter(int level, const rocksdb::Slice & /*key*/, const rocksdb::Slice &existing_value, std::string *new_value, + bool *value_changed) const override { + return existing_value.empty(); + } + bool FilterMergeOperand(int, const rocksdb::Slice & /*key*/, const rocksdb::Slice &operand) const override { + return operand.empty(); + } + + // Name of the compaction filter + const char *Name() const override { + return "CompactionFilterEraseEmptyValues"; + } +}; +auto to_td(rocksdb::Slice value) -> td::Slice { + return td::Slice(value.data(), value.size()); +} + +struct MergeOperatorAddCellRefcnt : public rocksdb::MergeOperator { + const char *Name() const override { + return "MergeOperatorAddCellRefcnt"; + } + bool FullMergeV2(const MergeOperationInput &merge_in, MergeOperationOutput *merge_out) const override { + CHECK(merge_in.existing_value); + auto &value = *merge_in.existing_value; + CHECK(merge_in.operand_list.size() >= 1); + td::Slice diff; + std::string diff_buf; + if (merge_in.operand_list.size() == 1) { + diff = to_td(merge_in.operand_list[0]); + } else { + diff_buf = merge_in.operand_list[0].ToString(); + for (size_t i = 1; i < merge_in.operand_list.size(); ++i) { + CellStorer::merge_refcnt_diffs(diff_buf, to_td(merge_in.operand_list[i])); + } + diff = diff_buf; + } + + merge_out->new_value = value.ToString(); + CellStorer::merge_value_and_refcnt_diff(merge_out->new_value, diff); + return true; + } + bool PartialMerge(const rocksdb::Slice & /*key*/, const rocksdb::Slice &left, const rocksdb::Slice &right, + std::string *new_value, rocksdb::Logger *logger) const override { + *new_value = left.ToString(); + CellStorer::merge_refcnt_diffs(*new_value, to_td(right)); + return true; + } +}; + +struct DB { + std::unique_ptr dboc; + std::shared_ptr kv; + void reset_loader() { + dboc->set_loader(std::make_unique(kv->snapshot())); + } +}; struct BocOptions { - std::shared_ptr async_executor; - std::optional o_in_memory; + using AsyncExecutor = DynamicBagOfCellsDb::AsyncExecutor; + + using CreateInMemoryOptions = DynamicBagOfCellsDb::CreateInMemoryOptions; + using CreateV1Options = DynamicBagOfCellsDb::CreateV1Options; + using CreateV2Options = DynamicBagOfCellsDb::CreateV2Options; + + std::shared_ptr async_executor; + struct KvOptions { + enum KvType { InMemory, RocksDb } kv_type{InMemory}; + bool experimental{false}; + bool no_transactions{false}; + size_t cache_size{0}; + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const KvOptions &kv_options) { + if (kv_options.kv_type == KvType::InMemory) { + return sb << "InMemory{}"; + } + return sb << "RockDb{cache_size=" << kv_options.cache_size << ", no_transactions=" << kv_options.no_transactions + << ", experimental=" << kv_options.experimental << "}"; + } + }; + KvOptions kv_options; + std::variant options; td::uint64 seed{123}; - auto create_dboc(td::KeyValueReader *kv, std::optional o_root_n) { - if (o_in_memory) { - auto res = DynamicBagOfCellsDb::create_in_memory(kv, *o_in_memory); - auto stats = res->get_stats().move_as_ok(); - if (o_root_n) { - ASSERT_EQ(*o_root_n, stats.roots_total_count); + std::shared_ptr create_kv(std::shared_ptr old_key_value, bool no_reads = false) { + if (kv_options.kv_type == KvOptions::InMemory) { + if (old_key_value) { + return old_key_value; + } + return std::make_shared(std::make_shared()); + } else if (kv_options.kv_type == KvOptions::RocksDb) { + auto merge_operator = std::make_shared(); + static const CompactionFilterEraseEmptyValues compaction_filter; + CHECK(!old_key_value || old_key_value.use_count() == 1); + std::string db_path = "test_celldb"; + if (old_key_value) { + //LOG(ERROR) << "Reload rocksdb"; + old_key_value.reset(); + } else { + //LOG(ERROR) << "New rocksdb"; + td::RocksDb::destroy(db_path).ensure(); + } + auto db_options = td::RocksDbOptions{ + .block_cache = {}, + .merge_operator = merge_operator, + .compaction_filter = &compaction_filter, + .experimental = kv_options.experimental, + .no_reads = no_reads, + .no_transactions = kv_options.no_transactions, + .use_direct_reads = true, + .no_block_cache = true, + }; + if (kv_options.cache_size != 0) { + db_options.no_block_cache = false; + db_options.block_cache = rocksdb::NewLRUCache(kv_options.cache_size); } - VLOG(boc) << "reset roots_n=" << stats.roots_total_count << " cells_n=" << stats.cells_total_count; - return res; + return std::make_shared(td::RocksDb::open(db_path, std::move(db_options)).move_as_ok()); + } else { + UNREACHABLE(); + } + } + void check_kv_is_empty(KeyValue &kv) { + if (kv_options.kv_type == KvOptions::InMemory) { + ASSERT_EQ(0u, kv.count("").move_as_ok()); + return; } - return DynamicBagOfCellsDb::create(); + + size_t non_empty_values = 0; + kv.for_each([&](auto key, auto value) { + non_empty_values += !value.empty(); + return td::Status::OK(); + }); + if (non_empty_values != 0) { + kv.for_each([&](auto key, auto value) { + LOG(ERROR) << "Key: " << td::hex_encode(key) << " Value: " << td::hex_encode(value); + std::string x; + LOG(ERROR) << int(kv.get(key, x).move_as_ok()); + return td::Status::OK(); + }); + } + ASSERT_EQ(0u, non_empty_values); + } + + [[nodiscard]] auto create_db(DB db, std::optional o_root_n) { + auto old_boc = std::move(db.dboc); + auto old_kv = std::move(db.kv); + old_boc.reset(); + using ResT = DB; + return std::visit(td::overloaded( + [&](CreateV1Options &) -> ResT { + auto new_kv = create_kv(std::move(old_kv)); + auto res = DynamicBagOfCellsDb::create(); + res->set_loader(std::make_unique(new_kv->snapshot())); + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }, + [&](CreateV2Options &options) -> ResT { + auto new_kv = create_kv(std::move(old_kv)); + auto res = DynamicBagOfCellsDb::create_v2(options); + res->set_loader(std::make_unique(new_kv->snapshot())); + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }, + [&](CreateInMemoryOptions &options) -> ResT { + auto read_kv = create_kv(std::move(old_kv), false); + auto res = DynamicBagOfCellsDb::create_in_memory(read_kv.get(), options); + auto new_kv = create_kv(std::move(read_kv), true); + res->set_loader(std::make_unique(new_kv->snapshot())); + auto stats = res->get_stats().move_as_ok(); + if (o_root_n) { + ASSERT_EQ(*o_root_n, stats.roots_total_count); + } + VLOG(boc) << "reset roots_n=" << stats.roots_total_count + << " cells_n=" << stats.cells_total_count; + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }), + options); }; void prepare_commit(DynamicBagOfCellsDb &dboc) { + td::PerfWarningTimer warning_timer("test_db_prepare_commit"); if (async_executor) { - async_executor->inc_generation(); std::latch latch(1); td::Result res; async_executor->execute_sync([&] { @@ -993,70 +1192,175 @@ struct BocOptions { }); latch.wait(); async_executor->execute_sync([&] {}); - async_executor->inc_generation(); + res.ensure(); } else { dboc.prepare_commit(); } } + enum CacheAction { ResetCache, KeepCache }; + void write_commit(DynamicBagOfCellsDb &dboc, std::shared_ptr kv, CacheAction action) { + td::PerfWarningTimer warning_timer("test_db_write_commit"); + kv->begin_write_batch().ensure(); + CellStorer cell_storer(*kv); + { + td::PerfWarningTimer timer("test_db_commit"); + dboc.commit(cell_storer).ensure(); + } + { + td::PerfWarningTimer timer("test_db_commit_write_batch"); + kv->commit_write_batch().ensure(); + } + switch (action) { + case ResetCache: { + td::PerfWarningTimer timer("test_db_reset_cache"); + dboc.set_loader(std::make_unique(kv->snapshot())); + break; + } + case KeepCache: + break; + } + } + + void commit(DB &db, CacheAction action = ResetCache) { + prepare_commit(*db.dboc); + write_commit(*db.dboc, db.kv, action); + } + + std::string description() const { + td::StringBuilder sb; + + sb << "DBOC(type="; + std::visit(td::overloaded([&](const CreateV1Options &) { sb << "V1"; }, + [&](const CreateV2Options &options) { + sb << "V2(concurrency=" << options.extra_threads + 1; + if (options.executor) { + sb << ", executor=" << options.executor->describe(); + } else { + sb << ", executor=threads"; + } + sb << ")"; + }, + [&](const CreateInMemoryOptions &options) { + sb << "InMemory(use_arena=" << options.use_arena + << ", less_memory=" << options.use_less_memory_during_creation << ")"; + }), + options); + sb << kv_options; + if (async_executor) { + sb << ", executor=" << async_executor->describe(); + } + sb << ")"; + + return sb.as_cslice().str(); + } }; template -void with_all_boc_options(F &&f, size_t tests_n = 500) { +void with_all_boc_options(F &&f, size_t tests_n, bool single_thread = false) { LOG(INFO) << "Test dynamic boc"; auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; + std::map>> benches; auto run = [&](BocOptions options) { - LOG(INFO) << "\t" << (options.o_in_memory ? "in memory" : "on disk") << (options.async_executor ? " async" : ""); - if (options.o_in_memory) { - LOG(INFO) << "\t\tuse_arena=" << options.o_in_memory->use_arena - << " less_memory=" << options.o_in_memory->use_less_memory_during_creation; - } + auto description = options.description(); + LOG(INFO) << "Running " << description; + auto start = td::Timestamp::now(); + DynamicBagOfCellsDb::Stats stats; + auto o_in_memory = std::get_if(&options.options); for (td::uint32 i = 0; i < tests_n; i++) { auto before = counter(); + options.seed = i == 0 ? 123 : i; - f(options); + auto stats_diff = f(options); + stats.apply_diff(stats_diff); + auto after = counter(); - LOG_CHECK((options.o_in_memory && options.o_in_memory->use_arena) || before == after) - << before << " vs " << after; + LOG_CHECK((o_in_memory && o_in_memory->use_arena) || before == after) << before << " vs " << after; } + LOG(INFO) << "\ttook " << td::Timestamp::now().at() - start.at() << " seconds"; + LOG(INFO) << stats; + for (auto &[key, value] : stats.named_stats.stats_int) { + if (td::begins_with(key, "bench_")) { + benches[key].emplace_back(value, description); + } + } + }; + + // NB: use .experimental to play with different RocksDb parameters + // Note, that new benchmark are necessary to fully understand the effect of different RocksDb options + std::vector kv_options_list = { + // BocOptions::KvOptions{.kv_type = BocOptions::KvOptions::InMemory}, + // BocOptions::KvOptions{.kv_type = BocOptions::KvOptions::RocksDb, .experimental = false, .cache_size = 0}, + BocOptions::KvOptions{ + .kv_type = BocOptions::KvOptions::RocksDb, .experimental = false, .cache_size = size_t{128 << 20}}, }; - run({.async_executor = std::make_shared(4)}); - run({}); - for (auto use_arena : {false, true}) { - for (auto less_memory : {false, true}) { - run({.o_in_memory = - DynamicBagOfCellsDb::CreateInMemoryOptions{.extra_threads = std::thread::hardware_concurrency(), - .verbose = false, - .use_arena = use_arena, - .use_less_memory_during_creation = less_memory}}); + std::vector has_executor_options = {false, true}; + for (auto kv_options : kv_options_list) { + for (bool has_executor : has_executor_options) { + std::shared_ptr executor; + if (has_executor) { + executor = std::make_shared( + 4); // 4 - to compare V1 and V2, because V1 has parallel_load = 4 by default + } + // V2 - 4 threads + run({.async_executor = executor, + .kv_options = kv_options, + .options = DynamicBagOfCellsDb::CreateV2Options{ + .extra_threads = 3, .executor = executor, .cache_ttl_max = 5}}); + + // V1 + run({.async_executor = executor, .kv_options = kv_options, .options = DynamicBagOfCellsDb::CreateV1Options{}}); + + // V2 - one thread + run({.async_executor = executor, + .kv_options = kv_options, + .options = + DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 0, .executor = executor, .cache_ttl_max = 5}}); + + // InMemory + for (auto use_arena : {false, true}) { + for (auto less_memory : {false, true}) { + run({.async_executor = executor, + .kv_options = kv_options, + .options = + DynamicBagOfCellsDb::CreateInMemoryOptions{.extra_threads = std::thread::hardware_concurrency(), + .verbose = false, + .use_arena = use_arena, + .use_less_memory_during_creation = less_memory}}); + } + } + } + } + + for (auto &[name, v] : benches) { + std::sort(v.begin(), v.end()); + LOG(INFO) << "Bench " << name; + for (auto &[t, name] : v) { + LOG(INFO) << "\t" << name << " " << double(t) / 1000 << "s"; } } } -void test_dynamic_boc(BocOptions options) { - auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; - auto before = counter(); - SCOPE_EXIT { - LOG_CHECK((options.o_in_memory && options.o_in_memory->use_arena) || before == counter()) - << before << " vs " << counter(); - }; +DynamicBagOfCellsDb::Stats test_dynamic_boc(BocOptions options) { + DynamicBagOfCellsDb::Stats stats; td::Random::Xorshift128plus rnd{options.seed}; std::string old_root_hash; std::string old_root_serialization; - auto kv = std::make_shared(); - auto create_dboc = [&]() { + DB db; + auto reload_db = [&]() { auto roots_n = old_root_hash.empty() ? 0 : 1; - return options.create_dboc(kv.get(), roots_n); + db = options.create_db(std::move(db), roots_n); }; - auto dboc = create_dboc(); - dboc->set_loader(std::make_unique(kv)); + reload_db(); for (int t = 1000; t >= 0; t--) { if (rnd() % 10 == 0) { - dboc = create_dboc(); + reload_db(); } - dboc->set_loader(std::make_unique(kv)); + db.dboc->load_cell(vm::CellHash{}.as_slice()).ensure_error(); + + db.reset_loader(); Ref old_root; if (!old_root_hash.empty()) { - old_root = dboc->load_cell(old_root_hash).move_as_ok(); + old_root = db.dboc->load_cell(old_root_hash).move_as_ok(); auto serialization = serialize_boc(old_root); ASSERT_EQ(old_root_serialization, serialization); } @@ -1071,47 +1375,61 @@ void test_dynamic_boc(BocOptions options) { ->get_root_cell(0) .move_as_ok(); - dboc->dec(old_root); + db.dboc->dec(old_root); if (t != 0) { - dboc->inc(cell); - } - dboc->prepare_commit().ensure(); - { - CellStorer cell_storer(*kv); - dboc->commit(cell_storer).ensure(); + db.dboc->inc(cell); } + options.commit(db, BocOptions::ResetCache); } - ASSERT_EQ(0u, kv->count("").ok()); + options.check_kv_is_empty(*db.kv); + + stats.named_stats.apply_diff(db.kv->get_usage_stats().to_named_stats()); + return stats; } TEST(TonDb, DynamicBoc) { with_all_boc_options(test_dynamic_boc, 1); }; -void test_dynamic_boc2(BocOptions options) { +DynamicBagOfCellsDb::Stats test_dynamic_boc2(BocOptions options) { td::Random::Xorshift128plus rnd{options.seed}; + DynamicBagOfCellsDb::Stats stats; - int total_roots = rnd.fast(1, !rnd.fast(0, 10) * 100 + 10); + int total_roots = rnd.fast(1, !rnd.fast(0, 30) * 100 + 10); int max_roots = rnd.fast(1, 20); + int max_cells = 20; + + // VERBOSITY_NAME(boc) = 1; + // LOG(WARNING) << "====================================================\n\n"; + // max_roots = 2; + // total_roots = 2; + // max_cells = 2; + + auto meta_key = [](size_t i) { return PSTRING() << "meta." << i; }; + std::array meta; + int last_commit_at = 0; int first_root_id = 0; int last_root_id = 0; - auto kv = std::make_shared(); - auto create_dboc = [&](td::int64 root_n) { return options.create_dboc(kv.get(), root_n); }; - auto dboc = create_dboc(0); - dboc->set_loader(std::make_unique(kv)); + DB db; + auto reload_db = [&](td::int64 root_n) { db = options.create_db(std::move(db), root_n); }; + reload_db(0); auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; auto before = counter(); - SCOPE_EXIT{ - // LOG_CHECK((options.o_in_memory && options.o_in_memory->use_arena) || before == counter()) - // << before << " vs " << counter(); + SCOPE_EXIT { + bool skip_check = false; + if (std::holds_alternative(options.options) && + std::get(options.options).use_arena) { + skip_check = true; + } + LOG_IF(FATAL, !(skip_check || before == counter())) << before << " vs " << counter(); }; std::vector> roots(max_roots); std::vector root_hashes(max_roots); auto add_root = [&](Ref root) { - dboc->inc(root); + db.dboc->inc(root); root_hashes[last_root_id % max_roots] = (root->get_hash().as_slice().str()); roots[last_root_id % max_roots] = root; last_root_id++; @@ -1124,9 +1442,9 @@ void test_dynamic_boc2(BocOptions options) { VLOG(boc) << " from db"; auto from_root_hash = root_hashes[root_id % max_roots]; if (rnd() % 2 == 0) { - from_root = dboc->load_root(from_root_hash).move_as_ok(); + from_root = db.dboc->load_root(from_root_hash).move_as_ok(); } else { - from_root = dboc->load_cell(from_root_hash).move_as_ok(); + from_root = db.dboc->load_cell(from_root_hash).move_as_ok(); } } else { VLOG(boc) << "FROM MEMORY"; @@ -1147,31 +1465,69 @@ void test_dynamic_boc2(BocOptions options) { from_root = get_root(rnd.fast(first_root_id, last_root_id - 1)); } VLOG(boc) << " ..."; - auto new_root = gen_random_cell(rnd.fast(1, 20), from_root, rnd); - root_cnt[new_root->get_hash()]++; - add_root(std::move(new_root)); + auto new_root_cell = gen_random_cell(rnd.fast(1, max_cells), from_root, rnd); + root_cnt[new_root_cell->get_hash()]++; + add_root(std::move(new_root_cell)); VLOG(boc) << " OK"; }; - auto commit = [&] { - VLOG(boc) << "commit"; - //rnd.fast(0, 1); - options.prepare_commit(*dboc); - { - CellStorer cell_storer(*kv); - dboc->commit(cell_storer); + td::UsageStats commit_stats{}; + auto commit = [&](bool finish = false) { + for (size_t i = 0; i < meta.size(); i++) { + std::string value; + auto status = db.dboc->meta_get(meta_key(i), value).move_as_ok(); + if (status == KeyValue::GetStatus::Ok) { + ASSERT_EQ(value, meta[i]); + ASSERT_TRUE(!meta[i].empty()); + } else { + ASSERT_TRUE(meta[i].empty()); + } + + if (meta[i].empty()) { + if (!finish && rnd() % 2 == 0) { + meta[i] = td::to_string(rnd()); + db.dboc->meta_set(meta_key(i), meta[i]); + VLOG(boc) << "meta set " << meta_key(i) << " " << meta[i]; + } + } else { + auto f = finish ? 1 : rnd() % 3; + if (f == 0) { + meta[i] = td::to_string(rnd()); + db.dboc->meta_set(meta_key(i), meta[i]); + VLOG(boc) << "meta set " << meta_key(i) << " " << meta[i]; + } else if (f == 1) { + meta[i] = ""; + db.dboc->meta_erase(meta_key(i)); + VLOG(boc) << "meta erase " << meta_key(i); + } + } } - dboc->set_loader(std::make_unique(kv)); + + VLOG(boc) << "before commit cells_in_db=" << db.kv->count(""); + //rnd.fast(0, 1); + auto stats_before = db.kv->get_usage_stats(); + options.commit(db, BocOptions::ResetCache); + auto stats_after = db.kv->get_usage_stats(); + commit_stats = commit_stats + stats_after - stats_before; + VLOG(boc) << "after commit cells_in_db=" << db.kv->count(""); + + // db.reset_loader(); for (int i = last_commit_at; i < last_root_id; i++) { roots[i % max_roots].clear(); } last_commit_at = last_root_id; }; - auto reset = [&] { + auto reset = [&](bool force_full = false) { VLOG(boc) << "reset"; commit(); - dboc = create_dboc(td::int64(root_cnt.size())); - dboc->set_loader(std::make_unique(kv)); + if (rnd() % 3 == 0 || force_full) { + // very slow for rocksdb + auto r_stats = db.dboc->get_stats(); + if (r_stats.is_ok()) { + stats.apply_diff(r_stats.ok()); + } + reload_db(root_cnt.size()); + } }; auto delete_root = [&] { @@ -1187,22 +1543,28 @@ void test_dynamic_boc2(BocOptions options) { root_cnt.erase(it); } - dboc->dec(std::move(old_root)); + db.dboc->dec(std::move(old_root)); first_root_id++; VLOG(boc) << " OK"; }; td::RandomSteps steps({{new_root, 10}, {delete_root, 9}, {commit, 2}, {reset, 1}}); while (first_root_id != total_roots) { - VLOG(boc) << first_root_id << " " << last_root_id << " " << kv->count("").ok(); + VLOG(boc) << first_root_id << " " << last_root_id; // << " " << db.kv->count("").ok(); steps.step(rnd); } - commit(); - ASSERT_EQ(0u, kv->count("").ok()); + commit(true); + options.check_kv_is_empty(*db.kv); + + // auto stats = kv->get_usage_stats(); + // LOG(ERROR) << "total: " << stats; + reset(true); + stats.named_stats.apply_diff(db.kv->get_usage_stats().to_named_stats()); + return stats; } TEST(TonDb, DynamicBoc2) { - with_all_boc_options(test_dynamic_boc2); + with_all_boc_options(test_dynamic_boc2, 50); } template @@ -1341,6 +1703,10 @@ class CompactArray { size_t size() const { return size_; } + void reset() { + size_ = 0; + root_ = {}; + } Ref merkle_proof(std::vector keys) { std::set hashes; @@ -1435,6 +1801,283 @@ class FastCompactArray { std::vector v_; }; +struct BocTestHelper { + public: + BocTestHelper() = default; + BocTestHelper(td::int64 seed) : rnd_(seed) { + } + + CompactArray create_array(size_t size, td::uint64 max_value) { + std::vector v(size); + td::Random::Xorshift128plus rnd{123}; + for (auto &x : v) { + x = rnd() % max_value; + } + return CompactArray(v); + } + + private: + td::Random::Xorshift128plus rnd_{123}; +}; + +DynamicBagOfCellsDb::Stats bench_dboc_get_and_set(BocOptions options) { + BocTestHelper helper(options.seed); + size_t n = 1 << 20; + size_t max_value = 1 << 26; + auto arr = helper.create_array(n, max_value); + + // auto kv = std::make_shared(); + td::Slice db_path = "compact_array_db"; + td::RocksDb::destroy(db_path).ensure(); + + DB db = options.create_db({}, {}); + DynamicBagOfCellsDb::Stats stats; + + td::Timer total_timer; + + auto bench = [&](td::Slice desc, auto &&f) { + auto before = db.dboc->get_stats().move_as_ok(); + td::Timer timer; + LOG(ERROR) << "Benchmarking " << desc; + f(); + stats.named_stats.stats_int[desc.str()] = td::int64(timer.elapsed() * 1000); + LOG(ERROR) << "Benchmarking " << desc << " done: " << timer.elapsed() << "s\n"; + auto after = db.dboc->get_stats().move_as_ok(); + after.named_stats.subtract_diff(before.named_stats); + LOG(ERROR) << after; + }; + + td::VectorQueue roots; + // Save array in db + bench(PSLICE() << "bench_inc_large_db(n=" << n << ")", [&] { + db.dboc->inc(arr.root()); + roots.push(arr.root()->get_hash()); + options.commit(db, BocOptions::ResetCache); + }); + bench("bench_compactify", [&] { + auto status = dynamic_cast(*db.kv).raw_db()->CompactRange({}, nullptr, nullptr); + LOG_IF(FATAL, !status.ok()) << status.ToString(); + }); + db = options.create_db(std::move(db), {}); + + bench(PSLICE() << "bench_inc_large_existed_db(n=" << n << ")", [&] { + db.dboc->inc(arr.root()); + roots.push(arr.root()->get_hash()); + options.commit(db, BocOptions::ResetCache); + }); + + td::Random::Xorshift128plus rnd{123}; + while (false) { + auto hash = arr.root()->get_hash(); + arr = CompactArray{n, db.dboc->load_root(hash.as_slice()).move_as_ok()}; + td::Timer timer; + for (size_t i = 0; i < 10000; i++) { + auto pos = rnd() % n; + arr.get(pos); + } + LOG(ERROR) << timer.elapsed() << "s\n"; + db.reset_loader(); + } + + for (auto p : + std::vector>{{10000, 0}, {10000, 5}, {5000, 5000}, {5, 10000}, {0, 10000}}) { + auto get_n = p.first; + auto set_n = p.second; + auto hash = arr.root()->get_hash(); + arr = CompactArray{n, db.dboc->load_root(hash.as_slice()).move_as_ok()}; + bench(PSTRING() << "bench_changes(get_n=" << get_n << ", set_n=" << set_n << ")", [&] { + for (size_t i = 0; i < get_n; i++) { + auto pos = rnd() % n; + arr.get(pos); + } + for (size_t i = 0; i < set_n; i++) { + auto pos = rnd() % n; + auto value = rnd() % max_value; + arr.set(pos, value); + } + }); + bench(PSTRING() << "bench_commit(get_n=" << get_n << ", set_n=" << set_n << ")", [&] { + db.dboc->inc(arr.root()); + roots.push(arr.root()->get_hash()); + options.commit(db, BocOptions::ResetCache); + }); + } + arr.reset(); + + bench(PSLICE() << "bench_dec_some_roots()", [&] { + while (roots.size() > 1) { + auto hash = roots.pop(); + auto cell = db.dboc->load_cell(hash.as_slice()).move_as_ok(); + db.dboc->dec(cell); + } + options.commit(db, BocOptions::ResetCache); + }); + + db = options.create_db(std::move(db), {}); + + bench(PSLICE() << "bench_dec_large_root(n=" << n << ")", [&] { + while (!roots.empty()) { + auto hash = roots.pop(); + auto cell = db.dboc->load_cell(hash.as_slice()).move_as_ok(); + db.dboc->dec(cell); + + /* + do { + auto cell = db.dboc->load_cell(hash.as_slice()).move_as_ok(); + db.dboc->dec(cell); + cell = {}; + options.prepare_commit(*db.dboc); + //db.dboc->prepare_commit().ensure(); + db.reset_loader(); + db = options.create_db(std::move(db), {}); + } while (true); + */ + } + options.commit(db, BocOptions::ResetCache); + }); + stats.named_stats.stats_int["bench_total"] = td::int64(total_timer.elapsed() * 1000); + + return stats; +} + +TEST(TonDb, BenchDynamicBocGetAndSet) { + with_all_boc_options(bench_dboc_get_and_set, 1); +} + +TEST(TonDb, DynamicBocIncSimple) { + auto kv = std::make_shared(std::make_shared()); + auto db = DynamicBagOfCellsDb::create_v2({.extra_threads = 0}); + db->set_loader(std::make_unique(kv)); + + td::Random::Xorshift128plus rnd(123); + size_t size = 4; + std::vector values(size); + for (auto &v : values) { + //v = rnd() % 2; + v = rnd(); + } + // 1. Create large dictionary and store it in db + auto arr_ptr = std::make_unique(values); + auto &arr = *arr_ptr; + td::VectorQueue queue; + auto push = [&]() { + //LOG(ERROR) << "PUSH ROOT"; + auto begin_stats = kv->get_usage_stats(); + db->inc(arr.root()); + queue.push(arr.root()->get_hash()); + vm::CellStorer cell_storer(*kv); + db->commit(cell_storer); + auto end_stats = kv->get_usage_stats(); + LOG(ERROR) << end_stats - begin_stats; + db->set_loader(std::make_unique(kv)); + auto hash = arr.root()->get_hash(); + arr = CompactArray{size, db->load_root(hash.as_slice()).move_as_ok()}; + //LOG(ERROR) << "CELLS IN DB: " << kv->count("").move_as_ok(); + }; + auto pop = [&]() { + if (queue.empty()) { + return; + } + //LOG(ERROR) << "POP ROOT"; + auto begin_stats = kv->get_usage_stats(); + auto cell = db->load_cell(queue.pop().as_slice()).move_as_ok(); + db->dec(cell); + vm::CellStorer cell_storer(*kv); + db->commit(cell_storer); + auto end_stats = kv->get_usage_stats(); + db->set_loader(std::make_unique(kv)); + //LOG(ERROR) << end_stats - begin_stats; + //LOG(ERROR) << "CELLS IN DB: " << kv->count("").move_as_ok(); + }; + auto upd = [&] { + for (int i = 0; i < 20; i++) { + auto pos = rnd.fast(0, td::narrow_cast(size) - 1); + if (rnd() % 2) { + auto value = rnd() % 2; + arr.set(pos, value); + } else { + arr.get(pos); + } + } + }; + + //LOG(ERROR) << "Created compact array"; + push(); + pop(); + //CHECK(kv->count("").move_as_ok() == 0); + + // 2. Lets change first 20 keys and read last 20 keys + /* + for (size_t i = 0; i < 20 && i < size; i++) { + arr.set(i, rnd()); + } + */ + //arr.set(0, rnd()); + arr.set(size - 1, rnd()); + for (size_t i = 0; i < 20 && i < size; i++) { + arr.get(size - i - 1); + } + + // 3. And now commit diff with stats + push(); + push(); + upd(); + upd(); + push(); + push(); + upd(); + pop(); + pop(); + upd(); + push(); + push(); + while (!queue.empty()) { + pop(); + } + LOG(ERROR) << "CELLS IN DB: " << kv->count("").move_as_ok(); +} + +class BenchCellStorerMergeRefcntDiffs : public td::Benchmark { + public: + std::string get_description() const override { + return PSTRING() << "bench_cells_storer_merge_refcnt_diffs"; + } + + void run(int n) override { + auto cell = vm::CellBuilder().store_bytes(std::string(32, 'A')).finalize(); + auto left_update = CellStorer::serialize_refcnt_diffs(1); + auto right_update = CellStorer::serialize_refcnt_diffs(1); + for (int i = 0; i < n; i++) { + CellStorer::merge_refcnt_diffs(left_update, right_update); + } + } + + private: + size_t tn_{}; +}; +class BenchCellStorerMergeValueAndRefcntDiff : public td::Benchmark { + public: + std::string get_description() const override { + return PSTRING() << "bench_cells_storer_merge_value_and_refcnt_diffs"; + } + + void run(int n) override { + auto cell = vm::CellBuilder().store_bytes(std::string(32, 'A')).finalize(); + auto value = CellStorer::serialize_value(10, cell, false); + auto update = CellStorer::serialize_refcnt_diffs(1); + for (int i = 0; i < n; i++) { + CellStorer::merge_value_and_refcnt_diff(value, update); + } + } + + private: + size_t tn_{}; +}; +TEST(Bench, CellStorerMerge) { + bench(BenchCellStorerMergeRefcntDiffs()); + bench(BenchCellStorerMergeValueAndRefcntDiff()); +} + TEST(Cell, BocHands) { serialize_boc(CellBuilder{}.store_bytes("AAAAAAAA").finalize()); auto a = CellBuilder{}.store_bytes("abcd").store_ref(CellBuilder{}.store_bytes("???").finalize()).finalize(); @@ -2262,7 +2905,37 @@ TEST(TonDb, BocRespectsUsageCell) { ASSERT_STREQ(serialization, serialization_of_virtualized_cell); } -void test_dynamic_boc_respectes_usage_cell(vm::BocOptions options) { +TEST(UsageTree, ThreadSafe) { + size_t test_n = 100; + td::Random::Xorshift128plus rnd(123); + for (size_t test_i = 0; test_i < test_n; test_i++) { + auto cell = vm::gen_random_cell(rnd.fast(2, 100), rnd, false); + auto usage_tree = std::make_shared(); + auto usage_cell = vm::UsageCell::create(cell, usage_tree->root_ptr()); + std::ptrdiff_t threads_n = 1; // TODO: when CellUsageTree is thread safe, change it to 4 + auto barrier = std::barrier{threads_n}; + std::vector threads; + std::vector explorations(threads_n); + for (std::ptrdiff_t i = 0; i < threads_n; i++) { + threads.emplace_back([&, i = i]() { + barrier.arrive_and_wait(); + explorations[i] = vm::CellExplorer::random_explore(usage_cell, rnd); + }); + } + for (auto &thread : threads) { + thread.join(); + } + auto proof = vm::MerkleProof::generate(cell, usage_tree.get()); + auto virtualized_proof = vm::MerkleProof::virtualize(proof, 1); + for (auto &exploration : explorations) { + auto new_exploration = vm::CellExplorer::explore(virtualized_proof, exploration.ops); + ASSERT_EQ(exploration.log, new_exploration.log); + } + } +} + +/* +vm::DynamicBagOfCellsDb::Stats test_dynamic_boc_respects_usage_cell(vm::BocOptions options) { td::Random::Xorshift128plus rnd(options.seed); auto cell = vm::gen_random_cell(20, rnd, true); auto usage_tree = std::make_shared(); @@ -2283,11 +2956,14 @@ void test_dynamic_boc_respectes_usage_cell(vm::BocOptions options) { auto serialization_of_virtualized_cell = serialize_boc(virtualized_proof); auto serialization = serialize_boc(cell); ASSERT_STREQ(serialization, serialization_of_virtualized_cell); + vm::DynamicBagOfCellsDb::Stats stats; + return stats; } TEST(TonDb, DynamicBocRespectsUsageCell) { - vm::with_all_boc_options(test_dynamic_boc_respectes_usage_cell, 20); + vm::with_all_boc_options(test_dynamic_boc_respects_usage_cell, 20, true); } +*/ TEST(TonDb, LargeBocSerializer) { td::Random::Xorshift128plus rnd{123}; diff --git a/crypto/vm/cells/Cell.h b/crypto/vm/cells/Cell.h index a75371dbb7..e2b47ffc01 100644 --- a/crypto/vm/cells/Cell.h +++ b/crypto/vm/cells/Cell.h @@ -55,6 +55,7 @@ class Cell : public CellTraits { } // load interface + virtual td::Status set_data_cell(Ref &&data_cell) const = 0; virtual td::Result load_cell() const = 0; virtual Ref virtualize(VirtualizationParameters virt) const; virtual td::uint32 get_virtualization() const = 0; diff --git a/crypto/vm/cells/DataCell.cpp b/crypto/vm/cells/DataCell.cpp index e5f21b0130..5134123d8c 100644 --- a/crypto/vm/cells/DataCell.cpp +++ b/crypto/vm/cells/DataCell.cpp @@ -36,7 +36,8 @@ struct ArenaAllocator { T* obj = new (ptr) T(std::forward(args)...); return std::unique_ptr(obj); } -private: + + private: td::MutableSlice alloc_batch() { size_t batch_size = 1 << 20; auto batch = std::make_unique(batch_size); @@ -53,7 +54,7 @@ struct ArenaAllocator { return res; } }; -} +} // namespace std::unique_ptr DataCell::create_empty_data_cell(Info info) { if (use_arena) { ArenaAllocator allocator; diff --git a/crypto/vm/cells/DataCell.h b/crypto/vm/cells/DataCell.h index 6d3c845fc4..b39ee1d4b2 100644 --- a/crypto/vm/cells/DataCell.h +++ b/crypto/vm/cells/DataCell.h @@ -31,6 +31,9 @@ class DataCell : public Cell { static thread_local bool use_arena; DataCell(const DataCell& other) = delete; + DataCell(DataCell&& other) = delete; + DataCell& operator=(const DataCell& other) = delete; + DataCell& operator=(DataCell&& other) = delete; ~DataCell() override; static void store_depth(td::uint8* dest, td::uint16 depth) { @@ -126,6 +129,10 @@ class DataCell : public Cell { explicit DataCell(Info info); public: + td::Status set_data_cell(Ref&& data_cell) const override { + CHECK(get_hash() == data_cell->get_hash()); + return td::Status::OK(); + } td::Result load_cell() const override { return LoadedCell{Ref{this}, {}, {}}; } @@ -228,4 +235,3 @@ inline CellHash as_cell_hash(const Ref& cell) { } } // namespace vm - diff --git a/crypto/vm/cells/ExtCell.h b/crypto/vm/cells/ExtCell.h index 401bb04831..dbbd8575b7 100644 --- a/crypto/vm/cells/ExtCell.h +++ b/crypto/vm/cells/ExtCell.h @@ -65,6 +65,9 @@ class ExtCell : public Cell { bool is_loaded() const override { return CellView(this)->is_loaded(); } + Ref> get_prunned_cell() const { + return prunned_cell_.load(); + } private: mutable td::AtomicRef data_cell_; @@ -112,6 +115,23 @@ class ExtCell : public Cell { return CellView(this)->get_depth(level); } + td::Status set_data_cell(Ref&& new_data_cell) const override { + auto prunned_cell = prunned_cell_.load(); + if (prunned_cell.is_null()) { + auto old_data_cell = data_cell_.get_unsafe(); + DCHECK(old_data_cell); + TRY_STATUS(old_data_cell->check_equals_unloaded(new_data_cell)); + return td::Status::OK(); + } + + TRY_STATUS(prunned_cell->check_equals_unloaded(new_data_cell)); + if (data_cell_.store_if_empty(new_data_cell)) { + prunned_cell_.store({}); + get_thread_safe_counter_unloaded().add(-1); + } + return td::Status::OK(); + } + td::Result> load_data_cell() const { auto data_cell = data_cell_.get_unsafe(); if (data_cell) { diff --git a/crypto/vm/cells/PrunnedCell.h b/crypto/vm/cells/PrunnedCell.h index a58b245ccf..6e8b77093d 100644 --- a/crypto/vm/cells/PrunnedCell.h +++ b/crypto/vm/cells/PrunnedCell.h @@ -142,6 +142,10 @@ class PrunnedCell : public Cell { return info_.get_depth(get_storage())[get_level_mask().apply(level).get_hash_i()]; } + td::Status set_data_cell(Ref &&data_cell) const override { + return td::Status::OK(); + } + td::Result load_cell() const override { return td::Status::Error("Can't load prunned branch"); } diff --git a/crypto/vm/cells/UsageCell.h b/crypto/vm/cells/UsageCell.h index 3e6e88982a..978b91f763 100644 --- a/crypto/vm/cells/UsageCell.h +++ b/crypto/vm/cells/UsageCell.h @@ -36,6 +36,9 @@ class UsageCell : public Cell { return Ref{true, std::move(cell), std::move(tree_node), PrivateTag{}}; } + td::Status set_data_cell(Ref &&data_cell) const override { + return cell_->set_data_cell(std::move(data_cell)); + } // load interface td::Result load_cell() const override { TRY_RESULT(loaded_cell, cell_->load_cell()); diff --git a/crypto/vm/cells/VirtualCell.h b/crypto/vm/cells/VirtualCell.h index 02abc1c881..a75bdf9de3 100644 --- a/crypto/vm/cells/VirtualCell.h +++ b/crypto/vm/cells/VirtualCell.h @@ -37,6 +37,9 @@ class VirtualCell : public Cell { } // load interface + td::Status set_data_cell(Ref &&data_cell) const override { + return cell_->set_data_cell(std::move(data_cell)); + } td::Result load_cell() const override { TRY_RESULT(loaded_cell, cell_->load_cell()); loaded_cell.virt = loaded_cell.virt.apply(virt_); diff --git a/crypto/vm/db/CellHashTable.h b/crypto/vm/db/CellHashTable.h index 522c987be6..a389806380 100644 --- a/crypto/vm/db/CellHashTable.h +++ b/crypto/vm/db/CellHashTable.h @@ -40,6 +40,17 @@ class CellHashTable { return res; } + template + std::pair emplace(td::Slice hash, ArgsT &&...args) { + auto it = set_.find(hash); + if (it != set_.end()) { + return std::pair(const_cast(*it), false); + } + auto res = set_.emplace(std::forward(args)...); + CHECK(res.second); + return std::pair(const_cast(*res.first), res.second); + } + template void for_each(F &&f) { for (auto &info : set_) { @@ -64,7 +75,7 @@ class CellHashTable { size_t size() const { return set_.size(); } - InfoT* get_if_exists(td::Slice hash) { + InfoT *get_if_exists(td::Slice hash) { auto it = set_.find(hash); if (it != set_.end()) { return &const_cast(*it); diff --git a/crypto/vm/db/CellStorage.cpp b/crypto/vm/db/CellStorage.cpp index 06df461ef6..a07d85e870 100644 --- a/crypto/vm/db/CellStorage.cpp +++ b/crypto/vm/db/CellStorage.cpp @@ -17,14 +17,19 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "vm/db/CellStorage.h" + +#include "td/utils/Parser.h" #include "vm/db/DynamicBagOfCellsDb.h" #include "vm/boc.h" #include "td/utils/base64.h" #include "td/utils/tl_parsers.h" #include "td/utils/tl_helpers.h" +#include + namespace vm { namespace { + class RefcntCellStorer { public: RefcntCellStorer(td::int32 refcnt, const td::Ref &cell, bool as_boc) @@ -43,7 +48,9 @@ class RefcntCellStorer { storer.store_slice(data); return; } + CHECK(refcnt_ > 0); store(refcnt_, storer); + CHECK(cell_.not_null()) store(*cell_, storer); for (unsigned i = 0; i < cell_->size_refs(); i++) { auto cell = cell_->get_ref(i); @@ -91,6 +98,7 @@ class RefcntCellParser { stored_boc_ = true; parse(refcnt, parser); } + CHECK(refcnt > 0); if (!need_data_) { return; } @@ -159,6 +167,9 @@ td::Result CellLoader::load(td::Slice hash, bool need_da DCHECK(get_status == KeyValue::GetStatus::NotFound); return LoadResult{}; } + if (serialized.empty()) { + return LoadResult{}; + } TRY_RESULT(res, load(hash, serialized, need_data, ext_cell_creator)); if (on_load_callback_) { on_load_callback_(res); @@ -198,6 +209,7 @@ td::Result CellLoader::load_refcnt(td::Slice hash) { if (res.refcnt_ == -1) { parse(res.refcnt_, parser); } + CHECK(res.refcnt_ > 0); TRY_STATUS(parser.get_status()); return res; } @@ -216,4 +228,77 @@ std::string CellStorer::serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc) { return kv_.set(cell->get_hash().as_slice(), serialize_value(refcnt, cell, as_boc)); } + +td::Status CellStorer::merge(td::Slice hash, td::int32 refcnt_diff) { + return kv_.merge(hash, serialize_refcnt_diffs(refcnt_diff)); +} + +void CellStorer::merge_value_and_refcnt_diff(std::string &left, td::Slice right) { + if (right.empty()) { + return; + } + CHECK(left.size() > 4); + CHECK(right.size() == 4); + + td::int32 left_refcnt = td::as(left.data()); + size_t shift = 0; + if (left_refcnt == -1) { + CHECK(left.size() >= 8); + left_refcnt = td::as(left.data() + 4); + shift = 4; + } + td::int32 right_refcnt_diff = td::as(right.data()); + td::int32 new_refcnt = left_refcnt + right_refcnt_diff; + CHECK(new_refcnt > 0); + td::as(left.data() + shift) = new_refcnt; +} +void CellStorer::merge_refcnt_diffs(std::string &left, td::Slice right) { + if (right.empty()) { + return; + } + if (left.empty()) { + left = right.str(); + return; + } + CHECK(left.size() == 4); + CHECK(right.size() == 4); + td::int32 left_refcnt_diff = td::as(left.data()); + td::int32 right_refcnt_diff = td::as(right.data()); + td::int32 total_refcnt_diff = left_refcnt_diff + right_refcnt_diff; + td::as(left.data()) = total_refcnt_diff; +} + +std::string CellStorer::serialize_refcnt_diffs(td::int32 refcnt_diff) { + TD_PERF_COUNTER(cell_store_refcnt_diff); + std::string s(4, 0); + td::as(s.data()) = refcnt_diff; + return s; +} + +td::Status CellStorer::apply_diff(const Diff &diff) { + switch (diff.type) { + case Diff::Set: + return kv_.set(diff.key.as_slice(), diff.value); + case Diff::Erase: + return kv_.erase(diff.key.as_slice()); + case Diff::Merge: + return kv_.merge(diff.key.as_slice(), diff.value); + default: + UNREACHABLE(); + } +} +td::Status CellStorer::apply_meta_diff(const MetaDiff &diff) { + switch (diff.type) { + case MetaDiff::Set: + CHECK(diff.key.size() != CellTraits::hash_bytes); + CHECK(!diff.value.empty()); + return kv_.set(diff.key, diff.value); + case MetaDiff::Erase: + CHECK(diff.key.size() != CellTraits::hash_bytes); + CHECK(diff.value.empty()); + return kv_.erase(diff.key); + default: + UNREACHABLE(); + } +} } // namespace vm diff --git a/crypto/vm/db/CellStorage.h b/crypto/vm/db/CellStorage.h index cabd7fdcbd..ca32a8007d 100644 --- a/crypto/vm/db/CellStorage.h +++ b/crypto/vm/db/CellStorage.h @@ -51,6 +51,9 @@ class CellLoader { td::Result load(td::Slice hash, bool need_data, ExtCellCreator &ext_cell_creator); static td::Result load(td::Slice hash, td::Slice value, bool need_data, ExtCellCreator &ext_cell_creator); td::Result load_refcnt(td::Slice hash); // This only loads refcnt_, cell_ == null + KeyValueReader &key_value_reader() const { + return *reader_; + } private: std::shared_ptr reader_; @@ -62,8 +65,28 @@ class CellStorer { CellStorer(KeyValue &kv); td::Status erase(td::Slice hash); td::Status set(td::int32 refcnt, const td::Ref &cell, bool as_boc); + td::Status merge(td::Slice hash, td::int32 refcnt_diff); + + static void merge_value_and_refcnt_diff(std::string &value, td::Slice right); + static void merge_refcnt_diffs(std::string &left, td::Slice right); + static std::string serialize_refcnt_diffs(td::int32 refcnt_diff); + static std::string serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc); + struct Diff { + enum Type { Set, Erase, Merge } type{Set}; + CellHash key; + std::string value{}; + }; + td::Status apply_diff(const Diff &diff); + + struct MetaDiff { + enum Type { Set, Erase } type{Set}; + std::string key; + std::string value{}; + }; + td::Status apply_meta_diff(const MetaDiff &diff); + private: KeyValue &kv_; }; diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index 093037583b..d6731b0390 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -66,19 +66,27 @@ struct CellInfo { struct Eq { using is_transparent = void; // Pred to use - bool operator()(const CellInfo &info, const CellInfo &other_info) const { return info.key() == other_info.key();} - bool operator()(const CellInfo &info, td::Slice hash) const { return info.key().as_slice() == hash;} - bool operator()(td::Slice hash, const CellInfo &info) const { return info.key().as_slice() == hash;} - + bool operator()(const CellInfo &info, const CellInfo &other_info) const { + return info.key() == other_info.key(); + } + bool operator()(const CellInfo &info, td::Slice hash) const { + return info.key().as_slice() == hash; + } + bool operator()(td::Slice hash, const CellInfo &info) const { + return info.key().as_slice() == hash; + } }; struct Hash { using is_transparent = void; // Pred to use using transparent_key_equal = Eq; - size_t operator()(td::Slice hash) const { return cell_hash_slice_hash(hash); } - size_t operator()(const CellInfo &info) const { return cell_hash_slice_hash(info.key().as_slice());} + size_t operator()(td::Slice hash) const { + return cell_hash_slice_hash(hash); + } + size_t operator()(const CellInfo &info) const { + return cell_hash_slice_hash(info.key().as_slice()); + } }; }; - bool operator<(const CellInfo &a, td::Slice b) { return a.key().as_slice() < b; } @@ -99,6 +107,36 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { return get_cell_info_lazy(level_mask, hash, depth).cell; } + td::Result>> meta_get_all(size_t max_count) const override { + std::vector> result; + auto s = loader_->key_value_reader().for_each_in_range("desc", "desd", + [&](const td::Slice &key, const td::Slice &value) { + if (result.size() >= max_count) { + return td::Status::Error("COUNT_LIMIT"); + } + if (td::begins_with(key, "desc") && key.size() != 32) { + result.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + if (s.message() == "COUNT_LIMIT") { + s = td::Status::OK(); + } + TRY_STATUS(std::move(s)); + return result; + } + td::Result meta_get(td::Slice key, std::string &value) override { + return loader_->key_value_reader().get(key, value); + } + td::Status meta_set(td::Slice key, td::Slice value) override { + meta_diffs_.push_back( + CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Set, .key = key.str(), .value = value.str()}); + return td::Status::OK(); + } + td::Status meta_erase(td::Slice key) override { + meta_diffs_.push_back(CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Erase, .key = key.str()}); + return td::Status::OK(); + } td::Result> load_cell(td::Slice hash) override { auto info = hash_table_.get_if_exists(hash); if (info && info->sync_with_db) { @@ -198,21 +236,29 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat if (is_prepared_for_commit()) { return td::Status::OK(); } + td::PerfWarningTimer timer_dfs_new_cells_in_db("dfs_new_cells_in_db"); for (auto &new_cell : to_inc_) { auto &new_cell_info = get_cell_info(new_cell); dfs_new_cells_in_db(new_cell_info); } + timer_dfs_new_cells_in_db.reset(); + td::PerfWarningTimer timer_dfs_new_cells("dfs_new_cells"); for (auto &new_cell : to_inc_) { auto &new_cell_info = get_cell_info(new_cell); dfs_new_cells(new_cell_info); } + timer_dfs_new_cells.reset(); + td::PerfWarningTimer timer_dfs_old_cells("dfs_old_cells"); for (auto &old_cell : to_dec_) { auto &old_cell_info = get_cell_info(old_cell); dfs_old_cells(old_cell_info); } + timer_dfs_old_cells.reset(); + td::PerfWarningTimer timer_save_diff_prepare("save_diff_prepare"); save_diff_prepare(); + timer_save_diff_prepare.reset(); to_inc_.clear(); to_dec_.clear(); @@ -222,6 +268,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat td::Status commit(CellStorer &storer) override { prepare_commit(); + td::PerfWarningTimer times_save_diff("save diff", 0.01); save_diff(storer); // Some elements are erased from hash table, to keep it small. // Hash table is no longer represents the difference between the loader and @@ -249,7 +296,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat celldb_compress_depth_ = value; } - vm::ExtCellCreator& as_ext_cell_creator() override { + vm::ExtCellCreator &as_ext_cell_creator() override { return *this; } @@ -259,6 +306,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat std::vector> to_dec_; CellHashTable hash_table_; std::vector visited_; + std::vector meta_diffs_; Stats stats_diff_; td::uint32 celldb_compress_depth_{0}; @@ -269,8 +317,9 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat class SimpleExtCellCreator : public ExtCellCreator { public: - explicit SimpleExtCellCreator(std::shared_ptr cell_db_reader) : - cell_db_reader_(std::move(cell_db_reader)) {} + explicit SimpleExtCellCreator(std::shared_ptr cell_db_reader) + : cell_db_reader_(std::move(cell_db_reader)) { + } td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { TRY_RESULT(ext_cell, DynamicBocExtCell::create(PrunnedCellInfo{level_mask, hash, depth}, @@ -279,7 +328,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return std::move(ext_cell); } - std::vector>& get_created_cells() { + std::vector> &get_created_cells() { return created_cells_; } @@ -382,8 +431,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat } bool not_in_db = false; - for_each( - info, [¬_in_db, this](auto &child_info) { not_in_db |= !dfs_new_cells_in_db(child_info); }, false); + for_each(info, [¬_in_db, this](auto &child_info) { not_in_db |= !dfs_new_cells_in_db(child_info); }, false); if (not_in_db) { CHECK(!info.in_db); @@ -441,6 +489,10 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat for (auto info_ptr : visited_) { save_cell(*info_ptr, storer); } + for (auto meta_diff : meta_diffs_) { + storer.apply_meta_diff(meta_diff); + } + meta_diffs_.clear(); visited_.clear(); } @@ -558,6 +610,8 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat } auto res = r_res.move_as_ok(); if (res.status != CellLoader::LoadResult::Ok) { + LOG_CHECK(info.cell.not_null()) << "Trying to load nonexistent cell from db " + << CellHash::from_slice(hash).to_hex(); break; } info.cell = std::move(res.cell()); @@ -651,7 +705,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat CellHashTable cells_; - std::queue load_queue_; + std::queue load_queue_; td::uint32 active_load_ = 0; td::uint32 max_parallel_load_ = 4; }; @@ -814,11 +868,10 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat pca_state_->promise_.set_result(td::Unit()); pca_state_ = {}; } - }; } // namespace -std::unique_ptr DynamicBagOfCellsDb::create() { +std::unique_ptr DynamicBagOfCellsDb::create(CreateV1Options) { return std::make_unique(); } } // namespace vm diff --git a/crypto/vm/db/DynamicBagOfCellsDb.h b/crypto/vm/db/DynamicBagOfCellsDb.h index 62864ad979..82028f3fe4 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.h +++ b/crypto/vm/db/DynamicBagOfCellsDb.h @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "td/db/KeyValue.h" #include "vm/cells.h" #include "td/utils/Slice.h" @@ -49,13 +50,23 @@ class CellDbReader { class DynamicBagOfCellsDb { public: virtual ~DynamicBagOfCellsDb() = default; + + virtual td::Result>> meta_get_all(size_t max_count) const = 0; + virtual td::Result meta_get(td::Slice key, std::string &value) = 0; + virtual td::Status meta_set(td::Slice key, td::Slice value) = 0; + virtual td::Status meta_erase(td::Slice key) = 0; + virtual td::Result> load_cell(td::Slice hash) = 0; virtual td::Result> load_root(td::Slice hash) = 0; virtual td::Result> load_root_thread_safe(td::Slice hash) const = 0; + virtual td::Result>> load_known_roots() const { + return std::vector>(); + } struct Stats { td::int64 roots_total_count{0}; td::int64 cells_total_count{0}; td::int64 cells_total_size{0}; + td::NamedStats named_stats; std::vector> custom_stats; void apply_diff(const Stats &diff) { roots_total_count += diff.roots_total_count; @@ -64,6 +75,20 @@ class DynamicBagOfCellsDb { CHECK(roots_total_count >= 0); CHECK(cells_total_count >= 0); CHECK(cells_total_size >= 0); + named_stats.apply_diff(diff.named_stats); + } + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const Stats &stats) { + sb << "STATS\n"; + for (auto &p : stats.custom_stats) { + sb << "\t" << p.first << "\t" << p.second << "\n"; + } + for (auto &p : stats.named_stats.stats_int) { + sb << "\t" << p.first << "\t" << p.second << "\n"; + } + for (auto &p : stats.named_stats.stats_str) { + sb << "\t" << p.first << "\t" << p.second << "\n"; + } + return sb; } }; virtual void inc(const Ref &old_root) = 0; @@ -72,7 +97,7 @@ class DynamicBagOfCellsDb { virtual td::Status prepare_commit() = 0; virtual Stats get_stats_diff() = 0; virtual td::Result get_stats() { - return td::Status::Error("Not implemented"); + return Stats{}; } virtual td::Status commit(CellStorer &) = 0; virtual std::shared_ptr get_cell_db_reader() = 0; @@ -83,25 +108,49 @@ class DynamicBagOfCellsDb { virtual void set_celldb_compress_depth(td::uint32 value) = 0; virtual vm::ExtCellCreator &as_ext_cell_creator() = 0; - static std::unique_ptr create(); + class AsyncExecutor { + public: + virtual ~AsyncExecutor() { + } + virtual void execute_async(std::function f) = 0; + virtual void execute_sync(std::function f) = 0; + virtual std::string describe() const { + return "AsyncExecutor"; + } + }; + + struct CreateV1Options { + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const CreateV1Options &options) { + return sb << "V1{}"; + } + }; + static std::unique_ptr create(CreateV1Options = {}); + + struct CreateV2Options { + size_t extra_threads{std::thread::hardware_concurrency()}; + std::shared_ptr executor{}; + size_t cache_ttl_max{2000}; + size_t cache_size_max{1000000}; + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const CreateV2Options &options) { + return sb << "V2{extra_threads=" << options.extra_threads << ", cache_ttl_max=" << options.cache_ttl_max + << ", cache_size_max=" << options.cache_size_max << "}"; + } + }; + static std::unique_ptr create_v2(CreateV2Options options); struct CreateInMemoryOptions { size_t extra_threads{std::thread::hardware_concurrency()}; bool verbose{true}; - // Allocated DataCels will never be deleted + // Allocated DataCells will never be deleted bool use_arena{false}; // Almost no overhead in memory during creation, but will scan database twice bool use_less_memory_during_creation{true}; - }; - static std::unique_ptr create_in_memory(td::KeyValueReader *kv, CreateInMemoryOptions options); - - class AsyncExecutor { - public: - virtual ~AsyncExecutor() { + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const CreateInMemoryOptions &options) { + return sb << "InMemory{extra_threads=" << options.extra_threads << ", use_arena=" << options.use_arena + << ", use_less_memory_during_creation=" << options.use_less_memory_during_creation << "}"; } - virtual void execute_async(std::function f) = 0; - virtual void execute_sync(std::function f) = 0; }; + static std::unique_ptr create_in_memory(td::KeyValueReader *kv, CreateInMemoryOptions options); virtual void load_cell_async(td::Slice hash, std::shared_ptr executor, td::Promise> promise) = 0; diff --git a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp new file mode 100644 index 0000000000..eff74e214d --- /dev/null +++ b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp @@ -0,0 +1,1511 @@ +#include "vm/db/DynamicBagOfCellsDb.h" +#include "vm/db/CellStorage.h" +#include "vm/db/CellHashTable.h" + +#include "vm/cells/ExtCell.h" + +#include "td/utils/base64.h" +#include "td/utils/format.h" +#include "td/utils/ThreadSafeCounter.h" +#include "td/utils/misc.h" +#include "validator/validator.h" + +#include "vm/cellslice.h" + +#include + +namespace vm { +namespace { + +// Very stupid Vector/MpmcQueue +template +struct TsVector { + TsVector() { + first_block_size_ = 64; + blocks_[0].data.resize(first_block_size_); + blocks_[0].is_ready = true; + } + TsVector(std::vector base) { + first_block_size_ = base.size(); + blocks_[0].data = std::move(base); + blocks_[0].is_ready = true; + } + struct Block { + std::mutex mutex; + std::atomic is_ready{false}; + std::vector data; + }; + T &at(size_t i) { + td::uint64 j = i / first_block_size_; + td::int32 hb = 63 - td::count_leading_zeroes64(j); // hb = -1 if j=0, else hb>=0 + + // If j=0, hb<0, so hb>>31 = -1 => mask=0 + // If j>0, hb>=0, so hb>>31=0 => mask=~0 (all ones) + td::uint64 mask = ~(td::uint64)(hb >> 31); + + size_t block_i = hb + 1; + uint64_t shift = hb & 63ULL; + uint64_t start = ((1ULL << shift) * first_block_size_) & mask; + size_t pos_in_block = i - start; + auto &block = blocks_[block_i]; + if (block.is_ready.load(std::memory_order_acquire)) { + return block.data.at(pos_in_block); + } + + std::unique_lock lock(block.mutex); + if (block.is_ready.load(std::memory_order_acquire)) { + return block.data.at(pos_in_block); + } + block.resize(start); + block.is_ready.store(true, std::memory_order_release); + return block.data.at(pos_in_block); + } + template + void push_back(S &&value) { + at(end_.fetch_add(1, std::memory_order_relaxed)) = std::forward(value); + } + T pop_front() { + auto pos = begin_.fetch_add(1, std::memory_order_relaxed); + while (pos >= end_.load(std::memory_order_acquire)) { + // This may (or may not) use too much CPU + td::this_thread::yield(); + } + return std::move(at(pos)); + } + size_t size() const { + return end_.load(); + } + + std::array blocks_; + size_t first_block_size_{0}; + std::atomic begin_{0}; + std::atomic end_{0}; +}; +struct CellInfo; + +class CellDbReaderExt; +struct DynamicBocExtCellExtra { + std::shared_ptr reader; +}; + +class DynamicBocCellLoader { + public: + static td::Result> load_data_cell(const ExtCell &cell, + const DynamicBocExtCellExtra &extra); +}; +using DynamicBocExtCell = ExtCell; + +class CellDbReaderExt : public CellDbReader { + public: + virtual td::Result> load_ext_cell(Ref cell) = 0; +}; + +td::Result> DynamicBocCellLoader::load_data_cell(const DynamicBocExtCell &cell, + const DynamicBocExtCellExtra &extra) { + return extra.reader->load_ext_cell(Ref(&cell)); +} + +#define S(x) \ + td::NamedThreadSafeCounter::CounterRef x { \ + nc.get_counter(#x) \ + } + +struct CacheStats { + td::NamedThreadSafeCounter nc; + S(load_cell_ext); + S(load_cell_ext_cache_hits); + S(load_cell_sync); + S(load_cell_sync_cache_hits); + S(load_cell_async); + S(load_cell_async_cache_hits); + S(ext_cells); + S(ext_cells_load); + S(ext_cells_load_cache_hits); + + S(kv_read_found); + S(kv_read_not_found); + + S(sync_with_db); + S(sync_with_db_only_ref); + S(load_cell_no_cache); +}; + +struct CommitStats { + td::NamedThreadSafeCounter nc; + + S(to_inc); + S(to_dec); + + S(gather_new_cells_calls); + S(gather_new_cells_calls_it); + S(update_parents_calls); + S(update_parents_calls_it); + S(dec_calls); + S(dec_calls_it); + + S(new_cells); + S(new_cells_leaves); + + S(new_cells_loaded_not_in_db); + S(new_cells_loaded_in_db); + S(new_cells_not_in_db_fast); + + S(dec_loaded); + S(dec_to_zero); + + S(changes_loaded); + + // new diff logic + S(diff_zero); + S(diff_full); + S(diff_erase); + S(diff_ref_cnt); + + // old full data logic + S(inc_save); + S(inc_save_full); + S(inc_save_only_ref_cnt); + S(inc_new_cell); + S(inc_just_ref_cnt); + + S(dec_save); + S(dec_save_full); + S(dec_save_only_refcnt); + S(dec_save_erase); + S(dec_erase_cell); + S(dec_just_ref_cnt); +}; + +template +struct AtomicPod { + T load() const { + while (true) { + if (auto res = try_read_stable()) { + return res->second; + } + } + } + + template + std::pair update(F &&f) { + while (true) { + auto res = try_read_stable(); + if (!res) { + continue; + } + auto [before, old_data] = *res; + + auto o_new_data = f(old_data); + if (!o_new_data) { + return {old_data, false}; + } + + if (!lock_.compare_exchange_weak(before, before + 1, std::memory_order_acq_rel, std::memory_order_relaxed)) { + continue; + } + + data_ = *o_new_data; // relaxed store inside lock + lock_.fetch_add(1, std::memory_order_release); + return {*o_new_data, true}; + } + } + + private: + mutable std::atomic lock_{0}; + T data_{}; + + std::optional> try_read_stable() const { + auto before = lock_.load(std::memory_order_acquire); + if (before % 2 == 1) { + return std::nullopt; + } + T temp = data_; // relaxed read is ok, checked by versioning + auto after = lock_.load(std::memory_order_acquire); + if (after != before) { + return std::nullopt; + } + return std::make_pair(before, temp); + } +}; + +struct InDbInfo { + std::vector parents; + std::atomic pending_children{0}; + std::atomic maybe_in_db{true}; + std::atomic visited_in_gather_new_cells{false}; +}; +td::StringBuilder &operator<<(td::StringBuilder &sb, const InDbInfo &info) { + sb << "mb_in_db:" << info.maybe_in_db.load() << " chld_n:" << info.pending_children + << " prnt_n:" << info.parents.size(); + return sb; +} +struct CellInfo { + struct State { + // db_ref_cnt and in_db are correct + bool sync_with_db{false}; + + // ignore if sync_with_db is false + td::int32 db_ref_cnt{0}; + td::int32 db_refcnt_fixup{0}; + + // if true - cell is definitely in db + // if false - we know that cell is not in db only is sync_with_db=true + bool in_db{false}; + + // diff to be applied + }; + AtomicPod state; + std::atomic ref_cnt_diff{0}; + + std::atomic visited{false}; + td::unique_ptr in_db_info_ptr; + std::mutex mutex; + + // Could be AtomicRef, but is am not sure that it is worth it + const Ref cell; + + explicit CellInfo(Ref cell) : cell(std::move(cell)) { + } + + InDbInfo &in_db_info() { + return *in_db_info_ptr; + } + const InDbInfo &in_db_info() const { + return *in_db_info_ptr; + } + InDbInfo &in_db_info_create() { // NOT thread safe + if (!in_db_info_ptr) { + in_db_info_ptr = td::make_unique(); + } + return in_db_info(); + } + InDbInfo &in_db_info_create(CellInfo *parent) { // Thread Safe + std::unique_lock lock(mutex); + if (!in_db_info_ptr) { + in_db_info_ptr = td::make_unique(); + } + auto &res = *in_db_info_ptr; + if (parent != nullptr) { + res.parents.emplace_back(parent); + } + lock.unlock(); + return res; + } + void in_db_info_destroy() { + in_db_info_ptr = nullptr; + } + td::int32 inc_ref_cnt() { + return ref_cnt_diff.fetch_add(1, std::memory_order_relaxed) + 1; + } + td::int32 dec_ref_cnt() { + return ref_cnt_diff.fetch_sub(1, std::memory_order_relaxed) - 1; + } + td::int32 get_ref_cnt_diff() const { + return ref_cnt_diff.load(std::memory_order_relaxed); + } + + void set_not_in_db() { + state.update([&](State state) -> std::optional { + if (state.sync_with_db) { + CHECK(state.db_ref_cnt == 0); + CHECK(!state.in_db); + return {}; + } + state.sync_with_db = true; + state.in_db = false; + state.db_ref_cnt = 0; + return state; + }); + } + void set_in_db() { + state.update([&](State state) -> std::optional { + if (state.sync_with_db) { + //LOG_CHECK(state.in_db) << *this; + return {}; + } + state.in_db = true; + return state; + }); + } + void synced_with_db(td::int32 db_ref_cnt) { + state.update([&](State state) -> std::optional { + if (state.sync_with_db) { + CHECK(state.in_db); + CHECK(state.db_ref_cnt == db_ref_cnt); + return {}; + } + state.in_db = true; + state.db_ref_cnt = db_ref_cnt; + return state; + }); + } + bool visit() { + return !visited.exchange(true); + } + void on_written_to_db() { + auto diff = ref_cnt_diff.exchange(0); + state.update([&](State state) -> std::optional { + if (diff == 0) { + return {}; + } + if (state.sync_with_db) { + state.db_ref_cnt += diff; + CHECK(state.db_ref_cnt >= 0); + state.in_db = state.db_ref_cnt > 0; + } else { + CHECK(diff > 0); + state.in_db = true; + state.db_refcnt_fixup += diff; + } + return state; + }); + } + + td::Result> get_data_cell() { + TRY_RESULT(loaded_cell, cell->load_cell()); + return loaded_cell.data_cell; + } + Cell::Hash key() const { + return cell->get_hash(); + } + bool operator<(const CellInfo &other) const { + return key() < other.key(); + } + + struct Eq { + using is_transparent = void; // Pred to use + bool operator()(const CellInfo &info, const CellInfo &other_info) const { + return info.key() == other_info.key(); + } + bool operator()(const CellInfo &info, td::Slice hash) const { + return info.key().as_slice() == hash; + } + bool operator()(td::Slice hash, const CellInfo &info) const { + return info.key().as_slice() == hash; + } + }; + struct Hash { + using is_transparent = void; // Pred to use + using transparent_key_equal = Eq; + size_t operator()(td::Slice hash) const { + return cell_hash_slice_hash(hash); + } + size_t operator()(const CellInfo &info) const { + return cell_hash_slice_hash(info.key().as_slice()); + } + }; +}; +td::StringBuilder &operator<<(td::StringBuilder &sb, const CellInfo &info) { + if (info.cell->is_loaded()) { + auto data_cell = info.cell->load_cell().move_as_ok().data_cell; + vm::CellSlice cs(vm::NoVm{}, data_cell); + sb << data_cell->get_hash().to_hex().substr(0, 8) << " refs:" << data_cell->size_refs() + << " data:" << cs.data_bits().to_hex(cs.size()) << " data_ptr=" << data_cell.get() << " data_ref_cnt(" + << data_cell->get_refcnt() << ")"; + } else { + sb << info.cell->get_hash().to_hex().substr(0, 8); + } + auto state = info.state.load(); + sb << " " << &info; + sb << "\n\tin_db=" << state.in_db << " sync_with_db=" << state.sync_with_db + << " ref_cnt_diff=" << info.get_ref_cnt_diff() << " db_ref_cnt=" << state.db_ref_cnt + << " db_ref_cnt_fixup=" << state.db_refcnt_fixup; + if (state.sync_with_db) { + sb << " REFS(" << info.get_ref_cnt_diff() + state.db_ref_cnt << ")"; + } + if (info.in_db_info_ptr) { + sb << " " << info.in_db_info(); + } + sb << " visited=" << info.visited.load(); + return sb; +} + +struct ExecutorOptions { + size_t extra_threads_n{0}; + std::shared_ptr async_executor; +}; +template +class ExecutorImpl { + public: + ExecutorImpl(ExecutorOptions options) : options_(options) { + } + ExecutorOptions options_; + using InputData = std::vector>; + using OutputData = std::vector>; + struct InputChunk { + td::Span infos; + size_t begin{}; + size_t end{}; + }; + + template + OutputData process(const InputData &data, const F &process_task_f) { + if (options_.extra_threads_n > 0) { + return process_parallel(data, process_task_f); + } else { + return process_sequential(data, process_task_f); + } + } + template + struct SingleThreadWorker { + const F &process_task_f; + mutable std::vector results{}; + void add_task(InputT input) const { + process_task_f(input, *this); + } + void add_result(OutputT output) const { + results.push_back(output); + } + }; + template + OutputData process_sequential(const InputData &data, const F &process_task_f) { + auto w = SingleThreadWorker{process_task_f}; + for (auto &chunk : data) { + for (auto &info : chunk) { + process_task_f(info, w); + } + } + + return {std::move(w.results)}; + } + + template + struct Shared; + + template + struct Worker { + size_t worker_i{}; + std::shared_ptr> shared; + + void add_task(InputT input) const { + shared->delay_or_process_task(input, *this); + } + void add_result(OutputT value) const { + shared->add_result(value, worker_i); + } + void loop() const { + shared->loop(*this); + } + }; + + template + struct Shared { + Shared(size_t workers_n, const InputData &input_data, const ProcessTaskF &process_task_f) + : input_chunks(prepare_input_chunks(input_data)) + , workers_n(workers_n) + , input_size(input_chunks.empty() ? 0 : input_chunks.back().end) + , batch_size(std::clamp(input_size / workers_n / 4, size_t(1), size_t(128))) + , process_task_f(process_task_f) { + } + + const std::vector input_chunks; + + const size_t workers_n{0}; + const size_t input_size{0}; + const size_t batch_size{128}; + + const ProcessTaskF &process_task_f; + + // Position in input + std::atomic next_input_i{0}; + + // Shared queue + // Probably a simpler queue would also work fine + td::MpmcQueue mpmc_queue{workers_n}; + using Waiter = td::MpmcSleepyWaiter; + Waiter waiter; + std::atomic mpmc_queue_size{workers_n}; // guard + + // Output vectors + struct ThreadData { + std::vector output; + char pad[TD_CONCURRENCY_PAD - sizeof(output)]; + }; + std::vector thread_data{workers_n}; + + auto prepare_input_chunks(const InputData &input_data) { + std::vector chunks; + for (auto &chunk : input_data) { + size_t prev_end = chunks.empty() ? 0 : chunks.back().end; + chunks.push_back({.infos = td::as_span(chunk), .begin = prev_end, .end = prev_end + chunk.size()}); + } + return chunks; + } + + void delay_or_process_task(InputT input, const Worker &worker) { + // if there is enough tasks in queue, we continue recursion + if (mpmc_queue_size.load(std::memory_order_acquire) > 256) { + process_task_f(input, worker); + } else { + mpmc_queue_size.fetch_add(1, std::memory_order_acq_rel); + mpmc_queue.push(input, worker.worker_i); + waiter.notify(); + } + } + + void add_result(OutputT result, size_t worker_i) { + thread_data[worker_i].output.push_back(std::move(result)); + } + + void process_initial_input(const Worker &worker) { + size_t input_chunk_i = 0; + while (true) { + auto begin_i = next_input_i.fetch_add(batch_size, std::memory_order_relaxed); + auto end_i = begin_i + batch_size; + if (begin_i >= input_size) { + break; + } + for (size_t i = begin_i; i < end_i && i < input_size; i++) { + while (input_chunks[input_chunk_i].end <= i) { + input_chunk_i++; + } + auto offset = i - input_chunks[input_chunk_i].begin; + auto task = input_chunks[input_chunk_i].infos[offset]; + process_task_f(task, worker); + } + } + } + + void on_processed_task_from_queue(size_t worker_i) { + if (mpmc_queue_size.fetch_add(-1, std::memory_order_acq_rel) == 1) { + for (size_t i = 0; i < workers_n; i++) { + mpmc_queue.push(nullptr, worker_i); + waiter.notify(); + } + } + } + + void process_queue(const Worker &worker) { + on_processed_task_from_queue(worker.worker_i); + + Waiter::Slot slot; + waiter.init_slot(slot, td::narrow_cast(worker.worker_i)); + + while (true) { + InputT input{}; + if (mpmc_queue.try_pop(input, worker.worker_i)) { + waiter.stop_wait(slot); + if (!input) { + break; + } + process_task_f(input, worker); + on_processed_task_from_queue(worker.worker_i); + } else { + waiter.wait(slot); + } + } + } + void loop(const Worker &worker) { + process_initial_input(worker); + process_queue(worker); + } + void finish() const { + CHECK(mpmc_queue_size == 0); + } + }; + + template + OutputData process_parallel(const InputData &input_data, const F &process_task_f) { + const size_t workers_n = options_.extra_threads_n + 1; + auto shared = std::make_shared>(workers_n, input_data, process_task_f); + + CHECK(workers_n >= 1); + for (size_t i = 0; i < workers_n; i++) { + auto to_run = [worker = Worker{.worker_i = i, .shared = shared}] { worker.loop(); }; + + if (i + 1 == workers_n) { + to_run(); + } else if (options_.async_executor) { + options_.async_executor->execute_async(std::move(to_run)); + } else { + // NB: td::thread, NOT std::thread + td::thread(std::move(to_run)).detach(); + } + } + shared->finish(); + return td::transform(shared->thread_data, [](auto &&x) { return std::move(x.output); }); + } +}; +struct Executor { + Executor(ExecutorOptions options = {}) : options_(options) { + } + + template + auto operator()(const std::vector> &data, const F &process_task_f) { + return ExecutorImpl(options_).process(data, process_task_f); + } + + private: + ExecutorOptions options_; +}; + +// Thread safe storage for CellInfo +// Will be used by everybody as shared cache. Yes there is some overhead, but it don't want to create other hash table +struct CellInfoStorage { + public: + // All methods are thead safe + // All CellInfo pointers lives as long as CellInfoStorage + + // returns CellInfo, only if it is already exists + CellInfo *get_cell_info(td::Slice hash) { + return lock(hash)->hash_table.get_if_exists(hash); + } + + CellInfo &create_cell_info_from_db(Ref data_cell, td::int32 ref_cnt) { + auto &info = create_cell_info_from_data_cell(std::move(data_cell)); + info.synced_with_db(ref_cnt); + return info; + } + + // Creates CellInfo from data_cell, or updates existing CellInfo if is not yet loaded + CellInfo &create_cell_info_from_data_cell(Ref cell) { + CHECK(cell.not_null()); + CHECK(cell->is_loaded()); + + auto hash = cell->get_hash(); + auto [info, created] = lock(hash.as_slice())->hash_table.emplace(hash.as_slice(), std::move(cell)); + + if (!created) { + info.cell->set_data_cell(std::move(cell)); + } + return info; + } + + // Creates CellInfo from cell. If cell is loaded, it will be used to rewrite or udpate current cell + CellInfo &create_cell_info(Ref cell, CellDbReaderExt *from_reader, CacheStats &stats) { + if (cell->is_loaded()) { + return create_cell_info_from_data_cell(cell->load_cell().move_as_ok().data_cell); + } + + bool our_ext_cell = false; + auto ext_cell = dynamic_cast(cell.get()); + if (ext_cell) { + auto prunned_cell = ext_cell->get_prunned_cell(); + if (prunned_cell.not_null()) { + our_ext_cell = prunned_cell->get_extra().reader.get() == from_reader; + } + our_ext_cell = true; + } else if (!cell->is_loaded()) { + // if we cached cell from OTHER db is good idea to drop it ASAP + force_drop_cache_.store(true, std::memory_order_relaxed); + } + + auto hash = cell->get_hash(); + auto [info, created] = lock(hash.as_slice())->hash_table.emplace(hash.as_slice(), std::move(cell)); + if (our_ext_cell) { + stats.ext_cells_load.inc(); + if (info.cell->is_loaded()) { + stats.ext_cells_load_cache_hits.inc(); + } + info.set_in_db(); + } + return info; + } + + void dump() { + LOG(ERROR) << "===========BEGIN DUMP==========="; + for (auto &bucket : buckets_) { + std::lock_guard guard(bucket.mutex); + bucket.hash_table.for_each([&](auto &info) { LOG(INFO) << info; }); + } + LOG(ERROR) << "===========END DUMP==========="; + } + + size_t cache_size() { + size_t res = 0; + for (auto &bucket : buckets_) { + std::lock_guard guard(bucket.mutex); + res += bucket.hash_table.size(); + } + return res; + } + bool force_drop_cache() { + return force_drop_cache_.load(std::memory_order_relaxed); + } + + private: + struct Bucket { + std::mutex mutex; + CellHashTable hash_table; + }; + constexpr static size_t buckets_n = 8192; + std::array bucket_; + + struct Unlock { + void operator()(Bucket *bucket) const { + bucket->mutex.unlock(); + } + }; + std::array buckets_{}; + std::atomic force_drop_cache_{false}; + + std::unique_ptr lock(Bucket &bucket) { + bucket.mutex.lock(); + return std::unique_ptr(&bucket); + } + std::unique_ptr lock(td::Slice key) { + auto hash = td::as(key.substr(16, 8).ubegin()); + auto bucket_i = hash % buckets_n; + return lock(buckets_[bucket_i]); + } +}; + +class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { + public: + explicit DynamicBagOfCellsDbImplV2(CreateV2Options options) : options_(options) { + get_thread_safe_counter().inc(); + // LOG(ERROR) << "Constructor called for DynamicBagOfCellsDbImplV2"; + } + ~DynamicBagOfCellsDbImplV2() { + // LOG(ERROR) << "Destructor called for DynamicBagOfCellsDbImplV2"; + get_thread_safe_counter().add(-1); + + if (cell_db_reader_) { + cell_db_reader_->drop_cache(); + } + } + + td::Result>> meta_get_all(size_t max_count) const override { + CHECK(meta_db_fixup_.empty()); + std::vector> result; + auto s = cell_db_reader_->key_value_reader().for_each_in_range( + "desc", "desd", [&](const td::Slice &key, const td::Slice &value) { + if (result.size() >= max_count) { + return td::Status::Error("COUNT_LIMIT"); + } + if (td::begins_with(key, "desc") && key.size() != 32) { + result.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + if (s.message() == "COUNT_LIMIT") { + s = td::Status::OK(); + } + TRY_STATUS(std::move(s)); + return result; + } + td::Result meta_get(td::Slice key, std::string &value) override { + auto it = meta_db_fixup_.find(key); + if (it != meta_db_fixup_.end()) { + if (it->second.empty()) { + return KeyValue::GetStatus::NotFound; + } + value = it->second; + return KeyValue::GetStatus::Ok; + } + return cell_db_reader_->key_value_reader().get(key, value); + } + td::Status meta_set(td::Slice key, td::Slice value) override { + meta_diffs_.push_back( + CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Set, .key = key.str(), .value = value.str()}); + return td::Status::OK(); + } + td::Status meta_erase(td::Slice key) override { + meta_diffs_.push_back(CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Erase, .key = key.str()}); + return td::Status::OK(); + } + td::Result> load_cell(td::Slice hash) override { + CHECK(cell_db_reader_); + return cell_db_reader_->load_cell(hash); + } + td::Result> load_root(td::Slice hash) override { + return load_cell(hash); + } + td::Result> load_root_thread_safe(td::Slice hash) const override { + // TODO: it is better to use AtomicRef, or atomic shared pointer + // But to use AtomicRef we need a little refactoring + // And std::atomic> is still unsupported by clang + std::unique_lock lock(atomic_cell_db_reader_mutex_); + auto reader = atomic_cell_db_reader_; + lock.unlock(); + if (!reader) { + return td::Status::Error("Empty reader"); + } + return reader->load_cell(hash); + } + void load_cell_async(td::Slice hash, std::shared_ptr executor, + td::Promise> promise) override { + CHECK(cell_db_reader_); + return cell_db_reader_->load_cell_async(hash, std::move(executor), std::move(promise)); + } + void prepare_commit_async(std::shared_ptr executor, td::Promise promise) override { + auto promise_ptr = std::make_shared>(std::move(promise)); + executor->execute_async([this, promise_ptr = std::move(promise_ptr)] { + prepare_commit(); + promise_ptr->set_value(td::Unit()); + }); + } + + void inc(const Ref &cell) override { + if (cell.is_null()) { + return; + } + if (cell->get_virtualization() != 0) { + return; + } + to_inc_.push_back(cell); + } + void dec(const Ref &cell) override { + if (cell.is_null()) { + return; + } + if (cell->get_virtualization() != 0) { + return; + } + to_dec_.push_back(cell); + } + + bool is_prepared_for_commit() { + return to_inc_.empty() && to_dec_.empty(); + } + + Stats get_stats_diff() override { + return {}; + } + + td::Status prepare_commit() override { + if (is_prepared_for_commit()) { + return td::Status::OK(); + } + // NB: we don't use options.executor, because it is prone to deadlocks. We need extra_threads_n threads + // available for blocking + Executor executor{{.extra_threads_n = options_.extra_threads, .async_executor = {}}}; + // calculate in_db for all vertices reachable from to_inc_ roots + // - for ext cells we already know they are in db + // - calculate in_db up from leaves + // - if at least one child is not in db, then the cell is definitely not in db + // - so in best case only leaves will be loaded from db + // - this is optional step. All other logic must work in any case + // - only already loaded cells are loaded from db + + stats_.to_inc.add(to_inc_.size()); + stats_.to_dec.add(to_dec_.size()); + + std::vector> visited_cells; + auto add_visited_cells = [&](std::vector> new_visited_cells) { + for (auto &x : new_visited_cells) { + visited_cells.push_back(std::move(x)); + } + }; + + std::vector> new_cells_leaves; + { + td::PerfWarningTimer timer("celldb_v2: gather_new_cells"); + std::vector prepared_to_inc; + std::vector visited_roots; + for (auto &cell : to_inc_) { + auto &info = cell_db_reader_->cell_info(cell); + if (info.inc_ref_cnt() == 1 && info.visit()) { + visited_roots.push_back(&info); + } + if (info.state.load().in_db) { + continue; + } + auto &in_db_info = info.in_db_info_create(nullptr); + if (!in_db_info.visited_in_gather_new_cells.exchange(true)) { + prepared_to_inc.push_back(&info); + } + } + new_cells_leaves = + executor({std::move(prepared_to_inc)}, [&](CellInfo *info, auto &worker) { gather_new_cells(info, worker); }); + visited_cells.push_back(std::move(visited_roots)); + } + + // LOG(WARNING) << "new_cells_leaves: " << new_cells_leaves.size(); + { + td::PerfWarningTimer timer("celldb_v2: update_parents"); + add_visited_cells( + executor({std::move(new_cells_leaves)}, [&](CellInfo *info, auto &worker) { update_parents(info, worker); })); + } + { + td::PerfWarningTimer timer("dec"); + std::vector prepared_to_dec; + for (auto &cell : to_dec_) { + auto &info = cell_db_reader_->cell_info(cell); + prepared_to_dec.push_back(&info); + } + add_visited_cells( + executor({std::move(prepared_to_dec)}, [&](CellInfo *info, auto &worker) { dec_cell(info, worker); })); + } + + td::PerfWarningTimer timer_serialize("celldb_v2: save_diff_serialize", 0.01); + // LOG(INFO) << "threads_n = " << options_.extra_threads + 1; + diff_chunks_ = executor.operator()( + visited_cells, [&](CellInfo *info, auto &worker) { serialize_diff(info, worker); }); + timer_serialize.reset(); + + { + td::PerfWarningTimer timer("celldb_v2: clear"); + to_inc_.clear(); + to_dec_.clear(); + } + + //cell_db_reader_->dump(); + return td::Status::OK(); + } + + td::Status commit(CellStorer &storer) override { + prepare_commit(); + save_diff(storer); + // We DON'T delete entries from cache, so cache actually represents diff with snapshot in reader + // But we don't want took keep old snapshot forever + LOG_IF(ERROR, dbg) << "clear cell_db_reader"; + //cell_db_reader_->dump(); + //TODO: call drop_cache reliably via rtti + + constexpr bool always_drop_cache = false; + if (always_drop_cache) { + td::PerfWarningTimer timer("celldb_v2: reset reader"); + cell_db_reader_->drop_cache(); + cache_stats_.apply_diff(cell_db_reader_->get_stats()); + cache_stats_.stats_int["commits"] += 1; + cell_db_reader_ = {}; + // keep atomic reader, to it will be reused + } + return td::Status::OK(); + } + + std::shared_ptr get_cell_db_reader() override { + CHECK(cell_db_reader_); + return cell_db_reader_; + } + + td::Status set_loader(std::unique_ptr loader) override { + if (cell_db_reader_) { + auto cache_size = cell_db_reader_->cache_size(); + bool force_drop_cache = cell_db_reader_->force_drop_cache(); + if (loader && cache_size < options_.cache_size_max && cell_db_reader_ttl_ < options_.cache_ttl_max && + !force_drop_cache) { + // keep cache + cell_db_reader_ttl_++; + return td::Status::OK(); + } + + td::PerfWarningTimer timer(PSTRING() << "celldb_v2: reset reader, TTL=" << cell_db_reader_ttl_ << "/" + << options_.cache_ttl_max << ", cache_size=" << cache_size + << ", force_drop_cache=" << force_drop_cache); + cache_stats_.apply_diff(cell_db_reader_->get_stats()); + cell_db_reader_->drop_cache(); + cell_db_reader_ = {}; + meta_db_fixup_ = {}; + cell_db_reader_ttl_ = 0; + } + + if (loader) { + cell_db_reader_ = std::make_shared(std::move(loader)); + cell_db_reader_ttl_ = 0; + } + + { + std::lock_guard guard(atomic_cell_db_reader_mutex_); + atomic_cell_db_reader_ = cell_db_reader_; + } + return td::Status::OK(); + } + + void set_celldb_compress_depth(td::uint32 value) override { + CHECK(value == 0); + } + + vm::ExtCellCreator &as_ext_cell_creator() override { + CHECK(cell_db_reader_); + return *cell_db_reader_; + } + td::Result get_stats() override { + auto ps = stats_.nc.get_stats().with_prefix("storage_"); + ps.apply_diff(cache_stats_.with_prefix("cache_cum_")); + if (cell_db_reader_) { + ps.apply_diff(cell_db_reader_->get_stats().with_prefix("cache_now_")); + ps.apply_diff(cell_db_reader_->get_stats().with_prefix("cache_cum_")); + } + Stats res; + res.named_stats = std::move(ps); + res.named_stats.stats_int["cache.size"] = cell_db_reader_ ? cell_db_reader_->cache_size() : 0; + res.named_stats.stats_int["cache.size_max"] = options_.cache_size_max; + res.named_stats.stats_int["cache.ttl"] = cell_db_reader_ttl_; + res.named_stats.stats_int["cache.ttl_max"] = options_.cache_ttl_max; + return res; + } + + private: + static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() { + static auto res = td::NamedThreadSafeCounter::get_default().get_counter("DynamicBagOfCellsDb"); + return res; + } + + class CellDbReaderImpl : public CellDbReaderExt, + public ExtCellCreator, + public std::enable_shared_from_this { + public: + explicit CellDbReaderImpl(std::unique_ptr cell_loader) : cell_loader_(std::move(cell_loader)) { + } + + size_t cache_size() const { + // NOT thread safe + if (internal_storage_) { + return internal_storage_->cache_size(); + } + return 0; + } + bool force_drop_cache() const { + // NOT thread safe + if (internal_storage_) { + return internal_storage_->force_drop_cache(); + } + return false; + } + void drop_cache() { + // NOT thread safe + internal_storage_.reset(); + } + + td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { + // thread safe function + stats_.ext_cells.inc(); + TRY_RESULT(ext_cell, DynamicBocExtCell::create(PrunnedCellInfo{level_mask, hash, depth}, + DynamicBocExtCellExtra{shared_from_this()})); + + return ext_cell; + } + CellInfo *register_ext_cell_inner(Ref ext_cell, CellInfoStorage &storage) { + auto &info = storage.create_cell_info(std::move(ext_cell), this, stats_); + return &info; + } + + void load_cell_async(td::Slice hash, std::shared_ptr executor, td::Promise> promise) { + // thread safe function + stats_.load_cell_async.inc(); + auto maybe_cell = load_cell_fast_path(hash, false, nullptr); + if (maybe_cell.not_null()) { + stats_.load_cell_async_cache_hits.inc(); + return promise.set_value(std::move(maybe_cell)); + } + auto promise_ptr = std::make_shared>>(std::move(promise)); + + executor->execute_async( + [self = shared_from_this(), promise_ptr = std::move(promise_ptr), hash = CellHash::from_slice(hash)]() { + promise_ptr->set_result(self->load_cell(hash.as_slice())); + }); + } + + td::Result> load_cell(td::Slice hash) override { + // thread safe function + stats_.load_cell_sync.inc(); + bool loaded{false}; + auto maybe_cell = load_cell_fast_path(hash, true, &loaded); + if (maybe_cell.not_null()) { + if (!loaded) { + stats_.load_cell_sync_cache_hits.inc(); + } + return maybe_cell; + } + return load_cell_slow_path(hash); + } + + td::Result> load_ext_cell(Ref ext_cell) override { + // thread safe function. + // Called by external cell + stats_.load_cell_ext.inc(); + auto storage = weak_storage_.lock(); + if (!storage) { + TRY_RESULT(load_result, load_cell_no_cache(ext_cell->get_hash().as_slice())); + return load_result.cell_; + } + // we delayed registering ext cell till this moment + auto cell_info = register_ext_cell_inner(std::move(ext_cell), *storage); + + CHECK(cell_info != nullptr); // currently all ext_cells are registered in cache + if (!cell_info->cell->is_loaded()) { + sync_with_db(*cell_info, true); + CHECK(cell_info->cell->is_loaded()); // critical, better to fail + } else { + stats_.load_cell_ext_cache_hits.inc(); + } + return cell_info->cell->load_cell().move_as_ok().data_cell; + } + + CellInfo &cell_info(Ref cell) { + // thread safe function, but called only by DB + CHECK(internal_storage_) + return internal_storage_->create_cell_info(std::move(cell), this, stats_); + } + + std::pair sync_with_db(CellInfo &info, bool need_data) { + // thread safe function, but called only by DB + auto effective_need_data = need_data; + if (info.cell->is_loaded()) { + effective_need_data = false; + } + return info.state.update([&](CellInfo::State state) -> std::optional { + if (state.sync_with_db) { + return {}; + } + stats_.sync_with_db.inc(); + if (!effective_need_data) { + stats_.sync_with_db_only_ref.inc(); + } + auto load_result = + cell_loader_->load(info.cell->get_hash().as_slice(), effective_need_data, *this).move_as_ok(); + + state.sync_with_db = true; + if (load_result.status == CellLoader::LoadResult::NotFound) { + CHECK(state.in_db == false); + CHECK(state.db_ref_cnt == 0); + stats_.kv_read_not_found.inc(); + return state; + } + stats_.kv_read_found.inc(); + + state.in_db = true; + state.db_ref_cnt = load_result.refcnt() + state.db_refcnt_fixup; + if (load_result.cell().not_null()) { + info.cell->set_data_cell(std::move(load_result.cell())); + } + CHECK(!need_data || info.cell->is_loaded()); + return state; + }); + } + + void dump() { + internal_storage_->dump(); + } + + td::NamedStats get_stats() const { + return stats_.nc.get_stats(); + } + td::KeyValueReader &key_value_reader() { + return cell_loader_->key_value_reader(); + } + + private: + static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() { + static auto res = td::NamedThreadSafeCounter::get_default().get_counter("DynamicBagOfCellsDbLoader"); + return res; + } + std::shared_ptr internal_storage_{std::make_shared()}; + std::weak_ptr weak_storage_{internal_storage_}; + std::unique_ptr cell_loader_; + CacheStats stats_; + + Ref load_cell_fast_path(td::Slice hash, bool may_block, bool *loaded) { + auto storage = weak_storage_.lock(); + if (!storage) { + return {}; + } + auto cell_info = storage->get_cell_info(hash); + if (cell_info != nullptr) { + if (!cell_info->cell->is_loaded()) { + if (may_block) { + if (loaded) { + *loaded = true; + } + CHECK(cell_info->state.load().in_db); + sync_with_db(*cell_info, true); + CHECK(cell_info->cell->is_loaded()); + } else { + return {}; + } + } + return cell_info->cell->load_cell().move_as_ok().data_cell; + } + return {}; + } + td::Result load_cell_no_cache(td::Slice hash) { + stats_.load_cell_no_cache.inc(); + TRY_RESULT(load_result, cell_loader_->load(hash, true, *this)); + if (load_result.status == CellLoader::LoadResult::NotFound) { + stats_.kv_read_not_found.inc(); + return td::Status::Error("Cell load failed: not in db"); + } + stats_.kv_read_found.inc(); + return load_result; + } + td::Result> load_cell_slow_path(td::Slice hash) { + TRY_RESULT(load_result, load_cell_no_cache(hash)); + auto storage = weak_storage_.lock(); + if (!storage) { + return load_result.cell_; + } + auto &cell_info = storage->create_cell_info_from_db(std::move(load_result.cell()), load_result.refcnt()); + return cell_info.cell->load_cell().move_as_ok().data_cell; + } + }; + + CreateV2Options options_; + std::vector> to_inc_; + std::vector> to_dec_; + std::vector> diff_chunks_; + std::vector meta_diffs_; + std::map> meta_db_fixup_; + + mutable std::mutex atomic_cell_db_reader_mutex_; + std::shared_ptr atomic_cell_db_reader_; + + std::shared_ptr cell_db_reader_; + size_t cell_db_reader_ttl_{0}; + td::NamedStats cache_stats_; + CommitStats stats_; + bool dbg{false}; + + template + void gather_new_cells(CellInfo *info, WorkerT &worker) { + stats_.gather_new_cells_calls.inc(); + do { + // invariant: info is not in DB; with created in_db_info + // we enter into each root only once + stats_.gather_new_cells_calls_it.inc(); + stats_.new_cells.inc(); + auto &in_db_info = info->in_db_info(); + + CellSlice cs(vm::NoVm{}, info->cell); // ensure cell is loaded + CellInfo *prev_child_info = nullptr; + while (cs.have_refs()) { + auto *child_info = &cell_db_reader_->cell_info(cs.fetch_ref()); + auto child_state = child_info->state.load(); + + if (child_state.in_db) { + LOG_IF(INFO, dbg) << "gather_new_cells: IN DB\n\tchld: " << *child_info; + continue; + } + + auto &child_in_db_info = child_info->in_db_info_create(info); + in_db_info.pending_children.fetch_add(1, std::memory_order_relaxed); + + if (child_in_db_info.visited_in_gather_new_cells.exchange(true)) { + continue; + } + + if (prev_child_info != nullptr) { + worker.add_task(prev_child_info); + } + prev_child_info = child_info; + } + LOG_IF(INFO, dbg) << "gather_new_cells: NOT IN DB\n\t" << *info; + if (in_db_info.pending_children.load(std::memory_order_relaxed) == 0) { + worker.add_result(info); + stats_.new_cells_leaves.inc(); + LOG_IF(WARNING, dbg) << "gather_new_cells: ADD LEAVE\n\t" << *info; + } + info = prev_child_info; + } while (info != nullptr); + } + + template + void update_parents(CellInfo *info, const WorkerT &worker) { + stats_.update_parents_calls.inc(); + size_t it = 0; + do { + stats_.update_parents_calls_it.inc(); + it++; + //LOG(INFO) << "update_parents: it=" << it << "\n\t"; + auto &in_db_info = info->in_db_info(); + bool in_db = false; + if (in_db_info.maybe_in_db.load(std::memory_order_relaxed)) { + auto [state, loaded] = cell_db_reader_->sync_with_db(*info, false); + in_db = state.in_db; + if (in_db) { + stats_.new_cells_loaded_in_db.inc(); + } else { + stats_.new_cells_loaded_not_in_db.inc(); + } + } else { + stats_.new_cells_not_in_db_fast.inc(); + info->set_not_in_db(); + } + LOG_IF(INFO, dbg) << "update_parents: it=" << it << "\n\t" << *info; + + CellInfo *prev_parent{nullptr}; + for (auto &parent : in_db_info.parents) { + auto &parent_in_db_info = parent->in_db_info(); + if (!in_db) { + parent_in_db_info.maybe_in_db.store(false, std::memory_order_relaxed); + } + if (parent_in_db_info.pending_children.fetch_sub(1, std::memory_order_release) == 1) { + if (prev_parent) { + worker.add_task(prev_parent); + } + prev_parent = parent; + } + } + if (!in_db) { + CellSlice cs(vm::NoVm{}, info->cell); + while (cs.have_refs()) { + auto child = cs.fetch_ref(); + auto &child_info = cell_db_reader_->cell_info(std::move(child)); + if (child_info.inc_ref_cnt() == 1 && child_info.visit()) { + worker.add_result(&child_info); + } + } + } + info->in_db_info_destroy(); + info = prev_parent; + } while (info); + } + + template + void dec_cell(CellInfo *info, WorkerT &worker) { + stats_.dec_calls.inc(); + + while (true) { + stats_.dec_calls_it.inc(); + if (info->visit()) { + worker.add_result(info); + } + auto ref_cnt_diff = info->dec_ref_cnt(); + if (ref_cnt_diff > 0) { + LOG_IF(INFO, dbg) << "NOT DEC" + << "\n\t" << info; + break; + } + auto state = info->state.load(); + if (ref_cnt_diff == 0 && state.in_db) { + LOG_IF(INFO, dbg) << "NOT DEC (in_db) " + << "\n\t" << info; + break; + } + if (!state.sync_with_db) { + state = cell_db_reader_->sync_with_db(*info, true).first; + stats_.dec_loaded.inc(); + CHECK(ref_cnt_diff == 0 || state.in_db); + } + auto ref_cnt = state.db_ref_cnt + ref_cnt_diff; + if (ref_cnt > 0) { + LOG_IF(INFO, dbg) << "DEC " << ref_cnt << "\n\t" << info; + } else { + LOG_IF(ERROR, dbg) << "DEC " << ref_cnt << "\n\t" << info; + } + CHECK(ref_cnt >= 0); + if (ref_cnt > 0) { + break; + } + stats_.dec_to_zero.inc(); + CellSlice cs(vm::NoVm{}, info->cell); + if (!cs.have_refs()) { + break; + } + while (cs.size_refs() > 1) { + worker.add_task(&cell_db_reader_->cell_info(cs.fetch_ref())); + } + info = &cell_db_reader_->cell_info(cs.fetch_ref()); + } + } + + template + void serialize_diff(CellInfo *info, Worker &worker) { + info->visited.store(false, std::memory_order_relaxed); + auto ref_cnt_diff = info->get_ref_cnt_diff(); + if (ref_cnt_diff == 0) { + stats_.diff_zero.inc(); + return; + } + + bool merge_supported = true; + if (merge_supported) { + auto state = info->state.load(); + if (ref_cnt_diff < 0) { + CHECK(state.sync_with_db); + /* + if (state.db_ref_cnt + ref_cnt_diff == 0) { + LOG(ERROR) << "DEC ERASE " << info->cell->get_hash().to_hex(); + } else { + LOG(ERROR) << "DEC MERGE " << info->cell->get_hash().to_hex() << *info; + } + */ + } + if (ref_cnt_diff < 0 && state.sync_with_db && state.db_ref_cnt + ref_cnt_diff == 0) { + // Erase is better than Merge+CompactionFilter + // So I see no reason for CompactionFilter at all + worker.add_result({.type = CellStorer::Diff::Erase, .key = info->cell->get_hash()}); + stats_.diff_erase.inc(); + } else { + bool with_data = ref_cnt_diff > 0 && !state.in_db; + if (with_data) { + CHECK(state.sync_with_db); + auto data_cell = info->cell->load_cell().move_as_ok().data_cell; + stats_.diff_full.inc(); + worker.add_result({.type = CellStorer::Diff::Set, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_value(ref_cnt_diff + state.db_ref_cnt, data_cell, false)}); + } else { + stats_.diff_ref_cnt.inc(); + worker.add_result({.type = CellStorer::Diff::Merge, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_refcnt_diffs(ref_cnt_diff)}); + } + } + info->on_written_to_db(); + return; + } + + auto state = info->state.load(); + if (!state.sync_with_db) { + stats_.changes_loaded.inc(); + state = cell_db_reader_->sync_with_db(*info, true).first; + } + CHECK(state.sync_with_db); + auto new_ref_cnt = ref_cnt_diff + state.db_ref_cnt; + + if (ref_cnt_diff < 0) { + stats_.dec_save.inc(); + if (new_ref_cnt == 0) { + stats_.dec_erase_cell.inc(); + + LOG_IF(ERROR, dbg) << "DEC ERASE " << *info; + worker.add_result({.type = CellStorer::Diff::Erase, .key = info->cell->get_hash()}); + stats_.dec_save_erase.inc(); + } else { + stats_.dec_just_ref_cnt.inc(); + + LOG_IF(ERROR, dbg) << "DEC REFCNT " << *info; + CHECK(info->cell->is_loaded()); + worker.add_result( + {.type = CellStorer::Diff::Set, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, false)}); + stats_.dec_save_full.inc(); + } + } else { + stats_.inc_save.inc(); + CHECK(info->cell->is_loaded()); + if (state.db_ref_cnt == 0) { + stats_.inc_new_cell.inc(); + LOG_IF(ERROR, dbg) << "INC CREATE " << *info; + } else { + stats_.inc_just_ref_cnt.inc(); + LOG_IF(ERROR, dbg) << "INC REFCNT " << *info; + } + + worker.add_result( + {.type = CellStorer::Diff::Set, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, false)}); + stats_.inc_save_full.inc(); + } + } + + void save_diff(CellStorer &storer) { + td::PerfWarningTimer timer("celldb_v2: save_diff"); + td::PerfWarningTimer timer_store_to_db("celldb_v2: save_diff_store_to_db", 0.01); + // Have no idea hot to parallelize this in case of rocksdb + for (auto &diffs : diff_chunks_) { + for (auto &diff : diffs) { + storer.apply_diff(diff).ensure(); + } + } + for (auto &meta_diff : meta_diffs_) { + meta_db_fixup_[meta_diff.key] = meta_diff.value; + storer.apply_meta_diff(meta_diff).ensure(); + } + timer_store_to_db.reset(); + td::PerfWarningTimer timer_clear("celldb_v2: save_diff_clear"); + diff_chunks_.clear(); + meta_diffs_.clear(); + timer_clear.reset(); + } +}; +} // namespace + +std::unique_ptr DynamicBagOfCellsDb::create_v2(CreateV2Options options) { + return std::make_unique(options); +} +} // namespace vm diff --git a/crypto/vm/db/InMemoryBagOfCellsDb.cpp b/crypto/vm/db/InMemoryBagOfCellsDb.cpp index 03cad0934e..e43cfde4e0 100644 --- a/crypto/vm/db/InMemoryBagOfCellsDb.cpp +++ b/crypto/vm/db/InMemoryBagOfCellsDb.cpp @@ -413,6 +413,7 @@ class CellStorage { size_t dense_ht_size = 0; size_t new_ht_size = 0; for_each_bucket(0, [&](auto bucket_id, CellBucket &bucket) { + // TODO: this leads to CE when use_dense_hash_map == false dense_ht_capacity += bucket.infos_.dense_ht_values_.size(); dense_ht_size += bucket.infos_.dense_ht_size_; new_ht_capacity += bucket.infos_.new_ht_.bucket_count(); @@ -468,6 +469,14 @@ class CellStorage { } return td::Status::Error("not found"); } + td::Result>> load_known_roots_local() const { + auto lock = local_access_.lock(); + std::vector> result; + for (auto &root : roots_) { + result.emplace_back(root); + } + return result; + } td::Result> load_root_shared(const CellHash &hash) const { std::lock_guard lock(root_mutex_); if (auto it = roots_.find(hash); it != roots_.end()) { @@ -620,7 +629,7 @@ class CellStorage { sb << "\n\t" << key << "=" << value; } LOG_IF(ERROR, desc_count != 0 && desc_count != stats.roots_total_count + 1) - << "desc<> keys count is " << desc_count << " wich is different from roots count " << stats.roots_total_count; + << "desc<> keys count is " << desc_count << " which is different from roots count " << stats.roots_total_count; LOG_IF(WARNING, verbose) << P << "done in " << full_timer.elapsed() << "\n\troots_count=" << stats.roots_total_count << "\n\t" << desc_count << "\n\tcells_count=" << stats.cells_total_count @@ -757,15 +766,84 @@ class CellStorage { } }; +class MetaStorage { + public: + explicit MetaStorage(std::vector> values) + : meta_(std::move_iterator(values.begin()), std::move_iterator(values.end())) { + for (auto &p : meta_) { + CHECK(p.first.size() != CellTraits::hash_bytes); + } + } + std::vector> meta_get_all(size_t max_count) const { + std::vector> res; + for (const auto &[k, v] : meta_) { + if (res.size() >= max_count) { + break; + } + res.emplace_back(k, v); + } + return res; + } + KeyValue::GetStatus meta_get(td::Slice key, std::string &value) const { + auto lock = local_access_.lock(); + auto it = meta_.find(key.str()); + if (it == meta_.end()) { + return KeyValue::GetStatus::NotFound; + } + value = it->second; + return KeyValue::GetStatus::Ok; + } + void meta_set(td::Slice key, td::Slice value) { + auto lock = local_access_.lock(); + meta_[key.str()] = value.str(); + meta_diffs_.push_back( + CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Set, .key = key.str(), .value = value.str()}); + } + void meta_erase(td::Slice key) { + auto lock = local_access_.lock(); + meta_.erase(key.str()); + meta_diffs_.push_back(CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Erase, .key = key.str()}); + } + std::vector extract_diffs() { + auto lock = local_access_.lock(); + return std::move(meta_diffs_); + } + + private: + mutable UniqueAccess local_access_; + std::unordered_map meta_; + std::vector meta_diffs_; +}; + class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { public: - explicit InMemoryBagOfCellsDb(td::unique_ptr storage) : storage_(std::move(storage)) { + explicit InMemoryBagOfCellsDb(td::unique_ptr storage, td::unique_ptr meta_storage) + : storage_(std::move(storage)), meta_storage_(std::move(meta_storage)) { + } + + td::Result>> meta_get_all(size_t max_count) const override { + return meta_storage_->meta_get_all(max_count); + } + td::Result meta_get(td::Slice key, std::string &value) override { + CHECK(key.size() != CellTraits::hash_bytes); + return meta_storage_->meta_get(key, value); + } + td::Status meta_set(td::Slice key, td::Slice value) override { + meta_storage_->meta_set(key, value); + return td::Status::OK(); + } + td::Status meta_erase(td::Slice key) override { + meta_storage_->meta_erase(key); + return td::Status::OK(); } td::Result> load_cell(td::Slice hash) override { return storage_->load_cell(CellHash::from_slice(hash)); } + td::Result>> load_known_roots() const override { + return storage_->load_known_roots_local(); + } td::Result> load_root(td::Slice hash) override { return storage_->load_root_local(CellHash::from_slice(hash)); } @@ -798,29 +876,37 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { TRY_STATUS(prepare_commit()); } + td::PerfWarningTimer times_save_diff("save diff"); Stats diff; CHECK(to_dec_.empty()); - for (auto &it : info_) { - auto &info = it.second; + for (auto &info : info_) { if (info.diff_refcnt == 0) { continue; } auto refcnt = td::narrow_cast(static_cast(info.db_refcnt) + info.diff_refcnt); - CHECK(refcnt >= 0); + LOG_CHECK(refcnt >= 0) << info.db_refcnt << " + " << info.diff_refcnt; if (refcnt > 0) { - cell_storer.set(refcnt, info.cell, false); + if (info.db_refcnt == 0) { + TRY_STATUS(cell_storer.set(refcnt, info.cell, false)); + } else { + TRY_STATUS(cell_storer.merge(info.cell->get_hash().as_slice(), info.diff_refcnt)); + } storage_->set(refcnt, info.cell); if (info.db_refcnt == 0) { diff.cells_total_count++; diff.cells_total_size += static_cast(info.cell->get_storage_size()); } } else { - cell_storer.erase(info.cell->get_hash().as_slice()); + TRY_STATUS(cell_storer.erase(info.cell->get_hash().as_slice())); storage_->erase(info.cell->get_hash()); diff.cells_total_count--; diff.cells_total_size -= static_cast(info.cell->get_storage_size()); } } + auto meta_diffs = meta_storage_->extract_diffs(); + for (const auto &meta_diff : meta_diffs) { + TRY_STATUS(cell_storer.apply_meta_diff(meta_diff)); + } storage_->apply_stats_diff(diff); info_ = {}; return td::Status::OK(); @@ -872,13 +958,39 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { private: td::unique_ptr storage_; + td::unique_ptr meta_storage_; struct Info { - td::int32 db_refcnt{0}; - td::int32 diff_refcnt{0}; + mutable td::int32 db_refcnt{0}; + mutable td::int32 diff_refcnt{0}; Ref cell; + vm::CellHash key() const { + return cell->get_hash(); + } + struct Eq { + using is_transparent = void; // Pred to use + bool operator()(const Info &info, const Info &other_info) const { + return info.key() == other_info.key(); + } + bool operator()(const Info &info, td::Slice hash) const { + return info.key().as_slice() == hash; + } + bool operator()(td::Slice hash, const Info &info) const { + return info.key().as_slice() == hash; + } + }; + struct Hash { + using is_transparent = void; // Pred to use + using transparent_key_equal = Eq; + size_t operator()(td::Slice hash) const { + return cell_hash_slice_hash(hash); + } + size_t operator()(const Info &info) const { + return cell_hash_slice_hash(info.key().as_slice()); + } + }; }; - td::HashMap info_; + td::HashSet info_; std::unique_ptr loader_; std::vector> to_inc_; @@ -886,13 +998,13 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { Ref do_inc(Ref cell) { auto cell_hash = cell->get_hash(); - if (auto it = info_.find(cell_hash); it != info_.end()) { - CHECK(it->second.diff_refcnt != std::numeric_limits::max()); - it->second.diff_refcnt++; - return it->second.cell; + if (auto it = info_.find(cell_hash.as_slice()); it != info_.end()) { + CHECK(it->diff_refcnt != std::numeric_limits::max()); + it->diff_refcnt++; + return it->cell; } if (auto o_info = storage_->get_info(cell_hash)) { - info_.emplace(cell_hash, Info{.db_refcnt = o_info->db_refcnt, .diff_refcnt = 1, .cell = o_info->cell}); + info_.emplace(Info{.db_refcnt = o_info->db_refcnt, .diff_refcnt = 1, .cell = o_info->cell}); return std::move(o_info->cell); } @@ -905,21 +1017,21 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { } auto res = cb.finalize(cs.is_special()); CHECK(res->get_hash() == cell_hash); - info_.emplace(cell_hash, Info{.db_refcnt = 0, .diff_refcnt = 1, .cell = res}); + info_.emplace(Info{.db_refcnt = 0, .diff_refcnt = 1, .cell = res}); return res; } void do_dec(Ref cell) { auto cell_hash = cell->get_hash(); - auto it = info_.find(cell_hash); + auto it = info_.find(cell_hash.as_slice()); if (it != info_.end()) { - CHECK(it->second.diff_refcnt != std::numeric_limits::min()); - --it->second.diff_refcnt; + CHECK(it->diff_refcnt != std::numeric_limits::min()); + --it->diff_refcnt; } else { auto info = *storage_->get_info(cell_hash); - it = info_.emplace(cell_hash, Info{.db_refcnt = info.db_refcnt, .diff_refcnt = -1, .cell = info.cell}).first; + it = info_.emplace(Info{.db_refcnt = info.db_refcnt, .diff_refcnt = -1, .cell = info.cell}).first; } - if (it->second.diff_refcnt + it->second.db_refcnt != 0) { + if (it->diff_refcnt + it->db_refcnt != 0) { return; } CellSlice cs(NoVm{}, std::move(cell)); @@ -936,7 +1048,8 @@ std::unique_ptr DynamicBagOfCellsDb::create_in_memory(td::K if (kv == nullptr) { LOG_IF(WARNING, options.verbose) << "Create empty in-memory cells database (no key value is given)"; auto storage = CellStorage::build(options, [](auto, auto, auto) { return std::make_pair(0, 0); }); - return std::make_unique(std::move(storage)); + auto meta_storage = td::make_unique(std::vector>{}); + return std::make_unique(std::move(storage), std::move(meta_storage)); } std::vector keys; @@ -962,6 +1075,9 @@ std::unique_ptr DynamicBagOfCellsDb::create_in_memory(td::K local_desc_count++; return td::Status::OK(); } + if (key.size() != 32) { + return td::Status::OK(); + } auto r_res = CellLoader::load(key, value.str(), true, pc_creator); if (r_res.is_error()) { LOG(ERROR) << r_res.error() << " at " << td::format::escaped(key); @@ -983,6 +1099,24 @@ std::unique_ptr DynamicBagOfCellsDb::create_in_memory(td::K }; auto storage = CellStorage::build(options, parallel_scan_cells); - return std::make_unique(std::move(storage)); + + std::vector> meta; + // NB: it scans 1/(2^32) of the database which is not much + kv->for_each_in_range("desc", "desd", [&meta](td::Slice key, td::Slice value) { + if (key.size() != 32) { + meta.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + // this is for tests mostly. desc* keys are expected to correspond to roots + kv->for_each_in_range("meta", "metb", [&meta](td::Slice key, td::Slice value) { + if (key.size() != 32) { + meta.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + auto meta_storage = td::make_unique(std::move(meta)); + + return std::make_unique(std::move(storage), std::move(meta_storage)); } } // namespace vm diff --git a/crypto/vm/db/StaticBagOfCellsDb.cpp b/crypto/vm/db/StaticBagOfCellsDb.cpp index 80dbfbf0b2..c65d2624ff 100644 --- a/crypto/vm/db/StaticBagOfCellsDb.cpp +++ b/crypto/vm/db/StaticBagOfCellsDb.cpp @@ -40,6 +40,9 @@ class RootCell : public Cell { struct PrivateTag {}; public: + td::Status set_data_cell(Ref &&data_cell) const override { + return cell_->set_data_cell(std::move(data_cell)); + } td::Result load_cell() const override { return cell_->load_cell(); } @@ -94,11 +97,11 @@ class DataCellCacheNoop { class DataCellCacheMutex { public: Ref store(int idx, Ref cell) { - auto lock = cells_rw_mutex_.lock_write(); + std::lock_guard lock(mutex_); return cells_.emplace(idx, std::move(cell)).first->second; } Ref load(int idx) { - auto lock = cells_rw_mutex_.lock_read(); + std::lock_guard lock(mutex_); auto it = cells_.find(idx); if (it != cells_.end()) { return it->second; @@ -106,12 +109,13 @@ class DataCellCacheMutex { return {}; } void clear() { - auto guard = cells_rw_mutex_.lock_write(); + std::lock_guard lock(mutex_); cells_.clear(); } private: - td::RwMutex cells_rw_mutex_; + std::mutex mutex_; + // NB: in case of high contention, one should use multiple buckets with per bucket mutexes td::HashMap> cells_; }; @@ -246,7 +250,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { BagOfCells::Info info_; std::mutex index_i_mutex_; - td::RwMutex index_data_rw_mutex_; + std::mutex index_mutex_; std::string index_data_; std::atomic index_i_{0}; size_t index_offset_{0}; @@ -319,7 +323,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { info_.index_offset + (td::int64)idx * info_.offset_byte_size)); offset_view = new_offset_view; } else { - guard = index_data_rw_mutex_.lock_read().move_as_ok(); + std::lock_guard guard(index_mutex_); offset_view = td::Slice(index_data_).substr((td::int64)idx * info_.offset_byte_size, info_.offset_byte_size); } @@ -432,7 +436,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { } td::uint8 tmp[8]; info_.write_offset(tmp, index_offset_); - auto guard = index_data_rw_mutex_.lock_write(); + std::lock_guard guard(index_mutex_); index_data_.append(reinterpret_cast(tmp), info_.offset_byte_size); } return td::Status::OK(); diff --git a/tddb/td/db/KeyValue.h b/tddb/td/db/KeyValue.h index 12c3a4f8d7..c3f83919bf 100644 --- a/tddb/td/db/KeyValue.h +++ b/tddb/td/db/KeyValue.h @@ -20,19 +20,51 @@ #include "td/utils/Status.h" #include "td/utils/Time.h" #include "td/utils/logging.h" +#include "td/utils/ThreadSafeCounter.h" #include namespace td { +struct UsageStats { + size_t get_count{}; + size_t get_found_count{}; + size_t get_not_found_count{}; + size_t set_count{}; + UsageStats operator+(const UsageStats& other) const { + return UsageStats{.get_count = get_count + other.get_count, + .get_found_count = get_found_count + other.get_found_count, + .get_not_found_count = get_not_found_count + other.get_not_found_count, + .set_count = set_count + other.set_count}; + } + UsageStats operator-(const UsageStats& other) const { + return UsageStats{.get_count = get_count - other.get_count, + .get_found_count = get_found_count - other.get_found_count, + .get_not_found_count = get_not_found_count - other.get_not_found_count, + .set_count = set_count - other.set_count}; + } + NamedStats to_named_stats() const { + NamedStats ns; + ns.stats_int["usage_get_count"] += get_count; + ns.stats_int["usage_get_found_count"] += get_found_count; + ns.stats_int["usage_get_not_found_count"] += get_not_found_count; + ns.stats_int["usage_set_count"] += set_count; + return ns; + } +}; +inline td::StringBuilder& operator<<(td::StringBuilder& sb, const UsageStats& stats) { + sb << "get: " << stats.get_count << ", +" << stats.get_found_count << ", -" << stats.get_not_found_count; + return sb; +} + class KeyValueReader { public: virtual ~KeyValueReader() = default; enum class GetStatus : int32 { Ok, NotFound }; - virtual Result get(Slice key, std::string &value) = 0; + virtual Result get(Slice key, std::string& value) = 0; virtual Result count(Slice prefix) = 0; virtual Status for_each(std::function f) { return Status::Error("for_each is not supported"); } - virtual Status for_each_in_range (Slice begin, Slice end, std::function f) { + virtual Status for_each_in_range(Slice begin, Slice end, std::function f) { return td::Status::Error("foreach_range is not supported"); } }; @@ -42,7 +74,7 @@ class PrefixedKeyValueReader : public KeyValueReader { PrefixedKeyValueReader(std::shared_ptr reader, Slice prefix) : reader_(std::move(reader)), prefix_(prefix.str()) { } - Result get(Slice key, std::string &value) override { + Result get(Slice key, std::string& value) override { return reader_->get(PSLICE() << prefix_ << key, value); } Result count(Slice prefix) override { @@ -54,14 +86,16 @@ class PrefixedKeyValueReader : public KeyValueReader { std::string prefix_; }; -class KeyValueUtils { - public: -}; - class KeyValue : public KeyValueReader { public: virtual Status set(Slice key, Slice value) = 0; virtual Status erase(Slice key) = 0; + virtual Status merge(Slice key, Slice value) { + return Status::Error("merge is not supported"); + } + virtual Status run_gc() { + return Status::OK(); + } virtual Status begin_write_batch() = 0; virtual Status commit_write_batch() = 0; @@ -80,12 +114,15 @@ class KeyValue : public KeyValueReader { virtual Status flush() { return Status::OK(); } + virtual UsageStats get_usage_stats() { + return {}; + } }; class PrefixedKeyValue : public KeyValue { public: PrefixedKeyValue(std::shared_ptr kv, Slice prefix) : kv_(std::move(kv)), prefix_(prefix.str()) { } - Result get(Slice key, std::string &value) override { + Result get(Slice key, std::string& value) override { return kv_->get(PSLICE() << prefix_ << key, value); } Result count(Slice prefix) override { diff --git a/tddb/td/db/MemoryKeyValue.cpp b/tddb/td/db/MemoryKeyValue.cpp index 0801336024..7105f72b94 100644 --- a/tddb/td/db/MemoryKeyValue.cpp +++ b/tddb/td/db/MemoryKeyValue.cpp @@ -22,57 +22,99 @@ namespace td { Result MemoryKeyValue::get(Slice key, std::string &value) { - auto it = map_.find(key); - if (it == map_.end()) { + auto bucket = lock(key); + auto &map = bucket->map; + + usage_stats_.get_count++; + auto it = map.find(key); + if (it == map.end()) { + usage_stats_.get_not_found_count++; return GetStatus::NotFound; } value = it->second; + usage_stats_.get_found_count++; return GetStatus::Ok; } +std::unique_ptr MemoryKeyValue::lock(td::Slice key) { + auto bucket_id = std::hash()(std::string_view(key.data(), key.size())) % buckets_.size(); + return lock(buckets_[bucket_id]); +} + Status MemoryKeyValue::for_each(std::function f) { - for (auto &it : map_) { - TRY_STATUS(f(it.first, it.second)); + for (auto &unlocked_bucket : buckets_) { + auto bucket = lock(unlocked_bucket); + for (auto &it : bucket->map) { + TRY_STATUS(f(it.first, it.second)); + } } return Status::OK(); } Status MemoryKeyValue::for_each_in_range(Slice begin, Slice end, std::function f) { - for (auto it = map_.lower_bound(begin); it != map_.end(); it++) { - if (it->first < end) { - TRY_STATUS(f(it->first, it->second)); - } else { - break; + for (auto &unlocked_bucket : buckets_) { + auto bucket = lock(unlocked_bucket); + auto &map = bucket->map; + for (auto it = map.lower_bound(begin); it != map.end(); it++) { + if (it->first < end) { + TRY_STATUS(f(it->first, it->second)); + } else { + break; + } } } return Status::OK(); } Status MemoryKeyValue::set(Slice key, Slice value) { - map_[key.str()] = value.str(); + auto bucket = lock(key); + auto &map = bucket->map; + + usage_stats_.set_count++; + map[key.str()] = value.str(); return Status::OK(); } +Status MemoryKeyValue::merge(Slice key, Slice update) { + CHECK(merger_); + auto bucket = lock(key); + auto &map = bucket->map; + auto &value = map[key.str()]; + merger_->merge_value_and_update(value, update); + if (value.empty()) { + map.erase(key.str()); + } + return td::Status::OK(); +} Status MemoryKeyValue::erase(Slice key) { - auto it = map_.find(key); - if (it != map_.end()) { - map_.erase(it); + auto bucket = lock(key); + auto &map = bucket->map; + auto it = map.find(key); + if (it != map.end()) { + map.erase(it); } return Status::OK(); } Result MemoryKeyValue::count(Slice prefix) { size_t res = 0; - for (auto it = map_.lower_bound(prefix); it != map_.end(); it++) { - if (Slice(it->first).truncate(prefix.size()) != prefix) { - break; + for (auto &unlocked_bucket : buckets_) { + auto bucket = lock(unlocked_bucket); + auto &map = bucket->map; + for (auto it = map.lower_bound(prefix); it != map.end(); it++) { + if (Slice(it->first).truncate(prefix.size()) != prefix) { + break; + } + res++; } - res++; } return res; } std::unique_ptr MemoryKeyValue::snapshot() { auto res = std::make_unique(); - res->map_ = map_; + for (size_t i = 0; i < buckets_.size(); i++) { + auto bucket = lock(buckets_[i]); + res->buckets_[i].map = bucket->map; + } return std::move(res); } @@ -80,10 +122,10 @@ std::string MemoryKeyValue::stats() const { return PSTRING() << "MemoryKeyValueStats{" << tag("get_count", get_count_) << "}"; } Status MemoryKeyValue::begin_write_batch() { - UNREACHABLE(); + return Status::OK(); } Status MemoryKeyValue::commit_write_batch() { - UNREACHABLE(); + return Status::OK(); } Status MemoryKeyValue::abort_write_batch() { UNREACHABLE(); diff --git a/tddb/td/db/MemoryKeyValue.h b/tddb/td/db/MemoryKeyValue.h index f0b5faa08b..cf896095d8 100644 --- a/tddb/td/db/MemoryKeyValue.h +++ b/tddb/td/db/MemoryKeyValue.h @@ -22,12 +22,22 @@ #include namespace td { + +struct Merger { + virtual ~Merger() = default; + virtual void merge_value_and_update(std::string &value, Slice update) = 0; + virtual void merge_update_and_update(std::string &left_update, Slice right_update) = 0; +}; class MemoryKeyValue : public KeyValue { public: - Result get(Slice key, std::string &value) override; + MemoryKeyValue() = default; + MemoryKeyValue(std::shared_ptr merger) : merger_(std::move(merger)) { + } + Result get(Slice key, std::string& value) override; Status for_each(std::function f) override; Status for_each_in_range(Slice begin, Slice end, std::function f) override; Status set(Slice key, Slice value) override; + Status merge(Slice key, Slice value) override; Status erase(Slice key) override; Result count(Slice prefix) override; @@ -43,8 +53,30 @@ class MemoryKeyValue : public KeyValue { std::string stats() const override; + UsageStats get_usage_stats() override { + return usage_stats_; + } + private: - std::map> map_; + static constexpr size_t buckets_n = 64; + struct Bucket { + std::mutex mutex; + std::map> map; + }; + struct Unlock { + void operator()(Bucket* bucket) const { + bucket->mutex.unlock(); + } + }; + std::array buckets_{}; int64 get_count_{0}; + UsageStats usage_stats_{}; + std::shared_ptr merger_; + + std::unique_ptr lock(Bucket& bucket) { + bucket.mutex.lock(); + return std::unique_ptr(&bucket); + } + std::unique_ptr lock(td::Slice key); }; } // namespace td diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index f1aa64a5d7..b56f3b145f 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -24,10 +24,13 @@ #include "rocksdb/write_batch.h" #include "rocksdb/utilities/optimistic_transaction_db.h" #include "rocksdb/utilities/transaction.h" +#include "td/utils/misc.h" + +#include namespace td { namespace { -static Status from_rocksdb(rocksdb::Status status) { +static Status from_rocksdb(const rocksdb::Status &status) { if (status.ok()) { return Status::OK(); } @@ -56,62 +59,83 @@ RocksDb::~RocksDb() { } RocksDb RocksDb::clone() const { + if (transaction_db_) { + return RocksDb{transaction_db_, options_}; + } return RocksDb{db_, options_}; } Result RocksDb::open(std::string path, RocksDbOptions options) { - rocksdb::OptimisticTransactionDB *db; - { - rocksdb::Options db_options; + rocksdb::Options db_options; + db_options.merge_operator = options.merge_operator; + db_options.compaction_filter = options.compaction_filter; - static auto default_cache = rocksdb::NewLRUCache(1 << 30); - if (!options.no_block_cache && options.block_cache == nullptr) { - options.block_cache = default_cache; - } + static auto default_cache = rocksdb::NewLRUCache(1 << 30); + if (!options.no_block_cache && options.block_cache == nullptr) { + options.block_cache = default_cache; + } - rocksdb::BlockBasedTableOptions table_options; - if (options.no_block_cache) { - table_options.no_block_cache = true; - } else { - table_options.block_cache = options.block_cache; - } - db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); - - db_options.use_direct_reads = options.use_direct_reads; - db_options.manual_wal_flush = true; - db_options.create_if_missing = true; - db_options.max_background_compactions = 4; - db_options.max_background_flushes = 2; - db_options.bytes_per_sync = 1 << 20; - db_options.writable_file_max_buffer_size = 2 << 14; - db_options.statistics = options.statistics; - db_options.max_log_file_size = 100 << 20; - db_options.keep_log_file_num = 1; - rocksdb::OptimisticTransactionDBOptions occ_options; - occ_options.validate_policy = rocksdb::OccValidationPolicy::kValidateSerial; + rocksdb::BlockBasedTableOptions table_options; + if (options.no_block_cache) { + table_options.no_block_cache = true; + } else { + table_options.block_cache = options.block_cache; + } + db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); + + // table_options.block_align = true; + if (options.no_reads) { + db_options.memtable_factory.reset(new rocksdb::VectorRepFactory()); + db_options.allow_concurrent_memtable_write = false; + } + + db_options.wal_recovery_mode = rocksdb::WALRecoveryMode::kTolerateCorruptedTailRecords; + db_options.use_direct_reads = options.use_direct_reads; + db_options.manual_wal_flush = true; + db_options.create_if_missing = true; + db_options.max_background_compactions = 4; + db_options.max_background_flushes = 2; + db_options.bytes_per_sync = 1 << 20; + db_options.writable_file_max_buffer_size = 2 << 14; + db_options.statistics = options.statistics; + db_options.max_log_file_size = 100 << 20; + db_options.keep_log_file_num = 1; + + if (options.experimental) { + // Place your experimental options here + } + + if (options.no_transactions) { + rocksdb::DB *db{nullptr}; + TRY_STATUS(from_rocksdb(rocksdb::DB::Open(db_options, std::move(path), &db))); + return RocksDb(std::shared_ptr(db), std::move(options)); + } else { + rocksdb::OptimisticTransactionDB *db{nullptr}; rocksdb::ColumnFamilyOptions cf_options(db_options); std::vector column_families; column_families.push_back(rocksdb::ColumnFamilyDescriptor(rocksdb::kDefaultColumnFamilyName, cf_options)); std::vector handles; + rocksdb::OptimisticTransactionDBOptions occ_options; + occ_options.validate_policy = rocksdb::OccValidationPolicy::kValidateSerial; TRY_STATUS(from_rocksdb(rocksdb::OptimisticTransactionDB::Open(db_options, occ_options, std::move(path), column_families, &handles, &db))); CHECK(handles.size() == 1); // i can delete the handle since DBImpl is always holding a reference to // default column family delete handles[0]; + return RocksDb(std::shared_ptr(db), std::move(options)); } - return RocksDb(std::shared_ptr(db), std::move(options)); } std::shared_ptr RocksDb::create_statistics() { return rocksdb::CreateDBStatistics(); } -std::string RocksDb::statistics_to_string(const std::shared_ptr statistics) { +std::string RocksDb::statistics_to_string(const std::shared_ptr &statistics) { return statistics->ToString(); } -void RocksDb::reset_statistics(const std::shared_ptr statistics) { +void RocksDb::reset_statistics(const std::shared_ptr &statistics) { statistics->Reset(); } @@ -133,7 +157,9 @@ std::string RocksDb::stats() const { } Result RocksDb::get(Slice key, std::string &value) { - //LOG(ERROR) << "GET"; + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::Status status; if (snapshot_) { rocksdb::ReadOptions options; @@ -162,6 +188,18 @@ Status RocksDb::set(Slice key, Slice value) { } return from_rocksdb(db_->Put({}, to_rocksdb(key), to_rocksdb(value))); } +Status RocksDb::merge(Slice key, Slice value) { + if (write_batch_) { + return from_rocksdb(write_batch_->Merge(to_rocksdb(key), to_rocksdb(value))); + } + if (transaction_) { + return from_rocksdb(transaction_->Merge(to_rocksdb(key), to_rocksdb(value))); + } + return from_rocksdb(db_->Merge({}, to_rocksdb(key), to_rocksdb(value))); +} +Status RocksDb::run_gc() { + return from_rocksdb(db_->CompactRange({}, nullptr, nullptr)); +} Status RocksDb::erase(Slice key) { if (write_batch_) { @@ -174,7 +212,11 @@ Status RocksDb::erase(Slice key) { } Result RocksDb::count(Slice prefix) { + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::ReadOptions options; + options.auto_prefix_mode = true; options.snapshot = snapshot_.get(); std::unique_ptr iterator; if (snapshot_ || !transaction_) { @@ -197,7 +239,11 @@ Result RocksDb::count(Slice prefix) { } Status RocksDb::for_each(std::function f) { + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::ReadOptions options; + options.auto_prefix_mode = true; options.snapshot = snapshot_.get(); std::unique_ptr iterator; if (snapshot_ || !transaction_) { @@ -219,7 +265,11 @@ Status RocksDb::for_each(std::function f) { } Status RocksDb::for_each_in_range(Slice begin, Slice end, std::function f) { + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::ReadOptions options; + options.auto_prefix_mode = true; options.snapshot = snapshot_.get(); std::unique_ptr iterator; if (snapshot_ || !transaction_) { @@ -252,9 +302,10 @@ Status RocksDb::begin_write_batch() { Status RocksDb::begin_transaction() { CHECK(!write_batch_); + CHECK(transaction_db_); rocksdb::WriteOptions options; options.sync = true; - transaction_.reset(db_->BeginTransaction(options, {})); + transaction_.reset(transaction_db_->BeginTransaction(options, {})); return Status::OK(); } @@ -307,7 +358,11 @@ Status RocksDb::end_snapshot() { } RocksDb::RocksDb(std::shared_ptr db, RocksDbOptions options) - : db_(std::move(db)), options_(options) { + : transaction_db_{db}, db_(std::move(db)), options_(std::move(options)) { +} + +RocksDb::RocksDb(std::shared_ptr db, RocksDbOptions options) + : db_(std::move(db)), options_(std::move(options)) { } void RocksDbSnapshotStatistics::begin_snapshot(const rocksdb::Snapshot *snapshot) { diff --git a/tddb/td/db/RocksDb.h b/tddb/td/db/RocksDb.h index 499a332810..d24a20dd7b 100644 --- a/tddb/td/db/RocksDb.h +++ b/tddb/td/db/RocksDb.h @@ -36,12 +36,16 @@ #include namespace rocksdb { +class DB; +class Comparator; class Cache; class OptimisticTransactionDB; class Transaction; class WriteBatch; class Snapshot; class Statistics; +class MergeOperator; +class CompactionFilter; } // namespace rocksdb namespace td { @@ -61,6 +65,14 @@ struct RocksDbOptions { std::shared_ptr statistics = nullptr; std::shared_ptr block_cache; // Default - one 1G cache for all RocksDb std::shared_ptr snapshot_statistics = nullptr; + + std::shared_ptr merge_operator = nullptr; + const rocksdb::CompactionFilter *compaction_filter = nullptr; + + bool experimental = false; + bool no_reads = false; + bool no_transactions = false; + bool use_direct_reads = false; bool no_block_cache = false; }; @@ -73,10 +85,12 @@ class RocksDb : public KeyValue { Result get(Slice key, std::string &value) override; Status set(Slice key, Slice value) override; + Status merge(Slice key, Slice value) override; Status erase(Slice key) override; + Status run_gc() override; Result count(Slice prefix) override; Status for_each(std::function f) override; - Status for_each_in_range (Slice begin, Slice end, std::function f) override; + Status for_each_in_range(Slice begin, Slice end, std::function f) override; Status begin_write_batch() override; Status commit_write_batch() override; @@ -94,8 +108,8 @@ class RocksDb : public KeyValue { std::string stats() const override; static std::shared_ptr create_statistics(); - static std::string statistics_to_string(const std::shared_ptr statistics); - static void reset_statistics(const std::shared_ptr statistics); + static std::string statistics_to_string(const std::shared_ptr &statistics); + static void reset_statistics(const std::shared_ptr &statistics); static std::shared_ptr create_cache(size_t capacity); @@ -103,12 +117,13 @@ class RocksDb : public KeyValue { RocksDb &operator=(RocksDb &&); ~RocksDb(); - std::shared_ptr raw_db() const { + std::shared_ptr raw_db() const { return db_; }; private: - std::shared_ptr db_; + std::shared_ptr transaction_db_; + std::shared_ptr db_; RocksDbOptions options_; std::unique_ptr transaction_; @@ -123,5 +138,6 @@ class RocksDb : public KeyValue { std::unique_ptr snapshot_; explicit RocksDb(std::shared_ptr db, RocksDbOptions options); + explicit RocksDb(std::shared_ptr db, RocksDbOptions options); }; } // namespace td diff --git a/tdutils/td/utils/MpmcQueue.h b/tdutils/td/utils/MpmcQueue.h index e6504e358c..1a5f8fa366 100644 --- a/tdutils/td/utils/MpmcQueue.h +++ b/tdutils/td/utils/MpmcQueue.h @@ -414,7 +414,9 @@ class MpmcQueue { while (true) { auto node = hazard_pointers_.protect(thread_id, 0, read_pos_); auto &block = node->block; - if (block.write_pos <= block.read_pos && node->next.load(std::memory_order_relaxed) == nullptr) { + auto read_pos = block.read_pos.load(); + auto write_pos = block.write_pos.load(); + if (write_pos <= read_pos && node->next.load(std::memory_order_relaxed) == nullptr) { return false; } auto pos = block.read_pos++; diff --git a/tdutils/td/utils/Status.h b/tdutils/td/utils/Status.h index cff8081437..f75de466a9 100644 --- a/tdutils/td/utils/Status.h +++ b/tdutils/td/utils/Status.h @@ -619,6 +619,13 @@ inline Result::Result(Status &&status) : status_(std::move(status)) { inline StringBuilder &operator<<(StringBuilder &string_builder, const Status &status) { return status.print(string_builder); } +template +StringBuilder &operator<<(StringBuilder &sb, const Result &result) { + if (result.is_ok()) { + return sb << "Ok{" << result.ok() << "}"; + } + return sb << result.error(); +} namespace detail { diff --git a/tdutils/td/utils/ThreadSafeCounter.h b/tdutils/td/utils/ThreadSafeCounter.h index aa976b2fbb..46dc16bf70 100644 --- a/tdutils/td/utils/ThreadSafeCounter.h +++ b/tdutils/td/utils/ThreadSafeCounter.h @@ -19,6 +19,7 @@ #pragma once +#include "port/thread.h" #include "td/utils/common.h" #include "td/utils/Slice.h" #include "td/utils/StringBuilder.h" @@ -26,6 +27,7 @@ #include #include +#include #include namespace td { @@ -69,6 +71,50 @@ class ThreadSafeCounter { ThreadSafeMultiCounter<1> counter_; }; +struct NamedStats { + std::map stats_int; + std::map stats_str; + + NamedStats with_suffix(const std::string &suffix) const { + NamedStats res; + for (auto &p : stats_int) { + res.stats_int[p.first + suffix] = p.second; + } + for (auto &p : stats_str) { + res.stats_str[p.first + suffix] = p.second; + } + return res; + } + NamedStats with_prefix(const std::string &prefix) const { + NamedStats res; + for (auto &p : stats_int) { + res.stats_int[prefix + p.first] = p.second; + } + for (auto &p : stats_str) { + res.stats_str[prefix + p.first] = p.second; + } + return res; + } + void apply_diff(const NamedStats &other) { + for (auto &p : other.stats_int) { + stats_int[p.first] += p.second; + } + for (auto &p : other.stats_str) { + stats_str[p.first] = p.second; + } + } + void subtract_diff(const NamedStats &other) { + for (auto &p : other.stats_int) { + stats_int[p.first] -= p.second; + } + } + NamedStats combine_with(const NamedStats &other) const { + NamedStats res = *this; + res.apply_diff(other); + return res; + } +}; + class NamedThreadSafeCounter { static constexpr int N = 128; using Counter = ThreadSafeMultiCounter; @@ -79,6 +125,9 @@ class NamedThreadSafeCounter { CounterRef() = default; CounterRef(size_t index, Counter *counter) : index_(index), counter_(counter) { } + void inc() { + add(1); + } void add(int64 diff) { counter_->add(index_, diff); } @@ -119,6 +168,11 @@ class NamedThreadSafeCounter { f(names_[i], counter_.sum(i)); } } + NamedStats get_stats() const { + NamedStats res; + for_each([&](Slice name, int64 cnt) { res.stats_int.emplace(name.str(), cnt); }); + return res; + } void clear() { std::unique_lock guard(mutex_); @@ -181,11 +235,11 @@ struct NamedPerfCounter { } // namespace td -#define TD_PERF_COUNTER(name) \ +#define TD_PERF_COUNTER(name) \ static auto perf_##name = td::NamedPerfCounter::get_default().get_counter(td::Slice(#name)); \ auto scoped_perf_##name = td::NamedPerfCounter::ScopedPerfCounterRef{.perf_counter = perf_##name}; -#define TD_PERF_COUNTER_SINCE(name, since) \ +#define TD_PERF_COUNTER_SINCE(name, since) \ static auto perf_##name = td::NamedPerfCounter::get_default().get_counter(td::Slice(#name)); \ - auto scoped_perf_##name = \ + auto scoped_perf_##name = \ td::NamedPerfCounter::ScopedPerfCounterRef{.perf_counter = perf_##name, .started_at_ticks = since}; diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 2ea04e1833..cbcc3ab1f0 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1414,6 +1414,9 @@ td::Status ValidatorEngine::load_global_config() { if (zero_state.root_hash.is_zero() || zero_state.file_hash.is_zero()) { return td::Status::Error(ton::ErrorCode::error, "[validator] section contains incomplete [zero_state]"); } + if (celldb_in_memory_ && celldb_v2_) { + return td::Status::Error(ton::ErrorCode::error, "at most one of --celldb-in-memory --celldb-v2 could be used"); + } ton::BlockIdExt init_block; if (!conf.validator_->init_block_) { @@ -1461,11 +1464,12 @@ td::Status ValidatorEngine::load_global_config() { if (!session_logs_file_.empty()) { validator_options_.write().set_session_logs_file(session_logs_file_); } - if (celldb_in_memory_) { + if (celldb_in_memory_ || celldb_v2_) { celldb_compress_depth_ = 0; } validator_options_.write().set_celldb_compress_depth(celldb_compress_depth_); validator_options_.write().set_celldb_in_memory(celldb_in_memory_); + validator_options_.write().set_celldb_v2(celldb_v2_); validator_options_.write().set_max_open_archive_files(max_open_archive_files_); validator_options_.write().set_archive_preload_period(archive_preload_period_); validator_options_.write().set_disable_rocksdb_stats(disable_rocksdb_stats_); @@ -4526,6 +4530,12 @@ int main(int argc, char *argv[]) { [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_in_memory, true); }); }); + p.add_option( + '\0', "celldb-v2", + "use new version off celldb", + [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_v2, true); }); + }); p.add_checked_option( '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.4)", [&](td::Slice s) -> td::Status { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index e0dc91f137..5a1db7f3f5 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -218,6 +218,7 @@ class ValidatorEngine : public td::actor::Actor { bool celldb_direct_io_ = false; bool celldb_preload_all_ = false; bool celldb_in_memory_ = false; + bool celldb_v2_ = false; td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; bool read_config_ = false; bool started_keyring_ = false; @@ -311,6 +312,9 @@ class ValidatorEngine : public td::actor::Actor { void set_celldb_in_memory(bool value) { celldb_in_memory_ = value; } + void set_celldb_v2(bool value) { + celldb_v2_ = value; + } void set_catchain_max_block_delay(double value) { catchain_max_block_delay_ = value; } diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index e86a373d1d..90c659cc46 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -28,6 +28,9 @@ #include "ton/ton-io.hpp" #include "common/delay.h" +#include +#include + namespace ton { namespace validator { @@ -73,6 +76,101 @@ CellDbIn::CellDbIn(td::actor::ActorId root_db, td::actor::ActorId td::Slice { + return td::Slice(value.data(), value.size()); + } + bool FullMergeV2(const MergeOperationInput& merge_in, MergeOperationOutput* merge_out) const override { + CHECK(merge_in.existing_value); + auto& value = *merge_in.existing_value; + CHECK(merge_in.operand_list.size() >= 1); + td::Slice diff; + std::string diff_buf; + if (merge_in.operand_list.size() == 1) { + diff = to_td(merge_in.operand_list[0]); + } else { + diff_buf = merge_in.operand_list[0].ToString(); + for (size_t i = 1; i < merge_in.operand_list.size(); ++i) { + vm::CellStorer::merge_refcnt_diffs(diff_buf, to_td(merge_in.operand_list[i])); + } + diff = diff_buf; + } + + merge_out->new_value = value.ToString(); + vm::CellStorer::merge_value_and_refcnt_diff(merge_out->new_value, diff); + return true; + } + bool PartialMerge(const rocksdb::Slice& /*key*/, const rocksdb::Slice& left, const rocksdb::Slice& right, + std::string* new_value, rocksdb::Logger* logger) const override { + *new_value = left.ToString(); + vm::CellStorer::merge_refcnt_diffs(*new_value, to_td(right)); + return true; + } +}; + +void CellDbIn::validate_meta() { + LOG(INFO) << "Validating metadata\n"; + size_t max_meta_keys_loaded = opts_->get_celldb_in_memory() ? std::numeric_limits::max() : 10000; + auto meta = boc_->meta_get_all(max_meta_keys_loaded).move_as_ok(); + bool partial_check = meta.size() == max_meta_keys_loaded; + if (partial_check) { + LOG(ERROR) << "Too much metadata in the database, do only partial check"; + } + size_t missing_roots = 0; + size_t unknown_roots = 0; + std::set root_hashes; + for (auto [k, v] : meta) { + if (k == "desczero") { + continue; + } + auto obj = fetch_tl_object(td::BufferSlice{v}, true); + obj.ensure(); + auto entry = DbEntry{obj.move_as_ok()}; + root_hashes.insert(vm::CellHash::from_slice(entry.root_hash.as_slice())); + auto cell = boc_->load_cell(entry.root_hash.as_slice()); + missing_roots += cell.is_error(); + LOG_IF(ERROR, cell.is_error()) << "Cannot load root from meta: " << entry.block_id.to_str() << " " << cell.error(); + } + + // load_known_roots is only supported by InMemory database, so it is ok to check all known roots here + auto known_roots = boc_->load_known_roots().move_as_ok(); + for (auto& root : known_roots) { + block::gen::ShardStateUnsplit::Record info; + block::gen::OutMsgQueueInfo::Record qinfo; + block::ShardId shard; + if (!(tlb::unpack_cell(root, info) && shard.deserialize(info.shard_id.write()) && + tlb::unpack_cell(info.out_msg_queue_info, qinfo))) { + LOG(FATAL) << "cannot create ShardDescr from a root in celldb"; + } + if (!partial_check && !root_hashes.contains(root->get_hash())) { + unknown_roots++; + LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no; + constexpr bool delete_unknown_roots = false; + if (delete_unknown_roots) { + vm::CellStorer stor{*cell_db_}; + cell_db_->begin_write_batch().ensure(); + boc_->dec(root); + boc_->commit(stor).ensure(); + cell_db_->commit_write_batch().ensure(); + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + } + LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no << " REMOVED"; + } + } + } + + LOG_IF(ERROR, missing_roots != 0) << "Missing root hashes: " << missing_roots; + LOG_IF(ERROR, unknown_roots != 0) << "Unknown roots: " << unknown_roots; + + LOG_IF(FATAL, missing_roots != 0) << "Missing root hashes: " << missing_roots; + LOG_IF(FATAL, unknown_roots != 0) << "Unknown roots: " << unknown_roots; + LOG(INFO) << "Validating metadata: OK\n"; +} + void CellDbIn::start_up() { on_load_callback_ = [actor = std::make_shared>( td::actor::create_actor("celldbmigration", actor_id(this))), @@ -96,44 +194,101 @@ void CellDbIn::start_up() { db_options.snapshot_statistics = snapshot_statistics_; } db_options.statistics = statistics_; - if (opts_->get_celldb_cache_size()) { - db_options.block_cache = td::RocksDb::create_cache(opts_->get_celldb_cache_size().value()); - LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(opts_->get_celldb_cache_size().value()); + auto o_celldb_cache_size = opts_->get_celldb_cache_size(); + + std::optional boc_in_memory_options; + std::optional boc_v1_options; + std::optional boc_v2_options; + + if (opts_->get_celldb_v2()) { + boc_v2_options = vm::DynamicBagOfCellsDb::CreateV2Options{ + .extra_threads = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 8u), + .executor = {}, + .cache_ttl_max = 2000, + .cache_size_max = 1000000}; + size_t min_rocksdb_cache = std::max(size_t{1} << 30, boc_v2_options->cache_size_max * 5000); + if (!o_celldb_cache_size || o_celldb_cache_size.value() < min_rocksdb_cache) { + LOG(WARNING) << "Increase CellDb block cache size to " << td::format::as_size(min_rocksdb_cache) << " from " + << td::format::as_size(o_celldb_cache_size.value()); + o_celldb_cache_size = min_rocksdb_cache; + } + LOG(WARNING) << "Using V2 DynamicBagOfCells with options " << *boc_v2_options; + } else if (opts_->get_celldb_in_memory()) { + // default options + boc_in_memory_options = vm::DynamicBagOfCellsDb::CreateInMemoryOptions{ + .extra_threads = std::thread::hardware_concurrency(), + .verbose = true, + .use_arena = false, + .use_less_memory_during_creation = true, + }; + LOG(WARNING) << "Using InMemory DynamicBagOfCells with options " << *boc_v2_options; + } else { + boc_v1_options = vm::DynamicBagOfCellsDb::CreateV1Options{}; + LOG(WARNING) << "Using V1 DynamicBagOfCells with options " << *boc_v1_options; + } + + if (o_celldb_cache_size) { + db_options.block_cache = td::RocksDb::create_cache(o_celldb_cache_size.value()); + LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(o_celldb_cache_size.value()); } db_options.use_direct_reads = opts_->get_celldb_direct_io(); + // NB: from now on we MUST use this merge operator + // Only V2 and InMemory BoC actually use them, but it still should be kept for V1, + // to handle updates written by V2 or InMemory BoCs + db_options.merge_operator = std::make_shared(); + if (opts_->get_celldb_in_memory()) { td::RocksDbOptions read_db_options; read_db_options.use_direct_reads = true; read_db_options.no_block_cache = true; read_db_options.block_cache = {}; + read_db_options.merge_operator = std::make_shared(); LOG(WARNING) << "Loading all cells in memory (because of --celldb-in-memory)"; td::Timer timer; auto read_cell_db = std::make_shared(td::RocksDb::open(path_, std::move(read_db_options)).move_as_ok()); - boc_ = vm::DynamicBagOfCellsDb::create_in_memory(read_cell_db.get(), {}); + boc_ = vm::DynamicBagOfCellsDb::create_in_memory(read_cell_db.get(), *boc_in_memory_options); in_memory_load_time_ = timer.elapsed(); - td::actor::send_closure(parent_, &CellDb::set_in_memory_boc, boc_); + + // no reads will be allowed from rocksdb, only writes + db_options.no_reads = true; } auto rocks_db = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); rocks_db_ = rocks_db->raw_db(); cell_db_ = std::move(rocks_db); if (!opts_->get_celldb_in_memory()) { - boc_ = vm::DynamicBagOfCellsDb::create(); + if (opts_->get_celldb_v2()) { + boc_ = vm::DynamicBagOfCellsDb::create_v2(*boc_v2_options); + } else { + boc_ = vm::DynamicBagOfCellsDb::create(*boc_v1_options); + } boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth()); boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); - td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); } + validate_meta(); + alarm_timestamp() = td::Timestamp::in(10.0); auto empty = get_empty_key_hash(); if (get_block(empty).is_error()) { DbEntry e{get_empty_key(), empty, empty, RootHash::zero()}; + vm::CellStorer stor{*cell_db_}; cell_db_->begin_write_batch().ensure(); set_block(empty, std::move(e)); + boc_->commit(stor); cell_db_->commit_write_batch().ensure(); + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + } + } + + if (opts_->get_celldb_v2() || opts_->get_celldb_in_memory()) { + send_closure(parent_, &CellDb::set_thread_safe_boc, boc_); + } else { + send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); } if (opts_->get_celldb_preload_all()) { @@ -161,7 +316,7 @@ void CellDbIn::start_up() { { std::string key = "stats.last_deleted_mc_seqno", value; - auto R = cell_db_->get(td::as_slice(key), value); + auto R = boc_->meta_get(td::as_slice(key), value); R.ensure(); if (R.ok() == td::KeyValue::GetStatus::Ok) { auto r_value = td::to_integer_safe(value); @@ -240,10 +395,10 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi td::Timer timer_write; vm::CellStorer stor{*cell_db_}; cell_db_->begin_write_batch().ensure(); - boc_->commit(stor).ensure(); set_block(get_empty_key_hash(), std::move(E)); set_block(D.prev, std::move(P)); set_block(key_hash, std::move(D)); + boc_->commit(stor).ensure(); cell_db_->commit_write_batch().ensure(); timer_write.pause(); @@ -266,11 +421,10 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi void CellDbIn::get_cell_db_reader(td::Promise> promise) { if (db_busy_) { - action_queue_.push( - [self = this, promise = std::move(promise)](td::Result R) mutable { - R.ensure(); - self->get_cell_db_reader(std::move(promise)); - }); + action_queue_.push([self = this, promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + self->get_cell_db_reader(std::move(promise)); + }); return; } promise.set_result(boc_->get_cell_db_reader()); @@ -440,9 +594,16 @@ void CellDbIn::gc_cont2(BlockHandle handle) { timer_get_keys.reset(); td::PerfWarningTimer timer_boc{"gccell_boc", 0.05}; - auto cell = boc_->load_cell(F.root_hash.as_slice()).move_as_ok(); + auto r_cell = boc_->load_cell(F.root_hash.as_slice()); + td::Ref cell; + if (r_cell.is_ok()) { + cell = r_cell.move_as_ok(); + boc_->dec(cell); + LOG(ERROR) << "GC of " << handle->id().to_str(); + } else { + LOG(ERROR) << "GC of UNKNOWN root: " << handle->id().to_str(); + } - boc_->dec(cell); db_busy_ = true; boc_->prepare_commit_async( async_executor, [this, SelfId = actor_id(this), timer_boc = std::move(timer_boc), F = std::move(F), key_hash, @@ -458,17 +619,19 @@ void CellDbIn::gc_cont2(BlockHandle handle) { td::PerfWarningTimer timer_write_batch{"gccell_write_batch", 0.05}; cell_db_->begin_write_batch().ensure(); - boc_->commit(stor).ensure(); - cell_db_->erase(get_key(key_hash)).ensure(); + boc_->meta_erase(get_key(key_hash)).ensure(); set_block(F.prev, std::move(P)); set_block(F.next, std::move(N)); if (handle->id().is_masterchain()) { last_deleted_mc_state_ = handle->id().seqno(); std::string key = "stats.last_deleted_mc_seqno", value = td::to_string(last_deleted_mc_state_); - cell_db_->set(td::as_slice(key), td::as_slice(value)); + boc_->meta_set(td::as_slice(key), td::as_slice(value)); } + + boc_->commit(stor).ensure(); cell_db_->commit_write_batch().ensure(); + alarm_timestamp() = td::Timestamp::now(); timer_write_batch.reset(); @@ -530,7 +693,7 @@ CellDbIn::KeyHash CellDbIn::get_empty_key_hash() { td::Result CellDbIn::get_block(KeyHash key_hash) { const auto key = get_key(key_hash); std::string value; - auto R = cell_db_->get(td::as_slice(key), value); + auto R = boc_->meta_get(td::as_slice(key), value); R.ensure(); auto S = R.move_as_ok(); if (S == td::KeyValue::GetStatus::NotFound) { @@ -543,7 +706,7 @@ td::Result CellDbIn::get_block(KeyHash key_hash) { void CellDbIn::set_block(KeyHash key_hash, DbEntry e) { const auto key = get_key(key_hash); - cell_db_->set(td::as_slice(key), e.release()).ensure(); + boc_->meta_set(td::as_slice(key), e.release()); } void CellDbIn::migrate_cell(td::Bits256 hash) { @@ -631,12 +794,14 @@ void CellDb::alarm() { } void CellDb::load_cell(RootHash hash, td::Promise> promise) { - if (in_memory_boc_) { - auto result = in_memory_boc_->load_root_thread_safe(hash.as_slice()); + if (thread_safe_boc_) { + auto result = thread_safe_boc_->load_root_thread_safe(hash.as_slice()); if (result.is_ok()) { return async_apply("load_cell_result", std::move(promise), std::move(result)); } else { LOG(ERROR) << "load_root_thread_safe failed - this is suspicious"; + send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); + return; } } if (!started_) { @@ -710,6 +875,13 @@ std::vector> CellDbIn::CellDbStatistics::pre for (auto& [key, value] : boc_stats_->custom_stats) { stats.emplace_back(key, value); } + + for (auto& [key, value] : boc_stats_->named_stats.stats_str) { + stats.emplace_back(key, value); + } + for (auto& [key, value] : boc_stats_->named_stats.stats_int) { + stats.emplace_back(key, td::to_string(value)); + } } return stats; } diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 5639b9748b..1e1ccddabf 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -74,6 +74,7 @@ class CellDbIn : public CellDbBase { CellDbIn(td::actor::ActorId root_db, td::actor::ActorId parent, std::string path, td::Ref opts); + void validate_meta(); void start_up() override; void alarm() override; @@ -195,13 +196,13 @@ class CellDb : public CellDbBase { started_ = true; boc_->set_loader(std::make_unique(std::move(snapshot), on_load_callback_)).ensure(); } - void set_in_memory_boc(std::shared_ptr in_memory_boc) { - CHECK(opts_->get_celldb_in_memory()); + void set_thread_safe_boc(std::shared_ptr thread_safe_boc) { + CHECK(opts_->get_celldb_in_memory() || opts_->get_celldb_v2()); if (!started_) { alarm(); } started_ = true; - in_memory_boc_ = std::move(in_memory_boc); + thread_safe_boc_ = std::move(thread_safe_boc); } void get_cell_db_reader(td::Promise> promise); @@ -219,7 +220,7 @@ class CellDb : public CellDbBase { td::actor::ActorOwn cell_db_; std::unique_ptr boc_; - std::shared_ptr in_memory_boc_; + std::shared_ptr thread_safe_boc_; bool started_ = false; std::vector> prepared_stats_{{"started", "false"}}; diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index ace6b1066f..45b8d7ec20 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -139,6 +139,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool get_celldb_in_memory() const override { return celldb_in_memory_; } + bool get_celldb_v2() const override { + return celldb_v2_; + } td::optional get_catchain_max_block_delay() const override { return catchain_max_block_delay_; } @@ -237,6 +240,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_celldb_in_memory(bool value) override { celldb_in_memory_ = value; } + void set_celldb_v2(bool value) override { + celldb_v2_ = value; + } void set_catchain_max_block_delay(double value) override { catchain_max_block_delay_ = value; } @@ -304,6 +310,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool celldb_direct_io_ = false; bool celldb_preload_all_ = false; bool celldb_in_memory_ = false; + bool celldb_v2_ = false; td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; bool state_serializer_enabled_ = true; td::Ref collator_options_{true}; diff --git a/validator/validator.h b/validator/validator.h index eecb30c078..9e1dba7db8 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -104,6 +104,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual std::string get_session_logs_file() const = 0; virtual td::uint32 get_celldb_compress_depth() const = 0; virtual bool get_celldb_in_memory() const = 0; + virtual bool get_celldb_v2() const = 0; virtual size_t get_max_open_archive_files() const = 0; virtual double get_archive_preload_period() const = 0; virtual bool get_disable_rocksdb_stats() const = 0; @@ -144,6 +145,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_celldb_direct_io(bool value) = 0; virtual void set_celldb_preload_all(bool value) = 0; virtual void set_celldb_in_memory(bool value) = 0; + virtual void set_celldb_v2(bool value) = 0; virtual void set_catchain_max_block_delay(double value) = 0; virtual void set_catchain_max_block_delay_slow(double value) = 0; virtual void set_state_serializer_enabled(bool value) = 0; From 81d24492fa732c2a992224adc44ae059067eadcb Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Sun, 30 Mar 2025 09:12:42 -0400 Subject: [PATCH 185/524] Optimize layout of and simplify vm::DataCell (#1570) * Add missing includes to td/utils/ThreadSafeCounter.h This allows the file to be included first in other headers. * Optimize layout of and simplify vm::DataCell This is a complete rewrite of vm::DataCell initially undertaken to make optimizer life easier when working with DataCell. In particular, the commit makes references to child cells reside at the known offset from `this` of the data cell, removes a bunch of unneeded complexity from the class definition itself (like `Info` struct), and uses placement new to allocate class with a trailer instead of relying on template trickery from `CellWithStorage.h`. Additionally, DataCell verification is rewritten from scratch and collected into a nice `CellChecker` class to make it easier to follow through and reason about both for humans and compilers. All in all, this is 3.5% time speedup on the verification benchmark (which is somewhat hard to attribute to a particular change) and average 5.5% memory consumption improvement (most likely coming from better packing of the fields). * Store variable-sized data in `char trailer_[]` in vm::PrunnedCell This is similar to how it is done in DataCell. * Remove now unused vm/cells/CellWithStorage.h * Make DynamicBagOfCellsDb::validate_bucket_a agnostic to use_arena option This prevents test failures if we actually start destructing (but not freeing) arena-allocated cells. --- crypto/CMakeLists.txt | 1 - crypto/vm/cells/CellBuilder.cpp | 5 +- crypto/vm/cells/CellWithStorage.h | 105 ----- crypto/vm/cells/DataCell.cpp | 578 ++++++++++++++------------ crypto/vm/cells/DataCell.h | 248 +++++------ crypto/vm/cells/PrunnedCell.h | 50 ++- crypto/vm/db/InMemoryBagOfCellsDb.cpp | 20 +- crypto/vm/db/StaticBagOfCellsDb.cpp | 1 - tdutils/td/utils/ThreadSafeCounter.h | 4 +- 9 files changed, 463 insertions(+), 549 deletions(-) delete mode 100644 crypto/vm/cells/CellWithStorage.h diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 2988f501ff..defe2b58e3 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -70,7 +70,6 @@ set(TON_CRYPTO_CORE_SOURCE vm/cells/CellString.h vm/cells/CellTraits.h vm/cells/CellUsageTree.h - vm/cells/CellWithStorage.h vm/cells/DataCell.h vm/cells/ExtCell.h vm/cells/LevelMask.h diff --git a/crypto/vm/cells/CellBuilder.cpp b/crypto/vm/cells/CellBuilder.cpp index 772b5f6b7a..a9ad449e1c 100644 --- a/crypto/vm/cells/CellBuilder.cpp +++ b/crypto/vm/cells/CellBuilder.cpp @@ -50,7 +50,7 @@ Ref CellBuilder::finalize_copy(bool special) const { if (vm_state_interface) { vm_state_interface->register_cell_create(); } - auto res = DataCell::create(data, size(), td::span(refs.data(), size_refs()), special); + auto res = DataCell::create(td::Slice{data, Cell::max_bytes}, size(), td::span(refs.data(), size_refs()), special); if (res.is_error()) { LOG(DEBUG) << res.error(); throw CellWriteError{}; @@ -68,7 +68,8 @@ Ref CellBuilder::finalize_copy(bool special) const { } td::Result> CellBuilder::finalize_novm_nothrow(bool special) { - auto res = DataCell::create(data, size(), td::mutable_span(refs.data(), size_refs()), special); + auto res = + DataCell::create(td::Slice{data, Cell::max_bytes}, size(), td::mutable_span(refs.data(), size_refs()), special); bits = refs_cnt = 0; return res; } diff --git a/crypto/vm/cells/CellWithStorage.h b/crypto/vm/cells/CellWithStorage.h deleted file mode 100644 index f3fedfc31e..0000000000 --- a/crypto/vm/cells/CellWithStorage.h +++ /dev/null @@ -1,105 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -namespace vm { -namespace detail { - -template -struct DefaultAllocator { - template - std::unique_ptr make_unique(ArgsT&&... args) { - return std::make_unique(std::forward(args)...); - } -}; - -template -class CellWithArrayStorage : public CellT { - public: - template - CellWithArrayStorage(ArgsT&&... args) : CellT(std::forward(args)...) { - } - ~CellWithArrayStorage() { - CellT::destroy_storage(get_storage()); - } - template - static auto create(Allocator allocator, size_t storage_size, ArgsT&&... args) { - static_assert(CellT::max_storage_size <= 40 * 8, ""); - //size = 128 + 32 + 8; - auto size = (storage_size + 7) / 8; -#define CASE(size) \ - case (size): \ - return allocator. template make_unique>(std::forward(args)...); -#define CASE2(offset) CASE(offset) CASE(offset + 1) -#define CASE8(offset) CASE2(offset) CASE2(offset + 2) CASE2(offset + 4) CASE2(offset + 6) -#define CASE32(offset) CASE8(offset) CASE8(offset + 8) CASE8(offset + 16) CASE8(offset + 24) - switch (size) { CASE32(0) CASE8(32) } -#undef CASE -#undef CASE2 -#undef CASE8 -#undef CASE32 - LOG(FATAL) << "TOO BIG " << storage_size; - UNREACHABLE(); - } - template - static std::unique_ptr create(size_t storage_size, ArgsT&&... args) { - return create(DefaultAllocator{}, storage_size, std::forward(args)...); - } - - private: - alignas(alignof(void*)) char storage_[Size]; - - const char* get_storage() const final { - return storage_; - } - char* get_storage() final { - return storage_; - } -}; - -template -class CellWithUniquePtrStorage : public CellT { - public: - template - CellWithUniquePtrStorage(size_t storage_size, ArgsT&&... args) - : CellT(std::forward(args)...), storage_(std::make_unique(storage_size)) { - } - ~CellWithUniquePtrStorage() { - CellT::destroy_storage(get_storage()); - } - - template - static std::unique_ptr create(size_t storage_size, ArgsT&&... args) { - return std::make_unique(storage_size, std::forward(args)...); - } - - private: - std::unique_ptr storage_; - - const char* get_storage() const final { - CHECK(storage_); - return storage_.get(); - } - char* get_storage() final { - CHECK(storage_); - return storage_.get(); - } -}; -} // namespace detail -} // namespace vm diff --git a/crypto/vm/cells/DataCell.cpp b/crypto/vm/cells/DataCell.cpp index 5134123d8c..14da8e85ac 100644 --- a/crypto/vm/cells/DataCell.cpp +++ b/crypto/vm/cells/DataCell.cpp @@ -16,335 +16,368 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/cells/DataCell.h" #include "openssl/digest.hpp" - -#include "td/utils/ScopeGuard.h" - -#include "vm/cells/CellWithStorage.h" +#include "vm/cells/DataCell.h" namespace vm { -thread_local bool DataCell::use_arena = false; namespace { -template -struct ArenaAllocator { - template - std::unique_ptr make_unique(ArgsT&&... args) { - auto* ptr = fast_alloc(sizeof(T)); - T* obj = new (ptr) T(std::forward(args)...); - return std::unique_ptr(obj); - } - private: - td::MutableSlice alloc_batch() { - size_t batch_size = 1 << 20; - auto batch = std::make_unique(batch_size); - return td::MutableSlice(batch.release(), batch_size); +class CellChecker { + public: + CellChecker(bool is_special, td::Slice data, int bit_length, td::Span> refs) + : is_special_(is_special) + , refs_(refs) + , refs_cnt_(static_cast(refs.size())) + , data_(data) + , bit_length_(bit_length) { } - char* fast_alloc(size_t size) { - thread_local td::MutableSlice batch; - auto aligned_size = (size + 7) / 8 * 8; - if (batch.size() < size) { - batch = alloc_batch(); + + td::Status check_and_compute_level_info() { + // First, we figure out what is the type of the cell. + type_ = Cell::SpecialType::Ordinary; + + if (is_special_) { + if (bit_length_ < 8) { + return td::Status::Error("Not enough data for a special cell"); + } + + type_ = static_cast(read_byte(0)); + if (type_ == Cell::SpecialType::Ordinary) { + return td::Status::Error("Invalid special cell type"); + } } - auto res = batch.begin(); - batch.remove_prefix(aligned_size); - return res; - } -}; -} // namespace -std::unique_ptr DataCell::create_empty_data_cell(Info info) { - if (use_arena) { - ArenaAllocator allocator; - auto res = detail::CellWithArrayStorage::create(allocator, info.get_storage_size(), info); - // this is dangerous - Ref(res.get()).release(); - return res; + + // Next, we populate everything except for virtualization and hashes. `check_*` functions also + // perform type-specific checks. + switch (type_) { + case Cell::SpecialType::Ordinary: + TRY_STATUS(check_ordinary_cell()); + break; + case Cell::SpecialType::PrunnedBranch: + TRY_STATUS(check_pruned_branch()); + break; + case Cell::SpecialType::Library: + TRY_STATUS(check_library()); + break; + case Cell::SpecialType::MerkleProof: + TRY_STATUS(check_merkle_proof()); + break; + case Cell::SpecialType::MerkleUpdate: + TRY_STATUS(check_merkle_update()); + break; + default: + return td::Status::Error("Invalid special cell type"); + } + + // Afterwards, we do some common checks and compute virtualization level. + if (*std::max_element(depth_.begin(), depth_.end()) > CellTraits::max_depth) { + return td::Status::Error("Depth is too big"); + } + + for (int i = 0; i < refs_cnt_; ++i) { + virtualization_ = std::max(virtualization_, refs_[i]->get_virtualization()); + } + if (virtualization_ > std::numeric_limits::max()) { + return td::Status::Error("Virtualization is too big to be stored in vm::DataCell"); + } + + // And finally, we compute cell hashes. + // NOTE: Hash computation algorithm is not described correctly (or at all) in the documentation. + int last_computed_hash = -1; + + for (int i = 0; i <= max_level; ++i) { + if (!level_mask_.is_significant(i + 1) && i != max_level) { + continue; + } + + compute_hash(i, last_computed_hash); + for (int j = last_computed_hash + 1; j < i; ++j) { + hash_[j] = hash_[i]; + } + last_computed_hash = i; + } + + return {}; } - return detail::CellWithUniquePtrStorage::create(info.get_storage_size(), info); -} + // Getters for computed values + Cell::SpecialType type() const { + return type_; + } -DataCell::DataCell(Info info) : info_(std::move(info)) { - get_thread_safe_counter().add(1); -} -DataCell::~DataCell() { - get_thread_safe_counter().add(-1); -} + Cell::LevelMask level_mask() const { + return level_mask_; + } -void DataCell::destroy_storage(char* storage) { - auto* refs = info_.get_refs(storage); - for (size_t i = 0; i < get_refs_cnt(); i++) { - Ref(refs[i], Ref::acquire_t{}); // call destructor + td::uint8 virtualization() const { + return static_cast(virtualization_); } -} -td::Result> DataCell::create(td::ConstBitPtr data, unsigned bits, td::Span> refs, - bool special) { - std::array, max_refs> copied_refs; - CHECK(refs.size() <= copied_refs.size()); - for (size_t i = 0; i < refs.size(); i++) { - copied_refs[i] = refs[i]; + std::array const& depths() const { + return depth_; } - return create(std::move(data), bits, td::MutableSpan>(copied_refs.data(), refs.size()), special); -} -DataCell::SpecialType DataCell::special_type() const { - if (is_special()) { - return static_cast(td::bitstring::bits_load_ulong(get_data(), 8)); + std::array const& hashes() const { + return hash_; } - return SpecialType::Ordinary; -} -td::Result> DataCell::create(td::ConstBitPtr data, unsigned bits, td::MutableSpan> refs, - bool special) { - for (auto& ref : refs) { - if (ref.is_null()) { - return td::Status::Error("Has null cell reference"); - } + private: + static constexpr int max_level = CellTraits::max_level; + + static constexpr int hash_bytes = CellTraits::hash_bytes; + static_assert(hash_bytes == sizeof(CellHash)); + + static constexpr int depth_bytes = CellTraits::depth_bytes; + static_assert(depth_bytes == 2); + + td::uint8 read_byte(size_t i) { + return data_[i]; } - SpecialType type = SpecialType::Ordinary; - if (special) { - if (bits < 8) { - return td::Status::Error("Not enough data for a special cell"); + td::Status check_ordinary_cell() { + for (int i = 0; i < refs_cnt_; ++i) { + level_mask_ = level_mask_.apply_or(refs_[i]->get_level_mask()); + + for (int j = 0; j <= max_level; ++j) { + depth_[j] = std::max(depth_[j], refs_[i]->get_depth(j)); + } } - type = static_cast(td::bitstring::bits_load_ulong(data, 8)); - if (type == SpecialType::Ordinary) { - return td::Status::Error("Special cell has Ordinary type"); + + if (refs_cnt_ != 0) { + for (auto& depth : depth_) { + ++depth; + } } + + return {}; } - LevelMask level_mask; - td::uint32 virtualization = 0; - switch (type) { - case SpecialType::Ordinary: { - for (auto& ref : refs) { - level_mask = level_mask.apply_or(ref->get_level_mask()); - virtualization = td::max(virtualization, ref->get_virtualization()); - } - break; + td::Status check_pruned_branch() { + if (refs_cnt_ != 0) { + return td::Status::Error("Pruned branch cannot have references"); + } + if (bit_length_ < 16) { + return td::Status::Error("Length mismatch in a pruned branch"); } - case SpecialType::PrunnedBranch: { - if (refs.size() != 0) { - return td::Status::Error("PrunnedBranch special cell has a cell reference"); - } - if (bits < 16) { - return td::Status::Error("Not enough data for a PrunnedBranch special cell"); - } - level_mask = LevelMask((td::bitstring::bits_load_ulong(data + 8, 8)) & 0xff); - auto level = level_mask.get_level(); - if (level > max_level || level == 0) { - return td::Status::Error("Prunned Branch has an invalid level"); - } - if (bits != (2 + level_mask.apply(level - 1).get_hashes_count() * (hash_bytes + depth_bytes)) * 8) { - return td::Status::Error("Not enouch data for a PrunnedBranch special cell"); - } - // depth will be checked later! - break; + level_mask_ = Cell::LevelMask{read_byte(1)}; + if (level_mask_.get_level() == 0 || level_mask_.get_level() > max_level) { + return td::Status::Error("Invalid level mask in a pruned branch"); } - case SpecialType::Library: { - if (bits != 8 + hash_bytes * 8) { - return td::Status::Error("Not enouch data for a Library special cell"); - } - if (!refs.empty()) { - return td::Status::Error("Library special cell has a cell reference"); - } - break; + int hashes_count = level_mask_.get_hash_i(); + auto expected_byte_size = 2 + hashes_count * (hash_bytes + depth_bytes); + + if (bit_length_ != static_cast(expected_byte_size * 8)) { + return td::Status::Error("Length mismatch in a pruned branch"); } - case SpecialType::MerkleProof: { - if (bits != 8 + (hash_bytes + depth_bytes) * 8) { - return td::Status::Error("Not enouch data for a MerkleProof special cell"); - } - if (refs.size() != 1) { - return td::Status::Error("Wrong references count for a MerkleProof special cell"); - } - if (td::bitstring::bits_memcmp(data + 8, refs[0]->get_hash(0).as_bitslice().get_ptr(), hash_bits) != 0) { - return td::Status::Error("Hash mismatch in a MerkleProof special cell"); - } - if (td::bitstring::bits_load_ulong(data + 8 + hash_bits, depth_bytes * 8) != refs[0]->get_depth(0)) { - return td::Status::Error("Depth mismatch in a MerkleProof special cell"); + // depth[max_level] = 0; + + for (int i = max_level; i--;) { + if (level_mask_.is_significant(i + 1)) { + int hashes_before = level_mask_.apply(i).get_hash_i(); + auto offset = 2 + hashes_count * hash_bytes + hashes_before * depth_bytes; + depth_[i] = DataCell::load_depth(data_.ubegin() + offset); + } else { + depth_[i] = depth_[i + 1]; } - level_mask = refs[0]->get_level_mask().shift_right(); - virtualization = refs[0]->get_virtualization(); - break; } - case SpecialType::MerkleUpdate: { - if (bits != 8 + (hash_bytes + depth_bytes) * 8 * 2) { - return td::Status::Error("Not enouch data for a MerkleUpdate special cell"); - } - if (refs.size() != 2) { - return td::Status::Error("Wrong references count for a MerkleUpdate special cell"); - } - if (td::bitstring::bits_memcmp(data + 8, refs[0]->get_hash(0).as_bitslice().get_ptr(), hash_bits) != 0) { - return td::Status::Error("First hash mismatch in a MerkleProof special cell"); - } - if (td::bitstring::bits_memcmp(data + 8 + hash_bits, refs[1]->get_hash(0).as_bitslice().get_ptr(), hash_bits) != - 0) { - return td::Status::Error("Second hash mismatch in a MerkleProof special cell"); - } - if (td::bitstring::bits_load_ulong(data + 8 + 2 * hash_bits, depth_bytes * 8) != refs[0]->get_depth(0)) { - return td::Status::Error("First depth mismatch in a MerkleProof special cell"); - } - if (td::bitstring::bits_load_ulong(data + 8 + 2 * hash_bits + depth_bytes * 8, depth_bytes * 8) != - refs[1]->get_depth(0)) { - return td::Status::Error("Second depth mismatch in a MerkleProof special cell"); - } + return {}; + } - level_mask = refs[0]->get_level_mask().apply_or(refs[1]->get_level_mask()).shift_right(); - virtualization = td::max(refs[0]->get_virtualization(), refs[1]->get_virtualization()); - break; + td::Status check_library() { + if (refs_cnt_ != 0) { + return td::Status::Error("Library cell cannot have references"); + } + if (bit_length_ != 8 * (1 + hash_bytes)) { + return td::Status::Error("Length mismatch in a library cell"); } - default: - return td::Status::Error("Unknown special cell type"); + return {}; } - Info info; - if (td::unlikely(bits > max_bits)) { - return td::Status::Error("Too many bits"); - } - if (td::unlikely(refs.size() > max_refs)) { - return td::Status::Error("Too many cell references"); - } - if (td::unlikely(virtualization > max_virtualization)) { - return td::Status::Error("Too big virtualization"); - } + td::Status check_merkle_child(int child_idx, int hash_offset, int depth_offset) { + CellHash stored_hash; + std::memcpy(&stored_hash, data_.begin() + hash_offset, hash_bytes); + if (stored_hash != refs_[child_idx]->get_hash(0)) { + return td::Status::Error("Invalid hash in a Merkle proof or update"); + } - CHECK(level_mask.get_level() <= max_level); - - auto hash_count = type == SpecialType::PrunnedBranch ? 1 : level_mask.get_hashes_count(); - DCHECK(hash_count <= max_level + 1); - - info.bits_ = bits; - info.refs_count_ = refs.size() & 7; - info.is_special_ = special; - info.level_mask_ = level_mask.get_mask() & 7; - info.hash_count_ = hash_count & 7; - info.virtualization_ = virtualization & 7; - - auto data_cell = create_empty_data_cell(info); - auto* storage = data_cell->get_storage(); - - // init data - auto* data_ptr = info.get_data(storage); - td::BitPtr{data_ptr}.copy_from(data, bits); - // prepare for serialization - if (bits & 7) { - int m = (0x80 >> (bits & 7)); - unsigned l = bits / 8; - data_ptr[l] = static_cast((data_ptr[l] & -m) | m); - } + td::uint16 stored_depth = DataCell::load_depth(data_.ubegin() + depth_offset); + if (stored_depth != refs_[child_idx]->get_depth(0)) { + return td::Status::Error("Invalid depth in a Merkle proof or update"); + } - // init refs - auto refs_ptr = info.get_refs(storage); - for (size_t i = 0; i < refs.size(); i++) { - refs_ptr[i] = refs[i].release(); - } + for (int i = 0; i <= max_level; ++i) { + depth_[i] = std::max(depth_[i], refs_[child_idx]->get_depth(i + 1) + 1); + } - // init hashes and depth - auto* hashes_ptr = info.get_hashes(storage); - auto* depth_ptr = info.get_depth(storage); + return {}; + } - // NB: be careful with special cells - auto total_hash_count = level_mask.get_hashes_count(); - auto hash_i_offset = total_hash_count - hash_count; - for (td::uint32 level_i = 0, hash_i = 0, level = level_mask.get_level(); level_i <= level; level_i++) { - if (!level_mask.is_significant(level_i)) { - continue; + td::Status check_merkle_proof() { + if (refs_cnt_ != 1) { + return td::Status::Error("Merkle proof must have exactly one reference"); } - SCOPE_EXIT { - hash_i++; - }; - if (hash_i < hash_i_offset) { - continue; + if (bit_length_ != 8 * (1 + hash_bytes + depth_bytes)) { + return td::Status::Error("Length mismatch in a Merkle proof"); } - unsigned char tmp[2]; - tmp[0] = info.d1(level_mask.apply(level_i)); - tmp[1] = info.d2(); - digest::SHA256 hasher; + TRY_STATUS(check_merkle_child(0, 1, 1 + hash_bytes)); - hasher.feed(td::Slice(tmp, 2)); + level_mask_ = refs_[0]->get_level_mask().shift_right(); - if (hash_i == hash_i_offset) { - DCHECK(level_i == 0 || type == SpecialType::PrunnedBranch); - hasher.feed(td::Slice(data_ptr, (bits + 7) >> 3)); - } else { - DCHECK(level_i != 0 && type != SpecialType::PrunnedBranch); - hasher.feed(hashes_ptr[hash_i - hash_i_offset - 1].as_slice()); + return {}; + } + + td::Status check_merkle_update() { + if (refs_cnt_ != 2) { + return td::Status::Error("Merkle update must have exactly two references"); + } + if (bit_length_ != 8 * (1 + (hash_bytes + depth_bytes) * 2)) { + return td::Status::Error("Length mismatch in a Merkle update"); } - auto dest_i = hash_i - hash_i_offset; + TRY_STATUS(check_merkle_child(0, 1, 1 + 2 * hash_bytes)); + TRY_STATUS(check_merkle_child(1, 1 + hash_bytes, 1 + 2 * hash_bytes + 2)); - // calc depth - td::uint16 depth = 0; - for (int i = 0; i < info.refs_count_; i++) { - td::uint16 child_depth = 0; - if (type == SpecialType::MerkleProof || type == SpecialType::MerkleUpdate) { - child_depth = refs_ptr[i]->get_depth(level_i + 1); - } else { - child_depth = refs_ptr[i]->get_depth(level_i); - } + level_mask_ = refs_[0]->get_level_mask().apply_or(refs_[1]->get_level_mask()).shift_right(); - // add depth into hash - td::uint8 child_depth_buf[depth_bytes]; - store_depth(child_depth_buf, child_depth); - hasher.feed(td::Slice(child_depth_buf, depth_bytes)); + return {}; + } - depth = std::max(depth, child_depth); + void compute_hash(int level, int last_computed_hash) { + if (level != max_level && type_ == Cell::SpecialType::PrunnedBranch) { + int hashes_before = level_mask_.apply(level).get_hash_i(); + auto offset = 2 + hashes_before * hash_bytes; + std::memcpy(&hash_[level], data_.begin() + offset, hash_bytes); + return; } - if (info.refs_count_ != 0) { - if (depth >= max_depth) { - return td::Status::Error("Depth is too big"); + + digest::SHA256 hasher; + + auto d1 = refs_cnt_ + (is_special_ << 3) + (level_mask_.apply(level).get_mask() << 5); + auto d2 = (bit_length_ >> 3 << 1) + ((bit_length_ & 7) != 0); + td::uint8 header[] = {static_cast(d1), static_cast(d2)}; + hasher.feed(header, 2); + + if (last_computed_hash != -1 && type_ != Cell::SpecialType::PrunnedBranch) { + hasher.feed(hash_[last_computed_hash].as_slice()); + } else { + hasher.feed(data_.substr(0, bit_length_ / 8)); + // If we are not byte-aligned, some bit gymnastics is required to correctly pad the last byte. + if (bit_length_ % 8 != 0) { + td::uint8 last_byte = data_[bit_length_ / 8]; + last_byte >>= 7 - bit_length_ % 8; + last_byte |= 1; + last_byte <<= 7 - bit_length_ % 8; + hasher.feed({&last_byte, 1}); } - depth++; } - depth_ptr[dest_i] = depth; - // children hash - for (int i = 0; i < info.refs_count_; i++) { - if (type == SpecialType::MerkleProof || type == SpecialType::MerkleUpdate) { - hasher.feed(refs_ptr[i]->get_hash(level_i + 1).as_slice()); - } else { - hasher.feed(refs_ptr[i]->get_hash(level_i).as_slice()); - } + bool is_merkle_node = type_ == Cell::SpecialType::MerkleUpdate || type_ == Cell::SpecialType::MerkleProof; + auto child_level = (is_merkle_node ? std::min(max_level, level + 1) : level); + + for (int i = 0; i < refs_cnt_; ++i) { + td::uint8 stored_depth[depth_bytes]; + DataCell::store_depth(stored_depth, refs_[i]->get_depth(child_level)); + + hasher.feed(stored_depth, depth_bytes); + } + + for (int i = 0; i < refs_cnt_; ++i) { + hasher.feed(refs_[i]->get_hash(child_level).as_slice()); } - auto extracted_size = hasher.extract(hashes_ptr[dest_i].as_slice()); - DCHECK(extracted_size == hash_bytes); + + hasher.extract(hash_[level].as_slice()); } - return Ref(data_cell.release(), Ref::acquire_t{}); + bool is_special_; + Cell::SpecialType type_; + td::Span> refs_; + int refs_cnt_; + td::Slice data_; + int bit_length_; + + Cell::LevelMask level_mask_; + td::uint32 virtualization_{0}; + std::array depth_{}; + std::array hash_{}; +}; + +char* allocate_in_arena(size_t size) { + constexpr size_t batch_size = 1 << 20; + thread_local td::MutableSlice batch; + + auto aligned_size = (size + 7) / 8 * 8; + if (batch.size() < size) { + batch = td::MutableSlice(new char[batch_size], batch_size); + } + auto res = batch.begin(); + batch.remove_prefix(aligned_size); + return res; } -const DataCell::Hash DataCell::do_get_hash(td::uint32 level) const { - auto hash_i = get_level_mask().apply(level).get_hash_i(); - if (special_type() == SpecialType::PrunnedBranch) { - auto this_hash_i = get_level_mask().get_hash_i(); - if (hash_i != this_hash_i) { - return reinterpret_cast(info_.get_data(get_storage()) + 2)[hash_i]; - } - hash_i = 0; +} // namespace + +thread_local bool DataCell::use_arena = false; + +td::Result> DataCell::create(td::Slice data, int bit_length, td::Span> refs, bool is_special) { + CHECK(bit_length >= 0 && data.size() * 8 >= static_cast(bit_length)); + if (refs.size() > CellTraits::max_refs) { + return td::Status::Error("Too many references"); + } + if (bit_length > CellTraits::max_bits) { + return td::Status::Error("Too many data bits"); + } + + CellChecker checker{is_special, data, bit_length, refs}; + TRY_STATUS(checker.check_and_compute_level_info()); + + auto level_info_size = sizeof(detail::LevelInfo) * (checker.level_mask().get_level() + 1); + auto cell_size = sizeof(DataCell) + level_info_size + (bit_length + 7) / 8; + + void* storage = use_arena ? allocate_in_arena(cell_size) : ::operator new(cell_size); + DataCell* allocated_cell = new (storage) + DataCell{bit_length, refs.size(), checker.type(), checker.level_mask(), use_arena, checker.virtualization()}; + auto& cell = *allocated_cell; + + auto mutable_data = cell.trailer_ + level_info_size; + + std::memcpy(mutable_data, data.data(), (bit_length + 7) / 8); + if (bit_length % 8 != 0) { + auto& last_byte = mutable_data[bit_length / 8]; + // This is the same padding as was used in CellChecker::compute_hash above. + last_byte >>= (7 - bit_length % 8); + last_byte |= 1; + last_byte <<= (7 - bit_length % 8); + } + + auto level_info = new (cell.trailer_) detail::LevelInfo[checker.level_mask().get_level() + 1]; + + for (int i = 0; i <= cell.level_; ++i) { + level_info[i] = { + .hash = checker.hashes()[i], + .depth = checker.depths()[i], + }; + } + + for (int i = 0; i < cell.refs_cnt_; ++i) { + cell.refs_[i] = refs[i]; } - return info_.get_hashes(get_storage())[hash_i]; + + return Ref{allocated_cell, Ref::acquire_t{}}; } -td::uint16 DataCell::do_get_depth(td::uint32 level) const { - auto hash_i = get_level_mask().apply(level).get_hash_i(); - if (special_type() == SpecialType::PrunnedBranch) { - auto this_hash_i = get_level_mask().get_hash_i(); - if (hash_i != this_hash_i) { - return load_depth(info_.get_data(get_storage()) + 2 + hash_bytes * this_hash_i + hash_i * depth_bytes); - } - hash_i = 0; +DataCell::~DataCell() { + for (size_t i = 0; i < level_ + 1; ++i) { + level_info()[i].~LevelInfo(); } - return info_.get_depth(get_storage())[hash_i]; + get_thread_safe_counter().add(-1); } int DataCell::serialize(unsigned char* buff, int buff_size, bool with_hashes) const { @@ -352,8 +385,8 @@ int DataCell::serialize(unsigned char* buff, int buff_size, bool with_hashes) co if (len > buff_size) { return 0; } - buff[0] = static_cast(info_.d1() | (with_hashes * 16)); - buff[1] = info_.d2(); + buff[0] = static_cast(construct_d1(max_level) | (with_hashes * 16)); + buff[1] = construct_d2(); int hs = 0; if (with_hashes) { hs = (get_level_mask().get_hashes_count()) * (hash_bytes + depth_bytes); @@ -361,7 +394,7 @@ int DataCell::serialize(unsigned char* buff, int buff_size, bool with_hashes) co std::memset(buff + 2, 0, hs); auto dest = td::MutableSlice(buff + 2, hs); auto level = get_level(); - // TODO: optimize for prunned brandh + // TODO: optimize for pruned branch for (unsigned i = 0; i <= level; i++) { if (!get_level_mask().is_significant(i)) { continue; @@ -376,7 +409,6 @@ int DataCell::serialize(unsigned char* buff, int buff_size, bool with_hashes) co store_depth(dest.ubegin(), get_depth(i)); dest.remove_prefix(depth_bytes); } - // buff[2] = 0; // for testing hash verification in deserialization buff += hs; len -= hs; } @@ -400,8 +432,16 @@ std::string DataCell::to_hex() const { return hex_buff; } -std::ostream& operator<<(std::ostream& os, const DataCell& c) { - return os << c.to_hex(); +DataCell::DataCell(int bit_length, size_t refs_cnt, Cell::SpecialType type, LevelMask level_mask, + bool allocated_in_arena, td::uint8 virtualization) + : bit_length_(bit_length) + , refs_cnt_(static_cast(refs_cnt)) + , type_(static_cast(type)) + , level_(static_cast(level_mask.get_level())) + , level_mask_(level_mask.get_mask()) + , allocated_in_arena_(allocated_in_arena) + , virtualization_(virtualization) { + get_thread_safe_counter().add(1); } } // namespace vm diff --git a/crypto/vm/cells/DataCell.h b/crypto/vm/cells/DataCell.h index b39ee1d4b2..172b3390a7 100644 --- a/crypto/vm/cells/DataCell.h +++ b/crypto/vm/cells/DataCell.h @@ -17,219 +17,197 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "vm/cells/Cell.h" #include "td/utils/Span.h" - #include "td/utils/ThreadSafeCounter.h" +#include "vm/cells/Cell.h" namespace vm { -class DataCell : public Cell { +namespace detail { + +struct LevelInfo { + CellHash hash; + td::uint16 depth; +}; + +} // namespace detail + +class DataCell final : public Cell { public: // NB: cells created with use_arena=true are never freed static thread_local bool use_arena; - DataCell(const DataCell& other) = delete; - DataCell(DataCell&& other) = delete; - DataCell& operator=(const DataCell& other) = delete; - DataCell& operator=(DataCell&& other) = delete; - ~DataCell() override; + static td::Result> create(td::Slice data, int bit_length, td::Span> refs, bool is_special); static void store_depth(td::uint8* dest, td::uint16 depth) { td::bitstring::bits_store_long(dest, depth, depth_bits); } + static td::uint16 load_depth(const td::uint8* src) { return td::bitstring::bits_load_ulong(src, depth_bits) & 0xffff; } - protected: - struct Info { - unsigned bits_; - - // d1 - unsigned char refs_count_ : 3; - bool is_special_ : 1; - unsigned char level_mask_ : 3; - - unsigned char hash_count_ : 3; - - unsigned char virtualization_ : 3; - - unsigned char d1() const { - return d1(LevelMask{level_mask_}); - } - unsigned char d1(LevelMask level_mask) const { - // d1 = refs_count + 8 * is_special + 32 * level - // + 16 * with_hashes - for seriazlization - // d1 = 7 + 16 + 32 * l - for absent cells - return static_cast(refs_count_ + 8 * is_special_ + 32 * level_mask.get_mask()); - } - unsigned char d2() const { - auto res = static_cast((bits_ / 8) * 2); - if ((bits_ & 7) != 0) { - return static_cast(res + 1); - } - return res; - } - size_t get_hashes_offset() const { - return 0; - } - size_t get_refs_offset() const { - return get_hashes_offset() + hash_bytes * hash_count_; - } - size_t get_depth_offset() const { - return get_refs_offset() + refs_count_ * sizeof(Cell*); - } - size_t get_data_offset() const { - return get_depth_offset() + sizeof(td::uint16) * hash_count_; - } - size_t get_storage_size() const { - return get_data_offset() + (bits_ + 7) / 8; - } - - const Hash* get_hashes(const char* storage) const { - return reinterpret_cast(storage + get_hashes_offset()); - } - - Hash* get_hashes(char* storage) const { - return reinterpret_cast(storage + get_hashes_offset()); - } - - const td::uint16* get_depth(const char* storage) const { - return reinterpret_cast(storage + get_depth_offset()); + void operator delete(DataCell* ptr, std::destroying_delete_t) { + bool allocated_in_arena = ptr->allocated_in_arena_; + ptr->~DataCell(); + if (!allocated_in_arena) { + ::operator delete(ptr); } + } - td::uint16* get_depth(char* storage) const { - return reinterpret_cast(storage + get_depth_offset()); - } + DataCell(DataCell const&) = delete; + DataCell(DataCell&&) = delete; - const unsigned char* get_data(const char* storage) const { - return reinterpret_cast(storage + get_data_offset()); - } - unsigned char* get_data(char* storage) const { - return reinterpret_cast(storage + get_data_offset()); - } + ~DataCell(); - Cell* const* get_refs(const char* storage) const { - return reinterpret_cast(storage + get_refs_offset()); - } - Cell** get_refs(char* storage) const { - return reinterpret_cast(storage + get_refs_offset()); - } - }; + virtual td::Status set_data_cell(Ref&& data_cell) const override { + CHECK(get_hash() == data_cell->get_hash()); + return td::Status::OK(); + } - Info info_; - virtual char* get_storage() = 0; - virtual const char* get_storage() const = 0; - // TODO: we may also save three different pointers + virtual td::Result load_cell() const override { + return LoadedCell{ + .data_cell = Ref{this}, + .virt = {}, + .tree_node = {}, + }; + } - void destroy_storage(char* storage); + virtual td::uint32 get_virtualization() const override { + return virtualization_; + } - explicit DataCell(Info info); + virtual CellUsageTree::NodePtr get_tree_node() const override { + return {}; + } - public: - td::Status set_data_cell(Ref&& data_cell) const override { - CHECK(get_hash() == data_cell->get_hash()); - return td::Status::OK(); + virtual bool is_loaded() const override { + return true; } - td::Result load_cell() const override { - return LoadedCell{Ref{this}, {}, {}}; + + virtual LevelMask get_level_mask() const override { + return LevelMask{level_mask_}; } + unsigned get_refs_cnt() const { - return info_.refs_count_; + return refs_cnt_; } + unsigned get_bits() const { - return info_.bits_; + return bit_length_; } + unsigned size_refs() const { - return info_.refs_count_; + return refs_cnt_; } + unsigned size() const { - return info_.bits_; + return bit_length_; } - const unsigned char* get_data() const { - return info_.get_data(get_storage()); + + unsigned char const* get_data() const { + return reinterpret_cast(trailer_ + sizeof(detail::LevelInfo) * (level_ + 1)); } + Ref get_ref(unsigned idx) const { - if (idx >= get_refs_cnt()) { - return Ref{}; + if (idx >= refs_cnt_) { + return {}; } - return Ref(get_ref_raw_ptr(idx)); + return refs_[idx]; } Cell* get_ref_raw_ptr(unsigned idx) const { - DCHECK(idx < get_refs_cnt()); - return info_.get_refs(get_storage())[idx]; + DCHECK(idx < refs_cnt_); + return const_cast(refs_[idx].get()); } Ref reset_ref_unsafe(unsigned idx, Ref ref, bool check_hash = true) { CHECK(idx < get_refs_cnt()); - auto refs = info_.get_refs(get_storage()); - CHECK(!check_hash || refs[idx]->get_hash() == ref->get_hash()); - auto res = Ref(refs[idx], Ref::acquire_t{}); // call destructor - refs[idx] = ref.release(); - return res; + CHECK(!check_hash || refs_[idx]->get_hash() == ref->get_hash()); + return std::exchange(refs_[idx], std::move(ref)); } - td::uint32 get_virtualization() const override { - return info_.virtualization_; - } - CellUsageTree::NodePtr get_tree_node() const override { - return {}; - } - bool is_loaded() const override { - return true; - } - LevelMask get_level_mask() const override { - return LevelMask{info_.level_mask_}; + bool is_special() const { + return type_ != static_cast(SpecialType::Ordinary); } - bool is_special() const { - return info_.is_special_; + SpecialType special_type() const { + return static_cast(type_); } - SpecialType special_type() const; + int get_serialized_size(bool with_hashes = false) const { return ((get_bits() + 23) >> 3) + (with_hashes ? get_level_mask().get_hashes_count() * (hash_bytes + depth_bytes) : 0); } + size_t get_storage_size() const { - return info_.get_storage_size(); + return sizeof(DataCell) + sizeof(detail::LevelInfo) * (level_ + 1) + (bit_length_ + 7) / 8; } + int serialize(unsigned char* buff, int buff_size, bool with_hashes = false) const; + std::string serialize() const; + std::string to_hex() const; + static td::int64 get_total_data_cells() { return get_thread_safe_counter().sum(); } template void store(StorerT& storer) const { - storer.template store_binary(info_.d1()); - storer.template store_binary(info_.d2()); + storer.template store_binary(construct_d1(max_level)); + storer.template store_binary(construct_d2()); storer.store_slice(td::Slice(get_data(), (get_bits() + 7) / 8)); } - protected: - static constexpr auto max_storage_size = max_refs * sizeof(void*) + (max_level + 1) * hash_bytes + max_bytes; - private: static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() { static auto res = td::NamedThreadSafeCounter::get_default().get_counter("DataCell"); return res; } - static std::unique_ptr create_empty_data_cell(Info info); - const Hash do_get_hash(td::uint32 level) const override; - td::uint16 do_get_depth(td::uint32 level) const override; + DataCell(int bit_length, size_t refs_cnt, Cell::SpecialType type, LevelMask level_mask, bool allocated_in_arena, + td::uint8 virtualization); - friend class CellBuilder; - static td::Result> create(td::ConstBitPtr data, unsigned bits, td::Span> refs, bool special); - static td::Result> create(td::ConstBitPtr data, unsigned bits, td::MutableSpan> refs, - bool special); + detail::LevelInfo const* level_info() const { + return reinterpret_cast(trailer_); + } + + virtual td::uint16 do_get_depth(td::uint32 level) const override { + return level_info()[std::min(level_, level)].depth; + } + + virtual const Hash do_get_hash(td::uint32 level) const override { + return level_info()[std::min(level_, level)].hash; + } + + td::uint8 construct_d1(td::uint32 level) const { + return static_cast(refs_cnt_ + (is_special() << 3) + (get_level_mask().apply(level).get_mask() << 5)); + } + + td::uint8 construct_d2() const { + return static_cast(bit_length_ / 8 + (bit_length_ + 7) / 8); + } + + unsigned bit_length_ : 11; + unsigned refs_cnt_ : 3; + unsigned type_ : 3; + unsigned level_ : 3; + unsigned level_mask_ : 3; + unsigned allocated_in_arena_ : 1; + unsigned virtualization_ : 8; + + std::array, max_refs> refs_{}; + + alignas(detail::LevelInfo) char trailer_[]; }; -std::ostream& operator<<(std::ostream& os, const DataCell& c); +inline std::ostream& operator<<(std::ostream& os, const DataCell& c) { + return os << c.to_hex(); +} + inline CellHash as_cell_hash(const Ref& cell) { return cell->get_hash(); } diff --git a/crypto/vm/cells/PrunnedCell.h b/crypto/vm/cells/PrunnedCell.h index 6e8b77093d..f4d40c2853 100644 --- a/crypto/vm/cells/PrunnedCell.h +++ b/crypto/vm/cells/PrunnedCell.h @@ -17,8 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "vm/cells/CellWithStorage.h" -#include "vm/cells/Cell.h" + +#include "vm/cells/DataCell.h" namespace vm { struct PrunnedCellInfo { @@ -28,7 +28,7 @@ struct PrunnedCellInfo { }; template -class PrunnedCell : public Cell { +class PrunnedCell final : public Cell { public: ExtraT& get_extra() { return extra_; @@ -37,22 +37,33 @@ class PrunnedCell : public Cell { return extra_; } + void operator delete(PrunnedCell* ptr, std::destroying_delete_t) { + bool allocated_in_arena = ptr->info_.allocated_in_arena_; + ptr->~PrunnedCell(); + if (!allocated_in_arena) { + ::operator delete(ptr); + } + } + static td::Result>> create(const PrunnedCellInfo& prunned_cell_info, ExtraT&& extra) { - return create(detail::DefaultAllocator>(), prunned_cell_info, std::forward(extra)); + auto allocator = [](size_t bytes) { return ::operator new(bytes); }; + return create(allocator, true, prunned_cell_info, std::move(extra)); } - template - static td::Result>> create(AllocatorT allocator, const PrunnedCellInfo& prunned_cell_info, - ExtraT&& extra) { + template + static td::Result>> create(AllocatorFunc&& allocator, bool should_free, + const PrunnedCellInfo& prunned_cell_info, ExtraT&& extra) { auto level_mask = prunned_cell_info.level_mask; if (level_mask.get_level() > max_level) { return td::Status::Error("Level is too big"); } Info info(level_mask); - auto prunned_cell = - detail::CellWithArrayStorage>::create(allocator, info.get_storage_size(), info, std::move(extra)); - TRY_STATUS(prunned_cell->init(prunned_cell_info)); - return Ref>(prunned_cell.release(), typename Ref>::acquire_t{}); + + auto storage = allocator(sizeof(PrunnedCell) + info.get_storage_size()); + auto* result = new (storage) PrunnedCell{info, std::move(extra)}; + result->info_.allocated_in_arena_ = !should_free; + TRY_STATUS(result->init(prunned_cell_info)); + return Ref>(result, typename Ref>::acquire_t{}); } LevelMask get_level_mask() const override { @@ -68,6 +79,7 @@ class PrunnedCell : public Cell { } unsigned char level_mask_ : 3; unsigned char hash_count_ : 3; + unsigned char allocated_in_arena_ : 1; size_t get_hashes_offset() const { return 0; } @@ -93,16 +105,10 @@ class PrunnedCell : public Cell { Info info_; ExtraT extra_; - virtual char* get_storage() = 0; - virtual const char* get_storage() const = 0; - void destroy_storage(char* storage) { - // noop - } td::Status init(const PrunnedCellInfo& prunned_cell_info) { - auto storage = get_storage(); auto& new_hash = prunned_cell_info.hash; - auto* hash = info_.get_hashes(storage); + auto* hash = info_.get_hashes(trailer_); size_t n = prunned_cell_info.level_mask.get_hashes_count(); CHECK(new_hash.size() == n * hash_bytes); for (td::uint32 i = 0; i < n; i++) { @@ -111,7 +117,7 @@ class PrunnedCell : public Cell { auto& new_depth = prunned_cell_info.depth; CHECK(new_depth.size() == n * depth_bytes); - auto* depth = info_.get_depth(storage); + auto* depth = info_.get_depth(trailer_); for (td::uint32 i = 0; i < n; i++) { depth[i] = DataCell::load_depth(new_depth.substr(i * Cell::depth_bytes, Cell::depth_bytes).ubegin()); if (depth[i] > max_depth) { @@ -135,11 +141,11 @@ class PrunnedCell : public Cell { private: const Hash do_get_hash(td::uint32 level) const override { - return info_.get_hashes(get_storage())[get_level_mask().apply(level).get_hash_i()]; + return info_.get_hashes(trailer_)[get_level_mask().apply(level).get_hash_i()]; } td::uint16 do_get_depth(td::uint32 level) const override { - return info_.get_depth(get_storage())[get_level_mask().apply(level).get_hash_i()]; + return info_.get_depth(trailer_)[get_level_mask().apply(level).get_hash_i()]; } td::Status set_data_cell(Ref &&data_cell) const override { @@ -149,5 +155,7 @@ class PrunnedCell : public Cell { td::Result load_cell() const override { return td::Status::Error("Can't load prunned branch"); } + + char trailer_[]; }; } // namespace vm diff --git a/crypto/vm/db/InMemoryBagOfCellsDb.cpp b/crypto/vm/db/InMemoryBagOfCellsDb.cpp index e43cfde4e0..44be4ca8b5 100644 --- a/crypto/vm/db/InMemoryBagOfCellsDb.cpp +++ b/crypto/vm/db/InMemoryBagOfCellsDb.cpp @@ -157,17 +157,9 @@ class ArenaPrunnedCellCreator : public ExtCellCreator { } }; - struct Allocator { - template - std::unique_ptr> make_unique(ArgsT &&...args) { - auto *ptr = arena_.alloc(sizeof(T)); - T *obj = new (ptr) T(std::forward(args)...); - return std::unique_ptr(obj); - } - }; td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { - Allocator allocator; - TRY_RESULT(cell, PrunnedCell::create(allocator, PrunnedCellInfo{level_mask, hash, depth}, Counter())); + TRY_RESULT(cell, PrunnedCell::create([&](size_t bytes) { return arena_.alloc(bytes); }, false, + PrunnedCellInfo{level_mask, hash, depth}, Counter())); return cell; } static td::int64 count() { @@ -601,7 +593,7 @@ class CellStorage { std::vector bucket_stats(buckets_.size()); std::atomic boc_count{0}; for_each_bucket(options.extra_threads, [&](size_t bucket_id, auto &bucket) { - bucket_stats[bucket_id] = validate_bucket_a(bucket, options.use_arena); + bucket_stats[bucket_id] = validate_bucket_a(bucket); boc_count += bucket.boc_count_; }); for_each_bucket(options.extra_threads, [&](size_t bucket_id, auto &bucket) { validate_bucket_b(bucket); }); @@ -730,12 +722,12 @@ class CellStorage { }); } - DynamicBagOfCellsDb::Stats validate_bucket_a(CellBucket &bucket, bool use_arena) { + DynamicBagOfCellsDb::Stats validate_bucket_a(CellBucket &bucket) { DynamicBagOfCellsDb::Stats stats; bucket.infos_.for_each([&](auto &it) { int cell_ref_cnt = it.cell->get_refcnt(); - CHECK(it.db_refcnt + 1 + use_arena >= cell_ref_cnt); - auto extra_refcnt = it.db_refcnt + 1 + use_arena - cell_ref_cnt; + CHECK(it.db_refcnt + 1 >= cell_ref_cnt); + auto extra_refcnt = it.db_refcnt + 1 - cell_ref_cnt; if (extra_refcnt != 0) { bucket.roots_.push_back(it.cell); stats.roots_total_count++; diff --git a/crypto/vm/db/StaticBagOfCellsDb.cpp b/crypto/vm/db/StaticBagOfCellsDb.cpp index c65d2624ff..215dcfa3bc 100644 --- a/crypto/vm/db/StaticBagOfCellsDb.cpp +++ b/crypto/vm/db/StaticBagOfCellsDb.cpp @@ -18,7 +18,6 @@ */ #include "vm/db/StaticBagOfCellsDb.h" -#include "vm/cells/CellWithStorage.h" #include "vm/boc.h" #include "vm/cells/ExtCell.h" diff --git a/tdutils/td/utils/ThreadSafeCounter.h b/tdutils/td/utils/ThreadSafeCounter.h index 46dc16bf70..8601275a9e 100644 --- a/tdutils/td/utils/ThreadSafeCounter.h +++ b/tdutils/td/utils/ThreadSafeCounter.h @@ -20,10 +20,12 @@ #pragma once #include "port/thread.h" -#include "td/utils/common.h" #include "td/utils/Slice.h" #include "td/utils/StringBuilder.h" #include "td/utils/ThreadLocalStorage.h" +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/port/Clocks.h" #include #include From 9c9198253fc600b5ec69c7c298a7d2244803ae6f Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Sat, 29 Mar 2025 15:48:00 -0400 Subject: [PATCH 186/524] Avoid CellBuilder during deserialization in CellSerializationInfo Another free 2.5% of speed-up on block verification. --- crypto/vm/boc.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 72afb99885..6377674d7e 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -89,14 +89,9 @@ td::Result CellSerializationInfo::get_bits(td::Slice cell) const { // TODO: check usage when result is empty td::Result> CellSerializationInfo::create_data_cell(td::Slice cell_slice, td::Span> refs) const { - CellBuilder cb; - TRY_RESULT(bits, get_bits(cell_slice)); - cb.store_bits(cell_slice.ubegin() + data_offset, bits); DCHECK(refs_cnt == (td::int64)refs.size()); - for (int k = 0; k < refs_cnt; k++) { - cb.store_ref(std::move(refs[k])); - } - TRY_RESULT(res, cb.finalize_novm_nothrow(special)); + TRY_RESULT(bits, get_bits(cell_slice)); + TRY_RESULT(res, DataCell::create(cell_slice.substr(data_offset), bits, refs, special)); CHECK(!res.is_null()); if (res->is_special() != special) { return td::Status::Error("is_special mismatch"); From 22003ae9ff1859b7f64f72df02ca41b4079a47d7 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Mon, 31 Mar 2025 14:35:01 -0400 Subject: [PATCH 187/524] Do not recompute validator set in MasterchainStateQ::mc_reinit This does not improve performance (as unpack_validator_set result is cached regardless) but makes the code clearer. --- crypto/block/mc-config.h | 4 ++-- validator/impl/shard.cpp | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index a44522f11c..8af2877dbf 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -643,8 +643,8 @@ class Config { const WorkchainSet& get_workchain_list() const { return workchains_; } - const ValidatorSet* get_cur_validator_set() const { - return cur_validators_.get(); + std::shared_ptr const& get_cur_validator_set() const& { + return cur_validators_; } std::pair get_validator_set_start_stop(int next = 0) const; ton::ValidatorSessionConfig get_consensus_config() const; diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index d2d402dd66..22091a0f71 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -388,11 +388,8 @@ td::Status MasterchainStateQ::mc_reinit() { CHECK(config_); CHECK(config_->set_block_id_ext(get_block_id())); - auto cv_root = config_->get_config_param(35, 34); - if (cv_root.not_null()) { - TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(cv_root), true)); - cur_validators_ = std::move(validators); - } + cur_validators_ = config_->get_cur_validator_set(); + auto nv_root = config_->get_config_param(37, 36); if (nv_root.not_null()) { TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root), true)); From 1ac7247fbbb95840f662105ca14a25e5a090a61e Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Mon, 31 Mar 2025 14:30:35 -0400 Subject: [PATCH 188/524] Do a single SHA256::feed call in CellChecker::compute_hash This speeds up DataCell::create by 20% and the verification benchmark by 3.5%. --- crypto/vm/cells/DataCell.cpp | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/crypto/vm/cells/DataCell.cpp b/crypto/vm/cells/DataCell.cpp index 14da8e85ac..b6ff8021f3 100644 --- a/crypto/vm/cells/DataCell.cpp +++ b/crypto/vm/cells/DataCell.cpp @@ -258,24 +258,33 @@ class CellChecker { return; } - digest::SHA256 hasher; + static_assert(2 + CellTraits::max_bytes + CellTraits::max_refs * (hash_bytes + depth_bytes) <= 512); + char data_to_hash[512]; + int pointer = 0; + + auto add_byte_to_hash = [&](char byte) { data_to_hash[pointer++] = byte; }; + + auto add_slice_to_hash = [&](td::Slice slice) { + std::memcpy(data_to_hash + pointer, slice.data(), slice.size()); + pointer += slice.size(); + }; auto d1 = refs_cnt_ + (is_special_ << 3) + (level_mask_.apply(level).get_mask() << 5); + add_byte_to_hash(static_cast(d1)); auto d2 = (bit_length_ >> 3 << 1) + ((bit_length_ & 7) != 0); - td::uint8 header[] = {static_cast(d1), static_cast(d2)}; - hasher.feed(header, 2); + add_byte_to_hash(static_cast(d2)); if (last_computed_hash != -1 && type_ != Cell::SpecialType::PrunnedBranch) { - hasher.feed(hash_[last_computed_hash].as_slice()); + add_slice_to_hash(hash_[last_computed_hash].as_slice()); } else { - hasher.feed(data_.substr(0, bit_length_ / 8)); + add_slice_to_hash(data_.substr(0, bit_length_ / 8)); // If we are not byte-aligned, some bit gymnastics is required to correctly pad the last byte. if (bit_length_ % 8 != 0) { td::uint8 last_byte = data_[bit_length_ / 8]; last_byte >>= 7 - bit_length_ % 8; last_byte |= 1; last_byte <<= 7 - bit_length_ % 8; - hasher.feed({&last_byte, 1}); + add_byte_to_hash(last_byte); } } @@ -283,16 +292,17 @@ class CellChecker { auto child_level = (is_merkle_node ? std::min(max_level, level + 1) : level); for (int i = 0; i < refs_cnt_; ++i) { - td::uint8 stored_depth[depth_bytes]; - DataCell::store_depth(stored_depth, refs_[i]->get_depth(child_level)); - - hasher.feed(stored_depth, depth_bytes); + auto depth = refs_[i]->get_depth(child_level); + add_byte_to_hash((depth >> 8) & 255); + add_byte_to_hash((depth >> 0) & 255); } for (int i = 0; i < refs_cnt_; ++i) { - hasher.feed(refs_[i]->get_hash(child_level).as_slice()); + add_slice_to_hash(refs_[i]->get_hash(child_level).as_slice()); } + digest::SHA256 hasher; + hasher.feed(data_to_hash, pointer); hasher.extract(hash_[level].as_slice()); } From c69315fee589cf298de4f2066485ca33c7a76f2c Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Tue, 1 Apr 2025 08:32:13 -0400 Subject: [PATCH 189/524] Set CMAKE_POLICY_VERSION_MINIMUM to 3.10 for crc32 This fixes Wasm CI job. --- CMakeLists.txt | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c65d66a95..ad1aaf5063 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -87,16 +87,15 @@ set(CRC32C_BUILD_BENCHMARKS OFF CACHE BOOL "Build CRC32C's benchmarks") set(CRC32C_USE_GLOG OFF CACHE BOOL "Build CRC32C's tests with Google Logging") set(CRC32C_INSTALL OFF CACHE BOOL "Install CRC32C's header and library") message("Add crc32c") -if (NOT MSVC) - set(OLD_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) - # fix aarch64 build @ crc32c/src/crc32c_arm64_linux_check.h - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=address") - add_subdirectory(third-party/crc32c EXCLUDE_FROM_ALL) - set(CMAKE_CXX_FLAGS ${OLD_CMAKE_CXX_FLAGS}) - unset(OLD_CMAKE_CXX_FLAGS) -else() +function(crc32_scope) + set(CMAKE_POLICY_VERSION_MINIMUM "3.10") + if (NOT MSVC) + # fix aarch64 build @ crc32c/src/crc32c_arm64_linux_check.h + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=address") + endif() add_subdirectory(third-party/crc32c EXCLUDE_FROM_ALL) -endif() +endfunction() +crc32_scope() set(CRC32C_FOUND 1) if (TON_USE_ROCKSDB) From 751ce0e044edb981fbe280f12af3edab776a90ba Mon Sep 17 00:00:00 2001 From: neodix42 Date: Tue, 1 Apr 2025 21:46:51 +0400 Subject: [PATCH 190/524] Merge pull request #1598 from neodix42/improve-docker-img-1 Speed up docker image arm64 build --- .../workflows/docker-ubuntu-branch-image.yml | 83 +++++++++++++--- .github/workflows/docker-ubuntu-image.yml | 97 ++++++++++++++++--- 2 files changed, 154 insertions(+), 26 deletions(-) diff --git a/.github/workflows/docker-ubuntu-branch-image.yml b/.github/workflows/docker-ubuntu-branch-image.yml index 00aa5015e2..bd97ed233f 100644 --- a/.github/workflows/docker-ubuntu-branch-image.yml +++ b/.github/workflows/docker-ubuntu-branch-image.yml @@ -11,21 +11,26 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: - build-and-push: - runs-on: ubuntu-22.04 + build-arm64: + runs-on: ubuntu-22.04-arm steps: - name: Check out repository uses: actions/checkout@v3 with: submodules: 'recursive' + - name: Get tag as branch name + id: tag + run: | + echo "TAG=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT + - name: Set up QEMU uses: docker/setup-qemu-action@v3.5.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3.10.0 with: - driver-opts: image=moby/buildkit:v0.11.0 + driver-opts: image=moby/buildkit:v0.20.2 - name: Login to GitHub Container Registry uses: docker/login-action@v3 @@ -34,28 +39,80 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and export to Docker + - name: Build and push uses: docker/build-push-action@v6 with: - load: true + platforms: linux/arm64 + push: true context: ./ - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-arm64 - - name: Test - run: | - docker run --rm -e "TEST=1" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test + + build-amd64: + runs-on: ubuntu-22.04 + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' - name: Get tag as branch name id: tag run: | echo "TAG=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.5.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.10.0 + with: + driver-opts: image=moby/buildkit:v0.20.2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push - id: docker_build uses: docker/build-push-action@v6 with: - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 push: true context: ./ - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }} + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-amd64 + + + merge: + runs-on: ubuntu-22.04 + needs: + - build-amd64 + - build-arm64 + steps: + - name: Get tag as branch name + id: tag + run: | + echo "TAG=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.10.0 + + - name: Create manifest list and push + run: | + docker buildx imagetools create \ + --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }} \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-arm64 \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-amd64 + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }} diff --git a/.github/workflows/docker-ubuntu-image.yml b/.github/workflows/docker-ubuntu-image.yml index aa4eaeef5b..0284dcd545 100644 --- a/.github/workflows/docker-ubuntu-image.yml +++ b/.github/workflows/docker-ubuntu-image.yml @@ -11,8 +11,8 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: - build-and-push: - runs-on: ubuntu-22.04 + build-arm64: + runs-on: ubuntu-22.04-arm steps: - name: Check out repository uses: actions/checkout@v3 @@ -32,16 +32,47 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and export to Docker + - name: Get next tag + id: tag + run: | + git fetch --all --tags + git tag -l + NEW_TAG=v$(date +'%Y.%m') + FOUND=$(git tag -l | grep $NEW_TAG | wc -l) + if [ $FOUND -eq 0 ]; then + echo "TAG=$NEW_TAG" >> $GITHUB_OUTPUT + else + echo "TAG=$NEW_TAG-$FOUND" >> $GITHUB_OUTPUT + fi + + - name: Build and push uses: docker/build-push-action@v6 with: - load: true + platforms: linux/arm64 + push: true context: ./ - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-arm64 - - name: Test - run: | - docker run --rm -e "TEST=1" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test + build-amd64: + runs-on: ubuntu-22.04 + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.5.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.10.0 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Get next tag id: tag @@ -57,12 +88,52 @@ jobs: fi - name: Build and push - id: docker_build uses: docker/build-push-action@v6 with: - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 push: true context: ./ - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }} + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-amd64 + + merge: + runs-on: ubuntu-22.04 + needs: + - build-amd64 + - build-arm64 + steps: + - name: Check out repository + uses: actions/checkout@v3 + + - name: Get next tag + id: tag + run: | + git fetch --all --tags + git tag -l + NEW_TAG=v$(date +'%Y.%m') + FOUND=$(git tag -l | grep $NEW_TAG | wc -l) + if [ $FOUND -eq 0 ]; then + echo "TAG=$NEW_TAG" >> $GITHUB_OUTPUT + else + echo "TAG=$NEW_TAG-$FOUND" >> $GITHUB_OUTPUT + fi + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.10.0 + + - name: Create manifest list and push + run: | + docker buildx imagetools create \ + --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-arm64 \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-amd64 + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest From 4366b6a66ab49f70a62646823a4178df95cbbabb Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 2 Apr 2025 11:01:46 +0300 Subject: [PATCH 191/524] Don't calculate account storage dict when not needed --- crypto/block/account-storage-stat.cpp | 153 +++++++++++++++----------- crypto/block/account-storage-stat.h | 72 ++++-------- crypto/block/transaction.cpp | 42 ++++--- 3 files changed, 142 insertions(+), 125 deletions(-) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index 918d2ac005..9fab8f8802 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -23,44 +23,29 @@ AccountStorageStat::AccountStorageStat() : AccountStorageStat({}, {}, 0, 0) { AccountStorageStat::AccountStorageStat(Ref dict_root, std::vector> roots, td::uint64 total_cells, td::uint64 total_bits) - : dict_(std::move(dict_root), 256), roots_(std::move(roots)), total_cells_(total_cells), total_bits_(total_bits) { + : dict_(std::move(dict_root), 256), total_cells_(total_cells), total_bits_(total_bits), roots_(std::move(roots)) { } -AccountStorageStat::AccountStorageStat(const AccountStorageStat& other) - : AccountStorageStat(other.dict_.get_root_cell(), other.roots_, other.total_cells_, other.total_bits_) { +AccountStorageStat::AccountStorageStat(const AccountStorageStat* parent) + : dict_(parent->dict_) + , dict_up_to_date_(parent->dict_up_to_date_) + , total_cells_(parent->total_cells_) + , total_bits_(parent->total_bits_) + , roots_(parent->roots_) + , parent_(parent) { + CHECK(parent_->parent_ == nullptr); } -AccountStorageStat::AccountStorageStat(AccountStorageStat&& other) - : AccountStorageStat(other.dict_.get_root_cell(), std::move(other.roots_), other.total_cells_, other.total_bits_) { - cache_ = std::move(other.cache_); -} - -AccountStorageStat& AccountStorageStat::operator=(const AccountStorageStat& other) { - dict_ = other.dict_; - total_cells_ = other.total_cells_; - total_bits_ = other.total_bits_; - roots_ = other.roots_; - cache_ = {}; - return *this; -} - -AccountStorageStat& AccountStorageStat::operator=(AccountStorageStat&& other) { - dict_ = std::move(other.dict_); - total_cells_ = other.total_cells_; - total_bits_ = other.total_bits_; - roots_ = std::move(other.roots_); - cache_ = std::move(other.cache_); - return *this; -} - -td::Result AccountStorageStat::replace_roots(std::vector> new_roots) { +td::Status AccountStorageStat::replace_roots(std::vector> new_roots, bool check_merkle_depth) { std::erase_if(new_roots, [](const Ref& c) { return c.is_null(); }); if (new_roots.empty()) { roots_.clear(); total_bits_ = total_cells_ = 0; dict_ = vm::Dictionary{256}; cache_ = {}; - return CellInfo{}; + dict_up_to_date_ = true; + parent_ = nullptr; + return td::Status::OK(); } auto cmp = [](const Ref& c1, const Ref& c2) { return c1->get_hash() < c2->get_hash(); }; @@ -71,29 +56,26 @@ td::Result AccountStorageStat::replace_roots(std:: cmp); std::set_difference(roots_.begin(), roots_.end(), new_roots.begin(), new_roots.end(), std::back_inserter(to_del), cmp); + if (to_add.empty() && to_del.empty()) { + return td::Status::OK(); + } - td::uint32 max_merkle_depth = 0; for (const Ref& root : to_add) { TRY_RESULT(info, add_cell(root)); - max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); + if (check_merkle_depth && info.max_merkle_depth > MAX_MERKLE_DEPTH) { + return td::Status::Error("too big Merkle depth"); + } } for (const Ref& root : to_del) { TRY_STATUS(remove_cell(root)); } roots_ = std::move(new_roots); - td::Status S = td::Status::OK(); - std::vector>> values; - cache_.for_each([&](Entry& e) { - if (S.is_ok()) { - S = commit_entry(e, values); - } - }); - TRY_STATUS(std::move(S)); - if (!dict_.multiset(values)) { - return td::Status::Error("failed to update dictionary"); + dict_up_to_date_ = false; + for (auto& [_, e] : cache_) { + TRY_STATUS(finalize_entry(e)); } - return CellInfo{max_merkle_depth}; + return td::Status::OK(); } void AccountStorageStat::add_hint(const td::HashSet& hint) { @@ -105,7 +87,7 @@ void AccountStorageStat::add_hint(const td::HashSet& hint) { Entry& e = get_entry(cell); e.exists = e.exists_known = true; if (is_root) { - fetch_entry(e).ignore(); + fetch_from_dict(e).ignore(); if (e.max_merkle_depth && e.max_merkle_depth.value() != 0) { return; } @@ -126,12 +108,12 @@ void AccountStorageStat::add_hint(const td::HashSet& hint) { td::Result AccountStorageStat::add_cell(const Ref& cell) { Entry& e = get_entry(cell); if (!e.exists_known || e.refcnt_diff < 0) { - TRY_STATUS(fetch_entry(e)); + TRY_STATUS(fetch_from_dict(e)); } ++e.refcnt_diff; if (e.exists || e.refcnt_diff > 1 || (e.refcnt && e.refcnt.value() + e.refcnt_diff != 1)) { if (!e.max_merkle_depth) { - TRY_STATUS(fetch_entry(e)); + TRY_STATUS(fetch_from_dict(e)); if (!e.max_merkle_depth) { return td::Status::Error(PSTRING() << "unexpected unknown Merkle depth of cell " << cell->get_hash()); } @@ -159,7 +141,7 @@ td::Result AccountStorageStat::add_cell(const Ref< td::Status AccountStorageStat::remove_cell(const Ref& cell) { Entry& e = get_entry(cell); if (!e.exists_known) { - TRY_STATUS(fetch_entry(e)); + TRY_STATUS(fetch_from_dict(e)); } if (!e.exists) { return td::Status::Error(PSTRING() << "Failed to remove cell " << cell->get_hash().to_hex() @@ -167,7 +149,7 @@ td::Status AccountStorageStat::remove_cell(const Ref& cell) { } --e.refcnt_diff; if (e.refcnt_diff < 0 && !e.refcnt) { - TRY_STATUS(fetch_entry(e)); + TRY_STATUS(fetch_from_dict(e)); } if (e.refcnt.value() + e.refcnt_diff != 0) { return td::Status::OK(); @@ -180,18 +162,70 @@ td::Status AccountStorageStat::remove_cell(const Ref& cell) { return td::Status::OK(); } +td::Result> AccountStorageStat::get_dict_root() { + if (!dict_up_to_date_) { + std::vector>> values; + for (auto& [_, e] : cache_) { + if (e.dict_refcnt_diff == 0) { + continue; + } + if (!e.exists_known || !e.refcnt || (e.exists && !e.max_merkle_depth)) { + return td::Status::Error("unexpected state of storage stat"); + } + if (e.exists) { + Ref cbr{true}; + auto& cb = cbr.write(); + CHECK(cb.store_long_bool(e.refcnt.value(), 32) && cb.store_long_bool(e.max_merkle_depth.value(), 2)); + values.emplace_back(e.hash.bits(), std::move(cbr)); + } else { + values.emplace_back(e.hash.bits(), Ref{}); + } + e.dict_refcnt_diff = 0; + } + if (!dict_.multiset(values)) { + return td::Status::Error("failed to update dictionary"); + } + dict_up_to_date_ = true; + } + return dict_.get_root_cell(); +} + +void AccountStorageStat::apply_child_stat(AccountStorageStat&& child) { + CHECK(parent_ == nullptr); + if (child.parent_ == nullptr) { + *this = std::move(child); + return; + } + CHECK(child.parent_ == this); + total_bits_ = child.total_bits_; + total_cells_ = child.total_cells_; + dict_ = std::move(child.dict_); + dict_up_to_date_ = child.dict_up_to_date_; + roots_ = std::move(child.roots_); + for (auto& [hash, e] : child.cache_) { + cache_[hash] = std::move(e); + } +} + AccountStorageStat::Entry& AccountStorageStat::get_entry(const Ref& cell) { - return cache_.apply(cell->get_hash().as_slice(), [&](Entry& e) { - if (e.inited) { - return; + Entry& e = cache_[cell->get_hash()]; + if (e.cell.not_null()) { + return e; + } + if (parent_) { + auto it = parent_->cache_.find(cell->get_hash()); + if (it != parent_->cache_.end()) { + CHECK(it->second.cell.not_null()); + e = it->second; + return e; } - e.inited = true; - e.cell = cell; - e.hash = cell->get_hash(); - }); + } + e.cell = cell; + e.hash = cell->get_hash(); + return e; } -td::Status AccountStorageStat::fetch_entry(Entry& e) { +td::Status AccountStorageStat::fetch_from_dict(Entry& e) { if (e.exists_known && e.refcnt && (!e.exists || e.max_merkle_depth)) { return td::Status::OK(); } @@ -214,20 +248,19 @@ td::Status AccountStorageStat::fetch_entry(Entry& e) { return td::Status::OK(); } -td::Status AccountStorageStat::commit_entry(Entry& e, - std::vector>>& values) { +td::Status AccountStorageStat::finalize_entry(Entry& e) { if (e.refcnt_diff == 0) { return td::Status::OK(); } - TRY_STATUS(fetch_entry(e)); + TRY_STATUS(fetch_from_dict(e)); e.refcnt.value() += e.refcnt_diff; + e.dict_refcnt_diff += e.refcnt_diff; e.refcnt_diff = 0; bool spec; if (e.refcnt.value() == 0) { --total_cells_; total_bits_ -= vm::load_cell_slice_special(e.cell, spec).size(); e.exists = false; - values.emplace_back(e.hash.bits(), td::Ref{}); } else { if (!e.exists) { ++total_cells_; @@ -237,10 +270,6 @@ td::Status AccountStorageStat::commit_entry(Entry& e, if (!e.max_merkle_depth) { return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown merkle depth"); } - td::Ref cbr{true}; - auto& cb = cbr.write(); - CHECK(cb.store_long_bool(e.refcnt.value(), 32) && cb.store_long_bool(e.max_merkle_depth.value(), 2)); - values.emplace_back(e.hash.bits(), std::move(cbr)); } return td::Status::OK(); } diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h index 83eb3af5ae..a7da8288f8 100644 --- a/crypto/block/account-storage-stat.h +++ b/crypto/block/account-storage-stat.h @@ -28,19 +28,19 @@ using td::Ref; class AccountStorageStat { public: - struct CellInfo { - td::uint32 max_merkle_depth = 0; - }; - AccountStorageStat(); AccountStorageStat(Ref dict_root, std::vector> roots, td::uint64 total_cells, td::uint64 total_bits); - AccountStorageStat(const AccountStorageStat &other); - AccountStorageStat(AccountStorageStat &&other); + explicit AccountStorageStat(const AccountStorageStat *parent); + AccountStorageStat(const AccountStorageStat &other) = delete; + AccountStorageStat(AccountStorageStat &&other) = default; ~AccountStorageStat() = default; - AccountStorageStat &operator=(const AccountStorageStat &other); - AccountStorageStat &operator=(AccountStorageStat &&other); + AccountStorageStat &operator=(const AccountStorageStat &other) = delete; + AccountStorageStat &operator=(AccountStorageStat &&other) = default; + + td::Status replace_roots(std::vector> new_roots, bool check_merkle_depth = false); + void add_hint(const td::HashSet &visited); td::uint64 get_total_cells() const { return total_cells_; @@ -50,70 +50,46 @@ class AccountStorageStat { return total_bits_; } - Ref get_dict_root() const { - return dict_.get_root_cell(); - } + td::Result> get_dict_root(); - td::Bits256 get_dict_hash() const { - return dict_.is_empty() ? td::Bits256::zero() : td::Bits256{dict_.get_root_cell()->get_hash().bits()}; + td::Result get_dict_hash() { + TRY_RESULT(root, get_dict_root()); + return root.is_null() ? td::Bits256::zero() : td::Bits256{root->get_hash().bits()}; } - td::Result replace_roots(std::vector> hint); - void add_hint(const td::HashSet &visited); + void apply_child_stat(AccountStorageStat &&child); private: vm::Dictionary dict_; + bool dict_up_to_date_ = true; td::uint64 total_cells_, total_bits_; std::vector> roots_; + const AccountStorageStat *parent_ = nullptr; + + struct CellInfo { + td::uint32 max_merkle_depth = 0; + }; td::Result add_cell(const Ref &cell); td::Status remove_cell(const Ref &cell); struct Entry { - bool inited = false; vm::CellHash hash; Ref cell; bool exists_known = false; bool exists = false; td::optional refcnt, max_merkle_depth; td::int32 refcnt_diff = 0; - - vm::Cell::Hash key() const { - return hash; - } - bool operator<(const Entry &other) const { - return key() < other.key(); - } - struct Eq { - using is_transparent = void; // Pred to use - bool operator()(const Entry &info, const Entry &other_info) const { - return info.key() == other_info.key(); - } - bool operator()(const Entry &info, td::Slice hash) const { - return info.key().as_slice() == hash; - } - bool operator()(td::Slice hash, const Entry &info) const { - return info.key().as_slice() == hash; - } - }; - struct Hash { - using is_transparent = void; // Pred to use - using transparent_key_equal = Eq; - size_t operator()(td::Slice hash) const { - return cell_hash_slice_hash(hash); - } - size_t operator()(const Entry &info) const { - return cell_hash_slice_hash(info.key().as_slice()); - } - }; + td::int32 dict_refcnt_diff = 0; }; - vm::CellHashTable cache_; + td::HashMap cache_; Entry &get_entry(const Ref &cell); - td::Status fetch_entry(Entry &e); - td::Status commit_entry(Entry &e, std::vector>> &values); + td::Status fetch_from_dict(Entry &e); + td::Status finalize_entry(Entry &e); static constexpr td::uint32 MERKLE_DEPTH_LIMIT = 3; + static constexpr td::uint32 MAX_MERKLE_DEPTH = 2; }; class StorageStatCalculationContext : public td::Context { diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 7dbc5249f7..f24a036764 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -2951,15 +2951,12 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, } AccountStorageStat storage_stat; if (update_storage_stat && account.account_storage_stat) { - storage_stat = account.account_storage_stat.value(); + storage_stat = AccountStorageStat{&account.account_storage_stat.value()}; } { TD_PERF_COUNTER(transaction_storage_stat_a); td::Timer timer; - TRY_RESULT(info, storage_stat.replace_roots({new_code, new_data, new_library})); - if (info.max_merkle_depth > max_allowed_merkle_depth) { - return td::Status::Error("too big Merkle depth"); - } + TRY_STATUS(storage_stat.replace_roots({new_code, new_data, new_library}, /* check_merkle_depth = */ true)); if (timer.elapsed() > 0.1) { LOG(INFO) << "Compute used storage (1) took " << timer.elapsed() << "s"; } @@ -3276,23 +3273,33 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } } - if (storage_refs_changed || (cfg.store_storage_dict_hash && !account.storage_dict_hash)) { + if (storage_refs_changed || + (cfg.store_storage_dict_hash && !account.storage_dict_hash && account.storage_used.cells > 25)) { TD_PERF_COUNTER(transaction_storage_stat_b); td::Timer timer; if (!new_account_storage_stat && account.account_storage_stat) { - new_account_storage_stat = account.account_storage_stat; + new_account_storage_stat = AccountStorageStat(&account.account_storage_stat.value()); } AccountStorageStat& stats = new_account_storage_stat.value_force(); // Don't check Merkle depth and size here - they were checked in check_state_limits - auto S = stats.replace_roots(new_storage_for_stat->prefetch_all_refs()).move_as_status(); + td::Status S = stats.replace_roots(new_storage_for_stat->prefetch_all_refs()); if (S.is_error()) { LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error(); return false; } - new_storage_dict_hash = stats.get_dict_hash(); // Root of AccountStorage is not counted in AccountStorageStat new_storage_used.cells = stats.get_total_cells() + 1; new_storage_used.bits = stats.get_total_bits() + new_storage_for_stat->size(); + // TODO: think about this limit (25) + if (cfg.store_storage_dict_hash && new_storage_used.cells > 25) { + auto r_hash = stats.get_dict_hash(); + if (r_hash.is_error()) { + LOG(ERROR) << "Cannot compute storage dict hash for account " << account.addr.to_hex() << ": " + << r_hash.move_as_error(); + return false; + } + new_storage_dict_hash = r_hash.move_as_ok(); + } if (timer.elapsed() > 0.1) { LOG(INFO) << "Compute used storage (2) took " << timer.elapsed() << "s"; } @@ -3300,11 +3307,10 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { new_storage_used = account.storage_used; new_storage_used.bits -= old_storage_for_stat->size(); new_storage_used.bits += new_storage_for_stat->size(); - new_storage_dict_hash = account.storage_dict_hash; - new_account_storage_stat = account.account_storage_stat; - } - if (!cfg.store_storage_dict_hash || (new_storage_dict_hash && new_storage_dict_hash.value().is_zero())) { - new_storage_dict_hash = {}; + new_account_storage_stat = {}; + if (cfg.store_storage_dict_hash) { + new_storage_dict_hash = account.storage_dict_hash; + } } CHECK(cb.store_long_bool(1, 1) // account$1 @@ -3683,7 +3689,13 @@ Ref Transaction::commit(Account& acc) { acc.last_trans_hash_ = root->get_hash().bits(); acc.last_paid = last_paid; acc.storage_used = new_storage_used; - acc.account_storage_stat = std::move(new_account_storage_stat); + if (new_account_storage_stat) { + if (acc.account_storage_stat) { + acc.account_storage_stat.value().apply_child_stat(std::move(new_account_storage_stat.value())); + } else { + acc.account_storage_stat = std::move(new_account_storage_stat); + } + } acc.storage_dict_hash = new_storage_dict_hash; acc.storage = new_storage; acc.balance = std::move(balance); From 5c777002bc55cc25b9605ae42f912ba8b85ca53d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 2 Apr 2025 12:17:30 +0300 Subject: [PATCH 192/524] Fix compilation errors --- crypto/block/transaction.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index e0345c5370..bc89611645 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -617,7 +617,7 @@ td::Result> Account::compute_account_storage_dict() const { if (storage_for_stat.is_null()) { return td::Status::Error("cannot compute storage dict: invalid storage"); } - TRY_STATUS(stat.replace_roots(storage_for_stat->prefetch_all_refs()).move_as_status()); + TRY_STATUS(stat.replace_roots(storage_for_stat->prefetch_all_refs())); // Root of AccountStorage is not counted in AccountStorageStat td::uint64 expected_cells = stat.get_total_cells() + 1; td::uint64 expected_bits = stat.get_total_bits() + storage->size(); @@ -626,9 +626,10 @@ td::Result> Account::compute_account_storage_dict() const { << " bits=" << expected_bits << ", found cells" << storage_used.cells << " bits=" << storage_used.bits); } - if (storage_dict_hash.value() != stat.get_dict_hash()) { - return td::Status::Error(PSTRING() << "invalid storage dict hash: computed " << stat.get_dict_hash().to_hex() - << ", found " << storage_dict_hash.value().to_hex()); + TRY_RESULT(root_hash, stat.get_dict_hash()); + if (storage_dict_hash.value() != root_hash) { + return td::Status::Error(PSTRING() << "invalid storage dict hash: computed " << root_hash.to_hex() << ", found " + << storage_dict_hash.value().to_hex()); } return stat.get_dict_root(); } @@ -662,9 +663,10 @@ td::Status Account::init_account_storage_stat(Ref dict_root) { } AccountStorageStat new_stat(std::move(dict_root), storage->prefetch_all_refs(), storage_used.cells - 1, storage_used.bits - storage->size()); - if (storage_dict_hash.value() != new_stat.get_dict_hash()) { - return td::Status::Error(PSTRING() << "invalid storage dict hash: computed " << new_stat.get_dict_hash().to_hex() - << ", found " << storage_dict_hash.value().to_hex()); + TRY_RESULT(root_hash, new_stat.get_dict_hash()); + if (storage_dict_hash.value() != root_hash) { + return td::Status::Error(PSTRING() << "invalid storage dict hash: computed " << root_hash.to_hex() << ", found " + << storage_dict_hash.value().to_hex()); } account_storage_stat = std::move(new_stat); return td::Status::OK(); From dd5068163c253f6e925dfecd5137fc4bca8f3843 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 3 Apr 2025 12:59:24 +0300 Subject: [PATCH 193/524] Optimize AccountStorageStat::add_hint call --- crypto/block/transaction.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index bc89611645..f316bca960 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -3363,12 +3363,12 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { td::Timer timer; if (!new_account_storage_stat && account.account_storage_stat) { new_account_storage_stat = AccountStorageStat(&account.account_storage_stat.value()); + if (compute_phase) { + new_account_storage_stat.value().add_hint(compute_phase->vm_loaded_cells); + } } AccountStorageStat& stats = new_account_storage_stat.value_force(); // Don't check Merkle depth and size here - they were checked in check_state_limits - if (compute_phase) { - stats.add_hint(compute_phase->vm_loaded_cells); - } auto roots = new_storage_for_stat->prefetch_all_refs(); storage_stats_updates.insert(storage_stats_updates.end(), roots.begin(), roots.end()); { From 3b4b00c0752d3ecd9926a567d561522a3b674912 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 3 Apr 2025 16:05:57 +0300 Subject: [PATCH 194/524] Optimize cell loading in account storage stat --- crypto/block/account-storage-stat.cpp | 20 ++++++++++++++------ crypto/block/account-storage-stat.h | 6 ++++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index 9fab8f8802..56cc8f1e1b 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -95,6 +95,7 @@ void AccountStorageStat::add_hint(const td::HashSet& hint) { if (hint.contains(cell->get_hash())) { bool spec; vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); + e.size_bits = cs.size(); for (unsigned i = 0; i < cs.size_refs(); ++i) { dfs(cs.prefetch_ref(i), false); } @@ -124,6 +125,7 @@ td::Result AccountStorageStat::add_cell(const Ref< td::uint32 max_merkle_depth = 0; bool spec; vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); + e.size_bits = cs.size(); for (unsigned i = 0; i < cs.size_refs(); ++i) { TRY_RESULT(info, add_cell(cs.prefetch_ref(i))); max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); @@ -156,6 +158,7 @@ td::Status AccountStorageStat::remove_cell(const Ref& cell) { } bool spec; vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); + e.size_bits = cs.size(); for (unsigned i = 0; i < cs.size_refs(); ++i) { TRY_STATUS(remove_cell(cs.prefetch_ref(i))); } @@ -209,18 +212,18 @@ void AccountStorageStat::apply_child_stat(AccountStorageStat&& child) { AccountStorageStat::Entry& AccountStorageStat::get_entry(const Ref& cell) { Entry& e = cache_[cell->get_hash()]; - if (e.cell.not_null()) { + if (e.inited) { return e; } if (parent_) { auto it = parent_->cache_.find(cell->get_hash()); if (it != parent_->cache_.end()) { - CHECK(it->second.cell.not_null()); + CHECK(it->second.inited); e = it->second; return e; } } - e.cell = cell; + e.inited = true; e.hash = cell->get_hash(); return e; } @@ -256,15 +259,20 @@ td::Status AccountStorageStat::finalize_entry(Entry& e) { e.refcnt.value() += e.refcnt_diff; e.dict_refcnt_diff += e.refcnt_diff; e.refcnt_diff = 0; - bool spec; if (e.refcnt.value() == 0) { + if (!e.size_bits) { + return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown cell bits"); + } --total_cells_; - total_bits_ -= vm::load_cell_slice_special(e.cell, spec).size(); + total_bits_ -= e.size_bits.value(); e.exists = false; } else { if (!e.exists) { + if (!e.size_bits) { + return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown cell bits"); + } ++total_cells_; - total_bits_ += vm::load_cell_slice_special(e.cell, spec).size(); + total_bits_ += e.size_bits.value(); } e.exists = true; if (!e.max_merkle_depth) { diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h index a7da8288f8..645f0744d8 100644 --- a/crypto/block/account-storage-stat.h +++ b/crypto/block/account-storage-stat.h @@ -74,15 +74,17 @@ class AccountStorageStat { td::Status remove_cell(const Ref &cell); struct Entry { + bool inited = false; vm::CellHash hash; - Ref cell; + td::optional size_bits; bool exists_known = false; bool exists = false; td::optional refcnt, max_merkle_depth; td::int32 refcnt_diff = 0; td::int32 dict_refcnt_diff = 0; }; - td::HashMap cache_; + + td::HashMap> cache_; Entry &get_entry(const Ref &cell); td::Status fetch_from_dict(Entry &e); From 0e2c9bf75644f5720b0b31242e8be87557085b19 Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Thu, 3 Apr 2025 16:46:13 +0200 Subject: [PATCH 195/524] Fix calculating normalized hash (#1602) * fix calculating normalized hash * add get_ext_in_msg_hash_norm to .h --- tonlib/tonlib/TonlibClient.cpp | 32 +++++++++++++++++--------------- tonlib/tonlib/TonlibClient.h | 1 + 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index b137ee7699..00dcf22dba 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -3308,14 +3308,9 @@ td::Status TonlibClient::do_request(const tonlib_api::raw_sendMessage& request, return td::Status::OK(); } -td::Status TonlibClient::do_request(const tonlib_api::raw_sendMessageReturnHash& request, - td::Promise>&& promise) { - TRY_RESULT_PREFIX(body, vm::std_boc_deserialize(request.body_), TonlibError::InvalidBagOfCells("body")); - auto hash = body->get_hash().as_slice().str(); - - // compute hash normalized +td::Result get_ext_in_msg_hash_norm(td::Ref ext_in_msg_cell) { block::gen::Message::Record message; - if (!tlb::type_unpack_cell(body, block::gen::t_Message_Any, message)) { + if (!tlb::type_unpack_cell(ext_in_msg_cell, block::gen::t_Message_Any, message)) { return td::Status::Error("Failed to unpack Message"); } auto tag = block::gen::CommonMsgInfo().get_tag(*message.info); @@ -3327,32 +3322,39 @@ td::Status TonlibClient::do_request(const tonlib_api::raw_sendMessageReturnHash& return td::Status::Error("Failed to unpack CommonMsgInfo::ext_in_msg_info"); } - td::Ref body_norm; + td::Ref body; auto body_cs = message.body.write(); - if (body_cs.fetch_long(1) == 1) { - body_norm = body_cs.fetch_ref(); + if (body_cs.fetch_ulong(1) == 1) { + body = body_cs.fetch_ref(); } else { - body_norm = vm::CellBuilder().append_cellslice(body_cs).finalize(); + body = vm::CellBuilder().append_cellslice(body_cs).finalize(); } auto cb = vm::CellBuilder(); - bool status = + bool status = cb.store_long_bool(2, 2) && // message$_ -> info:CommonMsgInfo -> ext_in_msg_info$10 cb.store_long_bool(0, 2) && // message$_ -> info:CommonMsgInfo -> src:MsgAddressExt -> addr_none$00 cb.append_cellslice_bool(msg_info.dest) && // message$_ -> info:CommonMsgInfo -> dest:MsgAddressInt cb.store_long_bool(0, 4) && // message$_ -> info:CommonMsgInfo -> import_fee:Grams -> 0 cb.store_long_bool(0, 1) && // message$_ -> init:(Maybe (Either StateInit ^StateInit)) -> nothing$0 cb.store_long_bool(1, 1) && // message$_ -> body:(Either X ^X) -> right$1 - cb.store_ref_bool(body_norm); + cb.store_ref_bool(body); if (!status) { return td::Status::Error("Failed to build normalized message"); } - auto hash_norm = cb.finalize()->get_hash().as_slice().str(); + return cb.finalize()->get_hash().bits(); +} + +td::Status TonlibClient::do_request(const tonlib_api::raw_sendMessageReturnHash& request, + td::Promise>&& promise) { + TRY_RESULT_PREFIX(body, vm::std_boc_deserialize(request.body_), TonlibError::InvalidBagOfCells("body")); + auto hash = body->get_hash().as_slice().str(); + TRY_RESULT(hash_norm, get_ext_in_msg_hash_norm(body)); make_request(int_api::SendMessage{std::move(body)}, promise.wrap([hash = std::move(hash), hash_norm = std::move(hash_norm)](auto res) { - return tonlib_api::make_object(std::move(hash), std::move(hash_norm)); + return tonlib_api::make_object(std::move(hash), hash_norm.as_slice().str()); })); return td::Status::OK(); } diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 28239a8a4e..ed19be65e7 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -59,6 +59,7 @@ class RunEmulator; td::Result> to_tonlib_api( const ton::ManualDns::EntryData& entry_data); td::Result to_dns_entry_data(tonlib_api::dns_EntryData& entry_data); +td::Result get_ext_in_msg_hash_norm(td::Ref ext_in_msg_cell); class TonlibClient : public td::actor::Actor { public: From 5661301db5c0d7a6facea4906fe160c9802bf655 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sat, 5 Apr 2025 10:38:50 +0300 Subject: [PATCH 196/524] Fix archive-ttl overflow, remove unused code, clarify validator-engine flags (#1605) --- validator-engine/validator-engine.cpp | 8 +- validator/db/archive-manager.cpp | 6 +- validator/db/archive-manager.hpp | 4 +- validator/db/rootdb.cpp | 6 +- validator/db/rootdb.hpp | 4 +- validator/interfaces/db.h | 2 +- validator/interfaces/validator-manager.h | 9 -- validator/manager-disk.hpp | 25 ----- validator/manager-hardfork.hpp | 25 ----- validator/manager.cpp | 119 +---------------------- validator/manager.hpp | 26 ----- 11 files changed, 15 insertions(+), 219 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index cbcc3ab1f0..4b4e47348e 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -4359,7 +4359,7 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); }); return td::Status::OK(); }); - p.add_checked_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=86400", + p.add_checked_option('b', "block-ttl", "deprecated", [&](td::Slice fname) { auto v = td::to_double(fname); if (v <= 0) { @@ -4369,7 +4369,9 @@ int main(int argc, char *argv[]) { return td::Status::OK(); }); p.add_checked_option( - 'A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=7*86400", + 'A', "archive-ttl", + "ttl for archived blocks (in seconds) default=7*86400. Note: archived blocks are gc'd after state-ttl + " + "archive-ttl seconds", [&](td::Slice fname) { auto v = td::to_double(fname); if (v <= 0) { @@ -4379,7 +4381,7 @@ int main(int argc, char *argv[]) { return td::Status::OK(); }); p.add_checked_option( - 'K', "key-proof-ttl", "key blocks will be deleted after this time (in seconds) default=365*86400*10", + 'K', "key-proof-ttl", "deprecated", [&](td::Slice fname) { auto v = td::to_double(fname); if (v <= 0) { diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 9b1ebf6c2f..9e6ea797e3 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -965,8 +965,8 @@ void ArchiveManager::alarm() { } } -void ArchiveManager::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) { - auto p = get_temp_package_id_by_unixtime(mc_ts - TEMP_PACKAGES_TTL); +void ArchiveManager::run_gc(UnixTime mc_ts, UnixTime gc_ts, double archive_ttl) { + auto p = get_temp_package_id_by_unixtime((double)mc_ts - TEMP_PACKAGES_TTL); std::vector vec; for (auto &x : temp_files_) { if (x.first < p) { @@ -994,7 +994,7 @@ void ArchiveManager::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl if (it == desc.first_blocks.end()) { continue; } - if (it->second.ts < gc_ts - archive_ttl) { + if ((double)it->second.ts < (double)gc_ts - archive_ttl) { vec.push_back(f.first); } } diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index cd79ccc6e8..33dd1f3d6f 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -61,7 +61,7 @@ class ArchiveManager : public td::actor::Actor { void truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise); //void truncate_continue(BlockSeqno masterchain_seqno, td::Promise promise); - void run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl); + void run_gc(UnixTime mc_ts, UnixTime gc_ts, double archive_ttl); /* from LTDB */ void get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise promise); @@ -240,7 +240,7 @@ class ArchiveManager : public td::actor::Actor { void update_permanent_slices(); - static const td::uint32 TEMP_PACKAGES_TTL = 3600; + static constexpr double TEMP_PACKAGES_TTL = 3600; }; } // namespace validator diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 2b370eb065..90546fbef9 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -431,10 +431,6 @@ void RootDb::allow_state_gc(BlockIdExt block_id, td::Promise promise) { td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_state_gc, block_id, std::move(promise)); } -void RootDb::allow_block_gc(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_info_gc, block_id, std::move(promise)); -} - void RootDb::prepare_stats(td::Promise>> promise) { auto merger = StatsMerger::create(std::move(promise)); td::actor::send_closure(cell_db_, &CellDb::prepare_stats, merger.make_promise("celldb.")); @@ -519,7 +515,7 @@ void RootDb::set_async_mode(bool mode, td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::set_async_mode, mode, std::move(promise)); } -void RootDb::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) { +void RootDb::run_gc(UnixTime mc_ts, UnixTime gc_ts, double archive_ttl) { td::actor::send_closure(archive_db_, &ArchiveManager::run_gc, mc_ts, gc_ts, archive_ttl); } diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 9e5b0b302e..2ccd7f942d 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -118,8 +118,6 @@ class RootDb : public Db { void archive(BlockHandle handle, td::Promise promise) override; void allow_state_gc(BlockIdExt block_id, td::Promise promise); - void allow_block_gc(BlockIdExt block_id, td::Promise promise); - //void allow_gc(FileDb::RefId ref_id, bool is_archive, td::Promise promise); void prepare_stats(td::Promise>> promise) override; @@ -137,7 +135,7 @@ class RootDb : public Db { td::Promise promise) override; void set_async_mode(bool mode, td::Promise promise) override; - void run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) override; + void run_gc(UnixTime mc_ts, UnixTime gc_ts, double archive_ttl) override; void add_persistent_state_description(td::Ref desc, td::Promise promise) override; void get_persistent_state_descriptions(td::Promise>> promise) override; diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 9a2eea56e0..f21c6f9028 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -123,7 +123,7 @@ class Db : public td::actor::Actor { td::Promise promise) = 0; virtual void set_async_mode(bool mode, td::Promise promise) = 0; - virtual void run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) = 0; + virtual void run_gc(UnixTime mc_ts, UnixTime gc_ts, double archive_ttl) = 0; virtual void add_persistent_state_description(td::Ref desc, td::Promise promise) = 0; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 00fb77e1eb..1f08c4ef5c 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -160,16 +160,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void try_get_static_file(FileHash file_hash, td::Promise promise) = 0; - virtual void allow_block_data_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) = 0; virtual void allow_block_state_gc(BlockIdExt block_id, td::Promise promise) = 0; - virtual void allow_zero_state_file_gc(BlockIdExt block_id, td::Promise promise) = 0; - virtual void allow_persistent_state_file_gc(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) = 0; - virtual void allow_block_signatures_gc(BlockIdExt block_id, td::Promise promise) = 0; - virtual void allow_block_proof_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) = 0; - virtual void allow_block_proof_link_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) = 0; - virtual void allow_block_candidate_gc(BlockIdExt block_id, td::Promise promise) = 0; - virtual void allow_block_info_gc(BlockIdExt block_id, td::Promise promise) = 0; virtual void archive(BlockHandle handle, td::Promise promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 62f54b2e95..8526ffa4d6 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -341,34 +341,9 @@ class ValidatorManagerImpl : public ValidatorManager { void update_gc_block_handle(BlockHandle handle, td::Promise promise) override { promise.set_value(td::Unit()); } - void allow_block_data_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - promise.set_result(false); - } void allow_block_state_gc(BlockIdExt block_id, td::Promise promise) override { promise.set_result(false); } - void allow_zero_state_file_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_persistent_state_file_gc(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override { - promise.set_result(false); - } - void allow_block_signatures_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_proof_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_proof_link_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_candidate_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_info_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } void archive(BlockHandle handle, td::Promise promise) override { td::actor::send_closure(db_, &Db::archive, std::move(handle), std::move(promise)); } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 486c185f7a..b2bd9250b8 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -406,34 +406,9 @@ class ValidatorManagerImpl : public ValidatorManager { void update_gc_block_handle(BlockHandle handle, td::Promise promise) override { promise.set_value(td::Unit()); } - void allow_block_data_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - promise.set_result(false); - } void allow_block_state_gc(BlockIdExt block_id, td::Promise promise) override { promise.set_result(false); } - void allow_zero_state_file_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_persistent_state_file_gc(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override { - promise.set_result(false); - } - void allow_block_signatures_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_proof_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_proof_link_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_candidate_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_info_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } void archive(BlockHandle handle, td::Promise promise) override { UNREACHABLE(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 12702ddd8a..85f3a30abc 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2374,7 +2374,7 @@ void ValidatorManagerImpl::try_advance_gc_masterchain_block() { gc_masterchain_handle_->id().id.seqno < last_masterchain_state_->last_key_block_id().seqno() && gc_masterchain_handle_->id().id.seqno < min_confirmed_masterchain_seqno_ && gc_masterchain_handle_->id().id.seqno < state_serializer_masterchain_seqno_ && - gc_masterchain_state_->get_unix_time() < td::Clocks::system() - state_ttl()) { + (double)gc_masterchain_state_->get_unix_time() < td::Clocks::system() - state_ttl()) { gc_advancing_ = true; auto block_id = gc_masterchain_handle_->one_next(true); @@ -2386,100 +2386,6 @@ void ValidatorManagerImpl::try_advance_gc_masterchain_block() { } } -void ValidatorManagerImpl::allow_persistent_state_file_gc(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - if (!gc_masterchain_handle_) { - promise.set_result(false); - return; - } - if (masterchain_block_id.seqno() == 0) { - promise.set_result(false); - return; - } - if (masterchain_block_id.seqno() >= gc_masterchain_handle_->id().seqno()) { - promise.set_result(false); - return; - } - auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { - R.ensure(); - auto handle = R.move_as_ok(); - CHECK(handle->is_key_block()); - promise.set_result(ValidatorManager::persistent_state_ttl(handle->unix_time()) < td::Clocks::system()); - }); - get_block_handle(masterchain_block_id, false, std::move(P)); -} - -void ValidatorManagerImpl::allow_archive(BlockIdExt block_id, td::Promise promise) { - /*if (!gc_masterchain_handle_) { - promise.set_result(false); - return; - } - if (!block_id.is_masterchain()) { - if (!gc_masterchain_state_->workchain_is_active(block_id.id.workchain)) { - promise.set_result(false); - return; - } - bool found = false; - auto S = gc_masterchain_state_->get_shard_from_config(block_id.shard_full()); - if (S.not_null()) { - if (block_id.id.seqno >= S->top_block_id().id.seqno) { - promise.set_result(false); - return; - } - found = true; - } else { - auto shards = gc_masterchain_state_->get_shards(); - for (auto shard : shards) { - if (shard_intersects(shard->shard(), block_id.shard_full())) { - if (block_id.id.seqno >= shard->top_block_id().id.seqno) { - promise.set_result(false); - return; - } - found = true; - } - } - } - CHECK(found); - } else { - if (block_id.id.seqno >= gc_masterchain_handle_->id().id.seqno) { - promise.set_result(false); - return; - } - } - auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - promise.set_result(true); - } - }); - td::actor::send_closure(db_, &Db::archive, block_id, std::move(P));*/ - promise.set_result(false); -} - -void ValidatorManagerImpl::allow_delete(BlockIdExt block_id, td::Promise promise) { - auto key_ttl = td::Clocks::system() - opts_->key_proof_ttl(); - auto ttl = td::Clocks::system() - opts_->archive_ttl(); - auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), promise = std::move(promise), ttl, key_ttl](td::Result R) mutable { - if (R.is_error()) { - promise.set_result(true); - return; - } - auto handle = R.move_as_ok(); - if (!handle->inited_unix_time()) { - promise.set_result(true); - return; - } - if (!handle->inited_is_key_block() || !handle->is_key_block()) { - promise.set_result(handle->unix_time() <= ttl); - } else { - promise.set_result(handle->unix_time() <= key_ttl); - } - }); - get_block_handle(block_id, false, std::move(P)); -} - void ValidatorManagerImpl::allow_block_state_gc(BlockIdExt block_id, td::Promise promise) { if (!gc_masterchain_handle_) { promise.set_result(false); @@ -2508,27 +2414,6 @@ void ValidatorManagerImpl::allow_block_state_gc(BlockIdExt block_id, td::Promise UNREACHABLE(); } -void ValidatorManagerImpl::allow_block_info_gc(BlockIdExt block_id, td::Promise promise) { - auto P = - td::PromiseCreator::lambda([db = db_.get(), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_result(false); - } else { - auto handle = R.move_as_ok(); - if (!handle->moved_to_archive() || !handle->is_applied()) { - promise.set_result(false); - } else { - auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { - R.ensure(); - promise.set_result(true); - }); - td::actor::send_closure(db, &Db::store_block_handle, handle, std::move(P)); - } - } - }); - get_block_handle(block_id, false, std::move(P)); -} - void ValidatorManagerImpl::got_next_gc_masterchain_handle(BlockHandle handle) { CHECK(gc_advancing_); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { @@ -2610,7 +2495,7 @@ void ValidatorManagerImpl::alarm() { alarm_timestamp() = td::Timestamp::in(1.0); if (shard_client_handle_ && gc_masterchain_handle_) { td::actor::send_closure(db_, &Db::run_gc, shard_client_handle_->unix_time(), gc_masterchain_handle_->unix_time(), - static_cast(opts_->archive_ttl())); + opts_->archive_ttl()); } if (log_status_at_.is_in_past()) { if (last_masterchain_block_handle_) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 347b8f7ef9..c406d36b04 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -568,30 +568,7 @@ class ValidatorManagerImpl : public ValidatorManager { } public: - void allow_delete(BlockIdExt block_id, td::Promise promise); - void allow_archive(BlockIdExt block_id, td::Promise promise); - void allow_block_data_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - allow_archive(block_id, std::move(promise)); - } void allow_block_state_gc(BlockIdExt block_id, td::Promise promise) override; - void allow_zero_state_file_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_persistent_state_file_gc(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override; - void allow_block_signatures_gc(BlockIdExt block_id, td::Promise promise) override { - allow_archive(block_id, std::move(promise)); - } - void allow_block_proof_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - allow_archive(block_id, std::move(promise)); - } - void allow_block_proof_link_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - allow_archive(block_id, std::move(promise)); - } - void allow_block_candidate_gc(BlockIdExt block_id, td::Promise promise) override { - allow_block_state_gc(block_id, std::move(promise)); - } - void allow_block_info_gc(BlockIdExt block_id, td::Promise promise) override; void archive(BlockHandle handle, td::Promise promise) override { td::actor::send_closure(db_, &Db::archive, std::move(handle), std::move(promise)); } @@ -716,9 +693,6 @@ class ValidatorManagerImpl : public ValidatorManager { double state_ttl() const { return opts_->state_ttl(); } - double block_ttl() const { - return opts_->block_ttl(); - } double max_mempool_num() const { return opts_->max_mempool_num(); } From c3160e37f4d445a871bcb9a52e4cac3557023e35 Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Sat, 5 Apr 2025 09:47:38 +0200 Subject: [PATCH 197/524] Enable bloom filters for CellDB (#1523) * enable bloom filters for celldb * 2 level index and filter for state ttl >= 30 days * set block cache size to 16gb in case of 2 level index * fix rebase artifacts * refactor setting block cache size --- tddb/td/db/RocksDb.cpp | 12 ++++++++++-- tddb/td/db/RocksDb.h | 2 ++ validator-engine/validator-engine.cpp | 7 +++++++ validator-engine/validator-engine.hpp | 4 ++++ validator/db/celldb.cpp | 7 +++++++ validator/validator-options.hpp | 7 +++++++ validator/validator.h | 2 ++ 7 files changed, 39 insertions(+), 2 deletions(-) diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index b56f3b145f..d9040b695b 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -24,10 +24,9 @@ #include "rocksdb/write_batch.h" #include "rocksdb/utilities/optimistic_transaction_db.h" #include "rocksdb/utilities/transaction.h" +#include "rocksdb/filter_policy.h" #include "td/utils/misc.h" -#include - namespace td { namespace { static Status from_rocksdb(const rocksdb::Status &status) { @@ -81,6 +80,15 @@ Result RocksDb::open(std::string path, RocksDbOptions options) { } else { table_options.block_cache = options.block_cache; } + if (options.enable_bloom_filter) { + table_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false)); + if (options.two_level_index_and_filter) { + table_options.index_type = rocksdb::BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + table_options.partition_filters = true; + table_options.cache_index_and_filter_blocks = true; + table_options.pin_l0_filter_and_index_blocks_in_cache = true; + } + } db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); // table_options.block_align = true; diff --git a/tddb/td/db/RocksDb.h b/tddb/td/db/RocksDb.h index d24a20dd7b..71ece1e2aa 100644 --- a/tddb/td/db/RocksDb.h +++ b/tddb/td/db/RocksDb.h @@ -75,6 +75,8 @@ struct RocksDbOptions { bool use_direct_reads = false; bool no_block_cache = false; + bool enable_bloom_filter = false; + bool two_level_index_and_filter = false; }; class RocksDb : public KeyValue { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 4b4e47348e..945d5456f4 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1470,6 +1470,7 @@ td::Status ValidatorEngine::load_global_config() { validator_options_.write().set_celldb_compress_depth(celldb_compress_depth_); validator_options_.write().set_celldb_in_memory(celldb_in_memory_); validator_options_.write().set_celldb_v2(celldb_v2_); + validator_options_.write().set_celldb_disable_bloom_filter(celldb_disable_bloom_filter_); validator_options_.write().set_max_open_archive_files(max_open_archive_files_); validator_options_.write().set_archive_preload_period(archive_preload_period_); validator_options_.write().set_disable_rocksdb_stats(disable_rocksdb_stats_); @@ -4538,6 +4539,12 @@ int main(int argc, char *argv[]) { [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_v2, true); }); }); + p.add_option( + '\0', "celldb-disable-bloom-filter", + "disable using bloom filter in CellDb. Enabled bloom filter reduces read latency, but increases memory usage", + [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_disable_bloom_filter, true); }); + }); p.add_checked_option( '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.4)", [&](td::Slice s) -> td::Status { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 5a1db7f3f5..77030d68be 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -219,6 +219,7 @@ class ValidatorEngine : public td::actor::Actor { bool celldb_preload_all_ = false; bool celldb_in_memory_ = false; bool celldb_v2_ = false; + bool celldb_disable_bloom_filter_ = false; td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; bool read_config_ = false; bool started_keyring_ = false; @@ -315,6 +316,9 @@ class ValidatorEngine : public td::actor::Actor { void set_celldb_v2(bool value) { celldb_v2_ = value; } + void set_celldb_disable_bloom_filter(bool value) { + celldb_disable_bloom_filter_ = value; + } void set_catchain_max_block_delay(double value) { catchain_max_block_delay_ = value; } diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 90c659cc46..c67203f089 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -227,6 +227,13 @@ void CellDbIn::start_up() { LOG(WARNING) << "Using V1 DynamicBagOfCells with options " << *boc_v1_options; } + db_options.enable_bloom_filter = !opts_->get_celldb_disable_bloom_filter(); + db_options.two_level_index_and_filter = db_options.enable_bloom_filter + && opts_->state_ttl() >= 60 * 60 * 24 * 30; // 30 days + if (db_options.two_level_index_and_filter && !opts_->get_celldb_in_memory()) { + o_celldb_cache_size = std::max(o_celldb_cache_size ? o_celldb_cache_size.value() : 0UL, 16UL << 30); + } + if (o_celldb_cache_size) { db_options.block_cache = td::RocksDb::create_cache(o_celldb_cache_size.value()); LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(o_celldb_cache_size.value()); diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 45b8d7ec20..55552635c0 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -142,6 +142,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool get_celldb_v2() const override { return celldb_v2_; } + bool get_celldb_disable_bloom_filter() const override { + return celldb_disable_bloom_filter_; + } td::optional get_catchain_max_block_delay() const override { return catchain_max_block_delay_; } @@ -243,6 +246,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_celldb_v2(bool value) override { celldb_v2_ = value; } + void set_celldb_disable_bloom_filter(bool value) override { + celldb_disable_bloom_filter_ = value; + } void set_catchain_max_block_delay(double value) override { catchain_max_block_delay_ = value; } @@ -311,6 +317,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool celldb_preload_all_ = false; bool celldb_in_memory_ = false; bool celldb_v2_ = false; + bool celldb_disable_bloom_filter_ = false; td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; bool state_serializer_enabled_ = true; td::Ref collator_options_{true}; diff --git a/validator/validator.h b/validator/validator.h index 9e1dba7db8..7868bc16f8 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -112,6 +112,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual td::optional get_celldb_cache_size() const = 0; virtual bool get_celldb_direct_io() const = 0; virtual bool get_celldb_preload_all() const = 0; + virtual bool get_celldb_disable_bloom_filter() const = 0; virtual td::optional get_catchain_max_block_delay() const = 0; virtual td::optional get_catchain_max_block_delay_slow() const = 0; virtual bool get_state_serializer_enabled() const = 0; @@ -146,6 +147,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_celldb_preload_all(bool value) = 0; virtual void set_celldb_in_memory(bool value) = 0; virtual void set_celldb_v2(bool value) = 0; + virtual void set_celldb_disable_bloom_filter(bool value) = 0; virtual void set_catchain_max_block_delay(double value) = 0; virtual void set_catchain_max_block_delay_slow(double value) = 0; virtual void set_state_serializer_enabled(bool value) = 0; From 724ec937208a1a3049773b451541bd0dbad2abbd Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Sat, 5 Apr 2025 13:50:25 +0200 Subject: [PATCH 198/524] Speed up LargeBocSerializer with bulk cells reading (#1533) * serialize boc with bfs (with multiget) * revert in-memory implementation * std_boc_serialize_to_file_large -> boc_serialize_to_file_large * provide load_bulk as default and for celldbv2 * remove excessive check --- crypto/test/test-db.cpp | 7 +- crypto/vm/boc.cpp | 6 +- crypto/vm/boc.h | 19 ++- crypto/vm/db/CellStorage.cpp | 22 +++ crypto/vm/db/CellStorage.h | 1 + crypto/vm/db/DynamicBagOfCellsDb.cpp | 46 ++++-- crypto/vm/db/DynamicBagOfCellsDb.h | 2 + crypto/vm/db/DynamicBagOfCellsDbV2.cpp | 18 +++ crypto/vm/db/InMemoryBagOfCellsDb.cpp | 13 ++ crypto/vm/large-boc-serializer.cpp | 214 ++++++++++++++++++------- tddb/td/db/KeyValue.h | 20 ++- tddb/td/db/MemoryKeyValue.cpp | 12 ++ tddb/td/db/MemoryKeyValue.h | 1 + tddb/td/db/RocksDb.cpp | 34 ++++ tddb/td/db/RocksDb.h | 1 + validator/state-serializer.cpp | 37 ++++- 16 files changed, 367 insertions(+), 86 deletions(-) diff --git a/crypto/test/test-db.cpp b/crypto/test/test-db.cpp index 2b77e71182..47d833d56c 100644 --- a/crypto/test/test-db.cpp +++ b/crypto/test/test-db.cpp @@ -2991,10 +2991,13 @@ TEST(TonDb, LargeBocSerializer) { td::unlink(path).ignore(); fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write) .move_as_ok(); - std_boc_serialize_to_file_large(dboc->get_cell_db_reader(), root->get_hash(), fd, 31); + boc_serialize_to_file_large(dboc->get_cell_db_reader(), root->get_hash(), fd, 31); fd.close(); auto b = td::read_file_str(path).move_as_ok(); - CHECK(a == b); + + auto a_cell = vm::deserialize_boc(td::BufferSlice(a)); + auto b_cell = vm::deserialize_boc(td::BufferSlice(b)); + ASSERT_EQ(a_cell->get_hash(), b_cell->get_hash()); } TEST(TonDb, DoNotMakeListsPrunned) { diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 6377674d7e..3aef7dd10a 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -209,7 +209,7 @@ td::Result BagOfCells::import_cell(td::Ref cell, int depth) { return td::Status::Error("error while importing a cell into a bag of cells: cell is null"); } if (logger_ptr_) { - TRY_STATUS(logger_ptr_->on_cell_processed()); + TRY_STATUS(logger_ptr_->on_cells_processed(1)); } auto it = cells.find(cell->get_hash()); if (it != cells.end()) { @@ -555,7 +555,7 @@ td::Result BagOfCells::serialize_to_impl(WriterT& writer, int mode) } store_offset(fixed_offset); if (logger_ptr_) { - TRY_STATUS(logger_ptr_->on_cell_processed()); + TRY_STATUS(logger_ptr_->on_cells_processed(1)); } } if (logger_ptr_) { @@ -588,7 +588,7 @@ td::Result BagOfCells::serialize_to_impl(WriterT& writer, int mode) } // std::cerr << std::endl; if (logger_ptr_) { - TRY_STATUS(logger_ptr_->on_cell_processed()); + TRY_STATUS(logger_ptr_->on_cells_processed(1)); } } writer.chk(); diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 17e7eb69de..4f3fe5fcf2 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -210,6 +210,7 @@ class BagOfCellsLogger { void start_stage(std::string stage) { log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD); + last_speed_log_ = td::Timestamp::now(); processed_cells_ = 0; timer_ = {}; stage_ = std::move(stage); @@ -217,15 +218,19 @@ class BagOfCellsLogger { void finish_stage(td::Slice desc) { LOG(ERROR) << "serializer: " << stage_ << " took " << timer_.elapsed() << "s, " << desc; } - td::Status on_cell_processed() { - ++processed_cells_; - if (processed_cells_ % 1000 == 0) { + td::Status on_cells_processed(size_t count) { + processed_cells_ += count; + if (processed_cells_ / 1000 > last_token_check_) { TRY_STATUS(cancellation_token_.check()); + last_token_check_ = processed_cells_ / 1000; } if (log_speed_at_.is_in_past()) { - log_speed_at_ += LOG_SPEED_PERIOD; - LOG(WARNING) << "serializer: " << stage_ << " " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s"; + double period = td::Timestamp::now().at() - last_speed_log_.at(); + + LOG(WARNING) << "serializer: " << stage_ << " " << (double)processed_cells_ / period << " cells/s"; processed_cells_ = 0; + last_speed_log_ = td::Timestamp::now(); + log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD); } return td::Status::OK(); } @@ -236,6 +241,8 @@ class BagOfCellsLogger { td::CancellationToken cancellation_token_; td::Timestamp log_speed_at_; size_t processed_cells_ = 0; + size_t last_token_check_ = 0; + td::Timestamp last_speed_log_; static constexpr double LOG_SPEED_PERIOD = 120.0; }; class BagOfCells { @@ -390,7 +397,7 @@ td::Result std_boc_serialize_multi(std::vector> root, td::Status std_boc_serialize_to_file(Ref root, td::FileFd& fd, int mode = 0, td::CancellationToken cancellation_token = {}); -td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, +td::Status boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, int mode = 0, td::CancellationToken cancellation_token = {}); } // namespace vm diff --git a/crypto/vm/db/CellStorage.cpp b/crypto/vm/db/CellStorage.cpp index a07d85e870..f93e3fa59c 100644 --- a/crypto/vm/db/CellStorage.cpp +++ b/crypto/vm/db/CellStorage.cpp @@ -177,6 +177,28 @@ td::Result CellLoader::load(td::Slice hash, bool need_da return res; } +td::Result> CellLoader::load_bulk(td::Span hashes, bool need_data, + ExtCellCreator &ext_cell_creator) { + std::vector values; + TRY_RESULT(get_statuses, reader_->get_multi(hashes, &values)); + std::vector res; + res.reserve(hashes.size()); + for (size_t i = 0; i < hashes.size(); i++) { + auto get_status = get_statuses[i]; + if (get_status != KeyValue::GetStatus::Ok) { + DCHECK(get_status == KeyValue::GetStatus::NotFound); + res.push_back(LoadResult{}); + continue; + } + TRY_RESULT(load_res, load(hashes[i], values[i], need_data, ext_cell_creator)); + if (on_load_callback_) { + on_load_callback_(load_res); + } + res.push_back(std::move(load_res)); + } + return res; +} + td::Result CellLoader::load(td::Slice hash, td::Slice value, bool need_data, ExtCellCreator &ext_cell_creator) { LoadResult res; diff --git a/crypto/vm/db/CellStorage.h b/crypto/vm/db/CellStorage.h index ca32a8007d..7ae5867933 100644 --- a/crypto/vm/db/CellStorage.h +++ b/crypto/vm/db/CellStorage.h @@ -49,6 +49,7 @@ class CellLoader { }; CellLoader(std::shared_ptr reader, std::function on_load_callback = {}); td::Result load(td::Slice hash, bool need_data, ExtCellCreator &ext_cell_creator); + td::Result> load_bulk(td::Span hashes, bool need_data, ExtCellCreator &ext_cell_creator); static td::Result load(td::Slice hash, td::Slice value, bool need_data, ExtCellCreator &ext_cell_creator); td::Result load_refcnt(td::Slice hash); // This only loads refcnt_, cell_ == null KeyValueReader &key_value_reader() const { diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index d6731b0390..1a30f76d40 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -138,22 +138,24 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return td::Status::OK(); } td::Result> load_cell(td::Slice hash) override { - auto info = hash_table_.get_if_exists(hash); - if (info && info->sync_with_db) { - TRY_RESULT(loaded_cell, info->cell->load_cell()); - return std::move(loaded_cell.data_cell); - } - TRY_RESULT(res, loader_->load(hash, true, *this)); - if (res.status != CellLoader::LoadResult::Ok) { - return td::Status::Error("cell not found"); - } - Ref cell = res.cell(); - hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_loaded(info, hash, std::move(res)); }); - return cell; + TRY_RESULT(loaded_cell, get_cell_info_force(hash).cell->load_cell()); + return std::move(loaded_cell.data_cell); } td::Result> load_root(td::Slice hash) override { return load_cell(hash); } + td::Result>> load_bulk(td::Span hashes) override { + std::vector> result; + result.reserve(hashes.size()); + for (auto &hash : hashes) { + auto cell = load_cell(hash); + if (cell.is_error()) { + return cell.move_as_error(); + } + result.push_back(cell.move_as_ok()); + } + return result; + } td::Result> load_root_thread_safe(td::Slice hash) const override { return td::Status::Error("Not implemented"); } @@ -193,6 +195,9 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat promise->set_result(std::move(cell)); }); } + CellInfo &get_cell_info_force(td::Slice hash) { + return hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_force(info, hash); }); + } CellInfo &get_cell_info_lazy(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) { return hash_table_.apply(hash.substr(hash.size() - Cell::hash_bytes), [&](CellInfo &info) { update_cell_info_lazy(info, level_mask, hash, depth); }); @@ -383,6 +388,23 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return std::move(load_result.cell()); } + td::Result>> load_bulk(td::Span hashes) override { + if (db_) { + return db_->load_bulk(hashes); + } + TRY_RESULT(load_result, cell_loader_->load_bulk(hashes, true, *this)); + + std::vector> res; + res.reserve(load_result.size()); + for (auto &load_res : load_result) { + if (load_res.status != CellLoader::LoadResult::Ok) { + return td::Status::Error("cell not found"); + } + res.push_back(std::move(load_res.cell())); + } + return res; + } + private: static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() { static auto res = td::NamedThreadSafeCounter::get_default().get_counter("DynamicBagOfCellsDbLoader"); diff --git a/crypto/vm/db/DynamicBagOfCellsDb.h b/crypto/vm/db/DynamicBagOfCellsDb.h index 82028f3fe4..ec0aee9c14 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.h +++ b/crypto/vm/db/DynamicBagOfCellsDb.h @@ -45,6 +45,7 @@ class CellDbReader { public: virtual ~CellDbReader() = default; virtual td::Result> load_cell(td::Slice hash) = 0; + virtual td::Result>> load_bulk(td::Span hashes) = 0; }; class DynamicBagOfCellsDb { @@ -57,6 +58,7 @@ class DynamicBagOfCellsDb { virtual td::Status meta_erase(td::Slice key) = 0; virtual td::Result> load_cell(td::Slice hash) = 0; + virtual td::Result>> load_bulk(td::Span hashes) = 0; virtual td::Result> load_root(td::Slice hash) = 0; virtual td::Result> load_root_thread_safe(td::Slice hash) const = 0; virtual td::Result>> load_known_roots() const { diff --git a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp index eff74e214d..ecf3b76cbf 100644 --- a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp @@ -811,6 +811,10 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { td::Result> load_root(td::Slice hash) override { return load_cell(hash); } + td::Result>> load_bulk(td::Span hashes) override { + CHECK(cell_db_reader_); + return cell_db_reader_->load_bulk(hashes); + } td::Result> load_root_thread_safe(td::Slice hash) const override { // TODO: it is better to use AtomicRef, or atomic shared pointer // But to use AtomicRef we need a little refactoring @@ -1102,6 +1106,20 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { return load_cell_slow_path(hash); } + td::Result>> load_bulk(td::Span hashes) override { + // thread safe function + std::vector> result; + result.reserve(hashes.size()); + for (auto &hash : hashes) { + auto maybe_cell = load_cell(hash); + if (maybe_cell.is_error()) { + return maybe_cell.move_as_error(); + } + result.push_back(maybe_cell.move_as_ok()); + } + return result; + } + td::Result> load_ext_cell(Ref ext_cell) override { // thread safe function. // Called by external cell diff --git a/crypto/vm/db/InMemoryBagOfCellsDb.cpp b/crypto/vm/db/InMemoryBagOfCellsDb.cpp index 44be4ca8b5..829ed38d8b 100644 --- a/crypto/vm/db/InMemoryBagOfCellsDb.cpp +++ b/crypto/vm/db/InMemoryBagOfCellsDb.cpp @@ -454,6 +454,16 @@ class CellStorage { return td::Status::Error("not found"); } + td::Result>> load_bulk(td::Span hashes) const { + std::vector> res; + res.reserve(hashes.size()); + for (auto &hash : hashes) { + TRY_RESULT(cell, load_cell(hash)); + res.push_back(std::move(cell)); + } + return res; + } + td::Result> load_root_local(const CellHash &hash) const { auto lock = local_access_.lock(); if (auto it = local_roots_.find(hash); it != local_roots_.end()) { @@ -839,6 +849,9 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { td::Result> load_root(td::Slice hash) override { return storage_->load_root_local(CellHash::from_slice(hash)); } + td::Result>> load_bulk(td::Span hashes) override { + return storage_->load_bulk(td::transform(hashes, [](auto &hash) { return CellHash::from_slice(hash); })); + } td::Result> load_root_thread_safe(td::Slice hash) const override { return storage_->load_root_shared(CellHash::from_slice(hash)); } diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp index d209c88ed6..839e6235ad 100644 --- a/crypto/vm/large-boc-serializer.cpp +++ b/crypto/vm/large-boc-serializer.cpp @@ -32,6 +32,7 @@ namespace { class LargeBocSerializer { public: using Hash = Cell::Hash; + constexpr static int load_batch_size = 4'000'000; explicit LargeBocSerializer(std::shared_ptr reader) : reader(std::move(reader)) { } @@ -46,7 +47,6 @@ class LargeBocSerializer { private: std::shared_ptr reader; struct CellInfo { - Cell::Hash hash; std::array ref_idx; int idx; unsigned short serialized_size; @@ -95,6 +95,8 @@ void LargeBocSerializer::add_root(Hash root) { roots.emplace_back(root, -1); } +// Unlike crypto/vm/boc.cpp this implementation does not load all cells into memory +// and traverses them in BFS order to utilize bulk load of cells on the same level. td::Status LargeBocSerializer::import_cells() { if (logger_ptr_) { logger_ptr_->start_stage("import_cells"); @@ -111,46 +113,124 @@ td::Status LargeBocSerializer::import_cells() { return td::Status::OK(); } -td::Result LargeBocSerializer::import_cell(Hash hash, int depth) { - if (depth > Cell::max_depth) { - return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large"); +td::Result LargeBocSerializer::import_cell(Hash root_hash, int root_depth) { + const int start_ind = cell_count; + td::HashMap> current_depth_hashes; + + auto existing_it = cells.find(root_hash); + if (existing_it != cells.end()) { + existing_it->second.should_cache = true; + } else { + current_depth_hashes.emplace(root_hash, std::make_pair(cell_count, false)); + } + int current_depth = root_depth; + int next_child_idx = cell_count + 1; + while (!current_depth_hashes.empty()) { + if (current_depth > Cell::max_depth) { + return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large"); + } + + cell_list.resize(cell_list.size() + current_depth_hashes.size()); + td::HashMap> next_depth_hashes; + auto batch_start = current_depth_hashes.begin(); + while (batch_start != current_depth_hashes.end()) { + std::vector batch_hashes; + batch_hashes.reserve(load_batch_size); + std::vector*> batch_idxs_should_cache; + batch_idxs_should_cache.reserve(load_batch_size); + + while (batch_hashes.size() < load_batch_size && batch_start != current_depth_hashes.end()) { + batch_hashes.push_back(batch_start->first.as_slice()); + batch_idxs_should_cache.push_back(&batch_start->second); + ++batch_start; + } + + TRY_RESULT_PREFIX(loaded_results, reader->load_bulk(batch_hashes), + "error while importing a cell into a bag of cells: "); + DCHECK(loaded_results.size() == batch_hashes.size()); + + for (size_t i = 0; i < loaded_results.size(); ++i) { + auto& cell = loaded_results[i]; + + if (cell->get_virtualization() != 0) { + return td::Status::Error( + "error while importing a cell into a bag of cells: cell has non-zero virtualization level"); + } + + const auto hash = cell->get_hash(); + CellSlice cs(std::move(cell)); + + DCHECK(cs.size_refs() <= 4); + std::array refs{-1, -1, -1, -1}; + for (unsigned j = 0; j < cs.size_refs(); j++) { + auto child = cs.prefetch_ref(j); + const auto child_hash = child->get_hash(); + + auto existing_global_it = cells.find(child_hash); + if (existing_global_it != cells.end()) { + existing_global_it->second.should_cache = true; + refs[j] = existing_global_it->second.idx; + continue; + } + auto current_depth_it = current_depth_hashes.find(child_hash); + if (current_depth_it != current_depth_hashes.end()) { + current_depth_it->second.second = true; + refs[j] = current_depth_it->second.first; + continue; + } + auto next_depth_it = next_depth_hashes.find(child_hash); + if (next_depth_it != next_depth_hashes.end()) { + next_depth_it->second.second = true; + refs[j] = next_depth_it->second.first; + continue; + } + auto res = next_depth_hashes.emplace(child_hash, std::make_pair(next_child_idx, false)); + refs[j] = next_child_idx++; + } + + auto dc = cs.move_as_loaded_cell().data_cell; + auto idx_should_cache = batch_idxs_should_cache[i]; + auto res = cells.emplace(hash, CellInfo(idx_should_cache->first, std::move(refs))); + DCHECK(res.second); + cell_list[idx_should_cache->first] = &*res.first; + CellInfo& dc_info = res.first->second; + dc_info.should_cache = idx_should_cache->second; + dc_info.hcnt = static_cast(dc->get_level_mask().get_hashes_count()); + DCHECK(dc_info.hcnt <= 4); + dc_info.wt = 0; // will be calculated after traversing + TRY_RESULT(serialized_size, td::narrow_cast_safe(dc->get_serialized_size())); + data_bytes += dc_info.serialized_size = serialized_size; + cell_count++; + } + if (logger_ptr_) { + TRY_STATUS(logger_ptr_->on_cells_processed(batch_hashes.size())); + } + } + + current_depth_hashes = std::move(next_depth_hashes); + next_depth_hashes.clear(); + current_depth++; + } + DCHECK(next_child_idx == cell_count); + + for (int idx = cell_count - 1; idx >= start_ind; --idx) { + CellInfo& cell_info = cell_list[idx]->second; + + unsigned sum_child_wt = 1; + for (size_t j = 0; j < cell_info.ref_idx.size(); ++j) { + int child_idx = cell_info.ref_idx[j]; + if (child_idx == -1) { + continue; + } + sum_child_wt += cell_list[child_idx]->second.wt; + ++int_refs; + } + cell_info.wt = static_cast(std::min(0xffU, sum_child_wt)); } - if (logger_ptr_) { - TRY_STATUS(logger_ptr_->on_cell_processed()); - } - auto it = cells.find(hash); - if (it != cells.end()) { - it->second.should_cache = true; - return it->second.idx; - } - TRY_RESULT(cell, reader->load_cell(hash.as_slice())); - if (cell->get_virtualization() != 0) { - return td::Status::Error( - "error while importing a cell into a bag of cells: cell has non-zero virtualization level"); - } - CellSlice cs(std::move(cell)); - std::array refs; - std::fill(refs.begin(), refs.end(), -1); - DCHECK(cs.size_refs() <= 4); - unsigned sum_child_wt = 1; - for (unsigned i = 0; i < cs.size_refs(); i++) { - TRY_RESULT(ref, import_cell(cs.prefetch_ref(i)->get_hash(), depth + 1)); - refs[i] = ref; - sum_child_wt += cell_list[ref]->second.wt; - ++int_refs; - } - auto dc = cs.move_as_loaded_cell().data_cell; - auto res = cells.emplace(hash, CellInfo(cell_count, refs)); - DCHECK(res.second); - cell_list.push_back(&*res.first); - CellInfo& dc_info = res.first->second; - dc_info.wt = (unsigned char)std::min(0xffU, sum_child_wt); - unsigned hcnt = dc->get_level_mask().get_hashes_count(); - DCHECK(hcnt <= 4); - dc_info.hcnt = (unsigned char)hcnt; - TRY_RESULT(serialized_size, td::narrow_cast_safe(dc->get_serialized_size())); - data_bytes += dc_info.serialized_size = serialized_size; - return cell_count++; + + auto root_it = cells.find(root_hash); + DCHECK(root_it != cells.end()); + return root_it->second.idx; } void LargeBocSerializer::reorder_cells() { @@ -386,7 +466,7 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { } store_offset(fixed_offset); if (logger_ptr_) { - TRY_STATUS(logger_ptr_->on_cell_processed()); + TRY_STATUS(logger_ptr_->on_cells_processed(1)); } } DCHECK(offs == info.data_size); @@ -399,26 +479,42 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { if (logger_ptr_) { logger_ptr_->start_stage("serialize"); } - for (int i = 0; i < cell_count; ++i) { - auto hash = cell_list[cell_count - 1 - i]->first; - const auto& dc_info = cell_list[cell_count - 1 - i]->second; - TRY_RESULT(dc, reader->load_cell(hash.as_slice())); - bool with_hash = (mode & Mode::WithIntHashes) && !dc_info.wt; - if (dc_info.is_root_cell && (mode & Mode::WithTopHash)) { - with_hash = true; + for (int batch_start = 0; batch_start < cell_count; batch_start += load_batch_size) { + int batch_end = std::min(batch_start + static_cast(load_batch_size), cell_count); + + std::vector batch_hashes; + batch_hashes.reserve(batch_end - batch_start); + for (int i = batch_start; i < batch_end; ++i) { + int cell_index = cell_count - 1 - i; + batch_hashes.push_back(cell_list[cell_index]->first.as_slice()); } - unsigned char buf[256]; - int s = dc->serialize(buf, 256, with_hash); - writer.store_bytes(buf, s); - DCHECK(dc->size_refs() == dc_info.get_ref_num()); - unsigned ref_num = dc_info.get_ref_num(); - for (unsigned j = 0; j < ref_num; ++j) { - int k = cell_count - 1 - dc_info.ref_idx[j]; - DCHECK(k > i && k < cell_count); - store_ref(k); + + TRY_RESULT(batch_cells, reader->load_bulk(std::move(batch_hashes))); + + for (int i = batch_start; i < batch_end; ++i) { + int idx_in_batch = i - batch_start; + int cell_index = cell_count - 1 - i; + + const auto& dc_info = cell_list[cell_index]->second; + auto& dc = batch_cells[idx_in_batch]; + + bool with_hash = (mode & Mode::WithIntHashes) && !dc_info.wt; + if (dc_info.is_root_cell && (mode & Mode::WithTopHash)) { + with_hash = true; + } + unsigned char buf[256]; + int s = dc->serialize(buf, 256, with_hash); + writer.store_bytes(buf, s); + DCHECK(dc->size_refs() == dc_info.get_ref_num()); + unsigned ref_num = dc_info.get_ref_num(); + for (unsigned j = 0; j < ref_num; ++j) { + int k = cell_count - 1 - dc_info.ref_idx[j]; + DCHECK(k > i && k < cell_count); + store_ref(k); + } } if (logger_ptr_) { - TRY_STATUS(logger_ptr_->on_cell_processed()); + TRY_STATUS(logger_ptr_->on_cells_processed(batch_hashes.size())); } } DCHECK(writer.position() - keep_position == info.data_size); @@ -435,7 +531,7 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { } } // namespace -td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, +td::Status boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, int mode, td::CancellationToken cancellation_token) { td::Timer timer; CHECK(reader != nullptr) diff --git a/tddb/td/db/KeyValue.h b/tddb/td/db/KeyValue.h index c3f83919bf..6bbe7df3cf 100644 --- a/tddb/td/db/KeyValue.h +++ b/tddb/td/db/KeyValue.h @@ -20,6 +20,7 @@ #include "td/utils/Status.h" #include "td/utils/Time.h" #include "td/utils/logging.h" +#include "td/utils/Span.h" #include "td/utils/ThreadSafeCounter.h" #include namespace td { @@ -59,7 +60,8 @@ class KeyValueReader { virtual ~KeyValueReader() = default; enum class GetStatus : int32 { Ok, NotFound }; - virtual Result get(Slice key, std::string& value) = 0; + virtual Result get(Slice key, std::string &value) = 0; + virtual Result> get_multi(td::Span keys, std::vector *values) = 0; virtual Result count(Slice prefix) = 0; virtual Status for_each(std::function f) { return Status::Error("for_each is not supported"); @@ -77,6 +79,14 @@ class PrefixedKeyValueReader : public KeyValueReader { Result get(Slice key, std::string& value) override { return reader_->get(PSLICE() << prefix_ << key, value); } + Result> get_multi(td::Span keys, std::vector *values) override { + std::vector prefixed_keys; + prefixed_keys.reserve(keys.size()); + for (auto &key : keys) { + prefixed_keys.push_back(PSLICE() << prefix_ << key); + } + return reader_->get_multi(prefixed_keys, values); + } Result count(Slice prefix) override { return reader_->count(PSLICE() << prefix_ << prefix); } @@ -125,6 +135,14 @@ class PrefixedKeyValue : public KeyValue { Result get(Slice key, std::string& value) override { return kv_->get(PSLICE() << prefix_ << key, value); } + Result> get_multi(td::Span keys, std::vector *values) override { + std::vector prefixed_keys; + prefixed_keys.reserve(keys.size()); + for (auto &key : keys) { + prefixed_keys.push_back(PSLICE() << prefix_ << key); + } + return kv_->get_multi(prefixed_keys, values); + } Result count(Slice prefix) override { return kv_->count(PSLICE() << prefix_ << prefix); } diff --git a/tddb/td/db/MemoryKeyValue.cpp b/tddb/td/db/MemoryKeyValue.cpp index 7105f72b94..c79daccea9 100644 --- a/tddb/td/db/MemoryKeyValue.cpp +++ b/tddb/td/db/MemoryKeyValue.cpp @@ -19,6 +19,7 @@ #include "td/db/MemoryKeyValue.h" #include "td/utils/format.h" +#include "td/utils/Span.h" namespace td { Result MemoryKeyValue::get(Slice key, std::string &value) { @@ -41,6 +42,17 @@ std::unique_ptr MemoryKeyValue:: return lock(buckets_[bucket_id]); } +Result> MemoryKeyValue::get_multi(td::Span keys, + std::vector *values) { + values->resize(keys.size()); + std::vector res; + res.reserve(keys.size()); + for (size_t i = 0; i < keys.size(); i++) { + res.push_back(get(keys[i], (*values)[i]).move_as_ok()); + } + return res; +} + Status MemoryKeyValue::for_each(std::function f) { for (auto &unlocked_bucket : buckets_) { auto bucket = lock(unlocked_bucket); diff --git a/tddb/td/db/MemoryKeyValue.h b/tddb/td/db/MemoryKeyValue.h index cf896095d8..6a5be7866d 100644 --- a/tddb/td/db/MemoryKeyValue.h +++ b/tddb/td/db/MemoryKeyValue.h @@ -34,6 +34,7 @@ class MemoryKeyValue : public KeyValue { MemoryKeyValue(std::shared_ptr merger) : merger_(std::move(merger)) { } Result get(Slice key, std::string& value) override; + Result> get_multi(td::Span keys, std::vector *values) override; Status for_each(std::function f) override; Status for_each_in_range(Slice begin, Slice end, std::function f) override; Status set(Slice key, Slice value) override; diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index d9040b695b..660381a31d 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -187,6 +187,40 @@ Result RocksDb::get(Slice key, std::string &value) { return from_rocksdb(status); } +Result> RocksDb::get_multi(td::Span keys, std::vector *values) { + std::vector statuses(keys.size()); + std::vector keys_rocksdb; + keys_rocksdb.reserve(keys.size()); + for (auto &key : keys) { + keys_rocksdb.push_back(to_rocksdb(key)); + } + std::vector values_rocksdb(keys.size()); + rocksdb::ReadOptions options; + if (snapshot_) { + options.snapshot = snapshot_.get(); + db_->MultiGet(options, db_->DefaultColumnFamily(), keys_rocksdb.size(), keys_rocksdb.data(), values_rocksdb.data(), statuses.data()); + } else if (transaction_) { + transaction_->MultiGet(options, db_->DefaultColumnFamily(), keys_rocksdb.size(), keys_rocksdb.data(), values_rocksdb.data(), statuses.data()); + } else { + db_->MultiGet(options, db_->DefaultColumnFamily(), keys_rocksdb.size(), keys_rocksdb.data(), values_rocksdb.data(), statuses.data()); + } + std::vector res(statuses.size()); + values->resize(statuses.size()); + for (size_t i = 0; i < statuses.size(); i++) { + auto &status = statuses[i]; + if (status.ok()) { + res[i] = GetStatus::Ok; + values->at(i) = values_rocksdb[i].ToString(); + } else if (status.code() == rocksdb::Status::kNotFound) { + res[i] = GetStatus::NotFound; + values->at(i) = ""; + } else { + return from_rocksdb(status); + } + } + return res; +} + Status RocksDb::set(Slice key, Slice value) { if (write_batch_) { return from_rocksdb(write_batch_->Put(to_rocksdb(key), to_rocksdb(value))); diff --git a/tddb/td/db/RocksDb.h b/tddb/td/db/RocksDb.h index 71ece1e2aa..c9fa93e10e 100644 --- a/tddb/td/db/RocksDb.h +++ b/tddb/td/db/RocksDb.h @@ -86,6 +86,7 @@ class RocksDb : public KeyValue { static Result open(std::string path, RocksDbOptions options = {}); Result get(Slice key, std::string &value) override; + Result> get_multi(td::Span keys, std::vector *values) override; Status set(Slice key, Slice value) override; Status merge(Slice key, Slice value) override; Status erase(Slice key) override; diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index bc3d7b5e0d..b8fd84d171 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -282,8 +282,38 @@ class CachedCellDbReader : public vm::CellDbReader { } return parent_->load_cell(hash); } + td::Result>> load_bulk(td::Span hashes) override { + total_reqs_ += hashes.size(); + if (!cache_) { + ++bulk_reqs_; + return parent_->load_bulk(hashes); + } + std::vector missing_hashes; + std::vector missing_indices; + std::vector> res(hashes.size()); + for (size_t i = 0; i < hashes.size(); i++) { + auto it = cache_->find(hashes[i]); + if (it != cache_->end()) { + ++cached_reqs_; + TRY_RESULT(loaded_cell, (*it)->load_cell()); + res[i] = loaded_cell.data_cell; + continue; + } + missing_hashes.push_back(hashes[i]); + missing_indices.push_back(i); + } + if (missing_hashes.empty()) { + return std::move(res); + } + TRY_RESULT(missing_cells, parent_->load_bulk(missing_hashes)); + for (size_t i = 0; i < missing_indices.size(); i++) { + res[missing_indices[i]] = missing_cells[i]; + } + return res; + }; void print_stats() const { - LOG(WARNING) << "CachedCellDbReader stats : " << total_reqs_ << " reads, " << cached_reqs_ << " cached"; + LOG(WARNING) << "CachedCellDbReader stats : " << total_reqs_ << " reads, " << cached_reqs_ << " cached, " + << bulk_reqs_ << " bulk reqs"; } private: std::shared_ptr parent_; @@ -291,6 +321,7 @@ class CachedCellDbReader : public vm::CellDbReader { td::uint64 total_reqs_ = 0; td::uint64 cached_reqs_ = 0; + td::uint64 bulk_reqs_ = 0; }; void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard) { @@ -373,7 +404,7 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state previous_state_cache->prepare_cache(shard); } auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache->cache); - auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); + auto res = vm::boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); new_cell_db_reader->print_stats(); return res; }; @@ -443,7 +474,7 @@ void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Refprepare_cache(shard); } auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache->cache); - auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); + auto res = vm::boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); new_cell_db_reader->print_stats(); return res; }; From 80ecebd5a1157176c0c18f9f29f39d9b497f5c44 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Sat, 5 Apr 2025 15:51:32 +0400 Subject: [PATCH 199/524] Fix failing windows build (#1603) * test builds - testnet * set CMAKE_POLICY_VERSION_MINIMUM for WIN32 too * adjust as per review comment --- .github/workflows/docker-ubuntu-image.yml | 1 - CMakeLists.txt | 7 +++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-ubuntu-image.yml b/.github/workflows/docker-ubuntu-image.yml index 0284dcd545..7a5eee10ad 100644 --- a/.github/workflows/docker-ubuntu-image.yml +++ b/.github/workflows/docker-ubuntu-image.yml @@ -5,7 +5,6 @@ on: push: branches: - 'master' - env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} diff --git a/CMakeLists.txt b/CMakeLists.txt index ad1aaf5063..b002b2844c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,9 +77,12 @@ endif() if (WIN32) message("Add wingetopt") - add_subdirectory(third-party/wingetopt EXCLUDE_FROM_ALL) + function(wingetopt_scope) + set(CMAKE_POLICY_VERSION_MINIMUM "3.10") + add_subdirectory(third-party/wingetopt EXCLUDE_FROM_ALL) + endfunction() + wingetopt_scope() set(WINGETOPT_FOUND 1) - message(STATUS "Use wingetopt") endif() set(CRC32C_BUILD_TESTS OFF CACHE BOOL "Build CRC32C's unit tests") From e745696757a298594e54d3fedbca604c3eb59a3f Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sat, 5 Apr 2025 16:58:38 +0300 Subject: [PATCH 200/524] [Tolk] Type aliases `type NewName = ` An alias creates a new name for an existing type but remains interchangeable with it: > type UserId = int32; > type MaybeOwnerHash = bytes32?; > type MyStorage = (int, int, slice, cell); Aliases never occur at runtime, they are compile-time only (they exist as types until IR generation). That's why lots of code comparing types use `type->unwrap_alias()` or `try_as`. In stdlib, a special alias was introduced: > type dict = cell?; Functions working with dictionaries use `dict`, improving code clarity. --- crypto/smartcont/tolk-stdlib/common.tolk | 18 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 140 ++++++------- tolk-tester/tests/constants-tests.tolk | 13 +- tolk-tester/tests/generics-1.tolk | 52 ++++- tolk-tester/tests/indexed-access.tolk | 15 +- tolk-tester/tests/inference-tests.tolk | 2 +- .../tests/invalid-declaration/err-1483.tolk | 8 + .../tests/invalid-semantics/err-4304.tolk | 10 + .../tests/invalid-syntax/err-3482.tolk | 6 + .../tests/invalid-typing/err-6188.tolk | 7 +- .../tests/invalid-typing/err-6337.tolk | 2 +- .../tests/invalid-typing/err-6450.tolk | 6 +- .../tests/invalid-typing/err-6553.tolk | 6 +- .../tests/invalid-typing/err-6580.tolk | 4 +- .../tests/invalid-typing/err-6734.tolk | 6 +- .../tests/invalid-typing/err-6737.tolk | 4 +- tolk-tester/tests/nullable-tensors.tolk | 30 ++- tolk-tester/tests/nullable-types.tolk | 15 +- tolk-tester/tests/smart-cast-tests.tolk | 4 +- tolk-tester/tests/type-aliases-tests.tolk | 189 ++++++++++++++++++ tolk-tester/tests/var-apply-tests.tolk | 34 +++- tolk/abscode.cpp | 2 + tolk/ast-from-tokens.cpp | 19 ++ tolk/ast-stringifier.h | 4 + tolk/ast.cpp | 8 + tolk/ast.h | 20 ++ tolk/fwd-declarations.h | 2 + tolk/generics-helpers.cpp | 8 +- tolk/pipe-ast-to-legacy.cpp | 37 +++- tolk/pipe-check-inferred-types.cpp | 53 ++++- tolk/pipe-infer-types-and-calls.cpp | 32 +-- tolk/pipe-optimize-boolean-expr.cpp | 29 ++- tolk/pipe-register-symbols.cpp | 10 + tolk/pipe-resolve-identifiers.cpp | 41 +++- tolk/smart-casts-cfg.cpp | 15 +- tolk/symtable.cpp | 12 ++ tolk/symtable.h | 13 ++ tolk/type-system.cpp | 163 ++++++++++++++- tolk/type-system.h | 35 ++++ 39 files changed, 909 insertions(+), 165 deletions(-) create mode 100644 tolk-tester/tests/invalid-declaration/err-1483.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4304.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3482.tolk create mode 100644 tolk-tester/tests/type-aliases-tests.tolk diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 0075764b66..dc0ec4a61f 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -3,6 +3,12 @@ // More specific functions are required to be imported explicitly, like "@stdlib/tvm-dicts". tolk 0.10 +/// In Tolk v1.x there would be a type `map`. +/// Currently, working with dictionaries is still low-level, with raw cells. +/// But just for clarity, we use "dict" instead of a "cell?" where a cell-dictionary is assumed. +/// Every dictionary object can be null. TVM NULL is essentially "empty dictionary". +type dict = cell?; + /** Tuple manipulation primitives. Elements of a tuple can be of arbitrary type. @@ -144,9 +150,9 @@ fun getMyOriginalBalance(): int /// Same as [getMyOriginalBalance], but returns a tuple: /// `int` — balance in nanotoncoins; -/// `cell` — a dictionary with 32-bit keys representing the balance of "extra currencies". +/// `dict` — a dictionary with 32-bit keys representing the balance of "extra currencies". @pure -fun getMyOriginalBalanceWithExtraCurrencies(): [int, cell?] +fun getMyOriginalBalanceWithExtraCurrencies(): [int, dict] asm "BALANCE"; /// Returns the logical time of the current transaction. @@ -476,12 +482,12 @@ fun getLastBits(self: slice, len: int): slice /// Loads a dictionary (TL HashMapE structure, represented as TVM cell) from a slice. /// Returns `null` if `nothing` constructor is used. @pure -fun loadDict(mutate self: slice): cell? +fun loadDict(mutate self: slice): dict asm( -> 1 0) "LDDICT"; /// Preloads a dictionary (cell) from a slice. @pure -fun preloadDict(self: slice): cell? +fun preloadDict(self: slice): dict asm "PLDDICT"; /// Loads a dictionary as [loadDict], but returns only the remainder of the slice. @@ -556,7 +562,7 @@ fun storeBool(mutate self: builder, x: bool): self /// Stores dictionary (represented by TVM `cell` or `null`) into a builder. /// In other words, stores a `1`-bit and a reference to [c] if [c] is not `null` and `0`-bit otherwise. @pure -fun storeDict(mutate self: builder, c: cell?): self +fun storeDict(mutate self: builder, c: dict): self asm(c self) "STDICT"; /// Stores (Maybe ^Cell) into a builder. @@ -720,7 +726,7 @@ fun reserveToncoinsOnBalance(nanoTonCoins: int, reserveMode: int): void /// Similar to [reserveToncoinsOnBalance], but also accepts a dictionary extraAmount (represented by a cell or null) /// with extra currencies. In this way currencies other than Toncoin can be reserved. -fun reserveExtraCurrenciesOnBalance(nanoTonCoins: int, extraAmount: cell?, reserveMode: int): void +fun reserveExtraCurrenciesOnBalance(nanoTonCoins: int, extraAmount: dict, reserveMode: int): void asm "RAWRESERVEX"; diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index d4babd1efb..83fe705c49 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -13,285 +13,289 @@ tolk 0.10 Every dictionary object (`self` parameter) can be null. TVM NULL is essentially "empty dictionary". */ +/// In @stdlib/common.tolk, there is a type alias: +/// `type dict = cell?;` +/// For clarity, we use "dict" instead of a "cell?" where a cell-dictionary is assumed. + /// Creates an empty dictionary, which is actually a null value. Equivalent to PUSHNULL @pure -fun createEmptyDict(): cell? +fun createEmptyDict(): dict asm "NEWDICT"; /// Checks whether a dictionary is empty. @pure -fun dictIsEmpty(self: cell?): bool +fun dictIsEmpty(self: dict): bool asm "DICTEMPTY"; @pure -fun iDictGet(self: cell?, keyLen: int, key: int): (slice?, bool) +fun iDictGet(self: dict, keyLen: int, key: int): (slice?, bool) asm(key self keyLen) "DICTIGET" "NULLSWAPIFNOT"; @pure -fun uDictGet(self: cell?, keyLen: int, key: int): (slice?, bool) +fun uDictGet(self: dict, keyLen: int, key: int): (slice?, bool) asm(key self keyLen) "DICTUGET" "NULLSWAPIFNOT"; @pure -fun sDictGet(self: cell?, keyLen: int, key: slice): (slice?, bool) +fun sDictGet(self: dict, keyLen: int, key: slice): (slice?, bool) asm(key self keyLen) "DICTGET" "NULLSWAPIFNOT"; @pure -fun iDictSet(mutate self: cell?, keyLen: int, key: int, value: slice): void +fun iDictSet(mutate self: dict, keyLen: int, key: int, value: slice): void asm(value key self keyLen) "DICTISET"; @pure -fun uDictSet(mutate self: cell?, keyLen: int, key: int, value: slice): void +fun uDictSet(mutate self: dict, keyLen: int, key: int, value: slice): void asm(value key self keyLen) "DICTUSET"; @pure -fun sDictSet(mutate self: cell?, keyLen: int, key: slice, value: slice): void +fun sDictSet(mutate self: dict, keyLen: int, key: slice, value: slice): void asm(value key self keyLen) "DICTSET"; @pure -fun iDictSetRef(mutate self: cell?, keyLen: int, key: int, value: cell): void +fun iDictSetRef(mutate self: dict, keyLen: int, key: int, value: cell): void asm(value key self keyLen) "DICTISETREF"; @pure -fun uDictSetRef(mutate self: cell?, keyLen: int, key: int, value: cell): void +fun uDictSetRef(mutate self: dict, keyLen: int, key: int, value: cell): void asm(value key self keyLen) "DICTUSETREF"; @pure -fun sDictSetRef(mutate self: cell?, keyLen: int, key: slice, value: cell): void +fun sDictSetRef(mutate self: dict, keyLen: int, key: slice, value: cell): void asm(value key self keyLen) "DICTSETREF"; @pure -fun iDictSetIfNotExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool +fun iDictSetIfNotExists(mutate self: dict, keyLen: int, key: int, value: slice): bool asm(value key self keyLen) "DICTIADD"; @pure -fun uDictSetIfNotExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool +fun uDictSetIfNotExists(mutate self: dict, keyLen: int, key: int, value: slice): bool asm(value key self keyLen) "DICTUADD"; @pure -fun iDictSetIfExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool +fun iDictSetIfExists(mutate self: dict, keyLen: int, key: int, value: slice): bool asm(value key self keyLen) "DICTIREPLACE"; @pure -fun uDictSetIfExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool +fun uDictSetIfExists(mutate self: dict, keyLen: int, key: int, value: slice): bool asm(value key self keyLen) "DICTUREPLACE"; @pure -fun iDictGetRef(self: cell?, keyLen: int, key: int): (cell?, bool) +fun iDictGetRef(self: dict, keyLen: int, key: int): (dict, bool) asm(key self keyLen) "DICTIGETREF" "NULLSWAPIFNOT"; @pure -fun uDictGetRef(self: cell?, keyLen: int, key: int): (cell?, bool) +fun uDictGetRef(self: dict, keyLen: int, key: int): (dict, bool) asm(key self keyLen) "DICTUGETREF" "NULLSWAPIFNOT"; @pure -fun sDictGetRef(self: cell?, keyLen: int, key: slice): (cell?, bool) +fun sDictGetRef(self: dict, keyLen: int, key: slice): (dict, bool) asm(key self keyLen) "DICTGETREF" "NULLSWAPIFNOT"; @pure -fun iDictGetRefOrNull(self: cell?, keyLen: int, key: int): cell? +fun iDictGetRefOrNull(self: dict, keyLen: int, key: int): dict asm(key self keyLen) "DICTIGETOPTREF"; @pure -fun uDictGetRefOrNull(self: cell?, keyLen: int, key: int): cell? +fun uDictGetRefOrNull(self: dict, keyLen: int, key: int): dict asm(key self keyLen) "DICTUGETOPTREF"; @pure -fun sDictGetRefOrNull(self: cell?, keyLen: int, key: slice): cell? +fun sDictGetRefOrNull(self: dict, keyLen: int, key: slice): dict asm(key self keyLen) "DICTGETOPTREF"; @pure -fun iDictDelete(mutate self: cell?, keyLen: int, key: int): bool +fun iDictDelete(mutate self: dict, keyLen: int, key: int): bool asm(key self keyLen) "DICTIDEL"; @pure -fun uDictDelete(mutate self: cell?, keyLen: int, key: int): bool +fun uDictDelete(mutate self: dict, keyLen: int, key: int): bool asm(key self keyLen) "DICTUDEL"; @pure -fun sDictDelete(mutate self: cell?, keyLen: int, key: slice): bool +fun sDictDelete(mutate self: dict, keyLen: int, key: slice): bool asm(key self keyLen) "DICTDEL"; @pure -fun iDictSetAndGet(mutate self: cell?, keyLen: int, key: int, value: slice): (slice?, bool) +fun iDictSetAndGet(mutate self: dict, keyLen: int, key: int, value: slice): (slice?, bool) asm(value key self keyLen) "DICTISETGET" "NULLSWAPIFNOT"; @pure -fun uDictSetAndGet(mutate self: cell?, keyLen: int, key: int, value: slice): (slice?, bool) +fun uDictSetAndGet(mutate self: dict, keyLen: int, key: int, value: slice): (slice?, bool) asm(value key self keyLen) "DICTUSETGET" "NULLSWAPIFNOT"; @pure -fun sDictSetAndGet(mutate self: cell?, keyLen: int, key: slice, value: slice): (slice?, bool) +fun sDictSetAndGet(mutate self: dict, keyLen: int, key: slice, value: slice): (slice?, bool) asm(value key self keyLen) "DICTSETGET" "NULLSWAPIFNOT"; @pure -fun iDictSetAndGetRefOrNull(mutate self: cell?, keyLen: int, key: int, value: cell): cell? +fun iDictSetAndGetRefOrNull(mutate self: dict, keyLen: int, key: int, value: cell): dict asm(value key self keyLen) "DICTISETGETOPTREF"; @pure -fun uDictSetAndGetRefOrNull(mutate self: cell?, keyLen: int, key: int, value: cell): cell? +fun uDictSetAndGetRefOrNull(mutate self: dict, keyLen: int, key: int, value: cell): dict asm(value key self keyLen) "DICTUSETGETOPTREF"; @pure -fun iDictDeleteAndGet(mutate self: cell?, keyLen: int, key: int): (slice?, bool) +fun iDictDeleteAndGet(mutate self: dict, keyLen: int, key: int): (slice?, bool) asm(key self keyLen) "DICTIDELGET" "NULLSWAPIFNOT"; @pure -fun uDictDeleteAndGet(mutate self: cell?, keyLen: int, key: int): (slice?, bool) +fun uDictDeleteAndGet(mutate self: dict, keyLen: int, key: int): (slice?, bool) asm(key self keyLen) "DICTUDELGET" "NULLSWAPIFNOT"; @pure -fun sDictDeleteAndGet(mutate self: cell?, keyLen: int, key: slice): (slice?, bool) +fun sDictDeleteAndGet(mutate self: dict, keyLen: int, key: slice): (slice?, bool) asm(key self keyLen) "DICTDELGET" "NULLSWAPIFNOT"; @pure -fun iDictSetBuilder(mutate self: cell?, keyLen: int, key: int, value: builder): void +fun iDictSetBuilder(mutate self: dict, keyLen: int, key: int, value: builder): void asm(value key self keyLen) "DICTISETB"; @pure -fun uDictSetBuilder(mutate self: cell?, keyLen: int, key: int, value: builder): void +fun uDictSetBuilder(mutate self: dict, keyLen: int, key: int, value: builder): void asm(value key self keyLen) "DICTUSETB"; @pure -fun sDictSetBuilder(mutate self: cell?, keyLen: int, key: slice, value: builder): void +fun sDictSetBuilder(mutate self: dict, keyLen: int, key: slice, value: builder): void asm(value key self keyLen) "DICTSETB"; @pure -fun iDictSetBuilderIfNotExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool +fun iDictSetBuilderIfNotExists(mutate self: dict, keyLen: int, key: int, value: builder): bool asm(value key self keyLen) "DICTIADDB"; @pure -fun uDictSetBuilderIfNotExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool +fun uDictSetBuilderIfNotExists(mutate self: dict, keyLen: int, key: int, value: builder): bool asm(value key self keyLen) "DICTUADDB"; @pure -fun iDictSetBuilderIfExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool +fun iDictSetBuilderIfExists(mutate self: dict, keyLen: int, key: int, value: builder): bool asm(value key self keyLen) "DICTIREPLACEB"; @pure -fun uDictSetBuilderIfExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool +fun uDictSetBuilderIfExists(mutate self: dict, keyLen: int, key: int, value: builder): bool asm(value key self keyLen) "DICTUREPLACEB"; @pure -fun iDictDeleteFirstAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) +fun iDictDeleteFirstAndGet(mutate self: dict, keyLen: int): (int?, slice?, bool) asm(-> 0 2 1 3) "DICTIREMMIN" "NULLSWAPIFNOT2"; @pure -fun uDictDeleteFirstAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) +fun uDictDeleteFirstAndGet(mutate self: dict, keyLen: int): (int?, slice?, bool) asm(-> 0 2 1 3) "DICTUREMMIN" "NULLSWAPIFNOT2"; @pure -fun sDictDeleteFirstAndGet(mutate self: cell?, keyLen: int): (slice?, slice?, bool) +fun sDictDeleteFirstAndGet(mutate self: dict, keyLen: int): (slice?, slice?, bool) asm(-> 0 2 1 3) "DICTREMMIN" "NULLSWAPIFNOT2"; @pure -fun iDictDeleteLastAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) +fun iDictDeleteLastAndGet(mutate self: dict, keyLen: int): (int?, slice?, bool) asm(-> 0 2 1 3) "DICTIREMMAX" "NULLSWAPIFNOT2"; @pure -fun uDictDeleteLastAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) +fun uDictDeleteLastAndGet(mutate self: dict, keyLen: int): (int?, slice?, bool) asm(-> 0 2 1 3) "DICTUREMMAX" "NULLSWAPIFNOT2"; @pure -fun sDictDeleteLastAndGet(mutate self: cell?, keyLen: int): (slice?, slice?, bool) +fun sDictDeleteLastAndGet(mutate self: dict, keyLen: int): (slice?, slice?, bool) asm(-> 0 2 1 3) "DICTREMMAX" "NULLSWAPIFNOT2"; @pure -fun iDictGetFirst(self: cell?, keyLen: int): (int?, slice?, bool) +fun iDictGetFirst(self: dict, keyLen: int): (int?, slice?, bool) asm (-> 1 0 2) "DICTIMIN" "NULLSWAPIFNOT2"; @pure -fun uDictGetFirst(self: cell?, keyLen: int): (int?, slice?, bool) +fun uDictGetFirst(self: dict, keyLen: int): (int?, slice?, bool) asm (-> 1 0 2) "DICTUMIN" "NULLSWAPIFNOT2"; @pure -fun sDictGetFirst(self: cell?, keyLen: int): (slice?, slice?, bool) +fun sDictGetFirst(self: dict, keyLen: int): (slice?, slice?, bool) asm (-> 1 0 2) "DICTMIN" "NULLSWAPIFNOT2"; @pure -fun iDictGetFirstAsRef(self: cell?, keyLen: int): (int?, cell?, bool) +fun iDictGetFirstAsRef(self: dict, keyLen: int): (int?, dict, bool) asm (-> 1 0 2) "DICTIMINREF" "NULLSWAPIFNOT2"; @pure -fun uDictGetFirstAsRef(self: cell?, keyLen: int): (int?, cell?, bool) +fun uDictGetFirstAsRef(self: dict, keyLen: int): (int?, dict, bool) asm (-> 1 0 2) "DICTUMINREF" "NULLSWAPIFNOT2"; @pure -fun sDictGetFirstAsRef(self: cell?, keyLen: int): (slice?, cell?, bool) +fun sDictGetFirstAsRef(self: dict, keyLen: int): (slice?, dict, bool) asm (-> 1 0 2) "DICTMINREF" "NULLSWAPIFNOT2"; @pure -fun iDictGetLast(self: cell?, keyLen: int): (int?, slice?, bool) +fun iDictGetLast(self: dict, keyLen: int): (int?, slice?, bool) asm (-> 1 0 2) "DICTIMAX" "NULLSWAPIFNOT2"; @pure -fun uDictGetLast(self: cell?, keyLen: int): (int?, slice?, bool) +fun uDictGetLast(self: dict, keyLen: int): (int?, slice?, bool) asm (-> 1 0 2) "DICTUMAX" "NULLSWAPIFNOT2"; @pure -fun sDictGetLast(self: cell?, keyLen: int): (slice?, slice?, bool) +fun sDictGetLast(self: dict, keyLen: int): (slice?, slice?, bool) asm (-> 1 0 2) "DICTMAX" "NULLSWAPIFNOT2"; @pure -fun iDictGetLastAsRef(self: cell?, keyLen: int): (int?, cell?, bool) +fun iDictGetLastAsRef(self: dict, keyLen: int): (int?, dict, bool) asm (-> 1 0 2) "DICTIMAXREF" "NULLSWAPIFNOT2"; @pure -fun uDictGetLastAsRef(self: cell?, keyLen: int): (int?, cell?, bool) +fun uDictGetLastAsRef(self: dict, keyLen: int): (int?, dict, bool) asm (-> 1 0 2) "DICTUMAXREF" "NULLSWAPIFNOT2"; @pure -fun sDictGetLastAsRef(self: cell?, keyLen: int): (slice?, cell?, bool) +fun sDictGetLastAsRef(self: dict, keyLen: int): (slice?, dict, bool) asm (-> 1 0 2) "DICTMAXREF" "NULLSWAPIFNOT2"; @pure -fun iDictGetNext(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) +fun iDictGetNext(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXT" "NULLSWAPIFNOT2"; @pure -fun uDictGetNext(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) +fun uDictGetNext(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXT" "NULLSWAPIFNOT2"; @pure -fun iDictGetNextOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) +fun iDictGetNextOrEqual(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXTEQ" "NULLSWAPIFNOT2"; @pure -fun uDictGetNextOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) +fun uDictGetNextOrEqual(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXTEQ" "NULLSWAPIFNOT2"; @pure -fun iDictGetPrev(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) +fun iDictGetPrev(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETPREV" "NULLSWAPIFNOT2"; @pure -fun uDictGetPrev(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) +fun uDictGetPrev(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETPREV" "NULLSWAPIFNOT2"; @pure -fun iDictGetPrevOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) +fun iDictGetPrevOrEqual(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETPREVEQ" "NULLSWAPIFNOT2"; @pure -fun uDictGetPrevOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) +fun uDictGetPrevOrEqual(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETPREVEQ" "NULLSWAPIFNOT2"; @@ -300,13 +304,13 @@ fun uDictGetPrevOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bo */ @pure -fun prefixDictGet(self: cell?, keyLen: int, key: slice): (slice, slice?, slice?, bool) +fun prefixDictGet(self: dict, keyLen: int, key: slice): (slice, slice?, slice?, bool) asm(key self keyLen) "PFXDICTGETQ" "NULLSWAPIFNOT2"; @pure -fun prefixDictSet(mutate self: cell?, keyLen: int, key: slice, value: slice): bool +fun prefixDictSet(mutate self: dict, keyLen: int, key: slice, value: slice): bool asm(value key self keyLen) "PFXDICTSET"; @pure -fun prefixDictDelete(mutate self: cell?, keyLen: int, key: slice): bool +fun prefixDictDelete(mutate self: dict, keyLen: int, key: slice): bool asm(key self keyLen) "PFXDICTDEL"; diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index e145306eaa..d5519b0ef9 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -1,13 +1,16 @@ +type MInt = int; +type MSlice = slice; + const int1 = 1; const int2 = 2; const int101: int = 101; -const int111: int = 111; +const int111: MInt = 111; const int1r = int1; const str1 = "const1"; -const str2 = stringHexToSlice("aabbcc"); +const str2: MSlice = stringHexToSlice("aabbcc"); const str2r: slice = str2; @@ -17,7 +20,7 @@ const str2int = 0xAABBCC; const nibbles: int = 4; const strange_zero = (!10 as int); -const strange_minus_1: int = (!0 as int); +const strange_minus_1: MInt = (!0 as int); const true1 = true; const true2 = !!true; @@ -38,7 +41,7 @@ fun sget2r(): slice { return str2r; } const int240: int = ((int1+int2)*10)<<3; -fun iget240(): int { return int240; } +fun iget240(): MInt { return int240; } @pure fun newc(): builder @@ -47,7 +50,7 @@ asm "NEWC"; fun endcs(b: builder): slice asm "ENDC" "CTOS"; @pure -fun sdeq(s1: slice, s2: slice): int +fun sdeq(s1: slice, s2: slice): MInt asm "SDEQ"; @pure fun stslicer(b: builder, s: slice): builder diff --git a/tolk-tester/tests/generics-1.tolk b/tolk-tester/tests/generics-1.tolk index ca31092752..70cd2371c9 100644 --- a/tolk-tester/tests/generics-1.tolk +++ b/tolk-tester/tests/generics-1.tolk @@ -1,3 +1,7 @@ +type MInt = int; +type Tensor2Int = (int, int); +type Tensor2IntN = (int, int)?; + fun eq1(value: X): X { return value; } fun eq2(value: X) { return value; } fun eq3(value: X): X { var cp: [X] = [eq1(value)]; var ((([v: X]))) = cp; return v; } @@ -26,6 +30,7 @@ fun test102(): (int, int, int, [int, int]) { var ab_tup = [0, [1, 2]]; ab_tup.0 = getTwo(); ab_tup.1.1 = getTwo(); + var ab_tens_al: Tensor2Int = (getTwo(), getTwo()); return (eq1(a), eq2(b), takeInt(getTwo()), [getTwo(), ab_tens.1.1]); } @@ -88,11 +93,12 @@ fun test106() { fun callTupleFirst(t: X): Y { return t.tupleFirst(); } fun callTuplePush(mutate self: T, v1: V, v2: V): self { self.tuplePush(v1); tuplePush(mutate self, v2); return self; } fun getTupleLastInt(t: tuple) { return t.tupleLast(); } -fun getTupleSize(t: tuple) { return t.tupleSize(); } +fun getTupleSize(t: MTuple) { return t.tupleSize(); } fun callAnyFn(f: (TObj) -> TResult, arg: TObj) { return f(arg); } fun callAnyFn2(f: TCallback, arg: tuple) { return f(arg); } -global t107: tuple; +global t107: MTuple; +type MTuple = tuple; @method_id(107) fun test107() { @@ -105,7 +111,7 @@ fun test107() { callAnyFn2(getTupleSize, t107), first, callTupleFirst(t107) as int, - callAnyFn(getTupleLastInt, t107), + callAnyFn(getTupleLastInt, t107 as tuple), callAnyFn2(getTupleLastInt, t107) ); } @@ -131,7 +137,41 @@ fun test108() { return g108; } -fun main(x: int): (int, [[int, int]]) { +@method_id(109) +fun test109(initialX: int32) { + var x: MInt = initialX; + __expect_type(eq1(x), "MInt"); + __expect_type(eq1(x), "int"); + __expect_type(eq1(x), "int8?"); + return (eq1(x), eq1(null), eq1(null)); +} + +@method_id(110) +fun test110() { + var ab_tens_al: Tensor2Int = (getTwo(), getTwo()); + ab_tens_al.0 = getTwo(); + var ab_tens_al_n: Tensor2Int? = (5, 6); + ab_tens_al_n.1 = getTwo(); + return (ab_tens_al, ab_tens_al_n); +} + +type Tup2Int = [int, int]; + +fun isSomeNullableNull(v: T?) { return v == null; } + +type MIntN = int?; +type Tup2IntN = Tup2Int?; + +@method_id(111) +fun test111() { + var i1: MIntN = 3; + var i2: Tup2IntN = [1, 2]; + return (isSomeNullableNull(i1), isSomeNullableNull(i2), isSomeNullableNull(i1), isSomeNullableNull(i1)); +} + +fun main(x: int): (int, [Tup2Int]) { + __expect_type(test110, "() -> (Tensor2Int, Tensor2Int)"); + try { if(x) { throw (1, x); } } catch (excNo, arg) { return (arg as int, [[eq2(arg as int), getTwo()]]); } return (0, [[x, 1]]); @@ -147,11 +187,15 @@ fun main(x: int): (int, [[int, int]]) { @testcase | 106 | | [ 106 2 6 6 ] @testcase | 107 | | 6 6 1 1 6 6 @testcase | 108 | | 45 +@testcase | 109 | 5 | 5 (null) (null) +@testcase | 110 | | 2 2 5 2 +@testcase | 111 | | 0 0 0 0 @fif_codegen DECLPROC eq1 @fif_codegen DECLPROC eq1 @fif_codegen DECLPROC eq1<(int,int)> @fif_codegen DECLPROC eq1<[int,int]> +@fif_codegen DECLPROC eq1 @fif_codegen DECLPROC getTwo @fif_codegen_avoid DECLPROC eq1 diff --git a/tolk-tester/tests/indexed-access.tolk b/tolk-tester/tests/indexed-access.tolk index e2bd3dd93d..4b3a0a4b26 100644 --- a/tolk-tester/tests/indexed-access.tolk +++ b/tolk-tester/tests/indexed-access.tolk @@ -1,5 +1,8 @@ +type Tup2Int = [int, MInt]; +type Tensor2Int = (int, MInt); +type MInt = int; -fun increment(mutate self: int) { +fun increment(mutate self: MInt) { self += 1; } @@ -8,7 +11,7 @@ fun increment2(mutate a: int, mutate b: int) { b += 1; } -fun assign1020(mutate a: int, mutate b: int) { +fun assign1020(mutate a: int, mutate b: MInt) { a = 10; b = 20; } @@ -51,7 +54,7 @@ fun test101() { return t; } -global t102: (int, (int, int), [int, int, [int, int]], int); +global t102: (int, (int, MInt), [int, int, Tup2Int], MInt); @method_id(102) fun test102() { @@ -137,7 +140,7 @@ fun test110(f: int, s: int) { return (x, x.1, x.1.plus(x.1 / 20), x.1, x.1 = x.1 * 2, x.1, x.1 += 1, x.1, x); } -global xx: (int, int); +global xx: Tensor2Int; @method_id(111) fun test111(x: (int, int)) { @@ -154,7 +157,7 @@ fun test112(f: int, s: int) { } @pure -fun getConstTuple() { +fun getConstTuple(): Tup2Int { return [1,2]; } @@ -197,7 +200,7 @@ fun test117() { var t = createEmptyTuple(); t.tuplePush(1); try { - return (t.0 as tuple).0 as int; + return (t.0 as tuple).0 as MInt; } catch(excNo) { return excNo; } diff --git a/tolk-tester/tests/inference-tests.tolk b/tolk-tester/tests/inference-tests.tolk index 4368483170..affe42ee1e 100644 --- a/tolk-tester/tests/inference-tests.tolk +++ b/tolk-tester/tests/inference-tests.tolk @@ -75,7 +75,7 @@ fun test5(x: int) { __expect_type([x, x >= 1], "[int, bool]"); __expect_type([x, x >= 1, null as slice?], "[int, bool, slice?]"); __expect_type((x, [x], [[x], x]), "(int, [int], [[int], int])"); - __expect_type(getMyOriginalBalanceWithExtraCurrencies(), "[int, cell?]"); + __expect_type(getMyOriginalBalanceWithExtraCurrencies(), "[int, dict]"); } fun test6() { diff --git a/tolk-tester/tests/invalid-declaration/err-1483.tolk b/tolk-tester/tests/invalid-declaration/err-1483.tolk new file mode 100644 index 0000000000..01812fedcd --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1483.tolk @@ -0,0 +1,8 @@ +type A = B; +type B = (int, A); + +/** +@compilation_should_fail +@stderr type `A` circularly references itself +@stderr type A = B + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4304.tolk b/tolk-tester/tests/invalid-semantics/err-4304.tolk new file mode 100644 index 0000000000..9636934140 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4304.tolk @@ -0,0 +1,10 @@ +type MInt = int; + +fun main() { + var value = MInt; +} + +/** +@compilation_should_fail +@stderr type `MInt` can not be used as a value +*/ diff --git a/tolk-tester/tests/invalid-syntax/err-3482.tolk b/tolk-tester/tests/invalid-syntax/err-3482.tolk new file mode 100644 index 0000000000..cfe2d38846 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3482.tolk @@ -0,0 +1,6 @@ +type MInt int; + +/** +@compilation_should_fail +@stderr expected `=`, got `int` +*/ diff --git a/tolk-tester/tests/invalid-typing/err-6188.tolk b/tolk-tester/tests/invalid-typing/err-6188.tolk index 8c1df4a26d..7a7fb15e3a 100644 --- a/tolk-tester/tests/invalid-typing/err-6188.tolk +++ b/tolk-tester/tests/invalid-typing/err-6188.tolk @@ -1,8 +1,11 @@ +type MInt = int; + fun failMathOnBoolean(c: cell) { - return (null == c) * 10; + var ten: MInt = 10; + return (null == c) * ten; } /** @compilation_should_fail -@stderr can not apply operator `*` to `bool` and `int` +@stderr can not apply operator `*` to `bool` and `MInt` */ diff --git a/tolk-tester/tests/invalid-typing/err-6337.tolk b/tolk-tester/tests/invalid-typing/err-6337.tolk index 1dca7822c8..5c9cbdc4c8 100644 --- a/tolk-tester/tests/invalid-typing/err-6337.tolk +++ b/tolk-tester/tests/invalid-typing/err-6337.tolk @@ -9,5 +9,5 @@ fun testCantCallDictMethodsOnNullable(c: cell) { /** @compilation_should_fail -@stderr can not call method for `cell` with object of type `cell?` +@stderr can not call method for `cell` with object of type `dict` */ diff --git a/tolk-tester/tests/invalid-typing/err-6450.tolk b/tolk-tester/tests/invalid-typing/err-6450.tolk index 052596e4ca..b14f3b6d51 100644 --- a/tolk-tester/tests/invalid-typing/err-6450.tolk +++ b/tolk-tester/tests/invalid-typing/err-6450.tolk @@ -1,9 +1,11 @@ +type MInt = int; + fun main() { - var tri: (int, int) = (10, false); + var tri: (int, MInt) = (10, false); return; } /** @compilation_should_fail -@stderr can not assign `(int, bool)` to variable of type `(int, int)` +@stderr can not assign `(int, bool)` to variable of type `(int, MInt)` */ diff --git a/tolk-tester/tests/invalid-typing/err-6553.tolk b/tolk-tester/tests/invalid-typing/err-6553.tolk index bf5a1165c2..ba6807713a 100644 --- a/tolk-tester/tests/invalid-typing/err-6553.tolk +++ b/tolk-tester/tests/invalid-typing/err-6553.tolk @@ -1,4 +1,6 @@ -fun getNullableInt(): int? { return 5; } +type MInt = int; + +fun getNullableInt(): MInt? { return 5; } fun testAssertThrowIsConditional() { var (x, y) = (getNullableInt(), getNullableInt()); @@ -8,5 +10,5 @@ fun testAssertThrowIsConditional() { /** @compilation_should_fail -@stderr can not apply operator `+` to `int` and `int?` +@stderr can not apply operator `+` to `MInt` and `MInt?` */ diff --git a/tolk-tester/tests/invalid-typing/err-6580.tolk b/tolk-tester/tests/invalid-typing/err-6580.tolk index d2a815eeeb..e2c088b046 100644 --- a/tolk-tester/tests/invalid-typing/err-6580.tolk +++ b/tolk-tester/tests/invalid-typing/err-6580.tolk @@ -1,4 +1,6 @@ -fun getNullableInt(): int? { return 5; } +type MIntN = int?; + +fun getNullableInt(): MIntN { return 5; } fun testNeverTypeOccurs() { var x: int? = getNullableInt(); diff --git a/tolk-tester/tests/invalid-typing/err-6734.tolk b/tolk-tester/tests/invalid-typing/err-6734.tolk index 3a5b1fe289..0f805cbe50 100644 --- a/tolk-tester/tests/invalid-typing/err-6734.tolk +++ b/tolk-tester/tests/invalid-typing/err-6734.tolk @@ -1,10 +1,12 @@ +type MTensor = (int, int); + fun failAssignNullToTensor() { - var ab = (1, 2); + var ab: MTensor = (1, 2); ab = null; return ab; } /** @compilation_should_fail -@stderr can not assign `null` to variable of type `(int, int)` +@stderr can not assign `null` to variable of type `MTensor` */ diff --git a/tolk-tester/tests/invalid-typing/err-6737.tolk b/tolk-tester/tests/invalid-typing/err-6737.tolk index d112b36068..cc4bc2f59a 100644 --- a/tolk-tester/tests/invalid-typing/err-6737.tolk +++ b/tolk-tester/tests/invalid-typing/err-6737.tolk @@ -1,5 +1,7 @@ +type MInt32 = int32; + fun testAssignBetweenDifferentIntN(op: int32, qid: uint64) { - op = qid as int32; // ok + op = qid as MInt32; // ok op = qid; } diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index 6b53306b68..9e3a269b5c 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -1,3 +1,7 @@ +type TensorOf2IntNull = (int?, int?); +type MInt = int; +type MIntN = int?; + fun getNullableInt(): int? { return 5; } fun sumOfNullableTensorComponents(t: (int, int)?): int { @@ -385,7 +389,9 @@ fun getNormalNullableTensorWidth1(vLess100: int?): ([int?], ())? { return ([vLess100], ()); // such a nullable tensor can store NULL in the same slot } -fun getTrickyNullableTensorWidth1(vLess100: int?): (int?, ())? { +type TrickyActually2SlotTensor = (MIntN, ())?; + +fun getTrickyNullableTensorWidth1(vLess100: int?): TrickyActually2SlotTensor { if (vLess100 != null && vLess100 >= 100) { return null; } @@ -429,6 +435,26 @@ fun test140(x: int8, y: int16) { return (t1, t2); } +@method_id(141) +fun test141() { + var t: TensorOf2IntNull = (1, 2); + if (t.0 != null) { + t.0 = null; + } + t.1 != null ? t.1 = 10 : t.1 = null; + return t; +} + +type T142 = (int, MInt)?; + +@method_id(142) +fun test142() { + var t = (1, 2) as T142; + __expect_type(t!, "(int, MInt)"); + var sum1 = sumOfNullableTensorComponents(t); + var sum2 = sumOfTensor(t!); + return (sum1, sum2, t!); +} fun main(){} @@ -482,6 +508,8 @@ fun main(){} @testcase | 136 | 9 | 9 0 @testcase | 136 | null | (null) -1 @testcase | 140 | 8 9 | 8 9 -1 (null) (null) 0 +@testcase | 141 | | (null) 10 +@testcase | 142 | | 3 3 1 2 @fif_codegen """ diff --git a/tolk-tester/tests/nullable-types.tolk b/tolk-tester/tests/nullable-types.tolk index 24aa7f8ae1..f5ee8b550f 100644 --- a/tolk-tester/tests/nullable-types.tolk +++ b/tolk-tester/tests/nullable-types.tolk @@ -1,5 +1,6 @@ +type MIntN = int?; -fun getNullable4(): int? { return 4; } +fun getNullable4(): MIntN { return 4; } fun getNullableIntNull(): int? asm "PUSHNULL"; fun eqInt(x: int) { return x; } @@ -36,7 +37,7 @@ fun test103(x: int?): (bool, bool, int) { } @method_id(104) -fun test104(x: int?) { +fun test104(x: MIntN) { var x2 = eq(x = 10); var ab = (x2, getNullableIntNull()); return (unwrap(ab.0) + (ab.1 == null ? -100 : ab.1!), ab.1); @@ -44,7 +45,7 @@ fun test104(x: int?) { @method_id(105) fun test105() { - var xy: (int?, int?) = (5, null); + var xy: (int?, MIntN) = (5, null); var ab = [1 ? [xy.0, xy.1] : null]; ab.0!.0 = intOr0(ab.0!.0); ab.0!.1 = intOr0(ab.0!.1); @@ -52,7 +53,7 @@ fun test105() { } global gTup106: tuple?; -global gInt106: int?; +global gInt106: MIntN; @method_id(106) fun test106() { @@ -91,7 +92,11 @@ fun test109() { return ([a, b] = [3, 4], a, b); } -fun main(x: int?, y: int?) { +fun main(x: int?, y: MIntN) { + __expect_type(x! == y!, "bool"); + __expect_type(5 != y!, "bool"); + __expect_type(y == null, "bool"); + __expect_type(true == (true as bool?)!, "bool"); } /** diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 4d71bb63a9..8132180267 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -9,10 +9,12 @@ fun increment(mutate self: int) { self += 1; } fun assignToInt(mutate self: int, value: int) { self = value; } fun assignToNullableInt(mutate self: int?, value: int) { self = value; } fun sameTensor(t: (int, int)) { return t; } -fun sameTensor2(t: (int?, (slice, slice, slice, builder)?)) { return t; } +fun sameTensor2(t: (int?, Pair4N)) { return t; } fun eq(v: T) { return v; } fun getTwo(): X { return 2 as X; } +type Pair4N = (slice, slice, slice, builder)?; + fun test1(): int { var x = getNullableInt(); var y = getNullableInt(); diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk new file mode 100644 index 0000000000..54899a0fc2 --- /dev/null +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -0,0 +1,189 @@ +type MIntN = MInt?; +type MInt = int; +type MInt_v2 = int; +type MVoid = void; +type Pair2_v1 = (int, int); +type Pair2_v2 = (MInt, MInt); +type MBool = bool; +type Tuple2Int = [int, int]; + + +fun test1(x: MInt): MVoid { + var y = x; + var z: MInt = 2; + __expect_type(x, "MInt"); + __expect_type(y, "MInt"); + __expect_type(z, "MInt"); + + __expect_type(x + y, "int"); + __expect_type((x + y) as MInt, "MInt"); + __expect_type(x!, "MInt"); + __expect_type(~x, "int"); + __expect_type(x as int, "int"); + __expect_type(x as int8, "int8"); + __expect_type(random() ? x : y, "MInt"); + __expect_type((x, 1, y), "(MInt, int, MInt)"); + + __expect_type(random() ? (1, 2) : (1, 2) as Pair2_v1, "(int, int)"); + __expect_type(random() ? (1, 2) : (1, 2) as Pair2_v2, "(int, int)"); + __expect_type(random() ? (1, 2) as Pair2_v1 : (1, 2), "Pair2_v1"); + __expect_type(random() ? (1, 2) as Pair2_v1 : (1, 2) as Pair2_v2, "Pair2_v1"); + __expect_type(random() ? (1, 2) as Pair2_v2 : (1, 2) as Pair2_v2, "Pair2_v2"); + + __expect_type(!x, "bool"); + + __expect_type(x as int?, "int?"); + __expect_type(x as MInt?, "MInt?"); + __expect_type(x as MIntN, "MIntN"); + + if (x) { return; } +} + +fun test2() { + __expect_type(test1, "(MInt) -> MVoid"); + __expect_type(test1(1), "MVoid"); +} + +fun test3(x: MIntN, y: MInt?) { + __expect_type(x, "MIntN"); + __expect_type(x!, "MInt"); + __expect_type(y!, "MInt"); + if (x != null) { + __expect_type(x, "MInt"); + } + __expect_type(x, "MInt?"); // merge null + MInt = MInt?, not MIntN as original + var (z1, z2) = (x, x!, ); + __expect_type(z1, "MInt?"); + __expect_type(z2, "MInt"); +} + +@method_id(104) +fun test4(x: MIntN, y: MIntN) { + if (x != null && y != null) { + __expect_type(x, "MInt"); + __expect_type(x + y, "int"); + return x + y; + } + __expect_type(x!, "MInt"); + __expect_type(random() ? x : y, "MInt?"); + __expect_type(random() ? x : 0, "MInt?"); + __expect_type(random() ? 0 : x, "MInt?"); + __expect_type(random() ? 0 : x!, "int"); + return y!; +} + +fun takeTensor_v1(v: Pair2_v1) { return v.0 + v.1; } +fun takeTensor_v2(v: Pair2_v2) { return v.0 + v.1; } +fun takeTensor_v3(v: (int, int)) { return v.0 + v.1; } + +fun test5() { + var x: Pair2_v1 = (1, 2); + var y = (3, 4) as Pair2_v2; + var z = (5, 6); + + __expect_type(x, "Pair2_v1"); + __expect_type(y, "Pair2_v2"); + + takeTensor_v1(x); takeTensor_v2(x); takeTensor_v3(x); + takeTensor_v1(y); takeTensor_v2(y); takeTensor_v3(y); + takeTensor_v1(z); takeTensor_v2(z); takeTensor_v3(z); + + var t = (y, x); + return t.0.1 + t.1.0; +} + +fun test6() { + var (x1: MInt?, x2: MIntN) = (5, 5); // smart cast + __expect_type(x1, "MInt"); + __expect_type(x2, "MInt"); + var (y1: MInt?, y2: MIntN) = (null, null); + __expect_type(y1, "null"); + __expect_type(y2, "null"); + var z: Pair2_v2? = (5, 5); + __expect_type(z, "Pair2_v2"); + __expect_type(z.0, "MInt"); +} + +fun someFn1(v: MInt): MIntN { return v; } + +fun test7() { + var f1: (int) -> int? = someFn1; + var f2: (int) -> MInt? = someFn1; + var f3: (MInt_v2) -> MInt_v2? = someFn1; + + f1 = f2; f1 = f3; + f2 = f1; f2 = f3; + f3 = f1; f3 = f2; +} + +fun test8() { + 0 as MInt; + 0 as MInt?; + 0 as MIntN; + + (1, 2) as Pair2_v2; + (1, 2) as Pair2_v2?; + (((1, 2) as Pair2_v2?) as (int, int)?) as Pair2_v1?; + + someFn1 as (int) -> int?; + someFn1 as (int) -> MInt?; + someFn1 as (MInt_v2) -> MInt_v2?; +} + +fun test9(b: MBool): MBool { + if (!b) { + __expect_type(b, "MBool"); + return !b; + } + return !!(5 as MInt); +} + +@method_id(110) +fun test10() { + var x1: Pair2_v1 = (5, 6); + var (a1, b1) = x1; + __expect_type(a1, "int"); + var x2: Pair2_v2? = x1; + var (a2, b2) = x2; + __expect_type(a2, "MInt"); + var x3: Tuple2Int = [9, 10]; + var [a3, b3] = x3; + return a1 + a2 + a3 + b1 + b2 + b3; +} + +fun analyzeTensor1(a: (T1, T2)): (T1, T2) { return a; } +fun analyzeTensor2(a: (T1, T2)?): (T1, T2)? { return a; } + +fun test11() { + var (x1: (int8, int16), x2: Pair2_v1, x3: Pair2_v2) = ((1,2), (3,4), (5,6)); + __expect_type(analyzeTensor1(x1), "(int8, int16)"); + __expect_type(analyzeTensor1(x2), "(int, int)"); + __expect_type(analyzeTensor1(x3), "(MInt, MInt)"); + __expect_type(analyzeTensor2(x1), "(int8, int16)?"); + __expect_type(analyzeTensor2(x2), "(int, int)?"); + __expect_type(analyzeTensor2(x3), "(MInt, MInt)?"); +} + + +fun main(x: MInt, y: MInt?) { + return y == null ? x : x + y; +} + +/** +@testcase | 0 | 3 4 | 7 +@testcase | 104 | 1 2 | 3 +@testcase | 110 | | 41 + +@fif_codegen +""" + test9 PROC:<{ + // b + DUP // b b + IFNOTJMP:<{ // b + NOT // '2 + }> // b + DROP // + TRUE // '5 + }> +""" + */ diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index 96b4d134de..76fdab21f0 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -1,3 +1,8 @@ +type MInt = int; +type MIntN = int?; +type MBuilder = builder; +type CallbackIntToInt = int -> int; + fun getBeginCell() { return beginCell; } @@ -9,7 +14,7 @@ fun getBeginParse() { @method_id(101) fun testVarApply1() { var (_, f_end_cell) = (0, endCell); - var b: builder = (getBeginCell())().storeInt(1, 32); + var b: MBuilder = (getBeginCell())().storeInt(1, 32); b.storeInt(2, 32); var s = (getBeginParse())(f_end_cell(b)); return (s.loadInt(32), s.loadInt(32)); @@ -55,7 +60,7 @@ fun demo_handler(op: int, query_id: int, a: int, b: int): int { return 0; // result not used, we test that func is nevertheless called } if (op == 0xF4) { - val func = query_id % 2 == 0 ? sum : mul; + val func: (MInt, int) -> MInt = query_id % 2 == 0 ? sum : mul; val result = func(a, b); return result; } @@ -92,7 +97,7 @@ fun always_throw2(x: int) { throw 239 + x; } -global global_f: int -> void; +global global_f: MInt -> void; @method_id(104) fun testGlobalVarApply() { @@ -124,7 +129,7 @@ fun testVarApply3() { t.tuplePush(1); t.tuplePush([2]); var getIntAt = t.tupleAt; - var getTupleFirstInt = createEmptyTuple().tupleFirst; + var getTupleFirstInt = createEmptyTuple().tupleFirst; var getTupleLastTuple = getTupleLastGetter(); return (getIntAt(t, 0), getTupleFirstInt(t), getTupleLastTuple(t), getTupleLastGetter()(t)); } @@ -143,9 +148,9 @@ fun myBeginCell(): builder? asm "NEWC"; @method_id(108) fun testCallingNotNull() { - var n4: () -> int? = getNullable4; + var n4: () -> MIntN = getNullable4; var creator: (() -> builder?)? = myBeginCell; - var end2: [int, (builder -> cell)?] = [0, endCell]; + var end2: [int, (MBuilder -> cell)?] = [0, endCell]; var c: cell = end2.1!((creator!()!)!.storeInt(getNullable4()!, 32)); return c.beginParse().loadInt(32); } @@ -186,8 +191,8 @@ fun testApplyNativePlus(x: int, y: int, z: int): bool { global op: (int, int) -> int; -fun check_assoc_2(a: int, b: int, c: int): bool { - return op(op(a, b), c) == op(a, op(b, c)); +fun check_assoc_2(a: int, b: MInt, c: MIntN): bool { + return op(op(a, b), c!) == op(a, op(b, c!)); } @method_id(112) @@ -197,6 +202,15 @@ fun testApplyGlobalVar(x: int, y: int, z: int): bool? { return check_assoc_2(x, y, z); } +fun justAdd2(x: MInt): int64 { return x + 2; } + +@method_id(113) +fun testCallbackAlias(secondNull: bool) { + var adder1: CallbackIntToInt = justAdd2; + var adder2: CallbackIntToInt? = secondNull ? null : justAdd2; + return (adder1(10), adder2 == null ? -1 : adder2(10)); +} + fun main() {} /** @@ -217,4 +231,6 @@ fun main() {} @testcase | 112 | 2 3 9 | -1 @testcase | 112 | 11 22 44 | -1 @testcase | 112 | -1 -10 -20 | -1 -*/ +@testcase | 113 | 0 | 12 12 +@testcase | 113 | -1 | 12 -1 + */ diff --git a/tolk/abscode.cpp b/tolk/abscode.cpp index fc16098464..eb2e9a0a66 100644 --- a/tolk/abscode.cpp +++ b/tolk/abscode.cpp @@ -414,6 +414,8 @@ std::vector CodeBlob::create_var(TypePtr var_type, SrcLocation loc, s std::string null_flag_name = name.empty() ? name : name + ".NNFlag"; ir_idx = create_var(t_nullable->inner, loc, std::move(name)); ir_idx.emplace_back(create_var(TypeDataBool::create(), loc, std::move(null_flag_name))[0]); + } else if (const TypeDataAlias* t_alias = var_type->try_as()) { + ir_idx = create_var(t_alias->underlying_type, loc, std::move(name)); } else if (var_type != TypeDataVoid::create() && var_type != TypeDataNever::create()) { #ifdef TOLK_DEBUG tolk_assert(stack_w == 1); diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 94ef848c17..729204579f 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -227,6 +227,21 @@ static AnyV parse_constant_declaration(Lexer& lex, const std::vector(loc, v_ident, declared_type, init_value); } +static AnyV parse_type_alias_declaration(Lexer& lex, const std::vector>& annotations) { + if (!annotations.empty()) { + lex.error("@annotations are not applicable to type alias declaration"); + } + SrcLocation loc = lex.cur_location(); + lex.expect(tok_type, "`type`"); + lex.check(tok_identifier, "type name"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + lex.expect(tok_assign, "`=`"); + TypePtr underlying_type = parse_type_from_tokens(lex); + lex.expect(tok_semicolon, "`;`"); + return createV(loc, v_ident, underlying_type); +} + // "parameters" are at function declaration: `fun f(param1: int, mutate param2: slice)` static V parse_parameter_list(Lexer& lex) { SrcLocation loc = lex.cur_location(); @@ -1180,6 +1195,10 @@ AnyV parse_src_file_to_ast(const SrcFile* file) { toplevel_declarations.push_back(parse_constant_declaration(lex, annotations)); annotations.clear(); break; + case tok_type: + toplevel_declarations.push_back(parse_type_alias_declaration(lex, annotations)); + annotations.clear(); + break; case tok_fun: case tok_get: toplevel_declarations.push_back(parse_function_declaration(lex, annotations)); diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index 07f03745ca..77390c1caa 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -81,6 +81,7 @@ class ASTStringifier final : public ASTVisitor { {ast_function_declaration, "ast_function_declaration"}, {ast_global_var_declaration, "ast_global_var_declaration"}, {ast_constant_declaration, "ast_constant_declaration"}, + {ast_type_alias_declaration, "ast_type_alias_declaration"}, {ast_tolk_required_version, "ast_tolk_required_version"}, {ast_import_directive, "ast_import_directive"}, {ast_tolk_file, "ast_tolk_file"}, @@ -157,6 +158,8 @@ class ASTStringifier final : public ASTVisitor { return static_cast(v->as()->get_identifier()->name); case ast_constant_declaration: return static_cast(v->as()->get_identifier()->name); + case ast_type_alias_declaration: + return "type " + static_cast(v->as()->get_identifier()->name); case ast_assign: return "="; case ast_set_assign: @@ -291,6 +294,7 @@ class ASTStringifier final : public ASTVisitor { case ast_function_declaration: return handle_vertex(v->as()); case ast_global_var_declaration: return handle_vertex(v->as()); case ast_constant_declaration: return handle_vertex(v->as()); + case ast_type_alias_declaration: return handle_vertex(v->as()); case ast_tolk_required_version: return handle_vertex(v->as()); case ast_import_directive: return handle_vertex(v->as()); case ast_tolk_file: return handle_vertex(v->as()); diff --git a/tolk/ast.cpp b/tolk/ast.cpp index ac93d21c30..f383ebcc71 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -154,6 +154,14 @@ void Vertex::assign_resolved_type(TypePtr declared_typ this->declared_type = declared_type; } +void Vertex::assign_alias_ref(AliasDefPtr alias_ref) { + this->alias_ref = alias_ref; +} + +void Vertex::assign_resolved_type(TypePtr underlying_type) { + this->underlying_type = underlying_type; +} + void Vertex::assign_resolved_type(TypePtr substituted_type) { this->substituted_type = substituted_type; } diff --git a/tolk/ast.h b/tolk/ast.h index 385e1ff895..b62aa4c532 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -113,6 +113,7 @@ enum ASTNodeType { ast_function_declaration, ast_global_var_declaration, ast_constant_declaration, + ast_type_alias_declaration, ast_tolk_required_version, ast_import_directive, ast_tolk_file, @@ -1036,6 +1037,25 @@ struct Vertex final : ASTOtherVararg { , declared_type(declared_type) {} }; +template<> +// ast_type_alias_declaration is declaring a structural type alias (fully interchangeable with original type) +// example: `type UserId = int;` +// see TypeDataAlias in type-system.h +struct Vertex final : ASTOtherVararg { + AliasDefPtr alias_ref = nullptr; // filled after register, contains TypeDataAlias + TypePtr underlying_type; // at the right of `=` + + auto get_identifier() const { return children.at(0)->as(); } + + Vertex* mutate() const { return const_cast(this); } + void assign_alias_ref(AliasDefPtr alias_ref); + void assign_resolved_type(TypePtr underlying_type); + + Vertex(SrcLocation loc, V name_identifier, TypePtr underlying_type) + : ASTOtherVararg(ast_type_alias_declaration, loc, {name_identifier}) + , underlying_type(underlying_type) {} +}; + template<> // ast_tolk_required_version is a preamble fixating compiler's version at the top of the file // example: `tolk 0.6` diff --git a/tolk/fwd-declarations.h b/tolk/fwd-declarations.h index 8d3b24a8f2..7255a85884 100644 --- a/tolk/fwd-declarations.h +++ b/tolk/fwd-declarations.h @@ -31,11 +31,13 @@ struct LocalVarData; struct FunctionData; struct GlobalVarData; struct GlobalConstData; +struct AliasDefData; using LocalVarPtr = const LocalVarData*; using FunctionPtr = const FunctionData*; using GlobalVarPtr = const GlobalVarData*; using GlobalConstPtr = const GlobalConstData*; +using AliasDefPtr = const AliasDefData*; class TypeData; using TypePtr = const TypeData*; diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index 9dae3f009c..b335683f33 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -83,7 +83,7 @@ void GenericSubstitutionsDeduceForCall::consider_next_condition(TypePtr param_ty provide_deducedT(asT->nameT, arg_type); } else if (const auto* p_nullable = param_type->try_as()) { // `arg: T?` called as `f(nullableInt)` => T is int - if (const auto* a_nullable = arg_type->try_as()) { + if (const auto* a_nullable = arg_type->unwrap_alias()->try_as()) { consider_next_condition(p_nullable->inner, a_nullable->inner); } // `arg: T?` called as `f(int)` => T is int @@ -92,21 +92,21 @@ void GenericSubstitutionsDeduceForCall::consider_next_condition(TypePtr param_ty } } else if (const auto* p_tensor = param_type->try_as()) { // `arg: (int, T)` called as `f((5, cs))` => T is slice - if (const auto* a_tensor = arg_type->try_as(); a_tensor && a_tensor->size() == p_tensor->size()) { + if (const auto* a_tensor = arg_type->unwrap_alias()->try_as(); a_tensor && a_tensor->size() == p_tensor->size()) { for (int i = 0; i < a_tensor->size(); ++i) { consider_next_condition(p_tensor->items[i], a_tensor->items[i]); } } } else if (const auto* p_tuple = param_type->try_as()) { // `arg: [int, T]` called as `f([5, cs])` => T is slice - if (const auto* a_tuple = arg_type->try_as(); a_tuple && a_tuple->size() == p_tuple->size()) { + if (const auto* a_tuple = arg_type->unwrap_alias()->try_as(); a_tuple && a_tuple->size() == p_tuple->size()) { for (int i = 0; i < a_tuple->size(); ++i) { consider_next_condition(p_tuple->items[i], a_tuple->items[i]); } } } else if (const auto* p_callable = param_type->try_as()) { // `arg: fun(TArg) -> TResult` called as `f(calcTupleLen)` => TArg is tuple, TResult is int - if (const auto* a_callable = arg_type->try_as(); a_callable && a_callable->params_size() == p_callable->params_size()) { + if (const auto* a_callable = arg_type->unwrap_alias()->try_as(); a_callable && a_callable->params_size() == p_callable->params_size()) { for (int i = 0; i < a_callable->params_size(); ++i) { consider_next_condition(p_callable->params_types[i], a_callable->params_types[i]); } diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index b26c9d2c4d..10647de946 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -150,7 +150,7 @@ class LValContext { LValContext local_lval; local_lval.enter_rval_inside_lval(); std::vector obj_ir_idx = pre_compile_expr(tensor_obj, code, nullptr, &local_lval); - const TypeDataTensor* t_tensor = tensor_obj->inferred_type->try_as(); + const TypeDataTensor* t_tensor = tensor_obj->inferred_type->unwrap_alias()->try_as(); tolk_assert(t_tensor); int stack_width = t_tensor->items[index_at]->get_width_on_stack(); int stack_offset = 0; @@ -384,7 +384,7 @@ static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyE std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); vars_modification_watcher.trigger_callbacks(left, loc); std::vector right = pre_compile_expr(rhs, code, nullptr); - const TypeDataTypedTuple* inferred_tuple = rhs->inferred_type->try_as(); + const TypeDataTypedTuple* inferred_tuple = rhs->inferred_type->unwrap_alias()->try_as(); std::vector types_list = inferred_tuple->items; std::vector rvect = code.create_tmp_var(TypeDataTensor::create(std::move(types_list)), rhs->loc, "(unpack-tuple)"); code.emplace_back(lhs->loc, Op::_UnTuple, rvect, std::move(right)); @@ -432,6 +432,13 @@ static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcL // Here rvect is a list of IR vars for inferred_type, probably patched due to target_type. GNU_ATTRIBUTE_NOINLINE static std::vector transition_expr_to_runtime_type_impl(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc) { +#ifdef TOLK_DEBUG + tolk_assert(static_cast(rvect.size()) == original_type->get_width_on_stack()); +#endif + + original_type = original_type->unwrap_alias(); + target_type = target_type->unwrap_alias(); + // pass `T` to `T` // could occur for passing tensor `(..., T, ...)` to `(..., T, ...)` while traversing tensor's components if (target_type == original_type) { @@ -635,6 +642,22 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as()) { + return transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, target_alias->underlying_type, loc); + } + // pass `UserId` (or some other alias) to `T` + if (const TypeDataAlias* orig_alias = original_type->try_as()) { + return transition_expr_to_runtime_type_impl(std::move(rvect), code, orig_alias->underlying_type, target_type, loc); + } + + // pass callable to callable + // their types aren't exactly equal, but they match (containing aliases, for example) + if (original_type->try_as() && target_type->try_as()) { + tolk_assert(rvect.size() == 1); + return rvect; + } + throw Fatal("unhandled transition_expr_to_runtime_type_impl() combination"); } @@ -819,7 +842,7 @@ static std::vector process_cast_as_operator(V v } static std::vector process_not_null_operator(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { - TypePtr child_target_type = v->get_expr()->inferred_type; + TypePtr child_target_type = v->get_expr()->inferred_type->unwrap_alias(); if (const auto* as_nullable = child_target_type->try_as()) { child_target_type = as_nullable->inner; } @@ -830,7 +853,7 @@ static std::vector process_not_null_operator(V static std::vector process_is_null_check(V v, CodeBlob& code, TypePtr target_type) { std::vector expr_ir_idx = pre_compile_expr(v->get_expr(), code, nullptr); std::vector isnull_ir_idx = code.create_tmp_var(TypeDataBool::create(), v->loc, "(is-null)"); - TypePtr expr_type = v->get_expr()->inferred_type; + TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); if (const TypeDataNullable* t_nullable = expr_type->try_as()) { if (!t_nullable->is_primitive_nullable()) { @@ -858,7 +881,7 @@ static std::vector process_dot_access(V v, CodeBlob& // it's NOT a method call `t.tupleSize()` (since such cases are handled by process_function_call) // it's `t.0`, `getUser().id`, and `t.tupleSize` (as a reference, not as a call) if (!v->is_target_fun_ref()) { - TypePtr obj_type = v->get_obj()->inferred_type; + TypePtr obj_type = v->get_obj()->inferred_type->unwrap_alias(); int index_at = std::get(v->target); // `tensorVar.0` if (const auto* t_tensor = obj_type->try_as()) { @@ -930,7 +953,7 @@ static std::vector process_function_call(V v, Code for (int i = 0; i < v->get_num_args(); ++i) { args.push_back(v->get_arg(i)->get_expr()); } - std::vector params_types = v->get_callee()->inferred_type->try_as()->params_types; + std::vector params_types = v->get_callee()->inferred_type->unwrap_alias()->try_as()->params_types; const TypeDataTensor* tensor_tt = TypeDataTensor::create(std::move(params_types))->try_as(); std::vector> vars_per_arg = pre_compile_tensor_inner(code, args, tensor_tt, nullptr); std::vector args_vars; @@ -1249,7 +1272,7 @@ static void process_do_while_statement(V v, CodeBlob& co until_cond = createV(cond->loc, "<", tok_lt, v_geq->get_lhs(), v_geq->get_rhs()); } else if (auto v_gt = cond->try_as(); v_gt && v_gt->tok == tok_gt) { until_cond = createV(cond->loc, "<=", tok_geq, v_gt->get_lhs(), v_gt->get_rhs()); - } else if (cond->inferred_type == TypeDataBool::create()) { + } else if (cond->inferred_type->unwrap_alias() == TypeDataBool::create()) { until_cond = createV(cond->loc, "!b", tok_logical_not, cond); } else { until_cond = createV(cond->loc, "!", tok_logical_not, cond); diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 4b5abbc8bc..927cfc4e19 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -118,6 +118,19 @@ static void handle_possible_compiler_internal_call(FunctionPtr cur_f, Vname == "__expect_type") { tolk_assert(v->get_num_args() == 2); TypePtr expected_type = parse_type_from_string(v->get_arg(1)->get_expr()->as()->str_val); + if (expected_type->has_unresolved_inside()) { // only aliases can be inside + expected_type = expected_type->replace_children_custom([](TypePtr child) -> TypePtr { + if (const auto* as_unresolved = child->try_as()) { + const Symbol* sym = lookup_global_symbol(as_unresolved->text); + tolk_assert(sym); + if (AliasDefPtr alias_ref = sym->try_as()) { + return TypeDataAlias::create(alias_ref->name, alias_ref->underlying_type); + } + tolk_assert(false); + } + return child; + }); + } TypePtr expr_type = v->get_arg(0)->inferred_type; if (expected_type != expr_type) { fire(cur_f, v->loc, "__expect_type failed: expected " + to_string(expected_type) + ", got " + to_string(expr_type)); @@ -125,14 +138,36 @@ static void handle_possible_compiler_internal_call(FunctionPtr cur_f, Vtry_as() || inferred_type == TypeDataCoins::create()) { + return true; + } + if (const TypeDataAlias* as_alias = inferred_type->try_as()) { + return expect_integer(as_alias->underlying_type); + } + return false; +} + static bool expect_integer(AnyExprV v_inferred) { - return v_inferred->inferred_type == TypeDataInt::create() || v_inferred->inferred_type->try_as() || v_inferred->inferred_type == TypeDataCoins::create(); + return expect_integer(v_inferred->inferred_type); } -static bool expect_boolean(AnyExprV v_inferred) { - return v_inferred->inferred_type == TypeDataBool::create(); +static bool expect_boolean(TypePtr inferred_type) { + if (inferred_type == TypeDataBool::create()) { + return true; + } + if (const TypeDataAlias* as_alias = inferred_type->try_as()) { + return expect_boolean(as_alias->underlying_type); + } + return false; } +static bool expect_boolean(AnyExprV v_inferred) { + return expect_boolean(v_inferred->inferred_type); +} class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { FunctionPtr cur_f = nullptr; // may be nullptr if checking `const a = ...` init_value @@ -282,9 +317,9 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { parent::visit(v); - TypePtr obj_type = v->get_obj()->inferred_type; if (v->is_target_indexed_access()) { - if (obj_type->try_as() && v->inferred_type->get_width_on_stack() != 1) { + TypePtr obj_type = v->get_obj()->inferred_type; + if (obj_type->unwrap_alias()->try_as() && v->inferred_type->get_width_on_stack() != 1) { fire_error_cannot_put_non1_stack_width_arg_to_tuple(cur_f, v->loc, v->inferred_type); } } @@ -296,7 +331,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { FunctionPtr fun_ref = v->fun_maybe; if (!fun_ref) { // `local_var(args)` and similar - const TypeDataFunCallable* f_callable = v->get_callee()->inferred_type->try_as(); + const TypeDataFunCallable* f_callable = v->get_callee()->inferred_type->unwrap_alias()->try_as(); tolk_assert(f_callable && f_callable->params_size() == v->get_num_args()); for (int i = 0; i < v->get_num_args(); ++i) { auto arg_i = v->get_arg(i)->get_expr(); @@ -381,7 +416,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // `(v1, v2) = rhs` / `var (v1, v2) = rhs` (rhs may be `(1,2)` or `tensorVar` or `someF()`, doesn't matter) // dig recursively into v1 and v2 with corresponding rhs i-th item of a tensor if (auto lhs_tensor = lhs->try_as()) { - const TypeDataTensor* rhs_type_tensor = rhs_type->try_as(); + const TypeDataTensor* rhs_type_tensor = rhs_type->unwrap_alias()->try_as(); if (!rhs_type_tensor) { fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to a tensor"); } @@ -398,7 +433,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // `[v1, v2] = rhs` / `var [v1, v2] = rhs` (rhs may be `[1,2]` or `tupleVar` or `someF()`, doesn't matter) // dig recursively into v1 and v2 with corresponding rhs i-th item of a tuple if (auto lhs_tuple = lhs->try_as()) { - const TypeDataTypedTuple* rhs_type_tuple = rhs_type->try_as(); + const TypeDataTypedTuple* rhs_type_tuple = rhs_type->unwrap_alias()->try_as(); if (!rhs_type_tuple) { fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to a tuple"); } @@ -414,7 +449,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // check `untypedTuple.0 = rhs_tensor` and other non-1 width elements if (auto lhs_dot = lhs->try_as()) { - if (lhs_dot->is_target_indexed_access() && lhs_dot->get_obj()->inferred_type == TypeDataTuple::create()) { + if (lhs_dot->is_target_indexed_access() && lhs_dot->get_obj()->inferred_type->unwrap_alias() == TypeDataTuple::create()) { if (rhs_type->get_width_on_stack() != 1) { fire_error_cannot_put_non1_stack_width_arg_to_tuple(cur_f, err_loc->loc, rhs_type); } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 9db82f5767..236037c3e7 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -401,7 +401,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // `(v1, v2) = rhs` / `var (v1, v2) = rhs` (rhs may be `(1,2)` or `tensorVar` or `someF()`, doesn't matter) // dig recursively into v1 and v2 with corresponding rhs i-th item of a tensor if (auto lhs_tensor = lhs->try_as()) { - const TypeDataTensor* rhs_type_tensor = rhs_type->try_as(); + const TypeDataTensor* rhs_type_tensor = rhs_type->unwrap_alias()->try_as(); std::vector types_list; types_list.reserve(lhs_tensor->size()); for (int i = 0; i < lhs_tensor->size(); ++i) { @@ -416,7 +416,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // `[v1, v2] = rhs` / `var [v1, v2] = rhs` (rhs may be `[1,2]` or `tupleVar` or `someF()`, doesn't matter) // dig recursively into v1 and v2 with corresponding rhs i-th item of a tuple if (auto lhs_tuple = lhs->try_as()) { - const TypeDataTypedTuple* rhs_type_tuple = rhs_type->try_as(); + const TypeDataTypedTuple* rhs_type_tuple = rhs_type->unwrap_alias()->try_as(); std::vector types_list; types_list.reserve(lhs_tuple->size()); for (int i = 0; i < lhs_tuple->size(); ++i) { @@ -478,7 +478,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, TypeDataInt::create()); break; case tok_logical_not: - if (rhs->inferred_type == TypeDataBool::create()) { + if (rhs->inferred_type->unwrap_alias() == TypeDataBool::create()) { builtin_func = "!b"; // "overloaded" for bool } assign_inferred_type(v, TypeDataBool::create()); @@ -520,7 +520,7 @@ class InferTypesAndCallsAndFieldsVisitor final { case tok_bitwise_xor: flow = infer_any_expr(lhs, std::move(flow), false).out_flow; flow = infer_any_expr(rhs, std::move(flow), false).out_flow; - if (lhs->inferred_type == TypeDataBool::create() && rhs->inferred_type == TypeDataBool::create()) { + if (lhs->inferred_type->unwrap_alias() == TypeDataBool::create() && rhs->inferred_type->unwrap_alias() == TypeDataBool::create()) { assign_inferred_type(v, TypeDataBool::create()); } else { assign_inferred_type(v, TypeDataInt::create()); @@ -615,7 +615,7 @@ class InferTypesAndCallsAndFieldsVisitor final { ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); assign_inferred_type(v, TypeDataBool::create()); - TypePtr expr_type = v->get_expr()->inferred_type; + TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); TypePtr non_null_type = calculate_type_subtract_null(expr_type); if (expr_type == TypeDataNullLiteral::create()) { // `expr == null` is always true v->mutate()->assign_always_true_or_false(v->is_negated ? 2 : 1); @@ -652,7 +652,7 @@ class InferTypesAndCallsAndFieldsVisitor final { ExprFlow infer_not_null_operator(V v, FlowContext&& flow, bool used_as_condition) { ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); - if (const auto* as_nullable = v->get_expr()->inferred_type->try_as()) { + if (const auto* as_nullable = v->get_expr()->inferred_type->unwrap_alias()->try_as()) { assign_inferred_type(v, as_nullable->inner); } else { assign_inferred_type(v, v->get_expr()); @@ -710,6 +710,9 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, fun_ref->inferred_full_type); return ExprFlow(std::move(flow), used_as_condition); + } else if (AliasDefPtr alias_ref = v->sym->try_as()) { + fire(cur_f, v->loc, "type `" + alias_ref->name + "` can not be used as a value"); + } else { tolk_assert(false); } @@ -766,6 +769,7 @@ class InferTypesAndCallsAndFieldsVisitor final { flow = infer_any_expr(v->get_obj(), std::move(flow), false).out_flow; TypePtr obj_type = v->get_obj()->inferred_type; + TypePtr unwrapped_obj_type = obj_type->unwrap_alias(); // our goal is to fill v->target knowing type of obj V v_ident = v->get_identifier(); // field/method name vertex V v_instantiationTs = v->get_instantiationTs(); @@ -775,7 +779,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // at first, check for indexed access if (field_name[0] >= '0' && field_name[0] <= '9') { int index_at = std::stoi(std::string(field_name)); - if (const auto* t_tensor = obj_type->try_as()) { + if (const auto* t_tensor = unwrapped_obj_type->try_as()) { if (index_at >= t_tensor->size()) { fire(cur_f, v_ident->loc, "invalid tensor index, expected 0.." + std::to_string(t_tensor->items.size() - 1)); } @@ -789,7 +793,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); } - if (const auto* t_tuple = obj_type->try_as()) { + if (const auto* t_tuple = unwrapped_obj_type->try_as()) { if (index_at >= t_tuple->size()) { fire(cur_f, v_ident->loc, "invalid tuple index, expected 0.." + std::to_string(t_tuple->items.size() - 1)); } @@ -803,7 +807,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); } - if (obj_type->try_as()) { + if (unwrapped_obj_type->try_as()) { TypePtr item_type = nullptr; if (v->is_lvalue && !hint) { // left side of assignment item_type = TypeDataUnknown::create(); @@ -899,7 +903,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // treat callee like a usual expression flow = infer_any_expr(callee, std::move(flow), false).out_flow; // it must have "callable" inferred type - const TypeDataFunCallable* f_callable = callee->inferred_type->try_as(); + const TypeDataFunCallable* f_callable = callee->inferred_type->unwrap_alias()->try_as(); if (!f_callable) { // `5()` / `SOME_CONST()` / `null()` fire(cur_f, v->loc, "calling a non-function " + to_string(callee->inferred_type)); } @@ -951,7 +955,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (param_type->has_genericT_inside()) { param_type = deducingTs->auto_deduce_from_argument(cur_f, dot_obj->loc, param_type, dot_obj->inferred_type); } - if (param_0.is_mutate_parameter() && dot_obj->inferred_type != param_type) { + if (param_0.is_mutate_parameter() && dot_obj->inferred_type->unwrap_alias() != param_type->unwrap_alias()) { if (SinkExpression s_expr = extract_sink_expression_from_vertex(dot_obj)) { assign_inferred_type(dot_obj, calc_declared_type_before_smart_cast(dot_obj)); flow.register_known_type(s_expr, param_type); @@ -974,7 +978,7 @@ class InferTypesAndCallsAndFieldsVisitor final { flow = infer_any_expr(arg_i, std::move(flow), false, param_type).out_flow; } assign_inferred_type(v->get_arg(i), arg_i); // arg itself is an expression - if (param_i.is_mutate_parameter() && arg_i->inferred_type != param_type) { + if (param_i.is_mutate_parameter() && arg_i->inferred_type->unwrap_alias() != param_type->unwrap_alias()) { if (SinkExpression s_expr = extract_sink_expression_from_vertex(arg_i)) { assign_inferred_type(arg_i, calc_declared_type_before_smart_cast(arg_i)); flow.register_known_type(s_expr, param_type); @@ -1023,7 +1027,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } ExprFlow infer_tensor(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { - const TypeDataTensor* tensor_hint = hint ? hint->try_as() : nullptr; + const TypeDataTensor* tensor_hint = hint ? hint->unwrap_alias()->try_as() : nullptr; std::vector types_list; types_list.reserve(v->get_items().size()); for (int i = 0; i < v->size(); ++i) { @@ -1036,7 +1040,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } ExprFlow infer_typed_tuple(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { - const TypeDataTypedTuple* tuple_hint = hint ? hint->try_as() : nullptr; + const TypeDataTypedTuple* tuple_hint = hint ? hint->unwrap_alias()->try_as() : nullptr; std::vector types_list; types_list.reserve(v->get_items().size()); for (int i = 0; i < v->size(); ++i) { diff --git a/tolk/pipe-optimize-boolean-expr.cpp b/tolk/pipe-optimize-boolean-expr.cpp index c4c5d1dcf5..642defa035 100644 --- a/tolk/pipe-optimize-boolean-expr.cpp +++ b/tolk/pipe-optimize-boolean-expr.cpp @@ -57,6 +57,29 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { return v_not; } + static bool expect_integer(TypePtr inferred_type) { + if (inferred_type == TypeDataInt::create()) { + return true; + } + if (inferred_type->try_as() || inferred_type == TypeDataCoins::create()) { + return true; + } + if (const TypeDataAlias* as_alias = inferred_type->try_as()) { + return expect_integer(as_alias->underlying_type); + } + return false; + } + + static bool expect_boolean(TypePtr inferred_type) { + if (inferred_type == TypeDataBool::create()) { + return true; + } + if (const TypeDataAlias* as_alias = inferred_type->try_as()) { + return expect_boolean(as_alias->underlying_type); + } + return false; + } + protected: AnyExprV replace(V v) override { @@ -66,11 +89,11 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { if (auto inner_not = v->get_rhs()->try_as(); inner_not && inner_not->tok == tok_logical_not) { AnyExprV cond_not_not = inner_not->get_rhs(); // `!!boolVar` => `boolVar` - if (cond_not_not->inferred_type == TypeDataBool::create()) { + if (expect_boolean(cond_not_not->inferred_type)) { return cond_not_not; } // `!!intVar` => `intVar != 0` - if (cond_not_not->inferred_type == TypeDataInt::create()) { + if (expect_integer(cond_not_not->inferred_type)) { auto v_zero = create_int_const(v->loc, td::make_refint(0)); auto v_neq = createV(v->loc, "!=", tok_neq, cond_not_not, v_zero); v_neq->mutate()->assign_rvalue_true(); @@ -94,7 +117,7 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { if (v->tok == tok_eq || v->tok == tok_neq) { AnyExprV lhs = v->get_lhs(); AnyExprV rhs = v->get_rhs(); - if (lhs->inferred_type == TypeDataBool::create() && rhs->type == ast_bool_const) { + if (expect_boolean(lhs->inferred_type) && rhs->type == ast_bool_const) { // `boolVar == true` / `boolVar != false` if (rhs->as()->bool_val ^ (v->tok == tok_neq)) { return lhs; diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index ea4cc8110b..e76f5b4796 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -133,6 +133,13 @@ static void register_global_var(V v) { v->mutate()->assign_var_ref(g_sym); } +static void register_type_alias(V v) { + AliasDefData* a_sym = new AliasDefData(static_cast(v->get_identifier()->name), v->loc, v->underlying_type); + + G.symtable.add_type_alias(a_sym); + v->mutate()->assign_alias_ref(a_sym); +} + static LocalVarData register_parameter(V v, int idx) { if (v->is_underscore()) { return {"", v->loc, v->declared_type, 0, idx}; @@ -238,6 +245,9 @@ static void iterate_through_file_symbols(const SrcFile* file) { case ast_global_var_declaration: register_global_var(v->as()); break; + case ast_type_alias_declaration: + register_type_alias(v->as()); + break; case ast_function_declaration: register_function(v->as()); break; diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 2ace2e7193..3f8f622c0b 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -136,6 +136,15 @@ struct NameAndScopeResolver { }; struct TypeDataResolver { + static TypePtr finalize_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { + if (type_data) { + if (type_data->has_unresolved_inside()) { + type_data = resolve_identifiers_in_type_data(cur_f, type_data, genericTs); + } + } + return type_data; + } + GNU_ATTRIBUTE_NOINLINE static TypePtr resolve_identifiers_in_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { return type_data->replace_children_custom([cur_f, genericTs](TypePtr child) { @@ -144,6 +153,14 @@ struct TypeDataResolver { std::string nameT = un->text; return TypeDataGenericT::create(std::move(nameT)); } + if (const Symbol* sym = lookup_global_symbol(un->text)) { + if (AliasDefPtr alias_ref = sym->try_as()) { + if (alias_ref->underlying_type->has_unresolved_inside()) { + resolve_and_mutate_type_alias(alias_ref); + } + return TypeDataAlias::create(alias_ref->name, alias_ref->underlying_type); + } + } if (un->text == "auto") { throw ParseError(cur_f, un->loc, "`auto` type does not exist; just omit a type for local variable (will be inferred from assignment); parameters should always be typed"); } @@ -155,13 +172,25 @@ struct TypeDataResolver { return child; }); } + + static void resolve_and_mutate_type_alias(AliasDefPtr alias_ref) { + static std::vector called_stack; + + // prevent recursion like `type A = B; type B = A` + bool contains = std::find(called_stack.begin(), called_stack.end(), alias_ref) != called_stack.end(); + if (contains) { + throw ParseError(alias_ref->loc, "type `" + alias_ref->name + "` circularly references itself"); + } + + called_stack.push_back(alias_ref); + TypePtr underlying_type = finalize_type_data(nullptr, alias_ref->underlying_type, nullptr); + alias_ref->mutate()->assign_resolved_type(underlying_type); + called_stack.pop_back(); + } }; static TypePtr finalize_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { - if (!type_data || !type_data->has_unresolved_inside()) { - return type_data; - } - return TypeDataResolver::resolve_identifiers_in_type_data(cur_f, type_data, genericTs); + return TypeDataResolver::finalize_type_data(cur_f, type_data, genericTs); } @@ -344,6 +373,10 @@ void pipeline_resolve_identifiers_and_assign_symbols() { v_const->mutate()->assign_resolved_type(declared_type); v_const->const_ref->mutate()->assign_resolved_type(declared_type); } + + } else if (auto v_alias = v->try_as()) { + TypeDataResolver::resolve_and_mutate_type_alias(v_alias->alias_ref); + v_alias->mutate()->assign_resolved_type(v_alias->alias_ref->underlying_type); } } } diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index 9a2f7ef4df..d3ddb5af75 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -196,6 +196,13 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { return TypeDataInt::create(); } + if (const auto* a_alias = a->try_as()) { + return calculate_type_lca(a_alias->underlying_type, b); + } + if (const auto* b_alias = b->try_as()) { + return calculate_type_lca(a, b_alias->underlying_type); + } + return nullptr; } @@ -352,7 +359,7 @@ FlowContext FlowContext::merge_flow(FlowContext&& c1, FlowContext&& c2) { // return `T`, so that `T?` = type // what for: `if (x != null)`, to smart cast x inside if TypePtr calculate_type_subtract_null(TypePtr type) { - if (const auto* as_nullable = type->try_as()) { + if (const auto* as_nullable = type->unwrap_alias()->try_as()) { return as_nullable->inner; } // union types will be handled here @@ -415,7 +422,7 @@ TypePtr calc_declared_type_before_smart_cast(AnyExprV v) { } if (auto as_dot = v->try_as(); as_dot && as_dot->is_target_indexed_access()) { - TypePtr obj_type = as_dot->get_obj()->inferred_type; // v already inferred; hence, index_at is correct + TypePtr obj_type = as_dot->get_obj()->inferred_type->unwrap_alias(); // v already inferred; hence, index_at is correct int index_at = std::get(as_dot->target); if (const auto* t_tensor = obj_type->try_as()) { return t_tensor->items[index_at]; @@ -436,7 +443,7 @@ TypePtr calc_declared_type_before_smart_cast(AnyExprV v) { TypePtr calc_smart_cast_type_on_assignment(TypePtr lhs_declared_type, TypePtr rhs_inferred_type) { // assign `T` to `T?` (or at least "assignable-to-T" to "T?") // smart cast to `T` - if (const auto* lhs_nullable = lhs_declared_type->try_as()) { + if (const auto* lhs_nullable = lhs_declared_type->unwrap_alias()->try_as()) { if (lhs_nullable->inner->can_rhs_be_assigned(rhs_inferred_type)) { return lhs_nullable->inner; } @@ -444,7 +451,7 @@ TypePtr calc_smart_cast_type_on_assignment(TypePtr lhs_declared_type, TypePtr rh // assign `null` to `T?` // smart cast to `null` - if (lhs_declared_type->try_as() && rhs_inferred_type == TypeDataNullLiteral::create()) { + if (lhs_declared_type->unwrap_alias()->try_as() && rhs_inferred_type == TypeDataNullLiteral::create()) { return TypeDataNullLiteral::create(); } diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index ea009957a5..4942fe8701 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -113,6 +113,10 @@ void LocalVarData::assign_inferred_type(TypePtr inferred_type) { this->declared_type = inferred_type; } +void AliasDefData::assign_resolved_type(TypePtr underlying_type) { + this->underlying_type = underlying_type; +} + GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_redefinition_of_symbol(SrcLocation loc, const Symbol* previous) { SrcLocation prev_loc = previous->loc; @@ -149,6 +153,14 @@ void GlobalSymbolTable::add_global_const(GlobalConstPtr c_sym) { } } +void GlobalSymbolTable::add_type_alias(AliasDefPtr a_sym) { + auto key = key_hash(a_sym->name); + auto [it, inserted] = entries.emplace(key, a_sym); + if (!inserted) { + fire_error_redefinition_of_symbol(a_sym->loc, it->second); + } +} + const Symbol* lookup_global_symbol(std::string_view name) { return G.symtable.lookup(name); } diff --git a/tolk/symtable.h b/tolk/symtable.h index e78eef6e81..a27aecd44d 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -216,6 +216,18 @@ struct GlobalConstData final : Symbol { void assign_const_value(ConstantValue&& value); }; +struct AliasDefData final : Symbol { + TypePtr underlying_type; + + AliasDefData(std::string name, SrcLocation loc, TypePtr underlying_type) + : Symbol(std::move(name), loc) + , underlying_type(underlying_type) { + } + + AliasDefData* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr underlying_type); +}; + class GlobalSymbolTable { std::unordered_map entries; @@ -227,6 +239,7 @@ class GlobalSymbolTable { void add_function(FunctionPtr f_sym); void add_global_var(GlobalVarPtr g_sym); void add_global_const(GlobalConstPtr c_sym); + void add_type_alias(AliasDefPtr a_sym); const Symbol* lookup(std::string_view name) const { const auto it = entries.find(key_hash(name)); diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index ebcfc6fe6f..459ad8136d 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -113,6 +113,18 @@ void type_system_init() { // and creates an object only if it isn't found in a global hashtable // +TypePtr TypeDataAlias::create(const std::string& alias_name, TypePtr underlying_type) { + TypeDataTypeIdCalculation hash(5694590762732189561ULL); + hash.feed_string(alias_name); + hash.feed_child(underlying_type); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + std::string name_copy = alias_name; + return hash.register_unique(new TypeDataAlias(hash.type_id(), hash.children_flags(), std::move(name_copy), underlying_type)); +} + TypePtr TypeDataNullable::create(TypePtr inner) { TypeDataTypeIdCalculation hash(1774084920039440885ULL); hash.feed_child(inner); @@ -286,6 +298,11 @@ std::string TypeDataBytesN::as_human_readable() const { // only non-trivial implementations are here; by default (no children), `callback(this)` is executed // +void TypeDataAlias::traverse(const TraverserCallbackT& callback) const { + callback(this); + underlying_type->traverse(callback); +} + void TypeDataNullable::traverse(const TraverserCallbackT& callback) const { callback(this); inner->traverse(callback); @@ -322,6 +339,10 @@ void TypeDataTypedTuple::traverse(const TraverserCallbackT& callback) const { // only non-trivial implementations are here; by default (no children), `return callback(this)` is executed // +TypePtr TypeDataAlias::replace_children_custom(const ReplacerCallbackT& callback) const { + return callback(create(alias_name, underlying_type->replace_children_custom(callback))); +} + TypePtr TypeDataNullable::replace_children_custom(const ReplacerCallbackT& callback) const { return callback(create(inner->replace_children_custom(callback))); } @@ -361,6 +382,13 @@ TypePtr TypeDataTypedTuple::replace_children_custom(const ReplacerCallbackT& cal // the same goes for passing arguments, returning values, etc. — where the "receiver" (lhs) checks "applier" (rhs) // +bool TypeDataAlias::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return underlying_type->can_rhs_be_assigned(rhs); +} + bool TypeDataInt::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; @@ -371,6 +399,9 @@ bool TypeDataInt::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == TypeDataCoins::create()) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -378,6 +409,9 @@ bool TypeDataBool::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -385,6 +419,9 @@ bool TypeDataCell::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -392,6 +429,9 @@ bool TypeDataSlice::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); // note, that bytesN is NOT automatically cast to slice without `as` operator } @@ -399,6 +439,9 @@ bool TypeDataBuilder::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -406,6 +449,9 @@ bool TypeDataTuple::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -413,6 +459,9 @@ bool TypeDataContinuation::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -420,6 +469,9 @@ bool TypeDataNullLiteral::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -436,6 +488,9 @@ bool TypeDataNullable::can_rhs_be_assigned(TypePtr rhs) const { if (inner->can_rhs_be_assigned(rhs)) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -443,6 +498,23 @@ bool TypeDataFunCallable::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataFunCallable* rhs_callable = rhs->try_as()) { + if (rhs_callable->params_size() != params_size()) { + return false; + } + for (int i = 0; i < params_size(); ++i) { + if (!rhs_callable->params_types[i]->can_rhs_be_assigned(params_types[i])) { + return false; + } + if (!params_types[i]->can_rhs_be_assigned(rhs_callable->params_types[i])) { + return false; + } + } + return return_type->can_rhs_be_assigned(rhs_callable->return_type) && rhs_callable->return_type->can_rhs_be_assigned(return_type); + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -460,6 +532,9 @@ bool TypeDataTensor::can_rhs_be_assigned(TypePtr rhs) const { } return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -472,6 +547,9 @@ bool TypeDataTypedTuple::can_rhs_be_assigned(TypePtr rhs) const { } return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -482,6 +560,9 @@ bool TypeDataIntN::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == TypeDataInt::create()) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); // `int8` is NOT assignable to `int32` without `as` } @@ -491,7 +572,10 @@ bool TypeDataBytesN::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } - return rhs == TypeDataNever::create(); + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } + return false; } bool TypeDataCoins::can_rhs_be_assigned(TypePtr rhs) const { @@ -501,6 +585,9 @@ bool TypeDataCoins::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == TypeDataInt::create()) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -521,6 +608,9 @@ bool TypeDataVoid::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -532,6 +622,10 @@ bool TypeDataVoid::can_rhs_be_assigned(TypePtr rhs) const { // note, that it's not auto-casts `var lhs: = rhs`, it's an expression `rhs as ` // +bool TypeDataAlias::can_be_casted_with_as_operator(TypePtr cast_to) const { + return underlying_type->can_be_casted_with_as_operator(cast_to); +} + bool TypeDataInt::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { // `int` as `int?` return can_be_casted_with_as_operator(to_nullable->inner); @@ -542,6 +636,9 @@ bool TypeDataInt::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to == TypeDataCoins::create()) { // `int` as `coins` return true; } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return cast_to == this; } @@ -555,6 +652,9 @@ bool TypeDataBool::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_intN = cast_to->try_as()) { return !to_intN->is_unsigned; // `bool` as `int8` ok, `bool` as `uintN` not (true is -1) } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return cast_to == this; } @@ -562,6 +662,9 @@ bool TypeDataCell::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return cast_to == this; } @@ -572,6 +675,9 @@ bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return cast_to == this; } @@ -579,6 +685,9 @@ bool TypeDataBuilder::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return cast_to == this; } @@ -586,6 +695,9 @@ bool TypeDataTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return cast_to == this; } @@ -593,17 +705,29 @@ bool TypeDataContinuation::can_be_casted_with_as_operator(TypePtr cast_to) const if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return cast_to == this; } bool TypeDataNullLiteral::can_be_casted_with_as_operator(TypePtr cast_to) const { - return cast_to == this || cast_to->try_as(); + if (cast_to->try_as()) { + return true; + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + return cast_to == this; } bool TypeDataNullable::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { return inner->can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return false; } @@ -611,7 +735,21 @@ bool TypeDataFunCallable::can_be_casted_with_as_operator(TypePtr cast_to) const if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } - return this == cast_to; + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + if (const TypeDataFunCallable* to_callable = cast_to->try_as()) { + if (to_callable->params_size() != params_size()) { + return false; + } + for (int i = 0; i < params_size(); ++i) { + if (!params_types[i]->can_be_casted_with_as_operator(to_callable->params_types[i])) { + return false; + } + } + return return_type->can_be_casted_with_as_operator(to_callable->return_type); + } + return false; } bool TypeDataGenericT::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -630,6 +768,9 @@ bool TypeDataTensor::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return false; } @@ -645,6 +786,9 @@ bool TypeDataTypedTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return false; } @@ -655,6 +799,9 @@ bool TypeDataIntN::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { // `int8` as `int32?` return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return cast_to == TypeDataInt::create() || cast_to == TypeDataCoins::create(); } @@ -665,6 +812,9 @@ bool TypeDataBytesN::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { // `bytes8` as `slice?` return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return cast_to == TypeDataSlice::create(); } @@ -675,6 +825,9 @@ bool TypeDataCoins::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { // `coins` as `coins?` / `coins` as `int?` return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } if (cast_to == TypeDataInt::create()) { return true; } @@ -708,6 +861,10 @@ bool TypeDataVoid::can_be_casted_with_as_operator(TypePtr cast_to) const { // though still, tricky situations like `(int, ())?` can still "embed" TVM NULL in parallel with original value // +bool TypeDataAlias::can_hold_tvm_null_instead() const { + return underlying_type->can_hold_tvm_null_instead(); +} + bool TypeDataNullable::can_hold_tvm_null_instead() const { if (get_width_on_stack() != 1) { // `(int, int)?` / `()?` can not hold null instead return false; // only `int?` / `cell?` / `StructWith1IntField?` can diff --git a/tolk/type-system.h b/tolk/type-system.h index 07a2496ca0..ebb25312e5 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -83,6 +83,8 @@ class TypeData { bool has_genericT_inside() const { return flags & flag_contains_genericT_inside; } bool has_unresolved_inside() const { return flags & flag_contains_unresolved_inside; } + inline TypePtr unwrap_alias() const; + using TraverserCallbackT = std::function; using ReplacerCallbackT = std::function; @@ -103,6 +105,32 @@ class TypeData { } }; +/* + * `type AliasName = underlying_type` is an alias, which is fully interchangeable with its original type. + * It never occurs at runtime: at IR generation it's erased, replaced by an underlying type. + * But until IR generation, aliases exists, and `var t: MyTensor2 = (1,2)` is alias "MyTensor", not tensor (int,int). + * That's why lots of code comparing types use `type->unwrap_alias()` or `try_as`. + */ +class TypeDataAlias final : public TypeData { + explicit TypeDataAlias(uint64_t type_id, int children_flags, std::string&& alias_name, TypePtr underlying_type) + : TypeData(type_id, children_flags, underlying_type->get_width_on_stack()) + , alias_name(std::move(alias_name)) + , underlying_type(underlying_type) {} + +public: + const std::string alias_name; + const TypePtr underlying_type; + + static TypePtr create(const std::string& alias_name, TypePtr underlying_type); + + std::string as_human_readable() const override { return alias_name; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + void traverse(const TraverserCallbackT& callback) const override; + TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; + bool can_hold_tvm_null_instead() const override; +}; + /* * `int` is TypeDataInt, representation of TVM int. */ @@ -519,6 +547,13 @@ class TypeDataVoid final : public TypeData { // -------------------------------------------- +inline TypePtr TypeData::unwrap_alias() const { + TypePtr unwrapped = this; + while (const TypeDataAlias* as_alias = unwrapped->try_as()) { + unwrapped = as_alias->underlying_type; + } + return unwrapped; +} class Lexer; TypePtr parse_type_from_tokens(Lexer& lex); From 249badf4b39851db69fd1cc0351d7088f768a7bd Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sat, 5 Apr 2025 17:11:48 +0300 Subject: [PATCH 201/524] [Tolk] Union types `T1 | T2 | ...` and pattern matching Union types allow a variable to hold multiple possible types. They are key to message parsing and routing in future struct-based designs. At the TVM level, they are tagged unions, like enums in Rust. Pattern matching is the only way of handling union types: > match (result) { > slice => { /* result is smart-casted to slice */ } > UserId => { /* result is smart-casted to UserId */ } > } `match` must cover all union cases (should be exhaustive). It can also be used as an expression. Variable declaration inside `match` is allowed. Besides `match`, you can test a union type by `is`: > if (v is cell) { > // v is smart-casted to cell --- tolk-tester/tests/generics-1.tolk | 28 + tolk-tester/tests/intN-tests.tolk | 16 +- .../tests/invalid-declaration/err-1140.tolk | 26 + .../tests/invalid-declaration/err-1473.tolk | 16 + .../tests/invalid-semantics/err-4355.tolk | 21 + .../tests/invalid-symbol/err-2208.tolk | 13 + .../tests/invalid-symbol/err-2355.tolk | 12 + .../tests/invalid-symbol/err-2898.tolk | 16 + .../tests/invalid-syntax/err-3360.tolk | 10 + .../tests/invalid-syntax/err-3390.tolk | 4 +- .../tests/invalid-syntax/err-3457.tolk | 18 + .../tests/invalid-typing/err-6089.tolk | 8 + .../tests/invalid-typing/err-6110.tolk | 15 + .../tests/invalid-typing/err-6144.tolk | 13 + .../tests/invalid-typing/err-6204.tolk | 7 +- .../tests/invalid-typing/err-6274.tolk | 19 + .../tests/invalid-typing/err-6353.tolk | 30 + .../tests/invalid-typing/err-6366.tolk | 13 + .../tests/invalid-typing/err-6427.tolk | 13 + .../tests/invalid-typing/err-6460.tolk | 19 + .../tests/invalid-typing/err-6498.tolk | 14 + .../tests/invalid-typing/err-6502.tolk | 11 + .../tests/invalid-typing/err-6518.tolk | 16 + .../tests/invalid-typing/err-6583.tolk | 12 + .../tests/invalid-typing/err-6659.tolk | 18 + .../tests/invalid-typing/err-6673.tolk | 14 + .../tests/invalid-typing/err-6690.tolk | 12 + .../tests/invalid-typing/err-6712.tolk | 12 + .../tests/invalid-typing/err-6767.tolk | 10 + .../tests/invalid-typing/err-6801.tolk | 9 + .../tests/invalid-typing/err-6808.tolk | 11 + .../tests/invalid-typing/err-6829.tolk | 16 + .../tests/invalid-typing/err-6853.tolk | 8 + .../tests/invalid-typing/err-6920.tolk | 12 + .../tests/invalid-typing/err-6930.tolk | 15 + tolk-tester/tests/nullable-tensors.tolk | 65 +- tolk-tester/tests/smart-cast-tests.tolk | 30 +- tolk-tester/tests/type-aliases-tests.tolk | 11 +- tolk-tester/tests/union-types-tests.tolk | 873 ++++++++++++++++++ tolk-tester/tests/unreachable-5.tolk | 24 + tolk-tester/tests/var-apply-tests.tolk | 17 +- .../warnings-not-errors/unreachable-3.tolk | 2 +- .../tests/warnings-not-errors/warnings-1.tolk | 8 +- .../tests/warnings-not-errors/warnings-2.tolk | 8 +- .../tests/warnings-not-errors/warnings-3.tolk | 26 + tolk-tester/tolk-tester.py | 2 +- tolk/abscode.cpp | 15 +- tolk/ast-from-tokens.cpp | 317 ++++--- tolk/ast-replacer.h | 16 +- tolk/ast-replicator.h | 18 +- tolk/ast-stringifier.h | 22 +- tolk/ast-visitor.h | 14 +- tolk/ast.cpp | 18 +- tolk/ast.h | 95 +- tolk/generics-helpers.cpp | 15 +- tolk/lexer.cpp | 6 +- tolk/lexer.h | 4 + tolk/pipe-ast-to-legacy.cpp | 464 +++++++--- tolk/pipe-calc-rvalue-lvalue.cpp | 141 ++- tolk/pipe-check-inferred-types.cpp | 116 ++- tolk/pipe-check-rvalue-lvalue.cpp | 90 +- tolk/pipe-constant-folding.cpp | 4 +- tolk/pipe-infer-types-and-calls.cpp | 130 ++- tolk/pipe-optimize-boolean-expr.cpp | 6 +- tolk/pipe-resolve-identifiers.cpp | 50 +- tolk/smart-casts-cfg.cpp | 159 ++-- tolk/smart-casts-cfg.h | 7 +- tolk/type-system.cpp | 586 +++++++++--- tolk/type-system.h | 201 ++-- 69 files changed, 3361 insertions(+), 706 deletions(-) create mode 100644 tolk-tester/tests/invalid-declaration/err-1140.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1473.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4355.tolk create mode 100644 tolk-tester/tests/invalid-symbol/err-2208.tolk create mode 100644 tolk-tester/tests/invalid-symbol/err-2355.tolk create mode 100644 tolk-tester/tests/invalid-symbol/err-2898.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3360.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3457.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6089.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6110.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6144.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6274.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6353.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6366.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6427.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6460.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6498.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6502.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6518.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6583.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6659.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6673.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6690.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6712.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6767.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6801.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6808.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6829.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6853.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6920.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6930.tolk create mode 100644 tolk-tester/tests/union-types-tests.tolk create mode 100644 tolk-tester/tests/unreachable-5.tolk create mode 100644 tolk-tester/tests/warnings-not-errors/warnings-3.tolk diff --git a/tolk-tester/tests/generics-1.tolk b/tolk-tester/tests/generics-1.tolk index 70cd2371c9..feb7a8393e 100644 --- a/tolk-tester/tests/generics-1.tolk +++ b/tolk-tester/tests/generics-1.tolk @@ -24,6 +24,7 @@ fun test102(): (int, int, int, [int, int]) { var b = getTwo() as int; var c: int = 1 ? getTwo() : getTwo(); var c redef = getTwo(); + var d: int = match (1 as int|slice|builder) { int => getTwo(), slice => getTwo(), builder => getTwo() }; var ab_tens = (0, (1, 2)); ab_tens.0 = getTwo(); ab_tens.1.1 = getTwo(); @@ -169,6 +170,29 @@ fun test111() { return (isSomeNullableNull(i1), isSomeNullableNull(i2), isSomeNullableNull(i1), isSomeNullableNull(i1)); } +@method_id(112) +fun test112(v: int | slice?) { + return (eq1(v), eq4(v == null)); +} + +fun makeNullable(a: T1 | T2): T1 | T2 | null { + return a; +} + +@method_id(113) +fun test113(a: int | slice) { + var b: int | (int, int) = match (a) { + int => a, + slice => (a.loadInt(32), -1) + }; + __expect_type(makeNullable(b), "int | (int, int) | null"); + return ( + makeNullable(a), -100, makeNullable(b), -100, makeNullable(9), -100, makeNullable(null), -100, + makeNullable(b), -100, makeNullable(null) + ); +} + + fun main(x: int): (int, [Tup2Int]) { __expect_type(test110, "() -> (Tensor2Int, Tensor2Int)"); @@ -190,12 +214,16 @@ fun main(x: int): (int, [Tup2Int]) { @testcase | 109 | 5 | 5 (null) (null) @testcase | 110 | | 2 2 5 2 @testcase | 111 | | 0 0 0 0 +@testcase | 112 | 5 1 | 5 1 0 +@testcase | 112 | 0 0 | 0 0 -1 +@testcase | 113 | 5 1 | 5 1 -100 (null) 5 1 -100 9 -100 (null) -100 (null) 5 1 -100 (null) (null) (null) 0 @fif_codegen DECLPROC eq1 @fif_codegen DECLPROC eq1 @fif_codegen DECLPROC eq1<(int,int)> @fif_codegen DECLPROC eq1<[int,int]> @fif_codegen DECLPROC eq1 +@fif_codegen DECLPROC eq1 @fif_codegen DECLPROC getTwo @fif_codegen_avoid DECLPROC eq1 diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index abafdf1d72..d12436f5e0 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -23,13 +23,13 @@ fun autoInferUInt16Opt(x: uint16) { return null; } -fun autoInferInt_v1() { +fun inferInt_v1(): int { if (random()) { return 0; } else if (random()) { return 0 as uint16; } else { return 0 as int8; } } -fun autoInferInt_v2() { +fun inferInt_v2(): int { if (random()) { return 0 as uint1; } else if (random()) { return 0 as int123; } else { return 0 as int8; } @@ -87,8 +87,8 @@ fun test3(op: int32, qid: uint64) { __expect_type(autoInferInt8, "(int8) -> int8"); __expect_type(autoInferUInt16Opt(0), "uint16?"); - __expect_type(autoInferInt_v1(), "int"); - __expect_type(autoInferInt_v2(), "int"); + __expect_type(inferInt_v1(), "int"); + __expect_type(inferInt_v2(), "int"); var amount: uint100 = 1000; var percent: uint8 = 50; @@ -124,7 +124,7 @@ fun test5() { fun test6() { var x: int11 = ~(0 as int22); while (x) {} - return x ? x : 0; + return x ? x as int : 0; } fun test7() { @@ -211,7 +211,7 @@ fun test13(x1: int?, x2: int8?, x3: int, x4: int8): (int8?, int8?, int8?, int8?, @method_id(114) fun test14(firstComponent: int8?): (int, int16)? { - return firstComponent! < 10 ? null : (firstComponent!, 2); + return firstComponent! < 10 ? null : (firstComponent! as int, 2 as int16); } fun assign0(mutate v: T) { v = 0; } @@ -232,7 +232,7 @@ fun main() { __expect_type(0 as coins, "coins"); __expect_type(someN_coins as coins?, "coins?"); __expect_type(someN_coins as int8?, "int8?"); - __expect_type(10>3 ? 0 as uint8 : 0 as uint16, "int"); + __expect_type(10>3 ? (0 as uint8) as uint8 | uint16 : 0 as uint16, "uint8 | uint16"); return t; } @@ -245,7 +245,7 @@ fun main() { @testcase | 110 | | 50000000 50000100 1234000000 51000000 @testcase | 111 | | [ 1000000000 1000000000 1000000000 -321123456789 321123456789 1100000000 ] @testcase | 114 | 5 | (null) (null) 0 -@testcase | 114 | 15 | 15 2 -1 +@testcase | 114 | 15 | 15 2 130 @fif_codegen DECLPROC assign0 @fif_codegen DECLPROC assign0 diff --git a/tolk-tester/tests/invalid-declaration/err-1140.tolk b/tolk-tester/tests/invalid-declaration/err-1140.tolk new file mode 100644 index 0000000000..eb1ca04e57 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1140.tolk @@ -0,0 +1,26 @@ +fun okWithJustInt(a: int8, b: int16): int { + if (a > 10) { + return a; + } + return b; +} + +fun okWithManualUnion(a: int8, b: int16): int16 | int8 { + if (a > 10) { + return a; + } + return b; +} + +fun autoInferUnionType(a: int8, b: int16) { + if (a > 10) { + return a; + } + return b; +} + +/** +@compilation_should_fail +@stderr function `autoInferUnionType` calculated return type is `int8 | int16`; probably, it's not what you expected; declare `fun (...): ` manually +@stderr fun autoInferUnionType + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1473.tolk b/tolk-tester/tests/invalid-declaration/err-1473.tolk new file mode 100644 index 0000000000..ef79b50336 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1473.tolk @@ -0,0 +1,16 @@ +fun getIntOrSlice() { return 5 as int | slice; } + +fun autoInferUnionType(a: int | slice | null) { + var cc = match (a) { + int => a = getIntOrSlice(), + slice => return beginCell(), + null => return -20 + }; + return cc; +} + +/** +@compilation_should_fail +@stderr function `autoInferUnionType` calculated return type is `builder | int | slice`; probably, it's not what you expected +@stderr fun autoInferUnionType + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4355.tolk b/tolk-tester/tests/invalid-semantics/err-4355.tolk new file mode 100644 index 0000000000..bb648dfa43 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4355.tolk @@ -0,0 +1,21 @@ +fun increment(mutate x: int) { + x += 1; +} + +fun cantModifyValDeclaredInMatch() { + match (var a = (0 + 10 + 90) as int | slice) { + int => increment(mutate a), + slice => {}, + }; + match (val b = (0 + 10 + 90) as int | slice) { + int => increment(mutate b), + slice => {}, + }; +} + +/** +@compilation_should_fail +@stderr modifying immutable variable `b` +@stderr in function `cantModifyValDeclaredInMatch` +@stderr increment(mutate b) + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2208.tolk b/tolk-tester/tests/invalid-symbol/err-2208.tolk new file mode 100644 index 0000000000..b918e0abfc --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2208.tolk @@ -0,0 +1,13 @@ +fun main(x: int | slice) { + match (var a = x) { // a variable declared in `match` subject exists only within a match + int => a + 0, + slice => a.loadInt(32), + } + return a; +} + +/** +@compilation_should_fail +@stderr undefined symbol `a` +@stderr return a; + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2355.tolk b/tolk-tester/tests/invalid-symbol/err-2355.tolk new file mode 100644 index 0000000000..f85dc25916 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2355.tolk @@ -0,0 +1,12 @@ + +fun main() { + match (10) { + asdf => 1, + }; +} + +/** +@compilation_should_fail +@stderr unknown type name `asdf` +@stderr asdf => 1 + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2898.tolk b/tolk-tester/tests/invalid-symbol/err-2898.tolk new file mode 100644 index 0000000000..ba1de77517 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2898.tolk @@ -0,0 +1,16 @@ +fun main(a: int | slice) { + match (a) { + int => { + var cc: int = a; + }, + slice => { + return cc; + }, + }; +} + +/** +@compilation_should_fail +@stderr undefined symbol `cc` +@stderr return cc; + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3360.tolk b/tolk-tester/tests/invalid-syntax/err-3360.tolk new file mode 100644 index 0000000000..7f1b466021 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3360.tolk @@ -0,0 +1,10 @@ + +fun main(r: int | slice) { + // after `is` we expect a type, parse `int?`, and expression becomes incorrect + return r is int ? 10 : 20; +} + +/** +@compilation_should_fail +@stderr expected `;`, got `10` + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3390.tolk b/tolk-tester/tests/invalid-syntax/err-3390.tolk index 58927d3a04..404aa959cc 100644 --- a/tolk-tester/tests/invalid-syntax/err-3390.tolk +++ b/tolk-tester/tests/invalid-syntax/err-3390.tolk @@ -1,8 +1,8 @@ fun main(): int { - ;; here is not a comment + ;; here not a comment } /** @compilation_should_fail -@stderr error: expected `;`, got `is` +@stderr error: expected `;`, got `not` */ diff --git a/tolk-tester/tests/invalid-syntax/err-3457.tolk b/tolk-tester/tests/invalid-syntax/err-3457.tolk new file mode 100644 index 0000000000..a2ef6e96b0 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3457.tolk @@ -0,0 +1,18 @@ +type asdf = int; + +fun main() { + match (10) { + asdf => 1, // it's match by type + }; + + var asdf = 5; + match (10) { + asdf => 2, // it's match by expression now + }; +} + +/** +@compilation_should_fail +@stderr `match` by expression is not supported yet +@stderr asdf => 2 + */ diff --git a/tolk-tester/tests/invalid-typing/err-6089.tolk b/tolk-tester/tests/invalid-typing/err-6089.tolk new file mode 100644 index 0000000000..8548d1781a --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6089.tolk @@ -0,0 +1,8 @@ +fun unionTypesNotAllowedInIs(a: int | slice | builder) { + if (a is int | slice) {} +} + +/** +@compilation_should_fail +@stderr union types are not allowed, use concrete types in `is` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6110.tolk b/tolk-tester/tests/invalid-typing/err-6110.tolk new file mode 100644 index 0000000000..fbef5abe67 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6110.tolk @@ -0,0 +1,15 @@ +fun makeNullable(a: T1 | T2): T1 | T2 | null { + return a; +} + +fun cantDeduceDefiniteUnion(a: int | slice | builder) { + makeNullable(a); // ok + makeNullable(a); // ok + makeNullable(a); +} + +/** +@compilation_should_fail +@stderr can not deduce T1 for generic function `makeNullable` +@stderr makeNullable(a); + */ diff --git a/tolk-tester/tests/invalid-typing/err-6144.tolk b/tolk-tester/tests/invalid-typing/err-6144.tolk new file mode 100644 index 0000000000..6fe610effe --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6144.tolk @@ -0,0 +1,13 @@ +type IntOrSlice = int | slice; + +fun unionTypesNotAllowedInMatch(a: int | slice) { + match (a) { + IntOrSlice => 1, + }; +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: union types are not allowed, use concrete types in `match` +@stderr IntOrSlice + */ diff --git a/tolk-tester/tests/invalid-typing/err-6204.tolk b/tolk-tester/tests/invalid-typing/err-6204.tolk index 58208caab3..8f2b540da0 100644 --- a/tolk-tester/tests/invalid-typing/err-6204.tolk +++ b/tolk-tester/tests/invalid-typing/err-6204.tolk @@ -1,10 +1,13 @@ fun cantUnifyCoinsAndUInt8(n: int8, c: coins) { __expect_type(random() ? n : c as int8, "int8"); - __expect_type(random() ? n : c as int16, "int"); - random() ? n : c; + __expect_type(random() ? n as int16 : c as int16, "int16"); + + var withHint: int = random() ? n : c; // ok + var withoutHint = random() ? n : c; // error } /** @compilation_should_fail @stderr types of ternary branches are incompatible: `int8` and `coins` +@stderr var withoutHint */ diff --git a/tolk-tester/tests/invalid-typing/err-6274.tolk b/tolk-tester/tests/invalid-typing/err-6274.tolk new file mode 100644 index 0000000000..d586efe410 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6274.tolk @@ -0,0 +1,19 @@ +type MInt = int; +type MInt2 = MInt | int; +type MSlice = slice; +type MCell = cell; +type MBuilder = builder; + +fun cantMatchAgainstNotVariant(a: int | slice | MBuilder) { + match (a) { + MInt2 => 10, + MSlice => 20, + MCell => 30, + }; +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: `cell` is not a variant of `int | slice | MBuilder` +@stderr MCell + */ diff --git a/tolk-tester/tests/invalid-typing/err-6353.tolk b/tolk-tester/tests/invalid-typing/err-6353.tolk new file mode 100644 index 0000000000..fb2537592c --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6353.tolk @@ -0,0 +1,30 @@ +fun cantInferUnionTypeWithoutHint(a: int | slice | builder) { + var ok1: builder | (slice | int | int) = match (a) { + int => a, + slice => a, + builder => a, + }; + var ok2 = match (a) { + int => a, + slice => a, + builder => a, + } as builder | (slice | int | int); + var ok3 = match (a) { + int => a as builder | slice | int, + slice => a as builder | slice | int, + builder => a as builder | slice, + }; + + var err = match (a) { + int => a!, + slice => a!, + builder => a!, + }; +} + +/** +@compilation_should_fail +@stderr type of `match` was inferred as `int | slice | builder`; probably, it's not what you expected +@stderr assign it to a variable `var v: = match (...) { ... }` manually +@stderr err = match (a) + */ diff --git a/tolk-tester/tests/invalid-typing/err-6366.tolk b/tolk-tester/tests/invalid-typing/err-6366.tolk new file mode 100644 index 0000000000..cc6052d99e --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6366.tolk @@ -0,0 +1,13 @@ + +fun main() { + match (0 as int | slice) { + int => 1, + else => 10, + }; +} + +/** +@compilation_should_fail +@stderr `else` is not allowed in `match` by type; you should cover all possible types +@stderr else => 10 + */ diff --git a/tolk-tester/tests/invalid-typing/err-6427.tolk b/tolk-tester/tests/invalid-typing/err-6427.tolk new file mode 100644 index 0000000000..37df89a1e4 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6427.tolk @@ -0,0 +1,13 @@ +fun cantReturnVoidAndNonVoid(a: int | slice | builder) { + match (a) { + int => { return; } + slice => {} + builder => "" + }; + return 123; +} + +/** +@compilation_should_fail +@stderr mixing void and non-void returns in function + */ diff --git a/tolk-tester/tests/invalid-typing/err-6460.tolk b/tolk-tester/tests/invalid-typing/err-6460.tolk new file mode 100644 index 0000000000..554f8b4d37 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6460.tolk @@ -0,0 +1,19 @@ +type MNull = null; +type MNull2 = MNull; + +fun main(t: MNull2) { + t as null; + t as MNull; + t as MNull2; + t as int?; + t as int | slice | null; + t as (int, int) | slice | MNull; + + t as int; // error +} + +/** +@compilation_should_fail +@stderr type `null` can not be cast to `int` +@stderr t as int + */ diff --git a/tolk-tester/tests/invalid-typing/err-6498.tolk b/tolk-tester/tests/invalid-typing/err-6498.tolk new file mode 100644 index 0000000000..79da03b65d --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6498.tolk @@ -0,0 +1,14 @@ + +fun main() { + var a = 0 as int | slice; + var bb = match (a) { + }; + __expect_type(a, "int | slice"); + return a; +} + +/** +@compilation_should_fail +@stderr empty `match` can't be used as expression +@stderr var bb = match + */ diff --git a/tolk-tester/tests/invalid-typing/err-6502.tolk b/tolk-tester/tests/invalid-typing/err-6502.tolk new file mode 100644 index 0000000000..2f9e664d8e --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6502.tolk @@ -0,0 +1,11 @@ +fun cantAutoInferUnionVariant() { + var a: (int?, int?) | (int, int?) | int = 5; + a = (1, 2) as (int?, int?); // ok + a = (1 as int?, 2 as int?); // ok + a = (1, 2); +} + +/** +@compilation_should_fail +@stderr can not assign `(int, int)` to variable of type `(int?, int?) | (int, int?) | int` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6518.tolk b/tolk-tester/tests/invalid-typing/err-6518.tolk new file mode 100644 index 0000000000..3827677446 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6518.tolk @@ -0,0 +1,16 @@ +type MNull = null; +type IntOr16 = int | int16; + +fun main(t: int?) { + t as int? | null; + t as int? | slice; + t as (int, int) | MNull | int8 | IntOr16; + + t as slice?; +} + +/** +@compilation_should_fail +@stderr type `int?` can not be cast to `slice?` +@stderr t as slice? + */ diff --git a/tolk-tester/tests/invalid-typing/err-6583.tolk b/tolk-tester/tests/invalid-typing/err-6583.tolk new file mode 100644 index 0000000000..530ff95ccb --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6583.tolk @@ -0,0 +1,12 @@ +fun cantAssignIncompatibleUnionTypes(a: int | slice | builder) { + var b: int | slice = match (a) { + int => a, + slice => a, + builder => null, + }; +} + +/** +@compilation_should_fail +@stderr can not assign `int | slice | null` to variable of type `int | slice` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6659.tolk b/tolk-tester/tests/invalid-typing/err-6659.tolk new file mode 100644 index 0000000000..82fa462b6a --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6659.tolk @@ -0,0 +1,18 @@ +fun cantAutoInferUnionInTernary() { + var dd: int | slice = random() ? 1 : ""; // ok + var cc = (random() ? 1 : "") as int|slice; // ok + random() ? 1 as int|slice : "" as int|slice; // ok + random() ? 1 as int|slice : ""; // ok + random() ? 1 : "" as int|slice|builder; // ok + random() ? 1 as int|slice|null : "" as slice?; // ok + + var sub = random() ? (1, 2 as int|slice) : (1, ""); + __expect_type(sub, "(int, int | slice)"); + + random() ? 1 : ""; +} + +/** +@compilation_should_fail +@stderr types of ternary branches are incompatible: `int` and `slice` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6673.tolk b/tolk-tester/tests/invalid-typing/err-6673.tolk new file mode 100644 index 0000000000..59a6ba2833 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6673.tolk @@ -0,0 +1,14 @@ +fun cantAutoInferUnionVariant() { + var a: int8 | int16 = 0 as int8; // ok + a = 0 as int16; // ok + if (a is int8) { + a = 1 as int8; // ok + } + a = 2; +} + +/** +@compilation_should_fail +@stderr can not assign `int` to variable of type `int8 | int16` +@stderr a = 2; + */ diff --git a/tolk-tester/tests/invalid-typing/err-6690.tolk b/tolk-tester/tests/invalid-typing/err-6690.tolk new file mode 100644 index 0000000000..cfbcd8cf48 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6690.tolk @@ -0,0 +1,12 @@ +fun main(t: int?) { + t!; + t! as int; + + t as int; +} + +/** +@compilation_should_fail +@stderr type `int?` can not be cast to `int` +@stderr t as int; + */ diff --git a/tolk-tester/tests/invalid-typing/err-6712.tolk b/tolk-tester/tests/invalid-typing/err-6712.tolk new file mode 100644 index 0000000000..c0210a7bc9 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6712.tolk @@ -0,0 +1,12 @@ +fun matchDoesntCoverAllCases(a: int | slice?) { + match (a) { + int => {} + slice => {} + }; +} + +/** +@compilation_should_fail +@stderr `match` does not cover all possible types; missing types are: `null` +@stderr match (a) + */ diff --git a/tolk-tester/tests/invalid-typing/err-6767.tolk b/tolk-tester/tests/invalid-typing/err-6767.tolk new file mode 100644 index 0000000000..f9ddf7273c --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6767.tolk @@ -0,0 +1,10 @@ +type MInt = int; + +fun main() { + return (5 as int8?) as MInt | slice | null; +} + +/** +@compilation_should_fail +@stderr type `int8?` can not be cast to `MInt | slice | null` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6801.tolk b/tolk-tester/tests/invalid-typing/err-6801.tolk new file mode 100644 index 0000000000..4646bd7be6 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6801.tolk @@ -0,0 +1,9 @@ + +fun main(a: slice | null | int8) { + a as slice | null; +} + +/** +@compilation_should_fail +@stderr type `slice | null | int8` can not be cast to `slice?` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6808.tolk b/tolk-tester/tests/invalid-typing/err-6808.tolk new file mode 100644 index 0000000000..46db0d6de1 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6808.tolk @@ -0,0 +1,11 @@ +fun main() { + (1 as int16) as (int | int16); // ok, exact match + (1 as int8) as (int | int32); // ok, int accepts int8, but int32 not + + 1 as (int16 | int32); // error, ambiguous +} + +/** +@compilation_should_fail +@stderr type `int` can not be cast to `int16 | int32` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6829.tolk b/tolk-tester/tests/invalid-typing/err-6829.tolk new file mode 100644 index 0000000000..fbf4623674 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6829.tolk @@ -0,0 +1,16 @@ +type Pair2 = (int, int); +type Pair2Or3 = Pair2 | (int, int, int); + +fun matchDoesntCoverAllCases(a: Pair2Or3 | slice | [[int]] | builder) { + if (a !is slice) { + match (a) { + (int, int) => {} + [[int]] => {} + } + } +} + +/** +@compilation_should_fail +@stderr `match` does not cover all possible types; missing types are: `(int, int, int)`, `builder` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6853.tolk b/tolk-tester/tests/invalid-typing/err-6853.tolk new file mode 100644 index 0000000000..f901ad48fd --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6853.tolk @@ -0,0 +1,8 @@ +fun cantAssignIncompatiblePrimitiveNullable(m: int?) { + var a: slice? = m; +} + +/** +@compilation_should_fail +@stderr can not assign `int?` to variable of type `slice?` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6920.tolk b/tolk-tester/tests/invalid-typing/err-6920.tolk new file mode 100644 index 0000000000..dc2a5ffcec --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6920.tolk @@ -0,0 +1,12 @@ +fun matchOverNotSubtypeIsNever(m: slice | cell) { + match (m) { + int => return m + 0 + }; + return 3; +} + +/** +@compilation_should_fail +@stderr `int` is not a variant of `slice | cell` +@stderr int => + */ diff --git a/tolk-tester/tests/invalid-typing/err-6930.tolk b/tolk-tester/tests/invalid-typing/err-6930.tolk new file mode 100644 index 0000000000..80269c3dc1 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6930.tolk @@ -0,0 +1,15 @@ +type MInt = int; + +fun f1(v: int): int { return v; } + +fun main(t: int?) { + f1 as (MInt) -> MInt; + f1 as (int8) -> uint16; + + f1 as ((int | slice) -> int); +} + +/** +@compilation_should_fail +@stderr type `(int) -> int` can not be cast to `(int | slice) -> int` + */ diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index 9e3a269b5c..c1de8c0b7b 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -430,8 +430,8 @@ fun test136(x: int?, ) { @method_id(140) fun test140(x: int8, y: int16) { - var t1: (int32, int)? = (x, y) as (int32, int64)?; - var t2: (int32, int)? = null as (int32, int64)?; + var t1: (int32, int64)? = ((x, y) as (int32, int64)) as (int32, int64)?; + var t2: (int32, int64)? = null as (int32, int64)?; return (t1, t2); } @@ -456,66 +456,83 @@ fun test142() { return (sum1, sum2, t!); } +@method_id(143) +fun test143(setBNull: bool, setANullMid: bool) { + var a: (int, int | slice | null, int)? = (1, 2, 3); + var b = setBNull ? null : a; + if (setANullMid) { + a.1 = null; + } + if (a.1 != null && b != null) { + b.0 += match (a.1) { int => 100, slice => 200 }; + } + __expect_type(a, "(int, int | slice | null, int)"); + __expect_type(b, "(int, int | slice | null, int)?"); + return (a, 777, b); +} + fun main(){} /** -@testcase | 101 | | 1 2 -1 -@testcase | 102 | | 1 2 -1 (null) (null) 0 +@testcase | 101 | | 1 2 129 +@testcase | 102 | | 1 2 129 (null) (null) 0 @testcase | 103 | 1 2 | 3 3 0 1 2 @testcase | 104 | | 1 2 (null) (null) 0 -@testcase | 105 | | (null) (null) (null) 0 1 2 3 -1 +@testcase | 105 | | (null) (null) (null) 0 1 2 3 131 @testcase | 106 | | 1 2 @testcase | 107 | | 0 0 -1 0 0 -1 -@testcase | 108 | 5 6 | 7 8 10 11 -1 (null) (null) 0 +@testcase | 108 | 5 6 | 7 8 10 11 129 (null) (null) 0 @testcase | 109 | | 0 0 -1 0 -1 0 0 -1 -1 -@testcase | 110 | | 3 4 (null) (null) 0 6 7 -1 +@testcase | 110 | | 3 4 (null) (null) 0 6 7 129 @testcase | 111 | | 50 30 70 90 100 @testcase | 112 | | 12 22 @testcase | 113 | | -1 @testcase | 114 | | (null) (null) (null) 0 (null) (null) (null) 0 -@testcase | 115 | | 2 3 7 (null) (null) 0 5 0 -1 0 +@testcase | 115 | | 2 3 7 (null) (null) 0 5 0 129 0 @testcase | 116 | -1 | (null) (null) 0 (null) (null) 0 -@testcase | 116 | 0 | 1 2 -1 1 2 -1 +@testcase | 116 | 0 | 1 2 129 1 2 129 @testcase | 117 | | (null) 1 3 -@testcase | 118 | 5 | 5 10 -1 +@testcase | 118 | 5 | 5 10 129 @testcase | 118 | null | (null) (null) 0 -@testcase | 119 | | (null) (null) 1 2 -1 100 +@testcase | 119 | | (null) (null) 1 2 129 100 @testcase | 120 | -1 | (null) (null) 0 -@testcase | 120 | 0 | 1 2 -1 +@testcase | 120 | 0 | 1 2 129 @testcase | 121 | | [ 1 [ 3 4 ] ] @testcase | 122 | 0 | [ 1 [ 3 4 ] 4 (null) ] @testcase | 122 | -1 | [ 1 (null) 4 (null) ] -@testcase | 123 | | 1 3 4 -1 -@testcase | 124 | 0 | 1 3 4 -1 4 (null) (null) 0 +@testcase | 123 | | 1 3 4 132 +@testcase | 124 | 0 | 1 3 4 132 4 (null) (null) 0 @testcase | 124 | -1 | 1 (null) (null) 0 4 (null) (null) 0 @testcase | 125 | | 3 @testcase | 126 | | 1 (null) 2 @testcase | 127 | 1 | 1 (null) (null) 0 2 -@testcase | 127 | 2 | 1 2 3 -1 4 +@testcase | 127 | 2 | 1 2 3 129 4 @testcase | 127 | 3 | 1 (null) (null) 0 5 -@testcase | 128 | 1 | 1 (null) (null) 0 2 -1 +@testcase | 128 | 1 | 1 (null) (null) 0 2 139 @testcase | 128 | 2 | (null) (null) (null) (null) (null) 0 -@testcase | 128 | 3 | 1 2 3 -1 4 -1 +@testcase | 128 | 3 | 1 2 3 129 4 139 @testcase | 129 | 0 | 5 5 0 -1 1 2 0 -1 @testcase | 129 | -1 | 5 5 0 -1 (null) (null) 0 -1 -@testcase | 130 | 0 | 1 2 3 -1 +@testcase | 130 | 0 | 1 2 3 129 @testcase | 130 | -1 | 1 (null) (null) 0 -@testcase | 131 | | -1 777 0 777 777 777 0 0 -1 -1 777 -1 -1 -1 777 +@testcase | 131 | | 140 777 0 777 777 777 0 0 140 140 777 140 140 141 777 @testcase | 132 | | -1 0 -1 0 777 (null) (null) -1 0 0 @testcase | 133 | | 60 -@testcase | 134 | | 11 21 -1 -@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 -1 (null) -1 (null) 0 777 10 -1 (null) -1 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 +@testcase | 134 | | 11 21 129 +@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 146 (null) 146 (null) 0 777 10 143 (null) 143 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 @testcase | 136 | 9 | 9 0 @testcase | 136 | null | (null) -1 -@testcase | 140 | 8 9 | 8 9 -1 (null) (null) 0 +@testcase | 140 | 8 9 | 8 9 144 (null) (null) 0 @testcase | 141 | | (null) 10 @testcase | 142 | | 3 3 1 2 +@testcase | 143 | -1 0 | 1 2 1 3 777 (null) (null) (null) (null) 0 +@testcase | 143 | 0 -1 | 1 (null) 0 3 777 1 2 1 3 145 @fif_codegen """ isTensorNull PROC:<{ - // t.0 t.1 t.NNFlag - 2 1 BLKDROP2 // t.NNFlag + // t.0 t.1 t.UTag + 2 1 BLKDROP2 // t.UTag 0 EQINT // '3 }> """ diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 8132180267..ffedcb4413 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -645,6 +645,18 @@ fun test59() { __expect_type(x1, "int"); } +@method_id(160) +fun test60() { + var cc1 = match (val a: int | slice = 0 + 10 + 90) { + int => a + 1, + }; + var cc2 = match (var a = (0 + 10 + 90) as int | slice) { + int => (a = 108) + 1, + slice => 120, + }; + return (cc1, cc2); +} + fun main(x: int?): int { @@ -663,18 +675,28 @@ fun main(x: int?): int { @testcase | 139 | | 16 @testcase | 140 | 5 | 25 @testcase | 141 | | 1 2 -@testcase | 142 | | 5 3 (null) (null) 0 -1 3 (null) (null) 0 +@testcase | 142 | | 5 3 (null) (null) 0 132 3 (null) (null) 0 @testcase | 143 | | 10 11 (null) 10 11 (null) (null) 0 @testcase | 144 | | 10 11 (null) 10 11 (null) (null) 0 -@testcase | 145 | | 5 3 (null) (null) 0 -1 3 (null) (null) (null) (null) 0 3 (null) (null) 0 -@testcase | 146 | | 3 4 5 3 4 5 -1 +@testcase | 145 | | 5 3 (null) (null) 0 132 3 (null) (null) (null) (null) 0 3 (null) (null) 0 +@testcase | 146 | | 3 4 5 3 4 5 131 @testcase | 147 | | (null) (null) 100 (null) 100 (null) (null) 0 @testcase | 158 | | 123 10 123 5 +@testcase | 160 | | 101 109 -@stderr warning: expression of type `int` is always not null, this condition is always true +@stderr warning: expression of type `int` can never be `null`, this condition is always true @stderr warning: unreachable code @stderr var t2 redef = getNullableInt()!; @fif_codegen eq55 PROC:<{ @fif_codegen eq55 PROC:<{ + +@fif_codegen +""" + test60 PROC:<{ + // + 101 PUSHINT // cc1 + 109 PUSHINT // cc1 cc2 + }> +""" */ diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk index 54899a0fc2..aa67bcad73 100644 --- a/tolk-tester/tests/type-aliases-tests.tolk +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -27,7 +27,7 @@ fun test1(x: MInt): MVoid { __expect_type(random() ? (1, 2) : (1, 2) as Pair2_v1, "(int, int)"); __expect_type(random() ? (1, 2) : (1, 2) as Pair2_v2, "(int, int)"); __expect_type(random() ? (1, 2) as Pair2_v1 : (1, 2), "Pair2_v1"); - __expect_type(random() ? (1, 2) as Pair2_v1 : (1, 2) as Pair2_v2, "Pair2_v1"); + __expect_type(random() ? (1, 2) as Pair2_v1 : (1, 2) as Pair2_v2, "(int, int)"); __expect_type(random() ? (1, 2) as Pair2_v2 : (1, 2) as Pair2_v2, "Pair2_v2"); __expect_type(!x, "bool"); @@ -40,8 +40,8 @@ fun test1(x: MInt): MVoid { } fun test2() { - __expect_type(test1, "(MInt) -> MVoid"); - __expect_type(test1(1), "MVoid"); + __expect_type(test1, "(MInt) -> void"); + __expect_type(test1(1), "void"); } fun test3(x: MIntN, y: MInt?) { @@ -51,7 +51,7 @@ fun test3(x: MIntN, y: MInt?) { if (x != null) { __expect_type(x, "MInt"); } - __expect_type(x, "MInt?"); // merge null + MInt = MInt?, not MIntN as original + __expect_type(x, "MInt?"); var (z1, z2) = (x, x!, ); __expect_type(z1, "MInt?"); __expect_type(z2, "MInt"); @@ -64,10 +64,11 @@ fun test4(x: MIntN, y: MIntN) { __expect_type(x + y, "int"); return x + y; } + __expect_type(x, "MInt?"); __expect_type(x!, "MInt"); __expect_type(random() ? x : y, "MInt?"); __expect_type(random() ? x : 0, "MInt?"); - __expect_type(random() ? 0 : x, "MInt?"); + __expect_type(random() ? 0 : x, "int?"); __expect_type(random() ? 0 : x!, "int"); return y!; } diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk new file mode 100644 index 0000000000..04446f8e73 --- /dev/null +++ b/tolk-tester/tests/union-types-tests.tolk @@ -0,0 +1,873 @@ +type MInt = int; +type MNull = null; +type MSlice = slice; +type MIntN = int | MNull; +type IntOrSlice = MInt | slice; +type IntOrSliceN = null | slice | int; +type IntOrSliceOrBuilder = int | slice | builder; + +type Pair2 = (int, int); +type Pair2Or3 = Pair2 | (int, int, int); +type SomeIntBits = int8 | int9 | int10 | int32 | uint64; + +fun getSlice32(): slice { return beginCell().storeInt(32, 32).endCell().beginParse(); } +fun getIntOrSlice(): int | slice { return 5; } + +fun testFlatten() { + __expect_type(0 as (int | int), "int"); + __expect_type(0 as (int | int | slice | builder | slice), "int | slice | builder"); + __expect_type(0 as (int? | int), "int?"); + __expect_type(0 as (slice | int? | slice? | cell), "slice | int | null | cell"); + __expect_type(0 as (int | (int | int)), "int"); + __expect_type(0 as (slice | (slice | (slice | int) | (cell | slice?))), "slice | int | cell | null"); + + __expect_type(0 as (MInt | MNull), "MInt?"); + __expect_type(0 as (MInt | slice | int), "MInt | slice"); + __expect_type(0 as (MIntN), "MIntN"); + __expect_type(0 as (MIntN | null), "int?"); + __expect_type(0 as (MIntN | MNull | MSlice), "int | null | MSlice"); + __expect_type(0 as (MIntN | MNull? | MSlice), "int | null | MSlice"); + __expect_type(0 as (MInt | IntOrSliceOrBuilder | MSlice), "MInt | slice | builder"); + __expect_type((1, 2) as (Pair2 | Pair2Or3 | (slice, int?)), "Pair2 | (int, int, int) | (slice, int?)"); + + __expect_type(0 as (int | int8), "int | int8"); + __expect_type(0 as (MInt | int8), "MInt | int8"); + __expect_type((0 as MInt) as (int | int8), "int | int8"); + __expect_type((0 as MInt) as (MInt | int8), "MInt | int8"); + + __expect_type(0 as IntOrSlice, "IntOrSlice"); + __expect_type(0 as IntOrSlice?, "MInt | slice | null"); + __expect_type(0 as IntOrSliceN, "IntOrSliceN"); + __expect_type(0 as IntOrSliceN?, "null | slice | int"); + + __expect_type(0 as (int?)?, "int?"); + __expect_type(0 as (int?)? | (int)?, "int?"); + + __expect_type(0 is builder, "bool"); + __expect_type(0 !is builder, "bool"); + __expect_type(0!!is builder, "bool"); + + __expect_type(match (0) { int => [] }, "[]"); + __expect_type(match (0 as int|slice) { int => 0, slice => 0 }, "int"); + __expect_type(match (0 as int|slice) { int => 0, slice => 0 as int8 } as int|int8, "int | int8"); + __expect_type(match (0 as int?) { int => "", null => null }, "slice?"); + __expect_type(match (0 as int?) { + int => 10>3 ? match(0) { int => 0 } : match (10>3 ? 6 : null) { null => 0, int => { return; } }, + null => match(null) { null => 0 } + }, "int"); +} + +@method_id(101) +fun test1() { + var a = getIntOrSlice(); + var b: MInt | MSlice = getIntOrSlice(); + var c: IntOrSlice = getIntOrSlice(); + __expect_type(a, "int | slice"); + __expect_type(b, "MInt | MSlice"); + __expect_type(c, "IntOrSlice"); + return (a, b, c); +} + +@method_id(102) +fun test2() { + var a: int | slice = 4; + __expect_type(a, "int"); + a = getIntOrSlice(); + __expect_type(a, "int | slice"); + a = getSlice32(); + __expect_type(a, "slice"); + return a.loadInt(32); +} + +@method_id(103) +fun test3(case: int): int | slice | null { + var a: int | slice | null = 4; + if (case == 0) { + a = getSlice32(); + } else if (case == 1) { + a = 10; + } else { + a = null; + } + return a; +} + +@method_id(104) +fun test4() { + return ( + 5 as int | int, + (5 as int8) as int8 | int8, + true as bool | bool, + 5 as int | int | (int | slice), + (5 as int | int | (int | slice)) as int | slice | null + ); +} + +@method_id(105) +fun test5() { + var a: int | int | IntOrSlice = getIntOrSlice(); + __expect_type(a, "int | slice"); + a = 5; + return (a, a as IntOrSlice, a as IntOrSliceOrBuilder, a as MIntN); +} + +@method_id(106) +fun test6(case: int): int | slice | builder | null { + var a: int? | slice? | builder = null; + __expect_type(a, "null"); + if (case == 1) { + a = null as slice? | builder; + __expect_type(a, "null | slice | builder"); + __expect_type(a!, "slice | builder"); + return a; + } + if (case == 2) { + __expect_type(a = 2 as MInt, "MInt"); + __expect_type(a, "int"); + a = 2; + __expect_type(a, "int"); + return a + 0; + } + if (case == 3) { + a = null as slice?; + __expect_type(a, "slice?"); + return a; + } + a = null as MNull; + __expect_type(a, "null"); + return a; +} + +@method_id(107) +fun test7(case: int) { + var a = (2, 3); + var b: Pair2 = (4, 5); + var c: Pair2Or3 = case == 1 ? a : case == 2 ? b : (6, 7, 8); + __expect_type(c, "Pair2Or3"); + c as Pair2Or3; + c as Pair2Or3 | Pair2Or3; + c as (int, int, int) | Pair2; + c as (int, int) | (int, int, int); + return (c, c as (int, int)? | (int, int, int) | null); +} + +@method_id(108) +fun test8() { + return ( + (2, 3) as (int, int) | (int, int, int) | null, + null as (int, int) | null | (int, int, int), + (2, 3) as Pair2Or3 | null, + (2, 3) as null | Pair2Or3, + null as Pair2Or3? + ); +} + +@method_id(109) +fun test9() { + var c: null | Pair2Or3 = (6, 7, 8) as Pair2Or3?; + __expect_type(c, "null | Pair2 | (int, int, int)"); + __expect_type(c!, "Pair2 | (int, int, int)"); + c! as Pair2Or3; + c! as (int, int, int) | Pair2; + (c! as (int, int) | (int, int, int)) as Pair2Or3; + return (c, c == null, c = null, c == null, c = null as Pair2Or3?, c == null, c as Pair2? | (int, int, int)? | null); +} + +@method_id(110) +fun test10() { + var t = getSlice32() as (int, (int, int)?, (int?, int)?) | slice; + if (10 > 3) { + t = (1, null, null) as (int, (int, int)?, (int?, int)?); + } + return t; +} + +@method_id(120) +fun test20() { + var a = getIntOrSlice(); + if (a is int) { + __expect_type(a, "int"); + return a + 0; + } + __expect_type(a, "slice"); + return a.loadInt(32); +} + +@method_id(121) +fun test21() { + var a = getIntOrSlice(); + if (a is int) { + if (a !is int8) { + __expect_type(a, "int"); + } + if (a is slice) { + __expect_type(a, "never"); + } + if (a !is slice) { + __expect_type(a, "int"); + } + if (a !is builder) { + __expect_type(a, "int"); + } + } + if (a !is int && a !is builder && a !is int && a !is null) { + __expect_type(a, "slice"); + } + if (a !is int || (a !is int && a !is builder)) { + __expect_type(a, "slice"); + } + if (a is int || a is slice) { + __expect_type(a, "int | slice"); + } else { + __expect_type(a, "never"); + } + if (a is slice || a is int) { + __expect_type(a, "slice | int"); + } else { + __expect_type(a, "never"); + } + if (a is int || a is slice || a is builder) { + __expect_type(a, "int | slice"); + } + if (a !is int && a !is slice && a !is builder) { + __expect_type(a, "never"); + } + return (a is int, a !is int, a is slice, a !is slice, a is int || a is slice || a is builder, a is builder); +} + +@method_id(122) +fun test22(a: int, b: int | slice) { + var result = 0; + if (a is int) { result += 1; } + if (a is int || a is slice) { result += 2; } + if (a is builder || a is slice) { result += 3; } + if (b is int) { result += 10; } + if ((b is int) | (b is slice)) { result += 11; } + if ((b is builder) | (b is int)) { result += 12; } + return result; +} + +@method_id(123) +fun test23(p2: bool) { + var p: Pair2Or3 = p2 ? (1, 2) : (3, 4, 5); + __expect_type(p, "Pair2Or3"); + if (p is (int, int) && 10 < 0) { + __expect_type(p, "(int, int)"); + } + __expect_type(p, "(int, int) | (int, int, int)"); + if (p is int) { + __expect_type(p, "never"); + return (0, 0); + } + __expect_type(p, "(int, int) | (int, int, int)"); + if (p !is (int, int)) { + __expect_type(p, "(int, int, int)"); + p = (10, 20); + } + __expect_type(p, "Pair2"); + return p; +} + +fun eqUnion2(v: T1 | T2): T2 | T1 { return v; } + +fun detectUnionSide(v: T1 | T2) { + if (v is T1) { + assert(v is T1 && v !is T2 && v !is (T1, T2), 101); + return 0; + } + assert(v is T2 && v !is T1 && !(v is null || v is never), 102); + return 1; +} + +@method_id(124) +fun test24() { + var bc = beginCell() as builder | cell; + var mi1 = 5 as MInt | int8; + var mi2 = 5 as int8 | MInt; + var mi3 = (5 as int8) as int8 | MInt; + __expect_type(eqUnion2(bc), "cell | builder"); + __expect_type(eqUnion2(mi1), "int8 | MInt"); + return ( + detectUnionSide(5), + detectUnionSide(5), + detectUnionSide(getIntOrSlice()), + detectUnionSide((getIntOrSlice() is int) ? getSlice32() : beginCell()), + detectUnionSide(bc), + detectUnionSide(mi1), + detectUnionSide(mi2), + detectUnionSide(mi3) + ); +} + +@method_id(125) +fun test25(a: int? | slice) { + var result = 0; + if (a is null) { result |= 0x0001; } + if (a is int || a == null) { result |= 0x0002; } + if ((a is null) | a is slice) { result |= 0x0004; } + if (a is int) { result |= 0x0008; } + if ((a is builder) | a is null) { result |= 0x0010; } + if (a is int || a is builder) { result |= 0x0020; } + if (a is int8 || a is null || a is int16) { result |= 0x0040; } + return result; +} + +@method_id(126) +fun test26() { + var a = 1 as int? | slice; + var result = 0; + if (a is null) { result |= 0x0001; } + if (a is int || a == null) { result |= 0x0002; } + if ((a is null) | a is slice) { result |= 0x0004; } + if (a is int) { result |= 0x0008; } + if ((a is builder) | a == null) { result |= 0x0010; } + if ((a is int) | a is builder) { result |= 0x0020; } + if ((a is int8) | (a is int16) | (a is null)) { result |= 0x0040; } + return result; +} + +@method_id(127) +fun test27() { + var a: (MInt, int) | slice = (1, 2); + __expect_type(a, "(MInt, int)"); + a = (1 as MInt, 2 as MInt); + __expect_type(a, "(MInt, int)"); + var b: IntOrSliceOrBuilder = 5 as (MInt | slice); + return (a, b); +} + +@method_id(128) +fun test28() { + var t = null as int | (int, int)?; + return (t, 777, 2 as int | (int, int)?, 777, (3, 4) as int | (int, int)?); +} + +@method_id(129) +fun test29() { + var t1 = (null as int?) as (int, int) | int | null; + var t2 = (5 as int?) as (int, int) | int | null; + return (t1, t2); +} + +fun get5OrNull(getNull: bool) { return getNull ? null : 5; } + +@method_id(130) +fun test30(getNull: bool) { + var t1 = get5OrNull(getNull) as (int, int) | int | null; + var t2 = (null as (int, int)?) as (int, int) | int | null; + var t3 = ((1, 2) as (int, int)?) as (int, int) | int | null; + var t4 = ([6] as [int]?) as (int, int) | [int] | null; + var t5 = (null as [int]?) as (int, int) | [int] | null; + return (t1, 777, t2, 777, t3, 777, t4, 777, t5); +} + +@method_id(131) +fun test31() { + // all of them are smart casted (narrowed from a wide union type to a subtype) + var t1: (int, int) | int | null = null; + var t2: (int, int) | int | null = 5; + var t3: (int, int) | int | null = (1, 2); + var t4: (int, int) | int | null = null as int?; + var t5: (int, int) | int | null = 5 as int?; + var t6: (int, int) | int | null = null as (int, int)?; + var t7: (int, int) | int | null = (1, 2) as (int, int)?; + __expect_type(t1, "null"); + __expect_type(t2, "int"); + __expect_type(t3, "(int, int)"); + __expect_type(t4, "int?"); + __expect_type(t6, "(int, int)?"); + return (t1, 777, t2, 777, t3, 777, t4, 777, t5, 777, t6, 777, t7); +} + +@method_id(132) +fun test32() { + var t1 = 5 as (int, slice?, int) | (int, int) | int | int8 | null; + var t2 = (5 as int8) as (int, slice?, int) | (int, int) | int | int8 | null; + var t3 = (4, 5) as (int, slice?, int) | (int, int) | int | int8 | null; + var t4 = null as (int, slice?, int) | (int, int) | int | int8 | null; + return (t1, 777, t2, 777, t3, 777, t4); +} + +@method_id(133) +fun test33() { + var t1 = (5 as (int, MInt) | int) as (int, slice?, int) | (int, int) | MInt | int8 | null; + var t2 = ((5 as int8) as null | int8 | int) as (int, slice?, int) | (MInt, int) | int | int8 | null; + var t3 = ((4, 5) as (int, MInt) | MInt) as (int, slice?, int) | (MInt, int) | int | int8 | null; + var t4 = (null as MInt | int8 | null) as (int, slice?, int) | (int, MInt) | int | int8 | null; + return (t1, 777, t2, 777, t3, 777, t4); +} + +@method_id(134) +fun test34() { + // all of them are smart casted (narrowed from a wide union type to a subtype) + var t1: (int, cell, slice?, int) | (int, int) | int | int8 | null = 5 as (int, int) | int; + var t2: (int, cell, slice?, int) | (int, MInt) | int | int8 | null = 5 as (int, int) | MInt | int8; + var t3: (int, cell, slice?, int) | (int, int) | MInt | int8 | null = (1, 2) as (int, int) | MInt; + var t4: (int, cell, slice?, int) | (MInt, MInt) | int | int8 | null = (1, 2) as (int, int) | int | null; + var t5: (int, cell, slice?, int) | (int, int) | int | int8 | null = (5 as int8) as (int, MInt) | int | int8; + var t6: (int, cell, slice?, int) | (MInt, int) | MInt | int8 | null = (5 as int8) as int8 | int; + var t7: (int, cell, slice?, int) | (MInt, MInt) | int | int8 | null = (5 as int8) as int8 | MInt | null; + __expect_type(t1, "(int, int) | int"); + __expect_type(t7, "int | int8 | null"); + return (t1, 777, t2, 777, t3, 777, t4, 777, t5, 777, t6, 777, t7); +} + +@method_id(135) +fun test35() { + var t1 = 5 as int | (int, MInt) | null; + var t2 = (1, 2) as int | (int, int) | (int, MInt, slice, slice); + var t3 = (1, 2) as int | null | (MInt, int) | (MInt, int, slice, slice); + var t4 = null as int? | (MInt, MInt)?; + __expect_type(t1!, "int | (int, MInt)"); + __expect_type(t2!, "int | (int, int) | (int, MInt, slice, slice)"); + __expect_type(t3!, "int | (MInt, int) | (MInt, int, slice, slice)"); + __expect_type(t4!, "int | (MInt, MInt)"); + return (t1!, 777, t2!, 777, t3!, 777, t4!); +} + +@method_id(136) +fun test36() { + var t1 = (1 as int8, 2 as int16) as (int, int) | slice; + var t2 = (1 as int8, 2 as int16) as (int, MInt) | (int8, int16) | slice; + var t3 = (1 as int8, 2 as int16) as (int?, MInt?) | builder | null; + var t4 = (1 as int8, 2 as int16) as (int8?, int16) | (int, int, int) | (cell, cell, cell); + return (t1, 777, t2, 777, t3, 777, t4); +} + +@method_id(137) +fun test37(one: int8, two: int16) { + var t1: (int, MInt) | slice = (one, two); + var t2: (int, int) | (int8, int16) | slice = (one, two); + var t3: (int?, int?) | builder | null = (one, two); + var t4: [int8?, int16] | (int, int, int) | (cell, cell, cell) = [one, two]; + __expect_type(t1, "(int, MInt)"); + __expect_type(t2, "(int8, int16)"); + __expect_type(t3, "(int?, int?)"); + __expect_type(t4, "[int8?, int16]"); + return (t1, 777, t2, 777, t3, 777, t4); +} + +@method_id(138) +fun test38() { + var t1 = (1, null, 2) as (int, (int, MInt, int, int)?, int) | slice; + var t2 = (1, null, 2) as (int, MInt | slice | null, int) | slice; + var t3 = (1, 5, 2) as (int, int | slice | null, int) | slice; + var t4 = (1, 5, 2) as (int, int | slice | (int, int), int) | slice; + return (t1, 777, t2, 777, t3, 777, t4); +} + +@method_id(139) +fun test39() { + var t1: (int, (MInt, int, int, MInt)?, int) | slice = (1, null, 2); + var t2: (int, int | slice | null, int) | slice = (1, null, 2); + var t3: (int, MInt | slice | null, int) | slice = (1, 5, 2); + var t4: (int, int | slice | (int, int), int) | slice = (1, 5, 2); + __expect_type(t1, "(int, (MInt, int, int, MInt)?, int)"); + __expect_type(t3, "(int, MInt | slice | null, int)"); + return (t1, 777, t2, 777, t3, 777, t4); +} + +@method_id(140) +fun test40(x: int): SomeIntBits { + if (x < 10) { return x as int8; } + if (x < 20) { return x as int9; } + if (x < 30) { return x as int10; } + return x as uint64; +} + +@method_id(141) +fun test41(x: int?, y: MInt | MNull) { + var t1 = x as int | MNull; + var t2 = x as int | slice | MNull; + var t3 = y as int?; + var t4 = y as int? | slice; + return (t1, t2, t3, t4); +} + +@method_id(142) +fun test42(x: int?, y: MInt | MNull) { + var t1: int | MNull = x; + var t2: int | slice | MNull = x; + var t3: int? = y; + var t4: int? | slice = y; + __expect_type(t1, "int?"); + __expect_type(t2, "int?"); + __expect_type(t3, "int?"); + __expect_type(t4, "int?"); + return (t1, t2, t3, t4); +} + +@method_id(143) +fun test43() { + var a: MNull = null; + __expect_type(a, "null"); + return (a as null, a as int?, a as MIntN, a as (int, int)?, a as (int, int) | slice | MNull); +} + +@method_id(144) +fun test44(a: int) { + return (a is slice, a is int, a is null, a != null, a is builder, a !is (int, int), a is int8); +} + +@method_id(145) +fun test45(a: slice?) { + return (a is slice, a is null, a != null, a is builder, a !is (int, int), a is int); +} + +@method_id(146) +fun test46(a: (int, int) | int) { + return (a is slice, a is null, a != null, a is builder, a !is (int, int), a is int && a == 0); +} + +fun checkSimple(a: int | slice | builder) { + return match (a) { + int => 1, + slice => 2, + builder => 3, + }; +} + +@method_id(147) +fun test47() { + __expect_type(checkSimple, "(int | slice | builder) -> int"); + return (checkSimple(1), checkSimple(beginCell().endCell().beginParse()), checkSimple(100), checkSimple(beginCell())); +} + +fun checkGeneric3(a: T1 | T2 | T3): (int, T1 | T2?) { + var vv: T1? = match (a) { T1 => a, T2 => null, T3 => null }; + return match (a) { + T1 => (1, a), + T2 => (2, match(a) { T2 => a }), + T3 => (3, null), + }; +} + +type IntOrInt8OrInt16 = int | int8 | int16; + +@method_id(148) +fun test48(a: IntOrInt8OrInt16) { + var b = a; + __expect_type(b, "IntOrInt8OrInt16"); + match (b) { MInt => {} int8 => {} int16 => {} } + __expect_type(b, "int | int8 | int16"); + match (b) { int => {}, int8 => { return ((0, null), (0, null)); }, int16 => "" }; + __expect_type(b, "int | int16"); + return (checkGeneric3(a), checkGeneric3(5 as int | slice | builder)); +} + +@method_id(149) +fun test49(a: slice? | int) { + if (a !is int) { + return match (a) { + slice => 1, + null => 2, + }; + } + return a + 0; +} + +type VeryComplexType = int | (MInt, int)? | builder? | int; + +@method_id(150) +fun test50() { + var a = 5 as int | slice | null; + var b = 5 as VeryComplexType; + test49(a); + test49(match (a) { int => a, slice => a, null => a, }); + __expect_type(b, "VeryComplexType"); + match (b) { int => {} Pair2 => {} builder => {} null => {} } + __expect_type(b, "int | (int, int) | builder | null"); + __expect_type(b!, "int | (int, int) | builder"); + match (b) { int => 100, Pair2 => {} builder => return -2, null => "null" } + __expect_type(b, "int | (int, int) | null"); + if (b !is null) { + match (b) { int => {}, Pair2 => {} } + } else { + match (b) { null => {} } + } + match (b is null) { + bool => {} + } + match (b = 5 as VeryComplexType) { int => b, (int, int) => { return -1; } null => {} builder => return -2 } + __expect_type(b, "int?"); + b is int && b!is int && b!!is int; + if (match(b) { int => 1, null => null } !is int) { + return 100; + } + return (b! + (match (a) { int => a, slice => a, null => a } as int|slice?).test49()); +} + +@method_id(151) +fun test51(init: int) { + var a = init as int | slice; + match(a){} + __expect_type(a, "int | slice"); + match (a) { + int => { + a = init; + match (a) {}; + __expect_type(a, "int"); + a = a > 10 ? beginCell().storeInt(init, 32).endCell().beginParse() : 10; + __expect_type(a, "int | slice"); + if (a is slice) { + a = a.loadInt(32); + } + } + slice => { + if (a !is int) { + a = a.loadInt(32) as int8; + } + } + } + __expect_type(a, "int"); + return a; +} + +@method_id(152) +fun test52(init: int): int | builder { + var m = get5OrNull(false); + if (init > 10 && init < 15) { + return init > 20 ? init : beginCell(); + } + return match (init as int?) { + int => init, + null => beginCell() + }; +} + +fun inc53(mutate x: int) { + x += 1; + return x; +} + +@method_id(153) +fun test53(a: int | builder) { + match (var a = 5 as int | slice) { + int => {} + slice => { __expect_type(a, "slice"); var b: slice? = a; return b.loadInt(32); } + } + match (val a = getIntOrSlice()) { + int => a + 0, + slice => {} + } + __expect_type(a, "int | builder"); + var cc = match (var a = getIntOrSlice()) { + int => inc53(mutate a), + slice => { return 777; } + }; + __expect_type(cc, "int"); + match (var (a, (b, c)) = (10, (20, 30))) { + (int, Pair2) => {} + } + return cc; +} + +@method_id(154) +fun test54(): int|slice { + match (var a = getIntOrSlice()) { + int => { + __expect_type(a, "int"); + a = beginCell().storeInt(100, 32).endCell().beginParse(); + __expect_type(a, "slice"); + if (a.getRemainingBitsCount() < 100) { a = a.loadInt(32); } + else { a = 0; } + __expect_type(a, "int"); + return a; + } + slice => throw 123 + } + return 777; +} + +@method_id(155) +fun test55() { + var a = getIntOrSlice() as int | slice | null; + var cc = match (a) { + int => a = getIntOrSlice(), + slice => return (-10, 0), + null => throw 123 + }; + __expect_type(a, "int | slice"); + return (a, cc); +} + +@method_id(156) +fun test56(a: int8, b: int16, useSecond: bool) { + var r1: int8 | int16 = useSecond ? b: a; + var r2: int = useSecond ? b: a; + var r3: int8 | (int, int) | int16? = useSecond ? b : a; + var r4 = (useSecond ? b : a) as int; + var r5 = (useSecond ? b : a) as int8? | int16; + var r6: int8 | int16 = match (val c: int16 | int8 | int8 = useSecond ? b : a) { int8 => c, int16 => c }; + var r7 = match (val c: int16 | int8 = useSecond ? b : a) { int8 => c, int16 => c } as int; + var r8 = match (val c: int16 | int8 = useSecond ? b : a) { int8 => c as int, int16 => c as int }; + var r9: int = match (val c = (useSecond ? b : a) as IntOrInt8OrInt16) { int8 => c, int16 => c, int => c }; + return (r1, r2, r3, r4, r5, r6, r7, r8, r9); +} + +@method_id(157) +fun test57(r3v: int) { + var r1 = null as int | () | null; + var r2 = () as int | () | null; + var r3 = () as (int, int) | () | null; + var r4 = null as (int, int) | () | null; + if (r3v == 1) { + r3 = (1, 2); + } + return (r1, r2, 777, r3, 777, r4 = r3v == 1 ? () : null, r4, 777, r1 is int, r1 is (), r1 is null, r3 is int, r3 is (int,int), r3 is (), r3 == null, 777, r4 is (), r4 is null); +} + +@method_id(158) +fun test58() { + var tn = () as ()?; + var aln: null = (tn = null); + __expect_type(tn, "null"); + return ((()) as () | (int, int) | null, (() as () | null) as () | (int, int) | null, tn, tn as ()?, aln); +} + +fun alwaysThrows123(): never { throw 123; } + +@method_id(159) +fun test59(a: int | slice | builder | null) { + try { + var b = 0; + var c = match (a) { + int => b + a, + slice => { + b = 10; + throw 456; + } + builder => alwaysThrows123(), + null => { + alwaysThrows123(); + } + }; + __expect_type(c, "int"); + return b + c; + } catch (excCode) { + return excCode; + } +} + +fun takeIntReturnVoid(a: int) { } + +fun returnVoidOrNull(willBeVoid: bool) { + var (i: int?, _: builder) = (willBeVoid ? 10 : null, beginCell()); + + var res = i != null ? takeIntReturnVoid(i) : null; + __expect_type(res, "void?"); + return res; +} + +@method_id(160) +fun test60() { + return (returnVoidOrNull(true), returnVoidOrNull(false)); +} + + + +fun main() { + return 0; +} + +/** +@testcase | 0 | | 0 +@testcase | 101 | | 5 1 5 1 5 1 +@testcase | 102 | | 32 +@testcase | 103 | 1 | 10 1 +@testcase | 103 | 2 | (null) 0 +@testcase | 104 | | 5 5 -1 5 1 5 1 +@testcase | 105 | | 5 5 1 5 1 5 +@testcase | 106 | 1 | (null) 0 +@testcase | 106 | 2 | 2 1 +@testcase | 107 | 1 | (null) 2 3 131 (null) 2 3 131 +@testcase | 107 | 3 | 6 7 8 132 6 7 8 132 +@testcase | 108 | | (null) 2 3 131 (null) (null) (null) 0 (null) 2 3 131 (null) 2 3 131 (null) (null) (null) 0 +@testcase | 109 | | 6 7 8 132 0 (null) -1 (null) (null) (null) 0 -1 (null) (null) (null) 0 +@testcase | 110 | | 1 (null) (null) 0 (null) (null) 0 134 +@testcase | 120 | | 5 +@testcase | 121 | | -1 0 0 -1 -1 0 +@testcase | 122 | 0 0 1 | 36 +@testcase | 122 | 0 0 4 | 14 +@testcase | 123 | 1 | 1 2 +@testcase | 123 | 0 | 10 20 +@testcase | 124 | | 0 1 0 1 0 0 1 0 +@testcase | 125 | null 0 | 87 +@testcase | 125 | 1 1 | 42 +@testcase | 126 | | 42 +@testcase | 127 | | 1 2 5 1 +@testcase | 128 | | (null) (null) 0 777 (null) 2 1 777 3 4 131 +@testcase | 129 | | (null) (null) 0 (null) 5 1 +@testcase | 130 | 0 | (null) 5 1 777 (null) (null) 0 777 1 2 131 777 (null) [ 6 ] 135 777 (null) (null) 0 +@testcase | 130 | -1 | (null) (null) 0 777 (null) (null) 0 777 1 2 131 777 (null) [ 6 ] 135 777 (null) (null) 0 +@testcase | 131 | | (null) 777 5 777 1 2 777 (null) 777 5 777 (null) (null) 0 777 1 2 131 +@testcase | 132 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 131 777 (null) (null) (null) 0 +@testcase | 133 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 131 777 (null) (null) (null) 0 +@testcase | 134 | | (null) 5 1 777 (null) 5 1 777 1 2 131 777 1 2 131 777 (null) 5 42 777 5 42 777 5 42 +@testcase | 135 | | (null) 5 1 777 (null) (null) 1 2 131 777 (null) (null) 1 2 131 777 (null) (null) 0 +@testcase | 136 | | 1 2 131 777 1 2 140 777 1 2 141 777 (null) 1 2 138 +@testcase | 137 | 1 2 | 1 2 777 1 2 777 1 2 777 [ 1 2 ] +@testcase | 138 | | 1 (null) (null) (null) (null) 0 2 149 777 1 (null) 0 2 143 777 1 5 1 2 143 777 1 (null) 5 1 2 144 +@testcase | 139 | | 1 (null) (null) (null) (null) 0 2 777 1 (null) 0 2 777 1 5 1 2 777 1 (null) 5 1 2 +@testcase | 140 | 5 | 5 42 +@testcase | 140 | 15 | 15 129 +@testcase | 140 | 90 | 90 49 +@testcase | 141 | 7 8 | 7 7 1 8 8 1 +@testcase | 141 | null null | (null) (null) 0 (null) (null) 0 +@testcase | 142 | 7 8 | 7 7 8 8 +@testcase | 142 | null null | (null) (null) (null) (null) +@testcase | 143 | | (null) (null) (null) (null) (null) 0 (null) (null) 0 +@testcase | 144 | 0 | 0 -1 0 -1 0 -1 0 +@testcase | 145 | null | 0 -1 0 0 -1 0 +@testcase | 146 | null 0 1 | 0 0 -1 0 -1 -1 +@testcase | 147 | | 1 2 1 3 +@testcase | 148 | 9 1 | 1 9 1 1 5 1 +@testcase | 148 | 9 44 | 3 (null) 0 1 5 1 +@testcase | 149 | null 0 | 2 +@testcase | 150 | | 10 +@testcase | 151 | 88 | 88 +@testcase | 152 | 52 | 52 1 +@testcase | 153 | 1 1 | 6 +@testcase | 154 | | 100 1 +@testcase | 155 | | 5 1 5 1 +@testcase | 156 | 1 2 -1 | 2 44 2 2 44 2 2 44 2 44 2 2 2 +@testcase | 157 | 1 | (null) 0 (null) 145 777 1 2 131 777 145 145 777 0 0 -1 0 -1 0 0 777 -1 0 +@testcase | 157 | 0 | (null) 0 (null) 145 777 (null) (null) 145 777 0 0 777 0 0 -1 0 0 -1 0 777 0 -1 +@testcase | 158 | | (null) (null) 145 (null) (null) 145 (null) 0 (null) +@testcase | 159 | 0 4 | 456 +@testcase | 159 | null 0 | 123 +@testcase | 160 | | 10 0 + + +@fif_codegen +""" + test26 PROC:<{ + // + 42 PUSHINT // result + }> +""" + +@fif_codegen +""" + checkSimple PROC:<{ + // a.USlot1 a.UTag + NIP // a.UTag + DUP // a.UTag a.UTag + 1 EQINT // a.UTag '3 + IF:<{ // a.UTag + DROP // + 1 PUSHINT // '2=1 + }>ELSE<{ // a.UTag + 4 EQINT // '6 + IF:<{ // + 2 PUSHINT // '2=2 + }>ELSE<{ // + 3 PUSHINT // '2=3 + }> + }> + }> +""" + +@fif_codegen DECLPROC checkGeneric3 + + */ diff --git a/tolk-tester/tests/unreachable-5.tolk b/tolk-tester/tests/unreachable-5.tolk new file mode 100644 index 0000000000..9371ee1275 --- /dev/null +++ b/tolk-tester/tests/unreachable-5.tolk @@ -0,0 +1,24 @@ +fun main(x: int | slice | null) { + if (x is builder) { + __expect_type(x, "never"); + return 10; + } + if (x !is cell) { + __expect_type(x, "int | slice | null"); + } + if (x is int) { + return x; + } + return (x is null) ? -1 : 20; +} + +/** +@testcase | 0 | 0 1 | 0 +@testcase | 0 | null 0 | -1 +@testcase | 0 | -1 2 | 20 + +@stderr warning: variable `x` of type `int | slice | null` can never be `builder` +@stderr if (x is builder) +@stderr warning: variable `x` of type `int | slice | null` can never be `cell` +@stderr if (x !is cell) + */ diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index 76fdab21f0..11241bf13b 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -202,7 +202,7 @@ fun testApplyGlobalVar(x: int, y: int, z: int): bool? { return check_assoc_2(x, y, z); } -fun justAdd2(x: MInt): int64 { return x + 2; } +fun justAdd2(x: MInt): int { return x + 2; } @method_id(113) fun testCallbackAlias(secondNull: bool) { @@ -211,6 +211,17 @@ fun testCallbackAlias(secondNull: bool) { return (adder1(10), adder2 == null ? -1 : adder2(10)); } +fun justGetTensor2() { return (2, 3); } + +@method_id(114) +fun testCallbackUnion(call1st: bool): (int, int) | int { + var cb: (int -> int) | (() -> (MInt, int)) = call1st ? justGetTensor2 : justAdd2; + if (cb is () -> (int, int)) { + return cb(); + } + return cb(10) as int; +} + fun main() {} /** @@ -223,7 +234,7 @@ fun main() {} @testcase | 107 | | 65537 @testcase | 108 | | 4 @testcase | 109 | | 0 3 0 7 -@testcase | 110 | 5 | 5 10 100 100 100 -1 +@testcase | 110 | 5 | 5 10 100 100 100 134 @testcase | 110 | 0 | (null) (null) (null) (null) (null) 0 @testcase | 111 | 2 3 9 | -1 @testcase | 111 | 11 22 44 | -1 @@ -233,4 +244,6 @@ fun main() {} @testcase | 112 | -1 -10 -20 | -1 @testcase | 113 | 0 | 12 12 @testcase | 113 | -1 | 12 -1 +@testcase | 114 | -1 | 2 3 130 +@testcase | 114 | 0 | (null) 12 1 */ diff --git a/tolk-tester/tests/warnings-not-errors/unreachable-3.tolk b/tolk-tester/tests/warnings-not-errors/unreachable-3.tolk index fab21fd24a..14bc432644 100644 --- a/tolk-tester/tests/warnings-not-errors/unreachable-3.tolk +++ b/tolk-tester/tests/warnings-not-errors/unreachable-3.tolk @@ -15,7 +15,7 @@ fun main(x: int?) { @testcase | 0 | 5 | -2 @testcase | 0 | null | -1 -@stderr warning: variable `x` of type `int` is always not null +@stderr warning: variable `x` of type `int` can never be `null`, this condition is always true @stderr if (x != null) @stderr warning: unreachable code @stderr return 3 + 4 diff --git a/tolk-tester/tests/warnings-not-errors/warnings-1.tolk b/tolk-tester/tests/warnings-not-errors/warnings-1.tolk index 040057d173..5dc3acdc3e 100644 --- a/tolk-tester/tests/warnings-not-errors/warnings-1.tolk +++ b/tolk-tester/tests/warnings-not-errors/warnings-1.tolk @@ -21,8 +21,8 @@ fun main() { /** @testcase | 0 | | 0 -@stderr warning: variable `c` of type `int` is always not null, this condition is always false -@stderr warning: variable `d` of type `int` is always not null, this condition is always true -@stderr warning: variable `e` is always null, this condition is always true -@stderr warning: expression is always null, this condition is always false +@stderr warning: variable `c` of type `int` can never be `null`, this condition is always false +@stderr warning: variable `d` of type `int` can never be `null`, this condition is always true +@stderr warning: variable `e` is always `null`, this condition is always true +@stderr warning: expression is always `null`, this condition is always false */ diff --git a/tolk-tester/tests/warnings-not-errors/warnings-2.tolk b/tolk-tester/tests/warnings-not-errors/warnings-2.tolk index 57ecb21ac8..2c5d97e848 100644 --- a/tolk-tester/tests/warnings-not-errors/warnings-2.tolk +++ b/tolk-tester/tests/warnings-not-errors/warnings-2.tolk @@ -15,12 +15,12 @@ fun main() { /** @testcase | 0 | | -1 -@stderr warning: variable `a` of type `int` is always not null, this condition is always true +@stderr warning: variable `a` of type `int` can never be `null`, this condition is always true @stderr warning: condition of ternary operator is always true -@stderr warning: variable `c` of type `slice` is always not null, this condition is always false +@stderr warning: variable `c` of type `slice` can never be `null`, this condition is always false @stderr warning: condition of `if` is always true -@stderr warning: variable `b` of type `builder` is always not null, this condition is always false +@stderr warning: variable `b` of type `builder` can never be `null`, this condition is always false @stderr warning: condition of `assert` is always false @stderr warning: condition of `while` is always false -@stderr warning: expression of type `bool` is always not null, this condition is always true +@stderr warning: expression of type `bool` can never be `null`, this condition is always true */ diff --git a/tolk-tester/tests/warnings-not-errors/warnings-3.tolk b/tolk-tester/tests/warnings-not-errors/warnings-3.tolk new file mode 100644 index 0000000000..1198bc1049 --- /dev/null +++ b/tolk-tester/tests/warnings-not-errors/warnings-3.tolk @@ -0,0 +1,26 @@ +fun main() { + var x = 0 as int | slice; + if (x !is builder) {} + while ((x = x) is builder) {} + + match (val tt = x) { + int => { + assert (tt is int, 505); + } + slice => {} + } + assert(x is int) throw 400; + __expect_type(x, "int"); + return x == null; +} + +/** +@testcase | 0 | | 0 + +@stderr warning: variable `x` of type `int | slice` can never be `builder`, this condition is always true +@stderr warning: condition of `if` is always true +@stderr warning: expression of type `int | slice` can never be `builder`, this condition is always false +@stderr warning: condition of `while` is always false +@stderr warning: variable `tt` is always `int`, this condition is always true +@stderr warning: condition of `assert` is always true + */ diff --git a/tolk-tester/tolk-tester.py b/tolk-tester/tolk-tester.py index 3b4fb5b7f6..f8a7a5512d 100644 --- a/tolk-tester/tolk-tester.py +++ b/tolk-tester/tolk-tester.py @@ -402,7 +402,7 @@ def run_all_tests(tests: List[str]): if testcase.compilation_should_fail: print(" OK, stderr match", file=sys.stderr) else: - print(" OK, %d cases, gas %d" % (len(testcase.input_output), gas_used), file=sys.stderr) + print(" OK, %d cases" % (len(testcase.input_output)), file=sys.stderr) except ParseInputError as e: print(" Error parsing input (cur line #%d):" % (testcase.line_idx + 1), e, file=sys.stderr) exit(2) diff --git a/tolk/abscode.cpp b/tolk/abscode.cpp index eb2e9a0a66..c33fcd89db 100644 --- a/tolk/abscode.cpp +++ b/tolk/abscode.cpp @@ -410,12 +410,19 @@ std::vector CodeBlob::create_var(TypePtr var_type, SrcLocation loc, s std::vector nested = create_var(t_tensor->items[i], loc, std::move(sub_name)); ir_idx.insert(ir_idx.end(), nested.begin(), nested.end()); } - } else if (const TypeDataNullable* t_nullable = var_type->try_as(); t_nullable && stack_w != 1) { - std::string null_flag_name = name.empty() ? name : name + ".NNFlag"; - ir_idx = create_var(t_nullable->inner, loc, std::move(name)); - ir_idx.emplace_back(create_var(TypeDataBool::create(), loc, std::move(null_flag_name))[0]); } else if (const TypeDataAlias* t_alias = var_type->try_as()) { ir_idx = create_var(t_alias->underlying_type, loc, std::move(name)); + } else if (const TypeDataUnion* t_union = var_type->try_as(); t_union && stack_w != 1) { + std::string utag_name = name.empty() ? "'UTag" : name + ".UTag"; + if (t_union->or_null) { // in stack comments, `a:(int,int)?` will be "a.0 a.1 a.UTag" + ir_idx = create_var(t_union->or_null, loc, std::move(name)); + } else { // in stack comments, `a:int|slice` will be "a.USlot1 a.UTag" + for (int i = 0; i < stack_w - 1; ++i) { + std::string slot_name = name.empty() ? "'USlot" + std::to_string(i + 1) : name + ".USlot" + std::to_string(i + 1); + ir_idx.emplace_back(create_var(TypeDataUnknown::create(), loc, std::move(slot_name))[0]); + } + } + ir_idx.emplace_back(create_var(TypeDataInt::create(), loc, std::move(utag_name))[0]); } else if (var_type != TypeDataVoid::create() && var_type != TypeDataNever::create()) { #ifdef TOLK_DEBUG tolk_assert(stack_w == 1); diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 729204579f..36e6ce5a2f 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -111,7 +111,7 @@ static void diagnose_addition_in_bitshift(SrcLocation loc, std::string_view bits } } -// replace (a == null) and similar to ast_is_null_check(a) (special AST vertex) +// replace (a == null) and similar to ast_is_type_operator(a, null) (as if `a is null` was written) static AnyExprV maybe_replace_eq_null_with_isNull_check(V v) { bool has_null = v->get_lhs()->type == ast_null_keyword || v->get_rhs()->type == ast_null_keyword; bool replace = has_null && (v->tok == tok_eq || v->tok == tok_neq); @@ -120,7 +120,7 @@ static AnyExprV maybe_replace_eq_null_with_isNull_check(V v } AnyExprV v_nullable = v->get_lhs()->type == ast_null_keyword ? v->get_rhs() : v->get_lhs(); - return createV(v->loc, v_nullable, v->tok == tok_neq); + return createV(v->loc, v_nullable, TypeDataNullLiteral::create(), v->tok == tok_neq); } // parse `123` / `0xFF` / `0b10001` to td::RefInt256 @@ -151,6 +151,7 @@ static td::RefInt256 parse_tok_int_const(std::string_view text) { AnyExprV parse_expr(Lexer& lex); +AnyV parse_statement(Lexer& lex); static AnyV parse_parameter(Lexer& lex, bool is_first) { SrcLocation loc = lex.cur_location(); @@ -242,6 +243,80 @@ static AnyV parse_type_alias_declaration(Lexer& lex, const std::vector(loc, v_ident, underlying_type); } +static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { + SrcLocation loc = lex.cur_location(); + if (lex.tok() == tok_oppar) { + lex.next(); + AnyExprV first = parse_var_declaration_lhs(lex, is_immutable); + if (lex.tok() == tok_clpar) { + lex.next(); + return first; + } + std::vector args(1, first); + while (lex.tok() == tok_comma) { + lex.next(); + args.push_back(parse_var_declaration_lhs(lex, is_immutable)); + } + lex.expect(tok_clpar, "`)`"); + return createV(loc, std::move(args)); + } + if (lex.tok() == tok_opbracket) { + lex.next(); + std::vector args(1, parse_var_declaration_lhs(lex, is_immutable)); + while (lex.tok() == tok_comma) { + lex.next(); + args.push_back(parse_var_declaration_lhs(lex, is_immutable)); + } + lex.expect(tok_clbracket, "`]`"); + return createV(loc, std::move(args)); + } + if (lex.tok() == tok_identifier) { + auto v_ident = createV(loc, lex.cur_str()); + TypePtr declared_type = nullptr; + bool marked_as_redef = false; + lex.next(); + if (lex.tok() == tok_colon) { + lex.next(); + declared_type = parse_type_from_tokens(lex); + } else if (lex.tok() == tok_redef) { + lex.next(); + marked_as_redef = true; + } + return createV(loc, v_ident, declared_type, is_immutable, marked_as_redef); + } + if (lex.tok() == tok_underscore) { + TypePtr declared_type = nullptr; + lex.next(); + if (lex.tok() == tok_colon) { + lex.next(); + declared_type = parse_type_from_tokens(lex); + } + return createV(loc, createV(loc, ""), declared_type, true, false); + } + lex.unexpected("variable name"); +} + +static AnyExprV parse_local_vars_declaration_assignment(Lexer& lex, bool is_statement) { + SrcLocation loc = lex.cur_location(); + bool is_immutable = lex.tok() == tok_val; + lex.next(); + + AnyExprV lhs = createV(loc, parse_var_declaration_lhs(lex, is_immutable)); + if (lex.tok() != tok_assign) { + lex.error("variables declaration must be followed by assignment: `var xxx = ...`"); + } + lex.next(); + AnyExprV rhs = parse_expr(lex); + + if (lex.tok() == tok_comma) { + lex.error("multiple declarations are not allowed, split variables on separate lines"); + } + if (is_statement) { + lex.expect(tok_semicolon, "`;`"); + } + return createV(loc, lhs, rhs); +} + // "parameters" are at function declaration: `fun f(param1: int, mutate param2: slice)` static V parse_parameter_list(Lexer& lex) { SrcLocation loc = lex.cur_location(); @@ -314,6 +389,123 @@ static V parse_maybe_instantiationTs_after_identifier(L } } +// `throw code` / `throw (code)` / `throw (code, arg)` +// it's technically a statement (can't occur "in any place of expression"), +// but inside `match` arm it can appear without semicolon: `pattern => throw 123` +static AnyV parse_throw_expression(Lexer& lex, bool is_statement) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_throw, "`throw`"); + + AnyExprV thrown_code, thrown_arg; + if (lex.tok() == tok_oppar) { // throw (code) or throw (code, arg) + lex.next(); + thrown_code = parse_expr(lex); + if (lex.tok() == tok_comma) { + lex.next(); + thrown_arg = parse_expr(lex); + } else { + thrown_arg = createV(loc); + } + lex.expect(tok_clpar, "`)`"); + } else { // throw code + thrown_code = parse_expr(lex); + thrown_arg = createV(loc); + } + + if (is_statement) { + lex.expect(tok_semicolon, "`;`"); + } + return createV(loc, thrown_code, thrown_arg); +} + +// `pattern => body` inside `match` +static V parse_match_arm(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + MatchArmKind pattern_kind = static_cast(-1); + TypePtr exact_type = nullptr; + AnyExprV pattern_expr = nullptr; + + Lexer::SavedPositionForLookahead backup = lex.save_parsing_position(); + try { + exact_type = parse_type_from_tokens(lex); + pattern_kind = MatchArmKind::exact_type; + } catch (const ParseError&) { + } + if (!exact_type || lex.tok() != tok_double_arrow) { + exact_type = nullptr; + lex.restore_position(backup); + try { + pattern_expr = parse_expr(lex); + pattern_kind = MatchArmKind::const_expression; // any expr at parsing, should result in const int/bool + } catch (const ParseError&) { + } + } + if (!exact_type && !pattern_expr && lex.tok() == tok_else) { + lex.next(); + pattern_kind = MatchArmKind::else_branch; + } + + if (pattern_kind == static_cast(-1)) { + lex.restore_position(backup); + throw ParseError(loc, "expected or in `match` before `=>`"); + } + lex.expect(tok_double_arrow, "`=>`"); + + AnyExprV body; + if (lex.tok() == tok_opbrace) { // pattern => { ... } + AnyV v_seq = parse_statement(lex); + body = createV(v_seq->loc, v_seq); + } else if (lex.tok() == tok_throw) { // pattern => throw 123 (allow without braces) + AnyV v_throw = parse_throw_expression(lex, false); + AnyV v_seq = createV(v_throw->loc, v_throw->loc, {v_throw}); + body = createV(v_seq->loc, v_seq); + } else if (lex.tok() == tok_return) { // pattern => return 123 (allow without braces, like throw) + lex.next(); + AnyV v_return = createV(lex.cur_location(), parse_expr(lex)); + AnyV v_seq = createV(v_return->loc, v_return->loc, {v_return}); + body = createV(v_seq->loc, v_seq); + } else { + body = parse_expr(lex); + } + + if (pattern_expr == nullptr) { // for match by type / default case, empty vertex, not nullptr + pattern_expr = createV(loc); + } + return createV(loc, pattern_kind, exact_type, pattern_expr, body); +} + +static V parse_match_expression(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_match, "`match`"); + + lex.expect(tok_oppar, "`(`"); + AnyExprV subject = lex.tok() == tok_var || lex.tok() == tok_val // `match (var x = rhs)` + ? parse_local_vars_declaration_assignment(lex, false) + : parse_expr(lex); + lex.expect(tok_clpar, "`)`"); + + std::vector subject_and_arms = {subject}; + lex.expect(tok_opbrace, "`{`"); + while (lex.tok() != tok_clbrace) { + auto v_arm = parse_match_arm(lex); + subject_and_arms.push_back(v_arm); + + // after `pattern => { ... }` comma is optional, after `pattern => expr` mandatory + bool was_comma = lex.tok() == tok_comma; // trailing comma is allowed always + if (was_comma) { + lex.next(); + } + if (lex.tok() == tok_clbrace) { + break; + } + if (!was_comma && v_arm->get_body()->type != ast_braced_expression) { + lex.unexpected("`,`"); + } + } + lex.expect(tok_clbrace, "`}`"); + return createV(loc, std::move(subject_and_arms)); +} + // parse (expr) / [expr] / identifier / number static AnyExprV parse_expr100(Lexer& lex) { SrcLocation loc = lex.cur_location(); @@ -404,6 +596,8 @@ static AnyExprV parse_expr100(Lexer& lex) { } return createV(loc, v_ident, v_instantiationTs); } + case tok_match: + return parse_match_expression(lex); default: lex.unexpected(""); } @@ -474,7 +668,7 @@ static AnyExprV parse_expr75(Lexer& lex) { return parse_expr80(lex); } -// parse E as +// parse E as / is / !is static AnyExprV parse_expr40(Lexer& lex) { AnyExprV lhs = parse_expr75(lex); if (lex.tok() == tok_as) { @@ -482,6 +676,15 @@ static AnyExprV parse_expr40(Lexer& lex) { lex.next(); TypePtr cast_to_type = parse_type_from_tokens(lex); lhs = createV(loc, lhs, cast_to_type); + } else if (lex.tok() == tok_is) { + SrcLocation loc = lex.cur_location(); + lex.next(); + TypePtr rhs_type = parse_type_from_tokens(lex); + bool is_negated = lhs->type == ast_not_null_operator; // `a !is ...`, now lhs = `a!` + if (is_negated) { + lhs = lhs->as()->get_expr(); + } + lhs = createV(loc, lhs, rhs_type, is_negated); } return lhs; } @@ -616,80 +819,6 @@ AnyExprV parse_expr(Lexer& lex) { return parse_expr10(lex); } -AnyV parse_statement(Lexer& lex); - -static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { - SrcLocation loc = lex.cur_location(); - if (lex.tok() == tok_oppar) { - lex.next(); - AnyExprV first = parse_var_declaration_lhs(lex, is_immutable); - if (lex.tok() == tok_clpar) { - lex.next(); - return first; - } - std::vector args(1, first); - while (lex.tok() == tok_comma) { - lex.next(); - args.push_back(parse_var_declaration_lhs(lex, is_immutable)); - } - lex.expect(tok_clpar, "`)`"); - return createV(loc, std::move(args)); - } - if (lex.tok() == tok_opbracket) { - lex.next(); - std::vector args(1, parse_var_declaration_lhs(lex, is_immutable)); - while (lex.tok() == tok_comma) { - lex.next(); - args.push_back(parse_var_declaration_lhs(lex, is_immutable)); - } - lex.expect(tok_clbracket, "`]`"); - return createV(loc, std::move(args)); - } - if (lex.tok() == tok_identifier) { - auto v_ident = createV(loc, lex.cur_str()); - TypePtr declared_type = nullptr; - bool marked_as_redef = false; - lex.next(); - if (lex.tok() == tok_colon) { - lex.next(); - declared_type = parse_type_from_tokens(lex); - } else if (lex.tok() == tok_redef) { - lex.next(); - marked_as_redef = true; - } - return createV(loc, v_ident, declared_type, is_immutable, marked_as_redef); - } - if (lex.tok() == tok_underscore) { - TypePtr declared_type = nullptr; - lex.next(); - if (lex.tok() == tok_colon) { - lex.next(); - declared_type = parse_type_from_tokens(lex); - } - return createV(loc, createV(loc, ""), declared_type, true, false); - } - lex.unexpected("variable name"); -} - -static AnyV parse_local_vars_declaration_assignment(Lexer& lex) { - SrcLocation loc = lex.cur_location(); - bool is_immutable = lex.tok() == tok_val; - lex.next(); - - AnyExprV lhs = createV(loc, parse_var_declaration_lhs(lex, is_immutable)); - if (lex.tok() != tok_assign) { - lex.error("variables declaration must be followed by assignment: `var xxx = ...`"); - } - lex.next(); - AnyExprV rhs = parse_expr(lex); - - if (lex.tok() == tok_comma) { - lex.error("multiple declarations are not allowed, split variables on separate lines"); - } - lex.expect(tok_semicolon, "`;`"); - return createV(loc, lhs, rhs); -} - static V parse_sequence(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.expect(tok_opbrace, "`{`"); @@ -699,7 +828,7 @@ static V parse_sequence(Lexer& lex) { } SrcLocation loc_end = lex.cur_location(); lex.expect(tok_clbrace, "`}`"); - return createV(loc, loc_end, items); + return createV(loc, loc_end, std::move(items)); } static AnyV parse_return_statement(Lexer& lex) { @@ -789,30 +918,6 @@ static AnyExprV create_catch_underscore_variable(const Lexer& lex) { return createV(lex.cur_location(), v_ident, nullptr); } -static AnyV parse_throw_statement(Lexer& lex) { - SrcLocation loc = lex.cur_location(); - lex.expect(tok_throw, "`throw`"); - - AnyExprV thrown_code, thrown_arg; - if (lex.tok() == tok_oppar) { // throw (code) or throw (code, arg) - lex.next(); - thrown_code = parse_expr(lex); - if (lex.tok() == tok_comma) { - lex.next(); - thrown_arg = parse_expr(lex); - } else { - thrown_arg = createV(loc); - } - lex.expect(tok_clpar, "`)`"); - } else { // throw code - thrown_code = parse_expr(lex); - thrown_arg = createV(loc); - } - - lex.expect(tok_semicolon, "`;`"); - return createV(loc, thrown_code, thrown_arg); -} - static AnyV parse_assert_statement(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.expect(tok_assert, "`assert`"); @@ -866,7 +971,7 @@ AnyV parse_statement(Lexer& lex) { switch (lex.tok()) { case tok_var: // `var x = 0` is technically an expression, but can not appear in "any place", case tok_val: // only as a separate declaration - return parse_local_vars_declaration_assignment(lex); + return parse_local_vars_declaration_assignment(lex, true); case tok_opbrace: return parse_sequence(lex); case tok_return: @@ -880,7 +985,7 @@ AnyV parse_statement(Lexer& lex) { case tok_while: return parse_while_statement(lex); case tok_throw: - return parse_throw_statement(lex); + return parse_throw_expression(lex, true); case tok_assert: return parse_assert_statement(lex); case tok_try: @@ -895,7 +1000,9 @@ AnyV parse_statement(Lexer& lex) { lex.error("break/continue from loops are not supported yet"); default: { AnyExprV expr = parse_expr(lex); - lex.expect(tok_semicolon, "`;`"); + if (expr->type != ast_match_expression) { // match statement, as if statement, doesn't require semicolon after + lex.expect(tok_semicolon, "`;`"); + } return expr; } } diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index d1ab4c4507..5efae1e2f4 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -60,6 +60,12 @@ class ASTReplacer { return v_mutable; } + GNU_ATTRIBUTE_ALWAYS_INLINE AnyExprV replace_children(const ASTExprBlockOfStatements* v) { + auto* v_mutable = const_cast(v); + v_mutable->child_sequence = replace(v_mutable->child_sequence->as()); + return v_mutable; + } + GNU_ATTRIBUTE_ALWAYS_INLINE AnyV replace_children(const ASTStatementUnary* v) { auto* v_mutable = const_cast(v); v_mutable->child = replace(v_mutable->child); @@ -88,6 +94,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { // expressions virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } @@ -108,8 +115,10 @@ class ASTReplacerInFunctionBody : public ASTReplacer { virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } - virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } // statements virtual AnyV replace(V v) { return replace_children(v); } virtual AnyV replace(V v) { return replace_children(v); } @@ -126,6 +135,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { switch (v->type) { case ast_empty_expression: return replace(v->as()); case ast_parenthesized_expression: return replace(v->as()); + case ast_braced_expression: return replace(v->as()); case ast_tensor: return replace(v->as()); case ast_typed_tuple: return replace(v->as()); case ast_reference: return replace(v->as()); @@ -146,8 +156,10 @@ class ASTReplacerInFunctionBody : public ASTReplacer { case ast_binary_operator: return replace(v->as()); case ast_ternary_operator: return replace(v->as()); case ast_cast_as_operator: return replace(v->as()); + case ast_is_type_operator: return replace(v->as()); case ast_not_null_operator: return replace(v->as()); - case ast_is_null_check: return replace(v->as()); + case ast_match_expression: return replace(v->as()); + case ast_match_arm: return replace(v->as()); default: throw UnexpectedASTNodeType(v, "ASTReplacerInFunctionBody::replace"); } diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index bf48c2389b..9f0738d3f6 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -61,6 +61,9 @@ class ASTReplicatorFunction : public ASTReplicator { virtual V clone(V v) { return createV(v->loc, clone(v->get_expr())); } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_sequence())); + } virtual V clone(V v) { return createV(v->loc, clone(v->get_items())); } @@ -121,11 +124,17 @@ class ASTReplicatorFunction : public ASTReplicator { virtual V clone(V v) { return createV(v->loc, clone(v->get_expr()), clone(v->cast_to_type)); } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_expr()), clone(v->rhs_type), v->is_negated); + } virtual V clone(V v) { return createV(v->loc, clone(v->get_expr())); } - virtual V clone(V v) { - return createV(v->loc, clone(v->get_expr()), v->is_negated); + virtual V clone(V v) { + return createV(v->loc, clone(v->get_all_children())); + } + virtual V clone(V v) { + return createV(v->loc, v->pattern_kind, clone(v->exact_type), clone(v->get_pattern_expr()), clone(v->get_body())); } // statements @@ -186,6 +195,7 @@ class ASTReplicatorFunction : public ASTReplicator { switch (v->type) { case ast_empty_expression: return clone(v->as()); case ast_parenthesized_expression: return clone(v->as()); + case ast_braced_expression: return clone(v->as()); case ast_tensor: return clone(v->as()); case ast_typed_tuple: return clone(v->as()); case ast_reference: return clone(v->as()); @@ -206,8 +216,10 @@ class ASTReplicatorFunction : public ASTReplicator { case ast_binary_operator: return clone(v->as()); case ast_ternary_operator: return clone(v->as()); case ast_cast_as_operator: return clone(v->as()); + case ast_is_type_operator: return clone(v->as()); case ast_not_null_operator: return clone(v->as()); - case ast_is_null_check: return clone(v->as()); + case ast_match_expression: return clone(v->as()); + case ast_match_arm: return clone(v->as()); default: throw UnexpectedASTNodeType(v, "ASTReplicatorFunction::clone"); } diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index 77390c1caa..33082a6e9b 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -36,6 +36,7 @@ class ASTStringifier final : public ASTVisitor { // expressions {ast_empty_expression, "ast_empty_expression"}, {ast_parenthesized_expression, "ast_parenthesized_expression"}, + {ast_braced_expression, "ast_braced_expression"}, {ast_tensor, "ast_tensor"}, {ast_typed_tuple, "ast_typed_tuple"}, {ast_reference, "ast_reference"}, @@ -56,8 +57,10 @@ class ASTStringifier final : public ASTVisitor { {ast_binary_operator, "ast_binary_operator"}, {ast_ternary_operator, "ast_ternary_operator"}, {ast_cast_as_operator, "ast_cast_as_operator"}, + {ast_is_type_operator, "ast_is_type_operator"}, {ast_not_null_operator, "ast_not_null_operator"}, - {ast_is_null_check, "ast_is_null_check"}, + {ast_match_expression, "ast_match_expression"}, + {ast_match_arm, "ast_match_arm"}, // statements {ast_empty_statement, "ast_empty_statement"}, {ast_sequence, "ast_sequence"}, @@ -170,6 +173,10 @@ class ASTStringifier final : public ASTVisitor { return static_cast(v->as()->operator_name); case ast_cast_as_operator: return v->as()->cast_to_type->as_human_readable(); + case ast_is_type_operator: { + std::string prefix = v->as()->is_negated ? "!is " : "is "; + return prefix + v->as()->rhs_type->as_human_readable(); + } case ast_sequence: return "↓" + std::to_string(v->as()->get_items().size()); case ast_instantiationT_item: @@ -209,6 +216,14 @@ class ASTStringifier final : public ASTVisitor { } return result + ">"; } + case ast_match_arm: + if (v->as()->pattern_kind == MatchArmKind::exact_type) { + return v->as()->exact_type->as_human_readable(); + } + if (v->as()->pattern_kind == MatchArmKind::const_expression) { + return "(expression)"; + } + return "(else)"; case ast_tolk_required_version: return static_cast(v->as()->semver); case ast_import_directive: @@ -249,6 +264,7 @@ class ASTStringifier final : public ASTVisitor { // expressions case ast_empty_expression: return handle_vertex(v->as()); case ast_parenthesized_expression: return handle_vertex(v->as()); + case ast_braced_expression: return handle_vertex(v->as()); case ast_tensor: return handle_vertex(v->as()); case ast_typed_tuple: return handle_vertex(v->as()); case ast_reference: return handle_vertex(v->as()); @@ -269,8 +285,10 @@ class ASTStringifier final : public ASTVisitor { case ast_binary_operator: return handle_vertex(v->as()); case ast_ternary_operator: return handle_vertex(v->as()); case ast_cast_as_operator: return handle_vertex(v->as()); + case ast_is_type_operator: return handle_vertex(v->as()); case ast_not_null_operator: return handle_vertex(v->as()); - case ast_is_null_check: return handle_vertex(v->as()); + case ast_match_expression: return handle_vertex(v->as()); + case ast_match_arm: return handle_vertex(v->as()); // statements case ast_empty_statement: return handle_vertex(v->as()); case ast_sequence: return handle_vertex(v->as()); diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index 737646bb5b..2f5f269f78 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -56,6 +56,10 @@ class ASTVisitor { } } + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTExprBlockOfStatements* v) { + visit_children(v->child_sequence->as()); + } + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTStatementUnary* v) { visit(v->child); } @@ -89,6 +93,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { // expressions virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } @@ -109,8 +114,10 @@ class ASTVisitorFunctionBody : public ASTVisitor { virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } - virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } // statements virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } @@ -128,6 +135,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { // expressions case ast_empty_expression: return visit(v->as()); case ast_parenthesized_expression: return visit(v->as()); + case ast_braced_expression: return visit(v->as()); case ast_tensor: return visit(v->as()); case ast_typed_tuple: return visit(v->as()); case ast_reference: return visit(v->as()); @@ -148,8 +156,10 @@ class ASTVisitorFunctionBody : public ASTVisitor { case ast_binary_operator: return visit(v->as()); case ast_ternary_operator: return visit(v->as()); case ast_cast_as_operator: return visit(v->as()); + case ast_is_type_operator: return visit(v->as()); case ast_not_null_operator: return visit(v->as()); - case ast_is_null_check: return visit(v->as()); + case ast_match_expression: return visit(v->as()); + case ast_match_arm: return visit(v->as()); // statements case ast_empty_statement: return visit(v->as()); case ast_sequence: return visit(v->as()); diff --git a/tolk/ast.cpp b/tolk/ast.cpp index f383ebcc71..d740970da9 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -138,6 +138,20 @@ void Vertex::assign_resolved_type(TypePtr cast_to_type) { this->cast_to_type = cast_to_type; } +void Vertex::assign_resolved_type(TypePtr rhs_type) { + this->rhs_type = rhs_type; +} + +void Vertex::assign_is_negated(bool is_negated) { + this->is_negated = is_negated; +} + +void Vertex::assign_resolved_pattern(MatchArmKind pattern_kind, TypePtr exact_type, AnyExprV pattern_expr) { + this->pattern_kind = pattern_kind; + this->exact_type = exact_type; + this->lhs = pattern_expr; +} + void Vertex::assign_var_ref(GlobalVarPtr var_ref) { this->var_ref = var_ref; } @@ -186,10 +200,6 @@ void Vertex::assign_fun_ref(FunctionPtr fun_ref) { this->fun_ref = fun_ref; } -void Vertex::assign_is_negated(bool is_negated) { - this->is_negated = is_negated; -} - void Vertex::assign_first_unreachable(AnyV first_unreachable) { this->first_unreachable = first_unreachable; } diff --git a/tolk/ast.h b/tolk/ast.h index b62aa4c532..fea11ebcb9 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -68,6 +68,7 @@ enum ASTNodeType { // expressions ast_empty_expression, ast_parenthesized_expression, + ast_braced_expression, ast_tensor, ast_typed_tuple, ast_reference, @@ -88,8 +89,10 @@ enum ASTNodeType { ast_binary_operator, ast_ternary_operator, ast_cast_as_operator, + ast_is_type_operator, ast_not_null_operator, - ast_is_null_check, + ast_match_expression, + ast_match_arm, // statements ast_empty_statement, ast_sequence, @@ -128,6 +131,12 @@ enum class AnnotationKind { unknown, }; +enum class MatchArmKind { // for `match` expression, each of arms `pattern => body` can be: + const_expression, // `-1 => body` / `SOME_CONST + ton("0.05") => body` (any expr at parsing, resulting in const) + exact_type, // `int => body` / `User | slice => body` + else_branch, // `else => body` +}; + template struct Vertex; @@ -244,7 +253,7 @@ struct ASTExprVararg : ASTNodeExpressionBase { AnyExprV child(int i) const { return children.at(i); } - ASTExprVararg(ASTNodeType type, SrcLocation loc, std::vector children) + ASTExprVararg(ASTNodeType type, SrcLocation loc, std::vector&& children) : ASTNodeExpressionBase(type, loc), children(std::move(children)) {} public: @@ -252,6 +261,17 @@ struct ASTExprVararg : ASTNodeExpressionBase { bool empty() const { return children.empty(); } }; +struct ASTExprBlockOfStatements : ASTNodeExpressionBase { + friend class ASTVisitor; + friend class ASTReplacer; + +protected: + AnyV child_sequence; + + ASTExprBlockOfStatements(ASTNodeType type, SrcLocation loc, AnyV child_sequence) + : ASTNodeExpressionBase(type, loc), child_sequence(child_sequence) {} +}; + struct ASTStatementUnary : ASTNodeStatementBase { friend class ASTVisitor; friend class ASTReplacer; @@ -351,6 +371,18 @@ struct Vertex final : ASTExprUnary { : ASTExprUnary(ast_parenthesized_expression, loc, expr) {} }; +template<> +// ast_braced_expression is a sequence, but in a context of expression (it has a type) +// it can contain arbitrary statements inside +// it can occur only in special places within the input code, not anywhere +// example: `match (intV) { 0 => { ... } }` rhs of 0 is braced expression +struct Vertex final : ASTExprBlockOfStatements { + auto get_sequence() const { return child_sequence->as(); } + + Vertex(SrcLocation loc, AnyV child_sequence) + : ASTExprBlockOfStatements(ast_braced_expression, loc, child_sequence) {} +}; + template<> // ast_tensor is a set of expressions embraced by (parenthesis) // in most languages, it's called "tuple", but in TVM, "tuple" is a TVM primitive, that's why "tensor" @@ -683,6 +715,24 @@ struct Vertex final : ASTExprUnary { , cast_to_type(cast_to_type) {} }; +template<> +// ast_is_type_operator is type matching with "is" or "!is" keywords and "== null" / "!= null" (same as "is null") +// examples: `v is SomeStruct` / `getF() !is slice` / `v == null` / `v !is null` +struct Vertex final : ASTExprUnary { + AnyExprV get_expr() const { return child; } + + TypePtr rhs_type; + bool is_negated; // `!is type`, `!= null` + + Vertex* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr rhs_type); + void assign_is_negated(bool is_negated); + + Vertex(SrcLocation loc, AnyExprV expr, TypePtr rhs_type, bool is_negated) + : ASTExprUnary(ast_is_type_operator, loc, expr) + , rhs_type(rhs_type), is_negated(is_negated) {} +}; + template<> // ast_not_null_operator is non-null assertion: like TypeScript ! or Kotlin !! // examples: `nullableInt!` / `getNullableBuilder()!` @@ -694,19 +744,40 @@ struct Vertex final : ASTExprUnary { }; template<> -// ast_is_null_check is an artificial vertex for "expr == null" / "expr != null" / same but null on the left -// it's created instead of a general binary expression to emphasize its purpose -struct Vertex final : ASTExprUnary { - bool is_negated; +// ast_match_expression is `match (subject) { ... arms ... }`, used either as a statement or as an expression +// example: `match (intOrSliceVar) { int => 1, slice => 2 }` +// example: `match (var c = getIntOrSlice()) { int => return 0, slice => throw 123 }` +struct Vertex final : ASTExprVararg { + AnyExprV get_subject() const { return child(0); } + int get_arms_count() const { return size() - 1; } + auto get_arm(int i) const { return child(i + 1)->as(); } + const std::vector& get_all_children() const { return children; } - AnyExprV get_expr() const { return child; } + bool is_statement() const { return !is_rvalue && !is_lvalue; } + + Vertex(SrcLocation loc, std::vector&& subject_and_arms) + : ASTExprVararg(ast_match_expression, loc, std::move(subject_and_arms)) {} +}; + +template<> +// ast_match_arm is one `pattern => body` inside `match` expression/statement +// pattern can be a custom expression / a type / `else` (see comments in MatchArmKind) +// body can be any expression; particularly, braced expression `{ ... }` +// example: `int => variable` (match by type, inferred_type of body variable's type) +// example: `a+b => { ...; return 0; }` (match by expression, inferred_type of body is "never" (unreachable end)) +struct Vertex final : ASTExprBinary { + MatchArmKind pattern_kind; + TypePtr exact_type; // for MatchArmKind::exact_type; otherwise, nullptr + + AnyExprV get_pattern_expr() const { return lhs; } + AnyExprV get_body() const { return rhs; } // remember, it may be V Vertex* mutate() const { return const_cast(this); } - void assign_is_negated(bool is_negated); + void assign_resolved_pattern(MatchArmKind pattern_kind, TypePtr exact_type, AnyExprV pattern_expr); - Vertex(SrcLocation loc, AnyExprV expr, bool is_negated) - : ASTExprUnary(ast_is_null_check, loc, expr) - , is_negated(is_negated) {} + Vertex(SrcLocation loc, MatchArmKind pattern_kind, TypePtr exact_type, AnyExprV pattern_expr, AnyExprV body) + : ASTExprBinary(ast_match_arm, loc, pattern_expr, body) + , pattern_kind(pattern_kind), exact_type(exact_type) {} }; @@ -739,7 +810,7 @@ struct Vertex final : ASTStatementVararg { Vertex* mutate() const { return const_cast(this); } void assign_first_unreachable(AnyV first_unreachable); - Vertex(SrcLocation loc, SrcLocation loc_end, std::vector items) + Vertex(SrcLocation loc, SrcLocation loc_end, std::vector&& items) : ASTStatementVararg(ast_sequence, loc, std::move(items)) , loc_end(loc_end) {} }; diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index b335683f33..94a43e9fc6 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -81,14 +81,14 @@ void GenericSubstitutionsDeduceForCall::consider_next_condition(TypePtr param_ty if (const auto* asT = param_type->try_as()) { // `(arg: T)` called as `f([1, 2])` => T is [int, int] provide_deducedT(asT->nameT, arg_type); - } else if (const auto* p_nullable = param_type->try_as()) { + } else if (const auto* p_nullable = param_type->try_as(); p_nullable && p_nullable->or_null) { // `arg: T?` called as `f(nullableInt)` => T is int - if (const auto* a_nullable = arg_type->unwrap_alias()->try_as()) { - consider_next_condition(p_nullable->inner, a_nullable->inner); + if (const auto* a_nullable = arg_type->unwrap_alias()->try_as(); a_nullable && a_nullable->or_null) { + consider_next_condition(p_nullable->or_null, a_nullable->or_null); } // `arg: T?` called as `f(int)` => T is int else { - consider_next_condition(p_nullable->inner, arg_type); + consider_next_condition(p_nullable->or_null, arg_type); } } else if (const auto* p_tensor = param_type->try_as()) { // `arg: (int, T)` called as `f((5, cs))` => T is slice @@ -112,6 +112,13 @@ void GenericSubstitutionsDeduceForCall::consider_next_condition(TypePtr param_ty } consider_next_condition(p_callable->return_type, a_callable->return_type); } + } else if (const auto* p_union = param_type->try_as()) { + // `arg: T1 | T2` called as `f(intOrBuilder)` => T1 is int, T2 is builder + if (const auto* a_union = arg_type->unwrap_alias()->try_as(); a_union && a_union->variants.size() == p_union->variants.size()) { + for (int i = 0; i < static_cast(p_union->variants.size()); ++i) { + consider_next_condition(p_union->variants[i], a_union->variants[i]); + } + } } } diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index 961be1674a..822abe931f 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -337,6 +337,7 @@ struct ChunkIdentifierOrKeyword final : ChunkLexerBase { case 2: if (str == "do") return tok_do; if (str == "if") return tok_if; + if (str == "is") return tok_is; if (str == "as") return tok_as; break; case 3: @@ -359,6 +360,7 @@ struct ChunkIdentifierOrKeyword final : ChunkLexerBase { case 5: if (str == "const") return tok_const; if (str == "false") return tok_false; + if (str == "match") return tok_match; if (str == "redef") return tok_redef; if (str == "while") return tok_while; if (str == "break") return tok_break; @@ -521,6 +523,7 @@ struct TolkLanguageGrammar { register_token("|=", 2, tok_set_bitwise_or); register_token("^=", 2, tok_set_bitwise_xor); register_token("->", 2, tok_arrow); + register_token("=>", 2, tok_double_arrow); register_token("<=>", 3, tok_spaceship); register_token("~>>", 3, tok_rshiftR); register_token("^>>", 3, tok_rshiftC); @@ -587,13 +590,14 @@ void Lexer::next_special(TokenType parse_next_as, const char* str_expected) { } Lexer::SavedPositionForLookahead Lexer::save_parsing_position() const { - return {p_next, cur_token_idx, cur_token}; + return {p_next, cur_token_idx, cur_token, location}; } void Lexer::restore_position(SavedPositionForLookahead saved) { p_next = saved.p_next; cur_token_idx = last_token_idx = saved.cur_token_idx; cur_token = saved.cur_token; + location = saved.loc; } void Lexer::error(const std::string& err_msg) const { diff --git a/tolk/lexer.h b/tolk/lexer.h index 34c792ebcc..6e9ab6a41b 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -116,9 +116,12 @@ enum TokenType { tok_assert, tok_if, tok_else, + tok_match, tok_arrow, + tok_double_arrow, tok_as, + tok_is, tok_tolk, tok_semver, @@ -161,6 +164,7 @@ class Lexer { const char* p_next = nullptr; int cur_token_idx = 0; Token cur_token; + SrcLocation loc; }; explicit Lexer(const SrcFile* file); diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 10647de946..167c561477 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -21,6 +21,7 @@ #include "type-system.h" #include "common/refint.h" #include "constant-evaluator.h" +#include "smart-casts-cfg.h" #include /* @@ -41,7 +42,7 @@ * Example: `nullableInt!`; for `nullableInt` inferred_type is `int?`, and target_type is `int` * (this doesn't lead to stack reorganization, but in case `nullableTensor!` does) * (inferred_type of `nullableInt!` is `int`, and its target_type depends on its usage). - * The same mechanism will work for union types in the future. + * Example: `var a: int|slice = 5`. This `5` should be extended as "5 1" (5 for value, 1 for type_id of `int`). */ namespace tolk { @@ -410,6 +411,39 @@ static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyE return right; } +static std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc) { + FunctionPtr eq_sym = lookup_global_symbol("_==_")->try_as(); + FunctionPtr isnull_sym = lookup_global_symbol("__isNull")->try_as(); + FunctionPtr not_sym = lookup_global_symbol("!b_")->try_as(); + std::vector result_ir_idx = code.create_tmp_var(TypeDataBool::create(), loc, debug_desc); + + const TypeDataUnion* lhs_union = expr_type->try_as(); + if (!lhs_union) { + // `int` is `int` / `int` is `builder`, it's compile-time, either 0, or -1 + bool types_eq = expr_type->get_type_id() == cmp_type->get_type_id(); + code.emplace_back(loc, Op::_IntConst, result_ir_idx, td::make_refint(types_eq ? -1 : 0)); + } else if (lhs_union->is_primitive_nullable() && cmp_type == TypeDataNullLiteral::create()) { + // `int?` is `null` for primitive 1-slot nullables, they hold either value of TVM NULL, no extra union tag slot + code.emplace_back(loc, Op::_Call, result_ir_idx, expr_ir_idx, isnull_sym); + } else if (lhs_union->is_primitive_nullable()) { + // `int?` is `int` (check for null actually) / `int?` is `builder` (compile-time false actually) + bool cant_happen = lhs_union->or_null->get_type_id() != cmp_type->get_type_id(); + if (cant_happen) { + code.emplace_back(loc, Op::_IntConst, result_ir_idx, td::make_refint(0)); + } else { + code.emplace_back(loc, Op::_Call, result_ir_idx, expr_ir_idx, isnull_sym); + code.emplace_back(loc, Op::_Call, result_ir_idx, result_ir_idx, not_sym); + } + } else { + // `int | slice` is `int`, check type id + std::vector typeid_ir_idx = code.create_tmp_var(TypeDataInt::create(), loc, "(type-id)"); + code.emplace_back(loc, Op::_IntConst, typeid_ir_idx, td::make_refint(cmp_type->get_type_id())); + code.emplace_back(loc, Op::_Call, result_ir_idx, std::vector{typeid_ir_idx[0], expr_ir_idx.back()}, eq_sym); + } + + return result_ir_idx; +} + static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcLocation loc, std::vector&& args_vars, FunctionPtr fun_ref, const char* debug_desc) { std::vector rvect = code.create_tmp_var(ret_type, loc, debug_desc); @@ -428,8 +462,8 @@ static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcL // `null` (inferred_type) is 1 stack slots, but target_type is 3, we should add 2 nulls. // Another example: `var t1 = (1, null); var t2: (int, (int,int)?) = t1;`. // Then t1's rvect is 2 vars (1 and null), but t1's `null` should be converted to 3 stack slots (resulting in 4 total). -// The same mechanism will work for union types in the future. -// Here rvect is a list of IR vars for inferred_type, probably patched due to target_type. +// The same mechanism works for union types, but there is a union tag (UTag) slot instead of null flag. +// Another example: `var i: int|slice = 5;`. This "5" is represented as "5 1" (5 for value, 1 is type_id of `int`). GNU_ATTRIBUTE_NOINLINE static std::vector transition_expr_to_runtime_type_impl(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc) { #ifdef TOLK_DEBUG @@ -446,176 +480,299 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectorget_width_on_stack(); - const TypeDataNullable* t_nullable = target_type->try_as(); - const TypeDataNullable* o_nullable = original_type->try_as(); + int orig_w = original_type->get_width_on_stack(); // = rvect.size() + const TypeDataUnion* t_union = target_type->try_as(); + const TypeDataUnion* o_union = original_type->try_as(); + + // most common case, simple nullability: + // - `int` to `int?` + // - `null` to `int?` + // - `int8?` to `int16?` + // - `null` to `StructWith1Int?` + // in general, pass `T1` to `T2?` when `T2` is a primitive and `T2?` still occupies 1 stack slot (value or TVM NULL) + if (t_union && target_w == 1 && orig_w == 1 && t_union->or_null->get_width_on_stack() == 1) { + // a generalized union type "T | ..." takes at least 2 slots, but "T | null" can be 1 + tolk_assert(t_union->or_null); + // rvect has 1 slot, either value or TVM NULL + return rvect; + } - // handle `never` - // it may occur due to smart cast and in unreachable branches - // we can't do anything reasonable here, but (hopefully) execution will never reach this point, and stack won't be polluted - if (original_type == TypeDataNever::create()) { - std::vector dummy_rvect; - dummy_rvect.reserve(target_w); - for (int i = 0; i < target_w; ++i) { - dummy_rvect.push_back(code.create_tmp_var(TypeDataUnknown::create(), loc, "(never)")[0]); - } - return dummy_rvect; + // smart cast of a primitive 1-slot nullable: + // - `int?` to `int` + // - `int?` to `null` + // - `StructWith1Int?` to `null` + // this value (one slot) is either a TVM primitive or TVM NULL at runtime + if (o_union && orig_w == 1 && target_w == 1 && o_union->or_null->get_width_on_stack() == 1) { + tolk_assert(o_union->or_null); + return rvect; } - if (target_type == TypeDataNever::create()) { - return {}; + + // pass `T` to `never` + // it occurs due to smart cast, in unreachable branches, for example `if (intVal == null) { return intVal; }` + // we can't do anything reasonable here, but (hopefully) execution will never reach this point, and stack won't be polluted + if (target_type == TypeDataNever::create() || original_type == TypeDataNever::create() || target_type == TypeDataUnknown::create()) { + return rvect; } - // pass `null` to `T?` - // for primitives like `int?`, no changes in rvect, null occupies the same TVM slot - // for tensors like `(int,int)?`, `null` is represented as N nulls + 1 null flag, insert N nulls - if (t_nullable && original_type == TypeDataNullLiteral::create()) { - tolk_assert(rvect.size() == 1); - if (target_w == 1 && !t_nullable->is_primitive_nullable()) { // `null` to `()?` - rvect = code.create_tmp_var(TypeDataInt::create(), loc, "(NNFlag)"); - code.emplace_back(loc, Op::_IntConst, rvect, td::make_refint(0)); - } - if (target_w > 1) { - FunctionPtr builtin_sym = lookup_global_symbol("__null")->try_as(); - rvect.reserve(target_w + 1); - for (int i = 1; i < target_w - 1; ++i) { - std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); - code.emplace_back(loc, Op::_Call, ith_null, std::vector{}, builtin_sym); - rvect.push_back(ith_null[0]); - } - std::vector null_flag_ir = code.create_tmp_var(TypeDataInt::create(), loc, "(NNFlag)"); - var_idx_t null_flag_ir_idx = null_flag_ir[0]; - code.emplace_back(loc, Op::_IntConst, std::move(null_flag_ir), td::make_refint(0)); - rvect.push_back(null_flag_ir_idx); + // smart cast to a primitive 1-slot nullable: + // - `int | slice | null` to `slice?` + // - `A | int | null` to `int?` + // so, originally a type occupies N slots, but needs to be converted to 1 slot + if (t_union && target_w == 1 && orig_w > 0 && t_union->or_null->get_width_on_stack() == 1) { + // nothing except "T1 | T2 | ... null" can be cast to 1-slot nullable `T1?` + tolk_assert(o_union && o_union->has_null() && o_union->has_variant_with_type_id(t_union->or_null)); + // here we exploit rvect shape, how union types and multi-slot nullables are stored on a stack + // `T1 | T2 | ... | null` occupies N+1 slots, where the last is for UTag + // when it holds null value, N slots are null, and UTag slot is 0 (it's type_id of TypeDataNullLiteral) + return {rvect[rvect.size() - 2]}; + } + + // pass `null` to `T?` when T is not a primitive (1-slot T was handled above) + // - `null` to `(int, int)?` + // - `null` to `int | slice | null` + // to represent a non-primitive null value, we need N nulls + 1 null flag (UTag=0, type_id of TypeDataNullLiteral) + if (t_union && target_w > 1 && original_type == TypeDataNullLiteral::create()) { + tolk_assert(t_union->has_null()); + FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); + rvect.reserve(target_w); // keep rvect[0], it's already null + for (int i = 1; i < target_w - 1; ++i) { + std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); + code.emplace_back(loc, Op::_Call, ith_null, std::vector{}, null_sym); + rvect.push_back(ith_null[0]); } + std::vector last_null = code.create_tmp_var(TypeDataInt::create(), loc, "(UTag)"); + code.emplace_back(loc, Op::_IntConst, last_null, td::make_refint(0)); + rvect.push_back(last_null[0]); return rvect; } - // pass `T` to `T?` - // for primitives like `int?`, no changes in rvect: `int` and `int?` occupy the same TVM slot (null is represented as NULL TVM value) - // for passing `(int, int)` to `(int, int)?` / `(int, null)` to `(int, (int,int)?)?`, add a null flag equals to 0 - if (t_nullable && !o_nullable) { - if (!t_nullable->is_primitive_nullable()) { - rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, t_nullable->inner, loc); - tolk_assert(target_w == static_cast(rvect.size() + 1)); - std::vector null_flag_ir = code.create_tmp_var(TypeDataInt::create(), loc, "(NNFlag)"); - var_idx_t null_flag_ir_idx = null_flag_ir[0]; - code.emplace_back(loc, Op::_IntConst, std::move(null_flag_ir), td::make_refint(-1)); - rvect.push_back(null_flag_ir_idx); + + // pass `null` to nullable empty tensor + // - `null` to `()?` + // - `null` to `EmptyStruct?` + // so, rvect contains TVM NULL, but instead, we should push UTag=0 + if (t_union && original_type == TypeDataNullLiteral::create()) { + tolk_assert(t_union->or_null && t_union->or_null->get_width_on_stack() == 0); + std::vector new_rvect = code.create_tmp_var(TypeDataInt::create(), loc, "(UTag)"); + code.emplace_back(loc, Op::_IntConst, new_rvect, td::make_refint(0)); + return new_rvect; + } + + // smart cast of a wide nullable union to plain `null` + // - `(int, int)?` to `null` + // - `int | slice | null` to `null` + if (o_union && target_type == TypeDataNullLiteral::create() && orig_w > 1) { + tolk_assert(o_union->has_null()); + // if we are here, it's guaranteed that original value holds null + // it means, that its shape is N nulls + 1 UTag (equals 0) + return {rvect[rvect.size() - 2]}; + } + + // smart cast of nullable empty tensor to plain `null` + // - `()?` to `null` + // `EmptyStruct?` to `null` + // so, rvect contains UTag, we need TVM NULL + if (o_union && target_type == TypeDataNullLiteral::create()) { + tolk_assert(orig_w == 1 && o_union->or_null); + FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); + std::vector new_rvect = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); + code.emplace_back(loc, Op::_Call, new_rvect, std::vector{}, null_sym); + return new_rvect; + } + + // pass primitive 1-slot `T?` to a wider nullable union + // - `int?` to `int | slice | null` + // - `slice?` to `(int, int) | slice | builder | null` + // so, originally `T?` is 1-slot, but needs to be converted to N+1 slots, keeping its value + if (o_union && orig_w == 1 && t_union && o_union->or_null->get_width_on_stack() == 1) { + tolk_assert(t_union->has_null() && t_union->has_variant_with_type_id(o_union->or_null) && target_w > 1); + // the transformation is tricky: + // when value is null, we need to achieve "... (null) 0" (value is already null, so "... value 0") + // when value is not null, we need to get "... value {type_id}" + // this can be done only via IFs at runtime; luckily, this case is very uncommon in practice + // for "...", we might need N-1 nulls: `int?` to `(int,int,int) | int | null` is `(null) (null) value/(null) 0/1` + FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); + std::vector new_rvect; + new_rvect.resize(target_w); + for (int i = 0; i < target_w - 2; ++i) { // N-1 nulls + std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); + code.emplace_back(loc, Op::_Call, ith_null, std::vector{}, null_sym); + new_rvect[i] = ith_null[0]; } + new_rvect[target_w - 2] = rvect[0]; // value + new_rvect[target_w - 1] = code.create_tmp_var(TypeDataInt::create(), loc, "(UTag)")[0]; + + std::vector eq_null_cond = code.create_tmp_var(TypeDataBool::create(), loc, "(value-is-null)"); + FunctionPtr isnull_sym = lookup_global_symbol("__isNull")->try_as(); + code.emplace_back(loc, Op::_Call, eq_null_cond, rvect, isnull_sym); + Op& if_op = code.emplace_back(loc, Op::_If, eq_null_cond); + code.push_set_cur(if_op.block0); + code.emplace_back(loc, Op::_IntConst, std::vector{new_rvect[target_w - 1]}, td::make_refint(0)); + code.close_pop_cur(loc); + code.push_set_cur(if_op.block1); + code.emplace_back(loc, Op::_IntConst, std::vector{new_rvect[target_w - 1]}, td::make_refint(o_union->or_null->get_type_id())); + code.close_pop_cur(loc); + return new_rvect; + } + + // extend a single type into a union type + // - `int` to `int | slice` + // - `int` to `int | (int, int) | null` + // - `(int, int)` to `(int, int, cell) | builder | (int, int)` + // - `(int, null)` to `(int, (int, int)?) | ...`: mind transition + // - `(int, null)` to `(int, int | slice | null) | ...`: mind transition + // so, probably need to prepend some nulls, and need to append UTag + if (t_union && !o_union) { + TypePtr t_subtype = t_union->calculate_exact_variant_to_fit_rhs(original_type); + tolk_assert(t_subtype && target_w > t_subtype->get_width_on_stack()); + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, t_subtype, loc); + std::vector prepend_nulls; + prepend_nulls.reserve(target_w - t_subtype->get_width_on_stack() - 1); + for (int i = 0; i < target_w - t_subtype->get_width_on_stack() - 1; ++i) { + FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); + std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(UVar.null)"); + prepend_nulls.push_back(ith_null[0]); + code.emplace_back(loc, Op::_Call, std::move(ith_null), std::vector{}, null_sym); + } + rvect.insert(rvect.begin(), prepend_nulls.begin(), prepend_nulls.end()); + + std::vector last_utag = code.create_tmp_var(TypeDataInt::create(), loc, "(UTag)"); + code.emplace_back(loc, Op::_IntConst, last_utag, td::make_refint(t_subtype->get_type_id())); + rvect.push_back(last_utag[0]); return rvect; } - // pass `T1?` to `T2?` - // for example, `int8?` to `int16?` - // transition inner types, leaving nullable flag unchanged for tensors - if (t_nullable && o_nullable) { - if (target_w > 1) { - var_idx_t null_flag_ir_idx = rvect.back(); - rvect.pop_back(); - rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, o_nullable->inner, t_nullable->inner, loc); - rvect.push_back(null_flag_ir_idx); - } + + // smart cast a union type to a single type + // - `int | slice` to `int` + // - `int | (int, int) | null` to `int` + // - `(int, (int, int)?) | ...` to `(int, null)`: mind transition + // so, cut off UTag and probably some unused tags from the start + if (o_union && !t_union) { + TypePtr o_subtype = o_union->calculate_exact_variant_to_fit_rhs(target_type); + tolk_assert(o_subtype && orig_w > o_subtype->get_width_on_stack()); + rvect = std::vector(rvect.begin() + orig_w - o_subtype->get_width_on_stack() - 1, rvect.end() - 1); + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, o_subtype, target_type, loc); return rvect; } - // pass `T?` to `null` - // it may occur due to smart cast, when a `T?` variable is guaranteed to be always null - // (for instance, always-null `(int,int)?` will be represented as 1 TVM NULL value, not 3) - if (target_type == TypeDataNullLiteral::create() && original_type->can_rhs_be_assigned(target_type)) { - tolk_assert(o_nullable || original_type == TypeDataUnknown::create()); - if (o_nullable && !o_nullable->is_primitive_nullable()) { - FunctionPtr builtin_sym = lookup_global_symbol("__null")->try_as(); - rvect = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); - code.emplace_back(loc, Op::_Call, rvect, std::vector{}, builtin_sym); + + // extend a union type to a wider one + // - `int | slice` to `int | slice | builder` + // - `int | slice` to `int | (int, int) | slice | null` + // so, both original and target have UTag slot, but rvect probably needs to be prepended by nulls + if (t_union && o_union && t_union->variants.size() >= o_union->variants.size()) { + tolk_assert(target_w >= orig_w && t_union->has_all_variants_of(o_union)); + std::vector prepend_nulls; + prepend_nulls.reserve(target_w - orig_w); + for (int i = 0; i < target_w - orig_w; ++i) { + FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); + std::vector ith_null_ir_idx = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(UVar.null)"); + prepend_nulls.push_back(ith_null_ir_idx[0]); + code.emplace_back(loc, Op::_Call, std::move(ith_null_ir_idx), std::vector{}, null_sym); } + rvect.insert(rvect.begin(), prepend_nulls.begin(), prepend_nulls.end()); return rvect; } - // pass `T?` to `T` (or, more generally, `T1?` to `T2`) - // it may occur due to operator `!` or smart cast - // for primitives like `int?`, no changes in rvect - // for passing `(int, int)?` to `(int, int)`, drop the null flag from the tail - // for complex scenarios like passing `(int, (int,int)?)?` to `(int, null)`, recurse the call - // (it may occur on `someF(t = (3,null))` when `(3,null)` at first targeted to lhs, but actually its result is rhs) - if (!t_nullable && o_nullable) { - if (!o_nullable->is_primitive_nullable()) { - rvect.pop_back(); - rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type->try_as()->inner, target_type, loc); - } + + // smart cast a wider union type to a narrow one + // - `int | slice | builder` to `int | slice` + // - `int | (int, int) | slice | null` to `int | slice` + // so, both original and target have UTag slot, but rvect needs to be cut off from the left + if (t_union && o_union) { + tolk_assert(target_w <= orig_w && o_union->has_all_variants_of(t_union)); + rvect = std::vector(rvect.begin() + (orig_w - target_w), rvect.end()); return rvect; } + // pass `bool` to `int` // in code, it's done via `as` operator, like `boolVar as int` // no changes in rvect, boolVar is guaranteed to be -1 or 0 at TVM level if (original_type == TypeDataBool::create() && target_type == TypeDataInt::create()) { return rvect; } + // pass `bool` to `int8` // same as above if (original_type == TypeDataBool::create() && target_type->try_as()) { return rvect; } + // pass `int8` to `int` // it comes from auto cast when an integer (even a literal) is assigned to intN // to changes in rvect, intN is int at TVM level if (target_type == TypeDataInt::create() && original_type->try_as()) { return rvect; } + // pass `coins` to `int` // same as above if (target_type == TypeDataInt::create() && original_type == TypeDataCoins::create()) { return rvect; } + // pass `int` to `int8` // in code, it's probably done with `as` operator // no changes in rvect if (original_type == TypeDataInt::create() && target_type->try_as()) { return rvect; } + // pass `int` to `coins` // same as above if (original_type == TypeDataInt::create() && target_type == TypeDataCoins::create()) { return rvect; } + // pass `int8` to `int16` / `int8` to `uint8` // in code, it's probably done with `as` operator // no changes in rvect if (original_type->try_as() && target_type->try_as()) { return rvect; } + // pass `int8` to `coins` // same as above if (target_type == TypeDataCoins::create() && original_type->try_as()) { return rvect; } + // pass `coins` to `int8` // same as above if (original_type == TypeDataCoins::create() && target_type->try_as()) { return rvect; } + // pass `bytes32` to `slice` // in code, it's probably done with `as` operator // no changes in rvect, since bytesN is slice at TVM level if (target_type == TypeDataSlice::create() && original_type->try_as()) { return rvect; } + // pass `slice` to `bytes32` // same as above if (original_type == TypeDataSlice::create() && target_type->try_as()) { return rvect; } + // pass `bytes32` to `bytes64` / `bits128` to `bytes16` // no changes in rvect if (original_type->try_as() && target_type->try_as()) { return rvect; } + // pass something to `unknown` // probably, it comes from `_ = rhs`, type of `_` is unknown, it's target_type of rhs // no changes in rvect if (target_type == TypeDataUnknown::create()) { return rvect; } + // pass `unknown` to something // probably, it comes from `arg` in exception, it's inferred as `unknown` and could be cast to any value if (original_type == TypeDataUnknown::create()) { tolk_assert(rvect.size() == 1); return rvect; } + // pass tensor to tensor, e.g. `(1, null)` to `(int, slice?)` / `(1, null)` to `(int, (int,int)?)` // every element of rhs tensor should be transitioned if (target_type->try_as() && original_type->try_as()) { @@ -635,22 +792,14 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as() && original_type->try_as()) { - tolk_assert(target_type->get_width_on_stack() == original_type->get_width_on_stack()); + tolk_assert(target_w == 1 && orig_w == 1); return rvect; } - // pass `T` to `UserId` (or some other alias) - if (const TypeDataAlias* target_alias = target_type->try_as()) { - return transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, target_alias->underlying_type, loc); - } - // pass `UserId` (or some other alias) to `T` - if (const TypeDataAlias* orig_alias = original_type->try_as()) { - return transition_expr_to_runtime_type_impl(std::move(rvect), code, orig_alias->underlying_type, target_type, loc); - } - // pass callable to callable // their types aren't exactly equal, but they match (containing aliases, for example) if (original_type->try_as() && target_type->try_as()) { @@ -817,7 +966,7 @@ static std::vector process_ternary_operator(V v std::vector cond = pre_compile_expr(v->get_cond(), code, nullptr); tolk_assert(cond.size() == 1); std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(cond)"); - + if (v->get_cond()->is_always_true) { code.emplace_back(v->get_when_true()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_true(), code, v->inferred_type)); } else if (v->get_cond()->is_always_false) { @@ -841,40 +990,91 @@ static std::vector process_cast_as_operator(V v return transition_to_target_type(std::move(rvect), code, target_type, v); } -static std::vector process_not_null_operator(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { - TypePtr child_target_type = v->get_expr()->inferred_type->unwrap_alias(); - if (const auto* as_nullable = child_target_type->try_as()) { - child_target_type = as_nullable->inner; +static std::vector process_is_type_operator(V v, CodeBlob& code, TypePtr target_type) { + TypePtr lhs_type = v->get_expr()->inferred_type->unwrap_alias(); + TypePtr cmp_type = v->rhs_type->unwrap_alias(); + bool is_null_check = cmp_type == TypeDataNullLiteral::create(); // `v == null`, not `v is T` + tolk_assert(!cmp_type->try_as()); // `v is int|slice` is a type checker error + + std::vector expr_ir_idx = pre_compile_expr(v->get_expr(), code, nullptr); + std::vector result_ir_idx = pre_compile_is_type(code, lhs_type, cmp_type, expr_ir_idx, v->loc, is_null_check ? "(is-null)" : "(is-type)"); + + if (v->is_negated) { + FunctionPtr not_sym = lookup_global_symbol("!b_")->try_as(); + code.emplace_back(v->loc, Op::_Call, result_ir_idx, result_ir_idx, not_sym); } + return transition_to_target_type(std::move(result_ir_idx), code, target_type, v); +} + +static std::vector process_not_null_operator(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); + TypePtr without_null_type = calculate_type_subtract_rhs_type(expr_type, TypeDataNullLiteral::create()); + TypePtr child_target_type = without_null_type != TypeDataNever::create() ? without_null_type : expr_type; + std::vector rvect = pre_compile_expr(v->get_expr(), code, child_target_type, lval_ctx); return transition_to_target_type(std::move(rvect), code, target_type, v); } -static std::vector process_is_null_check(V v, CodeBlob& code, TypePtr target_type) { - std::vector expr_ir_idx = pre_compile_expr(v->get_expr(), code, nullptr); - std::vector isnull_ir_idx = code.create_tmp_var(TypeDataBool::create(), v->loc, "(is-null)"); - TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); +static std::vector process_match_expression(V v, CodeBlob& code, TypePtr target_type) { + TypePtr lhs_type = v->get_subject()->inferred_type->unwrap_alias(); + + std::vector subj_ir_idx = pre_compile_expr(v->get_subject(), code, nullptr); + std::vector result_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(match-expression)"); + + // detect whether it's `match` by type covering all cases; type checker has already checked that + // match by type can not be mixed with match by expression and can't contain `else`, and that types are distinct and cover all variants + bool is_exhaustive_match_by_type = v->get_arms_count() > 0 && v->get_arm(0)->pattern_kind == MatchArmKind::exact_type; + if (is_exhaustive_match_by_type) { + // example: `match (v) { int => ... slice => ... builder => ... }` + // construct nested IFs: IF is int { ... } ELSE { IF is slice { ... } ELSE { ... } } + for (int i = 0; i < v->get_arms_count() - 1; ++i) { + auto v_ith_arm = v->get_arm(i); + TypePtr cmp_type = v_ith_arm->exact_type->unwrap_alias(); + tolk_assert(!cmp_type->try_as()); // `match` over `int|slice` is a type checker error + std::vector eq_ith_ir_idx = pre_compile_is_type(code, lhs_type, cmp_type, subj_ir_idx, v_ith_arm->loc, "(arm-cond-eq)"); + Op& if_op = code.emplace_back(v_ith_arm->loc, Op::_If, std::move(eq_ith_ir_idx)); + code.push_set_cur(if_op.block0); + if (v->is_statement()) { + pre_compile_expr(v_ith_arm->get_body(), code); + } else { + std::vector arm_ir_idx = pre_compile_expr(v_ith_arm->get_body(), code, v->inferred_type); + code.emplace_back(v->loc, Op::_Let, result_ir_idx, std::move(arm_ir_idx)); + } + code.close_pop_cur(v->loc); + code.push_set_cur(if_op.block1); // open ELSE + } - if (const TypeDataNullable* t_nullable = expr_type->try_as()) { - if (!t_nullable->is_primitive_nullable()) { - std::vector zero_ir_idx = code.create_tmp_var(TypeDataInt::create(), v->loc, "(zero)"); - code.emplace_back(v->loc, Op::_IntConst, zero_ir_idx, td::make_refint(0)); - FunctionPtr eq_sym = lookup_global_symbol("_==_")->try_as(); - code.emplace_back(v->loc, Op::_Call, isnull_ir_idx, std::vector{expr_ir_idx.back(), zero_ir_idx[0]}, eq_sym); + // we're inside the last ELSE, close all outer IFs + auto v_last_arm = v->get_arm(v->get_arms_count() - 1); + if (v->is_statement()) { + pre_compile_expr(v_last_arm->get_body(), code); } else { - FunctionPtr builtin_sym = lookup_global_symbol("__isNull")->try_as(); - code.emplace_back(v->loc, Op::_Call, isnull_ir_idx, expr_ir_idx, builtin_sym); + std::vector arm_ir_idx = pre_compile_expr(v_last_arm->get_body(), code, v->inferred_type); + code.emplace_back(v->loc, Op::_Let, result_ir_idx, std::move(arm_ir_idx)); + } + for (int i = 0; i < v->get_arms_count() - 1; ++i) { + code.close_pop_cur(v->loc); } + } else { - bool always_null = expr_type == TypeDataNullLiteral::create(); - code.emplace_back(v->loc, Op::_IntConst, isnull_ir_idx, td::make_refint(always_null ? -1 : 0)); + // not exhaustive `match` by type + for (int i = 0; i < v->get_arms_count(); ++i) { + auto v_arm = v->get_arm(i); + switch (v_arm->pattern_kind) { + case MatchArmKind::const_expression: + tolk_assert(false); // match by expressions is not supported yet, checked earlier + break; + case MatchArmKind::exact_type: + tolk_assert(false); // match by type is exhaustive, checked earlier + break; + case MatchArmKind::else_branch: + tolk_assert(false); // `else` in match can be only inside match by expression, unsupported yet + break; + } + } } - if (v->is_negated) { - FunctionPtr not_sym = lookup_global_symbol("!b_")->try_as(); - code.emplace_back(v->loc, Op::_Call, isnull_ir_idx, std::vector{isnull_ir_idx}, not_sym); - } - return transition_to_target_type(std::move(isnull_ir_idx), code, target_type, v); + return transition_to_target_type(std::move(result_ir_idx), code, target_type, v); } static std::vector process_dot_access(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { @@ -1048,6 +1248,15 @@ static std::vector process_function_call(V v, Code return transition_to_target_type(std::move(rvect_apply), code, target_type, v); } +static std::vector process_braced_expression(V v, CodeBlob& code, TypePtr target_type) { + // `{ ... }` used as an expression can not return a value currently (there is no syntax in a language) + // that's why it can appear only in special places, and its usage correctness has been checked + tolk_assert(v->inferred_type == TypeDataVoid::create() || v->inferred_type == TypeDataNever::create()); + process_any_statement(v->get_sequence(), code); + static_cast(target_type); + return {}; +} + static std::vector process_tensor(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { // tensor is compiled "as is", for example `(1, null)` occupies 2 slots // and if assigned/passed to something other, like `(int, (int,int)?)`, a whole tensor is transitioned, it works @@ -1068,6 +1277,8 @@ static std::vector process_typed_tuple(V v, CodeBlob static std::vector process_int_const(V v, CodeBlob& code, TypePtr target_type) { std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(int-const)"); code.emplace_back(v->loc, Op::_IntConst, rvect, v->intval); + // here, like everywhere, even for just `int`, there might be a potential transition due to union types + // example: passing `1` to `int | slice` puts actually "1 5" on a stack (1 for value, 5 for UTag = type_id of `int`) return transition_to_target_type(std::move(rvect), code, target_type, v); } @@ -1113,6 +1324,11 @@ static std::vector process_underscore(V v, CodeBlob& return code.create_tmp_var(v->inferred_type, v->loc, "(underscore)"); } +static std::vector process_empty_expression(V v, CodeBlob& code, TypePtr target_type) { + std::vector empty_rvect; + return transition_to_target_type(std::move(empty_rvect), code, target_type, v); +} + std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { switch (v->type) { case ast_reference: @@ -1129,16 +1345,20 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ return process_ternary_operator(v->as(), code, target_type); case ast_cast_as_operator: return process_cast_as_operator(v->as(), code, target_type, lval_ctx); + case ast_is_type_operator: + return process_is_type_operator(v->as(), code, target_type); case ast_not_null_operator: return process_not_null_operator(v->as(), code, target_type, lval_ctx); - case ast_is_null_check: - return process_is_null_check(v->as(), code, target_type); + case ast_match_expression: + return process_match_expression(v->as(), code, target_type); case ast_dot_access: return process_dot_access(v->as(), code, target_type, lval_ctx); case ast_function_call: return process_function_call(v->as(), code, target_type); case ast_parenthesized_expression: return pre_compile_expr(v->as()->get_expr(), code, target_type, lval_ctx); + case ast_braced_expression: + return process_braced_expression(v->as(), code, target_type); case ast_tensor: return process_tensor(v->as(), code, target_type, lval_ctx); case ast_typed_tuple: @@ -1157,6 +1377,8 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ return process_local_vars_declaration(v->as(), code); case ast_underscore: return process_underscore(v->as(), code); + case ast_empty_expression: + return process_empty_expression(v->as(), code, target_type); default: throw UnexpectedASTNodeType(v, "pre_compile_expr"); } @@ -1313,14 +1535,12 @@ static void process_throw_statement(V v, CodeBlob& code) { } static void process_return_statement(V v, CodeBlob& code) { - std::vector return_vars; - if (v->has_return_value()) { - TypePtr child_target_type = code.fun_ref->inferred_return_type; - if (code.fun_ref->does_return_self()) { - child_target_type = code.fun_ref->parameters[0].declared_type; - } - return_vars = pre_compile_expr(v->get_return_value(), code, child_target_type); + TypePtr child_target_type = code.fun_ref->inferred_return_type; + if (code.fun_ref->does_return_self()) { + child_target_type = code.fun_ref->parameters[0].declared_type; } + std::vector return_vars = pre_compile_expr(v->get_return_value(), code, child_target_type); + if (code.fun_ref->does_return_self()) { return_vars = {}; } diff --git a/tolk/pipe-calc-rvalue-lvalue.cpp b/tolk/pipe-calc-rvalue-lvalue.cpp index 1f374bc813..769d31c010 100644 --- a/tolk/pipe-calc-rvalue-lvalue.cpp +++ b/tolk/pipe-calc-rvalue-lvalue.cpp @@ -24,6 +24,8 @@ * * Example: `a = b`, `a` is lvalue, `b` is rvalue. * Example: `a + b`, both are rvalue. + * Example: `a;`, it's none (not rvalue, not lvalue). + * For instance, difference between none and rvalue helps detect whether `match` is expression or statement. * * Note, that this pass only assigns, not checks. So, for `f() = 4`, expr `f()` is lvalue. * Checking (firing this as incorrect later) is performed after type inferring, see pipe-check-rvalue-lvalue. @@ -47,79 +49,101 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { return saved; } + MarkingState enter_rvalue_if_none() { + if (cur_state != MarkingState::None) { + return cur_state; + } + cur_state = MarkingState::RValue; + return MarkingState::None; + } + void restore_state(MarkingState saved) { cur_state = saved; } - void mark_vertex_cur_or_rvalue(AnyExprV v) const { + void mark_vertex(AnyExprV v) const { if (cur_state == MarkingState::LValue || cur_state == MarkingState::LValueAndRValue) { v->mutate()->assign_lvalue_true(); } - if (cur_state == MarkingState::RValue || cur_state == MarkingState::LValueAndRValue || cur_state == MarkingState::None) { + if (cur_state == MarkingState::RValue || cur_state == MarkingState::LValueAndRValue) { v->mutate()->assign_rvalue_true(); } } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); + MarkingState saved = enter_rvalue_if_none(); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + mark_vertex(v); + MarkingState saved = enter_state(MarkingState::None); parent::visit(v); + restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); + MarkingState saved = enter_rvalue_if_none(); parent::visit(v); + restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); + MarkingState saved = enter_rvalue_if_none(); parent::visit(v); + restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(v->passed_as_mutate ? MarkingState::LValueAndRValue : MarkingState::RValue); parent::visit(v); restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + tolk_assert(cur_state == MarkingState::RValue); + mark_vertex(v); parent::visit(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); parent::visit(v->get_obj()); restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); parent::visit(v); restore_state(saved); @@ -130,11 +154,11 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { // so, if current state is "lvalue", `_` will be marked as lvalue, and ok // but if used incorrectly, like `f(_)` or just `_;`, it will be marked rvalue // and will fire an error later, in pipe lvalue/rvalue check - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::LValue); parent::visit(v->get_lhs()); enter_state(MarkingState::RValue); @@ -143,7 +167,7 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::LValueAndRValue); parent::visit(v->get_lhs()); enter_state(MarkingState::RValue); @@ -152,53 +176,114 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); parent::visit(v); restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); parent::visit(v); restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); parent::visit(v); // both cond, when_true and when_false are rvalue, `(cond ? a : b) = 5` prohibited restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); parent::visit(v->get_expr()); // leave lvalue state unchanged, for `mutate (t.0 as int)` both `t.0 as int` and `t.0` are lvalue } + void visit(V v) override { + mark_vertex(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v->get_expr()); + restore_state(saved); + } + void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); parent::visit(v->get_expr()); // leave lvalue state unchanged, for `mutate x!` both `x!` and `x` are lvalue } - void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + void visit(V v) override { + mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); - parent::visit(v->get_expr()); + parent::visit(v); restore_state(saved); } + void visit(V v) override { + tolk_assert(cur_state == MarkingState::RValue); + mark_vertex(v); + parent::visit(v); + } + void visit(V v) override { tolk_assert(cur_state == MarkingState::LValue); - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); parent::visit(v); } void visit(V v) override { tolk_assert(cur_state == MarkingState::LValue); - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); + parent::visit(v); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::None); parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); } void visit(V v) override { diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 927cfc4e19..2174560fdb 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -119,12 +119,14 @@ static void handle_possible_compiler_internal_call(FunctionPtr cur_f, Vget_num_args() == 2); TypePtr expected_type = parse_type_from_string(v->get_arg(1)->get_expr()->as()->str_val); if (expected_type->has_unresolved_inside()) { // only aliases can be inside - expected_type = expected_type->replace_children_custom([](TypePtr child) -> TypePtr { + expected_type = expected_type->replace_children_custom([cur_f, v](TypePtr child) -> TypePtr { if (const auto* as_unresolved = child->try_as()) { const Symbol* sym = lookup_global_symbol(as_unresolved->text); - tolk_assert(sym); + if (!sym) { + throw ParseError(cur_f, v->loc, "invalid __expect_type"); + } if (AliasDefPtr alias_ref = sym->try_as()) { - return TypeDataAlias::create(alias_ref->name, alias_ref->underlying_type); + return TypeDataAlias::create(alias_ref); } tolk_assert(false); } @@ -132,7 +134,7 @@ static void handle_possible_compiler_internal_call(FunctionPtr cur_f, Vget_arg(0)->inferred_type; - if (expected_type != expr_type) { + if (expected_type->as_human_readable() != expr_type->as_human_readable()) { fire(cur_f, v->loc, "__expect_type failed: expected " + to_string(expected_type) + ", got " + to_string(expr_type)); } } @@ -282,25 +284,29 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } - void visit(V v) override { + void visit(V v) override { parent::visit(v->get_expr()); - if (v->get_expr()->inferred_type == TypeDataNullLiteral::create()) { - // operator `!` used for always-null (proven by smart casts, for example), it's an error - fire(cur_f, v->loc, "operator `!` used for always null expression"); + if (v->rhs_type->unwrap_alias()->try_as()) { // `v is T1 | T2` / `v is T?` is disallowed + fire(cur_f, v->loc, "union types are not allowed, use concrete types in `is`"); } - // if operator `!` used for non-nullable, probably a warning should be printed - } - - void visit(V v) override { - parent::visit(v->get_expr()); if ((v->is_always_true && !v->is_negated) || (v->is_always_false && v->is_negated)) { - v->loc.show_warning(expression_as_string(v->get_expr()) + " is always null, this condition is always " + (v->is_always_true ? "true" : "false")); + v->loc.show_warning(expression_as_string(v->get_expr()) + " is always " + to_string(v->rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); } if ((v->is_always_false && !v->is_negated) || (v->is_always_true && v->is_negated)) { - v->loc.show_warning(expression_as_string(v->get_expr()) + " of type " + to_string(v->get_expr()) + " is always not null, this condition is always " + (v->is_always_true ? "true" : "false")); + v->loc.show_warning(expression_as_string(v->get_expr()) + " of type " + to_string(v->get_expr()) + " can never be " + to_string(v->rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); + } + } + + void visit(V v) override { + parent::visit(v->get_expr()); + + if (v->get_expr()->inferred_type == TypeDataNullLiteral::create()) { + // operator `!` used for always-null (proven by smart casts, for example), it's an error + fire(cur_f, v->loc, "operator `!` used for always null expression"); } + // if operator `!` used for non-nullable, probably a warning should be printed } void visit(V v) override { @@ -318,8 +324,8 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { parent::visit(v); if (v->is_target_indexed_access()) { - TypePtr obj_type = v->get_obj()->inferred_type; - if (obj_type->unwrap_alias()->try_as() && v->inferred_type->get_width_on_stack() != 1) { + TypePtr obj_type = v->get_obj()->inferred_type->unwrap_alias(); + if (v->inferred_type->get_width_on_stack() != 1 && (obj_type->try_as() || obj_type->try_as())) { fire_error_cannot_put_non1_stack_width_arg_to_tuple(cur_f, v->loc, v->inferred_type); } } @@ -513,6 +519,82 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } + void visit(V v) override { + parent::visit(v); + + bool has_type_arm = false; + bool has_expr_arm = false; + bool has_else_arm = false; + TypePtr subject_type = v->get_subject()->inferred_type->unwrap_alias(); + const TypeDataUnion* subject_union = subject_type->try_as(); + + std::vector covered_type_ids; // for type-based `match`, what types are on the left of `=>` + + for (int i = 0; i < v->get_arms_count(); ++i) { + auto v_arm = v->get_arm(i); + switch (v_arm->pattern_kind) { + case MatchArmKind::exact_type: { + if (has_expr_arm) { + fire(cur_f, v_arm->loc, "can not mix type and expression patterns in `match`"); + } + if (has_else_arm) { + fire(cur_f, v_arm->loc, "`else` branch should be the last"); + } + has_type_arm = true; + + TypePtr lhs_type = v_arm->exact_type->unwrap_alias(); // `lhs_type => ...` + if (lhs_type->try_as()) { + fire(cur_f, v_arm->loc, "wrong pattern matching: union types are not allowed, use concrete types in `match`"); + } + bool can_happen = (subject_union && subject_union->has_variant_with_type_id(lhs_type)) || + (!subject_union && subject_type->equal_to(lhs_type)); + if (!can_happen) { + fire(cur_f, v_arm->loc, "wrong pattern matching: " + to_string(lhs_type) + " is not a variant of " + to_string(subject_type)); + } + if (std::find(covered_type_ids.begin(), covered_type_ids.end(), lhs_type->get_type_id()) != covered_type_ids.end()) { + fire(cur_f, v_arm->loc, "wrong pattern matching: duplicated " + to_string(lhs_type)); + } + covered_type_ids.push_back(lhs_type->get_type_id()); + break; + } + case MatchArmKind::const_expression: { + if (has_type_arm) { + fire(cur_f, v_arm->loc, "can not mix type and expression patterns in `match`"); + } + if (has_else_arm) { + fire(cur_f, v_arm->loc, "`else` branch should be the last"); + } + has_expr_arm = true; + fire(cur_f, v_arm->loc, "`match` by expression is not supported yet"); + break; + } + default: + if (has_else_arm) { + fire(cur_f, v_arm->loc, "duplicated `else` branch"); + } + if (has_type_arm) { + fire(cur_f, v_arm->loc, "`else` is not allowed in `match` by type; you should cover all possible types"); + } + has_else_arm = true; + fire(cur_f, v_arm->loc, "`match` by expression is not supported yet"); + } + } + + // fire if `match` by type is not exhaustive + if (has_type_arm && subject_union && subject_union->variants.size() != covered_type_ids.size()) { + std::string missing; + for (TypePtr variant : subject_union->variants) { + if (std::find(covered_type_ids.begin(), covered_type_ids.end(), variant->get_type_id()) == covered_type_ids.end()) { + if (!missing.empty()) { + missing += ", "; + } + missing += to_string(variant); + } + } + throw ParseError(cur_f, v->loc, "`match` does not cover all possible types; missing types are: " + missing); + } + } + void visit(V v) override { parent::visit(v); diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp index 3ec47a16b5..8860911276 100644 --- a/tolk/pipe-check-rvalue-lvalue.cpp +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -29,65 +29,80 @@ namespace tolk { +// fire a general error, just a wrapper over `throw` GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_cannot_be_used_as_lvalue(AnyV v, const std::string& details) { +static void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { + throw ParseError(cur_f, loc, message); +} + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_cannot_be_used_as_lvalue(FunctionPtr cur_f, AnyV v, const std::string& details) { // example: `f() = 32` // example: `loadUint(c.beginParse(), 32)` (since `loadUint()` mutates the first argument) - v->error(details + " can not be used as lvalue"); + throw ParseError(cur_f, v->loc, details + " can not be used as lvalue"); } GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_modifying_immutable_variable(AnyExprV v, LocalVarPtr var_ref) { +static void fire_error_modifying_immutable_variable(FunctionPtr cur_f, AnyExprV v, LocalVarPtr var_ref) { if (var_ref->param_idx == 0 && var_ref->name == "self") { - v->error("modifying `self`, which is immutable by default; probably, you want to declare `mutate self`"); + throw ParseError(cur_f, v->loc, "modifying `self`, which is immutable by default; probably, you want to declare `mutate self`"); } else { - v->error("modifying immutable variable `" + var_ref->name + "`"); + throw ParseError(cur_f, v->loc, "modifying immutable variable `" + var_ref->name + "`"); } } // validate a function used as rvalue, like `var cb = f` // it's not a generic function (ensured earlier at type inferring) and has some more restrictions -static void validate_function_used_as_noncall(AnyExprV v, FunctionPtr fun_ref) { +static void validate_function_used_as_noncall(FunctionPtr cur_f, AnyExprV v, FunctionPtr fun_ref) { if (!fun_ref->arg_order.empty() || !fun_ref->ret_order.empty()) { - v->error("saving `" + fun_ref->name + "` into a variable will most likely lead to invalid usage, since it changes the order of variables on the stack"); + fire(cur_f, v->loc, "saving `" + fun_ref->name + "` into a variable will most likely lead to invalid usage, since it changes the order of variables on the stack"); } if (fun_ref->has_mutate_params()) { - v->error("saving `" + fun_ref->name + "` into a variable is impossible, since it has `mutate` parameters and thus can only be called directly"); + fire(cur_f, v->loc, "saving `" + fun_ref->name + "` into a variable is impossible, since it has `mutate` parameters and thus can only be called directly"); } } class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(cur_f, v, "braced expression"); + } + parent::visit(v); + } + void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "assignment"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "assignment"); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "assignment"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "assignment"); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "operator " + static_cast(v->operator_name)); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "operator " + static_cast(v->operator_name)); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "operator " + static_cast(v->operator_name)); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "operator " + static_cast(v->operator_name)); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "operator ?:"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "operator ?:"); } parent::visit(v); } @@ -97,52 +112,52 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { parent::visit(v->get_expr()); } - void visit(V v) override { - // if `x!` is lvalue, then `x` is also lvalue, so check that `x` is ok + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(cur_f, v, v->is_negated ? "operator !is" : "operator is"); + } parent::visit(v->get_expr()); } - void visit(V v) override { - if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, v->is_negated ? "operator !=" : "operator =="); - } + void visit(V v) override { + // if `x!` is lvalue, then `x` is also lvalue, so check that `x` is ok parent::visit(v->get_expr()); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "literal"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "literal"); } } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "literal"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "literal"); } } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "literal"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "literal"); } } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "literal"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "literal"); } } void visit(V v) override { // a reference to a method used as rvalue, like `var v = t.tupleAt` if (v->is_rvalue && v->is_target_fun_ref()) { - validate_function_used_as_noncall(v, std::get(v->target)); + validate_function_used_as_noncall(cur_f, v, std::get(v->target)); } } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "function call"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "function call"); } if (!v->fun_maybe) { parent::visit(v->get_callee()); @@ -158,11 +173,18 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { } } + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(cur_f, v, "`match` expression"); + } + parent::visit(v); + } + void visit(V v) override { if (v->marked_as_redef) { tolk_assert(v->var_ref); if (v->var_ref->is_immutable()) { - v->error("`redef` for immutable variable"); + fire(cur_f, v->loc, "`redef` for immutable variable"); } } } @@ -171,23 +193,23 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { if (v->is_lvalue) { tolk_assert(v->sym); if (LocalVarPtr var_ref = v->sym->try_as(); var_ref && var_ref->is_immutable()) { - fire_error_modifying_immutable_variable(v, var_ref); + fire_error_modifying_immutable_variable(cur_f, v, var_ref); } else if (v->sym->try_as()) { - v->error("modifying immutable constant"); + fire(cur_f, v->loc, "modifying immutable constant"); } else if (v->sym->try_as()) { - v->error("function can't be used as lvalue"); + fire(cur_f, v->loc, "function can't be used as lvalue"); } } // a reference to a function used as rvalue, like `var v = someFunction` if (FunctionPtr fun_ref = v->sym->try_as(); fun_ref && v->is_rvalue) { - validate_function_used_as_noncall(v, fun_ref); + validate_function_used_as_noncall(cur_f, v, fun_ref); } } void visit(V v) override { if (v->is_rvalue) { - v->error("`_` can't be used as a value; it's a placeholder for a left side of assignment"); + fire(cur_f, v->loc, "`_` can't be used as a value; it's a placeholder for a left side of assignment"); } } @@ -201,6 +223,12 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + cur_f = nullptr; + } }; void pipeline_check_rvalue_lvalue() { diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index 13ecece516..1a071f4923 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -96,11 +96,11 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { return v; } - AnyExprV replace(V v) override { + AnyExprV replace(V v) override { parent::replace(v); // `null == null` / `null != null` - if (v->get_expr()->type == ast_null_keyword) { + if (v->get_expr()->type == ast_null_keyword && v->rhs_type == TypeDataNullLiteral::create()) { return create_bool_const(v->loc, !v->is_negated); } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 236037c3e7..c0c7c1d39d 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -238,12 +238,14 @@ class InferTypesAndCallsAndFieldsVisitor final { return infer_ternary_operator(v->as(), std::move(flow), used_as_condition, hint); case ast_cast_as_operator: return infer_cast_as_operator(v->as(), std::move(flow), used_as_condition); + case ast_is_type_operator: + return infer_is_type_operator(v->as(), std::move(flow), used_as_condition); case ast_not_null_operator: return infer_not_null_operator(v->as(), std::move(flow), used_as_condition); - case ast_is_null_check: - return infer_is_null_check(v->as(), std::move(flow), used_as_condition); case ast_parenthesized_expression: return infer_parenthesized(v->as(), std::move(flow), used_as_condition, hint); + case ast_braced_expression: + return infer_braced_expression(v->as(), std::move(flow), used_as_condition); case ast_reference: return infer_reference(v->as(), std::move(flow), used_as_condition); case ast_dot_access: @@ -256,6 +258,8 @@ class InferTypesAndCallsAndFieldsVisitor final { return infer_typed_tuple(v->as(), std::move(flow), used_as_condition, hint); case ast_null_keyword: return infer_null_keyword(v->as(), std::move(flow), used_as_condition); + case ast_match_expression: + return infer_match_expression(v->as(), std::move(flow), used_as_condition, hint); case ast_underscore: return infer_underscore(v->as(), std::move(flow), used_as_condition, hint); case ast_empty_expression: @@ -589,12 +593,18 @@ class InferTypesAndCallsAndFieldsVisitor final { return after_false; } - TypeInferringUnifyStrategy tern_type; - tern_type.unify_with(v->get_when_true()->inferred_type); - if (!tern_type.unify_with(v->get_when_false()->inferred_type)) { - fire(cur_f, v->loc, "types of ternary branches are incompatible: " + to_string(v->get_when_true()) + " and " + to_string(v->get_when_false())); + TypeInferringUnifyStrategy branches_unifier; + branches_unifier.unify_with(v->get_when_true()->inferred_type, hint); + branches_unifier.unify_with(v->get_when_false()->inferred_type, hint); + if (branches_unifier.is_union_of_different_types()) { + // `... ? intVar : sliceVar` results in `int | slice`, probably it's not what the user expected + // example: `var v = ternary`, show an inference error + // do NOT show an error for `var v: T = ternary` (T is hint); it will be checked by type checker later + if (hint == nullptr || hint == TypeDataUnknown::create()) { + fire(cur_f, v->loc, "types of ternary branches are incompatible: " + to_string(v->get_when_true()) + " and " + to_string(v->get_when_false())); + } } - assign_inferred_type(v, tern_type.get_result()); + assign_inferred_type(v, branches_unifier.get_result()); FlowContext out_flow = FlowContext::merge_flow(std::move(after_true.out_flow), std::move(after_false.out_flow)); return ExprFlow(std::move(out_flow), std::move(after_true.true_flow), std::move(after_false.false_flow)); @@ -611,15 +621,16 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(after_expr.out_flow), true); } - ExprFlow infer_is_null_check(V v, FlowContext&& flow, bool used_as_condition) { + ExprFlow infer_is_type_operator(V v, FlowContext&& flow, bool used_as_condition) { ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); assign_inferred_type(v, TypeDataBool::create()); + TypePtr rhs_type = v->rhs_type->unwrap_alias(); TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); - TypePtr non_null_type = calculate_type_subtract_null(expr_type); - if (expr_type == TypeDataNullLiteral::create()) { // `expr == null` is always true + TypePtr non_rhs_type = calculate_type_subtract_rhs_type(expr_type, rhs_type); + if (expr_type->equal_to(v->rhs_type)) { // `expr is ` is always true v->mutate()->assign_always_true_or_false(v->is_negated ? 2 : 1); - } else if (non_null_type == TypeDataNever::create()) { // `expr == null` is always false + } else if (non_rhs_type == TypeDataNever::create()) { // `expr is ` is always false v->mutate()->assign_always_true_or_false(v->is_negated ? 1 : 2); } else { v->mutate()->assign_always_true_or_false(0); @@ -639,11 +650,11 @@ class InferTypesAndCallsAndFieldsVisitor final { true_flow.mark_unreachable(UnreachableKind::CantHappen); true_flow.register_known_type(s_expr, TypeDataNever::create()); } else if (!v->is_negated) { - true_flow.register_known_type(s_expr, TypeDataNullLiteral::create()); - false_flow.register_known_type(s_expr, non_null_type); + true_flow.register_known_type(s_expr, rhs_type); + false_flow.register_known_type(s_expr, non_rhs_type); } else { - true_flow.register_known_type(s_expr, non_null_type); - false_flow.register_known_type(s_expr, TypeDataNullLiteral::create()); + true_flow.register_known_type(s_expr, non_rhs_type); + false_flow.register_known_type(s_expr, rhs_type); } } return ExprFlow(std::move(after_expr.out_flow), std::move(true_flow), std::move(false_flow)); @@ -651,12 +662,9 @@ class InferTypesAndCallsAndFieldsVisitor final { ExprFlow infer_not_null_operator(V v, FlowContext&& flow, bool used_as_condition) { ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); - - if (const auto* as_nullable = v->get_expr()->inferred_type->unwrap_alias()->try_as()) { - assign_inferred_type(v, as_nullable->inner); - } else { - assign_inferred_type(v, v->get_expr()); - } + TypePtr expr_type = v->get_expr()->inferred_type; + TypePtr without_null_type = calculate_type_subtract_rhs_type(expr_type->unwrap_alias(), TypeDataNullLiteral::create()); + assign_inferred_type(v, without_null_type != TypeDataNever::create() ? without_null_type : expr_type); if (!used_as_condition) { return after_expr; @@ -670,6 +678,13 @@ class InferTypesAndCallsAndFieldsVisitor final { return after_expr; } + ExprFlow infer_braced_expression(V v, FlowContext&& flow, bool used_as_condition) { + // `{ ... }` used as an expression can not return a value currently (there is no syntax in a language) + flow = process_any_statement(v->get_sequence(), std::move(flow)); + assign_inferred_type(v, flow.is_unreachable() ? TypeDataNever::create() : TypeDataVoid::create()); + return ExprFlow(std::move(flow), used_as_condition); + } + ExprFlow infer_reference(V v, FlowContext&& flow, bool used_as_condition) { if (LocalVarPtr var_ref = v->sym->try_as()) { TypePtr declared_or_smart_casted = flow.smart_cast_if_exists(SinkExpression(var_ref)); @@ -1058,6 +1073,45 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } + ExprFlow infer_match_expression(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + flow = infer_any_expr(v->get_subject(), std::move(flow), false).out_flow; + SinkExpression s_expr = extract_sink_expression_from_vertex(v->get_subject()); + TypeInferringUnifyStrategy branches_unifier; + FlowContext arms_entry_facts = flow.clone(); + FlowContext match_out_flow; + + for (int i = 0; i < v->get_arms_count(); ++i) { + auto v_arm = v->get_arm(i); + FlowContext arm_flow = infer_any_expr(v_arm->get_pattern_expr(), arms_entry_facts.clone(), false, nullptr).out_flow; + if (s_expr && v_arm->pattern_kind == MatchArmKind::exact_type) { + arm_flow.register_known_type(s_expr, v_arm->exact_type); + } + arm_flow = infer_any_expr(v_arm->get_body(), std::move(arm_flow), false, hint).out_flow; + match_out_flow = i == 0 ? std::move(arm_flow) : FlowContext::merge_flow(std::move(match_out_flow), std::move(arm_flow)); + branches_unifier.unify_with(v_arm->get_body()->inferred_type, hint); + } + if (v->get_arms_count() == 0) { + match_out_flow = std::move(arms_entry_facts); + } + + if (v->is_statement()) { + assign_inferred_type(v, TypeDataVoid::create()); + } else { + if (v->get_arms_count() == 0) { // still allow empty `match` statements, for probable codegen + fire(cur_f, v->loc, "empty `match` can't be used as expression"); + } + if (branches_unifier.is_union_of_different_types()) { + // same as in ternary: `match (...) { t1 => someSlice, t2 => someInt }` is `int|slice`, probably unexpected + if (hint == nullptr || hint == TypeDataUnknown::create()) { + fire(cur_f, v->loc, "type of `match` was inferred as " + to_string(branches_unifier.get_result()) + "; probably, it's not what you expected; assign it to a variable `var v: = match (...) { ... }` manually"); + } + } + assign_inferred_type(v, branches_unifier.get_result()); + } + + return ExprFlow(std::move(match_out_flow), used_as_condition); + } + static ExprFlow infer_underscore(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { // if execution is here, underscore is either used as lhs of assignment, or incorrectly, like `f(_)` // more precise is to always set unknown here, but for incorrect usages, instead of an error @@ -1229,21 +1283,35 @@ class InferTypesAndCallsAndFieldsVisitor final { if (fun_ref->does_return_self()) { return_unifier.unify_with(fun_ref->parameters[0].declared_type); } - for (AnyExprV return_value : return_statements) { - if (!return_unifier.unify_with(return_value->inferred_type)) { - fire(cur_f, return_value->loc, "can not unify type " + to_string(return_value) + " with previous return type " + to_string(return_unifier.get_result())); - } - } - if (!body_end.is_unreachable()) { - if (!return_unifier.unify_with_implicit_return_void()) { - fire(cur_f, v_function->get_body()->as()->loc_end, "missing return"); - } + bool has_void_returns = false; + bool has_non_void_returns = false; + for (AnyExprV return_expr : return_statements) { + TypePtr cur_type = return_expr->inferred_type; // `return expr` - type of expr; `return` - void + return_unifier.unify_with(cur_type); + has_void_returns |= cur_type == TypeDataVoid::create(); + has_non_void_returns |= cur_type != TypeDataVoid::create(); } inferred_return_type = return_unifier.get_result(); - if (inferred_return_type == nullptr && body_end.is_unreachable()) { + if (inferred_return_type == nullptr) { // if no return statements at all inferred_return_type = TypeDataVoid::create(); } + + if (!body_end.is_unreachable() && inferred_return_type != TypeDataVoid::create()) { + fire(fun_ref, v_function->get_body()->as()->loc_end, "missing return"); + } + if (has_void_returns && has_non_void_returns) { + for (AnyExprV return_expr : return_statements) { + if (return_expr->inferred_type == TypeDataVoid::create()) { + fire(fun_ref, return_expr->loc, "mixing void and non-void returns in function `" + fun_ref->as_human_readable() + "`"); + } + } + } + if (return_unifier.is_union_of_different_types()) { + // `return intVar` + `return sliceVar` results in `int | slice`, probably unexpected + fire(fun_ref, v_function->get_body()->loc, "function `" + fun_ref->as_human_readable() + "` calculated return type is " + to_string(inferred_return_type) + "; probably, it's not what you expected; declare `fun (...): ` manually"); + } } + } else { // asm functions should be strictly typed, this was checked earlier tolk_assert(fun_ref->declared_return_type); diff --git a/tolk/pipe-optimize-boolean-expr.cpp b/tolk/pipe-optimize-boolean-expr.cpp index 642defa035..09282cfd99 100644 --- a/tolk/pipe-optimize-boolean-expr.cpp +++ b/tolk/pipe-optimize-boolean-expr.cpp @@ -141,9 +141,9 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { v = createV(v->loc, !v->is_ifnot, v_cond_unary->get_rhs(), v->get_if_body(), v->get_else_body()); } // `if (x != null)` -> ifnot(x == null) - if (auto v_cond_isnull = v->get_cond()->try_as(); v_cond_isnull && v_cond_isnull->is_negated) { - v_cond_isnull->mutate()->assign_is_negated(!v_cond_isnull->is_negated); - v = createV(v->loc, !v->is_ifnot, v_cond_isnull, v->get_if_body(), v->get_else_body()); + if (auto v_cond_istype = v->get_cond()->try_as(); v_cond_istype && v_cond_istype->is_negated) { + v_cond_istype->mutate()->assign_is_negated(!v_cond_istype->is_negated); + v = createV(v->loc, !v->is_ifnot, v_cond_istype, v->get_if_body(), v->get_else_body()); } return v; diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 3f8f622c0b..0b116bbe80 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -158,7 +158,7 @@ struct TypeDataResolver { if (alias_ref->underlying_type->has_unresolved_inside()) { resolve_and_mutate_type_alias(alias_ref); } - return TypeDataAlias::create(alias_ref->name, alias_ref->underlying_type); + return TypeDataAlias::create(alias_ref); } } if (un->text == "auto") { @@ -260,6 +260,48 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { } } + void visit(V v) override { + current_scope.open_scope(v->loc); + parent::visit(v->get_sequence()); + current_scope.close_scope(v->loc); + } + + void visit(V v) override { + current_scope.open_scope(v->loc); // `match (var a = init_val) { ... }` + parent::visit(v); // then `a` exists only inside `match` arms + current_scope.close_scope(v->loc); + } + + void visit(V v) override { + // resolve identifiers after => at first + parent::visit(v->get_body()); + // because handling lhs of => is comprehensive + + switch (v->pattern_kind) { + case MatchArmKind::exact_type: { + if (const TypeDataUnresolved* maybe_ident = v->exact_type->try_as()) { + if (const Symbol* sym = current_scope.lookup_symbol(maybe_ident->text); sym && !sym->try_as()) { + auto v_ident = createV(v->loc, sym->name); + AnyExprV pattern_expr = createV(v->loc, v_ident, nullptr); + parent::visit(pattern_expr); + v->mutate()->assign_resolved_pattern(MatchArmKind::const_expression, nullptr, pattern_expr); + return; + } + } + TypePtr resolved_exact_type = finalize_type_data(cur_f, v->exact_type, current_genericTs); + v->mutate()->assign_resolved_pattern(MatchArmKind::exact_type, resolved_exact_type, v->get_pattern_expr()); + break; + } + case MatchArmKind::const_expression: { + parent::visit(v->get_pattern_expr()); + break; + } + default: + // for `else` match branch, do nothing: its body was already traversed above + break; + } + } + void visit(V v) override { // for `t.tupleAt` / `obj.method`, resolve "MyAlias" and "T" // (for function call `t.tupleAt()`, this v (ast_dot_access `t.tupleAt`) is callee) @@ -278,6 +320,12 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { parent::visit(v->get_expr()); } + void visit(V v) override { + TypePtr rhs_type = finalize_type_data(cur_f, v->rhs_type, current_genericTs); + v->mutate()->assign_resolved_type(rhs_type); + parent::visit(v->get_expr()); + } + void visit(V v) override { if (v->empty()) { return; diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index d3ddb5af75..65ab7fb800 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -133,10 +133,15 @@ static AnyExprV unwrap_not_null_operator(AnyExprV expr) { // 3) when two data flows rejoin // example: `if (tensorVar != null) ... else ...` rejoin `(int,int)` and `null` into `(int,int)?` // when lca can't be calculated (example: `(int,int)` and `(int,int,int)`), nullptr is returned -static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { - if (a == b) { +static TypePtr calculate_type_lca(TypePtr a, TypePtr b, bool* became_union = nullptr) { + if (a->equal_to(b)) { return a; } + + if (a == TypeDataUnknown::create() || b == TypeDataUnknown::create()) { + return TypeDataUnknown::create(); + } + if (a == TypeDataNever::create()) { return b; } @@ -144,22 +149,11 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { return a; } - if (a->can_rhs_be_assigned(b)) { - return a; - } - if (b->can_rhs_be_assigned(a)) { - return b; - } - - if (a == TypeDataUnknown::create() || b == TypeDataUnknown::create()) { - return TypeDataUnknown::create(); - } - if (a == TypeDataNullLiteral::create()) { - return TypeDataNullable::create(b); + return TypeDataUnion::create_nullable(b); } if (b == TypeDataNullLiteral::create()) { - return TypeDataNullable::create(a); + return TypeDataUnion::create_nullable(a); } const auto* tensor1 = a->try_as(); @@ -168,7 +162,7 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { std::vector types_lca; types_lca.reserve(tensor1->size()); for (int i = 0; i < tensor1->size(); ++i) { - TypePtr next = calculate_type_lca(tensor1->items[i], tensor2->items[i]); + TypePtr next = calculate_type_lca(tensor1->items[i], tensor2->items[i], became_union); if (next == nullptr) { return nullptr; } @@ -183,7 +177,7 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { std::vector types_lca; types_lca.reserve(tuple1->size()); for (int i = 0; i < tuple1->size(); ++i) { - TypePtr next = calculate_type_lca(tuple1->items[i], tuple2->items[i]); + TypePtr next = calculate_type_lca(tuple1->items[i], tuple2->items[i], became_union); if (next == nullptr) { return nullptr; } @@ -192,18 +186,18 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { return TypeDataTypedTuple::create(std::move(types_lca)); } - if (a->try_as() && b->try_as()) { // cond ? int32 : int64 - return TypeDataInt::create(); - } - if (const auto* a_alias = a->try_as()) { - return calculate_type_lca(a_alias->underlying_type, b); + return calculate_type_lca(a_alias->underlying_type, b, became_union); } if (const auto* b_alias = b->try_as()) { - return calculate_type_lca(a, b_alias->underlying_type); + return calculate_type_lca(a, b_alias->underlying_type, became_union); } - return nullptr; + TypePtr resulting_union = TypeDataUnion::create(std::vector{a, b}); + if (became_union != nullptr && !a->equal_to(resulting_union) && !b->equal_to(resulting_union)) { + *became_union = true; + } + return resulting_union; } // merge (unify) of two sign states: what sign do we definitely have @@ -248,31 +242,28 @@ BoolState calculate_bool_lca(BoolState a, BoolState b) { // see comments above TypeInferringUnifyStrategy // this function calculates lca or currently stored result and next -bool TypeInferringUnifyStrategy::unify_with(TypePtr next) { +void TypeInferringUnifyStrategy::unify_with(TypePtr next, TypePtr dest_hint) { + // example: `var r = ... ? int8 : int16`, will be inferred as `int8 | int16` (via unification) + // but `var r: int = ... ? int8 : int16`, will be inferred as `int` (it's dest_hint) + if (dest_hint && dest_hint != TypeDataUnknown::create() && !dest_hint->unwrap_alias()->try_as()) { + if (dest_hint->can_rhs_be_assigned(next)) { + next = dest_hint; + } + } + if (unified_result == nullptr) { unified_result = next; - return true; + return; } if (unified_result == next) { - return true; + return; } - TypePtr combined = calculate_type_lca(unified_result, next); - if (!combined) { - return false; - } + bool became_union = false; + TypePtr combined = calculate_type_lca(unified_result, next, &became_union); + different_types_became_union |= became_union; unified_result = combined; - return true; -} - -bool TypeInferringUnifyStrategy::unify_with_implicit_return_void() { - if (unified_result == nullptr) { - unified_result = TypeDataVoid::create(); - return true; - } - - return unified_result == TypeDataVoid::create(); } // invalidate knowledge about sub-fields of a variable or its field @@ -356,14 +347,43 @@ FlowContext FlowContext::merge_flow(FlowContext&& c1, FlowContext&& c2) { return FlowContext(std::move(unified), c1.unreachable && c2.unreachable); } -// return `T`, so that `T?` = type -// what for: `if (x != null)`, to smart cast x inside if -TypePtr calculate_type_subtract_null(TypePtr type) { - if (const auto* as_nullable = type->unwrap_alias()->try_as()) { - return as_nullable->inner; +// return `T`, so that `T + subtract_type` = type +// example: `int?` - `null` = `int` +// example: `int | slice | builder | bool` - `bool | slice` = `int | builder` +// what for: `if (x != null)` / `if (x is T)`, to smart cast x inside if +TypePtr calculate_type_subtract_rhs_type(TypePtr type, TypePtr subtract_type) { + const TypeDataUnion* lhs_union = type->try_as(); + if (!lhs_union) { + return TypeDataNever::create(); + } + + std::vector rest_variants; + + if (const TypeDataUnion* sub_union = subtract_type->try_as()) { + if (lhs_union->has_all_variants_of(sub_union)) { + rest_variants.reserve(lhs_union->variants.size() - sub_union->variants.size()); + for (TypePtr lhs_variant : lhs_union->variants) { + if (!sub_union->has_variant_with_type_id(lhs_variant)) { + rest_variants.push_back(lhs_variant); + } + } + } + } else if (lhs_union->has_variant_with_type_id(subtract_type)) { + rest_variants.reserve(lhs_union->variants.size() - 1); + for (TypePtr lhs_variant : lhs_union->variants) { + if (lhs_variant->get_type_id() != subtract_type->get_type_id()) { + rest_variants.push_back(lhs_variant); + } + } + } + + if (rest_variants.empty()) { + return TypeDataNever::create(); + } + if (rest_variants.size() == 1) { + return rest_variants[0]; } - // union types will be handled here - return TypeDataNever::create(); + return TypeDataUnion::create(std::move(rest_variants)); } // given any expression vertex, extract SinkExpression is possible @@ -407,6 +427,13 @@ SinkExpression extract_sink_expression_from_vertex(AnyExprV v) { return extract_sink_expression_from_vertex(as_assign->get_lhs()); } + if (auto as_decl = v->try_as()) { + if (auto decl_var = as_decl->get_expr()->try_as()) { + tolk_assert(decl_var->var_ref); + return SinkExpression(decl_var->var_ref); + } + } + return {}; } @@ -441,18 +468,32 @@ TypePtr calc_declared_type_before_smart_cast(AnyExprV v) { // obvious example: `var x: (int,int)? = null`, it's `null` (`x == null` is always true, `x` can be passed to any `T?`) // not obvious example: `var x: (int?, int?)? = (3,null)`, result is `(int?,int?)`, whereas type of rhs is `(int,null)` TypePtr calc_smart_cast_type_on_assignment(TypePtr lhs_declared_type, TypePtr rhs_inferred_type) { - // assign `T` to `T?` (or at least "assignable-to-T" to "T?") - // smart cast to `T` - if (const auto* lhs_nullable = lhs_declared_type->unwrap_alias()->try_as()) { - if (lhs_nullable->inner->can_rhs_be_assigned(rhs_inferred_type)) { - return lhs_nullable->inner; + if (const TypeDataUnion* lhs_union = lhs_declared_type->unwrap_alias()->try_as()) { + // example: `var x: T? = null`, result is null + // example: `var x: int | (int, User?) = (5, null)`, result is `(int, User?)` + if (TypePtr lhs_subtype = lhs_union->calculate_exact_variant_to_fit_rhs(rhs_inferred_type)) { + return lhs_subtype; + } + // example: `var x: int | slice | cell = 4`, result is int + // example: `var x: T1 | T2 | T3 = y as T3 | T1`, result is `T1 | T3` + if (const TypeDataUnion* rhs_union = rhs_inferred_type->try_as()) { + bool lhs_has_all_variants_of_rhs = true; + for (TypePtr rhs_variant : rhs_union->variants) { + lhs_has_all_variants_of_rhs &= lhs_union->has_variant_with_type_id(rhs_variant); + } + if (lhs_has_all_variants_of_rhs && rhs_union->variants.size() < lhs_union->variants.size()) { + std::vector subtypes_of_lhs; + for (TypePtr lhs_variant : lhs_union->variants) { + if (rhs_union->has_variant_with_type_id(lhs_variant)) { + subtypes_of_lhs.push_back(lhs_variant); + } + } + if (subtypes_of_lhs.size() == 1) { + return subtypes_of_lhs[0]; + } + return TypeDataUnion::create(std::move(subtypes_of_lhs)); + } } - } - - // assign `null` to `T?` - // smart cast to `null` - if (lhs_declared_type->unwrap_alias()->try_as() && rhs_inferred_type == TypeDataNullLiteral::create()) { - return TypeDataNullLiteral::create(); } // no smart cast, type is the same as declared diff --git a/tolk/smart-casts-cfg.h b/tolk/smart-casts-cfg.h index b97c8864cb..31d4ebe0d4 100644 --- a/tolk/smart-casts-cfg.h +++ b/tolk/smart-casts-cfg.h @@ -34,12 +34,13 @@ namespace tolk { */ class TypeInferringUnifyStrategy { TypePtr unified_result = nullptr; + bool different_types_became_union = false; public: - bool unify_with(TypePtr next); - bool unify_with_implicit_return_void(); + void unify_with(TypePtr next, TypePtr dest_hint = nullptr); TypePtr get_result() const { return unified_result; } + bool is_union_of_different_types() const { return different_types_became_union; } }; /* @@ -200,7 +201,7 @@ struct ExprFlow { std::ostream& operator<<(std::ostream& os, const FactsAboutExpr& facts); std::ostream& operator<<(std::ostream& os, const FlowContext& flow); -TypePtr calculate_type_subtract_null(TypePtr type); +TypePtr calculate_type_subtract_rhs_type(TypePtr type, TypePtr subtract_type); SinkExpression extract_sink_expression_from_vertex(AnyExprV v); TypePtr calc_declared_type_before_smart_cast(AnyExprV v); TypePtr calc_smart_cast_type_on_assignment(TypePtr lhs_declared_type, TypePtr rhs_inferred_type); diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index 459ad8136d..a63d555c1d 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -24,18 +24,19 @@ namespace tolk { /* - * This class stores a big hashtable [hash => TypeData] + * This class stores a big hashtable [hash => TypePtr] * Every non-trivial TypeData*::create() method at first looks here, and allocates an object only if not found. - * That's why all allocated TypeData objects are unique, storing unique type_id. + * That's why all allocated TypeData objects are unique, and can be compared as pointers. + * But compare pointers carefully due to type aliases. */ -class TypeDataTypeIdCalculation { +class TypeDataHasherForUnique { uint64_t cur_hash; int children_flags_mask = 0; static std::unordered_map all_unique_occurred_types; public: - explicit TypeDataTypeIdCalculation(uint64_t initial_arbitrary_unique_number) + explicit TypeDataHasherForUnique(uint64_t initial_arbitrary_unique_number) : cur_hash(initial_arbitrary_unique_number) {} void feed_hash(uint64_t val) { @@ -47,14 +48,10 @@ class TypeDataTypeIdCalculation { } void feed_child(TypePtr inner) { - feed_hash(inner->type_id); + feed_hash(reinterpret_cast(inner)); children_flags_mask |= inner->flags; } - uint64_t type_id() const { - return cur_hash; - } - int children_flags() const { return children_flags_mask; } @@ -68,14 +65,52 @@ class TypeDataTypeIdCalculation { GNU_ATTRIBUTE_NOINLINE TypePtr register_unique(TypePtr newly_created) const { #ifdef TOLK_DEBUG - assert(newly_created->type_id == cur_hash); + assert(all_unique_occurred_types.find(cur_hash) == all_unique_occurred_types.end()); #endif all_unique_occurred_types[cur_hash] = newly_created; return newly_created; } }; -std::unordered_map TypeDataTypeIdCalculation::all_unique_occurred_types; +/* + * This class stores a hashtable [TypePtr => type_id] + * We need type_id to support union types, that are stored as tagged unions on a stack. + * Every type that can be contained inside a union, has type_id. + * Some type_id are predefined (1 = int, etc.), but all user-defined types are assigned type_id. + */ +class TypeIdCalculation { + static int last_type_id; + static std::unordered_map map_ptr_to_type_id; + +public: + static int assign_type_id(TypePtr self) { + if (self->has_type_alias_inside()) { // type_id is calculated without aliases + self = unwrap_type_alias_deeply(self); // `(int,int)` equals `(IntAlias,IntAlias)`. + } + if (auto it = map_ptr_to_type_id.find(self); it != map_ptr_to_type_id.end()) { + return it->second; + } + + int type_id = ++last_type_id; + map_ptr_to_type_id[self] = type_id; + return type_id; + } + + static TypePtr unwrap_type_alias_deeply(TypePtr type) { + return type->replace_children_custom([](TypePtr child) { + if (const TypeDataAlias* as_alias = child->try_as()) { + return as_alias->underlying_type->unwrap_alias(); + } + return child; + }); + } +}; + + +int TypeIdCalculation::last_type_id = 128; // below 128 reserved for built-in types +std::unordered_map TypeDataHasherForUnique::all_unique_occurred_types; +std::unordered_map TypeIdCalculation::map_ptr_to_type_id; + TypePtr TypeDataInt::singleton; TypePtr TypeDataBool::singleton; TypePtr TypeDataCell::singleton; @@ -105,41 +140,61 @@ void type_system_init() { } +bool TypeData::equal_to_slow_path(TypePtr lhs, TypePtr rhs) { + if (lhs->has_type_alias_inside()) { + lhs = TypeIdCalculation::unwrap_type_alias_deeply(lhs); + } + if (rhs->has_type_alias_inside()) { + rhs = TypeIdCalculation::unwrap_type_alias_deeply(rhs); + } + if (lhs == rhs) { + return true; + } + + if (const TypeDataUnion* lhs_union = lhs->try_as()) { + if (const TypeDataUnion* rhs_union = rhs->try_as()) { + return lhs_union->variants.size() == rhs_union->variants.size() && lhs_union->has_all_variants_of(rhs_union); + } + } + return false; +} + +TypePtr TypeData::unwrap_alias_slow_path(TypePtr lhs) { + TypePtr unwrapped = lhs; + while (const TypeDataAlias* as_alias = unwrapped->try_as()) { + unwrapped = as_alias->underlying_type; + } + return unwrapped; +} + + // -------------------------------------------- // create() // // all constructors of TypeData classes are private, only TypeData*::create() is allowed -// each non-trivial create() method calculates hash (type_id) +// each non-trivial create() method calculates hash // and creates an object only if it isn't found in a global hashtable // -TypePtr TypeDataAlias::create(const std::string& alias_name, TypePtr underlying_type) { - TypeDataTypeIdCalculation hash(5694590762732189561ULL); - hash.feed_string(alias_name); - hash.feed_child(underlying_type); +TypePtr TypeDataAlias::create(AliasDefPtr alias_ref) { + TypeDataHasherForUnique hash(5694590762732189561ULL); + hash.feed_string(alias_ref->name); + hash.feed_child(alias_ref->underlying_type); if (TypePtr existing = hash.get_existing()) { return existing; } - std::string name_copy = alias_name; - return hash.register_unique(new TypeDataAlias(hash.type_id(), hash.children_flags(), std::move(name_copy), underlying_type)); -} - -TypePtr TypeDataNullable::create(TypePtr inner) { - TypeDataTypeIdCalculation hash(1774084920039440885ULL); - hash.feed_child(inner); - if (TypePtr existing = hash.get_existing()) { - return existing; + TypePtr underlying_type = alias_ref->underlying_type; + if (underlying_type == TypeDataNullLiteral::create() || underlying_type == TypeDataNever::create() || underlying_type == TypeDataVoid::create()) { + return underlying_type; // aliasing these types is strange, don't store an alias } - // most types (int?, slice?, etc.), when nullable, still occupy 1 stack slot (holding TVM NULL at runtime) - // but for example for `(int, int)` we need an extra stack slot "null flag" - int width_on_stack = inner->can_hold_tvm_null_instead() ? 1 : inner->get_width_on_stack() + 1; - return hash.register_unique(new TypeDataNullable(hash.type_id(), hash.children_flags(), width_on_stack, inner)); + + return hash.register_unique(new TypeDataAlias(hash.children_flags(), alias_ref, underlying_type)); } TypePtr TypeDataFunCallable::create(std::vector&& params_types, TypePtr return_type) { - TypeDataTypeIdCalculation hash(3184039965511020991ULL); + TypeDataHasherForUnique hash(3184039965511020991ULL); for (TypePtr param : params_types) { hash.feed_child(param); hash.feed_hash(767721); @@ -150,21 +205,21 @@ TypePtr TypeDataFunCallable::create(std::vector&& params_types, TypePtr if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataFunCallable(hash.type_id(), hash.children_flags(), std::move(params_types), return_type)); + return hash.register_unique(new TypeDataFunCallable(hash.children_flags(), std::move(params_types), return_type)); } TypePtr TypeDataGenericT::create(std::string&& nameT) { - TypeDataTypeIdCalculation hash(9145033724911680012ULL); + TypeDataHasherForUnique hash(9145033724911680012ULL); hash.feed_string(nameT); if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataGenericT(hash.type_id(), std::move(nameT))); + return hash.register_unique(new TypeDataGenericT(std::move(nameT))); } TypePtr TypeDataTensor::create(std::vector&& items) { - TypeDataTypeIdCalculation hash(3159238551239480381ULL); + TypeDataHasherForUnique hash(3159238551239480381ULL); for (TypePtr item : items) { hash.feed_child(item); hash.feed_hash(819613); @@ -177,11 +232,11 @@ TypePtr TypeDataTensor::create(std::vector&& items) { for (TypePtr item : items) { width_on_stack += item->get_width_on_stack(); } - return hash.register_unique(new TypeDataTensor(hash.type_id(), hash.children_flags(), width_on_stack, std::move(items))); + return hash.register_unique(new TypeDataTensor(hash.children_flags(), width_on_stack, std::move(items))); } TypePtr TypeDataTypedTuple::create(std::vector&& items) { - TypeDataTypeIdCalculation hash(9189266157349499320ULL); + TypeDataHasherForUnique hash(9189266157349499320ULL); for (TypePtr item : items) { hash.feed_child(item); hash.feed_hash(735911); @@ -190,11 +245,11 @@ TypePtr TypeDataTypedTuple::create(std::vector&& items) { if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataTypedTuple(hash.type_id(), hash.children_flags(), std::move(items))); + return hash.register_unique(new TypeDataTypedTuple(hash.children_flags(), std::move(items))); } TypePtr TypeDataIntN::create(bool is_unsigned, bool is_variadic, int n_bits) { - TypeDataTypeIdCalculation hash(1678330938771108027ULL); + TypeDataHasherForUnique hash(1678330938771108027ULL); hash.feed_hash(is_unsigned); hash.feed_hash(is_variadic); hash.feed_hash(n_bits); @@ -202,29 +257,173 @@ TypePtr TypeDataIntN::create(bool is_unsigned, bool is_variadic, int n_bits) { if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataIntN(hash.type_id(), is_unsigned, is_variadic, n_bits)); + return hash.register_unique(new TypeDataIntN(is_unsigned, is_variadic, n_bits)); } TypePtr TypeDataBytesN::create(bool is_bits, int n_width) { - TypeDataTypeIdCalculation hash(7810988137199333041ULL); + TypeDataHasherForUnique hash(7810988137199333041ULL); hash.feed_hash(is_bits); hash.feed_hash(n_width); if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataBytesN(hash.type_id(), is_bits, n_width)); + return hash.register_unique(new TypeDataBytesN(is_bits, n_width)); +} + +TypePtr TypeDataUnion::create(std::vector&& variants) { + TypeDataHasherForUnique hash(8719233194368471403ULL); + for (TypePtr variant : variants) { + hash.feed_child(variant); + hash.feed_hash(817663); + } + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + + // at the moment of parsing, union type can contain unresolved symbols + // in this case, don't try to flatten: we have no info + // after symbols resolving, a new union type (with resolved variants) will be created + bool not_ready_yet = false; + for (TypePtr variant : variants) { + not_ready_yet |= variant->has_unresolved_inside() || variant->has_genericT_inside(); + } + if (not_ready_yet) { + TypePtr or_null = nullptr; + if (variants.size() == 2) { + if (variants[0] == TypeDataNullLiteral::create() || variants[1] == TypeDataNullLiteral::create()) { + or_null = variants[variants[0] == TypeDataNullLiteral::create()]; + } + } + return hash.register_unique(new TypeDataUnion(hash.children_flags(), -999999, or_null, std::move(variants))); + } + + // flatten variants and remove duplicates + // note, that `int | slice` and `int | int | slice` are different TypePtr, but actually the same variants + std::vector flat_variants; + flat_variants.reserve(variants.size()); + for (TypePtr variant : variants) { + if (const TypeDataUnion* nested_union = variant->unwrap_alias()->try_as()) { + for (TypePtr nested_variant : nested_union->variants) { + append_union_type_variant(nested_variant, flat_variants); + } + } else { + append_union_type_variant(variant, flat_variants); + } + } + // detect, whether it's `T?` or `T1 | T2 | ...` + TypePtr or_null = nullptr; + if (flat_variants.size() == 2) { + if (flat_variants[0] == TypeDataNullLiteral::create() || flat_variants[1] == TypeDataNullLiteral::create()) { + or_null = flat_variants[flat_variants[0] == TypeDataNullLiteral::create()]; + } + } + + int width_on_stack; + if (or_null && or_null->can_hold_tvm_null_instead()) { + width_on_stack = 1; + } else { + // `T1 | T2 | ...` occupy max(W[i]) + 1 slot for UTag (stores type_id or 0 for null) + int max_child_width = 0; + for (TypePtr i : flat_variants) { + if (i != TypeDataNullLiteral::create()) { // `Empty | () | null` totally should be 1 (0 + 1 for UTag) + max_child_width = std::max(max_child_width, i->get_width_on_stack()); + } + } + width_on_stack = max_child_width + 1; + } + + if (flat_variants.size() == 1) { // `int | int` + return flat_variants[0]; + } + return hash.register_unique(new TypeDataUnion(hash.children_flags(), width_on_stack, or_null, std::move(flat_variants))); +} + +TypePtr TypeDataUnion::create_nullable(TypePtr nullable) { + // calculate exactly the same hash as for `T | null` to create std::vector only if type seen the first time + TypeDataHasherForUnique hash(8719233194368471403ULL); + hash.feed_child(nullable); + hash.feed_hash(817663); + hash.feed_child(TypeDataNullLiteral::create()); + hash.feed_hash(817663); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + return create({nullable, TypeDataNullLiteral::create()}); } TypePtr TypeDataUnresolved::create(std::string&& text, SrcLocation loc) { - TypeDataTypeIdCalculation hash(3680147223540048162ULL); + TypeDataHasherForUnique hash(3680147223540048162ULL); hash.feed_string(text); // hash.feed_hash(*reinterpret_cast(&loc)); if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataUnresolved(hash.type_id(), std::move(text), loc)); + return hash.register_unique(new TypeDataUnresolved(std::move(text), loc)); +} + + +// -------------------------------------------- +// get_type_id() +// +// in order to support union types, every type that can be stored inside a union has a unique type_id +// some are predefined (1 = int, etc. in .h file), the others are here +// + +int TypeDataAlias::get_type_id() const { + return underlying_type->get_type_id(); +} + +int TypeDataFunCallable::get_type_id() const { + return TypeIdCalculation::assign_type_id(this); +} + +int TypeDataGenericT::get_type_id() const { + return TypeIdCalculation::assign_type_id(this); +} + +int TypeDataTensor::get_type_id() const { + assert(!has_genericT_inside()); + return TypeIdCalculation::assign_type_id(this); +} + +int TypeDataTypedTuple::get_type_id() const { + assert(!has_genericT_inside()); + return TypeIdCalculation::assign_type_id(this); +} + +int TypeDataIntN::get_type_id() const { + switch (n_bits) { + case 8: return 42 + is_unsigned; // for common intN, use predefined small numbers + case 16: return 44 + is_unsigned; + case 32: return 46 + is_unsigned; + case 64: return 48 + is_unsigned; + case 128: return 50 + is_unsigned; + case 256: return 52 + is_unsigned; + default: return TypeIdCalculation::assign_type_id(this); + } +} + +int TypeDataBytesN::get_type_id() const { + return TypeIdCalculation::assign_type_id(this); +} + +int TypeDataUnion::get_type_id() const { + assert(false); // a union can not be inside a union + throw Fatal("unexpected get_type_id() call"); +} + +int TypeDataUnknown::get_type_id() const { + assert(false); // unknown can not be inside a union + throw Fatal("unexpected get_type_id() call"); +} + +int TypeDataUnresolved::get_type_id() const { + assert(false); // unresolved can be inside a union at parsing, but is resolved is advance + throw Fatal("unexpected get_type_id() call"); } @@ -235,10 +434,8 @@ TypePtr TypeDataUnresolved::create(std::string&& text, SrcLocation loc) { // only non-trivial implementations are here; trivial are defined in .h file // -std::string TypeDataNullable::as_human_readable() const { - std::string nested = inner->as_human_readable(); - bool embrace = inner->try_as(); - return embrace ? "(" + nested + ")?" : nested + "?"; +std::string TypeDataAlias::as_human_readable() const { + return alias_ref->name; } std::string TypeDataFunCallable::as_human_readable() const { @@ -290,6 +487,30 @@ std::string TypeDataBytesN::as_human_readable() const { return s_bytes + std::to_string(n_width); } +std::string TypeDataUnion::as_human_readable() const { + // stringify `T?`, not `T | null` + if (or_null) { + bool embrace = or_null->try_as(); + return embrace ? "(" + or_null->as_human_readable() + ")?" : or_null->as_human_readable() + "?"; + } + + std::string result; + for (TypePtr variant : variants) { + if (!result.empty()) { + result += " | "; + } + bool embrace = variant->try_as(); + if (embrace) { + result += "("; + } + result += variant->as_human_readable(); + if (embrace) { + result += ")"; + } + } + return result; +} + // -------------------------------------------- // traverse() @@ -303,11 +524,6 @@ void TypeDataAlias::traverse(const TraverserCallbackT& callback) const { underlying_type->traverse(callback); } -void TypeDataNullable::traverse(const TraverserCallbackT& callback) const { - callback(this); - inner->traverse(callback); -} - void TypeDataFunCallable::traverse(const TraverserCallbackT& callback) const { callback(this); for (TypePtr param : params_types) { @@ -330,6 +546,13 @@ void TypeDataTypedTuple::traverse(const TraverserCallbackT& callback) const { } } +void TypeDataUnion::traverse(const TraverserCallbackT& callback) const { + callback(this); + for (TypePtr variant : variants) { + variant->traverse(callback); + } +} + // -------------------------------------------- // replace_children_custom() @@ -339,14 +562,6 @@ void TypeDataTypedTuple::traverse(const TraverserCallbackT& callback) const { // only non-trivial implementations are here; by default (no children), `return callback(this)` is executed // -TypePtr TypeDataAlias::replace_children_custom(const ReplacerCallbackT& callback) const { - return callback(create(alias_name, underlying_type->replace_children_custom(callback))); -} - -TypePtr TypeDataNullable::replace_children_custom(const ReplacerCallbackT& callback) const { - return callback(create(inner->replace_children_custom(callback))); -} - TypePtr TypeDataFunCallable::replace_children_custom(const ReplacerCallbackT& callback) const { std::vector mapped; mapped.reserve(params_types.size()); @@ -374,12 +589,23 @@ TypePtr TypeDataTypedTuple::replace_children_custom(const ReplacerCallbackT& cal return callback(create(std::move(mapped))); } +TypePtr TypeDataUnion::replace_children_custom(const ReplacerCallbackT& callback) const { + std::vector mapped; + mapped.reserve(variants.size()); + for (TypePtr variant : variants) { + mapped.push_back(variant->replace_children_custom(callback)); + } + return callback(create(std::move(mapped))); +} + // -------------------------------------------- // can_rhs_be_assigned() // // on `var lhs: = rhs`, having inferred rhs_type, check that it can be assigned without any casts // the same goes for passing arguments, returning values, etc. — where the "receiver" (lhs) checks "applier" (rhs) +// note, that `int8 | int16` is not assignable to `int` (even though both are assignable), +// because the only way to work with union types is to use `match`/`is` operators // bool TypeDataAlias::can_rhs_be_assigned(TypePtr rhs) const { @@ -475,25 +701,6 @@ bool TypeDataNullLiteral::can_rhs_be_assigned(TypePtr rhs) const { return rhs == TypeDataNever::create(); } -bool TypeDataNullable::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { - return true; - } - if (rhs == TypeDataNullLiteral::create()) { - return true; - } - if (const TypeDataNullable* rhs_nullable = rhs->try_as()) { - return inner->can_rhs_be_assigned(rhs_nullable->inner); - } - if (inner->can_rhs_be_assigned(rhs)) { - return true; - } - if (const TypeDataAlias* rhs_alias = rhs->try_as()) { - return can_rhs_be_assigned(rhs_alias->underlying_type); - } - return rhs == TypeDataNever::create(); -} - bool TypeDataFunCallable::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; @@ -510,7 +717,8 @@ bool TypeDataFunCallable::can_rhs_be_assigned(TypePtr rhs) const { return false; } } - return return_type->can_rhs_be_assigned(rhs_callable->return_type) && rhs_callable->return_type->can_rhs_be_assigned(return_type); + return return_type->can_rhs_be_assigned(rhs_callable->return_type) && + rhs_callable->return_type->can_rhs_be_assigned(return_type); } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { return can_rhs_be_assigned(rhs_alias->underlying_type); @@ -591,6 +799,23 @@ bool TypeDataCoins::can_rhs_be_assigned(TypePtr rhs) const { return rhs == TypeDataNever::create(); } +bool TypeDataUnion::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + if (calculate_exact_variant_to_fit_rhs(rhs)) { // `int` to `int | slice`, `int?` to `int8?`, `(int, null)` to `(int, T?) | slice` + return true; + } + if (const TypeDataUnion* rhs_union = rhs->try_as()) { + return has_all_variants_of(rhs_union); + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } + + return rhs == TypeDataNever::create(); +} + bool TypeDataUnknown::can_rhs_be_assigned(TypePtr rhs) const { return true; } @@ -608,9 +833,6 @@ bool TypeDataVoid::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } - if (const TypeDataAlias* rhs_alias = rhs->try_as()) { - return can_rhs_be_assigned(rhs_alias->underlying_type); - } return rhs == TypeDataNever::create(); } @@ -622,13 +844,27 @@ bool TypeDataVoid::can_rhs_be_assigned(TypePtr rhs) const { // note, that it's not auto-casts `var lhs: = rhs`, it's an expression `rhs as ` // +// common helper for union types: +// - `int as int?` is ok +// - `int8 as int16?` is ok (primitive 1-slot nullable don't store UTag, rules are less strict) +// - `int as int | int16` is ok (exact match one of types) +// - `int as slice | null` is NOT ok (no rhs subtype fits) +// - `int as int8 | int16` is NOT ok (ambiguity) +static bool can_be_casted_to_union(TypePtr self, const TypeDataUnion* rhs_union) { + if (rhs_union->is_primitive_nullable()) { // casting to primitive 1-slot nullable + return self == TypeDataNullLiteral::create() || self->can_be_casted_with_as_operator(rhs_union->or_null); + } + + return rhs_union->calculate_exact_variant_to_fit_rhs(self) != nullptr; +} + bool TypeDataAlias::can_be_casted_with_as_operator(TypePtr cast_to) const { return underlying_type->can_be_casted_with_as_operator(cast_to); } bool TypeDataInt::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { // `int` as `int?` - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { // `int` as `int?` / `int` as `int | slice` + return can_be_casted_to_union(this, to_union); } if (cast_to->try_as()) { // `int` as `int8` / `int` as `uint2` return true; @@ -646,8 +882,8 @@ bool TypeDataBool::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to == TypeDataInt::create()) { return true; } - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); } if (const auto* to_intN = cast_to->try_as()) { return !to_intN->is_unsigned; // `bool` as `int8` ok, `bool` as `uintN` not (true is -1) @@ -659,8 +895,8 @@ bool TypeDataBool::can_be_casted_with_as_operator(TypePtr cast_to) const { } bool TypeDataCell::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -672,8 +908,8 @@ bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to->try_as()) { // `slice` to `bytes32` / `slice` to `bits8` return true; } - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -682,8 +918,8 @@ bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { } bool TypeDataBuilder::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -692,8 +928,8 @@ bool TypeDataBuilder::can_be_casted_with_as_operator(TypePtr cast_to) const { } bool TypeDataTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -702,8 +938,8 @@ bool TypeDataTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { } bool TypeDataContinuation::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -712,8 +948,8 @@ bool TypeDataContinuation::can_be_casted_with_as_operator(TypePtr cast_to) const } bool TypeDataNullLiteral::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (cast_to->try_as()) { - return true; + if (const TypeDataUnion* to_union = cast_to->try_as()) { // `null` to `T?` / `null` to `... | null` + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -721,19 +957,9 @@ bool TypeDataNullLiteral::can_be_casted_with_as_operator(TypePtr cast_to) const return cast_to == this; } -bool TypeDataNullable::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return inner->can_be_casted_with_as_operator(to_nullable->inner); - } - if (const TypeDataAlias* to_alias = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_alias->underlying_type); - } - return false; -} - bool TypeDataFunCallable::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -746,8 +972,12 @@ bool TypeDataFunCallable::can_be_casted_with_as_operator(TypePtr cast_to) const if (!params_types[i]->can_be_casted_with_as_operator(to_callable->params_types[i])) { return false; } + if (!to_callable->params_types[i]->can_be_casted_with_as_operator(params_types[i])) { + return false; + } } - return return_type->can_be_casted_with_as_operator(to_callable->return_type); + return return_type->can_be_casted_with_as_operator(to_callable->return_type) && + to_callable->return_type->can_be_casted_with_as_operator(return_type); } return false; } @@ -765,8 +995,8 @@ bool TypeDataTensor::can_be_casted_with_as_operator(TypePtr cast_to) const { } return true; } - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -783,8 +1013,8 @@ bool TypeDataTypedTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { } return true; } - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -796,8 +1026,8 @@ bool TypeDataIntN::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to->try_as()) { // `int8` as `int32`, `int256` as `uint5`, anything return true; } - if (const auto* to_nullable = cast_to->try_as()) { // `int8` as `int32?` - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { // `int8` as `int32?` + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -809,8 +1039,8 @@ bool TypeDataBytesN::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to->try_as()) { // `bytes256` as `bytes512`, `bits1` as `bytes8` return true; } - if (const auto* to_nullable = cast_to->try_as()) { // `bytes8` as `slice?` - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { // `bytes8` as `slice?` + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -822,8 +1052,8 @@ bool TypeDataCoins::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to->try_as()) { // `coins` as `int8` return true; } - if (const auto* to_nullable = cast_to->try_as()) { // `coins` as `coins?` / `coins` as `int?` - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { // `coins` as `coins?` / `coins` as `int?` + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -834,6 +1064,19 @@ bool TypeDataCoins::can_be_casted_with_as_operator(TypePtr cast_to) const { return cast_to == this; } +bool TypeDataUnion::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const TypeDataUnion* to_union = cast_to->try_as()) { // `int8 | int16` as `int16 | int8 | slice` + if (to_union->is_primitive_nullable()) { + return or_null && or_null->can_be_casted_with_as_operator(to_union->or_null); + } + return to_union->has_all_variants_of(this); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + return false; +} + bool TypeDataUnknown::can_be_casted_with_as_operator(TypePtr cast_to) const { // 'unknown' can be cast to any TVM value return cast_to->get_width_on_stack() == 1; @@ -865,13 +1108,6 @@ bool TypeDataAlias::can_hold_tvm_null_instead() const { return underlying_type->can_hold_tvm_null_instead(); } -bool TypeDataNullable::can_hold_tvm_null_instead() const { - if (get_width_on_stack() != 1) { // `(int, int)?` / `()?` can not hold null instead - return false; // only `int?` / `cell?` / `StructWith1IntField?` can - } // and some tricky situations like `(int, ())?`, but not `(int?, ())?` - return !inner->can_hold_tvm_null_instead(); -} - bool TypeDataTensor::can_hold_tvm_null_instead() const { if (get_width_on_stack() != 1) { // `(int, int)` / `()` can not hold null instead, since null is 1 slot return false; // only `((), int)` and similar can: @@ -884,6 +1120,13 @@ bool TypeDataTensor::can_hold_tvm_null_instead() const { return true; } +bool TypeDataUnion::can_hold_tvm_null_instead() const { + if (get_width_on_stack() != 1) { // `(int, int)?` / `()?` can not hold null instead + return false; // only `int?` / `cell?` / `StructWith1IntField?` can + } // and some tricky situations like `(int, ())?`, but not `(int?, ())?` + return or_null && !or_null->can_hold_tvm_null_instead(); +} + bool TypeDataNever::can_hold_tvm_null_instead() const { return false; } @@ -893,12 +1136,73 @@ bool TypeDataVoid::can_hold_tvm_null_instead() const { } +// union types creation is a bit tricky: nested unions are flattened, duplicates are removed +// so, a resolved union type has variants, each with unique type_id +// (type_id is calculated with aliases erasure) +void TypeDataUnion::append_union_type_variant(TypePtr variant, std::vector& out_unique_variants) { + for (TypePtr existing : out_unique_variants) { + if (existing->get_type_id() == variant->get_type_id()) { + return; + } + } + + out_unique_variants.push_back(variant); +} + +bool TypeDataUnion::has_variant_with_type_id(int type_id) const { + for (TypePtr self_variant : variants) { + if (self_variant->get_type_id() == type_id) { + return true; + } + } + return false; +} + +bool TypeDataUnion::has_all_variants_of(const TypeDataUnion* rhs_type) const { + for (TypePtr rhs_variant : rhs_type->variants) { + if (!has_variant_with_type_id(rhs_variant->get_type_id())) { + return false; + } + } + return true; +} + +// given this = `T1 | T2 | ...` and rhs_type, find the only (not ambiguous) T_i that can accept it +TypePtr TypeDataUnion::calculate_exact_variant_to_fit_rhs(TypePtr rhs_type) const { + // primitive 1-slot nullable don't store type_id, they can be assigned less strict, like `int?` to `int16?` + if (const TypeDataUnion* rhs_union = rhs_type->unwrap_alias()->try_as()) { + if (is_primitive_nullable() && rhs_union->is_primitive_nullable() && or_null->can_rhs_be_assigned(rhs_union->or_null)) { + return this; + } + return nullptr; + } + // `int` to `int | int8` is okay: exact type matching + for (TypePtr variant : variants) { + if (variant->get_type_id() == rhs_type->get_type_id()) { + return variant; + } + } + + // find the only T_i; it would also be used for transition at IR generation, like `(int,null)` to `(int, User?) | int` + TypePtr first_covering = nullptr; + for (TypePtr variant : variants) { + if (variant->can_rhs_be_assigned(rhs_type)) { + if (first_covering) { + return nullptr; + } + first_covering = variant; + } + } + return first_covering; +} + + // -------------------------------------------- // parsing type from tokens // // here we implement parsing types (mostly after colon) to TypeData // example: `var v: int` is TypeDataInt -// example: `var v: (builder?, [cell])` is TypeDataTensor(TypeDataNullable(TypeDataBuilder), TypeDataTypedTuple(TypeDataCell)) +// example: `var v: (builder?, [cell])` is TypeDataTensor(TypeDataUnion(TypeDataBuilder,TypeDataNullLiteral), TypeDataTypedTuple(TypeDataCell)) // example: `fun f(): ()` is TypeDataTensor() (an empty one) // // note, that unrecognized type names (MyEnum, MyStruct, T) are parsed as TypeDataUnresolved, @@ -1036,7 +1340,7 @@ static TypePtr parse_type_nullable(Lexer& lex) { if (lex.tok() == tok_question) { lex.next(); - result = TypeDataNullable::create(result); + result = TypeDataUnion::create_nullable(result); } return result; @@ -1045,21 +1349,27 @@ static TypePtr parse_type_nullable(Lexer& lex) { static TypePtr parse_type_expression(Lexer& lex) { TypePtr result = parse_type_nullable(lex); - if (lex.tok() == tok_arrow) { // `int -> int`, `(cell, slice) -> void` + if (lex.tok() == tok_bitwise_or) { // `int | slice`, `Pair2 | (Pair3 | null)` + std::vector items; + items.emplace_back(result); + while (lex.tok() == tok_bitwise_or) { + lex.next(); + items.emplace_back(parse_type_nullable(lex)); + } + result = TypeDataUnion::create(std::move(items)); + } + + if (lex.tok() == tok_arrow) { // `int -> int`, `(cell, slice) -> void`, `int -> int -> int`, `int | cell -> void` lex.next(); TypePtr return_type = parse_type_expression(lex); std::vector params_types = {result}; if (const auto* as_tensor = result->try_as()) { params_types = as_tensor->items; } - return TypeDataFunCallable::create(std::move(params_types), return_type); + result = TypeDataFunCallable::create(std::move(params_types), return_type); } - if (lex.tok() != tok_bitwise_or) { - return result; - } - - lex.error("union types are not supported yet"); + return result; } TypePtr parse_type_from_tokens(Lexer& lex) { diff --git a/tolk/type-system.h b/tolk/type-system.h index ebb25312e5..56fd02e5ce 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -27,7 +27,7 @@ namespace tolk { * TypeData is both a user-given and an inferred type representation. * `int`, `cell`, `T`, `(int, [tuple])` are instances of TypeData. * Every unique TypeData is created only once, so for example TypeDataTensor::create(int, int) - * returns one and the same pointer always. This "uniqueness" is called type_id, calculated before creation. + * returns one and the same pointer always. * * In Tolk code, types after colon `var v: (int, T)` are parsed to TypeData. * See parse_type_from_tokens(). @@ -46,28 +46,29 @@ namespace tolk { * For instance, `fun f(v: T)`, at first "T" of `v` is unresolved, and then converted to TypeDataGenericT. */ class TypeData { - // all unique types have unique type_id; it's used both for allocating memory once and for tagged unions - const uint64_t type_id; // bits of flag_mask, to store often-used properties and return them without tree traversing const int flags; // how many slots on a stack this type occupies (calculated on creation), e.g. `int`=1, `(int,int)`=2, `(int,int)?`=3 const int width_on_stack; - friend class TypeDataTypeIdCalculation; + friend class TypeDataHasherForUnique; protected: enum flag_mask { flag_contains_unknown_inside = 1 << 1, flag_contains_genericT_inside = 1 << 2, flag_contains_unresolved_inside = 1 << 3, + flag_contains_type_alias_inside = 1 << 4, }; - explicit TypeData(uint64_t type_id, int flags_with_children, int width_on_stack) - : type_id(type_id) - , flags(flags_with_children) + explicit TypeData(int flags_with_children, int width_on_stack) + : flags(flags_with_children) , width_on_stack(width_on_stack) { } + static bool equal_to_slow_path(TypePtr lhs, TypePtr rhs); + static TypePtr unwrap_alias_slow_path(TypePtr lhs); + public: virtual ~TypeData() = default; @@ -76,18 +77,24 @@ class TypeData { return dynamic_cast(this); } - uint64_t get_type_id() const { return type_id; } int get_width_on_stack() const { return width_on_stack; } + bool equal_to(TypePtr rhs) const { + return this == rhs || equal_to_slow_path(this, rhs); + } + TypePtr unwrap_alias() const { + return has_type_alias_inside() ? unwrap_alias_slow_path(this) : this; + } + bool has_unknown_inside() const { return flags & flag_contains_unknown_inside; } bool has_genericT_inside() const { return flags & flag_contains_genericT_inside; } bool has_unresolved_inside() const { return flags & flag_contains_unresolved_inside; } - - inline TypePtr unwrap_alias() const; + bool has_type_alias_inside() const { return flags & flag_contains_type_alias_inside; } using TraverserCallbackT = std::function; using ReplacerCallbackT = std::function; + virtual int get_type_id() const = 0; virtual std::string as_human_readable() const = 0; virtual bool can_rhs_be_assigned(TypePtr rhs) const = 0; virtual bool can_be_casted_with_as_operator(TypePtr cast_to) const = 0; @@ -112,22 +119,22 @@ class TypeData { * That's why lots of code comparing types use `type->unwrap_alias()` or `try_as`. */ class TypeDataAlias final : public TypeData { - explicit TypeDataAlias(uint64_t type_id, int children_flags, std::string&& alias_name, TypePtr underlying_type) - : TypeData(type_id, children_flags, underlying_type->get_width_on_stack()) - , alias_name(std::move(alias_name)) + explicit TypeDataAlias(int children_flags, AliasDefPtr alias_ref, TypePtr underlying_type) + : TypeData(children_flags | flag_contains_type_alias_inside, underlying_type->get_width_on_stack()) + , alias_ref(alias_ref) , underlying_type(underlying_type) {} public: - const std::string alias_name; + const AliasDefPtr alias_ref; const TypePtr underlying_type; - static TypePtr create(const std::string& alias_name, TypePtr underlying_type); + static TypePtr create(AliasDefPtr alias_ref); - std::string as_human_readable() const override { return alias_name; } + int get_type_id() const override; + std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; void traverse(const TraverserCallbackT& callback) const override; - TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; bool can_hold_tvm_null_instead() const override; }; @@ -135,7 +142,7 @@ class TypeDataAlias final : public TypeData { * `int` is TypeDataInt, representation of TVM int. */ class TypeDataInt final : public TypeData { - TypeDataInt() : TypeData(1ULL, 0, 1) {} + TypeDataInt() : TypeData(0, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -143,6 +150,7 @@ class TypeDataInt final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 1; } std::string as_human_readable() const override { return "int"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -153,7 +161,7 @@ class TypeDataInt final : public TypeData { * From the type system point of view, int and bool are different, not-autocastable types. */ class TypeDataBool final : public TypeData { - TypeDataBool() : TypeData(2ULL, 0, 1) {} + TypeDataBool() : TypeData(0, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -161,6 +169,7 @@ class TypeDataBool final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 2; } std::string as_human_readable() const override { return "bool"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -170,7 +179,7 @@ class TypeDataBool final : public TypeData { * `cell` is TypeDataCell, representation of TVM cell. */ class TypeDataCell final : public TypeData { - TypeDataCell() : TypeData(3ULL, 0, 1) {} + TypeDataCell() : TypeData(0, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -178,6 +187,7 @@ class TypeDataCell final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 3; } std::string as_human_readable() const override { return "cell"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -187,7 +197,7 @@ class TypeDataCell final : public TypeData { * `slice` is TypeDataSlice, representation of TVM slice. */ class TypeDataSlice final : public TypeData { - TypeDataSlice() : TypeData(4ULL, 0, 1) {} + TypeDataSlice() : TypeData(0, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -195,6 +205,7 @@ class TypeDataSlice final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 4; } std::string as_human_readable() const override { return "slice"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -204,7 +215,7 @@ class TypeDataSlice final : public TypeData { * `builder` is TypeDataBuilder, representation of TVM builder. */ class TypeDataBuilder final : public TypeData { - TypeDataBuilder() : TypeData(5ULL, 0, 1) {} + TypeDataBuilder() : TypeData(0, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -212,6 +223,7 @@ class TypeDataBuilder final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 5; } std::string as_human_readable() const override { return "builder"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -223,7 +235,7 @@ class TypeDataBuilder final : public TypeData { * so getting its element results in TypeDataUnknown (which must be assigned/cast explicitly). */ class TypeDataTuple final : public TypeData { - TypeDataTuple() : TypeData(6ULL, 0, 1) {} + TypeDataTuple() : TypeData(0, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -231,6 +243,7 @@ class TypeDataTuple final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 6; } std::string as_human_readable() const override { return "tuple"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -241,7 +254,7 @@ class TypeDataTuple final : public TypeData { * It's like "untyped callable", not compatible with other types. */ class TypeDataContinuation final : public TypeData { - TypeDataContinuation() : TypeData(7ULL, 0, 1) {} + TypeDataContinuation() : TypeData(0, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -249,6 +262,7 @@ class TypeDataContinuation final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 7; } std::string as_human_readable() const override { return "continuation"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -261,7 +275,7 @@ class TypeDataContinuation final : public TypeData { * (it's much better for user to see an error here than when he passes this variable somewhere). */ class TypeDataNullLiteral final : public TypeData { - TypeDataNullLiteral() : TypeData(8ULL, 0, 1) {} + TypeDataNullLiteral() : TypeData(0, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -269,43 +283,20 @@ class TypeDataNullLiteral final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 0; } std::string as_human_readable() const override { return "null"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; -/* - * `T?` is "nullable T". - * It can be converted to T either with ! (non-null assertion operator) or with smart casts. - */ -class TypeDataNullable final : public TypeData { - TypeDataNullable(uint64_t type_id, int children_flags, int width_on_stack, TypePtr inner) - : TypeData(type_id, children_flags, width_on_stack) - , inner(inner) {} - -public: - const TypePtr inner; - - static TypePtr create(TypePtr inner); - - bool is_primitive_nullable() const { return get_width_on_stack() == 1 && inner->get_width_on_stack() == 1; } - - std::string as_human_readable() const override; - bool can_rhs_be_assigned(TypePtr rhs) const override; - bool can_be_casted_with_as_operator(TypePtr cast_to) const override; - void traverse(const TraverserCallbackT& callback) const override; - TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; - bool can_hold_tvm_null_instead() const override; -}; - /* * `fun(int, int) -> void` is TypeDataFunCallable, think of is as a typed continuation. * A type of function `fun f(x: int) { return x; }` is actually `fun(int) -> int`. * So, when assigning it to a variable `var cb = f`, this variable also has this type. */ class TypeDataFunCallable final : public TypeData { - TypeDataFunCallable(uint64_t type_id, int children_flags, std::vector&& params_types, TypePtr return_type) - : TypeData(type_id, children_flags, 1) + TypeDataFunCallable(int children_flags, std::vector&& params_types, TypePtr return_type) + : TypeData(children_flags, 1) , params_types(std::move(params_types)) , return_type(return_type) {} @@ -317,6 +308,7 @@ class TypeDataFunCallable final : public TypeData { int params_size() const { return static_cast(params_types.size()); } + int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -330,8 +322,8 @@ class TypeDataFunCallable final : public TypeData { * On instantiation like `f(1,"")`, a new function `f` is created with type `fun(int,slice)->[int,slice]`. */ class TypeDataGenericT final : public TypeData { - TypeDataGenericT(uint64_t type_id, std::string&& nameT) - : TypeData(type_id, flag_contains_genericT_inside, -999999) // width undefined until instantiated + explicit TypeDataGenericT(std::string&& nameT) + : TypeData(flag_contains_genericT_inside, -999999) // width undefined until instantiated , nameT(std::move(nameT)) {} public: @@ -339,6 +331,7 @@ class TypeDataGenericT final : public TypeData { static TypePtr create(std::string&& nameT); + int get_type_id() const override; std::string as_human_readable() const override { return nameT; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -351,8 +344,8 @@ class TypeDataGenericT final : public TypeData { * A tensor can be empty. */ class TypeDataTensor final : public TypeData { - TypeDataTensor(uint64_t type_id, int children_flags, int width_on_stack, std::vector&& items) - : TypeData(type_id, children_flags, width_on_stack) + TypeDataTensor(int children_flags, int width_on_stack, std::vector&& items) + : TypeData(children_flags, width_on_stack) , items(std::move(items)) {} public: @@ -362,6 +355,7 @@ class TypeDataTensor final : public TypeData { int size() const { return static_cast(items.size()); } + int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -376,8 +370,8 @@ class TypeDataTensor final : public TypeData { * `var [i, cs] = [0, ""]` (where a and b become two separate variables on a stack, int and slice). */ class TypeDataTypedTuple final : public TypeData { - TypeDataTypedTuple(uint64_t type_id, int children_flags, std::vector&& items) - : TypeData(type_id, children_flags, 1) + TypeDataTypedTuple(int children_flags, std::vector&& items) + : TypeData(children_flags, 1) , items(std::move(items)) {} public: @@ -387,6 +381,7 @@ class TypeDataTypedTuple final : public TypeData { int size() const { return static_cast(items.size()); } + int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -401,8 +396,8 @@ class TypeDataTypedTuple final : public TypeData { * intN is smoothly cast from/to plain int, mathematical operators on intN also "fall back" to general int. */ class TypeDataIntN final : public TypeData { - TypeDataIntN(uint64_t type_id, bool is_unsigned, bool is_variadic, int n_bits) - : TypeData(type_id, 0, 1) + TypeDataIntN(bool is_unsigned, bool is_variadic, int n_bits) + : TypeData(0, 1) , is_unsigned(is_unsigned) , is_variadic(is_variadic) , n_bits(n_bits) {} @@ -414,6 +409,7 @@ class TypeDataIntN final : public TypeData { static TypePtr create(bool is_unsigned, bool is_variadic, int n_bits); + int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -424,7 +420,7 @@ class TypeDataIntN final : public TypeData { * Example: `var cost = ton("0.05")` has type `coins`. */ class TypeDataCoins final : public TypeData { - TypeDataCoins() : TypeData(17ULL, 0, 1) {} + TypeDataCoins() : TypeData(0, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -432,6 +428,7 @@ class TypeDataCoins final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 17; } std::string as_human_readable() const override { return "coins"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -444,8 +441,8 @@ class TypeDataCoins final : public TypeData { * Note, that unlike intN automatically cast to/from int, bytesN does NOT auto cast to slice (without `as`). */ class TypeDataBytesN final : public TypeData { - TypeDataBytesN(uint64_t type_id, bool is_bits, int n_width) - : TypeData(type_id, 0, 1) + TypeDataBytesN(bool is_bits, int n_width) + : TypeData(0, 1) , is_bits(is_bits) , n_width(n_width) {} @@ -455,11 +452,69 @@ class TypeDataBytesN final : public TypeData { static TypePtr create(bool is_bits, int n_width); + int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; +/* + * `T1 | T2 | ...` is a union type. Unions are supported in Tolk, similar to TypeScript, stored like enums in Rust. + * `T | null` (denoted as `T?`) is also a union from a type system point of view. + * There is no TypeDataNullable, because mixing unions and nullables would result in a mess. + * At TVM level: + * - `T | null`, if T is 1 slot (like `int | null`), then it's still 1 slot + * - `T | null`, if T is N slots (like `(int, int)?`), it's stored as N+1 slots (the last for type_id if T or 0 if null) + * - `T1 | T2 | ...` is a tagged union: occupy max(T_i)+1 slots (1 for type_id) + */ +class TypeDataUnion final : public TypeData { + TypeDataUnion(int children_flags, int width_on_stack, TypePtr or_null, std::vector&& variants) + : TypeData(children_flags, width_on_stack) + , or_null(or_null) + , variants(std::move(variants)) {} + + bool has_variant_with_type_id(int type_id) const; + static void append_union_type_variant(TypePtr variant, std::vector& out_unique_variants); + +public: + const TypePtr or_null; // if `T | null`, then T is here (variants = [T, null] then); otherwise, nullptr + const std::vector variants; // T_i, flattened, no duplicates; may include aliases, but not other unions + + static TypePtr create(std::vector&& variants); + static TypePtr create_nullable(TypePtr nullable); + + // "primitive nullable" is `T?` which holds TVM NULL in the same slot (it other words, has no UTag slot) + // true : `int?`, `slice?`, `StructWith1IntField?` + // false: `(int, int)?`, `ComplexStruct?`, `()?` + bool is_primitive_nullable() const { + return get_width_on_stack() == 1 && or_null != nullptr && or_null->get_width_on_stack() == 1; + } + bool has_null() const { + if (or_null) { + return true; + } + return has_variant_with_type_id(0); + } + bool has_variant_with_type_id(TypePtr rhs_type) const { + int type_id = rhs_type->get_type_id(); + if (or_null) { + return type_id == 0 || type_id == or_null->get_type_id(); + } + return has_variant_with_type_id(type_id); + } + + TypePtr calculate_exact_variant_to_fit_rhs(TypePtr rhs_type) const; + bool has_all_variants_of(const TypeDataUnion* rhs_type) const; + + int get_type_id() const override; + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + void traverse(const TraverserCallbackT& callback) const override; + TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; + bool can_hold_tvm_null_instead() const override; +}; + /* * `unknown` is a special type, which can appear in corner cases. * The type of exception argument (which can hold any TVM value at runtime) is unknown. @@ -467,7 +522,7 @@ class TypeDataBytesN final : public TypeData { * The only thing available to do with unknown is to cast it: `catch (excNo, arg) { var i = arg as int; }` */ class TypeDataUnknown final : public TypeData { - TypeDataUnknown() : TypeData(20ULL, flag_contains_unknown_inside, 1) {} + TypeDataUnknown() : TypeData(flag_contains_unknown_inside, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -475,6 +530,7 @@ class TypeDataUnknown final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override; std::string as_human_readable() const override { return "unknown"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -487,8 +543,8 @@ class TypeDataUnknown final : public TypeData { * Example: `fun f(v: T)` at first v is TypeDataUnresolved("T"), later becomes TypeDataGenericT. */ class TypeDataUnresolved final : public TypeData { - TypeDataUnresolved(uint64_t type_id, std::string&& text, SrcLocation loc) - : TypeData(type_id, flag_contains_unresolved_inside, -999999) + TypeDataUnresolved(std::string&& text, SrcLocation loc) + : TypeData(flag_contains_unresolved_inside, -999999) , text(std::move(text)) , loc(loc) {} @@ -498,6 +554,7 @@ class TypeDataUnresolved final : public TypeData { static TypePtr create(std::string&& text, SrcLocation loc); + int get_type_id() const override; std::string as_human_readable() const override { return text + "*"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -510,7 +567,7 @@ class TypeDataUnresolved final : public TypeData { * Such variables can not be cast to any other types, all their usage will trigger type mismatch errors. */ class TypeDataNever final : public TypeData { - TypeDataNever() : TypeData(19ULL, 0, 0) {} + TypeDataNever() : TypeData(0, 0) {} static TypePtr singleton; friend void type_system_init(); @@ -518,6 +575,7 @@ class TypeDataNever final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 19; } std::string as_human_readable() const override { return "never"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -530,7 +588,7 @@ class TypeDataNever final : public TypeData { * Empty tensor is not compatible with void, although at IR level they are similar, 0 stack slots. */ class TypeDataVoid final : public TypeData { - TypeDataVoid() : TypeData(10ULL, 0, 0) {} + TypeDataVoid() : TypeData(0, 0) {} static TypePtr singleton; friend void type_system_init(); @@ -538,6 +596,7 @@ class TypeDataVoid final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 10; } std::string as_human_readable() const override { return "void"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -547,14 +606,6 @@ class TypeDataVoid final : public TypeData { // -------------------------------------------- -inline TypePtr TypeData::unwrap_alias() const { - TypePtr unwrapped = this; - while (const TypeDataAlias* as_alias = unwrapped->try_as()) { - unwrapped = as_alias->underlying_type; - } - return unwrapped; -} - class Lexer; TypePtr parse_type_from_tokens(Lexer& lex); TypePtr parse_type_from_string(std::string_view text); From ac57626d342d656af3f120764949294bdb2b801d Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sat, 5 Apr 2025 17:15:07 +0300 Subject: [PATCH 202/524] [Tolk] Pattern matching for expressions (switch-like behavior) `match` can also be used for constant expressions, similar to `switch`: > val nextValue = match (curValue) { > 1 => 0, > 0 => 1, > else => -1 > }; Only constant expressions are allowed on the left-hand side. `else` is required for expression form but optional for statement form. --- tolk-tester/tests/if-else-tests.tolk | 107 +++++++++ .../tests/invalid-declaration/err-1808.tolk | 6 + .../tests/invalid-semantics/err-4258.tolk | 13 ++ .../tests/invalid-semantics/err-4517.tolk | 14 ++ .../tests/invalid-syntax/err-3080.tolk | 11 + .../tests/invalid-syntax/err-3419.tolk | 12 + .../tests/invalid-syntax/err-3457.tolk | 2 +- .../tests/invalid-syntax/err-3837.tolk | 14 ++ .../tests/invalid-syntax/err-3913.tolk | 8 + tolk-tester/tests/match-by-expr-tests.tolk | 205 ++++++++++++++++++ tolk-tester/tests/union-types-tests.tolk | 29 +++ tolk/analyzer.cpp | 27 ++- tolk/constant-evaluator.cpp | 22 +- tolk/constant-evaluator.h | 1 + tolk/pipe-ast-to-legacy.cpp | 89 ++++---- tolk/pipe-check-inferred-types.cpp | 29 ++- tolk/pipe-constant-folding.cpp | 17 +- 17 files changed, 551 insertions(+), 55 deletions(-) create mode 100644 tolk-tester/tests/invalid-declaration/err-1808.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4258.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4517.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3080.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3419.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3837.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3913.tolk create mode 100644 tolk-tester/tests/match-by-expr-tests.tolk diff --git a/tolk-tester/tests/if-else-tests.tolk b/tolk-tester/tests/if-else-tests.tolk index 0f1d31f942..b16aef15ee 100644 --- a/tolk-tester/tests/if-else-tests.tolk +++ b/tolk-tester/tests/if-else-tests.tolk @@ -72,6 +72,33 @@ fun test7() { // here we test that no "missing return" error } +fun withNoElse(op: int): int { + if (op == 123) { return 100; } + if (op == 234) { return 200; } + if (op == 345) { return 300; } + throw 0xFF; +} + +fun withElse(op: int): int { + if (op == 123) { return 100; } + else if (op == 234) { return 200; } + else if (op == 345) { return 300; } + return 0xFF; +} + +fun withMatch(op: int): int { + match (op) { + 123 => return 100, + 234 => return 200, + 345 => return 300, + } + throw 0xFF; +} + +@method_id(108) +fun test8(op: int) { + return (withNoElse(op), withElse(op), withMatch(op)); +} fun main() { @@ -96,6 +123,8 @@ fun main() { @testcase | 105 | 3 | 7 222 @testcase | 106 | -5 | -1 @testcase | 106 | 5 | 5000 +@testcase | 108 | 123 | 100 100 100 +@testcase | 108 | 345 | 300 300 300 @fif_codegen """ @@ -111,4 +140,82 @@ fun main() { 50 EQINT // x '5 IFNOTJMP:<{ // x """ + +@fif_codegen +""" + withNoElse PROC:<{ + // op + DUP + 123 EQINT + IFJMP:<{ + DROP + 100 PUSHINT + }> + DUP + 234 PUSHINT + EQUAL + IFJMP:<{ + DROP + 200 PUSHINT + }> + 345 PUSHINT + EQUAL + IFJMP:<{ + 300 PUSHINT + }> + 255 THROW + }> +""" + +@fif_codegen +""" + withElse PROC:<{ + // op + DUP + 123 EQINT + IFJMP:<{ + DROP + 100 PUSHINT + }> + DUP + 234 PUSHINT + EQUAL + IFJMP:<{ + DROP + 200 PUSHINT + }> + 345 PUSHINT + EQUAL + IFJMP:<{ + 300 PUSHINT + }> + 8 PUSHPOW2DEC + }> +""" + +@fif_codegen +""" + withMatch PROC:<{ + // op + DUP + 123 EQINT + IFJMP:<{ + DROP + 100 PUSHINT + }> + DUP + 234 PUSHINT + EQUAL + IFJMP:<{ + DROP + 200 PUSHINT + }> + 345 PUSHINT + EQUAL + IFJMP:<{ + 300 PUSHINT + }> + 255 THROW + }> +""" */ diff --git a/tolk-tester/tests/invalid-declaration/err-1808.tolk b/tolk-tester/tests/invalid-declaration/err-1808.tolk new file mode 100644 index 0000000000..a7faa6ff3d --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1808.tolk @@ -0,0 +1,6 @@ +const C = match (10) { 10 => 0, else => -1 }; + +/** +@compilation_should_fail +@stderr not a constant expression + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4258.tolk b/tolk-tester/tests/invalid-semantics/err-4258.tolk new file mode 100644 index 0000000000..d60d1adf37 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4258.tolk @@ -0,0 +1,13 @@ +fun f(x: int): int { + return match(x) { + -1 => -1, + f(0) => 0, + else => 1, + }; +} + +/** +@compilation_should_fail +@stderr not a constant expression +@stderr f(0) => 0 + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4517.tolk b/tolk-tester/tests/invalid-semantics/err-4517.tolk new file mode 100644 index 0000000000..f84e3ac1b9 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4517.tolk @@ -0,0 +1,14 @@ +global g: int; + +fun f(x: int): int { + match(x) { + -1 => -1, + g => 0, + }; + return 10; +} + +/** +@compilation_should_fail +@stderr symbol `g` is not a constant + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3080.tolk b/tolk-tester/tests/invalid-syntax/err-3080.tolk new file mode 100644 index 0000000000..e79e8ee95d --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3080.tolk @@ -0,0 +1,11 @@ +fun f(x: int) { + return match(x) { + -1 => -1, + 0 => 0 + }; +} + +/** +@compilation_should_fail +@stderr `match` expression should have `else` branch + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3419.tolk b/tolk-tester/tests/invalid-syntax/err-3419.tolk new file mode 100644 index 0000000000..a59ca0e68e --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3419.tolk @@ -0,0 +1,12 @@ +fun f(x: int) { + return match(x) { + -1 => -1, + else => 0, + 0 => 0 + }; +} + +/** +@compilation_should_fail +@stderr `else` branch should be the last + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3457.tolk b/tolk-tester/tests/invalid-syntax/err-3457.tolk index a2ef6e96b0..8cfcf8e6ca 100644 --- a/tolk-tester/tests/invalid-syntax/err-3457.tolk +++ b/tolk-tester/tests/invalid-syntax/err-3457.tolk @@ -13,6 +13,6 @@ fun main() { /** @compilation_should_fail -@stderr `match` by expression is not supported yet +@stderr symbol `asdf` is not a constant @stderr asdf => 2 */ diff --git a/tolk-tester/tests/invalid-syntax/err-3837.tolk b/tolk-tester/tests/invalid-syntax/err-3837.tolk new file mode 100644 index 0000000000..c0ee50b30a --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3837.tolk @@ -0,0 +1,14 @@ +fun f(x: int) { + match(x) { + -1 => -1, + 0 => 0, + else => 0, + else => -1 + }; +} + +/** +@compilation_should_fail +@stderr duplicated `else` branch +@stderr else => -1 + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3913.tolk b/tolk-tester/tests/invalid-syntax/err-3913.tolk new file mode 100644 index 0000000000..033743259f --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3913.tolk @@ -0,0 +1,8 @@ +fun f(x: bool) { + return match (x) { true => 0, true => -1 }; +} + +/** +@compilation_should_fail +@stderr `match` expression should have `else` branch + */ diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk new file mode 100644 index 0000000000..4bb71c6648 --- /dev/null +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -0,0 +1,205 @@ +const C1 = 1; +const C2 = C1 + 1; + +@method_id(101) +fun test1(x: int) { + return match (x) { + 1 => 100, + C2 => 200, + 2 + 1 => 300, + else => 400 + }; +} + +@method_id(102) +fun test2() { + var x = 3; + return match (x) { + 1 => 100, + C2 => 200, + 2 + 1 => 300, + else => 400 + }; +} + +fun isGt10(x: int) { return x > 10; } + +@method_id(103) +fun test3(init: int) { + var r1 = match (isGt10(init)) { true => 10, false => 20 }; + var r2 = match (isGt10(init)) { !!true => 10, !!false => 20, else => 8 }; + var r3 = match (isGt10(init)) { 10 > 3 => 10, 10 < 3 => 20, 100 > 1 => 30, else => 8 }; + return (r1, r2, r3); +} + +@method_id(104) +fun test4(x: int?) { + return match(match(x) { null => 0, int => x * 2 }) { 0 => 0, 30 => 30, else => throw 123 }; +} + +@method_id(105) +fun test5(x: int) { + try { + return match (val r = x * 2) { + 10/C1 => 10, + C2*10 => throw 500, + else => 50 + }; + } catch (excCode) { + return excCode; + } +} + +@method_id(106) +fun test6(x: coins) { + var result1 = 0; + match (x) { + ton("0.05") => { result1 = 5; } + ton("0.1") => { result1 = 10; } + else => { result1 = 20; } + } + var result2 = 0; + match (x) { + ton("0.05") => { result2 = 5; } + ton("0.1") => { result2 = 10; } + } + return (result1, result2); +} + +@method_id(107) +fun test7(x: int8) { + match (x) { 10 => {} } + return match (x) { + 10 => 10, + 20 as int8 => 20, + 30 as int16 => 30, + 40 as uint255 => 40, + else => 50, + }; +} + +@method_id(108) +fun test8(x: int) { + var r1 = match (x) { else => 10 }; + match (x) { } + match (x) { else => { r1 += 20; } } + return r1; +} + +@method_id(109) +fun test9(x: int | slice) { + if (x is slice) { + match (x.loadInt(32)) { -1 => {}, 0 => {} } + } + return match (x) { + int => match (x) { 10 => 5, else => -1 }, + slice => match (x.loadInt(32)) { -1 => 0, 0 => 0, else => throw 20 } + }; +} + +@method_id(110) +fun test10(x: int) { + var result1 = match (x) { + 10 => throw 10, + 20 => return 20, + 30 => { x += 10; return x; } + else => x + }; + return result1 * 2; +} + +@method_id(111) +fun test11(x: bool) { + return match (x) { + true => false, + else => !x, + }; +} + + +fun main() {} + +/** +@testcase | 101 | 1 | 100 +@testcase | 101 | 2 | 200 +@testcase | 101 | 3 | 300 +@testcase | 101 | 4 | 400 +@testcase | 102 | | 300 +@testcase | 103 | 5 | 20 20 20 +@testcase | 103 | 15 | 10 10 10 +@testcase | 104 | 15 | 30 +@testcase | 104 | null | 0 +@testcase | 105 | 10 | 500 +@testcase | 105 | 11 | 50 +@testcase | 106 | 50000000 | 5 5 +@testcase | 106 | 300 | 20 0 +@testcase | 107 | 30 | 30 +@testcase | 107 | 40 | 40 +@testcase | 107 | 41 | 50 +@testcase | 108 | 5 | 30 +@testcase | 109 | 1 1 | -1 +@testcase | 110 | 20 | 20 +@testcase | 110 | 30 | 40 +@testcase | 110 | 50 | 100 +@testcase | 111 | 0 | -1 +@testcase | 111 | -1 | 0 + +@fif_codegen +""" + test1 PROC:<{ + // x + DUP // x x + 1 EQINT // x '3 + IF:<{ // x + DROP // + 100 PUSHINT // '1=100 + }>ELSE<{ // x + DUP // x x + 2 EQINT // x '6 + IF:<{ // x + DROP // + 200 PUSHINT // '1=200 + }>ELSE<{ // x + 3 EQINT // '9 + IF:<{ // + 300 PUSHINT // '1=300 + }>ELSE<{ // + 400 PUSHINT // '1=400 + }> + }> + }> + }> +""" + +@fif_codegen +""" + test2 PROC:<{ + // + 300 PUSHINT + }> +""" + +@fif_codegen +""" + test3 PROC:<{ + // init + DUP // init init + isGt10 CALLDICT // init '2 + -1 EQINT // init '5 + IF:<{ // init + 10 PUSHINT // init '3=10 + }>ELSE<{ // init + 20 PUSHINT // init '3=20 + }> // init r1 +""" + +@fif_codegen +""" + test8 PROC:<{ + // x + DROP // + 30 PUSHINT // r1 + }> +""" + + */ diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 04446f8e73..e451f13ddd 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -764,6 +764,15 @@ fun test60() { return (returnVoidOrNull(true), returnVoidOrNull(false)); } +@method_id(161) +fun test61(i: int32 | int64 | int) { + match (i) { + int32 => { return 10; } + int64 => { return 20; } + int => { return 30; } + } +} + fun main() { @@ -837,6 +846,7 @@ fun main() { @testcase | 159 | 0 4 | 456 @testcase | 159 | null 0 | 123 @testcase | 160 | | 10 0 +@testcase | 161 | 100 48 | 20 @fif_codegen @@ -868,6 +878,25 @@ fun main() { }> """ +@fif_codegen +""" + test61 PROC:<{ + // i.USlot1 i.UTag + NIP + DUP + 46 EQINT + IFJMP:<{ + DROP + 10 PUSHINT + }> + 48 EQINT + IFJMP:<{ + 20 PUSHINT + }> + 30 PUSHINT + }> +""" + @fif_codegen DECLPROC checkGeneric3 */ diff --git a/tolk/analyzer.cpp b/tolk/analyzer.cpp index c38b0bfa40..2cd65c419e 100644 --- a/tolk/analyzer.cpp +++ b/tolk/analyzer.cpp @@ -874,7 +874,32 @@ bool Op::mark_noreturn() { return set_noreturn(); case _Call: return set_noreturn(next->mark_noreturn() || does_function_always_throw(f_sym)); - case _If: + case _If: { + // this very-not-beautiful code does the following: + // replace `if (cond) { return; } else { block1; } next;` with `if (cond) { return; } block1; next` + // purpose: to make code like `if (...) { ... return; } else if (...) { ... return; } ...` act like without else + // similarly, `match (...) { v1 => { ... return; } ...}` is internally transformed to IF-ELSE + // (that's why such transformation is done at IR level, not in AST) + // without these code (without else removed), extra RETALT instructions are inserted, not necessary actually + // implementation is UGLY, because currently there is no way to perform IR replacements + // in the future, anyway, IR implementation should be rewritten, for easier traversing and replacement + // btw, now it doesn't work with `if (!...)` (v->is_ifnot), else "keyword" is not removed + if (block0->mark_noreturn() && !block1->is_empty()) { + VarDescrList block1_var_info = block1->var_info; // important to keep it + Op* last_in_block1 = block1.get(); + while (last_in_block1->next->cl != Op::_Nop) { // find the tail of a forward list of Ops + last_in_block1 = last_in_block1->next.get(); + } + last_in_block1->next = std::move(next); + next = std::move(block1); + block1 = std::make_unique(where, Op::_Nop); + block1->var_info = std::move(block1_var_info); + } else { + block1->mark_noreturn(); + } + bool next_noreturn = next->mark_noreturn(); + return set_noreturn((block0->noreturn() && block1->noreturn()) || next_noreturn); + } case _TryCatch: // note, that & | (not && ||) here and below is mandatory to invoke both left and right calls return set_noreturn((static_cast(block0->mark_noreturn()) & static_cast(block1 && block1->mark_noreturn())) | static_cast(next->mark_noreturn())); diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index 7daebc1da3..ff8dbbce18 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -384,12 +384,11 @@ struct ConstantEvaluator { v->error("not a constant expression"); } - // evaluate `const a = 2 + 3` into 5 - // type inferring has already passed, to types are correct, `const a = 1 + ""` can not occur - // recursive initializers `const a = b; const b = a` also 100% don't exist, checked on type inferring - // if init_value is not a constant expression like `const a = foo()`, an exception is thrown - static ConstantValue eval_const_init_value(GlobalConstPtr const_ref) { - return visit(const_ref->init_value); + // evaluate `2 + 3` into 5 + // type inferring has already passed, to types are correct, `1 + ""` can not occur + // if v is not a constant expression like `foo()`, an exception is thrown + static ConstantValue eval_expression_expected_to_be_constant(AnyExprV v) { + return visit(v); } }; @@ -405,8 +404,17 @@ ConstantValue eval_call_to_compile_time_function(AnyExprV v_call) { return parse_vertex_call_to_compile_time_function(v, v->fun_maybe->name); } +// evaluate `2 + 3` into `5` +// if v is not a constant expression like `gVar`, an exception is thrown +ConstantValue eval_expression_expected_to_be_constant(AnyExprV v) { + return ConstantEvaluator::eval_expression_expected_to_be_constant(v); +} + +// evaluate `const a = 2 + 3` into `const a = 5` +// recursive initializers `const a = b; const b = a` don't exist, checked on type inferring +// if init_value is not a constant expression like `const a = foo()`, an exception is thrown void eval_and_assign_const_init_value(GlobalConstPtr const_ref) { - ConstantValue init_value = ConstantEvaluator::eval_const_init_value(const_ref); + ConstantValue init_value = ConstantEvaluator::eval_expression_expected_to_be_constant(const_ref->init_value); const_ref->mutate()->assign_const_value(std::move(init_value)); } diff --git a/tolk/constant-evaluator.h b/tolk/constant-evaluator.h index b065e6f49b..1a7f364fbe 100644 --- a/tolk/constant-evaluator.h +++ b/tolk/constant-evaluator.h @@ -49,6 +49,7 @@ class ConstantValue { ConstantValue eval_string_const_standalone(AnyExprV v_string); ConstantValue eval_call_to_compile_time_function(AnyExprV v_call); +ConstantValue eval_expression_expected_to_be_constant(AnyExprV v); void eval_and_assign_const_init_value(GlobalConstPtr const_ref); } // namespace tolk diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 167c561477..bba79ea923 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1018,60 +1018,67 @@ static std::vector process_not_null_operator(V static std::vector process_match_expression(V v, CodeBlob& code, TypePtr target_type) { TypePtr lhs_type = v->get_subject()->inferred_type->unwrap_alias(); + int n_arms = v->get_arms_count(); std::vector subj_ir_idx = pre_compile_expr(v->get_subject(), code, nullptr); std::vector result_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(match-expression)"); - // detect whether it's `match` by type covering all cases; type checker has already checked that - // match by type can not be mixed with match by expression and can't contain `else`, and that types are distinct and cover all variants - bool is_exhaustive_match_by_type = v->get_arms_count() > 0 && v->get_arm(0)->pattern_kind == MatchArmKind::exact_type; - if (is_exhaustive_match_by_type) { - // example: `match (v) { int => ... slice => ... builder => ... }` - // construct nested IFs: IF is int { ... } ELSE { IF is slice { ... } ELSE { ... } } - for (int i = 0; i < v->get_arms_count() - 1; ++i) { - auto v_ith_arm = v->get_arm(i); + if (!n_arms) { // `match (subject) {}` + tolk_assert(v->is_statement()); + return {}; + } + + // it's either `match` by type (all arms patterns are types) or `match` by expression + bool is_match_by_type = v->get_arm(0)->pattern_kind == MatchArmKind::exact_type; + // detect whether `match` is exhaustive + bool is_exhaustive = is_match_by_type // match by type covers all cases, checked earlier + || !v->is_statement() // match by expression is guaranteely exhaustive, checked earlier + || v->get_arm(n_arms - 1)->pattern_kind == MatchArmKind::else_branch; + + // example 1 (exhaustive): `match (v) { int => ... slice => ... builder => ... }` + // construct nested IFs: IF is int { ... } ELSE { IF is slice { ... } ELSE { ... } } + // example 2 (exhaustive): `match (v) { -1 => ... 0 => ... else => ... }` + // construct nested IFs: IF is int { ... } ELSE { IF is slice { ... } ELSE { ... } } + // example 3 (not exhaustive): `match (v) { -1 => ... 0 => ... 1 => ... }` + // construct nested IFs: IF == -1 { ... } ELSE { IF == 0 { ... } ELSE { IF == 1 { ... } } } + FunctionPtr eq_sym = lookup_global_symbol("_==_")->try_as(); + for (int i = 0; i < n_arms - is_exhaustive; ++i) { + auto v_ith_arm = v->get_arm(i); + std::vector eq_ith_ir_idx; + if (is_match_by_type) { TypePtr cmp_type = v_ith_arm->exact_type->unwrap_alias(); tolk_assert(!cmp_type->try_as()); // `match` over `int|slice` is a type checker error - std::vector eq_ith_ir_idx = pre_compile_is_type(code, lhs_type, cmp_type, subj_ir_idx, v_ith_arm->loc, "(arm-cond-eq)"); - Op& if_op = code.emplace_back(v_ith_arm->loc, Op::_If, std::move(eq_ith_ir_idx)); - code.push_set_cur(if_op.block0); - if (v->is_statement()) { - pre_compile_expr(v_ith_arm->get_body(), code); - } else { - std::vector arm_ir_idx = pre_compile_expr(v_ith_arm->get_body(), code, v->inferred_type); - code.emplace_back(v->loc, Op::_Let, result_ir_idx, std::move(arm_ir_idx)); - } - code.close_pop_cur(v->loc); - code.push_set_cur(if_op.block1); // open ELSE + eq_ith_ir_idx = pre_compile_is_type(code, lhs_type, cmp_type, subj_ir_idx, v_ith_arm->loc, "(arm-cond-eq)"); + } else { + std::vector ith_ir_idx = pre_compile_expr(v_ith_arm->get_pattern_expr(), code); + tolk_assert(subj_ir_idx.size() == 1 && ith_ir_idx.size() == 1); + eq_ith_ir_idx = code.create_tmp_var(TypeDataBool::create(), v_ith_arm->loc, "(arm-cond-eq)"); + code.emplace_back(v_ith_arm->loc, Op::_Call, eq_ith_ir_idx, std::vector{subj_ir_idx[0], ith_ir_idx[0]}, eq_sym); + } + Op& if_op = code.emplace_back(v_ith_arm->loc, Op::_If, std::move(eq_ith_ir_idx)); + code.push_set_cur(if_op.block0); + if (v->is_statement()) { + pre_compile_expr(v_ith_arm->get_body(), code); + } else { + std::vector arm_ir_idx = pre_compile_expr(v_ith_arm->get_body(), code, v->inferred_type); + code.emplace_back(v->loc, Op::_Let, result_ir_idx, std::move(arm_ir_idx)); } + code.close_pop_cur(v->loc); + code.push_set_cur(if_op.block1); // open ELSE + } - // we're inside the last ELSE, close all outer IFs - auto v_last_arm = v->get_arm(v->get_arms_count() - 1); + if (is_exhaustive) { + // we're inside the last ELSE + auto v_last_arm = v->get_arm(n_arms - 1); if (v->is_statement()) { pre_compile_expr(v_last_arm->get_body(), code); } else { std::vector arm_ir_idx = pre_compile_expr(v_last_arm->get_body(), code, v->inferred_type); code.emplace_back(v->loc, Op::_Let, result_ir_idx, std::move(arm_ir_idx)); } - for (int i = 0; i < v->get_arms_count() - 1; ++i) { - code.close_pop_cur(v->loc); - } - - } else { - // not exhaustive `match` by type - for (int i = 0; i < v->get_arms_count(); ++i) { - auto v_arm = v->get_arm(i); - switch (v_arm->pattern_kind) { - case MatchArmKind::const_expression: - tolk_assert(false); // match by expressions is not supported yet, checked earlier - break; - case MatchArmKind::exact_type: - tolk_assert(false); // match by type is exhaustive, checked earlier - break; - case MatchArmKind::else_branch: - tolk_assert(false); // `else` in match can be only inside match by expression, unsupported yet - break; - } - } + } + // close all outer IFs + for (int i = 0; i < n_arms - is_exhaustive; ++i) { + code.close_pop_cur(v->loc); } return transition_to_target_type(std::move(result_ir_idx), code, target_type, v); diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 2174560fdb..b9bcdaced8 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -227,7 +227,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { bool both_int = expect_integer(lhs) && expect_integer(rhs); bool both_bool = expect_boolean(lhs) && expect_boolean(rhs); if (!both_int && !both_bool) { - if (lhs->inferred_type == rhs->inferred_type) { // compare slice with slice, int? with int? + if (lhs->inferred_type->equal_to(rhs->inferred_type)) { // compare slice with slice, int? with int? fire(cur_f, v->loc, "type " + to_string(lhs) + " can not be compared with `== !=`"); } else { fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); @@ -525,7 +525,8 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { bool has_type_arm = false; bool has_expr_arm = false; bool has_else_arm = false; - TypePtr subject_type = v->get_subject()->inferred_type->unwrap_alias(); + AnyExprV v_subject = v->get_subject(); + TypePtr subject_type = v_subject->inferred_type->unwrap_alias(); const TypeDataUnion* subject_union = subject_type->try_as(); std::vector covered_type_ids; // for type-based `match`, what types are on the left of `=>` @@ -565,7 +566,16 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { fire(cur_f, v_arm->loc, "`else` branch should be the last"); } has_expr_arm = true; - fire(cur_f, v_arm->loc, "`match` by expression is not supported yet"); + TypePtr pattern_type = v_arm->get_pattern_expr()->inferred_type->unwrap_alias(); + bool both_int = expect_integer(pattern_type) && expect_integer(subject_type); + bool both_bool = expect_boolean(pattern_type) && expect_boolean(subject_type); + if (!both_int && !both_bool) { + if (pattern_type->equal_to(subject_type)) { // `match` over `slice` etc., where operator `==` can't be applied + fire(cur_f, v_arm->loc, "wrong pattern matching: can not compare type " + to_string(subject_type) + " in `match`"); + } else { + fire(cur_f, v_arm->loc, "wrong pattern matching: can not compare type " + to_string(v_arm->get_pattern_expr()) + " with match subject of type " + to_string(v_subject)); + } + } break; } default: @@ -576,7 +586,6 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { fire(cur_f, v_arm->loc, "`else` is not allowed in `match` by type; you should cover all possible types"); } has_else_arm = true; - fire(cur_f, v_arm->loc, "`match` by expression is not supported yet"); } } @@ -593,6 +602,18 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } throw ParseError(cur_f, v->loc, "`match` does not cover all possible types; missing types are: " + missing); } + // `match` by expression, if it's not statement, should have `else` (unless it's match over bool with const true/false) + if (has_expr_arm && !has_else_arm && !v->is_statement()) { + bool needs_else_branch = true; + if (expect_boolean(subject_type) && v->get_arms_count() == 2) { + auto arm0 = v->get_arm(0)->get_pattern_expr()->try_as(); + auto arm1 = v->get_arm(1)->get_pattern_expr()->try_as(); + needs_else_branch = !(arm0 && arm1 && arm0->bool_val != arm1->bool_val); + } + if (needs_else_branch) { + fire(cur_f, v->loc, "`match` expression should have `else` branch"); + } + } } void visit(V v) override { diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index 1a071f4923..c64f7bff50 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -134,7 +134,22 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { return v; } - public: + AnyExprV replace(V v) override { + parent::replace(v); + + // replace `2 + 3 => ...` with `5 => ...` + // non-constant expressions like `foo() => ...` fire an error here + if (v->pattern_kind == MatchArmKind::const_expression && v->get_pattern_expr()->type != ast_int_const) { + ConstantValue value = eval_expression_expected_to_be_constant(v->get_pattern_expr()); + tolk_assert(value.is_int()); + AnyExprV v_new_pattern = create_int_const(v->get_pattern_expr()->loc, value.as_int()); + v->mutate()->assign_resolved_pattern(MatchArmKind::const_expression, nullptr, v_new_pattern); + } + + return v; + } + +public: bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } From 9c0f893f6a53661e71dbb2c2502167c763bdb5ea Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 28 Mar 2025 13:26:40 +0300 Subject: [PATCH 203/524] [Tolk] Optional semicolon for the last statement in a block While semicolons are required between statements to disambiguate sequencing, the trailing semicolon on the last statement is now optional. This change is backward-compatible and does not affect existing code. --- tolk-tester/tests/a-tests.tolk | 14 ++-- tolk-tester/tests/if-else-tests.tolk | 20 ++--- tolk-tester/tests/logical-operators.tolk | 6 +- tolk-tester/tests/match-by-expr-tests.tolk | 18 ++--- tolk-tester/tests/no-spaces.tolk | 16 ++-- tolk-tester/tests/nullable-types.tolk | 4 +- tolk-tester/tests/smart-cast-tests.tolk | 8 +- tolk-tester/tests/unbalanced-ret.tolk | 6 +- tolk-tester/tests/union-types-tests.tolk | 74 +++++++++--------- tolk-tester/tests/var-apply-tests.tolk | 26 +++---- tolk/ast-from-tokens.cpp | 88 ++++++++++------------ tolk/ast-from-tokens.h | 25 ------ tolk/ast-replacer.h | 6 +- tolk/ast-replicator.h | 10 +-- tolk/ast-stringifier.h | 8 +- tolk/ast-visitor.h | 6 +- tolk/ast.cpp | 2 +- tolk/ast.h | 52 ++++++------- tolk/pipe-ast-to-legacy.cpp | 10 +-- tolk/pipe-calc-rvalue-lvalue.cpp | 2 +- tolk/pipe-check-inferred-types.cpp | 4 +- tolk/pipe-discover-parse-sources.cpp | 5 +- tolk/pipe-infer-types-and-calls.cpp | 12 +-- tolk/pipe-register-symbols.cpp | 2 +- tolk/pipe-resolve-identifiers.cpp | 10 +-- 25 files changed, 201 insertions(+), 233 deletions(-) delete mode 100644 tolk/ast-from-tokens.h diff --git a/tolk-tester/tests/a-tests.tolk b/tolk-tester/tests/a-tests.tolk index 65bff5ad11..077303d648 100644 --- a/tolk-tester/tests/a-tests.tolk +++ b/tolk-tester/tests/a-tests.tolk @@ -4,19 +4,19 @@ fun f(a: int, b: int, c: int, d: int, e: int, f: int): (int, int) { __expect_type(D, "int"); __expect_type(D*D, "int"); __expect_type(calc_phi, "() -> int"); - return (Dx/D,Dy/D,); + return (Dx/D,Dy/D,) };;;; fun calc_phi(): int { var n = 1; - repeat (70) { n*=10; }; + repeat (70) { n*=10 }; var p= 1; var `q`=1; _=`q`; do { - (p,q)=(q,p+q); + (p,q)=(q,p+q) } while (q <= n); //;; - return mulDivRound(p, n, q,); + return mulDivRound(p, n, q,) } fun calc_sqrt2(): int { @@ -43,7 +43,7 @@ fun calc_root(m: int,) { k+=1; (a1, b1, c1) = (a, b, c); c+=b; - c += b += a; + c += b += a } while (c <= 0); (a, b, c) = (-c1, -b1, -a1); (p1, q1) = (k * p1+q1, p1); @@ -67,12 +67,12 @@ fun ataninv(base: int, q: int): int { // computes base*atan(1/q) fun calc_pi(): int { var base: int = 64; - repeat (70) { base *= 10; } + repeat (70) { base *= 10 } return (ataninv(base << 2, 5) - ataninv(base, 239))~>>4; } fun main(): int { - return calc_pi(); + return calc_pi() } /** diff --git a/tolk-tester/tests/if-else-tests.tolk b/tolk-tester/tests/if-else-tests.tolk index b16aef15ee..f40336a9ec 100644 --- a/tolk-tester/tests/if-else-tests.tolk +++ b/tolk-tester/tests/if-else-tests.tolk @@ -68,22 +68,22 @@ fun doNothing(): void {} @method_id(107) fun test7() { - if (random()) { return doNothing(); } + if (random()) { return doNothing() } // here we test that no "missing return" error } fun withNoElse(op: int): int { - if (op == 123) { return 100; } - if (op == 234) { return 200; } - if (op == 345) { return 300; } - throw 0xFF; + if (op == 123) { return 100 } + if (op == 234) { return 200 } + if (op == 345) { return 300 } + throw 0xFF } fun withElse(op: int): int { - if (op == 123) { return 100; } - else if (op == 234) { return 200; } - else if (op == 345) { return 300; } - return 0xFF; + if (op == 123) { return 100 } + else if (op == 234) { return 200 } + else if (op == 345) { return 300 } + return 0xFF } fun withMatch(op: int): int { @@ -92,7 +92,7 @@ fun withMatch(op: int): int { 234 => return 200, 345 => return 300, } - throw 0xFF; + throw 0xFF } @method_id(108) diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk index 700f2a3c57..cbc82f15e8 100644 --- a/tolk-tester/tests/logical-operators.tolk +++ b/tolk-tester/tests/logical-operators.tolk @@ -13,9 +13,9 @@ fun compileTimeEval1(x: int) { @method_id(101) fun withIfNot(x: int, y: int) { - if (!x) { return 10; } - else if (!y) { return 20; } - return x+y; + if (!x) { return 10 } + else if (!y) { { return 20 } } + return x+y } @method_id(102) diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk index 4bb71c6648..d9ecb3cfec 100644 --- a/tolk-tester/tests/match-by-expr-tests.tolk +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -8,7 +8,7 @@ fun test1(x: int) { C2 => 200, 2 + 1 => 300, else => 400 - }; + } } @method_id(102) @@ -19,7 +19,7 @@ fun test2() { C2 => 200, 2 + 1 => 300, else => 400 - }; + } } fun isGt10(x: int) { return x > 10; } @@ -34,7 +34,7 @@ fun test3(init: int) { @method_id(104) fun test4(x: int?) { - return match(match(x) { null => 0, int => x * 2 }) { 0 => 0, 30 => 30, else => throw 123 }; + return match(match(x) { null => 0, int => x * 2 }) { 0 => 0, 30 => 30, else => throw 123 } } @method_id(105) @@ -44,9 +44,9 @@ fun test5(x: int) { 10/C1 => 10, C2*10 => throw 500, else => 50 - }; + } } catch (excCode) { - return excCode; + return excCode } } @@ -60,8 +60,8 @@ fun test6(x: coins) { } var result2 = 0; match (x) { - ton("0.05") => { result2 = 5; } - ton("0.1") => { result2 = 10; } + ton("0.05") => { result2 = 5 } + ton("0.1") => { result2 = 10 } } return (result1, result2); } @@ -94,7 +94,7 @@ fun test9(x: int | slice) { return match (x) { int => match (x) { 10 => 5, else => -1 }, slice => match (x.loadInt(32)) { -1 => 0, 0 => 0, else => throw 20 } - }; + } } @method_id(110) @@ -113,7 +113,7 @@ fun test11(x: bool) { return match (x) { true => false, else => !x, - }; + } } diff --git a/tolk-tester/tests/no-spaces.tolk b/tolk-tester/tests/no-spaces.tolk index d63ca14110..3fe2573a94 100644 --- a/tolk-tester/tests/no-spaces.tolk +++ b/tolk-tester/tests/no-spaces.tolk @@ -1,13 +1,13 @@ const int10:int=10; fun just10(): int { return int10; } -fun eq(v: int): int { return`v`; } +fun eq(v: int): int { return`v` } -@method_id(101,) fun `get_-1` (): int {return-1;} -@method_id(102,) fun `get_--1` (): int {return--1;} -@method_id(103,) fun `get_---1`(): int {return---1;} -@method_id(104,) fun `get_+++1`(): int {return+++1;} -@method_id(105,) fun `get_+-+1`(): int {return+-+1;} +@method_id(101,) fun `get_-1` (): int {return-1} +@method_id(102,) fun `get_--1` (): int {return--1} +@method_id(103,) fun `get_---1`(): int {return---1} +@method_id(104,) fun `get_+++1`(): int {return+++1} +@method_id(105,) fun `get_+-+1`(): int {return+-+1} global `some()var`:int; @@ -26,7 +26,7 @@ global `some()var`:int; return[ (just10()-3==just10()-(4)--1)|((2==2)&(eq(eq(10)) -3==just10()--13)), ((flags&0xFF)!=0) - ]; + ] } @method_id(113)fun`unary+bitwise-constant`():[int,int,int]{ @@ -48,7 +48,7 @@ fun add3(a: int, b: int, c: int) { return a+b+c; } } fun `load:u32`(mutate self: slice): int { - return self.loadUint(32); + return self.loadUint(32) } @method_id(116) fun `call_~_via_backticks`():[int,int,int,int] { diff --git a/tolk-tester/tests/nullable-types.tolk b/tolk-tester/tests/nullable-types.tolk index f5ee8b550f..74e8f1b1be 100644 --- a/tolk-tester/tests/nullable-types.tolk +++ b/tolk-tester/tests/nullable-types.tolk @@ -3,8 +3,8 @@ type MIntN = int?; fun getNullable4(): MIntN { return 4; } fun getNullableIntNull(): int? asm "PUSHNULL"; -fun eqInt(x: int) { return x; } -fun eq(x: T) { return x; } +fun eqInt(x: int) { return x } +fun eq(x: T) { return x } fun unwrap(x: T?): T { return x!; } fun intOr0(x: int?): int { return null == x ? 0 : x!; } diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index ffedcb4413..62477c1355 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -47,8 +47,8 @@ fun test3(): int { return x; } if (random() > -1) { - if (y == null) { return -1; } - else { return y; } + if (y == null) { return -1 } + else { return y } } return 0; } @@ -344,8 +344,8 @@ fun test33(): int { fun test34() { var (x, y) = (getNullableInt(), getNullableInt()); - if (random()) { throw (x = 1, y = 2); } - else { throw (x = 3, y = (1, getNullableInt()!).1); } + if (random()) { throw (x = 1, y = 2) } + else { throw (x = 3, y = (1, getNullableInt()!).1) } return x + y; } diff --git a/tolk-tester/tests/unbalanced-ret.tolk b/tolk-tester/tests/unbalanced-ret.tolk index eca5dd2a58..05c7f46d98 100644 --- a/tolk-tester/tests/unbalanced-ret.tolk +++ b/tolk-tester/tests/unbalanced-ret.tolk @@ -4,10 +4,10 @@ fun main(x: int): (int, int) { x *= 2; y += 1; if (x == -10) { - return (111, 0); + return (111, 0) } } - return (x + 1, y); + return (x + 1, y) } @inline @@ -41,7 +41,7 @@ fun bar2(x: int, y: int): (int, int) { y = foo2(y); x *= 2; if (x == -10) { - return (111, y); + return (111, y) } } return (x + 1, y); diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index e451f13ddd..8cd97f9419 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -83,11 +83,11 @@ fun test2() { fun test3(case: int): int | slice | null { var a: int | slice | null = 4; if (case == 0) { - a = getSlice32(); + a = getSlice32() } else if (case == 1) { - a = 10; + a = 10 } else { - a = null; + a = null } return a; } @@ -99,8 +99,8 @@ fun test4() { (5 as int8) as int8 | int8, true as bool | bool, 5 as int | int | (int | slice), - (5 as int | int | (int | slice)) as int | slice | null - ); + (5 as int | int | (int | slice)) as int | slice | null, + ) } @method_id(105) @@ -119,19 +119,19 @@ fun test6(case: int): int | slice | builder | null { a = null as slice? | builder; __expect_type(a, "null | slice | builder"); __expect_type(a!, "slice | builder"); - return a; + return a } if (case == 2) { __expect_type(a = 2 as MInt, "MInt"); __expect_type(a, "int"); a = 2; __expect_type(a, "int"); - return a + 0; + return a + 0 } if (case == 3) { a = null as slice?; __expect_type(a, "slice?"); - return a; + return a } a = null as MNull; __expect_type(a, "null"); @@ -159,7 +159,7 @@ fun test8() { (2, 3) as Pair2Or3 | null, (2, 3) as null | Pair2Or3, null as Pair2Or3? - ); + ) } @method_id(109) @@ -177,7 +177,7 @@ fun test9() { fun test10() { var t = getSlice32() as (int, (int, int)?, (int?, int)?) | slice; if (10 > 3) { - t = (1, null, null) as (int, (int, int)?, (int?, int)?); + t = (1, null, null) as (int, (int, int)?, (int?, int)?) } return t; } @@ -222,9 +222,9 @@ fun test21() { __expect_type(a, "never"); } if (a is slice || a is int) { - __expect_type(a, "slice | int"); + __expect_type(a, "slice | int") } else { - __expect_type(a, "never"); + __expect_type(a, "never") } if (a is int || a is slice || a is builder) { __expect_type(a, "int | slice"); @@ -238,13 +238,13 @@ fun test21() { @method_id(122) fun test22(a: int, b: int | slice) { var result = 0; - if (a is int) { result += 1; } - if (a is int || a is slice) { result += 2; } - if (a is builder || a is slice) { result += 3; } - if (b is int) { result += 10; } - if ((b is int) | (b is slice)) { result += 11; } - if ((b is builder) | (b is int)) { result += 12; } - return result; + if (a is int) { result += 1 } + if (a is int || a is slice) { result += 2 } + if (a is builder || a is slice) { result += 3 } + if (b is int) { result += 10 } + if ((b is int) | (b is slice)) { result += 11 } + if ((b is builder) | (b is int)) { result += 12 } + return result } @method_id(123) @@ -268,7 +268,7 @@ fun test23(p2: bool) { return p; } -fun eqUnion2(v: T1 | T2): T2 | T1 { return v; } +fun eqUnion2(v: T1 | T2): T2 | T1 { return v } fun detectUnionSide(v: T1 | T2) { if (v is T1) { @@ -371,11 +371,13 @@ fun test31() { var t5: (int, int) | int | null = 5 as int?; var t6: (int, int) | int | null = null as (int, int)?; var t7: (int, int) | int | null = (1, 2) as (int, int)?; - __expect_type(t1, "null"); - __expect_type(t2, "int"); - __expect_type(t3, "(int, int)"); - __expect_type(t4, "int?"); - __expect_type(t6, "(int, int)?"); + { + __expect_type(t1, "null"); + __expect_type(t2, "int"); + __expect_type(t3, "(int, int)"); + __expect_type(t4, "int?"); + __expect_type(t6, "(int, int)?") + } return (t1, 777, t2, 777, t3, 777, t4, 777, t5, 777, t6, 777, t7); } @@ -469,9 +471,9 @@ fun test39() { @method_id(140) fun test40(x: int): SomeIntBits { - if (x < 10) { return x as int8; } - if (x < 20) { return x as int9; } - if (x < 30) { return x as int10; } + if (x < 10) { return x as int8 } + if (x < 20) { return x as int9 } + if (x < 30) { return x as int10 } return x as uint64; } @@ -524,7 +526,7 @@ fun checkSimple(a: int | slice | builder) { int => 1, slice => 2, builder => 3, - }; + } } @method_id(147) @@ -539,7 +541,7 @@ fun checkGeneric3(a: T1 | T2 | T3): (int, T1 | T2?) { T1 => (1, a), T2 => (2, match(a) { T2 => a }), T3 => (3, null), - }; + } } type IntOrInt8OrInt16 = int | int8 | int16; @@ -561,9 +563,9 @@ fun test49(a: slice? | int) { return match (a) { slice => 1, null => 2, - }; + } } - return a + 0; + return a + 0 } type VeryComplexType = int | (MInt, int)? | builder? | int; @@ -592,7 +594,7 @@ fun test50() { __expect_type(b, "int?"); b is int && b!is int && b!!is int; if (match(b) { int => 1, null => null } !is int) { - return 100; + return 100 } return (b! + (match (a) { int => a, slice => a, null => a } as int|slice?).test49()); } @@ -610,7 +612,7 @@ fun test51(init: int) { a = a > 10 ? beginCell().storeInt(init, 32).endCell().beginParse() : 10; __expect_type(a, "int | slice"); if (a is slice) { - a = a.loadInt(32); + a = a.loadInt(32) } } slice => { @@ -632,7 +634,7 @@ fun test52(init: int): int | builder { return match (init as int?) { int => init, null => beginCell() - }; + } } fun inc53(mutate x: int) { @@ -659,7 +661,7 @@ fun test53(a: int | builder) { match (var (a, (b, c)) = (10, (20, 30))) { (int, Pair2) => {} } - return cc; + return cc } @method_id(154) diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index 11241bf13b..3411f4a066 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -4,11 +4,11 @@ type MBuilder = builder; type CallbackIntToInt = int -> int; fun getBeginCell() { - return beginCell; + return beginCell } fun getBeginParse() { - return beginParse; + return beginParse } @method_id(101) @@ -76,25 +76,25 @@ fun testVarApplyInTernary() { t.tuplePush(code); } try { - t.tuplePush(demo_handler(0xF4, 122, 100, 200)); + t.tuplePush(demo_handler(0xF4, 122, 100, 200)) } catch(code) { - t.tuplePush(code); + t.tuplePush(code) } try { - t.tuplePush(demo_handler(0xF2, 122, 10, 10)); + t.tuplePush(demo_handler(0xF2, 122, 10, 10)) } catch(code) { - t.tuplePush(code); + t.tuplePush(code) } try { - t.tuplePush(demo_handler(0xF2, 123, 10, 10)); + t.tuplePush(demo_handler(0xF2, 123, 10, 10)) } catch(code) { - t.tuplePush(code); + t.tuplePush(code) } return t; } fun always_throw2(x: int) { - throw 239 + x; + throw 239 + x } global global_f: MInt -> void; @@ -143,7 +143,7 @@ fun testIndexedAccessApply() { return functions2.0(functions1.1(b)).loadInt(32); } -fun getNullable4(): int? { return 4; } +fun getNullable4(): int? { return 4 } fun myBeginCell(): builder? asm "NEWC"; @method_id(108) @@ -211,15 +211,15 @@ fun testCallbackAlias(secondNull: bool) { return (adder1(10), adder2 == null ? -1 : adder2(10)); } -fun justGetTensor2() { return (2, 3); } +fun justGetTensor2() { return (2, 3) } @method_id(114) fun testCallbackUnion(call1st: bool): (int, int) | int { var cb: (int -> int) | (() -> (MInt, int)) = call1st ? justGetTensor2 : justAdd2; if (cb is () -> (int, int)) { - return cb(); + return cb() } - return cb(10) as int; + return cb(10) as int } fun main() {} diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 36e6ce5a2f..ec9d689d47 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -14,7 +14,6 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "ast-from-tokens.h" #include "ast.h" #include "type-system.h" #include "platform-utils.h" @@ -296,7 +295,7 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { lex.unexpected("variable name"); } -static AnyExprV parse_local_vars_declaration_assignment(Lexer& lex, bool is_statement) { +static AnyExprV parse_local_vars_declaration_assignment(Lexer& lex) { SrcLocation loc = lex.cur_location(); bool is_immutable = lex.tok() == tok_val; lex.next(); @@ -311,9 +310,6 @@ static AnyExprV parse_local_vars_declaration_assignment(Lexer& lex, bool is_stat if (lex.tok() == tok_comma) { lex.error("multiple declarations are not allowed, split variables on separate lines"); } - if (is_statement) { - lex.expect(tok_semicolon, "`;`"); - } return createV(loc, lhs, rhs); } @@ -392,7 +388,7 @@ static V parse_maybe_instantiationTs_after_identifier(L // `throw code` / `throw (code)` / `throw (code, arg)` // it's technically a statement (can't occur "in any place of expression"), // but inside `match` arm it can appear without semicolon: `pattern => throw 123` -static AnyV parse_throw_expression(Lexer& lex, bool is_statement) { +static AnyV parse_throw_expression(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.expect(tok_throw, "`throw`"); @@ -412,9 +408,6 @@ static AnyV parse_throw_expression(Lexer& lex, bool is_statement) { thrown_arg = createV(loc); } - if (is_statement) { - lex.expect(tok_semicolon, "`;`"); - } return createV(loc, thrown_code, thrown_arg); } @@ -453,17 +446,17 @@ static V parse_match_arm(Lexer& lex) { AnyExprV body; if (lex.tok() == tok_opbrace) { // pattern => { ... } - AnyV v_seq = parse_statement(lex); - body = createV(v_seq->loc, v_seq); + AnyV v_block = parse_statement(lex); + body = createV(v_block->loc, v_block); } else if (lex.tok() == tok_throw) { // pattern => throw 123 (allow without braces) - AnyV v_throw = parse_throw_expression(lex, false); - AnyV v_seq = createV(v_throw->loc, v_throw->loc, {v_throw}); - body = createV(v_seq->loc, v_seq); + AnyV v_throw = parse_throw_expression(lex); + AnyV v_block = createV(v_throw->loc, v_throw->loc, {v_throw}); + body = createV(v_block->loc, v_block); } else if (lex.tok() == tok_return) { // pattern => return 123 (allow without braces, like throw) lex.next(); AnyV v_return = createV(lex.cur_location(), parse_expr(lex)); - AnyV v_seq = createV(v_return->loc, v_return->loc, {v_return}); - body = createV(v_seq->loc, v_seq); + AnyV v_block = createV(v_return->loc, v_return->loc, {v_return}); + body = createV(v_block->loc, v_block); } else { body = parse_expr(lex); } @@ -480,7 +473,7 @@ static V parse_match_expression(Lexer& lex) { lex.expect(tok_oppar, "`(`"); AnyExprV subject = lex.tok() == tok_var || lex.tok() == tok_val // `match (var x = rhs)` - ? parse_local_vars_declaration_assignment(lex, false) + ? parse_local_vars_declaration_assignment(lex) : parse_expr(lex); lex.expect(tok_clpar, "`)`"); @@ -819,16 +812,24 @@ AnyExprV parse_expr(Lexer& lex) { return parse_expr10(lex); } -static V parse_sequence(Lexer& lex) { +static V parse_block_statement(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.expect(tok_opbrace, "`{`"); std::vector items; while (lex.tok() != tok_clbrace) { - items.push_back(parse_statement(lex)); + AnyV v = parse_statement(lex); + items.push_back(v); + if (lex.tok() == tok_clbrace) { + break; + } + bool does_end_with_brace = v->type == ast_if_statement || v->type == ast_while_statement || v->type == ast_match_expression || v->type == ast_try_catch_statement || v->type == ast_repeat_statement || v->type == ast_block_statement; + if (!does_end_with_brace) { + lex.expect(tok_semicolon, "`;`"); + } } SrcLocation loc_end = lex.cur_location(); lex.expect(tok_clbrace, "`}`"); - return createV(loc, loc_end, std::move(items)); + return createV(loc, loc_end, std::move(items)); } static AnyV parse_return_statement(Lexer& lex) { @@ -837,7 +838,6 @@ static AnyV parse_return_statement(Lexer& lex) { AnyExprV child = lex.tok() == tok_semicolon // `return;` actually means "nothing" (inferred as void) ? createV(lex.cur_location()) : parse_expr(lex); - lex.expect(tok_semicolon, "`;`"); return createV(loc, child); } @@ -849,18 +849,18 @@ static AnyV parse_if_statement(Lexer& lex) { AnyExprV cond = parse_expr(lex); lex.expect(tok_clpar, "`)`"); - V if_body = parse_sequence(lex); - V else_body = nullptr; + V if_body = parse_block_statement(lex); + V else_body = nullptr; if (lex.tok() == tok_else) { // else if(e) { } or else { } lex.next(); if (lex.tok() == tok_if) { AnyV v_inner_if = parse_if_statement(lex); - else_body = createV(v_inner_if->loc, lex.cur_location(), {v_inner_if}); + else_body = createV(v_inner_if->loc, lex.cur_location(), {v_inner_if}); } else { - else_body = parse_sequence(lex); + else_body = parse_block_statement(lex); } } else { // no 'else', create empty block - else_body = createV(lex.cur_location(), lex.cur_location(), {}); + else_body = createV(lex.cur_location(), lex.cur_location(), {}); } return createV(loc, false, cond, if_body, else_body); } @@ -871,7 +871,7 @@ static AnyV parse_repeat_statement(Lexer& lex) { lex.expect(tok_oppar, "`(`"); AnyExprV cond = parse_expr(lex); lex.expect(tok_clpar, "`)`"); - V body = parse_sequence(lex); + V body = parse_block_statement(lex); return createV(loc, cond, body); } @@ -881,19 +881,18 @@ static AnyV parse_while_statement(Lexer& lex) { lex.expect(tok_oppar, "`(`"); AnyExprV cond = parse_expr(lex); lex.expect(tok_clpar, "`)`"); - V body = parse_sequence(lex); + V body = parse_block_statement(lex); return createV(loc, cond, body); } static AnyV parse_do_while_statement(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.expect(tok_do, "`do`"); - V body = parse_sequence(lex); + V body = parse_block_statement(lex); lex.expect(tok_while, "`while`"); lex.expect(tok_oppar, "`(`"); AnyExprV cond = parse_expr(lex); lex.expect(tok_clpar, "`)`"); - lex.expect(tok_semicolon, "`;`"); return createV(loc, body, cond); } @@ -935,14 +934,13 @@ static AnyV parse_assert_statement(Lexer& lex) { thrown_code = parse_expr(lex); } - lex.expect(tok_semicolon, "`;`"); return createV(loc, cond, thrown_code); } static AnyV parse_try_catch_statement(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.expect(tok_try, "`try`"); - V try_body = parse_sequence(lex); + V try_body = parse_block_statement(lex); std::vector catch_args; lex.expect(tok_catch, "`catch`"); @@ -963,7 +961,7 @@ static AnyV parse_try_catch_statement(Lexer& lex) { } V catch_expr = createV(catch_loc, std::move(catch_args)); - V catch_body = parse_sequence(lex); + V catch_body = parse_block_statement(lex); return createV(loc, try_body, catch_expr, catch_body); } @@ -971,9 +969,9 @@ AnyV parse_statement(Lexer& lex) { switch (lex.tok()) { case tok_var: // `var x = 0` is technically an expression, but can not appear in "any place", case tok_val: // only as a separate declaration - return parse_local_vars_declaration_assignment(lex, true); + return parse_local_vars_declaration_assignment(lex); case tok_opbrace: - return parse_sequence(lex); + return parse_block_statement(lex); case tok_return: return parse_return_statement(lex); case tok_if: @@ -985,31 +983,23 @@ AnyV parse_statement(Lexer& lex) { case tok_while: return parse_while_statement(lex); case tok_throw: - return parse_throw_expression(lex, true); + return parse_throw_expression(lex); case tok_assert: return parse_assert_statement(lex); case tok_try: return parse_try_catch_statement(lex); - case tok_semicolon: { - SrcLocation loc = lex.cur_location(); - lex.next(); - return createV(loc); - } + case tok_semicolon: + return createV(lex.cur_location()); case tok_break: case tok_continue: lex.error("break/continue from loops are not supported yet"); - default: { - AnyExprV expr = parse_expr(lex); - if (expr->type != ast_match_expression) { // match statement, as if statement, doesn't require semicolon after - lex.expect(tok_semicolon, "`;`"); - } - return expr; - } + default: + return parse_expr(lex); } } static AnyV parse_func_body(Lexer& lex) { - return parse_sequence(lex); + return parse_block_statement(lex); } static AnyV parse_asm_func_body(Lexer& lex, V param_list) { diff --git a/tolk/ast-from-tokens.h b/tolk/ast-from-tokens.h deleted file mode 100644 index 39574f9c2b..0000000000 --- a/tolk/ast-from-tokens.h +++ /dev/null @@ -1,25 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . -*/ -#pragma once - -#include "fwd-declarations.h" - -namespace tolk { - -AnyV parse_src_file_to_ast(const SrcFile* file); - -} // namespace tolk diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 5efae1e2f4..417ff411c2 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -62,7 +62,7 @@ class ASTReplacer { GNU_ATTRIBUTE_ALWAYS_INLINE AnyExprV replace_children(const ASTExprBlockOfStatements* v) { auto* v_mutable = const_cast(v); - v_mutable->child_sequence = replace(v_mutable->child_sequence->as()); + v_mutable->child_block_statement = replace(v_mutable->child_block_statement->as()); return v_mutable; } @@ -121,7 +121,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { virtual AnyExprV replace(V v) { return replace_children(v); } // statements virtual AnyV replace(V v) { return replace_children(v); } - virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } virtual AnyV replace(V v) { return replace_children(v); } virtual AnyV replace(V v) { return replace_children(v); } virtual AnyV replace(V v) { return replace_children(v); } @@ -168,7 +168,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { AnyV replace(AnyV v) final { switch (v->type) { case ast_empty_statement: return replace(v->as()); - case ast_sequence: return replace(v->as()); + case ast_block_statement: return replace(v->as()); case ast_return_statement: return replace(v->as()); case ast_if_statement: return replace(v->as()); case ast_repeat_statement: return replace(v->as()); diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 9f0738d3f6..66781f1f0d 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -62,7 +62,7 @@ class ASTReplicatorFunction : public ASTReplicator { return createV(v->loc, clone(v->get_expr())); } virtual V clone(V v) { - return createV(v->loc, clone(v->get_sequence())); + return createV(v->loc, clone(v->get_block_statement())); } virtual V clone(V v) { return createV(v->loc, clone(v->get_items())); @@ -142,8 +142,8 @@ class ASTReplicatorFunction : public ASTReplicator { virtual V clone(V v) { return createV(v->loc); } - virtual V clone(V v) { - return createV(v->loc, v->loc_end, clone(v->get_items())); + virtual V clone(V v) { + return createV(v->loc, v->loc_end, clone(v->get_items())); } virtual V clone(V v) { return createV(v->loc, clone(v->get_return_value())); @@ -228,7 +228,7 @@ class ASTReplicatorFunction : public ASTReplicator { AnyV clone(AnyV v) final { switch (v->type) { case ast_empty_statement: return clone(v->as()); - case ast_sequence: return clone(v->as()); + case ast_block_statement: return clone(v->as()); case ast_return_statement: return clone(v->as()); case ast_if_statement: return clone(v->as()); case ast_repeat_statement: return clone(v->as()); @@ -263,7 +263,7 @@ class ASTReplicatorFunction : public ASTReplicator { v_function->loc, clone(v_function->get_identifier()), clone(v_function->get_param_list()), - clone(v_function->get_body()->as()), + clone(v_function->get_body()->as()), clone(v_function->declared_return_type), v_function->genericsT_list, v_function->method_id, diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index 33082a6e9b..a232888f28 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -63,7 +63,7 @@ class ASTStringifier final : public ASTVisitor { {ast_match_arm, "ast_match_arm"}, // statements {ast_empty_statement, "ast_empty_statement"}, - {ast_sequence, "ast_sequence"}, + {ast_block_statement, "ast_block_statement"}, {ast_return_statement, "ast_return_statement"}, {ast_if_statement, "ast_if_statement"}, {ast_repeat_statement, "ast_repeat_statement"}, @@ -177,8 +177,8 @@ class ASTStringifier final : public ASTVisitor { std::string prefix = v->as()->is_negated ? "!is " : "is "; return prefix + v->as()->rhs_type->as_human_readable(); } - case ast_sequence: - return "↓" + std::to_string(v->as()->get_items().size()); + case ast_block_statement: + return "↓" + std::to_string(v->as()->get_items().size()); case ast_instantiationT_item: return v->as()->substituted_type->as_human_readable(); case ast_if_statement: @@ -291,7 +291,7 @@ class ASTStringifier final : public ASTVisitor { case ast_match_arm: return handle_vertex(v->as()); // statements case ast_empty_statement: return handle_vertex(v->as()); - case ast_sequence: return handle_vertex(v->as()); + case ast_block_statement: return handle_vertex(v->as()); case ast_return_statement: return handle_vertex(v->as()); case ast_if_statement: return handle_vertex(v->as()); case ast_repeat_statement: return handle_vertex(v->as()); diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index 2f5f269f78..eac7e98ef7 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -57,7 +57,7 @@ class ASTVisitor { } GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTExprBlockOfStatements* v) { - visit_children(v->child_sequence->as()); + visit_children(v->child_block_statement->as()); } GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTStatementUnary* v) { @@ -120,7 +120,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { virtual void visit(V v) { return visit_children(v); } // statements virtual void visit(V v) { return visit_children(v); } - virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } @@ -162,7 +162,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { case ast_match_arm: return visit(v->as()); // statements case ast_empty_statement: return visit(v->as()); - case ast_sequence: return visit(v->as()); + case ast_block_statement: return visit(v->as()); case ast_return_statement: return visit(v->as()); case ast_if_statement: return visit(v->as()); case ast_repeat_statement: return visit(v->as()); diff --git a/tolk/ast.cpp b/tolk/ast.cpp index d740970da9..9fadd5f6c7 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -200,7 +200,7 @@ void Vertex::assign_fun_ref(FunctionPtr fun_ref) { this->fun_ref = fun_ref; } -void Vertex::assign_first_unreachable(AnyV first_unreachable) { +void Vertex::assign_first_unreachable(AnyV first_unreachable) { this->first_unreachable = first_unreachable; } diff --git a/tolk/ast.h b/tolk/ast.h index fea11ebcb9..9dce458ecd 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -95,7 +95,7 @@ enum ASTNodeType { ast_match_arm, // statements ast_empty_statement, - ast_sequence, + ast_block_statement, ast_return_statement, ast_if_statement, ast_repeat_statement, @@ -266,10 +266,10 @@ struct ASTExprBlockOfStatements : ASTNodeExpressionBase { friend class ASTReplacer; protected: - AnyV child_sequence; + AnyV child_block_statement; - ASTExprBlockOfStatements(ASTNodeType type, SrcLocation loc, AnyV child_sequence) - : ASTNodeExpressionBase(type, loc), child_sequence(child_sequence) {} + ASTExprBlockOfStatements(ASTNodeType type, SrcLocation loc, AnyV child_block_statement) + : ASTNodeExpressionBase(type, loc), child_block_statement(child_block_statement) {} }; struct ASTStatementUnary : ASTNodeStatementBase { @@ -377,10 +377,10 @@ template<> // it can occur only in special places within the input code, not anywhere // example: `match (intV) { 0 => { ... } }` rhs of 0 is braced expression struct Vertex final : ASTExprBlockOfStatements { - auto get_sequence() const { return child_sequence->as(); } + auto get_block_statement() const { return child_block_statement->as(); } - Vertex(SrcLocation loc, AnyV child_sequence) - : ASTExprBlockOfStatements(ast_braced_expression, loc, child_sequence) {} + Vertex(SrcLocation loc, AnyV child_block_statement) + : ASTExprBlockOfStatements(ast_braced_expression, loc, child_block_statement) {} }; template<> @@ -797,10 +797,10 @@ struct Vertex final : ASTStatementVararg { }; template<> -// ast_sequence is "some sequence of statements" -// example: function body is a sequence -// example: do while body is a sequence -struct Vertex final : ASTStatementVararg { +// ast_block_statement is "{ statement; statement }" (trailing semicolon is optional) +// example: function body is a block +// example: do while body is a block +struct Vertex final : ASTStatementVararg { SrcLocation loc_end; AnyV first_unreachable = nullptr; @@ -811,7 +811,7 @@ struct Vertex final : ASTStatementVararg { void assign_first_unreachable(AnyV first_unreachable); Vertex(SrcLocation loc, SrcLocation loc_end, std::vector&& items) - : ASTStatementVararg(ast_sequence, loc, std::move(items)) + : ASTStatementVararg(ast_block_statement, loc, std::move(items)) , loc_end(loc_end) {} }; @@ -836,10 +836,10 @@ struct Vertex final : ASTStatementVararg { bool is_ifnot; // if(!cond), to generate more optimal fift code AnyExprV get_cond() const { return child_as_expr(0); } - auto get_if_body() const { return children.at(1)->as(); } - auto get_else_body() const { return children.at(2)->as(); } // always exists (when else omitted, it's empty) + auto get_if_body() const { return children.at(1)->as(); } + auto get_else_body() const { return children.at(2)->as(); } // always exists (when else omitted, it's empty) - Vertex(SrcLocation loc, bool is_ifnot, AnyExprV cond, V if_body, V else_body) + Vertex(SrcLocation loc, bool is_ifnot, AnyExprV cond, V if_body, V else_body) : ASTStatementVararg(ast_if_statement, loc, {cond, if_body, else_body}) , is_ifnot(is_ifnot) {} }; @@ -849,9 +849,9 @@ template<> // example: `repeat (10) { ... }` struct Vertex final : ASTStatementVararg { AnyExprV get_cond() const { return child_as_expr(0); } - auto get_body() const { return children.at(1)->as(); } + auto get_body() const { return children.at(1)->as(); } - Vertex(SrcLocation loc, AnyExprV cond, V body) + Vertex(SrcLocation loc, AnyExprV cond, V body) : ASTStatementVararg(ast_repeat_statement, loc, {cond, body}) {} }; @@ -860,9 +860,9 @@ template<> // example: `while (x > 0) { ... }` struct Vertex final : ASTStatementVararg { AnyExprV get_cond() const { return child_as_expr(0); } - auto get_body() const { return children.at(1)->as(); } + auto get_body() const { return children.at(1)->as(); } - Vertex(SrcLocation loc, AnyExprV cond, V body) + Vertex(SrcLocation loc, AnyExprV cond, V body) : ASTStatementVararg(ast_while_statement, loc, {cond, body}) {} }; @@ -870,10 +870,10 @@ template<> // ast_do_while_statement is a standard "do while" loop // example: `do { ... } while (x > 0);` struct Vertex final : ASTStatementVararg { - auto get_body() const { return children.at(0)->as(); } + auto get_body() const { return children.at(0)->as(); } AnyExprV get_cond() const { return child_as_expr(1); } - Vertex(SrcLocation loc, V body, AnyExprV cond) + Vertex(SrcLocation loc, V body, AnyExprV cond) : ASTStatementVararg(ast_do_while_statement, loc, {body, cond}) {} }; @@ -907,11 +907,11 @@ template<> // there are two formal "arguments" of catch: excNo and arg, but both can be omitted // when omitted, they are stored as underscores, so len of a catch tensor is always 2 struct Vertex final : ASTStatementVararg { - auto get_try_body() const { return children.at(0)->as(); } + auto get_try_body() const { return children.at(0)->as(); } auto get_catch_expr() const { return children.at(1)->as(); } // (excNo, arg), always len 2 - auto get_catch_body() const { return children.at(2)->as(); } + auto get_catch_body() const { return children.at(2)->as(); } - Vertex(SrcLocation loc, V try_body, V catch_expr, V catch_body) + Vertex(SrcLocation loc, V try_body, V catch_expr, V catch_body) : ASTStatementVararg(ast_try_catch_statement, loc, {try_body, catch_expr, catch_body}) {} }; @@ -1049,7 +1049,7 @@ struct Vertex final : ASTOtherVararg { int get_num_params() const { return children.at(1)->as()->size(); } auto get_param_list() const { return children.at(1)->as(); } auto get_param(int i) const { return children.at(1)->as()->get_param(i); } - AnyV get_body() const { return children.at(2); } // ast_sequence / ast_asm_body + AnyV get_body() const { return children.at(2); } // ast_block_statement / ast_asm_body FunctionPtr fun_ref = nullptr; // filled after register TypePtr declared_return_type; // filled at ast parsing; if unspecified (nullptr), means "auto infer" @@ -1058,7 +1058,7 @@ struct Vertex final : ASTOtherVararg { int flags; // from enum in FunctionData bool is_asm_function() const { return children.at(2)->type == ast_asm_body; } - bool is_code_function() const { return children.at(2)->type == ast_sequence; } + bool is_code_function() const { return children.at(2)->type == ast_block_statement; } bool is_builtin_function() const { return children.at(2)->type == ast_empty_statement; } Vertex* mutate() const { return const_cast(this); } diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index bba79ea923..3778b8cab6 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1259,7 +1259,7 @@ static std::vector process_braced_expression(V // `{ ... }` used as an expression can not return a value currently (there is no syntax in a language) // that's why it can appear only in special places, and its usage correctness has been checked tolk_assert(v->inferred_type == TypeDataVoid::create() || v->inferred_type == TypeDataNever::create()); - process_any_statement(v->get_sequence(), code); + process_any_statement(v->get_block_statement(), code); static_cast(target_type); return {}; } @@ -1392,7 +1392,7 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ } -static void process_sequence(V v, CodeBlob& code) { +static void process_block_statement(V v, CodeBlob& code) { for (AnyV item : v->get_items()) { process_any_statement(item, code); } @@ -1582,8 +1582,8 @@ static void append_implicit_return_statement(SrcLocation loc_end, CodeBlob& code void process_any_statement(AnyV v, CodeBlob& code) { switch (v->type) { - case ast_sequence: - return process_sequence(v->as(), code); + case ast_block_statement: + return process_block_statement(v->as(), code); case ast_return_statement: return process_return_statement(v->as(), code); case ast_repeat_statement: @@ -1608,7 +1608,7 @@ void process_any_statement(AnyV v, CodeBlob& code) { } static void convert_function_body_to_CodeBlob(FunctionPtr fun_ref, FunctionBodyCode* code_body) { - auto v_body = fun_ref->ast_root->as()->get_body()->as(); + auto v_body = fun_ref->ast_root->as()->get_body()->as(); CodeBlob* blob = new CodeBlob{fun_ref->name, fun_ref->loc, fun_ref}; std::vector rvect_import; diff --git a/tolk/pipe-calc-rvalue-lvalue.cpp b/tolk/pipe-calc-rvalue-lvalue.cpp index 769d31c010..b910eface6 100644 --- a/tolk/pipe-calc-rvalue-lvalue.cpp +++ b/tolk/pipe-calc-rvalue-lvalue.cpp @@ -238,7 +238,7 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { parent::visit(v); } - void visit(V v) override { + void visit(V v) override { MarkingState saved = enter_state(MarkingState::None); parent::visit(v); restore_state(saved); diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index b9bcdaced8..4b0f8d53f5 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -691,7 +691,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } - void visit(V v) override { + void visit(V v) override { parent::visit(v); if (v->first_unreachable) { @@ -715,7 +715,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (fun_ref->is_implicit_return() && fun_ref->declared_return_type) { if (!fun_ref->declared_return_type->can_rhs_be_assigned(TypeDataVoid::create()) || fun_ref->does_return_self()) { - fire(fun_ref, v_function->get_body()->as()->loc_end, "missing return"); + fire(fun_ref, v_function->get_body()->as()->loc_end, "missing return"); } } } diff --git a/tolk/pipe-discover-parse-sources.cpp b/tolk/pipe-discover-parse-sources.cpp index d31348ba91..2f7c02d6d2 100644 --- a/tolk/pipe-discover-parse-sources.cpp +++ b/tolk/pipe-discover-parse-sources.cpp @@ -25,7 +25,6 @@ */ #include "tolk.h" #include "ast.h" -#include "ast-from-tokens.h" #include "compiler-state.h" /* @@ -38,6 +37,8 @@ namespace tolk { +AnyV parse_src_file_to_ast(const SrcFile* file); + void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, const std::string& entrypoint_filename) { G.all_src_files.locate_and_register_source_file(stdlib_filename, {}); G.all_src_files.locate_and_register_source_file(entrypoint_filename, {}); @@ -64,7 +65,7 @@ void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, con } // todo #ifdef TOLK_PROFILING - lexer_measure_performance(G.all_src_files); + // lexer_measure_performance(G.all_src_files); } } // namespace tolk diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index c0c7c1d39d..0e1c5af983 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -186,8 +186,8 @@ class InferTypesAndCallsAndFieldsVisitor final { // traverse children in any statement FlowContext process_any_statement(AnyV v, FlowContext&& flow) { switch (v->type) { - case ast_sequence: - return process_sequence(v->as(), std::move(flow)); + case ast_block_statement: + return process_block_statement(v->as(), std::move(flow)); case ast_return_statement: return process_return_statement(v->as(), std::move(flow)); case ast_if_statement: @@ -680,7 +680,7 @@ class InferTypesAndCallsAndFieldsVisitor final { ExprFlow infer_braced_expression(V v, FlowContext&& flow, bool used_as_condition) { // `{ ... }` used as an expression can not return a value currently (there is no syntax in a language) - flow = process_any_statement(v->get_sequence(), std::move(flow)); + flow = process_any_statement(v->get_block_statement(), std::move(flow)); assign_inferred_type(v, flow.is_unreachable() ? TypeDataNever::create() : TypeDataVoid::create()); return ExprFlow(std::move(flow), used_as_condition); } @@ -1125,7 +1125,7 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } - FlowContext process_sequence(V v, FlowContext&& flow) { + FlowContext process_block_statement(V v, FlowContext&& flow) { // we'll print a warning if after some statement, control flow became unreachable // (but don't print a warning if it's already unreachable, for example we're inside always-false if) bool initially_unreachable = flow.is_unreachable(); @@ -1274,7 +1274,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (!body_end.is_unreachable()) { fun_ref->mutate()->assign_is_implicit_return(); if (fun_ref->declared_return_type == TypeDataNever::create()) { // `never` can only be declared, it can't be inferred - fire(fun_ref, v_function->get_body()->as()->loc_end, "a function returning `never` can not have a reachable endpoint"); + fire(fun_ref, v_function->get_body()->as()->loc_end, "a function returning `never` can not have a reachable endpoint"); } } @@ -1297,7 +1297,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } if (!body_end.is_unreachable() && inferred_return_type != TypeDataVoid::create()) { - fire(fun_ref, v_function->get_body()->as()->loc_end, "missing return"); + fire(fun_ref, v_function->get_body()->as()->loc_end, "missing return"); } if (has_void_returns && has_non_void_returns) { for (AnyExprV return_expr : return_statements) { diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index e76f5b4796..363952cbdc 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -190,7 +190,7 @@ static void register_function(V v) { std::cerr << "fun " << func_name << " : " << v->declared_return_type << std::endl; } - FunctionBody f_body = v->get_body()->type == ast_sequence ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); + FunctionBody f_body = v->get_body()->type == ast_block_statement ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); FunctionData* f_sym = new FunctionData(static_cast(func_name), v->loc, v->declared_return_type, std::move(parameters), 0, genericTs, nullptr, f_body, v); if (const auto* v_asm = v->get_body()->try_as()) { diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 0b116bbe80..797795afd9 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -262,7 +262,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { current_scope.open_scope(v->loc); - parent::visit(v->get_sequence()); + parent::visit(v->get_block_statement()); current_scope.close_scope(v->loc); } @@ -326,7 +326,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { parent::visit(v->get_expr()); } - void visit(V v) override { + void visit(V v) override { if (v->empty()) { return; } @@ -377,13 +377,13 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { fun_ref->mutate()->assign_resolved_type(return_type); if (fun_ref->is_code_function()) { - auto v_seq = v->get_body()->as(); + auto v_block = v->get_body()->as(); current_scope.open_scope(v->loc); for (int i = 0; i < v->get_num_params(); ++i) { current_scope.add_local_var(&fun_ref->parameters[i]); } - parent::visit(v_seq); - current_scope.close_scope(v_seq->loc_end); + parent::visit(v_block); + current_scope.close_scope(v_block->loc_end); tolk_assert(current_scope.scopes.empty()); } From cadb66e9a3d403b56d9a5db36e87cbbc4f9ae49c Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sat, 5 Apr 2025 17:20:17 +0300 Subject: [PATCH 204/524] [Tolk] Bump version to v0.11 --- crypto/smartcont/tolk-stdlib/common.tolk | 2 +- crypto/smartcont/tolk-stdlib/gas-payments.tolk | 2 +- crypto/smartcont/tolk-stdlib/lisp-lists.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk | 2 +- tolk/tolk-version.h | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index dc0ec4a61f..6b737ef3ee 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -1,7 +1,7 @@ // Standard library for Tolk (LGPL licence). // It contains common functions that are available out of the box, the user doesn't have to import anything. // More specific functions are required to be imported explicitly, like "@stdlib/tvm-dicts". -tolk 0.10 +tolk 0.11 /// In Tolk v1.x there would be a type `map`. /// Currently, working with dictionaries is still low-level, with raw cells. diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 11e3db7ff1..0e8305e1fd 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.10 +tolk 0.11 /** Gas and payment related primitives. diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk index fd6d2c8943..a08b2c11c8 100644 --- a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.10 +tolk 0.11 /** Lisp-style lists are nested 2-elements tuples: `(1, (2, (3, null)))` represents list `[1, 2, 3]`. diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index 83fe705c49..00ae594c6d 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.10 +tolk 0.11 /** Dictionaries are represented as `cell` data type (cells can store anything, dicts in particular). diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index 48bacec6c4..a7f1582485 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.10 +tolk 0.11 /// Usually `c3` has a continuation initialized by the whole code of the contract. It is used for function calls. /// The primitive returns the current value of `c3`. diff --git a/tolk/tolk-version.h b/tolk/tolk-version.h index be161a393f..b8f2893eb2 100644 --- a/tolk/tolk-version.h +++ b/tolk/tolk-version.h @@ -18,6 +18,6 @@ namespace tolk { -constexpr const char* TOLK_VERSION = "0.10.0"; +constexpr const char* TOLK_VERSION = "0.11.0"; } // namespace tolk From 7b0881292472e9c8957f23dfd3636b0589c9f6bc Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 7 Apr 2025 11:22:19 +0300 Subject: [PATCH 205/524] Don't store storage dict hash in masterchain --- crypto/block/transaction.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index f24a036764..417ce0cf46 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -3273,8 +3273,9 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } } + bool store_storage_dict_hash = cfg.store_storage_dict_hash && !account.is_masterchain(); if (storage_refs_changed || - (cfg.store_storage_dict_hash && !account.storage_dict_hash && account.storage_used.cells > 25)) { + (store_storage_dict_hash && !account.storage_dict_hash && account.storage_used.cells > 25)) { TD_PERF_COUNTER(transaction_storage_stat_b); td::Timer timer; if (!new_account_storage_stat && account.account_storage_stat) { @@ -3291,7 +3292,7 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { new_storage_used.cells = stats.get_total_cells() + 1; new_storage_used.bits = stats.get_total_bits() + new_storage_for_stat->size(); // TODO: think about this limit (25) - if (cfg.store_storage_dict_hash && new_storage_used.cells > 25) { + if (store_storage_dict_hash && new_storage_used.cells > 25) { auto r_hash = stats.get_dict_hash(); if (r_hash.is_error()) { LOG(ERROR) << "Cannot compute storage dict hash for account " << account.addr.to_hex() << ": " @@ -3308,7 +3309,7 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { new_storage_used.bits -= old_storage_for_stat->size(); new_storage_used.bits += new_storage_for_stat->size(); new_account_storage_stat = {}; - if (cfg.store_storage_dict_hash) { + if (store_storage_dict_hash) { new_storage_dict_hash = account.storage_dict_hash; } } From 73751cdda6e69453536cf2690794e069c1384c29 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 8 Apr 2025 08:55:45 +0300 Subject: [PATCH 206/524] Fix compilation error and test-db (#1609) * Fix compilation error * Fix error processing in load_cell * Decrease verbosity of certain ADNL logs --- adnl/adnl-ext-server.cpp | 2 +- adnl/adnl-peer.cpp | 4 ++-- crypto/vm/db/DynamicBagOfCellsDb.cpp | 17 ++++++++++++----- tdnet/td/net/TcpListener.cpp | 2 +- validator/db/celldb.cpp | 2 +- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/adnl/adnl-ext-server.cpp b/adnl/adnl-ext-server.cpp index 162a53afb9..627a25cb51 100644 --- a/adnl/adnl-ext-server.cpp +++ b/adnl/adnl-ext-server.cpp @@ -31,7 +31,7 @@ td::Status AdnlInboundConnection::process_packet(td::BufferSlice data) { td::PromiseCreator::lambda([SelfId = actor_id(this), query_id = f->query_id_](td::Result R) { if (R.is_error()) { auto S = R.move_as_error(); - LOG(WARNING) << "failed ext query: " << S; + LOG(INFO) << "failed ext query: " << S; } else { auto B = create_tl_object(query_id, R.move_as_ok()); td::actor::send_closure(SelfId, &AdnlInboundConnection::send, serialize_tl_object(B, true)); diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index ab46005819..88a38f7b04 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -598,11 +598,11 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessagePart &messa respond_with_nop(); auto size = message.total_size(); if (size > huge_packet_max_size()) { - VLOG(ADNL_WARNING) << this << ": dropping too big huge message: size=" << size; + VLOG(ADNL_INFO) << this << ": dropping too big huge message: size=" << size; return; } if (message.hash().is_zero()) { - VLOG(ADNL_WARNING) << this << ": dropping huge message with zero hash"; + VLOG(ADNL_INFO) << this << ": dropping huge message with zero hash"; return; } if (message.hash() != huge_message_hash_) { diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index 1a30f76d40..697e4d67f8 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -138,8 +138,18 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return td::Status::OK(); } td::Result> load_cell(td::Slice hash) override { - TRY_RESULT(loaded_cell, get_cell_info_force(hash).cell->load_cell()); - return std::move(loaded_cell.data_cell); + auto info = hash_table_.get_if_exists(hash); + if (info && info->sync_with_db) { + TRY_RESULT(loaded_cell, info->cell->load_cell()); + return std::move(loaded_cell.data_cell); + } + TRY_RESULT(res, loader_->load(hash, true, *this)); + if (res.status != CellLoader::LoadResult::Ok) { + return td::Status::Error("cell not found"); + } + Ref cell = res.cell(); + hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_loaded(info, hash, std::move(res)); }); + return cell; } td::Result> load_root(td::Slice hash) override { return load_cell(hash); @@ -195,9 +205,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat promise->set_result(std::move(cell)); }); } - CellInfo &get_cell_info_force(td::Slice hash) { - return hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_force(info, hash); }); - } CellInfo &get_cell_info_lazy(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) { return hash_table_.apply(hash.substr(hash.size() - Cell::hash_bytes), [&](CellInfo &info) { update_cell_info_lazy(info, level_mask, hash, depth); }); diff --git a/tdnet/td/net/TcpListener.cpp b/tdnet/td/net/TcpListener.cpp index e711cbbda2..c9bebe2d4c 100644 --- a/tdnet/td/net/TcpListener.cpp +++ b/tdnet/td/net/TcpListener.cpp @@ -61,7 +61,7 @@ void TcpListener::loop() { break; } TRY_RESULT(client_socket, std::move(r_socket)); - LOG(ERROR) << "Accept"; + LOG(INFO) << "Accept"; callback_->accept(std::move(client_socket)); } if (td::can_close(server_socket_fd_)) { diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index c67203f089..fe7af04e77 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -231,7 +231,7 @@ void CellDbIn::start_up() { db_options.two_level_index_and_filter = db_options.enable_bloom_filter && opts_->state_ttl() >= 60 * 60 * 24 * 30; // 30 days if (db_options.two_level_index_and_filter && !opts_->get_celldb_in_memory()) { - o_celldb_cache_size = std::max(o_celldb_cache_size ? o_celldb_cache_size.value() : 0UL, 16UL << 30); + o_celldb_cache_size = std::max(o_celldb_cache_size ? o_celldb_cache_size.value() : 0UL, 16UL << 30); } if (o_celldb_cache_size) { From 5c2461b98c56a7fe930b15fc0c6f22c6f7a0d1b8 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Tue, 8 Apr 2025 12:12:11 +0400 Subject: [PATCH 207/524] not always gh mac runners have nproc utility (#1612) --- assembly/native/build-macos-portable.sh | 10 +++++----- assembly/native/build-macos-shared.sh | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index 81ac507b76..2779730886 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -55,7 +55,7 @@ git clone https://github.com/lz4/lz4.git ../3pp/lz4 cd ../3pp/lz4 lz4Path=`pwd` git checkout v1.9.4 -make -j$(nproc) +make -j4 test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } cd ../../build # ./lib/liblz4.a @@ -73,7 +73,7 @@ if [ ! -d "../3pp/libsodium" ]; then git checkout 1.0.18 ./autogen.sh ./configure --with-pic --enable-static - make -j$(nproc) + make -j4 test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } cd ../../build else @@ -87,7 +87,7 @@ if [ ! -d "../3pp/openssl_3" ]; then opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make build_libs -j$(nproc) + make build_libs -j4 test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } cd ../../build else @@ -100,7 +100,7 @@ if [ ! -d "../3pp/zlib" ]; then cd ../3pp/zlib zlibPath=`pwd` ./configure --static - make -j$(nproc) + make -j4 test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } cd ../../build else @@ -116,7 +116,7 @@ if [ ! -d "../3pp/libmicrohttpd" ]; then cd libmicrohttpd-1.0.1 libmicrohttpdPath=`pwd` ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - make -j$(nproc) + make -j4 test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } cd ../../../build else diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index dbb4e7d29a..595c44e3e0 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -53,7 +53,7 @@ if [ ! -d "lz4" ]; then cd lz4 lz4Path=`pwd` git checkout v1.9.4 - make -j$(nproc) + make -j4 test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } cd .. else From 7528a883696f3a74afa3e8731575b19e52e97f33 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sat, 22 Mar 2025 13:51:15 +0300 Subject: [PATCH 208/524] Cache validator sets for create_shard_state --- crypto/block/mc-config.cpp | 71 ++++++++++++++++++++++++++++++-- crypto/block/mc-config.h | 4 +- lite-client/lite-client.h | 2 +- validator/impl/shard.cpp | 4 +- validator/impl/validator-set.hpp | 2 +- 5 files changed, 73 insertions(+), 10 deletions(-) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 0f019b068b..7b4f326f8d 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -38,6 +38,7 @@ #include "openssl/digest.hpp" #include #include +#include namespace block { using namespace std::literals::string_literals; @@ -253,7 +254,7 @@ td::Status Config::unpack() { } config_dict = std::make_unique(config_root, 32); if (mode & needValidatorSet) { - auto vset_res = unpack_validator_set(get_config_param(35, 34)); + auto vset_res = unpack_validator_set(get_config_param(35, 34), true); if (vset_res.is_error()) { return vset_res.move_as_error(); } @@ -486,15 +487,74 @@ td::Result Config::unpack_workchain_list(Ref root) { return std::move(pair.first); } -td::Result> Config::unpack_validator_set(Ref vset_root) { +class ValidatorSetCache { +public: + ValidatorSetCache() { + cache_.reserve(MAX_CACHE_SIZE + 1); + } + + std::shared_ptr get(const vm::CellHash& hash) { + std::lock_guard lock{mutex_}; + auto it = cache_.find(hash); + if (it == cache_.end()) { + return {}; + } + auto entry = it->second.get(); + entry->remove(); + lru_.put(entry); + return entry->value; + } + + void set(const vm::CellHash& hash, std::shared_ptr vset) { + std::lock_guard lock{mutex_}; + std::unique_ptr& entry = cache_[hash]; + if (entry == nullptr) { + entry = std::make_unique(hash, std::move(vset)); + } else { + entry->value = std::move(vset); + entry->remove(); + } + lru_.put(entry.get()); + if (cache_.size() > MAX_CACHE_SIZE) { + auto to_remove = (CacheEntry*)lru_.get(); + CHECK(to_remove); + to_remove->remove(); + cache_.erase(to_remove->key); + } + } + +private: + std::mutex mutex_; + + struct CacheEntry : td::ListNode { + explicit CacheEntry(vm::CellHash key, std::shared_ptr value) : key(key), value(std::move(value)) { + } + vm::CellHash key; + std::shared_ptr value; + }; + td::HashMap> cache_; + td::ListNode lru_; + + static constexpr size_t MAX_CACHE_SIZE = 100; +}; + +td::Result> Config::unpack_validator_set(Ref vset_root, bool use_cache) { if (vset_root.is_null()) { return td::Status::Error("validator set is absent"); } + static ValidatorSetCache cache; + if (use_cache) { + auto result = cache.get(vset_root->get_hash()); + if (result) { + return result; + } + } + gen::ValidatorSet::Record_validators_ext rec; Ref dict_root; if (!tlb::unpack_cell(vset_root, rec)) { gen::ValidatorSet::Record_validators rec0; - if (!tlb::unpack_cell(std::move(vset_root), rec0)) { + if (!tlb::unpack_cell(vset_root, rec0)) { return td::Status::Error("validator set is invalid"); } rec.utime_since = rec0.utime_since; @@ -515,7 +575,7 @@ td::Result> Config::unpack_validator_set(Ref(rec.utime_since, rec.utime_until, rec.total, rec.main); + auto ptr = std::make_shared(rec.utime_since, rec.utime_until, rec.total, rec.main); for (int i = 0; i < rec.total; i++) { key_buffer.store_ulong(i); auto descr_cs = dict.lookup(key_buffer.bits(), 16); @@ -548,6 +608,9 @@ td::Result> Config::unpack_validator_set(Reftotal_weight) { return td::Status::Error("validator set declares incorrect total weight"); } + if (use_cache) { + cache.set(vset_root->get_hash(), ptr); + } return std::move(ptr); } diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 98e6a26df1..a44522f11c 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -559,7 +559,7 @@ class Config { td::BitArray<256> config_addr; Ref config_root; std::unique_ptr config_dict; - std::unique_ptr cur_validators_; + std::shared_ptr cur_validators_; std::unique_ptr workchains_dict_; WorkchainSet workchains_; int version_{-1}; @@ -623,7 +623,7 @@ class Config { bool set_block_id_ext(const ton::BlockIdExt& block_id_ext); td::Result> get_special_smartcontracts(bool without_config = false) const; bool is_special_smartcontract(const ton::StdSmcAddress& addr) const; - static td::Result> unpack_validator_set(Ref valset_root); + static td::Result> unpack_validator_set(Ref valset_root, bool use_cache = false); td::Result> get_storage_prices() const; static td::Result do_get_one_storage_prices(vm::CellSlice cs); td::Result get_gas_limits_prices(bool is_masterchain = false) const; diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 721d2b20d6..3284f255d6 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -143,7 +143,7 @@ class TestNode : public td::actor::Actor { ton::LogicalTime end_lt{0}; ton::Bits256 vset_hash; Ref vset_root; - std::unique_ptr vset; + std::shared_ptr vset; std::map vset_map; int special_idx{-1}; std::pair created_total, created_special; diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 0ab216f712..d2d402dd66 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -390,12 +390,12 @@ td::Status MasterchainStateQ::mc_reinit() { auto cv_root = config_->get_config_param(35, 34); if (cv_root.not_null()) { - TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(cv_root))); + TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(cv_root), true)); cur_validators_ = std::move(validators); } auto nv_root = config_->get_config_param(37, 36); if (nv_root.not_null()) { - TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root))); + TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root), true)); next_validators_ = std::move(validators); } diff --git a/validator/impl/validator-set.hpp b/validator/impl/validator-set.hpp index 951ca4b715..81c0bd5183 100644 --- a/validator/impl/validator-set.hpp +++ b/validator/impl/validator-set.hpp @@ -74,7 +74,7 @@ class ValidatorSetCompute { private: const block::Config* config_{nullptr}; - std::unique_ptr cur_validators_, next_validators_; + std::shared_ptr cur_validators_, next_validators_; td::Ref compute_validator_set(ShardIdFull shard, const block::ValidatorSet& vset, UnixTime time, CatchainSeqno seqno) const; }; From 21719999796a7d6454a8d903e859c8a3b7e288a6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 8 Apr 2025 17:12:16 +0300 Subject: [PATCH 209/524] Avoid copying Account object --- emulator/emulator-extern.cpp | 5 +++-- emulator/transaction-emulator.cpp | 2 +- emulator/transaction-emulator.h | 12 ++++++++---- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/emulator/emulator-extern.cpp b/emulator/emulator-extern.cpp index eb5ff9f9e5..c5f3c04e00 100644 --- a/emulator/emulator-extern.cpp +++ b/emulator/emulator-extern.cpp @@ -187,7 +187,7 @@ const char *transaction_emulator_emulate_transaction(void *transaction_emulator, external_not_accepted->elapsed_time); } - auto emulation_success = dynamic_cast(*emulation_result); + auto emulation_success = std::move(dynamic_cast(*emulation_result)); auto trans_boc_b64 = cell_to_boc_b64(std::move(emulation_success.transaction)); if (trans_boc_b64.is_error()) { ERROR_RESPONSE(PSTRING() << "Can't serialize Transaction to boc " << trans_boc_b64.move_as_error()); @@ -260,7 +260,8 @@ const char *transaction_emulator_emulate_tick_tock_transaction(void *transaction } auto emulation_result = result.move_as_ok(); - auto emulation_success = dynamic_cast(*emulation_result); + auto emulation_success = + std::move(dynamic_cast(*emulation_result)); auto trans_boc_b64 = cell_to_boc_b64(std::move(emulation_success.transaction)); if (trans_boc_b64.is_error()) { ERROR_RESPONSE(PSTRING() << "Can't serialize Transaction to boc " << trans_boc_b64.move_as_error()); diff --git a/emulator/transaction-emulator.cpp b/emulator/transaction-emulator.cpp index 6267f9bd0b..abdf0d2cfa 100644 --- a/emulator/transaction-emulator.cpp +++ b/emulator/transaction-emulator.cpp @@ -141,7 +141,7 @@ td::Result TransactionEmulator::emulate_t return td::Status::Error("account hash mismatch"); } - return emulation_result; + return std::move(emulation_result); } else if (auto emulation_not_accepted_ptr = dynamic_cast(emulation.get())) { return td::Status::Error( PSTRING() diff --git a/emulator/transaction-emulator.h b/emulator/transaction-emulator.h index eae109f40a..848548c32b 100644 --- a/emulator/transaction-emulator.h +++ b/emulator/transaction-emulator.h @@ -33,14 +33,18 @@ class TransactionEmulator { virtual ~EmulationResult() = default; }; - struct EmulationSuccess: EmulationResult { + struct EmulationSuccess : EmulationResult { td::Ref transaction; block::Account account; td::Ref actions; - EmulationSuccess(td::Ref transaction_, block::Account account_, std::string vm_log_, td::Ref actions_, double elapsed_time_) : - EmulationResult(vm_log_, elapsed_time_), transaction(transaction_), account(account_) , actions(actions_) - {} + EmulationSuccess(td::Ref transaction_, block::Account account_, std::string vm_log_, + td::Ref actions_, double elapsed_time_) + : EmulationResult(vm_log_, elapsed_time_) + , transaction(transaction_) + , account(std::move(account_)) + , actions(actions_) { + } }; struct EmulationExternalNotAccepted: EmulationResult { From e873d272ac900c7e854bdd6421aae4f3c9efdffb Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Thu, 10 Apr 2025 10:22:04 +0400 Subject: [PATCH 210/524] Native logger for go tunnel --- tdnet/td/net/UdpServer.cpp | 22 +++++++++++++++++++++- tdnet/td/net/tunnel/libtunnel.h | 8 +++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/tdnet/td/net/UdpServer.cpp b/tdnet/td/net/UdpServer.cpp index fdb94da77c..8d05a2bb89 100644 --- a/tdnet/td/net/UdpServer.cpp +++ b/tdnet/td/net/UdpServer.cpp @@ -67,6 +67,26 @@ class UdpServerTunnelImpl : public UdpServer { static void on_recv_batch(void *next, uint8_t *data, size_t num); static void on_reinit(void *next, sockaddr *addr); + static void log(const char *text, const size_t len, const int level) { + const string str(text, len); + switch (level) { + case 0: + LOG(FATAL) << "[TUNNEL] " << str; + break; + case 1: + LOG(ERROR) << "[TUNNEL] " << str; + break; + case 2: + LOG(WARNING) << "[TUNNEL] " << str; + break; + case 3: + LOG(INFO) << "[TUNNEL] " << str; + break; + default: + LOG(DEBUG) << "[TUNNEL] " << str; + break; + } + } }; void UdpServerTunnelImpl::send(td::UdpMessage &&message) { @@ -129,7 +149,7 @@ void UdpServerTunnelImpl::start_up() { auto global_cfg = global_conf_data_R.move_as_ok(); LOG(INFO) << "Initializing ADNL Tunnel..."; - const auto res = PrepareTunnel(&on_recv_batch, &on_reinit, callback_.get(), callback_.get(), tunnel_config_.data(), tunnel_config_.size(), global_cfg.data(), global_cfg.size()); + const auto res = PrepareTunnel(&log, &on_recv_batch, &on_reinit, callback_.get(), callback_.get(), tunnel_config_.data(), tunnel_config_.size(), global_cfg.data(), global_cfg.size()); if (!res.index) { // the reason will be displayed in logs from lib part exit(1); diff --git a/tdnet/td/net/tunnel/libtunnel.h b/tdnet/td/net/tunnel/libtunnel.h index 30e377f0ba..3c6aa31ecd 100644 --- a/tdnet/td/net/tunnel/libtunnel.h +++ b/tdnet/td/net/tunnel/libtunnel.h @@ -35,6 +35,8 @@ typedef void (*RecvCallback)(void* next, uint8_t* data, size_t num); typedef void (*ReinitCallback)(void* next, struct sockaddr* data); +typedef void (*Logger)(const char *text, const size_t len, const int level); + // we need it because we cannot call C func by pointer directly from go static inline void on_recv_batch_ready(RecvCallback cb, void* next, void* data, size_t num) { @@ -45,6 +47,10 @@ static inline void on_reinit(ReinitCallback cb, void* next, void* data) { cb(next, (struct sockaddr*)data); } +static inline void write_log(Logger log, const char *text, const size_t len, const int level) { + log(text, len, level); +} + #line 1 "cgo-generated-wrapper" @@ -103,7 +109,7 @@ extern "C" { //goland:noinspection ALL -extern Tunnel PrepareTunnel(RecvCallback onRecv, ReinitCallback onReinit, void* nextOnRecv, void* nextOnReinit, char* configJson, int configJsonLen, char* networkConfigJson, int networkConfigJsonLen); +extern Tunnel PrepareTunnel(Logger logger, RecvCallback onRecv, ReinitCallback onReinit, void* nextOnRecv, void* nextOnReinit, char* configPath, int configPathLen, char* networkConfigJson, int networkConfigJsonLen); extern int WriteTunnel(size_t tunIdx, uint8_t* data, size_t num); #ifdef __cplusplus From 10e7b8faac31e8f5b59e59fb8d4c0f1ce463d97e Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Thu, 10 Apr 2025 11:18:49 +0400 Subject: [PATCH 211/524] Not run DHT server when tunnel enabled --- validator-engine/validator-engine.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index ff542ecd14..3da6b516c2 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1976,7 +1976,12 @@ void ValidatorEngine::started_adnl() { } void ValidatorEngine::add_dht(ton::PublicKeyHash id) { - auto D = ton::dht::Dht::create(ton::adnl::AdnlNodeIdShort{id}, db_root_, dht_config_, keyring_.get(), adnl_.get()); + auto creator = ton::dht::Dht::create; + if (!tunnel_config_.empty()) { + creator = ton::dht::Dht::create_client; + } + + auto D = creator(ton::adnl::AdnlNodeIdShort{id}, db_root_, dht_config_, keyring_.get(), adnl_.get()); D.ensure(); dht_nodes_[id] = D.move_as_ok(); From 756a628f98dbb69133c8895ba59edafdcff817c2 Mon Sep 17 00:00:00 2001 From: birydrad Date: Mon, 14 Apr 2025 10:48:04 +0400 Subject: [PATCH 212/524] Support celldb_compression_depth in celldbv2 (#1619) Co-authored-by: birydrad <> --- crypto/test/test-db.cpp | 150 ++++++++++++++----------- crypto/vm/db/DynamicBagOfCellsDbV2.cpp | 10 +- validator-engine/validator-engine.cpp | 2 +- 3 files changed, 92 insertions(+), 70 deletions(-) diff --git a/crypto/test/test-db.cpp b/crypto/test/test-db.cpp index 47d833d56c..f4dde8df71 100644 --- a/crypto/test/test-db.cpp +++ b/crypto/test/test-db.cpp @@ -71,7 +71,6 @@ #include "td/actor/actor.h" #include "td/utils/overloaded.h" - class ActorExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { public: ActorExecutor(size_t tn) : tn_(tn) { @@ -1085,7 +1084,9 @@ struct BocOptions { }; KvOptions kv_options; std::variant options; + std::pair compress_depth_range{0, 0}; td::uint64 seed{123}; + td::Random::Xorshift128plus rnd{123}; std::shared_ptr create_kv(std::shared_ptr old_key_value, bool no_reads = false) { if (kv_options.kv_type == KvOptions::InMemory) { @@ -1151,33 +1152,37 @@ struct BocOptions { auto old_kv = std::move(db.kv); old_boc.reset(); using ResT = DB; - return std::visit(td::overloaded( - [&](CreateV1Options &) -> ResT { - auto new_kv = create_kv(std::move(old_kv)); - auto res = DynamicBagOfCellsDb::create(); - res->set_loader(std::make_unique(new_kv->snapshot())); - return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; - }, - [&](CreateV2Options &options) -> ResT { - auto new_kv = create_kv(std::move(old_kv)); - auto res = DynamicBagOfCellsDb::create_v2(options); - res->set_loader(std::make_unique(new_kv->snapshot())); - return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; - }, - [&](CreateInMemoryOptions &options) -> ResT { - auto read_kv = create_kv(std::move(old_kv), false); - auto res = DynamicBagOfCellsDb::create_in_memory(read_kv.get(), options); - auto new_kv = create_kv(std::move(read_kv), true); - res->set_loader(std::make_unique(new_kv->snapshot())); - auto stats = res->get_stats().move_as_ok(); - if (o_root_n) { - ASSERT_EQ(*o_root_n, stats.roots_total_count); - } - VLOG(boc) << "reset roots_n=" << stats.roots_total_count - << " cells_n=" << stats.cells_total_count; - return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; - }), - options); + auto res = std::visit(td::overloaded( + [&](CreateV1Options &) -> ResT { + auto new_kv = create_kv(std::move(old_kv)); + auto res = DynamicBagOfCellsDb::create(); + res->set_loader(std::make_unique(new_kv->snapshot())); + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }, + [&](CreateV2Options &options) -> ResT { + auto new_kv = create_kv(std::move(old_kv)); + auto res = DynamicBagOfCellsDb::create_v2(options); + res->set_loader(std::make_unique(new_kv->snapshot())); + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }, + [&](CreateInMemoryOptions &options) -> ResT { + auto read_kv = create_kv(std::move(old_kv), false); + auto res = DynamicBagOfCellsDb::create_in_memory(read_kv.get(), options); + auto new_kv = create_kv(std::move(read_kv), true); + res->set_loader(std::make_unique(new_kv->snapshot())); + auto stats = res->get_stats().move_as_ok(); + if (o_root_n) { + ASSERT_EQ(*o_root_n, stats.roots_total_count); + } + VLOG(boc) << "reset roots_n=" << stats.roots_total_count + << " cells_n=" << stats.cells_total_count; + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }), + options); + if (compress_depth_range.second != 0) { + res.dboc->set_celldb_compress_depth(rnd.fast(compress_depth_range.first, compress_depth_range.second)); + } + return res; }; void prepare_commit(DynamicBagOfCellsDb &dboc) { td::PerfWarningTimer warning_timer("test_db_prepare_commit"); @@ -1249,6 +1254,9 @@ struct BocOptions { if (async_executor) { sb << ", executor=" << async_executor->describe(); } + if (compress_depth_range.second != 0) { + sb << ", compress_depth=[" << compress_depth_range.first << ";" << compress_depth_range.second << "]"; + } sb << ")"; return sb.as_cslice().str(); @@ -1270,6 +1278,7 @@ void with_all_boc_options(F &&f, size_t tests_n, bool single_thread = false) { auto before = counter(); options.seed = i == 0 ? 123 : i; + options.rnd = td::Random::Xorshift128plus{options.seed}; auto stats_diff = f(options); stats.apply_diff(stats_diff); @@ -1293,39 +1302,51 @@ void with_all_boc_options(F &&f, size_t tests_n, bool single_thread = false) { BocOptions::KvOptions{ .kv_type = BocOptions::KvOptions::RocksDb, .experimental = false, .cache_size = size_t{128 << 20}}, }; + std::vector> compress_depth_ranges = {{0, 5}, {5, 5}, {0, 0}}; std::vector has_executor_options = {false, true}; - for (auto kv_options : kv_options_list) { - for (bool has_executor : has_executor_options) { - std::shared_ptr executor; - if (has_executor) { - executor = std::make_shared( - 4); // 4 - to compare V1 and V2, because V1 has parallel_load = 4 by default - } - // V2 - 4 threads - run({.async_executor = executor, - .kv_options = kv_options, - .options = DynamicBagOfCellsDb::CreateV2Options{ - .extra_threads = 3, .executor = executor, .cache_ttl_max = 5}}); - - // V1 - run({.async_executor = executor, .kv_options = kv_options, .options = DynamicBagOfCellsDb::CreateV1Options{}}); - - // V2 - one thread - run({.async_executor = executor, - .kv_options = kv_options, - .options = - DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 0, .executor = executor, .cache_ttl_max = 5}}); - - // InMemory - for (auto use_arena : {false, true}) { - for (auto less_memory : {false, true}) { - run({.async_executor = executor, - .kv_options = kv_options, - .options = - DynamicBagOfCellsDb::CreateInMemoryOptions{.extra_threads = std::thread::hardware_concurrency(), - .verbose = false, - .use_arena = use_arena, - .use_less_memory_during_creation = less_memory}}); + for (auto compress_depth_range : compress_depth_ranges) { + for (auto kv_options : kv_options_list) { + for (bool has_executor : has_executor_options) { + std::shared_ptr executor; + if (has_executor) { + executor = std::make_shared( + 4); // 4 - to compare V1 and V2, because V1 has parallel_load = 4 by default + } + // V2 - 4 threads + run({ + .async_executor = executor, + .kv_options = kv_options, + .options = + DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 3, .executor = executor, .cache_ttl_max = 5}, + .compress_depth_range = compress_depth_range, + }); + + // V1 + run({.async_executor = executor, + .kv_options = kv_options, + .options = DynamicBagOfCellsDb::CreateV1Options{}, + .compress_depth_range = compress_depth_range}); + + // V2 - one thread + run({.async_executor = executor, + .kv_options = kv_options, + .options = + DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 0, .executor = executor, .cache_ttl_max = 5}, + .compress_depth_range = compress_depth_range}); + + // InMemory + if (compress_depth_range.second == 0) { + for (auto use_arena : {false, true}) { + for (auto less_memory : {false, true}) { + run({.async_executor = executor, + .kv_options = kv_options, + .options = + DynamicBagOfCellsDb::CreateInMemoryOptions{.extra_threads = std::thread::hardware_concurrency(), + .verbose = false, + .use_arena = use_arena, + .use_less_memory_during_creation = less_memory}}); + } + } } } } @@ -1342,7 +1363,7 @@ void with_all_boc_options(F &&f, size_t tests_n, bool single_thread = false) { DynamicBagOfCellsDb::Stats test_dynamic_boc(BocOptions options) { DynamicBagOfCellsDb::Stats stats; - td::Random::Xorshift128plus rnd{options.seed}; + auto &rnd = options.rnd; std::string old_root_hash; std::string old_root_serialization; DB db; @@ -1392,7 +1413,7 @@ TEST(TonDb, DynamicBoc) { }; DynamicBagOfCellsDb::Stats test_dynamic_boc2(BocOptions options) { - td::Random::Xorshift128plus rnd{options.seed}; + auto &rnd = options.rnd; DynamicBagOfCellsDb::Stats stats; int total_roots = rnd.fast(1, !rnd.fast(0, 30) * 100 + 10); @@ -1879,8 +1900,7 @@ DynamicBagOfCellsDb::Stats bench_dboc_get_and_set(BocOptions options) { db.reset_loader(); } - for (auto p : - std::vector>{{10000, 0}, {10000, 5}, {5000, 5000}, {5, 10000}, {0, 10000}}) { + for (auto p : std::vector>{{10000, 0}, {10000, 5}, {5000, 5000}, {5, 10000}, {0, 10000}}) { auto get_n = p.first; auto set_n = p.second; auto hash = arr.root()->get_hash(); @@ -2912,7 +2932,7 @@ TEST(UsageTree, ThreadSafe) { auto cell = vm::gen_random_cell(rnd.fast(2, 100), rnd, false); auto usage_tree = std::make_shared(); auto usage_cell = vm::UsageCell::create(cell, usage_tree->root_ptr()); - std::ptrdiff_t threads_n = 1; // TODO: when CellUsageTree is thread safe, change it to 4 + std::ptrdiff_t threads_n = 1; // TODO: when CellUsageTree is thread safe, change it to 4 auto barrier = std::barrier{threads_n}; std::vector threads; std::vector explorations(threads_n); diff --git a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp index ecf3b76cbf..3b7d7c2336 100644 --- a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp @@ -1008,7 +1008,7 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { } void set_celldb_compress_depth(td::uint32 value) override { - CHECK(value == 0); + celldb_compress_depth_ = value; } vm::ExtCellCreator &as_ext_cell_creator() override { @@ -1250,6 +1250,7 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { }; CreateV2Options options_; + td::int32 celldb_compress_depth_{0}; std::vector> to_inc_; std::vector> to_dec_; std::vector> diff_chunks_; @@ -1416,6 +1417,7 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { stats_.diff_zero.inc(); return; } + auto should_compress = celldb_compress_depth_ != 0 && info->cell->get_depth() == celldb_compress_depth_; bool merge_supported = true; if (merge_supported) { @@ -1443,7 +1445,7 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { stats_.diff_full.inc(); worker.add_result({.type = CellStorer::Diff::Set, .key = info->cell->get_hash(), - .value = CellStorer::serialize_value(ref_cnt_diff + state.db_ref_cnt, data_cell, false)}); + .value = CellStorer::serialize_value(ref_cnt_diff + state.db_ref_cnt, data_cell, should_compress)}); } else { stats_.diff_ref_cnt.inc(); worker.add_result({.type = CellStorer::Diff::Merge, @@ -1479,7 +1481,7 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { worker.add_result( {.type = CellStorer::Diff::Set, .key = info->cell->get_hash(), - .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, false)}); + .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, should_compress)}); stats_.dec_save_full.inc(); } } else { @@ -1496,7 +1498,7 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { worker.add_result( {.type = CellStorer::Diff::Set, .key = info->cell->get_hash(), - .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, false)}); + .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, should_compress)}); stats_.inc_save_full.inc(); } } diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 945d5456f4..6f47f77417 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1464,7 +1464,7 @@ td::Status ValidatorEngine::load_global_config() { if (!session_logs_file_.empty()) { validator_options_.write().set_session_logs_file(session_logs_file_); } - if (celldb_in_memory_ || celldb_v2_) { + if (celldb_in_memory_) { celldb_compress_depth_ = 0; } validator_options_.write().set_celldb_compress_depth(celldb_compress_depth_); From 688c695ed131fc08cef020a2bd32d8f0caa81708 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 14 Apr 2025 09:56:21 +0300 Subject: [PATCH 213/524] New account storage stat (#1568) * New account storage stat * Fix computing storage stat * Don't store zero dict hash * Dictionary::multiset; optimize account storage stat * Don't calculate account storage dict when not needed * Optimize cell loading in account storage stat * Don't store storage dict hash in masterchain * Avoid copying Account object --- common/global-version.h | 2 +- crypto/CMakeLists.txt | 2 + crypto/block/account-storage-stat.cpp | 285 ++++++++++++++++++++++++++ crypto/block/account-storage-stat.h | 109 ++++++++++ crypto/block/block-parse.cpp | 45 ++-- crypto/block/block-parse.h | 7 - crypto/block/block.tlb | 17 +- crypto/block/create-state.cpp | 7 +- crypto/block/mc-config.h | 6 +- crypto/block/transaction.cpp | 238 ++++++++++----------- crypto/block/transaction.h | 12 +- crypto/common/bitstring.cpp | 3 + crypto/vm/boc.h | 8 +- crypto/vm/cells/CellSlice.cpp | 8 + crypto/vm/cells/CellSlice.h | 1 + crypto/vm/dict.cpp | 254 +++++++++++++++++++++++ crypto/vm/dict.h | 4 +- doc/GlobalVersions.md | 7 + emulator/emulator-extern.cpp | 5 +- emulator/transaction-emulator.cpp | 2 +- emulator/transaction-emulator.h | 12 +- tonlib/tonlib/TonlibClient.cpp | 15 +- validator/impl/validate-query.cpp | 1 + 23 files changed, 866 insertions(+), 184 deletions(-) create mode 100644 crypto/block/account-storage-stat.cpp create mode 100644 crypto/block/account-storage-stat.h diff --git a/common/global-version.h b/common/global-version.h index 2308ce3e9c..b54a3bdc52 100644 --- a/common/global-version.h +++ b/common/global-version.h @@ -19,6 +19,6 @@ namespace ton { // See doc/GlobalVersions.md -constexpr int SUPPORTED_VERSION = 10; +constexpr int SUPPORTED_VERSION = 11; } diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index defe2b58e3..79d1117eaa 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -212,6 +212,8 @@ set(BLOCK_SOURCE block/mc-config.cpp block/output-queue-merger.cpp block/transaction.cpp + block/account-storage-stat.h + block/account-storage-stat.cpp block/precompiled-smc/PrecompiledSmartContract.cpp ${TLB_BLOCK_AUTO} diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp new file mode 100644 index 0000000000..56cc8f1e1b --- /dev/null +++ b/crypto/block/account-storage-stat.cpp @@ -0,0 +1,285 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "account-storage-stat.h" + +namespace block { + +AccountStorageStat::AccountStorageStat() : AccountStorageStat({}, {}, 0, 0) { +} + +AccountStorageStat::AccountStorageStat(Ref dict_root, std::vector> roots, + td::uint64 total_cells, td::uint64 total_bits) + : dict_(std::move(dict_root), 256), total_cells_(total_cells), total_bits_(total_bits), roots_(std::move(roots)) { +} + +AccountStorageStat::AccountStorageStat(const AccountStorageStat* parent) + : dict_(parent->dict_) + , dict_up_to_date_(parent->dict_up_to_date_) + , total_cells_(parent->total_cells_) + , total_bits_(parent->total_bits_) + , roots_(parent->roots_) + , parent_(parent) { + CHECK(parent_->parent_ == nullptr); +} + +td::Status AccountStorageStat::replace_roots(std::vector> new_roots, bool check_merkle_depth) { + std::erase_if(new_roots, [](const Ref& c) { return c.is_null(); }); + if (new_roots.empty()) { + roots_.clear(); + total_bits_ = total_cells_ = 0; + dict_ = vm::Dictionary{256}; + cache_ = {}; + dict_up_to_date_ = true; + parent_ = nullptr; + return td::Status::OK(); + } + + auto cmp = [](const Ref& c1, const Ref& c2) { return c1->get_hash() < c2->get_hash(); }; + std::sort(new_roots.begin(), new_roots.end(), cmp); + std::sort(roots_.begin(), roots_.end(), cmp); + std::vector> to_add, to_del; + std::set_difference(new_roots.begin(), new_roots.end(), roots_.begin(), roots_.end(), std::back_inserter(to_add), + cmp); + std::set_difference(roots_.begin(), roots_.end(), new_roots.begin(), new_roots.end(), std::back_inserter(to_del), + cmp); + if (to_add.empty() && to_del.empty()) { + return td::Status::OK(); + } + + for (const Ref& root : to_add) { + TRY_RESULT(info, add_cell(root)); + if (check_merkle_depth && info.max_merkle_depth > MAX_MERKLE_DEPTH) { + return td::Status::Error("too big Merkle depth"); + } + } + for (const Ref& root : to_del) { + TRY_STATUS(remove_cell(root)); + } + + roots_ = std::move(new_roots); + dict_up_to_date_ = false; + for (auto& [_, e] : cache_) { + TRY_STATUS(finalize_entry(e)); + } + return td::Status::OK(); +} + +void AccountStorageStat::add_hint(const td::HashSet& hint) { + td::HashSet visited; + std::function&, bool)> dfs = [&](const Ref& cell, bool is_root) { + if (!visited.insert(cell->get_hash()).second) { + return; + } + Entry& e = get_entry(cell); + e.exists = e.exists_known = true; + if (is_root) { + fetch_from_dict(e).ignore(); + if (e.max_merkle_depth && e.max_merkle_depth.value() != 0) { + return; + } + } + if (hint.contains(cell->get_hash())) { + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); + e.size_bits = cs.size(); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + dfs(cs.prefetch_ref(i), false); + } + } + }; + for (const Ref& root : roots_) { + dfs(root, true); + } +} + +td::Result AccountStorageStat::add_cell(const Ref& cell) { + Entry& e = get_entry(cell); + if (!e.exists_known || e.refcnt_diff < 0) { + TRY_STATUS(fetch_from_dict(e)); + } + ++e.refcnt_diff; + if (e.exists || e.refcnt_diff > 1 || (e.refcnt && e.refcnt.value() + e.refcnt_diff != 1)) { + if (!e.max_merkle_depth) { + TRY_STATUS(fetch_from_dict(e)); + if (!e.max_merkle_depth) { + return td::Status::Error(PSTRING() << "unexpected unknown Merkle depth of cell " << cell->get_hash()); + } + } + return CellInfo{e.max_merkle_depth.value()}; + } + + td::uint32 max_merkle_depth = 0; + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); + e.size_bits = cs.size(); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + TRY_RESULT(info, add_cell(cs.prefetch_ref(i))); + max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); + } + if (cs.special_type() == vm::CellTraits::SpecialType::MerkleProof || + cs.special_type() == vm::CellTraits::SpecialType::MerkleUpdate) { + ++max_merkle_depth; + } + max_merkle_depth = std::min(max_merkle_depth, MERKLE_DEPTH_LIMIT); + Entry& e2 = get_entry(cell); + e2.max_merkle_depth = max_merkle_depth; + return CellInfo{max_merkle_depth}; +} + +td::Status AccountStorageStat::remove_cell(const Ref& cell) { + Entry& e = get_entry(cell); + if (!e.exists_known) { + TRY_STATUS(fetch_from_dict(e)); + } + if (!e.exists) { + return td::Status::Error(PSTRING() << "Failed to remove cell " << cell->get_hash().to_hex() + << " : does not exist in the dict"); + } + --e.refcnt_diff; + if (e.refcnt_diff < 0 && !e.refcnt) { + TRY_STATUS(fetch_from_dict(e)); + } + if (e.refcnt.value() + e.refcnt_diff != 0) { + return td::Status::OK(); + } + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); + e.size_bits = cs.size(); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + TRY_STATUS(remove_cell(cs.prefetch_ref(i))); + } + return td::Status::OK(); +} + +td::Result> AccountStorageStat::get_dict_root() { + if (!dict_up_to_date_) { + std::vector>> values; + for (auto& [_, e] : cache_) { + if (e.dict_refcnt_diff == 0) { + continue; + } + if (!e.exists_known || !e.refcnt || (e.exists && !e.max_merkle_depth)) { + return td::Status::Error("unexpected state of storage stat"); + } + if (e.exists) { + Ref cbr{true}; + auto& cb = cbr.write(); + CHECK(cb.store_long_bool(e.refcnt.value(), 32) && cb.store_long_bool(e.max_merkle_depth.value(), 2)); + values.emplace_back(e.hash.bits(), std::move(cbr)); + } else { + values.emplace_back(e.hash.bits(), Ref{}); + } + e.dict_refcnt_diff = 0; + } + if (!dict_.multiset(values)) { + return td::Status::Error("failed to update dictionary"); + } + dict_up_to_date_ = true; + } + return dict_.get_root_cell(); +} + +void AccountStorageStat::apply_child_stat(AccountStorageStat&& child) { + CHECK(parent_ == nullptr); + if (child.parent_ == nullptr) { + *this = std::move(child); + return; + } + CHECK(child.parent_ == this); + total_bits_ = child.total_bits_; + total_cells_ = child.total_cells_; + dict_ = std::move(child.dict_); + dict_up_to_date_ = child.dict_up_to_date_; + roots_ = std::move(child.roots_); + for (auto& [hash, e] : child.cache_) { + cache_[hash] = std::move(e); + } +} + +AccountStorageStat::Entry& AccountStorageStat::get_entry(const Ref& cell) { + Entry& e = cache_[cell->get_hash()]; + if (e.inited) { + return e; + } + if (parent_) { + auto it = parent_->cache_.find(cell->get_hash()); + if (it != parent_->cache_.end()) { + CHECK(it->second.inited); + e = it->second; + return e; + } + } + e.inited = true; + e.hash = cell->get_hash(); + return e; +} + +td::Status AccountStorageStat::fetch_from_dict(Entry& e) { + if (e.exists_known && e.refcnt && (!e.exists || e.max_merkle_depth)) { + return td::Status::OK(); + } + auto cs = dict_.lookup(e.hash.as_bitslice()); + if (cs.is_null()) { + e.exists = false; + e.refcnt = 0; + } else { + if (cs->size_ext() != 32 + 2) { + return td::Status::Error(PSTRING() << "invalid record for cell " << e.hash.to_hex()); + } + e.exists = true; + e.refcnt = (td::uint32)cs.write().fetch_ulong(32); + e.max_merkle_depth = (td::uint32)cs.write().fetch_ulong(2); + if (e.refcnt.value() == 0) { + return td::Status::Error(PSTRING() << "invalid refcnt=0 for cell " << e.hash.to_hex()); + } + } + e.exists_known = true; + return td::Status::OK(); +} + +td::Status AccountStorageStat::finalize_entry(Entry& e) { + if (e.refcnt_diff == 0) { + return td::Status::OK(); + } + TRY_STATUS(fetch_from_dict(e)); + e.refcnt.value() += e.refcnt_diff; + e.dict_refcnt_diff += e.refcnt_diff; + e.refcnt_diff = 0; + if (e.refcnt.value() == 0) { + if (!e.size_bits) { + return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown cell bits"); + } + --total_cells_; + total_bits_ -= e.size_bits.value(); + e.exists = false; + } else { + if (!e.exists) { + if (!e.size_bits) { + return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown cell bits"); + } + ++total_cells_; + total_bits_ += e.size_bits.value(); + } + e.exists = true; + if (!e.max_merkle_depth) { + return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown merkle depth"); + } + } + return td::Status::OK(); +} + +} // namespace block \ No newline at end of file diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h new file mode 100644 index 0000000000..645f0744d8 --- /dev/null +++ b/crypto/block/account-storage-stat.h @@ -0,0 +1,109 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#pragma once +#include "common/refcnt.hpp" +#include "vm/dict.h" +#include "ton/ton-types.h" +#include "ton/ton-shard.h" +#include "common/bitstring.h" +#include "block.h" +#include "vm/db/CellHashTable.h" + +namespace block { +using td::Ref; + +class AccountStorageStat { + public: + AccountStorageStat(); + AccountStorageStat(Ref dict_root, std::vector> roots, td::uint64 total_cells, + td::uint64 total_bits); + explicit AccountStorageStat(const AccountStorageStat *parent); + AccountStorageStat(const AccountStorageStat &other) = delete; + AccountStorageStat(AccountStorageStat &&other) = default; + ~AccountStorageStat() = default; + + AccountStorageStat &operator=(const AccountStorageStat &other) = delete; + AccountStorageStat &operator=(AccountStorageStat &&other) = default; + + td::Status replace_roots(std::vector> new_roots, bool check_merkle_depth = false); + void add_hint(const td::HashSet &visited); + + td::uint64 get_total_cells() const { + return total_cells_; + } + + td::uint64 get_total_bits() const { + return total_bits_; + } + + td::Result> get_dict_root(); + + td::Result get_dict_hash() { + TRY_RESULT(root, get_dict_root()); + return root.is_null() ? td::Bits256::zero() : td::Bits256{root->get_hash().bits()}; + } + + void apply_child_stat(AccountStorageStat &&child); + + private: + vm::Dictionary dict_; + bool dict_up_to_date_ = true; + td::uint64 total_cells_, total_bits_; + std::vector> roots_; + const AccountStorageStat *parent_ = nullptr; + + struct CellInfo { + td::uint32 max_merkle_depth = 0; + }; + + td::Result add_cell(const Ref &cell); + td::Status remove_cell(const Ref &cell); + + struct Entry { + bool inited = false; + vm::CellHash hash; + td::optional size_bits; + bool exists_known = false; + bool exists = false; + td::optional refcnt, max_merkle_depth; + td::int32 refcnt_diff = 0; + td::int32 dict_refcnt_diff = 0; + }; + + td::HashMap> cache_; + + Entry &get_entry(const Ref &cell); + td::Status fetch_from_dict(Entry &e); + td::Status finalize_entry(Entry &e); + + static constexpr td::uint32 MERKLE_DEPTH_LIMIT = 3; + static constexpr td::uint32 MAX_MERKLE_DEPTH = 2; +}; + +class StorageStatCalculationContext : public td::Context { + public: + explicit StorageStatCalculationContext(bool active) : active_(active) { + } + bool calculating_storage_stat() const { + return active_; + } + + private: + bool active_ = false; +}; + +} // namespace block diff --git a/crypto/block/block-parse.cpp b/crypto/block/block-parse.cpp index 50851c7956..6d645ac79d 100644 --- a/crypto/block/block-parse.cpp +++ b/crypto/block/block-parse.cpp @@ -960,41 +960,34 @@ const MsgEnvelope t_MsgEnvelope; const RefTo t_Ref_MsgEnvelope; bool StorageUsed::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { - return t_VarUInteger_7.validate_skip(ops, cs, weak) // cells:(VarUInteger 7) - && t_VarUInteger_7.validate_skip(ops, cs, weak) // bits:(VarUInteger 7) - && t_VarUInteger_7.validate_skip(ops, cs, weak); // public_cells:(VarUInteger 7) -} - -bool StorageUsed::skip(vm::CellSlice& cs) const { - return t_VarUInteger_7.skip(cs) // cells:(VarUInteger 7) - && t_VarUInteger_7.skip(cs) // bits:(VarUInteger 7) - && t_VarUInteger_7.skip(cs); // public_cells:(VarUInteger 7) -} - -const StorageUsed t_StorageUsed; - -bool StorageUsedShort::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return t_VarUInteger_7.validate_skip(ops, cs, weak) // cells:(VarUInteger 7) && t_VarUInteger_7.validate_skip(ops, cs, weak); // bits:(VarUInteger 7) } -bool StorageUsedShort::skip(vm::CellSlice& cs) const { +bool StorageUsed::skip(vm::CellSlice& cs) const { return t_VarUInteger_7.skip(cs) // cells:(VarUInteger 7) && t_VarUInteger_7.skip(cs); // bits:(VarUInteger 7) } -const StorageUsedShort t_StorageUsedShort; +const StorageUsed t_StorageUsed; const Maybe t_Maybe_Grams; bool StorageInfo::skip(vm::CellSlice& cs) const { - return t_StorageUsed.skip(cs) // used:StorageUsed - && cs.advance(32) // last_paid:uint32 - && t_Maybe_Grams.skip(cs); // due_payment:(Maybe Grams) + int extra_tag = 0; + return t_StorageUsed.skip(cs) // used:StorageUsed + && cs.fetch_uint_to(3, extra_tag) // storage_extra:StorageExtraInfo + && (extra_tag == 0 || cs.advance(256)) // storage_extra_info$001 dict_hash:uint256 = StorageExtraInfo; + && cs.advance(32) // last_paid:uint32 + && t_Maybe_Grams.skip(cs); // due_payment:(Maybe Grams) } bool StorageInfo::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { - return t_StorageUsed.validate_skip(ops, cs, weak) // used:StorageUsed + int extra_tag = 0; + return t_StorageUsed.validate_skip(ops, cs, weak) // used:StorageUsed + && cs.fetch_uint_to(3, extra_tag) // storage_extra:StorageExtraInfo + && (extra_tag == 0 || + (extra_tag == 1 && cs.advance(256))) // storage_extra_info$001 dict_hash:uint256 = StorageExtraInfo; && cs.advance(32) // last_paid:uint32 && t_Maybe_Grams.validate_skip(ops, cs, weak); // due_payment:(Maybe Grams) } @@ -1368,7 +1361,7 @@ bool TrActionPhase::skip(vm::CellSlice& cs) const { && cs.advance(16 * 4 + 256) // tot_actions:uint16 spec_actions:uint16 // skipped_actions:uint16 msgs_created:uint16 // action_list_hash:uint256 - && t_StorageUsedShort.skip(cs); // tot_msg_size:StorageUsedShort + && t_StorageUsed.skip(cs); // tot_msg_size:StorageUsed } bool TrActionPhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { @@ -1381,7 +1374,7 @@ bool TrActionPhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const && cs.advance(16 * 4 + 256) // tot_actions:uint16 spec_actions:uint16 // skipped_actions:uint16 msgs_created:uint16 // action_list_hash:uint256 - && t_StorageUsedShort.validate_skip(ops, cs, weak); // tot_msg_size:StorageUsed + && t_StorageUsed.validate_skip(ops, cs, weak); // tot_msg_size:StorageUsed } const TrActionPhase t_TrActionPhase; @@ -1392,11 +1385,11 @@ bool TrBouncePhase::skip(vm::CellSlice& cs) const { return cs.advance(2); // tr_phase_bounce_negfunds$00 case tr_phase_bounce_nofunds: return cs.advance(2) // tr_phase_bounce_nofunds$01 - && t_StorageUsedShort.skip(cs) // msg_size:StorageUsedShort + && t_StorageUsed.skip(cs) // msg_size:StorageUsed && t_Grams.skip(cs); // req_fwd_fees:Grams case tr_phase_bounce_ok: return cs.advance(1) // tr_phase_bounce_ok$1 - && t_StorageUsedShort.skip(cs) // msg_size:StorageUsedShort + && t_StorageUsed.skip(cs) // msg_size:StorageUsed && t_Grams.skip(cs) // msg_fees:Grams && t_Grams.skip(cs); // fwd_fees:Grams } @@ -1409,11 +1402,11 @@ bool TrBouncePhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const return cs.advance(2); // tr_phase_bounce_negfunds$00 case tr_phase_bounce_nofunds: return cs.advance(2) // tr_phase_bounce_nofunds$01 - && t_StorageUsedShort.validate_skip(ops, cs, weak) // msg_size:StorageUsedShort + && t_StorageUsed.validate_skip(ops, cs, weak) // msg_size:StorageUsed && t_Grams.validate_skip(ops, cs, weak); // req_fwd_fees:Grams case tr_phase_bounce_ok: return cs.advance(1) // tr_phase_bounce_ok$1 - && t_StorageUsedShort.validate_skip(ops, cs, weak) // msg_size:StorageUsedShort + && t_StorageUsed.validate_skip(ops, cs, weak) // msg_size:StorageUsed && t_Grams.validate_skip(ops, cs, weak) // msg_fees:Grams && t_Grams.validate_skip(ops, cs, weak); // fwd_fees:Grams } diff --git a/crypto/block/block-parse.h b/crypto/block/block-parse.h index 65f8b91fe2..fd17c6579f 100644 --- a/crypto/block/block-parse.h +++ b/crypto/block/block-parse.h @@ -493,13 +493,6 @@ struct StorageUsed final : TLB_Complex { extern const StorageUsed t_StorageUsed; -struct StorageUsedShort final : TLB_Complex { - bool skip(vm::CellSlice& cs) const override; - bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override; -}; - -extern const StorageUsedShort t_StorageUsedShort; - struct StorageInfo final : TLB_Complex { bool skip(vm::CellSlice& cs) const override; bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override; diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 4a8bbc065d..5cba3c69f1 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -246,14 +246,13 @@ out_msg_queue_extra#0 dispatch_queue:DispatchQueue out_queue_size:(Maybe uint48) _ out_queue:OutMsgQueue proc_info:ProcessedInfo extra:(Maybe OutMsgQueueExtra) = OutMsgQueueInfo; -// -storage_used$_ cells:(VarUInteger 7) bits:(VarUInteger 7) - public_cells:(VarUInteger 7) = StorageUsed; -storage_used_short$_ cells:(VarUInteger 7) - bits:(VarUInteger 7) = StorageUsedShort; +storage_extra_none$000 = StorageExtraInfo; +storage_extra_info$001 dict_hash:uint256 = StorageExtraInfo; + +storage_used$_ cells:(VarUInteger 7) bits:(VarUInteger 7) = StorageUsed; -storage_info$_ used:StorageUsed last_paid:uint32 +storage_info$_ used:StorageUsed storage_extra:StorageExtraInfo last_paid:uint32 due_payment:(Maybe Grams) = StorageInfo; account_none$0 = Account; @@ -341,13 +340,13 @@ tr_phase_action$_ success:Bool valid:Bool no_funds:Bool total_fwd_fees:(Maybe Grams) total_action_fees:(Maybe Grams) result_code:int32 result_arg:(Maybe int32) tot_actions:uint16 spec_actions:uint16 skipped_actions:uint16 msgs_created:uint16 - action_list_hash:bits256 tot_msg_size:StorageUsedShort + action_list_hash:bits256 tot_msg_size:StorageUsed = TrActionPhase; tr_phase_bounce_negfunds$00 = TrBouncePhase; -tr_phase_bounce_nofunds$01 msg_size:StorageUsedShort +tr_phase_bounce_nofunds$01 msg_size:StorageUsed req_fwd_fees:Grams = TrBouncePhase; -tr_phase_bounce_ok$1 msg_size:StorageUsedShort +tr_phase_bounce_ok$1 msg_size:StorageUsed msg_fees:Grams fwd_fees:Grams = TrBouncePhase; // trans_ord$0000 credit_first:Bool diff --git a/crypto/block/create-state.cpp b/crypto/block/create-state.cpp index c8c8b970da..4a74ac0f60 100644 --- a/crypto/block/create-state.cpp +++ b/crypto/block/create-state.cpp @@ -338,10 +338,11 @@ td::RefInt256 create_smartcontract(td::RefInt256 smc_addr, Ref code, R PDO(cb.store_long_rchk_bool(workchain_id, ctor == 2 ? 8 : 32) && cb.store_bits_bool(addr.cbits(), 256)); THRERR("Cannot serialize addr:MsgAddressInt of the new smart contract"); // storage_stat:StorageInfo -> storage_stat.used:StorageUsed - PDO(block::store_UInt7(cb, stats.cells) // cells:(VarUInteger 7) - && block::store_UInt7(cb, stats.bits) // bits:(VarUInteger 7) - && block::store_UInt7(cb, stats.public_cells)); // public_cells:(VarUInteger 7) + PDO(block::store_UInt7(cb, stats.cells) // cells:(VarUInteger 7) + && block::store_UInt7(cb, stats.bits)) // bits:(VarUInteger 7) THRERR("Cannot serialize used:StorageUsed of the new smart contract"); + PDO(cb.store_zeroes_bool(3)); // extra:StorageExtraInfo + THRERR("Cannot serialize storage_extra:StorageExtraInfo of the new smart contract"); PDO(cb.store_long_bool(0, 33)); // last_paid:uint32 due_payment:(Maybe Grams) PDO(cb.append_data_cell_bool(storage)); // storage:AccountStorage THRERR("Cannot create Account of the new smart contract"); diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index a44522f11c..8c1afd965b 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -156,6 +156,10 @@ class McShardHashI : public td::CntObject { virtual bool before_merge() const = 0; }; +struct StorageUsed { + td::uint64 cells = 0, bits = 0; +}; + struct McShardHash : public McShardHashI { ton::BlockIdExt blk_; ton::LogicalTime start_lt_, end_lt_; @@ -336,7 +340,7 @@ struct StoragePrices { , mc_cell_price(_mc_cprice) { } static td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector& pricing, - const vm::CellStorageStat& storage_stat, ton::UnixTime last_paid, + const StorageUsed& storage_used, ton::UnixTime last_paid, bool is_special, bool is_masterchain); }; diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 34d2351144..417ce0cf46 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -257,6 +257,11 @@ bool Account::unpack_storage_info(vm::CellSlice& cs) { return false; } last_paid = info.last_paid; + if (info.storage_extra.write().fetch_long(3) == 1) { + info.storage_extra->prefetch_bits_to(storage_dict_hash.value_force()); + } else { + storage_dict_hash = {}; + } if (info.due_payment->prefetch_ulong(1) == 1) { vm::CellSlice& cs2 = info.due_payment.write(); cs2.advance(1); @@ -268,11 +273,9 @@ bool Account::unpack_storage_info(vm::CellSlice& cs) { due_payment = td::zero_refint(); } unsigned long long u = 0; - u |= storage_stat.cells = block::tlb::t_VarUInteger_7.as_uint(*used.cells); - u |= storage_stat.bits = block::tlb::t_VarUInteger_7.as_uint(*used.bits); - u |= storage_stat.public_cells = block::tlb::t_VarUInteger_7.as_uint(*used.public_cells); - LOG(DEBUG) << "last_paid=" << last_paid << "; cells=" << storage_stat.cells << " bits=" << storage_stat.bits - << " public_cells=" << storage_stat.public_cells; + u |= storage_used.cells = block::tlb::t_VarUInteger_7.as_uint(*used.cells); + u |= storage_used.bits = block::tlb::t_VarUInteger_7.as_uint(*used.bits); + LOG(DEBUG) << "last_paid=" << last_paid << "; cells=" << storage_used.cells << " bits=" << storage_used.bits; return (u != std::numeric_limits::max()); } @@ -527,7 +530,8 @@ bool Account::init_new(ton::UnixTime now) { last_trans_hash_.set_zero(); now_ = now; last_paid = 0; - storage_stat.clear(); + storage_used = {}; + storage_dict_hash = {}; due_payment = td::zero_refint(); balance.set_zero(); if (my_addr_exact.is_null()) { @@ -617,12 +621,12 @@ bool Account::belongs_to_shard(ton::ShardIdFull shard) const { * @param payment The total sum to be updated. * @param delta The time delta for which the payment is calculated. * @param prices The storage prices. - * @param storage Account storage statistics. + * @param storage_used Account storage statistics. * @param is_mc A flag indicating whether the account is in the masterchain. */ void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, const block::StoragePrices& prices, - const vm::CellStorageStat& storage, bool is_mc) { - td::BigInt256 c{(long long)storage.cells}, b{(long long)storage.bits}; + const StorageUsed& storage_used, bool is_mc) { + td::BigInt256 c{(long long)storage_used.cells}, b{(long long)storage_used.bits}; if (is_mc) { // storage.cells * prices.mc_cell_price + storage.bits * prices.mc_bit_price; c.mul_short(prices.mc_cell_price); @@ -643,7 +647,7 @@ void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, co * * @param now The current Unix time. * @param pricing The vector of storage prices. - * @param storage_stat Account storage statistics. + * @param storage_used Account storage statistics. * @param last_paid The Unix time when the last payment was made. * @param is_special A flag indicating if the account is special. * @param is_masterchain A flag indicating if the account is in the masterchain. @@ -651,7 +655,7 @@ void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, co * @returns The computed storage fees as RefInt256. */ td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std::vector& pricing, - const vm::CellStorageStat& storage_stat, ton::UnixTime last_paid, + const StorageUsed& storage_used, ton::UnixTime last_paid, bool is_special, bool is_masterchain) { if (now <= last_paid || !last_paid || is_special || pricing.empty() || now <= pricing[0].valid_since) { return td::zero_refint(); @@ -669,7 +673,7 @@ td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std:: ton::UnixTime valid_until = (i < n - 1 ? std::min(now, pricing[i + 1].valid_since) : now); if (upto < valid_until) { assert(upto >= pricing[i].valid_since); - add_partial_storage_payment(total.unique_write(), valid_until - upto, pricing[i], storage_stat, is_masterchain); + add_partial_storage_payment(total.unique_write(), valid_until - upto, pricing[i], storage_used, is_masterchain); } upto = valid_until; } @@ -685,7 +689,7 @@ td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std:: * @returns The computed storage fees as RefInt256. */ td::RefInt256 Account::compute_storage_fees(ton::UnixTime now, const std::vector& pricing) const { - return StoragePrices::compute_storage_fees(now, pricing, storage_stat, last_paid, is_special, is_masterchain()); + return StoragePrices::compute_storage_fees(now, pricing, storage_used, last_paid, is_special, is_masterchain()); } namespace transaction { @@ -1848,7 +1852,7 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { if (S.is_error()) { // Rollback changes to state, fail action phase LOG(INFO) << "Account state size exceeded limits: " << S.move_as_error(); - new_storage_stat.clear(); + new_account_storage_stat = {}; new_code = old_code; new_data = old_data; new_library = old_library; @@ -2104,7 +2108,7 @@ int Transaction::try_action_change_library(vm::CellSlice& cs, ActionPhase& ap, c LOG(DEBUG) << "added " << ((rec.mode >> 1) ? "public" : "private") << " library with hash " << hash.to_hex(); } new_library = std::move(dict).extract_root_cell(); - } catch (vm::VmError& vme) { + } catch (vm::VmError&) { return 42; } ap.spec_actions++; @@ -2931,7 +2935,7 @@ static td::uint32 get_public_libraries_diff_count(const td::Ref& old_l * This function is not called for special accounts. * * @param size_limits The size limits configuration. - * @param update_storage_stat Store storage stat in the Transaction's CellStorageStat. + * @param update_storage_stat Store storage stat in the Transaction's AccountStorageStat. * * @returns A `td::Status` indicating the result of the check. * - If the state limits are within the allowed range, returns OK. @@ -2939,60 +2943,44 @@ static td::uint32 get_public_libraries_diff_count(const td::Ref& old_l */ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, bool update_storage_stat) { auto cell_equal = [](const td::Ref& a, const td::Ref& b) -> bool { - if (a.is_null()) { - return b.is_null(); - } - if (b.is_null()) { - return false; - } - return a->get_hash() == b->get_hash(); + return a.is_null() || b.is_null() ? a.is_null() == b.is_null() : a->get_hash() == b->get_hash(); }; if (cell_equal(account.code, new_code) && cell_equal(account.data, new_data) && cell_equal(account.library, new_library)) { return td::Status::OK(); } - vm::CellStorageStat storage_stat; - storage_stat.limit_cells = size_limits.max_acc_state_cells; - storage_stat.limit_bits = size_limits.max_acc_state_bits; + AccountStorageStat storage_stat; + if (update_storage_stat && account.account_storage_stat) { + storage_stat = AccountStorageStat{&account.account_storage_stat.value()}; + } { TD_PERF_COUNTER(transaction_storage_stat_a); td::Timer timer; - auto add_used_storage = [&](const td::Ref& cell) -> td::Status { - if (cell.not_null()) { - TRY_RESULT(res, storage_stat.add_used_storage(cell)); - if (res.max_merkle_depth > max_allowed_merkle_depth) { - return td::Status::Error("too big merkle depth"); - } - } - return td::Status::OK(); - }; - TRY_STATUS(add_used_storage(new_code)); - TRY_STATUS(add_used_storage(new_data)); - TRY_STATUS(add_used_storage(new_library)); + TRY_STATUS(storage_stat.replace_roots({new_code, new_data, new_library}, /* check_merkle_depth = */ true)); if (timer.elapsed() > 0.1) { - LOG(INFO) << "Compute used storage took " << timer.elapsed() << "s"; + LOG(INFO) << "Compute used storage (1) took " << timer.elapsed() << "s"; } } - if (acc_status == Account::acc_active) { - storage_stat.clear_limit(); - } else { - storage_stat.clear(); - } - td::Status res; - if (storage_stat.cells > size_limits.max_acc_state_cells || storage_stat.bits > size_limits.max_acc_state_bits) { - res = td::Status::Error(PSTRING() << "account state is too big"); - } else if (account.is_masterchain() && !cell_equal(account.library, new_library) && - get_public_libraries_count(new_library) > size_limits.max_acc_public_libraries) { - res = td::Status::Error("too many public libraries"); - } else { - res = td::Status::OK(); + if (storage_stat.get_total_cells() > size_limits.max_acc_state_cells || + storage_stat.get_total_bits() > size_limits.max_acc_state_bits) { + return td::Status::Error(PSTRING() << "account state is too big: cells=" << storage_stat.get_total_cells() + << ", bits=" << storage_stat.get_total_bits() + << " (max cells=" << size_limits.max_acc_state_cells + << ", max bits=" << size_limits.max_acc_state_bits << ")"); + } + if (account.is_masterchain() && !cell_equal(account.library, new_library)) { + auto libraries_count = get_public_libraries_count(new_library); + if (libraries_count > size_limits.max_acc_public_libraries) { + return td::Status::Error(PSTRING() << "too many public libraries: " << libraries_count << " (max " + << size_limits.max_acc_public_libraries << ")"); + } } if (update_storage_stat) { // storage_stat will be reused in compute_state() - new_storage_stat = std::move(storage_stat); + new_account_storage_stat.value_force() = std::move(storage_stat); } - return res; + return td::Status::OK(); } /** @@ -3140,44 +3128,6 @@ bool Account::store_acc_status(vm::CellBuilder& cb, int acc_status) const { return cb.store_long_bool(v, 2); } -/** - * Tries to update the storage statistics based on the old storage statistics and old account state without fully recomputing it. - * - * It succeeds if only root cell of AccountStorage is changed. - * old_cs and new_cell are AccountStorage without extra currencies (if global_version >= 10). - * - * @param old_stat The old storage statistics. - * @param old_cs The old AccountStorage. - * @param new_cell The new AccountStorage. - * - * @returns An optional value of type vm::CellStorageStat. If the update is successful, it returns the new storage statistics. Otherwise, it returns an empty optional. - */ -static td::optional try_update_storage_stat(const vm::CellStorageStat& old_stat, - td::Ref old_cs, - td::Ref new_cell) { - if (old_stat.cells == 0 || old_cs.is_null()) { - return {}; - } - vm::CellSlice new_cs = vm::CellSlice(vm::NoVm(), new_cell); - if (old_cs->size_refs() != new_cs.size_refs()) { - return {}; - } - for (unsigned i = 0; i < old_cs->size_refs(); ++i) { - if (old_cs->prefetch_ref(i)->get_hash() != new_cs.prefetch_ref(i)->get_hash()) { - return {}; - } - } - if (old_stat.bits < old_cs->size()) { - return {}; - } - - vm::CellStorageStat new_stat; - new_stat.cells = old_stat.cells; - new_stat.bits = old_stat.bits - old_cs->size() + new_cs.size(); - new_stat.public_cells = old_stat.public_cells; - return new_stat; -} - /** * Removes extra currencies dict from AccountStorage. * @@ -3185,9 +3135,9 @@ static td::optional try_update_storage_stat(const vm::CellS * * @param storage_cs AccountStorage as CellSlice. * - * @returns AccountStorage without extra currencies as Cell. + * @returns AccountStorage without extra currencies as CellSlice. */ -static td::Ref storage_without_extra_currencies(td::Ref storage_cs) { +static td::Ref storage_without_extra_currencies(td::Ref storage_cs) { block::gen::AccountStorage::Record rec; if (!block::gen::csr_unpack(storage_cs, rec)) { LOG(ERROR) << "failed to unpack AccountStorage"; @@ -3205,18 +3155,20 @@ static td::Ref storage_without_extra_currencies(td::Ref return {}; } } - td::Ref cell; - if (!block::gen::pack_cell(cell, rec)) { + td::Ref result; + if (!block::gen::csr_pack(result, rec)) { LOG(ERROR) << "failed to pack AccountStorage"; return {}; } - return cell; + return result; } namespace transaction { /** * Computes the new state of the account. * + * @param cfg The configuration for the serialization phase. + * * @returns True if the state computation is successful, false otherwise. */ bool Transaction::compute_state(const SerializeConfig& cfg) { @@ -3290,45 +3242,92 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } else { new_inner_state.clear(); } - vm::CellStorageStat& stats = new_storage_stat; + td::Ref old_storage_for_stat = account.storage; - td::Ref new_storage_for_stat = storage; + td::Ref new_storage_for_stat = new_storage; if (cfg.extra_currency_v2) { new_storage_for_stat = storage_without_extra_currencies(new_storage); if (new_storage_for_stat.is_null()) { return false; } if (old_storage_for_stat.not_null()) { - old_storage_for_stat = vm::load_cell_slice_ref(storage_without_extra_currencies(old_storage_for_stat)); + old_storage_for_stat = storage_without_extra_currencies(old_storage_for_stat); if (old_storage_for_stat.is_null()) { return false; } } + } else if (cfg.store_storage_dict_hash) { + LOG(ERROR) << "unsupported store_storage_dict_hash=true, extra_currency_v2=false"; + return false; } - auto new_stats = try_update_storage_stat(account.storage_stat, old_storage_for_stat, storage); - if (new_stats) { - stats = new_stats.unwrap(); + + bool storage_refs_changed = false; + if (old_storage_for_stat.is_null() || new_storage_for_stat->size_refs() != old_storage_for_stat->size_refs()) { + storage_refs_changed = true; } else { + for (unsigned i = 0; i < new_storage_for_stat->size_refs(); i++) { + if (new_storage_for_stat->prefetch_ref(i)->get_hash() != old_storage_for_stat->prefetch_ref(i)->get_hash()) { + storage_refs_changed = true; + break; + } + } + } + + bool store_storage_dict_hash = cfg.store_storage_dict_hash && !account.is_masterchain(); + if (storage_refs_changed || + (store_storage_dict_hash && !account.storage_dict_hash && account.storage_used.cells > 25)) { TD_PERF_COUNTER(transaction_storage_stat_b); td::Timer timer; - stats.add_used_storage(new_storage_for_stat).ensure(); + if (!new_account_storage_stat && account.account_storage_stat) { + new_account_storage_stat = AccountStorageStat(&account.account_storage_stat.value()); + } + AccountStorageStat& stats = new_account_storage_stat.value_force(); + // Don't check Merkle depth and size here - they were checked in check_state_limits + td::Status S = stats.replace_roots(new_storage_for_stat->prefetch_all_refs()); + if (S.is_error()) { + LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error(); + return false; + } + // Root of AccountStorage is not counted in AccountStorageStat + new_storage_used.cells = stats.get_total_cells() + 1; + new_storage_used.bits = stats.get_total_bits() + new_storage_for_stat->size(); + // TODO: think about this limit (25) + if (store_storage_dict_hash && new_storage_used.cells > 25) { + auto r_hash = stats.get_dict_hash(); + if (r_hash.is_error()) { + LOG(ERROR) << "Cannot compute storage dict hash for account " << account.addr.to_hex() << ": " + << r_hash.move_as_error(); + return false; + } + new_storage_dict_hash = r_hash.move_as_ok(); + } if (timer.elapsed() > 0.1) { - LOG(INFO) << "Compute used storage took " << timer.elapsed() << "s"; + LOG(INFO) << "Compute used storage (2) took " << timer.elapsed() << "s"; } - } - CHECK(cb.store_long_bool(1, 1) // account$1 - && cb.append_cellslice_bool(account.my_addr) // addr:MsgAddressInt - && block::store_UInt7(cb, stats.cells) // storage_used$_ cells:(VarUInteger 7) - && block::store_UInt7(cb, stats.bits) // bits:(VarUInteger 7) - && block::store_UInt7(cb, stats.public_cells) // public_cells:(VarUInteger 7) - && cb.store_long_bool(last_paid, 32)); // last_paid:uint32 + } else { + new_storage_used = account.storage_used; + new_storage_used.bits -= old_storage_for_stat->size(); + new_storage_used.bits += new_storage_for_stat->size(); + new_account_storage_stat = {}; + if (store_storage_dict_hash) { + new_storage_dict_hash = account.storage_dict_hash; + } + } + + CHECK(cb.store_long_bool(1, 1) // account$1 + && cb.append_cellslice_bool(account.my_addr) // addr:MsgAddressInt + && block::store_UInt7(cb, new_storage_used.cells) // storage_used$_ cells:(VarUInteger 7) + && block::store_UInt7(cb, new_storage_used.bits) // bits:(VarUInteger 7) + && cb.store_long_bool(new_storage_dict_hash ? 1 : 0, 3) // extra:StorageExtraInfo + && (!new_storage_dict_hash || cb.store_bits_bool(new_storage_dict_hash.value())) // dict_hash:uint256 + && cb.store_long_bool(last_paid, 32)); // last_paid:uint32 if (due_payment.not_null() && td::sgn(due_payment) != 0) { CHECK(cb.store_long_bool(1, 1) && block::tlb::t_Grams.store_integer_ref(cb, due_payment)); // due_payment:(Maybe Grams) } else { CHECK(cb.store_long_bool(0, 1)); } - CHECK(cb.append_data_cell_bool(std::move(storage))); + CHECK(cb.append_cellslice_bool(new_storage)); new_total_state = cb.finalize(); if (verbosity > 2) { FLOG(INFO) { @@ -3345,6 +3344,8 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { * * Updates root. * + * @param cfg The configuration for the serialization. + * * @returns True if the serialization is successful, False otherwise. */ bool Transaction::serialize(const SerializeConfig& cfg) { @@ -3688,7 +3689,15 @@ Ref Transaction::commit(Account& acc) { acc.last_trans_end_lt_ = end_lt; acc.last_trans_hash_ = root->get_hash().bits(); acc.last_paid = last_paid; - acc.storage_stat = new_storage_stat; + acc.storage_used = new_storage_used; + if (new_account_storage_stat) { + if (acc.account_storage_stat) { + acc.account_storage_stat.value().apply_child_stat(std::move(new_account_storage_stat.value())); + } else { + acc.account_storage_stat = std::move(new_account_storage_stat); + } + } + acc.storage_dict_hash = new_storage_dict_hash; acc.storage = new_storage; acc.balance = std::move(balance); acc.due_payment = std::move(due_payment); @@ -3936,6 +3945,7 @@ td::Status FetchConfigParams::fetch_config_params( } { serialize_cfg->extra_currency_v2 = config.get_global_version() >= 10; + serialize_cfg->store_storage_dict_hash = config.get_global_version() >= 11; } { // fetch block_grams_created diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 8e612e6a5b..aa08719a8f 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "account-storage-stat.h" #include "common/refcnt.hpp" #include "common/refint.h" #include "vm/cells.h" @@ -179,6 +180,7 @@ struct ActionPhaseConfig { struct SerializeConfig { bool extra_currency_v2{false}; + bool store_storage_dict_hash{false}; }; struct CreditPhase { @@ -266,8 +268,12 @@ struct Account { ton::LogicalTime last_trans_lt_; ton::Bits256 last_trans_hash_; ton::LogicalTime block_lt; + ton::UnixTime last_paid; - vm::CellStorageStat storage_stat; + StorageUsed storage_used; + td::optional storage_dict_hash; + td::optional account_storage_stat; + block::CurrencyCollection balance; td::RefInt256 due_payment; Ref orig_total_state; // ^Account @@ -377,7 +383,9 @@ struct Transaction { std::unique_ptr compute_phase; std::unique_ptr action_phase; std::unique_ptr bounce_phase; - vm::CellStorageStat new_storage_stat; + StorageUsed new_storage_used; + td::optional new_account_storage_stat; + td::optional new_storage_dict_hash; bool gas_limit_overridden{false}; Transaction(const Account& _account, int ttype, ton::LogicalTime req_start_lt, ton::UnixTime _now, Ref _inmsg = {}); diff --git a/crypto/common/bitstring.cpp b/crypto/common/bitstring.cpp index 52e57c9a8a..3a6f331191 100644 --- a/crypto/common/bitstring.cpp +++ b/crypto/common/bitstring.cpp @@ -347,6 +347,9 @@ std::size_t bits_memscan_rev(ConstBitPtr bs, std::size_t bit_count, bool cmp_to) int bits_memcmp(const unsigned char* bs1, int bs1_offs, const unsigned char* bs2, int bs2_offs, std::size_t bit_count, std::size_t* same_upto) { if (!bit_count) { + if (same_upto) { + *same_upto = 0; + } return 0; } bs1 += (bs1_offs >> 3); diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 4f3fe5fcf2..be1de25ca5 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -113,21 +113,19 @@ class NewCellStorageStat { struct CellStorageStat { unsigned long long cells; unsigned long long bits; - unsigned long long public_cells; struct CellInfo { td::uint32 max_merkle_depth = 0; }; td::HashMap seen; - CellStorageStat() : cells(0), bits(0), public_cells(0) { + CellStorageStat() : cells(0), bits(0) { } - explicit CellStorageStat(unsigned long long limit_cells) - : cells(0), bits(0), public_cells(0), limit_cells(limit_cells) { + explicit CellStorageStat(unsigned long long limit_cells) : cells(0), bits(0), limit_cells(limit_cells) { } void clear_seen() { seen.clear(); } void clear() { - cells = bits = public_cells = 0; + cells = bits = 0; clear_limit(); clear_seen(); } diff --git a/crypto/vm/cells/CellSlice.cpp b/crypto/vm/cells/CellSlice.cpp index 9cd3e931aa..bea20f95d1 100644 --- a/crypto/vm/cells/CellSlice.cpp +++ b/crypto/vm/cells/CellSlice.cpp @@ -773,6 +773,14 @@ bool CellSlice::prefetch_maybe_ref(Ref& res) const { } } +std::vector> CellSlice::prefetch_all_refs() const { + std::vector> res(size_refs()); + for (unsigned i = 0; i < size_refs(); ++i) { + res[i] = prefetch_ref(i); + } + return res; +} + bool CellSlice::fetch_maybe_ref(Ref& res) { auto z = prefetch_ulong(1); if (!z) { diff --git a/crypto/vm/cells/CellSlice.h b/crypto/vm/cells/CellSlice.h index ecce30f5c5..7525272b59 100644 --- a/crypto/vm/cells/CellSlice.h +++ b/crypto/vm/cells/CellSlice.h @@ -190,6 +190,7 @@ class CellSlice : public td::CntObject { } bool fetch_maybe_ref(Ref& ref); bool prefetch_maybe_ref(Ref& ref) const; + std::vector> prefetch_all_refs() const; td::BitSlice fetch_bits(unsigned bits); td::BitSlice prefetch_bits(unsigned bits) const; td::Ref fetch_subslice(unsigned bits, unsigned refs = 0); diff --git a/crypto/vm/dict.cpp b/crypto/vm/dict.cpp index 41f9c3396f..e1e683fe36 100644 --- a/crypto/vm/dict.cpp +++ b/crypto/vm/dict.cpp @@ -1772,6 +1772,260 @@ void Dictionary::map(const simple_map_func_t& simple_map_func) { map(map_func); } +bool Dictionary::multiset(td::MutableSpan>> new_values) { + force_validate(); + auto cmp = [&](const std::pair>& a, + const std::pair>& b) { + return td::bitstring::bits_memcmp(a.first, b.first, key_bits) < 0; + }; + if (!std::is_sorted(new_values.begin(), new_values.end(), cmp)) { + std::sort(new_values.begin(), new_values.end(), cmp); + } + for (size_t i = 0; i + 1 < new_values.size(); ++i) { + if (td::bitstring::bits_memcmp(new_values[i].first, new_values[i + 1].first, key_bits) == 0) { + return false; + } + } + unsigned char key_buffer[max_key_bytes]; + try { + Ref root = dict_multiset(get_root_cell(), new_values, key_buffer, key_bits, key_bits, 0); + set_root_cell(std::move(root)); + return true; + } catch (CombineError) { + return false; + } +} + +static Ref dict_build(td::Span>> values, int total_key_len, + int prefix_len) { + if (values.empty()) { + return {}; + } + if (values.size() == 1) { + if (values[0].second.is_null()) { + throw CombineError{}; + } + CellBuilder cb; + append_dict_label(cb, values[0].first + prefix_len, total_key_len - prefix_len, total_key_len - prefix_len); + if (!cb.append_builder_bool(values[0].second)) { + throw VmError{Excno::cell_ov, "cannot store new value into a dictionary cell"}; + } + return cb.finalize(); + } + size_t common_prefix_len; + td::bitstring::bits_memcmp(values.front().first + prefix_len, values.back().first + prefix_len, + total_key_len - prefix_len, &common_prefix_len); + CHECK(prefix_len + common_prefix_len < total_key_len); + size_t idx = 0; + while (values[idx].first[prefix_len + common_prefix_len] == 0) { + ++idx; + } + Ref left = dict_build(values.substr(0, idx), total_key_len, prefix_len + common_prefix_len + 1); + Ref right = dict_build(values.substr(idx), total_key_len, prefix_len + common_prefix_len + 1); + CellBuilder cb; + append_dict_label(cb, values[0].first + prefix_len, common_prefix_len, total_key_len - prefix_len); + if (!(cb.store_ref_bool(std::move(left)) && cb.store_ref_bool(std::move(right)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); +} + +// Based on DictionaryFixed::dict_combine_with, but dict2 is replaced with a list of values, mode is false +Ref Dictionary::dict_multiset(Ref dict1, td::Span>> values2, + td::BitPtr key_buffer, int n, int total_key_len, int skip1) { + int prefix_len = total_key_len - n; + for (auto& [k, _] : values2) { + CHECK(td::bitstring::bits_memcmp(k, key_buffer - prefix_len, prefix_len) == 0); + } + if (dict1.is_null()) { + return dict_build(values2, total_key_len, prefix_len); + } + if (values2.empty()) { + assert(!skip1); + return dict1; + } + size_t common_prefix_len; + td::bitstring::bits_memcmp(values2.front().first + prefix_len, values2.back().first + prefix_len, + total_key_len - prefix_len, &common_prefix_len); + assert(prefix_len + common_prefix_len < total_key_len || values2.size() == 1); + // both dictionaries non-empty + // skip1: remove that much first bits from all keys in dictionary dict1 (its keys are actually n + skip1 bits long) + // resulting dictionary will have n-bit keys + LabelParser label1{dict1, n + skip1, LabelParser::chk_all}; + int l1 = label1.l_bits - skip1, l2 = (int)common_prefix_len; + assert(l1 >= 0 && l2 >= 0); + assert(!skip1 || label1.common_prefix_len(key_buffer - skip1, skip1) == skip1); + int c = label1.common_prefix_len(values2[0].first + prefix_len - skip1, skip1 + l2) - skip1; + label1.extract_label_to(key_buffer - skip1); + assert(c >= 0 && c <= l1 && c <= l2); + if (c < l1 && c < l2) { + // the two dictionaries have disjoint keys + CellBuilder cb; + append_dict_label(cb, key_buffer + c + 1, l1 - c - 1, n - c - 1); + if (!cell_builder_add_slice_bool(cb, *label1.remainder)) { + throw VmError{Excno::cell_ov, "cannot prune label of an old dictionary cell while merging dictionaries"}; + } + label1.remainder.clear(); + dict1 = cb.finalize(); + // cb.reset(); // included into finalize(); + // now dict1 has been "pruned" -- first skip1+c+1 bits removed from its root egde label + Ref dict2 = dict_build(values2, total_key_len, prefix_len + c + 1); + if (!values2[0].first[prefix_len + c]) { + std::swap(dict1, dict2); + } + // put dict1 into the left tree (with smaller labels), dict2 into the right tree + append_dict_label(cb, key_buffer, c, n); + if (!(cb.store_ref_bool(std::move(dict1)) && cb.store_ref_bool(std::move(dict2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + + auto combine_func = [&](CellBuilder& cb, const Ref& cb2) -> bool { + if (cb2.is_null()) { + return false; + } + cb.append_builder(*cb2); + return true; + }; + size_t idx = 0; + while (prefix_len + common_prefix_len < total_key_len && idx < values2.size() && + values2[idx].first[prefix_len + common_prefix_len] == 0) { + ++idx; + } + auto values2_left = values2.substr(0, idx); + auto values2_right = values2.substr(idx); + + if (c == l1 && c == l2) { + // funny enough, the non-skipped parts of labels of l1 and l2 match + CellBuilder cb; + append_dict_label(cb, key_buffer, c, n); + if (c == n) { + // our two dictionaries are in fact leafs with matching edge labels (keys) + if (!combine_func(cb, values2[0].second)) { + // alas, the two values did not combine, this key will be absent from resulting dictionary + return {}; + } + return cb.finalize(); + } + assert(c < n); + key_buffer += c + 1; + key_buffer[-1] = 0; + // combine left subtrees + auto c1 = dict_multiset(label1.remainder->prefetch_ref(0), values2_left, key_buffer, n - c - 1, total_key_len, 0); + key_buffer[-1] = 1; + // combine right subtrees + auto c2 = dict_multiset(label1.remainder->prefetch_ref(1), values2_right, key_buffer, n - c - 1, total_key_len, 0); + label1.remainder.clear(); + // c1 and c2 are merged left and right children of dict1 and dict2 + if (!c1.is_null() && !c2.is_null()) { + // both children non-empty, simply put them into the new node + if (!(cb.store_ref_bool(std::move(c1)) && cb.store_ref_bool(std::move(c2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + if (c1.is_null() && c2.is_null()) { + return {}; // both children empty, resulting dictionary also empty + } + // exactly one of c1 and c2 is non-empty, have to merge labels + bool sw = c1.is_null(); + key_buffer[-1] = sw; + if (sw) { + c1 = std::move(c2); + } + LabelParser label3{std::move(c1), n - c - 1, LabelParser::chk_all}; + label3.extract_label_to(key_buffer); + key_buffer -= c + 1; + // store combined label for the new edge + cb.reset(); + append_dict_label(cb, key_buffer, c + 1 + label3.l_bits, n); + // store payload + if (!cell_builder_add_slice_bool(cb, *label3.remainder)) { + throw VmError{Excno::cell_ov, "cannot change label of an old dictionary cell while merging edges"}; + } + return cb.finalize(); + } + if (c == l1) { + assert(c < l2); + dict1.clear(); + // children of root node of dict1 + auto c1 = label1.remainder->prefetch_ref(0); + auto c2 = label1.remainder->prefetch_ref(1); + label1.remainder.clear(); + // have to merge dict2 with one of the children of dict1 + td::bitstring::bits_memcpy(key_buffer, values2[0].first + prefix_len, l2); // dict2 has longer label, extract it + bool sw = key_buffer[c]; + if (!sw) { + // merge c1 with dict2 + c1 = dict_multiset(std::move(c1), values2, key_buffer + c + 1, n - c - 1, total_key_len, 0); + } else { + // merge c2 with dict2 + c2 = dict_multiset(std::move(c2), values2, key_buffer + c + 1, n - c - 1, total_key_len, 0); + } + if (!c1.is_null() && !c2.is_null()) { + CellBuilder cb; + append_dict_label(cb, key_buffer, c, n); + if (!(cb.store_ref_bool(std::move(c1)) && cb.store_ref_bool(std::move(c2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + // one of children is empty, have to merge root edges + key_buffer[c] = !sw; + if (!sw) { + std::swap(c1, c2); + } + assert(!c1.is_null() && c2.is_null()); + LabelParser label3{std::move(c1), n - c - 1, LabelParser::chk_all}; + label3.extract_label_to(key_buffer + c + 1); + CellBuilder cb; + append_dict_label(cb, key_buffer, c + 1 + label3.l_bits, n); + // store payload + if (!cell_builder_add_slice_bool(cb, *label3.remainder)) { + throw VmError{Excno::cell_ov, "cannot change label of an old dictionary cell while merging edges"}; + } + return cb.finalize(); + } else { + assert(c == l2 && c < l1); + // have to merge dict1 with one of the children of dict2 + bool sw = key_buffer[c]; + Ref c1, c2; + if (!sw) { + // merge dict1 with c1 + c1 = dict_multiset(std::move(dict1), values2_left, key_buffer + c + 1, n - c - 1, total_key_len, skip1 + c + 1); + c2 = dict_build(values2_right, total_key_len, prefix_len + l2 + 1); + } else { + // merge dict1 with c2 + c2 = dict_multiset(std::move(dict1), values2_right, key_buffer + c + 1, n - c - 1, total_key_len, skip1 + c + 1); + c1 = dict_build(values2_left, total_key_len, prefix_len + l2 + 1); + } + if (!c1.is_null() && !c2.is_null()) { + CellBuilder cb; + append_dict_label(cb, key_buffer, c, n); + if (!(cb.store_ref_bool(std::move(c1)) && cb.store_ref_bool(std::move(c2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + // one of children is empty, have to merge root edges + key_buffer[c] = !sw; + if (!sw) { + std::swap(c1, c2); + } + assert(!c1.is_null() && c2.is_null()); + LabelParser label3{std::move(c1), n - c - 1, LabelParser::chk_all}; + label3.extract_label_to(key_buffer + c + 1); + CellBuilder cb; + append_dict_label(cb, key_buffer, c + 1 + label3.l_bits, n); + // store payload + if (!cell_builder_add_slice_bool(cb, *label3.remainder)) { + throw VmError{Excno::cell_ov, "cannot change label of an old dictionary cell while merging edges"}; + } + return cb.finalize(); + } +} + // mode: +1 = forbid empty dict1 with non-empty dict2 // +2 = forbid empty dict2 with non-empty dict1 Ref DictionaryFixed::dict_combine_with(Ref dict1, Ref dict2, td::BitPtr key_buffer, int n, diff --git a/crypto/vm/dict.h b/crypto/vm/dict.h index c4044963f5..e22b909465 100644 --- a/crypto/vm/dict.h +++ b/crypto/vm/dict.h @@ -527,13 +527,15 @@ class Dictionary final : public DictionaryFixed { auto range(bool rev = false, bool sgnd = false) { return dict_range(*this, rev, sgnd); } + bool multiset(td::MutableSpan>> new_values); private: bool check_fork(CellSlice& cs, Ref c1, Ref c2, int n) const override { return cs.empty_ext(); } static Ref extract_value_ref(Ref cs); - std::pair, int> dict_filter(Ref dict, td::BitPtr key, int n, const filter_func_t& check_leaf) const; + static Ref dict_multiset(Ref dict1, td::Span>> values2, + td::BitPtr key_buffer, int n, int total_key_len, int skip1); }; class PrefixDictionary final : public DictionaryBase { diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 77963e9594..c0be0b108a 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -156,3 +156,10 @@ Example: if the last masterchain block seqno is `19071` then the list contains b ### TVM changes - `SENDMSG` calculates messages size and fees without extra currencies, uses new +64 and +128 mode behavior. - `SENDMSG` does not check the number of extra currencies. + +## Version 11 +### New account storage stat +Along with the storage stat (cells and bits count), each account now stores the hash of the **storage dict**. + +**Storage dict** is the dictionary that stores refcnt for each cell in the account state. +This is required to help computing storage stats in the future, after collator-validator separation. \ No newline at end of file diff --git a/emulator/emulator-extern.cpp b/emulator/emulator-extern.cpp index eb5ff9f9e5..c5f3c04e00 100644 --- a/emulator/emulator-extern.cpp +++ b/emulator/emulator-extern.cpp @@ -187,7 +187,7 @@ const char *transaction_emulator_emulate_transaction(void *transaction_emulator, external_not_accepted->elapsed_time); } - auto emulation_success = dynamic_cast(*emulation_result); + auto emulation_success = std::move(dynamic_cast(*emulation_result)); auto trans_boc_b64 = cell_to_boc_b64(std::move(emulation_success.transaction)); if (trans_boc_b64.is_error()) { ERROR_RESPONSE(PSTRING() << "Can't serialize Transaction to boc " << trans_boc_b64.move_as_error()); @@ -260,7 +260,8 @@ const char *transaction_emulator_emulate_tick_tock_transaction(void *transaction } auto emulation_result = result.move_as_ok(); - auto emulation_success = dynamic_cast(*emulation_result); + auto emulation_success = + std::move(dynamic_cast(*emulation_result)); auto trans_boc_b64 = cell_to_boc_b64(std::move(emulation_success.transaction)); if (trans_boc_b64.is_error()) { ERROR_RESPONSE(PSTRING() << "Can't serialize Transaction to boc " << trans_boc_b64.move_as_error()); diff --git a/emulator/transaction-emulator.cpp b/emulator/transaction-emulator.cpp index 6267f9bd0b..abdf0d2cfa 100644 --- a/emulator/transaction-emulator.cpp +++ b/emulator/transaction-emulator.cpp @@ -141,7 +141,7 @@ td::Result TransactionEmulator::emulate_t return td::Status::Error("account hash mismatch"); } - return emulation_result; + return std::move(emulation_result); } else if (auto emulation_not_accepted_ptr = dynamic_cast(emulation.get())) { return td::Status::Error( PSTRING() diff --git a/emulator/transaction-emulator.h b/emulator/transaction-emulator.h index eae109f40a..848548c32b 100644 --- a/emulator/transaction-emulator.h +++ b/emulator/transaction-emulator.h @@ -33,14 +33,18 @@ class TransactionEmulator { virtual ~EmulationResult() = default; }; - struct EmulationSuccess: EmulationResult { + struct EmulationSuccess : EmulationResult { td::Ref transaction; block::Account account; td::Ref actions; - EmulationSuccess(td::Ref transaction_, block::Account account_, std::string vm_log_, td::Ref actions_, double elapsed_time_) : - EmulationResult(vm_log_, elapsed_time_), transaction(transaction_), account(account_) , actions(actions_) - {} + EmulationSuccess(td::Ref transaction_, block::Account account_, std::string vm_log_, + td::Ref actions_, double elapsed_time_) + : EmulationResult(vm_log_, elapsed_time_) + , transaction(transaction_) + , account(std::move(account_)) + , actions(actions_) { + } }; struct EmulationExternalNotAccepted: EmulationResult { diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 00dcf22dba..26a98408d1 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -180,7 +180,7 @@ struct RawAccountState { td::Ref extra_currencies; ton::UnixTime storage_last_paid{0}; - vm::CellStorageStat storage_stat; + block::StorageUsed storage_used; td::Ref code; td::Ref data; @@ -1036,7 +1036,7 @@ class Query { TRY_RESULT(basechain_msg_prices, cfg->get_msg_prices(false)); block::MsgPrices* msg_prices[2] = {&basechain_msg_prices, &masterchain_msg_prices}; auto storage_fee_256 = block::StoragePrices::compute_storage_fees( - raw_.source->get_sync_time(), storage_prices, raw_.source->raw().storage_stat, + raw_.source->get_sync_time(), storage_prices, raw_.source->raw().storage_used, raw_.source->raw().storage_last_paid, false, is_masterchain); auto storage_fee = storage_fee_256.is_null() ? 0 : storage_fee_256->to_long(); @@ -1085,7 +1085,7 @@ class Query { TRY_RESULT(dest_gas_limits_prices, cfg->get_gas_limits_prices(dest_is_masterchain)); auto dest_storage_fee_256 = destination ? block::StoragePrices::compute_storage_fees( - destination->get_sync_time(), storage_prices, destination->raw().storage_stat, + destination->get_sync_time(), storage_prices, destination->raw().storage_used, destination->raw().storage_last_paid, false, is_masterchain) : td::make_refint(0); Fee dst_fee; @@ -1399,17 +1399,16 @@ class GetRawAccountState : public td::actor::Actor { return td::Status::Error("Failed to unpack StorageInfo"); } unsigned long long u = 0; - vm::CellStorageStat storage_stat; + block::StorageUsed storage_stat; u |= storage_stat.cells = block::tlb::t_VarUInteger_7.as_uint(*storage_used.cells); u |= storage_stat.bits = block::tlb::t_VarUInteger_7.as_uint(*storage_used.bits); - u |= storage_stat.public_cells = block::tlb::t_VarUInteger_7.as_uint(*storage_used.public_cells); //LOG(DEBUG) << "last_paid=" << res.storage_last_paid << "; cells=" << storage_stat.cells - //<< " bits=" << storage_stat.bits << " public_cells=" << storage_stat.public_cells; + //<< " bits=" << storage_stat.bits; if (u == std::numeric_limits::max()) { return td::Status::Error("Failed to unpack StorageStat"); } - res.storage_stat = storage_stat; + res.storage_used = storage_stat; } block::gen::AccountStorage::Record storage; @@ -2089,7 +2088,7 @@ class RunEmulator : public TonlibQueryActor { raw.balance = balance.grams->to_long(); raw.extra_currencies = balance.extra; raw.storage_last_paid = std::move(account.last_paid); - raw.storage_stat = std::move(account.storage_stat); + raw.storage_used = account.storage_used; raw.code = std::move(account.code); raw.data = std::move(account.data); raw.state = std::move(account.total_state); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 90966d8203..5f2ba66151 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1008,6 +1008,7 @@ bool ValidateQuery::fetch_config_params() { } { serialize_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; + serialize_cfg_.store_storage_dict_hash = config_->get_global_version() >= 11; } { // fetch block_grams_created From 56506aeab2bb6ea4d707efa8fd10a54f7a53a27f Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 14 Apr 2025 10:10:39 +0300 Subject: [PATCH 214/524] Fix finish_getDispatchQueueMessages with messages boc mask (#1623) Co-authored-by: ice-charon --- validator/impl/liteserver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 06f40b8fda..8457d234fa 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -3672,7 +3672,7 @@ void LiteQuery::finish_getDispatchQueueMessages(StdSmcAddress addr, LogicalTime fatal_error(r_messages_boc.move_as_error()); return; } - messages_boc = std::move(messages_boc); + messages_boc = r_messages_boc.move_as_ok(); } LOG(INFO) << "getDispatchQueueMessages(" << blk_id_.to_str() << ", " << mode_ << ") query completed"; auto b = ton::create_serialize_tl_object( From 86e6726e4125c2fcec7ca7ea3f3be6143b09524c Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 14 Apr 2025 10:11:14 +0300 Subject: [PATCH 215/524] Report error to correct promise in tonlib dns request (#1622) Co-authored-by: ice-charon --- tonlib/tonlib/TonlibClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 26a98408d1..4ddcbbfabb 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -5043,7 +5043,7 @@ void TonlibClient::do_dns_request(std::string name, td::Bits256 category, td::in return; } - TRY_RESULT_PROMISE(promise, args, ton::DnsInterface::resolve_args(name, category, address)); + TRY_RESULT_PROMISE(new_promise, args, ton::DnsInterface::resolve_args(name, category, address)); int_api::RemoteRunSmcMethod query; query.address = std::move(address); query.args = std::move(args); From 92f865b94f77ab8d2bc286cfdcbdfaa2b2ad3771 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Fri, 21 Mar 2025 08:22:12 -0400 Subject: [PATCH 216/524] Use check_for_each for iterating over validator set This is negligible but still positive (~0.5%) improvement. --- crypto/block/mc-config.cpp | 40 ++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 7b4f326f8d..d5aa65d5e1 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -576,34 +576,50 @@ td::Result> Config::unpack_validator_set(Ref(rec.utime_since, rec.utime_until, rec.total, rec.main); - for (int i = 0; i < rec.total; i++) { - key_buffer.store_ulong(i); - auto descr_cs = dict.lookup(key_buffer.bits(), 16); - if (descr_cs.is_null()) { - return td::Status::Error("indices in a validator set dictionary must be integers 0..total-1"); - } + + std::vector seen_keys(rec.total); + td::Status error; + + auto validator_set_check_fn = [&](Ref descr_cs, td::ConstBitPtr key, int n) -> bool { + int i = static_cast(key.get_uint(n)); + CHECK(i >= 0 && i < rec.total && !seen_keys[i]); + seen_keys[i] = true; + gen::ValidatorDescr::Record_validator_addr descr; if (!tlb::csr_unpack(descr_cs, descr)) { descr.adnl_addr.set_zero(); if (!(gen::t_ValidatorDescr.unpack_validator(descr_cs.write(), descr.public_key, descr.weight) && descr_cs->empty_ext())) { - return td::Status::Error(PSLICE() << "validator #" << i - << " has an invalid ValidatorDescr record in the validator set dictionary"); + error = td::Status::Error(PSLICE() << "validator #" << i + << " has an invalid ValidatorDescr record in the validator set dictionary"); + return false; } } gen::SigPubKey::Record sig_pubkey; if (!tlb::csr_unpack(std::move(descr.public_key), sig_pubkey)) { - return td::Status::Error(PSLICE() << "validator #" << i - << " has no public key or its public key is in unsupported format"); + error = td::Status::Error(PSLICE() << "validator #" << i + << " has no public key or its public key is in unsupported format"); + return false; } if (!descr.weight) { - return td::Status::Error(PSLICE() << "validator #" << i << " has zero weight"); + error = td::Status::Error(PSLICE() << "validator #" << i << " has zero weight"); + return false; } if (descr.weight > ~(ptr->total_weight)) { - return td::Status::Error("total weight of all validators in validator set exceeds 2^64"); + error = td::Status::Error("total weight of all validators in validator set exceeds 2^64"); + return false; } ptr->list.emplace_back(sig_pubkey.pubkey, descr.weight, ptr->total_weight, descr.adnl_addr); ptr->total_weight += descr.weight; + return true; + }; + + if (!dict.check_for_each(validator_set_check_fn)) { + CHECK(error.is_error()); + return error; + } + if (std::find(seen_keys.begin(), seen_keys.end(), false) != seen_keys.end()) { + return td::Status::Error("indices in a validator set dictionary must be integers 0..total-1"); } if (rec.total_weight && rec.total_weight != ptr->total_weight) { return td::Status::Error("validator set declares incorrect total weight"); From 2c5e8ea71d3fe78c51a349a329885a354a4fbaec Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 16 Apr 2025 20:58:18 +0300 Subject: [PATCH 217/524] Fix SDBEGINS(Q) in Asm.fif (#1626) --- crypto/fift/lib/Asm.fif | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index 976093f809..4d7e2b6fe5 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -2,7 +2,7 @@ library TVM_Asm // simple TVM Assembler namespace Asm Asm definitions -"0.4.5" constant asm-fif-version +"0.4.6" constant asm-fif-version variable @atend variable @was-split @@ -750,26 +750,37 @@ x{D723} @Defop SDSKIPLAST x{D724} @Defop SDSUBSTR x{D726} @Defop SDBEGINSX x{D727} @Defop SDBEGINSXQ + +/* +if (rembits >= opcodebits) or (rembits <= 18) + just write +else + PUSHSLICE SDBEGINSXQ + +--- +18 - size of PUSHREFSLICE +*/ + { tuck sbits tuck 5 + 3 >> swap x{D72A_} s, over 7 u, 3 roll s, -rot 3 << 3 + swap - @scomplete } : SDBEGINS:imm -{ tuck sbitrefs abort"no references allowed in slice" dup 26 <= - { drop > 8 * 3 + 21 + // slice builder opcodebits ((sbits + 5) // 8) * 8 + 3 + 21 + over brembits <= // total op bits <= remaining free bits in slice + { > swap x{D72E_} s, over 7 u, 3 roll s, -rot 3 << 3 + swap - @scomplete } : SDBEGINSQ:imm -{ tuck sbitrefs abort"no references allowed in slice" dup 26 <= - { drop > 8 * 3 + 21 + // slice builder opcodebits + over brembits <= // total op bits <= remaining free bits in slice + { Date: Thu, 17 Apr 2025 12:54:16 +0300 Subject: [PATCH 218/524] Process virtualization in cell load callback --- crypto/vm/boc.cpp | 8 +++++--- crypto/vm/boc.h | 2 +- crypto/vm/cells/Cell.h | 12 +++++++----- crypto/vm/cells/CellUsageTree.cpp | 9 +++++---- crypto/vm/cells/CellUsageTree.h | 9 +++++---- crypto/vm/cells/MerkleProof.h | 2 +- crypto/vm/cells/UsageCell.h | 2 +- validator/impl/collator-impl.h | 2 +- validator/impl/collator.cpp | 14 +++++++------- 9 files changed, 33 insertions(+), 27 deletions(-) diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index cfe37c662b..d3ef57b742 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -1254,16 +1254,18 @@ bool VmStorageStat::add_storage(const CellSlice& cs) { return true; } -void ProofStorageStat::add_loaded_cell(const Ref& cell) { - auto& [status, size] = cells_[cell->get_hash()]; +void ProofStorageStat::add_loaded_cell(const Ref& cell, td::uint8 max_level) { + auto& [status, size] = cells_[cell->get_hash(max_level)]; if (status == c_loaded) { return; } proof_size_ -= size; status = c_loaded; proof_size_ += size = estimate_serialized_size(cell); + max_level += (cell->special_type() == CellTraits::SpecialType::MerkleProof || + cell->special_type() == CellTraits::SpecialType::MerkleUpdate); for (unsigned i = 0; i < cell->size_refs(); ++i) { - auto& [child_status, child_size] = cells_[cell->get_ref(i)->get_hash()]; + auto& [child_status, child_size] = cells_[cell->get_ref(i)->get_hash(max_level)]; if (child_status == c_none) { child_status = c_prunned; proof_size_ += child_size = estimate_prunned_size(); diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 7e718ce26a..fbf395538d 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -165,7 +165,7 @@ struct VmStorageStat { class ProofStorageStat { public: - void add_loaded_cell(const Ref& cell); + void add_loaded_cell(const Ref& cell, td::uint8 max_level = Cell::max_level); void add_loaded_cells(const ProofStorageStat& other); td::uint64 estimate_proof_size() const; diff --git a/crypto/vm/cells/Cell.h b/crypto/vm/cells/Cell.h index e2b47ffc01..25b71fcb24 100644 --- a/crypto/vm/cells/Cell.h +++ b/crypto/vm/cells/Cell.h @@ -35,15 +35,17 @@ namespace vm { using td::Ref; class DataCell; +struct LoadedCell { + Ref data_cell; + detail::VirtualizationParameters virt; + CellUsageTree::NodePtr tree_node; // TODO: inline_vector? +}; + class Cell : public CellTraits { public: using LevelMask = detail::LevelMask; using VirtualizationParameters = detail::VirtualizationParameters; - struct LoadedCell { - Ref data_cell; - VirtualizationParameters virt; - CellUsageTree::NodePtr tree_node; // TODO: inline_vector? - }; + using LoadedCell = vm::LoadedCell; using Hash = CellHash; static_assert(std::is_standard_layout::value, "Cell::Hash is not a standard layout type"); diff --git a/crypto/vm/cells/CellUsageTree.cpp b/crypto/vm/cells/CellUsageTree.cpp index 3874998c2c..301ee7da49 100644 --- a/crypto/vm/cells/CellUsageTree.cpp +++ b/crypto/vm/cells/CellUsageTree.cpp @@ -17,17 +17,18 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "vm/cells/CellUsageTree.h" +#include "DataCell.h" namespace vm { // // CellUsageTree::NodePtr // -bool CellUsageTree::NodePtr::on_load(const td::Ref& cell) const { +bool CellUsageTree::NodePtr::on_load(const Cell::LoadedCell& loaded_cell) const { auto tree = tree_weak_.lock(); if (!tree) { return false; } - tree->on_load(node_id_, cell); + tree->on_load(node_id_, loaded_cell); return true; } @@ -111,13 +112,13 @@ void CellUsageTree::set_use_mark_for_is_loaded(bool use_mark) { use_mark_ = use_mark; } -void CellUsageTree::on_load(NodeId node_id, const td::Ref& cell) { +void CellUsageTree::on_load(NodeId node_id, const Cell::LoadedCell& loaded_cell) { if (ignore_loads_ || nodes_[node_id].is_loaded) { return; } nodes_[node_id].is_loaded = true; if (cell_load_callback_) { - cell_load_callback_(cell); + cell_load_callback_(loaded_cell); } } diff --git a/crypto/vm/cells/CellUsageTree.h b/crypto/vm/cells/CellUsageTree.h index d845be10d7..902b98d751 100644 --- a/crypto/vm/cells/CellUsageTree.h +++ b/crypto/vm/cells/CellUsageTree.h @@ -27,6 +27,7 @@ namespace vm { class DataCell; +struct LoadedCell; class CellUsageTree : public std::enable_shared_from_this { public: @@ -42,7 +43,7 @@ class CellUsageTree : public std::enable_shared_from_this { return node_id_ == 0 || tree_weak_.expired(); } - bool on_load(const td::Ref& cell) const; + bool on_load(const LoadedCell& loaded_cell) const; NodePtr create_child(unsigned ref_id) const; bool mark_path(CellUsageTree* master_tree) const; bool is_from_tree(const CellUsageTree* master_tree) const; @@ -63,7 +64,7 @@ class CellUsageTree : public std::enable_shared_from_this { void set_use_mark_for_is_loaded(bool use_mark = true); NodeId create_child(NodeId node_id, unsigned ref_id); - void set_cell_load_callback(std::function&)> f) { + void set_cell_load_callback(std::function f) { cell_load_callback_ = std::move(f); } void set_ignore_loads(bool value) { @@ -83,9 +84,9 @@ class CellUsageTree : public std::enable_shared_from_this { }; bool use_mark_{false}; std::vector nodes_{2}; - std::function&)> cell_load_callback_; + std::function cell_load_callback_; - void on_load(NodeId node_id, const td::Ref& cell); + void on_load(NodeId node_id, const LoadedCell& loaded_cell); NodeId create_node(NodeId parent); int ignore_loads_ = 0; }; diff --git a/crypto/vm/cells/MerkleProof.h b/crypto/vm/cells/MerkleProof.h index 96a31d64e7..0a56be050a 100644 --- a/crypto/vm/cells/MerkleProof.h +++ b/crypto/vm/cells/MerkleProof.h @@ -70,7 +70,7 @@ class MerkleProofBuilder { bool extract_proof_to(Ref &proof_root) const; td::Result extract_proof_boc() const; - void set_cell_load_callback(std::function&)> f) { + void set_cell_load_callback(std::function f) { usage_tree->set_cell_load_callback(std::move(f)); } }; diff --git a/crypto/vm/cells/UsageCell.h b/crypto/vm/cells/UsageCell.h index 978b91f763..c634bc9090 100644 --- a/crypto/vm/cells/UsageCell.h +++ b/crypto/vm/cells/UsageCell.h @@ -42,7 +42,7 @@ class UsageCell : public Cell { // load interface td::Result load_cell() const override { TRY_RESULT(loaded_cell, cell_->load_cell()); - if (tree_node_.on_load(loaded_cell.data_cell)) { + if (tree_node_.on_load(loaded_cell)) { CHECK(loaded_cell.tree_node.empty()); loaded_cell.tree_node = tree_node_; } diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 6e5f876eeb..405fad3346 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -408,7 +408,7 @@ class Collator final : public td::actor::Actor { AccountStorageDict* current_tx_storage_dict_ = nullptr; - void on_cell_loaded(const Ref& cell); + void on_cell_loaded(const vm::LoadedCell& cell); void set_current_tx_storage_dict(const block::Account& account); }; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index e01031f9ca..6e7a5b87aa 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -944,7 +944,7 @@ void Collator::got_neighbor_msg_queue(unsigned i, Ref res) { neighbor_proof_builders_.push_back(vm::MerkleProofBuilder{res->state_root_}); state_root = neighbor_proof_builders_.back().root(); if (full_collated_data_ && !block_id.is_masterchain()) { - neighbor_proof_builders_.back().set_cell_load_callback([&](const Ref& cell) { + neighbor_proof_builders_.back().set_cell_load_callback([&](const vm::LoadedCell& cell) { on_cell_loaded(cell); }); } @@ -1054,7 +1054,7 @@ bool Collator::unpack_merge_last_state() { // 1. prepare for creating a MerkleUpdate based on previous state state_usage_tree_ = std::make_shared(); if (full_collated_data_ && !is_masterchain()) { - state_usage_tree_->set_cell_load_callback([&](const Ref& cell) { + state_usage_tree_->set_cell_load_callback([&](const vm::LoadedCell& cell) { on_cell_loaded(cell); }); } @@ -1103,7 +1103,7 @@ bool Collator::unpack_last_state() { // prepare for creating a MerkleUpdate based on previous state state_usage_tree_ = std::make_shared(); if (full_collated_data_ && !is_masterchain()) { - state_usage_tree_->set_cell_load_callback([&](const Ref& cell) { + state_usage_tree_->set_cell_load_callback([&](const vm::LoadedCell& cell) { on_cell_loaded(cell); }); } @@ -2648,7 +2648,7 @@ bool Collator::init_account_storage_dict(block::Account& account) { << ": dict is empty"); } dict.mpb = vm::MerkleProofBuilder(res.move_as_ok()); - dict.mpb.set_cell_load_callback([&](const Ref& cell) { + dict.mpb.set_cell_load_callback([&](const vm::LoadedCell& cell) { on_cell_loaded(cell); }); } @@ -6382,9 +6382,9 @@ void Collator::finalize_stats() { * When storage stat is calculated, StorageStatCalculationContext::calculating_storage_stat() returns true. * In this case, loaded cell is stored in a separate StorageStat for the storage dict of the account (if it exists). * - * @param cell Loaded cell + * @param loaded_cell Loaded cell */ -void Collator::on_cell_loaded(const Ref& cell) { +void Collator::on_cell_loaded(const vm::LoadedCell& loaded_cell) { auto context = block::StorageStatCalculationContext::get(); vm::ProofStorageStat* stat = (context && context->calculating_storage_stat() && current_tx_storage_dict_ ? ¤t_tx_storage_dict_->proof_stat @@ -6392,7 +6392,7 @@ void Collator::on_cell_loaded(const Ref& cell) { if (block_limit_status_) { block_limit_status_->collated_data_size_estimate -= stat->estimate_proof_size(); } - stat->add_loaded_cell(cell); + stat->add_loaded_cell(loaded_cell.data_cell, loaded_cell.virt.get_level()); if (block_limit_status_) { block_limit_status_->collated_data_size_estimate += stat->estimate_proof_size(); } From 5fe152621bf9a255a07391a7f4444a715815de4a Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sat, 19 Apr 2025 17:41:55 +0300 Subject: [PATCH 219/524] Version 10: extra currencies, anycasts (#1595) * Disable anycast address, allow deploying with "fixed prefix length" * GETEXTRABALANCE instruction, changes in reserve * Change exitcode for "too many extra currencies" * Revert end_lt and collect action fine if state limits are exceeded * Fix consuming gas in RUNVM * Fix checking message balance * Add "supported-version" and capability constants to fift * Add unpacked in msg params to TVM c7 * Remove excessive logs in celldb * Fix moving libraries and log in runvm * Move "in msg params" in c7 to version 11 * Add msg value (with extra) to INMSGPARAMS --- crypto/block/block-parse.cpp | 11 +- crypto/block/block.tlb | 15 +- crypto/block/create-state.cpp | 28 +-- crypto/block/mc-config.cpp | 1 + crypto/block/mc-config.h | 1 + crypto/block/transaction.cpp | 332 +++++++++++++++++-------- crypto/block/transaction.h | 38 +-- crypto/fift/lib/Asm.fif | 17 ++ crypto/fift/words.cpp | 5 + crypto/smartcont/CreateState.fif | 3 + crypto/smc-envelope/SmartContract.cpp | 6 +- crypto/test/fift.cpp | 4 + crypto/test/fift/get_extra_balance.fif | 80 ++++++ crypto/vm/tonops.cpp | 181 +++++++++++--- crypto/vm/tonops.h | 9 +- crypto/vm/vm.cpp | 30 ++- crypto/vm/vm.h | 17 +- doc/GlobalVersions.md | 60 ++++- test/regression-tests.ans | 1 + validator/db/celldb.cpp | 4 - validator/impl/liteserver.cpp | 5 +- validator/impl/validate-query.cpp | 3 + 22 files changed, 650 insertions(+), 201 deletions(-) create mode 100644 crypto/test/fift/get_extra_balance.fif diff --git a/crypto/block/block-parse.cpp b/crypto/block/block-parse.cpp index 6d645ac79d..71ec2bcf51 100644 --- a/crypto/block/block-parse.cpp +++ b/crypto/block/block-parse.cpp @@ -738,7 +738,7 @@ const TickTock t_TickTock; const RefAnything t_RefCell; bool StateInit::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { - return Maybe{5}.validate_skip(ops, cs, weak) // split_depth:(Maybe (## 5)) + return Maybe{5}.validate_skip(ops, cs, weak) // fixed_prefix_length:(Maybe (## 5)) && Maybe{}.validate_skip(ops, cs, weak) // special:(Maybe TickTock) && Maybe{}.validate_skip(ops, cs, weak) // code:(Maybe ^Cell) && Maybe{}.validate_skip(ops, cs, weak) // data:(Maybe ^Cell) @@ -1078,7 +1078,7 @@ bool Account::skip_copy_depth_balance(vm::CellBuilder& cb, vm::CellSlice& cs) co case account: return cs.advance(1) // account$1 && t_MsgAddressInt.skip_get_depth(cs, depth) // addr:MsgAddressInt - && cb.store_uint_leq(30, depth) // -> store split_depth:(#<= 30) + && cb.store_uint_leq(30, depth) // -> store fixed_prefix_length:(#<= 30) && t_StorageInfo.skip(cs) // storage_stat:StorageInfo && t_AccountStorage.skip_copy_balance(cb, cs); // storage:AccountStorage } @@ -1223,13 +1223,14 @@ bool HashmapAugE::extract_extra(vm::CellSlice& cs) const { bool DepthBalanceInfo::skip(vm::CellSlice& cs) const { return cs.advance(5) && t_CurrencyCollection.skip( - cs); // depth_balance$_ split_depth:(#<= 30) balance:CurrencyCollection = DepthBalanceInfo; + cs); // depth_balance$_ fixed_prefix_length:(#<= 30) balance:CurrencyCollection = DepthBalanceInfo; } bool DepthBalanceInfo::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return cs.fetch_ulong(5) <= 30 && - t_CurrencyCollection.validate_skip(ops, cs, - weak); // depth_balance$_ split_depth:(#<= 30) balance:CurrencyCollection + t_CurrencyCollection.validate_skip( + ops, cs, + weak); // depth_balance$_ fixed_prefix_length:(#<= 30) balance:CurrencyCollection } bool DepthBalanceInfo::null_value(vm::CellBuilder& cb) const { diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 5cba3c69f1..99dfdb5223 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -141,12 +141,12 @@ ext_out_msg_info$11 src:MsgAddress dest:MsgAddressExt tick_tock$_ tick:Bool tock:Bool = TickTock; -_ split_depth:(Maybe (## 5)) special:(Maybe TickTock) +_ fixed_prefix_length:(Maybe (## 5)) special:(Maybe TickTock) code:(Maybe ^Cell) data:(Maybe ^Cell) library:(Maybe ^Cell) = StateInit; // StateInitWithLibs is used to validate sent and received messages -_ split_depth:(Maybe (## 5)) special:(Maybe TickTock) +_ fixed_prefix_length:(Maybe (## 5)) special:(Maybe TickTock) code:(Maybe ^Cell) data:(Maybe ^Cell) library:(HashmapE 256 SimpleLib) = StateInitWithLibs; @@ -272,14 +272,6 @@ acc_state_frozen$01 = AccountStatus; acc_state_active$10 = AccountStatus; acc_state_nonexist$11 = AccountStatus; -/* duplicates -tick_tock$_ tick:Bool tock:Bool = TickTock; - -_ split_depth:(Maybe (## 5)) special:(Maybe TickTock) - code:(Maybe ^Cell) data:(Maybe ^Cell) - library:(Maybe ^Cell) = StateInit; -*/ - account_descr$_ account:^Account last_trans_hash:bits256 last_trans_lt:uint64 = ShardAccount; @@ -800,7 +792,8 @@ size_limits_config#01 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells max_ext_msg_size:uint32 max_ext_msg_depth:uint16 = SizeLimitsConfig; size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_acc_state_bits:uint32 - max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 max_msg_extra_currencies:uint32 = SizeLimitsConfig; + max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 max_msg_extra_currencies:uint32 + max_acc_fixed_prefix_length:uint8 = SizeLimitsConfig; _ SizeLimitsConfig = ConfigParam 43; // key is [ wc:int32 addr:uint256 ] diff --git a/crypto/block/create-state.cpp b/crypto/block/create-state.cpp index 4a74ac0f60..a32564a215 100644 --- a/crypto/block/create-state.cpp +++ b/crypto/block/create-state.cpp @@ -94,12 +94,12 @@ typedef td::BitArray<256> hash_t; struct SmcDescr { hash_t addr; - int split_depth; + int fixed_prefix_length; bool preinit_only; td::RefInt256 gram_balance; Ref state_init; // StateInit Ref account; // Account - SmcDescr(const hash_t& _addr) : addr(_addr), split_depth(0), preinit_only(false) { + SmcDescr(const hash_t& _addr) : addr(_addr), fixed_prefix_length(0), preinit_only(false) { } }; @@ -123,7 +123,7 @@ vm::Dictionary config_dict{32}; ton::UnixTime now; bool set_config_smc(const SmcDescr& smc) { - if (config_addr_set || smc.preinit_only || workchain_id != wc_master || smc.split_depth) { + if (config_addr_set || smc.preinit_only || workchain_id != wc_master || smc.fixed_prefix_length) { return false; } vm::CellSlice cs = load_cell_slice(smc.state_init); @@ -221,7 +221,7 @@ bool add_public_library(hash_t lib_addr, hash_t smc_addr, Ref lib_root } td::RefInt256 create_smartcontract(td::RefInt256 smc_addr, Ref code, Ref data, - Ref library, td::RefInt256 balance, int special, int split_depth, + Ref library, td::RefInt256 balance, int special, int fixed_prefix_length, int mode) { if (is_empty_cell(code)) { code.clear(); @@ -238,12 +238,12 @@ td::RefInt256 create_smartcontract(td::RefInt256 smc_addr, Ref code, R THRERR("not a valid library collection"); } vm::CellBuilder cb; - if (!split_depth) { + if (!fixed_prefix_length) { PDO(cb.store_long_bool(0, 1)); } else { - PDO(cb.store_long_bool(1, 1) && cb.store_ulong_rchk_bool(split_depth, 5)); + PDO(cb.store_long_bool(1, 1) && cb.store_ulong_rchk_bool(fixed_prefix_length, 5)); } - THRERR("invalid split_depth for a smart contract"); + THRERR("invalid fixed_prefix_length for a smart contract"); if (!special) { PDO(cb.store_long_bool(0, 1)); } else { @@ -287,7 +287,7 @@ td::RefInt256 create_smartcontract(td::RefInt256 smc_addr, Ref code, R auto ins = smart_contracts.emplace(addr, addr); assert(ins.second); SmcDescr& smc = ins.first->second; - smc.split_depth = split_depth; + smc.fixed_prefix_length = fixed_prefix_length; smc.preinit_only = (mode == 1); smc.gram_balance = balance; total_smc_balance += balance; @@ -328,10 +328,10 @@ td::RefInt256 create_smartcontract(td::RefInt256 smc_addr, Ref code, R ctor = 2; // addr_std$10 } PDO(cb.store_long_bool(ctor, 2)); // addr_std$10 or addr_var$11 - if (split_depth) { + if (fixed_prefix_length) { PDO(cb.store_long_bool(1, 1) // just$1 - && cb.store_ulong_rchk_bool(split_depth, 5) // depth:(## 5) - && cb.store_bits_bool(addr.cbits(), split_depth)); // rewrite pfx:(depth * Bit) + && cb.store_ulong_rchk_bool(fixed_prefix_length, 5) // depth:(## 5) + && cb.store_bits_bool(addr.cbits(), fixed_prefix_length)); // rewrite pfx:(depth * Bit) } else { PDO(cb.store_long_bool(0, 1)); // nothing$0 } @@ -515,7 +515,7 @@ Ref create_state() { // data (cell) // library (cell) // balance (int) -// split_depth (int 0..32) +// fixed_prefix_length (int 0..32) // special (int 0..3, +2 = tick, +1 = tock) // [ address (uint256) ] // mode (0 = compute address only, 1 = create uninit, 2 = create complete; +4 = with specified address) @@ -537,7 +537,7 @@ void interpret_register_smartcontract(vm::Stack& stack) { if (special && workchain_id != wc_master) { throw fift::IntError{"cannot create special smartcontracts outside of the masterchain"}; } - int split_depth = stack.pop_smallint_range(32); + int fixed_prefix_length = stack.pop_smallint_range(32); td::RefInt256 balance = stack.pop_int_finite(); if (sgn(balance) < 0) { throw fift::IntError{"initial balance of a smartcontract cannot be negative"}; @@ -549,7 +549,7 @@ void interpret_register_smartcontract(vm::Stack& stack) { Ref data = stack.pop_cell(); Ref code = stack.pop_cell(); td::RefInt256 addr = create_smartcontract(std::move(spec_addr), std::move(code), std::move(data), std::move(library), - std::move(balance), special, split_depth, mode); + std::move(balance), special, fixed_prefix_length, mode); if (addr.is_null()) { throw fift::IntError{"internal error while creating smartcontract"}; } diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index d5aa65d5e1..d51f20599f 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2040,6 +2040,7 @@ td::Result Config::do_get_size_limits_config(td::Ref 30) { - return false; // invalid value for split_depth +bool Account::set_addr_rewrite_length(int new_length) { + if (new_length < 0 || new_length > 30) { + return false; // invalid value } - if (split_depth_set_) { - return split_depth_ == new_split_depth; + if (addr_rewrite_length_set) { + return addr_rewrite_length == new_length; } else { - split_depth_ = (unsigned char)new_split_depth; - split_depth_set_ = true; + addr_rewrite_length = (unsigned char)new_length; + addr_rewrite_length_set = true; return true; } } /** - * Checks if the given split depth is valid for the Account. + * Checks if the given addr rewrite length is valid for the Account. * - * @param split_depth The split depth to be checked. + * @param length The addr rewrite length to be checked. * - * @returns True if the split depth is valid, False otherwise. + * @returns True if the addr rewrite length is valid, False otherwise. */ -bool Account::check_split_depth(int split_depth) const { - return split_depth_set_ ? (split_depth == split_depth_) : (split_depth >= 0 && split_depth <= 30); +bool Account::check_addr_rewrite_length(int length) const { + return addr_rewrite_length_set ? (length == addr_rewrite_length) : (length >= 0 && length <= 30); } /** * Parses anycast data of the account address. * - * Initializes split_depth and addr_rewrite. + * Initializes addr_rewrite. * - * @param cs The cell slice containing partially-parsed account addressa. + * @param cs The cell slice containing partially-parsed account address. * * @returns True if parsing was successful, false otherwise. */ @@ -159,13 +159,13 @@ bool Account::parse_maybe_anycast(vm::CellSlice& cs) { if (t < 0) { return false; } else if (!t) { - return set_split_depth(0); + return set_addr_rewrite_length(0); } int depth; return cs.fetch_uint_leq(30, depth) // anycast_info$_ depth:(#<= 30) && depth // { depth >= 1 } && cs.fetch_bits_to(addr_rewrite.bits(), depth) // rewrite_pfx:(bits depth) - && set_split_depth(depth); + && set_addr_rewrite_length(depth); } /** @@ -176,12 +176,12 @@ bool Account::parse_maybe_anycast(vm::CellSlice& cs) { * @returns True if the anycast information was successfully stored, false otherwise. */ bool Account::store_maybe_anycast(vm::CellBuilder& cb) const { - if (!split_depth_set_ || !split_depth_) { + if (!addr_rewrite_length_set || !addr_rewrite_length) { return cb.store_bool_bool(false); } return cb.store_bool_bool(true) // just$1 - && cb.store_uint_leq(30, split_depth_) // depth:(#<= 30) - && cb.store_bits_bool(addr_rewrite.cbits(), split_depth_); // rewrite_pfx:(bits depth) + && cb.store_uint_leq(30, addr_rewrite_length) // depth:(#<= 30) + && cb.store_bits_bool(addr_rewrite.cbits(), addr_rewrite_length); // rewrite_pfx:(bits depth) } /** @@ -214,14 +214,14 @@ bool Account::unpack_address(vm::CellSlice& addr_cs) { if (workchain == ton::workchainInvalid) { workchain = new_wc; addr = addr_orig; - addr.bits().copy_from(addr_rewrite.cbits(), split_depth_); - } else if (split_depth_) { + addr.bits().copy_from(addr_rewrite.cbits(), addr_rewrite_length); + } else if (addr_rewrite_length) { ton::StdSmcAddress new_addr = addr_orig; - new_addr.bits().copy_from(addr_rewrite.cbits(), split_depth_); + new_addr.bits().copy_from(addr_rewrite.cbits(), addr_rewrite_length); if (new_addr != addr) { LOG(ERROR) << "error unpacking account " << workchain << ":" << addr.to_hex() << " : account header contains different address " << new_addr.to_hex() << " (with splitting depth " - << (int)split_depth_ << ")"; + << (int)addr_rewrite_length << ")"; return false; } } else if (addr != addr_orig) { @@ -235,7 +235,7 @@ bool Account::unpack_address(vm::CellSlice& addr_cs) { return false; } addr_rewrite = addr.bits(); // initialize all 32 bits of addr_rewrite - if (!split_depth_) { + if (!addr_rewrite_length) { my_addr_exact = my_addr; } return true; @@ -283,7 +283,7 @@ bool Account::unpack_storage_info(vm::CellSlice& cs) { * Unpacks the state of an Account from a CellSlice. * * State is serialized using StateInit TLB-scheme. - * Initializes split_depth (from account state - StateInit) + * Initializes fixed_prefix_length (from account state - StateInit) * * @param cs The CellSlice containing the serialized state. * @@ -294,12 +294,9 @@ bool Account::unpack_state(vm::CellSlice& cs) { if (!tlb::unpack_exact(cs, state)) { return false; } - int sd = 0; - if (state.split_depth->size() == 6) { - sd = (int)state.split_depth->prefetch_ulong(6) - 32; - } - if (!set_split_depth(sd)) { - return false; + fixed_prefix_length = 0; + if (state.fixed_prefix_length->size() == 6) { + fixed_prefix_length = (int)state.fixed_prefix_length->prefetch_ulong(6) - 32; } if (state.special->size() > 1) { int z = (int)state.special->prefetch_ulong(3); @@ -366,23 +363,25 @@ bool Account::compute_my_addr(bool force) { /** * Computes the address of the Account. * + * Legacy (used only if global_version < 10). + * * @param tmp_addr A reference to the CellSlice for the result. - * @param split_depth The split depth for the address. - * @param orig_addr_rewrite Address prefox of length split_depth. + * @param fixed_prefix_length The fixed prefix length for the address. + * @param orig_addr_rewrite Address prefix of length fixed_prefix_length. * * @returns True if the address was successfully computed, false otherwise. */ -bool Account::recompute_tmp_addr(Ref& tmp_addr, int split_depth, +bool Account::recompute_tmp_addr(Ref& tmp_addr, int fixed_prefix_length, td::ConstBitPtr orig_addr_rewrite) const { - if (!split_depth && my_addr_exact.not_null()) { + if (!fixed_prefix_length && my_addr_exact.not_null()) { tmp_addr = my_addr_exact; return true; } - if (split_depth == split_depth_ && my_addr.not_null()) { + if (fixed_prefix_length == addr_rewrite_length && my_addr.not_null()) { tmp_addr = my_addr; return true; } - if (split_depth < 0 || split_depth > 30) { + if (fixed_prefix_length < 0 || fixed_prefix_length > 30) { return false; } vm::CellBuilder cb; @@ -390,13 +389,13 @@ bool Account::recompute_tmp_addr(Ref& tmp_addr, int split_depth, if (!cb.store_long_bool(std ? 2 : 3, 2)) { // addr_std$10 or addr_var$11 return false; } - if (!split_depth) { + if (!fixed_prefix_length) { if (!cb.store_bool_bool(false)) { // anycast:(Maybe Anycast) return false; } } else if (!(cb.store_bool_bool(true) // just$1 - && cb.store_long_bool(split_depth, 5) // depth:(#<= 30) - && cb.store_bits_bool(addr.bits(), split_depth))) { // rewrite_pfx:(bits depth) + && cb.store_long_bool(fixed_prefix_length, 5) // depth:(#<= 30) + && cb.store_bits_bool(addr.bits(), fixed_prefix_length))) { // rewrite_pfx:(bits depth) return false; } if (std) { @@ -408,26 +407,26 @@ bool Account::recompute_tmp_addr(Ref& tmp_addr, int split_depth, return false; } Ref cell; - return cb.store_bits_bool(orig_addr_rewrite, split_depth) // address:(bits addr_len) or bits256 - && cb.store_bits_bool(addr.bits() + split_depth, 256 - split_depth) && cb.finalize_to(cell) && + return cb.store_bits_bool(orig_addr_rewrite, fixed_prefix_length) // address:(bits addr_len) or bits256 + && cb.store_bits_bool(addr.bits() + fixed_prefix_length, 256 - fixed_prefix_length) && cb.finalize_to(cell) && (tmp_addr = vm::load_cell_slice_ref(std::move(cell))).not_null(); } /** * Sets address rewriting info for a newly-activated account. * - * @param split_depth The split depth for the account address. - * @param orig_addr_rewrite Address frepix of length split_depth. + * @param rewrite_length The fixed prefix length for the account address. + * @param orig_addr_rewrite Address prefix of length fixed_prefix_length. * * @returns True if the rewriting info was successfully set, false otherwise. */ -bool Account::init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewrite) { - if (split_depth_set_ || !set_split_depth(split_depth)) { +bool Account::init_rewrite_addr(int rewrite_length, td::ConstBitPtr orig_addr_rewrite) { + if (addr_rewrite_length_set || !set_addr_rewrite_length(rewrite_length)) { return false; } addr_orig = addr; addr_rewrite = addr.bits(); - addr_orig.bits().copy_from(orig_addr_rewrite, split_depth); + addr_orig.bits().copy_from(orig_addr_rewrite, rewrite_length); return compute_my_addr(true); } @@ -483,7 +482,7 @@ bool Account::unpack(Ref shard_account, ton::UnixTime now, bool s case block::gen::AccountState::account_uninit: status = orig_status = acc_uninit; state_hash = addr; - forget_split_depth(); + forget_addr_rewrite_length(); break; case block::gen::AccountState::account_frozen: status = orig_status = acc_frozen; @@ -558,18 +557,18 @@ bool Account::init_new(ton::UnixTime now) { } state_hash = addr_orig; status = orig_status = acc_nonexist; - split_depth_set_ = false; + addr_rewrite_length_set = false; return true; } /** - * Resets the split depth of the account. + * Resets the fixed prefix length of the account. * - * @returns True if the split depth was successfully reset, false otherwise. + * @returns True if the fixed prefix length was successfully reset, false otherwise. */ -bool Account::forget_split_depth() { - split_depth_set_ = false; - split_depth_ = 0; +bool Account::forget_addr_rewrite_length() { + addr_rewrite_length_set = false; + addr_rewrite_length = 0; addr_orig = addr; my_addr = my_addr_exact; addr_rewrite = addr.bits(); @@ -587,9 +586,10 @@ bool Account::deactivate() { } // forget special (tick/tock) info tick = tock = false; + fixed_prefix_length = 0; if (status == acc_nonexist || status == acc_uninit) { - // forget split depth and address rewriting info - forget_split_depth(); + // forget fixed prefix length and address rewriting info + forget_addr_rewrite_length(); // forget specific state hash for deleted or uninitialized accounts (revert to addr) state_hash = addr; } @@ -710,6 +710,7 @@ Transaction::Transaction(const Account& _account, int ttype, ton::LogicalTime re , is_first(_account.transactions.empty()) , new_tick(_account.tick) , new_tock(_account.tock) + , new_fixed_prefix_length(_account.fixed_prefix_length) , now(_now) , account(_account) , my_addr(_account.my_addr) @@ -750,42 +751,52 @@ bool Transaction::unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* }; } auto cs = vm::load_cell_slice(in_msg); - int tag = block::gen::t_CommonMsgInfo.get_tag(cs); - Ref src_addr, dest_addr; + int tag = gen::t_CommonMsgInfo.get_tag(cs); switch (tag) { - case block::gen::CommonMsgInfo::int_msg_info: { - block::gen::CommonMsgInfo::Record_int_msg_info info; - if (!(tlb::unpack(cs, info) && msg_balance_remaining.unpack(std::move(info.value)))) { + case gen::CommonMsgInfo::int_msg_info: { + if (!(tlb::unpack(cs, in_msg_info) && msg_balance_remaining.unpack(in_msg_info.value))) { return false; } - if (info.ihr_disabled && ihr_delivered) { + if (in_msg_info.ihr_disabled && ihr_delivered) { return false; } - bounce_enabled = info.bounce; - src_addr = std::move(info.src); - dest_addr = std::move(info.dest); + bounce_enabled = in_msg_info.bounce; in_msg_type = 1; - td::RefInt256 ihr_fee = block::tlb::t_Grams.as_integer(std::move(info.ihr_fee)); + td::RefInt256 ihr_fee = block::tlb::t_Grams.as_integer(in_msg_info.ihr_fee); if (ihr_delivered) { in_fwd_fee = std::move(ihr_fee); } else { in_fwd_fee = td::zero_refint(); msg_balance_remaining += std::move(ihr_fee); } - if (info.created_lt >= start_lt) { - start_lt = info.created_lt + 1; + if (in_msg_info.created_lt >= start_lt) { + start_lt = in_msg_info.created_lt + 1; end_lt = start_lt + 1; } // ... break; } - case block::gen::CommonMsgInfo::ext_in_msg_info: { - block::gen::CommonMsgInfo::Record_ext_in_msg_info info; + case gen::CommonMsgInfo::ext_in_msg_info: { + gen::CommonMsgInfo::Record_ext_in_msg_info info; if (!tlb::unpack(cs, info)) { return false; } - src_addr = std::move(info.src); - dest_addr = std::move(info.dest); + in_msg_info.ihr_disabled = in_msg_info.bounce = in_msg_info.bounced = false; + in_msg_info.src = info.src; + in_msg_info.dest = info.dest; + in_msg_info.created_at = in_msg_info.created_lt = 0; + if (cfg->disable_anycast) { + // Check that dest is addr_std without anycast + gen::MsgAddressInt::Record_addr_std rec; + if (!gen::csr_unpack(info.dest, rec)) { + LOG(DEBUG) << "destination address of the external message is not a valid addr_std"; + return false; + } + if (rec.anycast->size() > 1) { + LOG(DEBUG) << "destination address of the external message is an anycast address"; + return false; + } + } in_msg_type = 2; in_msg_extern = true; // compute forwarding fees for this external message @@ -1397,11 +1408,63 @@ Ref Transaction::prepare_vm_c7(const ComputePhaseConfig& cfg) const { ? vm::StackEntry(td::make_refint(compute_phase->precompiled_gas_usage.value())) : vm::StackEntry()); // precompiled_gas_usage:Integer } + if (cfg.global_version >= 11) { + // in_msg_params:[...] + tuple.push_back(prepare_in_msg_params_tuple(trans_type == tr_ord ? &in_msg_info : nullptr, in_msg_state, + msg_balance_remaining)); + } auto tuple_ref = td::make_cnt_ref>(std::move(tuple)); LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple_ref).to_string(); return vm::make_tuple_ref(std::move(tuple_ref)); } +/** + * Prepares tuple with unpacked parameters of the inbound message (for the 17th element of c7). + * `info` is: + * - For internal messages - just int_msg_info of the message + * - For external messages - artificial int_msg_info based on ext_msg_info of the messages. + * - For tick-tock transactions and get methods - nullptr. + * + * @param info Pointer to the message info. + * @param state_init State init of the message (null if absent). + * @param msg_balance_remaining Remaining balance of the message (it's sometimes different from value in info). + * + * @returns Tuple with message parameters. + */ +Ref Transaction::prepare_in_msg_params_tuple(const gen::CommonMsgInfo::Record_int_msg_info* info, + const Ref& state_init, + const CurrencyCollection& msg_balance_remaining) { + std::vector in_msg_params(10); + if (info != nullptr) { + in_msg_params[0] = td::make_refint(info->bounce ? -1 : 0); // bounce + in_msg_params[1] = td::make_refint(info->bounced ? -1 : 0); // bounced + in_msg_params[2] = info->src; // src_addr + in_msg_params[3] = info->fwd_fee.is_null() ? td::zero_refint() : tlb::t_Grams.as_integer(info->fwd_fee); // fwd_fee + in_msg_params[4] = td::make_refint(info->created_lt); // created_lt + in_msg_params[5] = td::make_refint(info->created_at); // created_at + auto value = info->value; + in_msg_params[6] = + info->value.is_null() ? td::zero_refint() : tlb::t_Grams.as_integer_skip(value.write()); // original value + in_msg_params[7] = msg_balance_remaining.is_valid() ? msg_balance_remaining.grams : td::zero_refint(); // value + in_msg_params[8] = msg_balance_remaining.is_valid() ? vm::StackEntry::maybe(msg_balance_remaining.extra) + : vm::StackEntry{}; // value extra + in_msg_params[9] = vm::StackEntry::maybe(state_init); // state_init + } else { + in_msg_params[0] = td::zero_refint(); // bounce + in_msg_params[1] = td::zero_refint(); // bounced + static Ref addr_none = vm::CellBuilder{}.store_zeroes(2).as_cellslice_ref(); + in_msg_params[2] = addr_none; // src_addr + in_msg_params[3] = td::zero_refint(); // fed_fee + in_msg_params[4] = td::zero_refint(); // created_lt + in_msg_params[5] = td::zero_refint(); // created_at + in_msg_params[6] = td::zero_refint(); // original value + in_msg_params[7] = td::zero_refint(); // value + in_msg_params[8] = vm::StackEntry{}; // value extra + in_msg_params[9] = vm::StackEntry{}; // state_init + } + return td::make_cnt_ref>(std::move(in_msg_params)); +} + /** * Computes the number of output actions in a list. * @@ -1442,10 +1505,13 @@ bool Transaction::unpack_msg_state(const ComputePhaseConfig& cfg, bool lib_only, in_msg_library = state.library->prefetch_ref(); return true; } - if (state.split_depth->size() == 6) { - new_split_depth = (signed char)(state.split_depth->prefetch_ulong(6) - 32); + if (state.fixed_prefix_length->size() == 6) { + new_fixed_prefix_length = (signed char)(state.fixed_prefix_length->prefetch_ulong(6) - 32); } else { - new_split_depth = 0; + new_fixed_prefix_length = 0; + } + if (!cfg.disable_anycast) { + new_addr_rewrite_length = new_fixed_prefix_length; } if (state.special->size() > 1) { int z = (int)state.special->prefetch_ulong(3); @@ -1500,19 +1566,26 @@ std::vector> Transaction::compute_vm_libraries(const ComputePhaseC /** * Checks if the input message StateInit hash corresponds to the account address. * + * @param cfg The configuration for the compute phase. + * * @returns True if the input message state hash is valid, False otherwise. */ -bool Transaction::check_in_msg_state_hash() { +bool Transaction::check_in_msg_state_hash(const ComputePhaseConfig& cfg) { CHECK(in_msg_state.not_null()); - CHECK(new_split_depth >= 0 && new_split_depth < 32); + CHECK(new_fixed_prefix_length >= 0 && new_fixed_prefix_length < 32); td::Bits256 in_state_hash = in_msg_state->get_hash().bits(); - int d = new_split_depth; + int d = new_fixed_prefix_length; if ((in_state_hash.bits() + d).compare(account.addr.bits() + d, 256 - d)) { return false; } orig_addr_rewrite = in_state_hash.bits(); orig_addr_rewrite_set = true; - return account.recompute_tmp_addr(my_addr, d, orig_addr_rewrite.bits()); + if (cfg.disable_anycast) { + my_addr = my_addr_exact; + return true; + } else { + return account.recompute_tmp_addr(my_addr, d, orig_addr_rewrite.bits()); + } } /** @@ -1643,16 +1716,24 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { use_msg_state = true; bool forbid_public_libs = acc_status == Account::acc_uninit && account.is_masterchain(); // Forbid for deploying, allow for unfreezing - if (!(unpack_msg_state(cfg, false, forbid_public_libs) && account.check_split_depth(new_split_depth))) { - LOG(DEBUG) << "cannot unpack in_msg_state, or it has bad split_depth; cannot init account state"; + if (!(unpack_msg_state(cfg, false, forbid_public_libs) && + account.check_addr_rewrite_length(new_fixed_prefix_length))) { + LOG(DEBUG) << "cannot unpack in_msg_state, or it has bad fixed_prefix_length; cannot init account state"; cp.skip_reason = ComputePhase::sk_bad_state; return true; } - if (acc_status == Account::acc_uninit && !check_in_msg_state_hash()) { + if (acc_status == Account::acc_uninit && !check_in_msg_state_hash(cfg)) { LOG(DEBUG) << "in_msg_state hash mismatch, cannot init account state"; cp.skip_reason = ComputePhase::sk_bad_state; return true; } + if (cfg.disable_anycast && acc_status == Account::acc_uninit && + new_fixed_prefix_length > cfg.size_limits.max_acc_fixed_prefix_length) { + LOG(DEBUG) << "cannot init account state: too big fixed prefix length (" << new_fixed_prefix_length << ", max " + << cfg.size_limits.max_acc_fixed_prefix_length << ")"; + cp.skip_reason = ComputePhase::sk_bad_state; + return true; + } } else if (acc_status != Account::acc_active) { // no state, cannot perform transactions cp.skip_reason = in_msg_state.not_null() ? ComputePhase::sk_bad_state : ComputePhase::sk_no_state; @@ -1675,6 +1756,11 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { return true; } } + if (cfg.disable_anycast) { + my_addr = my_addr_exact; + new_addr_rewrite_length = 0; + force_remove_anycast_address = true; + } td::optional precompiled; if (new_code.not_null() && trans_type == tr_ord) { @@ -1970,7 +2056,7 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { ap.no_funds = true; } LOG(DEBUG) << "invalid action " << ap.result_arg << " in action list: error code " << ap.result_code; - // This is required here because changes to libraries are applied even if actipn phase fails + // This is required here because changes to libraries are applied even if action phase fails enforce_state_limits(); if (cfg.action_fine_enabled) { ap.action_fine = std::min(ap.action_fine, balance.grams); @@ -1994,6 +2080,15 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { } new_data = compute_phase->new_data; // tentative persistent data update applied if (!enforce_state_limits()) { + if (cfg.extra_currency_v2) { + end_lt = ap.end_lt = start_lt + 1; + if (cfg.action_fine_enabled) { + ap.action_fine = std::min(ap.action_fine, balance.grams); + ap.total_action_fees = ap.action_fine; + balance.grams -= ap.action_fine; + total_fees += ap.action_fine; + } + } return true; } @@ -2241,11 +2336,12 @@ bool Transaction::check_replace_src_addr(Ref& src_addr) const { * @param dest_addr A reference to the destination address of the transaction. * @param cfg The configuration for the action phase. * @param is_mc A pointer to a boolean where it will be stored whether the destination is in the masterchain. + * @param allow_anycast Allow anycast the address. * * @returns True if the destination address is valid, false otherwise. */ bool Transaction::check_rewrite_dest_addr(Ref& dest_addr, const ActionPhaseConfig& cfg, - bool* is_mc) const { + bool* is_mc, bool allow_anycast) const { if (!dest_addr->prefetch_ulong(1)) { // all external addresses allowed if (is_mc) { @@ -2305,6 +2401,9 @@ bool Transaction::check_rewrite_dest_addr(Ref& dest_addr, const A } } if (rec.anycast->size() > 1) { + if (!allow_anycast) { + return false; + } // destination address is an anycast vm::CellSlice cs{*rec.anycast}; int d = (int)cs.fetch_ulong(6) - 32; @@ -2472,11 +2571,11 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, return 35; // invalid source address } bool to_mc = false; - if (!check_rewrite_dest_addr(info.dest, cfg, &to_mc)) { + if (!check_rewrite_dest_addr(info.dest, cfg, &to_mc, !cfg.disable_anycast)) { LOG(DEBUG) << "invalid destination address in a proposed outbound message"; return check_skip_invalid(36); // invalid destination address } - if (cfg.extra_currency_v2) { + if (!ext_msg && cfg.extra_currency_v2) { CurrencyCollection value; if (!value.unpack(info.value)) { LOG(DEBUG) << "invalid value:ExtraCurrencies in a proposed outbound message"; @@ -2486,7 +2585,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, LOG(DEBUG) << "invalid value:ExtraCurrencies in a proposed outbound message: too many currencies (max " << cfg.size_limits.max_msg_extra_currencies << ")"; // Dict should be valid, since it was checked in t_OutListNode.validate_ref, so error here means limit exceeded - return check_skip_invalid(41); // invalid value:CurrencyCollection : too many extra currencies + return check_skip_invalid(44); // invalid value:CurrencyCollection : too many extra currencies } info.value = value.pack(); } @@ -2667,7 +2766,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (cfg.extra_currency_v2 && !req.check_extra_currency_limit(cfg.size_limits.max_msg_extra_currencies)) { LOG(DEBUG) << "too many extra currencies in the message : max " << cfg.size_limits.max_msg_extra_currencies; - return check_skip_invalid(41); // to many extra currencies + return check_skip_invalid(44); // to many extra currencies } Ref new_extra; @@ -2832,13 +2931,25 @@ int Transaction::try_action_reserve_currency(vm::CellSlice& cs, ActionPhase& ap, LOG(DEBUG) << "cannot parse currency field in action_reserve_currency"; return -1; } + if (cfg.extra_currency_v2 && reserve.has_extra()) { + LOG(DEBUG) << "cannot reserve extra currencies"; + return -1; + } LOG(DEBUG) << "action_reserve_currency: mode=" << mode << ", reserve=" << reserve.to_str() << ", balance=" << ap.remaining_balance.to_str() << ", original balance=" << original_balance.to_str(); if (mode & 4) { if (mode & 8) { - reserve = original_balance - reserve; + if (cfg.extra_currency_v2) { + reserve.grams = original_balance.grams - reserve.grams; + } else { + reserve = original_balance - reserve; + } } else { - reserve += original_balance; + if (cfg.extra_currency_v2) { + reserve.grams += original_balance.grams; + } else { + reserve += original_balance; + } } } else if (mode & 8) { LOG(DEBUG) << "invalid reserve mode " << mode; @@ -2851,7 +2962,7 @@ int Transaction::try_action_reserve_currency(vm::CellSlice& cs, ActionPhase& ap, if (mode & 2) { if (cfg.reserve_extra_enabled) { if (!reserve.clamp(ap.remaining_balance)) { - LOG(DEBUG) << "failed to clamp reserve amount" << mode; + LOG(DEBUG) << "failed to clamp reserve amount " << mode; return -1; } } else { @@ -2872,7 +2983,11 @@ int Transaction::try_action_reserve_currency(vm::CellSlice& cs, ActionPhase& ap, newc.grams = ap.remaining_balance.grams - reserve.grams; if (mode & 1) { // leave only res_grams, reserve everything else - std::swap(newc, reserve); + if (cfg.extra_currency_v2) { + std::swap(newc.grams, reserve.grams); + } else { + std::swap(newc, reserve); + } } // set remaining_balance to new_grams and new_extra ap.remaining_balance = std::move(newc); @@ -3192,13 +3307,14 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { && balance.store(cb)); // balance:CurrencyCollection int ticktock = new_tick * 2 + new_tock; unsigned si_pos = 0; + int fixed_prefix_length = cfg.disable_anycast ? new_fixed_prefix_length : account.addr_rewrite_length; if (acc_status == Account::acc_uninit) { CHECK(cb.store_long_bool(0, 2)); // account_uninit$00 = AccountState } else if (acc_status == Account::acc_frozen) { if (was_frozen) { vm::CellBuilder cb2; - CHECK(account.split_depth_ ? cb2.store_long_bool(account.split_depth_ + 32, 6) // _ ... = StateInit - : cb2.store_long_bool(0, 1)); // ... split_depth:(Maybe (## 5)) + CHECK(fixed_prefix_length ? cb2.store_long_bool(fixed_prefix_length + 32, 6) // _ ... = StateInit + : cb2.store_long_bool(0, 1)); // ... fixed_prefix_length:(Maybe (## 5)) CHECK(ticktock ? cb2.store_long_bool(ticktock | 4, 3) : cb2.store_long_bool(0, 1)); // special:(Maybe TickTock) CHECK(cb2.store_maybe_ref(new_code) && cb2.store_maybe_ref(new_data) && cb2.store_maybe_ref(new_library)); // code:(Maybe ^Cell) data:(Maybe ^Cell) library:(HashmapE 256 SimpleLib) @@ -3227,8 +3343,8 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } else { CHECK(acc_status == Account::acc_active && !was_frozen && !was_deleted); si_pos = cb.size_ext() + 1; - CHECK(account.split_depth_ ? cb.store_long_bool(account.split_depth_ + 96, 7) // account_active$1 _:StateInit - : cb.store_long_bool(2, 2)); // ... split_depth:(Maybe (## 5)) + CHECK(fixed_prefix_length ? cb.store_long_bool(fixed_prefix_length + 96, 7) // account_active$1 _:StateInit + : cb.store_long_bool(2, 2)); // ... fixed_prefix_length:(Maybe (## 5)) CHECK(ticktock ? cb.store_long_bool(ticktock | 4, 3) : cb.store_long_bool(0, 1)); // special:(Maybe TickTock) CHECK(cb.store_maybe_ref(new_code) && cb.store_maybe_ref(new_data) && cb.store_maybe_ref(new_library)); // code:(Maybe ^Cell) data:(Maybe ^Cell) library:(HashmapE 256 SimpleLib) @@ -3314,8 +3430,8 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } } - CHECK(cb.store_long_bool(1, 1) // account$1 - && cb.append_cellslice_bool(account.my_addr) // addr:MsgAddressInt + CHECK(cb.store_long_bool(1, 1) // account$1 + && cb.append_cellslice_bool(cfg.disable_anycast ? my_addr : account.my_addr) // addr:MsgAddressInt && block::store_UInt7(cb, new_storage_used.cells) // storage_used$_ cells:(VarUInteger 7) && block::store_UInt7(cb, new_storage_used.bits) // bits:(VarUInteger 7) && cb.store_long_bool(new_storage_dict_hash ? 1 : 0, 3) // extra:StorageExtraInfo @@ -3677,12 +3793,14 @@ Ref Transaction::commit(Account& acc) { CHECK((const void*)&acc == (const void*)&account); // export all fields modified by the Transaction into original account // NB: this is the only method that modifies account - if (orig_addr_rewrite_set && new_split_depth >= 0 && acc.status != Account::acc_active && + if (force_remove_anycast_address) { + CHECK(acc.forget_addr_rewrite_length()); + } else if (orig_addr_rewrite_set && new_addr_rewrite_length >= 0 && acc.status != Account::acc_active && acc_status == Account::acc_active) { LOG(DEBUG) << "setting address rewriting info for newly-activated account " << acc.addr.to_hex() - << " with split_depth=" << new_split_depth - << ", orig_addr_rewrite=" << orig_addr_rewrite.bits().to_hex(new_split_depth); - CHECK(acc.init_rewrite_addr(new_split_depth, orig_addr_rewrite.bits())); + << " with addr_rewrite_length=" << new_addr_rewrite_length + << ", orig_addr_rewrite=" << orig_addr_rewrite.bits().to_hex(new_addr_rewrite_length); + CHECK(acc.init_rewrite_addr(new_addr_rewrite_length, orig_addr_rewrite.bits())); } acc.status = (acc_status == Account::acc_deleted ? Account::acc_nonexist : acc_status); acc.last_trans_lt_ = start_lt; @@ -3714,6 +3832,7 @@ Ref Transaction::commit(Account& acc) { if (acc.status == Account::acc_active) { acc.tick = new_tick; acc.tock = new_tock; + acc.fixed_prefix_length = new_fixed_prefix_length; } else { CHECK(acc.deactivate()); } @@ -3914,6 +4033,7 @@ td::Status FetchConfigParams::fetch_config_params( compute_phase_cfg->size_limits = size_limits; compute_phase_cfg->precompiled_contracts = config.get_precompiled_contracts_config(); compute_phase_cfg->allow_external_unfreeze = compute_phase_cfg->global_version >= 8; + compute_phase_cfg->disable_anycast = config.get_global_version() >= 10; } { // compute action_phase_cfg @@ -3942,9 +4062,11 @@ td::Status FetchConfigParams::fetch_config_params( action_phase_cfg->reserve_extra_enabled = config.get_global_version() >= 9; action_phase_cfg->mc_blackhole_addr = config.get_burning_config().blackhole_addr; action_phase_cfg->extra_currency_v2 = config.get_global_version() >= 10; + action_phase_cfg->disable_anycast = config.get_global_version() >= 10; } { serialize_cfg->extra_currency_v2 = config.get_global_version() >= 10; + serialize_cfg->disable_anycast = config.get_global_version() >= 10; serialize_cfg->store_storage_dict_hash = config.get_global_version() >= 11; } { diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index aa08719a8f..fe70060067 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -31,6 +31,7 @@ #include "block/block.h" #include "block/mc-config.h" #include "precompiled-smc/PrecompiledSmartContract.h" +#include "block/block-auto.h" namespace block { using td::Ref; @@ -131,6 +132,7 @@ struct ComputePhaseConfig { PrecompiledContractsConfig precompiled_contracts; bool dont_run_precompiled_ = false; bool allow_external_unfreeze{false}; + bool disable_anycast{false}; ComputePhaseConfig() : gas_price(0), gas_limit(0), special_gas_limit(0), gas_credit(0) { compute_threshold(); @@ -173,6 +175,7 @@ struct ActionPhaseConfig { bool reserve_extra_enabled{false}; bool extra_currency_v2{false}; td::optional mc_blackhole_addr; + bool disable_anycast{false}; const MsgPrices& fetch_msg_prices(bool is_masterchain) const { return is_masterchain ? fwd_mc : fwd_std; } @@ -180,6 +183,7 @@ struct ActionPhaseConfig { struct SerializeConfig { bool extra_currency_v2{false}; + bool disable_anycast{false}; bool store_storage_dict_hash{false}; }; @@ -254,12 +258,13 @@ struct Account { bool is_special{false}; bool tick{false}; bool tock{false}; - bool split_depth_set_{false}; - unsigned char split_depth_{0}; + int fixed_prefix_length{0}; int verbosity{3 * 0}; ton::UnixTime now_{0}; ton::WorkchainId workchain{ton::workchainInvalid}; - td::BitArray<32> addr_rewrite; // rewrite (anycast) data, split_depth bits + td::BitArray<32> addr_rewrite; // rewrite (anycast) data, addr_rewrite_length bits + bool addr_rewrite_length_set{false}; + unsigned char addr_rewrite_length{0}; ton::StdSmcAddress addr; // rewritten address (by replacing a prefix of `addr_orig` with `addr_rewrite`); it is the key in ShardAccounts ton::StdSmcAddress addr_orig; // address indicated in smart-contract data (must coincide with hash of StateInit) Ref my_addr; // address as stored in the smart contract (MsgAddressInt); corresponds to `addr_orig` + anycast info @@ -286,9 +291,6 @@ struct Account { Account() = default; Account(ton::WorkchainId wc, td::ConstBitPtr _addr) : workchain(wc), addr(_addr) { } - Account(ton::WorkchainId wc, td::ConstBitPtr _addr, int depth) - : split_depth_set_(true), split_depth_((unsigned char)depth), workchain(wc), addr(_addr) { - } block::CurrencyCollection get_balance() const { return balance; } @@ -296,7 +298,7 @@ struct Account { bool unpack(Ref account, ton::UnixTime now, bool special); bool init_new(ton::UnixTime now); bool deactivate(); - bool recompute_tmp_addr(Ref& tmp_addr, int split_depth, td::ConstBitPtr orig_addr_rewrite) const; + bool recompute_tmp_addr(Ref& tmp_addr, int fixed_prefix_length, td::ConstBitPtr orig_addr_rewrite) const; td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector& pricing) const; bool is_masterchain() const { return workchain == ton::masterchainId; @@ -312,10 +314,10 @@ struct Account { protected: friend struct transaction::Transaction; - bool set_split_depth(int split_depth); - bool check_split_depth(int split_depth) const; - bool forget_split_depth(); - bool init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewrite); + bool set_addr_rewrite_length(int new_length); + bool check_addr_rewrite_length(int length) const; + bool forget_addr_rewrite_length(); + bool init_rewrite_addr(int rewrite_length, td::ConstBitPtr orig_addr_rewrite); private: bool unpack_address(vm::CellSlice& addr_cs); @@ -347,12 +349,15 @@ struct Transaction { bool was_created{false}; bool bounce_enabled{false}; bool in_msg_extern{false}; + gen::CommonMsgInfo::Record_int_msg_info in_msg_info; bool use_msg_state{false}; bool is_first{false}; bool orig_addr_rewrite_set{false}; bool new_tick; bool new_tock; - signed char new_split_depth{-1}; + int new_fixed_prefix_length{-1}; + int new_addr_rewrite_length{-1}; + bool force_remove_anycast_address = false; ton::UnixTime now; int acc_status; int verbosity{3 * 0}; @@ -390,7 +395,7 @@ struct Transaction { Transaction(const Account& _account, int ttype, ton::LogicalTime req_start_lt, ton::UnixTime _now, Ref _inmsg = {}); bool unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* cfg); - bool check_in_msg_state_hash(); + bool check_in_msg_state_hash(const ComputePhaseConfig& cfg); bool prepare_storage_phase(const StoragePhaseConfig& cfg, bool force_collect = true, bool adjust_msg_value = false); bool prepare_credit_phase(); td::uint64 gas_bought_for(const ComputePhaseConfig& cfg, td::RefInt256 nanograms); @@ -426,13 +431,18 @@ struct Transaction { int try_action_reserve_currency(vm::CellSlice& cs, ActionPhase& ap, const ActionPhaseConfig& cfg); bool check_replace_src_addr(Ref& src_addr) const; bool check_rewrite_dest_addr(Ref& dest_addr, const ActionPhaseConfig& cfg, - bool* is_mc = nullptr) const; + bool* is_mc = nullptr, bool allow_anycast = true) const; bool serialize_storage_phase(vm::CellBuilder& cb); bool serialize_credit_phase(vm::CellBuilder& cb); bool serialize_compute_phase(vm::CellBuilder& cb); bool serialize_action_phase(vm::CellBuilder& cb); bool serialize_bounce_phase(vm::CellBuilder& cb); bool unpack_msg_state(const ComputePhaseConfig& cfg, bool lib_only = false, bool forbid_public_libs = false); + + public: + static Ref prepare_in_msg_params_tuple(const gen::CommonMsgInfo::Record_int_msg_info* info, + const Ref& state_init, + const CurrencyCollection& msg_balance_remaining); }; } // namespace transaction diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index 4d7e2b6fe5..9413513972 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -1305,6 +1305,7 @@ x{F814} @Defop SETRAND x{F815} dup @Defop ADDRAND @Defop RANDOMIZE x{F82} @Defop(4u) GETPARAM +x{F881} @Defop(8u) GETPARAMLONG x{F823} @Defop NOW x{F824} @Defop BLOCKLT x{F825} @Defop LTIME @@ -1318,6 +1319,8 @@ x{F82C} @Defop STORAGEFEES x{F82D} @Defop PREVBLOCKSINFOTUPLE x{F82E} @Defop UNPACKEDCONFIGTUPLE x{F82F} @Defop DUEPAYMENT +x{F88111} @Defop INMSGPARAMS + x{F830} @Defop CONFIGDICT x{F832} @Defop CONFIGPARAM x{F833} @Defop CONFIGOPTPARAM @@ -1333,11 +1336,25 @@ x{F83A} @Defop GETORIGINALFWDFEE x{F83B} @Defop GETGASFEESIMPLE x{F83C} @Defop GETFORWARDFEESIMPLE +x{F89} @Defop(4u) INMSGPARAM +x{F890} @Defop INMSG_BOUNCE +x{F891} @Defop INMSG_BOUNCED +x{F892} @Defop INMSG_SRC +x{F893} @Defop INMSG_FWDFEE +x{F894} @Defop INMSG_LT +x{F895} @Defop INMSG_UTIME +x{F896} @Defop INMSG_ORIGVALUE +x{F897} @Defop INMSG_VALUE +x{F898} @Defop INMSG_VALUEEXTRA +x{F899} @Defop INMSG_STATEINIT + x{F840} @Defop GETGLOBVAR { dup 1 31 @rangechk 16 config! } : config.validator_num! diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index 4d860fbab8..db2ab74ab8 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -19,6 +19,7 @@ #include "SmartContract.h" #include "GenericAccount.h" +#include "transaction.h" #include "block/block.h" #include "block/block-auto.h" @@ -175,13 +176,16 @@ td::Ref prepare_vm_c7(SmartContract::Args args, td::Ref cod if (args.config && args.config.value()->get_global_version() >= 6) { tuple.push_back(args.config.value()->get_unpacked_config_tuple(now)); // unpacked_config_tuple tuple.push_back(td::zero_refint()); // due_payment - // precomiled_gas_usage:(Maybe Integer) + // precompiled_gas_usage:(Maybe Integer) td::optional precompiled; if (code.not_null()) { precompiled = args.config.value()->get_precompiled_contracts_config().get_contract(code->get_hash().bits()); } tuple.push_back(precompiled ? td::make_refint(precompiled.value().gas_usage) : vm::StackEntry()); } + if (args.config && args.config.value()->get_global_version() >= 11) { + tuple.push_back(block::transaction::Transaction::prepare_in_msg_params_tuple(nullptr, {}, {})); + } auto tuple_ref = td::make_cnt_ref>(std::move(tuple)); //LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple).to_string(); return vm::make_tuple_ref(std::move(tuple_ref)); diff --git a/crypto/test/fift.cpp b/crypto/test/fift.cpp index 9a92e0b1eb..38a9ddd92b 100644 --- a/crypto/test/fift.cpp +++ b/crypto/test/fift.cpp @@ -171,3 +171,7 @@ TEST(Fift, test_levels) { TEST(Fift, test_secp256k1) { run_fift("secp256k1.fif"); } + +TEST(Fift, test_get_extra_balance) { + run_fift("get_extra_balance.fif"); +} diff --git a/crypto/test/fift/get_extra_balance.fif b/crypto/test/fift/get_extra_balance.fif new file mode 100644 index 0000000000..2b475a7ed8 --- /dev/null +++ b/crypto/test/fift/get_extra_balance.fif @@ -0,0 +1,80 @@ +"Asm.fif" include +"FiftExt.fif" include +"TonUtil.fif" include + +null constant extras +{ null } 7 times +10 extras 2 tuple +8 tuple 1 tuple constant c7 +<{ + GASCONSUMED 1 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 2 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 3 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 4 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 5 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 6 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 7 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 8 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 9 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 10 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 0 INT <{ }>s SLICE 128 RUNVM GASCONSUMED ROT SUB 26 INT SUB +}>s c7 16 runvmx abort"exitcode != 0" +.s { drop } depth 1- times + +dictnew + s SLICE 128 RUNVM GASCONSUMED ROT SUB 26 INT SUB +}>s c7 16 runvmx abort"exitcode != 0" +.s { drop } depth 1- times + +dictnew 1 +{ + =: idx =: d + s SLICE 128 RUNVM GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 1 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 2 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 3 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 4 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 5 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 6 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 0 INT <{ }>s SLICE 128 RUNVM GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 0 INT <{ }>s SLICE 128 RUNVM GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 0 INT <{ }>s SLICE 128 RUNVM GASCONSUMED ROT SUB 26 INT SUB +}>s c7 16 runvmx abort"exitcode != 0" +.s { drop } depth 1- times diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index aab1711f46..b62e5fb991 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -63,6 +63,9 @@ bool debug(int x) { #define DEB_START DBG_START #define DEB DBG +static constexpr int randseed_idx = 6; +static constexpr int inmsgparams_idx = 17; + int exec_set_gas_generic(VmState* st, long long new_gas_limit) { if (new_gas_limit < st->gas_consumed()) { throw VmNoGas{}; @@ -152,6 +155,27 @@ int exec_get_var_param(VmState* st, unsigned idx) { return exec_get_param(st, idx, nullptr); } +int exec_get_var_param_long(VmState* st, unsigned idx) { + idx &= 255; + VM_LOG(st) << "execute GETPARAMLONG " << idx; + return exec_get_param(st, idx, nullptr); +} + +int exec_get_in_msg_param(VmState* st, unsigned idx, const char* name) { + if (name) { + VM_LOG(st) << "execute " << name; + } + Ref t = get_param(st, inmsgparams_idx).as_tuple(); + st->get_stack().push(tuple_index(t, idx)); + return 0; +} + +int exec_get_var_in_msg_param(VmState* st, unsigned idx) { + idx &= 15; + VM_LOG(st) << "execute INMSGPARAM " << idx; + return exec_get_in_msg_param(st, idx, nullptr); +} + int exec_get_config_dict(VmState* st) { exec_get_param(st, 9, "CONFIGDICT"); st->get_stack().push_smallint(32); @@ -358,6 +382,75 @@ int exec_get_forward_fee_simple(VmState* st) { return 0; } +int exec_get_extra_currency_balance(VmState* st) { + VM_LOG(st) << "execute GETEXTRABALANCE"; + Stack& stack = st->get_stack(); + auto id = (td::uint32)stack.pop_long_range((1LL << 32) - 1); + + auto tuple = st->get_c7(); + tuple = tuple_index(tuple, 0).as_tuple_range(255); + if (tuple.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; + } + tuple = tuple_index(tuple, 7).as_tuple_range(255); // Balance + if (tuple.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; + } + auto dict_root = tuple_index(tuple, 1); + if (!dict_root.is_cell() && !dict_root.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not cell or null"}; + } + + class LocalVmState : public VmStateInterface { + public: + explicit LocalVmState(VmState* st) : st_(st) { + } + ~LocalVmState() override = default; + + Ref load_library(td::ConstBitPtr hash) override { + return st_->load_library(hash); + } + void register_cell_load(const CellHash& cell_hash) override { + auto new_cell = st_->register_cell_load_free(cell_hash); + consume_gas(new_cell ? VmState::cell_load_gas_price : VmState::cell_reload_gas_price); + } + void register_cell_create() override { + // Not expected in this operation + } + int get_global_version() const override { + return st_->get_global_version(); + } + + private: + VmState* st_; + long long remaining = VmState::get_extra_balance_cheap_max_gas_price; + + void consume_gas(long long gas) { + long long consumed = std::min(gas, remaining); + st_->consume_gas(consumed); + remaining -= consumed; + if (remaining == 0) { + st_->consume_free_gas(gas - consumed); + } + } + }; + bool cheap = st->register_get_extra_balance_call(); + LocalVmState local_vm_state{st}; + VmStateInterface::Guard guard{cheap ? (VmStateInterface*)&local_vm_state : st}; + + Dictionary dict{dict_root.as_cell(), 32}; + Ref cs = dict.lookup(td::BitArray<32>(id)); + if (cs.is_null()) { + stack.push_smallint(0); + } else { + td::RefInt256 x; + util::load_var_integer_q(cs.write(), x, /* len_bits = */ 5, /* sgnd = */ false, /* quiet = */ false); + stack.push_int(std::move(x)); + } + + return 0; +} + void register_ton_config_ops(OpcodeTable& cp0) { using namespace std::placeholders; cp0.insert(OpcodeInstr::mkfixedrange(0xf820, 0xf823, 16, 4, instr::dump_1c("GETPARAM "), exec_get_var_param)) @@ -391,11 +484,24 @@ void register_ton_config_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xf840, 16, "GETGLOBVAR", exec_get_global_var)) .insert(OpcodeInstr::mkfixedrange(0xf841, 0xf860, 16, 5, instr::dump_1c_and(31, "GETGLOB "), exec_get_global)) .insert(OpcodeInstr::mksimple(0xf860, 16, "SETGLOBVAR", exec_set_global_var)) - .insert(OpcodeInstr::mkfixedrange(0xf861, 0xf880, 16, 5, instr::dump_1c_and(31, "SETGLOB "), exec_set_global)); + .insert(OpcodeInstr::mkfixedrange(0xf861, 0xf880, 16, 5, instr::dump_1c_and(31, "SETGLOB "), exec_set_global)) + .insert(OpcodeInstr::mksimple(0xf880, 16, "GETEXTRABALANCE", exec_get_extra_currency_balance)->require_version(10)) + .insert(OpcodeInstr::mkfixedrange(0xf88100, 0xf88111, 24, 8, instr::dump_1c_l_add(0, "GETPARAMLONG "), exec_get_var_param_long)->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf88111, 24, "INMSGPARAMS", std::bind(exec_get_param, _1, 17, "INMSGPARAMS"))->require_version(11)) + .insert(OpcodeInstr::mkfixedrange(0xf88112, 0xf881ff, 24, 8, instr::dump_1c_l_add(0, "GETPARAMLONG "), exec_get_var_param_long)->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf890, 16, "INMSG_BOUNCE", std::bind(exec_get_in_msg_param, _1, 0, "INMSG_BOUNCE"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf891, 16, "INMSG_BOUNCED", std::bind(exec_get_in_msg_param, _1, 1, "INMSG_BOUNCED"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf892, 16, "INMSG_SRC", std::bind(exec_get_in_msg_param, _1, 2, "INMSG_SRC"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf893, 16, "INMSG_FWDFEE", std::bind(exec_get_in_msg_param, _1, 3, "INMSG_FWDFEE"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf894, 16, "INMSG_LT", std::bind(exec_get_in_msg_param, _1, 4, "INMSG_LT"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf895, 16, "INMSG_UTIME", std::bind(exec_get_in_msg_param, _1, 5, "INMSG_UTIME"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf896, 16, "INMSG_ORIGVALUE", std::bind(exec_get_in_msg_param, _1, 6, "INMSG_ORIGVALUE"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf897, 16, "INMSG_VALUE", std::bind(exec_get_in_msg_param, _1, 7, "INMSG_VALUE"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf898, 16, "INMSG_VALUEEXTRA", std::bind(exec_get_in_msg_param, _1, 8, "INMSG_VALUEEXTRA"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf899, 16, "INMSG_STATEINIT", std::bind(exec_get_in_msg_param, _1, 9, "INMSG_STATEINIT"))->require_version(11)) + .insert(OpcodeInstr::mkfixedrange(0xf89a, 0xf8a0, 16, 4, instr::dump_1c("INMSGPARAM "), exec_get_var_in_msg_param)->require_version(11)); } -static constexpr int randseed_idx = 6; - td::RefInt256 generate_randu256(VmState* st) { auto tuple = st->get_c7(); auto t1 = tuple_index(tuple, 0).as_tuple_range(255); @@ -1380,10 +1486,13 @@ int exec_store_var_integer(VmState* st, int len_bits, bool sgnd, bool quiet) { return 0; } -bool skip_maybe_anycast(CellSlice& cs) { +bool skip_maybe_anycast(CellSlice& cs, int global_version) { if (cs.prefetch_ulong(1) != 1) { return cs.advance(1); } + if (global_version >= 10) { + return false; + } unsigned depth; return cs.advance(1) // just$1 && cs.fetch_uint_leq(30, depth) // anycast_info$_ depth:(#<= 30) @@ -1391,7 +1500,7 @@ bool skip_maybe_anycast(CellSlice& cs) { && cs.advance(depth); // rewrite_pfx:(bits depth) = Anycast; } -bool skip_message_addr(CellSlice& cs) { +bool skip_message_addr(CellSlice& cs, int global_version) { switch ((unsigned)cs.fetch_ulong(2)) { case 0: // addr_none$00 = MsgAddressExt; return true; @@ -1400,15 +1509,18 @@ bool skip_message_addr(CellSlice& cs) { return cs.fetch_uint_to(9, len) // len:(## 9) && cs.advance(len); // external_address:(bits len) = MsgAddressExt; } - case 2: { // addr_std$10 - return skip_maybe_anycast(cs) // anycast:(Maybe Anycast) - && cs.advance(8 + 256); // workchain_id:int8 address:bits256 = MsgAddressInt; + case 2: { // addr_std$10 + return skip_maybe_anycast(cs, global_version) // anycast:(Maybe Anycast) + && cs.advance(8 + 256); // workchain_id:int8 address:bits256 = MsgAddressInt; } case 3: { // addr_var$11 + if (global_version >= 10) { + return false; + } unsigned len; - return skip_maybe_anycast(cs) // anycast:(Maybe Anycast) - && cs.fetch_uint_to(9, len) // addr_len:(## 9) - && cs.advance(32 + len); // workchain_id:int32 address:(bits addr_len) = MsgAddressInt; + return skip_maybe_anycast(cs, global_version) // anycast:(Maybe Anycast) + && cs.fetch_uint_to(9, len) // addr_len:(## 9) + && cs.advance(32 + len); // workchain_id:int32 address:(bits addr_len) = MsgAddressInt; } default: return false; @@ -1420,7 +1532,7 @@ int exec_load_message_addr(VmState* st, bool quiet) { Stack& stack = st->get_stack(); auto csr = stack.pop_cellslice(); td::Ref addr{true}; - if (util::load_msg_addr_q(csr.write(), addr.write(), quiet)) { + if (util::load_msg_addr_q(csr.write(), addr.write(), st->get_global_version(), quiet)) { stack.push_cellslice(std::move(addr)); stack.push_cellslice(std::move(csr)); if (quiet) { @@ -1433,11 +1545,14 @@ int exec_load_message_addr(VmState* st, bool quiet) { return 0; } -bool parse_maybe_anycast(CellSlice& cs, StackEntry& res) { +bool parse_maybe_anycast(CellSlice& cs, StackEntry& res, int global_version) { res = StackEntry{}; if (cs.prefetch_ulong(1) != 1) { return cs.advance(1); } + if (global_version >= 10) { + return false; + } unsigned depth; Ref pfx; if (cs.advance(1) // just$1 @@ -1450,7 +1565,7 @@ bool parse_maybe_anycast(CellSlice& cs, StackEntry& res) { return false; } -bool parse_message_addr(CellSlice& cs, std::vector& res) { +bool parse_message_addr(CellSlice& cs, std::vector& res, int global_version) { res.clear(); switch ((unsigned)cs.fetch_ulong(2)) { case 0: // addr_none$00 = MsgAddressExt; @@ -1471,9 +1586,9 @@ bool parse_message_addr(CellSlice& cs, std::vector& res) { StackEntry v; int workchain; Ref addr; - if (parse_maybe_anycast(cs, v) // anycast:(Maybe Anycast) - && cs.fetch_int_to(8, workchain) // workchain_id:int8 - && cs.fetch_subslice_to(256, addr)) { // address:bits256 = MsgAddressInt; + if (parse_maybe_anycast(cs, v, global_version) // anycast:(Maybe Anycast) + && cs.fetch_int_to(8, workchain) // workchain_id:int8 + && cs.fetch_subslice_to(256, addr)) { // address:bits256 = MsgAddressInt; res.emplace_back(td::make_refint(2)); res.emplace_back(std::move(v)); res.emplace_back(td::make_refint(workchain)); @@ -1483,13 +1598,16 @@ bool parse_message_addr(CellSlice& cs, std::vector& res) { break; } case 3: { // addr_var$11 + if (global_version >= 10) { + return false; + } StackEntry v; int len, workchain; Ref addr; - if (parse_maybe_anycast(cs, v) // anycast:(Maybe Anycast) - && cs.fetch_uint_to(9, len) // addr_len:(## 9) - && cs.fetch_int_to(32, workchain) // workchain_id:int32 - && cs.fetch_subslice_to(len, addr)) { // address:(bits addr_len) = MsgAddressInt; + if (parse_maybe_anycast(cs, v, global_version) // anycast:(Maybe Anycast) + && cs.fetch_uint_to(9, len) // addr_len:(## 9) + && cs.fetch_int_to(32, workchain) // workchain_id:int32 + && cs.fetch_subslice_to(len, addr)) { // address:(bits addr_len) = MsgAddressInt; res.emplace_back(td::make_refint(3)); res.emplace_back(std::move(v)); res.emplace_back(td::make_refint(workchain)); @@ -1508,7 +1626,7 @@ int exec_parse_message_addr(VmState* st, bool quiet) { auto csr = stack.pop_cellslice(); auto& cs = csr.write(); std::vector res; - if (!(parse_message_addr(cs, res) && cs.empty_ext())) { + if (!(parse_message_addr(cs, res, st->get_global_version()) && cs.empty_ext())) { if (quiet) { stack.push_bool(false); } else { @@ -1548,7 +1666,7 @@ int exec_rewrite_message_addr(VmState* st, bool allow_var_addr, bool quiet) { auto csr = stack.pop_cellslice(); auto& cs = csr.write(); std::vector tuple; - if (!(parse_message_addr(cs, tuple) && cs.empty_ext())) { + if (!(parse_message_addr(cs, tuple, st->get_global_version()) && cs.empty_ext())) { if (quiet) { stack.push_bool(false); return 0; @@ -2026,9 +2144,9 @@ bool load_var_integer_q(CellSlice& cs, td::RefInt256& res, int len_bits, bool sg bool load_coins_q(CellSlice& cs, td::RefInt256& res, bool quiet) { return load_var_integer_q(cs, res, 4, false, quiet); } -bool load_msg_addr_q(CellSlice& cs, CellSlice& res, bool quiet) { +bool load_msg_addr_q(CellSlice& cs, CellSlice& res, int global_version, bool quiet) { res = cs; - if (!skip_message_addr(cs)) { + if (!skip_message_addr(cs, global_version)) { cs = res; if (quiet) { return false; @@ -2038,10 +2156,11 @@ bool load_msg_addr_q(CellSlice& cs, CellSlice& res, bool quiet) { res.cut_tail(cs); return true; } -bool parse_std_addr_q(CellSlice cs, ton::WorkchainId& res_wc, ton::StdSmcAddress& res_addr, bool quiet) { +bool parse_std_addr_q(CellSlice cs, ton::WorkchainId& res_wc, ton::StdSmcAddress& res_addr, int global_version, + bool quiet) { // Like exec_rewrite_message_addr, but for std address case std::vector tuple; - if (!(parse_message_addr(cs, tuple) && cs.empty_ext())) { + if (!(parse_message_addr(cs, tuple, global_version) && cs.empty_ext())) { if (quiet) { return false; } @@ -2076,14 +2195,14 @@ td::RefInt256 load_var_integer(CellSlice& cs, int len_bits, bool sgnd) { td::RefInt256 load_coins(CellSlice& cs) { return load_var_integer(cs, 4, false); } -CellSlice load_msg_addr(CellSlice& cs) { +CellSlice load_msg_addr(CellSlice& cs, int global_version) { CellSlice addr; - load_msg_addr_q(cs, addr, false); + load_msg_addr_q(cs, addr, global_version, false); return addr; } -std::pair parse_std_addr(CellSlice cs) { +std::pair parse_std_addr(CellSlice cs, int global_version) { std::pair res; - parse_std_addr_q(std::move(cs), res.first, res.second, false); + parse_std_addr_q(std::move(cs), res.first, res.second, global_version, false); return res; } diff --git a/crypto/vm/tonops.h b/crypto/vm/tonops.h index bbac078f27..d335050858 100644 --- a/crypto/vm/tonops.h +++ b/crypto/vm/tonops.h @@ -32,14 +32,15 @@ namespace util { // "_q" functions throw on error if not quiet, return false if quiet (leaving cs unchanged) bool load_var_integer_q(CellSlice& cs, td::RefInt256& res, int len_bits, bool sgnd, bool quiet); bool load_coins_q(CellSlice& cs, td::RefInt256& res, bool quiet); -bool load_msg_addr_q(CellSlice& cs, CellSlice& res, bool quiet); -bool parse_std_addr_q(CellSlice cs, ton::WorkchainId& res_wc, ton::StdSmcAddress& res_addr, bool quiet); +bool load_msg_addr_q(CellSlice& cs, CellSlice& res, int global_version, bool quiet); +bool parse_std_addr_q(CellSlice cs, ton::WorkchainId& res_wc, ton::StdSmcAddress& res_addr, int global_version, + bool quiet); // Non-"_q" functions throw on error td::RefInt256 load_var_integer(CellSlice& cs, int len_bits, bool sgnd); td::RefInt256 load_coins(CellSlice& cs); -CellSlice load_msg_addr(CellSlice& cs); -std::pair parse_std_addr(CellSlice cs); +CellSlice load_msg_addr(CellSlice& cs, int global_version); +std::pair parse_std_addr(CellSlice cs, int global_version); // store_... functions throw on error if not quiet, return false if quiet (leaving cb unchanged) bool store_var_integer(CellBuilder& cb, const td::RefInt256& x, int len_bits, bool sgnd, bool quiet = false); diff --git a/crypto/vm/vm.cpp b/crypto/vm/vm.cpp index 3c1118c603..33d37973b7 100644 --- a/crypto/vm/vm.cpp +++ b/crypto/vm/vm.cpp @@ -656,12 +656,12 @@ bool VmState::register_library_collection(Ref lib) { } void VmState::register_cell_load(const CellHash& cell_hash) { - if (cell_load_gas_price == cell_reload_gas_price) { - consume_gas(cell_load_gas_price); - } else { - auto ok = loaded_cells.insert(cell_hash); // check whether this is the first time this cell is loaded - consume_gas(ok.second ? cell_load_gas_price : cell_reload_gas_price); - } + auto new_cell = loaded_cells.insert(cell_hash).second; // check whether this is the first time this cell is loaded + consume_gas(new_cell ? cell_load_gas_price : cell_reload_gas_price); +} + +bool VmState::register_cell_load_free(const CellHash& cell_hash) { + return loaded_cells.insert(cell_hash).second; } void VmState::register_cell_create() { @@ -708,17 +708,27 @@ Ref lookup_library_in(td::ConstBitPtr key, Ref lib_root) { void VmState::run_child_vm(VmState&& new_state, bool return_data, bool return_actions, bool return_gas, bool isolate_gas, int ret_vals) { - new_state.log = std::move(log); - new_state.libraries = std::move(libraries); + if (global_version < 10) { + new_state.log = std::move(log); + new_state.libraries = std::move(libraries); + } new_state.stack_trace = stack_trace; new_state.max_data_depth = max_data_depth; if (!isolate_gas) { new_state.loaded_cells = std::move(loaded_cells); } else { - consume_gas(std::min(chksgn_counter, chksgn_free_count) * chksgn_gas_price); + consume_gas(free_gas_consumed); chksgn_counter = 0; + get_extra_balance_counter = 0; + free_gas_consumed = 0; + } + if (global_version >= 10) { + new_state.log = std::move(log); + new_state.libraries = std::move(libraries); } new_state.chksgn_counter = chksgn_counter; + new_state.free_gas_consumed = free_gas_consumed; + new_state.get_extra_balance_counter = get_extra_balance_counter; auto new_parent = std::make_unique(); new_parent->return_data = return_data; @@ -743,6 +753,8 @@ void VmState::restore_parent_vm(int res) { loaded_cells = std::move(child_state.loaded_cells); } chksgn_counter = child_state.chksgn_counter; + get_extra_balance_counter = child_state.get_extra_balance_counter; + free_gas_consumed = child_state.free_gas_consumed; VM_LOG(this) << "Child VM finished. res: " << res << ", steps: " << child_state.steps << ", gas: " << child_state.gas_consumed(); diff --git a/crypto/vm/vm.h b/crypto/vm/vm.h index a171ef27e6..a0ca4a4b3b 100644 --- a/crypto/vm/vm.h +++ b/crypto/vm/vm.h @@ -103,6 +103,8 @@ class VmState final : public VmStateInterface { td::uint16 max_data_depth = 512; // Default value int global_version{0}; size_t chksgn_counter = 0; + size_t get_extra_balance_counter = 0; + long long free_gas_consumed = 0; std::unique_ptr parent = nullptr; public: @@ -161,7 +163,10 @@ class VmState final : public VmStateInterface { bls_g2_multiexp_coef2_gas_price = 22840, bls_pairing_base_gas_price = 20000, - bls_pairing_element_gas_price = 11800 + bls_pairing_element_gas_price = 11800, + + get_extra_balance_cheap_count = 5, + get_extra_balance_cheap_max_gas_price = 200 }; VmState(); VmState(Ref _code, int global_version, Ref _stack, const GasLimits& _gas, int flags = 0, Ref _data = {}, @@ -214,6 +219,9 @@ class VmState final : public VmStateInterface { consume_stack_gas((unsigned)stk->depth()); } } + void consume_free_gas(long long amount) { + free_gas_consumed += amount; + } GasLimits get_gas_limits() const { return gas; } @@ -226,6 +234,7 @@ class VmState final : public VmStateInterface { Ref load_library( td::ConstBitPtr hash) override; // may throw a dictionary exception; returns nullptr if library is not found void register_cell_load(const CellHash& cell_hash) override; + bool register_cell_load_free(const CellHash& cell_hash); void register_cell_create() override; bool init_cp(int new_cp); bool set_cp(int new_cp); @@ -420,9 +429,15 @@ class VmState final : public VmStateInterface { ++chksgn_counter; if (chksgn_counter > chksgn_free_count) { consume_gas(chksgn_gas_price); + } else { + consume_free_gas(chksgn_gas_price); } } } + bool register_get_extra_balance_call() { + ++get_extra_balance_counter; + return get_extra_balance_counter <= get_extra_balance_cheap_count; + } private: void init_cregs(bool same_c3 = false, bool push_0 = true); diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index c0be0b108a..c3205f3c8a 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -3,7 +3,9 @@ Global version is a parameter specified in `ConfigParam 8` ([block.tlb](https:// Various features are enabled depending on the global version. ## Version 4 -New features of version 4 are desctibed in detail in [the documentation](https://docs.ton.org/v3/documentation/tvm/changelog/tvm-upgrade-2023-07). +__Enabled in mainnet on 2023-12-20__ + +New features of version 4 are described in detail in [the documentation](https://docs.ton.org/v3/documentation/tvm/changelog/tvm-upgrade-2023-07). ### New TVM instructions * `PREVMCBLOCKS`, `PREVKEYBLOCK` @@ -40,6 +42,7 @@ intermediate value before division (e.g. `(xy+w)/z`). * Unpaid storage fee is now saved to `due_payment` ## Version 5 +__Enabled in mainnet on 2024-02-03__ ### Gas limits Version 5 enables higher gas limits for special contracts. @@ -57,6 +60,7 @@ See [this post](https://t.me/tonstatus/88) for details. * `XLOAD` now works differently. When it takes a library cell, it returns the cell that it points to. This allows loading "nested libraries", if needed. ## Version 6 +__Enabled in mainnet on 2024-03-16__ ### c7 tuple **c7** tuple extended from 14 to 17 elements: @@ -101,10 +105,12 @@ Operations for working with Merkle proofs, where cells can have non-zero level a ## Version 7 +__Enabled in mainnet on 2024-04-18__ [Explicitly nullify](https://github.com/ton-blockchain/ton/pull/957/files) `due_payment` after due reimbursment. ## Version 8 +__Enabled in mainnet on 2024-08-25__ - Check mode on invalid `action_send_msg`. Ignore action if `IGNORE_ERROR` (+2) bit is set, bounce if `BOUNCE_ON_FAIL` (+16) bit is set. - Slightly change random seed generation to fix mix of `addr_rewrite` and `addr`. @@ -113,6 +119,7 @@ Operations for working with Merkle proofs, where cells can have non-zero level a - Don't use user-provided `fwd_fee` and `ihr_fee` for internal messages. ## Version 9 +__Enabled in mainnet on 2025-02-13__ ### c7 tuple c7 tuple parameter number **13** (previous blocks info tuple) now has the third element. It contains ids of the 16 last masterchain blocks with seqno divisible by 100. @@ -152,12 +159,63 @@ Example: if the last masterchain block seqno is `19071` then the list contains b - `SENDMSG` does not check the number of extra currencies. - Extra currency dictionary is not counted in the account size and does not affect storage fees. - Accounts with already existing extra currencies will get their sizes recomputed without EC only after modifying `AccountState`. +- Reserve action cannot reserve extra currencies. +Reserve modes `+1`, `+4` and `+8` ("reserve all except", "add original balance" and "negate amount") now only affect TONs, but not extra currencies. + +### Anycast addresses and address rewrite +- Anycast addresses are not allowed in `dest` of internal and external messages. +- `addr_var` are not allowed in `dest` of external messages. + - Note: as before, `addr_var` in `dest` of internal messages are automatically replaced with `addr_std`. +- TVM instructions `LDMSGADDR(Q)`, `PARSEMSGADDR(Q)`, `REWRITESTDADDR(Q)`, `REWRITEVARADDR(Q)` no more support anycast addresses and `addr_var`. +- `addr:MsgAddressInt` in `Account` cannot be an anycast address. + - Therefore, `src` of outbound messages cannot be an anycast address. + - Existing accounts with anycast addresses change to non-anycast addresses in the first transaction. +- When deploying an account with `fixed_prefix_length` in `StateInit` of the message (it was called `split_depth` before), the first `fixed_prefix_length` bits of the address are not compared against the state hash. + - This allows deploying an account to an arbitrary shard regardless of the hash of state init. + - `fixed_prefix_length` remains in the account state. + - `fixed_prefix_length` of the account can be at most 8. The limit can be changed in size limits config (`ConfigParam 43`). ### TVM changes - `SENDMSG` calculates messages size and fees without extra currencies, uses new +64 and +128 mode behavior. - `SENDMSG` does not check the number of extra currencies. +- New instruction `GETEXTRABALANCE` (`id - amount`). Takes id of the extra currency (integer in range `0..2^32-1`), returns the amount of this extra currency on the account balance. + - This is equivalent to taking the extra currency dictionary (`BALANCE SECOND`), loading value (`UDICTGET`) and parsing it (`LDVARUINT32`). If `id` is not present in the dictionary, `0` is returned. + - `GETEXTRABALANCE` has special gas cost that allows writing gas-efficient code with predictable gas usage even if there are a lot of different extra currencies. + - The full gas cost of `GETEXTRABALANCE` is `26` (normal instruction cost) plus gas for loading cells (up to `3300` if the dictionary has maximum depth). + - However, the first `5` executions of `GETEXTRABALANCE` cost at most `26+200` gas units. All subsequent executions cost the full price. + - `RUNVM` interacts with this instructions in the following way: + - Without "isolate gas" mode, the child VM shares `GETEXTRABALANCE` counter with the parent vm. + - With "isolate gas" mode, in the beginning of `RUNVM` the parent VM spends full gas for all already executed `GETEXTRABALANCE` and resets the counter. +- `LDMSGADDR(Q)`, `PARSEMSGADDR(Q)`, `REWRITESTDADDR(Q)`, `REWRITEVARADDR(Q)` no more support anycast addresses and `addr_var`. +- Fixed bug in `RUNVM` caused by throwing out-of-gas exception with "isolate gas" enabled. + +### Other changes +- Exceeding state limits in transaction now reverts `end_lt` back to `start_lt + 1` and collects action fines. ## Version 11 + +### c7 tuple +**c7** tuple extended from 17 to 18 elements: +* **17**: tuple with inbound message parameters. Asm opcode: `INMSGPARAMS`. + * The tuple contains: + * `bounce` (boolean) + * `bounced` (boolean) + * `src_addr` (slice) + * `fwd_fee` (int) + * `created_lt` (int) + * `created_at` (int) + * `orig_value` (int) - this is sometimes different from the value in `INCOMINGVALUE` and TVM stack because of storage fees + * `value` (int) - same as in `INCOMINGVALUE` and TVM stack. + * `value_extra` (cell or null) - same as in `INCOMINGVALUE`. + * `state_init` (cell or null) + * For external messages, tick-tock transactions and get methods: `bounce`, `bounced`, `fwd_fee`, `created_lt`, `created_at`, `orig_value`, `value` are 0, `value_extra` is null. + * For tick-tock transactions and get methods: `src_addr` is `addr_none`. + +### New TVM instructions +- `x GETPARAMLONG` - same as `x GETPARAM`, but `x` is in range `[0..254]`. Gas cost: `34`. +- `x INMSGPARAM` - equivalent to `INMSGPARAMS` `x INDEX`. Gas cost: `26`. + - Aliases: `INMSG_BOUNCE`, `INMSG_BOUNCED`, `INMSG_SRC`, `INMSG_FWDFEE`, `INMSG_LT`, `INMSG_UTIME`, `INMSG_ORIGVALUE`, `INMSG_VALUE`, `INMSG_VALUEEXTRA`, `INMSG_STATEINIT`. + ### New account storage stat Along with the storage stat (cells and bits count), each account now stores the hash of the **storage dict**. diff --git a/test/regression-tests.ans b/test/regression-tests.ans index 14d5958b2e..22e7826eac 100644 --- a/test/regression-tests.ans +++ b/test/regression-tests.ans @@ -16,6 +16,7 @@ Test_Fift_test_dict_default a9c8cbcfdece5573185022cea07f59f1bc404e5d879e5157a574 Test_Fift_test_disasm_default 412cf37d37c5d9d81f44dbf4e3d3e7cda173c23b890614eb8a3bc5f2b92f13e6 Test_Fift_test_fiftext_default 2b0db5d4d4bfbc705b959cc787540d7b3a21a71469eac54756e76953f0d9afca Test_Fift_test_fixed_default 278a19d56b773102caf5c1fe2997ea6c8d0d9e720eff8503feede6398a197eec +Test_Fift_test_get_extra_balance_default cc2428172b660ae66489e1b4868786654195137cc29524022cd1cfcf6708336d Test_Fift_test_hash_ext_default 686fc5680feca5b3bb207768215b27f6872a95128762dee0d7f2c88bc492d62d Test_Fift_test_hmap_default c269246882039824bb5822e896c3e6e82ef8e1251b6b251f5af8ea9fb8d05067 Test_Fift_test_levels_default 9fba4a7c98aec9000f42846d6e5fd820343ba61d68f9139dd16c88ccda757cf3 diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index fe7af04e77..fc8735d318 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -606,9 +606,6 @@ void CellDbIn::gc_cont2(BlockHandle handle) { if (r_cell.is_ok()) { cell = r_cell.move_as_ok(); boc_->dec(cell); - LOG(ERROR) << "GC of " << handle->id().to_str(); - } else { - LOG(ERROR) << "GC of UNKNOWN root: " << handle->id().to_str(); } db_busy_ = true; @@ -806,7 +803,6 @@ void CellDb::load_cell(RootHash hash, td::Promise> promise if (result.is_ok()) { return async_apply("load_cell_result", std::move(promise), std::move(result)); } else { - LOG(ERROR) << "load_root_thread_safe failed - this is suspicious"; send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); return; } diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 8457d234fa..e9592f6264 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -1432,13 +1432,16 @@ static td::Ref prepare_vm_c7(ton::UnixTime now, ton::LogicalTime lt, if (config && config->get_global_version() >= 6) { tuple.push_back(vm::StackEntry::maybe(config->get_unpacked_config_tuple(now))); // unpacked_config_tuple:[...] tuple.push_back(due_payment); // due_payment:Integer - // precomiled_gas_usage:(Maybe Integer) + // precompiled_gas_usage:(Maybe Integer) td::optional precompiled; if (my_code.not_null()) { precompiled = config->get_precompiled_contracts_config().get_contract(my_code->get_hash().bits()); } tuple.push_back(precompiled ? td::make_refint(precompiled.value().gas_usage) : vm::StackEntry()); } + if (config && config->get_global_version() >= 11) { + tuple.push_back(block::transaction::Transaction::prepare_in_msg_params_tuple(nullptr, {}, {})); + } auto tuple_ref = td::make_cnt_ref>(std::move(tuple)); LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple_ref).to_string(); return vm::make_tuple_ref(std::move(tuple_ref)); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 5f2ba66151..88ae031e2c 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -977,6 +977,7 @@ bool ValidateQuery::fetch_config_params() { compute_phase_cfg_.size_limits = size_limits; compute_phase_cfg_.precompiled_contracts = config_->get_precompiled_contracts_config(); compute_phase_cfg_.allow_external_unfreeze = compute_phase_cfg_.global_version >= 8; + compute_phase_cfg_.disable_anycast = config_->get_global_version() >= 10; } { // compute action_phase_cfg @@ -1005,9 +1006,11 @@ bool ValidateQuery::fetch_config_params() { action_phase_cfg_.reserve_extra_enabled = config_->get_global_version() >= 9; action_phase_cfg_.mc_blackhole_addr = config_->get_burning_config().blackhole_addr; action_phase_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; + action_phase_cfg_.disable_anycast = config_->get_global_version() >= 10; } { serialize_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; + serialize_cfg_.disable_anycast = config_->get_global_version() >= 10; serialize_cfg_.store_storage_dict_hash = config_->get_global_version() >= 11; } { From 982c38a1fed43b710bb4efa4449f0b4c449a67c3 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 21 Apr 2025 12:20:54 +0300 Subject: [PATCH 220/524] Increase default broadcast speed multiplier to 3.33 (#1631) --- validator-engine/validator-engine.cpp | 6 +++--- validator-engine/validator-engine.hpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 6f47f77417..5829027829 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -4588,7 +4588,7 @@ int main(int argc, char *argv[]) { }); p.add_checked_option( '\0', "broadcast-speed-catchain", - "multiplier for broadcast speed in catchain overlays (experimental, default is 1.0, which is ~300 KB/s)", + "multiplier for broadcast speed in catchain overlays (experimental, default is 3.33, which is ~1 MB/s)", [&](td::Slice s) -> td::Status { auto v = td::to_double(s); if (v <= 0.0) { @@ -4600,7 +4600,7 @@ int main(int argc, char *argv[]) { }); p.add_checked_option( '\0', "broadcast-speed-public", - "multiplier for broadcast speed in public shard overlays (experimental, default is 1.0, which is ~300 KB/s)", + "multiplier for broadcast speed in public shard overlays (experimental, default is 3.33, which is ~1 MB/s)", [&](td::Slice s) -> td::Status { auto v = td::to_double(s); if (v <= 0.0) { @@ -4612,7 +4612,7 @@ int main(int argc, char *argv[]) { }); p.add_checked_option( '\0', "broadcast-speed-private", - "multiplier for broadcast speed in private block overlays (experimental, default is 1.0, which is ~300 KB/s)", + "multiplier for broadcast speed in private block overlays (experimental, default is 3.33, which is ~1 MB/s)", [&](td::Slice s) -> td::Status { auto v = td::to_double(s); if (v <= 0.0) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 77030d68be..de6c3ce193 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -231,9 +231,9 @@ class ValidatorEngine : public td::actor::Actor { bool not_all_shards_ = false; std::vector add_shard_cmds_; bool state_serializer_disabled_flag_ = false; - double broadcast_speed_multiplier_catchain_ = 1.0; - double broadcast_speed_multiplier_public_ = 1.0; - double broadcast_speed_multiplier_private_ = 1.0; + double broadcast_speed_multiplier_catchain_ = 3.33; + double broadcast_speed_multiplier_public_ = 3.33; + double broadcast_speed_multiplier_private_ = 3.33; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; From d065e51d90e1bd845983acd8e1155a79b757250e Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 21 Apr 2025 15:05:14 +0300 Subject: [PATCH 221/524] Update Changelog (#1633) --- Changelog.md | 12 ++++++++++++ recent_changelog.md | 23 +++++++++++------------ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/Changelog.md b/Changelog.md index 4dce39fcfd..344b26cd3e 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,15 @@ +## 2025.04 Update + +1. Introduced substantial improvements of CellDB performance: celldb-v2, bloom filters. +2. Accelerated a number of intrinsic node operations: SHA256, cell operations, large boc serialization, validator set checks. +3. [TVM version v10](./doc/GlobalVersions.md) +4. Overlay broadcast speed up and improved network stats. +5. Fixed some issues in tonlib +6. [Added normalized hash](https://github.com/ton-blockchain/TEPs/pull/467) +7. Fix SDBEGINS(Q) in Asm.fif + +Besides the work of the core team, this update is based on the efforts of @Stanislav-Povolotsky (tonlib fixes), @ice-charon (tonlib fixes), RSquad team (due payments improvements in v10), Arayz @ TonBit (improvements in RUNVM), @Skydev0h and @pyAndr3w (Asm.fif). + ## 2025.03 Update 1. New extracurrency behavior introduced, check [GlobalVersions.md](./doc/GlobalVersions.md#version-10) 2. Optmization of validation process, in particular CellStorageStat. diff --git a/recent_changelog.md b/recent_changelog.md index 820d2aa423..5589603184 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,13 +1,12 @@ -## 2025.03 Update -1. New extracurrency behavior introduced, check [GlobalVersions.md](./doc/GlobalVersions.md#version-10) -2. Optmization of validation process, in particular CellStorageStat. -3. Flag for speeding up broadcasts in various overlays. -4. Fixes for static builds for emulator and tonlibjson -5. Improving getstats output: add - * Liteserver queries count - * Collated/validated blocks count, number of active sessions - * Persistent state sizes - * Initial sync progress -6. Fixes in logging, TON Storage, external message checking, persistent state downloading, UB in tonlib +## 2025.04 Update + +1. Introduced substantial improvements of CellDB performance: celldb-v2, bloom filters. +2. Accelerated a number of intrinsic node operations: SHA256, cell operations, large boc serialization, validator set checks. +3. [TVM version v10](./doc/GlobalVersions.md) +4. Overlay broadcast speed up and improved network stats. +5. Fixed some issues in tonlib +6. [Added normalized hash](https://github.com/ton-blockchain/TEPs/pull/467) +7. Fix SDBEGINS(Q) in Asm.fif + +Besides the work of the core team, this update is based on the efforts of @Stanislav-Povolotsky (tonlib fixes), @ice-charon (tonlib fixes), RSquad team (due payments improvements in v10), Arayz @ TonBit (improvements in RUNVM), @Skydev0h and @pyAndr3w (Asm.fif). -Besides the work of the core team, this update is based on the efforts of @Sild from StonFi(UB in tonlib). From 1835d84602bbaaa1593270d7ab3bb0b499920416 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 22 Apr 2025 20:14:07 +0300 Subject: [PATCH 222/524] Fixed setting gas limits in RUNVM (#1635) --- crypto/vm/vm.cpp | 4 ++++ doc/GlobalVersions.md | 1 + 2 files changed, 5 insertions(+) diff --git a/crypto/vm/vm.cpp b/crypto/vm/vm.cpp index 33d37973b7..272e3930e6 100644 --- a/crypto/vm/vm.cpp +++ b/crypto/vm/vm.cpp @@ -729,6 +729,10 @@ void VmState::run_child_vm(VmState&& new_state, bool return_data, bool return_ac new_state.chksgn_counter = chksgn_counter; new_state.free_gas_consumed = free_gas_consumed; new_state.get_extra_balance_counter = get_extra_balance_counter; + if (global_version >= 10) { + new_state.gas = GasLimits{std::min(new_state.gas.gas_limit, gas.gas_remaining), + std::min(new_state.gas.gas_max, gas.gas_remaining)}; + } auto new_parent = std::make_unique(); new_parent->return_data = return_data; diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index c3205f3c8a..7bd9b0cc98 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -188,6 +188,7 @@ Reserve modes `+1`, `+4` and `+8` ("reserve all except", "add original balance" - With "isolate gas" mode, in the beginning of `RUNVM` the parent VM spends full gas for all already executed `GETEXTRABALANCE` and resets the counter. - `LDMSGADDR(Q)`, `PARSEMSGADDR(Q)`, `REWRITESTDADDR(Q)`, `REWRITEVARADDR(Q)` no more support anycast addresses and `addr_var`. - Fixed bug in `RUNVM` caused by throwing out-of-gas exception with "isolate gas" enabled. +- Fixed setting gas limits in `RUNVM` after consuming "free gas" (e.g. after `CHKSIGN` instructions). ### Other changes - Exceeding state limits in transaction now reverts `end_lt` back to `start_lt + 1` and collects action fines. From e03b11025e74b642a9aa7c46594f3799dcf3951f Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 22 Apr 2025 20:14:57 +0300 Subject: [PATCH 223/524] Don't delete accounts with remaining balance (#1632) --- crypto/block/transaction.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index c9890203f5..b892b5a216 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -3295,6 +3295,9 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { acc_status = Account::acc_nonexist; was_created = false; } + if (acc_status == Account::acc_deleted && !balance.is_zero()) { + acc_status = Account::acc_uninit; + } if (acc_status == Account::acc_nonexist || acc_status == Account::acc_deleted) { CHECK(balance.is_zero()); vm::CellBuilder cb; From 0cc297ba58002a3312f72d787908e0a359e41f60 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 23 Apr 2025 21:47:10 +0300 Subject: [PATCH 224/524] Disable validator set cache during Merkle proof generation (#1636) --- crypto/block/mc-config.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index d51f20599f..78e2294f88 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -542,6 +542,11 @@ td::Result> Config::unpack_validator_set(Refload_cell()); + if (!loaded_root.tree_node.empty()) { + // Do not use cache during Merkle proof generation + use_cache = false; + } static ValidatorSetCache cache; if (use_cache) { auto result = cache.get(vset_root->get_hash()); From 413b89837a3ae3a517d2de9372e298f6efa44818 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Mon, 28 Apr 2025 10:15:18 +0400 Subject: [PATCH 225/524] Deprecate Ubuntu 20.04 (#1641) * not always gh mac runners have nproc utility * deprecate Ubuntu 20.04 * fix linking issues of portable tonlibjson.dylib * fix libemulator.dylib linking issues --- .../build-ton-linux-arm64-appimage.yml | 14 ++++++------- .../build-ton-linux-x86-64-appimage.yml | 20 +++++++------------ .../build-ton-linux-x86-64-shared.yml | 9 +-------- assembly/native/build-macos-portable.sh | 1 - assembly/native/build-macos-shared.sh | 1 - emulator/CMakeLists.txt | 11 ++++++++++ tonlib/CMakeLists.txt | 11 ++++++++++ 7 files changed, 37 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build-ton-linux-arm64-appimage.yml b/.github/workflows/build-ton-linux-arm64-appimage.yml index 289cd2a7e2..a0ba4a022c 100644 --- a/.github/workflows/build-ton-linux-arm64-appimage.yml +++ b/.github/workflows/build-ton-linux-arm64-appimage.yml @@ -36,21 +36,21 @@ jobs: uses: actions/cache@v4 with: path: 3pp - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-3pp-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-3pp-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} - name: Cache OpenSSL id: cache-openssl uses: actions/cache@v4 with: path: openssl_3 - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} - name: Restore cache TON uses: actions/cache/restore@v4 with: path: ~/.ccache - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} - restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-portable-ccache - name: Build TON run: | @@ -65,7 +65,7 @@ jobs: uses: actions/cache/save@v4 with: path: ~/.ccache - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} - name: Make AppImages run: | @@ -80,8 +80,8 @@ jobs: uses: actions/cache@v4 with: path: ~/.ccache - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-libs-ccache-${{ steps.date-stamp.outputs.timestamp }} - restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-libs-ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-portable-libs-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-portable-libs-ccache - name: Build TON libs run: | diff --git a/.github/workflows/build-ton-linux-x86-64-appimage.yml b/.github/workflows/build-ton-linux-x86-64-appimage.yml index 0f5645a3ad..43d9892432 100644 --- a/.github/workflows/build-ton-linux-x86-64-appimage.yml +++ b/.github/workflows/build-ton-linux-x86-64-appimage.yml @@ -4,7 +4,7 @@ on: [push,workflow_dispatch,workflow_call] jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Check out repository @@ -25,12 +25,6 @@ jobs: sudo apt remove libgsl-dev mkdir ~/.ccache 3pp - - name: Install gcc-11 g++-11 - run: | - sudo apt install -y manpages-dev software-properties-common - sudo add-apt-repository ppa:ubuntu-toolchain-r/test - sudo apt update && sudo apt install gcc-11 g++-11 - - name: Install clang-16 run: | wget https://apt.llvm.org/llvm.sh @@ -42,20 +36,20 @@ jobs: uses: actions/cache@v4 with: path: 3pp - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-3pp-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-3pp-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} - name: Cache OpenSSL id: cache-openssl uses: actions/cache@v4 with: path: openssl_3 - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} - name: Restore cache TON uses: actions/cache/restore@v4 with: path: ~/.ccache - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-portable-ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-portable-ccache - name: Build TON run: | @@ -70,7 +64,7 @@ jobs: uses: actions/cache/save@v4 with: path: ~/.ccache - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} - name: Make AppImages run: | @@ -85,8 +79,8 @@ jobs: uses: actions/cache@v4 with: path: ~/.ccache - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-portable-libs-ccache-${{ steps.date-stamp.outputs.timestamp }} - restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-portable-libs-ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-portable-libs-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-portable-libs-ccache - name: Build TON libs run: | diff --git a/.github/workflows/build-ton-linux-x86-64-shared.yml b/.github/workflows/build-ton-linux-x86-64-shared.yml index 37eddf3695..ed05d18aa7 100644 --- a/.github/workflows/build-ton-linux-x86-64-shared.yml +++ b/.github/workflows/build-ton-linux-x86-64-shared.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] + os: [ubuntu-22.04, ubuntu-24.04] runs-on: ${{ matrix.os }} steps: @@ -28,13 +28,6 @@ jobs: sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev ccache mkdir ~/.ccache - - if: matrix.os == 'ubuntu-20.04' - name: Install gcc-11 - run: | - sudo apt install -y manpages-dev software-properties-common - sudo add-apt-repository ppa:ubuntu-toolchain-r/test - sudo apt update && sudo apt install gcc-11 g++-11 - - if: matrix.os != 'ubuntu-24.04' run: | wget https://apt.llvm.org/llvm.sh diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index 2779730886..c46ea416e4 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -127,7 +127,6 @@ fi cmake -GNinja .. \ -DPORTABLE=1 \ -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=$OSX_TARGET \ --DCMAKE_CXX_FLAGS="-stdlib=libc++" \ -DCMAKE_BUILD_TYPE=Release \ -DOPENSSL_FOUND=1 \ -DOPENSSL_INCLUDE_DIR=$opensslPath/include \ diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index 595c44e3e0..a326694907 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -66,7 +66,6 @@ brew install openssl@3 brew unlink openssl@3 && brew link --overwrite openssl@3 cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \ --DCMAKE_CXX_FLAGS="-stdlib=libc++" \ -DLZ4_FOUND=1 \ -DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a \ -DLZ4_INCLUDE_DIRS=$lz4Path/lib diff --git a/emulator/CMakeLists.txt b/emulator/CMakeLists.txt index e1bc38f520..63cb28d6a9 100644 --- a/emulator/CMakeLists.txt +++ b/emulator/CMakeLists.txt @@ -30,6 +30,17 @@ else() target_link_libraries(emulator PUBLIC emulator_static git) endif() +if (APPLE) + set(CMAKE_MACOSX_RPATH ON) + set(CMAKE_CXX_VISIBILITY_PRESET hidden) + set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) +endif() + +if (APPLE AND PORTABLE) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libstdc++") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-libstdc++") +endif() + generate_export_header(emulator EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/emulator_export.h) if (EMULATOR_STATIC OR USE_EMSCRIPTEN) target_compile_definitions(emulator PUBLIC EMULATOR_STATIC_DEFINE) diff --git a/tonlib/CMakeLists.txt b/tonlib/CMakeLists.txt index 2e3c583d43..f86364411e 100644 --- a/tonlib/CMakeLists.txt +++ b/tonlib/CMakeLists.txt @@ -102,6 +102,17 @@ else() target_link_libraries(tonlibjson PRIVATE tonlibjson_private) endif() +if (APPLE) + set(CMAKE_MACOSX_RPATH ON) + set(CMAKE_CXX_VISIBILITY_PRESET hidden) + set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) +endif() + +if (APPLE AND PORTABLE) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libstdc++") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-libstdc++") +endif() + generate_export_header(tonlibjson EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/tonlib/tonlibjson_export.h) if (TONLIBJSON_STATIC OR USE_EMSCRIPTEN) target_compile_definitions(tonlibjson PUBLIC TONLIBJSON_STATIC_DEFINE) From 6a1d8832154be8da8cbbd6aba8d742dab03c0d46 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 28 Apr 2025 18:24:08 +0300 Subject: [PATCH 226/524] Shard block verifier for masterchain validators --- tl/generate/scheme/ton_api.tl | 13 ++ tl/generate/scheme/ton_api.tlo | Bin 112248 -> 113468 bytes .../validator-engine-console-query.cpp | 74 +++++++ .../validator-engine-console-query.h | 60 ++++++ .../validator-engine-console.cpp | 3 + validator-engine/validator-engine.cpp | 100 +++++++++ validator-engine/validator-engine.hpp | 13 ++ validator/CMakeLists.txt | 4 + validator/impl/shard.cpp | 4 +- validator/impl/shard.hpp | 2 +- validator/impl/top-shard-descr.hpp | 3 + validator/impl/validate-query.cpp | 46 +++- validator/impl/validate-query.hpp | 3 +- validator/interfaces/shard-block.h | 1 + validator/interfaces/shard.h | 2 +- validator/interfaces/validator-manager.h | 4 + validator/manager.cpp | 199 ++++++++++++------ validator/manager.hpp | 23 +- validator/shard-block-retainer.cpp | 179 ++++++++++++++++ validator/shard-block-retainer.hpp | 71 +++++++ validator/shard-block-verifier.cpp | 194 +++++++++++++++++ validator/shard-block-verifier.hpp | 90 ++++++++ validator/validator-options.cpp | 28 +++ validator/validator-options.hpp | 7 + validator/validator.h | 17 ++ 25 files changed, 1065 insertions(+), 75 deletions(-) create mode 100644 validator/shard-block-retainer.cpp create mode 100644 validator/shard-block-retainer.hpp create mode 100644 validator/shard-block-verifier.cpp create mode 100644 validator/shard-block-verifier.hpp diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index ac0241a1f4..c5bc4afe3d 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -696,6 +696,11 @@ engine.validator.collatorsList.shard shard_id:tonNode.shardId collators:(vector self_collate:Bool select_mode:string = engine.validator.collatorsList.Shard; engine.validator.collatorsList shards:(vector engine.validator.collatorsList.shard) = engine.validator.CollatorsList; +engine.validator.shardBlockVerifierConfig.shard shard_id:(tonNode.shardId) trusted_nodes:(vector int256) + required_confirms:int = engine.validator.shardBlockVerifierConfig.Shard; +engine.validator.shardBlockVerifierConfig shards:(vector engine.validator.shardBlockVerifierConfig.shard) + = engine.validator.ShardBlockVerifierConfig; + ---functions--- ---types--- @@ -855,6 +860,9 @@ engine.validator.importFastSyncMemberCertificate adnl_id:int256 certificate:over engine.validator.addFastSyncClient adnl_id:int256 slot:int = engine.validator.Success; engine.validator.delFastSyncClient adnl_id:int256 = engine.validator.Success; +engine.validator.setShardBlockVerifierConfig config:engine.validator.shardBlockVerifierConfig = engine.validator.Success; +engine.validator.showShardBlockVerifierConfig = engine.validator.ShardBlockVerifierConfig; + ---types--- storage.pong = storage.Pong; @@ -962,11 +970,16 @@ collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt collatorNode.pong flags:# = collatorNode.Pong; collatorNode.error code:int message:string = collatorNode.Error; +shardBlockVerifier.subscribed flags:# = shardBlockVerifier.Subscribed; +shardBlockVerifier.confirmBlocks blocks:(vector tonNode.blockIdExt) = shardBlockVerifier.ConfirmBlocks; + ---functions--- collatorNode.generateBlock shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt) creator:int256 round:int first_block_round:int priority:int = collatorNode.Candidate; collatorNode.ping flags:# = collatorNode.Pong; +shardBlockVerifier.subscribe shard:tonNode.shardId flags:# = shardBlockVerifier.Subscribed; + ---types--- storage.db.key.torrentList = storage.db.key.TorrentList; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 6708e8f0a610115e169ad65c345c7c9c301102fe..744323b6de215969871ddd09ac1cfb41e3619ed6 100644 GIT binary patch delta 801 zcmezIhHcL`Hr_|G^{p77z;Pq5xSU?IX1}XuYF>I~UaDSMVoqjCVo83HUT{WYQHoPe zesXqLYEfodW@?dheqLH;`owion^olgFqs79oLZ%ip{*E)Hi&r>H%M$&P~5?;dP8|p zz5=FNJ+K``xnO&XCo6JGEArlK;+MsuGPpFUIJqb@DRnZVrS#?-8X4k@=Qle#u(z9F z_YbBK5dSbRfYeVuv)(`g2TP#9Rx{kwmH6o-IhfEg6#w8xp_>;hk3H$ zIc-kflA_Y$lGK#=$%(I&Cp#?XnOqRYA}CmtT3DJ{1QbpN1#nUB>tMky*tye)o%lz;)CTew$OxwCj#Z4xlP{cM Tnf_-oqXyW&+e0QXs;B?}7&0kf delta 87 zcmdn5LL80JNMT8UO$Q diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 711d8968ae..eaece0e874 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1946,3 +1946,77 @@ td::Status DelFastSyncOverlayClientQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "success\n"; return td::Status::OK(); } + +td::Status SetShardBlockVerifierConfigQuery::run() { + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status SetShardBlockVerifierConfigQuery::send() { + TRY_RESULT(data, td::read_file(file_name_)); + TRY_RESULT(json, td::json_decode(data.as_slice())); + auto list = ton::create_tl_object(); + TRY_STATUS(ton::ton_api::from_json(*list, json.get_object())); + auto b = ton::create_serialize_tl_object(std::move(list)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SetShardBlockVerifierConfigQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status ClearShardBlockVerifierConfigQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status ClearShardBlockVerifierConfigQuery::send() { + auto list = ton::create_tl_object(); + auto b = ton::create_serialize_tl_object(std::move(list)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ClearShardBlockVerifierConfigQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status ShowShardBlockVerifierConfigQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status ShowShardBlockVerifierConfigQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ShowShardBlockVerifierConfigQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX( + config, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "Shard block verifier config:\n"; + if (config->shards_.empty()) { + td::TerminalIO::out() << "Config is empty\n"; + return td::Status::OK(); + } + for (const auto &shard : config->shards_) { + td::TerminalIO::out() << "Shard " << create_shard_id(shard->shard_id_).to_str() << "\n"; + td::TerminalIO::out() << " Required confirms = " << shard->required_confirms_ << "/" + << shard->trusted_nodes_.size() << "\n"; + td::TerminalIO::out() << " Trusted nodes:\n"; + for (const auto &node : shard->trusted_nodes_) { + td::TerminalIO::out() << " " << node << "\n"; + } + } + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index ad9f7c3b3d..c06924d9a1 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1747,3 +1747,63 @@ class DelFastSyncOverlayClientQuery : public Query { private: td::Bits256 adnl_id_; }; + +class SetShardBlockVerifierConfigQuery : public Query { + public: + SetShardBlockVerifierConfigQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "set-shard-block-verifier-config"; + } + static std::string get_help() { + return "set-shard-block-verifier-config \tset config for shard block verifier from file "; + } + std::string name() const override { + return get_name(); + } + + private: + std::string file_name_; +}; + +class ClearShardBlockVerifierConfigQuery : public Query { + public: + ClearShardBlockVerifierConfigQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "clear-shard-block-verifier-config"; + } + static std::string get_help() { + return "clear-shard-block-verifier-config \treset config for shard block verifier"; + } + std::string name() const override { + return get_name(); + } +}; + +class ShowShardBlockVerifierConfigQuery : public Query { + public: + ShowShardBlockVerifierConfigQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "show-shard-block-verifier-config"; + } + static std::string get_help() { + return "show-shard-block-verifier-config\tshow config of shard block verifier"; + } + std::string name() const override { + return get_name(); + } +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 9520dd111d..a747a648f1 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -170,6 +170,9 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index ea137a94be..223ba78643 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1708,6 +1708,36 @@ void ValidatorEngine::load_collators_list() { } } +void ValidatorEngine::load_shard_block_verifier_config() { + shard_block_verifier_config_ = {}; + auto data_R = td::read_file(shard_block_verifier_config_file()); + if (data_R.is_error()) { + return; + } + auto data = data_R.move_as_ok(); + auto json_R = td::json_decode(data.as_slice()); + if (json_R.is_error()) { + LOG(ERROR) << "Failed to parse shard block verifier config: " << json_R.move_as_error(); + return; + } + auto json = json_R.move_as_ok(); + shard_block_verifier_config_ = ton::create_tl_object(); + auto S = ton::ton_api::from_json(*shard_block_verifier_config_, json.get_object()); + if (S.is_error()) { + LOG(ERROR) << "Failed to parse shard block verifier config: " << S; + shard_block_verifier_config_ = {}; + return; + } + td::Ref config{true}; + S = config.write().unpack(*shard_block_verifier_config_); + if (S.is_ok()) { + validator_options_.write().set_shard_block_verifier_config(std::move(config)); + } else { + LOG(ERROR) << "Invalid shard block verifier config: " << S.move_as_error(); + shard_block_verifier_config_ = {}; + } +} + void ValidatorEngine::load_empty_local_config(td::Promise promise) { auto ret_promise = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { @@ -2018,6 +2048,7 @@ void ValidatorEngine::got_key(ton::PublicKey key) { void ValidatorEngine::start() { set_shard_check_function(); load_collators_list(); + load_shard_block_verifier_config(); read_config_ = true; start_adnl(); } @@ -2153,6 +2184,15 @@ void ValidatorEngine::start_validator() { } } + if (!shard_block_retainer_adnl_id_.is_zero()) { + if (config_.adnl_ids.contains(shard_block_retainer_adnl_id_.pubkey_hash())) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_shard_block_retainer, + shard_block_retainer_adnl_id_); + } else { + LOG(ERROR) << "Cannot start shard block retainer: no adnl id " << shard_block_retainer_adnl_id_; + } + } + started_validator(); } @@ -4974,6 +5014,55 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delFastSy }); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setShardBlockVerifierConfig &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + td::Ref config{true}; + auto S = config.write().unpack(*query.config_); + if (S.is_error()) { + promise.set_value(create_control_query_error(S.move_as_error_prefix("Invalid config: "))); + return; + } + auto s = td::json_encode(td::ToJson(*query.config_), true); + S = td::write_file(shard_block_verifier_config_file(), s); + if (S.is_error()) { + promise.set_value(create_control_query_error(std::move(S))); + return; + } + shard_block_verifier_config_ = std::move(query.config_); + validator_options_.write().set_shard_block_verifier_config(std::move(config)); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showShardBlockVerifierConfig &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + if (shard_block_verifier_config_) { + promise.set_value(ton::serialize_tl_object(shard_block_verifier_config_, true)); + } else { + promise.set_value(create_control_query_error(td::Status::Error("shard block verifier config is empty"))); + } +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { @@ -5461,6 +5550,17 @@ int main(int argc, char *argv[]) { [&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_broadcast_speed_multiplier_private, v); }); return td::Status::OK(); }); + p.add_checked_option( + '\0', "shard-block-retainer", "adnl id for shard block retainer (hex)", [&](td::Slice arg) -> td::Status { + td::Bits256 v; + if (v.from_hex(arg) != 256) { + return td::Status::Error("invalid adnl-id"); + } + acts.push_back([&x, v]() { + td::actor::send_closure(x, &ValidatorEngine::set_shard_block_retainer_adnl_id, ton::adnl::AdnlNodeIdShort{v}); + }); + return td::Status::OK(); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index b5155052c6..1893cf517b 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -193,6 +193,7 @@ class ValidatorEngine : public td::actor::Actor { Config config_; ton::tl_object_ptr custom_overlays_config_; ton::tl_object_ptr collators_list_; + ton::tl_object_ptr shard_block_verifier_config_; std::set running_gc_; @@ -254,6 +255,7 @@ class ValidatorEngine : public td::actor::Actor { double broadcast_speed_multiplier_catchain_ = 3.33; double broadcast_speed_multiplier_public_ = 3.33; double broadcast_speed_multiplier_private_ = 3.33; + ton::adnl::AdnlNodeIdShort shard_block_retainer_adnl_id_ = ton::adnl::AdnlNodeIdShort::zero(); std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -369,6 +371,9 @@ class ValidatorEngine : public td::actor::Actor { void set_broadcast_speed_multiplier_private(double value) { broadcast_speed_multiplier_private_ = value; } + void set_shard_block_retainer_adnl_id(ton::adnl::AdnlNodeIdShort id) { + shard_block_retainer_adnl_id_ = id; + } void start_up() override; ValidatorEngine() { @@ -381,6 +386,7 @@ class ValidatorEngine : public td::actor::Actor { void load_config(td::Promise promise); void set_shard_check_function(); void load_collators_list(); + void load_shard_block_verifier_config(); void start(); @@ -479,6 +485,9 @@ class ValidatorEngine : public td::actor::Actor { std::string collators_list_file() const { return db_root_ + "/collators-list.json"; } + std::string shard_block_verifier_config_file() const { + return db_root_ + "/shard-block-verifier-config.json"; + } void load_custom_overlays_config(); td::Status write_custom_overlays_config(); @@ -614,6 +623,10 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_delFastSyncClient &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_setShardBlockVerifierConfig &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_showShardBlockVerifierConfig &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 003d9b1b08..753ae9d29b 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -56,6 +56,8 @@ set(VALIDATOR_HEADERS import-db-slice.hpp queue-size-counter.hpp validator-telemetry.hpp + shard-block-verifier.hpp + shard-block-retainer.hpp collation-manager.hpp collator-node.hpp @@ -87,6 +89,8 @@ set(VALIDATOR_SOURCE validator-options.cpp queue-size-counter.cpp validator-telemetry.cpp + shard-block-verifier.cpp + shard-block-retainer.cpp downloaders/wait-block-data.cpp downloaders/wait-block-state.cpp diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index d2d402dd66..7302b2611c 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -501,11 +501,11 @@ std::vector> MasterchainStateQ::get_shards() const { return v; } -td::Ref MasterchainStateQ::get_shard_from_config(ShardIdFull shard) const { +td::Ref MasterchainStateQ::get_shard_from_config(ShardIdFull shard, bool exact) const { if (!config_) { return {}; } - return config_->get_shard_hash(shard); + return config_->get_shard_hash(shard, exact); } bool MasterchainStateQ::rotated_all_shards() const { diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index b13be02195..1dd6804be4 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -114,7 +114,7 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { Ref get_validator_set(ShardIdFull shard, UnixTime ts, CatchainSeqno cc_seqno) const; bool rotated_all_shards() const override; std::vector> get_shards() const override; - td::Ref get_shard_from_config(ShardIdFull shard) const override; + td::Ref get_shard_from_config(ShardIdFull shard, bool exact) const override; bool ancestor_is_valid(BlockIdExt id) const override { return check_old_mc_block_id(id); } diff --git a/validator/impl/top-shard-descr.hpp b/validator/impl/top-shard-descr.hpp index 5c866a6e33..e1fd5778f0 100644 --- a/validator/impl/top-shard-descr.hpp +++ b/validator/impl/top-shard-descr.hpp @@ -74,6 +74,9 @@ class ShardTopBlockDescrQ final : public ShardTopBlockDescrQBase { std::size_t size() const { return chain_blk_ids_.size(); } + const std::vector& get_chain_blocks() const override { + return chain_blk_ids_; + } UnixTime generated_at() const override { return gen_utime_; } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index aae7183606..dffdf21bc6 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1801,11 +1801,13 @@ void ValidateQuery::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result< * @param sibling The sibling shard information. * @param wc_info The workchain information. * @param ccvc The Catchain validators configuration. + * @param is_new Set to true if the top shard block is new, false if it existed in the previous shard configuration. * * @returns True if the validation wasa successful, false otherwise. */ bool ValidateQuery::check_one_shard(const block::McShardHash& info, const block::McShardHash* sibling, - const block::WorkchainInfo* wc_info, const block::CatchainValidatorsConfig& ccvc) { + const block::WorkchainInfo* wc_info, const block::CatchainValidatorsConfig& ccvc, + bool& is_new) { auto shard = info.shard(); LOG(DEBUG) << "checking shard " << shard.to_str() << " in new shard configuration"; if (info.next_validator_shard_ != shard.shard) { @@ -1814,6 +1816,7 @@ bool ValidateQuery::check_one_shard(const block::McShardHash& info, const block: ton::ShardIdFull{shard.workchain, info.next_validator_shard_}.to_str()); } auto old = old_shard_conf_->get_shard_hash(shard - 1, false); + is_new = (old.is_null() || old->top_block_id() != info.top_block_id()); Ref prev; CatchainSeqno cc_seqno; bool old_before_merge = false, fsm_inherited = false, workchain_created = false; @@ -2113,8 +2116,10 @@ bool ValidateQuery::check_shard_layout() { WorkchainId wc_id{ton::workchainInvalid}; Ref wc_info; - if (!new_shard_conf_->process_sibling_shard_hashes([self = this, &wc_set, &wc_id, &wc_info, &ccvc]( - block::McShardHash& cur, const block::McShardHash* sibling) { + std::vector new_top_shard_blocks; + if (!new_shard_conf_->process_sibling_shard_hashes([self = this, &new_top_shard_blocks, &wc_set, &wc_id, &wc_info, + &ccvc](block::McShardHash& cur, + const block::McShardHash* sibling) { if (!cur.is_valid()) { return -2; } @@ -2131,7 +2136,15 @@ bool ValidateQuery::check_shard_layout() { wc_info = it->second; } } - return self->check_one_shard(cur, sibling, wc_info.get(), ccvc) ? 0 : -1; + bool is_new; + int res = self->check_one_shard(cur, sibling, wc_info.get(), ccvc, is_new); + if (!res) { + return -1; + } + if (is_new) { + new_top_shard_blocks.push_back(cur.top_block_id()); + } + return 0; })) { return reject_query("new shard configuration is invalid"); } @@ -2148,6 +2161,14 @@ bool ValidateQuery::check_shard_layout() { << " is active, but is absent from new shard configuration"); } } + if (!new_top_shard_blocks.empty()) { + ++pending; + td::actor::send_closure(manager, &ValidatorManager::wait_verify_shard_blocks, std::move(new_top_shard_blocks), + [SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &ValidateQuery::verified_shard_blocks, + R.move_as_status()); + }); + } return check_mc_validator_info(is_key_block_ || (now_ / ccvc.mc_cc_lifetime > prev_now_ / ccvc.mc_cc_lifetime)); } @@ -2351,6 +2372,23 @@ void ValidateQuery::got_out_queue_size(size_t i, td::Result res) { try_validate(); } +/** + * Handles the result of ValidatorManager::wait_verify_shard_blocks. + * + * This is called after new top shard blocks were confirmed by trusted nodes. + * + * @param S The status of the operation (OK on success). + */ +void ValidateQuery::verified_shard_blocks(td::Status S) { + --pending; + if (S.is_error()) { + fatal_error(S.move_as_error_prefix("failed to verify shard blocks: ")); + return; + } + LOG(DEBUG) << "Verified shard blocks"; + try_validate(); +} + /* * * METHODS CALLED FROM try_validate() stage 1 diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index e42f885487..9c408d12a5 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -323,7 +323,7 @@ class ValidateQuery : public td::actor::Actor { void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res); bool check_one_shard(const block::McShardHash& info, const block::McShardHash* sibling, - const block::WorkchainInfo* wc_info, const block::CatchainValidatorsConfig& ccvc); + const block::WorkchainInfo* wc_info, const block::CatchainValidatorsConfig& ccvc, bool& is_new); bool check_shard_layout(); bool register_shard_block_creators(std::vector creator_list); bool check_cur_validator_set(); @@ -331,6 +331,7 @@ class ValidateQuery : public td::actor::Actor { bool check_utime_lt(); bool prepare_out_msg_queue_size(); void got_out_queue_size(size_t i, td::Result res); + void verified_shard_blocks(td::Status S); bool fix_one_processed_upto(block::MsgProcessedUpto& proc, ton::ShardIdFull owner, bool allow_cur = false); bool fix_processed_upto(block::MsgProcessedUptoCollection& upto, bool allow_cur = false); diff --git a/validator/interfaces/shard-block.h b/validator/interfaces/shard-block.h index e88a970bba..6e31cc136c 100644 --- a/validator/interfaces/shard-block.h +++ b/validator/interfaces/shard-block.h @@ -35,6 +35,7 @@ class ShardTopBlockDescription : public td::CntObject { virtual bool after_split() const = 0; virtual bool after_merge() const = 0; virtual CatchainSeqno catchain_seqno() const = 0; + virtual const std::vector& get_chain_blocks() const = 0; virtual UnixTime generated_at() const = 0; // if method returns false this shard block description is discarded diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 3546f0a39b..dff077f4c3 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -69,7 +69,7 @@ class MasterchainState : virtual public ShardState { virtual td::Ref get_total_validator_set(int next) const = 0; // next = -1 -> prev, next = 0 -> cur virtual bool rotated_all_shards() const = 0; virtual std::vector> get_shards() const = 0; - virtual td::Ref get_shard_from_config(ShardIdFull shard) const = 0; + virtual td::Ref get_shard_from_config(ShardIdFull shard, bool exact = true) const = 0; virtual bool workchain_is_active(WorkchainId workchain_id) const = 0; virtual td::uint32 monitor_min_split_depth(WorkchainId workchain_id) const = 0; virtual td::uint32 min_split_depth(WorkchainId workchain_id) const = 0; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 65f6f48a89..140f01ff20 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -270,6 +270,10 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void add_persistent_state_description(td::Ref desc) = 0; + virtual void wait_verify_shard_blocks(std::vector blocks, td::Promise promise) { + promise.set_result(td::Unit()); + } + static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 535f638ce0..f68cdf941b 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -42,6 +42,7 @@ #include "td/utils/JsonBuilder.h" #include "common/delay.h" +#include "td/actor/MultiPromise.h" #include "td/utils/filesystem.h" #include "validator/stats-merger.h" @@ -312,8 +313,7 @@ void ValidatorManagerImpl::get_zero_state(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, - std::move(promise)); + td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, std::move(promise)); } void ValidatorManagerImpl::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) { @@ -495,7 +495,8 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - if (!is_validator() && !is_shard_collator(block_id.shard_full()) && !cached_block_candidates_.count(block_id)) { + if (!is_validator() && !is_shard_collator(block_id.shard_full()) && !cached_block_candidates_.count(block_id) && + shard_block_retainers_.empty()) { return; } if (!last_masterchain_block_handle_) { @@ -544,6 +545,9 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refmay_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { return; } + for (auto &[_, actor] : shard_block_retainers_) { + td::actor::send_closure(actor, &ShardBlockRetainer::new_shard_block_description, desc); + } auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second.latest_desc->block_id().id.seqno) { VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; @@ -565,9 +569,22 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id(), 0, td::Timestamp::in(60.0), std::move(P)); } if (validating_masterchain()) { - preload_msg_queue_to_masterchain(desc); + td::MultiPromise mp; + auto ig = mp.init_guard(); + preload_msg_queue_to_masterchain(desc, ig.get_promise()); + wait_verify_shard_blocks({desc->block_id()}, ig.get_promise().wrap([desc](td::Unit) { + VLOG(VALIDATOR_DEBUG) << "verified top shard block " << desc->block_id().to_str(); + return td::Unit{}; + })); + ig.add_promise([SelfId = actor_id(this), desc](td::Result R) mutable { + if (R.is_error()) { + VLOG(VALIDATOR_DEBUG) << "shard block description: " << R.move_as_error(); + return; + } + td::actor::send_closure(SelfId, &ValidatorManagerImpl::set_shard_block_description_ready, std::move(desc)); + }); } - for (auto& [_, collator_node] : collator_nodes_) { + for (auto &[_, collator_node] : collator_nodes_) { if (collator_node.can_collate_shard(desc->shard())) { td::actor::send_closure(collator_node.actor, &CollatorNode::update_validator_group_info, desc->shard(), std::vector{desc->block_id()}, desc->catchain_seqno()); @@ -575,32 +592,48 @@ void ValidatorManagerImpl::add_shard_block_description(td::Ref desc) { +void ValidatorManagerImpl::preload_msg_queue_to_masterchain(td::Ref desc, + td::Promise promise) { + if (!validating_masterchain()) { + promise.set_error(td::Status::Error("not validating masterchain")); + return; + } auto id = ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}; auto it = shard_blocks_.find(id); - if (!validating_masterchain() || it == shard_blocks_.end() || - it->second.latest_desc->block_id() != desc->block_id()) { + if (it == shard_blocks_.end() || it->second.latest_desc->block_id() != desc->block_id()) { + promise.set_error(td::Status::Error("shard block description is outdated")); return; } wait_neighbor_msg_queue_proofs( ShardIdFull{masterchainId}, {desc->block_id()}, td::Timestamp::in(10.0), - [=, SelfId = actor_id(this), - retry_at = td::Timestamp::in(1.0)](td::Result>> R) { + [=, SelfId = actor_id(this), retry_at = td::Timestamp::in(1.0), + promise = std::move(promise)](td::Result>> R) mutable { if (R.is_error()) { delay_action( - [=]() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::preload_msg_queue_to_masterchain, desc); }, + [=, promise = std::move(promise)]() mutable { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::preload_msg_queue_to_masterchain, desc, + std::move(promise)); + }, retry_at); return; } auto res = R.move_as_ok(); auto &queue = res[desc->block_id()]; CHECK(queue.not_null()); - td::actor::send_closure(SelfId, &ValidatorManagerImpl::loaded_msg_queue_to_masterchain, desc, std::move(queue)); + td::actor::send_closure(SelfId, &ValidatorManagerImpl::loaded_msg_queue_to_masterchain, desc, std::move(queue), + std::move(promise)); }); } void ValidatorManagerImpl::loaded_msg_queue_to_masterchain(td::Ref desc, - td::Ref res) { + td::Ref res, + td::Promise promise) { + VLOG(VALIDATOR_DEBUG) << "loaded out msg queue to masterchain from " << desc->block_id().to_str(); + cached_msg_queue_to_masterchain_[desc->block_id()] = std::move(res); + promise.set_value(td::Unit()); +} + +void ValidatorManagerImpl::set_shard_block_description_ready(td::Ref desc) { auto id = ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}; auto it = shard_blocks_.find(id); if (it == shard_blocks_.end()) { @@ -608,12 +641,8 @@ void ValidatorManagerImpl::loaded_msg_queue_to_masterchain(td::Refsecond; if (info.ready_desc.is_null() || info.ready_desc->block_id().seqno() < desc->block_id().seqno()) { - VLOG(VALIDATOR_DEBUG) << "loaded out msg queue to masterchain from " << desc->block_id(); - if (info.ready_desc.not_null()) { - cached_msg_queue_to_masterchain_.erase(info.ready_desc->block_id()); - } + VLOG(VALIDATOR_DEBUG) << "top shard block description is ready: " << desc->block_id().to_str(); info.ready_desc = desc; - cached_msg_queue_to_masterchain_[desc->block_id()] = std::move(res); } } @@ -895,8 +924,7 @@ void ValidatorManagerImpl::wait_block_data_short(BlockIdExt block_id, td::uint32 void ValidatorManagerImpl::wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) { if (last_masterchain_state_.not_null() && !opts_->need_monitor(left_id.shard_full(), last_masterchain_state_)) { - return promise.set_error( - td::Status::Error(PSTRING() << "not monitoring shard " << left_id.shard_full().to_str())); + return promise.set_error(td::Status::Error(PSTRING() << "not monitoring shard " << left_id.shard_full().to_str())); } td::actor::create_actor("merge", left_id, right_id, priority, actor_id(this), timeout, std::move(promise)) @@ -1356,7 +1384,7 @@ void ValidatorManagerImpl::store_persistent_state_file(BlockIdExt block_id, Bloc } void ValidatorManagerImpl::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + std::function write_data, td::Promise promise) { td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id, std::move(write_data), std::move(promise)); @@ -2142,6 +2170,13 @@ void ValidatorManagerImpl::new_masterchain_block() { for (auto &c : collator_nodes_) { td::actor::send_closure(c.second.actor, &CollatorNode::new_masterchain_block_notification, last_masterchain_state_); } + if (!shard_block_verifier_.empty()) { + td::actor::send_closure(shard_block_verifier_, &ShardBlockVerifier::update_masterchain_state, + last_masterchain_state_); + } + for (auto &[_, actor] : shard_block_retainers_) { + td::actor::send_closure(actor, &ShardBlockRetainer::update_masterchain_state, last_masterchain_state_); + } if (last_masterchain_seqno_ % 1024 == 0) { LOG(WARNING) << "applied masterchain block " << last_masterchain_block_id_; } @@ -2152,9 +2187,9 @@ void ValidatorManagerImpl::update_shard_overlays() { std::set shards_to_monitor; shards_to_monitor.insert(ShardIdFull{masterchainId}); std::set workchains; - for (const auto& shard : last_masterchain_state_->get_shards()) { + for (const auto &shard : last_masterchain_state_->get_shards()) { workchains.insert(shard->shard().workchain); - if (opts_->need_monitor(shard->shard(),last_masterchain_state_)) { + if (opts_->need_monitor(shard->shard(), last_masterchain_state_)) { shards_to_monitor.insert(shard->shard()); } } @@ -2178,7 +2213,7 @@ void ValidatorManagerImpl::update_shards() { td::uint32 threshold = 9407194; bool force_group_id_upgrade = last_masterchain_seqno_ == threshold; auto legacy_opts_hash = opts.get_hash(); - if (last_masterchain_seqno_ >= threshold) { //TODO move to get_consensus_config() + if (last_masterchain_seqno_ >= threshold) { //TODO move to get_consensus_config() opts.proto_version = std::max(opts.proto_version, 1); } auto opts_hash = opts.get_hash(); @@ -2269,7 +2304,6 @@ void ValidatorManagerImpl::update_shards() { auto legacy_val_group_id = get_validator_set_id(shard, val_set, legacy_opts_hash, key_seqno, opts); auto val_group_id = get_validator_set_id(shard, val_set, opts_hash, key_seqno, opts); - auto it = validator_groups_.find(legacy_val_group_id); if (it != validator_groups_.end()) { new_validator_groups_.emplace(val_group_id, std::move(it->second)); @@ -2293,6 +2327,7 @@ void ValidatorManagerImpl::update_shards() { } active_validator_groups_master_ = active_validator_groups_shard_ = 0; + adnl::AdnlNodeIdShort mc_validator_adnl_id = adnl::AdnlNodeIdShort::zero(); if (allow_validate_) { for (auto &desc : new_shards) { auto shard = desc.first; @@ -2344,6 +2379,12 @@ void ValidatorManagerImpl::update_shards() { new_validator_groups_.emplace(val_group_id, ValidatorGroupEntry{std::move(G), shard}); } } + if (shard.is_masterchain()) { + mc_validator_adnl_id = adnl::AdnlNodeIdShort{val_set->get_validator(validator_id.bits256_value())->addr}; + if (mc_validator_adnl_id.is_zero()) { + mc_validator_adnl_id = adnl::AdnlNodeIdShort{validator_id.bits256_value()}; + } + } } } } @@ -2365,6 +2406,12 @@ void ValidatorManagerImpl::update_shards() { val_group_id, ValidatorGroupEntry{ create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_), shard}); } + if (shard.is_masterchain() && mc_validator_adnl_id.is_zero()) { + mc_validator_adnl_id = adnl::AdnlNodeIdShort{val_set->get_validator(validator_id.bits256_value())->addr}; + if (mc_validator_adnl_id.is_zero()) { + mc_validator_adnl_id = adnl::AdnlNodeIdShort{validator_id.bits256_value()}; + } + } } } @@ -2407,6 +2454,7 @@ void ValidatorManagerImpl::update_shards() { td::actor::send_closure(serializer_, &AsyncStateSerializer::auto_disable_serializer, is_validator() && last_masterchain_state_->get_global_id() == -239); // mainnet only } + init_shard_block_verifier(mc_validator_adnl_id); } void ValidatorManagerImpl::written_destroyed_validator_sessions(std::vector> list) { @@ -2416,36 +2464,36 @@ void ValidatorManagerImpl::written_destroyed_validator_sessions(std::vectorsecond; - if (!B.latest_desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { - if (B.ready_desc.not_null()) { - cached_msg_queue_to_masterchain_.erase(B.ready_desc->block_id()); - } - it = shard_blocks_.erase(it); - continue; - } - if (B.ready_desc.not_null() && - !B.ready_desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { - cached_msg_queue_to_masterchain_.erase(B.ready_desc->block_id()); - B.ready_desc = {}; - } - ++it; + for (auto it = shard_blocks_.begin(); it != shard_blocks_.end();) { + auto &B = it->second; + if (!B.latest_desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { + it = shard_blocks_.erase(it); + continue; } + if (B.ready_desc.not_null() && + !B.ready_desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { + B.ready_desc = {}; + } + ++it; } - { - auto it = out_shard_blocks_.begin(); - while (it != out_shard_blocks_.end()) { - auto &B = it->second; - if (!B->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { - auto it2 = it++; - out_shard_blocks_.erase(it2); - } else { - ++it; - } + for (auto it = cached_msg_queue_to_masterchain_.begin(); it != cached_msg_queue_to_masterchain_.end();) { + const BlockIdExt &block_id = it->first; + auto shard = last_masterchain_state_->get_shard_from_config(block_id.shard_full(), false); + if (shard.is_null() || shard->top_block_id().seqno() >= block_id.seqno()) { + it = cached_msg_queue_to_masterchain_.erase(it); + continue; + } + ++it; + } + + for (auto it = out_shard_blocks_.begin(); it != out_shard_blocks_.end();) { + auto &B = it->second; + if (!B->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { + auto it2 = it++; + out_shard_blocks_.erase(it2); + } else { + ++it; } } } @@ -2865,7 +2913,6 @@ bool ValidatorManagerImpl::Collator::can_collate_shard(ShardIdFull shard) const [&](const ShardIdFull &our_shard) { return shard_intersects(shard, our_shard); }); } - void ValidatorManagerImpl::got_next_key_blocks(std::vector r) { if (r.size() == 0) { delay_action([SelfId = actor_id( @@ -2931,9 +2978,8 @@ void ValidatorManagerImpl::prepare_stats(td::Promise R) mutable { @@ -3333,6 +3379,9 @@ void ValidatorManagerImpl::update_options(td::Ref opts) for (auto &[_, c] : collation_managers_) { td::actor::send_closure(c, &CollationManager::update_options, opts); } + if (!shard_block_verifier_.empty()) { + td::actor::send_closure(shard_block_verifier_, &ShardBlockVerifier::update_options, opts); + } opts_ = std::move(opts); init_session_stats(); } @@ -3563,8 +3612,8 @@ void ValidatorManagerImpl::init_validator_telemetry() { return; } std::set processed; - for (auto& key : temp_keys_) { - if (const ValidatorDescr* desc = validator_set->get_validator(key.bits256_value())) { + for (auto &key : temp_keys_) { + if (const ValidatorDescr *desc = validator_set->get_validator(key.bits256_value())) { processed.insert(key); adnl::AdnlNodeIdShort adnl_id; if (desc->addr.is_zero()) { @@ -3572,10 +3621,10 @@ void ValidatorManagerImpl::init_validator_telemetry() { } else { adnl_id = adnl::AdnlNodeIdShort{desc->addr}; } - auto& telemetry = validator_telemetry_[key]; + auto &telemetry = validator_telemetry_[key]; if (telemetry.empty()) { - telemetry = td::actor::create_actor( - "telemetry", key, adnl_id, opts_->zero_block_id().file_hash, actor_id(this)); + telemetry = td::actor::create_actor("telemetry", key, adnl_id, + opts_->zero_block_id().file_hash, actor_id(this)); } } } @@ -3610,7 +3659,7 @@ void ValidatorManagerImpl::init_session_stats() { session_stats_enabled_ = true; } -template +template void ValidatorManagerImpl::write_session_stats(const T &obj) { if (!session_stats_enabled_) { return; @@ -3635,6 +3684,32 @@ void ValidatorManagerImpl::write_session_stats(const T &obj) { } } +void ValidatorManagerImpl::init_shard_block_verifier(adnl::AdnlNodeIdShort local_id) { + if (local_id != shard_block_verifier_local_id_) { + shard_block_verifier_local_id_ = local_id; + if (local_id.is_zero()) { + shard_block_verifier_ = {}; + } else { + shard_block_verifier_ = td::actor::create_actor( + "shardblockverifier", local_id, last_masterchain_state_, opts_, actor_id(this), adnl_, rldp_); + } + } +} + +void ValidatorManagerImpl::wait_verify_shard_blocks(std::vector blocks, td::Promise promise) { + if (shard_block_verifier_.empty()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "shard block verifier not inited")); + return; + } + td::actor::send_closure(shard_block_verifier_, &ShardBlockVerifier::wait_shard_blocks, std::move(blocks), + std::move(promise)); +} + +void ValidatorManagerImpl::add_shard_block_retainer(adnl::AdnlNodeIdShort id) { + shard_block_retainers_[id] = td::actor::create_actor( + "shardblockretainer", id, last_masterchain_state_, opts_, actor_id(this), adnl_, rldp_); +} + } // namespace validator } // namespace ton diff --git a/validator/manager.hpp b/validator/manager.hpp index 20aaf04e57..fce2ceee4c 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -37,6 +37,8 @@ #include "validator-telemetry.hpp" #include "impl/candidates-buffer.hpp" #include "collator-node.hpp" +#include "shard-block-verifier.hpp" +#include "shard-block-retainer.hpp" #include #include @@ -235,9 +237,10 @@ class ValidatorManagerImpl : public ValidatorManager { } }; // DATA FOR COLLATOR - // Shard block will not be used until queue is ready (to avoid too long masterchain collation) + // Shard block will not be used until it is confirmed by trusted nodes (see ShardBlockVerifier) and + // msg queue to masterchain is ready (to avoid too long masterchain collation) // latest_desc - latest known block - // ready_desc - block with ready msg queue (may be null) + // ready_desc - block ready to be used (may be null) struct ShardTopBlock { td::Ref latest_desc; td::Ref ready_desc; @@ -562,8 +565,10 @@ class ValidatorManagerImpl : public ValidatorManager { void add_shard_block_description(td::Ref desc); void add_cached_block_candidate(ReceivedBlock block); - void preload_msg_queue_to_masterchain(td::Ref desc); - void loaded_msg_queue_to_masterchain(td::Ref desc, td::Ref res); + void preload_msg_queue_to_masterchain(td::Ref desc, td::Promise promise); + void loaded_msg_queue_to_masterchain(td::Ref desc, td::Ref res, + td::Promise promise); + void set_shard_block_description_ready(td::Ref desc); void register_block_handle(BlockHandle handle); @@ -769,6 +774,10 @@ class ValidatorManagerImpl : public ValidatorManager { std::function>>)> callback) override; void unregister_stats_provider(td::uint64 idx) override; + void wait_verify_shard_blocks(std::vector blocks, td::Promise promise) override; + + void add_shard_block_retainer(adnl::AdnlNodeIdShort id) override; + std::map> validator_telemetry_; void init_validator_telemetry(); @@ -795,6 +804,12 @@ class ValidatorManagerImpl : public ValidatorManager { void init_session_stats(); template void write_session_stats(const T &obj); + + td::actor::ActorOwn shard_block_verifier_; + adnl::AdnlNodeIdShort shard_block_verifier_local_id_ = adnl::AdnlNodeIdShort::zero(); + std::map> shard_block_retainers_; + + void init_shard_block_verifier(adnl::AdnlNodeIdShort local_id); }; } // namespace validator diff --git a/validator/shard-block-retainer.cpp b/validator/shard-block-retainer.cpp new file mode 100644 index 0000000000..4fb96502e7 --- /dev/null +++ b/validator/shard-block-retainer.cpp @@ -0,0 +1,179 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "shard-block-retainer.hpp" + +namespace ton::validator { + +void ShardBlockRetainer::start_up() { + if (last_masterchain_state_.not_null()) { + update_masterchain_state(last_masterchain_state_); + } + + class Callback : public adnl::Adnl::Callback { + public: + explicit Callback(td::actor::ActorId id) : id_(std::move(id)) { + } + void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(id_, &ShardBlockRetainer::process_query, src, std::move(data), std::move(promise)); + } + + private: + td::actor::ActorId id_; + }; + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::shardBlockVerifier_subscribe::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); +} + +void ShardBlockRetainer::tear_down() { + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::shardBlockVerifier_subscribe::ID)); +} + +void ShardBlockRetainer::update_masterchain_state(td::Ref state) { + last_masterchain_state_ = state; + for (auto it = confirmed_blocks_.begin(); it != confirmed_blocks_.end();) { + if (is_block_outdated(*it)) { + it = confirmed_blocks_.erase(it); + } else { + ++it; + } + } + if (last_masterchain_state_->is_key_state() || !inited_) { + validator_adnl_ids_.clear(); + for (int next = 0; next <= 1; ++next) { + auto vset = last_masterchain_state_->get_total_validator_set(next); + if (vset.is_null()) { + continue; + } + for (auto& val : vset->export_vector()) { + adnl::AdnlNodeIdShort adnl_id{val.addr}; + if (adnl_id.is_zero()) { + adnl_id = adnl::AdnlNodeIdShort{ValidatorFullId{val.key}.short_id()}; + validator_adnl_ids_.insert(adnl_id); + } + } + } + LOG(INFO) << "Updating validator set: " << validator_adnl_ids_.size() << " adnl ids"; + for (auto it = subscribers_.begin(); it != subscribers_.end();) { + if (it->second.is_in_past()) { + LOG(INFO) << "Unsubscribed " << it->first.first << " for " << it->first.second.to_str() << " (expired)"; + it = subscribers_.erase(it); + continue; + } + if (!validator_adnl_ids_.contains(it->first.first)) { + LOG(INFO) << "Unsubscribed " << it->first.first << " for " << it->first.second.to_str() << " (not a validator)"; + it = subscribers_.erase(it); + continue; + } + ++it; + } + } + inited_ = true; +} + +void ShardBlockRetainer::new_shard_block_description(td::Ref desc) { + if (last_masterchain_state_.is_null() || is_block_outdated(desc->block_id()) || + !opts_->need_monitor(desc->shard(), last_masterchain_state_)) { + return; + } + td::actor::send_closure( + manager_, &ValidatorManager::wait_block_state_short, desc->block_id(), 0, td::Timestamp::in(30.0), + [SelfId = actor_id(this), desc](td::Result> R) { + if (R.is_error()) { + LOG(WARNING) << "Wait block state for " << desc->block_id().to_str() << " : " << R.move_as_error(); + td::actor::send_closure(SelfId, &ShardBlockRetainer::new_shard_block_description, std::move(desc)); + return; + } + td::actor::send_closure(SelfId, &ShardBlockRetainer::confirm_shard_block_description, std::move(desc)); + }); +} + +void ShardBlockRetainer::process_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, + td::Promise promise) { + TRY_RESULT_PROMISE(promise, query, fetch_tl_object(data, true)) + ShardIdFull shard = create_shard_id(query->shard_); + if (!shard.is_valid_ext() || shard.is_masterchain()) { + promise.set_error(td::Status::Error(PSTRING() << "invalid shard " << shard.to_str())); + return; + } + if (!validator_adnl_ids_.contains(src)) { + promise.set_error(td::Status::Error(PSTRING() << "unauthorized src " << src)); + return; + } + td::Timestamp& ttl = subscribers_[{src, shard}]; + if (!ttl) { + std::vector blocks; + for (const BlockIdExt& block : confirmed_blocks_) { + if (shard_intersects(block.shard_full(), shard)) { + blocks.push_back(block); + } + } + LOG(INFO) << "New subscriber " << src << " for " << shard.to_str() << ", sending " << blocks.size() << " blocks"; + send_confirmations(src, std::move(blocks)); + } + ttl = td::Timestamp::in(SUBSCRIPTION_TTL); + promise.set_value(create_serialize_tl_object(0)); +} + +void ShardBlockRetainer::send_confirmations(adnl::AdnlNodeIdShort dst, std::vector blocks) { + for (size_t l = 0; l < blocks.size(); l += MAX_BLOCKS_PER_MESSAGE) { + size_t r = std::min(l + MAX_BLOCKS_PER_MESSAGE, blocks.size()); + auto query = create_tl_object(); + for (size_t i = l; i < r; ++i) { + query->blocks_.push_back(create_tl_block_id(blocks[i])); + } + td::actor::send_closure(rldp_, &rldp::Rldp::send_message, local_id_, dst, serialize_tl_object(query, true)); + } +} + +void ShardBlockRetainer::confirm_shard_block_description(td::Ref desc) { + for (const BlockIdExt& block_id : desc->get_chain_blocks()) { + confirm_block(block_id); + } +} + +void ShardBlockRetainer::confirm_block(BlockIdExt block_id) { + if (is_block_outdated(block_id) || !confirmed_blocks_.insert(block_id).second) { + return; + } + size_t sent = 0; + for (auto it = subscribers_.begin(); it != subscribers_.end();) { + if (it->second.is_in_past()) { + LOG(INFO) << "Unsubscribed " << it->first.first << " for " << it->first.second.to_str() << " (expired)"; + it = subscribers_.erase(it); + continue; + } + if (shard_intersects(it->first.second, block_id.shard_full())) { + ++sent; + send_confirmations(it->first.first, {block_id}); + } + ++it; + } + LOG(INFO) << "Confirmed block " << block_id.to_str() << ", sending " << sent << " confirmations"; +} + +bool ShardBlockRetainer::is_block_outdated(const BlockIdExt& block_id) const { + auto shard_desc = last_masterchain_state_->get_shard_from_config(block_id.shard_full(), false); + return shard_desc.not_null() && shard_desc->top_block_id().seqno() >= block_id.seqno(); +} + +} // namespace ton::validator diff --git a/validator/shard-block-retainer.hpp b/validator/shard-block-retainer.hpp new file mode 100644 index 0000000000..14a6cfa8ae --- /dev/null +++ b/validator/shard-block-retainer.hpp @@ -0,0 +1,71 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "interfaces/validator-manager.h" +#include "rldp.h" +#include + +namespace ton::validator { + +class ShardBlockRetainer : public td::actor::Actor { + public: + ShardBlockRetainer(adnl::AdnlNodeIdShort local_id, td::Ref last_masterchain_state, + td::Ref opts, td::actor::ActorId manager, + td::actor::ActorId adnl, td::actor::ActorId rldp) + : local_id_(local_id) + , last_masterchain_state_(last_masterchain_state) + , opts_(std::move(opts)) + , manager_(std::move(manager)) + , adnl_(std::move(adnl)) + , rldp_(std::move(rldp)) { + } + + void start_up() override; + void tear_down() override; + void update_masterchain_state(td::Ref state); + void new_shard_block_description(td::Ref desc); + + void update_options(td::Ref opts) { + opts_ = std::move(opts); + } + + private: + adnl::AdnlNodeIdShort local_id_; + td::Ref last_masterchain_state_; + td::Ref opts_; + td::actor::ActorId manager_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + + bool inited_ = false; + std::set validator_adnl_ids_; + std::map, td::Timestamp> subscribers_; + std::set confirmed_blocks_; + + void process_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); + void send_confirmations(adnl::AdnlNodeIdShort dst, std::vector blocks); + + void confirm_shard_block_description(td::Ref desc); + void confirm_block(BlockIdExt block_id); + + bool is_block_outdated(const BlockIdExt& block_id) const; + + static constexpr double SUBSCRIPTION_TTL = 60.0; + static constexpr size_t MAX_BLOCKS_PER_MESSAGE = 8; +}; + +} // namespace ton::validator diff --git a/validator/shard-block-verifier.cpp b/validator/shard-block-verifier.cpp new file mode 100644 index 0000000000..5beb1e9a3d --- /dev/null +++ b/validator/shard-block-verifier.cpp @@ -0,0 +1,194 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "shard-block-verifier.hpp" + +#include "td/actor/MultiPromise.h" + +namespace ton::validator { + +void ShardBlockVerifier::start_up() { + update_config(opts_->get_shard_block_verifier_config()); + update_masterchain_state(last_masterchain_state_); + + class Callback : public adnl::Adnl::Callback { + public: + explicit Callback(td::actor::ActorId id) : id_(std::move(id)) { + } + void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + td::actor::send_closure(id_, &ShardBlockVerifier::process_message, src, std::move(data)); + } + void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + } + + private: + td::actor::ActorId id_; + }; + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::shardBlockVerifier_confirmBlocks::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); +} + +void ShardBlockVerifier::tear_down() { + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::shardBlockVerifier_confirmBlocks::ID)); +} + +void ShardBlockVerifier::update_masterchain_state(td::Ref state) { + last_masterchain_state_ = std::move(state); + for (auto it = blocks_.begin(); it != blocks_.end();) { + if (is_block_outdated(it->first)) { + it->second.finalize_promises(); + it = blocks_.erase(it); + } else { + ++it; + } + } +} + +void ShardBlockVerifier::wait_shard_blocks(std::vector blocks, td::Promise promise) { + td::MultiPromise mp; + auto ig = mp.init_guard(); + ig.add_promise(std::move(promise)); + for (const BlockIdExt& block_id : blocks) { + BlockInfo* info = get_block_info(block_id); + if (info && !info->confirmed) { + info->promises.push_back(ig.get_promise()); + } + } +} + +void ShardBlockVerifier::update_config(td::Ref new_config) { + auto old_config = std::move(config_); + config_ = std::move(new_config); + auto old_blocks = std::move(blocks_); + blocks_.clear(); + for (auto& [block_id, old_info] : old_blocks) { + BlockInfo* new_info = get_block_info(block_id); + if (new_info == nullptr) { + old_info.finalize_promises(); + continue; + } + new_info->promises = std::move(old_info.promises); + if (new_info->confirmed) { + new_info->finalize_promises(); + } + for (size_t old_src_idx = 0; old_src_idx < old_info.confirmed_by.size(); ++old_src_idx) { + if (old_info.confirmed_by[old_src_idx]) { + set_block_confirmed(old_config->shards[old_info.config_shard_idx].trusted_nodes[old_src_idx], block_id); + } + } + } + + alarm_timestamp().relax(send_subscribe_at_ = td::Timestamp::now()); +} + +void ShardBlockVerifier::alarm() { + if (send_subscribe_at_ && send_subscribe_at_.is_in_past()) { + for (auto& shard_config : config_->shards) { + for (auto& node_id : shard_config.trusted_nodes) { + td::Promise P = [shard = shard_config.shard_id, node_id](td::Result R) { + if (R.is_error()) { + LOG(WARNING) << "Subscribe to " << node_id << " for " << shard.to_str() << " : " << R.move_as_error(); + } + }; + td::actor::send_closure(rldp_, &rldp::Rldp::send_query, local_id_, node_id, "subscribe", std::move(P), + td::Timestamp::in(3.0), + create_serialize_tl_object( + create_tl_shard_id(shard_config.shard_id), 0)); + } + } + send_subscribe_at_ = td::Timestamp::in(SEND_SUBSCRIBE_PERIOD); + } + alarm_timestamp().relax(send_subscribe_at_); +} + +void ShardBlockVerifier::process_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) { + auto r_obj = fetch_tl_object(data, true); + if (r_obj.is_error()) { + return; + } + for (const auto& b : r_obj.ok()->blocks_) { + set_block_confirmed(src, create_block_id(b)); + } +} + +int ShardBlockVerifier::get_config_shard_idx(const ShardIdFull& shard_id) const { + for (size_t i = 0; i < config_->shards.size(); i++) { + if (shard_intersects(shard_id, config_->shards[i].shard_id)) { + return i; + } + } + return -1; +} + +bool ShardBlockVerifier::is_block_outdated(const BlockIdExt& block_id) const { + auto shard_desc = last_masterchain_state_->get_shard_from_config(block_id.shard_full(), false); + return shard_desc.not_null() && shard_desc->top_block_id().seqno() >= block_id.seqno(); +} + +ShardBlockVerifier::BlockInfo* ShardBlockVerifier::get_block_info(const BlockIdExt& block_id) { + auto it = blocks_.find(block_id); + if (it != blocks_.end()) { + return &it->second; + } + int config_shard_idx = get_config_shard_idx(block_id.shard_full()); + if (config_shard_idx < 0 || is_block_outdated(block_id)) { + return nullptr; + } + auto& shard_config = config_->shards[config_shard_idx]; + BlockInfo& info = blocks_[block_id]; + info.config_shard_idx = config_shard_idx; + info.confirmed_by.resize(shard_config.trusted_nodes.size(), false); + info.confirmed = (shard_config.required_confirms == 0); + return &info; +} + +void ShardBlockVerifier::set_block_confirmed(adnl::AdnlNodeIdShort src, BlockIdExt block_id) { + BlockInfo* info = get_block_info(block_id); + if (info == nullptr) { + LOG(INFO) << "Confirm for " << block_id.to_str() << " from " << src << " : ignored"; + return; + } + auto& shard_config = config_->shards[info->config_shard_idx]; + size_t src_idx = 0; + while (src_idx < shard_config.trusted_nodes.size() && shard_config.trusted_nodes[src_idx] != src) { + ++src_idx; + } + if (src_idx == shard_config.trusted_nodes.size()) { + LOG(INFO) << "Confirm for " << block_id.to_str() << " from " << src << " : unknown src"; + return; + } + if (info->confirmed_by[src_idx]) { + LOG(INFO) << "Confirm for " << block_id.to_str() << " from " << src << " #" << src_idx << " : duplicate"; + return; + } + info->confirmed_by[src_idx] = true; + ++info->confirmed_by_cnt; + LOG(INFO) << "Confirm for " << block_id.to_str() << " from " << src << " #" << src_idx << " : accepted (" + << info->confirmed_by_cnt << "/" << shard_config.required_confirms << "/" + << shard_config.trusted_nodes.size() << ")" + << (info->confirmed_by_cnt == shard_config.required_confirms ? ", CONFIRMED" : ""); + if (info->confirmed_by_cnt == shard_config.required_confirms) { + info->confirmed = true; + info->finalize_promises(); + info->promises.clear(); + } +} + +} // namespace ton::validator diff --git a/validator/shard-block-verifier.hpp b/validator/shard-block-verifier.hpp new file mode 100644 index 0000000000..82c1e3738c --- /dev/null +++ b/validator/shard-block-verifier.hpp @@ -0,0 +1,90 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "interfaces/validator-manager.h" +#include "rldp.h" +#include + +namespace ton::validator { + +class ShardBlockVerifier : public td::actor::Actor { + public: + ShardBlockVerifier(adnl::AdnlNodeIdShort local_id, td::Ref last_masterchain_state, + td::Ref opts, td::actor::ActorId manager, + td::actor::ActorId adnl, td::actor::ActorId rldp) + : local_id_(local_id) + , last_masterchain_state_(last_masterchain_state) + , opts_(std::move(opts)) + , manager_(std::move(manager)) + , adnl_(std::move(adnl)) + , rldp_(std::move(rldp)) { + } + + void start_up() override; + void tear_down() override; + void update_masterchain_state(td::Ref state); + void wait_shard_blocks(std::vector blocks, td::Promise promise); + + void update_options(td::Ref opts) { + if (config_ != opts->get_shard_block_verifier_config()) { + update_config(opts->get_shard_block_verifier_config()); + } + opts_ = std::move(opts); + } + + void alarm() override; + + private: + adnl::AdnlNodeIdShort local_id_; + td::Ref last_masterchain_state_; + td::Ref opts_; + td::actor::ActorId manager_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + + td::Ref config_; + + td::Timestamp send_subscribe_at_ = td::Timestamp::never(); + + struct BlockInfo { + size_t config_shard_idx = 0; + std::vector confirmed_by; + td::uint32 confirmed_by_cnt = 0; + bool confirmed = false; + std::vector> promises; + + void finalize_promises() { + for (auto& promise : promises) { + promise.set_value(td::Unit()); + } + } + }; + std::map blocks_; + + void update_config(td::Ref new_config); + void process_message(adnl::AdnlNodeIdShort src, td::BufferSlice data); + + int get_config_shard_idx(const ShardIdFull& shard_id) const; + bool is_block_outdated(const BlockIdExt& block_id) const; + BlockInfo* get_block_info(const BlockIdExt& block_id); + + void set_block_confirmed(adnl::AdnlNodeIdShort src, BlockIdExt block_id); + + static constexpr double SEND_SUBSCRIBE_PERIOD = 10.0; +}; + +} // namespace ton::validator diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index 230a5df469..52047a1d8b 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -69,6 +69,34 @@ CollatorsList CollatorsList::default_list() { return list; } +td::Status ShardBlockVerifierConfig::unpack(const ton_api::engine_validator_shardBlockVerifierConfig& obj) { + shards.clear(); + for (const auto& shard_obj : obj.shards_) { + Shard shard; + shard.shard_id = create_shard_id(shard_obj->shard_id_); + if (shard.shard_id.is_masterchain() || !shard.shard_id.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << shard.shard_id.to_str()); + } + std::set trusted_nodes; + for (const td::Bits256& id : shard_obj->trusted_nodes_) { + adnl::AdnlNodeIdShort node_id{id}; + if (!trusted_nodes.insert(node_id).second) { + return td::Status::Error(PSTRING() << "duplicate node " << node_id); + } + shard.trusted_nodes.push_back(node_id); + if (shard_obj->required_confirms_ < 0 || shard_obj->required_confirms_ > (int)shard.trusted_nodes.size()) { + return td::Status::Error(PSTRING() + << "invalid required_confirms " << shard_obj->required_confirms_ << " for shard " + << shard.shard_id.to_str() << " (nodes: " << shard.trusted_nodes.size() << ")"); + } + shard.required_confirms = shard_obj->required_confirms_; + shards.push_back(std::move(shard)); + } + } + return td::Status::OK(); +} + + td::Ref ValidatorManagerOptions::create(BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard, bool allow_blockchain_init, double sync_blocks_before, diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 0c6f936330..607c8890e7 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -169,6 +169,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool check_collator_node_whitelist(adnl::AdnlNodeIdShort id) const override { return !collator_node_whitelist_enabled_ || collator_node_whitelist_.contains(id); } + td::Ref get_shard_block_verifier_config() const override { + return shard_block_verifier_config_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -286,6 +289,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_collator_node_whitelist_enabled(bool enabled) override { collator_node_whitelist_enabled_ = enabled; } + void set_shard_block_verifier_config(td::Ref config) override { + shard_block_verifier_config_ = std::move(config); + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); @@ -345,6 +351,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { td::Ref collators_list_{true, CollatorsList::default_list()}; std::set collator_node_whitelist_; bool collator_node_whitelist_enabled_ = false; + td::Ref shard_block_verifier_config_{true}; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 2318d53e5b..09b790a35c 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -95,6 +95,17 @@ struct CollatorsList : public td::CntObject { static CollatorsList default_list(); }; +struct ShardBlockVerifierConfig : public td::CntObject { + struct Shard { + ShardIdFull shard_id; + std::vector trusted_nodes; + td::uint32 required_confirms; + }; + std::vector shards; + + td::Status unpack(const ton_api::engine_validator_shardBlockVerifierConfig& obj); +}; + struct ValidatorManagerOptions : public td::CntObject { public: virtual BlockIdExt zero_block_id() const = 0; @@ -141,6 +152,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual double get_catchain_broadcast_speed_multiplier() const = 0; virtual td::Ref get_collators_list() const = 0; virtual bool check_collator_node_whitelist(adnl::AdnlNodeIdShort id) const = 0; + virtual td::Ref get_shard_block_verifier_config() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; @@ -179,6 +191,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_collators_list(td::Ref list) = 0; virtual void set_collator_node_whitelisted_validator(adnl::AdnlNodeIdShort id, bool add) = 0; virtual void set_collator_node_whitelist_enabled(bool enabled) = 0; + virtual void set_shard_block_verifier_config(td::Ref config) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, @@ -348,6 +361,10 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_collation_manager_stats( td::Promise> promise) = 0; + + virtual void add_shard_block_retainer(adnl::AdnlNodeIdShort id) { + LOG(ERROR) << "Unimplemented add_shard_block_retainer"; + } }; } // namespace validator From ebde55ea343bba9c7b00947c4e6590c2899c5899 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 25 Apr 2025 13:28:37 +0400 Subject: [PATCH 227/524] [Tolk] Simplify VarDescr, drop _Const enum value Now, only VarDescr::int_const represents whether it's a constant, without being combined (and potentially become unsync) val bit. --- tolk-tester/tests/logical-operators.tolk | 27 +++++++++++++++++++ tolk-tester/tests/smart-cast-tests.tolk | 4 ++- tolk/abscode.cpp | 33 +++++++++++------------- tolk/analyzer.cpp | 4 +-- tolk/tolk.h | 17 +++++++----- 5 files changed, 57 insertions(+), 28 deletions(-) diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk index cbc82f15e8..7e3168c7c9 100644 --- a/tolk-tester/tests/logical-operators.tolk +++ b/tolk-tester/tests/logical-operators.tolk @@ -148,6 +148,31 @@ fun testConvertIfToIfnot(x: bool) { return -4; } +@pure +fun get123(): int + asm "123 PUSHINT"; + +@method_id(114) +fun test114() { + val fals = (get123() < 0) as int; + return (fals ? -1 : fals) != 0; +} + +@method_id(115) +fun test115() { + val tru = get123() >= 0; + val fals = get123() < 0; + + if ((true || false) && (false || false)) { + throw 123; + } + if ((tru || fals) && (fals || fals)) { + throw 456; + } + return (tru, fals); +} + + fun main() { } @@ -187,6 +212,8 @@ fun main() { @testcase | 112 | 5 0 | 0 -1 0 -1 -1 @testcase | 112 | 0 -1 | -1 -1 -1 -1 -1 @testcase | 113 | 0 | 1 +@testcase | 114 | | 0 +@testcase | 115 | | -1 0 @fif_codegen """ diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 62477c1355..946c00222a 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -148,9 +148,10 @@ fun test10(): int { return x + y; } +@method_id(111) fun test11() { var [x, y] = [getNullableInt(), getNullableInt()]; - if (random()) { return x == null || y == null ? -1 : x + y; } + if (eq(10) < 0) { return x == null || y == null ? -1 : x + y; } if (true && (x == null || y == null) && !!true) { return 0; } return x + y; } @@ -665,6 +666,7 @@ fun main(x: int?): int { /** @testcase | 0 | 1 | 1 +@testcase | 111 | | 10 @testcase | 123 | | 7 @testcase | 124 | 4 | 6 @testcase | 124 | null | 15 diff --git a/tolk/abscode.cpp b/tolk/abscode.cpp index c33fcd89db..90c7fbcf17 100644 --- a/tolk/abscode.cpp +++ b/tolk/abscode.cpp @@ -59,9 +59,6 @@ void VarDescr::show_value(std::ostream& os) const { if (val & _Int) { os << 'i'; } - if (val & _Const) { - os << 'c'; - } if (val & _Zero) { os << '0'; } @@ -114,7 +111,7 @@ void VarDescr::set_const(td::RefInt256 value) { if (!int_const->signed_fits_bits(257)) { int_const.write().invalidate(); } - val = _Const | _Int; + val = _Int; int s = sgn(int_const); if (s < -1) { val |= _Nan | _NonZero; @@ -130,41 +127,41 @@ void VarDescr::set_const(td::RefInt256 value) { } } -void VarDescr::set_const(std::string value) { - str_const = value; - val = _Const; +void VarDescr::set_const(const std::string&) { + int_const.clear(); + val = 0; } void VarDescr::operator|=(const VarDescr& y) { - val &= y.val; - if (is_int_const() && y.is_int_const() && cmp(int_const, y.int_const) != 0) { - val &= ~_Const; - } - if (!(val & _Const)) { - int_const.clear(); + if (is_int_const()) { + bool y_same = y.is_int_const() && *int_const == *y.int_const; + if (!y_same) { + int_const.clear(); + } } + val &= y.val; } void VarDescr::operator&=(const VarDescr& y) { - val |= y.val; - if (y.int_const.not_null() && int_const.is_null()) { + if (y.is_int_const()) { int_const = y.int_const; } + val |= y.val; } void VarDescr::set_value(const VarDescr& y) { - val = y.val; int_const = y.int_const; + val = y.val; } void VarDescr::set_value(VarDescr&& y) { - val = y.val; int_const = std::move(y.int_const); + val = y.val; } void VarDescr::clear_value() { - val = 0; int_const.clear(); + val = 0; } void VarDescrList::show(std::ostream& os) const { diff --git a/tolk/analyzer.cpp b/tolk/analyzer.cpp index 2cd65c419e..935c8b02c7 100644 --- a/tolk/analyzer.cpp +++ b/tolk/analyzer.cpp @@ -62,10 +62,10 @@ bool operator==(const VarDescrList& x, const VarDescrList& y) { } bool same_values(const VarDescr& x, const VarDescr& y) { - if (x.val != y.val || x.int_const.is_null() != y.int_const.is_null()) { + if (x.val != y.val || x.is_int_const() != y.is_int_const()) { return false; } - if (x.int_const.not_null() && cmp(x.int_const, y.int_const) != 0) { + if (x.is_int_const() && *x.int_const != *y.int_const) { return false; } return true; diff --git a/tolk/tolk.h b/tolk/tolk.h index 3f00d0d4bc..74d8fdd9fc 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -68,7 +68,6 @@ struct VarDescr { enum { _Last = 1, _Unused = 2 }; int flags; enum { - _Const = 16, _Int = 32, _Zero = 64, _NonZero = 128, @@ -79,16 +78,15 @@ struct VarDescr { _Even = 16384, _Odd = 32768, }; - static constexpr int ConstZero = _Const | _Int | _Zero | _Pos | _Neg | _Finite | _Even; - static constexpr int ConstOne = _Const | _Int | _NonZero | _Pos | _Finite | _Odd; - static constexpr int ConstTrue = _Const | _Int | _NonZero | _Neg | _Finite | _Odd; + static constexpr int ConstZero = _Int | _Zero | _Pos | _Neg | _Finite | _Even; + static constexpr int ConstOne = _Int | _NonZero | _Pos | _Finite | _Odd; + static constexpr int ConstTrue = _Int | _NonZero | _Neg | _Finite | _Odd; static constexpr int ValBit = _Int | _Pos | _Finite; static constexpr int ValBool = _Int | _Neg | _Finite; static constexpr int FiniteInt = _Int | _Finite; static constexpr int FiniteUInt = _Int | _Finite | _Pos; int val; td::RefInt256 int_const; - std::string str_const; explicit VarDescr(var_idx_t _idx = -1, int _flags = 0, int _val = 0) : idx(_idx), flags(_flags), val(_val) { } @@ -120,7 +118,12 @@ struct VarDescr { return val & _Odd; } bool is_int_const() const { - return (val & (_Int | _Const)) == (_Int | _Const) && int_const.not_null(); +#ifdef TOLK_DEBUG + if (int_const.not_null()) { + tolk_assert(val & _Int); + } +#endif + return int_const.not_null(); } bool always_nonpos() const { return val & _Neg; @@ -151,7 +154,7 @@ struct VarDescr { } void set_const(long long value); void set_const(td::RefInt256 value); - void set_const(std::string value); + void set_const(const std::string& value); void operator+=(const VarDescr& y) { flags &= y.flags; } From 9e6c1fc9875e9ca683a5b88e7e824c74dfb95a0a Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 17 Feb 2025 18:01:23 +0300 Subject: [PATCH 228/524] [Tolk] Implement structures that work like tensors on a stack Support declaring structures with arbitrary fields and creating objects very similar to TS syntax. Structs are actually tensors on a stack, and accessing their fields behave exactly as accessing tensor's elements by index. As a consequence, recurring struct fields doesn't work; it will lead to infinity size. All features of type system (nullables, unions) seamlessly work with structures. A struct with 1 field has no extra overhead, it's just a field on a stack. --- .../tests/allow-post-modification.tolk | 13 +- .../tests/invalid-declaration/err-1585.tolk | 10 + .../tests/invalid-semantics/err-4085.tolk | 22 + .../tests/invalid-semantics/err-4150.tolk | 13 + .../tests/invalid-semantics/err-4193.tolk | 13 + .../tests/invalid-semantics/err-4260.tolk | 15 + .../tests/invalid-semantics/err-4391.tolk | 13 + .../tests/invalid-semantics/err-4440.tolk | 21 + .../tests/invalid-semantics/err-4519.tolk | 12 + .../tests/invalid-semantics/err-4572.tolk | 9 + .../tests/invalid-semantics/err-4682.tolk | 13 + .../tests/invalid-semantics/err-4712.tolk | 10 + .../tests/invalid-semantics/err-4822.tolk | 23 + .../tests/invalid-syntax/err-3034.tolk | 8 + .../tests/invalid-syntax/err-3150.tolk | 9 + .../tests/invalid-syntax/err-3690.tolk | 12 + .../tests/invalid-syntax/err-3802.tolk | 9 + .../tests/invalid-typing/err-6078.tolk | 11 + .../tests/invalid-typing/err-6184.tolk | 11 + .../tests/invalid-typing/err-6230.tolk | 10 + .../tests/invalid-typing/err-6709.tolk | 13 + .../tests/invalid-typing/err-6829.tolk | 6 +- .../tests/invalid-typing/err-6845.tolk | 12 + .../tests/invalid-typing/err-6942.tolk | 19 + tolk-tester/tests/meaningful-1.tolk | 43 ++ tolk-tester/tests/smart-cast-tests.tolk | 52 ++ tolk-tester/tests/struct-tests.tolk | 519 ++++++++++++++++++ tolk-tester/tests/type-aliases-tests.tolk | 7 + tolk-tester/tests/var-apply-tests.tolk | 21 + tolk/abscode.cpp | 9 +- tolk/ast-from-tokens.cpp | 91 ++- tolk/ast-replacer.h | 6 + tolk/ast-replicator.h | 12 + tolk/ast-stringifier.h | 20 + tolk/ast-visitor.h | 6 + tolk/ast.cpp | 20 + tolk/ast.h | 109 +++- tolk/fwd-declarations.h | 4 + tolk/pipe-ast-to-legacy.cpp | 139 +++-- tolk/pipe-calc-rvalue-lvalue.cpp | 19 + tolk/pipe-check-inferred-types.cpp | 13 + tolk/pipe-check-rvalue-lvalue.cpp | 25 + tolk/pipe-infer-types-and-calls.cpp | 129 ++++- tolk/pipe-register-symbols.cpp | 21 + tolk/pipe-resolve-identifiers.cpp | 37 +- tolk/smart-casts-cfg.cpp | 37 +- tolk/symtable.cpp | 25 + tolk/symtable.h | 32 ++ tolk/type-system.cpp | 54 ++ tolk/type-system.h | 23 + 50 files changed, 1719 insertions(+), 61 deletions(-) create mode 100644 tolk-tester/tests/invalid-declaration/err-1585.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4085.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4150.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4193.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4260.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4391.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4440.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4519.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4572.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4682.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4712.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4822.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3034.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3150.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3690.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3802.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6078.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6184.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6230.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6709.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6845.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6942.tolk create mode 100644 tolk-tester/tests/meaningful-1.tolk create mode 100644 tolk-tester/tests/struct-tests.tolk diff --git a/tolk-tester/tests/allow-post-modification.tolk b/tolk-tester/tests/allow-post-modification.tolk index 180654e327..10c2437407 100644 --- a/tolk-tester/tests/allow-post-modification.tolk +++ b/tolk-tester/tests/allow-post-modification.tolk @@ -112,6 +112,16 @@ fun test_assign_tensor_global(x: (int, int)) { return (x, x = (20, 30), fs = x.multens((1, 2)), fs.multens(multens(mutate x, (-1, -1))), x, fs); } +struct Point1D { + c: int; +} + +@method_id(25) +fun test_with_single_slot_struct(x: Point1D) { + var result = (x, x.c += 10, [x, x.c += 20, eq(x.c -= 50), x.c], eq2((x.c, x.c *= eq(x.c /= 2)))); + return result; +} + fun main() { } @@ -132,6 +142,7 @@ fun main() { @testcase | 22 | 100 | 100 210 4200 630 @testcase | 23 | 1 1 | 1 1 20 30 20 60 -400 -3600 -20 -60 -400 -3600 @testcase | 24 | 1 1 | 1 1 20 30 20 60 -400 -3600 -20 -60 -400 -3600 +@testcase | 25 | 100 | 100 110 [ 110 130 80 80 ] 80 3200 @fif_codegen """ @@ -147,5 +158,5 @@ fun main() { // x.0 x.1 """ -@code_hash 61280273714870328160131559159866470128402169974050439159015534193532598351244 +@code_hash 6737917279814799680932710799951154408447028229503449454536845752635763933556 */ diff --git a/tolk-tester/tests/invalid-declaration/err-1585.tolk b/tolk-tester/tests/invalid-declaration/err-1585.tolk new file mode 100644 index 0000000000..0c1fef3328 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1585.tolk @@ -0,0 +1,10 @@ +struct B { a: A } +struct A { b: BAlias } + +type BAlias = B; + +/** +@compilation_should_fail +@stderr struct `B` size is infinity due to recursive fields +@stderr struct B { + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4085.tolk b/tolk-tester/tests/invalid-semantics/err-4085.tolk new file mode 100644 index 0000000000..11c4039088 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4085.tolk @@ -0,0 +1,22 @@ +struct A { + value: int; +} + +struct B { + data: slice; +} + +fun main() { + var r1: A | B = A { value: 10 }; + var r2: A | null = { value: 10 }; + var r3: A | int | builder = { value: 10 }; + + var r4: A | B = { value: 20 }; +} + +/** +@compilation_should_fail +@stderr can not detect struct name +@stderr use either `var v: StructName = { ... }` or `var v = StructName { ... }` +@stderr { value: 20 } + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4150.tolk b/tolk-tester/tests/invalid-semantics/err-4150.tolk new file mode 100644 index 0000000000..d542578728 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4150.tolk @@ -0,0 +1,13 @@ +fun increment(mutate x: int) { + x += 1; +} + +fun checkCantMutateFieldOfImmutableTensor() { + val t = (0, 1); + increment(mutate ((t!)).0); +} + +/** +@compilation_should_fail +@stderr modifying immutable variable `t` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4193.tolk b/tolk-tester/tests/invalid-semantics/err-4193.tolk new file mode 100644 index 0000000000..b0b31fa7da --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4193.tolk @@ -0,0 +1,13 @@ +struct User { + id: int; +} + +fun cantInstantiateNonGenericStruct() { + var u = User { id: 3 }; +} + +/** +@compilation_should_fail +@stderr generic T not expected here +@stderr User + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4260.tolk b/tolk-tester/tests/invalid-semantics/err-4260.tolk new file mode 100644 index 0000000000..6f174a595c --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4260.tolk @@ -0,0 +1,15 @@ +struct User { + id: int; +} + +type UserAlias = User; + +fun cantInstantiateNonGenericStruct() { + var u = UserAlias { id: 3 }; +} + +/** +@compilation_should_fail +@stderr generic T not expected here +@stderr UserAlias + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4391.tolk b/tolk-tester/tests/invalid-semantics/err-4391.tolk new file mode 100644 index 0000000000..80baf9b835 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4391.tolk @@ -0,0 +1,13 @@ +struct User { + id: int; + name: slice; +} + +fun main() { + var u: User = { id: 1 }; +} + +/** +@compilation_should_fail +@stderr field `name` missed in initialization of struct `User` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4440.tolk b/tolk-tester/tests/invalid-semantics/err-4440.tolk new file mode 100644 index 0000000000..9c59f2c507 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4440.tolk @@ -0,0 +1,21 @@ +struct NumberFields { + `-1`: int; + `0`: int; + `100500`: int; +} + +fun main() { + var b: NumberFields = { `-1`: -1, `0`: 0, `100500`: 100500 }; + b.0; // ok + b.`0`; // ok + b.`-1`; // ok + b.`100500`; // ok + b.100500; // ok + b.123; +} + +/** +@compilation_should_fail +@stderr field `123` doesn't exist in type `NumberFields` +@stderr b.123 + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4519.tolk b/tolk-tester/tests/invalid-semantics/err-4519.tolk new file mode 100644 index 0000000000..b0377b7199 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4519.tolk @@ -0,0 +1,12 @@ +struct value { id: int; } + +fun main() { + var value = 100; + var obj = value { id: 100 }; +} + +/** +@compilation_should_fail +@stderr `value` does not name a struct +@stderr obj = value + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4572.tolk b/tolk-tester/tests/invalid-semantics/err-4572.tolk new file mode 100644 index 0000000000..a62b63a0fa --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4572.tolk @@ -0,0 +1,9 @@ +fun checkCantMutateFieldOfImmutableTuple() { + val ks = null as [int, [int, [int]]]?; + ks!.1.1.0 = 10; +} + +/** +@compilation_should_fail +@stderr modifying immutable variable `ks` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4682.tolk b/tolk-tester/tests/invalid-semantics/err-4682.tolk new file mode 100644 index 0000000000..054a6a4032 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4682.tolk @@ -0,0 +1,13 @@ +struct User { id: int; } + +fun absk(v: int) { return v > 0 ? v : -v; } + +fun main() { + absk(User); +} + +/** +@compilation_should_fail +@stderr struct `User` can not be used as a value +@stderr absk(User) +*/ diff --git a/tolk-tester/tests/invalid-semantics/err-4712.tolk b/tolk-tester/tests/invalid-semantics/err-4712.tolk new file mode 100644 index 0000000000..2e210eb4b3 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4712.tolk @@ -0,0 +1,10 @@ +fun main() { + var t: tuple = createEmptyTuple(); + t.asdf = 10; +} + +/** +@compilation_should_fail +@stderr field `asdf` doesn't exist in type `tuple` +@stderr in function `main` +*/ diff --git a/tolk-tester/tests/invalid-semantics/err-4822.tolk b/tolk-tester/tests/invalid-semantics/err-4822.tolk new file mode 100644 index 0000000000..85cf4073be --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4822.tolk @@ -0,0 +1,23 @@ +struct Point { + x: int; + y: int; +} + +struct Coord2 { + p1: Point?; + p2: Point; +} + +fun increment(mutate x: int) { + x += 1; +} + +fun checkCantMutateFieldOfImmutableTuple(p1: Point?, p2: Point) { + val c2: Coord2 = { p1, p2 }; + (c2!.p1!).x.increment(); +} + +/** +@compilation_should_fail +@modifying immutable variable `c2` + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3034.tolk b/tolk-tester/tests/invalid-syntax/err-3034.tolk new file mode 100644 index 0000000000..af02af80a6 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3034.tolk @@ -0,0 +1,8 @@ +struct A { + id; +} + +/** +@compilation_should_fail +@stderr expected `: `, got `;` +*/ diff --git a/tolk-tester/tests/invalid-syntax/err-3150.tolk b/tolk-tester/tests/invalid-syntax/err-3150.tolk new file mode 100644 index 0000000000..8828fcadff --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3150.tolk @@ -0,0 +1,9 @@ +struct A { + id: int = 3; +} + +/** +@compilation_should_fail +@stderr default values for fields not supported yet +@stderr id: int = 3 +*/ diff --git a/tolk-tester/tests/invalid-syntax/err-3690.tolk b/tolk-tester/tests/invalid-syntax/err-3690.tolk new file mode 100644 index 0000000000..934e185418 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3690.tolk @@ -0,0 +1,12 @@ +struct A { + id: int; +} + +fun main() { + A { id: 1, id: 2 }; +} + +/** +@compilation_should_fail +@stderr duplicate field initialization +*/ diff --git a/tolk-tester/tests/invalid-syntax/err-3802.tolk b/tolk-tester/tests/invalid-syntax/err-3802.tolk new file mode 100644 index 0000000000..fec840c981 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3802.tolk @@ -0,0 +1,9 @@ +struct A { + id1: int + id2: int +} + +/** +@compilation_should_fail +@stderr expected `;` or `,`, got `id2` +*/ diff --git a/tolk-tester/tests/invalid-typing/err-6078.tolk b/tolk-tester/tests/invalid-typing/err-6078.tolk new file mode 100644 index 0000000000..1390866437 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6078.tolk @@ -0,0 +1,11 @@ +struct S { v: int; } + +fun cantAssignIncorrectTypeToField() { + var s: S = { v: 0 }; + s.v = createEmptyTuple(); +} + +/** +@compilation_should_fail +@stderr can not assign `tuple` to field of type `int` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6184.tolk b/tolk-tester/tests/invalid-typing/err-6184.tolk new file mode 100644 index 0000000000..16ba338620 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6184.tolk @@ -0,0 +1,11 @@ +struct S { v: () -> int; } + +fun cantCreateIncorrectTypeToField() { + var s: S = { v: beginCell }; +} + +/** +@compilation_should_fail +@stderr can not assign `() -> builder` to field of type `() -> int` +@stderr v: beginCell + */ diff --git a/tolk-tester/tests/invalid-typing/err-6230.tolk b/tolk-tester/tests/invalid-typing/err-6230.tolk new file mode 100644 index 0000000000..03c80df87f --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6230.tolk @@ -0,0 +1,10 @@ +struct S { a: int; b: int; } + +fun cantCastStructToTensor(s: S) { + s as (int, int); +} + +/** +@compilation_should_fail +@stderr type `S` can not be cast to `(int, int)` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6709.tolk b/tolk-tester/tests/invalid-typing/err-6709.tolk new file mode 100644 index 0000000000..d1fe412810 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6709.tolk @@ -0,0 +1,13 @@ +struct A { id: int; } +struct B { id: int; } +type AAlias = A; + +fun cantAssignDifferentStructures(a: A) { + var a2: AAlias = a; // ok + var b: B = a; +} + +/** +@compilation_should_fail +@stderr can not assign `A` to variable of type `B` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6829.tolk b/tolk-tester/tests/invalid-typing/err-6829.tolk index fbf4623674..d3f2e505c8 100644 --- a/tolk-tester/tests/invalid-typing/err-6829.tolk +++ b/tolk-tester/tests/invalid-typing/err-6829.tolk @@ -1,7 +1,9 @@ type Pair2 = (int, int); type Pair2Or3 = Pair2 | (int, int, int); -fun matchDoesntCoverAllCases(a: Pair2Or3 | slice | [[int]] | builder) { +struct A {} + +fun matchDoesntCoverAllCases(a: Pair2Or3 | slice | [[int]] | A) { if (a !is slice) { match (a) { (int, int) => {} @@ -12,5 +14,5 @@ fun matchDoesntCoverAllCases(a: Pair2Or3 | slice | [[int]] | builder) { /** @compilation_should_fail -@stderr `match` does not cover all possible types; missing types are: `(int, int, int)`, `builder` +@stderr `match` does not cover all possible types; missing types are: `(int, int, int)`, `A` */ diff --git a/tolk-tester/tests/invalid-typing/err-6845.tolk b/tolk-tester/tests/invalid-typing/err-6845.tolk new file mode 100644 index 0000000000..b739ce52ab --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6845.tolk @@ -0,0 +1,12 @@ +struct A { id: int; value: int; } + +fun takeTensor(v: (int, int)) {} + +fun cantPassStructAsTensor() { + takeTensor(A{id: 1, value:2}); +} + +/** +@compilation_should_fail +@stderr can not pass `A` to `(int, int)` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6942.tolk b/tolk-tester/tests/invalid-typing/err-6942.tolk new file mode 100644 index 0000000000..0ffe5c218b --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6942.tolk @@ -0,0 +1,19 @@ +struct Point { x: int; y: int; } + +fun methodOverNullable(p: Point?) {} +fun method(p: Point) {} + +fun cantAssignNullableObjectField(a: Point?) { + if (a != null) { + a.x = 10; // ok + } + a.methodOverNullable(); // ok + a.method(); // also ok, would have failed on type checking, but doesn't reach type checking + a.y = 20; +} + +/** +@compilation_should_fail +@stderr can not access field `y` of a possibly nullable object `Point?` +@stderr check it via `obj != null` or use non-null assertion `obj!` operator + */ diff --git a/tolk-tester/tests/meaningful-1.tolk b/tolk-tester/tests/meaningful-1.tolk new file mode 100644 index 0000000000..19356dc52c --- /dev/null +++ b/tolk-tester/tests/meaningful-1.tolk @@ -0,0 +1,43 @@ +struct CounterReset { + initialValue: int64; +} + +struct CounterIncrement1 { +} + +struct CounterIncrement { + byValue: int32; +} + +type MyMessage = + CounterIncrement + | CounterIncrement1 + | CounterReset; + +fun fakeParseMessage(mode: int): MyMessage { + if (mode == 1) { + return CounterIncrement1 {}; + } else if (mode > 0) { + return CounterIncrement { byValue: mode }; + } else { + return CounterReset { initialValue: 0 }; + } +} + +fun onInternalMessage(curCounter: int, mode: int) { + var m = fakeParseMessage(mode); + __expect_type(m, "MyMessage"); + + val nextCounter: int = match (m) { + CounterReset => m.initialValue, + CounterIncrement => curCounter + m.byValue, + CounterIncrement1 => curCounter + 1, + }; + return nextCounter; +} + +/** +@testcase | 0 | 50 0 | 0 +@testcase | 0 | 50 1 | 51 +@testcase | 0 | 50 9 | 59 + */ diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 946c00222a..77374bdcef 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -1,6 +1,17 @@ // the goal of this file is not only to @testcase results — // but to check that this file compiles +struct Point { + x: int; + y: int; +} + +struct PointOfNulls { + x: int?; + y: int?; + sub: Point?; +} + fun getNullableInt(): int? { return 5; } fun getNullableSlice(): slice? { return null; } fun takeNullableInt(a: int?) {} @@ -12,6 +23,7 @@ fun sameTensor(t: (int, int)) { return t; } fun sameTensor2(t: (int?, Pair4N)) { return t; } fun eq(v: T) { return v; } fun getTwo(): X { return 2 as X; } +fun assignNullTo(mutate v: T?) { v = null; } type Pair4N = (slice, slice, slice, builder)?; @@ -659,6 +671,44 @@ fun test60() { } +@method_id(161) +fun test61(p: Point) { + var p2: Point? = p; + __expect_type(p2, "Point"); + if (p.x < 10) { + assignNullTo(mutate p2); + } + __expect_type(p2, "Point?"); + return (p2, p2!); +} + +fun test62() { + var p = PointOfNulls { x: 1, y: null, sub: null }; + __expect_type(p.x, "int?"); + __expect_type(p.sub, "Point?"); + if (p.sub == null) { + p.x! > 10 ? p.sub = { x: 1, y : 2 } : Point { x: 3, y: 4 }; + __expect_type(p.sub, "Point?"); + p.x! > 10 ? p.sub = { x: 1, y : 2 } : p.sub = eq({ x: 3, y: 4 }); + } + __expect_type(p.sub, "Point"); + __expect_type(p!!.sub!!, "Point"); + p.sub.x + p.x! + p.y!; + return random() ? null : p; +} + +fun test63() { + __expect_type(test62, "() -> PointOfNulls?"); + var r = test62(); + __expect_type(r!.sub, "Point?"); + if (r != null) { + r.sub = { x: 1, y: 2 }; + } + __expect_type(r!.sub, "Point?"); + r!.sub = { x: 1, y: 2 }; + __expect_type(r!.sub, "Point"); +} + fun main(x: int?): int { return x == null ? -1 : x; @@ -685,6 +735,8 @@ fun main(x: int?): int { @testcase | 147 | | (null) (null) 100 (null) 100 (null) (null) 0 @testcase | 158 | | 123 10 123 5 @testcase | 160 | | 101 109 +@testcase | 161 | 9 9 | (null) (null) 0 (null) (null) +@testcase | 161 | 19 0 | 19 0 133 19 0 @stderr warning: expression of type `int` can never be `null`, this condition is always true @stderr warning: unreachable code diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk new file mode 100644 index 0000000000..91dc005435 --- /dev/null +++ b/tolk-tester/tests/struct-tests.tolk @@ -0,0 +1,519 @@ +struct JustInt { + value: int; +} + +struct JustIntWrapper { + int: JustInt; +} + +type MInt = int; +type JustIntAlias = JustInt; +type JustIntAlias2 = JustIntAlias; + +struct Storage { + owner: User, + lastPoint: Point, +} + +struct Point { + x: int; + y: MInt +} + +struct User { + id: int; + name: slice; +} + +struct OuterStorage { + st: Storage, + stMaybe: Storage? +} + +struct WithTensorInside { + coords: (int, int); + tup: [int, int]; + otherCoords: (int, int)?; + otherTup: tuple; +} + +fun sumCoords(p: Point) { + return p.x + p.y; +} + +fun getStorage1(x: int): Storage { + val owner = User { id: 1, name: "u" }; + return { + owner, + lastPoint: { x, y: 10, }, + }; +} + +fun generatePoint(x: int, y: int): Point { + return { x, y }; +} + +fun assignCoords(mutate self: Point) { + self.x = 10; + self.y = 20; +} + +@method_id(101) +fun test1() { + var i1 = JustInt { value: 1 }; + var i2: JustInt = JustInt { value: 2, }; + var i3: JustInt = { `value`: 3 }; + __expect_type(i3, "JustInt"); + __expect_type(i3!, "JustInt"); + return (i1, i2, i3, [i1, JustInt{value:5}]); +} + +@method_id(102) +fun test2() { + var i1 = JustIntAlias2 { value: 1 }; + var i2: JustInt = JustIntAlias2 { value: 2, }; + var i3: JustIntAlias2 = { `value`: 3 }; + __expect_type(i1.value, "int"); + __expect_type(i1.value + i2.value + i3.value, "int"); + __expect_type(i3, "JustIntAlias2"); + __expect_type([i2, i3], "[JustInt, JustIntAlias2]"); + __expect_type([i2] as [JustIntAlias], "[JustIntAlias]"); + return (i1, i2, i3, [i1, JustIntAlias{value:5}]); +} + +@method_id(103) +fun test3() { + return generatePoint(5, 6); +} + +@method_id(104) +fun test4() { + var p: Point = { x: 10, y: 20 }; + return (p == null, p, p = {x:30,y:40}, p != null); +} + +@method_id(105) +fun test5() { + var b = PointAlias { y: 20, x: 10 }; + var p: PointAlias = Point { x: b.x, y: b.y }; + p.x += 5; + return (sumCoords(p), p.y += 10, p.sumCoords()); +} + +struct BacktickNames { + `my()id`: MInt, + `100500`: int; +} + +@method_id(106) +fun test6() { + val b: BacktickNames = { `100500`: 20, `my()id`: 10 }; + var p: Point = { x: 0, y: 0 }; + p.x += b.`my()id`; + p.y += b.100500; + return p; +} + +@method_id(107) +fun test7() { + var s = getStorage1(5); + s.owner.name = beginCell().storeInt(s.lastPoint.x, 32).endCell().beginParse(); + var s2 = s; + assignCoords(mutate s.lastPoint); + return (s.owner.name.loadInt(32), s2.owner.name.loadInt(32), s.lastPoint, s2.lastPoint.sumCoords()); +} + +struct Empty{} +struct OuterEmpty { nothing: EmptyAlias } +type EmptyAlias = Empty; + +@method_id(108) +fun test8() { + var e1: Empty = {}; + var o1: OuterEmpty = { nothing: e1 }; + var o2 = OuterEmpty { nothing: {} }; + __expect_type(o2.nothing, "EmptyAlias"); + var o3 = Empty{} as Empty?; + var o4 = null as Empty?; + var o5: EmptyAlias? = o3!; + var o6: OuterEmpty? = { nothing: {} }; + var o7 = o3! as EmptyAlias?; + var o8 = o6 as OuterEmpty?; + __expect_type(o3, "Empty?"); + __expect_type(o6, "OuterEmpty"); + __expect_type(o8, "OuterEmpty?"); + return (e1, Empty{}, EmptyAlias{}, o2, o1, 777, o3, 777, o4, 777, o5, 777, o6, 777, o7, 777, o8, 777, o3!, 777); +} + +fun maxCoord(p: Point): int8 { + return p.x > p.y ? p.x : p.y; +} + +@method_id(109) +fun test9() { + var p = Point { x: 80, y: 100 }; + p.assignCoords(); + return (maxCoord({y: 70, x: 30}), maxCoord(generatePoint(30, 20)), maxCoord(p), p.maxCoord(), Point{x:-80,y:-80}.maxCoord()); +} + +@method_id(110) +fun test10(notNull: bool) { + var p: Point? = null; + __expect_type(p, "null"); + if (notNull) { + p = { x: 1, y: 2 }; + __expect_type(p, "Point"); + __expect_type(p.x, "int"); + __expect_type(p.y, "MInt"); + __expect_type(p.x + p.y, "int"); + assignCoords(mutate p); + } + __expect_type(p, "Point?"); + return p; +} + +@method_id(111) +fun test11(notNull: bool) { + var os: OuterStorage = { + st: { owner: { id: 0, name: "" }, lastPoint: { x: 0, y: 0 } }, + stMaybe: notNull ? { owner: { id: 1, name: "" }, lastPoint: { x: 1, y: 1 } } : null, + }; + if (os.stMaybe == null) { + os!.st!.lastPoint!.y = 2; + os.stMaybe = { owner: { id: 2, name: "" }, lastPoint: { x: 2, y: 3 } }; + } else { + os.stMaybe.owner.id += 1; + os.stMaybe.lastPoint = { x: 3, y: 4 }; + } + if (!notNull) { + assignCoords(mutate os.stMaybe.lastPoint); + os.stMaybe.lastPoint.assignCoords(); + } + return (os.st.lastPoint, os.stMaybe.lastPoint, os.stMaybe.lastPoint.sumCoords()); +} + +@method_id(112) +fun test12(notNull: bool): Point? { + return notNull ? { x: 1, y : 2 } : null; +} + +@method_id(113) +fun test13() { + return null as User?; +} + +@method_id(114) +fun test14() { + var p: Point? = { y: 2, x: 1 }; + __expect_type(p, "Point"); + __expect_type(p as Point, "Point"); + __expect_type(p as Point?, "Point?"); + return p; +} + +@method_id(115) +fun test15() { + var os = OuterStorage { + st: { owner: { id: 0, name: "" }, lastPoint: { x: 0, y: 0 } }, + stMaybe: null, + }; + __expect_type(os.stMaybe, "Storage?"); + return os.stMaybe; +} + +fun pushZero(mutate t: tuple) { + t.tuplePush(0); +} + +@method_id(116) +fun test16(): WithTensorInside? { + var wt: WithTensorInside = { + coords: (1, 2), + tup: [3, 4], + otherCoords: null, + otherTup: createEmptyTuple(), + }; + wt.coords = (-1, -2); + wt.coords.1 -= 2; + wt.tup.0 += 5; + wt.otherCoords!.0 = 7; + wt.otherTup.tuplePush(0); + pushZero(mutate wt.otherTup); + (wt.otherTup.1, wt.otherTup.0) = (11, 10); + return wt; +} + +@method_id(117) +fun test17(x: JustInt?) { + var w1: JustIntWrapper = { int: { value: x == null ? -1 : x.value } }; + var w2: JustIntWrapper? = x == null ? x : { int: { value: x.value } }; + return (x, x!, x!.value, w1, w2); +} + +fun sumXY

(point: P) { return point.x + point.y; } @method_id(118) @@ -488,11 +489,11 @@ fun test34() { return o1; } -@pure fun getXPure() { return 1; } -@pure fun getYPure() { return 2; } +@pure @noinline fun getXPure() { return 1; } +@pure @noinline fun getYPure() { return 2; } global t_impure: tuple; -fun getXImpure() { t_impure.push(1); return 1; } -fun getYImpure() { t_impure.push(2); return 2; } +@noinline fun getXImpure() { t_impure.push(1); return 1; } +@noinline fun getYImpure() { t_impure.push(2); return 2; } @method_id(135) fun test35() { diff --git a/tolk-tester/tests/try-catch-tests.tolk b/tolk-tester/tests/try-catch-tests.tolk index 8acb6f2c78..b1878955e9 100644 --- a/tolk-tester/tests/try-catch-tests.tolk +++ b/tolk-tester/tests/try-catch-tests.tolk @@ -164,6 +164,7 @@ fun test109(): (int, int) { return (g_reg, l_reg); } +@noinline fun alwaysThrow123(): never { throw 123; } @@ -173,6 +174,7 @@ fun alwaysThrowX(x: int): never { else { throw (x, null); } } +@noinline fun anotherNever(throw123: bool): never { if (throw123) { alwaysThrow123(); } alwaysThrowX(456); @@ -224,9 +226,13 @@ fun test111(b: bool) { } } -fun mySetCode(newCode: slice): void +fun mySetCodeAsm(newCode: slice): void asm "SETCODE"; +fun mySetCode(newCode: slice) { + mySetCodeAsm(newCode); +} + fun testCodegen3(numberId: int, paramVal: cell) { if (numberId == -1000) { var cs = paramVal.beginParse(); @@ -288,7 +294,7 @@ fun main() { @testcase | 112 | 5 | 138 @testcase | 113 | | 2048 -@code_hash 100812963733861852648745958727639570706776197131971804296829155862118445447691 +@code_hash 91375323652563311931660098165979913104421063599680251769401655783153201389325 @fif_codegen """ diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 422b08bdb2..53e6caefa2 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -535,6 +535,7 @@ fun test47() { return (checkSimple(1), checkSimple(beginCell().endCell().beginParse()), checkSimple(100), checkSimple(beginCell())); } +@noinline fun checkGeneric3(a: T1 | T2 | T3): (int, T1 | T2?) { var vv: T1? = match (a) { T1 => a, T2 => null, T3 => null }; return match (a) { diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index 2e0bf0cad3..4db9233a9e 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -22,6 +22,7 @@ set(TOLK_SOURCE pipe-check-serialized-fields.cpp pipe-constant-folding.cpp pipe-optimize-boolean-expr.cpp + pipe-detect-inline-in-place.cpp pipe-ast-to-legacy.cpp pipe-find-unused-symbols.cpp pipe-generate-fif-output.cpp diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 9a3a4e6fb0..5c5953415b 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -1347,6 +1347,7 @@ static V parse_annotation(Lexer& lex) { throw ParseError(loc, "unknown annotation " + static_cast(name)); case AnnotationKind::inline_simple: case AnnotationKind::inline_ref: + case AnnotationKind::noinline: case AnnotationKind::pure: if (v_arg) { throw ParseError(v_arg->loc, "arguments aren't allowed for " + static_cast(name)); @@ -1471,13 +1472,17 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vectorkind) { case AnnotationKind::inline_simple: - flags |= FunctionData::flagInline; + inline_mode = FunctionInlineMode::inlineViaFif; // maybe will be replaced by inlineInPlace later break; case AnnotationKind::inline_ref: - flags |= FunctionData::flagInlineRef; + inline_mode = FunctionInlineMode::inlineRef; + break; + case AnnotationKind::noinline: + inline_mode = FunctionInlineMode::noInline; break; case AnnotationKind::pure: flags |= FunctionData::flagMarkedAsPure; @@ -1502,7 +1507,7 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(loc, v_ident, v_param_list, v_body, receiver_type, ret_type, genericsT_list, std::move(tvm_method_id), flags); + return createV(loc, v_ident, v_param_list, v_body, receiver_type, ret_type, genericsT_list, std::move(tvm_method_id), flags, inline_mode); } static AnyV parse_struct_field(Lexer& lex) { diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 03a9b0a53b..2aabec2b6f 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -335,7 +335,8 @@ class ASTReplicator final { clone(v_orig->return_type_node), v_orig->genericsT_list ? clone(v_orig->genericsT_list) : nullptr, v_orig->tvm_method_id, - v_orig->flags + v_orig->flags, + v_orig->inline_mode ); } diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 59a18a1631..27c7e4c13b 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -60,6 +60,9 @@ AnnotationKind Vertex::parse_kind(std::string_view name) { if (name == "@inline_ref") { return AnnotationKind::inline_ref; } + if (name == "@noinline") { + return AnnotationKind::noinline; + } if (name == "@method_id") { return AnnotationKind::method_id; } diff --git a/tolk/ast.h b/tolk/ast.h index cf82a0a178..400c849b1f 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -138,6 +138,7 @@ enum ASTNodeKind { enum class AnnotationKind { inline_simple, inline_ref, + noinline, method_id, pure, deprecated, @@ -1243,6 +1244,7 @@ struct Vertex final : ASTOtherVararg { V genericsT_list; // for non-generics it's nullptr td::RefInt256 tvm_method_id; // specified via @method_id annotation int flags; // from enum in FunctionData + FunctionInlineMode inline_mode; // from annotations like `@inline` or auto-detected "in-place" bool is_asm_function() const { return children.at(2)->kind == ast_asm_body; } bool is_code_function() const { return children.at(2)->kind == ast_block_statement; } @@ -1251,9 +1253,9 @@ struct Vertex final : ASTOtherVararg { Vertex* mutate() const { return const_cast(this); } void assign_fun_ref(FunctionPtr fun_ref); - Vertex(SrcLocation loc, V name_identifier, V parameters, AnyV body, AnyTypeV receiver_type_node, AnyTypeV return_type_node, V genericsT_list, td::RefInt256 tvm_method_id, int flags) + Vertex(SrcLocation loc, V name_identifier, V parameters, AnyV body, AnyTypeV receiver_type_node, AnyTypeV return_type_node, V genericsT_list, td::RefInt256 tvm_method_id, int flags, FunctionInlineMode inline_mode) : ASTOtherVararg(ast_function_declaration, loc, {name_identifier, parameters, body}) - , receiver_type_node(receiver_type_node), return_type_node(return_type_node), genericsT_list(genericsT_list), tvm_method_id(std::move(tvm_method_id)), flags(flags) {} + , receiver_type_node(receiver_type_node), return_type_node(return_type_node), genericsT_list(genericsT_list), tvm_method_id(std::move(tvm_method_id)), flags(flags), inline_mode(inline_mode) {} }; template<> diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 3696d550e6..d4db1edec4 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -41,14 +41,14 @@ static std::vector define_builtin_parameters(const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags) { - auto* f_sym = new FunctionData(name, {}, "", nullptr, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); + auto* f_sym = new FunctionData(name, {}, "", nullptr, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); G.symtable.add_function(f_sym); } static void define_builtin_method(const std::string& name, TypePtr receiver_type, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags, std::initializer_list arg_order = {}, std::initializer_list ret_order = {}) { std::string method_name = name.substr(name.find('.') + 1); - auto* f_sym = new FunctionData(name, {}, std::move(method_name), receiver_type, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); + auto* f_sym = new FunctionData(name, {}, std::move(method_name), receiver_type, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); f_sym->arg_order = arg_order; f_sym->ret_order = ret_order; G.symtable.add_function(f_sym); @@ -1517,6 +1517,9 @@ void define_builtins() { define_builtin_func("__expect_type", {TypeDataUnknown::create(), Slice}, Unit, nullptr, compile_expect_type, FunctionData::flagMarkedAsPure); + define_builtin_func("__expect_inline", {Bool}, Unit, nullptr, + compile_expect_type, + FunctionData::flagMarkedAsPure); define_builtin_method("T.__toTuple", typeT, {typeT}, TypeDataTuple::create(), declReceiverT, compile_any_object_to_tuple, FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp index c635c7483a..d392a58b8b 100644 --- a/tolk/codegen.cpp +++ b/tolk/codegen.cpp @@ -517,7 +517,7 @@ bool Op::generate_code_step(Stack& stack) { } } } else { - if (f_sym->is_inline() || f_sym->is_inline_ref()) { + if (f_sym->inline_mode == FunctionInlineMode::inlineViaFif || f_sym->inline_mode == FunctionInlineMode::inlineRef) { stack.o << AsmOp::Custom(loc, f_sym->name + " INLINECALLDICT", (int)right.size(), (int)left.size()); } else if (f_sym->is_code_function() && std::get(f_sym->body)->code->require_callxargs) { stack.o << AsmOp::Custom(loc, f_sym->name + (" PREPAREDICT"), 0, 2); diff --git a/tolk/fwd-declarations.h b/tolk/fwd-declarations.h index 97cba7200b..981eaa4a67 100644 --- a/tolk/fwd-declarations.h +++ b/tolk/fwd-declarations.h @@ -52,4 +52,12 @@ struct GenericsSubstitutions; struct SrcFile; +enum class FunctionInlineMode { + notCalculated, + inlineViaFif, + inlineRef, + inlineInPlace, + noInline, +}; + } // namespace tolk diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index c2c7a9b546..ae51cf7521 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -370,7 +370,7 @@ FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstituti } TypePtr new_return_type = replace_genericT_with_deduced(fun_ref->declared_return_type, allocatedTs); TypePtr new_receiver_type = replace_genericT_with_deduced(fun_ref->receiver_type, allocatedTs); - FunctionData* new_fun_ref = new FunctionData(new_name, fun_ref->loc, fun_ref->method_name, new_receiver_type, new_return_type, std::move(new_parameters), fun_ref->flags, nullptr, allocatedTs, fun_ref->body, fun_ref->ast_root); + FunctionData* new_fun_ref = new FunctionData(new_name, fun_ref->loc, fun_ref->method_name, new_receiver_type, new_return_type, std::move(new_parameters), fun_ref->flags, fun_ref->inline_mode, nullptr, allocatedTs, fun_ref->body, fun_ref->ast_root); new_fun_ref->arg_order = fun_ref->arg_order; new_fun_ref->ret_order = fun_ref->ret_order; new_fun_ref->base_fun_ref = fun_ref; diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 8a6c2911b6..f97d7a6898 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -337,6 +337,32 @@ class CheckReorderingForAsmArgOrderIsSafeVisitor final : public ASTVisitorFuncti } }; +// when a call to `f()` was inlined, f's body was processed, leaving some state +// that should be cleared upon next inlining; +// for instance, ir_idx of local variables point to caller (where f was inlined) +class ClearStateAfterInlineInPlace final : public ASTVisitorFunctionBody { + void visit(V v) override { + if (!v->marked_as_redef) { + v->var_ref->mutate()->assign_ir_idx({}); + } + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + tolk_assert(false); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + tolk_assert(fun_ref->is_inlined_in_place()); + + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + fun_ref->get_param(i).mutate()->assign_ir_idx({}); + } + + parent::visit(v_function->get_body()); + } +}; + // given `{some_expr}!`, return some_expr static AnyExprV unwrap_not_null_operator(AnyExprV v) { while (auto v_notnull = v->try_as()) { @@ -609,6 +635,54 @@ static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob tolk_assert(false); } +static std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, V v_call, const std::vector>& vars_per_arg) { + FunctionPtr fun_ref = v_call->fun_maybe; + tolk_assert(vars_per_arg.size() == fun_ref->parameters.size()); + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + const LocalVarData& param_i = fun_ref->get_param(i); + if (!param_i.is_used_as_lval() && !param_i.is_mutate_parameter()) { + // if param used for reading only, pass the same ir_idx as for an argument + // it decreases number of tmp variables and leads to better optimizations + // (being honest, it's quite strange that copy+LET may lead to more stack permutations) + param_i.mutate()->assign_ir_idx(std::vector(vars_per_arg[i])); + } else { + std::vector ith_param = code.create_var(param_i.declared_type, v_call->loc, param_i.name); + code.emplace_back(v_call->loc, Op::_Let, ith_param, vars_per_arg[i]); + param_i.mutate()->assign_ir_idx(std::move(ith_param)); + } + } + + std::vector rvect_call = code.create_tmp_var(ret_type, v_call->loc, "(inlined-return)"); + std::vector* backup_outer_inline = code.inline_rvect_out; + FunctionPtr backup_cur_fun = code.fun_ref; + bool backup_inline_before_return = code.inlining_before_immediate_return; + code.inline_rvect_out = &rvect_call; + code.inlining_before_immediate_return = stmt_before_immediate_return == v_call; + code.fun_ref = fun_ref; + + auto v_ast_root = fun_ref->ast_root->as(); + auto v_block = v_ast_root->get_body()->as(); + process_any_statement(v_block, code); + + if (fun_ref->has_mutate_params() && fun_ref->inferred_return_type == TypeDataVoid::create()) { + std::vector mutated_vars; + for (const LocalVarData& p_sym: fun_ref->parameters) { + if (p_sym.is_mutate_parameter()) { + mutated_vars.insert(mutated_vars.end(), p_sym.ir_idx.begin(), p_sym.ir_idx.end()); + } + } + code.emplace_back(v_call->loc, Op::_Let, rvect_call, std::move(mutated_vars)); + } + + ClearStateAfterInlineInPlace visitor; + visitor.start_visiting_function(fun_ref, v_ast_root); + + code.fun_ref = backup_cur_fun; + code.inline_rvect_out = backup_outer_inline; + code.inlining_before_immediate_return = backup_inline_before_return; + return rvect_call; +} + // "Transition to target (runtime) type" is the following process. // Imagine `fun analyze(t: (int,int)?)` and a call `analyze((1,2))`. // `(1,2)` (inferred_type) is 2 stack slots, but `t` (target_type) is 3 (one for null-flag). @@ -1468,9 +1542,14 @@ static std::vector process_function_call(V v, Code for (const std::vector& list : vars_per_arg) { args_vars.insert(args_vars.end(), list.cbegin(), list.cend()); } - std::vector rvect_apply = fun_ref->is_compile_time_special_gen() - ? gen_compile_time_code_instead_of_fun_call(code, v->loc, vars_per_arg, fun_ref) - : gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)", arg_order_already_equals_asm); + std::vector rvect_call; + if (fun_ref->is_compile_time_special_gen()) { + rvect_call = gen_compile_time_code_instead_of_fun_call(code, v->loc, vars_per_arg, fun_ref); + } else if (fun_ref->is_inlined_in_place() && fun_ref->is_code_function()) { + rvect_call = gen_inline_fun_call_in_place(code, op_call_type, v, vars_per_arg); + } else { + rvect_call = gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)", arg_order_already_equals_asm); + } if (fun_ref->has_mutate_params()) { LValContext local_lval; @@ -1491,20 +1570,20 @@ static std::vector process_function_call(V v, Code std::vector rvect = code.create_tmp_var(real_ret_type, v->loc, "(fun-call)"); left.insert(left.end(), rvect.begin(), rvect.end()); vars_modification_watcher.trigger_callbacks(left, v->loc); - code.emplace_back(v->loc, Op::_Let, left, rvect_apply); + code.emplace_back(v->loc, Op::_Let, left, rvect_call); local_lval.after_let(std::move(left), code, v->loc); - rvect_apply = rvect; + rvect_call = rvect; } if (obj_leftmost && fun_ref->does_return_self()) { if (obj_leftmost->is_lvalue) { // to handle if obj is global var, potentially re-assigned inside a chain - rvect_apply = pre_compile_expr(obj_leftmost, code, nullptr); + rvect_call = pre_compile_expr(obj_leftmost, code, nullptr); } else { // temporary object, not lvalue, pre_compile_expr - rvect_apply = vars_per_arg[asm_self_idx]; + rvect_call = vars_per_arg[asm_self_idx]; } } - return transition_to_target_type(std::move(rvect_apply), code, target_type, v); + return transition_to_target_type(std::move(rvect_call), code, target_type, v); } static std::vector process_braced_expression(V v, CodeBlob& code, TypePtr target_type) { @@ -1772,6 +1851,7 @@ static void process_block_statement(V v, CodeBlob& code) { FunctionPtr cur_f = code.fun_ref; bool does_f_return_nothing = cur_f->inferred_return_type == TypeDataVoid::create() && !cur_f->does_return_self() && !cur_f->has_mutate_params(); bool is_toplevel_block = v == cur_f->ast_root->as()->get_body(); + bool inlining_doesnt_prevent = code.inline_rvect_out == nullptr || code.inlining_before_immediate_return; // we want to optimize `match` and `if/else`: if it's the last statement, implicitly add "return" to every branch // (to generate IFJMP instead of nested IF ELSE); @@ -1782,11 +1862,11 @@ static void process_block_statement(V v, CodeBlob& code) { AnyV stmt = v->get_item(i); AnyV next_stmt = v->get_item(i + 1); bool next_is_empty_return = next_stmt->kind == ast_return_statement && !next_stmt->as()->has_return_value(); - stmt_before_immediate_return = next_is_empty_return && does_f_return_nothing ? stmt : nullptr; + stmt_before_immediate_return = next_is_empty_return && does_f_return_nothing && inlining_doesnt_prevent ? stmt : nullptr; process_any_statement(stmt, code); } AnyV last_stmt = v->get_item(v->size() - 1); - stmt_before_immediate_return = is_toplevel_block && does_f_return_nothing ? last_stmt : nullptr; + stmt_before_immediate_return = is_toplevel_block && does_f_return_nothing && inlining_doesnt_prevent ? last_stmt : nullptr; process_any_statement(last_stmt, code); stmt_before_immediate_return = backup; } @@ -1941,25 +2021,35 @@ static void process_throw_statement(V v, CodeBlob& code) { } static void process_return_statement(V v, CodeBlob& code) { - TypePtr child_target_type = code.fun_ref->inferred_return_type; - if (code.fun_ref->does_return_self()) { - child_target_type = code.fun_ref->parameters[0].declared_type; + // it's a function we're traversing AST of; + // probably, it's called and inlined into another (outer) function, we handle this below + FunctionPtr fun_ref = code.fun_ref; + + TypePtr child_target_type = fun_ref->inferred_return_type; + if (fun_ref->does_return_self()) { + child_target_type = fun_ref->parameters[0].declared_type; } std::vector return_vars = pre_compile_expr(v->get_return_value(), code, child_target_type); - if (code.fun_ref->does_return_self()) { + if (fun_ref->does_return_self()) { return_vars = {}; } - if (code.fun_ref->has_mutate_params()) { + if (fun_ref->has_mutate_params()) { std::vector mutated_vars; - for (const LocalVarData& p_sym: code.fun_ref->parameters) { + for (const LocalVarData& p_sym: fun_ref->parameters) { if (p_sym.is_mutate_parameter()) { mutated_vars.insert(mutated_vars.end(), p_sym.ir_idx.begin(), p_sym.ir_idx.end()); } } return_vars.insert(return_vars.begin(), mutated_vars.begin(), mutated_vars.end()); } - code.emplace_back(v->loc, Op::_Return, std::move(return_vars)); + + // if fun_ref is called and inlined into a parent, assign a result instead of generating a return statement + if (code.inline_rvect_out) { + code.emplace_back(v->loc, Op::_Let, *code.inline_rvect_out, std::move(return_vars)); + } else { + code.emplace_back(v->loc, Op::_Return, std::move(return_vars)); + } } // append "return" (void) to the end of the function @@ -2123,7 +2213,7 @@ class ConvertASTToLegacyOpVisitor final { static void start_visiting_function(FunctionPtr fun_ref, V) { tolk_assert(fun_ref->is_type_inferring_done()); - if (fun_ref->is_code_function()) { + if (fun_ref->is_code_function() && !fun_ref->is_inlined_in_place()) { convert_function_body_to_CodeBlob(fun_ref, std::get(fun_ref->body)); } else if (fun_ref->is_asm_function()) { convert_asm_body_to_AsmOp(fun_ref, std::get(fun_ref->body)); diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp index be4db27499..075e999820 100644 --- a/tolk/pipe-check-rvalue-lvalue.cpp +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -37,11 +37,11 @@ static void fire_error_cannot_be_used_as_lvalue(FunctionPtr cur_f, AnyV v, const } GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_modifying_immutable_variable(FunctionPtr cur_f, AnyExprV v, LocalVarPtr var_ref) { +static void fire_error_modifying_immutable_variable(FunctionPtr cur_f, SrcLocation loc, LocalVarPtr var_ref) { if (var_ref->param_idx == 0 && var_ref->name == "self") { - throw ParseError(cur_f, v->loc, "modifying `self`, which is immutable by default; probably, you want to declare `mutate self`"); + throw ParseError(cur_f, loc, "modifying `self`, which is immutable by default; probably, you want to declare `mutate self`"); } else { - throw ParseError(cur_f, v->loc, "modifying immutable variable `" + var_ref->name + "`"); + throw ParseError(cur_f, loc, "modifying immutable variable `" + var_ref->name + "`"); } } @@ -59,6 +59,13 @@ static void validate_function_used_as_noncall(FunctionPtr cur_f, AnyExprV v, Fun class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { FunctionPtr cur_f = nullptr; + void on_var_used_as_lvalue(SrcLocation loc, LocalVarPtr var_ref) const { + if (var_ref->is_immutable()) { + fire_error_modifying_immutable_variable(cur_f, loc, var_ref); + } + var_ref->mutate()->assign_used_as_lval(); + } + void visit(V v) override { if (v->is_lvalue) { fire_error_cannot_be_used_as_lvalue(cur_f, v, "braced expression"); @@ -162,8 +169,8 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { } if (auto as_ref = leftmost_obj->try_as()) { - if (LocalVarPtr var_ref = as_ref->sym->try_as(); var_ref && var_ref->is_immutable()) { - fire_error_modifying_immutable_variable(cur_f, leftmost_obj, var_ref); + if (LocalVarPtr var_ref = as_ref->sym->try_as()) { + on_var_used_as_lvalue(leftmost_obj->loc, var_ref); } } } @@ -211,8 +218,8 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { if (v->is_lvalue) { tolk_assert(v->sym); - if (LocalVarPtr var_ref = v->sym->try_as(); var_ref && var_ref->is_immutable()) { - fire_error_modifying_immutable_variable(cur_f, v, var_ref); + if (LocalVarPtr var_ref = v->sym->try_as()) { + on_var_used_as_lvalue(v->loc, var_ref); } else if (v->sym->try_as()) { fire(cur_f, v->loc, "modifying immutable constant"); } else if (v->sym->try_as()) { diff --git a/tolk/pipe-detect-inline-in-place.cpp b/tolk/pipe-detect-inline-in-place.cpp new file mode 100644 index 0000000000..21001c2aa5 --- /dev/null +++ b/tolk/pipe-detect-inline-in-place.cpp @@ -0,0 +1,299 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-visitor.h" +#include + +/* + * This pipe detects whether each function can be inlined in-place or not. + * Outcome: call `fun_ref->assign_inline_mode_in_place()` for "lightweight" or "called only once" functions, + * and they will be inlined in-place while converting AST to IR (to Ops), and won't be generated to Fift. + * + * Given AST only, there is no definite algorithm to predict whether a function is "simple" ("lightweight"), + * so that inlining it will do better. There are no correct metrics at AST level that can be mapped onto TVM complexity. + * So, instead of overcomplicating and fine-tuning an algorithm, we're heading a simple way: + * - if a function is tiny, inline it always + * - if a function is called only once, inline it (only there, obviously) + * - if a function is marked `@inline` (intended by the user), inline it in place (if possible) + * - see `should_auto_inline_if_not_prevented()` + * + * What can prevent a function from inlining? Several reasons: + * - it's recursive + * - it's used as non-call (a reference to it is taken) + * - see `is_inlining_prevented_even_if_annotated()` + * + * About `@inline` annotation. It means "user intention", so the compiler tries to inline it in-place + * without considering AST metrics. But anyway, something may prevent inlining (middle returns, for example). + * In this case, the desired flag is just not set; inline_mode remains inlineViaFif, we'll generate `PROCINLINE`. + * + * Besides inline detection, this pipe populates `fun_ref->n_times_called` (while building call graph). + * It's used in Fift output inside comments. + */ + +namespace tolk { + +// to calculate recursions, at first we populate the call graph +// (purpose: functions in recursive call chains can't be inlined) +static std::unordered_map> call_graph; + +// when traversing a function, collect some AST metrics used to detect whether it's lightweight +struct StateWhileTraversingFunction { + FunctionPtr fun_ref; + bool has_returns_in_the_middle = false; + int n_statements = 0; + int n_function_calls = 0; + int n_binary_operators = 0; + int n_control_flow = 0; + int n_globals = 0; + int max_block_depth = 0; + + explicit StateWhileTraversingFunction(FunctionPtr fun_ref) + : fun_ref(fun_ref) {} + + int calculate_ast_cost() const { + return n_function_calls + n_binary_operators + n_statements * 2 + + n_control_flow * 10 + n_globals * 5 + (max_block_depth - 1) * 10; + } + + bool is_inlining_prevented_even_if_annotated() const { + // even if user specified `@inline`, we can't do anything about recursions, for example; + // in this case, in-place inlining won't happen, we'll generate `PROCINLINE` to Fift + bool is_inside_recursion = fun_ref->n_times_called >= 9999; + return has_returns_in_the_middle || is_inside_recursion || fun_ref->is_used_as_noncall() || !fun_ref->is_code_function(); + } + + bool should_auto_inline_if_not_prevented() const { + // if a function is called only once, inline it regardless of its size + // (to prevent this, `@inline_ref` can be used, for example) + if (fun_ref->n_times_called == 1) { + return true; + } + + // if a function is lightweight, inline in regardless of how many times it's called + // (for instance, `Storage.load` is always inlined) + int approx_cost_per_call = calculate_ast_cost(); + if (approx_cost_per_call < 30) { + return true; + } + + // try to _somehow_ detect whether to inline it or not + return approx_cost_per_call * fun_ref->n_times_called < 150; + } +}; + +// traverse the AST, collect metrics, and in the end, probably set the inline flag +class DetectIfToInlineFunctionInPlaceVisitor final : ASTVisitorFunctionBody { + StateWhileTraversingFunction cur_state{nullptr}; + int block_depth = 0; + std::vector> collected_expect_inline; // `__expect_inline()` compiler assertions + +protected: + void visit(V v) override { + if (v->fun_maybe && v->fun_maybe->is_builtin_function() && v->fun_maybe->name == "__expect_inline") { + collected_expect_inline.push_back(v); + } else { + cur_state.n_function_calls++; + } + parent::visit(v); + } + + void visit(V v) override { + if (v->tok == tok_logical_and || v->tok == tok_logical_not) { + cur_state.n_control_flow++; + } else { + cur_state.n_binary_operators++; + } + parent::visit(v); + } + + void visit(V v) override { + if (v->sym->try_as()) { + cur_state.n_globals++; + } + } + + void visit(V v) override { + block_depth++; + cur_state.n_statements += v->size(); + cur_state.max_block_depth = std::max(cur_state.max_block_depth, block_depth); + parent::visit(v); + block_depth--; + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + // detect if `return` the last return statement in a function's body + // (currently in-place inlining for functions with returns in the middle is not supported) + auto body_block = cur_state.fun_ref->ast_root->as()->get_body()->as(); + bool is_last_statement = body_block->get_item(body_block->size() - 1) == v; + cur_state.has_returns_in_the_middle |= !is_last_statement; + parent::visit(v); + } + + public: + bool should_visit_function(FunctionPtr fun_ref) override { + // unsupported or no-sense cases + if (fun_ref->is_builtin_function() || fun_ref->is_asm_function() || fun_ref->is_generic_function() || + fun_ref->has_tvm_method_id() || !fun_ref->arg_order.empty() || !fun_ref->ret_order.empty() || + fun_ref->is_used_as_noncall()) { + return false; + } + // disabled by the user + if (fun_ref->inline_mode == FunctionInlineMode::noInline || fun_ref->inline_mode == FunctionInlineMode::inlineRef) { + return false; + } + // okay, start auto-detection; + // for functions marked `@inline` (inlineViaFif), probably we'll change to inlineInPlace + return true; + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_state = StateWhileTraversingFunction(fun_ref); + collected_expect_inline.clear(); + parent::visit(v_function->get_body()); + + bool prevented_anyway = cur_state.is_inlining_prevented_even_if_annotated(); + bool will_inline = false; + if (fun_ref->inline_mode == FunctionInlineMode::inlineViaFif) { + // if a function is marked `@inline`, so the user requested in to be inlined; + // if it's possible, do it; otherwise, leave it as `PROCINLINE` to Fift + will_inline = !prevented_anyway; + } else { + // a function is not marked `@inline` / `@inline_ref` / etc., so automatically decide + will_inline = !prevented_anyway && cur_state.should_auto_inline_if_not_prevented(); + } + + // handle `__expect_inline(true)` (assertions inside compiler tests) + for (auto v_expect : collected_expect_inline) { + tolk_assert(v_expect->get_num_args() == 1 && v_expect->get_arg(0)->get_expr()->kind == ast_bool_const); + bool expected = v_expect->get_arg(0)->get_expr()->as()->bool_val; + if (expected != will_inline) { + fire(fun_ref, v_expect->loc, "__expect_inline failed"); + } + } + + // okay, this function will be inlined, mark the flag + if (will_inline && fun_ref->n_times_called) { + fun_ref->mutate()->assign_inline_mode_in_place(); + } + } +}; + +// this visitor (called once for a function): +// 1) fills call_graph[cur_f] (all function calls from cur_f) +// 2) increments n_times_called +// as a result of applying it to every function, we get a full call graph and how many times each function was called; +// we'll use this call graph to detect recursive components (functions within recursions can not be inlined) +class CallGraphBuilderVisitor final : ASTVisitorFunctionBody { + FunctionPtr cur_f{nullptr}; + +protected: + void visit(V v) override { + if (FunctionPtr called_f = v->fun_maybe) { + if (called_f->is_code_function()) { + call_graph[cur_f].emplace_back(called_f); + } + called_f->mutate()->n_times_called++; + } + parent::visit(v); + } + + public: + bool should_visit_function(FunctionPtr fun_ref) override { + // don't include asm functions, we don't need them in calculations + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + call_graph[cur_f] = std::vector{}; + parent::visit(v_function->get_body()); + } +}; + +static void detect_recursive_functions() { + // 1) build call_graph (and calculate n_times_called also) + visit_ast_of_all_functions(); + + // 2) using call_graph, detect cycles (the smallest, non-optimized algorithm, okay for our needs) + for (const auto& it : call_graph) { + FunctionPtr f_start_from = it.first; + std::unordered_set visited; + std::function is_recursive_dfs = [&](FunctionPtr cur) -> bool { + for (FunctionPtr f_called : call_graph[cur]) { + if (f_called == f_start_from) + return true; + if (!visited.insert(f_called).second) + continue; + if (is_recursive_dfs(f_called)) + return true; + } + return false; + }; + if (!it.second.empty() && is_recursive_dfs(f_start_from)) { + f_start_from->mutate()->n_times_called = 9999; // means "recursive" + } + } +} + +void pipeline_detect_inline_in_place() { + detect_recursive_functions(); + visit_ast_of_all_functions(); + call_graph.clear(); +} + +} // namespace tolk diff --git a/tolk/pipe-find-unused-symbols.cpp b/tolk/pipe-find-unused-symbols.cpp index cce03cc901..dad565e0e0 100644 --- a/tolk/pipe-find-unused-symbols.cpp +++ b/tolk/pipe-find-unused-symbols.cpp @@ -37,7 +37,7 @@ namespace tolk { static void mark_function_used_dfs(const std::unique_ptr& op); static void mark_function_used(FunctionPtr fun_ref) { - if (!fun_ref->is_code_function() || fun_ref->is_really_used()) { // already handled + if (!fun_ref->is_code_function() || fun_ref->is_really_used() || fun_ref->is_inlined_in_place()) { // already handled return; } diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp index 6cfd506831..ca82cd3fbd 100644 --- a/tolk/pipe-generate-fif-output.cpp +++ b/tolk/pipe-generate-fif-output.cpp @@ -79,13 +79,17 @@ static void generate_output_func(FunctionPtr fun_ref) { std::cerr << "\n---------- resulting code for " << fun_ref->name << " -------------\n"; } const char* modifier = ""; - if (fun_ref->is_inline()) { + if (fun_ref->inline_mode == FunctionInlineMode::inlineViaFif) { modifier = "INLINE"; - } else if (fun_ref->is_inline_ref()) { + } else if (fun_ref->inline_mode == FunctionInlineMode::inlineRef) { modifier = "REF"; } if (G.settings.tolk_src_as_line_comments) { - std::cout << " // " << fun_ref->loc << std::endl; + std::cout << " // " << fun_ref->loc; + if (!fun_ref->n_times_called && !fun_ref->is_used_as_noncall() && !fun_ref->is_entrypoint() && !fun_ref->has_tvm_method_id()) { + std::cout << " (note: function never called!)"; + } + std::cout << std::endl; } std::cout << " " << fun_ref->name << " PROC" << modifier << ":<{"; int mode = 0; @@ -103,10 +107,10 @@ static void generate_output_func(FunctionPtr fun_ref) { if (G.settings.tolk_src_as_line_comments) { mode |= Stack::_LineComments; } - if (fun_ref->is_inline() && code->ops->noreturn()) { + if (fun_ref->inline_mode == FunctionInlineMode::inlineViaFif && code->ops->noreturn()) { mode |= Stack::_InlineFunc; } - if (fun_ref->is_inline() || fun_ref->is_inline_ref()) { + if (fun_ref->inline_mode == FunctionInlineMode::inlineViaFif || fun_ref->inline_mode == FunctionInlineMode::inlineRef) { mode |= Stack::_InlineAny; } code->generate_code(std::cout, mode, 2); @@ -133,11 +137,13 @@ void pipeline_generate_fif_output_to_std_cout() { std::cout << "PROGRAM{\n"; bool has_main_procedure = false; + int n_inlined_in_place = 0; for (FunctionPtr fun_ref : G.all_functions) { - if (!fun_ref->does_need_codegen()) { + if (fun_ref->is_asm_function() || !fun_ref->does_need_codegen()) { if (G.is_verbosity(2) && fun_ref->is_code_function()) { std::cerr << fun_ref->name << ": code not generated, function does not need codegen\n"; } + n_inlined_in_place += fun_ref->is_inlined_in_place(); continue; } @@ -157,6 +163,15 @@ void pipeline_generate_fif_output_to_std_cout() { throw Fatal("the contract has no entrypoint; forgot `fun onInternalMessage(...)`?"); } + if (n_inlined_in_place) { + std::cout << " // " << n_inlined_in_place << " functions inlined in-place:" << "\n"; + for (FunctionPtr fun_ref : G.all_functions) { + if (fun_ref->is_inlined_in_place()) { + std::cout << " // - " << fun_ref->name << " (" << fun_ref->n_times_called << (fun_ref->n_times_called == 1 ? " call" : " calls") << ")\n"; + } + } + } + for (GlobalVarPtr var_ref : G.all_global_vars) { if (!var_ref->is_really_used() && G.settings.remove_unused_functions) { if (G.is_verbosity(2)) { @@ -169,7 +184,7 @@ void pipeline_generate_fif_output_to_std_cout() { } for (FunctionPtr fun_ref : G.all_functions) { - if (!fun_ref->does_need_codegen()) { + if (fun_ref->is_asm_function() || !fun_ref->does_need_codegen()) { continue; } generate_output_func(fun_ref); diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 7eaa6ac91b..b4394a2d8e 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -215,7 +215,7 @@ static FunctionPtr register_function(V v, FunctionPtr const GenericsDeclaration* genericTs = nullptr; // at registering it's null; will be assigned after types resolving FunctionBody f_body = v->get_body()->kind == ast_block_statement ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); - FunctionData* f_sym = new FunctionData(std::move(name), v->loc, std::move(method_name), v->receiver_type_node, v->return_type_node, std::move(parameters), 0, genericTs, substitutedTs, f_body, v); + FunctionData* f_sym = new FunctionData(std::move(name), v->loc, std::move(method_name), v->receiver_type_node, v->return_type_node, std::move(parameters), 0, v->inline_mode, genericTs, substitutedTs, f_body, v); f_sym->base_fun_ref = base_fun_ref; // for `f`, here is `f` if (auto v_asm = v->get_body()->try_as()) { diff --git a/tolk/pipeline.h b/tolk/pipeline.h index 4e5beca20e..e4448deb14 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -44,6 +44,7 @@ void pipeline_check_pure_impure_operations(); void pipeline_check_serialized_fields(); void pipeline_constant_folding(); void pipeline_optimize_boolean_expressions(); +void pipeline_detect_inline_in_place(); void pipeline_convert_ast_to_legacy_Expr_Op(); void pipeline_find_unused_symbols(); diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 46d3949383..b9b4b1fa85 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -60,6 +60,10 @@ bool FunctionData::does_need_codegen() const { if (is_generic_function()) { return false; } + // if calls to this function were inlined in place, the function itself is omitted from fif + if (is_inlined_in_place()) { + return false; + } // currently, there is no inlining, all functions are codegenerated // (but actually, unused ones are later removed by Fift) // in the future, we may want to implement a true AST inlining for "simple" functions @@ -105,6 +109,10 @@ void FunctionData::assign_is_really_used() { this->flags |= flagReallyUsed; } +void FunctionData::assign_inline_mode_in_place() { + this->inline_mode = FunctionInlineMode::inlineInPlace; +} + void FunctionData::assign_arg_order(std::vector&& arg_order) { this->arg_order = std::move(arg_order); } @@ -129,6 +137,10 @@ void GlobalConstData::assign_init_value(AnyExprV init_value) { this->init_value = init_value; } +void LocalVarData::assign_used_as_lval() { + this->flags |= flagUsedAsLVal; +} + void LocalVarData::assign_ir_idx(std::vector&& ir_idx) { this->ir_idx = std::move(ir_idx); } diff --git a/tolk/symtable.h b/tolk/symtable.h index 15a52d148b..243c38a607 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -50,6 +50,7 @@ struct LocalVarData final : Symbol { flagMutateParameter = 1, // parameter was declared with `mutate` keyword flagImmutable = 2, // variable was declared via `val` (not `var`) flagLateInit = 4, // variable was declared via `lateinit` (not assigned at declaration) + flagUsedAsLVal = 8, // variable is assigned or in another way used as lvalue inside a function }; AnyTypeV type_node; // either at declaration `var x:int`, or if omitted, from assigned value `var x=2` @@ -80,9 +81,11 @@ struct LocalVarData final : Symbol { bool is_immutable() const { return flags & flagImmutable; } bool is_lateinit() const { return flags & flagLateInit; } bool is_mutate_parameter() const { return flags & flagMutateParameter; } + bool is_used_as_lval() const { return flags & flagUsedAsLVal; } bool has_default_value() const { return default_value != nullptr; } LocalVarData* mutate() const { return const_cast(this); } + void assign_used_as_lval(); void assign_ir_idx(std::vector&& ir_idx); void assign_resolved_type(TypePtr declared_type); void assign_inferred_type(TypePtr inferred_type); @@ -104,8 +107,6 @@ struct FunctionData final : Symbol { static constexpr int EMPTY_TVM_METHOD_ID = -10; enum { - flagInline = 1, // marked `@inline` - flagInlineRef = 2, // marked `@inline_ref` flagTypeInferringDone = 4, // type inferring step of function's body (all AST nodes assigning v->inferred_type) is done flagUsedAsNonCall = 8, // used not only as `f()`, but as a 1-st class function (assigned to var, pushed to tuple, etc.) flagMarkedAsPure = 16, // declared as `pure`, can't call impure and access globals, unused invocations are optimized out @@ -123,6 +124,8 @@ struct FunctionData final : Symbol { int tvm_method_id = EMPTY_TVM_METHOD_ID; int flags; + FunctionInlineMode inline_mode; + int n_times_called = 0; // calculated while building call graph; 9999 for recursions std::string method_name; // for `fun Container.store` here is "store" AnyTypeV receiver_type_node; // for `fun Container.store` here is `Container` @@ -141,9 +144,10 @@ struct FunctionData final : Symbol { FunctionBody body; AnyV ast_root; // V for user-defined (not builtin) - FunctionData(std::string name, SrcLocation loc, std::string method_name, AnyTypeV receiver_type_node, AnyTypeV return_type_node, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) + FunctionData(std::string name, SrcLocation loc, std::string method_name, AnyTypeV receiver_type_node, AnyTypeV return_type_node, std::vector parameters, int initial_flags, FunctionInlineMode inline_mode, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) : Symbol(std::move(name), loc) , flags(initial_flags) + , inline_mode(inline_mode) , method_name(std::move(method_name)) , receiver_type_node(receiver_type_node) , parameters(std::move(parameters)) @@ -153,9 +157,10 @@ struct FunctionData final : Symbol { , body(body) , ast_root(ast_root) { } - FunctionData(std::string name, SrcLocation loc, std::string method_name, TypePtr receiver_type, TypePtr declared_return_type, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) + FunctionData(std::string name, SrcLocation loc, std::string method_name, TypePtr receiver_type, TypePtr declared_return_type, std::vector parameters, int initial_flags, FunctionInlineMode inline_mode, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) : Symbol(std::move(name), loc) , flags(initial_flags) + , inline_mode(inline_mode) , method_name(std::move(method_name)) , receiver_type_node(nullptr) , receiver_type(receiver_type) @@ -189,8 +194,7 @@ struct FunctionData final : Symbol { bool is_generic_function() const { return genericTs != nullptr; } bool is_instantiation_of_generic_function() const { return substitutedTs != nullptr; } - bool is_inline() const { return flags & flagInline; } - bool is_inline_ref() const { return flags & flagInlineRef; } + bool is_inlined_in_place() const { return inline_mode == FunctionInlineMode::inlineInPlace; } bool is_type_inferring_done() const { return flags & flagTypeInferringDone; } bool is_used_as_noncall() const { return flags & flagUsedAsNonCall; } bool is_marked_as_pure() const { return flags & flagMarkedAsPure; } @@ -218,6 +222,7 @@ struct FunctionData final : Symbol { void assign_is_implicit_return(); void assign_is_type_inferring_done(); void assign_is_really_used(); + void assign_inline_mode_in_place(); void assign_arg_order(std::vector&& arg_order); }; diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp index 6ead2b149e..6f8bbba912 100644 --- a/tolk/tolk.cpp +++ b/tolk/tolk.cpp @@ -67,6 +67,7 @@ int tolk_proceed(const std::string &entrypoint_filename) { pipeline_check_serialized_fields(); pipeline_constant_folding(); pipeline_optimize_boolean_expressions(); + pipeline_detect_inline_in_place(); pipeline_convert_ast_to_legacy_Expr_Op(); pipeline_find_unused_symbols(); diff --git a/tolk/tolk.h b/tolk/tolk.h index a329b1d557..3f6dcadd82 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -810,7 +810,7 @@ bool apply_op(StackTransform& trans, const AsmOp& op); */ struct Optimizer { - static constexpr int optimize_depth = 20; + static constexpr int optimize_depth = 30; AsmOpConsList code_; int l_{0}, l2_{0}, p_, pb_, q_, indent_; bool debug_{false}; @@ -1048,6 +1048,8 @@ struct CodeBlob { std::string name; SrcLocation forced_loc; std::vector vars; + std::vector* inline_rvect_out = nullptr; + bool inlining_before_immediate_return = false; std::unique_ptr ops; std::unique_ptr* cur_ops; #ifdef TOLK_DEBUG From 37c9fcd6608ce5dbe9a2569190725c64b8ab9a97 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 15 Jun 2025 12:41:03 +0700 Subject: [PATCH 329/524] [Tolk] Change STSLICER to STSLICE in stdlib In real-world code, it requires fewer stack permutations, because slices are not constant --- crypto/smartcont/tolk-stdlib/common.tolk | 6 +++--- tolk-tester/tests/pack-unpack-6.tolk | 10 ++++------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 30aabed3a7..86995717f4 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -759,12 +759,12 @@ fun builder.storeUint(mutate self, x: int, len: int): self /// Stores a slice into a builder. @pure fun builder.storeSlice(mutate self, s: slice): self - asm "STSLICER"; + asm(s self) "STSLICE"; /// Stores an address into a builder. @pure fun builder.storeAddress(mutate self, addr: address): self - asm "STSLICER"; + asm(addr self) "STSLICE"; /// Stores amount of Toncoins into a builder. @pure @@ -792,7 +792,7 @@ fun builder.storeMaybeRef(mutate self, c: cell?): self /// Concatenates two builders. @pure fun builder.storeBuilder(mutate self, from: builder): self - asm "STBR"; + asm(from self) "STB"; /// Stores a slice representing TL addr_none$00 (two `0` bits). @pure diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk index 07ec41f852..34e40578cf 100644 --- a/tolk-tester/tests/pack-unpack-6.tolk +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -131,9 +131,9 @@ RETALT @fif_codegen """ test3 PROC:<{ - NEWC x{0109ab} PUSHSLICE - STSLICER + NEWC + STSLICE ENDC CTOS x{01} SDBEGINSQ @@ -168,8 +168,7 @@ IF:<{ 9 THROWIF 8 EQINT 9 THROWIFNOT - SWAP - STSLICER + STSLICE """ @fif_codegen @@ -177,8 +176,7 @@ IF:<{ test6 PROC:<{ x{11} PUSHSLICE NEWC - SWAP - STSLICER + STSLICE ENDC HASHCU """ From bce0e5e797de86f7ef8f243059c13bb10d12f73b Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 15 Jun 2025 22:19:53 +0700 Subject: [PATCH 330/524] [Tolk] The magic `lazy` keyword Lazy loading, partial loading, partial updating See pipe-lazy-load-insertions.cpp for comments --- crypto/smartcont/tolk-stdlib/common.tolk | 34 +- .../tests/invalid-semantics/err-4460.tolk | 14 + .../tests/invalid-serialization/err-7300.tolk | 15 + .../tests/invalid-serialization/err-7301.tolk | 14 + .../tests/invalid-serialization/err-7302.tolk | 16 + .../tests/invalid-serialization/err-7303.tolk | 11 + .../tests/invalid-serialization/err-7304.tolk | 22 + .../tests/invalid-serialization/err-7305.tolk | 19 + .../tests/invalid-serialization/err-7306.tolk | 17 + .../tests/invalid-serialization/err-7307.tolk | 17 + .../tests/invalid-serialization/err-7308.tolk | 18 + .../tests/invalid-serialization/err-7683.tolk | 12 + .../tests/invalid-serialization/err-7684.tolk | 11 + .../tests/invalid-serialization/err-7720.tolk | 15 + .../tests/invalid-serialization/err-7800.tolk | 17 + .../tests/invalid-serialization/err-7923.tolk | 14 + .../tests/invalid-typing/err-6366.tolk | 1 + tolk-tester/tests/lazy-algo-tests.tolk | 1505 +++++++++++++ tolk-tester/tests/lazy-load-tests.tolk | 1981 +++++++++++++++++ tolk-tester/tests/struct-tests.tolk | 4 +- tolk/CMakeLists.txt | 2 + tolk/ast-aux-data.h | 24 + tolk/ast-from-tokens.cpp | 39 +- tolk/ast-replacer.h | 4 + tolk/ast-replicator.h | 8 + tolk/ast-stringifier.h | 4 + tolk/ast-visitor.h | 6 +- tolk/ast.cpp | 8 + tolk/ast.h | 33 +- tolk/builtins.cpp | 6 + tolk/fwd-declarations.h | 2 + tolk/lazy-helpers.cpp | 101 + tolk/lazy-helpers.h | 91 + tolk/lexer.cpp | 1 + tolk/lexer.h | 1 + tolk/pack-unpack-api.cpp | 129 ++ tolk/pack-unpack-api.h | 9 + tolk/pack-unpack-serializers.cpp | 168 +- tolk/pack-unpack-serializers.h | 18 + tolk/pipe-ast-to-legacy.cpp | 247 +- tolk/pipe-calc-rvalue-lvalue.cpp | 14 + tolk/pipe-check-inferred-types.cpp | 4 +- tolk/pipe-check-rvalue-lvalue.cpp | 7 + tolk/pipe-generate-fif-output.cpp | 4 +- tolk/pipe-infer-types-and-calls.cpp | 30 +- tolk/pipe-lazy-load-insertions.cpp | 1032 +++++++++ tolk/pipe-resolve-identifiers.cpp | 2 +- tolk/pipeline.h | 1 + tolk/smart-casts-cfg.cpp | 11 + tolk/smart-casts-cfg.h | 1 + tolk/tolk.cpp | 1 + tolk/tolk.h | 17 +- tolk/type-system.cpp | 9 + tolk/type-system.h | 1 + 54 files changed, 5722 insertions(+), 70 deletions(-) create mode 100644 tolk-tester/tests/invalid-semantics/err-4460.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7300.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7301.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7302.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7303.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7304.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7305.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7306.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7307.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7308.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7683.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7684.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7720.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7800.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7923.tolk create mode 100644 tolk-tester/tests/lazy-algo-tests.tolk create mode 100644 tolk-tester/tests/lazy-load-tests.tolk create mode 100644 tolk/lazy-helpers.cpp create mode 100644 tolk/lazy-helpers.h create mode 100644 tolk/pipe-lazy-load-insertions.cpp diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 86995717f4..c2674a4c52 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -233,15 +233,17 @@ struct PackOptions { /// UnpackOptions allows you to control behavior of `MyStruct.fromCell(c)` and similar functions. struct UnpackOptions { - // after finished reading all fields from a cell/slice, call [slice.assertEnd] to ensure no remaining data left; - // it's the default behavior, it ensures that you've fully described data you're reading with a struct; - // example: `struct Point { x: int8; y: int8 }`, input "0102" is ok, "0102FF" will throw excno 9; - // note: setting this to false does not decrease gas (DROP from a stack and ENDS cost the same); - // note: this option controls [T.fromCell] and [T.fromSlice], but is ignored by [slice.loadAny] + /// after finished reading all fields from a cell/slice, call [slice.assertEnd] to ensure no remaining data left; + /// it's the default behavior, it ensures that you've fully described data you're reading with a struct; + /// example: `struct Point { x: int8; y: int8 }`, input "0102" is ok, "0102FF" will throw excno 9; + /// note: setting this to false does not decrease gas (DROP from a stack and ENDS cost the same); + /// note: this option controls [T.fromCell] and [T.fromSlice], but is ignored by [slice.loadAny]; + /// note: `lazy` ignores this option, because it reads fields on demand or even skips them assertEndAfterReading: bool = true, /// this excNo is thrown if a prefix doesn't match, e.g. for `struct (0x01) A` given input "88..."; - /// similarly, for a union type, this is thrown when none of the opcodes match + /// similarly, for a union type, this is thrown when none of the opcodes match; + /// note: `lazy` ignores this option if you have `else` in `match` (you write custom logic there) throwIfOpcodeDoesNotMatch: int = 63, } @@ -327,6 +329,26 @@ fun T.getDeclaredPackPrefix(): int fun T.getDeclaredPackPrefixLen(): int builtin; +/// Forces an object created by `lazy` to load fully. Returns the remaining slice (having read all fields). +/// Since `options.assertEndAfterReading` is ignored by `lazy` (fields are loaded on demand), +/// this method can help you overcome this, if you really need to check input consistency. +/// Example: +/// ``` +/// val msg = lazy CounterMessage.fromSlice(s); +/// match (msg) { // it's a lazy match, without creating a union on the stack +/// CounterIncrement => { +/// ... +/// newCounter = curCounter + msg.incBy; // `incBy` loaded here, on demand +/// msg.forceLoadLazyObject().assertEnd() // the purpose: get remainder +/// } +/// } +/// ``` +/// Note: while [slice.assertEnd] may seem reasonable, these checks are avoided in practice, +/// because the purpose of `lazy` is to auto-detect and load only necessary fields, not up to the end. +@pure +fun T.forceLoadLazyObject(self): slice + builtin; + /// Cell represents a typed cell reference (as opposed to untyped `cell`). /// Example: /// ``` diff --git a/tolk-tester/tests/invalid-semantics/err-4460.tolk b/tolk-tester/tests/invalid-semantics/err-4460.tolk new file mode 100644 index 0000000000..1cae8c7a96 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4460.tolk @@ -0,0 +1,14 @@ +struct Point { + x: int8; + y: int8; +} + +fun main() { + var p = lazy Point.fromSlice(stringHexToSlice("0A14")); + p.asdf; +} + +/** +@compilation_should_fail +@stderr field `asdf` doesn't exist in type `Point` + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7300.tolk b/tolk-tester/tests/invalid-serialization/err-7300.tolk new file mode 100644 index 0000000000..31930736e2 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7300.tolk @@ -0,0 +1,15 @@ +struct A { a1: int7; a2: int8; a3: int8; } +struct B { b1: int7; b2: int8; b3: int8; } +type AOrB = A | B; + +fun f() { + val o = lazy AOrB.fromSlice(""); + __expect_lazy(""); + return (o is A) ? o.a1 : o.b1; +} + +/** +@compilation_should_fail +@stderr `lazy` will not work here, because variable `o` it's used in a non-lazy manner +@stderr hint: lazy union may be used only in `match` statement + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7301.tolk b/tolk-tester/tests/invalid-serialization/err-7301.tolk new file mode 100644 index 0000000000..1843bc4837 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7301.tolk @@ -0,0 +1,14 @@ +type Int32Or64 = int32 | int64; + +fun f() { + val o = lazy Int32Or64.fromSlice(""); + match (o) { + int32 => return o + 1, + int64 => throw o, + } +} + +/** +@compilation_should_fail +@stderr `lazy` union should contain only structures, but it contains `int32` + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7302.tolk b/tolk-tester/tests/invalid-serialization/err-7302.tolk new file mode 100644 index 0000000000..7885f6d0c8 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7302.tolk @@ -0,0 +1,16 @@ +struct A{} +type StructAndNot = A | int8; + +fun f() { + var o = lazy StructAndNot.fromSlice(""); // neg5 + __expect_lazy(""); + match (o) { + A => {} + int8 => {} + } +} + +/** +@compilation_should_fail +@stderr `lazy` union should contain only structures, but it contains `int8` + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7303.tolk b/tolk-tester/tests/invalid-serialization/err-7303.tolk new file mode 100644 index 0000000000..3fe0302174 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7303.tolk @@ -0,0 +1,11 @@ +type PairInt8 = (int8, int8); + +fun f() { + var i = lazy PairInt8.fromSlice(stringHexToSlice("0102")); + return (i.0, i.1); +} + +/** +@compilation_should_fail +@stderr `lazy` is applicable to structs, not to `PairInt8` + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7304.tolk b/tolk-tester/tests/invalid-serialization/err-7304.tolk new file mode 100644 index 0000000000..4659caf80b --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7304.tolk @@ -0,0 +1,22 @@ +struct Counter7Increment { byValue: int7 } +struct Counter7Decrement { byValue: int7 } +type MsgEitherCounter = Counter7Increment | Counter7Decrement; + +fun f(c: cell) { + // because 2 matches + var msg = lazy MsgEitherCounter.fromCell(c); // neg8 + var t = createEmptyTuple(); + match (msg) { + Counter7Increment => {} + Counter7Decrement => {} + } + match (msg) { + Counter7Increment => {} + Counter7Decrement => {} + } +} + +/** +@compilation_should_fail +@stderr `lazy` will not work here, because variable `msg` it's used in a non-lazy manner + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7305.tolk b/tolk-tester/tests/invalid-serialization/err-7305.tolk new file mode 100644 index 0000000000..59fb5ba50b --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7305.tolk @@ -0,0 +1,19 @@ +struct Counter7Increment { byValue: int7 } +struct Counter7Decrement { byValue: int7 } +type MsgEitherCounter = Counter7Increment | Counter7Decrement; + +fun f() { + // because .toCell() + var msg = lazy MsgEitherCounter.fromSlice(""); // neg9 + match (msg) { + Counter7Increment => {} + Counter7Decrement => {} + } + return msg.toCell(); +} + +/** +@compilation_should_fail +@stderr `lazy` will not work here, because variable `msg` it's used in a non-lazy manner +@stderr lazy MsgEitherCounter + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7306.tolk b/tolk-tester/tests/invalid-serialization/err-7306.tolk new file mode 100644 index 0000000000..d515c7d494 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7306.tolk @@ -0,0 +1,17 @@ +struct Counter7Increment { byValue: int7 } +struct Counter7Decrement { byValue: int7 } +type MsgEitherCounter = Counter7Increment | Counter7Decrement; + +fun f() { + // because in a complex expression + var msg = lazy MsgEitherCounter.fromSlice(""); + match (msg) { + Counter7Increment => 1 as int32, + Counter7Decrement => 2 as int32, + }.toCell() +} + +/** +@compilation_should_fail +@stderr `lazy` will not work here, because variable `msg` it's used in a non-lazy manner + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7307.tolk b/tolk-tester/tests/invalid-serialization/err-7307.tolk new file mode 100644 index 0000000000..9265cf1ce3 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7307.tolk @@ -0,0 +1,17 @@ +struct Counter7Increment { byValue: int7 } +struct Counter7Decrement { byValue: int7 } +type MsgEitherCounter = Counter7Increment | Counter7Decrement; + +fun f() { + var msg = lazy MsgEitherCounter.fromSlice(""); // neg10 + match (msg) { + Counter7Increment => 1 as int32, + Counter7Decrement => 2 as int32, + else => throw 123, // not allowed in either + } +} + +/** +@compilation_should_fail +@stderr `else` is unreachable, because this `match` has only two options (0/1 prefixes) + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7308.tolk b/tolk-tester/tests/invalid-serialization/err-7308.tolk new file mode 100644 index 0000000000..a46964a86f --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7308.tolk @@ -0,0 +1,18 @@ +struct (0x01) A { } +struct (0x02) B { } +struct (0x03) C { } +type MyInput = A | B | C; + +fun f() { + var msg = lazy MyInput.fromSlice(""); + match (msg) { + A => {} + B => {} + else => throw 123, + } +} + +/** +@compilation_should_fail +@stderr `match` does not cover all possible types; missing types are: `C` + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7683.tolk b/tolk-tester/tests/invalid-serialization/err-7683.tolk new file mode 100644 index 0000000000..f82924c908 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7683.tolk @@ -0,0 +1,12 @@ +struct A { +} + +fun neg4() { + // because compiler doesn't analyze deep assignments + var (a, b) = (1, lazy A.fromSlice("")); +} + +/** +@compilation_should_fail +@stderr error: incorrect `lazy` operator usage, it's not directly assigned to a variable + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7684.tolk b/tolk-tester/tests/invalid-serialization/err-7684.tolk new file mode 100644 index 0000000000..cfb49b3aec --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7684.tolk @@ -0,0 +1,11 @@ +struct St { +} + +fun getStorage() { + return lazy St.fromCell(contract.getData()); +} + +/** +@compilation_should_fail +@stderr error: incorrect `lazy` operator usage, it's not directly assigned to a variable + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7720.tolk b/tolk-tester/tests/invalid-serialization/err-7720.tolk new file mode 100644 index 0000000000..ec1bc4d355 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7720.tolk @@ -0,0 +1,15 @@ +struct Point { + x: int8; + y: int8; +} + +fun neg11(s: slice) { + // mutating function can't be lazy + var p = lazy s.loadAny(); + return p.x; +} + +/** +@compilation_should_fail +@stderr `lazy` operator can only be used with built-in functions like fromCell/fromSlice or simple wrappers over them +*/ diff --git a/tolk-tester/tests/invalid-serialization/err-7800.tolk b/tolk-tester/tests/invalid-serialization/err-7800.tolk new file mode 100644 index 0000000000..1df8e5f5ca --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7800.tolk @@ -0,0 +1,17 @@ +struct Point { + x: int32; + y: int32; +} + +fun loadPoint(s: slice): Point { + return Point.fromSlice(s); +} + +fun main() { + var p = lazy loadPoint(stringHexToSlice("0000000100000002")); +} + +/** +@compilation_should_fail +@stderr `lazy` operator can only be used with built-in functions like fromCell/fromSlice + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7923.tolk b/tolk-tester/tests/invalid-serialization/err-7923.tolk new file mode 100644 index 0000000000..8b809560df --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7923.tolk @@ -0,0 +1,14 @@ +struct Point { + x: int8; + y: int8; +} + +fun main() { + var p = Point.fromSlice(stringHexToSlice("0102")); + p.forceLoadLazyObject(); +} + +/** +@compilation_should_fail +@stderr this method is applicable to lazy variables only + */ diff --git a/tolk-tester/tests/invalid-typing/err-6366.tolk b/tolk-tester/tests/invalid-typing/err-6366.tolk index cc6052d99e..cba32e7708 100644 --- a/tolk-tester/tests/invalid-typing/err-6366.tolk +++ b/tolk-tester/tests/invalid-typing/err-6366.tolk @@ -2,6 +2,7 @@ fun main() { match (0 as int | slice) { int => 1, + slice => 2, else => 10, }; } diff --git a/tolk-tester/tests/lazy-algo-tests.tolk b/tolk-tester/tests/lazy-algo-tests.tolk new file mode 100644 index 0000000000..363640ed37 --- /dev/null +++ b/tolk-tester/tests/lazy-algo-tests.tolk @@ -0,0 +1,1505 @@ +struct A { a1: int7; a2: int8; a3: int8; } +struct B { b1: int7; b2: int8; b3: int8; } +type AOrB = A | B; + +@noinline +fun A.getA1(self) { + return self.a1; +} + +fun AOrB.doMatch(self) { + __expect_inline(true); + match (self) { + A => {} + B => {} + } +} + +struct Fields9 { + f1: uint8; f2: uint8; f3: uint8; + f4: uint8; f5: uint8; f6: uint8; + f7: uint8; f8: uint8; f9: uint8; +} + +fun uint8.doNothing(self) {} + +@noinline +fun Fields9.method(self) {} + +fun Fields9.useAll(self) { + __expect_inline(true); + if (true) { self.method() } +} + +struct WithEitherABMiddle { + f1: int32; + f2: int32; + ab: A | B; + f4: int32; + f5: int32; + f6: int32; +} + +struct WithEitherABEnd { + f1: int32; + f2: int32; + f3: int32; + f4: int32; + f5: int32; + ab: A | B; +} + +fun WithEitherABEnd.checkF3(self, against: int) { + assert(self.f3 == against, 100); +} + +fun WithEitherABEnd.doMatch(self) { + __expect_inline(true); + match (self.ab) { + A => {} + B => {} + } +} + +fun rand() { return random.uint256() } + +struct Point { + x: int8; + y: int8; +} + +fun Point.assignX(mutate self, x: int) { + self.x = x; +} + +fun Point.getX(self) { + return self.x; +} + +fun Point.getXNoInline(self) { + __expect_inline(false); // due to returns in the middle + if (10>3) { + return self.x; + } + throw 123; +} + +fun Point.getCoord(self, getX: bool) { + return getX ? self.x : self.y; +} + +fun Point.getCoordWrapper(self, getX: bool) { + if (10<3) { throw 123 } + return self.getCoord(getX) +} + +fun getXOfPoint(p: Point) { + return p.x +} + +struct Point3d { + x: int32; + y: int32; + z: int32; +} + +struct Point78ForMaybe { + x: int7; + y: int8; +} + +type OptionalPoint78ForMaybe = Point78ForMaybe?; + +struct ComplexWithPoint { + f1: int8; + p: Point; + f3: int8; +} + +struct ComplexWithMaybePoint { + f1: int8; + p: Point78ForMaybe?; + f3: int8; +} + +struct Counter7Increment { byValue: int7 } +struct Counter7Decrement { byValue: int7 } +type MsgEitherCounter = Counter7Increment | Counter7Decrement; + +struct FieldsAndEitherCounter { + f1: int8; + f2: int8; + c: MsgEitherCounter; +} + +struct WalletStorage { + isSignatureAllowed: bool; + seqno: uint32; + walletId: uint32; + publicKey: uint256; + extensions: dict; +} + +@overflow1023_policy("suppress") +struct NotFixedWidthStorage { + f1: int8; + f2: int8?; + f3: int8; + f4: bool; + f5: cell; + f6: int8 | bool; + f7: address; + f8: int8; + f9: bits200; + f10: (bits400, ()); + f11: coins; +} + +struct WithTensor { + a1: int8; + t: (int8, int8); + a3: int8; +} + +struct WithMaybeTensor { + a1: int8; + a2: int8; + t: (int8, int8)?; +} + +struct MixedDataAndRefs { + i1: int8; + i2: bool; + a1: address; + e: (); + r1: cell; + r2: ((), cell); + i4: Point; + rm: cell?; + r4: Cell?; +} + +struct (0x01) CounterIncrement { byValue: int8 } +struct (0x02) CounterDecrement { byValue: int8 } +struct (0x03) CounterDecrementBy1 { } +struct (0x04) CounterReset { initialValue: int32 } +type MsgFullCounter = CounterIncrement | CounterDecrement | CounterDecrementBy1 | CounterReset; + +struct ItemState { + ownerAddress: address; + content: cell; + auction: cell?; + royaltyParams: cell; +} + +struct HasIncrementEnd { + a: int8; + inc: CounterIncrement; +} + + +fun loadWalletStorage() { + return WalletStorage.fromSlice(stringHexToSlice("8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_")); +} + +fun WalletStorage.load() { + return WalletStorage.fromCell(createEmptyCell()); +} + + + +fun algo1() { + val o = lazy Fields9.fromSlice(""); + if (rand()) { + __expect_lazy("[o] load f1"); + o.f1; + } +} + +fun algo2() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1"); + if (rand()) { + o.f1; + } else { + o.f1 + o.f1; + } +} + +fun algo3() { + val o = lazy Fields9.fromSlice(""); + if (rand()) { + return; + } + __expect_lazy("[o] load f1"); + o.f1.doNothing(); +} + +fun algo4() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1 f2"); + if (rand()) { + o.f1; + } else { + o.f2; + } +} + +fun algo5() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] skip (bits8), load f2, skip (bits8), load f4"); + o.f4; + o.f2; +} + +fun algo6() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1 f2, skip (bits40), load f8"); + if (rand()) { + o.f8; + } + o.f2; + o.f1 * o.f1 + o.f1 + o.f2 * o.f2; +} + +fun algo7() { + val o = lazy Fields9.fromSlice(""); + if (rand()) { + __expect_lazy("[o] load f1 f2 f3 f4 f5 f6 f7 f8 f9"); + o; + } else { + } +} + +fun algo8() { + if (rand()) { + val o: Fields9 = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1 f2 f3 f4 f5 f6 f7 f8 f9"); + o.method(); + } else { + } +} + +fun algo9() { + val p = lazy Point3d.fromCell(createEmptyCell()); + __expect_lazy("[p] load x y z"); + if (p.x && p.y) { + p.x; + p.y; + } else { + p.z; + } +} + +fun algo10() { + val p = lazy Point3d.fromSlice(""); + __expect_lazy("[p] load x, skip (bits32), load z"); + p.z; + if (rand()) { + p.x; + return; + } +} + +fun algo11() { + val p = lazy Point3d.fromSlice(""); + var num = 0; + if (10 > 2) { + num += 1; + } + __expect_lazy("[p] load x"); + rand() + ? p.x as int + : p.x + num; +} + +fun algo12() { + try { + val p = lazy Point3d.fromSlice(""); + try {} + catch {} + assert(10>3, 1); + __expect_lazy("[p] skip (bits32), load y"); + p.y * p.y; + } catch { + } +} + +fun algo13() { + val o = lazy Fields9.fromSlice(""); + if (10 > 3) { + __expect_lazy("[o] load f1"); + assert(o.f1 > 0) throw 400; + } +} + +fun algo14() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x"); + while (rand()) { + p.x; + } +} + +fun algo15() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] skip (bits8), load y"); + do { + p.y; + } while (rand()); +} + +fun algo16() { + val p = lazy Point3d.fromSlice(""); + __expect_lazy("[p] load x, skip (bits32), load z"); + do { + p.x; + } while (p.z); +} + +fun algo17() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x y"); + repeat(p.x) { + p.y; + } +} + +fun algo18() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1 f2, skip (bits16), load f5 f6"); + if (o.f1) { + if (o.f2) { + throw o.f6 + } + } + else { o.f5 } +} + +fun algo19() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x y"); + var p2 = p; + return (p2.x, p2.y); +} + +fun algo20() { + val c = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[c] skip (bits8), load p"); + return c.p.x; +} + +fun algo21() { + val c = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[c] load f1 p"); + if (c.f1 > 0) { + return c.p.y; + } + throw 0; +} + +fun algo22() { + var w = lazy WithTensor.fromSlice(""); + __expect_lazy("[w] skip (bits8), load t"); + w.t.0 = 10; +} + +fun algo23() { + var w = lazy WithMaybeTensor.fromSlice(""); + __expect_lazy("[w] skip (bits8), load a2 t"); + if (w.t == null) { + return w.a2; + } + return w.t.0; +} + +fun algo24() { + val w = lazy WalletStorage.fromSlice(""); + try { + __expect_lazy("[w] skip (bits1), load seqno"); + return w.seqno as int; + } catch (excno) { + return excno; + } +} + +fun algo25() { + val w = lazy WalletStorage.fromSlice(""); + __expect_lazy("[w] skip (bits1), load seqno"); + try { + return w.seqno as int; + } catch (excno) { + if (excno > 0) { + return w.seqno as int; + } + return excno; + } +} + +fun algo26() { + val w = lazy WalletStorage.fromSlice(""); + __expect_lazy("[w] skip (bits1), load seqno"); + try { + return w.seqno + 1; + } catch (excno) { + } + return w.seqno - 1; +} + +fun algo27() { + val w = lazy WalletStorage.fromSlice(""); + __expect_lazy("[w] skip (bits1), load seqno walletId"); + try { + return w.seqno + 1; + } catch (excno) { + } + if (10 > 3) { + return w.walletId as int; + } + return -1; +} + +fun algo28() { + val w = lazy WalletStorage.fromSlice(""); + try { + try { + return 1; + } catch { + __expect_lazy("[w] skip (bits1), load seqno walletId"); + return w.walletId as int; + w.seqno; + } + } catch (excno) { + return -1; + } +} + +fun algo29() { + val w = lazy WalletStorage.fromSlice(""); + try { + __expect_lazy("[w] skip (bits1), load seqno walletId"); + try { + return 1; + } catch { + return w.walletId as int; + } + w.seqno; + } catch (excno) { + return -1; + } +} + +fun algo30() { + val w = lazy WalletStorage.fromSlice(""); + if (true) { + try { + __expect_lazy("[w] skip (bits1), load seqno walletId"); + try { + return 1; + } catch { + return w.walletId as int; + } + w.seqno; + } catch (excno) { + return -1; + } + } +} + +fun algo31() { + val w = lazy WalletStorage.fromSlice(""); + if (false) {} + else { + __expect_lazy("[w] skip (bits65), load publicKey"); + while (true) { + w.publicKey + } + } +} + +fun algo32() { + val w = lazy WalletStorage.fromSlice(""); + __expect_lazy("[w] load isSignatureAllowed"); + match (true) { + true => { w.isSignatureAllowed } + false => {} + } +} + +fun algo33() { + val p = lazy ComplexWithPoint.fromSlice(""); + try { if(true) {} else { + __expect_lazy("[p] skip (bits8), load p"); + p.p.x; + }} catch{} +} + +fun algo34() { + val p = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[p] skip (bits8), load p f3"); + if (p.p.x) { } + else { p.f3 } +} + +fun algo35() { + var p = lazy ComplexWithPoint.fromSlice(""); + try { + __expect_lazy("[p] load f1 p"); + if (p.p.x) { } + else { p.p = {x:1, y:2} } + if (true) { + try {} catch (excno) { match (excno + p.f1) { } } + } + } catch {} +} + +fun algo36() { + var p1 = lazy ComplexWithPoint.fromSlice(""); + var p2 = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[p1] skip (bits8), load p"); + try { + __expect_lazy("[p2] load f1 p f3"); + match (true) { + true => { p2.f3 } + else => p2.p.x ? p2.f1 : p1.p.x + } + } catch { p1.p.y -= 1 } +} + +fun algo37() { + var p1 = lazy ComplexWithPoint.fromSlice(""); + var p2 = lazy ComplexWithPoint.fromSlice(""); + try { + if (10>3) { + __expect_lazy("[p2] load f1 p f3"); + var p1 = p2; + } + } catch { + __expect_lazy("[p1] skip (bits8), load p"); + p1.p.y -= 1 + } +} + +fun algo38() { + var p1 = lazy ComplexWithPoint.fromSlice(""); + var p2 = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[p1] skip (bits8), load p f3"); + try { + if (10>3) { + __expect_lazy("[p2] skip (bits24), load f3"); + p1.f3 = p2.f3; + } + } catch { + p1.p.y -= 1 + } +} + +fun algo39() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x"); + return p.getX(); +} + +@method_id(140) +fun algo40() { + val p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + return p.getCoord(false); +} + +@method_id(141) +fun algo41(getX: bool) { + val p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + return p.getCoordWrapper(getX); +} + +fun algo42() { + var p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x y"); // calling a mutating method — not available for lazy + p.assignX(10); +} + +fun algo43() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x y"); // not a method — not available for lazy (not implemented) + return getXOfPoint(p); +} + +fun algo44() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x y"); // not an inlined method + return p.getXNoInline(); +} + +fun algo46() { + var w = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[w] skip (bits64), load f3, skip (bits64), load ab"); + w.checkF3(0); + w.doMatch(); +} + +fun algo47() { + var f = lazy Fields9.fromSlice(""); + __expect_lazy("[f] load f1 f2 f3 f4 f5 f6 f7 f8 f9"); + f.useAll(); +} + +fun algo48() { + var p = lazy Point.fromSlice(""); + contract.getData(); + __expect_lazy("[p] load x y"); + p.forceLoadLazyObject(); +} + +fun algo49(forceLoad: bool) { + val l = lazy Fields9.fromSlice(""); + if (forceLoad) { + __expect_lazy("[l] load f1 f2 f3 f4 f5 f6 f7 f8 f9"); + try { l.forceLoadLazyObject() } + catch { l.f2 } + } +} + +fun algo50() { + var st = lazy NotFixedWidthStorage.fromSlice(""); + __expect_lazy("[st] skip (bits8) (int8?) (bits9) (int8 | bool) (address) (bits608), load f11"); + st.f11; +} + +fun algo51() { + var w1 = lazy MixedDataAndRefs.fromSlice(""); + var w2 = lazy MixedDataAndRefs.fromSlice(""); + var w3 = lazy MixedDataAndRefs.fromSlice(""); + var w4 = lazy MixedDataAndRefs.fromSlice(""); + var w5 = lazy MixedDataAndRefs.fromSlice(""); + var w6 = lazy MixedDataAndRefs.fromSlice(""); + + __expect_lazy("[w1] load r1"); + w1.r1; + __expect_lazy("[w2] skip (cell), load r2"); + w2.r2; + __expect_lazy("[w3] skip (bits9) (address) (cell) (((), cell)) (bits16), load rm"); + w3.rm; + __expect_lazy("[w4] skip (bits9) (address), load i4"); + w4.i4; + __expect_lazy("[w5] skip (bits8), load i2, skip (address), load e r1 i4"); + w5.i2; w5.e; w5.r1; w5.i4; +} + +fun algo52(cc: Cell?) { + val itemState = lazy cc!.load(); + __expect_lazy("[itemState] skip (address) (cell) (cell?), load royaltyParams"); + return itemState.royaltyParams; +} + +fun algo53() { + val i = lazy ItemState.fromSlice(""); + __expect_lazy("[i] load content"); + return i.content.beginParse(); +} + + +fun algo100() { + var o = lazy AOrB.fromSlice(""); + __expect_lazy("[o] lazy match"); + match (o) { + A => { + __expect_lazy("[o] load a1, skip (bits8), load a3"); + o.a1; + __expect_lazy(""); + o.a3 + } + B => { + __expect_lazy("[o] load b1 b2"); + o.b2; + __expect_lazy(""); + o.b1 + } + } +} + +fun algo101() { + val o = lazy WithEitherABMiddle.fromSlice(""); + __expect_lazy("[o] skip (bits64), load ab, skip (bits32), load f5"); + o.f5; + o.ab; +} + +fun algo102() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits128), load f5 ab"); + o.f5; + o.ab; +} + +fun algo103() { + val o = lazy WithEitherABMiddle.fromSlice(""); + __expect_lazy("[o] skip (bits64), load ab"); + match (o.ab) { + A => {} + B => {} + } +} + +fun algo104() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits160), lazy match ab"); + match (o.ab) { + A => {} + B => {} + } +} + +fun algo105() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] load f1 f2 f3, skip (bits64), lazy match ab"); + match (o.ab) { + A => { + assert(o.f1 > o.f2, o.f3); + // __expect_lazy("[o.ab] load a1 a2"); + return o.ab.a1 + o.ab.a2; + } + B => { + // __expect_lazy("[o.ab] load b1 b2 b3"); + val cp = o.ab; + return cp.b1 + cp.b2; + } + } +} + +fun algo106() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits96), load f4, skip (bits32), lazy match ab"); + if (rand()) { + o.f4; + } else { + match (o.ab) { A => {} B => { + o.ab.b2; + } } + o.f4; + } +} + +fun algo107() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits160), load ab"); + if (o.ab is B) {} + match (o.ab) {} +} + +fun algo108(): int7 | int8 { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits160), lazy match ab"); + return match (o.ab) { + A => { + // __expect_lazy("[o.ab] load a1"); + return o.ab.a1; + } + B => { + // __expect_lazy("[o.ab] skip b1, load b2"); + return o.ab.b2; + } + } +} + +fun algo109(): int { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] load f1 f2, skip (bits32), load f4, skip (bits32), lazy match ab"); + match (o.ab) { + A => { + assert(o.f1 > o.f2, 100); + // __expect_lazy("[o.ab] load a1"); + return o.f1 + o.ab.a1; + } + B => { + return o.f2 + o.f4; + } + } +} + +fun algo110() { + val p = lazy Point3d.fromSlice(""); + __expect_lazy("[p] load x y z"); + match (p.x) { + 1 => p.y, + 2 => p.z, + } +} + +fun algo111() { + val p = lazy Point3d.fromSlice(""); + __expect_lazy("[p] load x y z"); + match (p.x) { + 1 => 1, + else => { var x = p.y + p.y } + } + p.z; +} + +fun algo112() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1 f2, skip (bits8), load f4 f5 f6"); + match (o.f2) { + uint8 => { + o.f1 + match(o.f4) { 1 => o.f4, else => o.f5 } + } + } + o.f6; +} + +fun algo113() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1"); + match (o.f1) { + uint8 => { + debug.print(o.f1); + } + } + o.f1; +} + +type PointOrPoint3d = Point | Point3d; + +fun algo114() { + val o = lazy PointOrPoint3d.fromSlice(""); + __expect_lazy("[o] lazy match"); + match (o) { + Point => { + __expect_lazy("[o] load x y"); + return o.x + o.y; + } + Point3d => { + __expect_lazy("[o] load x y z"); + var p = o; + return p.x + p.y; + } + } +} + +fun algo115() { + val o = lazy WithEitherABMiddle.fromSlice(""); + assert(10>3, 1); + __expect_lazy("[o] load f1 f2 ab f4"); + o.f1 + o.f2; + match (o.ab) { + A => { o.f1 + o.ab.a1 } + B => { o.f1 + o.f2 + o.ab.b1 } + } + o.f4; +} + +fun algo116() { + val o = lazy WithEitherABMiddle.fromSlice(""); + __expect_lazy("[o] load f1 f2 ab f4 f5"); + o.f1 + o.f2; + match (o.ab) { + A => { return o.f4 } + B => throw o.f5 + } +} + +fun algo117() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] load f1 f2, skip (bits32), load f4 f5, lazy match ab"); + o.f1 + o.f2; + match (o.ab) { + A => return o.f4, + B => throw o.f5 + } +} + +fun algo118() { + val o = lazy WithEitherABEnd.fromSlice(""); + if (10>3) { + __expect_lazy("[o] load f1 f2, skip (bits96), lazy match ab"); + match (o.ab) { + A => { return o.f1 } + B => { throw o.f2 } + } + } + throw 123; +} + +fun algo119() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] load f1 f2, skip (bits96), lazy match ab"); + match (o.ab) { + A => { + return o.ab.a2 + o.f2; + } + B => { + assert(o.ab.b1) throw o.f1; + } + } + o.f1; + return o.f2 as int; +} + +fun algo120() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] load f1, skip (bits128), lazy match ab"); + match (o.ab) { + A => return o.ab.a2 * 1, + B => return o.ab.b1 << 0, + } + o.f1; +} + +fun algo121() { + val o1 = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o1] load f1 f2 f3 f4 f5, lazy match ab"); + match (o1.ab) { + A => { (o1.f1, o1.f2, o1.f3, o1.f4, o1.f5) } + B => {} + } + val o2 = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o2] load f1 f2 f3 f4 f5 ab"); + match (o2.ab) { + A => { (o2) } + B => {} + } +} + + +type AOrWithEitherABEnd = A | WithEitherABEnd; + +fun algo122() { + val o = lazy AOrWithEitherABEnd.fromSlice(""); + match (o) { + A => { + __expect_lazy("[o] load a1"); + o.a1; + } + WithEitherABEnd => { + __expect_lazy("[o] skip (bits160), load ab"); // not lazy match, nesting not supported + match (o.ab) { + A => { o.ab.a1 } + B => {} + } + } + } +} + +fun algo123() { + val o = lazy AOrB.fromSlice(""); + return match (o) { + A => { + __expect_lazy("[o] load a1"); + throw o.a1; + } + B => { + __expect_lazy("[o] load b1 b2"); + assert(10>o.b2) throw 100; + return o.b1; + } + } +} + +fun algo124() { + val o = lazy AOrB.fromSlice(""); + return match (o) { + A => { + assert(10>3, 100); + assert(10>3, 100); + __expect_lazy("[o] load a1 a2 a3"); + throw (100, o.getA1()); + } + B => { + assert(10>3, 100); + __expect_lazy("[o] load b1 b2 b3"); + return o is B; + } + } +} + +fun algo127() { + val w = lazy WithMaybeTensor.fromSlice(""); + __expect_lazy("[w] skip (bits16), load t"); + match (w.t) { + null => {} + (int8, int8) => {} + } +} + +fun algo128() { + val o = lazy AOrB.fromSlice(""); + __expect_lazy("[o] lazy match"); + match (o) { + A => { + match (o) { A => {} } + } + B => {} + } +} + +fun algo129() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits128), load f5, lazy match ab"); + match (o.ab) { + A => {} + B => { + match (o.ab) { B => { o.f5 } } + } + } +} + +fun algo130() { + var o = lazy AOrB.fromSlice(""); + try { + return 1; + } catch (excno) { + __expect_lazy("[o] lazy match"); + match (o) { + A => { + __expect_lazy("[o] load a1"); + return o.a1 + excno; + } + B => { + __expect_lazy("[o] load b1 b2 b3"); + if (o.b1 > 0) { o = B { b1: 1, b2: 2, b3: 3 } } + return o.b2 as int; + } + } + } +} + +fun algo131() { + val o = lazy AOrB.fromSlice(""); + match (o) { + A => { + try { + __expect_lazy("[o] skip (bits15), load a3"); + return o.a3 as int; + } catch (excno) { return excno } + } + B => { + if (10 > 3) { return -1; } + else { + if (10 > 3) { return -1; } + else { + __expect_lazy("[o] load b1"); + return o.b1 * 0; + } + } + } + } +} + +fun algo132(): int { + val o = lazy AOrB.fromSlice(""); + match (o) { + B => { + __expect_lazy("[o] load b1 b2"); + try { + return o.b1; + } catch { + return o.b2; + } + } + A => { + __expect_lazy("[o] skip (bits7), load a2 a3"); + try { + return o.a3; + } catch { + throw 123; + } + return o.a2; + } + } +} + +fun algo133(): void { + val o = lazy WithEitherABEnd.fromSlice(""); + try {} + catch { + try { + __expect_lazy("[o] load f1, skip (bits128), lazy match ab"); + match (o.ab) { + A => { } + B => { o.f1 } + } + } catch { throw 123 } + } +} + +fun algo134(): void { + val o = lazy WithEitherABEnd.fromSlice(""); + try { try { + __expect_lazy("[o] load f1 f2, skip (bits96), lazy match ab"); + try { + match (o.ab) { + A => { } + B => { } + } + } catch { throw o.f1 + o.f2 } + } catch {} } catch {} +} + +fun algo135() { + val o = lazy AOrB.fromSlice(""); + match (o) { + A => { + __expect_lazy("[o] load a1 a2 a3"); + o.a1; + o.forceLoadLazyObject().assertEnd() + } + B => { + __expect_lazy("[o] load b1 b2"); + if (o.b1) { + o.b2; + } + } + } +} + +fun algo136() { + while (true) { + val obj = lazy AOrB.fromSlice(""); + __expect_lazy("[obj] lazy match"); + match (obj) { + A => { __expect_lazy("[obj] load a1"); obj.a1 } + B => { __expect_lazy("[obj] skip (bits7), load b2"); obj.b2 } + } + } +} + + +fun algo150() { + val o = lazy WithEitherABEnd.fromSlice(""); + var found = false; + __expect_lazy("[o] skip (bits160), load ab"); + while (!found) { + match (o.ab) { + A => {} + B => { found = true } + } + } +} + +fun algo151() { + val o = lazy WithEitherABEnd.fromSlice(""); + var found = false; + if (!found) { + __expect_lazy("[o] skip (bits160), lazy match ab"); + match (o.ab) { + A => {} + B => { found = true } + } + } +} + +fun algo152() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits160), load ab"); + 1 + match (o.ab) { + A => 1, + B => -1, + } +} + +fun algo153() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits160), load ab"); + 1 && match (o.ab) { + A => 1, + B => -1, + } +} + +fun algo224(): void { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits32), load f2, skip (bits32), load f4 f5 ab"); + match (o.ab) { + A => {} + B => { o.f2; } + } + match (o.ab) { + A => {} + B => { if (rand()) { o.f4; } return; } + } + if (o.ab is A) { + if (rand()) { + o.f5; + } + } +} + +fun algo225() { + __expect_lazy("[c] lazy match"); + match (val c = lazy MsgFullCounter.fromSlice("")) { + CounterIncrement => { + var d = lazy WalletStorage.fromSlice(""); + __expect_lazy("[d] load (bits1) seqno, save immutable (tail)"); + d.seqno += 1; + d.toCell(); + __expect_lazy("[c] load byValue"); + 1 + c.byValue; + } + CounterDecrement => { + val c = lazy MsgFullCounter.fromSlice(""); + __expect_lazy("[c] lazy match"); + match (c) { + CounterIncrement => {} + CounterDecrement => {} + CounterDecrementBy1 => {} + CounterReset => { + assert(10>3, 101); + __expect_lazy("[c] load initialValue"); + while (10>3) { + if (10>3) { c.initialValue } + } + } + } + } + CounterDecrementBy1 => { + } + CounterReset => { + if (10 > 3) { + __expect_lazy("[c] load initialValue"); + c.initialValue + } + } + else => throw 123, // `else` is allowed in a lazy `match` + } +} + +fun algo226() { + var st = lazy loadWalletStorage(); + __expect_lazy("[st] skip (bits1), load seqno"); + return st.seqno; +} + +fun algo250() { + var f = lazy Fields9.fromSlice(""); + __expect_lazy("[f] skip (bits8), load f2"); + match (f.f2) { + uint8 => {} + } +} + +fun algo251() { + var f = lazy Fields9.fromSlice(""); + __expect_lazy("[f] skip (bits8), load f2"); + match (f.f2) { + 0 => 1, + 1 => 0, + else => throw 0, + } +} + +fun algo252() { + var p = lazy Point.fromSlice(""); + __expect_lazy("[p] skip (bits8), load y"); + match (p.y) { + int8 => {} + } +} + +fun algo253() { + var p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x y"); + match (p.y) { + 0 => p.x, + 1 => 0, + else => throw 0, + } +} + +fun algo254() { + var p = lazy Point.fromSlice(""); + __expect_lazy("[p] lazy match"); + match (p) { + Point => { + return 1; + } + } +} + +fun algo255() { + var p = lazy Point.fromSlice(""); + __expect_lazy("[p] lazy match"); + match (p) { + Point => { + __expect_lazy("[p] load x"); + return p.x; + } + else => throw 123 + } +} + +fun algo256() { + var p = lazy Point.fromSlice(""); + match (p) { + Point => { + __expect_lazy("[p] load x"); + match (p.x) { int8 => {} } + } + } +} + +fun algo257(): int { + val k = lazy HasIncrementEnd.fromSlice(""); + __expect_lazy("[k] skip (bits8), load inc"); + match (k.inc) { + CounterIncrement => { + __expect_lazy(""); + return k.inc.byValue; + } + } +} + + +fun algo300() { + val p = lazy AOrB.fromSlice(stringHexToSlice("8F020304")); + __expect_lazy("[p] lazy match"); + match (p) { + A => { + __expect_lazy("[p] skip (bits7), load a2"); + debug.print(p.a2); + } + B => { + __expect_lazy("[p] load b1 b2"); + debug.print(p.b1 + p.b2); + } + } +} + +@method_id(301) +fun algo301() { + __expect_lazy("[p] lazy match"); + match (val p = lazy AOrB.fromSlice(stringHexToSlice("8F020304"))) { + A => { + __expect_lazy("[p] skip (bits7), load a2"); + return p.a2 as int; + } + B => { + __expect_lazy("[p] load b1 b2"); + return p.b1 + p.b2; + } + } +} + +fun algo302() { + var c = Point{x:1,y:2}.toCell(); + var o = lazy Point.fromCell(c); + __expect_lazy("[o] load x y"); + o.x; + o.y; + return o; +} + +@method_id(303) +fun algo303() { + var c = Point{x:1,y:2}.toCell(); + var o = lazy c.load(); + __expect_lazy("[o] load x y"); + return o; +} + +@method_id(304) +fun algo304() { + var o = lazy Point{x:1,y:2}.toCell().load(); + __expect_lazy("[o] load x y"); + return o; +} + +@method_id(305) +fun algo305() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + assert(10>3, 100); + __expect_lazy("[p] load x y"); + { + assert(p.x == 1, 100); + assert(p.y == 2, 100); + } + return p; +} + +fun algo306() { + var st = lazy WalletStorage.load(); + __expect_lazy("[st] load (bits1) seqno, save immutable (tail)"); + st.seqno += 1; + return st.toCell(); +} + +fun algo307() { + var c = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[c] load (bits8) p, save immutable (tail)"); + c.p.assignX(10); + c.toCell(); +} + +fun algo308() { + var c = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[c] load (bits8) p, save immutable (tail)"); + c.p.x = 10; + c.toCell(); +} + +fun algo309() { + var c = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[c] load f1 p f3"); // object used for writing and toCell, load all for safety + c = { f1: 10, p: {x:0,y:0}, f3: c.f3 }; + c.toCell(); +} + +fun algo310() { + var c = lazy ComplexWithPoint.fromSlice(""); + if (true) { + __expect_lazy("[c] load (bits8) p, save immutable (tail)"); + debug.print(c.p.y += 2); + c.toCell(); + } +} + +fun algo311() { + var st = lazy WalletStorage.load(); + __expect_lazy("[st] load isSignatureAllowed (bits320) extensions"); + st.isSignatureAllowed = true; + st.extensions = null; + return st.toCell(); +} + +fun algo312() { + var st = lazy NotFixedWidthStorage.fromSlice(""); + __expect_lazy("[st] load (bits8) (int8?) (bits9) (cell) (int8 | bool) (address) f8, save immutable (tail)"); + if (rand()) { + st.f8 += 1; + } + st.toCell(); +} + +fun algo313() { + var st = lazy NotFixedWidthStorage.fromSlice(""); + if (rand()) { + __expect_lazy("[st] load (bits8) (int8?) (bits9) f5, save immutable (tail), skip (int8 | bool) (address) (bits8), load f9"); + st.f5 = createEmptyCell(); + st.f9; + st.toCell(); + } +} + + +fun main() { +} + +/** +@testcase | 140 | | 2 +@testcase | 141 | -1 | 1 +@testcase | 301 | | 17 +@testcase | 303 | | 1 2 +@testcase | 304 | | 1 2 +@testcase | 305 | | 1 2 + +@fif_codegen +""" + algo40 PROC:<{ + x{0102} PUSHSLICE + 8 LDI + NIP + 8 LDI + DROP + }> +""" + */ diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk new file mode 100644 index 0000000000..36d7491d48 --- /dev/null +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -0,0 +1,1981 @@ +import "@stdlib/tvm-dicts.tolk" + +struct A { a1: int7; a2: uint8; a3: uint8; } +struct B { b1: int7; b2: uint8; b3: uint8; } +type AOrB = A | B; + +@noinline +fun A.getA1(self) { + return self.a1; +} + +struct Fields9 { + f1: uint8; f2: uint8; f3: uint8; + f4: uint8; f5: uint8; f6: uint8; + f7: uint8; f8: uint8; f9: uint8; +} + +@noinline +fun Fields9.method(self) {} + +fun Fields9.getF2(self) { return self.f2 } +fun Fields9.getF5(self) { return self.f5 } + +struct Small { s1: int7; } +struct Big { b1: int7; b2: int8; b3: int8; } +type SmallOrBig = Small | Big; + +struct StructF1F2AB { + f1: int8; + f2: int8; + ab: A | B; +} + +struct StructF1SBF3 { + f1: int8; + sb: Small | Big; + f3: int8; +} + +struct WithEitherABMiddle { + f1: int32; + f2: int32; + ab: A | B; + f4: int32; + f5: int32; + f6: int32; +} + +struct WithEitherABEnd { + f1: int8; + f2: int8; + f3: int8; + f4: int8; + f5: int8; + ab: A | B; +} + +struct Point { + x: int8; + y: int8; +} + +fun Point.getX(self) { + return self.x +} + +fun Point.makeCell(self) { + return self.toCell() +} + +fun Point.makeCellWrapped(self) { + val cc = self.makeCell(); + self.x; + return cc +} + +fun Point.incXAndMakeCell(mutate self) { + __expect_inline(true); + self.x += 1; + return self.toCell() +} + +struct Point3d { + x: int32; + y: int32; + z: int32; +} + +type PointOrPoint3d = Point | Point3d; + +struct Point78ForMaybe { + x: int7; + y: int8; +} + +type OptionalPoint78ForMaybe = Point78ForMaybe?; + +struct ComplexWithPoint { + f1: int8; + p: Point; + f3: int8; +} + +struct ComplexWithMaybePoint { + f1: int8; + p: Point78ForMaybe?; + f3: int8; +} + +struct Counter7Increment { byValue: int7 } +struct Counter7Decrement { byValue: int7 } +type MsgEitherCounter = Counter7Increment | Counter7Decrement; + +struct FieldsAndEitherCounter { + f1: int8; + f2: int8; + c: MsgEitherCounter; +} + +struct WalletStorage { + isSignatureAllowed: bool; + seqno: uint32; + walletId: uint32; + publicKey: uint256; + extensions: dict; +} + +fun WalletStorage.save(self) { + contract.setData(self.toCell()) +} + +@overflow1023_policy("suppress") +struct NotFixedWidthStorage { + f1: int8; + f2: int8?; + f3: int8; + f4: bool; + f5: cell; + f6: int8 | bool; + f7: address; + f8: int8; + f9: bits200; + f10: (bits400, ()); + f11: coins; +} + +struct Inner1 { i1: int8; i2: int8; } +struct Inner2 { i1: int8; i2: int8; } +type Inner2Alias = Inner2; +struct Outer { + inner1: Inner1; + inner2: Inner2Alias; + wa: StructF1F2AB; +} + +struct WithMaybe { + f1: int8; + big: Big?; + f3: int2; +} + +struct WithRefs { + r1: cell?; + f2: int8; + r3: Cell; + f4: int8; +} + +struct (0x01) CounterIncrement { byValue: int8 } +struct (0x02) CounterDecrement { byValue: int8 } +struct (0x03) CounterDecrementBy1 { } +struct (0x04) CounterReset { initialValue: int32 } +type MsgFullCounter = CounterIncrement | CounterDecrement | CounterDecrementBy1 | CounterReset; + +struct(0x12345678) Counter32Increment { byValue: int32; } +struct(0x23456789) Counter32DecrementBy1 {} +struct(0x34567890) Counter32Reset { initial: uint32; f2: int8; f3: int8; f4: int8; f5: int8; } +type MyMsg32 = Counter32Increment | Counter32DecrementBy1 | Counter32Reset; + +struct AutoP1 { f1: int8; f2: int8; } +struct AutoP2 { f1: int8; f2: int8; f3: int8; } +struct AutoP3; +type AutoPrefixedBin2 = AutoP1 | AutoP2 | AutoP3; // 00 / 01 / 10 + +struct(0b0111) ShortP1 { f1: int8; } +struct(0b0100) ShortP2 { f1: int8; f2: int8; } +type ShortPrefixed = ShortP1 | ShortP2; + +struct(0b01) NotEqPrefixedS1 { f: uint6; } +struct(0b0001) NotEqPrefixedS2 { f: uint4; } +struct(0b101) NotEqPrefixedS3 { f: int5; } +type NotEqPrefixed = NotEqPrefixedS1 | NotEqPrefixedS2 | NotEqPrefixedS3; + +struct WithBitsField { + bin16: bits16; + bin8: bits8; +} + +struct WithOneRestField { + i32: int32; + rest: RemainingBitsAndRefs; +} + +struct VestingStorage { + vestingParameters: Cell<(uint64, int32)>; + seqno: uint32; + subwalletId: uint32; + publicKey: uint256; + whitelist: dict; +} + +struct OneIntOneRef { + z: cell? = null; + o: cell; + e: () = (); + i: int32 = 0; + r: cell; +} + + +@noinline +fun contract.getFakeData(seqno: int): Cell { + return WalletStorage { + isSignatureAllowed: true, + seqno: seqno, + walletId: 999, + publicKey: 1 << 100, + extensions: null, + }.toCell() +} + +fun getPointCell(x: int, y: int): Cell { + val p: Point = {x,y}; + return p.toCell(); +} + +fun loadWalletStorage() { + return WalletStorage.fromSlice(stringHexToSlice("8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_")); +} + +fun WalletStorage.load() { + return WalletStorage.fromCell(contract.getFakeData(777)); +} + +@method_id(101) +fun demo101() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x"); + return p.x; +} + +@method_id(102) +fun demo102(takeX: bool) { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + var result: int; + __expect_lazy("[p] load x y"); + if (takeX) { + result = p.x; + } else { + result = p.y; + } + return result; +} + +@method_id(103) +fun demo103() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] skip (bits8), load y"); + return p.y * p.y * p.y; +} + +@method_id(104) +fun demo104() { + var p2 = lazy Point.fromSlice(stringHexToSlice("0A14")); + __expect_lazy("[p2] load x y"); + assert(p2.x < 100) throw 100; + assert(p2.y < 100) throw 100; + + var c = lazy ComplexWithPoint.fromSlice(stringHexToSlice("01020304")); + __expect_lazy("[c] load f1, skip (bits16), load f3"); + return (c.f1, 0 + c.f3); +} + +@method_id(105) +fun demo105(s: slice) { + var c = lazy ComplexWithMaybePoint.fromSlice(s); + __expect_lazy("[c] load f1, skip (Point78ForMaybe?), load f3"); + return (c.f1, c.f3); +} + +@method_id(106) +fun demo106(s: slice) { + var c = lazy ComplexWithMaybePoint.fromSlice(s); + var x: int? = null; + __expect_lazy("[c] skip (bits8), load p"); + if (c.p != null) { + x = c.p.x; + } + return (c.p == null, x); +} + +@method_id(107) +fun demo107() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + return p; +} + +@method_id(108) +fun demo108() { + var o = lazy Fields9.fromSlice(stringHexToSlice("010203040506070809")); + __expect_lazy("[o] load f1 f2 f3 f4 f5 f6 f7 f8 f9"); + { + assert(o.f3 == 3) throw 100; + assert(o.f9 == 9) throw 100; + } + + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + p.y = 15; + return (p, o); +} + +@method_id(109) +fun demo109() { + var st = lazy WalletStorage.fromSlice(stringHexToSlice("8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_")); + __expect_lazy("[st] skip (bits65), load publicKey"); + return st.publicKey; +} + +@method_id(110) +fun demo110(s: slice) { + var msg = lazy MsgEitherCounter.fromSlice(s); + var t = createEmptyTuple(); + __expect_lazy("[msg] lazy match"); + match (msg) { + Counter7Increment => { + __expect_lazy("[msg] load byValue"); + t.push(+msg.byValue) + } + Counter7Decrement => { + __expect_lazy("[msg] load byValue"); + t.push(-msg.byValue) + } + } + return t; +} + +@method_id(111) +fun demo111(s: slice, x: int) { + var msg = lazy MsgEitherCounter.fromSlice(s); + var t = createEmptyTuple(); + match (msg) { + Counter7Increment => { + assert(x >= 0, 123); + __expect_lazy("[msg] load byValue"); + t.push(+msg.byValue) + } + Counter7Decrement => { + assert(x >= 1, 456); + assert(t.size() == 0, 789); + __expect_lazy("[msg] load byValue"); + t.push(-msg.byValue) + } + } + return t; +} + +@method_id(112) +fun demo112(s: slice) { + var msg = lazy MsgFullCounter.fromSlice(s); + var curX = 0; + __expect_lazy("[msg] lazy match"); + match (msg) { + CounterIncrement => curX += msg.byValue, + CounterDecrement => curX -= msg.byValue, + CounterDecrementBy1 => curX = -1, + CounterReset => curX = msg.initialValue, + } + return curX; +} + +@method_id(113) +fun demo113(s: slice) { + var msg = lazy MsgFullCounter.fromSlice(s); + var curX = 0; + var newX: int = match (msg) { + CounterIncrement => curX + msg.byValue, + CounterDecrement => curX - msg.byValue, + CounterDecrementBy1 => -1, + CounterReset => msg.initialValue, + }; + return newX; +} + +@method_id(114) +fun demo114(s: slice) { + var msg = lazy MsgFullCounter.fromSlice(s); + var curX = 0; + var newX: int | [int] = match (msg) { + CounterDecrementBy1 => -1, + CounterReset => [msg.initialValue as int], + CounterDecrement => [curX - msg.byValue], + CounterIncrement => curX + msg.byValue, + }; + return newX; +} + +@method_id(115) +fun demo115(s: slice): int8 | bool { + var msg = lazy MsgEitherCounter.fromSlice(s); + return match (msg) { + Counter7Decrement => msg.byValue < 5, + Counter7Increment => msg.byValue as int8, + } +} + +@method_id(118) +fun demo118(s: slice) { + match (var msg = lazy MsgEitherCounter.fromSlice(s)) { + Counter7Increment => return +msg.byValue, + Counter7Decrement => { + assert(s.remainingRefsCount() == 0, 456); + __expect_lazy("[msg] load byValue"); + return -msg.byValue + } + } +} + +@method_id(119) +fun demo119(s: slice) { + var o = lazy SmallOrBig.fromSlice(s); + var s1 = -1; + var b1 = -1; + match (o) { + Small => { + __expect_lazy("[o] load s1"); + s1 = o.s1; + } + Big => { + __expect_lazy("[o] skip (bits7), load b2"); + b1 = o.b2; + } + } + return (s1, b1); +} + +@method_id(120) +fun demo120(s: slice) { + var msg = lazy FieldsAndEitherCounter.fromSlice(s); + var t = createEmptyTuple(); + __expect_lazy("[msg] load f1, skip (bits8), lazy match c"); + match (msg.c) { + Counter7Increment => { t.push(+msg.c.byValue) } + Counter7Decrement => { t.push(-msg.c.byValue) } + } + t.push(msg.f1); + return t; +} + +@method_id(121) +fun demo121() { + var c = FieldsAndEitherCounter{f1:1, f2:2, c:Counter7Decrement{byValue:20}}; + var o = lazy FieldsAndEitherCounter.fromCell(c.toCell()); + __expect_lazy("[o] load f1 f2 c"); // load c, not lazy match + return (o.c is Counter7Increment) ? o.f1 + o.c.byValue : o.f2 + o.c.byValue; +} + +@method_id(122) +fun demo122(s: slice): int | slice { + var o = lazy FieldsAndEitherCounter.fromSlice(s); + __expect_lazy("[o] load f1, skip (bits8), lazy match c"); + return match(o.c) { + Counter7Decrement => -o.c.byValue, + Counter7Increment => o.f1 + o.c.byValue, + } +} + +@method_id(123) +fun demo123(s: slice) { + var o = lazy FieldsAndEitherCounter.fromSlice(s); + __expect_lazy("[o] load f1 f2 c"); // not lazy match, used in a complex expression + return o.f1 + match (o.c) { + Counter7Increment => o.f1 as int, + Counter7Decrement => o.f2 * o.f2, + } +} + +@method_id(124) +fun demo124(s: slice) { + var msg = lazy FieldsAndEitherCounter.fromSlice(s); + var t = createEmptyTuple(); + __expect_lazy("[msg] skip (bits8), load f2, lazy match c"); + match (msg.c) { + Counter7Increment => { + var mm = msg.c; + t.push(+mm.byValue) + } + Counter7Decrement => { + try {} + catch {} + var mm = msg.c; + t.push(-mm.byValue); + t.push(msg.f2); + } + } + return t; +} + +@method_id(125) +fun demo125() { + var o = lazy StructF1F2AB.fromSlice(stringHexToSlice("7F200F0203")); + var (a1, a3, b2) = (-1,-1,-1); + __expect_lazy("[o] load f1 f2 ab"); + match (o.ab) { + A => { + a1 = o.ab.a1; + a3 = o.ab.a3; + } + B => { + b2 = o.ab.b2; + } + } + return (o, a1, a3, b2); +} + +@method_id(126) +fun demo126(s: slice) { + var o = lazy StructF1SBF3.fromSlice(s); + var (s1, b2) = (-1, -1); + __expect_lazy("[o] load f1 sb f3"); + match (o.sb) { + Small => { + s1 = o.sb.s1; + } + Big => { + b2 = o.sb.b2; + } + } + return (o, s1, b2); +} + +@method_id(127) +fun demo127() { + val o = lazy Outer.fromSlice(stringHexToSlice("0102030408FF7FFFFF")); + __expect_lazy("[o] load inner1, skip (bits16), load wa"); + var inner1_i1 = o.inner1.i1; + var inner1_i2 = o.inner1.i2; + + var wa_is_A: bool; + var wa_last: int; + match (o.wa.ab) { + A => { wa_is_A = true; __expect_lazy(""); wa_last = o.wa.ab.a3; } + B => { wa_is_A = false; __expect_lazy(""); wa_last = o.wa.ab.b3; } + } + return (inner1_i1, inner1_i2, wa_is_A, wa_last); +} + +@method_id(128) +fun demo128(s: slice) { + var o = lazy WithMaybe.fromSlice(s); + __expect_lazy("[o] load f1 big"); + assert (o.f1 > 0) throw 123; + var isnull = true; + var b3 = -1; + match (o.big) { + null => { + debug.printString("big is null"); + } + Big => { + isnull = false; + assert(o.big.b1 != 0, 123); + b3 = o.big.b3; + } + } + return (isnull, b3); +} + +@method_id(129) +fun demo129() { + var o = lazy AOrB.fromSlice(stringHexToSlice("0F0203")); + var (a1, wasA) = (-1, false); + __expect_lazy("[o] lazy match"); + match (o) { + A => { + __expect_lazy("[o] load a1 a2 a3"); + a1 = o.getA1(); + wasA = true; + } + B => { + __expect_lazy("[o] skip (bits7), load b2"); + a1 = o.b2; + } + } + return (wasA, a1); +} + +@method_id(130) +fun demo130() { + var st = lazy loadWalletStorage(); + __expect_lazy("[st] skip (bits33), load walletId"); + return st.walletId; +} + +@method_id(131) +fun demo131() { + var st = lazy WalletStorage.load(); + __expect_lazy("[st] skip (bits1), load seqno, skip (bits32), load publicKey"); + return (st.seqno, st.publicKey >> 100); +} + +@method_id(132) +fun demo132(incSeqno: bool) { + var st = lazy WalletStorage.fromCell(contract.getFakeData(888)); + __expect_lazy("[st] skip (bits1), load seqno, skip (bits288), load extensions"); + if (incSeqno) { + return (st.seqno + 1, st.extensions == null); + } else { + return (st.seqno as int, st.extensions == null); + } +} + +@method_id(133) +fun demo133() { + var extensions = createEmptyDict(); + extensions.iDictSet(32, 8, stringHexToSlice("12")); + var wCell = WalletStorage { + isSignatureAllowed: true, + seqno: 0, + walletId: 123, + publicKey: 1 << 88, + extensions, + }.toCell(); + if (10 > 3) { + var st = lazy wCell.load(); + __expect_lazy("[st] skip (bits321), load extensions"); + while (!false) { + assert(st.extensions != null, 123); + return st.extensions.iDictGet(32, 8).0!.loadUint(8); + } + } else { + throw 0; + } +} + +@method_id(134) +fun demo134() { + var cc = lazy CounterIncrement.fromSlice(stringHexToSlice("010f"), {throwIfOpcodeDoesNotMatch: 134}); + return cc.byValue; +} + +@method_id(135) +fun demo135(skip: bool) { + var cc = lazy WithBitsField.fromSlice(stringHexToSlice("123456")); + __expect_lazy("[cc] load bin16, save immutable (tail)"); + cc.bin16 = stringHexToSlice("00000000") as bits16; + try { + return cc.toCell({ + skipBitsNFieldsValidation: skip + }).beginParse().remainingBitsCount(); + } catch (excno) { + return excno; + } +} + +@method_id(136) +fun demo136(skip: bool): int { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + if (skip) { + return -1; + } else { + __expect_lazy("[p] skip (bits8), load y"); + return p.y; + } +} + +@method_id(137) +fun demo137(s: slice): int { + val p = lazy Point.fromSlice(s); + try { + __expect_lazy("[p] load x"); + return p.x; // will be ok even if s doesn't contain y + } catch (excno) { + return 100 + excno; + } +} + +@method_id(138) +fun demo138(s: slice): int { + val p = lazy Point.fromSlice(s); + try { + __expect_lazy("[p] skip (bits8), load y"); + return p.y; + } catch (excno) { + return 100 + excno; + } +} + +@method_id(139) +fun demo139(s: slice, skip: bool): int { + var msg = lazy MsgFullCounter.fromSlice(s); + match (msg) { + CounterIncrement => { + try { + if (s.isEnd()) { return 100; } + __expect_lazy("[msg] load byValue"); + return msg.byValue; + } catch (excno) { return excno } + } + CounterReset => throw 123, + CounterDecrementBy1 => throw 456, + CounterDecrement => { + if (!skip) { + __expect_lazy("[msg] load byValue"); + return msg.byValue; + } + return 100; + } + } +} + +@method_id(140) +fun demo140(getF4: bool) { + val o = lazy WithEitherABEnd.fromSlice(stringHexToSlice("010203040580FF00")); + var result: int; + __expect_lazy("[o] skip (bits24), load f4, skip (bits8), lazy match ab"); + if (getF4) { + result = o.f4; + } else { + match (o.ab) { A => throw 123, B => { + result = o.ab.b2; + } } + if (o.f4 < 0) { result *= 100; } + } + return result; +} + +@method_id(141) +fun demo141(eval: bool): int { + val o = lazy WithEitherABEnd.fromSlice(stringHexToSlice("010203040500FF00")); + if (eval) { + try { + __expect_lazy("[o] load f1 f2, skip (bits24), lazy match ab"); + match (o.ab) { + A => { return o.f1 } + B => { throw o.f2 } + } + } catch(excno) { return excno * 100 } + } + throw 123; +} + +@method_id(142) +fun demo142(s: slice, ignore: bool) { + val o = lazy AOrB.fromSlice(s); + match (o) { + A => { + try { + __expect_lazy("[o] skip (bits15), load a3"); + return o.a3 as int; + } catch (excno) { return excno } + } + B => { + if (ignore) { return -1; } + else { + if (ignore) { return -1; } + else { + __expect_lazy("[o] load b1"); + return o.b1 * 1; + } + } + } + } +} + +@method_id(143) +fun demo143(): int { + val o = lazy WithEitherABEnd.fromSlice(stringHexToSlice("0809000000")); // ab missing + try { try { + __expect_lazy("[o] load f1 f2, skip (bits24), lazy match ab"); + try { + match (o.ab) { + A => { return 1 } + B => { return 2 } + } + } catch { throw o.f1 + o.f2 } + } catch(excno) { return excno } } catch { return -1 } +} + +@method_id(144) +fun demo144(dummy: bool) { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + var result = -1; + if (!dummy) { + __expect_lazy("[p] load x"); + result = p.getX(); + } + return result; +} + +@method_id(145) +fun demo145(getF5: bool): int { + var p = lazy Fields9.fromSlice(stringHexToSlice("010203040506070809")); + try { + __expect_lazy("[p] skip (bits8), load f2, skip (bits16), load f5"); + if (getF5) { + return p.getF5(); + } + return 1 * p.getF2(); + } catch { throw 123 } +} + +@method_id(146) +fun demo146() { + val p = lazy WithRefs.fromSlice(stringHexToSlice("0000000")); // no mandatory ref exists, but since we don't access it, it doesn't fail + __expect_lazy("[p] load r1"); + return p.r1 == null; +} + +@method_id(147) +fun demo147() { + val p = lazy WithRefs.fromSlice(stringHexToSlice("0000000")); + if (false) {} else { + try { + __expect_lazy("[p] skip (cell?) (bits8), load f4"); + return (true, p.f4 as int); // even though `r3` (Cell) doesn't exist, it's ignored to reach `f4` + } catch (excno) { + return (false, excno); + }} +} + +@method_id(148) +fun demo148() { + var o = lazy AOrB.fromSlice(stringHexToSlice("0F0203FF")); + match (o) { + A => { + var bits: int; + __expect_lazy("[o] load a1 a2 a3"); + bits = o.forceLoadLazyObject().remainingBitsCount(); + return bits; + } + B => throw 123 + } +} + +@method_id(149) +fun demo149() { + var o = lazy AOrB.fromSlice(stringHexToSlice("8F02")); // corrupted, too small + match (o) { + A => throw 123, + B => { + try { + __expect_lazy("[o] load b1 b2 b3"); + o.forceLoadLazyObject().assertEnd(); + return o.b1 << 8 + } catch (excno) { + return excno + } + } + } +} + +@method_id(150) +fun demo150() { + val o = lazy WithOneRestField.fromSlice(stringHexToSlice("00000001FFFF")); + return (o.i32, o.rest.remainingBitsCount()); +} + +@method_id(151) +fun demo151() { + val c = VestingStorage { + whitelist: createEmptyDict(), + subwalletId: 123, + publicKey: 456, + vestingParameters: (1 as uint64, 2 as int32).toCell(), + seqno: 10, + }.toCell(); + var st = lazy c.load(); + __expect_lazy("[st] skip (Cell<(uint64, int32)>) (bits320), load whitelist"); + return st.whitelist == null; // need to skip prev bits, because dict requires 1 bit +} + +@method_id(152) +fun demo152() { + val c = OneIntOneRef { o: createEmptyCell(), r: (2 as int32).toCell() }.toCell(); + val st = lazy c.load(); + __expect_lazy("[st] skip (cell?) (cell), load r"); + return st.r.beginParse().remainingBitsCount(); +} + +@method_id(153) +fun demo153() { + val c = OneIntOneRef { o: createEmptyCell(), r: (2 as int32).toCell() }.toCell(); + val st = lazy c.load(); + __expect_lazy("[st] skip (cell?), load i"); + return st.i; +} + +global gTup: tuple; + +@noinline +fun pushToGlobalTup(v: T) { gTup.push(v.__toTuple()); } + +@method_id(154) +fun demo154() { + gTup = createEmptyTuple(); + val p = lazy Point.fromSlice(stringHexToSlice("0102")); + pushToGlobalTup(p); + return gTup; +} + +fun inlinedFunctionWithLazy(s: slice) { + __expect_inline(true); + val p = lazy Point.fromSlice(s); + assert(p.y >= 0, 555); +} + +@method_id(155) +fun demo155() { + inlinedFunctionWithLazy(stringHexToSlice("0102")); + inlinedFunctionWithLazy(stringHexToSlice("0304")); + try { + inlinedFunctionWithLazy(stringHexToSlice("03FF")); + } catch (excno) { return excno; } + return -1; +} + +@method_id(156) +fun demo156(s: slice): int { + val d = lazy CounterIncrement.fromSlice(s); + __expect_lazy("[d] lazy match"); + match (d) { + CounterIncrement => { + __expect_lazy("[d] load byValue"); + return d.byValue; + } + else => { + return -100; + } + } +} + + +@method_id(200) +fun demo200() { + var st = lazy WalletStorage.fromCell(contract.getFakeData(888)); + var b = beginCell(); + if (10>3) { + __expect_lazy("[st] load isSignatureAllowed seqno walletId publicKey extensions"); + assert(st.isSignatureAllowed, 123); + b.storeAny(st); + var st2 = lazy WalletStorage.fromCell(b.toCell()); + __expect_lazy("[st2] load isSignatureAllowed"); + if (st2.isSignatureAllowed) { + return 1; + } + } + throw 123; +} + +@method_id(201) +fun demo201() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + var c1 = p.toCell(); + var x = p.x; + var c2 = p.toCell(); + var y = p.y; + var c3 = p.toCell(); + assert(c1.beginParse().bitsEqual(c2.beginParse()), 100); + assert(c1.beginParse().bitsEqual(c3.beginParse()), 100); + return (x, y, p is Point); +} + +@method_id(202) +fun demo202() { + var o = lazy Fields9.fromSlice(stringHexToSlice("010203040506070809")); + __expect_lazy("[o] load f1 f2, save immutable (tail), skip (bits24), load f6"); + if (o.f1 == 1) { + o.f2 = 127; + } + var f6 = o.f6; + var o2 = lazy o.toCell().load(); + __expect_lazy("[o2] skip (bits8), load f2"); + return (o.f2, o2.f2, f6); +} + +@method_id(203) +fun demo203() { + var o = lazy Fields9.fromSlice(stringHexToSlice("010203040506070809")); + if (o.f1 == 1) { + return o.toCell().load(); + } + throw 123; +} + +@method_id(204) +fun demo204() { + var o = lazy Fields9.fromSlice(stringHexToSlice("010203040506070809")); + __expect_lazy("[o] load f1 f2 f3 f4, save immutable (tail)"); + if (o.f1 != 1) { + return o.toCell().load(); + } else { + o.f1 = 10; + } + if (o.f2 != 2) { + return o.toCell().load(); + } else { + o.f2 = 20; + } + if (o.f3 != 3) { + return o.toCell().load(); + } else { + o.f4 = 40; + } + return o.toCell().load(); +} + +@method_id(205) +fun demo205() { + var o = lazy WithMaybe.fromSlice(stringHexToSlice("551_")); + __expect_lazy("[o] save immutable (tail), skip (bits8), load big"); + match (o.big) { null => {} Big => {} } + return o.toCell().beginParse().bitsEqual(stringHexToSlice("551_")); +} + +@method_id(206) +fun demo206() { + var o = lazy Fields9.fromSlice(stringHexToSlice("010203040506070809")); + o.method(); + return (o.toCell() is Cell, o.toCell().beginParse().remainingBitsCount()); +} + +@method_id(207) +fun demo207() { + var o = lazy WithRefs.fromCell(WithRefs { + r1: null, + f2: 123, + r3: getPointCell(10, 20), + f4: 55, + }.toCell()); + o.r1 = getPointCell(60, 70); + o.f2 = 90; + return o.toCell(); +} + +@method_id(208) +fun demo208(s: slice) { + var o = lazy AOrB.fromSlice(s); + __expect_lazy("[o] lazy match"); + match (o) { + A => { + __expect_lazy("[o] load a1 a2 a3"); + var bc = o.toCell().beginParse().remainingBitsCount(); + return (bc, (o as A|B).toCell().beginParse().remainingBitsCount()); + } + B => { + assert(10>3, 101); + __expect_lazy("[o] load b1 b2 b3"); // loaded because of toCell + return (o.b1 as int, o.toCell().hash() as int); + } + } +} + +@method_id(209) +fun demo209() { + var o = lazy StructF1F2AB.fromSlice(stringHexToSlice("7F200F0203")); + __expect_lazy("[o] skip (bits16), lazy match ab"); + match (o.ab) { + A => { + return o.ab.toCell().beginParse().remainingBitsCount(); + } + B => throw 0xFFFF + } +} + +@method_id(210) +fun demo210() { + var o = lazy StructF1F2AB.fromSlice(stringHexToSlice("7F200F0203")); + __expect_lazy("[o] save immutable (tail), skip (bits16), load ab"); // not lazy match because of nested o.toCell() + match (o.ab) { + A => { + return o.toCell().beginParse().remainingBitsCount(); + } + B => throw 0xFFFF + } +} + +@method_id(211) +fun demo211() { + var o = lazy Counter32Reset.fromSlice(stringHexToSlice("34567890000000FF02030405")); + var f3 = -1; + __expect_lazy("[o] save immutable (tail), load initial, skip (bits8), load f3"); + if (o.initial > 100) { + f3 = o.f3; + } + assert(o.toCell().beginParse().bitsEqual(stringHexToSlice("34567890000000FF02030405")), 100); + return o.toCell(); +} + +@method_id(212) +fun demo212() { + var o = lazy Counter32Reset.fromSlice(stringHexToSlice("34567890000000FF02030405")); + __expect_lazy("[o] load initial (bits8) f3, save immutable (tail), load f4"); + if (o.initial > 100 && o.initial < 10000) { + o.f3 = o.f4; + } + return o.toCell().beginParse().bitsEqual(stringHexToSlice("34567890000000FF02040405")); +} + +@method_id(213) +fun demo213(eval: bool) { + var c = lazy ComplexWithPoint.fromSlice(stringHexToSlice("01080903FFFF")); // more than needed + if (eval && 10>3) { + __expect_lazy("[c] load (bits8) p, save immutable (tail)"); + c.p.y += 2; + var s = c.toCell().beginParse(); + return (s.remainingBitsCount(), s.skipBits(8+8).preloadUint(8)); + } else { + throw 123; + } +} + +@method_id(214) +fun demo214() { + val p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] save immutable (tail), load x"); + var s = p.makeCellWrapped().beginParse(); + return (s.loadUint(16), s.remainingBitsCount()); +} + +@method_id(215) +fun demo215() { + val p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + var s = Point.makeCell(p).beginParse(); + return (s.loadUint(16), s.remainingBitsCount()); +} + +@method_id(216) +fun demo216() { + var st = lazy WalletStorage.fromSlice(stringHexToSlice("8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_")); + __expect_lazy("[st] load (bits1) seqno, save immutable (tail), load walletId"); + st.seqno += 1; + val w = st.walletId; + st.save(); + return (contract.getData().hash() & 0xFFFF, w); +} + +@method_id(217) +fun demo217() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); // because call mutating method + return p.incXAndMakeCell().beginParse().loadUint(16); +} + +@method_id(218) +fun demo218() { + var o = lazy WithRefs.fromCell(WithRefs { + r1: (1 as int32).toCell(), + f2: 2, + r3: Point{x: 10, y: 20}.toCell(), + f4: 4, + }.toCell()); + __expect_lazy("[o] load (cell?) f2, save immutable (tail), load r3"); + val (f2, pointY) = (o.f2, o.r3.load().y); + o.f2 *= 10; + return (f2, pointY, o.toCell().hash() & 0xFFFF, o.f2); +} + + +@method_id(300) +fun demo300() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] save immutable (tail)"); + return p.toCell().beginParse().bitsEqual(stringHexToSlice("0102")); +} + +@method_id(301) +fun demo301() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] save immutable (tail), load x"); + var x = p.x; + return (x, p.toCell().beginParse().bitsEqual(stringHexToSlice("0102"))); +} + +@method_id(302) +fun demo302() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + p.x = 15; + return p.toCell().beginParse().bitsEqual(stringHexToSlice("0F02")); +} + +@method_id(303) +fun demo303() { + var st = lazy WalletStorage.fromSlice(stringHexToSlice("8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_")); + __expect_lazy("[st] load isSignatureAllowed seqno, save immutable (tail)"); + st.isSignatureAllowed = false; + st.seqno += 1; + return st.toCell().hash(); +} + +@method_id(304) +fun demo304() { + var st = lazy WalletStorage.fromSlice(stringHexToSlice("8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_")); + __expect_lazy("[st] load isSignatureAllowed seqno, save immutable (tail), load walletId publicKey extensions"); + var st2 = st; + (st.isSignatureAllowed, st.seqno) = (false, st.seqno + 1); + return (st2, st.toCell().hash()); +} + +@method_id(305) +fun demo305(setX: int?) { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x, save immutable (tail)"); + if (setX != null) { + p.x = setX; + } + return p.toCell().beginParse().loadUint(8); +} + +@method_id(306) +fun demo306(setX: int?) { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + if (setX != null) { + p.x = setX; + } + return p; +} + +@method_id(307) +fun demo307() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + var cc = (p as PointOrPoint3d).toCell(); + assert(cc.beginParse().remainingBitsCount() == 1+8+8, 100); + return cc.hash(); +} + +@method_id(308) +fun demo308() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + var cc = (p as PointOrPoint3d).toCell(); + var ll = lazy cc.load(); + assert(cc.beginParse().remainingBitsCount() == 1+8+8, 100); + __expect_lazy("[ll] lazy match"); + match (ll) { + Point => { + assert(p.x == 1, 100); + p.y += 1; + __expect_lazy("[ll] load x y"); + ll.y += 1; + assert(p.y == ll.y, 101); + ll.y -= 1; + return (p, (ll as PointOrPoint3d).toCell().hash()); + } + Point3d => throw 0 + } +} + +@method_id(309) +fun demo309() { + var st = lazy WalletStorage.load(); + __expect_lazy("[st] load (bits65) publicKey, save immutable (tail)"); + st.publicKey += 1; + return st.toCell(); +} + +@method_id(310) +fun demo310() { + val cell = NotFixedWidthStorage { + f1: 10, + f2: null, + f3: 20, + f4: true, + f5: createEmptyCell(), + f6: 10, + f7: createAddressNone(), + f8: 9, + f9: "" as bits200, + f10: ("" as bits400, ()), + f11: 0, + }.toCell({skipBitsNFieldsValidation: true}); + var st = lazy cell.load(); + var sm = lazy cell.load(); + var sk = lazy cell.load(); + __expect_lazy("[st] load (bits8) (int8?) (bits9) (cell) f6, save immutable (tail), skip (address), load f8"); + match (st.f6) { int8 => st.f6 += 90, bool => throw 123 }; + __expect_lazy("[sm] load f5"); + sm.f5; // previous (data-only) fields are not needed to reach a ref + __expect_lazy("[sk] skip (bits8) (int8?) (bits9) (int8 | bool), load f7"); + sk.f7; // ref (f5) is not needed to be skipped to reach data + return (st.toCell().hash() & 0xFFFFFFFF, st.f8); +} + +@method_id(311) +fun demo311() { + var st = lazy WalletStorage.load(); + __expect_lazy("[st] load isSignatureAllowed (bits320) extensions"); + st.isSignatureAllowed = false; + st.extensions = null; + return st.toCell(); +} + + +@method_id(400) +fun demo400() { + var t = createEmptyTuple(); + t.push(stringHexToSlice("0102")); + t.push(stringHexToSlice("0304")); + t.push(stringHexToSlice("0506")); + var corrX: int = -1; + while (t.size() && corrX == -1) { + var p = lazy Point.fromSlice(t.pop()); + __expect_lazy("[p] load x y"); + if (p.y == 6) { + corrX = p.x; + } + } + if (corrX == -1) { + throw 123; + } + return corrX; +} + +fun helper401(s: slice, fakeHandle: bool): never { + val msg = lazy MsgFullCounter.fromCell(beginCell().storeSlice(s).endCell()); + match (msg) { + CounterDecrementBy1 => throw 0, + CounterReset => throw 0, + CounterDecrement => { + var newX = 100; + __expect_lazy("[msg] load byValue"); + if (!fakeHandle) { + newX -= msg.byValue; + } else { + newX += msg.byValue; + } + throw newX; + } + CounterIncrement => { + var newX = 50; + if (!fakeHandle) { + __expect_lazy("[msg] load byValue"); + newX += msg.byValue; + } + throw newX; + } + else => throw 0xFFFF + } +} + +@method_id(401) +fun demo401(s: slice, fakeHandle: bool) { + try { + helper401(s, fakeHandle); + } catch (excNo) { + return excNo; + } +} + +@method_id(402) +fun demo402(s: slice) { + try { + val msg = lazy MsgFullCounter.fromSlice(s, {throwIfOpcodeDoesNotMatch: 0xFFFF}); + __expect_lazy("[msg] lazy match"); + match (msg) { + CounterDecrementBy1 => {} + CounterReset => {} + CounterDecrement => {} + CounterIncrement => {} + // without else, 0xFFFF is thrown by default + } + return 0; + } catch (excno) { + return excno; + } +} + +@method_id(403) +fun demo403(s: slice) { + val msg = lazy MsgFullCounter.fromSlice(s); + var st = lazy loadWalletStorage(); + var t = createEmptyTuple(); + __expect_lazy("[st] skip (bits1), load seqno walletId"); + match (msg) { + CounterIncrement => t.push(st.seqno), + CounterDecrementBy1 => throw 0, + CounterReset => throw 0, + CounterDecrement => { + t.push(st.walletId); + if (t.size() == 1) { + __expect_lazy("[msg] load byValue"); + t.push(msg.byValue); + } + } + } + return t; +} + +@method_id(404) +fun demo404(s: slice) { + var msg = lazy MyMsg32.fromSlice(s, {throwIfOpcodeDoesNotMatch: 0xFFFF}); + var curCounter = 10; + __expect_lazy("[msg] lazy match"); + match (msg) { + Counter32Increment => { + __expect_lazy("[msg] load byValue"); + curCounter += msg.byValue; + } + Counter32DecrementBy1 => { + curCounter -= 1; + } + Counter32Reset => { + __expect_lazy("[msg] load initial, skip (bits8), load f3"); + assert (msg.initial >= 0) throw 0xFF; + if (msg.initial > 100) { + msg.initial = 100; + } + curCounter = msg.initial; + if (msg.f3 == 100) { + curCounter *= 2; + } + } + } + return curCounter; +} + + +@method_id(405) +fun demo405(s: slice) { + var sum = 0; + try { var msg = lazy AutoPrefixedBin2.fromSlice(s); match (msg) { + AutoP3 => { + sum = -1; + } + AutoP1 => { + __expect_lazy("[msg] load f1 f2"); + sum = msg.f1 + msg.f2; + } + AutoP2 => { + __expect_lazy("[msg] load f1 f2 f3"); + sum = msg.f1; + sum += msg.f2; + if (sum > 0) { + sum += msg.f3; + } + } + } } catch { sum = -2 } + return sum; +} + + +@method_id(406) +fun demo406(s: slice) { + try { + match (val msg = lazy ShortPrefixed.fromSlice(s)) { + ShortP1 => { + return 1; + } + ShortP2 => { + __expect_lazy("[msg] skip (bits8), load f2"); + var sum = 2 + msg.f2; + return sum; + } + else => { throw 9999; } + } + } catch(excno) { return excno } +} + +@method_id(407) +fun demo407(s: slice): int { + match (val msg = lazy NotEqPrefixed.fromSlice(s)) { + NotEqPrefixedS1 => return msg.f, + NotEqPrefixedS2 => return msg.f, + NotEqPrefixedS3 => return msg.f, + } + throw 0xFF; +} + +@method_id(408) +fun demo408() { + var o = lazy Outer.fromSlice(stringHexToSlice("0102030408FF7FFFFF")); + __expect_lazy("[o] skip (bits16), load inner2 wa"); + if (10>3) { + o.inner2.i2 = 10; + } + return o.wa.ab.toCell().beginParse().remainingBitsCount(); +} + +@method_id(409) +fun demo409() { + var o = lazy Counter32Reset.fromSlice(stringHexToSlice("34567890000000FF02030405")); + var f3 = -1; + __expect_lazy("[o] load initial, skip (bits8), load f3"); + if (o.initial > 100) { + f3 = o.f3; + } + return (o.initial, f3); +} + +@method_id(410) +fun demo410(s: slice) { + val msg = lazy MsgFullCounter.fromSlice(s); + // via codegen, check that "return" is implicitly added, and IFJMP is generated + match (msg) { + CounterIncrement => { debug.print(1) } + CounterDecrement => { debug.dumpStack() } + CounterReset => { random.setSeed(10) } + CounterDecrementBy1 => { reserveToncoinsOnBalance(10, 1) } + else => { /* do nothing, "just accept tons" */ } + } +} + +@method_id(411) +fun demo411(s: slice) { + val msg = lazy AOrB.fromSlice(s); + // the same, but for either + match (msg) { + A => { debug.print(1) } + B => { debug.dumpStack() } + } +} + + +fun main() { + val s = stringHexToSlice("0102"); + var p = lazy Point.fromSlice(s); + __expect_lazy("[p] skip (bits8), load y"); + return (p.y, s.remainingBitsCount()); // s not changed +} + + +/** +@testcase | 0 | | 2 16 +@testcase | 101 | | 1 +@testcase | 102 | 0 | 2 +@testcase | 103 | | 8 +@testcase | 104 | | 1 4 +@testcase | 105 | x{01041_} | 1 8 +@testcase | 105 | x{01820304} | 1 4 +@testcase | 106 | x{01041_} | -1 (null) +@testcase | 106 | x{01820304} | 0 2 +@testcase | 107 | | 1 2 +@testcase | 108 | | 1 15 1 2 3 4 5 6 7 8 9 +@testcase | 109 | | 0 +@testcase | 110 | x{08} | [ 8 ] +@testcase | 110 | x{88} | [ -8 ] +@testcase | 111 | x{08} 50 | [ 8 ] +@testcase | 111 | x{88} 50 | [ -8 ] +@testcase | 112 | x{0110} | 16 +@testcase | 112 | x{020f} | -15 +@testcase | 112 | x{03} | -1 +@testcase | 112 | x{03ffff} | -1 +@testcase | 112 | x{04000000FF} | 255 +@testcase | 113 | x{0110} | 16 +@testcase | 113 | x{020f} | -15 +@testcase | 113 | x{03} | -1 +@testcase | 113 | x{03ffff} | -1 +@testcase | 113 | x{04000000FF} | 255 +@testcase | 114 | x{0110} | 16 1 +@testcase | 114 | x{020f} | [ -15 ] typeid-25 +@testcase | 114 | x{03} | -1 1 +@testcase | 114 | x{03ffff} | -1 1 +@testcase | 114 | x{04000000FF} | [ 255 ] typeid-25 +@testcase | 115 | x{08} | 8 42 +@testcase | 115 | x{88} | 0 2 +@testcase | 115 | x{83} | -1 2 +@testcase | 118 | x{08} | 8 +@testcase | 118 | x{88} | -8 +@testcase | 119 | x{0F} | 15 -1 +@testcase | 119 | x{8F0203} | -1 2 +@testcase | 120 | x{010208} | [ 8 1 ] +@testcase | 120 | x{010288} | [ -8 1 ] +@testcase | 121 | | 22 +@testcase | 122 | x{010208} | 9 1 +@testcase | 122 | x{010288} | -8 1 +@testcase | 123 | x{010280} | 5 +@testcase | 124 | x{010283} | [ -3 2 ] +@testcase | 125 | | 127 32 15 2 3 typeid-1 15 3 -1 +@testcase | 126 | x{7F0F20} | 127 (null) (null) 15 typeid-3 32 15 -1 +@testcase | 126 | x{7F8F020320} | 127 15 2 3 typeid-4 32 -1 2 +@testcase | 127 | | 1 2 -1 255 +@testcase | 128 | x{5500} | -1 -1 +@testcase | 128 | x{558F01024} | 0 2 +@testcase | 129 | | -1 15 +@testcase | 130 | | 456 +@testcase | 131 | | 777 1 +@testcase | 132 | -1 | 889 -1 +@testcase | 132 | 0 | 888 -1 +@testcase | 133 | | 18 +@testcase | 134 | | 15 +@testcase | 135 | -1 | 40 +@testcase | 135 | 0 | 9 +@testcase | 136 | -1 | -1 +@testcase | 136 | 0 | 2 +@testcase | 137 | x{0102} | 1 +@testcase | 137 | x{03} | 3 +@testcase | 137 | x{1} | 109 +@testcase | 138 | x{0102} | 2 +@testcase | 138 | x{03} | 109 +@testcase | 139 | x{0105} 0 | 5 +@testcase | 139 | x{01} 0 | 9 +@testcase | 139 | x{0230} 0 | 48 +@testcase | 139 | x{0230} -1 | 100 +@testcase | 140 | -1 | 4 +@testcase | 140 | 0 | 255 +@testcase | 141 | -1 | 1 +@testcase | 142 | x{010203} 0 | 3 +@testcase | 142 | x{880203} 0 | 8 +@testcase | 142 | x{0102} 0 | 9 +@testcase | 143 | | 17 +@testcase | 144 | 0 | 1 +@testcase | 145 | 0 | 2 +@testcase | 145 | -1 | 5 +@testcase | 146 | | -1 +@testcase | 147 | | -1 0 +@testcase | 148 | | 8 +@testcase | 149 | | 9 +@testcase | 150 | | 1 16 +@testcase | 151 | | -1 +@testcase | 152 | | 32 +@testcase | 153 | | 0 +@testcase | 154 | | [ [ 1 2 ] ] +@testcase | 155 | | 555 +@testcase | 156 | x{0110} | 16 +@testcase | 156 | x{FF10} | -100 + +@testcase | 200 | | 1 +@testcase | 201 | | 1 2 -1 +@testcase | 202 | | 127 127 6 +@testcase | 203 | | 1 2 3 4 5 6 7 8 9 +@testcase | 204 | | 10 20 3 40 5 6 7 8 9 +@testcase | 205 | | -1 +@testcase | 206 | | -1 72 +@testcase | 207 | | C{11A18FA6FB377116ABE1827A4469944C29484C8B00B33BDC4E560D405D5BFDAD} +@testcase | 208 | x{0F0203} | 23 24 +@testcase | 209 | | 23 +@testcase | 210 | | 40 +@testcase | 211 | | C{32B95044B7193DA671CFD5822D3CAA23DF5055A7E75622EBCB16D31C23C50AA8} +@testcase | 212 | | -1 +@testcase | 213 | -1 | 48 11 +@testcase | 214 | | 258 0 +@testcase | 215 | | 258 0 +@testcase | 216 | | 12826 456 +@testcase | 217 | | 514 +@testcase | 218 | | 2 20 11636 20 + +@testcase | 300 | | -1 +@testcase | 301 | | 1 -1 +@testcase | 302 | | -1 +@testcase | 303 | | 31704026347210867689619328142399241654002690318751313036781135835273797831937 +@testcase | 304 | | -1 123 456 0 (null) 31704026347210867689619328142399241654002690318751313036781135835273797831937 +@testcase | 305 | null | 1 +@testcase | 305 | 88 | 88 +@testcase | 306 | null | 1 2 +@testcase | 306 | 88 | 88 2 +@testcase | 307 | | 79635787059049818071080046066850883641967849619803682012164346688156057263776 +@testcase | 308 | | 1 3 79635787059049818071080046066850883641967849619803682012164346688156057263776 +@testcase | 309 | | C{EE7C06B69DDCEFEBED507FE35F603A9019D4AE79A13F670FE4E08928AF484838} +@testcase | 310 | | 3332695883 9 +@testcase | 311 | | C{16A4757FA412AD5E89418F1263904CD57FA4DF0E73EE8A53DC60F862C72BCD9D} + +@testcase | 400 | | 5 +@testcase | 401 | x{0110} -1 | 50 +@testcase | 401 | x{020f} 0 | 85 +@testcase | 402 | x{020f} | 0 +@testcase | 402 | x{ffffffff} | 65535 +@testcase | 403 | x{0110} | [ 123 ] +@testcase | 403 | x{020f} | [ 456 15 ] +@testcase | 404 | x{1234567800000010} | 26 +@testcase | 404 | x{23456789} | 9 +@testcase | 404 | x{345678900000000002030405} | 0 +@testcase | 405 | x{01c220_} | 15 +@testcase | 405 | x{41c20260_} | 24 +@testcase | 405 | x{a0_} | -1 +@testcase | 405 | x{ffff} | -2 +@testcase | 406 | x{7} | 1 +@testcase | 406 | x{40102} | 4 +@testcase | 406 | x{99} | 9999 +@testcase | 407 | x{7F} | 63 +@testcase | 407 | x{1F} | 15 +@testcase | 407 | x{BF} | -1 +@testcase | 408 | | 24 +@testcase | 409 | | 255 3 + +@fif_codegen +""" + demo101 PROC:<{ // + x{0102} PUSHSLICE // lazyS + 8 LDI // '9 lazyS + DROP // p.x + }> +""" + +@fif_codegen +""" + demo103 PROC:<{ // + x{0102} PUSHSLICE // lazyS + 8 PUSHINT + SDSKIPFIRST // lazyS + 8 LDI // '10 lazyS + DROP // p.y +""" + +@fif_codegen +""" + demo106 PROC:<{ // s + // lazyS + PUSHNULL // lazyS x + SWAP // x lazyS + 8 PUSHINT + SDSKIPFIRST // x lazyS + 1 LDU // x '18 lazyS +""" + +@fif_codegen +""" + demo109 PROC:<{ // + x{8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_} PUSHSLICE // lazyS + 65 PUSHINT + SDSKIPFIRST // lazyS + 256 LDU // '13 lazyS + DROP // st.publicKey + }> +""" + +@fif_codegen +""" + demo113 PROC:<{ + // lazyS + x{01} SDBEGINSQ + IF:<{ + 8 LDI + DROP + }>ELSE<{ + x{02} SDBEGINSQ + IF:<{ + 8 LDI + DROP + NEGATE + }>ELSE<{ + x{03} SDBEGINSQ + IF:<{ + DROP + -1 PUSHINT + }>ELSE<{ + x{04} SDBEGINSQ + IFNOTJMP:<{ + 63 THROW + }> + 32 LDI + DROP + }> + }> + }> + }> +""" + +@fif_codegen +""" + demo134 PROC:<{ // + x{010f} PUSHSLICE // lazyS + x{01} SDBEGINSQ // lazyS '6 + 134 THROWIFNOT // lazyS + 8 LDI // '11 lazyS + DROP // cc + }> +""" + +@fif_codegen +""" + demo136 PROC:<{ + x{0102} PUSHSLICE + SWAP + IFJMP:<{ + DROP + -1 PUSHINT + }> + 8 PUSHINT + SDSKIPFIRST + 8 LDI + DROP + }> +""" + +@fif_codegen +""" + demo146 PROC:<{ + x{0000000} PUSHSLICE + LDOPTREF + DROP + ISNULL + }> +""" + +@fif_codegen +""" + demo201 PROC:<{ + x{0102} PUSHSLICE + DUP + 8 LDI + 8 LDI + DROP + s2 PUSH + NEWC + STSLICE + ENDC + s3 PUSH + NEWC + STSLICE + ENDC + s0 s4 XCHG + NEWC + STSLICE + ENDC +""" + + +@fif_codegen +""" + demo203 PROC:<{ // + x{010203040506070809} PUSHSLICE // lazyS + DUP // '14 lazyS + 8 LDU // '14 '16 lazyS + DROP // '14 o.f1 + 1 EQINT // '14 '18 + IFJMP:<{ // '14 + NEWC // '14 b + STSLICE // b + ENDC // '21 + CTOS // s +""" + +@fif_codegen +""" + demo206 PROC:<{ // + x{010203040506070809} PUSHSLICE // lazyS + DUP // '14 lazyS + 8 LDU // '14 o.f1 lazyS + 8 LDU // '14 o.f1 o.f2 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 o.f8 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 o.f8 '32 lazyS + DROP // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 o.f8 o.f9 +""" + +@fif_codegen +""" + demo215 PROC:<{ + x{0102} PUSHSLICE + 8 LDI + 8 LDI + DROP + SWAP + NEWC + 8 STI + 8 STI + ENDC +""" + +@fif_codegen +""" + demo302 PROC:<{ + x{0102} PUSHSLICE + 8 LDI + NIP + 15 PUSHINT + NEWC + 8 STI + STSLICE +""" + +@fif_codegen +""" + demo311 PROC:<{ // + 777 PUSHINT // '6=777 + contract.getFakeData CALLDICT // '7 + CTOS // lazyS + 1 LDI // '13 lazyS + NIP // lazyS + 320 PUSHINT // lazyS '14=320 + LDSLICEX // '15 lazyS + DROP // '15 + PUSHNULL // '15 st.extensions + 0 PUSHINT + NEWC + 1 STU // '15 st.extensions b + s1 s2 XCHG // st.extensions '15 b + STSLICE // st.extensions b + STOPTREF // b + ENDC // '21 + }> +""" + +@fif_codegen +""" + demo404 PROC:<{ + // lazyS + x{12345678} SDBEGINSQ + IF:<{ + 32 LDI + DROP + 10 ADDCONST + }>ELSE<{ + x{23456789} SDBEGINSQ + IF:<{ + DROP + 9 PUSHINT + }>ELSE<{ + x{34567890} SDBEGINSQ + IFNOTJMP:<{ + 16 PUSHPOW2DEC + THROWANY + }> + 32 LDU + 8 PUSHINT + SDSKIPFIRST + 8 LDI + DROP + OVER + 100 GTINT + IF:<{ + 100 PUSHINT + s2 POP + }> + 100 EQINT + IF:<{ + 1 LSHIFT# + }> + }> + }> + }> +""" + +@fif_codegen +""" + demo410 PROC:<{ + // lazyS + x{01} SDBEGINSQ + IFJMP:<{ + ... + }> + x{02} SDBEGINSQ + IFJMP:<{ + DROP + DUMPSTK + }> + x{04} SDBEGINSQ + IFJMP:<{ + ... + }> + x{03} SDBEGINSQ + NIP + IFJMP:<{ + ... + RAWRESERVE + }> + }> +""" + +@fif_codegen +""" + demo411 PROC:<{ // s + // lazyS + 1 LDU // '10 lazyS + DROP // '10 + IFJMP:<{ // + DUMPSTK // + }> // + 1 PUSHINT // '11=1 + s0 DUMP DROP // + }> +""" + +@fif_codegen DECLPROC pushToGlobalTup + */ diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 4a9a431e7e..7c224bab4b 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -404,8 +404,8 @@ fun test28(getEmpty: bool) { return (t1, t1 is Empty, t1 is (int, int), t1 is builder, 777, t2, t2 is Empty, t2 is Has2EmptyNullable, t2 == null); } -struct Empty1 {} -struct Empty2 {} +struct Empty1; +struct Empty2; @method_id(129) fun test29() { diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index 4db9233a9e..0705876f30 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -23,12 +23,14 @@ set(TOLK_SOURCE pipe-constant-folding.cpp pipe-optimize-boolean-expr.cpp pipe-detect-inline-in-place.cpp + pipe-lazy-load-insertions.cpp pipe-ast-to-legacy.cpp pipe-find-unused-symbols.cpp pipe-generate-fif-output.cpp type-system.cpp smart-casts-cfg.cpp generics-helpers.cpp + lazy-helpers.cpp abscode.cpp analyzer.cpp asmops.cpp diff --git a/tolk/ast-aux-data.h b/tolk/ast-aux-data.h index 59006b1fcc..37cbcfa844 100644 --- a/tolk/ast-aux-data.h +++ b/tolk/ast-aux-data.h @@ -17,6 +17,7 @@ #pragma once #include "ast.h" +#include "lazy-helpers.h" /* * This file contains a schema of aux_data inside ast_artificial_aux_vertex @@ -35,4 +36,27 @@ struct AuxData_ForceFiftLocation final : ASTAuxData { } }; +// AuxData_LazyObjectLoadFields is a special auto-inserted vertex to load fields of a lazy struct; +// example: `var p = lazy Point.fromSlice(s); aux "load x"; return p.x` +struct AuxData_LazyObjectLoadFields final : ASTAuxData { + LocalVarPtr var_ref; // comes from `lazy` + TypePtr union_variant; // not just `o` but `match(o) { V1 => here }` + StructFieldPtr field_ref; // not just `o` but `match(o.field) { V1 => here }` + LazyStructLoadInfo load_info; // instructions, which fields to load, which to skip, etc. + + AuxData_LazyObjectLoadFields(LocalVarPtr var_ref, TypePtr union_variant, StructFieldPtr field_ref, LazyStructLoadInfo load_info) + : var_ref(var_ref), union_variant(union_variant), field_ref(field_ref), load_info(std::move(load_info)) { + } +}; + +// AuxData_LazyMatchForUnion wraps `match(lazy_var)` or its field +struct AuxData_LazyMatchForUnion final : ASTAuxData { + LocalVarPtr var_ref; // comes from `lazy` + StructFieldPtr field_ref; // not `match(o)`, but `match(o.field)` + + AuxData_LazyMatchForUnion(LocalVarPtr var_ref, StructFieldPtr field_ref) + : var_ref(var_ref), field_ref(field_ref) { + } +}; + } // namespace tolk diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 5c5953415b..4ad6ea9548 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -690,7 +690,7 @@ static V parse_match_arm(Lexer& lex) { } lex.expect(tok_double_arrow, "`=>`"); - AnyExprV body; + V body = nullptr; if (lex.tok() == tok_opbrace) { // pattern => { ... } AnyV v_block = parse_statement(lex); body = createV(v_block->loc, v_block); @@ -704,7 +704,9 @@ static V parse_match_arm(Lexer& lex) { AnyV v_block = createV(v_return->loc, v_return->loc, {v_return}); body = createV(v_block->loc, v_block); } else { - body = parse_expr(lex); + AnyExprV unbraced_expr = parse_expr(lex); + AnyV v_block = createV(unbraced_expr->loc, unbraced_expr->loc, {createV(unbraced_expr->loc, unbraced_expr)}); + body = createV(unbraced_expr->loc, v_block); } if (pattern_expr == nullptr) { // for match by type / default case, empty vertex, not nullptr @@ -731,13 +733,14 @@ static V parse_match_expression(Lexer& lex) { // after `pattern => { ... }` comma is optional, after `pattern => expr` mandatory bool was_comma = lex.tok() == tok_comma; // trailing comma is allowed always + bool was_unbraced = v_arm->get_body()->get_block_statement()->size() == 1 && v_arm->get_body()->get_block_statement()->get_item(0)->kind == ast_braced_yield_result; if (was_comma) { lex.next(); } if (lex.tok() == tok_clbrace) { break; } - if (!was_comma && v_arm->get_body()->kind != ast_braced_expression) { + if (!was_comma && was_unbraced) { lex.unexpected("`,`"); } } @@ -745,6 +748,14 @@ static V parse_match_expression(Lexer& lex) { return createV(loc, std::move(subject_and_arms)); } +static V parse_lazy_operator(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_lazy, "`lazy`"); + + AnyExprV expr = parse_expr(lex); + return createV(loc, expr); +} + // parse (expr) / [expr] / identifier / number static AnyExprV parse_expr100(Lexer& lex) { SrcLocation loc = lex.cur_location(); @@ -852,6 +863,8 @@ static AnyExprV parse_expr100(Lexer& lex) { return createV(loc, nullptr, parse_object_body(lex)); case tok_match: return parse_match_expression(lex); + case tok_lazy: + return parse_lazy_operator(lex); default: lex.unexpected(""); } @@ -1529,18 +1542,20 @@ static AnyV parse_struct_field(Lexer& lex) { static V parse_struct_body(Lexer& lex) { SrcLocation loc = lex.cur_location(); - lex.expect(tok_opbrace, "`{`"); - std::vector fields; - while (lex.tok() != tok_clbrace) { - fields.push_back(parse_struct_field(lex)); - if (lex.tok() == tok_comma || lex.tok() == tok_semicolon) { - lex.next(); - } else if (lex.tok() != tok_clbrace) { - lex.unexpected("`;` or `,`"); + + if (lex.tok() != tok_semicolon) { // `struct A;` equal to `struct A {}` + lex.expect(tok_opbrace, "`{`"); + while (lex.tok() != tok_clbrace) { + fields.push_back(parse_struct_field(lex)); + if (lex.tok() == tok_comma || lex.tok() == tok_semicolon) { + lex.next(); + } else if (lex.tok() != tok_clbrace) { + lex.unexpected("`;` or `,`"); + } } + lex.expect(tok_clbrace, "`}`"); } - lex.expect(tok_clbrace, "`}`"); return createV(loc, std::move(fields)); } diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 03e7d43206..581babee31 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -95,6 +95,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } @@ -118,6 +119,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } @@ -140,6 +142,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { case ast_empty_expression: return replace(v->as()); case ast_parenthesized_expression: return replace(v->as()); case ast_braced_expression: return replace(v->as()); + case ast_braced_yield_result: return replace(v->as()); case ast_artificial_aux_vertex: return replace(v->as()); case ast_tensor: return replace(v->as()); case ast_bracket_tuple: return replace(v->as()); @@ -163,6 +166,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { case ast_cast_as_operator: return replace(v->as()); case ast_is_type_operator: return replace(v->as()); case ast_not_null_operator: return replace(v->as()); + case ast_lazy_operator: return replace(v->as()); case ast_match_expression: return replace(v->as()); case ast_match_arm: return replace(v->as()); case ast_object_field: return replace(v->as()); diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 2aabec2b6f..19956e4675 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -84,6 +84,9 @@ class ASTReplicator final { static V clone(V v) { return createV(v->loc, clone(v->get_block_statement())); } + static V clone(V v) { + return createV(v->loc, clone(v->get_expr())); + } static V clone(V v) { return createV(v->loc, clone(v->get_wrapped_expr()), v->aux_data, v->inferred_type); } @@ -153,6 +156,9 @@ class ASTReplicator final { static V clone(V v) { return createV(v->loc, clone(v->get_expr())); } + static V clone(V v) { + return createV(v->loc, clone(v->get_expr())); + } static V clone(V v) { return createV(v->loc, clone(v->get_all_children())); } @@ -273,6 +279,7 @@ class ASTReplicator final { case ast_empty_expression: return clone(v->as()); case ast_parenthesized_expression: return clone(v->as()); case ast_braced_expression: return clone(v->as()); + case ast_braced_yield_result: return clone(v->as()); case ast_artificial_aux_vertex: return clone(v->as()); case ast_tensor: return clone(v->as()); case ast_bracket_tuple: return clone(v->as()); @@ -296,6 +303,7 @@ class ASTReplicator final { case ast_cast_as_operator: return clone(v->as()); case ast_is_type_operator: return clone(v->as()); case ast_not_null_operator: return clone(v->as()); + case ast_lazy_operator: return clone(v->as()); case ast_match_expression: return clone(v->as()); case ast_match_arm: return clone(v->as()); case ast_object_field: return clone(v->as()); diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index 395f956811..b9ee069793 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -45,6 +45,7 @@ class ASTStringifier final : public ASTVisitor { {ast_empty_expression, "ast_empty_expression"}, {ast_parenthesized_expression, "ast_parenthesized_expression"}, {ast_braced_expression, "ast_braced_expression"}, + {ast_braced_yield_result, "ast_braced_yield_result"}, {ast_artificial_aux_vertex, "ast_artificial_aux_vertex"}, {ast_tensor, "ast_tensor"}, {ast_bracket_tuple, "ast_bracket_tuple"}, @@ -68,6 +69,7 @@ class ASTStringifier final : public ASTVisitor { {ast_cast_as_operator, "ast_cast_as_operator"}, {ast_is_type_operator, "ast_is_type_operator"}, {ast_not_null_operator, "ast_not_null_operator"}, + {ast_lazy_operator, "ast_lazy_operator"}, {ast_match_expression, "ast_match_expression"}, {ast_match_arm, "ast_match_arm"}, {ast_object_field, "ast_object_field"}, @@ -302,6 +304,7 @@ class ASTStringifier final : public ASTVisitor { case ast_empty_expression: return handle_vertex(v->as()); case ast_parenthesized_expression: return handle_vertex(v->as()); case ast_braced_expression: return handle_vertex(v->as()); + case ast_braced_yield_result: return handle_vertex(v->as()); case ast_artificial_aux_vertex: return handle_vertex(v->as()); case ast_tensor: return handle_vertex(v->as()); case ast_bracket_tuple: return handle_vertex(v->as()); @@ -325,6 +328,7 @@ class ASTStringifier final : public ASTVisitor { case ast_cast_as_operator: return handle_vertex(v->as()); case ast_is_type_operator: return handle_vertex(v->as()); case ast_not_null_operator: return handle_vertex(v->as()); + case ast_lazy_operator: return handle_vertex(v->as()); case ast_match_expression: return handle_vertex(v->as()); case ast_match_arm: return handle_vertex(v->as()); case ast_object_field: return handle_vertex(v->as()); diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index 926ec5808a..6e38ba7929 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -67,7 +67,7 @@ class ASTVisitor { } GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTExprBlockOfStatements* v) { - visit_children(v->child_block_statement->as()); + visit(v->child_block_statement->as()); } GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTStatementUnary* v) { @@ -104,6 +104,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } @@ -127,6 +128,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } @@ -150,6 +152,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { case ast_empty_expression: return visit(v->as()); case ast_parenthesized_expression: return visit(v->as()); case ast_braced_expression: return visit(v->as()); + case ast_braced_yield_result: return visit(v->as()); case ast_artificial_aux_vertex: return visit(v->as()); case ast_tensor: return visit(v->as()); case ast_bracket_tuple: return visit(v->as()); @@ -173,6 +176,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { case ast_cast_as_operator: return visit(v->as()); case ast_is_type_operator: return visit(v->as()); case ast_not_null_operator: return visit(v->as()); + case ast_lazy_operator: return visit(v->as()); case ast_match_expression: return visit(v->as()); case ast_match_arm: return visit(v->as()); case ast_object_field: return visit(v->as()); diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 27c7e4c13b..6e3b3b8902 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -152,6 +152,10 @@ void Vertex::assign_is_negated(bool is_negated) { this->is_negated = is_negated; } +void Vertex::assign_dest_var_ref(LocalVarPtr dest_var_ref) { + this->dest_var_ref = dest_var_ref; +} + void Vertex::assign_resolved_pattern(MatchArmKind pattern_kind, AnyExprV pattern_expr) { this->pattern_type_node = nullptr; this->pattern_kind = pattern_kind; @@ -190,6 +194,10 @@ void Vertex::assign_first_unreachable(AnyV first_unreachabl this->first_unreachable = first_unreachable; } +void Vertex::assign_new_children(std::vector&& children) { + this->children = std::move(children); +} + void Vertex::assign_target(const DotTarget& target) { this->target = target; } diff --git a/tolk/ast.h b/tolk/ast.h index 400c849b1f..1da1d6e6c2 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -75,6 +75,7 @@ enum ASTNodeKind { ast_empty_expression, ast_parenthesized_expression, ast_braced_expression, + ast_braced_yield_result, ast_artificial_aux_vertex, ast_tensor, ast_bracket_tuple, @@ -98,6 +99,7 @@ enum ASTNodeKind { ast_cast_as_operator, ast_is_type_operator, ast_not_null_operator, + ast_lazy_operator, ast_match_expression, ast_match_arm, ast_object_field, @@ -499,6 +501,7 @@ template<> // it can contain arbitrary statements inside // it can occur only in special places within the input code, not anywhere // example: `match (intV) { 0 => { ... } }` rhs of 0 is braced expression +// example: `match (intV) { 0 => 1 }` rhs is implicitly wrapped to a braced expression with `1` yielded struct Vertex final : ASTExprBlockOfStatements { auto get_block_statement() const { return child_block_statement->as(); } @@ -506,6 +509,16 @@ struct Vertex final : ASTExprBlockOfStatements { : ASTExprBlockOfStatements(ast_braced_expression, loc, child_block_statement) {} }; +template<> +// ast_braced_yield_result is a special vertex to return a value from a braced expression +// example: `match (intV) { 0 => 1 }` rhs of 0 is implicitly wrapped to a braced expression with `1` yielded +struct Vertex final : ASTExprUnary { + AnyExprV get_expr() const { return child; } + + Vertex(SrcLocation loc, AnyExprV expr) + : ASTExprUnary(ast_braced_yield_result, loc, expr) {} +}; + template<> // ast_artificial_aux_vertex is a compiler-inserted vertex that can't occur in source code // example: special vertex to force location in Fift output at constant usage @@ -880,6 +893,21 @@ struct Vertex final : ASTExprUnary { : ASTExprUnary(ast_not_null_operator, loc, expr) {} }; +template<> +// ast_lazy_operator is `lazy (loading-expr)` for lazy/partial loading +// example: `lazy Storage.fromCell(contract.getData())` +struct Vertex final : ASTExprUnary { + LocalVarPtr dest_var_ref = nullptr; // `var st = lazy Storage.load()` + + AnyExprV get_expr() const { return child; } + + Vertex* mutate() const { return const_cast(this); } + void assign_dest_var_ref(LocalVarPtr dest_var_ref); + + Vertex(SrcLocation loc, AnyExprV expr) + : ASTExprUnary(ast_lazy_operator, loc, expr) {} +}; + template<> // ast_match_expression is `match (subject) { ... arms ... }`, used either as a statement or as an expression // example: `match (intOrSliceVar) { int => 1, slice => 2 }` @@ -907,12 +935,12 @@ struct Vertex final : ASTExprBinary { AnyTypeV pattern_type_node; // for MatchArmKind::exact_type; otherwise nullptr AnyExprV get_pattern_expr() const { return lhs; } - AnyExprV get_body() const { return rhs; } // remember, it may be V + auto get_body() const { return rhs->as(); } Vertex* mutate() const { return const_cast(this); } void assign_resolved_pattern(MatchArmKind pattern_kind, AnyExprV pattern_expr); - Vertex(SrcLocation loc, MatchArmKind pattern_kind, AnyTypeV pattern_type_node, AnyExprV pattern_expr, AnyExprV body) + Vertex(SrcLocation loc, MatchArmKind pattern_kind, AnyTypeV pattern_type_node, AnyExprV pattern_expr, V body) : ASTExprBinary(ast_match_arm, loc, pattern_expr, body) , pattern_kind(pattern_kind), pattern_type_node(pattern_type_node) {} }; @@ -999,6 +1027,7 @@ struct Vertex final : ASTStatementVararg { Vertex* mutate() const { return const_cast(this); } void assign_first_unreachable(AnyV first_unreachable); + void assign_new_children(std::vector&& children); Vertex(SrcLocation loc, SrcLocation loc_end, std::vector&& items) : ASTStatementVararg(ast_block_statement, loc, std::move(items)) diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index d4db1edec4..f0d8de7cb2 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1491,6 +1491,9 @@ void define_builtins() { define_builtin_method("T.getDeclaredPackPrefixLen", typeT, {}, Int, declReceiverT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal | FunctionData::flagAllowAnyWidthT); + define_builtin_method("T.forceLoadLazyObject", typeT, {typeT}, Slice, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); define_builtin_method("Cell.load", CellT, {CellT, UnpackOptions}, typeT, declReceiverT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); @@ -1520,6 +1523,9 @@ void define_builtins() { define_builtin_func("__expect_inline", {Bool}, Unit, nullptr, compile_expect_type, FunctionData::flagMarkedAsPure); + define_builtin_func("__expect_lazy", {Slice}, Unit, nullptr, + compile_expect_type, + FunctionData::flagMarkedAsPure); define_builtin_method("T.__toTuple", typeT, {typeT}, TypeDataTuple::create(), declReceiverT, compile_any_object_to_tuple, FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); diff --git a/tolk/fwd-declarations.h b/tolk/fwd-declarations.h index 981eaa4a67..f46def4ac3 100644 --- a/tolk/fwd-declarations.h +++ b/tolk/fwd-declarations.h @@ -60,4 +60,6 @@ enum class FunctionInlineMode { noInline, }; +typedef int var_idx_t; + } // namespace tolk diff --git a/tolk/lazy-helpers.cpp b/tolk/lazy-helpers.cpp new file mode 100644 index 0000000000..22ef5fff27 --- /dev/null +++ b/tolk/lazy-helpers.cpp @@ -0,0 +1,101 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "lazy-helpers.h" +#include "tolk.h" +#include "type-system.h" + +namespace tolk { + +/* + * This file contains "lazy" state across multiple files. + * They all are used after `lazy` operators have been processed and "load xxx" vertices have been inserted. + * Particularly, while transforming AST to Ops. + * For comments about laziness, see pipe-lazy-load-insertions.cpp. + */ + +LazyVariableLoadedState::LazyVariableLoadedState(TypePtr declared_type, std::vector&& ir_slice, std::vector&& ir_options) + : declared_type(declared_type) + , ir_slice(std::move(ir_slice)) + , ir_options(std::move(ir_options)) + , loaded_state(declared_type->unwrap_alias()->try_as() ? declared_type->unwrap_alias()->try_as()->struct_ref : nullptr) { + // fill loaded_variants: variants of a lazy union or the last field of a struct if it's a union + const TypeDataUnion* t_union = declared_type->unwrap_alias()->try_as(); + if (is_struct() && loaded_state.original_struct->get_num_fields()) { + t_union = loaded_state.original_struct->fields.back()->declared_type->unwrap_alias()->try_as(); + } + if (t_union) { + variants_state.reserve(t_union->size()); + for (TypePtr variant : t_union->variants) { + const TypeDataStruct* variant_struct = variant->unwrap_alias()->try_as(); + variants_state.emplace_back(variant_struct ? variant_struct->struct_ref : nullptr); + } + } +} + +const LazyStructLoadedState* LazyVariableLoadedState::get_struct_state(StructPtr original_struct) const { + if (loaded_state.original_struct == original_struct) { + return &loaded_state; + } + for (const LazyStructLoadedState& struct_state : variants_state) { + if (struct_state.original_struct == original_struct) { + return &struct_state; + } + } + return nullptr; +} + +void LazyVariableLoadedState::assert_field_loaded(StructPtr original_struct, StructFieldPtr original_field) const { + // on field access `point.x`, ensure that it's loaded, so the value on a stack is not an occasional null + const LazyStructLoadedState* struct_state = get_struct_state(original_struct); + tolk_assert(struct_state); + tolk_assert(struct_state->was_loaded_once()); + StructFieldPtr hidden_field = struct_state->hidden_struct->find_field(original_field->name); + tolk_assert(hidden_field); + tolk_assert(struct_state->ith_field_was_loaded[hidden_field->field_idx]); +} + +void LazyStructLoadedState::on_started_loading(StructPtr hidden_struct) { + this->hidden_struct = hidden_struct; + this->ith_field_was_loaded.resize(hidden_struct->get_num_fields()); // initially false +} + +void LazyStructLoadedState::on_original_field_loaded(StructFieldPtr hidden_field) { + this->ith_field_was_loaded[hidden_field->field_idx] = true; + // for example, `var p = lazy Point; aux "load x"; return p.x`; + // we are at "load x", it exists in Point, here just save it was loaded (for assertions and debugging); + // apart from saving, stack is also updated when loading, `p` becomes `valueX null` +} + +void LazyStructLoadedState::on_aside_field_loaded(StructFieldPtr hidden_field, std::vector&& ir_field_gap) { + this->ith_field_was_loaded[hidden_field->field_idx] = true; + this->aside_gaps_and_tail.emplace_back(hidden_field, std::move(ir_field_gap)); + // for example, `var st = lazy Storage; aux "load gap, load seqno"; st.seqno += 1; st.toCell()`; + // we are at "load gap", it does not exist in Storage, so save loaded value separately +} + +std::vector LazyStructLoadedState::get_ir_loaded_aside_field(StructFieldPtr hidden_field) const { + // for example, `var st = lazy Storage; aux "load gap, load seqno"; st.seqno += 1; st.toCell()`; + // we are at "st.toCell()" that stores immutable gap before modified "seqno" + for (const auto& [gap_field, ir_field_gap] : this->aside_gaps_and_tail) { + if (gap_field == hidden_field) { + return ir_field_gap; + } + } + tolk_assert(false); +} + +} // namespace tolk diff --git a/tolk/lazy-helpers.h b/tolk/lazy-helpers.h new file mode 100644 index 0000000000..8fea95b7bd --- /dev/null +++ b/tolk/lazy-helpers.h @@ -0,0 +1,91 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "fwd-declarations.h" +#include + +namespace tolk { + +// LazyStructLoadInfo describes how to load a struct: which fields to load, which to skip. +// It's calculated based on variable usages and passed through the pipeline carried by auxiliary AST vertices. +// Based on it, lazy loading Ops are generated in pack-unpack api. +// To understand `hidden_struct`, read pipe-lazy-load-insertions.cpp. +struct LazyStructLoadInfo { + enum ActionWithField { + LoadField, + SkipField, + LazyMatchField, + SaveImmutableTail, + }; + + StructPtr original_struct; // original (e.g. `Point`) + StructPtr hidden_struct; // "lazy Point" — only requested fields, matching binary shape + std::vector ith_field_action; // each for corresponding field of a struct + + LazyStructLoadInfo(StructPtr original_struct, StructPtr hidden_struct, std::vector&& ith_field_action) + : original_struct(original_struct) + , hidden_struct(hidden_struct) + , ith_field_action(std::move(ith_field_action)) { + } +}; + +// LazyStructLoadedState represents state (which fields were already loaded) while generating AST to Ops. +// For example, variable `var p = lazy Point.fromSlice(s); aux "load x"; return p.x` is initially "nothing loaded", +// and after "load x" ith_field_action[0] becomes true (and `p` is updated on a stack and becomes `valueX null`). +struct LazyStructLoadedState { + StructPtr original_struct; // original (e.g. `Point`) + StructPtr hidden_struct = nullptr; // "lazy Point" — only requested fields, matching binary shape + std::vector ith_field_was_loaded; // each for corresponding field of hidden_struct + std::vector>> aside_gaps_and_tail; + + explicit LazyStructLoadedState(StructPtr original_struct) + : original_struct(original_struct) {} + + void on_started_loading(StructPtr hidden_struct); + void on_original_field_loaded(StructFieldPtr hidden_field); + void on_aside_field_loaded(StructFieldPtr hidden_field, std::vector&& ir_field_gap); + + bool was_loaded_once() const { return hidden_struct != nullptr; } + std::vector get_ir_loaded_aside_field(StructFieldPtr hidden_field) const; + + LazyStructLoadedState* mutate() const { return const_cast(this); } +}; + +// LazyVariableLoadedState contains a state of a whole lazy variable while generating AST to Ops. +// For example, `var p = lazy Point.fromSlice(s)` contains one struct. +// But `var msg = lazy MyMsgUnion.fromSlice(s)` contains N variants, each with own state, but common lazy slice `s`. +// When inlining a function, like `p.getX()`, `self` of `getX` also becomes a lazy variable pointing to the same state. +struct LazyVariableLoadedState { + TypePtr declared_type; + std::vector ir_slice; // filled by `lazy` operator + std::vector ir_options; // same, comes from `lazy T.fromSlice(s, options)` + LazyStructLoadedState loaded_state; // for struct: filled; for union: empty + std::vector variants_state; // variants of a lazy union or the last field if it's a union + + LazyVariableLoadedState(TypePtr declared_type, std::vector&& ir_slice, std::vector&& ir_options); + + bool is_struct() const { return loaded_state.original_struct != nullptr; } + bool is_union() const { return loaded_state.original_struct == nullptr; } + + const LazyStructLoadedState* get_struct_state(StructPtr original_struct) const; + void assert_field_loaded(StructPtr original_struct, StructFieldPtr original_field) const; +}; + + +} // namespace tolk + diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index 94620dc25e..44d1667686 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -354,6 +354,7 @@ struct ChunkIdentifierOrKeyword final : ChunkLexerBase { if (str == "self") return tok_self; if (str == "tolk") return tok_tolk; if (str == "type") return tok_type; + if (str == "lazy") return tok_lazy; if (str == "enum") return tok_enum; break; case 5: diff --git a/tolk/lexer.h b/tolk/lexer.h index dbe0f34b71..63f0e6a306 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -118,6 +118,7 @@ enum TokenType { tok_if, tok_else, tok_match, + tok_lazy, tok_arrow, tok_double_arrow, diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index 107130d598..62ef8d7bbe 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -16,6 +16,7 @@ */ #include "pack-unpack-api.h" #include "generics-helpers.h" +#include "lazy-helpers.h" #include "type-system.h" #include @@ -182,6 +183,14 @@ bool check_struct_can_be_packed_or_unpacked(TypePtr any_type, bool is_pack, std: return true; } +static int calc_offset_on_stack(StructPtr struct_ref, int field_idx) { + int stack_offset = 0; + for (int i = 0; i < field_idx; ++i) { + stack_offset += struct_ref->get_field(i)->declared_type->get_width_on_stack(); + } + return stack_offset; +} + // -------------------------------------------- // high-level API for outer code @@ -254,6 +263,126 @@ std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation return ir_slice; // return mutated slice } +void generate_lazy_struct_from_slice(CodeBlob& code, SrcLocation loc, const LazyVariableLoadedState* lazy_variable, const LazyStructLoadInfo& load_info, const std::vector& ir_obj) { + StructPtr original_struct = load_info.original_struct; + StructPtr hidden_struct = load_info.hidden_struct; + tolk_assert(hidden_struct->fields.size() == load_info.ith_field_action.size()); + + const LazyStructLoadedState* loaded_state = lazy_variable->get_struct_state(original_struct); + tolk_assert(loaded_state && !loaded_state->was_loaded_once()); + loaded_state->mutate()->on_started_loading(hidden_struct); + + UnpackContext ctx(code, loc, lazy_variable->ir_slice, lazy_variable->ir_options); + + if (hidden_struct->opcode.exists()) { + ctx.loadAndCheckOpcode(hidden_struct->opcode); + } + + for (int field_idx = 0; field_idx < hidden_struct->get_num_fields(); ++field_idx) { + StructFieldPtr hidden_field = hidden_struct->get_field(field_idx); + tolk_assert(!loaded_state->ith_field_was_loaded[field_idx]); + + // note that as opposed to regular loading, lazy loading doesn't return rvect, it fills stack slots (ir_obj) instead + switch (load_info.ith_field_action[field_idx]) { + case LazyStructLoadInfo::LoadField: { + if (StructFieldPtr original_field = original_struct->find_field(hidden_field->name)) { + tolk_assert(hidden_field->declared_type == original_field->declared_type); + std::vector ir_field = ctx.generate_unpack_any(hidden_field->declared_type); + int stack_offset = calc_offset_on_stack(original_struct, original_field->field_idx); + int stack_width = hidden_field->declared_type->get_width_on_stack(); + code.emplace_back(loc, Op::_Let, std::vector(ir_obj.begin() + stack_offset, ir_obj.begin() + stack_offset + stack_width), std::move(ir_field)); + loaded_state->mutate()->on_original_field_loaded(hidden_field); + } else { + tolk_assert(hidden_field->name == "(gap)"); + std::vector ir_gap = ctx.generate_unpack_any(hidden_field->declared_type); + loaded_state->mutate()->on_aside_field_loaded(hidden_field, std::move(ir_gap)); + } + break; + } + case LazyStructLoadInfo::SkipField: { + ctx.generate_skip_any(hidden_field->declared_type); + break; + } + case LazyStructLoadInfo::LazyMatchField: { + StructFieldPtr original_field = original_struct->find_field(hidden_field->name); + tolk_assert(original_field && hidden_field->declared_type == original_field->declared_type); + loaded_state->mutate()->on_original_field_loaded(hidden_field); + break; + } + case LazyStructLoadInfo::SaveImmutableTail: { + std::vector ir_immutable_tail = code.create_tmp_var(TypeDataSlice::create(), loc, "(lazy-tail-slice)"); + code.emplace_back(loc, Op::_Let, ir_immutable_tail, lazy_variable->ir_slice); + loaded_state->mutate()->on_aside_field_loaded(hidden_field, std::move(ir_immutable_tail)); + break; + } + } + } + + // options.assertEndAfterReading is ignored by `lazy`, because tail fields may be skipped, it's okay +} + +std::vector generate_lazy_struct_to_cell(CodeBlob& code, SrcLocation loc, const LazyStructLoadedState* loaded_state, std::vector&& ir_obj, const std::vector& ir_options) { + StructPtr original_struct = loaded_state->original_struct; + StructPtr hidden_struct = loaded_state->hidden_struct; + + std::vector rvect_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); + code.emplace_back(loc, Op::_Call, rvect_builder, std::vector{}, lookup_function("beginCell")); + + PackContext ctx(code, loc, rvect_builder, ir_options); + + if (hidden_struct->opcode.exists()) { + ctx.storeUint(code.create_int(loc, hidden_struct->opcode.pack_prefix, "(struct-prefix)"), hidden_struct->opcode.prefix_len); + } + + for (int field_idx = 0; field_idx < hidden_struct->get_num_fields(); ++field_idx) { + StructFieldPtr hidden_field = hidden_struct->get_field(field_idx); + tolk_assert(loaded_state->ith_field_was_loaded[field_idx]); + + if (StructFieldPtr original_field = original_struct->find_field(hidden_field->name)) { + int stack_offset = calc_offset_on_stack(original_struct, original_field->field_idx); + int stack_width = hidden_field->declared_type->get_width_on_stack(); + std::vector ir_field(ir_obj.begin() + stack_offset, ir_obj.begin() + stack_offset + stack_width); + ctx.generate_pack_any(hidden_field->declared_type, std::move(ir_field)); + } else { + std::vector ir_gap_or_tail = loaded_state->get_ir_loaded_aside_field(hidden_field); + if (hidden_field->declared_type->unwrap_alias()->try_as()) { + ctx.storeSlice(ir_gap_or_tail[0]); + } else { + ctx.generate_pack_any(hidden_field->declared_type, std::move(ir_gap_or_tail)); + } + if (hidden_field->name == "(tail)") { + break; + } + } + } + + std::vector rvect_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(cell)"); + code.emplace_back(loc, Op::_Call, rvect_cell, std::move(rvect_builder), lookup_function("builder.endCell")); + + return rvect_cell; +} + +std::vector generate_lazy_match_for_union(CodeBlob& code, SrcLocation loc, TypePtr union_type, const LazyVariableLoadedState* lazy_variable, const LazyMatchOptions& options) { + tolk_assert(lazy_variable->ir_options.size() == 2); + UnpackContext ctx(code, loc, lazy_variable->ir_slice, lazy_variable->ir_options); + std::vector rvect_match = ctx.generate_lazy_match_any(union_type, options); + + return rvect_match; +} + +std::vector generate_lazy_object_finish_loading(CodeBlob& code, SrcLocation loc, const LazyVariableLoadedState* lazy_variable, std::vector&& ir_obj) { + tolk_assert(lazy_variable->ir_slice.size() == 1); + + // the call to `lazy_var.forceLoadLazyObject()` does not do anything: at the moment of analyzing, + // it had marked all the object as "used", all fields where loaded, and the slice points after the last field; + // so, just return the held slice + static_cast(code); + static_cast(loc); + static_cast(ir_obj); + + return lazy_variable->ir_slice; +} + PackSize estimate_serialization_size(TypePtr any_type) { EstimateContext ctx; return ctx.estimate_any(any_type); diff --git a/tolk/pack-unpack-api.h b/tolk/pack-unpack-api.h index 7737125d40..7b4d02f66a 100644 --- a/tolk/pack-unpack-api.h +++ b/tolk/pack-unpack-api.h @@ -32,4 +32,13 @@ std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation PackSize estimate_serialization_size(TypePtr any_type); std::vector generate_estimate_size_call(CodeBlob& code, SrcLocation loc, TypePtr any_type); +struct LazyStructLoadInfo; +struct LazyStructLoadedState; +struct LazyVariableLoadedState; + +void generate_lazy_struct_from_slice(CodeBlob& code, SrcLocation loc, const LazyVariableLoadedState* lazy_variable, const LazyStructLoadInfo& load_info, const std::vector& ir_obj); +std::vector generate_lazy_struct_to_cell(CodeBlob& code, SrcLocation loc, const LazyStructLoadedState* loaded_state, std::vector&& ir_obj, const std::vector& ir_options); +std::vector generate_lazy_match_for_union(CodeBlob& code, SrcLocation loc, TypePtr union_type, const LazyVariableLoadedState* lazy_variable, const LazyMatchOptions& options); +std::vector generate_lazy_object_finish_loading(CodeBlob& code, SrcLocation loc, const LazyVariableLoadedState* lazy_variable, std::vector&& ir_obj); + } // namespace tolk diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index d139a45266..770ec56b90 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -36,7 +36,8 @@ namespace tolk { - +class LValContext; +std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr target_type = nullptr, LValContext* lval_ctx = nullptr); std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc); std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc); @@ -182,6 +183,34 @@ void UnpackContext::assertEndIfOption() const { } } +void UnpackContext::throwInvalidOpcode() const { + std::vector args_throw = { option_throwIfOpcodeDoesNotMatch }; + Op& op_throw = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_throw), lookup_function("__throw")); + op_throw.set_impure_flag(); +} + +const LazyMatchOptions::MatchBlock* LazyMatchOptions::find_match_block(TypePtr variant) const { + for (const MatchBlock& b : match_blocks) { + if (b.arm_variant->get_type_id() == variant->get_type_id()) { + return &b; + } + } + tolk_assert(false); +} + +void LazyMatchOptions::save_match_result_on_arm_end(CodeBlob& code, SrcLocation loc, const MatchBlock* arm_block, std::vector&& ir_arm_result, const std::vector& ir_match_expr_result) const { + if (!is_statement) { + // if it's `match` expression (not statement), then every arm has a result, assigned to a whole `match` result + ir_arm_result = transition_to_target_type(std::move(ir_arm_result), code, arm_block->block_expr_type, match_expr_type, loc); + code.emplace_back(loc, Op::_Let, ir_match_expr_result, std::move(ir_arm_result)); + } else if (add_return_to_all_arms) { + // if it's `match` statement, even if an arm is an expression, it's void, actually + // moreover, if it's the last statement in a function, add implicit "return" to all match cases to produce IFJMP + code.emplace_back(loc, Op::_Return); + } +} + + // -------------------------------------------- // serializers with pack/unpack/skip/estimate // @@ -638,6 +667,34 @@ struct S_Either final : ISerializer { return ir_result; } + std::vector lazy_match(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc, const LazyMatchOptions& options) const { + for (const LazyMatchOptions::MatchBlock& m : options.match_blocks) { + if (m.arm_variant == nullptr) { // `else => ...` not allowed for Either + // it's not the best place to fire an error, but let it be + throw ParseError(loc, "`else` is unreachable, because this `match` has only two options (0/1 prefixes)"); + } + } + tolk_assert(options.match_blocks.size() == 2); + std::vector ir_result = code.create_tmp_var(options.match_expr_type, loc, "(match-expression)"); + std::vector ir_is_right = ctx->loadUint(1, "(eitherBit)"); + Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_is_right)); + { + code.push_set_cur(if_op.block0); + const LazyMatchOptions::MatchBlock* m_block = options.find_match_block(t_right); + std::vector ith_result = pre_compile_expr(m_block->v_body, code); + options.save_match_result_on_arm_end(code, loc, m_block, std::move(ith_result), ir_result); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + const LazyMatchOptions::MatchBlock* m_block = options.find_match_block(t_left); + std::vector ith_result = pre_compile_expr(m_block->v_body, code); + options.save_match_result_on_arm_end(code, loc, m_block, std::move(ith_result), ir_result); + code.close_pop_cur(loc); + } + return ir_result; + } + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { std::vector ir_is_right = ctx->loadUint(1, "(eitherBit)"); Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_is_right)); @@ -714,12 +771,54 @@ struct S_MultipleConstructors final : ISerializer { } // we're inside last ELSE - std::vector args_throw = { ctx->option_throwIfOpcodeDoesNotMatch }; - Op& op_throw = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_throw), lookup_function("__throw")); - op_throw.set_impure_flag(); + ctx->throwInvalidOpcode(); + for (int j = 0; j < t_union->size(); ++j) { + code.close_pop_cur(loc); // close all outer IFs + } + return ir_result; + } + + std::vector lazy_match(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc, const LazyMatchOptions& options) const { + std::vector opcodes_order_mapping(t_union->size(), -1); + const LazyMatchOptions::MatchBlock* else_block = nullptr; + for (int i = 0; i < static_cast(options.match_blocks.size()); ++i) { + if (options.match_blocks[i].arm_variant) { + int variant_idx = t_union->get_variant_idx(options.match_blocks[i].arm_variant); + tolk_assert(variant_idx != -1); + opcodes_order_mapping[i] = variant_idx; + } else { + tolk_assert(else_block == nullptr); + else_block = &options.match_blocks[i]; + } + } + + FunctionPtr f_tryStripPrefix = lookup_function("slice.tryStripPrefix"); + + std::vector ir_result = code.create_tmp_var(options.match_expr_type, loc, "(match-expression)"); + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); + + for (int i = 0; i < t_union->size(); ++i) { + StructData::PackOpcode opcode = opcodes[opcodes_order_mapping[i]]; + std::vector args = { ctx->ir_slice0, code.create_int(loc, opcode.pack_prefix, "(pack-prefix)"), code.create_int(loc, opcode.prefix_len, "(prefix-len)") }; + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), f_tryStripPrefix); + Op& if_op = code.emplace_back(loc, Op::_If, ir_prefix_eq); + code.push_set_cur(if_op.block0); + std::vector ith_result = pre_compile_expr(options.match_blocks[i].v_body, code); + options.save_match_result_on_arm_end(code, loc, &options.match_blocks[i], std::move(ith_result), ir_result); + code.close_pop_cur(loc); + code.push_set_cur(if_op.block1); // open ELSE + } + + if (else_block) { + std::vector else_result = pre_compile_expr(else_block->v_body, code); + options.save_match_result_on_arm_end(code, loc, else_block, std::move(else_result), ir_result); + } else { + ctx->throwInvalidOpcode(); + } for (int j = 0; j < t_union->size(); ++j) { code.close_pop_cur(loc); // close all outer IFs } + return ir_result; } @@ -739,9 +838,7 @@ struct S_MultipleConstructors final : ISerializer { } // we're inside last ELSE - std::vector args_throw = { ctx->option_throwIfOpcodeDoesNotMatch }; - Op& op_throw = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_throw), lookup_function("__throw")); - op_throw.set_impure_flag(); + ctx->throwInvalidOpcode(); for (int j = 0; j < t_union->size(); ++j) { code.close_pop_cur(loc); // close all outer IFs } @@ -842,6 +939,49 @@ struct S_CustomStruct final : ISerializer { return ir_struct; } + std::vector lazy_match(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc, const LazyMatchOptions& options) const { + const LazyMatchOptions::MatchBlock* when_block = nullptr; // Point => ... + const LazyMatchOptions::MatchBlock* else_block = nullptr; // else => ... + for (const LazyMatchOptions::MatchBlock& match_block : options.match_blocks) { + if (match_block.arm_variant) { + tolk_assert(match_block.arm_variant->equal_to(TypeDataStruct::create(struct_ref))); + when_block = &match_block; + } else { + else_block = &match_block; + } + } + + std::vector ir_result = code.create_tmp_var(options.match_expr_type, loc, "(match-expression)"); + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); + + StructData::PackOpcode opcode = struct_ref->opcode; + if (opcode.exists()) { // it's `match` over a struct (makes sense for a struct with prefix and `else` branch) + std::vector args = { ctx->ir_slice0, code.create_int(loc, opcode.pack_prefix, "(pack-prefix)"), code.create_int(loc, opcode.prefix_len, "(prefix-len)") }; + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), lookup_function("slice.tryStripPrefix")); + } else { + code.emplace_back(loc, Op::_Let, ir_prefix_eq, std::vector{code.create_int(loc, -1, "(true)")}); + } + Op& if_op = code.emplace_back(loc, Op::_If, ir_prefix_eq); + { + code.push_set_cur(if_op.block0); + std::vector when_result = pre_compile_expr(when_block->v_body, code); + options.save_match_result_on_arm_end(code, loc, when_block, std::move(when_result), ir_result); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + if (else_block) { + std::vector else_result = pre_compile_expr(else_block->v_body, code); + options.save_match_result_on_arm_end(code, loc, else_block, std::move(else_result), ir_result); + } else { + ctx->throwInvalidOpcode(); + } + code.close_pop_cur(loc); + } + + return ir_result; + } + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { if (struct_ref->opcode.exists() && ctx->get_prefix_mode() == PrefixReadMode::LoadAndCheck) { ctx->loadAndCheckOpcode(struct_ref->opcode); @@ -1048,6 +1188,20 @@ void UnpackContext::generate_skip_any(TypePtr any_type, PrefixReadMode prefix_mo this->prefix_mode = backup; } +std::vector UnpackContext::generate_lazy_match_any(TypePtr any_type, const LazyMatchOptions& options) const { + std::unique_ptr serializer = get_serializer_for_type(any_type); + if (auto* s = dynamic_cast(serializer.get())) { + return s->lazy_match(this, code, loc, options); + } + if (auto* s = dynamic_cast(serializer.get())) { + return s->lazy_match(this, code, loc, options); + } + if (auto* s = dynamic_cast(serializer.get())) { + return s->lazy_match(this, code, loc, options); + } + tolk_assert(false); +} + PackSize EstimateContext::estimate_any(TypePtr any_type, PrefixEstimateMode prefix_mode) const { PrefixEstimateMode backup = this->prefix_mode; this->prefix_mode = prefix_mode; diff --git a/tolk/pack-unpack-serializers.h b/tolk/pack-unpack-serializers.h index 06163153ba..0e664b9a11 100644 --- a/tolk/pack-unpack-serializers.h +++ b/tolk/pack-unpack-serializers.h @@ -91,6 +91,22 @@ enum class PrefixReadMode { DoNothingAlreadyLoaded, }; +struct LazyMatchOptions { + struct MatchBlock { + TypePtr arm_variant; // left of `V => ...`; nullptr for `else => ...` + AnyExprV v_body; // right of `V => ...` + TypePtr block_expr_type; // for match expression, if `V => expr`, it's expr's inferred_type + }; + + TypePtr match_expr_type; // type of `match` expression, `void` for statement + bool is_statement; // it's `match` statement, not expression, so it does not return any result + bool add_return_to_all_arms; // it's the last statement in a function, add "return" to its cases for better Fift code + std::vector match_blocks; + + const MatchBlock* find_match_block(TypePtr variant) const; + void save_match_result_on_arm_end(CodeBlob& code, SrcLocation loc, const MatchBlock* arm_block, std::vector&& ir_arm_result, const std::vector& ir_match_expr_result) const; +}; + class UnpackContext { CodeBlob& code; SrcLocation loc; @@ -115,9 +131,11 @@ class UnpackContext { void skipBits(int len) const; void skipBits_var(var_idx_t ir_len) const; void assertEndIfOption() const; + void throwInvalidOpcode() const; std::vector generate_unpack_any(TypePtr any_type, PrefixReadMode prefix_mode = PrefixReadMode::LoadAndCheck) const; void generate_skip_any(TypePtr any_type, PrefixReadMode prefix_mode = PrefixReadMode::LoadAndCheck) const; + std::vector generate_lazy_match_any(TypePtr any_type, const LazyMatchOptions& options) const; }; diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index f97d7a6898..a6d84a30c8 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -111,10 +111,10 @@ static int calc_offset_on_stack(const TypeDataTensor* t_tensor, int index_at) { return stack_offset; } -static int calc_offset_on_stack(const TypeDataStruct* t_struct, int field_idx) { +static int calc_offset_on_stack(StructPtr struct_ref, int field_idx) { int stack_offset = 0; for (int i = 0; i < field_idx; ++i) { - stack_offset += t_struct->struct_ref->fields[i]->declared_type->get_width_on_stack(); + stack_offset += struct_ref->get_field(i)->declared_type->get_width_on_stack(); } return stack_offset; } @@ -179,7 +179,7 @@ class LValContext { stack_offset = calc_offset_on_stack(t_tensor, index_at); } else if (const TypeDataStruct* t_struct = obj_type->try_as()) { stack_width = t_struct->struct_ref->get_field(index_at)->declared_type->get_width_on_stack(); - stack_offset = calc_offset_on_stack(t_struct, index_at); + stack_offset = calc_offset_on_stack(t_struct->struct_ref, index_at); } else { tolk_assert(false); } @@ -363,6 +363,28 @@ class ClearStateAfterInlineInPlace final : public ASTVisitorFunctionBody { } }; + +// CodeBlob has a mapping [st => ptr] +const LazyVariableLoadedState* CodeBlob::get_lazy_variable(LocalVarPtr var_ref) const { + for (const LazyVarRefAtCodegen& stored : lazy_variables) { + if (stored.var_ref == var_ref) { + return stored.var_state; + } + } + return nullptr; +} + +// detect `st` by vertex "st" +const LazyVariableLoadedState* CodeBlob::get_lazy_variable(AnyExprV v) const { + if (auto as_ref = v->try_as()) { + if (LocalVarPtr var_ref = as_ref->sym->try_as()) { + return get_lazy_variable(var_ref); + } + } + return nullptr; +} + + // given `{some_expr}!`, return some_expr static AnyExprV unwrap_not_null_operator(AnyExprV v) { while (auto v_notnull = v->try_as()) { @@ -567,11 +589,21 @@ static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcL return rvect; } -static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob& code, SrcLocation loc, const std::vector>& vars_per_arg, FunctionPtr called_f) { - if (called_f->is_instantiation_of_generic_function()) { +static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob& code, V v_call, const std::vector>& vars_per_arg) { + SrcLocation loc = v_call->loc; + FunctionPtr called_f = v_call->fun_maybe; + + if (called_f->is_method() && called_f->is_instantiation_of_generic_function()) { std::string_view f_name = called_f->base_fun_ref->name; TypePtr typeT = called_f->substitutedTs->typeT_at(0); + const LazyVariableLoadedState* lazy_variable = v_call->dot_obj_is_self ? code.get_lazy_variable(v_call->get_self_obj()) : nullptr; + + if (f_name == "T.toCell" && lazy_variable && lazy_variable->is_struct()) { + // in: object Lazy (partially loaded), out: Cell + std::vector ir_obj = vars_per_arg[0]; // = lazy_var_ref->ir_idx + return generate_lazy_struct_to_cell(code, loc, &lazy_variable->loaded_state, std::move(ir_obj), vars_per_arg[1]); + } if (f_name == "T.toCell") { // in: object T, out: Cell (just a cell, wrapped) std::vector ir_obj = vars_per_arg[0]; @@ -615,6 +647,19 @@ static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob if (f_name == "T.estimatePackSize") { return generate_estimate_size_call(code, loc, typeT); } + if (f_name == "T.forceLoadLazyObject") { + // in: object T, out: slice (same slice that a lazy variable holds, after loading/skipping all its fields) + if (!lazy_variable) { + fire(code.fun_ref, v_call->loc, "this method is applicable to lazy variables only"); + } + std::vector ir_obj = vars_per_arg[0]; + return generate_lazy_object_finish_loading(code, loc, lazy_variable, std::move(ir_obj)); + } + } + + if (called_f->is_instantiation_of_generic_function()) { + std::string_view f_name = called_f->base_fun_ref->name; + TypePtr typeT = called_f->substitutedTs->typeT_at(0); if (f_name == "createMessage") { std::vector ir_msg_params = vars_per_arg[0]; @@ -635,51 +680,59 @@ static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob tolk_assert(false); } -static std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, V v_call, const std::vector>& vars_per_arg) { - FunctionPtr fun_ref = v_call->fun_maybe; - tolk_assert(vars_per_arg.size() == fun_ref->parameters.size()); - for (int i = 0; i < fun_ref->get_num_params(); ++i) { - const LocalVarData& param_i = fun_ref->get_param(i); +std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, SrcLocation loc, FunctionPtr f_inlined, AnyExprV self_obj, bool is_before_immediate_return, const std::vector>& vars_per_arg) { + tolk_assert(vars_per_arg.size() == f_inlined->parameters.size()); + for (int i = 0; i < f_inlined->get_num_params(); ++i) { + const LocalVarData& param_i = f_inlined->get_param(i); if (!param_i.is_used_as_lval() && !param_i.is_mutate_parameter()) { // if param used for reading only, pass the same ir_idx as for an argument // it decreases number of tmp variables and leads to better optimizations // (being honest, it's quite strange that copy+LET may lead to more stack permutations) param_i.mutate()->assign_ir_idx(std::vector(vars_per_arg[i])); } else { - std::vector ith_param = code.create_var(param_i.declared_type, v_call->loc, param_i.name); - code.emplace_back(v_call->loc, Op::_Let, ith_param, vars_per_arg[i]); + std::vector ith_param = code.create_var(param_i.declared_type, loc, param_i.name); + code.emplace_back(loc, Op::_Let, ith_param, vars_per_arg[i]); param_i.mutate()->assign_ir_idx(std::move(ith_param)); } } - std::vector rvect_call = code.create_tmp_var(ret_type, v_call->loc, "(inlined-return)"); + std::vector rvect_call = code.create_tmp_var(ret_type, loc, "(inlined-return)"); std::vector* backup_outer_inline = code.inline_rvect_out; FunctionPtr backup_cur_fun = code.fun_ref; bool backup_inline_before_return = code.inlining_before_immediate_return; + auto backup_lazy_variables = code.lazy_variables; code.inline_rvect_out = &rvect_call; - code.inlining_before_immediate_return = stmt_before_immediate_return == v_call; - code.fun_ref = fun_ref; + code.inlining_before_immediate_return = is_before_immediate_return; + code.fun_ref = f_inlined; + // specially handle `point.getX()` if point is a lazy var: to make `self.toCell()` work and `self.x` asserted; + // (only methods preserve lazy, `getXOf(point)` does not, though theoretically can be done) + const LazyVariableLoadedState* lazy_receiver = self_obj ? code.get_lazy_variable(self_obj) : nullptr; + if (lazy_receiver) { + LocalVarPtr self_var_ref = &f_inlined->parameters[0]; // `self` becomes lazy while inlining + code.lazy_variables.emplace_back(self_var_ref, lazy_receiver); // (points to the same slice, immutable tail, etc.) + } - auto v_ast_root = fun_ref->ast_root->as(); + auto v_ast_root = f_inlined->ast_root->as(); auto v_block = v_ast_root->get_body()->as(); process_any_statement(v_block, code); - if (fun_ref->has_mutate_params() && fun_ref->inferred_return_type == TypeDataVoid::create()) { + if (f_inlined->has_mutate_params() && f_inlined->inferred_return_type == TypeDataVoid::create()) { std::vector mutated_vars; - for (const LocalVarData& p_sym: fun_ref->parameters) { + for (const LocalVarData& p_sym: f_inlined->parameters) { if (p_sym.is_mutate_parameter()) { mutated_vars.insert(mutated_vars.end(), p_sym.ir_idx.begin(), p_sym.ir_idx.end()); } } - code.emplace_back(v_call->loc, Op::_Let, rvect_call, std::move(mutated_vars)); + code.emplace_back(loc, Op::_Let, rvect_call, std::move(mutated_vars)); } ClearStateAfterInlineInPlace visitor; - visitor.start_visiting_function(fun_ref, v_ast_root); + visitor.start_visiting_function(f_inlined, v_ast_root); code.fun_ref = backup_cur_fun; code.inline_rvect_out = backup_outer_inline; code.inlining_before_immediate_return = backup_inline_before_return; + code.lazy_variables = std::move(backup_lazy_variables); return rvect_call; } @@ -1283,6 +1336,58 @@ static std::vector process_not_null_operator(V return transition_to_target_type(std::move(rvect), code, target_type, v); } +static std::vector process_lazy_operator(V v, CodeBlob& code, TypePtr target_type) { + // `lazy Storage.fromSlice(s)` does not load anything here, it only saves a slice for future loads; + // "future loads" are special auxiliary AST vertices "load x" that were inserted in pipe-lazy-load-insertions.cpp + auto v_call = v->get_expr()->try_as(); + tolk_assert(v_call && v_call->fun_maybe); + + FunctionPtr called_f = v_call->fun_maybe; + if (called_f->is_code_function()) { // `lazy loadStorage()` is allowed, it contains just `return ...`, inline it here + auto f_body = called_f->ast_root->as()->get_body()->as(); + tolk_assert(f_body->size() == 1 && f_body->get_item(0)->kind == ast_return_statement); + auto f_returns = f_body->get_item(0)->as(); + v_call = f_returns->get_return_value()->try_as(); + tolk_assert(v_call && v_call->fun_maybe && v_call->fun_maybe->is_builtin_function()); + called_f = v_call->fun_maybe; + } + + // only predefined built-in functions are allowed for lazy loading + tolk_assert(called_f->is_builtin_function() && called_f->is_instantiation_of_generic_function()); + std::string_view f_name = called_f->base_fun_ref->name; + std::vector ir_slice = code.create_var(TypeDataSlice::create(), v->loc, "lazyS"); + bool has_passed_options = false; + if (f_name == "T.fromSlice") { + std::vector passed_slice = pre_compile_expr(v_call->get_arg(0)->get_expr(), code); + code.emplace_back(v->loc, Op::_Let, ir_slice, std::move(passed_slice)); + has_passed_options = v_call->get_num_args() == 2; + } else if (f_name == "T.fromCell") { + std::vector ir_cell = pre_compile_expr(v_call->get_arg(0)->get_expr(), code); + code.emplace_back(v->loc, Op::_Call, ir_slice, ir_cell, lookup_function("cell.beginParse")); + has_passed_options = v_call->get_num_args() == 2; + } else if (f_name == "Cell.load") { + std::vector ir_cell = pre_compile_expr(v_call->get_callee()->try_as()->get_obj(), code); + code.emplace_back(v->loc, Op::_Call, ir_slice, ir_cell, lookup_function("cell.beginParse")); + has_passed_options = v_call->get_num_args() == 1; + } else { + tolk_assert(false); + } + + // on `var p = lazy Point.fromSlice(s, options)`, save s and options (lazy_variable) + AnyExprV v_options = has_passed_options ? v_call->get_arg(v_call->get_num_args() - 1)->get_expr() : called_f->parameters.back().default_value; + std::vector ir_options = pre_compile_expr(v_options, code, called_f->parameters[1].declared_type); + const LazyVariableLoadedState* lazy_variable = new LazyVariableLoadedState(v->dest_var_ref->declared_type, std::move(ir_slice), std::move(ir_options)); + code.lazy_variables.emplace_back(v->dest_var_ref, lazy_variable); + + // initially, all contents of `p` is filled by nulls, but before `p.x` or any other field usages, + // they will be loaded by separate AST aux vertices; + // same for unions: `val msg = lazy MyMsgUnion`, msg is N+1 nulls, but next lazy `match` will transition slots, + // which will be filled by loads + std::vector ir_null = gen_op_call(code, TypeDataNullLiteral::create(), v->loc, {}, lookup_function("__null"), "(init-null)"); + std::vector ir_initial_nulls(v->dest_var_ref->ir_idx.size(), ir_null[0]); + return transition_to_target_type(std::move(ir_initial_nulls), code, target_type, v); +} + static std::vector process_match_expression(V v, CodeBlob& code, TypePtr target_type) { TypePtr lhs_type = v->get_subject()->inferred_type->unwrap_alias(); @@ -1297,10 +1402,18 @@ static std::vector process_match_expression(V v // it's either `match` by type (all arms patterns are types) or `match` by expression bool is_match_by_type = v->get_arm(0)->pattern_kind == MatchArmKind::exact_type; + bool last_else_branch = v->get_arm(n_arms - 1)->pattern_kind == MatchArmKind::else_branch; // detect whether `match` is exhaustive bool is_exhaustive = is_match_by_type // match by type covers all cases, checked earlier || !v->is_statement() // match by expression is guaranteely exhaustive, checked earlier - || v->get_arm(n_arms - 1)->pattern_kind == MatchArmKind::else_branch; + || last_else_branch; + + // `else` is not allowed in `match` by type; this was not fired at type checking, + // because it might turned out to be a lazy match, where `else` is allowed; + // if we are here, it's not a lazy match, it's a regular one (the lazy one is handled specially, in aux vertex) + if (is_match_by_type && last_else_branch) { + throw ParseError(v->get_arm(n_arms - 1)->loc, "`else` is not allowed in `match` by type; you should cover all possible types"); + } // example 1 (exhaustive): `match (v) { int => ... slice => ... builder => ... }` // construct nested IFs: IF is int { ... } ELSE { IF is slice { ... } ELSE { ... } } @@ -1374,9 +1487,14 @@ static std::vector process_dot_access(V v, CodeBlob& return lval_ir_idx; } } + // handle `lazyPoint.x`, assert that slot for "x" is loaded (ensure lazy-loading correctness); + // same for `val msg = lazy MyMsgUnion; match(...) msg.field` inside a specific variant (struct_ref) + if (const LazyVariableLoadedState* lazy_variable = code.get_lazy_variable(v->get_obj())) { + lazy_variable->assert_field_loaded(t_struct->struct_ref, field_ref); + } std::vector lhs_vars = pre_compile_expr(v->get_obj(), code, nullptr, lval_ctx); int stack_width = field_ref->declared_type->get_width_on_stack(); - int stack_offset = calc_offset_on_stack(t_struct, field_ref->field_idx); + int stack_offset = calc_offset_on_stack(t_struct->struct_ref, field_ref->field_idx); std::vector rvect{lhs_vars.begin() + stack_offset, lhs_vars.begin() + stack_offset + stack_width}; // an object field might be smart cast at this point, for example we're in `if (user.t != null)` // it means that we must drop the null flag (if `user.t` is a tensor), or maybe perform other stack transformations @@ -1544,9 +1662,9 @@ static std::vector process_function_call(V v, Code } std::vector rvect_call; if (fun_ref->is_compile_time_special_gen()) { - rvect_call = gen_compile_time_code_instead_of_fun_call(code, v->loc, vars_per_arg, fun_ref); + rvect_call = gen_compile_time_code_instead_of_fun_call(code, v, vars_per_arg); } else if (fun_ref->is_inlined_in_place() && fun_ref->is_code_function()) { - rvect_call = gen_inline_fun_call_in_place(code, op_call_type, v, vars_per_arg); + rvect_call = gen_inline_fun_call_in_place(code, op_call_type, v->loc, v->fun_maybe, v->get_self_obj(), v == stmt_before_immediate_return, vars_per_arg); } else { rvect_call = gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)", arg_order_already_equals_asm); } @@ -1587,12 +1705,18 @@ static std::vector process_function_call(V v, Code } static std::vector process_braced_expression(V v, CodeBlob& code, TypePtr target_type) { - // `{ ... }` used as an expression can not return a value currently (there is no syntax in a language) - // that's why it can appear only in special places, and its usage correctness has been checked - tolk_assert(v->inferred_type == TypeDataVoid::create() || v->inferred_type == TypeDataNever::create()); - process_any_statement(v->get_block_statement(), code); - static_cast(target_type); - return {}; + // generally, `{ ... }` is a block statement not returning a value; it's used to represent `match` braced arms; + // unless it's a special vertex "braced expression" (currently, only `match` arms) + std::vector implicit_rvect; + for (AnyV item : v->get_block_statement()->get_items()) { + if (auto v_return = item->try_as()) { + tolk_assert(implicit_rvect.empty()); + implicit_rvect = pre_compile_expr(v_return->get_expr(), code); + } else { + process_any_statement(item, code); + } + } + return transition_to_target_type(std::move(implicit_rvect), code, target_type, v); } static std::vector process_tensor(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { @@ -1780,6 +1904,67 @@ static std::vector process_artificial_aux_vertex(V(v->aux_data)) { + const LazyVariableLoadedState* lazy_variable = code.get_lazy_variable(data->var_ref); + tolk_assert(lazy_variable); + + std::vector ir_obj = data->var_ref->ir_idx; // loading will update stack slots of `p` + TypePtr t_orig = data->var_ref->declared_type; + + if (data->field_ref) { // extract a field from a whole lazy variable + tolk_assert(lazy_variable->is_struct()); + int stack_offset = calc_offset_on_stack(lazy_variable->loaded_state.original_struct, data->field_ref->field_idx); + int stack_width = data->field_ref->declared_type->get_width_on_stack(); + ir_obj = std::vector(ir_obj.begin() + stack_offset, ir_obj.begin() + stack_offset + stack_width); + t_orig = data->field_ref->declared_type; + } + + if (data->union_variant) { // extract a variant from a union (a union variable or a union field of a struct) + ir_obj = transition_to_target_type(std::move(ir_obj), code, t_orig, data->union_variant, v->loc); + } + + // `load_info` contains instructions to skip, load, save tail, etc.; + // it generates LETs to ir_obj, so stack slots of lazy_variable will contain loaded data + generate_lazy_struct_from_slice(code, v->loc, lazy_variable, data->load_info, ir_obj); + return transition_to_target_type({}, code, target_type, v); + } + + // aux "match(lazyUnion)" / aux "match(obj.lastUnionField)" + if (const auto* data = dynamic_cast(v->aux_data)) { + V v_match = v->get_wrapped_expr()->as(); + pre_compile_expr(v_match->get_subject(), code, nullptr); + + const LazyVariableLoadedState* lazy_variable = code.get_lazy_variable(data->var_ref); + tolk_assert(lazy_variable); + TypePtr t_union = data->field_ref ? data->field_ref->declared_type : data->var_ref->declared_type; + + std::vector match_blocks; + match_blocks.reserve(v_match->get_arms_count()); + for (int i = 0; i < v_match->get_arms_count(); ++i) { + auto v_arm = v_match->get_arm(i); + TypePtr arm_variant = nullptr; + if (v_arm->pattern_kind == MatchArmKind::exact_type) { + arm_variant = v_arm->pattern_type_node->resolved_type->unwrap_alias(); + } else { + tolk_assert(v_arm->pattern_kind == MatchArmKind::else_branch); // `else` allowed in a lazy match + } + match_blocks.emplace_back(LazyMatchOptions::MatchBlock{arm_variant, v_arm->get_body(), v_arm->get_body()->inferred_type}); + } + + LazyMatchOptions options = { + .match_expr_type = v->inferred_type, + .is_statement = v_match->is_statement(), + .add_return_to_all_arms = v == stmt_before_immediate_return, + .match_blocks = std::move(match_blocks), + }; + + // it will generate match by a slice prefix, and for each `match` arm, invoke pre_compile_expr(), + // which contains "aux load" particularly + std::vector ir_match = generate_lazy_match_for_union(code, v->loc, t_union, lazy_variable, options); + return transition_to_target_type(std::move(ir_match), code, target_type, v); + } + tolk_assert(false); } @@ -1803,6 +1988,8 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ return process_is_type_operator(v->as(), code, target_type); case ast_not_null_operator: return process_not_null_operator(v->as(), code, target_type, lval_ctx); + case ast_lazy_operator: + return process_lazy_operator(v->as(), code, target_type); case ast_match_expression: return process_match_expression(v->as(), code, target_type); case ast_dot_access: diff --git a/tolk/pipe-calc-rvalue-lvalue.cpp b/tolk/pipe-calc-rvalue-lvalue.cpp index d451a7c601..dc269a342e 100644 --- a/tolk/pipe-calc-rvalue-lvalue.cpp +++ b/tolk/pipe-calc-rvalue-lvalue.cpp @@ -88,6 +88,13 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { restore_state(saved); } + void visit(V v) override { + mark_vertex(v); + MarkingState saved = enter_rvalue_if_none(); + parent::visit(v); + restore_state(saved); + } + void visit(V v) override { mark_vertex(v); MarkingState saved = enter_rvalue_if_none(); @@ -213,6 +220,13 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { parent::visit(v->get_expr()); // leave lvalue state unchanged, for `mutate x!` both `x!` and `x` are lvalue } + void visit(V v) override { + mark_vertex(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + void visit(V v) override { mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index ba15825592..b47513ee4d 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -587,7 +587,9 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { fire(cur_f, v_arm->loc, "duplicated `else` branch"); } if (has_type_arm) { - fire(cur_f, v_arm->loc, "`else` is not allowed in `match` by type; you should cover all possible types"); + // `else` is not allowed in `match` by type, but we don't fire an error here, + // because it might turn out to be a lazy `match`, where `else` is allowed; + // if it's not lazy, an error is fired later } has_else_arm = true; } diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp index 075e999820..cb80676281 100644 --- a/tolk/pipe-check-rvalue-lvalue.cpp +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -125,6 +125,13 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { parent::visit(v->get_expr()); } + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(cur_f, v, "lazy expression"); + } + parent::visit(v->get_expr()); + } + void visit(V v) override { if (v->is_lvalue) { fire_error_cannot_be_used_as_lvalue(cur_f, v, "literal"); diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp index ca82cd3fbd..f0bd7c5a36 100644 --- a/tolk/pipe-generate-fif-output.cpp +++ b/tolk/pipe-generate-fif-output.cpp @@ -47,7 +47,7 @@ static void generate_output_func(FunctionPtr fun_ref) { CodeBlob* code = std::get(fun_ref->body)->code; if (G.is_verbosity(3)) { - code->print(std::cerr, 9); + code->print(std::cerr, 0); } code->prune_unreachable_code(); if (G.is_verbosity(5)) { @@ -73,7 +73,7 @@ static void generate_output_func(FunctionPtr fun_ref) { } code->mark_noreturn(); if (G.is_verbosity(3)) { - code->print(std::cerr, 15); + // code->print(std::cerr, 15); } if (G.is_verbosity(2)) { std::cerr << "\n---------- resulting code for " << fun_ref->name << " -------------\n"; diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 0345ac0aa1..2d396762cc 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -376,10 +376,12 @@ class InferTypesAndCallsAndFieldsVisitor final { return infer_is_type_operator(v->as(), std::move(flow), used_as_condition); case ast_not_null_operator: return infer_not_null_operator(v->as(), std::move(flow), used_as_condition); + case ast_lazy_operator: + return infer_lazy_operator(v->as(), std::move(flow), used_as_condition); case ast_parenthesized_expression: return infer_parenthesized(v->as(), std::move(flow), used_as_condition, hint); case ast_braced_expression: - return infer_braced_expression(v->as(), std::move(flow), used_as_condition); + return infer_braced_expression(v->as(), std::move(flow), used_as_condition, hint); case ast_reference: return infer_reference(v->as(), std::move(flow), used_as_condition, hint); case ast_dot_access: @@ -830,16 +832,34 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(after_expr.out_flow), true); } + ExprFlow infer_lazy_operator(V v, FlowContext&& flow, bool used_as_condition) { + ExprFlow lazy_expr = infer_any_expr(v->get_expr(), std::move(flow), used_as_condition); + assign_inferred_type(v, v->get_expr()); // there is no Lazy, so `lazy expr` is just typeof expr + return lazy_expr; + } + ExprFlow infer_parenthesized(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), used_as_condition, hint); assign_inferred_type(v, v->get_expr()); return after_expr; } - ExprFlow infer_braced_expression(V v, FlowContext&& flow, bool used_as_condition) { - // `{ ... }` used as an expression can not return a value currently (there is no syntax in a language) - flow = process_any_statement(v->get_block_statement(), std::move(flow)); - assign_inferred_type(v, flow.is_unreachable() ? TypeDataNever::create() : TypeDataVoid::create()); + ExprFlow infer_braced_expression(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + // generally, `{ ... }` is a block statement not returning a value; it's used to represent `match` braced arms; + // unless it's a special vertex (used to represent a non-braced `match` arm) + AnyExprV implicit_result = nullptr; + for (AnyV item : v->get_block_statement()->get_items()) { + if (auto v_return = item->try_as()) { + tolk_assert(implicit_result == nullptr); + implicit_result = v_return; + flow = infer_any_expr(v_return->get_expr(), std::move(flow), false, hint).out_flow; + assign_inferred_type(v_return, v_return->get_expr()); + } else { + flow = process_any_statement(item, std::move(flow)); + } + } + + assign_inferred_type(v, flow.is_unreachable() ? TypeDataNever::create() : implicit_result ? implicit_result->inferred_type : TypeDataVoid::create()); return ExprFlow(std::move(flow), used_as_condition); } diff --git a/tolk/pipe-lazy-load-insertions.cpp b/tolk/pipe-lazy-load-insertions.cpp new file mode 100644 index 0000000000..e7510023a2 --- /dev/null +++ b/tolk/pipe-lazy-load-insertions.cpp @@ -0,0 +1,1032 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "lazy-helpers.h" +#include "ast.h" +#include "ast-aux-data.h" +#include "ast-visitor.h" +#include "ast-replacer.h" +#include "type-system.h" +#include "smart-casts-cfg.h" +#include "pack-unpack-api.h" + +namespace tolk { + +/* + * This pipe finds `lazy` operators and inserts loading points to load only required fields just before being used. + * It happens after type inferring/checking. While inferring, `lazy expr` was inferred just as expr. + * There is no dedicated `Lazy` type in the type system. All the magic of laziness is calculated here. + * + * This is the second version of the algorithm. The first one (which didn't reach production) attempted to + * calculate precise loading locations right before every field usage. + * E.g., `assert(obj.f1 > 0); assert(obj.f2 > 0)` — it detected "load f1" + assert1 + "load f2" + assert2. + * However, it turned out to increase gas consumption. In practice, when a structure has only a few fields (99% cases), + * it's better to load them ALL AT ONCE rather than on demand: this results in fewer stack manipulations. + * + * `lazy` is not about lazy/partial loading, but also about partial updating. + * As opposed to non-lazy `var st = Storage.fromCell(c); st.xxx = yyy; st.toCell()`, which loads and writes all fields, + * partial updating detects immutable portions of a struct, saves them separately, and reuses on toCell(). + * + * All in all, the algorithm focuses on: + * - Identifying which fields are used for a lazy variable. + * - Loading all required fields at once (and skipping unused ones). + * - Doing lazy matching for unions, to avoid constructing heavy union types on the stack. + * - Calculating and loading used fields at the right place for every union variant. + * - Analyzing modification and gathering immutable portions at loading to be reused on saving. + * + * Example "lazy union": + * > val msg = lazy MyMessage.fromSlice(msgBody); // doesn't construct a union, actually + * > match (msg) { // not a match by type, but a lazy match by a slice prefix + * > CounterReset => { + * > assert(senderAddress == storage.owner) throw 403; + * > // <-- here "load msg.initial" is inserted + * > storage.counter = msg.initial; + * > } + * > } + * + * Example "skip unused": + * > get seqno() { + * > val storage = lazy Storage.fromCell(contract.getData()); + * > // <-- here "skip all fields before seqno, load seqno" is inserted + * > return storage.seqno; + * > } + * + * Example "nesting into try/if": + * > val st = lazy Storage.fromSlice(s); + * > try { + * > // <-- here "load necessary fields" is inserted, if they are used only in `try` body + * > st.someField + * > } + * + * Example "bubbling to closest (lca) statement": + * > val st = lazy Storage.fromSlice(s); + * > // <-- here "load necessary fields" is inserted, because they are used both in `try` and after + * > try { ... st.someField } catch {} + * > st.anotherField + * + * Example "modification and immutable tail": + * > val st = lazy loadStorage(); + * > // <-- here "load f1 f2, save immutable tail, load rest" is inserted + * > ... read all fields + * > st.f2 = newValue; // only f2 is modified, others are only read + * > contract.setData(st.toCell()); // st.toCell() writes f1 f2 and immutable tail + * + * Example "modification and immutable gap": + * > struct Storage { a: int32; b: int32; c: int32; seqno: int32; } + * > var st = lazy loadStorage(); + * > // <-- here "load 96 bits, load seqno" is inserted (note: "abc" are grouped into "96 bits") + * > st.seqno += 1; // only seqno is accessed, "abc" not, that's why they are grouped + * > st.toCell(); // writes 96 bits (grouped "abc") and seqno + * + * Implementation: "original struct" and "hidden struct". + * To group fields for loading/skipping, for every lazy variable, "hidden struct" is created, containing: + * - fields from original struct that are used + * - gaps and artificial fields that are not used, to match binary representation + * + * Example 1: + * | struct Point { x: int8, y: int8 } | hidden: struct lazyPoint { gap: bits8; y: int8 } + * | val p = lazy Point | p is initially `null null` on a stack + * | <-- "skip 8 bits, load y" | field gap skipped, y loaded AND mapped onto a stack to match p.y + * | p.y | p is now `null yValue` + * + * Example 2: + * | struct St { a,b,c; seqno; ... } | hidden: struct lazyStorage { gap: bits96; seqno: int32; tail: slice } + * | val st = lazy St | st is initially `null null null null` + * | <-- "load 96 bits, seqno, save tail" | field gap loaded, seqno loaded, tail (rest fields) saved + * | st.seqno += 1 | st is now `null null null seqno`, "gap" and "tail" kept aside st's ir_idx + * | st.toCell() | writes gap (96 bits, grouped "abc") + modified seqno + storeSlice tail + * + * In a similar way, it works for unions: `lazy UnionType` is represented exactly as `UnionType` on a stack, + * that's why type transitions and methods inlining work natively (when transforming AST to Ops). + * But for each variant, its own "lazyVariant" hidden_struct is created. Used fields are loaded and placed + * into correct placed on a stack, gaps are skipped or placed aside. + * + * Some highlights and considerations: + * - `lazy A.fromSlice(s)` does NOT read from slice immediately; instead, it saves the slice pointer and reads on demand + * (at "loading points" inserted by the compiler in the current pipe). + * - Options can be passed: `lazy A.fromSlice(s, {...})`, but `assertEndAfterReading` is ignored, it doesn't make sense + * (because fields are read later, only required ones, and the last is preloaded rather than loaded). + * There is a special method `a.forceLoadLazyObject()`, can be used inside `match` to load the variant fully. + * - The compiler detects which properties are accessed and inserts "load x y z" as close to the first usage as possible; + * it does NOT split loading into multiple instructions (first x y, then z somewhere below): it has a negative effect. + * - Inlined methods preserve laziness (e.g. `point.getX()` / `storage.save()`): the compiler analyzes the bodies of + * those methods to detect usages of `self`, and marks used fields to be loaded in advance. + * - Perfectly works for unions if a union is used only in `match`; then a union is not even created on a stack: + * instead, this `match` becomes lazy and works but cutting a slice prefix. + * - Effective toCell(): the compiler tracks which fields are modified and reuses an immutable tail slice on writing. + * - Has optimization "to reach a ref, no need to load preceding bits". + * E.g. `struct St { v: int32; content: cell; }` and `st = lazy St; st.content` does only LDREF, no "skip 32 bits". + * - The last requested field is preloaded, not loaded. + * + * There are some drawbacks (probable possible enhancements): + * - When a union is used in some way except match, lazy is inapplicable (compilation error). + * - When a union has some primitives besides structures, lazy is inapplicable (for `T?`, particularly). + * Possible enhancement: handle unions where structs are mixed with primitives. + * - When a struct has a union field, it can be lazily matched only if it's the last. + * Possible enhancement: allow lazy match for union fields in the middle. + * - When `match` is used inside complex expressions, it's not lazy for safety. + * Possible enhancement: `cond && match(...)` is unsafe to be lazy, but `1 + match(...)` could be lazy. + * - Only methods preserve laziness (`p.getX()`), functions do not. + * Possible enhancement: `getXOfPoint(p)` could also be lazy for inlined functions (now `p` is read as a whole). + */ + +// Given fun_ref = "A.fromSlice" from `lazy A.fromSlice(s)` check it's correct to be inside the `lazy` operator. +static bool does_function_satisfy_for_lazy_operator(FunctionPtr fun_ref, bool allow_wrapper = true) { + if (!fun_ref) { + return false; + } + // allow `lazy SomeStruct.fromSlice(s)`; these functions are also handled while transforming AST to Ops + if (fun_ref->is_builtin_function() && fun_ref->is_instantiation_of_generic_function()) { + std::string_view f_name = fun_ref->base_fun_ref->name; + return f_name == "T.fromSlice" || f_name == "T.fromCell" || f_name == "Cell.load"; + } + // allow `lazy loadData()`, where loadData() is a simple wrapper like + // `fun loadData() { return SomeStruct.fromCell(contract.getData()) }` + if (allow_wrapper && fun_ref->is_code_function() && fun_ref->get_num_params() == 0) { + auto f_body = fun_ref->ast_root->as()->get_body()->as(); + if (f_body->size() == 1) { + if (auto f_returns = f_body->get_item(0)->try_as(); f_returns && f_returns->has_return_value()) { + if (auto f_returns_call = f_returns->get_return_value()->try_as()) { + return does_function_satisfy_for_lazy_operator(f_returns_call->fun_maybe, false); + } + } + } + } + return false; +} + +// Currently, only `A | B | ...` (only structures) can be lazily loaded; later steps rely on StructPtr. +// For example, `(int32, int32) | ...` or `T?` are incompatible with `lazy`. +// If structs don't have prefixes, a prefix tree is built for a union, it also works. +static TypePtr is_union_type_prevented_from_lazy_loading(const TypeDataUnion* t_union) { + for (TypePtr variant : t_union->variants) { + bool is_struct = variant->unwrap_alias()->try_as(); + if (!is_struct) { + return variant; + } + } + return nullptr; +} + +// Given `lazy `, check that expr is correct: a valid function call with valid types. +// If not, fire an error. +static void check_lazy_operator_used_correctly(FunctionPtr cur_f, V v) { + bool is_ok_call = v->get_expr()->kind == ast_function_call + && does_function_satisfy_for_lazy_operator(v->get_expr()->as()->fun_maybe); + if (!is_ok_call) { + fire(cur_f, v->loc, "`lazy` operator can only be used with built-in functions like fromCell/fromSlice or simple wrappers over them"); + } + + // it should be either a struct or a union of structs + TypePtr expr_type = v->inferred_type; + if (expr_type->unwrap_alias()->try_as()) { + return; + } + if (const TypeDataUnion* expr_union = expr_type->unwrap_alias()->try_as()) { + if (TypePtr wrong_variant = is_union_type_prevented_from_lazy_loading(expr_union)) { + fire(cur_f, v->loc, "`lazy` union should contain only structures, but it contains `" + wrong_variant->as_human_readable() + "`"); + } + return; + } + fire(cur_f, v->loc, "`lazy` is applicable to structs, not to `" + expr_type->as_human_readable() + "`"); +} + +// Given `storage.save()` for a lazy `storage` variable, check if `self` inside should gain laziness. +// If yes, the body of the method is also traversed to detect usages. +// If no, it's assumed that all fields of `storage` are used (an object used "as a whole"). +static bool can_method_be_inlined_preserving_lazy(FunctionPtr method_ref) { + if (method_ref->is_builtin_function() && method_ref->is_instantiation_of_generic_function()) { + std::string_view f_name = method_ref->base_fun_ref->name; + return f_name == "T.toCell" || f_name == "T.forceLoadLazyObject"; + } + + return method_ref->is_inlined_in_place() && // only AST-inlined methods can be lazy + !method_ref->has_mutate_params() && + !method_ref->does_return_self(); +} + +// The first stage of an algorithm is to collect every lazy expression, every field, every union variant. +// This "collecting" is done inside a block, considering all nested statements. +// As a result, we know how many times a variable (and every field independently) is used for reading, writing, etc. +struct ExprUsagesWhileCollecting { + std::string name_str; // "v" / "v.field" / "v.field.nested"; for debugging only + TypePtr expr_type; // either type of variable/field or its narrowed type inside `match` + StructPtr struct_ref; // if it's a struct, otherwise, nullptr + + int used_for_reading = 0; + int used_for_writing = 0; + int used_for_matching = 0; + int used_for_toCell = 0; + int total_usages_with_fields = 0; + std::vector needed_above_stmt; + std::vector> used_as_match_subj; + + std::vector fields; // for struct: every field; otherwise: empty + std::vector variants; // for union: every variant; otherwise: itself (for match over non-union) + + ExprUsagesWhileCollecting(std::string name_for_debugging, TypePtr expr_type, bool is_variant_of_itself = false) + : name_str(std::move(name_for_debugging)) + , expr_type(expr_type) + , struct_ref(nullptr) { + if (const TypeDataUnion* expr_union = expr_type->unwrap_alias()->try_as()) { + variants.reserve(expr_union->size()); + for (int i = 0; i < expr_union->size(); ++i) { + variants.emplace_back(name_str + "(#" + std::to_string(i) + ")", expr_union->variants[i]); + } + return; + } + if (const TypeDataStruct* t_struct = expr_type->unwrap_alias()->try_as()) { + struct_ref = t_struct->struct_ref; + fields.reserve(struct_ref->get_num_fields()); + for (int i = 0; i < struct_ref->get_num_fields(); ++i) { + StructFieldPtr field_ref = struct_ref->get_field(i); + fields.emplace_back(name_str + "." + field_ref->name, field_ref->declared_type); + } + } + // to allow code like + // > val msg = lazy Counter.fromSlice(s) <-- struct! not union + // > match (msg) { Counter => {} else => {} } + // we track `msg` inside `match` as a single variant — not over union, but over itself + if (!is_variant_of_itself) { + variants.emplace_back(name_str, expr_type, true); + } + } + + void merge_with_sub_block(const ExprUsagesWhileCollecting& rhs) { + tolk_assert(expr_type->equal_to(rhs.expr_type) && struct_ref == rhs.struct_ref); + used_for_reading += rhs.used_for_reading; + used_for_writing += rhs.used_for_writing; + used_for_matching += rhs.used_for_matching; + used_for_toCell += rhs.used_for_toCell; + total_usages_with_fields += rhs.total_usages_with_fields; + needed_above_stmt.insert(needed_above_stmt.end(), rhs.needed_above_stmt.begin(), rhs.needed_above_stmt.end()); + used_as_match_subj.insert(used_as_match_subj.end(), rhs.used_as_match_subj.begin(), rhs.used_as_match_subj.end()); + for (int i = 0; i < static_cast(fields.size()); ++i) { + fields[i].merge_with_sub_block(rhs.fields[i]); + } + for (int i = 0; i < static_cast(variants.size()); ++i) { + variants[i].merge_with_sub_block(rhs.variants[i]); + } + } + + void on_used_rw(bool is_lvalue) { + is_lvalue ? used_for_writing++ : used_for_reading++; + total_usages_with_fields++; + } + + void on_used_toCell() { + used_for_toCell++; + total_usages_with_fields++; + } + + void on_used_as_match_subj(V v_match) { + used_as_match_subj.push_back(v_match); + used_for_matching++; + total_usages_with_fields++; + } + + bool is_self_or_field_used_for_reading() const { + if (used_for_reading) { + return true; + } + bool any = false; + for (const ExprUsagesWhileCollecting& field_usages : fields) { + any |= field_usages.is_self_or_field_used_for_reading() || field_usages.is_self_or_child_used_for_matching(); + } + return any; + } + + bool is_self_or_field_used_for_toCell() const { + if (used_for_toCell) { + return true; + } + bool any = false; + for (const ExprUsagesWhileCollecting& field_usages : fields) { + any |= field_usages.is_self_or_field_used_for_toCell(); + } + return any; + } + + bool is_self_or_child_used_for_writing() const { + if (used_for_writing) { + return true; + } + bool any = false; + for (const ExprUsagesWhileCollecting& field_usages : fields) { + any |= field_usages.is_self_or_child_used_for_writing(); + } + for (const ExprUsagesWhileCollecting& variant_usages : variants) { + any |= variant_usages.is_self_or_child_used_for_writing(); + } + return any; + } + + bool is_self_or_child_used_for_matching() const { + if (used_for_matching) { + return true; + } + bool any = false; + for (const ExprUsagesWhileCollecting& field_usages : fields) { + any |= field_usages.is_self_or_child_used_for_matching(); + } + for (const ExprUsagesWhileCollecting& variant_usages : variants) { + any |= variant_usages.is_self_or_child_used_for_matching(); + } + return any; + } + + void treat_match_like_read() { + if (used_for_matching) { + used_for_reading++; + total_usages_with_fields++; + } + for (ExprUsagesWhileCollecting& field_usages : fields) { + field_usages.treat_match_like_read(); + } + for (ExprUsagesWhileCollecting& variant_usages : variants) { + variant_usages.treat_match_like_read(); + } + } + + LazyStructLoadInfo generate_hidden_struct_load_all(SrcLocation loc, bool is_variant_of_union) const { + tolk_assert(struct_ref); + + StructPtr hidden_struct = new StructData( + "(lazy)" + struct_ref->name, + loc, + std::vector(struct_ref->fields), + is_variant_of_union ? StructData::PackOpcode(0, 0) : struct_ref->opcode, + struct_ref->overflow1023_policy, + nullptr, + nullptr, + struct_ref->ast_root + ); + std::vector all_fields_load_actions(struct_ref->get_num_fields(), LazyStructLoadInfo::LoadField); + + return LazyStructLoadInfo(struct_ref, hidden_struct, std::move(all_fields_load_actions)); + } + + // for every field of a struct, after calculating all usages, determine: which fields to load, and which to skip + LazyStructLoadInfo calculate_hidden_struct(SrcLocation loc, bool is_variant_of_union) const { + tolk_assert(struct_ref); + + struct FutureField { + LazyStructLoadInfo::ActionWithField action; + std::string_view field_name; + TypePtr field_type; + PackSize pack_size; + + FutureField(LazyStructLoadInfo::ActionWithField action, std::string_view field_name, TypePtr field_type) + : action(action), field_name(field_name), field_type(field_type) + , pack_size(estimate_serialization_size(field_type)) {} + }; + + std::vector future_fields; + + bool object_used_as_a_whole = used_for_reading || used_for_writing || (used_for_toCell && is_variant_of_union); + + // if used as toCell(), detect last modified field_idx: after it, immutable tail can be saved + bool need_immutable_tail = used_for_toCell && !is_variant_of_union; + int last_modified_field_idx = -1; + for (int field_idx = struct_ref->get_num_fields() - 1; field_idx >= 0; --field_idx) { + if (used_for_writing || fields[field_idx].is_self_or_child_used_for_writing()) { + last_modified_field_idx = field_idx; + break; + } + } + + // fill future_fields + for (int field_idx = 0; field_idx < struct_ref->get_num_fields(); ++field_idx) { + StructFieldPtr orig_field = struct_ref->get_field(field_idx); + TypePtr field_type = orig_field->declared_type; + const ExprUsagesWhileCollecting& field_usages = fields[field_idx]; + bool used_anyhow_but_match = field_usages.is_self_or_field_used_for_reading() || field_usages.is_self_or_child_used_for_writing() || field_usages.is_self_or_field_used_for_toCell(); + + if (need_immutable_tail && field_idx == last_modified_field_idx + 1) { + future_fields.emplace_back(LazyStructLoadInfo::SaveImmutableTail, "(tail)", TypeDataSlice::create()); + } + + if (field_usages.used_for_matching == 1 && !used_anyhow_but_match && !object_used_as_a_whole && !used_for_toCell && !is_variant_of_union && field_idx == struct_ref->get_num_fields() - 1 && + field_type->unwrap_alias()->try_as() && !is_union_type_prevented_from_lazy_loading(field_type->unwrap_alias()->try_as())) { + future_fields.emplace_back(LazyStructLoadInfo::LazyMatchField, orig_field->name, orig_field->declared_type); + continue; + } + if (used_anyhow_but_match || field_usages.is_self_or_child_used_for_matching() || object_used_as_a_whole) { + future_fields.emplace_back(LazyStructLoadInfo::LoadField, orig_field->name, orig_field->declared_type); + continue; + } + + // okay, this field is not needed; we should skip it; + // try to merge "skip 8 bits" + "skip 16 bits" into a single "skip 24 bits" + if (!future_fields.empty() && future_fields.back().action == LazyStructLoadInfo::SkipField) { + if (const TypeDataBytesN* last_bitsN = future_fields.back().field_type->try_as()) { + PackSize cur_size = estimate_serialization_size(field_type); + if (cur_size.min_bits == cur_size.max_bits && cur_size.max_refs == 0) { + TypePtr total_bitsN = TypeDataBytesN::create(true, last_bitsN->n_width + cur_size.max_bits); + future_fields.back().field_type = total_bitsN; + continue; + } + } + } + + // generate "skip 8 bits" instead of "skip int8" (it's more effective, and it can be merged with next) + TypePtr skip_type = field_type; + PackSize skip_size = estimate_serialization_size(field_type); + if (skip_size.min_bits == skip_size.max_bits && skip_size.max_refs == 0) { + skip_type = TypeDataBytesN::create(true, skip_size.max_bits); + } + future_fields.emplace_back(LazyStructLoadInfo::SkipField, "(gap)", skip_type); + } + + // if we need tail, we should load all fields before it (even if they aren't used) + if (need_immutable_tail) { + for (FutureField& f : future_fields) { + if (f.action == LazyStructLoadInfo::SaveImmutableTail) { + break; + } + f.action = LazyStructLoadInfo::LoadField; + } + } + + // here we drop "skip field" if we actually don't need even to skip it, just ignore, like it does not exist; + // example: unused fields in the end `load a; skip b; skip c` -> `load a`; + // example: `skip bits8; load ref` - `load ref`, because to reach a ref, no need to skip preceding bits; + for (size_t i = future_fields.size(); i-- > 0; ) { + if (FutureField f = future_fields[i]; f.action == LazyStructLoadInfo::SkipField) { + PackSize s_cur = f.pack_size; + PackSize s_after(0); + for (size_t j = i + 1; j < future_fields.size(); ++j) { + s_after = EstimateContext::sum(s_after, future_fields[j].pack_size); + } + bool ignore = (s_after.max_bits == 0 && s_after.max_refs == 0) // nothing is loaded after — no need to skip cur + || (s_after.max_bits == 0 && s_cur.max_refs == 0) // no reach ref, no need to skip bits + || (s_after.max_refs == 0 && s_cur.max_bits == 0) // and vice versa: no reach data, to need to skip refs + || (s_cur.max_bits == 0 && s_cur.max_refs == 0); // empty struct/tensor, no need "bits0 skip" + if (ignore) { + future_fields.erase(future_fields.begin() + static_cast(i)); + } + } + } + + // okay, we're done calculating; transform future_fields to hidden_struct + std::vector hidden_fields; + std::vector ith_field_action; + hidden_fields.reserve(future_fields.size()); + ith_field_action.reserve(future_fields.size()); + for (int field_idx = 0; field_idx < static_cast(future_fields.size()); ++field_idx) { + FutureField f = future_fields[field_idx]; + StructFieldPtr created = new StructFieldData(static_cast(f.field_name), {}, field_idx, nullptr, nullptr); + created->mutate()->assign_resolved_type(f.field_type); + hidden_fields.push_back(created); + ith_field_action.push_back(f.action); + } + + StructPtr hidden_struct = new StructData( + "(lazy)" + struct_ref->name, + loc, + std::move(hidden_fields), + is_variant_of_union ? StructData::PackOpcode(0, 0) : struct_ref->opcode, + struct_ref->overflow1023_policy, + nullptr, + nullptr, + struct_ref->ast_root + ); + + return LazyStructLoadInfo(struct_ref, hidden_struct, std::move(ith_field_action)); + } +}; + +// After collecting all vars/fields/variants usages, we should store, where exactly (in AST) which fields to load. +// Every insertion point is represented as this class, it's transformed to an AST auxiliary vertex by a replacer. +struct OneLoadingInsertionPoint { + std::vector all_stmts_where_used; + TypePtr union_variant; + StructFieldPtr field_ref; + LazyStructLoadInfo load_info; + mutable bool was_inserted_to_ast = false; + + OneLoadingInsertionPoint(std::vector&& all_stmts_where_used, TypePtr union_variant, StructFieldPtr field_ref, LazyStructLoadInfo&& load_info) + : all_stmts_where_used(std::move(all_stmts_where_used)) + , union_variant(union_variant) + , field_ref(field_ref) + , load_info(std::move(load_info)) {} + + void mark_inserted_to_ast() const { + was_inserted_to_ast = true; + } + + bool is_mentioned_in_stmt(AnyV stmt) const { + return std::find(all_stmts_where_used.begin(), all_stmts_where_used.end(), stmt) != all_stmts_where_used.end(); + } +}; + +// Every `lazy` operator must be assigned to a variable: `var st = lazy getStorage()`. +// Then `st` is a lazy variable, for which all calculations are done, after which it's stored as this class. +struct LazyVarInFunction { + LocalVarPtr var_ref; + V created_by_lazy_op; + V v_lazy_match_var_itself = nullptr; // lazy `match` for the variable itself + V v_lazy_match_last_field = nullptr; // lazy `match` for the last field of a struct + std::vector load_points; // a set of points where AST should be updated + + // convert already calculated usages of "st" variable and all its fields to a final immutable representation + LazyVarInFunction(FunctionPtr cur_f, LocalVarPtr var_ref, V created_by_lazy_op, ExprUsagesWhileCollecting&& var_usages) + : var_ref(var_ref) + , created_by_lazy_op(created_by_lazy_op) { + + // handle if `msg` is used only in `match (msg) { ... }` + // (it may even be not a union, just a struct with opcode, and `match` with `else`) + bool used_only_as_match = var_usages.used_for_matching == 1 && !var_usages.used_for_reading && !var_usages.used_for_toCell && !var_usages.used_for_writing; + if (used_only_as_match) { + v_lazy_match_var_itself = var_usages.used_as_match_subj.front(); + load_points.reserve(var_usages.variants.size()); + for (ExprUsagesWhileCollecting& variant_usages : var_usages.variants) { + LazyStructLoadInfo load_info = variant_usages.calculate_hidden_struct(created_by_lazy_op->loc, true); + load_points.emplace_back(std::move(variant_usages.needed_above_stmt), variant_usages.expr_type, nullptr, std::move(load_info)); + } + return; + } + + // okay, variable is used not only as `match`; + // prohibit this to a union: lazy union may only be matched, nothing more (`msg is A` etc. don't work) + const TypeDataStruct* t_struct = var_ref->declared_type->unwrap_alias()->try_as(); + bool is_union = t_struct == nullptr; + if (is_union) { + fire(cur_f, created_by_lazy_op->loc, "`lazy` will not work here, because variable `" + var_ref->name + "` it's used in a non-lazy manner\nhint: lazy union may be used only in `match` statement"); + } + + // so, it's just a struct, `lazy Point`; we've already calculated all statements where its fields are used + LazyStructLoadInfo load_info = var_usages.calculate_hidden_struct(created_by_lazy_op->loc, false); + bool is_lazy_match_last_field = !load_info.ith_field_action.empty() && load_info.ith_field_action.back() == LazyStructLoadInfo::LazyMatchField; + load_points.emplace_back(std::move(var_usages.needed_above_stmt), nullptr, nullptr, std::move(load_info)); + + // but probably, there is `match (lazyObj.lastField)` which is lazy; + if (is_lazy_match_last_field) { + StructFieldPtr field_ref = t_struct->struct_ref->fields.back(); + const ExprUsagesWhileCollecting& last_field_usages = var_usages.fields.back(); + v_lazy_match_last_field = last_field_usages.used_as_match_subj.front(); + // inside `match` over a field, loading locations were not detected: insert "load all fields" into every arm + for (int i = 0; i < v_lazy_match_last_field->get_arms_count(); ++i) { + if (auto v_arm = v_lazy_match_last_field->get_arm(i); v_arm->pattern_kind == MatchArmKind::exact_type) { + TypePtr union_variant = v_arm->pattern_type_node->resolved_type; + auto v_arm_body = v_arm->get_body()->get_block_statement(); + if (!v_arm_body->empty()) { + const TypeDataUnion* t_union = field_ref->declared_type->unwrap_alias()->try_as(); + int variant_idx = t_union->get_variant_idx(union_variant); + LazyStructLoadInfo load_all = last_field_usages.variants[variant_idx].generate_hidden_struct_load_all(created_by_lazy_op->loc, true); + load_points.emplace_back(std::vector{v_arm_body->get_item(0)}, union_variant, field_ref, std::move(load_all)); + } + } + } + } + } +}; + +static std::unordered_map> functions_with_lazy_vars; + + +static ExprUsagesWhileCollecting collect_expr_usages_in_block(std::string name_for_debugging, SinkExpression s_expr, TypePtr expr_type, V v_block); + +// This visitor finds usages of "v" / "v.field" / etc. in ONE statement or expression and populates lazy_expr data. +// For every struct, all its fields are also populated; for a union — all its variants. +// Since AST vertices don't have "parent_node", we need to remember some details while traversing top-down. +class CollectUsagesInStatementVisitor final : ASTVisitorFunctionBody { + AnyV cur_stmt; + SinkExpression s_expr; + ExprUsagesWhileCollecting* lazy_expr; + V parent_dot = nullptr; + + CollectUsagesInStatementVisitor(AnyV cur_stmt, SinkExpression s_expr, ExprUsagesWhileCollecting* lazy_expr) + : cur_stmt(cur_stmt), s_expr(s_expr), lazy_expr(lazy_expr) {} + +protected: + void visit(V v) override { + if (extract_sink_expression_from_vertex(v) == s_expr) { + bool is_subj_of_dot = parent_dot && parent_dot->is_target_struct_field() && parent_dot->get_obj() == v; + if (!is_subj_of_dot) { + lazy_expr->on_used_rw(v->is_lvalue); + } + } + } + + void visit(V v) override { + if (extract_sink_expression_from_vertex(v) == s_expr) { + bool is_subj_of_dot = parent_dot && parent_dot->is_target_struct_field() && parent_dot->get_obj() == v; + if (!is_subj_of_dot) { + lazy_expr->on_used_rw(v->is_lvalue); + } + } + auto backup = parent_dot; + parent_dot = v; + parent::visit(v); + parent_dot = backup; + } + + void visit(V v) override { + FunctionPtr fun_ref = v->fun_maybe; + if (fun_ref && fun_ref->does_accept_self() && can_method_be_inlined_preserving_lazy(fun_ref)) { + AnyExprV dot_obj = v->get_callee()->as()->get_obj(); + if (extract_sink_expression_from_vertex(dot_obj) == s_expr) { + // handle built-in functions specially + if (fun_ref->is_builtin_function() && fun_ref->base_fun_ref->name == "T.toCell") { + lazy_expr->on_used_toCell(); + return; + } + if (fun_ref->is_builtin_function() && fun_ref->base_fun_ref->name == "T.forceLoadLazyObject") { + lazy_expr->on_used_rw(false); // just use the object as a whole, slice will point to its end + return; + } + if (s_expr.index_path) { // obj.f.method(), mark obj.f as used anyway + lazy_expr->on_used_rw(false); + } + tolk_assert(fun_ref->is_code_function()); + + // okay, we have `st.save()` / `p.getX()`, which will be inlined when transforming to IR; + // dig into that method's body to fetch used fields `self.x` etc. + auto v_body_block = fun_ref->ast_root->try_as()->get_body()->try_as(); + ExprUsagesWhileCollecting inner_usages = collect_expr_usages_in_block(lazy_expr->name_str + "(=self)", SinkExpression(&fun_ref->parameters[0]), lazy_expr->expr_type, v_body_block); + inner_usages.treat_match_like_read(); // nested lazy match in inlined functions doesn't work, it's not wrapped into aux vertex + lazy_expr->merge_with_sub_block(inner_usages); + return; + } + } + + parent::visit(v); + } + + void visit(V v) override { + AnyExprV subj = v->get_subject(); + bool is_match_by_cur = extract_sink_expression_from_vertex(subj) == s_expr; + + // `match` statement over current expression is okay (it will be lazy if it's the only, and other conditions satisfy); + // `match` expression, generally, is not safe to be lazy, e.g. `return cond && match(...)`, + // but simply `return match(...)` / `var result = match(...)` is okay + bool is_safe = false; + if (v->is_statement()) { + is_safe = cur_stmt == v; + } else if (auto v_return = cur_stmt->try_as()) { + is_safe = v_return->get_return_value() == v; + } else if (auto v_assign = cur_stmt->try_as()) { + is_safe = v_assign->get_rhs() == v; + } else if (auto v_set_assign = cur_stmt->try_as()) { + is_safe = v_set_assign->get_rhs() == v; + } + + if (is_match_by_cur && is_safe) { + lazy_expr->on_used_as_match_subj(v); + const TypeDataUnion* expr_as_union = lazy_expr->expr_type->unwrap_alias()->try_as(); + for (int i = 0; i < v->get_arms_count(); ++i) { + if (auto v_arm = v->get_arm(i); v_arm->pattern_kind == MatchArmKind::exact_type) { + TypePtr exact_type = v_arm->pattern_type_node->resolved_type; + auto v_block = v_arm->get_body()->get_block_statement(); + ExprUsagesWhileCollecting variant_usages = collect_expr_usages_in_block(lazy_expr->name_str, s_expr, exact_type, v_block); + int variant_idx = expr_as_union ? expr_as_union->get_variant_idx(exact_type) : 0; // match over non-union is ok + lazy_expr->variants[variant_idx].merge_with_sub_block(variant_usages); + } + } + return; + } + + parent::visit(v); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + tolk_assert(false); + } + + static void collect_usages_in_expression(ExprUsagesWhileCollecting* out, SinkExpression s_expr, AnyV v_expr) { + CollectUsagesInStatementVisitor visitor(v_expr, s_expr, out); + visitor.parent::visit(v_expr); + if (out->struct_ref) { + for (int field_idx = 0; field_idx < out->struct_ref->get_num_fields(); ++field_idx) { + collect_usages_in_expression(&out->fields[field_idx], s_expr.get_child_s_expr(field_idx), v_expr); + out->total_usages_with_fields += out->fields[field_idx].total_usages_with_fields; + } + } + } +}; + +// This visitor analyzes A WHOLE BLOCK, statement by statement, and detects all statements where lazy_expr is used. +// It takes care of nested try/catch, etc. +// Ideally, it should calculate the only "lca" AST vertex of all usages, but it's not as easy as it seems. +// Instead, `lazy_expr->needed_above_stmt` contains all statements where expr is "mentioned" (and needs to be loaded before). +// And later, traversing top-down, the first occurrence is taken, inserting an AST aux vertex right before it. +class CollectUsagesInBlockBottomUp { + ExprUsagesWhileCollecting* lazy_expr; + SinkExpression s_expr; + + CollectUsagesInBlockBottomUp(ExprUsagesWhileCollecting* lazy_expr, SinkExpression s_expr) + : lazy_expr(lazy_expr), s_expr(s_expr) {} + + void visit_try_catch_statement(V v) const { + ExprUsagesWhileCollecting u_try = visit_sub_block(v->get_try_body()); + ExprUsagesWhileCollecting u_catch = visit_sub_block(v->get_catch_body()); + + if (u_try.total_usages_with_fields && u_catch.total_usages_with_fields) { + lazy_expr->needed_above_stmt.push_back(v); + } + if (u_try.total_usages_with_fields || u_catch.total_usages_with_fields) { + if (lazy_expr->total_usages_with_fields) { + lazy_expr->needed_above_stmt.push_back(v); + } + } + + lazy_expr->merge_with_sub_block(u_try); + lazy_expr->merge_with_sub_block(u_catch); + } + + void visit_if_statement(V v) const { + ExprUsagesWhileCollecting u_cond = visit_other(v->get_cond()); + ExprUsagesWhileCollecting u_then = visit_sub_block(v->get_if_body()); + ExprUsagesWhileCollecting u_else = visit_sub_block(v->get_else_body()); + + if (u_cond.total_usages_with_fields) { + lazy_expr->needed_above_stmt.push_back(v); + } + if (u_then.total_usages_with_fields && u_else.total_usages_with_fields) { + lazy_expr->needed_above_stmt.push_back(v); + } + if (u_then.total_usages_with_fields || u_else.total_usages_with_fields) { + if (lazy_expr->total_usages_with_fields) { + lazy_expr->needed_above_stmt.push_back(v); + } + } + + lazy_expr->merge_with_sub_block(u_cond); + lazy_expr->merge_with_sub_block(u_then); + lazy_expr->merge_with_sub_block(u_else); + } + + void visit_block_statement(V v) const { + for (int i = v->size() - 1; i >= 0; --i) { + AnyV ith_statement = v->get_item(i); + switch (ith_statement->kind) { + case ast_try_catch_statement: + visit_try_catch_statement(ith_statement->as()); + continue; + case ast_if_statement: + visit_if_statement(ith_statement->as()); + continue; + default: + break; + } + + ExprUsagesWhileCollecting u_ith = visit_other(ith_statement); + if (u_ith.total_usages_with_fields) { + u_ith.needed_above_stmt.push_back(ith_statement); + } + lazy_expr->merge_with_sub_block(u_ith); + } + } + + ExprUsagesWhileCollecting visit_sub_block(V v_block) const { + return visit_block_bottom_up(lazy_expr->name_str, s_expr, lazy_expr->expr_type, v_block); + } + + ExprUsagesWhileCollecting visit_other(AnyV v) const { + ExprUsagesWhileCollecting result(lazy_expr->name_str, lazy_expr->expr_type); + CollectUsagesInStatementVisitor::collect_usages_in_expression(&result, s_expr, v); + return result; + } + +public: + static ExprUsagesWhileCollecting visit_block_bottom_up(std::string name_for_debugging, SinkExpression s_expr, TypePtr expr_type, V v_block) { + ExprUsagesWhileCollecting lazy_expr(std::move(name_for_debugging), expr_type); + CollectUsagesInBlockBottomUp visitor(&lazy_expr, s_expr); + visitor.visit_block_statement(v_block); + return lazy_expr; + } +}; + +static ExprUsagesWhileCollecting collect_expr_usages_in_block(std::string name_for_debugging, SinkExpression s_expr, TypePtr expr_type, V v_block) { + return CollectUsagesInBlockBottomUp::visit_block_bottom_up(std::move(name_for_debugging), s_expr, expr_type, v_block); +} + +// Step 1: +// This visitor finds `var st = lazy expr`, launches finding usages for `st`, +// and adds `st` as LazyVarInFunction to a global list. +class CollectAllLazyObjectsAndFieldsVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; + V parent_block = nullptr; + +protected: + void visit(V v) override { + auto backup = parent_block; + parent_block = v; + parent::visit(v); + parent_block = backup; + } + + // `var st = lazy ...` + void visit(V v) override { + if (auto rhs_lazy = v->get_rhs()->try_as()) { + check_lazy_operator_used_correctly(cur_f, rhs_lazy); + + if (auto lhs_var_decl = v->get_lhs()->try_as()) { + auto lhs_var = lhs_var_decl->get_expr()->try_as(); + if (!lhs_var->marked_as_redef) { + // collect usages of a lazy var inside the same block statement where it's declared + LocalVarPtr var_ref = lhs_var->var_ref; + ExprUsagesWhileCollecting var_usages = collect_expr_usages_in_block(var_ref->name, SinkExpression(var_ref), var_ref->declared_type, parent_block); + LazyVarInFunction lazy_var(cur_f, var_ref, rhs_lazy, std::move(var_usages)); + functions_with_lazy_vars[cur_f].emplace_back(std::move(lazy_var)); + } + } + } + + parent::visit(v); + } + + // check that `lazy` operator used in a correct pattern with a correct expression + void visit(V v) override { + for (const LazyVarInFunction& lazy_var : functions_with_lazy_vars[cur_f]) { + if (lazy_var.created_by_lazy_op == v) { + parent::visit(v); + return; + } + } + + // for `return lazy ...` and other cases except allowed + fire(cur_f, v->loc, "incorrect `lazy` operator usage, it's not directly assigned to a variable\nhint: use `lazy` like this:\n> var st = lazy MyStorage.fromSlice(...)"); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + } +}; + +// Step 2: +// After visiting all functions and finding all lazy variables, this replacer updates AST, +// inserting (already calculated) load vertices. They are auxiliary vertices holding special data. +// They are handled later when transforming AST to Ops. +class LazyLoadInsertionsReplacer final : public ASTReplacerInFunctionBody { + FunctionPtr cur_f = nullptr; + V f_body = nullptr; + +protected: + // `var st = lazy expr` -> save "st" (it will be used in codegen to assert "st.x" that "x" is loaded) + AnyExprV replace(V v) override { + for (const LazyVarInFunction& lazy_var : functions_with_lazy_vars[cur_f]) { + if (lazy_var.created_by_lazy_op == v) { + v->mutate()->assign_dest_var_ref(lazy_var.var_ref); + return parent::replace(v); + } + } + + tolk_assert(false); // all `lazy` operators where detected and handled + } + + // `{ ... }` -> `{ ... load ... }` + AnyV replace(V v) override { + std::vector new_children; // since we don't have "parent_node" and "next_child" in AST, + new_children.reserve(v->size()); // traverse every block statement and insert "load" in the middle + + for (AnyV stmt : v->get_items()) { + for (const LazyVarInFunction& lazy_var : functions_with_lazy_vars[cur_f]) { + for (const OneLoadingInsertionPoint& ins : lazy_var.load_points) { + if (!ins.was_inserted_to_ast && ins.is_mentioned_in_stmt(stmt)) { + ASTAuxData* aux_data = new AuxData_LazyObjectLoadFields(lazy_var.var_ref, ins.union_variant, ins.field_ref, ins.load_info); + new_children.push_back(createV(stmt->loc, createV(stmt->loc), aux_data, TypeDataVoid::create())); + ins.mark_inserted_to_ast(); + } + } + } + new_children.push_back(parent::replace(stmt)); + } + + v->mutate()->assign_new_children(std::move(new_children)); + return v; + } + + // `match (lazy_obj)` / `match (lazy_obj.field)` -> wrap with aux + AnyExprV replace(V v) override { + for (const LazyVarInFunction& lazy_var : functions_with_lazy_vars[cur_f]) { + bool is_lazy_match_for_union = lazy_var.v_lazy_match_var_itself == v; + if (is_lazy_match_for_union) { + ASTAuxData* aux_data = new AuxData_LazyMatchForUnion(lazy_var.var_ref, nullptr); + return createV(v->loc, parent::replace(v), aux_data, v->inferred_type); + } + + bool is_lazy_match_for_last_field = lazy_var.v_lazy_match_last_field == v; + if (is_lazy_match_for_last_field) { + StructPtr struct_ref = lazy_var.var_ref->declared_type->unwrap_alias()->try_as()->struct_ref; + ASTAuxData* aux_data = new AuxData_LazyMatchForUnion(lazy_var.var_ref, struct_ref->fields.back()); + return createV(v->loc, parent::replace(v), aux_data, v->inferred_type); + } + } + + return parent::replace(v); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && functions_with_lazy_vars.contains(fun_ref); + } + + void start_replacing_in_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + f_body = v_function->get_body()->as(); + parent::replace(v_function->get_body()); + } +}; + +// Step 3: +// After modifying AST (inserting loads, lazy match, etc.), +// check __expect_lazy() calls, used in compiler tests as assertions. +class CheckExpectLazyAssertionsVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; + + static std::string stringify_lazy_load_above_stmt(const AuxData_LazyObjectLoadFields* aux_load) { + static const char* action_to_str[] = {"load", "skip", "lazy match", "save immutable"}; + + const LazyStructLoadInfo& load_info = aux_load->load_info; + StructPtr struct_ref = load_info.hidden_struct; + std::string_view last_action; + + std::string result = "[" + aux_load->var_ref->name + "] "; + for (int i = 0; i < struct_ref->get_num_fields(); ++i) { + std::string field_name = struct_ref->get_field(i)->name; + if (field_name == "(gap)") { + field_name = "(" + struct_ref->get_field(i)->declared_type->as_human_readable() + ")"; + } + std::string_view action = action_to_str[load_info.ith_field_action[i]]; + if (action != last_action) { + if (result[result.size() - 2] != ']') { + result += ", "; + } + result += action; + last_action = action; + } + result += " "; + result += field_name; + } + return result; + } + + void visit(V v) override { + // again, given "__expect_lazy(...)", we have no "next sibling", so traverse block statements + for (int i = 0; i < v->size(); ++i) { + AnyV cur_stmt = v->get_item(i); + if (auto v_call = cur_stmt->try_as()) { + if (v_call->fun_maybe && v_call->fun_maybe->is_builtin_function() && v_call->fun_maybe->name == "__expect_lazy") { + AnyV next_stmt = v->get_item(i + 1); + std::string_view expected = v_call->get_arg(0)->get_expr()->as()->str_val; + std::string actual; + if (auto next_aux = next_stmt->try_as()) { + if (const auto* aux_load = dynamic_cast(next_aux->aux_data)) { + actual = stringify_lazy_load_above_stmt(aux_load); + } + if (const auto* aux_match = dynamic_cast(next_aux->aux_data)) { + actual = "[" + aux_match->var_ref->name + "] " + "lazy match"; + } + } + + if (actual != expected) { + throw ParseError(cur_stmt->loc, "__expect_lazy failed: actual \"" + actual + "\""); + } + } + } + parent::visit(cur_stmt); + } + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && functions_with_lazy_vars.contains(fun_ref); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + } +}; + +void pipeline_lazy_load_insertions() { + visit_ast_of_all_functions(); + replace_ast_of_all_functions(); + visit_ast_of_all_functions(); + functions_with_lazy_vars.clear(); +} + +} // namespace tolk diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 2f6694469e..fad233c7f9 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -211,7 +211,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { // resolve identifiers after => at first - parent::visit(v->get_body()); + visit(v->get_body()); // because handling lhs of => is comprehensive switch (v->pattern_kind) { diff --git a/tolk/pipeline.h b/tolk/pipeline.h index e4448deb14..001228bee1 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -45,6 +45,7 @@ void pipeline_check_serialized_fields(); void pipeline_constant_folding(); void pipeline_optimize_boolean_expressions(); void pipeline_detect_inline_in_place(); +void pipeline_lazy_load_insertions(); void pipeline_convert_ast_to_legacy_Expr_Op(); void pipeline_find_unused_symbols(); diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index 31b19558fc..6597efe95e 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -113,6 +113,17 @@ std::string SinkExpression::to_string() const { return result; } +SinkExpression SinkExpression::get_child_s_expr(int field_idx) const { + uint64_t new_index_path = index_path; // if we have c.1 (index_path = 2) and construct c.1.N, calc (N<<8 + 2) + for (int empty_byte = 0; empty_byte < 8; ++empty_byte) { + if ((index_path & (0xFF << (empty_byte*8))) == 0) { + new_index_path += (field_idx + 1) << (empty_byte*8); + break; + } + } + return SinkExpression(var_ref, new_index_path); +} + static std::string to_string(SignState s) { static const char* txt[6 + 1] = {"sign=unknown", ">0", "<0", "=0", ">=0", "<=0", "sign=never"}; return txt[static_cast(s)]; diff --git a/tolk/smart-casts-cfg.h b/tolk/smart-casts-cfg.h index 1e24a7b696..b579e8f31d 100644 --- a/tolk/smart-casts-cfg.h +++ b/tolk/smart-casts-cfg.h @@ -71,6 +71,7 @@ struct SinkExpression { explicit operator bool() const { return var_ref != nullptr; } std::string to_string() const; + SinkExpression get_child_s_expr(int field_idx) const; }; // UnreachableKind is a reason of why control flow is unreachable or interrupted diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp index 6f8bbba912..cbad23fe3a 100644 --- a/tolk/tolk.cpp +++ b/tolk/tolk.cpp @@ -68,6 +68,7 @@ int tolk_proceed(const std::string &entrypoint_filename) { pipeline_constant_folding(); pipeline_optimize_boolean_expressions(); pipeline_detect_inline_in_place(); + pipeline_lazy_load_insertions(); pipeline_convert_ast_to_legacy_Expr_Op(); pipeline_find_unused_symbols(); diff --git a/tolk/tolk.h b/tolk/tolk.h index 3f6dcadd82..9bd21c99d6 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -46,7 +46,6 @@ inline void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) * */ -typedef int var_idx_t; typedef int const_idx_t; struct TmpVar { @@ -1042,12 +1041,26 @@ struct FunctionBodyAsm { void compile(AsmOpList& dest, SrcLocation loc) const; }; +struct LazyVariableLoadedState; + +// LazyVarRefAtCodegen is a mutable state of a variable assigned by `lazy` operator: +// > var p = lazy Point.fromSlice(s) +// When inlining a method `p.getX()`, `self` also becomes lazy, pointing to the same state. +struct LazyVarRefAtCodegen { + LocalVarPtr var_ref; + const LazyVariableLoadedState* var_state; + + LazyVarRefAtCodegen(LocalVarPtr var_ref, const LazyVariableLoadedState* var_state) + : var_ref(var_ref), var_state(var_state) {} +}; + struct CodeBlob { int var_cnt, in_var_cnt; FunctionPtr fun_ref; std::string name; SrcLocation forced_loc; std::vector vars; + std::vector lazy_variables; std::vector* inline_rvect_out = nullptr; bool inlining_before_immediate_return = false; std::unique_ptr ops; @@ -1101,6 +1114,8 @@ struct CodeBlob { close_blk(location); pop_cur(); } + const LazyVariableLoadedState* get_lazy_variable(LocalVarPtr var_ref) const; + const LazyVariableLoadedState* get_lazy_variable(AnyExprV v) const; void prune_unreachable_code(); void fwd_analyze(); void mark_noreturn(); diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index d37a657784..b6ffd6022e 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -1316,6 +1316,15 @@ bool TypeDataUnion::has_all_variants_of(const TypeDataUnion* rhs_type) const { return true; } +int TypeDataUnion::get_variant_idx(TypePtr lookup_variant) const { + for (int i = 0; i < size(); ++i) { + if (variants[i]->equal_to(lookup_variant)) { + return i; + } + } + return -1; +} + // given this = `T1 | T2 | ...` and rhs_type, find the only (not ambiguous) T_i that can accept it TypePtr TypeDataUnion::calculate_exact_variant_to_fit_rhs(TypePtr rhs_type) const { // primitive 1-slot nullable don't store type_id, they can be assigned less strict, like `int?` to `int16?` diff --git a/tolk/type-system.h b/tolk/type-system.h index c1f9cbb6db..62841893ac 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -569,6 +569,7 @@ class TypeDataUnion final : public TypeData { TypePtr calculate_exact_variant_to_fit_rhs(TypePtr rhs_type) const; bool has_all_variants_of(const TypeDataUnion* rhs_type) const; + int get_variant_idx(TypePtr lookup_variant) const; int get_width_on_stack() const override; int get_type_id() const override; From dc328acd348b380c44e4a823a93527f4b7827f5b Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 12 Jun 2025 13:23:48 +0700 Subject: [PATCH 331/524] [Tolk] Deprecate and comment some functions in stdlib --- crypto/smartcont/tolk-stdlib/common.tolk | 55 +++++++++++++++++------- tolk-tester/tests/cells-slices.tolk | 5 --- tolk-tester/tests/send-msg-1.tolk | 4 +- tolk-tester/tests/send-msg-2.tolk | 2 +- tolk-tester/tests/send-msg-3.tolk | 2 +- tolk/builtins.cpp | 10 ++--- tolk/send-message-api.cpp | 2 +- 7 files changed, 50 insertions(+), 30 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index c2674a4c52..3eefbbc5f8 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -388,7 +388,7 @@ fun Cell.beginParse(self): slice /// Returns hash of a typed cell, same as [cell.hash]. @pure -fun Cell.hash(self): slice +fun Cell.hash(self): uint256 asm "HASHCU"; /// RemainingBitsAndRefs is a special built-in type to get "all the rest" slice tail on reading. @@ -458,20 +458,20 @@ fun stringToBase256(constString: slice): int /// Computes the representation hash of a `cell` and returns it as a 256-bit unsigned integer `x`. /// Useful for signing and checking signatures of arbitrary entities represented by a tree of cells. @pure -fun cell.hash(self): int +fun cell.hash(self): uint256 asm "HASHCU"; /// Computes the hash of a `slice` and returns it as a 256-bit unsigned integer `x`. /// The result is the same as if an ordinary cell containing only data and references from `s` had been created /// and its hash computed by [cell.hash]. @pure -fun slice.hash(self): int +fun slice.hash(self): uint256 asm "HASHSU"; /// Computes sha256 of the data bits of a `slice`. If the bit length of `s` is not divisible by eight, /// throws a cell underflow exception. The hash value is returned as a 256-bit unsigned integer `x`. @pure -fun slice.bitsHash(self): int +fun slice.bitsHash(self): uint256 asm "SHA256U"; /// Checks the Ed25519-`signature` of a `hash` (a 256-bit unsigned integer, usually computed as the hash of some data) @@ -1119,7 +1119,7 @@ struct (0x00) ExtOutLogBucket { /// Options for creating an external outgoing message. /// Consider [createExternalLogMessage] for examples. -struct createExternalLogMessageOptions { +struct CreateExternalLogMessageOptions { /// destination is either an external address or a pattern to calculate it dest: address | // either some valid external/none address (not internal!) builder | // ... or a manually constructed builder with a valid external address @@ -1143,7 +1143,7 @@ struct createExternalLogMessageOptions { /// (if body is small, it will be inlined without an expensive cell creation) /// (if body is large, the compiler will automatically wrap it into a cell) @pure -fun createExternalLogMessage(options: createExternalLogMessageOptions): OutMessage +fun createExternalLogMessage(options: CreateExternalLogMessageOptions): OutMessage builtin; /// UnsafeBodyNoRef is used to prevent default behavior: when message body is potentially large, @@ -1157,13 +1157,13 @@ fun createExternalLogMessage(options: createExternalLogMessageOptions { - bodyForceNoRef: T; + forceInline: T; } /// OutMessage is a result of [createMessage]. @@ -1202,6 +1202,13 @@ struct StateInit { } /// Calculates a hash of StateInit if only code+data are set. +/// Example: +/// ``` +/// val addrHash = StateInit.calcHashCodeData(codeCell, dataCell); +/// createMessage({ +/// dest: (workchain, addrHash), +/// ... +/// ``` @pure fun StateInit.calcHashCodeData(code: cell, data: cell): uint256 asm """ // code data @@ -1304,23 +1311,41 @@ fun sendRawMessage(msg: cell, mode: int): void Receiving and handling messages. */ +/// Skip 0xFFFFFFFF prefix (when a message is bounced). +@pure +fun slice.skipBouncedPrefix(mutate self): self + asm "32 LDU" "NIP"; + /// Load msgFlags from incoming message body (4 bits). @pure +@deprecated("use `onInternalMessage(in: InMessage)` and `in.isBounced` instead of parsing msgCell manually") fun slice.loadMessageFlags(mutate self): int asm( -> 1 0) "4 LDU"; -/// Skip 0xFFFFFFFF prefix (when a message is bounced). -@pure -fun slice.skipBouncedPrefix(mutate self): self - asm "32 PUSHINT" "SDSKIPFIRST"; - -/// The guideline recommends to start the body of an internal message with uint32 `op` and uint64 `queryId`. +/// Loads a message opcode (32-bit integer) when parsing message body manually. @pure fun slice.loadMessageOp(mutate self): int asm( -> 1 0) "32 LDU"; -/// The guideline recommends that uint64 `queryId` should follow uint32 `op`. +/// Stores a message opcode (32-bit integer) when composing an output message manually. +@pure +@deprecated("use `createMessage` instead of composing messages manually") +fun builder.storeMessageOp(mutate self, op: int): self + asm(op self) "32 STU"; + +/// Loads a message queryId (64-bit integer, immediately following the opcode) when parsing message body manually. @pure +@deprecated("use structures and lazy match instead of parsing messages manually") fun slice.loadMessageQueryId(mutate self): int asm( -> 1 0) "64 LDU"; +@pure +@deprecated("use structures and lazy loading instead of parsing messages manually") +fun slice.skipMessageQueryId(mutate self): self + asm "64 LDU" "NIP"; + +@pure +@deprecated("use `createMessage` instead of composing messages manually") +fun builder.storeMessageQueryId(mutate self, queryId: int): self + asm(queryId self) "64 STU"; + diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 2cbb786470..d8e62bc6cb 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -1,11 +1,6 @@ fun builder.store_u32(mutate self, value: int): self { return self.storeUint(value, 32); } -fun builder.storeMessageOp(mutate self, op: int): self - asm(op self) "32 STU"; -fun builder.storeMessageQueryId(mutate self, queryId: int): self - asm(queryId self) "64 STU"; - fun slice.load_u32(mutate self): int { return self.loadUint(32); } diff --git a/tolk-tester/tests/send-msg-1.tolk b/tolk-tester/tests/send-msg-1.tolk index a7be4ec058..3a889dcf43 100644 --- a/tolk-tester/tests/send-msg-1.tolk +++ b/tolk-tester/tests/send-msg-1.tolk @@ -643,7 +643,7 @@ fun test22(op: uint32, queryId: uint64) { dest: address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"), value: ton("0.08"), body: UnsafeBodyNoRef { - bodyForceNoRef: bodyB + forceInline: bodyB }, }); assert(b.hash() == test22_manual(bodyB).hash(), 122); @@ -690,7 +690,7 @@ fun test23(state32: uint32) { var b = createMessage({ bounce: true, body: UnsafeBodyNoRef { - bodyForceNoRef: bd, + forceInline: bd, }, dest: { workchain: 9, stateInit: init }, value: ton("0.10009"), diff --git a/tolk-tester/tests/send-msg-2.tolk b/tolk-tester/tests/send-msg-2.tolk index 1d11752db3..050cd84322 100644 --- a/tolk-tester/tests/send-msg-2.tolk +++ b/tolk-tester/tests/send-msg-2.tolk @@ -294,7 +294,7 @@ fun test5() { closeTo: pivot, } }, - body: UnsafeBodyNoRef { bodyForceNoRef: body }, + body: UnsafeBodyNoRef { forceInline: body }, value: ton("0.1"), }); return b.hash() diff --git a/tolk-tester/tests/send-msg-3.tolk b/tolk-tester/tests/send-msg-3.tolk index 07265ec965..8db47589d1 100644 --- a/tolk-tester/tests/send-msg-3.tolk +++ b/tolk-tester/tests/send-msg-3.tolk @@ -171,7 +171,7 @@ fun test9(topic: int) { }; var b = createExternalLogMessage({ body: UnsafeBodyNoRef { - bodyForceNoRef: bd, + forceInline: bd, }, dest: ExtOutLogBucket { topic: topic }, }); diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index f0d8de7cb2..d15288564b 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1248,7 +1248,7 @@ void define_builtins() { TypePtr PackOptions = TypeDataUnknown::create(); TypePtr UnpackOptions = TypeDataUnknown::create(); TypePtr CreateMessageOptions = TypeDataUnknown::create(); - TypePtr createExternalLogMessageOptions = TypeDataUnknown::create(); + TypePtr CreateExternalLogMessageOptions = TypeDataUnknown::create(); TypePtr OutMessage = TypeDataUnknown::create(); TypePtr AddressShardingOptions = TypeDataUnknown::create(); const GenericsDeclaration* declTBody = new GenericsDeclaration(std::vector{{"TBody", nullptr}}, 0); @@ -1510,7 +1510,7 @@ void define_builtins() { define_builtin_func("createMessage", {CreateMessageOptions}, OutMessage, declTBody, compile_time_only_function, FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); - define_builtin_func("createExternalLogMessage", {createExternalLogMessageOptions}, OutMessage, declTBody, + define_builtin_func("createExternalLogMessage", {CreateExternalLogMessageOptions}, OutMessage, declTBody, compile_time_only_function, FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); @@ -1584,15 +1584,15 @@ void patch_builtins_after_stdlib_loaded() { lookup_function("builder.storeAny")->mutate()->parameters[2].default_value = v_empty_PackOptions; StructPtr struct_ref_CreateMessageOptions = lookup_global_symbol("CreateMessageOptions")->try_as(); - StructPtr struct_ref_createExternalLogMessageOptions = lookup_global_symbol("createExternalLogMessageOptions")->try_as(); + StructPtr struct_ref_CreateExternalLogMessageOptions = lookup_global_symbol("CreateExternalLogMessageOptions")->try_as(); StructPtr struct_ref_OutMessage = lookup_global_symbol("OutMessage")->try_as(); TypePtr CreateMessageOptions = TypeDataGenericTypeWithTs::create(struct_ref_CreateMessageOptions, nullptr, {TypeDataGenericT::create("TBody")}); - TypePtr createExternalLogMessageOptions = TypeDataGenericTypeWithTs::create(struct_ref_createExternalLogMessageOptions, nullptr, {TypeDataGenericT::create("TBody")}); + TypePtr CreateExternalLogMessageOptions = TypeDataGenericTypeWithTs::create(struct_ref_CreateExternalLogMessageOptions, nullptr, {TypeDataGenericT::create("TBody")}); TypePtr OutMessage = TypeDataStruct::create(struct_ref_OutMessage); lookup_function("createMessage")->mutate()->parameters[0].declared_type = CreateMessageOptions; lookup_function("createMessage")->mutate()->declared_return_type = OutMessage; - lookup_function("createExternalLogMessage")->mutate()->parameters[0].declared_type = createExternalLogMessageOptions; + lookup_function("createExternalLogMessage")->mutate()->parameters[0].declared_type = CreateExternalLogMessageOptions; lookup_function("createExternalLogMessage")->mutate()->declared_return_type = OutMessage; } diff --git a/tolk/send-message-api.cpp b/tolk/send-message-api.cpp index 4fd8ab8465..6d0b535240 100644 --- a/tolk/send-message-api.cpp +++ b/tolk/send-message-api.cpp @@ -329,7 +329,7 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T } std::vector generate_createExternalLogMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect) { - StructPtr s_Options = lookup_global_symbol("createExternalLogMessageOptions")->try_as(); + StructPtr s_Options = lookup_global_symbol("CreateExternalLogMessageOptions")->try_as(); StructPtr s_ExtOutLogBucket = lookup_global_symbol("ExtOutLogBucket")->try_as(); const TypeDataUnion* t_dest = s_Options->find_field("dest")->declared_type->try_as(); From d9bb6bb0c09f11712495a2b2600d819c9bbb7ba6 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Wed, 18 Jun 2025 17:04:17 +0300 Subject: [PATCH 332/524] [Tolk] Use parenthesis `()` as a suffix for Fift generation For every function f, name it `f()` instead of `f` in Fift output. > DECLPROC f() > f() CALLDICT This avoids collisions with Fift reserved names like execute/swap/etc. For global vars, `$` prefix is now used: > DECLGLOBVAR $name --- .../tests/allow-post-modification.tolk | 6 +-- tolk-tester/tests/asm-arg-order.tolk | 24 +++++----- tolk-tester/tests/assignment-tests.tolk | 2 +- tolk-tester/tests/bit-operators.tolk | 14 +++--- tolk-tester/tests/cells-slices.tolk | 16 +++---- tolk-tester/tests/codegen-check-demo.tolk | 8 ++-- tolk-tester/tests/comments-tests.tolk | 2 +- tolk-tester/tests/generics-1.tolk | 22 ++++----- tolk-tester/tests/generics-2.tolk | 6 +-- tolk-tester/tests/generics-3.tolk | 2 +- tolk-tester/tests/generics-4.tolk | 14 +++--- tolk-tester/tests/if-else-tests.tolk | 46 +++++++++---------- tolk-tester/tests/indexed-access.tolk | 24 +++++----- tolk-tester/tests/inline-tests.tolk | 16 +++---- tolk-tester/tests/intN-tests.tolk | 8 ++-- tolk-tester/tests/lazy-algo-tests.tolk | 2 +- tolk-tester/tests/lazy-load-tests.tolk | 38 +++++++-------- tolk-tester/tests/logical-operators.tolk | 16 +++---- tolk-tester/tests/match-by-expr-tests.tolk | 20 ++++---- tolk-tester/tests/methods-tests.tolk | 6 +-- tolk-tester/tests/mutate-methods.tolk | 14 +++--- tolk-tester/tests/no-spaces.tolk | 10 ++-- tolk-tester/tests/null-keyword.tolk | 6 +-- tolk-tester/tests/nullable-tensors.tolk | 6 +-- tolk-tester/tests/op-priority.tolk | 6 +-- tolk-tester/tests/pack-unpack-1.tolk | 2 +- tolk-tester/tests/pack-unpack-4.tolk | 2 +- tolk-tester/tests/pack-unpack-6.tolk | 10 ++-- tolk-tester/tests/parse-address.tolk | 4 +- .../tests/remove-unused-functions.tolk | 16 +++---- tolk-tester/tests/self-keyword.tolk | 28 +++++------ tolk-tester/tests/smart-cast-tests.tolk | 6 +-- tolk-tester/tests/some-tests-2.tolk | 24 +++++----- tolk-tester/tests/special-fun-names.tolk | 32 +++++++++++-- tolk-tester/tests/struct-tests.tolk | 24 +++++----- tolk-tester/tests/try-catch-tests.tolk | 10 ++-- tolk-tester/tests/type-aliases-tests.tolk | 2 +- tolk-tester/tests/union-types-tests.tolk | 8 ++-- tolk-tester/tests/use-before-declare.tolk | 4 +- tolk-tester/tests/var-apply-tests.tolk | 13 ++++++ tolk/codegen.cpp | 12 ++--- tolk/pipe-generate-fif-output.cpp | 10 ++-- 42 files changed, 289 insertions(+), 252 deletions(-) diff --git a/tolk-tester/tests/allow-post-modification.tolk b/tolk-tester/tests/allow-post-modification.tolk index 0f8603cc66..a74b2e1ecd 100644 --- a/tolk-tester/tests/allow-post-modification.tolk +++ b/tolk-tester/tests/allow-post-modification.tolk @@ -154,14 +154,14 @@ fun main() { @fif_codegen """ - int.~inc PROC:<{ // self y - inc CALLDICT // self newY + int.~inc() PROC:<{ // self y + inc() CALLDICT // self newY }> """ @fif_codegen """ - test_assign_tensor_global PROC:<{ // x.0 x.1 + test_assign_tensor_global() PROC:<{ // x.0 x.1 """ @code_hash 6737917279814799680932710799951154408447028229503449454536845752635763933556 diff --git a/tolk-tester/tests/asm-arg-order.tolk b/tolk-tester/tests/asm-arg-order.tolk index b8622060a2..1558a3c721 100644 --- a/tolk-tester/tests/asm-arg-order.tolk +++ b/tolk-tester/tests/asm-arg-order.tolk @@ -219,7 +219,7 @@ fun main() { @fif_codegen """ - test27 PROC:<{ + test27() PROC:<{ 10 PUSHINT 2 PUSHINT 1 ADDCONST MUL @@ -228,18 +228,18 @@ fun main() { @fif_codegen """ - test28 PROC:<{ - get10Pure CALLDICT - get2Pure CALLDICT + test28() PROC:<{ + get10Pure() CALLDICT + get2Pure() CALLDICT 1 ADDCONST MUL }> """ @fif_codegen """ - test29 PROC:<{ - get2Impure CALLDICT - get10Impure CALLDICT + test29() PROC:<{ + get2Impure() CALLDICT + get10Impure() CALLDICT SWAP 1 ADDCONST MUL }> @@ -247,17 +247,17 @@ fun main() { @fif_codegen """ - test30 PROC:<{ + test30() PROC:<{ ... - g10 GETGLOB - g2 GETGLOB + $g10 GETGLOB + $g2 GETGLOB 1 ADDCONST MUL }> """ @fif_codegen """ - test32 PROC:<{ + test32() PROC:<{ 10 PUSHINT 2 PUSHINT 1 ADDCONST MUL @@ -266,7 +266,7 @@ fun main() { @fif_codegen """ - test34 PROC:<{ + test34() PROC:<{ x{020a} PUSHSLICE 8 LDU 8 LDU diff --git a/tolk-tester/tests/assignment-tests.tolk b/tolk-tester/tests/assignment-tests.tolk index e53020d050..bd9b28d9b4 100644 --- a/tolk-tester/tests/assignment-tests.tolk +++ b/tolk-tester/tests/assignment-tests.tolk @@ -283,7 +283,7 @@ fun main(value: int, ) { @fif_codegen """ - test116 PROC:<{ // + test116() PROC:<{ // 1 PUSHINT // '10=1 2 PUSHINT // '10=1 '11=2 3 PUSHINT // '10=1 '11=2 '12=3 diff --git a/tolk-tester/tests/bit-operators.tolk b/tolk-tester/tests/bit-operators.tolk index 692275bf64..c734740dfd 100644 --- a/tolk-tester/tests/bit-operators.tolk +++ b/tolk-tester/tests/bit-operators.tolk @@ -126,7 +126,7 @@ fun testBoolCompareOptimized(x: bool) { @fif_codegen """ - boolWithBitwiseConst PROC:<{ + boolWithBitwiseConst() PROC:<{ 0 PUSHINT // '3 -1 PUSHINT // '3 '5 0 PUSHINT // '3 '5 '7 @@ -136,7 +136,7 @@ fun testBoolCompareOptimized(x: bool) { @fif_codegen """ - testDoUntilCodegen PROC:<{ // i n + testDoUntilCodegen() PROC:<{ // i n 0 PUSHINT // i n cnt=0 UNTIL:<{ INC // i n cnt @@ -162,7 +162,7 @@ fun testBoolCompareOptimized(x: bool) { @fif_codegen """ - testConstNegateCodegen PROC:<{ + testConstNegateCodegen() PROC:<{ TRUE // '0 FALSE // '0 '1 FALSE // '0 '1 '2 @@ -174,7 +174,7 @@ fun testBoolCompareOptimized(x: bool) { @fif_codegen """ - testBoolNegateOptimized PROC:<{ // x + testBoolNegateOptimized() PROC:<{ // x DUP // x x NOT // x '1 OVER // x '1 x @@ -186,14 +186,14 @@ fun testBoolCompareOptimized(x: bool) { @fif_codegen """ - testBoolCompareOptimized PROC:<{ // x + testBoolCompareOptimized() PROC:<{ // x DUP // x x NOT // x '1 OVER // x '1 x - eqX CALLDICT // x '1 '2 + eqX() CALLDICT // x '1 '2 NOT // x '1 '3 s2 PUSH // x '1 '3 x - eqX CALLDICT // x '1 '3 '4 + eqX() CALLDICT // x '1 '3 '4 s3 PUSH // x '1 '3 '4 x }> """ diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index d8e62bc6cb..fa96bd5295 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -352,7 +352,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - test6 PROC:<{ + test6() PROC:<{ 18446744082299486211 PUSHINT NEWC 96 STU // '2 @@ -361,7 +361,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - test17 PROC:<{ + test17() PROC:<{ 4219 PUSHINT NEWC 16 STU @@ -369,7 +369,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - test18 PROC:<{ + test18() PROC:<{ 421 PUSHINT NEWC 26 STU @@ -377,7 +377,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - test19 PROC:<{ + test19() PROC:<{ NEWC 123 PUSHINT SWAP @@ -392,7 +392,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - test20 PROC:<{ + test20() PROC:<{ 40976 PUSHINT NEWC 16 STU @@ -400,7 +400,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - test21 PROC:<{ + test21() PROC:<{ 14 PUSHINT SDSKIPFIRST }> @@ -408,7 +408,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - test22 PROC:<{ + test22() PROC:<{ NEWC // '2 NEWC // b1 b2 SWAP // b2 b1 @@ -421,7 +421,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - test23 PROC:<{ + test23() PROC:<{ NEWC SWAP IF:<{ diff --git a/tolk-tester/tests/codegen-check-demo.tolk b/tolk-tester/tests/codegen-check-demo.tolk index ff2ca3c2cc..dec38a8857 100644 --- a/tolk-tester/tests/codegen-check-demo.tolk +++ b/tolk-tester/tests/codegen-check-demo.tolk @@ -33,7 +33,7 @@ Below, I just give examples of @fif_codegen tag: @fif_codegen """ -main PROC:<{ // s +main() PROC:<{ // s 17 PUSHINT // s '3=17 OVER // s z=17 t WHILE:<{ @@ -49,7 +49,7 @@ main PROC:<{ // s @fif_codegen """ -main PROC:<{ +main() PROC:<{ ... WHILE:<{ ... @@ -76,13 +76,13 @@ main PROC:<{ @fif_codegen """ -test1 PROC:<{ +test1() PROC:<{ FALSE }> """ @fif_codegen NOT // '8 -@fif_codegen main PROC:<{ +@fif_codegen main() PROC:<{ @fif_codegen_avoid PROCINLINE @fif_codegen_avoid END c diff --git a/tolk-tester/tests/comments-tests.tolk b/tolk-tester/tests/comments-tests.tolk index 3d00a0b1a8..96caefe924 100644 --- a/tolk-tester/tests/comments-tests.tolk +++ b/tolk-tester/tests/comments-tests.tolk @@ -27,7 +27,7 @@ fun main() { @fif_codegen_enable_comments @fif_codegen """ - demo_fields_def PROC:<{ // x + demo_fields_def() PROC:<{ // x // 13: f1: x + C_20 20 ADDCONST // '6 10 PUSHINT // '6 '7=10 diff --git a/tolk-tester/tests/generics-1.tolk b/tolk-tester/tests/generics-1.tolk index 39ddb2cdf3..b92f27a6e3 100644 --- a/tolk-tester/tests/generics-1.tolk +++ b/tolk-tester/tests/generics-1.tolk @@ -234,19 +234,19 @@ fun main(x: int): (int, [Tup2Int]) { @testcase | 112 | 0 0 | 0 0 -1 @testcase | 113 | 5 1 | 5 1 -100 (null) 5 1 -100 9 -100 (null) -100 (null) 5 1 -100 (null) (null) (null) 0 -@fif_codegen DECLPROC eq1 -@fif_codegen DECLPROC eq1 -@fif_codegen DECLPROC eq1<(int,int)> -@fif_codegen DECLPROC eq1<[int,int]> -@fif_codegen DECLPROC eq1 -@fif_codegen DECLPROC eq1 -@fif_codegen DECLPROC eq1>> +@fif_codegen DECLPROC eq1() +@fif_codegen DECLPROC eq1() +@fif_codegen DECLPROC eq1<(int,int)>() +@fif_codegen DECLPROC eq1<[int,int]>() +@fif_codegen DECLPROC eq1() +@fif_codegen DECLPROC eq1() +@fif_codegen DECLPROC eq1>>() // was inlined -@fif_codegen_avoid DECLPROC getTwo +@fif_codegen_avoid DECLPROC getTwo() @fif_codegen_avoid getTwo -@fif_codegen_avoid DECLPROC eq1 -@fif_codegen_avoid DECLPROC eq2 -@fif_codegen_avoid DECLPROC eq3 +@fif_codegen_avoid DECLPROC eq1() +@fif_codegen_avoid DECLPROC eq2() +@fif_codegen_avoid DECLPROC eq3() */ diff --git a/tolk-tester/tests/generics-2.tolk b/tolk-tester/tests/generics-2.tolk index 65d76cebfe..2a6684f0dd 100644 --- a/tolk-tester/tests/generics-2.tolk +++ b/tolk-tester/tests/generics-2.tolk @@ -475,12 +475,12 @@ fun main(c: Wrapper, d: WrappedInt) { @fif_codegen """ - test1 PROC:<{ + test1() PROC:<{ 23 PUSHINT // c1=23 DUP // c1=23 c2=23 }> """ -@fif_codegen DECLPROC getWrappervalue1> -@fif_codegen DECLPROC getWrappervalue2>> +@fif_codegen DECLPROC getWrappervalue1>() +@fif_codegen DECLPROC getWrappervalue2>>() */ diff --git a/tolk-tester/tests/generics-3.tolk b/tolk-tester/tests/generics-3.tolk index 691b878356..494e48ba25 100644 --- a/tolk-tester/tests/generics-3.tolk +++ b/tolk-tester/tests/generics-3.tolk @@ -136,7 +136,7 @@ fun main() { @fif_codegen """ - test3 PROC:<{ + test3() PROC:<{ 10 PUSHINT // r.USlot1=10 -1 PUSHINT // r.USlot1=10 '22=-1 FALSE // r.USlot1=10 '22=-1 '24 diff --git a/tolk-tester/tests/generics-4.tolk b/tolk-tester/tests/generics-4.tolk index 2869ad1f57..c5f3e038c3 100644 --- a/tolk-tester/tests/generics-4.tolk +++ b/tolk-tester/tests/generics-4.tolk @@ -180,16 +180,16 @@ fun main() { @testcase | 107 | | 1 10 110 117 @testcase | 108 | | 10 (null) 20 -@fif_codegen DECLPROC eqUnusedT -@fif_codegen DECLPROC eqUnusedU -@fif_codegen DECLPROC mySend -@fif_codegen DECLPROC mySend -@fif_codegen DECLPROC mySend -@fif_codegen DECLPROC mySend +@fif_codegen DECLPROC eqUnusedT() +@fif_codegen DECLPROC eqUnusedU() +@fif_codegen DECLPROC mySend() +@fif_codegen DECLPROC mySend() +@fif_codegen DECLPROC mySend() +@fif_codegen DECLPROC mySend() @fif_codegen """ - test6 PROC:<{ // + test6() PROC:<{ // NEWC // '8 ENDC // p.init.USlot2 -1 PUSHINT // p.init.USlot2 '11=-1 diff --git a/tolk-tester/tests/if-else-tests.tolk b/tolk-tester/tests/if-else-tests.tolk index d34dc178b2..d510dd4a47 100644 --- a/tolk-tester/tests/if-else-tests.tolk +++ b/tolk-tester/tests/if-else-tests.tolk @@ -256,7 +256,7 @@ fun main() { @fif_codegen """ - test3 PROC:<{ // x + test3() PROC:<{ // x DUP // x x 20 NEQINT // x '2 IFNOTJMP:<{ // x @@ -270,7 +270,7 @@ fun main() { @fif_codegen """ - withNoElse PROC:<{ + withNoElse() PROC:<{ DUP 123 EQINT IFJMP:<{ @@ -295,7 +295,7 @@ fun main() { @fif_codegen """ - withElse PROC:<{ + withElse() PROC:<{ DUP 123 EQINT IFJMP:<{ @@ -320,7 +320,7 @@ fun main() { @fif_codegen """ - withMatch PROC:<{ + withMatch() PROC:<{ DUP 123 EQINT IFJMP:<{ @@ -345,59 +345,59 @@ fun main() { @fif_codegen """ - demo9_1 PROC:<{ + demo9_1() PROC:<{ 0 GTINT IFJMP:<{ - t GETGLOB + $t GETGLOB 123 PUSHINT TPUSH - t SETGLOB + $t SETGLOB }> - t GETGLOB + $t GETGLOB 456 PUSHINT TPUSH - t SETGLOB + $t SETGLOB }> """ @fif_codegen """ - demo9_2 PROC:<{ + demo9_2() PROC:<{ 0 GTINT IFJMP:<{ - t GETGLOB + $t GETGLOB 123 PUSHINT TPUSH - t SETGLOB + $t SETGLOB }> }> """ @fif_codegen """ - demo9_3 PROC:<{ + demo9_3() PROC:<{ 0 GTINT IFJMP:<{ - t GETGLOB + $t GETGLOB 123 PUSHINT TPUSH - t SETGLOB + $t SETGLOB }> }> """ @fif_codegen """ - demo9_4 PROC:<{ + demo9_4() PROC:<{ DUP -100 NEQINT IFJMP:<{ 0 GTINT IFJMP:<{ - t GETGLOB + $t GETGLOB 123 PUSHINT TPUSH - t SETGLOB + $t SETGLOB }> }> DROP @@ -406,22 +406,22 @@ fun main() { @fif_codegen """ - demo10_1 PROC:<{ + demo10_1() PROC:<{ DUP 0 EQINT IFJMP:<{ DROP - t GETGLOB + $t GETGLOB 123 PUSHINT TPUSH - t SETGLOB + $t SETGLOB }> 1 EQINT IFJMP:<{ - t GETGLOB + $t GETGLOB 456 PUSHINT TPUSH - t SETGLOB + $t SETGLOB }> }> """ diff --git a/tolk-tester/tests/indexed-access.tolk b/tolk-tester/tests/indexed-access.tolk index 8cd4a74d1c..dd0400c281 100644 --- a/tolk-tester/tests/indexed-access.tolk +++ b/tolk-tester/tests/indexed-access.tolk @@ -307,7 +307,7 @@ fun main(){} @fif_codegen """ - testCodegenSimple PROC:<{ + testCodegenSimple() PROC:<{ 1 PUSHINT // '2=1 SINGLE // t1 2 PUSHINT // t1 '3=2 @@ -327,43 +327,43 @@ fun main(){} STRDUMP DROP 1 PUSHINT // t1 t2 '20=1 SINGLE // t1 t2 '18 - gTup SETGLOB + $gTup SETGLOB 2 PUSHINT // t1 t2 '21=2 - gTup GETGLOB // t1 t2 '21=2 g_gTup + $gTup GETGLOB // t1 t2 '21=2 g_gTup SWAP // t1 t2 g_gTup '21=2 0 SETINDEX // t1 t2 g_gTup - gTup SETGLOB + $gTup SETGLOB x{} PUSHSLICE // t1 t2 '25 STRDUMP DROP 1 PUSHINT // t1 t2 '28=1 2 PUSHINT // t1 t2 '26=1 '27=2 PAIR - gTens SETGLOB + $gTens SETGLOB 4 PUSHINT // t1 t2 g_gTens.1=4 - gTens GETGLOB + $gTens GETGLOB UNPAIR // t1 t2 g_gTens.1=4 g_gTens.0 g_gTens.1 DROP // t1 t2 g_gTens.1=4 g_gTens.0 SWAP // t1 t2 g_gTens.0 g_gTens.1=4 PAIR - gTens SETGLOB + $gTens SETGLOB x{} PUSHSLICE // t1 t2 '36 STRDUMP DROP - gTup GETGLOB // t1 t2 g_gTup - gTens GETGLOB + $gTup GETGLOB // t1 t2 g_gTup + $gTens GETGLOB UNPAIR // t1 t2 g_gTup g_gTens.0 g_gTens.1 }> """ @fif_codegen """ - testCodegenNoPureIndexedAccess PROC:<{ + testCodegenNoPureIndexedAccess() PROC:<{ 0 PUSHINT // '8=0 }> """ @fif_codegen """ - testCodegenIndexPostfix1 PROC:<{ // x.0 x.1 + testCodegenIndexPostfix1() PROC:<{ // x.0 x.1 // ab.1 ab.0 SWAP // ab.0 ab.1 }> @@ -371,7 +371,7 @@ fun main(){} @fif_codegen """ - testCodegenIndexPostfix2 PROC:<{ // x.0 x.1.0 x.1.1 x.2 + testCodegenIndexPostfix2() PROC:<{ // x.0 x.1.0 x.1.1 x.2 s2 POP // y.0 y.2 y.1.1 s1 s2 XCHG // y.2 y.0 y.1.1 }> diff --git a/tolk-tester/tests/inline-tests.tolk b/tolk-tester/tests/inline-tests.tolk index 0645546a60..243652b462 100644 --- a/tolk-tester/tests/inline-tests.tolk +++ b/tolk-tester/tests/inline-tests.tolk @@ -450,7 +450,7 @@ fun usedIn10ButDeclaredBelow(x: int) { @fif_codegen_avoid Storage.save @fif_codegen """ - test5 PROC:<{ // + test5() PROC:<{ // c4 PUSH // '7 CTOS // s LDMSGADDR // '11 s @@ -458,7 +458,7 @@ fun usedIn10ButDeclaredBelow(x: int) { @fif_codegen """ - test6 PROC:<{ + test6() PROC:<{ 11 PUSHINT DUP }> @@ -466,7 +466,7 @@ fun usedIn10ButDeclaredBelow(x: int) { @fif_codegen """ - test8 PROC:<{ // + test8() PROC:<{ // 5 PUSHINT NEWC 32 STU // '3 @@ -480,7 +480,7 @@ fun usedIn10ButDeclaredBelow(x: int) { @fif_codegen """ - test9 PROC:<{ // x + test9() PROC:<{ // x 0 PUSHINT // p1.x p1.y=0 s0 s1 PUSH2 // p1.x p1.y=0 p2.x=0 p2.y NIL // x p1.y=0 p2.x=0 p2.y self @@ -506,7 +506,7 @@ fun usedIn10ButDeclaredBelow(x: int) { @fif_codegen """ - test10 PROC:<{ + test10() PROC:<{ 5 PUSHINT // '3=5 DUP // '3=5 y=5 5 PUSHINT @@ -518,7 +518,7 @@ fun usedIn10ButDeclaredBelow(x: int) { @fif_codegen """ - test12 PROC:<{ // x + test12() PROC:<{ // x 40 PUSHINT // x r1 SWAP // r1 x 2 ADDCONST // r1 y @@ -527,7 +527,7 @@ fun usedIn10ButDeclaredBelow(x: int) { @fif_codegen """ - wrapperNotInlined PROC:<{ // x + wrapperNotInlined() PROC:<{ // x DUP // x x ISNULL // x '1 IFNOTJMP:<{ // x @@ -538,7 +538,7 @@ fun usedIn10ButDeclaredBelow(x: int) { @fif_codegen """ - wrap16 PROC:<{ + wrap16() PROC:<{ IFJMP:<{ 139 PUSHINT OVER diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index bf39811125..9cdc469889 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -249,13 +249,13 @@ fun main() { @testcase | 114 | 5 | (null) (null) 0 @testcase | 114 | 15 | 15 2 typeid-2 -@fif_codegen DECLPROC assign0 -@fif_codegen DECLPROC assign0 -@fif_codegen DECLPROC assign0 +@fif_codegen DECLPROC assign0() +@fif_codegen DECLPROC assign0() +@fif_codegen DECLPROC assign0() @fif_codegen """ - test12 PROC:<{ + test12() PROC:<{ 100000000 PUSHINT // '4=100000000 }> """ diff --git a/tolk-tester/tests/lazy-algo-tests.tolk b/tolk-tester/tests/lazy-algo-tests.tolk index 363640ed37..27214077b3 100644 --- a/tolk-tester/tests/lazy-algo-tests.tolk +++ b/tolk-tester/tests/lazy-algo-tests.tolk @@ -1494,7 +1494,7 @@ fun main() { @fif_codegen """ - algo40 PROC:<{ + algo40() PROC:<{ x{0102} PUSHSLICE 8 LDI NIP diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index 36d7491d48..b7eba14fac 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -1686,7 +1686,7 @@ fun main() { @fif_codegen """ - demo101 PROC:<{ // + demo101() PROC:<{ // x{0102} PUSHSLICE // lazyS 8 LDI // '9 lazyS DROP // p.x @@ -1695,7 +1695,7 @@ fun main() { @fif_codegen """ - demo103 PROC:<{ // + demo103() PROC:<{ // x{0102} PUSHSLICE // lazyS 8 PUSHINT SDSKIPFIRST // lazyS @@ -1705,7 +1705,7 @@ fun main() { @fif_codegen """ - demo106 PROC:<{ // s + demo106() PROC:<{ // s // lazyS PUSHNULL // lazyS x SWAP // x lazyS @@ -1716,7 +1716,7 @@ fun main() { @fif_codegen """ - demo109 PROC:<{ // + demo109() PROC:<{ // x{8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_} PUSHSLICE // lazyS 65 PUSHINT SDSKIPFIRST // lazyS @@ -1727,7 +1727,7 @@ fun main() { @fif_codegen """ - demo113 PROC:<{ + demo113() PROC:<{ // lazyS x{01} SDBEGINSQ IF:<{ @@ -1759,7 +1759,7 @@ fun main() { @fif_codegen """ - demo134 PROC:<{ // + demo134() PROC:<{ // x{010f} PUSHSLICE // lazyS x{01} SDBEGINSQ // lazyS '6 134 THROWIFNOT // lazyS @@ -1770,7 +1770,7 @@ fun main() { @fif_codegen """ - demo136 PROC:<{ + demo136() PROC:<{ x{0102} PUSHSLICE SWAP IFJMP:<{ @@ -1786,7 +1786,7 @@ fun main() { @fif_codegen """ - demo146 PROC:<{ + demo146() PROC:<{ x{0000000} PUSHSLICE LDOPTREF DROP @@ -1796,7 +1796,7 @@ fun main() { @fif_codegen """ - demo201 PROC:<{ + demo201() PROC:<{ x{0102} PUSHSLICE DUP 8 LDI @@ -1819,7 +1819,7 @@ fun main() { @fif_codegen """ - demo203 PROC:<{ // + demo203() PROC:<{ // x{010203040506070809} PUSHSLICE // lazyS DUP // '14 lazyS 8 LDU // '14 '16 lazyS @@ -1834,7 +1834,7 @@ fun main() { @fif_codegen """ - demo206 PROC:<{ // + demo206() PROC:<{ // x{010203040506070809} PUSHSLICE // lazyS DUP // '14 lazyS 8 LDU // '14 o.f1 lazyS @@ -1851,7 +1851,7 @@ fun main() { @fif_codegen """ - demo215 PROC:<{ + demo215() PROC:<{ x{0102} PUSHSLICE 8 LDI 8 LDI @@ -1865,7 +1865,7 @@ fun main() { @fif_codegen """ - demo302 PROC:<{ + demo302() PROC:<{ x{0102} PUSHSLICE 8 LDI NIP @@ -1877,9 +1877,9 @@ fun main() { @fif_codegen """ - demo311 PROC:<{ // + demo311() PROC:<{ // 777 PUSHINT // '6=777 - contract.getFakeData CALLDICT // '7 + contract.getFakeData() CALLDICT // '7 CTOS // lazyS 1 LDI // '13 lazyS NIP // lazyS @@ -1899,7 +1899,7 @@ fun main() { @fif_codegen """ - demo404 PROC:<{ + demo404() PROC:<{ // lazyS x{12345678} SDBEGINSQ IF:<{ @@ -1939,7 +1939,7 @@ fun main() { @fif_codegen """ - demo410 PROC:<{ + demo410() PROC:<{ // lazyS x{01} SDBEGINSQ IFJMP:<{ @@ -1965,7 +1965,7 @@ fun main() { @fif_codegen """ - demo411 PROC:<{ // s + demo411() PROC:<{ // s // lazyS 1 LDU // '10 lazyS DROP // '10 @@ -1977,5 +1977,5 @@ fun main() { }> """ -@fif_codegen DECLPROC pushToGlobalTup +@fif_codegen DECLPROC pushToGlobalTup() */ diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk index c925b8c1a8..8242d5ff4e 100644 --- a/tolk-tester/tests/logical-operators.tolk +++ b/tolk-tester/tests/logical-operators.tolk @@ -217,7 +217,7 @@ fun main() { @fif_codegen """ - simpleAllConst PROC:<{ + simpleAllConst() PROC:<{ TRUE 0 PUSHINT TRUE @@ -232,7 +232,7 @@ fun main() { @fif_codegen """ - compileTimeEval1 PROC:<{ // x + compileTimeEval1() PROC:<{ // x DUP // x x 0 EQINT // x '1 FALSE // x '1 '4 @@ -248,7 +248,7 @@ fun main() { @fif_codegen """ - withIfNot PROC:<{ // x y + withIfNot() PROC:<{ // x y c2 SAVE SAMEALTSAVE OVER // x y x @@ -268,7 +268,7 @@ fun main() { @fif_codegen """ - testAndConstCodegen PROC:<{ + testAndConstCodegen() PROC:<{ FALSE 0 PUSHINT DUP @@ -290,7 +290,7 @@ fun main() { @fif_codegen """ - testOrConstCodegen PROC:<{ + testOrConstCodegen() PROC:<{ -1 PUSHINT TRUE FALSE @@ -315,7 +315,7 @@ For example, `a && b` can be expressed without IFs. These are moments of future optimizations. For now, it's more than enough. @fif_codegen """ - testAndSimpleCodegen PROC:<{ // a b + testAndSimpleCodegen() PROC:<{ // a b SWAP // b a IF:<{ // b 0 NEQINT // '2 @@ -328,7 +328,7 @@ These are moments of future optimizations. For now, it's more than enough. @fif_codegen """ - testOrSimpleCodegen PROC:<{ // a b + testOrSimpleCodegen() PROC:<{ // a b SWAP // b a 0 GTINT // b '3 IF:<{ // b @@ -343,7 +343,7 @@ These are moments of future optimizations. For now, it's more than enough. @fif_codegen """ - testConvertIfToIfnot PROC:<{ // x + testConvertIfToIfnot() PROC:<{ // x DUP // x x 100 THROWIF DUP // x x diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk index ca3e21d839..489cdf7489 100644 --- a/tolk-tester/tests/match-by-expr-tests.tolk +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -181,7 +181,7 @@ fun main() { @fif_codegen """ - test1 PROC:<{ // x + test1() PROC:<{ // x DUP // x x 1 EQINT // x '3 IF:<{ // x @@ -207,16 +207,16 @@ fun main() { @fif_codegen """ - test2 PROC:<{ + test2() PROC:<{ 300 PUSHINT }> """ @fif_codegen """ - test3 PROC:<{ // init + test3() PROC:<{ // init DUP // init init - isGt10 CALLDICT // init '2 + isGt10() CALLDICT // init '2 -1 EQINT // init '5 IF:<{ // init 10 PUSHINT // init '3=10 @@ -227,7 +227,7 @@ fun main() { @fif_codegen """ - test8 PROC:<{ // x + test8() PROC:<{ // x DROP // 30 PUSHINT // r1 }> @@ -235,29 +235,29 @@ fun main() { @fif_codegen """ - helper12 PROC:<{ + helper12() PROC:<{ DUP 1 EQINT IFJMP:<{ DROP 1 PUSHINT - g12 SETGLOB + $g12 SETGLOB }> DUP 2 EQINT IFJMP:<{ DROP 2 PUSHINT - g12 SETGLOB + $g12 SETGLOB }> DUP 3 EQINT IFJMP:<{ DROP 3 PUSHINT - g12 SETGLOB + $g12 SETGLOB }> - g12 SETGLOB + $g12 SETGLOB }> """ diff --git a/tolk-tester/tests/methods-tests.tolk b/tolk-tester/tests/methods-tests.tolk index 83118ef369..db3c3e9cdd 100644 --- a/tolk-tester/tests/methods-tests.tolk +++ b/tolk-tester/tests/methods-tests.tolk @@ -309,7 +309,7 @@ fun main() {} @testcase | 117 | | 3 @testcase | 118 | | 10 -@fif_codegen DECLPROC Pair.createFrom -@fif_codegen DECLPROC Pair.compareWith -@fif_codegen DECLPROC CounterMsg.onInternalMessage +@fif_codegen DECLPROC Pair.createFrom() +@fif_codegen DECLPROC Pair.compareWith() +@fif_codegen DECLPROC CounterMsg.onInternalMessage() */ diff --git a/tolk-tester/tests/mutate-methods.tolk b/tolk-tester/tests/mutate-methods.tolk index a53b2c15c0..892a625cec 100644 --- a/tolk-tester/tests/mutate-methods.tolk +++ b/tolk-tester/tests/mutate-methods.tolk @@ -326,26 +326,26 @@ fun main(){} @fif_codegen """ - int.incrementInPlace PROC:<{ // self byValue + int.incrementInPlace() PROC:<{ // self byValue ADD // self }> """ @fif_codegen """ - testIncrement2 PROC:<{ + testIncrement2() PROC:<{ ... - incrementTwoInPlace CALLDICT // x y sum1 + incrementTwoInPlace() CALLDICT // x y sum1 -ROT 10 PUSHINT // sum1 x y '11=10 - int.incrementTwoInPlace CALLDICT // sum1 x y sum2 + int.incrementTwoInPlace() CALLDICT // sum1 x y sum2 s1 s3 s0 XCHG3 // x y sum1 sum2 }> """ @fif_codegen """ - load_next PROC:<{ // cs + load_next() PROC:<{ // cs 32 LDI // '4 cs SWAP // cs '4 }> @@ -353,14 +353,14 @@ fun main(){} @fif_codegen """ - testStoreUintPureUnusedResult PROC:<{ + testStoreUintPureUnusedResult() PROC:<{ 0 PUSHINT // '11=0 }> """ @fif_codegen """ - testStoreUintImpureUnusedResult PROC:<{ + testStoreUintImpureUnusedResult() PROC:<{ NEWC // b STIX // '2 DROP // diff --git a/tolk-tester/tests/no-spaces.tolk b/tolk-tester/tests/no-spaces.tolk index 016cf478a5..732de8011d 100644 --- a/tolk-tester/tests/no-spaces.tolk +++ b/tolk-tester/tests/no-spaces.tolk @@ -80,14 +80,14 @@ fun`main`(){} @fif_codegen """ - get_+-+1 PROC:<{ + get_+-+1() PROC:<{ -1 PUSHINT }> """ @fif_codegen """ - unary+bitwise-constant PROC:<{ + unary+bitwise-constant() PROC:<{ -4 PUSHINT 6 PUSHINT -4 PUSHINT @@ -97,14 +97,14 @@ fun`main`(){} @fif_codegen """ - unary_const_check PROC:<{ + unary_const_check() PROC:<{ -1 PUSHINT // fst1=-1 DUP // fst1=-1 snd1=-1 2 PUSHINT // fst1=-1 snd1=-1 trd1=2 s1 s1 s0 PUSH3 // fst1=-1 snd1=-1 trd1=2 fst2=-1 snd2=-1 trd2=2 - add3 CALLDICT // fst1=-1 snd1=-1 trd1=2 '13 + add3() CALLDICT // fst1=-1 snd1=-1 trd1=2 '13 3 -ROLL // '13 fst1=-1 snd1=-1 trd1=2 - add3 CALLDICT // '13 '14 + add3() CALLDICT // '13 '14 PAIR // '12 }> """ diff --git a/tolk-tester/tests/null-keyword.tolk b/tolk-tester/tests/null-keyword.tolk index 98b90811f2..c36a66c26b 100644 --- a/tolk-tester/tests/null-keyword.tolk +++ b/tolk-tester/tests/null-keyword.tolk @@ -92,7 +92,7 @@ fun main() { @fif_codegen """ - test1 PROC:<{ + test1() PROC:<{ PUSHNULL // numbers 1 PUSHINT // numbers '2=1 SWAP // '2=1 numbers @@ -113,14 +113,14 @@ fun main() { @fif_codegen """ - main PROC:<{ + main() PROC:<{ 1 PUSHINT // '3=1 }> """ @fif_codegen """ - test7 PROC:<{ + test7() PROC:<{ ... LDOPTREF // b '9 '8 DROP // b c diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index 56e111b827..5b03945425 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -555,7 +555,7 @@ fun main(){} @fif_codegen """ - isTensorNull PROC:<{ // t.0 t.1 t.UTag + isTensorNull() PROC:<{// t.0 t.1 t.UTag 2 1 BLKDROP2 // t.UTag 0 EQINT // '3 }> @@ -563,14 +563,14 @@ fun main(){} @fif_codegen """ - test113 PROC:<{ + test113() PROC:<{ 1 PUSHINT // '2=1 PUSHNULL // '2=1 '3 PAIR // t 1 INDEX // '5 PUSHNULL // '5 '6 0 PUSHINT // '5 '6 '7=0 - isTensorNull CALLDICT // '8 + isTensorNull() CALLDICT // '8 }> """ */ diff --git a/tolk-tester/tests/op-priority.tolk b/tolk-tester/tests/op-priority.tolk index 27b30c988d..e635d54234 100644 --- a/tolk-tester/tests/op-priority.tolk +++ b/tolk-tester/tests/op-priority.tolk @@ -95,21 +95,21 @@ fun main() { @fif_codegen """ - unary_minus_1 PROC:<{ // a b c + unary_minus_1() PROC:<{ // a b c -ROT // c a b ADD // c '3 NEGATE // c '4 SWAP // '4 c MUL // '5 }> - unary_minus_2 PROC:<{ // a b c + unary_minus_2() PROC:<{ // a b c -ROT // c a b ADD // c '3 NEGATE // c '4 SWAP // '4 c MUL // '5 }> - unary_minus_3 PROC:<{ // a b c + unary_minus_3() PROC:<{ // a b c -ROT // c a b ADD // c '3 SWAP // '3 c diff --git a/tolk-tester/tests/pack-unpack-1.tolk b/tolk-tester/tests/pack-unpack-1.tolk index 379055c46f..68ce0bc107 100644 --- a/tolk-tester/tests/pack-unpack-1.tolk +++ b/tolk-tester/tests/pack-unpack-1.tolk @@ -106,7 +106,7 @@ fun main(c: cell) { @fif_codegen """ - test3 PROC:<{ + test3() PROC:<{ 42949672980 PUSHINT NEWC 64 STI // b diff --git a/tolk-tester/tests/pack-unpack-4.tolk b/tolk-tester/tests/pack-unpack-4.tolk index d2bcfb6187..6d913f584f 100644 --- a/tolk-tester/tests/pack-unpack-4.tolk +++ b/tolk-tester/tests/pack-unpack-4.tolk @@ -124,7 +124,7 @@ fun main() {} @fif_codegen """ - test6 PROC:<{ // + test6() PROC:<{ // b{010000000000001111} PUSHSLICE // s b{00} SDBEGINSQ // s '8 IF:<{ // s diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk index 34e40578cf..51c1f7c133 100644 --- a/tolk-tester/tests/pack-unpack-6.tolk +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -130,7 +130,7 @@ RETALT @fif_codegen """ - test3 PROC:<{ + test3() PROC:<{ x{0109ab} PUSHSLICE NEWC STSLICE @@ -160,7 +160,7 @@ IF:<{ @fif_codegen """ - test5 PROC:<{ + test5() PROC:<{ x{11} PUSHSLICE NEWC OVER @@ -173,7 +173,7 @@ IF:<{ @fif_codegen """ - test6 PROC:<{ + test6() PROC:<{ x{11} PUSHSLICE NEWC STSLICE @@ -183,7 +183,7 @@ IF:<{ @fif_codegen """ - test8 PROC:<{ // + test8() PROC:<{ // x{010f} PUSHSLICE // '0 16 PUSHPOW2DEC // s '2=65535 SWAP // '2=65535 s @@ -197,7 +197,7 @@ IF:<{ @fif_codegen """ - test9 PROC:<{ + test9() PROC:<{ x{010f} PUSHSLICE x{01} SDBEGINSQ IF:<{ diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index 36fb2ec2b9..c489527ade 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -241,7 +241,7 @@ fun main() { @fif_codegen """ - check0 PROC:<{ // addr + check0() PROC:<{ // addr REWRITESTDADDR DROP // '2 0 EQINT // '4 @@ -251,7 +251,7 @@ fun main() { @fif_codegen """ - codegenAddrEq PROC:<{ // a b + codegenAddrEq() PROC:<{ // a b 2DUP // a b a b SDEQ // a b '2 IFJMP:<{ // a b diff --git a/tolk-tester/tests/remove-unused-functions.tolk b/tolk-tester/tests/remove-unused-functions.tolk index cd72fd08c0..a6b910009f 100644 --- a/tolk-tester/tests/remove-unused-functions.tolk +++ b/tolk-tester/tests/remove-unused-functions.tolk @@ -32,18 +32,18 @@ fun main(): (int, int, int) { @testcase | 0 | | 3 10 20 -@fif_codegen DECLPROC used_as_noncall1 -@fif_codegen DECLGLOBVAR used_gv +@fif_codegen DECLPROC used_as_noncall1() +@fif_codegen DECLGLOBVAR $used_gv -@fif_codegen_avoid DECLPROC unused1 -@fif_codegen_avoid DECLPROC unused2 -@fif_codegen_avoid DECLPROC unused3 -@fif_codegen_avoid DECLGLOBVAR unused_gv +@fif_codegen_avoid DECLPROC unused1() +@fif_codegen_avoid DECLPROC unused2() +@fif_codegen_avoid DECLPROC unused3() +@fif_codegen_avoid DECLGLOBVAR $unused_gv Note, that `usedButOptimizedOut()` (a pure function which result is unused) is currently codegenerated, since it's formally reachable. This is because optimizing code is a moment of codegen for now (later than marking unused symbols). -@fif_codegen DECLPROC usedButOptimizedOut -@fif_codegen_avoid usedButOptimizedOut CALLDICT +@fif_codegen DECLPROC usedButOptimizedOut() +@fif_codegen_avoid usedButOptimizedOut() CALLDICT */ diff --git a/tolk-tester/tests/self-keyword.tolk b/tolk-tester/tests/self-keyword.tolk index e5941dc81f..5fa5538ddd 100644 --- a/tolk-tester/tests/self-keyword.tolk +++ b/tolk-tester/tests/self-keyword.tolk @@ -226,27 +226,29 @@ fun main() { } @fif_codegen """ - int.incChained PROC:<{ // self - INC // self + int.incChained() PROC:<{ // self + INC // self }> - int.incChained2 PROC:<{ // self - int.incChained CALLDICT // self + int.incChained2() PROC:<{ // self + int.incChained() CALLDICT // self }> - int.incChained3 PROC:<{ // self - int.incChained CALLDICT // self + int.incChained3() PROC:<{ // self + int.incChained() CALLDICT // self }> - int.incChained4 PROC:<{ // self - int.incChained CALLDICT // self + int.incChained4() PROC:<{ // self + int.incChained() CALLDICT // self }> """ @fif_codegen """ - testIncChainedCodegen PROC:<{ // x - int.incChained CALLDICT // x - int.incChained2 CALLDICT // x - int.incChained3 CALLDICT // x - int.incChained4 CALLDICT // x + testIncChainedCodegen() PROC:<{ // x + int.incChained() CALLDICT // x + int.incChained2() CALLDICT // x + int.incChained3() CALLDICT // x + int.incChained4() CALLDICT // x }> """ + +@fif_codegen int.checkNotEqChainableMutateAnother() PREPAREDICT */ diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 247431eb0c..1a0f96a0fa 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -749,12 +749,12 @@ fun main(x: int?): int { @stderr warning: unreachable code @stderr var t2 redef = getNullableInt()!; -@fif_codegen eq55 PROC:<{ -@fif_codegen eq55 PROC:<{ +@fif_codegen eq55() PROC:<{ +@fif_codegen eq55() PROC:<{ @fif_codegen """ - test60 PROC:<{ + test60() PROC:<{ 101 PUSHINT // cc1 109 PUSHINT // cc1 cc2 }> diff --git a/tolk-tester/tests/some-tests-2.tolk b/tolk-tester/tests/some-tests-2.tolk index 3568da56ca..9a4cb8cd02 100644 --- a/tolk-tester/tests/some-tests-2.tolk +++ b/tolk-tester/tests/some-tests-2.tolk @@ -114,14 +114,14 @@ fun test96(p: Point) { @fif_codegen """ - touchCodegen2 PROC:<{ - get10 CALLDICT // f + touchCodegen2() PROC:<{ + get10() CALLDICT // f }> """ @fif_codegen """ - testDumpDontPolluteStack PROC:<{ + testDumpDontPolluteStack() PROC:<{ ... DUMPSTK x{6d79} PUSHSLICE // f s '5 @@ -132,7 +132,7 @@ fun test96(p: Point) { @fif_codegen """ - testStartBalanceCodegen1 PROC:<{ + testStartBalanceCodegen1() PROC:<{ BALANCE // t FIRST // first }> @@ -140,7 +140,7 @@ fun test96(p: Point) { @fif_codegen """ - testStartBalanceCodegen2 PROC:<{ + testStartBalanceCodegen2() PROC:<{ BALANCE FIRST // first }> @@ -148,24 +148,24 @@ fun test96(p: Point) { @fif_codegen """ - test95 PROC:<{ + test95() PROC:<{ ... - next GETGLOB // g_next + $next GETGLOB // g_next 3 PUSHINT // g_next '14=3 4 PUSHINT // g_next '14=3 '15=4 5 PUSHINT // g_next '14=3 '15=4 '16=5 TRIPLE // '10 '11 SWAP - cur SETGLOB - next SETGLOB - cur GETGLOB // g_cur - next GETGLOB // g_cur g_next + $cur SETGLOB + $next SETGLOB + $cur GETGLOB // g_cur + $next GETGLOB // g_cur g_next }> """ @fif_codegen """ - test96 PROC:<{ + test96() PROC:<{ s1 DUMP s0 DUMP 2 BLKDROP }> """ diff --git a/tolk-tester/tests/special-fun-names.tolk b/tolk-tester/tests/special-fun-names.tolk index 8fae6d5dbf..11c34f98d9 100644 --- a/tolk-tester/tests/special-fun-names.tolk +++ b/tolk-tester/tests/special-fun-names.tolk @@ -4,6 +4,23 @@ fun onRunTickTock() { return -2; } fun onSplitPrepare() { return -3; } fun onSplitInstall() { return -4; } +// 'execute' and many other names are reserved words in Fift, +// but since they are codegenerated like `execute()` (it's a valid Fift name!), no collisions +@noinline +fun execute() { return 123 } +@noinline +fun `PUSHINT`() { return 456 } +@noinline +fun `@havebits`() { return 789 } + +global swap: int; + +@method_id(101) +fun test101() { + swap = 1; + return execute() + `PUSHINT`() + `@havebits`() + swap; +} + /** @experimental_options remove-unused-functions @@ -12,13 +29,18 @@ fun onSplitInstall() { return -4; } @testcase | -2 | | -2 @testcase | -3 | | -3 @testcase | -4 | | -4 +@testcase | 101 | | 1369 @fif_codegen """ - 0 DECLMETHOD onInternalMessage - -1 DECLMETHOD onExternalMessage - -2 DECLMETHOD onRunTickTock - -3 DECLMETHOD onSplitPrepare - -4 DECLMETHOD onSplitInstall + 0 DECLMETHOD onInternalMessage() + -1 DECLMETHOD onExternalMessage() + -2 DECLMETHOD onRunTickTock() + -3 DECLMETHOD onSplitPrepare() + -4 DECLMETHOD onSplitInstall() """ + +@fif_codegen DECLPROC execute() +@fif_codegen execute() CALLDICT +@fif_codegen DECLGLOBVAR $swap */ diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 7c224bab4b..18d596c34d 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -627,7 +627,7 @@ type PointAlias = Point; @fif_codegen """ - test6 PROC:<{ + test6() PROC:<{ 10 PUSHINT // p.x 20 PUSHINT // p.x p.y }> @@ -635,7 +635,7 @@ type PointAlias = Point; @fif_codegen """ - sumXY PROC:<{ // point.x point.y + sumXY() PROC:<{ // point.x point.y ADD // '2 }> """ @@ -643,7 +643,7 @@ type PointAlias = Point; @fif_codegen """ - test23 PROC:<{ // p.x p.y + test23() PROC:<{ // p.x p.y 2DROP 20 PUSHINT // '10=20 }> @@ -651,27 +651,27 @@ type PointAlias = Point; @fif_codegen """ - test29 PROC:<{ + test29() PROC:<{ PUSHNULL // '5 }> """ @fif_codegen """ - test35 PROC:<{ // - getXPure CALLDICT // '2 - getYPure CALLDICT // p.x p.y + test35() PROC:<{ // + getXPure() CALLDICT // '2 + getYPure() CALLDICT // p.x p.y }> """ @fif_codegen """ - test36 PROC:<{ // + test36() PROC:<{ // NIL // '0 - t_impure SETGLOB // - getYImpure CALLDICT // '4 - getXImpure CALLDICT // p.y p.x - t_impure GETGLOB // p.y p.x g_t_impure + $t_impure SETGLOB // + getYImpure() CALLDICT // '4 + getXImpure() CALLDICT // p.y p.x + $t_impure GETGLOB // p.y p.x g_t_impure s1 s2 XCHG // p.x p.y g_t_impure }> """ diff --git a/tolk-tester/tests/try-catch-tests.tolk b/tolk-tester/tests/try-catch-tests.tolk index b1878955e9..04e32531cb 100644 --- a/tolk-tester/tests/try-catch-tests.tolk +++ b/tolk-tester/tests/try-catch-tests.tolk @@ -298,7 +298,7 @@ fun main() { @fif_codegen """ - testCodegen1 PROC:<{ // x + testCodegen1() PROC:<{ // x DUP // x x 10 GTINT // x '2 IFJMP:<{ // x @@ -316,17 +316,17 @@ fun main() { @fif_codegen """ - testCodegen2 PROC:<{ // x + testCodegen2() PROC:<{ // x DUP // x x 10 GTINT // x '2 IFJMP:<{ // x DROP // - alwaysThrow123 CALLDICT + alwaysThrow123() CALLDICT }> // x 10 LESSINT // '5 IFJMP:<{ // FALSE // '6 - anotherNever CALLDICT + anotherNever() CALLDICT }> // 0 PUSHINT // '8=0 }> @@ -334,7 +334,7 @@ fun main() { @fif_codegen """ - testCodegen3 PROC:<{ // numberId paramVal + testCodegen3() PROC:<{ // numberId paramVal SWAP -1000 PUSHINT // paramVal numberId '2=-1000 EQUAL // paramVal '3 diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk index 5d725ecf5a..3fc5345050 100644 --- a/tolk-tester/tests/type-aliases-tests.tolk +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -212,7 +212,7 @@ fun main(x: MInt, y: MInt?) { @fif_codegen """ - test9 PROC:<{ // b + test9() PROC:<{ // b DUP // b b IFNOTJMP:<{ // b NOT // '2 diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 53e6caefa2..5dc3c9fca6 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -854,14 +854,14 @@ fun main() { @fif_codegen """ - test26 PROC:<{ + test26() PROC:<{ 42 PUSHINT // result }> """ @fif_codegen """ - checkSimple PROC:<{ // a.USlot1 a.UTag + checkSimple() PROC:<{ // a.USlot1 a.UTag NIP // a.UTag DUP // a.UTag a.UTag 1 EQINT // a.UTag '3 @@ -881,7 +881,7 @@ fun main() { @fif_codegen """ - test61 PROC:<{ + test61() PROC:<{ NIP DUP 46 EQINT @@ -897,6 +897,6 @@ fun main() { }> """ -@fif_codegen DECLPROC checkGeneric3 +@fif_codegen DECLPROC checkGeneric3() */ diff --git a/tolk-tester/tests/use-before-declare.tolk b/tolk-tester/tests/use-before-declare.tolk index 42afe8b00a..0c59c67158 100644 --- a/tolk-tester/tests/use-before-declare.tolk +++ b/tolk-tester/tests/use-before-declare.tolk @@ -48,14 +48,14 @@ const first = 1; @fif_codegen """ - test1 PROC:<{ + test1() PROC:<{ 30 PUSHINT // '10 }> """ @fif_codegen """ - test2 PROC:<{ + test2() PROC:<{ 2 PUSHINT // '2 }> """ diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index d1361bd0c8..702c9fddc9 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -325,6 +325,18 @@ fun testSavingFunWithDefaults() { return cc(1, 2, 3); } +@noinline +fun tupleCreator() { return createEmptyTuple() } +@noinline +fun getTupleCreator() { return tupleCreator } + +@method_id(120) +fun testNoInlineSaving() { + var creator = getTupleCreator(); + __expect_type(creator, "() -> tuple"); + return creator(); +} + fun main() {} @@ -355,4 +367,5 @@ fun main() {} @testcase | 117 | | 6 8 @testcase | 118 | | 0 10 -1 (null) @testcase | 119 | | 6 +@testcase | 120 | | [] */ diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp index d392a58b8b..f5ef89de3b 100644 --- a/tolk/codegen.cpp +++ b/tolk/codegen.cpp @@ -338,7 +338,7 @@ bool Op::generate_code_step(Stack& stack) { if (!used || disabled()) { return true; } - stack.o << AsmOp::Custom(loc, g_sym->name + " GETGLOB", 0, 1); + stack.o << AsmOp::Custom(loc, "$" + g_sym->name + " GETGLOB", 0, 1); if (left.size() != 1) { tolk_assert(left.size() <= 15); stack.o << AsmOp::UnTuple(loc, (int)left.size()); @@ -376,7 +376,7 @@ bool Op::generate_code_step(Stack& stack) { std::get(f_sym->body)->compile(stack.o, res, args0, loc); // compile res := f (args0) } } else { - stack.o << AsmOp::Custom(loc, f_sym->name + " CALLDICT", (int)right.size(), (int)left.size()); + stack.o << AsmOp::Custom(loc, f_sym->name + "() CALLDICT", (int)right.size(), (int)left.size()); } stack.o.undent(); stack.o << AsmOp::Custom({}, "}>"); @@ -518,12 +518,12 @@ bool Op::generate_code_step(Stack& stack) { } } else { if (f_sym->inline_mode == FunctionInlineMode::inlineViaFif || f_sym->inline_mode == FunctionInlineMode::inlineRef) { - stack.o << AsmOp::Custom(loc, f_sym->name + " INLINECALLDICT", (int)right.size(), (int)left.size()); + stack.o << AsmOp::Custom(loc, f_sym->name + "() INLINECALLDICT", (int)right.size(), (int)left.size()); } else if (f_sym->is_code_function() && std::get(f_sym->body)->code->require_callxargs) { - stack.o << AsmOp::Custom(loc, f_sym->name + (" PREPAREDICT"), 0, 2); + stack.o << AsmOp::Custom(loc, f_sym->name + ("() PREPAREDICT"), 0, 2); exec_callxargs((int)right.size() + 1, (int)left.size()); } else { - stack.o << AsmOp::Custom(loc, f_sym->name + " CALLDICT", (int)right.size(), (int)left.size()); + stack.o << AsmOp::Custom(loc, f_sym->name + "() CALLDICT", (int)right.size(), (int)left.size()); } } stack.modified(); @@ -551,7 +551,7 @@ bool Op::generate_code_step(Stack& stack) { stack.o << AsmOp::Tuple(loc, (int)right.size()); } if (!right.empty()) { - stack.o << AsmOp::Custom(loc, g_sym->name + " SETGLOB", 1, 0); + stack.o << AsmOp::Custom(loc, "$" + g_sym->name + " SETGLOB", 1, 0); stack.modified(); } stack.s.resize(k); diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp index f0bd7c5a36..237649f45f 100644 --- a/tolk/pipe-generate-fif-output.cpp +++ b/tolk/pipe-generate-fif-output.cpp @@ -91,13 +91,13 @@ static void generate_output_func(FunctionPtr fun_ref) { } std::cout << std::endl; } - std::cout << " " << fun_ref->name << " PROC" << modifier << ":<{"; + std::cout << " " << fun_ref->name << "() PROC" << modifier << ":<{"; int mode = 0; if (G.settings.stack_layout_comments) { mode |= Stack::_StackComments; size_t len = 2 + fun_ref->name.size() + 5 + std::strlen(modifier) + 3; while (len < 28) { // a bit weird, but okay for now: - std::cout << ' '; // insert space after "xxx PROC" before `// stack state` + std::cout << ' '; // insert space after "xxx() PROC" before `// stack state` len++; // (the first AsmOp-comment that will be code generated) } // space is the same as used to align comments in asmops.cpp std::cout << '\t'; @@ -153,9 +153,9 @@ void pipeline_generate_fif_output_to_std_cout() { std::cout << " "; if (fun_ref->has_tvm_method_id()) { - std::cout << fun_ref->tvm_method_id << " DECLMETHOD " << fun_ref->name << "\n"; + std::cout << fun_ref->tvm_method_id << " DECLMETHOD " << fun_ref->name << "()\n"; } else { - std::cout << "DECLPROC " << fun_ref->name << "\n"; + std::cout << "DECLPROC " << fun_ref->name << "()\n"; } } @@ -180,7 +180,7 @@ void pipeline_generate_fif_output_to_std_cout() { continue; } - std::cout << " " << "DECLGLOBVAR " << var_ref->name << "\n"; + std::cout << " " << "DECLGLOBVAR $" << var_ref->name << "\n"; } for (FunctionPtr fun_ref : G.all_functions) { From 6354018b27c979c9597fcf6ff7880f84bd58758a Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 22 Jun 2025 09:25:12 +0300 Subject: [PATCH 333/524] [Tolk] Better handle nested unknown types at assignment Fix an issue with a ternary operator that returns (and is assigned to) multiple variables --- tolk-tester/tests/dicts-demo.tolk | 13 +++++++++++++ tolk/smart-casts-cfg.cpp | 18 +++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/tolk-tester/tests/dicts-demo.tolk b/tolk-tester/tests/dicts-demo.tolk index 1ff4cd8c3e..f53debb7f7 100644 --- a/tolk-tester/tests/dicts-demo.tolk +++ b/tolk-tester/tests/dicts-demo.tolk @@ -93,6 +93,17 @@ fun test104() { ); } +@method_id(105) +fun test105(takeNext: bool) { + var dict = createEmptyDict(); + dict.uDictSet(32, 0, stringHexToSlice("01")); + dict.uDictSet(32, 8, stringHexToSlice("02")); + var (next, data, found) = takeNext ? dict.uDictGetNext(32, -1) : dict.uDictGetPrev(32, 9); + __expect_type(found, "bool"); + assert(found, 10); + return data!.loadUint(8); +} + fun main() {} /** @@ -102,4 +113,6 @@ fun main() {} @testcase | 102 | | [ [ 4 104 ] [ 3 103 ] [ 2 102 ] [ 1 101 ] ] @testcase | 103 | | 1 1 2 1 3 (null) @testcase | 104 | | 12 34 56 78 0 +@testcase | 105 | -1 | 1 +@testcase | 105 | 0 | 2 */ diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index 6597efe95e..7dd60ee975 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -218,6 +218,22 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b, bool* became_union = nul return resulting_union; } +// when `var v = rhs`, `v` is `unknown` before assignment (before rhs->inferred_type is assigned to it); +// when `var (v1,v2,v3) = rhs`, left side is `(unknown,unknown,unknown)` +static bool is_type_unknown_from_var_lhs_decl(TypePtr t) { + if (t == TypeDataUnknown::create()) { + return true; + } + if (const auto* t_tensor = t->try_as()) { + bool all_unknown = true; + for (TypePtr item : t_tensor->items) { + all_unknown &=is_type_unknown_from_var_lhs_decl(item); + } + return all_unknown; + } + return false; +} + // merge (unify) of two sign states: what sign do we definitely have // it's used on data flow rejoin // example: `if (x > 0) ... else ...`; lca(Positive, NonPositive) = Unknown @@ -263,7 +279,7 @@ BoolState calculate_bool_lca(BoolState a, BoolState b) { void TypeInferringUnifyStrategy::unify_with(TypePtr next, TypePtr dest_hint) { // example: `var r = ... ? int8 : int16`, will be inferred as `int8 | int16` (via unification) // but `var r: int = ... ? int8 : int16`, will be inferred as `int` (it's dest_hint) - if (dest_hint && dest_hint != TypeDataUnknown::create() && !dest_hint->unwrap_alias()->try_as()) { + if (dest_hint && !is_type_unknown_from_var_lhs_decl(dest_hint) && !dest_hint->unwrap_alias()->try_as()) { if (dest_hint->can_rhs_be_assigned(next)) { next = dest_hint; } From 1774c05ac670ec90a712f9f84e2b0b71cacdb8b9 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 14 Apr 2025 22:03:13 +0300 Subject: [PATCH 334/524] [Tolk] Modern `onInternalMessage` and `onBouncedMessage` Instead of "FunC-style" parsing msg_cell, use `in.senderAddress`, `in.originalForwardFee`, that are mapped onto TVM-11 instructions --- crypto/smartcont/tolk-stdlib/common.tolk | 44 +++++- .../smartcont/tolk-stdlib/gas-payments.tolk | 11 +- tolk-tester/tests/cells-slices.tolk | 4 +- tolk-tester/tests/handle-msg-1.tolk | 134 ++++++++++++++++++ tolk-tester/tests/handle-msg-2.tolk | 23 +++ tolk-tester/tests/handle-msg-3.tolk | 32 +++++ tolk-tester/tests/handle-msg-4.tolk | 40 ++++++ tolk-tester/tests/handle-msg-5.tolk | 39 +++++ tolk-tester/tests/handle-msg-6.tolk | 47 ++++++ tolk-tester/tests/handle-msg-7.tolk | 22 +++ .../tests/invalid-declaration/err-1906.tolk | 9 ++ .../tests/invalid-declaration/err-1907.tolk | 8 ++ .../tests/invalid-declaration/err-1908.tolk | 12 ++ .../tests/invalid-semantics/err-4618.tolk | 11 ++ .../tests/invalid-semantics/err-4790.tolk | 11 ++ .../tests/invalid-semantics/err-4890.tolk | 11 ++ tolk-tester/tests/lazy-load-tests.tolk | 2 +- tolk-tester/tests/meaningful-1.tolk | 6 + tolk-tester/tolk-tester.py | 2 + tolk/CMakeLists.txt | 2 + tolk/ast-aux-data.h | 13 ++ tolk/ast-from-tokens.cpp | 18 ++- tolk/ast.cpp | 3 + tolk/ast.h | 2 + tolk/builtins.cpp | 24 ++++ tolk/gen-entrypoints.cpp | 120 ++++++++++++++++ tolk/gen-entrypoints.h | 28 ++++ tolk/pipe-ast-to-legacy.cpp | 11 ++ tolk/pipe-detect-inline-in-place.cpp | 4 +- tolk/pipe-infer-types-and-calls.cpp | 8 ++ tolk/pipe-register-symbols.cpp | 3 + tolk/pipe-transform-on-message.cpp | 119 ++++++++++++++++ tolk/pipeline.h | 1 + tolk/symtable.cpp | 9 ++ tolk/symtable.h | 5 +- tolk/tolk.cpp | 1 + 36 files changed, 825 insertions(+), 14 deletions(-) create mode 100644 tolk-tester/tests/handle-msg-1.tolk create mode 100644 tolk-tester/tests/handle-msg-2.tolk create mode 100644 tolk-tester/tests/handle-msg-3.tolk create mode 100644 tolk-tester/tests/handle-msg-4.tolk create mode 100644 tolk-tester/tests/handle-msg-5.tolk create mode 100644 tolk-tester/tests/handle-msg-6.tolk create mode 100644 tolk-tester/tests/handle-msg-7.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1906.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1907.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1908.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4618.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4790.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4890.tolk create mode 100644 tolk/gen-entrypoints.cpp create mode 100644 tolk/gen-entrypoints.h create mode 100644 tolk/pipe-transform-on-message.cpp diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 3eefbbc5f8..9c57bf18fc 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -843,7 +843,7 @@ fun slice.remainingBitsAndRefsCount(self): (int, int) /// Checks whether a slice is empty (i.e., contains no bits of data and no cell references). @pure -fun slice.isEnd(self): bool +fun slice.isEmpty(self): bool asm "SEMPTY"; /// Checks whether a slice has no bits of data. @@ -1311,6 +1311,45 @@ fun sendRawMessage(msg: cell, mode: int): void Receiving and handling messages. */ +/// InMessage is an input for `onInternalMessage` — when your contract accepts a non-bounced message. +/// Internally, some data exist on the stack, some can be acquired with TVM instructions. +/// The compiler replaces `in.someField` and gets a value correctly. +/// Example: +/// ``` +/// fun onInternalMessage(in: InMessage) { +/// in.senderAddress // actually calls `INMSG_SRC` asm instruction +/// in.body // actually gets from a TVM stack +/// ``` +struct InMessage { + senderAddress: address; // an internal address from which the message arrived + valueCoins: coins; // ton amount attached to an incoming message + valueExtra: dict; // extra currencies attached to an incoming message + originalForwardFee: coins; // fee that was paid by the sender + createdLt: uint64; // logical time when a message was created + createdAt: uint32; // unixtime when a message was created + body: slice; // message body, parse it with `lazy AllowedMsg.fromSlice(in.body)` +} + +/// InMessageBounced is an input for `onBouncedMessage`. +/// Very similar to a non-bounced input [InMessage]. +/// Note, that `bouncedBody` is currently 256 bits: 0xFFFFFFFF + 224 bits from the originally sent message. +/// Parse it with care! +/// Example: +/// ``` +/// fun onBouncedMessage(in: InMessageBounced) { +/// in.bouncedBody.skipBouncedPrefix(); +/// val originalOpcode = in.bouncedBody.loadUint(32); +/// ``` +struct InMessageBounced { + senderAddress: address; // an internal address from which the message was bounced + valueCoins: coins; // ton amount attached to a message + valueExtra: dict; // extra currencies attached to a message + originalForwardFee: coins; // comission that the sender has payed to send this message + createdLt: uint64; // logical time when a message was created (and bounced) + createdAt: uint32; // unixtime when a message was created (and bounced) + bouncedBody: slice; // currently 256 bits: 0xFFFFFFFF + 224 bits sent originally +} + /// Skip 0xFFFFFFFF prefix (when a message is bounced). @pure fun slice.skipBouncedPrefix(mutate self): self @@ -1318,7 +1357,7 @@ fun slice.skipBouncedPrefix(mutate self): self /// Load msgFlags from incoming message body (4 bits). @pure -@deprecated("use `onInternalMessage(in: InMessage)` and `in.isBounced` instead of parsing msgCell manually") +@deprecated("use `onBouncedMessage` handler instead of parsing msgCell flags manually") fun slice.loadMessageFlags(mutate self): int asm( -> 1 0) "4 LDU"; @@ -1349,3 +1388,4 @@ fun slice.skipMessageQueryId(mutate self): self fun builder.storeMessageQueryId(mutate self, queryId: int): self asm(queryId self) "64 STU"; + diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 21de803caa..5aafedbe1d 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -45,16 +45,17 @@ fun calculateGasFeeWithoutFlatPrice(workchain: int8, gasUsed: coins): coins fun calculateStorageFee(workchain: int8, seconds: int, bits: int, cells: int): coins asm(cells bits seconds workchain) "GETSTORAGEFEE"; -/// Calculates amount of nanotoncoins you should pay to send a message of specified size. -fun calculateMessageFee(workchain: int8, bits: int, cells: int): coins +/// Calculates amount of nanotoncoins you should pay to send a message of a specified size. +fun calculateForwardFee(workchain: int8, bits: int, cells: int): coins asm(cells bits workchain) "GETFORWARDFEE"; -/// Same as [calculateMessageFee], but without lump price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) -fun calculateMessageFeeWithoutLumpPrice(workchain: int8, bits: int, cells: int): coins +/// Same as [calculateForwardFee], but without lump price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) +fun calculateForwardFeeWithoutLumpPrice(workchain: int8, bits: int, cells: int): coins asm(cells bits workchain) "GETFORWARDFEESIMPLE"; /// Calculates fee that was paid by the sender of an incoming internal message. -fun calculateOriginalMessageFee(workchain: int8, incomingFwdFee: coins): coins +@deprecated("use modern `onInternalMessage` and access `in.originalForwardFee` directly") +fun calculateOriginalForwardFee(workchain: int8, incomingFwdFee: coins): coins asm(incomingFwdFee workchain) "GETORIGINALFWDFEE"; /// Returns the amount of nanotoncoins current contract debts for storage. ("due" and "debt" are synonyms) diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index fa96bd5295..8fee091b84 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -112,11 +112,11 @@ fun slice.sumNumbersInSlice(mutate self): int { fun test10() { var ref = beginCell().storeInt(100, 32).endCell(); var s: slice = beginCell().storeInt(1, 32).storeInt(2, 32).storeRef(ref).endCell().beginParse(); - var result = (s.remainingBitsCount(), s.sumNumbersInSlice(), s.remainingBitsCount(), s.isEnd(), s.isEndOfBits(), s.isEndOfRefs()); + var result = (s.remainingBitsCount(), s.sumNumbersInSlice(), s.remainingBitsCount(), s.isEmpty(), s.isEndOfBits(), s.isEndOfRefs()); var ref2: cell = s.loadRef(); var s2: slice = ref2.beginParse(); s.assertEnd(); - return (result, s2.loadInt(32), s2.isEnd()); + return (result, s2.loadInt(32), s2.isEmpty()); } @method_id(111) diff --git a/tolk-tester/tests/handle-msg-1.tolk b/tolk-tester/tests/handle-msg-1.tolk new file mode 100644 index 0000000000..1e08082d55 --- /dev/null +++ b/tolk-tester/tests/handle-msg-1.tolk @@ -0,0 +1,134 @@ + +fun setTvmRegisterC7(c7: [tuple]): void + asm "c7 POP"; + +@noinline +fun emulateC7PresentForTests(bounced: bool, senderAddress: address, fwdFee: coins, createdLt: int, createdAt: int, valueCoins: coins, valueExtra: dict) { + var c7_inner = createEmptyTuple(); + repeat (17) { + c7_inner.push(null); + } + + var inmsgparams = createEmptyTuple(); + inmsgparams.push(null); // bounce (not present in InMessage) + inmsgparams.push(bounced); + inmsgparams.push(senderAddress); + inmsgparams.push(fwdFee); + inmsgparams.push(createdLt); + inmsgparams.push(createdAt); + inmsgparams.push(null); // orig value (not present in InMessage) + inmsgparams.push(valueCoins); // in onInternalMessage, from a stack; in onBouncedMessage, from TVM + inmsgparams.push(valueExtra); + inmsgparams.push(null); // state init (not present in InMessage) + c7_inner.push(inmsgparams); + + setTvmRegisterC7([c7_inner]); +} + +fun invokeTest(body: slice, method_id: int): void + asm "c3 PUSH" "EXECUTE"; // stack: body + +@method_id(101) +fun handle1(in: InMessage) { + return in.senderAddress.getWorkchain(); +} + +@method_id(201) +fun entrypoint1() { + emulateC7PresentForTests(true, address("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"), 100, 20, 30, 900, null); + invokeTest("", 101); +} + +@method_id(102) +fun handle2(in: InMessage) { + return (in.valueCoins, in.senderAddress.isInternal()); +} + +@method_id(202) +fun entrypoint2() { + emulateC7PresentForTests(false, address("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"), 100, 20, 30, 888, null); + invokeTest("", 102); +} + +@method_id(103) +fun handle3(`in()`: InMessage) { + return ( + `in()`.senderAddress.getWorkchainAndHash(), + `in()`.valueCoins, + `in()`.valueExtra, + `in()`.createdLt, + `in()`.createdAt, + `in()`.body.isEmpty(), + ) +} + +@method_id(203) +fun entrypoint3() { + emulateC7PresentForTests(true, address("-1:000000000000000000000000000000000000000000000000000000000000FFFF"), 100, 20, 30, 999, null); + invokeTest("00", 103); +} + +fun onInternalMessage(in: InMessage) { + __expect_type(in.body, "slice"); + __expect_type(in.senderAddress, "address"); + __expect_type(in.valueCoins, "coins"); + return in.originalForwardFee; +} + +/** +@testcase | 201 | | 0 +@testcase | 202 | | 888 -1 +@testcase | 203 | | -1 65535 999 (null) 20 30 0 + +@fif_codegen +""" + handle1() PROC:<{ // in.body + DROP // + INMSG_SRC // '1 + REWRITESTDADDR + DROP // '3 + }> +""" + +@fif_codegen +""" + handle2() PROC:<{ // in.body + DROP // + INMSG_VALUE // '1 + INMSG_SRC // '1 '3 + b{10} SDBEGINSQ + NIP // '1 '5 + }> +""" + +@fif_codegen +""" + handle3() PROC:<{ + INMSG_SRC + REWRITESTDADDR + INMSG_VALUE + INMSG_VALUEEXTRA + INMSG_LT + INMSG_UTIME + s0 s6 XCHG + SEMPTY + s5 s6 XCHG + s4 s5 XCHG + s3 s4 XCHG + s1 s3 s0 XCHG3 + }> +""" + +@fif_codegen +""" + onInternalMessage() PROC:<{ // in.body + DROP + INMSG_BOUNCED + 0 THROWIF + INMSG_FWDFEE + 0 PUSHINT + GETORIGINALFWDFEE + }> +""" + + */ diff --git a/tolk-tester/tests/handle-msg-2.tolk b/tolk-tester/tests/handle-msg-2.tolk new file mode 100644 index 0000000000..2f8a8b29f5 --- /dev/null +++ b/tolk-tester/tests/handle-msg-2.tolk @@ -0,0 +1,23 @@ + +@method_id(101) +fun test1() { + return 0; +} + +fun onInternalMessage(in: InMessage) { + return in.valueExtra; +} + +/** +@testcase | 101 | | 0 + +@fif_codegen +""" + onInternalMessage() PROC:<{ + DROP + INMSG_BOUNCED + 0 THROWIF + INMSG_VALUEEXTRA + }> +""" +*/ \ No newline at end of file diff --git a/tolk-tester/tests/handle-msg-3.tolk b/tolk-tester/tests/handle-msg-3.tolk new file mode 100644 index 0000000000..78bb301ddd --- /dev/null +++ b/tolk-tester/tests/handle-msg-3.tolk @@ -0,0 +1,32 @@ + +@method_id(101) +fun test1() { + return 0; +} + +fun onBouncedMessage(in: InMessageBounced) { + in.bouncedBody.skipBouncedPrefix(); + throw in.bouncedBody.loadUint(32); +} + +fun onInternalMessage(in: InMessage) { + return in.body; +} + +/** +@testcase | 101 | | 0 + +@fif_codegen +""" + onInternalMessage() PROC:<{ + INMSG_BOUNCED + IFJMP:<{ + 32 LDU + NIP + 32 LDU + DROP + THROWANY + }> // in.body + }> +""" +*/ \ No newline at end of file diff --git a/tolk-tester/tests/handle-msg-4.tolk b/tolk-tester/tests/handle-msg-4.tolk new file mode 100644 index 0000000000..ac772992dc --- /dev/null +++ b/tolk-tester/tests/handle-msg-4.tolk @@ -0,0 +1,40 @@ + +@method_id(101) +fun test1() { + return 0; +} + +@inline_ref +fun onBouncedMessage(in: InMessageBounced) { + contract.setData(createEmptyCell()); +} + +fun onInternalMessage(in: InMessage) { + return 123; +} + +/** +@testcase | 101 | | 0 + +@fif_codegen +""" + onBouncedMessage() PROCREF:<{ // in.body + DROP + PUSHREF + c4 POP + }> +""" + +@fif_codegen +""" + onInternalMessage() PROC:<{ // in.body + INMSG_BOUNCED // in.body '1 + IFJMP:<{ // in.body + onBouncedMessage() INLINECALLDICT + }> + DROP + 123 PUSHINT + }> +""" + + */ diff --git a/tolk-tester/tests/handle-msg-5.tolk b/tolk-tester/tests/handle-msg-5.tolk new file mode 100644 index 0000000000..d5a30f8f98 --- /dev/null +++ b/tolk-tester/tests/handle-msg-5.tolk @@ -0,0 +1,39 @@ + +@method_id(101) +fun test1() { + return 0; +} + +@noinline +fun onBouncedMessage(in: InMessageBounced) { + throw in.valueCoins; +} + +fun onInternalMessage(in: InMessage) { + return in.valueCoins; +} + +/** +@testcase | 101 | | 0 + +@fif_codegen +""" + onBouncedMessage() PROC:<{ + DROP + INMSG_VALUE + THROWANY + }> +""" + +@fif_codegen +""" + onInternalMessage() PROC:<{ // in.body + INMSG_BOUNCED // in.body '1 + IFJMP:<{ // in.body + onBouncedMessage() CALLDICT // + }> // in.body + DROP // + INMSG_VALUE // '3 + }> +""" + */ diff --git a/tolk-tester/tests/handle-msg-6.tolk b/tolk-tester/tests/handle-msg-6.tolk new file mode 100644 index 0000000000..0d349f9cca --- /dev/null +++ b/tolk-tester/tests/handle-msg-6.tolk @@ -0,0 +1,47 @@ + +@method_id(101) +fun test1() { + return 0; +} + +fun onBouncedMessage(in: InMessageBounced) { + if (in.valueExtra != null) { + return; + } + throw in.originalForwardFee; +} + +fun onInternalMessage(in: InMessage) { + return in.senderAddress; +} + +/** +@testcase | 101 | | 0 + +@fif_codegen +""" + onBouncedMessage() PROC:<{ + DROP + INMSG_VALUEEXTRA + ISNULL + IFNOTJMP:<{ + }> + INMSG_FWDFEE + 0 PUSHINT + GETORIGINALFWDFEE + THROWANY + }> +""" + +@fif_codegen +""" + onInternalMessage() PROC:<{ + INMSG_BOUNCED + IFJMP:<{ + onBouncedMessage() CALLDICT + }> + DROP + INMSG_SRC + }> +""" + */ diff --git a/tolk-tester/tests/handle-msg-7.tolk b/tolk-tester/tests/handle-msg-7.tolk new file mode 100644 index 0000000000..8410d6f3d6 --- /dev/null +++ b/tolk-tester/tests/handle-msg-7.tolk @@ -0,0 +1,22 @@ + +@method_id(101) +fun test1() { + return 0; +} + +@on_bounced_policy("manual") +fun onInternalMessage(in: InMessage) { + return in.valueExtra; +} + +/** +@testcase | 101 | | 0 + +@fif_codegen +""" + onInternalMessage() PROC:<{ + DROP + INMSG_VALUEEXTRA + }> +""" +*/ \ No newline at end of file diff --git a/tolk-tester/tests/invalid-declaration/err-1906.tolk b/tolk-tester/tests/invalid-declaration/err-1906.tolk new file mode 100644 index 0000000000..2b7e458cff --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1906.tolk @@ -0,0 +1,9 @@ + +fun onBouncedMessage(in: InMessageBounced) { + return in.valueExtra; +} + +/** +@compilation_should_fail +@stderr `onBouncedMessage` should return `void` + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1907.tolk b/tolk-tester/tests/invalid-declaration/err-1907.tolk new file mode 100644 index 0000000000..945a8bf487 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1907.tolk @@ -0,0 +1,8 @@ + +fun onBouncedMessage() { +} + +/** +@compilation_should_fail +@stderr `onBouncedMessage` should have one parameter `InMessageBounced` + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1908.tolk b/tolk-tester/tests/invalid-declaration/err-1908.tolk new file mode 100644 index 0000000000..3904418134 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1908.tolk @@ -0,0 +1,12 @@ + +fun handle(in: InMessage) { +} + +fun onInternalMessage(in: InMessage) { + return handle(in); +} + +/** +@compilation_should_fail +@stderr using `in` as an object is prohibited, because `InMessage` is a built-in struct, its fields are mapped to TVM instructions + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4618.tolk b/tolk-tester/tests/invalid-semantics/err-4618.tolk new file mode 100644 index 0000000000..6fb6419a8a --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4618.tolk @@ -0,0 +1,11 @@ +fun onInternalMessage() { + onExternalMessage(""); +} + +fun onExternalMessage(body: slice) { +} + +/** +@compilation_should_fail +@stderr onExternalMessage is a special entrypoint, it can not be called as a regular function + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4790.tolk b/tolk-tester/tests/invalid-semantics/err-4790.tolk new file mode 100644 index 0000000000..910d741147 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4790.tolk @@ -0,0 +1,11 @@ +fun onInternalMessage() { + var f = onExternalMessage; +} + +fun onExternalMessage(body: slice) { +} + +/** +@compilation_should_fail +@stderr can not get reference to this function, it's a special entrypoint + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4890.tolk b/tolk-tester/tests/invalid-semantics/err-4890.tolk new file mode 100644 index 0000000000..d913652a05 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4890.tolk @@ -0,0 +1,11 @@ + +fun onInternalMessage(in: InMessage) { + in.valueCoins = 123; + return in.valueCoins; +} + +/** +@compilation_should_fail +@stderr modifying an immutable variable +@stderr hint: fields of InMessage can be used for reading only + */ diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index b7eba14fac..63c40f1523 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -701,7 +701,7 @@ fun demo139(s: slice, skip: bool): int { match (msg) { CounterIncrement => { try { - if (s.isEnd()) { return 100; } + if (s.isEmpty()) { return 100; } __expect_lazy("[msg] load byValue"); return msg.byValue; } catch (excno) { return excno } diff --git a/tolk-tester/tests/meaningful-1.tolk b/tolk-tester/tests/meaningful-1.tolk index 19356dc52c..45278ad647 100644 --- a/tolk-tester/tests/meaningful-1.tolk +++ b/tolk-tester/tests/meaningful-1.tolk @@ -40,4 +40,10 @@ fun onInternalMessage(curCounter: int, mode: int) { @testcase | 0 | 50 0 | 0 @testcase | 0 | 50 1 | 51 @testcase | 0 | 50 9 | 59 + +@fif_codegen +""" + onInternalMessage() PROC:<{ // curCounter mode + fakeParseMessage() CALLDICT // curCounter m.USlot1 m.UTag +""" */ diff --git a/tolk-tester/tolk-tester.py b/tolk-tester/tolk-tester.py index f39d2d3ab8..da81f87fe1 100644 --- a/tolk-tester/tolk-tester.py +++ b/tolk-tester/tolk-tester.py @@ -116,6 +116,8 @@ def __init__(self, method_id_str: str, input_str: str, output_str: str): continue elif in_arg.startswith("x{") or TolkTestCaseInputOutput.reJustNumber.fullmatch(in_arg): processed_inputs.append(in_arg) + elif in_arg.startswith("cell{"): + processed_inputs.append("") elif TolkTestCaseInputOutput.reMathExpr.fullmatch(in_arg): processed_inputs.append(str(eval(in_arg))) elif in_arg == "null": diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index 0705876f30..bc8a6daa01 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -6,6 +6,7 @@ set(TOLK_SOURCE ast.cpp ast-from-tokens.cpp constant-evaluator.cpp + gen-entrypoints.cpp pack-unpack-api.cpp pack-unpack-serializers.cpp send-message-api.cpp @@ -24,6 +25,7 @@ set(TOLK_SOURCE pipe-optimize-boolean-expr.cpp pipe-detect-inline-in-place.cpp pipe-lazy-load-insertions.cpp + pipe-transform-on-message.cpp pipe-ast-to-legacy.cpp pipe-find-unused-symbols.cpp pipe-generate-fif-output.cpp diff --git a/tolk/ast-aux-data.h b/tolk/ast-aux-data.h index 37cbcfa844..36a806a091 100644 --- a/tolk/ast-aux-data.h +++ b/tolk/ast-aux-data.h @@ -18,6 +18,7 @@ #include "ast.h" #include "lazy-helpers.h" +#include "tolk.h" /* * This file contains a schema of aux_data inside ast_artificial_aux_vertex @@ -59,4 +60,16 @@ struct AuxData_LazyMatchForUnion final : ASTAuxData { } }; +struct AuxData_OnInternalMessage_getField final : ASTAuxData { + FunctionPtr f_onInternalMessage; + const std::string_view field_name; + + AuxData_OnInternalMessage_getField(FunctionPtr f_onInternalMessage, std::string_view field_name) + : f_onInternalMessage(f_onInternalMessage) + , field_name(field_name) { + } + + std::vector generate_get_InMessage_field(CodeBlob& code, SrcLocation loc) const; +}; + } // namespace tolk diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 4ad6ea9548..98d4e546ec 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -1376,7 +1376,8 @@ static V parse_annotation(Lexer& lex) { throw ParseError(loc, "expecting `(number)` after " + static_cast(name)); } break; - case AnnotationKind::overflow1023_policy: { + case AnnotationKind::overflow1023_policy: + case AnnotationKind::on_bounced_policy: { if (!v_arg || v_arg->size() != 1 || v_arg->get_item(0)->kind != ast_string_const) { throw ParseError(loc, "expecting `(\"policy_name\")` after " + static_cast(name)); } @@ -1410,7 +1411,8 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vectorintval; break; } + case AnnotationKind::on_bounced_policy: { + std::string_view str = v_annotation->get_arg()->get_item(0)->as()->str_val; + if (str == "manual") { + flags |= FunctionData::flagManualOnBounce; + } else { + v_annotation->error("incorrect value for " + static_cast(v_annotation->name)); + } + if (f_name != "onInternalMessage") { + v_annotation->error("this annotation is applicable only to onInternalMessage()"); + } + break; + } case AnnotationKind::deprecated: case AnnotationKind::custom: break; diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 6e3b3b8902..b22f9d31f9 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -75,6 +75,9 @@ AnnotationKind Vertex::parse_kind(std::string_view name) { if (name == "@overflow1023_policy") { return AnnotationKind::overflow1023_policy; } + if (name == "@on_bounced_policy") { + return AnnotationKind::on_bounced_policy; + } return AnnotationKind::unknown; } diff --git a/tolk/ast.h b/tolk/ast.h index 1da1d6e6c2..8836d2a068 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -146,6 +146,7 @@ enum class AnnotationKind { deprecated, custom, overflow1023_policy, + on_bounced_policy, unknown, }; @@ -522,6 +523,7 @@ struct Vertex final : ASTExprUnary { template<> // ast_artificial_aux_vertex is a compiler-inserted vertex that can't occur in source code // example: special vertex to force location in Fift output at constant usage +// example: `msg.isBounced` / `msg.xxx` in onInternalMessage are handled specially struct Vertex final : ASTExprUnary { const ASTAuxData* aux_data; // custom payload, see ast-aux-data.h diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index d15288564b..59c07da951 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -975,6 +975,24 @@ static AsmOp compile_throw_if_unless(std::vector& res, std::vector& res, std::vector& args, SrcLocation loc) { + return exec_op(loc, "GETORIGINALFWDFEE", 2); +} + +static AsmOp compile_calc_InMessage_getInMsgParam(std::vector& res, std::vector& args, SrcLocation loc) { + // instead of "0 INMSGPARAM", generate "INMSG_BOUNCE", etc. — these are aliases in Asm.fif + static const char* aliases[] = { + "INMSG_BOUNCE", "INMSG_BOUNCED", "INMSG_SRC", "INMSG_FWDFEE", "INMSG_LT", "INMSG_UTIME", "INMSG_ORIGVALUE", "INMSG_VALUE", "INMSG_VALUEEXTRA", "INMSG_STATEINIT", + }; + tolk_assert(res.size() == 1 && args.size() == 1 && args[0].is_int_const()); + args[0].unused(); + size_t idx = args[0].int_const->to_long(); + if (idx < std::size(aliases)) { + return exec_op(loc, aliases[idx], 0); + } + return exec_arg_op(loc, "INMSGPARAM", args[0].int_const, 1); +} + static AsmOp compile_throw_arg(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.empty() && args.size() == 2); VarDescr &x = args[1]; @@ -1364,6 +1382,12 @@ void define_builtins() { define_builtin_func("__throw_if_unless", ParamsInt3, Unit, nullptr, compile_throw_if_unless, 0); + define_builtin_func("__InMessage.originalForwardFee", ParamsInt2, Int, nullptr, + compile_calc_InMessage_originalForwardFee, + 0); + define_builtin_func("__InMessage.getInMsgParam", ParamsInt1, Int, nullptr, + compile_calc_InMessage_getInMsgParam, + 0); // compile-time only functions, evaluated essentially at compile-time, no runtime implementation // they are placed in stdlib and marked as `builtin` diff --git a/tolk/gen-entrypoints.cpp b/tolk/gen-entrypoints.cpp new file mode 100644 index 0000000000..196b3dd162 --- /dev/null +++ b/tolk/gen-entrypoints.cpp @@ -0,0 +1,120 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "gen-entrypoints.h" +#include "type-system.h" + +/* + * This module is responsible for `onInternalMessage` at IR generation. + * In FunC, `recv_internal()` entrypoint was declared in such a way: + * > () recv_internal(int my_balance, int msg_value, cell in_msg_full, slice in_msg_body) + * Whenever the user wanted to check whether the message is bounced, he had to parse the cell manually. + * + * In Tolk: + * > fun onInternalMessage(in: InMessage) + * And to use `in.senderAddress`, `in.body`, `in.originalForwardFee`, etc. in the function. + * Under the hood, `in.senderAddress` is transformed into `INMSG_SRC`, and so on. + * + * Also, if `onBouncedMessage` exists, it's embedded directly, like + * > if (INMSG_BOUNCED) { onBouncedMessage(in.body); return; } + */ + +namespace tolk { + +// implemented in ast-from-legacy.cpp +std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, SrcLocation loc, FunctionPtr f_inlined, AnyExprV self_obj, bool is_before_immediate_return, const std::vector>& vars_per_arg); + + +// check for "modern" `fun onInternalMessage(in: InMessage)`, +// because a "FunC-style" (msgCell: cell, msgBody: slice) also works; +// after transformation `in.xxx` to TVM aux vertices, the last parameter is named `in.body` +static bool is_modern_onInternalMessage(FunctionPtr f_onInternalMessage) { + return f_onInternalMessage->get_num_params() == 1 && f_onInternalMessage->get_param(0).name == "in.body"; +} + +std::vector AuxData_OnInternalMessage_getField::generate_get_InMessage_field(CodeBlob& code, SrcLocation loc) const { + if (field_name == "body") { // take `in.body` from a stack + return f_onInternalMessage->find_param("in.body")->ir_idx; + } + if (field_name == "bouncedBody") { // we're in onBouncedMessage() + return f_onInternalMessage->find_param("in.body")->ir_idx; + } + + int idx = -1; + if (field_name == "isBounced") idx = 1; + else if (field_name == "senderAddress") idx = 2; + else if (field_name == "originalForwardFee") idx = 3; + else if (field_name == "createdLt") idx = 4; + else if (field_name == "createdAt") idx = 5; + else if (field_name == "valueCoins") idx = 7; + else if (field_name == "valueExtra") idx = 8; + tolk_assert(idx != -1); + + std::vector ir_msgparam = code.create_tmp_var(TypeDataInt::create(), loc, field_name.data()); + code.emplace_back(loc, Op::_Call, ir_msgparam, std::vector{code.create_int(loc, idx, "(param-idx)")}, lookup_function("__InMessage.getInMsgParam")); + + if (field_name == "originalForwardFee") { + code.emplace_back(loc, Op::_Call, ir_msgparam, std::vector{ir_msgparam[0], code.create_int(loc, 0, "(basechain)")}, lookup_function("__InMessage.originalForwardFee")); + } + + return ir_msgparam; +} + +void handle_onInternalMessage_codegen_start(FunctionPtr f_onInternalMessage, const std::vector& rvect_params, CodeBlob& code, SrcLocation loc) { + // ignore FunC-style `onInternalMessage(msgCell, msgBody)` + if (!is_modern_onInternalMessage(f_onInternalMessage)) { + return; + } + // ignore `@on_bounced_policy("manual")`, don't insert "if (isBounced) return" + if (f_onInternalMessage->is_manual_on_bounce()) { + return; + } + + const Symbol* sym = lookup_global_symbol("onBouncedMessage"); + FunctionPtr f_onBouncedMessage = sym ? sym->try_as() : nullptr; + + AuxData_OnInternalMessage_getField get_isBounced(f_onInternalMessage, "isBounced"); + std::vector ir_isBounced = get_isBounced.generate_get_InMessage_field(code, loc); + + if (f_onBouncedMessage) { + // generate: `if (isBounced) { onBouncedMessage(); return; } + tolk_assert(f_onBouncedMessage->inferred_return_type->get_width_on_stack() == 0); + Op& if_isBounced = code.emplace_back(loc, Op::_If, ir_isBounced); + { + code.push_set_cur(if_isBounced.block0); + std::vector ir_bodySlice(rvect_params.end() - 1, rvect_params.end()); + if (f_onBouncedMessage->is_inlined_in_place()) { + gen_inline_fun_call_in_place(code, TypeDataVoid::create(), loc, f_onBouncedMessage, nullptr, true, {ir_bodySlice}); + } else { + Op& op_call = code.emplace_back(loc, Op::_Call, std::vector{}, ir_bodySlice, f_onBouncedMessage); + op_call.set_impure_flag(); + } + code.emplace_back(loc, Op::_Return, std::vector{}); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_isBounced.block1); + code.close_pop_cur(loc); + } + } else { + // generate: `assert (!isBounced) throw 0` + std::vector args_throw0if = { code.create_int(loc, 0, "(exit-0)"), ir_isBounced[0], code.create_int(loc, 1, "") }; + Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_throw0if), lookup_function("__throw_if_unless")); + op_assert.set_impure_flag(); + } +} + +} // namespace tolk diff --git a/tolk/gen-entrypoints.h b/tolk/gen-entrypoints.h new file mode 100644 index 0000000000..90f62b9bc2 --- /dev/null +++ b/tolk/gen-entrypoints.h @@ -0,0 +1,28 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "ast-aux-data.h" +#include "fwd-declarations.h" +#include "tolk.h" + +namespace tolk { + +void handle_onInternalMessage_codegen_start(FunctionPtr f_onInternalMessage, const std::vector& rvect_params, CodeBlob& code, SrcLocation loc); +std::vector generate_get_requested_field_parsing_on_demand(const AuxData_OnInternalMessage_getField* aux_data, CodeBlob& code, SrcLocation loc); + +} // namespace tolk diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index a6d84a30c8..aa2e2dd3a9 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -23,8 +23,10 @@ #include "common/refint.h" #include "smart-casts-cfg.h" #include "pack-unpack-api.h" +#include "gen-entrypoints.h" #include "generics-helpers.h" #include "send-message-api.h" +#include "gen-entrypoints.h" /* * This pipe is the last one operating AST: it transforms AST to IR. @@ -1965,6 +1967,11 @@ static std::vector process_artificial_aux_vertex(V(v->aux_data)) { + std::vector rvect = data->generate_get_InMessage_field(code, v->loc); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } + tolk_assert(false); } @@ -2304,6 +2311,10 @@ static void convert_function_body_to_CodeBlob(FunctionPtr fun_ref, FunctionBodyC blob->in_var_cnt = blob->var_cnt; tolk_assert(blob->var_cnt == total_arg_width); + if (fun_ref->name == "onInternalMessage") { + handle_onInternalMessage_codegen_start(fun_ref, rvect_import, *blob, fun_ref->loc); + } + process_block_statement(v_body, *blob); append_implicit_return_statement(v_body->loc_end, *blob); diff --git a/tolk/pipe-detect-inline-in-place.cpp b/tolk/pipe-detect-inline-in-place.cpp index 21001c2aa5..4221347f90 100644 --- a/tolk/pipe-detect-inline-in-place.cpp +++ b/tolk/pipe-detect-inline-in-place.cpp @@ -227,7 +227,9 @@ class DetectIfToInlineFunctionInPlaceVisitor final : ASTVisitorFunctionBody { } // okay, this function will be inlined, mark the flag - if (will_inline && fun_ref->n_times_called) { + bool is_called = fun_ref->n_times_called + || fun_ref->name == "onBouncedMessage"; // implicitly called by the compiler from onInternalMessage() + if (will_inline && is_called) { fun_ref->mutate()->assign_inline_mode_in_place(); } } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 2d396762cc..432802279a 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -953,6 +953,9 @@ class InferTypesAndCallsAndFieldsVisitor final { if (fun_ref->is_compile_time_const_val() || fun_ref->is_compile_time_special_gen()) { fire(cur_f, v->loc, "can not get reference to this function, it's compile-time only"); } + if (fun_ref->is_entrypoint()) { + fire(cur_f, v->loc, "can not get reference to this function, it's a special entrypoint"); + } fun_ref->mutate()->assign_is_used_as_noncall(); get_or_infer_return_type(fun_ref); assign_inferred_type(v, fun_ref->inferred_full_type); @@ -1159,6 +1162,11 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } + // prevent calling `onBouncedMessage()` and other special functions directly + if (fun_ref->is_entrypoint()) { + fire(cur_f, v->loc, fun_ref->name + " is a special entrypoint, it can not be called as a regular function"); + } + // so, we have a call `f(args)` or `obj.f(args)`, f is fun_ref (function / method) (code / asm / builtin) // we're going to iterate over passed arguments, and (if generic) infer substitutedTs // at first, check argument count diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index b4394a2d8e..9177229465 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -52,6 +52,9 @@ static int calculate_tvm_method_id_for_entrypoint(std::string_view func_name) { if (func_name == "onSplitInstall") { return -4; } + if (func_name == "onBouncedMessage") { + return FunctionData::EMPTY_TVM_METHOD_ID; + } tolk_assert(false); } diff --git a/tolk/pipe-transform-on-message.cpp b/tolk/pipe-transform-on-message.cpp new file mode 100644 index 0000000000..68aff5060e --- /dev/null +++ b/tolk/pipe-transform-on-message.cpp @@ -0,0 +1,119 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-aux-data.h" +#include "ast-replacer.h" +#include "type-system.h" + +/* + * This pipe analyzes the body of + * > fun onInternalMessage(in: InMessage) + * and replaces `in.senderAddress` / etc. with aux AST vertices (handled specially at IR generation). + * + * This function is transformed to + * > fun onInternalMessage(in.body) + * so, + * - accessing `in.body` actually will take an element from a stack + * - accessing `in.senderAddress` will emit `INMSG_SRC` TVM instruction. + */ + +namespace tolk { + +// handle all functions having a prototype `fun f(var: InMessage)` (for testing purposes) +static bool is_onInternalMessage(FunctionPtr fun_ref) { + if (fun_ref->is_entrypoint() || fun_ref->has_tvm_method_id()) { + if (fun_ref->get_num_params() == 1) { + const auto* t_param = fun_ref->get_param(0).declared_type->try_as(); + return t_param && t_param->struct_ref->name == "InMessage"; + } + } + return false; +} + +// `onBouncedMessage` is only one, it's automatically embedded into `onInternalMessage` if exists +static bool is_onBouncedMessage(FunctionPtr fun_ref) { + return fun_ref->is_entrypoint() && fun_ref->name == "onBouncedMessage"; +} + + +struct TransformOnInternalMessageReplacer final : ASTReplacerInFunctionBody { + FunctionPtr cur_f = nullptr; + LocalVarPtr param_ref = nullptr; // `in` for `fun onInternalMessage(in: InMessage)` + + static void validate_onBouncedMessage(FunctionPtr f) { + if (f->inferred_return_type != TypeDataVoid::create() && f->inferred_return_type != TypeDataNever::create()) { + fire(f, f->loc, "`onBouncedMessage` should return `void`"); + } + if (f->get_num_params() != 1) { + fire(f, f->loc, "`onBouncedMessage` should have one parameter `InMessageBounced`"); + } + const auto* t_struct = f->get_param(0).declared_type->try_as(); + if (!t_struct || t_struct->struct_ref->name != "InMessageBounced") { + fire(f, f->loc, "`onBouncedMessage` should have one parameter `InMessageBounced`"); + } + } + +protected: + AnyExprV replace(V v) override { + // don't allow `var v = in` or passing `in` to another function (only `in.someField` is allowed) + if (v->sym == param_ref) { + fire(cur_f, v->loc, "using `" + param_ref->name + "` as an object is prohibited, because `InMessage` is a built-in struct, its fields are mapped to TVM instructions\nhint: use `" + param_ref->name + ".senderAddress` and other fields directly"); + } + return parent::replace(v); + } + + AnyExprV replace(V v) override { + // replace `in.senderAddress` / `in.valueCoins` with an aux vertex + if (v->get_obj()->kind == ast_reference && v->get_obj()->as()->sym == param_ref) { + if (v->is_lvalue && v->get_field_name() != "body" && v->get_field_name() != "bouncedBody") { + fire(cur_f, v->loc, "modifying an immutable variable\nhint: fields of InMessage can be used for reading only"); + } + + ASTAuxData* aux_getField = new AuxData_OnInternalMessage_getField(cur_f, v->get_field_name()); + return createV(v->loc, v, aux_getField, v->inferred_type); + } + + return parent::replace(v); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return is_onInternalMessage(fun_ref) || is_onBouncedMessage(fun_ref); + } + + void start_replacing_in_function(FunctionPtr fun_ref, V v_function) override { + if (fun_ref->name == "onBouncedMessage") { + validate_onBouncedMessage(fun_ref); + } + + cur_f = fun_ref; + param_ref = &fun_ref->parameters[0]; + + parent::replace(v_function->get_body()); + + std::vector new_parameters; + new_parameters.emplace_back("in.body", fun_ref->loc, TypeDataSlice::create(), nullptr, 0, 0); + fun_ref->mutate()->parameters = std::move(new_parameters); + } +}; + +void pipeline_transform_onInternalMessage() { + replace_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipeline.h b/tolk/pipeline.h index 001228bee1..eb1c153b4d 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -46,6 +46,7 @@ void pipeline_constant_folding(); void pipeline_optimize_boolean_expressions(); void pipeline_detect_inline_in_place(); void pipeline_lazy_load_insertions(); +void pipeline_transform_onInternalMessage(); void pipeline_convert_ast_to_legacy_Expr_Op(); void pipeline_find_unused_symbols(); diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index b9b4b1fa85..03d0cda133 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -42,6 +42,15 @@ std::string StructData::as_human_readable() const { return name + genericTs->as_human_readable(); } +LocalVarPtr FunctionData::find_param(std::string_view name) const { + for (const LocalVarData& param_data : parameters) { + if (param_data.name == name) { + return ¶m_data; + } + } + return nullptr; +} + bool FunctionData::does_need_codegen() const { // when a function is declared, but not referenced from code in any way, don't generate its body if (!is_really_used() && G.settings.remove_unused_functions) { diff --git a/tolk/symtable.h b/tolk/symtable.h index 243c38a607..50ab271be5 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -112,7 +112,7 @@ struct FunctionData final : Symbol { flagMarkedAsPure = 16, // declared as `pure`, can't call impure and access globals, unused invocations are optimized out flagImplicitReturn = 32, // control flow reaches end of function, so it needs implicit return at the end flagContractGetter = 64, // was declared via `get func(): T`, tvm_method_id is auto-assigned - flagIsEntrypoint = 128, // it's `main` / `onExternalMessage` / etc. + flagIsEntrypoint = 128, // it's `main` / `onExternalMessage` / etc. flagHasMutateParams = 256, // has parameters declared as `mutate` flagAcceptsSelf = 512, // is a member function (has `self` first parameter) flagReturnsSelf = 1024, // return type is `self` (returns the mutated 1st argument), calls can be chainable @@ -120,6 +120,7 @@ struct FunctionData final : Symbol { flagCompileTimeVal = 4096, // calculated only at compile-time for constant arguments: `ton("0.05")`, `stringCrc32`, and others flagCompileTimeGen = 8192, // at compile-time it's handled specially, not as a regular function: `T.toCell`, etc. flagAllowAnyWidthT = 16384, // for built-in generic functions that is not restricted to be 1-slot type + flagManualOnBounce = 32768, // for onInternalMessage, don't insert "if (isBounced) return" }; int tvm_method_id = EMPTY_TVM_METHOD_ID; @@ -184,6 +185,7 @@ struct FunctionData final : Symbol { int get_num_params() const { return static_cast(parameters.size()); } const LocalVarData& get_param(int idx) const { return parameters[idx]; } + LocalVarPtr find_param(std::string_view name) const; bool is_code_function() const { return std::holds_alternative(body); } bool is_asm_function() const { return std::holds_alternative(body); } @@ -210,6 +212,7 @@ struct FunctionData final : Symbol { bool is_compile_time_const_val() const { return flags & flagCompileTimeVal; } bool is_compile_time_special_gen() const { return flags & flagCompileTimeGen; } bool is_variadic_width_T_allowed() const { return flags & flagAllowAnyWidthT; } + bool is_manual_on_bounce() const { return flags & flagManualOnBounce; } bool does_need_codegen() const; diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp index cbad23fe3a..63a2dd7f25 100644 --- a/tolk/tolk.cpp +++ b/tolk/tolk.cpp @@ -69,6 +69,7 @@ int tolk_proceed(const std::string &entrypoint_filename) { pipeline_optimize_boolean_expressions(); pipeline_detect_inline_in_place(); pipeline_lazy_load_insertions(); + pipeline_transform_onInternalMessage(); pipeline_convert_ast_to_legacy_Expr_Op(); pipeline_find_unused_symbols(); From 72e0ae29a2fa2e947738c8e4910217c376150581 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 26 Jun 2025 19:45:55 +0300 Subject: [PATCH 335/524] [Tolk] Correctly handle `import` without ".tolk" extension Also, output abs paths in error messages. --- tolk-tester/tests/imports-tests.tolk | 18 ++++++ tolk-tester/tests/imports/has-tinf-err.tolk | 4 ++ tolk-tester/tests/imports/some-helpers.tolk | 1 + tolk-tester/tests/imports/some-storage.tolk | 1 + .../{use-dicts.tolk => use-dicts.ext.tolk} | 0 .../tests/invalid-symbol/err-2205.tolk | 2 +- .../tests/invalid-symbol/err-2539.tolk | 6 +- .../tests/invalid-symbol/err-2540.tolk | 11 ++++ .../tests/invalid-typing/err-6722.tolk | 7 +++ tolk-tester/tests/logical-operators.tolk | 2 +- tolk/ast-stringifier.h | 2 +- tolk/compiler-state.h | 2 +- tolk/pipe-discover-parse-sources.cpp | 5 +- tolk/pipe-generate-fif-output.cpp | 4 +- tolk/pipe-resolve-identifiers.cpp | 20 +------ tolk/pipe-resolve-types.cpp | 3 + tolk/src-file.cpp | 57 ++++++++++++------- tolk/src-file.h | 19 ++++--- tolk/symtable.cpp | 13 +++++ tolk/symtable.h | 2 + tolk/tolk-main.cpp | 20 +++---- tolk/tolk-wasm.cpp | 2 +- 22 files changed, 131 insertions(+), 70 deletions(-) create mode 100644 tolk-tester/tests/imports-tests.tolk create mode 100644 tolk-tester/tests/imports/has-tinf-err.tolk create mode 100644 tolk-tester/tests/imports/some-helpers.tolk create mode 100644 tolk-tester/tests/imports/some-storage.tolk rename tolk-tester/tests/imports/{use-dicts.tolk => use-dicts.ext.tolk} (100%) create mode 100644 tolk-tester/tests/invalid-symbol/err-2540.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6722.tolk diff --git a/tolk-tester/tests/imports-tests.tolk b/tolk-tester/tests/imports-tests.tolk new file mode 100644 index 0000000000..c4dc934788 --- /dev/null +++ b/tolk-tester/tests/imports-tests.tolk @@ -0,0 +1,18 @@ +import "imports/some-math" +import "./imports/use-dicts.ext" + +fun main() { + prepareDict_3_30_4_40_5_x(4); + return someAdd(1, 2); +} + +/** +@testcase | 0 | | 3 + +@fif_codegen +""" + main() PROC:<{ + 3 PUSHINT + }> +""" + */ diff --git a/tolk-tester/tests/imports/has-tinf-err.tolk b/tolk-tester/tests/imports/has-tinf-err.tolk new file mode 100644 index 0000000000..bc9e5c8b19 --- /dev/null +++ b/tolk-tester/tests/imports/has-tinf-err.tolk @@ -0,0 +1,4 @@ + +fun asdf() { + var i: int = ""; +} diff --git a/tolk-tester/tests/imports/some-helpers.tolk b/tolk-tester/tests/imports/some-helpers.tolk new file mode 100644 index 0000000000..a5caa25158 --- /dev/null +++ b/tolk-tester/tests/imports/some-helpers.tolk @@ -0,0 +1 @@ +import "./some-storage" diff --git a/tolk-tester/tests/imports/some-storage.tolk b/tolk-tester/tests/imports/some-storage.tolk new file mode 100644 index 0000000000..93a692987b --- /dev/null +++ b/tolk-tester/tests/imports/some-storage.tolk @@ -0,0 +1 @@ +struct SomeStorage {} diff --git a/tolk-tester/tests/imports/use-dicts.tolk b/tolk-tester/tests/imports/use-dicts.ext.tolk similarity index 100% rename from tolk-tester/tests/imports/use-dicts.tolk rename to tolk-tester/tests/imports/use-dicts.ext.tolk diff --git a/tolk-tester/tests/invalid-symbol/err-2205.tolk b/tolk-tester/tests/invalid-symbol/err-2205.tolk index 3c9d72204f..c3d47a672e 100644 --- a/tolk-tester/tests/invalid-symbol/err-2205.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2205.tolk @@ -3,6 +3,6 @@ import "../imports/invalid-no-import.tolk"; /** @compilation_should_fail -@stderr ../imports/invalid-no-import.tolk:2:13 +@stderr tests/imports/invalid-no-import.tolk:2:13 @stderr Using a non-imported symbol `someAdd` */ diff --git a/tolk-tester/tests/invalid-symbol/err-2539.tolk b/tolk-tester/tests/invalid-symbol/err-2539.tolk index f7c66c2582..65c02115b9 100644 --- a/tolk-tester/tests/invalid-symbol/err-2539.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2539.tolk @@ -1,9 +1,9 @@ import "@stdlib/tvm-dicts" -import "../imports/use-dicts-err.tolk" +import "../imports/use-dicts-err" /** @compilation_should_fail -@stderr ../imports/use-dicts-err.tolk:2:22 +@stderr tests/imports/use-dicts-err.tolk:2:22 @stderr Using a non-imported symbol `createEmptyDict` -@stderr Forgot to import "@stdlib/tvm-dicts"? +@stderr hint: forgot to import "@stdlib/tvm-dicts"? */ diff --git a/tolk-tester/tests/invalid-symbol/err-2540.tolk b/tolk-tester/tests/invalid-symbol/err-2540.tolk new file mode 100644 index 0000000000..0b14df1b89 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2540.tolk @@ -0,0 +1,11 @@ +import "../imports/some-helpers" + +fun usingNonImported() { + var s: SomeStorage = {}; +} + +/** +@compilation_should_fail +@stderr Using a non-imported symbol `SomeStorage` +@stderr forgot to import "some-storage.tolk"? + */ diff --git a/tolk-tester/tests/invalid-typing/err-6722.tolk b/tolk-tester/tests/invalid-typing/err-6722.tolk new file mode 100644 index 0000000000..854e211675 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6722.tolk @@ -0,0 +1,7 @@ +import "../imports/./has-tinf-err" + +/** +@compilation_should_fail +@stderr imports/has-tinf-err.tolk:3:18 +@stderr can not assign `slice` to variable of type `int` + */ diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk index 8242d5ff4e..c184da72c6 100644 --- a/tolk-tester/tests/logical-operators.tolk +++ b/tolk-tester/tests/logical-operators.tolk @@ -1,4 +1,4 @@ -import "imports/use-dicts.tolk" +import "imports/use-dicts.ext.tolk" fun simpleAllConst() { return (!0, !!0 & !false, !!!0, !1, !!1, !-1, !!-1, (!5 as int == 0) == !0, !0 == true); diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index b9ee069793..fd19401e95 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -247,7 +247,7 @@ class ASTStringifier final : public ASTVisitor { case ast_import_directive: return static_cast(v->as()->get_file_leaf()->str_val); case ast_tolk_file: - return v->as()->file->rel_filename; + return v->as()->file->realpath; default: return {}; } diff --git a/tolk/compiler-state.h b/tolk/compiler-state.h index 2193503a22..6685e092a7 100644 --- a/tolk/compiler-state.h +++ b/tolk/compiler-state.h @@ -56,7 +56,7 @@ struct CompilerSettings { std::string output_filename; std::string boc_output_filename; - std::string stdlib_folder; // a path to tolk-stdlib/; files imported via @stdlib/xxx are there + std::string stdlib_folder; // path to tolk-stdlib/; note: from tolk-js it's empty! tolk-js reads files via js callback FsReadCallback read_callback; diff --git a/tolk/pipe-discover-parse-sources.cpp b/tolk/pipe-discover-parse-sources.cpp index 2f7c02d6d2..1cadf07f4d 100644 --- a/tolk/pipe-discover-parse-sources.cpp +++ b/tolk/pipe-discover-parse-sources.cpp @@ -52,10 +52,9 @@ void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, con for (AnyV v_toplevel : file->ast->as()->get_toplevel_declarations()) { if (auto v_import = v_toplevel->try_as()) { std::string imported_str = v_import->get_file_name(); - size_t cur_slash_pos = file->rel_filename.rfind('/'); - std::string rel_filename = cur_slash_pos == std::string::npos || imported_str[0] == '@' + std::string rel_filename = imported_str[0] == '@' ? std::move(imported_str) - : file->rel_filename.substr(0, cur_slash_pos + 1) + imported_str; + : file->extract_dirname() + imported_str; const SrcFile* imported = G.all_src_files.locate_and_register_source_file(rel_filename, v_import->loc); file->imports.push_back(SrcFile::ImportDirective{imported}); diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp index 237649f45f..1dc513f054 100644 --- a/tolk/pipe-generate-fif-output.cpp +++ b/tolk/pipe-generate-fif-output.cpp @@ -125,11 +125,11 @@ void pipeline_generate_fif_output_to_std_cout() { std::cout << "// automatically generated from "; bool need_comma = false; for (const SrcFile* file : G.all_src_files) { - if (!file->is_stdlib_file()) { + if (!file->is_stdlib_file) { if (need_comma) { std::cout << ", "; } - std::cout << file->rel_filename; + std::cout << file->extract_short_name(); need_comma = true; } } diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index fad233c7f9..46710cb639 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -64,22 +64,6 @@ static void fire_error_type_used_as_symbol(FunctionPtr cur_f, V } } -static void check_import_exists_when_using_sym(FunctionPtr cur_f, AnyV v_usage, const Symbol* used_sym) { - SrcLocation sym_loc = used_sym->loc; - if (!v_usage->loc.is_symbol_from_same_or_builtin_file(sym_loc)) { - const SrcFile* declared_in = sym_loc.get_src_file(); - bool has_import = false; - for (const SrcFile::ImportDirective& import : v_usage->loc.get_src_file()->imports) { - if (import.imported_file == declared_in) { - has_import = true; - } - } - if (!has_import) { - throw ParseError(cur_f, v_usage->loc, "Using a non-imported symbol `" + used_sym->name + "`. Forgot to import \"" + declared_in->rel_filename + "\"?"); - } - } -} - struct NameAndScopeResolver { std::vector> scopes; @@ -180,7 +164,9 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { // for global functions, global vars and constants, `import` must exist if (!sym->try_as()) { - check_import_exists_when_using_sym(cur_f, v, sym); + if (!v->loc.is_symbol_from_same_or_builtin_file(sym->loc)) { + sym->check_import_exists_when_used_from(cur_f, v->loc); + } } } diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 194cc4998c..2a9bb20288 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -168,6 +168,9 @@ class TypeNodesVisitorResolver { } if (const Symbol* sym = lookup_global_symbol(text)) { if (TypePtr custom_type = try_resolve_user_defined_type(cur_f, loc, sym, allow_without_type_arguments)) { + if (!v->loc.is_symbol_from_same_or_builtin_file(sym->loc)) { + sym->check_import_exists_when_used_from(cur_f, v->loc); + } return custom_type; } } diff --git a/tolk/src-file.cpp b/tolk/src-file.cpp index ab54ce7b0c..07db0eb88b 100644 --- a/tolk/src-file.cpp +++ b/tolk/src-file.cpp @@ -24,41 +24,44 @@ namespace tolk { static_assert(sizeof(SrcLocation) == 8); -const SrcFile* AllRegisteredSrcFiles::find_file(const std::string& abs_filename) const { +const SrcFile* AllRegisteredSrcFiles::find_file(const std::string& realpath) const { + // files with the same realpath are considered equal for (const SrcFile* file : all_src_files) { - if (file->abs_filename == abs_filename) { + if (file->realpath == realpath) { return file; } } return nullptr; } -const SrcFile* AllRegisteredSrcFiles::locate_and_register_source_file(const std::string& rel_filename, SrcLocation included_from) { - td::Result path = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::Realpath, rel_filename.c_str()); +const SrcFile* AllRegisteredSrcFiles::locate_and_register_source_file(const std::string& filename, SrcLocation included_from) { + bool is_stdlib = filename.size() > 8 && filename.starts_with("@stdlib/"); + + td::Result path = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::Realpath, filename.c_str()); if (path.is_error()) { if (included_from.is_defined()) { throw ParseError(included_from, "Failed to import: " + path.move_as_error().message().str()); } - throw Fatal("Failed to locate " + rel_filename + ": " + path.move_as_error().message().str()); + throw Fatal("Failed to locate " + filename + ": " + path.move_as_error().message().str()); } - std::string abs_filename = path.move_as_ok(); - if (const SrcFile* file = find_file(abs_filename)) { + std::string realpath = path.move_as_ok(); + if (const SrcFile* file = find_file(realpath)) { return file; } - td::Result text = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::ReadFile, abs_filename.c_str()); + td::Result text = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::ReadFile, realpath.c_str()); if (text.is_error()) { if (included_from.is_defined()) { throw ParseError(included_from, "Failed to import: " + text.move_as_error().message().str()); } - throw Fatal("Failed to read " + rel_filename + ": " + text.move_as_error().message().str()); + throw Fatal("Failed to read " + realpath + ": " + text.move_as_error().message().str()); } int file_id = static_cast(all_src_files.size()); // SrcFile::file_id is the index in all files - SrcFile* created = new SrcFile(file_id, rel_filename, std::move(abs_filename), text.move_as_ok()); + SrcFile* created = new SrcFile(file_id, is_stdlib, std::move(realpath), text.move_as_ok()); if (G.is_verbosity(1)) { - std::cerr << "register file_id " << created->file_id << " " << created->abs_filename << std::endl; + std::cerr << "register file_id " << created->file_id << " " << created->realpath << std::endl; } all_src_files.push_back(created); return created; @@ -72,11 +75,6 @@ SrcFile* AllRegisteredSrcFiles::get_next_unparsed_file() { return const_cast(all_src_files[++last_parsed_file_id]); } -bool SrcFile::is_stdlib_file() const { - std::string_view rel(rel_filename); - return rel.size() > 10 && rel.substr(0, 8) == "@stdlib/"; // common.tolk, tvm-dicts.tolk, etc -} - bool SrcFile::is_offset_valid(int offset) const { return offset >= 0 && offset < static_cast(text.size()); } @@ -116,9 +114,30 @@ SrcFile::SrcPosition SrcFile::convert_offset(int offset) const { return SrcPosition{offset, line_idx + 1, char_idx + 1, line_str}; } +std::string SrcFile::extract_short_name() const { + size_t last_slash = realpath.find_last_of("/\\"); + if (last_slash == std::string::npos) { + return realpath; + } + std::string short_name = realpath.substr(last_slash + 1); // "file.tolk" (no path) + + if (is_stdlib_file) { // not "common.tolk", but "@stdlib/common" + return "@stdlib/" + short_name.substr(0, short_name.size() - 5); + } + return short_name; +} + +std::string SrcFile::extract_dirname() const { + size_t last_slash = realpath.find_last_of("/\\"); + if (last_slash == std::string::npos) { + return ""; + } + return realpath.substr(0, last_slash + 1); +} + std::ostream& operator<<(std::ostream& os, const SrcFile* src_file) { - return os << (src_file ? src_file->rel_filename : "unknown-location"); + return os << (src_file ? src_file->realpath : "unknown-location"); } std::ostream& operator<<(std::ostream& os, const Fatal& fatal) { @@ -233,7 +252,7 @@ void ParseError::show(std::ostream& os) const { // print "location: line1 \n (spaces) line2 \n ..." std::string_view message = this->message; std::string loc_text = loc.to_string(); - std::string loc_spaces(std::min(static_cast(loc_text.size()), 30), ' '); + std::string loc_spaces(std::min(static_cast(loc_text.size()), 9), ' '); size_t start = 0, end; os << loc_text << ": error: "; while ((end = message.find('\n', start)) != std::string::npos) { @@ -248,7 +267,7 @@ void ParseError::show(std::ostream& os) const { } } if (current_function) { - os << " // in function `" << current_function->as_human_readable() << "`" << std::endl; + os << std::endl << " // in function `" << current_function->as_human_readable() << "`" << std::endl; } loc.show_context(os); } diff --git a/tolk/src-file.h b/tolk/src-file.h index 1f3eb030e8..9b39a0accc 100644 --- a/tolk/src-file.h +++ b/tolk/src-file.h @@ -35,25 +35,26 @@ struct SrcFile { }; int file_id; // an incremental counter through all parsed files - std::string rel_filename; // relative to cwd - std::string abs_filename; // absolute from root + bool is_stdlib_file; // is a part of Tolk distribution, imported via "@stdlib/..." + std::string realpath; // what "realpath" returned to locate (either abs path or what tolk-js returns) std::string text; // file contents loaded into memory, every Token::str_val points inside it AnyV ast = nullptr; // when a file has been parsed, its ast_tolk_file is kept here std::vector imports; // to check strictness (can't use a symbol without importing its file) - SrcFile(int file_id, std::string rel_filename, std::string abs_filename, std::string&& text) + SrcFile(int file_id, bool is_stdlib_file, std::string realpath, std::string&& text) : file_id(file_id) - , rel_filename(std::move(rel_filename)) - , abs_filename(std::move(abs_filename)) + , is_stdlib_file(is_stdlib_file) + , realpath(std::move(realpath)) , text(std::move(text)) { } SrcFile(const SrcFile& other) = delete; SrcFile &operator=(const SrcFile&) = delete; - bool is_stdlib_file() const; - bool is_offset_valid(int offset) const; SrcPosition convert_offset(int offset) const; + + std::string extract_short_name() const; + std::string extract_dirname() const; }; @@ -102,9 +103,9 @@ class AllRegisteredSrcFiles { public: const SrcFile* get_file(int file_id) const { return all_src_files.at(file_id); } - const SrcFile* find_file(const std::string& abs_filename) const; + const SrcFile* find_file(const std::string& realpath) const; - const SrcFile* locate_and_register_source_file(const std::string& rel_filename, SrcLocation included_from); + const SrcFile* locate_and_register_source_file(const std::string& filename, SrcLocation included_from); SrcFile* get_next_unparsed_file(); auto begin() const { return all_src_files.begin(); } diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 03d0cda133..348fac8399 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -21,6 +21,19 @@ namespace tolk { +void Symbol::check_import_exists_when_used_from(FunctionPtr cur_f, SrcLocation used_loc) const { + const SrcFile* declared_in = loc.get_src_file(); + bool has_import = false; + for (const SrcFile::ImportDirective& import : used_loc.get_src_file()->imports) { + if (import.imported_file == declared_in) { + has_import = true; + } + } + if (!has_import) { + throw ParseError(cur_f, used_loc, "Using a non-imported symbol `" + name + "`\nhint: forgot to import \"" + declared_in->extract_short_name() + "\"?"); + } +} + std::string FunctionData::as_human_readable() const { if (!is_generic_function()) { return name; // if it's generic instantiation like `f`, its name is "f", not "f" diff --git a/tolk/symtable.h b/tolk/symtable.h index 50ab271be5..268b474d3a 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -43,6 +43,8 @@ struct Symbol { #endif return dynamic_cast(this); } + + void check_import_exists_when_used_from(FunctionPtr cur_f, SrcLocation used_loc) const; }; struct LocalVarData final : Symbol { diff --git a/tolk/tolk-main.cpp b/tolk/tolk-main.cpp index 4e9c1f4d86..17a2b4c845 100644 --- a/tolk/tolk-main.cpp +++ b/tolk/tolk-main.cpp @@ -149,19 +149,15 @@ static std::string auto_discover_stdlib_folder() { td::Result fs_read_callback(CompilerSettings::FsReadCallbackKind kind, const char* query) { switch (kind) { case CompilerSettings::FsReadCallbackKind::Realpath: { - td::Result res_realpath; - if (query[0] == '@' && strlen(query) > 8 && !strncmp(query, "@stdlib/", 8)) { - // import "@stdlib/filename" or import "@stdlib/filename.tolk" - std::string path = G.settings.stdlib_folder + static_cast(query + 7); - if (strncmp(path.c_str() + path.size() - 5, ".tolk", 5) != 0) { - path += ".tolk"; - } - res_realpath = td::realpath(td::CSlice(path.c_str())); - } else { - // import "relative/to/cwd/path.tolk" - res_realpath = td::realpath(td::CSlice(query)); - } + bool is_stdlib = query[0] == '@' && strlen(query) > 8 && !strncmp(query, "@stdlib/", 8); + std::string path = is_stdlib + ? G.settings.stdlib_folder + static_cast(query + 7) + : static_cast(query); + if (strncmp(path.c_str() + path.size() - 5, ".tolk", 5) != 0) { + path += ".tolk"; + } + td::Result res_realpath = td::realpath(td::CSlice(path.c_str())); if (res_realpath.is_error()) { // note, that for non-existing files, `realpath()` on Linux/Mac returns an error, // whereas on Windows, it returns okay, but fails after, on reading, with a message "cannot open file" diff --git a/tolk/tolk-wasm.cpp b/tolk/tolk-wasm.cpp index 3c8cb87dd7..b5149481bb 100644 --- a/tolk/tolk-wasm.cpp +++ b/tolk/tolk-wasm.cpp @@ -57,7 +57,7 @@ static td::Result compile_internal(char *config_json) { std::cerr.rdbuf(errs.rdbuf()); int exit_code = tolk_proceed(entrypoint_filename); if (exit_code != 0) { - return td::Status::Error("Tolk compilation error: " + errs.str()); + return td::Status::Error(errs.str()); } TRY_RESULT(fift_res, fift::compile_asm_program(outs.str(), "/fiftlib/")); From 2bb89b461ffc5ef193429823ee1c9e17dab75818 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 29 Jun 2025 21:38:10 +0300 Subject: [PATCH 336/524] [Tolk] Correctly handle variadic ints at serialization Occasionally, `varint32` was not handled at packing to/from cells and worked like a regular `int32`. This MR fixes the issue, and also: - adds missing varintN types - fix intN/uintN limits - rename bytesN to bitsN, since "bits" are more frequent in practice --- crypto/smartcont/tolk-stdlib/common.tolk | 4 +- tolk-tester/tests/intN-tests.tolk | 4 +- tolk-tester/tests/lazy-load-tests.tolk | 17 ++++++- tolk-tester/tests/pack-unpack-2.tolk | 47 +++++++++++++++++++ tolk-tester/tests/pack-unpack-5.tolk | 3 +- tolk-tester/tests/pack-unpack-6.tolk | 4 +- tolk/builtins.cpp | 45 ++++++++++++++++++ tolk/pack-unpack-api.cpp | 4 +- tolk/pack-unpack-serializers.cpp | 60 +++++++++++++++++++----- tolk/pack-unpack-serializers.h | 2 +- tolk/pipe-ast-to-legacy.cpp | 8 ++-- tolk/pipe-lazy-load-insertions.cpp | 6 +-- tolk/pipe-resolve-types.cpp | 36 +++++++------- tolk/type-system.cpp | 32 ++++++------- tolk/type-system.h | 30 ++++++------ 15 files changed, 223 insertions(+), 79 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 9c57bf18fc..7cf61f5623 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -227,8 +227,8 @@ struct PackOptions { /// however, if you guarantee that a slice is valid (for example, it comes from trusted sources), /// set this option to true to disable runtime checks; /// note: `int32` and other are always validated for overflow without any extra gas, - /// so this flag controls only rarely used `bytesN` / `bitsN` types - skipBitsNFieldsValidation: bool = false, + /// so this flag controls only rarely used `bitsN` type + skipBitsNValidation: bool = false, } /// UnpackOptions allows you to control behavior of `MyStruct.fromCell(c)` and similar functions. diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index 9cdc469889..cd557092ed 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -80,7 +80,7 @@ fun test3(op: int32, qid: uint64) { if (op == qid) {} if ((op as int32?)! == qid) {} if (op == (qid as uint64)) {} - if ((op as uint257?)! != (qid as int256?)!) {} + if ((op as int257?)! != (qid as uint256?)!) {} __expect_type(op << qid, "int"); takeAnyInt(op); @@ -227,7 +227,7 @@ fun main() { assign0(mutate t.1 as uint16); assign0(mutate t.2 as int); __expect_type(t.0 as int1, "int1"); - __expect_type(t.0 as uint257, "uint257"); + __expect_type(t.0 as int257, "int257"); __expect_type(0 as int32, "int32"); __expect_type((0 as int32) as uint64?, "uint64?"); __expect_type(null as (int8, [uint16?])?, "(int8, [uint16?])?"); diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index 63c40f1523..208c892412 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -217,6 +217,11 @@ struct OneIntOneRef { r: cell; } +struct WithVariadicInts { + i1: varuint32; + i2: varuint32; +} + @noinline fun contract.getFakeData(seqno: int): Cell { @@ -655,7 +660,7 @@ fun demo135(skip: bool) { cc.bin16 = stringHexToSlice("00000000") as bits16; try { return cc.toCell({ - skipBitsNFieldsValidation: skip + skipBitsNValidation: skip }).beginParse().remainingBitsCount(); } catch (excno) { return excno; @@ -939,6 +944,13 @@ fun demo156(s: slice): int { } } +@method_id(157) +fun demo157() { + var c: Cell = WithVariadicInts { i1: 1 << 100, i2: 1 << 200 }.toCell(); + val d = lazy c.load(); + return d.i2 == (1 << 200); +} + @method_id(200) fun demo200() { @@ -1280,7 +1292,7 @@ fun demo310() { f9: "" as bits200, f10: ("" as bits400, ()), f11: 0, - }.toCell({skipBitsNFieldsValidation: true}); + }.toCell({skipBitsNValidation: true}); var st = lazy cell.load(); var sm = lazy cell.load(); var sk = lazy cell.load(); @@ -1625,6 +1637,7 @@ fun main() { @testcase | 155 | | 555 @testcase | 156 | x{0110} | 16 @testcase | 156 | x{FF10} | -100 +@testcase | 157 | | -1 @testcase | 200 | | 1 @testcase | 201 | | 1 2 -1 diff --git a/tolk-tester/tests/pack-unpack-2.tolk b/tolk-tester/tests/pack-unpack-2.tolk index 51432d0b4d..cb01fbfeb0 100644 --- a/tolk-tester/tests/pack-unpack-2.tolk +++ b/tolk-tester/tests/pack-unpack-2.tolk @@ -334,6 +334,30 @@ struct DifferentMix3 { pairm: (int32, int64)?; } +/* + ui16: VarUInteger 16 + i16: VarInteger 16 + ui32: VarUInteger 32 + i32: VarInteger 32 + */ + +struct WithVariadicInts { + ui16: varuint16; + i16: varint16; + ui32: varuint32; + i32: varint32; +} + +/* + Test MIN_INT and MAX_INT + */ + +struct EdgeCaseInts { + maxUint: uint256 = +115792089237316195423570985008687907853269984665640564039457584007913129639935; + maxInt: int257 = +115792089237316195423570985008687907853269984665640564039457584007913129639935; + minInt: int257 = -115792089237316195423570985008687907853269984665640564039457584007913129639935 - 1; +} + /* Test: write with builder, read with other struct (type `builder` is available for writing, but not for reading) @@ -670,6 +694,27 @@ fun test_mutatingRemainder() { return obj.rest.remainingBitsCount(); } +@method_id(228) +fun test_VariadicIntegers() { + var t: WithVariadicInts = { + ui16: (1 << 120) - 1, + i16: -(1 << 119), + ui32: (1 << 248) - 1, + i32: -(1 << 247), + }; + run(t, stringHexToSlice("ffffffffffffffffffffffffffffffff800000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe000000000000000000000000000000000000000000000000000000000000020_")); + run({ui16: 0, i16: 0, ui32: 0, i32: 0}, stringHexToSlice("000020_")); + + var c = t.toCell().load(); + return (c.ui16 == t.ui16) & (c.i16 == t.i16) & (c.ui32 == t.ui32) & (c.i32 == t.i32) +} + +@method_id(229) +fun test_EdgeCaseIntegers() { + var edge: EdgeCaseInts = {}; + var manual = beginCell().storeUint(edge.maxUint, 256).storeInt(edge.maxInt, 257).storeInt(edge.minInt, 257); + return edge.toCell().hash() == manual.endCell().hash() +} fun main() { var t: JustInt32 = { value: 10 }; @@ -709,4 +754,6 @@ fun main() { @testcase | 225 | | 5 40 65535 8 50000000 @testcase | 226 | | 1 16 0 @testcase | 227 | | 16 +@testcase | 228 | | -1 +@testcase | 229 | | -1 */ diff --git a/tolk-tester/tests/pack-unpack-5.tolk b/tolk-tester/tests/pack-unpack-5.tolk index 46b2a77b54..af397181e5 100644 --- a/tolk-tester/tests/pack-unpack-5.tolk +++ b/tolk-tester/tests/pack-unpack-5.tolk @@ -22,6 +22,7 @@ fun test1() { coins.estimatePackSize(), bits6.estimatePackSize(), bytes8.estimatePackSize(), + varint32.estimatePackSize(), ); } @@ -204,7 +205,7 @@ fun main() { } /** -@testcase | 101 | | [ 32 32 0 0 ] [ 64 64 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 0 9999 0 4 ] [ 4 124 0 0 ] [ 6 6 0 0 ] [ 64 64 0 0 ] +@testcase | 101 | | [ 32 32 0 0 ] [ 64 64 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 0 9999 0 4 ] [ 4 124 0 0 ] [ 6 6 0 0 ] [ 64 64 0 0 ] [ 5 253 0 0 ] @testcase | 102 | | [ 32 32 0 0 ] [ 1 33 0 0 ] [ 1 33 0 0 ] [ 17 33 0 0 ] @testcase | 103 | | [ 67 199 0 0 ] [ 69 599 0 0 ] @testcase | 104 | | [ 4 534 0 0 ] [ 2 536 0 0 ] [ 6 1070 0 0 ] diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk index 51c1f7c133..71f6ab0b40 100644 --- a/tolk-tester/tests/pack-unpack-6.tolk +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -50,13 +50,13 @@ fun test5() { @method_id(106) fun test6() { - return SomeBytesFields { f1: stringHexToSlice("11") as bytes1 }.toCell({skipBitsNFieldsValidation: true}).hash() & 0xFFFF; + return SomeBytesFields { f1: stringHexToSlice("11") as bytes1 }.toCell({skipBitsNValidation: true}).hash() & 0xFFFF; } @method_id(107) fun test7(believe: bool) { try { - return SomeBytesFields { f1: stringHexToSlice("ffff") as bytes1 }.toCell({skipBitsNFieldsValidation: believe}).hash() & 0xFFFF; + return SomeBytesFields { f1: stringHexToSlice("ffff") as bytes1 }.toCell({skipBitsNValidation: believe}).hash() & 0xFFFF; } catch (excno) { return excno; } diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 59c07da951..cce907beb6 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1036,6 +1036,25 @@ static AsmOp compile_fetch_int(std::vector& res, std::vector return exec_op(loc, (fetch ? "LD"s : "PLD"s) + (sgnd ? "IX" : "UX"), 2, 1 + (unsigned)fetch); } +// fun slice.__loadVarInt(mutate self, bits: int, unsigned: bool): int +static AsmOp compile_fetch_varint(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(args.size() == 3 && res.size() == 2); + // it's a hidden function for auto-serialization (not exposed to stdlib), to bits/unsigned are not dynamic + tolk_assert(args[1].is_int_const() && args[2].is_int_const()); + long n_bits = args[1].int_const->to_long(); + long is_unsigned = args[2].int_const->to_long(); + + args[1].unused(); + args[2].unused(); + if (n_bits == 16) { + return exec_op(loc, is_unsigned ? "LDVARUINT16" : "LDVARINT16", 1, 2); + } + if (n_bits == 32) { + return exec_op(loc, is_unsigned ? "LDVARUINT32" : "LDVARINT32", 1, 2); + } + tolk_assert(false); +} + // fun builder.storeInt (mutate self, x: int, len: int): self asm(x b len) "STIX"; // fun builder.storeUint (mutate self, x: int, len: int): self asm(x b len) "STUX"; static AsmOp compile_store_int(std::vector& res, std::vector& args, SrcLocation loc, bool sgnd) { @@ -1063,6 +1082,25 @@ static AsmOp compile_store_int(std::vector& res, std::vector return exec_op(loc, sgnd ? "STIX" : "STUX", 3, 1); } +// fun builder.__storeVarInt (mutate self, x: int, bits: int, unsigned: bool): self +static AsmOp compile_store_varint(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(args.size() == 4 && res.size() == 1); + // it's a hidden function for auto-serialization (not exposed to stdlib), to bits/unsigned are not dynamic + tolk_assert(args[2].is_int_const() && args[3].is_int_const()); + long n_bits = args[2].int_const->to_long(); + long is_unsigned = args[3].int_const->to_long(); + + args[2].unused(); + args[3].unused(); + if (n_bits == 16) { + return exec_op(loc, is_unsigned ? "STVARUINT16" : "STVARINT16", 2, 1); + } + if (n_bits == 32) { + return exec_op(loc, is_unsigned ? "STVARUINT32" : "STVARINT32", 2, 1); + } + tolk_assert(false); +} + // fun builder.storeBool(mutate self, value: bool): self asm( -> 1 0) "1 STI"; static AsmOp compile_store_bool(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 2 && res.size() == 1); @@ -1388,6 +1426,13 @@ void define_builtins() { define_builtin_func("__InMessage.getInMsgParam", ParamsInt1, Int, nullptr, compile_calc_InMessage_getInMsgParam, 0); + define_builtin_method("builder.__storeVarInt", Builder, {Builder, Int, Int, Bool}, Unit, nullptr, + compile_store_varint, // not exposed to stdlib, used in auto-serialization + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf); + define_builtin_method("slice.__loadVarInt", Slice, {Slice, Int, Bool}, Int, nullptr, + compile_fetch_varint, // not exposed to stdlib, used in auto-serialization + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, + {}, {1, 0}); // compile-time only functions, evaluated essentially at compile-time, no runtime implementation // they are placed in stdlib and marked as `builtin` diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index 62ef8d7bbe..083537710c 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -58,7 +58,7 @@ class PackUnpackAvailabilityChecker { if (any_type->try_as()) { return {}; } - if (any_type->try_as()) { + if (any_type->try_as()) { return {}; } if (any_type == TypeDataCoins::create()) { @@ -345,7 +345,7 @@ std::vector generate_lazy_struct_to_cell(CodeBlob& code, SrcLocation ctx.generate_pack_any(hidden_field->declared_type, std::move(ir_field)); } else { std::vector ir_gap_or_tail = loaded_state->get_ir_loaded_aside_field(hidden_field); - if (hidden_field->declared_type->unwrap_alias()->try_as()) { + if (hidden_field->declared_type->unwrap_alias()->try_as()) { ctx.storeSlice(ir_gap_or_tail[0]); } else { ctx.generate_pack_any(hidden_field->declared_type, std::move(ir_gap_or_tail)); diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index 770ec56b90..f38f5d2114 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -65,7 +65,7 @@ PackContext::PackContext(CodeBlob& code, SrcLocation loc, std::vector , f_storeUint(lookup_function("builder.storeUint")) , ir_builder(std::move(ir_builder)) , ir_builder0(this->ir_builder[0]) - , option_skipBitsNFieldsValidation(ir_options[0]) { + , option_skipBitsNValidation(ir_options[0]) { } void PackContext::storeInt(var_idx_t ir_idx, int len) const { @@ -264,16 +264,51 @@ struct S_IntN final : ISerializer { } }; -struct S_BytesN final : ISerializer { +struct S_VariadicIntN final : ISerializer { + const int n_bits; // only 16 and 32 available + const bool is_unsigned; + + explicit S_VariadicIntN(int n_bits, bool is_unsigned) + : n_bits(n_bits), is_unsigned(is_unsigned) {} + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + FunctionPtr f_storeVarInt = lookup_function("builder.__storeVarInt"); + std::vector args = { ctx->ir_builder0, rvect[0], code.create_int(loc, n_bits, "(n-bits)"), code.create_int(loc, is_unsigned, "(is-unsigned)") }; + code.emplace_back(loc, Op::_Call, ctx->ir_builder, std::move(args), f_storeVarInt); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadVarInt = lookup_function("slice.__loadVarInt"); + std::vector args = { ctx->ir_slice0, code.create_int(loc, n_bits, "(n-bits)"), code.create_int(loc, is_unsigned, "(is-unsigned)") }; + std::vector result = code.create_tmp_var(TypeDataInt::create(), loc, "(loaded-varint)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, result[0]}, std::move(args), f_loadVarInt); + return result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + // no TVM instruction to skip, just load but don't use the result + unpack(ctx, code, loc); + } + + PackSize estimate(const EstimateContext* ctx) override { + if (n_bits == 32) { + return PackSize(5, 253); + } else { + return PackSize(4, 124); // same as `coins` + } + } +}; + +struct S_BitsN final : ISerializer { const int n_bits; - explicit S_BytesN(int n_width, bool is_bits) + explicit S_BitsN(int n_width, bool is_bits) : n_bits(is_bits ? n_width : n_width * 8) {} void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { tolk_assert(rvect.size() == 1); - Op& if_disabled_by_user = code.emplace_back(loc, Op::_If, std::vector{ctx->option_skipBitsNFieldsValidation}); + Op& if_disabled_by_user = code.emplace_back(loc, Op::_If, std::vector{ctx->option_skipBitsNValidation}); { code.push_set_cur(if_disabled_by_user.block0); code.close_pop_cur(loc); @@ -400,10 +435,8 @@ struct S_Coins final : ISerializer { } void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { - FunctionPtr f_loadCoins = lookup_function("slice.loadCoins"); - std::vector args = ctx->ir_slice; - std::vector dummy_loaded = code.create_tmp_var(TypeDataInt::create(), loc, "(loaded-coins)"); - code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, dummy_loaded[0]}, std::move(args), f_loadCoins); + // no TVM instruction to skip, just load but don't use the result + unpack(ctx, code, loc); } PackSize estimate(const EstimateContext* ctx) override { @@ -428,9 +461,7 @@ struct S_Address final : ISerializer { // we can't do just // ctx->skipBits(2 + 1 + 8 + 256); // because it may be addr_none or addr_extern; there is no "skip address" in TVM, so just load it - FunctionPtr f_loadAddress = lookup_function("slice.loadAddress"); - std::vector ir_address = code.create_tmp_var(TypeDataSlice::create(), loc, "(tmp-addr)"); - code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_address[0]}, ctx->ir_slice, f_loadAddress); + unpack(ctx, code, loc); } PackSize estimate(const EstimateContext* ctx) override { @@ -1088,10 +1119,13 @@ std::vector auto_generate_opcodes_for_union(TypePtr union_type, std: static std::unique_ptr get_serializer_for_type(TypePtr any_type) { if (const auto* t_intN = any_type->try_as()) { + if (t_intN->is_variadic) { + return std::make_unique(t_intN->n_bits, t_intN->is_unsigned); + } return std::make_unique(t_intN->n_bits, t_intN->is_unsigned); } - if (const auto* t_bytesN = any_type->try_as()) { - return std::make_unique(t_bytesN->n_width, t_bytesN->is_bits); + if (const auto* t_bitsN = any_type->try_as()) { + return std::make_unique(t_bitsN->n_width, t_bitsN->is_bits); } if (any_type == TypeDataCoins::create()) { return std::make_unique(); diff --git a/tolk/pack-unpack-serializers.h b/tolk/pack-unpack-serializers.h index 0e664b9a11..b3427953bc 100644 --- a/tolk/pack-unpack-serializers.h +++ b/tolk/pack-unpack-serializers.h @@ -64,7 +64,7 @@ class PackContext { public: const std::vector ir_builder; const var_idx_t ir_builder0; - const var_idx_t option_skipBitsNFieldsValidation; + const var_idx_t option_skipBitsNValidation; PackContext(CodeBlob& code, SrcLocation loc, std::vector ir_builder, const std::vector& ir_options); diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index aa2e2dd3a9..38a7c492ac 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1026,20 +1026,20 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as()) { + // no changes in rvect, since bitsN is slice at TVM level + if (target_type == TypeDataSlice::create() && original_type->try_as()) { return rvect; } // pass `slice` to `bytes32` // same as above - if (original_type == TypeDataSlice::create() && target_type->try_as()) { + if (original_type == TypeDataSlice::create() && target_type->try_as()) { return rvect; } // pass `bytes32` to `bytes64` / `bits128` to `bytes16` // no changes in rvect - if (original_type->try_as() && target_type->try_as()) { + if (original_type->try_as() && target_type->try_as()) { return rvect; } diff --git a/tolk/pipe-lazy-load-insertions.cpp b/tolk/pipe-lazy-load-insertions.cpp index e7510023a2..64438645b7 100644 --- a/tolk/pipe-lazy-load-insertions.cpp +++ b/tolk/pipe-lazy-load-insertions.cpp @@ -433,10 +433,10 @@ struct ExprUsagesWhileCollecting { // okay, this field is not needed; we should skip it; // try to merge "skip 8 bits" + "skip 16 bits" into a single "skip 24 bits" if (!future_fields.empty() && future_fields.back().action == LazyStructLoadInfo::SkipField) { - if (const TypeDataBytesN* last_bitsN = future_fields.back().field_type->try_as()) { + if (const TypeDataBitsN* last_bitsN = future_fields.back().field_type->try_as()) { PackSize cur_size = estimate_serialization_size(field_type); if (cur_size.min_bits == cur_size.max_bits && cur_size.max_refs == 0) { - TypePtr total_bitsN = TypeDataBytesN::create(true, last_bitsN->n_width + cur_size.max_bits); + TypePtr total_bitsN = TypeDataBitsN::create(last_bitsN->n_width + cur_size.max_bits, true); future_fields.back().field_type = total_bitsN; continue; } @@ -447,7 +447,7 @@ struct ExprUsagesWhileCollecting { TypePtr skip_type = field_type; PackSize skip_size = estimate_serialization_size(field_type); if (skip_size.min_bits == skip_size.max_bits && skip_size.max_refs == 0) { - skip_type = TypeDataBytesN::create(true, skip_size.max_bits); + skip_type = TypeDataBitsN::create(skip_size.max_bits, true); } future_fields.emplace_back(LazyStructLoadInfo::SkipField, "(gap)", skip_type); } diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 2a9bb20288..7ca035cfd4 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -71,24 +71,24 @@ static void fire_error_generic_type_used_without_T(FunctionPtr cur_f, SrcLocatio fire(cur_f, loc, "type `" + type_name_with_Ts + "` is generic, you should provide type arguments"); } -static TypePtr parse_intN(std::string_view strN, bool is_unsigned) { +static TypePtr parse_intN_uintN(std::string_view strN, bool is_unsigned) { int n; - auto result = std::from_chars(strN.data() + 3 + static_cast(is_unsigned), strN.data() + strN.size(), n); + auto result = std::from_chars(strN.data(), strN.data() + strN.size(), n); bool parsed = result.ec == std::errc() && result.ptr == strN.data() + strN.size(); - if (!parsed || n <= 0 || n > 256 + static_cast(is_unsigned)) { + if (!parsed || n <= 0 || n > 257 - static_cast(is_unsigned)) { return nullptr; // `int1000`, maybe it's user-defined alias, let it be unresolved } - return TypeDataIntN::create(is_unsigned, false, n); + return TypeDataIntN::create(n, is_unsigned, false); } -static TypePtr parse_bytesN(std::string_view strN, bool is_bits) { +static TypePtr parse_bytesN_bitsN(std::string_view strN, bool is_bits) { int n; - auto result = std::from_chars(strN.data() + 5 - static_cast(is_bits), strN.data() + strN.size(), n); + auto result = std::from_chars(strN.data(), strN.data() + strN.size(), n); bool parsed = result.ec == std::errc() && result.ptr == strN.data() + strN.size(); if (!parsed || n <= 0 || n > 1024) { return nullptr; // `bytes9999`, maybe it's user-defined alias, let it be unresolved } - return TypeDataBytesN::create(is_bits, n); + return TypeDataBitsN::create(n, is_bits); } static TypePtr try_parse_predefined_type(std::string_view str) { @@ -113,8 +113,12 @@ static TypePtr try_parse_predefined_type(std::string_view str) { if (str == "address") return TypeDataAddress::create(); break; case 8: - if (str == "varint16") return TypeDataIntN::create(false, true, 16); - if (str == "varint32") return TypeDataIntN::create(false, true, 32); + if (str == "varint16") return TypeDataIntN::create(16, false, true); + if (str == "varint32") return TypeDataIntN::create(32, false, true); + break; + case 9: + if (str == "varuint16") return TypeDataIntN::create(16, true, true); + if (str == "varuint32") return TypeDataIntN::create(32, true, true); break; case 12: if (str == "continuation") return TypeDataContinuation::create(); @@ -124,22 +128,22 @@ static TypePtr try_parse_predefined_type(std::string_view str) { } if (str.starts_with("int")) { - if (TypePtr intN = parse_intN(str, false)) { + if (TypePtr intN = parse_intN_uintN(str.substr(3), false)) { return intN; } } - if (str.size() > 4 && str.starts_with("uint")) { - if (TypePtr uintN = parse_intN(str, true)) { + if (str.starts_with("uint")) { + if (TypePtr uintN = parse_intN_uintN(str.substr(4), true)) { return uintN; } } - if (str.size() > 4 && str.starts_with("bits")) { - if (TypePtr bitsN = parse_bytesN(str, true)) { + if (str.starts_with("bits")) { + if (TypePtr bitsN = parse_bytesN_bitsN(str.substr(4), true)) { return bitsN; } } - if (str.size() > 5 && str.starts_with("bytes")) { - if (TypePtr bytesN = parse_bytesN(str, false)) { + if (str.starts_with("bytes")) { + if (TypePtr bytesN = parse_bytesN_bitsN(str.substr(5), false)) { return bytesN; } } diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index b6ffd6022e..6253537e27 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -295,27 +295,27 @@ TypePtr TypeDataBrackets::create(std::vector&& items) { return hash.register_unique(new TypeDataBrackets(hash.children_flags(), std::move(items))); } -TypePtr TypeDataIntN::create(bool is_unsigned, bool is_variadic, int n_bits) { +TypePtr TypeDataIntN::create(int n_bits, bool is_unsigned, bool is_variadic) { TypeDataHasherForUnique hash(1678330938771108027ULL); + hash.feed_hash(n_bits); hash.feed_hash(is_unsigned); hash.feed_hash(is_variadic); - hash.feed_hash(n_bits); if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataIntN(is_unsigned, is_variadic, n_bits)); + return hash.register_unique(new TypeDataIntN(n_bits, is_unsigned, is_variadic)); } -TypePtr TypeDataBytesN::create(bool is_bits, int n_width) { +TypePtr TypeDataBitsN::create(int n_width, bool is_bits) { TypeDataHasherForUnique hash(7810988137199333041ULL); - hash.feed_hash(is_bits); hash.feed_hash(n_width); + hash.feed_hash(is_bits); if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataBytesN(is_bits, n_width)); + return hash.register_unique(new TypeDataBitsN(n_width, is_bits)); } TypePtr TypeDataUnion::create(std::vector&& variants) { @@ -504,7 +504,7 @@ int TypeDataIntN::get_type_id() const { } } -int TypeDataBytesN::get_type_id() const { +int TypeDataBitsN::get_type_id() const { return TypeIdCalculation::assign_type_id(this); } @@ -591,9 +591,9 @@ std::string TypeDataIntN::as_human_readable() const { return s_int + std::to_string(n_bits); } -std::string TypeDataBytesN::as_human_readable() const { - std::string s_bytes = is_bits ? "bits" : "bytes"; - return s_bytes + std::to_string(n_width); +std::string TypeDataBitsN::as_human_readable() const { + std::string s_bits = is_bits ? "bits" : "bytes"; + return s_bits + std::to_string(n_width); } std::string TypeDataUnion::as_human_readable() const { @@ -739,7 +739,7 @@ bool TypeDataSlice::can_rhs_be_assigned(TypePtr rhs) const { if (const TypeDataAlias* rhs_alias = rhs->try_as()) { return can_rhs_be_assigned(rhs_alias->underlying_type); } - return rhs == TypeDataNever::create(); // note, that bytesN/address is NOT automatically cast to slice without `as` operator + return rhs == TypeDataNever::create(); // note, that bitsN/address is NOT automatically cast to slice without `as` operator } bool TypeDataBuilder::can_rhs_be_assigned(TypePtr rhs) const { @@ -881,8 +881,8 @@ bool TypeDataIntN::can_rhs_be_assigned(TypePtr rhs) const { return rhs == TypeDataNever::create(); // `int8` is NOT assignable to `int32` without `as` } -bool TypeDataBytesN::can_rhs_be_assigned(TypePtr rhs) const { - // `slice` is NOT assignable to bytesN without `as` +bool TypeDataBitsN::can_rhs_be_assigned(TypePtr rhs) const { + // `slice` is NOT assignable to bitsN without `as` // `bytes32` is NOT assignable to `bytes256` and even to `bits256` without `as` if (rhs == this) { return true; @@ -1010,7 +1010,7 @@ bool TypeDataCell::can_be_casted_with_as_operator(TypePtr cast_to) const { } bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (cast_to->try_as()) { // `slice` to `bytes32` / `slice` to `bits8` + if (cast_to->try_as()) { // `slice` to `bytes32` / `slice` to `bits8` return true; } if (cast_to == TypeDataAddress::create()) { @@ -1176,8 +1176,8 @@ bool TypeDataIntN::can_be_casted_with_as_operator(TypePtr cast_to) const { return cast_to == TypeDataInt::create() || cast_to == TypeDataCoins::create(); } -bool TypeDataBytesN::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (cast_to->try_as()) { // `bytes256` as `bytes512`, `bits1` as `bytes8` +bool TypeDataBitsN::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to->try_as()) { // `bytes256` as `bytes512`, `bits1` as `bytes8` return true; } if (const TypeDataUnion* to_union = cast_to->try_as()) { // `bytes8` as `slice?` diff --git a/tolk/type-system.h b/tolk/type-system.h index 62841893ac..e9a63c2135 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -458,18 +458,18 @@ class TypeDataBrackets final : public TypeData { * intN is smoothly cast from/to plain int, mathematical operators on intN also "fall back" to general int. */ class TypeDataIntN final : public TypeData { - TypeDataIntN(bool is_unsigned, bool is_variadic, int n_bits) + TypeDataIntN(int n_bits, bool is_unsigned, bool is_variadic) : TypeData(0) + , n_bits(n_bits) , is_unsigned(is_unsigned) - , is_variadic(is_variadic) - , n_bits(n_bits) {} + , is_variadic(is_variadic) {} public: + const int n_bits; const bool is_unsigned; const bool is_variadic; - const int n_bits; - static TypePtr create(bool is_unsigned, bool is_variadic, int n_bits); + static TypePtr create(int n_bits, bool is_unsigned, bool is_variadic); int get_type_id() const override; std::string as_human_readable() const override; @@ -497,22 +497,22 @@ class TypeDataCoins final : public TypeData { }; /* - * `bytes256`, `bits512`, `bytes8` are TypeDataBytesN. At TVM level, it's just slice. - * The purpose of bytesN is to be used in struct fields, describing the way of serialization (n bytes / n bits). - * In this essence, bytesN is very similar to intN. - * Note, that unlike intN automatically cast to/from int, bytesN does NOT auto cast to slice (without `as`). + * `bits512`, `bytes8`, `bits9` are TypeDataBitsN. At TVM level, it's just slice. + * The purpose of bitsN is to be used in struct fields, describing the way of serialization (n bytes / n bits). + * In this essence, bitsN is very similar to intN. + * Note that unlike intN automatically cast to/from int, bitsN does NOT auto cast to slice (without `as`). */ -class TypeDataBytesN final : public TypeData { - TypeDataBytesN(bool is_bits, int n_width) +class TypeDataBitsN final : public TypeData { + TypeDataBitsN(int n_width, bool is_bits) : TypeData(0) - , is_bits(is_bits) - , n_width(n_width) {} + , n_width(n_width) + , is_bits(is_bits) {} public: + const int n_width; // either in bits, or in bytes const bool is_bits; - const int n_width; - static TypePtr create(bool is_bits, int n_width); + static TypePtr create(int n_width, bool is_bits); int get_type_id() const override; std::string as_human_readable() const override; From 8d650aa174ac592ccad1225a0919f06e041990a3 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 29 Jun 2025 23:18:13 +0300 Subject: [PATCH 337/524] [Tolk] Support custom pack/unpack serializers for any T For any type alias, one can now specify > fun T.packToBuilder > fun T.unpackFromSlice That will be called whenever that type is serialized. Perfect to express custom TL/B behavior, like > len: (## 8) > data: (bits (len*8)) --- .../tests/invalid-serialization/err-7750.tolk | 22 +++ .../tests/invalid-serialization/err-7751.tolk | 20 ++ .../tests/invalid-serialization/err-7752.tolk | 20 ++ .../tests/invalid-serialization/err-7753.tolk | 19 ++ .../tests/invalid-serialization/err-7754.tolk | 22 +++ tolk-tester/tests/lazy-load-tests.tolk | 31 +++ tolk-tester/tests/pack-unpack-7.tolk | 182 ++++++++++++++++++ tolk/CMakeLists.txt | 2 +- tolk/pack-unpack-api.cpp | 40 ++++ tolk/pack-unpack-serializers.cpp | 55 ++++++ tolk/pack-unpack-serializers.h | 1 + tolk/pipe-check-serialized-fields.cpp | 6 +- tolk/pipe-detect-inline-in-place.cpp | 16 +- tolk/pipe-infer-types-and-calls.cpp | 2 +- tolk/pipeline.h | 2 +- tolk/tolk.cpp | 2 +- 16 files changed, 433 insertions(+), 9 deletions(-) create mode 100644 tolk-tester/tests/invalid-serialization/err-7750.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7751.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7752.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7753.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7754.tolk create mode 100644 tolk-tester/tests/pack-unpack-7.tolk diff --git a/tolk-tester/tests/invalid-serialization/err-7750.tolk b/tolk-tester/tests/invalid-serialization/err-7750.tolk new file mode 100644 index 0000000000..fcc5329ab3 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7750.tolk @@ -0,0 +1,22 @@ +type CustomInt = int; + +fun CustomInt.packToBuilder(self, mutate b: builder) { +} + +struct WithC { + a: int32; + b: CustomInt; +} + +fun main() { + WithC.fromSlice(""); +} + +/** +@compilation_should_fail +@stderr auto-serialization via fromSlice() is not available for type `WithC` +@stderr because field `WithC.b` of type `CustomInt` can't be serialized +@stderr because type `CustomInt` defines a custom pack function, but does not define unpack +@stderr hint: declare unpacker like this: +@stderr fun CustomInt.unpackFromSlice(mutate s: slice): CustomInt + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7751.tolk b/tolk-tester/tests/invalid-serialization/err-7751.tolk new file mode 100644 index 0000000000..41ddd142e6 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7751.tolk @@ -0,0 +1,20 @@ +type CustomInt = int; + +fun CustomInt.packToBuilder(self, b: builder) { +} + +fun CustomInt.unpackFromSlice(mutate s: slice) { +} + + +fun main() { + (5 as CustomInt).toCell() +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `CustomInt` +@stderr because `CustomInt.packToBuilder()` is declared incorrectly +@stderr hint: it must accept 2 parameters and return nothing: +@stderr fun CustomInt.packToBuilder(self, mutate b: builder) + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7752.tolk b/tolk-tester/tests/invalid-serialization/err-7752.tolk new file mode 100644 index 0000000000..b8dc0b9ecb --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7752.tolk @@ -0,0 +1,20 @@ +type CustomInt = int; + +fun CustomInt.packToBuilder(self, mutate b: builder) { +} + +fun CustomInt.unpackFromSlice(self, mutate s: slice) { +} + + +fun main() { + (5 as CustomInt).toCell() +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `CustomInt` +@stderr because `CustomInt.unpackFromSlice()` is declared incorrectly +@stderr hint: it must accept 1 parameter and return an object: +@stderr fun CustomInt.unpackFromSlice(mutate s: slice) + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7753.tolk b/tolk-tester/tests/invalid-serialization/err-7753.tolk new file mode 100644 index 0000000000..6e30b474b7 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7753.tolk @@ -0,0 +1,19 @@ +type CustomInt = int; + +fun CustomInt.packToBuilder(self, mutate b: builder) { +} + +fun CustomInt.unpackFromSlice(mutate s: slice) { + return (1, 2, 3) +} + +fun main() { + (5 as CustomInt).toCell() +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `CustomInt` +@stderr because `CustomInt.unpackFromSlice()` is declared incorrectly +@stderr hint: it must accept 1 parameter and return an object: + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7754.tolk b/tolk-tester/tests/invalid-serialization/err-7754.tolk new file mode 100644 index 0000000000..21127dd47a --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7754.tolk @@ -0,0 +1,22 @@ +type CustomInt = int; + +fun CustomInt.packToBuilder(self, mutate b: builder) { +} + +fun CustomInt.unpackFromSlice(mutate s: slice) { + if (s.remainingBitsCount() > 10) { + return 123; + } + return 600; +} + +fun main() { + var s = ""; + s.loadAny(); +} + +/** +@compilation_should_fail +@stderr auto-serialization via loadAny() is not available for type `CustomInt` +@stderr because `CustomInt.unpackFromSlice()` can't be inlined; probably, it contains `return` in the middle + */ diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index 208c892412..7a5643a548 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -222,6 +222,25 @@ struct WithVariadicInts { i2: varuint32; } +global gModByCustom: int; + +type MagicGlobalModifier = (); + +fun MagicGlobalModifier.packToBuilder(self, mutate b: builder) { + b.storeUint(gModByCustom, 8); +} + +fun MagicGlobalModifier.unpackFromSlice(mutate s: slice) { + gModByCustom = s.loadUint(8); + return (); +} + +struct WithGlobalModifier { + a: int8; + g: MagicGlobalModifier = (); + n: int8; +} + @noinline fun contract.getFakeData(seqno: int): Cell { @@ -951,6 +970,17 @@ fun demo157() { return d.i2 == (1 << 200); } +@method_id(158) +fun demo158() { + val m1 = lazy WithGlobalModifier.fromSlice(stringHexToSlice("01FF")); + __expect_lazy("[m1] skip (bits8), load g"); + m1.g; // modifies gModByCustom by a custom unpack function + val after1 = gModByCustom; + val m2 = lazy WithGlobalModifier.fromSlice(stringHexToSlice("010A02")); + __expect_lazy("[m2] skip (bits8) (MagicGlobalModifier), load n"); + return (m2.n, gModByCustom, after1); // also modifies by skipping (unpack called) +} + @method_id(200) fun demo200() { @@ -1638,6 +1668,7 @@ fun main() { @testcase | 156 | x{0110} | 16 @testcase | 156 | x{FF10} | -100 @testcase | 157 | | -1 +@testcase | 158 | | 2 10 255 @testcase | 200 | | 1 @testcase | 201 | | 1 2 -1 diff --git a/tolk-tester/tests/pack-unpack-7.tolk b/tolk-tester/tests/pack-unpack-7.tolk new file mode 100644 index 0000000000..07407ff91f --- /dev/null +++ b/tolk-tester/tests/pack-unpack-7.tolk @@ -0,0 +1,182 @@ +// encoded as TL/B `len: (## 8) data: (bits (len*8))` +type TelegramString = slice; + +fun TelegramString.packToBuilder(self, mutate b: builder) { + val bytes = self.remainingBitsCount() / 8; + b.storeUint(bytes, 8); + b.storeSlice(self); +} + +fun TelegramString.unpackFromSlice(mutate s: slice) { + val bytes = s.loadUint(8); + return s.loadBits(bytes * 8); +} + +type Custom8 = int; + +fun Custom8.packToBuilder(self, mutate b: builder) { + b.storeUint(self, 8) +} + +fun Custom8.unpackFromSlice(mutate s: slice) { + return s.loadUint(8) +} + +struct StorWithStr { + a: int32; + str: TelegramString; + b: int32; +} + +struct PointWithCustomInt { + a: Custom8; + b: int8; +} + + +type MyBorderedInt = int; + +fun MyBorderedInt.packToBuilder(self, mutate b: builder) { + if (self > 10) { b.storeUint(1, 4) } + else if (self > 0) { b.storeUint(2, 4) } + else { b.storeUint(3, 4) } +} + +fun MyBorderedInt.unpackFromSlice(mutate s: slice) { + return match (s.loadUint(4)) { + 1 => 10, + 2 => 0, + 3 => -1, + else => throw 123 + } +} + +struct WithMyBorder { + a: int8; + b: MyBorderedInt; +} + +type MyCustomNothing = (); + +struct WithFakeWriter { + a: int8; + fake: MyCustomNothing = (); + b: int8; +} + +fun MyCustomNothing.packToBuilder(self, mutate b: builder) { + b.storeUint(123, 32); + b.storeRef(createEmptyCell()); +} + +global gModByCustom: int; + +type MagicGlobalModifier = (); + +fun MagicGlobalModifier.packToBuilder(self, mutate b: builder) { + b.storeUint(gModByCustom, 8); +} + +fun MagicGlobalModifier.unpackFromSlice(mutate s: slice) { + gModByCustom = s.loadUint(8); + return (); +} + +struct WithGlobalModifier { + a: int8; + g: MagicGlobalModifier = (); + n: int8; +} + +type Tensor3Skipping1 = (int, int, int); + +fun Tensor3Skipping1.unpackFromSlice(mutate s: slice): Tensor3Skipping1 { + val e0 = s.loadUint(8); + val e2 = s.loadUint(8); + return (e0, 0, e2); +} + +fun Tensor3Skipping1.packToBuilder(self, mutate b: builder) { + b.storeUint(self.0, 8).storeUint(self.2, 8); +} + + +@method_id(101) +fun test1() { + var t: StorWithStr = { a: 10, str: "abc", b: 20 }; + var c = t.toCell(); + var back = c.load(); + return ((t.b == back.b) & (t.str.bitsEqual(back.str)), back.str.remainingBitsCount(), c.hash() & 0xFFFF); +} + +@method_id(102) +fun test2() { + var s = ("" as TelegramString, "" as TelegramString); + return beginCell().storeAny(s).endCell().beginParse().remainingBitsCount(); +} + +@method_id(103) +fun test3() { + return PointWithCustomInt.fromSlice(stringHexToSlice("0102")); +} + +@method_id(104) +fun test4(initialInt: int) { + var c = WithMyBorder { a: 0, b: initialInt }.toCell(); + return c.load().b; +} + +@method_id(105) +fun test5() { + var f: WithFakeWriter = { a: 10, b: 20 }; + var s = beginCell().storeAny(f).endCell().beginParse(); + val size = s.remainingBitsAndRefsCount(); + return (s.skipBits(8).loadUint(32), size.1); +} + +@method_id(106) +fun test6() { + gModByCustom = 6; + val m1: WithGlobalModifier = { a: 8, n: 16 }; + val c1 = m1.toCell(); + val r2 = WithGlobalModifier.fromSlice(stringHexToSlice("01FF02")); + val gAfter2 = gModByCustom; + WithGlobalModifier.fromSlice(stringHexToSlice("010002")); // not deleted, sets 0 + val gAfter3 = gModByCustom; + var s = stringHexToSlice("010902FF"); + s.skipAny(); // custom loader also called + return (c1.beginParse().skipBits(8).loadUint(8), r2.n, gAfter2, gAfter3, s.remainingBitsCount(), gModByCustom); +} + +@method_id(107) +fun test7() { + val t = (1, 2, 3) as Tensor3Skipping1; + __expect_type(t.toCell(), "Cell"); + return t.toCell().load(); +} + +fun main() { + __expect_type("" as TelegramString, "TelegramString"); +} + +/** +@testcase | 101 | | -1 24 5203 +@testcase | 102 | | 16 +@testcase | 103 | | 1 2 +@testcase | 104 | 55 | 10 +@testcase | 104 | 8 | 0 +@testcase | 104 | -5 | -1 +@testcase | 105 | | 123 1 +@testcase | 106 | | 6 2 255 0 8 9 +@testcase | 107 | | 1 0 3 + +@fif_codegen +""" + test3() PROC:<{ + x{0102} PUSHSLICE + 8 LDU + 8 LDI + DROP + }> +""" + */ diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index bc8a6daa01..033456e7fa 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -20,10 +20,10 @@ set(TOLK_SOURCE pipe-refine-lvalue-for-mutate.cpp pipe-check-rvalue-lvalue.cpp pipe-check-pure-impure.cpp - pipe-check-serialized-fields.cpp pipe-constant-folding.cpp pipe-optimize-boolean-expr.cpp pipe-detect-inline-in-place.cpp + pipe-check-serialized-fields.cpp pipe-lazy-load-insertions.cpp pipe-transform-on-message.cpp pipe-ast-to-legacy.cpp diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index 083537710c..abab637e97 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -53,6 +53,26 @@ struct CantSerializeBecause { class PackUnpackAvailabilityChecker { std::vector already_checked; + static bool check_declared_packToBuilder(TypePtr receiver_type, FunctionPtr f_pack) { + if (!f_pack->does_accept_self() || f_pack->does_mutate_self() || f_pack->get_num_params() != 2) { + return false; + } + if (f_pack->get_param(1).declared_type != TypeDataBuilder::create() || !f_pack->has_mutate_params()) { + return false; + } + return f_pack->inferred_return_type->get_width_on_stack() == 0; + } + + static bool check_declared_unpackFromSlice(TypePtr receiver_type, FunctionPtr f_unpack) { + if (f_unpack->does_accept_self() || f_unpack->get_num_params() != 1) { + return false; + } + if (f_unpack->get_param(0).declared_type != TypeDataSlice::create() || !f_unpack->has_mutate_params()) { + return false; + } + return f_unpack->inferred_return_type->equal_to(receiver_type); + } + public: std::optional detect_why_cant_serialize(TypePtr any_type, bool is_pack) { if (any_type->try_as()) { @@ -137,6 +157,26 @@ class PackUnpackAvailabilityChecker { if (t_alias->alias_ref->name == "RemainingBitsAndRefs") { // it's built-in RemainingBitsAndRefs (slice) return {}; } + if (FunctionPtr f_pack = get_custom_pack_unpack_function(t_alias, true)) { + std::string receiver_name = t_alias->alias_ref->as_human_readable(); + if (!check_declared_packToBuilder(t_alias, f_pack)) { + return CantSerializeBecause("because `" + receiver_name + ".packToBuilder()` is declared incorrectly\nhint: it must accept 2 parameters and return nothing:\n> fun " + receiver_name + ".packToBuilder(self, mutate b: builder)"); + } + if (!f_pack->is_inlined_in_place()) { + return CantSerializeBecause("because `" + receiver_name + ".packToBuilder()` can't be inlined; probably, it contains `return` in the middle"); + } + if (FunctionPtr f_unpack = get_custom_pack_unpack_function(t_alias, false)) { + if (!check_declared_unpackFromSlice(t_alias, f_unpack)) { + return CantSerializeBecause("because `" + receiver_name + ".unpackFromSlice()` is declared incorrectly\nhint: it must accept 1 parameter and return an object:\n> fun " + receiver_name + ".unpackFromSlice(mutate s: slice): " + receiver_name); + } + if (!f_unpack->is_inlined_in_place()) { + return CantSerializeBecause("because `" + receiver_name + ".unpackFromSlice()` can't be inlined; probably, it contains `return` in the middle"); + } + } else if (!is_pack) { + return CantSerializeBecause("because type `" + receiver_name + "` defines a custom pack function, but does not define unpack\nhint: declare unpacker like this:\n> fun " + receiver_name + ".unpackFromSlice(mutate s: slice): " + receiver_name); + } + return {}; + } if (auto why = detect_why_cant_serialize(t_alias->underlying_type, is_pack)) { return CantSerializeBecause("because alias `" + t_alias->as_human_readable() + "` expands to `" + t_alias->underlying_type->as_human_readable() + "`", why.value()); } diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index f38f5d2114..ecad405021 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -40,6 +40,7 @@ class LValContext; std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr target_type = nullptr, LValContext* lval_ctx = nullptr); std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc); std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc); +std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, SrcLocation loc, FunctionPtr f_inlined, AnyExprV self_obj, bool is_before_immediate_return, const std::vector>& vars_per_arg); bool is_type_cellT(TypePtr any_type) { if (const TypeDataStruct* t_struct = any_type->try_as()) { @@ -49,6 +50,24 @@ bool is_type_cellT(TypePtr any_type) { return false; } +// For any type alias, one can declare custom pack/unpack functions: +// > type TelegramString = slice +// > fun TelegramString.packToBuilder(self, mutate b: builder) { ... } +// > fun TelegramString.unpackFromSlice(mutate s: slice): TelegramString { ... } +// It's externally checked in advance that it's declared correctly. +FunctionPtr get_custom_pack_unpack_function(TypePtr receiver_type, bool is_pack) { + if (const TypeDataAlias* t_alias = receiver_type->try_as()) { + if (t_alias->alias_ref->is_instantiation_of_generic_alias()) { + // does not work for generic aliases currently, because `MyAlias.pack` was not instantiated earlier + return nullptr; + } + std::string receiver_name = t_alias->alias_ref->as_human_readable(); + if (const Symbol* sym = lookup_global_symbol(receiver_name + (is_pack ? ".packToBuilder" : ".unpackFromSlice"))) { + return sym->try_as(); + } + } + return nullptr; +} // -------------------------------------------- // options, context, common helpers @@ -1037,6 +1056,39 @@ struct S_CustomStruct final : ISerializer { } }; +struct S_CustomReceiverForPackUnpack final : ISerializer { + TypePtr receiver_type; + + explicit S_CustomReceiverForPackUnpack(TypePtr receiver_type) + : receiver_type(receiver_type) {} + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + FunctionPtr f_pack = get_custom_pack_unpack_function(receiver_type, true); + tolk_assert(f_pack && f_pack->does_accept_self() && f_pack->inferred_return_type->get_width_on_stack() == 0); + std::vector vars_per_arg = { std::move(rvect), ctx->ir_builder }; + std::vector ir_mutated_builder = gen_inline_fun_call_in_place(code, TypeDataBuilder::create(), loc, f_pack, nullptr, false, vars_per_arg); + code.emplace_back(loc, Op::_Let, ctx->ir_builder, std::move(ir_mutated_builder)); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_unpack = get_custom_pack_unpack_function(receiver_type, false); + tolk_assert(f_unpack && f_unpack->inferred_return_type->get_width_on_stack() == receiver_type->get_width_on_stack()); + TypePtr ret_type = TypeDataTensor::create({TypeDataSlice::create(), receiver_type}); + std::vector ir_slice_and_res = gen_inline_fun_call_in_place(code, ret_type, loc, f_unpack, nullptr, false, {ctx->ir_slice}); + code.emplace_back(loc, Op::_Let, ctx->ir_slice, std::vector{ir_slice_and_res.front()}); + return std::vector(ir_slice_and_res.begin() + 1, ir_slice_and_res.end()); + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + // just load and ignore the result + unpack(ctx, code, loc); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize::unpredictable_infinity(); + } +}; + // -------------------------------------------- // automatically generate opcodes @@ -1192,6 +1244,9 @@ static std::unique_ptr get_serializer_for_type(TypePtr any_type) { if (t_alias->alias_ref->name == "RemainingBitsAndRefs") { return std::make_unique(); } + if (get_custom_pack_unpack_function(t_alias, true)) { + return std::make_unique(t_alias); + } return get_serializer_for_type(t_alias->underlying_type); } diff --git a/tolk/pack-unpack-serializers.h b/tolk/pack-unpack-serializers.h index b3427953bc..ab11733aa5 100644 --- a/tolk/pack-unpack-serializers.h +++ b/tolk/pack-unpack-serializers.h @@ -163,6 +163,7 @@ class EstimateContext { bool is_type_cellT(TypePtr any_type); +FunctionPtr get_custom_pack_unpack_function(TypePtr receiver_type, bool is_pack); std::vector auto_generate_opcodes_for_union(TypePtr union_type, std::string& because_msg); } // namespace tolk diff --git a/tolk/pipe-check-serialized-fields.cpp b/tolk/pipe-check-serialized-fields.cpp index a13032e372..f8b9b9cd02 100644 --- a/tolk/pipe-check-serialized-fields.cpp +++ b/tolk/pipe-check-serialized-fields.cpp @@ -82,10 +82,10 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody TypePtr serialized_type = nullptr; bool is_pack = false; if (f_name == "Cell.load" || f_name == "T.fromSlice" || f_name == "T.fromCell" || f_name == "T.toCell" || - f_name == "T.loadAny" || f_name == "slice.skipAny" || f_name == "slice.storeAny" || f_name == "T.estimatePackSize" || + f_name == "T.loadAny" || f_name == "slice.skipAny" || f_name == "slice.loadAny" || f_name == "builder.storeAny" || f_name == "T.estimatePackSize" || f_name == "createMessage" || f_name == "createExternalLogMessage") { serialized_type = fun_ref->substitutedTs->typeT_at(0); - is_pack = f_name == "T.toCell" || f_name == "slice.storeAny" || f_name == "T.estimatePackSize" || f_name == "createMessage" || f_name == "createExternalLogMessage"; + is_pack = f_name == "T.toCell" || f_name == "builder.storeAny" || f_name == "T.estimatePackSize" || f_name == "createMessage" || f_name == "createExternalLogMessage"; } else { return; // not a serialization function } @@ -111,7 +111,7 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody }; void pipeline_check_serialized_fields() { - visit_ast_of_all_functions(); + visit_ast_of_all_functions(); } } // namespace tolk diff --git a/tolk/pipe-detect-inline-in-place.cpp b/tolk/pipe-detect-inline-in-place.cpp index 4221347f90..7e892b687f 100644 --- a/tolk/pipe-detect-inline-in-place.cpp +++ b/tolk/pipe-detect-inline-in-place.cpp @@ -51,6 +51,19 @@ namespace tolk { // (purpose: functions in recursive call chains can't be inlined) static std::unordered_map> call_graph; +static bool is_called_implicitly_by_compiler(FunctionPtr f) { + if (f->name == "onBouncedMessage") { + return true; + } + if (f->is_method() && f->method_name == "packToBuilder") { + return f->does_accept_self() && !f->does_mutate_self() && f->get_num_params() == 2 && f->has_mutate_params(); + } + if (f->is_method() && f->method_name == "unpackFromSlice") { + return !f->does_accept_self() && f->get_num_params() == 1 && f->has_mutate_params(); + } + return false; +} + // when traversing a function, collect some AST metrics used to detect whether it's lightweight struct StateWhileTraversingFunction { FunctionPtr fun_ref; @@ -227,8 +240,7 @@ class DetectIfToInlineFunctionInPlaceVisitor final : ASTVisitorFunctionBody { } // okay, this function will be inlined, mark the flag - bool is_called = fun_ref->n_times_called - || fun_ref->name == "onBouncedMessage"; // implicitly called by the compiler from onInternalMessage() + bool is_called = fun_ref->n_times_called || is_called_implicitly_by_compiler(fun_ref); if (will_inline && is_called) { fun_ref->mutate()->assign_inline_mode_in_place(); } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 432802279a..1253cdffc3 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -1073,7 +1073,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // check for method (`t.size` / `user.getId`); even `i.0()` can be here if `fun int.0(self)` exists // for `T.copy` / `Container.create`, substitution for T is also returned if (!fun_ref) { - std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, dot_obj->loc, obj_type, field_name); + std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, dot_obj->loc, dot_obj->inferred_type, field_name); } // not a field, not a method — fire an error diff --git a/tolk/pipeline.h b/tolk/pipeline.h index eb1c153b4d..641beb06b1 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -41,10 +41,10 @@ void pipeline_check_inferred_types(); void pipeline_refine_lvalue_for_mutate_arguments(); void pipeline_check_rvalue_lvalue(); void pipeline_check_pure_impure_operations(); -void pipeline_check_serialized_fields(); void pipeline_constant_folding(); void pipeline_optimize_boolean_expressions(); void pipeline_detect_inline_in_place(); +void pipeline_check_serialized_fields(); void pipeline_lazy_load_insertions(); void pipeline_transform_onInternalMessage(); void pipeline_convert_ast_to_legacy_Expr_Op(); diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp index 63a2dd7f25..69b03f20ad 100644 --- a/tolk/tolk.cpp +++ b/tolk/tolk.cpp @@ -64,10 +64,10 @@ int tolk_proceed(const std::string &entrypoint_filename) { pipeline_refine_lvalue_for_mutate_arguments(); pipeline_check_rvalue_lvalue(); pipeline_check_pure_impure_operations(); - pipeline_check_serialized_fields(); pipeline_constant_folding(); pipeline_optimize_boolean_expressions(); pipeline_detect_inline_in_place(); + pipeline_check_serialized_fields(); pipeline_lazy_load_insertions(); pipeline_transform_onInternalMessage(); pipeline_convert_ast_to_legacy_Expr_Op(); From 63670d2449991b99be40bb5fb6119036e817770b Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 30 Jun 2025 20:52:55 +0300 Subject: [PATCH 338/524] [Tolk] Use of `AutoDeployAddress` for calculation aside createMessage New functions in stdlib: > AutoDeployAddress.buildAddress > AutoDeployAddress.addressMatches Based on the same philosophy as createMessage --- crypto/smartcont/tolk-stdlib/common.tolk | 28 +- tolk-tester/tests/build-addr-tests.tolk | 310 +++++++++++++++++++++++ tolk-tester/tests/parse-address.tolk | 91 ------- tolk/builtins.cpp | 13 + tolk/pipe-ast-to-legacy.cpp | 9 + tolk/send-message-api.cpp | 253 +++++++++++++++--- tolk/send-message-api.h | 3 + 7 files changed, 576 insertions(+), 131 deletions(-) create mode 100644 tolk-tester/tests/build-addr-tests.tolk diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 7cf61f5623..da52bfb98f 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -1056,17 +1056,39 @@ struct AddressShardingOptions { /// }, /// ... /// ``` -/// It's often called "deployment", but it's inaccurate. TON Blockchain doesn't have a dedicated -/// deployment mechanism. You just send a message to some address — and if it doesn't exist, but you've -/// attached a way to initialize it (code+data) — it's initialized immediately, and accepts your message. /// You just provide code+data, and the compiler automatically calculates the destination address, /// because in TON, the address of a contract, by definition, is a hash of its initial state. +/// You can also use it without sending a message. See [buildAddress] and [addressMatches]. struct AutoDeployAddress { workchain: int8 = BASECHAIN; stateInit: ContractState | cell; toShard: AddressShardingOptions? = null; } +/// Constructs an address that a deployed contract will have. +/// For example, from a jetton minter, you want to calculate an address of a jetton wallet: +/// ``` +/// val jwDeployed = calcDeployedJettonWallet(...); +/// val jwAddrBuilt = jwDeployed.buildAddress(); +/// ``` +/// Just instead of `dest` for [createMessage], you use it is to calculate a jetton/nft address. +/// Note: returns `builder`, not `address`! It's cheap. +/// If you really need `address`, use `address.fromValidBuilder(res)`. +@pure +fun AutoDeployAddress.buildAddress(self): builder + builtin; + +/// Checks that an address matches a deployed contract. +/// For example, from a jetton minter, you're checking whether a message is from a jetton wallet: +/// ``` +/// val jwDeployed = calcDeployedJettonWallet(...); +/// val fromJW = jwDeployed.addressMatches(senderAddress); +/// ``` +/// Just instead of `dest` for [createMessage], you use it is to check whether a sender is a jetton/nft. +@pure +fun AutoDeployAddress.addressMatches(self, addr: address): bool + builtin; + /// Options for creating an outgoing message. /// Consider [createMessage] for examples. struct CreateMessageOptions { diff --git a/tolk-tester/tests/build-addr-tests.tolk b/tolk-tester/tests/build-addr-tests.tolk new file mode 100644 index 0000000000..788450a556 --- /dev/null +++ b/tolk-tester/tests/build-addr-tests.tolk @@ -0,0 +1,310 @@ +const SHARD_DEPTH = 8; + +fun getMyAddressDev(): address + asm "x{80194DC6438F99D3D9DBE151944925D90B2492954BF6B9C070FBFF2DDED5F30547D_} PUSHSLICE"; + +struct WalletStorage { + balance: coins; + ownerAddress: address; + minterAddress: address; +} + +fun WalletStorage.generateEmptyData(ownerAddress: address, minterAddress: address) { + val emptyWalletStorage: WalletStorage = { + balance: 0, + ownerAddress, + minterAddress, + }; + return emptyWalletStorage.toCell(); +} + +@inline_ref +fun buildAddrInShard_manual(a: address, options: AddressShardingOptions) { + var sb = options.closeTo as slice; + sb.skipBits(3); // addr_std$10 + anycast 0 + val wc_b = sb.loadInt(8); + val shardPrefix = sb.loadUint(options.fixedPrefixLength); + + var sa = a as slice; + sa.skipBits(3 + 8 + options.fixedPrefixLength); + + return beginCell() + .storeUint(0b100, 3) // addr_std$10 + anycast 0 + .storeInt(wc_b, 8) + .storeUint(shardPrefix, options.fixedPrefixLength) + .storeSlice(sa); +} + +@method_id(101) +fun test1() { + val a = address("0:00000000000000000000000000000000000000000000000000000000000000FF"); + val b = address("1:1100000000000000000000000000000000000000000000000000000000000000"); + val dd1 = buildAddrInShard_manual(a, {fixedPrefixLength: 8, closeTo: b}); + val dd2 = a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: b}); + assert(dd1.endCell().hash() == dd2.endCell().hash(), 400); + return dd1.endCell().beginParse() as address + == address("1:11000000000000000000000000000000000000000000000000000000000000FF"); +} + +@method_id(102) +fun test2() { + val a = address("0:1234aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + val b = address("0:FFFFbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + val dd1 = buildAddrInShard_manual(a, {closeTo: b, fixedPrefixLength: 16}); + val dd2 = a.buildSameAddressInAnotherShard({fixedPrefixLength: 16, closeTo: b}); + assert((a as slice).remainingBitsCount() == 267, 267); + assert((b as slice).remainingBitsCount() == 267, 267); + assert(dd1.endCell().hash() == dd2.endCell().hash(), 400); + return address.fromValidBuilder(dd2) + == address("0:FFFFaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); +} + +@method_id(103) +fun test3() { + var t1_a = address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"); + var t1_b = address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); + var t2_a = address("EQCe4AYIBce1pAk2qJJPSs1OzyZRlKjkfq8zuC8D7erv6DUP"); + var t2_b = address("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"); + var t3_a = address("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"); + var t3_b = address("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"); + return ( + t1_a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: t1_b}).endCell().hash() & 0xFFFF, + t1_a.buildSameAddressInAnotherShard({fixedPrefixLength: 4, closeTo: t1_b}).endCell().hash() & 0xFFFF, + t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 2, closeTo: t2_b}).endCell().hash() & 0xFFFF, + t3_a.buildSameAddressInAnotherShard({fixedPrefixLength: 30, closeTo: t2_b}).endCell().hash() & 0xFFFF, + t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 1, closeTo: t3_b}).endCell().hash() & 0xFFFF, + t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 0, closeTo: t3_b}).endCell().hash() & 0xFFFF, + ) +} + +@method_id(104) +fun test4(shardDepth: int) { + var a = address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"); + var b = address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); + return a.buildSameAddressInAnotherShard({fixedPrefixLength: shardDepth, closeTo: b}).endCell().hash() & 0xFFFF; +} + +@method_id(105) +fun test5() { + var b = beginCell() + .storeUint(0b100, 3) // std addr no anycast + .storeInt(MASTERCHAIN, 8) + .storeUint(0xFFFF, 256); + val a = address("-1:000000000000000000000000000000000000000000000000000000000000FFFF"); + return address.fromValidBuilder(b) == a; +} + +@method_id(106) +fun test6() { + val a = address("0:00000000000000000000000000000000000000000000000000000000000000FF"); + val b = address("1:1100000000000000000000000000000000000000000000000000000000000000"); + return address.fromValidBuilder(a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: b})) == + address("1:11000000000000000000000000000000000000000000000000000000000000FF"); +} + + +fun manual_buildAddressOfJettonWallet_plain(ownerAddress: address, minterAddress: address, jettonWalletCode: cell): builder { + val stateInitHash = StateInit.calcHashCodeData( + jettonWalletCode, + WalletStorage.generateEmptyData(ownerAddress, minterAddress) + ); + return beginCell() + .storeUint(0b100, 3) + .storeUint(BASECHAIN, 8) + .storeUint(stateInitHash, 256) +} + +fun manual_buildAddressOfJettonWallet_sharded(ownerAddress: address, minterAddress: address, jettonWalletCode: cell): builder { + val stateInitHash = StateInit.calcHashPrefixCodeData( + SHARD_DEPTH, + jettonWalletCode, + WalletStorage.generateEmptyData(ownerAddress, minterAddress) + ); + var mask = stateInitHash & ((1 << (256 - SHARD_DEPTH)) - 1); + val shard_prefix = (ownerAddress as slice).getMiddleBits(3 + 8, SHARD_DEPTH); + return beginCell() + .storeUint(0b100, 3) + .storeUint(BASECHAIN, 8) + .storeSlice(shard_prefix) + .storeUint( mask, 256 - SHARD_DEPTH); +} + +fun address.manual_isAddressOfJettonWallet_plain(self, ownerAddress: address, minterAddress: address, jettonWalletCode: cell) { + val stateInitHash = StateInit.calcHashCodeData( + jettonWalletCode, + WalletStorage.generateEmptyData(ownerAddress, minterAddress) + ); + val (wc, hash) = self.getWorkchainAndHash(); + return (stateInitHash == hash) & (BASECHAIN == wc); +} + +fun address.manual_isAddressOfJettonWallet_sharded(self, ownerAddress: address, minterAddress: address, jettonWalletCode: cell) { + var stateInitHash = StateInit.calcHashPrefixCodeData( + SHARD_DEPTH, + jettonWalletCode, + WalletStorage.generateEmptyData(ownerAddress, minterAddress) + ); + val mask = (1 << (256 - SHARD_DEPTH)) - 1; + stateInitHash = stateInitHash & mask; + var (wc, hash) = self.getWorkchainAndHash(); + hash = hash & mask; + return (stateInitHash == hash) & (BASECHAIN == wc); +} + +@method_id(110) +fun test10() { + val jwCode: cell = beginCell().storeInt(0x273849723892, 94).endCell(); + val ownerAddress = address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); + val minterAddress = getMyAddressDev(); + + val b1 = manual_buildAddressOfJettonWallet_plain(ownerAddress, minterAddress, jwCode); + val si2: AutoDeployAddress = { + stateInit: { code: jwCode, data: WalletStorage.generateEmptyData(ownerAddress, minterAddress) } + }; + val b2 = si2.buildAddress(); + assert (b1.endCell().hash() == b2.endCell().hash()) throw 123; + + val checkedAddr = b1.endCell().beginParse() as address; + assert (checkedAddr.manual_isAddressOfJettonWallet_plain(ownerAddress, minterAddress, jwCode)) throw 123; + assert (si2.addressMatches(checkedAddr)) throw 123; + + return ( + si2.addressMatches(ownerAddress), + AutoDeployAddress { + workchain: -1, + stateInit: { code: jwCode, data: WalletStorage.generateEmptyData(ownerAddress, minterAddress) } + }.addressMatches(checkedAddr) + ) +} + +@method_id(111) +fun test11() { + val jwCode: cell = beginCell().storeInt(0x273849723892, 94).endCell(); + val ownerAddress = address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); + val minterAddress = getMyAddressDev(); + + val b1 = manual_buildAddressOfJettonWallet_sharded(ownerAddress, minterAddress, jwCode); + val si2: AutoDeployAddress = { + stateInit: { code: jwCode, data: WalletStorage.generateEmptyData(ownerAddress, minterAddress) }, + toShard: { fixedPrefixLength: SHARD_DEPTH, closeTo: ownerAddress } + }; + val b2 = si2.buildAddress(); + assert (b1.endCell().hash() == b2.endCell().hash()) throw 123; + + val checkedAddr = b1.endCell().beginParse() as address; + assert (checkedAddr.manual_isAddressOfJettonWallet_sharded(ownerAddress, minterAddress, jwCode)) throw 123; + assert (si2.addressMatches(checkedAddr)) throw 123; + + return ( + AutoDeployAddress { + stateInit: { code: jwCode, data: WalletStorage.generateEmptyData(ownerAddress, minterAddress) }, + }.addressMatches(checkedAddr), + si2.addressMatches(ownerAddress), + AutoDeployAddress { + workchain: -1, + stateInit: { code: jwCode, data: WalletStorage.generateEmptyData(ownerAddress, minterAddress) }, + toShard: { fixedPrefixLength: SHARD_DEPTH, closeTo: ownerAddress } + }.addressMatches(checkedAddr) + ) +} + +@method_id(112) +fun test12() { + val jwCode: cell = beginCell().storeInt(0x273849723892, 94).endCell(); + val ownerAddress = address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); + var i = 1; + while (i < 10) { + var params: AutoDeployAddress = { + workchain: -1, + stateInit: { code: jwCode, data: createEmptyCell() }, + toShard: { fixedPrefixLength: i, closeTo: ownerAddress } + }; + val addrBuilt = params.buildAddress(); + val addr = address.fromValidBuilder(addrBuilt); + assert (params.addressMatches(addr)) throw 123; + params.workchain = 0; + assert (!params.addressMatches(addr)) throw 123; + params.workchain = -1; + params.toShard!.fixedPrefixLength += 1; + assert (!params.addressMatches(addr)) throw 123; + i += 1 + } + return i; +} + +@method_id(113) +fun test13() { + val stateInitCell = StateInit { + fixedPrefixLength: null, + code: createEmptyCell(), + data: beginCell().storeUint(123, 32).endCell(), + special: null, + library: null, + }.toCell(); + + val manualBuilt = beginCell().storeUint(0b100, 3).storeUint(0, 8) + .storeUint(stateInitCell.hash(), 256); + val si: AutoDeployAddress = { + stateInit: stateInitCell + }; + val addrBuilt = si.buildAddress(); + assert (manualBuilt.endCell().hash() == addrBuilt.endCell().hash()) throw 123; + + val addr = addrBuilt.endCell().beginParse() as address; + return ( + si.addressMatches(addr), + AutoDeployAddress {workchain: 99, stateInit: stateInitCell}.addressMatches(addr), + ) +} + +@method_id(114) +fun test14() { + val ownerAddress = address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); + val stateInitCell = StateInit { + fixedPrefixLength: 10, + code: createEmptyCell(), + data: beginCell().storeUint(123, 32).endCell(), + special: null, + library: null, + }.toCell(); + + val manualBuilt = beginCell().storeUint(0b100, 3).storeUint(9, 8) + .storeSlice((ownerAddress as slice).getMiddleBits(3+8, 10)) + .storeUint(stateInitCell.hash() & ((1 << 246) - 1), 246); + val si: AutoDeployAddress = { + workchain: 9, + stateInit: stateInitCell, + toShard: { fixedPrefixLength: 10, closeTo: ownerAddress } + }; + val addrBuilt = si.buildAddress(); + val addr = addrBuilt.endCell().beginParse() as address; + + val stateInitHash = stateInitCell.hash(); + val (wc, hash) = addr.getWorkchainAndHash(); + val manual_isAddrOfContract = ((stateInitHash & ((1 << 246) - 1)) == (hash & ((1 << 246) - 1))) & (9 == wc); + + return ( + manualBuilt.endCell().hash() == addrBuilt.endCell().hash(), + manual_isAddrOfContract, + si.addressMatches(addr), + ) +} + + + +fun main() {} + +/** +@testcase | 101 | | -1 +@testcase | 102 | | -1 +@testcase | 103 | | 39876 24338 15241 50719 11252 15241 +@testcase | 104 | 2 | 24338 +@testcase | 104 | 9 | 39876 +@testcase | 105 | | -1 +@testcase | 106 | | -1 +@testcase | 110 | | 0 0 +@testcase | 111 | | 0 0 0 +@testcase | 112 | | 10 +@testcase | 113 | | -1 0 +@testcase | 114 | | -1 -1 -1 + */ diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index c489527ade..0079871a62 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -36,90 +36,6 @@ fun codegenAddrEq(a: address, b: address) { return 3; } -@inline_ref -fun buildAddrInShard_manual(a: address, options: AddressShardingOptions) { - var sb = options.closeTo as slice; - sb.skipBits(3); // addr_std$10 + anycast 0 - val wc_b = sb.loadInt(8); - val shardPrefix = sb.loadUint(options.fixedPrefixLength); - - var sa = a as slice; - sa.skipBits(3 + 8 + options.fixedPrefixLength); - - return beginCell() - .storeUint(0b100, 3) // addr_std$10 + anycast 0 - .storeInt(wc_b, 8) - .storeUint(shardPrefix, options.fixedPrefixLength) - .storeSlice(sa); -} - -@method_id(101) -fun test1() { - val a = address("0:00000000000000000000000000000000000000000000000000000000000000FF"); - val b = address("1:1100000000000000000000000000000000000000000000000000000000000000"); - val dd1 = buildAddrInShard_manual(a, {fixedPrefixLength: 8, closeTo: b}); - val dd2 = a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: b}); - assert(dd1.endCell().hash() == dd2.endCell().hash(), 400); - return dd1.endCell().beginParse() as address - == address("1:11000000000000000000000000000000000000000000000000000000000000FF"); -} - -@method_id(102) -fun test2() { - val a = address("0:1234aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - val b = address("0:FFFFbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); - val dd1 = buildAddrInShard_manual(a, {closeTo: b, fixedPrefixLength: 16}); - val dd2 = a.buildSameAddressInAnotherShard({fixedPrefixLength: 16, closeTo: b}); - assert((a as slice).remainingBitsCount() == 267, 267); - assert((b as slice).remainingBitsCount() == 267, 267); - assert(dd1.endCell().hash() == dd2.endCell().hash(), 400); - return address.fromValidBuilder(dd2) - == address("0:FFFFaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); -} - -@method_id(103) -fun test3() { - var t1_a = address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"); - var t1_b = address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); - var t2_a = address("EQCe4AYIBce1pAk2qJJPSs1OzyZRlKjkfq8zuC8D7erv6DUP"); - var t2_b = address("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"); - var t3_a = address("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"); - var t3_b = address("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"); - return ( - t1_a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: t1_b}).endCell().hash() & 0xFFFF, - t1_a.buildSameAddressInAnotherShard({fixedPrefixLength: 4, closeTo: t1_b}).endCell().hash() & 0xFFFF, - t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 2, closeTo: t2_b}).endCell().hash() & 0xFFFF, - t3_a.buildSameAddressInAnotherShard({fixedPrefixLength: 30, closeTo: t2_b}).endCell().hash() & 0xFFFF, - t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 1, closeTo: t3_b}).endCell().hash() & 0xFFFF, - t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 0, closeTo: t3_b}).endCell().hash() & 0xFFFF, - ) -} - -@method_id(104) -fun test4(shardDepth: int) { - var a = address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"); - var b = address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); - return a.buildSameAddressInAnotherShard({fixedPrefixLength: shardDepth, closeTo: b}).endCell().hash() & 0xFFFF; -} - -@method_id(105) -fun test5() { - var b = beginCell() - .storeUint(0b100, 3) // std addr no anycast - .storeInt(MASTERCHAIN, 8) - .storeUint(0xFFFF, 256); - val a = address("-1:000000000000000000000000000000000000000000000000000000000000FFFF"); - return address.fromValidBuilder(b) == a; -} - -@method_id(106) -fun test6() { - val a = address("0:00000000000000000000000000000000000000000000000000000000000000FF"); - val b = address("1:1100000000000000000000000000000000000000000000000000000000000000"); - return address.fromValidBuilder(a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: b})) == - address("1:11000000000000000000000000000000000000000000000000000000000000FF"); -} - fun main() { __expect_type(cc1, "address"); @@ -231,13 +147,6 @@ fun main() { /** @testcase | 0 | | -1 0 -1 0 -1 -@testcase | 101 | | -1 -@testcase | 102 | | -1 -@testcase | 103 | | 39876 24338 15241 50719 11252 15241 -@testcase | 104 | 2 | 24338 -@testcase | 104 | 9 | 39876 -@testcase | 105 | | -1 -@testcase | 106 | | -1 @fif_codegen """ diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index cce907beb6..22d743753b 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1307,6 +1307,7 @@ void define_builtins() { TypePtr CreateExternalLogMessageOptions = TypeDataUnknown::create(); TypePtr OutMessage = TypeDataUnknown::create(); TypePtr AddressShardingOptions = TypeDataUnknown::create(); + TypePtr AutoDeployAddress = TypeDataUnknown::create(); const GenericsDeclaration* declTBody = new GenericsDeclaration(std::vector{{"TBody", nullptr}}, 0); // builtin operators @@ -1582,6 +1583,12 @@ void define_builtins() { define_builtin_func("createExternalLogMessage", {CreateExternalLogMessageOptions}, OutMessage, declTBody, compile_time_only_function, FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + define_builtin_method("AutoDeployAddress.buildAddress", AutoDeployAddress, {AutoDeployAddress}, Builder, nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagCompileTimeGen); + define_builtin_method("AutoDeployAddress.addressMatches", AutoDeployAddress, {AutoDeployAddress, Address}, Bool, nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagCompileTimeGen); // functions not presented in stdlib at all // used in tolk-tester to check/expose internal compiler state @@ -1614,9 +1621,15 @@ void patch_builtins_after_stdlib_loaded() { lookup_function("debug.dumpStack")->mutate()->receiver_type = debug; StructPtr struct_ref_AddressShardingOptions = lookup_global_symbol("AddressShardingOptions")->try_as(); + StructPtr struct_ref_AutoDeployAddress = lookup_global_symbol("AutoDeployAddress")->try_as(); TypePtr AddressShardingOptions = TypeDataStruct::create(struct_ref_AddressShardingOptions); + TypePtr AutoDeployAddress = TypeDataStruct::create(struct_ref_AutoDeployAddress); lookup_function("address.buildSameAddressInAnotherShard")->mutate()->parameters[1].declared_type = AddressShardingOptions; + lookup_function("AutoDeployAddress.buildAddress")->mutate()->receiver_type = AutoDeployAddress; + lookup_function("AutoDeployAddress.buildAddress")->mutate()->parameters[0].declared_type = AutoDeployAddress; + lookup_function("AutoDeployAddress.addressMatches")->mutate()->receiver_type = AutoDeployAddress; + lookup_function("AutoDeployAddress.addressMatches")->mutate()->parameters[0].declared_type = AutoDeployAddress; StructPtr struct_ref_CellT = lookup_global_symbol("Cell")->try_as(); StructPtr struct_ref_PackOptions = lookup_global_symbol("PackOptions")->try_as(); diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 38a7c492ac..3f525287bb 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -678,6 +678,15 @@ static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob std::vector ir_shard_options = vars_per_arg[1]; return generate_address_buildInAnotherShard(code, loc, std::move(ir_self_address), std::move(ir_shard_options)); } + if (called_f->name == "AutoDeployAddress.buildAddress") { + std::vector ir_self = vars_per_arg[0]; + return generate_AutoDeployAddress_buildAddress(code, loc, std::move(ir_self)); + } + if (called_f->name == "AutoDeployAddress.addressMatches") { + std::vector ir_self = vars_per_arg[0]; + std::vector ir_address = vars_per_arg[1]; + return generate_AutoDeployAddress_addressMatches(code, loc, std::move(ir_self), std::move(ir_address)); + } tolk_assert(false); } diff --git a/tolk/send-message-api.cpp b/tolk/send-message-api.cpp index 6d0b535240..2b4d80be8e 100644 --- a/tolk/send-message-api.cpp +++ b/tolk/send-message-api.cpp @@ -43,6 +43,49 @@ static std::vector create_default_PackOptions(CodeBlob& code, SrcLoca return ir_options; } +// calculate `addrHash &= mask` where mask = `(1 << (256 - SHARD_DEPTH)) - 1` +static void append_bitwise_and_shard_mask(CodeBlob& code, SrcLocation loc, var_idx_t ir_addr_hash, var_idx_t ir_shard_depth) { + var_idx_t ir_one = code.create_int(loc, 1, "(one)"); + std::vector ir_mask = code.create_tmp_var(TypeDataInt::create(), loc, "(mask)"); + code.emplace_back(loc, Op::_Call, ir_mask, std::vector{code.create_int(loc, 256, ""), ir_shard_depth}, lookup_function("_-_")); + code.emplace_back(loc, Op::_Call, ir_mask, std::vector{ir_one, ir_mask[0]}, lookup_function("_<<_")); + code.emplace_back(loc, Op::_Call, ir_mask, std::vector{ir_mask[0], ir_one}, lookup_function("_-_")); + code.emplace_back(loc, Op::_Call, std::vector{ir_addr_hash}, std::vector{ir_addr_hash, ir_mask[0]}, lookup_function("_&_")); +} + +// struct AutoDeployAddress { workchain: int8; stateInit: ContractState | cell; toShard: AddressShardingOptions?; } +struct IR_AutoDeployAddress { + std::vector is_ContractState, // stateInit is ContractState + is_AddressSharding; // toShard is not null + var_idx_t workchain, // workchain + stateInitCode, stateInitData, stateInitCell, // stateInit + ir_shardDepth, ir_closeTo; // toShard + + IR_AutoDeployAddress(CodeBlob& code, SrcLocation loc, const std::vector& ir_vars) { + StructPtr s_AutoDeployAddress = lookup_global_symbol("AutoDeployAddress")->try_as(); + const TypeDataUnion* t_stateInit = s_AutoDeployAddress->find_field("stateInit")->declared_type->try_as(); + const TypeDataUnion* t_toShard = s_AutoDeployAddress->find_field("toShard")->declared_type->try_as(); + tolk_assert(ir_vars.size() == 1 + 3 + 3); + tolk_assert(t_stateInit && t_stateInit->get_width_on_stack() == (2+1) && t_stateInit->size() == 2); + tolk_assert(t_toShard && t_toShard->get_width_on_stack() == (2+1) && t_toShard->or_null); + + workchain = ir_vars[0]; + + std::vector ir_stateInitUnion(ir_vars.begin() + 1, ir_vars.begin() + 1 + 3); + is_ContractState = pre_compile_is_type(code, t_stateInit, t_stateInit->variants[0], ir_stateInitUnion, loc, "(is-ContractState)"); + std::vector ir_ContractState = transition_to_target_type(std::vector(ir_stateInitUnion), code, t_stateInit, t_stateInit->variants[0], loc); + stateInitCode = ir_ContractState[0]; + stateInitData = ir_ContractState[1]; + stateInitCell = transition_to_target_type(std::vector(ir_stateInitUnion), code, t_stateInit, t_stateInit->variants[1], loc)[0]; + + std::vector ir_toShardOrNull(ir_vars.begin() + 1 + 3, ir_vars.begin() + 1 + 3 + 3); + is_AddressSharding = pre_compile_is_type(code, t_toShard, t_toShard->or_null, ir_toShardOrNull, loc, "(is-AddressSharding)"); + std::vector ir_AddressSharding = transition_to_target_type(std::vector(ir_toShardOrNull), code, t_toShard, t_toShard->or_null, loc); + ir_shardDepth = ir_AddressSharding[0]; + ir_closeTo = ir_AddressSharding[1]; + } +}; + std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect) { StructPtr s_Options = lookup_global_symbol("CreateMessageOptions")->try_as(); StructPtr s_AutoDeployAddress = lookup_global_symbol("AutoDeployAddress")->try_as(); @@ -50,13 +93,9 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T const TypeDataBool* t_bounce = s_Options->find_field("bounce")->declared_type->try_as(); const TypeDataUnion* t_dest = s_Options->find_field("dest")->declared_type->try_as(); const TypeDataUnion* t_value = s_Options->find_field("value")->declared_type->try_as(); - const TypeDataUnion* t_stateInit = s_AutoDeployAddress->find_field("stateInit")->declared_type->try_as(); - const TypeDataUnion* t_toShard = s_AutoDeployAddress->find_field("toShard")->declared_type->try_as(); tolk_assert(t_bounce); tolk_assert(t_dest && t_dest->get_width_on_stack() == (1+3+3+1) && t_dest->size() == 4); tolk_assert(t_value && t_value->get_width_on_stack() == (2+1) && t_value->size() == 2); - tolk_assert(t_stateInit && t_stateInit->get_width_on_stack() == (2+1) && t_stateInit->size() == 2); - tolk_assert(t_toShard && t_toShard->get_width_on_stack() == (2+1) && t_toShard->or_null); int offset = 0; auto next_slice = [&rvect, &offset](int width) -> std::vector { @@ -79,17 +118,7 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T std::vector ir_dest_is_AutoDeploy = pre_compile_is_type(code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), ir_dest, loc, "(is-address)"); std::vector ir_dest_is_builder = pre_compile_is_type(code, t_dest, TypeDataBuilder::create(), ir_dest, loc, "(is-builder)"); std::vector ir_dest_AutoDeployAddress = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), loc); - // dest.workchain - std::vector ir_dest_workchain(ir_dest_AutoDeployAddress.begin(), ir_dest_AutoDeployAddress.begin() + 1); - // dest.stateInit (code+data or cell) - std::vector ir_dest_stateInitUnion(ir_dest_AutoDeployAddress.begin() + 1, ir_dest_AutoDeployAddress.begin() + 1 + 3); - std::vector ir_is_ContractState = pre_compile_is_type(code, t_stateInit, t_stateInit->variants[0], ir_dest_stateInitUnion, loc, "(is-ContractState)"); - std::vector ir_ContractState = transition_to_target_type(std::vector(ir_dest_stateInitUnion), code, t_stateInit, t_stateInit->variants[0], loc); - std::vector ir_StateInitCell = transition_to_target_type(std::vector(ir_dest_stateInitUnion), code, t_stateInit, t_stateInit->variants[1], loc); - // dest.toShard (shardPrefix+closeTo or null) - std::vector ir_dest_toShardOrNull(ir_dest_AutoDeployAddress.begin() + 1 + 3, ir_dest_AutoDeployAddress.begin() + 1 + 3 + 3); - std::vector ir_is_AddressSharding = pre_compile_is_type(code, t_toShard, t_toShard->or_null, ir_dest_toShardOrNull, loc, "(is-AddressSharding)"); - std::vector ir_AddressSharding = transition_to_target_type(std::vector(ir_dest_toShardOrNull), code, t_toShard, t_toShard->or_null, loc); + IR_AutoDeployAddress ir_dest_ad(code, loc, ir_dest_AutoDeployAddress); // currently, there is no way to pass PackOptions, defaults are used std::vector ir_options = create_default_PackOptions(code, loc); @@ -154,26 +183,27 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T // and, if toShard, take first D bits from dest.toShard.closeTo and mix with 256-D bits of hash code.push_set_cur(if_AutoDeploy.block0); ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); // addr_std$10 + 0 anycast - ctx.storeInt(ir_dest_workchain[0], 8); + ctx.storeInt(ir_dest_ad.workchain, 8); std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); - Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_is_ContractState); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_dest_ad.is_ContractState); { // input is `dest: { ... stateInit: { code, data } }` code.push_set_cur(if_ContractState.block0); - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_is_AddressSharding); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding); { // input is `dest: { ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } }; // then stateInitHash = (hash of StateInit = 0b1(depth)0110 (prefix + code + data)) code.push_set_cur(if_sharded.block0); - std::vector ir_args_depth_code_data = { ir_AddressSharding[0], ir_ContractState[0], ir_ContractState[1] }; - code.emplace_back(loc, Op::_Call, ir_hash, std::move(ir_args_depth_code_data), lookup_function("StateInit.calcHashPrefixCodeData")); + std::vector args = { ir_dest_ad.ir_shardDepth, ir_dest_ad.stateInitCode, ir_dest_ad.stateInitData }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashPrefixCodeData")); code.close_pop_cur(loc); } { // input is: `dest: { ... stateInit: { code, data } }` (toShard is null); // then hash = (hash of StateInit = 0b00110 (only code + data)) code.push_set_cur(if_sharded.block1); - code.emplace_back(loc, Op::_Call, ir_hash, ir_ContractState, lookup_function("StateInit.calcHashCodeData")); + std::vector args = { ir_dest_ad.stateInitCode, ir_dest_ad.stateInitData }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashCodeData")); code.close_pop_cur(loc); } code.close_pop_cur(loc); @@ -181,10 +211,11 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T { // input is `dest: { ... stateInit: cell }` code.push_set_cur(if_ContractState.block1); - code.emplace_back(loc, Op::_Call, ir_hash, ir_StateInitCell, lookup_function("cell.hash")); + std::vector args = { ir_dest_ad.stateInitCell }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("cell.hash")); code.close_pop_cur(loc); } - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_is_AddressSharding); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding); { // input is `dest: { ... toShard: { fixedPrefixLength, closeTo } }` // we already calculated stateInitHash (ir_hash): either cell.hash() or based on prefix+code+data; @@ -196,17 +227,14 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T // | hash (result) | 01010101...yyy | // remember, that closeTo is addr_std$10 + 0 + workchain + xxx...xxx, so skip 11 bits and read 8 code.push_set_cur(if_sharded.block0); + append_bitwise_and_shard_mask(code, loc, ir_hash[0], ir_dest_ad.ir_shardDepth); + std::vector ir_lowerD = code.create_tmp_var(TypeDataInt::create(), loc, "(lowerD)"); + code.emplace_back(loc, Op::_Call, ir_lowerD, std::vector{code.create_int(loc, 256, ""), ir_dest_ad.ir_shardDepth}, lookup_function("_-_")); std::vector ir_shardPrefix = code.create_tmp_var(TypeDataSlice::create(), loc, "(shardPrefix)"); - std::vector args_subslice = { ir_AddressSharding[1], code.create_int(loc, 3+8, ""), ir_AddressSharding[0] }; + std::vector args_subslice = { ir_dest_ad.ir_closeTo, code.create_int(loc, 3+8, ""), ir_dest_ad.ir_shardDepth }; code.emplace_back(loc, Op::_Call, ir_shardPrefix, std::move(args_subslice), lookup_function("slice.getMiddleBits")); ctx.storeSlice(ir_shardPrefix[0]); // first 8 bits of closeTo hash - std::vector ir_mask = code.create_tmp_var(TypeDataInt::create(), loc, "(mask)"); - code.emplace_back(loc, Op::_Call, ir_mask, std::vector{code.create_int(loc, 256, ""), ir_AddressSharding[0]}, lookup_function("_-_")); - code.emplace_back(loc, Op::_Call, ir_mask, std::vector{ir_one, ir_mask[0]}, lookup_function("_<<_")); - code.emplace_back(loc, Op::_Call, ir_mask, std::vector{ir_mask[0], ir_one}, lookup_function("_-_")); - code.emplace_back(loc, Op::_Call, ir_hash, std::vector{ir_hash[0], ir_mask[0]}, lookup_function("_&_")); - code.emplace_back(loc, Op::_Call, ir_mask, std::vector{code.create_int(loc, 256, ""), ir_AddressSharding[0]}, lookup_function("_-_")); - ctx.storeUint_var(ir_hash[0], ir_mask[0]); // 248 STU (stateInitHash & mask) + ctx.storeUint_var(ir_hash[0], ir_lowerD[0]); // 248 STU (stateInitHash & mask) code.close_pop_cur(loc); } { @@ -275,17 +303,17 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T } { code.push_set_cur(if_no_init.block0); - Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_is_ContractState); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_dest_ad.is_ContractState); { // input is `dest: { ... stateInit: { code, data } }` and need to compose TL/B StateInit; // it's either just code+data OR (if `toShard: { ... }` is set) fixedPrefixLength+code+data code.push_set_cur(if_ContractState.block0); - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_is_AddressSharding); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding); { // 1 (maybe true) + 0 (either left) + 1 (maybe true of StateInit) + fixedPrefixLength + 0110 + body ref or not code.push_set_cur(if_sharded.block0); ctx.storeUint(code.create_int(loc, 0b101, ""), 1 + 1 + 1); - ctx.storeUint(ir_AddressSharding[0], 5); // fixedPrefixLength (shard depth) + ctx.storeUint(ir_dest_ad.ir_shardDepth, 5); // fixedPrefixLength (shard depth) ctx.storeUint(code.create_int(loc, 0b01100 + body_store_as_ref, ""), 4 + 1); code.close_pop_cur(loc); // also, we used dest.toShard to fill CommonMsgInfoRelaxed.dest.address (with a mask for stateInitHash, see above) @@ -297,8 +325,8 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T ctx.storeUint(ir_rest_bits, 1 + 1 + 5 + 1); code.close_pop_cur(loc); } - ctx.storeRef(ir_ContractState[0]); // dest.stateInit.code - ctx.storeRef(ir_ContractState[1]); // dest.stateInit.data + ctx.storeRef(ir_dest_ad.stateInitCode); + ctx.storeRef(ir_dest_ad.stateInitData); code.close_pop_cur(loc); } { @@ -307,7 +335,7 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T code.push_set_cur(if_ContractState.block1); var_idx_t ir_rest_bits = code.create_int(loc, 0b110 + body_store_as_ref, "(rest-bits)"); ctx.storeUint(ir_rest_bits, 1 + 1 + 1); - ctx.storeRef(ir_StateInitCell[0]); + ctx.storeRef(ir_dest_ad.stateInitCell); code.close_pop_cur(loc); } code.close_pop_cur(loc); @@ -493,4 +521,155 @@ std::vector generate_address_buildInAnotherShard(CodeBlob& code, SrcL return ir_builder; } +std::vector generate_AutoDeployAddress_buildAddress(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy) { + IR_AutoDeployAddress ir_self(code, loc, ir_auto_deploy); + + std::vector ir_builder = code.create_tmp_var(TypeDataSlice::create(), loc, "(addr-b)"); + // important! unlike `createMessage()`, we calculate hash and shard prefix BEFORE creating a cell + // (for fewer stack manipulations) + + // calculate stateInitHash = (hash of StateInit cell would be, but without constructing a cell) + std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_self.is_ContractState); + { + // called `{ ... stateInit: { code, data } }` + code.push_set_cur(if_ContractState.block0); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + { + // called `{ ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } } + code.push_set_cur(if_sharded.block0); + std::vector args = { ir_self.ir_shardDepth, ir_self.stateInitCode, ir_self.stateInitData }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashPrefixCodeData")); + code.close_pop_cur(loc); + } + { + // called `{ ... stateInit: { code, data } }` (toShard is null) + code.push_set_cur(if_sharded.block1); + std::vector args = { ir_self.stateInitCode, ir_self.stateInitData }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashCodeData")); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + { + // called `{ ... stateInit: cell }` + code.push_set_cur(if_ContractState.block1); + std::vector args = { ir_self.stateInitCell }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("cell.hash")); + code.close_pop_cur(loc); + } + + // now, if toShard, perform bitwise calculations with hashes (order on a stack matters) + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + { + // called `{ ... toShard: { fixedPrefixLength, closeTo } }` + // we already calculated stateInitHash (ir_hash): either cell.hash() or based on prefix+code+data; + // keep hash = (last 256-D bits from stateInitHash) = `hash & mask` + code.push_set_cur(if_sharded.block0); + append_bitwise_and_shard_mask(code, loc, ir_hash[0], ir_self.ir_shardDepth); + std::vector ir_lowerD = code.create_tmp_var(TypeDataInt::create(), loc, "(lowerD)"); + code.emplace_back(loc, Op::_Call, ir_lowerD, std::vector{code.create_int(loc, 256, ""), ir_self.ir_shardDepth}, lookup_function("_-_")); + + // calculate shard_prefix = (first D bits from dest.toShard.closeTo) + std::vector ir_shardPrefix = code.create_tmp_var(TypeDataSlice::create(), loc, "(shardPrefix)"); + std::vector args_subslice = { ir_self.ir_closeTo, code.create_int(loc, 3+8, ""), ir_self.ir_shardDepth }; + code.emplace_back(loc, Op::_Call, ir_shardPrefix, std::move(args_subslice), lookup_function("slice.getMiddleBits")); + + // on a stack: stateInitHash & mask; shard prefix; create a cell and store all + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); + PackContext ctx(code, loc, ir_builder, create_default_PackOptions(code, loc)); + ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); // addr_std$10 + 0 anycast + ctx.storeInt(ir_self.workchain, 8); + ctx.storeSlice(ir_shardPrefix[0]); // first 8 bits of closeTo hash + ctx.storeUint_var(ir_hash[0], ir_lowerD[0]); // 248 STU (stateInitHash & mask) + code.close_pop_cur(loc); + } + { + // called `{ workchain, stateInit }` (toShard is null); + // on a stack: hash (already calculated); create a cell and store all + code.push_set_cur(if_sharded.block1); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); + PackContext ctx(code, loc, ir_builder, create_default_PackOptions(code, loc)); + ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); // addr_std$10 + 0 anycast + ctx.storeInt(ir_self.workchain, 8); + ctx.storeUint(ir_hash[0], 256); + code.close_pop_cur(loc); + } + + return ir_builder; +} + +std::vector generate_AutoDeployAddress_addressMatches(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy, std::vector&& ir_address) { + IR_AutoDeployAddress ir_self(code, loc, ir_auto_deploy); + + // at first, calculate stateInitHash = (hash of StateInit cell would be, but without constructing a cell) + std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_self.is_ContractState); + { + // called `{ ... stateInit: { code, data } }` + code.push_set_cur(if_ContractState.block0); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + { + // called `{ ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } } + code.push_set_cur(if_sharded.block0); + std::vector args = { ir_self.ir_shardDepth, ir_self.stateInitCode, ir_self.stateInitData }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashPrefixCodeData")); + code.close_pop_cur(loc); + } + { + // called `{ ... stateInit: { code, data } }` (toShard is null) + code.push_set_cur(if_sharded.block1); + std::vector args = { ir_self.stateInitCode, ir_self.stateInitData }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashCodeData")); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + { + // called `{ ... stateInit: cell }` + code.push_set_cur(if_ContractState.block1); + std::vector args = { ir_self.stateInitCell }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("cell.hash")); + code.close_pop_cur(loc); + } + + // now calculate `stateInitHash &= mask` where mask = `(1 << (256 - SHARD_DEPTH)) - 1` + Op& if_sharded1 = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + { + code.push_set_cur(if_sharded1.block0); + append_bitwise_and_shard_mask(code, loc, ir_hash[0], ir_self.ir_shardDepth); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_sharded1.block1); + code.close_pop_cur(loc); + } + + // now do `(wc, hash) = addr.getWorkchainAndHash()` + std::vector ir_addr_wc_hash = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create()}), loc, "(self-wc-hash)"); + code.emplace_back(loc, Op::_Call, ir_addr_wc_hash, ir_address, lookup_function("address.getWorkchainAndHash")); + + // now calculate `hash &= mask` (the same as we did earlier for stateInitHash) + Op& if_sharded2 = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + { + code.push_set_cur(if_sharded2.block0); + append_bitwise_and_shard_mask(code, loc, ir_addr_wc_hash[1], ir_self.ir_shardDepth); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_sharded2.block1); + code.close_pop_cur(loc); + } + + // finally, eval `(hash == stateInitHash) & (wc == workchain)` + std::vector ir_eq_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(eq-hash)"); + code.emplace_back(loc, Op::_Call, ir_eq_hash, std::vector{ir_addr_wc_hash[1], ir_hash[0]}, lookup_function("_==_")); + std::vector ir_eq_wc = code.create_tmp_var(TypeDataInt::create(), loc, "(eq-wc)"); + code.emplace_back(loc, Op::_Call, ir_eq_wc, std::vector{ir_addr_wc_hash[0], ir_self.workchain}, lookup_function("_==_")); + + std::vector ir_bool_result = code.create_tmp_var(TypeDataBool::create(), loc, "(is-addr-result)"); + code.emplace_back(loc, Op::_Call, ir_bool_result, std::vector{ir_eq_hash[0], ir_eq_wc[0]}, lookup_function("_&_")); + return ir_bool_result; +} + } // namespace tolk diff --git a/tolk/send-message-api.h b/tolk/send-message-api.h index 1af7389d49..f86be5446b 100644 --- a/tolk/send-message-api.h +++ b/tolk/send-message-api.h @@ -25,4 +25,7 @@ std::vector generate_createExternalLogMessage(CodeBlob& code, SrcLoca std::vector generate_address_buildInAnotherShard(CodeBlob& code, SrcLocation loc, std::vector&& ir_self_address, std::vector&& ir_shard_options); +std::vector generate_AutoDeployAddress_buildAddress(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy); +std::vector generate_AutoDeployAddress_addressMatches(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy, std::vector&& ir_address); + } // namespace tolk From 6f5fb744606ef09a7fcfa5e4b192f62040bb09ef Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 3 Jul 2025 18:46:40 +0300 Subject: [PATCH 339/524] [Tolk] Make a semicolon optional for top-level declarations After declaring type aliases, constants, etc. semicolon is now optional. Struct fields can also be separated with a newline only. --- crypto/smartcont/tolk-stdlib/common.tolk | 461 ++++++++---------- .../smartcont/tolk-stdlib/gas-payments.tolk | 24 +- crypto/smartcont/tolk-stdlib/lisp-lists.tolk | 10 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 138 +++--- .../smartcont/tolk-stdlib/tvm-lowlevel.tolk | 8 +- tolk-tester/tests/assignment-tests.tolk | 8 +- tolk-tester/tests/constants-tests.tolk | 13 +- tolk-tester/tests/indexed-access.tolk | 4 +- .../tests/invalid-declaration/err-1043.tolk | 6 + .../tests/invalid-declaration/err-1794.tolk | 2 +- .../tests/invalid-syntax/err-3618.tolk | 2 +- .../tests/invalid-syntax/err-3802.tolk | 9 - tolk-tester/tests/methods-tests.tolk | 8 +- tolk-tester/tests/mutate-methods.tolk | 4 +- tolk-tester/tests/no-spaces.tolk | 6 +- tolk-tester/tests/pack-unpack-3.tolk | 13 +- tolk-tester/tests/pack-unpack-5.tolk | 8 +- tolk-tester/tests/smart-cast-tests.tolk | 2 +- tolk-tester/tests/strings-tests.tolk | 10 +- tolk-tester/tests/struct-tests.tolk | 26 +- tolk-tester/tests/type-aliases-tests.tolk | 18 +- tolk-tester/tests/union-types-tests.tolk | 22 +- tolk/ast-from-tokens.cpp | 38 +- 23 files changed, 402 insertions(+), 438 deletions(-) create mode 100644 tolk-tester/tests/invalid-declaration/err-1043.tolk delete mode 100644 tolk-tester/tests/invalid-syntax/err-3802.tolk diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index da52bfb98f..a72718246a 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -7,7 +7,7 @@ tolk 0.99 /// Currently, working with dictionaries is still low-level, with raw cells. /// But just for clarity, we use "dict" instead of a "cell?" where a cell-dictionary is assumed. /// Every dictionary object can be null. TVM NULL is essentially "empty dictionary". -type dict = cell?; +type dict = cell? /** Tuple manipulation primitives. @@ -18,47 +18,47 @@ type dict = cell?; /// Creates a tuple with zero elements. @pure fun createEmptyTuple(): tuple - asm "NIL"; + asm "NIL" /// Appends a value to tuple, resulting in `Tuple t' = (x1, ..., xn, value)`. /// If its size exceeds 255, throws a type check exception. @pure fun tuple.push(mutate self, value: T): void - asm "TPUSH"; + asm "TPUSH" /// Returns the first element of a non-empty tuple. /// `t.0` is actually the same as `t.first()` @pure fun tuple.first(self): T - asm "FIRST"; + asm "FIRST" /// Returns the [`index`]-th element of a tuple. /// `t.i` is actually the same as `t.get(i)` @pure fun tuple.get(self, index: int): T - builtin; + builtin /// Sets the [`index`]-th element of a tuple to a specified value /// (element with this index must already exist, a new element isn't created). /// `t.i = value` is actually the same as `t.set(value, i)` @pure fun tuple.set(mutate self, value: T, index: int): void - builtin; + builtin /// Returns the size of a tuple (elements count in it). @pure fun tuple.size(self): int - asm "TLEN"; + asm "TLEN" /// Returns the last element of a non-empty tuple. @pure fun tuple.last(self): T - asm "LAST"; + asm "LAST" /// Pops and returns the last element of a non-empty tuple. @pure fun tuple.pop(mutate self): T - asm "TPOP"; + asm "TPOP" /** @@ -70,64 +70,64 @@ fun tuple.pop(mutate self): T /// Note, that `ton()` requires a constant string; `ton(some_var)` is an error @pure fun ton(floatString: slice): coins - builtin; + builtin /// Computes the minimum of two integers. @pure fun min(x: int, y: int): int - asm "MIN"; + asm "MIN" /// Computes the maximum of two integers. @pure fun max(x: int, y: int): int - asm "MAX"; + asm "MAX" /// Sorts two integers. /// Example: `minMax(x, y)` with (x=20, y=10) and with (x=10, y=20) returns (10, 20) @pure fun minMax(x: int, y: int): (int, int) - asm "MINMAX"; + asm "MINMAX" /// Computes the absolute value of an integer. @pure fun abs(x: int): int - asm "ABS"; + asm "ABS" /// Returns the sign of an integer: `-1` if x < 0, `0` if x == 0, `1` if x > 0. @pure fun sign(x: int): int - asm "SGN"; + asm "SGN" /// Computes the quotient and remainder of [x] / [y]. Example: divMod(112,3) = (37,1) @pure fun divMod(x: int, y: int): (int, int) - asm "DIVMOD"; + asm "DIVMOD" /// Computes the remainder and quotient of [x] / [y]. Example: modDiv(112,3) = (1,37) @pure fun modDiv(x: int, y: int): (int, int) - asm(-> 1 0) "DIVMOD"; + asm(-> 1 0) "DIVMOD" /// Computes multiple-then-divide: floor([x] * [y] / [z]). /// The intermediate result is stored in a 513-bit integer to prevent precision loss. @pure fun mulDivFloor(x: int, y: int, z: int): int - builtin; + builtin /// Similar to `mulDivFloor`, but rounds the result: round([x] * [y] / [z]). @pure fun mulDivRound(x: int, y: int, z: int): int - builtin; + builtin /// Similar to `mulDivFloor`, but ceils the result: ceil([x] * [y] / [z]). @pure fun mulDivCeil(x: int, y: int, z: int): int - builtin; + builtin /// Computes the quotient and remainder of ([x] * [y] / [z]). Example: mulDivMod(112,3,10) = (33,6) @pure fun mulDivMod(x: int, y: int, z: int): (int, int) - builtin; + builtin /** @@ -136,83 +136,83 @@ fun mulDivMod(x: int, y: int, z: int): (int, int) /// `contract` is a built-in struct, it has only static methods. /// Example: `contract.getCode()` and other methods. -struct contract {} +struct contract /// Returns the internal address of the current smart contract. /// If necessary, it can be parsed further using [address.getWorkchain] and others. @pure fun contract.getAddress(): address - asm "MYADDR"; + asm "MYADDR" /// Returns the balance (in nanotoncoins) of the smart contract at the start of Computation Phase. /// Note that RAW primitives such as [sendMessage] do not update this field. @pure fun contract.getOriginalBalance(): coins - asm "BALANCE" "FIRST"; + asm "BALANCE" "FIRST" /// Same as [contract.getOriginalBalance], but returns a tuple: /// `int` — balance in nanotoncoins; /// `dict` — a dictionary with 32-bit keys representing the balance of "extra currencies". @pure fun contract.getOriginalBalanceWithExtraCurrencies(): [coins, dict] - asm "BALANCE"; + asm "BALANCE" /// Returns the persistent contract storage cell. It can be parsed or modified with slice and builder primitives later. @pure fun contract.getData(): cell - asm "c4 PUSH"; + asm "c4 PUSH" /// Sets `cell` [c] as persistent contract data. You can update persistent contract storage with this primitive. fun contract.setData(c: cell): void - asm "c4 POP"; + asm "c4 POP" /// Retrieves code of smart-contract from c7 @pure fun contract.getCode(): cell - asm "MYCODE"; + asm "MYCODE" /// Creates an output action that would change this smart contract code to that given by cell [newCode]. /// Notice that this change will take effect only after the successful termination of the current run of the smart contract. fun contract.setCodePostponed(newCode: cell): void - asm "SETCODE"; + asm "SETCODE" /** Global getters of current blockchain (environment) state. */ -const MASTERCHAIN = -1; -const BASECHAIN = 0; +const MASTERCHAIN = -1 +const BASECHAIN = 0 /// `blockchain` is a built-in struct, it has only static methods. /// Example: `blockchain.configParam(16)` and other methods. -struct blockchain {} +struct blockchain /// Returns current Unix timestamp (in seconds). @pure fun blockchain.now(): int - asm "NOW"; + asm "NOW" /// Returns the logical time of the current transaction. @pure fun blockchain.logicalTime(): int - asm "LTIME"; + asm "LTIME" /// Returns the starting logical time of the current block. @pure fun blockchain.currentBlockLogicalTime(): int - asm "BLOCKLT"; + asm "BLOCKLT" /// Returns the value of the global configuration parameter with integer index `i` as a `cell` or `null` value. @pure fun blockchain.configParam(x: int): cell? - asm "CONFIGOPTPARAM"; + asm "CONFIGOPTPARAM" /// Commits the current state of registers `c4` (“persistent data”) and `c5` (“actions”) /// so that the current execution is considered “successful” with the saved values even if an exception /// in Computation Phase is thrown later. fun commitContractDataAndActions(): void - asm "COMMIT"; + asm "COMMIT" /** @@ -257,7 +257,7 @@ struct UnpackOptions { /// (beginCell() + serialize fields + endCell()). @pure fun T.toCell(self, options: PackOptions = {}): Cell - builtin; + builtin /// Parse anything from a cell (most likely, you'll call it for structures). /// Example: @@ -268,7 +268,7 @@ fun T.toCell(self, options: PackOptions = {}): Cell /// (packedCell.beginParse() + read from slice). @pure fun T.fromCell(packedCell: cell, options: UnpackOptions = {}): T - builtin; + builtin /// Parse anything from a slice (most likely, you'll call it for structures). /// Example: @@ -281,7 +281,7 @@ fun T.fromCell(packedCell: cell, options: UnpackOptions = {}): T /// If you need to mutate it, like `cs.loadInt()`, consider calling `cs.loadAny()`. @pure fun T.fromSlice(rawSlice: slice, options: UnpackOptions = {}): T - builtin; + builtin /// Parse anything from a slice, shifting its internal pointer. /// Similar to `slice.loadUint()` and others, but allows loading structures. @@ -294,7 +294,7 @@ fun T.fromSlice(rawSlice: slice, options: UnpackOptions = {}): T /// to read data from the middle. @pure fun slice.loadAny(mutate self, options: UnpackOptions = {}): T - builtin; + builtin /// Skip anything in a slice, shifting its internal pointer. /// Similar to `slice.skipBits()` and others, but allows skipping structures. @@ -305,7 +305,7 @@ fun slice.loadAny(mutate self, options: UnpackOptions = {}): T /// ``` @pure fun slice.skipAny(mutate self, options: UnpackOptions = {}): self - builtin; + builtin /// Store anything to a builder. /// Similar to `builder.storeUint()` and others, but allows storing structures. @@ -315,19 +315,19 @@ fun slice.skipAny(mutate self, options: UnpackOptions = {}): self /// ``` @pure fun builder.storeAny(mutate self, v: T, options: PackOptions = {}): self - builtin; + builtin /// Returns serialization prefix of a struct. Works at compile-time. /// Example: for `struct (0xF0) AssetRegular { ... }` will return `240`. @pure fun T.getDeclaredPackPrefix(): int - builtin; + builtin /// Returns serialization prefix length of a struct. Works at compile-time. /// Example: for `struct (0xF0) AssetRegular { ... }` will return `16`. @pure fun T.getDeclaredPackPrefixLen(): int - builtin; + builtin /// Forces an object created by `lazy` to load fully. Returns the remaining slice (having read all fields). /// Since `options.assertEndAfterReading` is ignored by `lazy` (fields are loaded on demand), @@ -347,7 +347,7 @@ fun T.getDeclaredPackPrefixLen(): int /// because the purpose of `lazy` is to auto-detect and load only necessary fields, not up to the end. @pure fun T.forceLoadLazyObject(self): slice - builtin; + builtin /// Cell represents a typed cell reference (as opposed to untyped `cell`). /// Example: @@ -365,7 +365,7 @@ fun T.forceLoadLazyObject(self): slice /// Note, that `st = MyStorage.fromSlice(s)` does NOT deep-load any refs; `st.extra` is `Cell`, not `T`; /// you should manually call `st.extra.load()` to get T (ExtraData in this example). struct Cell { - tvmCell: cell; + tvmCell: cell } /// Parse data from already loaded cell reference. @@ -379,17 +379,17 @@ struct Cell { /// ``` @pure fun Cell.load(self, options: UnpackOptions = {}): T - builtin; + builtin /// Converts a typed cell into a slice. @pure fun Cell.beginParse(self): slice - asm "CTOS"; + asm "CTOS" /// Returns hash of a typed cell, same as [cell.hash]. @pure fun Cell.hash(self): uint256 - asm "HASHCU"; + asm "HASHCU" /// RemainingBitsAndRefs is a special built-in type to get "all the rest" slice tail on reading. /// Example: @@ -400,19 +400,19 @@ fun Cell.hash(self): uint256 /// } /// ``` /// When you deserialize JettonMessage, forwardPayload contains "everything left after reading fields above". -type RemainingBitsAndRefs = slice; +type RemainingBitsAndRefs = slice /// Creates a cell with zero bits and references. /// Equivalent to `beginCell().endCell()` but cheaper. @pure fun createEmptyCell(): cell - asm " PUSHREF"; + asm " PUSHREF" /// Creates a slice with zero remaining bits and references. /// Equivalent to `beginCell().endCell().beginParse()` but cheaper. @pure fun createEmptySlice(): slice - asm "x{} PUSHSLICE"; + asm "x{} PUSHSLICE" /** @@ -424,14 +424,14 @@ fun createEmptySlice(): slice /// Note: stringCrc32(slice_var) does not work! It accepts a constant string and works at compile-time. @pure fun stringCrc32(constString: slice): int - builtin; + builtin /// Compile-time function that calculates crc16 (XMODEM) of a constant string. /// Example: `const op = stringCrc16("some_str")` = 53407 = 0xD09F /// Note: stringCrc16(slice_var) does not work! It accepts a constant string and works at compile-time. @pure fun stringCrc16(constString: slice): int - builtin; + builtin /// Compile-time function that calculates sha256 of a constant string and returns 256-bit integer. /// Example: `const hash = stringSha256("some_crypto_key")` @@ -439,40 +439,40 @@ fun stringCrc16(constString: slice): int /// Use `sliceBitsHash` or `sliceHash` (declared below) to hash a slice without/with its refs at runtime. @pure fun stringSha256(constString: slice): int - builtin; + builtin /// Compile-time function that calculates sha256 of a constant string and takes the first 32 bits. /// Example: `const minihash = stringSha256_32("some_crypto_key")` /// Note: stringSha256_32(slice_var) does not work! It accepts a constant string and works at compile-time. @pure fun stringSha256_32(constString: slice): int - builtin; + builtin /// Compile-time function that takes N-chars ascii string and interprets it as a number in base 256. /// Example: `const value = stringToBase256("AB")` = 16706 (65*256 + 66) /// Note: stringToBase256(slice_var) does not work! It accepts a constant string and works at compile-time. @pure fun stringToBase256(constString: slice): int - builtin; + builtin /// Computes the representation hash of a `cell` and returns it as a 256-bit unsigned integer `x`. /// Useful for signing and checking signatures of arbitrary entities represented by a tree of cells. @pure fun cell.hash(self): uint256 - asm "HASHCU"; + asm "HASHCU" /// Computes the hash of a `slice` and returns it as a 256-bit unsigned integer `x`. /// The result is the same as if an ordinary cell containing only data and references from `s` had been created /// and its hash computed by [cell.hash]. @pure fun slice.hash(self): uint256 - asm "HASHSU"; + asm "HASHSU" /// Computes sha256 of the data bits of a `slice`. If the bit length of `s` is not divisible by eight, /// throws a cell underflow exception. The hash value is returned as a 256-bit unsigned integer `x`. @pure fun slice.bitsHash(self): uint256 - asm "SHA256U"; + asm "SHA256U" /// Checks the Ed25519-`signature` of a `hash` (a 256-bit unsigned integer, usually computed as the hash of some data) /// using [publicKey] (also represented by a 256-bit unsigned integer). @@ -483,7 +483,7 @@ fun slice.bitsHash(self): uint256 /// the second hashing occurring inside `CHKSIGNS`. @pure fun isSignatureValid(hash: int, signature: slice, publicKey: int): bool - asm "CHKSIGNU"; + asm "CHKSIGNU" /// Checks whether [signature] is a valid Ed25519-signature of the data portion of `slice data` using `publicKey`, /// similarly to [isSignatureValid]. @@ -492,40 +492,40 @@ fun isSignatureValid(hash: int, signature: slice, publicKey: int): bool /// with sha256 used to reduce [data] to the 256-bit number that is actually signed. @pure fun isSliceSignatureValid(data: slice, signature: slice, publicKey: int): bool - asm "CHKSIGNS"; + asm "CHKSIGNS" /// `random` is a built-in struct, it has only static methods. /// Example: `random.uint256()` and other methods. -struct random {} +struct random /// Generates a new pseudo-random unsigned 256-bit integer x. /// Ensure you've called [random.initialize] in advance to make it unpredictable! fun random.uint256(): uint256 - asm "RANDU256"; + asm "RANDU256" /// Generates a new pseudo-random integer z in the range 0..limit−1 (or limit..−1, if upto < 0). /// More precisely, an unsigned random value x is generated as in random; then z := x * limit / 2^256 is computed. /// Ensure you've called [random.initialize] in advance to make it unpredictable! fun random.range(limit: int): int - asm "RAND"; + asm "RAND" /// Returns the current random seed used to generate pseudo-random numbers. @pure fun random.getSeed(): uint256 - asm "RANDSEED"; + asm "RANDSEED" /// Sets the random seed to the provided value. fun random.setSeed(seed: uint256): void - asm "SETRAND"; + asm "SETRAND" /// Initializes (mixes) random seed with the provided value. fun random.initializeBy(mixSeedWith: uint256): void - asm "ADDRAND"; + asm "ADDRAND" /// Initializes random seed with current time to make random generation unpredictable. /// Typically, you call this function once before calling [random.uint256] / [random.range]. fun random.initialize(): void - asm "LTIME" "ADDRAND"; + asm "LTIME" "ADDRAND" /** @@ -544,22 +544,22 @@ fun random.initialize(): void /// a zero flag is returned to indicate failure. If [c] is `null`, returns `x = y = z = 0`. @pure fun cell.calculateSize(self, maxCells: int): (int, int, int, bool) - asm "CDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT"; + asm "CDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT" /// Similar to [cell.calculateSize], but accepting a `slice` [s] instead of a `cell`. /// The returned value of `x` does not take into account the cell that contains the `slice` [s] itself; /// however, the data bits and the cell references of [s] are accounted for in `y` and `z`. @pure fun slice.calculateSize(self, maxCells: int): (int, int, int, bool) - asm "SDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT"; + asm "SDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT" /// A non-quiet version of [cell.calculateSize] that throws a cell overflow exception (`8`) on failure. fun cell.calculateSizeStrict(self, maxCells: int): (int, int, int) - asm "CDATASIZE"; + asm "CDATASIZE" /// A non-quiet version of [cell.calculateSize] that throws a cell overflow exception (`8`) on failure. fun slice.calculateSizeStrict(self, maxCells: int): (int, int, int) - asm "SDATASIZE"; + asm "SDATASIZE" /// Returns the depth of a `cell`. /// If [c] has no references, then return `0`; @@ -567,21 +567,21 @@ fun slice.calculateSizeStrict(self, maxCells: int): (int, int, int) /// If [c] is a `null` instead of a cell, returns zero. @pure fun cell?.depth(self): int - asm "CDEPTH"; + asm "CDEPTH" /// Returns the depth of a `slice`. /// If [s] has no references, then returns `0`; /// otherwise the returned value is one plus the maximum of depths of cells referred to from [s]. @pure fun slice.depth(self): int - asm "SDEPTH"; + asm "SDEPTH" /// Returns the depth of a `builder`. /// If no cell references are stored in [b], then returns 0; /// otherwise the returned value is one plus the maximum of depths of cells referred to from [b]. @pure fun builder.depth(self): int - asm "BDEPTH"; + asm "BDEPTH" /// Returns the number of stack slots anyVariable occupies (works at compile-time). /// Example: sizeof(nullableInt) = 1, because `int?` is 1 TVM slot holding either NULL or a value. @@ -589,7 +589,7 @@ fun builder.depth(self): int /// Useful for debugging or when preparing stack contents for RUNVM. @pure fun sizeof(anyVariable: T): int - builtin; + builtin /** @@ -599,19 +599,19 @@ fun sizeof(anyVariable: T): int /// `debug` is a built-in struct, it has only static methods. /// Example: `debug.print(v)` and other methods. -struct debug {} +struct debug /// Dump a variable [x] to the debug log. fun debug.print(x: T): void - builtin; + builtin /// Dump a string [x] to the debug log. fun debug.printString(x: T): void - builtin; + builtin /// Dumps the stack (at most the top 255 values) and shows the total stack depth. fun debug.dumpStack(): void - builtin; + builtin /** @@ -625,126 +625,126 @@ fun debug.dumpStack(): void /// Note: stringHexToSlice(slice_var) does not work! It accepts a constant string and works at compile-time. @pure fun stringHexToSlice(constStringBytesHex: slice): slice - builtin; + builtin /// Converts a `cell` into a `slice`. Notice that [c] must be either an ordinary cell, /// or an exotic cell (see [TVM.pdf](https://ton-blockchain.github.io/docs/tvm.pdf), 3.1.2) /// which is automatically loaded to yield an ordinary cell `c'`, converted into a `slice` afterwards. @pure fun cell.beginParse(self): slice - asm "CTOS"; + asm "CTOS" /// Checks if slice is empty. If not, throws an exception with code 9. fun slice.assertEnd(self): void - asm "ENDS"; + asm "ENDS" /// Loads the next reference from the slice. @pure fun slice.loadRef(mutate self): cell - asm( -> 1 0) "LDREF"; + asm( -> 1 0) "LDREF" /// Preloads the next reference from the slice. @pure fun slice.preloadRef(self): cell - asm "PLDREF"; + asm "PLDREF" /// Loads a signed [len]-bit integer from a slice. @pure fun slice.loadInt(mutate self, len: int): int - builtin; + builtin /// Loads an unsigned [len]-bit integer from a slice. @pure fun slice.loadUint(mutate self, len: int): int - builtin; + builtin /// Loads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate slice `s''`. @pure fun slice.loadBits(mutate self, len: int): slice - builtin; + builtin /// Preloads a signed [len]-bit integer from a slice. @pure fun slice.preloadInt(self, len: int): int - builtin; + builtin /// Preloads an unsigned [len]-bit integer from a slice. @pure fun slice.preloadUint(self, len: int): int - builtin; + builtin /// Preloads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate slice. @pure fun slice.preloadBits(self, len: int): slice - builtin; + builtin /// Loads serialized amount of Toncoins (any unsigned integer up to `2^120 - 1`). @pure fun slice.loadCoins(mutate self): coins - asm( -> 1 0) "LDGRAMS"; + asm( -> 1 0) "LDGRAMS" /// Loads bool (-1 or 0) from a slice @pure fun slice.loadBool(mutate self): bool - asm( -> 1 0) "1 LDI"; + asm( -> 1 0) "1 LDI" /// Shifts a slice pointer to [len] bits forward, mutating the slice. @pure fun slice.skipBits(mutate self, len: int): self - builtin; + builtin /// Returns the first `0 ≤ len ≤ 1023` bits of a slice. @pure fun slice.getFirstBits(self, len: int): slice - asm "SDCUTFIRST"; + asm "SDCUTFIRST" /// Returns all but the last `0 ≤ len ≤ 1023` bits of a slice. @pure fun slice.removeLastBits(mutate self, len: int): self - asm "SDSKIPLAST"; + asm "SDSKIPLAST" /// Returns the last `0 ≤ len ≤ 1023` bits of a slice. @pure fun slice.getLastBits(self, len: int): slice - asm "SDCUTLAST"; + asm "SDCUTLAST" /// Returns `0 ≤ len ≤ 1023` bits of a slice starting from `0 ≤ offset ≤ 1023`. /// (in other words, extracts a bit substring `[offset, len)` out of the slice) @pure fun slice.getMiddleBits(self, offset: int, len: int): slice - asm "SDSUBSTR"; + asm "SDSUBSTR" /// Loads a dictionary (TL HashMapE structure, represented as TVM cell) from a slice. /// Returns `null` if `nothing` constructor is used. @pure fun slice.loadDict(mutate self): dict - asm( -> 1 0) "LDDICT"; + asm( -> 1 0) "LDDICT" /// Preloads a dictionary (cell) from a slice. @pure fun slice.preloadDict(self): dict - asm "PLDDICT"; + asm "PLDDICT" /// Loads a dictionary as [slice.loadDict], but returns only the remainder of the slice. @pure fun slice.skipDict(mutate self): self - asm "SKIPDICT"; + asm "SKIPDICT" /// Loads (Maybe ^Cell) from a slice. /// In other words, loads 1 bit: if it's true, loads the first ref, otherwise returns `null`. @pure fun slice.loadMaybeRef(mutate self): cell? - asm( -> 1 0) "LDOPTREF"; + asm( -> 1 0) "LDOPTREF" /// Preloads (Maybe ^Cell) from a slice. @pure fun slice.preloadMaybeRef(self): cell? - asm "PLDOPTREF"; + asm "PLDOPTREF" /// Loads (Maybe ^Cell), but returns only the remainder of the slice. @pure fun slice.skipMaybeRef(mutate self): self - asm "SKIPOPTREF"; + asm "SKIPOPTREF" /** Builder primitives: constructing cells. @@ -756,70 +756,70 @@ fun slice.skipMaybeRef(mutate self): self /// Creates a new empty builder. @pure fun beginCell(): builder - asm "NEWC"; + asm "NEWC" /// Converts a builder into an ordinary `cell`. @pure fun builder.endCell(self): cell - asm "ENDC"; + asm "ENDC" /// Stores a reference to a cell into a builder. @pure fun builder.storeRef(mutate self, c: cell): self - asm(c self) "STREF"; + asm(c self) "STREF" /// Stores a signed [len]-bit integer into a builder (`0 ≤ len ≤ 257`). @pure fun builder.storeInt(mutate self, x: int, len: int): self - builtin; + builtin /// Stores an unsigned [len]-bit integer into a builder (`0 ≤ len ≤ 256`). @pure fun builder.storeUint(mutate self, x: int, len: int): self - builtin; + builtin /// Stores a slice into a builder. @pure fun builder.storeSlice(mutate self, s: slice): self - asm(s self) "STSLICE"; + asm(s self) "STSLICE" /// Stores an address into a builder. @pure fun builder.storeAddress(mutate self, addr: address): self - asm(addr self) "STSLICE"; + asm(addr self) "STSLICE" /// Stores amount of Toncoins into a builder. @pure fun builder.storeCoins(mutate self, x: coins): self - builtin; + builtin /// Stores bool (-1 or 0) into a builder. /// Attention: true value is `-1`, not 1! If you pass `1` here, TVM will throw an exception. @pure fun builder.storeBool(mutate self, x: bool): self - builtin; + builtin /// Stores dictionary (represented by TVM `cell` or `null`) into a builder. /// In other words, stores a `1`-bit and a reference to [c] if [c] is not `null` and `0`-bit otherwise. @pure fun builder.storeDict(mutate self, c: dict): self - asm(c self) "STDICT"; + asm(c self) "STDICT" /// Stores (Maybe ^Cell) into a builder. /// In other words, if cell is `null`, store '0' bit; otherwise, store '1' and a ref to [c]. @pure fun builder.storeMaybeRef(mutate self, c: cell?): self - asm(c self) "STOPTREF"; + asm(c self) "STOPTREF" /// Concatenates two builders. @pure fun builder.storeBuilder(mutate self, from: builder): self - asm(from self) "STB"; + asm(from self) "STB" /// Stores a slice representing TL addr_none$00 (two `0` bits). @pure fun builder.storeAddressNone(mutate self): self - asm "b{00} STSLICECONST"; + asm "b{00} STSLICECONST" /** @@ -829,84 +829,51 @@ fun builder.storeAddressNone(mutate self): self /// Returns the number of references in a slice. @pure fun slice.remainingRefsCount(self): int - asm "SREFS"; + asm "SREFS" /// Returns the number of data bits in a slice. @pure fun slice.remainingBitsCount(self): int - asm "SBITS"; + asm "SBITS" /// Returns both the number of data bits and the number of references in a slice. @pure fun slice.remainingBitsAndRefsCount(self): (int, int) - asm "SBITREFS"; + asm "SBITREFS" /// Checks whether a slice is empty (i.e., contains no bits of data and no cell references). @pure fun slice.isEmpty(self): bool - asm "SEMPTY"; + asm "SEMPTY" /// Checks whether a slice has no bits of data. @pure fun slice.isEndOfBits(self): bool - asm "SDEMPTY"; + asm "SDEMPTY" /// Checks whether a slice has no references. @pure fun slice.isEndOfRefs(self): bool - asm "SREMPTY"; + asm "SREMPTY" /// Checks whether data parts of two slices coinside. @pure fun slice.bitsEqual(self, b: slice): bool - asm "SDEQ"; + asm "SDEQ" /// Returns the number of cell references already stored in a builder. @pure fun builder.refsCount(self): int - asm "BREFS"; + asm "BREFS" /// Returns the number of data bits already stored in a builder. @pure fun builder.bitsCount(self): int - asm "BBITS"; + asm "BBITS" /** Address manipulation primitives. - The address manipulation primitives listed below serialize and deserialize values according to the following TL-B scheme: - ```TL-B - addr_none$00 = MsgAddressExt; - addr_extern$01 len:(## 8) external_address:(bits len) - = MsgAddressExt; - anycast_info$_ depth:(#<= 30) { depth >= 1 } - rewrite_pfx:(bits depth) = Anycast; - addr_std$10 anycast:(Maybe Anycast) - workchain_id:int8 address:bits256 = MsgAddressInt; - addr_var$11 anycast:(Maybe Anycast) addr_len:(## 9) - workchain_id:int32 address:(bits addr_len) = MsgAddressInt; - _ _:MsgAddressInt = MsgAddress; - _ _:MsgAddressExt = MsgAddress; - - int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool - src:MsgAddress dest:MsgAddressInt - value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams - created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; - ext_out_msg_info$11 src:MsgAddress dest:MsgAddressExt - created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; - ``` - A deserialized `MsgAddress` is represented by a tuple `t` as follows: - - - `addr_none` is represented by `t = (0)`, - i.e., a tuple containing exactly one integer equal to zero. - - `addr_extern` is represented by `t = (1, s)`, - where slice `s` contains the field `external_address`. In other words, ` - t` is a pair (a tuple consisting of two entries), containing an integer equal to one and slice `s`. - - `addr_std` is represented by `t = (2, u, x, s)`, - where `u` is either a `null` (if `anycast` is absent) or a slice `s'` containing `rewrite_pfx` (if anycast is present). - Next, integer `x` is the `workchain_id`, and slice `s` contains the address. - - `addr_var` is represented by `t = (3, u, x, s)`, - where `u`, `x`, and `s` have the same meaning as for `addr_std`. */ /// Compile-time function that parses a valid contract address. @@ -915,19 +882,19 @@ fun builder.bitsCount(self): int /// Returns `address`, which can be stored in a builder, compared with `==`, etc. @pure fun address(stdAddress: slice): address - builtin; + builtin /// Creates a slice representing "none address" (TL addr_none$00 — two zero bits). @pure fun createAddressNone(): address - asm "b{00} PUSHSLICE"; + asm "b{00} PUSHSLICE" /// Returns if it's an empty address. /// Don't confuse it with null! Empty address is a slice with two `0` bits. /// In TL/B, it's addr_none$00. @pure fun address.isNone(self): bool - asm "b{00} SDBEGINSQ" "NIP"; + asm "b{00} SDBEGINSQ" "NIP" /// Returns if it's a standard (internal) address. Such addresses contain workchain (8 bits) and hash (256 bits). /// All contract addresses are internal, so it's the most practical use case. @@ -935,25 +902,25 @@ fun address.isNone(self): bool /// For internal addresses, you can call [address.getWorkchain] and [address.getWorkchainAndHash]. @pure fun address.isInternal(self): bool - asm "b{10} SDBEGINSQ" "NIP"; + asm "b{10} SDBEGINSQ" "NIP" /// Returns if it's an external address, used to communication with the outside world. /// In TL/B it's addr_extern$01. @pure fun address.isExternal(self): bool - asm "b{01} SDBEGINSQ" "NIP"; + asm "b{01} SDBEGINSQ" "NIP" /// Extracts workchain and hash from a standard (internal) address. /// If the address is not internal, throws a cell deserialization exception. @pure fun address.getWorkchainAndHash(self): (int8, uint256) - asm "REWRITESTDADDR"; + asm "REWRITESTDADDR" /// Extracts workchain from a standard (internal) address. /// If the address is not internal, throws a cell deserialization exception. @pure fun address.getWorkchain(self): int8 - asm "REWRITESTDADDR" "DROP"; + asm "REWRITESTDADDR" "DROP" /// Checks whether two addresses are equal. Equivalent to `a == b`. /// Deprecated! Left for smoother transition from FunC, where you used `slice` everywhere. @@ -961,12 +928,12 @@ fun address.getWorkchain(self): int8 @pure @deprecated("use `senderAddress == ownerAddress`, not `senderAddress.bitsEqual(ownerAddress)`") fun address.bitsEqual(self, b: address): bool - asm "SDEQ"; + asm "SDEQ" /// Loads from slice [s] a valid `MsgAddress` (none/internal/external). @pure fun slice.loadAddress(mutate self): address - asm( -> 1 0) "LDMSGADDR"; + asm( -> 1 0) "LDMSGADDR" /** @@ -974,27 +941,27 @@ fun slice.loadAddress(mutate self): address */ /// mode = 0: Reserve exact amount of nanotoncoins -const RESERVE_MODE_EXACT_AMOUNT = 0; +const RESERVE_MODE_EXACT_AMOUNT = 0 /// +1: Actually reserves all but amount, meaning `currentContractBalance - amount` -const RESERVE_MODE_ALL_BUT_AMOUNT = 1; +const RESERVE_MODE_ALL_BUT_AMOUNT = 1 /// +2: Actually set `min(amount, currentContractBalance)` (without this mode, if amount is greater, the action will fail) -const RESERVE_MODE_AT_MOST = 2; +const RESERVE_MODE_AT_MOST = 2 /// +4: [amount] is increased by the _original_ balance of the current account (before the compute phase). -const RESERVE_MODE_INCREASE_BY_ORIGINAL_BALANCE = 4; +const RESERVE_MODE_INCREASE_BY_ORIGINAL_BALANCE = 4 /// +8: Actually sets `amount = -amount` before performing any further actions. -const RESERVE_MODE_NEGATE_AMOUNT = 8; +const RESERVE_MODE_NEGATE_AMOUNT = 8 /// +16: If this action fails, the transaction will be bounced. -const RESERVE_MODE_BOUNCE_ON_ACTION_FAIL = 16; +const RESERVE_MODE_BOUNCE_ON_ACTION_FAIL = 16 /// Creates an output action which would reserve Toncoins on balance. /// For [reserveMode] consider constants above. fun reserveToncoinsOnBalance(nanoTonCoins: coins, reserveMode: int): void - asm "RAWRESERVE"; + asm "RAWRESERVE" /// Similar to [reserveToncoinsOnBalance], but also accepts a dictionary extraAmount (represented by a cell or null) /// with extra currencies. In this way currencies other than Toncoin can be reserved. fun reserveExtraCurrenciesOnBalance(nanoTonCoins: coins, extraAmount: dict, reserveMode: int): void - asm "RAWRESERVEX"; + asm "RAWRESERVEX" /** @@ -1007,39 +974,39 @@ fun reserveExtraCurrenciesOnBalance(nanoTonCoins: coins, extraAmount: dict, rese */ /// mode = 0 is used for ordinary messages; the gas fees are deducted from the senging amount; action phaes should NOT be ignored. -const SEND_MODE_REGULAR = 0; +const SEND_MODE_REGULAR = 0 /// +1 means that the sender wants to pay transfer fees separately. -const SEND_MODE_PAY_FEES_SEPARATELY = 1; +const SEND_MODE_PAY_FEES_SEPARATELY = 1 /// +2 means that any errors arising while processing this message during the action phase should be ignored. -const SEND_MODE_IGNORE_ERRORS = 2; +const SEND_MODE_IGNORE_ERRORS = 2 /// in the case of action fail - bounce transaction. No effect if SEND_MODE_IGNORE_ERRORS (+2) is used. TVM UPGRADE 2023-07. https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages -const SEND_MODE_BOUNCE_ON_ACTION_FAIL = 16; +const SEND_MODE_BOUNCE_ON_ACTION_FAIL = 16 /// mode = 32 means that the current account must be destroyed if its resulting balance is zero. -const SEND_MODE_DESTROY = 32; +const SEND_MODE_DESTROY = 32 /// mode = 64 is used for messages that carry all the remaining value of the inbound message in addition to the value initially indicated in the new message. -const SEND_MODE_CARRY_ALL_REMAINING_MESSAGE_VALUE = 64; +const SEND_MODE_CARRY_ALL_REMAINING_MESSAGE_VALUE = 64 /// mode = 128 is used for messages that are to carry all the remaining balance of the current smart contract (instead of the value originally indicated in the message). -const SEND_MODE_CARRY_ALL_BALANCE = 128; +const SEND_MODE_CARRY_ALL_BALANCE = 128 /// do not create an action, only estimate fee. TVM UPGRADE 2023-07. https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages -const SEND_MODE_ESTIMATE_FEE_ONLY = 1024; +const SEND_MODE_ESTIMATE_FEE_ONLY = 1024 /// Other modes affect the fee calculation as follows: /// +64 substitutes the entire balance of the incoming message as an outcoming value (slightly inaccurate, gas expenses that cannot be estimated before the computation is completed are not taken into account). /// +128 substitutes the value of the entire balance of the contract before the start of the computation phase (slightly inaccurate, since gas expenses that cannot be estimated before the completion of the computation phase are not taken into account). -type ExtraCurrenciesDict = dict; +type ExtraCurrenciesDict = dict /// ContractState is "code + data" of a contract. /// Used in outgoing messages (StateInit) to initialize a destination contract. struct ContractState { - code: cell; - data: cell; + code: cell + data: cell } /// AddressShardingOptions provides settings to calculate an address in another shard. /// Consider [createMessage] and [address.buildSameAddressInAnotherShard] for usage. struct AddressShardingOptions { - fixedPrefixLength: uint5; // shard depth, formerly splitDepth - closeTo: address; + fixedPrefixLength: uint5 // shard depth, formerly splitDepth + closeTo: address } /// AutoDeployAddress is a destination that initializes a receiver contract if it does not exist yet. @@ -1060,9 +1027,9 @@ struct AddressShardingOptions { /// because in TON, the address of a contract, by definition, is a hash of its initial state. /// You can also use it without sending a message. See [buildAddress] and [addressMatches]. struct AutoDeployAddress { - workchain: int8 = BASECHAIN; - stateInit: ContractState | cell; - toShard: AddressShardingOptions? = null; + workchain: int8 = BASECHAIN + stateInit: ContractState | cell + toShard: AddressShardingOptions? = null } /// Constructs an address that a deployed contract will have. @@ -1076,7 +1043,7 @@ struct AutoDeployAddress { /// If you really need `address`, use `address.fromValidBuilder(res)`. @pure fun AutoDeployAddress.buildAddress(self): builder - builtin; + builtin /// Checks that an address matches a deployed contract. /// For example, from a jetton minter, you're checking whether a message is from a jetton wallet: @@ -1087,22 +1054,22 @@ fun AutoDeployAddress.buildAddress(self): builder /// Just instead of `dest` for [createMessage], you use it is to check whether a sender is a jetton/nft. @pure fun AutoDeployAddress.addressMatches(self, addr: address): bool - builtin; + builtin /// Options for creating an outgoing message. /// Consider [createMessage] for examples. struct CreateMessageOptions { /// whether a message will bounce back on error - bounce: bool; + bounce: bool /// message value: attached tons (or tons + extra currencies) - value: coins | (coins, ExtraCurrenciesDict); + value: coins | (coins, ExtraCurrenciesDict) /// destination is either a provided address, or is auto-calculated by stateInit - dest: address | // either just send a message to some address - builder | // ... or a manually constructed builder with a valid address - (int8, uint256) | // ... or to workchain + hash (also known as accountID) - AutoDeployAddress; // ... or "send to stateInit" aka deploy (address auto-calculated) + dest: address // either just send a message to some address + | builder // ... or a manually constructed builder with a valid address + | (int8, uint256) // ... or to workchain + hash (also known as accountID) + | AutoDeployAddress // ... or "send to stateInit" aka deploy (address auto-calculated) /// body is any serializable object (or just miss this field for empty body) - body: TBody; + body: TBody } /// Creates a message (`OutMessage`) — a well-formatted message cell. @@ -1123,7 +1090,7 @@ struct CreateMessageOptions { /// If you need an empty body, just miss the field `body`, fill other 3 fields. @pure fun createMessage(options: CreateMessageOptions): OutMessage - builtin; + builtin /// ExtOutLogBucket is a variant of a custom external address for emitting logs "to the outer world". @@ -1136,18 +1103,18 @@ fun createMessage(options: CreateMessageOptions): OutMessage /// > 0x01 ExtOutOffchainContract { adnl: uint256 | bits256 } /// Serialization details: '01' (addr_extern) + 256 (len) + 0x00 (prefix) + 248 bits = 267 in total struct (0x00) ExtOutLogBucket { - topic: uint248 | bits248; + topic: uint248 | bits248 } /// Options for creating an external outgoing message. /// Consider [createExternalLogMessage] for examples. struct CreateExternalLogMessageOptions { /// destination is either an external address or a pattern to calculate it - dest: address | // either some valid external/none address (not internal!) - builder | // ... or a manually constructed builder with a valid external address - ExtOutLogBucket; // ... or encode topic/eventID in destination + dest: address // either some valid external/none address (not internal!) + | builder // ... or a manually constructed builder with a valid external address + | ExtOutLogBucket // ... or encode topic/eventID in destination /// body is any serializable object (or just miss this field for empty body) - body: TBody; + body: TBody } /// Creates an external message (`OutMessage`) — a well-formatted message cell. @@ -1166,7 +1133,7 @@ struct CreateExternalLogMessageOptions { /// (if body is large, the compiler will automatically wrap it into a cell) @pure fun createExternalLogMessage(options: CreateExternalLogMessageOptions): OutMessage - builtin; + builtin /// UnsafeBodyNoRef is used to prevent default behavior: when message body is potentially large, /// it's packed into a separate ref. Wrapping body with this struct tells the compiler: @@ -1185,30 +1152,30 @@ fun createExternalLogMessage(options: CreateExternalLogMessageOptions { - forceInline: T; + forceInline: T } /// OutMessage is a result of [createMessage]. /// Essentially, it's a composed message cell, ready to be sent. struct OutMessage { - messageCell: cell; + messageCell: cell } /// Sends a ready message cell. /// For `sendMode`, see the constants above (SEND_MODE_*). fun OutMessage.send(self, sendMode: int): void - asm "SENDRAWMSG"; + asm "SENDRAWMSG" fun OutMessage.sendAndEstimateFee(self, sendMode: int): coins - asm "SENDMSG"; + asm "SENDMSG" @pure fun OutMessage.estimateFeeWithoutSending(self, sendMode: int): coins - asm "10 PUSHPOW2" "OR" "SENDMSG"; + asm "10 PUSHPOW2" "OR" "SENDMSG" @pure fun OutMessage.hash(self): uint256 - asm "HASHCU"; + asm "HASHCU" /// StateInit is a "canonical TL/B representation from block.tlb" of a contract initial state. @@ -1216,11 +1183,11 @@ fun OutMessage.hash(self): uint256 /// To represent code+data, consider [ContractState]. /// To represent sharding (fixedPrefixLength, formerly splitDepth), consider [AutoDeployAddress]. struct StateInit { - fixedPrefixLength: uint5?; - special: (bool, bool)?; - code: cell?; - data: cell?; - library: cell?; + fixedPrefixLength: uint5? + special: (bool, bool)? + code: cell? + data: cell? + library: cell? } /// Calculates a hash of StateInit if only code+data are set. @@ -1251,7 +1218,7 @@ fun StateInit.calcHashCodeData(code: cell, data: cell): uint256 asm 256 STU // store dataHash ONE HASHEXT_SHA256 -"""; +""" /// Calculates a hash of StateInit if fixedPrefixLength+code+data are set. @pure @@ -1280,7 +1247,7 @@ fun StateInit.calcHashPrefixCodeData(fixedPrefixLength: uint5, code: cell, data: ONE HASHEXT_SHA256 // pDepth hash NIP -"""; +""" /// Given an internal address A="aaaa...a" returns "bbaa...a" (D bits from address B, 256-D from A). @@ -1299,7 +1266,7 @@ fun StateInit.calcHashPrefixCodeData(fixedPrefixLength: uint5, code: cell, data: /// If you really need `address`, use `address.fromValidBuilder(resBuilder)`. @pure fun address.buildSameAddressInAnotherShard(self, options: AddressShardingOptions): builder - builtin; + builtin /// Converts a builder containing a valid address (internal/external/none) to `address` (slice under the hood). /// Gas-expensive! Because or a cell creation: essentially, it's `b.endCell().beginParse()`. @@ -1319,13 +1286,13 @@ fun address.buildSameAddressInAnotherShard(self, options: AddressShardingOptions /// ``` @pure fun address.fromValidBuilder(b: builder): address - asm "ENDC" "CTOS"; + asm "ENDC" "CTOS" /// Sends a raw message — a correctly serialized TL object `Message X`. /// In practice, you'll use a high-level wrapper [createMessage]. fun sendRawMessage(msg: cell, mode: int): void - asm "SENDRAWMSG"; + asm "SENDRAWMSG" @@ -1343,13 +1310,13 @@ fun sendRawMessage(msg: cell, mode: int): void /// in.body // actually gets from a TVM stack /// ``` struct InMessage { - senderAddress: address; // an internal address from which the message arrived - valueCoins: coins; // ton amount attached to an incoming message - valueExtra: dict; // extra currencies attached to an incoming message - originalForwardFee: coins; // fee that was paid by the sender - createdLt: uint64; // logical time when a message was created - createdAt: uint32; // unixtime when a message was created - body: slice; // message body, parse it with `lazy AllowedMsg.fromSlice(in.body)` + senderAddress: address // an internal address from which the message arrived + valueCoins: coins // ton amount attached to an incoming message + valueExtra: dict // extra currencies attached to an incoming message + originalForwardFee: coins // fee that was paid by the sender + createdLt: uint64 // logical time when a message was created + createdAt: uint32 // unixtime when a message was created + body: slice // message body, parse it with `lazy AllowedMsg.fromSlice(in.body)` } /// InMessageBounced is an input for `onBouncedMessage`. @@ -1363,51 +1330,51 @@ struct InMessage { /// val originalOpcode = in.bouncedBody.loadUint(32); /// ``` struct InMessageBounced { - senderAddress: address; // an internal address from which the message was bounced - valueCoins: coins; // ton amount attached to a message - valueExtra: dict; // extra currencies attached to a message - originalForwardFee: coins; // comission that the sender has payed to send this message - createdLt: uint64; // logical time when a message was created (and bounced) - createdAt: uint32; // unixtime when a message was created (and bounced) - bouncedBody: slice; // currently 256 bits: 0xFFFFFFFF + 224 bits sent originally + senderAddress: address // an internal address from which the message was bounced + valueCoins: coins // ton amount attached to a message + valueExtra: dict // extra currencies attached to a message + originalForwardFee: coins // comission that the sender has payed to send this message + createdLt: uint64 // logical time when a message was created (and bounced) + createdAt: uint32 // unixtime when a message was created (and bounced) + bouncedBody: slice // currently 256 bits: 0xFFFFFFFF + 224 bits sent originally } /// Skip 0xFFFFFFFF prefix (when a message is bounced). @pure fun slice.skipBouncedPrefix(mutate self): self - asm "32 LDU" "NIP"; + asm "32 LDU" "NIP" /// Load msgFlags from incoming message body (4 bits). @pure @deprecated("use `onBouncedMessage` handler instead of parsing msgCell flags manually") fun slice.loadMessageFlags(mutate self): int - asm( -> 1 0) "4 LDU"; + asm( -> 1 0) "4 LDU" /// Loads a message opcode (32-bit integer) when parsing message body manually. @pure fun slice.loadMessageOp(mutate self): int - asm( -> 1 0) "32 LDU"; + asm( -> 1 0) "32 LDU" /// Stores a message opcode (32-bit integer) when composing an output message manually. @pure @deprecated("use `createMessage` instead of composing messages manually") fun builder.storeMessageOp(mutate self, op: int): self - asm(op self) "32 STU"; + asm(op self) "32 STU" /// Loads a message queryId (64-bit integer, immediately following the opcode) when parsing message body manually. @pure @deprecated("use structures and lazy match instead of parsing messages manually") fun slice.loadMessageQueryId(mutate self): int - asm( -> 1 0) "64 LDU"; + asm( -> 1 0) "64 LDU" @pure @deprecated("use structures and lazy loading instead of parsing messages manually") fun slice.skipMessageQueryId(mutate self): self - asm "64 LDU" "NIP"; + asm "64 LDU" "NIP" @pure @deprecated("use `createMessage` instead of composing messages manually") fun builder.storeMessageQueryId(mutate self, queryId: int): self - asm(queryId self) "64 STU"; + asm(queryId self) "64 STU" diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 5aafedbe1d..977fa6f2d2 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -7,7 +7,7 @@ tolk 0.99 /// Returns amount of gas (in gas units) consumed in current Computation Phase. fun getGasConsumedAtTheMoment(): int - asm "GASCONSUMED"; + asm "GASCONSUMED" /// This function is required to be called when you process an external message (from an outer world) /// and "accept" it to blockchain. @@ -15,14 +15,14 @@ fun getGasConsumedAtTheMoment(): int /// As an effect, the current smart contract agrees to buy some gas to finish the current transaction. /// For more details, check [accept_message effects](https://ton.org/docs/#/smart-contracts/accept). fun acceptExternalMessage(): void - asm "ACCEPT"; + asm "ACCEPT" /// When processing an internal message, by default, the limit of gas consumption is determined by incoming message. /// Functions [setGasLimit] and [setGasLimitToMaximum] allow you to change this behavior. /// Sets current gas limit `gl` to its maximal allowed value `gm`, and resets the gas credit `gc` to zero, /// decreasing the value of `gr` by `gc` in the process. fun setGasLimitToMaximum(): void - asm "ACCEPT"; + asm "ACCEPT" /// When processing an internal message, by default, the limit of gas consumption is determined by incoming message. /// Functions [setGasLimit] and [setGasLimitToMaximum] allow you to change this behavior. @@ -30,41 +30,41 @@ fun setGasLimitToMaximum(): void /// If the gas consumed so far (including the present instruction) exceeds the resulting value of `gl`, /// an (unhandled) out of gas exception is thrown before setting new gas limits. fun setGasLimit(limit: int): void - asm "SETGASLIMIT"; + asm "SETGASLIMIT" /// Calculates fee (amount in nanotoncoins to be paid) for a transaction which consumed [gasUsed] gas units. fun calculateGasFee(workchain: int8, gasUsed: int): coins - asm(gasUsed workchain) "GETGASFEE"; + asm(gasUsed workchain) "GETGASFEE" /// Same as [calculateGasFee], but without flat price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) fun calculateGasFeeWithoutFlatPrice(workchain: int8, gasUsed: coins): coins - asm(gasUsed workchain) "GETGASFEESIMPLE"; + asm(gasUsed workchain) "GETGASFEESIMPLE" /// Calculates amount of nanotoncoins you should pay for storing a contract of provided size for [seconds]. /// [bits] and [cells] represent contract state (code + data). fun calculateStorageFee(workchain: int8, seconds: int, bits: int, cells: int): coins - asm(cells bits seconds workchain) "GETSTORAGEFEE"; + asm(cells bits seconds workchain) "GETSTORAGEFEE" /// Calculates amount of nanotoncoins you should pay to send a message of a specified size. fun calculateForwardFee(workchain: int8, bits: int, cells: int): coins - asm(cells bits workchain) "GETFORWARDFEE"; + asm(cells bits workchain) "GETFORWARDFEE" /// Same as [calculateForwardFee], but without lump price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) fun calculateForwardFeeWithoutLumpPrice(workchain: int8, bits: int, cells: int): coins - asm(cells bits workchain) "GETFORWARDFEESIMPLE"; + asm(cells bits workchain) "GETFORWARDFEESIMPLE" /// Calculates fee that was paid by the sender of an incoming internal message. @deprecated("use modern `onInternalMessage` and access `in.originalForwardFee` directly") fun calculateOriginalForwardFee(workchain: int8, incomingFwdFee: coins): coins - asm(incomingFwdFee workchain) "GETORIGINALFWDFEE"; + asm(incomingFwdFee workchain) "GETORIGINALFWDFEE" /// Returns the amount of nanotoncoins current contract debts for storage. ("due" and "debt" are synonyms) /// If it has no debt, `0` is returned. fun contract.getStorageDuePayment(): coins - asm "DUEPAYMENT"; + asm "DUEPAYMENT" /// Returns the amount of nanotoncoins charged for storage. /// (during storage phase preceeding to current computation phase) @pure fun contract.getStoragePaidPayment(): coins - asm "STORAGEFEES"; + asm "STORAGEFEES" diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk index 13625c5caf..2853abef03 100644 --- a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -9,25 +9,25 @@ tolk 0.99 @pure fun createEmptyList(): tuple - asm "PUSHNULL"; + asm "PUSHNULL" /// Adds an element to the beginning of lisp-style list. /// Note, that it does not mutate the list: instead, it returns a new one (it's a lisp pattern). @pure fun listPrepend(head: X, tail: tuple?): tuple - asm "CONS"; + asm "CONS" /// Extracts the head and the tail of lisp-style list. @pure fun listSplit(list: tuple): (X, tuple?) - asm "UNCONS"; + asm "UNCONS" /// Returns the head of lisp-style list. @pure fun listGetHead(list: tuple): X - asm "CAR"; + asm "CAR" /// Returns the tail of lisp-style list. @pure fun listGetTail(list: tuple): tuple? - asm "CDR"; + asm "CDR" diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index dc2d3fd6b5..14723682a0 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -14,289 +14,289 @@ tolk 0.99 */ /// In @stdlib/common.tolk, there is a type alias: -/// `type dict = cell?;` +/// `type dict = cell?` /// For clarity, we use "dict" instead of a "cell?" where a cell-dictionary is assumed. /// Creates an empty dictionary, which is actually a null value. Equivalent to PUSHNULL @pure fun createEmptyDict(): dict - asm "NEWDICT"; + asm "NEWDICT" /// Checks whether a dictionary is empty. @pure fun dict.dictIsEmpty(self): bool - asm "DICTEMPTY"; + asm "DICTEMPTY" @pure fun dict.iDictGet(self, keyLen: int, key: int): (slice?, bool) - asm(key self keyLen) "DICTIGET" "NULLSWAPIFNOT"; + asm(key self keyLen) "DICTIGET" "NULLSWAPIFNOT" @pure fun dict.uDictGet(self, keyLen: int, key: int): (slice?, bool) - asm(key self keyLen) "DICTUGET" "NULLSWAPIFNOT"; + asm(key self keyLen) "DICTUGET" "NULLSWAPIFNOT" @pure fun dict.sDictGet(self, keyLen: int, key: slice): (slice?, bool) - asm(key self keyLen) "DICTGET" "NULLSWAPIFNOT"; + asm(key self keyLen) "DICTGET" "NULLSWAPIFNOT" @pure fun dict.iDictSet(mutate self, keyLen: int, key: int, value: slice): void - asm(value key self keyLen) "DICTISET"; + asm(value key self keyLen) "DICTISET" @pure fun dict.uDictSet(mutate self, keyLen: int, key: int, value: slice): void - asm(value key self keyLen) "DICTUSET"; + asm(value key self keyLen) "DICTUSET" @pure fun dict.sDictSet(mutate self, keyLen: int, key: slice, value: slice): void - asm(value key self keyLen) "DICTSET"; + asm(value key self keyLen) "DICTSET" @pure fun dict.iDictSetRef(mutate self, keyLen: int, key: int, value: cell): void - asm(value key self keyLen) "DICTISETREF"; + asm(value key self keyLen) "DICTISETREF" @pure fun dict.uDictSetRef(mutate self, keyLen: int, key: int, value: cell): void - asm(value key self keyLen) "DICTUSETREF"; + asm(value key self keyLen) "DICTUSETREF" @pure fun dict.sDictSetRef(mutate self, keyLen: int, key: slice, value: cell): void - asm(value key self keyLen) "DICTSETREF"; + asm(value key self keyLen) "DICTSETREF" @pure fun dict.iDictSetIfNotExists(mutate self, keyLen: int, key: int, value: slice): bool - asm(value key self keyLen) "DICTIADD"; + asm(value key self keyLen) "DICTIADD" @pure fun dict.uDictSetIfNotExists(mutate self, keyLen: int, key: int, value: slice): bool - asm(value key self keyLen) "DICTUADD"; + asm(value key self keyLen) "DICTUADD" @pure fun dict.iDictSetIfExists(mutate self, keyLen: int, key: int, value: slice): bool - asm(value key self keyLen) "DICTIREPLACE"; + asm(value key self keyLen) "DICTIREPLACE" @pure fun dict.uDictSetIfExists(mutate self, keyLen: int, key: int, value: slice): bool - asm(value key self keyLen) "DICTUREPLACE"; + asm(value key self keyLen) "DICTUREPLACE" @pure fun dict.iDictGetRef(self, keyLen: int, key: int): (dict, bool) - asm(key self keyLen) "DICTIGETREF" "NULLSWAPIFNOT"; + asm(key self keyLen) "DICTIGETREF" "NULLSWAPIFNOT" @pure fun dict.uDictGetRef(self, keyLen: int, key: int): (dict, bool) - asm(key self keyLen) "DICTUGETREF" "NULLSWAPIFNOT"; + asm(key self keyLen) "DICTUGETREF" "NULLSWAPIFNOT" @pure fun dict.sDictGetRef(self, keyLen: int, key: slice): (dict, bool) - asm(key self keyLen) "DICTGETREF" "NULLSWAPIFNOT"; + asm(key self keyLen) "DICTGETREF" "NULLSWAPIFNOT" @pure fun dict.iDictGetRefOrNull(self, keyLen: int, key: int): dict - asm(key self keyLen) "DICTIGETOPTREF"; + asm(key self keyLen) "DICTIGETOPTREF" @pure fun dict.uDictGetRefOrNull(self, keyLen: int, key: int): dict - asm(key self keyLen) "DICTUGETOPTREF"; + asm(key self keyLen) "DICTUGETOPTREF" @pure fun dict.sDictGetRefOrNull(self, keyLen: int, key: slice): dict - asm(key self keyLen) "DICTGETOPTREF"; + asm(key self keyLen) "DICTGETOPTREF" @pure fun dict.iDictDelete(mutate self, keyLen: int, key: int): bool - asm(key self keyLen) "DICTIDEL"; + asm(key self keyLen) "DICTIDEL" @pure fun dict.uDictDelete(mutate self, keyLen: int, key: int): bool - asm(key self keyLen) "DICTUDEL"; + asm(key self keyLen) "DICTUDEL" @pure fun dict.sDictDelete(mutate self, keyLen: int, key: slice): bool - asm(key self keyLen) "DICTDEL"; + asm(key self keyLen) "DICTDEL" @pure fun dict.iDictSetAndGet(mutate self, keyLen: int, key: int, value: slice): (slice?, bool) - asm(value key self keyLen) "DICTISETGET" "NULLSWAPIFNOT"; + asm(value key self keyLen) "DICTISETGET" "NULLSWAPIFNOT" @pure fun dict.uDictSetAndGet(mutate self, keyLen: int, key: int, value: slice): (slice?, bool) - asm(value key self keyLen) "DICTUSETGET" "NULLSWAPIFNOT"; + asm(value key self keyLen) "DICTUSETGET" "NULLSWAPIFNOT" @pure fun dict.sDictSetAndGet(mutate self, keyLen: int, key: slice, value: slice): (slice?, bool) - asm(value key self keyLen) "DICTSETGET" "NULLSWAPIFNOT"; + asm(value key self keyLen) "DICTSETGET" "NULLSWAPIFNOT" @pure fun dict.iDictSetAndGetRefOrNull(mutate self, keyLen: int, key: int, value: cell): dict - asm(value key self keyLen) "DICTISETGETOPTREF"; + asm(value key self keyLen) "DICTISETGETOPTREF" @pure fun dict.uDictSetAndGetRefOrNull(mutate self, keyLen: int, key: int, value: cell): dict - asm(value key self keyLen) "DICTUSETGETOPTREF"; + asm(value key self keyLen) "DICTUSETGETOPTREF" @pure fun dict.iDictDeleteAndGet(mutate self, keyLen: int, key: int): (slice?, bool) - asm(key self keyLen) "DICTIDELGET" "NULLSWAPIFNOT"; + asm(key self keyLen) "DICTIDELGET" "NULLSWAPIFNOT" @pure fun dict.uDictDeleteAndGet(mutate self, keyLen: int, key: int): (slice?, bool) - asm(key self keyLen) "DICTUDELGET" "NULLSWAPIFNOT"; + asm(key self keyLen) "DICTUDELGET" "NULLSWAPIFNOT" @pure fun dict.sDictDeleteAndGet(mutate self, keyLen: int, key: slice): (slice?, bool) - asm(key self keyLen) "DICTDELGET" "NULLSWAPIFNOT"; + asm(key self keyLen) "DICTDELGET" "NULLSWAPIFNOT" @pure fun dict.iDictSetBuilder(mutate self, keyLen: int, key: int, value: builder): void - asm(value key self keyLen) "DICTISETB"; + asm(value key self keyLen) "DICTISETB" @pure fun dict.uDictSetBuilder(mutate self, keyLen: int, key: int, value: builder): void - asm(value key self keyLen) "DICTUSETB"; + asm(value key self keyLen) "DICTUSETB" @pure fun dict.sDictSetBuilder(mutate self, keyLen: int, key: slice, value: builder): void - asm(value key self keyLen) "DICTSETB"; + asm(value key self keyLen) "DICTSETB" @pure fun dict.iDictSetBuilderIfNotExists(mutate self, keyLen: int, key: int, value: builder): bool - asm(value key self keyLen) "DICTIADDB"; + asm(value key self keyLen) "DICTIADDB" @pure fun dict.uDictSetBuilderIfNotExists(mutate self, keyLen: int, key: int, value: builder): bool - asm(value key self keyLen) "DICTUADDB"; + asm(value key self keyLen) "DICTUADDB" @pure fun dict.iDictSetBuilderIfExists(mutate self, keyLen: int, key: int, value: builder): bool - asm(value key self keyLen) "DICTIREPLACEB"; + asm(value key self keyLen) "DICTIREPLACEB" @pure fun dict.uDictSetBuilderIfExists(mutate self, keyLen: int, key: int, value: builder): bool - asm(value key self keyLen) "DICTUREPLACEB"; + asm(value key self keyLen) "DICTUREPLACEB" @pure fun dict.iDictDeleteFirstAndGet(mutate self, keyLen: int): (int?, slice?, bool) - asm(-> 0 2 1 3) "DICTIREMMIN" "NULLSWAPIFNOT2"; + asm(-> 0 2 1 3) "DICTIREMMIN" "NULLSWAPIFNOT2" @pure fun dict.uDictDeleteFirstAndGet(mutate self, keyLen: int): (int?, slice?, bool) - asm(-> 0 2 1 3) "DICTUREMMIN" "NULLSWAPIFNOT2"; + asm(-> 0 2 1 3) "DICTUREMMIN" "NULLSWAPIFNOT2" @pure fun dict.sDictDeleteFirstAndGet(mutate self, keyLen: int): (slice?, slice?, bool) - asm(-> 0 2 1 3) "DICTREMMIN" "NULLSWAPIFNOT2"; + asm(-> 0 2 1 3) "DICTREMMIN" "NULLSWAPIFNOT2" @pure fun dict.iDictDeleteLastAndGet(mutate self, keyLen: int): (int?, slice?, bool) - asm(-> 0 2 1 3) "DICTIREMMAX" "NULLSWAPIFNOT2"; + asm(-> 0 2 1 3) "DICTIREMMAX" "NULLSWAPIFNOT2" @pure fun dict.uDictDeleteLastAndGet(mutate self, keyLen: int): (int?, slice?, bool) - asm(-> 0 2 1 3) "DICTUREMMAX" "NULLSWAPIFNOT2"; + asm(-> 0 2 1 3) "DICTUREMMAX" "NULLSWAPIFNOT2" @pure fun dict.sDictDeleteLastAndGet(mutate self, keyLen: int): (slice?, slice?, bool) - asm(-> 0 2 1 3) "DICTREMMAX" "NULLSWAPIFNOT2"; + asm(-> 0 2 1 3) "DICTREMMAX" "NULLSWAPIFNOT2" @pure fun dict.iDictGetFirst(self, keyLen: int): (int?, slice?, bool) - asm (-> 1 0 2) "DICTIMIN" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTIMIN" "NULLSWAPIFNOT2" @pure fun dict.uDictGetFirst(self, keyLen: int): (int?, slice?, bool) - asm (-> 1 0 2) "DICTUMIN" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTUMIN" "NULLSWAPIFNOT2" @pure fun dict.sDictGetFirst(self, keyLen: int): (slice?, slice?, bool) - asm (-> 1 0 2) "DICTMIN" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTMIN" "NULLSWAPIFNOT2" @pure fun dict.iDictGetFirstAsRef(self, keyLen: int): (int?, dict, bool) - asm (-> 1 0 2) "DICTIMINREF" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTIMINREF" "NULLSWAPIFNOT2" @pure fun dict.uDictGetFirstAsRef(self, keyLen: int): (int?, dict, bool) - asm (-> 1 0 2) "DICTUMINREF" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTUMINREF" "NULLSWAPIFNOT2" @pure fun dict.sDictGetFirstAsRef(self, keyLen: int): (slice?, dict, bool) - asm (-> 1 0 2) "DICTMINREF" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTMINREF" "NULLSWAPIFNOT2" @pure fun dict.iDictGetLast(self, keyLen: int): (int?, slice?, bool) - asm (-> 1 0 2) "DICTIMAX" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTIMAX" "NULLSWAPIFNOT2" @pure fun dict.uDictGetLast(self, keyLen: int): (int?, slice?, bool) - asm (-> 1 0 2) "DICTUMAX" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTUMAX" "NULLSWAPIFNOT2" @pure fun dict.sDictGetLast(self, keyLen: int): (slice?, slice?, bool) - asm (-> 1 0 2) "DICTMAX" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTMAX" "NULLSWAPIFNOT2" @pure fun dict.iDictGetLastAsRef(self, keyLen: int): (int?, dict, bool) - asm (-> 1 0 2) "DICTIMAXREF" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTIMAXREF" "NULLSWAPIFNOT2" @pure fun dict.uDictGetLastAsRef(self, keyLen: int): (int?, dict, bool) - asm (-> 1 0 2) "DICTUMAXREF" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTUMAXREF" "NULLSWAPIFNOT2" @pure fun dict.sDictGetLastAsRef(self, keyLen: int): (slice?, dict, bool) - asm (-> 1 0 2) "DICTMAXREF" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTMAXREF" "NULLSWAPIFNOT2" @pure fun dict.iDictGetNext(self, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXT" "NULLSWAPIFNOT2"; + asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXT" "NULLSWAPIFNOT2" @pure fun dict.uDictGetNext(self, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXT" "NULLSWAPIFNOT2"; + asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXT" "NULLSWAPIFNOT2" @pure fun dict.iDictGetNextOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXTEQ" "NULLSWAPIFNOT2"; + asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXTEQ" "NULLSWAPIFNOT2" @pure fun dict.uDictGetNextOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXTEQ" "NULLSWAPIFNOT2"; + asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXTEQ" "NULLSWAPIFNOT2" @pure fun dict.iDictGetPrev(self, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTIGETPREV" "NULLSWAPIFNOT2"; + asm(pivot self keyLen -> 1 0 2) "DICTIGETPREV" "NULLSWAPIFNOT2" @pure fun dict.uDictGetPrev(self, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTUGETPREV" "NULLSWAPIFNOT2"; + asm(pivot self keyLen -> 1 0 2) "DICTUGETPREV" "NULLSWAPIFNOT2" @pure fun dict.iDictGetPrevOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTIGETPREVEQ" "NULLSWAPIFNOT2"; + asm(pivot self keyLen -> 1 0 2) "DICTIGETPREVEQ" "NULLSWAPIFNOT2" @pure fun dict.uDictGetPrevOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTUGETPREVEQ" "NULLSWAPIFNOT2"; + asm(pivot self keyLen -> 1 0 2) "DICTUGETPREVEQ" "NULLSWAPIFNOT2" /** @@ -305,12 +305,12 @@ fun dict.uDictGetPrevOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool @pure fun dict.prefixDictGet(self, keyLen: int, key: slice): (slice, slice?, slice?, bool) - asm(key self keyLen) "PFXDICTGETQ" "NULLSWAPIFNOT2"; + asm(key self keyLen) "PFXDICTGETQ" "NULLSWAPIFNOT2" @pure fun dict.prefixDictSet(mutate self, keyLen: int, key: slice, value: slice): bool - asm(value key self keyLen) "PFXDICTSET"; + asm(value key self keyLen) "PFXDICTSET" @pure fun dict.prefixDictDelete(mutate self, keyLen: int, key: slice): bool - asm(key self keyLen) "PFXDICTDEL"; + asm(key self keyLen) "PFXDICTDEL" diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index 724464b706..18a0e159fd 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -5,21 +5,21 @@ tolk 0.99 /// The primitive returns the current value of `c3`. @pure fun getTvmRegisterC3(): continuation - asm "c3 PUSH"; + asm "c3 PUSH" /// Updates the current value of `c3`. Usually, it is used for updating smart contract code in run-time. /// Note that after execution of this primitive the current code /// (and the stack of recursive function calls) won't change, /// but any other function call will use a function from the new code. fun setTvmRegisterC3(c: continuation): void - asm "c3 POP"; + asm "c3 POP" /// Transforms a `slice` [s] into a simple ordinary continuation `c`, with `c.code = s` and an empty stack and savelist. @pure fun transformSliceToContinuation(s: slice): continuation - asm "BLESS"; + asm "BLESS" /// Moves a variable or a value [x] to the top of the stack. @pure fun T.stackMoveToTop(mutate self): void - asm "NOP"; + asm "NOP" diff --git a/tolk-tester/tests/assignment-tests.tolk b/tolk-tester/tests/assignment-tests.tolk index bd9b28d9b4..5b8c2094c4 100644 --- a/tolk-tester/tests/assignment-tests.tolk +++ b/tolk-tester/tests/assignment-tests.tolk @@ -1,5 +1,5 @@ fun extractFromTypedTuple(params: [int]) { - var [payload: int] = params; + var [payload: int, ] = params; return payload + 10; } @@ -111,7 +111,7 @@ global t107: (int, int); @method_id(107) fun test107() { - ((t107 = (1, 2)).0, (t107 = (3, 4)).1) = (5, 6); + ((t107 = (1, 2)).0, (t107 = (3, 4)).1, ) = (5, 6, ); return t107; } @@ -168,7 +168,7 @@ fun test112() { @method_id(113) fun test113() { - var (a, t, z) = (1, [2,3], (-1,-1)); + var (a, t, z, ) = (1, [2,3], (-1,-1)); (a, t, a, z, t.1, z.1) = (10, [a,12], 13, (a, t.1), 14, t.1); return (a, t, z); } @@ -252,7 +252,7 @@ fun test118(x: int) { fun main(value: int, ) { - var (x: int?, y) = (autoInferIntNull(value), autoInferIntNull(value * 2)); + var (x: int?, y,) = (autoInferIntNull(value), autoInferIntNull(value * 2)); if (x == null && y == null) { return null; } return x == null || y == null ? -1 : x! + y!; } diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index 607cccf0a9..cafd2ffde8 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -1,8 +1,8 @@ -type MInt = int; -type MSlice = slice; +type MInt = int +type MSlice = slice -const int1 = 1; -const int2 = 2; +const int1 = 1 +const int2 = 2 const int101: int = 101; const int111: MInt = 111; @@ -24,9 +24,8 @@ const strange_minus_1: MInt = (!0 as int); const addr1 = address("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"); -const true1 = true; -const true2 = !!true; -const true3 = true1 && true2; +// since `;` is not mandatory, this is correct from a syntax point of view +const true1 = true const true2 = !!true const true3 = true1 && true2 const false1 = !true; const false2 = false1 || false; diff --git a/tolk-tester/tests/indexed-access.tolk b/tolk-tester/tests/indexed-access.tolk index dd0400c281..af6888df78 100644 --- a/tolk-tester/tests/indexed-access.tolk +++ b/tolk-tester/tests/indexed-access.tolk @@ -41,8 +41,8 @@ fun plus(mutate x: int, y: int): int { fun eq(v: X): X { return v; } -global gTup: [int]; -global gTens: (int, int); +global gTup: [int] +global gTens: (int, int) @method_id(100) fun testCodegenSimple() { diff --git a/tolk-tester/tests/invalid-declaration/err-1043.tolk b/tolk-tester/tests/invalid-declaration/err-1043.tolk new file mode 100644 index 0000000000..4ef7d292fd --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1043.tolk @@ -0,0 +1,6 @@ +get id() { return 123; } + +/** +@compilation_should_fail +@stderr expected `fun`, got `id` + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1794.tolk b/tolk-tester/tests/invalid-declaration/err-1794.tolk index a064a633c2..3505572cf4 100644 --- a/tolk-tester/tests/invalid-declaration/err-1794.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1794.tolk @@ -1,4 +1,4 @@ -get int.seqno(self) { +get fun int.seqno(self) { return 0; } diff --git a/tolk-tester/tests/invalid-syntax/err-3618.tolk b/tolk-tester/tests/invalid-syntax/err-3618.tolk index 7751c4a95f..0c3fa7d3f5 100644 --- a/tolk-tester/tests/invalid-syntax/err-3618.tolk +++ b/tolk-tester/tests/invalid-syntax/err-3618.tolk @@ -2,5 +2,5 @@ const x = 0b0012; /** @compilation_should_fail -@stderr expected `;`, got `2` +@stderr expected top-level declaration, got `2` */ diff --git a/tolk-tester/tests/invalid-syntax/err-3802.tolk b/tolk-tester/tests/invalid-syntax/err-3802.tolk deleted file mode 100644 index fec840c981..0000000000 --- a/tolk-tester/tests/invalid-syntax/err-3802.tolk +++ /dev/null @@ -1,9 +0,0 @@ -struct A { - id1: int - id2: int -} - -/** -@compilation_should_fail -@stderr expected `;` or `,`, got `id2` -*/ diff --git a/tolk-tester/tests/methods-tests.tolk b/tolk-tester/tests/methods-tests.tolk index db3c3e9cdd..4b09a53d6a 100644 --- a/tolk-tester/tests/methods-tests.tolk +++ b/tolk-tester/tests/methods-tests.tolk @@ -207,12 +207,12 @@ fun test12() { __expect_type((null as slice?).arbitraryMethod(), "never"); } -type BuilderOrInt = builder|int; -fun (int|builder).isInt(self) { return self is int; } +type BuilderOrInt = |builder|int +fun|int|builder.isInt(self) { return self is int; } @method_id(113) fun test13() { - var u1 = beginCell() as builder|int; + var u1 = beginCell() as|builder|int; var u2 = 10 as int|builder; var u3 = 20 as BuilderOrInt; return (u1.isInt(), u2.isInt(), u3.isInt(), 10.isInt(), beginCell().isInt(), (8 as int8).isInt()); @@ -224,7 +224,7 @@ fun (int|builder|T).isBuilder(self) { return self is builder; } fun test14() { var b1 = beginCell() as BuilderOrInt | slice; var b2 = beginCell() as builder|slice|int|Point; - var i1 = 5 as int|builder|continuation|(int, int); + var i1 = 5 as |int|builder|continuation|(int, int); var c = beginCell().endCell() as cell?|builder|int; var cn = null as cell?|int|int8|builder; return (b1.isBuilder(), b2.isBuilder(), i1.isBuilder(), c.isBuilder(), cn.isBuilder()); diff --git a/tolk-tester/tests/mutate-methods.tolk b/tolk-tester/tests/mutate-methods.tolk index 892a625cec..f3e93b694d 100644 --- a/tolk-tester/tests/mutate-methods.tolk +++ b/tolk-tester/tests/mutate-methods.tolk @@ -138,8 +138,8 @@ fun updateTwoItems(mutate pair: Pair2, byValue: int) { pair = (first + byValue, second + byValue); } -global t107_1: int; -global t107_2: int; +global t107_1: int +global t107_2: int @method_id(107) fun testMutableTensor() { diff --git a/tolk-tester/tests/no-spaces.tolk b/tolk-tester/tests/no-spaces.tolk index 732de8011d..3a703dc7c3 100644 --- a/tolk-tester/tests/no-spaces.tolk +++ b/tolk-tester/tests/no-spaces.tolk @@ -1,8 +1,10 @@ -const int10:int=10; +const int10:int=10 const dd=1 fun just10(): int { return int10; } fun eq(v: int): int { return`v` } +const ONE = 1struct Point{x:int y:int} + @method_id(101,) fun `get_-1` (): int {return-1} @method_id(102,) fun `get_--1` (): int {return- -1} @method_id(103,) fun `get_---1`(): int {return-(-(-1))} @@ -13,7 +15,7 @@ global `some()var`:int; @method_id(110) fun `some_math`(): int { `some()var`=- -6; - return 1*-2*-3*-4*just10()*-5+-`some()var`+-(-`some()var`)-(-(-`some()var`)); + return ONE*-2*-3*-4*just10()*-5+-`some()var`+-(-`some()var`)-(-(-`some()var`)); } @method_id(111) fun `negative_nums`(a:int):int { diff --git a/tolk-tester/tests/pack-unpack-3.tolk b/tolk-tester/tests/pack-unpack-3.tolk index 310a3ed6cc..793f714b9d 100644 --- a/tolk-tester/tests/pack-unpack-3.tolk +++ b/tolk-tester/tests/pack-unpack-3.tolk @@ -114,11 +114,10 @@ struct(0x00184300) CounterResetTo { } type MsgCounter1 = - CounterIncrement | - CounterDecrement | - CounterReset0 | - CounterResetTo | -; + | CounterIncrement + | CounterDecrement + | CounterReset0 + | CounterResetTo /* bodyPayload1$001 @@ -146,7 +145,7 @@ struct(0b01) BodyPayload2 { owner_address: address; } -type BodyPayload = BodyPayload1 | BodyPayload2; +type BodyPayload = BodyPayload1 | BodyPayload2 /* sayHiAndGoodbye#89 @@ -172,7 +171,7 @@ struct(0x0013) SayStoreInChain { contents: Cell; } -type MsgExternal1 = SayHiAndGoodbye | SayStoreInChain; +type MsgExternal1 = SayHiAndGoodbye | SayStoreInChain /* transferParams1#794 diff --git a/tolk-tester/tests/pack-unpack-5.tolk b/tolk-tester/tests/pack-unpack-5.tolk index af397181e5..1baa240642 100644 --- a/tolk-tester/tests/pack-unpack-5.tolk +++ b/tolk-tester/tests/pack-unpack-5.tolk @@ -118,11 +118,11 @@ fun test6() { return (Test6_1.estimatePackSize(), Test6_2.estimatePackSize(), Test6_or.estimatePackSize()); } -struct(0x1020) Test7_1 { } -struct(0x1030) Test7_2 { } -struct(0x1040) Test7_3 { } +struct(0x1020) Test7_1; +struct(0x1030) Test7_2; +struct(0x1040) Test7_3; -type Test7_or = Test7_1 | Test7_2 | Test7_3 |; +type Test7_or = | Test7_1 | Test7_2 | Test7_3 @method_id(107) fun test7() { diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 1a0f96a0fa..90f1ae6446 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -168,7 +168,7 @@ fun test10(): int { @method_id(111) fun test11() { - var [x, y] = [getNullableInt(), getNullableInt()]; + var [x, y,] = [getNullableInt(), getNullableInt()]; if (eq(10) < 0) { return x == null || y == null ? -1 : x + y; } if (true && (x == null || y == null) && !!true) { return 0; } return x + y; diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index ab2facf8a5..863a8aeff4 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -1,20 +1,20 @@ -get ascii_slice(): slice { +get fun ascii_slice(): slice { return"string"; } -get raw_slice(): slice { +get fun raw_slice(): slice { return stringHexToSlice("abcdef"); } -get addr_slice(): address { +get fun addr_slice(): address { return address("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"); } -get string_hex(): int { +get fun string_hex(): int { return stringToBase256("ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"); } -get fun string_minihash(): int { // 'get' and 'get fun' both possible +get fun string_minihash(): int { return stringSha256_32("transfer(slice, int)"); } diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 18d596c34d..8eba3b4274 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -6,8 +6,8 @@ struct JustIntWrapper { int: JustInt; } -type MInt = int; -type JustIntAlias = JustInt; +type MInt = int +type JustIntAlias = JustInt type JustIntAlias2 = JustIntAlias; struct Storage { @@ -16,13 +16,13 @@ struct Storage { } struct Point { - x: int; + x: int y: MInt } struct User { - id: int; - name: slice; + id: int + name: slice } struct OuterStorage { @@ -31,10 +31,10 @@ struct OuterStorage { } struct WithTensorInside { - coords: (int, int); - tup: [int, int]; - otherCoords: (int, int)?; - otherTup: tuple; + coords: (int, int) + tup: [int, int] + otherCoords: (int, int)? + otherTup: tuple } fun sumCoords(p: Point) { @@ -133,9 +133,12 @@ fun test7() { return (s.owner.name.loadInt(32), s2.owner.name.loadInt(32), s.lastPoint, s2.lastPoint.sumCoords()); } -struct Empty{} +struct Empty struct OuterEmpty { nothing: EmptyAlias } -type EmptyAlias = Empty; +type EmptyAlias = Empty + +struct EmptyGeneric1; +struct EmptyGeneric2 @method_id(108) fun test8() { @@ -152,6 +155,7 @@ fun test8() { __expect_type(o3, "Empty?"); __expect_type(o6, "OuterEmpty"); __expect_type(o8, "OuterEmpty?"); + __expect_type(EmptyGeneric2{}, "EmptyGeneric2"); return (e1, Empty{}, EmptyAlias{}, o2, o1, 777, o3, 777, o4, 777, o5, 777, o6, 777, o7, 777, o8, 777, o3!, 777); } diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk index 3fc5345050..3bc0e51142 100644 --- a/tolk-tester/tests/type-aliases-tests.tolk +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -1,13 +1,13 @@ import "@stdlib/tvm-dicts.tolk" -type MIntN = MInt?; -type MInt = int; -type MInt_v2 = int; -type MVoid = void; -type Pair2_v1 = (int, int); -type Pair2_v2 = (MInt, MInt); -type MBool = bool; -type Tuple2Int = [int, int]; +type MIntN = MInt? +type MInt = int +type MInt_v2 = int +type MVoid = void +type Pair2_v1 = | (int, int) +type Pair2_v2 = (MInt, MInt) +type MBool = | bool +type Tuple2Int = [int, int] struct Point { x: int; y: int } type PointAlias = Point; @@ -160,7 +160,7 @@ fun test10() { var (a2, b2) = x2; __expect_type(a2, "MInt"); var x3: Tuple2Int = [9, 10]; - var [a3, b3] = x3; + var [a3, b3, ] = x3; return a1 + a2 + a3 + b1 + b2 + b3; } diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 5dc3c9fca6..7b9d52b61e 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -1,10 +1,10 @@ -type MInt = int; -type MNull = null; -type MSlice = slice; -type MIntN = int | MNull; -type IntOrSlice = MInt | slice; -type IntOrSliceN = null | slice | int; -type IntOrSliceOrBuilder = int | slice | builder; +type MInt = int +type MNull = | null +type MSlice = slice +type MIntN = int | MNull +type IntOrSlice = MInt | slice +type IntOrSliceN = null | slice | int +type IntOrSliceOrBuilder = | int | slice | builder type Pair2 = (int, int); type Pair2Or3 = Pair2 | (int, int, int); @@ -18,8 +18,8 @@ fun testFlatten() { __expect_type(0 as (int | int | slice | builder | slice), "int | slice | builder"); __expect_type(0 as (int? | int), "int?"); __expect_type(0 as (slice | int? | slice? | cell), "slice | int | null | cell"); - __expect_type(0 as (int | (int | int)), "int"); - __expect_type(0 as (slice | (slice | (slice | int) | (cell | slice?))), "slice | int | cell | null"); + __expect_type(0 as (| int | (int | int)), "int"); + __expect_type(0 as (slice | (slice | (slice | int) | (| cell | slice?))), "slice | int | cell | null"); __expect_type(0 as (MInt | MNull), "MInt?"); __expect_type(0 as (MInt | slice | int), "MInt | slice"); @@ -43,7 +43,7 @@ fun testFlatten() { __expect_type(0 as (int?)?, "int?"); __expect_type(0 as (int?)? | (int)?, "int?"); - __expect_type(0 is builder, "bool"); + __expect_type(0 is (|builder), "bool"); __expect_type(0 !is builder, "bool"); __expect_type(0!!is builder, "bool"); @@ -80,7 +80,7 @@ fun test2() { } @method_id(103) -fun test3(case: int): int | slice | null { +fun test3(case: int): | int | slice | null { var a: int | slice | null = 4; if (case == 0) { a = getSlice32() diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 98d4e546ec..39cef55b7c 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -236,6 +236,9 @@ static AnyTypeV parse_type_nullable(Lexer& lex) { } static AnyTypeV parse_type_expression(Lexer& lex) { + if (lex.tok() == tok_bitwise_or) { // allow leading `|`, like in TypeScript + lex.next(); + } AnyTypeV result = parse_type_nullable(lex); if (lex.tok() == tok_bitwise_or) { // `int | slice`, `Pair2 | (Pair3 | null)` @@ -243,9 +246,6 @@ static AnyTypeV parse_type_expression(Lexer& lex) { items.emplace_back(result); while (lex.tok() == tok_bitwise_or) { lex.next(); - if (lex.tok() == tok_clpar || lex.tok() == tok_clbracket || lex.tok() == tok_semicolon) { - break; // allow trailing `|` (not leading, like in TypeScript, because of tree-sitter) - } items.emplace_back(parse_type_nullable(lex)); } result = createV(result->loc, std::move(items)); @@ -368,7 +368,6 @@ static AnyV parse_global_var_declaration(Lexer& lex, const std::vectorkind) { @@ -399,7 +398,6 @@ static AnyV parse_constant_declaration(Lexer& lex, const std::vectorkind) { @@ -428,7 +426,6 @@ static AnyV parse_type_alias_declaration(Lexer& lex, const std::vectorkind) { @@ -455,6 +452,9 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable, bool al std::vector args(1, first); while (lex.tok() == tok_comma) { lex.next(); + if (lex.tok() == tok_clpar) { // trailing comma + break; + } args.push_back(parse_var_declaration_lhs(lex, is_immutable, false)); } lex.expect(tok_clpar, "`)`"); @@ -465,6 +465,9 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable, bool al std::vector args(1, parse_var_declaration_lhs(lex, is_immutable, false)); while (lex.tok() == tok_comma) { lex.next(); + if (lex.tok() == tok_clbracket) { // trailing comma + break; + } args.push_back(parse_var_declaration_lhs(lex, is_immutable, false)); } lex.expect(tok_clbracket, "`]`"); @@ -1327,7 +1330,6 @@ static AnyV parse_asm_func_body(Lexer& lex, V param_list) { asm_commands.push_back(createV(lex.cur_location(), asm_command)); lex.next(); } - lex.expect(tok_semicolon, "`;`"); return createV(loc, std::move(arg_order), std::move(ret_order), std::move(asm_commands)); } @@ -1388,13 +1390,9 @@ static V parse_annotation(Lexer& lex) { return createV(loc, name, kind, v_arg); } -static AnyV parse_function_declaration(Lexer& lex, const std::vector>& annotations) { +static AnyV parse_function_declaration(Lexer& lex, const std::vector>& annotations, bool is_contract_getter) { SrcLocation loc = lex.cur_location(); - bool is_contract_getter = lex.cur_str() == "get"; - lex.next(); - if (is_contract_getter && lex.tok() == tok_fun) { - lex.next(); // 'get f()' and 'get fun f()' both correct - } + lex.expect(tok_fun, "`fun`"); AnyTypeV receiver_type = nullptr; auto backup = lex.save_parsing_position(); @@ -1460,7 +1458,6 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(lex.cur_location()); lex.next(); - lex.expect(tok_semicolon, "`;`"); } else if (lex.tok() == tok_opbrace) { v_body = parse_func_body(lex); } else if (lex.tok() == tok_asm) { @@ -1558,14 +1555,12 @@ static V parse_struct_body(Lexer& lex) { SrcLocation loc = lex.cur_location(); std::vector fields; - if (lex.tok() != tok_semicolon) { // `struct A;` equal to `struct A {}` - lex.expect(tok_opbrace, "`{`"); + if (lex.tok() == tok_opbrace) { // `struct A` equal to `struct A {}` + lex.next(); while (lex.tok() != tok_clbrace) { fields.push_back(parse_struct_field(lex)); if (lex.tok() == tok_comma || lex.tok() == tok_semicolon) { lex.next(); - } else if (lex.tok() != tok_clbrace) { - lex.unexpected("`;` or `,`"); } } lex.expect(tok_clbrace, "`}`"); @@ -1700,7 +1695,7 @@ AnyV parse_src_file_to_ast(const SrcFile* file) { annotations.clear(); break; case tok_fun: - toplevel_declarations.push_back(parse_function_declaration(lex, annotations)); + toplevel_declarations.push_back(parse_function_declaration(lex, annotations, false)); annotations.clear(); break; case tok_struct: @@ -1715,8 +1710,9 @@ AnyV parse_src_file_to_ast(const SrcFile* file) { lex.error("`" + static_cast(lex.cur_str()) +"` is not supported yet"); case tok_identifier: - if (lex.cur_str() == "get") { // tok-level "get", contract getter - toplevel_declarations.push_back(parse_function_declaration(lex, annotations)); + if (lex.cur_str() == "get") { // top-level "get", contract getter + lex.next(); + toplevel_declarations.push_back(parse_function_declaration(lex, annotations, true)); annotations.clear(); break; } From 06e0544376a1866a7befd96617757c737e27a11d Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 4 Jul 2025 14:38:41 +0300 Subject: [PATCH 340/524] [Tolk] Gas optimization: replace ternary with CONDSEL when it's safe --- tolk-tester/tests/ternary-tests.tolk | 367 +++++++++++++++++++++++++++ tolk/builtins.cpp | 20 ++ tolk/pipe-ast-to-legacy.cpp | 28 ++ 3 files changed, 415 insertions(+) create mode 100644 tolk-tester/tests/ternary-tests.tolk diff --git a/tolk-tester/tests/ternary-tests.tolk b/tolk-tester/tests/ternary-tests.tolk new file mode 100644 index 0000000000..a1e88a8e84 --- /dev/null +++ b/tolk-tester/tests/ternary-tests.tolk @@ -0,0 +1,367 @@ +const TWO = 2; +global gTup: tuple; + +fun get1() { gTup.push(1); return 1; } +fun get2() { gTup.push(2); return 2; } + +struct Point { + x: int; + y: int; +} + +@noinline +fun Point.create0NoInline(): Point { + return { x: 0, y: 0 } +} + +fun Point.create0(): Point { + return { x: 0, y: 0 } +} + +@method_id(101) +fun test101(x: int) { + return (x ? 2 : 1, x == 5 ? TWO : 1); +} + +@method_id(102) +fun test102(x: int?) { + return x == null ? (0!) : ((x)); +} + +@method_id(103) +fun test103(x: bool) { + if (x ? -1 : +1) { + return x ? -2 : +2; + } + throw 123; +} + +@method_id(104) +fun test104(x: (int, int)) { + return x.0 ? true : null; +} + +@method_id(105) +fun test105(getX: bool) { + var p: Point = { x: 10, y: 20 }; + return getX ? p.x : p!.y; +} + +@noinline +fun helper106(glob: bool, t: tuple) { + return glob ? gTup : t; +} + +@method_id(106) +fun test106() { + gTup = createEmptyTuple(); + gTup.push(1); + return helper106(true, createEmptyTuple()); +} + +@noinline +fun helper107(t: (int, Point)) { + return t.0 ? t.0 : t.1.y; +} + +@method_id(107) +fun test107() { + return (helper107((1, {x: 10, y:20})), helper107((0, {x: 10, y: 20}))); +} + +@method_id(108) +fun test108(calcMin: bool) { + var cb = calcMin ? min : max; + return cb(10, 20); +} + +@method_id(109) +fun test109(bigCost: bool) { + return bigCost ? ton("0.05") : (ton("0.001")); +} + +@method_id(110) +fun test110(k: int) { + val c: [int, int, int] = [k, 6, 7]; + return c.0 > 0 ? c.1 : c.2; +} + +@method_id(111) +fun test111(g: bool) { + return g ? (1, 2) : (3, 4); +} + + +@method_id(200) +fun test200(x: int) { + return 10 > 3 ? 200 : 100; +} + +@method_id(201) +fun test201(x: int) { + return (x is slice) ? 200 : x; +} + +@method_id(202) +fun test202(x: int) { + x > 10 ? x : 0; + x < 10 ? x! : -10; + return x; +} + +@method_id(203) +fun test203() { + var xx = false; + if (!xx) { + return xx ? 100 : 200; + } + return 300; +} + + +@method_id(220) +fun test220(p: Point?, getNull: bool) { + if (p == null) { + return getNull ? p : 20; + } + return p.y; +} + +@method_id(221) +fun test221(c: (int, (int,int)?)) { + if (c.1 != null) { + return c.0 > 5 ? c.1.0 : c.1.1; + } + return c.0 > 5 ? 0 : c.1; +} + + +@method_id(300) +fun test300(x: int): tuple | int { + // no condsel, because more that 1 slot + return x > 0 ? x : gTup; +} + +@method_id(301) +fun test301(first: bool) { + var opt1 = (10, 20); + var opt2 = (30, 40); + // no condsel, because not 1 slot arguments + return first ? opt1 : opt2; +} + +@method_id(302) +fun test302(x: int?) { + // no condsel, because not trivial (unary operator) + return x == null ? 0 : -x; +} + +@method_id(303) +fun test303(first: bool) { + gTup = createEmptyTuple(); + // no condsel, because not trivial + return (first ? get1() : get2(), gTup); +} + +@method_id(304) +fun test304(y: bool) { + return y ? Point.create0NoInline().y : 5; +} + +@method_id(305) +fun test305(y: bool) { + return y ? Point.create0().y : 5; +} + + +fun main() {} + +/** +@testcase | 101 | 0 | 1 1 +@testcase | 101 | 5 | 2 2 +@testcase | 101 | 9 | 2 1 +@testcase | 102 | 8 | 8 +@testcase | 102 | null | 0 +@testcase | 103 | -1 | -2 +@testcase | 104 | 1 2 | -1 +@testcase | 104 | 0 2 | (null) +@testcase | 105 | -1 | 10 +@testcase | 106 | | [ 1 ] +@testcase | 107 | | 1 20 +@testcase | 108 | 0 | 20 +@testcase | 109 | -1 | 50000000 +@testcase | 110 | 5 | 6 +@testcase | 110 | -5 | 7 +@testcase | 111 | -1 | 1 2 +@testcase | 111 | 0 | 3 4 + +@testcase | 200 | 20 | 200 +@testcase | 201 | 5 | 5 +@testcase | 202 | 5 | 5 +@testcase | 203 | | 200 + +@testcase | 220 | null null 0 -1 | (null) +@testcase | 220 | null null 0 0 | 20 +@testcase | 220 | 9 8 123 5 | 8 +@testcase | 221 | 10 null null 0 | 0 +@testcase | 221 | 3 null null 0 | (null) +@testcase | 221 | 10 3 4 100 | 3 +@testcase | 221 | 3 3 4 100 | 4 + +@testcase | 300 | 5 | 5 1 +@testcase | 301 | -1 | 10 20 +@testcase | 302 | null | 0 +@testcase | 303 | -1 | 1 [ 1 ] +@testcase | 304 | -1 | 0 +@testcase | 304 | 0 | 5 +@testcase | 305 | 0 | 5 +@testcase | 305 | 0 | 5 + +@fif_codegen +""" + test101() PROC:<{ + DUP + 2 PUSHINT + 1 PUSHINT + CONDSEL + SWAP + 5 EQINT + 2 PUSHINT + 1 PUSHINT + CONDSEL + }> +""" + +@fif_codegen +""" + test102() PROC:<{ // x + DUP // x x + ISNULL // x '1 + 0 PUSHINT + ROT // '1 '3=0 x + CONDSEL // '2 + }> +""" + +@fif_codegen +""" + test103() PROC:<{ + -2 PUSHINT + 2 PUSHINT + CONDSEL + }> +""" + +@fif_codegen +""" + test105() PROC:<{ // getX + 10 PUSHINT // getX '3=10 + 20 PUSHINT // getX p.x=10 p.y=20 + CONDSEL // '5 + }> +""" + +@fif_codegen +""" + helper106() PROC:<{ + $gTup GETGLOB + SWAP + CONDSEL + }> +""" + +@fif_codegen +""" + helper107() PROC:<{ + NIP + s1 s(-1) PUXC + CONDSEL + }> +""" + +@fif_codegen +""" + test108() PROC:<{ + CONT:<{ + MIN + }> + CONT:<{ + MAX + }> + CONDSEL + 10 PUSHINT + 20 PUSHINT + ROT + 2 1 CALLXARGS + }> +""" + +@fif_codegen +""" + test109() PROC:<{ + 50000000 PUSHINT + 1000000 PUSHINT + CONDSEL + }> +""" + +@fif_codegen +""" + test200() PROC:<{ + DROP + 200 PUSHINT + }> +""" + +@fif_codegen +""" + test201() PROC:<{ + // '2 + }> +""" + +@fif_codegen +""" + test202() PROC:<{ + }> +""" + +@fif_codegen +""" + test203() PROC:<{ + 200 PUSHINT + }> +""" + +@fif_codegen +""" + test220() PROC:<{ + s3 POP + 0 EQINT + IFJMP:<{ + 20 PUSHINT + CONDSEL + }> + NIP + }> +""" + +@fif_codegen +""" + test304() PROC:<{ + IF:<{ + Point.create0NoInline() CALLDICT +""" + +@fif_codegen +""" + test305() PROC:<{ + IF:<{ + 0 PUSHINT + }>ELSE<{ + 5 PUSHINT + }> + }> +""" + +*/ diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 22d743753b..dc2795c898 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1004,6 +1004,23 @@ static AsmOp compile_throw_arg(std::vector& res, std::vector } } +// `x ? y : z` can be compiled as `CONDSEL` asm instruction if y and z are don't require evaluation +static AsmOp compile_ternary_as_condsel(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 1 && args.size() == 3); + VarDescr& cond = args[0]; // args = [ cond, when_true, when_false ] + if (cond.always_true()) { + cond.unused(); + args[2].unused(); + return AsmOp::Nop(loc); + } + if (cond.always_false()) { + cond.unused(); + args[1].unused(); + return AsmOp::Nop(loc); + } + return exec_op(loc, "CONDSEL", 3, 1); +} + static AsmOp compile_bool_const(std::vector& res, std::vector& args, SrcLocation loc, bool val) { tolk_assert(res.size() == 1 && args.empty()); VarDescr& r = res[0]; @@ -1434,6 +1451,9 @@ void define_builtins() { compile_fetch_varint, // not exposed to stdlib, used in auto-serialization FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, {}, {1, 0}); + define_builtin_func("__condsel", ParamsInt3, Int, nullptr, + compile_ternary_as_condsel, + 0); // compile-time only functions, evaluated essentially at compile-time, no runtime implementation // they are placed in stdlib and marked as `builtin` diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 3f525287bb..732dc3e180 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -409,6 +409,29 @@ static V calc_sink_leftmost_obj(V v) { return leftmost_obj->kind == ast_reference ? leftmost_obj->as() : nullptr; } +// ternary `x ? y : z` can be optimized to asm `CONDSEL` (not IF/ELSE), if y and z don't require evaluation; +// example when can: `cond ? 2 : null`, `x == null ? some_var : obj.field`; +// example when not: `cond ? f() : g()` and other non-trivial arguments +static bool is_ternary_arg_trivial_for_condsel(AnyExprV v, bool require_1slot = true) { + if (require_1slot && v->inferred_type->get_width_on_stack() != 1) { + return false; + } + if (v->kind == ast_int_const || v->kind == ast_string_const || v->kind == ast_bool_const || + v->kind == ast_null_keyword || v->kind == ast_reference) { + return true; + } + if (auto v_par = v->try_as()) { + return is_ternary_arg_trivial_for_condsel(v_par->get_expr(), require_1slot); + } + if (auto v_dot = v->try_as()) { + return is_ternary_arg_trivial_for_condsel(v_dot->get_obj(), false); + } + if (auto v_cast = v->try_as()) { + return is_ternary_arg_trivial_for_condsel(v_cast->get_expr(), require_1slot); + } + return false; +} + static std::vector> pre_compile_tensor_inner(CodeBlob& code, const std::vector& args, const TypeDataTensor* tensor_target_type, LValContext* lval_ctx) { @@ -1303,6 +1326,11 @@ static std::vector process_ternary_operator(V v code.emplace_back(v->get_when_true()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_true(), code, v->inferred_type)); } else if (v->get_cond()->is_always_false) { code.emplace_back(v->get_when_false()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_false(), code, v->inferred_type)); + } else if (v->inferred_type->get_width_on_stack() == 1 && is_ternary_arg_trivial_for_condsel(v->get_when_true()) && is_ternary_arg_trivial_for_condsel(v->get_when_false())) { + std::vector ir_true = pre_compile_expr(v->get_when_true(), code, v->inferred_type); + std::vector ir_false = pre_compile_expr(v->get_when_false(), code, v->inferred_type); + std::vector condsel_args = { cond[0], ir_true[0], ir_false[0] }; + code.emplace_back(v->loc, Op::_Call, rvect, std::move(condsel_args), lookup_function("__condsel")); } else { Op& if_op = code.emplace_back(v->loc, Op::_If, cond); code.push_set_cur(if_op.block0); From a045893afb8bc2a2d034dd0fe0eb537407540d11 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 4 Jul 2025 13:13:40 +0300 Subject: [PATCH 341/524] [Tolk] Peephole optimization: SDSKIPFIRST to LDU+NIP It's a bit better for gas usage, always --- tolk-tester/tests/cells-slices.tolk | 4 ++-- tolk-tester/tests/lazy-load-tests.tolk | 20 ++++++++++---------- tolk-tester/tests/pack-unpack-1.tolk | 4 ++-- tolk/optimize.cpp | 9 +++++++-- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 8fee091b84..6a298d2348 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -401,8 +401,8 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ test21() PROC:<{ - 14 PUSHINT - SDSKIPFIRST + 14 LDU + NIP }> """ diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index 7a5643a548..72083fc85f 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -1741,8 +1741,8 @@ fun main() { """ demo103() PROC:<{ // x{0102} PUSHSLICE // lazyS - 8 PUSHINT - SDSKIPFIRST // lazyS + 8 LDU + NIP // lazyS 8 LDI // '10 lazyS DROP // p.y """ @@ -1753,8 +1753,8 @@ fun main() { // lazyS PUSHNULL // lazyS x SWAP // x lazyS - 8 PUSHINT - SDSKIPFIRST // x lazyS + 8 LDU + NIP // x lazyS 1 LDU // x '18 lazyS """ @@ -1762,8 +1762,8 @@ fun main() { """ demo109() PROC:<{ // x{8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_} PUSHSLICE // lazyS - 65 PUSHINT - SDSKIPFIRST // lazyS + 65 LDU + NIP // lazyS 256 LDU // '13 lazyS DROP // st.publicKey }> @@ -1821,8 +1821,8 @@ fun main() { DROP -1 PUSHINT }> - 8 PUSHINT - SDSKIPFIRST + 8 LDU + NIP 8 LDI DROP }> @@ -1962,8 +1962,8 @@ fun main() { THROWANY }> 32 LDU - 8 PUSHINT - SDSKIPFIRST + 8 LDU + NIP 8 LDI DROP OVER diff --git a/tolk-tester/tests/pack-unpack-1.tolk b/tolk-tester/tests/pack-unpack-1.tolk index 68ce0bc107..406307cfd7 100644 --- a/tolk-tester/tests/pack-unpack-1.tolk +++ b/tolk-tester/tests/pack-unpack-1.tolk @@ -112,8 +112,8 @@ fun main(c: cell) { 64 STI // b ENDC // c CTOS // s - 64 PUSHINT - SDSKIPFIRST // s + 64 LDU + NIP // s SBITS // '17 }> """ diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index c8fd8fc967..dab931d84f 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -231,8 +231,13 @@ bool Optimizer::detect_rewrite_MY_skip_bits() { p_ = n_merged; q_ = 2; - oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, td::make_refint(total_skip_bits))); - oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, "SDSKIPFIRST")); + if (total_skip_bits <= 256) { + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, std::to_string(total_skip_bits) + " LDU")); + oq_[1] = std::make_unique(AsmOp::Pop(op_[0]->loc, 1)); + } else { + oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, td::make_refint(total_skip_bits))); + oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, "SDSKIPFIRST")); + } return true; } From b297b6e48724b860172acff497e16045b50514fa Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 4 Jul 2025 16:33:03 +0300 Subject: [PATCH 342/524] [Tolk] Peephole optimization: LD+DROP to PLD Peephole optimizations: - `N LDU` + `DROP` -> `N PLDU` - same for LDI -> PLDI, LDREF -> PLDREF, etc. --- tolk-tester/tests/asm-arg-order.tolk | 5 +-- tolk-tester/tests/handle-msg-3.tolk | 3 +- tolk-tester/tests/lazy-algo-tests.tolk | 3 +- tolk-tester/tests/lazy-load-tests.tolk | 51 +++++++++----------------- tolk-tester/tests/null-keyword.tolk | 3 +- tolk-tester/tests/pack-unpack-6.tolk | 3 +- tolk-tester/tests/pack-unpack-7.tolk | 3 +- tolk-tester/tests/some-tests-3.tolk | 2 +- tolk/optimize.cpp | 35 ++++++++++++++++++ tolk/tolk.h | 1 + 10 files changed, 61 insertions(+), 48 deletions(-) diff --git a/tolk-tester/tests/asm-arg-order.tolk b/tolk-tester/tests/asm-arg-order.tolk index 1558a3c721..c42c1f9588 100644 --- a/tolk-tester/tests/asm-arg-order.tolk +++ b/tolk-tester/tests/asm-arg-order.tolk @@ -269,12 +269,11 @@ fun main() { test34() PROC:<{ x{020a} PUSHSLICE 8 LDU - 8 LDU - DROP + 8 PLDU SWAP 1 ADDCONST MUL }> """ -@code_hash 88343225778124778743440967192570181022747908733235053033508198051414837052492 +@code_hash 38717859169035491454494966803813683158800226121601078320213761069394202390827 */ diff --git a/tolk-tester/tests/handle-msg-3.tolk b/tolk-tester/tests/handle-msg-3.tolk index 78bb301ddd..41d3c1734b 100644 --- a/tolk-tester/tests/handle-msg-3.tolk +++ b/tolk-tester/tests/handle-msg-3.tolk @@ -23,8 +23,7 @@ fun onInternalMessage(in: InMessage) { IFJMP:<{ 32 LDU NIP - 32 LDU - DROP + 32 PLDU THROWANY }> // in.body }> diff --git a/tolk-tester/tests/lazy-algo-tests.tolk b/tolk-tester/tests/lazy-algo-tests.tolk index 27214077b3..ac8e9fcd53 100644 --- a/tolk-tester/tests/lazy-algo-tests.tolk +++ b/tolk-tester/tests/lazy-algo-tests.tolk @@ -1498,8 +1498,7 @@ fun main() { x{0102} PUSHSLICE 8 LDI NIP - 8 LDI - DROP + 8 PLDI }> """ */ diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index 72083fc85f..3e6a67befc 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -1732,8 +1732,7 @@ fun main() { """ demo101() PROC:<{ // x{0102} PUSHSLICE // lazyS - 8 LDI // '9 lazyS - DROP // p.x + 8 PLDI // p.x }> """ @@ -1743,8 +1742,7 @@ fun main() { x{0102} PUSHSLICE // lazyS 8 LDU NIP // lazyS - 8 LDI // '10 lazyS - DROP // p.y + 8 PLDI // p.y """ @fif_codegen @@ -1764,8 +1762,7 @@ fun main() { x{8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_} PUSHSLICE // lazyS 65 LDU NIP // lazyS - 256 LDU // '13 lazyS - DROP // st.publicKey + 256 PLDU // st.publicKey }> """ @@ -1775,13 +1772,11 @@ fun main() { // lazyS x{01} SDBEGINSQ IF:<{ - 8 LDI - DROP + 8 PLDI }>ELSE<{ x{02} SDBEGINSQ IF:<{ - 8 LDI - DROP + 8 PLDI NEGATE }>ELSE<{ x{03} SDBEGINSQ @@ -1793,8 +1788,7 @@ fun main() { IFNOTJMP:<{ 63 THROW }> - 32 LDI - DROP + 32 PLDI }> }> }> @@ -1807,8 +1801,7 @@ fun main() { x{010f} PUSHSLICE // lazyS x{01} SDBEGINSQ // lazyS '6 134 THROWIFNOT // lazyS - 8 LDI // '11 lazyS - DROP // cc + 8 PLDI // cc }> """ @@ -1823,8 +1816,7 @@ fun main() { }> 8 LDU NIP - 8 LDI - DROP + 8 PLDI }> """ @@ -1832,8 +1824,7 @@ fun main() { """ demo146() PROC:<{ x{0000000} PUSHSLICE - LDOPTREF - DROP + PLDOPTREF ISNULL }> """ @@ -1844,8 +1835,7 @@ fun main() { x{0102} PUSHSLICE DUP 8 LDI - 8 LDI - DROP + 8 PLDI s2 PUSH NEWC STSLICE @@ -1866,8 +1856,7 @@ fun main() { demo203() PROC:<{ // x{010203040506070809} PUSHSLICE // lazyS DUP // '14 lazyS - 8 LDU // '14 '16 lazyS - DROP // '14 o.f1 + 8 PLDU // '14 o.f1 1 EQINT // '14 '18 IFJMP:<{ // '14 NEWC // '14 b @@ -1889,8 +1878,7 @@ fun main() { 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 lazyS 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 lazyS 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 o.f8 lazyS - 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 o.f8 '32 lazyS - DROP // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 o.f8 o.f9 + 8 PLDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 o.f8 o.f9 """ @fif_codegen @@ -1898,8 +1886,7 @@ fun main() { demo215() PROC:<{ x{0102} PUSHSLICE 8 LDI - 8 LDI - DROP + 8 PLDI SWAP NEWC 8 STI @@ -1928,8 +1915,7 @@ fun main() { 1 LDI // '13 lazyS NIP // lazyS 320 PUSHINT // lazyS '14=320 - LDSLICEX // '15 lazyS - DROP // '15 + PLDSLICEX // '15 PUSHNULL // '15 st.extensions 0 PUSHINT NEWC @@ -1947,8 +1933,7 @@ fun main() { // lazyS x{12345678} SDBEGINSQ IF:<{ - 32 LDI - DROP + 32 PLDI 10 ADDCONST }>ELSE<{ x{23456789} SDBEGINSQ @@ -1964,8 +1949,7 @@ fun main() { 32 LDU 8 LDU NIP - 8 LDI - DROP + 8 PLDI OVER 100 GTINT IF:<{ @@ -2011,8 +1995,7 @@ fun main() { """ demo411() PROC:<{ // s // lazyS - 1 LDU // '10 lazyS - DROP // '10 + 1 PLDU // '10 IFJMP:<{ // DUMPSTK // }> // diff --git a/tolk-tester/tests/null-keyword.tolk b/tolk-tester/tests/null-keyword.tolk index c36a66c26b..c21da1d0e8 100644 --- a/tolk-tester/tests/null-keyword.tolk +++ b/tolk-tester/tests/null-keyword.tolk @@ -122,8 +122,7 @@ fun main() { """ test7() PROC:<{ ... - LDOPTREF // b '9 '8 - DROP // b c + PLDOPTREF // b c ISNULL // b '11 10 MULCONST // b '13 SWAP // '13 b diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk index 71f6ab0b40..47c0b062d6 100644 --- a/tolk-tester/tests/pack-unpack-6.tolk +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -138,8 +138,7 @@ RETALT CTOS x{01} SDBEGINSQ 63 THROWIFNOT - 8 LDI - DROP + 8 PLDI }> """ diff --git a/tolk-tester/tests/pack-unpack-7.tolk b/tolk-tester/tests/pack-unpack-7.tolk index 07407ff91f..60e9c2b5cf 100644 --- a/tolk-tester/tests/pack-unpack-7.tolk +++ b/tolk-tester/tests/pack-unpack-7.tolk @@ -175,8 +175,7 @@ fun main() { test3() PROC:<{ x{0102} PUSHSLICE 8 LDU - 8 LDI - DROP + 8 PLDI }> """ */ diff --git a/tolk-tester/tests/some-tests-3.tolk b/tolk-tester/tests/some-tests-3.tolk index 8c53e33bf3..5c99c4123d 100644 --- a/tolk-tester/tests/some-tests-3.tolk +++ b/tolk-tester/tests/some-tests-3.tolk @@ -31,5 +31,5 @@ fun f(cs: slice) { @testcase | 101 | x{000102030405060708090a0b0c0d0e0f10111213} | 6 @testcase | 0 | x{000102030405060708090a0b0c0d0e0f10111213} | 0 1 2 3 -@code_hash 58474889199998908444151060994149070836199913191952040273624197630531731101157 +@code_hash 110094925152266492480367791654603701827254965953641340034522027582587727126150 */ diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index dab931d84f..732d04baf0 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -264,6 +264,40 @@ bool Optimizer::detect_rewrite_NEWC_PUSH_STUR() { return true; } +// pattern `N LDU` + `DROP` -> `N PLDU` (common after loading the last field manually or by `lazy`); +// the same for LDI -> PLDI, LDREF -> PLDREF, etc. +bool Optimizer::detect_rewrite_LDxx_DROP() { + bool second_drop = pb_ > 1 && op_[1]->is_pop() && op_[1]->a == 0; + if (!second_drop || !op_[0]->is_custom()) { + return false; + } + + static const char* ends_with[] = { " LDI", " LDU", " LDBITS"}; + static const char* repl_with[] = {" PLDI", " PLDU", " PLDBITS"}; + static const char* equl_to[] = { "LDREF", "LDDICT", "LDOPTREF", "LDSLICEX"}; + static const char* repl_to[] = {"PLDREF", "PLDDICT", "PLDOPTREF", "PLDSLICEX"}; + + std::string_view f = op_[0]->op; + for (size_t i = 0; i < std::size(ends_with); ++i) { + if (f.ends_with(ends_with[i])) { + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, op_[0]->op.substr(0, f.rfind(' ')) + repl_with[i], 0, 1)); + return true; + } + } + for (size_t i = 0; i < std::size(equl_to); ++i) { + if (f == equl_to[i]) { + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, repl_to[i], 0, 1)); + return true; + } + } + + return false; +} + bool Optimizer::is_push_const(int* i, int* c) const { return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_push_const(i, c); @@ -685,6 +719,7 @@ bool Optimizer::find_at_least(int pb) { (is_xchg_xchg(&i, &j, &k, &l) && rewrite(AsmOp::Xchg(loc, i, j), AsmOp::Xchg(loc, k, l))) || detect_rewrite_big_THROW() || detect_rewrite_MY_store_int() || detect_rewrite_MY_skip_bits() || detect_rewrite_NEWC_PUSH_STUR() || + detect_rewrite_LDxx_DROP() || (!(mode_ & 1) && ((is_rot() && rewrite(AsmOp::Custom(loc, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(loc, "-ROT", 3, 3))) || (is_2dup() && rewrite(AsmOp::Custom(loc, "2DUP", 2, 4))) || diff --git a/tolk/tolk.h b/tolk/tolk.h index 9bd21c99d6..45b553d8a8 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -902,6 +902,7 @@ struct Optimizer { bool detect_rewrite_MY_store_int(); bool detect_rewrite_MY_skip_bits(); bool detect_rewrite_NEWC_PUSH_STUR(); + bool detect_rewrite_LDxx_DROP(); AsmOpConsList extract_code(); }; From 55c2530c826f8f6fcc620297b1f7ae158b8875a1 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 4 Jul 2025 17:56:01 +0300 Subject: [PATCH 343/524] [Tolk] Peephole optimization: SWAP+any to any/anyR - SWAP + EQUAL -> EQUAL - same for other symmetric operators (NEQ, MUL, etc.) - SWAP + xxx PUSHINT + 32 STUR -> xxx PUSHINT + ROT + 32 STU - SWAP + STSLICER => STSLICE and vice versa - same for other storing operators (STU, STB, STREF, etc.) - SWAP + LESS/LEQ => GREATER/GEQ --- tolk-tester/tests/a-tests.tolk | 2 +- .../tests/allow-post-modification.tolk | 2 +- tolk-tester/tests/calls-tests.tolk | 63 ++++++ tolk-tester/tests/cells-slices.tolk | 213 +++++++++++++++++- tolk-tester/tests/inline-tests.tolk | 4 +- tolk-tester/tests/op-priority.tolk | 9 +- tolk-tester/tests/pack-unpack-6.tolk | 1 - tolk-tester/tests/some-tests-3.tolk | 2 +- tolk-tester/tests/strings-tests.tolk | 2 +- tolk-tester/tests/try-catch-tests.tolk | 2 +- tolk/asmops.cpp | 2 + tolk/optimize.cpp | 77 +++++++ tolk/tolk.h | 3 + 13 files changed, 362 insertions(+), 20 deletions(-) diff --git a/tolk-tester/tests/a-tests.tolk b/tolk-tester/tests/a-tests.tolk index a122ca39dc..fb553f7e5b 100644 --- a/tolk-tester/tests/a-tests.tolk +++ b/tolk-tester/tests/a-tests.tolk @@ -79,5 +79,5 @@ fun main(): int { method_id | in | out @testcase | 0 | | 31415926535897932384626433832795028841971693993751058209749445923078164 -@code_hash 103119459254804391561326789266022754176632268076661379738543746273668564328534 +@code_hash 85818713521853656486584648797214567489479452958868213061669627117427586818086 */ diff --git a/tolk-tester/tests/allow-post-modification.tolk b/tolk-tester/tests/allow-post-modification.tolk index a74b2e1ecd..f825d12f81 100644 --- a/tolk-tester/tests/allow-post-modification.tolk +++ b/tolk-tester/tests/allow-post-modification.tolk @@ -164,5 +164,5 @@ fun main() { test_assign_tensor_global() PROC:<{ // x.0 x.1 """ -@code_hash 6737917279814799680932710799951154408447028229503449454536845752635763933556 +@code_hash 77620375659834063567341916666636335765114427483375544778177892407598637223147 */ diff --git a/tolk-tester/tests/calls-tests.tolk b/tolk-tester/tests/calls-tests.tolk index a03cd5a301..9405c9ead0 100644 --- a/tolk-tester/tests/calls-tests.tolk +++ b/tolk-tester/tests/calls-tests.tolk @@ -159,6 +159,26 @@ fun test9() { return int(10) + int(5).plus1() + int.create0() + int.create0().plus1(); } +@noinline fun cmp1(a: int, b: int) { return a == b } +@noinline fun cmp2(a: int, b: int) { return b == a } +@noinline fun cmp3(a: int, b: int) { return a != b } +@noinline fun cmp4(a: int, b: int) { return b != a } + +@method_id(110) +fun test10(a: int, b: int) { + return b - a +} + +@noinline fun leq1(a: int, b: int) { return b < a } +@noinline fun leq2(a: int, b: int) { return b <= a } +@noinline fun leq3(a: int, b: int) { return b > a } +@noinline fun leq4(a: int, b: int) { return b >= a } + +@method_id(111) +fun test11(a: int, b: int) { + return ((b < a), (b <= a), (b > a), (b >= a)); +} + fun main() {} /** @@ -171,4 +191,47 @@ fun main() {} @testcase | 107 | | 16 5 @testcase | 108 | | [ 1 2 3 ] [ 5 8 1 ] @testcase | 109 | | 17 +@testcase | 110 | 5 8 | 3 +@testcase | 111 | 5 8 | 0 0 -1 -1 +@testcase | 111 | 7 7 | 0 -1 0 -1 + +@fif_codegen +""" + cmp1() PROC:<{ + EQUAL + }> + cmp2() PROC:<{ + EQUAL + }> + cmp3() PROC:<{ + NEQ + }> + cmp4() PROC:<{ + NEQ + }> +""" + +@fif_codegen +""" + test10() PROC:<{ + SUBR + }> +""" + +@fif_codegen +""" + leq1() PROC:<{ + GREATER + }> + leq2() PROC:<{ + GEQ + }> + leq3() PROC:<{ + LESS + }> + leq4() PROC:<{ + LEQ + }> +""" + */ diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 6a298d2348..2963177dec 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -318,6 +318,121 @@ fun test24(uns: bool) { } } +fun builder.my_storeRef(mutate self, value: cell): self + asm(self value) "STREFR"; + +fun builder.my_storeAddress(mutate self, value: address): self + asm(value self) "STSLICE"; + +fun builder.my_storeBuilder(mutate self, value: builder): self + asm(value self) "STB"; + +fun my_PUSHINT_SWAP_STU(mutate b: builder): void + asm "5 PUSHINT" "SWAP" "8 STU"; + + +@noinline +fun demo30(value: cell, dest: builder) { + // SWAP + STREFR => STREF + return dest.my_storeRef(value); +} + +@noinline +fun demo31(dest: builder, value: cell) { + // SWAP + STREF => STREFR + return dest.storeRef(value); +} + +@noinline +fun demo32(dest: builder, value: builder) { + // SWAP + STB => STBR + return dest.my_storeBuilder(value); +} + +@noinline +fun demo33(value: builder, dest: builder) { + // SWAP + STBR => STB + return dest.storeBuilder(value); +} + +@noinline +fun demo34(dest: builder, value: address) { + // SWAP + STSLICE => STSLICER + return dest.my_storeAddress(value); +} + +@noinline +fun demo35(value: slice, dest: builder) { + // SWAP + STSLICER => STSLICE + return dest.storeSlice(value); +} + +@noinline +fun demo36(x: int) { + var b = beginCell(); + if (x > 0) { + // inside IF: N PUSHINT + SWAP + L STU => N PUSHINT + L STUR + my_PUSHINT_SWAP_STU(mutate b); + } + return b; +} + +@noinline +fun demo37(op: int32, u: int8 | int256) { + var b = beginCell(); + // b.storeUint(input.op, 32); + if (u is int8) { + // inside IF: SWAP + 0 PUSHINT + 3 STUR => 0 PUSHINT + 3 STU + b.storeUint(0, 3); + b.storeUint(u, 8); + } + return b; +} + +@method_id(130) +fun test30() { + return (demo37(10, 88 as int8), demo36(50)); +} + +@method_id(131) +fun test31() { + var s = createAddressNone(); + var dest = beginCell().storeUint(3, 32); + assert(demo35(s as slice, dest).endCell().hash() == demo34(dest, s).endCell().hash()) throw 300; + var r1 = demo35(s as slice, dest).endCell().hash(); + var b = beginCell().storeUint(0xFF, 16); + assert(demo33(b, dest).endCell().hash() == demo32(dest, b).endCell().hash()) throw 301; + var r2 = demo33(b, dest).endCell().hash(); + var r = createEmptyCell(); + assert(demo30(r, dest).endCell().hash() == demo31(dest, r).endCell().hash()) throw 302; + var r3 = demo30(r, dest).endCell().hash(); + return [r1 & 0xFFFFFFFF, r2 & 0xFFFFFFFF, r3 & 0xFFFFFFFF]; +} + +@method_id(132) +fun test32(p: int) { + var b = beginCell(); + var x = p + 10; + b.storeUint(x, 32); // SWAP + n STU => n STUR + return b; +} + +@method_id(133) +fun test33(p: int) { + var b = beginCell(); + var x = p + 10; + b.storeInt(x, 8); // SWAP + n STI => n STIR + return b; +} + +@method_id(134) +fun test34(p: int, n: int) { + var b = beginCell(); + var x = p + 10; + b.storeInt(x, n); // no changes, it's STIX + return b; +} + fun main(): int { return 0; } @@ -347,6 +462,11 @@ fun main(): int { @testcase | 123 | 0 | 255 @testcase | 124 | -1 | 8 @testcase | 124 | 0 | 10 +@testcase | 130 | | BC{00030b10} BC{000205} +@testcase | 131 | | [ 225345949 901620178 1560646286 ] +@testcase | 132 | 0 | BC{00080000000a} +@testcase | 133 | 0 | BC{00020a} +@testcase | 134 | 0 8 | BC{00020a} We test that consequtive storeInt/storeUint with constants are joined into a single number @@ -378,15 +498,13 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ test19() PROC:<{ - NEWC 123 PUSHINT - SWAP + NEWC 4 STI 16 PUSHPOW2DEC 16 STUR -1 PUSHINT - SWAP - 8 STI + 8 STIR }> """ @@ -411,9 +529,9 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin test22() PROC:<{ NEWC // '2 NEWC // b1 b2 - SWAP // b2 b1 2056 PUSHINT - 24 STUR // b2 b1 + ROT + 24 STU // b2 b1 SWAP // b1 b2 10141514286835656557042350424064 PUSHINTX 132 STUR // b1 b2 @@ -437,4 +555,87 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin }> """ +@fif_codegen +""" + demo30() PROC:<{ // value dest + STREF // dest + }> + demo31() PROC:<{ // dest value + STREFR // dest + }> + demo32() PROC:<{ // dest value + STBR // dest + }> + demo33() PROC:<{ // value dest + STB // dest + }> + demo34() PROC:<{ // dest value + STSLICER // dest + }> + demo35() PROC:<{ // value dest + STSLICE // dest + }> +""" + +@fif_codegen +""" + demo36() PROC:<{ // x + NEWC // x b + SWAP // b x + 0 GTINT // b '4 + IF:<{ // b + 5 PUSHINT + 8 STUR // b + }> // b + }> +""" + +@fif_codegen +""" + demo37() PROC:<{ + NEWC + s3 POP + 42 EQINT + IF:<{ + 0 PUSHINT + ROT + 3 STU + 8 STU // b + }>ELSE<{ + DROP // b + }> + }> +""" + +@fif_codegen +""" + test32() PROC:<{ // p + NEWC // p b + SWAP // b p + 10 ADDCONST // b x + 32 STUR // b + }> +""" + +@fif_codegen +""" + test33() PROC:<{ // p + NEWC // p b + SWAP // b p + 10 ADDCONST // b x + 8 STIR // b + }> +""" + +@fif_codegen +""" + test34() PROC:<{ + NEWC + s0 s2 XCHG + 10 ADDCONST + -ROT + STIX + }> +""" + */ diff --git a/tolk-tester/tests/inline-tests.tolk b/tolk-tester/tests/inline-tests.tolk index 243652b462..fc48ddaa34 100644 --- a/tolk-tester/tests/inline-tests.tolk +++ b/tolk-tester/tests/inline-tests.tolk @@ -471,9 +471,9 @@ fun usedIn10ButDeclaredBelow(x: int) { NEWC 32 STU // '3 25 PUSHINT // '3 '9 - SWAP // x self 107374182425 PUSHINT - 64 STUR // x '3 + ROT + 64 STU // x '3 SWAP // '3 x }> """ diff --git a/tolk-tester/tests/op-priority.tolk b/tolk-tester/tests/op-priority.tolk index e635d54234..79c8376934 100644 --- a/tolk-tester/tests/op-priority.tolk +++ b/tolk-tester/tests/op-priority.tolk @@ -96,23 +96,20 @@ fun main() { @fif_codegen """ unary_minus_1() PROC:<{ // a b c - -ROT // c a b + s0 s2 XCHG ADD // c '3 NEGATE // c '4 - SWAP // '4 c MUL // '5 }> unary_minus_2() PROC:<{ // a b c - -ROT // c a b + s0 s2 XCHG ADD // c '3 NEGATE // c '4 - SWAP // '4 c MUL // '5 }> unary_minus_3() PROC:<{ // a b c - -ROT // c a b + s0 s2 XCHG ADD // c '3 - SWAP // '3 c MUL // '4 NEGATE // '5 }> diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk index 47c0b062d6..364d026a27 100644 --- a/tolk-tester/tests/pack-unpack-6.tolk +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -212,7 +212,6 @@ IF:<{ 140 PUSHINT }> 139 PUSHINT - SWAP EQUAL }> """ diff --git a/tolk-tester/tests/some-tests-3.tolk b/tolk-tester/tests/some-tests-3.tolk index 5c99c4123d..7208295fe4 100644 --- a/tolk-tester/tests/some-tests-3.tolk +++ b/tolk-tester/tests/some-tests-3.tolk @@ -31,5 +31,5 @@ fun f(cs: slice) { @testcase | 101 | x{000102030405060708090a0b0c0d0e0f10111213} | 6 @testcase | 0 | x{000102030405060708090a0b0c0d0e0f10111213} | 0 1 2 3 -@code_hash 110094925152266492480367791654603701827254965953641340034522027582587727126150 +@code_hash 110494322917670257289501183090765859207486616122530510808569169535563021169176 */ diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index 863a8aeff4..69d0fb2d51 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -78,5 +78,5 @@ fun test1() { @testcase | 0 | | 0 @testcase | 101 | | [ 65 66 67 68 ] -@code_hash 15952581052628192793433894459656689089299796554419234117286198843831925928625 +@code_hash 33596158548631231081906594921712427436857221384530839008514203474433205250299 */ diff --git a/tolk-tester/tests/try-catch-tests.tolk b/tolk-tester/tests/try-catch-tests.tolk index 04e32531cb..136c606d2a 100644 --- a/tolk-tester/tests/try-catch-tests.tolk +++ b/tolk-tester/tests/try-catch-tests.tolk @@ -294,7 +294,7 @@ fun main() { @testcase | 112 | 5 | 138 @testcase | 113 | | 2048 -@code_hash 91375323652563311931660098165979913104421063599680251769401655783153201389325 +@code_hash 3924051084946061509165180039638830343498397714643311802900659572522552334228 @fif_codegen """ diff --git a/tolk/asmops.cpp b/tolk/asmops.cpp index 44887633bb..3dc39239b4 100644 --- a/tolk/asmops.cpp +++ b/tolk/asmops.cpp @@ -223,6 +223,8 @@ AsmOp AsmOp::Parse(SrcLocation loc, const std::string& custom_op) { return AsmOp::Push(loc, 0); } else if (custom_op == "OVER") { return AsmOp::Push(loc, 1); + } else if (custom_op.ends_with(" PUSHINT") && custom_op[0] >= '1' && custom_op[0] <= '9' && custom_op.find(' ') == custom_op.rfind(' ')) { + return AsmOp::IntConst(loc, td::string_to_int256(custom_op.substr(0, custom_op.find(' ')))); } else { return AsmOp::Custom(loc, custom_op); } diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index 732d04baf0..d9e7608beb 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -298,6 +298,82 @@ bool Optimizer::detect_rewrite_LDxx_DROP() { return false; } +// pattern `SWAP` + `EQUAL` -> `EQUAL` +// and other symmetric operators: NEQ, MUL, etc. +bool Optimizer::detect_rewrite_SWAP_symmetric() { + bool first_swap = op_[0]->is_swap(); + if (!first_swap || pb_ < 2 || !op_[1]->is_custom()) { + return false; + } + std::string_view n = op_[1]->op; + bool next_symmetric = n == "EQUAL" || n == "NEQ" || n == "SDEQ" || n == "AND" || n == "OR" + || n == "ADD" || n == "MUL" || n == "MIN" || n == "MAX"; + if (!next_symmetric) { + return false; + } + + p_ = 2; + q_ = 1; + oq_[0] = std::move(op_[1]); + return true; +} + +// pattern `SWAP` + `xxx PUSHINT` + `32 STUR` -> `xxx PUSHINT` + `ROT` + `32 STU` +bool Optimizer::detect_rewrite_SWAP_PUSH_STUR() { + bool first_swap = op_[0]->is_swap(); + if (!first_swap || pb_ < 3) { + return false; + } + bool next_push = op_[1]->is_const() && op_[1]->op.ends_with(" PUSHINT"); + if (!next_push) { + return false; + } + bool next_stu_r = op_[2]->is_custom() && (op_[2]->op.ends_with(" STUR") || op_[2]->op.ends_with(" STIR")); + if (!next_stu_r) { + return false; + } + + p_ = 3; + q_ = 3; + oq_[0] = std::move(op_[1]); + oq_[1] = std::make_unique(AsmOp::BlkSwap(oq_[0]->loc, 1, 2)); // ROT + oq_[2] = std::make_unique(AsmOp::Custom(oq_[0]->loc, op_[2]->op.substr(0, op_[2]->op.size() - 1), 1, 1)); + return true; +} + +// pattern `SWAP` + `STSLICER` -> `STSLICE` and vice versa: `SWAP` + `STSLICE` => `STSLICER`; +// same for `STB` / `STREF` / `n STU` / `n STI` +bool Optimizer::detect_rewrite_SWAP_STxxxR() { + bool first_swap = op_[0]->is_swap(); + if (!first_swap || pb_ < 2 || !op_[1]->is_custom()) { + return false; + } + + static const char* ends_with[] = {" STU", " STI", " STUR", " STIR"}; + static const char* repl_with[] = {" STUR", " STIR", " STU", " STI"}; + static const char* equl_to[] = {"STSLICE", "STSLICER", "STB", "STBR", "SUB", "SUBR", "STREF", "STREFR", "LESS", "LEQ", "GREATER", "GEQ"}; + static const char* repl_to[] = {"STSLICER", "STSLICE", "STBR", "STB", "SUBR", "SUB", "STREFR", "STREF", "GREATER", "GEQ", "LESS", "LEQ"}; + + std::string_view f = op_[1]->op; + for (size_t i = 0; i < std::size(ends_with); ++i) { + if (f.ends_with(ends_with[i])) { + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, op_[1]->op.substr(0, f.rfind(' ')) + repl_with[i], 1, 1)); + return true; + } + } + for (size_t i = 0; i < std::size(equl_to); ++i) { + if (f == equl_to[i]) { + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, repl_to[i], 0, 1)); + return true; + } + } + + return false; +} bool Optimizer::is_push_const(int* i, int* c) const { return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_push_const(i, c); @@ -720,6 +796,7 @@ bool Optimizer::find_at_least(int pb) { detect_rewrite_big_THROW() || detect_rewrite_MY_store_int() || detect_rewrite_MY_skip_bits() || detect_rewrite_NEWC_PUSH_STUR() || detect_rewrite_LDxx_DROP() || + detect_rewrite_SWAP_symmetric() || detect_rewrite_SWAP_PUSH_STUR() || detect_rewrite_SWAP_STxxxR() || (!(mode_ & 1) && ((is_rot() && rewrite(AsmOp::Custom(loc, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(loc, "-ROT", 3, 3))) || (is_2dup() && rewrite(AsmOp::Custom(loc, "2DUP", 2, 4))) || diff --git a/tolk/tolk.h b/tolk/tolk.h index 45b553d8a8..32c6843f8a 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -903,6 +903,9 @@ struct Optimizer { bool detect_rewrite_MY_skip_bits(); bool detect_rewrite_NEWC_PUSH_STUR(); bool detect_rewrite_LDxx_DROP(); + bool detect_rewrite_SWAP_symmetric(); + bool detect_rewrite_SWAP_PUSH_STUR(); + bool detect_rewrite_SWAP_STxxxR(); AsmOpConsList extract_code(); }; From eb8cdfb5414ab8794342f23e7abe453102be42d3 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 4 Jul 2025 23:20:56 +0300 Subject: [PATCH 344/524] [Tolk] Peephole optimization: PUSHINT+STU to STSLICECONST Now, `b.storeInt(123, 32)` generates not `123 PUSHINT; SWAP; STI`, but `x{...} STSLICECONST`. Although, for certain conditions, still generate STUR/STUR targeting smaller bytecode. --- tolk-tester/tests/cells-slices.tolk | 34 ++++++++-------------- tolk-tester/tests/constants-tests.tolk | 2 +- tolk-tester/tests/inline-tests.tolk | 11 +++---- tolk-tester/tests/lazy-load-tests.tolk | 6 ++-- tolk-tester/tests/pack-unpack-1.tolk | 3 +- tolk-tester/tests/strings-tests.tolk | 2 +- tolk/optimize.cpp | 40 ++++++++++++++++++++------ 7 files changed, 53 insertions(+), 45 deletions(-) diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 2963177dec..57a92f03b6 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -473,26 +473,23 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ test6() PROC:<{ - 18446744082299486211 PUSHINT NEWC - 96 STU // '2 + x{000000010000000200000003} STSLICECONST ENDC // '11 """ @fif_codegen """ test17() PROC:<{ - 4219 PUSHINT NEWC - 16 STU + x{107b} STSLICECONST """ @fif_codegen """ test18() PROC:<{ - 421 PUSHINT NEWC - 26 STU + b{00000000000000000110100101} STSLICECONST """ @fif_codegen @@ -501,8 +498,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin 123 PUSHINT NEWC 4 STI - 16 PUSHPOW2DEC - 16 STUR + x{ffff} STSLICECONST -1 PUSHINT 8 STIR }> @@ -511,9 +507,8 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ test20() PROC:<{ - 40976 PUSHINT NEWC - 16 STU + x{a010} STSLICECONST """ @fif_codegen @@ -529,12 +524,10 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin test22() PROC:<{ NEWC // '2 NEWC // b1 b2 - 2056 PUSHINT - ROT - 24 STU // b2 b1 + SWAP // b2 b1 + x{000808} STSLICECONST // b2 b1 SWAP // b1 b2 - 10141514286835656557042350424064 PUSHINTX - 132 STUR // b1 b2 + x{000000080010000000000000000000000} STSLICECONST // b1 b2 """ @fif_codegen @@ -543,13 +536,11 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin NEWC SWAP IF:<{ - 91343852333181432387730302044911803916571639814 PUSHINT - 256 STUR + x{0000000000000000000000001000000000000000000000000200000000000006} STSLICECONST 8 PUSHINT 256 STUR }>ELSE<{ - 56539106072908298546665520023773392506479484700019806659963456035401760775 PUSHINT - 255 STIR + b{000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000111} STSLICECONST }> BBITS }> @@ -597,9 +588,8 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin s3 POP 42 EQINT IF:<{ - 0 PUSHINT - ROT - 3 STU + SWAP + b{000} STSLICECONST 8 STU // b }>ELSE<{ DROP // b diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index cafd2ffde8..5a386cd4b2 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -126,5 +126,5 @@ fun main() { @testcase | 105 | | -1 0 7 48 @testcase | 106 | | -1 0 0 -1 -@code_hash 39394999917297360167950120811797040756088634052063904555277102239104815314650 +@code_hash 32362412747322136329528616455651783746542516198110452861733590440068294458753 */ diff --git a/tolk-tester/tests/inline-tests.tolk b/tolk-tester/tests/inline-tests.tolk index fc48ddaa34..d85c5543b8 100644 --- a/tolk-tester/tests/inline-tests.tolk +++ b/tolk-tester/tests/inline-tests.tolk @@ -467,13 +467,11 @@ fun usedIn10ButDeclaredBelow(x: int) { @fif_codegen """ test8() PROC:<{ // - 5 PUSHINT NEWC - 32 STU // '3 + x{00000005} STSLICECONST // '3 25 PUSHINT // '3 '9 - 107374182425 PUSHINT - ROT - 64 STU // x '3 + SWAP // x self + x{0000001900000019} STSLICECONST // x '3 SWAP // '3 x }> """ @@ -509,9 +507,8 @@ fun usedIn10ButDeclaredBelow(x: int) { test10() PROC:<{ 5 PUSHINT // '3=5 DUP // '3=5 y=5 - 5 PUSHINT NEWC - 32 STU // a=5 b=5 c + x{00000005} STSLICECONST // a=5 b=5 c BBITS // a=5 b=5 '18 }> """ diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index 3e6a67befc..4d097a215e 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -1900,9 +1900,8 @@ fun main() { x{0102} PUSHSLICE 8 LDI NIP - 15 PUSHINT NEWC - 8 STI + x{0f} STSLICECONST STSLICE """ @@ -1917,9 +1916,8 @@ fun main() { 320 PUSHINT // lazyS '14=320 PLDSLICEX // '15 PUSHNULL // '15 st.extensions - 0 PUSHINT NEWC - 1 STU // '15 st.extensions b + b{0} STSLICECONST // '15 st.extensions b s1 s2 XCHG // st.extensions '15 b STSLICE // st.extensions b STOPTREF // b diff --git a/tolk-tester/tests/pack-unpack-1.tolk b/tolk-tester/tests/pack-unpack-1.tolk index 406307cfd7..9c9acf8219 100644 --- a/tolk-tester/tests/pack-unpack-1.tolk +++ b/tolk-tester/tests/pack-unpack-1.tolk @@ -107,9 +107,8 @@ fun main(c: cell) { @fif_codegen """ test3() PROC:<{ - 42949672980 PUSHINT NEWC - 64 STI // b + x{0000000a00000014} STSLICECONST // b ENDC // c CTOS // s 64 LDU diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index 69d0fb2d51..99a4221610 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -78,5 +78,5 @@ fun test1() { @testcase | 0 | | 0 @testcase | 101 | | [ 65 66 67 68 ] -@code_hash 33596158548631231081906594921712427436857221384530839008514203474433205250299 +@code_hash 52991558142486159683575375539594116045551936710713942466413832119934760926753 */ diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index d9e7608beb..605329bcd4 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -164,8 +164,8 @@ bool Optimizer::detect_rewrite_big_THROW() { return true; } -// purpose 1: for one constant b.storeInt(123, 32) generate not "123 PUSHINT; SWAP; STI", but "123 PUSHINT; STIR" -// purpose 2: consecutive b.storeUint(ff, 16).storeUint(ff, 16) generate one "00ff00ff" STU +// purpose 1: for b.storeInt(123, 32) generate not "123 PUSHINT; SWAP; STI", but "x{...} STSLICECONST" +// purpose 2: consecutive b.storeUint(ff, 16).storeUint(ff, 16) generate one "x{00ff00ff} STSLICECONST" // (since it works at IR level, it also works for const variables and auto-serialization) bool Optimizer::detect_rewrite_MY_store_int() { bool first_my_store = op_[0]->is_custom() && op_[0]->op.starts_with("MY_store_int"); @@ -198,14 +198,38 @@ bool Optimizer::detect_rewrite_MY_store_int() { n_merged++; } - p_ = n_merged; - q_ = 2; - oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, total_number)); - if (total_number == 0 && total_len == 4 && first_unsigned) { // "STGRAMS" stores four 0-bits cheaper than "4 STUR" - oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, "STGRAMS", 1, 1)); - } else { + // we do not want to always use STSLICECONST; for example, storing "0" 64-bit via x{00...} is more effective + // for a single operation, but in practice, total bytecode becomes larger, which has a cumulative negative effect; + // here is a heuristic "when to use STSLICECONST, when leave PUSHINT + STUR", based on real contracts measurements + bool use_stsliceconst = total_len <= 32 || (total_len <= 48 && total_number >= 256) || (total_len <= 64 && total_number >= 65536) + || (total_len <= 96 && total_number >= (1ULL<<32)) || (total_number > (1ULL<<62)); + if (!use_stsliceconst) { + p_ = n_merged; + q_ = 2; + oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, total_number)); oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, std::to_string(total_len) + (first_unsigned ? " STUR" : " STIR"), 1, 1)); + return true; } + + p_ = n_merged; + q_ = 1; + + // output "x{...}" or "b{...}" (if length not divisible by 4) + const td::RefInt256 base = td::make_refint(total_len % 4 == 0 ? 16 : 2); + const int s_len = base == 16 ? total_len / 4 : total_len; + const char* digits = "0123456789abcdef"; + + std::string result(s_len + 3, '0'); + result[0] = base == 16 ? 'x' : 'b'; + result[1] = '{'; + result[s_len + 3 - 1] = '}'; + for (int i = s_len - 1; i >= 0 && total_number != 0; --i) { + result[2 + i] = digits[(total_number % base)->to_long()]; + total_number /= base; + } + + result += " STSLICECONST"; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, result, 0, 1)); return true; } From 83ca804ea68a56b2af6092599ae293e7550f29ee Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 4 Jul 2025 23:52:15 +0300 Subject: [PATCH 345/524] [Tolk] Peephole optimization: NOT+THROWIFNOT to THROWIF --- tolk-tester/tests/logical-operators.tolk | 6 +++--- tolk/optimize.cpp | 24 ++++++++++++++++++++++++ tolk/pipe-ast-to-legacy.cpp | 17 ++++------------- tolk/tolk.h | 1 + 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk index c184da72c6..9b70ae40c9 100644 --- a/tolk-tester/tests/logical-operators.tolk +++ b/tolk-tester/tests/logical-operators.tolk @@ -351,15 +351,15 @@ These are moments of future optimizations. For now, it's more than enough. DUP // x x IFNOTJMP:<{ // x DROP // - 1 PUSHINT // '5=1 + 1 PUSHINT // '7=1 }> // x DUP // x x IFNOTJMP:<{ // x DROP // - 1 PUSHINT // '6=1 + 1 PUSHINT // '8=1 }> // x 100 THROWIFNOT - -4 PUSHINT // '9=-4 + -4 PUSHINT // '11=-4 }> """ diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index 605329bcd4..93c6e20675 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -399,6 +399,29 @@ bool Optimizer::detect_rewrite_SWAP_STxxxR() { return false; } +// pattern `NOT` + `123 THROWIFNOT` -> `123 THROWIF` (and THROWIF -> THROWIFNOT) +bool Optimizer::detect_rewrite_NOT_THROWIF() { + bool first_not = op_[0]->is_custom() && op_[0]->op == "NOT"; + if (!first_not || pb_ < 2 || !op_[1]->is_custom()) { + return false; + } + + static const char* ends_with[] = {" THROWIF", " THROWIFNOT"}; + static const char* repl_with[] = {" THROWIFNOT", " THROWIF"}; + + std::string_view f = op_[1]->op; + for (size_t i = 0; i < std::size(ends_with); ++i) { + if (f.ends_with(ends_with[i])) { + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, op_[1]->op.substr(0, f.rfind(' ')) + repl_with[i], 1, 0)); + return true; + } + } + + return false; +} + bool Optimizer::is_push_const(int* i, int* c) const { return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_push_const(i, c); } @@ -821,6 +844,7 @@ bool Optimizer::find_at_least(int pb) { detect_rewrite_MY_store_int() || detect_rewrite_MY_skip_bits() || detect_rewrite_NEWC_PUSH_STUR() || detect_rewrite_LDxx_DROP() || detect_rewrite_SWAP_symmetric() || detect_rewrite_SWAP_PUSH_STUR() || detect_rewrite_SWAP_STxxxR() || + detect_rewrite_NOT_THROWIF() || (!(mode_ & 1) && ((is_rot() && rewrite(AsmOp::Custom(loc, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(loc, "-ROT", 3, 3))) || (is_2dup() && rewrite(AsmOp::Custom(loc, "2DUP", 2, 4))) || diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 732dc3e180..7d1d2e1b61 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -2103,21 +2103,12 @@ static void process_block_statement(V v, CodeBlob& code) { } static void process_assert_statement(V v, CodeBlob& code) { - std::vector args(3); - if (auto v_not = v->get_cond()->try_as(); v_not && v_not->tok == tok_logical_not) { - args[0] = v->get_thrown_code(); - args[1] = v->get_cond()->as()->get_rhs(); - args[2] = createV(v->loc, true); - args[2]->mutate()->assign_inferred_type(TypeDataInt::create()); - } else { - args[0] = v->get_thrown_code(); - args[1] = v->get_cond(); - args[2] = createV(v->loc, false); - args[2]->mutate()->assign_inferred_type(TypeDataInt::create()); - } + std::vector ir_thrown_code = pre_compile_expr(v->get_thrown_code(), code); + std::vector ir_cond = pre_compile_expr(v->get_cond(), code); + tolk_assert(ir_cond.size() == 1 && ir_thrown_code.size() == 1); + std::vector args_vars = { ir_thrown_code[0], ir_cond[0], code.create_int(v->loc, 0, "(assert-0)") }; FunctionPtr builtin_sym = lookup_function("__throw_if_unless"); - std::vector args_vars = pre_compile_tensor(code, args); gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); } diff --git a/tolk/tolk.h b/tolk/tolk.h index 32c6843f8a..9bef3c4dfb 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -906,6 +906,7 @@ struct Optimizer { bool detect_rewrite_SWAP_symmetric(); bool detect_rewrite_SWAP_PUSH_STUR(); bool detect_rewrite_SWAP_STxxxR(); + bool detect_rewrite_NOT_THROWIF(); AsmOpConsList extract_code(); }; From 38a95cb9e38e8f759f38b9c19441788da9dc2729 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sat, 5 Jul 2025 16:33:28 +0300 Subject: [PATCH 346/524] [Tolk] Better error messages on type mismatch For example, when passing `bits8` to `slice`, the compiler now suggests to use `as` operator. --- tolk-tester/tests/assignment-tests.tolk | 9 ++++ .../tests/invalid-typing/err-6010.tolk | 12 +++++ .../tests/invalid-typing/err-6087.tolk | 1 + .../tests/invalid-typing/err-6135.tolk | 1 + .../tests/invalid-typing/err-6249.tolk | 1 + .../tests/invalid-typing/err-6390.tolk | 8 +++ .../tests/invalid-typing/err-6595.tolk | 10 ++++ .../tests/invalid-typing/err-6603.tolk | 14 +++++ .../tests/invalid-typing/err-6633.tolk | 11 ++++ .../tests/invalid-typing/err-6716.tolk | 7 +++ .../tests/invalid-typing/err-6737.tolk | 1 + .../tests/invalid-typing/err-6803.tolk | 1 + .../tests/invalid-typing/err-6839.tolk | 1 + .../tests/invalid-typing/err-6937.tolk | 1 + tolk/pipe-ast-to-legacy.cpp | 5 ++ tolk/pipe-check-inferred-types.cpp | 54 +++++++++++++++---- tolk/pipe-infer-types-and-calls.cpp | 6 +-- tolk/type-system.cpp | 3 ++ 18 files changed, 132 insertions(+), 14 deletions(-) create mode 100644 tolk-tester/tests/invalid-typing/err-6010.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6390.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6595.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6603.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6633.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6716.tolk diff --git a/tolk-tester/tests/assignment-tests.tolk b/tolk-tester/tests/assignment-tests.tolk index 5b8c2094c4..0343bdf507 100644 --- a/tolk-tester/tests/assignment-tests.tolk +++ b/tolk-tester/tests/assignment-tests.tolk @@ -249,6 +249,14 @@ fun test118(x: int) { return (i1, i2, st.owner.remainingBitsCount(), i3.0); } +@method_id(119) +fun test119(a: int, b: int) { + var t: tuple = [a + 1, b * 2] as tuple; + val l = t.size(); + val [c,d:int] = [t.0 as int, t.1]; + return (l, c, d); +} + fun main(value: int, ) { @@ -279,6 +287,7 @@ fun main(value: int, ) { @testcase | 116 | | 1 2 3 4 [ 1 2 3 4 ] @testcase | 117 | | [ 20 ] @testcase | 118 | 3 | 10 3 0 1 +@testcase | 119 | 1 2 | 2 2 4 @fif_codegen diff --git a/tolk-tester/tests/invalid-typing/err-6010.tolk b/tolk-tester/tests/invalid-typing/err-6010.tolk new file mode 100644 index 0000000000..ece391173c --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6010.tolk @@ -0,0 +1,12 @@ +fun analyze(t: slice) {} + +fun main(t: slice?) { + analyze(t); +} + +/** +@compilation_should_fail +@stderr can not pass `slice?` to `slice` +@stderr hint: probably, you should check on null +@stderr hint: alternatively, use `!` operator to bypass nullability checks: `!` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6087.tolk b/tolk-tester/tests/invalid-typing/err-6087.tolk index aabd956eac..dfc3ee62fc 100644 --- a/tolk-tester/tests/invalid-typing/err-6087.tolk +++ b/tolk-tester/tests/invalid-typing/err-6087.tolk @@ -6,4 +6,5 @@ fun cantAutoCastBytesNToSlice() { /** @compilation_should_fail @stderr method `loadInt` not found for type `bits32` +@stderr but it exists for type `slice` */ diff --git a/tolk-tester/tests/invalid-typing/err-6135.tolk b/tolk-tester/tests/invalid-typing/err-6135.tolk index f55c6dd484..7bf5808719 100644 --- a/tolk-tester/tests/invalid-typing/err-6135.tolk +++ b/tolk-tester/tests/invalid-typing/err-6135.tolk @@ -8,4 +8,5 @@ fun cantAssignIntNToCoins(n: int8, c: coins) { @compilation_should_fail @stderr can not assign `coins` to variable of type `int8` @stderr n = c; +@stderr hint: use `as` operator for unsafe casting: ` as int8` */ diff --git a/tolk-tester/tests/invalid-typing/err-6249.tolk b/tolk-tester/tests/invalid-typing/err-6249.tolk index 1553bf75ee..f5f4722d95 100644 --- a/tolk-tester/tests/invalid-typing/err-6249.tolk +++ b/tolk-tester/tests/invalid-typing/err-6249.tolk @@ -7,4 +7,5 @@ fun cantUnifyBytesNAndSlice(n: slice, c: bytes16) { /** @compilation_should_fail @stderr types of ternary branches are incompatible: `slice` and `bytes16` +@stderr hint: maybe, you should use ` as ` to make them identical */ diff --git a/tolk-tester/tests/invalid-typing/err-6390.tolk b/tolk-tester/tests/invalid-typing/err-6390.tolk new file mode 100644 index 0000000000..969cfd843a --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6390.tolk @@ -0,0 +1,8 @@ +fun main(k: (int, slice)) { + var (a, b, c: int?) = k; +} + +/** +@compilation_should_fail +@stderr can not assign `(int, slice)`, sizes mismatch + */ diff --git a/tolk-tester/tests/invalid-typing/err-6595.tolk b/tolk-tester/tests/invalid-typing/err-6595.tolk new file mode 100644 index 0000000000..c016ae187d --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6595.tolk @@ -0,0 +1,10 @@ +fun main() { + blockchain.configParam(true); +} + +/** +@compilation_should_fail +@stderr can not pass `bool` to `int` +@stderr hint: use `as` operator for unsafe casting: ` as int` +@stderr caution! in TVM, bool TRUE is -1, not 1 + */ diff --git a/tolk-tester/tests/invalid-typing/err-6603.tolk b/tolk-tester/tests/invalid-typing/err-6603.tolk new file mode 100644 index 0000000000..5f6275c350 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6603.tolk @@ -0,0 +1,14 @@ +struct A { + kk: tuple + dd: [int, int] +} + +fun main(a: A) { + a.kk = a.dd +} + +/** +@compilation_should_fail +@stderr can not assign `[int, int]` to field of type `tuple` +@stderr hint: use `as` operator for unsafe casting: ` as tuple` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6633.tolk b/tolk-tester/tests/invalid-typing/err-6633.tolk new file mode 100644 index 0000000000..d8ed9c2980 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6633.tolk @@ -0,0 +1,11 @@ +fun main(c: cell) { + var sender_address: slice = c.beginParse().loadAddress(); +} + +/** +@compilation_should_fail +@stderr can not assign `address` to variable of type `slice` +@stderr hint: unlike FunC, Tolk has a special type `address` (which is slice at the TVM level); +@stderr most likely, you just need `address` everywhere +@stderr hint: alternatively, use `as` operator for unsafe casting: ` as slice` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6716.tolk b/tolk-tester/tests/invalid-typing/err-6716.tolk new file mode 100644 index 0000000000..134fb9d14f --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6716.tolk @@ -0,0 +1,7 @@ +fun f(d: bits8 = "") {} + +/** +@compilation_should_fail +@stderr can not assign `slice` to `bits8` +@stderr hint: use `as` operator for unsafe casting: ` as bits8` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6737.tolk b/tolk-tester/tests/invalid-typing/err-6737.tolk index cc4bc2f59a..86934f3c8f 100644 --- a/tolk-tester/tests/invalid-typing/err-6737.tolk +++ b/tolk-tester/tests/invalid-typing/err-6737.tolk @@ -9,4 +9,5 @@ fun testAssignBetweenDifferentIntN(op: int32, qid: uint64) { @compilation_should_fail @stderr can not assign `uint64` to variable of type `int32` @stderr op = qid; +@stderr hint: use `as` operator for unsafe casting: ` as int32` */ diff --git a/tolk-tester/tests/invalid-typing/err-6803.tolk b/tolk-tester/tests/invalid-typing/err-6803.tolk index b04e8d974f..1e9592a68f 100644 --- a/tolk-tester/tests/invalid-typing/err-6803.tolk +++ b/tolk-tester/tests/invalid-typing/err-6803.tolk @@ -9,4 +9,5 @@ fun cantPassBytesNToSlice(c: bits16) { @compilation_should_fail @stderr can not pass `bits16` to `slice` @stderr takeAnySlice(c); +@stderr hint: use `as` operator for unsafe casting: ` as slice` */ diff --git a/tolk-tester/tests/invalid-typing/err-6839.tolk b/tolk-tester/tests/invalid-typing/err-6839.tolk index 91cca9c4fd..e8a4fd7c0a 100644 --- a/tolk-tester/tests/invalid-typing/err-6839.tolk +++ b/tolk-tester/tests/invalid-typing/err-6839.tolk @@ -16,4 +16,5 @@ fun cantMixDifferentThis() { /** @compilation_should_fail @stderr method `appendBuilder` not found for type `int` +@stderr (but it exists for type `builder`) */ diff --git a/tolk-tester/tests/invalid-typing/err-6937.tolk b/tolk-tester/tests/invalid-typing/err-6937.tolk index d380279a94..6038ea656c 100644 --- a/tolk-tester/tests/invalid-typing/err-6937.tolk +++ b/tolk-tester/tests/invalid-typing/err-6937.tolk @@ -9,4 +9,5 @@ fun cantAssignSliceToBytesN() { @compilation_should_fail @stderr can not assign `slice` to variable of type `bytes16` @stderr var c2: bytes16 = getAnySlice(); +@stderr hint: use `as` operator for unsafe casting: ` as bytes16` */ diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 7d1d2e1b61..f22b0ab445 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1075,6 +1075,11 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as() && target_type->try_as()) { + return rvect; + } + // pass something to `unknown` // probably, it comes from `_ = rhs`, type of `_` is unknown, it's target_type of rhs // no changes in rvect diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index b47513ee4d..97a953f4f6 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -49,6 +49,38 @@ static std::string expression_as_string(AnyExprV v) { return "expression"; } +// fire a general error on type mismatch; for example, "can not assign `cell` to `slice`"; +// for instance, if `as` operator is applicable, compiler will suggest it +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_type_mismatch(FunctionPtr cur_f, SrcLocation loc, const char* text_tpl, TypePtr src, TypePtr dst) { +#ifdef TOLK_DEBUG + tolk_assert(!dst->can_rhs_be_assigned(src)); +#endif + std::string message = text_tpl; + message.replace(message.find("{src}"), 5, to_string(src)); + message.replace(message.find("{dst}"), 5, to_string(dst)); + if (src->can_be_casted_with_as_operator(dst)) { + bool suggest_as = !dst->try_as() && !dst->try_as(); + if ((src == TypeDataAddress::create() || src == TypeDataSlice::create()) && (dst == TypeDataAddress::create() || dst == TypeDataSlice::create())) { + message += "\nhint: unlike FunC, Tolk has a special type `address` (which is slice at the TVM level);"; + message += "\n most likely, you just need `address` everywhere"; + message += "\nhint: alternatively, use `as` operator for unsafe casting: ` as " + dst->as_human_readable() + "`"; + } else if (suggest_as) { + message += "\nhint: use `as` operator for unsafe casting: ` as " + dst->as_human_readable() + "`"; + } + if (src == TypeDataBool::create() && dst == TypeDataInt::create()) { + message += "\ncaution! in TVM, bool TRUE is -1, not 1"; + } + } + if (const TypeDataUnion* src_nullable = src->try_as(); src_nullable && src_nullable->or_null) { + if (dst->can_rhs_be_assigned(src_nullable->or_null)) { + message += "\nhint: probably, you should check on null"; + message += "\nhint: alternatively, use `!` operator to bypass nullability checks: `!`"; + } + } + fire(cur_f, loc, message); +} + // fire an error on `!cell` / `+slice` GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_cannot_apply_operator(FunctionPtr cur_f, SrcLocation loc, std::string_view operator_name, AnyExprV unary_expr) { @@ -78,7 +110,7 @@ static void check_function_argument_passed(FunctionPtr cur_f, TypePtr param_type if (is_obj_of_dot_call) { fire(cur_f, ith_arg->loc, "can not call method for " + to_string(param_type) + " with object of type " + to_string(ith_arg)); } else { - fire(cur_f, ith_arg->loc, "can not pass " + to_string(ith_arg) + " to " + to_string(param_type)); + fire_error_type_mismatch(cur_f, ith_arg->loc, "can not pass {src} to {dst}", ith_arg->inferred_type, param_type); } } } @@ -349,7 +381,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { auto arg_i = v->get_arg(i)->get_expr(); TypePtr param_type = f_callable->params_types[i]; if (!param_type->can_rhs_be_assigned(arg_i->inferred_type)) { - fire(cur_f, arg_i->loc, "can not pass " + to_string(arg_i) + " to " + to_string(param_type)); + fire_error_type_mismatch(cur_f, arg_i->loc, "can not pass {src} to {dst}", arg_i->inferred_type, param_type); } } return; @@ -411,7 +443,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } if (declared_type) { if (!declared_type->can_rhs_be_assigned(rhs_type)) { - fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to variable of type " + to_string(declared_type)); + fire_error_type_mismatch(cur_f, err_loc->loc, "can not assign {src} to variable of type {dst}", rhs_type, declared_type); } } else { if (rhs_type == TypeDataNullLiteral::create()) { @@ -469,11 +501,11 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // for strange lhs like `f() = rhs` type checking will pass, but will fail lvalue check later if (!lhs->inferred_type->can_rhs_be_assigned(rhs_type)) { if (lhs->try_as()) { - fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to variable of type " + to_string(lhs)); + fire_error_type_mismatch(cur_f, err_loc->loc, "can not assign {src} to variable of type {dst}", rhs_type, lhs->inferred_type); } else if (lhs->try_as()) { - fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to field of type " + to_string(lhs)); + fire_error_type_mismatch(cur_f, err_loc->loc, "can not assign {src} to field of type {dst}", rhs_type, lhs->inferred_type); } else { - fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to " + to_string(lhs)); + fire_error_type_mismatch(cur_f, err_loc->loc, "can not assign {src} to {dst}", rhs_type, lhs->inferred_type); } } } @@ -490,7 +522,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { TypePtr expr_type = v->get_return_value()->inferred_type; if (!cur_f->inferred_return_type->can_rhs_be_assigned(expr_type)) { - fire(cur_f, v->get_return_value()->loc, "can not convert type " + to_string(expr_type) + " to return type " + to_string(cur_f->inferred_return_type)); + fire_error_type_mismatch(cur_f, v->get_return_value()->loc, "can not convert type {src} to return type {dst}", expr_type, cur_f->inferred_return_type); } } @@ -626,7 +658,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { parent::visit(v->get_init_val()); if (!v->field_ref->declared_type->can_rhs_be_assigned(v->get_init_val()->inferred_type)) { - fire(cur_f, v->get_init_val()->loc, "can not assign " + to_string(v->get_init_val()) + " to field of type " + to_string(v->field_ref->declared_type)); + fire_error_type_mismatch(cur_f, v->get_init_val()->loc, "can not assign {src} to field of type {dst}", v->get_init_val()->inferred_type, v->field_ref->declared_type); } } @@ -740,7 +772,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { TypePtr inferred_type = param_ref->default_value->inferred_type; if (!param_ref->declared_type->can_rhs_be_assigned(inferred_type)) { - throw ParseError(param_ref->loc, "can not assign " + to_string(inferred_type) + " to " + to_string(param_ref->declared_type)); + fire_error_type_mismatch(fun_ref, param_ref->loc, "can not assign {src} to {dst}", inferred_type, param_ref->declared_type); } } } @@ -756,7 +788,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (const_ref->declared_type) { // `const a: int = ...` TypePtr inferred_type = const_ref->init_value->inferred_type; if (!const_ref->declared_type->can_rhs_be_assigned(inferred_type)) { - throw ParseError(const_ref->loc, "can not assign " + to_string(inferred_type) + " to " + to_string(const_ref->declared_type)); + fire_error_type_mismatch(nullptr, const_ref->loc, "can not assign {src} to {dst}", inferred_type, const_ref->declared_type); } } } @@ -767,7 +799,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { TypePtr inferred_type = field_ref->default_value->inferred_type; if (!field_ref->declared_type->can_rhs_be_assigned(inferred_type)) { - throw ParseError(field_ref->loc, "can not assign " + to_string(inferred_type) + " to " + to_string(field_ref->declared_type)); + fire_error_type_mismatch(nullptr, field_ref->loc, "can not assign {src} to {dst}", inferred_type, field_ref->declared_type); } } }; diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 1253cdffc3..1bca8908ad 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -536,7 +536,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(lhs_var, rhs_type); assign_inferred_type(lhs_var->var_ref, rhs_type); } - TypePtr smart_casted_type = declared_type ? calc_smart_cast_type_on_assignment(declared_type, rhs_type) : rhs_type; + TypePtr smart_casted_type = declared_type && rhs_type != TypeDataUnknown::create() ? calc_smart_cast_type_on_assignment(declared_type, rhs_type) : rhs_type; out_flow.register_known_type(SinkExpression(lhs_var->var_ref), smart_casted_type); return; } @@ -740,7 +740,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // example: `var v = ternary`, show an inference error // do NOT show an error for `var v: T = ternary` (T is hint); it will be checked by type checker later if (hint == nullptr || hint == TypeDataUnknown::create() || hint->has_genericT_inside()) { - fire(cur_f, v->loc, "types of ternary branches are incompatible: " + to_string(v->get_when_true()) + " and " + to_string(v->get_when_false())); + fire(cur_f, v->loc, "types of ternary branches are incompatible: " + to_string(v->get_when_true()) + " and " + to_string(v->get_when_false()) + "\nhint: maybe, you should use ` as ` to make them identical"); } } assign_inferred_type(v, branches_unifier.get_result()); @@ -1081,7 +1081,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // as a special case, handle accessing fields of nullable objects, to show a more precise error if (const TypeDataUnion* as_union = obj_type->try_as(); as_union && as_union->or_null) { if (const TypeDataStruct* n_struct = as_union->or_null->try_as(); n_struct && n_struct->struct_ref->find_field(field_name)) { - fire(cur_f, v_ident->loc, "can not access field `" + static_cast(field_name) + "` of a possibly nullable object " + to_string(dot_obj) + "\ncheck it via `obj != null` or use non-null assertion `obj!` operator"); + fire(cur_f, v_ident->loc, "can not access field `" + static_cast(field_name) + "` of a possibly nullable object " + to_string(dot_obj) + "\nhint: check it via `obj != null` or use non-null assertion `obj!` operator"); } } if (out_f_called) { diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index 6253537e27..cf89edb0b7 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -1146,6 +1146,9 @@ bool TypeDataTensor::can_be_casted_with_as_operator(TypePtr cast_to) const { } bool TypeDataBrackets::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to->try_as()) { // `[int, int]` as `tuple` + return true; + } if (const auto* to_tuple = cast_to->try_as(); to_tuple && to_tuple->size() == size()) { for (int i = 0; i < size(); ++i) { if (!items[i]->can_be_casted_with_as_operator(to_tuple->items[i])) { From 705a419edafadb5e821f83eb29844234e6ceb77f Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 7 Jul 2025 03:49:08 +0300 Subject: [PATCH 347/524] [Tolk] More correctly generate a serialization prefix tree for nullable unions Now, treat `T?` as `Maybe` even if T is a union. Example: A|B|C|D|null => 0 | 100+A | 101+B | 110+C | 111+D. If no `null`, just distribute sequentially: A|B|C => 00+A | 01+B | 10+C --- tolk-tester/tests/lazy-load-tests.tolk | 32 ++++++++++++++++++++++++-- tolk-tester/tests/pack-unpack-4.tolk | 16 ++++++------- tolk-tester/tests/pack-unpack-5.tolk | 4 ++-- tolk/pack-unpack-serializers.cpp | 13 +++++++---- 4 files changed, 48 insertions(+), 17 deletions(-) diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index 4d097a215e..f686fe67c9 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -241,6 +241,18 @@ struct WithGlobalModifier { n: int8; } +struct WithN5 { n: uint5 } +type U111 = int8 | int16 | WithN5 | bits100 +struct WithU111 { + a: U111 + b: U111? + bit: bool +} +fun WithU111.getSlice_a8_bnull(): slice asm "b{000000000101} PUSHSLICE" +fun WithU111.getSlice_a16_b8(): slice asm "b{010000000000000001100000000011} PUSHSLICE" +fun WithU111.getSlice_a8_b5(): slice asm "b{0000000001110111111} PUSHSLICE" +fun WithU111.getSlice_a8_b5_nobit(): slice asm "b{000000000111011111} PUSHSLICE" + @noinline fun contract.getFakeData(seqno: int): Cell { @@ -981,6 +993,21 @@ fun demo158() { return (m2.n, gModByCustom, after1); // also modifies by skipping (unpack called) } +@method_id(159) +fun demo159() { + val p1 = lazy WithU111.fromSlice(WithU111.getSlice_a8_bnull()); + val p2 = lazy WithU111.fromSlice(WithU111.getSlice_a16_b8()); + val p3 = lazy WithU111.fromSlice(WithU111.getSlice_a8_b5()); + val p4 = lazy WithU111.fromSlice(WithU111.getSlice_a8_b5_nobit()); + + val p1_bit = p1.bit; + val p2_bit = p2.bit; + val p3_bit = p3.bit; + val p4_b = p4.b; + + return (p1_bit, p2_bit, (p3.b is WithN5) ? p3.b.n as int : 777, p3_bit, (p4_b is WithN5) ? p4_b.n as int: 777); +} + @method_id(200) fun demo200() { @@ -1601,10 +1628,10 @@ fun main() { @testcase | 113 | x{03ffff} | -1 @testcase | 113 | x{04000000FF} | 255 @testcase | 114 | x{0110} | 16 1 -@testcase | 114 | x{020f} | [ -15 ] typeid-25 +@testcase | 114 | x{020f} | [ -15 ] typeid-27 @testcase | 114 | x{03} | -1 1 @testcase | 114 | x{03ffff} | -1 1 -@testcase | 114 | x{04000000FF} | [ 255 ] typeid-25 +@testcase | 114 | x{04000000FF} | [ 255 ] typeid-27 @testcase | 115 | x{08} | 8 42 @testcase | 115 | x{88} | 0 2 @testcase | 115 | x{83} | -1 2 @@ -1669,6 +1696,7 @@ fun main() { @testcase | 156 | x{FF10} | -100 @testcase | 157 | | -1 @testcase | 158 | | 2 10 255 +@testcase | 159 | | -1 -1 31 -1 31 @testcase | 200 | | 1 @testcase | 201 | | 1 2 -1 diff --git a/tolk-tester/tests/pack-unpack-4.tolk b/tolk-tester/tests/pack-unpack-4.tolk index 6d913f584f..08a1b43281 100644 --- a/tolk-tester/tests/pack-unpack-4.tolk +++ b/tolk-tester/tests/pack-unpack-4.tolk @@ -38,10 +38,10 @@ fun test1() { type Union_8_16_32_n = int8 | null | int16 | int32; -fun ans102n_8(): slice asm "b{0100001111} PUSHSLICE"; -fun ans102n_16(): slice asm "b{100000000000001111} PUSHSLICE"; -fun ans102n_32(): slice asm "b{1100000000000000000000000000001111} PUSHSLICE"; -fun ans102n_null(): slice asm "b{00} PUSHSLICE"; +fun ans102n_8(): slice asm "b{10000001111} PUSHSLICE"; +fun ans102n_16(): slice asm "b{1010000000000001111} PUSHSLICE"; +fun ans102n_32(): slice asm "b{11000000000000000000000000000001111} PUSHSLICE"; +fun ans102n_null(): slice asm "b{0} PUSHSLICE"; @method_id(102) fun test2() { @@ -65,10 +65,10 @@ fun ans104_8(): slice asm "b{0000001111} PUSHSLICE"; fun ans104_16(): slice asm "b{010000000000001111} PUSHSLICE"; fun ans104_32(): slice asm "b{1000000000000000000000000000001111} PUSHSLICE"; -fun ans104n_8(): slice asm "b{0100001111} PUSHSLICE"; -fun ans104n_16(): slice asm "b{100000000000001111} PUSHSLICE"; -fun ans104n_32(): slice asm "b{1100000000000000000000000000001111} PUSHSLICE"; -fun ans104n_null(): slice asm "b{00} PUSHSLICE"; +fun ans104n_8(): slice asm "b{10000001111} PUSHSLICE"; +fun ans104n_16(): slice asm "b{1010000000000001111} PUSHSLICE"; +fun ans104n_32(): slice asm "b{11000000000000000000000000000001111} PUSHSLICE"; +fun ans104n_null(): slice asm "b{0} PUSHSLICE"; @method_id(104) fun test4() { diff --git a/tolk-tester/tests/pack-unpack-5.tolk b/tolk-tester/tests/pack-unpack-5.tolk index 1baa240642..97b740b481 100644 --- a/tolk-tester/tests/pack-unpack-5.tolk +++ b/tolk-tester/tests/pack-unpack-5.tolk @@ -160,7 +160,7 @@ struct Test9_bits4 { f: bits4; } type Test9_f1 = int32 | int64 | int128; // auto-generated 2-bit prefix type Test9_f2 = int32 | Inner8_2; // auto-generated 1-bit prefix (Either) type Test9_f3 = bits1 | Test9_bits2 | bits3 | bits4 | bits5; // auto-generated 3-bit prefix -type Test9_f4 = bits1 | Test9_bits2 | bits3 | Test9_bits4?; // auto-generated 3-bit prefix +type Test9_f4 = bits1 | Test9_bits2 | bits3 | Test9_bits4?; // auto-generated 0 / 100/101/110/111 @method_id(109) fun test9() { @@ -213,7 +213,7 @@ fun main() { @testcase | 106 | | [ 64 64 0 0 ] [ 5 37 1 1 ] [ 5 65 0 1 ] @testcase | 107 | | [ 16 16 0 0 ] [ 16 16 0 0 ] @testcase | 108 | | [ 10 275 0 0 ] [ 34 158 0 0 ] [ 47 1143 0 1 ] -@testcase | 109 | | [ 34 130 0 0 ] [ 33 159 0 0 ] [ 4 8 0 0 ] [ 3 7 0 0 ] +@testcase | 109 | | [ 34 130 0 0 ] [ 33 159 0 0 ] [ 4 8 0 0 ] [ 1 7 0 0 ] @testcase | 110 | | [ 32 9999 0 4 ] [ 33 9999 0 8 ] @testcase | 111 | | [ 1023 1023 0 1 ] [ 0 0 2 2 ] @testcase | 120 | | 16 4128 1 1 diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index ecad405021..5f1ebcc8f5 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -1146,13 +1146,16 @@ std::vector auto_generate_opcodes_for_union(TypePtr union_type, std: // okay, none of the opcodes are specified, generate a prefix tree; // examples: `int32 | int64 | int128` / `int32 | A | null` / `A | B` / `A | B | C`; - // create prefixes `0b00 0b01 0b10` / `0b01 0b10 0b00` (use 0b00 for null if exists): - // for 3/4 variants — two bits, for 5 — three - int prefix_len = static_cast(std::ceil(std::log2(t_union->size()))); - int cur_prefix = has_null ? 1 : 0; // will use 0b00 for null, so start with 0b01 + // if `null` exists, it's 0, all others are 1+tree: A|B|C|D|null => 0 | 100+A | 101+B | 110+C | 111+D; + // if no `null`, just distribute sequentially: A|B|C => 00+A | 01+B | 10+C + int n_without_null = t_union->size() - has_null; + int prefix_len = static_cast(std::ceil(std::log2(n_without_null))); + int cur_prefix = 0; for (TypePtr variant : t_union->variants) { if (variant == TypeDataNullLiteral::create()) { - result.emplace_back(0, prefix_len); + result.emplace_back(0, 1); + } else if (has_null) { + result.emplace_back((1< Date: Mon, 7 Jul 2025 04:02:48 +0300 Subject: [PATCH 348/524] [Tolk] Bump version to v1.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finally, after 7 months of development — the release --- crypto/smartcont/tolk-stdlib/common.tolk | 2 +- crypto/smartcont/tolk-stdlib/gas-payments.tolk | 2 +- crypto/smartcont/tolk-stdlib/lisp-lists.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk | 2 +- tolk/tolk-version.h | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index a72718246a..7d21dd7d9d 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -1,7 +1,7 @@ // Standard library for Tolk (LGPL licence). // It contains common functions that are available out of the box, the user doesn't have to import anything. // More specific functions are required to be imported explicitly, like "@stdlib/tvm-dicts". -tolk 0.99 +tolk 1.0 /// In Tolk v1.x there would be a type `map`. /// Currently, working with dictionaries is still low-level, with raw cells. diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 977fa6f2d2..916cefc6c1 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.99 +tolk 1.0 /** Gas and payment related primitives. diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk index 2853abef03..212586b1d4 100644 --- a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.99 +tolk 1.0 /** Lisp-style lists are nested 2-elements tuples: `[1, [2, [3, null]]]` represents list `[1, 2, 3]`. diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index 14723682a0..41e091f0aa 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.99 +tolk 1.0 /** Dictionaries are represented as `cell` data type (cells can store anything, dicts in particular). diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index 18a0e159fd..d578199c7d 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.99 +tolk 1.0 /// Usually `c3` has a continuation initialized by the whole code of the contract. It is used for function calls. /// The primitive returns the current value of `c3`. diff --git a/tolk/tolk-version.h b/tolk/tolk-version.h index aaf1255d20..77d745eac9 100644 --- a/tolk/tolk-version.h +++ b/tolk/tolk-version.h @@ -18,6 +18,6 @@ namespace tolk { -constexpr const char* TOLK_VERSION = "0.99.0"; +constexpr const char* TOLK_VERSION = "1.0.0"; } // namespace tolk From a89cafb4e941478d34357e58e3e82a05d3b75b02 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 9 Jul 2025 11:52:38 +0300 Subject: [PATCH 349/524] Move limit for storage dict hash to config --- crypto/block/block.tlb | 2 +- crypto/block/mc-config.cpp | 1 + crypto/block/mc-config.h | 1 + crypto/block/transaction.cpp | 4 ++-- crypto/block/transaction.h | 1 + validator/impl/validate-query.cpp | 1 + 6 files changed, 7 insertions(+), 3 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 83ec61ed58..7bff1ac177 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -798,7 +798,7 @@ size_limits_config#01 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_acc_state_bits:uint32 max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 max_msg_extra_currencies:uint32 - max_acc_fixed_prefix_length:uint8 = SizeLimitsConfig; + max_acc_fixed_prefix_length:uint8 acc_state_cells_for_storage_dict:uint32 = SizeLimitsConfig; _ SizeLimitsConfig = ConfigParam 43; // key is [ wc:int32 addr:uint256 ] diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 35c3df0055..bd9b6289dc 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2046,6 +2046,7 @@ td::Result Config::do_get_size_limits_config(td::Refsize(); - // TODO: think about this limit (25) - if (store_storage_dict_hash && new_storage_used.cells > 25) { + if (store_storage_dict_hash && new_storage_used.cells >= cfg.size_limits.acc_state_cells_for_storage_dict) { auto r_hash = stats.get_dict_hash(); if (r_hash.is_error()) { LOG(ERROR) << "Cannot compute storage dict hash for account " << account.addr.to_hex() << ": " @@ -4184,6 +4183,7 @@ td::Status FetchConfigParams::fetch_config_params( serialize_cfg->extra_currency_v2 = config.get_global_version() >= 10; serialize_cfg->disable_anycast = config.get_global_version() >= 10; serialize_cfg->store_storage_dict_hash = config.get_global_version() >= 11; + serialize_cfg->size_limits = size_limits; } { // fetch block_grams_created diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 1a7af2d125..9ef06e3c36 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -186,6 +186,7 @@ struct SerializeConfig { bool extra_currency_v2{false}; bool disable_anycast{false}; bool store_storage_dict_hash{false}; + SizeLimitsConfig size_limits; }; struct CreditPhase { diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index ac7f400f34..58576e0272 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1079,6 +1079,7 @@ bool ValidateQuery::fetch_config_params() { serialize_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; serialize_cfg_.disable_anycast = config_->get_global_version() >= 10; serialize_cfg_.store_storage_dict_hash = config_->get_global_version() >= 11; + serialize_cfg_.size_limits = size_limits; } { // fetch block_grams_created From a969178974d0755e9f30737e7883fba9e0b359e8 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 9 Jul 2025 11:53:08 +0300 Subject: [PATCH 350/524] BTOS opcode; decrease gas usage for HASHSU --- common/global-version.h | 2 +- crypto/fift/lib/Asm.fif | 1 + crypto/vm/cellops.cpp | 10 ++++++++++ crypto/vm/tonops.cpp | 9 ++++++--- doc/GlobalVersions.md | 12 +++++++++++- 5 files changed, 29 insertions(+), 5 deletions(-) diff --git a/common/global-version.h b/common/global-version.h index b54a3bdc52..d88a22c32a 100644 --- a/common/global-version.h +++ b/common/global-version.h @@ -19,6 +19,6 @@ namespace ton { // See doc/GlobalVersions.md -constexpr int SUPPORTED_VERSION = 11; +constexpr int SUPPORTED_VERSION = 12; } diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index 9413513972..3a9e3e447b 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -697,6 +697,7 @@ x{CF3F} @Defop BCHKBITREFSQ x{CF40} @Defop STZEROES x{CF41} @Defop STONES x{CF42} @Defop STSAME +x{CF50} @Defop BTOS { tuck sbitrefs swap 22 + swap @havebitrefs not { swap PUSHSLICE STSLICER } { over sbitrefs 2dup 57 3 2x<= diff --git a/crypto/vm/cellops.cpp b/crypto/vm/cellops.cpp index 61ffe5c554..b76439fac6 100644 --- a/crypto/vm/cellops.cpp +++ b/crypto/vm/cellops.cpp @@ -764,6 +764,15 @@ int exec_store_same(VmState* st, const char* name, int val) { return 0; } +int exec_builder_to_slice(VmState* st) { + Stack& stack = st->get_stack(); + VM_LOG(st) << "execute BTOS"; + stack.check_underflow(1); + Ref cell = stack.pop_builder().write().finalize_novm(); + stack.push_cellslice(Ref{true, NoVm(), cell}); + return 0; +} + int exec_store_const_slice(VmState* st, CellSlice& cs, unsigned args, int pfx_bits) { unsigned refs = (args >> 3) & 3; unsigned data_bits = (args & 7) * 8 + 2; @@ -866,6 +875,7 @@ void register_cell_serialize_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xcf40, 16, "STZEROES", std::bind(exec_store_same, _1, "STZEROES", 0))) .insert(OpcodeInstr::mksimple(0xcf41, 16, "STONES", std::bind(exec_store_same, _1, "STONES", 1))) .insert(OpcodeInstr::mksimple(0xcf42, 16, "STSAME", std::bind(exec_store_same, _1, "STSAME", -1))) + .insert(OpcodeInstr::mksimple(0xcf50, 16, "BTOS", exec_builder_to_slice)->require_version(12)) .insert(OpcodeInstr::mkext(0xcf80 >> 7, 9, 5, dump_store_const_slice, exec_store_const_slice, compute_len_store_const_slice)); } diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index b06fb1b77b..3173f9d1c5 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -618,10 +618,13 @@ int exec_compute_hash(VmState* st, int mode) { hash = cell->get_hash().as_array(); } else { auto cs = stack.pop_cellslice(); - vm::CellBuilder cb; + CellBuilder cb; CHECK(cb.append_cellslice_bool(std::move(cs))); - // TODO: use cb.get_hash() instead - hash = cb.finalize()->get_hash().as_array(); + if (st->get_global_version() >= 12) { + hash = cb.finalize_novm()->get_hash().as_array(); + } else { + hash = cb.finalize()->get_hash().as_array(); + } } td::RefInt256 res{true}; CHECK(res.write().import_bytes(hash.data(), hash.size(), false)); diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index ad5eba948f..674fba2662 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -144,6 +144,7 @@ Example: if the last masterchain block seqno is `19071` then the list contains b - Fix recursive jump to continuations with non-null control data. ## Version 10 +__Enabled in mainnet on 2025-05-07__ ### Extra currencies - Internal messages cannot carry more than 2 different extra currencies. The limit can be changed in size limits config (`ConfigParam 43`). @@ -194,6 +195,7 @@ Reserve modes `+1`, `+4` and `+8` ("reserve all except", "add original balance" - Exceeding state limits in transaction now reverts `end_lt` back to `start_lt + 1` and collects action fines. ## Version 11 +__Enabled in mainnet on 2025-07-05__ ### c7 tuple **c7** tuple extended from 17 to 18 elements: @@ -225,4 +227,12 @@ This is required to help computing storage stats in the future, after collator-v ### Other changes - Fix returning `null` as `c4` and `c5` (when VM state is not committed) in `RUNVM`. -- In new internal messages `ihr_disabled` is automatically set to `1`, `ihr_fee` is always zero. \ No newline at end of file +- In new internal messages `ihr_disabled` is automatically set to `1`, `ihr_fee` is always zero. + +## Version 12 + +### New TVM instructions +- `BTOS` (`b - s`) - same as `ENDC CTOS`, but without gas cost for cell creation and loading. Gas cost: `26`. + +### Other TVM changes +- `HASHSU` (`s - hash`) now does not spend gas for cell creation. Gas cost: `26`. \ No newline at end of file From 07f2565e8751784c8bed5c51cad5786c0d7de86e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 9 Jul 2025 14:24:49 +0300 Subject: [PATCH 351/524] HASHBU opcode --- crypto/fift/lib/Asm.fif | 1 + crypto/vm/tonops.cpp | 10 +++++++--- doc/GlobalVersions.md | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index 3a9e3e447b..a4b1e144d3 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -1391,6 +1391,7 @@ x{F912} @Defop ECRECOVER x{F913} @Defop SECP256K1_XONLY_PUBKEY_TWEAK_ADD x{F914} @Defop P256_CHKSIGNU x{F915} @Defop P256_CHKSIGNS +x{F916} @Defop HASHBU x{F920} @Defop RIST255_FROMHASH x{F921} @Defop RIST255_VALIDATE diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index 3173f9d1c5..981f7c5b33 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -610,13 +610,13 @@ void register_prng_ops(OpcodeTable& cp0) { } int exec_compute_hash(VmState* st, int mode) { - VM_LOG(st) << "execute HASH" << (mode & 1 ? 'S' : 'C') << 'U'; + VM_LOG(st) << "execute HASH" << "CSB"[mode] << 'U'; Stack& stack = st->get_stack(); std::array hash; - if (!(mode & 1)) { + if (mode == 0) { // cell auto cell = stack.pop_cell(); hash = cell->get_hash().as_array(); - } else { + } else if (mode == 1) { // slice auto cs = stack.pop_cellslice(); CellBuilder cb; CHECK(cb.append_cellslice_bool(std::move(cs))); @@ -625,6 +625,9 @@ int exec_compute_hash(VmState* st, int mode) { } else { hash = cb.finalize()->get_hash().as_array(); } + } else { // builder + auto cb = stack.pop_builder(); + hash = cb.write().finalize_novm()->get_hash().as_array(); } td::RefInt256 res{true}; CHECK(res.write().import_bytes(hash.data(), hash.size(), false)); @@ -1368,6 +1371,7 @@ void register_ton_crypto_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xf913, 16, "SECP256K1_XONLY_PUBKEY_TWEAK_ADD", exec_secp256k1_xonly_pubkey_tweak_add)->require_version(9)) .insert(OpcodeInstr::mksimple(0xf914, 16, "P256_CHKSIGNU", std::bind(exec_p256_chksign, _1, false))->require_version(4)) .insert(OpcodeInstr::mksimple(0xf915, 16, "P256_CHKSIGNS", std::bind(exec_p256_chksign, _1, true))->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf916, 16, "HASHBU", std::bind(exec_compute_hash, _1, 2))->require_version(12)) .insert(OpcodeInstr::mksimple(0xf920, 16, "RIST255_FROMHASH", exec_ristretto255_from_hash)->require_version(4)) .insert(OpcodeInstr::mksimple(0xf921, 16, "RIST255_VALIDATE", std::bind(exec_ristretto255_validate, _1, false))->require_version(4)) diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 674fba2662..47683c4314 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -233,6 +233,7 @@ This is required to help computing storage stats in the future, after collator-v ### New TVM instructions - `BTOS` (`b - s`) - same as `ENDC CTOS`, but without gas cost for cell creation and loading. Gas cost: `26`. +- `HASHBU` (`b - hash`) - same as `ENDC HASHCU`, but without gas cost for cell creation. Gas cost: `26`. ### Other TVM changes - `HASHSU` (`s - hash`) now does not spend gas for cell creation. Gas cost: `26`. \ No newline at end of file From 298db547f868e933e70efcef3d228fe09dca9162 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 9 Jul 2025 15:28:30 +0300 Subject: [PATCH 352/524] New account size limit for masterchain --- crypto/block/block.tlb | 2 +- crypto/block/mc-config.cpp | 2 +- crypto/block/mc-config.h | 2 +- crypto/block/transaction.cpp | 18 ++++++++++-------- crypto/block/transaction.h | 3 ++- doc/GlobalVersions.md | 6 +++++- validator/impl/validate-query.cpp | 1 + 7 files changed, 21 insertions(+), 13 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 7bff1ac177..1967ca0239 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -796,7 +796,7 @@ _ MisbehaviourPunishmentConfig = ConfigParam 40; size_limits_config#01 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 max_ext_msg_size:uint32 max_ext_msg_depth:uint16 = SizeLimitsConfig; size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 - max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_acc_state_bits:uint32 + max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_mc_acc_state_cells:uint32 max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 max_msg_extra_currencies:uint32 max_acc_fixed_prefix_length:uint8 acc_state_cells_for_storage_dict:uint32 = SizeLimitsConfig; _ SizeLimitsConfig = ConfigParam 43; diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index bd9b6289dc..d7570863e3 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2040,8 +2040,8 @@ td::Result Config::do_get_size_limits_config(td::Ref& old_l * This function is not called for special accounts. * * @param size_limits The size limits configuration. + * @param global_version Global version (ConfigParam 8). * @param is_account_stat Store storage stat in the Transaction's AccountStorageStat. * * @returns A `td::Status` indicating the result of the check. @@ -3182,7 +3183,8 @@ static td::uint32 get_public_libraries_diff_count(const td::Ref& old_l * - If the state limits exceed the maximum allowed range, returns an error with AccountStorageStat::errorcode_limits_exceeded code. * - If an error occurred during storage stat calculation, returns other error. */ -td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, bool is_account_stat) { +td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, int global_version, + bool is_account_stat) { auto cell_equal = [](const td::Ref& a, const td::Ref& b) -> bool { return a.is_null() || b.is_null() ? a.is_null() == b.is_null() : a->get_hash() == b->get_hash(); }; @@ -3213,13 +3215,12 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, } } - if (storage_stat.get_total_cells() > size_limits.max_acc_state_cells || - storage_stat.get_total_bits() > size_limits.max_acc_state_bits) { + td::uint32 max_cells = account.is_masterchain() && global_version >= 12 ? size_limits.max_mc_acc_state_cells + : size_limits.max_acc_state_cells; + if (storage_stat.get_total_cells() > max_cells) { return td::Status::Error(AccountStorageStat::errorcode_limits_exceeded, PSTRING() << "account state is too big: cells=" << storage_stat.get_total_cells() - << ", bits=" << storage_stat.get_total_bits() - << " (max cells=" << size_limits.max_acc_state_cells - << ", max bits=" << size_limits.max_acc_state_bits << ")"); + << " (max cells=" << max_cells << ")"); } if (account.is_masterchain() && !cell_equal(account.library, new_library)) { auto libraries_count = get_public_libraries_count(new_library); @@ -4178,6 +4179,7 @@ td::Status FetchConfigParams::fetch_config_params( action_phase_cfg->extra_currency_v2 = config.get_global_version() >= 10; action_phase_cfg->disable_anycast = config.get_global_version() >= 10; action_phase_cfg->disable_ihr_flag = config.get_global_version() >= 11; + action_phase_cfg->global_version = config.get_global_version(); } { serialize_cfg->extra_currency_v2 = config.get_global_version() >= 10; diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 9ef06e3c36..5492ee6bf2 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -177,6 +177,7 @@ struct ActionPhaseConfig { bool disable_ihr_flag{false}; td::optional mc_blackhole_addr; bool disable_anycast{false}; + int global_version = 0; const MsgPrices& fetch_msg_prices(bool is_masterchain) const { return is_masterchain ? fwd_mc : fwd_std; } @@ -413,7 +414,7 @@ struct Transaction { bool run_precompiled_contract(const ComputePhaseConfig& cfg, precompiled::PrecompiledSmartContract& precompiled); bool prepare_compute_phase(const ComputePhaseConfig& cfg); bool prepare_action_phase(const ActionPhaseConfig& cfg); - td::Status check_state_limits(const SizeLimitsConfig& size_limits, bool is_account_stat = true); + td::Status check_state_limits(const SizeLimitsConfig& size_limits, int global_version, bool is_account_stat = true); bool prepare_bounce_phase(const ActionPhaseConfig& cfg); bool compute_state(const SerializeConfig& cfg); bool serialize(const SerializeConfig& cfg); diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 47683c4314..6281735e08 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -236,4 +236,8 @@ This is required to help computing storage stats in the future, after collator-v - `HASHBU` (`b - hash`) - same as `ENDC HASHCU`, but without gas cost for cell creation. Gas cost: `26`. ### Other TVM changes -- `HASHSU` (`s - hash`) now does not spend gas for cell creation. Gas cost: `26`. \ No newline at end of file +- `HASHSU` (`s - hash`) now does not spend gas for cell creation. Gas cost: `26`. + +### Other changes +- Account size in masterchain is now limited to `2048` cells. This can be configured in size limits config (`ConfigParam 43`). + - The previous limit was the same as in basechain (`65536`). \ No newline at end of file diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 58576e0272..4bef90395c 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1074,6 +1074,7 @@ bool ValidateQuery::fetch_config_params() { action_phase_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; action_phase_cfg_.disable_anycast = config_->get_global_version() >= 10; action_phase_cfg_.disable_ihr_flag = config_->get_global_version() >= 11; + action_phase_cfg_.global_version = config_->get_global_version(); } { serialize_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; From 126ac76a50d07eab5cd5df914478d8de9dec161c Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 23 Jul 2025 16:08:41 +0300 Subject: [PATCH 353/524] New bounce format --- crypto/block/block-parse.cpp | 46 ++++++-------- crypto/block/block-parse.h | 42 +++++++----- crypto/block/block.tlb | 14 +++- crypto/block/transaction.cpp | 92 +++++++++++++++++++-------- crypto/block/transaction.h | 4 +- crypto/smc-envelope/SmartContract.cpp | 4 +- crypto/vm/boc.cpp | 10 +++ crypto/vm/boc.h | 2 + crypto/vm/cells/CellBuilder.h | 3 + crypto/vm/tonops.cpp | 18 ++++-- doc/GlobalVersions.md | 33 ++++++++++ emulator/test/emulator-tests.cpp | 2 +- tonlib/tonlib/TonlibClient.cpp | 9 +-- validator/impl/collator-impl.h | 4 ++ validator/impl/collator.cpp | 8 ++- validator/impl/ihr-message.cpp | 2 +- validator/impl/liteserver.cpp | 6 +- validator/impl/validate-query.cpp | 40 +++++++++--- validator/impl/validate-query.hpp | 4 ++ 19 files changed, 242 insertions(+), 101 deletions(-) diff --git a/crypto/block/block-parse.cpp b/crypto/block/block-parse.cpp index 71ec2bcf51..9443b69f37 100644 --- a/crypto/block/block-parse.cpp +++ b/crypto/block/block-parse.cpp @@ -665,7 +665,7 @@ bool CommonMsgInfo::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const && t_MsgAddressInt.validate_skip(ops, cs, weak) // src && t_MsgAddressInt.validate_skip(ops, cs, weak) // dest && t_CurrencyCollection.validate_skip(ops, cs, weak) // value - && t_Grams.validate_skip(ops, cs, weak) // ihr_fee + && t_Grams.validate_skip(ops, cs, weak) // extra_flags && t_Grams.validate_skip(ops, cs, weak) // fwd_fee && cs.advance(64 + 32); // created_lt:uint64 created_at:uint32 case ext_in_msg_info: @@ -684,7 +684,7 @@ bool CommonMsgInfo::unpack(vm::CellSlice& cs, CommonMsgInfo::Record_int_msg_info return get_tag(cs) == int_msg_info && cs.advance(1) && cs.fetch_bool_to(data.ihr_disabled) && cs.fetch_bool_to(data.bounce) && cs.fetch_bool_to(data.bounced) && t_MsgAddressInt.fetch_to(cs, data.src) && t_MsgAddressInt.fetch_to(cs, data.dest) && t_CurrencyCollection.fetch_to(cs, data.value) && - t_Grams.fetch_to(cs, data.ihr_fee) && t_Grams.fetch_to(cs, data.fwd_fee) && + t_Grams.fetch_to(cs, data.extra_flags) && t_Grams.fetch_to(cs, data.fwd_fee) && cs.fetch_uint_to(64, data.created_lt) && cs.fetch_uint_to(32, data.created_at); } @@ -696,7 +696,7 @@ bool CommonMsgInfo::skip(vm::CellSlice& cs) const { && t_MsgAddressInt.skip(cs) // src && t_MsgAddressInt.skip(cs) // dest && t_CurrencyCollection.skip(cs) // value - && t_Grams.skip(cs) // ihr_fee + && t_Grams.skip(cs) // extra_flags && t_Grams.skip(cs) // fwd_fee && cs.advance(64 + 32); // created_lt:uint64 created_at:uint32 case ext_in_msg_info: @@ -718,7 +718,7 @@ bool CommonMsgInfo::get_created_lt(vm::CellSlice& cs, unsigned long long& create && t_MsgAddressInt.skip(cs) // src && t_MsgAddressInt.skip(cs) // dest && t_CurrencyCollection.skip(cs) // value - && t_Grams.skip(cs) // ihr_fee + && t_Grams.skip(cs) // extra_flags && t_Grams.skip(cs) // fwd_fee && cs.fetch_ulong_bool(64, created_lt) // created_lt:uint64 && cs.advance(32); // created_at:uint32 @@ -1843,26 +1843,18 @@ bool InMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return false; } -bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { +static td::RefInt256 get_ihr_fee(const CommonMsgInfo::Record_int_msg_info &info, int global_version) { + // Legacy: extra_flags was previously ihr_fee + return global_version >= 12 ? td::zero_refint() : t_Grams.as_integer(std::move(info.extra_flags)); +} + +bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs, int global_version) const { int tag = get_tag(cs); switch (tag) { case msg_import_ext: // inbound external message return t_ImportFees.null_value(cb); // external messages have no value and no import fees case msg_import_ihr: // IHR-forwarded internal message to its final destination - if (cs.advance(3) && cs.size_refs() >= 3) { - auto msg_cs = load_cell_slice(cs.fetch_ref()); - CommonMsgInfo::Record_int_msg_info msg_info; - td::RefInt256 ihr_fee; - vm::CellBuilder aux; - // sort of Prolog-style in C++ - return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) && - cs.fetch_ref().not_null() && (ihr_fee = t_Grams.as_integer_skip(cs)).not_null() && - cs.fetch_ref().not_null() && !cmp(ihr_fee, t_Grams.as_integer(*msg_info.ihr_fee)) && - cb.append_cellslice_bool(msg_info.ihr_fee) // fees_collected := ihr_fee - && aux.append_cellslice_bool(msg_info.ihr_fee) && t_ExtraCurrencyCollection.null_value(aux) && - t_CurrencyCollection.add_values(cb, aux.as_cellslice_ref().write(), - msg_info.value.write()); // value_imported := ihr_fee + value - } + // IHR is not implemented return false; case msg_import_imm: // internal message re-imported from this very block if (cs.advance(3) && cs.size_refs() >= 2) { @@ -1888,7 +1880,7 @@ bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) && cb.append_cellslice_bool(in_msg.fwd_fee_remaining) // fees_collected := fwd_fee_remaining && t_Grams.as_integer_skip_to(msg_info.value.write(), value_grams) && - (ihr_fee = t_Grams.as_integer(std::move(msg_info.ihr_fee))).not_null() && + (ihr_fee = get_ihr_fee(msg_info, global_version)).not_null() && t_Grams.store_integer_ref(cb, value_grams + ihr_fee + fwd_fee_remaining) && cb.append_cellslice_bool( msg_info.value.write()); // value_imported = msg.value + msg.ihr_fee + fwd_fee_remaining @@ -1911,7 +1903,7 @@ bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) && t_Grams.store_integer_ref(cb, std::move(transit_fee)) // fees_collected := transit_fees && t_Grams.as_integer_skip_to(msg_info.value.write(), value_grams) && - (ihr_fee = t_Grams.as_integer(std::move(msg_info.ihr_fee))).not_null() && + (ihr_fee = get_ihr_fee(msg_info, global_version)).not_null() && t_Grams.store_integer_ref(cb, value_grams + ihr_fee + fwd_fee_remaining) && cb.append_cellslice_bool( msg_info.value.write()); // value_imported = msg.value + msg.ihr_fee + fwd_fee_remaining @@ -1941,8 +1933,8 @@ bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { const InMsg t_InMsg; -const Aug_InMsgDescr aug_InMsgDescr; -const InMsgDescr t_InMsgDescr; +const Aug_InMsgDescr aug_InMsgDescrDefault(ton::SUPPORTED_VERSION); +const InMsgDescr t_InMsgDescrDefault(ton::SUPPORTED_VERSION); bool OutMsg::skip(vm::CellSlice& cs) const { switch (get_tag(cs)) { @@ -2038,7 +2030,7 @@ bool OutMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return false; } -bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const { +bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs, int global_version) const { auto tag = get_tag(cs); switch (tag) { case msg_export_ext: // external outbound message carries no value @@ -2071,7 +2063,7 @@ bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const { td::RefInt256 value_grams, ihr_fee, fwd_fee_remaining; return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) && (value_grams = t_Grams.as_integer_skip(msg_info.value.write())).not_null() && - (ihr_fee = t_Grams.as_integer(std::move(msg_info.ihr_fee))).not_null() && + (ihr_fee = get_ihr_fee(msg_info, global_version)).not_null() && (fwd_fee_remaining = t_Grams.as_integer(out_msg.fwd_fee_remaining)).not_null() && t_Grams.store_integer_ref(cb, value_grams + ihr_fee + fwd_fee_remaining) && cb.append_cellslice_bool(std::move(msg_info.value)); @@ -2112,8 +2104,8 @@ bool OutMsg::get_emitted_lt(vm::CellSlice& cs, unsigned long long& emitted_lt) c const OutMsg t_OutMsg; -const Aug_OutMsgDescr aug_OutMsgDescr; -const OutMsgDescr t_OutMsgDescr; +const Aug_OutMsgDescr aug_OutMsgDescrDefault(ton::SUPPORTED_VERSION); +const OutMsgDescr t_OutMsgDescrDefault(ton::SUPPORTED_VERSION); bool EnqueuedMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return cs.advance(64) && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak); diff --git a/crypto/block/block-parse.h b/crypto/block/block-parse.h index fd17c6579f..2eb8c5d778 100644 --- a/crypto/block/block-parse.h +++ b/crypto/block/block-parse.h @@ -405,7 +405,7 @@ struct CommonMsgInfo final : TLB_Complex { struct CommonMsgInfo::Record_int_msg_info { bool ihr_disabled, bounce, bounced; - Ref src, dest, value, ihr_fee, fwd_fee; + Ref src, dest, value, extra_flags, fwd_fee; unsigned long long created_lt; unsigned created_at; }; @@ -604,7 +604,7 @@ extern const Aug_ShardAccounts aug_ShardAccounts; struct ShardAccounts final : TLB_Complex { HashmapAugE dict_type; - ShardAccounts() : dict_type(256, aug_ShardAccounts){}; + ShardAccounts() : dict_type(256, aug_ShardAccounts) {}; bool skip(vm::CellSlice& cs) const override { return dict_type.skip(cs); } @@ -814,7 +814,7 @@ struct InMsg final : TLB_Complex { } return (int)cs.prefetch_ulong(5) - 0b00100 + 8; } - bool get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const; + bool get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs, int global_version) const; }; extern const InMsg t_InMsg; @@ -844,7 +844,7 @@ struct OutMsg final : TLB_Complex { } return t; } - bool get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const; + bool get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs, int global_version) const; bool get_emitted_lt(vm::CellSlice& cs, unsigned long long& emitted_lt) const; }; @@ -853,18 +853,22 @@ extern const OutMsg t_OutMsg; // next: InMsgDescr, OutMsgDescr, OutMsgQueue, and their augmentations struct Aug_InMsgDescr final : AugmentationCheckData { - Aug_InMsgDescr() : AugmentationCheckData(t_InMsg, t_ImportFees) { + explicit Aug_InMsgDescr(int global_version) + : AugmentationCheckData(t_InMsg, t_ImportFees), global_version(global_version) { } bool eval_leaf(vm::CellBuilder& cb, vm::CellSlice& cs) const override { - return t_InMsg.get_import_fees(cb, cs); + return t_InMsg.get_import_fees(cb, cs, global_version); } + int global_version; }; -extern const Aug_InMsgDescr aug_InMsgDescr; +extern const Aug_InMsgDescr aug_InMsgDescrDefault; struct InMsgDescr final : TLB_Complex { + Aug_InMsgDescr aug; HashmapAugE dict_type; - InMsgDescr() : dict_type(256, aug_InMsgDescr){}; + explicit InMsgDescr(int global_version) : aug(global_version), dict_type(256, aug) { + } bool skip(vm::CellSlice& cs) const override { return dict_type.skip(cs); } @@ -873,21 +877,25 @@ struct InMsgDescr final : TLB_Complex { } }; -extern const InMsgDescr t_InMsgDescr; +extern const InMsgDescr t_InMsgDescrDefault; struct Aug_OutMsgDescr final : AugmentationCheckData { - Aug_OutMsgDescr() : AugmentationCheckData(t_OutMsg, t_CurrencyCollection) { + explicit Aug_OutMsgDescr(int global_version) + : AugmentationCheckData(t_OutMsg, t_CurrencyCollection), global_version(global_version) { } bool eval_leaf(vm::CellBuilder& cb, vm::CellSlice& cs) const override { - return t_OutMsg.get_export_value(cb, cs); + return t_OutMsg.get_export_value(cb, cs, global_version); } + int global_version; }; -extern const Aug_OutMsgDescr aug_OutMsgDescr; +extern const Aug_OutMsgDescr aug_OutMsgDescrDefault; struct OutMsgDescr final : TLB_Complex { + Aug_OutMsgDescr aug; HashmapAugE dict_type; - OutMsgDescr() : dict_type(256, aug_OutMsgDescr){}; + explicit OutMsgDescr(int global_version) : aug(global_version), dict_type(256, aug) { + } bool skip(vm::CellSlice& cs) const override { return dict_type.skip(cs); } @@ -896,7 +904,7 @@ struct OutMsgDescr final : TLB_Complex { } }; -extern const OutMsgDescr t_OutMsgDescr; +extern const OutMsgDescr t_OutMsgDescrDefault; struct EnqueuedMsg final : TLB_Complex { int get_size(const vm::CellSlice& cs) const override { @@ -935,7 +943,7 @@ extern const Aug_DispatchQueue aug_DispatchQueue; struct OutMsgQueue final : TLB_Complex { HashmapAugE dict_type; - OutMsgQueue() : dict_type(32 + 64 + 256, aug_OutMsgQueue){}; + OutMsgQueue() : dict_type(32 + 64 + 256, aug_OutMsgQueue) {}; bool skip(vm::CellSlice& cs) const override { return dict_type.skip(cs); } @@ -1138,8 +1146,8 @@ struct Aug_ShardFees final : AugmentationCheckData { extern const Aug_ShardFees aug_ShardFees; // Validate dict of libraries in message: used when sending and receiving message -bool validate_message_libs(const td::Ref &cell); -bool validate_message_relaxed_libs(const td::Ref &cell); +bool validate_message_libs(const td::Ref& cell); +bool validate_message_relaxed_libs(const td::Ref& cell); } // namespace tlb } // namespace block diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 1967ca0239..9b4af3c1e0 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -125,7 +125,7 @@ currencies$_ grams:Grams other:ExtraCurrencyCollection // int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool src:MsgAddressInt dest:MsgAddressInt - value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + value:CurrencyCollection extra_flags:(VarUInteger 16) fwd_fee:Grams created_lt:uint64 created_at:uint32 = CommonMsgInfo; ext_in_msg_info$10 src:MsgAddressExt dest:MsgAddressInt import_fee:Grams = CommonMsgInfo; @@ -134,7 +134,7 @@ ext_out_msg_info$11 src:MsgAddressInt dest:MsgAddressExt int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool src:MsgAddress dest:MsgAddressInt - value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + value:CurrencyCollection extra_flags:(VarUInteger 16) fwd_fee:Grams created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; ext_out_msg_info$11 src:MsgAddress dest:MsgAddressExt created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; @@ -162,6 +162,16 @@ message$_ {X:Type} info:CommonMsgInfoRelaxed _ (Message Any) = MessageAny; + +_ value:CurrencyCollection created_lt:uint64 created_at:uint32 = NewBounceOriginalInfo; +_ gas_used:uint32 vm_steps:uint32 = NewBounceComputePhaseInfo; +new_bounce_body#fffffffe + original_body:(Maybe ^Cell) + original_info:^NewBounceOriginalInfo + bounced_by_phase:uint8 exit_code:int32 + compute_phase:(Maybe NewBounceComputePhaseInfo) + = NewBounceBody; + // interm_addr_regular$0 use_dest_bits:(#<= 96) = IntermediateAddress; diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index e751cbd999..b5f13b625d 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -117,7 +117,7 @@ bool Account::set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr) { /** * Sets the length of anycast prefix length in the account address. * - * @param new_length The new rewrite lingth. + * @param new_length The new rewrite length. * * @returns True if the length was successfully set, False otherwise. */ @@ -872,7 +872,16 @@ bool Transaction::unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* } bounce_enabled = in_msg_info.bounce; in_msg_type = 1; - td::RefInt256 ihr_fee = block::tlb::t_Grams.as_integer(in_msg_info.ihr_fee); + td::RefInt256 ihr_fee; + if (cfg->global_version >= 12) { + ihr_fee = td::zero_refint(); + td::RefInt256 extra_flags = tlb::t_Grams.as_integer(in_msg_info.extra_flags); + new_bounce_format = extra_flags->get_bit(0); + new_bounce_format_with_body = extra_flags->get_bit(1); + } else { + // Legacy: extra_flags was previously ihr_fee + ihr_fee = tlb::t_Grams.as_integer(in_msg_info.extra_flags); + } if (ihr_delivered) { in_fwd_fee = std::move(ihr_fee); } else { @@ -1277,7 +1286,7 @@ namespace transaction { * Checks if it is required to increase gas_limit (from GasLimitsPrices config) for the transaction * * In January 2024 a highload wallet of @wallet Telegram bot in mainnet was stuck because current gas limit (1M) is - * not enough to clean up old queires, thus locking funds inside. + * not enough to clean up old queries, thus locking funds inside. * See comment in crypto/smartcont/highload-wallet-v2-code.fc for details on why this happened. * Account address: EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu * It was proposed to validators to increase gas limit for this account to 70M for a limited amount @@ -1507,7 +1516,7 @@ Ref Transaction::prepare_vm_c7(const ComputePhaseConfig& cfg) const { // The only context where PrevBlocksInfo (13 parameter of c7) is null is inside emulator // where it need to be set via transaction_emulator_set_prev_blocks_info (see emulator/emulator-extern.cpp) // Inside validator, collator and liteserver checking external message contexts - // prev_blocks_info is always not null, since get_prev_blocks_info() + // prev_blocks_info is always not null, since get_prev_blocks_info() // may only return tuple or raise Error (See crypto/block/mc-config.cpp#2223) tuple.push_back(vm::StackEntry::maybe(cfg.prev_blocks_info)); } @@ -2677,8 +2686,8 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (cfg.disable_custom_fess) { fwd_fee = ihr_fee = td::zero_refint(); } else { - fwd_fee = block::tlb::t_Grams.as_integer(info.fwd_fee); - ihr_fee = block::tlb::t_Grams.as_integer(info.ihr_fee); + fwd_fee = tlb::t_Grams.as_integer(info.fwd_fee); + ihr_fee = cfg.global_version >= 12 ? td::zero_refint() : tlb::t_Grams.as_integer(info.extra_flags); } if (cfg.disable_ihr_flag) { info.ihr_disabled = true; @@ -2916,7 +2925,9 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, // re-pack message value CHECK(req.pack_to(info.value)); CHECK(block::tlb::t_Grams.pack_integer(info.fwd_fee, fwd_fee_remain)); - CHECK(block::tlb::t_Grams.pack_integer(info.ihr_fee, ihr_fee)); + if (cfg.global_version < 12) { + CHECK(block::tlb::t_Grams.pack_integer(info.extra_flags, ihr_fee)); + } // serialize message CHECK(tlb::csr_pack(msg.info, info)); @@ -3250,8 +3261,8 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { } bounce_phase = std::make_unique(); BouncePhase& bp = *bounce_phase; - block::gen::Message::Record msg; - block::gen::CommonMsgInfo::Record_int_msg_info info; + gen::Message::Record msg; + gen::CommonMsgInfo::Record_int_msg_info info; auto cs = vm::load_cell_slice(in_msg); if (!(tlb::unpack(cs, info) && gen::t_Maybe_Either_StateInit_Ref_StateInit.skip(cs) && cs.have(1) && cs.have_refs((int)cs.prefetch_ulong(1)))) { @@ -3261,6 +3272,45 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { if (cs.fetch_ulong(1)) { cs = vm::load_cell_slice(cs.prefetch_ref()); } + + vm::CellBuilder body; + if (new_bounce_format) { + body.store_long(0xfffffffeU, 32); // new_bounce_body#fffffffe + if (new_bounce_format_with_body) { // original_body:(Maybe ^Cell) + body.store_long(1, 1); + body.store_ref(vm::CellBuilder().append_cellslice(in_msg_body).finalize_novm()); + } else { + body.store_long(0, 1); + } + body.store_ref(vm::CellBuilder() + .append_cellslice(in_msg_info.value) // value:CurrencyCollection + .store_long(in_msg_info.created_lt, 64) // created_lt:uint64 + .store_long(in_msg_info.created_at, 32) // created_at:uint32 + .finalize_novm()); // original_info:^NewBounceOriginalInfo + if (compute_phase->skip_reason != ComputePhase::sk_none) { + body.store_long(0, 8); // bounced_by_phase:uint8 + body.store_long(compute_phase->skip_reason - 1, 32); // exit_code:int32 + } else if (!compute_phase->success) { + body.store_long(1, 8); // bounced_by_phase:uint8 + body.store_long(compute_phase->exit_code, 32); // exit_code:int32 + } else { + body.store_long(2, 8); // bounced_by_phase:uint8 + body.store_long(action_phase->result_code, 32); // exit_code:int32 + } + // compute_phase:(Maybe NewBounceComputePhaseInfo) + if (compute_phase->skip_reason != ComputePhase::sk_none) { + body.store_long(0, 1); + } else { + body.store_long(1, 1); + body.store_long(compute_phase->gas_used, 32); // gas_used:uint32 + body.store_long(compute_phase->vm_steps, 32); // vm_steps:uint32 + } + } else if (cfg.bounce_msg_body) { + int body_bits = std::min((int)cs.size(), cfg.bounce_msg_body); + body.store_long_bool(-1, 32); // 0xffffffff tag + body.append_bitslice(cs.prefetch_bits(body_bits)); // truncated message body + } + info.ihr_disabled = true; info.bounce = false; info.bounced = true; @@ -3276,7 +3326,8 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { // compute size of message vm::CellStorageStat sstat; // for message size // preliminary storage estimation of the resulting message - sstat.compute_used_storage(info.value->prefetch_ref()); + sstat.add_used_storage(info.value->prefetch_ref()); + sstat.add_used_storage(body.get_refs()); bp.msg_bits = sstat.bits; bp.msg_cells = sstat.cells; // compute forwarding fees @@ -3312,26 +3363,17 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { && cb.append_cellslice_bool(info.src) // src:MsgAddressInt && cb.append_cellslice_bool(info.dest) // dest:MsgAddressInt && msg_balance.store(cb) // value:CurrencyCollection - && block::tlb::t_Grams.store_long(cb, 0) // ihr_fee:Grams + && block::tlb::t_Grams.store_long(cb, 0) // extra_flags:(VarUInteger 16) && block::tlb::t_Grams.store_long(cb, bp.fwd_fees) // fwd_fee:Grams && cb.store_long_bool(info.created_lt, 64) // created_lt:uint64 && cb.store_long_bool(info.created_at, 32) // created_at:uint32 && cb.store_bool_bool(false)); // init:(Maybe ...) - if (cfg.bounce_msg_body) { - int body_bits = std::min((int)cs.size(), cfg.bounce_msg_body); - if (cb.remaining_bits() >= body_bits + 33u) { - CHECK(cb.store_bool_bool(false) // body:(Either X ^X) -> left X - && cb.store_long_bool(-1, 32) // int = -1 ("message type") - && cb.append_bitslice(cs.prefetch_bits(body_bits))); // truncated message body - } else { - vm::CellBuilder cb2; - CHECK(cb.store_bool_bool(true) // body:(Either X ^X) -> right ^X - && cb2.store_long_bool(-1, 32) // int = -1 ("message type") - && cb2.append_bitslice(cs.prefetch_bits(body_bits)) // truncated message body - && cb.store_builder_ref_bool(std::move(cb2))); // ^X - } + if (cb.can_extend_by(1 + body.size(), body.size_refs())) { + // body:(Either X ^X) -> left X + CHECK(cb.store_bool_bool(false) && cb.append_builder_bool(body)); } else { - CHECK(cb.store_bool_bool(false)); // body:(Either ..) + // body:(Either X ^X) -> right ^X + CHECK(cb.store_bool_bool(true) && cb.store_builder_ref_bool(std::move(body))); } CHECK(cb.finalize_to(bp.out_msg)); if (verbosity > 2) { diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 5492ee6bf2..e987645db0 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -196,7 +196,7 @@ struct CreditPhase { }; struct ComputePhase { - enum { sk_none, sk_no_state, sk_bad_state, sk_no_gas, sk_suspended }; + enum { sk_none = 0, sk_no_state = 1, sk_bad_state = 2, sk_no_gas = 3, sk_suspended = 4 }; int skip_reason{sk_none}; bool success{false}; bool msg_state_used{false}; @@ -358,6 +358,8 @@ struct Transaction { bool bounce_enabled{false}; bool in_msg_extern{false}; gen::CommonMsgInfo::Record_int_msg_info in_msg_info; + bool new_bounce_format{false}; + bool new_bounce_format_with_body{false}; bool use_msg_state{false}; bool is_first{false}; bool orig_addr_rewrite_set{false}; diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index 265c602d79..a89cfa6d95 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -61,9 +61,9 @@ td::Ref build_internal_message(td::RefInt256 amount, td::Refbit_size(false) + 7) >> 3); b.store_long_bool(len, 4) && b.store_int256_bool(*amount, len * 8, false); // grams:Grams - b.store_zeroes(1 + 4 + 4 + 64 + 32 + 1); // extre, ihr_fee, fwd_fee, created_lt, created_at, init + b.store_zeroes(1 + 4 + 4 + 64 + 32 + 1); // extra currencies, extra_flags, fwd_fee, created_lt, created_at, init // body:(Either X ^X) - if (b.remaining_bits() >= 1 + (*body).size() && b.remaining_refs() >= (*body).size_refs()) { + if (b.remaining_bits() >= 1 + body->size() && b.remaining_refs() >= body->size_refs()) { b.store_zeroes(1); b.append_cellslice(body); } else { diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 837624563a..93fc0a89d9 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -1156,6 +1156,16 @@ td::Result CellStorageStat::add_used_storage(Ref CellStorageStat::add_used_storage(td::Span> cells, bool kill_dup, + unsigned skip_count_root) { + CellInfo result; + for (const auto& cell : cells) { + TRY_RESULT(info, add_used_storage(cell, kill_dup, skip_count_root)); + result.max_merkle_depth = std::max(result.max_merkle_depth, info.max_merkle_depth); + } + return result; +} + void NewCellStorageStat::add_cell(Ref cell) { dfs(std::move(cell), true, false); } diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index fbf395538d..5bf45c9224 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -143,6 +143,8 @@ struct CellStorageStat { td::Result add_used_storage(const CellSlice& cs, bool kill_dup = true, unsigned skip_count_root = 0); td::Result add_used_storage(CellSlice&& cs, bool kill_dup = true, unsigned skip_count_root = 0); td::Result add_used_storage(Ref cell, bool kill_dup = true, unsigned skip_count_root = 0); + td::Result add_used_storage(td::Span> cells, bool kill_dup = true, + unsigned skip_count_root = 0); unsigned long long limit_cells = std::numeric_limits::max(); unsigned long long limit_bits = std::numeric_limits::max(); diff --git a/crypto/vm/cells/CellBuilder.h b/crypto/vm/cells/CellBuilder.h index 954a1ac084..4c701c7d2d 100644 --- a/crypto/vm/cells/CellBuilder.h +++ b/crypto/vm/cells/CellBuilder.h @@ -85,6 +85,9 @@ class CellBuilder : public td::CntObject { Ref get_ref(unsigned idx) const { return idx < refs_cnt ? refs[idx] : Ref{}; } + td::Span> get_refs() const { + return {refs.data(), refs_cnt}; + } void reset(); bool reset_bool() { reset(); diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index 981f7c5b33..d74ecb17f2 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -1826,6 +1826,7 @@ int exec_send_message(VmState* st) { Ref dest; td::RefInt256 value; td::RefInt256 user_fwd_fee, user_ihr_fee; + unsigned extra_flags_len = 0; bool have_extra_currencies = false; bool ext_msg = msg.info->prefetch_ulong(1); if (ext_msg) { // External message @@ -1843,17 +1844,23 @@ int exec_send_message(VmState* st) { } ihr_disabled = info.ihr_disabled || st->get_global_version() >= 11; dest = std::move(info.dest); - Ref extra; + Ref extra; if (!block::tlb::t_CurrencyCollection.unpack_special(info.value.write(), value, extra)) { throw VmError{Excno::unknown, "invalid message"}; } have_extra_currencies = !extra.is_null(); user_fwd_fee = block::tlb::t_Grams.as_integer(info.fwd_fee); - user_ihr_fee = block::tlb::t_Grams.as_integer(info.ihr_fee); + if (st->get_global_version() >= 12) { + user_ihr_fee = td::zero_refint(); + extra_flags_len = info.extra_flags->size(); + } else { + // Legacy: extra_flags was previously ihr_fee + user_ihr_fee = block::tlb::t_Grams.as_integer(info.extra_flags); + } } bool is_masterchain = parse_addr_workchain(*my_addr) == -1 || (!ext_msg && parse_addr_workchain(*dest) == -1); - td::Ref prices_cs; + Ref prices_cs; if (st->get_global_version() >= 6) { prices_cs = tuple_index(get_unpacked_config_tuple(st), is_masterchain ? 4 : 5).as_slice(); } else { @@ -1886,7 +1893,7 @@ int exec_send_message(VmState* st) { } else { max_cells = 1 << 13; } - vm::VmStorageStat stat(max_cells); + VmStorageStat stat(max_cells); CellSlice cs = load_cell_slice(msg_cell); cs.skip_first(cs.size()); if (st->get_global_version() >= 10 && have_extra_currencies) { @@ -1967,7 +1974,8 @@ int exec_send_message(VmState* st) { bits = 4 + my_addr->size() + dest->size() + stored_grams_len(value) + 1 + 32 + 64; td::RefInt256 fwd_fee_first = (fwd_fee * prices.first_frac) >> 16; bits += stored_grams_len(fwd_fee - fwd_fee_first); - bits += stored_grams_len(ihr_fee); + // Legacy: extra_flags was previously ihr_fee + bits += st->get_global_version() >= 12 ? extra_flags_len : stored_grams_len(ihr_fee); } // init bits++; diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 6281735e08..be0fe380b0 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -231,12 +231,45 @@ This is required to help computing storage stats in the future, after collator-v ## Version 12 +### Extra message flags and new bounce format +Field `ihr_fee:Grams` in internal message is now called `extra_flags:(VarUInteger 16)` (it's the same format). +This field does not represent fees. `ihr_fee` is always zero since version 11, so this field was essentially unused. + +`(extra_flags & 1) = 1` enables the new bounce format for the message. The bounced message contains information about the transaction. +If `(extra_flags & 3) = 3`, the bounced message also contains the whole body of the original message. +When the message with new bounce flag is bounced, the bounced message body has the following format (`new_bounce_body`): +``` +_ value:CurrencyCollection created_lt:uint64 created_at:uint32 = NewBounceOriginalInfo; +_ gas_used:uint32 vm_steps:uint32 = NewBounceComputePhaseInfo; + +new_bounce_body#fffffffe + original_body:(Maybe ^Cell) + original_info:^NewBounceOriginalInfo + bounced_by_phase:uint8 exit_code:int32 + compute_phase:(Maybe NewBounceComputePhaseInfo) + = NewBounceBody; +``` +- `original_body` - cell that contains the body of the original message (if `extra_flags & 2`) or nothing (if not `extra_flags & 2`). +- `original_info` - value, lt and unixtime of the original message. +- `bounced_by_phase`: + - `0` - compute phase was skipped. `exit_code` denotes the skip reason: + - `exit_code = 0` - no state (account is uninit or frozen, and no state init is present in the message). + - `exit_code = 1` - bad state (account is uninit or frozen, and state init in the message has the wrong hash). + - `exit_code = 2` - no gas. + - `exit_code = 3` - account is suspended. + - `1` - compute phase failed. `exit_code` is the value from the compute phase. + - `2` - action phase failed. `exit_code` is the value from the action phase. +- `exit_code` - 32-bit exit code, see above. +- `compute_phase` - exists if it was not skipped (`bounced_by_phase > 0`): + - `gas_used`, `vm_steps` - same as in `TrComputePhase` of the transaction. + ### New TVM instructions - `BTOS` (`b - s`) - same as `ENDC CTOS`, but without gas cost for cell creation and loading. Gas cost: `26`. - `HASHBU` (`b - hash`) - same as `ENDC HASHCU`, but without gas cost for cell creation. Gas cost: `26`. ### Other TVM changes - `HASHSU` (`s - hash`) now does not spend gas for cell creation. Gas cost: `26`. +- `SENDMSG` instruction treats `extra_flags` field accordingly (see above). ### Other changes - Account size in masterchain is now limited to `2048` cells. This can be configured in size limits config (`ConfigParam 43`). diff --git a/emulator/test/emulator-tests.cpp b/emulator/test/emulator-tests.cpp index ae273ddfde..4057f58161 100644 --- a/emulator/test/emulator-tests.cpp +++ b/emulator/test/emulator-tests.cpp @@ -221,7 +221,7 @@ TEST(Emulator, wallet_int_and_ext_msg) { { vm::CellBuilder cb; block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(0)); - msg_info.ihr_fee = cb.as_cellslice_ref(); + msg_info.extra_flags = cb.as_cellslice_ref(); } msg_info.created_lt = 0; msg_info.created_at = static_cast(utime); diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 7e868480b0..f95ac33bfa 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -3225,15 +3225,12 @@ struct ToRawTransactions { TRY_RESULT(src, to_std_address(msg_info.src)); TRY_RESULT(dest, to_std_address(msg_info.dest)); TRY_RESULT(fwd_fee, to_balance(msg_info.fwd_fee)); - TRY_RESULT(ihr_fee, to_balance(msg_info.ihr_fee)); auto created_lt = static_cast(msg_info.created_lt); return tonlib_api::make_object( - msg_hash, - tonlib_api::make_object(src), - tonlib_api::make_object(std::move(dest)), balance, - std::move(extra_currencies), fwd_fee, ihr_fee, created_lt, std::move(body_hash), - get_data(src)); + msg_hash, tonlib_api::make_object(src), + tonlib_api::make_object(std::move(dest)), balance, std::move(extra_currencies), + fwd_fee, /* ihr_fee = */ 0, created_lt, std::move(body_hash), get_data(src)); } case block::gen::CommonMsgInfo::ext_in_msg_info: { block::gen::CommonMsgInfo::Record_ext_in_msg_info msg_info; diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index fcc95caade..47c627fd76 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "block-parse.h" #include "interfaces/validator-manager.h" #include "shard.hpp" #include "top-shard-descr.hpp" @@ -156,6 +157,7 @@ class Collator final : public td::actor::Actor { ton::LogicalTime shards_max_end_lt_{0}; ton::UnixTime prev_state_utime_; int global_id_{0}; + int global_version_{0}; ton::BlockSeqno min_ref_mc_seqno_{~0U}; ton::BlockSeqno vert_seqno_{~0U}, prev_vert_seqno_{~0U}; ton::BlockIdExt prev_key_block_; @@ -203,6 +205,8 @@ class Collator final : public td::actor::Actor { std::vector ext_msg_list_; std::priority_queue, std::greater> new_msgs; std::pair last_proc_int_msg_, first_unproc_int_msg_; + block::tlb::Aug_InMsgDescr aug_InMsgDescr{0}; + block::tlb::Aug_OutMsgDescr aug_OutMsgDescr{0}; std::unique_ptr in_msg_dict, out_msg_dict, old_out_msg_queue_, out_msg_queue_, sibling_out_msg_queue_; std::map unprocessed_deferred_messages_; // number of messages from dispatch queue in new_msgs diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index acf9075b4a..401f721d17 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -778,6 +778,7 @@ bool Collator::unpack_last_mc_state() { CHECK(config_); config_->set_block_id_ext(mc_block_id_); global_id_ = config_->get_global_blockchain_id(); + global_version_ = config_->get_global_version(); ihr_enabled_ = config_->ihr_enabled(); create_stats_enabled_ = config_->create_stats_enabled(); report_version_ = config_->has_capability(ton::capReportVersion); @@ -2292,8 +2293,9 @@ bool Collator::do_collate() { return fatal_error("cannot fetch required configuration parameters from masterchain state"); } LOG(DEBUG) << "config parameters fetched, creating message dictionaries"; - in_msg_dict = std::make_unique(256, block::tlb::aug_InMsgDescr); - out_msg_dict = std::make_unique(256, block::tlb::aug_OutMsgDescr); + aug_InMsgDescr.global_version = aug_OutMsgDescr.global_version = global_version_; + in_msg_dict = std::make_unique(256, aug_InMsgDescr); + out_msg_dict = std::make_unique(256, aug_OutMsgDescr); LOG(DEBUG) << "message dictionaries created"; if (max_lt == start_lt) { ++max_lt; @@ -3118,7 +3120,7 @@ bool Collator::create_special_transaction(block::CurrencyCollection amount, Ref< && cb.store_long_bool(0x4ff, 11) // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 && cb.store_bits_bool(addr) // address:bits256 => dest:MsgAddressInt && amount.store(cb) // value:CurrencyCollection - && cb.store_zeroes_bool(4 + 4) // ihr_fee:Grams fwd_fee:Grams + && cb.store_zeroes_bool(4 + 4) // extra_flags:(VarUInteger 16) fwd_fee:Grams && cb.store_long_bool(lt, 64) // created_lt:uint64 && cb.store_long_bool(now_, 32) // created_at:uint32 && cb.store_zeroes_bool(2) // init:(Maybe ...) body:(Either X ^X) = Message X diff --git a/validator/impl/ihr-message.cpp b/validator/impl/ihr-message.cpp index 4327b5dd38..75cf08bc72 100644 --- a/validator/impl/ihr-message.cpp +++ b/validator/impl/ihr-message.cpp @@ -99,7 +99,7 @@ td::Result> IhrMessageQ::create_ihr_message(td::BufferSlice dat "block header in the Merkle proof of an IHR message does not belong to the declared source block"); } vm::AugmentedDictionary out_msg_dict{vm::load_cell_slice_ref(extra.out_msg_descr), 256, - block::tlb::aug_OutMsgDescr}; + block::tlb::aug_OutMsgDescrDefault}; Bits256 key{ihr_msg->get_hash().bits()}; auto descr = out_msg_dict.lookup(key); out_msg_dict.reset(); diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index e9592f6264..c369d1f53d 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -2390,7 +2390,8 @@ void LiteQuery::perform_listBlockTransactions(BlockIdExt blkid, int mode, int co static td::Result> get_in_msg_metadata( const Ref& in_msg_descr_root, const Ref& trans_root) { - vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, block::tlb::aug_InMsgDescr}; + vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, + block::tlb::aug_InMsgDescrDefault}; block::gen::Transaction::Record transaction; if (!block::tlb::unpack_cell(trans_root, transaction)) { return td::Status::Error("invalid Transaction in block"); @@ -2549,7 +2550,8 @@ void LiteQuery::perform_listBlockTransactionsExt(BlockIdExt blkid, int mode, int static td::Status process_all_in_msg_metadata(const Ref& in_msg_descr_root, const std::vector>& trans_roots) { - vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, block::tlb::aug_InMsgDescr}; + vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, + block::tlb::aug_InMsgDescrDefault}; for (const Ref& trans_root : trans_roots) { block::gen::Transaction::Record transaction; if (!block::tlb::unpack_cell(trans_root, transaction)) { diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 4bef90395c..fd822d2bc8 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -943,6 +943,7 @@ bool ValidateQuery::try_unpack_mc_state() { return reject_query(PSTRING() << "vertical seqno mismatch: new block has " << vert_seqno_ << " while the masterchain configuration expects " << config_->get_vert_seqno()); } + global_version_ = config_->get_global_version(); prev_key_block_exists_ = config_->get_last_key_block(prev_key_block_, prev_key_block_lt_); if (prev_key_block_exists_) { prev_key_block_seqno_ = prev_key_block_.seqno(); @@ -2704,17 +2705,19 @@ bool ValidateQuery::unpack_block_data() { auto outmsg_cs = vm::load_cell_slice_ref(std::move(extra.out_msg_descr)); // run some hand-written checks from block::tlb:: // (automatic tests from block::gen:: have been already run for the entire block) - if (!block::tlb::t_InMsgDescr.validate_upto(10000000, *inmsg_cs)) { + t_InMsgDescr.aug.global_version = global_version_; + t_OutMsgDescr.aug.global_version = global_version_; + if (!t_InMsgDescr.validate_upto(10000000, *inmsg_cs)) { return reject_query("InMsgDescr of the new block failed to pass handwritten validity tests"); } - if (!block::tlb::t_OutMsgDescr.validate_upto(10000000, *outmsg_cs)) { + if (!t_OutMsgDescr.validate_upto(10000000, *outmsg_cs)) { return reject_query("OutMsgDescr of the new block failed to pass handwritten validity tests"); } if (!block::tlb::t_ShardAccountBlocks.validate_ref(10000000, extra.account_blocks)) { return reject_query("ShardAccountBlocks of the new block failed to pass handwritten validity tests"); } - in_msg_dict_ = std::make_unique(std::move(inmsg_cs), 256, block::tlb::aug_InMsgDescr); - out_msg_dict_ = std::make_unique(std::move(outmsg_cs), 256, block::tlb::aug_OutMsgDescr); + in_msg_dict_ = std::make_unique(std::move(inmsg_cs), 256, t_InMsgDescr.aug); + out_msg_dict_ = std::make_unique(std::move(outmsg_cs), 256, t_OutMsgDescr.aug); account_blocks_dict_ = std::make_unique( vm::load_cell_slice_ref(std::move(extra.account_blocks)), 256, block::tlb::aug_ShardAccountBlocks); LOG(DEBUG) << "validating InMsgDescr"; @@ -3773,7 +3776,7 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) block::tlb::MsgEnvelope::Record_std env; // int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool // src:MsgAddressInt dest:MsgAddressInt - // value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + // value:CurrencyCollection extra_flags:(VarUInteger 16) fwd_fee:Grams // created_lt:uint64 created_at:uint32 = CommonMsgInfo; block::gen::CommonMsgInfo::Record_int_msg_info info; ton::AccountIdPrefixFull src_prefix, dest_prefix, cur_prefix, next_prefix; @@ -4335,7 +4338,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms block::tlb::MsgEnvelope::Record_std env; // int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool // src:MsgAddressInt dest:MsgAddressInt - // value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + // value:CurrencyCollection extra_flags:(VarUInteger 16) fwd_fee:Grams // created_lt:uint64 created_at:uint32 = CommonMsgInfo; block::gen::CommonMsgInfo::Record_int_msg_info info; ton::AccountIdPrefixFull src_prefix, dest_prefix, cur_prefix, next_prefix; @@ -5377,6 +5380,22 @@ std::unique_ptr ValidateQuery::unpack_account(td::ConstBitPtr ad return new_acc; } +/** + * Gets IHR fee of the internal message + * + * In earlier versions (before 12) the field extra_flags was ihr_fee. + * Since version 12 ihr_fee is always zero. + * + * @param info CommonMsgInfo of the internal message + * @param global_version global version from ConfigParam 8 + * + * @returns IHR fee + */ +static td::RefInt256 get_ihr_fee(const block::gen::CommonMsgInfo::Record_int_msg_info &info, int global_version) { + // Legacy: extra_flags was previously ihr_fee + return global_version >= 12 ? td::zero_refint() : block::tlb::t_Grams.as_integer(std::move(info.extra_flags)); +} + /** * Checks the validity of a single transaction for a given account. * Performs transaction execution. @@ -5464,7 +5483,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT CHECK(money_imported.validate_unpack(info.value)); ihr_delivered = (in_msg_tag == block::gen::InMsg::msg_import_ihr); if (!ihr_delivered) { - money_imported += block::tlb::t_Grams.as_integer(info.ihr_fee); + money_imported += get_ihr_fee(info, global_version_); } CHECK(money_imported.is_valid()); } @@ -5533,7 +5552,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT // unpack exported message value (from this transaction) block::CurrencyCollection msg_export_value; CHECK(msg_export_value.unpack(info.value)); - msg_export_value += block::tlb::t_Grams.as_integer(info.ihr_fee); + msg_export_value += get_ihr_fee(info, global_version_); msg_export_value += msg_env.fwd_fee_remaining; CHECK(msg_export_value.is_valid()); money_exported += msg_export_value; @@ -6145,9 +6164,12 @@ bool ValidateQuery::check_special_message(Ref in_msg_root, const block if (block::tlb::t_Grams.as_integer(info.fwd_fee)->sgn()) { return reject_query("special message with hash "s + msg_hash.to_hex() + " has a non-zero fwd_fee"); } - if (block::tlb::t_Grams.as_integer(info.ihr_fee)->sgn()) { + if (get_ihr_fee(info, global_version_)->sgn()) { return reject_query("special message with hash "s + msg_hash.to_hex() + " has a non-zero ihr_fee"); } + if (block::tlb::t_Grams.as_integer(info.extra_flags)->sgn()) { + return reject_query("special message with hash "s + msg_hash.to_hex() + " has a non-zero extra_flags"); + } block::CurrencyCollection value; if (!value.validate_unpack(info.value)) { return reject_query("special message with hash "s + msg_hash.to_hex() + " has an invalid value"); diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 95e81cdec1..f172de95d5 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -18,6 +18,7 @@ */ #pragma once +#include "block-parse.h" #include "interfaces/validator-manager.h" #include "vm/cells.h" #include "vm/dict.h" @@ -190,6 +191,7 @@ class ValidateQuery : public td::actor::Actor { ton::LogicalTime max_shard_lt_{0}; int global_id_{0}; + int global_version_{0}; ton::BlockSeqno vert_seqno_{~0U}; bool ihr_enabled_{false}; bool create_stats_enabled_{false}; @@ -222,6 +224,8 @@ class ValidateQuery : public td::actor::Actor { std::map block_create_count_; unsigned block_create_total_{0}; + block::tlb::InMsgDescr t_InMsgDescr{0}; + block::tlb::OutMsgDescr t_OutMsgDescr{0}; std::unique_ptr in_msg_dict_, out_msg_dict_, account_blocks_dict_; block::ValueFlow value_flow_; block::CurrencyCollection import_created_, transaction_fees_, total_burned_{0}, fees_burned_{0}; From 8a971be7fee91636bb36d816b7ac0c4e14d57ef7 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Fri, 25 Jul 2025 12:35:28 +0300 Subject: [PATCH 354/524] Update changelog --- Changelog.md | 4 ++++ recent_changelog.md | 12 ++---------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Changelog.md b/Changelog.md index d2f940c00c..b104d10ad2 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,7 @@ +## 2025.07 Accelerator Update + +Separation of validation and collation processes that allows to host them on independent machines and achieve full horizontal scaling. [More details in documentation](https://docs.ton.org/v3/documentation/infra/nodes/validation/collators) + ## 2025.06 Update 1. ADNL and candidate broadcast optimization diff --git a/recent_changelog.md b/recent_changelog.md index a178f38fa6..6b65d21836 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,11 +1,3 @@ -## 2025.06 Update - -1. ADNL and candidate broadcast optimization -2. [TVM version v11](./doc/GlobalVersions.md): new opcodes, and `c7` entry to improve developer experience. It also activates storage stats and `ihr_fee` nullification. -3. Fixed `start_lt` of tick transactions [see details on 01.06.2025 incident](https://telegra.ph/Report-on-June-1-2025-Operation-Incident-06-02). -4. Introduction of persistent state sharding, as well as making serialization of large BOCs more deterministic -5. Emulator improvements: in get methods, set config from provided `c7`; allow retrieval of logs from emulator runs for get methods -6. Optimized package import for archive nodes - -Besides the work of the core team, this update is based on the efforts of the RSquad team (deterministic large BOC serialization); AArayz, wy666444, Robinlzw, Lucian-code233 from TonBit (early discovery of the TVM 11 bug); @Skydev0h (uninitialized `BLOCKLT` in get methods); and @yma-het from TONWhales (emulator improvements). +## 2025.07 Accelerator Update +Separation of validation and collation processes that allows to host them on independent machines and achieve full horizontal scaling. [More details in documentation](https://docs.ton.org/v3/documentation/infra/nodes/validation/collators) From 273663701cb202dc7fbed5fc437f26947919c26f Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 25 Jul 2025 14:16:50 +0300 Subject: [PATCH 355/524] Change exit codes for compute phase skip reason in bounce msg --- crypto/block/transaction.cpp | 4 ++-- doc/GlobalVersions.md | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index b5f13b625d..8d220c7b98 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -3288,8 +3288,8 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { .store_long(in_msg_info.created_at, 32) // created_at:uint32 .finalize_novm()); // original_info:^NewBounceOriginalInfo if (compute_phase->skip_reason != ComputePhase::sk_none) { - body.store_long(0, 8); // bounced_by_phase:uint8 - body.store_long(compute_phase->skip_reason - 1, 32); // exit_code:int32 + body.store_long(0, 8); // bounced_by_phase:uint8 + body.store_long(-compute_phase->skip_reason, 32); // exit_code:int32 } else if (!compute_phase->success) { body.store_long(1, 8); // bounced_by_phase:uint8 body.store_long(compute_phase->exit_code, 32); // exit_code:int32 diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index be0fe380b0..6e59955c4e 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -253,10 +253,10 @@ new_bounce_body#fffffffe - `original_info` - value, lt and unixtime of the original message. - `bounced_by_phase`: - `0` - compute phase was skipped. `exit_code` denotes the skip reason: - - `exit_code = 0` - no state (account is uninit or frozen, and no state init is present in the message). - - `exit_code = 1` - bad state (account is uninit or frozen, and state init in the message has the wrong hash). - - `exit_code = 2` - no gas. - - `exit_code = 3` - account is suspended. + - `exit_code = -1` - no state (account is uninit or frozen, and no state init is present in the message). + - `exit_code = -2` - bad state (account is uninit or frozen, and state init in the message has the wrong hash). + - `exit_code = -3` - no gas. + - `exit_code = -4` - account is suspended. - `1` - compute phase failed. `exit_code` is the value from the compute phase. - `2` - action phase failed. `exit_code` is the value from the action phase. - `exit_code` - 32-bit exit code, see above. From 8c7a5a67dbc689f4c5382e952104544b33b4df24 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 31 Jul 2025 10:00:49 +0300 Subject: [PATCH 356/524] "Skip public msg send" flag for custom overlays (#1757) --- tl/generate/scheme/ton_api.tl | 2 +- tl/generate/scheme/ton_api.tlo | Bin 116068 -> 116116 bytes .../validator-engine-console-query.cpp | 3 +++ validator/full-node.cpp | 20 ++++++++++++------ validator/full-node.h | 1 + 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 7db6ac0e73..ebfd25defd 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -691,7 +691,7 @@ engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector eng engine.validator.customOverlayNode adnl_id:int256 msg_sender:Bool msg_sender_priority:int block_sender:Bool = engine.validator.CustomOverlayNode; engine.validator.customOverlay name:string nodes:(vector engine.validator.customOverlayNode) sender_shards:(vector tonNode.shardId) - = engine.validator.CustomOverlay; + skip_public_msg_send:Bool = engine.validator.CustomOverlay; engine.validator.customOverlaysConfig overlays:(vector engine.validator.customOverlay) = engine.validator.CustomOverlaysConfig; engine.validator.collatorOptions diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index b4d84dd96804cee1ac61c758da3f1a1307cfb32f..8c13ffcb66d6296e175a89e22c09305c8f8267c7 100644 GIT binary patch delta 123 zcmaFT#XhB*eS?4m>*A7I-{mF;O2}e-cZ0L+>` Op^8yqyG%8shZg{@!ZjrT delta 92 zcmbQz&Hki|eS?4m>)D!=C32GkCFC|6OPDTT0dXhi>`(wRrhi~%)R}%jpHXG1| hdmeItg*T_{3}J+@_Uy_z08x6Nicw*^Pc@^57XScQC$Rtk diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index eaece0e874..15f1264221 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1294,6 +1294,9 @@ td::Status ShowCustomOverlaysQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << " " << ton::create_shard_id(shard).to_str() << "\n"; } } + if (overlay->skip_public_msg_send_) { + td::TerminalIO::out() << "Don't send external messages to public overlays\n"; + } td::TerminalIO::out() << "\n"; } return td::Status::OK(); diff --git a/validator/full-node.cpp b/validator/full-node.cpp index df91237619..4e75ddede7 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -318,21 +318,28 @@ void FullNodeImpl::send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice dat } void FullNodeImpl::send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) { - auto shard = get_shard(dst); - if (shard.empty()) { - VLOG(FULL_NODE_WARNING) << "dropping OUT ext message to unknown shard"; - return; - } + bool skip_public = false; for (auto &[_, private_overlay] : custom_overlays_) { if (private_overlay.params_.send_shard(dst.as_leaf_shard())) { for (auto &[local_id, actor] : private_overlay.actors_) { if (private_overlay.params_.msg_senders_.contains(local_id)) { td::actor::send_closure(actor, &FullNodeCustomOverlay::send_external_message, data.clone()); + if (private_overlay.params_.skip_public_msg_send_) { + skip_public = true; + } } } } } - td::actor::send_closure(shard, &FullNodeShard::send_external_message, std::move(data)); + + if (!skip_public) { + auto shard = get_shard(dst); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping OUT ext message to unknown shard"; + return; + } + td::actor::send_closure(shard, &FullNodeShard::send_external_message, std::move(data)); + } } void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { @@ -964,6 +971,7 @@ CustomOverlayParams CustomOverlayParams::fetch(const ton_api::engine_validator_c for (const auto &shard : f.sender_shards_) { c.sender_shards_.push_back(create_shard_id(shard)); } + c.skip_public_msg_send_ = f.skip_public_msg_send_; return c; } diff --git a/validator/full-node.h b/validator/full-node.h index dc29f67ca2..4c11c1fbd8 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -67,6 +67,7 @@ struct CustomOverlayParams { std::map msg_senders_; std::set block_senders_; std::vector sender_shards_; + bool skip_public_msg_send_ = false; bool send_shard(const ShardIdFull& shard) const; static CustomOverlayParams fetch(const ton_api::engine_validator_customOverlay& f); From 7d1995ec0a03bac280bdf63c91f005696d3ab816 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Thu, 31 Jul 2025 10:01:13 +0300 Subject: [PATCH 357/524] Improve portable linux binaries (#1759) * add libgslcblas into portable binaries * remove commented line * fix portable linux binaries --- .github/workflows/build-ton-linux-arm64-appimage.yml | 3 +-- .github/workflows/build-ton-linux-x86-64-appimage.yml | 2 +- assembly/appimage/create-appimages.sh | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-ton-linux-arm64-appimage.yml b/.github/workflows/build-ton-linux-arm64-appimage.yml index a0ba4a022c..c8147677e1 100644 --- a/.github/workflows/build-ton-linux-arm64-appimage.yml +++ b/.github/workflows/build-ton-linux-arm64-appimage.yml @@ -21,8 +21,7 @@ jobs: - name: Install system libraries run: | sudo apt update - sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache - sudo apt remove libgsl-dev + sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache libgsl-dev libblas-dev libgslcblas0 mkdir ~/.ccache 3pp - name: Install clang-16 diff --git a/.github/workflows/build-ton-linux-x86-64-appimage.yml b/.github/workflows/build-ton-linux-x86-64-appimage.yml index 7ad873d22e..cb02b7bc20 100644 --- a/.github/workflows/build-ton-linux-x86-64-appimage.yml +++ b/.github/workflows/build-ton-linux-x86-64-appimage.yml @@ -21,7 +21,7 @@ jobs: - name: Install system libraries run: | sudo apt update - sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache libgsl-dev libblas-dev + sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache libgsl-dev libblas-dev libgslcblas0 mkdir ~/.ccache 3pp - name: Install clang-16 diff --git a/assembly/appimage/create-appimages.sh b/assembly/appimage/create-appimages.sh index 7bd68b6fbf..f6e2690727 100644 --- a/assembly/appimage/create-appimages.sh +++ b/assembly/appimage/create-appimages.sh @@ -27,7 +27,7 @@ for file in ../artifacts/*; do printf '[Desktop Entry]\nName='$appName'\nExec='$appName'\nIcon='$appName'\nType=Application\nCategories=Utility;\n' > $appName.AppDir/$appName.desktop cp ../ton.png $appName.AppDir/$appName.png cp $file $appName.AppDir/usr/bin/ - cp ../build/openssl_3/libcrypto.so.3 \ + cp ../openssl_3/libcrypto.so.3 \ /lib/$ARCH-linux-gnu/libatomic.so.1 \ /lib/$ARCH-linux-gnu/libsodium.so.23 \ /lib/$ARCH-linux-gnu/libz.so.1 \ @@ -37,6 +37,7 @@ for file in ../artifacts/*; do /lib/$ARCH-linux-gnu/libstdc++.so.6 \ /lib/$ARCH-linux-gnu/libgsl.so.27 \ /lib/$ARCH-linux-gnu/libblas.so.3 \ + /lib/$ARCH-linux-gnu/libgslcblas.so.0 \ $appName.AppDir/usr/lib/ chmod +x ./$appName.AppDir/usr/bin/$appName From 3d49e8bb57bb6fd7400e8eab213426522d658e9e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 31 Jul 2025 10:01:52 +0300 Subject: [PATCH 358/524] Fix selecting shard overlays after decreasing monitor_min_split (#1756) When node is out of sync (e.g., during initial sync), it may try to use shard overlays that do not exist anymore because monitor min split was decreased. "Download state", "download archive" and "download proof link" are the types of requests that are necessary to sync. --- validator/full-node.cpp | 16 ++++++++++------ validator/full-node.hpp | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 4e75ddede7..890fd7a2eb 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -444,7 +444,7 @@ void FullNodeImpl::download_zero_state(BlockIdExt id, td::uint32 priority, td::T void FullNodeImpl::download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { - auto shard = get_shard(id.shard_full()); + auto shard = get_shard(id.shard_full(), /* historical = */ true); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping download state diff query to unknown shard"; promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); @@ -467,7 +467,7 @@ void FullNodeImpl::download_block_proof(BlockIdExt block_id, td::uint32 priority void FullNodeImpl::download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { - auto shard = get_shard(block_id.shard_full()); + auto shard = get_shard(block_id.shard_full(), /* historical = */ true); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping download proof link query to unknown shard"; promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); @@ -490,7 +490,7 @@ void FullNodeImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeou void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) { - auto shard = get_shard(shard_prefix); + auto shard = get_shard(shard_prefix, /* historical = */ true); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping download archive query to unknown shard"; promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); @@ -519,7 +519,7 @@ void FullNodeImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vect timeout, std::move(promise)); } -td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { +td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard, bool historical) { if (shard.is_masterchain()) { return shards_[ShardIdFull{masterchainId}].actor.get(); } @@ -527,8 +527,12 @@ td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { return {}; } int pfx_len = shard.pfx_len(); - if (pfx_len > wc_monitor_min_split_) { - shard = shard_prefix(shard, wc_monitor_min_split_); + int min_split = wc_monitor_min_split_; + if (historical) { + min_split = td::Random::fast(0, min_split); + } + if (pfx_len > min_split) { + shard = shard_prefix(shard, min_split); } while (true) { auto it = shards_.find(shard); diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 9a70d5c0ce..dfd12f054c 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -131,7 +131,7 @@ class FullNodeImpl : public FullNode { FileHash zero_state_file_hash_; td::actor::ActorId get_shard(AccountIdPrefixFull dst); - td::actor::ActorId get_shard(ShardIdFull shard); + td::actor::ActorId get_shard(ShardIdFull shard, bool historical = false); std::map shards_; int wc_monitor_min_split_ = 0; From 4ec6711be40fad6a3dada187d7d87d34dc407f6a Mon Sep 17 00:00:00 2001 From: neodix42 Date: Thu, 31 Jul 2025 10:02:17 +0300 Subject: [PATCH 359/524] Bind tolk-stdlib to tolk release... (#1752) * add talk-stdlib to tolk release * add talk-stdlib to tolk release --- .github/workflows/create-tolk-release.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/create-tolk-release.yml b/.github/workflows/create-tolk-release.yml index fb8438a120..4cd30ef6f0 100644 --- a/.github/workflows/create-tolk-release.yml +++ b/.github/workflows/create-tolk-release.yml @@ -9,6 +9,9 @@ on: permissions: write-all +env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + jobs: create-release: runs-on: ubuntu-22.04 @@ -152,3 +155,11 @@ jobs: asset_name: ton-wasm.zip tag: ${{ inputs.tag }} + - name: Upload stdlib + run: | + mkdir smartcont_lib + cd smartcont_lib + cp -r ../artifacts/ton-x86_64-linux/{smartcont,lib} . + zip -r smartcont_lib.zip . + gh release upload ${{ inputs.tag }} smartcont_lib.zip + From 76144d440dcaa5eff82bacf00dd0d0cf6a325375 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 31 Jul 2025 10:24:07 +0300 Subject: [PATCH 360/524] Session stats improvements (#1746) * Detailed session stats for work time and storage stat cache * Bring back the old session state file writing --- crypto/block/transaction.cpp | 27 ++-- crypto/block/transaction.h | 1 + tdutils/td/utils/Timer.h | 46 ++++++ tl/generate/scheme/ton_api.tl | 18 ++- tl/generate/scheme/ton_api.tlo | Bin 116116 -> 117588 bytes validator/impl/collator-impl.h | 5 +- validator/impl/collator.cpp | 170 ++++++++++++++--------- validator/impl/validate-query.cpp | 45 +++--- validator/impl/validate-query.hpp | 4 +- validator/interfaces/validator-manager.h | 60 +++++++- validator/manager.cpp | 48 +------ validator/manager.hpp | 5 - 12 files changed, 277 insertions(+), 152 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index ca34e14af8..75ee477bac 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -1710,12 +1710,12 @@ bool Transaction::run_precompiled_contract(const ComputePhaseConfig& cfg, precom ComputePhase& cp = *compute_phase; CHECK(cp.precompiled_gas_usage); td::uint64 gas_usage = cp.precompiled_gas_usage.value(); - td::Timer timer; + td::RealCpuTimer timer; auto result = impl.run(my_addr, now, start_lt, balance, new_data, *in_msg_body, in_msg, msg_balance_remaining, in_msg_extern, compute_vm_libraries(cfg), cfg.global_version, cfg.max_vm_data_depth, new_code, cfg.unpacked_config_tuple, due_payment.not_null() ? due_payment : td::zero_refint(), gas_usage); - double elapsed = timer.elapsed(); + time_tvm = timer.elapsed_both(); cp.vm_init_state_hash = td::Bits256::zero(); cp.exit_code = result.exit_code; cp.out_of_gas = false; @@ -1726,7 +1726,7 @@ bool Transaction::run_precompiled_contract(const ComputePhaseConfig& cfg, precom cp.success = (cp.accepted && result.committed); LOG(INFO) << "Running precompiled smart contract " << impl.get_name() << ": exit_code=" << result.exit_code << " accepted=" << result.accepted << " success=" << cp.success << " gas_used=" << gas_usage - << " time=" << elapsed << "s"; + << " time=" << time_tvm.real << "s cpu_time=" << time_tvm.cpu; if (cp.accepted & use_msg_state) { was_activated = true; acc_status = Account::acc_active; @@ -1734,7 +1734,7 @@ bool Transaction::run_precompiled_contract(const ComputePhaseConfig& cfg, precom if (cfg.with_vm_log) { cp.vm_log = PSTRING() << "Running precompiled smart contract " << impl.get_name() << ": exit_code=" << result.exit_code << " accepted=" << result.accepted - << " success=" << cp.success << " gas_used=" << gas_usage << " time=" << elapsed << "s"; + << " success=" << cp.success << " gas_used=" << gas_usage << " time=" << time_tvm.real << "s"; } if (cp.success) { cp.new_data = impl.get_c4(); @@ -1943,9 +1943,9 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { LOG(DEBUG) << "starting VM"; cp.vm_init_state_hash = vm.get_state_hash(); - td::Timer timer; + td::RealCpuTimer timer; cp.exit_code = ~vm.run(); - double elapsed = timer.elapsed(); + time_tvm = timer.elapsed_both(); LOG(DEBUG) << "VM terminated with exit code " << cp.exit_code; cp.out_of_gas = (cp.exit_code == ~(int)vm::Excno::out_of_gas); cp.vm_final_state_hash = vm.get_final_state_hash(cp.exit_code); @@ -1971,7 +1971,7 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { LOG(INFO) << "steps: " << vm.get_steps_count() << " gas: used=" << gas.gas_consumed() << ", max=" << gas.gas_max << ", limit=" << gas.gas_limit << ", credit=" << gas.gas_credit; LOG(INFO) << "out_of_gas=" << cp.out_of_gas << ", accepted=" << cp.accepted << ", success=" << cp.success - << ", time=" << elapsed << "s"; + << ", time=" << time_tvm.real << "s, cpu_time=" << time_tvm.cpu; if (logger != nullptr) { cp.vm_log = logger->get_log(); } @@ -3196,7 +3196,13 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, } { TD_PERF_COUNTER(transaction_storage_stat_a); - td::Timer timer; + td::RealCpuTimer timer; + SCOPE_EXIT { + LOG_IF(INFO, timer.elapsed_real() > 0.1) << "Compute used storage (1) took " << timer.elapsed_real() << "s"; + if (is_account_stat) { + time_storage_stat += timer.elapsed_both(); + } + }; if (is_account_stat && compute_phase) { storage_stat.add_hint(compute_phase->vm_loaded_cells); } @@ -3208,9 +3214,6 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, storage_stat_updates.push_back(new_library); } TRY_STATUS(storage_stat.replace_roots({new_code, new_data, new_library}, /* check_merkle_depth = */ true)); - if (timer.elapsed() > 0.1) { - LOG(INFO) << "Compute used storage (1) took " << timer.elapsed() << "s"; - } } if (storage_stat.get_total_cells() > size_limits.max_acc_state_cells || @@ -3511,9 +3514,11 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { auto roots = new_storage_for_stat->prefetch_all_refs(); storage_stat_updates.insert(storage_stat_updates.end(), roots.begin(), roots.end()); { + td::RealCpuTimer timer; StorageStatCalculationContext context{true}; StorageStatCalculationContext::Guard guard{&context}; td::Status S = stats.replace_roots(roots); + time_storage_stat += timer.elapsed_both(); if (S.is_error()) { LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error(); return false; diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 1a7af2d125..83b7891638 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -399,6 +399,7 @@ struct Transaction { td::optional new_storage_dict_hash; bool gas_limit_overridden{false}; std::vector> storage_stat_updates; + td::RealCpuTimer::Time time_tvm, time_storage_stat; Transaction(const Account& _account, int ttype, ton::LogicalTime req_start_lt, ton::UnixTime _now, Ref _inmsg = {}); bool unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* cfg); diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index d787f1ca22..d1ba104b09 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -81,6 +81,52 @@ class ThreadCpuTimer { bool is_paused_{false}; }; +class RealCpuTimer { + public: + RealCpuTimer() : RealCpuTimer(false) { + } + explicit RealCpuTimer(bool is_paused) : real_(is_paused), cpu_(is_paused) { + } + RealCpuTimer(const RealCpuTimer &other) = default; + RealCpuTimer &operator=(const RealCpuTimer &other) = default; + + double elapsed_real() const { + return real_.elapsed(); + } + double elapsed_cpu() const { + return cpu_.elapsed(); + } + struct Time { + double real = 0.0, cpu = 0.0; + double get(bool is_cpu) const { + return is_cpu ? cpu : real; + } + Time &operator+=(const Time &other) { + real += other.real; + cpu += other.cpu; + return *this; + } + Time operator-(const Time &other) const { + return {.real = real - other.real, .cpu = cpu - other.cpu}; + } + }; + Time elapsed_both() const { + return {.real = real_.elapsed(), .cpu = cpu_.elapsed()}; + } + void pause() { + real_.pause(); + cpu_.pause(); + } + void resume() { + real_.resume(); + cpu_.resume(); + } + + private: + Timer real_; + ThreadCpuTimer cpu_; +}; + class PerfLog; struct EmptyDeleter { template diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index ebfd25defd..1b8e27365b 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -955,21 +955,35 @@ validatorStats.blockStats ext_msgs:validatorStats.blockStats.extMsgsStats transactions:int shard_configuration:(vector tonNode.blockIdExt) old_out_msg_queue_size:long new_out_msg_queue_size:long msg_queue_cleaned:int neighbors:(vector validatorStats.blockStats.neighborStats) = validatorStats.BlockStats; +validatorStats.collateWorkTimeStats + total:double queue_cleanup:double prelim_storage_stat:double trx_tvm:double trx_storage_stat:double + trx_other:double final_storage_stat:double create_block:double create_collated_data:double create_block_candidate:double + = validatorStats.CollateWorkTimeStats; +validatorStats.storageStatCacheStats + small_cnt:long small_cells:long hit_cnt:long hit_cells:long miss_cnt:long miss_cells:long = validatorStats.StorageStatCacheStats; validatorStats.collatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 cc_seqno:int collated_at:double bytes:int collated_data_bytes:int attempt:int self:int256 is_validator:Bool total_time:double work_time:double cpu_work_time:double time_stats:string + work_time_real_stats:validatorStats.collateWorkTimeStats + work_time_cpu_stats:validatorStats.collateWorkTimeStats block_limits:validatorStats.blockLimitsStatus - block_stats:validatorStats.blockStats = validatorSession.stats.CollatedBlock; + block_stats:validatorStats.blockStats + storage_stat_cache:validatorStats.storageStatCacheStats = validatorSession.stats.CollatedBlock; +validatorStats.validateWorkTimeStats + total:double trx_tvm:double trx_storage_stat:double trx_other:double = validatorStats.ValidateWorkTimeStats; validatorStats.validatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 validated_at:double self:int256 valid:Bool comment:string bytes:int collated_data_bytes:int - total_time:double work_time:double cpu_work_time:double = validatorStats.ValidatedBlock; + total_time:double work_time:double cpu_work_time:double + work_time_real_stats:validatorStats.validateWorkTimeStats + work_time_cpu_stats:validatorStats.validateWorkTimeStats + storage_stat_cache:validatorStats.storageStatCacheStats = validatorStats.ValidatedBlock; validatorStats.newValidatorGroup.node id:int256 pubkey:PublicKey adnl_id:int256 weight:long = validatorStats.newValidatorGroup.Node; validatorStats.newValidatorGroup session_id:int256 shard:tonNode.shardId cc_seqno:int diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 8c13ffcb66d6296e175a89e22c09305c8f8267c7..9a5241d8084e1eb2ff75ea71d00b934fec92235b 100644 GIT binary patch delta 1122 zcma)5&ubGw6!yW|{Ag2bno8qR74%?0>s16}ZA26dh%KfS?PlCe%#zt`x;v@fq$nbI z35>W0L6H6fh6>wjFH$@!sFxBCUOafIAfB8z*$^87aSqG8H{ZPXy>GsK@jUYJWhC+K z)!Fk0!gaNk*vM-1!?U({m)$#+Z~eMx5n?AGhbMx~Y2l)CpKOf}h zVPi`1^!EyIJcf+FyiZmg)vMVOw~aos?e?K=0;9q4(SC;w_!`m4wBRyASvQ- zqPQ%KEe~>s6Xw5O27PR`K^xRC86~xb-!6fknykB&S+?Y(o?gx~3dv^?vMsn0-Rg?7AHZ22Uz*=Ba5TBW$@+&jwOtY*PBJ1 zw~IP6E*1qVnZ752QGpYr_~tPoAO7hNlNhzP3q&&RVVtfJ&FH~X msg_root, block::Account* acc, UnixTime utime, LogicalTime lt, block::StoragePhaseConfig* storage_phase_cfg, block::ComputePhaseConfig* compute_phase_cfg, block::ActionPhaseConfig* action_phase_cfg, block::SerializeConfig* serialize_cfg, bool external, - LogicalTime after_lt); + LogicalTime after_lt, CollationStats* stats = nullptr); private: void start_up() override; @@ -404,8 +404,7 @@ class Collator final : public td::actor::Actor { static td::uint32 get_skip_externals_queue_size(); private: - td::Timer work_timer_{true}; - td::ThreadCpuTimer cpu_work_timer_{true}; + td::RealCpuTimer work_timer_{true}; CollationStats stats_; void finalize_stats(); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index acf9075b4a..5d0cbd2d6a 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1963,10 +1963,8 @@ bool Collator::register_shard_block_creators(std::vector creator_li */ bool Collator::try_collate() { work_timer_.resume(); - cpu_work_timer_.resume(); SCOPE_EXIT { work_timer_.pause(); - cpu_work_timer_.pause(); }; if (!preinit_complete) { LOG(WARNING) << "running do_preinit()"; @@ -2450,8 +2448,10 @@ bool Collator::dequeue_message(Ref msg_envelope, ton::LogicalTime deli * @returns True if the cleanup operation was successful, false otherwise. */ bool Collator::out_msg_queue_cleanup() { + td::RealCpuTimer timer; SCOPE_EXIT { stats_.load_fraction_queue_cleanup = block_limit_status_->load_fraction(block::ParamLimits::cl_normal); + stats_.work_time.queue_cleanup = timer.elapsed_both(); }; LOG(INFO) << "cleaning outbound queue from messages already imported by neighbors"; if (verbosity >= 2) { @@ -2677,6 +2677,10 @@ bool Collator::init_account_storage_dict(block::Account& account) { if (storage_dict_hash.is_zero()) { return true; } + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.prelim_storage_stat = timer.elapsed_both(); + }; td::Ref cached_dict_root = storage_stat_cache_ ? storage_stat_cache_(storage_dict_hash) : td::Ref{}; if (cached_dict_root.not_null()) { @@ -2684,6 +2688,14 @@ bool Collator::init_account_storage_dict(block::Account& account) { LOG(DEBUG) << "Inited storage stat from cache for account " << account.addr.to_hex() << " (" << account.storage_used.cells << " cells)"; storage_stat_cache_update_.emplace_back(cached_dict_root, account.storage_used.cells); + stats_.storage_stat_cache.hit_cnt++; + stats_.storage_stat_cache.hit_cells += account.storage_used.cells; + } else if (account.storage_used.cells >= StorageStatCache::MIN_ACCOUNT_CELLS) { + stats_.storage_stat_cache.miss_cnt++; + stats_.storage_stat_cache.miss_cells += account.storage_used.cells; + } else { + stats_.storage_stat_cache.small_cnt++; + stats_.storage_stat_cache.small_cells += account.storage_used.cells; } if (!full_collated_data_ || is_masterchain()) { if (cached_dict_root.not_null()) { @@ -2847,6 +2859,10 @@ static td::Ref clean_usage_cells(td::Ref old_root, td::Ref= StorageStatCache::MIN_ACCOUNT_CELLS; @@ -3188,6 +3204,12 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t std::unique_ptr trans = std::make_unique( *acc, mask == 2 ? block::transaction::Transaction::tr_tick : block::transaction::Transaction::tr_tock, req_start_lt, now_); + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.trx_tvm += trans->time_tvm; + stats_.work_time.trx_storage_stat += trans->time_storage_stat; + stats_.work_time.trx_other += timer.elapsed_both() - trans->time_tvm - trans->time_storage_stat; + }; if (!trans->prepare_storage_phase(storage_phase_cfg_, true)) { return fatal_error(td::Status::Error( -666, std::string{"cannot create storage phase of a new transaction for smart contract "} + smc_addr.to_hex())); @@ -3292,7 +3314,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, } set_current_tx_storage_dict(*acc); auto res = impl_create_ordinary_transaction(msg_root, acc, now_, start_lt, &storage_phase_cfg_, &compute_phase_cfg_, - &action_phase_cfg_, &serialize_cfg_, external, after_lt); + &action_phase_cfg_, &serialize_cfg_, external, after_lt, &stats_); if (res.is_error()) { auto error = res.move_as_error(); if (error.code() == -701) { @@ -3348,19 +3370,17 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, * @param serialize_cfg The configuration for the serialization of the transaction. * @param external Flag indicating if the message is external. * @param after_lt The logical time after which the transaction should occur. Used only for external messages. + * @param collation_stats Stats to write real/cpu time to (optional) * * @returns A Result object containing the created transaction. * Returns error_code == 669 if the error is fatal and the block can not be produced. * Returns error_code == 701 if the transaction can not be included into block, but it's ok (external or too early internal). */ -td::Result> Collator::impl_create_ordinary_transaction(Ref msg_root, - block::Account* acc, - UnixTime utime, LogicalTime lt, - block::StoragePhaseConfig* storage_phase_cfg, - block::ComputePhaseConfig* compute_phase_cfg, - block::ActionPhaseConfig* action_phase_cfg, - block::SerializeConfig* serialize_cfg, - bool external, LogicalTime after_lt) { +td::Result> Collator::impl_create_ordinary_transaction( + Ref msg_root, block::Account* acc, UnixTime utime, LogicalTime lt, + block::StoragePhaseConfig* storage_phase_cfg, block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, block::SerializeConfig* serialize_cfg, bool external, + LogicalTime after_lt, CollationStats* stats) { if (acc->last_trans_end_lt_ >= lt && acc->transactions.empty()) { return td::Status::Error(-669, PSTRING() << "last transaction time in the state of account " << acc->workchain << ":" << acc->addr.to_hex() << " is too large"); @@ -3372,64 +3392,74 @@ td::Result> Collator::impl_crea std::unique_ptr trans = std::make_unique( *acc, block::transaction::Transaction::tr_ord, trans_min_lt + 1, utime, msg_root); - bool ihr_delivered = false; // FIXME - if (!trans->unpack_input_msg(ihr_delivered, action_phase_cfg)) { - if (external) { - // inbound external message was not accepted - return td::Status::Error(-701, "inbound external message rejected by account "s + acc->addr.to_hex() + - " before smart-contract execution"); + { + td::RealCpuTimer timer; + SCOPE_EXIT { + if (stats) { + stats->work_time.trx_tvm += trans->time_tvm; + stats->work_time.trx_storage_stat += trans->time_storage_stat; + stats->work_time.trx_other += timer.elapsed_both() - trans->time_tvm - trans->time_storage_stat; + } + }; + bool ihr_delivered = false; // FIXME + if (!trans->unpack_input_msg(ihr_delivered, action_phase_cfg)) { + if (external) { + // inbound external message was not accepted + return td::Status::Error(-701, "inbound external message rejected by account "s + acc->addr.to_hex() + + " before smart-contract execution"); + } + return td::Status::Error(-669, "cannot unpack input message for a new transaction"); } - return td::Status::Error(-669, "cannot unpack input message for a new transaction"); - } - if (trans->bounce_enabled) { - if (!trans->prepare_storage_phase(*storage_phase_cfg, true)) { - return td::Status::Error( - -669, "cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + if (trans->bounce_enabled) { + if (!trans->prepare_storage_phase(*storage_phase_cfg, true)) { + return td::Status::Error( + -669, "cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!external && !trans->prepare_credit_phase()) { + return td::Status::Error( + -669, "cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + } else { + if (!external && !trans->prepare_credit_phase()) { + return td::Status::Error( + -669, "cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!trans->prepare_storage_phase(*storage_phase_cfg, true, true)) { + return td::Status::Error( + -669, "cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } } - if (!external && !trans->prepare_credit_phase()) { + if (!trans->prepare_compute_phase(*compute_phase_cfg)) { return td::Status::Error( - -669, "cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + -669, "cannot create compute phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!trans->compute_phase->accepted) { + if (external) { + // inbound external message was not accepted + auto const& cp = *trans->compute_phase; + return td::Status::Error( + -701, PSLICE() << "inbound external message rejected by transaction " << acc->addr.to_hex() << ":\n" + << "exitcode=" << cp.exit_code << ", steps=" << cp.vm_steps << ", gas_used=" << cp.gas_used + << (cp.vm_log.empty() ? "" : "\nVM Log (truncated):\n..." + cp.vm_log)); + } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { + return td::Status::Error(-669, "new ordinary transaction for smart contract "s + acc->addr.to_hex() + + " has not been accepted by the smart contract (?)"); + } } - } else { - if (!external && !trans->prepare_credit_phase()) { + if (trans->compute_phase->success && !trans->prepare_action_phase(*action_phase_cfg)) { return td::Status::Error( - -669, "cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + -669, "cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex()); } - if (!trans->prepare_storage_phase(*storage_phase_cfg, true, true)) { + if (trans->bounce_enabled && + (!trans->compute_phase->success || trans->action_phase->state_exceeds_limits || trans->action_phase->bounce) && + !trans->prepare_bounce_phase(*action_phase_cfg)) { return td::Status::Error( - -669, "cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + -669, "cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex()); } - } - if (!trans->prepare_compute_phase(*compute_phase_cfg)) { - return td::Status::Error( - -669, "cannot create compute phase of a new transaction for smart contract "s + acc->addr.to_hex()); - } - if (!trans->compute_phase->accepted) { - if (external) { - // inbound external message was not accepted - auto const& cp = *trans->compute_phase; - return td::Status::Error( - -701, PSLICE() << "inbound external message rejected by transaction " << acc->addr.to_hex() << ":\n" - << "exitcode=" << cp.exit_code << ", steps=" << cp.vm_steps << ", gas_used=" << cp.gas_used - << (cp.vm_log.empty() ? "" : "\nVM Log (truncated):\n..." + cp.vm_log)); - } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { - return td::Status::Error(-669, "new ordinary transaction for smart contract "s + acc->addr.to_hex() + - " has not been accepted by the smart contract (?)"); + if (!trans->serialize(*serialize_cfg)) { + return td::Status::Error(-669, "cannot serialize new transaction for smart contract "s + acc->addr.to_hex()); } } - if (trans->compute_phase->success && !trans->prepare_action_phase(*action_phase_cfg)) { - return td::Status::Error( - -669, "cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex()); - } - if (trans->bounce_enabled && - (!trans->compute_phase->success || trans->action_phase->state_exceeds_limits || trans->action_phase->bounce) && - !trans->prepare_bounce_phase(*action_phase_cfg)) { - return td::Status::Error( - -669, "cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex()); - } - if (!trans->serialize(*serialize_cfg)) { - return td::Status::Error(-669, "cannot serialize new transaction for smart contract "s + acc->addr.to_hex()); - } return std::move(trans); } @@ -6018,6 +6048,10 @@ bool Collator::create_mc_block_extra(Ref& mc_block_extra) { * @returns True if the new block is successfully created, false otherwise. */ bool Collator::create_block() { + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.create_block += timer.elapsed_both(); + }; Ref block_info, extra; if (!create_block_info(block_info)) { return fatal_error("cannot create BlockInfo for the new block"); @@ -6095,7 +6129,7 @@ Ref Collator::collate_shard_block_descr_set() { /** * Visits certain cells in out msg queue and dispatch queue to add them to the proof * - * @returns True on success, Falise if error occurred + * @returns True on success, False if error occurred */ bool Collator::prepare_msg_queue_proof() { auto res = old_out_msg_queue_->scan_diff( @@ -6156,6 +6190,10 @@ bool Collator::prepare_msg_queue_proof() { * @returns True if the collated data was successfully created, false otherwise. */ bool Collator::create_collated_data() { + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.create_collated_data += timer.elapsed_both(); + }; // 1. store the set of used shard block descriptions if (!used_shard_block_descr_.empty()) { auto cell = collate_shard_block_descr_set(); @@ -6253,6 +6291,10 @@ bool Collator::create_collated_data() { * @returns True if the block candidate was created successfully, false otherwise. */ bool Collator::create_block_candidate() { + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.create_block_candidate += timer.elapsed_both(); + }; auto consensus_config = config_->get_consensus_config(); // 1. serialize block LOG(INFO) << "serializing new Block"; @@ -6524,9 +6566,8 @@ td::uint32 Collator::get_skip_externals_queue_size() { } void Collator::finalize_stats() { - double work_time = work_timer_.elapsed(); - double cpu_work_time = cpu_work_timer_.elapsed(); - LOG(WARNING) << "Collate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; + auto work_time = work_timer_.elapsed_both(); + LOG(WARNING) << "Collate query work time = " << work_time.real << "s, cpu time = " << work_time.cpu << "s"; if (block_candidate) { stats_.block_id = block_candidate->id; stats_.collated_data_hash = block_candidate->collated_file_hash; @@ -6553,8 +6594,7 @@ void Collator::finalize_stats() { block_limit_status_->limits.classify_collated_data_size(stats_.estimated_collated_data_bytes); } stats_.total_time = perf_timer_.elapsed(); - stats_.work_time = work_time; - stats_.cpu_work_time = cpu_work_time; + stats_.work_time.total = work_time; stats_.time_stats = (PSTRING() << perf_log_); if (is_masterchain() && shard_conf_) { shard_conf_->process_shard_hashes([&](const block::McShardHash& shard) { diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index ac7f400f34..fbaa06aaca 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -5369,6 +5369,14 @@ std::unique_ptr ValidateQuery::unpack_account(td::ConstBitPtr ad LOG(DEBUG) << "Inited storage stat from cache for account " << addr.to_hex(256) << " (" << new_acc->storage_used.cells << " cells)"; storage_stat_cache_update_.emplace_back(dict_root, new_acc->storage_used.cells); + stats_.storage_stat_cache.hit_cnt++; + stats_.storage_stat_cache.hit_cells += new_acc->storage_used.cells; + } else if (new_acc->storage_used.cells >= StorageStatCache::MIN_ACCOUNT_CELLS) { + stats_.storage_stat_cache.miss_cnt++; + stats_.storage_stat_cache.miss_cells += new_acc->storage_used.cells; + } else { + stats_.storage_stat_cache.small_cnt++; + stats_.storage_stat_cache.small_cells += new_acc->storage_used.cells; } } } @@ -5780,6 +5788,12 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT // .... std::unique_ptr trs = std::make_unique(account, trans_type, lt, now_, in_msg_root); + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.trx_tvm += trs->time_tvm; + stats_.work_time.trx_storage_stat += trs->time_storage_stat; + stats_.work_time.trx_other += timer.elapsed_both() - trs->time_tvm - trs->time_storage_stat; + }; if (in_msg_root.not_null()) { if (!trs->unpack_input_msg(ihr_delivered, &action_phase_cfg_)) { // inbound external message was not accepted @@ -7029,10 +7043,8 @@ bool ValidateQuery::try_validate() { return true; } work_timer_.resume(); - cpu_work_timer_.resume(); SCOPE_EXIT { work_timer_.pause(); - cpu_work_timer_.pause(); }; try { if (!stage_) { @@ -7181,25 +7193,24 @@ void ValidateQuery::written_candidate() { * Sends validation work time to manager. */ void ValidateQuery::record_stats(bool valid, std::string error_message) { - ValidationStats stats; - stats.block_id = id_; - stats.collated_data_hash = block_candidate.collated_file_hash; - stats.validated_at = td::Clocks::system(); - stats.self = local_validator_id_; - stats.valid = valid; + stats_.block_id = id_; + stats_.collated_data_hash = block_candidate.collated_file_hash; + stats_.validated_at = td::Clocks::system(); + stats_.self = local_validator_id_; + stats_.valid = valid; if (valid) { - stats.comment = (PSTRING() << "OK ts=" << now_); + stats_.comment = (PSTRING() << "OK ts=" << now_); } else { - stats.comment = std::move(error_message); + stats_.comment = std::move(error_message); } - stats.actual_bytes = block_candidate.data.size(); - stats.actual_collated_data_bytes = block_candidate.collated_data.size(); - stats.total_time = perf_timer_.elapsed(); - stats.work_time = work_timer_.elapsed(); - stats.cpu_work_time = cpu_work_timer_.elapsed(); + stats_.actual_bytes = block_candidate.data.size(); + stats_.actual_collated_data_bytes = block_candidate.collated_data.size(); + stats_.total_time = perf_timer_.elapsed(); + stats_.work_time.total = work_timer_.elapsed_both(); LOG(WARNING) << "validation took " << perf_timer_.elapsed() << "s"; - LOG(WARNING) << "Validate query work time = " << stats.work_time << "s, cpu time = " << stats.cpu_work_time << "s"; - td::actor::send_closure(manager, &ValidatorManager::log_validate_query_stats, std::move(stats)); + LOG(WARNING) << "Validate query work time = " << stats_.work_time.total.real + << "s, cpu time = " << stats_.work_time.total.cpu << "s"; + td::actor::send_closure(manager, &ValidatorManager::log_validate_query_stats, std::move(stats_)); } } // namespace validator diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 95e81cdec1..e43d782f53 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -413,8 +413,8 @@ class ValidateQuery : public td::actor::Actor { return true; } - td::Timer work_timer_{true}; - td::ThreadCpuTimer cpu_work_timer_{true}; + td::RealCpuTimer work_timer_{true}; + ValidationStats stats_; void record_stats(bool valid, std::string error_message = ""); }; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 6c5e8ce9ec..78248610ac 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -55,6 +55,17 @@ struct AsyncSerializerState { UnixTime last_written_block_ts; }; +struct StorageStatCacheStats { + td::uint64 small_cnt = 0, small_cells = 0; + td::uint64 hit_cnt = 0, hit_cells = 0; + td::uint64 miss_cnt = 0, miss_cells = 0; + + tl_object_ptr tl() const { + return create_tl_object(small_cnt, small_cells, hit_cnt, hit_cells, + miss_cnt, miss_cells); + } +}; + struct CollationStats { BlockIdExt block_id{workchainInvalid, 0, 0, RootHash::zero(), FileHash::zero()}; td::Status status = td::Status::OK(); @@ -69,7 +80,7 @@ struct CollationStats { td::uint32 estimated_bytes = 0, gas = 0, lt_delta = 0, estimated_collated_data_bytes = 0; int cat_bytes = 0, cat_gas = 0, cat_lt_delta = 0, cat_collated_data_bytes = 0; std::string limits_log; - double total_time = 0.0, work_time = 0.0, cpu_work_time = 0.0; + double total_time = 0.0; std::string time_stats; td::uint32 transactions = 0; @@ -104,6 +115,28 @@ struct CollationStats { double load_fraction_externals = -1.0; double load_fraction_new_msgs = -1.0; + struct WorkTimeStats { + td::RealCpuTimer::Time total; + td::RealCpuTimer::Time queue_cleanup; + td::RealCpuTimer::Time prelim_storage_stat; + td::RealCpuTimer::Time trx_tvm; + td::RealCpuTimer::Time trx_storage_stat; + td::RealCpuTimer::Time trx_other; + td::RealCpuTimer::Time final_storage_stat; + td::RealCpuTimer::Time create_block; + td::RealCpuTimer::Time create_collated_data; + td::RealCpuTimer::Time create_block_candidate; + + tl_object_ptr tl(bool is_cpu) const { + return create_tl_object( + total.get(is_cpu), queue_cleanup.get(is_cpu), prelim_storage_stat.get(is_cpu), trx_tvm.get(is_cpu), + trx_storage_stat.get(is_cpu), trx_other.get(is_cpu), final_storage_stat.get(is_cpu), create_block.get(is_cpu), + create_collated_data.get(is_cpu), create_block_candidate.get(is_cpu)); + } + }; + WorkTimeStats work_time; + StorageStatCacheStats storage_stat_cache; + tl_object_ptr tl() const { std::vector> shards_obj; for (const BlockIdExt& block_id : shard_configuration) { @@ -120,13 +153,13 @@ struct CollationStats { std::move(neighbors_obj)); return create_tl_object( create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes, - actual_collated_data_bytes, attempt, self.bits256_value(), is_validator, total_time, work_time, cpu_work_time, - time_stats, + actual_collated_data_bytes, attempt, self.bits256_value(), is_validator, total_time, work_time.total.real, + work_time.total.cpu, time_stats, work_time.tl(false), work_time.tl(true), create_tl_object( estimated_bytes, gas, lt_delta, estimated_collated_data_bytes, cat_bytes, cat_gas, cat_lt_delta, cat_collated_data_bytes, load_fraction_queue_cleanup, load_fraction_dispatch, load_fraction_internals, load_fraction_externals, load_fraction_new_msgs, limits_log), - std::move(block_stats)); + std::move(block_stats), storage_stat_cache.tl()); } }; @@ -138,12 +171,27 @@ struct ValidationStats { bool valid = false; std::string comment; td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0; - double total_time = 0.0, work_time = 0.0, cpu_work_time = 0.0; + double total_time = 0.0; + + struct WorkTimeStats { + td::RealCpuTimer::Time total; + td::RealCpuTimer::Time trx_tvm; + td::RealCpuTimer::Time trx_storage_stat; + td::RealCpuTimer::Time trx_other; + + tl_object_ptr tl(bool is_cpu) const { + return create_tl_object( + total.get(is_cpu), trx_tvm.get(is_cpu), trx_storage_stat.get(is_cpu), trx_other.get(is_cpu)); + } + }; + WorkTimeStats work_time; + StorageStatCacheStats storage_stat_cache; tl_object_ptr tl() const { return create_tl_object( create_tl_block_id(block_id), collated_data_hash, validated_at, self.bits256_value(), valid, comment, - actual_bytes, actual_collated_data_bytes, total_time, work_time, cpu_work_time); + actual_bytes, actual_collated_data_bytes, total_time, work_time.total.real, work_time.total.cpu, + work_time.tl(false), work_time.tl(true), storage_stat_cache.tl()); } }; diff --git a/validator/manager.cpp b/validator/manager.cpp index 6e3780fef4..1fcd6a520e 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1992,7 +1992,6 @@ void ValidatorManagerImpl::start_up() { } validator_manager_init(opts_, actor_id(this), db_.get(), std::move(P)); - init_session_stats(); check_waiters_at_ = td::Timestamp::in(1.0); alarm_timestamp().relax(check_waiters_at_); @@ -3485,7 +3484,6 @@ void ValidatorManagerImpl::update_options(td::Ref opts) td::actor::send_closure(shard_block_verifier_, &ShardBlockVerifier::update_options, opts); } opts_ = std::move(opts); - init_session_stats(); } void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) { @@ -3739,51 +3737,19 @@ void ValidatorManagerImpl::init_validator_telemetry() { } } -void ValidatorManagerImpl::init_session_stats() { - if (opts_->get_session_logs_file() == session_stats_filename_) { - return; - } - session_stats_filename_ = opts_->get_session_logs_file(); - if (session_stats_filename_.empty()) { - session_stats_enabled_ = false; - session_stats_fd_.close(); - return; - } - auto r_fd = td::FileFd::open(session_stats_filename_, - td::FileFd::Flags::Write | td::FileFd::Flags::Append | td::FileFd::Create); - if (r_fd.is_error()) { - LOG(ERROR) << "Failed to open session stats file for writing: " << r_fd.move_as_error(); - session_stats_filename_.clear(); - session_stats_enabled_ = false; - return; - } - session_stats_fd_ = r_fd.move_as_ok(); - session_stats_enabled_ = true; -} - template void ValidatorManagerImpl::write_session_stats(const T &obj) { - if (!session_stats_enabled_) { + std::string fname = opts_->get_session_logs_file(); + if (fname.empty()) { return; } auto s = td::json_encode(td::ToJson(*obj.tl()), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - s += '\n'; - td::Slice slice{s}; - while (!slice.empty()) { - auto R = session_stats_fd_.write(slice); - if (R.is_error()) { - LOG(WARNING) << "Failed to write to session stats: " << R.move_as_error(); - } - if (R.ok() == 0) { - LOG(WARNING) << "Failed to write to session stats"; - } - slice.remove_prefix(R.ok()); - } - auto S = session_stats_fd_.sync(); - if (S.is_error()) { - LOG(WARNING) << "Failed to write to session stats: " << S; - } + + std::ofstream file; + file.open(fname, std::ios_base::app); + file << s << "\n"; + file.close(); } void ValidatorManagerImpl::init_shard_block_verifier(adnl::AdnlNodeIdShort local_id) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 9d1863c3da..952fcf49b5 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -827,11 +827,6 @@ class ValidatorManagerImpl : public ValidatorManager { td::actor::ActorOwn storage_stat_cache_; - bool session_stats_enabled_ = false; - std::string session_stats_filename_; - td::FileFd session_stats_fd_; - - void init_session_stats(); template void write_session_stats(const T &obj); From 0326fc5479cec096a128a53368fa83aae6c11f8f Mon Sep 17 00:00:00 2001 From: Oleg Vallas Date: Fri, 1 Aug 2025 11:09:03 +0200 Subject: [PATCH 361/524] Improve block compression (#1710) * [improve-block-compression] Add improved block compression algorithm, saving 15% more space with almost the same performance. Add dynamic compression algorithm configuration. * [improve-block-compression] Add max_decompressed_size to decompression interface * [improve-block-compression] Fix encoding of Prunned branch cells. Add support for collated data format. * [improve-block-compression] Change compression interface to support multiple roots. Save algorithm flag in the first byte. * [improve-block-compression] Replace compression with a new one in method compress_candidate_data() * [improve-block-compression] Store decompression size at the start of the compressed data * [improve-block-compression] Remove passing max decompression size to new compression method * [improve-block-compression] Use new compression algorithm in serialize_block_full() * [improve-block-compression] Use new compression algorithm in serialize_block_candidate_broadcast() * [improve-block-compression] Small performance optimization * [improve-block-compression] Replace vector of vectors with vector of arrays for faster operations with BOC graph. Speed up by ~10%. * [improve-block-compression] Use new compression approach in serialize_block_broadcast() method * [improve-block-compression] Remove redundant debug output * [improve-block-compression] Add checks and validations for improved compression * [improve-block-compression] Remove redundant size calculation * [improve-block-compression] Add all necessary checks and validations to improved decompression method * [improve-block-compression] Add all necessary checks and validations to base compression methods * [improve-block-compression] Introduce tl objects for improved compression and integrate them into all compression usages. * [improve-block-compression] Fix support of prunned branch cells with level up to 7. * [improve-block-compression] Replace size literals with a constant. Add checks for baseline decompress. * [improve-block-compression] Add check against negative cell data length * [improve-block-compression] Catch CellWriteError from cell_builder.finalize() * [improve-block-compression] Move bits allocation and uint write into a single append function. * [improve-block-compression] Move buffer capacity check and bits reading to a separate method * [improve-block-compression] Revert original implementation of the compression methods, while keeping updated decompression methods * [improve-block-compression] Revert original implementation of the compression methods, while keeping updated decompression methods * [improve-block-compression] Store signatures directly inside tonNode_blockBroadcastCompressedV2 tl object. * [improve-block-compression] Pass max_decompression_size from config to boc_compress method. Use it to check limit on decompressed block. * [improve-block-compression] Increase max cell count to 2^32. Add node count validity check. * [improve-block-compression] Refactor types. Use unsigned types where possible. * [Compress-block-using-round-cache] Remove limitation of 2^16 root cells in BOC. * [Compress-block-using-round-cache] Regenerate ton_api.tlo after merging testnet branch. --- crypto/CMakeLists.txt | 2 + crypto/vm/boc-compression.cpp | 624 +++++++++++++++++++++ crypto/vm/boc-compression.h | 42 ++ tl/generate/scheme/ton_api.tl | 7 + tl/generate/scheme/ton_api.tlo | Bin 117588 -> 118872 bytes validator-session/candidate-serializer.cpp | 54 +- validator-session/candidate-serializer.h | 2 + validator/collator-node.cpp | 16 +- validator/full-node-fast-sync-overlays.cpp | 9 + validator/full-node-fast-sync-overlays.hpp | 2 + validator/full-node-private-overlay.cpp | 19 + validator/full-node-private-overlay.hpp | 4 + validator/full-node-serializer.cpp | 65 +++ validator/full-node-shard.cpp | 9 + validator/full-node-shard.hpp | 2 + 15 files changed, 844 insertions(+), 13 deletions(-) create mode 100644 crypto/vm/boc-compression.cpp create mode 100644 crypto/vm/boc-compression.h diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index fd3c41c09d..50f8903941 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -14,6 +14,7 @@ set(TON_CRYPTO_CORE_SOURCE openssl/residue.cpp openssl/rand.cpp vm/boc.cpp + vm/boc-compression.cpp vm/large-boc-serializer.cpp tl/tlblib.cpp @@ -108,6 +109,7 @@ set(TON_CRYPTO_SOURCE vm/arithops.h vm/atom.h vm/boc.h + vm/boc-compression.h vm/boc-writers.h vm/box.hpp vm/cellops.h diff --git a/crypto/vm/boc-compression.cpp b/crypto/vm/boc-compression.cpp new file mode 100644 index 0000000000..63a834e086 --- /dev/null +++ b/crypto/vm/boc-compression.cpp @@ -0,0 +1,624 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ +#include "boc-compression.h" + +#include +#include "vm/boc.h" +#include "vm/boc-writers.h" +#include "vm/cells.h" +#include "vm/cellslice.h" +#include "td/utils/Slice-decl.h" +#include "td/utils/lz4.h" + +namespace vm { + +td::Result boc_compress_baseline_lz4(const std::vector>& boc_roots) { + TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(boc_roots), 2)); + td::BufferSlice compressed = td::lz4_compress(data); + + // Add decompressed size at the beginning + td::BufferSlice compressed_with_size(compressed.size() + kDecompressedSizeBytes); + auto size_slice = td::BitSliceWrite(compressed_with_size.as_slice().ubegin(), kDecompressedSizeBytes * 8); + size_slice.bits().store_uint(data.size(), kDecompressedSizeBytes * 8); + memcpy(compressed_with_size.data() + kDecompressedSizeBytes, compressed.data(), compressed.size()); + + return compressed_with_size; +} + +td::Result>> boc_decompress_baseline_lz4(td::Slice compressed, int max_decompressed_size) { + // Check minimum input size for decompressed size header + if (compressed.size() < kDecompressedSizeBytes) { + return td::Status::Error("BOC decompression failed: input too small for header"); + } + + // Read decompressed size + constexpr size_t kSizeBits = kDecompressedSizeBytes * 8; + int decompressed_size = td::BitSlice(compressed.ubegin(), kSizeBits).bits().get_uint(kSizeBits); + compressed.remove_prefix(kDecompressedSizeBytes); + if (decompressed_size <= 0 || decompressed_size > max_decompressed_size) { + return td::Status::Error("BOC decompression failed: invalid decompressed size"); + } + + TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size)); + TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed)); + return roots; +} + +inline void append_uint(td::BitString& bs, unsigned val, unsigned n) { + bs.reserve_bitslice(n).bits().store_uint(val, n); +} + +inline td::Result read_uint(td::BitSlice& bs, int bits) { + // Check if there enough bits available + if (bs.size() < bits) { + return td::Status::Error("BOC decompression failed: not enough bits to read"); + } + unsigned result = bs.bits().get_uint(bits); + bs.advance(bits); + return result; +} + +td::Result boc_compress_improved_structure_lz4(const std::vector>& boc_roots) { + // Input validation + if (boc_roots.empty()) { + return td::Status::Error("No root cells were provided for serialization"); + } + for (const auto& root : boc_roots) { + if (root.is_null()) { + return td::Status::Error("Cannot serialize a null cell reference into a bag of cells"); + } + } + + // Initialize data structures for graph representation + td::HashMap cell_hashes; + std::vector> boc_graph; + std::vector refs_cnt; + std::vector cell_data; + std::vector cell_type; + std::vector prunned_branch_level; + std::vector root_indexes; + size_t total_size_estimate = 0; + + // Build graph representation using recursive lambda + const auto build_graph = [&](auto&& self, td::Ref cell) -> td::Result { + if (cell.is_null()) { + return td::Status::Error("Error while importing a cell during serialization: cell is null"); + } + + auto cell_hash = cell->get_hash(); + auto it = cell_hashes.find(cell_hash); + if (it != cell_hashes.end()) { + return it->second; + } + + size_t current_cell_id = boc_graph.size(); + cell_hashes.emplace(cell_hash, current_cell_id); + + bool is_special = false; + vm::CellSlice cell_slice = vm::load_cell_slice_special(cell, is_special); + if (!cell_slice.is_valid()) { + return td::Status::Error("Invalid loaded cell data"); + } + td::BitSlice cell_bitslice = cell_slice.as_bitslice(); + + // Initialize new cell in graph + boc_graph.emplace_back(); + refs_cnt.emplace_back(cell_slice.size_refs()); + cell_type.emplace_back(size_t(cell_slice.special_type())); + prunned_branch_level.push_back(0); + + DCHECK(cell_slice.size_refs() <= 4); + + // Process special cell of type PrunnedBranch + if (cell_slice.special_type() == vm::CellTraits::SpecialType::PrunnedBranch) { + DCHECK(cell_slice.size() >= 16); + cell_data.emplace_back(cell_bitslice.subslice(16, cell_bitslice.size() - 16)); + prunned_branch_level.back() = cell_slice.data()[1]; + } else { + cell_data.emplace_back(cell_bitslice); + } + total_size_estimate += cell_bitslice.size(); + + // Process cell references + for (int i = 0; i < cell_slice.size_refs(); ++i) { + TRY_RESULT(child_id, self(self, cell_slice.prefetch_ref(i))); + boc_graph[current_cell_id][i] = child_id; + } + + return current_cell_id; + }; + + // Build the graph starting from roots + for (auto root : boc_roots) { + TRY_RESULT(root_cell_id, build_graph(build_graph, root)); + root_indexes.push_back(root_cell_id); + } + + // Check graph properties + const size_t node_count = boc_graph.size(); + std::vector> reverse_graph(node_count); + size_t edge_count = 0; + + // Build reverse graph + for (int i = 0; i < node_count; ++i) { + for (size_t child_index = 0; child_index < refs_cnt[i]; ++child_index) { + size_t child = boc_graph[i][child_index]; + ++edge_count; + reverse_graph[child].push_back(i); + } + } + + // Process cell data sizes + std::vector is_data_small(node_count, 0); + for (int i = 0; i < node_count; ++i) { + if (cell_type[i] != 1) { + is_data_small[i] = cell_data[i].size() < 128; + } + } + + // Perform topological sort + std::vector topo_order, rank(node_count); + const auto topological_sort = [&]() -> td::Status { + std::vector> queue; + queue.reserve(node_count); + std::vector in_degree(node_count); + + // Calculate in-degrees and initialize queue + for (int i = 0; i < node_count; ++i) { + in_degree[i] = refs_cnt[i]; + if (in_degree[i] == 0) { + queue.emplace_back(cell_type[i] == 0, -int(cell_data[i].size()), -i); + } + } + + if (queue.empty()) { + return td::Status::Error("Cycle detected in cell references"); + } + + std::sort(queue.begin(), queue.end()); + + // Process queue + while (!queue.empty()) { + int node = -std::get<2>(queue.back()); + queue.pop_back(); + topo_order.push_back(node); + + for (int parent : reverse_graph[node]) { + if (--in_degree[parent] == 0) { + queue.emplace_back(0, 0, -parent); + } + } + } + + if (topo_order.size() != node_count) { + return td::Status::Error("Invalid graph structure"); + } + + std::reverse(topo_order.begin(), topo_order.end()); + return td::Status::OK(); + }; + + TRY_STATUS(topological_sort()); + + // Calculate index of vertices in topsort + for (int i = 0; i < node_count; ++i) { + rank[topo_order[i]] = i; + } + + // Build compressed representation + td::BitString result; + total_size_estimate += (node_count * 10 * 8); + result.reserve_bits(total_size_estimate); + + // Store roots information + append_uint(result, root_indexes.size(), 32); + for (int root_ind : root_indexes) { + append_uint(result, rank[root_ind], 32); + } + + // Store node count + append_uint(result, node_count, 32); + + // Store cell types and sizes + for (int i = 0; i < node_count; ++i) { + size_t node = topo_order[i]; + size_t currrent_cell_type = bool(cell_type[node]) + prunned_branch_level[node]; + append_uint(result, currrent_cell_type, 4); + append_uint(result, refs_cnt[node], 4); + + if (cell_type[node] != 1) { + if (is_data_small[node]) { + append_uint(result, 1, 1); + append_uint(result, cell_data[node].size(), 7); + } else { + append_uint(result, 0, 1); + append_uint(result, 1 + cell_data[node].size() / 8, 7); + } + } + } + + // Store edge information + auto edge_bits = result.reserve_bitslice(edge_count).bits(); + for (int i = 0; i < node_count; ++i) { + size_t node = topo_order[i]; + for (size_t child_index = 0; child_index < refs_cnt[node]; ++child_index) { + size_t child = boc_graph[node][child_index]; + edge_bits.store_uint(rank[child] == i + 1, 1); + ++edge_bits; + } + } + + // Store cell data + for (size_t node : topo_order) { + if (cell_type[node] != 1 && !is_data_small[node]) { + continue; + } + result.append(cell_data[node].subslice(0, cell_data[node].size() % 8)); + } + + // Store BOC graph with optimized encoding + for (size_t i = 0; i < node_count; ++i) { + size_t node = topo_order[i]; + if (node_count <= i + 3) + continue; + + for (int j = 0; j < refs_cnt[node]; ++j) { + if (rank[boc_graph[node][j]] <= i + 1) + continue; + + int delta = rank[boc_graph[node][j]] - i - 2; // Always >= 0 because of above check + size_t required_bits = 1 + (31 ^ __builtin_clz(node_count - i - 3)); + + if (required_bits < 8 - (result.size() + 1) % 8 + 1) { + append_uint(result, delta, required_bits); + } else if (delta < (1 << (8 - (result.size() + 1) % 8))) { + size_t available_bits = 8 - (result.size() + 1) % 8; + append_uint(result, 1, 1); + append_uint(result, delta, available_bits); + } else { + append_uint(result, 0, 1); + append_uint(result, delta, required_bits); + } + } + } + + // Pad result to byte boundary + while (result.size() % 8) { + append_uint(result, 0, 1); + } + + // Store remaining cell data + for (size_t node : topo_order) { + if (cell_type[node] == 1 || is_data_small[node]) { + size_t prefix_size = cell_data[node].size() % 8; + result.append(cell_data[node].subslice(prefix_size, cell_data[node].size() - prefix_size)); + } else { + size_t data_size = cell_data[node].size() + 1; + size_t padding = (8 - data_size % 8) % 8; + + if (padding) { + append_uint(result, 0, padding); + } + append_uint(result, 1, 1); + result.append(cell_data[node]); + } + } + + // Final padding + while (result.size() % 8) { + append_uint(result, 0, 1); + } + + // Create final compressed buffer + td::BufferSlice serialized((const char*)result.bits().get_byte_ptr(), result.size() / 8); + + td::BufferSlice compressed = td::lz4_compress(serialized); + + // Add decompressed size at the beginning + td::BufferSlice compressed_with_size(compressed.size() + kDecompressedSizeBytes); + auto size_slice = td::BitSliceWrite(compressed_with_size.as_slice().ubegin(), kDecompressedSizeBytes * 8); + size_slice.bits().store_uint(serialized.size(), kDecompressedSizeBytes * 8); + memcpy(compressed_with_size.data() + kDecompressedSizeBytes, compressed.data(), compressed.size()); + + return compressed_with_size; +} + +td::Result>> boc_decompress_improved_structure_lz4(td::Slice compressed, int max_decompressed_size) { + constexpr size_t kMaxCellDataLengthBits = 1024; + + // Check minimum input size for decompressed size header + if (compressed.size() < kDecompressedSizeBytes) { + return td::Status::Error("BOC decompression failed: input too small for header"); + } + + // Read decompressed size + constexpr size_t kSizeBits = kDecompressedSizeBytes * 8; + size_t decompressed_size = td::BitSlice(compressed.ubegin(), kSizeBits).bits().get_uint(kSizeBits); + compressed.remove_prefix(kDecompressedSizeBytes); + if (decompressed_size > max_decompressed_size) { + return td::Status::Error("BOC decompression failed: invalid decompressed size"); + } + + // Decompress LZ4 data + TRY_RESULT(serialized, td::lz4_decompress(compressed, decompressed_size)); + + if (serialized.size() != decompressed_size) { + return td::Status::Error("BOC decompression failed: decompressed size mismatch"); + } + + // Initialize bit reader + td::BitSlice bit_reader(serialized.as_slice().ubegin(), serialized.as_slice().size() * 8); + size_t orig_size = bit_reader.size(); + + // Read root count + TRY_RESULT(root_count, read_uint(bit_reader, 32)); + // We assume that each cell should take at least 1 byte, even effectively serialized + // Otherwise it means that provided root_count is incorrect + if (root_count < 1 || root_count > decompressed_size) { + return td::Status::Error("BOC decompression failed: invalid root count"); + } + + std::vector root_indexes(root_count); + for (int i = 0; i < root_count; ++i) { + TRY_RESULT_ASSIGN(root_indexes[i], read_uint(bit_reader, 32)); + } + + // Read number of nodes from header + TRY_RESULT(node_count, read_uint(bit_reader, 32)); + if (node_count < 1) { + return td::Status::Error("BOC decompression failed: invalid node count"); + } + + // We assume that each cell should take at least 1 byte, even effectively serialized + // Otherwise it means that provided node_count is incorrect + if (node_count > decompressed_size) { + return td::Status::Error("BOC decompression failed: incorrect node count provided"); + } + + + // Validate root indexes + for (int i = 0; i < root_count; ++i) { + if (root_indexes[i] >= node_count) { + return td::Status::Error("BOC decompression failed: invalid root index"); + } + } + + // Initialize data structures + std::vector cell_data_length(node_count), is_data_small(node_count); + std::vector is_special(node_count), cell_refs_cnt(node_count); + std::vector prunned_branch_level(node_count, 0); + + std::vector cell_builders(node_count); + std::vector> boc_graph(node_count); + + // Read cell metadata + for (int i = 0; i < node_count; ++i) { + // Check enough bits for cell type and refs count + if (bit_reader.size() < 8) { + return td::Status::Error("BOC decompression failed: not enough bits for cell metadata"); + } + + size_t cell_type = bit_reader.bits().get_uint(4); + is_special[i] = bool(cell_type); + if (is_special[i]) { + prunned_branch_level[i] = cell_type - 1; + } + bit_reader.advance(4); + + cell_refs_cnt[i] = bit_reader.bits().get_uint(4); + bit_reader.advance(4); + if (cell_refs_cnt[i] > 4) { + return td::Status::Error("BOC decompression failed: invalid cell refs count"); + } + + if (prunned_branch_level[i]) { + size_t coef = std::bitset<4>(prunned_branch_level[i]).count(); + cell_data_length[i] = (256 + 16) * coef; + } else { + // Check enough bits for data length metadata + if (bit_reader.size() < 8) { + return td::Status::Error("BOC decompression failed: not enough bits for data length"); + } + + is_data_small[i] = bit_reader.bits().get_uint(1); + bit_reader.advance(1); + cell_data_length[i] = bit_reader.bits().get_uint(7); + bit_reader.advance(7); + + if (!is_data_small[i]) { + cell_data_length[i] *= 8; + if (!cell_data_length[i]) { + cell_data_length[i] += 1024; + } + } + } + + // Validate cell data length + if (cell_data_length[i] > kMaxCellDataLengthBits) { + return td::Status::Error("BOC decompression failed: invalid cell data length"); + } + } + + // Read direct edge connections + for (int i = 0; i < node_count; ++i) { + for (int j = 0; j < cell_refs_cnt[i]; ++j) { + TRY_RESULT(edge_connection, read_uint(bit_reader, 1)); + if (edge_connection) { + boc_graph[i][j] = i + 1; + } + } + } + + // Read initial cell data + for (int i = 0; i < node_count; ++i) { + if (prunned_branch_level[i]) { + cell_builders[i].store_long((1 << 8) + prunned_branch_level[i], 16); + } + + size_t remainder_bits = cell_data_length[i] % 8; + if (bit_reader.size() < remainder_bits) { + return td::Status::Error("BOC decompression failed: not enough bits for initial cell data"); + } + cell_builders[i].store_bits(bit_reader.subslice(0, remainder_bits)); + bit_reader.advance(remainder_bits); + cell_data_length[i] -= remainder_bits; + } + + // Decode remaining edge connections + for (size_t i = 0; i < node_count; ++i) { + if (node_count <= i + 3) { + for (int j = 0; j < cell_refs_cnt[i]; ++j) { + if (!boc_graph[i][j]) { + boc_graph[i][j] = i + 2; + } + } + continue; + } + + for (int j = 0; j < cell_refs_cnt[i]; ++j) { + if (!boc_graph[i][j]) { + size_t pref_size = (orig_size - bit_reader.size()); + size_t required_bits = 1 + (31 ^ __builtin_clz(node_count - i - 3)); + + if (required_bits < 8 - (pref_size + 1) % 8 + 1) { + TRY_RESULT_ASSIGN(boc_graph[i][j], read_uint(bit_reader, required_bits)); + boc_graph[i][j] += i + 2; + } else { + TRY_RESULT(edge_connection, read_uint(bit_reader, 1)); + if (edge_connection) { + pref_size = (orig_size - bit_reader.size()); + size_t available_bits = 8 - pref_size % 8; + TRY_RESULT_ASSIGN(boc_graph[i][j], read_uint(bit_reader, available_bits)); + boc_graph[i][j] += i + 2; + } else { + TRY_RESULT_ASSIGN(boc_graph[i][j], read_uint(bit_reader, required_bits)); + boc_graph[i][j] += i + 2; + } + } + } + } + } + + // Check if all graph connections are valid + for (int node = 0; node < node_count; ++node) { + for (int j = 0; j < cell_refs_cnt[node]; ++j) { + size_t child_node = boc_graph[node][j]; + if (child_node >= node_count) { + return td::Status::Error("BOC decompression failed: invalid graph connection"); + } + if (child_node <= node) { + return td::Status::Error("BOC decompression failed: circular reference in graph"); + } + } + } + + // Align to byte boundary + while ((orig_size - bit_reader.size()) % 8) { + TRY_RESULT(bit, read_uint(bit_reader, 1)); + } + + // Read remaining cell data + for (int i = 0; i < node_count; ++i) { + size_t padding_bits = 0; + if (!prunned_branch_level[i] && !is_data_small[i]) { + while (bit_reader.size() > 0 && bit_reader.bits()[0] == 0) { + ++padding_bits; + bit_reader.advance(1); + } + TRY_RESULT(bit, read_uint(bit_reader, 1)); + ++padding_bits; + } + if (cell_data_length[i] < padding_bits) { + return td::Status::Error("BOC decompression failed: invalid cell data length"); + } + size_t remaining_data_bits = cell_data_length[i] - padding_bits; + if (bit_reader.size() < remaining_data_bits) { + return td::Status::Error("BOC decompression failed: not enough bits for remaining cell data"); + } + + cell_builders[i].store_bits(bit_reader.subslice(0, remaining_data_bits)); + bit_reader.advance(remaining_data_bits); + } + + // Build cell tree + std::vector> nodes(node_count); + for (int i = node_count - 1; i >= 0; --i) { + try { + for (int child_index = 0; child_index < cell_refs_cnt[i]; ++child_index) { + size_t child = boc_graph[i][child_index]; + cell_builders[i].store_ref(nodes[child]); + } + try { + nodes[i] = cell_builders[i].finalize(is_special[i]); + } catch (vm::CellBuilder::CellWriteError& e) { + return td::Status::Error("BOC decompression failed: write error while finalizing cell."); + } + } catch (vm::VmError& e) { + return td::Status::Error("BOC decompression failed: VM error during cell construction"); + } + } + + std::vector> root_nodes; + root_nodes.reserve(root_count); + for (size_t index : root_indexes) { + root_nodes.push_back(nodes[index]); + } + + return root_nodes; +} + +td::Result boc_compress(const std::vector>& boc_roots, CompressionAlgorithm algo) { + // Check for empty input + if (boc_roots.empty()) { + return td::Status::Error("Cannot compress empty BOC roots"); + } + + td::BufferSlice compressed; + if (algo == CompressionAlgorithm::BaselineLZ4) { + TRY_RESULT_ASSIGN(compressed, boc_compress_baseline_lz4(boc_roots)); + } else if (algo == CompressionAlgorithm::ImprovedStructureLZ4) { + TRY_RESULT_ASSIGN(compressed, boc_compress_improved_structure_lz4(boc_roots)); + } else { + return td::Status::Error("Unknown compression algorithm"); + } + + td::BufferSlice compressed_with_algo(compressed.size() + 1); + compressed_with_algo.data()[0] = int(algo); + memcpy(compressed_with_algo.data() + 1, compressed.data(), compressed.size()); + return compressed_with_algo; +} + +td::Result>> boc_decompress(td::Slice compressed, int max_decompressed_size) { + if (compressed.size() == 0) { + return td::Status::Error("Can't decompress empty data"); + } + + int algo = int(compressed[0]); + compressed.remove_prefix(1); + + switch (algo) { + case int(CompressionAlgorithm::BaselineLZ4): + return boc_decompress_baseline_lz4(compressed, max_decompressed_size); + case int(CompressionAlgorithm::ImprovedStructureLZ4): + return boc_decompress_improved_structure_lz4(compressed, max_decompressed_size); + } + return td::Status::Error("Unknown compression algorithm"); +} + +} // namespace vm diff --git a/crypto/vm/boc-compression.h b/crypto/vm/boc-compression.h new file mode 100644 index 0000000000..f143edd1fb --- /dev/null +++ b/crypto/vm/boc-compression.h @@ -0,0 +1,42 @@ +/* +This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ +#pragma once + +#include "td/utils/Status.h" +#include "td/utils/buffer.h" +#include "vm/cells/CellSlice.h" +#include "vm/cells/CellBuilder.h" +#include "vm/excno.hpp" + +namespace vm { + +constexpr size_t kDecompressedSizeBytes = 4; + +enum class CompressionAlgorithm : int { BaselineLZ4 = 0, ImprovedStructureLZ4 = 1 }; + +td::Result boc_compress_baseline_lz4(const std::vector>& boc_roots); +td::Result>> boc_decompress_baseline_lz4(td::Slice compressed, int max_decompressed_size); + +td::Result boc_compress_improved_structure_lz4(const std::vector>& boc_roots); +td::Result>> boc_decompress_improved_structure_lz4(td::Slice compressed, int max_decompressed_size); + +td::Result boc_compress(const std::vector>& boc_roots, CompressionAlgorithm algo = CompressionAlgorithm::BaselineLZ4); +td::Result>> boc_decompress(td::Slice compressed, int max_decompressed_size); + +} // namespace vm diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 1b8e27365b..a6359a9877 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -343,6 +343,7 @@ validatorSession.candidateId src:int256 root_hash:int256 file_hash:int256 collat validatorSession.blockUpdate ts:long actions:(vector validatorSession.round.Message) state:int = validatorSession.BlockUpdate; validatorSession.candidate src:int256 round:int root_hash:int256 data:bytes collated_data:bytes = validatorSession.Candidate; validatorSession.compressedCandidate flags:# src:int256 round:int root_hash:int256 decompressed_size:int data:bytes = validatorSession.Candidate; +validatorSession.compressedCandidateV2 flags:# src:int256 round:int root_hash:int256 data:bytes = validatorSession.Candidate; validatorSession.config catchain_idle_timeout:double catchain_max_deps:int round_candidates:int next_candidate_delay:double round_attempt_duration:int max_round_attempts:int max_block_size:int max_collated_data_size:int = validatorSession.Config; @@ -430,6 +431,8 @@ tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_ha proof:bytes data:bytes = tonNode.Broadcast; tonNode.blockBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int flags:# compressed:bytes = tonNode.Broadcast; +tonNode.blockBroadcastCompressedV2 id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + signatures:(vector tonNode.blockSignature) flags:# compressed:bytes = tonNode.Broadcast; tonNode.ihrMessageBroadcast message:tonNode.ihrMessage = tonNode.Broadcast; tonNode.externalMessageBroadcast message:tonNode.externalMessage = tonNode.Broadcast; tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast; @@ -438,6 +441,8 @@ tonNode.newBlockCandidateBroadcast id:tonNode.blockIdExt catchain_seqno:int vali collator_signature:tonNode.blockSignature data:bytes = tonNode.Broadcast; tonNode.newBlockCandidateBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int collator_signature:tonNode.blockSignature flags:# compressed:bytes = tonNode.Broadcast; +tonNode.newBlockCandidateBroadcastCompressedV2 id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + collator_signature:tonNode.blockSignature flags:# compressed:bytes = tonNode.Broadcast; // optimistic broadcast of response to tonNode.getOutMsgQueueProof with dst_shard, block and limits arguments tonNode.outMsgQueueProofBroadcast dst_shard:tonNode.shardId block:tonNode.blockIdExt @@ -456,6 +461,7 @@ ton.blockIdApprove root_cell_hash:int256 file_hash:int256 = ton.BlockId; tonNode.dataFull id:tonNode.blockIdExt proof:bytes block:bytes is_link:Bool = tonNode.DataFull; tonNode.dataFullCompressed id:tonNode.blockIdExt flags:# compressed:bytes is_link:Bool = tonNode.DataFull; +tonNode.dataFullCompressedV2 id:tonNode.blockIdExt flags:# compressed:bytes is_link:Bool = tonNode.DataFull; tonNode.dataFullEmpty = tonNode.DataFull; tonNode.capabilities#f5bf60c0 version_major:int version_minor:int flags:# = tonNode.Capabilities; @@ -1000,6 +1006,7 @@ validatorStats.collatorNodeResponse self:int256 validator_id:int256 timestamp:do collatorNode.candidate source:PublicKey id:tonNode.blockIdExt data:bytes collated_data:bytes = collatorNode.Candidate; collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt decompressed_size:int data:bytes = collatorNode.Candidate; +collatorNode.compressedCandidateV2 flags:# source:PublicKey id:tonNode.blockIdExt data:bytes = collatorNode.Candidate; collatorNode.pong flags:# = collatorNode.Pong; collatorNode.error code:int message:string = collatorNode.Error; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 9a5241d8084e1eb2ff75ea71d00b934fec92235b..6475aede86c9bd31385620bc807bacdf14c86cd4 100644 GIT binary patch delta 810 zcmZWnO=uHA7|n+p^P?7H6O0;kDYc}{?z)Ys%^`uK_y?|UDHrB07u1|IQ7kCxj8ZDwY;N@B+*8gy zWP6PsjC+A21I=;4ZWk}Q4jxBJy#G<$$#nSmt{pzteaW|DY(uBA%0NYrFkt9SA6ahe zp7$r+H}8GV{Tx8|b>49;Y$HXSt$~a>dJM@+gLJO0&96M;h{f74{GkB<9fk-)ngCU7 zj6lXmI(ns>Hx{1^;6o!owY&2#u_(bMrWOK$X~-dv8B#-*uH3S?wpB&hVgU03cv){$ zuzF1o+l)(ykf6x4TCGV?`OnqZ-M)1RUllZ1OR#La~`D9^ME9Oe2oN9V8 z<0e7(s;~`fVF>z3?cd(3rMS&jB9+DlDNv=yFY%KwqCt!}K49j8nj3+b0+?eqn?J$%pR}(-)X9mVkNF zB}^H05aJ1@j5QGPHD-(&5}+Ui`GWxnik#VWIBeSvg2cAJF=I>-W&s7~eMr`I6!Ppm}dZHY|q%wcmd+d d={5&}o&eiba*$Dj4`Swm^Nc> deserialize_candi if (!compression_enabled) { return fetch_tl_object(data, true); } - TRY_RESULT(f, fetch_tl_object(data, true)); - if (f->decompressed_size_ > max_decompressed_data_size) { - return td::Status::Error("decompressed size is too big"); - } - TRY_RESULT(p, decompress_candidate_data(f->data_, f->decompressed_size_, proto_version)); - return create_tl_object(f->src_, f->round_, f->root_hash_, std::move(p.first), - std::move(p.second)); + TRY_RESULT(f, fetch_tl_object(data, true)); + td::Result> res; + ton_api::downcast_call(*f, td::overloaded( + [&](ton_api::validatorSession_candidate& c) { + res = td::Status::Error("Received decompressed tl object, while compression_enabled=true"); + }, + [&](ton_api::validatorSession_compressedCandidate& c) { + res = [&]() -> td::Result> { + if (c.decompressed_size_ > max_decompressed_data_size) { + return td::Status::Error("decompressed size is too big"); + } + TRY_RESULT(p, decompress_candidate_data(c.data_, false, c.decompressed_size_, + max_decompressed_data_size, proto_version)); + return create_tl_object(c.src_, c.round_, c.root_hash_, std::move(p.first), + std::move(p.second)); + }(); + }, + [&](ton_api::validatorSession_compressedCandidateV2& c) { + res = [&]() -> td::Result> { + if (c.data_.size() > max_decompressed_data_size) { + return td::Status::Error("Compressed data is too big"); + } + TRY_RESULT(p, decompress_candidate_data(c.data_, true, 0, + max_decompressed_data_size, proto_version)); + return create_tl_object(c.src_, c.round_, c.root_hash_, std::move(p.first), + std::move(p.second)); + }(); + })); + return res; } td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, @@ -69,13 +92,20 @@ td::Result compress_candidate_data(td::Slice block, td::Slice c } td::Result> decompress_candidate_data(td::Slice compressed, + bool improved_compression, int decompressed_size, + int max_decompressed_size, int proto_version) { - TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size)); - if (decompressed.size() != (size_t)decompressed_size) { - return td::Status::Error("decompressed size mismatch"); + std::vector> roots; + if (!improved_compression) { + TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size)); + if (decompressed.size() != (size_t)decompressed_size) { + return td::Status::Error("decompressed size mismatch"); + } + TRY_RESULT_ASSIGN(roots, vm::std_boc_deserialize_multi(decompressed)); + } else { + TRY_RESULT_ASSIGN(roots, vm::boc_decompress(compressed, max_decompressed_size)); } - TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed)); if (roots.empty()) { return td::Status::Error("boc is empty"); } @@ -83,7 +113,7 @@ td::Result> decompress_candidate_dat roots.erase(roots.begin()); int collated_data_mode = proto_version >= 5 ? 2 : 31; TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), collated_data_mode)); - LOG(DEBUG) << "Decompressing block candidate: " << compressed.size() << " -> " + LOG(DEBUG) << "Decompressing block candidate " << (improved_compression ? "V2:" : ":") << compressed.size() << " -> " << block_data.size() + collated_data.size(); return std::make_pair(std::move(block_data), std::move(collated_data)); } diff --git a/validator-session/candidate-serializer.h b/validator-session/candidate-serializer.h index 7cc77f0be3..9ab53cc89c 100644 --- a/validator-session/candidate-serializer.h +++ b/validator-session/candidate-serializer.h @@ -30,7 +30,9 @@ td::Result> deserialize_candi td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, size_t& decompressed_size); td::Result> decompress_candidate_data(td::Slice compressed, + bool improved_compression, int decompressed_size, + int max_decompressed_size, int proto_version); } // namespace ton::validatorsession diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 4b0d7e140b..82ca99cd23 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -664,7 +664,21 @@ td::Result CollatorNode::deserialize_candidate(tl_object_ptr td::Result { + TRY_RESULT(p, validatorsession::decompress_candidate_data( + c.data_, true, 0, max_decompressed_data_size, proto_version)); auto collated_data_hash = td::sha256_bits256(p.second); auto key = ton::PublicKey{c.source_}; if (!key.is_ed25519()) { diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index fa2e03b2e1..3b4a36c6db 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -35,6 +35,10 @@ void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonN process_block_broadcast(src, query); } +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query) { + process_block_broadcast(src, query); +} + void FullNodeFastSyncOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); if (B.is_error()) { @@ -94,6 +98,11 @@ void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, process_block_candidate_broadcast(src, query); } +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query) { + process_block_candidate_broadcast(src, query); +} + void FullNodeFastSyncOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { BlockIdExt block_id; CatchainSeqno cc_seqno; diff --git a/validator/full-node-fast-sync-overlays.hpp b/validator/full-node-fast-sync-overlays.hpp index e0db87b7fe..41a45cbc71 100644 --- a/validator/full-node-fast-sync-overlays.hpp +++ b/validator/full-node-fast-sync-overlays.hpp @@ -25,6 +25,7 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { public: void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast& query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed& query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2& query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_outMsgQueueProofBroadcast& query); void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast& query); @@ -32,6 +33,7 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast& query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed& query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressedV2& query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast& query); void process_telemetry_broadcast(adnl::AdnlNodeIdShort src, diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index 11c3e9d611..fd5e970ba0 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -34,6 +34,11 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, process_block_broadcast(src, query); } +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_blockBroadcastCompressedV2 &query) { + process_block_broadcast(src, query); +} + void FullNodePrivateBlockOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); if (B.is_error()) { @@ -63,6 +68,11 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, process_block_candidate_broadcast(src, query); } +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query) { + process_block_candidate_broadcast(src, query); +} + void FullNodePrivateBlockOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { BlockIdExt block_id; @@ -292,6 +302,10 @@ void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNod process_block_broadcast(src, query); } +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query) { + process_block_broadcast(src, query); +} + void FullNodeCustomOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { VLOG(FULL_NODE_DEBUG) << "Dropping block broadcast in private overlay \"" << name_ << "\" from unauthorized sender " @@ -330,6 +344,11 @@ void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, process_block_candidate_broadcast(src, query); } +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query) { + process_block_candidate_broadcast(src, query); +} + void FullNodeCustomOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { VLOG(FULL_NODE_DEBUG) << "Dropping block candidate broadcast in private overlay \"" << name_ diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index 70e196ea68..355e448587 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -25,12 +25,14 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { public: void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query); void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_telemetry_broadcast(PublicKeyHash src, const tl_object_ptr& telemetry); @@ -106,12 +108,14 @@ class FullNodeCustomOverlay : public td::actor::Actor { public: void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query); void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); template diff --git a/validator/full-node-serializer.cpp b/validator/full-node-serializer.cpp index 94dc2155e8..5665cc2339 100644 --- a/validator/full-node-serializer.cpp +++ b/validator/full-node-serializer.cpp @@ -20,6 +20,7 @@ #include "auto/tl/ton_api.hpp" #include "tl-utils/tl-utils.hpp" #include "vm/boc.h" +#include "vm/boc-compression.h" #include "td/utils/lz4.h" #include "full-node.h" #include "td/utils/overloaded.h" @@ -88,6 +89,28 @@ static td::Result deserialize_block_broadcast(ton_api::tonNode_b std::move(proof)}; } +static td::Result deserialize_block_broadcast(ton_api::tonNode_blockBroadcastCompressedV2& f, + int max_decompressed_size) { + std::vector signatures; + for (auto& sig : f.signatures_) { + signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); + } + TRY_RESULT(roots, vm::boc_decompress(f.compressed_, max_decompressed_size)); + if (roots.size() != 2) { + return td::Status::Error("expected 2 roots in boc"); + } + TRY_RESULT(proof, vm::std_boc_serialize(roots[0], 0)); + TRY_RESULT(data, vm::std_boc_serialize(roots[1], 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block broadcast: " << f.compressed_.size() << " -> " + << data.size() + proof.size() + signatures.size() * 96; + return BlockBroadcast{create_block_id(f.id_), + std::move(signatures), + static_cast(f.catchain_seqno_), + static_cast(f.validator_set_hash_), + std::move(data), + std::move(proof)}; +} + td::Result deserialize_block_broadcast(ton_api::tonNode_Broadcast& obj, int max_decompressed_data_size) { td::Result B; @@ -96,6 +119,9 @@ td::Result deserialize_block_broadcast(ton_api::tonNode_Broadcas [&](ton_api::tonNode_blockBroadcastCompressed& f) { B = deserialize_block_broadcast(f, max_decompressed_data_size); }, + [&](ton_api::tonNode_blockBroadcastCompressedV2& f) { + B = deserialize_block_broadcast(f, max_decompressed_data_size); + }, [&](auto&) { B = td::Status::Error("unknown broadcast type"); })); return B; } @@ -139,6 +165,20 @@ static td::Status deserialize_block_full(ton_api::tonNode_dataFullCompressed& f, return td::Status::OK(); } +static td::Status deserialize_block_full(ton_api::tonNode_dataFullCompressedV2& f, BlockIdExt& id, td::BufferSlice& proof, + td::BufferSlice& data, bool& is_proof_link, int max_decompressed_size) { + TRY_RESULT(roots, vm::boc_decompress(f.compressed_, max_decompressed_size)); + if (roots.size() != 2) { + return td::Status::Error("expected 2 roots in boc"); + } + TRY_RESULT_ASSIGN(proof, vm::std_boc_serialize(roots[0], 0)); + TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(roots[1], 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block full V2: " << f.compressed_.size() << " -> " << data.size() + proof.size(); + id = create_block_id(f.id_); + is_proof_link = f.is_link_; + return td::Status::OK(); +} + td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id, td::BufferSlice& proof, td::BufferSlice& data, bool& is_proof_link, int max_decompressed_data_size) { td::Status S; @@ -148,6 +188,9 @@ td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id [&](ton_api::tonNode_dataFullCompressed& f) { S = deserialize_block_full(f, id, proof, data, is_proof_link, max_decompressed_data_size); }, + [&](ton_api::tonNode_dataFullCompressedV2& f) { + S = deserialize_block_full(f, id, proof, data, is_proof_link, max_decompressed_data_size); + }, [&](auto&) { S = td::Status::Error("unknown data type"); })); return S; } @@ -194,6 +237,24 @@ static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBloc return td::Status::OK(); } +static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBlockCandidateBroadcastCompressedV2& obj, + BlockIdExt& block_id, CatchainSeqno& cc_seqno, + td::uint32& validator_set_hash, td::BufferSlice& data, + int max_decompressed_data_size) { + block_id = create_block_id(obj.id_); + cc_seqno = obj.catchain_seqno_; + validator_set_hash = obj.validator_set_hash_; + TRY_RESULT(roots, vm::boc_decompress(obj.compressed_, max_decompressed_data_size)); + if (roots.size() != 1) { + return td::Status::Error("expected 1 root in boc"); + } + auto root = std::move(roots[0]); + TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(root, 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block candidate broadcast V2: " << obj.compressed_.size() << " -> " + << data.size(); + return td::Status::OK(); +} + td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_Broadcast& obj, BlockIdExt& block_id, CatchainSeqno& cc_seqno, td::uint32& validator_set_hash, td::BufferSlice& data, int max_decompressed_data_size) { @@ -207,6 +268,10 @@ td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_Broadcast& obj S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, data, max_decompressed_data_size); }, + [&](ton_api::tonNode_newBlockCandidateBroadcastCompressedV2& f) { + S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, + data, max_decompressed_data_size); + }, [&](auto&) { S = td::Status::Error("unknown data type"); })); return S; } diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index f5a1e0b168..771b28be38 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -804,6 +804,11 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, process_block_candidate_broadcast(src, query); } +void FullNodeShardImpl::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query) { + process_block_candidate_broadcast(src, query); +} + void FullNodeShardImpl::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { BlockIdExt block_id; CatchainSeqno cc_seqno; @@ -832,6 +837,10 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_bl process_block_broadcast(src, query); } +void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query) { + process_block_broadcast(src, query); +} + void FullNodeShardImpl::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); if (B.is_error()) { diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 534693d959..a90460adb5 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -152,6 +152,7 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query); void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query); @@ -163,6 +164,7 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void receive_broadcast(PublicKeyHash src, td::BufferSlice query); From f4468b88f6ed3cfd6562a5d64603b899e2db6388 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 1 Aug 2025 16:48:36 +0300 Subject: [PATCH 362/524] Send broadcasts earlier in AcceptBlock --- validator/impl/accept-block.cpp | 29 +++++++++++++++-------------- validator/impl/accept-block.hpp | 1 + 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index cbb03bac13..b9aea1e7f0 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -494,6 +494,17 @@ void AcceptBlockQuery::written_block_signatures() { void AcceptBlockQuery::written_block_info() { VLOG(VALIDATOR_DEBUG) << "written block info"; if (data_.not_null()) { + block_root_ = data_->root_cell(); + if (block_root_.is_null()) { + fatal_error("block data does not contain a root cell"); + return; + } + // generate proof + if (!create_new_proof()) { + fatal_error("cannot generate proof for block "s + id_.to_str()); + return; + } + send_broadcasts(); if (!apply_) { written_state({}); return; @@ -563,17 +574,6 @@ void AcceptBlockQuery::written_state(td::Ref upd_state) { CHECK(data_.not_null()); state_ = std::move(upd_state); - block_root_ = data_->root_cell(); - if (block_root_.is_null()) { - fatal_error("block data does not contain a root cell"); - return; - } - // generate proof - if (!create_new_proof()) { - fatal_error("cannot generate proof for block "s + id_.to_str()); - return; - } - if (apply_ && state_keep_old_hash_ != state_old_hash_) { fatal_error(PSTRING() << "invalid previous state hash in newly-created proof: expected " << state_->root_hash().to_hex() << ", found in update " << state_old_hash_.to_hex()); @@ -935,8 +935,11 @@ void AcceptBlockQuery::written_block_info_2() { } void AcceptBlockQuery::applied() { + finish_query(); +} + +void AcceptBlockQuery::send_broadcasts() { if (send_broadcast_mode_ == 0) { - finish_query(); return; } BlockBroadcast b; @@ -964,8 +967,6 @@ void AcceptBlockQuery::applied() { // td::actor::send_closure(manager_, &ValidatorManager::send_block_candidate_broadcast, id_, // validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), // std::move(b.data), send_broadcast_mode_); - - finish_query(); } } // namespace validator diff --git a/validator/impl/accept-block.hpp b/validator/impl/accept-block.hpp index d8b763ba03..e582650c18 100644 --- a/validator/impl/accept-block.hpp +++ b/validator/impl/accept-block.hpp @@ -89,6 +89,7 @@ class AcceptBlockQuery : public td::actor::Actor { void written_block_next(); void written_block_info_2(); void applied(); + void send_broadcasts(); private: BlockIdExt id_; From 5fa41216b969649658476694ee7bb52465e04ae8 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Mon, 4 Aug 2025 18:27:05 +0400 Subject: [PATCH 363/524] Tunnel small refactor --- adnl/adnl-network-manager.cpp | 2 +- tdnet/td/net/UdpServer.cpp | 6 ++--- validator-engine/validator-engine.cpp | 36 ++++++++++++++++----------- validator-engine/validator-engine.hpp | 1 + 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/adnl/adnl-network-manager.cpp b/adnl/adnl-network-manager.cpp index 99a74eb3b6..6d748e34ab 100644 --- a/adnl/adnl-network-manager.cpp +++ b/adnl/adnl-network-manager.cpp @@ -76,7 +76,7 @@ size_t AdnlNetworkManagerImpl::add_listening_udp_port(td::uint16 port) { return idx; } -#define TUNNEL_FAKE_PORT 1 +constexpr int TUNNEL_FAKE_PORT = 0; size_t AdnlNetworkManagerImpl::add_tunnel_udp_port(std::string global_config, std::string tunnel_config, td::Promise on_ready, td::actor::Scheduler *scheduler) { diff --git a/tdnet/td/net/UdpServer.cpp b/tdnet/td/net/UdpServer.cpp index 8d05a2bb89..cae5c4d514 100644 --- a/tdnet/td/net/UdpServer.cpp +++ b/tdnet/td/net/UdpServer.cpp @@ -35,9 +35,9 @@ int VERBOSITY_NAME(udp_server) = VERBOSITY_NAME(DEBUG) + 10; } namespace detail { -#define TUNNEL_BUFFER_SZ_PACKETS 100 -#define TUNNEL_MAX_PACKET_MTU 1500 -#define TUNNEL_ALARM_EVERY 0.01 +constexpr int TUNNEL_BUFFER_SZ_PACKETS = 100; +constexpr int TUNNEL_MAX_PACKET_MTU = 1500; +constexpr double TUNNEL_ALARM_EVERY = 0.01; class UdpServerTunnelImpl : public UdpServer { public: diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 8645d2280d..87e454d2ad 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -2094,31 +2094,24 @@ void ValidatorEngine::start_adnl() { class Handler : public ton::adnl::AdnlNetworkManager::TunnelEventsHandler { public: - Handler(ValidatorEngine *scheduler) - : validator_engine_(scheduler) { + Handler(ValidatorEngine *scheduler, const td::actor::ActorId &actor) + : validator_engine_(scheduler), validator_engine_actor_(actor) { } private: ValidatorEngine *validator_engine_; - void on_in_addr_update(td::IPAddress ip) override { - validator_engine_->scheduler_->run_in_context_external([&] { - LOG(INFO) << "[EVENT] Tunnel reinitialized, addr: " << ip; + td::actor::ActorId validator_engine_actor_; - validator_engine_->addr_lists_.clear(); - validator_engine_->add_addr(Config::Addr{}, Config::AddrCats{ - .in_addr = ip, - .is_tunnel = true, - .cats = {0, 1, 2, 3}, - }); + void on_in_addr_update(td::IPAddress ip) override { + LOG(INFO) << "[EVENT] Tunnel reinitialized, addr: " << ip; - for (auto &adnl : validator_engine_->config_.adnl_ids) { - validator_engine_->add_adnl(adnl.first, adnl.second); - } + validator_engine_->scheduler_->run_in_context_external([&] { + td::actor::send_closure(validator_engine_actor_, &ValidatorEngine::reinit_tunnel, ip); }); } }; - td::actor::send_closure(adnl_network_manager_, &ton::adnl::AdnlNetworkManager::install_tunnel_events_handler, std::make_unique(this)); + td::actor::send_closure(adnl_network_manager_, &ton::adnl::AdnlNetworkManager::install_tunnel_events_handler, std::make_unique(this, actor_id(this))); ton::adnl::AdnlCategoryMask cat_mask; for (int i = 0; i <= 3; i++) { @@ -2142,6 +2135,19 @@ void ValidatorEngine::start_adnl() { started_adnl(); } +void ValidatorEngine::reinit_tunnel(td::IPAddress ip) { + this->addr_lists_.clear(); + this->add_addr(Config::Addr{}, Config::AddrCats{ + .in_addr = ip, + .is_tunnel = true, + .cats = {0, 1, 2, 3}, + }); + + for (auto &adnl : this->config_.adnl_ids) { + this->add_adnl(adnl.first, adnl.second); + } +} + void ValidatorEngine::add_addr(const Config::Addr &addr, const Config::AddrCats &cats) { ton::adnl::AdnlCategoryMask cat_mask; for (auto cat : cats.cats) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 1568068913..1ea4a372fa 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -213,6 +213,7 @@ class ValidatorEngine : public td::actor::Actor { void got_state(td::Ref state); void write_config(td::Promise promise); + void reinit_tunnel(td::IPAddress ip); std::map addr_lists_; std::map prio_addr_lists_; From 5f25056c8ff59469f93e0dab396ed62603e92a4b Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Mon, 4 Aug 2025 19:09:40 +0400 Subject: [PATCH 364/524] Pass scheduler to tunnel handler instead of engine --- validator-engine/validator-engine.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 87e454d2ad..cdeb5d9176 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -2094,24 +2094,24 @@ void ValidatorEngine::start_adnl() { class Handler : public ton::adnl::AdnlNetworkManager::TunnelEventsHandler { public: - Handler(ValidatorEngine *scheduler, const td::actor::ActorId &actor) - : validator_engine_(scheduler), validator_engine_actor_(actor) { + Handler(td::actor::Scheduler *scheduler, const td::actor::ActorId &actor) + : scheduler_(scheduler), validator_engine_actor_(actor) { } private: - ValidatorEngine *validator_engine_; + td::actor::Scheduler* scheduler_; td::actor::ActorId validator_engine_actor_; void on_in_addr_update(td::IPAddress ip) override { LOG(INFO) << "[EVENT] Tunnel reinitialized, addr: " << ip; - validator_engine_->scheduler_->run_in_context_external([&] { + scheduler_->run_in_context_external([&] { td::actor::send_closure(validator_engine_actor_, &ValidatorEngine::reinit_tunnel, ip); }); } }; - td::actor::send_closure(adnl_network_manager_, &ton::adnl::AdnlNetworkManager::install_tunnel_events_handler, std::make_unique(this, actor_id(this))); + td::actor::send_closure(adnl_network_manager_, &ton::adnl::AdnlNetworkManager::install_tunnel_events_handler, std::make_unique(this->scheduler_, actor_id(this))); ton::adnl::AdnlCategoryMask cat_mask; for (int i = 0; i <= 3; i++) { From d2cc797857bdfafaeeb5363b9414a48e02d208c2 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 4 Aug 2025 18:28:58 +0300 Subject: [PATCH 365/524] Optimistic block generation --- tdutils/td/utils/Timer.h | 3 + tl/generate/scheme/ton_api.tl | 3 +- tl/generate/scheme/ton_api.tlo | Bin 118872 -> 119192 bytes validator-session/validator-session.cpp | 200 ++++++++++++++++---- validator-session/validator-session.h | 3 + validator-session/validator-session.hpp | 35 ++++ validator/collation-manager.cpp | 45 ++++- validator/collation-manager.hpp | 6 + validator/collator-node.cpp | 27 +-- validator/fabric.h | 28 ++- validator/impl/collator-impl.h | 22 ++- validator/impl/collator.cpp | 231 ++++++++++++++++------- validator/impl/fabric.cpp | 35 +--- validator/interfaces/validator-manager.h | 7 +- validator/manager-disk.cpp | 9 +- validator/manager-hardfork.cpp | 4 +- validator/validator-group.cpp | 120 ++++++++++-- validator/validator-group.hpp | 23 +++ 18 files changed, 606 insertions(+), 195 deletions(-) diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index d1ba104b09..de793aec07 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -160,6 +160,9 @@ class PerfLog { }; template double PerfLogAction::finish(const T &result) { + if (!perf_log_) { + return 0.0; + } if (result.is_ok()) { return perf_log_->finish_action(i_, td::Status::OK()); } else { diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index a6359a9877..38dc33489b 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -344,6 +344,7 @@ validatorSession.blockUpdate ts:long actions:(vector validatorSession.round.Mess validatorSession.candidate src:int256 round:int root_hash:int256 data:bytes collated_data:bytes = validatorSession.Candidate; validatorSession.compressedCandidate flags:# src:int256 round:int root_hash:int256 decompressed_size:int data:bytes = validatorSession.Candidate; validatorSession.compressedCandidateV2 flags:# src:int256 round:int root_hash:int256 data:bytes = validatorSession.Candidate; +validatorSession.optimisticCandidateBroadcast flags:# prev_candidate_id:int256 data:bytes = validatorSession.OptimisticCandidateBroadcast; validatorSession.config catchain_idle_timeout:double catchain_max_deps:int round_candidates:int next_candidate_delay:double round_attempt_duration:int max_round_attempts:int max_block_size:int max_collated_data_size:int = validatorSession.Config; @@ -962,7 +963,7 @@ validatorStats.blockStats old_out_msg_queue_size:long new_out_msg_queue_size:long msg_queue_cleaned:int neighbors:(vector validatorStats.blockStats.neighborStats) = validatorStats.BlockStats; validatorStats.collateWorkTimeStats - total:double queue_cleanup:double prelim_storage_stat:double trx_tvm:double trx_storage_stat:double + total:double optimistic_apply:double queue_cleanup:double prelim_storage_stat:double trx_tvm:double trx_storage_stat:double trx_other:double final_storage_stat:double create_block:double create_collated_data:double create_block_candidate:double = validatorStats.CollateWorkTimeStats; validatorStats.storageStatCacheStats diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 6475aede86c9bd31385620bc807bacdf14c86cd4..3b64f7517b1ad5fa9ef9bf964ec3917078c2d098 100644 GIT binary patch delta 273 zcmcbyfPKbdcHT#`^{p77z;h$-Z(D`5L+lQ^Wr;bNDTyWdMZu}X#hLkedj16^nYo$8 zC7H?76ZIH{CpQaAZsxFCz{oP|uJ4b{YwR auto to_approve = real_state_->choose_blocks_to_approve(description(), local_idx()); for (auto &block : to_approve) { auto id = SentBlock::get_block_id(block); - if (approved_.count(id) && approved_[id].first <= td::Clocks::system()) { + if (approved_.contains(id) && approved_[id].first <= td::Clocks::system()) { msgs.emplace_back(create_tl_object( cur_round_, id, approved_[id].second.clone())); cnt++; @@ -223,6 +222,19 @@ bool ValidatorSessionImpl::ensure_candidate_unique(td::uint32 src_idx, td::uint3 void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice data, td::optional expected_id, bool is_overlay_broadcast, bool is_startup) { + bool is_optimistic = false; + ValidatorSessionCandidateId optimistic_prev_candidate = ValidatorSessionCandidateId::zero(); + if (is_overlay_broadcast) { + auto R = fetch_tl_object(data, true); + if (R.is_ok()) { + if (src == local_id()) { + return; + } + is_optimistic = true; + optimistic_prev_candidate = R.ok_ref()->prev_candidate_id_; + data = std::move(R.ok_ref()->data_); + } + } // Note: src is not necessarily equal to the sender of this message: // If requested using get_broadcast_p2p, src is the creator of the block, sender possibly is some other node. auto src_idx = description().get_source_idx(src); @@ -262,13 +274,59 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice return; } + if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || + block_round >= cur_round_ + MAX_FUTURE_ROUND_BLOCK || (block_round == 0 && is_optimistic)) { + VLOG(VALIDATOR_SESSION_NOTICE) << this << "[node " << src << "][broadcast " << block_id + << "]: bad round=" << block_round << " cur_round" << cur_round_; + return; + } + + BroadcastInfo broadcast_info{.candidate_id = block_id, + .received_at = td::Clocks::system(), + .deserialize_time = deserialize_time, + .serialized_size = data.size(), + .file_hash = file_hash, + .collated_data_hash = collated_data_file_hash}; + + if (is_optimistic) { + if (optimistic_broadcasts_.contains({block_round, src_idx})) { + VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) + << "]: duplicate optimistic broadcast for round " << block_round; + return; + } + if (block_round > cur_round_) { + OptimisticBroadcast &optimistic_broadcast = optimistic_broadcasts_[{block_round, src_idx}]; + optimistic_broadcast.candidate = std::move(candidate); + optimistic_broadcast.prev_candidate_id = optimistic_prev_candidate; + optimistic_broadcast.broadcast_info = broadcast_info; + VLOG(VALIDATOR_SESSION_WARNING) << this << ": received optimistic broadcast " << block_id << " from " << src + << ", round " << block_round; + return; + } + if (SentBlock::get_block_id(real_state_->get_committed_block(description(), block_round - 1)) != + optimistic_prev_candidate) { + VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) + << "]: dropping optimistic broadcast for round " << block_round + << " - prev candidate mismatch"; + return; + } + } + process_received_block(block_round, src, src_idx, std::move(candidate), broadcast_info, is_overlay_broadcast, + is_startup); +} + +void ValidatorSessionImpl::process_received_block(td::uint32 block_round, PublicKeyHash src, td::uint32 src_idx, + tl_object_ptr candidate, + const BroadcastInfo &info, bool is_overlay_broadcast, + bool is_startup) { + ValidatorSessionCandidateId block_id = info.candidate_id; auto stat = stats_get_candidate_stat(block_round, src, block_id); if (stat) { if (stat->block_status == ValidatorSessionStats::status_none) { stat->block_status = ValidatorSessionStats::status_received; } if (stat->got_block_at <= 0.0) { - stat->got_block_at = td::Clocks::system(); + stat->got_block_at = info.received_at; if (is_overlay_broadcast) { stat->got_block_by = ValidatorSessionStats::recv_broadcast; } else if (is_startup) { @@ -277,19 +335,13 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice stat->got_block_by = ValidatorSessionStats::recv_query; } } - stat->deserialize_time = deserialize_time; - stat->serialized_size = data.size(); + stat->deserialize_time = info.deserialize_time; + stat->serialized_size = info.serialized_size; stat->block_id.root_hash = candidate->root_hash_; - stat->block_id.file_hash = file_hash; - stat->collated_data_hash = collated_data_file_hash; + stat->block_id.file_hash = info.file_hash; + stat->collated_data_hash = info.collated_data_hash; } - if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || - block_round >= cur_round_ + MAX_FUTURE_ROUND_BLOCK) { - VLOG(VALIDATOR_SESSION_NOTICE) << this << "[node " << src << "][broadcast " << block_id - << "]: bad round=" << block_round << " cur_round" << cur_round_; - return; - } auto it = blocks_.find(block_id); if (it != blocks_.end()) { it->second->round_ = std::max(it->second->round_, block_round); @@ -315,10 +367,10 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice return; } - CHECK(!pending_approve_.count(block_id)); - CHECK(!approved_.count(block_id)); - CHECK(!pending_reject_.count(block_id)); - CHECK(!rejected_.count(block_id)); + CHECK(!pending_approve_.contains(block_id)); + CHECK(!approved_.contains(block_id)); + CHECK(!pending_reject_.contains(block_id)); + CHECK(!rejected_.contains(block_id)); auto v = virtual_state_->choose_blocks_to_approve(description(), local_idx()); for (auto &b : v) { @@ -487,19 +539,15 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, GeneratedCandidate if (round != cur_round_) { return; } - td::Timer serialize_timer; - auto b = create_tl_object(local_id().tl(), round, c.candidate.id.root_hash, - std::move(c.candidate.data), - std::move(c.candidate.collated_data)); - auto B = serialize_candidate(b, compress_block_candidates_).move_as_ok(); + SentCandidateStats &send_stats = send_candidate_broadcast(round, c.candidate); if (stat) { - stat->serialize_time = serialize_timer.elapsed(); - stat->serialized_size = B.size(); + stat->serialize_time = send_stats.serialize_time; + stat->serialized_size = send_stats.serialized_size; } - td::actor::send_closure(catchain_, &catchain::CatChain::send_broadcast, std::move(B)); - - blocks_.emplace(block_id, std::move(b)); + blocks_[block_id] = create_tl_object( + local_id().tl(), round, c.candidate.id.root_hash, std::move(c.candidate.data), + std::move(c.candidate.collated_data)); pending_generate_ = false; generated_ = true; generated_block_ = block_id; @@ -507,6 +555,31 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, GeneratedCandidate request_new_block(true); } +ValidatorSessionImpl::SentCandidateStats &ValidatorSessionImpl::send_candidate_broadcast( + td::uint32 round, const BlockCandidate &candidate, + td::optional optimistic_prev_candidate) { + ValidatorSessionCandidateId candidate_id = description().candidate_id( + local_idx(), candidate.id.root_hash, candidate.id.file_hash, candidate.collated_file_hash); + SentCandidateStats &stats = sent_candidates_[candidate_id]; + if (stats.sent) { + return stats; + } + stats.sent = true; + td::Timer serialize_timer; + auto b = create_tl_object( + local_id().tl(), round, candidate.id.root_hash, candidate.data.clone(), candidate.collated_data.clone()); + auto data = serialize_candidate(b, compress_block_candidates_).move_as_ok(); + stats.serialize_time = serialize_timer.elapsed(); + stats.sent_at = td::Clocks::system(); + stats.serialized_size = data.size(); + if (optimistic_prev_candidate) { + data = create_serialize_tl_object( + 0, optimistic_prev_candidate.value(), std::move(data)); + } + td::actor::send_closure(catchain_, &catchain::CatChain::send_broadcast, std::move(data)); + return stats; +} + void ValidatorSessionImpl::signed_block(td::uint32 round, ValidatorSessionCandidateId hash, td::BufferSlice signature) { if (round != cur_round_) { return; @@ -586,7 +659,7 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { return; } } - if (pending_approve_.count(block_id) || rejected_.count(block_id)) { + if (pending_approve_.contains(block_id) || rejected_.contains(block_id)) { return; } @@ -651,7 +724,7 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { description().get_node_priority(block->get_src_idx(), cur_round_)}}, B->root_hash_, B->data_.clone(), B->collated_data_.clone(), std::move(P)); } else if (T.is_in_past()) { - if (!active_requests_.count(block_id)) { + if (!active_requests_.contains(block_id)) { auto v = virtual_state_->get_block_approvers(description(), block_id); if (v.size() > 0) { auto id = description().get_source_id(v[td::Random::fast(0, static_cast(v.size() - 1))]); @@ -796,9 +869,9 @@ void ValidatorSessionImpl::check_all() { for (auto &B : to_approve) { if (B) { auto block_id = SentBlock::get_block_id(B); - auto pending = pending_approve_.count(block_id) == 1; - auto rejected = rejected_.count(block_id) == 1; - auto accepted = approved_.count(block_id) == 1; + auto pending = pending_approve_.contains(block_id); + auto rejected = rejected_.contains(block_id); + auto accepted = approved_.contains(block_id); sb << " " << block_id << " pending: " << pending << " rejected: " << rejected << " accepted: " << accepted << "\n"; } else { @@ -960,6 +1033,18 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { round_started_at_ = td::Timestamp::now(); round_debug_at_ = td::Timestamp::in(60.0); + for (auto it = optimistic_broadcasts_.begin(); it != optimistic_broadcasts_.end() && it->first.first <= cur_round_;) { + OptimisticBroadcast &optimistic_broadcast = it->second; + auto [block_round, src_idx] = it->first; + if (SentBlock::get_block_id(real_state_->get_committed_block(description(), block_round - 1)) == + optimistic_broadcast.prev_candidate_id) { + process_received_block(block_round, description().get_source_id(src_idx), src_idx, + std::move(optimistic_broadcast.candidate), optimistic_broadcast.broadcast_info, true, + false); + } + it = optimistic_broadcasts_.erase(it); + } + check_all(); } @@ -1010,6 +1095,7 @@ ValidatorSessionImpl::ValidatorSessionImpl(catchain::CatChainSessionId session_i , overlay_manager_(overlays) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) { compress_block_candidates_ = opts.proto_version >= 4; + allow_optimistic_generation_ = opts.proto_version >= 5; description_ = ValidatorSessionDescription::create(std::move(opts), nodes, local_id); src_round_candidate_.resize(description_->get_total_nodes()); } @@ -1200,7 +1286,7 @@ ValidatorSessionStats::Producer *ValidatorSessionImpl::stats_get_candidate_stat( auto it2 = stats_pending_approve_.find({round, it->candidate_id}); if (it2 != stats_pending_approve_.end()) { for (td::uint32 node_id : it2->second) { - it->set_approved_by(node_id, description().get_node_weight(node_id), description().get_total_weight()); + process_approve(node_id, round, it->candidate_id); } stats_pending_approve_.erase(it2); } @@ -1249,13 +1335,7 @@ void ValidatorSessionImpl::stats_process_action(td::uint32 node_id, ton_api::val if (obj.candidate_ == skip_round_candidate_id()) { return; } - auto stat = stats_get_candidate_stat_by_id(obj.round_, obj.candidate_); - if (stat) { - stat->set_approved_by(node_id, description().get_node_weight(node_id), - description().get_total_weight()); - } else { - stats_pending_approve_[{obj.round_, obj.candidate_}].push_back(node_id); - } + process_approve(node_id, obj.round_, obj.candidate_); }, [&](const ton_api::validatorSession_message_commit &obj) { if (obj.candidate_ == skip_round_candidate_id()) { @@ -1272,6 +1352,46 @@ void ValidatorSessionImpl::stats_process_action(td::uint32 node_id, ton_api::val [](const auto &) {})); } +void ValidatorSessionImpl::process_approve(td::uint32 node_id, td::uint32 round, + ValidatorSessionCandidateId candidate_id) { + auto stat = stats_get_candidate_stat_by_id(round, candidate_id); + if (!stat) { + stats_pending_approve_[{round, candidate_id}].push_back(node_id); + return; + } + + bool was_approved_66pct = stat->approved_66pct_at > 0.0; + stat->set_approved_by(node_id, description().get_node_weight(node_id), description().get_total_weight()); + bool is_approved_66pct = stat->approved_66pct_at > 0.0; + + if (allow_optimistic_generation_ && !was_approved_66pct && is_approved_66pct && cur_round_ == round && + description().get_node_priority(local_idx(), round + 1) == 0 && blocks_.contains(candidate_id)) { + auto &block = blocks_[candidate_id]; + if (cur_round_ == first_block_round_ && + description().get_node_priority(description().get_source_idx(PublicKeyHash{block->src_}), cur_round_) == 0) { + callback_->generate_block_optimistic(BlockSourceInfo{description().get_source_public_key(local_idx()), + BlockCandidatePriority{round + 1, round + 1, 0}}, + block->data_.clone(), block->root_hash_, stat->block_id.file_hash, + [=, SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + return; + } + td::actor::send_closure( + SelfId, &ValidatorSessionImpl::generated_optimistic_candidate, + round + 1, R.move_as_ok(), candidate_id); + }); + } + } +} + +void ValidatorSessionImpl::generated_optimistic_candidate(td::uint32 round, GeneratedCandidate candidate, + ValidatorSessionCandidateId prev_candidate) { + if (cur_round_ > round) { + return; + } + send_candidate_broadcast(round, candidate.candidate, prev_candidate); +} + td::actor::ActorOwn ValidatorSession::create( catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index 641fb4866c..5e226f3578 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -93,6 +93,9 @@ class ValidatorSession : public td::actor::Actor { ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash, td::Promise promise) = 0; + virtual void generate_block_optimistic(BlockSourceInfo source_info, td::BufferSlice prev_block, RootHash prev_root_hash, + FileHash prev_file_hash, td::Promise promise) { + } virtual ~Callback() = default; }; diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 04fdc875c5..ef0e3fba89 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -161,6 +161,7 @@ class ValidatorSessionImpl : public ValidatorSession { bool catchain_started_ = false; bool allow_unsafe_self_blocks_resync_; bool compress_block_candidates_ = false; + bool allow_optimistic_generation_ = false; ValidatorSessionStats cur_stats_; bool stats_inited_ = false; @@ -168,6 +169,30 @@ class ValidatorSessionImpl : public ValidatorSession { stats_pending_approve_; // round, candidate_id -> approvers std::map, std::vector> stats_pending_sign_; // round, candidate_id -> signers + + struct SentCandidateStats { + bool sent = false; + double sent_at = -1.0; + double serialize_time = -1.0; + size_t serialized_size = 0; + }; + std::map sent_candidates_; + + struct BroadcastInfo { + ValidatorSessionCandidateId candidate_id; + double received_at = -1.0; + double deserialize_time = -1.0; + size_t serialized_size = 0; + ValidatorSessionFileHash file_hash; + ValidatorSessionCollatedDataFileHash collated_data_hash; + }; + struct OptimisticBroadcast { + tl_object_ptr candidate; + ValidatorSessionCandidateId prev_candidate_id; + BroadcastInfo broadcast_info; + }; + std::map, OptimisticBroadcast> optimistic_broadcasts_; // round, src -> broadcast + void stats_init(); void stats_add_round(); ValidatorSessionStats::Producer *stats_get_candidate_stat( @@ -176,6 +201,10 @@ class ValidatorSessionImpl : public ValidatorSession { ValidatorSessionStats::Producer *stats_get_candidate_stat_by_id(td::uint32 round, ValidatorSessionCandidateId candidate_id); void stats_process_action(td::uint32 node_id, ton_api::validatorSession_round_Message &action); + void process_approve(td::uint32 node_id, td::uint32 round, ValidatorSessionCandidateId candidate_id); + + void generated_optimistic_candidate(td::uint32 round, GeneratedCandidate candidate, + ValidatorSessionCandidateId prev_candidate); public: ValidatorSessionImpl(catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, @@ -205,6 +234,9 @@ class ValidatorSessionImpl : public ValidatorSession { bool ensure_candidate_unique(td::uint32 src_idx, td::uint32 round, ValidatorSessionCandidateId block_id); void process_broadcast(PublicKeyHash src, td::BufferSlice data, td::optional expected_id, bool is_overlay_broadcast, bool is_startup); + void process_received_block(td::uint32 block_round, PublicKeyHash src, td::uint32 src_idx, + tl_object_ptr candidate, const BroadcastInfo &info, + bool is_overlay_broadcast, bool is_startup); void process_message(PublicKeyHash src, td::BufferSlice data); void process_query(PublicKeyHash src, td::BufferSlice data, td::Promise promise); @@ -218,6 +250,9 @@ class ValidatorSessionImpl : public ValidatorSession { td::BufferSlice signature); void generated_block(td::uint32 round, GeneratedCandidate c, double collation_time); + SentCandidateStats &send_candidate_broadcast( + td::uint32 round, const BlockCandidate &candidate, + td::optional optimistic_prev_candidate = {}); void signed_block(td::uint32 round, ValidatorSessionCandidateId hash, td::BufferSlice signature); void end_request(td::uint32 round, ValidatorSessionCandidateId block_id) { diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index 853ac37aea..9eadd3f0af 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -38,11 +38,15 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha td::Promise promise, int proto_version) { if (shard.is_masterchain()) { run_collate_query( - shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), - opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), promise.wrap([](BlockCandidate&& candidate) { + CollateParams{.shard = shard, + .min_masterchain_block_id = min_masterchain_block_id, + .prev = std::move(prev), + .creator = creator, + .validator_set = std::move(validator_set), + .collator_opts = opts_->get_collator_options()}, + manager_, td::Timestamp::in(10.0), std::move(cancellation_token), promise.wrap([](BlockCandidate&& candidate) { return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; - }), - adnl::AdnlNodeIdShort::zero(), std::move(cancellation_token), 0); + })); return; } collate_shard_block(shard, min_masterchain_block_id, std::move(prev), creator, priority, std::move(validator_set), @@ -50,6 +54,27 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha proto_version); } +void CollationManager::collate_next_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, + BlockIdExt prev_block_id, td::BufferSlice prev_block, + Ed25519_PublicKey creator, BlockCandidatePriority priority, + td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, + td::Promise promise, int proto_version) { + TRY_RESULT_PROMISE(promise, prev_block_data, create_block(prev_block_id, std::move(prev_block))); + run_collate_query( + CollateParams{.shard = shard, + .min_masterchain_block_id = min_masterchain_block_id, + .prev = {prev_block_id}, + .creator = creator, + .validator_set = std::move(validator_set), + .collator_opts = opts_->get_collator_options(), + .optimistic_prev_block_ = std::move(prev_block_data)}, + manager_, td::Timestamp::in(10.0), std::move(cancellation_token), promise.wrap([](BlockCandidate&& candidate) { + return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; + })); + // TODO: request to collator node +} + void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, @@ -109,11 +134,15 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas if (selected_collator.is_zero() && s->self_collate) { run_collate_query( - shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), - opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), promise.wrap([](BlockCandidate&& candidate) { + CollateParams{.shard = shard, + .min_masterchain_block_id = min_masterchain_block_id, + .prev = std::move(prev), + .creator = creator, + .validator_set = std::move(validator_set), + .collator_opts = opts_->get_collator_options()}, + manager_, td::Timestamp::in(10.0), std::move(cancellation_token), promise.wrap([](BlockCandidate&& candidate) { return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; - }), - adnl::AdnlNodeIdShort::zero(), std::move(cancellation_token), 0); + })); return; } diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp index 5b8d2b7a07..a64d432892 100644 --- a/validator/collation-manager.hpp +++ b/validator/collation-manager.hpp @@ -39,6 +39,12 @@ class CollationManager : public td::actor::Actor { td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise, int proto_version); + void collate_next_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, BlockIdExt prev_block_id, + td::BufferSlice prev_block, Ed25519_PublicKey creator, BlockCandidatePriority priority, + td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, td::Promise promise, + int proto_version); + void update_options(td::Ref opts); void validator_group_started(ShardIdFull shard); diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 82ca99cd23..ba3ff01ebd 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -539,18 +539,21 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std }; cache_entry->started = true; cache_entry->block_seqno = block_seqno; - run_collate_query( - shard, last_masterchain_state_->get_block_id(), std::move(prev_blocks), Ed25519_PublicKey{td::Bits256::zero()}, - last_masterchain_state_->get_validator_set(shard), opts_->get_collator_options(), manager_, timeout, - [=, SelfId = actor_id(this), timer = td::Timer{}](td::Result R) { - FLOG(INFO) { - prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); - sb << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); - }; - td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); - }, - local_id_, cache_entry->cancellation_token_source.get_cancellation_token(), - CollateMode::skip_store_candidate | CollateMode::from_collator_node); + run_collate_query(CollateParams{.shard = shard, + .min_masterchain_block_id = last_masterchain_state_->get_block_id(), + .prev = std::move(prev_blocks), + .validator_set = last_masterchain_state_->get_validator_set(shard), + .collator_opts = opts_->get_collator_options(), + .collator_node_id = local_id_, + .skip_store_candidate = true}, + manager_, timeout, cache_entry->cancellation_token_source.get_cancellation_token(), + [=, SelfId = actor_id(this), timer = td::Timer{}](td::Result R) { + FLOG(INFO) { + prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); + sb << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); + }; + td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); + }); } void CollatorNode::process_result(std::shared_ptr cache_entry, td::Result R) { diff --git a/validator/fabric.h b/validator/fabric.h index 19319f36e2..825c3b2482 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -26,8 +26,23 @@ namespace ton { namespace validator { +struct CollateParams { + ShardIdFull shard; + BlockIdExt min_masterchain_block_id; + std::vector prev; + bool is_hardfork = false; + Ed25519_PublicKey creator{td::Bits256::zero()}; + td::Ref validator_set = {}; + td::Ref collator_opts = {}; + adnl::AdnlNodeIdShort collator_node_id = adnl::AdnlNodeIdShort::zero(); + bool skip_store_candidate = false; + int attempt_idx = 0; + + // Optional - used for optimistic collation + Ref optimistic_prev_block_ = {}; +}; + enum ValidateMode { fake = 1 }; -enum CollateMode { skip_store_candidate = 1, from_collator_node = 2 }; td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_, td::Ref opts); @@ -85,15 +100,8 @@ void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, BlockCandidate candidate, td::Ref validator_set, PublicKeyHash local_validator_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, unsigned mode = 0); -void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - Ed25519_PublicKey creator, td::Ref validator_set, - td::Ref collator_opts, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise, - adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode, - int attempt_idx = 0); -void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise); +void run_collate_query(CollateParams params, td::actor::ActorId manager, td::Timestamp timeout, + td::CancellationToken cancellation_token, td::Promise promise); void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise); void run_fetch_account_state(WorkchainId wc, StdSmcAddress addr, td::actor::ActorId manager, diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 0080011573..f9e4244794 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -33,6 +33,7 @@ #include #include #include "common/global-version.h" +#include "fabric.h" namespace ton { @@ -80,7 +81,8 @@ class Collator final : public td::actor::Actor { td::Timestamp queue_cleanup_timeout_, soft_timeout_, medium_timeout_; td::Promise main_promise; adnl::AdnlNodeIdShort collator_node_id_ = adnl::AdnlNodeIdShort::zero(); - unsigned mode_ = 0; + bool skip_store_candidate_ = false; + Ref optimistic_prev_block_; int attempt_idx_; bool allow_repeat_collation_ = false; ton::BlockSeqno last_block_seqno{0}; @@ -95,11 +97,8 @@ class Collator final : public td::actor::Actor { static constexpr bool shard_splitting_enabled = true; public: - Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, - Ref validator_set, Ed25519_PublicKey collator_id, Ref collator_opts, - td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, - adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode, - int attempt_idx); + Collator(CollateParams params, td::actor::ActorId manager, td::Timestamp timeout, + td::CancellationToken cancellation_token, td::Promise promise); ~Collator() override = default; bool is_busy() const { return busy_; @@ -122,6 +121,8 @@ class Collator final : public td::actor::Actor { private: void start_up() override; + void load_prev_states_blocks(); + void process_optimistic_prev_block(); void alarm() override; int verbosity{3 * 0}; int verify{1}; @@ -276,6 +277,7 @@ class Collator final : public td::actor::Actor { void after_get_shard_blocks(td::Result>> res, td::PerfLogAction token); void after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res, td::PerfLogAction token); + void after_get_shard_state_optimistic(td::Result> res, td::PerfLogAction token); bool preprocess_prev_mc_state(); bool register_mc_state(Ref other_mc_state); bool request_aux_mc_state(BlockSeqno seqno, Ref& state); @@ -326,6 +328,14 @@ class Collator final : public td::actor::Actor { bool is_masterchain() const { return shard_.is_masterchain(); } + int prev_block_idx(const BlockIdExt& id) const { + for (size_t i = 0; i < prev_blocks.size(); ++i) { + if (prev_blocks[i] == id) { + return i; + } + } + return -1; + } bool is_our_address(Ref addr_ref) const; bool is_our_address(ton::AccountIdPrefixFull addr_prefix) const; bool is_our_address(const ton::StdSmcAddress& addr) const; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 5d0cbd2d6a..3ded80b6df 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -58,33 +58,21 @@ static constexpr int MAX_ATTEMPTS = 5; /** * Constructs a Collator object. * - * @param shard The shard of the new block. - * @param is_hardfork A boolean indicating whether the new block is a hardfork. - * @param min_masterchain_block_id The the minimum reference masterchain block. - * @param prev A vector of BlockIdExt representing the previous blocks. - * @param validator_set A reference to the ValidatorSet. - * @param collator_id The public key of the block creator. - * @param collator_opts A reference to CollatorOptions. + * @param params Collator parameters * @param manager The ActorId of the ValidatorManager. * @param timeout The timeout for the collator. - * @param promise The promise to return the result. - * @param collator_node_id ADNL id of the collator node that generates the block (zero if it's not a collator node) * @param cancellation_token Token to cancel collation. - * @param mode +1 - skip storing candidate to disk, +2 - called from CollatorNode. - * @param attempt_idx The index of the attempt, starting from 0. On later attempts collator decreases block limits and skips some steps. - */ -Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, - std::vector prev, td::Ref validator_set, Ed25519_PublicKey collator_id, - Ref collator_opts, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise, adnl::AdnlNodeIdShort collator_node_id, - td::CancellationToken cancellation_token, unsigned mode, int attempt_idx) - : shard_(shard) - , is_hardfork_(is_hardfork) - , min_mc_block_id{min_masterchain_block_id} - , prev_blocks(std::move(prev)) - , created_by_(collator_id) - , collator_opts_(collator_opts) - , validator_set_(std::move(validator_set)) + * @param promise The promise to return the result. + */ +Collator::Collator(CollateParams params, td::actor::ActorId manager, td::Timestamp timeout, + td::CancellationToken cancellation_token, td::Promise promise) + : shard_(params.shard) + , is_hardfork_(params.is_hardfork) + , min_mc_block_id{params.min_masterchain_block_id} + , prev_blocks(std::move(params.prev)) + , created_by_(params.creator) + , collator_opts_(params.collator_opts.is_null() ? td::Ref{true} : params.collator_opts) + , validator_set_(std::move(params.validator_set)) , manager(manager) , timeout(timeout) // default timeout is 10 seconds, declared in validator/validator-group.cpp:generate_block_candidate:run_collate_query @@ -92,9 +80,10 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha , soft_timeout_(td::Timestamp::at(timeout.at() - 3.0)) , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) , main_promise(std::move(promise)) - , collator_node_id_(collator_node_id) - , mode_(mode) - , attempt_idx_(attempt_idx) + , collator_node_id_(params.collator_node_id) + , skip_store_candidate_(params.skip_store_candidate) + , optimistic_prev_block_(std::move(params.optimistic_prev_block_)) + , attempt_idx_(params.attempt_idx) , perf_timer_("collate", 0.1, [manager](double duration) { send_closure(manager, &ValidatorManager::add_perf_timer_stat, "collate", duration); @@ -204,6 +193,16 @@ void Collator::start_up() { return; } } + if (optimistic_prev_block_.not_null()) { + if (prev_blocks.size() != 1) { + fatal_error(-666, "optimistic prev block is not null, which is not allowed after merge"); + return; + } + if (prev_blocks[0] != optimistic_prev_block_->block_id()) { + fatal_error(-666, "optimistic prev block is not null, but has invalid block id"); + return; + } + } busy_ = true; step = 1; if (!is_masterchain()) { @@ -239,32 +238,10 @@ void Collator::start_up() { // 3. load previous block(s) and corresponding state(s) prev_states.resize(prev_blocks.size()); prev_block_data.resize(prev_blocks.size()); - for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { - // 3.1. load state - LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; - ++pending; - auto token = perf_log_.start_action(PSTRING() << "wait_block_state #" << i); - td::actor::send_closure_later( - manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), timeout, - [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { - LOG(DEBUG) << "got answer to wait_block_state query #" << i; - td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state, i, std::move(res), - std::move(token)); - }); - if (prev_blocks[i].seqno()) { - // 3.2. load block - // NB: we need the block itself only for extracting start_lt and end_lt to create correct prev_blk:ExtBlkRef and related Merkle proofs - LOG(DEBUG) << "sending wait_block_data() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; - ++pending; - auto token = perf_log_.start_action(PSTRING() << "wait_block_data #" << i); - td::actor::send_closure_later( - manager, &ValidatorManager::wait_block_data_short, prev_blocks[i], priority(), timeout, - [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { - LOG(DEBUG) << "got answer to wait_block_data query #" << i; - td::actor::send_closure_later(std::move(self), &Collator::after_get_block_data, i, std::move(res), - std::move(token)); - }); - } + if (optimistic_prev_block_.is_null()) { + load_prev_states_blocks(); + } else { + process_optimistic_prev_block(); } if (is_hardfork_) { LOG(WARNING) << "generating a hardfork block"; @@ -312,6 +289,86 @@ void Collator::start_up() { CHECK(pending); } +/** + * Load previous states and blocks from DB + */ +void Collator::load_prev_states_blocks() { + for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { + // 3.1. load state + LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; + ++pending; + auto token = perf_log_.start_action(PSTRING() << "wait_block_state #" << i); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), timeout, + [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query #" << i; + td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state, i, std::move(res), + std::move(token)); + }); + if (prev_blocks[i].seqno()) { + // 3.2. load block + LOG(DEBUG) << "sending wait_block_data() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; + ++pending; + auto token = perf_log_.start_action(PSTRING() << "wait_block_data #" << i); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_data_short, prev_blocks[i], priority(), timeout, + [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_data query #" << i; + td::actor::send_closure_later(std::move(self), &Collator::after_get_block_data, i, std::move(res), + std::move(token)); + }); + } + } +} + +/** + * Write optimistic prev block as block data, load previous state to apply Merkle update to it + */ +void Collator::process_optimistic_prev_block() { + std::vector prev_prev; + BlockIdExt mc_blkid; + bool after_split; + auto S = block::unpack_block_prev_blk_try(optimistic_prev_block_->root_cell(), optimistic_prev_block_->block_id(), + prev_prev, mc_blkid, after_split); + if (S.is_error()) { + fatal_error(S.move_as_error_prefix("failed to unpack optimistic prev block: ")); + return; + } + // 3.1. load state + if (prev_prev.size() == 1) { + LOG(DEBUG) << "sending wait_block_state() query for " << prev_prev[0].to_str() << " to Manager (opt)"; + ++pending; + auto token = perf_log_.start_action("opt wait_block_state"); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, prev_prev[0], priority(), timeout, + [self = get_self(), token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query (opt)"; + td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state_optimistic, std::move(res), + std::move(token)); + }); + } else { + CHECK(prev_prev.size() == 2); + LOG(DEBUG) << "sending wait_block_state_merge() query for " << prev_prev[0].to_str() << " and " + << prev_prev[1].to_str() << " to Manager (opt)"; + ++pending; + auto token = perf_log_.start_action("opt wait_block_state_merge"); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_merge, prev_prev[0], prev_prev[1], priority(), timeout, + [self = get_self(), token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state_merge query (opt)"; + td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state_optimistic, std::move(res), + std::move(token)); + }); + } + // 3.2. load block + LOG(DEBUG) << "use optimistic prev block " << prev_blocks[0].to_str(); + ++pending; + auto token = perf_log_.start_action(PSTRING() << "opt wait_block_data"); + td::actor::send_closure_later(actor_id(this), &Collator::after_get_block_data, 0, optimistic_prev_block_, + std::move(token)); +} + + /** * Raises an error when timeout is reached. */ @@ -377,9 +434,18 @@ bool Collator::fatal_error(td::Status error) { if (allow_repeat_collation_ && error.code() != ErrorCode::cancelled && attempt_idx_ + 1 < MAX_ATTEMPTS && !is_hardfork_ && !timeout.is_in_past()) { LOG(WARNING) << "Repeating collation (attempt #" << attempt_idx_ + 1 << ")"; - run_collate_query(shard_, min_mc_block_id, prev_blocks, created_by_, validator_set_, collator_opts_, manager, - td::Timestamp::in(10.0), std::move(main_promise), collator_node_id_, - std::move(cancellation_token_), mode_, attempt_idx_ + 1); + run_collate_query(CollateParams{.shard = shard_, + .min_masterchain_block_id = min_mc_block_id, + .prev = prev_blocks, + .is_hardfork = false, + .creator = created_by_, + .validator_set = validator_set_, + .collator_opts = collator_opts_, + .collator_node_id = collator_node_id_, + .skip_store_candidate = skip_store_candidate_, + .attempt_idx = attempt_idx_ + 1, + .optimistic_prev_block_ = optimistic_prev_block_}, + manager, td::Timestamp::in(10.0), std::move(cancellation_token_), std::move(main_promise)); } else { LOG(INFO) << "collation failed in " << perf_timer_.elapsed() << " s " << error; LOG(INFO) << perf_log_; @@ -678,7 +744,7 @@ void Collator::after_get_shard_state(int idx, td::Result> res, t * Callback function called after retrieving block data for a previous block. * * @param idx The index of the previous block (0 or 1). - * @param res The retreved block data. + * @param res The retrieved block data. */ void Collator::after_get_block_data(int idx, td::Result> res, td::PerfLogAction token) { LOG(DEBUG) << "in Collator::after_get_block_data(" << idx << ")"; @@ -757,6 +823,31 @@ void Collator::after_get_storage_stat_cache(td::Result> res, td::PerfLogAction token) { + LOG(DEBUG) << "in Collator::after_get_shard_state_optimistic()"; + token.finish(res); + if (res.is_error()) { + fatal_error(res.move_as_error()); + return; + } + td::RealCpuTimer timer; + work_timer_.resume(); + auto state = res.move_as_ok(); + auto S = state.write().apply_block(optimistic_prev_block_->block_id(), optimistic_prev_block_); + if (S.is_error()) { + fatal_error(S.move_as_error_prefix("apply error: ")); + return; + } + work_timer_.pause(); + stats_.work_time.optimistic_apply = timer.elapsed_both(); + after_get_shard_state(0, std::move(state), {}); +} + /** * Unpacks the last masterchain state and initializes the Collator object with the extracted configuration. * @@ -896,7 +987,9 @@ bool Collator::request_neighbor_msg_queues() { unsigned i = 0; for (block::McShardDescr& descr : neighbors_) { LOG(DEBUG) << "neighbor #" << i << " : " << descr.blk_.to_str(); - top_blocks.push_back(descr.blk_); + if (prev_block_idx(descr.blk_) == -1) { + top_blocks.push_back(descr.blk_); + } ++i; } ++pending; @@ -935,8 +1028,7 @@ bool Collator::request_out_msg_queue_size() { /** * Handles the result of obtaining the outbound queue for a neighbor. * - * @param i The index of the neighbor. - * @param res The obtained outbound queue. + * @param R The result of retrieving neighbor message queues (top block id -> queue). */ void Collator::got_neighbor_msg_queues(td::Result>> R, td::PerfLogAction token) { @@ -952,12 +1044,17 @@ void Collator::got_neighbor_msg_queues(td::Result= 0) { + got_neighbor_msg_queue( + i, Ref{true, descr.blk_, prev_states[prev_idx]->root_cell(), td::Ref{}, true}); + } else { + auto it = res.find(descr.blk_); + if (it == res.end()) { + fatal_error(PSTRING() << "no msg queue from neighbor #" << i); + return; + } + got_neighbor_msg_queue(i, it->second); } - got_neighbor_msg_queue(i, it->second); ++i; } check_pending(); @@ -6393,7 +6490,7 @@ bool Collator::create_block_candidate() { << consensus_config.max_collated_data_size << ")"); } // 4. save block candidate - if (mode_ & CollateMode::skip_store_candidate) { + if (skip_store_candidate_) { td::actor::send_closure_later(actor_id(this), &Collator::return_block_candidate, td::Unit()); } else { LOG(INFO) << "saving new BlockCandidate"; @@ -6579,7 +6676,7 @@ void Collator::finalize_stats() { stats_.cc_seqno = validator_set_.not_null() ? validator_set_->get_catchain_seqno() : 0; stats_.collated_at = td::Clocks::system(); stats_.attempt = attempt_idx_; - stats_.is_validator = !(mode_ & CollateMode::from_collator_node); + stats_.is_validator = collator_node_id_.is_zero(); stats_.self = stats_.is_validator ? PublicKey(pubkeys::Ed25519(created_by_)).compute_short_id() : collator_node_id_.pubkey_hash(); if (block_limit_status_) { diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 2f6ddf1ebf..f38f773204 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -212,39 +212,18 @@ void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, .release(); } -void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - Ed25519_PublicKey creator, td::Ref validator_set, - td::Ref collator_opts, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise, - adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode, - int attempt_idx) { +void run_collate_query(CollateParams params, td::actor::ActorId manager, td::Timestamp timeout, + td::CancellationToken cancellation_token, td::Promise promise) { BlockSeqno seqno = 0; - for (auto& p : prev) { - if (p.seqno() > seqno) { - seqno = p.seqno(); - } - } - td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1) - << (attempt_idx ? "_" + td::to_string(attempt_idx) : ""), - shard, false, min_masterchain_block_id, std::move(prev), std::move(validator_set), - creator, std::move(collator_opts), std::move(manager), timeout, std::move(promise), - collator_node_id, std::move(cancellation_token), mode, attempt_idx) - .release(); -} - -void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise) { - BlockSeqno seqno = 0; - for (auto& p : prev) { + for (auto& p : params.prev) { if (p.seqno() > seqno) { seqno = p.seqno(); } } - td::actor::create_actor( - PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, min_masterchain_block_id, - std::move(prev), td::Ref{}, Ed25519_PublicKey{Bits256::zero()}, td::Ref{true}, - std::move(manager), timeout, std::move(promise), adnl::AdnlNodeIdShort::zero(), td::CancellationToken{}, 0, 0) + td::actor::create_actor(PSTRING() << "collate" << params.shard.to_str() << ":" << (seqno + 1) + << (params.attempt_idx ? "_" + td::to_string(params.attempt_idx) : ""), + std::move(params), std::move(manager), timeout, std::move(cancellation_token), + std::move(promise)) .release(); } diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 78248610ac..90e11a35ba 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -117,6 +117,7 @@ struct CollationStats { struct WorkTimeStats { td::RealCpuTimer::Time total; + td::RealCpuTimer::Time optimistic_apply; td::RealCpuTimer::Time queue_cleanup; td::RealCpuTimer::Time prelim_storage_stat; td::RealCpuTimer::Time trx_tvm; @@ -129,9 +130,9 @@ struct CollationStats { tl_object_ptr tl(bool is_cpu) const { return create_tl_object( - total.get(is_cpu), queue_cleanup.get(is_cpu), prelim_storage_stat.get(is_cpu), trx_tvm.get(is_cpu), - trx_storage_stat.get(is_cpu), trx_other.get(is_cpu), final_storage_stat.get(is_cpu), create_block.get(is_cpu), - create_collated_data.get(is_cpu), create_block_candidate.get(is_cpu)); + total.get(is_cpu), optimistic_apply.get(is_cpu), queue_cleanup.get(is_cpu), prelim_storage_stat.get(is_cpu), + trx_tvm.get(is_cpu), trx_storage_stat.get(is_cpu), trx_other.get(is_cpu), final_storage_stat.get(is_cpu), + create_block.get(is_cpu), create_collated_data.get(is_cpu), create_block_candidate.get(is_cpu)); } }; WorkTimeStats work_time; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 15271aedbc..a4124904ea 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -128,9 +128,12 @@ void ValidatorManagerImpl::sync_complete(td::Promise promise) { } Ed25519_PublicKey created_by{td::Bits256::zero()}; td::as(created_by.as_bits256().data() + 32 - 4) = ((unsigned)std::time(nullptr) >> 8); - run_collate_query(shard_id, last_masterchain_block_id_, prev, created_by, val_set, td::Ref{true}, - actor_id(this), td::Timestamp::in(10.0), std::move(P), adnl::AdnlNodeIdShort::zero(), - td::CancellationToken{}, 0); + run_collate_query(CollateParams{.shard = shard_id, + .min_masterchain_block_id = last_masterchain_block_id_, + .prev = prev, + .creator = created_by, + .validator_set = val_set}, + actor_id(this), td::Timestamp::in(10.0), {}, std::move(P)); } void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector prev, BlockIdExt last, diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index 3798803bbf..de0390ece0 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -55,7 +55,9 @@ void ValidatorManagerImpl::sync_complete(td::Promise promise) { }); LOG(ERROR) << "running collate query"; - run_collate_hardfork(shard_id, block_id, prev, actor_id(this), td::Timestamp::in(10.0), std::move(P)); + run_collate_query( + CollateParams{.shard = shard_id, .min_masterchain_block_id = block_id, .prev = prev, .is_hardfork = true}, + actor_id(this), td::Timestamp::in(10.0), {}, std::move(P)); } void ValidatorManagerImpl::created_candidate(BlockCandidate candidate) { diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index d2db0ae992..6a6148a1b8 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -23,7 +23,6 @@ #include "td/utils/overloaded.h" #include "common/delay.h" #include "ton/lite-tl.hpp" -#include "ton/ton-tl.hpp" #include "td/utils/Random.h" #include "collator-node.hpp" @@ -43,9 +42,7 @@ void ValidatorGroup::generate_block_candidate(validatorsession::BlockSourceInfo return; } td::uint32 round_id = source_info.priority.round; - if (round_id > last_known_round_id_) { - last_known_round_id_ = round_id; - } + update_round_id(round_id); if (!started_) { promise.set_error(td::Status::Error(ErrorCode::notready, "cannot collate block: group not started")); return; @@ -70,11 +67,38 @@ void ValidatorGroup::generate_block_candidate(validatorsession::BlockSourceInfo td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, source_info, std::move(cache), std::move(R)); }; + + if (optimistic_generation_ && prev_block_ids_.size() == 1 && optimistic_generation_->prev == prev_block_ids_[0] && + optimistic_generation_->round == round_id) { + if (optimistic_generation_->result) { + P.set_value(optimistic_generation_->result.value().clone()); + } else { + optimistic_generation_->promises.push_back( + [=, SelfId = actor_id(this), P = std::move(P), + cancellation_token = + cancellation_token_source_.get_cancellation_token()](td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ValidatorGroup::generate_block_candidate_cont, source_info, std::move(P), + std::move(cancellation_token)); + } else { + P.set_value(R.move_as_ok()); + } + }); + } + return; + } + generate_block_candidate_cont(source_info, std::move(P), cancellation_token_source_.get_cancellation_token()); +} + +void ValidatorGroup::generate_block_candidate_cont(validatorsession::BlockSourceInfo source_info, + td::Promise promise, + td::CancellationToken cancellation_token) { + TRY_STATUS_PROMISE(promise, cancellation_token.check()); td::uint64 max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; td::actor::send_closure(collation_manager_, &CollationManager::collate_block, shard_, min_masterchain_block_id_, prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, - source_info.priority, validator_set_, max_answer_size, - cancellation_token_source_.get_cancellation_token(), std::move(P), config_.proto_version); + source_info.priority, validator_set_, max_answer_size, std::move(cancellation_token), + std::move(promise), config_.proto_version); } void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo source_info, @@ -108,9 +132,7 @@ void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo return; } td::uint32 round_id = source_info.priority.round; - if (round_id > last_known_round_id_) { - last_known_round_id_ = round_id; - } + update_round_id(round_id); if (round_id < last_known_round_id_) { promise.set_error(td::Status::Error(ErrorCode::notready, "too old")); return; @@ -179,9 +201,7 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so td::Promise promise) { stats.cc_seqno = validator_set_->get_catchain_seqno(); td::uint32 round_id = source_info.priority.round; - if (round_id >= last_known_round_id_) { - last_known_round_id_ = round_id + 1; - } + update_round_id(round_id + 1); auto sig_set = create_signature_set(std::move(signatures)); validator_set_->check_signatures(root_hash, file_hash, sig_set).ensure(); auto approve_sig_set = create_signature_set(std::move(approve_signatures)); @@ -236,6 +256,10 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so cached_collated_block_ = nullptr; approved_candidates_cache_.clear(); cancellation_token_source_.cancel(); + if (optimistic_generation_ && optimistic_generation_->round == last_known_round_id_ && + optimistic_generation_->prev != next_block_id) { + optimistic_generation_ = {}; + } } void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, @@ -262,9 +286,7 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref } void ValidatorGroup::skip_round(td::uint32 round_id) { - if (round_id >= last_known_round_id_) { - last_known_round_id_ = round_id + 1; - } + update_round_id(round_id + 1); } void ValidatorGroup::get_approved_candidate(PublicKey source, RootHash root_hash, FileHash file_hash, @@ -275,6 +297,66 @@ void ValidatorGroup::get_approved_candidate(PublicKey source, RootHash root_hash std::move(promise)); } +void ValidatorGroup::generate_block_optimistic(validatorsession::BlockSourceInfo source_info, + td::BufferSlice prev_block, RootHash prev_root_hash, + FileHash prev_file_hash, td::Promise promise) { + if (destroying_) { + return; + } + if (last_known_round_id_ + 1 != source_info.priority.round) { + return; + } + if (optimistic_generation_ && optimistic_generation_->round >= source_info.priority.round) { + return; + } + BlockIdExt block_id{create_next_block_id_simple(), prev_root_hash, prev_file_hash}; + optimistic_generation_ = std::make_unique(); + optimistic_generation_->round = source_info.priority.round; + optimistic_generation_->prev = BlockIdExt{create_next_block_id_simple(), prev_root_hash, prev_file_hash}; + optimistic_generation_->promises.push_back(std::move(promise)); + + td::Promise P = [=, SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_optimistic, source_info, std::move(R)); + }; + LOG(WARNING) << "Optimistically generating next block after " << block_id.to_str(); + td::uint64 max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; + td::actor::send_closure(collation_manager_, &CollationManager::collate_next_block, shard_, min_masterchain_block_id_, + block_id, std::move(prev_block), Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, + source_info.priority, validator_set_, max_answer_size, + optimistic_generation_->cancellation_token_source.get_cancellation_token(), std::move(P), + config_.proto_version); +} + +void ValidatorGroup::generated_block_optimistic(validatorsession::BlockSourceInfo source_info, + td::Result R) { + if (!optimistic_generation_ || optimistic_generation_->round != source_info.priority.round) { + return; + } + if (R.is_error()) { + LOG(WARNING) << "Optimistic generation failed: " << R.move_as_error(); + for (auto &promise : optimistic_generation_->promises) { + promise.set_error(R.error().clone()); + } + optimistic_generation_ = {}; + return; + } + optimistic_generation_->result = R.move_as_ok(); + for (auto &promise : optimistic_generation_->promises) { + promise.set_result(optimistic_generation_->result.value().clone()); + } + optimistic_generation_->promises.clear(); +} + +void ValidatorGroup::update_round_id(td::uint32 round) { + if (last_known_round_id_ >= round) { + return; + } + last_known_round_id_ = round; + if (optimistic_generation_ && optimistic_generation_->round < round) { + optimistic_generation_ = {}; + } +} + BlockIdExt ValidatorGroup::create_next_block_id(RootHash root_hash, FileHash file_hash) const { return BlockIdExt{create_next_block_id_simple(), root_hash, file_hash}; } @@ -352,6 +434,12 @@ std::unique_ptr ValidatorGroup::ma td::actor::send_closure(id_, &ValidatorGroup::get_approved_candidate, source, root_hash, file_hash, collated_data_file_hash, std::move(promise)); } + void generate_block_optimistic(validatorsession::BlockSourceInfo source_info, td::BufferSlice prev_block, + RootHash prev_root_hash, FileHash prev_file_hash, + td::Promise promise) override { + td::actor::send_closure(id_, &ValidatorGroup::generate_block_optimistic, source_info, std::move(prev_block), + prev_root_hash, prev_file_hash, std::move(promise)); + } private: td::actor::ActorId id_; @@ -534,7 +622,7 @@ void ValidatorGroup::get_validator_group_info_for_litequery_cont( BlockIdExt id{next_block_id, candidate->id_->block_id_->root_hash_, candidate->id_->block_id_->file_hash_}; candidate->id_->block_id_ = create_tl_lite_block_id(id); candidate->available_ = - available_block_candidates_.count({candidate->id_->creator_, id, candidate->id_->collated_data_hash_}); + available_block_candidates_.contains({candidate->id_->creator_, id, candidate->id_->collated_data_hash_}); } auto result = create_tl_object(); diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index a7456cf9f0..17bd23a4df 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -37,6 +37,8 @@ class ValidatorManager; class ValidatorGroup : public td::actor::Actor { public: void generate_block_candidate(validatorsession::BlockSourceInfo source_info, td::Promise promise); + void generate_block_candidate_cont(validatorsession::BlockSourceInfo source_info, + td::Promise promise, td::CancellationToken cancellation_token); void validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block, td::Promise> promise); void accept_block_candidate(validatorsession::BlockSourceInfo source_info, td::BufferSlice block, RootHash root_hash, @@ -52,6 +54,10 @@ class ValidatorGroup : public td::actor::Actor { BlockIdExt create_next_block_id(RootHash root_hash, FileHash file_hash) const; BlockId create_next_block_id_simple() const; + void generate_block_optimistic(validatorsession::BlockSourceInfo source_info, td::BufferSlice prev_block, + RootHash prev_root_hash, FileHash prev_file_hash, td::Promise promise); + void generated_block_optimistic(validatorsession::BlockSourceInfo source_info, td::Result R); + void start(std::vector prev, BlockIdExt min_masterchain_block_id); void create_session(); void destroy(); @@ -156,6 +162,8 @@ class ValidatorGroup : public td::actor::Actor { std::shared_ptr cached_collated_block_; td::CancellationTokenSource cancellation_token_source_; + void update_round_id(td::uint32 round); + void generated_block_candidate(validatorsession::BlockSourceInfo source_info, std::shared_ptr cache, td::Result R); @@ -181,6 +189,21 @@ class ValidatorGroup : public td::actor::Actor { std::set sent_candidate_broadcasts_; void send_block_candidate_broadcast(BlockIdExt id, td::BufferSlice data); + + struct OptimisticGeneration { + td::uint32 round = 0; + BlockIdExt prev; + td::optional result; + td::CancellationTokenSource cancellation_token_source; + std::vector> promises; + + ~OptimisticGeneration() { + for (auto& promise : promises) { + promise.set_error(td::Status::Error(ErrorCode::cancelled, "Cancelled")); + } + } + }; + std::unique_ptr optimistic_generation_; }; } // namespace validator From dd130963520fb5e27691916b903d2dcf82fad29e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 4 Aug 2025 18:46:31 +0300 Subject: [PATCH 366/524] Don't wait for storing state to celldb in most cases --- validator/apply-block.cpp | 2 +- .../downloaders/wait-block-state-merge.cpp | 4 +- validator/downloaders/wait-block-state.cpp | 47 +++++--- validator/downloaders/wait-block-state.hpp | 9 +- validator/impl/accept-block.cpp | 2 +- validator/impl/check-proof.cpp | 2 +- validator/impl/collator.cpp | 6 +- validator/impl/out-msg-queue-proof.cpp | 2 +- validator/impl/validate-query.cpp | 6 +- validator/manager-disk.cpp | 17 +-- validator/manager-disk.hpp | 4 +- validator/manager-hardfork.cpp | 17 +-- validator/manager-hardfork.hpp | 4 +- validator/manager-init.cpp | 2 +- validator/manager.cpp | 104 +++++++++++------- validator/manager.hpp | 44 +++++--- validator/queue-size-counter.cpp | 6 +- validator/shard-block-retainer.cpp | 2 +- validator/shard-client.cpp | 6 +- validator/validator.h | 4 +- 20 files changed, 174 insertions(+), 116 deletions(-) diff --git a/validator/apply-block.cpp b/validator/apply-block.cpp index a35b74ce8a..3c6a931cd7 100644 --- a/validator/apply-block.cpp +++ b/validator/apply-block.cpp @@ -166,7 +166,7 @@ void ApplyBlock::written_block_data() { }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state, handle_, apply_block_priority(), timeout_, - std::move(P)); + true, std::move(P)); } } diff --git a/validator/downloaders/wait-block-state-merge.cpp b/validator/downloaders/wait-block-state-merge.cpp index 2b19616109..c5f1cfa7f8 100644 --- a/validator/downloaders/wait-block-state-merge.cpp +++ b/validator/downloaders/wait-block-state-merge.cpp @@ -55,7 +55,7 @@ void WaitBlockStateMerge::start_up() { } }); - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, left_, priority_, timeout_, + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, left_, priority_, timeout_, false, std::move(P_l)); auto P_r = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { @@ -66,7 +66,7 @@ void WaitBlockStateMerge::start_up() { } }); - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, right_, priority_, timeout_, + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, right_, priority_, timeout_, false, std::move(P_r)); } diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index 92a0720bc7..7ca2bbd2eb 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -32,7 +32,11 @@ void WaitBlockState::alarm() { } void WaitBlockState::abort_query(td::Status reason) { - if (promise_) { + if (promise_no_store_) { + promise_no_store_.set_error( + reason.move_as_error_prefix(PSTRING() << "failed to download state " << handle_->id() << ": ")); + } + if (promise_final_) { if (priority_ > 0 || (reason.code() != ErrorCode::timeout && reason.code() != ErrorCode::notready)) { LOG(WARNING) << "aborting wait block state query for " << handle_->id() << " priority=" << priority_ << ": " << reason; @@ -40,18 +44,19 @@ void WaitBlockState::abort_query(td::Status reason) { LOG(DEBUG) << "aborting wait block state query for " << handle_->id() << " priority=" << priority_ << ": " << reason; } - promise_.set_error(reason.move_as_error_prefix(PSTRING() << "failed to download state " << handle_->id() << ": ")); + promise_final_.set_error( + reason.move_as_error_prefix(PSTRING() << "failed to download state " << handle_->id() << ": ")); } stop(); } void WaitBlockState::finish_query() { CHECK(handle_->received_state()); - /*if (handle_->id().is_masterchain() && handle_->inited_proof()) { - td::actor::send_closure(manager_, &ValidatorManager::new_block, handle_, prev_state_, [](td::Unit) {}); - }*/ - if (promise_) { - promise_.set_result(prev_state_); + if (promise_no_store_) { + promise_no_store_.set_result(prev_state_); + } + if (promise_final_) { + promise_final_.set_result(prev_state_); } stop(); } @@ -275,10 +280,16 @@ void WaitBlockState::got_block_data(td::Ref data) { } void WaitBlockState::apply() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &WaitBlockState::abort_query, R.move_as_error_prefix("db set error: ")); + } else { + td::actor::send_closure(SelfId, &WaitBlockState::written_state, R.move_as_ok()); + } + }); + if (opts_->get_permanent_celldb()) { - td::actor::send_closure(manager_, &ValidatorManager::set_block_state_from_data, handle_, block_, - std::move(promise_)); - stop(); + td::actor::send_closure(manager_, &ValidatorManager::set_block_state_from_data, handle_, block_, std::move(P)); return; } TD_PERF_COUNTER(apply_block_to_state); @@ -289,15 +300,11 @@ void WaitBlockState::apply() { return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &WaitBlockState::abort_query, R.move_as_error_prefix("db set error: ")); - } else { - td::actor::send_closure(SelfId, &WaitBlockState::written_state, R.move_as_ok()); - } - }); - td::actor::send_closure(manager_, &ValidatorManager::set_block_state, handle_, prev_state_, std::move(P)); + if (promise_no_store_) { + promise_no_store_.set_result(prev_state_); + promise_no_store_ = {}; + } } void WaitBlockState::written_state(td::Ref upd_state) { @@ -317,6 +324,10 @@ void WaitBlockState::got_state_from_db(td::Ref state) { }); td::actor::send_closure(manager_, &ValidatorManager::set_block_state, handle_, prev_state_, std::move(P)); + if (promise_no_store_) { + promise_no_store_.set_result(prev_state_); + promise_no_store_ = {}; + } } else { finish_query(); } diff --git a/validator/downloaders/wait-block-state.hpp b/validator/downloaders/wait-block-state.hpp index a9317381c6..cf556e33d4 100644 --- a/validator/downloaders/wait-block-state.hpp +++ b/validator/downloaders/wait-block-state.hpp @@ -28,7 +28,8 @@ class WaitBlockState : public td::actor::Actor { public: WaitBlockState(BlockHandle handle, td::uint32 priority, td::Ref opts, td::Ref last_masterchain_state, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise> promise, + td::Timestamp timeout, td::Promise> promise_no_store, + td::Promise> promise_final, td::Ref persistent_state_desc = {}) : handle_(std::move(handle)) , priority_(priority) @@ -36,7 +37,8 @@ class WaitBlockState : public td::actor::Actor { , last_masterchain_state_(last_masterchain_state) , manager_(manager) , timeout_(timeout) - , promise_(std::move(promise)) + , promise_no_store_(std::move(promise_no_store)) + , promise_final_(std::move(promise_final)) , persistent_state_desc_(std::move(persistent_state_desc)) , perf_timer_("waitstate", 1.0, [manager](double duration) { send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitstate", duration); @@ -96,7 +98,8 @@ class WaitBlockState : public td::actor::Actor { td::Ref last_masterchain_state_; td::actor::ActorId manager_; td::Timestamp timeout_; - td::Promise> promise_; + td::Promise> promise_no_store_; + td::Promise> promise_final_; td::Ref persistent_state_desc_; td::Ref prev_state_; diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index b9aea1e7f0..8839940895 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -626,7 +626,7 @@ void AcceptBlockQuery::got_last_mc_block(std::pair, Bl VLOG(VALIDATOR_DEBUG) << "shardchain block refers to newer masterchain block " << mc_blkid_.to_str() << ", trying to obtain it"; td::actor::send_closure_later(manager_, &ValidatorManager::wait_block_state_short, mc_blkid_, priority(), timeout_, - [SelfId = actor_id(this)](td::Result> R) { + false, [SelfId = actor_id(this)](td::Result> R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::got_mc_state, R.move_as_ok()); diff --git a/validator/impl/check-proof.cpp b/validator/impl/check-proof.cpp index 30a13c0891..12d336ad18 100644 --- a/validator/impl/check-proof.cpp +++ b/validator/impl/check-proof.cpp @@ -344,7 +344,7 @@ void CheckProof::got_block_handle(BlockHandle handle) { process_masterchain_state(); return; } - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, prev_[0], priority(), timeout_, + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, prev_[0], priority(), timeout_, false, [SelfId = actor_id(this)](td::Result> R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &CheckProof::got_masterchain_state, diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 3ded80b6df..c2ca14f999 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -299,7 +299,7 @@ void Collator::load_prev_states_blocks() { ++pending; auto token = perf_log_.start_action(PSTRING() << "wait_block_state #" << i); td::actor::send_closure_later( - manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), timeout, + manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), timeout, false, [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { LOG(DEBUG) << "got answer to wait_block_state query #" << i; td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state, i, std::move(res), @@ -340,7 +340,7 @@ void Collator::process_optimistic_prev_block() { ++pending; auto token = perf_log_.start_action("opt wait_block_state"); td::actor::send_closure_later( - manager, &ValidatorManager::wait_block_state_short, prev_prev[0], priority(), timeout, + manager, &ValidatorManager::wait_block_state_short, prev_prev[0], priority(), timeout, false, [self = get_self(), token = std::move(token)](td::Result> res) mutable { LOG(DEBUG) << "got answer to wait_block_state query (opt)"; td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state_optimistic, std::move(res), @@ -574,7 +574,7 @@ bool Collator::request_aux_mc_state(BlockSeqno seqno, Ref& st ++pending; auto token = perf_log_.start_action(PSTRING() << "auxiliary wait_block_state " << blkid.to_str()); td::actor::send_closure_later( - manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout, + manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout, false, [self = get_self(), blkid, token = std::move(token)](td::Result> res) mutable { LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str(); td::actor::send_closure_later(std::move(self), &Collator::after_get_aux_shard_state, blkid, std::move(res), diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index e3330ea42b..dc56997a15 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -334,7 +334,7 @@ void OutMsgQueueImporter::get_proof_local(std::shared_ptr entry, Blo return; } td::actor::send_closure( - manager_, &ValidatorManager::wait_block_state_short, block, 0, entry->timeout, + manager_, &ValidatorManager::wait_block_state_short, block, 0, entry->timeout, false, [=, SelfId = actor_id(this), manager = manager_, timeout = entry->timeout, retry_after = td::Timestamp::in(0.1)](td::Result> R) mutable { if (R.is_error()) { diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index fbaa06aaca..01ded4dfa3 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -372,7 +372,7 @@ void ValidateQuery::start_up() { LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; ++pending; td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), - timeout, [self = get_self(), i](td::Result> res) -> void { + timeout, false, [self = get_self(), i](td::Result> res) -> void { LOG(DEBUG) << "got answer to wait_block_state_short query #" << i; td::actor::send_closure_later( std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res)); @@ -784,7 +784,7 @@ void ValidateQuery::got_mc_handle(td::Result res) { } auto mc_handle = res.move_as_ok(); td::actor::send_closure_later( - manager, &ValidatorManager::wait_block_state, mc_handle, priority(), timeout, + manager, &ValidatorManager::wait_block_state, mc_handle, priority(), timeout, false, [self = get_self(), id = id_, mc_handle](td::Result> res) { LOG(DEBUG) << "got answer to wait_block_state() query for masterchain block"; if (res.is_ok() && mc_handle->id().seqno() > 0 && !mc_handle->inited_proof()) { @@ -1764,7 +1764,7 @@ bool ValidateQuery::request_aux_mc_state(BlockSeqno seqno, Ref> res) { LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str(); td::actor::send_closure_later(std::move(self), diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index a4124904ea..be57a632d3 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -321,14 +321,15 @@ void ValidatorManagerImpl::dec_pending_new_blocks() { } void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto it = wait_state_.find(handle->id()); if (it == wait_state_.end()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle->id(), std::move(R)); }); auto id = td::actor::create_actor("waitstate", handle, 0, opts_, last_masterchain_state_, - actor_id(this), td::Timestamp::in(10.0), std::move(P)) + actor_id(this), td::Timestamp::in(10.0), + td::Promise>{}, std::move(P)) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); @@ -340,14 +341,14 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior } void ValidatorManagerImpl::wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), timeout, promise = std::move(promise)](td::Result R) mutable { + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); return; } - td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), 0, timeout, + td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), 0, timeout, wait_store, std::move(promise)); }); get_block_handle(block_id, true, std::move(P)); @@ -401,7 +402,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 auto shard = handle->id().shard_full(); auto prev_shard = handle->one_prev(true).shard_full(); if (shard == prev_shard) { - wait_block_state_short(handle->one_prev(true), 0, timeout, std::move(promise)); + wait_block_state_short(handle->one_prev(true), 0, timeout, true, std::move(promise)); } else { CHECK(shard_parent(shard) == prev_shard); bool left = shard_child(prev_shard, true) == shard; @@ -420,7 +421,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 } } }); - wait_block_state_short(handle->one_prev(true), 0, timeout, std::move(P)); + wait_block_state_short(handle->one_prev(true), 0, timeout, true, std::move(P)); } } else { wait_block_state_merge(handle->one_prev(true), handle->one_prev(false), 0, timeout, std::move(promise)); @@ -495,7 +496,7 @@ void ValidatorManagerImpl::wait_block_message_queue(BlockHandle handle, td::uint } }); - wait_block_state(handle, 0, timeout, std::move(P)); + wait_block_state(handle, 0, timeout, true, std::move(P)); } void ValidatorManagerImpl::wait_block_message_queue_short(BlockIdExt block_id, td::uint32 priority, diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 6c8aafd727..bdde74b2d9 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -163,9 +163,9 @@ class ValidatorManagerImpl : public ValidatorManager { std::function write_data, td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; - void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; - void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, td::Promise>> promise) override { diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index de0390ece0..717ab3aa55 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -167,14 +167,15 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto it = wait_state_.find(handle->id()); if (it == wait_state_.end()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle->id(), std::move(R)); }); auto id = td::actor::create_actor("waitstate", handle, 0, opts_, last_masterchain_state_, - actor_id(this), td::Timestamp::in(10.0), std::move(P)) + actor_id(this), td::Timestamp::in(10.0), + td::Promise>{}, std::move(P)) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); @@ -186,14 +187,14 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior } void ValidatorManagerImpl::wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), timeout, promise = std::move(promise)](td::Result R) mutable { + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); return; } - td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), 0, timeout, + td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), 0, timeout, wait_store, std::move(promise)); }); get_block_handle(block_id, true, std::move(P)); @@ -247,7 +248,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 auto shard = handle->id().shard_full(); auto prev_shard = handle->one_prev(true).shard_full(); if (shard == prev_shard) { - wait_block_state_short(handle->one_prev(true), 0, timeout, std::move(promise)); + wait_block_state_short(handle->one_prev(true), 0, timeout, false, std::move(promise)); } else { CHECK(shard_parent(shard) == prev_shard); bool left = shard_child(prev_shard, true) == shard; @@ -266,7 +267,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 } } }); - wait_block_state_short(handle->one_prev(true), 0, timeout, std::move(P)); + wait_block_state_short(handle->one_prev(true), 0, timeout, false, std::move(P)); } } else { wait_block_state_merge(handle->one_prev(true), handle->one_prev(false), 0, timeout, std::move(promise)); @@ -341,7 +342,7 @@ void ValidatorManagerImpl::wait_block_message_queue(BlockHandle handle, td::uint } }); - wait_block_state(handle, 0, timeout, std::move(P)); + wait_block_state(handle, 0, timeout, true, std::move(P)); } void ValidatorManagerImpl::wait_block_message_queue_short(BlockIdExt block_id, td::uint32 priority, diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index b3a6c0b2f5..70876084a8 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -201,9 +201,9 @@ class ValidatorManagerImpl : public ValidatorManager { void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override { UNREACHABLE(); } - void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; - void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, td::Promise>> promise) override { diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index f9a55c6ae3..886a71544e 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -348,7 +348,7 @@ void ValidatorManagerMasterchainStarter::got_init_block_handle(BlockHandle handl handle_ = std::move(handle); if (!handle_->received_state()) { LOG(ERROR) << "db inconsistent: last state ( " << handle_->id() << " ) not received"; - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state, handle_, 1, td::Timestamp::in(600.0), + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state, handle_, 1, td::Timestamp::in(600.0), true, [SelfId = actor_id(this), handle = handle_](td::Result> R) { td::actor::send_closure( SelfId, &ValidatorManagerMasterchainStarter::got_init_block_handle, handle); diff --git a/validator/manager.cpp b/validator/manager.cpp index 1fcd6a520e..6a11b6bfa2 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -579,7 +579,7 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id(), 0, td::Timestamp::in(60.0), std::move(P)); + wait_block_state_short(desc->block_id(), 0, td::Timestamp::in(60.0), true, std::move(P)); } if (validating_masterchain()) { td::MultiPromise mp; @@ -796,7 +796,7 @@ void ValidatorManagerImpl::run_ext_query(td::BufferSlice data, td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto it0 = block_state_cache_.find(handle->id()); if (it0 != block_state_cache_.end()) { it0->second.ttl_ = td::Timestamp::in(30.0); @@ -805,33 +805,43 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior } auto it = wait_state_.find(handle->id()); if (it == wait_state_.end()) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); + auto P1 = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R), true); + }); + auto P2 = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R), false); }); auto id = td::actor::create_actor("waitstate", handle, priority, opts_, last_masterchain_state_, - actor_id(this), td::Timestamp::at(timeout.at() + 10.0), std::move(P), - get_block_persistent_state_to_download(handle->id())) + actor_id(this), td::Timestamp::at(timeout.at() + 10.0), std::move(P1), + std::move(P2), get_block_persistent_state_to_download(handle->id())) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); } - it->second.waiting_.emplace_back(timeout, priority, std::move(promise)); + if (wait_store) { + it->second.waiting_.emplace_back(timeout, priority, std::move(promise)); + } else if (it->second.preliminary_done_) { + promise.set_result(it->second.preliminary_result_); + return; + } else { + it->second.waiting_preliminary_.emplace_back(timeout, priority, std::move(promise)); + } auto X = it->second.get_timeout(); td::actor::send_closure(it->second.actor_, &WaitBlockState::update_timeout, X.first, X.second); } void ValidatorManagerImpl::wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), priority, timeout, promise = std::move(promise)](td::Result R) mutable { + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); return; } td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), priority, timeout, - std::move(promise)); + wait_store, std::move(promise)); }); get_block_handle(block_id, true, std::move(P)); } @@ -954,7 +964,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 auto shard = handle->id().shard_full(); auto prev_shard = handle->one_prev(true).shard_full(); if (shard == prev_shard) { - wait_block_state_short(handle->one_prev(true), priority, timeout, std::move(promise)); + wait_block_state_short(handle->one_prev(true), priority, timeout, false, std::move(promise)); } else { CHECK(shard_parent(shard) == prev_shard); bool left = shard_child(prev_shard, true) == shard; @@ -973,7 +983,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 } } }); - wait_block_state_short(handle->one_prev(true), priority, timeout, std::move(P)); + wait_block_state_short(handle->one_prev(true), priority, timeout, false, std::move(P)); } } else { wait_block_state_merge(handle->one_prev(true), handle->one_prev(false), priority, timeout, std::move(promise)); @@ -1048,7 +1058,7 @@ void ValidatorManagerImpl::wait_block_message_queue(BlockHandle handle, td::uint } }); - wait_block_state(handle, priority, timeout, std::move(P)); + wait_block_state(handle, priority, timeout, false, std::move(P)); } void ValidatorManagerImpl::wait_block_message_queue_short(BlockIdExt block_id, td::uint32 priority, @@ -1309,37 +1319,51 @@ void ValidatorManagerImpl::get_block_by_seqno_from_db(AccountIdPrefixFull accoun td::actor::send_closure(db_, &Db::get_block_by_seqno, account, seqno, std::move(promise)); } -void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Result> R) { - if (R.is_ok()) { - block_state_cache_[handle->id()] = {R.ok(), td::Timestamp::in(30.0)}; - } +void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Result> R, + bool preliminary) { auto it = wait_state_.find(handle->id()); - if (it != wait_state_.end()) { - if (R.is_error()) { - auto S = R.move_as_error(); - if (S.code() != ErrorCode::timeout) { - for (auto &X : it->second.waiting_) { - X.promise.set_error(S.clone()); - } - } else if (it->second.waiting_.size() != 0) { - auto X = it->second.get_timeout(); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); - }); - auto id = td::actor::create_actor("waitstate", handle, X.second, opts_, last_masterchain_state_, - actor_id(this), X.first, std::move(P), - get_block_persistent_state_to_download(handle->id())) - .release(); - it->second.actor_ = id; - return; - } - } else { - auto r = R.move_as_ok(); + if (it == wait_state_.end()) { + return; + } + if (R.is_ok()) { + auto r = R.move_as_ok(); + for (auto &X : it->second.waiting_preliminary_) { + X.promise.set_result(r); + } + it->second.preliminary_done_ = true; + it->second.preliminary_result_ = r; + it->second.waiting_preliminary_.clear(); + if (!preliminary) { + block_state_cache_[handle->id()] = {r, td::Timestamp::in(30.0)}; for (auto &X : it->second.waiting_) { X.promise.set_result(r); } + wait_state_.erase(it); + } + } else if (!preliminary) { + auto S = R.move_as_error(); + if (S.code() != ErrorCode::timeout) { + for (auto &X : it->second.waiting_) { + X.promise.set_error(S.clone()); + } + for (auto &X : it->second.waiting_preliminary_) { + X.promise.set_error(S.clone()); + } + wait_state_.erase(it); + } else if (!it->second.waiting_.empty() || !it->second.waiting_preliminary_.empty()) { + auto X = it->second.get_timeout(); + auto P1 = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R), true); + }); + auto P2 = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R), false); + }); + auto id = td::actor::create_actor("waitstate", handle, X.second, opts_, last_masterchain_state_, + actor_id(this), X.first, std::move(P1), std::move(P2), + get_block_persistent_state_to_download(handle->id())) + .release(); + it->second.actor_ = id; } - wait_state_.erase(it); } } @@ -2756,7 +2780,7 @@ void ValidatorManagerImpl::got_next_gc_masterchain_handle(BlockHandle handle) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::got_next_gc_masterchain_state, std::move(handle), td::Ref{R.move_as_ok()}); }); - wait_block_state(handle, 0, td::Timestamp::in(60.0), std::move(P)); + wait_block_state(handle, 0, td::Timestamp::in(60.0), true, std::move(P)); } void ValidatorManagerImpl::got_next_gc_masterchain_state(BlockHandle handle, td::Ref state) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 952fcf49b5..6f7b73447a 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -166,9 +166,17 @@ class ValidatorManagerImpl : public ValidatorManager { WaitList() = default; std::pair get_timeout() const { + return get_timeout_impl(waiting_); + } + void check_timers() { + check_timers_impl(waiting_); + } + + protected: + static std::pair get_timeout_impl(const std::vector> &waiting) { td::Timestamp t = td::Timestamp::now(); td::uint32 prio = 0; - for (auto &v : waiting_) { + for (auto &v : waiting) { if (v.timeout.at() > t.at()) { t = v.timeout; } @@ -178,10 +186,10 @@ class ValidatorManagerImpl : public ValidatorManager { } return {td::Timestamp::at(t.at() + 10.0), prio}; } - void check_timers() { + static void check_timers_impl(std::vector> &waiting) { td::uint32 j = 0; - auto f = waiting_.begin(); - auto t = waiting_.end(); + auto f = waiting.begin(); + auto t = waiting.end(); while (f < t) { if (f->timeout.is_in_past()) { f->promise.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); @@ -192,16 +200,26 @@ class ValidatorManagerImpl : public ValidatorManager { j++; } } - waiting_.resize(j); + waiting.resize(j); } }; template - struct WaitListCaching : public WaitList { - bool done_ = false; - ResType result_; - td::Timestamp remove_at_; + struct WaitListPreliminary : WaitList { + std::vector> waiting_preliminary_; + bool preliminary_done_ = false; + ResType preliminary_result_; + + std::pair get_timeout() const { + auto t1 = WaitList::get_timeout_impl(this->waiting_); + auto t2 = WaitList::get_timeout_impl(waiting_preliminary_); + return {std::max(t1.first, t2.first), std::max(t1.second, t2.second)}; + } + void check_timers() { + WaitList::check_timers_impl(this->waiting_); + WaitList::check_timers_impl(waiting_preliminary_); + } }; - std::map>> wait_state_; + std::map>> wait_state_; std::map>> wait_block_data_; struct CachedBlockState { @@ -441,9 +459,9 @@ class ValidatorManagerImpl : public ValidatorManager { std::function write_data, td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; - void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; - void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, td::Promise>> promise) override; @@ -585,7 +603,7 @@ class ValidatorManagerImpl : public ValidatorManager { void register_block_handle(BlockHandle handle); - void finished_wait_state(BlockHandle handle, td::Result> R); + void finished_wait_state(BlockHandle handle, td::Result> R, bool preliminary); void finished_wait_data(BlockHandle handle, td::Result> R); void start_up() override; diff --git a/validator/queue-size-counter.cpp b/validator/queue-size-counter.cpp index 4fe55ae31b..487bdefc80 100644 --- a/validator/queue-size-counter.cpp +++ b/validator/queue-size-counter.cpp @@ -112,7 +112,7 @@ void QueueSizeCounter::get_queue_size_ex(ton::BlockIdExt block_id, bool calc_who } BlockHandle handle = R.move_as_ok(); td::actor::send_closure( - manager, &ValidatorManager::wait_block_state, handle, 0, td::Timestamp::in(10.0), + manager, &ValidatorManager::wait_block_state, handle, 0, td::Timestamp::in(10.0), false, [SelfId, handle](td::Result> R) mutable { if (R.is_error()) { td::actor::send_closure(SelfId, &QueueSizeCounter::on_error, handle->id(), @@ -159,7 +159,7 @@ void QueueSizeCounter::get_queue_size_cont(BlockHandle handle, td::Ref> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &QueueSizeCounter::on_error, state->get_block_id(), R.move_as_error()); @@ -213,7 +213,7 @@ void QueueSizeCounter::process_top_shard_blocks() { return; } td::actor::send_closure( - manager, &ValidatorManager::wait_block_state_short, R.ok()->id(), 0, td::Timestamp::in(10.0), + manager, &ValidatorManager::wait_block_state_short, R.ok()->id(), 0, td::Timestamp::in(10.0), false, [=](td::Result> R) { if (R.is_error()) { LOG(WARNING) << "Failed to get masterchain state: " << R.move_as_error(); diff --git a/validator/shard-block-retainer.cpp b/validator/shard-block-retainer.cpp index 528f6bcf78..8ef4d8f21e 100644 --- a/validator/shard-block-retainer.cpp +++ b/validator/shard-block-retainer.cpp @@ -109,7 +109,7 @@ void ShardBlockRetainer::new_shard_block_description(td::Refblock_id(), 0, td::Timestamp::in(30.0), + manager_, &ValidatorManager::wait_block_state_short, desc->block_id(), 0, td::Timestamp::in(30.0), true, [SelfId = actor_id(this), desc](td::Result> R) { if (R.is_error()) { LOG(WARNING) << "Wait block state for " << desc->block_id().to_str() << " : " << R.move_as_error(); diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index e5fd0a2ac9..49ee7b6e8a 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -161,7 +161,7 @@ void ShardClient::download_masterchain_state() { } }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state, masterchain_block_handle_, - shard_client_priority(), td::Timestamp::in(600), std::move(P)); + shard_client_priority(), td::Timestamp::in(600), true, std::move(P)); } void ShardClient::got_masterchain_block_state(td::Ref state) { @@ -201,7 +201,7 @@ void ShardClient::apply_all_shards() { } }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, shard->top_block_id(), - shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); + shard_client_priority(), td::Timestamp::in(1500), true, std::move(Q)); } } for (const auto &[wc, desc] : masterchain_state_->get_workchain_list()) { @@ -216,7 +216,7 @@ void ShardClient::apply_all_shards() { }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, BlockIdExt{wc, shardIdAll, 0, desc->zerostate_root_hash, desc->zerostate_file_hash}, - shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); + shard_client_priority(), td::Timestamp::in(1500), true, std::move(Q)); } } } diff --git a/validator/validator.h b/validator/validator.h index c04f835a7b..e736149500 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -325,9 +325,9 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise promise) = 0; - virtual void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + virtual void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) = 0; - virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) = 0; virtual void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, From 6cbae0344edc6d2a206a45d6aa7fe8220e9e7b1b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 5 Aug 2025 23:08:10 +0300 Subject: [PATCH 367/524] Refactor CollatorNode --- validator/CMakeLists.txt | 6 +- validator/collation-manager.cpp | 2 +- .../collator-node/collator-node-session.cpp | 237 ++++++++++ .../collator-node/collator-node-session.hpp | 79 ++++ .../{ => collator-node}/collator-node.cpp | 419 ++++++------------ .../{ => collator-node}/collator-node.hpp | 40 +- validator/manager.cpp | 7 +- validator/manager.hpp | 2 +- validator/validator-group.cpp | 4 +- 9 files changed, 471 insertions(+), 325 deletions(-) create mode 100644 validator/collator-node/collator-node-session.cpp create mode 100644 validator/collator-node/collator-node-session.hpp rename validator/{ => collator-node}/collator-node.cpp (53%) rename validator/{ => collator-node}/collator-node.hpp (74%) diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 6f68f55fd7..98987c78a6 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -64,7 +64,8 @@ set(VALIDATOR_HEADERS shard-block-retainer.hpp collation-manager.hpp - collator-node.hpp + collator-node/collator-node.hpp + collator-node/collator-node-session.hpp manager-disk.h manager-disk.hpp manager-init.h @@ -81,7 +82,8 @@ set(VALIDATOR_SOURCE apply-block.cpp block-handle.cpp collation-manager.cpp - collator-node.cpp + collator-node/collator-node.cpp + collator-node/collator-node-session.cpp get-next-key-blocks.cpp import-db-slice.cpp import-db-slice-local.cpp diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index 9eadd3f0af..9260dfb50b 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -16,7 +16,7 @@ */ #include "collation-manager.hpp" -#include "collator-node.hpp" +#include "collator-node/collator-node.hpp" #include "fabric.h" #include "td/utils/Random.h" diff --git a/validator/collator-node/collator-node-session.cpp b/validator/collator-node/collator-node-session.cpp new file mode 100644 index 0000000000..76c6f99741 --- /dev/null +++ b/validator/collator-node/collator-node-session.cpp @@ -0,0 +1,237 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#include "collator-node-session.hpp" + +#include "fabric.h" + +namespace ton::validator { + +static BlockSeqno get_next_block_seqno(const std::vector& prev) { + if (prev.size() == 1) { + return prev[0].seqno() + 1; + } + CHECK(prev.size() == 2); + return std::max(prev[0].seqno(), prev[1].seqno()) + 1; +} + +CollatorNodeSession::CollatorNodeSession(ShardIdFull shard, std::vector prev, + td::Ref validator_set, BlockIdExt min_masterchain_block_id, + bool can_generate, adnl::AdnlNodeIdShort local_id, + td::Ref opts, + td::actor::ActorId manager, + td::actor::ActorId adnl, td::actor::ActorId rldp) + : shard_(shard) + , prev_(std::move(prev)) + , validator_set_(validator_set) + , min_masterchain_block_id_(min_masterchain_block_id) + , can_generate_(can_generate) + , local_id_(local_id) + , opts_(opts) + , manager_(manager) + , adnl_(adnl) + , rldp_(rldp) + , next_block_seqno_(get_next_block_seqno(prev_)) { +} + +void CollatorNodeSession::start_up() { + LOG(INFO) << "Starting collator node session, shard " << shard_.to_str() << ", cc_seqno " + << validator_set_->get_catchain_seqno() << ", next block seqno " << next_block_seqno_; + + if (can_generate_) { + generate_block(prev_, {}, td::Timestamp::in(10.0), [](td::Result) {}); + } +} + +void CollatorNodeSession::tear_down() { + LOG(INFO) << "Finishing collator node session, shard " << shard_.to_str() << ", cc_seqno " + << validator_set_->get_catchain_seqno(); + for (auto& [_, entry] : cache_) { + entry->cancel(td::Status::Error("validator session finished")); + } +} + +void CollatorNodeSession::new_shard_block_accepted(BlockIdExt block_id, bool can_generate) { + CHECK(block_id.shard_full() == shard_); + can_generate_ = can_generate; + if (next_block_seqno_ > block_id.seqno()) { + return; + } + LOG(DEBUG) << "New shard block " << block_id.to_str(); + next_block_seqno_ = block_id.seqno() + 1; + prev_ = {block_id}; + + while (!cache_.empty()) { + auto& [cache_prev, entry] = *cache_.begin(); + if (entry->block_seqno < next_block_seqno_) { + entry->cancel(td::Status::Error(PSTRING() << "next block seqno " << entry->block_seqno << " is too old, expected " + << next_block_seqno_)); + } else if (entry->block_seqno == next_block_seqno_ && prev_ != cache_prev) { + entry->cancel(td::Status::Error(PSTRING() << "invalid prev blocks for seqno " << entry->block_seqno)); + } else { + break; + } + if (!entry->has_external_query_at && entry->has_internal_query_at) { + LOG(INFO) << "generate block query" + << ": shard=" << shard_.to_str() << ", cc_seqno=" << validator_set_->get_catchain_seqno() + << ", next_block_seqno=" << entry->block_seqno + << ": nobody asked for block, but we tried to generate it"; + } + if (entry->has_external_query_at && !entry->has_internal_query_at) { + LOG(INFO) << "generate block query" + << ": shard=" << shard_.to_str() << ", cc_seqno=" << validator_set_->get_catchain_seqno() + << ", next_block_seqno=" << entry->block_seqno + << ": somebody asked for block we didn't even try to generate"; + } + cache_.erase(cache_.begin()); + } + + if (can_generate_) { + generate_block(prev_, {}, td::Timestamp::in(10.0), [](td::Result) {}); + } +} + +void CollatorNodeSession::generate_block(std::vector prev_blocks, + td::optional o_priority, td::Timestamp timeout, + td::Promise promise) { + bool is_external = !o_priority; + BlockSeqno block_seqno = get_next_block_seqno(prev_blocks); + if (next_block_seqno_ > block_seqno) { + promise.set_error(td::Status::Error(PSTRING() << "next block seqno " << block_seqno << " is too old, expected " + << next_block_seqno_)); + return; + } + if (next_block_seqno_ == block_seqno && prev_ != prev_blocks) { + promise.set_error(td::Status::Error("invalid prev_blocks")); + return; + } + if (next_block_seqno_ + 10 < block_seqno) { + promise.set_error(td::Status::Error(PSTRING() << "next block seqno " << block_seqno << " is too new, current is " + << next_block_seqno_)); + return; + } + + static auto prefix_inner = [](td::StringBuilder& sb, const ShardIdFull& shard, CatchainSeqno cc_seqno, + BlockSeqno block_seqno, const td::optional& o_priority) { + sb << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno; + if (o_priority) { + sb << " external{"; + sb << "round_offset=" << o_priority.value().round - o_priority.value().first_block_round + << ",priority=" << o_priority.value().priority; + sb << ",first_block_round=" << o_priority.value().first_block_round; + sb << "}"; + } else { + sb << " internal"; + } + }; + auto prefix = [&](td::StringBuilder& sb) { + prefix_inner(sb, shard_, validator_set_->get_catchain_seqno(), block_seqno, o_priority); + }; + + auto cache_entry = cache_[prev_blocks]; + if (cache_entry == nullptr) { + cache_entry = cache_[prev_blocks] = std::make_shared(); + } + if (is_external && !cache_entry->has_external_query_at) { + cache_entry->has_external_query_at = td::Timestamp::now(); + if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { + FLOG(INFO) { + prefix(sb); + sb << ": got external query " << cache_entry->has_external_query_at - cache_entry->has_internal_query_at + << "s after internal query [WON]"; + }; + } + } + if (!is_external && !cache_entry->has_internal_query_at) { + cache_entry->has_internal_query_at = td::Timestamp::now(); + if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { + FLOG(INFO) { + prefix(sb); + sb << ": got internal query " << cache_entry->has_internal_query_at - cache_entry->has_external_query_at + << "s after external query [LOST]"; + }; + } + } + if (cache_entry->result) { + auto has_result_ago = td::Timestamp::now() - cache_entry->has_result_at; + FLOG(INFO) { + prefix(sb); + sb << ": using cached result " << " generated " << has_result_ago << "s ago"; + sb << (is_external ? " for external query [WON]" : " for internal query "); + }; + promise.set_result(cache_entry->result.value().clone()); + return; + } + cache_entry->promises.push_back(std::move(promise)); + + if (cache_entry->started) { + FLOG(INFO) { + prefix(sb); + sb << ": collation in progress, waiting"; + }; + return; + } + FLOG(INFO) { + prefix(sb); + sb << ": starting collation"; + }; + cache_entry->started = true; + cache_entry->block_seqno = block_seqno; + run_collate_query(CollateParams{.shard = shard_, + .min_masterchain_block_id = min_masterchain_block_id_, + .prev = std::move(prev_blocks), + .validator_set = validator_set_, + .collator_opts = opts_->get_collator_options(), + .collator_node_id = local_id_, + .skip_store_candidate = true}, + manager_, timeout, cache_entry->cancellation_token_source.get_cancellation_token(), + [=, shard = shard_, cc_seqno = validator_set_->get_catchain_seqno(), SelfId = actor_id(this), + timer = td::Timer{}](td::Result R) mutable { + FLOG(INFO) { + prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); + sb << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); + }; + td::actor::send_closure(SelfId, &CollatorNodeSession::process_result, cache_entry, std::move(R)); + }); +} + +void CollatorNodeSession::process_result(std::shared_ptr cache_entry, td::Result R) { + if (R.is_error()) { + cache_entry->started = false; + for (auto& p : cache_entry->promises) { + p.set_error(R.error().clone()); + } + } else { + cache_entry->result = R.move_as_ok(); + cache_entry->has_result_at = td::Timestamp::now(); + for (auto& p : cache_entry->promises) { + p.set_result(cache_entry->result.value().clone()); + } + } + cache_entry->promises.clear(); +} + +void CollatorNodeSession::CacheEntry::cancel(td::Status reason) { + for (auto& promise : promises) { + promise.set_error(reason.clone()); + } + promises.clear(); + cancellation_token_source.cancel(); +} + +} // namespace ton::validator diff --git a/validator/collator-node/collator-node-session.hpp b/validator/collator-node/collator-node-session.hpp new file mode 100644 index 0000000000..51e1b7cf6c --- /dev/null +++ b/validator/collator-node/collator-node-session.hpp @@ -0,0 +1,79 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "interfaces/validator-manager.h" +#include "rldp/rldp.h" +#include "rldp2/rldp.h" +#include +#include + +namespace ton::validator { + +class ValidatorManager; + +class CollatorNodeSession : public td::actor::Actor { + public: + CollatorNodeSession(ShardIdFull shard, std::vector prev, td::Ref validator_set, + BlockIdExt min_masterchain_block_id, bool can_generate, adnl::AdnlNodeIdShort local_id, + td::Ref opts, td::actor::ActorId manager, + td::actor::ActorId adnl, td::actor::ActorId rldp); + + void start_up() override; + void tear_down() override; + + void update_options(td::Ref opts) { + opts_ = std::move(opts); + } + + void new_shard_block_accepted(BlockIdExt block_id, bool can_generate); + + void generate_block(std::vector prev_blocks, td::optional o_priority, + td::Timestamp timeout, td::Promise promise); + + private: + ShardIdFull shard_; + std::vector prev_; + td::Ref validator_set_; + BlockIdExt min_masterchain_block_id_; + bool can_generate_; + adnl::AdnlNodeIdShort local_id_; + td::Ref opts_; + td::actor::ActorId manager_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + + struct CacheEntry { + bool started = false; + td::Timestamp has_internal_query_at; + td::Timestamp has_external_query_at; + td::Timestamp has_result_at; + BlockSeqno block_seqno = 0; + td::optional result; + td::CancellationTokenSource cancellation_token_source; + std::vector> promises; + + void cancel(td::Status reason); + }; + + BlockSeqno next_block_seqno_; + std::map, std::shared_ptr> cache_; + + void process_result(std::shared_ptr cache_entry, td::Result R); +}; + +} // namespace ton::validator diff --git a/validator/collator-node.cpp b/validator/collator-node/collator-node.cpp similarity index 53% rename from validator/collator-node.cpp rename to validator/collator-node/collator-node.cpp index ba3ff01ebd..6f0cae4191 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node/collator-node.cpp @@ -75,6 +75,17 @@ void CollatorNode::add_shard(ShardIdFull shard) { } LOG(INFO) << "Collator node: local_id=" << local_id_ << " , shard=" << shard.to_str(); collating_shards_.push_back(shard); + if (last_masterchain_state_.is_null()) { + return; + } + for (auto& [group_shard, validator_group] : validator_groups_) { + if (validator_group.actor.empty() && shard_intersects(shard, group_shard)) { + validator_group.actor = td::actor::create_actor( + PSTRING() << "collatornode" << shard.to_str(), shard, validator_group.prev, + last_masterchain_state_->get_validator_set(group_shard), last_masterchain_state_->get_block_id(), + can_generate(), local_id_, opts_, manager_, adnl_, rldp_); + } + } } void CollatorNode::del_shard(ShardIdFull shard) { @@ -82,6 +93,20 @@ void CollatorNode::del_shard(ShardIdFull shard) { if (it != collating_shards_.end()) { collating_shards_.erase(it); } + for (auto& [group_shard, validator_group] : validator_groups_) { + if (!validator_group.actor.empty() && shard_intersects(shard, group_shard) && !can_collate_shard(group_shard)) { + validator_group.actor = {}; + } + } +} + +void CollatorNode::update_options(td::Ref opts) { + for (auto& [_, shard] : validator_groups_) { + if (!shard.actor.empty()) { + td::actor::send_closure(shard.actor, &CollatorNodeSession::update_options, opts); + } + } + opts_ = std::move(opts); } void CollatorNode::new_masterchain_block_notification(td::Ref state) { @@ -91,7 +116,7 @@ void CollatorNode::new_masterchain_block_notification(td::Ref last_key_block_seqno_ = state->last_key_block_id().seqno(); mc_config_status_ = check_mc_config(); if (mc_config_status_.is_error()) { - LOG(ERROR) << "Cannot validate masterchain config (possibly outdated software):" << mc_config_status_; + LOG(ERROR) << "Cannot validate masterchain config (possibly outdated software): " << mc_config_status_; } } @@ -131,36 +156,42 @@ void CollatorNode::new_masterchain_block_notification(td::Ref new_shards.emplace(shard, std::vector{v->top_block_id()}); } } - - for (auto& [shard, prev] : new_shards) { - CatchainSeqno cc_seqno = state->get_validator_set(shard)->get_catchain_seqno(); - auto it = validator_groups_.emplace(shard, ValidatorGroupInfo{}); - ValidatorGroupInfo& info = it.first->second; - if (it.second || info.cc_seqno != cc_seqno) { - info.cleanup(); - info.cc_seqno = cc_seqno; - } - } for (auto it = validator_groups_.begin(); it != validator_groups_.end();) { if (new_shards.contains(it->first)) { ++it; } else { - it->second.cleanup(); it = validator_groups_.erase(it); } } for (auto& [shard, prev] : new_shards) { - ValidatorGroupInfo& info = validator_groups_[shard]; - update_validator_group_info(shard, std::move(prev), info.cc_seqno); - auto it = future_validator_groups_.find({shard, info.cc_seqno}); - if (it != future_validator_groups_.end()) { - for (auto& new_prev : it->second.pending_blocks) { - update_validator_group_info(shard, std::move(new_prev), info.cc_seqno); + auto validator_set = state->get_validator_set(shard); + CatchainSeqno cc_seqno = validator_set->get_catchain_seqno(); + auto [it, created] = validator_groups_.emplace(shard, ValidatorGroupInfo{}); + it->second.prev = std::move(prev); + if (created || it->second.cc_seqno != cc_seqno) { + it->second.cc_seqno = cc_seqno; + if (can_collate_shard(shard)) { + it->second.actor = td::actor::create_actor( + PSTRING() << "collatornode" << shard.to_str(), shard, it->second.prev, validator_set, + last_masterchain_state_->get_block_id(), can_generate(), local_id_, opts_, manager_, adnl_, rldp_); } - for (auto& promise : it->second.promises) { + } else if (!it->second.actor.empty() && prev.size() == 1) { + td::actor::send_closure(it->second.actor, &CollatorNodeSession::new_shard_block_accepted, prev[0], + can_generate()); + } + auto it2 = future_validator_groups_.find({shard, cc_seqno}); + if (it2 != future_validator_groups_.end()) { + FutureValidatorGroup& future_group = it2->second; + if (!it->second.actor.empty()) { + for (const BlockIdExt& block_id : future_group.pending_blocks) { + td::actor::send_closure(it->second.actor, &CollatorNodeSession::new_shard_block_accepted, block_id, + can_generate()); + } + } + for (auto& promise : future_group.promises) { promise.set_value(td::Unit()); } - future_validator_groups_.erase(it); + future_validator_groups_.erase(it2); } } @@ -181,82 +212,24 @@ void CollatorNode::update_shard_client_handle(BlockHandle shard_client_handle) { shard_client_handle_ = shard_client_handle; } -void CollatorNode::update_validator_group_info(ShardIdFull shard, std::vector prev, - CatchainSeqno cc_seqno) { - if (!can_collate_shard(shard)) { +void CollatorNode::new_shard_block_accepted(BlockIdExt block_id, CatchainSeqno cc_seqno) { + if (!can_collate_shard(block_id.shard_full())) { return; } - CHECK(prev.size() == 1 || prev.size() == 2); - BlockSeqno next_block_seqno = prev[0].seqno() + 1; - if (prev.size() == 2) { - next_block_seqno = std::max(next_block_seqno, prev[1].seqno() + 1); - } - auto it = validator_groups_.find(shard); - if (it != validator_groups_.end()) { - ValidatorGroupInfo& info = it->second; - if (info.cc_seqno == cc_seqno) { // block from currently known validator group - if (info.next_block_seqno < next_block_seqno) { - LOG(DEBUG) << "updated validator group info: shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno - << ", next_block_seqno=" << next_block_seqno; - info.next_block_seqno = next_block_seqno; - info.prev = std::move(prev); - for (auto cache_it = info.cache.begin(); cache_it != info.cache.end();) { - auto& [cached_prev, cache_entry] = *cache_it; - if (cache_entry->block_seqno < info.next_block_seqno) { - cache_entry->cancel(td::Status::Error(PSTRING() << "next block seqno " << cache_entry->block_seqno - << " is too small, expected " << info.next_block_seqno)); - if (!cache_entry->has_external_query_at && cache_entry->has_internal_query_at) { - LOG(INFO) << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno - << ", next_block_seqno=" << cache_entry->block_seqno - << ": nobody asked for block, but we tried to generate it"; - } - if (cache_entry->has_external_query_at && !cache_entry->has_internal_query_at) { - LOG(INFO) << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno - << ", next_block_seqno=" << cache_entry->block_seqno - << ": somebody asked for block we didn't even tried to generate"; - } - cache_it = info.cache.erase(cache_it); - continue; - } - if (cache_entry->block_seqno == info.next_block_seqno && cached_prev != info.prev) { - cache_entry->cancel(td::Status::Error("invalid prev blocks")); - if (!cache_entry->has_external_query_at && cache_entry->has_internal_query_at) { - LOG(INFO) << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno - << ", next_block_seqno=" << cache_entry->block_seqno - << ": nobody asked for block, but we tried to generate it"; - } - if (cache_entry->has_external_query_at && !cache_entry->has_internal_query_at) { - LOG(INFO) << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno - << ", next_block_seqno=" << cache_entry->block_seqno - << ": somebody asked for block we didn't even tried to generate"; - } - cache_it = info.cache.erase(cache_it); - continue; - } - ++cache_it; - } - auto S = check_out_of_sync(); - if (S.is_error()) { - LOG(DEBUG) << "not generating block automatically: " << S; - return; - } - if (mc_config_status_.is_error()) { - LOG(DEBUG) << "not generating block automatically: unsupported mc config: " << mc_config_status_; - return; - } - generate_block(shard, cc_seqno, info.prev, {}, td::Timestamp::in(10.0), [](td::Result) {}); - } - return; + auto it = validator_groups_.find(block_id.shard_full()); + if (it == validator_groups_.end() || it->second.cc_seqno != cc_seqno) { + auto future_group = get_future_validator_group(block_id.shard_full(), cc_seqno); + if (future_group.is_error()) { + LOG(DEBUG) << "Dropping new shard block " << block_id.to_str() << " cc_seqno=" << cc_seqno << " : " + << future_group.error(); + } else { + LOG(DEBUG) << "New shard block in future validator group " << block_id.to_str() << " cc_seqno=" << cc_seqno; + future_group.ok()->pending_blocks.push_back(block_id); } + return; } - auto future_validator_group = get_future_validator_group(shard, cc_seqno); - if (future_validator_group.is_ok()) { - // future validator group, remember for later - future_validator_group.ok()->pending_blocks.push_back(std::move(prev)); + if (!it->second.actor.empty()) { + td::actor::send_closure(it->second.actor, &CollatorNodeSession::new_shard_block_accepted, block_id, can_generate()); } } @@ -276,34 +249,17 @@ td::Result CollatorNode::get_future_validat return td::Status::Error("no such shard"); } if (cc_seqno < it->second.cc_seqno) { // past validator group - return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " is outdated (current is" << it->second.cc_seqno - << ")"); + return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " for shard " << shard.to_str() + << " is outdated (current is" << it->second.cc_seqno << ")"); } if (cc_seqno - it->second.cc_seqno > 1) { // future validator group, cc_seqno too big - return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " is too big (currently known is" - << it->second.cc_seqno << ")"); + return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " for shard " << shard.to_str() + << " is too big (currently known is" << it->second.cc_seqno << ")"); } // future validator group return &future_validator_groups_[{shard, cc_seqno}]; } -void CollatorNode::ValidatorGroupInfo::cleanup() { - prev.clear(); - next_block_seqno = 0; - for (auto& [_, cache_entry] : cache) { - cache_entry->cancel(td::Status::Error("validator group is outdated")); - } - cache.clear(); -} - -void CollatorNode::CacheEntry::cancel(td::Status reason) { - for (auto& promise : promises) { - promise.set_error(reason.clone()); - } - promises.clear(); - cancellation_token_source.cancel(); -} - static td::BufferSlice serialize_error(td::Status error) { return create_serialize_tl_object(error.code(), error.message().c_str()); } @@ -423,19 +379,19 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data return; } LOG(INFO) << "got adnl query from " << src << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno; - generate_block(shard, cc_seqno, std::move(prev_blocks), priority, td::Timestamp::in(10.0), std::move(new_promise)); + process_generate_block_query(shard, cc_seqno, std::move(prev_blocks), priority, td::Timestamp::in(10.0), + std::move(new_promise)); } -void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, - std::optional o_priority, td::Timestamp timeout, - td::Promise promise) { - bool is_external = !o_priority; +void CollatorNode::process_generate_block_query(ShardIdFull shard, CatchainSeqno cc_seqno, + std::vector prev_blocks, BlockCandidatePriority priority, + td::Timestamp timeout, td::Promise promise) { if (last_masterchain_state_.is_null()) { promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); return; } - if (!can_collate_shard(shard)) { - promise.set_error(td::Status::Error(PSTRING() << "this node can't collate shard " << shard.to_str())); + if (timeout.is_in_past()) { + promise.set_error(td::Status::Error(ErrorCode::timeout)); return; } auto it = validator_groups_.find(shard); @@ -443,133 +399,19 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std TRY_RESULT_PROMISE(promise, future_validator_group, get_future_validator_group(shard, cc_seqno)); future_validator_group->promises.push_back([=, SelfId = actor_id(this), prev_blocks = std::move(prev_blocks), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - return; - } - if (timeout.is_in_past()) { - promise.set_error(td::Status::Error(ErrorCode::timeout)); - return; - } - td::actor::send_closure(SelfId, &CollatorNode::generate_block, shard, cc_seqno, std::move(prev_blocks), - std::move(o_priority), timeout, std::move(promise)); + TRY_STATUS_PROMISE(promise, R.move_as_status()); + td::actor::send_closure(SelfId, &CollatorNode::process_generate_block_query, shard, cc_seqno, + std::move(prev_blocks), std::move(priority), timeout, std::move(promise)); }); return; } ValidatorGroupInfo& validator_group_info = it->second; - BlockSeqno block_seqno = prev_blocks.at(0).seqno() + 1; - if (prev_blocks.size() == 2) { - block_seqno = std::max(block_seqno, prev_blocks.at(1).seqno() + 1); - } - if (validator_group_info.next_block_seqno > block_seqno) { - promise.set_error(td::Status::Error(PSTRING() << "next block seqno " << block_seqno << " is too small, expected " - << validator_group_info.next_block_seqno)); - return; - } - if (validator_group_info.next_block_seqno == block_seqno && validator_group_info.prev != prev_blocks) { - promise.set_error(td::Status::Error("invalid prev_blocks")); + if (validator_group_info.actor.empty()) { + promise.set_error(td::Status::Error(PSTRING() << "cannot collate shard " << shard.to_str())); return; } - - static auto prefix_inner = [](auto& sb, auto& shard, auto cc_seqno, auto block_seqno, - const std::optional& o_priority) { - sb << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno; - if (o_priority) { - sb << " external{"; - sb << "round_offset=" << o_priority->round - o_priority->first_block_round - << ",priority=" << o_priority->priority; - sb << ",first_block_round=" << o_priority->first_block_round; - sb << "}"; - } else { - sb << " internal"; - } - }; - auto prefix = [&](auto& sb) { prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); }; - - auto cache_entry = validator_group_info.cache[prev_blocks]; - if (cache_entry == nullptr) { - cache_entry = validator_group_info.cache[prev_blocks] = std::make_shared(); - } - if (is_external && !cache_entry->has_external_query_at) { - cache_entry->has_external_query_at = td::Timestamp::now(); - if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { - FLOG(INFO) { - prefix(sb); - sb << ": got external query " - << cache_entry->has_external_query_at.at() - cache_entry->has_internal_query_at.at() - << "s after internal query [WON]"; - }; - } - } - if (!is_external && !cache_entry->has_internal_query_at) { - cache_entry->has_internal_query_at = td::Timestamp::now(); - if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { - FLOG(INFO) { - prefix(sb); - sb << ": got internal query " - << cache_entry->has_internal_query_at.at() - cache_entry->has_external_query_at.at() - << "s after external query [LOST]"; - }; - } - } - if (cache_entry->result) { - auto has_result_ago = td::Timestamp::now().at() - cache_entry->has_result_at.at(); - FLOG(INFO) { - prefix(sb); - sb << ": using cached result " << " generated " << has_result_ago << "s ago"; - sb << (is_external ? " for external query [WON]" : " for internal query "); - }; - - promise.set_result(cache_entry->result.value().clone()); - return; - } - cache_entry->promises.push_back(std::move(promise)); - - if (cache_entry->started) { - FLOG(INFO) { - prefix(sb); - sb << ": collation in progress, waiting"; - }; - return; - } - FLOG(INFO) { - prefix(sb); - sb << ": starting collation"; - }; - cache_entry->started = true; - cache_entry->block_seqno = block_seqno; - run_collate_query(CollateParams{.shard = shard, - .min_masterchain_block_id = last_masterchain_state_->get_block_id(), - .prev = std::move(prev_blocks), - .validator_set = last_masterchain_state_->get_validator_set(shard), - .collator_opts = opts_->get_collator_options(), - .collator_node_id = local_id_, - .skip_store_candidate = true}, - manager_, timeout, cache_entry->cancellation_token_source.get_cancellation_token(), - [=, SelfId = actor_id(this), timer = td::Timer{}](td::Result R) { - FLOG(INFO) { - prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); - sb << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); - }; - td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); - }); -} - -void CollatorNode::process_result(std::shared_ptr cache_entry, td::Result R) { - if (R.is_error()) { - cache_entry->started = false; - for (auto& p : cache_entry->promises) { - p.set_error(R.error().clone()); - } - } else { - cache_entry->result = R.move_as_ok(); - cache_entry->has_result_at = td::Timestamp::now(); - for (auto& p : cache_entry->promises) { - p.set_result(cache_entry->result.value().clone()); - } - } - cache_entry->promises.clear(); + td::actor::send_closure(validator_group_info.actor, &CollatorNodeSession::generate_block, std::move(prev_blocks), + priority, timeout, std::move(promise)); } td::Status CollatorNode::check_out_of_sync() { @@ -645,53 +487,54 @@ tl_object_ptr CollatorNode::serialize_candidate td::Result CollatorNode::deserialize_candidate(tl_object_ptr f, int max_decompressed_data_size, int proto_version) { td::Result res; - ton_api::downcast_call(*f, td::overloaded( - [&](ton_api::collatorNode_candidate& c) { - res = [&]() -> td::Result { - auto hash = td::sha256_bits256(c.collated_data_); - auto key = ton::PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), hash, std::move(c.data_), - std::move(c.collated_data_)}; - }(); - }, - [&](ton_api::collatorNode_compressedCandidate& c) { - res = [&]() -> td::Result { - if (c.decompressed_size_ <= 0) { - return td::Status::Error("invalid decompressed size"); - } - if (c.decompressed_size_ > max_decompressed_data_size) { - return td::Status::Error("decompressed size is too big"); - } - TRY_RESULT(p, validatorsession::decompress_candidate_data( - c.data_, false, c.decompressed_size_, max_decompressed_data_size, proto_version)); - auto collated_data_hash = td::sha256_bits256(p.second); - auto key = ton::PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, - std::move(p.first), std::move(p.second)}; - }(); - }, - [&](ton_api::collatorNode_compressedCandidateV2& c) { - res = [&]() -> td::Result { - TRY_RESULT(p, validatorsession::decompress_candidate_data( - c.data_, true, 0, max_decompressed_data_size, proto_version)); - auto collated_data_hash = td::sha256_bits256(p.second); - auto key = ton::PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, - std::move(p.first), std::move(p.second)}; - }(); - })); + ton_api::downcast_call( + *f, td::overloaded( + [&](ton_api::collatorNode_candidate& c) { + res = [&]() -> td::Result { + auto hash = td::sha256_bits256(c.collated_data_); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), hash, std::move(c.data_), + std::move(c.collated_data_)}; + }(); + }, + [&](ton_api::collatorNode_compressedCandidate& c) { + res = [&]() -> td::Result { + if (c.decompressed_size_ <= 0) { + return td::Status::Error("invalid decompressed size"); + } + if (c.decompressed_size_ > max_decompressed_data_size) { + return td::Status::Error("decompressed size is too big"); + } + TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, false, c.decompressed_size_, + max_decompressed_data_size, proto_version)); + auto collated_data_hash = td::sha256_bits256(p.second); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), + std::move(p.second)}; + }(); + }, + [&](ton_api::collatorNode_compressedCandidateV2& c) { + res = [&]() -> td::Result { + TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, true, 0, + max_decompressed_data_size, proto_version)); + auto collated_data_hash = td::sha256_bits256(p.second); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), + std::move(p.second)}; + }(); + })); return res; } diff --git a/validator/collator-node.hpp b/validator/collator-node/collator-node.hpp similarity index 74% rename from validator/collator-node.hpp rename to validator/collator-node/collator-node.hpp index a4df036816..d07116979f 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node/collator-node.hpp @@ -16,6 +16,7 @@ */ #pragma once +#include "collator-node-session.hpp" #include "interfaces/validator-manager.h" #include "rldp/rldp.h" #include "rldp2/rldp.h" @@ -36,16 +37,17 @@ class CollatorNode : public td::actor::Actor { void add_shard(ShardIdFull shard); void del_shard(ShardIdFull shard); + void update_options(td::Ref opts); + void new_masterchain_block_notification(td::Ref state); void update_shard_client_handle(BlockHandle shard_client_handle); - void update_validator_group_info(ShardIdFull shard, std::vector prev, CatchainSeqno cc_seqno); - - void update_options(td::Ref opts) { - opts_ = std::move(opts); - } + void new_shard_block_accepted(BlockIdExt block_id, CatchainSeqno cc_seqno); private: void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); + void process_generate_block_query(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, + BlockCandidatePriority priority, td::Timestamp timeout, + td::Promise promise); void process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode_ping& ping, td::Promise promise); bool can_collate_shard(ShardIdFull shard) const; @@ -58,28 +60,13 @@ class CollatorNode : public td::actor::Actor { std::vector collating_shards_; std::set validator_adnl_ids_; - struct CacheEntry { - bool started = false; - td::Timestamp has_internal_query_at; - td::Timestamp has_external_query_at; - td::Timestamp has_result_at; - BlockSeqno block_seqno = 0; - td::optional result; - td::CancellationTokenSource cancellation_token_source; - std::vector> promises; - - void cancel(td::Status reason); - }; struct ValidatorGroupInfo { CatchainSeqno cc_seqno{0}; std::vector prev; - BlockSeqno next_block_seqno{0}; - std::map, std::shared_ptr> cache; - - void cleanup(); + td::actor::ActorOwn actor; }; struct FutureValidatorGroup { - std::vector> pending_blocks; + std::vector pending_blocks; std::vector> promises; }; std::map validator_groups_; @@ -93,14 +80,13 @@ class CollatorNode : public td::actor::Actor { td::Result get_future_validator_group(ShardIdFull shard, CatchainSeqno cc_seqno); - void generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, - std::optional o_priority, td::Timestamp timeout, - td::Promise promise); - void process_result(std::shared_ptr cache_entry, td::Result R); - td::Status check_out_of_sync(); td::Status check_mc_config(); + bool can_generate() { + return check_out_of_sync().is_ok() && mc_config_status_.is_ok(); + } + public: static tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress); static td::Result deserialize_candidate(tl_object_ptr f, diff --git a/validator/manager.cpp b/validator/manager.cpp index 6a11b6bfa2..4862c47273 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -238,8 +238,7 @@ void ValidatorManagerImpl::new_block_broadcast(BlockBroadcast broadcast, td::Pro void ValidatorManagerImpl::validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno) { for (auto &[_, collator_node] : collator_nodes_) { if (collator_node.can_collate_shard(block_id.shard_full())) { - td::actor::send_closure(collator_node.actor, &CollatorNode::update_validator_group_info, block_id.shard_full(), - std::vector{block_id}, cc_seqno); + td::actor::send_closure(collator_node.actor, &CollatorNode::new_shard_block_accepted, block_id, cc_seqno); } } } @@ -599,8 +598,8 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refshard())) { - td::actor::send_closure(collator_node.actor, &CollatorNode::update_validator_group_info, desc->shard(), - std::vector{desc->block_id()}, desc->catchain_seqno()); + td::actor::send_closure(collator_node.actor, &CollatorNode::new_shard_block_accepted, desc->block_id(), + desc->catchain_seqno()); } } } diff --git a/validator/manager.hpp b/validator/manager.hpp index 6f7b73447a..68799c3726 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -38,7 +38,7 @@ #include "storage-stat-cache.hpp" #include "validator-telemetry.hpp" #include "impl/candidates-buffer.hpp" -#include "collator-node.hpp" +#include "collator-node/collator-node.hpp" #include "shard-block-verifier.hpp" #include "shard-block-retainer.hpp" #include "td/utils/LRUCache.h" diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 6a6148a1b8..163eb90bfd 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -24,7 +24,7 @@ #include "common/delay.h" #include "ton/lite-tl.hpp" #include "td/utils/Random.h" -#include "collator-node.hpp" +#include "collator-node/collator-node.hpp" namespace ton { @@ -213,7 +213,7 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so return; } auto next_block_id = create_next_block_id(root_hash, file_hash); - LOG(WARNING) << "Accepted block " << next_block_id; + LOG(WARNING) << "Accepted block " << next_block_id.to_str(); stats.block_id = next_block_id; td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, std::move(stats)); auto block = From badc4fd4ff96ab80f690d9800d13982e14ab444e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 6 Aug 2025 15:10:34 +0300 Subject: [PATCH 368/524] Enable optimistic collation in proto version 6 --- validator-session/validator-session.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index f0dbbfc8de..40ecf1b2ed 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -1095,7 +1095,7 @@ ValidatorSessionImpl::ValidatorSessionImpl(catchain::CatChainSessionId session_i , overlay_manager_(overlays) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) { compress_block_candidates_ = opts.proto_version >= 4; - allow_optimistic_generation_ = opts.proto_version >= 5; + allow_optimistic_generation_ = opts.proto_version >= 6; description_ = ValidatorSessionDescription::create(std::move(opts), nodes, local_id); src_round_candidate_.resize(description_->get_total_nodes()); } From 8a49651b611c3b0ca18a8f92c8598a6fcca2c50f Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 7 Aug 2025 12:42:03 +0300 Subject: [PATCH 369/524] Allow hostnames instead of IPs for liteservers (#1771) --- adnl/adnl-ext-client.cpp | 14 ++++++++++++++ adnl/adnl-ext-client.h | 2 ++ adnl/adnl-ext-client.hpp | 4 ++++ .../blockchain-explorer-query.cpp | 6 +++--- blockchain-explorer/blockchain-explorer.cpp | 8 ++++---- blockchain-explorer/blockchain-explorer.hpp | 2 +- lite-client/ext-client.cpp | 11 +++++------ lite-client/lite-client.cpp | 2 +- lite-client/query-utils.cpp | 13 +++++++++++-- lite-client/query-utils.hpp | 9 ++++++--- tl/generate/scheme/ton_api.tl | 4 ++-- tl/generate/scheme/ton_api.tlo | Bin 119192 -> 119264 bytes utils/proxy-liteserver.cpp | 7 +++---- 13 files changed, 56 insertions(+), 26 deletions(-) diff --git a/adnl/adnl-ext-client.cpp b/adnl/adnl-ext-client.cpp index 9602b521e8..873d34b110 100644 --- a/adnl/adnl-ext-client.cpp +++ b/adnl/adnl-ext-client.cpp @@ -31,6 +31,14 @@ void AdnlExtClientImpl::alarm() { next_create_at_ = td::Timestamp::in(10.0); alarm_timestamp() = next_create_at_; + if (!dst_host_.empty()) { + auto S = dst_addr_.init_host_port(dst_host_); + if (S.is_error()) { + LOG(INFO) << "failed to connect to " << dst_host_ << ": " << S; + return; + } + LOG(DEBUG) << "resolved " << dst_host_ << " -> " << dst_addr_; + } auto fd = td::SocketFd::open(dst_addr_); if (fd.is_error()) { LOG(INFO) << "failed to connect to " << dst_addr_ << ": " << fd.move_as_error(); @@ -166,6 +174,12 @@ td::actor::ActorOwn AdnlExtClient::create(AdnlNodeIdFull dst, td: return td::actor::create_actor("extclient", std::move(dst), dst_addr, std::move(callback)); } +td::actor::ActorOwn AdnlExtClient::create(AdnlNodeIdFull dst, std::string dst_host, + std::unique_ptr callback) { + return td::actor::create_actor("extclient", std::move(dst), std::move(dst_host), + std::move(callback)); +} + td::actor::ActorOwn AdnlExtClient::create(AdnlNodeIdFull dst, PrivateKey local_id, td::IPAddress dst_addr, std::unique_ptr callback) { diff --git a/adnl/adnl-ext-client.h b/adnl/adnl-ext-client.h index b9a5d570ad..9e537a5a51 100644 --- a/adnl/adnl-ext-client.h +++ b/adnl/adnl-ext-client.h @@ -39,6 +39,8 @@ class AdnlExtClient : public td::actor::Actor { td::Promise promise) = 0; static td::actor::ActorOwn create(AdnlNodeIdFull dst, td::IPAddress dst_addr, std::unique_ptr callback); + static td::actor::ActorOwn create(AdnlNodeIdFull dst, std::string dst_host, + std::unique_ptr callback); static td::actor::ActorOwn create(AdnlNodeIdFull dst, PrivateKey local_id, td::IPAddress dst_addr, std::unique_ptr callback); }; diff --git a/adnl/adnl-ext-client.hpp b/adnl/adnl-ext-client.hpp index 1dd7d2ba3f..76de83ab44 100644 --- a/adnl/adnl-ext-client.hpp +++ b/adnl/adnl-ext-client.hpp @@ -71,6 +71,9 @@ class AdnlExtClientImpl : public AdnlExtClient { AdnlExtClientImpl(AdnlNodeIdFull dst_id, td::IPAddress dst_addr, std::unique_ptr callback) : dst_(std::move(dst_id)), dst_addr_(dst_addr), callback_(std::move(callback)) { } + AdnlExtClientImpl(AdnlNodeIdFull dst_id, std::string dst_host, std::unique_ptr callback) + : dst_(std::move(dst_id)), dst_host_(std::move(dst_host)), callback_(std::move(callback)) { + } AdnlExtClientImpl(AdnlNodeIdFull dst_id, PrivateKey local_id, td::IPAddress dst_addr, std::unique_ptr callback) : dst_(std::move(dst_id)), local_id_(local_id), dst_addr_(dst_addr), callback_(std::move(callback)) { @@ -133,6 +136,7 @@ class AdnlExtClientImpl : public AdnlExtClient { AdnlNodeIdFull dst_; PrivateKey local_id_; td::IPAddress dst_addr_; + std::string dst_host_; std::unique_ptr callback_; diff --git a/blockchain-explorer/blockchain-explorer-query.cpp b/blockchain-explorer/blockchain-explorer-query.cpp index 26a6787e1e..d540d48782 100644 --- a/blockchain-explorer/blockchain-explorer-query.cpp +++ b/blockchain-explorer/blockchain-explorer-query.cpp @@ -1429,10 +1429,10 @@ void HttpQueryStatus::finish_query() { A << "" << static_cast(x->ts_.at_unix()) << ""; } A << "\n"; - for (td::uint32 i = 0; i < results_.ips.size(); i++) { + for (td::uint32 i = 0; i < results_.addrs.size(); i++) { A << ""; - if (results_.ips[i].is_valid()) { - A << "" << results_.ips[i].get_ip_str() << ":" << results_.ips[i].get_port() << ""; + if (!results_.addrs[i].empty()) { + A << "" << results_.addrs[i] << ""; } else { A << "hidden"; } diff --git a/blockchain-explorer/blockchain-explorer.cpp b/blockchain-explorer/blockchain-explorer.cpp index ca50d52665..d21bc465ce 100644 --- a/blockchain-explorer/blockchain-explorer.cpp +++ b/blockchain-explorer/blockchain-explorer.cpp @@ -187,7 +187,7 @@ class CoreActor : public CoreActorInterface { std::mutex queue_mutex_; std::mutex res_mutex_; std::map> results_; - std::vector addrs_; + std::vector addrs_; static CoreActor* instance_; td::actor::ActorId self_id_; @@ -220,7 +220,7 @@ class CoreActor : public CoreActorInterface { } void get_results(td::uint32 max, td::Promise promise) override { RemoteNodeStatusList r; - r.ips = hide_ips_ ? std::vector{addrs_.size()} : addrs_; + r.addrs = hide_ips_ ? std::vector{addrs_.size()} : addrs_; auto it = results_.rbegin(); while (it != results_.rend() && r.results.size() < max) { r.results.push_back(it->second); @@ -445,14 +445,14 @@ class CoreActor : public CoreActorInterface { r_servers.ensure(); servers = r_servers.move_as_ok(); for (const auto& serv : servers) { - addrs_.push_back(serv.addr); + addrs_.push_back(serv.hostname); } } else { if (!remote_addr_.is_valid()) { LOG(FATAL) << "remote addr not set"; } - addrs_.push_back(remote_addr_); servers.push_back(liteclient::LiteServerConfig{ton::adnl::AdnlNodeIdFull{remote_public_key_}, remote_addr_}); + addrs_.push_back(servers.back().hostname); } n_servers_ = servers.size(); client_ = liteclient::ExtClient::create(std::move(servers), make_callback(), true); diff --git a/blockchain-explorer/blockchain-explorer.hpp b/blockchain-explorer/blockchain-explorer.hpp index 1bae362dee..abbbde9e9b 100644 --- a/blockchain-explorer/blockchain-explorer.hpp +++ b/blockchain-explorer/blockchain-explorer.hpp @@ -59,7 +59,7 @@ class CoreActorInterface : public td::actor::Actor { }; struct RemoteNodeStatusList { - std::vector ips; + std::vector addrs; std::vector> results; }; virtual ~CoreActorInterface() = default; diff --git a/lite-client/ext-client.cpp b/lite-client/ext-client.cpp index fd624ff8cc..d6b7982226 100644 --- a/lite-client/ext-client.cpp +++ b/lite-client/ext-client.cpp @@ -104,7 +104,7 @@ class ExtClientImpl : public ExtClient { promise.set_result(std::move(R)); }; LOG(DEBUG) << "Sending query " << query_info.to_str() << " to server #" << server.idx << " (" - << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; + << server.config.hostname << ")"; send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, std::move(P)); } @@ -173,9 +173,9 @@ class ExtClientImpl : public ExtClient { td::actor::ActorId parent_; size_t idx_; }; - LOG(INFO) << "Connecting to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() << ":" - << server.config.addr.get_port() << ") for query " << (query_info ? query_info->to_str() : "[none]"); - server.client = ton::adnl::AdnlExtClient::create(server.config.adnl_id, server.config.addr, + LOG(INFO) << "Connecting to liteserver #" << server.idx << " (" << server.config.hostname << ") for query " + << (query_info ? query_info->to_str() : "[none]"); + server.client = ton::adnl::AdnlExtClient::create(server.config.adnl_id, server.config.hostname, std::make_unique(actor_id(this), server_idx)); } @@ -204,8 +204,7 @@ class ExtClientImpl : public ExtClient { continue; } if (server.timeout.is_in_past()) { - LOG(INFO) << "Closing connection to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() - << ":" << server.config.addr.get_port() << ")"; + LOG(INFO) << "Closing connection to liteserver #" << server.idx << " (" << server.config.hostname << ")"; server.client.reset(); server.alive = false; server.ignore_until = {}; diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index b4cc2709da..9e260dea09 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -108,7 +108,7 @@ void TestNode::run() { if (single_liteserver_idx_ != -1) { // Use single liteserver from config CHECK(single_liteserver_idx_ >= 0 && (size_t)single_liteserver_idx_ < servers.size()); td::TerminalIO::out() << "using liteserver #" << single_liteserver_idx_ << " with addr " - << servers[single_liteserver_idx_].addr << "\n"; + << servers[single_liteserver_idx_].hostname << "\n"; servers = {servers[single_liteserver_idx_]}; } } diff --git a/lite-client/query-utils.cpp b/lite-client/query-utils.cpp index b46d46a5c1..7c9e7cfc73 100644 --- a/lite-client/query-utils.cpp +++ b/lite-client/query-utils.cpp @@ -318,16 +318,25 @@ bool LiteServerConfig::Slice::accepts_query(const QueryInfo& query_info) const { td::Result> LiteServerConfig::parse_global_config( const ton_api::liteclient_config_global& config) { std::vector servers; + auto get_hostname = [](const auto& f) -> std::string { + if (f->hostname_.empty()) { + return PSTRING() << td::IPAddress::ipv4_to_str(f->ip_) << ":" << f->port_; + } + if (f->port_ == 0) { + return f->hostname_; + } + return PSTRING() << f->hostname_ << ":" << f->port_; + }; for (const auto& f : config.liteservers_) { LiteServerConfig server; - TRY_STATUS(server.addr.init_host_port(td::IPAddress::ipv4_to_str(f->ip_), f->port_)); + server.hostname = get_hostname(f); server.adnl_id = adnl::AdnlNodeIdFull{PublicKey{f->id_}}; server.is_full = true; servers.push_back(std::move(server)); } for (const auto& f : config.liteservers_v2_) { LiteServerConfig server; - TRY_STATUS(server.addr.init_host_port(td::IPAddress::ipv4_to_str(f->ip_), f->port_)); + server.hostname = get_hostname(f); server.adnl_id = adnl::AdnlNodeIdFull{PublicKey{f->id_}}; server.is_full = false; for (const auto& slice_obj : f->slices_) { diff --git a/lite-client/query-utils.hpp b/lite-client/query-utils.hpp index 28500e2664..d214246951 100644 --- a/lite-client/query-utils.hpp +++ b/lite-client/query-utils.hpp @@ -73,11 +73,14 @@ struct LiteServerConfig { public: ton::adnl::AdnlNodeIdFull adnl_id; - td::IPAddress addr; + std::string hostname; LiteServerConfig() = default; - LiteServerConfig(ton::adnl::AdnlNodeIdFull adnl_id, td::IPAddress addr) - : is_full(true), adnl_id(adnl_id), addr(addr) { + LiteServerConfig(ton::adnl::AdnlNodeIdFull adnl_id, std::string hostname) + : is_full(true), adnl_id(adnl_id), hostname(std::move(hostname)) { + } + LiteServerConfig(ton::adnl::AdnlNodeIdFull adnl_id, td::IPAddress ip) + : is_full(true), adnl_id(adnl_id), hostname(PSTRING() << ip.get_ip_str() << ":" << ip.get_port()) { } bool accepts_query(const QueryInfo& query_info) const; diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 38dc33489b..d89000b2d7 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -657,8 +657,8 @@ liteserver.descV2.sliceSimple shards:(vector tonNode.shardId) = liteserver.descV liteserver.descV2.shardInfo shard_id:tonNode.shardId seqno:int utime:int lt:long = liteserver.descV2.ShardInfo; liteserver.descV2.sliceTimed shards_from:(vector liteserver.descV2.shardInfo) shards_to:(vector liteserver.descV2.shardInfo) = liteserver.descV2.Slice; -liteserver.desc id:PublicKey ip:int port:int = liteserver.Desc; -liteserver.descV2 id:PublicKey ip:int port:int slices:(vector liteserver.descV2.Slice) = liteserver.DescV2; +liteserver.desc id:PublicKey ip:int port:int hostname:string = liteserver.Desc; +liteserver.descV2 id:PublicKey ip:int port:int hostname:string slices:(vector liteserver.descV2.Slice) = liteserver.DescV2; liteclient.config.global liteservers:(vector liteserver.desc) liteservers_v2:(vector liteserver.descV2) validator:validator.config.global = liteclient.config.Global; engine.adnl id:int256 category:int = engine.Adnl; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 3b64f7517b1ad5fa9ef9bf964ec3917078c2d098..80ba81372a715addfc20adb7639f91fea625f8b4 100644 GIT binary patch delta 177 zcmbQSnEkADhdgxo004nAN09&k delta 130 zcmaE`n0>}#_6-UuEG0`kk4#ok(O~J8;rHGgpklHcBGhr;0xUH7$bA7H1k(=i{ Hbol@P)c!H} diff --git a/utils/proxy-liteserver.cpp b/utils/proxy-liteserver.cpp index a9baa75956..e89cb1bbf9 100644 --- a/utils/proxy-liteserver.cpp +++ b/utils/proxy-liteserver.cpp @@ -184,7 +184,7 @@ class ProxyLiteserver : public td::actor::Actor { for (size_t i = 0; i < servers_.size(); ++i) { Server& server = servers_[i]; - server.client = adnl::AdnlExtClient::create(server.config.adnl_id, server.config.addr, + server.client = adnl::AdnlExtClient::create(server.config.adnl_id, server.config.hostname, std::make_unique(actor_id(this), i)); server.alive = false; } @@ -197,7 +197,7 @@ class ProxyLiteserver : public td::actor::Actor { } server.alive = ready; LOG(WARNING) << (ready ? "Connected to" : "Disconnected from") << " server #" << idx << " (" - << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; + << server.config.hostname << ")"; } void create_ext_server() { @@ -295,8 +295,7 @@ class ProxyLiteserver : public td::actor::Actor { Server& server = servers_[server_idx]; LOG(INFO) << "Sending query " << query_info.to_str() << (wait_mc_seqno_obj ? PSTRING() << " (wait seqno " << wait_mc_seqno_obj->seqno_ << ")" : "") - << ", size=" << data.size() << ", to server #" << server_idx << " (" << server.config.addr.get_ip_str() - << ":" << server.config.addr.get_port() << ")"; + << ", size=" << data.size() << ", to server #" << server_idx << " (" << server.config.hostname << ")"; BlockSeqno wait_mc_seqno = wait_mc_seqno_obj ? wait_mc_seqno_obj->seqno_ : 0; wait_mc_seqno = std::max(wait_mc_seqno, last_known_masterchain_seqno_); From 501790f3518887fff86991dbf893f82151a1fc3d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 7 Aug 2025 15:37:21 +0300 Subject: [PATCH 370/524] Optimistic collation on collator nodes --- tl/generate/scheme/ton_api.tl | 5 +- tl/generate/scheme/ton_api.tlo | Bin 119264 -> 119800 bytes ton/ton-types.h | 2 + validator-session/validator-session-types.h | 4 +- validator-session/validator-session.cpp | 10 +- validator/CMakeLists.txt | 2 + validator/collation-manager.cpp | 162 ++++++++++++++---- validator/collation-manager.hpp | 28 ++- .../collator-node/collator-node-session.cpp | 95 +++++++++- .../collator-node/collator-node-session.hpp | 26 ++- validator/collator-node/collator-node.cpp | 159 +++++++---------- validator/collator-node/collator-node.hpp | 12 +- validator/collator-node/utils.cpp | 93 ++++++++++ validator/collator-node/utils.hpp | 27 +++ validator/fabric.h | 2 +- validator/impl/collator-impl.h | 2 +- validator/impl/collator.cpp | 17 +- validator/impl/validate-query.cpp | 4 +- validator/manager-hardfork.hpp | 5 +- validator/manager.cpp | 2 +- validator/shard-block-verifier.cpp | 2 +- validator/validator-group.cpp | 2 +- 22 files changed, 480 insertions(+), 181 deletions(-) create mode 100644 validator/collator-node/utils.cpp create mode 100644 validator/collator-node/utils.hpp diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index d89000b2d7..3f07579933 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -1008,7 +1008,7 @@ validatorStats.collatorNodeResponse self:int256 validator_id:int256 timestamp:do collatorNode.candidate source:PublicKey id:tonNode.blockIdExt data:bytes collated_data:bytes = collatorNode.Candidate; collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt decompressed_size:int data:bytes = collatorNode.Candidate; collatorNode.compressedCandidateV2 flags:# source:PublicKey id:tonNode.blockIdExt data:bytes = collatorNode.Candidate; -collatorNode.pong flags:# = collatorNode.Pong; +collatorNode.pong#5bbf0521 flags:# version:flags.0?int = collatorNode.Pong; collatorNode.error code:int message:string = collatorNode.Error; shardBlockVerifier.subscribed flags:# = shardBlockVerifier.Subscribed; @@ -1017,6 +1017,9 @@ shardBlockVerifier.confirmBlocks blocks:(vector tonNode.blockIdExt) = shardBlock ---functions--- collatorNode.generateBlock shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt) creator:int256 round:int first_block_round:int priority:int = collatorNode.Candidate; +collatorNode.generateBlockOptimistic shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt) + creator:int256 round:int first_block_round:int priority:int = collatorNode.Candidate; +collatorNode.requestBlockCallback flags:# block_id:tonNode.BlockIdExt = collatorNode.Candidate; collatorNode.ping flags:# = collatorNode.Pong; shardBlockVerifier.subscribe shard:tonNode.shardId flags:# = shardBlockVerifier.Subscribed; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 80ba81372a715addfc20adb7639f91fea625f8b4..ce74dbb37275ffa2745966d24b2e1a77a6d32679 100644 GIT binary patch delta 166 zcmaE`nEl6c_6-I#j7*amtF$M-`N1(=MUYWovX7TYb3)Dbgc`;djEsx6d)6_QGifVK z(fp{AoS&1ESdw4lm!FcVm!6uJT9jCl>Xeh8ob6vwl9`)XT#}j0z_7i)o^c|h;iCN= zR}@in6r~oHrWTigH8>~c prev; td::uint32 self_idx = 0; PublicKeyHash self = PublicKeyHash::zero(); - std::vector nodes; + std::vector nodes{}; tl_object_ptr tl() const { std::vector> prev_arr; @@ -257,7 +257,7 @@ struct EndValidatorGroupStats { ValidatorSessionId session_id = ValidatorSessionId::zero(); double timestamp = -1.0; PublicKeyHash self = PublicKeyHash::zero(); - std::vector nodes; + std::vector nodes{}; tl_object_ptr tl() const { std::vector> nodes_arr; diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 40ecf1b2ed..4a06314077 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -336,7 +336,7 @@ void ValidatorSessionImpl::process_received_block(td::uint32 block_round, Public } } stat->deserialize_time = info.deserialize_time; - stat->serialized_size = info.serialized_size; + stat->serialized_size = (int)info.serialized_size; stat->block_id.root_hash = candidate->root_hash_; stat->block_id.file_hash = info.file_hash; stat->collated_data_hash = info.collated_data_hash; @@ -473,7 +473,7 @@ void ValidatorSessionImpl::candidate_decision_ok(td::uint32 round, ValidatorSess stat->block_status = ValidatorSessionStats::status_approved; stat->comment = PSTRING() << "ts=" << ok_from; stat->validation_time = validation_time; - stat->gen_utime = (double)ok_from; + stat->gen_utime = (int)ok_from; stat->validated_at = td::Clocks::system(); stat->validation_cached = validation_cached; } @@ -528,7 +528,11 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, GeneratedCandidate stat->block_status = ValidatorSessionStats::status_received; stat->collation_time = collation_time; stat->collated_at = td::Clocks::system(); - stat->got_block_at = td::Clocks::system(); + if (auto it = sent_candidates_.find(block_id); it != sent_candidates_.end()) { + stat->got_block_at = it->second.sent_at; + } else { + stat->got_block_at = td::Clocks::system(); + } stat->got_block_by = ValidatorSessionStats::recv_collated; stat->collation_cached = c.is_cached; stat->self_collated = c.self_collated; diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 98987c78a6..e33010fe14 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -66,6 +66,7 @@ set(VALIDATOR_HEADERS collation-manager.hpp collator-node/collator-node.hpp collator-node/collator-node-session.hpp + collator-node/utils.hpp manager-disk.h manager-disk.hpp manager-init.h @@ -84,6 +85,7 @@ set(VALIDATOR_SOURCE collation-manager.cpp collator-node/collator-node.cpp collator-node/collator-node-session.cpp + collator-node/utils.cpp get-next-key-blocks.cpp import-db-slice.cpp import-db-slice-local.cpp diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index 9260dfb50b..e842b76d46 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -17,6 +17,7 @@ #include "collation-manager.hpp" #include "collator-node/collator-node.hpp" +#include "collator-node/utils.hpp" #include "fabric.h" #include "td/utils/Random.h" @@ -29,6 +30,29 @@ namespace ton::validator { void CollationManager::start_up() { td::actor::send_closure(rldp_, &rldp2::Rldp::add_id, local_id_); update_collators_list(*opts_->get_collators_list()); + + class Cb : public adnl::Adnl::Callback { + public: + explicit Cb(td::actor::ActorId id) : id_(std::move(id)) { + } + void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(id_, &CollationManager::receive_query, src, std::move(data), std::move(promise)); + } + + private: + td::actor::ActorId id_; + }; + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_requestBlockCallback::ID), + std::make_unique(actor_id(this))); +} + +void CollationManager::tear_down() { + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_requestBlockCallback::ID)); } void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, @@ -54,25 +78,47 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha proto_version); } -void CollationManager::collate_next_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, - BlockIdExt prev_block_id, td::BufferSlice prev_block, - Ed25519_PublicKey creator, BlockCandidatePriority priority, - td::Ref validator_set, td::uint64 max_answer_size, - td::CancellationToken cancellation_token, - td::Promise promise, int proto_version) { - TRY_RESULT_PROMISE(promise, prev_block_data, create_block(prev_block_id, std::move(prev_block))); - run_collate_query( - CollateParams{.shard = shard, - .min_masterchain_block_id = min_masterchain_block_id, - .prev = {prev_block_id}, - .creator = creator, - .validator_set = std::move(validator_set), - .collator_opts = opts_->get_collator_options(), - .optimistic_prev_block_ = std::move(prev_block_data)}, - manager_, td::Timestamp::in(10.0), std::move(cancellation_token), promise.wrap([](BlockCandidate&& candidate) { - return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; - })); - // TODO: request to collator node +void CollationManager::collate_block_optimistic(ShardIdFull shard, BlockIdExt min_masterchain_block_id, + BlockIdExt prev_block_id, td::BufferSlice prev_block, + Ed25519_PublicKey creator, BlockCandidatePriority priority, + td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, + td::Promise promise, int proto_version) { + if (shard.is_masterchain()) { + TRY_RESULT_PROMISE(promise, prev_block_data, create_block(prev_block_id, std::move(prev_block))); + run_collate_query( + CollateParams{.shard = shard, + .min_masterchain_block_id = min_masterchain_block_id, + .prev = {prev_block_id}, + .creator = creator, + .validator_set = std::move(validator_set), + .collator_opts = opts_->get_collator_options(), + .optimistic_prev_block = std::move(prev_block_data)}, + manager_, td::Timestamp::in(10.0), std::move(cancellation_token), promise.wrap([](BlockCandidate&& candidate) { + return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; + })); + return; + } + + auto& entry = optimistic_prev_cache_[prev_block_id]; + entry.block_data = std::move(prev_block); + ++entry.refcnt; + promise = [this, SelfId = actor_id(this), prev_block_id, + promise = std::move(promise)](td::Result R) mutable { + promise.set_result(std::move(R)); + td::actor::send_lambda_later(SelfId, [=, this]() { + auto it = optimistic_prev_cache_.find(prev_block_id); + CHECK(it != optimistic_prev_cache_.end()); + CHECK(it->second.refcnt > 0); + if (--it->second.refcnt == 0) { + optimistic_prev_cache_.erase(it); + } + }); + }; + + collate_shard_block(shard, min_masterchain_block_id, {prev_block_id}, creator, priority, std::move(validator_set), + max_answer_size, std::move(cancellation_token), std::move(promise), td::Timestamp::in(10.0), + proto_version, true); } void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, @@ -80,7 +126,7 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise, td::Timestamp timeout, - int proto_version) { + int proto_version, bool is_optimistic) { TRY_STATUS_PROMISE(promise, cancellation_token.check()); ShardInfo* s = select_shard_info(shard); if (s == nullptr) { @@ -91,12 +137,22 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas adnl::AdnlNodeIdShort selected_collator = adnl::AdnlNodeIdShort::zero(); size_t selected_idx = 0; + auto check_collator = [&](const adnl::AdnlNodeIdShort& id) -> bool { + auto& collator = collators_[id]; + if (!collator.alive) { + return false; + } + if (is_optimistic && collator.version < CollatorNode::VERSION_OPTIMISTIC_COLLATE) { + return false; + } + return true; + }; switch (s->select_mode) { case CollatorsList::mode_random: { int cnt = 0; for (size_t i = 0; i < s->collators.size(); ++i) { adnl::AdnlNodeIdShort collator = s->collators[i]; - if (collators_[collator].alive) { + if (check_collator(collator)) { ++cnt; if (td::Random::fast(1, cnt) == 1) { selected_collator = collator; @@ -109,7 +165,7 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas case CollatorsList::mode_ordered: { for (size_t i = 0; i < s->collators.size(); ++i) { adnl::AdnlNodeIdShort collator = s->collators[i]; - if (collators_[collator].alive) { + if (check_collator(collator)) { selected_collator = collator; selected_idx = i; break; @@ -121,7 +177,7 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas size_t iters = 0; for (size_t i = s->cur_idx; iters < s->collators.size(); (++i) %= s->collators.size(), ++iters) { adnl::AdnlNodeIdShort& collator = s->collators[i]; - if (collators_[collator].alive) { + if (check_collator(collator)) { selected_collator = collator; selected_idx = i; s->cur_idx = (i + 1) % s->collators.size(); @@ -133,13 +189,20 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas } if (selected_collator.is_zero() && s->self_collate) { + td::Ref optimistic_prev_block; + if (is_optimistic) { + CHECK(prev.size() == 1); + TRY_RESULT_PROMISE_ASSIGN(promise, optimistic_prev_block, + create_block(prev[0], optimistic_prev_cache_.at(prev[0]).block_data.clone())); + } run_collate_query( CollateParams{.shard = shard, .min_masterchain_block_id = min_masterchain_block_id, .prev = std::move(prev), .creator = creator, .validator_set = std::move(validator_set), - .collator_opts = opts_->get_collator_options()}, + .collator_opts = opts_->get_collator_options(), + .optimistic_prev_block = std::move(optimistic_prev_block)}, manager_, td::Timestamp::in(10.0), std::move(cancellation_token), promise.wrap([](BlockCandidate&& candidate) { return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; })); @@ -175,19 +238,26 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas [=, promise = std::move(promise)]() mutable { td::actor::send_closure(SelfId, &CollationManager::collate_shard_block, shard, min_masterchain_block_id, prev, creator, priority, validator_set, max_answer_size, cancellation_token, - std::move(promise), timeout, proto_version); + std::move(promise), timeout, proto_version, is_optimistic); }, retry_at); }; if (selected_collator.is_zero()) { - P.set_error(td::Status::Error(PSTRING() << "shard " << shard.to_str() << " has no alive collator node")); + P.set_error(td::Status::Error(PSTRING() << "shard " << shard.to_str() << " has no suitable collator node")); return; } - td::BufferSlice query = create_serialize_tl_object( - create_tl_shard_id(shard), validator_set->get_catchain_seqno(), std::move(prev_blocks), creator.as_bits256(), - priority.round, priority.first_block_round, priority.priority); + td::BufferSlice query; + if (is_optimistic) { + query = create_serialize_tl_object( + create_tl_shard_id(shard), validator_set->get_catchain_seqno(), std::move(prev_blocks), creator.as_bits256(), + priority.round, priority.first_block_round, priority.priority); + } else { + query = create_serialize_tl_object( + create_tl_shard_id(shard), validator_set->get_catchain_seqno(), std::move(prev_blocks), creator.as_bits256(), + priority.round, priority.first_block_round, priority.priority); + } LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to #" << selected_idx << "(" << selected_collator << ")"; @@ -201,9 +271,8 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas return; } TRY_RESULT_PROMISE(P, f, fetch_tl_object(data, true)); - TRY_RESULT_PROMISE( - P, candidate, - CollatorNode::deserialize_candidate(std::move(f), td::narrow_cast(max_answer_size), proto_version)); + TRY_RESULT_PROMISE(P, candidate, + deserialize_candidate(std::move(f), td::narrow_cast(max_answer_size), proto_version)); if (candidate.pubkey.as_bits256() != creator.as_bits256()) { P.set_error(td::Status::Error("collate query: block candidate source mismatch")); return; @@ -363,7 +432,8 @@ void CollationManager::alarm() { } if (collator.ping_at.is_in_past()) { collator.sent_ping = true; - td::BufferSlice query = create_serialize_tl_object(0); + td::BufferSlice query = + create_serialize_tl_object(ton_api::collatorNode_pong::VERSION_MASK); td::Promise P = [=, id = id, SelfId = actor_id(this)](td::Result R) mutable { td::actor::send_closure(SelfId, &CollationManager::got_pong, id, std::move(R)); }; @@ -399,9 +469,11 @@ void CollationManager::got_pong(adnl::AdnlNodeIdShort id, td::Resultflags_ & ton_api::collatorNode_pong::VERSION_MASK ? pong->version_ : 0; + LOG(DEBUG) << "pong from " << id << " : OK, version=" << collator.version; } collator.ping_at = td::Timestamp::in(td::Random::fast(10.0, 20.0)); if (collator.active_cnt && !collator.sent_ping) { @@ -421,4 +493,26 @@ void CollationManager::on_collate_query_error(adnl::AdnlNodeIdShort id) { } } +void CollationManager::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, + td::Promise promise) { + if (!collators_.contains(src)) { + promise.set_error(td::Status::Error("got request from unknown collator")); + return; + } + TRY_RESULT_PROMISE(promise, query, fetch_tl_object(data, true)); + BlockIdExt block_id = create_block_id(query->block_id_); + auto it = optimistic_prev_cache_.find(block_id); + if (it == optimistic_prev_cache_.end()) { + LOG(INFO) << "collatorNode.requestBlockCallback from " << src << " block " << block_id.to_str() << " : not found"; + promise.set_error(td::Status::Error("block not found")); + return; + } + LOG(INFO) << "collatorNode.requestBlockCallback from " << src << " block " << block_id.to_str() << " : OK"; + promise.set_value( + serialize_tl_object(serialize_candidate(BlockCandidate(Ed25519_PublicKey{td::Bits256::zero()}, block_id, + td::Bits256::zero(), it->second.block_data.clone(), {}), + true), + true)); +} + } // namespace ton::validator diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp index a64d432892..37a45fb38a 100644 --- a/validator/collation-manager.hpp +++ b/validator/collation-manager.hpp @@ -27,11 +27,13 @@ class ValidatorManager; class CollationManager : public td::actor::Actor { public: CollationManager(adnl::AdnlNodeIdShort local_id, td::Ref opts, - td::actor::ActorId manager, td::actor::ActorId rldp) - : local_id_(local_id), opts_(opts), manager_(manager), rldp_(rldp) { + td::actor::ActorId manager, td::actor::ActorId adnl, + td::actor::ActorId rldp) + : local_id_(local_id), opts_(opts), manager_(manager), adnl_(adnl), rldp_(rldp) { } void start_up() override; + void tear_down() override; void alarm() override; void collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, @@ -39,11 +41,11 @@ class CollationManager : public td::actor::Actor { td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise, int proto_version); - void collate_next_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, BlockIdExt prev_block_id, - td::BufferSlice prev_block, Ed25519_PublicKey creator, BlockCandidatePriority priority, - td::Ref validator_set, td::uint64 max_answer_size, - td::CancellationToken cancellation_token, td::Promise promise, - int proto_version); + void collate_block_optimistic(ShardIdFull shard, BlockIdExt min_masterchain_block_id, BlockIdExt prev_block_id, + td::BufferSlice prev_block, Ed25519_PublicKey creator, BlockCandidatePriority priority, + td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, td::Promise promise, + int proto_version); void update_options(td::Ref opts); @@ -56,13 +58,14 @@ class CollationManager : public td::actor::Actor { adnl::AdnlNodeIdShort local_id_; td::Ref opts_; td::actor::ActorId manager_; + td::actor::ActorId adnl_; td::actor::ActorId rldp_; void collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise, - td::Timestamp timeout, int proto_version); + td::Timestamp timeout, int proto_version, bool is_optimistic = false); void update_collators_list(const CollatorsList& collators_list); @@ -73,6 +76,7 @@ class CollationManager : public td::actor::Actor { size_t active_cnt = 0; td::Timestamp last_ping_at = td::Timestamp::never(); td::Status last_ping_status = td::Status::Error("not pinged"); + int version = -1; }; std::map collators_; @@ -92,6 +96,14 @@ class CollationManager : public td::actor::Actor { ShardInfo* select_shard_info(ShardIdFull shard); void got_pong(adnl::AdnlNodeIdShort id, td::Result R); void on_collate_query_error(adnl::AdnlNodeIdShort id); + + void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); + + struct OptimisticPrevCache { + td::BufferSlice block_data; + size_t refcnt = 0; + }; + std::map optimistic_prev_cache_; }; } // namespace ton::validator diff --git a/validator/collator-node/collator-node-session.cpp b/validator/collator-node/collator-node-session.cpp index 76c6f99741..8e131f3c0c 100644 --- a/validator/collator-node/collator-node-session.cpp +++ b/validator/collator-node/collator-node-session.cpp @@ -17,7 +17,9 @@ #include "collator-node-session.hpp" +#include "collator-node.hpp" #include "fabric.h" +#include "utils.hpp" namespace ton::validator { @@ -31,7 +33,7 @@ static BlockSeqno get_next_block_seqno(const std::vector& prev) { CollatorNodeSession::CollatorNodeSession(ShardIdFull shard, std::vector prev, td::Ref validator_set, BlockIdExt min_masterchain_block_id, - bool can_generate, adnl::AdnlNodeIdShort local_id, + bool can_generate, Ref state, adnl::AdnlNodeIdShort local_id, td::Ref opts, td::actor::ActorId manager, td::actor::ActorId adnl, td::actor::ActorId rldp) @@ -46,6 +48,7 @@ CollatorNodeSession::CollatorNodeSession(ShardIdFull shard, std::vectorget_catchain_seqno() << ", next block seqno " << next_block_seqno_; if (can_generate_) { - generate_block(prev_, {}, td::Timestamp::in(10.0), [](td::Result) {}); + generate_block(prev_, {}, {}, td::Timestamp::in(10.0), [](td::Result) {}); } } @@ -101,14 +104,22 @@ void CollatorNodeSession::new_shard_block_accepted(BlockIdExt block_id, bool can } if (can_generate_) { - generate_block(prev_, {}, td::Timestamp::in(10.0), [](td::Result) {}); + generate_block(prev_, {}, {}, td::Timestamp::in(10.0), [](td::Result) {}); } } +void CollatorNodeSession::update_masterchain_config(td::Ref state) { + ValidatorSessionConfig config = state->get_consensus_config(); + proto_version_ = config.proto_version; + max_candidate_size_ = config.max_block_size + config.max_collated_data_size + 1024; +} + void CollatorNodeSession::generate_block(std::vector prev_blocks, - td::optional o_priority, td::Timestamp timeout, + td::optional o_priority, + td::Ref o_optimistic_prev_block, td::Timestamp timeout, td::Promise promise) { bool is_external = !o_priority; + bool is_optimistic = o_optimistic_prev_block.not_null(); BlockSeqno block_seqno = get_next_block_seqno(prev_blocks); if (next_block_seqno_ > block_seqno) { promise.set_error(td::Status::Error(PSTRING() << "next block seqno " << block_seqno << " is too old, expected " @@ -126,7 +137,8 @@ void CollatorNodeSession::generate_block(std::vector prev_blocks, } static auto prefix_inner = [](td::StringBuilder& sb, const ShardIdFull& shard, CatchainSeqno cc_seqno, - BlockSeqno block_seqno, const td::optional& o_priority) { + BlockSeqno block_seqno, const td::optional& o_priority, + bool is_optimistic) { sb << "generate block query" << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno; if (o_priority) { @@ -138,9 +150,12 @@ void CollatorNodeSession::generate_block(std::vector prev_blocks, } else { sb << " internal"; } + if (is_optimistic) { + sb << " opt"; + } }; auto prefix = [&](td::StringBuilder& sb) { - prefix_inner(sb, shard_, validator_set_->get_catchain_seqno(), block_seqno, o_priority); + prefix_inner(sb, shard_, validator_set_->get_catchain_seqno(), block_seqno, o_priority, is_optimistic); }; auto cache_entry = cache_[prev_blocks]; @@ -198,13 +213,14 @@ void CollatorNodeSession::generate_block(std::vector prev_blocks, .validator_set = validator_set_, .collator_opts = opts_->get_collator_options(), .collator_node_id = local_id_, - .skip_store_candidate = true}, + .skip_store_candidate = true, + .optimistic_prev_block = o_optimistic_prev_block}, manager_, timeout, cache_entry->cancellation_token_source.get_cancellation_token(), [=, shard = shard_, cc_seqno = validator_set_->get_catchain_seqno(), SelfId = actor_id(this), timer = td::Timer{}](td::Result R) mutable { FLOG(INFO) { - prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); - sb << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); + prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority, is_optimistic); + sb << ": " << (R.is_ok() ? "OK" : R.error().to_string()) << " time=" << timer.elapsed(); }; td::actor::send_closure(SelfId, &CollatorNodeSession::process_result, cache_entry, std::move(R)); }); @@ -226,6 +242,67 @@ void CollatorNodeSession::process_result(std::shared_ptr cache_entry cache_entry->promises.clear(); } +void CollatorNodeSession::process_request(adnl::AdnlNodeIdShort src, std::vector prev_blocks, + BlockCandidatePriority priority, bool is_optimistic, td::Timestamp timeout, + td::Promise promise) { + if (is_optimistic) { + if (prev_blocks.size() != 1) { + promise.set_error(td::Status::Error("optimistic collation, expected 1 prev block")); + return; + } + auto it = cache_.find(prev_blocks); + if (it == cache_.end() || it->second->started) { + BlockIdExt prev_block = prev_blocks[0]; + td::actor::send_closure( + manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, prev_block, + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + td::actor::send_closure(SelfId, &CollatorNodeSession::process_request_optimistic_cont, src, prev_block, + priority, timeout, std::move(promise), std::move(R)); + }); + return; + } + } + generate_block(std::move(prev_blocks), priority, {}, timeout, std::move(promise)); +} + +void CollatorNodeSession::process_request_optimistic_cont(adnl::AdnlNodeIdShort src, BlockIdExt prev_block_id, + BlockCandidatePriority priority, td::Timestamp timeout, + td::Promise promise, + td::Result prev_block_data) { + if (prev_block_data.is_ok()) { + TRY_RESULT_PROMISE_PREFIX(promise, prev_block, create_block(prev_block_id, prev_block_data.move_as_ok()), + "invalid prev block data in db: "); + LOG(INFO) << "got prev block from db for optimistic collation: " << prev_block_id.to_str(); + generate_block({prev_block_id}, priority, prev_block, timeout, std::move(promise)); + return; + } + td::actor::send_closure( + rldp_, &rldp2::Rldp::send_query_ex, local_id_, src, "getprevblock", + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + td::actor::send_closure(SelfId, &CollatorNodeSession::process_request_optimistic_cont2, prev_block_id, priority, + timeout, std::move(promise), std::move(R)); + }, + timeout, + create_serialize_tl_object(0, create_tl_block_id(prev_block_id)), + max_candidate_size_); +} + +void CollatorNodeSession::process_request_optimistic_cont2(BlockIdExt prev_block_id, BlockCandidatePriority priority, + td::Timestamp timeout, td::Promise promise, + td::Result R) { + TRY_RESULT_PROMISE_PREFIX(promise, response, std::move(R), + "failed to download prev block data for optimistic collation: "); + TRY_RESULT_PROMISE_PREFIX(promise, f, fetch_tl_object(response, true), + "failed to download prev block data for optimistic collation: "); + TRY_RESULT_PROMISE_PREFIX(promise, candidate, + deserialize_candidate(std::move(f), max_candidate_size_, proto_version_), + "failed to download prev block data for optimistic collation: "); + TRY_RESULT_PROMISE_PREFIX(promise, prev_block, create_block(prev_block_id, std::move(candidate.data)), + "invalid prev block data from validator: "); + LOG(INFO) << "got prev block from validator for optimistic collation: " << prev_block_id.to_str(); + generate_block({prev_block_id}, priority, prev_block, timeout, std::move(promise)); +} + void CollatorNodeSession::CacheEntry::cancel(td::Status reason) { for (auto& promise : promises) { promise.set_error(reason.clone()); diff --git a/validator/collator-node/collator-node-session.hpp b/validator/collator-node/collator-node-session.hpp index 51e1b7cf6c..0f9f03d30f 100644 --- a/validator/collator-node/collator-node-session.hpp +++ b/validator/collator-node/collator-node-session.hpp @@ -29,9 +29,10 @@ class ValidatorManager; class CollatorNodeSession : public td::actor::Actor { public: CollatorNodeSession(ShardIdFull shard, std::vector prev, td::Ref validator_set, - BlockIdExt min_masterchain_block_id, bool can_generate, adnl::AdnlNodeIdShort local_id, - td::Ref opts, td::actor::ActorId manager, - td::actor::ActorId adnl, td::actor::ActorId rldp); + BlockIdExt min_masterchain_block_id, bool can_generate, td::Ref state, + adnl::AdnlNodeIdShort local_id, td::Ref opts, + td::actor::ActorId manager, td::actor::ActorId adnl, + td::actor::ActorId rldp); void start_up() override; void tear_down() override; @@ -42,8 +43,9 @@ class CollatorNodeSession : public td::actor::Actor { void new_shard_block_accepted(BlockIdExt block_id, bool can_generate); - void generate_block(std::vector prev_blocks, td::optional o_priority, - td::Timestamp timeout, td::Promise promise); + void process_request(adnl::AdnlNodeIdShort src, std::vector prev_blocks, BlockCandidatePriority priority, + bool is_optimistic, td::Timestamp timeout, td::Promise promise); + void update_masterchain_config(td::Ref state); private: ShardIdFull shard_; @@ -73,7 +75,21 @@ class CollatorNodeSession : public td::actor::Actor { BlockSeqno next_block_seqno_; std::map, std::shared_ptr> cache_; + td::uint32 proto_version_ = 0; + td::uint32 max_candidate_size_ = 0; + + void generate_block(std::vector prev_blocks, td::optional o_priority, + td::Ref o_optimistic_prev_block, td::Timestamp timeout, + td::Promise promise); void process_result(std::shared_ptr cache_entry, td::Result R); + + void process_request_optimistic_cont(adnl::AdnlNodeIdShort src, BlockIdExt prev_block_id, + BlockCandidatePriority priority, td::Timestamp timeout, + td::Promise promise, + td::Result prev_block_data); + void process_request_optimistic_cont2(BlockIdExt prev_block_id, BlockCandidatePriority priority, + td::Timestamp timeout, td::Promise promise, + td::Result R); }; } // namespace ton::validator diff --git a/validator/collator-node/collator-node.cpp b/validator/collator-node/collator-node.cpp index 6f0cae4191..dd0dfb7927 100644 --- a/validator/collator-node/collator-node.cpp +++ b/validator/collator-node/collator-node.cpp @@ -23,7 +23,7 @@ #include "checksum.h" #include "impl/collator-impl.h" #include "impl/shard.hpp" -#include "validator-session/candidate-serializer.h" +#include "utils.hpp" namespace ton::validator { @@ -55,6 +55,9 @@ void CollatorNode::start_up() { td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlock::ID), std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlockOptimistic::ID), + std::make_unique(actor_id(this))); td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, adnl::Adnl::int_to_bytestring(ton_api::collatorNode_ping::ID), std::make_unique(actor_id(this))); @@ -64,6 +67,8 @@ void CollatorNode::start_up() { void CollatorNode::tear_down() { td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlock::ID)); + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlockOptimistic::ID)); td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, adnl::Adnl::int_to_bytestring(ton_api::collatorNode_ping::ID)); } @@ -83,7 +88,7 @@ void CollatorNode::add_shard(ShardIdFull shard) { validator_group.actor = td::actor::create_actor( PSTRING() << "collatornode" << shard.to_str(), shard, validator_group.prev, last_masterchain_state_->get_validator_set(group_shard), last_masterchain_state_->get_block_id(), - can_generate(), local_id_, opts_, manager_, adnl_, rldp_); + can_generate(), last_masterchain_state_, local_id_, opts_, manager_, adnl_, rldp_); } } } @@ -118,9 +123,7 @@ void CollatorNode::new_masterchain_block_notification(td::Ref if (mc_config_status_.is_error()) { LOG(ERROR) << "Cannot validate masterchain config (possibly outdated software): " << mc_config_status_; } - } - if (validator_adnl_ids_.empty() || state->is_key_state()) { validator_adnl_ids_.clear(); for (int next : {-1, 0, 1}) { td::Ref vals = state->get_total_validator_set(next); @@ -135,6 +138,11 @@ void CollatorNode::new_masterchain_block_notification(td::Ref } } } + for (auto& [_, group] : validator_groups_) { + if (!group.actor.empty()) { + td::actor::send_closure(group.actor, &CollatorNodeSession::update_masterchain_config, state); + } + } } std::map> new_shards; @@ -173,7 +181,8 @@ void CollatorNode::new_masterchain_block_notification(td::Ref if (can_collate_shard(shard)) { it->second.actor = td::actor::create_actor( PSTRING() << "collatornode" << shard.to_str(), shard, it->second.prev, validator_set, - last_masterchain_state_->get_block_id(), can_generate(), local_id_, opts_, manager_, adnl_, rldp_); + last_masterchain_state_->get_block_id(), can_generate(), last_masterchain_state_, local_id_, opts_, + manager_, adnl_, rldp_); } } else if (!it->second.actor.empty() && prev.size() == 1) { td::actor::send_closure(it->second.actor, &CollatorNodeSession::new_shard_block_accepted, prev[0], @@ -250,11 +259,11 @@ td::Result CollatorNode::get_future_validat } if (cc_seqno < it->second.cc_seqno) { // past validator group return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " for shard " << shard.to_str() - << " is outdated (current is" << it->second.cc_seqno << ")"); + << " is outdated (current is " << it->second.cc_seqno << ")"); } if (cc_seqno - it->second.cc_seqno > 1) { // future validator group, cc_seqno too big return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " for shard " << shard.to_str() - << " is too big (currently known is" << it->second.cc_seqno << ")"); + << " is too big (currently known is " << it->second.cc_seqno << ")"); } // future validator group return &future_validator_groups_[{shard, cc_seqno}]; @@ -326,17 +335,39 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data return; } - TRY_RESULT_PROMISE(promise, f, fetch_tl_object(data, true)); - ShardIdFull shard = create_shard_id(f->shard_); - CatchainSeqno cc_seqno = f->cc_seqno_; + bool is_optimistic = false; + ShardIdFull shard; + CatchainSeqno cc_seqno; std::vector prev_blocks; - for (const auto& b : f->prev_blocks_) { - prev_blocks.push_back(create_block_id(b)); + BlockCandidatePriority priority; + Ed25519_PublicKey creator; + if (auto R = fetch_tl_object(data, true); R.is_ok()) { + auto f = R.move_as_ok(); + shard = create_shard_id(f->shard_); + cc_seqno = f->cc_seqno_; + for (const auto& b : f->prev_blocks_) { + prev_blocks.push_back(create_block_id(b)); + } + priority = BlockCandidatePriority{.round = static_cast(f->round_), + .first_block_round = static_cast(f->first_block_round_), + .priority = f->priority_}; + creator = Ed25519_PublicKey(f->creator_); + } else if (auto R = fetch_tl_object(data, true); R.is_ok()) { + is_optimistic = true; + auto f = R.move_as_ok(); + shard = create_shard_id(f->shard_); + cc_seqno = f->cc_seqno_; + for (const auto& b : f->prev_blocks_) { + prev_blocks.push_back(create_block_id(b)); + } + priority = BlockCandidatePriority{.round = static_cast(f->round_), + .first_block_round = static_cast(f->first_block_round_), + .priority = f->priority_}; + creator = Ed25519_PublicKey(f->creator_); + } else { + promise.set_error(td::Status::Error("cannot parse request")); + return; } - auto priority = BlockCandidatePriority{.round = static_cast(f->round_), - .first_block_round = static_cast(f->first_block_round_), - .priority = f->priority_}; - Ed25519_PublicKey creator(f->creator_); td::Promise new_promise = [promise = std::move(promise), src, shard](td::Result R) mutable { if (R.is_error()) { @@ -378,14 +409,16 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data new_promise.set_error(td::Status::Error(PSTRING() << "invalid size of prev_blocks: " << prev_blocks.size())); return; } - LOG(INFO) << "got adnl query from " << src << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno; - process_generate_block_query(shard, cc_seqno, std::move(prev_blocks), priority, td::Timestamp::in(10.0), - std::move(new_promise)); + LOG(INFO) << "got adnl query from " << src << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno + << (is_optimistic ? ", optimistic" : ""); + process_generate_block_query(src, shard, cc_seqno, std::move(prev_blocks), priority, is_optimistic, + td::Timestamp::in(10.0), std::move(new_promise)); } -void CollatorNode::process_generate_block_query(ShardIdFull shard, CatchainSeqno cc_seqno, +void CollatorNode::process_generate_block_query(adnl::AdnlNodeIdShort src, ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, BlockCandidatePriority priority, - td::Timestamp timeout, td::Promise promise) { + bool is_optimistic, td::Timestamp timeout, + td::Promise promise) { if (last_masterchain_state_.is_null()) { promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); return; @@ -400,8 +433,8 @@ void CollatorNode::process_generate_block_query(ShardIdFull shard, CatchainSeqno future_validator_group->promises.push_back([=, SelfId = actor_id(this), prev_blocks = std::move(prev_blocks), promise = std::move(promise)](td::Result R) mutable { TRY_STATUS_PROMISE(promise, R.move_as_status()); - td::actor::send_closure(SelfId, &CollatorNode::process_generate_block_query, shard, cc_seqno, - std::move(prev_blocks), std::move(priority), timeout, std::move(promise)); + td::actor::send_closure(SelfId, &CollatorNode::process_generate_block_query, src, shard, cc_seqno, + std::move(prev_blocks), std::move(priority), is_optimistic, timeout, std::move(promise)); }); return; } @@ -410,8 +443,8 @@ void CollatorNode::process_generate_block_query(ShardIdFull shard, CatchainSeqno promise.set_error(td::Status::Error(PSTRING() << "cannot collate shard " << shard.to_str())); return; } - td::actor::send_closure(validator_group_info.actor, &CollatorNodeSession::generate_block, std::move(prev_blocks), - priority, timeout, std::move(promise)); + td::actor::send_closure(validator_group_info.actor, &CollatorNodeSession::process_request, src, + std::move(prev_blocks), priority, is_optimistic, timeout, std::move(promise)); } td::Status CollatorNode::check_out_of_sync() { @@ -461,7 +494,12 @@ void CollatorNode::process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode LOG(DEBUG) << "got ping from " << src; TRY_STATUS_PROMISE(promise, check_out_of_sync()); TRY_STATUS_PROMISE_PREFIX(promise, mc_config_status_.clone(), "unsupported mc config: "); - promise.set_result(create_serialize_tl_object(0)); + auto pong = create_tl_object(); + if (ping.flags_ & ton_api::collatorNode_pong::VERSION_MASK) { + pong->flags_ |= ton_api::collatorNode_pong::VERSION_MASK; + pong->version_ = COLLATOR_NODE_VERSION; + } + promise.set_result(serialize_tl_object(pong, true)); } bool CollatorNode::can_collate_shard(ShardIdFull shard) const { @@ -469,73 +507,4 @@ bool CollatorNode::can_collate_shard(ShardIdFull shard) const { [&](const ShardIdFull& our_shard) { return shard_intersects(shard, our_shard); }); } -tl_object_ptr CollatorNode::serialize_candidate(const BlockCandidate& block, - bool compress) { - if (!compress) { - return create_tl_object( - PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), block.data.clone(), - block.collated_data.clone()); - } - size_t decompressed_size; - td::BufferSlice compressed = - validatorsession::compress_candidate_data(block.data, block.collated_data, decompressed_size).move_as_ok(); - return create_tl_object( - 0, PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), - (int)decompressed_size, std::move(compressed)); -} - -td::Result CollatorNode::deserialize_candidate(tl_object_ptr f, - int max_decompressed_data_size, int proto_version) { - td::Result res; - ton_api::downcast_call( - *f, td::overloaded( - [&](ton_api::collatorNode_candidate& c) { - res = [&]() -> td::Result { - auto hash = td::sha256_bits256(c.collated_data_); - auto key = PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), hash, std::move(c.data_), - std::move(c.collated_data_)}; - }(); - }, - [&](ton_api::collatorNode_compressedCandidate& c) { - res = [&]() -> td::Result { - if (c.decompressed_size_ <= 0) { - return td::Status::Error("invalid decompressed size"); - } - if (c.decompressed_size_ > max_decompressed_data_size) { - return td::Status::Error("decompressed size is too big"); - } - TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, false, c.decompressed_size_, - max_decompressed_data_size, proto_version)); - auto collated_data_hash = td::sha256_bits256(p.second); - auto key = PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), - std::move(p.second)}; - }(); - }, - [&](ton_api::collatorNode_compressedCandidateV2& c) { - res = [&]() -> td::Result { - TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, true, 0, - max_decompressed_data_size, proto_version)); - auto collated_data_hash = td::sha256_bits256(p.second); - auto key = PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), - std::move(p.second)}; - }(); - })); - return res; -} - } // namespace ton::validator diff --git a/validator/collator-node/collator-node.hpp b/validator/collator-node/collator-node.hpp index d07116979f..cb9cec47c5 100644 --- a/validator/collator-node/collator-node.hpp +++ b/validator/collator-node/collator-node.hpp @@ -45,9 +45,9 @@ class CollatorNode : public td::actor::Actor { private: void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); - void process_generate_block_query(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, - BlockCandidatePriority priority, td::Timestamp timeout, - td::Promise promise); + void process_generate_block_query(adnl::AdnlNodeIdShort src, ShardIdFull shard, CatchainSeqno cc_seqno, + std::vector prev_blocks, BlockCandidatePriority priority, + bool is_optimistic, td::Timestamp timeout, td::Promise promise); void process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode_ping& ping, td::Promise promise); bool can_collate_shard(ShardIdFull shard) const; @@ -87,10 +87,10 @@ class CollatorNode : public td::actor::Actor { return check_out_of_sync().is_ok() && mc_config_status_.is_ok(); } + static constexpr int COLLATOR_NODE_VERSION = 1; + public: - static tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress); - static td::Result deserialize_candidate(tl_object_ptr f, - int max_decompressed_data_size, int proto_version); + static constexpr int VERSION_OPTIMISTIC_COLLATE = 1; }; } // namespace ton::validator diff --git a/validator/collator-node/utils.cpp b/validator/collator-node/utils.cpp new file mode 100644 index 0000000000..23e2ec40f9 --- /dev/null +++ b/validator/collator-node/utils.cpp @@ -0,0 +1,93 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "utils.hpp" +#include "checksum.h" +#include "keys/keys.hpp" +#include "ton/ton-tl.hpp" +#include "validator-session/candidate-serializer.h" + +namespace ton::validator { + +tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress) { + if (!compress) { + return create_tl_object( + PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), block.data.clone(), + block.collated_data.clone()); + } + size_t decompressed_size; + td::BufferSlice compressed = + validatorsession::compress_candidate_data(block.data, block.collated_data, decompressed_size).move_as_ok(); + return create_tl_object( + 0, PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), + (int)decompressed_size, std::move(compressed)); +} + +td::Result deserialize_candidate(tl_object_ptr f, + int max_decompressed_data_size, int proto_version) { + td::Result res; + ton_api::downcast_call( + *f, td::overloaded( + [&](ton_api::collatorNode_candidate& c) { + res = [&]() -> td::Result { + auto hash = td::sha256_bits256(c.collated_data_); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), hash, std::move(c.data_), + std::move(c.collated_data_)}; + }(); + }, + [&](ton_api::collatorNode_compressedCandidate& c) { + res = [&]() -> td::Result { + if (c.decompressed_size_ <= 0) { + return td::Status::Error("invalid decompressed size"); + } + if (c.decompressed_size_ > max_decompressed_data_size) { + return td::Status::Error("decompressed size is too big"); + } + TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, false, c.decompressed_size_, + max_decompressed_data_size, proto_version)); + auto collated_data_hash = td::sha256_bits256(p.second); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), + std::move(p.second)}; + }(); + }, + [&](ton_api::collatorNode_compressedCandidateV2& c) { + res = [&]() -> td::Result { + TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, true, 0, + max_decompressed_data_size, proto_version)); + auto collated_data_hash = td::sha256_bits256(p.second); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), + std::move(p.second)}; + }(); + })); + return res; +} + +} // namespace ton::validator diff --git a/validator/collator-node/utils.hpp b/validator/collator-node/utils.hpp new file mode 100644 index 0000000000..26b67d5a97 --- /dev/null +++ b/validator/collator-node/utils.hpp @@ -0,0 +1,27 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "ton/ton-types.h" +#include "tl/generate/auto/tl/ton_api.h" + +namespace ton::validator { + +tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress); +td::Result deserialize_candidate(tl_object_ptr f, + int max_decompressed_data_size, int proto_version); +} // namespace ton::validator diff --git a/validator/fabric.h b/validator/fabric.h index 825c3b2482..e0088a0455 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -39,7 +39,7 @@ struct CollateParams { int attempt_idx = 0; // Optional - used for optimistic collation - Ref optimistic_prev_block_ = {}; + Ref optimistic_prev_block = {}; }; enum ValidateMode { fake = 1 }; diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index f9e4244794..0aa0e26adc 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -331,7 +331,7 @@ class Collator final : public td::actor::Actor { int prev_block_idx(const BlockIdExt& id) const { for (size_t i = 0; i < prev_blocks.size(); ++i) { if (prev_blocks[i] == id) { - return i; + return (int)i; } } return -1; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index c2ca14f999..b2e5c725b9 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -82,7 +82,7 @@ Collator::Collator(CollateParams params, td::actor::ActorId ma , main_promise(std::move(promise)) , collator_node_id_(params.collator_node_id) , skip_store_candidate_(params.skip_store_candidate) - , optimistic_prev_block_(std::move(params.optimistic_prev_block_)) + , optimistic_prev_block_(std::move(params.optimistic_prev_block)) , attempt_idx_(params.attempt_idx) , perf_timer_("collate", 0.1, [manager](double duration) { @@ -368,7 +368,6 @@ void Collator::process_optimistic_prev_block() { std::move(token)); } - /** * Raises an error when timeout is reached. */ @@ -444,7 +443,7 @@ bool Collator::fatal_error(td::Status error) { .collator_node_id = collator_node_id_, .skip_store_candidate = skip_store_candidate_, .attempt_idx = attempt_idx_ + 1, - .optimistic_prev_block_ = optimistic_prev_block_}, + .optimistic_prev_block = optimistic_prev_block_}, manager, td::Timestamp::in(10.0), std::move(cancellation_token_), std::move(main_promise)); } else { LOG(INFO) << "collation failed in " << perf_timer_.elapsed() << " s " << error; @@ -6668,8 +6667,8 @@ void Collator::finalize_stats() { if (block_candidate) { stats_.block_id = block_candidate->id; stats_.collated_data_hash = block_candidate->collated_file_hash; - stats_.actual_bytes = block_candidate->data.size(); - stats_.actual_collated_data_bytes = block_candidate->collated_data.size(); + stats_.actual_bytes = (td::uint32)block_candidate->data.size(); + stats_.actual_collated_data_bytes = (td::uint32)block_candidate->collated_data.size(); } else { stats_.block_id.id = new_id; } @@ -6680,10 +6679,10 @@ void Collator::finalize_stats() { stats_.self = stats_.is_validator ? PublicKey(pubkeys::Ed25519(created_by_)).compute_short_id() : collator_node_id_.pubkey_hash(); if (block_limit_status_) { - stats_.estimated_bytes = block_limit_status_->estimate_block_size(); - stats_.gas = block_limit_status_->gas_used; - stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; - stats_.estimated_collated_data_bytes = block_limit_status_->collated_data_size_estimate; + stats_.estimated_bytes = (td::uint32)block_limit_status_->estimate_block_size(); + stats_.gas = (td::uint32)block_limit_status_->gas_used; + stats_.lt_delta = (td::uint32)(block_limit_status_->cur_lt - block_limit_status_->limits.start_lt); + stats_.estimated_collated_data_bytes = (td::uint32)block_limit_status_->collated_data_size_estimate; stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.estimated_bytes); stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas); stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 01ded4dfa3..00337f3c7a 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -7203,8 +7203,8 @@ void ValidateQuery::record_stats(bool valid, std::string error_message) { } else { stats_.comment = std::move(error_message); } - stats_.actual_bytes = block_candidate.data.size(); - stats_.actual_collated_data_bytes = block_candidate.collated_data.size(); + stats_.actual_bytes = (td::uint32)block_candidate.data.size(); + stats_.actual_collated_data_bytes = (td::uint32)block_candidate.collated_data.size(); stats_.total_time = perf_timer_.elapsed(); stats_.work_time.total = work_timer_.elapsed_both(); LOG(WARNING) << "validation took " << perf_timer_.elapsed() << "s"; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 70876084a8..5757c2bd49 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -185,7 +185,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { UNREACHABLE(); } - void set_block_state_from_data_preliminary(std::vector> blocks, td::Promise promise) { + void set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) override { UNREACHABLE(); } void get_cell_db_reader(td::Promise> promise) override; @@ -246,7 +247,7 @@ class ValidatorManagerImpl : public ValidatorManager { promise.set_value(td::Unit()); } void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data, int mode) { + td::BufferSlice data, int mode) override { callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data), mode); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 4862c47273..f61bd5f0b4 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2675,7 +2675,7 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group td::actor::ActorId ValidatorManagerImpl::get_collation_manager(adnl::AdnlNodeIdShort adnl_id) { auto &actor = collation_managers_[adnl_id]; if (actor.empty()) { - actor = td::actor::create_actor("collation", adnl_id, opts_, actor_id(this), rldp2_); + actor = td::actor::create_actor("collation", adnl_id, opts_, actor_id(this), adnl_, rldp2_); } return actor.get(); } diff --git a/validator/shard-block-verifier.cpp b/validator/shard-block-verifier.cpp index ebd6111440..0403ff22aa 100644 --- a/validator/shard-block-verifier.cpp +++ b/validator/shard-block-verifier.cpp @@ -131,7 +131,7 @@ void ShardBlockVerifier::process_message(adnl::AdnlNodeIdShort src, td::BufferSl int ShardBlockVerifier::get_config_shard_idx(const ShardIdFull& shard_id) const { for (size_t i = 0; i < config_->shards.size(); i++) { if (shard_intersects(shard_id, config_->shards[i].shard_id)) { - return i; + return (int)i; } } return -1; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 163eb90bfd..7cc980c002 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -320,7 +320,7 @@ void ValidatorGroup::generate_block_optimistic(validatorsession::BlockSourceInfo }; LOG(WARNING) << "Optimistically generating next block after " << block_id.to_str(); td::uint64 max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; - td::actor::send_closure(collation_manager_, &CollationManager::collate_next_block, shard_, min_masterchain_block_id_, + td::actor::send_closure(collation_manager_, &CollationManager::collate_block_optimistic, shard_, min_masterchain_block_id_, block_id, std::move(prev_block), Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, source_info.priority, validator_set_, max_answer_size, optimistic_generation_->cancellation_token_source.get_cancellation_token(), std::move(P), From e173f11332bcea97835f7e3bbc3a030f490abf4b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 7 Aug 2025 16:13:58 +0300 Subject: [PATCH 371/524] Wait for block in validator-session.cpp --- validator-session/validator-session.cpp | 66 +++++++++++++++++++------ validator-session/validator-session.hpp | 3 +- 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 4a06314077..964f097e6c 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -361,6 +361,12 @@ void ValidatorSessionImpl::process_received_block(td::uint32 block_round, Public } blocks_[block_id] = std::move(candidate); + if (auto it = block_waiters_.find(block_id); it != block_waiters_.end()) { + for (auto &promise : it->second) { + promise.set_result(td::Unit()); + } + block_waiters_.erase(it); + } VLOG(VALIDATOR_SESSION_WARNING) << this << ": received broadcast " << block_id; if (block_round != cur_round_) { @@ -552,6 +558,12 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, GeneratedCandidate blocks_[block_id] = create_tl_object( local_id().tl(), round, c.candidate.id.root_hash, std::move(c.candidate.data), std::move(c.candidate.collated_data)); + if (auto it = block_waiters_.find(block_id); it != block_waiters_.end()) { + for (auto &promise : it->second) { + promise.set_result(td::Unit()); + } + block_waiters_.erase(it); + } pending_generate_ = false; generated_ = true; generated_block_ = block_id; @@ -742,7 +754,6 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { VLOG(VALIDATOR_SESSION_WARNING) << print_id << ": failed to get candidate " << hash << " from " << id << ": " << R.move_as_error(); } else { - LOG(ERROR) << "QQQQQ Got block " << R.ok().size(); td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src_id, R.move_as_ok(), candidate_id, false, false); } @@ -1369,25 +1380,48 @@ void ValidatorSessionImpl::process_approve(td::uint32 node_id, td::uint32 round, bool is_approved_66pct = stat->approved_66pct_at > 0.0; if (allow_optimistic_generation_ && !was_approved_66pct && is_approved_66pct && cur_round_ == round && - description().get_node_priority(local_idx(), round + 1) == 0 && blocks_.contains(candidate_id)) { - auto &block = blocks_[candidate_id]; - if (cur_round_ == first_block_round_ && - description().get_node_priority(description().get_source_idx(PublicKeyHash{block->src_}), cur_round_) == 0) { - callback_->generate_block_optimistic(BlockSourceInfo{description().get_source_public_key(local_idx()), - BlockCandidatePriority{round + 1, round + 1, 0}}, - block->data_.clone(), block->root_hash_, stat->block_id.file_hash, - [=, SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - return; - } - td::actor::send_closure( - SelfId, &ValidatorSessionImpl::generated_optimistic_candidate, - round + 1, R.move_as_ok(), candidate_id); - }); + cur_round_ == first_block_round_ && description().get_node_priority(local_idx(), round + 1) == 0 && + blocks_.contains(candidate_id) && + description().get_node_priority(description().get_source_idx(stat->validator_id), cur_round_) == 0) { + if (blocks_.contains(candidate_id)) { + generate_block_optimistic(round, candidate_id); + } else { + block_waiters_[candidate_id].push_back([=, SelfId = actor_id(this)](td::Result R) { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ValidatorSessionImpl::generate_block_optimistic, round, candidate_id); + } + }); } } } +void ValidatorSessionImpl::generate_block_optimistic(td::uint32 cur_round, + ValidatorSessionCandidateId prev_candidate_id) { + if (cur_round != cur_round_) { + return; + } + auto it = blocks_.find(prev_candidate_id); + if (it == blocks_.end()) { + return; + } + auto &block = it->second; + auto stat = stats_get_candidate_stat_by_id(cur_round, prev_candidate_id); + if (!stat) { + return; + } + callback_->generate_block_optimistic(BlockSourceInfo{description().get_source_public_key(local_idx()), + BlockCandidatePriority{cur_round + 1, cur_round + 1, 0}}, + block->data_.clone(), block->root_hash_, stat->block_id.file_hash, + [=, SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + return; + } + td::actor::send_closure(SelfId, + &ValidatorSessionImpl::generated_optimistic_candidate, + cur_round + 1, R.move_as_ok(), prev_candidate_id); + }); +} + void ValidatorSessionImpl::generated_optimistic_candidate(td::uint32 round, GeneratedCandidate candidate, ValidatorSessionCandidateId prev_candidate) { if (cur_round_ > round) { diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index ef0e3fba89..83189868ad 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -76,6 +76,7 @@ class ValidatorSessionImpl : public ValidatorSession { std::map> blocks_; // src_round_candidate_[src_id][round] -> candidate id std::vector> src_round_candidate_; + std::map>> block_waiters_; catchain::CatChainSessionId unique_hash_; @@ -202,7 +203,7 @@ class ValidatorSessionImpl : public ValidatorSession { ValidatorSessionCandidateId candidate_id); void stats_process_action(td::uint32 node_id, ton_api::validatorSession_round_Message &action); void process_approve(td::uint32 node_id, td::uint32 round, ValidatorSessionCandidateId candidate_id); - + void generate_block_optimistic(td::uint32 cur_round, ValidatorSessionCandidateId prev_candidate_id); void generated_optimistic_candidate(td::uint32 round, GeneratedCandidate candidate, ValidatorSessionCandidateId prev_candidate); From 90616d8cb75bd359dacab0be6c8e9fff13033b25 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 7 Aug 2025 16:18:14 +0300 Subject: [PATCH 372/524] Fix compilation error --- crypto/vm/boc-compression.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/vm/boc-compression.cpp b/crypto/vm/boc-compression.cpp index 63a834e086..db04f162f5 100644 --- a/crypto/vm/boc-compression.cpp +++ b/crypto/vm/boc-compression.cpp @@ -283,7 +283,7 @@ td::Result boc_compress_improved_structure_lz4(const std::vecto continue; int delta = rank[boc_graph[node][j]] - i - 2; // Always >= 0 because of above check - size_t required_bits = 1 + (31 ^ __builtin_clz(node_count - i - 3)); + size_t required_bits = 1 + (31 ^ td::count_leading_zeroes32(node_count - i - 3)); if (required_bits < 8 - (result.size() + 1) % 8 + 1) { append_uint(result, delta, required_bits); @@ -494,7 +494,7 @@ td::Result>> boc_decompress_improved_structure_lz4 for (int j = 0; j < cell_refs_cnt[i]; ++j) { if (!boc_graph[i][j]) { size_t pref_size = (orig_size - bit_reader.size()); - size_t required_bits = 1 + (31 ^ __builtin_clz(node_count - i - 3)); + size_t required_bits = 1 + (31 ^ td::count_leading_zeroes32(node_count - i - 3)); if (required_bits < 8 - (pref_size + 1) % 8 + 1) { TRY_RESULT_ASSIGN(boc_graph[i][j], read_uint(bit_reader, required_bits)); From 038db1b223ccd3efe8d0b1ace3411d6a40c49afb Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 11 Aug 2025 11:18:55 +0300 Subject: [PATCH 373/524] Bugfixes in storage daemon --- storage/NodeActor.cpp | 30 ++++++++++++++--------- storage/storage-daemon/storage-daemon.cpp | 3 +++ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/storage/NodeActor.cpp b/storage/NodeActor.cpp index e24fffff5a..92a83968fe 100644 --- a/storage/NodeActor.cpp +++ b/storage/NodeActor.cpp @@ -27,6 +27,7 @@ #include "td/utils/overloaded.h" #include "tl-utils/tl-utils.hpp" #include "auto/tl/ton_api.hpp" +#include "common/delay.h" #include "td/actor/MultiPromise.h" namespace ton { @@ -545,8 +546,9 @@ void NodeActor::loop_queries() { auto it = peers_.find(part.peer_id); CHECK(it != peers_.end()); auto &state = it->second.state; - CHECK(state->peer_state_ready_); - CHECK(state->peer_state_.load().will_upload); + if (!state->peer_state_ready_ || !state->peer_state_.load().will_upload) { + continue; + } CHECK(state->node_queries_active_.size() < MAX_PEER_TOTAL_QUERIES); auto part_id = part.part_id; if (state->node_queries_active_.insert(static_cast(part_id)).second) { @@ -787,15 +789,21 @@ void NodeActor::db_store_torrent_meta() { return; } next_db_store_meta_at_ = td::Timestamp::never(); - auto meta = torrent_.get_meta_str(); - db_->set(create_hash_tl_object(torrent_.get_hash()), td::BufferSlice(meta), - [new_count = (td::int64)torrent_.get_ready_parts_count(), SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &NodeActor::after_db_store_torrent_meta, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &NodeActor::after_db_store_torrent_meta, new_count); - } - }); + auto meta = torrent_.get_meta(); + delay_action( + [SelfId = actor_id(this), meta = std::move(meta), db = db_, hash = torrent_.get_hash(), + new_count = (td::int64)torrent_.get_ready_parts_count()]() { + auto meta_str = meta.serialize(); + db->set(create_hash_tl_object(hash), td::BufferSlice(meta_str), + [=](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &NodeActor::after_db_store_torrent_meta, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &NodeActor::after_db_store_torrent_meta, new_count); + } + }); + }, + td::Timestamp::now()); } void NodeActor::after_db_store_torrent_meta(td::Result R) { diff --git a/storage/storage-daemon/storage-daemon.cpp b/storage/storage-daemon/storage-daemon.cpp index 98388428d2..d1cbf37b59 100644 --- a/storage/storage-daemon/storage-daemon.cpp +++ b/storage/storage-daemon/storage-daemon.cpp @@ -42,6 +42,8 @@ #if TD_DARWIN || TD_LINUX #include #endif +#include "td/utils/port/rlimit.h" + #include namespace ton { @@ -950,6 +952,7 @@ int main(int argc, char *argv[]) { SCOPE_EXIT { td::log_interface = td::default_log_interface; }; + LOG_STATUS(td::change_maximize_rlimit(td::RlimitType::nofile, 786432)); td::IPAddress ip_addr; bool client_mode = false; From 37f10de155f9fb51037c048a8e3d3d94ef91b6cc Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 12 Aug 2025 13:54:32 +0300 Subject: [PATCH 374/524] Change new bounce format: include full or partial original body --- crypto/block/block.tlb | 2 +- crypto/block/transaction.cpp | 7 +++---- crypto/block/transaction.h | 2 +- doc/GlobalVersions.md | 6 +++--- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 9b4af3c1e0..03b92cd2b4 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -166,7 +166,7 @@ _ (Message Any) = MessageAny; _ value:CurrencyCollection created_lt:uint64 created_at:uint32 = NewBounceOriginalInfo; _ gas_used:uint32 vm_steps:uint32 = NewBounceComputePhaseInfo; new_bounce_body#fffffffe - original_body:(Maybe ^Cell) + original_body:^Cell original_info:^NewBounceOriginalInfo bounced_by_phase:uint8 exit_code:int32 compute_phase:(Maybe NewBounceComputePhaseInfo) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 8d220c7b98..3d83a93da4 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -877,7 +877,7 @@ bool Transaction::unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* ihr_fee = td::zero_refint(); td::RefInt256 extra_flags = tlb::t_Grams.as_integer(in_msg_info.extra_flags); new_bounce_format = extra_flags->get_bit(0); - new_bounce_format_with_body = extra_flags->get_bit(1); + new_bounce_format_full_body = extra_flags->get_bit(1); } else { // Legacy: extra_flags was previously ihr_fee ihr_fee = tlb::t_Grams.as_integer(in_msg_info.extra_flags); @@ -3276,11 +3276,10 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { vm::CellBuilder body; if (new_bounce_format) { body.store_long(0xfffffffeU, 32); // new_bounce_body#fffffffe - if (new_bounce_format_with_body) { // original_body:(Maybe ^Cell) - body.store_long(1, 1); + if (new_bounce_format_full_body) { // original_body:^Cell body.store_ref(vm::CellBuilder().append_cellslice(in_msg_body).finalize_novm()); } else { - body.store_long(0, 1); + body.store_ref(vm::CellBuilder().store_bits(in_msg_body->as_bitslice()).finalize_novm()); } body.store_ref(vm::CellBuilder() .append_cellslice(in_msg_info.value) // value:CurrencyCollection diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index e987645db0..d6ab6c870b 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -359,7 +359,7 @@ struct Transaction { bool in_msg_extern{false}; gen::CommonMsgInfo::Record_int_msg_info in_msg_info; bool new_bounce_format{false}; - bool new_bounce_format_with_body{false}; + bool new_bounce_format_full_body{false}; bool use_msg_state{false}; bool is_first{false}; bool orig_addr_rewrite_set{false}; diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 6e59955c4e..e2336e6948 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -236,20 +236,20 @@ Field `ihr_fee:Grams` in internal message is now called `extra_flags:(VarUIntege This field does not represent fees. `ihr_fee` is always zero since version 11, so this field was essentially unused. `(extra_flags & 1) = 1` enables the new bounce format for the message. The bounced message contains information about the transaction. -If `(extra_flags & 3) = 3`, the bounced message also contains the whole body of the original message. +If `(extra_flags & 3) = 3`, the bounced message contains the whole body of the original message. Otherwise, only the bits from the root of the original body are returned. When the message with new bounce flag is bounced, the bounced message body has the following format (`new_bounce_body`): ``` _ value:CurrencyCollection created_lt:uint64 created_at:uint32 = NewBounceOriginalInfo; _ gas_used:uint32 vm_steps:uint32 = NewBounceComputePhaseInfo; new_bounce_body#fffffffe - original_body:(Maybe ^Cell) + original_body:^Cell original_info:^NewBounceOriginalInfo bounced_by_phase:uint8 exit_code:int32 compute_phase:(Maybe NewBounceComputePhaseInfo) = NewBounceBody; ``` -- `original_body` - cell that contains the body of the original message (if `extra_flags & 2`) or nothing (if not `extra_flags & 2`). +- `original_body` - cell that contains the body of the original message. If `extra_flags & 2` then the whole body is returned, otherwise it is only the root without refs. - `original_info` - value, lt and unixtime of the original message. - `bounced_by_phase`: - `0` - compute phase was skipped. `exit_code` denotes the skip reason: From c7b6c2599b1612bc7589199d9424e6d3bffed7f1 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 12 Aug 2025 14:47:02 +0300 Subject: [PATCH 375/524] Use rldp2 for downloading candidates in validator session --- validator-session/CMakeLists.txt | 2 +- validator-session/validator-session.cpp | 5 ++--- validator-session/validator-session.h | 4 ++-- validator-session/validator-session.hpp | 4 ++-- validator/validator-group.cpp | 10 +++++----- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/validator-session/CMakeLists.txt b/validator-session/CMakeLists.txt index 94363c79cf..6d4b00bd61 100644 --- a/validator-session/CMakeLists.txt +++ b/validator-session/CMakeLists.txt @@ -26,4 +26,4 @@ target_include_directories(validatorsession PUBLIC $/.. ${OPENSSL_INCLUDE_DIR} ) -target_link_libraries(validatorsession PRIVATE tdutils tdactor adnl rldp tl_api dht tdfec overlay catchain) +target_link_libraries(validatorsession PRIVATE tdutils tdactor adnl rldp2 tl_api dht tdfec overlay catchain) diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 964f097e6c..9720ab34fc 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -1097,7 +1097,7 @@ ValidatorSessionImpl::ValidatorSessionImpl(catchain::CatChainSessionId session_i PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync) : unique_hash_(session_id) @@ -1221,7 +1221,6 @@ void ValidatorSessionImpl::start_up() { virtual_state_ = real_state_; check_all(); - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, description().get_source_adnl_id(local_idx())); } void ValidatorSessionImpl::stats_init() { @@ -1434,7 +1433,7 @@ td::actor::ActorOwn ValidatorSession::create( catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, + td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync) { return td::actor::create_actor("session", session_id, std::move(opts), local_id, std::move(nodes), std::move(callback), keyring, adnl, rldp, diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index 5e226f3578..dddd3092ee 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -20,7 +20,7 @@ #include "adnl/adnl.h" #include "adnl/utils.hpp" -#include "rldp/rldp.h" +#include "rldp2/rldp.h" #include "ton/ton-types.h" @@ -112,7 +112,7 @@ class ValidatorSession : public td::actor::Actor { catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, + td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync); virtual ~ValidatorSession() = default; }; diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 83189868ad..8af0fd0ec1 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -86,7 +86,7 @@ class ValidatorSessionImpl : public ValidatorSession { td::actor::ActorId keyring_; td::actor::ActorId adnl_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; td::actor::ActorId overlay_manager_; td::actor::ActorOwn catchain_; std::unique_ptr description_; @@ -211,7 +211,7 @@ class ValidatorSessionImpl : public ValidatorSession { ValidatorSessionImpl(catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, + td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync); void start_up() override; void alarm() override; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 7cc980c002..4a8310862a 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -473,14 +473,17 @@ void ValidatorGroup::create_session() { } CHECK(found); + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_adnl_id_); + td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_adnl_id_); + config_.catchain_opts.broadcast_speed_multiplier = opts_->get_catchain_broadcast_speed_multiplier(); if (!config_.new_catchain_ids) { session_ = validatorsession::ValidatorSession::create(session_id_, config_, local_id_, std::move(vec), - make_validator_session_callback(), keyring_, adnl_, rldp_, + make_validator_session_callback(), keyring_, adnl_, rldp2_, overlays_, db_root_, "-", allow_unsafe_self_blocks_resync_); } else { session_ = validatorsession::ValidatorSession::create( - session_id_, config_, local_id_, std::move(vec), make_validator_session_callback(), keyring_, adnl_, rldp_, + session_id_, config_, local_id_, std::move(vec), make_validator_session_callback(), keyring_, adnl_, rldp2_, overlays_, db_root_ + "/catchains/", PSTRING() << "." << shard_.workchain << "." << shard_.shard << "." << validator_set_->get_catchain_seqno() << ".", @@ -495,9 +498,6 @@ void ValidatorGroup::create_session() { if (started_) { td::actor::send_closure(session_, &validatorsession::ValidatorSession::start); } - - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_adnl_id_); - td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_adnl_id_); } void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterchain_block_id) { From a70bdfb53f188752b41add4ce7e13ae1472b7f84 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 12 Aug 2025 16:29:40 +0300 Subject: [PATCH 376/524] Disable optimistic generation in masterchain --- validator-session/validator-session.cpp | 1 + validator/validator-group.cpp | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 9720ab34fc..bddf56004a 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -1413,6 +1413,7 @@ void ValidatorSessionImpl::generate_block_optimistic(td::uint32 cur_round, block->data_.clone(), block->root_hash_, stat->block_id.file_hash, [=, SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { + LOG(DEBUG) << "Optimistic generation error: " << R.move_as_error(); return; } td::actor::send_closure(SelfId, diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 4a8310862a..450453788e 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -301,12 +301,19 @@ void ValidatorGroup::generate_block_optimistic(validatorsession::BlockSourceInfo td::BufferSlice prev_block, RootHash prev_root_hash, FileHash prev_file_hash, td::Promise promise) { if (destroying_) { + promise.set_error(td::Status::Error("validator session finished")); + return; + } + if (shard_.is_masterchain()) { + promise.set_error(td::Status::Error("no optimistic generation in masterchain")); return; } if (last_known_round_id_ + 1 != source_info.priority.round) { + promise.set_error(td::Status::Error("too old round")); return; } if (optimistic_generation_ && optimistic_generation_->round >= source_info.priority.round) { + promise.set_error(td::Status::Error("optimistic generation already in progress")); return; } BlockIdExt block_id{create_next_block_id_simple(), prev_root_hash, prev_file_hash}; From 3f216b6a83377d27d6df3fa27e3e4005ae4f7535 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 14 Aug 2025 17:14:12 +0300 Subject: [PATCH 377/524] Fix returning error in wait block state --- validator/downloaders/wait-block-state.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index 7ca2bbd2eb..a69e2b1308 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -34,7 +34,7 @@ void WaitBlockState::alarm() { void WaitBlockState::abort_query(td::Status reason) { if (promise_no_store_) { promise_no_store_.set_error( - reason.move_as_error_prefix(PSTRING() << "failed to download state " << handle_->id() << ": ")); + reason.clone().move_as_error_prefix(PSTRING() << "failed to download state " << handle_->id() << ": ")); } if (promise_final_) { if (priority_ > 0 || (reason.code() != ErrorCode::timeout && reason.code() != ErrorCode::notready)) { From c48413a2e8f38f9c6b4b726141049a1471653c74 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sat, 16 Aug 2025 17:54:35 +0300 Subject: [PATCH 378/524] Fix error processing in Collator::process_optimistic_prev_block --- validator/impl/collator-impl.h | 2 +- validator/impl/collator.cpp | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 0aa0e26adc..03eedb841f 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -122,7 +122,7 @@ class Collator final : public td::actor::Actor { private: void start_up() override; void load_prev_states_blocks(); - void process_optimistic_prev_block(); + bool process_optimistic_prev_block(); void alarm() override; int verbosity{3 * 0}; int verify{1}; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index b2e5c725b9..e38d30e1ff 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -241,7 +241,9 @@ void Collator::start_up() { if (optimistic_prev_block_.is_null()) { load_prev_states_blocks(); } else { - process_optimistic_prev_block(); + if (!process_optimistic_prev_block()) { + return; + } } if (is_hardfork_) { LOG(WARNING) << "generating a hardfork block"; @@ -324,15 +326,14 @@ void Collator::load_prev_states_blocks() { /** * Write optimistic prev block as block data, load previous state to apply Merkle update to it */ -void Collator::process_optimistic_prev_block() { +bool Collator::process_optimistic_prev_block() { std::vector prev_prev; BlockIdExt mc_blkid; bool after_split; auto S = block::unpack_block_prev_blk_try(optimistic_prev_block_->root_cell(), optimistic_prev_block_->block_id(), prev_prev, mc_blkid, after_split); if (S.is_error()) { - fatal_error(S.move_as_error_prefix("failed to unpack optimistic prev block: ")); - return; + return fatal_error(S.move_as_error_prefix("failed to unpack optimistic prev block: ")); } // 3.1. load state if (prev_prev.size() == 1) { @@ -366,6 +367,7 @@ void Collator::process_optimistic_prev_block() { auto token = perf_log_.start_action(PSTRING() << "opt wait_block_data"); td::actor::send_closure_later(actor_id(this), &Collator::after_get_block_data, 0, optimistic_prev_block_, std::move(token)); + return true; } /** From e169659d81d9f3962028c15fcd844155e0f73884 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 18 Aug 2025 11:08:56 +0300 Subject: [PATCH 379/524] Blockchain explorer: fix search by utime --- blockchain-explorer/blockchain-explorer-query.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/blockchain-explorer/blockchain-explorer-query.cpp b/blockchain-explorer/blockchain-explorer-query.cpp index d540d48782..6919c5c14c 100644 --- a/blockchain-explorer/blockchain-explorer-query.cpp +++ b/blockchain-explorer/blockchain-explorer-query.cpp @@ -525,8 +525,8 @@ HttpQueryBlockSearch::HttpQueryBlockSearch(std::map op } if (opts.count("utime") == 1) { try { - seqno_ = static_cast(std::stoull(opts["utime"])); - mode_ = 1; + utime_ = static_cast(std::stoull(opts["utime"])); + mode_ = 4; } catch (...) { error_ = td::Status::Error("cannot parse utime"); return; From f416812d3f94aa5b4d0d120f37e6a0027be8e9fd Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 19 Aug 2025 18:47:43 +0300 Subject: [PATCH 380/524] Optimistic validation --- validator-session/validator-session.cpp | 42 +++++++ validator-session/validator-session.h | 10 +- validator-session/validator-session.hpp | 3 + validator/fabric.h | 18 ++- validator/impl/fabric.cpp | 18 ++- validator/impl/validate-query.cpp | 152 +++++++++++++++++++----- validator/impl/validate-query.hpp | 24 +++- validator/manager-disk.cpp | 9 +- validator/validator-group.cpp | 141 ++++++++++++++++------ validator/validator-group.hpp | 3 +- 10 files changed, 324 insertions(+), 96 deletions(-) diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index bddf56004a..27c105929b 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -294,6 +294,12 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice << "]: duplicate optimistic broadcast for round " << block_round; return; } + int priority = description().get_node_priority(src_idx, block_round); + if (priority < 0) { + VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) + << "]: node is not a producer in round " << block_round; + return; + } if (block_round > cur_round_) { OptimisticBroadcast &optimistic_broadcast = optimistic_broadcasts_[{block_round, src_idx}]; optimistic_broadcast.candidate = std::move(candidate); @@ -301,6 +307,11 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice optimistic_broadcast.broadcast_info = broadcast_info; VLOG(VALIDATOR_SESSION_WARNING) << this << ": received optimistic broadcast " << block_id << " from " << src << ", round " << block_round; + validate_optimistic_broadcast( + BlockSourceInfo{description().get_source_public_key(src_idx), + BlockCandidatePriority{block_round, block_round, priority}}, + optimistic_broadcast.candidate->root_hash_, optimistic_broadcast.candidate->data_.clone(), + optimistic_broadcast.candidate->collated_data_.clone(), optimistic_broadcast.prev_candidate_id); return; } if (SentBlock::get_block_id(real_state_->get_committed_block(description(), block_round - 1)) != @@ -387,6 +398,37 @@ void ValidatorSessionImpl::process_received_block(td::uint32 block_round, Public } } +void ValidatorSessionImpl::validate_optimistic_broadcast(BlockSourceInfo source_info, + ValidatorSessionRootHash root_hash, td::BufferSlice data, + td::BufferSlice collated_data, + ValidatorSessionCandidateId prev_candidate_id) { + if (source_info.priority.round <= cur_round_) { + VLOG(VALIDATOR_SESSION_DEBUG) << this << ": validate optimistic broadcast from " + << source_info.source.compute_short_id() << " : too old"; + return; + } + auto it = blocks_.find(prev_candidate_id); + if (it == blocks_.end()) { + VLOG(VALIDATOR_SESSION_DEBUG) << this << ": validate optimistic broadcast from " + << source_info.source.compute_short_id() << " : wait for prev block"; + block_waiters_[prev_candidate_id].push_back( + [=, SelfId = actor_id(this), data = std::move(data), + collated_data = std::move(collated_data)](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ValidatorSessionImpl::validate_optimistic_broadcast, source_info, + root_hash, std::move(data), std::move(collated_data), prev_candidate_id); + } + }); + return; + } + VLOG(VALIDATOR_SESSION_DEBUG) << this << ": validate optimistic broadcast from " + << source_info.source.compute_short_id(); + callback_->on_optimistic_candidate( + source_info, root_hash, std::move(data), std::move(collated_data), + description().get_source_public_key(description().get_source_idx(PublicKeyHash{it->second->src_})), + it->second->root_hash_, it->second->data_.clone(), it->second->collated_data_.clone()); +} + void ValidatorSessionImpl::process_message(PublicKeyHash src, td::BufferSlice data) { } diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index dddd3092ee..f8e2d62e72 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -93,8 +93,14 @@ class ValidatorSession : public td::actor::Actor { ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash, td::Promise promise) = 0; - virtual void generate_block_optimistic(BlockSourceInfo source_info, td::BufferSlice prev_block, RootHash prev_root_hash, - FileHash prev_file_hash, td::Promise promise) { + virtual void generate_block_optimistic(BlockSourceInfo source_info, td::BufferSlice prev_block, + RootHash prev_root_hash, FileHash prev_file_hash, + td::Promise promise) { + } + virtual void on_optimistic_candidate(BlockSourceInfo source_info, ValidatorSessionRootHash root_hash, + td::BufferSlice data, td::BufferSlice collated_data, PublicKey prev_source, + ValidatorSessionRootHash prev_root_hash, td::BufferSlice prev_data, + td::BufferSlice prev_collated_data) { } virtual ~Callback() = default; }; diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 8af0fd0ec1..4861c52e17 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -238,6 +238,9 @@ class ValidatorSessionImpl : public ValidatorSession { void process_received_block(td::uint32 block_round, PublicKeyHash src, td::uint32 src_idx, tl_object_ptr candidate, const BroadcastInfo &info, bool is_overlay_broadcast, bool is_startup); + void validate_optimistic_broadcast(BlockSourceInfo source_info, ValidatorSessionRootHash root_hash, + td::BufferSlice data, td::BufferSlice collated_data, + ValidatorSessionCandidateId prev_candidate_id); void process_message(PublicKeyHash src, td::BufferSlice data); void process_query(PublicKeyHash src, td::BufferSlice data, td::Promise promise); diff --git a/validator/fabric.h b/validator/fabric.h index e0088a0455..8a3f6193fe 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -42,7 +42,17 @@ struct CollateParams { Ref optimistic_prev_block = {}; }; -enum ValidateMode { fake = 1 }; +struct ValidateParams { + ShardIdFull shard; + BlockIdExt min_masterchain_block_id; + std::vector prev; + td::Ref validator_set = {}; + PublicKeyHash local_validator_id = PublicKeyHash::zero();; + bool is_fake = false; + + // Optional - used for validation of optimistic candidates + Ref optimistic_prev_block = {}; +}; td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_, td::Ref opts); @@ -96,10 +106,8 @@ void run_check_proof_query(BlockIdExt id, td::Ref proof, td::actor::Actor td::Ref rel_key_block_proof, bool skip_check_signatures = false); void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); -void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - BlockCandidate candidate, td::Ref validator_set, PublicKeyHash local_validator_id, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, unsigned mode = 0); +void run_validate_query(BlockCandidate candidate, ValidateParams params, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise); void run_collate_query(CollateParams params, td::actor::ActorId manager, td::Timestamp timeout, td::CancellationToken cancellation_token, td::Promise promise); void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index f38f773204..4cf3349289 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -192,23 +192,19 @@ void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::act .release(); } -void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - BlockCandidate candidate, td::Ref validator_set, PublicKeyHash local_validator_id, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, unsigned mode) { +void run_validate_query(BlockCandidate candidate, ValidateParams params, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise) { BlockSeqno seqno = 0; - for (auto& p : prev) { + for (auto& p : params.prev) { if (p.seqno() > seqno) { seqno = p.seqno(); } } - bool is_fake = mode & ValidateMode::fake; static std::atomic idx; - td::actor::create_actor(PSTRING() << (is_fake ? "fakevalidate" : "validateblock") << shard.to_str() - << ":" << (seqno + 1) << "#" << idx.fetch_add(1), - shard, min_masterchain_block_id, std::move(prev), std::move(candidate), - std::move(validator_set), local_validator_id, std::move(manager), timeout, - std::move(promise), mode) + td::actor::create_actor( + PSTRING() << (params.is_fake ? "fakevalidate" : "validateblock") << params.shard.to_str() << ":" << (seqno + 1) + << "#" << idx.fetch_add(1), + std::move(candidate), std::move(params), std::move(manager), timeout, std::move(promise)) .release(); } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 00337f3c7a..adcf72eb25 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -59,33 +59,29 @@ std::string ErrorCtx::as_string() const { /** * Constructs a ValidateQuery object. * - * @param shard The shard of the block being validated. - * @param min_masterchain_block_id The minimum allowed masterchain block reference for the block. - * @param prev A vector of BlockIdExt representing the previous blocks. * @param candidate The BlockCandidate to be validated. - * @param validator_set A reference to the ValidatorSet. + * @param params Validation parameters * @param manager The ActorId of the ValidatorManager. * @param timeout The timeout for the validation. * @param promise The Promise to return the ValidateCandidateResult to. - * @param mode +1 - fake mode */ -ValidateQuery::ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - BlockCandidate candidate, Ref validator_set, - PublicKeyHash local_validator_id, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise, unsigned mode) - : shard_(shard) +ValidateQuery::ValidateQuery(BlockCandidate candidate, ValidateParams params, + td::actor::ActorId manager, td::Timestamp timeout, + td::Promise promise) + : shard_(params.shard) , id_(candidate.id) - , min_mc_block_id(min_masterchain_block_id) - , prev_blocks(std::move(prev)) + , min_mc_block_id(params.min_masterchain_block_id) + , prev_blocks(std::move(params.prev)) , block_candidate(std::move(candidate)) - , validator_set_(std::move(validator_set)) - , local_validator_id_(local_validator_id) + , validator_set_(std::move(params.validator_set)) + , local_validator_id_(params.local_validator_id) , manager(manager) , timeout(timeout) , main_promise(std::move(promise)) - , is_fake_(mode & ValidateMode::fake) + , is_fake_(params.is_fake) , shard_pfx_(shard_.shard) , shard_pfx_len_(ton::shard_prefix_length(shard_)) + , optimistic_prev_block_(std::move(params.optimistic_prev_block)) , perf_timer_("validateblock", 0.1, [manager](double duration) { send_closure(manager, &ValidatorManager::add_perf_timer_stat, "validateblock", duration); }) { @@ -350,6 +346,21 @@ void ValidateQuery::start_up() { // return; } } + if (optimistic_prev_block_.not_null()) { + if (is_masterchain()) { + fatal_error("optimistic validation in masterchain is not supported"); + return; + } + if (prev_blocks.size() != 1) { + fatal_error("optimistic prev block is not null, which is not allowed after merge"); + return; + } + if (prev_blocks[0] != optimistic_prev_block_->block_id()) { + fatal_error("optimistic prev block is not null, but has invalid block id"); + return; + } + LOG(WARNING) << "Optimistic prev block id = " << optimistic_prev_block_->block_id().to_str(); + } // 2. learn latest masterchain state and block id LOG(DEBUG) << "sending get_top_masterchain_state_block() to Manager"; ++pending; @@ -366,17 +377,13 @@ void ValidateQuery::start_up() { } // 4. load state(s) corresponding to previous block(s) (not full-collated-data or masterchain) prev_states.resize(prev_blocks.size()); - if (is_masterchain() || !full_collated_data_) { - for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { - // 4.1. load state - LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; - ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), - timeout, false, [self = get_self(), i](td::Result> res) -> void { - LOG(DEBUG) << "got answer to wait_block_state_short query #" << i; - td::actor::send_closure_later( - std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res)); - }); + if (is_masterchain() || !full_collated_data_) { + if (optimistic_prev_block_.is_null()) { + load_prev_states(); + } else { + if (!process_optimistic_prev_block()) { + return; + } } } // 4. request masterchain handle and state referred to in the block @@ -409,6 +416,84 @@ void ValidateQuery::start_up() { CHECK(pending); } +/** + * Load previous states from DB + */ +void ValidateQuery::load_prev_states() { + for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { + // 4.1. load state + LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; + ++pending; + td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), + timeout, false, [self = get_self(), i](td::Result> res) -> void { + LOG(DEBUG) << "got answer to wait_block_state_short query #" << i; + td::actor::send_closure_later( + std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res)); + }); + } +} + +/** + * Load previous state for optimistic prev block to apply Merkle update to it + */ +bool ValidateQuery::process_optimistic_prev_block() { + std::vector prev_prev; + BlockIdExt mc_blkid; + bool after_split; + auto S = block::unpack_block_prev_blk_try(optimistic_prev_block_->root_cell(), optimistic_prev_block_->block_id(), + prev_prev, mc_blkid, after_split); + if (S.is_error()) { + return fatal_error(S.move_as_error_prefix("failed to unpack optimistic prev block: ")); + } + // 4.1. load state + if (prev_prev.size() == 1) { + LOG(DEBUG) << "sending wait_block_state() query for " << prev_prev[0].to_str() << " to Manager (opt)"; + ++pending; + td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_prev[0], priority(), timeout, + false, [self = get_self()](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query (opt)"; + td::actor::send_closure_later(std::move(self), + &ValidateQuery::after_get_shard_state_optimistic, + std::move(res)); + }); + } else { + CHECK(prev_prev.size() == 2); + LOG(DEBUG) << "sending wait_block_state_merge() query for " << prev_prev[0].to_str() << " and " + << prev_prev[1].to_str() << " to Manager (opt)"; + ++pending; + td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_merge, prev_prev[0], prev_prev[1], + priority(), timeout, [self = get_self()](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state_merge query (opt)"; + td::actor::send_closure_later(std::move(self), + &ValidateQuery::after_get_shard_state_optimistic, + std::move(res)); + }); + } + return true; +} + +/** + * Callback function called after retrieving previous state for optimistic prev block + * + * @param res The retrieved state. + */ +void ValidateQuery::after_get_shard_state_optimistic(td::Result> res) { + LOG(DEBUG) << "in ValidateQuery::after_get_shard_state_optimistic()"; + if (res.is_error()) { + fatal_error(res.move_as_error()); + return; + } + work_timer_.resume(); + auto state = res.move_as_ok(); + auto S = state.write().apply_block(optimistic_prev_block_->block_id(), optimistic_prev_block_); + if (S.is_error()) { + fatal_error(S.move_as_error_prefix("apply error: ")); + return; + } + work_timer_.pause(); + after_get_shard_state(0, std::move(state)); +} + /** * Unpacks and validates a block candidate. * @@ -1619,11 +1704,16 @@ bool ValidateQuery::request_neighbor_queues() { for (block::McShardDescr& descr : neighbors_) { LOG(DEBUG) << "requesting outbound queue of neighbor #" << i << " : " << descr.blk_.to_str(); ++pending; - send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, - [self = get_self(), i](td::Result> res) { - td::actor::send_closure(std::move(self), &ValidateQuery::got_neighbor_out_queue, i, - std::move(res)); - }); + if (int prev_idx = prev_block_idx(descr.blk_); prev_idx >= 0) { + td::actor::send_closure(actor_id(this), &ValidateQuery::got_neighbor_out_queue, i, + prev_states.at(prev_idx)->message_queue()); + } else { + send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, + [self = get_self(), i](td::Result> res) { + td::actor::send_closure(std::move(self), &ValidateQuery::got_neighbor_out_queue, i, + std::move(res)); + }); + } ++i; } } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index e43d782f53..29cdbd8d29 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -18,6 +18,7 @@ */ #pragma once +#include "fabric.h" #include "interfaces/validator-manager.h" #include "vm/cells.h" #include "vm/dict.h" @@ -112,15 +113,13 @@ class ValidateQuery : public td::actor::Actor { return SUPPORTED_VERSION; } static constexpr long long supported_capabilities() { - return ton::capCreateStatsEnabled | ton::capBounceMsgBody | ton::capReportVersion | ton::capShortDequeue | - ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages | ton::capFullCollatedData; + return capCreateStatsEnabled | capBounceMsgBody | capReportVersion | capShortDequeue | capStoreOutMsgQueueSize | + capMsgMetadata | capDeferMessages | capFullCollatedData; } public: - ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - BlockCandidate candidate, td::Ref validator_set, PublicKeyHash local_validator_id, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, unsigned mode = 0); + ValidateQuery(BlockCandidate candidate, ValidateParams params, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise); private: int verbosity{3 * 1}; @@ -153,6 +152,7 @@ class ValidateQuery : public td::actor::Actor { td::BitArray<64> shard_pfx_; int shard_pfx_len_; td::Bits256 created_by_; + Ref optimistic_prev_block_; Ref prev_state_root_; Ref state_root_; @@ -269,6 +269,10 @@ class ValidateQuery : public td::actor::Actor { void alarm() override; void start_up() override; + void load_prev_states(); + bool process_optimistic_prev_block(); + void after_get_shard_state_optimistic(td::Result> res); + bool save_candidate(); void written_candidate(); @@ -290,6 +294,14 @@ class ValidateQuery : public td::actor::Actor { bool is_masterchain() const { return shard_.is_masterchain(); } + int prev_block_idx(const BlockIdExt& id) const { + for (size_t i = 0; i < prev_blocks.size(); ++i) { + if (prev_blocks[i] == id) { + return (int)i; + } + } + return -1; + } td::actor::ActorId get_self() { return actor_id(this); } diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index be57a632d3..eb06dd304a 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -156,8 +156,13 @@ void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector prev, BlockIdExt last, diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 450453788e..5261560e8b 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -126,20 +126,54 @@ void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo } void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block, - td::Promise> promise) { + td::Promise> promise, + td::optional optimistic_prev_block) { if (destroying_) { promise.set_error(td::Status::Error("validator session finished")); return; } + bool is_optimistic = (bool)optimistic_prev_block; + if (is_optimistic && shard_.is_masterchain()) { + promise.set_error(td::Status::Error("no optimistic validation in masterchain")); + return; + } td::uint32 round_id = source_info.priority.round; - update_round_id(round_id); + if (!is_optimistic) { + update_round_id(round_id); + } if (round_id < last_known_round_id_) { promise.set_error(td::Status::Error(ErrorCode::notready, "too old")); return; } + if (is_optimistic && round_id > last_known_round_id_ + 1) { + promise.set_error(td::Status::Error(ErrorCode::notready, "too new")); + return; + } + if (is_optimistic && shard_.is_masterchain()) { + promise.set_error(td::Status::Error("optimistic validation in masterchain is not supported")); + return; + } auto next_block_id = create_next_block_id(block.id.root_hash, block.id.file_hash); block.id = next_block_id; + auto prev = prev_block_ids_; + if (is_optimistic) { + if (round_id > last_known_round_id_) { + ++block.id.id.seqno; + } + optimistic_prev_block.value().id.id = block.id.id; + --optimistic_prev_block.value().id.id.seqno; + if (round_id == last_known_round_id_) { + if (prev_block_ids_ != std::vector{optimistic_prev_block.value().id}) { + promise.set_error(td::Status::Error("wrong prev block for optimistic validation")); + return; + } + optimistic_prev_block = {}; + is_optimistic = false; + } else { + prev = {optimistic_prev_block.value().id}; + } + } CacheKey cache_key = block_to_cache_key(block); auto it = approved_candidates_cache_.find(cache_key); @@ -148,44 +182,60 @@ void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), source_info, block = block.clone(), - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - auto S = R.move_as_error(); - if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { - LOG(ERROR) << "failed to validate candidate: " << S; - } - delay_action( - [SelfId, source_info, block = std::move(block), promise = std::move(promise)]() mutable { - td::actor::send_closure(SelfId, &ValidatorGroup::validate_block_candidate, std::move(source_info), - std::move(block), std::move(promise)); - }, - td::Timestamp::in(0.1)); - } else { - auto v = R.move_as_ok(); - v.visit(td::overloaded( - [&](UnixTime ts) { - td::actor::send_closure(SelfId, &ValidatorGroup::update_approve_cache, block_to_cache_key(block), ts); - td::actor::send_closure(SelfId, &ValidatorGroup::add_available_block_candidate, block.pubkey.as_bits256(), - block.id, block.collated_file_hash); - if (need_send_candidate_broadcast(source_info, block.id.is_masterchain())) { - td::actor::send_closure(SelfId, &ValidatorGroup::send_block_candidate_broadcast, block.id, - block.data.clone()); - } - promise.set_value({ts, false}); - }, - [&](CandidateReject reject) { - promise.set_error( - td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad candidate: " << reject.reason)); - })); - } - }); + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), source_info, block = block.clone(), + optimistic_prev_block = is_optimistic ? optimistic_prev_block.value().clone() : td::optional{}, + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + auto S = R.move_as_error(); + if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { + LOG(ERROR) << "failed to validate candidate: " << S; + } + delay_action( + [SelfId, source_info, block = std::move(block), promise = std::move(promise), + optimistic_prev_block = std::move(optimistic_prev_block)]() mutable { + td::actor::send_closure(SelfId, &ValidatorGroup::validate_block_candidate, std::move(source_info), + std::move(block), std::move(promise), std::move(optimistic_prev_block)); + }, + td::Timestamp::in(0.1)); + } else { + auto v = R.move_as_ok(); + v.visit(td::overloaded( + [&](UnixTime ts) { + td::actor::send_closure(SelfId, &ValidatorGroup::update_approve_cache, block_to_cache_key(block), ts); + td::actor::send_closure(SelfId, &ValidatorGroup::add_available_block_candidate, + block.pubkey.as_bits256(), block.id, block.collated_file_hash); + if (need_send_candidate_broadcast(source_info, block.id.is_masterchain())) { + td::actor::send_closure(SelfId, &ValidatorGroup::send_block_candidate_broadcast, block.id, + block.data.clone()); + } + promise.set_value({ts, false}); + }, + [&](CandidateReject reject) { + promise.set_error( + td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad candidate: " << reject.reason)); + })); + } + }); if (!started_) { P.set_error(td::Status::Error(ErrorCode::notready, "validator group not started")); return; } VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id; - run_validate_query(shard_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, local_id_, + td::Ref optimistic_prev_block_data; + if (is_optimistic) { + TRY_RESULT_PROMISE_PREFIX_ASSIGN( + P, optimistic_prev_block_data, + create_block(optimistic_prev_block.value().id, std::move(optimistic_prev_block.value().data)), + "failed to parse optimistic prev block: "); + } + run_validate_query(std::move(block), + ValidateParams{.shard = shard_, + .min_masterchain_block_id = min_masterchain_block_id_, + .prev = std::move(prev), + .validator_set = validator_set_, + .local_validator_id = local_id_, + .optimistic_prev_block = optimistic_prev_block_data}, manager_, td::Timestamp::in(15.0), std::move(P)); } @@ -254,7 +304,6 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so std::move(approve_sig_set), send_broadcast_mode, std::move(promise)); prev_block_ids_ = std::vector{next_block_id}; cached_collated_block_ = nullptr; - approved_candidates_cache_.clear(); cancellation_token_source_.cancel(); if (optimistic_generation_ && optimistic_generation_->round == last_known_round_id_ && optimistic_generation_->prev != next_block_id) { @@ -405,7 +454,7 @@ std::unique_ptr ValidatorGroup::ma sha256_bits256(collated_data.as_slice()), data.clone(), collated_data.clone()}; td::actor::send_closure(id_, &ValidatorGroup::validate_block_candidate, std::move(source_info), - std::move(candidate), std::move(P)); + std::move(candidate), std::move(P), td::optional{}); } void on_generate_slot(validatorsession::BlockSourceInfo source_info, td::Promise promise) override { @@ -447,6 +496,23 @@ std::unique_ptr ValidatorGroup::ma td::actor::send_closure(id_, &ValidatorGroup::generate_block_optimistic, source_info, std::move(prev_block), prev_root_hash, prev_file_hash, std::move(promise)); } + void on_optimistic_candidate(validatorsession::BlockSourceInfo source_info, + validatorsession::ValidatorSessionRootHash root_hash, td::BufferSlice data, + td::BufferSlice collated_data, PublicKey prev_source, + validatorsession::ValidatorSessionRootHash prev_root_hash, td::BufferSlice prev_data, + td::BufferSlice prev_collated_data) override { + BlockCandidate candidate{Ed25519_PublicKey{source_info.source.ed25519_value().raw()}, + BlockIdExt{0, 0, 0, root_hash, sha256_bits256(data.as_slice())}, + sha256_bits256(collated_data.as_slice()), data.clone(), collated_data.clone()}; + BlockCandidate prev_candidate{Ed25519_PublicKey{prev_source.ed25519_value().raw()}, + BlockIdExt{0, 0, 0, prev_root_hash, sha256_bits256(prev_data.as_slice())}, + sha256_bits256(prev_collated_data.as_slice()), prev_data.clone(), + prev_collated_data.clone()}; + + td::actor::send_closure( + id_, &ValidatorGroup::validate_block_candidate, std::move(source_info), std::move(candidate), + [](td::Result>) mutable {}, std::move(prev_candidate)); + } private: td::actor::ActorId id_; @@ -511,7 +577,6 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch prev_block_ids_ = prev; min_masterchain_block_id_ = min_masterchain_block_id; cached_collated_block_ = nullptr; - approved_candidates_cache_.clear(); started_ = true; if (init_) { diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 17bd23a4df..56d89dda45 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -40,7 +40,8 @@ class ValidatorGroup : public td::actor::Actor { void generate_block_candidate_cont(validatorsession::BlockSourceInfo source_info, td::Promise promise, td::CancellationToken cancellation_token); void validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block, - td::Promise> promise); + td::Promise> promise, + td::optional optimistic_prev_block); void accept_block_candidate(validatorsession::BlockSourceInfo source_info, td::BufferSlice block, RootHash root_hash, FileHash file_hash, std::vector signatures, std::vector approve_signatures, From ff010ef78ad7cc568363494e4f4abade60545033 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 19 Aug 2025 19:29:03 +0300 Subject: [PATCH 381/524] More session stats for collation/validation time --- tl/generate/scheme/ton_api.tl | 16 +- tl/generate/scheme/ton_api.tlo | Bin 119800 -> 118952 bytes validator/impl/collator-impl.h | 2 +- validator/impl/collator.cpp | 23 +-- validator/impl/validate-query.cpp | 209 ++++++++++++++--------- validator/impl/validate-query.hpp | 24 +-- validator/interfaces/validator-manager.h | 28 +-- 7 files changed, 175 insertions(+), 127 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 3f07579933..6e1a0fc4f0 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -962,10 +962,6 @@ validatorStats.blockStats ext_msgs:validatorStats.blockStats.extMsgsStats transactions:int shard_configuration:(vector tonNode.blockIdExt) old_out_msg_queue_size:long new_out_msg_queue_size:long msg_queue_cleaned:int neighbors:(vector validatorStats.blockStats.neighborStats) = validatorStats.BlockStats; -validatorStats.collateWorkTimeStats - total:double optimistic_apply:double queue_cleanup:double prelim_storage_stat:double trx_tvm:double trx_storage_stat:double - trx_other:double final_storage_stat:double create_block:double create_collated_data:double create_block_candidate:double - = validatorStats.CollateWorkTimeStats; validatorStats.storageStatCacheStats small_cnt:long small_cells:long hit_cnt:long hit_cells:long miss_cnt:long miss_cells:long = validatorStats.StorageStatCacheStats; @@ -974,22 +970,20 @@ validatorStats.collatedBlock bytes:int collated_data_bytes:int attempt:int self:int256 is_validator:Bool total_time:double work_time:double cpu_work_time:double time_stats:string - work_time_real_stats:validatorStats.collateWorkTimeStats - work_time_cpu_stats:validatorStats.collateWorkTimeStats + work_time_real_stats:string + work_time_cpu_stats:string block_limits:validatorStats.blockLimitsStatus block_stats:validatorStats.blockStats storage_stat_cache:validatorStats.storageStatCacheStats = validatorSession.stats.CollatedBlock; -validatorStats.validateWorkTimeStats - total:double trx_tvm:double trx_storage_stat:double trx_other:double = validatorStats.ValidateWorkTimeStats; validatorStats.validatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 validated_at:double self:int256 valid:Bool comment:string bytes:int collated_data_bytes:int - total_time:double work_time:double cpu_work_time:double - work_time_real_stats:validatorStats.validateWorkTimeStats - work_time_cpu_stats:validatorStats.validateWorkTimeStats + total_time:double work_time:double cpu_work_time:double time_stats:string + work_time_real_stats:string + work_time_cpu_stats:string storage_stat_cache:validatorStats.storageStatCacheStats = validatorStats.ValidatedBlock; validatorStats.newValidatorGroup.node id:int256 pubkey:PublicKey adnl_id:int256 weight:long = validatorStats.newValidatorGroup.Node; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index ce74dbb37275ffa2745966d24b2e1a77a6d32679..ee5fb5ce817f1079c69a32c1625e810f0248b3a2 100644 GIT binary patch delta 200 zcmeydoPEVYcHT#`^{p77zOBfmNH77c6 zPjqIyFAY{P-6ft;ffJEjg>m{BCPsnjGfEg`CjU4n zFnvQJV+~l_c7r6w9W2{_WH6rK02@AiNfFQ-kToE4_>s&}C}#9voNkcF=z?b0pF~Cj Lu-@fAN*Ln-c%w~@ delta 712 zcmZ3nkp0JUcHT#`^{p77z;h$-dpnlYQymW{*?&=aHwmj>HA{XiU}0w>4?H;)PV2u*(2qCGvQlu=}QPCBE=Gr0r`G8F#R-f;}<)U>u{EGR)CnXMn^( zR&WEoF8%DjCTmH4Nn+0Aclpu+`2{7JxtYZ!naS~q1qC^k(;sp(O7rm+mZp}b#wX{b zCgznEOn#RwB_do?QAQkDaeQ)OUJ5AcQW+*I9FqZu&i05j#w#4G;JBJ@n9nG#h7u;gpa4mMf`tVw zShA&v@DvZiQw(5BML@ saved); + void return_block_candidate(td::Result saved, td::PerfLogAction token); bool update_last_proc_int_msg(const std::pair& new_lt_hash); td::CancellationToken cancellation_token_; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index e38d30e1ff..608c8f67aa 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -573,7 +573,7 @@ bool Collator::request_aux_mc_state(BlockSeqno seqno, Ref& st CHECK(blkid.is_valid_ext() && blkid.is_masterchain()); LOG(DEBUG) << "sending auxiliary wait_block_state() query for " << blkid.to_str() << " to Manager"; ++pending; - auto token = perf_log_.start_action(PSTRING() << "auxiliary wait_block_state " << blkid.to_str()); + auto token = perf_log_.start_action(PSTRING() << "auxiliary wait_block_state " << blkid.seqno()); td::actor::send_closure_later( manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout, false, [self = get_self(), blkid, token = std::move(token)](td::Result> res) mutable { @@ -6492,16 +6492,18 @@ bool Collator::create_block_candidate() { } // 4. save block candidate if (skip_store_candidate_) { - td::actor::send_closure_later(actor_id(this), &Collator::return_block_candidate, td::Unit()); + td::actor::send_closure_later(actor_id(this), &Collator::return_block_candidate, td::Unit(), td::PerfLogAction{}); } else { LOG(INFO) << "saving new BlockCandidate"; - td::actor::send_closure_later( - manager, &ValidatorManager::set_block_candidate, block_candidate->id, block_candidate->clone(), - validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), - [self = get_self()](td::Result saved) -> void { - LOG(DEBUG) << "got answer to set_block_candidate"; - td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, std::move(saved)); - }); + auto token = perf_log_.start_action("set_block_candidate"); + td::actor::send_closure_later(manager, &ValidatorManager::set_block_candidate, block_candidate->id, + block_candidate->clone(), validator_set_->get_catchain_seqno(), + validator_set_->get_validator_set_hash(), + [self = get_self(), token = std::move(token)](td::Result saved) mutable { + LOG(DEBUG) << "got answer to set_block_candidate"; + td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, + std::move(saved), std::move(token)); + }); } // 5. communicate about bad and delayed external messages if (!bad_ext_msgs_.empty() || !delay_ext_msgs_.empty()) { @@ -6521,8 +6523,9 @@ bool Collator::create_block_candidate() { * * @param saved The result of saving the block candidate to the disk. */ -void Collator::return_block_candidate(td::Result saved) { +void Collator::return_block_candidate(td::Result saved, td::PerfLogAction token) { // 6. return data to the original "caller" + token.finish(saved); if (saved.is_error()) { auto err = saved.move_as_error(); LOG(ERROR) << "cannot save block candidate: " << err.to_string(); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index adcf72eb25..4285a385cd 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -365,10 +365,12 @@ void ValidateQuery::start_up() { LOG(DEBUG) << "sending get_top_masterchain_state_block() to Manager"; ++pending; td::actor::send_closure_later(manager, &ValidatorManager::get_top_masterchain_state_block, - [self = get_self()](td::Result, BlockIdExt>> res) { + [self = get_self(), token = perf_log_.start_action("get_top_masterchain_state_block")]( + td::Result, BlockIdExt>> res) mutable { LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; - td::actor::send_closure_later( - std::move(self), &ValidateQuery::after_get_latest_mc_state, std::move(res)); + td::actor::send_closure_later(std::move(self), + &ValidateQuery::after_get_latest_mc_state, + std::move(res), std::move(token)); }); // 3. unpack block candidate (while necessary data is being loaded) if (!unpack_block_candidate()) { @@ -389,12 +391,13 @@ void ValidateQuery::start_up() { // 4. request masterchain handle and state referred to in the block if (!is_masterchain()) { ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::get_block_handle, mc_blkid_, true, - [self = get_self()](td::Result res) { - LOG(DEBUG) << "got answer to get_block_handle() query for masterchain block"; - td::actor::send_closure_later(std::move(self), &ValidateQuery::got_mc_handle, - std::move(res)); - }); + td::actor::send_closure_later( + manager, &ValidatorManager::get_block_handle, mc_blkid_, true, + [self = get_self(), token = perf_log_.start_action("get_block_handle")](td::Result res) mutable { + LOG(DEBUG) << "got answer to get_block_handle() query for masterchain block"; + td::actor::send_closure_later(std::move(self), &ValidateQuery::got_mc_handle, std::move(res), + std::move(token)); + }); } else { if (prev_blocks[0] != mc_blkid_) { soft_reject_query("cannot validate masterchain block "s + id_.to_str() + @@ -406,12 +409,14 @@ void ValidateQuery::start_up() { // 5. get storage stat cache ++pending; LOG(DEBUG) << "sending get_storage_stat_cache() query to Manager"; - td::actor::send_closure_later( - manager, &ValidatorManager::get_storage_stat_cache, - [self = get_self()](td::Result(const td::Bits256&)>> res) { - LOG(DEBUG) << "got answer to get_storage_stat_cache() query"; - td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_storage_stat_cache, std::move(res)); - }); + td::actor::send_closure_later(manager, &ValidatorManager::get_storage_stat_cache, + [self = get_self(), token = perf_log_.start_action("get_storage_stat_cache")]( + td::Result(const td::Bits256&)>> res) mutable { + LOG(DEBUG) << "got answer to get_storage_stat_cache() query"; + td::actor::send_closure_later(std::move(self), + &ValidateQuery::after_get_storage_stat_cache, + std::move(res), std::move(token)); + }); // ... CHECK(pending); } @@ -424,12 +429,14 @@ void ValidateQuery::load_prev_states() { // 4.1. load state LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), - timeout, false, [self = get_self(), i](td::Result> res) -> void { - LOG(DEBUG) << "got answer to wait_block_state_short query #" << i; - td::actor::send_closure_later( - std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res)); - }); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), timeout, false, + [self = get_self(), i, token = perf_log_.start_action(PSTRING() << "wait_block_state #" << i)]( + td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state_short query #" << i; + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res), + std::move(token)); + }); } } @@ -449,25 +456,27 @@ bool ValidateQuery::process_optimistic_prev_block() { if (prev_prev.size() == 1) { LOG(DEBUG) << "sending wait_block_state() query for " << prev_prev[0].to_str() << " to Manager (opt)"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_prev[0], priority(), timeout, - false, [self = get_self()](td::Result> res) mutable { - LOG(DEBUG) << "got answer to wait_block_state query (opt)"; - td::actor::send_closure_later(std::move(self), - &ValidateQuery::after_get_shard_state_optimistic, - std::move(res)); - }); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, prev_prev[0], priority(), timeout, false, + [self = get_self(), + token = perf_log_.start_action("opt wait_block_state")](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query (opt)"; + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_shard_state_optimistic, + std::move(res), std::move(token)); + }); } else { CHECK(prev_prev.size() == 2); LOG(DEBUG) << "sending wait_block_state_merge() query for " << prev_prev[0].to_str() << " and " << prev_prev[1].to_str() << " to Manager (opt)"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_merge, prev_prev[0], prev_prev[1], - priority(), timeout, [self = get_self()](td::Result> res) mutable { - LOG(DEBUG) << "got answer to wait_block_state_merge query (opt)"; - td::actor::send_closure_later(std::move(self), - &ValidateQuery::after_get_shard_state_optimistic, - std::move(res)); - }); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_merge, prev_prev[0], prev_prev[1], priority(), timeout, + [self = get_self(), + token = perf_log_.start_action("opt wait_block_state_merge")](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state_merge query (opt)"; + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_shard_state_optimistic, + std::move(res), std::move(token)); + }); } return true; } @@ -477,12 +486,14 @@ bool ValidateQuery::process_optimistic_prev_block() { * * @param res The retrieved state. */ -void ValidateQuery::after_get_shard_state_optimistic(td::Result> res) { +void ValidateQuery::after_get_shard_state_optimistic(td::Result> res, td::PerfLogAction token) { + token.finish(res); LOG(DEBUG) << "in ValidateQuery::after_get_shard_state_optimistic()"; if (res.is_error()) { fatal_error(res.move_as_error()); return; } + td::RealCpuTimer timer; work_timer_.resume(); auto state = res.move_as_ok(); auto S = state.write().apply_block(optimistic_prev_block_->block_id(), optimistic_prev_block_); @@ -491,7 +502,8 @@ void ValidateQuery::after_get_shard_state_optimistic(td::Result> return; } work_timer_.pause(); - after_get_shard_state(0, std::move(state)); + stats_.work_time.optimistic_apply = timer.elapsed_both(); + after_get_shard_state(0, std::move(state), {}); } /** @@ -783,10 +795,12 @@ bool ValidateQuery::extract_collated_data() { void ValidateQuery::request_latest_mc_state() { ++pending; td::actor::send_closure_later(manager, &ValidatorManager::get_top_masterchain_state_block, - [self = get_self()](td::Result, BlockIdExt>> res) { + [self = get_self(), token = perf_log_.start_action("get_top_masterchain_state_block")]( + td::Result, BlockIdExt>> res) mutable { LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; - td::actor::send_closure_later( - std::move(self), &ValidateQuery::after_get_latest_mc_state, std::move(res)); + td::actor::send_closure_later(std::move(self), + &ValidateQuery::after_get_latest_mc_state, + std::move(res), std::move(token)); }); } @@ -795,7 +809,9 @@ void ValidateQuery::request_latest_mc_state() { * * @param res The result of the retrieval of the latest masterchain state. */ -void ValidateQuery::after_get_latest_mc_state(td::Result, BlockIdExt>> res) { +void ValidateQuery::after_get_latest_mc_state(td::Result, BlockIdExt>> res, + td::PerfLogAction token) { + token.finish(res); LOG(WARNING) << "in ValidateQuery::after_get_latest_mc_state()"; --pending; if (res.is_error()) { @@ -836,7 +852,8 @@ void ValidateQuery::after_get_latest_mc_state(td::Result> res) { +void ValidateQuery::after_get_mc_state(td::Result> res, td::PerfLogAction token) { + token.finish(res); CHECK(!is_masterchain()); LOG(WARNING) << "in ValidateQuery::after_get_mc_state() for " << mc_blkid_.to_str(); --pending; @@ -861,7 +878,8 @@ void ValidateQuery::after_get_mc_state(td::Result> res) { * * @param res The result of retrieving the masterchain block handle. */ -void ValidateQuery::got_mc_handle(td::Result res) { +void ValidateQuery::got_mc_handle(td::Result res, td::PerfLogAction token) { + token.finish(res); LOG(DEBUG) << "in ValidateQuery::got_mc_handle() for " << mc_blkid_.to_str(); if (res.is_error()) { fatal_error(res.move_as_error()); @@ -870,13 +888,15 @@ void ValidateQuery::got_mc_handle(td::Result res) { auto mc_handle = res.move_as_ok(); td::actor::send_closure_later( manager, &ValidatorManager::wait_block_state, mc_handle, priority(), timeout, false, - [self = get_self(), id = id_, mc_handle](td::Result> res) { + [self = get_self(), id = id_, mc_handle, + token = perf_log_.start_action("mc wait_block_state")](td::Result> res) mutable { LOG(DEBUG) << "got answer to wait_block_state() query for masterchain block"; if (res.is_ok() && mc_handle->id().seqno() > 0 && !mc_handle->inited_proof()) { res = td::Status::Error(-666, "reference masterchain block "s + mc_handle->id().to_str() + " for block " + id.to_str() + " does not have a valid proof"); } - td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_mc_state, std::move(res)); + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_mc_state, std::move(res), + std::move(token)); }); } @@ -885,7 +905,9 @@ void ValidateQuery::got_mc_handle(td::Result res) { * * @param res The retrieved storage stat cache. */ -void ValidateQuery::after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res) { +void ValidateQuery::after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res, + td::PerfLogAction token) { + token.finish(res); --pending; if (res.is_error()) { LOG(INFO) << "after_get_storage_stat_cache : " << res.error(); @@ -906,7 +928,8 @@ void ValidateQuery::after_get_storage_stat_cache(td::Result> res) { +void ValidateQuery::after_get_shard_state(int idx, td::Result> res, td::PerfLogAction token) { + token.finish(res); LOG(WARNING) << "in ValidateQuery::after_get_shard_state(" << idx << ")"; --pending; if (res.is_error()) { @@ -1684,7 +1707,8 @@ bool ValidateQuery::request_neighbor_queues() { return fatal_error("neighbor from masterchain is not the last mc block"); } ++pending; - send_closure_later(get_self(), &ValidateQuery::got_neighbor_out_queue, i, mc_state_->message_queue()); + send_closure_later(get_self(), &ValidateQuery::got_neighbor_out_queue, i, mc_state_->message_queue(), + td::PerfLogAction{}); ++i; continue; } @@ -1697,7 +1721,8 @@ bool ValidateQuery::request_neighbor_queues() { return reject_query("cannot fetch shard state from collated data", state.move_as_error()); } ++pending; - send_closure_later(get_self(), &ValidateQuery::got_neighbor_out_queue, i, state.move_as_ok()->message_queue()); + send_closure_later(get_self(), &ValidateQuery::got_neighbor_out_queue, i, state.move_as_ok()->message_queue(), + td::PerfLogAction{}); ++i; } } else { @@ -1706,13 +1731,15 @@ bool ValidateQuery::request_neighbor_queues() { ++pending; if (int prev_idx = prev_block_idx(descr.blk_); prev_idx >= 0) { td::actor::send_closure(actor_id(this), &ValidateQuery::got_neighbor_out_queue, i, - prev_states.at(prev_idx)->message_queue()); + prev_states.at(prev_idx)->message_queue(), td::PerfLogAction{}); } else { - send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, - [self = get_self(), i](td::Result> res) { - td::actor::send_closure(std::move(self), &ValidateQuery::got_neighbor_out_queue, i, - std::move(res)); - }); + send_closure_later( + manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, + [self = get_self(), i, token = perf_log_.start_action(PSTRING() << "wait_block_message_queue #" << i)]( + td::Result> res) mutable { + td::actor::send_closure(std::move(self), &ValidateQuery::got_neighbor_out_queue, i, std::move(res), + std::move(token)); + }); } ++i; } @@ -1727,7 +1754,8 @@ bool ValidateQuery::request_neighbor_queues() { * @param i The index of the neighbor. * @param res The obtained outbound queue. */ -void ValidateQuery::got_neighbor_out_queue(int i, td::Result> res) { +void ValidateQuery::got_neighbor_out_queue(int i, td::Result> res, td::PerfLogAction token) { + token.finish(res); --pending; if (res.is_error()) { fatal_error(res.move_as_error()); @@ -1854,13 +1882,15 @@ bool ValidateQuery::request_aux_mc_state(BlockSeqno seqno, Ref> res) { - LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str(); - td::actor::send_closure_later(std::move(self), - &ValidateQuery::after_get_aux_shard_state, blkid, - std::move(res)); - }); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout, false, + [self = get_self(), blkid, + token = perf_log_.start_action(PSTRING() << "auxiliary wait_block_state " << blkid.seqno())]( + td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str(); + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_aux_shard_state, blkid, std::move(res), + std::move(token)); + }); state.clear(); return true; } @@ -1890,7 +1920,9 @@ Ref ValidateQuery::get_aux_mc_state(BlockSeqno seqno) const { * @param blkid The BlockIdExt of the shard state. * @param res The result of retrieving the shard state. */ -void ValidateQuery::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res) { +void ValidateQuery::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res, + td::PerfLogAction token) { + token.finish(res); LOG(DEBUG) << "in ValidateQuery::after_get_aux_shard_state(" << blkid.to_str() << ")"; --pending; if (res.is_error()) { @@ -2286,11 +2318,12 @@ bool ValidateQuery::check_shard_layout() { } if (!new_top_shard_blocks.empty()) { ++pending; - td::actor::send_closure(manager, &ValidatorManager::wait_verify_shard_blocks, std::move(new_top_shard_blocks), - [SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ValidateQuery::verified_shard_blocks, - R.move_as_status()); - }); + td::actor::send_closure( + manager, &ValidatorManager::wait_verify_shard_blocks, std::move(new_top_shard_blocks), + [SelfId = actor_id(this), + token = perf_log_.start_action("wait_verify_shard_blocks")](td::Result R) mutable { + td::actor::send_closure(SelfId, &ValidateQuery::verified_shard_blocks, R.move_as_status(), std::move(token)); + }); } return check_mc_validator_info(is_key_block_ || (now_ / ccvc.mc_cc_lifetime > prev_now_ / ccvc.mc_cc_lifetime)); } @@ -2465,11 +2498,13 @@ bool ValidateQuery::prepare_out_msg_queue_size() { out_msg_queue_size_known_ = true; for (size_t i = 0; i < prev_blocks.size(); ++i) { ++pending; - send_closure_later(manager, &ValidatorManager::get_out_msg_queue_size, prev_blocks[i], - [self = get_self(), i](td::Result res) { - td::actor::send_closure(std::move(self), &ValidateQuery::got_out_queue_size, i, - std::move(res)); - }); + send_closure_later( + manager, &ValidatorManager::get_out_msg_queue_size, prev_blocks[i], + [self = get_self(), i, token = perf_log_.start_action(PSTRING() << "get_out_msg_queue_size #" << i)]( + td::Result res) mutable { + td::actor::send_closure(std::move(self), &ValidateQuery::got_out_queue_size, i, std::move(res), + std::move(token)); + }); } return true; } @@ -2482,7 +2517,8 @@ bool ValidateQuery::prepare_out_msg_queue_size() { * @param i The index of the previous block (0 or 1). * @param res The result object containing the size of the queue. */ -void ValidateQuery::got_out_queue_size(size_t i, td::Result res) { +void ValidateQuery::got_out_queue_size(size_t i, td::Result res, td::PerfLogAction token) { + token.finish(res); --pending; if (res.is_error()) { fatal_error( @@ -2502,7 +2538,8 @@ void ValidateQuery::got_out_queue_size(size_t i, td::Result res) { * * @param S The status of the operation (OK on success). */ -void ValidateQuery::verified_shard_blocks(td::Status S) { +void ValidateQuery::verified_shard_blocks(td::Status S, td::PerfLogAction token) { + token.finish(S); --pending; if (S.is_error()) { fatal_error(S.move_as_error_prefix("failed to verify shard blocks: ")); @@ -7254,13 +7291,14 @@ bool ValidateQuery::try_validate() { * @returns True. */ bool ValidateQuery::save_candidate() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &ValidateQuery::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &ValidateQuery::written_candidate); - } - }); + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), token = perf_log_.start_action("set_block_candidate")](td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ValidateQuery::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ValidateQuery::written_candidate, std::move(token)); + } + }); td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, id_, block_candidate.clone(), validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), std::move(P)); @@ -7275,7 +7313,8 @@ bool ValidateQuery::save_candidate() { * Callback function called after saving block candidate. * Finishes validation. */ -void ValidateQuery::written_candidate() { +void ValidateQuery::written_candidate(td::PerfLogAction token) { + token.finish(td::Status::OK()); finish_query(); } @@ -7297,9 +7336,11 @@ void ValidateQuery::record_stats(bool valid, std::string error_message) { stats_.actual_collated_data_bytes = (td::uint32)block_candidate.collated_data.size(); stats_.total_time = perf_timer_.elapsed(); stats_.work_time.total = work_timer_.elapsed_both(); + stats_.time_stats = (PSTRING() << perf_log_); LOG(WARNING) << "validation took " << perf_timer_.elapsed() << "s"; LOG(WARNING) << "Validate query work time = " << stats_.work_time.total.real << "s, cpu time = " << stats_.work_time.total.cpu << "s"; + LOG(WARNING) << perf_log_; td::actor::send_closure(manager, &ValidatorManager::log_validate_query_stats, std::move(stats_)); } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 29cdbd8d29..df19b24c1c 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -253,6 +253,7 @@ class ValidateQuery : public td::actor::Actor { bool have_unprocessed_account_dispatch_queue_ = false; td::PerfWarningTimer perf_timer_; + td::PerfLog perf_log_; static constexpr td::uint32 priority() { return 2; @@ -271,10 +272,10 @@ class ValidateQuery : public td::actor::Actor { void load_prev_states(); bool process_optimistic_prev_block(); - void after_get_shard_state_optimistic(td::Result> res); + void after_get_shard_state_optimistic(td::Result> res, td::PerfLogAction token); bool save_candidate(); - void written_candidate(); + void written_candidate(td::PerfLogAction token); bool fatal_error(td::Status error); bool fatal_error(int err_code, std::string err_msg); @@ -307,11 +308,12 @@ class ValidateQuery : public td::actor::Actor { } void request_latest_mc_state(); - void after_get_latest_mc_state(td::Result, BlockIdExt>> res); - void after_get_mc_state(td::Result> res); - void got_mc_handle(td::Result res); - void after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res); - void after_get_shard_state(int idx, td::Result> res); + void after_get_latest_mc_state(td::Result, BlockIdExt>> res, td::PerfLogAction token); + void after_get_mc_state(td::Result> res, td::PerfLogAction token); + void got_mc_handle(td::Result res, td::PerfLogAction token); + void after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res, + td::PerfLogAction token); + void after_get_shard_state(int idx, td::Result> res, td::PerfLogAction token); bool process_mc_state(Ref mc_state); bool try_unpack_mc_state(); bool fetch_config_params(); @@ -331,12 +333,12 @@ class ValidateQuery : public td::actor::Actor { bool unpack_one_prev_state(block::ShardState& ss, BlockIdExt blkid, Ref prev_state_root); bool split_prev_state(block::ShardState& ss); bool request_neighbor_queues(); - void got_neighbor_out_queue(int i, td::Result> res); + void got_neighbor_out_queue(int i, td::Result> res, td::PerfLogAction token); bool register_mc_state(Ref other_mc_state); bool request_aux_mc_state(BlockSeqno seqno, Ref& state); Ref get_aux_mc_state(BlockSeqno seqno) const; - void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res); + void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res, td::PerfLogAction token); bool check_one_shard(const block::McShardHash& info, const block::McShardHash* sibling, const block::WorkchainInfo* wc_info, const block::CatchainValidatorsConfig& ccvc, bool& is_new); @@ -346,8 +348,8 @@ class ValidateQuery : public td::actor::Actor { bool check_mc_validator_info(bool update_mc_cc); bool check_utime_lt(); bool prepare_out_msg_queue_size(); - void got_out_queue_size(size_t i, td::Result res); - void verified_shard_blocks(td::Status S); + void got_out_queue_size(size_t i, td::Result res, td::PerfLogAction token); + void verified_shard_blocks(td::Status S, td::PerfLogAction token); bool fix_one_processed_upto(block::MsgProcessedUpto& proc, ton::ShardIdFull owner, bool allow_cur = false); bool fix_processed_upto(block::MsgProcessedUptoCollection& upto, bool allow_cur = false); diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 90e11a35ba..ade27c4eee 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -128,11 +128,16 @@ struct CollationStats { td::RealCpuTimer::Time create_collated_data; td::RealCpuTimer::Time create_block_candidate; - tl_object_ptr tl(bool is_cpu) const { - return create_tl_object( - total.get(is_cpu), optimistic_apply.get(is_cpu), queue_cleanup.get(is_cpu), prelim_storage_stat.get(is_cpu), - trx_tvm.get(is_cpu), trx_storage_stat.get(is_cpu), trx_other.get(is_cpu), final_storage_stat.get(is_cpu), - create_block.get(is_cpu), create_collated_data.get(is_cpu), create_block_candidate.get(is_cpu)); + std::string to_str(bool is_cpu) const { + return PSTRING() << "total=" << total.get(is_cpu) << " optimistic_apply=" << optimistic_apply.get(is_cpu) + << " queue_cleanup=" << queue_cleanup.get(is_cpu) + << " prelim_storage_stat=" << prelim_storage_stat.get(is_cpu) + << " trx_tvm=" << trx_tvm.get(is_cpu) << " trx_storage_stat=" << trx_storage_stat.get(is_cpu) + << " trx_other=" << trx_other.get(is_cpu) + << " final_storage_stat=" << final_storage_stat.get(is_cpu) + << " create_block=" << create_block.get(is_cpu) + << " create_collated_data=" << create_collated_data.get(is_cpu) + << " create_block_candidate=" << create_block_candidate.get(is_cpu); } }; WorkTimeStats work_time; @@ -155,7 +160,7 @@ struct CollationStats { return create_tl_object( create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes, actual_collated_data_bytes, attempt, self.bits256_value(), is_validator, total_time, work_time.total.real, - work_time.total.cpu, time_stats, work_time.tl(false), work_time.tl(true), + work_time.total.cpu, time_stats, work_time.to_str(false), work_time.to_str(true), create_tl_object( estimated_bytes, gas, lt_delta, estimated_collated_data_bytes, cat_bytes, cat_gas, cat_lt_delta, cat_collated_data_bytes, load_fraction_queue_cleanup, load_fraction_dispatch, load_fraction_internals, @@ -173,16 +178,19 @@ struct ValidationStats { std::string comment; td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0; double total_time = 0.0; + std::string time_stats; struct WorkTimeStats { td::RealCpuTimer::Time total; + td::RealCpuTimer::Time optimistic_apply; td::RealCpuTimer::Time trx_tvm; td::RealCpuTimer::Time trx_storage_stat; td::RealCpuTimer::Time trx_other; - tl_object_ptr tl(bool is_cpu) const { - return create_tl_object( - total.get(is_cpu), trx_tvm.get(is_cpu), trx_storage_stat.get(is_cpu), trx_other.get(is_cpu)); + std::string to_str(bool is_cpu) const { + return PSTRING() << "total=" << total.get(is_cpu) << " optimistic_apply=" << optimistic_apply.get(is_cpu) + << " trx_tvm=" << trx_tvm.get(is_cpu) << " trx_storage_stat=" << trx_storage_stat.get(is_cpu) + << " trx_other=" << trx_other.get(is_cpu); } }; WorkTimeStats work_time; @@ -192,7 +200,7 @@ struct ValidationStats { return create_tl_object( create_tl_block_id(block_id), collated_data_hash, validated_at, self.bits256_value(), valid, comment, actual_bytes, actual_collated_data_bytes, total_time, work_time.total.real, work_time.total.cpu, - work_time.tl(false), work_time.tl(true), storage_stat_cache.tl()); + time_stats, work_time.to_str(false), work_time.to_str(true), storage_stat_cache.tl()); } }; From 9b987d48b198184b37e3c5957003f99efbbced18 Mon Sep 17 00:00:00 2001 From: birydrad <> Date: Mon, 8 Sep 2025 14:15:05 +0200 Subject: [PATCH 382/524] coroutines support in actors --- CMakeLists.txt | 16 +- assembly/native/build-macos-portable.sh | 2 +- assembly/native/build-macos-shared.sh | 2 +- assembly/native/build-ubuntu-appimages.sh | 2 +- assembly/native/build-ubuntu-portable.sh | 2 +- assembly/native/build-ubuntu-shared.sh | 2 +- assembly/native/build-windows-2019.bat | 2 +- assembly/native/build-windows-2022.bat | 2 +- assembly/native/build-windows.bat | 2 +- tdactor/CMakeLists.txt | 5 + tdactor/benchmark/CMakeLists.txt | 65 ++ tdactor/benchmark/benchmark-coro.cpp | 126 +++ tdactor/benchmark/benchmark.cpp | 4 + tdactor/benchmark/gbench-coro-folly.cpp | 105 ++ tdactor/benchmark/gbench-coro-yaclib.cpp | 189 ++++ tdactor/benchmark/gbench-coro.cpp | 1001 ++++++++++++++++++ tdactor/example/actor-example-coroutines.cpp | 182 ++++ tdactor/td/actor/PromiseFuture.h | 29 + tdactor/td/actor/actor.h | 132 ++- tdactor/td/actor/common.h | 105 +- tdactor/td/actor/core/ActorExecuteContext.h | 3 + tdactor/td/actor/core/CpuWorker.cpp | 50 +- tdactor/td/actor/core/CpuWorker.h | 16 +- tdactor/td/actor/core/Scheduler.cpp | 56 +- tdactor/td/actor/core/Scheduler.h | 7 +- tdactor/td/actor/core/SchedulerContext.h | 5 + tdactor/td/actor/coro.h | 11 + tdactor/td/actor/coro_awaitables.h | 234 ++++ tdactor/td/actor/coro_executor.h | 237 +++++ tdactor/td/actor/coro_task.h | 469 ++++++++ tdactor/td/actor/coro_types.h | 114 ++ tdactor/td/actor/coro_utils.h | 268 +++++ tdactor/test/CMakeLists.txt | 3 + tdactor/test/actors_core.cpp | 12 +- tdactor/test/test-coro.cpp | 877 +++++++++++++++ tdutils/td/utils/Closure.h | 4 +- tdutils/td/utils/Status.h | 29 + 37 files changed, 4206 insertions(+), 164 deletions(-) create mode 100644 tdactor/benchmark/benchmark-coro.cpp create mode 100644 tdactor/benchmark/gbench-coro-folly.cpp create mode 100644 tdactor/benchmark/gbench-coro-yaclib.cpp create mode 100644 tdactor/benchmark/gbench-coro.cpp create mode 100644 tdactor/example/actor-example-coroutines.cpp create mode 100644 tdactor/td/actor/coro.h create mode 100644 tdactor/td/actor/coro_awaitables.h create mode 100644 tdactor/td/actor/coro_executor.h create mode 100644 tdactor/td/actor/coro_task.h create mode 100644 tdactor/td/actor/coro_types.h create mode 100644 tdactor/td/actor/coro_utils.h create mode 100644 tdactor/test/CMakeLists.txt create mode 100644 tdactor/test/test-coro.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index c37f351d8c..f042368c10 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,6 +56,7 @@ option(TONLIB_ENABLE_JNI "Use \"ON\" to enable JNI-compatible TonLib API.") option(TON_USE_ASAN "Use \"ON\" to enable AddressSanitizer." OFF) option(TON_USE_TSAN "Use \"ON\" to enable ThreadSanitizer." OFF) option(TON_USE_UBSAN "Use \"ON\" to enable UndefinedBehaviorSanitizer." OFF) +option(TON_USE_COVERAGE "Use \"ON\" to enable code coverage with gcov." OFF) set(TON_ARCH "native" CACHE STRING "Architecture, will be passed to -march=") option(TON_PRINT_BACKTRACE_ON_CRASH "Attempt to print a backtrace when a fatal signal is caught" ON) @@ -251,7 +252,7 @@ elseif (CLANG OR GCC) set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gc-sections -Wl,--exclude-libs,ALL") endif() set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") - if (NOT TON_USE_ASAN AND NOT TON_USE_TSAN AND NOT MEMPROF) + if (NOT TON_USE_ASAN AND NOT TON_USE_TSAN AND NOT TON_USE_COVERAGE AND NOT MEMPROF) if (NOT USE_EMSCRIPTEN) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--exclude-libs,ALL") endif() @@ -337,6 +338,18 @@ endif() if (TON_USE_UBSAN) add_cxx_compiler_flag("-fsanitize=undefined") endif() +if (TON_USE_COVERAGE) + add_cxx_compiler_flag("-fprofile-arcs") + add_cxx_compiler_flag("-ftest-coverage") + add_cxx_compiler_flag("--coverage") + add_cxx_compiler_flag("-O0") + add_cxx_compiler_flag("-g") + add_cxx_compiler_flag("-fno-inline") + add_cxx_compiler_flag("-fno-inline-small-functions") + add_cxx_compiler_flag("-fno-default-inline") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage --coverage") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fprofile-arcs -ftest-coverage --coverage") +endif() #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread") #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined") @@ -554,6 +567,7 @@ add_test(test-cells test-cells ${TEST_OPTIONS}) add_test(test-smartcont test-smartcont) add_test(test-net test-net) add_test(test-actors test-tdactor) +add_test(test-actors-coro tdactor/test/test-coro) add_test(test-emulator test-emulator) #BEGIN tonlib diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index feb5bf72f2..87955fe3f9 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -153,7 +153,7 @@ if [ "$with_tests" = true ]; then lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont \ - test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ + test-net test-tdactor test-coro test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index f5aa15b4ac..592e5ee770 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -91,7 +91,7 @@ if [ "$with_tests" = true ]; then lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont \ - test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ + test-net test-tdactor test-coro test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else diff --git a/assembly/native/build-ubuntu-appimages.sh b/assembly/native/build-ubuntu-appimages.sh index 2cdc6c6521..0384714684 100644 --- a/assembly/native/build-ubuntu-appimages.sh +++ b/assembly/native/build-ubuntu-appimages.sh @@ -63,7 +63,7 @@ ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib- validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy dht-ping-servers dht-resolve \ adnl-proxy create-state emulator test-ed25519 test-bigint \ - test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ + test-vm test-fift test-cells test-smartcont test-net test-tdactor test-coro test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh index a4f0bb233a..c384c9c421 100644 --- a/assembly/native/build-ubuntu-portable.sh +++ b/assembly/native/build-ubuntu-portable.sh @@ -137,7 +137,7 @@ ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib- validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy dht-ping-servers dht-resolve \ adnl-proxy create-state emulator test-ed25519 test-bigint \ - test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ + test-vm test-fift test-cells test-smartcont test-net test-tdactor test-coro test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index 6ddbaf8856..c866168cc1 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -66,7 +66,7 @@ ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib- validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy dht-ping-servers dht-resolve \ adnl-proxy create-state emulator test-ed25519 test-bigint \ - test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ + test-vm test-fift test-cells test-smartcont test-net test-tdactor test-coro test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } diff --git a/assembly/native/build-windows-2019.bat b/assembly/native/build-windows-2019.bat index e671d277c8..1bc8bc68d4 100644 --- a/assembly/native/build-windows-2019.bat +++ b/assembly/native/build-windows-2019.bat @@ -148,7 +148,7 @@ ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonli tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^ -test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ +test-tdactor test-coro test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver dht-ping-servers dht-resolve IF %errorlevel% NEQ 0 ( echo Can't compile TON diff --git a/assembly/native/build-windows-2022.bat b/assembly/native/build-windows-2022.bat index 6ff7166af3..88236c7789 100644 --- a/assembly/native/build-windows-2022.bat +++ b/assembly/native/build-windows-2022.bat @@ -148,7 +148,7 @@ ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonli tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^ -test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ +test-tdactor test-coro test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver IF %errorlevel% NEQ 0 ( echo Can't compile TON diff --git a/assembly/native/build-windows.bat b/assembly/native/build-windows.bat index f0a65670b4..0f5db3019d 100644 --- a/assembly/native/build-windows.bat +++ b/assembly/native/build-windows.bat @@ -148,7 +148,7 @@ ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonli tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^ -test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ +test-tdactor test-coro test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver dht-ping-servers dht-resolve IF %errorlevel% NEQ 0 ( echo Can't compile TON diff --git a/tdactor/CMakeLists.txt b/tdactor/CMakeLists.txt index 44580dd597..8328cb1bc9 100644 --- a/tdactor/CMakeLists.txt +++ b/tdactor/CMakeLists.txt @@ -15,6 +15,7 @@ set(TDACTOR_SOURCE td/actor/ActorShared.h td/actor/ActorStats.h td/actor/common.h + td/actor/coro.h td/actor/PromiseFuture.h td/actor/MultiPromise.h @@ -52,8 +53,12 @@ add_library(tdactor STATIC ${TDACTOR_SOURCE}) target_include_directories(tdactor PUBLIC $) target_link_libraries(tdactor PUBLIC tdutils) +add_executable(tdactor-example-coroutines example/actor-example-coroutines.cpp) +target_link_libraries(tdactor-example-coroutines PUBLIC tdactor) + # BEGIN-INTERNAL add_subdirectory(benchmark) +add_subdirectory(test) # END-INTERNAL install(TARGETS tdactor EXPORT TdTargets diff --git a/tdactor/benchmark/CMakeLists.txt b/tdactor/benchmark/CMakeLists.txt index 662ae20c48..10360d4e4f 100644 --- a/tdactor/benchmark/CMakeLists.txt +++ b/tdactor/benchmark/CMakeLists.txt @@ -15,3 +15,68 @@ if (MSVC) set_property(SOURCE benchmark.cpp APPEND_STRING PROPERTY COMPILE_FLAGS " /wd4457 /wd4316") endif() +add_executable(benchmark-coro benchmark-coro.cpp) +target_link_libraries(benchmark-coro tdactor tdutils) + +# Google Benchmark based coroutine benchmarks +find_package(PkgConfig) +if(PkgConfig_FOUND) + pkg_check_modules(GBENCH benchmark) + if(GBENCH_FOUND) + add_executable(gbench-coro gbench-coro.cpp) + target_link_libraries(gbench-coro tdactor tdutils) + # Use full library paths to avoid conflicts with our "benchmark" executable + target_link_directories(gbench-coro PRIVATE ${GBENCH_LIBRARY_DIRS}) + target_link_libraries(gbench-coro -lbenchmark) + target_include_directories(gbench-coro PRIVATE ${GBENCH_INCLUDE_DIRS}) + target_compile_options(gbench-coro PRIVATE ${GBENCH_CFLAGS_OTHER}) + + message(STATUS "Google Benchmark found via pkg-config - gbench-coro target enabled") + + # Check for Folly + find_package(folly CONFIG REQUIRED) + if(folly_FOUND) + add_executable(gbench-coro-folly gbench-coro-folly.cpp) + target_link_libraries(gbench-coro-folly -lbenchmark) + target_link_directories(gbench-coro-folly PRIVATE ${GBENCH_LIBRARY_DIRS}) + target_include_directories(gbench-coro-folly PRIVATE ${GBENCH_INCLUDE_DIRS}) + target_compile_options(gbench-coro-folly PRIVATE ${GBENCH_CFLAGS_OTHER}) + + find_package(gflags REQUIRED) + target_link_libraries(gbench-coro-folly Folly::folly gflags) + + message(STATUS "Folly found - enabling Folly benchmarks in gbench-coro-folly") + else() + message(STATUS "Folly not found - Folly benchmarks disabled") + endif() + + add_executable(gbench-coro-yaclib gbench-coro-yaclib.cpp) + target_link_libraries(gbench-coro-yaclib -lbenchmark) + target_link_directories(gbench-coro-yaclib PRIVATE ${GBENCH_LIBRARY_DIRS}) + target_include_directories(gbench-coro-yaclib PRIVATE ${GBENCH_INCLUDE_DIRS}) + target_compile_options(gbench-coro-yaclib PRIVATE ${GBENCH_CFLAGS_OTHER}) + + # Check for YACLib + find_package(yaclib CONFIG QUIET) + if(yaclib_FOUND) + target_link_libraries(gbench-coro-yaclib yaclib::yaclib) + message(STATUS "YACLib found - enabling YACLib benchmarks in gbench-coro-yaclib") + else() + message(STATUS "YACLib not found - fetching via FetchContent for gbench-coro-yaclib") + include(FetchContent) + set(YACLIB_FLAGS "CORO" CACHE STRING "" FORCE) + FetchContent_Declare(yaclib + GIT_REPOSITORY https://github.com/YACLib/YACLib.git + GIT_TAG main + ) + FetchContent_MakeAvailable(yaclib) + target_link_libraries(gbench-coro-yaclib yaclib) + endif() + target_compile_definitions(gbench-coro-yaclib PRIVATE YACLIB_CORO=2) + + else() + message(STATUS "Google Benchmark not found via pkg-config - gbench-coro target disabled") + endif() +else() + message(STATUS "PkgConfig not found - gbench-coro target disabled") +endif() \ No newline at end of file diff --git a/tdactor/benchmark/benchmark-coro.cpp b/tdactor/benchmark/benchmark-coro.cpp new file mode 100644 index 0000000000..6b06c6a388 --- /dev/null +++ b/tdactor/benchmark/benchmark-coro.cpp @@ -0,0 +1,126 @@ +#include "td/actor/coro.h" +#include "td/actor/actor.h" +#include "td/utils/benchmark.h" + +#include + +using namespace td::actor; + +class BenchmarkDatabase final : public td::actor::Actor { + public: + Task square(size_t x) { + co_return x * x; + } +}; + +class CoroBenchmark final : public td::actor::Actor { + public: + void start_up() override { + db_ = td::actor::create_actor("BenchmarkDatabase").release(); + flow().start_immediate().detach(); + } + + Task run_benchmarks() { + LOG(INFO) << "=== Running benchmarks ==="; + std::vector thread_counts = {1, 10}; + size_t total_ops = 100000; + + for (size_t thread_count : thread_counts) { + size_t ops_per_thread = std::max(1, total_ops / thread_count); + std::vector> tasks; + + auto run_benchmark = [&](const char* name, bool immediate) -> Task { + td::Timer timer; + + for (size_t t = 0; t < thread_count; t++) { + auto task_name = PSTRING() << name << "_" << t; + tasks.push_back(spawn_actor(std::move(task_name), [](auto db, auto ops_per_thread, auto immediate) -> Task { + for (size_t i = 0; i < ops_per_thread; i++) { + if (immediate) { + (void)co_await ask_immediate(db, &BenchmarkDatabase::square, i); + } else { + (void)co_await ask(db, &BenchmarkDatabase::square, i); + } + } + co_return td::Unit(); + }(db_, ops_per_thread, immediate))); + } + + for (auto& task : tasks) { + co_await std::move(task); + } + tasks.clear(); + + auto elapsed = timer.elapsed(); + auto ops_per_sec = total_ops / elapsed; + LOG(INFO) << name << " " << ops_per_thread << " ops: " << elapsed << "s (threads=" << thread_count + << ", " << static_cast(ops_per_sec) << " ops/sec)"; + co_return td::Unit(); + }; + + co_await run_benchmark("Immediate", true); + co_await run_benchmark("Delayed", false); + } + + LOG(INFO) << "=== Optimized benchmarks (direct) ==="; + constexpr size_t single_thread = 1; + size_t ops_count = total_ops / single_thread; + + // Warm up + for (size_t i = 0; i < 1000; i++) { + (void)co_await ask(db_, &BenchmarkDatabase::square, i); + } + + td::Timer timer; + for (size_t i = 0; i < ops_count; i++) { + (void)co_await ask(db_, &BenchmarkDatabase::square, i); + } + auto elapsed = timer.elapsed(); + auto ops_per_sec = ops_count / elapsed; + LOG(INFO) << "Direct delayed " << ops_count << " ops: " << elapsed << "s (" + << static_cast(ops_per_sec) << " ops/sec)"; + + timer = {}; + for (size_t i = 0; i < ops_count; i++) { + (void)co_await ask_immediate(db_, &BenchmarkDatabase::square, i); + } + elapsed = timer.elapsed(); + ops_per_sec = ops_count / elapsed; + LOG(INFO) << "Direct immediate " << ops_count << " ops: " << elapsed << "s (" + << static_cast(ops_per_sec) << " ops/sec)"; + + timer = {}; + for (size_t i = 0; i < ops_count; i++) { + auto local_square = [](size_t x) -> Task { co_return x * x; }; + (void)co_await local_square(i); + } + elapsed = timer.elapsed(); + ops_per_sec = ops_count / elapsed; + LOG(INFO) << "Local coroutine " << ops_count << " ops: " << elapsed << "s (" + << static_cast(ops_per_sec) << " ops/sec)"; + + co_return td::Unit(); + } + + Task flow() { + LOG(INFO) << "Starting benchmarks"; + (void)co_await run_benchmarks(); + LOG(INFO) << "Benchmarks completed"; + td::actor::SchedulerContext::get()->stop(); + stop(); + co_return td::Unit(); + } + + private: + td::actor::ActorId db_; +}; + +int main() { + SET_VERBOSITY_LEVEL(VERBOSITY_NAME(INFO)); + td::actor::Scheduler scheduler({4}); + + scheduler.run_in_context([&] { td::actor::create_actor("CoroBenchmark").release(); }); + + scheduler.run(); + return 0; +} diff --git a/tdactor/benchmark/benchmark.cpp b/tdactor/benchmark/benchmark.cpp index deddcb22ce..a7d30373e4 100644 --- a/tdactor/benchmark/benchmark.cpp +++ b/tdactor/benchmark/benchmark.cpp @@ -313,6 +313,10 @@ class ActorExecutorBenchmark : public td::Benchmark { //queue.push_back(std::move(ptr)); q.push(ptr, 0); } + void add_token_to_cpu_queue(SchedulerToken token, SchedulerId scheduler_id) override { + SchedulerMessage::Raw *raw = reinterpret_cast(token); + q.push(SchedulerMessage(SchedulerMessage::acquire_t{}, raw), 0); + } void set_alarm_timestamp(const ActorInfoPtr &actor_info_ptr) override { UNREACHABLE(); } diff --git a/tdactor/benchmark/gbench-coro-folly.cpp b/tdactor/benchmark/gbench-coro-folly.cpp new file mode 100644 index 0000000000..1b5c808916 --- /dev/null +++ b/tdactor/benchmark/gbench-coro-folly.cpp @@ -0,0 +1,105 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + + +namespace { + +using folly::coro::Task; +struct DatabaseService { + folly::Executor::KeepAlive<> ex; + + Task query_user(int user_id) { + // Same "latency": cheap CPU work + int sum = 0; + for (int i = 0; i < 100; ++i) { + sum += user_id * i; + } + co_return "user_" + std::to_string(user_id) + "_data_" + std::to_string(sum); + } + + Task check_auth(int user_id) { + co_return (user_id % 2) == 0; // Even IDs are authorized + } +}; + +struct RequestHandler { + DatabaseService& db; + folly::Executor::KeepAlive<> ex; + + + Task handle_request(int request_id) { + const int user_id = request_id % 1000; + + const bool authorized = co_await co_withExecutor(db.ex, db.check_auth(user_id)); + if (!authorized) { + co_return "401 Unauthorized"; + } + + std::string user_data = co_await co_withExecutor(db.ex, db.query_user(user_id)); + co_return "200 OK: " + user_data; + } +}; + +} // namespace + +// Real-world benchmark: HTTP-like request handler with database lookups +// Simulates: Request -> Auth Check -> DB Query -> Response +static void BM_HttpRequestHandler_Folly(benchmark::State& state) { + const int concurrent_requests = static_cast(state.range(0)); + state.SetLabel("HttpHandler_Folly_" + std::to_string(concurrent_requests) + "_requests"); + + // Executor sized to available HW, capped by requested concurrency (avoid oversubscribe) + const int hw = std::max(1u, std::thread::hardware_concurrency()); + const int threads = 4; + folly::CPUThreadPoolExecutor exec(threads); + auto ex = folly::getKeepAliveToken(&exec); + + DatabaseService db{folly::SerialExecutor::create(ex)}; + RequestHandler handler{db, ex}; + + int request_counter = 0; + + while (state.KeepRunningBatch(concurrent_requests)) { + std::vector> requests; + requests.reserve(concurrent_requests); + + // Launch concurrent requests on the executor + for (int i = 0; i < concurrent_requests; ++i) { + requests.emplace_back(co_withExecutor(ex, handler.handle_request(request_counter++))); + } + + // Await all responses concurrently + auto responses = + folly::coro::blockingWait(folly::coro::collectAllRange(std::move(requests))); + + for (auto& r : responses) { + benchmark::DoNotOptimize(r); + } + } +} + +// Register HTTP handler benchmark with various concurrency levels +BENCHMARK(BM_HttpRequestHandler_Folly)->Arg(1)->UseRealTime()->MeasureProcessCPUTime(); +BENCHMARK(BM_HttpRequestHandler_Folly)->Arg(10)->UseRealTime()->MeasureProcessCPUTime(); +BENCHMARK(BM_HttpRequestHandler_Folly)->Arg(100)->UseRealTime()->MeasureProcessCPUTime(); +BENCHMARK(BM_HttpRequestHandler_Folly)->Arg(1000)->UseRealTime()->MeasureProcessCPUTime(); + +int main(int argc, char** argv) { + benchmark::Initialize(&argc, argv); + if (benchmark::ReportUnrecognizedArguments(argc, argv)) + return 1; + benchmark::SetBenchmarkFilter("Http"); + benchmark::RunSpecifiedBenchmarks(); + return 0; +} diff --git a/tdactor/benchmark/gbench-coro-yaclib.cpp b/tdactor/benchmark/gbench-coro-yaclib.cpp new file mode 100644 index 0000000000..c32e417bce --- /dev/null +++ b/tdactor/benchmark/gbench-coro-yaclib.cpp @@ -0,0 +1,189 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +struct DatabaseService { + DatabaseService(yaclib::IExecutorPtr executor) : strand_(executor) { + } + yaclib::Strand strand_; + + yaclib::Task check_auth(int user_id) { + co_await On(strand_); + // Simulate auth check + co_return (user_id % 2) == 0; // Even IDs are authorized + } +}; + +static void BM_HttpRequestHandler_Yaclib(benchmark::State& state) { + const int concurrent_requests = static_cast(state.range(0)); + state.SetLabel("HttpHandler_" + std::to_string(concurrent_requests) + "_requests"); + + // Thread pool backing the "DB"; strand enforces single-coro entry to DB section + auto db_pool = yaclib::MakeFairThreadPool(10); + auto db_strand = yaclib::MakeStrand(db_pool); + + auto query_user = [&](int user_id) -> yaclib::Task { + co_await On(*db_strand); + // Simulate DB latency with a small computation + int sum = 0; + for (int i = 0; i < 100; ++i) { + sum += user_id * i; + } + std::stringstream ss; + ss << "user_" << user_id << "_data_" << sum; + co_return ss.str(); + }; + auto check_auth = [&](int user_id) -> yaclib::Task { + co_await On(*db_strand); + // Simulate auth check + co_return (user_id % 2) == 0; // Even IDs are authorized + }; + + std::vector executors; + for (size_t i = 0; i < concurrent_requests; ++i) { + executors.emplace_back(yaclib::MakeStrand(db_pool)); + } + // "RequestHandler" + auto handle_request = [&](int request_id) -> yaclib::Task { + co_await On(*executors[request_id % executors.size()]); + const int user_id = request_id % 1000; + + const bool authorized = co_await check_auth(user_id); + if (!authorized) + co_return std::string{"401 Unauthorized"}; + + const std::string user_data = co_await query_user(user_id); + co_return std::string{"200 OK: "} + user_data; + }; + + auto run_test = [&]() -> yaclib::Task { + int request_counter = 0; + while (state.KeepRunningBatch(concurrent_requests)) { + std::vector> batch; + batch.reserve(concurrent_requests); + + for (int i = 0; i < concurrent_requests; ++i) { + // Convert lazy Task -> started Future + batch.emplace_back(handle_request(request_counter++).ToFuture()); + } + + // Wait for all; keeps concurrency + auto all = co_await yaclib::WhenAll(batch.begin(), batch.end()); + benchmark::DoNotOptimize(all); + } + co_return 0; + }; + + benchmark::DoNotOptimize(run_test().Get()); + + db_pool->Stop(); + db_pool->Wait(); +} + +// Register with various concurrency levels +BENCHMARK(BM_HttpRequestHandler_Yaclib)->Arg(1)->UseRealTime()->MeasureProcessCPUTime(); +BENCHMARK(BM_HttpRequestHandler_Yaclib)->Arg(10)->UseRealTime()->MeasureProcessCPUTime(); +BENCHMARK(BM_HttpRequestHandler_Yaclib)->Arg(100)->UseRealTime()->MeasureProcessCPUTime(); +BENCHMARK(BM_HttpRequestHandler_Yaclib)->Arg(1000)->UseRealTime()->MeasureProcessCPUTime(); + +static void BM_AskYaclib(benchmark::State& state) { + auto pool = yaclib::MakeFairThreadPool(10); + const int num_tasks = static_cast(state.range(0)); + std::vector executors; + for (int i = 0; i < num_tasks; ++i) { + executors.emplace_back(yaclib::MakeStrand(pool)); + } + + auto get = [&](size_t i) -> yaclib::Task { + co_await On(*executors[i % executors.size()]); + co_return 42; + }; + + // Set a clean label for this benchmark + auto run_test = [&]() -> yaclib::Task { + int request_counter = 0; + while (state.KeepRunningBatch(num_tasks)) { + if (num_tasks == 1) { + int result = co_await get(0); + benchmark::DoNotOptimize(result); + } else { + std::vector> tasks; + tasks.reserve(num_tasks); + for (int i = 0; i < num_tasks; ++i) { + auto task = get(i); + tasks.push_back(std::move(task)); + } + for (auto& task : tasks) { + co_await std::move(task); + } + } + } + co_return 0; + }; + + benchmark::DoNotOptimize(run_test().Get()); + + pool->Stop(); + pool->Wait(); +} + +// Register all combinations - the label is set inside the benchmark +// Single task +BENCHMARK(BM_AskYaclib)->Args({1})->UseRealTime()->MeasureProcessCPUTime(); +BENCHMARK(BM_AskYaclib)->Args({10})->UseRealTime()->MeasureProcessCPUTime(); +BENCHMARK(BM_AskYaclib)->Args({100})->UseRealTime()->MeasureProcessCPUTime(); + +yaclib::Task simple_task() { + co_return 42; +} +static void BM_TaskAwait(benchmark::State& state) { + auto db_pool = yaclib::MakeFairThreadPool(1); + auto run_test = [&]() -> yaclib::Task { + int sum = 0; + while (state.KeepRunning()) { + sum += co_await simple_task(); + } + benchmark::DoNotOptimize(sum); + co_return 0; + }; + + benchmark::DoNotOptimize(run_test().Get()); + + db_pool->Stop(); + db_pool->Wait(); +} + +// Register with various concurrency levels +BENCHMARK(BM_TaskAwait)->UseRealTime()->MeasureProcessCPUTime(); + +int main(int argc, char** argv) { + benchmark::Initialize(&argc, argv); + if (benchmark::ReportUnrecognizedArguments(argc, argv)) { + return 1; + } + benchmark::RunSpecifiedBenchmarks(); + return 0; +} diff --git a/tdactor/benchmark/gbench-coro.cpp b/tdactor/benchmark/gbench-coro.cpp new file mode 100644 index 0000000000..bc029046d3 --- /dev/null +++ b/tdactor/benchmark/gbench-coro.cpp @@ -0,0 +1,1001 @@ +#include +#include "td/actor/coro.h" +#include "td/actor/actor.h" + +#include +#include +#include +#include + +#if defined(__APPLE__) || defined(__linux__) +#include +#elif defined(_WIN32) +#include +#else +#include +#endif + +using namespace td::actor; + +struct SchedulerGuard { + Scheduler sched{std::vector{Scheduler::NodeInfo{10}}}; + + template + void run_until_done(td::Slice name, Task task) { + auto done = std::make_shared>(false); + td::actor::ActorOwn<> actor_id; + + sched.run_in_context([&] { + actor_id = td::actor::create_actor(name); + task.set_executor(Executor::on_actor(actor_id.get())); + + auto wrapped_task = [](auto done, Task task) -> Task { + auto r = co_await std::move(task).wrap(); + r.ensure(); + done->store(true, std::memory_order_release); + co_return td::Unit{}; + }(done, std::move(task)); + + (void)std::move(wrapped_task).start(); + }); + + while (!done->load(std::memory_order_acquire)) { + sched.run(0.001); + } + } +}; + +static SchedulerGuard& guard() { + static SchedulerGuard g; + return g; +} + +static inline double process_cpu_seconds() { +#if defined(_WIN32) + FILETIME creation, exit, kernel, user; + if (GetProcessTimes(GetCurrentProcess(), &creation, &exit, &kernel, &user)) { + auto to_seconds = [](const FILETIME& ft) { + ULARGE_INTEGER uli{.LowPart = ft.dwLowDateTime, .HighPart = ft.dwHighDateTime}; + return uli.QuadPart * 1e-7; // 100ns → seconds + }; + return to_seconds(kernel) + to_seconds(user); + } + return 0.0; +#elif defined(__APPLE__) || defined(__linux__) + rusage ru{}; + if (getrusage(RUSAGE_SELF, &ru) == 0) { + auto to_seconds = [](const timeval& tv) { return tv.tv_sec + tv.tv_usec * 1e-6; }; + return to_seconds(ru.ru_utime) + to_seconds(ru.ru_stime); + } + return 0.0; +#else + return static_cast(std::clock()) / CLOCKS_PER_SEC; +#endif +} + +template +void coro_benchmark(benchmark::State& state, F&& benchmark_code) { + const auto real_start = std::chrono::steady_clock::now(); + const double cpu_start = process_cpu_seconds(); + + auto task = benchmark_code(state); + guard().run_until_done(state.name(), std::move(task)); + + const double cpu_elapsed = process_cpu_seconds() - cpu_start; + const double real_elapsed = std::chrono::duration(std::chrono::steady_clock::now() - real_start).count(); + const double iterations = static_cast(state.iterations()); + + if (iterations > 0 && cpu_elapsed > 0 && real_elapsed > 0) { + state.counters["cpu_time"] = benchmark::Counter(cpu_elapsed / iterations); + state.counters["cpu_speed"] = benchmark::Counter(iterations / cpu_elapsed, benchmark::Counter::kIsRate); + state.counters["real_time"] = benchmark::Counter(real_elapsed / iterations); + state.counters["real_speed"] = benchmark::Counter(iterations / real_elapsed, benchmark::Counter::kIsRate); + state.counters.erase("items_per_second"); + } +} + +class BenchActor : public td::actor::core::Actor { + public: + Task compute_task(int x) { + co_return x * 2; + } + + td::Result compute_sync(int x) { + return x * 3; + } + + void compute_promise(int x, td::Promise promise) { + promise.set_value(x * 4); + } +}; + +Task simple_task() { + co_return 42; +} + +struct TestAwaitable { + int value; + bool is_ready{false}; + + bool await_ready() noexcept { + return is_ready; + } + std::coroutine_handle<> await_suspend(std::coroutine_handle<> h) noexcept { + return h; + } + td::Result await_resume() noexcept { + return value; + } +}; + +static void BM_RawTaskAwait(benchmark::State& state) { + coro_benchmark(state, [&](auto& state) -> Task { + co_await detach_from_actor(); + td::int64 sum = 0; + while (state.KeepRunning()) { + sum += (co_await SkipAwaitTransform{TestAwaitable{42}}).move_as_ok(); + } + benchmark::DoNotOptimize(sum); + co_return td::Unit(); + }); +} +BENCHMARK(BM_RawTaskAwait)->UseRealTime()->MeasureProcessCPUTime(); + +static void BM_DelayedTaskAwait(benchmark::State& state) { + coro_benchmark(state, [&](auto& state) -> Task { + co_await detach_from_actor(); + td::int64 sum = 0; + while (state.KeepRunning()) { + sum += (co_await SkipAwaitTransform{simple_task()}).move_as_ok(); + } + benchmark::DoNotOptimize(sum); + co_return td::Unit(); + }); +} +BENCHMARK(BM_DelayedTaskAwait)->UseRealTime()->MeasureProcessCPUTime(); + +static void BM_StartedTaskAwait(benchmark::State& state) { + coro_benchmark(state, [&](auto& state) -> Task { + co_await detach_from_actor(); + td::int64 sum = 0; + while (state.KeepRunning()) { + sum += (co_await SkipAwaitTransform{simple_task().start_immediate()}).move_as_ok(); + } + benchmark::DoNotOptimize(sum); + + co_return td::Unit(); + }); +} +BENCHMARK(BM_StartedTaskAwait)->UseRealTime()->MeasureProcessCPUTime(); + +static void BM_ScheduledTaskAwait(benchmark::State& state) { + coro_benchmark(state, [&](auto& state) -> Task { + co_await detach_from_actor(); + td::int64 sum = 0; + while (state.KeepRunning()) { + sum += (co_await SkipAwaitTransform{simple_task().start()}).move_as_ok(); + } + benchmark::DoNotOptimize(sum); + + co_return td::Unit(); + }); +} +BENCHMARK(BM_ScheduledTaskAwait)->UseRealTime()->MeasureProcessCPUTime(); + +enum class ResumeMethod { Raw, Pass, Try }; +enum class ResumeLocation { Actor, Scheduler, Any }; +enum class AwaitableState { Ready, Suspended }; + +static void BM_AwaitThenResume(benchmark::State& state) { + coro_benchmark(state, [](auto& state) -> Task { + const auto method = static_cast(state.range(0)); + const auto location = static_cast(state.range(1)); + const auto awaitable = static_cast(state.range(2)); + + // Set a clean label for this benchmark + const char* awaitable_name[] = {"Ready", "Suspended"}; + const char* method_names[] = {"Raw", "Pass", "Try"}; + const char* location_names[] = {"Actor", "Scheduler", "Any"}; + auto label = PSTRING() << method_names[static_cast(method)] << "_" + << location_names[static_cast(location)] << "_" + << awaitable_name[static_cast(awaitable)]; + state.SetLabel(label); + // Create an actor to run on if needed + td::actor::ActorOwn actor; + if (location == ResumeLocation::Actor) { + actor = td::actor::create_actor("bench_actor"); + } + + TestAwaitable aw{42, awaitable == AwaitableState::Ready}; + Executor executor; + if (location == ResumeLocation::Actor) { + executor = Executor::on_actor(actor.get()); + } else if (location == ResumeLocation::Scheduler) { + executor = Executor::on_scheduler(); + } else { + executor = Executor::on_any(); + } + co_await resume_on(executor); + + td::int64 sum = 0; + td::int32 total_iterations = 0; + + while (state.KeepRunning()) { + int value = 0; + + switch (method) { + case ResumeMethod::Raw: { + // Raw awaitable without any resume_on wrapper + value = (co_await SkipAwaitTransform{aw}).move_as_ok(); + break; + } + case ResumeMethod::Pass: { + value = (co_await SkipAwaitTransform{wrap_and_resume_on_current(aw)}).move_as_ok(); + break; + } + case ResumeMethod::Try: { + value = (co_await SkipAwaitTransform{unwrap_and_resume_on_current(aw)}); + break; + } + } + + sum += value; + total_iterations++; + } + + CHECK(state.iterations() == total_iterations); + + benchmark::DoNotOptimize(sum); + co_return td::Unit(); + }); +} + +// Register all combinations +static void ApplyAwaitThenResumeArgs(benchmark::internal::Benchmark* b) { + for (auto location : {ResumeLocation::Actor, ResumeLocation::Scheduler, ResumeLocation::Any}) { + for (auto awaitable : {AwaitableState::Ready, AwaitableState::Suspended}) { + for (auto method : {ResumeMethod::Pass, ResumeMethod::Try, ResumeMethod::Raw}) { + b->Args({static_cast(method), static_cast(location), static_cast(awaitable)}); + } + } + } +} +BENCHMARK(BM_AwaitThenResume)->Apply(ApplyAwaitThenResumeArgs)->UseRealTime()->MeasureProcessCPUTime(); + +// Simple benchmarks +static void BM_TaskCreation(benchmark::State& state) { + coro_benchmark(state, [](auto& state) -> Task { + while (state.KeepRunning()) { + auto task = []() -> Task { co_return 42; }(); + benchmark::DoNotOptimize(task); + // Task is not started - just measuring creation overhead + } + co_return td::Unit(); + }); +} +BENCHMARK(BM_TaskCreation)->UseRealTime()->MeasureProcessCPUTime(); + +static void BM_SimpleCompute(benchmark::State& state) { + coro_benchmark(state, [](auto& state) -> Task { + while (state.KeepRunning()) { + auto task = []() -> Task { + int sum = 0; + for (int j = 0; j < 100; j++) { + sum += j; + } + co_return sum; + }(); + // Task will auto-start when awaited + auto result = co_await std::move(task); + benchmark::DoNotOptimize(result); + } + co_return td::Unit(); + }); +} +BENCHMARK(BM_SimpleCompute)->UseRealTime()->MeasureProcessCPUTime(); + +static void BM_TaskChain(benchmark::State& state) { + coro_benchmark(state, [](auto& state) -> Task { + while (state.KeepRunning()) { + auto task1 = []() -> Task { co_return 10; }(); + auto task2 = [](Task task1) -> Task { + auto v = co_await std::move(task1); + co_return v * 2; + }(std::move(task1)); + auto result = co_await std::move(task2); + benchmark::DoNotOptimize(result); + } + co_return td::Unit(); + }); +} +BENCHMARK(BM_TaskChain)->UseRealTime()->MeasureProcessCPUTime(); + +static void BM_ErrorHandling(benchmark::State& state) { + coro_benchmark(state, [](auto& state) -> Task { + while (state.KeepRunning()) { + auto error_task = []() -> Task { co_return td::Status::Error("test error"); }(); + auto result = co_await std::move(error_task).wrap(); + int value = result.is_error() ? 0 : result.ok(); + benchmark::DoNotOptimize(value); + } + co_return td::Unit(); + }); +} +BENCHMARK(BM_ErrorHandling)->UseRealTime()->MeasureProcessCPUTime(); + +static void BM_SpawnCoroutineOld(benchmark::State& state) { + coro_benchmark(state, [](auto& state) -> Task { + while (state.KeepRunning()) { + auto result = co_await spawn_actor("test", []() -> Task { co_return 42; }()); + benchmark::DoNotOptimize(result); + } + + co_return td::Unit(); + }); +} +BENCHMARK(BM_SpawnCoroutineOld)->UseRealTime()->MeasureProcessCPUTime(); + +// Benchmark with multiple operations per iteration +static void BM_BatchTaskCreation(benchmark::State& state) { + coro_benchmark(state, [](auto& state) -> Task { + while (state.KeepRunning()) { + auto task = []() -> Task { co_return 42; }(); + benchmark::DoNotOptimize(task); + } + co_return td::Unit(); + }); +} +BENCHMARK(BM_BatchTaskCreation)->UseRealTime()->MeasureProcessCPUTime(); + +// Concurrent tasks +static void BM_ConcurrentTasks(benchmark::State& state) { + coro_benchmark(state, [](auto& state) -> Task { + const int num_tasks_int = static_cast(state.range(0)); + while (state.KeepRunningBatch(num_tasks_int)) { + std::vector> tasks; + tasks.reserve(num_tasks_int); + for (int i = 0; i < num_tasks_int; ++i) { + tasks.emplace_back([](int i) -> Task { co_return i * 2; }(i).start()); + } + int total = 0; + for (auto& task : tasks) { + auto result = co_await std::move(task); + total += result; + } + benchmark::DoNotOptimize(total); + } + co_return td::Unit(); + }); +} +BENCHMARK(BM_ConcurrentTasks)->RangeMultiplier(2)->Range(1, 64)->UseRealTime()->MeasureProcessCPUTime(); + +// Memory pattern +static void BM_MemoryPattern(benchmark::State& state) { + coro_benchmark(state, [](auto& state) -> Task { + const int num_tasks = static_cast(state.range(0)); + while (state.KeepRunningBatch(num_tasks)) { + std::vector> tasks; + tasks.reserve(num_tasks); + for (int i = 0; i < num_tasks; ++i) { + tasks.emplace_back([i]() -> Task { co_return i; }()); + } + int sum = 0; + for (auto& task : tasks) { + auto result = co_await std::move(task); + sum += result; + } + benchmark::DoNotOptimize(sum); + } + co_return td::Unit(); + }); +} +BENCHMARK(BM_MemoryPattern)->RangeMultiplier(4)->Range(1, 256)->UseRealTime()->MeasureProcessCPUTime(); + +// Unified benchmark for ask operations +enum class AskMethod { Task, TaskWrap, Promise, Sync, Call, TaskNew }; +enum class AskMode { Scheduled, Immediate }; + +static void BM_Ask(benchmark::State& state) { + coro_benchmark(state, [](auto& state) -> Task { + + const auto method = static_cast(state.range(0)); + const auto mode = static_cast(state.range(1)); + const int num_tasks = static_cast(state.range(2)); + + // Set a clean label for this benchmark + const char* method_names[] = {"Task", "TaskTry", "Promise", "Sync", "Call", "New"}; + const char* mode_names[] = {"Scheduled", "Immediate"}; + auto label = PSTRING() << method_names[static_cast(method)] << "_" << mode_names[static_cast(mode)] << "_" + << num_tasks; + state.SetLabel(label); + td::int32 total_tasks = 0; + std::vector> actors; + for (int i = 0; i < num_tasks; ++i) { + actors.push_back(td::actor::create_actor("bench_actor")); + + // just ensure that the following ask_immediate will be immediate + while (true) { + auto task = ask_immediate(actors.back(), &BenchActor::compute_task, 42); + if (task.await_ready()) { + break; + } + } + } + co_await Yield{}; + + while (state.KeepRunningBatch(num_tasks)) { + if (num_tasks == 1) { + // Single task - direct execution + int result = 0; + + if (mode == AskMode::Immediate) { + switch (method) { + case AskMethod::Task: { + result = co_await ask_immediate(actors[0], &BenchActor::compute_task, 42); + break; + } + case AskMethod::TaskWrap: { + result = (co_await ask_immediate(actors[0], &BenchActor::compute_task, 42).wrap()).move_as_ok(); + break; + } + case AskMethod::TaskNew: { + result = co_await ask_new_immediate(actors[0], &BenchActor::compute_task, 42); + break; + } + case AskMethod::Promise: { + result = co_await ask_immediate(actors[0], &BenchActor::compute_promise, 42); + break; + } + case AskMethod::Sync: { + result = co_await ask_immediate(actors[0], &BenchActor::compute_sync, 42); + break; + } + case AskMethod::Call: { + result = co_await actors[0].get_actor_unsafe().compute_task(42); + break; + } + } + } else { + switch (method) { + case AskMethod::Task: { + result = co_await ask(actors[0], &BenchActor::compute_task, 42); + break; + } + case AskMethod::TaskWrap: { + result = (co_await ask(actors[0], &BenchActor::compute_task, 42).wrap()).ok(); + break; + } + case AskMethod::TaskNew: { + result = co_await ask_new(actors[0], &BenchActor::compute_task, 42); + break; + } + case AskMethod::Promise: { + result = co_await ask(actors[0], &BenchActor::compute_promise, 42); + break; + } + case AskMethod::Sync: { + result = co_await ask(actors[0], &BenchActor::compute_sync, 42); + break; + } + case AskMethod::Call: { + UNREACHABLE(); + } + } + } + + total_tasks++; + benchmark::DoNotOptimize(result); + } else { + // Multiple tasks - launch concurrently + std::vector> tasks; + tasks.reserve(num_tasks); + + // Launch all tasks + for (int i = 0; i < num_tasks; ++i) { + if (mode == AskMode::Immediate) { + switch (method) { + case AskMethod::Task: + case AskMethod::TaskWrap: + tasks.emplace_back(ask_immediate(actors[i], &BenchActor::compute_task, 42 + i)); + break; + case AskMethod::TaskNew: + tasks.emplace_back(ask_new_immediate(actors[i], &BenchActor::compute_task, 42 + i)); + break; + case AskMethod::Promise: + tasks.emplace_back(ask_immediate(actors[i], &BenchActor::compute_promise, 42 + i)); + break; + case AskMethod::Sync: + tasks.emplace_back(ask_immediate(actors[i], &BenchActor::compute_sync, 42 + i)); + break; + case AskMethod::Call: { + UNREACHABLE(); + } + } + CHECK(tasks.back().await_ready()); + } else if (mode == AskMode::Scheduled) { + switch (method) { + case AskMethod::Task: + case AskMethod::TaskWrap: + tasks.emplace_back(ask(actors[i], &BenchActor::compute_task, 42 + i)); + break; + case AskMethod::TaskNew: + tasks.emplace_back(ask_new(actors[i], &BenchActor::compute_task, 42 + i)); + break; + case AskMethod::Promise: + tasks.emplace_back(ask(actors[i], &BenchActor::compute_promise, 42 + i)); + break; + case AskMethod::Sync: + tasks.emplace_back(ask(actors[i], &BenchActor::compute_sync, 42 + i)); + break; + case AskMethod::Call: { + UNREACHABLE(); + } + } + } else { + UNREACHABLE(); + } + } + + // Await all results + int total = 0; + for (auto& task : tasks) { + if (method == AskMethod::TaskWrap) { + total += (co_await std::move(task).wrap()).ok(); + } else { + auto result = co_await std::move(task); + total += result; + } + total_tasks++; + } + benchmark::DoNotOptimize(total); + } + } + + CHECK(state.iterations() == total_tasks); + co_return td::Unit(); + }); +} + +// Register all combinations - the label is set inside the benchmark +static void ApplyBMAskArgs(benchmark::internal::Benchmark* b) { + for (auto method : {AskMethod::TaskNew, AskMethod::TaskWrap, AskMethod::Task, AskMethod::Promise, AskMethod::Sync}) { + for (auto mode : {AskMode::Scheduled, AskMode::Immediate}) { + for (int n : {1, 10, 100}) { + b->Args({static_cast(method), static_cast(mode), n}); + } + } + } + // Call method is only meaningful for single-task immediate mode + b->Args({static_cast(AskMethod::Call), static_cast(AskMode::Immediate), 1}); +} +BENCHMARK(BM_Ask)->Apply(ApplyBMAskArgs)->UseRealTime()->MeasureProcessCPUTime()->Repetitions(10000); + +// Benchmark send_closure with promise callback using Worker pattern +static void BM_SendClosureWorker(benchmark::State& state) { + struct Worker : public td::actor::core::Actor { + public: + Worker(benchmark::State& state, td::Promise promise, bool immediate, int num_tasks) + : state_(state) + , promise_(std::move(promise)) + , immediate_(immediate) + , num_actors_(num_tasks) + , tasks_completed_(0) { + for (int i = 0; i < num_actors_; i++) { + childs_.push_back(td::actor::create_actor("bench_actor")); + } + } + + private: + benchmark::State& state_; + td::Promise promise_; + std::vector> childs_; + bool immediate_; + int num_actors_; + int tasks_completed_; + int total_ = 0; + + void loop() override { + if (state_.KeepRunningBatch(num_actors_)) { + tasks_completed_ = 0; + total_ = 0; + for (int i = 0; i < num_actors_; ++i) { + auto promise = td::promise_send_closure(actor_id(this), &Worker::done, i); + if (immediate_) { + send_closure_immediate(childs_[i], &BenchActor::compute_promise, 42 + i, std::move(promise)); + } else { + send_closure(childs_[i], &BenchActor::compute_promise, 42 + i, std::move(promise)); + } + } + } else { + promise_.set_value(7); + } + } + + void done(int task_id, td::Result result) { + if (result.is_ok()) { + total_ += result.ok(); + } + tasks_completed_++; + benchmark::DoNotOptimize(total_); + + if (tasks_completed_ == num_actors_) { + loop(); + } + } + }; + + coro_benchmark(state, [](auto& state) -> Task { + const bool immediate = static_cast(state.range(0)); + const int num_tasks = static_cast(state.range(1)); + + // Set a clean label for this benchmark + state.SetLabel(PSTRING() << "SendClosure_" << (immediate ? "Immediate" : "Scheduled") << "_" << num_tasks); + + auto [task, promise] = StartedTask::make_bridge(); + auto td_promise = + td::Promise([p = std::move(promise)](td::Result r) mutable { p.set_result(std::move(r)); }); + auto worker = td::actor::create_actor("worker", state, std::move(td_promise), immediate, num_tasks); + auto result = co_await std::move(task); + benchmark::DoNotOptimize(result); + + co_return td::Unit(); + }); +} + +// Register send_closure benchmarks - the label is set inside the benchmark +static void ApplySendClosureArgs(benchmark::internal::Benchmark* b) { + for (int immediate : {0, 1}) { + for (int n : {1, 10, 100}) { + b->Args({immediate, n}); + } + } +} +BENCHMARK(BM_SendClosureWorker)->Apply(ApplySendClosureArgs)->UseRealTime()->MeasureProcessCPUTime(); + +// Real-world benchmark: HTTP-like request handler with database lookups +// Simulates: Request -> Auth Check -> DB Query -> Response +static void BM_HttpRequestHandler(benchmark::State& state) { + // Simulated database service + struct DatabaseService : public td::actor::core::Actor { + Task query_user(int user_id) { + // Simulate DB latency with a small computation + int sum = 0; + for (int i = 0; i < 100; ++i) { + sum += user_id * i; + } + co_return PSTRING() << "user_" << user_id << "_data_" << sum; + } + + Task check_auth(int user_id) { + // Simulate auth check + co_return (user_id % 2) == 0; // Even IDs are authorized + } + }; + + // Request handler service + struct RequestHandler { + static Task handle_request(ActorId db, int request_id) { + int user_id = request_id % 1000; + + // Step 1: Check authorization + auto authorized = co_await ask_immediate(db, &DatabaseService::check_auth, user_id); + if (!authorized) { + co_return "401 Unauthorized"; + } + + // Step 2: Query database + auto user_data = co_await ask_immediate(db, &DatabaseService::query_user, user_id); + + // Step 3: Process and return response + co_return PSTRING() << "200 OK: " << user_data; + } + }; + + coro_benchmark(state, [](auto& state) -> Task { + const int concurrent_requests = static_cast(state.range(0)); + state.SetLabel(PSTRING() << "HttpHandler_" << concurrent_requests << "_requests"); + + auto db = td::actor::create_actor("database"); + + int request_counter = 0; + + while (state.KeepRunningBatch(concurrent_requests)) { + std::vector> requests; + requests.reserve(concurrent_requests); + + // Launch concurrent requests + for (int i = 0; i < concurrent_requests; ++i) { + // TODO or lazy coroutine? + auto task = RequestHandler::handle_request(db.get(), request_counter).start(); + requests.emplace_back(std::move(task)); + } + + // Await all responses + for (auto& request : requests) { + auto response = co_await std::move(request); + benchmark::DoNotOptimize(response); + } + } + + co_return td::Unit(); + }); +} + +// Register HTTP handler benchmark with various concurrency levels +BENCHMARK(BM_HttpRequestHandler)->RangeMultiplier(10)->Range(1, 1000)->UseRealTime()->MeasureProcessCPUTime(); + +static void BM_HttpRequestHandlerOld(benchmark::State& state) { + // Simulated database service + struct DatabaseService : public td::actor::core::Actor { + td::Result query_user(int user_id) { + // Simulate DB latency with a small computation + int sum = 0; + for (int i = 0; i < 100; ++i) { + sum += user_id * i; + } + return PSTRING() << "user_" << user_id << "_data_" << sum; + } + + td::Result check_auth(int user_id) { + // Simulate auth check + return (user_id % 2) == 0; // Even IDs are authorized + } + }; + + // Request handler service + struct RequestHandlerOld : public td::actor::core::Actor { + RequestHandlerOld(td::actor::ActorId db, td::Promise promise, td::int32 request_id) + : db_(db), promise_(std::move(promise)), user_id_(request_id % 1000) { + } + + void start_up() override { + // Step 1: Check authorization + send_closure_immediate(db_, &DatabaseService::check_auth, user_id_, + td::promise_send_closure(actor_id(this), &RequestHandlerOld::on_authorized)); + } + + void on_authorized(td::Result r_authorized) { + if (r_authorized.is_error()) { + promise_.set_error(r_authorized.move_as_error()); + return stop(); + } + auto authorized = r_authorized.ok(); + if (!authorized) { + promise_.set_value("401 anauthorized"); + return stop(); + } + + send_closure_immediate(db_, &DatabaseService::query_user, user_id_, + td::promise_send_closure(actor_id(this), &RequestHandlerOld::on_user)); + } + + void on_user(td::Result r_user_data) { + if (r_user_data.is_error()) { + promise_.set_error(r_user_data.move_as_error()); + return stop(); + } + auto user_data = r_user_data.ok(); + promise_.set_value(PSTRING() << "200 OK: " << user_data); + return stop(); + } + + private: + td::actor::ActorId db_; + td::Promise promise_; + td::int32 user_id_{}; + }; + + coro_benchmark(state, [](auto& state) -> Task { + const int concurrent_requests = static_cast(state.range(0)); + state.SetLabel(PSTRING() << "HttpHandlerOld_" << concurrent_requests << "_requests"); + + auto db = td::actor::create_actor("database"); + + int request_counter = 0; + + while (state.KeepRunningBatch(concurrent_requests)) { + std::vector> requests; + requests.reserve(concurrent_requests); + + // Launch concurrent requests + for (int i = 0; i < concurrent_requests; ++i) { + auto [task, promise] = StartedTask::make_bridge(); + td::actor::create_actor("handler", db.get(), std::move(promise), ++request_counter) + .release(); + requests.emplace_back(std::move(task)); + } + + // Await all responses + for (auto& request : requests) { + auto response = co_await std::move(request); + benchmark::DoNotOptimize(response); + } + } + + co_return td::Unit(); + }); +} + +// Register HTTP handler benchmark with various concurrency levels +BENCHMARK(BM_HttpRequestHandlerOld)->RangeMultiplier(10)->Range(1, 1000)->UseRealTime()->MeasureProcessCPUTime(); + +// Concurrent Pub-Sub benchmark: Publishers -> Broker -> Subscribers +// Multiple Publisher actors produce messages concurrently, Broker fans out to all Subscribers +static void BM_PubSubConcurrent(benchmark::State& state) { + struct Message { + std::string payload; + }; + + struct Subscriber : public td::actor::core::Actor { + explicit Subscriber(int id) : id_(id) { + } + + void process(Message m) { + int sum = 0; + for (char c : m.payload) { + sum += static_cast(c); + } + total_++; + benchmark::DoNotOptimize(sum); + } + + td::Result get_delivered_count() { + return total_; + } + + private: + int id_; + td::int64 total_{0}; + }; + + struct Broker : public td::actor::core::Actor { + void subscribe(td::actor::ActorId sub) { + subscribers_.push_back(sub); + } + + Task publish(Message m) { + for (auto& sub : subscribers_) { + send_closure(sub, &Subscriber::process, m); + } + co_return static_cast(subscribers_.size()); + } + + private: + std::vector> subscribers_; + }; + + struct Publisher : public td::actor::core::Actor { + Publisher(td::actor::ActorId broker, int id) : broker_(broker), id_(id) { + } + + Task produce(int count) { + int delivered_total = 0; + for (int j = 0; j < count; ++j) { + Message m{PSTRING() << "msg_" << id_ << "_" << j}; + auto delivered = co_await ask(broker_, &Broker::publish, std::move(m)); + delivered_total += delivered; + } + co_return delivered_total; + } + + private: + td::actor::ActorId broker_; + int id_; + }; + + coro_benchmark(state, [](auto& state) -> Task { + const int num_publishers = static_cast(state.range(0)); + const int num_subscribers = static_cast(state.range(1)); + const int num_brokers = static_cast(state.range(2)); + constexpr int messages_per_publisher = 10; + state.SetLabel(PSTRING() << "PubSubConcurrent_P" << num_publishers << "_S" << num_subscribers << "_B" + << num_brokers); + + // Create brokers (shards) + std::vector> brokers; + brokers.reserve(num_brokers); + for (int b = 0; b < num_brokers; ++b) { + brokers.push_back(td::actor::create_actor("broker")); + } + + // Create subscribers and subscribe them round-robin across brokers + std::vector> subscribers; + subscribers.reserve(num_subscribers); + for (int i = 0; i < num_subscribers; ++i) { + auto sub = td::actor::create_actor("subscriber", i); + for (auto& broker : brokers) { + send_closure(broker, &Broker::subscribe, sub.get()); + } + subscribers.push_back(std::move(sub)); + } + + // Create publishers and assign each to a broker (round-robin) + std::vector> publishers; + publishers.reserve(num_publishers); + for (int p = 0; p < num_publishers; ++p) { + auto& broker = brokers[p % num_brokers]; + publishers.push_back(td::actor::create_actor("publisher", broker.get(), p)); + } + + const td::int64 total_messages = num_publishers * messages_per_publisher * num_subscribers; + td::int64 iteration_count = 0; + while (state.KeepRunningBatch(total_messages)) { + iteration_count++; + std::vector> tasks; + tasks.reserve(num_publishers); + for (auto& pub : publishers) { + tasks.emplace_back(ask(pub, &Publisher::produce, messages_per_publisher)); + } + + int delivered_sum = 0; + for (auto& task : tasks) { + delivered_sum += co_await std::move(task); + } + benchmark::DoNotOptimize(delivered_sum); + } + + td::int64 total_delivered = 0; + const td::int64 expected_per_subscriber = iteration_count * num_publishers * messages_per_publisher; + for (auto& subscriber : subscribers) { + while (true) { + auto delivered_count = co_await ask(subscriber, &Subscriber::get_delivered_count); + if (delivered_count != expected_per_subscriber) { + LOG(ERROR) << "Subscriber delivered " << delivered_count << " != expected " << expected_per_subscriber; + continue; + } + total_delivered += delivered_count; + break; + } + } + CHECK(state.iterations() == total_delivered); + + co_return td::Unit(); + }); +} + +static void ApplyPubSubArgs(benchmark::internal::Benchmark* b) { + constexpr int combos[][3] = {{1, 10, 1}, {10, 10, 1}, {10, 100, 1}, {100, 100, 1}, {10, 100, 4}, {100, 100, 4}}; + for (const auto& combo : combos) { + b->Args({combo[0], combo[1], combo[2]}); + } +} +BENCHMARK(BM_PubSubConcurrent)->Apply(ApplyPubSubArgs)->UseRealTime()->MeasureProcessCPUTime(); + +static void BM_ConcurrentAsks(benchmark::State& state) { + coro_benchmark(state, [](auto& state) -> Task { + const int num_actors = static_cast(state.range(0)); + std::vector> actors; + actors.reserve(num_actors); + for (int i = 0; i < num_actors; ++i) { + actors.push_back(td::actor::create_actor("bench_actor")); + } + + while (state.KeepRunningBatch(num_actors)) { + std::vector> tasks; + tasks.reserve(num_actors); + + for (auto& actor : actors) { + tasks.emplace_back(ask(actor, &BenchActor::compute_task, 42)); + } + + for (auto& task : tasks) { + auto result = co_await std::move(task); + benchmark::DoNotOptimize(result); + } + } + + co_return td::Unit(); + }); +} +BENCHMARK(BM_ConcurrentAsks)->RangeMultiplier(4)->Range(1, 64)->UseRealTime()->MeasureProcessCPUTime(); + +int main(int argc, char** argv) { + benchmark::Initialize(&argc, argv); + if (benchmark::ReportUnrecognizedArguments(argc, argv)) { + return 1; + } + benchmark::SetBenchmarkFilter("BM_Ask/5/1/10/"); + //benchmark::SetBenchmarkFilter("BM_Ask"); + benchmark::RunSpecifiedBenchmarks(); + return 0; +} \ No newline at end of file diff --git a/tdactor/example/actor-example-coroutines.cpp b/tdactor/example/actor-example-coroutines.cpp new file mode 100644 index 0000000000..e41603e60a --- /dev/null +++ b/tdactor/example/actor-example-coroutines.cpp @@ -0,0 +1,182 @@ +#include "absl/status/status.h" +#include "td/actor/coro.h" +#include "td/actor/actor.h" +#include "td/utils/SharedSlice.h" +#include "td/utils/port/sleep.h" + +#include +#include +#include +#include +#include +#include + +using namespace td::actor; + +Task example_create() { + LOG(INFO) << "Detach"; + Task value = []() -> Task { + LOG(FATAL) << "This line will not be executed"; + co_return 17; + }(); + value.detach(); + + LOG(INFO) << "Simple co_await"; + // Task will be started after co await has been called + Task value2 = []() -> Task { co_return 17; }(); + CHECK(17 == co_await std::move(value2)); + + LOG(INFO) << "start_immediate than co_await"; + // Task is started immediately + StartedTask value3 = []() -> Task { + td::usleep_for(1000000); + co_return 17; + }() + .start_immediate(); + CHECK(value3.await_ready()); + CHECK(17 == (co_await std::move(value3))); + + LOG(INFO) << "start than co_await"; + // Task is started on scheduler + StartedTask value4 = []() -> Task { + td::usleep_for(1000000); + co_return 17; + }() + .start(); + CHECK(!value4.await_ready()); + CHECK(17 == (co_await std::move(value4))); + + StartedTask value5 = spawn_actor("worker", []() -> Task { + // This code will be run on some actor + // Main reason to use it is actor statistics + co_return 17; + }()); + CHECK(17 == (co_await std::move(value5))); + co_return td::Unit(); +} + +Task example_communicate() { + LOG(INFO) << "Communicate with actor"; + struct Worker : public Actor { + int square(int x) { + return x * x; + } + Task square_task(int x) { + co_return square(x); + } + void square_promise(int x, td::Promise promise) { + send_closure(actor_id(this), &Worker::square_task, x, std::move(promise)); + } + }; + auto worker = create_actor("worker"); + + StartedTask value6 = ask(worker, &Worker::square, 17); + CHECK(289 == (co_await std::move(value6))); + StartedTask value7 = ask(worker, &Worker::square_promise, 17); + CHECK(289 == (co_await std::move(value7))); + StartedTask value8 = ask(worker, &Worker::square_task, 17); + CHECK(289 == (co_await std::move(value8))); + co_return td::Unit(); +} + +Task task_error() { + co_return td::Status::Error("test error"); +} +td::Result result_error() { + return td::Status::Error("test error"); +} +Task pass_task_error() { + co_await task_error(); + co_return 17; +} +Task pass_result_error() { + co_await result_error(); + co_return 17; +} + +Task example_error_handling() { + // Error handling + (co_await pass_task_error().wrap()).ensure_error(); + (co_await pass_result_error().wrap()).ensure_error(); + (co_await result_error().wrap()).ensure_error(); + + co_return td::Unit(); +} + +Task example_actor() { + struct TaskActor : public Actor { + TaskActor(td::Promise promise) : promise_(std::move(promise)) { + + } + void start_up() override { + // it is usual actor all coroutines create FROM actor, will be executed ON actor + run().start().detach(); + } + Task run() { + state_ = 19; + finish(); + co_return td::Unit(); + } + private: + td::Promise promise_; + int state_ {17}; + + void finish() { + promise_.set_result(state_); + stop(); + } + }; + + auto [task, promise] = StartedTask::make_bridge(); + auto task_actor = create_actor("task_actor", std::move(promise)); + CHECK(19 == (co_await std::move(task))); + co_return td::Unit(); + +} + +Task example_all() { + std::vector> v; + int n = std::thread::hardware_concurrency(); + for (int i = 0; i < n; i++) { + v.push_back([](int i) -> Task { + td::usleep_for(1000000); + co_return i* i; + }(i) + .start()); + } + auto vv = co_await all(std::move(v)); + for (int i = 0; i < n; i++) { + CHECK(vv[i] == i * i); + } + co_return td::Unit(); +} + +Task run_all_examples() { + co_await example_create(); + co_await example_communicate(); + co_await example_error_handling(); + co_await example_actor(); + co_await example_all(); + co_return td::Unit(); +} + +Task example() { + LOG(INFO) << "Start example coroutine"; + + (co_await run_all_examples().wrap()).ensure(); + + LOG(INFO) << "Finish example coroutine and stop scheduler"; + td::actor::SchedulerContext::get()->stop(); + co_return td::Unit(); +} + +int main() { + SET_VERBOSITY_LEVEL(VERBOSITY_NAME(INFO)); + td::actor::Scheduler scheduler({std::thread::hardware_concurrency()}); + + scheduler.run_in_context([&] { (void)example().start(); }); + + scheduler.run(); + LOG(INFO) << "DONE"; + return 0; +} \ No newline at end of file diff --git a/tdactor/td/actor/PromiseFuture.h b/tdactor/td/actor/PromiseFuture.h index 2b8890d3f2..91cb916ccc 100644 --- a/tdactor/td/actor/PromiseFuture.h +++ b/tdactor/td/actor/PromiseFuture.h @@ -518,4 +518,33 @@ std::pair, Future> make_promise_future() { return std::make_pair(std::move(promise), std::move(future)); } +template +inline constexpr bool always_false = false; + +template +struct is_result : std::false_type {}; +template +struct is_result> : std::true_type {}; + +template +inline constexpr bool is_result_v = is_result::value; + +template +constexpr decltype(auto) connect(L &&l, R &&r) noexcept { + if constexpr (is_result_v>) { + if (r.is_error()) { + connect(std::forward(l), r.move_as_error()); + } else { + connect(std::forward(l), r.move_as_ok()); + } + } else if constexpr (requires { custom_connect(std::forward(l), std::forward(r)); }) { + // ADL will find overloads defined in the namespaces of L or R + return custom_connect(std::forward(l), std::forward(r)); + } else if constexpr (requires { std::forward(l)(std::forward(r)); }) { + return std::forward(l)(std::forward(r)); + } else { + static_assert(always_false, "no matching apply overload"); + } +} + } // namespace td diff --git a/tdactor/td/actor/actor.h b/tdactor/td/actor/actor.h index 1f4f6e9941..12721fca3c 100644 --- a/tdactor/td/actor/actor.h +++ b/tdactor/td/actor/actor.h @@ -26,84 +26,85 @@ namespace td { namespace actor { template -TD_WARN_UNUSED_RESULT ActorOwn create_actor(ActorOptions options, ArgsT &&... args) { +TD_WARN_UNUSED_RESULT ActorOwn create_actor(ActorOptions options, ArgsT &&...args) { return ActorOwn(ActorId::create(options, std::forward(args)...)); } template -TD_WARN_UNUSED_RESULT ActorOwn create_actor(Slice name, ArgsT &&... args) { +TD_WARN_UNUSED_RESULT ActorOwn create_actor(Slice name, ArgsT &&...args) { return ActorOwn(ActorId::create(ActorOptions().with_name(name), std::forward(args)...)); } -#define SEND_CLOSURE_LATER 1 -#ifndef SEND_CLOSURE_LATER +namespace internal { -template , - size_t argument_count = member_function_argument_count(), - std::enable_if_t with_promise = false> -void send_closure(ActorIdT &&actor_id, FunctionT function, ArgsT &&... args) { +template > +void send_closure_dispatch(ActorIdT &&actor_id, FunctionT function, ArgsT &&...args) { using ActorT = typename std::decay_t::ActorT; static_assert(std::is_base_of::value, "unsafe send_closure"); + constexpr size_t argument_count = member_function_argument_count(); + ActorIdT id = std::forward(actor_id); - detail::send_closure(id.as_actor_ref(), function, std::forward(args)...); + if constexpr (argument_count == sizeof...(ArgsT)) { + if constexpr (Later) { + detail::send_closure_later(id.as_actor_ref(), function, std::forward(args)...); + } else { + detail::send_closure(id.as_actor_ref(), function, std::forward(args)...); + } + } else { + auto closure = call_n_arguments( + [&function](auto &&...nargs) { + if constexpr (Later) { + return create_delayed_closure(function, std::forward(nargs)...); + } else { + return create_immediate_closure(function, std::forward(nargs)...); + } + }, + std::forward(args)...); + auto promise = get_last_argument(std::forward(args)...); + if constexpr (Later) { + detail::send_closure_with_promise_later(id.as_actor_ref(), std::move(closure), std::move(promise)); + } else { + detail::send_closure_with_promise(id.as_actor_ref(), std::move(closure), std::move(promise)); + } + } } +} // namespace internal + template , size_t argument_count = member_function_argument_count(), - std::enable_if_t with_promise = true> -void send_closure(ActorIdT &&actor_id, FunctionT function, ArgsT &&... args) { - using ActorT = typename std::decay_t::ActorT; - static_assert(std::is_base_of::value, "unsafe send_closure"); - - ActorIdT id = std::forward(actor_id); - detail::send_closure_with_promise(id.as_actor_ref(), - call_n_arguments( - [&function](auto &&... nargs) { - return create_immediate_closure(function, - std::forward(nargs)...); - }, - std::forward(args)...), - get_last_argument(std::forward(args)...)); + std::enable_if_t with_promise = false> +void send_closure_immediate(ActorIdT &&actor_id, FunctionT function, ArgsT &&...args) { + internal::send_closure_dispatch(std::forward(actor_id), function, std::forward(args)...); } -#else +template , + size_t argument_count = member_function_argument_count(), + std::enable_if_t with_promise = true> +void send_closure_immediate(ActorIdT &&actor_id, FunctionT function, ArgsT &&...args) { + internal::send_closure_dispatch(std::forward(actor_id), function, std::forward(args)...); +} template , size_t argument_count = member_function_argument_count(), std::enable_if_t with_promise = false> -void send_closure(ActorIdT &&actor_id, FunctionT function, ArgsT &&... args) { - using ActorT = typename std::decay_t::ActorT; - static_assert(std::is_base_of::value, "unsafe send_closure"); - - ActorIdT id = std::forward(actor_id); - detail::send_closure_later(id.as_actor_ref(), function, std::forward(args)...); +void send_closure(ActorIdT &&actor_id, FunctionT function, ArgsT &&...args) { + internal::send_closure_dispatch(std::forward(actor_id), function, std::forward(args)...); } template , size_t argument_count = member_function_argument_count(), std::enable_if_t with_promise = true> -void send_closure(ActorIdT &&actor_id, FunctionT function, ArgsT &&... args) { - using ActorT = typename std::decay_t::ActorT; - static_assert(std::is_base_of::value, "unsafe send_closure"); - - ActorIdT id = std::forward(actor_id); - detail::send_closure_with_promise_later(id.as_actor_ref(), - call_n_arguments( - [&function](auto &&... nargs) { - return create_delayed_closure(function, - std::forward(nargs)...); - }, - std::forward(args)...), - get_last_argument(std::forward(args)...)); +void send_closure(ActorIdT &&actor_id, FunctionT function, ArgsT &&...args) { + internal::send_closure_dispatch(std::forward(actor_id), function, std::forward(args)...); } -#endif - template , size_t argument_count = member_function_argument_count(), std::enable_if_t with_promise = false> -auto future_send_closure(ActorIdT &&actor_id, FunctionT function, ArgsT &&... args) { +auto future_send_closure(ActorIdT &&actor_id, FunctionT function, ArgsT &&...args) { using R = ::td::detail::get_ret_t>; auto pf = make_promise_future(); send_closure(std::forward(actor_id), std::move(function), std::forward(args)..., @@ -115,7 +116,7 @@ template , size_t argument_count = member_function_argument_count(), std::enable_if_t with_promise = true> -Future future_send_closure(ActorIdT &&actor_id, FunctionT function, ArgsT &&... args) { +Future future_send_closure(ActorIdT &&actor_id, FunctionT function, ArgsT &&...args) { auto pf = make_promise_future(); send_closure(std::forward(actor_id), std::move(function), std::forward(args)..., std::move(pf.first)); @@ -123,7 +124,7 @@ Future future_send_closure(ActorIdT &&actor_id, FunctionT function, ArgsT &&. } template -bool send_closure_bool(ActorIdT &&actor_id, FunctionT function, ArgsT &&... args) { +bool send_closure_bool(ActorIdT &&actor_id, FunctionT function, ArgsT &&...args) { send_closure(std::forward(actor_id), function, std::forward(args)...); return true; } @@ -131,45 +132,30 @@ bool send_closure_bool(ActorIdT &&actor_id, FunctionT function, ArgsT &&... args template , size_t argument_count = member_function_argument_count(), std::enable_if_t with_promise = false> -void send_closure_later(ActorIdT &&actor_id, FunctionT function, ArgsT &&... args) { - using ActorT = typename std::decay_t::ActorT; - static_assert(std::is_base_of::value, "unsafe send_closure"); - - ActorIdT id = std::forward(actor_id); - detail::send_closure_later(id.as_actor_ref(), function, std::forward(args)...); +void send_closure_later(ActorIdT &&actor_id, FunctionT function, ArgsT &&...args) { + internal::send_closure_dispatch(std::forward(actor_id), function, std::forward(args)...); } template , size_t argument_count = member_function_argument_count(), std::enable_if_t with_promise = true> -void send_closure_later(ActorIdT &&actor_id, FunctionT function, ArgsT &&... args) { - using ActorT = typename std::decay_t::ActorT; - static_assert(std::is_base_of::value, "unsafe send_closure"); - - ActorIdT id = std::forward(actor_id); - detail::send_closure_with_promise_later(id.as_actor_ref(), - call_n_arguments( - [&function](auto &&... nargs) { - return create_delayed_closure(function, - std::forward(nargs)...); - }, - std::forward(args)...), - get_last_argument(std::forward(args)...)); +void send_closure_later(ActorIdT &&actor_id, FunctionT function, ArgsT &&...args) { + internal::send_closure_dispatch(std::forward(actor_id), function, std::forward(args)...); } template -bool send_closure_later_bool(ActorIdT &&actor_id, FunctionT function, ArgsT &&... args) { +bool send_closure_later_bool(ActorIdT &&actor_id, FunctionT function, ArgsT &&...args) { send_closure_later(std::forward(actor_id), function, std::forward(args)...); return true; } template -void send_lambda(ActorIdT &&actor_id, ArgsT &&... args) { +void send_lambda(ActorIdT &&actor_id, ArgsT &&...args) { ActorIdT id = std::forward(actor_id); detail::send_lambda(id.as_actor_ref(), std::forward(args)...); } template -void send_lambda_later(ActorIdT &&actor_id, ArgsT &&... args) { +void send_lambda_later(ActorIdT &&actor_id, ArgsT &&...args) { ActorIdT id = std::forward(actor_id); detail::send_lambda_later(id.as_actor_ref(), std::forward(args)...); } @@ -188,14 +174,14 @@ void send_signals_later(ActorIdT &&actor_id, ActorSignals signals) { class SendClosure { public: template - void operator()(ArgsT &&... args) const { + void operator()(ArgsT &&...args) const { td::actor::send_closure(std::forward(args)...); } }; template template -auto Promise::send_closure(ArgsT &&... args) { +auto Promise::send_closure(ArgsT &&...args) { return [promise = std::move(*this), t = std::make_tuple(std::forward(args)...)](auto &&r_res) mutable { TRY_RESULT_PROMISE(promise, res, std::move(r_res)); td::call_tuple(SendClosure(), std::tuple_cat(std::move(t), std::make_tuple(std::move(res), std::move(promise)))); @@ -203,7 +189,7 @@ auto Promise::send_closure(ArgsT &&... args) { } template -auto promise_send_closure(ArgsT &&... args) { +auto promise_send_closure(ArgsT &&...args) { return [t = std::make_tuple(std::forward(args)...)](auto &&res) mutable { td::call_tuple(SendClosure(), std::tuple_cat(std::move(t), std::make_tuple(std::move(res)))); }; diff --git a/tdactor/td/actor/common.h b/tdactor/td/actor/common.h index 222db74369..d81d67645b 100644 --- a/tdactor/td/actor/common.h +++ b/tdactor/td/actor/common.h @@ -301,6 +301,12 @@ inline void send_message_later(ActorRef actor_ref, core::ActorMessage message) { send_message_later(actor_ref.actor_info, std::move(message)); } +template +auto run_on_current_actor(ClosureT &&closure) { + using ActorType = typename std::remove_cvref_t::ActorType; + return closure.run(¤t_actor()); +} + template void send_immediate(ActorRef actor_ref, ExecuteF &&execute, ToMessageF &&to_message) { auto scheduler_context_ptr = core::SchedulerContext::get(); @@ -320,7 +326,7 @@ void send_immediate(ActorRef actor_ref, ExecuteF &&execute, ToMessageF &&to_mess } template -void send_lambda(ActorRef actor_ref, F &&lambda) { +void send_lambda_immediate(ActorRef actor_ref, F &&lambda) { send_immediate(actor_ref, lambda, [&lambda]() mutable { return ActorMessageCreator::lambda(std::move(lambda)); }); } template @@ -328,14 +334,18 @@ void send_lambda_later(ActorRef actor_ref, F &&lambda) { send_message_later(actor_ref, ActorMessageCreator::lambda(std::move(lambda))); } +template +void send_lambda(ActorRef actor_ref, F &&lambda) { + send_lambda_immediate(actor_ref, std::forward(lambda)); +} + template void send_closure_impl(ActorRef actor_ref, ClosureT &&closure) { - using ActorType = typename ClosureT::ActorType; send_immediate( - actor_ref, [&closure]() mutable { closure.run(¤t_actor()); }, + actor_ref, [&closure]() mutable { run_on_current_actor(closure); }, [&closure]() mutable { return ActorMessageCreator::lambda( - [closure = to_delayed_closure(std::move(closure))]() mutable { closure.run(¤t_actor()); }); + [closure = to_delayed_closure(std::move(closure))]() mutable { run_on_current_actor(closure); }); }); } @@ -346,38 +356,77 @@ void send_closure(ActorRef actor_ref, ArgsT &&...args) { template void send_closure_later_impl(ActorRef actor_ref, ClosureT &&closure) { - using ActorType = typename ClosureT::ActorType; - send_message_later(actor_ref, - ActorMessageCreator::lambda([closure = to_delayed_closure(std::move(closure))]() mutable { - closure.run(¤t_actor()); - })); + send_message_later( + actor_ref, ActorMessageCreator::lambda( + [closure = to_delayed_closure(std::move(closure))]() mutable { run_on_current_actor(closure); })); } +// Helper to unwrap Result to T, otherwise keep the type as is +template +struct unwrap_result { + using type = T; +}; +template +struct unwrap_result> { + using type = T; +}; +template +using unwrap_result_t = typename unwrap_result::type; + template void send_closure_with_promise(ActorRef actor_ref, ClosureT &&closure, PromiseT &&promise) { - using ActorType = typename ClosureT::ActorType; - using ResultType = decltype(closure.run(¤t_actor())); - auto &&promise_i = promise_interface(std::forward(promise)); - send_immediate( - actor_ref, [&closure, &promise = promise_i]() mutable { promise(closure.run(¤t_actor())); }, - [&closure, &promise = promise_i]() mutable { - return ActorMessageCreator::lambda( - [closure = to_delayed_closure(std::move(closure)), promise = std::move(promise)]() mutable { - promise(closure.run(¤t_actor())); - }); - }); + using RawResultType = decltype(run_on_current_actor(closure)); + if constexpr (std::is_void_v) { + // Adapt void to td::Unit + auto &&promise_i = promise_interface(std::forward(promise)); + send_immediate( + actor_ref, + [&closure, &promise = promise_i]() mutable { + run_on_current_actor(closure); + promise(td::Unit()); + }, + [&closure, &promise = promise_i]() mutable { + return ActorMessageCreator::lambda( + [closure = to_delayed_closure(std::move(closure)), promise = std::move(promise)]() mutable { + run_on_current_actor(closure); + promise(td::Unit()); + }); + }); + } else { + using ResultType = unwrap_result_t; + auto &&promise_i = promise_interface(std::forward(promise)); + send_immediate( + actor_ref, + [&closure, &promise = promise_i]() mutable { connect(std::move(promise), run_on_current_actor(closure)); }, + [&closure, &promise = promise_i]() mutable { + return ActorMessageCreator::lambda( + [closure = to_delayed_closure(std::move(closure)), promise = std::move(promise)]() mutable { + connect(std::move(promise), run_on_current_actor(closure)); + }); + }); + } } template void send_closure_with_promise_later(ActorRef actor_ref, ClosureT &&closure, PromiseT &&promise) { - using ActorType = typename ClosureT::ActorType; - using ResultType = decltype(closure.run(¤t_actor())); - send_message_later( - actor_ref, - ActorMessageCreator::lambda([closure = to_delayed_closure(std::move(closure)), - promise = promise_interface(std::forward(promise))]() mutable { - promise(closure.run(¤t_actor())); - })); + using RawResultType = decltype(run_on_current_actor(closure)); + if constexpr (std::is_void_v) { + // Adapt void to td::Unit + send_message_later( + actor_ref, + ActorMessageCreator::lambda([closure = to_delayed_closure(std::move(closure)), + promise = promise_interface(std::forward(promise))]() mutable { + run_on_current_actor(closure); + promise(td::Unit()); + })); + } else { + using ResultType = unwrap_result_t; + send_message_later(actor_ref, ActorMessageCreator::lambda([closure = to_delayed_closure(std::move(closure)), + promise = promise_interface( + std::forward(promise))]() mutable { + connect(std::move(promise), run_on_current_actor(closure)); + })); + } } template diff --git a/tdactor/td/actor/core/ActorExecuteContext.h b/tdactor/td/actor/core/ActorExecuteContext.h index 37f7d856c8..37d80b5922 100644 --- a/tdactor/td/actor/core/ActorExecuteContext.h +++ b/tdactor/td/actor/core/ActorExecuteContext.h @@ -44,6 +44,9 @@ class ActorExecuteContext : public Context { CHECK(actor_); return *actor_; } + Actor *actor_ptr() const { + return actor_; + } bool has_flags() const { return flags_ != 0; } diff --git a/tdactor/td/actor/core/CpuWorker.cpp b/tdactor/td/actor/core/CpuWorker.cpp index d78660c6fc..b8cf7ce412 100644 --- a/tdactor/td/actor/core/CpuWorker.cpp +++ b/tdactor/td/actor/core/CpuWorker.cpp @@ -22,6 +22,7 @@ #include "td/actor/core/SchedulerContext.h" #include "td/actor/core/Scheduler.h" // FIXME: afer LocalQueue is in a separate file +#include namespace td { namespace actor { @@ -34,55 +35,54 @@ void CpuWorker::run() { waiter_.init_slot(slot, thread_id); auto &debug = dispatcher.get_debug(); while (true) { - SchedulerMessage message; - if (try_pop(message, thread_id)) { + SchedulerToken token = nullptr; + if (try_pop(token, thread_id)) { waiter_.stop_wait(slot); - if (!message) { + if (!token) { return; } - auto lock = debug.start(message->get_name()); - ActorExecutor executor(*message, dispatcher, ActorExecutor::Options().with_from_queue()); + auto encoded = reinterpret_cast(token); + if ((encoded & 1u) == 0u) { + // Regular actor message + auto raw_message = reinterpret_cast(token); + SchedulerMessage message(SchedulerMessage::acquire_t{}, raw_message); + auto lock = debug.start(message->get_name()); + ActorExecutor executor(*message, dispatcher, ActorExecutor::Options().with_from_queue()); + } else { + // Coroutine continuation + auto h = std::coroutine_handle<>::from_address(reinterpret_cast(encoded & ~uintptr_t(1))); + auto lock = debug.start("coro"); + h.resume(); + } } else { waiter_.wait(slot); } } } -bool CpuWorker::try_pop_local(SchedulerMessage &message) { - SchedulerMessage::Raw *raw_message; - if (local_queues_[id_].try_pop(raw_message)) { - message = SchedulerMessage(SchedulerMessage::acquire_t{}, raw_message); - return true; - } - return false; +bool CpuWorker::try_pop_local(SchedulerToken &token) { + return local_queues_[id_].try_pop(token); } -bool CpuWorker::try_pop_global(SchedulerMessage &message, size_t thread_id) { - SchedulerMessage::Raw *raw_message; - if (queue_.try_pop(raw_message, thread_id)) { - message = SchedulerMessage(SchedulerMessage::acquire_t{}, raw_message); - return true; - } - return false; +bool CpuWorker::try_pop_global(SchedulerToken &token, size_t thread_id) { + return queue_.try_pop(token, thread_id); } -bool CpuWorker::try_pop(SchedulerMessage &message, size_t thread_id) { +bool CpuWorker::try_pop(SchedulerToken &token, size_t thread_id) { if (++cnt_ == 51) { cnt_ = 0; - if (try_pop_global(message, thread_id) || try_pop_local(message)) { + if (try_pop_global(token, thread_id) || try_pop_local(token)) { return true; } } else { - if (try_pop_local(message) || try_pop_global(message, thread_id)) { + if (try_pop_local(token) || try_pop_global(token, thread_id)) { return true; } } for (size_t i = 1; i < local_queues_.size(); i++) { size_t pos = (i + id_) % local_queues_.size(); - SchedulerMessage::Raw *raw_message; - if (local_queues_[id_].steal(raw_message, local_queues_[pos])) { - message = SchedulerMessage(SchedulerMessage::acquire_t{}, raw_message); + if (local_queues_[id_].steal(token, local_queues_[pos])) { return true; } } diff --git a/tdactor/td/actor/core/CpuWorker.h b/tdactor/td/actor/core/CpuWorker.h index d9f32513b7..dd520241a5 100644 --- a/tdactor/td/actor/core/CpuWorker.h +++ b/tdactor/td/actor/core/CpuWorker.h @@ -19,6 +19,7 @@ #pragma once #include "td/actor/core/SchedulerMessage.h" +#include "td/actor/core/SchedulerContext.h" #include "td/utils/MpmcQueue.h" #include "td/utils/MpmcWaiter.h" @@ -31,23 +32,22 @@ template struct LocalQueue; class CpuWorker { public: - CpuWorker(MpmcQueue &queue, MpmcWaiter &waiter, size_t id, - MutableSpan> local_queues) + CpuWorker(MpmcQueue &queue, MpmcWaiter &waiter, size_t id, + MutableSpan> local_queues) : queue_(queue), waiter_(waiter), id_(id), local_queues_(local_queues) { } void run(); private: - MpmcQueue &queue_; + MpmcQueue &queue_; MpmcWaiter &waiter_; size_t id_; - MutableSpan> local_queues_; + MutableSpan> local_queues_; size_t cnt_{0}; - bool try_pop(SchedulerMessage &message, size_t thread_id); - - bool try_pop_local(SchedulerMessage &message); - bool try_pop_global(SchedulerMessage &message, size_t thread_id); + bool try_pop(SchedulerToken &token, size_t thread_id); + bool try_pop_local(SchedulerToken &token); + bool try_pop_global(SchedulerToken &token, size_t thread_id); }; } // namespace core } // namespace actor diff --git a/tdactor/td/actor/core/Scheduler.cpp b/tdactor/td/actor/core/Scheduler.cpp index 40f1e72cda..dfb5706389 100644 --- a/tdactor/td/actor/core/Scheduler.cpp +++ b/tdactor/td/actor/core/Scheduler.cpp @@ -21,6 +21,8 @@ #include "td/actor/core/CpuWorker.h" #include "td/actor/core/IoWorker.h" +#include + namespace td { namespace actor { namespace core { @@ -43,10 +45,10 @@ Scheduler::Scheduler(std::shared_ptr scheduler_group_info, S info_->id = id; if (cpu_threads_count != 0) { info_->cpu_threads_count = cpu_threads_count; - info_->cpu_queue = std::make_unique>(1024, max_thread_count()); + info_->cpu_queue = std::make_unique>(1024, max_thread_count()); info_->cpu_queue_waiter = std::make_unique(); - info_->cpu_local_queue = std::vector>(cpu_threads_count); + info_->cpu_local_queue = std::vector>(cpu_threads_count); } info_->io_queue = std::make_unique>(); info_->io_queue->init(); @@ -161,22 +163,38 @@ void Scheduler::ContextImpl::add_to_queue(ActorInfoPtr actor_info_ptr, Scheduler if (need_poll || !info.cpu_queue) { info.io_queue->writer_put(std::move(actor_info_ptr)); } else { + auto token = static_cast(actor_info_ptr.release()); if (scheduler_id == get_scheduler_id() && cpu_worker_id_.is_valid()) { - // may push local - CHECK(actor_info_ptr); - auto raw = actor_info_ptr.release(); auto should_notify = info.cpu_local_queue[cpu_worker_id_.value()].push( - raw, [&](auto value) { info.cpu_queue->push(value, get_thread_id()); }); + token, [&](auto value) { info.cpu_queue->push(value, get_thread_id()); }); if (should_notify) { info.cpu_queue_waiter->notify(); } return; } - info.cpu_queue->push(actor_info_ptr.release(), get_thread_id()); + info.cpu_queue->push(token, get_thread_id()); info.cpu_queue_waiter->notify(); } } +void Scheduler::ContextImpl::add_token_to_cpu_queue(SchedulerToken token, SchedulerId scheduler_id) { + if (!scheduler_id.is_valid()) { + scheduler_id = get_scheduler_id(); + } + auto &info = scheduler_group()->schedulers.at(scheduler_id.value()); + if (scheduler_id == get_scheduler_id() && cpu_worker_id_.is_valid()) { + auto should_notify = info.cpu_local_queue[cpu_worker_id_.value()].push(token, [&](auto value) { + info.cpu_queue->push(value, get_thread_id()); + }); + if (should_notify) { + info.cpu_queue_waiter->notify(); + } + return; + } + info.cpu_queue->push(token, get_thread_id()); + info.cpu_queue_waiter->notify(); +} + ActorInfoCreator &Scheduler::ContextImpl::get_actor_info_creator() { return *creator_; } @@ -245,7 +263,7 @@ void Scheduler::ContextImpl::stop() { for (auto &scheduler_info : group.schedulers) { scheduler_info.io_queue->writer_put({}); for (size_t i = 0; i < scheduler_info.cpu_threads_count; i++) { - scheduler_info.cpu_queue->push({}, get_thread_id()); + scheduler_info.cpu_queue->push(nullptr, get_thread_id()); scheduler_info.cpu_queue_waiter->notify(); } } @@ -294,11 +312,18 @@ void Scheduler::close_scheduler_group(SchedulerGroupInfo &group_info) { for (auto &q : scheduler_info.cpu_local_queue) { auto &cpu_queue = q; while (true) { - SchedulerMessage::Raw *raw_message; + SchedulerToken raw_message; if (!cpu_queue.try_pop(raw_message)) { break; } - SchedulerMessage(SchedulerMessage::acquire_t{}, raw_message); + auto encoded = reinterpret_cast(raw_message); + if ((encoded & 1u) == 0u) { + SchedulerMessage::Raw *raw = reinterpret_cast(raw_message); + SchedulerMessage(SchedulerMessage::acquire_t{}, raw); + } else { + auto h = std::coroutine_handle<>::from_address(reinterpret_cast(encoded & ~uintptr_t(1))); + h.destroy(); + } // message's destructor is called queues_are_empty = false; } @@ -306,11 +331,18 @@ void Scheduler::close_scheduler_group(SchedulerGroupInfo &group_info) { if (scheduler_info.cpu_queue) { auto &cpu_queue = *scheduler_info.cpu_queue; while (true) { - SchedulerMessage::Raw *raw_message; + SchedulerToken raw_message; if (!cpu_queue.try_pop(raw_message, get_thread_id())) { break; } - SchedulerMessage(SchedulerMessage::acquire_t{}, raw_message); + auto encoded = reinterpret_cast(raw_message); + if ((encoded & 1u) == 0u) { + SchedulerMessage::Raw *raw = reinterpret_cast(raw_message); + SchedulerMessage(SchedulerMessage::acquire_t{}, raw); + } else { + auto h = std::coroutine_handle<>::from_address(reinterpret_cast(encoded & ~uintptr_t(1))); + h.destroy(); + } // message's destructor is called queues_are_empty = false; } diff --git a/tdactor/td/actor/core/Scheduler.h b/tdactor/td/actor/core/Scheduler.h index e76b919e10..85458294fb 100644 --- a/tdactor/td/actor/core/Scheduler.h +++ b/tdactor/td/actor/core/Scheduler.h @@ -55,10 +55,8 @@ #include #include -#include #include #include -#include #include namespace td { @@ -151,10 +149,10 @@ struct LocalQueue { struct SchedulerInfo { SchedulerId id; // will be read by all workers is any thread - std::unique_ptr> cpu_queue; + std::unique_ptr> cpu_queue; std::unique_ptr cpu_queue_waiter; - std::vector> cpu_local_queue; + std::vector> cpu_local_queue; //std::vector> cpu_stealing_queue; // only scheduler itself may read from io_queue_ @@ -244,6 +242,7 @@ class Scheduler { SchedulerId get_scheduler_id() const override; void add_to_queue(ActorInfoPtr actor_info_ptr, SchedulerId scheduler_id, bool need_poll) override; + void add_token_to_cpu_queue(SchedulerToken token, SchedulerId scheduler_id) override; ActorInfoCreator &get_actor_info_creator() override; diff --git a/tdactor/td/actor/core/SchedulerContext.h b/tdactor/td/actor/core/SchedulerContext.h index 99b1922f5e..55dc8fa45c 100644 --- a/tdactor/td/actor/core/SchedulerContext.h +++ b/tdactor/td/actor/core/SchedulerContext.h @@ -28,6 +28,10 @@ namespace td { namespace actor { namespace core { + +// Token type for CPU queue - encodes either ActorInfo* (bit 0 = 0) or coroutine handle (bit 0 = 1) +using SchedulerToken = void*; + class SchedulerDispatcher { public: virtual ~SchedulerDispatcher() = default; @@ -35,6 +39,7 @@ class SchedulerDispatcher { virtual SchedulerId get_scheduler_id() const = 0; virtual void add_to_queue(ActorInfoPtr actor_info_ptr, SchedulerId scheduler_id, bool need_poll) = 0; virtual void set_alarm_timestamp(const ActorInfoPtr &actor_info_ptr) = 0; + virtual void add_token_to_cpu_queue(SchedulerToken token, SchedulerId scheduler_id) = 0; }; struct Debug; diff --git a/tdactor/td/actor/coro.h b/tdactor/td/actor/coro.h new file mode 100644 index 0000000000..3013b57348 --- /dev/null +++ b/tdactor/td/actor/coro.h @@ -0,0 +1,11 @@ +#pragma once + +#include "td/actor/coro_types.h" +#include "td/actor/coro_executor.h" +#include "td/actor/coro_awaitables.h" +#include "td/actor/coro_task.h" +#include "td/actor/coro_utils.h" + +namespace td::actor { + +} // namespace td::actor \ No newline at end of file diff --git a/tdactor/td/actor/coro_awaitables.h b/tdactor/td/actor/coro_awaitables.h new file mode 100644 index 0000000000..2296d73ea4 --- /dev/null +++ b/tdactor/td/actor/coro_awaitables.h @@ -0,0 +1,234 @@ +#pragma once + +#include "td/actor/coro_types.h" +#include "td/actor/coro_executor.h" +#include "td/utils/Status.h" + +#include +#include +#include +#include + +namespace td::actor { + +namespace detail { + +template +struct WrappedCoroutine { + struct promise_type { + Body* body{}; + std::coroutine_handle outer{}; + + promise_type(Body* b, std::coroutine_handle o) noexcept : body(b), outer(o) { + } + + WrappedCoroutine get_return_object() noexcept { + return WrappedCoroutine{std::coroutine_handle::from_promise(*this)}; + } + std::suspend_always initial_suspend() noexcept { + return {}; + } + auto final_suspend() noexcept { + struct A { + promise_type* p; + bool await_ready() noexcept { + return false; + } + std::coroutine_handle<> await_suspend(std::coroutine_handle self) noexcept { + auto next = p->body->route_resume(p->outer); + self.destroy(); + return next; + } + void await_resume() noexcept { + } + }; + return A{this}; + } + void return_void() noexcept { + } + void unhandled_exception() noexcept { + std::terminate(); + } + }; + + using handle = std::coroutine_handle; + handle h{}; + explicit WrappedCoroutine(handle hh) : h(hh) { + } + ~WrappedCoroutine() { + if (h) + h.destroy(); + } + WrappedCoroutine& operator=(WrappedCoroutine&& o) = delete; + WrappedCoroutine(const WrappedCoroutine&) = delete; + WrappedCoroutine& operator=(const WrappedCoroutine&) = delete; +}; + +template +[[nodiscard]] WrappedCoroutine make_wrapped_coroutine( + BodyT* b, std::coroutine_handle o) noexcept { + co_return; +} + +template +[[nodiscard]] std::coroutine_handle<> wrap_coroutine(BodyT* body, std::coroutine_handle outer) noexcept { + auto tmp = make_wrapped_coroutine(body, outer); + return std::exchange(tmp.h, {}); +} + +template +inline constexpr bool has_peek = requires(Aw& a) { a.await_resume_peek(); }; + +template +struct TaskUnwrapAwaiter { + using A = std::remove_cvref_t; + using Res = decltype(std::declval().await_resume()); + using Ok = decltype(std::declval().move_as_ok()); + + [[no_unique_address]] A aw; + + using Cache = std::conditional_t, td::Unit, std::optional>; + [[no_unique_address]] Cache ok_; + + bool await_ready() noexcept { + // If aw.await_ready() is true and not error, we may continue execution right away + if constexpr (has_peek) { + if (aw.await_ready() && !aw.await_resume_peek().is_error()) { + return true; + } + } + return false; + } + + template + std::coroutine_handle<> route_resume(std::coroutine_handle h) noexcept { + if constexpr (has_peek) { + const Res& r = aw.await_resume_peek(); + if (r.is_error()) { + auto rr = aw.await_resume(); + return h.promise().route_finish(std::move(rr).move_as_error()); + } + return h.promise().route_resume(); + } else { + Res r = aw.await_resume(); + if (r.is_error()) { + return h.promise().route_finish(std::move(r).move_as_error()); + } + ok_.emplace(std::move(r).move_as_ok()); + return h.promise().route_resume(); + } + } + + template + std::coroutine_handle<> await_suspend(std::coroutine_handle h) noexcept { + if constexpr (requires { aw.await_ready(); }) { + if (aw.await_ready()) { + return route_resume(h); + } + } + if (h.promise().is_immediate_execution_always_allowed()) { + return await_suspend_to(aw, h); + } + auto r_handle = wrap_coroutine(this, h); + return await_suspend_to(aw, r_handle); + } + + Ok await_resume() noexcept { + if constexpr (!has_peek) { + if (ok_) { + return std::move(*ok_); + } + } + Res r = aw.await_resume(); + return std::move(r).move_as_ok(); + } +}; + +template +struct TaskWrapAwaiter { + using A = std::remove_cvref_t; + using Res = decltype(std::declval().await_resume()); + + [[no_unique_address]] A aw; + + bool await_ready() noexcept { + return aw.await_ready(); + } + + template + std::coroutine_handle<> route_resume(std::coroutine_handle h) noexcept { + return h.promise().route_resume(); + } + + template + std::coroutine_handle<> await_suspend(std::coroutine_handle h) noexcept { + if constexpr (requires { aw.await_ready(); }) { + if (aw.await_ready()) { + return route_resume(h); + } + } + if (h.promise().is_immediate_execution_always_allowed()) { + return await_suspend_to(aw, h); + } + auto r_handle = wrap_coroutine(this, h); + return await_suspend_to(aw, r_handle); + } + + Res await_resume() noexcept { + return aw.await_resume(); + } +}; + +template +struct ResultUnwrapAwaiter { + using Res = std::remove_cvref_t; + using Ok = decltype(std::declval().move_as_ok()); + + Res result; + + bool await_ready() noexcept { + return result.is_ok(); + } + + template + std::coroutine_handle<> await_suspend(std::coroutine_handle h) noexcept { + if (result.is_error()) { + // This code is not generic and currently only used in Task and SharedTask implementations + h.promise().return_value(std::move(result).move_as_error()); + return h.promise().final_suspend().await_suspend(h); + } + return h; + } + + Ok await_resume() noexcept { + return std::move(result).move_as_ok(); + } +}; + +} // namespace detail + +// These helpers are used via await_transform to: +// 1) Ensure the awaiting coroutine resumes on the current task's scheduler. +// 2) Optionally unwrap td::Result. If it’s an error, propagate it to the parent +// as if using co_return error; so `co_await get_error();` is equivalent to `co_return get_error();`. +template +[[nodiscard]] auto unwrap_and_resume_on_current(Aw&& aw_) noexcept { + return detail::TaskUnwrapAwaiter{std::forward(aw_), {}}; +} + +template +[[nodiscard]] auto wrap_and_resume_on_current(Aw&& aw_) noexcept { + return detail::TaskWrapAwaiter{std::forward(aw_)}; +} + +template +[[nodiscard]] auto result_awaiter_unwrap(Result&& r) noexcept { + return detail::ResultUnwrapAwaiter>(std::move(r)); +} + +template +[[nodiscard]] auto result_awaiter_wrap(Result&& r) noexcept { + return detail::ReadyAwaitable>(std::move(r)); +} + +} // namespace td::actor diff --git a/tdactor/td/actor/coro_executor.h b/tdactor/td/actor/coro_executor.h new file mode 100644 index 0000000000..6871529e13 --- /dev/null +++ b/tdactor/td/actor/coro_executor.h @@ -0,0 +1,237 @@ +#pragma once + +#include "td/actor/actor.h" +#include "td/actor/core/SchedulerContext.h" +#include "td/actor/core/SchedulerId.h" +#include "td/actor/coro_types.h" + +#include +#include +#include + +namespace td::actor { + +namespace detail { + +inline ActorId<> get_current_actor_id() noexcept { + auto context = core::ActorExecuteContext::get(); + if (context && context->actor_ptr()) { + return actor_id(context->actor_ptr()); + } + return td::actor::ActorId<>{}; +} + +template +class ActorMessageCoroutineSafe : public core::ActorMessageImpl { + public: + explicit ActorMessageCoroutineSafe(std::coroutine_handle

continuation) : continuation_(continuation) { + } + void run() override { + continuation_.resume(); + continuation_ = {}; + } + ~ActorMessageCoroutineSafe() override; + + private: + std::coroutine_handle

continuation_; +}; + +// Executor +// - schedule (sometime, NOT right now) +// - execute_or_schedule - it is ok to start executing immediately +// - for any just execute +// - for scheduler just execute +// - for actor try to execute immediately and schedule if can't +// - resume_or_schedule - if we are already executing we may continue otherwise schedule. + +struct ActorExecutor { + td::actor::ActorId<> actor; + bool is_immediate_execution_allowed() const noexcept { + return actor == get_current_actor_id(); + } + bool is_immediate_execution_always_allowed() const noexcept { + return false; + } + template + [[nodiscard]] std::coroutine_handle<> resume_or_schedule(std::coroutine_handle

cont) noexcept { + if (is_immediate_execution_allowed()) { + return cont; + } + schedule(std::move(cont)); + return std::noop_coroutine(); + } + template + static auto to_message(std::coroutine_handle

cont) noexcept { + return td::actor::core::ActorMessage{std::make_unique>(std::move(cont))}; + } + template + [[nodiscard]] std::coroutine_handle<> execute_or_schedule(std::coroutine_handle

cont) noexcept { + if (is_immediate_execution_allowed()) { + return cont; + } + td::actor::detail::send_immediate( + actor.as_actor_ref(), [&] { cont.resume(); }, [&]() { return to_message(std::move(cont)); }); + return std::noop_coroutine(); + } + template + void schedule(std::coroutine_handle

cont) noexcept { + td::actor::detail::send_message_later(actor.as_actor_ref(), to_message(std::move(cont))); + } +}; + +struct SchedulerExecutor { + bool is_immediate_execution_allowed() const noexcept { + return get_current_actor_id().empty(); + } + bool is_immediate_execution_always_allowed() const noexcept { + return false; + } + [[nodiscard]] std::coroutine_handle<> resume_or_schedule(std::coroutine_handle<> cont) noexcept { + return execute_or_schedule(std::move(cont)); + } + [[nodiscard]] std::coroutine_handle<> execute_or_schedule(std::coroutine_handle<> cont) noexcept { + if (is_immediate_execution_allowed()) { + return cont; + } + schedule(std::move(cont)); + return std::noop_coroutine(); + } + void schedule(std::coroutine_handle<> cont) noexcept { + auto token = reinterpret_cast(encode_continuation(cont)); + auto ctx = td::actor::core::SchedulerContext::get(); + CHECK(ctx); + ctx->add_token_to_cpu_queue(token, td::actor::core::SchedulerId{}); + } +}; + +struct AnyExecutor { + bool is_immediate_execution_allowed() const noexcept { + return true; + } + bool is_immediate_execution_always_allowed() const noexcept { + return true; + } + [[nodiscard]] std::coroutine_handle<> resume_or_schedule(std::coroutine_handle<> cont) noexcept { + return execute_or_schedule(std::move(cont)); + } + [[nodiscard]] std::coroutine_handle<> execute_or_schedule(std::coroutine_handle<> cont) noexcept { + return cont; + } + void schedule(std::coroutine_handle<> cont) noexcept { + LOG(ERROR) << "Schedule on any executor"; + SchedulerExecutor{}.schedule(cont); + } +}; + +struct Executor { + std::variant executor_{SchedulerExecutor{}}; + + static Executor on_actor(td::actor::ActorId<> actor) noexcept { + return {ActorExecutor{std::move(actor)}}; + } + template + static Executor on_actor(const td::actor::ActorOwn& actor) noexcept { + return {ActorExecutor{actor.get()}}; + } + static Executor on_scheduler() noexcept { + return {SchedulerExecutor{}}; + } + static Executor on_any() noexcept { + return {AnyExecutor{}}; + } + static Executor on_current_actor() noexcept { + return on_actor(get_current_actor_id()); + } + static Executor on_default() noexcept { + auto current_actor_id = get_current_actor_id(); + return current_actor_id.empty() ? on_scheduler() : on_actor(std::move(current_actor_id)); + } + + bool is_immediate_execution_allowed() const noexcept { + return visit([](auto& v) { return v.is_immediate_execution_allowed(); }, executor_); + //return executor_.visit([&](auto& v) { return v.is_immediate_execution_allowed(); }); + } + bool is_immediate_execution_always_allowed() const noexcept { + return visit([](auto& v) { return v.is_immediate_execution_always_allowed(); }, executor_); + } + template + [[nodiscard]] std::coroutine_handle<> resume_or_schedule(std::coroutine_handle

cont) noexcept { + return visit([&](auto& v) { return v.resume_or_schedule(std::move(cont)); }, executor_); + } + template + [[nodiscard]] std::coroutine_handle<> execute_or_schedule(std::coroutine_handle

cont) noexcept { + return visit([&](auto& v) { return v.execute_or_schedule(std::move(cont)); }, executor_); + } + + template + void schedule(std::coroutine_handle

cont) noexcept { + return visit([&](auto& v) { return v.schedule(std::move(cont)); }, executor_); + } +}; + +template +ActorMessageCoroutineSafe

::~ActorMessageCoroutineSafe() { + if (continuation_) { + SchedulerExecutor{}.schedule(continuation_.promise().route_finish(td::Status::Error("Actor destroyed"))); + } +} + +struct ResumeOn { + Executor executor; + + bool await_ready() noexcept { + return executor.is_immediate_execution_allowed(); + } + + template + std::coroutine_handle<> await_suspend(std::coroutine_handle

cont) noexcept { + return executor.resume_or_schedule(std::move(cont)); + } + + void await_resume() noexcept { + } +}; + +struct YieldOn { + Executor executor; + bool await_ready() noexcept { + return false; + } + template + std::coroutine_handle<> await_suspend(std::coroutine_handle

self) noexcept { + executor.schedule(std::move(self)); + return std::noop_coroutine(); + } + void await_resume() noexcept { + } +}; + +} // namespace detail + +using Executor = detail::Executor; + +[[nodiscard]] inline auto resume_on(Executor executor) noexcept { + return detail::ResumeOn{std::move(executor)}; +} + +[[nodiscard]] inline auto yield_on(Executor executor) noexcept { + return detail::YieldOn{std::move(executor)}; +} + +inline auto attach_to_actor(td::actor::ActorId<> actor_id) noexcept { + return detail::ResumeOn{Executor::on_actor(actor_id)}; +} + +inline auto detach_from_actor() noexcept { + return detail::ResumeOn{Executor::on_scheduler()}; +} + +inline auto become_lightweight() noexcept { + return detail::ResumeOn{Executor::on_any()}; +} + +inline Yield yield_on_current() noexcept { + return Yield{}; +} + +} // namespace td::actor diff --git a/tdactor/td/actor/coro_task.h b/tdactor/td/actor/coro_task.h new file mode 100644 index 0000000000..accf714ca3 --- /dev/null +++ b/tdactor/td/actor/coro_task.h @@ -0,0 +1,469 @@ +#pragma once + +#include "td/actor/coro_types.h" +#include "td/actor/coro_executor.h" +#include "td/actor/coro_awaitables.h" +#include "td/actor/PromiseFuture.h" +#include "td/utils/Status.h" + +#include +#include +#include +#include +#include + +namespace td::actor { + +namespace detail { + +struct TaskStateManagerData { + Executor executor{Executor::on_default()}; + std::atomic flags{0}; + std::coroutine_handle<> continuation{}; + + enum Flags : uint8_t { + READY_FLAG = 1, + STARTED_FLAG = 2, + SUSPEND_FLAG = 4, + DETACH_FLAG = 8, + }; + + uint8_t set_flag(uint8_t new_flag) noexcept { + auto old_flags = flags.fetch_or(new_flag, std::memory_order_acq_rel); + CHECK((old_flags & new_flag) == 0); + return old_flags; + } + + [[nodiscard]] std::coroutine_handle<> on_ready(std::coroutine_handle<> self_handle) { + auto old_flags = set_flag(READY_FLAG); + + if (!(old_flags & STARTED_FLAG)) { + return continuation; + } + + std::coroutine_handle<> next = std::noop_coroutine(); + if (old_flags & SUSPEND_FLAG) { + next = continuation; + } + if (old_flags & DETACH_FLAG) { + self_handle.destroy(); + } + return next; + } + + void set_executor(Executor new_executor) noexcept { + executor = std::move(new_executor); + } +}; + +template +struct TaskStateManager { + using Data = TaskStateManagerData; + using Coro = std::coroutine_handle<>; + using Self = std::coroutine_handle

; + Data* data; + + void set_executor(Executor executor) noexcept { + data->executor = std::move(executor); + } + + void start(Self self) { + set_is_started(); + data->executor.schedule(self); + } + void start_immediate(Self self) { + set_is_started(); + data->executor.execute_or_schedule(self).resume(); + } + void start_external() { + set_is_started(); + } + + bool is_ready() const noexcept { + return false; + } + + [[nodiscard]] std::coroutine_handle<> on_suspend_and_start(Self self, Coro continuation) { + data->continuation = continuation; + return data->executor.execute_or_schedule(self); + } + + void on_detach(Self self) { + self.destroy(); + } + + private: + void set_is_started() { + data->flags.fetch_or(Data::STARTED_FLAG, std::memory_order_relaxed); + } +}; + +template +struct StartedTaskStateManager { + using Data = TaskStateManagerData; + using Coro = std::coroutine_handle<>; + using Self = std::coroutine_handle

; + Data* data; + + bool is_ready() const noexcept { + return data->flags.load(std::memory_order_acquire) & Data::READY_FLAG; + } + + [[nodiscard]] std::coroutine_handle<> on_suspend(Coro new_continuation) { + CHECK(!data->continuation); + data->continuation = new_continuation; + auto old_flags = data->set_flag(Data::SUSPEND_FLAG); + return (old_flags & Data::READY_FLAG) ? new_continuation : std::noop_coroutine(); + } + + void on_detach(Self self) { + auto old_flags = data->set_flag(Data::DETACH_FLAG); + if (old_flags & Data::READY_FLAG) { + self.destroy(); + } + } +}; + +} // namespace detail + +struct promise_common { + detail::TaskStateManagerData state_manager_data; +}; + +template +struct promise_value : promise_common { + [[no_unique_address]] ResultT result; + + template + void return_value(TT&& v) noexcept { + result = std::forward(v); + } + + struct ExternalResult {}; + void return_value(ExternalResult&&) noexcept { + } + + void unhandled_exception() noexcept { + result = td::Status::Error("unhandled exception in coroutine"); + } + + ResultT extract_result() noexcept { + return std::move(result); + } +}; + +template +struct Task; + +template +struct StartedTask; + +template +struct promise_type : promise_value> { + static_assert(!std::is_void_v, "Task is not supported; use Task instead"); + using Handle = std::coroutine_handle; + + auto self() noexcept { + return Handle::from_promise(*this); + } + + template + void external_return_value(TT&& v) noexcept + requires(!std::is_void_v) + { + route_finish(std::forward(v)).resume(); + } + + Task get_return_object() noexcept { + return Task{Handle::from_promise(*this)}; + } + + std::suspend_always initial_suspend() noexcept { + return {}; + } + + auto final_suspend() noexcept { + struct Final { + bool await_ready() noexcept { + return false; + } + std::coroutine_handle<> await_suspend(Handle self) noexcept { + return self.promise().state_manager_data.on_ready(self); + } + void await_resume() noexcept { + } + }; + return Final{}; + } + + auto await_transform(detail::YieldOn y) { + this->state_manager_data.set_executor(y.executor); + return y; + } + auto await_transform(detail::ResumeOn y) { + this->state_manager_data.set_executor(y.executor); + return y; + } + + template + auto await_transform(SkipAwaitTransform wrapped_aw) noexcept { + return std::move(wrapped_aw.awaitable); + } + + auto await_transform(Yield) noexcept { + return yield_on(this->state_manager_data.executor); + } + + auto await_transform(std::suspend_always) noexcept { + return std::suspend_always{}; + } + + auto await_transform(std::suspend_never) noexcept { + return std::suspend_always{}; + } + + template + auto await_transform(td::ResultUnwrap wrapped) noexcept { + return await_transform(std::move(wrapped.result)); + } + template + auto await_transform(td::ResultWrap wrapped) noexcept { + return await_transform(Wrapped>{std::move(wrapped.result)}); + } + + template + auto await_transform(Wrapped> wrapped) noexcept { + return result_awaiter_wrap(std::move(wrapped.value)); + } + + template + auto await_transform(td::Result&& result) noexcept { + return result_awaiter_unwrap(std::move(result)); + } + + template + auto await_transform(Task&& task) noexcept { + return unwrap_and_resume_on_current(std::move(task).start_immediate()); + } + template + auto await_transform(StartedTask&& task) noexcept { + return unwrap_and_resume_on_current(std::move(task)); + } + + template + auto await_transform(Wrapped>&& wrapped) noexcept { + return wrap_and_resume_on_current(std::move(wrapped.value)); + } + template + auto await_transform(Wrapped>&& wrapped) noexcept { + return wrap_and_resume_on_current(std::move(wrapped.value)); + } + + template + auto await_transform(Aw&& aw) noexcept { + return wrap_and_resume_on_current(std::forward(aw)); + } + + // API used by TaskWrapAwaiter and TaskUnwrapAwaiter + bool is_immediate_execution_always_allowed() const noexcept { + return this->state_manager_data.executor.is_immediate_execution_always_allowed(); + } + + std::coroutine_handle<> route_resume() { + return this->state_manager_data.executor.resume_or_schedule(self()); + } + + std::coroutine_handle<> route_finish(td::Result r) { + this->return_value(std::move(r)); + return final_suspend().await_suspend(self()); + } +}; + +template +struct [[nodiscard]] Task { + using value_type = T; + + using promise_type = promise_type; + using Handle = std::coroutine_handle; + Handle h{}; + + Task() = default; + explicit Task(Handle hh) : h(hh) { + } + Task(Task&& o) noexcept : h(std::exchange(o.h, {})) { + } + Task& operator=(Task&& o) = delete; + Task(const Task&) = delete; + Task& operator=(const Task&) = delete; + + auto sm() { + return detail::TaskStateManager{&h.promise().state_manager_data}; + } + + ~Task() noexcept { + detach(); + } + void detach() { + if (!h) { + return; + } + sm().on_detach(h); + h = {}; + } + + auto start() && { + sm().start(h); + return StartedTask{std::exchange(h, {})}; + } + auto start_immediate() && { + sm().start_immediate(h); + return StartedTask{std::exchange(h, {})}; + } + auto start_external() && { + sm().start_external(); + return StartedTask{std::exchange(h, {})}; + } + void set_executor(Executor new_executor) { + CHECK(h); + sm().set_executor(std::move(new_executor)); + } + + constexpr bool await_ready() noexcept { + return sm().is_ready(); + } + + std::coroutine_handle<> await_suspend(std::coroutine_handle<> continuation) noexcept { + CHECK(h); + return sm().on_suspend_and_start(h, continuation); + } + td::Result await_resume() noexcept { + CHECK(h); + return h.promise().extract_result(); + } + + const td::Result& await_resume_peek() const noexcept { + CHECK(h); + return h.promise().result; + } + + auto wrap() && { + return Wrapped{std::move(*this)}; + } +}; + +template +struct [[nodiscard]] StartedTask { + using value_type = T; + + using promise_type = promise_type; + using Handle = std::coroutine_handle; + Handle h{}; + + auto sm() { + CHECK(h); + return detail::StartedTaskStateManager{&h.promise().state_manager_data}; + } + StartedTask() = default; + explicit StartedTask(Handle hh) : h(hh) { + CHECK(h); + } + StartedTask(StartedTask&& o) noexcept : h(std::exchange(o.h, {})) { + } + StartedTask& operator=(StartedTask&& o) = delete; + StartedTask(const StartedTask&) = delete; + StartedTask& operator=(const StartedTask&) = delete; + + ~StartedTask() noexcept { + detach(); + } + void detach() { + if (!h) { + return; + } + sm().on_detach(h); + h = {}; + } + bool await_ready() noexcept { + return sm().is_ready(); + } + + std::coroutine_handle<> await_suspend(std::coroutine_handle<> continuation) noexcept { + return sm().on_suspend(continuation); + } + td::Result await_resume() noexcept { + return h.promise().extract_result(); + } + + const td::Result& await_resume_peek() const noexcept { + CHECK(h); + return h.promise().result; + } + + auto wrap() && { + return Wrapped{std::move(*this)}; + } + + template + auto then(F&& f) && { + using Self = StartedTask; + using FDecayed = std::decay_t; + using Awaitable = decltype(detail::make_awaitable(std::declval()(std::declval()))); + using Ret = decltype(std::declval().await_resume()); + using U = std::conditional_t, decltype(std::declval().move_as_ok()), Ret>; + return [](Self task, FDecayed fn) mutable -> Task { + co_await become_lightweight(); + auto value = co_await std::move(task); + co_return co_await detail::make_awaitable(fn(std::move(value))); + }(std::move(*this), std::forward(f)); + } + + struct ExternalPromise : public PromiseInterface { + ExternalPromise() = default; + explicit ExternalPromise(promise_type* p) : promise(p) { + } + void set_value(T&& value) override { + promise.release()->external_return_value(std::move(value)); + } + void set_error(Status&& error) override { + promise.release()->external_return_value(std::move(error)); + } + + operator bool() const { + return bool(promise); + } + + struct Deleter { + void operator()(promise_type* p) { + p->external_return_value(td::Status::Error("promise destroyed")); + } + }; + std::unique_ptr promise{}; + }; + + static std::pair make_bridge() { + auto task = []() -> Task { co_return typename promise_type::ExternalResult{}; }(); + task.set_executor(Executor::on_scheduler()); + auto promise = ExternalPromise(&task.h.promise()); + auto started_task = std::move(task).start_external(); + return std::make_pair(std::move(started_task), std::move(promise)); + } +}; + +template +void custom_connect(P&& p, StartedTask&& mt) noexcept { + if (mt.await_ready()) { + connect(std::move(p), mt.await_resume()); + return; + } + [](auto promise, auto mt) mutable -> detail::FireAndForget { + auto result = co_await mt; + connect(std::move(promise), std::move(result)); + }(std::forward

(p), std::move(mt)); +} + +template +void custom_connect(P&& p, Task&& t) noexcept { + connect(std::forward

(p), std::move(t).start_immediate()); +} + +} // namespace td::actor diff --git a/tdactor/td/actor/coro_types.h b/tdactor/td/actor/coro_types.h new file mode 100644 index 0000000000..ae65dd70ca --- /dev/null +++ b/tdactor/td/actor/coro_types.h @@ -0,0 +1,114 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace td::actor { + + +template +concept TDResultLike = requires(R r) { + { r.is_error() } -> std::convertible_to; + r.move_as_ok(); + r.move_as_error(); +}; + +template +concept IsAwaitable = requires(T& t, std::coroutine_handle<> h) { + t.await_suspend(h); + t.await_resume(); +}; + +template +concept CoroTask = requires(T t) { + typename T::promise_type; + typename T::value_type; + { t.await_suspend(std::coroutine_handle<>{}) }; + { t.await_resume() }; +}; + +namespace detail { +template +inline std::coroutine_handle<> await_suspend_to(Aw& aw, std::coroutine_handle<> cont) noexcept { + if constexpr (std::is_same_v) { + aw.await_suspend(cont); + return std::noop_coroutine(); + } else if constexpr (std::is_same_v) { + if (!aw.await_suspend(cont)) { + return cont; + } + return std::noop_coroutine(); + } else { + return aw.await_suspend(cont); + } +} + +struct FireAndForget { + struct promise_type { + FireAndForget get_return_object() noexcept { + return {}; + } + std::suspend_never initial_suspend() noexcept { + return {}; + } + std::suspend_never final_suspend() noexcept { + return {}; + } + void return_void() noexcept { + } + void unhandled_exception() noexcept { + } + }; +}; + +template +struct ReadyAwaitable { + [[no_unique_address]] T value; + constexpr bool await_ready() const noexcept { + return true; + } + constexpr void await_suspend(std::coroutine_handle<>) const noexcept { + } + T await_resume() noexcept { + return std::move(value); + } +}; + +template +auto make_awaitable(X&& x) { + using DX = std::decay_t; + if constexpr (IsAwaitable) { + return std::forward(x); + } else { + return ReadyAwaitable{std::forward(x)}; + } +} + +inline uintptr_t encode_continuation(std::coroutine_handle<> h) noexcept { + auto p = h.address(); + auto v = reinterpret_cast(p); + return v | 1u; +} + +inline std::coroutine_handle<> decode_continuation(uintptr_t token) noexcept { + return std::coroutine_handle<>::from_address(reinterpret_cast(token & ~uintptr_t(1))); +} + +} // namespace detail + +template +struct [[nodiscard]] SkipAwaitTransform { + [[no_unique_address]] T awaitable; +}; + +template +struct Wrapped { + [[no_unique_address]] T value; +}; + +struct [[nodiscard]] Yield {}; + +} // namespace td::actor diff --git a/tdactor/td/actor/coro_utils.h b/tdactor/td/actor/coro_utils.h new file mode 100644 index 0000000000..0abb729673 --- /dev/null +++ b/tdactor/td/actor/coro_utils.h @@ -0,0 +1,268 @@ +#pragma once + +#include "td/actor/coro_types.h" +#include "td/actor/coro_executor.h" +#include "td/actor/coro_task.h" +#include "td/actor/actor.h" +#include "td/utils/Status.h" +#include "td/utils/Slice.h" + +#include +#include +#include +#include + +namespace td::actor { + +namespace detail { + +template +struct memfn_meta; + +template +struct remove_memfn_const { + using type = T; +}; +template +struct remove_memfn_const { + using type = R (C::*)(Args...); +}; +template +using remove_memfn_const_t = remove_memfn_const::type; + +template +struct memfn_meta : memfn_meta> {}; +template +struct memfn_meta { + using cls = C; + using ret = R; +}; + +template +struct unwrap_promise { + using type = void; +}; +template +struct unwrap_promise> { + using type = T; +}; + +template +using last_t = std::tuple_element_t>; + +template +struct is_task : std::false_type {}; +template +struct is_task> : std::true_type {}; + +} // namespace detail + +template +td::Result> collect(std::vector>&& results) { + for (auto& result : results) { + if (result.is_error()) { + return result.move_as_error(); + } + } + std::vector values; + values.reserve(results.size()); + for (auto& result : results) { + values.push_back(result.move_as_ok()); + } + return values; +} + +template +td::Result> collect(std::tuple...> results) { + return std::apply( + [](auto&&... results) -> td::Result> { + td::Status error; + bool has_error = false; + + (void)((results.is_error() ? (has_error = true, error = results.move_as_error(), false) : true) && ...); + + if (has_error) { + return std::move(error); + } + + return std::tuple{std::move(results).move_as_ok()...}; + }, + std::move(results)); +} + +template +using await_result_t = + decltype(std::declval::promise_type>().await_transform(std::declval()).await_resume()); + +template 1), int> = 0> +auto all(Awaitables&&... awaitables) -> Task...>> { + co_await become_lightweight(); + co_return std::tuple...>{co_await std::forward(awaitables)...}; +} + +template 1), int> = 0> +auto all_wrap(Awaitables&&... awaitables) -> Task>...>> { + co_await become_lightweight(); + co_return std::tuple>...>{ + co_await Wrapped{std::forward(awaitables)}...}; +} + +template +Task>> all(std::vector tasks) { + co_await become_lightweight(); + std::vector> results; + results.reserve(tasks.size()); + // TODO: auto start + for (auto& task : tasks) { + results.push_back(co_await std::move(task)); + } + co_return results; +} + +template +Task>>> all_wrap(std::vector tasks) { + co_await become_lightweight(); + std::vector>> results; + // TODO: auto start + results.reserve(tasks.size()); + for (auto& task : tasks) { + results.push_back(co_await Wrapped{std::move(task)}); + } + co_return results; +} + +enum class UnifiedKind : uint8_t { None, Void, TaskReturn, PromiseArgument, ReturnValue }; + +template +struct unified_result; + +template +struct unified_result : unified_result> {}; + +template +struct unified_result (C::*)(Args...)> { + using type = T; + static constexpr UnifiedKind kind = UnifiedKind::ReturnValue; +}; + +template +struct unified_result (C::*)(Args...)> { + using type = T; + static constexpr UnifiedKind kind = UnifiedKind::TaskReturn; +}; + +template +struct unified_result { + using type = T; + static constexpr UnifiedKind kind = UnifiedKind::ReturnValue; +}; + +template +struct unified_result { + using Last = std::remove_cvref_t>; + static constexpr bool is_promise = std::is_same_v::type>>; + using type = std::conditional_t::type, void>; + static constexpr UnifiedKind kind = is_promise ? UnifiedKind::PromiseArgument : UnifiedKind::Void; +}; + +template +auto ask_impl(TargetId&& to, MemFn mf, Args&&... args) { + using Meta = unified_result; + using T = Meta::type; + using Ret = detail::memfn_meta::ret; + + using TT = std::conditional_t, td::Unit, T>; + + static_assert(Meta::kind == UnifiedKind::TaskReturn || Meta::kind == UnifiedKind::PromiseArgument || + Meta::kind == UnifiedKind::ReturnValue || Meta::kind == UnifiedKind::Void, + "ask: method must return Task or take td::Promise as last parameter"); + + if constexpr (Meta::kind == UnifiedKind::TaskReturn) { + return ask_new_impl(std::forward(to), mf, std::forward(args)...); + } + + auto [task, promise] = StartedTask::make_bridge(); + td::actor::internal::send_closure_dispatch(std::forward(to), mf, std::forward(args)..., + std::move(promise)); + return std::move(task); +} + +template +auto ask_new_impl(TargetId&& to, MemFn mf, Args&&... args) { + using Meta = unified_result; + using T = Meta::type; + static_assert(Meta::kind == UnifiedKind::TaskReturn, "ask: method must return Task"); + if constexpr (Later) { + auto task = [](auto closure) -> Task { + co_return co_await detail::run_on_current_actor(closure); + }(create_delayed_closure(mf, std::forward(args)...)); + task.set_executor(Executor::on_actor(to)); + return std::move(task).start(); + } else { + std::optional> o_task; + td::actor::detail::send_immediate( + to.as_actor_ref(), + [&] { + o_task.emplace(detail::run_on_current_actor(create_immediate_closure(mf, std::forward(args)...)) + .start_immediate()); + }, + [&]() { + auto task = [](auto closure) -> Task { + co_return co_await detail::run_on_current_actor(closure); + }(create_delayed_closure(mf, std::forward(args)...)); + task.set_executor(Executor::on_actor(to)); + o_task.emplace(std::move(task).start_external()); + return detail::ActorExecutor::to_message(o_task->h); + }); + return std::move(*o_task); + } +} + +template +auto ask_new(Args&&... args) { + return ask_new_impl(std::forward(args)...); +} + +template +auto ask_new_immediate(Args&&... args) { + return ask_new_impl(std::forward(args)...); +} + +template +auto ask(Args&&... args) { + return ask_impl(std::forward(args)...); +} + +template +auto ask_immediate(Args&&... args) { + return ask_impl(std::forward(args)...); +} + +template +auto ask_promise(Args&&... args) { + return ask(std::forward(args)...); +} + +template +auto spawn_actor(td::Slice name, TaskType task) { + using StartedTaskType = StartedTask; + using PromiseType = StartedTaskType::ExternalPromise; + auto [result_task, result_promise] = StartedTaskType::make_bridge(); + + struct TaskAwaiter : public td::actor::Actor { + TaskAwaiter(TaskType task, PromiseType promise) : task_(std::move(task)), promise_(std::move(promise)) { + } + + private: + TaskType task_; + PromiseType promise_; + void start_up() { + task_.set_executor(Executor::on_current_actor()); + connect(std::move(promise_), std::move(task_)); + } + }; + td::actor::create_actor(name, std::move(task), std::move(result_promise)).release(); + return std::move(result_task); +} + +} // namespace td::actor diff --git a/tdactor/test/CMakeLists.txt b/tdactor/test/CMakeLists.txt new file mode 100644 index 0000000000..17a71612c6 --- /dev/null +++ b/tdactor/test/CMakeLists.txt @@ -0,0 +1,3 @@ +add_executable(test-coro test-coro.cpp) +target_link_libraries(test-coro tdactor tdutils) +target_include_directories(test-coro PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../) diff --git a/tdactor/test/actors_core.cpp b/tdactor/test/actors_core.cpp index 1c56d1e5b0..43eb67dc46 100644 --- a/tdactor/test/actors_core.cpp +++ b/tdactor/test/actors_core.cpp @@ -279,6 +279,9 @@ TEST(Actor2, executor_simple) { void add_to_queue(ActorInfoPtr ptr, SchedulerId scheduler_id, bool need_poll) override { queue.push_back(std::move(ptr)); } + void add_token_to_cpu_queue(SchedulerToken token, SchedulerId scheduler_id) override { + UNREACHABLE(); + } void set_alarm_timestamp(const ActorInfoPtr &actor_info_ptr) override { UNREACHABLE(); } @@ -326,7 +329,9 @@ TEST(Actor2, executor_simple) { LOG_CHECK(sb.as_cslice() == "") << sb.as_cslice(); } CHECK(dispatcher.queue.size() == 1); - { ActorExecutor executor(*actor, dispatcher, ActorExecutor::Options().with_from_queue()); } + { + ActorExecutor executor(*actor, dispatcher, ActorExecutor::Options().with_from_queue()); + } CHECK(dispatcher.queue.size() == 1); dispatcher.queue.clear(); LOG_CHECK(sb.as_cslice() == "bigB") << sb.as_cslice(); @@ -1142,7 +1147,8 @@ TEST(Actor2, test_stats) { td::actor::create_actor("queue_worker").release(); } void alarm() override { - td::actor::send_closure(stats_, &ActorStats::prepare_stats, td::promise_send_closure(actor_id(this), &Master::on_stats)); + td::actor::send_closure(stats_, &ActorStats::prepare_stats, + td::promise_send_closure(actor_id(this), &Master::on_stats)); alarm_timestamp() = td::Timestamp::in(5); } void on_stats(td::Result r_stats) { @@ -1155,7 +1161,7 @@ TEST(Actor2, test_stats) { private: std::shared_ptr watcher_; td::actor::ActorOwn stats_; - int cnt_={2}; + int cnt_ = {2}; }; td::actor::create_actor("Master", watcher).release(); }); diff --git a/tdactor/test/test-coro.cpp b/tdactor/test/test-coro.cpp new file mode 100644 index 0000000000..27508d44cf --- /dev/null +++ b/tdactor/test/test-coro.cpp @@ -0,0 +1,877 @@ +#include "absl/strings/str_format.h" +#include "td/actor/coro.h" +#include "td/actor/actor.h" +#include "td/utils/Random.h" +#include "td/utils/tests.h" +#include "td/utils/port/sleep.h" + +#include +#include +#include +#include +#include +#include +#include + +using namespace td::actor; + +template +inline void expect_ok(const td::Result& r, const char* msg) { + LOG_CHECK(r.is_ok()) << msg; +} + +template +inline void expect_eq(const T& a, const U& b, const char* msg) { + LOG_CHECK(a == b) << msg; +} + +inline void expect_true(bool cond, const char* msg) { + LOG_CHECK(cond) << msg; +} + +inline void small_sleep_ms(int ms) { + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); +} + +// Minimal custom awaitables used to validate await_transform branches +struct HandleReturningAwaitable { + std::coroutine_handle<> stored_handle; + int value; + bool ready; + + explicit HandleReturningAwaitable(int v = 42, bool r = false) : value(v), ready(r) { + } + + bool await_ready() const noexcept { + return ready; + } + std::coroutine_handle<> await_suspend(std::coroutine_handle<> h) noexcept { + stored_handle = h; + return h; // symmetric transfer form + } + int await_resume() noexcept { + return value; + } +}; + +struct BoolReturningAwaitable { + std::coroutine_handle<> stored_handle; + int value; + bool ready; + bool should_suspend; + + explicit BoolReturningAwaitable(int v = 43, bool r = false, bool suspend = true) + : value(v), ready(r), should_suspend(suspend) { + } + + bool await_ready() const noexcept { + return ready; + } + bool await_suspend(std::coroutine_handle<> h) noexcept { + stored_handle = h; + if (should_suspend) { + detail::SchedulerExecutor{}.schedule(h); + } + return should_suspend; + } + int await_resume() noexcept { + return value; + } +}; + +struct VoidReturningAwaitable { + std::coroutine_handle<> stored_handle; + int value; + bool ready; + + explicit VoidReturningAwaitable(int v = 44, bool r = false) : value(v), ready(r) { + } + + bool await_ready() const noexcept { + return ready; + } + void await_suspend(std::coroutine_handle<> h) noexcept { + stored_handle = h; + detail::SchedulerExecutor{}.schedule(h); + } + int await_resume() noexcept { + return value; + } +}; + +// Simple utility actors used by tests +class TestLogger final : public td::actor::Actor { + public: + Task log(std::string msg) { + small_sleep_ms(10); + LOG(INFO) << "[Logger] " << msg; + co_return td::Unit(); + } + + void log_promise(std::string msg, td::Promise promise) { + small_sleep_ms(10); + LOG(INFO) << "[Logger Promise] " << msg; + promise.set_value(td::Unit()); + } + + Task multiply2(int x) { + co_return x * 2; + } + + void multiply3(int x, td::Promise promise) { + promise.set_value(x * 3); + } +}; + +class TestDatabase final : public td::actor::Actor { + public: + explicit TestDatabase(td::actor::ActorId logger) : logger_(logger) { + } + + Task calcA(const std::string& key) { + if (key.empty()) + co_return td::Status::Error("empty key"); + small_sleep_ms(5); + co_return static_cast(key.size()) * 10; + } + + Task square(size_t x) { + co_return x* x; + } + + Task get(std::string key) { + int ai = co_await calcA(key); // Tasks propagate errors by default + (void)co_await ask(logger_, &TestLogger::log, std::string("DB get ") + key); + small_sleep_ms(5); + if (key == "user") + co_return std::string("Alice:") + std::to_string(ai); + co_return td::Status::Error("not found"); + } + + private: + td::actor::ActorId logger_; +}; + +// 4) Tests grouped by topic +class CoroSpec final : public td::actor::Actor { + public: + void start_up() override { + logger_ = td::actor::create_actor("TestLogger").release(); + db_ = td::actor::create_actor("TestDatabase", logger_).release(); + [](Task test) -> Task { + (co_await std::move(test).wrap()).ensure(); + co_await yield_on_current(); + td::actor::SchedulerContext::get()->stop(); + co_return td::Unit{}; + }(run_all()) + .start_immediate() + .detach(); + } + + Task unified_queries() { + LOG(INFO) << "=== unified queries ==="; + + using Value = std::unique_ptr; + static auto make_value = []() { return std::make_unique(7); }; + class Uni : public td::actor::Actor { + public: + Value get_value() { + return make_value(); + } + td::Result get_result() { + return make_value(); + } + td::Result get_result_err() { + return td::Status::Error("error"); + } + Task get_task() { + co_return get_value(); + } + Task get_task_err() { + co_return td::Status::Error("error"); + } + void get_via_promise(td::Promise promise) { + promise.set_value(get_value()); + } + void get_via_promise_err(td::Promise promise) { + promise.set_error(td::Status::Error("error")); + } + void get_void() { + } + }; + auto uni = td::actor::create_actor("UnifiedResult"); + + auto check = [](td::Result v) { + CHECK(v.ok()); + CHECK(*v.ok() == 7); + }; + auto check_value = [](Value v) { CHECK(*v == 7); }; + auto check_ok = [](td::Result v) { v.ensure(); }; + auto check_err = [](td::Result v) { v.ensure_error(); }; + + auto meta_ask = [&](auto&&... args) -> Task { + LOG(INFO) << "meta_ask: ask(args...)"; + check(co_await ask(args...).wrap()); + LOG(INFO) << "meta_ask: ask_immediate(args...)"; + check(co_await ask_immediate(args...).wrap()); + LOG(INFO) << "meta_ask: co_try(ask_immediate(args...))"; + check_value(co_await ask_immediate(args...)); + LOG(INFO) << "meta_ask: co_try(ask(args...))"; + check_value(co_await ask(args...)); + //check(co_await ask_new(args...)); + co_return td::Unit{}; + }; + + auto meta_ask_err = [&](auto&&... args) -> Task { + LOG(INFO) << "meta_ask_err: ask(args...)"; + check_err(co_await ask(args...).wrap()); + LOG(INFO) << "meta_ask_err: ask_immediate(args...)"; + check_err(co_await ask_immediate(args...).wrap()); + + check_err(co_await [](auto&&... args) -> Task { + LOG(INFO) << "meta_ask_err: co_try(ask_immediate(args...))"; + co_return co_await ask_immediate(args...); + }(args...) + .wrap()); + check_err(co_await [](auto&&... args) -> Task { + LOG(INFO) << "meta_ask_err: co_try(ask_immediate(args...))"; + co_return co_await ask_immediate(args...); + }(args...) + .wrap()); + check_err(co_await [](auto&&... args) -> Task { + LOG(INFO) << "meta_ask_err: co_try(ask_immediate(args...))"; + co_await ask(args...); + co_return std::make_unique(17); + }(args...) + .wrap()); + + LOG(INFO) << "meta_ask_err: co_try(ask(args...))"; + //check(co_await ask_new(args...)); + co_return td::Unit{}; + }; + + // ask from coroutines + check_ok(co_await meta_ask(uni, &Uni::get_result)); + check_ok(co_await meta_ask(uni, &Uni::get_task)); + check_ok(co_await meta_ask(uni, &Uni::get_via_promise)); + check_ok(co_await meta_ask(uni, &Uni::get_value)); + co_await ask(uni, &Uni::get_void); + co_await ask_immediate(uni, &Uni::get_void); + + check_ok(co_await meta_ask_err(uni, &Uni::get_result_err)); + check_ok(co_await meta_ask_err(uni, &Uni::get_task_err)); + check_ok(co_await meta_ask_err(uni, &Uni::get_via_promise_err)); + + check(co_await ask_new(uni, &Uni::get_task)); + + static_assert(td::is_promise_interface::ExternalPromise>()); + + auto check_send_closure = [&](auto&& f) -> Task { + auto [task, task_promise] = StartedTask::make_bridge(); + td::Promise promise = [moved_task_promise = std::move(task_promise)](td::Result r) mutable { + moved_task_promise.set_result(std::move(r)); + }; + f(std::move(promise)); + send_closure(uni, &Uni::get_value, std::move(promise)); + check(co_await std::move(task)); + co_return td::Unit{}; + }; + + co_await check_send_closure([&](auto promise) { send_closure(uni, &Uni::get_result, std::move(promise)); }); + co_await check_send_closure([&](auto promise) { send_closure(uni, &Uni::get_value, std::move(promise)); }); + co_await check_send_closure([&](auto promise) { send_closure(uni, &Uni::get_via_promise, std::move(promise)); }); + co_await check_send_closure([&](auto promise) { send_closure(uni, &Uni::get_task, std::move(promise)); }); + + bool done = false; + LOG(INFO) << "Test send_closure_immediate"; + send_closure_immediate(uni, &Uni::get_void, [&](td::Result r) { + (void)r; + done = true; + }); + CHECK(done); + + co_return td::Unit(); + } + + // A. Awaitable branch coverage (handle/bool/void; ready/suspend) + Task awaitable_branches() { + LOG(INFO) << "=== Awaitable branches ==="; + + struct Case { + const char* name; + int expected; + std::function()> run; + }; + std::vector cases = { + {"handle:not_ready", 100, [&]() -> Task { co_return co_await HandleReturningAwaitable(100, false); }}, + {"handle:ready", 101, [&]() -> Task { co_return co_await HandleReturningAwaitable(101, true); }}, + {"bool:suspend", 200, [&]() -> Task { co_return co_await BoolReturningAwaitable(200, false, true); }}, + {"bool:immediate", 201, [&]() -> Task { co_return co_await BoolReturningAwaitable(201, false, false); }}, + {"bool:ready", 202, [&]() -> Task { co_return co_await BoolReturningAwaitable(202, true, true); }}, + {"void:not_ready", 300, [&]() -> Task { co_return co_await VoidReturningAwaitable(300, false); }}, + {"void:ready", 301, [&]() -> Task { co_return co_await VoidReturningAwaitable(301, true); }}, + }; + for (auto& c : cases) { + auto r = co_await c.run(); + expect_eq(r, c.expected, c.name); + } + + int sum = 0; + sum += co_await HandleReturningAwaitable(10, false); + sum += co_await BoolReturningAwaitable(20, false, true); + sum += co_await BoolReturningAwaitable(30, false, false); + sum += co_await VoidReturningAwaitable(40, false); + expect_eq(sum, 100, "mixed awaitables sum"); + + co_return td::Unit(); + } + + // B. Recursion via ask hop and direct recursion + Task rec_fast(int n) { + if (n == 0) + co_return 0; + int r = co_await rec_fast(n - 1); // Tasks propagate errors by default now + co_return r + 1; + } + Task rec_slow(int n) { + if (n == 0) + co_return 0; + int r = + co_await ask(actor_id(this), &CoroSpec::rec_slow, n - 1); // ask returns StartedTask which propagates errors + co_return r + 1; + } + Task recursion() { + LOG(INFO) << "=== Recursion ==="; + for (int depth : {5, 10}) { + int a = co_await rec_slow(depth); // Tasks propagate errors by default + expect_eq(a, depth, "recursion via ask"); + int b = co_await rec_fast(depth); // Tasks propagate errors by default + expect_eq(b, depth, "direct recursion"); + } + co_return td::Unit(); + } + + // C. ask()/ask_immediate and unified Task/Promise targets + Task asks() { + LOG(INFO) << "=== ask / ask_immediate ==="; + + auto delayed = ask(db_, &TestDatabase::square, 4); + expect_true(!delayed.await_ready(), "delayed ask is not ready"); + expect_eq(co_await std::move(delayed), static_cast(16), "delayed ask result"); + + co_await Yield{}; + for (int i = 0; i < 16; i++) { + auto immediate = ask_immediate(db_, &TestDatabase::square, 4); + expect_true(immediate.await_ready(), "immediate ask is ready"); + expect_eq(immediate.await_resume().ok(), static_cast(16), "immediate ask result"); + } + + auto user = co_await ask(db_, &TestDatabase::get, std::string("user")); + LOG(INFO) << "User: " << user; + + (void)co_await ask(logger_, &TestLogger::log, std::string("unified Task target")); + (void)co_await ask(logger_, &TestLogger::log_promise, std::string("unified Promise target")); + co_return td::Unit(); + } + + // C2. Modifiers: Yield, ChangeOwner attach/detach, yield_on + Task modifiers() { + LOG(INFO) << "=== Modifiers (Yield, ChangeOwner, yield_on) ==="; + + auto self = actor_id(this); + co_await attach_to_actor(self); // just in case + auto on_self = [self] { + if (self != td::actor::detail::get_current_actor_id()) { + return td::Status::Error("not on self"); + } + return td::Status::OK(); + }; + auto on_none = [] { + if (!td::actor::detail::get_current_actor_id().empty()) { + return td::Status::Error("not on none"); + } + return td::Status::OK(); + }; + + // Yield sequencing + { + td::Timer timer; + for (int i = 0; i < 1000; i++) { + co_await yield_on_current(); + on_self(); + } + LOG(INFO) << "yield_on_current (x100): " << timer.elapsed(); + timer = {}; + for (int i = 0; i < 1000; i++) { + co_await attach_to_actor(self); // noop + on_self(); + } + LOG(INFO) << "attach_to_actor (x100) : " << timer.elapsed(); + } + + // Attach to current actor and ensure suspended await resumes on same actor + { + co_await attach_to_actor(self); + int v = co_await BoolReturningAwaitable(123, false, true); + on_self(); + expect_eq(v, 123, "suspended await result"); + } + + // Detach (no specific owner), ensure we still resume and continue + { + co_await detach_from_actor(); + on_none(); + int v = co_await BoolReturningAwaitable(321, false, true); + expect_eq(v, 321, "detached suspended await result"); + co_await attach_to_actor(self); + on_self(); + } + + // Explicit yield_on to current actor + { + co_await yield_on_current(); + on_self(); + } + + co_return td::Unit(); + } + + // D. Concurrency and double-resumption surface + Task concurrency() { + LOG(INFO) << "=== Concurrency ==="; + auto self = actor_id(this); + co_await detach_from_actor(); + + for (int round = 0; round < 100; round++) { + co_await attach_to_actor(self); + co_await detach_from_actor(); + auto task = [](int value) -> Task { + td::usleep_for(td::Random::fast(0, 1000)); + co_return value * 2; + }(round) + .start(); + td::usleep_for(td::Random::fast(0, 1000)); + auto result = co_await std::move(task); + CHECK(result == round * 2); + } + + for (int round = 0; round < 100; round++) { + co_await attach_to_actor(self); + co_await detach_from_actor(); + auto task = [](int value) -> Task { + td::usleep_for(td::Random::fast(0, 1000)); + co_return value * 2; + }(round) + .start(); + td::usleep_for(td::Random::fast(0, 1000)); + task.detach(); + td::usleep_for(100); + } + + // Many parallel tasks + sum + std::vector> many; + size_t expect = 0; + for (size_t i = 0; i < 200; i++) { + auto t = [](size_t v) -> Task { co_return v; }(i).start(); + many.push_back(std::move(t)); + expect += i; + } + size_t got = 0; + for (auto& t : many) { + auto v = co_await std::move(t); // Tasks propagate errors by default + got += v; + } + expect_eq(got, expect, "many parallel sum"); + co_return td::Unit{}; + } + Task concurrency2() { + LOG(INFO) << "=== Concurrency 2 ==="; + // A few shapes with spawn_coroutine_old + std::vector> shapes; + for (int i = 0; i < 8000; i++) { + int m = i % 4; + if (m == 0) { + //shapes.push_back(spawn_actor("immediate", []() -> Task { co_return 1; })); + } else if (m == 1) { + shapes.push_back(spawn_actor("hop1", []() -> Task { + co_await spawn_actor("sub", []() -> Task { co_return td::Unit(); }()); + co_return 2; + }())); + } else if (m == 2) { + // intentionally left out heavy nested spawns variant + } else { + //shapes.push_back([]() -> Task { co_return 2; }().start_immediate()); + } + } + int s = 0; + for (auto& t : shapes) { + auto v = co_await std::move(t); // Tasks propagate errors by default + s += v; + } + LOG(INFO) << "shapes sum: " << s; + co_return td::Unit(); + } + + // F. Task lifecycle sanity (lazy start + await; explicit start) + Task lifecycle() { + LOG(INFO) << "=== Task lifecycle ==="; + auto make_task = []() -> Task { co_return 7; }; + + // Await without explicit start + { + auto v = co_await make_task(); // Tasks propagate errors by default + expect_eq(v, 7, "await without start"); + } + // Explicit start + { + auto t = make_task().start(); + auto v = co_await std::move(t); // Tasks propagate errors by default + expect_eq(v, 7, "await after start"); + } + co_return td::Unit(); + } + Task helpers() { + LOG(INFO) << "=== Task helper ==="; + CHECK(5 == co_await td::actor::detail::make_awaitable(5)); + auto get7 = []() -> Task { co_return 7; }; + CHECK(7 == co_await get7()); + auto square = [](size_t x) -> Task { co_return x* x; }; + auto res = co_await get7().start().then(square); + CHECK(res == 49); + co_return td::Unit(); + co_return td::Unit(); + } + + Task combinators() { + LOG(ERROR) << "Test combinators"; + + // Test all() with variadic arguments + { + auto make_task = [](int val, int delay_ms) -> Task { + small_sleep_ms(delay_ms); + co_return val; + }; + + auto tuple = co_await all(make_task(1, 10), make_task(2, 20), make_task(3, 30)); + auto a = std::move(std::get<0>(tuple)); + auto b = std::move(std::get<1>(tuple)); + auto c = std::move(std::get<2>(tuple)); + expect_eq(1, a, "all() first result"); + expect_eq(2, b, "all() second result"); + expect_eq(3, c, "all() third result"); + LOG(ERROR) << "all() variadic test passed"; + } + + // Test all() with vector + { + std::vector> tasks; + for (int i = 0; i < 5; ++i) { + tasks.push_back([](int val) -> Task { + small_sleep_ms(val * 10); + co_return val * 2; + }(i)); + } + + auto results = co_await all(std::move(tasks)); + expect_eq(5u, results.size(), "all() vector size"); + for (size_t i = 0; i < results.size(); ++i) { + expect_eq(static_cast(i * 2), results[i], "all() vector result"); + } + LOG(ERROR) << "all() vector test passed"; + } + + // Test all() with errors and collect_results + { + auto success_task = []() -> Task { co_return 42; }; + auto error_task = []() -> Task { co_return td::Status::Error("Test error"); }; + + auto tuple = co_await all(success_task().wrap(), error_task().wrap()); + + // Test that individual results can have errors + auto s = std::move(std::get<0>(tuple)); + auto e = std::move(std::get<1>(tuple)); + expect_eq(42, s.ok(), "all() with error - success task"); + expect_true(e.is_error(), "all() with error - error task"); + + // Test collect_results with tuple containing an error + auto tuple2 = co_await all(success_task().wrap(), error_task().wrap()); + auto collected = collect(std::move(tuple2)); + expect_true(collected.is_error(), "collect_results should return error if any task failed"); + LOG(ERROR) << "all() error handling test passed"; + } + + // Test collect_results with all successful tasks + { + auto task1 = []() -> Task { co_return 1; }; + auto task2 = []() -> Task { co_return 2; }; + auto task3 = []() -> Task { co_return 3; }; + + // Test with tuple + auto tuple = co_await all(task1().wrap(), task2().wrap(), task3().wrap()); + auto collected_tuple = collect(std::move(tuple)); + expect_ok(collected_tuple, "collect_results should succeed when all tasks succeed"); + auto [a, b, c] = collected_tuple.move_as_ok(); + expect_eq(1, a, "First value"); + expect_eq(2, b, "Second value"); + expect_eq(3, c, "Third value"); + + // Test with vector + std::vector> tasks; + for (int i = 0; i < 5; ++i) { + tasks.push_back([](int val) -> Task { co_return val; }(i)); + } + auto vec = co_await all_wrap(std::move(tasks)); + auto collected_vec = collect(std::move(vec)); + expect_ok(collected_vec, "collect_results should succeed for vector"); + auto& values = collected_vec.ok(); + expect_eq(5u, values.size(), "Vector size"); + for (size_t i = 0; i < values.size(); ++i) { + expect_eq(static_cast(i), values[i], "Vector element"); + } + LOG(ERROR) << "collect_results test passed"; + } + + co_return td::Unit{}; + } + + // G. co_try success and error propagation + Task try_awaitable() { + LOG(INFO) << "=== co_try ==="; + + // Success path: unwrap value + { + auto ok_task = []() -> Task { co_return 123; }; + int v = co_await ok_task(); + expect_eq(v, 123, "co_try unwraps ok value"); + } + + // Error path: propagate out of outer Task, so awaiting yields error + { + auto err_task = []() -> Task { co_return td::Status::Error("boom"); }; + auto r = co_await err_task().wrap(); // control: direct await to observe error pattern + expect_true(r.is_error(), "sanity: err_task returns error"); + + // Now check propagation via co_try inside an outer task + auto outer = [err_task]() -> Task { + int x = co_await err_task(); + co_return x + 1; // should never reach + }(); + auto rr = co_await std::move(outer).wrap(); + expect_true(rr.is_error(), "co_try propagates error to outer Task"); + } + + // Test try_unwrap() method on StartedTask + { + auto ok_task = []() -> Task { co_return 456; }; + auto started = ok_task().start_immediate(); + int v = co_await std::move(started); + expect_eq(v, 456, "try_unwrap() unwraps ok value from StartedTask"); + } + + // Test try_unwrap() error propagation + { + auto err_task = []() -> Task { co_return td::Status::Error("test error"); }; + auto outer = [err_task]() -> Task { + auto started = err_task().start_immediate(); + int x = co_await std::move(started); + co_return x + 1; // should never reach + }(); + auto result = co_await std::move(outer).wrap(); + expect_true(result.is_error(), "try_unwrap() propagates error from StartedTask"); + } + + // Test co_try() with Result values (non-awaitable) + { + auto outer = []() -> Task { + td::Result ok_result = 789; + int x = co_await std::move(ok_result); + co_return x + 1; + }(); + auto result = co_await std::move(outer).wrap(); // Use wrap() to get Result + expect_true(result.is_ok(), "co_try(Result) works with ok value"); + expect_eq(result.move_as_ok(), 790, "co_try(Result) returns correct value"); + } + + // Test co_try() error propagation with Result + { + auto outer = []() -> Task { + td::Result err_result = td::Status::Error("direct error"); + int x = co_await std::move(err_result); + co_return x + 1; // should never reach + }(); + auto result = co_await std::move(outer).wrap(); // Use wrap() to get Result + expect_true(result.is_error(), "co_try(Result) propagates error"); + } + + // Test co_try() with Result lvalue reference + { + auto outer = []() -> Task { + td::Result ok_result = 999; + // Test with lvalue reference (should work with the Storage type handling) + int x = co_await std::move(ok_result); + // Result should be moved out after co_try + co_return x + 2; + }(); + auto result = co_await std::move(outer).wrap(); // Use wrap() to get Result + expect_true(result.is_ok(), "co_try(Result&) works with lvalue reference"); + expect_eq(result.move_as_ok(), 1001, "co_try(Result&) returns correct value"); + } + + // Test default Result co_await (propagates errors) + { + auto outer = []() -> Task { + td::Result ok_result = 333; + int x = co_await std::move(ok_result); // Default: propagates error + co_return x * 2; + }(); + auto result = co_await std::move(outer).wrap(); // Use wrap() to get Result + expect_true(result.is_ok(), "Result default co_await works with ok value"); + expect_eq(result.move_as_ok(), 666, "Result default co_await returns correct value"); + } + + // Test default Result co_await error propagation + { + auto outer = []() -> Task { + td::Result err_result = td::Status::Error("unwrap error"); + int x = co_await std::move(err_result); // Default: propagates error + co_return x * 2; // should never reach + }(); + auto result = co_await std::move(outer).wrap(); // Use wrap() to get Result + expect_true(result.is_error(), "Result default co_await propagates error"); + } + + // Test Result::wrap() to prevent error propagation + { + auto outer = []() -> Task> { + td::Result err_result = td::Status::Error("wrapped error"); + auto full_result = co_await std::move(err_result).wrap(); // Explicit: no propagation + expect_true(full_result.is_error(), "wrap() preserves error in Result"); + co_return full_result; + }(); + auto result = co_await std::move(outer); + expect_true(result.is_error(), "wrap() preserved the error"); + } + + // Test Result::wrap() with ok value + { + auto outer = []() -> Task> { + td::Result ok_result = 555; + auto full_result = co_await std::move(ok_result).wrap(); // Explicit: no propagation + expect_true(full_result.is_ok(), "wrap() preserves ok value in Result"); + co_return full_result; + }(); + auto result = co_await std::move(outer).wrap(); // Need to wrap outer task too + expect_true(result.is_ok(), "Task completes successfully"); + auto inner_result = result.move_as_ok(); + expect_true(inner_result.is_ok(), "wrap() preserved the ok value"); + expect_eq(inner_result.move_as_ok(), 555, "wrap() preserved the correct value"); + } + + // Test Task default co_await (propagates errors) + { + auto inner = []() -> Task { co_return 888; }; + auto outer = [&inner]() -> Task { + int x = co_await inner(); // Default: propagates errors, returns T + co_return x + 1; + }(); + auto result = co_await std::move(outer).wrap(); // Wrap to get Result + expect_true(result.is_ok(), "Task default co_await works"); + expect_eq(result.move_as_ok(), 889, "Task default co_await returns correct value"); + } + + // Test Task default co_await error propagation + { + auto inner = []() -> Task { co_return td::Status::Error("task error"); }; + auto outer = [&inner]() -> Task { + int x = co_await inner(); // Default: propagates error + co_return x + 1; // should never reach + }(); + auto result = co_await std::move(outer).wrap(); // Wrap to get Result + expect_true(result.is_error(), "Task default co_await propagates error"); + } + + // Test Task::wrap() to prevent error propagation + { + auto inner = []() -> Task { co_return td::Status::Error("wrapped task error"); }; + auto outer = [&inner]() -> Task> { + auto full_result = co_await inner().wrap(); // Explicit: no propagation + expect_true(full_result.is_error(), "Task::wrap() preserves error"); + co_return full_result; + }(); + auto result = co_await std::move(outer).wrap(); // Wrap outer too + expect_true(result.is_ok(), "Outer task completes successfully"); + auto inner_result = result.move_as_ok(); + expect_true(inner_result.is_error(), "Task::wrap() preserved the error"); + } + + co_return td::Unit{}; + } + + static Task slow_task() { + td::usleep_for(2000000); + co_return td::Unit{}; + } + Task stop_actor() { + class StopActor : public td::actor::Actor { + public: + void start_up() override { + alarm_timestamp() = td::Timestamp::in(1); + } + void alarm() override { + LOG(INFO) << "alarm"; + stop(); + } + Task query() { + auto task = slow_task(); + task.set_executor(Executor::on_scheduler()); + // actor will stop before task is finished + co_await std::move(task); + // here we could access actor but we should v + LOG(FATAL) << "access stopped actor"; + co_return 1; + } + }; + auto a = create_actor("stop_actor"); + auto r = co_await ask(a, &StopActor::query).wrap(); + r.ensure_error(); + LOG(INFO) << "Got error from stopped actor " << r.error(); + co_return td::Unit{}; + } + + // Master runner + Task run_all() { + LOG(ERROR) << "Run tests"; + (void)co_await ask(logger_, &TestLogger::log, std::string("Starting coroutine tests")); + + co_await unified_queries(); + co_await concurrency(); + for (int i = 0; i < 10; i++) { + co_await concurrency2(); + } + co_await awaitable_branches(); + co_await recursion(); + co_await asks(); + co_await modifiers(); + co_await lifecycle(); + co_await helpers(); + co_await combinators(); + co_await try_awaitable(); + co_await stop_actor(); + + (void)co_await ask(logger_, &TestLogger::log, std::string("All tests passed")); + co_return td::Unit(); + } + + private: + td::actor::ActorId db_; + td::actor::ActorId logger_; +}; + +// 5) Runner +int main() { + SET_VERBOSITY_LEVEL(VERBOSITY_NAME(INFO)); + td::actor::Scheduler scheduler({4}); + scheduler.run_in_context([&] { td::actor::create_actor("CoroSpec").release(); }); + scheduler.run(); + return 0; +} diff --git a/tdutils/td/utils/Closure.h b/tdutils/td/utils/Closure.h index 3b2fc9de54..3407c75428 100644 --- a/tdutils/td/utils/Closure.h +++ b/tdutils/td/utils/Closure.h @@ -95,7 +95,7 @@ class ImmediateClosure { template ImmediateClosure create_immediate_closure( - ResultT (ActorT::*func)(DestArgsT...), SrcArgsT &&... args) { + ResultT (ActorT::*func)(DestArgsT...), SrcArgsT &&...args) { return ImmediateClosure(func, std::forward(args)...); } @@ -172,7 +172,7 @@ DelayedClosure to_delayed_closure(DelayedClosure &&other) { } template -auto create_delayed_closure(ResultT (ActorT::*func)(DestArgsT...), SrcArgsT &&... args) { +auto create_delayed_closure(ResultT (ActorT::*func)(DestArgsT...), SrcArgsT &&...args) { return DelayedClosure(func, std::forward(args)...); } diff --git a/tdutils/td/utils/Status.h b/tdutils/td/utils/Status.h index f75de466a9..27e09af59f 100644 --- a/tdutils/td/utils/Status.h +++ b/tdutils/td/utils/Status.h @@ -443,6 +443,12 @@ class Status { } }; +// Forward declarations for Result wrappers +template +struct ResultUnwrap; +template +struct ResultWrap; + template class Result { public: @@ -604,6 +610,16 @@ class Result { return f(move_as_ok()); } + // Returns a wrapper that can be co_awaited to propagate errors in coroutines + ResultUnwrap try_unwrap() && { + return ResultUnwrap(std::move(*this)); + } + + // Returns a wrapper that prevents error propagation when co_awaited + ResultWrap wrap() && { + return ResultWrap(std::move(*this)); + } + private: Status status_; union { @@ -611,6 +627,19 @@ class Result { }; }; + +// Wrapper to prevent error propagation when co_awaiting Result +template +struct ResultWrap { + Result result; +}; + +template +struct ResultUnwrap { + Result result; + +}; + template <> inline Result::Result(Status &&status) : status_(std::move(status)) { // no assert From d268ac0cb441c3d6e70aae9b6b7655c7e04bb0ff Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 10 Sep 2025 10:14:57 +0300 Subject: [PATCH 383/524] Set block id in ConfigInfo::extract_config (#1780) This fixes calculating prev_blocks_info in tonlib and similar places --- crypto/block/check-proof.cpp | 2 +- crypto/block/mc-config.cpp | 16 +++++----------- crypto/block/mc-config.h | 5 ++--- tonlib/tonlib/LastConfig.cpp | 5 +++-- tonlib/tonlib/TonlibClient.cpp | 2 +- validator/collator-node/collator-node.cpp | 3 ++- validator/impl/collator.cpp | 3 +-- validator/impl/liteserver.cpp | 19 ++++++++++--------- validator/impl/liteserver.hpp | 2 +- validator/impl/shard.cpp | 7 +++---- validator/impl/validate-query.cpp | 13 ++++++------- 11 files changed, 35 insertions(+), 42 deletions(-) diff --git a/crypto/block/check-proof.cpp b/crypto/block/check-proof.cpp index 431a03fec8..c3c9085da7 100644 --- a/crypto/block/check-proof.cpp +++ b/crypto/block/check-proof.cpp @@ -492,7 +492,7 @@ td::Status BlockProofLink::validate(td::uint32* save_utime) const { return td::Status::Error("BlockProofLink contains a state proof for "s + from.to_str() + " with incorrect root hash"); } - TRY_RESULT(config, block::ConfigInfo::extract_config(vstate_root, block::ConfigInfo::needPrevBlocks)); + TRY_RESULT(config, block::ConfigInfo::extract_config(vstate_root, from, block::ConfigInfo::needPrevBlocks)); if (!config->check_old_mc_block_id(to, true)) { return td::Status::Error("cannot check that "s + to.to_str() + " is indeed a previous masterchain block of " + from.to_str() + " using the presented Merkle proof of masterchain state"); diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 35c3df0055..ce11bcaec4 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -92,22 +92,16 @@ td::Result> Config::extract_from_state(Ref mc_ return unpack_config(std::move(extra.config), mode); } -td::Result> ConfigInfo::extract_config(std::shared_ptr static_boc, - int mode) { - TRY_RESULT(rc, static_boc->get_root_count()); - if (rc != 1) { - return td::Status::Error(-668, "Masterchain state BoC is invalid"); - } - TRY_RESULT(root, static_boc->get_root_cell(0)); - return extract_config(std::move(root), mode); -} - -td::Result> ConfigInfo::extract_config(Ref mc_state_root, int mode) { +td::Result> ConfigInfo::extract_config(Ref mc_state_root, + ton::BlockIdExt mc_block_id, int mode) { if (mc_state_root.is_null()) { return td::Status::Error("configuration state root cell is null"); } auto config = std::unique_ptr{new ConfigInfo(std::move(mc_state_root), mode)}; TRY_STATUS(config->unpack_wrapped()); + if (!config->set_block_id_ext(mc_block_id)) { + return td::Status::Error("failed to set mc block id"); + } return std::move(config); } diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index c5a6d7722f..03dda0b709 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -775,9 +775,8 @@ class ConfigInfo : public Config, public ShardConfig { std::vector compute_validator_set_cc(ton::ShardIdFull shard, ton::UnixTime time, ton::CatchainSeqno* cc_seqno_delta = nullptr) const; td::Result> get_prev_blocks_info() const; - static td::Result> extract_config(std::shared_ptr static_boc, - int mode = 0); - static td::Result> extract_config(Ref mc_state_root, int mode = 0); + static td::Result> extract_config(Ref mc_state_root, + ton::BlockIdExt mc_block_id, int mode = 0); private: ConfigInfo(Ref mc_state_root, int _mode = 0); diff --git a/tonlib/tonlib/LastConfig.cpp b/tonlib/tonlib/LastConfig.cpp index e972d84e04..45d6c9ebb3 100644 --- a/tonlib/tonlib/LastConfig.cpp +++ b/tonlib/tonlib/LastConfig.cpp @@ -93,8 +93,9 @@ td::Status LastConfig::process_config_proof(ton::ton_api::object_ptrstate_proof_.as_slice(), raw_config->config_proof_.as_slice())); - TRY_RESULT(config, block::ConfigInfo::extract_config( - std::move(state), block::ConfigInfo::needPrevBlocks | block::ConfigInfo::needCapabilities)); + TRY_RESULT(config, + block::ConfigInfo::extract_config( + std::move(state), blkid, block::ConfigInfo::needPrevBlocks | block::ConfigInfo::needCapabilities)); for (auto i : params_) { VLOG(last_config) << "ConfigParam(" << i << ") = "; diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 7e868480b0..3132bf766f 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -2168,7 +2168,7 @@ class RunEmulator : public TonlibQueryActor { } try { - auto r_config = block::ConfigInfo::extract_config(mc_state_root_, 0b11'11111111); + auto r_config = block::ConfigInfo::extract_config(mc_state_root_, block_id_.mc, 0b11'11111111); if (r_config.is_error()) { check(r_config.move_as_error()); return; diff --git a/validator/collator-node/collator-node.cpp b/validator/collator-node/collator-node.cpp index dd0dfb7927..9b03d9dc0b 100644 --- a/validator/collator-node/collator-node.cpp +++ b/validator/collator-node/collator-node.cpp @@ -465,7 +465,8 @@ td::Status CollatorNode::check_mc_config() { } TRY_RESULT_PREFIX( config, - block::ConfigInfo::extract_config(last_masterchain_state_->root_cell(), block::ConfigInfo::needCapabilities), + block::ConfigInfo::extract_config(last_masterchain_state_->root_cell(), last_masterchain_state_->get_block_id(), + block::ConfigInfo::needCapabilities), "cannot unpack masterchain config"); if (config->get_global_version() > Collator::supported_version()) { return td::Status::Error(PSTRING() << "unsupported global version " << config->get_global_version() diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 608c8f67aa..ead5ec0386 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -856,7 +856,7 @@ void Collator::after_get_shard_state_optimistic(td::Result> res, */ bool Collator::unpack_last_mc_state() { auto res = block::ConfigInfo::extract_config( - mc_state_root, + mc_state_root, mc_block_id_, block::ConfigInfo::needShardHashes | block::ConfigInfo::needLibraries | block::ConfigInfo::needValidatorSet | block::ConfigInfo::needWorkchainInfo | block::ConfigInfo::needCapabilities | block::ConfigInfo::needPrevBlocks | @@ -868,7 +868,6 @@ bool Collator::unpack_last_mc_state() { } config_ = res.move_as_ok(); CHECK(config_); - config_->set_block_id_ext(mc_block_id_); global_id_ = config_->get_global_blockchain_id(); ihr_enabled_ = config_->ihr_enabled(); create_stats_enabled_ = config_->create_stats_enabled(); diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index e9592f6264..2fabe8cb01 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -917,7 +917,8 @@ void LiteQuery::continue_getLibraries(Ref mc_s mc_state_ = Ref(std::move(mc_state)); CHECK(mc_state_.not_null()); - auto rconfig = block::ConfigInfo::extract_config(mc_state_->root_cell(), block::ConfigInfo::needLibraries); + auto rconfig = block::ConfigInfo::extract_config(mc_state_->root_cell(), mc_state_->get_block_id(), + block::ConfigInfo::needLibraries); if (rconfig.is_error()) { fatal_error("cannot extract library list block configuration from masterchain state"); return; @@ -1251,9 +1252,10 @@ bool LiteQuery::make_shard_info_proof(Ref& proof, BlockIdExt& blkid, A return true; } -bool LiteQuery::make_ancestor_block_proof(Ref& proof, Ref state_root, const BlockIdExt& old_blkid) { - vm::MerkleProofBuilder mpb{std::move(state_root)}; - auto rconfig = block::ConfigInfo::extract_config(mpb.root(), block::ConfigInfo::needPrevBlocks); +bool LiteQuery::make_ancestor_block_proof(Ref& proof, Ref mc_state, const BlockIdExt& old_blkid) { + vm::MerkleProofBuilder mpb{mc_state->root_cell()}; + auto rconfig = + block::ConfigInfo::extract_config(mpb.root(), mc_state->get_block_id(), block::ConfigInfo::needPrevBlocks); if (rconfig.is_error()) { return fatal_error( "cannot extract previous block configuration from masterchain state while constructing Merkle proof for "s + @@ -1319,13 +1321,12 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { vm::AugmentedDictionary accounts_dict{vm::load_cell_slice_ref(sstate.accounts), 256, block::tlb::aug_ShardAccounts}; auto acc_csr = accounts_dict.lookup(acc_addr_); if (mode_ & 0x80000000) { - auto config = block::ConfigInfo::extract_config(mc_state_->root_cell(), 0xFFFF); + auto config = block::ConfigInfo::extract_config(mc_state_->root_cell(), mc_state_->get_block_id(), 0xFFFF); if (config.is_error()) { fatal_error(config.move_as_error()); return; } auto rconfig = config.move_as_ok(); - rconfig->set_block_id_ext(mc_state_->get_block_id()); acc_state_promise_.set_value(std::make_tuple( std::move(acc_csr), sstate.gen_utime, sstate.gen_lt, std::move(rconfig) )); @@ -1502,7 +1503,7 @@ void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice LOG(DEBUG) << "creating VM with gas limit " << gas_limit; // **** INIT VM **** auto r_config = block::ConfigInfo::extract_config( - mc_state_->root_cell(), + mc_state_->root_cell(), mc_state_->get_block_id(), block::ConfigInfo::needLibraries | block::ConfigInfo::needCapabilities | block::ConfigInfo::needPrevBlocks); if (r_config.is_error()) { fatal_error(r_config.move_as_error()); @@ -1905,7 +1906,7 @@ void LiteQuery::continue_getConfigParams(int mode, std::vector param_list) if (mode & block::ConfigInfo::needPrevBlocks) { mode |= block::ConfigInfo::needCapabilities; } - auto res = block::ConfigInfo::extract_config(mpb.root(), mode); + auto res = block::ConfigInfo::extract_config(mpb.root(), keyblk ? base_blk_id_ : mc_state_->get_block_id(), mode); if (res.is_error()) { fatal_error(res.move_as_error()); return; @@ -3045,7 +3046,7 @@ bool LiteQuery::construct_proof_link_back_cont(ton::BlockIdExt cur, ton::BlockId return fatal_error("cannot construct proof for state of masterchain block "s + cur.to_str()); } // construct proof that `next` is listed in OldMcBlocksInfo of `mc_state_` - if (!make_ancestor_block_proof(state_proof, mc_state_->root_cell(), next)) { + if (!make_ancestor_block_proof(state_proof, mc_state_, next)) { return fatal_error("cannot prove that "s + next.to_str() + " is in the previous block set of the masterchain state of " + cur.to_str()); } diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index fc87353322..afeabb9c01 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -221,7 +221,7 @@ class LiteQuery : public td::actor::Actor { bool make_shard_info_proof(Ref& proof, Ref& info, ShardIdFull shard, bool exact = true); bool make_shard_info_proof(Ref& proof, Ref& info, AccountIdPrefixFull prefix); bool make_shard_info_proof(Ref& proof, BlockIdExt& blkid, AccountIdPrefixFull prefix); - bool make_ancestor_block_proof(Ref& proof, Ref state_root, const BlockIdExt& old_blkid); + bool make_ancestor_block_proof(Ref& proof, Ref mc_state, const BlockIdExt& old_blkid); }; } // namespace validator diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 69bbfdd624..5803eac89f 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -376,9 +376,9 @@ td::Status MasterchainStateQ::mc_init() { td::Status MasterchainStateQ::mc_reinit() { auto res = block::ConfigInfo::extract_config( - root_cell(), block::ConfigInfo::needStateRoot | block::ConfigInfo::needValidatorSet | - block::ConfigInfo::needShardHashes | block::ConfigInfo::needPrevBlocks | - block::ConfigInfo::needWorkchainInfo); + root_cell(), blkid, + block::ConfigInfo::needStateRoot | block::ConfigInfo::needValidatorSet | block::ConfigInfo::needShardHashes | + block::ConfigInfo::needPrevBlocks | block::ConfigInfo::needWorkchainInfo); cur_validators_.reset(); next_validators_.reset(); if (res.is_error()) { @@ -386,7 +386,6 @@ td::Status MasterchainStateQ::mc_reinit() { } config_ = res.move_as_ok(); CHECK(config_); - CHECK(config_->set_block_id_ext(get_block_id())); cur_validators_ = config_->get_cur_validator_set(); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 4285a385cd..da5d7bf736 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1009,7 +1009,7 @@ bool ValidateQuery::try_unpack_mc_state() { return fatal_error(-666, "latest masterchain state does not have a root cell"); } auto res = block::ConfigInfo::extract_config( - mc_state_root_, + mc_state_root_, mc_blkid_, block::ConfigInfo::needShardHashes | block::ConfigInfo::needLibraries | block::ConfigInfo::needValidatorSet | block::ConfigInfo::needWorkchainInfo | block::ConfigInfo::needStateExtraRoot | block::ConfigInfo::needCapabilities | block::ConfigInfo::needPrevBlocks | @@ -1020,7 +1020,6 @@ bool ValidateQuery::try_unpack_mc_state() { } config_ = res.move_as_ok(); CHECK(config_); - config_->set_block_id_ext(mc_blkid_); ihr_enabled_ = config_->ihr_enabled(); create_stats_enabled_ = config_->create_stats_enabled(); if (config_->has_capabilities() && (config_->get_capabilities() & ~supported_capabilities())) { @@ -1514,17 +1513,17 @@ bool ValidateQuery::compute_next_state() { } } auto r_config_info = block::ConfigInfo::extract_config( - state_root_, block::ConfigInfo::needShardHashes | block::ConfigInfo::needLibraries | - block::ConfigInfo::needValidatorSet | block::ConfigInfo::needWorkchainInfo | - block::ConfigInfo::needStateExtraRoot | block::ConfigInfo::needAccountsRoot | - block::ConfigInfo::needSpecialSmc | block::ConfigInfo::needCapabilities); + state_root_, id_, + block::ConfigInfo::needShardHashes | block::ConfigInfo::needLibraries | block::ConfigInfo::needValidatorSet | + block::ConfigInfo::needWorkchainInfo | block::ConfigInfo::needStateExtraRoot | + block::ConfigInfo::needAccountsRoot | block::ConfigInfo::needSpecialSmc | + block::ConfigInfo::needCapabilities); if (r_config_info.is_error()) { return reject_query("cannot extract configuration from new masterchain state "s + mc_blkid_.to_str() + " : " + r_config_info.error().to_string()); } new_config_ = r_config_info.move_as_ok(); CHECK(new_config_); - new_config_->set_block_id_ext(id_); } return true; } From 635bbd0d082d021483fab6f0bae89e6f81e1346d Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Tue, 29 Jul 2025 18:05:01 +0300 Subject: [PATCH 384/524] [Tolk] Refactor: IR generation of complex built-ins like T.toCell Instead of working by a string name, store a pointer to an ops-generator function inside a symbol --- tolk-tester/tests/pack-unpack-5.tolk | 51 +++++++------ tolk/analyzer.cpp | 2 +- tolk/builtins.cpp | 104 +++++++++++++++++---------- tolk/codegen.cpp | 4 +- tolk/pack-unpack-api.cpp | 96 +++++++++++++++---------- tolk/pack-unpack-api.h | 11 +-- tolk/pipe-ast-to-legacy.cpp | 81 +-------------------- tolk/send-message-api.cpp | 30 +++++--- tolk/send-message-api.h | 31 -------- tolk/symtable.h | 9 +-- tolk/tolk.h | 19 +++-- 11 files changed, 200 insertions(+), 238 deletions(-) delete mode 100644 tolk/send-message-api.h diff --git a/tolk-tester/tests/pack-unpack-5.tolk b/tolk-tester/tests/pack-unpack-5.tolk index 97b740b481..e643af6c31 100644 --- a/tolk-tester/tests/pack-unpack-5.tolk +++ b/tolk-tester/tests/pack-unpack-5.tolk @@ -10,29 +10,34 @@ struct JustMaybeInt32 { value: MaybeInt32; } +fun T.estimate(): [int, int, int, int] { + // convert a tensor to a tuple for better output readability + val (minBits, maxBits, minRefs, maxRefs) = T.estimatePackSize(); + return [minBits, maxBits, minRefs, maxRefs]; +} @method_id(101) fun test1() { return ( - int32.estimatePackSize(), - uint64.estimatePackSize(), - int1.estimatePackSize(), - bool.estimatePackSize(), - RemainingBitsAndRefs.estimatePackSize(), - coins.estimatePackSize(), - bits6.estimatePackSize(), - bytes8.estimatePackSize(), - varint32.estimatePackSize(), + int32.estimate(), + uint64.estimate(), + int1.estimate(), + bool.estimate(), + RemainingBitsAndRefs.estimate(), + coins.estimate(), + bits6.estimate(), + bytes8.estimate(), + varint32.estimate(), ); } @method_id(102) fun test2() { return ( - JustInt32.estimatePackSize(), - JustMaybeInt32.estimatePackSize(), - MaybeInt32.estimatePackSize(), - Int16Or32.estimatePackSize(), + JustInt32.estimate(), + JustMaybeInt32.estimate(), + MaybeInt32.estimate(), + Int16Or32.estimate(), ); } @@ -51,7 +56,7 @@ struct Test3_2 { @method_id(103) fun test3() { - return (Test3_1.estimatePackSize(), Test3_2.estimatePackSize()); + return (Test3_1.estimate(), Test3_2.estimate()); } struct Test4_1 { @@ -71,7 +76,7 @@ struct Test4_3 { @method_id(104) fun test4() { - return (Test4_1.estimatePackSize(), Test4_2.estimatePackSize(), Test4_3.estimatePackSize()); + return (Test4_1.estimate(), Test4_2.estimate(), Test4_3.estimate()); } struct Test5_1 { @@ -99,7 +104,7 @@ struct Test5_3 { @method_id(105) fun test5() { - return (Test5_1.estimatePackSize(), Test5_2.estimatePackSize(), Test5_3.estimatePackSize()); + return (Test5_1.estimate(), Test5_2.estimate(), Test5_3.estimate()); } struct(0x00112233) Test6_1 { @@ -115,7 +120,7 @@ type Test6_or = Test6_1 | Test6_2; @method_id(106) fun test6() { - return (Test6_1.estimatePackSize(), Test6_2.estimatePackSize(), Test6_or.estimatePackSize()); + return (Test6_1.estimate(), Test6_2.estimate(), Test6_or.estimate()); } struct(0x1020) Test7_1; @@ -127,7 +132,7 @@ type Test7_or = | Test7_1 | Test7_2 | Test7_3 @method_id(107) fun test7() { assert((Test7_1{} as Test7_or).toCell().beginParse().remainingBitsCount() == Test7_or.estimatePackSize().0, 400); - return (Test7_1.estimatePackSize(), Test7_or.estimatePackSize()); + return (Test7_1.estimate(), Test7_or.estimate()); } struct(0x10) Inner8_1 { @@ -151,7 +156,7 @@ struct Test8 { @method_id(108) fun test8() { - return (Inner8_1.estimatePackSize(), Inner8_2.estimatePackSize(), Test8.estimatePackSize()); + return (Inner8_1.estimate(), Inner8_2.estimate(), Test8.estimate()); } struct Test9_bits2 { f: bits2; } @@ -164,7 +169,7 @@ type Test9_f4 = bits1 | Test9_bits2 | bits3 | Test9_bits4?; // auto-generate @method_id(109) fun test9() { - return (Test9_f1.estimatePackSize(), Test9_f2.estimatePackSize(), Test9_f3.estimatePackSize(), Test9_f4.estimatePackSize()); + return (Test9_f1.estimate(), Test9_f2.estimate(), Test9_f3.estimate(), Test9_f4.estimate()); } struct Test10_1 { @@ -176,7 +181,7 @@ type Test10_2 = (Test10_1, bool?, RemainingBitsAndRefs); @method_id(110) fun test10() { - return (Test10_1.estimatePackSize(), Test10_2.estimatePackSize()); + return (Test10_1.estimate(), Test10_2.estimate()); } struct Test11_1 { @@ -191,7 +196,7 @@ struct Test11_2 { @method_id(111) fun test11() { - return (Test11_1.estimatePackSize(), Test11_2.estimatePackSize()); + return (Test11_1.estimate(), Test11_2.estimate()); } @method_id(120) @@ -201,7 +206,7 @@ fun test20() { } fun main() { - __expect_type(int8.estimatePackSize(), "[int, int, int, int]"); + __expect_type(int8.estimatePackSize(), "(int, int, int, int)"); } /** diff --git a/tolk/analyzer.cpp b/tolk/analyzer.cpp index 8e9fe914e7..ea6a9db22e 100644 --- a/tolk/analyzer.cpp +++ b/tolk/analyzer.cpp @@ -718,7 +718,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { if (arg_order_already_equals_asm()) { maybe_swap_builtin_args_to_compile(); } - std::get(f_sym->body)->compile(tmp, res, args, loc); + std::get(f_sym->body)->compile(tmp, res, args, loc); if (arg_order_already_equals_asm()) { maybe_swap_builtin_args_to_compile(); } diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index dc2795c898..f3aa3c5ef7 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -40,22 +40,34 @@ static std::vector define_builtin_parameters(const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags) { - auto* f_sym = new FunctionData(name, {}, "", nullptr, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); +static void define_builtin_func(const std::string& name, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const std::function& func, int flags) { + auto* f_sym = new FunctionData(name, {}, "", nullptr, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltinAsmOp(func), nullptr); G.symtable.add_function(f_sym); } -static void define_builtin_method(const std::string& name, TypePtr receiver_type, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags, +static void define_builtin_func(const std::string& name, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const std::function& func, int flags) { + auto* f_sym = new FunctionData(name, {}, "", nullptr, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltinGenerateOps(func), nullptr); + G.symtable.add_function(f_sym); +} + +static void define_builtin_method(const std::string& name, TypePtr receiver_type, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const std::function& func, int flags, std::initializer_list arg_order = {}, std::initializer_list ret_order = {}) { std::string method_name = name.substr(name.find('.') + 1); - auto* f_sym = new FunctionData(name, {}, std::move(method_name), receiver_type, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); + auto* f_sym = new FunctionData(name, {}, std::move(method_name), receiver_type, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltinAsmOp(func), nullptr); f_sym->arg_order = arg_order; f_sym->ret_order = ret_order; G.symtable.add_function(f_sym); G.all_methods.push_back(f_sym); } -void FunctionBodyBuiltin::compile(AsmOpList& dest, std::vector& out, std::vector& in, +void define_builtin_method(const std::string& name, TypePtr receiver_type, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const std::function& func, int flags) { + std::string method_name = name.substr(name.find('.') + 1); + auto* f_sym = new FunctionData(name, {}, std::move(method_name), receiver_type, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltinGenerateOps(func), nullptr); + G.symtable.add_function(f_sym); + G.all_methods.push_back(f_sym); +} + +void FunctionBodyBuiltinAsmOp::compile(AsmOpList& dest, std::vector& out, std::vector& in, SrcLocation loc) const { dest << simple_compile(out, in, loc); } @@ -1166,7 +1178,7 @@ static AsmOp compile_fetch_slice(std::vector& res, std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_slice_sdbeginsq(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 3 && res.size() == 2); auto& prefix = args[1]; auto& prefix_len = args[2]; @@ -1181,7 +1193,7 @@ AsmOp compile_slice_sdbeginsq(std::vector& res, std::vector& } // fun slice.skipBits(mutate self, len: int): self "SDSKIPFIRST" -AsmOp compile_skip_bits_in_slice(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_skip_bits_in_slice(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 2 && res.size() == 1); auto& len = args[1]; // same technique as for storeUint: @@ -1268,7 +1280,7 @@ static AsmOp compile_any_object_sizeof(std::vector& res, std::vector&, std::vector&, SrcLocation loc) { +static AsmOp compile_time_only_function(std::vector&, std::vector&, SrcLocation loc) { // all ton() invocations are constants, replaced by integers; no dynamic values allowed, no work at runtime tolk_assert(false); return AsmOp::Nop(loc); @@ -1292,6 +1304,24 @@ static AsmOp compile_expect_type(std::vector&, std::vector&, return AsmOp::Nop(loc); } +// implemented in dedicated files + +using GenerateOpsImpl = FunctionBodyBuiltinGenerateOps::GenerateOpsImpl; + +GenerateOpsImpl generate_T_toCell; +GenerateOpsImpl generate_builder_storeAny; +GenerateOpsImpl generate_T_fromSlice; +GenerateOpsImpl generate_slice_loadAny; +GenerateOpsImpl generate_T_fromCell; +GenerateOpsImpl generate_T_forceLoadLazyObject; +GenerateOpsImpl generate_slice_skipAny; +GenerateOpsImpl generate_T_estimatePackSize; + +GenerateOpsImpl generate_createMessage; +GenerateOpsImpl generate_createExternalLogMessage; +GenerateOpsImpl generate_address_buildInAnotherShard; +GenerateOpsImpl generate_AutoDeployAddress_buildAddress; +GenerateOpsImpl generate_AutoDeployAddress_addressMatches; void define_builtins() { using namespace std::placeholders; @@ -1546,8 +1576,8 @@ void define_builtins() { compile_tuple_set_at, FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf); define_builtin_method("address.buildSameAddressInAnotherShard", Address, {Address, AddressShardingOptions}, Builder, nullptr, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagCompileTimeGen); + generate_address_buildInAnotherShard, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); define_builtin_method("debug.print", debug, {typeT}, Unit, declGenericT, compile_debug_print_to_string, FunctionData::flagAllowAnyWidthT); @@ -1564,17 +1594,17 @@ void define_builtins() { // serialization/deserialization methods to/from cells (or, more low-level, slices/builders) // they work with structs (or, more low-level, with arbitrary types) define_builtin_method("T.toCell", typeT, {typeT, PackOptions}, CellT, declReceiverT, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + generate_T_toCell, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); define_builtin_method("T.fromCell", typeT, {TypeDataCell::create(), UnpackOptions}, typeT, declReceiverT, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + generate_T_fromCell, + FunctionData::flagMarkedAsPure | FunctionData::flagAllowAnyWidthT); define_builtin_method("T.fromSlice", typeT, {Slice, UnpackOptions}, typeT, declReceiverT, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); - define_builtin_method("T.estimatePackSize", typeT, {}, TypeDataBrackets::create({TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create()}), declReceiverT, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + generate_T_fromSlice, + FunctionData::flagMarkedAsPure | FunctionData::flagAllowAnyWidthT); + define_builtin_method("T.estimatePackSize", typeT, {}, TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create()}), declReceiverT, + generate_T_estimatePackSize, + FunctionData::flagMarkedAsPure | FunctionData::flagAllowAnyWidthT); define_builtin_method("T.getDeclaredPackPrefix", typeT, {}, Int, declReceiverT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal | FunctionData::flagAllowAnyWidthT); @@ -1582,33 +1612,33 @@ void define_builtins() { compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal | FunctionData::flagAllowAnyWidthT); define_builtin_method("T.forceLoadLazyObject", typeT, {typeT}, Slice, declReceiverT, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + generate_T_forceLoadLazyObject, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); define_builtin_method("Cell.load", CellT, {CellT, UnpackOptions}, typeT, declReceiverT, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + generate_T_fromCell, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); define_builtin_method("slice.loadAny", Slice, {Slice, UnpackOptions}, typeT, declGenericT, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + generate_slice_loadAny, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); define_builtin_method("slice.skipAny", Slice, {Slice, UnpackOptions}, Slice, declGenericT, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + generate_slice_skipAny, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); define_builtin_method("builder.storeAny", Builder, {Builder, typeT, PackOptions}, Builder, declGenericT, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + generate_builder_storeAny, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); define_builtin_func("createMessage", {CreateMessageOptions}, OutMessage, declTBody, - compile_time_only_function, - FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + generate_createMessage, + FunctionData::flagAllowAnyWidthT); define_builtin_func("createExternalLogMessage", {CreateExternalLogMessageOptions}, OutMessage, declTBody, - compile_time_only_function, - FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + generate_createExternalLogMessage, + FunctionData::flagAllowAnyWidthT); define_builtin_method("AutoDeployAddress.buildAddress", AutoDeployAddress, {AutoDeployAddress}, Builder, nullptr, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagCompileTimeGen); + generate_AutoDeployAddress_buildAddress, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); define_builtin_method("AutoDeployAddress.addressMatches", AutoDeployAddress, {AutoDeployAddress, Address}, Bool, nullptr, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagCompileTimeGen); + generate_AutoDeployAddress_addressMatches, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); // functions not presented in stdlib at all // used in tolk-tester to check/expose internal compiler state diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp index f5ef89de3b..8be3688105 100644 --- a/tolk/codegen.cpp +++ b/tolk/codegen.cpp @@ -373,7 +373,7 @@ bool Op::generate_code_step(Stack& stack) { if (f_sym->is_asm_function()) { std::get(f_sym->body)->compile(stack.o, loc); // compile res := f (args0) } else { - std::get(f_sym->body)->compile(stack.o, res, args0, loc); // compile res := f (args0) + std::get(f_sym->body)->compile(stack.o, res, args0, loc); // compile res := f (args0) } } else { stack.o << AsmOp::Custom(loc, f_sym->name + "() CALLDICT", (int)right.size(), (int)left.size()); @@ -511,7 +511,7 @@ bool Op::generate_code_step(Stack& stack) { if (arg_order_already_equals_asm()) { maybe_swap_builtin_args_to_compile(); } - std::get(f_sym->body)->compile(stack.o, res, args, loc); // compile res := f (args) + std::get(f_sym->body)->compile(stack.o, res, args, loc); // compile res := f (args) if (arg_order_already_equals_asm()) { maybe_swap_builtin_args_to_compile(); } diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index abab637e97..213476630a 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -237,15 +237,16 @@ static int calc_offset_on_stack(StructPtr struct_ref, int field_idx) { // -std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_obj, const std::vector& ir_options) { +// fun T.toCell(self, options: PackOptions): Cell +std::vector generate_T_toCell(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr typeT = called_f->substitutedTs->typeT_at(0); FunctionPtr f_beginCell = lookup_function("beginCell"); FunctionPtr f_endCell = lookup_function("builder.endCell"); std::vector rvect_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); code.emplace_back(loc, Op::_Call, rvect_builder, std::vector{}, f_beginCell); - tolk_assert(ir_options.size() == 1); // struct PackOptions - PackContext ctx(code, loc, rvect_builder, ir_options); - ctx.generate_pack_any(any_type, std::move(ir_obj)); + PackContext ctx(code, loc, rvect_builder, args[1]); + ctx.generate_pack_any(typeT, std::vector(args[0])); std::vector rvect_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(cell)"); code.emplace_back(loc, Op::_Call, rvect_cell, std::move(rvect_builder), f_endCell); @@ -253,54 +254,70 @@ std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation return rvect_cell; } -std::vector generate_pack_struct_to_builder(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_builder, std::vector&& ir_obj, const std::vector& ir_options) { - PackContext ctx(code, loc, ir_builder, ir_options); // mutate this builder - ctx.generate_pack_any(any_type, std::move(ir_obj)); +// fun builder.storeAny(mutate self, v: T, options: PackOptions = {}): self +std::vector generate_builder_storeAny(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr typeT = called_f->substitutedTs->typeT_at(0); + PackContext ctx(code, loc, args[0], args[2]); // mutate this builder + ctx.generate_pack_any(typeT, std::vector(args[1])); - return ir_builder; // return mutated builder + return args[0]; // return mutated builder } -std::vector generate_unpack_struct_from_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, bool mutate_slice, const std::vector& ir_options) { - if (!mutate_slice) { - std::vector slice_copy = code.create_var(TypeDataSlice::create(), loc, "s"); - code.emplace_back(loc, Op::_Let, slice_copy, std::move(ir_slice)); - ir_slice = std::move(slice_copy); - } +// fun T.fromSlice(rawSlice: slice, options: UnpackOptions): T +std::vector generate_T_fromSlice(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + std::vector slice_copy = code.create_var(TypeDataSlice::create(), loc, "s"); + code.emplace_back(loc, Op::_Let, slice_copy, args[0]); - tolk_assert(ir_options.size() == 2); // struct UnpackOptions - UnpackContext ctx(code, loc, std::move(ir_slice), ir_options); - std::vector rvect_struct = ctx.generate_unpack_any(any_type); - tolk_assert(any_type->get_width_on_stack() == static_cast(rvect_struct.size())); + TypePtr typeT = called_f->substitutedTs->typeT_at(0); + UnpackContext ctx(code, loc, std::move(slice_copy), args[1]); + std::vector rvect_struct = ctx.generate_unpack_any(typeT); + tolk_assert(typeT->get_width_on_stack() == static_cast(rvect_struct.size())); - // slice.loadAny() ignores options.assertEndAfterReading, because it's intended to read data in the middle - if (!mutate_slice && !estimate_serialization_size(any_type).is_unpredictable_infinity()) { + if (!estimate_serialization_size(typeT).is_unpredictable_infinity()) { ctx.assertEndIfOption(); } return rvect_struct; } -std::vector generate_unpack_struct_from_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_cell, const std::vector& ir_options) { +// fun slice.loadAny(mutate self, options: UnpackOptions): T +std::vector generate_slice_loadAny(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr typeT = called_f->substitutedTs->typeT_at(0); + UnpackContext ctx(code, loc, args[0], args[1]); + std::vector rvect_struct = ctx.generate_unpack_any(typeT); + tolk_assert(typeT->get_width_on_stack() == static_cast(rvect_struct.size())); + + // slice.loadAny() ignores options.assertEndAfterReading, because it's intended to read data in the middle + std::vector ir_slice_and_result = args[0]; + ir_slice_and_result.insert(ir_slice_and_result.end(), rvect_struct.begin(), rvect_struct.end()); + return ir_slice_and_result; +} + +// fun T.fromCell(packedCell: cell, options: UnpackOptions): T +// fun Cell.load(self, options: UnpackOptions): T +std::vector generate_T_fromCell(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr typeT = called_f->substitutedTs->typeT_at(0); FunctionPtr f_beginParse = lookup_function("cell.beginParse"); std::vector ir_slice = code.create_var(TypeDataSlice::create(), loc, "s"); - code.emplace_back(loc, Op::_Call, ir_slice, std::move(ir_cell), f_beginParse); + code.emplace_back(loc, Op::_Call, ir_slice, args[0], f_beginParse); - tolk_assert(ir_options.size() == 2); // struct UnpackOptions - UnpackContext ctx(code, loc, std::move(ir_slice), ir_options); - std::vector rvect_struct = ctx.generate_unpack_any(any_type); - tolk_assert(any_type->get_width_on_stack() == static_cast(rvect_struct.size())); + UnpackContext ctx(code, loc, std::move(ir_slice), args[1]); + std::vector rvect_struct = ctx.generate_unpack_any(typeT); + tolk_assert(typeT->get_width_on_stack() == static_cast(rvect_struct.size())); // if a struct has RemainingBitsAndRefs, don't test it for assertEnd - if (!estimate_serialization_size(any_type).is_unpredictable_infinity()) { + if (!estimate_serialization_size(typeT).is_unpredictable_infinity()) { ctx.assertEndIfOption(); } return rvect_struct; } -std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, const std::vector& ir_options) { - UnpackContext ctx(code, loc, ir_slice, ir_options); // mutate this slice - ctx.generate_skip_any(any_type); +// fun slice.skipAny(mutate self, options: UnpackOptions): self +std::vector generate_slice_skipAny(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr typeT = called_f->substitutedTs->typeT_at(0); + UnpackContext ctx(code, loc, args[0], args[1]); // mutate this slice + ctx.generate_skip_any(typeT); - return ir_slice; // return mutated slice + return args[0]; // return mutated slice } void generate_lazy_struct_from_slice(CodeBlob& code, SrcLocation loc, const LazyVariableLoadedState* lazy_variable, const LazyStructLoadInfo& load_info, const std::vector& ir_obj) { @@ -423,14 +440,19 @@ std::vector generate_lazy_object_finish_loading(CodeBlob& code, SrcLo return lazy_variable->ir_slice; } +std::vector generate_T_forceLoadLazyObject(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + // a call to `T.forceLoadLazyObject()` was handled separately + tolk_assert(false); +} + PackSize estimate_serialization_size(TypePtr any_type) { EstimateContext ctx; return ctx.estimate_any(any_type); } -std::vector generate_estimate_size_call(CodeBlob& code, SrcLocation loc, TypePtr any_type) { - EstimateContext ctx; - PackSize pack_size = ctx.estimate_any(any_type); +std::vector generate_T_estimatePackSize(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr typeT = called_f->substitutedTs->typeT_at(0); + PackSize pack_size = estimate_serialization_size(typeT); std::vector ir_tensor = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create()}), loc, "(result-tensor)"); code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[0]}, td::make_refint(pack_size.min_bits)); @@ -438,11 +460,7 @@ std::vector generate_estimate_size_call(CodeBlob& code, SrcLocation l code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[2]}, td::make_refint(pack_size.min_refs)); code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[3]}, td::make_refint(pack_size.max_refs)); - FunctionPtr f_toTuple = lookup_function("T.__toTuple"); - std::vector ir_tuple = code.create_tmp_var(TypeDataTuple::create(), loc, "(result-tuple)"); - code.emplace_back(loc, Op::_Call, ir_tuple, ir_tensor, f_toTuple); - - return ir_tuple; + return ir_tensor; } } // namespace tolk diff --git a/tolk/pack-unpack-api.h b/tolk/pack-unpack-api.h index 7b4d02f66a..45da429e56 100644 --- a/tolk/pack-unpack-api.h +++ b/tolk/pack-unpack-api.h @@ -22,15 +22,10 @@ namespace tolk { bool check_struct_can_be_packed_or_unpacked(TypePtr any_type, bool is_pack, std::string& because_msg); - -std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_obj, const std::vector& ir_options); -std::vector generate_pack_struct_to_builder(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_builder, std::vector&& ir_obj, const std::vector& ir_options); -std::vector generate_unpack_struct_from_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, bool mutate_slice, const std::vector& ir_options); -std::vector generate_unpack_struct_from_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_cell, const std::vector& ir_options); -std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, const std::vector& ir_options); - PackSize estimate_serialization_size(TypePtr any_type); -std::vector generate_estimate_size_call(CodeBlob& code, SrcLocation loc, TypePtr any_type); + +// functions like T.toCell() are not declared here: they are implemented in a .cpp file, +// and their prototypes exist and are referenced in `builtins.cpp` struct LazyStructLoadInfo; struct LazyStructLoadedState; diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index f22b0ab445..1f4ae232c3 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -24,9 +24,6 @@ #include "smart-casts-cfg.h" #include "pack-unpack-api.h" #include "gen-entrypoints.h" -#include "generics-helpers.h" -#include "send-message-api.h" -#include "gen-entrypoints.h" /* * This pipe is the last one operating AST: it transforms AST to IR. @@ -620,8 +617,6 @@ static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob if (called_f->is_method() && called_f->is_instantiation_of_generic_function()) { std::string_view f_name = called_f->base_fun_ref->name; - TypePtr typeT = called_f->substitutedTs->typeT_at(0); - const LazyVariableLoadedState* lazy_variable = v_call->dot_obj_is_self ? code.get_lazy_variable(v_call->get_self_obj()) : nullptr; if (f_name == "T.toCell" && lazy_variable && lazy_variable->is_struct()) { @@ -629,49 +624,6 @@ static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob std::vector ir_obj = vars_per_arg[0]; // = lazy_var_ref->ir_idx return generate_lazy_struct_to_cell(code, loc, &lazy_variable->loaded_state, std::move(ir_obj), vars_per_arg[1]); } - if (f_name == "T.toCell") { - // in: object T, out: Cell (just a cell, wrapped) - std::vector ir_obj = vars_per_arg[0]; - return generate_pack_struct_to_cell(code, loc, typeT, std::move(ir_obj), vars_per_arg[1]); - } - if (f_name == "T.fromCell") { - // in: cell, out: object T - std::vector ir_cell = vars_per_arg[0]; - return generate_unpack_struct_from_cell(code, loc, typeT, std::move(ir_cell), vars_per_arg[1]); - } - if (f_name == "T.fromSlice") { - // in: slice, out: object T, input slice NOT mutated - std::vector ir_slice = vars_per_arg[0]; - return generate_unpack_struct_from_slice(code, loc, typeT, std::move(ir_slice), false, vars_per_arg[1]); - } - if (f_name == "Cell.load") { - // in: cell, out: object T - std::vector ir_cell = vars_per_arg[0]; - return generate_unpack_struct_from_cell(code, loc, typeT, std::move(ir_cell), vars_per_arg[1]); - } - if (f_name == "slice.loadAny") { - // in: slice, out: object T, input slice is mutated, so prepend self before an object - var_idx_t ir_self = vars_per_arg[0][0]; - std::vector ir_slice = vars_per_arg[0]; - std::vector ir_obj = generate_unpack_struct_from_slice(code, loc, typeT, std::move(ir_slice), true, vars_per_arg[1]); - std::vector ir_result = {ir_self}; - ir_result.insert(ir_result.end(), ir_obj.begin(), ir_obj.end()); - return ir_result; - } - if (f_name == "slice.skipAny") { - // in: slice, out: the same slice, with a shifted pointer - std::vector ir_slice = vars_per_arg[0]; - return generate_skip_struct_in_slice(code, loc, typeT, std::move(ir_slice), vars_per_arg[1]); - } - if (f_name == "builder.storeAny") { - // in: builder and object T, out: mutated builder - std::vector ir_builder = vars_per_arg[0]; - std::vector ir_obj = vars_per_arg[1]; - return generate_pack_struct_to_builder(code, loc, typeT, std::move(ir_builder), std::move(ir_obj), vars_per_arg[2]); - } - if (f_name == "T.estimatePackSize") { - return generate_estimate_size_call(code, loc, typeT); - } if (f_name == "T.forceLoadLazyObject") { // in: object T, out: slice (same slice that a lazy variable holds, after loading/skipping all its fields) if (!lazy_variable) { @@ -682,36 +634,9 @@ static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob } } - if (called_f->is_instantiation_of_generic_function()) { - std::string_view f_name = called_f->base_fun_ref->name; - TypePtr typeT = called_f->substitutedTs->typeT_at(0); - - if (f_name == "createMessage") { - std::vector ir_msg_params = vars_per_arg[0]; - return generate_createMessage(code, loc, typeT->unwrap_alias(), std::move(ir_msg_params)); - } - if (f_name == "createExternalLogMessage") { - std::vector ir_msg_params = vars_per_arg[0]; - return generate_createExternalLogMessage(code, loc, typeT->unwrap_alias(), std::move(ir_msg_params)); - } - } - - if (called_f->name == "address.buildSameAddressInAnotherShard") { - std::vector ir_self_address = vars_per_arg[0]; - std::vector ir_shard_options = vars_per_arg[1]; - return generate_address_buildInAnotherShard(code, loc, std::move(ir_self_address), std::move(ir_shard_options)); - } - if (called_f->name == "AutoDeployAddress.buildAddress") { - std::vector ir_self = vars_per_arg[0]; - return generate_AutoDeployAddress_buildAddress(code, loc, std::move(ir_self)); - } - if (called_f->name == "AutoDeployAddress.addressMatches") { - std::vector ir_self = vars_per_arg[0]; - std::vector ir_address = vars_per_arg[1]; - return generate_AutoDeployAddress_addressMatches(code, loc, std::move(ir_self), std::move(ir_address)); - } - - tolk_assert(false); + auto gen = std::get_if(&called_f->body); + tolk_assert(gen); + return (*gen)->generate_ops(called_f, code, loc, vars_per_arg); } std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, SrcLocation loc, FunctionPtr f_inlined, AnyExprV self_obj, bool is_before_immediate_return, const std::vector>& vars_per_arg) { diff --git a/tolk/send-message-api.cpp b/tolk/send-message-api.cpp index 2b4d80be8e..e2323b17f0 100644 --- a/tolk/send-message-api.cpp +++ b/tolk/send-message-api.cpp @@ -14,8 +14,8 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "send-message-api.h" #include "pack-unpack-serializers.h" +#include "generics-helpers.h" #include "type-system.h" namespace tolk { @@ -86,7 +86,9 @@ struct IR_AutoDeployAddress { } }; -std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect) { +// fun createMessage(options: CreateMessageOptions): OutMessage +std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr bodyT = called_f->substitutedTs->typeT_at(0); StructPtr s_Options = lookup_global_symbol("CreateMessageOptions")->try_as(); StructPtr s_AutoDeployAddress = lookup_global_symbol("AutoDeployAddress")->try_as(); @@ -98,6 +100,7 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T tolk_assert(t_value && t_value->get_width_on_stack() == (2+1) && t_value->size() == 2); int offset = 0; + std::vector rvect = args[0]; auto next_slice = [&rvect, &offset](int width) -> std::vector { int start = offset; offset += width; @@ -356,7 +359,9 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T return ir_cell; } -std::vector generate_createExternalLogMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect) { +// fun createExternalLogMessage(options: CreateExternalLogMessageOptions): OutMessage +std::vector generate_createExternalLogMessage(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr bodyT = called_f->substitutedTs->typeT_at(0); StructPtr s_Options = lookup_global_symbol("CreateExternalLogMessageOptions")->try_as(); StructPtr s_ExtOutLogBucket = lookup_global_symbol("ExtOutLogBucket")->try_as(); @@ -366,6 +371,7 @@ std::vector generate_createExternalLogMessage(CodeBlob& code, SrcLoca tolk_assert(t_topic && t_topic->get_width_on_stack() == (1+1) && t_topic->size() == 2); int offset = 0; + std::vector rvect = args[0]; auto next_slice = [&rvect, &offset](int width) -> std::vector { int start = offset; offset += width; @@ -490,7 +496,9 @@ std::vector generate_createExternalLogMessage(CodeBlob& code, SrcLoca return ir_cell; } -std::vector generate_address_buildInAnotherShard(CodeBlob& code, SrcLocation loc, std::vector&& ir_self_address, std::vector&& ir_shard_options) { +// fun address.buildSameAddressInAnotherShard(self, options: AddressShardingOptions): builder +std::vector generate_address_buildInAnotherShard(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + std::vector ir_shard_options = args[1]; tolk_assert(ir_shard_options.size() == 2); // example for fixedPrefixLength (shard depth) = 8: @@ -515,14 +523,15 @@ std::vector generate_address_buildInAnotherShard(CodeBlob& code, SrcL std::vector ir_restLenA = {code.create_int(loc, 256, "(last-addrA)")}; code.emplace_back(loc, Op::_Call, ir_restLenA, std::vector{ir_restLenA[0], ir_shard_options[0]}, lookup_function("_-_")); std::vector ir_tailA = code.create_tmp_var(TypeDataSlice::create(), loc, "(tailA)"); - code.emplace_back(loc, Op::_Call, ir_tailA, std::vector{ir_self_address[0], ir_restLenA[0]}, lookup_function("slice.getLastBits")); + code.emplace_back(loc, Op::_Call, ir_tailA, std::vector{args[0][0], ir_restLenA[0]}, lookup_function("slice.getLastBits")); code.emplace_back(loc, Op::_Call, ir_builder, std::vector{ir_builder[0], ir_tailA[0]}, lookup_function("builder.storeSlice")); return ir_builder; } -std::vector generate_AutoDeployAddress_buildAddress(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy) { - IR_AutoDeployAddress ir_self(code, loc, ir_auto_deploy); +// fun AutoDeployAddress.buildAddress(self): builder +std::vector generate_AutoDeployAddress_buildAddress(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + IR_AutoDeployAddress ir_self(code, loc, args[0]); std::vector ir_builder = code.create_tmp_var(TypeDataSlice::create(), loc, "(addr-b)"); // important! unlike `createMessage()`, we calculate hash and shard prefix BEFORE creating a cell @@ -599,8 +608,9 @@ std::vector generate_AutoDeployAddress_buildAddress(CodeBlob& code, S return ir_builder; } -std::vector generate_AutoDeployAddress_addressMatches(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy, std::vector&& ir_address) { - IR_AutoDeployAddress ir_self(code, loc, ir_auto_deploy); +// fun AutoDeployAddress.addressMatches(self, addr: address): bool +std::vector generate_AutoDeployAddress_addressMatches(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + IR_AutoDeployAddress ir_self(code, loc, args[0]); // at first, calculate stateInitHash = (hash of StateInit cell would be, but without constructing a cell) std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); @@ -647,7 +657,7 @@ std::vector generate_AutoDeployAddress_addressMatches(CodeBlob& code, // now do `(wc, hash) = addr.getWorkchainAndHash()` std::vector ir_addr_wc_hash = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create()}), loc, "(self-wc-hash)"); - code.emplace_back(loc, Op::_Call, ir_addr_wc_hash, ir_address, lookup_function("address.getWorkchainAndHash")); + code.emplace_back(loc, Op::_Call, ir_addr_wc_hash, args[1], lookup_function("address.getWorkchainAndHash")); // now calculate `hash &= mask` (the same as we did earlier for stateInitHash) Op& if_sharded2 = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); diff --git a/tolk/send-message-api.h b/tolk/send-message-api.h deleted file mode 100644 index f86be5446b..0000000000 --- a/tolk/send-message-api.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . -*/ -#pragma once - -#include "tolk.h" - -namespace tolk { - -std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect); -std::vector generate_createExternalLogMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect); - -std::vector generate_address_buildInAnotherShard(CodeBlob& code, SrcLocation loc, std::vector&& ir_self_address, std::vector&& ir_shard_options); - -std::vector generate_AutoDeployAddress_buildAddress(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy); -std::vector generate_AutoDeployAddress_addressMatches(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy, std::vector&& ir_address); - -} // namespace tolk diff --git a/tolk/symtable.h b/tolk/symtable.h index 268b474d3a..32292b0b74 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -96,13 +96,15 @@ struct LocalVarData final : Symbol { struct FunctionBodyCode; struct FunctionBodyAsm; -struct FunctionBodyBuiltin; +struct FunctionBodyBuiltinAsmOp; +struct FunctionBodyBuiltinGenerateOps; struct GenericsDeclaration; typedef std::variant< FunctionBodyCode*, FunctionBodyAsm*, - FunctionBodyBuiltin* + FunctionBodyBuiltinAsmOp*, + FunctionBodyBuiltinGenerateOps* > FunctionBody; struct FunctionData final : Symbol { @@ -120,7 +122,6 @@ struct FunctionData final : Symbol { flagReturnsSelf = 1024, // return type is `self` (returns the mutated 1st argument), calls can be chainable flagReallyUsed = 2048, // calculated via dfs from used functions; declared but unused functions are not codegenerated flagCompileTimeVal = 4096, // calculated only at compile-time for constant arguments: `ton("0.05")`, `stringCrc32`, and others - flagCompileTimeGen = 8192, // at compile-time it's handled specially, not as a regular function: `T.toCell`, etc. flagAllowAnyWidthT = 16384, // for built-in generic functions that is not restricted to be 1-slot type flagManualOnBounce = 32768, // for onInternalMessage, don't insert "if (isBounced) return" }; @@ -212,7 +213,7 @@ struct FunctionData final : Symbol { bool does_mutate_self() const { return (flags & flagAcceptsSelf) && parameters[0].is_mutate_parameter(); } bool is_really_used() const { return flags & flagReallyUsed; } bool is_compile_time_const_val() const { return flags & flagCompileTimeVal; } - bool is_compile_time_special_gen() const { return flags & flagCompileTimeGen; } + bool is_compile_time_special_gen() const { return std::holds_alternative(body); } bool is_variadic_width_T_allowed() const { return flags & flagAllowAnyWidthT; } bool is_manual_on_bounce() const { return flags & flagManualOnBounce; } diff --git a/tolk/tolk.h b/tolk/tolk.h index 9bef3c4dfb..75b6aa4070 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -1028,17 +1028,26 @@ struct Stack { * */ -typedef std::function&, std::vector&, SrcLocation)> simple_compile_func_t; +struct FunctionBodyBuiltinAsmOp { + using CompileToAsmOpImpl = AsmOp(std::vector&, std::vector&, SrcLocation); + + std::function simple_compile; -struct FunctionBodyBuiltin { - simple_compile_func_t simple_compile; - - explicit FunctionBodyBuiltin(simple_compile_func_t compile) + explicit FunctionBodyBuiltinAsmOp(std::function compile) : simple_compile(std::move(compile)) {} void compile(AsmOpList& dest, std::vector& out, std::vector& in, SrcLocation loc) const; }; +struct FunctionBodyBuiltinGenerateOps { + using GenerateOpsImpl = std::vector(FunctionPtr, CodeBlob&, SrcLocation, const std::vector>&); + + std::function generate_ops; + + explicit FunctionBodyBuiltinGenerateOps(std::function generate_ops) + : generate_ops(std::move(generate_ops)) {} +}; + struct FunctionBodyAsm { std::vector ops; From 5756e736559d40382fd85faf890b8a7aa6c64a2a Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Tue, 29 Jul 2025 18:34:21 +0300 Subject: [PATCH 385/524] [Tolk] Don't emit unused functions to Fift code Drop experimental option "remove-unused-functions": it's always on now --- tolk-tester/tests/calls-tests.tolk | 6 +++++- tolk-tester/tests/cells-slices.tolk | 2 ++ tolk-tester/tests/comments-tests.tolk | 1 + tolk-tester/tests/generics-4.tolk | 1 + tolk-tester/tests/indexed-access.tolk | 5 ++++- tolk-tester/tests/intN-tests.tolk | 1 + tolk-tester/tests/logical-operators.tolk | 2 ++ tolk-tester/tests/mutate-methods.tolk | 7 ++++++- tolk-tester/tests/parse-address.tolk | 1 + tolk-tester/tests/remove-unused-functions.tolk | 2 -- tolk-tester/tests/smart-cast-tests.tolk | 9 ++++++--- tolk-tester/tests/special-fun-names.tolk | 2 -- tolk-tester/tests/try-catch-tests.tolk | 2 ++ tolk-tester/tests/type-aliases-tests.tolk | 1 + tolk-tester/tests/use-before-declare.tolk | 2 ++ tolk/compiler-state.cpp | 6 +++--- tolk/compiler-state.h | 2 +- tolk/pipe-generate-fif-output.cpp | 8 ++------ tolk/symtable.cpp | 2 +- 19 files changed, 41 insertions(+), 21 deletions(-) diff --git a/tolk-tester/tests/calls-tests.tolk b/tolk-tester/tests/calls-tests.tolk index 9405c9ead0..2177136f59 100644 --- a/tolk-tester/tests/calls-tests.tolk +++ b/tolk-tester/tests/calls-tests.tolk @@ -179,7 +179,11 @@ fun test11(a: int, b: int) { return ((b < a), (b <= a), (b > a), (b >= a)); } -fun main() {} +fun main() { + // mark used to codegen them + cmp1; cmp2; cmp3; cmp4; + leq1; leq2; leq3; leq4; +} /** @testcase | 101 | 10 | 10 15 100 12 3 110 300 diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 57a92f03b6..17fd0c12bc 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -249,6 +249,7 @@ fun test18() { return (s.loadUint(14), s.loadInt(8), s.loadUint(4)); } +@method_id(119) fun test19() { // numbers with potential overflow for STU are not joined, check via codegen var b = beginCell(); @@ -267,6 +268,7 @@ fun test20() { return (s.loadBool(), s.loadBool(), s.loadBool(), s.loadUint(8), s.loadBool(), s.loadCoins()); } +@method_id(121) fun test21(s: slice) { // successive skipBits are also joined var x = 8; diff --git a/tolk-tester/tests/comments-tests.tolk b/tolk-tester/tests/comments-tests.tolk index 96caefe924..8287808063 100644 --- a/tolk-tester/tests/comments-tests.tolk +++ b/tolk-tester/tests/comments-tests.tolk @@ -18,6 +18,7 @@ fun demo_fields_def(x: int) { } fun main() { + var cb = demo_fields_def; return 1; } diff --git a/tolk-tester/tests/generics-4.tolk b/tolk-tester/tests/generics-4.tolk index c5f3e038c3..4d65ac7046 100644 --- a/tolk-tester/tests/generics-4.tolk +++ b/tolk-tester/tests/generics-4.tolk @@ -63,6 +63,7 @@ fun mySend(p: Parameters): int { return total; } +@method_id(101) fun test1() { eqUnusedT(100); eqUnusedU(100); diff --git a/tolk-tester/tests/indexed-access.tolk b/tolk-tester/tests/indexed-access.tolk index af6888df78..c30911015f 100644 --- a/tolk-tester/tests/indexed-access.tolk +++ b/tolk-tester/tests/indexed-access.tolk @@ -179,6 +179,7 @@ fun getConstTuple(): Tup2Int { return [1,2]; } +@noinline fun testCodegenNoPureIndexedAccess() { (getConstTuple().1, getConstTuple().0) = (3, 4); return 0; @@ -277,7 +278,9 @@ fun test123() { return t; } -fun main(){} +fun main(){ + testCodegenNoPureIndexedAccess(); +} /** diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index cd557092ed..844b2e9121 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -201,6 +201,7 @@ fun test11() { ]; } +@method_id(112) fun test12() { var a = ton("1") + ton("2"); // check absence in fif codegen return ton("0.1"); diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk index 9b70ae40c9..0fcd3f25c0 100644 --- a/tolk-tester/tests/logical-operators.tolk +++ b/tolk-tester/tests/logical-operators.tolk @@ -1,9 +1,11 @@ import "imports/use-dicts.ext.tolk" +@method_id(90) fun simpleAllConst() { return (!0, !!0 & !false, !!!0, !1, !!1, !-1, !!-1, (!5 as int == 0) == !0, !0 == true); } +@method_id(91) fun compileTimeEval1(x: int) { // todo now compiler doesn't understand that bool can't be equal to number other than 0/-1 // (but understands that it can't be positive) diff --git a/tolk-tester/tests/mutate-methods.tolk b/tolk-tester/tests/mutate-methods.tolk index f3e93b694d..b31e91ac89 100644 --- a/tolk-tester/tests/mutate-methods.tolk +++ b/tolk-tester/tests/mutate-methods.tolk @@ -307,7 +307,12 @@ fun testExplicitReturn() { } -fun main(){} +fun main(){ + // mark used to codegen them + load_next; + testStoreUintPureUnusedResult; + testStoreUintImpureUnusedResult +} /** @testcase | 101 | | 70 60 diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index 0079871a62..05b95f4628 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -30,6 +30,7 @@ fun check0(addr: address) { assert (addr.getWorkchain() == 0) throw 111; } +@method_id(90) fun codegenAddrEq(a: address, b: address) { if (a == b) { return 1; } if (a != b) { return 2; } diff --git a/tolk-tester/tests/remove-unused-functions.tolk b/tolk-tester/tests/remove-unused-functions.tolk index a6b910009f..71f71d7622 100644 --- a/tolk-tester/tests/remove-unused-functions.tolk +++ b/tolk-tester/tests/remove-unused-functions.tolk @@ -28,8 +28,6 @@ fun main(): (int, int, int) { } /** -@experimental_options remove-unused-functions - @testcase | 0 | | 3 10 20 @fif_codegen DECLPROC used_as_noncall1() diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 90f1ae6446..17c3bce509 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -610,7 +610,7 @@ fun test55() { // in case of loops, type inferring is re-enterable // first iteration: x is int, eq instantiated // second (final) iteration: x is int?, eq instantiated - // (checked via codegen) + // (but since eq finally becomes unused, it's not generated to fift) eq55(x); __expect_type(x, "int?"); // types are checked (unlike generics instantiated) after inferring x = rand() ? 1 : null; @@ -718,6 +718,9 @@ fun test63() { fun main(x: int?): int { + // mark used to codegen them + test55; + return x == null ? -1 : x; } @@ -749,8 +752,8 @@ fun main(x: int?): int { @stderr warning: unreachable code @stderr var t2 redef = getNullableInt()!; -@fif_codegen eq55() PROC:<{ -@fif_codegen eq55() PROC:<{ +@fif_codegen eq55() PROC:<{ +@fif_codegen_avoid eq55() PROC:<{ @fif_codegen """ diff --git a/tolk-tester/tests/special-fun-names.tolk b/tolk-tester/tests/special-fun-names.tolk index 11c34f98d9..8771762ab1 100644 --- a/tolk-tester/tests/special-fun-names.tolk +++ b/tolk-tester/tests/special-fun-names.tolk @@ -22,8 +22,6 @@ fun test101() { } /** -@experimental_options remove-unused-functions - @testcase | 0 | | 0 @testcase | -1 | | -1 @testcase | -2 | | -2 diff --git a/tolk-tester/tests/try-catch-tests.tolk b/tolk-tester/tests/try-catch-tests.tolk index 136c606d2a..df2e7c6dd9 100644 --- a/tolk-tester/tests/try-catch-tests.tolk +++ b/tolk-tester/tests/try-catch-tests.tolk @@ -264,6 +264,8 @@ fun testBigExcno() { } fun main() { + // mark used to codegen them + testCodegen3; } /** diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk index 3bc0e51142..cf542310bc 100644 --- a/tolk-tester/tests/type-aliases-tests.tolk +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -143,6 +143,7 @@ fun test8() { someFn1 as (MInt_v2) -> MInt_v2?; } +@method_id(109) fun test9(b: MBool): MBool { if (!b) { __expect_type(b, "MBool"); diff --git a/tolk-tester/tests/use-before-declare.tolk b/tolk-tester/tests/use-before-declare.tolk index 0c59c67158..d5ecd40701 100644 --- a/tolk-tester/tests/use-before-declare.tolk +++ b/tolk-tester/tests/use-before-declare.tolk @@ -23,6 +23,7 @@ fun demo1(v: int): int { global demo_var: int; const demo_10: int = 10; +@method_id(101) fun test1(): int { var demo_var: int = demo_10; var demo_slice: int = demo_20; @@ -33,6 +34,7 @@ fun test1(): int { return demo_var + demo_slice; } +@method_id(102) fun test2() { return second; } diff --git a/tolk/compiler-state.cpp b/tolk/compiler-state.cpp index f5bda29e6f..e11af94497 100644 --- a/tolk/compiler-state.cpp +++ b/tolk/compiler-state.cpp @@ -43,9 +43,9 @@ void PersistentHeapAllocator::clear() { void CompilerSettings::enable_experimental_option(std::string_view name) { ExperimentalOption* to_enable = nullptr; - if (name == remove_unused_functions.name) { - to_enable = &remove_unused_functions; - } + // if (name == some_option.name) { + // to_enable = &some_option; + // } if (to_enable == nullptr) { std::cerr << "unknown experimental option: " << name << std::endl; diff --git a/tolk/compiler-state.h b/tolk/compiler-state.h index 6685e092a7..2f27692cf3 100644 --- a/tolk/compiler-state.h +++ b/tolk/compiler-state.h @@ -60,7 +60,7 @@ struct CompilerSettings { FsReadCallback read_callback; - ExperimentalOption remove_unused_functions{"remove-unused-functions"}; + // ExperimentalOption some_option{"some-option"}; void enable_experimental_option(std::string_view name); void parse_experimental_options_cmd_arg(const std::string& cmd_arg); diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp index 1dc513f054..8f55737d9d 100644 --- a/tolk/pipe-generate-fif-output.cpp +++ b/tolk/pipe-generate-fif-output.cpp @@ -85,11 +85,7 @@ static void generate_output_func(FunctionPtr fun_ref) { modifier = "REF"; } if (G.settings.tolk_src_as_line_comments) { - std::cout << " // " << fun_ref->loc; - if (!fun_ref->n_times_called && !fun_ref->is_used_as_noncall() && !fun_ref->is_entrypoint() && !fun_ref->has_tvm_method_id()) { - std::cout << " (note: function never called!)"; - } - std::cout << std::endl; + std::cout << " // " << fun_ref->loc << std::endl; } std::cout << " " << fun_ref->name << "() PROC" << modifier << ":<{"; int mode = 0; @@ -173,7 +169,7 @@ void pipeline_generate_fif_output_to_std_cout() { } for (GlobalVarPtr var_ref : G.all_global_vars) { - if (!var_ref->is_really_used() && G.settings.remove_unused_functions) { + if (!var_ref->is_really_used()) { if (G.is_verbosity(2)) { std::cerr << var_ref->name << ": variable not generated, it's unused\n"; } diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 348fac8399..b3d9516482 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -66,7 +66,7 @@ LocalVarPtr FunctionData::find_param(std::string_view name) const { bool FunctionData::does_need_codegen() const { // when a function is declared, but not referenced from code in any way, don't generate its body - if (!is_really_used() && G.settings.remove_unused_functions) { + if (!is_really_used()) { return false; } // functions with asm body don't need code generation From 6f60419f24f5b96734fd07f7e5d12d2ee8547e52 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 31 Jul 2025 13:09:21 +0300 Subject: [PATCH 386/524] [Tolk] Prevent exponential explosion while evaluating constants --- tolk-tester/tests/some-tests-4.tolk | 110 ++++++++++++++++++++++++++++ tolk/pipe-ast-to-legacy.cpp | 56 +++++++++++--- tolk/tolk.h | 9 +++ 3 files changed, 163 insertions(+), 12 deletions(-) create mode 100644 tolk-tester/tests/some-tests-4.tolk diff --git a/tolk-tester/tests/some-tests-4.tolk b/tolk-tester/tests/some-tests-4.tolk new file mode 100644 index 0000000000..74645e15e4 --- /dev/null +++ b/tolk-tester/tests/some-tests-4.tolk @@ -0,0 +1,110 @@ +const __precisionZeros = 1000000000000000000; // 18 zeros +const __base0 = 1000000043929018416; // 4x every year (3600 * (24 * 365 + 6) seconds) + +const __base1 = __base0 * __base0 / __precisionZeros; +const __base2 = __base1 * __base1 / __precisionZeros; +const __base3 = __base2 * __base2 / __precisionZeros; +const __base4 = __base3 * __base3 / __precisionZeros; +const __base5 = __base4 * __base4 / __precisionZeros; +const __base6 = __base5 * __base5 / __precisionZeros; +const __base7 = __base6 * __base6 / __precisionZeros; +const __base8 = __base7 * __base7 / __precisionZeros; +const __base9 = __base8 * __base8 / __precisionZeros; +const __base10 = __base9 * __base9 / __precisionZeros; +const __base11 = __base10 * __base10 / __precisionZeros; +const __base12 = __base11 * __base11 / __precisionZeros; +const __base13 = __base12 * __base12 / __precisionZeros; +const __base14 = __base13 * __base13 / __precisionZeros; +const __base15 = __base14 * __base14 / __precisionZeros; +const __base16 = __base15 * __base15 / __precisionZeros; +const __base17 = __base16 * __base16 / __precisionZeros; +const __base18 = __base17 * __base17 / __precisionZeros; +const __base19 = __base18 * __base18 / __precisionZeros; +const __base20 = __base19 * __base19 / __precisionZeros; +const __base21 = __base20 * __base20 / __precisionZeros; +const __base22 = __base21 * __base21 / __precisionZeros; +const __base23 = __base22 * __base22 / __precisionZeros; +const __base24 = __base23 * __base23 / __precisionZeros; +const __base25 = __base24 * __base24 / __precisionZeros; +const __base26 = __base25 * __base25 / __precisionZeros; +const __base27 = __base26 * __base26 / __precisionZeros; +const __base28 = __base27 * __base27 / __precisionZeros; +const __base29 = __base28 * __base28 / __precisionZeros; +const __base30 = __base29 * __base29 / __precisionZeros; + +const ERROR_EXP_TOO_LARGE = 10970; + +@method_id(101) +fun test1(value: int, exp: int): int { + assert (exp <= 0x7fffffff) throw ERROR_EXP_TOO_LARGE; + + if ((exp & 0x00000001) != 0) { value = mulDivFloor(value, __base0, __precisionZeros); } + if ((exp & 0x00000002) != 0) { value = mulDivFloor(value, __base1, __precisionZeros); } + if ((exp & 0x00000004) != 0) { value = mulDivFloor(value, __base2, __precisionZeros); } + if ((exp & 0x00000008) != 0) { value = mulDivFloor(value, __base3, __precisionZeros); } + + if ((exp & 0x00000010) != 0) { value = mulDivFloor(value, __base4, __precisionZeros); } + if ((exp & 0x00000020) != 0) { value = mulDivFloor(value, __base5, __precisionZeros); } + if ((exp & 0x00000040) != 0) { value = mulDivFloor(value, __base6, __precisionZeros); } + if ((exp & 0x00000080) != 0) { value = mulDivFloor(value, __base7, __precisionZeros); } + + if ((exp & 0x00000100) != 0) { value = mulDivFloor(value, __base8, __precisionZeros); } + if ((exp & 0x00000200) != 0) { value = mulDivFloor(value, __base9, __precisionZeros); } + if ((exp & 0x00000400) != 0) { value = mulDivFloor(value, __base10, __precisionZeros); } + if ((exp & 0x00000800) != 0) { value = mulDivFloor(value, __base11, __precisionZeros); } + + if ((exp & 0x00001000) != 0) { value = mulDivFloor(value, __base12, __precisionZeros); } + if ((exp & 0x00002000) != 0) { value = mulDivFloor(value, __base13, __precisionZeros); } + if ((exp & 0x00004000) != 0) { value = mulDivFloor(value, __base14, __precisionZeros); } + if ((exp & 0x00008000) != 0) { value = mulDivFloor(value, __base15, __precisionZeros); } + + if ((exp & 0x00010000) != 0) { value = mulDivFloor(value, __base16, __precisionZeros); } + if ((exp & 0x00020000) != 0) { value = mulDivFloor(value, __base17, __precisionZeros); } + if ((exp & 0x00040000) != 0) { value = mulDivFloor(value, __base18, __precisionZeros); } + if ((exp & 0x00080000) != 0) { value = mulDivFloor(value, __base19, __precisionZeros); } + + if ((exp & 0x00100000) != 0) { value = mulDivFloor(value, __base20, __precisionZeros); } + if ((exp & 0x00200000) != 0) { value = mulDivFloor(value, __base21, __precisionZeros); } + if ((exp & 0x00400000) != 0) { value = mulDivFloor(value, __base22, __precisionZeros); } + if ((exp & 0x00800000) != 0) { value = mulDivFloor(value, __base23, __precisionZeros); } + + if ((exp & 0x01000000) != 0) { value = mulDivFloor(value, __base24, __precisionZeros); } + if ((exp & 0x02000000) != 0) { value = mulDivFloor(value, __base25, __precisionZeros); } + if ((exp & 0x04000000) != 0) { value = mulDivFloor(value, __base26, __precisionZeros); } + if ((exp & 0x08000000) != 0) { value = mulDivFloor(value, __base27, __precisionZeros); } + + if ((exp & 0x10000000) != 0) { value = mulDivFloor(value, __base28, __precisionZeros); } + if ((exp & 0x20000000) != 0) { value = mulDivFloor(value, __base29, __precisionZeros); } + if ((exp & 0x40000000) != 0) { value = mulDivFloor(value, __base30, __precisionZeros); } + + return value; +} + +@method_id(102) +fun test2(g1: bool) { + if (g1) { + return __base30; + } + return __base30; +} + +fun main() { + return 0x7fffffff; +} + +/** +@testcase | 101 | 12345678901234567890 0 | 12345678901234567890 +@testcase | 101 | 12345678901234567890 2147483647 | 1152091301147391687695287037940761486473052236056853112522947 +@testcase | 102 | 0 | 305482242253527345283975278648594775678 +@testcase | 102 | -1 | 305482242253527345283975278648594775678 + +@fif_codegen +""" + test2() PROC:<{ + IFJMP:<{ + 305482242253527345283975278648594775678 PUSHINT // '62 + }> + 305482242253527345283975278648594775678 PUSHINT // '124 + }> +""" + */ diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 1f4ae232c3..9cfc5ee49a 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -383,6 +383,15 @@ const LazyVariableLoadedState* CodeBlob::get_lazy_variable(AnyExprV v) const { return nullptr; } +const CachedConstValueAtCodegen* CodeBlob::get_cached_const(GlobalConstPtr const_ref) const { + for (const CachedConstValueAtCodegen& c : cached_consts) { + if (c.const_ref == const_ref) { + return &c; + } + } + return nullptr; +} + // given `{some_expr}!`, return some_expr static AnyExprV unwrap_not_null_operator(AnyExprV v) { @@ -1111,6 +1120,38 @@ std::vector transition_to_target_type(std::vector&& rvect, std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, CodeBlob& code, LValContext* lval_ctx) { + // referencing a local variable (not its declaration, but its usage) + if (LocalVarPtr var_ref = sym->try_as()) { +#ifdef TOLK_DEBUG + tolk_assert(static_cast(var_ref->ir_idx.size()) == var_ref->declared_type->get_width_on_stack()); +#endif + return var_ref->ir_idx; + } + + // referencing a global constant, embed its init_value directly (so, it will be evaluated to a const val later) + if (GlobalConstPtr const_ref = sym->try_as()) { + tolk_assert(lval_ctx == nullptr); + // when evaluating `a1` in `const a2 = a1 + a1`, cache a1's IR vars to prevent exponential explosion + if (code.inside_evaluating_constant) { + if (const CachedConstValueAtCodegen* cached = code.get_cached_const(const_ref)) { + return cached->ir_idx; + } + std::vector ir_sub_const = pre_compile_expr(const_ref->init_value, code, const_ref->declared_type); + code.cached_consts.emplace_back(CachedConstValueAtCodegen{const_ref, ir_sub_const}); + return ir_sub_const; + } + // just referencing `a1` in a function body + // (note that `a1` occurred 2 times has 2 different ir_idx, caching above is done only within another const) + ASTAuxData* aux_data = new AuxData_ForceFiftLocation(loc); + auto v_force_loc = createV(loc, const_ref->init_value, aux_data, const_ref->inferred_type); + code.inside_evaluating_constant = true; + std::vector ir_const = pre_compile_expr(v_force_loc, code, const_ref->declared_type); + code.inside_evaluating_constant = false; + code.cached_consts.clear(); + return ir_const; + } + + // referencing a global variable, copy it to a local tmp var if (GlobalVarPtr glob_ref = sym->try_as()) { // handle `globalVar = rhs` / `mutate globalVar` if (lval_ctx && !lval_ctx->is_rval_inside_lval()) { @@ -1127,23 +1168,14 @@ std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, Co } return local_ir_idx; } - if (GlobalConstPtr const_ref = sym->try_as()) { - tolk_assert(lval_ctx == nullptr); - ASTAuxData* aux_data = new AuxData_ForceFiftLocation(loc); - auto v_force_loc = createV(loc, const_ref->init_value, aux_data, const_ref->inferred_type); - return pre_compile_expr(v_force_loc, code, const_ref->declared_type); - } + + // referencing a function (not calling it! using as a callback, works similar to a global var) if (FunctionPtr fun_ref = sym->try_as()) { std::vector rvect = code.create_tmp_var(fun_ref->inferred_full_type, loc, "(glob-var-fun)"); code.emplace_back(loc, Op::_GlobVar, rvect, std::vector{}, fun_ref); return rvect; } - if (LocalVarPtr var_ref = sym->try_as()) { -#ifdef TOLK_DEBUG - tolk_assert(static_cast(var_ref->ir_idx.size()) == var_ref->declared_type->get_width_on_stack()); -#endif - return var_ref->ir_idx; - } + throw Fatal("pre_compile_symbol"); } diff --git a/tolk/tolk.h b/tolk/tolk.h index 75b6aa4070..43bce76496 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -1068,6 +1068,12 @@ struct LazyVarRefAtCodegen { : var_ref(var_ref), var_state(var_state) {} }; +// CachedConstValueAtCodegen is used for a map [some_const => '5] +struct CachedConstValueAtCodegen { + GlobalConstPtr const_ref; + std::vector ir_idx; +}; + struct CodeBlob { int var_cnt, in_var_cnt; FunctionPtr fun_ref; @@ -1075,7 +1081,9 @@ struct CodeBlob { SrcLocation forced_loc; std::vector vars; std::vector lazy_variables; + std::vector cached_consts; std::vector* inline_rvect_out = nullptr; + bool inside_evaluating_constant = false; bool inlining_before_immediate_return = false; std::unique_ptr ops; std::unique_ptr* cur_ops; @@ -1130,6 +1138,7 @@ struct CodeBlob { } const LazyVariableLoadedState* get_lazy_variable(LocalVarPtr var_ref) const; const LazyVariableLoadedState* get_lazy_variable(AnyExprV v) const; + const CachedConstValueAtCodegen* get_cached_const(GlobalConstPtr const_ref) const; void prune_unreachable_code(); void fwd_analyze(); void mark_noreturn(); From 08c61dfc8209ee988cbc28aa58cae6d3d18effbd Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 4 Aug 2025 20:10:16 +0700 Subject: [PATCH 387/524] [Tolk] Fix createMessage() with a dynamic union destination --- tolk-tester/tests/send-msg-4.tolk | 60 +++++++++++++++++++++++ tolk/send-message-api.cpp | 79 +++++++++++++++++-------------- 2 files changed, 103 insertions(+), 36 deletions(-) create mode 100644 tolk-tester/tests/send-msg-4.tolk diff --git a/tolk-tester/tests/send-msg-4.tolk b/tolk-tester/tests/send-msg-4.tolk new file mode 100644 index 0000000000..84706fd71e --- /dev/null +++ b/tolk-tester/tests/send-msg-4.tolk @@ -0,0 +1,60 @@ +struct LiquidityDepositWithInitData { + code: cell + data: cell +} + +type LiquidityDepositAddress = address + +type LiquidityDepositDestination = LiquidityDepositWithInitData | LiquidityDepositAddress + +struct (0x1b434676) AddLiquidityPartTon { + destination: LiquidityDepositDestination +} + +type AllowedMessages = AddLiquidityPartTon + +fun main() { + val msgCell = beginCell(). + storeUint(0x1b434676, 32) + .storeBool(true) // either right + .storeAddress(address("Ef_vA6yRfmt2P4UHnxlrQUZFcBnKux8mL2eMqBgpeMFPorr4")) + .endCell(); + + val msg = lazy AllowedMessages.fromCell(msgCell); + + match (msg) { + AddLiquidityPartTon => { + // dynamic union: the compiler inserts lots of IFs at runtime to handle this + var destination: address | AutoDeployAddress; + + match (msg.destination) { + LiquidityDepositWithInitData => { + destination = { + stateInit: { code: msg.destination.code, data: msg.destination.data }, + }; + } + LiquidityDepositAddress => { + destination = msg.destination; + } + } + + val sendMsg = createMessage({ + bounce: false, + dest: destination, + value: 0, + }); + sendMsg.send(SEND_MODE_REGULAR); + } + } + + return 0; +} + +/** +@testcase | 0 | | 0 + +@fif_codegen ONE HASHEXT_SHA256 +@fif_codegen }>ELSE<{ +@fif_codegen HASHCU +@fif_codegen 256 STU + */ diff --git a/tolk/send-message-api.cpp b/tolk/send-message-api.cpp index e2323b17f0..2048322936 100644 --- a/tolk/send-message-api.cpp +++ b/tolk/send-message-api.cpp @@ -55,39 +55,49 @@ static void append_bitwise_and_shard_mask(CodeBlob& code, SrcLocation loc, var_i // struct AutoDeployAddress { workchain: int8; stateInit: ContractState | cell; toShard: AddressShardingOptions?; } struct IR_AutoDeployAddress { - std::vector is_ContractState, // stateInit is ContractState - is_AddressSharding; // toShard is not null + std::vector ir_stateInitField, + ir_toShardField; + const TypeDataUnion* t_stateInit; + const TypeDataUnion* t_toShard; var_idx_t workchain, // workchain stateInitCode, stateInitData, stateInitCell, // stateInit ir_shardDepth, ir_closeTo; // toShard IR_AutoDeployAddress(CodeBlob& code, SrcLocation loc, const std::vector& ir_vars) { StructPtr s_AutoDeployAddress = lookup_global_symbol("AutoDeployAddress")->try_as(); - const TypeDataUnion* t_stateInit = s_AutoDeployAddress->find_field("stateInit")->declared_type->try_as(); - const TypeDataUnion* t_toShard = s_AutoDeployAddress->find_field("toShard")->declared_type->try_as(); + t_stateInit = s_AutoDeployAddress->find_field("stateInit")->declared_type->try_as(); + t_toShard = s_AutoDeployAddress->find_field("toShard")->declared_type->try_as(); tolk_assert(ir_vars.size() == 1 + 3 + 3); tolk_assert(t_stateInit && t_stateInit->get_width_on_stack() == (2+1) && t_stateInit->size() == 2); tolk_assert(t_toShard && t_toShard->get_width_on_stack() == (2+1) && t_toShard->or_null); workchain = ir_vars[0]; - std::vector ir_stateInitUnion(ir_vars.begin() + 1, ir_vars.begin() + 1 + 3); - is_ContractState = pre_compile_is_type(code, t_stateInit, t_stateInit->variants[0], ir_stateInitUnion, loc, "(is-ContractState)"); - std::vector ir_ContractState = transition_to_target_type(std::vector(ir_stateInitUnion), code, t_stateInit, t_stateInit->variants[0], loc); + ir_stateInitField = std::vector(ir_vars.begin() + 1, ir_vars.begin() + 1 + 3); + std::vector ir_ContractState = transition_to_target_type(std::vector(ir_stateInitField), code, t_stateInit, t_stateInit->variants[0], loc); stateInitCode = ir_ContractState[0]; stateInitData = ir_ContractState[1]; - stateInitCell = transition_to_target_type(std::vector(ir_stateInitUnion), code, t_stateInit, t_stateInit->variants[1], loc)[0]; + stateInitCell = transition_to_target_type(std::vector(ir_stateInitField), code, t_stateInit, t_stateInit->variants[1], loc)[0]; - std::vector ir_toShardOrNull(ir_vars.begin() + 1 + 3, ir_vars.begin() + 1 + 3 + 3); - is_AddressSharding = pre_compile_is_type(code, t_toShard, t_toShard->or_null, ir_toShardOrNull, loc, "(is-AddressSharding)"); - std::vector ir_AddressSharding = transition_to_target_type(std::vector(ir_toShardOrNull), code, t_toShard, t_toShard->or_null, loc); + ir_toShardField = std::vector(ir_vars.begin() + 1 + 3, ir_vars.begin() + 1 + 3 + 3); + std::vector ir_AddressSharding = transition_to_target_type(std::vector(ir_toShardField), code, t_toShard, t_toShard->or_null, loc); ir_shardDepth = ir_AddressSharding[0]; ir_closeTo = ir_AddressSharding[1]; } + + // generate IR vars "stateInit is ContractState" + std::vector is_ContractState(CodeBlob& code, SrcLocation loc) const { + return pre_compile_is_type(code, t_stateInit, t_stateInit->variants[0], ir_stateInitField, loc, "(is-ContractState)"); + } + + // generate IR vars "toShard is not null" + std::vector is_AddressSharding(CodeBlob& code, SrcLocation loc) const { + return pre_compile_is_type(code, t_toShard, t_toShard->or_null, ir_toShardField, loc, "(is-AddressSharding)"); + } }; // fun createMessage(options: CreateMessageOptions): OutMessage -std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& ir_options) { TypePtr bodyT = called_f->substitutedTs->typeT_at(0); StructPtr s_Options = lookup_global_symbol("CreateMessageOptions")->try_as(); StructPtr s_AutoDeployAddress = lookup_global_symbol("AutoDeployAddress")->try_as(); @@ -100,7 +110,7 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co tolk_assert(t_value && t_value->get_width_on_stack() == (2+1) && t_value->size() == 2); int offset = 0; - std::vector rvect = args[0]; + std::vector rvect = ir_options[0]; auto next_slice = [&rvect, &offset](int width) -> std::vector { int start = offset; offset += width; @@ -118,14 +128,11 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co // struct ContractState { code: cell; data: cell; } // struct AddressShardingOptions { fixedPrefixLength: uint5; closeTo: address; } std::vector ir_dest_is_address = pre_compile_is_type(code, t_dest, TypeDataAddress::create(), ir_dest, loc, "(is-address)"); - std::vector ir_dest_is_AutoDeploy = pre_compile_is_type(code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), ir_dest, loc, "(is-address)"); + std::vector ir_dest_is_AutoDeploy = pre_compile_is_type(code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), ir_dest, loc, "(is-auto)"); std::vector ir_dest_is_builder = pre_compile_is_type(code, t_dest, TypeDataBuilder::create(), ir_dest, loc, "(is-builder)"); std::vector ir_dest_AutoDeployAddress = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), loc); IR_AutoDeployAddress ir_dest_ad(code, loc, ir_dest_AutoDeployAddress); - // currently, there is no way to pass PackOptions, defaults are used - std::vector ir_options = create_default_PackOptions(code, loc); - FunctionPtr f_beginCell = lookup_function("beginCell"); FunctionPtr f_endCell = lookup_function("builder.endCell"); @@ -146,7 +153,7 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co if (body_store_as_ref && !body_already_ref) { std::vector ir_ref_builder = code.create_var(TypeDataBuilder::create(), loc, "refb"); code.emplace_back(loc, Op::_Call, ir_ref_builder, std::vector{}, f_beginCell); - PackContext ref_ctx(code, loc, ir_ref_builder, ir_options); + PackContext ref_ctx(code, loc, ir_ref_builder, create_default_PackOptions(code, loc)); ref_ctx.generate_pack_any(bodyT, std::move(ir_body)); std::vector ir_ref_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(ref-cell)"); code.emplace_back(loc, Op::_Call, ir_ref_cell, std::move(ir_ref_builder), f_endCell); @@ -155,7 +162,7 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co std::vector ir_builder = code.create_var(TypeDataSlice::create(), loc, "b"); code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, f_beginCell); - PackContext ctx(code, loc, ir_builder, ir_options); + PackContext ctx(code, loc, ir_builder, create_default_PackOptions(code, loc)); var_idx_t ir_zero = code.create_int(loc, 0, "(zero)"); var_idx_t ir_one = code.create_int(loc, 1, "(one)"); @@ -188,11 +195,11 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); // addr_std$10 + 0 anycast ctx.storeInt(ir_dest_ad.workchain, 8); std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); - Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_dest_ad.is_ContractState); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_dest_ad.is_ContractState(code, loc)); { // input is `dest: { ... stateInit: { code, data } }` code.push_set_cur(if_ContractState.block0); - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding(code, loc)); { // input is `dest: { ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } }; // then stateInitHash = (hash of StateInit = 0b1(depth)0110 (prefix + code + data)) @@ -218,7 +225,7 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("cell.hash")); code.close_pop_cur(loc); } - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding(code, loc)); { // input is `dest: { ... toShard: { fixedPrefixLength, closeTo } }` // we already calculated stateInitHash (ir_hash): either cell.hash() or based on prefix+code+data; @@ -306,12 +313,12 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co } { code.push_set_cur(if_no_init.block0); - Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_dest_ad.is_ContractState); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_dest_ad.is_ContractState(code, loc)); { // input is `dest: { ... stateInit: { code, data } }` and need to compose TL/B StateInit; // it's either just code+data OR (if `toShard: { ... }` is set) fixedPrefixLength+code+data code.push_set_cur(if_ContractState.block0); - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding(code, loc)); { // 1 (maybe true) + 0 (either left) + 1 (maybe true of StateInit) + fixedPrefixLength + 0110 + body ref or not code.push_set_cur(if_sharded.block0); @@ -530,8 +537,8 @@ std::vector generate_address_buildInAnotherShard(FunctionPtr called_f } // fun AutoDeployAddress.buildAddress(self): builder -std::vector generate_AutoDeployAddress_buildAddress(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { - IR_AutoDeployAddress ir_self(code, loc, args[0]); +std::vector generate_AutoDeployAddress_buildAddress(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& ir_options) { + IR_AutoDeployAddress ir_self(code, loc, ir_options[0]); std::vector ir_builder = code.create_tmp_var(TypeDataSlice::create(), loc, "(addr-b)"); // important! unlike `createMessage()`, we calculate hash and shard prefix BEFORE creating a cell @@ -539,11 +546,11 @@ std::vector generate_AutoDeployAddress_buildAddress(FunctionPtr calle // calculate stateInitHash = (hash of StateInit cell would be, but without constructing a cell) std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); - Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_self.is_ContractState); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_self.is_ContractState(code, loc)); { // called `{ ... stateInit: { code, data } }` code.push_set_cur(if_ContractState.block0); - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding(code, loc)); { // called `{ ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } } code.push_set_cur(if_sharded.block0); @@ -569,7 +576,7 @@ std::vector generate_AutoDeployAddress_buildAddress(FunctionPtr calle } // now, if toShard, perform bitwise calculations with hashes (order on a stack matters) - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding(code, loc)); { // called `{ ... toShard: { fixedPrefixLength, closeTo } }` // we already calculated stateInitHash (ir_hash): either cell.hash() or based on prefix+code+data; @@ -609,16 +616,16 @@ std::vector generate_AutoDeployAddress_buildAddress(FunctionPtr calle } // fun AutoDeployAddress.addressMatches(self, addr: address): bool -std::vector generate_AutoDeployAddress_addressMatches(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { - IR_AutoDeployAddress ir_self(code, loc, args[0]); +std::vector generate_AutoDeployAddress_addressMatches(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& ir_self_and_addr) { + IR_AutoDeployAddress ir_self(code, loc, ir_self_and_addr[0]); // at first, calculate stateInitHash = (hash of StateInit cell would be, but without constructing a cell) std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); - Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_self.is_ContractState); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_self.is_ContractState(code, loc)); { // called `{ ... stateInit: { code, data } }` code.push_set_cur(if_ContractState.block0); - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding(code, loc)); { // called `{ ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } } code.push_set_cur(if_sharded.block0); @@ -644,7 +651,7 @@ std::vector generate_AutoDeployAddress_addressMatches(FunctionPtr cal } // now calculate `stateInitHash &= mask` where mask = `(1 << (256 - SHARD_DEPTH)) - 1` - Op& if_sharded1 = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + Op& if_sharded1 = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding(code, loc)); { code.push_set_cur(if_sharded1.block0); append_bitwise_and_shard_mask(code, loc, ir_hash[0], ir_self.ir_shardDepth); @@ -657,10 +664,10 @@ std::vector generate_AutoDeployAddress_addressMatches(FunctionPtr cal // now do `(wc, hash) = addr.getWorkchainAndHash()` std::vector ir_addr_wc_hash = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create()}), loc, "(self-wc-hash)"); - code.emplace_back(loc, Op::_Call, ir_addr_wc_hash, args[1], lookup_function("address.getWorkchainAndHash")); + code.emplace_back(loc, Op::_Call, ir_addr_wc_hash, ir_self_and_addr[1], lookup_function("address.getWorkchainAndHash")); // now calculate `hash &= mask` (the same as we did earlier for stateInitHash) - Op& if_sharded2 = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + Op& if_sharded2 = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding(code, loc)); { code.push_set_cur(if_sharded2.block0); append_bitwise_and_shard_mask(code, loc, ir_addr_wc_hash[1], ir_self.ir_shardDepth); From 7b89b2a38b4d59e1b2512c26342d438bdd6f2414 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 7 Aug 2025 15:43:20 +0700 Subject: [PATCH 388/524] [Tolk] Stricter type aliases and equal_to for types - Added `equal_to()` instead of comparing TypePtr pointers - Aliases referencing the same underlying type are now stricter: no silent interchangeability - Smart casts restore declared type after control flow --- tolk-tester/tests/generics-2.tolk | 18 +- tolk-tester/tests/generics-4.tolk | 2 +- tolk-tester/tests/inline-tests.tolk | 4 +- tolk-tester/tests/intN-tests.tolk | 2 +- .../tests/invalid-semantics/err-4081.tolk | 2 +- .../tests/invalid-semantics/err-4318.tolk | 13 + .../tests/invalid-semantics/err-4539.tolk | 25 + .../tests/invalid-semantics/err-4619.tolk | 15 + .../tests/invalid-semantics/err-4680.tolk | 15 + .../tests/invalid-semantics/err-4681.tolk | 15 + .../tests/invalid-typing/err-6274.tolk | 2 +- .../tests/invalid-typing/err-6513.tolk | 23 + .../tests/invalid-typing/err-6593.tolk | 13 + .../tests/invalid-typing/err-6834.tolk | 14 + tolk-tester/tests/lazy-load-tests.tolk | 10 +- tolk-tester/tests/match-by-expr-tests.tolk | 4 +- tolk-tester/tests/methods-tests.tolk | 19 + tolk-tester/tests/nullable-tensors.tolk | 18 +- tolk-tester/tests/pack-unpack-2.tolk | 8 +- tolk-tester/tests/pack-unpack-6.tolk | 10 +- tolk-tester/tests/smart-cast-tests.tolk | 67 ++- tolk-tester/tests/struct-tests.tolk | 26 +- tolk-tester/tests/type-aliases-tests.tolk | 215 +++++++- tolk-tester/tests/union-types-tests.tolk | 32 +- tolk-tester/tests/var-apply-tests.tolk | 4 +- tolk-tester/tolk-tester.js | 2 +- tolk-tester/tolk-tester.py | 2 +- tolk/ast-from-tokens.cpp | 2 +- tolk/generics-helpers.cpp | 6 +- tolk/pack-unpack-serializers.cpp | 2 +- tolk/pipe-ast-to-legacy.cpp | 24 +- tolk/pipe-check-inferred-types.cpp | 33 +- tolk/pipe-infer-types-and-calls.cpp | 24 +- tolk/pipe-resolve-identifiers.cpp | 14 +- tolk/smart-casts-cfg.cpp | 41 +- tolk/smart-casts-cfg.h | 6 +- tolk/type-system.cpp | 513 +++++++++--------- tolk/type-system.h | 38 +- 38 files changed, 844 insertions(+), 439 deletions(-) create mode 100644 tolk-tester/tests/invalid-semantics/err-4318.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4539.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4619.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4680.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4681.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6513.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6593.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6834.tolk diff --git a/tolk-tester/tests/generics-2.tolk b/tolk-tester/tests/generics-2.tolk index 2a6684f0dd..94ae2ea038 100644 --- a/tolk-tester/tests/generics-2.tolk +++ b/tolk-tester/tests/generics-2.tolk @@ -330,8 +330,8 @@ type StrangeAlsoInt = StrangeInt<()>; fun test19() { var i1: StrangeInt = 20; var i2: StrangeInt = 30; - var i3: StrangeInt> = 40 as StrangeAlsoInt; - i1 = i2; i2 = i1; i1 = i3; i3 = i1; + var i3: StrangeInt> = 40; + i1 = i2 as StrangeInt; i1 = i3 as StrangeInt; return (i1 as StrangeInt<()>, i3 as StrangeAlsoInt, i1 is StrangeInt, i1 is StrangeInt, i3 is StrangeInt, i3 is StrangeAlsoInt); } @@ -376,7 +376,7 @@ fun test22(w: int | WrapperAlias | slice) { WrapperAlias => {} slice => {} } - __expect_type(w, "int | Wrapper | slice"); + __expect_type(w, "int | WrapperAlias | slice"); match (w) { MyInt => {} slice => {} @@ -453,15 +453,15 @@ fun main(c: Wrapper, d: WrappedInt) { @testcase | 104 | | 11 11 13 13 13 2 @testcase | 105 | 0 | 80 @testcase | 105 | -1 | (null) -@testcase | 106 | | (null) (null) 0 777 1 2 typeid-14 +@testcase | 106 | | (null) (null) 0 777 1 2 typeid-12 @testcase | 107 | 5 | 20 20 5 5 20 777 (null) (null) (null) 0 -@testcase | 108 | 0 | 777 typeid-15 777 typeid-15 +@testcase | 108 | 0 | 777 typeid-1 777 typeid-1 @testcase | 108 | -1 | 777 0 777 0 @testcase | 109 | | 40 40 70 @testcase | 110 | | 20 1 20 42 -@testcase | 111 | | 5 1 typeid-4 5 1 typeid-4 5 1 typeid-4 5 1 typeid-4 -@testcase | 112 | -1 | 10 1 777 10 1 typeid-5 -@testcase | 112 | 0 | 20 1 777 (null) 0 typeid-6 +@testcase | 111 | | 5 1 typeid-2 5 1 typeid-2 5 1 typeid-2 5 1 typeid-2 +@testcase | 112 | -1 | 10 1 777 10 1 typeid-3 +@testcase | 112 | 0 | 20 1 777 (null) 0 typeid-4 @testcase | 113 | | 30 -1 @testcase | 114 | | 999 (null) 2 @testcase | 115 | | 10 0 200 1 -1 200 1 -1 @@ -469,7 +469,7 @@ fun main(c: Wrapper, d: WrappedInt) { @testcase | 117 | | 100 123 @testcase | 118 | | 123 1 777 123 1 -1 2 @testcase | 119 | | 40 40 -1 -1 -1 -1 -@testcase | 120 | | (null) typeid-9 777 (null) (null) 0 typeid-11 -1 -1 +@testcase | 120 | | (null) typeid-9 777 (null) (null) 0 typeid-10 -1 -1 @testcase | 123 | | 10 10 10 10 10 diff --git a/tolk-tester/tests/generics-4.tolk b/tolk-tester/tests/generics-4.tolk index 4d65ac7046..6833d3867e 100644 --- a/tolk-tester/tests/generics-4.tolk +++ b/tolk-tester/tests/generics-4.tolk @@ -176,7 +176,7 @@ fun main() { /** @testcase | 103 | | 10 20 30 777 40 40 @testcase | 104 | | (null) (null) (null) -@testcase | 105 | | -1 (null) 0 (null) typeid-6 777 0 123 0 456 typeid-5 +@testcase | 105 | | -1 (null) 0 (null) typeid-2 777 0 123 0 456 typeid-1 @testcase | 106 | | -1 0 -1 -1 @testcase | 107 | | 1 10 110 117 @testcase | 108 | | 10 (null) 20 diff --git a/tolk-tester/tests/inline-tests.tolk b/tolk-tester/tests/inline-tests.tolk index d85c5543b8..d27e79bb8c 100644 --- a/tolk-tester/tests/inline-tests.tolk +++ b/tolk-tester/tests/inline-tests.tolk @@ -537,7 +537,7 @@ fun usedIn10ButDeclaredBelow(x: int) { """ wrap16() PROC:<{ IFJMP:<{ - 139 PUSHINT + 129 PUSHINT OVER EQUAL IFJMP:<{ @@ -545,7 +545,7 @@ fun usedIn10ButDeclaredBelow(x: int) { 0 GTINT 100 THROWIFNOT }> - 140 PUSHINT + 130 PUSHINT s2 POP EQUAL IFJMP:<{ diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index 844b2e9121..2555d983c7 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -248,7 +248,7 @@ fun main() { @testcase | 110 | | 50000000 50000100 1234000000 51000000 @testcase | 111 | | [ 1000000000 1000000000 1000000000 -321123456789 321123456789 1100000000 ] @testcase | 114 | 5 | (null) (null) 0 -@testcase | 114 | 15 | 15 2 typeid-2 +@testcase | 114 | 15 | 15 2 typeid-1 @fif_codegen DECLPROC assign0() @fif_codegen DECLPROC assign0() diff --git a/tolk-tester/tests/invalid-semantics/err-4081.tolk b/tolk-tester/tests/invalid-semantics/err-4081.tolk index 4442cf2d50..7a3d9bcfb0 100644 --- a/tolk-tester/tests/invalid-semantics/err-4081.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4081.tolk @@ -15,6 +15,6 @@ fun main(w: int | WrapperAlias | WrapperAlias) { /** @compilation_should_fail -@stderr wrong pattern matching: duplicated `Wrapper` +@stderr wrong pattern matching: duplicated `WrapperAlias` @stderr WrapperAlias */ diff --git a/tolk-tester/tests/invalid-semantics/err-4318.tolk b/tolk-tester/tests/invalid-semantics/err-4318.tolk new file mode 100644 index 0000000000..9809b7c8f7 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4318.tolk @@ -0,0 +1,13 @@ +struct Point { + x: int + y: int +} + +fun Point.getX() { + return self.x +} + +/** +@compilation_should_fail +@stderr using `self` in a static method + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4539.tolk b/tolk-tester/tests/invalid-semantics/err-4539.tolk new file mode 100644 index 0000000000..cc15288ee6 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4539.tolk @@ -0,0 +1,25 @@ +import "@stdlib/tvm-dicts" + +type BalanceList = dict +type AssetList = dict + +fun BalanceList.validate(self): int1 { return 0 } +fun AssetList.validate(self): int2 { return 0 } + +fun f(b: BalanceList) { + if (b == null) { return } + else { b = createEmptyDict() } + __expect_type(b, "BalanceList"); + + if (b == null) { return } + __expect_type(b, "cell"); + + b.validate() +} + +/** +@compilation_should_fail +@stderr call to method `validate` for type `cell` is ambiguous +@stderr candidate function: `BalanceList.validate` +@stderr candidate function: `AssetList.validate` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4619.tolk b/tolk-tester/tests/invalid-semantics/err-4619.tolk new file mode 100644 index 0000000000..e62cc22b5e --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4619.tolk @@ -0,0 +1,15 @@ +struct Inc { v: int32 } + +type Msg = Inc + +fun main(s: slice) { + val msg = lazy Msg.fromSlice(s); + match (msg) { + else => throw 0xFFFF + } +} + +/** +@compilation_should_fail +@stderr `match` contains only `else`, but no variants + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4680.tolk b/tolk-tester/tests/invalid-semantics/err-4680.tolk new file mode 100644 index 0000000000..175ec6b659 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4680.tolk @@ -0,0 +1,15 @@ +type BalanceDict = dict + +fun dict.doSome(self) {} +fun BalanceDict.doSome(self) {} + +fun m(d: dict) { + d.doSome() +} + +/** +@compilation_should_fail +@stderr call to method `doSome` for type `dict` is ambiguous +@stderr candidate function: `dict.doSome` +@stderr candidate function: `BalanceDict.doSome` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4681.tolk b/tolk-tester/tests/invalid-semantics/err-4681.tolk new file mode 100644 index 0000000000..0974c50dde --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4681.tolk @@ -0,0 +1,15 @@ +type BalanceDict = dict + +fun dict.doSome(self) {} +fun BalanceDict.doSome(self) {} + +fun m(d: BalanceDict) { + d.doSome() +} + +/** +@compilation_should_fail +@stderr call to method `doSome` for type `BalanceDict` is ambiguous +@stderr candidate function: `dict.doSome` +@stderr candidate function: `BalanceDict.doSome` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6274.tolk b/tolk-tester/tests/invalid-typing/err-6274.tolk index d586efe410..d436c67c67 100644 --- a/tolk-tester/tests/invalid-typing/err-6274.tolk +++ b/tolk-tester/tests/invalid-typing/err-6274.tolk @@ -14,6 +14,6 @@ fun cantMatchAgainstNotVariant(a: int | slice | MBuilder) { /** @compilation_should_fail -@stderr wrong pattern matching: `cell` is not a variant of `int | slice | MBuilder` +@stderr wrong pattern matching: `MCell` is not a variant of `int | slice | MBuilder` @stderr MCell */ diff --git a/tolk-tester/tests/invalid-typing/err-6513.tolk b/tolk-tester/tests/invalid-typing/err-6513.tolk new file mode 100644 index 0000000000..e68bf89e5b --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6513.tolk @@ -0,0 +1,23 @@ +type UserId = int +type OwnerId = int + +fun main() { + var a = 10 as UserId | slice; + match (a) { + UserId => {} + slice => {} + } + __expect_type(a, "UserId | slice"); + + match (a) { + OwnerId => {} + slice => {} + } + return a; +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: `OwnerId` is not a variant of `UserId | slice` +@stderr OwnerId => {} + */ diff --git a/tolk-tester/tests/invalid-typing/err-6593.tolk b/tolk-tester/tests/invalid-typing/err-6593.tolk new file mode 100644 index 0000000000..f6e11190ef --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6593.tolk @@ -0,0 +1,13 @@ +type UserId = int +type OwnerId = int + +fun f(v: UserId) { + match (v) { + OwnerId => {} + } +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: `OwnerId` is not a variant of `UserId` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6834.tolk b/tolk-tester/tests/invalid-typing/err-6834.tolk new file mode 100644 index 0000000000..9936e96fa5 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6834.tolk @@ -0,0 +1,14 @@ +type SnakedCell = cell + +type MInt_v1 = int +type MInt_v2 = int + +fun m(a: SnakedCell, b: SnakedCell) { + a = b +} + +/** +@compilation_should_fail +@stderr can not assign `SnakedCell` to variable of type `SnakedCell` +@stderr hint: use `as` operator for unsafe casting: ` as SnakedCell` + */ diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index f686fe67c9..b850a72809 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -1628,10 +1628,10 @@ fun main() { @testcase | 113 | x{03ffff} | -1 @testcase | 113 | x{04000000FF} | 255 @testcase | 114 | x{0110} | 16 1 -@testcase | 114 | x{020f} | [ -15 ] typeid-27 +@testcase | 114 | x{020f} | [ -15 ] typeid-2 @testcase | 114 | x{03} | -1 1 @testcase | 114 | x{03ffff} | -1 1 -@testcase | 114 | x{04000000FF} | [ 255 ] typeid-27 +@testcase | 114 | x{04000000FF} | [ 255 ] typeid-2 @testcase | 115 | x{08} | 8 42 @testcase | 115 | x{88} | 0 2 @testcase | 115 | x{83} | -1 2 @@ -1646,9 +1646,9 @@ fun main() { @testcase | 122 | x{010288} | -8 1 @testcase | 123 | x{010280} | 5 @testcase | 124 | x{010283} | [ -3 2 ] -@testcase | 125 | | 127 32 15 2 3 typeid-1 15 3 -1 -@testcase | 126 | x{7F0F20} | 127 (null) (null) 15 typeid-3 32 15 -1 -@testcase | 126 | x{7F8F020320} | 127 15 2 3 typeid-4 32 -1 2 +@testcase | 125 | | 127 32 15 2 3 typeid-6 15 3 -1 +@testcase | 126 | x{7F0F20} | 127 (null) (null) 15 typeid-8 32 15 -1 +@testcase | 126 | x{7F8F020320} | 127 15 2 3 typeid-7 32 -1 2 @testcase | 127 | | 1 2 -1 255 @testcase | 128 | x{5500} | -1 -1 @testcase | 128 | x{558F01024} | 0 2 diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk index 489cdf7489..201c19f893 100644 --- a/tolk-tester/tests/match-by-expr-tests.tolk +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -81,9 +81,9 @@ fun test7(x: int8) { @method_id(108) fun test8(x: int) { - var r1 = match (x) { else => 10 }; + var r1 = match (x is int) { false => 0, else => 10 }; match (x) { } - match (x) { else => { r1 += 20; } } + match (x * 0) { 1 => {} else => { r1 += 20; } } return r1; } diff --git a/tolk-tester/tests/methods-tests.tolk b/tolk-tester/tests/methods-tests.tolk index 4b09a53d6a..27c6d17a18 100644 --- a/tolk-tester/tests/methods-tests.tolk +++ b/tolk-tester/tests/methods-tests.tolk @@ -288,6 +288,25 @@ fun test18() { return cb(10); } +type BalanceList = dict +type AssetList = dict + +fun BalanceList.validate(self): int1 { return 0 } +fun AssetList.validate(self): int2 { return 0 } + +fun test19(b: BalanceList, a: AssetList) { + __expect_type(b.validate(), "int1"); + __expect_type(a.validate(), "int2"); + + if (b == null) {} + match (a) { + null => {} + cell => {} + } + __expect_type(b, "BalanceList"); + __expect_type(a, "AssetList"); +} + fun main() {} /** diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index 5b03945425..4aa804b499 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -503,7 +503,7 @@ fun main(){} @testcase | 102 | | 1 2 typeid-1 (null) (null) 0 @testcase | 103 | 1 2 | 3 3 0 1 2 @testcase | 104 | | 1 2 (null) (null) 0 -@testcase | 105 | | (null) (null) (null) 0 1 2 3 typeid-3 +@testcase | 105 | | (null) (null) (null) 0 1 2 3 typeid-2 @testcase | 106 | | 1 2 @testcase | 107 | | 0 0 -1 0 0 -1 @testcase | 108 | 5 6 | 7 8 10 11 typeid-1 (null) (null) 0 @@ -525,33 +525,33 @@ fun main(){} @testcase | 121 | | [ 1 [ 3 4 ] ] @testcase | 122 | 0 | [ 1 [ 3 4 ] 4 (null) ] @testcase | 122 | -1 | [ 1 (null) 4 (null) ] -@testcase | 123 | | 1 3 4 typeid-4 -@testcase | 124 | 0 | 1 3 4 typeid-4 4 (null) (null) 0 +@testcase | 123 | | 1 3 4 typeid-3 +@testcase | 124 | 0 | 1 3 4 typeid-3 4 (null) (null) 0 @testcase | 124 | -1 | 1 (null) (null) 0 4 (null) (null) 0 @testcase | 125 | | 3 @testcase | 126 | | 1 (null) 2 @testcase | 127 | 1 | 1 (null) (null) 0 2 @testcase | 127 | 2 | 1 2 3 typeid-1 4 @testcase | 127 | 3 | 1 (null) (null) 0 5 -@testcase | 128 | 1 | 1 (null) (null) 0 2 typeid-11 +@testcase | 128 | 1 | 1 (null) (null) 0 2 typeid-6 @testcase | 128 | 2 | (null) (null) (null) (null) (null) 0 -@testcase | 128 | 3 | 1 2 3 typeid-1 4 typeid-11 +@testcase | 128 | 3 | 1 2 3 typeid-1 4 typeid-6 @testcase | 129 | 0 | 5 5 0 -1 1 2 0 -1 @testcase | 129 | -1 | 5 5 0 -1 (null) (null) 0 -1 @testcase | 130 | 0 | 1 2 3 typeid-1 @testcase | 130 | -1 | 1 (null) (null) 0 -@testcase | 131 | | typeid-12 777 0 777 777 777 0 0 typeid-12 typeid-12 777 typeid-12 typeid-12 typeid-13 777 +@testcase | 131 | | typeid-7 777 0 777 777 777 0 0 typeid-7 typeid-7 777 typeid-7 typeid-7 typeid-8 777 @testcase | 132 | | -1 0 -1 0 777 (null) (null) -1 0 0 @testcase | 133 | | 60 @testcase | 134 | | 11 21 typeid-1 -@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 typeid-15 (null) typeid-15 (null) 0 777 10 typeid-16 (null) typeid-16 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 +@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 typeid-9 (null) typeid-9 (null) 0 777 10 typeid-10 (null) typeid-10 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 @testcase | 136 | 9 | 9 0 @testcase | 136 | null | (null) -1 -@testcase | 140 | 8 9 | 8 9 typeid-17 (null) (null) 0 +@testcase | 140 | 8 9 | 8 9 typeid-11 (null) (null) 0 @testcase | 141 | | (null) 10 @testcase | 142 | | 3 3 1 2 @testcase | 143 | -1 0 | 1 2 1 3 777 (null) (null) (null) (null) 0 -@testcase | 143 | 0 -1 | 1 (null) 0 3 777 1 2 1 3 typeid-18 +@testcase | 143 | 0 -1 | 1 (null) 0 3 777 1 2 1 3 typeid-12 @fif_codegen """ diff --git a/tolk-tester/tests/pack-unpack-2.tolk b/tolk-tester/tests/pack-unpack-2.tolk index cb01fbfeb0..a920039b40 100644 --- a/tolk-tester/tests/pack-unpack-2.tolk +++ b/tolk-tester/tests/pack-unpack-2.tolk @@ -737,18 +737,18 @@ fun main() { @testcase | 208 | | -1 @testcase | 209 | | -1 @testcase | 210 | | 123 555 46 (null) -@testcase | 211 | | (null) typeid-4 123 -@testcase | 212 | | (null) typeid-5 123 +@testcase | 211 | | (null) typeid-3 123 +@testcase | 212 | | (null) typeid-4 123 @testcase | 213 | | -1 @testcase | 214 | | -1 @testcase | 215 | | -1 @testcase | 216 | | -1 @testcase | 217 | | -1 @testcase | 218 | | -1 -@testcase | 219 | | 44 (null) (null) 46 typeid-15 +@testcase | 219 | | 44 (null) (null) 46 typeid-11 @testcase | 220 | | 0 0 (null) 99 typeid-8 typeid-9 99 1234 (null) 889129 14 @testcase | 221 | | -1 -@testcase | 222 | | 510 567 9392843922 typeid-18 81923 81923 typeid-19 777 0 -1 (null) (null) 0 100000 100000 typeid-19 +@testcase | 222 | | 510 567 9392843922 typeid-13 81923 81923 typeid-14 777 0 -1 (null) (null) 0 100000 100000 typeid-14 @testcase | 223 | | 10 55 (null) @testcase | 224 | | 60 50000000 5 9 123 -1 @testcase | 225 | | 5 40 65535 8 50000000 diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk index 364d026a27..3b5074f566 100644 --- a/tolk-tester/tests/pack-unpack-6.tolk +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -146,14 +146,14 @@ RETALT """ IF:<{ DROP - 139 PUSHINT + 129 PUSHINT }>ELSE<{ x{03} SDBEGINSQ NIP IFNOTJMP:<{ 104 THROW }> - 140 PUSHINT + 130 PUSHINT }> """ @@ -201,7 +201,7 @@ IF:<{ x{01} SDBEGINSQ IF:<{ DROP - 139 PUSHINT + 129 PUSHINT }>ELSE<{ x{03} SDBEGINSQ NIP @@ -209,9 +209,9 @@ IF:<{ 16 PUSHPOW2DEC THROWANY }> - 140 PUSHINT + 130 PUSHINT }> - 139 PUSHINT + 129 PUSHINT EQUAL }> """ diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 17c3bce509..589bf65231 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -716,6 +716,65 @@ fun test63() { __expect_type(r!.sub, "Point"); } +type BalanceDict = dict + +fun cell.myDepth(self): int1 { return 0 } +fun BalanceDict.myDepth(self): int3 { return 0 } +fun null.myDepth(self): int4 { return 0 } + +fun test64(d: dict, b: BalanceDict) { + match (d) { + null => { __expect_type(d.myDepth(), "int4") } + cell => { __expect_type(d.myDepth(), "int1") } + } + __expect_type(d, "dict"); + __expect_type(d.myDepth(), "int3"); + + match (b) { + null => { __expect_type(b.myDepth(), "int4") } + cell => { __expect_type(b.myDepth(), "int1") } + } + __expect_type(b, "BalanceDict"); + __expect_type(b.myDepth(), "int3"); + + if (d == null) { return; } + if (b != null) { return; } + + __expect_type(d.myDepth(), "int1"); + __expect_type(b.myDepth(), "int4"); + + if ([0].0) { + (b, d) = (createEmptyCell(), null) + } + __expect_type(b, "BalanceDict"); + __expect_type(d, "dict"); +} + +struct WithDict { + d: dict + b: BalanceDict +} + +fun test65(w: WithDict, t: (dict, BalanceDict)) { + match (w.d) { + null => { __expect_type(w.d, "null") } + cell => { __expect_type(w.d, "cell") } + } + if (w.b == null) {} + else { __expect_type(w.b, "cell") } + + match (t.1) { + null => { __expect_type(t.1.myDepth(), "int4") } + cell => { __expect_type(t.1.myDepth(), "int1") } + } + if (t.0 != null) { __expect_type(t.0, "cell") } + + __expect_type(w.d, "dict"); + __expect_type(w.b, "BalanceDict"); + __expect_type(t.0, "dict"); + __expect_type(t.1, "BalanceDict"); +} + fun main(x: int?): int { // mark used to codegen them @@ -737,16 +796,16 @@ fun main(x: int?): int { @testcase | 139 | | 16 @testcase | 140 | 5 | 25 @testcase | 141 | | 1 2 -@testcase | 142 | | 5 3 (null) (null) 0 typeid-5 3 (null) (null) 0 +@testcase | 142 | | 5 3 (null) (null) 0 typeid-3 3 (null) (null) 0 @testcase | 143 | | 10 11 (null) 10 11 (null) (null) 0 @testcase | 144 | | 10 11 (null) 10 11 (null) (null) 0 -@testcase | 145 | | 5 3 (null) (null) 0 typeid-5 3 (null) (null) (null) (null) 0 3 (null) (null) 0 -@testcase | 146 | | 3 4 5 3 4 5 typeid-4 +@testcase | 145 | | 5 3 (null) (null) 0 typeid-3 3 (null) (null) (null) (null) 0 3 (null) (null) 0 +@testcase | 146 | | 3 4 5 3 4 5 typeid-2 @testcase | 147 | | (null) (null) 100 (null) 100 (null) (null) 0 @testcase | 158 | | 123 10 123 5 @testcase | 160 | | 101 109 @testcase | 161 | 9 9 | (null) (null) 0 (null) (null) -@testcase | 161 | 19 0 | 19 0 typeid-1 19 0 +@testcase | 161 | 19 0 | 19 0 typeid-4 19 0 @stderr warning: expression of type `int` can never be `null`, this condition is always true @stderr warning: unreachable code diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 8eba3b4274..ae8d51ac2a 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -586,37 +586,37 @@ type PointAlias = Point; @testcase | 105 | | 35 30 45 @testcase | 106 | | 10 20 @testcase | 107 | | 5 5 10 20 15 -@testcase | 108 | | 777 typeid-3 777 0 777 777 777 typeid-3 777 typeid-4 777 777 +@testcase | 108 | | 777 typeid-2 777 0 777 777 777 typeid-2 777 typeid-3 777 777 @testcase | 109 | | 70 30 20 20 -80 @testcase | 110 | 0 | (null) (null) 0 -@testcase | 110 | -1 | 10 20 typeid-5 +@testcase | 110 | -1 | 10 20 typeid-1 @testcase | 111 | 0 | 0 2 10 20 30 @testcase | 111 | -1 | 0 0 3 4 7 @testcase | 112 | 0 | (null) (null) 0 -@testcase | 112 | -1 | 1 2 typeid-5 +@testcase | 112 | -1 | 1 2 typeid-1 @testcase | 113 | | (null) (null) 0 @testcase | 114 | | 1 2 @testcase | 115 | | (null) (null) (null) (null) 0 -@testcase | 116 | | -1 -4 [ 8 4 ] 7 (null) 0 [ 10 11 ] typeid-7 +@testcase | 116 | | -1 -4 [ 8 4 ] 7 (null) 0 [ 10 11 ] typeid-5 @testcase | 117 | 5 | 5 5 5 5 5 @testcase | 117 | null | (null) (null) (null) -1 (null) @testcase | 118 | | 17 -@testcase | 119 | | 10 20 9 10 typeid-5 9 -@testcase | 120 | 0 | 80 9 8 80 typeid-5 +@testcase | 119 | | 10 20 9 10 typeid-1 9 +@testcase | 120 | 0 | 80 9 8 80 typeid-1 @testcase | 120 | -1 | 8 14 (null) (null) 0 @testcase | 121 | 0 | 17 17 8 136 @testcase | 121 | -1 | 8 (null) 8 8 @testcase | 122 | 100 | 101 50 106 212 100 101 101 @testcase | 124 | 66 | 66 -@testcase | 125 | | (null) (null) 40 60 typeid-5 +@testcase | 125 | | (null) (null) 40 60 typeid-1 @testcase | 126 | | 118 -@testcase | 127 | | 0 0 typeid-3 typeid-3 777 -1 -1 0 0 777 0 0 typeid-10 777 0 0 777 (null) (null) 0 -@testcase | 128 | -1 | (null) (null) typeid-3 -1 0 0 777 (null) (null) typeid-3 -1 0 0 -@testcase | 128 | 0 | 1 2 typeid-2 0 -1 0 777 0 typeid-3 typeid-10 0 -1 0 +@testcase | 127 | | 0 0 typeid-2 typeid-2 777 -1 -1 0 0 777 0 0 typeid-7 777 0 0 777 (null) (null) 0 +@testcase | 128 | -1 | (null) (null) typeid-2 -1 0 0 777 (null) (null) typeid-2 -1 0 0 +@testcase | 128 | 0 | 1 2 typeid-8 0 -1 0 777 0 typeid-2 typeid-7 0 -1 0 @testcase | 129 | | (null) -@testcase | 130 | -1 | typeid-12 777 (null) typeid-12 777 0 777 0 0 -@testcase | 130 | 0 | typeid-11 777 4 1 777 typeid-11 777 0 0 -@testcase | 131 | | 5 typeid-13 (null) typeid-13 (null) 0 777 0 0 -1 777 0 -1 +@testcase | 130 | -1 | typeid-9 777 (null) typeid-9 777 0 777 0 0 +@testcase | 130 | 0 | typeid-10 777 4 1 777 typeid-10 777 0 0 +@testcase | 131 | | 5 typeid-11 (null) typeid-11 (null) 0 777 0 0 -1 777 0 -1 @testcase | 132 | | 10 20 10 0 0 20 0 0 0 0 @testcase | 133 | | -1 0 5 (null) (null) (null) 0 0 46 @testcase | 134 | | 10 20 diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk index cf542310bc..587e16433e 100644 --- a/tolk-tester/tests/type-aliases-tests.tolk +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -8,6 +8,8 @@ type Pair2_v1 = | (int, int) type Pair2_v2 = (MInt, MInt) type MBool = | bool type Tuple2Int = [int, int] +type UserId = int +type OwnerId = int struct Point { x: int; y: int } type PointAlias = Point; @@ -35,9 +37,15 @@ fun test1(x: MInt): MVoid { __expect_type(rand() ? (1, 2) : (1, 2) as Pair2_v1, "(int, int)"); __expect_type(rand() ? (1, 2) : (1, 2) as Pair2_v2, "(int, int)"); __expect_type(rand() ? (1, 2) as Pair2_v1 : (1, 2), "Pair2_v1"); - __expect_type(rand() ? (1, 2) as Pair2_v1 : (1, 2) as Pair2_v2, "(int, int)"); __expect_type(rand() ? (1, 2) as Pair2_v2 : (1, 2) as Pair2_v2, "Pair2_v2"); + // since `Pair2_v1` and `Pair2_v2` underlying type is equal, a union of them can't be created + var d1 = (1, 2) as Pair2_v1 | Pair2_v2; + var d2 = (1, 2) as Pair2_v2 | Pair2_v1; + __expect_type(d1, "Pair2_v1"); + __expect_type(d2, "Pair2_v2"); + __expect_type(rand() ? (1, 2) as Pair2_v1 : (1, 2) as Pair2_v2, "Pair2_v1"); + __expect_type(!x, "bool"); __expect_type(x as int?, "int?"); @@ -63,9 +71,9 @@ fun test3(x: MIntN, y: MInt?) { if (x != null) { __expect_type(x, "MInt"); } - __expect_type(x, "MInt?"); + __expect_type(x, "MIntN"); var (z1, z2) = (x, x!, ); - __expect_type(z1, "MInt?"); + __expect_type(z1, "MIntN"); __expect_type(z2, "MInt"); } @@ -76,9 +84,9 @@ fun test4(x: MIntN, y: MIntN) { __expect_type(x + y, "int"); return x + y; } - __expect_type(x, "MInt?"); + __expect_type(x, "MIntN"); __expect_type(x!, "MInt"); - __expect_type(rand() ? x : y, "MInt?"); + __expect_type(rand() ? x : y, "MIntN"); __expect_type(rand() ? x : 0, "MInt?"); __expect_type(rand() ? 0 : x, "int?"); __expect_type(rand() ? 0 : x!, "int"); @@ -97,8 +105,8 @@ fun test5() { __expect_type(x, "Pair2_v1"); __expect_type(y, "Pair2_v2"); - takeTensor_v1(x); takeTensor_v2(x); takeTensor_v3(x); - takeTensor_v1(y); takeTensor_v2(y); takeTensor_v3(y); + takeTensor_v1(x); takeTensor_v2(x as Pair2_v2); takeTensor_v3(x); + takeTensor_v1(y as Pair2_v1); takeTensor_v2(y); takeTensor_v3(y); takeTensor_v1(z); takeTensor_v2(z); takeTensor_v3(z); var t = (y, x); @@ -117,16 +125,16 @@ fun test6() { __expect_type(z.0, "MInt"); } -fun someFn1(v: MInt): MIntN { return v; } +fun someFn1(v: MInt): MIntN { return v as MIntN; } fun test7() { var f1: (int) -> int? = someFn1; var f2: (int) -> MInt? = someFn1; - var f3: (MInt_v2) -> MInt_v2? = someFn1; + var f3: (MInt_v2) -> MInt_v2? = someFn1 as (MInt_v2) -> MInt_v2?; f1 = f2; f1 = f3; - f2 = f1; f2 = f3; - f3 = f1; f3 = f2; + f2 = f1; f2 = f3 as (int) -> MInt?; + f3 = f1; f3 = f2 as (MInt_v2) -> MInt_v2?; } fun test8() { @@ -157,7 +165,7 @@ fun test10() { var x1: Pair2_v1 = (5, 6); var (a1, b1) = x1; __expect_type(a1, "int"); - var x2: Pair2_v2? = x1; + var x2: Pair2_v2? = x1 as (int, int); var (a2, b2) = x2; __expect_type(a2, "MInt"); var x3: Tuple2Int = [9, 10]; @@ -199,6 +207,188 @@ fun test12(makeNotNull: bool) { return t; } +struct Wrapper { + value: T; +} +type WrapperOfInt = Wrapper +type WrapperOfMInt = Wrapper +type WrapperAlias = Wrapper +type WrapperAliasOfInt = WrapperAlias +type WrapperAliasOfMInt = WrapperAlias + +fun WrapperOfInt.myFn(self): int1 { return 0 } +fun WrapperOfMInt.myFn(self): int2 { return 0 } +fun WrapperAliasOfInt.myFn(self): int3 { return 0 } +fun WrapperAliasOfMInt.myFn(self): int4 { return 0 } + +fun test13(a: WrapperAlias, b: WrapperAlias, c: Wrapper, d: Wrapper, + e: WrapperAliasOfInt, f: WrapperAliasOfMInt, g: WrapperOfInt, h: WrapperOfMInt) { + __expect_type(g.myFn(), "int1"); + __expect_type(h.myFn(), "int2"); + __expect_type(e.myFn(), "int3"); + __expect_type(f.myFn(), "int4"); + + a = a; a = b; a = c; a = d; a = e; + b = a; b = b; b = c; b = d; b = f; + c = a; c = b; c = c; c = d; c = e; c = f; c = g; c = h; + d = a; d = b; d = c; d = d; d = e; d = f; d = g; d = h; + e = a; e = c; e = d; e = e; + f = b; f = c; f = d; f = f; + g = c; g = d; g = g; + h = c; h = d; h = h; + + a as WrapperAlias; + a as WrapperAlias; + a as Wrapper; + a as Wrapper; + a as WrapperOfInt; + a as WrapperOfMInt; + a as WrapperAliasOfInt; + a as WrapperAliasOfMInt; + + match (a) { WrapperAlias => {} } + match (a) { WrapperAlias => {} } + match (a) { Wrapper => {} } + match (a) { Wrapper => {} } + // match (a) { WrapperOfInt => {} } + // match (a) { WrapperOfMInt => {} } + match (a) { WrapperAliasOfInt => {} } + // match (a) { WrapperAliasOfMInt => {} } + + b as WrapperAlias; + b as WrapperAlias; + b as Wrapper; + b as Wrapper; + b as WrapperOfInt; + b as WrapperOfMInt; + b as WrapperAliasOfInt; + b as WrapperAliasOfMInt; + + match (b) { WrapperAlias => {} } + match (b) { WrapperAlias => {} } + match (b) { Wrapper => {} } + match (b) { Wrapper => {} } + // match (b) { WrapperOfInt => {} } + // match (b) { WrapperOfMInt => {} } + // match (b) { WrapperAliasOfInt => {} } + match (b) { WrapperAliasOfMInt => {} } + + match (e) { WrapperAliasOfInt => {} } + + match (f) { WrapperAliasOfMInt => {} } + match (f) { Wrapper => {} } + match (f) { Wrapper => {} } + // match (f) { WrapperAlias => {} } + match (f) { WrapperAlias => {} } + + __expect_type(a, "WrapperAlias"); + __expect_type(b, "WrapperAlias"); + __expect_type(c, "Wrapper"); + __expect_type(d, "Wrapper"); + __expect_type(e, "WrapperAliasOfInt"); + __expect_type(f, "WrapperAliasOfMInt"); + __expect_type(g, "WrapperOfInt"); + __expect_type(h, "WrapperOfMInt"); + + __expect_type(g.myFn(), "int1"); + __expect_type(h.myFn(), "int2"); + __expect_type(e.myFn(), "int3"); + __expect_type(f.myFn(), "int4"); +} + +type Payload14 = MInt | MBool | PointAlias2 + +fun test14(i: MInt, b: MBool, c: PointAlias2) { + var p: Payload14 = i; + p = b; + p = c; + p = i as int; + p = b as bool; + p = c as Point; + p = c as PointAlias; +} + +type MInt1 = int +type MInt2 = MInt1 + +fun int.plus0(self) { return self + 0 } +fun MInt1.plus1(self) { return self + 1 } +fun MInt2.plus2(self) { return self + 2 } + +@method_id(115) +fun test15() { + var i: int = 0; + var m: MInt1 = 10; + var t: MInt2 = 20; + + if (0) { + i = m; i = t; + m = i; m = t; + t = i; t = m; + } + + return ( + i.plus0(), i.plus1(), i.plus2(), + m.plus0(), m.plus1(), m.plus2(), + t.plus0(), t.plus1(), t.plus2(), + ) +} + +type StrangeUnionInt = int | UserId +type StrangeUnionInt2 = UserId | OwnerId + +struct ManyUnionFields { + f1: int | UserId + f2: StrangeUnionInt + f3: UserId | OwnerId + f4: OwnerId | UserId + f5: int | StrangeUnionInt2 + f6: StrangeUnionInt2 + + f10: slice | int | UserId + f11: slice | UserId | int + f12: OwnerId | () | int | UserId | slice + f13: OwnerId | cell | StrangeUnionInt2? +} + +fun test16(a: ManyUnionFields) { + __expect_type(a.f1, "int"); + __expect_type(a.f2, "StrangeUnionInt"); + __expect_type(a.f3, "UserId"); + __expect_type(a.f5, "int"); + __expect_type(a.f6, "StrangeUnionInt2"); + a.f1 + a.f2 + a.f3 + a.f4 + a.f5 + a.f6; + + __expect_type(a.f10, "slice | int"); + __expect_type(a.f11, "slice | UserId"); + __expect_type(a.f12, "OwnerId | () | slice"); + __expect_type(a.f13, "OwnerId | cell | null"); + + match (a.f13) { + int => {} + cell => {} + null => {} + } +} + +type SnakedCell = cell + +fun test17(a: SnakedCell, b: SnakedCell, c: SnakedCell) { + a = a; a = b; a = c; + b = a; + c = a; + + a as SnakedCell; + a as SnakedCell; + a as SnakedCell; + b as SnakedCell; + b as SnakedCell; + b as SnakedCell; + c as SnakedCell; + c as SnakedCell; + c as SnakedCell; +} + fun main(x: MInt, y: MInt?) { return y == null ? x : x + y; @@ -210,6 +400,7 @@ fun main(x: MInt, y: MInt?) { @testcase | 110 | | 41 @testcase | 112 | 0 | [ 1 1 ] @testcase | 112 | -1 | [ 2 1 ] +@testcase | 115 | | 0 1 2 10 11 12 20 21 22 @fif_codegen """ diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 7b9d52b61e..0f649f75ef 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -222,7 +222,7 @@ fun test21() { __expect_type(a, "never"); } if (a is slice || a is int) { - __expect_type(a, "slice | int") + __expect_type(a, "int | slice") } else { __expect_type(a, "never") } @@ -254,12 +254,12 @@ fun test23(p2: bool) { if (p is (int, int) && 10 < 0) { __expect_type(p, "(int, int)"); } - __expect_type(p, "(int, int) | (int, int, int)"); + __expect_type(p, "Pair2Or3"); if (p is int) { __expect_type(p, "never"); return (0, 0); } - __expect_type(p, "(int, int) | (int, int, int)"); + __expect_type(p, "Pair2Or3"); if (p !is (int, int)) { __expect_type(p, "(int, int, int)"); p = (10, 20); @@ -552,7 +552,7 @@ fun test48(a: IntOrInt8OrInt16) { var b = a; __expect_type(b, "IntOrInt8OrInt16"); match (b) { MInt => {} int8 => {} int16 => {} } - __expect_type(b, "int | int8 | int16"); + __expect_type(b, "IntOrInt8OrInt16"); match (b) { int => {}, int8 => { return ((0, null), (0, null)); }, int16 => "" }; __expect_type(b, "int | int16"); return (checkGeneric3(a), checkGeneric3(5 as int | slice | builder)); @@ -579,10 +579,10 @@ fun test50() { test49(match (a) { int => a, slice => a, null => a, }); __expect_type(b, "VeryComplexType"); match (b) { int => {} Pair2 => {} builder => {} null => {} } - __expect_type(b, "int | (int, int) | builder | null"); - __expect_type(b!, "int | (int, int) | builder"); + __expect_type(b, "VeryComplexType"); + __expect_type(b!, "int | (MInt, int) | builder"); match (b) { int => 100, Pair2 => {} builder => return -2, null => "null" } - __expect_type(b, "int | (int, int) | null"); + __expect_type(b, "int | Pair2 | null"); if (b !is null) { match (b) { int => {}, Pair2 => {} } } else { @@ -796,7 +796,7 @@ fun main() { @testcase | 107 | 3 | 6 7 8 typeid-2 6 7 8 typeid-2 @testcase | 108 | | (null) 2 3 typeid-1 (null) (null) (null) 0 (null) 2 3 typeid-1 (null) 2 3 typeid-1 (null) (null) (null) 0 @testcase | 109 | | 6 7 8 typeid-2 0 (null) -1 (null) (null) (null) 0 -1 (null) (null) (null) 0 -@testcase | 110 | | 1 (null) (null) 0 (null) (null) 0 typeid-7 +@testcase | 110 | | 1 (null) (null) 0 (null) (null) 0 typeid-3 @testcase | 120 | | 5 @testcase | 121 | | -1 0 0 -1 -1 0 @testcase | 122 | 0 0 1 | 36 @@ -810,19 +810,19 @@ fun main() { @testcase | 127 | | 1 2 5 1 @testcase | 128 | | (null) (null) 0 777 (null) 2 1 777 3 4 typeid-1 @testcase | 129 | | (null) (null) 0 (null) 5 1 -@testcase | 130 | 0 | (null) 5 1 777 (null) (null) 0 777 1 2 typeid-1 777 (null) [ 6 ] typeid-8 777 (null) (null) 0 -@testcase | 130 | -1 | (null) (null) 0 777 (null) (null) 0 777 1 2 typeid-1 777 (null) [ 6 ] typeid-8 777 (null) (null) 0 +@testcase | 130 | 0 | (null) 5 1 777 (null) (null) 0 777 1 2 typeid-1 777 (null) [ 6 ] typeid-4 777 (null) (null) 0 +@testcase | 130 | -1 | (null) (null) 0 777 (null) (null) 0 777 1 2 typeid-1 777 (null) [ 6 ] typeid-4 777 (null) (null) 0 @testcase | 131 | | (null) 777 5 777 1 2 777 (null) 777 5 777 (null) (null) 0 777 1 2 typeid-1 @testcase | 132 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 typeid-1 777 (null) (null) (null) 0 @testcase | 133 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 typeid-1 777 (null) (null) (null) 0 @testcase | 134 | | (null) 5 1 777 (null) 5 1 777 1 2 typeid-1 777 1 2 typeid-1 777 (null) 5 42 777 5 42 777 5 42 @testcase | 135 | | (null) 5 1 777 (null) (null) 1 2 typeid-1 777 (null) (null) 1 2 typeid-1 777 (null) (null) 0 -@testcase | 136 | | 1 2 typeid-1 777 1 2 typeid-12 777 1 2 typeid-13 777 (null) 1 2 typeid-14 +@testcase | 136 | | 1 2 typeid-1 777 1 2 typeid-5 777 1 2 typeid-6 777 (null) 1 2 typeid-7 @testcase | 137 | 1 2 | 1 2 777 1 2 777 1 2 777 [ 1 2 ] -@testcase | 138 | | 1 (null) (null) (null) (null) 0 2 typeid-18 777 1 (null) 0 2 typeid-19 777 1 5 1 2 typeid-19 777 1 (null) 5 1 2 typeid-20 +@testcase | 138 | | 1 (null) (null) (null) (null) 0 2 typeid-9 777 1 (null) 0 2 typeid-10 777 1 5 1 2 typeid-10 777 1 (null) 5 1 2 typeid-11 @testcase | 139 | | 1 (null) (null) (null) (null) 0 2 777 1 (null) 0 2 777 1 5 1 2 777 1 (null) 5 1 2 @testcase | 140 | 5 | 5 42 -@testcase | 140 | 15 | 15 typeid-3 +@testcase | 140 | 15 | 15 typeid-12 @testcase | 140 | 90 | 90 49 @testcase | 141 | 7 8 | 7 7 1 8 8 1 @testcase | 141 | null null | (null) (null) 0 (null) (null) 0 @@ -843,9 +843,9 @@ fun main() { @testcase | 154 | | 100 1 @testcase | 155 | | 5 1 5 1 @testcase | 156 | 1 2 -1 | 2 44 2 2 44 2 2 44 2 44 2 2 2 -@testcase | 157 | 1 | (null) 0 (null) typeid-21 777 1 2 typeid-1 777 typeid-21 typeid-21 777 0 0 -1 0 -1 0 0 777 -1 0 -@testcase | 157 | 0 | (null) 0 (null) typeid-21 777 (null) (null) typeid-21 777 0 0 777 0 0 -1 0 0 -1 0 777 0 -1 -@testcase | 158 | | (null) (null) typeid-21 (null) (null) typeid-21 (null) 0 (null) +@testcase | 157 | 1 | (null) 0 (null) typeid-14 777 1 2 typeid-1 777 typeid-14 typeid-14 777 0 0 -1 0 -1 0 0 777 -1 0 +@testcase | 157 | 0 | (null) 0 (null) typeid-14 777 (null) (null) typeid-14 777 0 0 777 0 0 -1 0 0 -1 0 777 0 -1 +@testcase | 158 | | (null) (null) typeid-14 (null) (null) typeid-14 (null) 0 (null) @testcase | 159 | 0 4 | 456 @testcase | 159 | null 0 | 123 @testcase | 160 | | 10 0 diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index 702c9fddc9..8bd23fe053 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -350,7 +350,7 @@ fun main() {} @testcase | 107 | | 65537 @testcase | 108 | | 4 @testcase | 109 | | 0 3 0 7 -@testcase | 110 | 5 | 5 10 100 100 100 typeid-6 +@testcase | 110 | 5 | 5 10 100 100 100 typeid-2 @testcase | 110 | 0 | (null) (null) (null) (null) (null) 0 @testcase | 111 | 2 3 9 | -1 @testcase | 111 | 11 22 44 | -1 @@ -360,7 +360,7 @@ fun main() {} @testcase | 112 | -1 -10 -20 | -1 @testcase | 113 | 0 | 12 12 @testcase | 113 | -1 | 12 -1 -@testcase | 114 | -1 | 2 3 typeid-3 +@testcase | 114 | -1 | 2 3 typeid-1 @testcase | 114 | 0 | (null) 12 1 @testcase | 115 | | 7 6 7 @testcase | 116 | | 7 11 80 diff --git a/tolk-tester/tolk-tester.js b/tolk-tester/tolk-tester.js index dcd667e6ec..ae07a53b8f 100644 --- a/tolk-tester/tolk-tester.js +++ b/tolk-tester/tolk-tester.js @@ -279,7 +279,7 @@ class TolkTestFile { /** @type {boolean} */ this.enable_tolk_lines_comments = false /** @type {number} */ - this.pivot_typeid = 138 // may be changed when stdlib introduces new union types + this.pivot_typeid = 128 } parse_input_from_tolk_file() { diff --git a/tolk-tester/tolk-tester.py b/tolk-tester/tolk-tester.py index da81f87fe1..4fba75448a 100644 --- a/tolk-tester/tolk-tester.py +++ b/tolk-tester/tolk-tester.py @@ -266,7 +266,7 @@ def __init__(self, tolk_filename: str, artifacts_folder: str): self.expected_hash: TolkTestCaseExpectedHash | None = None self.experimental_options: str | None = None self.enable_tolk_lines_comments = False - self.pivot_typeid = 138 # may be changed when stdlib introduces new union types + self.pivot_typeid = 128 def parse_input_from_tolk_file(self): with open(self.tolk_filename, "r") as fd: diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 39cef55b7c..b9c89bdcae 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -1122,7 +1122,7 @@ static V parse_block_statement(Lexer& lex) { static AnyV parse_return_statement(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.expect(tok_return, "`return`"); - AnyExprV child = lex.tok() == tok_semicolon // `return;` actually means "nothing" (inferred as void) + AnyExprV child = lex.tok() == tok_semicolon || lex.tok() == tok_clbrace ? createV(lex.cur_location()) : parse_expr(lex); return createV(loc, child); diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index ae51cf7521..4726db9f85 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -184,7 +184,9 @@ void GenericSubstitutionsDeducing::consider_next_condition(TypePtr param_type, T bool is_sub_correct = true; for (TypePtr p_variant : p_union->variants) { if (!p_variant->has_genericT_inside()) { - auto it = std::find(a_sub_p.begin(), a_sub_p.end(), p_variant); + auto it = std::find_if(a_sub_p.begin(), a_sub_p.end(), [p_variant](TypePtr a) { + return a->equal_to(p_variant); + }); if (it != a_sub_p.end()) { a_sub_p.erase(it); } else { @@ -210,7 +212,7 @@ void GenericSubstitutionsDeducing::consider_next_condition(TypePtr param_type, T } } else if (const auto* p_instSt = param_type->try_as(); p_instSt && p_instSt->struct_ref) { // `arg: Wrapper` called as `f(wrappedInt)` => T is int - if (const auto* a_struct = arg_type->try_as(); a_struct && a_struct->struct_ref->is_instantiation_of_generic_struct() && a_struct->struct_ref->base_struct_ref == p_instSt->struct_ref) { + if (const auto* a_struct = arg_type->unwrap_alias()->try_as(); a_struct && a_struct->struct_ref->is_instantiation_of_generic_struct() && a_struct->struct_ref->base_struct_ref == p_instSt->struct_ref) { tolk_assert(p_instSt->size() == a_struct->struct_ref->substitutedTs->size()); for (int i = 0; i < p_instSt->size(); ++i) { consider_next_condition(p_instSt->type_arguments[i], a_struct->struct_ref->substitutedTs->typeT_at(i)); diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index 5f1ebcc8f5..31df5e9665 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -210,7 +210,7 @@ void UnpackContext::throwInvalidOpcode() const { const LazyMatchOptions::MatchBlock* LazyMatchOptions::find_match_block(TypePtr variant) const { for (const MatchBlock& b : match_blocks) { - if (b.arm_variant->get_type_id() == variant->get_type_id()) { + if (b.arm_variant->equal_to(variant)) { return &b; } } diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 9cfc5ee49a..068403ce15 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -579,7 +579,7 @@ std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, Ty FunctionPtr not_sym = lookup_function("!b_"); std::vector result_ir_idx = code.create_tmp_var(TypeDataBool::create(), loc, debug_desc); - const TypeDataUnion* lhs_union = expr_type->try_as(); + const TypeDataUnion* lhs_union = expr_type->unwrap_alias()->try_as(); if (!lhs_union) { // `int` is `int` / `int` is `builder`, it's compile-time, either 0, or -1 bool types_eq = expr_type->get_type_id() == cmp_type->get_type_id(); @@ -720,12 +720,12 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector(rvect.size()) == original_type->get_width_on_stack()); #endif + // aliases are erased at the TVM level original_type = original_type->unwrap_alias(); target_type = target_type->unwrap_alias(); // pass `T` to `T` - // could occur for passing tensor `(..., T, ...)` to `(..., T, ...)` while traversing tensor's components - if (target_type == original_type) { + if (target_type->equal_to(original_type)) { return rvect; } @@ -768,7 +768,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectoris_primitive_nullable() && orig_w > 0) { // nothing except "T1 | T2 | ... null" can be cast to 1-slot nullable `T1?` - tolk_assert(o_union && o_union->has_null() && o_union->has_variant_with_type_id(t_union->or_null)); + tolk_assert(o_union && o_union->has_null() && o_union->has_variant_equal_to(t_union->or_null)); // here we exploit rvect shape, how union types and multi-slot nullables are stored on a stack // `T1 | T2 | ... | null` occupies N+1 slots, where the last is for UTag // when it holds null value, N slots are null, and UTag slot is 0 (it's type_id of TypeDataNullLiteral) @@ -834,7 +834,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectoris_primitive_nullable() && t_union) { - tolk_assert(t_union->has_null() && t_union->has_variant_with_type_id(o_union->or_null) && target_w > 1); + tolk_assert(t_union->has_null() && t_union->has_variant_equal_to(o_union->or_null) && target_w > 1); // the transformation is tricky: // when value is null, we need to achieve "... (null) 0" (value is already null, so "... value 0") // when value is not null, we need to get "... value {type_id}" @@ -1313,8 +1313,8 @@ static std::vector process_cast_as_operator(V v } static std::vector process_is_type_operator(V v, CodeBlob& code, TypePtr target_type) { - TypePtr lhs_type = v->get_expr()->inferred_type->unwrap_alias(); - TypePtr cmp_type = v->type_node->resolved_type->unwrap_alias(); + TypePtr lhs_type = v->get_expr()->inferred_type; + TypePtr cmp_type = v->type_node->resolved_type; bool is_null_check = cmp_type == TypeDataNullLiteral::create(); // `v == null`, not `v is T` tolk_assert(!cmp_type->try_as()); // `v is int|slice` is a type checker error @@ -1329,7 +1329,7 @@ static std::vector process_is_type_operator(V v } static std::vector process_not_null_operator(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { - TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); + TypePtr expr_type = v->get_expr()->inferred_type; TypePtr without_null_type = calculate_type_subtract_rhs_type(expr_type, TypeDataNullLiteral::create()); TypePtr child_target_type = without_null_type != TypeDataNever::create() ? without_null_type : expr_type; @@ -1390,7 +1390,7 @@ static std::vector process_lazy_operator(V v, Code } static std::vector process_match_expression(V v, CodeBlob& code, TypePtr target_type) { - TypePtr lhs_type = v->get_subject()->inferred_type->unwrap_alias(); + TypePtr lhs_type = v->get_subject()->inferred_type; int n_arms = v->get_arms_count(); std::vector subj_ir_idx = pre_compile_expr(v->get_subject(), code, nullptr); @@ -1427,8 +1427,8 @@ static std::vector process_match_expression(V v auto v_ith_arm = v->get_arm(i); std::vector eq_ith_ir_idx; if (is_match_by_type) { - TypePtr cmp_type = v_ith_arm->pattern_type_node->resolved_type->unwrap_alias(); - tolk_assert(!cmp_type->try_as()); // `match` over `int|slice` is a type checker error + TypePtr cmp_type = v_ith_arm->pattern_type_node->resolved_type; + tolk_assert(!cmp_type->unwrap_alias()->try_as()); // `match` over `int|slice` is a type checker error eq_ith_ir_idx = pre_compile_is_type(code, lhs_type, cmp_type, subj_ir_idx, v_ith_arm->loc, "(arm-cond-eq)"); } else { std::vector ith_ir_idx = pre_compile_expr(v_ith_arm->get_pattern_expr(), code); @@ -1946,7 +1946,7 @@ static std::vector process_artificial_aux_vertex(Vget_arm(i); TypePtr arm_variant = nullptr; if (v_arm->pattern_kind == MatchArmKind::exact_type) { - arm_variant = v_arm->pattern_type_node->resolved_type->unwrap_alias(); + arm_variant = v_arm->pattern_type_node->resolved_type; } else { tolk_assert(v_arm->pattern_kind == MatchArmKind::else_branch); // `else` allowed in a lazy match } diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 97a953f4f6..7880149c84 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -562,10 +562,10 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { bool has_expr_arm = false; bool has_else_arm = false; AnyExprV v_subject = v->get_subject(); - TypePtr subject_type = v_subject->inferred_type->unwrap_alias(); - const TypeDataUnion* subject_union = subject_type->try_as(); + TypePtr subject_type = v_subject->inferred_type; + const TypeDataUnion* subject_union = subject_type->unwrap_alias()->try_as(); - std::vector covered_type_ids; // for type-based `match`, what types are on the left of `=>` + std::vector covered_types; // for type-based `match`, what types are on the left of `=>` for (int i = 0; i < v->get_arms_count(); ++i) { auto v_arm = v->get_arm(i); @@ -579,19 +579,22 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } has_type_arm = true; - TypePtr lhs_type = v_arm->pattern_type_node->resolved_type->unwrap_alias(); // `lhs_type => ...` - if (lhs_type->try_as()) { + TypePtr lhs_type = v_arm->pattern_type_node->resolved_type; // `lhs_type => ...` + if (lhs_type->unwrap_alias()->try_as()) { fire(cur_f, v_arm->loc, "wrong pattern matching: union types are not allowed, use concrete types in `match`"); } - bool can_happen = (subject_union && subject_union->has_variant_with_type_id(lhs_type)) || + bool can_happen = (subject_union && subject_union->has_variant_equal_to(lhs_type)) || (!subject_union && subject_type->equal_to(lhs_type)); if (!can_happen) { fire(cur_f, v_arm->loc, "wrong pattern matching: " + to_string(lhs_type) + " is not a variant of " + to_string(subject_type)); } - if (std::find(covered_type_ids.begin(), covered_type_ids.end(), lhs_type->get_type_id()) != covered_type_ids.end()) { + auto it_mentioned = std::find_if(covered_types.begin(), covered_types.end(), [lhs_type](TypePtr existing) { + return existing->equal_to(lhs_type); + }); + if (it_mentioned != covered_types.end()) { fire(cur_f, v_arm->loc, "wrong pattern matching: duplicated " + to_string(lhs_type)); } - covered_type_ids.push_back(lhs_type->get_type_id()); + covered_types.push_back(lhs_type); break; } case MatchArmKind::const_expression: { @@ -602,7 +605,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { fire(cur_f, v_arm->loc, "`else` branch should be the last"); } has_expr_arm = true; - TypePtr pattern_type = v_arm->get_pattern_expr()->inferred_type->unwrap_alias(); + TypePtr pattern_type = v_arm->get_pattern_expr()->inferred_type; bool both_int = expect_integer(pattern_type) && expect_integer(subject_type); bool both_bool = expect_boolean(pattern_type) && expect_boolean(subject_type); if (!both_int && !both_bool) { @@ -627,11 +630,19 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } + // only `else` branch + if (has_else_arm && !has_type_arm && !has_expr_arm) { + fire(cur_f, v->loc, "`match` contains only `else`, but no variants"); + } + // fire if `match` by type is not exhaustive - if (has_type_arm && subject_union && subject_union->variants.size() != covered_type_ids.size()) { + if (has_type_arm && subject_union && subject_union->variants.size() != covered_types.size()) { std::string missing; for (TypePtr variant : subject_union->variants) { - if (std::find(covered_type_ids.begin(), covered_type_ids.end(), variant->get_type_id()) == covered_type_ids.end()) { + auto it_mentioned = std::find_if(covered_types.begin(), covered_types.end(), [variant](TypePtr existing) { + return existing->equal_to(variant); + }); + if (it_mentioned == covered_types.end()) { if (!missing.empty()) { missing += ", "; } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 1bca8908ad..5c10d44d59 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -786,7 +786,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } rhs_type = rhs_type->unwrap_alias(); - TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); + TypePtr expr_type = v->get_expr()->inferred_type; TypePtr non_rhs_type = calculate_type_subtract_rhs_type(expr_type, rhs_type); if (expr_type->equal_to(rhs_type)) { // `expr is ` is always true v->mutate()->assign_always_true_or_false(v->is_negated ? 2 : 1); @@ -823,7 +823,7 @@ class InferTypesAndCallsAndFieldsVisitor final { ExprFlow infer_not_null_operator(V v, FlowContext&& flow, bool used_as_condition) { ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); TypePtr expr_type = v->get_expr()->inferred_type; - TypePtr without_null_type = calculate_type_subtract_rhs_type(expr_type->unwrap_alias(), TypeDataNullLiteral::create()); + TypePtr without_null_type = calculate_type_subtract_rhs_type(expr_type, TypeDataNullLiteral::create()); assign_inferred_type(v, without_null_type != TypeDataNever::create() ? without_null_type : expr_type); if (!used_as_condition) { @@ -907,8 +907,8 @@ class InferTypesAndCallsAndFieldsVisitor final { // - or inside a call: `globalF()` / `genericFn()` / `genericFn()` / `local_var()` if (LocalVarPtr var_ref = v->sym->try_as()) { - TypePtr declared_or_smart_casted = flow.smart_cast_if_exists(SinkExpression(var_ref)); - tolk_assert(declared_or_smart_casted != nullptr); // all local vars are presented in flow + tolk_assert(flow.smart_cast_exists(SinkExpression(var_ref))); // all local vars are presented in flow + TypePtr declared_or_smart_casted = flow.smart_cast_or_original(SinkExpression(var_ref), var_ref->declared_type); assign_inferred_type(v, declared_or_smart_casted); if (var_ref->is_lateinit() && declared_or_smart_casted == TypeDataUnknown::create() && v->is_rvalue) { fire_error_using_lateinit_variable_uninitialized(cur_f, v->loc, v->get_name()); @@ -1010,9 +1010,7 @@ class InferTypesAndCallsAndFieldsVisitor final { v->mutate()->assign_target(field_ref); TypePtr inferred_type = field_ref->declared_type; if (SinkExpression s_expr = extract_sink_expression_from_vertex(v)) { - if (TypePtr smart_casted = flow.smart_cast_if_exists(s_expr)) { - inferred_type = smart_casted; - } + inferred_type = flow.smart_cast_or_original(s_expr, inferred_type); } assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); @@ -1033,9 +1031,7 @@ class InferTypesAndCallsAndFieldsVisitor final { v->mutate()->assign_target(index_at); TypePtr inferred_type = t_tensor->items[index_at]; if (SinkExpression s_expr = extract_sink_expression_from_vertex(v)) { - if (TypePtr smart_casted = flow.smart_cast_if_exists(s_expr)) { - inferred_type = smart_casted; - } + inferred_type = flow.smart_cast_or_original(s_expr, inferred_type); } assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); @@ -1047,9 +1043,7 @@ class InferTypesAndCallsAndFieldsVisitor final { v->mutate()->assign_target(index_at); TypePtr inferred_type = t_tuple->items[index_at]; if (SinkExpression s_expr = extract_sink_expression_from_vertex(v)) { - if (TypePtr smart_casted = flow.smart_cast_if_exists(s_expr)) { - inferred_type = smart_casted; - } + inferred_type = flow.smart_cast_or_original(s_expr, inferred_type); } assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); @@ -1185,7 +1179,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (param_type->has_genericT_inside()) { param_type = deducingTs.auto_deduce_from_argument(cur_f, self_obj->loc, param_type, self_obj->inferred_type); } - if (param_0.is_mutate_parameter() && self_obj->inferred_type->unwrap_alias() != param_type->unwrap_alias()) { + if (param_0.is_mutate_parameter() && !self_obj->inferred_type->equal_to(param_type)) { if (SinkExpression s_expr = extract_sink_expression_from_vertex(self_obj)) { assign_inferred_type(self_obj, calc_declared_type_before_smart_cast(self_obj)); flow.register_known_type(s_expr, param_type); @@ -1212,7 +1206,7 @@ class InferTypesAndCallsAndFieldsVisitor final { param_type = deducingTs.auto_deduce_from_argument(cur_f, arg_i->loc, param_type, arg_i->inferred_type); } assign_inferred_type(v->get_arg(i), arg_i); // arg itself is an expression - if (param_i.is_mutate_parameter() && arg_i->inferred_type->unwrap_alias() != param_type->unwrap_alias()) { + if (param_i.is_mutate_parameter() && !arg_i->inferred_type->equal_to(param_type)) { if (SinkExpression s_expr = extract_sink_expression_from_vertex(arg_i)) { assign_inferred_type(arg_i, calc_declared_type_before_smart_cast(arg_i)); flow.register_known_type(s_expr, param_type); diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 46710cb639..e0f448f2d1 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -64,6 +64,15 @@ static void fire_error_type_used_as_symbol(FunctionPtr cur_f, V } } +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_using_self_not_in_method(FunctionPtr cur_f, SrcLocation loc) { + if (cur_f->is_static_method()) { + fire(cur_f, loc, "using `self` in a static method"); + } else { + fire(cur_f, loc, "using `self` in a regular function (not a method)"); + } +} + struct NameAndScopeResolver { std::vector> scopes; @@ -174,9 +183,12 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { try { parent::visit(v->get_obj()); } catch (const ParseError&) { - if (v->get_obj()->kind == ast_reference) { + if (auto v_type_name = v->get_obj()->try_as()) { // for `Point.create` / `int.zero`, "undefined symbol" is fired for Point/int // suppress this exception till a later pipe, it will be tried to be resolved as a type + if (v_type_name->get_identifier()->name == "self") { + fire_error_using_self_not_in_method(cur_f, v_type_name->loc); + } return; } throw; diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index 7dd60ee975..1bcbe546b2 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -204,13 +204,6 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b, bool* became_union = nul return TypeDataBrackets::create(std::move(types_lca)); } - if (const auto* a_alias = a->try_as()) { - return calculate_type_lca(a_alias->underlying_type, b, became_union); - } - if (const auto* b_alias = b->try_as()) { - return calculate_type_lca(a, b_alias->underlying_type, became_union); - } - TypePtr resulting_union = TypeDataUnion::create(std::vector{a, b}); if (became_union != nullptr && !a->equal_to(resulting_union) && !b->equal_to(resulting_union)) { *became_union = true; @@ -314,6 +307,22 @@ void FlowContext::invalidate_all_subfields(LocalVarPtr var_ref, uint64_t parent_ } } +// get the resulting type of variable or struct field +TypePtr FlowContext::smart_cast_or_original(SinkExpression s_expr, TypePtr originally_declared_type) const { + auto it = known_facts.find(s_expr); + if (it == known_facts.end()) { + return originally_declared_type; + } + + TypePtr smart_casted = it->second.expr_type; + if (smart_casted->equal_to(originally_declared_type)) { + // given `var a: dict`, after merging control flow branches, restore `a: dict` instead of `a: cell?` + // (same for struct fields and other sink expressions) + return originally_declared_type; + } + return smart_casted; +} + // update current type of `local_var` / `tensorVar.0` / `obj.field` // example: `local_var = rhs` // example: `f(mutate obj.field)` @@ -386,26 +395,26 @@ FlowContext FlowContext::merge_flow(FlowContext&& c1, FlowContext&& c2) { // example: `int | slice | builder | bool` - `bool | slice` = `int | builder` // what for: `if (x != null)` / `if (x is T)`, to smart cast x inside if TypePtr calculate_type_subtract_rhs_type(TypePtr type, TypePtr subtract_type) { - const TypeDataUnion* lhs_union = type->try_as(); + const TypeDataUnion* lhs_union = type->unwrap_alias()->try_as(); if (!lhs_union) { return TypeDataNever::create(); } std::vector rest_variants; - if (const TypeDataUnion* sub_union = subtract_type->try_as()) { + if (const TypeDataUnion* sub_union = subtract_type->unwrap_alias()->try_as()) { if (lhs_union->has_all_variants_of(sub_union)) { rest_variants.reserve(lhs_union->size() - sub_union->size()); for (TypePtr lhs_variant : lhs_union->variants) { - if (!sub_union->has_variant_with_type_id(lhs_variant)) { + if (!sub_union->has_variant_equal_to(lhs_variant)) { rest_variants.push_back(lhs_variant); } } } - } else if (lhs_union->has_variant_with_type_id(subtract_type)) { + } else if (lhs_union->has_variant_equal_to(subtract_type)) { rest_variants.reserve(lhs_union->size() - 1); for (TypePtr lhs_variant : lhs_union->variants) { - if (lhs_variant->get_type_id() != subtract_type->get_type_id()) { + if (!lhs_variant->equal_to(subtract_type)) { rest_variants.push_back(lhs_variant); } } @@ -519,14 +528,10 @@ TypePtr calc_smart_cast_type_on_assignment(TypePtr lhs_declared_type, TypePtr rh // example: `var x: int | slice | cell = 4`, result is int // example: `var x: T1 | T2 | T3 = y as T3 | T1`, result is `T1 | T3` if (const TypeDataUnion* rhs_union = rhs_inferred_type->try_as()) { - bool lhs_has_all_variants_of_rhs = true; - for (TypePtr rhs_variant : rhs_union->variants) { - lhs_has_all_variants_of_rhs &= lhs_union->has_variant_with_type_id(rhs_variant); - } - if (lhs_has_all_variants_of_rhs && rhs_union->size() < lhs_union->size()) { + if (lhs_union->has_all_variants_of(rhs_union) && rhs_union->size() < lhs_union->size()) { std::vector subtypes_of_lhs; for (TypePtr lhs_variant : lhs_union->variants) { - if (rhs_union->has_variant_with_type_id(lhs_variant)) { + if (rhs_union->has_variant_equal_to(lhs_variant)) { subtypes_of_lhs.push_back(lhs_variant); } } diff --git a/tolk/smart-casts-cfg.h b/tolk/smart-casts-cfg.h index b579e8f31d..1fb491d8f3 100644 --- a/tolk/smart-casts-cfg.h +++ b/tolk/smart-casts-cfg.h @@ -158,10 +158,8 @@ class FlowContext { bool is_unreachable() const { return unreachable; } - TypePtr smart_cast_if_exists(SinkExpression s_expr) const { - auto it = known_facts.find(s_expr); - return it == known_facts.end() ? nullptr : it->second.expr_type; - } + bool smart_cast_exists(SinkExpression s_expr) const { return known_facts.find(s_expr) != known_facts.end(); } + TypePtr smart_cast_or_original(SinkExpression s_expr, TypePtr originally_declared_type) const; void register_known_type(SinkExpression s_expr, TypePtr assigned_type); void mark_unreachable(UnreachableKind reason); diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index cf89edb0b7..b72da00dab 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -24,109 +24,58 @@ namespace tolk { /* - * This class stores a big hashtable [hash => TypePtr] - * Every non-trivial TypeData*::create() method at first looks here, and allocates an object only if not found. - * That's why all allocated TypeData objects are unique, and can be compared as pointers. - * But compare pointers carefully due to type aliases. + * Every TypeData has children_flags (just a mask of all children), + * here we have an utility to calculate them at creation. */ -class TypeDataHasherForUnique { - uint64_t cur_hash; +class CalcChildrenFlags { int children_flags_mask = 0; - static std::unordered_map all_unique_occurred_types; - public: - explicit TypeDataHasherForUnique(uint64_t initial_arbitrary_unique_number) - : cur_hash(initial_arbitrary_unique_number) {} - - void feed_hash(uint64_t val) { - cur_hash = cur_hash * 56235515617499ULL + val; - } - - void feed_string(const std::string& s) { - feed_hash(std::hash{}(s)); - } - void feed_child(TypePtr inner) { - feed_hash(reinterpret_cast(inner)); children_flags_mask |= inner->flags; } - int children_flags() const { - return children_flags_mask; - } - - GNU_ATTRIBUTE_FLATTEN - TypePtr get_existing() const { - auto it = all_unique_occurred_types.find(cur_hash); - return it != all_unique_occurred_types.end() ? it->second : nullptr; + void feed_child(const std::vector& children) { + for (TypePtr inner : children) { + children_flags_mask |= inner->flags; + } } - GNU_ATTRIBUTE_NOINLINE - TypePtr register_unique(TypePtr newly_created) const { -#ifdef TOLK_DEBUG - assert(all_unique_occurred_types.find(cur_hash) == all_unique_occurred_types.end()); -#endif - all_unique_occurred_types[cur_hash] = newly_created; - return newly_created; + int children_flags() const { + return children_flags_mask; } }; /* * This class stores a hashtable [TypePtr => type_id] * We need type_id to support union types, that are stored as tagged unions on a stack. - * Every type that can be contained inside a union, has type_id. + * Every type actually contained inside a union, has type_id. * Some type_id are predefined (1 = int, etc.), but all user-defined types are assigned type_id. */ class TypeIdCalculation { static int last_type_id; static std::unordered_map map_ptr_to_type_id; - static std::vector instantiated_structs; public: static int assign_type_id(TypePtr self) { - if (self->has_type_alias_inside()) { // type_id is calculated without aliases - self = unwrap_type_alias_deeply(self); // `(int,int)` equals `(IntAlias,IntAlias)`. - } - if (auto it = map_ptr_to_type_id.find(self); it != map_ptr_to_type_id.end()) { + // type_id is calculated without aliases, based on "equal to"; + // for instance, `UserId` / `OwnerId` / `int` will have the same type_id without any runtime conversion + auto it = std::find_if(map_ptr_to_type_id.begin(), map_ptr_to_type_id.end(), [self](std::pair existing) { + return existing.first->equal_to(self); + }); + if (it != map_ptr_to_type_id.end()) { return it->second; } - // make `Wrapper>` and `Wrapper>` and `Wrapper` have equal type_id - if (const TypeDataStruct* t_struct = self->try_as(); t_struct && t_struct->struct_ref->is_instantiation_of_generic_struct()) { - StructPtr struct_ref = t_struct->struct_ref; - instantiated_structs.push_back(struct_ref); - for (StructPtr another_ref : instantiated_structs) { - if (struct_ref != another_ref && self->equal_to(TypeDataStruct::create(another_ref))) { - auto it = map_ptr_to_type_id.find(TypeDataStruct::create(another_ref)); - assert(it != map_ptr_to_type_id.end()); - int type_id = it->second; - map_ptr_to_type_id[self] = type_id; - return type_id; - } - } - } - int type_id = ++last_type_id; map_ptr_to_type_id[self] = type_id; return type_id; } - - static TypePtr unwrap_type_alias_deeply(TypePtr type) { - return type->replace_children_custom([](TypePtr child) { - if (const TypeDataAlias* as_alias = child->try_as()) { - return as_alias->underlying_type->unwrap_alias(); - } - return child; - }); - } }; int TypeIdCalculation::last_type_id = 128; // below 128 reserved for built-in types -std::unordered_map TypeDataHasherForUnique::all_unique_occurred_types; std::unordered_map TypeIdCalculation::map_ptr_to_type_id; -std::vector TypeIdCalculation::instantiated_structs; TypePtr TypeDataInt::singleton; TypePtr TypeDataBool::singleton; @@ -159,31 +108,6 @@ void type_system_init() { } -bool TypeData::equal_to_slow_path(TypePtr lhs, TypePtr rhs) { - if (lhs->has_type_alias_inside()) { - lhs = TypeIdCalculation::unwrap_type_alias_deeply(lhs); - } - if (rhs->has_type_alias_inside()) { - rhs = TypeIdCalculation::unwrap_type_alias_deeply(rhs); - } - if (lhs == rhs) { - return true; - } - - if (const TypeDataUnion* lhs_union = lhs->try_as()) { - if (const TypeDataUnion* rhs_union = rhs->try_as()) { - return lhs_union->variants.size() == rhs_union->variants.size() && lhs_union->has_all_variants_of(rhs_union); - } - } - if (const TypeDataStruct* lhs_struct = lhs->try_as(); lhs_struct && lhs_struct->struct_ref->is_instantiation_of_generic_struct()) { - if (const TypeDataStruct* rhs_struct = rhs->try_as(); rhs_struct && rhs_struct->struct_ref->is_instantiation_of_generic_struct()) { - return lhs_struct->struct_ref->base_struct_ref == rhs_struct->struct_ref->base_struct_ref - && lhs_struct->struct_ref->substitutedTs->equal_to(rhs_struct->struct_ref->substitutedTs); - } - } - return false; -} - TypePtr TypeData::unwrap_alias_slow_path(TypePtr lhs) { TypePtr unwrapped = lhs; while (const TypeDataAlias* as_alias = unwrapped->try_as()) { @@ -192,6 +116,27 @@ TypePtr TypeData::unwrap_alias_slow_path(TypePtr lhs) { return unwrapped; } +// having `type UserId = int` and `type OwnerId = int` (when their underlying types are equal), +// make `UserId` and `OwnerId` NOT equal and NOT assignable (although they'll have the same type_id); +// it allows overloading methods for these types independently, e.g. +// > type BalanceList = dict +// > type AssetList = dict +// > fun BalanceList.validate(self) +// > fun AssetList.validate(self) +static bool are_two_equal_type_aliases_different(const TypeDataAlias* t1, const TypeDataAlias* t2) { + if (t1->alias_ref == t2->alias_ref) { + return false; + } + if (t1->alias_ref->is_instantiation_of_generic_alias() && t2->alias_ref->is_instantiation_of_generic_alias()) { + return !t1->alias_ref->substitutedTs->equal_to(t2->alias_ref->substitutedTs); + } + // handle `type MInt2 = MInt1`, as well as `type BalanceList = dict`, then they are equal + const TypeDataAlias* t_und1 = t1->underlying_type->try_as(); + const TypeDataAlias* t_und2 = t2->underlying_type->try_as(); + bool one_aliases_another = (t_und1 && t_und1->alias_ref == t2->alias_ref) + || (t_und2 && t1->alias_ref == t_und2->alias_ref); + return !one_aliases_another; +} // -------------------------------------------- // create() @@ -202,152 +147,67 @@ TypePtr TypeData::unwrap_alias_slow_path(TypePtr lhs) { // TypePtr TypeDataAlias::create(AliasDefPtr alias_ref) { - TypeDataHasherForUnique hash(5694590762732189561ULL); - hash.feed_string(alias_ref->name); - hash.feed_child(alias_ref->underlying_type); - - if (TypePtr existing = hash.get_existing()) { - return existing; - } - TypePtr underlying_type = alias_ref->underlying_type; if (underlying_type == TypeDataNullLiteral::create() || underlying_type == TypeDataNever::create() || underlying_type == TypeDataVoid::create()) { return underlying_type; // aliasing these types is strange, don't store an alias } - return hash.register_unique(new TypeDataAlias(hash.children_flags(), alias_ref, underlying_type)); + CalcChildrenFlags reg; + reg.feed_child(alias_ref->underlying_type); + return new TypeDataAlias(reg.children_flags(), alias_ref, underlying_type); } TypePtr TypeDataFunCallable::create(std::vector&& params_types, TypePtr return_type) { - TypeDataHasherForUnique hash(3184039965511020991ULL); - for (TypePtr param : params_types) { - hash.feed_child(param); - hash.feed_hash(767721); - } - hash.feed_child(return_type); - hash.feed_hash(767722); - - if (TypePtr existing = hash.get_existing()) { - return existing; - } - return hash.register_unique(new TypeDataFunCallable(hash.children_flags(), std::move(params_types), return_type)); + CalcChildrenFlags reg; + reg.feed_child(params_types); + reg.feed_child(return_type); + return new TypeDataFunCallable(reg.children_flags(), std::move(params_types), return_type); } TypePtr TypeDataGenericT::create(std::string&& nameT) { - TypeDataHasherForUnique hash(9145033724911680012ULL); - hash.feed_string(nameT); - - if (TypePtr existing = hash.get_existing()) { - return existing; - } - return hash.register_unique(new TypeDataGenericT(std::move(nameT))); + return new TypeDataGenericT(std::move(nameT)); } TypePtr TypeDataGenericTypeWithTs::create(StructPtr struct_ref, AliasDefPtr alias_ref, std::vector&& type_arguments) { - TypeDataHasherForUnique hash(7451094818554079348ULL); // use it only to calculate children_flags if (struct_ref) { assert(alias_ref == nullptr && struct_ref->is_generic_struct()); - hash.feed_string(struct_ref->name); } else { assert(struct_ref == nullptr && alias_ref->is_generic_alias()); - hash.feed_string(alias_ref->name); - } - for (TypePtr argT : type_arguments) { - hash.feed_child(argT); } - return new TypeDataGenericTypeWithTs(hash.children_flags(), struct_ref, alias_ref, std::move(type_arguments)); + CalcChildrenFlags reg; + reg.feed_child(type_arguments); + return new TypeDataGenericTypeWithTs(reg.children_flags(), struct_ref, alias_ref, std::move(type_arguments)); } TypePtr TypeDataStruct::create(StructPtr struct_ref) { - TypeDataHasherForUnique hash(8315986401043319583ULL); - hash.feed_string(struct_ref->name); - - if (TypePtr existing = hash.get_existing()) { - return existing; - } - return hash.register_unique(new TypeDataStruct(struct_ref)); + return new TypeDataStruct(struct_ref); } TypePtr TypeDataTensor::create(std::vector&& items) { - TypeDataHasherForUnique hash(3159238551239480381ULL); - for (TypePtr item : items) { - hash.feed_child(item); - hash.feed_hash(819613); - } - - if (TypePtr existing = hash.get_existing()) { - return existing; - } - return hash.register_unique(new TypeDataTensor(hash.children_flags(), std::move(items))); + CalcChildrenFlags reg; + reg.feed_child(items); + return new TypeDataTensor(reg.children_flags(), std::move(items)); } TypePtr TypeDataBrackets::create(std::vector&& items) { - TypeDataHasherForUnique hash(9189266157349499320ULL); - for (TypePtr item : items) { - hash.feed_child(item); - hash.feed_hash(735911); - } - - if (TypePtr existing = hash.get_existing()) { - return existing; - } - return hash.register_unique(new TypeDataBrackets(hash.children_flags(), std::move(items))); + CalcChildrenFlags reg; + reg.feed_child(items); + return new TypeDataBrackets(reg.children_flags(), std::move(items)); } TypePtr TypeDataIntN::create(int n_bits, bool is_unsigned, bool is_variadic) { - TypeDataHasherForUnique hash(1678330938771108027ULL); - hash.feed_hash(n_bits); - hash.feed_hash(is_unsigned); - hash.feed_hash(is_variadic); - - if (TypePtr existing = hash.get_existing()) { - return existing; - } - return hash.register_unique(new TypeDataIntN(n_bits, is_unsigned, is_variadic)); + return new TypeDataIntN(n_bits, is_unsigned, is_variadic); } TypePtr TypeDataBitsN::create(int n_width, bool is_bits) { - TypeDataHasherForUnique hash(7810988137199333041ULL); - hash.feed_hash(n_width); - hash.feed_hash(is_bits); - - if (TypePtr existing = hash.get_existing()) { - return existing; - } - return hash.register_unique(new TypeDataBitsN(n_width, is_bits)); + return new TypeDataBitsN(n_width, is_bits); } TypePtr TypeDataUnion::create(std::vector&& variants) { - TypeDataHasherForUnique hash(8719233194368471403ULL); - for (TypePtr variant : variants) { - hash.feed_child(variant); - hash.feed_hash(817663); - } - - if (TypePtr existing = hash.get_existing()) { - return existing; - } - - // for generics, like `f(v: T?)`, no type_id exists - // in this case, don't try to flatten: we have no info - // after instantiation, a new union type (with resolved variants) will be created - bool not_ready_yet = false; - for (TypePtr variant : variants) { - not_ready_yet |= variant->has_genericT_inside(); - } - if (not_ready_yet) { - TypePtr or_null = nullptr; - if (variants.size() == 2) { - if (variants[0] == TypeDataNullLiteral::create() || variants[1] == TypeDataNullLiteral::create()) { - or_null = variants[variants[0] == TypeDataNullLiteral::create()]; - } - } - return hash.register_unique(new TypeDataUnion(hash.children_flags(), or_null, std::move(variants))); - } - // flatten variants and remove duplicates - // note, that `int | slice` and `int | int | slice` are different TypePtr, but actually the same variants + // note, that `int | slice` and `int | int | slice` are different TypePtr, but actually the same variants; + // note, that `UserId | OwnerId` (both are aliases to `int`) will emit `UserId` (OwnerId is a duplicate) std::vector flat_variants; flat_variants.reserve(variants.size()); for (TypePtr variant : variants) { @@ -370,21 +230,10 @@ TypePtr TypeDataUnion::create(std::vector&& variants) { if (flat_variants.size() == 1) { // `int | int` return flat_variants[0]; } - return hash.register_unique(new TypeDataUnion(hash.children_flags(), or_null, std::move(flat_variants))); -} - -TypePtr TypeDataUnion::create_nullable(TypePtr nullable) { - // calculate exactly the same hash as for `T | null` to create std::vector only if type seen the first time - TypeDataHasherForUnique hash(8719233194368471403ULL); - hash.feed_child(nullable); - hash.feed_hash(817663); - hash.feed_child(TypeDataNullLiteral::create()); - hash.feed_hash(817663); - if (TypePtr existing = hash.get_existing()) { - return existing; - } - return create({nullable, TypeDataNullLiteral::create()}); + CalcChildrenFlags reg; + reg.feed_child(flat_variants); + return new TypeDataUnion(reg.children_flags(), or_null, std::move(flat_variants)); } @@ -685,14 +534,18 @@ TypePtr TypeDataUnion::replace_children_custom(const ReplacerCallbackT& callback // bool TypeDataAlias::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { - return true; + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + // having `type UserId = int` and `type OwnerId = int`, make them NOT assignable without `as` + // (although they both have the same type_id) + if (underlying_type->equal_to(rhs_alias->underlying_type)) { + return !are_two_equal_type_aliases_different(this, rhs_alias); + } } return underlying_type->can_rhs_be_assigned(rhs); } bool TypeDataInt::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (rhs->try_as()) { @@ -708,7 +561,7 @@ bool TypeDataInt::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataBool::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { @@ -718,7 +571,7 @@ bool TypeDataBool::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataCell::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (const TypeDataStruct* rhs_struct = rhs->try_as()) { @@ -733,7 +586,7 @@ bool TypeDataCell::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataSlice::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { @@ -743,7 +596,7 @@ bool TypeDataSlice::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataBuilder::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { @@ -753,7 +606,7 @@ bool TypeDataBuilder::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataTuple::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { @@ -763,7 +616,7 @@ bool TypeDataTuple::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataContinuation::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { @@ -773,7 +626,7 @@ bool TypeDataContinuation::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataAddress::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { @@ -783,7 +636,7 @@ bool TypeDataAddress::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataNullLiteral::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { @@ -793,9 +646,6 @@ bool TypeDataNullLiteral::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataFunCallable::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { - return true; - } if (const TypeDataFunCallable* rhs_callable = rhs->try_as()) { if (rhs_callable->params_size() != params_size()) { return false; @@ -826,15 +676,12 @@ bool TypeDataGenericTypeWithTs::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataStruct::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { - return true; + if (const TypeDataStruct* rhs_struct = rhs->try_as()) { // C> = C + return struct_ref == rhs_struct->struct_ref || equal_to(rhs_struct); } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { return can_rhs_be_assigned(rhs_alias->underlying_type); } - if (const TypeDataStruct* rhs_struct = rhs->try_as()) { // C> = C - return equal_to(rhs_struct); - } return rhs == TypeDataNever::create(); } @@ -869,32 +716,33 @@ bool TypeDataBrackets::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataIntN::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { - return true; - } if (rhs == TypeDataInt::create()) { return true; } + if (const TypeDataIntN* rhs_intN = rhs->try_as()) { + // `int8` is NOT assignable to `int32` without `as` + return n_bits == rhs_intN->n_bits && is_unsigned == rhs_intN->is_unsigned && is_variadic == rhs_intN->is_variadic; + } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { return can_rhs_be_assigned(rhs_alias->underlying_type); } - return rhs == TypeDataNever::create(); // `int8` is NOT assignable to `int32` without `as` + return rhs == TypeDataNever::create(); } bool TypeDataBitsN::can_rhs_be_assigned(TypePtr rhs) const { - // `slice` is NOT assignable to bitsN without `as` - // `bytes32` is NOT assignable to `bytes256` and even to `bits256` without `as` - if (rhs == this) { - return true; + if (const TypeDataBitsN* rhs_bitsN = rhs->try_as()) { + // `slice` is NOT assignable to bitsN without `as` + // `bytes32` is NOT assignable to `bytes256` and even to `bits256` without `as` + return n_width == rhs_bitsN->n_width && is_bits == rhs_bitsN->is_bits; } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { return can_rhs_be_assigned(rhs_alias->underlying_type); } - return false; + return rhs == TypeDataNever::create(); } bool TypeDataCoins::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (rhs == TypeDataInt::create()) { @@ -907,9 +755,6 @@ bool TypeDataCoins::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataUnion::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { - return true; - } if (calculate_exact_variant_to_fit_rhs(rhs)) { // `int` to `int | slice`, `int?` to `int8?`, `(int, null)` to `(int, T?) | slice` return true; } @@ -919,7 +764,6 @@ bool TypeDataUnion::can_rhs_be_assigned(TypePtr rhs) const { if (const TypeDataAlias* rhs_alias = rhs->try_as()) { return can_rhs_be_assigned(rhs_alias->underlying_type); } - return rhs == TypeDataNever::create(); } @@ -928,11 +772,11 @@ bool TypeDataUnknown::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataNever::can_rhs_be_assigned(TypePtr rhs) const { - return rhs == TypeDataNever::create(); + return rhs == singleton; } bool TypeDataVoid::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } return rhs == TypeDataNever::create(); @@ -977,7 +821,7 @@ bool TypeDataInt::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == this; + return cast_to == singleton; } bool TypeDataBool::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -993,7 +837,7 @@ bool TypeDataBool::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == this; + return cast_to == singleton; } bool TypeDataCell::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1006,7 +850,7 @@ bool TypeDataCell::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == this; + return cast_to == singleton; } bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1022,7 +866,7 @@ bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == this; + return cast_to == singleton; } bool TypeDataBuilder::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1032,7 +876,7 @@ bool TypeDataBuilder::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == this; + return cast_to == singleton; } bool TypeDataTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1042,7 +886,7 @@ bool TypeDataTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == this; + return cast_to == singleton; } bool TypeDataContinuation::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1052,7 +896,7 @@ bool TypeDataContinuation::can_be_casted_with_as_operator(TypePtr cast_to) const if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == this; + return cast_to == singleton; } bool TypeDataAddress::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1065,7 +909,7 @@ bool TypeDataAddress::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == this; + return cast_to == singleton; } bool TypeDataNullLiteral::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1075,7 +919,7 @@ bool TypeDataNullLiteral::can_be_casted_with_as_operator(TypePtr cast_to) const if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == this; + return cast_to == singleton; } bool TypeDataFunCallable::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1124,7 +968,7 @@ bool TypeDataStruct::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataStruct* to_struct = cast_to->try_as()) { // C> as C return equal_to(to_struct); } - return cast_to == this; + return false; } bool TypeDataTensor::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1205,7 +1049,7 @@ bool TypeDataCoins::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to == TypeDataInt::create()) { return true; } - return cast_to == this; + return cast_to == singleton; } bool TypeDataUnion::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1231,7 +1075,7 @@ bool TypeDataNever::can_be_casted_with_as_operator(TypePtr cast_to) const { } bool TypeDataVoid::can_be_casted_with_as_operator(TypePtr cast_to) const { - return cast_to == this; + return cast_to == singleton; } @@ -1288,12 +1132,147 @@ bool TypeDataVoid::can_hold_tvm_null_instead() const { } +// -------------------------------------------- +// equal_to() +// +// comparing types for equality (when implementation differs from a default "compare pointers"); +// two types are EQUAL is a much more strict property than "assignable"; +// a union type can hold only non-equal types; for instance, having `type MyInt = int`, a union `int | MyInt` == `int`; +// searching for a compatible method for a receiver is also based on equal_to() as first priority +// + +bool TypeDataAlias::equal_to(TypePtr rhs) const { + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + // given `type UserId = int` and `type OwnerId = int`, treat them as NOT equal (they are also not assignable); + // (but nevertheless, they will have the same type_id, and `UserId | OwnerId` is not a valid union) + if (underlying_type->equal_to(rhs_alias->underlying_type)) { + return !are_two_equal_type_aliases_different(this, rhs_alias); + } + } + return underlying_type->equal_to(rhs); +} + +bool TypeDataFunCallable::equal_to(TypePtr rhs) const { + if (const TypeDataFunCallable* rhs_callable = rhs->try_as(); rhs_callable && rhs_callable->params_size() == params_size()) { + for (int i = 0; i < params_size(); ++i) { + if (!params_types[i]->equal_to(rhs_callable->params_types[i])) { + return false; + } + } + return return_type->equal_to(rhs_callable->return_type); + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + +bool TypeDataGenericT::equal_to(TypePtr rhs) const { + if (const TypeDataGenericT* rhs_T = rhs->try_as()) { + return nameT == rhs_T->nameT; + } + return false; +} + +bool TypeDataGenericTypeWithTs::equal_to(TypePtr rhs) const { + if (const TypeDataGenericTypeWithTs* rhs_Ts = rhs->try_as(); rhs_Ts && size() == rhs_Ts->size()) { + for (int i = 0; i < size(); ++i) { + if (!type_arguments[i]->equal_to(rhs_Ts->type_arguments[i])) { + return false; + } + } + return alias_ref == rhs_Ts->alias_ref && struct_ref == rhs_Ts->struct_ref; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + +bool TypeDataStruct::equal_to(TypePtr rhs) const { + if (const TypeDataStruct* rhs_struct = rhs->try_as()) { + if (struct_ref == rhs_struct->struct_ref) { + return true; + } + if (struct_ref->is_instantiation_of_generic_struct() && rhs_struct->struct_ref->is_instantiation_of_generic_struct()) { + return struct_ref->base_struct_ref == rhs_struct->struct_ref->base_struct_ref + && struct_ref->substitutedTs->equal_to(rhs_struct->struct_ref->substitutedTs); + } + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + +bool TypeDataTensor::equal_to(TypePtr rhs) const { + if (const TypeDataTensor* rhs_tensor = rhs->try_as(); rhs_tensor && size() == rhs_tensor->size()) { + for (int i = 0; i < size(); ++i) { + if (!items[i]->equal_to(rhs_tensor->items[i])) { + return false; + } + } + return true; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + +bool TypeDataBrackets::equal_to(TypePtr rhs) const { + if (const TypeDataBrackets* rhs_brackets = rhs->try_as(); rhs_brackets && size() == rhs_brackets->size()) { + for (int i = 0; i < size(); ++i) { + if (!items[i]->equal_to(rhs_brackets->items[i])) { + return false; + } + } + return true; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + +bool TypeDataIntN::equal_to(TypePtr rhs) const { + if (const TypeDataIntN* rhs_intN = rhs->try_as()) { + return n_bits == rhs_intN->n_bits && is_unsigned == rhs_intN->is_unsigned && is_variadic == rhs_intN->is_variadic; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + +bool TypeDataBitsN::equal_to(TypePtr rhs) const { + if (const TypeDataBitsN* rhs_bitsN = rhs->try_as()) { + return n_width == rhs_bitsN->n_width && is_bits == rhs_bitsN->is_bits; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + +bool TypeDataUnion::equal_to(TypePtr rhs) const { + if (const TypeDataUnion* rhs_union = rhs->try_as()) { + return variants.size() == rhs_union->variants.size() && has_all_variants_of(rhs_union); + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + + // union types creation is a bit tricky: nested unions are flattened, duplicates are removed -// so, a resolved union type has variants, each with unique type_id -// (type_id is calculated with aliases erasure) +// so, a resolved union type has variants, each will be assigned a unique type_id (tagged unions) void TypeDataUnion::append_union_type_variant(TypePtr variant, std::vector& out_unique_variants) { + // having `UserId | OwnerId` (both are aliases to `int`) merge them into just `UserId`, because underlying are equal + TypePtr underlying_variant = variant->unwrap_alias(); for (TypePtr existing : out_unique_variants) { - if (existing->get_type_id() == variant->get_type_id()) { + if (existing->equal_to(underlying_variant)) { return; } } @@ -1301,9 +1280,9 @@ void TypeDataUnion::append_union_type_variant(TypePtr variant, std::vectorget_type_id() == type_id) { + if (self_variant->equal_to(rhs_type)) { return true; } } @@ -1312,7 +1291,7 @@ bool TypeDataUnion::has_variant_with_type_id(int type_id) const { bool TypeDataUnion::has_all_variants_of(const TypeDataUnion* rhs_type) const { for (TypePtr rhs_variant : rhs_type->variants) { - if (!has_variant_with_type_id(rhs_variant->get_type_id())) { + if (!has_variant_equal_to(rhs_variant)) { return false; } } @@ -1339,7 +1318,7 @@ TypePtr TypeDataUnion::calculate_exact_variant_to_fit_rhs(TypePtr rhs_type) cons } // `int` to `int | int8` is okay: exact type matching for (TypePtr variant : variants) { - if (variant->get_type_id() == rhs_type->get_type_id()) { + if (variant->equal_to(rhs_type)) { return variant; } } diff --git a/tolk/type-system.h b/tolk/type-system.h index e9a63c2135..74a3bb5805 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -42,7 +42,7 @@ class TypeData { // bits of flag_mask, to store often-used properties and return them without tree traversing const int flags; - friend class TypeDataHasherForUnique; + friend class CalcChildrenFlags; protected: enum flag_mask { @@ -55,7 +55,6 @@ class TypeData { : flags(flags_with_children) { } - static bool equal_to_slow_path(TypePtr lhs, TypePtr rhs); static TypePtr unwrap_alias_slow_path(TypePtr lhs); public: @@ -71,9 +70,6 @@ class TypeData { return 1; // most types occupy 1 stack slot (int, cell, slice, etc.) } - bool equal_to(TypePtr rhs) const { - return this == rhs || equal_to_slow_path(this, rhs); - } TypePtr unwrap_alias() const { return has_type_alias_inside() ? unwrap_alias_slow_path(this) : this; } @@ -93,6 +89,10 @@ class TypeData { return true; } + virtual bool equal_to(TypePtr rhs) const { + return this == rhs->unwrap_alias(); + } + virtual TypePtr replace_children_custom(const ReplacerCallbackT& callback) const { return callback(this); } @@ -124,6 +124,7 @@ class TypeDataAlias final : public TypeData { bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; bool can_hold_tvm_null_instead() const override; + bool equal_to(TypePtr rhs) const override; }; /* @@ -320,6 +321,7 @@ class TypeDataFunCallable final : public TypeData { bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; + bool equal_to(TypePtr rhs) const override; }; /* @@ -343,6 +345,7 @@ class TypeDataGenericT final : public TypeData { std::string as_human_readable() const override { return nameT; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + bool equal_to(TypePtr rhs) const override; }; /* @@ -373,6 +376,7 @@ class TypeDataGenericTypeWithTs final : public TypeData { bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; + bool equal_to(TypePtr rhs) const override; }; /* @@ -398,6 +402,7 @@ class TypeDataStruct final : public TypeData { bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; bool can_hold_tvm_null_instead() const override; + bool equal_to(TypePtr rhs) const override; }; /* @@ -425,6 +430,7 @@ class TypeDataTensor final : public TypeData { bool can_be_casted_with_as_operator(TypePtr cast_to) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; bool can_hold_tvm_null_instead() const override; + bool equal_to(TypePtr rhs) const override; }; /* @@ -449,6 +455,7 @@ class TypeDataBrackets final : public TypeData { bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; + bool equal_to(TypePtr rhs) const override; }; /* @@ -475,6 +482,7 @@ class TypeDataIntN final : public TypeData { std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + bool equal_to(TypePtr rhs) const override; }; /* @@ -518,6 +526,7 @@ class TypeDataBitsN final : public TypeData { std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + bool equal_to(TypePtr rhs) const override; }; /* @@ -528,6 +537,8 @@ class TypeDataBitsN final : public TypeData { * - `T | null`, if T is 1 slot (like `int | null`), then it's still 1 slot * - `T | null`, if T is N slots (like `(int, int)?`), it's stored as N+1 slots (the last for type_id if T or 0 if null) * - `T1 | T2 | ...` is a tagged union: occupy max(T_i)+1 slots (1 for type_id) + * When a union is created, variants are flattened, duplicates are removed: `int | UserId | IntOrSlice` = `int | slice`, + * duplicates are detected based on `equal_to()`, and a union can be tested on having `has_variant_equal_to()`. */ class TypeDataUnion final : public TypeData { TypeDataUnion(int children_flags, TypePtr or_null, std::vector&& variants) @@ -535,7 +546,6 @@ class TypeDataUnion final : public TypeData { , or_null(or_null) , variants(std::move(variants)) {} - bool has_variant_with_type_id(int type_id) const; static void append_union_type_variant(TypePtr variant, std::vector& out_unique_variants); public: @@ -543,7 +553,7 @@ class TypeDataUnion final : public TypeData { const std::vector variants; // T_i, flattened, no duplicates; may include aliases, but not other unions static TypePtr create(std::vector&& variants); - static TypePtr create_nullable(TypePtr nullable); + static TypePtr create_nullable(TypePtr nullable) { return create({nullable, TypeDataNullLiteral::create()}); } int size() const { return static_cast(variants.size()); } @@ -554,20 +564,11 @@ class TypeDataUnion final : public TypeData { return get_width_on_stack() == 1 && or_null != nullptr && or_null->get_width_on_stack() == 1; } bool has_null() const { - if (or_null) { - return true; - } - return has_variant_with_type_id(0); - } - bool has_variant_with_type_id(TypePtr rhs_type) const { - int type_id = rhs_type->get_type_id(); - if (or_null) { - return type_id == 0 || type_id == or_null->get_type_id(); - } - return has_variant_with_type_id(type_id); + return or_null != nullptr || has_variant_equal_to(TypeDataNullLiteral::create()); } TypePtr calculate_exact_variant_to_fit_rhs(TypePtr rhs_type) const; + bool has_variant_equal_to(TypePtr rhs_type) const; bool has_all_variants_of(const TypeDataUnion* rhs_type) const; int get_variant_idx(TypePtr lookup_variant) const; @@ -578,6 +579,7 @@ class TypeDataUnion final : public TypeData { bool can_be_casted_with_as_operator(TypePtr cast_to) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; bool can_hold_tvm_null_instead() const override; + bool equal_to(TypePtr rhs) const override; }; /* From 4d59949c3ba4053f5ef3d4623633f419ac7a5ee0 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 7 Aug 2025 23:37:45 +0700 Subject: [PATCH 389/524] [Tolk] Enhance overload resolution rules for method receivers This commit focuses on overload resolution for generic methods: strict ordering, type shapes, etc. --- .../tests/invalid-semantics/err-4104.tolk | 2 +- .../tests/invalid-semantics/err-4560.tolk | 17 ++ .../tests/invalid-semantics/err-4900.tolk | 13 + .../tests/invalid-semantics/err-4901.tolk | 13 + .../tests/invalid-semantics/err-4902.tolk | 18 ++ .../tests/invalid-semantics/err-4903.tolk | 17 ++ .../tests/invalid-semantics/err-4904.tolk | 19 ++ .../tests/invalid-symbol/err-2258.tolk | 12 + .../tests/invalid-typing/err-6728.tolk | 17 ++ tolk-tester/tests/methods-tests.tolk | 10 +- tolk-tester/tests/overloads-tests.tolk | 260 ++++++++++++++++++ tolk-tester/tests/var-apply-tests.tolk | 2 +- tolk/CMakeLists.txt | 1 + tolk/generics-helpers.cpp | 113 +++----- tolk/generics-helpers.h | 6 +- tolk/overload-resolution.cpp | 231 ++++++++++++++++ tolk/overload-resolution.h | 45 +++ tolk/pipe-infer-types-and-calls.cpp | 22 +- tolk/type-system.h | 2 +- 19 files changed, 717 insertions(+), 103 deletions(-) create mode 100644 tolk-tester/tests/invalid-semantics/err-4560.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4900.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4901.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4902.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4903.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4904.tolk create mode 100644 tolk-tester/tests/invalid-symbol/err-2258.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6728.tolk create mode 100644 tolk-tester/tests/overloads-tests.tolk create mode 100644 tolk/overload-resolution.cpp create mode 100644 tolk/overload-resolution.h diff --git a/tolk-tester/tests/invalid-semantics/err-4104.tolk b/tolk-tester/tests/invalid-semantics/err-4104.tolk index 3ca7ef1bc0..7cb8c8ef9b 100644 --- a/tolk-tester/tests/invalid-semantics/err-4104.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4104.tolk @@ -8,7 +8,7 @@ fun Container.createFrom(item: U): Container { fun cantUseAsNonCall() { Container.createFrom; // ok - Container.createFrom>; // ok + Container.createFrom>; // ok Container.createFrom; } diff --git a/tolk-tester/tests/invalid-semantics/err-4560.tolk b/tolk-tester/tests/invalid-semantics/err-4560.tolk new file mode 100644 index 0000000000..507b9ccc2e --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4560.tolk @@ -0,0 +1,17 @@ +struct Wrapper { item: T } + +type AssetBuilder = builder +type BalanceBuilder = builder + +fun Wrapper.validate(self) { return 0 } + +fun main() { + var b: BalanceBuilder = beginCell(); + Wrapper{item:b}.validate(); +} + +/** +@compilation_should_fail +@stderr method `validate` not found for type `Wrapper` +@stderr but it exists for type `Wrapper` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4900.tolk b/tolk-tester/tests/invalid-semantics/err-4900.tolk new file mode 100644 index 0000000000..2922529316 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4900.tolk @@ -0,0 +1,13 @@ +fun int.realValue(self): int8 { return 1 } +fun int?.realValue(self): uint256 { return 1 } + +fun main() { + (5 as int8).realValue(); +} + +/** +@compilation_should_fail +@stderr call to method `realValue` for type `int8` is ambiguous +@stderr candidate function: `int.realValue` +@stderr candidate function: `int?.realValue` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4901.tolk b/tolk-tester/tests/invalid-semantics/err-4901.tolk new file mode 100644 index 0000000000..a21db6feed --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4901.tolk @@ -0,0 +1,13 @@ +fun int8.doSome2() {} +fun int16.doSome2() {} + +fun main() { + int.doSome2(); +} + +/** +@compilation_should_fail +@stderr call to method `doSome2` for type `int` is ambiguous +@stderr candidate function: `int8.doSome2` +@stderr candidate function: `int16.doSome2` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4902.tolk b/tolk-tester/tests/invalid-semantics/err-4902.tolk new file mode 100644 index 0000000000..c56be06ccc --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4902.tolk @@ -0,0 +1,18 @@ +struct MyMap + +fun MyMap.amb(self) { return 0 } +fun MyMap.amb(self) { return 0 } + +fun main() { + MyMap{}.amb(); // ok + MyMap{}.amb(); // ok + + MyMap{}.amb(); +} + +/** +@compilation_should_fail +@stderr call to method `amb` for type `MyMap` is ambiguous +@stderr candidate function: `MyMap.amb` with K=`int` +@stderr candidate function: `MyMap.amb` with V=`slice` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4903.tolk b/tolk-tester/tests/invalid-semantics/err-4903.tolk new file mode 100644 index 0000000000..4c8ab33f59 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4903.tolk @@ -0,0 +1,17 @@ +struct MyMap + +fun MyMap.amb(self) { return 0 } +fun MyMap.amb(self) { return 0 } + +fun main() { + MyMap<(int, int, MyMap), slice>{}.amb(); // ok + MyMap{}.amb(); // ok + + MyMap{}.amb(); +} + +/** +@compilation_should_fail +@stderr method `amb` not found for type `MyMap` +@stderr (but it exists for type `MyMap`) + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4904.tolk b/tolk-tester/tests/invalid-semantics/err-4904.tolk new file mode 100644 index 0000000000..062fa5af2a --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4904.tolk @@ -0,0 +1,19 @@ +type AssetBuilder = builder +type BalanceBuilder = builder + +fun AssetBuilder.validate(self) { return 0 } +fun BalanceBuilder.validate(self) { return 0 } + +fun main() { + (beginCell() as AssetBuilder).validate(); // ok + (beginCell() as BalanceBuilder).validate(); // ok + + beginCell().validate(); +} + +/** +@compilation_should_fail +@stderr call to method `validate` for type `builder` is ambiguous +@stderr candidate function: `AssetBuilder.validate` +@stderr candidate function: `BalanceBuilder.validate` + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2258.tolk b/tolk-tester/tests/invalid-symbol/err-2258.tolk new file mode 100644 index 0000000000..98e4224729 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2258.tolk @@ -0,0 +1,12 @@ +struct Wrapper { item: T; } +fun Wrapper.createNull(): Wrapper { return { item: null } } + +fun main() { + Wrapper.createNull(); +} + +/** +@compilation_should_fail +@stderr method `createNull` not found for type `Wrapper` +@stderr (but it exists for type `Wrapper`) + */ diff --git a/tolk-tester/tests/invalid-typing/err-6728.tolk b/tolk-tester/tests/invalid-typing/err-6728.tolk new file mode 100644 index 0000000000..6da940f0af --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6728.tolk @@ -0,0 +1,17 @@ +struct Point { + x: int8 + y: int8 +} + +fun Cell.myMethod(self, v: U): int + asm "DROP" "HASHCU" + +fun main() { + Point{x:10,y:20}.toCell().myMethod(4); // ok + Point{x:10,y:20}.toCell().myMethod((4, 5)); +} + +/** +@compilation_should_fail +@stderr can not call `Cell.myMethod` with U=(int, int), because it occupies 2 stack slots in TVM, not 1 + */ diff --git a/tolk-tester/tests/methods-tests.tolk b/tolk-tester/tests/methods-tests.tolk index 27c6d17a18..c4d5948bcc 100644 --- a/tolk-tester/tests/methods-tests.tolk +++ b/tolk-tester/tests/methods-tests.tolk @@ -191,11 +191,11 @@ fun T?.arbitraryMethod(self): never { throw 123; } fun test12() { __expect_type(10.arbitraryMethod(), "int?"); - __expect_type((10 as int8?).arbitraryMethod(), "int?"); __expect_type((10 as coins?)!.arbitraryMethod(), "int?"); - __expect_type(null.arbitraryMethod(), "int?"); - __expect_type((10 as coins?).arbitraryMethod(), "int?"); + __expect_type((10 as int8?).arbitraryMethod(), "never"); + __expect_type(null.arbitraryMethod(), "never"); + __expect_type((10 as coins?).arbitraryMethod(), "never"); __expect_type(Wrapper { item: 10 }.arbitraryMethod(), "Wrapper"); __expect_type(Wrapper { item: 10 as int8 }.arbitraryMethod(), "Wrapper"); @@ -280,7 +280,7 @@ fun Wrapper.createFrom(item: U): Wrapper { return {item}; } fun test18() { __expect_type(10.copy, "(int) -> int"); __expect_type(Wrapper{item:null as int8?}.copy, "(Wrapper) -> Wrapper"); - __expect_type(Wrapper.createNull, "() -> Wrapper"); + __expect_type(Wrapper.createNull, "() -> Wrapper"); __expect_type(Wrapper?>.createNull, "() -> Wrapper?>"); __expect_type(Wrapper.createFrom, "(int8) -> Wrapper"); @@ -319,7 +319,7 @@ fun main() {} @testcase | 107 | | 2 1 -1 @testcase | 108 | 5 | 17 5 @testcase | 109 | | 5 100500 20 30 5 5 31 -@testcase | 110 | | [ 2 3 1 1 3 1 1 ] -1 +@testcase | 110 | | [ 2 3 1 1 1 1 1 ] -1 @testcase | 111 | | [ 1 2 3 3 2 ] @testcase | 113 | | 0 -1 -1 -1 0 -1 @testcase | 114 | | -1 -1 0 0 0 diff --git a/tolk-tester/tests/overloads-tests.tolk b/tolk-tester/tests/overloads-tests.tolk new file mode 100644 index 0000000000..962fb06c4b --- /dev/null +++ b/tolk-tester/tests/overloads-tests.tolk @@ -0,0 +1,260 @@ +struct Wrapper { item: T } +struct Nothing {} +struct MyPair {} + +struct Point { x: int8; y: int8 } + +type MInt = int +type MSlice = slice +type PointAlias = Point + +fun T.realValue(self): int1 { return 0 } +fun TT?.realValue(self): int2 { return 0 } +fun int.realValue(self): int3 { return 0 } +fun int8.realValue(self): int4 { return 0 } +fun int?.realValue(self): int5 { return 0 } +fun Nothing.realValue(self): int6 { return 0 } +fun Nothing>.realValue(self): int7 { return 0 } +fun Nothing.realValue(self): int8 { return 0 } + +fun test1() { + __expect_type(beginCell().realValue(), "int1"); + __expect_type((5 as int|slice).realValue(), "int1"); + __expect_type((beginCell() as builder?).realValue(), "int2"); + __expect_type((null as null|Nothing).realValue(), "int2"); + __expect_type((null as Nothing|null).realValue(), "int2"); + __expect_type(5.realValue(), "int3"); + __expect_type((5 as int?).realValue(), "int5"); + __expect_type((5 as int8).realValue(), "int4"); + __expect_type((5 as int8?).realValue(), "int2"); + __expect_type(Nothing{}.realValue(), "int6"); + __expect_type(Nothing{}.realValue(), "int6"); + __expect_type(Nothing{}.realValue(), "int8"); + __expect_type(Nothing{}.realValue(), "int8"); + __expect_type(Nothing{}.realValue(), "int6"); + __expect_type(Nothing>{}.realValue(), "int7"); + __expect_type(Nothing>>{}.realValue(), "int7"); + __expect_type(Nothing>?>{}.realValue(), "int8"); +} + +fun T.doSome2(self): int { return 0 } +fun int8.doSome2(self): cell { return createEmptyCell() } +fun int16.doSome2(self): slice { return "" } + +fun test2() { + __expect_type(5.doSome2(), "int"); + __expect_type((5 as int8).doSome2(), "cell"); + __expect_type(((5 as int8) as int16).doSome2(), "slice"); + __expect_type((5 as int?).doSome2(), "int"); + __expect_type(5.doSome2().doSome2(), "int"); + __expect_type((1, 2).doSome2(), "int"); +} + +fun MyPair.mapFn(self): int1 { return 0 } +fun MyPair.mapFn(self): int2 { return 0 } +fun MyPair.mapFn(self): int3 { return 0 } +fun MyPair.mapFn(self): int4 { return 0 } + +fun test3() { + __expect_type(MyPair{}.mapFn(), "int1"); + __expect_type(MyPair{}.mapFn(), "int1"); + __expect_type(MyPair{}.mapFn(), "int2"); + __expect_type(MyPair{}.mapFn(), "int2"); + __expect_type(MyPair{}.mapFn(), "int3"); + __expect_type(MyPair{}.mapFn(), "int4"); + __expect_type(MyPair{}.mapFn(), "int4"); +} + +fun (int|slice).unionMix(self): slice { return "" } +fun int.unionMix(self): bool { return true } + +fun test4() { + __expect_type((5 as int).unionMix(), "bool"); + __expect_type(("".unionMix()), "slice"); + __expect_type(((5 as int|slice).unionMix()), "slice"); +} + +fun WrapA.dup(self): WrapA { return self } +fun WrapA>.dup(self): WrapA> { return self } +struct WrapA {} + +fun test5() { + __expect_type(WrapA{}.dup(), "WrapA"); + __expect_type(WrapA>{}.dup(), "WrapA>"); +} + +type AliasBuilder1 = builder +type AliasBuilder2 = builder + +fun AliasBuilder1.myBAlias(self): int1 { return 0 } +fun AliasBuilder2.myBAlias(self): int2 { return 0 } +fun builder?.myBAlias(self): int3 { return 0 } +fun T.myBAlias(self): int4 { return 0 } +fun U?.myBAlias(self): int5 { return 0 } + +fun test6(b: builder, b1: AliasBuilder1, b2: AliasBuilder2, b1n: AliasBuilder1?, b2n: AliasBuilder2?) { + __expect_type(b1.myBAlias(), "int1"); + __expect_type(b2.myBAlias(), "int2"); + __expect_type(b1n.myBAlias(), "int3"); + __expect_type((b2n as builder?).myBAlias(), "int3"); + + __expect_type((b as builder|slice).myBAlias(), "int4"); + __expect_type(0.myBAlias(), "int4"); + __expect_type((0 as int|int16).myBAlias(), "int4"); + __expect_type(b2n.myBAlias(), "int3"); + __expect_type((null as builder|slice?).myBAlias(), "int5"); + __expect_type((0 as int?).myBAlias(), "int5"); +} + +type AssetBuilder = builder +type BalanceBuilder = builder + +fun AssetBuilder.validate(self): int1 { return 0 } +fun BalanceBuilder.validate(self): int2 { return 0 } +fun Wrapper.validate(self): int3 { return 0 } +fun Wrapper.validate(self): int4 { return 0 } + +fun test7() { + var a: AssetBuilder = beginCell(); + var b: BalanceBuilder = beginCell(); + + __expect_type(a.validate(), "int1"); + __expect_type(b.validate(), "int2"); + __expect_type(Wrapper{item:a}.validate(), "int3"); + __expect_type(Wrapper{item:b}.validate(), "int4"); +} + +fun (slice|TT).supposeMe(self): TT { if (self is slice) { throw 123 } return self } +fun T.supposeMe(self): int2 { return 0 } +fun int|slice|null.supposeMe(self): int3 { return 0 } + +fun test8(a: cell|slice, b: cell|()|slice|builder, c:slice, d:int|slice, e:null|(slice|int), f: int?, g:cell) { + __expect_type(a.supposeMe(), "cell"); + __expect_type(b.supposeMe(), "cell | () | builder"); + __expect_type(c.supposeMe(), "slice"); + __expect_type(d.supposeMe(), "int"); + __expect_type(e.supposeMe(), "int3"); + __expect_type(f.supposeMe(), "int2"); + __expect_type(g.supposeMe(), "int2"); +} + +fun MyPair.mySize(self): int1 { return 0 } +fun MyPair.mySize(self): int2 { return 0 } +fun MyPair>.mySize(self): int3 { return 0 } +fun MyPair>.mySize(self): int4 { return 0 } +fun MyPair>.mySize(self): int5 { return 0 } + +fun test9( + m1: MyPair, m2: MyPair, + m3: MyPair>, m4: MyPair>, + m5: MyPair>, m6: MyPair>, + m7: MyPair>, m8: MyPair>, + m9: MyPair?>, m10: MyPair?>, + m11: MyPair>, m12: MyPair>, +) { + __expect_type(m1.mySize(), "int1"); + __expect_type(m2.mySize(), "int2"); + __expect_type(m3.mySize(), "int3"); + __expect_type(m4.mySize(), "int3"); + __expect_type(m5.mySize(), "int4"); + __expect_type(m6.mySize(), "int4"); + __expect_type(m7.mySize(), "int5"); + __expect_type(m8.mySize(), "int5"); + __expect_type(m9.mySize(), "int2"); + __expect_type(m10.mySize(), "int1"); + __expect_type(m11.mySize(), "int4"); + __expect_type(m12.mySize(), "int4"); +} + +fun MyPair>.h1(self): int1 { return 0 } +fun MyPair>.h1(self): int2 { return 0 } +fun MyPair>.h1(self): int3 { return 0 } +fun MyPair>.h1(self): int4 { return 0 } + +fun test10() { + __expect_type(MyPair>{}.h1(), "int1"); + __expect_type(MyPair>{}.h1(), "int2"); + __expect_type(MyPair>{}.h1(), "int3"); + __expect_type(MyPair>{}.h1(), "int4"); + __expect_type(MyPair>{}.h1(), "int4"); +} + +fun (int, U).p(self): int1 { return 0 } +fun (T, U).p(self): T { return self.0 } +fun T.p(self): int3 { return 0 } + +fun test11() { + __expect_type((5, "").p(), "int1"); + __expect_type((5 as int8, "").p(), "int8"); + __expect_type(("", "").p(), "slice"); + __expect_type((1, 2, 3).p(), "int3"); +} + +fun Wrapper.w(self): int1 { return 0 } +fun Wrapper.w(self): int2 { return 0 } + +fun test12() { + __expect_type(Wrapper{item:0}.w(), "int1"); + __expect_type(Wrapper{item:""}.w(), "int1"); + __expect_type(Wrapper>{item:{item:0}}.w(), "int2"); + // here the compiler finds T=slice resulting in Wrapper (it's correct) + // and chooses (1) due to a deeper shape + __expect_type(Wrapper{item:""}.w(), "int1"); +} + +type MyCell = cell +type MInt8 = int8 +type MInt32 = int32 + +fun MyCell.makeIterator(self): MyIterator { + return MyIterator.new(self) +} + +struct MyIterator { + data: slice +} + +fun MyIterator.new(c: cell): MyIterator { + return { data: c.beginParse() } +} + +fun MyIterator.empty(self): bool { + return self.data.isEmpty() +} + +fun MyIterator.next(mutate self): T { + return self.data.loadAny() +} + +fun MyCell.iterateAndAppend(self, mutate t: tuple) { + var it = self.makeIterator(); + while (!it.empty()) { + t.push(it.next()) + } +} + +fun MyCell.iterateAndAppend(self, mutate t: tuple) { + t.push(-777); +} + +@method_id(113) +fun test13() { + var c = beginCell().storeSlice(stringHexToSlice("0102")).endCell(); + var t = createEmptyTuple(); + (c as MyCell).iterateAndAppend(mutate t); + (c as MyCell).iterateAndAppend(mutate t); + (c as MyCell).iterateAndAppend(mutate t); + (c as MyCell).iterateAndAppend(mutate t); + return t; // [ 1 2 258 1 2 -777 ] +} + + +fun main() { + // besides these positive tests, there are many negative resulting in "ambiguous call" + return 0 +} + +/** +@testcase | 0 | | 0 +@testcase | 113 | | [ 1 2 258 1 2 -777 ] + */ diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index 8bd23fe053..e11235ed01 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -309,7 +309,7 @@ fun testMethodsOfGenericStruct() { __expect_type(w2, "Wrapper"); var eq = Wrapper.equalsNotNull; __expect_type(eq, "(Wrapper, Wrapper) -> bool"); - var creator3 = Wrapper.createFromNull; + var creator3 = Wrapper.createFromNull; __expect_type(creator3, "() -> Wrapper"); return (eq(w1, w2), w2.value = w1.value as int, eq(w1, w2), creator3()); } diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index 033456e7fa..2a68d02d8e 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -33,6 +33,7 @@ set(TOLK_SOURCE smart-casts-cfg.cpp generics-helpers.cpp lazy-helpers.cpp + overload-resolution.cpp abscode.cpp analyzer.cpp asmops.cpp diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index 4726db9f85..5c031e38de 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -89,7 +89,6 @@ void GenericsSubstitutions::set_typeT(std::string_view nameT, TypePtr typeT) { for (int i = 0; i < size(); ++i) { if (genericTs->get_nameT(i) == nameT) { if (valuesTs[i] == nullptr) { - tolk_assert(!typeT->has_genericT_inside()); valuesTs[i] = typeT; } break; @@ -126,6 +125,12 @@ GenericSubstitutionsDeducing::GenericSubstitutionsDeducing(StructPtr struct_ref) , deducedTs(struct_ref->genericTs) { } +GenericSubstitutionsDeducing::GenericSubstitutionsDeducing(const GenericsDeclaration* genericTs) + : fun_ref(nullptr) + , struct_ref(nullptr) + , deducedTs(genericTs) { +} + // purpose: having `f(value: T)` and call `f(5)`, deduce T = int // generally, there may be many generic Ts for declaration, and many arguments // for every argument, `consider_next_condition()` is called @@ -146,8 +151,16 @@ void GenericSubstitutionsDeducing::consider_next_condition(TypePtr param_type, T deducedTs.set_typeT(asT->nameT, arg_type); } else if (const auto* p_nullable = param_type->try_as(); p_nullable && p_nullable->or_null) { // `arg: T?` called as `f(nullableInt)` => T is int - if (const auto* a_nullable = arg_type->unwrap_alias()->try_as(); a_nullable && a_nullable->or_null) { - consider_next_condition(p_nullable->or_null, a_nullable->or_null); + // `arg: T?` called as `f(T1|T2|null)` => T is T1|T2 + if (const auto* a_nullable = arg_type->unwrap_alias()->try_as(); a_nullable && a_nullable->has_null()) { + std::vector rest_but_null; + rest_but_null.reserve(a_nullable->size() - 1); + for (TypePtr a_variant : a_nullable->variants) { + if (a_variant != TypeDataNullLiteral::create()) { + rest_but_null.push_back(a_variant); + } + } + consider_next_condition(p_nullable->or_null, TypeDataUnion::create(std::move(rest_but_null))); } // `arg: T?` called as `f(int)` => T is int else { @@ -218,6 +231,13 @@ void GenericSubstitutionsDeducing::consider_next_condition(TypePtr param_type, T consider_next_condition(p_instSt->type_arguments[i], a_struct->struct_ref->substitutedTs->typeT_at(i)); } } + // `arg: Wrapper` called as `f(Wrapper>)` => T is Wrapper + if (const auto* a_instSt = arg_type->try_as(); a_instSt && a_instSt->struct_ref == p_instSt->struct_ref) { + tolk_assert(p_instSt->size() == a_instSt->size()); + for (int i = 0; i < p_instSt->size(); ++i) { + consider_next_condition(p_instSt->type_arguments[i], a_instSt->type_arguments[i]); + } + } } else if (const auto* p_instAl = param_type->try_as(); p_instAl && p_instAl->alias_ref) { // `arg: WrapperAlias` called as `f(wrappedInt)` => T is int if (const auto* a_alias = arg_type->try_as(); a_alias && a_alias->alias_ref->is_instantiation_of_generic_alias() && a_alias->alias_ref->base_alias_ref == p_instAl->alias_ref) { @@ -444,86 +464,23 @@ AliasDefPtr instantiate_generic_alias(AliasDefPtr alias_ref, GenericsSubstitutio return new_alias_ref; } -// find `builder.storeInt` for called_receiver = "builder" and called_name = "storeInt" -// most practical case, when a direct method for receiver exists; -// note, that having an alias `type WorkchainNum = int` and methods `WorkchainNum.isMasterchain()`, -// it's okay to call `-1.isMasterchain()`, because int equals to any alias; -// currently there is no chance to change this logic, say, `type AssetList = dict` to have separate methods, -// due to smart casts, types merge of control flow rejoin, etc., which immediately become `cell?` -FunctionPtr match_exact_method_for_call_not_generic(TypePtr called_receiver, std::string_view called_name) { - FunctionPtr exact_found = nullptr; - - for (FunctionPtr method_ref : G.all_methods) { - if (method_ref->method_name == called_name && !method_ref->receiver_type->has_genericT_inside()) { - if (method_ref->receiver_type->equal_to(called_receiver)) { - if (exact_found) { - return nullptr; - } - exact_found = method_ref; - } - } - } - - return exact_found; -} - -// find `int?.copy` / `T.copy` for called_receiver = "int" and called_name = "copy" -std::vector match_methods_for_call_including_generic(TypePtr called_receiver, std::string_view called_name) { - std::vector candidates; - - // step1: find all methods where a receiver equals to provided, e.g. `MInt.copy` - for (FunctionPtr method_ref : G.all_methods) { - if (method_ref->method_name == called_name && !method_ref->receiver_type->has_genericT_inside()) { - if (method_ref->receiver_type->equal_to(called_receiver)) { - candidates.emplace_back(method_ref, GenericsSubstitutions(method_ref->genericTs)); - } - } - } - if (!candidates.empty()) { - return candidates; +// a function `tuple.push(self, v: T) asm "TPUSH"` can't be called with T=Point (2 stack slots); +// almost all asm/built-in generic functions expect one stack slot, but there are exceptions +bool is_allowed_asm_generic_function_with_non1_width_T(FunctionPtr fun_ref, int idxT) { + // if a built-in function is marked with a special flag + if (fun_ref->is_variadic_width_T_allowed()) { + return true; } - // step2: find all methods where a receiver can accept provided, e.g. `int8.copy` / `int?.copy` / `(int|slice).copy` - for (FunctionPtr method_ref : G.all_methods) { - if (method_ref->method_name == called_name && !method_ref->receiver_type->has_genericT_inside()) { - if (method_ref->receiver_type->can_rhs_be_assigned(called_receiver)) { - candidates.emplace_back(method_ref, GenericsSubstitutions(method_ref->genericTs)); - } + // allow "Cell.hash", "map.isEmpty" and other methods that don't depend on internal structure + if (fun_ref->is_method() && idxT < fun_ref->genericTs->n_from_receiver) { + TypePtr receiver = fun_ref->receiver_type->unwrap_alias(); + if (const auto* r_withTs = receiver->try_as()) { + return r_withTs->struct_ref && r_withTs->struct_ref->name == "Cell"; } } - if (!candidates.empty()) { - return candidates; - } - // step 3: try to match generic receivers, e.g. `Container.copy` / `(T?|slice).copy` but NOT `T.copy` - for (FunctionPtr method_ref : G.all_methods) { - if (method_ref->method_name == called_name && method_ref->receiver_type->has_genericT_inside() && !method_ref->receiver_type->try_as()) { - try { - GenericSubstitutionsDeducing deducingTs(method_ref); - TypePtr replaced = deducingTs.auto_deduce_from_argument(method_ref->receiver_type, called_receiver); - if (!replaced->has_genericT_inside()) { - candidates.emplace_back(method_ref, deducingTs.flush()); - } - } catch (...) {} - } - } - if (!candidates.empty()) { - return candidates; - } - - // step 4: try to match `T.copy` - for (FunctionPtr method_ref : G.all_methods) { - if (method_ref->method_name == called_name && method_ref->receiver_type->try_as()) { - try { - GenericSubstitutionsDeducing deducingTs(method_ref); - TypePtr replaced = deducingTs.auto_deduce_from_argument(method_ref->receiver_type, called_receiver); - if (!replaced->has_genericT_inside()) { - candidates.emplace_back(method_ref, deducingTs.flush()); - } - } catch (...) {} - } - } - return candidates; + return false; } } // namespace tolk diff --git a/tolk/generics-helpers.h b/tolk/generics-helpers.h index c8f46ff27d..4cec94cd28 100644 --- a/tolk/generics-helpers.h +++ b/tolk/generics-helpers.h @@ -97,6 +97,7 @@ class GenericSubstitutionsDeducing { public: explicit GenericSubstitutionsDeducing(FunctionPtr fun_ref); explicit GenericSubstitutionsDeducing(StructPtr struct_ref); + explicit GenericSubstitutionsDeducing(const GenericsDeclaration* genericTs); TypePtr replace_Ts_with_currently_deduced(TypePtr orig) const; TypePtr auto_deduce_from_argument(TypePtr param_type, TypePtr arg_type); @@ -110,13 +111,10 @@ class GenericSubstitutionsDeducing { } }; -typedef std::pair MethodCallCandidate; - FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs); StructPtr instantiate_generic_struct(StructPtr struct_ref, GenericsSubstitutions&& substitutedTs); AliasDefPtr instantiate_generic_alias(AliasDefPtr alias_ref, GenericsSubstitutions&& substitutedTs); -FunctionPtr match_exact_method_for_call_not_generic(TypePtr called_receiver, std::string_view called_name); -std::vector match_methods_for_call_including_generic(TypePtr called_receiver, std::string_view called_name); +bool is_allowed_asm_generic_function_with_non1_width_T(FunctionPtr fun_ref, int idxT); } // namespace tolk diff --git a/tolk/overload-resolution.cpp b/tolk/overload-resolution.cpp new file mode 100644 index 0000000000..e82db073bb --- /dev/null +++ b/tolk/overload-resolution.cpp @@ -0,0 +1,231 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "overload-resolution.h" +#include "type-system.h" +#include "compiler-state.h" + +/* + * Find an exact method having a receiver type. + * + * Given: int.copy, T.copy, Container.copy + * > 5.copy(); // 1 + * > (5 as int8).copy(); // 2 with T=int8 + * > containerOfInt.copy(); // 3 with T=int + * > nullableContainerOfInt.copy(); // 2 with T=Container? + * + */ + +namespace tolk { + +// each next shape kind is more specific than another; +// e.g., between `T.copy` and `int.copy` we choose the second; +enum class ShapeKind { + GenericT, // T + Union, // U|V, T? + Primitive, // int, slice, address, ... + Tensor, // (A,B,...) + Instantiated, // Map, Container, Struct, ... +}; + +// for every receiver, we calculate "score": how deep and specific it is; +// e.g., between `Container` and `T` we choose the first; +// e.g., between `map` and `map>` we choose the second; +struct ShapeScore { + ShapeKind kind; + int depth; + + bool is_shape_better_than(const ShapeScore& rhs) const { + if (kind != rhs.kind) { + return kind > rhs.kind; + } + return depth > rhs.depth; + } + + bool operator==(const ShapeScore& rhs) const = default; +}; + +// calculate score for a receiver; +// note: it's an original receiver, with generics, not an instantiated one +static ShapeScore calculate_shape_score(TypePtr t) { + if (t->try_as()) { + return {ShapeKind::GenericT, 1}; + } + + if (const auto* t_union = t->try_as()) { + int d = 0; + for (TypePtr variant : t_union->variants) { + d = std::max(d, calculate_shape_score(variant).depth); + } + return {ShapeKind::Union, 1 + d}; + } + + if (const auto* t_tensor = t->try_as()) { + int d = 0; + for (TypePtr item : t_tensor->items) { + d = std::max(d, calculate_shape_score(item).depth); + } + return {ShapeKind::Tensor, 1 + d}; + } + + if (const auto* t_brackets = t->try_as()) { + int d = 0; + for (TypePtr item : t_brackets->items) { + d = std::max(d, calculate_shape_score(item).depth); + } + return {ShapeKind::Tensor, 1 + d}; + } + + if (const auto* t_instTs = t->try_as()) { + int d = 0; + for (TypePtr typeT : t_instTs->type_arguments) { + d = std::max(d, calculate_shape_score(typeT).depth); + } + return {ShapeKind::Instantiated, 1 + d}; + } + + if (const auto* t_alias = t->try_as()) { + return calculate_shape_score(t_alias->underlying_type); + } + + return {ShapeKind::Primitive, 1}; +} + +// tries to find Ts in `pattern` to reach `actual`; +// example: pattern=`map`, actual=`map` => T=int +// example: pattern=`Container`, actual=`Container>` => T=Container +static bool can_substitute_Ts_to_reach_actual(TypePtr pattern, TypePtr actual, const GenericsDeclaration* genericTs) { + GenericSubstitutionsDeducing deducingTs(genericTs); + TypePtr replaced = deducingTs.auto_deduce_from_argument(pattern, actual); + return replaced->equal_to(actual); +} + +// checks whether a generic typeA is more specific than typeB; +// example: `map` dominates `map`; +// example: `map>` dominates `map>` dominates `map>`; +// example: `map` and `map` are not comparable; +static bool is_more_specific_generic(TypePtr typeA, TypePtr typeB, const GenericsDeclaration* genericTsA, const GenericsDeclaration* genericTsB) { + // exists θ: θ(B)=A && not exists φ: φ(A)=B + return can_substitute_Ts_to_reach_actual(typeB, typeA, genericTsB) + && !can_substitute_Ts_to_reach_actual(typeA, typeB, genericTsA); +} + +// the main "overload resolution" entrypoint: given `obj.method()`, find best applicable methods; +// if there are many (no one is better than others), a caller side will emit "ambiguous call" +std::vector resolve_methods_for_call(TypePtr provided_receiver, std::string_view called_name) { + // find all methods theoretically applicable; we'll filter them by priority; + // for instance, if there is `T.method`, it will be instantiated with T=provided_receiver + std::vector viable; + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == called_name) { + TypePtr receiver = method_ref->receiver_type; + if (receiver->has_genericT_inside()) { + try { // check whether exist some T to make it a valid call (probably with type coercion) + GenericSubstitutionsDeducing deducingTs(method_ref); + TypePtr replaced = deducingTs.auto_deduce_from_argument(receiver, provided_receiver); + if (replaced->can_rhs_be_assigned(provided_receiver)) { + viable.emplace_back(receiver, replaced, method_ref, deducingTs.flush()); + } + } catch (...) {} + } else if (receiver->can_rhs_be_assigned(provided_receiver)) { + viable.emplace_back(receiver, receiver, method_ref, GenericsSubstitutions(method_ref->genericTs)); + } + } + } + // if nothing found, return nothing; + // if the only found, it's the one + if (viable.size() <= 1) { + return viable; + } + // okay, we have multiple viable methods, and need to locate the better + + // 1) exact match candidates with equal_to() + // (for instance, an alias equals to its underlying type, as well as `T1|T2` equals to `T2|T1`) + std::vector exact; + for (const MethodCallCandidate& candidate : viable) { + if (candidate.instantiated_receiver->equal_to(provided_receiver)) { + exact.push_back(candidate); + } + } + if (exact.size() == 1) { + return exact; + } + if (!exact.empty()) { + viable = std::move(exact); + } + + // 2) if there are both generic and non-generic functions, filter out generic + size_t n_generics = 0; + for (const MethodCallCandidate& candidate : viable) { + n_generics += candidate.is_generic(); + } + if (n_generics < viable.size()) { + std::vector non_generic; + for (const MethodCallCandidate& candidate : viable) { + if (!candidate.is_generic()) { + non_generic.push_back(candidate); + } + } + // all the code below is dedicated to choosing between generic Ts, so return if non-generic + return non_generic; + } + + // 3) better shape in terms of structural depth + // (prefer `Container` over `T` and `map>` over `map`) + ShapeScore best_shape = {ShapeKind::GenericT, -999}; + for (const MethodCallCandidate& candidate : viable) { + ShapeScore s = calculate_shape_score(candidate.original_receiver); + if (s.is_shape_better_than(best_shape)) { + best_shape = s; + } + } + + std::vector best_by_shape; + for (const MethodCallCandidate& candidate : viable) { + if (calculate_shape_score(candidate.original_receiver) == best_shape) { + best_by_shape.push_back(candidate); + } + } + if (best_by_shape.size() == 1) { + return best_by_shape; + } + if (!best_by_shape.empty()) { + viable = std::move(best_by_shape); + } + + // 4) find the overload that dominates all others + // (prefer `Container` over `Container` and `map` over `map`) + const MethodCallCandidate* dominator = nullptr; + for (const MethodCallCandidate& candidate : viable) { + bool dominates_all = true; + for (const MethodCallCandidate& other : viable) { + if (candidate.method_ref != other.method_ref) { + dominates_all &= is_more_specific_generic(candidate.original_receiver, other.original_receiver, candidate.method_ref->genericTs, other.method_ref->genericTs); + } + } + if (dominates_all) { + assert(!dominator); + dominator = &candidate; + } + } + + if (dominator) { + return {*dominator}; + } + return viable; +} + +} // namespace tolk diff --git a/tolk/overload-resolution.h b/tolk/overload-resolution.h new file mode 100644 index 0000000000..6960604bbf --- /dev/null +++ b/tolk/overload-resolution.h @@ -0,0 +1,45 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include +#include "fwd-declarations.h" +#include "generics-helpers.h" + +namespace tolk { + +// when there are many methods with the same name, the overload resolution mechanism +// analyzes possible candidates to resolve the best match +struct MethodCallCandidate { + TypePtr original_receiver; + TypePtr instantiated_receiver; + FunctionPtr method_ref; + GenericsSubstitutions substitutedTs; + + MethodCallCandidate(TypePtr original_receiver, TypePtr instantiated_receiver, FunctionPtr method_ref, GenericsSubstitutions&& substitutedTs) + : original_receiver(original_receiver) + , instantiated_receiver(instantiated_receiver) + , method_ref(method_ref) + , substitutedTs(std::move(substitutedTs)) {} + + bool is_generic() const { return original_receiver != instantiated_receiver; } +}; + + +std::vector resolve_methods_for_call(TypePtr provided_receiver, std::string_view called_name); + +} // namespace tolk diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 5c10d44d59..2fc1385d53 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -19,6 +19,7 @@ #include "ast.h" #include "ast-visitor.h" #include "generics-helpers.h" +#include "overload-resolution.h" #include "type-system.h" #include "smart-casts-cfg.h" #include @@ -223,16 +224,10 @@ static TypePtr try_pick_instantiated_generic_from_hint(TypePtr hint, AliasDefPtr // given `p.create` (called_receiver = Point, called_name = "create") // look up a corresponding method (it may be `Point.create` / `Point?.create` / `T.create`) -static MethodCallCandidate choose_only_method_to_call(FunctionPtr cur_f, SrcLocation loc, TypePtr called_receiver, std::string_view called_name) { - // most practical cases: `builder.storeInt` etc., when a direct method for receiver exists - if (FunctionPtr exact_method = match_exact_method_for_call_not_generic(called_receiver, called_name)) { - return {exact_method, GenericsSubstitutions(exact_method->genericTs)}; - } - - // else, try to match, for example `10.copy` with `int?.copy`, with `T.copy`, etc. - std::vector candidates = match_methods_for_call_including_generic(called_receiver, called_name); +static std::pair choose_only_method_to_call(FunctionPtr cur_f, SrcLocation loc, TypePtr called_receiver, std::string_view called_name) { + std::vector candidates = resolve_methods_for_call(called_receiver, called_name); if (candidates.size() == 1) { - return candidates[0]; + return {candidates[0].method_ref, candidates[0].substitutedTs}; } if (candidates.empty()) { // return nullptr, the caller side decides how to react on this return {nullptr, GenericsSubstitutions(nullptr)}; @@ -240,10 +235,11 @@ static MethodCallCandidate choose_only_method_to_call(FunctionPtr cur_f, SrcLoca std::ostringstream msg; msg << "call to method `" << called_name << "` for type `" << called_receiver << "` is ambiguous\n"; - for (const auto& [method_ref, substitutedTs] : candidates) { + for (const MethodCallCandidate& candidate : candidates) { + FunctionPtr method_ref = candidate.method_ref; msg << "candidate function: `" << method_ref->as_human_readable() << "`"; if (method_ref->is_generic_function()) { - msg << " with " << substitutedTs.as_human_readable(false); + msg << " with " << candidate.substitutedTs.as_human_readable(false); } if (method_ref->loc.is_defined()) { msg << " (declared at " << method_ref->loc << ")\n"; @@ -891,7 +887,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // T for asm function must be a TVM primitive (width 1), otherwise, asm would act incorrectly if (fun_ref->is_asm_function() || fun_ref->is_builtin_function()) { for (int i = 0; i < substitutedTs.size(); ++i) { - if (substitutedTs.typeT_at(i)->get_width_on_stack() != 1 && !fun_ref->is_variadic_width_T_allowed()) { + if (substitutedTs.typeT_at(i)->get_width_on_stack() != 1 && !is_allowed_asm_generic_function_with_non1_width_T(fun_ref, i)) { fire_error_calling_asm_function_with_non1_stack_width_arg(cur_f, loc, fun_ref, substitutedTs, i); } } @@ -989,7 +985,7 @@ class InferTypesAndCallsAndFieldsVisitor final { TypePtr receiver_type = obj_as_type->resolved_type; std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, v_ident->loc, receiver_type, v->get_field_name()); if (!fun_ref) { - fire(cur_f, v_ident->loc, "method `" + to_string(v->get_field_name()) + "` not found for type " + to_string(receiver_type)); + fire_error_method_not_found(cur_f, v_ident->loc, receiver_type, v->get_field_name()); } } } diff --git a/tolk/type-system.h b/tolk/type-system.h index 74a3bb5805..0fdbc26e69 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -561,7 +561,7 @@ class TypeDataUnion final : public TypeData { // true : `int?`, `slice?`, `StructWith1IntField?` // false: `(int, int)?`, `ComplexStruct?`, `()?` bool is_primitive_nullable() const { - return get_width_on_stack() == 1 && or_null != nullptr && or_null->get_width_on_stack() == 1; + return !has_genericT_inside() && get_width_on_stack() == 1 && or_null != nullptr && or_null->get_width_on_stack() == 1; } bool has_null() const { return or_null != nullptr || has_variant_equal_to(TypeDataNullLiteral::create()); From 3c10903dc59140e7dd13673f13d813d31ea88006 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 15 Sep 2025 16:15:09 +0300 Subject: [PATCH 390/524] Send validator telemetry from all nodes in fast-sync overlay (#1798) --- create-hardfork/create-hardfork.cpp | 3 -- test/test-ton-collator.cpp | 3 -- validator/full-node-fast-sync-overlays.cpp | 15 ++++++++ validator/full-node-fast-sync-overlays.hpp | 3 ++ validator/full-node-private-overlay.cpp | 24 +++++++++---- validator/full-node-private-overlay.hpp | 3 ++ validator/full-node.cpp | 21 ----------- validator/full-node.hpp | 1 - validator/interfaces/validator-manager.h | 1 - validator/manager-disk.hpp | 2 -- validator/manager-hardfork.hpp | 2 -- validator/manager.cpp | 42 ---------------------- validator/manager.hpp | 8 ----- validator/validator-telemetry.cpp | 4 +-- validator/validator-telemetry.hpp | 25 +++++++------ validator/validator.h | 1 - 16 files changed, 52 insertions(+), 106 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index e2f77dd86c..aadf877ca1 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -280,9 +280,6 @@ class HardforkCreator : public td::actor::Actor { void new_key_block(ton::validator::BlockHandle handle) override { } - void send_validator_telemetry(ton::PublicKeyHash key, - ton::tl_object_ptr telemetry) override { - } }; td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::install_callback, diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 89be586e62..3ee723ec88 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -381,9 +381,6 @@ class TestNode : public td::actor::Actor { void new_key_block(ton::validator::BlockHandle handle) override { } - void send_validator_telemetry(ton::PublicKeyHash key, - ton::tl_object_ptr telemetry) override { - } }; td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::install_callback, diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index 3b4a36c6db..e9927518a5 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -325,6 +325,21 @@ void FullNodeFastSyncOverlay::init() { std::make_unique(actor_id(this)), rules, std::move(scope), options); inited_ = true; + if (shard_.is_masterchain()) { + class TelemetryCallback : public ValidatorTelemetry::Callback { + public: + explicit TelemetryCallback(td::actor::ActorId id) : id_(id) { + } + void send_telemetry(tl_object_ptr telemetry) override { + td::actor::send_closure(id_, &FullNodeFastSyncOverlay::send_validator_telemetry, std::move(telemetry)); + } + + private: + td::actor::ActorId id_; + }; + telemetry_sender_ = td::actor::create_actor( + "telemetry", local_id_, std::make_unique(actor_id(this))); + } } void FullNodeFastSyncOverlay::tear_down() { diff --git a/validator/full-node-fast-sync-overlays.hpp b/validator/full-node-fast-sync-overlays.hpp index 41a45cbc71..3505ae5eba 100644 --- a/validator/full-node-fast-sync-overlays.hpp +++ b/validator/full-node-fast-sync-overlays.hpp @@ -17,6 +17,8 @@ #pragma once #include "full-node.h" +#include "validator-telemetry.hpp" + #include namespace ton::validator::fullnode { @@ -108,6 +110,7 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { void init(); void get_stats_extra(td::Promise promise); + td::actor::ActorOwn telemetry_sender_; bool collect_telemetry_ = false; std::ofstream telemetry_file_; }; diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index fd5e970ba0..7ea813a466 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -99,7 +99,7 @@ void FullNodePrivateBlockOverlay::process_block_candidate_broadcast(PublicKeyHas } void FullNodePrivateBlockOverlay::process_telemetry_broadcast( - PublicKeyHash src, const tl_object_ptr& telemetry) { + PublicKeyHash src, const tl_object_ptr &telemetry) { if (telemetry->adnl_id_ != src.bits256_value()) { VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": adnl_id mismatch"; return; @@ -117,9 +117,7 @@ void FullNodePrivateBlockOverlay::process_telemetry_broadcast( } VLOG(FULL_NODE_DEBUG) << "Got telemetry broadcast from " << src; auto s = td::json_encode(td::ToJson(*telemetry), false); - std::erase_if(s, [](char c) { - return c == '\n' || c == '\r'; - }); + std::erase_if(s, [](char c) { return c == '\n' || c == '\r'; }); telemetry_file_ << s << "\n"; telemetry_file_.flush(); if (telemetry_file_.fail()) { @@ -141,9 +139,7 @@ void FullNodePrivateBlockOverlay::receive_broadcast(PublicKeyHash src, td::Buffe } return; } - ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto& obj) { - Self->process_broadcast(src, obj); - }); + ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); }); } void FullNodePrivateBlockOverlay::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, @@ -286,6 +282,20 @@ void FullNodePrivateBlockOverlay::init() { td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); inited_ = true; + + class TelemetryCallback : public ValidatorTelemetry::Callback { + public: + explicit TelemetryCallback(td::actor::ActorId id) : id_(id) { + } + void send_telemetry(tl_object_ptr telemetry) override { + td::actor::send_closure(id_, &FullNodePrivateBlockOverlay::send_validator_telemetry, std::move(telemetry)); + } + + private: + td::actor::ActorId id_; + }; + telemetry_sender_ = td::actor::create_actor("telemetry", local_id_, + std::make_unique(actor_id(this))); } void FullNodePrivateBlockOverlay::tear_down() { diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index 355e448587..2cbe25f686 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -17,6 +17,8 @@ #pragma once #include "full-node.h" +#include "validator-telemetry.hpp" + #include namespace ton::validator::fullnode { @@ -100,6 +102,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { void try_init(); void init(); + td::actor::ActorOwn telemetry_sender_; bool collect_telemetry_ = false; std::ofstream telemetry_file_; }; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 890fd7a2eb..d648fc821b 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -625,24 +625,6 @@ void FullNodeImpl::new_key_block(BlockHandle handle) { } } -void FullNodeImpl::send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) { - if (use_old_private_overlays_) { - auto it = private_block_overlays_.find(key); - if (it == private_block_overlays_.end()) { - VLOG(FULL_NODE_INFO) << "Cannot send validator telemetry for " << key << " : no private block overlay"; - return; - } - td::actor::send_closure(it->second, &FullNodePrivateBlockOverlay::send_validator_telemetry, std::move(telemetry)); - } else { - auto overlay = fast_sync_overlays_.get_masterchain_overlay_for(adnl::AdnlNodeIdShort{telemetry->adnl_id_}); - if (overlay.empty()) { - VLOG(FULL_NODE_INFO) << "Cannot send validator telemetry for adnl id " << key << " : no fast sync overlay"; - return; - } - td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::send_validator_telemetry, std::move(telemetry)); - } -} - void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast) { send_block_broadcast_to_custom_overlays(broadcast); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_block_broadcast, std::move(broadcast), @@ -788,9 +770,6 @@ void FullNodeImpl::start_up() { void new_key_block(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::new_key_block, std::move(handle)); } - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { - td::actor::send_closure(id_, &FullNodeImpl::send_validator_telemetry, key, std::move(telemetry)); - } explicit Callback(td::actor::ActorId id) : id_(id) { } diff --git a/validator/full-node.hpp b/validator/full-node.hpp index dfd12f054c..482a916bbe 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -90,7 +90,6 @@ class FullNodeImpl : public FullNode { void got_key_block_config(td::Ref config); void new_key_block(BlockHandle handle); - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry); void process_block_broadcast(BlockBroadcast broadcast) override; void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index ade27c4eee..90e5247566 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -311,7 +311,6 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_ihr_message(td::Ref message) = 0; virtual void send_top_shard_block_description(td::Ref desc) = 0; virtual void send_block_broadcast(BlockBroadcast broadcast, int mode) = 0; - virtual void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) = 0; virtual void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index bdde74b2d9..1c4d00d720 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -275,8 +275,6 @@ class ValidatorManagerImpl : public ValidatorManager { void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast, int mode) override { } - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { - } void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) override { diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 5757c2bd49..37b5d6dd39 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -352,8 +352,6 @@ class ValidatorManagerImpl : public ValidatorManager { } void send_block_broadcast(BlockBroadcast broadcast, int mode) override { } - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { - } void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) override { diff --git a/validator/manager.cpp b/validator/manager.cpp index f61bd5f0b4..a5315788dc 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1867,11 +1867,6 @@ void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast, int mo callback_->send_broadcast(std::move(broadcast), mode); } -void ValidatorManagerImpl::send_validator_telemetry(PublicKeyHash key, - tl_object_ptr telemetry) { - callback_->send_validator_telemetry(key, std::move(telemetry)); -} - void ValidatorManagerImpl::send_get_out_msg_queue_proof_request( ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) { @@ -2065,7 +2060,6 @@ void ValidatorManagerImpl::started(ValidatorManagerInitResult R) { if (opts_->nonfinal_ls_queries_enabled()) { candidates_buffer_ = td::actor::create_actor("candidates-buffer", actor_id(this)); } - init_validator_telemetry(); auto Q = td::PromiseCreator::lambda( [SelfId = actor_id(this)](td::Result>> R) { @@ -2263,7 +2257,6 @@ void ValidatorManagerImpl::new_masterchain_block() { td::actor::send_closure(serializer_, &AsyncStateSerializer::update_last_known_key_block_ts, last_key_block_handle_->unix_time()); } - init_validator_telemetry(); } update_shard_overlays(); @@ -3725,41 +3718,6 @@ void ValidatorManagerImpl::CheckedExtMsgCounter::before_query() { } } -void ValidatorManagerImpl::init_validator_telemetry() { - if (last_masterchain_state_.is_null()) { - return; - } - td::Ref validator_set = last_masterchain_state_->get_total_validator_set(0); - if (validator_set.is_null()) { - validator_telemetry_.clear(); - return; - } - std::set processed; - for (auto &key : temp_keys_) { - if (const ValidatorDescr *desc = validator_set->get_validator(key.bits256_value())) { - processed.insert(key); - adnl::AdnlNodeIdShort adnl_id; - if (desc->addr.is_zero()) { - adnl_id = adnl::AdnlNodeIdShort{ValidatorFullId{desc->key}.compute_short_id()}; - } else { - adnl_id = adnl::AdnlNodeIdShort{desc->addr}; - } - auto &telemetry = validator_telemetry_[key]; - if (telemetry.empty()) { - telemetry = td::actor::create_actor("telemetry", key, adnl_id, - opts_->zero_block_id().file_hash, actor_id(this)); - } - } - } - for (auto it = validator_telemetry_.begin(); it != validator_telemetry_.end();) { - if (processed.contains(it->first)) { - ++it; - } else { - it = validator_telemetry_.erase(it); - } - } -} - template void ValidatorManagerImpl::write_session_stats(const T &obj) { std::string fname = opts_->get_session_logs_file(); diff --git a/validator/manager.hpp b/validator/manager.hpp index 68799c3726..b3fdbc344c 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -36,7 +36,6 @@ #include "token-manager.h" #include "queue-size-counter.hpp" #include "storage-stat-cache.hpp" -#include "validator-telemetry.hpp" #include "impl/candidates-buffer.hpp" #include "collator-node/collator-node.hpp" #include "shard-block-verifier.hpp" @@ -384,7 +383,6 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_temp_key(PublicKeyHash key, td::Promise promise) override { temp_keys_.insert(key); - init_validator_telemetry(); promise.set_value(td::Unit()); } void del_permanent_key(PublicKeyHash key, td::Promise promise) override { @@ -393,7 +391,6 @@ class ValidatorManagerImpl : public ValidatorManager { } void del_temp_key(PublicKeyHash key, td::Promise promise) override { temp_keys_.erase(key); - init_validator_telemetry(); promise.set_value(td::Unit()); } @@ -558,7 +555,6 @@ class ValidatorManagerImpl : public ValidatorManager { void send_ihr_message(td::Ref message) override; void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast, int mode) override; - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override; void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) override; @@ -824,10 +820,6 @@ class ValidatorManagerImpl : public ValidatorManager { void iterate_temp_block_handles(std::function f) override; - std::map> validator_telemetry_; - - void init_validator_telemetry(); - struct Collator { td::actor::ActorOwn actor; std::set shards; diff --git a/validator/validator-telemetry.cpp b/validator/validator-telemetry.cpp index 403dd6f9f2..b870e8b84d 100644 --- a/validator/validator-telemetry.cpp +++ b/validator/validator-telemetry.cpp @@ -51,7 +51,7 @@ void ValidatorTelemetry::start_up() { cpu_cores_ = r_cpu_cores.move_as_ok(); } - LOG(DEBUG) << "Initializing validator telemetry, key = " << key_ << ", adnl_id = " << local_id_; + LOG(DEBUG) << "Initializing validator telemetry, adnl_id = " << local_id_; alarm_timestamp().relax(send_telemetry_at_ = td::Timestamp::in(td::Random::fast(30.0, 60.0))); } @@ -81,7 +81,7 @@ void ValidatorTelemetry::send_telemetry() { .cpu_threads_count; LOG(DEBUG) << "Sending validator telemetry for adnl id " << local_id_; - td::actor::send_closure(manager_, &ValidatorManager::send_validator_telemetry, key_, std::move(telemetry)); + callback_->send_telemetry(std::move(telemetry)); } } // namespace ton::validator diff --git a/validator/validator-telemetry.hpp b/validator/validator-telemetry.hpp index 73908bdd10..d83641dfe4 100644 --- a/validator/validator-telemetry.hpp +++ b/validator/validator-telemetry.hpp @@ -33,23 +33,23 @@ namespace ton::validator { class ValidatorManager; class ValidatorTelemetry : public td::actor::Actor { -public: - ValidatorTelemetry(PublicKeyHash key, adnl::AdnlNodeIdShort local_id, td::Bits256 zero_state_file_hash, - td::actor::ActorId manager) - : key_(key) - , local_id_(local_id) - , zero_state_file_hash_(zero_state_file_hash) - , manager_(std::move(manager)) { + public: + class Callback { + public: + virtual ~Callback() = default; + virtual void send_telemetry(tl_object_ptr telemetry) = 0; + }; + + ValidatorTelemetry(adnl::AdnlNodeIdShort local_id, std::unique_ptr callback) + : local_id_(local_id), callback_(std::move(callback)) { } void start_up() override; void alarm() override; -private: - PublicKeyHash key_; + private: adnl::AdnlNodeIdShort local_id_; - td::Bits256 zero_state_file_hash_; - td::actor::ActorId manager_; + std::unique_ptr callback_; std::string node_version_; std::string os_version_; @@ -61,6 +61,5 @@ class ValidatorTelemetry : public td::actor::Actor { void send_telemetry(); static constexpr double PERIOD = 600.0; - static constexpr td::uint32 MAX_SIZE = 8192; }; -} // namespace ton::validator \ No newline at end of file +} // namespace ton::validator diff --git a/validator/validator.h b/validator/validator.h index e736149500..826fc8f8f7 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -242,7 +242,6 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise>> promise) = 0; virtual void new_key_block(BlockHandle handle) = 0; - virtual void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) = 0; }; virtual ~ValidatorManagerInterface() = default; From a7c001036139326a83fe7dbcc346f196c6430024 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 15 Sep 2025 16:20:00 +0300 Subject: [PATCH 391/524] Ban collator node when it returns invalid block (#1797) --- tl/generate/scheme/ton_api.tl | 2 +- tl/generate/scheme/ton_api.tlo | Bin 118952 -> 118988 bytes .../validator-engine-console-query.cpp | 3 + validator/collation-manager.cpp | 104 +++++++++++------- validator/collation-manager.hpp | 6 + validator/validator-group.cpp | 22 +++- validator/validator-group.hpp | 1 + 7 files changed, 95 insertions(+), 43 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 6e1a0fc4f0..d737dea5a9 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -786,7 +786,7 @@ engine.validator.shardOutQueueSize size:long = engine.validator.ShardOutQueueSiz engine.validator.exportedPrivateKeys encrypted_data:bytes = engine.validator.ExportedPrivateKeys; engine.validator.collationManagerStats.shard shard_id:tonNode.shardId self_collate:Bool select_mode:string active:Bool collators:(vector int256) = engine.validator.collationManagerStats.Shard; -engine.validator.collationManagerStats.collator adnl_id:int256 active:Bool alive:Bool ping_in:double last_ping_ago:double last_ping_status:string = engine.validator.collationManagerStats.Collator; +engine.validator.collationManagerStats.collator adnl_id:int256 active:Bool alive:Bool ping_in:double last_ping_ago:double last_ping_status:string banned_for:double = engine.validator.collationManagerStats.Collator; engine.validator.collationManagerStats.localId adnl_id:int256 shards:(vector engine.validator.collationManagerStats.shard) collators:(vector engine.validator.collationManagerStats.collator) = engine.validator.collationManagerStats.LocalId; engine.validator.collationManagerStats local_ids:(vector engine.validator.collationManagerStats.localId) = engine.validator.CollationManagerStats; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index ee5fb5ce817f1079c69a32c1625e810f0248b3a2..bf531b7af9deb192d7628c25692399ab8c7691e8 100644 GIT binary patch delta 92 zcmZ3nkp0X;_6;rytj=m&`TCO|DyVNRQ24R|!JRzeA_pf(^5!uiANI)$FJ7PAaFB(E jD=9HAFEu4TEx(9ivcfSL7Lbz3Etk%K8Jj~chx`QqORytm delta 76 zcmX@JkbT8M_6;rytd5*Y;`);xDyVNRQ24R|!JRzeA_pf(^5!uiAGXO0FJ7O%fsK)a S1tc_i#icV~#^#F4A%6h~5*wEQ diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 15f1264221..1a9ca052e2 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1856,6 +1856,9 @@ td::Status GetCollationManagerStatsQuery::receive(td::BufferSlice data) { } sb << td::StringBuilder::FixedDouble(collator->last_ping_ago_, 3) << ": " << status; } + if (collator->banned_for_ > 0.0) { + sb << " banned_for=" << td::StringBuilder::FixedDouble(std::max(collator->banned_for_, 0.0), 3); + } td::TerminalIO::out() << sb.as_cslice() << "\n"; } } diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index e842b76d46..7d694af985 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -137,53 +137,61 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas adnl::AdnlNodeIdShort selected_collator = adnl::AdnlNodeIdShort::zero(); size_t selected_idx = 0; - auto check_collator = [&](const adnl::AdnlNodeIdShort& id) -> bool { - auto& collator = collators_[id]; - if (!collator.alive) { - return false; - } - if (is_optimistic && collator.version < CollatorNode::VERSION_OPTIMISTIC_COLLATE) { - return false; - } - return true; - }; - switch (s->select_mode) { - case CollatorsList::mode_random: { - int cnt = 0; - for (size_t i = 0; i < s->collators.size(); ++i) { - adnl::AdnlNodeIdShort collator = s->collators[i]; - if (check_collator(collator)) { - ++cnt; - if (td::Random::fast(1, cnt) == 1) { + for (int allow_banned = 0; allow_banned < 2; ++allow_banned) { + auto check_collator = [&](const adnl::AdnlNodeIdShort& id) -> bool { + auto& collator = collators_[id]; + if (!collator.alive) { + return false; + } + if (collator.banned_until && !allow_banned) { + return false; + } + if (is_optimistic && collator.version < CollatorNode::VERSION_OPTIMISTIC_COLLATE) { + return false; + } + return true; + }; + switch (s->select_mode) { + case CollatorsList::mode_random: { + int cnt = 0; + for (size_t i = 0; i < s->collators.size(); ++i) { + adnl::AdnlNodeIdShort collator = s->collators[i]; + if (check_collator(collator)) { + ++cnt; + if (td::Random::fast(1, cnt) == 1) { + selected_collator = collator; + selected_idx = i; + } + } + } + break; + } + case CollatorsList::mode_ordered: { + for (size_t i = 0; i < s->collators.size(); ++i) { + adnl::AdnlNodeIdShort collator = s->collators[i]; + if (check_collator(collator)) { selected_collator = collator; selected_idx = i; + break; } } + break; } - break; - } - case CollatorsList::mode_ordered: { - for (size_t i = 0; i < s->collators.size(); ++i) { - adnl::AdnlNodeIdShort collator = s->collators[i]; - if (check_collator(collator)) { - selected_collator = collator; - selected_idx = i; - break; + case CollatorsList::mode_round_robin: { + size_t iters = 0; + for (size_t i = s->cur_idx; iters < s->collators.size(); (++i) %= s->collators.size(), ++iters) { + adnl::AdnlNodeIdShort& collator = s->collators[i]; + if (check_collator(collator)) { + selected_collator = collator; + selected_idx = i; + s->cur_idx = (i + 1) % s->collators.size(); + break; + } } + break; } - break; } - case CollatorsList::mode_round_robin: { - size_t iters = 0; - for (size_t i = s->cur_idx; iters < s->collators.size(); (++i) %= s->collators.size(), ++iters) { - adnl::AdnlNodeIdShort& collator = s->collators[i]; - if (check_collator(collator)) { - selected_collator = collator; - selected_idx = i; - s->cur_idx = (i + 1) % s->collators.size(); - break; - } - } + if (!selected_collator.is_zero() || s->self_collate) { break; } } @@ -372,11 +380,21 @@ void CollationManager::get_stats( } obj->last_ping_ago_ = collator.last_ping_at ? td::Time::now() - collator.last_ping_at.at() : -1.0; obj->last_ping_status_ = collator.last_ping_status.is_ok() ? "OK" : collator.last_ping_status.message().str(); + obj->banned_for_ = collator.banned_until ? collator.banned_until.in() : -1.0; stats->collators_.push_back(std::move(obj)); } promise.set_value(std::move(stats)); } +void CollationManager::ban_collator(adnl::AdnlNodeIdShort collator_id, std::string reason) { + auto it = collators_.find(collator_id); + if (it == collators_.end()) { + return; + } + alarm_timestamp().relax(it->second.banned_until = td::Timestamp::in(BAN_DURATION)); + LOG(ERROR) << "Ban collator " << collator_id << " for " << BAN_DURATION << "s: " << reason; +} + void CollationManager::update_collators_list(const CollatorsList& collators_list) { shards_.clear(); for (auto& [_, collator] : collators_) { @@ -427,6 +445,14 @@ CollationManager::ShardInfo* CollationManager::select_shard_info(ShardIdFull sha void CollationManager::alarm() { alarm_timestamp() = td::Timestamp::never(); for (auto& [id, collator] : collators_) { + if (collator.banned_until) { + if (collator.banned_until.is_in_past()) { + collator.banned_until = {}; + LOG(ERROR) << "Unban collator " << id; + } else { + alarm_timestamp().relax(collator.banned_until); + } + } if (collator.active_cnt == 0 || collator.sent_ping) { continue; } diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp index 37a45fb38a..6691a387ea 100644 --- a/validator/collation-manager.hpp +++ b/validator/collation-manager.hpp @@ -54,6 +54,8 @@ class CollationManager : public td::actor::Actor { void get_stats(td::Promise> promise); + void ban_collator(adnl::AdnlNodeIdShort collator_id, std::string reason); + private: adnl::AdnlNodeIdShort local_id_; td::Ref opts_; @@ -77,6 +79,8 @@ class CollationManager : public td::actor::Actor { td::Timestamp last_ping_at = td::Timestamp::never(); td::Status last_ping_status = td::Status::Error("not pinged"); int version = -1; + // Collator is banned when in returns invalid block + td::Timestamp banned_until = td::Timestamp::never(); }; std::map collators_; @@ -104,6 +108,8 @@ class CollationManager : public td::actor::Actor { size_t refcnt = 0; }; std::map optimistic_prev_cache_; + + static constexpr double BAN_DURATION = 300.0; }; } // namespace ton::validator diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 5261560e8b..c79dbda4ac 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -117,6 +117,9 @@ void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo if (need_send_candidate_broadcast(source_info, shard_.is_masterchain())) { send_block_candidate_broadcast(c.candidate.id, c.candidate.data.clone()); } + if (!c.self_collated) { + block_collator_node_id_[c.candidate.id] = adnl::AdnlNodeIdShort{c.collator_node_id}; + } cache->result = std::move(c); for (auto &p : cache->promises) { p.set_value(cache->result.value().clone()); @@ -182,10 +185,15 @@ void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo return; } + auto it2 = block_collator_node_id_.find(block.id); + adnl::AdnlNodeIdShort collator_node_id = + it2 == block_collator_node_id_.end() ? adnl::AdnlNodeIdShort::zero() : it2->second; + auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), source_info, block = block.clone(), + [=, SelfId = actor_id(this), block = block.clone(), optimistic_prev_block = is_optimistic ? optimistic_prev_block.value().clone() : td::optional{}, - promise = std::move(promise)](td::Result R) mutable { + promise = std::move(promise), + collation_manager = collation_manager_](td::Result R) mutable { if (R.is_error()) { auto S = R.move_as_error(); if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { @@ -212,6 +220,10 @@ void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo promise.set_value({ts, false}); }, [&](CandidateReject reject) { + if (!collator_node_id.is_zero()) { + td::actor::send_closure(collation_manager, &CollationManager::ban_collator, collator_node_id, + PSTRING() << "bad candidate " << block.id.to_str() << " : " << reject.reason); + } promise.set_error( td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad candidate: " << reject.reason)); })); @@ -396,7 +408,11 @@ void ValidatorGroup::generated_block_optimistic(validatorsession::BlockSourceInf optimistic_generation_ = {}; return; } - optimistic_generation_->result = R.move_as_ok(); + GeneratedCandidate c = R.move_as_ok(); + if (!c.self_collated) { + block_collator_node_id_[c.candidate.id] = adnl::AdnlNodeIdShort{c.collator_node_id}; + } + optimistic_generation_->result = std::move(c); for (auto &promise : optimistic_generation_->promises) { promise.set_result(optimistic_generation_->result.value().clone()); } diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 56d89dda45..3df456efa3 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -188,6 +188,7 @@ class ValidatorGroup : public td::actor::Actor { } std::set sent_candidate_broadcasts_; + std::map block_collator_node_id_; void send_block_candidate_broadcast(BlockIdExt id, td::BufferSlice data); From 41849db9133dff529e6316015e9512035c05e0c8 Mon Sep 17 00:00:00 2001 From: mkiesel Date: Mon, 15 Sep 2025 20:46:12 +0200 Subject: [PATCH 392/524] avoid touching packfiles on startup (#1793) ArchiveFile::start_up() truncates packfiles unconditionally, causing unneeded filesystem writes (mtime) and complicating auditing on file level. Co-authored-by: mkiesel --- validator/db/package.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/validator/db/package.cpp b/validator/db/package.cpp index cc699baa82..be96164229 100644 --- a/validator/db/package.cpp +++ b/validator/db/package.cpp @@ -48,8 +48,16 @@ Package::Package(td::FileFd fd) : fd_(std::move(fd)) { } td::Status Package::truncate(td::uint64 size) { - TRY_STATUS(fd_.seek(size + header_size())); - return fd_.truncate_to_current_position(size + header_size()); + auto target_size = size + header_size(); + TRY_RESULT(current_size, fd_.get_size()); + + // Only truncate if the size actually differs to avoid updating mtime unnecessarily + if (current_size != target_size) { + TRY_STATUS(fd_.seek(target_size)); + return fd_.truncate_to_current_position(target_size); + } + + return td::Status::OK(); } td::uint64 Package::append(std::string filename, td::Slice data, bool sync) { From 37656974215bb52d914b85c465dd823f476c8894 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 8 Aug 2025 13:52:34 +0700 Subject: [PATCH 393/524] [Tolk] Built-in map as a high-level wrapper over TVM dicts --- crypto/smartcont/tolk-stdlib/common.tolk | 309 +++++++- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 51 +- tolk-tester/tests/dicts-demo.tolk | 21 + tolk-tester/tests/generics-2.tolk | 38 +- tolk-tester/tests/handle-msg-6.tolk | 4 +- tolk-tester/tests/inference-tests.tolk | 2 +- .../tests/invalid-declaration/err-1709.tolk | 8 + .../tests/invalid-declaration/err-1710.tolk | 14 + .../tests/invalid-declaration/err-1711.tolk | 13 + .../tests/invalid-declaration/err-1806.tolk | 12 + .../tests/invalid-declaration/err-1945.tolk | 22 + .../tests/invalid-semantics/err-4109.tolk | 17 + .../tests/invalid-typing/err-6639.tolk | 12 + .../tests/invalid-typing/err-6761.tolk | 15 + .../tests/invalid-typing/err-6802.tolk | 10 + tolk-tester/tests/maps-tests.tolk | 682 +++++++++++++++++ tolk-tester/tests/overloads-tests.tolk | 29 + tolk-tester/tests/pack-unpack-2.tolk | 58 ++ tolk-tester/tests/send-msg-1.tolk | 22 +- tolk/CMakeLists.txt | 1 + tolk/ast-visitor.h | 1 + tolk/builtins.cpp | 184 +++++ tolk/compiler-state.cpp | 4 + tolk/generics-helpers.cpp | 37 + tolk/maps-kv-api.cpp | 723 ++++++++++++++++++ tolk/maps-kv-api.h | 27 + tolk/optimize.cpp | 39 +- tolk/overload-resolution.cpp | 5 + tolk/pack-unpack-api.cpp | 3 + tolk/pack-unpack-serializers.cpp | 30 + tolk/pack-unpack-serializers.h | 3 + tolk/pipe-check-serialized-fields.cpp | 62 +- tolk/pipe-resolve-types.cpp | 12 + tolk/send-message-api.cpp | 11 - tolk/tolk.h | 1 + tolk/type-system.cpp | 57 ++ tolk/type-system.h | 29 + 37 files changed, 2506 insertions(+), 62 deletions(-) create mode 100644 tolk-tester/tests/invalid-declaration/err-1709.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1710.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1711.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1806.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1945.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4109.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6639.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6761.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6802.tolk create mode 100644 tolk-tester/tests/maps-tests.tolk create mode 100644 tolk/maps-kv-api.cpp create mode 100644 tolk/maps-kv-api.h diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 7d21dd7d9d..d42ff93661 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -1,14 +1,8 @@ // Standard library for Tolk (LGPL licence). // It contains common functions that are available out of the box, the user doesn't have to import anything. -// More specific functions are required to be imported explicitly, like "@stdlib/tvm-dicts". +// More specific functions are required to be imported explicitly, like "@stdlib/gas-payments". tolk 1.0 -/// In Tolk v1.x there would be a type `map`. -/// Currently, working with dictionaries is still low-level, with raw cells. -/// But just for clarity, we use "dict" instead of a "cell?" where a cell-dictionary is assumed. -/// Every dictionary object can be null. TVM NULL is essentially "empty dictionary". -type dict = cell? - /** Tuple manipulation primitives. Elements of a tuple can be of arbitrary type. @@ -150,11 +144,9 @@ fun contract.getAddress(): address fun contract.getOriginalBalance(): coins asm "BALANCE" "FIRST" -/// Same as [contract.getOriginalBalance], but returns a tuple: -/// `int` — balance in nanotoncoins; -/// `dict` — a dictionary with 32-bit keys representing the balance of "extra currencies". +/// Same as [contract.getOriginalBalance], but returns a tuple: balance in nanotoncoins and extra currencies @pure -fun contract.getOriginalBalanceWithExtraCurrencies(): [coins, dict] +fun contract.getOriginalBalanceWithExtraCurrencies(): [coins, ExtraCurrenciesMap] asm "BALANCE" /// Returns the persistent contract storage cell. It can be parsed or modified with slice and builder primitives later. @@ -614,6 +606,289 @@ fun debug.dumpStack(): void builtin +/* + Implementation of `map` and methods over it. + + Note: maps in TVM are called "dictionaries". They are stored as nested cells + (each value in a separate cell, probably with intermediate cells-nodes). + Constructing even small dicts is gas-expensive, because cell creation costs a lot. + + `map` is a "human-readable interface" over low-level dictionaries. The compiler, + knowing types of K and V, effectively generates asm instructions for storing/loading values. + But still, of course, it's a tree of cells, it's just a TVM dictionary. + */ + +/// `map` is a built-in high-level type. Internally, it's an optional cell. +/// - an empty map is represented as `null`. +/// - a non-empty map points to a root cell. +/// Note about some restrictions for K and V types: +/// - a key must be fixed-with; valid: int32, uint64, address, bits256, Point; invalid: int, coins +/// - a value must be serializable; valid: int32, coins, AnyStruct, Cell; invalid: int, builder +struct map { + tvmDict: cell? +} + +/// A low-level TVM dictionary is called "dict". +/// Think of it as "a map with unknown keys and unknown values". +type dict = cell? + +/// Returns an empty typed map. It's essentially "PUSHNULL", since TVM NULL represents an empty map. +/// Example: +/// ``` +/// var m: map = createEmptyMap(); +/// ``` +@pure +fun createEmptyMap(): map + builtin + +/// Converts a low-level TVM dictionary to a typed map. +/// Actually, does nothing: accepts an "optional cell" and returns the same "optional cell", +/// so if you specify key/value types incorrectly, it will fail later, at [map.get] and similar. +@pure +fun createMapFromLowLevelDict(d: dict): map + builtin + +/// Converts a high-level map to a low-level TVM dictionary. +/// Actually, does nothing: returns the same "optional cell". +@pure +fun map.toLowLevelDict(self): dict + asm "NOP" + +/// Checks whether a map is empty (whether a cell is null). +/// Note: a check `m == null` will not work, use `m.isEmpty()`. +@pure +fun map.isEmpty(self): bool + asm "DICTEMPTY" + +/// Checks whether a key exists in a map. +@pure +fun map.exists(self, key: K): bool + builtin + +/// Gets an element by key. If not found, does NOT throw, just returns isFound = false. +/// Example: +/// ``` +/// val r = m.get(123); +/// if (r.isFound) { +/// r.loadValue() +/// } +/// ``` +@pure +fun map.get(self, key: K): MapLookupResult + builtin + +/// Gets an element by key and throws if it doesn't exist. +@pure +fun map.mustGet(self, key: K, throwIfNotFound: int = 9): V + builtin + +/// Sets an element by key. +/// Example: +/// ``` +/// m.set(k, 3); +/// ``` +/// Since it returns `self`, calls may be chained. +@pure +fun map.set(mutate self, key: K, value: V): self + builtin + +/// Sets an element and returns the previous element at that key. If no previous, isFound = false. +/// Example: +/// ``` +/// val prev = m.setAndGetPrevious(k, 3); +/// if (prev.isFound) { +/// prev.loadValue() +/// } +/// ``` +@pure +fun map.setAndGetPrevious(mutate self, key: K, value: V): MapLookupResult + builtin + +/// Sets an element only if the key already exists. Returns whether an element was replaced. +@pure +fun map.replaceIfExists(mutate self, key: K, value: V): bool + builtin + +/// Sets an element only if the key already exists and returns the previous element at that key. +@pure +fun map.replaceAndGetPrevious(mutate self, key: K, value: V): MapLookupResult + builtin + +/// Sets an element only if the key does not exist. Returns whether an element was added. +@pure +fun map.addIfNotExists(mutate self, key: K, value: V): bool + builtin + +/// Sets an element only if the key does not exist. If exists, returns an old value. +@pure +fun map.addOrGetExisting(mutate self, key: K, value: V): MapLookupResult + builtin + +/// Delete an element at the key. Returns whether an element was deleted. +@pure +fun map.delete(mutate self, key: K): bool + builtin + +/// Delete an element at the key and returns the deleted element. If not exists, isFound = false. +/// Example: +/// ``` +/// val prev = m.deleteAndGetDeleted(k); +/// if (prev.isFound) { +/// prev.loadValue() +/// } +/// ``` +@pure +fun map.deleteAndGetDeleted(mutate self, key: K): MapLookupResult + builtin + +/// Finds the first (minimal) element in a map. If key are integers, it's the minimal integer. +/// If keys are addresses or complex structures (represented as slices), it's lexicographically smallest. +/// For an empty map, just returns isFound = false. +/// Useful for iterating over a map: +/// ``` +/// var r = m.findFirst(); +/// while (r.isFound) { +/// ... use r.getKey() and r.loadValue() +/// r = m.iterateNext(r) +/// } +/// ``` +@pure +fun map.findFirst(self): MapEntry + builtin + +/// Finds the last (maximal) element in a map. If key are integers, it's the maximal integer. +/// If keys are addresses or complex structures (represented as slices), it's lexicographically largest. +/// For an empty map, just returns isFound = false. +/// Useful for iterating over a map: +/// ``` +/// var r = m.findLast(); +/// while (r.isFound) { +/// ... use r.getKey() and r.loadValue() +/// r = m.iteratePrev(r) +/// } +/// ``` +@pure +fun map.findLast(self): MapEntry + builtin + +/// Finds an element with key > pivotKey. +/// Don't forget to check `isFound` before using `getKey()` and `loadValue()` of the result. +@pure +fun map.findKeyGreater(self, pivotKey: K): MapEntry + builtin + +/// Finds an element with key >= pivotKey. +/// Don't forget to check `isFound` before using `getKey()` and `loadValue()` of the result. +@pure +fun map.findKeyGreaterOrEqual(self, pivotKey: K): MapEntry + builtin + +/// Finds an element with key < pivotKey. +/// Don't forget to check `isFound` before using `getKey()` and `loadValue()` of the result. +@pure +fun map.findKeyLess(self, pivotKey: K): MapEntry + builtin + +/// Finds an element with key <= pivotKey. +/// Don't forget to check `isFound` before using `getKey()` and `loadValue()` of the result. +@pure +fun map.findKeyLessOrEqual(self, pivotKey: K): MapEntry + builtin + +/// Iterate over a map in ascending order. +/// Example: +/// ``` +/// // iterate for all keys >= 10 up to the end +/// var r = m.findKeyGreaterOrEqual(10); +/// while (r.isFound) { +/// ... use r.getKey() and r.loadValue() +/// r = m.iterateNext(r) +/// } +/// ``` +@pure +fun map.iterateNext(self, current: MapEntry): MapEntry + builtin + +/// Iterate over a map in reverse order. +/// Example: +/// ``` +/// // iterate for all keys < 10 down lo lowest +/// var r = m.findKeyLess(10); +/// while (r.isFound) { +/// ... use r.getKey() and r.loadValue() +/// r = m.iteratePrev(r) +/// } +/// ```@pure +fun map.iteratePrev(self, current: MapEntry): MapEntry + builtin + +/// MapLookupResult is a return value of [map.get], [map.setAndGetPrevious], and similar. +/// Instead of returning a nullable value (that you'd check on null before usage), +/// this struct is returned (and you check on `isFound` before usage). +/// Example: +/// ``` +/// val r = m.get(key); +/// if (r.isFound) { +/// r.loadValue() // unpacks a returned slice +/// } +/// ``` +struct MapLookupResult { + rawSlice: slice? // holds encoded value, present if isFound + isFound: bool +} + +@pure +fun MapLookupResult.loadValue(self): TValue { + // it's assumed that you check for `isFound` before calling `loadValue()`; + // note that `assertEnd` is called to ensure no remaining data left besides TValue + return TValue.fromSlice(self.rawSlice!, { + assertEndAfterReading: true + }) +} + +@pure +fun MapLookupResult.loadValue(self): slice { + return self.rawSlice! +} + +/// MapEntry is a return value of [map.findFirst], [map.iterateNext], and similar. +/// You should check for `isFound` before calling `getKey()` and `loadValue()`. +/// Example: +/// ``` +/// var r = m.findFirst(); +/// while (r.isFound) { +/// ... use r.getKey() and r.loadValue() +/// r = m.iterateNext(r) +/// } +/// ``` +struct MapEntry { + rawValue: slice? // holds encoded value, present if isFound + key: K + isFound: bool +} + +@pure +fun MapEntry.getKey(self): K { + // it's assumed that you check for `isFound` before calling `getKey()`; + // (otherwise, it will contain an incorrect stack slot with `null` value, not K) + return self.key +} + +@pure +fun MapEntry.loadValue(self): V { + // it's assumed that you check for `isFound` before calling `loadValue()`; + // note that `assertEnd` is inserted to ensure no remaining data left besides TValue + return V.fromSlice(self.rawValue!, { + assertEndAfterReading: true + }) +} + +@pure +fun MapEntry.loadValue(self): slice { + return self.rawValue! +} + + /** Slice primitives: parsing cells. When you _load_ some data, you mutate the slice (shifting an internal pointer on the stack). @@ -715,7 +990,6 @@ fun slice.getMiddleBits(self, offset: int, len: int): slice asm "SDSUBSTR" /// Loads a dictionary (TL HashMapE structure, represented as TVM cell) from a slice. -/// Returns `null` if `nothing` constructor is used. @pure fun slice.loadDict(mutate self): dict asm( -> 1 0) "LDDICT" @@ -993,7 +1267,10 @@ const SEND_MODE_ESTIMATE_FEE_ONLY = 1024 /// +64 substitutes the entire balance of the incoming message as an outcoming value (slightly inaccurate, gas expenses that cannot be estimated before the computation is completed are not taken into account). /// +128 substitutes the value of the entire balance of the contract before the start of the computation phase (slightly inaccurate, since gas expenses that cannot be estimated before the completion of the computation phase are not taken into account). -type ExtraCurrenciesDict = dict +/// `ExtraCurrenciesMap` represents a dictionary of "extra currencies" other than TON coin. +/// They can be attached to a message (incoming and outgoing) and stored on contract's balance. +/// It's a dictionary `[currencyID => amount]` (amount is like `coins`, but encoded with a high precision). +type ExtraCurrenciesMap = map /// ContractState is "code + data" of a contract. /// Used in outgoing messages (StateInit) to initialize a destination contract. @@ -1062,7 +1339,7 @@ struct CreateMessageOptions { /// whether a message will bounce back on error bounce: bool /// message value: attached tons (or tons + extra currencies) - value: coins | (coins, ExtraCurrenciesDict) + value: coins | (coins, ExtraCurrenciesMap) /// destination is either a provided address, or is auto-calculated by stateInit dest: address // either just send a message to some address | builder // ... or a manually constructed builder with a valid address @@ -1312,7 +1589,7 @@ fun sendRawMessage(msg: cell, mode: int): void struct InMessage { senderAddress: address // an internal address from which the message arrived valueCoins: coins // ton amount attached to an incoming message - valueExtra: dict // extra currencies attached to an incoming message + valueExtra: ExtraCurrenciesMap // extra currencies attached to an incoming message originalForwardFee: coins // fee that was paid by the sender createdLt: uint64 // logical time when a message was created createdAt: uint32 // unixtime when a message was created @@ -1332,7 +1609,7 @@ struct InMessage { struct InMessageBounced { senderAddress: address // an internal address from which the message was bounced valueCoins: coins // ton amount attached to a message - valueExtra: dict // extra currencies attached to a message + valueExtra: ExtraCurrenciesMap // extra currencies attached to a message originalForwardFee: coins // comission that the sender has payed to send this message createdLt: uint64 // logical time when a message was created (and bounced) createdAt: uint32 // unixtime when a message was created (and bounced) diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index 41e091f0aa..289c58e66b 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -2,8 +2,11 @@ tolk 1.0 /** - Dictionaries are represented as `cell` data type (cells can store anything, dicts in particular). - Currently, they have very low-level API very close to TVM internals. + Low-level API working with TVM dictionaries. + Not recommended to use, since it's very complicated and error-prone. + Use `map` instead, it's even more efficient then low-level dict API. + + Dictionaries are represented as `cell` data type. Most of functions are duplicated for three common cases: - iDict* - dicts with signed integer keys - uDict* - dicts with unsigned integer keys @@ -86,28 +89,28 @@ fun dict.uDictSetIfExists(mutate self, keyLen: int, key: int, value: slice): boo @pure -fun dict.iDictGetRef(self, keyLen: int, key: int): (dict, bool) +fun dict.iDictGetRef(self, keyLen: int, key: int): (cell?, bool) asm(key self keyLen) "DICTIGETREF" "NULLSWAPIFNOT" @pure -fun dict.uDictGetRef(self, keyLen: int, key: int): (dict, bool) +fun dict.uDictGetRef(self, keyLen: int, key: int): (cell?, bool) asm(key self keyLen) "DICTUGETREF" "NULLSWAPIFNOT" @pure -fun dict.sDictGetRef(self, keyLen: int, key: slice): (dict, bool) +fun dict.sDictGetRef(self, keyLen: int, key: slice): (cell?, bool) asm(key self keyLen) "DICTGETREF" "NULLSWAPIFNOT" @pure -fun dict.iDictGetRefOrNull(self, keyLen: int, key: int): dict +fun dict.iDictGetRefOrNull(self, keyLen: int, key: int): cell? asm(key self keyLen) "DICTIGETOPTREF" @pure -fun dict.uDictGetRefOrNull(self, keyLen: int, key: int): dict +fun dict.uDictGetRefOrNull(self, keyLen: int, key: int): cell? asm(key self keyLen) "DICTUGETOPTREF" @pure -fun dict.sDictGetRefOrNull(self, keyLen: int, key: slice): dict +fun dict.sDictGetRefOrNull(self, keyLen: int, key: slice): cell? asm(key self keyLen) "DICTGETOPTREF" @@ -138,11 +141,11 @@ fun dict.sDictSetAndGet(mutate self, keyLen: int, key: slice, value: slice): (sl @pure -fun dict.iDictSetAndGetRefOrNull(mutate self, keyLen: int, key: int, value: cell): dict +fun dict.iDictSetAndGetRefOrNull(mutate self, keyLen: int, key: int, value: cell): cell? asm(value key self keyLen) "DICTISETGETOPTREF" @pure -fun dict.uDictSetAndGetRefOrNull(mutate self, keyLen: int, key: int, value: cell): dict +fun dict.uDictSetAndGetRefOrNull(mutate self, keyLen: int, key: int, value: cell): cell? asm(value key self keyLen) "DICTUSETGETOPTREF" @@ -228,15 +231,15 @@ fun dict.sDictGetFirst(self, keyLen: int): (slice?, slice?, bool) asm (-> 1 0 2) "DICTMIN" "NULLSWAPIFNOT2" @pure -fun dict.iDictGetFirstAsRef(self, keyLen: int): (int?, dict, bool) +fun dict.iDictGetFirstAsRef(self, keyLen: int): (int?, cell?, bool) asm (-> 1 0 2) "DICTIMINREF" "NULLSWAPIFNOT2" @pure -fun dict.uDictGetFirstAsRef(self, keyLen: int): (int?, dict, bool) +fun dict.uDictGetFirstAsRef(self, keyLen: int): (int?, cell?, bool) asm (-> 1 0 2) "DICTUMINREF" "NULLSWAPIFNOT2" @pure -fun dict.sDictGetFirstAsRef(self, keyLen: int): (slice?, dict, bool) +fun dict.sDictGetFirstAsRef(self, keyLen: int): (slice?, cell?, bool) asm (-> 1 0 2) "DICTMINREF" "NULLSWAPIFNOT2" @@ -253,15 +256,15 @@ fun dict.sDictGetLast(self, keyLen: int): (slice?, slice?, bool) asm (-> 1 0 2) "DICTMAX" "NULLSWAPIFNOT2" @pure -fun dict.iDictGetLastAsRef(self, keyLen: int): (int?, dict, bool) +fun dict.iDictGetLastAsRef(self, keyLen: int): (int?, cell?, bool) asm (-> 1 0 2) "DICTIMAXREF" "NULLSWAPIFNOT2" @pure -fun dict.uDictGetLastAsRef(self, keyLen: int): (int?, dict, bool) +fun dict.uDictGetLastAsRef(self, keyLen: int): (int?, cell?, bool) asm (-> 1 0 2) "DICTUMAXREF" "NULLSWAPIFNOT2" @pure -fun dict.sDictGetLastAsRef(self, keyLen: int): (slice?, dict, bool) +fun dict.sDictGetLastAsRef(self, keyLen: int): (slice?, cell?, bool) asm (-> 1 0 2) "DICTMAXREF" "NULLSWAPIFNOT2" @@ -273,6 +276,10 @@ fun dict.iDictGetNext(self, keyLen: int, pivot: int): (int?, slice?, bool) fun dict.uDictGetNext(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXT" "NULLSWAPIFNOT2" +@pure +fun dict.sDictGetNext(self, keyLen: int, pivot: slice): (slice?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTGETNEXT" "NULLSWAPIFNOT2" + @pure fun dict.iDictGetNextOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXTEQ" "NULLSWAPIFNOT2" @@ -281,6 +288,10 @@ fun dict.iDictGetNextOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool fun dict.uDictGetNextOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXTEQ" "NULLSWAPIFNOT2" +@pure +fun dict.sDictGetNextOrEqual(self, keyLen: int, pivot: slice): (slice?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTGETNEXTEQ" "NULLSWAPIFNOT2" + @pure fun dict.iDictGetPrev(self, keyLen: int, pivot: int): (int?, slice?, bool) @@ -290,6 +301,10 @@ fun dict.iDictGetPrev(self, keyLen: int, pivot: int): (int?, slice?, bool) fun dict.uDictGetPrev(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETPREV" "NULLSWAPIFNOT2" +@pure +fun dict.sDictGetPrev(self, keyLen: int, pivot: slice): (slice?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTGETPREV" "NULLSWAPIFNOT2" + @pure fun dict.iDictGetPrevOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETPREVEQ" "NULLSWAPIFNOT2" @@ -298,6 +313,10 @@ fun dict.iDictGetPrevOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool fun dict.uDictGetPrevOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETPREVEQ" "NULLSWAPIFNOT2" +@pure +fun dict.sDictGetPrevOrEqual(self, keyLen: int, pivot: slice): (slice?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTGETPREVEQ" "NULLSWAPIFNOT2" + /** Prefix dictionary primitives. diff --git a/tolk-tester/tests/dicts-demo.tolk b/tolk-tester/tests/dicts-demo.tolk index f53debb7f7..cec83280b2 100644 --- a/tolk-tester/tests/dicts-demo.tolk +++ b/tolk-tester/tests/dicts-demo.tolk @@ -104,6 +104,26 @@ fun test105(takeNext: bool) { return data!.loadUint(8); } +@method_id(106) +fun test106() { + var dict = createEmptyDict(); + dict.addIntToIDict(2, 102); + dict.addIntToIDict(-1, 101); + dict.addIntToIDict(-4, 104); + dict.addIntToIDict(3, 103); + + var m = createMapFromLowLevelDict(dict); + val fk = m.findFirst().key; + val fv = m.findFirst().loadValue(); + var t = createEmptyTuple(); + var r = m.findKeyLess(3); + while (r.isFound) { + t.push(r.loadValue()); + r = m.iteratePrev(r); + } + return (fk, fv, t); +} + fun main() {} /** @@ -115,4 +135,5 @@ fun main() {} @testcase | 104 | | 12 34 56 78 0 @testcase | 105 | -1 | 1 @testcase | 105 | 0 | 2 +@testcase | 106 | | -4 104 [ 102 101 104 ] */ diff --git a/tolk-tester/tests/generics-2.tolk b/tolk-tester/tests/generics-2.tolk index 94ae2ea038..4966d111ba 100644 --- a/tolk-tester/tests/generics-2.tolk +++ b/tolk-tester/tests/generics-2.tolk @@ -34,11 +34,23 @@ fun wrap2(value: T) { return WrapperAlias { value }; } +fun wrap0(): Wrapper { + return { value: 0 } +} + +fun wrapNull(): Wrapper { + return { value: null } +} + fun makePair(a: A, b: B) { var c: Pair = { first: a, second: b }; return c; } +fun makePairOf0(): Pair { + return { first: 0, second: 0 } +} + @method_id(101) fun test1() { @@ -428,6 +440,29 @@ fun test25() { } } +@method_id(126) +fun test26() { + var m1: Wrapper? = wrap0(); + var m2 = wrap0() as Wrapper?; + var m3: WrapperAlias? = wrap0(); + var m4: Wrapper? = wrapNull(); + var m5 = wrap0() as Wrapper?; + var m6 = wrapNull() as WrapperAlias?; + var m7 = null as Wrapper?; + __expect_type(m1, "Wrapper"); + __expect_type(m2, "Wrapper?"); + __expect_type(m3, "WrapperAlias"); + __expect_type(m4, "Wrapper"); + return (m1, m2, m3.value, m4.value, 777, m5, m6, m7); +} + +fun test27() { + var pp1: Pair | int = makePairOf0(); + var pp2 = makePairOf0() as Pair | Wrapper; + __expect_type(pp1, "Pair"); + __expect_type(pp2, "Pair | Wrapper"); +} + fun main(c: Wrapper, d: WrappedInt) { __expect_type(c, "Wrapper"); @@ -453,7 +488,7 @@ fun main(c: Wrapper, d: WrappedInt) { @testcase | 104 | | 11 11 13 13 13 2 @testcase | 105 | 0 | 80 @testcase | 105 | -1 | (null) -@testcase | 106 | | (null) (null) 0 777 1 2 typeid-12 +@testcase | 106 | | (null) (null) 0 777 1 2 typeid-15 @testcase | 107 | 5 | 20 20 5 5 20 777 (null) (null) (null) 0 @testcase | 108 | 0 | 777 typeid-1 777 typeid-1 @testcase | 108 | -1 | 777 0 777 0 @@ -471,6 +506,7 @@ fun main(c: Wrapper, d: WrappedInt) { @testcase | 119 | | 40 40 -1 -1 -1 -1 @testcase | 120 | | (null) typeid-9 777 (null) (null) 0 typeid-10 -1 -1 @testcase | 123 | | 10 10 10 10 10 +@testcase | 126 | | 0 0 0 (null) 777 0 typeid-12 (null) typeid-12 (null) 0 @fif_codegen diff --git a/tolk-tester/tests/handle-msg-6.tolk b/tolk-tester/tests/handle-msg-6.tolk index 0d349f9cca..57668f2bf5 100644 --- a/tolk-tester/tests/handle-msg-6.tolk +++ b/tolk-tester/tests/handle-msg-6.tolk @@ -5,7 +5,7 @@ fun test1() { } fun onBouncedMessage(in: InMessageBounced) { - if (in.valueExtra != null) { + if (!in.valueExtra.isEmpty()) { return; } throw in.originalForwardFee; @@ -23,7 +23,7 @@ fun onInternalMessage(in: InMessage) { onBouncedMessage() PROC:<{ DROP INMSG_VALUEEXTRA - ISNULL + DICTEMPTY IFNOTJMP:<{ }> INMSG_FWDFEE diff --git a/tolk-tester/tests/inference-tests.tolk b/tolk-tester/tests/inference-tests.tolk index 9c836baf4f..279dd7191d 100644 --- a/tolk-tester/tests/inference-tests.tolk +++ b/tolk-tester/tests/inference-tests.tolk @@ -75,7 +75,7 @@ fun test5(x: int) { __expect_type([x, x >= 1], "[int, bool]"); __expect_type([x, x >= 1, null as slice?], "[int, bool, slice?]"); __expect_type((x, [x], [[x], x]), "(int, [int], [[int], int])"); - __expect_type(contract.getOriginalBalanceWithExtraCurrencies(), "[coins, dict]"); + __expect_type(contract.getOriginalBalanceWithExtraCurrencies(), "[coins, ExtraCurrenciesMap]"); } fun test6() { diff --git a/tolk-tester/tests/invalid-declaration/err-1709.tolk b/tolk-tester/tests/invalid-declaration/err-1709.tolk new file mode 100644 index 0000000000..cb3849e6cf --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1709.tolk @@ -0,0 +1,8 @@ +global g: map + +/** +@compilation_should_fail +@stderr invalid `map`: type `coins` can not be used as a key +@stderr because its binary size is not constant: it's 4..124 bits +@stderr global g + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1710.tolk b/tolk-tester/tests/invalid-declaration/err-1710.tolk new file mode 100644 index 0000000000..5235b94418 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1710.tolk @@ -0,0 +1,14 @@ +struct WithRef { + v: int32; + r: cell?; +} + +fun main(s: map) { + +} + +/** +@compilation_should_fail +@stderr invalid `map`: type `WithRef` can not be used as a key +@stderr because it may contain a cell reference, not only data bits + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1711.tolk b/tolk-tester/tests/invalid-declaration/err-1711.tolk new file mode 100644 index 0000000000..f00d618ff1 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1711.tolk @@ -0,0 +1,13 @@ +type ForwardPayload = RemainingBitsAndRefs + +struct WithPayload { + w: map +} + +/** +@compilation_should_fail +@stderr invalid `map`: type `ForwardPayload` can not be used as a key +@stderr because it does not specify keyLen for a dictionary +@stderr hint: use `address` if a key is an internal address +@stderr hint: use `bits128` and similar if a key represents fixed-width data + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1806.tolk b/tolk-tester/tests/invalid-declaration/err-1806.tolk new file mode 100644 index 0000000000..240d1267ce --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1806.tolk @@ -0,0 +1,12 @@ + +fun main() { + var m: map; +} + +/** +@compilation_should_fail +@stderr invalid `map`: type `int` can not be used as a value +@stderr because it can not be serialized +@stderr because type `int` is not serializable, it doesn't define binary width +@stderr hint: replace `int` with `int32` / `uint64` / `coins` / etc. + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1945.tolk b/tolk-tester/tests/invalid-declaration/err-1945.tolk new file mode 100644 index 0000000000..a1d825a4c9 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1945.tolk @@ -0,0 +1,22 @@ +type MInt = int +struct WI { v: MInt } + +struct P { + w: map +} + +fun main() { + +} + +/** +@compilation_should_fail +@stderr invalid `map`: type `(bool, WI)` can not be used as a value +@stderr because it can not be serialized +@stderr because element `tensor.1` of type `WI` can't be serialized +@stderr because field `WI.v` of type `MInt` can't be serialized +@stderr because alias `MInt` expands to `int` +@stderr because type `int` is not serializable, it doesn't define binary width +@stderr hint: replace `int` with `int32` / `uint64` / `coins` / etc. +@stderr w: map + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4109.tolk b/tolk-tester/tests/invalid-semantics/err-4109.tolk new file mode 100644 index 0000000000..9f4a5771bd --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4109.tolk @@ -0,0 +1,17 @@ +struct Wrapper { + value: T; +} + +fun wrap0(): Wrapper { + return { value: 0 } +} + +fun main() { + var m: Wrapper | Wrapper = wrap0(); +} + +/** +@compilation_should_fail +@stderr can not deduce T for generic function `wrap0` +@stderr instantiate it manually with `wrap0<...>()` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6639.tolk b/tolk-tester/tests/invalid-typing/err-6639.tolk new file mode 100644 index 0000000000..e642ad642f --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6639.tolk @@ -0,0 +1,12 @@ + +fun main() { + createEmptyMap(); +} + +/** +@compilation_should_fail +@stderr invalid `map`: type `slice` can not be used as a key +@stderr because it does not specify keyLen for a dictionary +@stderr hint: use `address` if a key is an internal address +@stderr hint: use `bits128` and similar if a key represents fixed-width data + */ diff --git a/tolk-tester/tests/invalid-typing/err-6761.tolk b/tolk-tester/tests/invalid-typing/err-6761.tolk new file mode 100644 index 0000000000..48bfb12f37 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6761.tolk @@ -0,0 +1,15 @@ +struct Point { + x: int8 + y: int32 +} + + +fun main() { + var m = createEmptyMap(); + m.set(123, createAddressNone()); +} + +/** +@compilation_should_fail +@stderr can not pass `int` to `Point` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6802.tolk b/tolk-tester/tests/invalid-typing/err-6802.tolk new file mode 100644 index 0000000000..4511252dcf --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6802.tolk @@ -0,0 +1,10 @@ + +fun main() { + var m = createEmptyMap(); + m.set(5, ""); +} + +/** +@compilation_should_fail +@stderr can not pass `slice` to `int32` + */ diff --git a/tolk-tester/tests/maps-tests.tolk b/tolk-tester/tests/maps-tests.tolk new file mode 100644 index 0000000000..5a0830b32d --- /dev/null +++ b/tolk-tester/tests/maps-tests.tolk @@ -0,0 +1,682 @@ + +struct JustInt32 { + v: MInt32 +} + +struct JustUint64 { + v: uint64 +} + +struct JustAddress { + v: address +} + +type MInt32 = int32 +type MSlice = slice + +struct Point { + x: int8 + y: int8 +} + +struct Point3d { + x: int8 + y: int8 + z: int8 +} + +struct DataWithCell { + a: address + v: coins + p: Cell? + payload: cell? = null +} + +fun map.calculateSize(self) { + var size = 0; + var r = self.findFirst(); + while (r.isFound) { + size += 1; + r = self.iterateNext(r); + } + return size; +} + + +@method_id(101) +fun test1() { + var m: map = createEmptyMap(); + m.set(1, 10); + return m.exists(1); +} + +@method_id(102) +fun test2(k1: int, v1: int, k2: int, v2: int) { + var m: map = createMapFromLowLevelDict(null); + m.set(k1, v1); + m.set(k2, v2); + return (m.exists(1), m.exists(k1), m.mustGet(k2)); +} + +@method_id(103) +fun test3(v1: int) { + var m = createEmptyMap(); + m.set(1, (1, 2)); + m.set(2, (3, v1)); + m.delete(1); + return (m.exists(1), m.mustGet(2)); +} + +@method_id(104) +fun test4() { + var m = createEmptyMap(); + m.set(1, 10); + m.set(2, 20); + m.set(3, 30); + val g1 = m.get(1); + val g4 = m.get(4); + return (g1.isFound, g4.isFound, g1.loadValue()); +} + +@method_id(105) +fun test5() { + var m = createEmptyMap(); + m.set(1, 10); + m.set(2, 20); + m.set(3, 30); + var t = createEmptyTuple(); + var r = m.findFirst(); + while (r.isFound) { + t.push(r.getKey()); + t.push(r.loadValue()); + r = m.iterateNext(r); + } + r = m.findLast(); + while (r.isFound) { + t.push(r.loadValue()); + r = m.iterateNext(r); + } + r = m.findLast(); + while (r.isFound) { + t.push(r.loadValue()); + r = m.iteratePrev(r); + } + return t; +} + +@method_id(106) +fun test6() { + var m = createEmptyMap(); + m.set(1, 10); + m.set(2, 20); + m.set(3, 30); + return ( + m.findKeyGreater(1).getKey(), m.findKeyGreaterOrEqual(1).getKey(), + m.findKeyLess(3).getKey(), m.findKeyLessOrEqual(3).getKey(), + m.findKeyLess(3).loadValue(), m.findKeyLessOrEqual(3).loadValue(), + m.findKeyLessOrEqual(3).isFound, m.findKeyGreater(4).isFound, + ) +} + +@method_id(107) +fun test7() { + var m1 = createEmptyMap(); + m1.addIfNotExists(1, true); + m1.addIfNotExists(1, false); + m1.replaceIfExists(2, true); + return (m1.mustGet(1), m1.replaceIfExists(2, false), m1.get(2).isFound); +} + +@method_id(108) +fun test8() { + var (a, m: map) = (1, createEmptyMap()); + val prev1 = m.setAndGetPrevious(a, createAddressNone()); + val prev2 = m.setAndGetPrevious(a, address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5")); + return (prev1.isFound, prev2.isFound, prev2.loadValue().isNone(), prev2.loadValue().isNone(), m.get(a).loadValue().getWorkchain()); +} + +@method_id(109) +fun test9() { + var m = createEmptyMap(); + m.set(1, (ton("1"), ())); + val replaced = m.replaceIfExists(1, (ton("0.05"), ())); + val prev = m.replaceAndGetPrevious(1, (ton("0.01"), ())); + return (replaced, prev.loadValue(), m.replaceAndGetPrevious(2, (0,())), m.mustGet(1).0); +} + +@method_id(110) +fun test10() { + var m = createEmptyMap(); + val added1 = m.addIfNotExists(1<<100, 1<<10); + val added2 = m.addIfNotExists(1<<100, 2<<10); + val b1 = m.addOrGetExisting(2<<100, 20); + val b2 = m.addOrGetExisting(2<<100, 30); + return (added1, added2, m.mustGet(1<<100), b1.isFound, b2.isFound, b2.loadValue()); +} + +@method_id(111) +fun test11() { + var m: map; + m = createEmptyMap(); + m.set(0, null); + m.set(1, 1); + m.set(2, null); + return (m.exists(0), m.exists(1), m.mustGet(0), m.mustGet(1), m.get(0).isFound, m.get(0).loadValue(), m.get(3)); +} + +@method_id(112) +fun test12() { + var m1: map = createEmptyMap(); + val s1 = m1.calculateSize(); + m1.set(1, 10); + m1.addIfNotExists(1, -100); + val s2 = m1.calculateSize(); + var m2 = m1; + m2.addIfNotExists(2, 20); + m2.addIfNotExists(3, 30); + var m3 = m2; + val wasDeleted = m2.delete(2); + var del1 = m1.deleteAndGetDeleted(-100); + var del2 = m2.deleteAndGetDeleted(2); + var del3 = m2.deleteAndGetDeleted(3); + return ( + s1, s2, m3.calculateSize(), + wasDeleted, del1.isFound, del2.isFound, del3.isFound, + del3.loadValue(), + ); +} + +@method_id(113) +fun test13(m: map) { + // key remains just int at the TVM level, no runtime packing, checked via codegen + return m.get({v: 123}) +} + +@method_id(114) +fun test14(fromV: int) { + var m = createEmptyMap(); + m.set({v: 1}, 10); + m.set({v: 2}, 20); + m.set({v: 3}, 30); + return m.findKeyGreater({v: fromV}).key +} + +@method_id(115) +fun test15() { + var m = createEmptyMap(); + m.set({v: address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA")}, 10); + m.set({v: address("Ef9LynHHKgBxY6-l-W_dWN-CtGT2_ji5rN3EzOI-p9zWEfq6")}, 20); + m.set({v: address("Ef9hMd78gzSiVsK0zz0AHtEja8x1UoB_NDZMjn-l86NQK_2Y")}, 30); + return m.mustGet({v:address("Ef9LynHHKgBxY6-l-W_dWN-CtGT2_ji5rN3EzOI-p9zWEfq6")}); +} + +@method_id(116) +fun test16() { + var m = createEmptyMap(); + m.set(100, {x: 10, y: 20}); + return (m.deleteAndGetDeleted(100).loadValue(), m.calculateSize(), m.isEmpty()); +} + +@method_id(117) +fun test17() { + var m = createEmptyMap(); + m.set(true, 123); + return m.findFirst().key; +} + +@method_id(118) +fun test18() { + var m: map = createEmptyMap(); + m.set({x: 1, y: 1}, {x: 10, y: 10}); + m.addIfNotExists({x: 1, y: 1}, {x: -100, y: -100}); + val f1 = m.get({x: 1, y: 1}); + m.replaceIfExists({x: 1, y: 1}, {x: 20, y: 20}); + val f2 = m.get({x: 1, y: 1}); + val deleted1 = m.delete({x: 1, y: 1}); + val deleted2 = m.delete({x: 1, y: 1}); + val f3 = m.get({x: 1, y: 1}); + return (m, f1.isFound, f2.isFound, f3.isFound, f1.loadValue(), f2.loadValue(), deleted1, deleted2, m.isEmpty()); +} + +@method_id(119) +fun test19() { + var m: map> = createEmptyMap(); + m.addIfNotExists(address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"), Point{x:10,y:10}.toCell()); + return m.get(address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA")).loadValue().load(); +} + +@method_id(120) +fun test20() { + var m: map> = createEmptyMap(); + m.addIfNotExists(address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"), Point{x:10,y:10}.toCell()); + return m.mustGet(address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA")).load(); +} + +@method_id(121) +fun test21() { + var m: map> = createEmptyMap(); + val r1 = m.addOrGetExisting(address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"), Point{x:10,y:10}.toCell()); + val r2 = m.addOrGetExisting(address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"), Point{x:20,y:20}.toCell()); + val d1 = m.deleteAndGetDeleted(address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA")); + val d2 = m.deleteAndGetDeleted(address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA")); + return (r1.isFound, r2.isFound, r2.loadValue().load(), d1.isFound, d2.isFound); +} + +@method_id(122) +fun test22() { + var m: map> = createEmptyMap(); + m.set(3, Point{x:30,y:30}.toCell()); + m.set(2, Point{x:20,y:20}.toCell()); + m.set(1, Point{x:10,y:10}.toCell()); + return (m.findFirst().key, m.findLast().key, m.findKeyGreater(2).key, m.findKeyGreaterOrEqual(2).loadValue().load(), m.mustGet(2).load().x); +} + +@method_id(123) +fun test23() { + var m: map = createEmptyMap(); + val p1 = m.setAndGetPrevious({x:1,y:1}, {a: createAddressNone(), v: ton("0.05"), p: Point{x:10,y:10}.toCell(), payload: null}); + val p2 = m.setAndGetPrevious({x:1,y:1}, {a: createAddressNone(), v: ton("0.1"), p: null, payload: createEmptyCell()}); + return (p1.isFound, p2.isFound, p2.loadValue().p!.load(), m.mustGet({x:1,y:1}).a.isNone(), m.get({x:1,y:1}).loadValue().payload.depth()); +} + +@method_id(124) +fun test24() { + var m = createEmptyMap(); + val p1 = m.deleteAndGetDeleted({x: 1, y: 1}); + m.set({x: 3, y: 3}, {a: createAddressNone(), v: ton("0.05"), p: Point{x:30,y:30}.toCell(), payload: createEmptyCell()}); + m.set({x: 1, y: 1}, {a: createAddressNone(), v: ton("0.05"), p: Point{x:10,y:10}.toCell(), payload: null}); + m.set({x: 2, y: 2}, {a: createAddressNone(), v: ton("0.05"), p: null, payload: null}); + val k2 = m.findKeyGreaterOrEqual({x: 2, y: 2}).getKey(); + var r = m.findFirst(); + var t = createEmptyTuple(); + while (r.isFound) { + t.push(r.getKey().x); + t.push(r.getKey().y); + t.push(r.loadValue().payload == null); + r = m.iterateNext(r); + } + return (p1, k2, t); +} + +@method_id(125) +fun test25() { + var m = createEmptyMap(); + m.set(1, stringHexToSlice("0101")); + return m; +} + +@method_id(126) +fun test26() { + var m = createEmptyMap(); + m.set(1, stringHexToSlice("0101")); + val f1 = m.setAndGetPrevious(1, stringHexToSlice("010101")); + val f2 = m.deleteAndGetDeleted(1); + val f3 = m.setAndGetPrevious(1, stringHexToSlice("01010101")); + return ( + f1.isFound, f2.isFound, f3.isFound, + f1.loadValue().remainingBitsCount(), f2.loadValue().remainingBitsCount(), + m.mustGet(1).remainingBitsCount(), + ); +} + +@method_id(127) +fun test27() { + var m = createEmptyMap(); + m.set(1, ""); + return (m.findFirst().key, m.findFirst().loadValue().remainingBitsCount()); +} + +@method_id(128) +fun test28() { + var m = createEmptyMap(); + m.set({x: 1, y: 1}, stringHexToSlice("01")); + m.set({x: 3, y: 3}, stringHexToSlice("03")); + m.set({x: 2, y: 2}, stringHexToSlice("02")); + var r = m.findKeyLessOrEqual({x: 4, y: 0}); + var sum = 0; + while (r.isFound) { + sum += r.loadValue().loadUint(8); + r = m.iteratePrev(r); + } + return (sum, m.calculateSize()); +} + +@method_id(129) +fun test29() { + var m = createEmptyMap(); + val a1 = m.addIfNotExists(1, DataWithCell{a: createAddressNone(), v: 0, p: null, payload: null}.toCell().beginParse()); + val a2 = m.addIfNotExists(2, DataWithCell{a: createAddressNone(), v: 0, p: Point{x:20,y:20}.toCell(), payload: null}.toCell().beginParse()); + val a3 = m.addIfNotExists(3, DataWithCell{a: createAddressNone(), v: 0, p: Point{x:30,y:30}.toCell(), payload: createEmptyCell()}.toCell().beginParse()); + val r4 = m.replaceAndGetPrevious(4, ""); + return ( + a1, a2, a3, r4, + DataWithCell.fromSlice(m.mustGet(2)).payload, + m.get(3).loadValue().loadAny().p!.load(), + m.get(3).loadValue().remainingRefsCount(), + m.findLast().key, + m.findLast().loadValue().remainingRefsCount(), + ); +} + +@method_id(130) +fun test30() { + var m = createEmptyMap(); + m.set(1, createEmptyCell()); + m.set(2, Point{x:20,y:20}.toCell()); + m.set(3, Point{x:30,y:30}.toCell()); + val c3 = m.get(3); + var r = m.findKeyLess(3); + var t = createEmptyTuple(); + while (r.isFound) { + t.push(r.loadValue().beginParse().remainingBitsCount()); + r = m.iteratePrev(r); + } + val p2 = lazy Point.fromCell(m.mustGet(2)); + return (c3.isFound, c3.loadValue().beginParse().loadAny(), t, p2.y); +} + +@method_id(131) +fun test31() { + var m = createEmptyMap(); + m.set(1, createEmptyCell()); + m.set(2, Point{x:20,y:20}.toCell()); + m.set(3, Point{x:30,y:30}.toCell()); + m.set(4, null); + val c4 = m.get(4); + val c5 = m.get(5); + var r = m.findLast(); + var t = createEmptyTuple(); + while (r.isFound) { + t.push(r.loadValue() == null); + r = m.iteratePrev(r); + } + return (m.mustGet(4), c4.loadValue(), c5.isFound, t); +} + +@method_id(132) +fun test32() { + var m = createEmptyMap(); + m.set(stringHexToSlice("0102") as bits16, stringHexToSlice("aaaa") as bits16); + m.set(stringHexToSlice("0304") as bits16, stringHexToSlice("bbbb") as bits16); + val repl = m.replaceAndGetPrevious(stringHexToSlice("0304") as bits16, stringHexToSlice("0000") as bits16); + var r = m.iteratePrev(m.findLast()); + return ((r.key as slice).remainingBitsCount(), (r.key as slice).loadAny(), (r.loadValue() as slice).remainingBitsCount(), repl.isFound, (repl.loadValue() as slice).loadUint(8)); +} + +@method_id(133) +fun test33() { + var m = createEmptyMap(); + m.set(123, {x: 10, y: 10}); + return m.mustGet(123); +} + +@method_id(134) +fun test34(z: int) { + var m = createEmptyMap(); + m.set(123, {x: 10, y: 10, z}); + return [m.mustGet(123).z, m.addIfNotExists(456, {x: 10, y: z, z}), m.mustGet(456).y]; +} + +@method_id(135) +fun test35() { + var m: map> = createEmptyMap(); + m.set(1, createEmptyMap()); + val f1 = m.mustGet(1).findFirst(); + assert (!f1.isFound) throw 123; + m.get(1).loadValue().set(10, 100); + assert (m.mustGet(1).isEmpty()) throw 123; // the previous set() changed a copy + + var nested = createEmptyMap(); + nested.set(1,10).set(2, 20); + m.replaceIfExists(1, nested); + nested.set(3, 30); + assert (nested.calculateSize() == 3) throw 123; + assert (m.mustGet(1).calculateSize() == 2) throw 123; + + val (k1, v2) = (m.mustGet(1).findFirst().getKey(), m.get(1).loadValue().findLast().loadValue()); + val s1 = m.findFirst().loadValue().calculateSize(); + m.mustGet(1).delete(10); + val d2 = m.mustGet(1).deleteAndGetDeleted(2); + return (f1.isFound, k1, v2, s1, d2.loadValue(), m.mustGet(1).isEmpty(), m.get(1).loadValue().calculateSize()); +} + +@method_id(136) +fun test36() { + val k = address("Ef_vA6yRfmt2P4UHnxlrQUZFcBnKux8mL2eMqBgpeMFPorr4"); + var m = createEmptyMap(); + m.set(k, 10); + return m.mustGet(k); +} + +@method_id(137) +fun test37() { + try { + var m = createEmptyMap(); + m.set(1, 10); + return m.mustGet(555) as int; + } catch (ex) { + return ex; + } +} + +@method_id(138) +fun test38(k: int) { + try { + var m = createEmptyMap(); + m.set(1, 10); + return m.mustGet(k, 55); + } catch (ex) { + return ex as int64; + } +} + +type Set = map + +fun createEmptySet(): Set { return createEmptyMap() } +fun Set.add(mutate self, k: T) { self.set(k, ()) } +fun Set.remove(mutate self, k: T) { self.delete(k) } + +@method_id(139) +fun test39() { + var s: Set = createEmptySet(); + s.add(10); s.add(10); + s.add(20); + s.add(30); + s.remove(20); + return (s.exists(20), s.exists(10), s.calculateSize()); +} + +@method_id(140) +fun test40() { + var `m!empty` = createEmptyMap() as map?; + var `m!filled` = createEmptyMap() as map?; + var `m_null` = null as map?; + + `m!filled`!.set(10, 20); + return ( + `m!empty` == null, `m!filled` == null, `m_null` == null, 777, + `m!empty`!.isEmpty(), `m!filled`!.isEmpty(), 777, + `m!empty`, `m_null`, + ); +} + +fun main() { + __expect_type(createEmptyMap, "() -> map"); +} + +/** +@testcase | 101 | | -1 +@testcase | 102 | 1 10 2 20 | -1 -1 20 +@testcase | 102 | 2 20 3 30 | 0 -1 30 +@testcase | 103 | 100 | 0 3 100 +@testcase | 104 | | -1 0 10 +@testcase | 105 | | [ 1 10 2 20 3 30 30 30 20 10 ] +@testcase | 106 | | 2 1 2 3 20 30 -1 0 +@testcase | 107 | | -1 0 0 +@testcase | 108 | | 0 -1 -1 -1 0 +@testcase | 109 | | -1 50000000 (null) 0 10000000 +@testcase | 110 | | -1 0 1024 0 -1 20 +@testcase | 111 | | -1 -1 (null) 1 -1 (null) (null) 0 +@testcase | 112 | | 0 1 3 -1 0 0 -1 30 +@testcase | 114 | 2 | 3 +@testcase | 114 | 3 | (null) +@testcase | 115 | | 20 +@testcase | 116 | | 10 20 0 -1 +@testcase | 117 | | -1 +@testcase | 118 | | (null) -1 -1 0 10 10 20 20 -1 0 -1 +@testcase | 119 | | 10 10 +@testcase | 120 | | 10 10 +@testcase | 121 | | 0 -1 10 10 -1 0 +@testcase | 122 | | 1 3 3 20 20 20 +@testcase | 123 | | 0 -1 10 10 -1 0 +@testcase | 124 | | (null) 0 2 2 [ 1 1 -1 2 2 -1 3 3 0 ] +@testcase | 126 | | -1 -1 0 16 24 32 +@testcase | 127 | | 1 0 +@testcase | 128 | | 6 3 +@testcase | 129 | | -1 -1 -1 (null) 0 (null) 30 30 2 3 2 +@testcase | 130 | | -1 30 30 [ 16 0 ] 20 +@testcase | 131 | | (null) (null) 0 [ -1 0 0 0 ] +@testcase | 132 | | 16 1 2 16 -1 187 +@testcase | 133 | | 10 10 +@testcase | 134 | 5 | [ 5 -1 5 ] +@testcase | 135 | | 0 1 20 2 20 0 2 +@testcase | 136 | | 10 +@testcase | 137 | | 9 +@testcase | 138 | 1 | 10 +@testcase | 138 | 4 | 55 +@testcase | 139 | | 0 -1 2 +@testcase | 140 | | 0 0 -1 777 -1 0 777 (null) typeid-1 (null) 0 + +@fif_codegen +""" + test1() PROC:<{ // + NEWDICT // m + 1 PUSHINT // m '2=1 + x{0000000a} PUSHSLICE + s0 s2 XCHG + 8 PUSHINT + DICTISET // m + 1 PUSHINT // m '11=1 + SWAP + 8 PUSHINT // '11=1 m '16=8 + DICTIGET NULLSWAPIFNOT // '12 '13 + NIP // '13 + }> +""" + +@fif_codegen +""" + test13() PROC:<{ // m + 123 PUSHINT // m '1=123 + SWAP + 32 PUSHINT // '1=123 m '6=32 + DICTIGET NULLSWAPIFNOT // '2 '3 + }> +""" + +@fif_codegen +""" + test14() PROC:<{ + ... + 64 PUSHINT + DICTUGETNEXT NULLSWAPIFNOT2 + DROP + NIP + }> +""" + +@fif_codegen +""" + test15() PROC:<{ + ... + x{9FE9794E38E5400E2C75F4BF2DFBAB1BF0568C9EDFC717359BB8999C47D4FB9AC23_} PUSHSLICE + SWAP + 267 PUSHINT + DICTGET + 9 THROWIFNOT + 8 LDI + ENDS + }> +""" + +@fif_codegen +""" + test16() PROC:<{ + NEWDICT + 100 PUSHINT + x{0a14} PUSHSLICE + s0 s2 XCHG + 32 PUSHINT + DICTISET +""" + +@fif_codegen +""" + test25() PROC:<{ // + NEWDICT // m + 1 PUSHINT // m '2=1 + x{0101} PUSHSLICE // m '2=1 '3 + s0 s2 XCHG + 8 PUSHINT // '3 '2=1 m '6=8 + DICTISET // m + }> +""" + +@fif_codegen +""" + test30() PROC:<{ + NEWDICT + 1 PUSHINT + PUSHREF + s0 s2 XCHG + 8 PUSHINT + DICTISETREF +""" + +@fif_codegen +""" + test33() PROC:<{ + NEWDICT + 123 PUSHINT + x{0a0a} PUSHSLICE + s0 s2 XCHG + 8 PUSHINT + DICTUSET + 123 PUSHINT + SWAP + 8 PUSHINT + DICTUGET + 9 THROWIFNOT + 8 LDI + 8 LDI + ENDS + }> +""" + +@fif_codegen +""" + test34() PROC:<{ + NEWDICT + 123 PUSHINT + NEWC + x{0a0a} STSLICECONST + s3 PUSH + 8 STIR + s0 s2 XCHG + 32 PUSHINT + DICTUSETB +""" + +@fif_codegen +""" + test36() PROC:<{ // + x{9FFDE075922FCD6EC7F0A0F3E32D6828C8AE03395763E4C5ECF19503052F1829F45_} PUSHSLICE // k + NEWDICT // k m + x{0000000a} PUSHSLICE + s2 s1 PUXC + 267 PUSHINT + DICTSET // k m +""" + +*/ + diff --git a/tolk-tester/tests/overloads-tests.tolk b/tolk-tester/tests/overloads-tests.tolk index 962fb06c4b..70baa48b58 100644 --- a/tolk-tester/tests/overloads-tests.tolk +++ b/tolk-tester/tests/overloads-tests.tolk @@ -249,6 +249,35 @@ fun test13() { } +fun map.mySize(self): int1 { return 0 } +fun map.mySize(self): int2 { return 0 } +fun map.mySize(self): int3 { return 0 } +fun map>.mySize(self): int4 { return 0 } +fun map>.mySize(self): int5 { return 0 } + +type MInt16 = int16 +type MInt16ov = MInt16 +type MMap32 = map> +type MMap8Of32 = map + +fun test14( + m1: map>, m2: map>, + m3: map>, m4: map, m5: MMap8Of32, + m6: map?>, + m7: map, m8: map, + m9: map, +) { + __expect_type(m1.mySize(), "int5"); + __expect_type(m2.mySize(), "int5"); + __expect_type(m3.mySize(), "int4"); + __expect_type(m4.mySize(), "int4"); + __expect_type(m5.mySize(), "int4"); + __expect_type(m6.mySize(), "int3"); + __expect_type(m7.mySize(), "int2"); + __expect_type(m8.mySize(), "int2"); + __expect_type(m9.mySize(), "int1"); +} + fun main() { // besides these positive tests, there are many negative resulting in "ambiguous call" return 0 diff --git a/tolk-tester/tests/pack-unpack-2.tolk b/tolk-tester/tests/pack-unpack-2.tolk index a920039b40..da5fba0d33 100644 --- a/tolk-tester/tests/pack-unpack-2.tolk +++ b/tolk-tester/tests/pack-unpack-2.tolk @@ -406,6 +406,17 @@ struct WithTwoRestFields { rest2: RemainingBitsAndRefs; } +struct WithMaps { + m1: map + m2: map +} + +struct WithNullableMaps { + m1: map? + m2: map? + m3: map? + m4: map | int32 +} // --------------------------------------------- @@ -716,6 +727,51 @@ fun test_EdgeCaseIntegers() { return edge.toCell().hash() == manual.endCell().hash() } +@method_id(230) +fun test_serializeMaps() { + var ( + m1: map, + m2: map, + ) = (createEmptyMap(), createEmptyMap()); + m1.set(1, ""); + m1.set(2, stringHexToSlice("01").appendRef(stringHexToSlice("02"))); + m2.set({value: 3}, (3, 30)); + m2.set({value: 4}, (4, 40)); + + var w = (WithMaps { m1, m2 }).toCell().load(); + w.m2.delete({value: 4}); + val d3 = w.m2.deleteAndGetDeleted({value: 3}); + var r = w.m1.findFirst(); + var kl = (0, 0); + while (r.isFound) { + kl = r.loadValue().remainingBitsAndRefsCount(); + r = w.m1.iterateNext(r); + } + return (w.m1.isEmpty(), w.m2.isEmpty(), d3.isFound, d3.loadValue(), kl); +} + +@method_id(231) +fun test_nullableMaps() { + var o: WithNullableMaps = { + m1: createEmptyMap(), + m2: createEmptyMap(), + m3: null, + m4: createEmptyMap(), + }; + if (o.m2 != null && o.m4 is map) { + o.m2.set(1, 10); + o.m4.set(1, 10); + } + var c = o.toCell(); + var s = c.beginParse(); + var (bits, refs) = s.remainingBitsAndRefsCount(); + assert(bits == 2+2+1+2) throw 123; + assert(refs == 0+1+0+1) throw 234; + + o = c.load(); + return (o.m1 == null, o.m1!.isEmpty(), o.m2 == null, o.m2!.isEmpty(), o.m3 == null, o.m4 is int32); +} + fun main() { var t: JustInt32 = { value: 10 }; var c = t.toCell(); @@ -756,4 +812,6 @@ fun main() { @testcase | 227 | | 16 @testcase | 228 | | -1 @testcase | 229 | | -1 +@testcase | 230 | | 0 -1 -1 3 30 8 1 +@testcase | 231 | | 0 -1 0 0 -1 0 */ diff --git a/tolk-tester/tests/send-msg-1.tolk b/tolk-tester/tests/send-msg-1.tolk index 3a889dcf43..c15b2d356d 100644 --- a/tolk-tester/tests/send-msg-1.tolk +++ b/tolk-tester/tests/send-msg-1.tolk @@ -170,13 +170,13 @@ fun test4(value: coins) { @noinline fun test5_manual() { - var ec_dict = createEmptyDict(); - ec_dict.iDictSet(32, 1, "ec1"); + var ec_dict: ExtraCurrenciesMap = createEmptyMap(); + ec_dict.set(1, 100); return beginCell() .storeUint(0x18, 6) // bounce .storeAddress(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA")) .storeCoins(123) // value.grams - .storeMaybeRef(ec_dict) // value.extra + .storeMaybeRef(ec_dict.toLowLevelDict()) // value.extra .storeUint(0, 0 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) // body: op + queryId .storeUint(0x12345678, 32) @@ -186,8 +186,8 @@ fun test5_manual() { @method_id(105) fun test5() { - var ec_dict = createEmptyDict(); - ec_dict.iDictSet(32, 1, "ec1"); + var ec_dict: ExtraCurrenciesMap = createEmptyMap(); + ec_dict.set(1, 100); val body: MyBody = { queryId: 800 }; var b = createMessage({ body, @@ -215,9 +215,9 @@ fun test6_manual(value: coins, ec_dict: dict) { @method_id(106) fun test6(value: coins, dictKey: int?) { - var ec_dict = createEmptyDict(); + var ec_dict: ExtraCurrenciesMap = createEmptyMap(); if (dictKey != null) { - ec_dict.iDictSet(32, dictKey, "ec1"); + ec_dict.set(dictKey, 100500); } val body: MyBody = { queryId: 800 }; var b = createMessage({ @@ -226,7 +226,7 @@ fun test6(value: coins, dictKey: int?) { value: (value, ec_dict), body, }); - assert(b.hash() == test6_manual(value, ec_dict).hash(), 106); + assert(b.hash() == test6_manual(value, ec_dict.toLowLevelDict()).hash(), 106); return b.hash(); } @@ -617,7 +617,7 @@ fun test21(queryId: uint64) { var b = createMessage({ bounce: false, dest: address("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi"), - value: (ton("0.1"), createEmptyDict()), + value: (ton("0.1"), createEmptyMap()), body, }); assert(b.hash() == test21_manual(body).hash(), 121); @@ -732,9 +732,9 @@ fun main() { @testcase | 102 | | 20192939504955637835708665496574868659039935190188156593169026529135727309085 @testcase | 103 | | 97654287980681401436727082042373998183264988661375688119718809500301413968039 @testcase | 104 | 600000 | 755636693689039391990782995030008885663781699175339033402019272057515711062 -@testcase | 105 | | 24816341499673835567887890014278904436471582322802948121781418622643959482495 +@testcase | 105 | | 24503969966890554363249789782591743702412487637374098978386746563683471749050 @testcase | 106 | 123 null | 104134380371273907780196393444164225714229635235007677971195651971972203592811 -@testcase | 106 | 456 8888 | 39337586036945718311402746340438400160817844833530971545330721291986281100430 +@testcase | 106 | 456 8888 | 6425272102977874346517989655071023150283043721608819726435980878339463285784 @testcase | 107 | 1000 | 55093441331748625324828489600632232039914212774002148634088483962817636598198 @testcase | 108 | 50000000 | 95023796475113775225029817428715936488418545169963429399979521091689824066088 @testcase | 109 | 0 | 55999621586681214992294941423256376619779969729861696464321825639854258502733 diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index 2a68d02d8e..0014d736e3 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -10,6 +10,7 @@ set(TOLK_SOURCE pack-unpack-api.cpp pack-unpack-serializers.cpp send-message-api.cpp + maps-kv-api.cpp pipe-discover-parse-sources.cpp pipe-register-symbols.cpp pipe-resolve-identifiers.cpp diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index 6e38ba7929..65f5608ed6 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -212,6 +212,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { const std::vector& get_all_not_builtin_functions(); +const std::vector& get_all_declared_global_vars(); const std::vector& get_all_declared_constants(); const std::vector& get_all_declared_structs(); diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index f3aa3c5ef7..a71ee7c88b 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1307,6 +1307,7 @@ static AsmOp compile_expect_type(std::vector&, std::vector&, // implemented in dedicated files using GenerateOpsImpl = FunctionBodyBuiltinGenerateOps::GenerateOpsImpl; +using CompileToAsmOpImpl = FunctionBodyBuiltinAsmOp::CompileToAsmOpImpl; GenerateOpsImpl generate_T_toCell; GenerateOpsImpl generate_builder_storeAny; @@ -1323,6 +1324,45 @@ GenerateOpsImpl generate_address_buildInAnotherShard; GenerateOpsImpl generate_AutoDeployAddress_buildAddress; GenerateOpsImpl generate_AutoDeployAddress_addressMatches; +GenerateOpsImpl generate_mapKV_exists; +GenerateOpsImpl generate_mapKV_get; +GenerateOpsImpl generate_mapKV_mustGet; +GenerateOpsImpl generate_mapKV_set; +GenerateOpsImpl generate_mapKV_setGet; +GenerateOpsImpl generate_mapKV_replace; +GenerateOpsImpl generate_mapKV_replaceGet; +GenerateOpsImpl generate_mapKV_add; +GenerateOpsImpl generate_mapKV_addGet; +GenerateOpsImpl generate_mapKV_del; +GenerateOpsImpl generate_mapKV_delGet; +GenerateOpsImpl generate_mapKV_findFirst; +GenerateOpsImpl generate_mapKV_findLast; +GenerateOpsImpl generate_mapKV_findKeyGreater; +GenerateOpsImpl generate_mapKV_findKeyGreaterOrEqual; +GenerateOpsImpl generate_mapKV_findKeyLess; +GenerateOpsImpl generate_mapKV_findKeyLessOrEqual; +GenerateOpsImpl generate_mapKV_iterateNext; +GenerateOpsImpl generate_mapKV_iteratePrev; + +CompileToAsmOpImpl compile_createEmptyMap; +CompileToAsmOpImpl compile_createMapFromLowLevelDict; +CompileToAsmOpImpl compile_dict_get; +CompileToAsmOpImpl compile_dict_mustGet; +CompileToAsmOpImpl compile_dict_getMin; +CompileToAsmOpImpl compile_dict_getMax; +CompileToAsmOpImpl compile_dict_getNext; +CompileToAsmOpImpl compile_dict_getNextEq; +CompileToAsmOpImpl compile_dict_getPrev; +CompileToAsmOpImpl compile_dict_getPrevEq; +CompileToAsmOpImpl compile_dict_set; +CompileToAsmOpImpl compile_dict_setGet; +CompileToAsmOpImpl compile_dict_replace; +CompileToAsmOpImpl compile_dict_replaceGet; +CompileToAsmOpImpl compile_dict_add; +CompileToAsmOpImpl compile_dict_addGet; +CompileToAsmOpImpl compile_dict_del; +CompileToAsmOpImpl compile_dict_delGet; + void define_builtins() { using namespace std::placeholders; @@ -1655,6 +1695,121 @@ void define_builtins() { define_builtin_method("T.__toTuple", typeT, {typeT}, TypeDataTuple::create(), declReceiverT, compile_any_object_to_tuple, FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + + TypePtr MapKV = TypeDataMapKV::create(TypeDataGenericT::create("K"), TypeDataGenericT::create("V")); + TypePtr TKey = TypeDataGenericT::create("K"); + TypePtr TValue = TypeDataGenericT::create("V"); + TypePtr LookupResultT = TypeDataUnknown::create(); + TypePtr EntryKV = TypeDataUnknown::create(); + const GenericsDeclaration* declGenericMapKV = new GenericsDeclaration(std::vector{{"K", nullptr}, {"V", nullptr}}, 0); + const GenericsDeclaration* declReceiverMapKV = new GenericsDeclaration(std::vector{{"K", nullptr}, {"V", nullptr}}, 2); + + // high-level methods for maps; + // they are generic, so all type checks are done automatically; + // but all calls to them are handled at generating Ops from AST, their "simple compile" is not called + define_builtin_func("createEmptyMap", {}, MapKV, declGenericMapKV, + compile_createEmptyMap, + FunctionData::flagMarkedAsPure | FunctionData::flagAllowAnyWidthT); + define_builtin_func("createMapFromLowLevelDict", {TypeDataUnion::create_nullable(TypeDataCell::create())}, MapKV, declGenericMapKV, + compile_createMapFromLowLevelDict, + FunctionData::flagMarkedAsPure | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.exists", MapKV, {MapKV, TKey}, TypeDataBool::create(), declReceiverMapKV, + generate_mapKV_exists, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.get", MapKV, {MapKV, TKey}, LookupResultT, declReceiverMapKV, + generate_mapKV_get, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.mustGet", MapKV, {MapKV, TKey, TypeDataInt::create()}, TValue, declReceiverMapKV, + generate_mapKV_mustGet, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.set", MapKV, {MapKV, TKey, TValue}, TypeDataVoid::create(), declReceiverMapKV, + generate_mapKV_set, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT | FunctionData::flagReturnsSelf); + define_builtin_method("map.setAndGetPrevious", MapKV, {MapKV, TKey, TValue}, LookupResultT, declReceiverMapKV, + generate_mapKV_setGet, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.replaceIfExists", MapKV, {MapKV, TKey, TValue}, TypeDataBool::create(), declReceiverMapKV, + generate_mapKV_replace, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.replaceAndGetPrevious", MapKV, {MapKV, TKey, TValue}, LookupResultT, declReceiverMapKV, + generate_mapKV_replaceGet, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.addIfNotExists", MapKV, {MapKV, TKey, TValue}, TypeDataBool::create(), declReceiverMapKV, + generate_mapKV_add, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.addOrGetExisting", MapKV, {MapKV, TKey, TValue}, LookupResultT, declReceiverMapKV, + generate_mapKV_addGet, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.delete", MapKV, {MapKV, TKey}, TypeDataBool::create(), declReceiverMapKV, + generate_mapKV_del, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.deleteAndGetDeleted", MapKV, {MapKV, TKey}, LookupResultT, declReceiverMapKV, + generate_mapKV_delGet, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.findFirst", MapKV, {MapKV}, EntryKV, declReceiverMapKV, + generate_mapKV_findFirst, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.findLast", MapKV, {MapKV}, EntryKV, declReceiverMapKV, + generate_mapKV_findLast, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.findKeyGreater", MapKV, {MapKV, TKey}, EntryKV, declReceiverMapKV, + generate_mapKV_findKeyGreater, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.findKeyGreaterOrEqual", MapKV, {MapKV, TKey}, EntryKV, declReceiverMapKV, + generate_mapKV_findKeyGreaterOrEqual, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.findKeyLess", MapKV, {MapKV, TKey}, EntryKV, declReceiverMapKV, + generate_mapKV_findKeyLess, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.findKeyLessOrEqual", MapKV, {MapKV, TKey}, EntryKV, declReceiverMapKV, + generate_mapKV_findKeyLessOrEqual, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.iterateNext", MapKV, {MapKV, EntryKV}, EntryKV, declReceiverMapKV, + generate_mapKV_iterateNext, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.iteratePrev", MapKV, {MapKV, EntryKV}, EntryKV, declReceiverMapKV, + generate_mapKV_iteratePrev, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + + // low-level functions that actually emit TVM assembly, they work on a "dict" level + TypePtr PlainDict = TypeDataCell::create(); + TypePtr KeySliceOrInt = TypeDataUnknown::create(); + TypePtr ValueSlice = TypeDataSlice::create(); + TypePtr ValueFound = TypeDataInt::create(); + TypePtr LookupSliceFound = TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create()}); + + define_builtin_func("__dict.get", {KeySliceOrInt, PlainDict, TypeDataInt::create()}, LookupSliceFound, nullptr, + compile_dict_get, 0); + define_builtin_func("__dict.mustGet", {KeySliceOrInt, PlainDict, TypeDataInt::create()}, LookupSliceFound, nullptr, + compile_dict_mustGet, 0); + define_builtin_func("__dict.getMin", {PlainDict}, TypeDataTensor::create({ValueSlice, KeySliceOrInt, ValueFound}), nullptr, + compile_dict_getMin, 0); + define_builtin_func("__dict.getMax", {PlainDict}, TypeDataTensor::create({ValueSlice, KeySliceOrInt, ValueFound}), nullptr, + compile_dict_getMax, 0); + define_builtin_func("__dict.getNext", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, TypeDataBool::create()}), nullptr, + compile_dict_getNext, 0); + define_builtin_func("__dict.getNextEq", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, TypeDataBool::create()}), nullptr, + compile_dict_getNextEq, 0); + define_builtin_func("__dict.getPrev", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, TypeDataBool::create()}), nullptr, + compile_dict_getPrev, 0); + define_builtin_func("__dict.getPrevEq", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, TypeDataBool::create()}), nullptr, + compile_dict_getPrevEq, 0); + define_builtin_func("__dict.set", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, PlainDict, nullptr, + compile_dict_set, 0); + define_builtin_func("__dict.setGet", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, LookupSliceFound}), nullptr, + compile_dict_setGet, 0); + define_builtin_func("__dict.replace", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, TypeDataBool::create()}), nullptr, + compile_dict_replace, 0); + define_builtin_func("__dict.replaceGet", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, LookupSliceFound}), nullptr, + compile_dict_replaceGet, 0); + define_builtin_func("__dict.add", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, TypeDataBool::create()}), nullptr, + compile_dict_add, 0); + define_builtin_func("__dict.addGet", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, LookupSliceFound}), nullptr, + compile_dict_addGet, 0); + define_builtin_func("__dict.del", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, TypeDataBool::create()}), nullptr, + compile_dict_del, 0); + define_builtin_func("__dict.delGet", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, LookupSliceFound}), nullptr, + compile_dict_delGet, 0); } // there are some built-in functions that operate on types declared in stdlib (like Cell) @@ -1726,6 +1881,35 @@ void patch_builtins_after_stdlib_loaded() { lookup_function("createMessage")->mutate()->declared_return_type = OutMessage; lookup_function("createExternalLogMessage")->mutate()->parameters[0].declared_type = CreateExternalLogMessageOptions; lookup_function("createExternalLogMessage")->mutate()->declared_return_type = OutMessage; + + if (!lookup_global_symbol("MapLookupResult")) return; + StructPtr struct_ref_LookupResultT = lookup_global_symbol("MapLookupResult")->try_as(); + StructPtr struct_ref_EntryKV = lookup_global_symbol("MapEntry")->try_as(); + TypePtr TKey = TypeDataGenericT::create("K"); + TypePtr TValue = TypeDataGenericT::create("V"); + TypePtr LookupResultT = TypeDataGenericTypeWithTs::create(struct_ref_LookupResultT, nullptr, {TValue}); + TypePtr EntryKV = TypeDataGenericTypeWithTs::create(struct_ref_EntryKV, nullptr, {TKey, TValue}); + + lookup_function("map.get")->mutate()->declared_return_type = LookupResultT; + lookup_function("map.setAndGetPrevious")->mutate()->declared_return_type = LookupResultT; + lookup_function("map.replaceAndGetPrevious")->mutate()->declared_return_type = LookupResultT; + lookup_function("map.addOrGetExisting")->mutate()->declared_return_type = LookupResultT; + lookup_function("map.deleteAndGetDeleted")->mutate()->declared_return_type = LookupResultT; + + auto v_def_throwCode = createV({}, td::make_refint(9), "9"); + v_def_throwCode->assign_inferred_type(TypeDataInt::create()); + lookup_function("map.mustGet")->mutate()->parameters[2].assign_default_value(v_def_throwCode); + + lookup_function("map.findFirst")->mutate()->declared_return_type = EntryKV; + lookup_function("map.findLast")->mutate()->declared_return_type = EntryKV; + lookup_function("map.findKeyGreater")->mutate()->declared_return_type = EntryKV; + lookup_function("map.findKeyGreaterOrEqual")->mutate()->declared_return_type = EntryKV; + lookup_function("map.findKeyLess")->mutate()->declared_return_type = EntryKV; + lookup_function("map.findKeyLessOrEqual")->mutate()->declared_return_type = EntryKV; + lookup_function("map.iterateNext")->mutate()->declared_return_type = EntryKV; + lookup_function("map.iterateNext")->parameters[1].mutate()->declared_type = EntryKV; + lookup_function("map.iteratePrev")->mutate()->declared_return_type = EntryKV; + lookup_function("map.iteratePrev")->parameters[1].mutate()->declared_type = EntryKV; } } // namespace tolk diff --git a/tolk/compiler-state.cpp b/tolk/compiler-state.cpp index e11af94497..9615e929f3 100644 --- a/tolk/compiler-state.cpp +++ b/tolk/compiler-state.cpp @@ -70,6 +70,10 @@ const std::vector& get_all_not_builtin_functions() { return G.all_functions; } +const std::vector& get_all_declared_global_vars() { + return G.all_global_vars; +} + const std::vector& get_all_declared_constants() { return G.all_constants; } diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index 5c031e38de..4f76bc86c4 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -238,6 +238,20 @@ void GenericSubstitutionsDeducing::consider_next_condition(TypePtr param_type, T consider_next_condition(p_instSt->type_arguments[i], a_instSt->type_arguments[i]); } } + // `arg: Wrapper?` called as `f(Wrapper)` => T is int + if (const auto* a_union = arg_type->unwrap_alias()->try_as()) { + TypePtr variant_matches = nullptr; + int n_matches = 0; + for (TypePtr a_variant : a_union->variants) { + if (const auto* a_struct = a_variant->unwrap_alias()->try_as(); a_struct && a_struct->struct_ref->is_instantiation_of_generic_struct() && a_struct->struct_ref->base_struct_ref == p_instSt->struct_ref) { + variant_matches = a_variant; + n_matches++; + } + } + if (n_matches == 1) { + consider_next_condition(param_type, variant_matches); + } + } } else if (const auto* p_instAl = param_type->try_as(); p_instAl && p_instAl->alias_ref) { // `arg: WrapperAlias` called as `f(wrappedInt)` => T is int if (const auto* a_alias = arg_type->try_as(); a_alias && a_alias->alias_ref->is_instantiation_of_generic_alias() && a_alias->alias_ref->base_alias_ref == p_instAl->alias_ref) { @@ -246,6 +260,26 @@ void GenericSubstitutionsDeducing::consider_next_condition(TypePtr param_type, T consider_next_condition(p_instAl->type_arguments[i], a_alias->alias_ref->substitutedTs->typeT_at(i)); } } + } else if (const auto* p_map = param_type->try_as()) { + // `arg: map` called as `f(someMapInt32Slice)` => K = int32, V = slice + if (const auto* a_map = arg_type->unwrap_alias()->try_as()) { + consider_next_condition(p_map->TKey, a_map->TKey); + consider_next_condition(p_map->TValue, a_map->TValue); + } + // `arg: map?` called as `f(someMapInt32Slice)` => K = int32, V = slice + if (const auto* a_union = arg_type->unwrap_alias()->try_as()) { + TypePtr variant_matches = nullptr; + int n_matches = 0; + for (TypePtr a_variant : a_union->variants) { + if (a_variant->unwrap_alias()->try_as()) { + variant_matches = a_variant; + n_matches++; + } + } + if (n_matches == 1) { + consider_next_condition(param_type, variant_matches); + } + } } } @@ -478,6 +512,9 @@ bool is_allowed_asm_generic_function_with_non1_width_T(FunctionPtr fun_ref, int if (const auto* r_withTs = receiver->try_as()) { return r_withTs->struct_ref && r_withTs->struct_ref->name == "Cell"; } + if (receiver->try_as()) { + return true; + } } return false; diff --git a/tolk/maps-kv-api.cpp b/tolk/maps-kv-api.cpp new file mode 100644 index 0000000000..770450ae3b --- /dev/null +++ b/tolk/maps-kv-api.cpp @@ -0,0 +1,723 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "maps-kv-api.h" +#include "type-system.h" +#include "generics-helpers.h" +#include "pack-unpack-api.h" + +/* + * `map` is a high-level wrapper over TVM dictionaries. + * The compiler automatically constructs correct DICT asm instructions, takes care of packing slices, etc. + * + * In practice, K is most likely intN or address, V is any serializable value. + * If K is numeric, DICTI* or DICTU* instructions are used. + * If K is address or bitsN, DICT* instructions are used (internal address assumed). + * If K is complex, it's automatically packed into a slice, and DICT* instructions are used. + * + * On writing, DICTSETB instructions (providing a builder). Later, if a value is constant (so that a slice pushed + * into this builder is constant), it's replaced no DICTSET (not B) with a peephole. + * For writing `map>`, DICTSETREF instruction is used, but ONLY for writing (not for getting!). + * REF instructions are not used for getting to maintain the same external interface `MapLookupResult` and `MapEntry` + * with `loadValue()` etc. + */ + +namespace tolk { + +enum class DictKeyKind { + IntKey, // DICTI* + UIntKey, // DICTU* + SliceKey, // DICT* +}; + +enum class DictValueKind { + SliceValue, // DICTSET + BuilderValue, // DICTSETB + CellRefValue, // DICTSETREF +}; + + +// map should use DICTI* instructions +// note that `struct UserId { v: int32 }` also optimized, since it's just a signed int on a stack +static int is_TKey_TVM_int(TypePtr TKey) { + if (const TypeDataIntN* t_intN = TKey->try_as(); t_intN && !t_intN->is_variadic && !t_intN->is_unsigned) { + return t_intN->n_bits; + } + if (const TypeDataAlias* t_alias = TKey->try_as()) { + return is_TKey_TVM_int(t_alias->underlying_type); + } + if (const TypeDataStruct* t_struct = TKey->try_as(); t_struct && t_struct->struct_ref->get_num_fields() == 1) { + return is_TKey_TVM_int(t_struct->struct_ref->get_field(0)->declared_type); + } + if (TKey == TypeDataBool::create()) { // allow `bool` as a key with `DICTI` instructions + return true; + } + return 0; +} + +// map should use DICTU* instructions +static int is_TKey_TVM_uint(TypePtr TKey) { + if (const TypeDataIntN* t_intN = TKey->try_as(); t_intN && !t_intN->is_variadic && t_intN->is_unsigned) { + return t_intN->n_bits; + } + if (const TypeDataAlias* t_alias = TKey->try_as()) { + return is_TKey_TVM_uint(t_alias->underlying_type); + } + if (const TypeDataStruct* t_struct = TKey->try_as(); t_struct && t_struct->struct_ref->get_num_fields() == 1) { + return is_TKey_TVM_uint(t_struct->struct_ref->get_field(0)->declared_type); + } + return 0; +} + +// map should use DICT* instructions +// note that map is forbidden, since a raw slice doesn't define binary width +static int is_TKey_TVM_slice(TypePtr TKey) { + if (TKey == TypeDataAddress::create()) { + return 3 + 8 + 256; // "internal address" assumed + } + if (const TypeDataBitsN* t_bitsN = TKey->try_as()) { + return t_bitsN->is_bits ? t_bitsN->n_width : t_bitsN->n_width * 8; + } + if (const TypeDataAlias* t_alias = TKey->try_as()) { + return is_TKey_TVM_slice(t_alias->underlying_type); + } + if (const TypeDataStruct* t_struct = TKey->try_as(); t_struct && t_struct->struct_ref->get_num_fields() == 1) { + return is_TKey_TVM_slice(t_struct->struct_ref->get_field(0)->declared_type); + } + return 0; +} + +// we allow `map` and handle it separately, because we don't need to unpack it +static bool is_TValue_raw_slice(TypePtr TValue) { + return TValue->unwrap_alias() == TypeDataSlice::create(); +} + +// `map>` can emit SETREF instructions +static bool is_TValue_cell_or_CellT(TypePtr TValue) { + return TValue->unwrap_alias() == TypeDataCell::create() || is_type_cellT(TValue->unwrap_alias()); +} + +bool check_mapKV_TKey_is_valid(TypePtr TKey, std::string& because_msg) { + if (is_TKey_TVM_int(TKey) || is_TKey_TVM_uint(TKey) || is_TKey_TVM_slice(TKey)) { + return true; + } + + // okay, not a trivial key — it must be a serializable struct of a constant size + if (TKey->unwrap_alias() == TypeDataSlice::create()) { // a dedicated error message for `map` + because_msg = "because it does not specify keyLen for a dictionary\nhint: use `address` if a key is an internal address\nhint: use `bits128` and similar if a key represents fixed-width data"; + return false; + } + if (!check_struct_can_be_packed_or_unpacked(TKey, false, because_msg)) { + because_msg = "because it can not be serialized to slice\n" + because_msg; + return false; + } + + PackSize pack_size = estimate_serialization_size(TKey); + if (pack_size.min_bits != pack_size.max_bits) { + because_msg += "because its binary size is not constant: it's " + std::to_string(pack_size.min_bits) + ".." + std::to_string(pack_size.max_bits) + " bits"; + return false; + } + if (pack_size.min_bits > 1023) { + because_msg += "because its binary size is too large: " + std::to_string(pack_size.min_bits) + " bits"; + return false; + } + if (pack_size.max_refs) { + because_msg += "because it may contain a cell reference, not only data bits"; + return false; + } + return true; +} + +bool check_mapKV_TValue_is_valid(TypePtr TValue, std::string& because_msg) { + // we allow `slice` and `RemainingBitsAndRefs` as a value + if (is_TValue_raw_slice(TValue)) { + return true; + } + // or something that can be packed to/from slice + if (!check_struct_can_be_packed_or_unpacked(TValue, false, because_msg)) { + because_msg = "because it can not be serialized\n" + because_msg; + return false; + } + // note that `struct A { s: slice }` can not be used as a value (not serializable), + // although `slice` can, because in stdlib behavior for TValue=slice is overloaded (no deserialization) + + return true; +} + + +// an internal helper, having TKey and TValue, generate IR variables passed to __dict.* built-in functions +class DictKeyValue { + CodeBlob& code; + SrcLocation loc; + + DictKeyKind key_kind; + int key_len; + var_idx_t key_irv = -1; + + DictValueKind value_kind; + var_idx_t value_irv = -1; + +public: + + var_idx_t ir_key_kind() const { + return code.create_int(loc, static_cast(key_kind), "(key-kind)"); + } + + var_idx_t ir_value_kind() const { + return code.create_int(loc, static_cast(value_kind), "(value-kind)"); + } + + var_idx_t ir_key_len() const { + return code.create_int(loc, key_len, "(key-len)"); + } + + var_idx_t ir_key_val() const { + tolk_assert(key_irv != -1); + return key_irv; + } + + var_idx_t ir_value_val() const { + tolk_assert(value_irv != -1); + return value_irv; + } + + DictKeyValue(CodeBlob& code, SrcLocation loc, TypePtr TKey, const std::vector* exact_key, TypePtr TValue, const std::vector* exact_value, bool allow_REF_TValue = false) + : code(code) + , loc(loc) { + if (int i_bits = is_TKey_TVM_int(TKey)) { + key_kind = DictKeyKind::IntKey; + key_len = i_bits; + if (exact_key != nullptr) { + tolk_assert(exact_key->size() == 1); + key_irv = exact_key->at(0); + } + } else if (int u_bits = is_TKey_TVM_uint(TKey)) { + key_kind = DictKeyKind::UIntKey; + key_len = u_bits; + if (exact_key != nullptr) { + tolk_assert(exact_key->size() == 1); + key_irv = exact_key->at(0); + } + } else if (int s_bits = is_TKey_TVM_slice(TKey)) { + key_kind = DictKeyKind::SliceKey; + key_len = s_bits; + if (exact_key != nullptr) { + tolk_assert(exact_key->size() == 1); + key_irv = exact_key->at(0); + } + } else { + key_kind = DictKeyKind::SliceKey; + PackSize pack_size = EstimateContext().estimate_any(TKey); + tolk_assert(pack_size.max_refs == 0 && pack_size.min_bits == pack_size.max_bits); + key_len = pack_size.max_bits; + + if (exact_key != nullptr) { + std::vector ir_builder = code.create_tmp_var(TypeDataBuilder::create(), loc, "(map-keyB)"); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); + PackContext ctx(code, loc, ir_builder, create_default_PackOptions(code, loc)); + ctx.generate_pack_any(TKey, std::vector(*exact_key)); + std::vector ir_slice = code.create_tmp_var(TypeDataSlice::create(), loc, "(map-key)"); + // todo BTOS + code.emplace_back(loc, Op::_Call, ir_slice, ir_builder, lookup_function("builder.endCell")); + code.emplace_back(loc, Op::_Call, ir_slice, ir_slice, lookup_function("cell.beginParse")); + key_irv = ir_slice[0]; + } + } + + if (is_TValue_raw_slice(TValue)) { + value_kind = DictValueKind::SliceValue; + if (exact_value != nullptr) { + value_irv = exact_value->at(0); + } + } else if (allow_REF_TValue && is_TValue_cell_or_CellT(TValue)) { + // note that we use CellRefValue for writing only (not for reading, not for "set+get"): + // we don't emit REF for getters to match typing of MapLookupResult and MapEntry, + // so that `loadValue()` implemented in stdlib works universally for any V (particularly, Cell) + // (given `map.get`, DICTGET will be emitted, and loadValue() will load a ref correctly) + value_kind = DictValueKind::CellRefValue; + if (exact_value != nullptr) { + value_irv = exact_value->at(0); + } + } else { + value_kind = DictValueKind::BuilderValue; + + if (exact_value != nullptr) { + std::vector ir_builder = code.create_tmp_var(TypeDataBuilder::create(), loc, "(valueB)"); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); + PackContext ctx(code, loc, ir_builder, create_default_PackOptions(code, loc)); + ctx.generate_pack_any(TValue, std::vector(*exact_value)); + value_irv = ir_builder[0]; + } + } + } +}; + +// MapEntry is a built-in struct { rawValue: slice, key: K, isFound: bool } +// when used for numeric K, tvm instructions DICTI* and DICTU* return an integer key onto the stack +// when used for address/bitsN, tvm instructions DICT* return a slice key onto the stack +// so, in practice, we don't need any transformations from a TVM result, +// but when K is complex (like struct Point), TVM instructions return a slice, which is needed to be unpacked to K +GNU_ATTRIBUTE_NOINLINE +static std::vector construct_MapEntry_with_non_trivial_key(CodeBlob& code, SrcLocation loc, std::vector&& ir_entry, TypePtr TKey) { + tolk_assert(ir_entry.size() == 3); // slice value, slice key, isFound + + std::vector ir_key = code.create_tmp_var(TKey, loc, "(entry-key)"); + Op& if_found = code.emplace_back(loc, Op::_If, std::vector{ir_entry[2]}); + { + code.push_set_cur(if_found.block0); + UnpackContext ctx(code, loc, std::vector(ir_entry.begin() + 1, ir_entry.begin() + 2), create_default_UnpackOptions(code, loc)); + std::vector ir_unpacked_key = ctx.generate_unpack_any(TKey); + code.emplace_back(loc, Op::_Let, ir_key, std::move(ir_unpacked_key)); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_found.block1); + for (var_idx_t ith_null : ir_key) { + code.emplace_back(loc, Op::_Let, std::vector{ith_null}, std::vector{ir_entry[1]}); + } + code.close_pop_cur(loc); + } + + std::vector ir_result; + ir_result.reserve(2 + ir_key.size()); + ir_result.push_back(ir_entry[0]); // rawSlice + ir_result.insert(ir_result.end(), ir_key.begin(), ir_key.end()); + ir_result.push_back(ir_entry[2]); // isFound + return ir_result; +} + +static std::vector create_ir_MapEntry(CodeBlob& code, SrcLocation loc) { + return code.create_tmp_var(TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create(), TypeDataInt::create()}), loc, "(entry)"); +} + +// see a comment above construct_MapEntry_with_non_trivial_key() +static std::vector finalize_ir_MapEntry(CodeBlob& code, SrcLocation loc, std::vector&& ir_entry, TypePtr TKey) { + if (!is_TKey_TVM_int(TKey) && !is_TKey_TVM_uint(TKey) && !is_TKey_TVM_slice(TKey)) { + ir_entry = construct_MapEntry_with_non_trivial_key(code, loc, std::move(ir_entry), TKey); + } + return ir_entry; +} + + +// ---------------------------------- +// generating AsmOp and IR code +// + + +// "DICTSET" -> "DICTISETB" having key_kind and value_kind +static std::string choose_dict_op(std::string_view op_slice, VarDescr& var_with_key_kind, VarDescr& var_with_value_kind) { + std::string op(op_slice); + + DictKeyKind key_kind = static_cast(var_with_key_kind.int_const->to_long()); + var_with_key_kind.unused(); + + if (key_kind == DictKeyKind::UIntKey) op = "DICTU" + op.substr(4); + if (key_kind == DictKeyKind::IntKey) op = "DICTI" + op.substr(4); + + DictValueKind value_kind = static_cast(var_with_value_kind.int_const->to_long()); + var_with_value_kind.unused(); + + if (value_kind == DictValueKind::BuilderValue) op += "B"; + if (value_kind == DictValueKind::CellRefValue) op += "REF"; + + return op; +} + +// "empty map" is just NULL in TVM; it's extracted as a built-in to check for K/V correctness in advance +AsmOp compile_createEmptyMap(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 1 && args.empty()); + return AsmOp::Custom(loc, "NEWDICT", 0, 1); +} + +// "convert dict to map" is just NOP; it's extracted as a built-in to allow non-1 width K/V +AsmOp compile_createMapFromLowLevelDict(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 1 && args.size() == 1); + return AsmOp::Parse(loc, "NOP"); +} + +// DICTGET: k D n => (x −1) OR (0); + NULLSWAPIFNOT => (x -1) OR (null 0) +AsmOp compile_dict_get(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 2 && args.size() == 2+3); + std::string op = choose_dict_op("DICTGET", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT", 3, 2); +} + +// DICTGET: k D n => (x −1) OR (0) +// but since it's MUST get, it's wrapped to assert(found), so assume x (value) will exist on a stack +AsmOp compile_dict_mustGet(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 2 && args.size() == 2+3); + std::string op = choose_dict_op("DICTGET", args[0], args[1]); + return AsmOp::Custom(loc, op, 3, 2); +} + +// DICTMIN: D n => (x k −1) OR (0); + NULLSWAPIFNOT2 => (x k -1) OR (null null 0) +AsmOp compile_dict_getMin(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+2); + std::string op = choose_dict_op("DICTMIN", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT2", 2, 3); +} + +// DICTMAX: D n => (x k −1) OR (0); + NULLSWAPIFNOT2 => (x k -1) OR (null null 0) +AsmOp compile_dict_getMax(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+2); + std::string op = choose_dict_op("DICTMAX", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT2", 2, 3); +} + +// DICTGETNEXT: k D n => (x k −1) OR (0); + NULLSWAPIFNOT2 => (x k -1) OR (null null 0) +AsmOp compile_dict_getNext(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+3); + std::string op = choose_dict_op("DICTGETNEXT", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT2", 3, 3); +} + +// DICTGETNEXTEQ: k D n => (x k −1) OR (0); + NULLSWAPIFNOT2 => (x k -1) OR (null null 0) +AsmOp compile_dict_getNextEq(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+3); + std::string op = choose_dict_op("DICTGETNEXTEQ", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT2", 3, 3); +} + +// DICTGETPREV: k D n => (x k −1) OR (0); + NULLSWAPIFNOT2 => (x k -1) OR (null null 0) +AsmOp compile_dict_getPrev(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+3); + std::string op = choose_dict_op("DICTGETPREV", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT2", 3, 3); +} + +// DICTGETPREVEQ: k D n => (x k −1) OR (0); + NULLSWAPIFNOT2 => (x k -1) OR (null null 0) +AsmOp compile_dict_getPrevEq(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+3); + std::string op = choose_dict_op("DICTGETPREVEQ", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT2", 3, 3); +} + +// DICTSET: x k D n => D' +AsmOp compile_dict_set(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 1 && args.size() == 2+4); + std::string op = choose_dict_op("DICTSET", args[0], args[1]); + return AsmOp::Custom(loc, op, 4, 1); +} + +// DICTSETGET: x k D n => (D' y −1) or (D' 0); + NULLSWAPIFNOT => (D' y -1) OR (D' null 0) +AsmOp compile_dict_setGet(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+4); + std::string op = choose_dict_op("DICTSETGET", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT", 4, 3); +} + +// DICTREPLACE: x k D n => (D' -1) OR (D 0) +AsmOp compile_dict_replace(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 2 && args.size() == 2+4); + std::string op = choose_dict_op("DICTREPLACE", args[0], args[1]); + return AsmOp::Custom(loc, op, 4, 2); +} + +// DICTREPLACEGET: x k D n => (D' y -1) OR (D 0); + NULLSWAPIFNOT => (D' y -1) OR (D null 0) +AsmOp compile_dict_replaceGet(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+4); + std::string op = choose_dict_op("DICTREPLACEGET", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT", 4, 3); +} + +// DICTADD: x k D n => (D' -1) OR (D 0) +AsmOp compile_dict_add(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 2 && args.size() == 2+4); + std::string op = choose_dict_op("DICTADD", args[0], args[1]); + return AsmOp::Custom(loc, op, 4, 2); +} + +// DICTADDGET: x k D n => (D' -1) OR (D y 0); + NULLSWAPIF + NOT => (D' null 0) OR (D y -1) (from "isAdded" to "isFound") +AsmOp compile_dict_addGet(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+4); + std::string op = choose_dict_op("DICTADDGET", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIF" + " NOT", 4, 3); +} + +// DICTDEL: k D n => (D' -1) OR (D 0) +AsmOp compile_dict_del(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 2 && args.size() == 2+3); + std::string op = choose_dict_op("DICTDEL", args[0], args[1]); + return AsmOp::Custom(loc, op, 3, 2); +} + +// DICTDELGET: k D n => (D' x -1) OR (D 0); + NULLSWAPIFNOT => (D' x -1) OR (D null 0) +AsmOp compile_dict_delGet(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+3); + std::string op = choose_dict_op("DICTDELGET", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT", 3, 3); +} + + +// fun map.exists(self, key: K): bool +std::vector generate_mapKV_exists(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + + std::vector ir_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create()}), loc, "(lookup)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_lookup, std::move(dict_args), lookup_function("__dict.get")); + + return {ir_lookup[1]}; // isFound from (sliceOrNull isFound) +} + +// fun map.get(self, key: K): MapLookupResult +std::vector generate_mapKV_get(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + + std::vector ir_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create()}), loc, "(lookup)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_lookup, std::move(dict_args), lookup_function("__dict.get")); + + // in all functions where we return MapLookupResult: + // on a stack we have (slice, found) - exactly the shape of MapLookupResult; + // the user manually calls `result.loadValue()` after checking result.isFound + return ir_lookup; +} + +// fun map.mustGet(self, key: K, throwIfNotFound: int = 9): V +std::vector generate_mapKV_mustGet(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + TypePtr TValue = called_f->substitutedTs->typeT_at(1); + // since we don't return MapLookupResult, for `map>` we can use DICTGETREF + bool use_DICTGETREF = is_TValue_cell_or_CellT(TValue); + DictKeyValue kv(code, loc, TKey, &args[1], use_DICTGETREF ? TValue : TypeDataSlice::create(), nullptr, true); + + std::vector ir_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create()}), loc, "(lookup)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_lookup, std::move(dict_args), lookup_function("__dict.mustGet")); + + std::vector args_assert = { args[2][0], ir_lookup[1], code.create_int(loc, 0, "") }; + Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); + op_assert.set_impure_flag(); + + std::vector ir_slice(ir_lookup.begin(), ir_lookup.begin() + 1); + if (is_TValue_raw_slice(TValue)) { + return ir_slice; + } + if (use_DICTGETREF) { // ir_slice holds a cell actually + return ir_slice; // (it's exactly what we need, we need to return Cell) + } + + // load TValue and check for assertEnd (it's the default behavior) + UnpackContext ctx(code, loc, std::move(ir_slice), create_default_UnpackOptions(code, loc)); + std::vector ir_value = ctx.generate_unpack_any(TValue); + ctx.assertEndIfOption(); + return ir_value; +} + +// fun map.set(mutate self, key: K, value: V): self +std::vector generate_mapKV_set(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + TypePtr TValue = called_f->substitutedTs->typeT_at(1); + DictKeyValue kv(code, loc, TKey, &args[1], TValue, &args[2], true); + + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_value_val(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, args[0], std::move(dict_args), lookup_function("__dict.set")); + + return args[0]; // return mutated map +} + +// fun map.setAndGetPrevious(mutate self, key: K, value: V): MapLookupResult +std::vector generate_mapKV_setGet(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + TypePtr TValue = called_f->substitutedTs->typeT_at(1); + DictKeyValue kv(code, loc, TKey, &args[1], TValue, &args[2]); + + std::vector ir_map_and_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataSlice::create(), TypeDataInt::create()}), loc, "(map-and-lookup)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_value_val(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_map_and_lookup, std::move(dict_args), lookup_function("__dict.setGet")); + + return ir_map_and_lookup; +} + +// fun map.replaceIfExists(mutate self, key: K, value: V): bool +std::vector generate_mapKV_replace(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + TypePtr TValue = called_f->substitutedTs->typeT_at(1); + DictKeyValue kv(code, loc, TKey, &args[1], TValue, &args[2], true); + + std::vector ir_map_and_was_added = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataBool::create()}), loc, "(map-and-was-added)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_value_val(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_map_and_was_added, std::move(dict_args), lookup_function("__dict.replace")); + + return ir_map_and_was_added; +} + +// fun map.replaceAndGetPrevious(mutate self, key: K, value: V): MapLookupResult +std::vector generate_mapKV_replaceGet(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + TypePtr TValue = called_f->substitutedTs->typeT_at(1); + DictKeyValue kv(code, loc, TKey, &args[1], TValue, &args[2]); + + std::vector ir_map_and_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataSlice::create(), TypeDataInt::create()}), loc, "(map-and-lookup)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_value_val(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_map_and_lookup, std::move(dict_args), lookup_function("__dict.replaceGet")); + + return ir_map_and_lookup; +} + +// fun map.addIfNotExists(mutate self, key: K, value: V): bool +std::vector generate_mapKV_add(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + TypePtr TValue = called_f->substitutedTs->typeT_at(1); + DictKeyValue kv(code, loc, TKey, &args[1], TValue, &args[2], true); + + std::vector ir_map_and_was_added = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataBool::create()}), loc, "(map-and-was-added)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_value_val(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_map_and_was_added, std::move(dict_args), lookup_function("__dict.add")); + + return ir_map_and_was_added; +} + +// fun map.addOrGetExisting(mutate self, key: K, value: V): MapLookupResult +std::vector generate_mapKV_addGet(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + TypePtr TValue = called_f->substitutedTs->typeT_at(1); + DictKeyValue kv(code, loc, TKey, &args[1], TValue, &args[2]); + + std::vector ir_map_and_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataSlice::create(), TypeDataInt::create()}), loc, "(lookup)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_value_val(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_map_and_lookup, std::move(dict_args), lookup_function("__dict.addGet")); + + return ir_map_and_lookup; +} + +// fun map.delete(mutate self, key: K): bool +std::vector generate_mapKV_del(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + + std::vector ir_map_and_was_deleted = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataBool::create()}), loc, "(map-and-was-added)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_map_and_was_deleted, std::move(dict_args), lookup_function("__dict.del")); + + return ir_map_and_was_deleted; +} + +// fun map.deleteAndGetDeleted(mutate self, key: K): MapLookupResult +std::vector generate_mapKV_delGet(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + + std::vector ir_map_and_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataSlice::create(), TypeDataInt::create()}), loc, "(lookup)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_map_and_lookup, std::move(dict_args), lookup_function("__dict.delGet")); + + return ir_map_and_lookup; +} + +// fun map.findFirst(): MapEntry +std::vector generate_mapKV_findFirst(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, nullptr, TypeDataSlice::create(), nullptr); + + // on a stack (ir_entry) we will have: either (x k −1) or (null null 0) + std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getMin")); + + return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); +} + +// fun map.findLast(): MapEntry +std::vector generate_mapKV_findLast(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, nullptr, TypeDataSlice::create(), nullptr); + + std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getMax")); + + return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); +} + +// fun map.findKeyGreater(pivotKey: K): MapEntry +std::vector generate_mapKV_findKeyGreater(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + + std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getNext")); + + return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); +} + +// fun map.findKeyGreaterOrEqual(pivotKey: K): MapEntry +std::vector generate_mapKV_findKeyGreaterOrEqual(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + + std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getNextEq")); + + return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); +} + +// fun map.findKeyLess(pivotKey: K): MapEntry +std::vector generate_mapKV_findKeyLess(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + + std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getPrev")); + + return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); +} + +// fun map.findKeyLessOrEqual(pivotKey: K): MapEntry +std::vector generate_mapKV_findKeyLessOrEqual(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + + std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getPrevEq")); + + return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); +} + +// fun map.iterateNext(current: MapEntry): MapEntry +std::vector generate_mapKV_iterateNext(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + std::vector ir_pivot_key(args[1].begin() + 1, args[1].end() - 1); + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &ir_pivot_key, TypeDataSlice::create(), nullptr); + + std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getNext")); + + return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); +} + +// fun map.iteratePrev(current: MapEntry): MapEntry +std::vector generate_mapKV_iteratePrev(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + std::vector ir_pivot_key(args[1].begin() + 1, args[1].end() - 1); + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &ir_pivot_key, TypeDataSlice::create(), nullptr); + + std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getPrev")); + + return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); +} + +} // namespace tolk diff --git a/tolk/maps-kv-api.h b/tolk/maps-kv-api.h new file mode 100644 index 0000000000..d6d64adc41 --- /dev/null +++ b/tolk/maps-kv-api.h @@ -0,0 +1,27 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "fwd-declarations.h" +#include "tolk.h" + +namespace tolk { + +bool check_mapKV_TKey_is_valid(TypePtr TKey, std::string& because_msg); +bool check_mapKV_TValue_is_valid(TypePtr TValue, std::string& because_msg); + +} // namespace tolk diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index 93c6e20675..a3a0e29162 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -422,6 +422,43 @@ bool Optimizer::detect_rewrite_NOT_THROWIF() { return false; } +// pattern `NEWC` + store const slice + XCHG + keyLen + DICTSETB -> push const slice + XCHG + keyLen + DICTSET +// (useful for `someMap.set(k, constVal)` where constVal is represented as a const slice) +bool Optimizer::detect_rewrite_DICTSETB_DICTSET() { + bool fifth_dict = pb_ >= 5 && op_[4]->is_custom() && op_[4]->op.starts_with("DICT"); + if (!fifth_dict) { + return false; + } + + bool first_newc = op_[0]->op == "NEWC"; + bool second_stsliceconst = op_[1]->op.ends_with(" STSLICECONST"); + bool third_xchg = op_[2]->is_xchg() || op_[2]->op == "ROT" || op_[2]->op == "-ROT" || op_[2]->op.ends_with(" PUXC"); + bool fourth_pushint = op_[3]->is_const() && op_[3]->op.ends_with(" PUSHINT"); + if (!first_newc || !second_stsliceconst || !third_xchg || !fourth_pushint) { + return false; + } + + static const char* contains_b[] = {"SETB", "REPLACEB", "ADDB", "GETB"}; + static const char* repl_with[] = {"SET", "REPLACE", "ADD", "GET" }; + + std::string new_op = op_[4]->op; // "DICTSET" / "DICTSETGET NULLSWAPIFNOT" + for (size_t i = 0; i < std::size(contains_b); ++i) { + if (size_t pos = new_op.find(contains_b[i]); pos == 4 || pos == 5) { + new_op.replace(pos, std::strlen(contains_b[i]), repl_with[i]); + p_ = 5; + q_ = 4; + oq_[0] = std::make_unique(AsmOp::Custom(op_[1]->loc, op_[1]->op.substr(0, op_[1]->op.rfind(' ')) + " PUSHSLICE", 0, 1)); + oq_[1] = std::move(op_[2]); + oq_[2] = std::move(op_[3]); + oq_[3] = std::make_unique(AsmOp::Custom(op_[4]->loc, new_op)); + return true; + } + } + + return false; +} + + bool Optimizer::is_push_const(int* i, int* c) const { return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_push_const(i, c); } @@ -844,7 +881,7 @@ bool Optimizer::find_at_least(int pb) { detect_rewrite_MY_store_int() || detect_rewrite_MY_skip_bits() || detect_rewrite_NEWC_PUSH_STUR() || detect_rewrite_LDxx_DROP() || detect_rewrite_SWAP_symmetric() || detect_rewrite_SWAP_PUSH_STUR() || detect_rewrite_SWAP_STxxxR() || - detect_rewrite_NOT_THROWIF() || + detect_rewrite_NOT_THROWIF() || detect_rewrite_DICTSETB_DICTSET() || (!(mode_ & 1) && ((is_rot() && rewrite(AsmOp::Custom(loc, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(loc, "-ROT", 3, 3))) || (is_2dup() && rewrite(AsmOp::Custom(loc, "2DUP", 2, 4))) || diff --git a/tolk/overload-resolution.cpp b/tolk/overload-resolution.cpp index e82db073bb..ed98bf250c 100644 --- a/tolk/overload-resolution.cpp +++ b/tolk/overload-resolution.cpp @@ -97,6 +97,11 @@ static ShapeScore calculate_shape_score(TypePtr t) { return {ShapeKind::Instantiated, 1 + d}; } + if (const auto* t_map = t->try_as()) { + int d = std::max(calculate_shape_score(t_map->TKey).depth, calculate_shape_score(t_map->TValue).depth); + return {ShapeKind::Instantiated, 1 + d}; + } + if (const auto* t_alias = t->try_as()) { return calculate_shape_score(t_alias->underlying_type); } diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index 213476630a..1c78222600 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -96,6 +96,9 @@ class PackUnpackAvailabilityChecker { if (any_type == TypeDataNever::create()) { return {}; } + if (any_type->try_as()) { + return {}; + } if (const auto* t_struct = any_type->try_as()) { StructPtr struct_ref = t_struct->struct_ref; diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index 31df5e9665..e65679712f 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -1163,6 +1163,33 @@ std::vector auto_generate_opcodes_for_union(TypePtr union_type, std: return result; } +// there is no way to pass custom pack options to createMessage / map.set / etc., using hardcoded ones +std::vector create_default_PackOptions(CodeBlob& code, SrcLocation loc) { + StructPtr s_PackOptions = lookup_global_symbol("PackOptions")->try_as(); + std::vector ir_options = code.create_tmp_var(TypeDataStruct::create(s_PackOptions), loc, "(pack-options)"); + tolk_assert(ir_options.size() == 1); + + std::vector ir_defaults = { + code.create_int(loc, 0, "(zero)"), // skipBitsNFieldsValidation + }; + code.emplace_back(loc, Op::_Let, ir_options, std::move(ir_defaults)); + return ir_options; +} + +// there is no way to pass custom unpack options to map.get / etc., using hardcoded ones +std::vector create_default_UnpackOptions(CodeBlob& code, SrcLocation loc) { + StructPtr s_UnpackOptions = lookup_global_symbol("UnpackOptions")->try_as(); + std::vector ir_options = code.create_tmp_var(TypeDataStruct::create(s_UnpackOptions), loc, "(unpack-options)"); + tolk_assert(ir_options.size() == 2); + + std::vector ir_defaults = { + code.create_int(loc, -1, "(true)"), // assertEndAfterReading + code.create_int(loc, 63, "(excno)"), // throwIfOpcodeDoesNotMatch + }; + code.emplace_back(loc, Op::_Let, ir_options, std::move(ir_defaults)); + return ir_options; +} + // -------------------------------------------- // detect serializer by TypePtr @@ -1207,6 +1234,9 @@ static std::unique_ptr get_serializer_for_type(TypePtr any_type) { return std::make_unique(); } + if (any_type->try_as()) { + return std::make_unique(); + } if (const auto* t_struct = any_type->try_as()) { return std::make_unique(t_struct->struct_ref); } diff --git a/tolk/pack-unpack-serializers.h b/tolk/pack-unpack-serializers.h index ab11733aa5..0f00bb0468 100644 --- a/tolk/pack-unpack-serializers.h +++ b/tolk/pack-unpack-serializers.h @@ -166,4 +166,7 @@ bool is_type_cellT(TypePtr any_type); FunctionPtr get_custom_pack_unpack_function(TypePtr receiver_type, bool is_pack); std::vector auto_generate_opcodes_for_union(TypePtr union_type, std::string& because_msg); +std::vector create_default_PackOptions(CodeBlob& code, SrcLocation loc); +std::vector create_default_UnpackOptions(CodeBlob& code, SrcLocation loc); + } // namespace tolk diff --git a/tolk/pipe-check-serialized-fields.cpp b/tolk/pipe-check-serialized-fields.cpp index f8b9b9cd02..762c29bacc 100644 --- a/tolk/pipe-check-serialized-fields.cpp +++ b/tolk/pipe-check-serialized-fields.cpp @@ -18,6 +18,7 @@ #include "ast.h" #include "ast-visitor.h" #include "pack-unpack-api.h" +#include "maps-kv-api.h" #include "generics-helpers.h" #include "type-system.h" @@ -42,6 +43,36 @@ static void fire_error_theoretical_overflow_1023(StructPtr struct_ref, PackSize ); } +GNU_ATTRIBUTE_NOINLINE +static void check_map_TKey_TValue(SrcLocation loc, TypePtr TKey, TypePtr TValue) { + std::string because_msg; + if (!check_mapKV_TKey_is_valid(TKey, because_msg)) { + fire(nullptr, loc, "invalid `map`: type `" + TKey->as_human_readable() + "` can not be used as a key\n" + because_msg); + } + if (!check_mapKV_TValue_is_valid(TValue, because_msg)) { + fire(nullptr, loc, "invalid `map`: type `" + TValue->as_human_readable() + "` can not be used as a value\n" + because_msg); + } +} + +GNU_ATTRIBUTE_NOINLINE +static void check_mapKV_inside_type(SrcLocation loc, TypePtr any_type) { + any_type->replace_children_custom([loc](TypePtr child) { + if (const TypeDataMapKV* t_map = child->try_as()) { + check_map_TKey_TValue(loc, t_map->TKey, t_map->TValue); + } + if (const TypeDataAlias* t_alias = child->try_as()) { + check_mapKV_inside_type(loc, t_alias->underlying_type); + } + return child; + }); +} + +static void check_mapKV_inside_type(AnyTypeV type_node) { + if (type_node && type_node->resolved_type->has_mapKV_inside()) { + check_mapKV_inside_type(type_node->loc, type_node->resolved_type); + } +} + class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody { FunctionPtr cur_f = nullptr; @@ -73,12 +104,19 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody } void visit(V v) override { + parent::visit(v); + FunctionPtr fun_ref = v->fun_maybe; - if (!fun_ref || !fun_ref->is_compile_time_special_gen() || !fun_ref->is_instantiation_of_generic_function()) { + if (!fun_ref || !fun_ref->is_builtin_function() || !fun_ref->is_instantiation_of_generic_function()) { return; } - std::string_view f_name = fun_ref->base_fun_ref->name; + + if (f_name == "createEmptyMap") { + check_map_TKey_TValue(v->loc, fun_ref->substitutedTs->typeT_at(0), fun_ref->substitutedTs->typeT_at(1)); + return; + } + TypePtr serialized_type = nullptr; bool is_pack = false; if (f_name == "Cell.load" || f_name == "T.fromSlice" || f_name == "T.fromCell" || f_name == "T.toCell" || @@ -99,19 +137,37 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody check_type_fits_cell_or_has_policy(serialized_type); } - public: + void visit(V v) override { + check_mapKV_inside_type(v->type_node); + } + +public: bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } void start_visiting_function(FunctionPtr fun_ref, V v_function) override { cur_f = fun_ref; + + for (int i = 0; i < cur_f->get_num_params(); ++i) { + check_mapKV_inside_type(cur_f->get_param(i).type_node); + } + parent::visit(v_function->get_body()); } }; void pipeline_check_serialized_fields() { visit_ast_of_all_functions(); + + for (StructPtr struct_ref : get_all_declared_structs()) { + for (StructFieldPtr field_ref : struct_ref->fields) { + check_mapKV_inside_type(field_ref->type_node); + } + } + for (GlobalVarPtr glob_ref : get_all_declared_global_vars()) { + check_mapKV_inside_type(glob_ref->type_node); + } } } // namespace tolk diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 7ca035cfd4..67f5dd0c7d 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -170,6 +170,12 @@ class TypeNodesVisitorResolver { // if we're inside `f`, replace "T" with TypeDataInt return substitutedTs->get_substitution_for_nameT(text); } + if (text == "map") { + if (!allow_without_type_arguments) { + fire_error_generic_type_used_without_T(cur_f, loc, "map"); + } + return TypeDataMapKV::create(TypeDataGenericT::create("K"), TypeDataGenericT::create("V")); + } if (const Symbol* sym = lookup_global_symbol(text)) { if (TypePtr custom_type = try_resolve_user_defined_type(cur_f, loc, sym, allow_without_type_arguments)) { if (!v->loc.is_symbol_from_same_or_builtin_file(sym->loc)) { @@ -298,6 +304,12 @@ class TypeNodesVisitorResolver { } return TypeDataAlias::create(instantiate_generic_alias(alias_ref, GenericsSubstitutions(alias_ref->genericTs, type_arguments))); } + if (const TypeDataMapKV* t_map = type_to_instantiate->try_as(); t_map && t_map->TKey->try_as()) { + if (type_arguments.size() != 2) { + fire(cur_f, loc, "type `map` expects 2 type arguments, but " + std::to_string(type_arguments.size()) + " provided"); + } + return TypeDataMapKV::create(type_arguments[0], type_arguments[1]); + } if (const TypeDataGenericT* asT = type_to_instantiate->try_as()) { fire_error_unknown_type_name(cur_f, loc, asT->nameT); } diff --git a/tolk/send-message-api.cpp b/tolk/send-message-api.cpp index 2048322936..b5ef61aa7a 100644 --- a/tolk/send-message-api.cpp +++ b/tolk/send-message-api.cpp @@ -32,17 +32,6 @@ static bool is_type_UnsafeBodyNoRef_T(TypePtr bodyT) { return false; } -// currently, there is no way to pass custom pack options to createMessage, using hardcoded ones -static std::vector create_default_PackOptions(CodeBlob& code, SrcLocation loc) { - StructPtr s_PackOptions = lookup_global_symbol("PackOptions")->try_as(); - std::vector ir_options = code.create_tmp_var(TypeDataStruct::create(s_PackOptions), loc, "(pack-options)"); - tolk_assert(ir_options.size() == 1); - - var_idx_t ir_zero = code.create_int(loc, 0, "(zero)"); - code.emplace_back(loc, Op::_Let, std::vector{ir_options[0]}, std::vector{ir_zero}); // skipBitsNFieldsValidation - return ir_options; -} - // calculate `addrHash &= mask` where mask = `(1 << (256 - SHARD_DEPTH)) - 1` static void append_bitwise_and_shard_mask(CodeBlob& code, SrcLocation loc, var_idx_t ir_addr_hash, var_idx_t ir_shard_depth) { var_idx_t ir_one = code.create_int(loc, 1, "(one)"); diff --git a/tolk/tolk.h b/tolk/tolk.h index 43bce76496..13a9acefe3 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -907,6 +907,7 @@ struct Optimizer { bool detect_rewrite_SWAP_PUSH_STUR(); bool detect_rewrite_SWAP_STxxxR(); bool detect_rewrite_NOT_THROWIF(); + bool detect_rewrite_DICTSETB_DICTSET(); AsmOpConsList extract_code(); }; diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index b72da00dab..edd422400c 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -236,6 +236,13 @@ TypePtr TypeDataUnion::create(std::vector&& variants) { return new TypeDataUnion(reg.children_flags(), or_null, std::move(flat_variants)); } +TypePtr TypeDataMapKV::create(TypePtr TKey, TypePtr TValue) { + CalcChildrenFlags reg; + reg.feed_child(TKey); + reg.feed_child(TValue); + return new TypeDataMapKV(reg.children_flags(), TKey, TValue); +} + // -------------------------------------------- // get_width_on_stack() @@ -362,6 +369,11 @@ int TypeDataUnion::get_type_id() const { throw Fatal("unexpected get_type_id() call"); } +int TypeDataMapKV::get_type_id() const { + assert(!has_genericT_inside()); + return TypeIdCalculation::assign_type_id(this); +} + int TypeDataUnknown::get_type_id() const { assert(false); // unknown can not be inside a union throw Fatal("unexpected get_type_id() call"); @@ -469,6 +481,10 @@ std::string TypeDataUnion::as_human_readable() const { return result; } +std::string TypeDataMapKV::as_human_readable() const { + return "map<" + TKey->as_human_readable() + ", " + TValue->as_human_readable() + ">"; +} + // -------------------------------------------- // replace_children_custom() @@ -523,6 +539,10 @@ TypePtr TypeDataUnion::replace_children_custom(const ReplacerCallbackT& callback return callback(create(std::move(mapped))); } +TypePtr TypeDataMapKV::replace_children_custom(const ReplacerCallbackT& callback) const { + return callback(create(TKey->replace_children_custom(callback), TValue->replace_children_custom(callback))); +} + // -------------------------------------------- // can_rhs_be_assigned() @@ -767,6 +787,16 @@ bool TypeDataUnion::can_rhs_be_assigned(TypePtr rhs) const { return rhs == TypeDataNever::create(); } +bool TypeDataMapKV::can_rhs_be_assigned(TypePtr rhs) const { + if (const TypeDataMapKV* rhs_map = rhs->try_as()) { + return TKey->equal_to(rhs_map->TKey) && TValue->equal_to(rhs_map->TValue); + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } + return rhs == TypeDataNever::create(); +} + bool TypeDataUnknown::can_rhs_be_assigned(TypePtr rhs) const { return true; } @@ -1065,6 +1095,19 @@ bool TypeDataUnion::can_be_casted_with_as_operator(TypePtr cast_to) const { return false; } +bool TypeDataMapKV::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const TypeDataMapKV* to_map = cast_to->try_as()) { + return TKey->equal_to(to_map->TKey) && TValue->equal_to(to_map->TValue); + } + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + return false; +} + bool TypeDataUnknown::can_be_casted_with_as_operator(TypePtr cast_to) const { // 'unknown' can be cast to any TVM value return cast_to->get_width_on_stack() == 1; @@ -1123,6 +1166,10 @@ bool TypeDataUnion::can_hold_tvm_null_instead() const { return or_null && !or_null->can_hold_tvm_null_instead(); } +bool TypeDataMapKV::can_hold_tvm_null_instead() const { + return false; // map is an optional cell, so `map?` requires a nullable presence slot +} + bool TypeDataNever::can_hold_tvm_null_instead() const { return false; } @@ -1265,6 +1312,16 @@ bool TypeDataUnion::equal_to(TypePtr rhs) const { return false; } +bool TypeDataMapKV::equal_to(TypePtr rhs) const { + if (const TypeDataMapKV* rhs_map = rhs->try_as()) { + return TKey->equal_to(rhs_map->TKey) && TValue->equal_to(rhs_map->TValue); + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + // union types creation is a bit tricky: nested unions are flattened, duplicates are removed // so, a resolved union type has variants, each will be assigned a unique type_id (tagged unions) diff --git a/tolk/type-system.h b/tolk/type-system.h index 0fdbc26e69..a6da3d8fa2 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -49,6 +49,7 @@ class TypeData { flag_contains_unknown_inside = 1 << 1, flag_contains_genericT_inside = 1 << 2, flag_contains_type_alias_inside = 1 << 3, + flag_contains_mapKV_inside = 1 << 4, }; explicit TypeData(int flags_with_children) @@ -77,6 +78,7 @@ class TypeData { bool has_unknown_inside() const { return flags & flag_contains_unknown_inside; } bool has_genericT_inside() const { return flags & flag_contains_genericT_inside; } bool has_type_alias_inside() const { return flags & flag_contains_type_alias_inside; } + bool has_mapKV_inside() const { return flags & flag_contains_mapKV_inside; } using ReplacerCallbackT = std::function; @@ -582,6 +584,33 @@ class TypeDataUnion final : public TypeData { bool equal_to(TypePtr rhs) const override; }; +/* + * `map` is a built-in type, a high-level wrapper over TVM dictionaries. + * Internally, a map is just a nullable cell: null represents an empty dict, otherwise it points to a root cell. + * The compiler checks that key-value types are correct, generates optimal TVM instructions, + * auto-serialization of V and non-primitive K to slices, etc. Deeply integrated with stdlib. + */ +class TypeDataMapKV final : public TypeData { + TypeDataMapKV(int children_flags, TypePtr TKey, TypePtr TValue) + : TypeData(children_flags | flag_contains_mapKV_inside) + , TKey(TKey) + , TValue(TValue) {} + +public: + TypePtr TKey; + TypePtr TValue; + + static TypePtr create(TypePtr TKey, TypePtr TValue); + + int get_type_id() const override; + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; + bool can_hold_tvm_null_instead() const override; + bool equal_to(TypePtr rhs) const override; +}; + /* * `unknown` is a special type, which can appear in corner cases. * The type of exception argument (which can hold any TVM value at runtime) is unknown. From 36fe5fd4b624eeebf2d698e5d0cc5f5a656d3ea3 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 8 Aug 2025 18:45:01 +0700 Subject: [PATCH 394/524] [Tolk] Modify stdlib for rendering, add built-in types --- crypto/smartcont/tolk-stdlib/common.tolk | 474 ++++++++++-------- .../smartcont/tolk-stdlib/gas-payments.tolk | 10 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 8 +- .../smartcont/tolk-stdlib/tvm-lowlevel.tolk | 4 +- tolk/ast-from-tokens.cpp | 5 + 5 files changed, 292 insertions(+), 209 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index d42ff93661..2228036d3f 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -3,6 +3,130 @@ // More specific functions are required to be imported explicitly, like "@stdlib/gas-payments". tolk 1.0 +/** + Built-in types. + */ + +/// `int` is the primitive 257-bit signed integer type. +type int = builtin + +/// `bool` is a classic boolean type, which can hold only two values: `true` and `false`. +/// At the TVM (TON virtual machine) level, it's an integer -1 or 0. +/// Note: `boolVar as int` is possible, but remember, that true is -1, not 1! +type bool = builtin + +/// `cell` is a data structure, which can hold of up to 1023 bits (not bytes!) +/// and up to 4 references (refs) to other cells. +/// Both contract code and contract state are represented by a tree of cells. +/// [See docs](https://docs.ton.org/v3/documentation/data-formats/tlb/cell-boc) +type cell = builtin + +/// `slice` is a "cell opened for reading". +/// When you call [cell.beginParse], you get a `slice`, from which you can load binary data +/// or high-level structures with [T.fromSlice]. +type slice = builtin + +/// `builder` is a "cell at the stage of creation". +/// When you call [beginCell], you get a `builder`, populate it with binary data or structures, +/// and after [builder.endCell], you get a `cell`. +type builder = builtin + +/// `continuation` is an "executable cell" representing executable TVM bytecode. +/// They are used to manage execution flow in TVM programs and serve as +/// the basis for function calls, exception handling, and control flow operations. +type continuation = builtin + +/// `tuple` is a collection from 0 to 255 elements of any type. +/// You can push, pop, access individual elements as `someTuple.0`. +/// A tuple occupies one stack slot regardless of its size. +type tuple = builtin + +/// `address` represents an internal/external/none address. +/// Most likely, you'll use it for internal addresses — "an address of a smart contract". +/// It's `slice` under the hood. `someAddress as slice` is also possible. +/// [See docs](https://docs.ton.org/learn/overviews/addresses#address-of-smart-contract) +type address = builtin + +/// `never` is a special type that represents computations that never complete normally. +/// A function that always throws an exception returns `never`, and the compiler knows +/// that any code after it is unreachable. +type never = builtin + +/// `int8`, `int32`, `int222`, etc. is "a fixed-width signed integer with N bits", N <= 257. +/// Note: it's still `int` at runtime, you can assign "100500" to "int8": +/// overflow will happen at serialization to a cell/builder, NOT at assignment. +type intN = builtin + +/// `uint32`, `uint64`, `uint111`, etc. is "a fixed-width unsigned integer with N bits", N <= 256. +/// Note: it's still `int` at runtime, you can assign "100500" to "uint8": +/// overflow will happen at serialization to a cell/builder, NOT at assignment. +type uintN = builtin + +/// `coins` is a special primitive representing "nanotoncoins". One TON = 10^9 nanotoncoins. +/// You can create coins with `ton()` function: `ton("0.05")` (actually, `int` 50000000 at runtime). +/// Arithmetic operations on `coins` degrade to 257-bit `int` type. +type coins = builtin + +/// `varint16` is `int` at runtime, but serialized as "variadic signed int", -2^119 <= X < 2^119. +type varint16 = builtin + +/// `varuint16` is `int` at runtime, but serialized as "variadic unsigned int", 0 <= X < 2^120. +type varuint16 = builtin + +/// `varint32` is `int` at runtime, but serialized as "variadic signed int", -2^247 <= X < 2^247. +type varint32 = builtin + +/// `varuint32` is `int` at runtime, but serialized as "variadic unsigned int", 0 <= X < 2^248. +type varuint32 = builtin + +/// `bits256`, `bits111`, etc. is "a fixed-width slice with N bits and 0 refs", N <= 1023. +/// Note: use `as` operator to convert `slice` to `bitsN`: `someSlice as bits256` +/// (manually writing `as` enforces you to think that this conversion is correct). +/// Note: similar to `intN`, you can assign an invalid slice to `bitsN`, +/// an error will be fired at serialization with [T.toCell] and similar, NOT at assignment. +type bitsN = builtin + +/// `bytes8`, `bytes99`, etc. is a convenient alias for `bits(N*8)` +type bytesN = builtin + +/// `map` is "a map from a key K to a value V". +/// Internally, it's an "optional cell": an empty map is `null`, a non-empty points to a root cell. +/// Restrictions for K and V types: +/// - a key must be fixed-width; valid: `int32`, `uint64`, `address`, `bits256`, `Point`; invalid: `int`, `coins` +/// - a value must be serializable; valid: `int32`, `coins`, `AnyStruct`, `Cell`; invalid: `int`, `builder` +struct map { + tvmDict: dict +} + +/// `dict` is a low-level TVM dictionary. +/// Think of it as "a map with unknown keys and unknown values". +/// Prefer using `map`, not `dict`. +type dict = cell? + +/// `void` is the unit type representing the absence of a meaningful value. +/// It's similar to both `void` and `unit` in other languages. +/// Note: a function without return type means "auto infer", NOT "void". +type void = builtin + +/// `Cell` represents a typed cell reference (as opposed to untyped `cell`). +/// ``` +/// struct ExtraData { ... } +/// +/// struct MyStorage { +/// ... +/// extra: Cell // TL-B `^ExtraData` +/// optional: Cell? // TL-B `(Maybe ^ExtraData)` +/// code: cell // TL-B `^Cell` +/// data: cell? // TL-B `(Maybe ^Cell)` +/// } +/// ``` +/// Note, that `st = MyStorage.fromSlice(s)` does NOT deep-load any refs; `st.extra` is `Cell`, not `T`; +/// you should manually call `st.extra.load()` to get `T` (ExtraData in this example). +struct Cell { + tvmCell: cell +} + + /** Tuple manipulation primitives. Elements of a tuple can be of arbitrary type. @@ -26,14 +150,14 @@ fun tuple.push(mutate self, value: T): void fun tuple.first(self): T asm "FIRST" -/// Returns the [`index`]-th element of a tuple. +/// Returns the `i`-th element of a tuple. /// `t.i` is actually the same as `t.get(i)` @pure fun tuple.get(self, index: int): T builtin -/// Sets the [`index`]-th element of a tuple to a specified value -/// (element with this index must already exist, a new element isn't created). +/// Sets the `i`-th element of a tuple to a specified value +/// (an element with this index must already exist, a new element is not created). /// `t.i = value` is actually the same as `t.set(value, i)` @pure fun tuple.set(mutate self, value: T, index: int): void @@ -61,7 +185,7 @@ fun tuple.pop(mutate self): T /// Converts a constant floating-point string to nanotoncoins. /// Example: `ton("0.05")` is equal to 50000000. -/// Note, that `ton()` requires a constant string; `ton(some_var)` is an error +/// Note, that `ton()` requires a constant string; `ton(some_var)` is an error. @pure fun ton(floatString: slice): coins builtin @@ -92,33 +216,33 @@ fun abs(x: int): int fun sign(x: int): int asm "SGN" -/// Computes the quotient and remainder of [x] / [y]. Example: divMod(112,3) = (37,1) +/// Computes the quotient and remainder of x / y. Example: divMod(112,3) = (37,1) @pure fun divMod(x: int, y: int): (int, int) asm "DIVMOD" -/// Computes the remainder and quotient of [x] / [y]. Example: modDiv(112,3) = (1,37) +/// Computes the remainder and quotient of x / y. Example: modDiv(112,3) = (1,37) @pure fun modDiv(x: int, y: int): (int, int) asm(-> 1 0) "DIVMOD" -/// Computes multiple-then-divide: floor([x] * [y] / [z]). +/// Computes multiple-then-divide: floor(x * y / z). /// The intermediate result is stored in a 513-bit integer to prevent precision loss. @pure fun mulDivFloor(x: int, y: int, z: int): int builtin -/// Similar to `mulDivFloor`, but rounds the result: round([x] * [y] / [z]). +/// Similar to [mulDivFloor], but rounds the result: round(x * y / z). @pure fun mulDivRound(x: int, y: int, z: int): int builtin -/// Similar to `mulDivFloor`, but ceils the result: ceil([x] * [y] / [z]). +/// Similar to [mulDivFloor], but ceils the result: ceil(x * y / z). @pure fun mulDivCeil(x: int, y: int, z: int): int builtin -/// Computes the quotient and remainder of ([x] * [y] / [z]). Example: mulDivMod(112,3,10) = (33,6) +/// Computes the quotient and remainder of (x * y / z). Example: mulDivMod(112,3,10) = (33,6) @pure fun mulDivMod(x: int, y: int, z: int): (int, int) builtin @@ -139,12 +263,11 @@ fun contract.getAddress(): address asm "MYADDR" /// Returns the balance (in nanotoncoins) of the smart contract at the start of Computation Phase. -/// Note that RAW primitives such as [sendMessage] do not update this field. @pure fun contract.getOriginalBalance(): coins asm "BALANCE" "FIRST" -/// Same as [contract.getOriginalBalance], but returns a tuple: balance in nanotoncoins and extra currencies +/// Same as [contract.getOriginalBalance], but returns a tuple: balance in nanotoncoins and extra currencies. @pure fun contract.getOriginalBalanceWithExtraCurrencies(): [coins, ExtraCurrenciesMap] asm "BALANCE" @@ -154,17 +277,16 @@ fun contract.getOriginalBalanceWithExtraCurrencies(): [coins, ExtraCurrenciesMap fun contract.getData(): cell asm "c4 PUSH" -/// Sets `cell` [c] as persistent contract data. You can update persistent contract storage with this primitive. +/// Sets the persistent contract storage. fun contract.setData(c: cell): void asm "c4 POP" -/// Retrieves code of smart-contract from c7 +/// Retrieves code of smart-contract from c7. @pure fun contract.getCode(): cell asm "MYCODE" -/// Creates an output action that would change this smart contract code to that given by cell [newCode]. -/// Notice that this change will take effect only after the successful termination of the current run of the smart contract. +/// Creates an output action that would change this smart contract code after successful termination of the current run. fun contract.setCodePostponed(newCode: cell): void asm "SETCODE" @@ -200,8 +322,8 @@ fun blockchain.currentBlockLogicalTime(): int fun blockchain.configParam(x: int): cell? asm "CONFIGOPTPARAM" -/// Commits the current state of registers `c4` (“persistent data”) and `c5` (“actions”) -/// so that the current execution is considered “successful” with the saved values even if an exception +/// Commits current state of registers `c4` (persistent data) and `c5` (actions) +/// so that the current execution is considered "successful" with the saved values even if an exception /// in Computation Phase is thrown later. fun commitContractDataAndActions(): void asm "COMMIT" @@ -220,7 +342,7 @@ struct PackOptions { /// set this option to true to disable runtime checks; /// note: `int32` and other are always validated for overflow without any extra gas, /// so this flag controls only rarely used `bitsN` type - skipBitsNValidation: bool = false, + skipBitsNValidation: bool = false } /// UnpackOptions allows you to control behavior of `MyStruct.fromCell(c)` and similar functions. @@ -231,16 +353,15 @@ struct UnpackOptions { /// note: setting this to false does not decrease gas (DROP from a stack and ENDS cost the same); /// note: this option controls [T.fromCell] and [T.fromSlice], but is ignored by [slice.loadAny]; /// note: `lazy` ignores this option, because it reads fields on demand or even skips them - assertEndAfterReading: bool = true, + assertEndAfterReading: bool = true /// this excNo is thrown if a prefix doesn't match, e.g. for `struct (0x01) A` given input "88..."; /// similarly, for a union type, this is thrown when none of the opcodes match; /// note: `lazy` ignores this option if you have `else` in `match` (you write custom logic there) - throwIfOpcodeDoesNotMatch: int = 63, + throwIfOpcodeDoesNotMatch: int = 63 } /// Convert anything to a cell (most likely, you'll call it for structures). -/// Example: /// ``` /// var st: MyStorage = { ... }; /// contract.setData(st.toCell()); @@ -252,7 +373,6 @@ fun T.toCell(self, options: PackOptions = {}): Cell builtin /// Parse anything from a cell (most likely, you'll call it for structures). -/// Example: /// ``` /// var st = MyStorage.fromCell(contract.getData()); /// ``` @@ -263,11 +383,10 @@ fun T.fromCell(packedCell: cell, options: UnpackOptions = {}): T builtin /// Parse anything from a slice (most likely, you'll call it for structures). -/// Example: /// ``` /// var msg = CounterIncrement.fromSlice(cs); /// ``` -/// All fields are read from a slice immediately. +/// All fields are read from a slice immediately unless `lazy`. /// If a slice is corrupted, an exception is thrown (most likely, excode 9 "cell underflow"). /// Note, that a passed slice is NOT mutated, its internal pointer is NOT shifted. /// If you need to mutate it, like `cs.loadInt()`, consider calling `cs.loadAny()`. @@ -277,12 +396,11 @@ fun T.fromSlice(rawSlice: slice, options: UnpackOptions = {}): T /// Parse anything from a slice, shifting its internal pointer. /// Similar to `slice.loadUint()` and others, but allows loading structures. -/// Example: /// ``` /// var st: MyStorage = cs.loadAny(); // or cs.loadAny() /// ``` /// Similar to `MyStorage.fromSlice(cs)`, but called as a slice method and mutates the slice. -/// Note: [options.assertEndAfterReading] is ignored by this function, because it's actually intended +/// Note: `options.assertEndAfterReading` is ignored by this function, because it's actually intended /// to read data from the middle. @pure fun slice.loadAny(mutate self, options: UnpackOptions = {}): T @@ -290,7 +408,6 @@ fun slice.loadAny(mutate self, options: UnpackOptions = {}): T /// Skip anything in a slice, shifting its internal pointer. /// Similar to `slice.skipBits()` and others, but allows skipping structures. -/// Example: /// ``` /// struct TwoInts { a: int32; b: int32; } /// cs.skipAny(); // skips 64 bits @@ -301,7 +418,6 @@ fun slice.skipAny(mutate self, options: UnpackOptions = {}): self /// Store anything to a builder. /// Similar to `builder.storeUint()` and others, but allows storing structures. -/// Example: /// ``` /// var b = beginCell().storeUint(32).storeAny(msgBody).endCell(); /// ``` @@ -324,7 +440,6 @@ fun T.getDeclaredPackPrefixLen(): int /// Forces an object created by `lazy` to load fully. Returns the remaining slice (having read all fields). /// Since `options.assertEndAfterReading` is ignored by `lazy` (fields are loaded on demand), /// this method can help you overcome this, if you really need to check input consistency. -/// Example: /// ``` /// val msg = lazy CounterMessage.fromSlice(s); /// match (msg) { // it's a lazy match, without creating a union on the stack @@ -341,29 +456,9 @@ fun T.getDeclaredPackPrefixLen(): int fun T.forceLoadLazyObject(self): slice builtin -/// Cell represents a typed cell reference (as opposed to untyped `cell`). -/// Example: -/// ``` -/// struct ExtraData { ... } -/// -/// struct MyStorage { -/// ... -/// extra: Cell; // TL-B `^ExtraData` -/// optional: Cell?; // TL-B `(Maybe ^ExtraData)` -/// code: cell; // TL-B `^Cell` -/// data: cell?; // TL-B `(Maybe ^Cell)` -/// } -/// ``` -/// Note, that `st = MyStorage.fromSlice(s)` does NOT deep-load any refs; `st.extra` is `Cell`, not `T`; -/// you should manually call `st.extra.load()` to get T (ExtraData in this example). -struct Cell { - tvmCell: cell -} - /// Parse data from already loaded cell reference. -/// Example: /// ``` -/// struct MyStorage { ... extra: Cell; } +/// struct MyStorage { ... extra: Cell } /// /// var st = MyStorage.fromCell(contract.getData()); /// // st.extra is cell; if we need to unpack it, we do @@ -383,8 +478,7 @@ fun Cell.beginParse(self): slice fun Cell.hash(self): uint256 asm "HASHCU" -/// RemainingBitsAndRefs is a special built-in type to get "all the rest" slice tail on reading. -/// Example: +/// `RemainingBitsAndRefs` is a special built-in type to get "all the rest" slice tail on reading. /// ``` /// struct JettonMessage { /// ... some fields @@ -395,13 +489,13 @@ fun Cell.hash(self): uint256 type RemainingBitsAndRefs = slice /// Creates a cell with zero bits and references. -/// Equivalent to `beginCell().endCell()` but cheaper. +/// Equivalent to `beginCell().endCell()`. @pure fun createEmptyCell(): cell asm " PUSHREF" /// Creates a slice with zero remaining bits and references. -/// Equivalent to `beginCell().endCell().beginParse()` but cheaper. +/// Equivalent to `beginCell().endCell().beginParse()`. @pure fun createEmptySlice(): slice asm "x{} PUSHSLICE" @@ -412,76 +506,84 @@ fun createEmptySlice(): slice */ /// Compile-time function that calculates crc32 of a constant string. -/// Example: `const op = stringCrc32("some_str")` = 4013618352 = 0xEF3AF4B0 -/// Note: stringCrc32(slice_var) does not work! It accepts a constant string and works at compile-time. +/// ``` +/// const op = stringCrc32("some_str") // = 4013618352 = 0xEF3AF4B0 +/// ``` +/// Note: `stringCrc32(slice_var)` does NOT work! It accepts a constant string and works at compile-time. @pure fun stringCrc32(constString: slice): int builtin /// Compile-time function that calculates crc16 (XMODEM) of a constant string. -/// Example: `const op = stringCrc16("some_str")` = 53407 = 0xD09F -/// Note: stringCrc16(slice_var) does not work! It accepts a constant string and works at compile-time. +/// ``` +/// const op = stringCrc16("some_str") // = 53407 = 0xD09F +/// ``` +/// Note: `stringCrc16(slice_var)` does NOT work! It accepts a constant string and works at compile-time. @pure fun stringCrc16(constString: slice): int builtin /// Compile-time function that calculates sha256 of a constant string and returns 256-bit integer. -/// Example: `const hash = stringSha256("some_crypto_key")` -/// Note: it's a compile-time function, `stringSha256(slice_var)` does not work. -/// Use `sliceBitsHash` or `sliceHash` (declared below) to hash a slice without/with its refs at runtime. +/// ``` +/// const hash = stringSha256("some_crypto_key") +/// ``` +/// Note: it's a compile-time function, `stringSha256(slice_var)` does NOT work! +/// Use [slice.bitsHash] or [slice.hash] (declared below) to hash a slice without/with its refs at runtime. @pure fun stringSha256(constString: slice): int builtin /// Compile-time function that calculates sha256 of a constant string and takes the first 32 bits. -/// Example: `const minihash = stringSha256_32("some_crypto_key")` -/// Note: stringSha256_32(slice_var) does not work! It accepts a constant string and works at compile-time. +/// ``` +/// const minihash = stringSha256_32("some_crypto_key") +/// ``` +/// Note: `stringSha256_32(slice_var)` does NOT work! It accepts a constant string and works at compile-time. @pure fun stringSha256_32(constString: slice): int builtin /// Compile-time function that takes N-chars ascii string and interprets it as a number in base 256. -/// Example: `const value = stringToBase256("AB")` = 16706 (65*256 + 66) -/// Note: stringToBase256(slice_var) does not work! It accepts a constant string and works at compile-time. +/// ``` +/// const value = stringToBase256("AB") // = 16706 (65*256 + 66) +/// ``` +/// Note: `stringToBase256(slice_var)` does NOT work! It accepts a constant string and works at compile-time. @pure fun stringToBase256(constString: slice): int builtin -/// Computes the representation hash of a `cell` and returns it as a 256-bit unsigned integer `x`. +/// Computes the representation hash of a cell and returns it as a 256-bit unsigned integer. /// Useful for signing and checking signatures of arbitrary entities represented by a tree of cells. @pure fun cell.hash(self): uint256 asm "HASHCU" -/// Computes the hash of a `slice` and returns it as a 256-bit unsigned integer `x`. -/// The result is the same as if an ordinary cell containing only data and references from `s` had been created -/// and its hash computed by [cell.hash]. +/// Computes the hash of a slice and returns it as a 256-bit unsigned integer. +/// The result is the same as [cell.hash] for a cell containing data and references from this slice. @pure fun slice.hash(self): uint256 asm "HASHSU" -/// Computes sha256 of the data bits of a `slice`. If the bit length of `s` is not divisible by eight, -/// throws a cell underflow exception. The hash value is returned as a 256-bit unsigned integer `x`. +/// Computes sha256 of the data bits of a slice (without refs). +/// If the bit length is not divisible by eight, throws a cell underflow exception. @pure fun slice.bitsHash(self): uint256 asm "SHA256U" /// Checks the Ed25519-`signature` of a `hash` (a 256-bit unsigned integer, usually computed as the hash of some data) -/// using [publicKey] (also represented by a 256-bit unsigned integer). +/// using `publicKey` (also represented by a 256-bit unsigned integer). /// The signature must contain at least 512 data bits; only the first 512 bits are used. -/// The result is `−1` if the signature is valid, `0` otherwise. /// Note that `CHKSIGNU` creates a 256-bit slice with the hash and calls `CHKSIGNS`. -/// That is, if [hash] is computed as the hash of some data, these data are hashed twice, +/// That is, if parameter `hash` is computed as the hash of some data, these data are hashed twice, /// the second hashing occurring inside `CHKSIGNS`. @pure fun isSignatureValid(hash: int, signature: slice, publicKey: int): bool asm "CHKSIGNU" -/// Checks whether [signature] is a valid Ed25519-signature of the data portion of `slice data` using `publicKey`, +/// Checks whether `signature` is a valid Ed25519-signature of the data portion of slice `data` using `publicKey`, /// similarly to [isSignatureValid]. -/// If the bit length of [data] is not divisible by eight, throws a cell underflow exception. +/// If the bit length of `data` is not divisible by eight, throws a cell underflow exception. /// The verification of Ed25519 signatures is the standard one, -/// with sha256 used to reduce [data] to the 256-bit number that is actually signed. +/// with sha256 used to reduce `data` to the 256-bit number that is actually signed. @pure fun isSliceSignatureValid(data: slice, signature: slice, publicKey: int): bool asm "CHKSIGNS" @@ -527,20 +629,20 @@ fun random.initialize(): void /// Returns `(x, y, z, -1)` or `(null, null, null, 0)`. /// Recursively computes the count of distinct cells `x`, data bits `y`, and cell references `z` -/// in the DAG rooted at `cell` [c], effectively returning the total storage used by this DAG taking into account +/// in a tree of cells, effectively returning the total storage used by this tree taking into account /// the identification of equal cells. -/// The values of `x`, `y`, and `z` are computed by a depth-first traversal of this DAG, +/// The values of `x`, `y`, and `z` are computed by a depth-first traversal of this tree, /// with a hash table of visited cell hashes used to prevent visits of already-visited cells. -/// The total count of visited cells `x` cannot exceed non-negative [maxCells]; +/// The total count of visited cells `x` cannot exceed non-negative `maxCells`; /// otherwise the computation is aborted before visiting the `(maxCells + 1)`-st cell and -/// a zero flag is returned to indicate failure. If [c] is `null`, returns `x = y = z = 0`. +/// a zero flag is returned to indicate failure. @pure fun cell.calculateSize(self, maxCells: int): (int, int, int, bool) asm "CDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT" -/// Similar to [cell.calculateSize], but accepting a `slice` [s] instead of a `cell`. -/// The returned value of `x` does not take into account the cell that contains the `slice` [s] itself; -/// however, the data bits and the cell references of [s] are accounted for in `y` and `z`. +/// Similar to [cell.calculateSize], but accepting a slice instead of a cell. +/// The returned value of `x` does not take into account the cell that contains the slice itself; +/// however, its data bits and cell references are accounted for in `y` and `z`. @pure fun slice.calculateSize(self, maxCells: int): (int, int, int, bool) asm "SDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT" @@ -549,35 +651,30 @@ fun slice.calculateSize(self, maxCells: int): (int, int, int, bool) fun cell.calculateSizeStrict(self, maxCells: int): (int, int, int) asm "CDATASIZE" -/// A non-quiet version of [cell.calculateSize] that throws a cell overflow exception (`8`) on failure. +/// A non-quiet version of [slice.calculateSize] that throws a cell overflow exception (`8`) on failure. fun slice.calculateSizeStrict(self, maxCells: int): (int, int, int) asm "SDATASIZE" -/// Returns the depth of a `cell`. -/// If [c] has no references, then return `0`; -/// otherwise the returned value is one plus the maximum of depths of cells referred to from [c]. -/// If [c] is a `null` instead of a cell, returns zero. +/// Returns the depth of a cell: 0 if no references, otherwise 1 + maximum of depths of all references. +/// When called for `null`, returns 0. @pure fun cell?.depth(self): int asm "CDEPTH" -/// Returns the depth of a `slice`. -/// If [s] has no references, then returns `0`; -/// otherwise the returned value is one plus the maximum of depths of cells referred to from [s]. +/// Returns the depth of a cell: 0 if no references, otherwise 1 + maximum of depths of all references. @pure fun slice.depth(self): int asm "SDEPTH" -/// Returns the depth of a `builder`. -/// If no cell references are stored in [b], then returns 0; -/// otherwise the returned value is one plus the maximum of depths of cells referred to from [b]. +/// Returns the depth of a builder: 0 if no references, otherwise 1 + maximum of depths of all references. @pure fun builder.depth(self): int asm "BDEPTH" -/// Returns the number of stack slots anyVariable occupies (works at compile-time). -/// Example: sizeof(nullableInt) = 1, because `int?` is 1 TVM slot holding either NULL or a value. -/// Example: sizeof(somePoint) = 2 for `struct Point { x:int, y: int }`: two fields one slot per each. +/// Returns the number of stack slots `anyVariable` occupies (works at compile-time). +/// - sizeof(nullableInt) = 1, because `int?` is 1 TVM slot holding either NULL or a value. +/// - sizeof(somePoint) = 2 for `struct Point { x:int, y:int }`: two fields one slot per each. +/// /// Useful for debugging or when preparing stack contents for RUNVM. @pure fun sizeof(anyVariable: T): int @@ -593,11 +690,11 @@ fun sizeof(anyVariable: T): int /// Example: `debug.print(v)` and other methods. struct debug -/// Dump a variable [x] to the debug log. +/// Dump a variable to the debug log. fun debug.print(x: T): void builtin -/// Dump a string [x] to the debug log. +/// Dump a string to the debug log. fun debug.printString(x: T): void builtin @@ -618,22 +715,7 @@ fun debug.dumpStack(): void But still, of course, it's a tree of cells, it's just a TVM dictionary. */ -/// `map` is a built-in high-level type. Internally, it's an optional cell. -/// - an empty map is represented as `null`. -/// - a non-empty map points to a root cell. -/// Note about some restrictions for K and V types: -/// - a key must be fixed-with; valid: int32, uint64, address, bits256, Point; invalid: int, coins -/// - a value must be serializable; valid: int32, coins, AnyStruct, Cell; invalid: int, builder -struct map { - tvmDict: cell? -} - -/// A low-level TVM dictionary is called "dict". -/// Think of it as "a map with unknown keys and unknown values". -type dict = cell? - /// Returns an empty typed map. It's essentially "PUSHNULL", since TVM NULL represents an empty map. -/// Example: /// ``` /// var m: map = createEmptyMap(); /// ``` @@ -643,7 +725,7 @@ fun createEmptyMap(): map /// Converts a low-level TVM dictionary to a typed map. /// Actually, does nothing: accepts an "optional cell" and returns the same "optional cell", -/// so if you specify key/value types incorrectly, it will fail later, at [map.get] and similar. +/// so if you specify key/value types incorrectly, it will fail later, at `map.get` and similar. @pure fun createMapFromLowLevelDict(d: dict): map builtin @@ -655,6 +737,7 @@ fun map.toLowLevelDict(self): dict asm "NOP" /// Checks whether a map is empty (whether a cell is null). +/// /// Note: a check `m == null` will not work, use `m.isEmpty()`. @pure fun map.isEmpty(self): bool @@ -666,7 +749,6 @@ fun map.exists(self, key: K): bool builtin /// Gets an element by key. If not found, does NOT throw, just returns isFound = false. -/// Example: /// ``` /// val r = m.get(123); /// if (r.isFound) { @@ -683,7 +765,6 @@ fun map.mustGet(self, key: K, throwIfNotFound: int = 9): V builtin /// Sets an element by key. -/// Example: /// ``` /// m.set(k, 3); /// ``` @@ -693,7 +774,6 @@ fun map.set(mutate self, key: K, value: V): self builtin /// Sets an element and returns the previous element at that key. If no previous, isFound = false. -/// Example: /// ``` /// val prev = m.setAndGetPrevious(k, 3); /// if (prev.isFound) { @@ -730,7 +810,6 @@ fun map.delete(mutate self, key: K): bool builtin /// Delete an element at the key and returns the deleted element. If not exists, isFound = false. -/// Example: /// ``` /// val prev = m.deleteAndGetDeleted(k); /// if (prev.isFound) { @@ -748,7 +827,7 @@ fun map.deleteAndGetDeleted(mutate self, key: K): MapLookupResult /// ``` /// var r = m.findFirst(); /// while (r.isFound) { -/// ... use r.getKey() and r.loadValue() +/// // ... use r.getKey() and r.loadValue() /// r = m.iterateNext(r) /// } /// ``` @@ -763,7 +842,7 @@ fun map.findFirst(self): MapEntry /// ``` /// var r = m.findLast(); /// while (r.isFound) { -/// ... use r.getKey() and r.loadValue() +/// // ... use r.getKey() and r.loadValue() /// r = m.iteratePrev(r) /// } /// ``` @@ -796,12 +875,11 @@ fun map.findKeyLessOrEqual(self, pivotKey: K): MapEntry builtin /// Iterate over a map in ascending order. -/// Example: /// ``` /// // iterate for all keys >= 10 up to the end /// var r = m.findKeyGreaterOrEqual(10); /// while (r.isFound) { -/// ... use r.getKey() and r.loadValue() +/// // ... use r.getKey() and r.loadValue() /// r = m.iterateNext(r) /// } /// ``` @@ -810,22 +888,21 @@ fun map.iterateNext(self, current: MapEntry): MapEntry builtin /// Iterate over a map in reverse order. -/// Example: /// ``` /// // iterate for all keys < 10 down lo lowest /// var r = m.findKeyLess(10); /// while (r.isFound) { -/// ... use r.getKey() and r.loadValue() +/// // ... use r.getKey() and r.loadValue() /// r = m.iteratePrev(r) /// } -/// ```@pure +/// ``` +@pure fun map.iteratePrev(self, current: MapEntry): MapEntry builtin -/// MapLookupResult is a return value of [map.get], [map.setAndGetPrevious], and similar. +/// `MapLookupResult` is a return value of `map.get`, `map.setAndGetPrevious`, and similar. /// Instead of returning a nullable value (that you'd check on null before usage), -/// this struct is returned (and you check on `isFound` before usage). -/// Example: +/// this struct is returned (and you check `isFound` before usage). /// ``` /// val r = m.get(key); /// if (r.isFound) { @@ -851,13 +928,12 @@ fun MapLookupResult.loadValue(self): slice { return self.rawSlice! } -/// MapEntry is a return value of [map.findFirst], [map.iterateNext], and similar. +/// `MapEntry` is a return value of `map.findFirst`, `map.iterateNext`, and similar. /// You should check for `isFound` before calling `getKey()` and `loadValue()`. -/// Example: /// ``` /// var r = m.findFirst(); /// while (r.isFound) { -/// ... use r.getKey() and r.loadValue() +/// // ... use r.getKey() and r.loadValue() /// r = m.iterateNext(r) /// } /// ``` @@ -896,74 +972,74 @@ fun MapEntry.loadValue(self): slice { */ /// Compile-time function that converts a constant hex-encoded string to N/2 bytes. -/// Example: `const v = stringHexToSlice("abcdef")` = slice with 3 bytes `[ 0xAB, 0xCD, 0xEF ]` -/// Note: stringHexToSlice(slice_var) does not work! It accepts a constant string and works at compile-time. +/// ``` +/// const v = stringHexToSlice("abcdef") // = slice with 3 bytes: 0xAB,0xCD,0xEF +/// ``` +/// Note: `stringHexToSlice(slice_var)` does NOT work! It accepts a constant string and works at compile-time. @pure fun stringHexToSlice(constStringBytesHex: slice): slice builtin -/// Converts a `cell` into a `slice`. Notice that [c] must be either an ordinary cell, -/// or an exotic cell (see [TVM.pdf](https://ton-blockchain.github.io/docs/tvm.pdf), 3.1.2) -/// which is automatically loaded to yield an ordinary cell `c'`, converted into a `slice` afterwards. +/// Converts a cell into a slice. @pure fun cell.beginParse(self): slice asm "CTOS" -/// Checks if slice is empty. If not, throws an exception with code 9. +/// Checks if a slice is empty. If not, throws an exception with code 9. fun slice.assertEnd(self): void asm "ENDS" -/// Loads the next reference from the slice. +/// Loads the next reference from a slice. @pure fun slice.loadRef(mutate self): cell asm( -> 1 0) "LDREF" -/// Preloads the next reference from the slice. +/// Preloads the next reference from a slice. @pure fun slice.preloadRef(self): cell asm "PLDREF" -/// Loads a signed [len]-bit integer from a slice. +/// Loads a signed len-bit integer from a slice. @pure fun slice.loadInt(mutate self, len: int): int builtin -/// Loads an unsigned [len]-bit integer from a slice. +/// Loads an unsigned len-bit integer from a slice. @pure fun slice.loadUint(mutate self, len: int): int builtin -/// Loads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate slice `s''`. +/// Loads the first `0 ≤ len ≤ 1023` bits from a slice. @pure fun slice.loadBits(mutate self, len: int): slice builtin -/// Preloads a signed [len]-bit integer from a slice. +/// Preloads a signed len-bit integer from a slice. @pure fun slice.preloadInt(self, len: int): int builtin -/// Preloads an unsigned [len]-bit integer from a slice. +/// Preloads an unsigned len-bit integer from a slice. @pure fun slice.preloadUint(self, len: int): int builtin -/// Preloads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate slice. +/// Preloads the first `0 ≤ len ≤ 1023` bits from a slice. @pure fun slice.preloadBits(self, len: int): slice builtin -/// Loads serialized amount of Toncoins (any unsigned integer up to `2^120 - 1`). +/// Loads the serialized amount of nanotoncoins (any unsigned integer up to `2^120 - 1`). @pure fun slice.loadCoins(mutate self): coins asm( -> 1 0) "LDGRAMS" -/// Loads bool (-1 or 0) from a slice +/// Loads a boolean from a slice. @pure fun slice.loadBool(mutate self): bool asm( -> 1 0) "1 LDI" -/// Shifts a slice pointer to [len] bits forward, mutating the slice. +/// Shifts a slice pointer to len bits forward. @pure fun slice.skipBits(mutate self, len: int): self builtin @@ -999,7 +1075,7 @@ fun slice.loadDict(mutate self): dict fun slice.preloadDict(self): dict asm "PLDDICT" -/// Loads a dictionary as [slice.loadDict], but returns only the remainder of the slice. +/// Skip a dictionary from a slice. @pure fun slice.skipDict(mutate self): self asm "SKIPDICT" @@ -1015,11 +1091,12 @@ fun slice.loadMaybeRef(mutate self): cell? fun slice.preloadMaybeRef(self): cell? asm "PLDOPTREF" -/// Loads (Maybe ^Cell), but returns only the remainder of the slice. +/// Skips (Maybe ^Cell) in a slice. @pure fun slice.skipMaybeRef(mutate self): self asm "SKIPOPTREF" + /** Builder primitives: constructing cells. When you _store_ some data, you mutate the builder (shifting an internal pointer on the stack). @@ -1032,7 +1109,7 @@ fun slice.skipMaybeRef(mutate self): self fun beginCell(): builder asm "NEWC" -/// Converts a builder into an ordinary `cell`. +/// Converts a builder into an ordinary cell. @pure fun builder.endCell(self): cell asm "ENDC" @@ -1042,12 +1119,12 @@ fun builder.endCell(self): cell fun builder.storeRef(mutate self, c: cell): self asm(c self) "STREF" -/// Stores a signed [len]-bit integer into a builder (`0 ≤ len ≤ 257`). +/// Stores a signed len-bit integer into a builder (`0 ≤ len ≤ 257`). @pure fun builder.storeInt(mutate self, x: int, len: int): self builtin -/// Stores an unsigned [len]-bit integer into a builder (`0 ≤ len ≤ 256`). +/// Stores an unsigned len-bit integer into a builder (`0 ≤ len ≤ 256`). @pure fun builder.storeUint(mutate self, x: int, len: int): self builtin @@ -1062,25 +1139,24 @@ fun builder.storeSlice(mutate self, s: slice): self fun builder.storeAddress(mutate self, addr: address): self asm(addr self) "STSLICE" -/// Stores amount of Toncoins into a builder. +/// Stores the amount of nanotoncoins into a builder. @pure fun builder.storeCoins(mutate self, x: coins): self builtin -/// Stores bool (-1 or 0) into a builder. -/// Attention: true value is `-1`, not 1! If you pass `1` here, TVM will throw an exception. +/// Stores a boolean into a builder. @pure fun builder.storeBool(mutate self, x: bool): self builtin -/// Stores dictionary (represented by TVM `cell` or `null`) into a builder. -/// In other words, stores a `1`-bit and a reference to [c] if [c] is not `null` and `0`-bit otherwise. +/// Stores a low-level TVM dictionary (optional cell) into a builder. +/// In other words, if `c` is null, stores `0`; if `c` is not null, stores `1` and a reference to `c`. @pure fun builder.storeDict(mutate self, c: dict): self asm(c self) "STDICT" /// Stores (Maybe ^Cell) into a builder. -/// In other words, if cell is `null`, store '0' bit; otherwise, store '1' and a ref to [c]. +/// In other words, if `c` is null, stores `0`; if `c` is not null, stores `1` and a reference to `c`. @pure fun builder.storeMaybeRef(mutate self, c: cell?): self asm(c self) "STOPTREF" @@ -1090,7 +1166,7 @@ fun builder.storeMaybeRef(mutate self, c: cell?): self fun builder.storeBuilder(mutate self, from: builder): self asm(from self) "STB" -/// Stores a slice representing TL addr_none$00 (two `0` bits). +/// Stores a slice representing "none address" (TL addr_none$00 — two zero bits). @pure fun builder.storeAddressNone(mutate self): self asm "b{00} STSLICECONST" @@ -1151,8 +1227,10 @@ fun builder.bitsCount(self): int */ /// Compile-time function that parses a valid contract address. -/// Example: address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5") -/// Example: address("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8") +/// ``` +/// address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5") +/// address("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8") +/// ``` /// Returns `address`, which can be stored in a builder, compared with `==`, etc. @pure fun address(stdAddress: slice): address @@ -1163,9 +1241,8 @@ fun address(stdAddress: slice): address fun createAddressNone(): address asm "b{00} PUSHSLICE" -/// Returns if it's an empty address. -/// Don't confuse it with null! Empty address is a slice with two `0` bits. -/// In TL/B, it's addr_none$00. +/// Returns if it's an empty address (= "none address"). +/// Don't confuse it with null! An empty address is a slice "00" (two zero bits). @pure fun address.isNone(self): bool asm "b{00} SDBEGINSQ" "NIP" @@ -1204,7 +1281,7 @@ fun address.getWorkchain(self): int8 fun address.bitsEqual(self, b: address): bool asm "SDEQ" -/// Loads from slice [s] a valid `MsgAddress` (none/internal/external). +/// Loads a valid address (internal/external/none). In TL/B, it's `MsgAddress`. @pure fun slice.loadAddress(mutate self): address asm( -> 1 0) "LDMSGADDR" @@ -1228,7 +1305,7 @@ const RESERVE_MODE_NEGATE_AMOUNT = 8 const RESERVE_MODE_BOUNCE_ON_ACTION_FAIL = 16 /// Creates an output action which would reserve Toncoins on balance. -/// For [reserveMode] consider constants above. +/// For `reserveMode` consider constants above. fun reserveToncoinsOnBalance(nanoTonCoins: coins, reserveMode: int): void asm "RAWRESERVE" @@ -1253,7 +1330,7 @@ const SEND_MODE_REGULAR = 0 const SEND_MODE_PAY_FEES_SEPARATELY = 1 /// +2 means that any errors arising while processing this message during the action phase should be ignored. const SEND_MODE_IGNORE_ERRORS = 2 -/// in the case of action fail - bounce transaction. No effect if SEND_MODE_IGNORE_ERRORS (+2) is used. TVM UPGRADE 2023-07. https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages +/// in the case of action fail - bounce transaction. No effect if SEND_MODE_IGNORE_ERRORS (+2) is used. [TVM UPGRADE 2023-07](https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages) const SEND_MODE_BOUNCE_ON_ACTION_FAIL = 16 /// mode = 32 means that the current account must be destroyed if its resulting balance is zero. const SEND_MODE_DESTROY = 32 @@ -1261,7 +1338,7 @@ const SEND_MODE_DESTROY = 32 const SEND_MODE_CARRY_ALL_REMAINING_MESSAGE_VALUE = 64 /// mode = 128 is used for messages that are to carry all the remaining balance of the current smart contract (instead of the value originally indicated in the message). const SEND_MODE_CARRY_ALL_BALANCE = 128 -/// do not create an action, only estimate fee. TVM UPGRADE 2023-07. https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages +/// do not create an action, only estimate fee. [TVM UPGRADE 2023-07](https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages) const SEND_MODE_ESTIMATE_FEE_ONLY = 1024 /// Other modes affect the fee calculation as follows: /// +64 substitutes the entire balance of the incoming message as an outcoming value (slightly inaccurate, gas expenses that cannot be estimated before the computation is completed are not taken into account). @@ -1272,23 +1349,22 @@ const SEND_MODE_ESTIMATE_FEE_ONLY = 1024 /// It's a dictionary `[currencyID => amount]` (amount is like `coins`, but encoded with a high precision). type ExtraCurrenciesMap = map -/// ContractState is "code + data" of a contract. +/// `ContractState` is "code + data" of a contract. /// Used in outgoing messages (StateInit) to initialize a destination contract. struct ContractState { code: cell data: cell } -/// AddressShardingOptions provides settings to calculate an address in another shard. +/// `AddressShardingOptions` provides settings to calculate an address in another shard. /// Consider [createMessage] and [address.buildSameAddressInAnotherShard] for usage. struct AddressShardingOptions { fixedPrefixLength: uint5 // shard depth, formerly splitDepth closeTo: address } -/// AutoDeployAddress is a destination that initializes a receiver contract if it does not exist yet. +/// `AutoDeployAddress` is a destination that initializes a receiver contract if it does not exist yet. /// In order to do this, it contains StateInit — and calculates an address by a hash of StateInit. -/// Example: /// ``` /// createMessage({ /// dest: { @@ -1302,7 +1378,8 @@ struct AddressShardingOptions { /// ``` /// You just provide code+data, and the compiler automatically calculates the destination address, /// because in TON, the address of a contract, by definition, is a hash of its initial state. -/// You can also use it without sending a message. See [buildAddress] and [addressMatches]. +/// You can also use it without sending a message. +/// See [AutoDeployAddress.buildAddress] and [AutoDeployAddress.addressMatches]. struct AutoDeployAddress { workchain: int8 = BASECHAIN stateInit: ContractState | cell @@ -1351,16 +1428,16 @@ struct CreateMessageOptions { /// Creates a message (`OutMessage`) — a well-formatted message cell. /// Typically, you just send it. In advanced scenarios, you can estimate fees or even postpone sending. -/// Example: /// ``` /// val reply = createMessage({ /// bounce: false, /// value: ton("0.05"), /// dest: senderAddress, -/// body: RequestedInfo { ... } // note: no `toCell`! just pass an object +/// body: RequestedInfo { ... } // note: no toCell! just pass an object /// }); /// reply.send(SEND_MODE_REGULAR); /// ``` +/// /// Hint: don't call `body.toCell()`, pass `body` directly! /// (if body is small, it will be inlined without an expensive cell creation) /// (if body is large, the compiler will automatically wrap it into a cell) @@ -1370,14 +1447,14 @@ fun createMessage(options: CreateMessageOptions): OutMessage builtin -/// ExtOutLogBucket is a variant of a custom external address for emitting logs "to the outer world". +/// `ExtOutLogBucket` is a variant of a custom external address for emitting logs "to the outer world". /// It includes some "topic" (arbitrary number), that determines the format of the message body. /// For example, you emit "deposit event" (reserving topic "deposit" = 123): -/// > `dest: ExtOutLogBucket { topic: 123 }, body: DepositData { ... }` +/// ``` +/// dest: ExtOutLogBucket { topic: 123 }, +/// body: DepositData { ... } +/// ``` /// and external indexers can index your emitted logs by destination address without parsing body. -/// Currently, external messages are used only for emitting logs (for viewing them in indexers). -/// In the future, there might be -/// > 0x01 ExtOutOffchainContract { adnl: uint256 | bits256 } /// Serialization details: '01' (addr_extern) + 256 (len) + 0x00 (prefix) + 248 bits = 267 in total struct (0x00) ExtOutLogBucket { topic: uint248 | bits248 @@ -1396,7 +1473,6 @@ struct CreateExternalLogMessageOptions { /// Creates an external message (`OutMessage`) — a well-formatted message cell. /// Typically, you just send it. In advanced scenarios, you can estimate fees or even postpone sending. -/// Example: /// ``` /// val emitMsg = createExternalLogMessage({ /// dest: createAddressNone(), @@ -1412,18 +1488,17 @@ struct CreateExternalLogMessageOptions { fun createExternalLogMessage(options: CreateExternalLogMessageOptions): OutMessage builtin -/// UnsafeBodyNoRef is used to prevent default behavior: when message body is potentially large, +/// `UnsafeBodyNoRef` is used to prevent default behavior: when message body is potentially large, /// it's packed into a separate ref. Wrapping body with this struct tells the compiler: /// "inline message body in-place, I guarantee it will fit". -/// Example: /// ``` -/// struct ProbablyLarge { a: (coins, coins, coins, coins, coins) } // max 620 bits +/// struct ProbablyLarge { a: (coins, coins, coins, coins, coins) } /// /// val contents: ProbablyLarge = { ... }; // you are sure: values are small /// createMessage({ -/// // body: contents, // by default, it will be stored a ref (it's large) +/// // body: contents, // by default, will be a ref (max 620 bits) /// body: UnsafeBodyNoRef { -/// forceInline: contents, // but wrapping forces the compiler to inline it +/// forceInline: contents, // but it forces the compiler to inline it /// } /// ``` /// Another example: your body contains `builder` or `RemainingBitsAndRefs`: unpredictable size. @@ -1455,7 +1530,7 @@ fun OutMessage.hash(self): uint256 asm "HASHCU" -/// StateInit is a "canonical TL/B representation from block.tlb" of a contract initial state. +/// `StateInit` is a "canonical TL/B representation from block.tlb" of a contract initial state. /// But for everyday tasks, it's too complicated. It is not used in practice. /// To represent code+data, consider [ContractState]. /// To represent sharding (fixedPrefixLength, formerly splitDepth), consider [AutoDeployAddress]. @@ -1468,7 +1543,6 @@ struct StateInit { } /// Calculates a hash of StateInit if only code+data are set. -/// Example: /// ``` /// val addrHash = StateInit.calcHashCodeData(codeCell, dataCell); /// createMessage({ @@ -1528,12 +1602,16 @@ fun StateInit.calcHashPrefixCodeData(fixedPrefixLength: uint5, code: cell, data: /// Given an internal address A="aaaa...a" returns "bbaa...a" (D bits from address B, 256-D from A). +/// /// Example for fixedPrefixLength (shard depth) = 8: -/// | self (A) | aaaaaaaaaaa...aaa | -/// | closeTo (B) | 01010101bbb...bbb | shardPrefix = 01010101 (depth 8) -/// | result | 01010101aaa...aaa | address of A in same shard as B +/// ``` +/// self (A) | aaaaaaaaaaa...aaa | +/// closeTo (B) | 01010101bbb...bbb | shardPrefix = 01010101 (depth 8) +/// result | 01010101aaa...aaa | address of A in same shard as B +/// ``` /// More precisely, self (input) is 267 bits: '100' (std addr no anycast) + workchainA + "aaaa...a". /// The result is also 267 bits: '100' + workchainB + "bb" (D bits) + "aa...a" (256-D bits). +/// /// Note: returns `builder`, not `address`! Because builder is cheap, and you can send a message to it: /// ``` /// createMessage({ @@ -1577,10 +1655,9 @@ fun sendRawMessage(msg: cell, mode: int): void Receiving and handling messages. */ -/// InMessage is an input for `onInternalMessage` — when your contract accepts a non-bounced message. +/// `InMessage` is an input for `onInternalMessage` — when your contract accepts a non-bounced message. /// Internally, some data exist on the stack, some can be acquired with TVM instructions. /// The compiler replaces `in.someField` and gets a value correctly. -/// Example: /// ``` /// fun onInternalMessage(in: InMessage) { /// in.senderAddress // actually calls `INMSG_SRC` asm instruction @@ -1596,11 +1673,10 @@ struct InMessage { body: slice // message body, parse it with `lazy AllowedMsg.fromSlice(in.body)` } -/// InMessageBounced is an input for `onBouncedMessage`. +/// `InMessageBounced` is an input for `onBouncedMessage`. /// Very similar to a non-bounced input [InMessage]. /// Note, that `bouncedBody` is currently 256 bits: 0xFFFFFFFF + 224 bits from the originally sent message. /// Parse it with care! -/// Example: /// ``` /// fun onBouncedMessage(in: InMessageBounced) { /// in.bouncedBody.skipBouncedPrefix(); diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 916cefc6c1..55c74006e1 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -32,16 +32,16 @@ fun setGasLimitToMaximum(): void fun setGasLimit(limit: int): void asm "SETGASLIMIT" -/// Calculates fee (amount in nanotoncoins to be paid) for a transaction which consumed [gasUsed] gas units. +/// Calculates fee (amount in nanotoncoins to be paid) for a transaction which consumed `gasUsed` gas units. fun calculateGasFee(workchain: int8, gasUsed: int): coins asm(gasUsed workchain) "GETGASFEE" -/// Same as [calculateGasFee], but without flat price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) +/// Same as [calculateGasFee], but without flat price. [Docs about fees](https://docs.ton.org/develop/howto/fees-low-level) fun calculateGasFeeWithoutFlatPrice(workchain: int8, gasUsed: coins): coins asm(gasUsed workchain) "GETGASFEESIMPLE" -/// Calculates amount of nanotoncoins you should pay for storing a contract of provided size for [seconds]. -/// [bits] and [cells] represent contract state (code + data). +/// Calculates amount of nanotoncoins you should pay for storing a contract of provided size for `seconds`. +/// `bits` and `cells` represent contract state (code + data). fun calculateStorageFee(workchain: int8, seconds: int, bits: int, cells: int): coins asm(cells bits seconds workchain) "GETSTORAGEFEE" @@ -49,7 +49,7 @@ fun calculateStorageFee(workchain: int8, seconds: int, bits: int, cells: int): c fun calculateForwardFee(workchain: int8, bits: int, cells: int): coins asm(cells bits workchain) "GETFORWARDFEE" -/// Same as [calculateForwardFee], but without lump price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) +/// Same as [calculateForwardFee], but without lump price. [Docs about fees](https://docs.ton.org/develop/howto/fees-low-level) fun calculateForwardFeeWithoutLumpPrice(workchain: int8, bits: int, cells: int): coins asm(cells bits workchain) "GETFORWARDFEESIMPLE" diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index 289c58e66b..504186f393 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -17,10 +17,12 @@ tolk 1.0 */ /// In @stdlib/common.tolk, there is a type alias: -/// `type dict = cell?` +/// ``` +/// type dict = cell? +/// ``` /// For clarity, we use "dict" instead of a "cell?" where a cell-dictionary is assumed. -/// Creates an empty dictionary, which is actually a null value. Equivalent to PUSHNULL +/// Creates an empty dictionary, which is actually a null value. Equivalent to `PUSHNULL` @pure fun createEmptyDict(): dict asm "NEWDICT" @@ -319,7 +321,7 @@ fun dict.sDictGetPrevOrEqual(self, keyLen: int, pivot: slice): (slice?, slice?, /** - Prefix dictionary primitives. + Prefix dictionary primitives. */ @pure diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index d578199c7d..de97654d8b 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -14,12 +14,12 @@ fun getTvmRegisterC3(): continuation fun setTvmRegisterC3(c: continuation): void asm "c3 POP" -/// Transforms a `slice` [s] into a simple ordinary continuation `c`, with `c.code = s` and an empty stack and savelist. +/// Transforms a slice into a simple ordinary continuation, with `c.code = s` and an empty stack and savelist. @pure fun transformSliceToContinuation(s: slice): continuation asm "BLESS" -/// Moves a variable or a value [x] to the top of the stack. +/// Moves a variable to the top of the stack. @pure fun T.stackMoveToTop(mutate self): void asm "NOP" diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index b9c89bdcae..35fd441716 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -425,6 +425,11 @@ static AnyV parse_type_alias_declaration(Lexer& lex, const std::vector = builtin + lex.next(); + return createV(loc); + } + AnyTypeV underlying_type = parse_type_from_tokens(lex); for (auto v_annotation : annotations) { From 6e9ce41baaebad1b1d441372a0a302125d442a21 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 18 Aug 2025 11:16:35 +0700 Subject: [PATCH 395/524] [Tolk] Fix overflow detection on constant storeUint --- tolk-tester/tests/cells-slices.tolk | 71 ++++++++++++++++++++++++++++ tolk-tester/tests/pack-unpack-1.tolk | 15 ++++++ tolk/builtins.cpp | 8 ++-- 3 files changed, 89 insertions(+), 5 deletions(-) diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 17fd0c12bc..e1c2ac9c39 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -16,6 +16,16 @@ fun endCell(b: builder): cell fun beginParse(c: cell): slice asm "CTOS"; +@noinline +fun triggerOverflowIntConst() { + return beginCell().storeInt(10, 4) +} + +@noinline +fun triggerOverflowUintConst() { + return beginCell().storeUint(123, 6) +} + @method_id(101) fun test1(): [int,int,int,int,int] { var b: builder = beginCell().storeUint(1, 32); @@ -435,6 +445,30 @@ fun test34(p: int, n: int) { return b; } +@method_id(135) +fun test35(overflowMode: int) { + try { + val b: builder = match (overflowMode) { + 1 => triggerOverflowIntConst(), + 2 => triggerOverflowUintConst(), + 3 => beginCell().storeUint(10, -4), + 4 => beginCell().storeInt(100, 1), + 5 => beginCell().storeInt(1, 0), + 6 => beginCell().storeInt(115792089237316195423570985008687907853269984665640564039457584007913129639935, 256), + 7 => beginCell().storeInt(15, 4), + 8 => beginCell().storeUint(1<<170, 169), + else => beginCell(), + }; + return b.endCell().beginParse().loadUint(1) * 1000 + } + catch (ex) { return ex } +} + +@method_id(136) +fun test36() { + return beginCell().storeInt(0, 0) +} + fun main(): int { return 0; } @@ -469,6 +503,15 @@ fun main(): int { @testcase | 132 | 0 | BC{00080000000a} @testcase | 133 | 0 | BC{00020a} @testcase | 134 | 0 8 | BC{00020a} +@testcase | 135 | 1 | 5 +@testcase | 135 | 2 | 5 +@testcase | 135 | 3 | 5 +@testcase | 135 | 4 | 5 +@testcase | 135 | 5 | 5 +@testcase | 135 | 6 | 5 +@testcase | 135 | 7 | 5 +@testcase | 135 | 8 | 5 +@testcase | 136 | | BC{0000} We test that consequtive storeInt/storeUint with constants are joined into a single number @@ -630,4 +673,32 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin }> """ +@fif_codegen +""" + triggerOverflowIntConst() PROC:<{ + 10 PUSHINT + NEWC + 4 STI + }> +""" + +@fif_codegen +""" + triggerOverflowUintConst() PROC:<{ + 123 PUSHINT + NEWC + 6 STU + }> +""" + +@fif_codegen +""" + test36() PROC:<{ + 0 PUSHINT + NEWC + OVER + STIX + }> +""" + */ diff --git a/tolk-tester/tests/pack-unpack-1.tolk b/tolk-tester/tests/pack-unpack-1.tolk index 9c9acf8219..3cc2060387 100644 --- a/tolk-tester/tests/pack-unpack-1.tolk +++ b/tolk-tester/tests/pack-unpack-1.tolk @@ -7,6 +7,11 @@ struct Point { y: int32; } +struct TwoU { + a: uint8 + b: uint8 +} + @method_id(101) fun test1(value: int) { var t: JustInt32 = { value }; @@ -84,6 +89,15 @@ fun test7(s: slice) { return (6, s.remainingBitsCount()); } +@method_id(108) +fun test8(): cell | int { + try { + return TwoU{a:1<<10, b:0}.toCell() + } catch (ex) { + return ex + } +} + fun main(c: cell) { c as Cell; (c as Cell) as cell; @@ -103,6 +117,7 @@ fun main(c: cell) { @testcase | 107 | x{09332} | 4 12 @testcase | 107 | x{2} | 5 1 @testcase | 107 | x{0234} | 6 16 +@testcase | 108 | | 5 1 @fif_codegen """ diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index a71ee7c88b..2149bc64d4 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1093,12 +1093,10 @@ static AsmOp compile_store_int(std::vector& res, std::vector // purpose: to merge consecutive `b.storeUint(0, 1).storeUint(1, 1)` into one "1 PUSHINT + 2 STU", // when constant arguments are passed, keep them as a separate (fake) instruction, to be handled by optimizer later bool value_and_len_is_const = z.is_int_const() && x.is_int_const(); - if (value_and_len_is_const && G.settings.optimization_level >= 2) { + if (value_and_len_is_const && x.int_const >= 0 && z.int_const > 0 && z.int_const <= 256 && G.settings.optimization_level >= 2) { // don't handle negative numbers or potential overflow, merging them is incorrect - bool value_is_safe = sgnd - ? x.int_const >= 0 && z.int_const < 64 && x.int_const < (1ULL << (z.int_const->to_long() - 1)) - : x.int_const >= 0; - if (value_is_safe && z.int_const > 0 && z.int_const <= (255 + !sgnd)) { + int len = static_cast(z.int_const->to_long()); + if (x.int_const->fits_bits(len, sgnd)) { z.unused(); x.unused(); return AsmOp::Custom(loc, "MY_store_int"s + (sgnd ? "I " : "U ") + x.int_const->to_dec_string() + " " + z.int_const->to_dec_string(), 1); From 1ebdc0a5c8e267771640eb02fb3061902d64b6e2 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 14 Aug 2025 01:01:52 +0700 Subject: [PATCH 396/524] [Tolk] Support numeric enums similar to TypeScript and C++ --- tolk-tester/tests/enums-tests.tolk | 307 ++++++++++++++++++ tolk-tester/tests/generics-1.tolk | 7 +- .../tests/invalid-declaration/err-1039.tolk | 7 + .../tests/invalid-declaration/err-1697.tolk | 7 + .../tests/invalid-declaration/err-1700.tolk | 11 + .../tests/invalid-declaration/err-1839.tolk | 8 + .../tests/invalid-semantics/err-4395.tolk | 10 + .../tests/invalid-semantics/err-4518.tolk | 13 + .../tests/invalid-semantics/err-4810.tolk | 10 + .../tests/invalid-symbol/err-2190.tolk | 14 + .../tests/invalid-symbol/err-2618.tolk | 9 + .../tests/invalid-symbol/err-2910.tolk | 10 + .../tests/invalid-syntax/err-3410.tolk | 11 + .../tests/invalid-syntax/err-3781.tolk | 6 - .../tests/invalid-typing/err-6120.tolk | 16 + .../tests/invalid-typing/err-6121.tolk | 17 + .../tests/invalid-typing/err-6122.tolk | 17 + .../tests/invalid-typing/err-6310.tolk | 12 + .../tests/invalid-typing/err-6311.tolk | 12 + .../tests/invalid-typing/err-6312.tolk | 14 + .../tests/invalid-typing/err-6511.tolk | 10 + .../tests/invalid-typing/err-6512.tolk | 10 + tolk-tester/tests/lazy-load-tests.tolk | 9 + tolk/ast-from-tokens.cpp | 57 +++- tolk/ast-replacer.h | 1 + tolk/ast-stringifier.h | 8 + tolk/ast-visitor.h | 1 + tolk/ast.cpp | 4 + tolk/ast.h | 50 ++- tolk/compiler-state.cpp | 4 + tolk/compiler-state.h | 1 + tolk/constant-evaluator.cpp | 86 ++++- tolk/constant-evaluator.h | 1 + tolk/fwd-declarations.h | 4 + tolk/pack-unpack-api.cpp | 3 + tolk/pipe-ast-to-legacy.cpp | 48 ++- tolk/pipe-check-inferred-types.cpp | 107 ++++-- tolk/pipe-check-rvalue-lvalue.cpp | 6 + tolk/pipe-constant-folding.cpp | 19 ++ tolk/pipe-discover-parse-sources.cpp | 2 +- tolk/pipe-infer-types-and-calls.cpp | 76 +++-- tolk/pipe-register-symbols.cpp | 36 +- tolk/pipe-resolve-identifiers.cpp | 17 +- tolk/pipe-resolve-types.cpp | 17 + tolk/symtable.cpp | 67 ++-- tolk/symtable.h | 38 ++- tolk/type-system.cpp | 51 +++ tolk/type-system.h | 21 ++ 48 files changed, 1149 insertions(+), 123 deletions(-) create mode 100644 tolk-tester/tests/enums-tests.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1039.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1697.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1700.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1839.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4395.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4518.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4810.tolk create mode 100644 tolk-tester/tests/invalid-symbol/err-2190.tolk create mode 100644 tolk-tester/tests/invalid-symbol/err-2618.tolk create mode 100644 tolk-tester/tests/invalid-symbol/err-2910.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3410.tolk delete mode 100644 tolk-tester/tests/invalid-syntax/err-3781.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6120.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6121.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6122.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6310.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6311.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6312.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6511.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6512.tolk diff --git a/tolk-tester/tests/enums-tests.tolk b/tolk-tester/tests/enums-tests.tolk new file mode 100644 index 0000000000..c61faada2b --- /dev/null +++ b/tolk-tester/tests/enums-tests.tolk @@ -0,0 +1,307 @@ +const SIX = 6 + +enum EEmpty {} + +enum Color { + Red, + Green, + Blue +} + +type ColorAlias = Color + +struct WithColor { + c1: Color + c2: Color = Color.Green +} + +enum EPartialAssigned { + M100 = -100, + M99, + P1 = (1 + 0) << 0; + P2 = ((+1 + 1) << 1) / 2; + P3 + P4 + P5 = 0b101, + P6 = SIX; + Max = 0x7FFFFFFF, +} + +enum Err { + TooLowId = 0x100, + TooHighId, + DisabledId, +} + +struct Box { + value: T +} + +fun checkIsBlue(c: Color) { + return c == Color.Blue +} + +fun Color.isBlue(self) { + return self == Color.Blue +} + +fun Color.createBlue() { + return Color.Blue +} + +const C_BLUE = Color.Blue +const C_TEN = 5 * (Color.Green as int) * 2 + +// these methods do not bother Color.Red and other +fun T.Red() { return 123 } +fun T.Max() { return -C_TEN } +fun int.Max(self) { return self } +fun Color.Max(): EPartialAssigned { return ((EPartialAssigned.Max as int) + 1) as EPartialAssigned } + +@noinline +fun checkIsColor(obj: Color | T) { return obj is Color } + +@inline_ref +fun Color.mapToNumber(self) { + // match over enum is exhaustive, it's like "if Red {} else if Green {} else {}" + return match (self) { + Color.Red => 100, + Color.Green => 200, + Color.Blue => 300, // this works like `else` and handles invalid values at runtime also + } +} + + +@method_id(101) +fun test1() { + var c = Color.Red; + + return ( + checkIsBlue(Color.Blue), checkIsBlue(c), checkIsBlue(Color.createBlue()), + Color.Blue.isBlue(), Color.Red.isBlue(), Color.createBlue()!!.isBlue(), C_BLUE.isBlue(), + Color.createBlue() + ); +} + +@method_id(102) +fun test2(c: Color) { + return (c is Color, c !is Color, c is EPartialAssigned, c is EEmpty, c is int, c is int8); +} + +@method_id(103) +fun test3() { + assert (EPartialAssigned.P4 as int == 4) throw 123; + assert (EPartialAssigned.M99 as int <= 0) throw 123; + assert (EPartialAssigned.P3 == (3 as EPartialAssigned)) throw 123; + assert (EPartialAssigned.P3 != (4 as EPartialAssigned)) throw 123; + return ( + EPartialAssigned.M100, + EPartialAssigned.M99, + EPartialAssigned.P1, + EPartialAssigned.P2, + EPartialAssigned.P3, + EPartialAssigned.P4, + EPartialAssigned.P5, + EPartialAssigned.P6, + EPartialAssigned.Max, + ) +} + +@method_id(104) +fun test4(chooseGreen: bool) { + var c: ColorAlias = chooseGreen ? Color.Green : ColorAlias.Blue; + return (c is Color, (c!) is ColorAlias, ColorAlias.Blue, ColorAlias.createBlue().isBlue(), c); +} + +@method_id(105) +fun test5() { + return (int.Red(), int8.Max(), 456.Max(), EPartialAssigned.Max, Color.Max()); +} + +@method_id(106) +fun test6(c: Color) { + val p = c as EPartialAssigned; + val e = c as EEmpty; + match (e) { EEmpty => {} } + return (p, e, c as int, c as int?, 100 as Color, -100 as EEmpty); +} + +@method_id(107) +fun test7() { + return ( + WithColor{c1: Color.Red}, + WithColor{c1: -100 as Color, c2: Color.createBlue()} + ) +} + +@method_id(108) +fun test8() { + var (c0: Color | int, c1: Color | int) = (Color.Red, 123); + var (u1, u2) = (Color.Red as Color | int, 0 as Color | int); + var (b1, b2, b3) = (Color.Red as Color | EPartialAssigned | builder, EPartialAssigned.Max as ColorAlias | builder | EPartialAssigned, beginCell() as builder | Color | EPartialAssigned); + return ( + checkIsColor(u1), checkIsColor(u2), + checkIsColor(b1), checkIsColor(b2), checkIsColor(b3), + b2 is EPartialAssigned, + u1, + match (u1) { int => -100, Color => -200 } + ) +} + +@method_id(109) +fun test9() { + return ( + Color.Red.mapToNumber(), Color.Green.mapToNumber(), Color.Blue.mapToNumber(), + // this is UB, the compiler may do any optimizations + (100 as Color).mapToNumber(), (-99 as Color).mapToNumber(), + ); +} + +@method_id(110) +fun test10(c: Color) { + var t = createEmptyTuple(); + match (c) { + Color.Red => { t.push(1) } + else => { t.push(-100) } + } + match (c) { + Color.Green => { t.push(2) } + Color.Blue => { t.push(3) } + Color.Red => { t.push(1) } + } + match (c is Color) { + true => { t.push(-100) } + } + match (c) { + Color => { t.push(88) } + } + match (c) { + Color.Blue => { t.push(3) } + Color.Green => { t.push(2) } + else => { t.push(-100) } + } + match (c) { + Color.Red => { t.push(1) } + Color.Blue => { t.push(3) } + Color.Green => { t.push(2) } + } + return t; +} + +@method_id(111) +fun test11(b1Value: Color) { + var b1: Box = { value: b1Value }; + var b2 = Box { value: Color.Green }; + var b3 = Box { value: Color.Blue }; + return (b1.value == b2.value, b1.value != b2.value, b1.value == b3.value, b1.value != b3.value) +} + +@method_id(112) +fun test12(c1: Color, c2: Color) { + var t = createEmptyTuple(); + + match (c1) { + Color.Red => { t.push(10 + c2.mapToNumber()) } + Color.Green => { t.push(20 + c2.mapToNumber()) } + Color.Blue => { t.push(30 + c2.mapToNumber()) } + } + + match (c2) { + Color.Red => { + match (c1) { + Color.Red => t.push(100), + Color.Blue => t.push(300), + Color.Green => t.push(200), + } + } + Color.Blue => match (c1) { + Color.Green => t.push(210), + Color.Red => t.push(110), + Color.Blue => t.push(310), + }, + Color.Green => match (c1) { + Color.Green => t.push(220), + Color.Red => t.push(120), + Color.Blue => t.push(320), + } + } + + t.push((999 as Color).mapToNumber()); + return t; +} + +@method_id(113) +fun test13(x: int) { + try { + if (x < 0) { throw Err.TooLowId } + assert (x < 100) throw Err.TooHighId; + assert (x != 10, Err.DisabledId); + return (0, false); + } catch (excno) { + return (excno, excno as Err == Err.TooLowId); + } +} + + +fun main(c: Color) { + __expect_type(c, "Color"); + __expect_type(C_BLUE, "Color"); + __expect_type(Color.createBlue(), "Color"); + __expect_type(ColorAlias.Red, "Color"); + __expect_type(EPartialAssigned.Max, "EPartialAssigned"); + __expect_type(Box{value:Color.Red}, "Box"); + return c; +} + + +/** +@testcase | 0 | 123 | 123 +@testcase | 101 | | -1 0 -1 -1 0 -1 -1 2 +@testcase | 102 | 1 | -1 0 0 0 0 0 +@testcase | 103 | | -100 -99 1 2 3 4 5 6 2147483647 +@testcase | 104 | -1 | -1 -1 2 -1 1 +@testcase | 105 | | 123 -10 456 2147483647 2147483648 +@testcase | 106 | 1 | 1 1 1 1 100 -100 +@testcase | 107 | | 0 1 -100 2 +@testcase | 108 | | -1 0 -1 0 0 -1 0 typeid-1 -200 +@testcase | 109 | | 100 200 300 300 300 +@testcase | 110 | 0 | [ 1 1 -100 88 -100 1 ] +@testcase | 110 | 1 | [ -100 2 -100 88 2 2 ] +@testcase | 110 | 2 | [ -100 3 -100 88 3 3 ] +@testcase | 110 | 44 | [ -100 1 -100 88 -100 2 ] +@testcase | 111 | 2 | 0 -1 -1 0 +@testcase | 112 | 0 1 | [ 210 120 300 ] +@testcase | 112 | 2 4 | [ 330 320 300 ] +@testcase | 113 | -8 | 256 -1 +@testcase | 113 | 111 | 257 0 +@testcase | 113 | 10 | 258 0 +@testcase | 113 | 50 | 0 0 + +@fif_codegen +""" + test1() PROC:<{ + TRUE + FALSE + TRUE + TRUE + FALSE + TRUE + TRUE + 2 PUSHINT + }> +""" + +@fif_codegen +""" + test2() PROC:<{ // c + DROP // + -1 PUSHINT // '1=-1 + 0 PUSHINT // '1=-1 '2 + 0 PUSHINT // '1=-1 '2 '3=0 + s0 s0 s0 PUSH3 // '1=-1 '2 '3=0 '4=0 '5=0 '6=0 + }> +""" + +@fif_codegen DECLPROC checkIsColor() + +*/ diff --git a/tolk-tester/tests/generics-1.tolk b/tolk-tester/tests/generics-1.tolk index b92f27a6e3..d44f083f8a 100644 --- a/tolk-tester/tests/generics-1.tolk +++ b/tolk-tester/tests/generics-1.tolk @@ -15,7 +15,7 @@ fun tuplePush(mutate t: tuple, value: T): void @method_id(101) fun test101(x: int) { var (a, b, c) = (x, (x,x), [x,x]); - return (eq1(a), eq1(b), eq1(c), eq2(a), eq2(b), eq2(c), eq3(a), eq4(b), eq3(createEmptyTuple())); + return (eq1(a), eq1(b), eq1(c), eq2(a), eq2(b), eq2(c), eq3(a), eq4(b), eq3(createEmptyTuple()), eq1(Color.Red)); } fun getTwo(): X { return 2 as X; } @@ -28,6 +28,7 @@ struct Wrapper { struct Err { errPayload: T } +enum Color { Red, Green, Blue } @method_id(102) fun test102(): (int, int, int, [int, int], Wrapper< Wrapper >) { @@ -211,6 +212,7 @@ fun test113(a: int | slice) { fun main(x: int): (int, [Tup2Int]) { __expect_type(test110, "() -> (Tensor2Int, Tensor2Int)"); + __expect_type(eq2(10 as Color), "Color"); try { if(x) { throw (1, x); } } catch (excNo, arg) { return (arg as int, [[eq2(arg as int), getTwo()]]); } @@ -219,7 +221,7 @@ fun main(x: int): (int, [Tup2Int]) { /** @testcase | 0 | 1 | 1 [ [ 1 2 ] ] -@testcase | 101 | 0 | 0 0 0 [ 0 0 ] 0 0 0 [ 0 0 ] 0 0 0 [] +@testcase | 101 | 0 | 0 0 0 [ 0 0 ] 0 0 0 [ 0 0 ] 0 0 0 [] 0 @testcase | 102 | | 2 2 2 [ 2 2 ] 2 @testcase | 103 | 0 | 0 100 100 @testcase | 104 | 0 | [ 1 (null) 2 ] [ 2 -1 0 ] @@ -241,6 +243,7 @@ fun main(x: int): (int, [Tup2Int]) { @fif_codegen DECLPROC eq1() @fif_codegen DECLPROC eq1() @fif_codegen DECLPROC eq1>>() +@fif_codegen DECLPROC eq1() // was inlined @fif_codegen_avoid DECLPROC getTwo() diff --git a/tolk-tester/tests/invalid-declaration/err-1039.tolk b/tolk-tester/tests/invalid-declaration/err-1039.tolk new file mode 100644 index 0000000000..52143ab754 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1039.tolk @@ -0,0 +1,7 @@ + +enum Color { Red = Color.Red } + +/** +@compilation_should_fail +@stderr referencing other members in initializers is not allowed yet + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1697.tolk b/tolk-tester/tests/invalid-declaration/err-1697.tolk new file mode 100644 index 0000000000..cc8a904327 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1697.tolk @@ -0,0 +1,7 @@ + +enum Color { Red = ton("0.05") as int } + +/** +@compilation_should_fail +@stderr unsupported operation in `enum` member value + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1700.tolk b/tolk-tester/tests/invalid-declaration/err-1700.tolk new file mode 100644 index 0000000000..018a583fe1 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1700.tolk @@ -0,0 +1,11 @@ +enum Over { + MaxIntMinus1 = +115792089237316195423570985008687907853269984665640564039457584007913129639935 - 1, + MaxInt, + MaxIntPlus1, +} + +/** +@compilation_should_fail +@stderr integer overflow +@stderr MaxIntPlus1 + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1839.tolk b/tolk-tester/tests/invalid-declaration/err-1839.tolk new file mode 100644 index 0000000000..595a853b58 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1839.tolk @@ -0,0 +1,8 @@ +enum Over { + A = (+115792089237316195423570985008687907853269984665640564039457584007913129639934 << 1) * 2, +} + +/** +@compilation_should_fail +@stderr integer overflow + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4395.tolk b/tolk-tester/tests/invalid-semantics/err-4395.tolk new file mode 100644 index 0000000000..c258ff3797 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4395.tolk @@ -0,0 +1,10 @@ +enum Color { Red, Green, Blue, } + +fun main() { + Color.Red = 123 as Color; +} + +/** +@compilation_should_fail +@stderr modifying immutable constant + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4518.tolk b/tolk-tester/tests/invalid-semantics/err-4518.tolk new file mode 100644 index 0000000000..7fbf9e01ae --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4518.tolk @@ -0,0 +1,13 @@ +struct Point { x: int, y: int } +fun Point.create() {} + +fun nnn() {} + +fun main() { + Point.create = nnn; +} + +/** +@compilation_should_fail +@stderr invalid left side of assignment + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4810.tolk b/tolk-tester/tests/invalid-semantics/err-4810.tolk new file mode 100644 index 0000000000..7c13cf87b2 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4810.tolk @@ -0,0 +1,10 @@ +enum Color { Red, Green, Blue } + +fun main() { + return Color.Red +} + +/** +@compilation_should_fail +@stderr type arguments not expected here + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2190.tolk b/tolk-tester/tests/invalid-symbol/err-2190.tolk new file mode 100644 index 0000000000..b15475538e --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2190.tolk @@ -0,0 +1,14 @@ +enum Color { + Red, + Green, + Blue, +} + +fun main(c: Color) { + c.Red +} + +/** +@compilation_should_fail +@stderr field `Red` doesn't exist in type `Color` + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2618.tolk b/tolk-tester/tests/invalid-symbol/err-2618.tolk new file mode 100644 index 0000000000..29745c2de9 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2618.tolk @@ -0,0 +1,9 @@ +enum Unknown { + A = KKK, +} + +/** +@compilation_should_fail +@stderr undefined symbol `KKK` +@stderr A = + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2910.tolk b/tolk-tester/tests/invalid-symbol/err-2910.tolk new file mode 100644 index 0000000000..1b5bc4d3f8 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2910.tolk @@ -0,0 +1,10 @@ +enum Color { Red, Green, Blue } + +fun main() { + Color.Yellow +} + +/** +@compilation_should_fail +@stderr member `Yellow` does not exist in enum `Color` + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3410.tolk b/tolk-tester/tests/invalid-syntax/err-3410.tolk new file mode 100644 index 0000000000..354a82de2f --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3410.tolk @@ -0,0 +1,11 @@ +enum Color { + Red, + Green, + Blue, + , +} + +/** +@compilation_should_fail +@stderr expected member name, got `,` + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3781.tolk b/tolk-tester/tests/invalid-syntax/err-3781.tolk deleted file mode 100644 index bf23d85704..0000000000 --- a/tolk-tester/tests/invalid-syntax/err-3781.tolk +++ /dev/null @@ -1,6 +0,0 @@ -enum MyKind { } - -/** -@compilation_should_fail -@stderr `enum` is not supported yet -*/ diff --git a/tolk-tester/tests/invalid-typing/err-6120.tolk b/tolk-tester/tests/invalid-typing/err-6120.tolk new file mode 100644 index 0000000000..ebe021ea66 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6120.tolk @@ -0,0 +1,16 @@ +enum Color { Red, Green, Blue } + +fun main(c: Color) { + match (c) { + Color.Red => {} + else => {} + } + match (c) { + Color.Green => {} + } +} + +/** +@compilation_should_fail +@stderr `match` does not cover all possible enum members; missing members are: Red, Blue + */ diff --git a/tolk-tester/tests/invalid-typing/err-6121.tolk b/tolk-tester/tests/invalid-typing/err-6121.tolk new file mode 100644 index 0000000000..25dd91b630 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6121.tolk @@ -0,0 +1,17 @@ +enum Color { Red, Green, Blue } +type ColorAlias = Color + +fun main(c: Color) { + match (c) { + Color.Red => {} + Color.Green => {} + ColorAlias.Red => {} + else => {} + } +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: duplicated enum member in `match` +@stderr ColorAlias.Red + */ diff --git a/tolk-tester/tests/invalid-typing/err-6122.tolk b/tolk-tester/tests/invalid-typing/err-6122.tolk new file mode 100644 index 0000000000..66e218b641 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6122.tolk @@ -0,0 +1,17 @@ +enum Color { Red, Green, Blue } +type ColorAlias = Color + +fun main(c: Color) { + match (c) { + Color.Red => {} + Color.Green => {} + ColorAlias.Blue => {} + else => {} + } +} + +/** +@compilation_should_fail +@stderr `match` already covers all possible enum members, `else` is invalid +@stderr else => {} + */ diff --git a/tolk-tester/tests/invalid-typing/err-6310.tolk b/tolk-tester/tests/invalid-typing/err-6310.tolk new file mode 100644 index 0000000000..901a04b6b1 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6310.tolk @@ -0,0 +1,12 @@ +enum Color { Red, Green, Blue } + +fun main(c: Color) { + match (c) { + 2 => {} + } +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: can not compare type `int` with match subject of type `Color` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6311.tolk b/tolk-tester/tests/invalid-typing/err-6311.tolk new file mode 100644 index 0000000000..56adc34cf2 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6311.tolk @@ -0,0 +1,12 @@ +enum Color { Red, Green, Blue } + +fun main() { + match (2 as int8) { + Color.Red => {} + } +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: can not compare type `Color` with match subject of type `int8` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6312.tolk b/tolk-tester/tests/invalid-typing/err-6312.tolk new file mode 100644 index 0000000000..7b9c0dfb1e --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6312.tolk @@ -0,0 +1,14 @@ +enum Color { Red, Green, Blue } +enum Shape { Circle, Square } + +fun main(c: Color) { + c as Shape; // ok + match (c) { + Shape.Circle => {} + } +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: can not compare type `Shape` with match subject of type `Color` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6511.tolk b/tolk-tester/tests/invalid-typing/err-6511.tolk new file mode 100644 index 0000000000..880587674c --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6511.tolk @@ -0,0 +1,10 @@ +enum Color { Red, Green, Blue } + +fun main(c: Color) { + c == 1 +} + +/** +@compilation_should_fail +@stderr can not apply operator `==` to `Color` and `int` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6512.tolk b/tolk-tester/tests/invalid-typing/err-6512.tolk new file mode 100644 index 0000000000..03b81b5f60 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6512.tolk @@ -0,0 +1,10 @@ +enum Color { Red, Green, Blue } + +fun main(c: Color) { + c + 1 +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `Color` and `int` + */ diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index b850a72809..51ec9148d9 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -1591,6 +1591,14 @@ fun demo411(s: slice) { } } +@method_id(412) +fun demo412() { + val msg = lazy AOrB.fromSlice(""); + match (msg) { + } + return 123; +} + fun main() { val s = stringHexToSlice("0102"); @@ -1755,6 +1763,7 @@ fun main() { @testcase | 407 | x{BF} | -1 @testcase | 408 | | 24 @testcase | 409 | | 255 3 +@testcase | 412 | | 123 @fif_codegen """ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 35fd441716..86de235293 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -1625,6 +1625,58 @@ static AnyV parse_struct_declaration(Lexer& lex, const std::vector(loc, v_ident, genericsT_list, overflow1023_policy, opcode, parse_struct_body(lex)); } +static AnyV parse_enum_member(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.check(tok_identifier, "member name"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + + AnyExprV init_value = nullptr; + if (lex.tok() == tok_assign) { // `Red = 1` + lex.next(); + init_value = parse_expr(lex); + } + + return createV(loc, v_ident, init_value); +} + +static V parse_enum_body(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_opbrace, "`{`"); + + std::vector members; + while (lex.tok() != tok_clbrace) { + members.push_back(parse_enum_member(lex)); + if (lex.tok() == tok_comma || lex.tok() == tok_semicolon) { + lex.next(); + } + } + lex.expect(tok_clbrace, "`}`"); + + return createV(loc, std::move(members)); +} + +static AnyV parse_enum_declaration(Lexer& lex, const std::vector>& annotations) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_enum, "`enum`"); + + lex.check(tok_identifier, "identifier"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + + for (auto v_annotation : annotations) { + switch (v_annotation->kind) { + case AnnotationKind::deprecated: + case AnnotationKind::custom: + break; + default: + v_annotation->error("this annotation is not applicable to enum"); + } + } + + return createV(loc, v_ident, parse_enum_body(lex)); +} + static AnyV parse_tolk_required_version(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.next_special(tok_semver, "semver"); // syntax: "tolk 0.6" @@ -1707,9 +1759,12 @@ AnyV parse_src_file_to_ast(const SrcFile* file) { toplevel_declarations.push_back(parse_struct_declaration(lex, annotations)); annotations.clear(); break; + case tok_enum: + toplevel_declarations.push_back(parse_enum_declaration(lex, annotations)); + annotations.clear(); + break; case tok_export: - case tok_enum: case tok_operator: case tok_infix: lex.error("`" + static_cast(lex.cur_str()) +"` is not supported yet"); diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 581babee31..41683571e7 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -213,6 +213,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { const std::vector& get_all_not_builtin_functions(); const std::vector& get_all_declared_constants(); const std::vector& get_all_declared_structs(); +const std::vector& get_all_declared_enums(); template void replace_ast_of_all_functions() { diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index fd19401e95..570ab048c7 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -102,6 +102,9 @@ class ASTStringifier final : public ASTVisitor { {ast_struct_field, "ast_struct_field"}, {ast_struct_body, "ast_struct_body"}, {ast_struct_declaration, "ast_struct_declaration"}, + {ast_enum_member, "ast_enum_member"}, + {ast_enum_body, "ast_enum_body"}, + {ast_enum_declaration, "ast_enum_declaration"}, {ast_tolk_required_version, "ast_tolk_required_version"}, {ast_import_directive, "ast_import_directive"}, {ast_tolk_file, "ast_tolk_file"}, @@ -176,6 +179,8 @@ class ASTStringifier final : public ASTVisitor { return static_cast(v->as()->get_identifier()->name) + ": " + ast_type_node_to_string(v->as()->type_node); case ast_struct_declaration: return "struct " + static_cast(v->as()->get_identifier()->name); + case ast_enum_declaration: + return "enum " + static_cast(v->as()->get_identifier()->name); case ast_assign: return "="; case ast_set_assign: @@ -361,6 +366,9 @@ class ASTStringifier final : public ASTVisitor { case ast_struct_field: return handle_vertex(v->as()); case ast_struct_body: return handle_vertex(v->as()); case ast_struct_declaration: return handle_vertex(v->as()); + case ast_enum_member: return handle_vertex(v->as()); + case ast_enum_body: return handle_vertex(v->as()); + case ast_enum_declaration: return handle_vertex(v->as()); case ast_tolk_required_version: return handle_vertex(v->as()); case ast_import_directive: return handle_vertex(v->as()); case ast_tolk_file: return handle_vertex(v->as()); diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index 65f5608ed6..165dc0ce45 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -215,6 +215,7 @@ const std::vector& get_all_not_builtin_functions(); const std::vector& get_all_declared_global_vars(); const std::vector& get_all_declared_constants(); const std::vector& get_all_declared_structs(); +const std::vector& get_all_declared_enums(); template void visit_ast_of_all_functions() { diff --git a/tolk/ast.cpp b/tolk/ast.cpp index b22f9d31f9..b13d90beb1 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -177,6 +177,10 @@ void Vertex::assign_alias_ref(AliasDefPtr alias_ref) this->alias_ref = alias_ref; } +void Vertex::assign_enum_ref(EnumDefPtr enum_ref) { + this->enum_ref = enum_ref; +} + void Vertex::assign_struct_ref(StructPtr struct_ref) { this->struct_ref = struct_ref; } diff --git a/tolk/ast.h b/tolk/ast.h index 8836d2a068..b802f847bc 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -132,6 +132,9 @@ enum ASTNodeKind { ast_struct_field, ast_struct_body, ast_struct_declaration, + ast_enum_member, + ast_enum_body, + ast_enum_declaration, ast_tolk_required_version, ast_import_directive, ast_tolk_file, @@ -716,12 +719,14 @@ struct Vertex final : ASTExprUnary { typedef std::variant< FunctionPtr, // for `t.tupleAt` target is `tupleAt` global function StructFieldPtr, // for `user.id` target is field `id` of struct `User` + EnumMemberPtr, // for `Color.Red` target is `Red` enum member int // for `t.0` target is "indexed access" 0 > DotTarget; DotTarget target = static_cast(nullptr); // filled at type inferring bool is_target_fun_ref() const { return std::holds_alternative(target); } bool is_target_struct_field() const { return std::holds_alternative(target); } + bool is_target_enum_member() const { return std::holds_alternative(target); } bool is_target_indexed_access() const { return std::holds_alternative(target); } AnyExprV get_obj() const { return child; } @@ -1366,8 +1371,8 @@ struct Vertex final : ASTOtherVararg { auto get_field(int i) const { return children.at(i)->as(); } const std::vector& get_all_fields() const { return children; } - Vertex(SrcLocation loc, std::vector&& fields) - : ASTOtherVararg(ast_struct_body, loc, std::move(fields)) {} + Vertex(SrcLocation loc, std::vector&& members) + : ASTOtherVararg(ast_struct_body, loc, std::move(members)) {} }; template<> @@ -1393,6 +1398,47 @@ struct Vertex final : ASTOtherVararg { , genericsT_list(genericsT_list), overflow1023_policy(overflow1023_policy) {} }; +template<> +// ast_enum_member is one member at enum declaration +// example: `enum Color { Red = 1, Green, Blue }` is enum declaration, its body contains 3 members +struct Vertex final : ASTOtherVararg { + AnyExprV init_value; // nullptr if no default + + auto get_identifier() const { return children.at(0)->as(); } + + Vertex(SrcLocation loc, V name_identifier, AnyExprV init_value) + : ASTOtherVararg(ast_enum_member, loc, {name_identifier}) + , init_value(init_value) {} +}; + +template<> +// ast_enum_body is `{ ... }` inside enum declaration, it contains enum members +// example: `enum Color { Red = 1, Green, Blue }` its body contains 3 members +struct Vertex final : ASTOtherVararg { + int get_num_members() const { return size(); } + auto get_member(int i) const { return children.at(i)->as(); } + const std::vector& get_all_members() const { return children; } + + Vertex(SrcLocation loc, std::vector&& members) + : ASTOtherVararg(ast_enum_body, loc, std::move(members)) {} +}; + +template<> +// ast_enum_declaration is a declaring a `enum` similar to TypeScript (not to Rust, we don't need structural enums, we have unions) +// example: `enum Color { Red, Green, Blue }` +struct Vertex final : ASTOtherVararg { + EnumDefPtr enum_ref = nullptr; // filled after register + + auto get_identifier() const { return children.at(0)->as(); } + auto get_enum_body() const { return children.at(1)->as(); } + + Vertex* mutate() const { return const_cast(this); } + void assign_enum_ref(EnumDefPtr enum_ref); + + Vertex(SrcLocation loc, V name_identifier, V enum_body) + : ASTOtherVararg(ast_enum_declaration, loc, {name_identifier, enum_body}) {} +}; + template<> // ast_tolk_required_version is a preamble fixating compiler's version at the top of the file // example: `tolk 0.6` diff --git a/tolk/compiler-state.cpp b/tolk/compiler-state.cpp index 9615e929f3..d7bea32bde 100644 --- a/tolk/compiler-state.cpp +++ b/tolk/compiler-state.cpp @@ -82,4 +82,8 @@ const std::vector& get_all_declared_structs() { return G.all_structs; } +const std::vector& get_all_declared_enums() { + return G.all_enums; +} + } // namespace tolk diff --git a/tolk/compiler-state.h b/tolk/compiler-state.h index 2f27692cf3..f56b30a434 100644 --- a/tolk/compiler-state.h +++ b/tolk/compiler-state.h @@ -102,6 +102,7 @@ struct CompilerState { std::vector all_global_vars; std::vector all_constants; std::vector all_structs; + std::vector all_enums; AllRegisteredSrcFiles all_src_files; bool is_verbosity(int gt_eq) const { return settings.verbosity >= gt_eq; } diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index 20c316fdb1..5952b8c060 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -315,8 +315,13 @@ struct ConstantExpressionChecker { if (auto v_cast_to = v->try_as()) { return visit(v_cast_to->get_expr()); } - if (auto v_dot = v->try_as(); v_dot && (v_dot->is_target_indexed_access() || v_dot->is_target_fun_ref())) { - return visit(v_dot->get_obj()); + if (auto v_dot = v->try_as()) { + if (v_dot->is_target_indexed_access()) { // anotherConst.0 + return visit(v_dot->get_obj()); + } + if (v_dot->is_target_enum_member()) { // Color.Red + return; + } } v->error("not a constant expression"); } @@ -329,6 +334,79 @@ struct ConstantExpressionChecker { } }; +struct EnumMemberEvaluator { + static td::RefInt256 handle_unary_operator(V v) { + td::RefInt256 rhs = visit(v->get_rhs()); + if (!rhs->is_valid()) return rhs; + + switch (v->tok) { + case tok_minus: return -rhs; + case tok_bitwise_not: return ~rhs; + case tok_plus: return rhs; + default: + fire(nullptr, v->loc, "unsupported operator in `enum` member value"); + } + } + + static td::RefInt256 handle_binary_operator(V v) { + td::RefInt256 lhs = visit(v->get_lhs()); + td::RefInt256 rhs = visit(v->get_rhs()); + if (!lhs->is_valid()) return lhs; + if (!rhs->is_valid()) return rhs; + + switch (v->tok) { + case tok_plus: return lhs + rhs; + case tok_minus: return lhs - rhs; + case tok_mul: return lhs * rhs; + case tok_div: return lhs / rhs; + case tok_mod: return lhs % rhs; + case tok_bitwise_and: return lhs & rhs; + case tok_bitwise_or: return lhs | rhs; + case tok_bitwise_xor: return lhs ^ rhs; + case tok_lshift: return lhs << static_cast(rhs->to_long()); + case tok_rshift: return lhs >> static_cast(rhs->to_long()); + default: + fire(nullptr, v->loc, "unsupported operator in `enum` member value"); + } + } + + // `enum { Red = SIX }`, we met `SIX` + static td::RefInt256 handle_reference(V v) { + GlobalConstPtr const_ref = v->sym->try_as(); + if (!const_ref) { + v->error("symbol `" + static_cast(v->get_name()) + "` is not a constant"); + } + return visit(const_ref->init_value); + } + + static td::RefInt256 visit(AnyExprV v) { + if (auto v_int = v->try_as()) { + return v_int->intval; + } + if (auto v_un = v->try_as()) { + return handle_unary_operator(v_un); + } + if (auto v_bin = v->try_as()) { + return handle_binary_operator(v_bin); + } + if (auto v_ref = v->try_as()) { + return handle_reference(v_ref); + } + if (auto v_par = v->try_as()) { + return visit(v_par->get_expr()); + } + if (auto v_dot = v->try_as()) { + // `enum E { V = AnotherEnum.A }` not allowed (can be allowed in the future, + // but remember to construct an initialization tree, since A may be auto-computed, and prevent recursion) + if (v_dot->is_target_enum_member()) { + fire(nullptr, v->loc, "referencing other members in initializers is not allowed yet"); + } + } + fire(nullptr, v->loc, "unsupported operation in `enum` member value"); + } +}; + + void check_expression_is_constant(AnyExprV v_expr) { ConstantExpressionChecker::check_expression_expected_to_be_constant(v_expr); } @@ -346,5 +424,9 @@ CompileTimeFunctionResult eval_call_to_compile_time_function(AnyExprV v_call) { return parse_vertex_call_to_compile_time_function(v, v->fun_maybe->name); } +td::RefInt256 eval_enum_member_init_value(AnyExprV init_value) { + return EnumMemberEvaluator::visit(init_value); +} + } // namespace tolk diff --git a/tolk/constant-evaluator.h b/tolk/constant-evaluator.h index b7a9fc7fe1..4f488951c6 100644 --- a/tolk/constant-evaluator.h +++ b/tolk/constant-evaluator.h @@ -27,5 +27,6 @@ typedef std::variant CompileTimeFunctionResult; void check_expression_is_constant(AnyExprV v_expr); std::string eval_string_const_standalone(AnyExprV v_string); CompileTimeFunctionResult eval_call_to_compile_time_function(AnyExprV v_call); +td::RefInt256 eval_enum_member_init_value(AnyExprV init_value); } // namespace tolk diff --git a/tolk/fwd-declarations.h b/tolk/fwd-declarations.h index f46def4ac3..8a5028edbb 100644 --- a/tolk/fwd-declarations.h +++ b/tolk/fwd-declarations.h @@ -36,6 +36,8 @@ struct GlobalConstData; struct AliasDefData; struct StructFieldData; struct StructData; +struct EnumMemberData; +struct EnumDefData; using LocalVarPtr = const LocalVarData*; using FunctionPtr = const FunctionData*; @@ -44,6 +46,8 @@ using GlobalConstPtr = const GlobalConstData*; using AliasDefPtr = const AliasDefData*; using StructFieldPtr = const StructFieldData*; using StructPtr = const StructData*; +using EnumMemberPtr = const EnumMemberData*; +using EnumDefPtr = const EnumDefData*; class TypeData; using TypePtr = const TypeData*; diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index 1c78222600..2b6c2d2145 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -424,6 +424,9 @@ std::vector generate_lazy_struct_to_cell(CodeBlob& code, SrcLocation std::vector generate_lazy_match_for_union(CodeBlob& code, SrcLocation loc, TypePtr union_type, const LazyVariableLoadedState* lazy_variable, const LazyMatchOptions& options) { tolk_assert(lazy_variable->ir_options.size() == 2); + if (options.match_blocks.empty()) { // empty `match` statement, no arms + return {}; + } UnpackContext ctx(code, loc, lazy_variable->ir_slice, lazy_variable->ir_options); std::vector rvect_match = ctx.generate_lazy_match_any(union_type, options); diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 068403ce15..965a032a63 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1092,6 +1092,18 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as()) { + return rvect; + } + if (original_type == TypeDataInt::create() && target_type->try_as()) { + return rvect; + } + // `Color.Red` as `BounceMode` (all enums are integers, they can be cast one to another) + if (original_type->try_as() && target_type->try_as()) { + return rvect; + } + throw Fatal("unhandled transition_expr_to_runtime_type_impl() combination"); } @@ -1390,7 +1402,8 @@ static std::vector process_lazy_operator(V v, Code } static std::vector process_match_expression(V v, CodeBlob& code, TypePtr target_type) { - TypePtr lhs_type = v->get_subject()->inferred_type; + TypePtr subject_type = v->get_subject()->inferred_type; + const TypeDataEnum* subject_enum = subject_type->unwrap_alias()->try_as(); int n_arms = v->get_arms_count(); std::vector subj_ir_idx = pre_compile_expr(v->get_subject(), code, nullptr); @@ -1401,18 +1414,28 @@ static std::vector process_match_expression(V v return {}; } + bool has_type_arm = false; + bool has_expr_arm = false; + bool has_else_arm = false; + for (int i = 0; i < n_arms; ++i) { + auto v_arm = v->get_arm(i); + has_type_arm |= v_arm->pattern_kind == MatchArmKind::exact_type; + has_expr_arm |= v_arm->pattern_kind == MatchArmKind::const_expression; + has_else_arm |= v_arm->pattern_kind == MatchArmKind::else_branch; + } + // it's either `match` by type (all arms patterns are types) or `match` by expression - bool is_match_by_type = v->get_arm(0)->pattern_kind == MatchArmKind::exact_type; - bool last_else_branch = v->get_arm(n_arms - 1)->pattern_kind == MatchArmKind::else_branch; + bool is_match_by_type = has_type_arm; // detect whether `match` is exhaustive bool is_exhaustive = is_match_by_type // match by type covers all cases, checked earlier - || !v->is_statement() // match by expression is guaranteely exhaustive, checked earlier - || last_else_branch; + || !v->is_statement() // match by expression is exhaustive, checked earlier + || (has_expr_arm && subject_enum) // match by enum (either covers all cases or contains else, checked earlier) + || has_else_arm; // `else` is not allowed in `match` by type; this was not fired at type checking, // because it might turned out to be a lazy match, where `else` is allowed; // if we are here, it's not a lazy match, it's a regular one (the lazy one is handled specially, in aux vertex) - if (is_match_by_type && last_else_branch) { + if (is_match_by_type && has_else_arm) { throw ParseError(v->get_arm(n_arms - 1)->loc, "`else` is not allowed in `match` by type; you should cover all possible types"); } @@ -1429,7 +1452,7 @@ static std::vector process_match_expression(V v if (is_match_by_type) { TypePtr cmp_type = v_ith_arm->pattern_type_node->resolved_type; tolk_assert(!cmp_type->unwrap_alias()->try_as()); // `match` over `int|slice` is a type checker error - eq_ith_ir_idx = pre_compile_is_type(code, lhs_type, cmp_type, subj_ir_idx, v_ith_arm->loc, "(arm-cond-eq)"); + eq_ith_ir_idx = pre_compile_is_type(code, subject_type, cmp_type, subj_ir_idx, v_ith_arm->loc, "(arm-cond-eq)"); } else { std::vector ith_ir_idx = pre_compile_expr(v_ith_arm->get_pattern_expr(), code); tolk_assert(subj_ir_idx.size() == 1 && ith_ir_idx.size() == 1); @@ -1475,7 +1498,7 @@ static std::vector process_match_expression(V v static std::vector process_dot_access(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { // it's NOT a method call `t.tupleSize()` (since such cases are handled by process_function_call) // it's `t.0`, `getUser().id`, and `t.tupleSize` (as a reference, not as a call) - if (!v->is_target_fun_ref()) { + if (v->is_target_indexed_access() || v->is_target_struct_field()) { TypePtr obj_type = v->get_obj()->inferred_type->unwrap_alias(); // `user.id`; internally, a struct (an object) is a tensor if (const auto* t_struct = obj_type->try_as()) { @@ -1551,6 +1574,15 @@ static std::vector process_dot_access(V v, CodeBlob& } tolk_assert(false); } + // `Color.Red` + if (v->is_target_enum_member()) { + // all enums are integers, and their integer values have already been assigned or auto-calculated + EnumMemberPtr member_ref = std::get(v->target); + tolk_assert(!member_ref->computed_value.is_null()); + std::vector enum_ir_idx = code.create_tmp_var(TypeDataInt::create(), v->get_identifier()->loc, "(enum-member)"); + code.emplace_back(v->loc, Op::_IntConst, enum_ir_idx, member_ref->computed_value); + return transition_to_target_type(std::move(enum_ir_idx), code, target_type, v); + } // okay, v->target refs a function, like `obj.method`, filled at type inferring // (currently, nothing except a global function can be referenced, no object-scope methods exist) diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 7880149c84..a7bbe549f1 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -191,18 +191,29 @@ static bool expect_boolean(AnyExprV v_inferred) { return expect_boolean(v_inferred->inferred_type); } -static bool expect_address(TypePtr inferred_type) { - if (inferred_type == TypeDataAddress::create()) { +static bool expect_thrown_code(TypePtr t_ex_no) { + return expect_integer(t_ex_no) || t_ex_no->unwrap_alias()->try_as(); +} + +static bool check_eq_neq_operator(TypePtr lhs_type, TypePtr rhs_type, bool& not_integer_comparison) { + if (expect_integer(lhs_type) && expect_integer(rhs_type)) { return true; } - if (const TypeDataAlias* as_alias = inferred_type->try_as()) { - return expect_address(as_alias->underlying_type); + if (expect_boolean(lhs_type) && expect_boolean(rhs_type)) { + return true; + } + if (lhs_type->equal_to(TypeDataAddress::create()) && rhs_type->equal_to(TypeDataAddress::create())) { + not_integer_comparison = true; // `address` can be compared with ==, but it's handled specially + return true; } - return false; -} -static bool expect_address(AnyExprV v_inferred) { - return expect_address(v_inferred->inferred_type); + const TypeDataEnum* lhs_enum = lhs_type->unwrap_alias()->try_as(); + const TypeDataEnum* rhs_enum = rhs_type->unwrap_alias()->try_as(); + if (lhs_enum && rhs_enum) { // allow `someColor == anotherColor`, don't allow `someColor == 123` + return lhs_enum->enum_ref == rhs_enum->enum_ref; + } + + return false; } class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { @@ -258,17 +269,16 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // (if to allow `int?`/`int8?` in the future, `==` must be expressed in a complicated Fift code considering TVM NULL) case tok_eq: case tok_neq: { - bool both_int = expect_integer(lhs) && expect_integer(rhs); - bool both_bool = expect_boolean(lhs) && expect_boolean(rhs); - if (!both_int && !both_bool) { - bool both_address = expect_address(lhs) && expect_address(rhs); - if (both_address) { // address can be compared with ==, but it's not integer comparison, it's handled specially - v->mutate()->assign_fun_ref(nullptr); - } else if (lhs->inferred_type->equal_to(rhs->inferred_type)) { // compare slice with slice, int? with int? + bool not_integer_comparison = false; + if (!check_eq_neq_operator(lhs->inferred_type, rhs->inferred_type, not_integer_comparison)) { + if (lhs->inferred_type->equal_to(rhs->inferred_type)) { // compare slice with slice, int? with int? fire(cur_f, v->loc, "type " + to_string(lhs) + " can not be compared with `== !=`"); } else { fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); } + } + if (not_integer_comparison) { // special handling at IR generation like for `address` + v->mutate()->assign_fun_ref(nullptr); } break; } @@ -563,9 +573,11 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { bool has_else_arm = false; AnyExprV v_subject = v->get_subject(); TypePtr subject_type = v_subject->inferred_type; + const TypeDataEnum* subject_enum = subject_type->unwrap_alias()->try_as(); const TypeDataUnion* subject_union = subject_type->unwrap_alias()->try_as(); - std::vector covered_types; // for type-based `match`, what types are on the left of `=>` + std::vector covered_types; // for type-based `match`, what types are on the left of `=>` + std::vector covered_enum; // for `match` over an enum, we want it to be exhaustive for (int i = 0; i < v->get_arms_count(); ++i) { auto v_arm = v->get_arm(i); @@ -606,15 +618,25 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } has_expr_arm = true; TypePtr pattern_type = v_arm->get_pattern_expr()->inferred_type; - bool both_int = expect_integer(pattern_type) && expect_integer(subject_type); - bool both_bool = expect_boolean(pattern_type) && expect_boolean(subject_type); - if (!both_int && !both_bool) { + bool not_integer_comparison = false; + if (!check_eq_neq_operator(pattern_type, subject_type, not_integer_comparison) || not_integer_comparison) { if (pattern_type->equal_to(subject_type)) { // `match` over `slice` etc., where operator `==` can't be applied fire(cur_f, v_arm->loc, "wrong pattern matching: can not compare type " + to_string(subject_type) + " in `match`"); } else { fire(cur_f, v_arm->loc, "wrong pattern matching: can not compare type " + to_string(v_arm->get_pattern_expr()) + " with match subject of type " + to_string(v_subject)); } } + if (subject_enum) { + auto l_dot = v_arm->get_pattern_expr()->try_as(); + if (!l_dot || !l_dot->is_target_enum_member()) { // match (someColor) { anotherColor => } + fire(cur_f, v_arm->loc, "wrong pattern matching: `match` should contain members of a enum"); + } + EnumMemberPtr member_ref = std::get(l_dot->target); + if (std::find(covered_enum.begin(), covered_enum.end(), member_ref) != covered_enum.end()) { + fire(cur_f, v_arm->loc, "wrong pattern matching: duplicated enum member in `match`"); + } + covered_enum.push_back(member_ref); + } break; } default: @@ -651,8 +673,30 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } fire(cur_f, v->loc, "`match` does not cover all possible types; missing types are: " + missing); } - // `match` by expression, if it's not statement, should have `else` (unless it's match over bool with const true/false) - if (has_expr_arm && !has_else_arm && !v->is_statement()) { + // fire if `match` by enum is not exhaustive + if (has_expr_arm && subject_enum && !has_else_arm && subject_enum->enum_ref->members.size() != covered_enum.size()) { + std::string missing; + for (EnumMemberPtr member_ref : subject_enum->enum_ref->members) { + if (std::find(covered_enum.begin(), covered_enum.end(), member_ref) == covered_enum.end()) { + if (!missing.empty()) { + missing += ", "; + } + missing += member_ref->name; + } + } + fire(cur_f, v->loc, "`match` does not cover all possible enum members; missing members are: " + missing); + } + // fire if `match` by enum covers all cases, but contains `else` + // (note that `else` for types could exist for a lazy match; for non-lazy, it's fired later) + if (has_expr_arm && subject_enum && has_else_arm && subject_enum->enum_ref->members.size() == covered_enum.size()) { + for (int i = 0; i < v->get_arms_count(); ++i) { + if (auto v_arm = v->get_arm(i); v_arm->pattern_kind == MatchArmKind::else_branch) { + fire(cur_f, v_arm->loc, "`match` already covers all possible enum members, `else` is invalid"); + } + } + } + // `match` by expression, if it's not a statement, should have `else` (unless exhaustive) + if (has_expr_arm && !has_else_arm && !v->is_statement() && !subject_enum) { bool needs_else_branch = true; if (expect_boolean(subject_type) && v->get_arms_count() == 2) { auto arm0 = v->get_arm(0)->get_pattern_expr()->try_as(); @@ -724,7 +768,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { parent::visit(v); - if (!expect_integer(v->get_thrown_code())) { + if (!expect_thrown_code(v->get_thrown_code()->inferred_type)) { fire(cur_f, v->get_thrown_code()->loc, "excNo of `throw` must be an integer, got " + to_string(v->get_thrown_code())); } if (v->has_thrown_arg() && v->get_thrown_arg()->inferred_type->get_width_on_stack() != 1) { @@ -739,7 +783,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (!expect_integer(cond) && !expect_boolean(cond)) { fire(cur_f, cond->loc, "can not use " + to_string(cond) + " as a boolean condition"); } - if (!expect_integer(v->get_thrown_code())) { + if (!expect_thrown_code(v->get_thrown_code()->inferred_type)) { fire(cur_f, v->get_thrown_code()->loc, "thrown excNo of `assert` must be an integer, got " + to_string(v->get_thrown_code())); } @@ -813,6 +857,16 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { fire_error_type_mismatch(nullptr, field_ref->loc, "can not assign {src} to {dst}", inferred_type, field_ref->declared_type); } } + + // given enum member `Red = 1` check types within its init_value and the whole expression itself + void start_visiting_enum_member(EnumDefPtr enum_ref, EnumMemberPtr member_ref) { + parent::visit(member_ref->init_value); + + TypePtr inferred_type = member_ref->init_value->inferred_type; + if (!inferred_type->equal_to(TypeDataInt::create()) && !inferred_type->equal_to(TypeDataEnum::create(enum_ref))) { + fire(nullptr, member_ref->loc, "enum member is " + to_string(inferred_type) + ", not `int`\nhint: all enums must be integers"); + } + } }; void pipeline_check_inferred_types() { @@ -829,6 +883,13 @@ void pipeline_check_inferred_types() { } } } + for (EnumDefPtr enum_ref : get_all_declared_enums()) { + for (EnumMemberPtr member_ref : enum_ref->members) { + if (member_ref->has_init_value()) { + visitor.start_visiting_enum_member(enum_ref, member_ref); + } + } + } } } // namespace tolk diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp index cb80676281..dd2f9a83e4 100644 --- a/tolk/pipe-check-rvalue-lvalue.cpp +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -179,6 +179,12 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { if (LocalVarPtr var_ref = as_ref->sym->try_as()) { on_var_used_as_lvalue(leftmost_obj->loc, var_ref); } + if (as_ref->sym->try_as()) { // `Point.create = f` + if (v->is_target_enum_member()) { + fire(cur_f, v->loc, "modifying immutable constant"); + } + fire(cur_f, v->loc, "invalid left side of assignment"); + } } } diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index 74fbb3453a..27cecd3c60 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -35,6 +35,21 @@ namespace tolk { +// assign `enum` members values (either auto-compute sequentially or use manual initializers) +static void assign_enum_members_values(EnumDefPtr enum_ref) { + td::RefInt256 prev_value = td::make_refint(-1); + for (EnumMemberPtr member_ref : enum_ref->members) { + td::RefInt256 cur_value = member_ref->init_value ? eval_enum_member_init_value(member_ref->init_value) : prev_value + 1; + if (!cur_value->is_valid() || !cur_value->signed_fits_bits(257)) { + fire(nullptr, member_ref->loc, "integer overflow"); + } + + member_ref->mutate()->assign_computed_value(cur_value); + prev_value = std::move(cur_value); + } +} + + class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { static V create_int_const(SrcLocation loc, td::RefInt256&& intval) { auto v_int = createV(loc, std::move(intval), {}); @@ -190,6 +205,10 @@ void pipeline_constant_folding() { } } } + // assign `enum` members values (either auto-compute sequentially or use manual initializers) + for (EnumDefPtr enum_ref : get_all_declared_enums()) { + assign_enum_members_values(enum_ref); + } replace_ast_of_all_functions(); } diff --git a/tolk/pipe-discover-parse-sources.cpp b/tolk/pipe-discover-parse-sources.cpp index 1cadf07f4d..da4a7850f4 100644 --- a/tolk/pipe-discover-parse-sources.cpp +++ b/tolk/pipe-discover-parse-sources.cpp @@ -47,7 +47,7 @@ void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, con tolk_assert(!file->ast); file->ast = parse_src_file_to_ast(file); - // if (!file->is_stdlib_file()) file->ast->debug_print(); + // if (!file->is_stdlib_file) file->ast->debug_print(); for (AnyV v_toplevel : file->ast->as()->get_toplevel_declarations()) { if (auto v_import = v_toplevel->try_as()) { diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 2fc1385d53..ec1ecf496f 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -160,14 +160,20 @@ static void fire_error_using_lateinit_variable_uninitialized(FunctionPtr cur_f, // fire an error when `obj.f()`, method `f` not found, try to locate a method for another type GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_method_not_found(FunctionPtr cur_f, SrcLocation loc, TypePtr receiver_type, std::string_view method_name) { - if (std::vector other = lookup_methods_with_name(method_name); !other.empty()) { - fire(cur_f, loc, "method `" + to_string(method_name) + "` not found for type " + to_string(receiver_type) + "\n(but it exists for type " + to_string(other.front()->receiver_type) + ")"); +static void fire_error_method_or_field_not_found(FunctionPtr cur_f, SrcLocation loc, TypePtr receiver_type, std::string_view field_name, bool called_as_method, bool is_static_dot) { + if (!called_as_method && is_static_dot && receiver_type->unwrap_alias()->try_as()) { + fire(cur_f, loc, "member `" + to_string(field_name) + "` does not exist in enum " + to_string(receiver_type)); } - if (const Symbol* sym = lookup_global_symbol(method_name); sym && sym->try_as()) { - fire(cur_f, loc, "method `" + to_string(method_name) + "` not found, but there is a global function named `" + to_string(method_name) + "`\n(a function should be called `foo(arg)`, not `arg.foo()`)"); + if (!called_as_method && !is_static_dot) { + fire(cur_f, loc, "field `" + to_string(field_name) + "` doesn't exist in type " + to_string(receiver_type)); + } + if (std::vector other = lookup_methods_with_name(field_name); !other.empty()) { + fire(cur_f, loc, "method `" + to_string(field_name) + "` not found for type " + to_string(receiver_type) + "\n(but it exists for type " + to_string(other.front()->receiver_type) + ")"); } - fire(cur_f, loc, "method `" + to_string(method_name) + "` not found"); + if (const Symbol* sym = lookup_global_symbol(field_name); sym && sym->try_as()) { + fire(cur_f, loc, "method `" + to_string(field_name) + "` not found, but there is a global function named `" + to_string(field_name) + "`\n(a function should be called `foo(arg)`, not `arg.foo()`)"); + } + fire(cur_f, loc, "method `" + to_string(field_name) + "` not found"); } // safe version of std::stoi that does not crash on long numbers @@ -250,6 +256,12 @@ static std::pair choose_only_method_to_call( fire(cur_f, loc, msg.str()); } +static void check_no_unexpected_type_arguments(FunctionPtr cur_f, V v_instantiationTs) { + if (v_instantiationTs != nullptr) { + fire(cur_f, v_instantiationTs->loc, "type arguments not expected here"); + } +} + // given fun `f` and a call `f(a,b,c)`, check that argument count is expected; // (parameters may have default values, so it's not as trivial as to compare params and args size) void check_arguments_count_at_fun_call(FunctionPtr cur_f, V v, FunctionPtr called_f, AnyExprV self_obj) { @@ -938,9 +950,9 @@ class InferTypesAndCallsAndFieldsVisitor final { fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); v->mutate()->assign_sym(fun_ref); - } else if (UNLIKELY(v_instantiationTs != nullptr)) { + } else { // non-generic function referenced like `return beginCell;` - fire(cur_f, v_instantiationTs->loc, "type arguments not expected here"); + check_no_unexpected_type_arguments(cur_f, v_instantiationTs); } if (out_f_called) { // so, it's `globalF()` / `genericFn()` / `genericFn()` @@ -963,9 +975,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } // for non-functions: `local_var` and similar not allowed - if (UNLIKELY(v->has_instantiationTs())) { - fire(cur_f, v->get_instantiationTs()->loc, "type arguments not expected here"); - } + check_no_unexpected_type_arguments(cur_f, v->get_instantiationTs()); return ExprFlow(std::move(flow), used_as_condition); } @@ -978,14 +988,25 @@ class InferTypesAndCallsAndFieldsVisitor final { FunctionPtr fun_ref = nullptr; // to be filled for `.method` / `Point.create` (both standalone or in a call) GenericsSubstitutions substitutedTs(nullptr); V v_ident = v->get_identifier(); // field/method name vertex + V v_instantiationTs = v->get_instantiationTs(); + std::string_view field_name = v_ident->name; - // handle `Point.create` / `Container.wrap`: no dot_obj expression actually, lhs is a type, looking up a method + // handle `Point.create` / `Container.wrap` / `Color.Red`: lhs is a type, looking up a constant/method if (auto obj_ref = v->get_obj()->try_as()) { if (const auto* obj_as_type = obj_ref->sym->try_as()) { TypePtr receiver_type = obj_as_type->resolved_type; - std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, v_ident->loc, receiver_type, v->get_field_name()); + // `Color.Red` (enum member) — just fill v->target and done + if (const TypeDataEnum* enum_receiver = receiver_type->unwrap_alias()->try_as()) { + if (EnumMemberPtr member_ref = enum_receiver->enum_ref->find_member(field_name)) { + v->mutate()->assign_target(member_ref); + check_no_unexpected_type_arguments(cur_f, v_instantiationTs); + assign_inferred_type(v, enum_receiver); // `Color.Red` is `Color` + return ExprFlow(std::move(flow), used_as_condition); + } + } + std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, v_ident->loc, receiver_type, field_name); if (!fun_ref) { - fire_error_method_not_found(cur_f, v_ident->loc, receiver_type, v->get_field_name()); + fire_error_method_or_field_not_found(cur_f, v_ident->loc, receiver_type, field_name, out_f_called != nullptr, true); } } } @@ -997,8 +1018,6 @@ class InferTypesAndCallsAndFieldsVisitor final { // our goal is to fill v->target (field/index/method) knowing type of obj TypePtr obj_type = dot_obj ? dot_obj->inferred_type->unwrap_alias() : TypeDataUnknown::create(); - V v_instantiationTs = v->get_instantiationTs(); - std::string_view field_name = v_ident->name; // check for field access (`user.id`), when obj is a struct if (const TypeDataStruct* obj_struct = obj_type->try_as()) { @@ -1008,6 +1027,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (SinkExpression s_expr = extract_sink_expression_from_vertex(v)) { inferred_type = flow.smart_cast_or_original(s_expr, inferred_type); } + check_no_unexpected_type_arguments(cur_f, v_instantiationTs); assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); } @@ -1074,11 +1094,7 @@ class InferTypesAndCallsAndFieldsVisitor final { fire(cur_f, v_ident->loc, "can not access field `" + static_cast(field_name) + "` of a possibly nullable object " + to_string(dot_obj) + "\nhint: check it via `obj != null` or use non-null assertion `obj!` operator"); } } - if (out_f_called) { - fire_error_method_not_found(cur_f, v_ident->loc, dot_obj->inferred_type, v->get_field_name()); - } else { - fire(cur_f, v_ident->loc, "field `" + static_cast(field_name) + "` doesn't exist in type " + to_string(dot_obj)); - } + fire_error_method_or_field_not_found(cur_f, v_ident->loc, dot_obj->inferred_type, field_name, out_f_called != nullptr, false); } // if `fun T.copy(self)` and reference `int.copy` — all generic parameters are determined by the receiver, we know it @@ -1097,9 +1113,9 @@ class InferTypesAndCallsAndFieldsVisitor final { substitutedTs.provide_type_arguments(collect_type_arguments_for_fun(v->loc, fun_ref->genericTs, v_instantiationTs)); fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); - } else if (UNLIKELY(v_instantiationTs != nullptr)) { + } else { // non-generic method referenced like `var cb = c.hash;` - fire(cur_f, v_instantiationTs->loc, "type arguments not expected here"); + check_no_unexpected_type_arguments(cur_f, v_instantiationTs); } if (out_f_called) { // so, it's `user.method()` / `t.tupleAt()` / `t.tupleAt()` / `Point.create` @@ -1697,6 +1713,11 @@ class InferTypesAndCallsAndFieldsVisitor final { } } } + + // given `enum Color { Red = 1 }` infer that it's int + void start_visiting_enum_member(EnumMemberPtr member_ref) { + infer_any_expr(member_ref->init_value, FlowContext(), false); + } }; class LaunchInferTypesAndMethodsOnce final { @@ -1775,6 +1796,15 @@ void pipeline_infer_types_and_calls_and_fields() { visitor.start_visiting_struct_fields(struct_ref); } } + + // infer types for init_value of enum members + for (EnumDefPtr enum_ref : get_all_declared_enums()) { + for (EnumMemberPtr member_ref : enum_ref->members) { + if (member_ref->has_init_value()) { + visitor.start_visiting_enum_member(member_ref); + } + } + } } void pipeline_infer_types_and_calls_and_fields(FunctionPtr fun_ref) { diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 9177229465..feb5c42524 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -98,7 +98,7 @@ static void validate_arg_ret_order_of_asm_function(V v_body, int n static GlobalConstPtr register_constant(V v) { GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->type_node, v->get_init_value()); - G.symtable.add_global_const(c_sym); + G.symtable.add_global_symbol(c_sym); G.all_constants.push_back(c_sym); v->mutate()->assign_const_ref(c_sym); return c_sym; @@ -107,7 +107,7 @@ static GlobalConstPtr register_constant(V v) { static GlobalVarPtr register_global_var(V v) { GlobalVarData* g_sym = new GlobalVarData(static_cast(v->get_identifier()->name), v->loc, v->type_node); - G.symtable.add_global_var(g_sym); + G.symtable.add_global_symbol(g_sym); G.all_global_vars.push_back(g_sym); v->mutate()->assign_glob_ref(g_sym); return g_sym; @@ -122,11 +122,36 @@ static AliasDefPtr register_type_alias(V v, AliasDef AliasDefData* a_sym = new AliasDefData(std::move(name), v->loc, v->underlying_type_node, genericTs, substitutedTs, v); a_sym->base_alias_ref = base_alias_ref; // for `Response`, here is `Response` - G.symtable.add_type_alias(a_sym); + G.symtable.add_global_symbol(a_sym); v->mutate()->assign_alias_ref(a_sym); return a_sym; } +static EnumDefPtr register_enum(V v) { + auto v_body = v->get_enum_body(); + + std::vector members; + members.reserve(v_body->get_num_members()); + for (int i = 0; i < v_body->get_num_members(); ++i) { + auto v_member = v_body->get_member(i); + std::string member_name = static_cast(v_member->get_identifier()->name); + + for (EnumMemberPtr prev : members) { + if (UNLIKELY(prev->name == member_name)) { + v_member->error("redeclaration of member `" + member_name + "`"); + } + } + members.emplace_back(new EnumMemberData(member_name, v_member->loc, v_member->init_value)); + } + + EnumDefData* e_sym = new EnumDefData(static_cast(v->get_identifier()->name), v->loc, std::move(members)); + + G.symtable.add_global_symbol(e_sym); + G.all_enums.push_back(e_sym); + v->mutate()->assign_enum_ref(e_sym); + return e_sym; +} + static StructPtr register_struct(V v, StructPtr base_struct_ref = nullptr, std::string override_name = {}, const GenericsSubstitutions* substitutedTs = nullptr) { auto v_body = v->get_struct_body(); @@ -170,7 +195,7 @@ static StructPtr register_struct(V v, StructPtr base_str StructData* s_sym = new StructData(std::move(name), v->loc, std::move(fields), opcode, v->overflow1023_policy, genericTs, substitutedTs, v); s_sym->base_struct_ref = base_struct_ref; // for `Container`, here is `Container` - G.symtable.add_struct(s_sym); + G.symtable.add_global_symbol(s_sym); G.all_structs.push_back(s_sym); v->mutate()->assign_struct_ref(s_sym); return s_sym; @@ -284,6 +309,9 @@ static void iterate_through_file_symbols(const SrcFile* file) { case ast_type_alias_declaration: register_type_alias(v->as()); break; + case ast_enum_declaration: + register_enum(v->as()); + break; case ast_struct_declaration: register_struct(v->as()); break; diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index e0f448f2d1..320ebf70c5 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -166,7 +166,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { if (!sym) { fire_error_undefined_symbol(cur_f, v->get_identifier()); } - if (sym->try_as() || sym->try_as()) { + if (sym->try_as() || sym->try_as() || sym->try_as()) { fire_error_type_used_as_symbol(cur_f, v->get_identifier()); } v->mutate()->assign_sym(sym); @@ -184,7 +184,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { parent::visit(v->get_obj()); } catch (const ParseError&) { if (auto v_type_name = v->get_obj()->try_as()) { - // for `Point.create` / `int.zero`, "undefined symbol" is fired for Point/int + // for `Point.create` / `int.zero` / `Color.Red`, "undefined symbol" is fired for Point/int/Color // suppress this exception till a later pipe, it will be tried to be resolved as a type if (v_type_name->get_identifier()->name == "self") { fire_error_using_self_not_in_method(cur_f, v_type_name->loc); @@ -298,6 +298,15 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { } } } + + void start_visiting_enum_members(EnumDefPtr enum_ref) { + // member `Red = Another.Blue`, resolve `Another` + for (EnumMemberPtr member_ref : enum_ref->members) { + if (member_ref->has_init_value()) { + parent::visit(member_ref->init_value); + } + } + } }; NameAndScopeResolver AssignSymInsideFunctionVisitor::current_scope; @@ -320,6 +329,10 @@ void pipeline_resolve_identifiers_and_assign_symbols() { } else if (auto v_struct = v->try_as()) { tolk_assert(v_struct->struct_ref); visitor.start_visiting_struct_fields(v_struct->struct_ref); + + } else if (auto v_enum = v->try_as()) { + tolk_assert(v_enum->enum_ref); + visitor.start_visiting_enum_members(v_enum->enum_ref); } } } diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 67f5dd0c7d..994c1e96a9 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -271,6 +271,9 @@ class TypeNodesVisitorResolver { } return TypeDataStruct::create(struct_ref); } + if (EnumDefPtr enum_ref = sym->try_as()) { + return TypeDataEnum::create(enum_ref); + } return nullptr; } @@ -592,6 +595,17 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { } } } + + void start_visiting_enum_members(EnumDefPtr enum_ref) { + type_nodes_visitor = TypeNodesVisitorResolver(nullptr, nullptr, nullptr, false); + + // same for struct field `v: int8 = 0 as int8` + for (EnumMemberPtr member_ref : enum_ref->members) { + if (member_ref->has_init_value()) { + parent::visit(member_ref->init_value); + } + } + } }; // prevent recursion like `struct A { field: A }`; @@ -664,6 +678,9 @@ void pipeline_resolve_types_and_aliases() { TypeNodesVisitorResolver::visit_symbol(v_struct->struct_ref); } visitor.start_visiting_struct_fields(v_struct->struct_ref); + + } else if (auto v_enum = v->try_as()) { + visitor.start_visiting_enum_members(v_enum->enum_ref); } } } diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index b3d9516482..07db40a429 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -55,6 +55,10 @@ std::string StructData::as_human_readable() const { return name + genericTs->as_human_readable(); } +std::string EnumDefData::as_human_readable() const { + return name; +} + LocalVarPtr FunctionData::find_param(std::string_view name) const { for (const LocalVarData& param_data : parameters) { if (param_data.name == name) { @@ -189,6 +193,14 @@ void AliasDefData::assign_resolved_type(TypePtr underlying_type) { this->underlying_type = underlying_type; } +void EnumMemberData::assign_init_value(AnyExprV init_value) { + this->init_value = init_value; +} + +void EnumMemberData::assign_computed_value(td::RefInt256 computed_value) { + this->computed_value = std::move(computed_value); +} + void StructFieldData::assign_resolved_type(TypePtr declared_type) { this->declared_type = declared_type; } @@ -230,6 +242,15 @@ std::string StructData::PackOpcode::format_as_slice() const { return result; } +EnumMemberPtr EnumDefData::find_member(std::string_view member_name) const { + for (EnumMemberPtr member_ref : members) { + if (member_ref->name == member_name) { + return member_ref; + } + } + return nullptr; +} + GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_redefinition_of_symbol(SrcLocation loc, const Symbol* previous) { SrcLocation prev_loc = previous->loc; @@ -242,52 +263,14 @@ static void fire_error_redefinition_of_symbol(SrcLocation loc, const Symbol* pre throw ParseError(loc, "redefinition of built-in symbol"); } -void GlobalSymbolTable::add_function(FunctionPtr f_sym) { - auto key = key_hash(f_sym->name); - auto [it, inserted] = entries.emplace(key, f_sym); - if (!inserted) { - fire_error_redefinition_of_symbol(f_sym->loc, it->second); - } -} - -void GlobalSymbolTable::add_global_var(GlobalVarPtr g_sym) { - auto key = key_hash(g_sym->name); - auto [it, inserted] = entries.emplace(key, g_sym); - if (!inserted) { - fire_error_redefinition_of_symbol(g_sym->loc, it->second); - } -} - -void GlobalSymbolTable::add_global_const(GlobalConstPtr c_sym) { - auto key = key_hash(c_sym->name); - auto [it, inserted] = entries.emplace(key, c_sym); +void GlobalSymbolTable::add_global_symbol(const Symbol* sym) { + auto key = key_hash(sym->name); + auto [it, inserted] = entries.emplace(key, sym); if (!inserted) { - fire_error_redefinition_of_symbol(c_sym->loc, it->second); + fire_error_redefinition_of_symbol(sym->loc, it->second); } } -void GlobalSymbolTable::add_type_alias(AliasDefPtr a_sym) { - auto key = key_hash(a_sym->name); - auto [it, inserted] = entries.emplace(key, a_sym); - if (!inserted) { - fire_error_redefinition_of_symbol(a_sym->loc, it->second); - } -} - -void GlobalSymbolTable::add_struct(StructPtr s_sym) { - auto key = key_hash(s_sym->name); - auto [it, inserted] = entries.emplace(key, s_sym); - if (!inserted) { - fire_error_redefinition_of_symbol(s_sym->loc, it->second); - } -} - -void GlobalSymbolTable::replace_function(FunctionPtr f_sym) { - auto key = key_hash(f_sym->name); - assert(entries.contains(key)); - entries[key] = f_sym; -} - const Symbol* lookup_global_symbol(std::string_view name) { return G.symtable.lookup(name); } diff --git a/tolk/symtable.h b/tolk/symtable.h index 32292b0b74..72a2131169 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -368,6 +368,35 @@ struct StructData final : Symbol { std::string as_human_readable() const; }; +struct EnumMemberData final : Symbol { + AnyExprV init_value; // nullptr if no init (`Red`, not `Red = 1`) + td::RefInt256 computed_value; // auto-calculated or assigned from init if integer + + bool has_init_value() const { return init_value != nullptr; } + + EnumMemberData(std::string name, SrcLocation loc, AnyExprV init_value) + : Symbol(std::move(name), loc) + , init_value(init_value) { + } + + EnumMemberData* mutate() const { return const_cast(this); } + void assign_init_value(AnyExprV init_value); + void assign_computed_value(td::RefInt256 computed_value); +}; + +struct EnumDefData final : Symbol { + std::vector members; + + EnumMemberPtr find_member(std::string_view member_name) const; + + EnumDefData(std::string name, SrcLocation loc, std::vector&& members) + : Symbol(std::move(name), loc) + , members(std::move(members)) { + } + + std::string as_human_readable() const; +}; + struct TypeReferenceUsedAsSymbol final : Symbol { TypePtr resolved_type; @@ -385,13 +414,8 @@ class GlobalSymbolTable { } public: - void add_function(FunctionPtr f_sym); - void add_global_var(GlobalVarPtr g_sym); - void add_global_const(GlobalConstPtr c_sym); - void add_type_alias(AliasDefPtr a_sym); - void add_struct(StructPtr s_sym); - - void replace_function(FunctionPtr f_sym); + void add_global_symbol(const Symbol* sym); + void add_function(FunctionPtr f_sym) { add_global_symbol(f_sym); } const Symbol* lookup(std::string_view name) const { const auto it = entries.find(key_hash(name)); diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index edd422400c..ef1d43ac40 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -184,6 +184,10 @@ TypePtr TypeDataStruct::create(StructPtr struct_ref) { return new TypeDataStruct(struct_ref); } +TypePtr TypeDataEnum::create(EnumDefPtr enum_ref) { + return new TypeDataEnum(enum_ref); +} + TypePtr TypeDataTensor::create(std::vector&& items) { CalcChildrenFlags reg; reg.feed_child(items); @@ -338,6 +342,10 @@ int TypeDataStruct::get_type_id() const { return TypeIdCalculation::assign_type_id(this); } +int TypeDataEnum::get_type_id() const { + return TypeIdCalculation::assign_type_id(this); +} + int TypeDataTensor::get_type_id() const { assert(!has_genericT_inside()); return TypeIdCalculation::assign_type_id(this); @@ -421,6 +429,10 @@ std::string TypeDataStruct::as_human_readable() const { return struct_ref->name; } +std::string TypeDataEnum::as_human_readable() const { + return enum_ref->name; +} + std::string TypeDataTensor::as_human_readable() const { std::string result = "("; for (TypePtr item : items) { @@ -705,6 +717,16 @@ bool TypeDataStruct::can_rhs_be_assigned(TypePtr rhs) const { return rhs == TypeDataNever::create(); } +bool TypeDataEnum::can_rhs_be_assigned(TypePtr rhs) const { + if (const TypeDataEnum* rhs_enum = rhs->try_as()) { + return enum_ref == rhs_enum->enum_ref; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } + return rhs == TypeDataNever::create(); +} + bool TypeDataTensor::can_rhs_be_assigned(TypePtr rhs) const { if (const auto* as_tensor = rhs->try_as(); as_tensor && as_tensor->size() == size()) { for (int i = 0; i < size(); ++i) { @@ -848,6 +870,9 @@ bool TypeDataInt::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to == TypeDataCoins::create()) { // `int` as `coins` return true; } + if (cast_to->try_as()) { // `int` as `Color` (all enums are integer) + return true; + } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } @@ -1001,6 +1026,22 @@ bool TypeDataStruct::can_be_casted_with_as_operator(TypePtr cast_to) const { return false; } +bool TypeDataEnum::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to == TypeDataInt::create()) { + return true; + } + if (cast_to->try_as()) { + return true; // all enums are integers, they can be `as` cast to each other + } + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + return false; +} + bool TypeDataTensor::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_tensor = cast_to->try_as(); to_tensor && to_tensor->size() == size()) { for (int i = 0; i < size(); ++i) { @@ -1252,6 +1293,16 @@ bool TypeDataStruct::equal_to(TypePtr rhs) const { return false; } +bool TypeDataEnum::equal_to(TypePtr rhs) const { + if (const TypeDataEnum* rhs_enum = rhs->try_as()) { + return enum_ref == rhs_enum->enum_ref; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + bool TypeDataTensor::equal_to(TypePtr rhs) const { if (const TypeDataTensor* rhs_tensor = rhs->try_as(); rhs_tensor && size() == rhs_tensor->size()) { for (int i = 0; i < size(); ++i) { diff --git a/tolk/type-system.h b/tolk/type-system.h index a6da3d8fa2..be043a643e 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -407,6 +407,27 @@ class TypeDataStruct final : public TypeData { bool equal_to(TypePtr rhs) const override; }; +/* + * `Color.Red`, `BounceMode.NoBounce` is TypeDataEnum. At TVM level, it's `int`. + * Its value is either assigned like `Red = 1` or auto-calculated. + */ +class TypeDataEnum final : public TypeData { + explicit TypeDataEnum(EnumDefPtr enum_ref) + : TypeData(0) + , enum_ref(enum_ref) {} + +public: + EnumDefPtr enum_ref; + + static TypePtr create(EnumDefPtr enum_ref); + + int get_type_id() const override; + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + bool equal_to(TypePtr rhs) const override; +}; + /* * `(int, slice)` is TypeDataTensor of 2 elements. Tensor of N elements occupies N stack slots. * Of course, there may be nested tensors, like `(int, (int, slice), cell)`. From 0fb8c1ca006c2a968e58fb2ce76f0c967c2db493 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 18 Aug 2025 16:34:50 +0700 Subject: [PATCH 397/524] [Tolk] Support serialization for enums --- .../tests/invalid-declaration/err-1480.tolk | 8 + tolk-tester/tests/maps-tests.tolk | 22 ++ tolk-tester/tests/pack-unpack-2.tolk | 21 +- tolk-tester/tests/pack-unpack-5.tolk | 18 ++ tolk-tester/tests/pack-unpack-7.tolk | 27 +++ tolk-tester/tests/pack-unpack-8.tolk | 215 ++++++++++++++++++ tolk/ast-from-tokens.cpp | 8 +- tolk/ast.h | 7 +- tolk/pack-unpack-api.cpp | 7 + tolk/pack-unpack-serializers.cpp | 111 +++++++++ tolk/pack-unpack-serializers.h | 1 + tolk/pipe-check-serialized-fields.cpp | 12 + tolk/pipe-register-symbols.cpp | 2 +- tolk/pipe-resolve-types.cpp | 6 + tolk/symtable.cpp | 4 + tolk/symtable.h | 8 +- 16 files changed, 471 insertions(+), 6 deletions(-) create mode 100644 tolk-tester/tests/invalid-declaration/err-1480.tolk create mode 100644 tolk-tester/tests/pack-unpack-8.tolk diff --git a/tolk-tester/tests/invalid-declaration/err-1480.tolk b/tolk-tester/tests/invalid-declaration/err-1480.tolk new file mode 100644 index 0000000000..667c5af2b9 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1480.tolk @@ -0,0 +1,8 @@ + +enum C3 : int { +} + +/** +@compilation_should_fail +@stderr serialization type of `enum` must be intN: `int8` / `uint32` / etc. + */ diff --git a/tolk-tester/tests/maps-tests.tolk b/tolk-tester/tests/maps-tests.tolk index 5a0830b32d..43255a97b7 100644 --- a/tolk-tester/tests/maps-tests.tolk +++ b/tolk-tester/tests/maps-tests.tolk @@ -32,6 +32,12 @@ struct DataWithCell { payload: cell? = null } +enum HashMode: uint16 { + NoHash = 0, + PartialHash = 80, + TotalHash = 255, +} + fun map.calculateSize(self) { var size = 0; var r = self.findFirst(); @@ -500,6 +506,21 @@ fun test40() { ); } +@method_id(141) +fun test41() { + var m = createEmptyMap(); + m.set(-7, HashMode.NoHash); + m.addIfNotExists(111, HashMode.TotalHash); + m.replaceIfExists(-7, HashMode.PartialHash); + + val k7 = m.get(-7); + return ( + m.exists(8), m.mustGet(111) == HashMode.TotalHash, + k7.isFound, k7.loadValue() == HashMode.NoHash, k7.loadValue() == HashMode.PartialHash + ); +} + + fun main() { __expect_type(createEmptyMap, "() -> map"); } @@ -546,6 +567,7 @@ fun main() { @testcase | 138 | 4 | 55 @testcase | 139 | | 0 -1 2 @testcase | 140 | | 0 0 -1 777 -1 0 777 (null) typeid-1 (null) 0 +@testcase | 141 | | 0 -1 -1 0 -1 @fif_codegen """ diff --git a/tolk-tester/tests/pack-unpack-2.tolk b/tolk-tester/tests/pack-unpack-2.tolk index da5fba0d33..dc122f2147 100644 --- a/tolk-tester/tests/pack-unpack-2.tolk +++ b/tolk-tester/tests/pack-unpack-2.tolk @@ -418,6 +418,15 @@ struct WithNullableMaps { m4: map | int32 } +enum EStoredAsInt8 { M100 = -100, Z = 0, P100 = 100 } +enum EStoredAsUint1 { ZERO, ONE } + +struct WithEnums { + e1: EStoredAsInt8 + e2: EStoredAsUint1 + rem: uint7 = 0 +} + // --------------------------------------------- @@ -749,7 +758,7 @@ fun test_serializeMaps() { } return (w.m1.isEmpty(), w.m2.isEmpty(), d3.isFound, d3.loadValue(), kl); } - + @method_id(231) fun test_nullableMaps() { var o: WithNullableMaps = { @@ -772,6 +781,15 @@ fun test_nullableMaps() { return (o.m1 == null, o.m1!.isEmpty(), o.m2 == null, o.m2!.isEmpty(), o.m3 == null, o.m4 is int32); } +@method_id(232) +fun test_SimpleEnums() { + run({e1: EStoredAsInt8.M100, e2: EStoredAsUint1.ZERO}, stringHexToSlice("9c00")); + run({e1: EStoredAsInt8.Z, e2: EStoredAsUint1.ZERO}, stringHexToSlice("0000")); + run({e1: EStoredAsInt8.P100, e2: EStoredAsUint1.ONE, rem: 1}, stringHexToSlice("6481")); + return -1; +} + + fun main() { var t: JustInt32 = { value: 10 }; var c = t.toCell(); @@ -814,4 +832,5 @@ fun main() { @testcase | 229 | | -1 @testcase | 230 | | 0 -1 -1 3 30 8 1 @testcase | 231 | | 0 -1 0 0 -1 0 +@testcase | 232 | | -1 */ diff --git a/tolk-tester/tests/pack-unpack-5.tolk b/tolk-tester/tests/pack-unpack-5.tolk index e643af6c31..05a9ee1e6c 100644 --- a/tolk-tester/tests/pack-unpack-5.tolk +++ b/tolk-tester/tests/pack-unpack-5.tolk @@ -205,6 +205,23 @@ fun test20() { CellInner8_1.getDeclaredPackPrefixLen(), CellInner8_1.getDeclaredPackPrefix()); } +enum Color { Red, Green, Blue } +enum EInt9 { P133 = 133, M1 = -1 } +enum EUint6 { P63 = 63, P30 = 30, P1 = 1 } +enum EUint1 { A, B } +enum EInt1 { A = -1, B } +enum EUint123 { Z, V5 = 5, E123 = 1<<122, E123_2 = (1<<122) + (1<<121) } +type EUint123Alias = EUint123 + +@method_id(121) +fun test21() { + return ( + Color.estimate(), EInt9.estimate(), EUint6.estimate(), + EUint1.estimate(), EInt1.estimate(), EInt1.estimate(), + EUint123.estimate(), EUint123Alias.estimate(), + ) +} + fun main() { __expect_type(int8.estimatePackSize(), "(int, int, int, int)"); } @@ -222,4 +239,5 @@ fun main() { @testcase | 110 | | [ 32 9999 0 4 ] [ 33 9999 0 8 ] @testcase | 111 | | [ 1023 1023 0 1 ] [ 0 0 2 2 ] @testcase | 120 | | 16 4128 1 1 +@testcase | 121 | | [ 2 2 0 0 ] [ 9 9 0 0 ] [ 6 6 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 123 123 0 0 ] [ 123 123 0 0 ] */ diff --git a/tolk-tester/tests/pack-unpack-7.tolk b/tolk-tester/tests/pack-unpack-7.tolk index 60e9c2b5cf..0a11b378a4 100644 --- a/tolk-tester/tests/pack-unpack-7.tolk +++ b/tolk-tester/tests/pack-unpack-7.tolk @@ -100,6 +100,22 @@ fun Tensor3Skipping1.packToBuilder(self, mutate b: builder) { b.storeUint(self.0, 8).storeUint(self.2, 8); } +enum Color { + Red, + Green, + Blue, +} + +type UnsafeColor = Color + +fun UnsafeColor.unpackFromSlice(mutate s: slice) { + return s.loadUint(2) as Color +} + +fun UnsafeColor.packToBuilder(self, mutate b: builder) { + b.storeUint(self as int, 2) +} + @method_id(101) fun test1() { @@ -155,6 +171,15 @@ fun test7() { return t.toCell().load(); } +@method_id(108) +fun test8(c: UnsafeColor) { + var way1 = beginCell().storeAny(c).endCell(); + var way2 = c.toCell(); + __expect_type(way2, "Cell"); + assert (way1.hash() == way2.hash()) throw 123; + return way2.load(); +} + fun main() { __expect_type("" as TelegramString, "TelegramString"); } @@ -169,6 +194,8 @@ fun main() { @testcase | 105 | | 123 1 @testcase | 106 | | 6 2 255 0 8 9 @testcase | 107 | | 1 0 3 +@testcase | 108 | 0 | 0 +@testcase | 108 | 3 | 3 @fif_codegen """ diff --git a/tolk-tester/tests/pack-unpack-8.tolk b/tolk-tester/tests/pack-unpack-8.tolk new file mode 100644 index 0000000000..a146996a13 --- /dev/null +++ b/tolk-tester/tests/pack-unpack-8.tolk @@ -0,0 +1,215 @@ +enum EFits2Bits { ZERO, ONE, TWO } + +fun EFits2Bits.sliceWithZero(): slice asm "b{00} PUSHSLICE" +fun EFits2Bits.sliceWithThree(): slice asm "b{11} PUSHSLICE" + +enum EStartFrom1 { + ONE = 1 + TWO + THREE +} + +enum EFillsAllUint3 { + p0, p1, p2, p3, p4, p5, p6, p7 +} + +enum EStartFromM2 { + M2 = -2, + M1, ZERO, P1, P2, P3 = 3, +} + +enum EFits8Bits { + E0 = 0; + E110 = 110; + E220 = 220; +} + +const MAX_UINT = 115792089237316195423570985008687907853269984665640564039457584007913129639935 + +enum EMinMax { + MIN_INT = -115792089237316195423570985008687907853269984665640564039457584007913129639935 - 1, + MAX_INT = +115792089237316195423570985008687907853269984665640564039457584007913129639935, +} + +enum E0Max { + ZERO, + MAX_INT = +115792089237316195423570985008687907853269984665640564039457584007913129639935, +} + +struct WithEnumsUnion { + u: EFits8Bits | EStartFromM2 +} + +enum Role: int8 { + Admin, User, +} + +fun (Role, Role).decode(mutate self, s: slice) { + self = s.loadAny() +} + +enum EncodedVari : varint16 { + ONE = 1, + TWO = 2, + MANY = 1<<100, +} + + +@method_id(101) +fun test1() { + var e = EFits2Bits.fromSlice(EFits2Bits.sliceWithZero()); + return (e == EFits2Bits.ZERO, e == EFits2Bits.TWO); +} + +@method_id(102) +fun test2(c: EFits2Bits) { + try { + return (c.toCell().beginParse().loadAny() == c) as int + } catch (ex) { + return ex * 10 + } +} + +@method_id(103) +fun test3() { + return EStartFrom1.fromCell(EStartFrom1.ONE.toCell()) +} + +@method_id(104) +fun test4(v: EFillsAllUint3) { + return EFillsAllUint3.fromCell(v.toCell()) +} + +@method_id(105) +fun test5(s: slice) { + return EStartFromM2.fromSlice(s, {assertEndAfterReading: false}) +} + +@method_id(106) +fun test6() { + try { + return (4 as EFits2Bits).toCell().beginParse().loadUint(1) + } catch (ex) { + return ex + } +} + +@method_id(107) +fun test7(v: EFits8Bits) { + try { return EFits8Bits.fromCell(v.toCell()) as int } + catch (ex) { return ex * 1000 } +} + +@method_id(108) +fun test8() { + var s_min = beginCell().storeInt(-MAX_UINT - 1, 257).endCell(); + var s_max = beginCell().storeInt(+MAX_UINT, 257).endCell(); + return ( + EMinMax.fromCell(s_min) == EMinMax.MIN_INT, EMinMax.fromCell(s_max) == EMinMax.MAX_INT, + EMinMax.fromCell(s_max) == EMinMax.MIN_INT, EMinMax.fromCell(s_min) == EMinMax.MAX_INT, + ) +} + +@method_id(109) +fun test9() { + var s_min = beginCell().storeUint(0, 256).endCell(); + var s_max = beginCell().storeUint(MAX_UINT, 256).endCell(); + return ( + E0Max.fromCell(s_min) == E0Max.ZERO, E0Max.fromCell(s_max) == E0Max.MAX_INT, + ) +} + +@method_id(110) +fun test10() { + var w1 = WithEnumsUnion{u: EFits8Bits.E110}.toCell().load(); + var w2 = WithEnumsUnion{u: (220 as EFits8Bits)}.toCell().load(); + var w3 = WithEnumsUnion{u: ([0] as tuple).get(0)}.toCell().load(); + var w4 = WithEnumsUnion{u: EStartFromM2.P3}.toCell().load(); + return ( + w1.u is EFits8Bits && w1.u == EFits8Bits.E110, + w2.u is EFits8Bits && w2.u == EFits8Bits.E220, + w3.u is EStartFromM2 && w3.u == EStartFromM2.ZERO, + w4.u is EStartFromM2 && w4.u == EStartFromM2.P3, + w4.u is EFits8Bits && w4.u == EFits8Bits.E220, + ) +} + +@method_id(111) +fun test11() { + var (r1, r2) = (-1 as Role, -1 as Role); + val before = (r1 == Role.Admin, r2 == Role.Admin); + (r1, r2).decode(stringHexToSlice("0001")); + return (before, r1, r2, r1 == Role.Admin, r2 == Role.Admin, r1 == Role.User, r2 == Role.User) +} + +@method_id(112) +fun test12() { + var c3_one = EncodedVari.fromSlice(stringHexToSlice("1018_")); + var c3_many = EncodedVari.fromSlice(stringHexToSlice("d100000000000000000000000008_")); + return (c3_one, c3_one == EncodedVari.MANY, c3_many == EncodedVari.MANY); +} + +fun main() { + +} + +/** +@testcase | 101 | | -1 0 +@testcase | 102 | 1 | -1 +@testcase | 102 | 3 | 50 +@testcase | 103 | | 1 +@testcase | 104 | 3 | 3 +@testcase | 104 | 7 | 7 +@testcase | 105 | x{d} | -2 +@testcase | 105 | x{7} | 3 +@testcase | 106 | | 5 +@testcase | 107 | 0 | 0 +@testcase | 107 | 110 | 110 +@testcase | 107 | 220 | 220 +@testcase | 107 | -50 | 5000 +@testcase | 107 | 1 | 5000 +@testcase | 107 | 222 | 5000 +@testcase | 108 | | -1 -1 0 0 +@testcase | 109 | | -1 -1 +@testcase | 110 | | -1 -1 -1 -1 0 +@testcase | 111 | | 0 0 0 1 -1 0 0 -1 +@testcase | 112 | | 1 0 -1 + +@fif_codegen +""" + test1() PROC:<{ + b{00} PUSHSLICE + 2 LDU + OVER + 2 GTINT + 5 THROWIF + ENDS +""" + +@fif_codegen +""" + test4() PROC:<{ // v + NEWC // v b + 3 STU // b + ENDC // '4 + CTOS // s + 3 LDU // '9 s + ENDS // '9 + }> +""" + +@fif_codegen +""" + test5() PROC:<{ + // s + 3 PLDI + DUP + -2 LESSINT + 5 THROWIF + DUP + 3 GTINT + 5 THROWIF + }> +""" + + */ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 86de235293..cf164ca3c2 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -1664,6 +1664,12 @@ static AnyV parse_enum_declaration(Lexer& lex, const std::vector(lex.cur_location(), lex.cur_str()); lex.next(); + AnyTypeV colon_type = nullptr; + if (lex.tok() == tok_colon) { // enum Role: int8 + lex.next(); + colon_type = parse_type_expression(lex); + } + for (auto v_annotation : annotations) { switch (v_annotation->kind) { case AnnotationKind::deprecated: @@ -1674,7 +1680,7 @@ static AnyV parse_enum_declaration(Lexer& lex, const std::vector(loc, v_ident, parse_enum_body(lex)); + return createV(loc, v_ident, colon_type, parse_enum_body(lex)); } static AnyV parse_tolk_required_version(Lexer& lex) { diff --git a/tolk/ast.h b/tolk/ast.h index b802f847bc..2d7ff884bd 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -1426,8 +1426,10 @@ struct Vertex final : ASTOtherVararg { template<> // ast_enum_declaration is a declaring a `enum` similar to TypeScript (not to Rust, we don't need structural enums, we have unions) // example: `enum Color { Red, Green, Blue }` +// example: `enum Role: int8 { Admin, User }` struct Vertex final : ASTOtherVararg { EnumDefPtr enum_ref = nullptr; // filled after register + AnyTypeV colon_type = nullptr; // serialization type after `:` if exists auto get_identifier() const { return children.at(0)->as(); } auto get_enum_body() const { return children.at(1)->as(); } @@ -1435,8 +1437,9 @@ struct Vertex final : ASTOtherVararg { Vertex* mutate() const { return const_cast(this); } void assign_enum_ref(EnumDefPtr enum_ref); - Vertex(SrcLocation loc, V name_identifier, V enum_body) - : ASTOtherVararg(ast_enum_declaration, loc, {name_identifier, enum_body}) {} + Vertex(SrcLocation loc, V name_identifier, AnyTypeV colon_type, V enum_body) + : ASTOtherVararg(ast_enum_declaration, loc, {name_identifier, enum_body}) + , colon_type(colon_type) {} }; template<> diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index 2b6c2d2145..f710d2e30a 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -121,6 +121,13 @@ class PackUnpackAvailabilityChecker { return {}; } + if (const auto* t_enum = any_type->try_as()) { + if (t_enum->enum_ref->members.empty()) { + return CantSerializeBecause("because `enum` is empty"); + } + return {}; + } + if (const auto* t_union = any_type->try_as()) { // a union can almost always be serialized if every of its variants can: // - `T?` is TL/B `(Maybe T)` diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index e65679712f..d000bfb6fe 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -1056,6 +1056,85 @@ struct S_CustomStruct final : ISerializer { } }; +struct S_IntegerEnum final : ISerializer { + EnumDefPtr enum_ref; + + explicit S_IntegerEnum(EnumDefPtr enum_ref) + : enum_ref(enum_ref) { + } + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + TypePtr intN = calculate_intN_to_serialize_enum(enum_ref); + return ctx->generate_pack_any(intN, std::move(rvect)); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + const TypeDataIntN* intN = calculate_intN_to_serialize_enum(enum_ref)->try_as(); + std::vector ir_num = ctx->generate_unpack_any(intN); + + // when reading an integer value, we need to validate that it's a valid enum member; + // at first, detect whether it's a sequence (A, A+1, ..., A+N) + bool is_sequence = true; + td::RefInt256 expected_cur = enum_ref->members.front()->computed_value; + for (EnumMemberPtr member_ref : enum_ref->members) { + is_sequence &= td::cmp(member_ref->computed_value, expected_cur) == 0; + expected_cur += 1; + } + + if (is_sequence) { + // enum's members are A...B one by one (probably, 0...M); + // then validation is: "throw if vB", but "LESSINT + THROWIF" 2 times is more generalized + td::RefInt256 min_value = enum_ref->members.front()->computed_value; + bool dont_check_min = intN->is_unsigned && min_value == 0; + if (!dont_check_min) { // LDU can't load < 0 + std::vector ir_min_value = code.create_tmp_var(TypeDataInt::create(), loc, "(enum-min)"); + code.emplace_back(loc, Op::_IntConst, ir_min_value, min_value); + std::vector ir_lt_min = code.create_tmp_var(TypeDataInt::create(), loc, "(enum-lt-min)"); + code.emplace_back(loc, Op::_Call, ir_lt_min, std::vector{ir_num[0], ir_min_value[0]}, lookup_function("_<_")); + std::vector args_assert1 = { code.create_int(loc, 5, "(excno)"), ir_lt_min[0], code.create_int(loc, 1, "") }; + Op& op_assert1 = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert1), lookup_function("__throw_if_unless")); + op_assert1.set_impure_flag(); + } + td::RefInt256 max_value = enum_ref->members.back()->computed_value; + bool dont_check_max = intN->is_unsigned && max_value == (1ULL << intN->n_bits) - 1; + if (!dont_check_max) { // LDU can't load >= 1<_")); + std::vector args_assert2 = { code.create_int(loc, 5, "(excno)"), ir_gt_max[0], code.create_int(loc, 1, "") }; + Op& op_assert2 = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert2), lookup_function("__throw_if_unless")); + op_assert2.set_impure_flag(); + } + } else { + // okay, enum is not a sequence, just a set of values; + // then validation is: "throw if v is not contained in V", check v==V_i and combine with OR + var_idx_t ir_any_of = code.create_int(loc, 0, "(any-of-equals)"); + for (EnumMemberPtr member_ref : enum_ref->members) { + std::vector ir_ith_value = code.create_tmp_var(TypeDataInt::create(), loc, "(enum-ith)"); + code.emplace_back(loc, Op::_IntConst, ir_ith_value, member_ref->computed_value); + std::vector ir_ith_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(enum-ith-eq)"); + code.emplace_back(loc, Op::_Call, ir_ith_eq, std::vector{ir_num[0], ir_ith_value[0]}, lookup_function("_==_")); + code.emplace_back(loc, Op::_Call, std::vector{ir_any_of}, std::vector{ir_any_of, ir_ith_eq[0]}, lookup_function("_|_")); + } + std::vector args_assert = { code.create_int(loc, 5, "(excno)"), ir_any_of, code.create_int(loc, 0, "") }; + Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); + op_assert.set_impure_flag(); + } + return ir_num; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + TypePtr intN = calculate_intN_to_serialize_enum(enum_ref); + return ctx->generate_skip_any(intN); + } + + PackSize estimate(const EstimateContext* ctx) override { + TypePtr intN = calculate_intN_to_serialize_enum(enum_ref); + return ctx->estimate_any(intN); + } +}; + struct S_CustomReceiverForPackUnpack final : ISerializer { TypePtr receiver_type; @@ -1163,6 +1242,35 @@ std::vector auto_generate_opcodes_for_union(TypePtr union_type, std: return result; } +// given a `enum`, calculate N bits enough to store all values; +// example: `enum Color { Red, Green, Blue }` is 00/01/10 - uint2 +// example: `enum Role: int8 { ... }` — manually specified +TypePtr calculate_intN_to_serialize_enum(EnumDefPtr enum_ref) { + if (enum_ref->colon_type) { + return enum_ref->colon_type; + } + + bool is_unsigned = false; + int n_bits = 1; + for (; n_bits <= 256; ++n_bits) { + bool fits_unsigned = std::all_of(enum_ref->members.begin(), enum_ref->members.end(), [n_bits](EnumMemberPtr member_ref) { + return member_ref->computed_value->unsigned_fits_bits(n_bits); + }); + if (fits_unsigned) { + is_unsigned = true; + break; + } + bool fits_signed = std::all_of(enum_ref->members.begin(), enum_ref->members.end(), [n_bits](EnumMemberPtr member_ref) { + return member_ref->computed_value->signed_fits_bits(n_bits); + }); + if (fits_signed) { + break; + } + } + + return TypeDataIntN::create(n_bits, is_unsigned, false); +} + // there is no way to pass custom pack options to createMessage / map.set / etc., using hardcoded ones std::vector create_default_PackOptions(CodeBlob& code, SrcLocation loc) { StructPtr s_PackOptions = lookup_global_symbol("PackOptions")->try_as(); @@ -1240,6 +1348,9 @@ static std::unique_ptr get_serializer_for_type(TypePtr any_type) { if (const auto* t_struct = any_type->try_as()) { return std::make_unique(t_struct->struct_ref); } + if (const auto* t_enum = any_type->try_as()) { + return std::make_unique(t_enum->enum_ref); + } if (const auto* t_union = any_type->try_as()) { // `T?` is always `(Maybe T)`, even if T has custom opcode (opcode will follow bit '1') diff --git a/tolk/pack-unpack-serializers.h b/tolk/pack-unpack-serializers.h index 0f00bb0468..a511f751e6 100644 --- a/tolk/pack-unpack-serializers.h +++ b/tolk/pack-unpack-serializers.h @@ -165,6 +165,7 @@ class EstimateContext { bool is_type_cellT(TypePtr any_type); FunctionPtr get_custom_pack_unpack_function(TypePtr receiver_type, bool is_pack); std::vector auto_generate_opcodes_for_union(TypePtr union_type, std::string& because_msg); +TypePtr calculate_intN_to_serialize_enum(EnumDefPtr enum_ref); std::vector create_default_PackOptions(CodeBlob& code, SrcLocation loc); std::vector create_default_UnpackOptions(CodeBlob& code, SrcLocation loc); diff --git a/tolk/pipe-check-serialized-fields.cpp b/tolk/pipe-check-serialized-fields.cpp index 762c29bacc..792a73d293 100644 --- a/tolk/pipe-check-serialized-fields.cpp +++ b/tolk/pipe-check-serialized-fields.cpp @@ -73,6 +73,13 @@ static void check_mapKV_inside_type(AnyTypeV type_node) { } } +// given `enum Role: int8` check colon type (not struct/slice etc.) +static void check_enum_colon_type_to_be_intN(AnyTypeV colon_type_node) { + if (!colon_type_node->resolved_type->try_as()) { + fire(nullptr, colon_type_node->loc, "serialization type of `enum` must be intN: `int8` / `uint32` / etc."); + } +} + class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody { FunctionPtr cur_f = nullptr; @@ -168,6 +175,11 @@ void pipeline_check_serialized_fields() { for (GlobalVarPtr glob_ref : get_all_declared_global_vars()) { check_mapKV_inside_type(glob_ref->type_node); } + for (EnumDefPtr enum_ref : get_all_declared_enums()) { + if (enum_ref->colon_type_node) { + check_enum_colon_type_to_be_intN(enum_ref->colon_type_node); + } + } } } // namespace tolk diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index feb5c42524..ee7d96ca0b 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -144,7 +144,7 @@ static EnumDefPtr register_enum(V v) { members.emplace_back(new EnumMemberData(member_name, v_member->loc, v_member->init_value)); } - EnumDefData* e_sym = new EnumDefData(static_cast(v->get_identifier()->name), v->loc, std::move(members)); + EnumDefData* e_sym = new EnumDefData(static_cast(v->get_identifier()->name), v->loc, v->colon_type, std::move(members)); G.symtable.add_global_symbol(e_sym); G.all_enums.push_back(e_sym); diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 994c1e96a9..8913403b13 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -605,6 +605,12 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { parent::visit(member_ref->init_value); } } + + // serialization type: `enum Role: int8` + if (enum_ref->colon_type_node) { + TypePtr colon_type = finalize_type_node(enum_ref->colon_type_node); + enum_ref->mutate()->assign_resolved_colon_type(colon_type); // later it will be checked to be intN + } } }; diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 07db40a429..20b9617b44 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -215,6 +215,10 @@ void StructData::assign_resolved_genericTs(const GenericsDeclaration* genericTs) } } +void EnumDefData::assign_resolved_colon_type(TypePtr colon_type) { + this->colon_type = colon_type; +} + StructFieldPtr StructData::find_field(std::string_view field_name) const { for (StructFieldPtr field : fields) { if (field->name == field_name) { diff --git a/tolk/symtable.h b/tolk/symtable.h index 72a2131169..c49aa6e56b 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -385,16 +385,22 @@ struct EnumMemberData final : Symbol { }; struct EnumDefData final : Symbol { + AnyTypeV colon_type_node; // nullptr if no serialization type after `:` + TypePtr colon_type = nullptr; // = resolved colon_type_node std::vector members; EnumMemberPtr find_member(std::string_view member_name) const; - EnumDefData(std::string name, SrcLocation loc, std::vector&& members) + EnumDefData(std::string name, SrcLocation loc, AnyTypeV colon_type_node, std::vector&& members) : Symbol(std::move(name), loc) + , colon_type_node(colon_type_node) , members(std::move(members)) { } std::string as_human_readable() const; + + EnumDefData* mutate() const { return const_cast(this); } + void assign_resolved_colon_type(TypePtr colon_type); }; struct TypeReferenceUsedAsSymbol final : Symbol { From b175341067d320b04a04a248f27ed118ce6db79c Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 28 Aug 2025 23:57:07 +0700 Subject: [PATCH 398/524] [Tolk] Fix parsing sequential `as` operator --- tolk-tester/tests/match-by-expr-tests.tolk | 2 +- tolk-tester/tests/parse-address.tolk | 6 ++++-- tolk/ast-from-tokens.cpp | 23 +++++++++++----------- tolk/pipe-ast-to-legacy.cpp | 10 ++++++++++ tolk/type-system.cpp | 2 +- 5 files changed, 28 insertions(+), 15 deletions(-) diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk index 201c19f893..588632051d 100644 --- a/tolk-tester/tests/match-by-expr-tests.tolk +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -81,7 +81,7 @@ fun test7(x: int8) { @method_id(108) fun test8(x: int) { - var r1 = match (x is int) { false => 0, else => 10 }; + var r1 = match (x is int && x is int is bool) { false => 0, else => 10 }; match (x) { } match (x * 0) { 1 => {} else => { r1 += 20; } } return r1; diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index 05b95f4628..b4613655ca 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -142,12 +142,14 @@ fun main() { (cc1 as slice).bitsEqual(((cc2 as slice) as address) as slice), createAddressNone() == cc1, createAddressNone() == address.createNone(), - check0(address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff")) + check0(address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff")), + cc1 == cc1 as bits267 as address, + cc1 == cc1 as slice as bits999 as address, ); } /** -@testcase | 0 | | -1 0 -1 0 -1 +@testcase | 0 | | -1 0 -1 0 -1 -1 -1 @fif_codegen """ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index cf164ca3c2..ec783e2758 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -951,23 +951,24 @@ static AnyExprV parse_expr75(Lexer& lex) { return parse_expr80(lex); } -// parse E as / is / !is +// parse E as / is / !is (left-to-right) static AnyExprV parse_expr40(Lexer& lex) { AnyExprV lhs = parse_expr75(lex); - if (lex.tok() == tok_as) { - SrcLocation loc = lex.cur_location(); - lex.next(); - AnyTypeV cast_to_type = parse_type_from_tokens(lex); - lhs = createV(loc, lhs, cast_to_type); - } else if (lex.tok() == tok_is) { + TokenType t = lex.tok(); + while (t == tok_as || t == tok_is) { SrcLocation loc = lex.cur_location(); lex.next(); AnyTypeV rhs_type = parse_type_from_tokens(lex); - bool is_negated = lhs->kind == ast_not_null_operator; // `a !is ...`, now lhs = `a!` - if (is_negated) { - lhs = lhs->as()->get_expr(); + if (t == tok_as) { + lhs = createV(loc, lhs, rhs_type); + } else { + bool is_negated = lhs->kind == ast_not_null_operator; // `a !is ...`, now lhs = `a!` + if (is_negated) { + lhs = lhs->as()->get_expr(); + } + lhs = createV(loc, lhs, rhs_type, is_negated); } - lhs = createV(loc, lhs, rhs_type, is_negated); + t = lex.tok(); } return lhs; } diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 965a032a63..568f4868d8 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1009,6 +1009,16 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as()) { + return rvect; + } + + // pass `address` to `bits267` + if (original_type == TypeDataAddress::create() && target_type->try_as()) { + return rvect; + } + // pass a typed tuple `[int, int]` to an untyped (via `as` operator) if (original_type->try_as() && target_type->try_as()) { return rvect; diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index ef1d43ac40..ba796defd6 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -955,7 +955,7 @@ bool TypeDataContinuation::can_be_casted_with_as_operator(TypePtr cast_to) const } bool TypeDataAddress::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (cast_to == TypeDataSlice::create()) { + if (cast_to == TypeDataSlice::create() || cast_to->try_as()) { return true; } if (const TypeDataUnion* to_union = cast_to->try_as()) { From a0e831644f9063f6f610b7586b9e9d6562dbb54d Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 1 Sep 2025 14:06:14 +0300 Subject: [PATCH 399/524] [Tolk] Readonly fields in structs --- tolk-tester/tests/assignment-tests.tolk | 28 +++++++++++++++++++ .../tests/invalid-semantics/err-4130.tolk | 22 +++++++++++++++ .../tests/invalid-semantics/err-4131.tolk | 12 ++++++++ .../tests/invalid-semantics/err-4132.tolk | 15 ++++++++++ .../tests/invalid-semantics/err-4572.tolk | 4 ++- .../tests/invalid-semantics/err-4822.tolk | 6 ++-- tolk/ast-from-tokens.cpp | 9 +++++- tolk/ast-replicator.h | 2 +- tolk/ast.h | 5 ++-- tolk/lexer.cpp | 1 + tolk/lexer.h | 1 + tolk/pipe-check-rvalue-lvalue.cpp | 20 ++++++++++++- tolk/pipe-lazy-load-insertions.cpp | 2 +- tolk/pipe-register-symbols.cpp | 2 +- tolk/symtable.h | 4 ++- 15 files changed, 121 insertions(+), 12 deletions(-) create mode 100644 tolk-tester/tests/invalid-semantics/err-4130.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4131.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4132.tolk diff --git a/tolk-tester/tests/assignment-tests.tolk b/tolk-tester/tests/assignment-tests.tolk index 0343bdf507..377f03a92a 100644 --- a/tolk-tester/tests/assignment-tests.tolk +++ b/tolk-tester/tests/assignment-tests.tolk @@ -257,6 +257,32 @@ fun test119(a: int, b: int) { return (l, c, d); } +struct Point { + x: int + y: int +} + +struct ROLine { + readonly from: Point + readonly to: Point +} + +fun eq(v: T) { return v } + +@method_id(120) +fun test120() { + val p: Point = { x: 10, y: 20 }; + eq(p).x = 30; + return p; +} + +@method_id(121) +fun test121(l: ROLine) { + eq(l.from).x = eq(l.to).y; + eq(l.from).y += 10; + return l +} + fun main(value: int, ) { @@ -288,6 +314,8 @@ fun main(value: int, ) { @testcase | 117 | | [ 20 ] @testcase | 118 | 3 | 10 3 0 1 @testcase | 119 | 1 2 | 2 2 4 +@testcase | 120 | | 10 20 +@testcase | 121 | null | (null) @fif_codegen diff --git a/tolk-tester/tests/invalid-semantics/err-4130.tolk b/tolk-tester/tests/invalid-semantics/err-4130.tolk new file mode 100644 index 0000000000..907727f98e --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4130.tolk @@ -0,0 +1,22 @@ +struct Point { + x: int + y: int +} + +struct ROLine { + readonly p1: Point + readonly p2: Point +} + +fun eq(v: T) { return v } + +fun f(l: ROLine) { + l.p1; l.p2; l.p1.x; eq(l.p2.y); eq(eq(l).p2).y; + + l.p2!.x += 1; +} + +/** +@compilation_should_fail +@stderr modifying readonly field `ROLine.p2` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4131.tolk b/tolk-tester/tests/invalid-semantics/err-4131.tolk new file mode 100644 index 0000000000..c3403b3b46 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4131.tolk @@ -0,0 +1,12 @@ +struct ImmSlice { + readonly s: slice +} + +fun ImmSlice.read(mutate self) { + self.s.loadRef() +} + +/** +@compilation_should_fail +@stderr modifying readonly field `ImmSlice.s` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4132.tolk b/tolk-tester/tests/invalid-semantics/err-4132.tolk new file mode 100644 index 0000000000..6de7eb5605 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4132.tolk @@ -0,0 +1,15 @@ +struct ROPoint { + readonly x: int + readonly y: int +} + +fun increment(mutate x: int) {} + +fun main(p1: ROPoint, p2: ROPoint) { + increment(mutate (p1 = p2).x); +} + +/** +@compilation_should_fail +@stderr modifying readonly field `ROPoint.x` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4572.tolk b/tolk-tester/tests/invalid-semantics/err-4572.tolk index a62b63a0fa..694f6d7049 100644 --- a/tolk-tester/tests/invalid-semantics/err-4572.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4572.tolk @@ -1,6 +1,8 @@ +fun eq(v: T) { return v } + fun checkCantMutateFieldOfImmutableTuple() { val ks = null as [int, [int, [int]]]?; - ks!.1.1.0 = 10; + eq(ks!.1.1 = [10]).0 = 10; } /** diff --git a/tolk-tester/tests/invalid-semantics/err-4822.tolk b/tolk-tester/tests/invalid-semantics/err-4822.tolk index 85cf4073be..108153c0c4 100644 --- a/tolk-tester/tests/invalid-semantics/err-4822.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4822.tolk @@ -8,8 +8,8 @@ struct Coord2 { p2: Point; } -fun increment(mutate x: int) { - x += 1; +fun int.increment(mutate self) { + self += 1; } fun checkCantMutateFieldOfImmutableTuple(p1: Point?, p2: Point) { @@ -19,5 +19,5 @@ fun checkCantMutateFieldOfImmutableTuple(p1: Point?, p2: Point) { /** @compilation_should_fail -@modifying immutable variable `c2` +@stderr modifying immutable variable `c2` */ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index ec783e2758..8737a7ad9a 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -1542,6 +1542,13 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(lex.cur_location(), lex.cur_str()); lex.next(); @@ -1554,7 +1561,7 @@ static AnyV parse_struct_field(Lexer& lex) { default_value = parse_expr(lex); } - return createV(loc, v_ident, default_value, declared_type); + return createV(loc, v_ident, is_readonly, default_value, declared_type); } static V parse_struct_body(Lexer& lex) { diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 19956e4675..9661e2b4bb 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -235,7 +235,7 @@ class ASTReplicator final { return createV(v->loc, clone(v->get_params())); } static V clone(V v) { - return createV(v->loc, clone(v->get_identifier()), v->default_value ? clone(v->default_value) : nullptr, clone(v->type_node)); + return createV(v->loc, clone(v->get_identifier()), v->is_readonly, v->default_value ? clone(v->default_value) : nullptr, clone(v->type_node)); } static V clone(V v) { return createV(v->loc, clone(v->get_all_fields())); diff --git a/tolk/ast.h b/tolk/ast.h index 2d7ff884bd..db39157204 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -1353,14 +1353,15 @@ template<> // ast_struct_field is one field at struct declaration // example: `struct Point { x: int, y: int }` is struct declaration, its body contains 2 fields struct Vertex final : ASTOtherVararg { + bool is_readonly; // declared as `readonly field: int` AnyTypeV type_node; // always exists, typing struct fields is mandatory AnyExprV default_value; // nullptr if no default auto get_identifier() const { return children.at(0)->as(); } - Vertex(SrcLocation loc, V name_identifier, AnyExprV default_value, AnyTypeV type_node) + Vertex(SrcLocation loc, V name_identifier, bool is_readonly, AnyExprV default_value, AnyTypeV type_node) : ASTOtherVararg(ast_struct_field, loc, {name_identifier}) - , type_node(type_node), default_value(default_value) {} + , is_readonly(is_readonly), type_node(type_node), default_value(default_value) {} }; template<> diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index 44d1667686..fa6d0bb59e 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -382,6 +382,7 @@ struct ChunkIdentifierOrKeyword final : ChunkLexerBase { if (str == "builtin") return tok_builtin; break; case 8: + if (str == "readonly") return tok_readonly; if (str == "continue") return tok_continue; if (str == "operator") return tok_operator; break; diff --git a/tolk/lexer.h b/tolk/lexer.h index 63f0e6a306..81d9175f5a 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -44,6 +44,7 @@ enum TokenType { tok_colon, tok_asm, tok_builtin, + tok_readonly, tok_int_const, tok_string_const, diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp index dd2f9a83e4..69251f3486 100644 --- a/tolk/pipe-check-rvalue-lvalue.cpp +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -18,6 +18,7 @@ #include "ast.h" #include "ast-visitor.h" #include "platform-utils.h" +#include "type-system.h" /* * This pipe checks lvalue/rvalue for validity. @@ -45,6 +46,11 @@ static void fire_error_modifying_immutable_variable(FunctionPtr cur_f, SrcLocati } } +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_modifying_readonly_field(FunctionPtr cur_f, SrcLocation loc, StructPtr struct_ref, StructFieldPtr field_ref) { + throw ParseError(cur_f, loc, "modifying readonly field `" + struct_ref->as_human_readable() + "." + field_ref->name + "`"); +} + // validate a function used as rvalue, like `var cb = f` // it's not a generic function (ensured earlier at type inferring) and has some more restrictions static void validate_function_used_as_noncall(FunctionPtr cur_f, AnyExprV v, FunctionPtr fun_ref) { @@ -159,10 +165,20 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { // check for `immutableVal.field = rhs` or any other mutation of an immutable tensor/tuple/object // don't allow cheating like `((immutableVal!)).field = rhs` + // same here: check for `obj.readonlyField = rhs` or any other mutation of a readonly field if (v->is_lvalue) { - AnyExprV leftmost_obj = v->get_obj(); + AnyExprV leftmost_obj = v; while (true) { if (auto as_dot = leftmost_obj->try_as()) { + if (as_dot->is_target_struct_field()) { + StructFieldPtr field_ref = std::get(as_dot->target); + const TypeDataStruct* obj_type = as_dot->get_obj()->inferred_type->unwrap_alias()->try_as(); + tolk_assert(obj_type); + if (field_ref->is_readonly) { + fire_error_modifying_readonly_field(cur_f, as_dot->loc, obj_type->struct_ref, field_ref); + } + } + leftmost_obj = as_dot->get_obj(); } else if (auto as_par = leftmost_obj->try_as()) { leftmost_obj = as_par->get_expr(); @@ -192,6 +208,8 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { if (v->is_rvalue && v->is_target_fun_ref()) { validate_function_used_as_noncall(cur_f, v, std::get(v->target)); } + + parent::visit(v); } void visit(V v) override { diff --git a/tolk/pipe-lazy-load-insertions.cpp b/tolk/pipe-lazy-load-insertions.cpp index 64438645b7..e6279b7220 100644 --- a/tolk/pipe-lazy-load-insertions.cpp +++ b/tolk/pipe-lazy-load-insertions.cpp @@ -489,7 +489,7 @@ struct ExprUsagesWhileCollecting { ith_field_action.reserve(future_fields.size()); for (int field_idx = 0; field_idx < static_cast(future_fields.size()); ++field_idx) { FutureField f = future_fields[field_idx]; - StructFieldPtr created = new StructFieldData(static_cast(f.field_name), {}, field_idx, nullptr, nullptr); + StructFieldPtr created = new StructFieldData(static_cast(f.field_name), {}, field_idx, false, nullptr, nullptr); created->mutate()->assign_resolved_type(f.field_type); hidden_fields.push_back(created); ith_field_action.push_back(f.action); diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index ee7d96ca0b..44315a132f 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -166,7 +166,7 @@ static StructPtr register_struct(V v, StructPtr base_str v_field->error("redeclaration of field `" + field_name + "`"); } } - fields.emplace_back(new StructFieldData(field_name, v_field->loc, i, v_field->type_node, v_field->default_value)); + fields.emplace_back(new StructFieldData(field_name, v_field->loc, i, v_field->is_readonly, v_field->type_node, v_field->default_value)); } PackOpcode opcode(0, 0); diff --git a/tolk/symtable.h b/tolk/symtable.h index c49aa6e56b..606015b4ed 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -300,6 +300,7 @@ struct AliasDefData final : Symbol { struct StructFieldData final : Symbol { int field_idx; + bool is_readonly; AnyTypeV type_node; TypePtr declared_type = nullptr; // = resolved type_node AnyExprV default_value; // nullptr if no default @@ -310,9 +311,10 @@ struct StructFieldData final : Symbol { void assign_resolved_type(TypePtr declared_type); void assign_default_value(AnyExprV default_value); - StructFieldData(std::string name, SrcLocation loc, int field_idx, AnyTypeV type_node, AnyExprV default_value) + StructFieldData(std::string name, SrcLocation loc, int field_idx, bool is_readonly, AnyTypeV type_node, AnyExprV default_value) : Symbol(std::move(name), loc) , field_idx(field_idx) + , is_readonly(is_readonly) , type_node(type_node) , default_value(default_value) { } From c08958e5092ecd973727c6088298b5871cede2f8 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 1 Sep 2025 17:42:32 +0300 Subject: [PATCH 400/524] [Tolk] Private fields in structs --- crypto/smartcont/tolk-stdlib/common.tolk | 10 +- tolk-tester/tests/dicts-demo.tolk | 2 +- tolk-tester/tests/generics-2.tolk | 44 +++++++++ .../tests/invalid-semantics/err-4359.tolk | 9 ++ .../tests/invalid-semantics/err-4414.tolk | 11 +++ .../tests/invalid-semantics/err-4415.tolk | 11 +++ .../tests/invalid-syntax/err-3710.tolk | 11 +++ tolk-tester/tests/maps-tests.tolk | 12 +-- tolk-tester/tests/pack-unpack-1.tolk | 2 +- tolk-tester/tests/pack-unpack-2.tolk | 4 +- tolk-tester/tests/pack-unpack-3.tolk | 14 +-- tolk-tester/tests/pack-unpack-6.tolk | 4 +- tolk-tester/tests/struct-tests.tolk | 43 +++++++++ tolk/CMakeLists.txt | 1 + tolk/ast-from-tokens.cpp | 10 +- tolk/ast-replicator.h | 2 +- tolk/ast.h | 5 +- tolk/lexer.cpp | 1 + tolk/lexer.h | 1 + tolk/pipe-check-private-fields.cpp | 95 +++++++++++++++++++ tolk/pipe-lazy-load-insertions.cpp | 2 +- tolk/pipe-register-symbols.cpp | 2 +- tolk/pipeline.h | 1 + tolk/symtable.h | 4 +- tolk/tolk.cpp | 1 + 25 files changed, 269 insertions(+), 33 deletions(-) create mode 100644 tolk-tester/tests/invalid-semantics/err-4359.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4414.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4415.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3710.tolk create mode 100644 tolk/pipe-check-private-fields.cpp diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 2228036d3f..0f39ca3749 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -95,7 +95,7 @@ type bytesN = builtin /// - a key must be fixed-width; valid: `int32`, `uint64`, `address`, `bits256`, `Point`; invalid: `int`, `coins` /// - a value must be serializable; valid: `int32`, `coins`, `AnyStruct`, `Cell`; invalid: `int`, `builder` struct map { - tvmDict: dict + private tvmDict: dict } /// `dict` is a low-level TVM dictionary. @@ -123,7 +123,7 @@ type void = builtin /// Note, that `st = MyStorage.fromSlice(s)` does NOT deep-load any refs; `st.extra` is `Cell`, not `T`; /// you should manually call `st.extra.load()` to get `T` (ExtraData in this example). struct Cell { - tvmCell: cell + private readonly tvmCell: cell } @@ -910,7 +910,7 @@ fun map.iteratePrev(self, current: MapEntry): MapEntry /// } /// ``` struct MapLookupResult { - rawSlice: slice? // holds encoded value, present if isFound + private readonly rawSlice: slice? // holds encoded value, present if isFound isFound: bool } @@ -938,8 +938,8 @@ fun MapLookupResult.loadValue(self): slice { /// } /// ``` struct MapEntry { - rawValue: slice? // holds encoded value, present if isFound - key: K + private readonly rawValue: slice? // holds encoded value, present if isFound + private readonly key: K isFound: bool } diff --git a/tolk-tester/tests/dicts-demo.tolk b/tolk-tester/tests/dicts-demo.tolk index cec83280b2..990d57a86b 100644 --- a/tolk-tester/tests/dicts-demo.tolk +++ b/tolk-tester/tests/dicts-demo.tolk @@ -113,7 +113,7 @@ fun test106() { dict.addIntToIDict(3, 103); var m = createMapFromLowLevelDict(dict); - val fk = m.findFirst().key; + val fk = m.findFirst().getKey(); val fv = m.findFirst().loadValue(); var t = createEmptyTuple(); var r = m.findKeyLess(3); diff --git a/tolk-tester/tests/generics-2.tolk b/tolk-tester/tests/generics-2.tolk index 4966d111ba..82683af83c 100644 --- a/tolk-tester/tests/generics-2.tolk +++ b/tolk-tester/tests/generics-2.tolk @@ -463,6 +463,49 @@ fun test27() { __expect_type(pp2, "Pair | Wrapper"); } +struct Box { + private readonly item: T +} + +fun Box.create(v: T) { + return Box { item: v } +} + +fun Box.itemEquals(self, cmp: T) { + return self.item == cmp +} + +fun Box.itemEqualsTo(self, cmp: Box) { + return self.item == cmp.item +} + +fun Box.clone(self) { + var b = Box { item: self.item as U }; + b.item; + return b; +} + +fun Box.areEqual(b1: Box, b2: Box) { + return b1.item == b2.item +} + +fun Box.specifiedGetItem(self) { + return self.item +} + +@method_id(128) +fun test28() { + var b1 = Box.create(5); + var b2 = Box.create(4); + var b3 = b2.clone(); + __expect_type(b3, "Box"); + return ( + b1.itemEquals(0), b2.itemEquals(4), + b1.itemEqualsTo(b2), b2.itemEqualsTo(b3), + Box.areEqual(b1, b1), + ); +} + fun main(c: Wrapper, d: WrappedInt) { __expect_type(c, "Wrapper"); @@ -507,6 +550,7 @@ fun main(c: Wrapper, d: WrappedInt) { @testcase | 120 | | (null) typeid-9 777 (null) (null) 0 typeid-10 -1 -1 @testcase | 123 | | 10 10 10 10 10 @testcase | 126 | | 0 0 0 (null) 777 0 typeid-12 (null) typeid-12 (null) 0 +@testcase | 128 | | 0 -1 0 -1 -1 @fif_codegen diff --git a/tolk-tester/tests/invalid-semantics/err-4359.tolk b/tolk-tester/tests/invalid-semantics/err-4359.tolk new file mode 100644 index 0000000000..4b487e2b79 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4359.tolk @@ -0,0 +1,9 @@ +fun m() { + var c = (12 as int64).toCell(); + var s = c.tvmCell.beginParse(); +} + +/** +@compilation_should_fail +@stderr field `Cell.tvmCell` is private + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4414.tolk b/tolk-tester/tests/invalid-semantics/err-4414.tolk new file mode 100644 index 0000000000..481ddcc3ab --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4414.tolk @@ -0,0 +1,11 @@ +struct Point { private x: int, private y: int } +struct Point3d { x: int, y: int, z: int } + +fun Point3d.create() { + return Point { x: 10, y: 20 } +} + +/** +@compilation_should_fail +@stderr field `Point.x` is private + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4415.tolk b/tolk-tester/tests/invalid-semantics/err-4415.tolk new file mode 100644 index 0000000000..db3faf01dc --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4415.tolk @@ -0,0 +1,11 @@ +struct Wrap { private item: T } + +fun m(w: Wrap) { + w = w; + return w.item!; +} + +/** +@compilation_should_fail +@stderr field `Wrap.item` is private + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3710.tolk b/tolk-tester/tests/invalid-syntax/err-3710.tolk new file mode 100644 index 0000000000..00ff2e865a --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3710.tolk @@ -0,0 +1,11 @@ +struct A { + private readonly a: () + readonly b: () + private private e: () +} + +/** +@compilation_should_fail +@stderr expected field name, got `private` +@stderr private private e + */ diff --git a/tolk-tester/tests/maps-tests.tolk b/tolk-tester/tests/maps-tests.tolk index 43255a97b7..e80d9c01b5 100644 --- a/tolk-tester/tests/maps-tests.tolk +++ b/tolk-tester/tests/maps-tests.tolk @@ -204,7 +204,7 @@ fun test14(fromV: int) { m.set({v: 1}, 10); m.set({v: 2}, 20); m.set({v: 3}, 30); - return m.findKeyGreater({v: fromV}).key + return m.findKeyGreater({v: fromV}).getKey() } @method_id(115) @@ -227,7 +227,7 @@ fun test16() { fun test17() { var m = createEmptyMap(); m.set(true, 123); - return m.findFirst().key; + return m.findFirst().getKey(); } @method_id(118) @@ -274,7 +274,7 @@ fun test22() { m.set(3, Point{x:30,y:30}.toCell()); m.set(2, Point{x:20,y:20}.toCell()); m.set(1, Point{x:10,y:10}.toCell()); - return (m.findFirst().key, m.findLast().key, m.findKeyGreater(2).key, m.findKeyGreaterOrEqual(2).loadValue().load(), m.mustGet(2).load().x); + return (m.findFirst().getKey(), m.findLast().getKey(), m.findKeyGreater(2).getKey(), m.findKeyGreaterOrEqual(2).loadValue().load(), m.mustGet(2).load().x); } @method_id(123) @@ -329,7 +329,7 @@ fun test26() { fun test27() { var m = createEmptyMap(); m.set(1, ""); - return (m.findFirst().key, m.findFirst().loadValue().remainingBitsCount()); + return (m.findFirst().getKey(), m.findFirst().loadValue().remainingBitsCount()); } @method_id(128) @@ -359,7 +359,7 @@ fun test29() { DataWithCell.fromSlice(m.mustGet(2)).payload, m.get(3).loadValue().loadAny().p!.load(), m.get(3).loadValue().remainingRefsCount(), - m.findLast().key, + m.findLast().getKey(), m.findLast().loadValue().remainingRefsCount(), ); } @@ -406,7 +406,7 @@ fun test32() { m.set(stringHexToSlice("0304") as bits16, stringHexToSlice("bbbb") as bits16); val repl = m.replaceAndGetPrevious(stringHexToSlice("0304") as bits16, stringHexToSlice("0000") as bits16); var r = m.iteratePrev(m.findLast()); - return ((r.key as slice).remainingBitsCount(), (r.key as slice).loadAny(), (r.loadValue() as slice).remainingBitsCount(), repl.isFound, (repl.loadValue() as slice).loadUint(8)); + return ((r.getKey() as slice).remainingBitsCount(), (r.getKey() as slice).loadAny(), (r.loadValue() as slice).remainingBitsCount(), repl.isFound, (repl.loadValue() as slice).loadUint(8)); } @method_id(133) diff --git a/tolk-tester/tests/pack-unpack-1.tolk b/tolk-tester/tests/pack-unpack-1.tolk index 3cc2060387..8239403190 100644 --- a/tolk-tester/tests/pack-unpack-1.tolk +++ b/tolk-tester/tests/pack-unpack-1.tolk @@ -34,7 +34,7 @@ fun test1(value: int) { fun test2() { var t: JustInt32 = { value: 10 }; var c = t.toCell(); - var s = c.tvmCell.beginParse(); + var s = c.beginParse(); __expect_type(s.skipAny(), "slice"); s.assertEnd(); return true; diff --git a/tolk-tester/tests/pack-unpack-2.tolk b/tolk-tester/tests/pack-unpack-2.tolk index dc122f2147..7bf0f247ca 100644 --- a/tolk-tester/tests/pack-unpack-2.tolk +++ b/tolk-tester/tests/pack-unpack-2.tolk @@ -468,7 +468,7 @@ fun test_TwoInts32And64() { @method_id(205) fun test_TwoInts32AndRef64() { - run({ op: 123, query_id_ref: { tvmCell: beginCell().storeUint(255,64).endCell() } }, stringHexToSlice("0000007B").appendRef(stringHexToSlice("00000000000000FF"))); + run({ op: 123, query_id_ref: (255 as uint64).toCell() }, stringHexToSlice("0000007B").appendRef(stringHexToSlice("00000000000000FF"))); return true; } @@ -627,7 +627,7 @@ fun test_DifferentMix2() { assert(iae1.query_id_maybe_ref == null, 400); assert_slice_is_44_and_ref45(r1.rest); - run({ iae: { tvmCell: IntAndEither32OrRef64{ op: 778, i32orRef: { tvmCell: Inner2{ i64_in_ref: 9919992 }.toCell() }, query_id_maybe_ref: Inner1{ query_id_ref: 889477 }.toCell() }.toCell() }, tic: { op: 123, amount: 500000 }, rest: beginCell().endCell().beginParse() }, + run({ iae: IntAndEither32OrRef64{ op: 778, i32orRef: Inner2{ i64_in_ref: 9919992 }.toCell(), query_id_maybe_ref: Inner1{ query_id_ref: 889477 }.toCell() }.toCell(), tic: { op: 123, amount: 500000 }, rest: beginCell().endCell().beginParse() }, stringHexToSlice("0000007B307A120").appendRef(stringHexToSlice("0000030AE_").appendRef(stringHexToSlice("0000000000975DF8")).appendRef(stringHexToSlice("00000000000D9285")))); var r2 = DifferentMix2.fromSlice(stringHexToSlice("0000007B307A120").appendRef(stringHexToSlice("0000030AE_").appendRef(stringHexToSlice("0000000000975DF8")).appendRef(stringHexToSlice("00000000000D9285")))); val iae2 = r2.iae.load(); diff --git a/tolk-tester/tests/pack-unpack-3.tolk b/tolk-tester/tests/pack-unpack-3.tolk index 793f714b9d..71331464f6 100644 --- a/tolk-tester/tests/pack-unpack-3.tolk +++ b/tolk-tester/tests/pack-unpack-3.tolk @@ -248,8 +248,8 @@ fun test_MsgExternal1() { run(SayHiAndGoodbye{ dest_addr: null, body: BodyPayload2 { master_id: 10, owner_address: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c") } }, stringHexToSlice("892150000000000000000000000000000000000000000000000000000000000000000002_")); run(SayHiAndGoodbye{ dest_addr: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), body: BodyPayload1 { should_forward: false, n_times: 85, content: generateSlice_44_with_ref45() } }, stringHexToSlice("89C0000000000000000000000000000000000000000000000000000000000000000002000000550000002C").appendRef(stringHexToSlice("0000002D"))); run(SayHiAndGoodbye{ dest_addr: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), body: BodyPayload2 { master_id: -5, owner_address: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi") } }, stringHexToSlice("89CFF28E8033DB1467CACEA8B158F0F61E682DE06E8A5947504C904F1F703D2BE4D9E7EE7F9474019ED8A33E5675458AC787B0F3416F037452CA3A82648278FB81E95F26CF4_")); - run(SayStoreInChain{ in_masterchain: true, contents: { tvmCell: BodyPayload1{ should_forward: true, n_times: 20, content: generateSlice_44_with_ref45() }.toCell()} }, stringHexToSlice("0013C_").appendRef(stringHexToSlice("3000000140000002C").appendRef(stringHexToSlice("0000002D")))); - run(SayStoreInChain{ in_masterchain: false, contents: { tvmCell: BodyPayload2{ master_id: 37, owner_address: address(("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c")) }.toCell()} }, stringHexToSlice("00134_").appendRef(stringHexToSlice("4960000000000000000000000000000000000000000000000000000000000000000004_"))); + run(SayStoreInChain{ in_masterchain: true, contents: BodyPayload1{ should_forward: true, n_times: 20, content: generateSlice_44_with_ref45() }.toCell() as cell as Cell }, stringHexToSlice("0013C_").appendRef(stringHexToSlice("3000000140000002C").appendRef(stringHexToSlice("0000002D")))); + run(SayStoreInChain{ in_masterchain: false, contents: BodyPayload2{ master_id: 37, owner_address: address(("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c")) }.toCell() as cell as Cell }, stringHexToSlice("00134_").appendRef(stringHexToSlice("4960000000000000000000000000000000000000000000000000000000000000000004_"))); val o = MsgExternal1.fromSlice(stringHexToSlice("00134_").appendRef(stringHexToSlice("4960000000000000000000000000000000000000000000000000000000000000000004_"))); assert(o is SayStoreInChain, 400); @@ -260,15 +260,15 @@ fun test_MsgExternal1() { @method_id(205) fun test_MsgTransfer() { run({ params: EitherLeft { value: TransferParams1 { dest_int: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), amount: 80000000, dest_ext: makeExternalAddress(1234,80) } } }, stringHexToSlice("FB3701FF3CA4FF28E8033DB1467CACEA8B158F0F61E682DE06E8A5947504C904F1F703D2BE4D9E404C4B4004A0000000000000000009A5_")); - run({ params: EitherLeft { value: TransferParams2 { intVector: (123, 1234567890123456, 1234567890123456), needs_more: {tvmCell: beginCell().storeBool(true).endCell()} } } }, stringHexToSlice("FB3701FF48000003DDC118B54F22AEB0000118B54F22AEB02_").appendRef(stringHexToSlice("C_"))); - run({ params: EitherRight { value: { tvmCell: TransferParams1{ dest_int: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), amount: 80000000, dest_ext: makeExternalAddress(1234,70) }.toCell()} } }, stringHexToSlice("FB3701FFC_").appendRef(stringHexToSlice("7948000000000000000000000000000000000000000000000000000000000000000000809896800918000000000000004D2"))); - run({ params: EitherRight { value: { tvmCell: TransferParams2{ intVector: (123, null, 0), needs_more: {tvmCell: beginCell().storeBool(false).endCell()} }.toCell()} } }, stringHexToSlice("FB3701FFC_").appendRef(stringHexToSlice("90000007B00000000000000004_").appendRef(stringHexToSlice("4_")))); + run({ params: EitherLeft { value: TransferParams2 { intVector: (123, 1234567890123456, 1234567890123456), needs_more: true.toCell() } } }, stringHexToSlice("FB3701FF48000003DDC118B54F22AEB0000118B54F22AEB02_").appendRef(stringHexToSlice("C_"))); + run({ params: EitherRight { value: TransferParams1{ dest_int: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), amount: 80000000, dest_ext: makeExternalAddress(1234,70) }.toCell() as cell as Cell } }, stringHexToSlice("FB3701FFC_").appendRef(stringHexToSlice("7948000000000000000000000000000000000000000000000000000000000000000000809896800918000000000000004D2"))); + run({ params: EitherRight { value: TransferParams2{ intVector: (123, null, 0), needs_more: false.toCell() }.toCell() as cell as Cell } }, stringHexToSlice("FB3701FFC_").appendRef(stringHexToSlice("90000007B00000000000000004_").appendRef(stringHexToSlice("4_")))); val o = MsgTransfer.fromSlice(stringHexToSlice("FB3701FFC_").appendRef(stringHexToSlice("90000007B00000000000000004_").appendRef(stringHexToSlice("4_")))); return ( o.params is EitherLeft, o.params is EitherRight, - o.params is EitherRight && TransferParams.fromCell(o.params.value.tvmCell) is TransferParams1, - o.params is EitherRight && TransferParams.fromCell(o.params.value.tvmCell) is TransferParams2, + o.params is EitherRight && TransferParams.fromCell(o.params.value) is TransferParams1, + o.params is EitherRight && TransferParams.fromCell(o.params.value) is TransferParams2, ); } diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk index 3b5074f566..4a261869c5 100644 --- a/tolk-tester/tests/pack-unpack-6.tolk +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -26,9 +26,7 @@ fun test2() { @method_id(103) fun test3() { - var cc: Cell = { - tvmCell: beginCell().storeSlice(stringHexToSlice("0109ab")).endCell() - }; + var cc = beginCell().storeSlice(stringHexToSlice("0109ab")).endCell() as Cell; return cc.load({assertEndAfterReading: false}); } diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index ae8d51ac2a..eb33de0e1c 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -569,6 +569,47 @@ fun test41(rev: bool) { } } +struct PrivPoint { + private readonly x: int + private y: int + z: int +} + +fun PrivPoint.create(x: int, y: int): PrivPoint { + return { x, y, z: 0 } +} + +fun PrivPoint.modifyY(mutate self, newY: int) { + self.y = newY +} + +fun PrivPoint.getSum(self) { + return self.x + self.y + self.z +} + +@method_id(142) +fun test42() { + var p1 = PrivPoint.create(10, 20); + p1.modifyY(80); + p1.z += 5; + return (p1, p1.getSum()); +} + +struct PrivWithDefaults { + private a: int32 = 0 + private b: bool = false +} + +fun PrivWithDefaults.createA(a: U) { + return PrivWithDefaults { a: a } +} + +@method_id(143) +fun test43() { + var obj: PrivWithDefaults = {}; + return (obj, PrivWithDefaults.createA(20)); +} + fun main(x: int8, y: MInt) { __expect_type(PointAlias{x,y}, "Point"); __expect_type(Point{x,y} as Point, "Point"); @@ -628,6 +669,8 @@ type PointAlias = Point; @testcase | 140 | | 2 1 4 3 @testcase | 141 | -1 | 4 3 2 1 @testcase | 141 | 0 | 3 4 1 2 +@testcase | 142 | | 10 80 5 95 +@testcase | 143 | | 0 0 20 0 @fif_codegen """ diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index 0014d736e3..6657be95df 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -20,6 +20,7 @@ set(TOLK_SOURCE pipe-check-inferred-types.cpp pipe-refine-lvalue-for-mutate.cpp pipe-check-rvalue-lvalue.cpp + pipe-check-private-fields.cpp pipe-check-pure-impure.cpp pipe-constant-folding.cpp pipe-optimize-boolean-expr.cpp diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 8737a7ad9a..03016a5fe8 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -1543,8 +1543,14 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(loc, v_ident, is_readonly, default_value, declared_type); + return createV(loc, v_ident, is_private, is_readonly, default_value, declared_type); } static V parse_struct_body(Lexer& lex) { diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 9661e2b4bb..1f41dc3d14 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -235,7 +235,7 @@ class ASTReplicator final { return createV(v->loc, clone(v->get_params())); } static V clone(V v) { - return createV(v->loc, clone(v->get_identifier()), v->is_readonly, v->default_value ? clone(v->default_value) : nullptr, clone(v->type_node)); + return createV(v->loc, clone(v->get_identifier()), v->is_private, v->is_readonly, v->default_value ? clone(v->default_value) : nullptr, clone(v->type_node)); } static V clone(V v) { return createV(v->loc, clone(v->get_all_fields())); diff --git a/tolk/ast.h b/tolk/ast.h index db39157204..1b748a58bd 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -1353,15 +1353,16 @@ template<> // ast_struct_field is one field at struct declaration // example: `struct Point { x: int, y: int }` is struct declaration, its body contains 2 fields struct Vertex final : ASTOtherVararg { + bool is_private; // declared as `private field: int` bool is_readonly; // declared as `readonly field: int` AnyTypeV type_node; // always exists, typing struct fields is mandatory AnyExprV default_value; // nullptr if no default auto get_identifier() const { return children.at(0)->as(); } - Vertex(SrcLocation loc, V name_identifier, bool is_readonly, AnyExprV default_value, AnyTypeV type_node) + Vertex(SrcLocation loc, V name_identifier, bool is_private, bool is_readonly, AnyExprV default_value, AnyTypeV type_node) : ASTOtherVararg(ast_struct_field, loc, {name_identifier}) - , is_readonly(is_readonly), type_node(type_node), default_value(default_value) {} + , is_private(is_private), is_readonly(is_readonly), type_node(type_node), default_value(default_value) {} }; template<> diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index fa6d0bb59e..c349a91ae3 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -380,6 +380,7 @@ struct ChunkIdentifierOrKeyword final : ChunkLexerBase { break; case 7: if (str == "builtin") return tok_builtin; + if (str == "private") return tok_private; break; case 8: if (str == "readonly") return tok_readonly; diff --git a/tolk/lexer.h b/tolk/lexer.h index 81d9175f5a..b92b2a1c29 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -44,6 +44,7 @@ enum TokenType { tok_colon, tok_asm, tok_builtin, + tok_private, tok_readonly, tok_int_const, diff --git a/tolk/pipe-check-private-fields.cpp b/tolk/pipe-check-private-fields.cpp new file mode 100644 index 0000000000..46373c7acf --- /dev/null +++ b/tolk/pipe-check-private-fields.cpp @@ -0,0 +1,95 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-visitor.h" +#include "platform-utils.h" +#include "type-system.h" + +namespace tolk { + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_private_field_used_outside_method(FunctionPtr cur_f, SrcLocation loc, StructPtr struct_ref, StructFieldPtr field_ref) { + throw ParseError(cur_f, loc, "field `" + struct_ref->as_human_readable() + "." + field_ref->name + "` is private"); +} + +static bool is_private_field_usage_allowed(FunctionPtr cur_f, StructPtr struct_ref) { + // private fields are accessible only inside methods for that struct + if (!cur_f->is_method()) { + return false; + } + const TypeDataStruct* receiver_struct = cur_f->receiver_type->unwrap_alias()->try_as(); + if (receiver_struct && receiver_struct->struct_ref == struct_ref) { + return true; + } + + // probably it's generic, e.g. struct_ref = `Box` and receiver = `Box` + if (struct_ref->is_instantiation_of_generic_struct() && cur_f->is_instantiation_of_generic_function()) { + const auto* receiver_Ts = cur_f->base_fun_ref->receiver_type->try_as(); + return receiver_Ts && receiver_Ts->struct_ref == struct_ref->base_struct_ref; + } + + return false; +} + +class CheckPrivateFieldsUsageVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; + +protected: + void visit(V v) override { + parent::visit(v); + + if (v->is_target_struct_field()) { + StructFieldPtr field_ref = std::get(v->target); + const TypeDataStruct* obj_type = v->get_obj()->inferred_type->unwrap_alias()->try_as(); + tolk_assert(obj_type); + if (field_ref->is_private && !is_private_field_usage_allowed(cur_f, obj_type->struct_ref)) { + fire_error_private_field_used_outside_method(cur_f, v->loc, obj_type->struct_ref, field_ref); + } + } + } + + void visit(V v) override { + parent::visit(v); + tolk_assert(v->struct_ref); + + for (int i = 0; i < v->get_body()->get_num_fields(); ++i) { + auto v_field = v->get_body()->get_field(i); + StructFieldPtr field_ref = v_field->field_ref; + if (field_ref->is_private && !is_private_field_usage_allowed(cur_f, v->struct_ref)) { + fire_error_private_field_used_outside_method(cur_f, v_field->loc, v->struct_ref, field_ref); + } + } + } + + public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + cur_f = nullptr; + } +}; + +void pipeline_check_private_fields_usage() { + visit_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-lazy-load-insertions.cpp b/tolk/pipe-lazy-load-insertions.cpp index e6279b7220..8eb0d76d07 100644 --- a/tolk/pipe-lazy-load-insertions.cpp +++ b/tolk/pipe-lazy-load-insertions.cpp @@ -489,7 +489,7 @@ struct ExprUsagesWhileCollecting { ith_field_action.reserve(future_fields.size()); for (int field_idx = 0; field_idx < static_cast(future_fields.size()); ++field_idx) { FutureField f = future_fields[field_idx]; - StructFieldPtr created = new StructFieldData(static_cast(f.field_name), {}, field_idx, false, nullptr, nullptr); + StructFieldPtr created = new StructFieldData(static_cast(f.field_name), {}, field_idx, false, false, nullptr, nullptr); created->mutate()->assign_resolved_type(f.field_type); hidden_fields.push_back(created); ith_field_action.push_back(f.action); diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 44315a132f..7f684259a0 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -166,7 +166,7 @@ static StructPtr register_struct(V v, StructPtr base_str v_field->error("redeclaration of field `" + field_name + "`"); } } - fields.emplace_back(new StructFieldData(field_name, v_field->loc, i, v_field->is_readonly, v_field->type_node, v_field->default_value)); + fields.emplace_back(new StructFieldData(field_name, v_field->loc, i, v_field->is_private, v_field->is_readonly, v_field->type_node, v_field->default_value)); } PackOpcode opcode(0, 0); diff --git a/tolk/pipeline.h b/tolk/pipeline.h index 641beb06b1..d433afcfca 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -40,6 +40,7 @@ void pipeline_infer_types_and_calls_and_fields(); void pipeline_check_inferred_types(); void pipeline_refine_lvalue_for_mutate_arguments(); void pipeline_check_rvalue_lvalue(); +void pipeline_check_private_fields_usage(); void pipeline_check_pure_impure_operations(); void pipeline_constant_folding(); void pipeline_optimize_boolean_expressions(); diff --git a/tolk/symtable.h b/tolk/symtable.h index 606015b4ed..09235eb353 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -300,6 +300,7 @@ struct AliasDefData final : Symbol { struct StructFieldData final : Symbol { int field_idx; + bool is_private; bool is_readonly; AnyTypeV type_node; TypePtr declared_type = nullptr; // = resolved type_node @@ -311,9 +312,10 @@ struct StructFieldData final : Symbol { void assign_resolved_type(TypePtr declared_type); void assign_default_value(AnyExprV default_value); - StructFieldData(std::string name, SrcLocation loc, int field_idx, bool is_readonly, AnyTypeV type_node, AnyExprV default_value) + StructFieldData(std::string name, SrcLocation loc, int field_idx, bool is_private, bool is_readonly, AnyTypeV type_node, AnyExprV default_value) : Symbol(std::move(name), loc) , field_idx(field_idx) + , is_private(is_private) , is_readonly(is_readonly) , type_node(type_node) , default_value(default_value) { diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp index 69b03f20ad..cef2365ecc 100644 --- a/tolk/tolk.cpp +++ b/tolk/tolk.cpp @@ -63,6 +63,7 @@ int tolk_proceed(const std::string &entrypoint_filename) { pipeline_check_inferred_types(); pipeline_refine_lvalue_for_mutate_arguments(); pipeline_check_rvalue_lvalue(); + pipeline_check_private_fields_usage(); pipeline_check_pure_impure_operations(); pipeline_constant_folding(); pipeline_optimize_boolean_expressions(); From 82ae5d9d53c1d9f3f90bb439aee0f076c50262fa Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 14 Sep 2025 15:03:43 +0500 Subject: [PATCH 401/524] [Tolk] Ensure that compiled wasm passes all tests --- .../tests/invalid-symbol/err-2980.tolk | 2 +- tolk-tester/tolk-tester.js | 44 ++++++++++--------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/tolk-tester/tests/invalid-symbol/err-2980.tolk b/tolk-tester/tests/invalid-symbol/err-2980.tolk index de29cc6578..65e87ff4a7 100644 --- a/tolk-tester/tests/invalid-symbol/err-2980.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2980.tolk @@ -6,6 +6,6 @@ @compilation_should_fail On Linux/Mac, `realpath()` returns an error, and the error message is "cannot find file" On Windows, it fails after, on reading, with a message "cannot open file" -@stderr err-2980.tolk:2:7: error: Failed to import: cannot +@stderr err-2980.tolk:2:7: error: Failed to import: @stderr import "unexisting.tolk"; */ diff --git a/tolk-tester/tolk-tester.js b/tolk-tester/tolk-tester.js index ae07a53b8f..fa17bd5fb3 100644 --- a/tolk-tester/tolk-tester.js +++ b/tolk-tester/tolk-tester.js @@ -500,29 +500,31 @@ function copyFromCString(mod, ptr) { function compileFile(mod, filename, experimentalOptions, withSrcLineComments) { // see tolk-wasm.cpp: typedef void (*WasmFsReadCallback)(int, char const*, char**, char**) const callbackPtr = mod.addFunction((kind, dataPtr, destContents, destError) => { - if (kind === 0) { // realpath - try { - let relative = copyFromCString(mod, dataPtr) - if (relative.startsWith('@stdlib/')) { - // import "@stdlib/filename" or import "@stdlib/filename.tolk" - relative = STDLIB_FOLDER + '/' + relative.substring(7) - if (!relative.endsWith('.tolk')) { - relative += '.tolk' + switch (kind) { // enum ReadCallback::Kind in C++ + case 0: // realpath + let relativeFilename = copyFromCString(mod, dataPtr) // from `import` statement, relative to cur file + if (!relativeFilename.endsWith('.tolk')) { + relativeFilename += '.tolk' + } + copyToCStringPtr(mod, path.normalize(relativeFilename), destContents) + break + case 1: // read file + try { + const filename = copyFromCString(mod, dataPtr) // already normalized (as returned above) + if (filename.startsWith('@stdlib/')) { + const contents = fs.readFileSync(STDLIB_FOLDER + '/' + filename.substring(8)).toString('utf-8'); + copyToCStringPtr(mod, contents, destContents) + } else { + const contents = fs.readFileSync(filename).toString('utf-8'); + copyToCStringPtr(mod, contents, destContents) } + } catch (err) { + copyToCStringPtr(mod, err.message || err.toString(), destError) } - copyToCStringPtr(mod, fs.realpathSync(relative), destContents); - } catch (err) { - copyToCStringPtr(mod, 'cannot find file', destError); - } - } else if (kind === 1) { // read file - try { - const absolute = copyFromCString(mod, dataPtr) // already normalized (as returned above) - copyToCStringPtr(mod, fs.readFileSync(absolute).toString('utf-8'), destContents); - } catch (err) { - copyToCStringPtr(mod, err.message || err.toString(), destError); - } - } else { - copyToCStringPtr(mod, 'Unknown callback kind=' + kind, destError); + break + default: + copyToCStringPtr(mod, 'Unknown callback kind=' + kind, destError) + break } }, 'viiii'); From 4c62653b44a09a163e0531e346ad267d73edf435 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 14 Sep 2025 17:37:55 +0500 Subject: [PATCH 402/524] [Tolk] Auto-infer `never` for always-throwing functions Previously, `never` could be specified, but was not auto-inferred. (because it's done the same way in TypeScript) Now, it's auto-inferred, which is more expected in our case. --- tolk-tester/tests/inference-tests.tolk | 4 ++-- tolk-tester/tests/invalid-typing/err-6427.tolk | 1 + tolk-tester/tests/never-type-tests.tolk | 11 +++++++++++ tolk-tester/tests/try-catch-tests.tolk | 2 +- tolk-tester/tests/var-apply-tests.tolk | 2 +- tolk/analyzer.cpp | 2 +- tolk/ast-replacer.h | 1 + tolk/ast-visitor.h | 1 + tolk/builtins.cpp | 6 +++++- tolk/compiler-state.cpp | 4 ++++ tolk/compiler-state.h | 1 + tolk/pipe-infer-types-and-calls.cpp | 12 ++++++++++-- 12 files changed, 39 insertions(+), 8 deletions(-) diff --git a/tolk-tester/tests/inference-tests.tolk b/tolk-tester/tests/inference-tests.tolk index 279dd7191d..7aa5a3cb30 100644 --- a/tolk-tester/tests/inference-tests.tolk +++ b/tolk-tester/tests/inference-tests.tolk @@ -111,8 +111,8 @@ fun alwaysThrowsNotAnnotated2() { alwaysThrows(); } fun test9() { __expect_type(alwaysThrows(), "never"); __expect_type(alwaysThrows, "() -> never"); - __expect_type(alwaysThrowsNotAnnotated(), "void"); - __expect_type(alwaysThrowsNotAnnotated2(), "void"); + __expect_type(alwaysThrowsNotAnnotated(), "never"); + __expect_type(alwaysThrowsNotAnnotated2(), "never"); } diff --git a/tolk-tester/tests/invalid-typing/err-6427.tolk b/tolk-tester/tests/invalid-typing/err-6427.tolk index 37df89a1e4..57d2951100 100644 --- a/tolk-tester/tests/invalid-typing/err-6427.tolk +++ b/tolk-tester/tests/invalid-typing/err-6427.tolk @@ -1,4 +1,5 @@ fun cantReturnVoidAndNonVoid(a: int | slice | builder) { + throw 123; match (a) { int => { return; } slice => {} diff --git a/tolk-tester/tests/never-type-tests.tolk b/tolk-tester/tests/never-type-tests.tolk index 894473893e..b19d88caf3 100644 --- a/tolk-tester/tests/never-type-tests.tolk +++ b/tolk-tester/tests/never-type-tests.tolk @@ -1,5 +1,14 @@ fun takeInt(a: int) {} +fun alwaysThrows() { + throw 123; +} + +fun alwaysThrowsButReturns() { + throw 123; + return 123; +} + @method_id(101) fun test1(x: int?) { if (x == null && x != null) { @@ -21,6 +30,8 @@ fun test1(x: int?) { fun main() { __expect_type(test1, "(int?) -> int"); + __expect_type(alwaysThrows(), "never"); + __expect_type(alwaysThrowsButReturns(), "int"); } /** diff --git a/tolk-tester/tests/try-catch-tests.tolk b/tolk-tester/tests/try-catch-tests.tolk index df2e7c6dd9..23ca55c466 100644 --- a/tolk-tester/tests/try-catch-tests.tolk +++ b/tolk-tester/tests/try-catch-tests.tolk @@ -175,7 +175,7 @@ fun alwaysThrowX(x: int): never { } @noinline -fun anotherNever(throw123: bool): never { +fun anotherNever(throw123: bool) { // auto-infer `never` if (throw123) { alwaysThrow123(); } alwaysThrowX(456); } diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index e11235ed01..f40e1718ef 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -101,7 +101,7 @@ fun testVarApplyInTernary() { return t; } -fun always_throw2(x: int) { +fun always_throw2(x: int): void { throw 239 + x } diff --git a/tolk/analyzer.cpp b/tolk/analyzer.cpp index ea6a9db22e..5d70b80649 100644 --- a/tolk/analyzer.cpp +++ b/tolk/analyzer.cpp @@ -24,7 +24,7 @@ namespace tolk { // for instance, variables after their call aren't considered used // its main purpose is `throw` statement, it's a call to a built-in `__throw` function static bool does_function_always_throw(FunctionPtr fun_ref) { - return fun_ref->declared_return_type == TypeDataNever::create(); + return fun_ref->inferred_return_type == TypeDataNever::create(); } /* diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 41683571e7..6b4a08f054 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -210,6 +210,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { }; +const std::vector& get_all_builtin_functions(); const std::vector& get_all_not_builtin_functions(); const std::vector& get_all_declared_constants(); const std::vector& get_all_declared_structs(); diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index 165dc0ce45..dc18496b8d 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -211,6 +211,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { }; +const std::vector& get_all_builtin_functions(); const std::vector& get_all_not_builtin_functions(); const std::vector& get_all_declared_global_vars(); const std::vector& get_all_declared_constants(); diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 2149bc64d4..fd48cc033c 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -43,11 +43,13 @@ static std::vector define_builtin_parameters(const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const std::function& func, int flags) { auto* f_sym = new FunctionData(name, {}, "", nullptr, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltinAsmOp(func), nullptr); G.symtable.add_function(f_sym); + G.all_builtins.push_back(f_sym); } static void define_builtin_func(const std::string& name, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const std::function& func, int flags) { auto* f_sym = new FunctionData(name, {}, "", nullptr, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltinGenerateOps(func), nullptr); G.symtable.add_function(f_sym); + G.all_builtins.push_back(f_sym); } static void define_builtin_method(const std::string& name, TypePtr receiver_type, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const std::function& func, int flags, @@ -57,6 +59,7 @@ static void define_builtin_method(const std::string& name, TypePtr receiver_type f_sym->arg_order = arg_order; f_sym->ret_order = ret_order; G.symtable.add_function(f_sym); + G.all_builtins.push_back(f_sym); G.all_methods.push_back(f_sym); } @@ -64,6 +67,7 @@ void define_builtin_method(const std::string& name, TypePtr receiver_type, const std::string method_name = name.substr(name.find('.') + 1); auto* f_sym = new FunctionData(name, {}, std::move(method_name), receiver_type, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltinGenerateOps(func), nullptr); G.symtable.add_function(f_sym); + G.all_builtins.push_back(f_sym); G.all_methods.push_back(f_sym); } @@ -1500,7 +1504,7 @@ void define_builtins() { define_builtin_func("__throw", ParamsInt1, Never, nullptr, compile_throw, 0); - define_builtin_func("__throw_arg", {typeT, Int}, Never, declGenericT, + define_builtin_func("__throw_arg", {TypeDataUnknown::create(), Int}, Never, nullptr, compile_throw_arg, 0); define_builtin_func("__throw_if_unless", ParamsInt3, Unit, nullptr, diff --git a/tolk/compiler-state.cpp b/tolk/compiler-state.cpp index d7bea32bde..f7e3919847 100644 --- a/tolk/compiler-state.cpp +++ b/tolk/compiler-state.cpp @@ -66,6 +66,10 @@ void CompilerSettings::parse_experimental_options_cmd_arg(const std::string& cmd } } +const std::vector& get_all_builtin_functions() { + return G.all_builtins; +} + const std::vector& get_all_not_builtin_functions() { return G.all_functions; } diff --git a/tolk/compiler-state.h b/tolk/compiler-state.h index f56b30a434..cf3a11ffaa 100644 --- a/tolk/compiler-state.h +++ b/tolk/compiler-state.h @@ -96,6 +96,7 @@ struct CompilerState { GlobalSymbolTable symtable; PersistentHeapAllocator persistent_mem; + std::vector all_builtins; // all built-in functions std::vector all_functions; // all user-defined (not built-in) global-scope functions, with generic instantiations std::vector all_methods; // all user-defined and built-in extension methods for arbitrary types (receivers) std::vector all_contract_getters; diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index ec1ecf496f..152e90666b 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -1640,7 +1640,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (!body_end.is_unreachable()) { fun_ref->mutate()->assign_is_implicit_return(); - if (fun_ref->declared_return_type == TypeDataNever::create()) { // `never` can only be declared, it can't be inferred + if (fun_ref->declared_return_type == TypeDataNever::create()) { fire(fun_ref, v_function->get_body()->as()->loc_end, "a function returning `never` can not have a reachable endpoint"); } } @@ -1660,7 +1660,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } inferred_return_type = return_unifier.get_result(); if (inferred_return_type == nullptr) { // if no return statements at all - inferred_return_type = TypeDataVoid::create(); + inferred_return_type = body_end.is_unreachable() ? TypeDataNever::create() : TypeDataVoid::create(); } if (!body_end.is_unreachable() && inferred_return_type != TypeDataVoid::create()) { @@ -1780,8 +1780,16 @@ static void infer_and_save_type_of_constant(GlobalConstPtr const_ref) { } void pipeline_infer_types_and_calls_and_fields() { + // loop over user-defined functions visit_ast_of_all_functions(); + // assign inferred_type to built-in functions like __throw() + for (FunctionPtr fun_ref : get_all_builtin_functions()) { + if (LaunchInferTypesAndMethodsOnce::should_visit_function(fun_ref)) { + infer_and_save_return_type_of_function(fun_ref); + } + } + // analyze constants that weren't referenced by any function InferTypesAndCallsAndFieldsVisitor visitor; for (GlobalConstPtr const_ref : get_all_declared_constants()) { From 506273c4b94e6c1fba3a81b57d9565d8f0670768 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 14 Sep 2025 17:38:54 +0500 Subject: [PATCH 403/524] [Tolk] Bump version to v1.1 --- crypto/smartcont/tolk-stdlib/common.tolk | 2 +- crypto/smartcont/tolk-stdlib/gas-payments.tolk | 2 +- crypto/smartcont/tolk-stdlib/lisp-lists.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk | 2 +- tolk/tolk-version.h | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 0f39ca3749..e0a81bfb9c 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -1,7 +1,7 @@ // Standard library for Tolk (LGPL licence). // It contains common functions that are available out of the box, the user doesn't have to import anything. // More specific functions are required to be imported explicitly, like "@stdlib/gas-payments". -tolk 1.0 +tolk 1.1 /** Built-in types. diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 55c74006e1..dd343508ff 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 1.0 +tolk 1.1 /** Gas and payment related primitives. diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk index 212586b1d4..85d44e1a00 100644 --- a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 1.0 +tolk 1.1 /** Lisp-style lists are nested 2-elements tuples: `[1, [2, [3, null]]]` represents list `[1, 2, 3]`. diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index 504186f393..dee3ea4d7d 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 1.0 +tolk 1.1 /** Low-level API working with TVM dictionaries. diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index de97654d8b..2885dc3120 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 1.0 +tolk 1.1 /// Usually `c3` has a continuation initialized by the whole code of the contract. It is used for function calls. /// The primitive returns the current value of `c3`. diff --git a/tolk/tolk-version.h b/tolk/tolk-version.h index 77d745eac9..1e5f3ba544 100644 --- a/tolk/tolk-version.h +++ b/tolk/tolk-version.h @@ -18,6 +18,6 @@ namespace tolk { -constexpr const char* TOLK_VERSION = "1.0.0"; +constexpr const char* TOLK_VERSION = "1.1.0"; } // namespace tolk From b8c878074521f0da8f65c4e2a815e593163c7a69 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 16 Sep 2025 16:00:50 +0300 Subject: [PATCH 404/524] Fix compilation error (#1801) --- crypto/vm/boc-compression.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/vm/boc-compression.cpp b/crypto/vm/boc-compression.cpp index db04f162f5..d37e0380ab 100644 --- a/crypto/vm/boc-compression.cpp +++ b/crypto/vm/boc-compression.cpp @@ -19,6 +19,7 @@ #include "boc-compression.h" #include +#include #include "vm/boc.h" #include "vm/boc-writers.h" #include "vm/cells.h" From 743deba3f465cc90b970ea9f0413f2762c6aec26 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Tue, 16 Sep 2025 17:01:20 +0400 Subject: [PATCH 405/524] Fix failing Github MacOS actions due to updated XCode (#1800) * add libgslcblas into portable binaries * remove commented line * update target OSX for macos 15 * ensure system headers come from the right macOS SDK. --- .github/workflows/build-ton-macos-15-arm64-shared.yml | 2 +- assembly/native/build-macos-portable.sh | 2 ++ assembly/native/build-macos-shared.sh | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-ton-macos-15-arm64-shared.yml b/.github/workflows/build-ton-macos-15-arm64-shared.yml index 1423b4eb04..a1e0782b70 100644 --- a/.github/workflows/build-ton-macos-15-arm64-shared.yml +++ b/.github/workflows/build-ton-macos-15-arm64-shared.yml @@ -36,7 +36,7 @@ jobs: git submodule update cp assembly/native/build-macos-shared.sh . chmod +x build-macos-shared.sh - ./build-macos-shared.sh -t -c -o 15.0 + ./build-macos-shared.sh -t -c -o 16.0 ccache -sp - name: Run Tests diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index feb5bf72f2..302effb711 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -127,6 +127,8 @@ fi cmake -GNinja .. \ -DPORTABLE=1 \ -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=$OSX_TARGET \ +-DCMAKE_CXX_FLAGS="-stdlib=libc++" \ +-DCMAKE_SYSROOT=$(xcrun --show-sdk-path) \ -DCMAKE_BUILD_TYPE=Release \ -DOPENSSL_FOUND=1 \ -DOPENSSL_INCLUDE_DIR=$opensslPath/include \ diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index f5aa15b4ac..c2aa09cedf 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -79,6 +79,8 @@ brew install openssl@3 brew unlink openssl@3 && brew link --overwrite openssl@3 cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \ +-DCMAKE_CXX_FLAGS="-stdlib=libc++" \ +-DCMAKE_SYSROOT=$(xcrun --show-sdk-path) \ -DLZ4_FOUND=1 \ -DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a \ -DLZ4_INCLUDE_DIRS=$lz4Path/lib From fe5e8733d84608dacb3dc3c212052eee2a937de3 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Tue, 16 Sep 2025 17:01:20 +0400 Subject: [PATCH 406/524] Fix failing Github MacOS actions due to updated XCode (#1800) * add libgslcblas into portable binaries * remove commented line * update target OSX for macos 15 * ensure system headers come from the right macOS SDK. --- .github/workflows/build-ton-macos-15-arm64-shared.yml | 2 +- assembly/native/build-macos-portable.sh | 2 ++ assembly/native/build-macos-shared.sh | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-ton-macos-15-arm64-shared.yml b/.github/workflows/build-ton-macos-15-arm64-shared.yml index 1423b4eb04..a1e0782b70 100644 --- a/.github/workflows/build-ton-macos-15-arm64-shared.yml +++ b/.github/workflows/build-ton-macos-15-arm64-shared.yml @@ -36,7 +36,7 @@ jobs: git submodule update cp assembly/native/build-macos-shared.sh . chmod +x build-macos-shared.sh - ./build-macos-shared.sh -t -c -o 15.0 + ./build-macos-shared.sh -t -c -o 16.0 ccache -sp - name: Run Tests diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index feb5bf72f2..302effb711 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -127,6 +127,8 @@ fi cmake -GNinja .. \ -DPORTABLE=1 \ -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=$OSX_TARGET \ +-DCMAKE_CXX_FLAGS="-stdlib=libc++" \ +-DCMAKE_SYSROOT=$(xcrun --show-sdk-path) \ -DCMAKE_BUILD_TYPE=Release \ -DOPENSSL_FOUND=1 \ -DOPENSSL_INCLUDE_DIR=$opensslPath/include \ diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index f5aa15b4ac..c2aa09cedf 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -79,6 +79,8 @@ brew install openssl@3 brew unlink openssl@3 && brew link --overwrite openssl@3 cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \ +-DCMAKE_CXX_FLAGS="-stdlib=libc++" \ +-DCMAKE_SYSROOT=$(xcrun --show-sdk-path) \ -DLZ4_FOUND=1 \ -DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a \ -DLZ4_INCLUDE_DIRS=$lz4Path/lib From 05bea13375448a401d8e07c6132b7f709f5e3a32 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Fri, 19 Sep 2025 16:07:08 +0400 Subject: [PATCH 407/524] Force jammy llvm-toolchain usage on Ubuntu 22.04 ARM github images (#1808) * add libgslcblas into portable binaries * remove commented line * fix portable linux binaries * force llvm jammy installation on ubuntu 22.04 --- .github/workflows/build-ton-linux-arm64-appimage.yml | 11 ++++++----- .github/workflows/build-ton-linux-arm64-shared.yml | 8 +++++--- .github/workflows/build-ton-linux-x86-64-appimage.yml | 2 +- assembly/appimage/create-appimages.sh | 3 ++- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-ton-linux-arm64-appimage.yml b/.github/workflows/build-ton-linux-arm64-appimage.yml index a0ba4a022c..b7ecdc7845 100644 --- a/.github/workflows/build-ton-linux-arm64-appimage.yml +++ b/.github/workflows/build-ton-linux-arm64-appimage.yml @@ -21,15 +21,16 @@ jobs: - name: Install system libraries run: | sudo apt update - sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache - sudo apt remove libgsl-dev + sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache libgsl-dev libblas-dev libgslcblas0 mkdir ~/.ccache 3pp - name: Install clang-16 run: | - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 16 clang + lsb_release -a + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-16 main" | sudo tee /etc/apt/sources.list.d/llvm.list + sudo apt-get -y update + sudo apt-get install -y clang-16 - name: Cache 3pp id: cache-3pp diff --git a/.github/workflows/build-ton-linux-arm64-shared.yml b/.github/workflows/build-ton-linux-arm64-shared.yml index 540fffece6..e7de5002bb 100644 --- a/.github/workflows/build-ton-linux-arm64-shared.yml +++ b/.github/workflows/build-ton-linux-arm64-shared.yml @@ -31,9 +31,11 @@ jobs: - if: matrix.os != 'ubuntu-24.04-arm' name: Install llvm-16 run: | - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 16 clang + lsb_release -a + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-16 main" | sudo tee /etc/apt/sources.list.d/llvm.list + sudo apt-get -y update + sudo apt-get install -y clang-16 - name: Cache OpenSSL id: cache-openssl diff --git a/.github/workflows/build-ton-linux-x86-64-appimage.yml b/.github/workflows/build-ton-linux-x86-64-appimage.yml index 7ad873d22e..cb02b7bc20 100644 --- a/.github/workflows/build-ton-linux-x86-64-appimage.yml +++ b/.github/workflows/build-ton-linux-x86-64-appimage.yml @@ -21,7 +21,7 @@ jobs: - name: Install system libraries run: | sudo apt update - sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache libgsl-dev libblas-dev + sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache libgsl-dev libblas-dev libgslcblas0 mkdir ~/.ccache 3pp - name: Install clang-16 diff --git a/assembly/appimage/create-appimages.sh b/assembly/appimage/create-appimages.sh index 7bd68b6fbf..f6e2690727 100644 --- a/assembly/appimage/create-appimages.sh +++ b/assembly/appimage/create-appimages.sh @@ -27,7 +27,7 @@ for file in ../artifacts/*; do printf '[Desktop Entry]\nName='$appName'\nExec='$appName'\nIcon='$appName'\nType=Application\nCategories=Utility;\n' > $appName.AppDir/$appName.desktop cp ../ton.png $appName.AppDir/$appName.png cp $file $appName.AppDir/usr/bin/ - cp ../build/openssl_3/libcrypto.so.3 \ + cp ../openssl_3/libcrypto.so.3 \ /lib/$ARCH-linux-gnu/libatomic.so.1 \ /lib/$ARCH-linux-gnu/libsodium.so.23 \ /lib/$ARCH-linux-gnu/libz.so.1 \ @@ -37,6 +37,7 @@ for file in ../artifacts/*; do /lib/$ARCH-linux-gnu/libstdc++.so.6 \ /lib/$ARCH-linux-gnu/libgsl.so.27 \ /lib/$ARCH-linux-gnu/libblas.so.3 \ + /lib/$ARCH-linux-gnu/libgslcblas.so.0 \ $appName.AppDir/usr/lib/ chmod +x ./$appName.AppDir/usr/bin/$appName From bb0af68fb61eed2df855df289cd5137e542c56f3 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Thu, 25 Sep 2025 12:44:49 +0300 Subject: [PATCH 408/524] Set default global version to SUPPORTED_VERSION for getmethods in emulator (#1814) Co-authored-by: SpyCheese --- crypto/smc-envelope/SmartContract.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index a89cfa6d95..c067b88ff5 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -162,7 +162,9 @@ td::Ref prepare_vm_c7(SmartContract::Args args, td::Ref cod vm::load_cell_slice_ref(address), // myself:MsgAddressInt vm::StackEntry::maybe(config) // vm::StackEntry::maybe(td::Ref()) }; - if (args.config && args.config.value()->get_global_version() >= 4) { + + int global_version = args.config ? args.config.value()->get_global_version() : SUPPORTED_VERSION; + if (global_version >= 4) { tuple.push_back(vm::StackEntry::maybe(code)); // code:Cell tuple.push_back(block::CurrencyCollection::zero().as_vm_tuple()); // in_msg_value:[Integer (Maybe Cell)] tuple.push_back(td::zero_refint()); // storage_fees:Integer @@ -173,17 +175,18 @@ td::Ref prepare_vm_c7(SmartContract::Args args, td::Ref cod // prev_key_block:BlockId ] : PrevBlocksInfo tuple.push_back(args.prev_blocks_info ? args.prev_blocks_info.value() : vm::StackEntry{}); // prev_block_info } - if (args.config && args.config.value()->get_global_version() >= 6) { - tuple.push_back(args.config.value()->get_unpacked_config_tuple(now)); // unpacked_config_tuple + if (global_version >= 6) { + tuple.push_back(args.config ? args.config.value()->get_unpacked_config_tuple(now) + : vm::StackEntry{}); // unpacked_config_tuple tuple.push_back(td::zero_refint()); // due_payment // precompiled_gas_usage:(Maybe Integer) td::optional precompiled; - if (code.not_null()) { + if (code.not_null() && args.config) { precompiled = args.config.value()->get_precompiled_contracts_config().get_contract(code->get_hash().bits()); } tuple.push_back(precompiled ? td::make_refint(precompiled.value().gas_usage) : vm::StackEntry()); } - if (args.config && args.config.value()->get_global_version() >= 11) { + if (global_version >= 11) { tuple.push_back(block::transaction::Transaction::prepare_in_msg_params_tuple(nullptr, {}, {})); } auto tuple_ref = td::make_cnt_ref>(std::move(tuple)); @@ -263,7 +266,7 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Refdump(os, 2); LOG(DEBUG) << "VM stack:\n" << os.str(); } - int global_version = config ? config->get_global_version() : 0; + int global_version = config ? config->get_global_version() : SUPPORTED_VERSION; vm::VmState vm{state.code, global_version, std::move(stack), gas, 1, state.data, log}; vm.set_c7(std::move(c7)); vm.set_chksig_always_succeed(ignore_chksig); From e25bd3133dc82944539273fa13b62f9632607f78 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Thu, 2 Oct 2025 19:01:44 +0300 Subject: [PATCH 409/524] Leave HASHSU gas coas as is (#1818) Co-authored-by: SpyCheese --- crypto/vm/tonops.cpp | 6 +----- doc/GlobalVersions.md | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index d74ecb17f2..16e24b1464 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -620,11 +620,7 @@ int exec_compute_hash(VmState* st, int mode) { auto cs = stack.pop_cellslice(); CellBuilder cb; CHECK(cb.append_cellslice_bool(std::move(cs))); - if (st->get_global_version() >= 12) { - hash = cb.finalize_novm()->get_hash().as_array(); - } else { - hash = cb.finalize()->get_hash().as_array(); - } + hash = cb.finalize()->get_hash().as_array(); } else { // builder auto cb = stack.pop_builder(); hash = cb.write().finalize_novm()->get_hash().as_array(); diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index e2336e6948..79dbc6bcc5 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -268,7 +268,6 @@ new_bounce_body#fffffffe - `HASHBU` (`b - hash`) - same as `ENDC HASHCU`, but without gas cost for cell creation. Gas cost: `26`. ### Other TVM changes -- `HASHSU` (`s - hash`) now does not spend gas for cell creation. Gas cost: `26`. - `SENDMSG` instruction treats `extra_flags` field accordingly (see above). ### Other changes From bca9c48ed538e7a713701e8ad113e13e0fce04af Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 6 Oct 2025 11:28:50 +0300 Subject: [PATCH 410/524] test-smartcont: decrease number of keys in multisig --- crypto/test/test-smartcont.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/test/test-smartcont.cpp b/crypto/test/test-smartcont.cpp index 7f512ceae9..a278407218 100644 --- a/crypto/test/test-smartcont.cpp +++ b/crypto/test/test-smartcont.cpp @@ -547,8 +547,8 @@ bool operator<(const ton::MultisigWallet::Mask& a, const ton::MultisigWallet::Ma TEST(Smartcon, Multisig) { auto ms_lib = ton::MultisigWallet::create(); - int n = 100; - int k = 99; + int n = 50; + int k = 49; td::uint32 wallet_id = std::numeric_limits::max() - 3; std::vector keys; for (int i = 0; i < n; i++) { @@ -613,10 +613,10 @@ TEST(Smartcon, Multisig) { LOG(INFO) << "CODE: " << ans.code; LOG(INFO) << "GAS: " << ans.gas_used; } - for (int i = 0; i + 1 < 50; i++) { + for (int i = 0; i + 1 < 25; i++) { qb.sign(i, keys[i]); } - auto query = qb.create(49, keys[49]); + auto query = qb.create(24, keys[24]); CHECK(ms->get_n_k() == std::make_pair(n, k)); auto ans = ms.write().send_external_message(query, args()); @@ -629,10 +629,10 @@ TEST(Smartcon, Multisig) { { ton::MultisigWallet::QueryBuilder qb(wallet_id, query_id, vm::CellBuilder().finalize()); - for (int i = 50; i + 1 < 100; i++) { + for (int i = 25; i + 1 < 50; i++) { qb.sign(i, keys[i]); } - query = qb.create(99, keys[99]); + query = qb.create(49, keys[49]); } ans = ms.write().send_external_message(query, args()); From 8194beb0f21b79f75f8e748a02086614021f2adc Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 8 Oct 2025 10:53:10 +0300 Subject: [PATCH 411/524] Limit ttl and number of values in DHT (#1826) --- crypto/vm/boc.cpp | 4 +- crypto/vm/large-boc-serializer.cpp | 4 +- dht/dht-in.hpp | 11 ++- dht/dht.cpp | 134 ++++++++++++++--------------- 4 files changed, 75 insertions(+), 78 deletions(-) diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 93fc0a89d9..29c6c50c2d 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -575,8 +575,8 @@ td::Result BagOfCells::serialize_to_impl(WriterT& writer, int mode) if (dc_info.is_root_cell && (mode & Mode::WithTopHash)) { with_hash = true; } - unsigned char buf[256]; - int s = dc->serialize(buf, 256, with_hash); + unsigned char buf[Cell::max_serialized_bytes]; + int s = dc->serialize(buf, Cell::max_serialized_bytes, with_hash); writer.store_bytes(buf, s); DCHECK(dc->size_refs() == dc_info.ref_num); // std::cerr << (dc_info.is_special() ? '*' : ' ') << i << '<' << (int)dc_info.wt << ">:"; diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp index f6e35c6fe8..ec7eee298d 100644 --- a/crypto/vm/large-boc-serializer.cpp +++ b/crypto/vm/large-boc-serializer.cpp @@ -502,8 +502,8 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { if (dc_info.is_root_cell && (mode & Mode::WithTopHash)) { with_hash = true; } - unsigned char buf[256]; - int s = dc->serialize(buf, 256, with_hash); + unsigned char buf[Cell::max_serialized_bytes]; + int s = dc->serialize(buf, Cell::max_serialized_bytes, with_hash); writer.store_bytes(buf, s); DCHECK(dc->size_refs() == dc_info.get_ref_num()); unsigned ref_num = dc_info.get_ref_num(); diff --git a/dht/dht-in.hpp b/dht/dht-in.hpp index 0d668d438f..2f4e70416f 100644 --- a/dht/dht-in.hpp +++ b/dht/dht-in.hpp @@ -47,8 +47,6 @@ class DhtMemberImpl : public DhtMember { td::uint32 k_; td::uint32 a_; td::int32 network_id_{-1}; - td::uint32 max_cache_time_ = 60; - td::uint32 max_cache_size_ = 100; std::vector buckets_; @@ -57,17 +55,14 @@ class DhtMemberImpl : public DhtMember { // to be republished once in a while std::map our_values_; - std::map cached_values_; - td::ListNode cached_values_lru_; - std::map values_; + std::set> values_ttl_order_; td::Timestamp fill_att_ = td::Timestamp::in(0); td::Timestamp republish_att_ = td::Timestamp::in(0); DhtKeyId last_republish_key_ = DhtKeyId::zero(); DhtKeyId last_check_key_ = DhtKeyId::zero(); - adnl::AdnlNodeIdShort last_check_reverse_conn_ = adnl::AdnlNodeIdShort::zero(); struct ReverseConnection { adnl::AdnlNodeIdShort dht_node_; @@ -76,6 +71,7 @@ class DhtMemberImpl : public DhtMember { }; std::map reverse_connections_; std::set our_reverse_connections_; + std::set> reverse_connections_ttl_order_; class Callback : public adnl::Adnl::Callback { public: @@ -193,6 +189,9 @@ class DhtMemberImpl : public DhtMember { } void get_self_node(td::Promise promise) override; + + static constexpr size_t MAX_VALUES = 100000; + static constexpr size_t MAX_REVERSE_CONNECTIONS = 100000; }; } // namespace dht diff --git a/dht/dht.cpp b/dht/dht.cpp index b774b5f91d..da3b336010 100644 --- a/dht/dht.cpp +++ b/dht/dht.cpp @@ -228,25 +228,23 @@ void DhtMemberImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::dht_findVa td::Promise promise) { find_value_queries_++; auto it = values_.find(DhtKeyId{query.key_}); - if (it != values_.end() && it->second.expired()) { - values_.erase(it); - it = values_.end(); - } - if (it != values_.end()) { + if (it != values_.end() && !it->second.expired()) { promise.set_value(create_serialize_tl_object(it->second.tl())); return; } auto k = static_cast(query.k_); - if (k > max_k()) { - k = max_k(); - } + k = std::min(k, max_k()); auto R = get_nearest_nodes(DhtKeyId{query.key_}, k); promise.set_value(create_serialize_tl_object(R.tl())); } td::Status DhtMemberImpl::store_in(DhtValue value) { + if (value.ttl() > (td::uint32)td::Clocks::system() + 3600 + 60) { + // clients typically set ttl = 1 hour + return td::Status::Error("ttl is too big"); + } if (value.expired()) { VLOG(DHT_INFO) << this << ": dropping expired value: " << value.key_id() << " expire_at = " << value.ttl(); return td::Status::OK(); @@ -259,10 +257,12 @@ td::Status DhtMemberImpl::store_in(DhtValue value) { if (dist < k_ + 10) { auto it = values_.find(key_id); if (it != values_.end()) { + CHECK(values_ttl_order_.erase({it->second.ttl(), it->first})); it->second.update(std::move(value)); } else { - values_.emplace(key_id, std::move(value)); + it = values_.emplace(key_id, std::move(value)).first; } + CHECK(values_ttl_order_.insert({it->second.ttl(), it->first}).second); } else { VLOG(DHT_INFO) << this << ": dropping too remote value: " << value.key_id() << " distance = " << dist; } @@ -324,7 +324,13 @@ void DhtMemberImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::dht_regist TRY_RESULT_PROMISE(promise, encryptor, pub.create_encryptor()); TRY_STATUS_PROMISE(promise, encryptor->check_signature(to_sign, query.signature_)); DhtKeyId key_id = get_reverse_connection_key(client_id).compute_key_id(); - reverse_connections_[client_id] = ReverseConnection{src, key_id, td::Timestamp::at_unix(std::min(ttl, now + 300))}; + auto it = reverse_connections_.find(client_id); + if (it != reverse_connections_.end()) { + CHECK(reverse_connections_ttl_order_.erase({it->second.ttl_, client_id})); + } + auto &entry = reverse_connections_[client_id] = + ReverseConnection{src, key_id, td::Timestamp::at_unix(std::min(ttl, now + 300))}; + CHECK(reverse_connections_ttl_order_.insert({entry.ttl_, client_id}).second); promise.set_value(create_serialize_tl_object()); } @@ -332,25 +338,18 @@ void DhtMemberImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::dht_reques td::Promise promise) { adnl::AdnlNodeIdShort client{query.client_}; auto it = reverse_connections_.find(client); - if (it != reverse_connections_.end()) { - if (it->second.ttl_.is_in_past()) { - reverse_connections_.erase(it); - } else { - PublicKey pub{query.target_->id_}; - TRY_RESULT_PROMISE(promise, encryptor, pub.create_encryptor()); - TRY_STATUS_PROMISE(promise, - encryptor->check_signature(serialize_tl_object(query.target_, true), query.signature_)); - td::actor::send_closure(adnl_, &adnl::Adnl::send_message, id_, it->second.dht_node_, - create_serialize_tl_object( - std::move(query.target_), std::move(query.signature_), query.client_)); - promise.set_result(create_serialize_tl_object()); - return; - } + if (it != reverse_connections_.end() && !it->second.ttl_.is_in_past()) { + PublicKey pub{query.target_->id_}; + TRY_RESULT_PROMISE(promise, encryptor, pub.create_encryptor()); + TRY_STATUS_PROMISE(promise, encryptor->check_signature(serialize_tl_object(query.target_, true), query.signature_)); + td::actor::send_closure(adnl_, &adnl::Adnl::send_message, id_, it->second.dht_node_, + create_serialize_tl_object( + std::move(query.target_), std::move(query.signature_), query.client_)); + promise.set_result(create_serialize_tl_object()); + return; } auto k = static_cast(query.k_); - if (k > max_k()) { - k = max_k(); - } + k = std::min(k, max_k()); auto R = get_nearest_nodes(get_reverse_connection_key(client).compute_key_id(), k); promise.set_value(create_serialize_tl_object(R.tl())); } @@ -434,7 +433,7 @@ void DhtMemberImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice d auto S = [&]() -> td::Status { auto f = F.move_as_ok(); adnl::AdnlNodeIdShort client{f->client_}; - if (!our_reverse_connections_.count(client)) { + if (!our_reverse_connections_.contains(client)) { return td::Status::Error(PSTRING() << ": unknown id for reverse ping: " << client); } TRY_RESULT_PREFIX(node, adnl::AdnlNode::create(f->target_), "failed to parse node: "); @@ -501,7 +500,8 @@ void DhtMemberImpl::register_reverse_connection(adnl::AdnlNodeIdFull client, td: td::actor::send_closure(keyring_, &keyring::Keyring::sign_message, client_short.pubkey_hash(), register_reverse_connection_to_sign(client_short, id_, ttl), [=, print_id = print_id(), list = get_nearest_nodes(key_id, k_ * 2), SelfId = actor_id(this), - promise = std::move(promise)](td::Result R) mutable { + promise = std::move(promise), id = id_, k = k_, a = a_, network_id = network_id_, + client_only = client_only_, adnl = adnl_](td::Result R) mutable { TRY_RESULT_PROMISE_PREFIX(promise, signature, std::move(R), "Failed to sign: "); td::actor::send_closure(SelfId, &DhtMemberImpl::get_self_node, [=, list = std::move(list), signature = std::move(signature), @@ -509,8 +509,8 @@ void DhtMemberImpl::register_reverse_connection(adnl::AdnlNodeIdFull client, td: R.ensure(); td::actor::create_actor( "RegisterReverseQuery", key_id, std::move(client), ttl, - std::move(signature), print_id, id_, std::move(list), k_, a_, - network_id_, R.move_as_ok(), client_only_, SelfId, adnl_, + std::move(signature), print_id, id, std::move(list), k, a, + network_id, R.move_as_ok(), client_only, SelfId, adnl, std::move(promise)) .release(); }); @@ -534,25 +534,22 @@ void DhtMemberImpl::request_reverse_ping(adnl::AdnlNode target, adnl::AdnlNodeId void DhtMemberImpl::request_reverse_ping_cont(adnl::AdnlNode target, td::BufferSlice signature, adnl::AdnlNodeIdShort client, td::Promise promise) { auto it = reverse_connections_.find(client); - if (it != reverse_connections_.end()) { - if (it->second.ttl_.is_in_past()) { - reverse_connections_.erase(it); - } else { - td::actor::send_closure(adnl_, &adnl::Adnl::send_message, id_, it->second.dht_node_, - create_serialize_tl_object( - target.tl(), std::move(signature), client.bits256_value())); - promise.set_result(td::Unit()); - return; - } + if (it != reverse_connections_.end() && !it->second.ttl_.is_in_past()) { + td::actor::send_closure(adnl_, &adnl::Adnl::send_message, id_, it->second.dht_node_, + create_serialize_tl_object( + target.tl(), std::move(signature), client.bits256_value())); + promise.set_result(td::Unit()); + return; } auto key_id = get_reverse_connection_key(client).compute_key_id(); get_self_node([=, target = std::move(target), signature = std::move(signature), promise = std::move(promise), SelfId = actor_id(this), print_id = print_id(), list = get_nearest_nodes(key_id, k_ * 2), - client_only = client_only_](td::Result R) mutable { + client_only = client_only_, id = id_, k = k_, a = a_, adnl = adnl_, + network_id = network_id_](td::Result R) mutable { R.ensure(); td::actor::create_actor( - "RequestReversePing", client, std::move(target), std::move(signature), print_id, id_, std::move(list), k_, a_, - network_id_, R.move_as_ok(), client_only, SelfId, adnl_, std::move(promise)) + "RequestReversePing", client, std::move(target), std::move(signature), print_id, id, std::move(list), k, a, + network_id, R.move_as_ok(), client_only, SelfId, adnl, std::move(promise)) .release(); }); } @@ -561,6 +558,9 @@ void DhtMemberImpl::check() { VLOG(DHT_INFO) << this << ": ping=" << ping_queries_ << " fnode=" << find_node_queries_ << " fvalue=" << find_value_queries_ << " store=" << store_queries_ << " addrlist=" << get_addr_list_queries_; + VLOG(DHT_INFO) << this << ": values=" << values_.size() << " our_values=" << our_values_.size(); + VLOG(DHT_INFO) << this << ": reverse_conns=" << reverse_connections_.size() + << " our_reverse_conns=" << our_reverse_connections_.size(); for (auto &bucket : buckets_) { bucket.check(client_only_, adnl_, actor_id(this), id_); } @@ -568,10 +568,15 @@ void DhtMemberImpl::check() { save_to_db(); } - if (values_.size() > 0) { + for (auto it = values_ttl_order_.begin(); + it != values_ttl_order_.end() && (it->first < td::Clocks::system() || values_.size() > MAX_VALUES);) { + CHECK(values_.erase(it->second)); + it = values_ttl_order_.erase(it); + } + if (!values_.empty()) { auto it = values_.lower_bound(last_check_key_); if (it != values_.end() && it->first == last_check_key_) { - it++; + ++it; } if (it == values_.end()) { it = values_.begin(); @@ -579,14 +584,10 @@ void DhtMemberImpl::check() { td::uint32 cnt = 0; auto s = last_check_key_; - while (values_.size() > 0 && cnt < 1 && it->first != s) { + while (!values_.empty() && cnt < 1 && it->first != s) { last_check_key_ = it->first; cnt++; - if (it->second.expired()) { - it = values_.erase(it); - - // do not republish soon-to-be-expired values - } else if (it->second.ttl() > td::Clocks::system() + 60) { + if (it->second.ttl() > td::Clocks::system() + 60) { auto dist = distance(it->first, k_ + 10); if (dist == 0) { @@ -598,16 +599,17 @@ void DhtMemberImpl::check() { }); send_store(it->second.clone(), std::move(P)); } - it++; + ++it; } else if (dist >= k_ + 10) { + CHECK(values_ttl_order_.erase({it->second.ttl(), it->first})); it = values_.erase(it); } else { - it++; + ++it; } } else { - it++; + ++it; } - if (values_.size() == 0) { + if (values_.empty()) { break; } if (it == values_.end()) { @@ -615,21 +617,17 @@ void DhtMemberImpl::check() { } } } - if (reverse_connections_.size() > 0) { - auto it = reverse_connections_.upper_bound(last_check_reverse_conn_); - if (it == reverse_connections_.end()) { - it = reverse_connections_.begin(); - } - last_check_reverse_conn_ = it->first; - if (it->second.ttl_.is_in_past()) { - reverse_connections_.erase(it); - } + for (auto it = reverse_connections_ttl_order_.begin(); + it != reverse_connections_ttl_order_.end() && + (it->first.is_in_past() || reverse_connections_.size() > MAX_REVERSE_CONNECTIONS);) { + CHECK(reverse_connections_.erase(it->second)); + it = reverse_connections_ttl_order_.erase(it); } if (republish_att_.is_in_past()) { auto it = our_values_.lower_bound(last_republish_key_); if (it != our_values_.end() && it->first == last_republish_key_) { - it++; + ++it; } if (it == our_values_.end()) { it = our_values_.begin(); @@ -705,8 +703,8 @@ void DhtMemberImpl::send_store(DhtValue value, td::Promise promise) { } void DhtMemberImpl::get_self_node(td::Promise promise) { - auto P = td::PromiseCreator::lambda([promise = std::move(promise), print_id = print_id(), id = id_, - keyring = keyring_, client_only = client_only_, + auto P = td::PromiseCreator::lambda([promise = std::move(promise), id = id_, keyring = keyring_, + client_only = client_only_, network_id = network_id_](td::Result R) mutable { R.ensure(); auto node = R.move_as_ok(); From 34823b1ea378edbe3bc59f3bcc48126480a0b768 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Wed, 8 Oct 2025 14:47:53 +0300 Subject: [PATCH 412/524] Add changelog --- Changelog.md | 12 ++++++++++++ recent_changelog.md | 10 ++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/Changelog.md b/Changelog.md index b104d10ad2..8c6ed697d7 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,15 @@ +## 2025.10 Update + +1. [TVM version v12](./doc/GlobalVersions.md): full bounces, new `BTOS` and `HASHBU` instuctions, limit on contract size in masterchain. +2. Optimistic collation/validation: allow nodes to generate and check block candidates before previous block is fully signed (not fully activated yet). +3. Introduced custom block compression algorithm. +4. Overlay improvements: improved overlay discovery on shard configuration update, private externals in custom overlays. +5. Various improvements: session stats, telemetry in fast-sync overlay, earlier block broadcasts, limiting ttl for values in DHT, fixing search by utime in native blockexplorer, faster downloading candidates in validator session, parallelization of storing to cell_db, avoiding touching packfiles on startup. + +Besides the work of the core team, this update is based on the efforts of the Tonstudio team: @hacker-volodya @Shvandre; and @mkiesel (avoiding touching packfiles on startup). + + + ## 2025.07 Accelerator Update Separation of validation and collation processes that allows to host them on independent machines and achieve full horizontal scaling. [More details in documentation](https://docs.ton.org/v3/documentation/infra/nodes/validation/collators) diff --git a/recent_changelog.md b/recent_changelog.md index 6b65d21836..fc4ae86c4a 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,3 +1,9 @@ -## 2025.07 Accelerator Update +## 2025.10 Update -Separation of validation and collation processes that allows to host them on independent machines and achieve full horizontal scaling. [More details in documentation](https://docs.ton.org/v3/documentation/infra/nodes/validation/collators) +1. [TVM version v12](./doc/GlobalVersions.md): full bounces, new `BTOS` and `HASHBU` instuctions, limit on contract size in masterchain. +2. Optimistic collation/validation: allow nodes to generate and check block candidates before previous block is fully signed (not fully activated yet). +3. Introduced custom block compression algorithm. +4. Overlay improvements: improved overlay discovery on shard configuration update, private externals in custom overlays. +5. Various improvements: session stats, telemetry in fast-sync overlay, earlier block broadcasts, limiting ttl for values in DHT, fixing search by utime in native blockexplorer, faster downloading candidates in validator session, parallelization of storing to cell_db, avoiding touching packfiles on startup. + +Besides the work of the core team, this update is based on the efforts of the Tonstudio team: @hacker-volodya @Shvandre; and @mkiesel (avoiding touching packfiles on startup). From 06675603e9f854b325026607b1c085ee0a05c158 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 20 Oct 2025 17:36:55 +0300 Subject: [PATCH 413/524] Debug logs for applying blocks --- validator/apply-block.cpp | 29 +++++++++++++++++++++-------- validator/impl/accept-block.cpp | 4 ++++ validator/impl/check-proof.cpp | 11 ++++++++++- validator/impl/fabric.cpp | 20 +++++++++++--------- validator/manager.cpp | 7 ++++--- validator/net/download-block.cpp | 2 +- validator/validate-broadcast.cpp | 22 +++++++++++++++++++--- 7 files changed, 70 insertions(+), 25 deletions(-) diff --git a/validator/apply-block.cpp b/validator/apply-block.cpp index 3c6a931cd7..55f5071029 100644 --- a/validator/apply-block.cpp +++ b/validator/apply-block.cpp @@ -29,7 +29,7 @@ namespace validator { void ApplyBlock::abort_query(td::Status reason) { if (promise_) { - VLOG(VALIDATOR_WARNING) << "aborting apply block query for " << id_ << ": " << reason; + VLOG(VALIDATOR_WARNING) << "aborting apply block query for " << id_.to_str() << ": " << reason; promise_.set_error(std::move(reason)); } stop(); @@ -51,7 +51,7 @@ void ApplyBlock::alarm() { } void ApplyBlock::start_up() { - VLOG(VALIDATOR_DEBUG) << "running apply_block for " << id_; + VLOG(VALIDATOR_DEBUG) << "running apply_block for " << id_.to_str() << ", mc_seqno=" << masterchain_block_id_.seqno(); if (id_.is_masterchain()) { masterchain_block_id_ = id_; @@ -71,6 +71,7 @@ void ApplyBlock::start_up() { } void ApplyBlock::got_block_handle(BlockHandle handle) { + VLOG(VALIDATOR_DEBUG) << "got_block_handle"; handle_ = std::move(handle); if (handle_->is_applied() && (!handle_->id().is_masterchain() || handle_->processed())) { @@ -79,6 +80,7 @@ void ApplyBlock::got_block_handle(BlockHandle handle) { } if (handle_->is_applied()) { + VLOG(VALIDATOR_DEBUG) << "already applied"; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), seqno = handle_->id().id.seqno](td::Result R) { R.ensure(); @@ -94,16 +96,19 @@ void ApplyBlock::got_block_handle(BlockHandle handle) { } if (handle_->id().id.seqno == 0) { + VLOG(VALIDATOR_DEBUG) << "seqno == 0"; written_block_data(); return; } if (handle_->is_archived()) { + VLOG(VALIDATOR_DEBUG) << "already archived"; finish_query(); return; } if (handle_->received()) { + VLOG(VALIDATOR_DEBUG) << "already received"; written_block_data(); return; } @@ -117,6 +122,7 @@ void ApplyBlock::got_block_handle(BlockHandle handle) { } }); + VLOG(VALIDATOR_DEBUG) << "storing block data"; td::actor::send_closure(manager_, &ValidatorManager::set_block_data, handle_, block_, std::move(P)); } else { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result> R) { @@ -128,13 +134,14 @@ void ApplyBlock::got_block_handle(BlockHandle handle) { } }); + VLOG(VALIDATOR_DEBUG) << "wait for block data"; td::actor::send_closure(manager_, &ValidatorManager::wait_block_data, handle_, apply_block_priority(), timeout_, std::move(P)); } } void ApplyBlock::written_block_data() { - VLOG(VALIDATOR_DEBUG) << "apply block: written block data for " << id_; + VLOG(VALIDATOR_DEBUG) << "written_block_data"; if (!handle_->id().seqno()) { CHECK(handle_->inited_split_after()); CHECK(handle_->inited_state_root_hash()); @@ -165,24 +172,26 @@ void ApplyBlock::written_block_data() { } }); + VLOG(VALIDATOR_DEBUG) << "wait_block_state"; td::actor::send_closure(manager_, &ValidatorManager::wait_block_state, handle_, apply_block_priority(), timeout_, true, std::move(P)); } } void ApplyBlock::got_cur_state(td::Ref state) { - VLOG(VALIDATOR_DEBUG) << "apply block: received state for " << id_; + VLOG(VALIDATOR_DEBUG) << "got_cur_state"; state_ = std::move(state); CHECK(handle_->received_state()); written_state(); } void ApplyBlock::written_state() { + VLOG(VALIDATOR_DEBUG) << "written_state"; if (handle_->is_applied() && handle_->processed()) { finish_query(); return; } - VLOG(VALIDATOR_DEBUG) << "apply block: setting next for parents of " << id_; + VLOG(VALIDATOR_DEBUG) << "setting next for parents"; if (handle_->id().id.seqno != 0 && !handle_->is_applied()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { @@ -208,12 +217,13 @@ void ApplyBlock::written_state() { } void ApplyBlock::written_next() { + VLOG(VALIDATOR_DEBUG) << "written_next"; if (handle_->is_applied() && handle_->processed()) { finish_query(); return; } - VLOG(VALIDATOR_DEBUG) << "apply block: applying parents of " << id_; + VLOG(VALIDATOR_DEBUG) << "applying parents"; if (handle_->id().id.seqno != 0 && !handle_->is_applied()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { @@ -241,7 +251,8 @@ void ApplyBlock::written_next() { } void ApplyBlock::applied_prev() { - VLOG(VALIDATOR_DEBUG) << "apply block: waiting manager's confirm for " << id_; + VLOG(VALIDATOR_DEBUG) << "applying parents"; + VLOG(VALIDATOR_DEBUG) << "applied_prev, waiting manager's confirm"; if (!id_.is_masterchain()) { handle_->set_masterchain_ref_block(masterchain_block_id_.seqno()); } @@ -253,10 +264,11 @@ void ApplyBlock::applied_prev() { } }); td::actor::send_closure(manager_, &ValidatorManager::new_block, handle_, state_, std::move(P)); + } void ApplyBlock::applied_set() { - VLOG(VALIDATOR_DEBUG) << "apply block: setting apply bit for " << id_; + VLOG(VALIDATOR_DEBUG) << "applied_set"; handle_->set_applied(); if (handle_->id().seqno() > 0) { CHECK(handle_->handle_moved_to_archive()); @@ -270,6 +282,7 @@ void ApplyBlock::applied_set() { td::actor::send_closure(SelfId, &ApplyBlock::finish_query); } }); + VLOG(VALIDATOR_DEBUG) << "flush handle"; handle_->flush(manager_, handle_, std::move(P)); } else { finish_query(); diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index 8839940895..8de97b6f4d 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -350,6 +350,7 @@ bool AcceptBlockQuery::check_send_error(td::actor::ActorId Sel } void AcceptBlockQuery::finish_query() { + VLOG(VALIDATOR_DEBUG) << "finish_query()"; if (apply_) { ValidatorInvariants::check_post_accept(handle_); } @@ -448,6 +449,7 @@ void AcceptBlockQuery::got_block_candidate_data(td::BufferSlice data) { } void AcceptBlockQuery::got_block_handle_cont() { + VLOG(VALIDATOR_DEBUG) << "got_block_handle_cont()"; if (data_.not_null() && !handle_->received()) { td::actor::send_closure( manager_, &ValidatorManager::set_block_data, handle_, data_, [SelfId = actor_id(this)](td::Result R) { @@ -514,9 +516,11 @@ void AcceptBlockQuery::written_block_info() { td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::got_prev_state, R.move_as_ok()); }); + VLOG(VALIDATOR_DEBUG) << "wait_prev_block_state"; td::actor::send_closure(manager_, &ValidatorManager::wait_prev_block_state, handle_, priority(), timeout_, std::move(P)); } else { + VLOG(VALIDATOR_DEBUG) << "wait_block_data"; td::actor::send_closure(manager_, &ValidatorManager::wait_block_data, handle_, priority(), timeout_, [SelfId = actor_id(this)](td::Result> R) { check_send_error(SelfId, R) || diff --git a/validator/impl/check-proof.cpp b/validator/impl/check-proof.cpp index 12d336ad18..05d5c84ba1 100644 --- a/validator/impl/check-proof.cpp +++ b/validator/impl/check-proof.cpp @@ -43,7 +43,7 @@ void CheckProof::alarm() { void CheckProof::abort_query(td::Status reason) { if (promise_) { - VLOG(VALIDATOR_WARNING) << "aborting check proof for " << id_ << " query: " << reason; + VLOG(VALIDATOR_WARNING) << "aborting check proof for " << id_.to_str() << " query: " << reason; promise_.set_error(std::move(reason)); } stop(); @@ -271,6 +271,7 @@ bool CheckProof::init_parse(bool is_aux) { } void CheckProof::start_up() { + VLOG(VALIDATOR_DEBUG) << "started check proof for " << id_.to_str() << ", mode=" << mode_; alarm_timestamp() = timeout_; auto res = vm::std_boc_deserialize(proof_->data()); @@ -324,6 +325,7 @@ void CheckProof::start_up() { } void CheckProof::got_block_handle(BlockHandle handle) { + VLOG(VALIDATOR_DEBUG) << "got_block_handle"; handle_ = std::move(handle); CHECK(handle_); if (!is_proof() || skip_check_signatures_) { @@ -353,6 +355,7 @@ void CheckProof::got_block_handle(BlockHandle handle) { } void CheckProof::got_masterchain_state(td::Ref state) { + VLOG(VALIDATOR_DEBUG) << "got_masterchain_state #" << state->get_seqno(); CHECK(is_proof()); state_ = std::move(state); @@ -366,6 +369,7 @@ void CheckProof::got_masterchain_state(td::Ref state) { } void CheckProof::process_masterchain_state() { + VLOG(VALIDATOR_DEBUG) << "process_masterchain_state"; CHECK(is_proof()); CHECK(state_.not_null()); @@ -396,6 +400,7 @@ void CheckProof::process_masterchain_state() { } void CheckProof::check_signatures(Ref s) { + VLOG(VALIDATOR_DEBUG) << "check_signatures"; if (s->get_catchain_seqno() != catchain_seqno_) { abort_query(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad validator catchain seqno: expected " << s->get_catchain_seqno() << ", found " @@ -449,6 +454,7 @@ void CheckProof::check_signatures(Ref s) { } void CheckProof::got_block_handle_2(BlockHandle handle) { + VLOG(VALIDATOR_DEBUG) << "got_block_handle_2 " << handle->id().id.to_str(); handle_ = std::move(handle); handle_->set_split(before_split_); @@ -468,16 +474,19 @@ void CheckProof::got_block_handle_2(BlockHandle handle) { // do not save proof if we skipped signatures auto proof = Ref(proof_); CHECK(proof.not_null()); + VLOG(VALIDATOR_DEBUG) << "set_block_proof"; td::actor::send_closure_later(manager_, &ValidatorManager::set_block_proof, handle_, std::move(proof), std::move(P)); } else if (is_proof()) { auto proof = Ref(proof_); CHECK(proof.not_null()); CHECK(sig_ok_); + VLOG(VALIDATOR_DEBUG) << "set_block_proof"; td::actor::send_closure_later(manager_, &ValidatorManager::set_block_proof, handle_, std::move(proof), std::move(P)); } else { CHECK(proof_.not_null()); + VLOG(VALIDATOR_DEBUG) << "set_block_proof_link"; td::actor::send_closure_later(manager_, &ValidatorManager::set_block_proof_link, handle_, proof_, std::move(P)); } } diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 4cf3349289..af171c3c9e 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -158,37 +158,39 @@ void run_hardfork_accept_block_query(BlockIdExt id, td::Ref data, void run_apply_block_query(BlockIdExt id, td::Ref block, BlockIdExt masterchain_block_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise) { - td::actor::create_actor(PSTRING() << "apply " << id, id, std::move(block), masterchain_block_id, manager, - timeout, std::move(promise)) + td::actor::create_actor(PSTRING() << "apply" << id.id.to_str(), id, std::move(block), + masterchain_block_id, manager, timeout, std::move(promise)) .release(); } void run_check_proof_query(BlockIdExt id, td::Ref proof, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, bool skip_check_signatures) { - td::actor::create_actor("checkproof", id, std::move(proof), manager, timeout, std::move(promise), - skip_check_signatures) + td::actor::create_actor(PSTRING() << "checkproof" << id.id.to_str(), id, std::move(proof), manager, + timeout, std::move(promise), skip_check_signatures) .release(); } void run_check_proof_query(BlockIdExt id, td::Ref proof, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, td::Ref rel_key_block_proof, bool skip_check_signatures) { - td::actor::create_actor("checkproof/key", id, std::move(proof), manager, timeout, std::move(promise), - skip_check_signatures, std::move(rel_key_block_proof)) + td::actor::create_actor(PSTRING() << "checkproof/key" << id.id.to_str(), id, std::move(proof), manager, + timeout, std::move(promise), skip_check_signatures, + std::move(rel_key_block_proof)) .release(); } void run_check_proof_query(BlockIdExt id, td::Ref proof, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, td::Ref rel_mc_state, bool skip_check_signatures) { - td::actor::create_actor("checkproof/st", id, std::move(proof), manager, timeout, std::move(promise), - skip_check_signatures, std::move(rel_mc_state)) + td::actor::create_actor(PSTRING() << "checkproof/st" << id.id.to_str(), id, std::move(proof), manager, + timeout, std::move(promise), skip_check_signatures, std::move(rel_mc_state)) .release(); } void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise) { - td::actor::create_actor("checkprooflink", id, std::move(proof), manager, timeout, std::move(promise)) + td::actor::create_actor(PSTRING() << "checkprooflink" << id.id.to_str(), id, std::move(proof), manager, + timeout, std::move(promise)) .release(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index a5315788dc..696e7b3691 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -229,9 +229,10 @@ void ValidatorManagerImpl::new_block_broadcast(BlockBroadcast broadcast, td::Pro } promise.set_result(std::move(R)); }; - td::actor::create_actor("broadcast", std::move(broadcast), last_masterchain_block_handle_, - last_masterchain_state_, last_known_key_block_handle_, actor_id(this), - td::Timestamp::in(2.0), std::move(promise)) + BlockIdExt block_id = broadcast.block_id; + td::actor::create_actor( + PSTRING() << "broadcast" << block_id.id.to_str(), std::move(broadcast), last_masterchain_block_handle_, + last_masterchain_state_, last_known_key_block_handle_, actor_id(this), td::Timestamp::in(2.0), std::move(promise)) .release(); } diff --git a/validator/net/download-block.cpp b/validator/net/download-block.cpp index c60955ed25..75dc57fc44 100644 --- a/validator/net/download-block.cpp +++ b/validator/net/download-block.cpp @@ -294,7 +294,7 @@ void DownloadBlock::got_block_partial_proof(td::BufferSlice proof) { } void DownloadBlock::checked_block_proof() { - VLOG(FULL_NODE_DEBUG) << "checked proof for " << block_id_; + VLOG(FULL_NODE_DEBUG) << "checked proof for " << block_id_.to_str(); if (!handle_) { CHECK(!short_); diff --git a/validator/validate-broadcast.cpp b/validator/validate-broadcast.cpp index 59cb1faca4..0450496973 100644 --- a/validator/validate-broadcast.cpp +++ b/validator/validate-broadcast.cpp @@ -28,7 +28,8 @@ namespace validator { void ValidateBroadcast::abort_query(td::Status reason) { if (promise_) { - VLOG(VALIDATOR_WARNING) << "aborting validate broadcast query for " << broadcast_.block_id << ": " << reason; + VLOG(VALIDATOR_WARNING) << "aborting validate broadcast query for " << broadcast_.block_id.to_str() << ": " + << reason; promise_.set_error(std::move(reason)); } stop(); @@ -36,7 +37,7 @@ void ValidateBroadcast::abort_query(td::Status reason) { void ValidateBroadcast::finish_query() { if (promise_) { - VLOG(VALIDATOR_DEBUG) << "validated broadcast for " << broadcast_.block_id; + VLOG(VALIDATOR_DEBUG) << "validated broadcast for " << broadcast_.block_id.to_str(); promise_.set_result(td::Unit()); } stop(); @@ -47,7 +48,9 @@ void ValidateBroadcast::alarm() { } void ValidateBroadcast::start_up() { - VLOG(VALIDATOR_DEBUG) << "received broadcast for " << broadcast_.block_id; + VLOG(VALIDATOR_DEBUG) << "received broadcast for " << broadcast_.block_id.to_str() + << " : last_mc_seqno=" << last_masterchain_state_->get_seqno() + << " last_key_block_seqno=" << last_known_masterchain_block_handle_->id().seqno(); alarm_timestamp() = timeout_; auto hash = sha256_bits256(broadcast_.data.as_slice()); @@ -127,6 +130,7 @@ void ValidateBroadcast::start_up() { } void ValidateBroadcast::got_key_block_id(BlockIdExt block_id) { + VLOG(VALIDATOR_DEBUG) << "got_key_block_id " << block_id.id.to_str(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &ValidateBroadcast::abort_query, @@ -139,6 +143,7 @@ void ValidateBroadcast::got_key_block_id(BlockIdExt block_id) { } void ValidateBroadcast::got_key_block_handle(ConstBlockHandle handle) { + VLOG(VALIDATOR_DEBUG) << "got_key_block_handle " << handle->id().id.to_str(); if (handle->id().seqno() == 0) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { @@ -172,6 +177,7 @@ void ValidateBroadcast::got_key_block_handle(ConstBlockHandle handle) { } void ValidateBroadcast::got_key_block_proof_link(td::Ref key_proof_link) { + VLOG(VALIDATOR_DEBUG) << "got_key_block_proof_link"; key_proof_link_ = key_proof_link; auto confR = key_proof_link->get_key_block_config(); if (confR.is_error()) { @@ -182,6 +188,7 @@ void ValidateBroadcast::got_key_block_proof_link(td::Ref key_proof_li } void ValidateBroadcast::got_zero_state(td::Ref state) { + VLOG(VALIDATOR_DEBUG) << "got_zero_state"; zero_state_ = state; auto confR = state->get_config_holder(); if (confR.is_error()) { @@ -192,6 +199,7 @@ void ValidateBroadcast::got_zero_state(td::Ref state) { } void ValidateBroadcast::check_signatures_common(td::Ref conf) { + VLOG(VALIDATOR_DEBUG) << "checking signatures"; auto val_set = conf->get_validator_set(broadcast_.block_id.shard_full(), header_info_.utime, header_info_.cc_seqno); if (val_set.is_null()) { abort_query(td::Status::Error(ErrorCode::notready, "failed to compute validator set")); @@ -216,6 +224,7 @@ void ValidateBroadcast::check_signatures_common(td::Ref conf) { } void ValidateBroadcast::checked_signatures() { + VLOG(VALIDATOR_DEBUG) << "checked_signatures"; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &ValidateBroadcast::abort_query, R.move_as_error_prefix("db error: ")); @@ -228,6 +237,7 @@ void ValidateBroadcast::checked_signatures() { } void ValidateBroadcast::got_block_handle(BlockHandle handle) { + VLOG(VALIDATOR_DEBUG) << "got_block_handle " << handle->id().id.to_str(); handle_ = std::move(handle); auto dataR = create_block(broadcast_.block_id, broadcast_.data.clone()); @@ -250,10 +260,12 @@ void ValidateBroadcast::got_block_handle(BlockHandle handle) { } }); + VLOG(VALIDATOR_DEBUG) << "writing block data for " << handle_->id().id.to_str(); td::actor::send_closure(manager_, &ValidatorManager::set_block_data, handle_, data_, std::move(P)); } void ValidateBroadcast::written_block_data() { + VLOG(VALIDATOR_DEBUG) << "written_block_data"; if (handle_->id().is_masterchain()) { if (handle_->inited_proof()) { checked_proof(); @@ -267,6 +279,7 @@ void ValidateBroadcast::written_block_data() { td::actor::send_closure(SelfId, &ValidateBroadcast::checked_proof); } }); + VLOG(VALIDATOR_DEBUG) << "checking proof"; if (!key_proof_link_.is_null()) { run_check_proof_query(broadcast_.block_id, proof_, manager_, timeout_, std::move(P), key_proof_link_); } else { @@ -288,11 +301,13 @@ void ValidateBroadcast::written_block_data() { td::actor::send_closure(SelfId, &ValidateBroadcast::checked_proof); } }); + VLOG(VALIDATOR_DEBUG) << "checking proof link"; run_check_proof_link_query(broadcast_.block_id, proof_link_, manager_, timeout_, std::move(P)); } } void ValidateBroadcast::checked_proof() { + VLOG(VALIDATOR_DEBUG) << "checked_proof"; if (handle_->inited_proof() && handle_->is_key_block()) { td::actor::send_closure(manager_, &ValidatorManager::update_last_known_key_block, handle_, false); } @@ -305,6 +320,7 @@ void ValidateBroadcast::checked_proof() { } }); + VLOG(VALIDATOR_DEBUG) << "apply block"; td::actor::create_actor("applyblock", handle_->id(), data_, handle_->id(), manager_, timeout_, std::move(P)) .release(); From 590f1c3fc4d27f6208f18e809a9df7f595293a6e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 20 Oct 2025 17:44:16 +0300 Subject: [PATCH 414/524] Change the behavior of extra_flags (#1846) --- crypto/block/transaction.cpp | 32 ++++++++++++++++++++------------ crypto/block/transaction.h | 1 + doc/GlobalVersions.md | 5 +++++ 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 2d65c72295..0ef94b0766 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -875,9 +875,9 @@ bool Transaction::unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* td::RefInt256 ihr_fee; if (cfg->global_version >= 12) { ihr_fee = td::zero_refint(); - td::RefInt256 extra_flags = tlb::t_Grams.as_integer(in_msg_info.extra_flags); - new_bounce_format = extra_flags->get_bit(0); - new_bounce_format_full_body = extra_flags->get_bit(1); + in_msg_extra_flags = tlb::t_Grams.as_integer(in_msg_info.extra_flags); + new_bounce_format = in_msg_extra_flags->get_bit(0); + new_bounce_format_full_body = in_msg_extra_flags->get_bit(1); } else { // Legacy: extra_flags was previously ihr_fee ihr_fee = tlb::t_Grams.as_integer(in_msg_info.extra_flags); @@ -2692,6 +2692,13 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (cfg.disable_ihr_flag) { info.ihr_disabled = true; } + if (cfg.global_version >= 12) { + td::RefInt256 extra_flags = tlb::t_Grams.as_integer(info.extra_flags); + if (td::cmp(extra_flags & td::make_refint(3), extra_flags) != 0) { + LOG(DEBUG) << "invalid extra_flags in a proposed outbound message"; + return check_skip_invalid(45); + } + } } // set created_at and created_lt to correct values info.created_at = now; @@ -3361,15 +3368,16 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { end_lt++; info.created_at = now; vm::CellBuilder cb; - CHECK(cb.store_long_bool(5, 4) // int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool - && cb.append_cellslice_bool(info.src) // src:MsgAddressInt - && cb.append_cellslice_bool(info.dest) // dest:MsgAddressInt - && msg_balance.store(cb) // value:CurrencyCollection - && block::tlb::t_Grams.store_long(cb, 0) // extra_flags:(VarUInteger 16) - && block::tlb::t_Grams.store_long(cb, bp.fwd_fees) // fwd_fee:Grams - && cb.store_long_bool(info.created_lt, 64) // created_lt:uint64 - && cb.store_long_bool(info.created_at, 32) // created_at:uint32 - && cb.store_bool_bool(false)); // init:(Maybe ...) + CHECK(cb.store_long_bool(5, 4) // int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool + && cb.append_cellslice_bool(info.src) // src:MsgAddressInt + && cb.append_cellslice_bool(info.dest) // dest:MsgAddressInt + && msg_balance.store(cb) // value:CurrencyCollection + && block::tlb::t_Grams.store_integer_ref( + cb, in_msg_extra_flags & td::make_refint(3)) // extra_flags:(VarUInteger 16) + && block::tlb::t_Grams.store_long(cb, bp.fwd_fees) // fwd_fee:Grams + && cb.store_long_bool(info.created_lt, 64) // created_lt:uint64 + && cb.store_long_bool(info.created_at, 32) // created_at:uint32 + && cb.store_bool_bool(false)); // init:(Maybe ...) if (cb.can_extend_by(1 + body.size(), body.size_refs())) { // body:(Either X ^X) -> left X CHECK(cb.store_bool_bool(false) && cb.append_builder_bool(body)); diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index c4c13a5587..5feb7c747f 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -358,6 +358,7 @@ struct Transaction { bool bounce_enabled{false}; bool in_msg_extern{false}; gen::CommonMsgInfo::Record_int_msg_info in_msg_info; + td::RefInt256 in_msg_extra_flags = td::zero_refint(); bool new_bounce_format{false}; bool new_bounce_format_full_body{false}; bool use_msg_state{false}; diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 79dbc6bcc5..bf62e6adc7 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -237,6 +237,9 @@ This field does not represent fees. `ihr_fee` is always zero since version 11, s `(extra_flags & 1) = 1` enables the new bounce format for the message. The bounced message contains information about the transaction. If `(extra_flags & 3) = 3`, the bounced message contains the whole body of the original message. Otherwise, only the bits from the root of the original body are returned. + +All other bits in `extra_flags` are reserved for future use and are not allowed now (internal messages with flags other than `0..3` are invalid). + When the message with new bounce flag is bounced, the bounced message body has the following format (`new_bounce_body`): ``` _ value:CurrencyCollection created_lt:uint64 created_at:uint32 = NewBounceOriginalInfo; @@ -263,6 +266,8 @@ new_bounce_body#fffffffe - `compute_phase` - exists if it was not skipped (`bounced_by_phase > 0`): - `gas_used`, `vm_steps` - same as in `TrComputePhase` of the transaction. +The bounced message has the same 0th and 1st bits in `extra_flags` as the original message. + ### New TVM instructions - `BTOS` (`b - s`) - same as `ENDC CTOS`, but without gas cost for cell creation and loading. Gas cost: `26`. - `HASHBU` (`b - hash`) - same as `ENDC HASHCU`, but without gas cost for cell creation. Gas cost: `26`. From 2aed6e42cc684f326c0261440a25b72143eadcf8 Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Mon, 20 Oct 2025 11:19:55 -0400 Subject: [PATCH 415/524] Run CI on pull request (#1844) --- .github/workflows/build-ton-linux-android-tonlib.yml | 7 ++++--- .github/workflows/build-ton-linux-arm64-appimage.yml | 6 ++++-- .github/workflows/build-ton-linux-arm64-shared.yml | 4 +++- .github/workflows/build-ton-linux-x86-64-appimage.yml | 6 ++++-- .github/workflows/build-ton-linux-x86-64-shared.yml | 8 +++++--- .github/workflows/build-ton-macos-13-x86-64-portable.yml | 6 ++++-- .github/workflows/build-ton-macos-14-arm64-portable.yml | 6 ++++-- .github/workflows/build-ton-macos-15-arm64-shared.yml | 6 ++++-- .github/workflows/build-ton-macos-arm64-shared.yml | 6 ++++-- .github/workflows/build-ton-macos-x86-64-shared.yml | 6 ++++-- .github/workflows/build-ton-wasm-emscripten.yml | 8 +++++--- .github/workflows/create-release.yml | 1 - .github/workflows/create-tolk-release.yml | 3 +-- .github/workflows/docker-ubuntu-branch-image.yml | 6 ++++++ .github/workflows/docker-ubuntu-image.yml | 6 ++++++ .github/workflows/ton-ccpcheck.yml | 6 ++++-- .github/workflows/ton-x86-64-windows.yml | 3 ++- 17 files changed, 64 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build-ton-linux-android-tonlib.yml b/.github/workflows/build-ton-linux-android-tonlib.yml index 5cd74e3639..b92ea9d409 100644 --- a/.github/workflows/build-ton-linux-android-tonlib.yml +++ b/.github/workflows/build-ton-linux-android-tonlib.yml @@ -1,15 +1,17 @@ name: Tonlib Android -on: [push,workflow_dispatch,workflow_call] +on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: + if: github.repository == 'ton-blockchain/ton' + runs-on: ubuntu-22.04 steps: - name: Check out repository uses: actions/checkout@v3 - with: + with: submodules: 'recursive' - name: Install system libraries @@ -18,7 +20,6 @@ jobs: sudo apt-get install -y build-essential git cmake ninja-build automake libtool texinfo autoconf libgflags-dev \ zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev \ libtool autoconf libsodium-dev libsecp256k1-dev liblz4-dev - - name: Cache Android NDK id: cache-android-ndk diff --git a/.github/workflows/build-ton-linux-arm64-appimage.yml b/.github/workflows/build-ton-linux-arm64-appimage.yml index b7ecdc7845..a258fdbb74 100644 --- a/.github/workflows/build-ton-linux-arm64-appimage.yml +++ b/.github/workflows/build-ton-linux-arm64-appimage.yml @@ -1,15 +1,17 @@ name: Ubuntu TON build (AppImages, arm64) -on: [push,workflow_dispatch,workflow_call] +on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: + if: github.repository == 'ton-blockchain/ton' + runs-on: ubuntu-22.04-arm steps: - name: Check out repository uses: actions/checkout@v3 - with: + with: submodules: 'recursive' - name: Date Stamp diff --git a/.github/workflows/build-ton-linux-arm64-shared.yml b/.github/workflows/build-ton-linux-arm64-shared.yml index e7de5002bb..40f63509a5 100644 --- a/.github/workflows/build-ton-linux-arm64-shared.yml +++ b/.github/workflows/build-ton-linux-arm64-shared.yml @@ -1,9 +1,11 @@ name: Ubuntu TON build (shared, arm64) -on: [push,workflow_dispatch,workflow_call] +on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: + if: github.repository == 'ton-blockchain/ton' + strategy: fail-fast: false matrix: diff --git a/.github/workflows/build-ton-linux-x86-64-appimage.yml b/.github/workflows/build-ton-linux-x86-64-appimage.yml index cb02b7bc20..36974b8044 100644 --- a/.github/workflows/build-ton-linux-x86-64-appimage.yml +++ b/.github/workflows/build-ton-linux-x86-64-appimage.yml @@ -1,15 +1,17 @@ name: Ubuntu TON build (AppImages, x86-64) -on: [push,workflow_dispatch,workflow_call] +on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: + if: github.repository == 'ton-blockchain/ton' + runs-on: ubuntu-22.04 steps: - name: Check out repository uses: actions/checkout@v3 - with: + with: submodules: 'recursive' - name: Date Stamp diff --git a/.github/workflows/build-ton-linux-x86-64-shared.yml b/.github/workflows/build-ton-linux-x86-64-shared.yml index ed05d18aa7..9a61c9f42f 100644 --- a/.github/workflows/build-ton-linux-x86-64-shared.yml +++ b/.github/workflows/build-ton-linux-x86-64-shared.yml @@ -1,9 +1,11 @@ name: Ubuntu TON build (shared, x86-64) -on: [push,workflow_dispatch,workflow_call] +on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: + if: github.repository == 'ton-blockchain/ton' + strategy: fail-fast: false matrix: @@ -13,7 +15,7 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v3 - with: + with: submodules: 'recursive' - name: Date Stamp @@ -59,5 +61,5 @@ jobs: - name: Run Tests run: | - cd build + cd build ctest --output-on-failure --timeout 1800 diff --git a/.github/workflows/build-ton-macos-13-x86-64-portable.yml b/.github/workflows/build-ton-macos-13-x86-64-portable.yml index 6a6a6a255f..8e4e044042 100644 --- a/.github/workflows/build-ton-macos-13-x86-64-portable.yml +++ b/.github/workflows/build-ton-macos-13-x86-64-portable.yml @@ -1,9 +1,11 @@ name: MacOS-13 TON build (portable, x86-64) -on: [push,workflow_dispatch,workflow_call] +on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: + if: github.repository == 'ton-blockchain/ton' + runs-on: macos-13 steps: @@ -49,7 +51,7 @@ jobs: - name: Run Tests run: | - cd build + cd build ctest --output-on-failure --timeout 1800 - name: Upload artifacts diff --git a/.github/workflows/build-ton-macos-14-arm64-portable.yml b/.github/workflows/build-ton-macos-14-arm64-portable.yml index 995ff1c404..3fa1a13da3 100644 --- a/.github/workflows/build-ton-macos-14-arm64-portable.yml +++ b/.github/workflows/build-ton-macos-14-arm64-portable.yml @@ -1,9 +1,11 @@ name: MacOS-14 TON build (portable, arm64) -on: [push,workflow_dispatch,workflow_call] +on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: + if: github.repository == 'ton-blockchain/ton' + runs-on: macos-14 steps: @@ -49,7 +51,7 @@ jobs: - name: Run Tests run: | - cd build + cd build ctest --output-on-failure --timeout 1800 - name: Upload artifacts diff --git a/.github/workflows/build-ton-macos-15-arm64-shared.yml b/.github/workflows/build-ton-macos-15-arm64-shared.yml index a1e0782b70..dabf3e7717 100644 --- a/.github/workflows/build-ton-macos-15-arm64-shared.yml +++ b/.github/workflows/build-ton-macos-15-arm64-shared.yml @@ -1,9 +1,11 @@ name: MacOS-15 TON build (shared, arm64) -on: [push,workflow_dispatch,workflow_call] +on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: + if: github.repository == 'ton-blockchain/ton' + runs-on: macos-15 steps: @@ -41,5 +43,5 @@ jobs: - name: Run Tests run: | - cd build + cd build ctest --output-on-failure --timeout 1800 diff --git a/.github/workflows/build-ton-macos-arm64-shared.yml b/.github/workflows/build-ton-macos-arm64-shared.yml index 5ef9266af8..be0d78b735 100644 --- a/.github/workflows/build-ton-macos-arm64-shared.yml +++ b/.github/workflows/build-ton-macos-arm64-shared.yml @@ -1,9 +1,11 @@ name: MacOS-14 TON build (shared, arm64) -on: [push,workflow_dispatch,workflow_call] +on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: + if: github.repository == 'ton-blockchain/ton' + runs-on: macos-14 steps: @@ -41,5 +43,5 @@ jobs: - name: Run Tests run: | - cd build + cd build ctest --output-on-failure --timeout 1800 diff --git a/.github/workflows/build-ton-macos-x86-64-shared.yml b/.github/workflows/build-ton-macos-x86-64-shared.yml index 6200af6633..98f71ec3f7 100644 --- a/.github/workflows/build-ton-macos-x86-64-shared.yml +++ b/.github/workflows/build-ton-macos-x86-64-shared.yml @@ -1,9 +1,11 @@ name: MacOS TON build (shared, x86-64) -on: [push,workflow_dispatch,workflow_call] +on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: + if: github.repository == 'ton-blockchain/ton' + runs-on: macos-13 steps: @@ -41,5 +43,5 @@ jobs: - name: Run Tests run: | - cd build + cd build ctest --output-on-failure --timeout 1800 diff --git a/.github/workflows/build-ton-wasm-emscripten.yml b/.github/workflows/build-ton-wasm-emscripten.yml index f9f8611485..7dcfbc9c53 100644 --- a/.github/workflows/build-ton-wasm-emscripten.yml +++ b/.github/workflows/build-ton-wasm-emscripten.yml @@ -1,9 +1,11 @@ name: Emscripten TON build (wasm) -on: [push,workflow_dispatch,workflow_call] +on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: + if: github.repository == 'ton-blockchain/ton' + runs-on: ubuntu-22.04 steps: @@ -50,7 +52,7 @@ jobs: run: | cp assembly/wasm/*.fc . git clone https://github.com/ton-community/func-js.git - cd func-js + cd func-js npm install npm run build npm link @@ -63,7 +65,7 @@ jobs: cp artifacts/funcfiftlib.wasm.js func-js/node_modules/@ton-community/func-js-bin/dist/funcfiftlib.wasm.js cp artifacts/funcfiftlib.js func-js/node_modules/@ton-community/func-js-bin/dist/funcfiftlib.js npx func-js stdlib.fc intrinsics.fc --fift ./output.f - + - name: Upload artifacts uses: actions/upload-artifact@master with: diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 05a3db265b..09e1c632d6 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -125,7 +125,6 @@ jobs: - name: Show all artifacts run: | tree artifacts - # create release diff --git a/.github/workflows/create-tolk-release.yml b/.github/workflows/create-tolk-release.yml index 4cd30ef6f0..963768444a 100644 --- a/.github/workflows/create-tolk-release.yml +++ b/.github/workflows/create-tolk-release.yml @@ -76,7 +76,6 @@ jobs: - name: Show all artifacts run: | tree artifacts - # create release - name: Get registration token @@ -159,7 +158,7 @@ jobs: run: | mkdir smartcont_lib cd smartcont_lib - cp -r ../artifacts/ton-x86_64-linux/{smartcont,lib} . + cp -r ../artifacts/ton-x86_64-linux/{smartcont,lib} . zip -r smartcont_lib.zip . gh release upload ${{ inputs.tag }} smartcont_lib.zip diff --git a/.github/workflows/docker-ubuntu-branch-image.yml b/.github/workflows/docker-ubuntu-branch-image.yml index bd97ed233f..f8e0f206c5 100644 --- a/.github/workflows/docker-ubuntu-branch-image.yml +++ b/.github/workflows/docker-ubuntu-branch-image.yml @@ -12,6 +12,8 @@ env: jobs: build-arm64: + if: github.repository == 'ton-blockchain/ton' + runs-on: ubuntu-22.04-arm steps: - name: Check out repository @@ -49,6 +51,8 @@ jobs: build-amd64: + if: github.repository == 'ton-blockchain/ton' + runs-on: ubuntu-22.04 steps: - name: Check out repository @@ -86,6 +90,8 @@ jobs: merge: + if: github.repository == 'ton-blockchain/ton' + runs-on: ubuntu-22.04 needs: - build-amd64 diff --git a/.github/workflows/docker-ubuntu-image.yml b/.github/workflows/docker-ubuntu-image.yml index 7a5eee10ad..76cf3db625 100644 --- a/.github/workflows/docker-ubuntu-image.yml +++ b/.github/workflows/docker-ubuntu-image.yml @@ -11,6 +11,8 @@ env: jobs: build-arm64: + if: github.repository == 'ton-blockchain/ton' + runs-on: ubuntu-22.04-arm steps: - name: Check out repository @@ -53,6 +55,8 @@ jobs: tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-arm64 build-amd64: + if: github.repository == 'ton-blockchain/ton' + runs-on: ubuntu-22.04 steps: - name: Check out repository @@ -95,6 +99,8 @@ jobs: tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-amd64 merge: + if: github.repository == 'ton-blockchain/ton' + runs-on: ubuntu-22.04 needs: - build-amd64 diff --git a/.github/workflows/ton-ccpcheck.yml b/.github/workflows/ton-ccpcheck.yml index 95bef5f3d1..9b2352108d 100644 --- a/.github/workflows/ton-ccpcheck.yml +++ b/.github/workflows/ton-ccpcheck.yml @@ -1,15 +1,17 @@ name: TON Static Code Analysis -on: [push,workflow_dispatch,workflow_call] +on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: + if: github.repository == 'ton-blockchain/ton' + runs-on: ubuntu-22.04 steps: - name: Check out repository uses: actions/checkout@v3 - with: + with: submodules: 'recursive' - name: Run Cppcheck diff --git a/.github/workflows/ton-x86-64-windows.yml b/.github/workflows/ton-x86-64-windows.yml index 8de9e11be9..0f52413ed4 100644 --- a/.github/workflows/ton-x86-64-windows.yml +++ b/.github/workflows/ton-x86-64-windows.yml @@ -1,6 +1,6 @@ name: Windows TON build (portable, x86-64) -on: [push,workflow_dispatch,workflow_call] +on: [push, pull_request, workflow_dispatch, workflow_call] defaults: run: @@ -8,6 +8,7 @@ defaults: jobs: build: + if: github.repository == 'ton-blockchain/ton' runs-on: windows-2022 From 2f352bc91b0a44d3c493897f5f827032d069b6af Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Mon, 20 Oct 2025 11:21:23 -0400 Subject: [PATCH 416/524] Add -Werror CI job (#1845) * Remove commented out directives from top-level CMakeLists * Use add_compiler_options instead of some weird helper in CMake * Suppress CMake policy version warning from Abseil We really should update it. * Suppress existing warnings This is not particularly precise as I disabled warnings on file level (and warning suppression mappings allow specifying function scope) but is good enough to add -Werror CI job in next commit. * Add -Werror CI job --- .../build-ton-linux-x86-64-werror.yml | 45 ++++++ CMake/AddCXXCompilerFlag.cmake | 74 ---------- CMakeLists.txt | 129 +++++++++--------- suppression_mappings.txt | 103 ++++++++++++++ 4 files changed, 213 insertions(+), 138 deletions(-) create mode 100644 .github/workflows/build-ton-linux-x86-64-werror.yml delete mode 100644 CMake/AddCXXCompilerFlag.cmake create mode 100644 suppression_mappings.txt diff --git a/.github/workflows/build-ton-linux-x86-64-werror.yml b/.github/workflows/build-ton-linux-x86-64-werror.yml new file mode 100644 index 0000000000..cf783bfda6 --- /dev/null +++ b/.github/workflows/build-ton-linux-x86-64-werror.yml @@ -0,0 +1,45 @@ +name: Ubuntu TON -Werror build (shared, x86-64) + +on: [push, pull_request, workflow_dispatch, workflow_call] + +jobs: + build: + if: github.repository == 'ton-blockchain/ton' + + runs-on: ubuntu-24.04 + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + + - name: Install system libraries + run: | + sudo apt-get update + sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev + + - name: Install Clang 21 + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 21 clang + + # TODO: Preserve ccache cache between runs once we get rid of warning suppression mappings. + + - name: Build TON + run: | + cmake -S . -B build -G Ninja \ + -DCMAKE_C_COMPILER=clang-21 \ + -DCMAKE_CXX_COMPILER=clang++-21 \ + -DCMAKE_BUILD_TYPE=Release \ + -DTON_USE_JEMALLOC=ON \ + -DTON_USE_LLD=On \ + -DTON_WERROR_BUILD=On + ninja -C build diff --git a/CMake/AddCXXCompilerFlag.cmake b/CMake/AddCXXCompilerFlag.cmake deleted file mode 100644 index 6fb615a1f7..0000000000 --- a/CMake/AddCXXCompilerFlag.cmake +++ /dev/null @@ -1,74 +0,0 @@ -# - Adds a compiler flag if it is supported by the compiler -# -# This function checks that the supplied compiler flag is supported and then -# adds it to the corresponding compiler flags -# -# add_cxx_compiler_flag( []) -# -# - Example -# -# include(AddCXXCompilerFlag) -# add_cxx_compiler_flag(-Wall) -# add_cxx_compiler_flag(-no-strict-aliasing RELEASE) -# Requires CMake 2.6+ - -if (__add_cxx_compiler_flag) - return() -endif() -set(__add_cxx_compiler_flag INCLUDED) - -include(CheckCXXCompilerFlag) - -function(mangle_compiler_flag FLAG OUTPUT) - string(TOUPPER "HAVE_CXX_FLAG_${FLAG}" SANITIZED_FLAG) - string(REPLACE "+" "X" SANITIZED_FLAG ${SANITIZED_FLAG}) - string(REGEX REPLACE "[^A-Za-z_0-9]" "_" SANITIZED_FLAG ${SANITIZED_FLAG}) - string(REGEX REPLACE "_+" "_" SANITIZED_FLAG ${SANITIZED_FLAG}) - set(${OUTPUT} "${SANITIZED_FLAG}" PARENT_SCOPE) -endfunction(mangle_compiler_flag) - -function(add_cxx_compiler_flag FLAG) - string(REPLACE "-Wno-" "-W" MAIN_FLAG ${FLAG}) - mangle_compiler_flag("${MAIN_FLAG}" MANGLED_FLAG_NAME) - if (DEFINED CMAKE_REQUIRED_FLAGS) - set(OLD_CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}") - set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${FLAG}") - else() - set(CMAKE_REQUIRED_FLAGS "${FLAG}") - endif() - check_cxx_compiler_flag("${MAIN_FLAG}" ${MANGLED_FLAG_NAME}) - if (DEFINED OLD_CMAKE_REQUIRED_FLAGS) - set(CMAKE_REQUIRED_FLAGS "${OLD_CMAKE_REQUIRED_FLAGS}") - else() - unset(CMAKE_REQUIRED_FLAGS) - endif() - if (${MANGLED_FLAG_NAME}) - set(VARIANT ${ARGV1}) - if (ARGV1) - string(TOUPPER "_${VARIANT}" VARIANT) - endif() - set(CMAKE_CXX_FLAGS${VARIANT} "${CMAKE_CXX_FLAGS${VARIANT}} ${FLAG}" PARENT_SCOPE) - endif() -endfunction() - -function(add_required_cxx_compiler_flag FLAG) - string(REPLACE "-Wno-" "-W" MAIN_FLAG ${FLAG}) - mangle_compiler_flag("${MAIN_FLAG}" MANGLED_FLAG_NAME) - set(OLD_CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}") - set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${FLAG}") - check_cxx_compiler_flag("${MAIN_FLAG}" ${MANGLED_FLAG_NAME}) - set(CMAKE_REQUIRED_FLAGS "${OLD_CMAKE_REQUIRED_FLAGS}") - if (${MANGLED_FLAG_NAME}) - set(VARIANT ${ARGV1}) - if (ARGV1) - string(TOUPPER "_${VARIANT}" VARIANT) - endif() - set(CMAKE_CXX_FLAGS${VARIANT} "${CMAKE_CXX_FLAGS${VARIANT}} ${FLAG}" PARENT_SCOPE) - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${FLAG}" PARENT_SCOPE) - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${FLAG}" PARENT_SCOPE) - set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${FLAG}" PARENT_SCOPE) - set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${FLAG}" PARENT_SCOPE) - else() - message(FATAL_ERROR "Required flag '${FLAG}' is not supported by the compiler") - endif() -endfunction() diff --git a/CMakeLists.txt b/CMakeLists.txt index c37f351d8c..6e758c457b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,6 @@ cmake_minimum_required(VERSION 3.16 FATAL_ERROR) project(TON VERSION 0.5 LANGUAGES C CXX) set(CMAKE_POSITION_INDEPENDENT_CODE ON) -#set(OPENSSL_USE_STATIC_LIBS TRUE) # Define the two required variables before including the source code for watching a git repository. set(PRE_CONFIGURE_FILE "git.cc.in") @@ -35,6 +34,16 @@ set(CMAKE_CXX_EXTENSIONS FALSE) set(CMAKE_COLOR_DIAGNOSTICS TRUE) +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(GCC 1) +elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set(CLANG 1) +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + set(INTEL 1) +elseif (NOT MSVC) + message(FATAL_ERROR "Compiler isn't supported") +endif() + #BEGIN internal option(BUILD_SHARED_LIBS "Use \"ON\" to build shared libraries instead of static where it's not specified (not recommended)" OFF) option(USE_EMSCRIPTEN "Use \"ON\" for config building wasm." OFF) @@ -77,16 +86,29 @@ if ((ARCHITECTURE MATCHES "arm64") AND (CMAKE_SYSTEM_NAME STREQUAL "Darwin") AND endif() #END M1 support +if (CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "20") + add_compile_options(--warning-suppression-mappings=${CMAKE_SOURCE_DIR}/suppression_mappings.txt) +endif() + +option(TON_WERROR_BUILD "Use -Werror" OFF) + +# If we want to check -Werror build in CI, we want to use it for third party projects as well but we +# don't want to force our warning options on them. +if (TON_WERROR_BUILD) + add_compile_options(-Werror) +endif() + if (TON_USE_ABSEIL) message("Add abseil-cpp") - set(ABSL_PROPAGATE_CXX_STD TRUE) - add_subdirectory(third-party/abseil-cpp EXCLUDE_FROM_ALL) + function(abseil_scope) + set(ABSL_PROPAGATE_CXX_STD TRUE) + set(CMAKE_POLICY_VERSION_MINIMUM "3.10") + add_subdirectory(third-party/abseil-cpp EXCLUDE_FROM_ALL) + endfunction() + abseil_scope() set(ABSL_FOUND 1) endif() -#add_subdirectory(third-party/libcuckoo EXCLUDE_FROM_ALL) -#add_subdirectory(third-party/junction EXCLUDE_FROM_ALL) - if (WIN32) message("Add wingetopt") function(wingetopt_scope) @@ -166,16 +188,6 @@ else() message(STATUS "Could NOT find ccache") endif() -if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(GCC 1) -elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") - set(CLANG 1) -elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - set(INTEL 1) -elseif (NOT MSVC) - message(FATAL_ERROR "Compiler isn't supported") -endif() - include(CheckCXXCompilerFlag) set(CMAKE_THREAD_PREFER_PTHREAD ON) @@ -282,33 +294,37 @@ set(TONLIB_COMPILE "0") set(TONLIB_COMPILE "1") #END tonlib -include(AddCXXCompilerFlag) if (MSVC) - add_cxx_compiler_flag("/experimental:external /external:anglebrackets /external:W0") -endif() -if (NOT MSVC) - add_cxx_compiler_flag("-Wall") - add_cxx_compiler_flag("-Wextra") -endif() - -add_cxx_compiler_flag("-Wimplicit-fallthrough=2") -add_cxx_compiler_flag("-Wpointer-arith") -add_cxx_compiler_flag("-Wcast-qual") -add_cxx_compiler_flag("-Wsign-compare") -add_cxx_compiler_flag("-Wduplicated-branches") -add_cxx_compiler_flag("-Wduplicated-cond") -add_cxx_compiler_flag("-Walloc-zero") -add_cxx_compiler_flag("-Wlogical-op") -add_cxx_compiler_flag("-Wno-tautological-compare") -add_cxx_compiler_flag("-Wpointer-arith") -add_cxx_compiler_flag("-Wvla") -add_cxx_compiler_flag("-Wnon-virtual-dtor") -add_cxx_compiler_flag("-Wno-unused-parameter") -add_cxx_compiler_flag("-Wconversion") -add_cxx_compiler_flag("-Wno-sign-conversion") -add_cxx_compiler_flag("-Qunused-arguments") -add_cxx_compiler_flag("-Wno-unused-private-field") -add_cxx_compiler_flag("-Wno-redundant-move") + add_compile_options(/experimental:external /external:anglebrackets /external:W0) +endif() + +if (GCC OR CLANG) + add_compile_options(-Wall) + add_compile_options(-Wextra) + + add_compile_options(-Wpointer-arith) + add_compile_options(-Wcast-qual) + add_compile_options(-Wsign-compare) + add_compile_options(-Wpointer-arith) + add_compile_options(-Wvla) + add_compile_options(-Wnon-virtual-dtor) + add_compile_options(-Wconversion) + + # TODO: Either justify all of these or remove. + add_compile_options(-Wno-tautological-compare) + add_compile_options(-Wno-unused-parameter) + add_compile_options(-Wno-sign-conversion) + add_compile_options(-Wno-unused-private-field) + add_compile_options(-Wno-redundant-move) + + if (GCC) + add_compile_options(-Wimplicit-fallthrough=2) + add_compile_options(-Wduplicated-branches) + add_compile_options(-Wduplicated-cond) + add_compile_options(-Walloc-zero) + add_compile_options(-Wlogical-op) + endif() +endif() if (GCC OR CLANG) if (CMAKE_BUILD_TYPE MATCHES "RelWithDebInfo") @@ -318,30 +334,19 @@ if (GCC OR CLANG) endif() endif() -#add_cxx_compiler_flag("-Wno-unused-function") -#add_cxx_compiler_flag("-Wno-unused-variable") -#add_cxx_compiler_flag("-Wno-shorten-64-to-32") -#add_cxx_compiler_flag("-Werror") - -#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem /usr/include/c++/v1") -if (CLANG) - #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") -endif() if (TON_USE_ASAN) - add_cxx_compiler_flag("-fsanitize=address") + add_compile_options(-fsanitize=address) + add_link_options(-fsanitize=address) add_definitions(-DTD_USE_ASAN=1) endif() if (TON_USE_TSAN) - add_cxx_compiler_flag("-fsanitize=thread") + add_compile_options(-fsanitize=thread) + add_link_options(-fsanitize=thread) endif() if (TON_USE_UBSAN) - add_cxx_compiler_flag("-fsanitize=undefined") + add_compile_options(-fsanitize=undefined) + add_link_options(-fsanitize=undefined) endif() -#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread") -#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") -#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined") -#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=leak") -#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -finstrument-functions") #Compilation database set(CMAKE_EXPORT_COMPILE_COMMANDS 1) @@ -451,10 +456,6 @@ target_link_libraries(test-fift PRIVATE fift-lib) add_executable(test-tdutils test/test-td-main.cpp ${TDUTILS_TEST_SOURCE}) target_link_libraries(test-tdutils PRIVATE tdutils ${CMAKE_THREAD_LIBS_INIT} memprof ${JEMALLOC_LIBRARIES}) -#target_link_libraries_system(test-tdutils absl::base absl::container absl::hash ) -#target_link_libraries_system(test-tdutils libcuckoo) -#target_include_directories(test-tdutils PRIVATE SYSTEM ${JUNCTION_ALL_INCLUDE_DIRS}) -#target_link_libraries(test-tdutils PRIVATE ${JUNCTION_ALL_LIBRARIES}) add_executable(test-tdactor test/test-td-main.cpp ${TDACTOR_TEST_SOURCE}) target_link_libraries(test-tdactor PRIVATE tdactor ${CMAKE_THREAD_LIBS_INIT}) @@ -535,7 +536,7 @@ if (HAS_PARENT) ${ED25519_TEST_SOURCE} ${TONDB_TEST_SOURCE} ${BIGNUM_TEST_SOURCE} - ${CELLS_TEST_SOURCE} # ${TONVM_TEST_SOURCE} ${FIFT_TEST_SOURCE} ${TONLIB_ONLINE_TEST_SOURCE} + ${CELLS_TEST_SOURCE} PARENT_SCOPE) endif() add_library(all_tests INTERFACE) diff --git a/suppression_mappings.txt b/suppression_mappings.txt new file mode 100644 index 0000000000..5822b3f183 --- /dev/null +++ b/suppression_mappings.txt @@ -0,0 +1,103 @@ +[#warnings] +src:*/ciso646 + +[deprecated-builtins] +src:*/third-party/abseil-cpp/absl/meta/type_traits.h + +[array-parameter] +src:*/third-party/abseil-cpp/absl/hash/internal/low_level_hash.cc + +[deprecated-literal-operator] +src:*/tdutils/td/utils/date.h + +[missing-field-initializers] +src:*/emulator/test/emulator-tests.cpp + +[missing-designated-field-initializers] +src:*/tdutils/td/utils/Timer.cpp +src:*/validator/db/permanent-celldb/permanent-celldb-utils.cpp +src:*/validator/validator-options.cpp + +[implicit-int-float-conversion] +src:*/overlay/overlay.cpp +src:*/tdactor/td/actor/ActorStats.cpp +src:*/tdactor/td/actor/ActorStats.h +src:*/tonlib/test/online.cpp + +[unused-variable] +src:*/crypto/block/precompiled-smc/PrecompiledSmartContract.cpp +src:*/crypto/test/test-db.cpp +src:*/crypto/test/test-smartcont.cpp +src:*/crypto/tl/tlbc-gen-cpp.cpp +src:*/crypto/vm/boc-compression.cpp +src:*/crypto/vm/large-boc-serializer.cpp +src:*/tdactor/td/actor/ActorStats.cpp +src:*/validator/full-node-fast-sync-overlays.cpp + +[unused-but-set-variable] +src:*/tdtl/td/tl/tl_config.cpp +src:*/tdtl/td/tl/tl_generate.cpp +src:*/tdutils/test/misc.cpp + +[return-type] +src:*/tl/generate/tl_writer_hpp.cpp + +[deprecated-declarations] +src:*/tdutils/test/crypto.cpp +src:*/third-party/rocksdb/db/memtable.cc + +[implicit-int-conversion] +src:*/crypto/vm/boc-compression.cpp +src:*/crypto/vm/boc.cpp + +[shorten-64-to-32] +src:*/crypto/parser/lexer.cpp +src:*/crypto/vm/boc-compression.cpp +src:*/crypto/vm/dict.cpp +src:*/lite-client/lite-client.cpp +src:*/tl/generate/auto/tl/tonlib_api.h +src:*/tl/tl/TlObject.h +src:*/tonlib/tonlib/TonlibClient.cpp +src:*/validator-session/validator-session.cpp + +[sign-compare] +src:*/crypto/block/transaction.cpp +src:*/crypto/vm/boc-compression.cpp +src:*/crypto/vm/dict.cpp +src:*/lite-client/lite-client.cpp +src:*/tonlib/tonlib/TonlibClient.cpp +src:*/validator-session/candidate-serializer.cpp +src:*/validator/db/package.cpp +src:*/validator/db/permanent-celldb/permanent-celldb-utils.cpp +src:*/validator/full-node-fast-sync-overlays.cpp + +[nontrivial-memcall] +src:*/third-party/rocksdb/port/mmap.cc + +[unused-const-variable] +src:*/catchain/catchain-receiver.cpp +src:*/crypto/block/block-parse.cpp + +[unused-function] +src:*/crypto/vm/db/DynamicBagOfCellsDb.cpp + +[logical-op-parentheses] +src:*/validator/import-db-slice-local.cpp + +[float-conversion] +src:*/validator-engine-console/validator-engine-console-query.cpp +src:*/validator/db/archive-manager.cpp + +[deprecated-this-capture] +src:*/tonlib/tonlib/TonlibClient.cpp +src:*/validator/impl/liteserver.cpp + +[unused-result] +src:*/crypto/test/test-cells.cpp + +[deprecated-enum-float-conversion] +src:*/validator/impl/liteserver.cpp + +[unused-lambda-capture] +src:*/lite-client/lite-client.cpp +src:*/storage/storage-daemon/StorageProvider.cpp From f9db7bd255d20141f748c5353ae800085c3d6901 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Thu, 23 Oct 2025 18:03:21 +0400 Subject: [PATCH 417/524] fix WASM artefacts by upgrading emscripten, forcing filesystem access and setting environment=node (#1850) Co-authored-by: EmelyanenkoK --- assembly/wasm/fift-func-wasm-build-ubuntu.sh | 4 ++-- emulator/CMakeLists.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/assembly/wasm/fift-func-wasm-build-ubuntu.sh b/assembly/wasm/fift-func-wasm-build-ubuntu.sh index a80430c1dd..a99f27392b 100644 --- a/assembly/wasm/fift-func-wasm-build-ubuntu.sh +++ b/assembly/wasm/fift-func-wasm-build-ubuntu.sh @@ -70,8 +70,8 @@ echo fi cd emsdk -./emsdk install 3.1.19 -./emsdk activate 3.1.19 +./emsdk install 4.0.17 +./emsdk activate 4.0.17 EMSDK_DIR=`pwd` . $EMSDK_DIR/emsdk_env.sh diff --git a/emulator/CMakeLists.txt b/emulator/CMakeLists.txt index 63cb28d6a9..a802d8df30 100644 --- a/emulator/CMakeLists.txt +++ b/emulator/CMakeLists.txt @@ -63,8 +63,8 @@ if (USE_EMSCRIPTEN) target_link_options(emulator-emscripten PRIVATE -sIGNORE_MISSING_MAIN=1) target_link_options(emulator-emscripten PRIVATE -sAUTO_NATIVE_LIBRARIES=0) target_link_options(emulator-emscripten PRIVATE -sMODULARIZE=1) - target_link_options(emulator-emscripten PRIVATE -sENVIRONMENT=web) - target_link_options(emulator-emscripten PRIVATE -sFILESYSTEM=0) + target_link_options(emulator-emscripten PRIVATE -sENVIRONMENT=node) + target_link_options(emulator-emscripten PRIVATE -sFORCE_FILESYSTEM=1) target_link_options(emulator-emscripten PRIVATE -sALLOW_MEMORY_GROWTH=1) target_link_options(emulator-emscripten PRIVATE -fexceptions) if (USE_EMSCRIPTEN_NO_WASM) From ecd8fbb833c408eb34ec1aa4516e9e4344b54a22 Mon Sep 17 00:00:00 2001 From: Petr Makhnev <51853996+i582@users.noreply.github.com> Date: Fri, 24 Oct 2025 15:37:06 +0400 Subject: [PATCH 418/524] [vm] Add new opcodes to store and load `addr_std$10` (#1849) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [vm] Add new opcodes to store and load `addr_std$10` LDSTDADDR — loads a MessageInt address or fail LDSTDADDRQ — quirly loads a MessageInt address LDOPTSTDADDR — loads a Maybe MessageInt address or fail LDOPTSTDADDRQ — quitly loads a Maybe MessageInt address STSTDADDR — stores a MessageInt address or fail STSTDADDRQ — quetly stores a MessageInt address STOPTSTDADDR — stores a Maybe MessageInt address or fail STOPTSTDADDRQ — quetly stores a Maybe MessageInt address * add description for STSTDADDR error * add check that address is fully valid * add require_version for the new opcodes * correct LDOPTSTDADDR(Q) behaviour * correct STOPTSTDADDR(Q) behaviour * run test-fift with --regression * add description to GlobalVersions.md * remove old +1 * fix stack description * fix `is_valid_std_msg_addr` * add `have_refs(0)` check to is_valid_std_msg_addr * check exact length and refs_length in is_valid_std_msg_addr * simplify fallback in is_valid_std_msg_addr * use push_bool, add check for refs --- crypto/fift/lib/Asm.fif | 8 + crypto/test/fift.cpp | 4 + crypto/test/fift/load-store-std-addr.fif | 405 +++++++++++++++++++++++ crypto/vm/tonops.cpp | 208 +++++++++++- crypto/vm/tonops.h | 1 + doc/GlobalVersions.md | 8 + test/regression-tests.ans | 1 + 7 files changed, 634 insertions(+), 1 deletion(-) create mode 100644 crypto/test/fift/load-store-std-addr.fif diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index a4b1e144d3..b69e582ea0 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -1458,6 +1458,14 @@ x{FA44} @Defop REWRITESTDADDR x{FA45} @Defop REWRITESTDADDRQ x{FA46} @Defop REWRITEVARADDR x{FA47} @Defop REWRITEVARADDRQ +x{FA48} @Defop LDSTDADDR +x{FA49} @Defop LDSTDADDRQ +x{FA50} @Defop LDOPTSTDADDR +x{FA51} @Defop LDOPTSTDADDRQ +x{FA52} @Defop STSTDADDR +x{FA53} @Defop STSTDADDRQ +x{FA54} @Defop STOPTSTDADDR +x{FA55} @Defop STOPTSTDADDRQ x{FB00} @Defop SENDRAWMSG x{FB02} @Defop RAWRESERVE diff --git a/crypto/test/fift.cpp b/crypto/test/fift.cpp index 55a6ded755..4d522ce738 100644 --- a/crypto/test/fift.cpp +++ b/crypto/test/fift.cpp @@ -179,3 +179,7 @@ TEST(Fift, test_get_extra_balance) { TEST(Fift, test_p256) { run_fift("p256.fif"); } + +TEST(Fift, test_load_store_std_addr) { + run_fift("load-store-std-addr.fif"); +} diff --git a/crypto/test/fift/load-store-std-addr.fif b/crypto/test/fift/load-store-std-addr.fif new file mode 100644 index 0000000000..18c3e39d63 --- /dev/null +++ b/crypto/test/fift/load-store-std-addr.fif @@ -0,0 +1,405 @@ +"Asm.fif" include + +{ { drop } depth 1- times } : clear-stack +{ ."Running test: " .dump cr 0 runvmx 0= { "test failed" abort } ifnot clear-stack } : run-test +{ ."Running test: " .dump cr 0 runvmx 0= { "test failed: exit code = 0 but expected to be non zero" abort } if clear-stack } : run-fail-test + +variable @addr_none +variable @valid_std_address +variable @valid_opt_std_address +variable @invalid_std_address + +b{00} @addr_none ! +x{8016316F1DAA41369BD888538DAA8F9D3BB8D449BA0F22AC0C882CFC98BAB534843_} @valid_std_address ! + s "should load address via LDSTDADDR" run-test + +<{ + @addr_none @ PUSHSLICE + LDSTDADDR + SBITS + 100 THROWIF // check that slice is empty + REWRITESTDADDR // check that address really is std + DROP2 +}>s "should not load addr_none address via LDSTDADDR" run-fail-test + +<{ + b{} PUSHSLICE + LDSTDADDR // will throw since no data available +}>s "should fail to load address via LDSTDADDR from empty slice" run-fail-test + +// ============================= LDSTDADDRQ ============================= + +<{ + @valid_std_address @ PUSHSLICE + LDSTDADDRQ + 100 THROWIFNOT // check that status is -1 + @valid_std_address @ PUSHSLICE + SDEQ // slice must be intact + DROP // drop slice + REWRITESTDADDR // check that address really is std + DROP2 +}>s "should load address via LDSTDADDRQ" run-test + +<{ + @addr_none @ PUSHSLICE + LDSTDADDRQ + 100 THROWIF // check that status is 0 + @addr_none @ PUSHSLICE + SDEQ // slice must be intact + DROP // drop slice +}>s "should not load addr_none address via LDSTDADDRQ" run-test + +<{ + b{} PUSHSLICE + LDSTDADDRQ // fail since no data available + 100 THROWIF // check that status is 0 +}>s "should fail to load address via LDSTDADDRQ from empty slice" run-test + +// ============================= LDOPTSTDADDR ============================= + +<{ + @valid_std_address @ PUSHSLICE + LDOPTSTDADDR + SBITS + 100 THROWIF // check that slice is empty + REWRITESTDADDR // check that address really is std + DROP2 +}>s "should load address via LDOPTSTDADDR" run-test + +<{ + @addr_none @ PUSHSLICE + LDOPTSTDADDR + SBITS + 100 THROWIF // check that slice is empty + ISNULL // loaded value must be null + 100 THROWIFNOT +}>s "should load null via LDOPTSTDADDR from addr_none slice" run-test + +<{ + b{10} PUSHSLICE + LDOPTSTDADDR // will throw since no data available for the address bpdy +}>s "should fail to load address via LDOPTSTDADDR from b{10} slice" run-fail-test + +<{ + b{1} PUSHSLICE + LDOPTSTDADDR // will throw since no data available for the tag +}>s "should fail to load address via LDOPTSTDADDR from b{1} slice" run-fail-test + +<{ + b{} PUSHSLICE + LDOPTSTDADDR // will throw since no data available +}>s "should fail to load address via LDOPTSTDADDR from empty slice" run-fail-test + +// ============================= LDOPTSTDADDRQ ============================= + +<{ + @valid_std_address @ PUSHSLICE + LDOPTSTDADDRQ + 100 THROWIFNOT // check that status is -1 + SBITS + 100 THROWIF // check that slice is empty + REWRITESTDADDR // check that address really is std + DROP2 +}>s "should load address via LDOPTSTDADDRQ" run-test + +<{ + @addr_none @ PUSHSLICE + LDOPTSTDADDRQ + 100 THROWIFNOT // check that status is -1 + SBITS + 100 THROWIF // check that slice is empty + ISNULL // loaded value must be null + 100 THROWIFNOT +}>s "should load null via LDOPTSTDADDRQ from addr_none" run-test + +<{ + b{10} PUSHSLICE + LDOPTSTDADDRQ // will return slice status=0 + 100 THROWIF // check that status is 0 + b{10} PUSHSLICE + SDEQ // slice must be intact +}>s "should not load address via LDOPTSTDADDRQ from b{10} slice" run-test + +<{ + b{1} PUSHSLICE + LDOPTSTDADDRQ // will return slice status=0 + 100 THROWIF // check that status is 0 + b{1} PUSHSLICE + SDEQ // slice must be intact +}>s "should not load address via LDOPTSTDADDRQ from b{1} slice" run-test + +<{ + b{} PUSHSLICE + LDOPTSTDADDRQ // fail since no data available + 100 THROWIF // check that status is 0 +}>s "should fail to load address via LDOPTSTDADDRQ from empty slice" run-test + +// ============================= STSTDADDR ============================= + +<{ + @valid_std_address @ PUSHSLICE + NEWC + STSTDADDR + DROP // drop builder +}>s "should store address via STSTDADDR" run-test + +<{ + @addr_none @ PUSHSLICE + NEWC + STSTDADDR // should fail +}>s "should fail to store addr_none address via STSTDADDR" run-fail-test + +<{ + @valid_opt_std_address @ PUSHSLICE + NEWC + STSTDADDR // should fail +}>s "should fail to store opt std address via STSTDADDR" run-fail-test + +<{ + @valid_std_address @ PUSHSLICE + NEWC + s "should fail to store opt std address via STSTDADDR to full builder" run-fail-test + +<{ + @invalid_std_address @ PUSHSLICE + NEWC + STSTDADDR // should fail +}>s "should fail to store too short slice via STSTDADDR" run-fail-test + +<{ + b{110011001111} PUSHSLICE + NEWC + STSTDADDR // should fail +}>s "should fail to store random slice via STSTDADDR" run-fail-test + +// ============================= STSTDADDRQ ============================= + +<{ + @valid_std_address @ PUSHSLICE + NEWC + STSTDADDRQ + 100 THROWIF // check that status is 0 (for some reason STSLICEQ returns 0 for success and -1 for fail) + DROP // drop builder +}>s "should store address via STSTDADDRQ" run-test + +<{ + @addr_none @ PUSHSLICE + NEWC + STSTDADDRQ + 100 THROWIFNOT // check that status is -1 (for some reason STSLICEQ returns 0 for success and -1 for fail) + DROP2 // drop builder and slice +}>s "should fail to store addr_none address via STSTDADDRQ" run-test + +<{ + @valid_opt_std_address @ PUSHSLICE + NEWC + STSTDADDRQ + 100 THROWIFNOT // check that status is -1 (for some reason STSLICEQ returns 0 for success and -1 for fail) + DROP2 // drop builder and slice +}>s "should fail to store opt std address via STSTDADDRQ" run-test + +<{ + @invalid_std_address @ PUSHSLICE + NEWC + STSTDADDRQ // should fail + 100 THROWIFNOT // check that status is -1 (for some reason STSLICEQ returns 0 for success and -1 for fail) + DROP2 // drop builder and slice +}>s "should fail to store too short slice via STSTDADDRQ" run-test + +<{ + b{110011001111} PUSHSLICE + NEWC + STSTDADDRQ // should fail + 100 THROWIFNOT // check that status is -1 (for some reason STSLICEQ returns 0 for success and -1 for fail) + DROP2 // drop builder and slice +}>s "should fail to store random slice via STSTDADDRQ" run-test + + +// ============================= STOPTSTDADDR ============================= + +<{ + @valid_std_address @ PUSHSLICE + DUP + NEWC + STOPTSTDADDR + ENDC + CTOS + LDOPTSTDADDR + DROP + SDEQ // check that address loaded back is equal to original one + 100 THROWIFNOT +}>s "should store address via STOPTSTDADDR" run-test + +<{ + PUSHNULL + NEWC + STOPTSTDADDR + ENDC + CTOS + LDOPTSTDADDR // we should be able to load address back + DROP + ISNULL // check that address loaded back is actually a Null + 100 THROWIFNOT +}>s "should store null via STOPTSTDADDR as addr_none" run-test + +<{ + PUSHNULL + NEWC + STOPTSTDADDR + ENDC + CTOS + b{00} PUSHSLICE + SDEQ // check that address stored as b{00} + 100 THROWIFNOT +}>s "should store null via STOPTSTDADDR as b{00}" run-test + +<{ + @addr_none @ PUSHSLICE + NEWC + STOPTSTDADDR // should fail +}>s "should fail to store addr_none address via STOPTSTDADDR" run-fail-test + +<{ + @valid_std_address @ PUSHSLICE + NEWC + s "should fail to store address via STOPTSTDADDR in full builder" run-fail-test + +<{ + PUSHNULL + NEWC + s "should fail to store null via STOPTSTDADDR in full builder" run-fail-test + +<{ + PUSHNULL + NEWC + s "should fail to store null via STOPTSTDADDR in almost full builder" run-fail-test + +<{ + @invalid_std_address @ PUSHSLICE + NEWC + STOPTSTDADDR // should fail +}>s "should fail to store too short slice via STOPTSTDADDR" run-fail-test + +<{ + b{110011001111} PUSHSLICE + NEWC + STOPTSTDADDR // should fail +}>s "should fail to store random slice via STOPTSTDADDR" run-fail-test + +<{ + 10 PUSHINT + NEWC + STOPTSTDADDR // should fail +}>s "should fail to store int via STOPTSTDADDR" run-fail-test + +// ============================= STOPTSTDADDRQ ============================= + +<{ + @valid_std_address @ PUSHSLICE + DUP + NEWC + STOPTSTDADDRQ + 100 THROWIF // check that status is 0 (for some reason STSLICEQ returns 0 for success and -1 for fail) + ENDC + CTOS + LDOPTSTDADDR + DROP + SDEQ // check that address loaded back is equal to original one + 100 THROWIFNOT +}>s "should store address via STOPTSTDADDRQ" run-test + +<{ + PUSHNULL + NEWC + STOPTSTDADDRQ // stored as b{00} + 100 THROWIF // check that status is 0 (for some reason STSLICEQ returns 0 for success and -1 for fail) + ENDC + CTOS // b{00} + LDOPTSTDADDR // null slice + SBITS + 100 THROWIF // check that slice is empty + ISNULL + 100 THROWIFNOT // check that loaded address is null +}>s "should store null via STOPTSTDADDRQ" run-test + +<{ + @addr_none @ PUSHSLICE + NEWC + STOPTSTDADDRQ // should fail + 100 THROWIFNOT // check that status is -1 (for some reason STSLICEQ returns 0 for success and -1 for fail) +}>s "should fail to store addr_none address via STOPTSTDADDRQ" run-test + +<{ + @valid_std_address @ PUSHSLICE + NEWC + s "should fail to store address via STOPTSTDADDRQ in full builder" run-test + +<{ + PUSHNULL + NEWC + s "should fail to store null via STOPTSTDADDR in full builder" run-test + +<{ + PUSHNULL + NEWC + s "should fail to store null via STOPTSTDADDR in almost full builder" run-test + +<{ + @invalid_std_address @ PUSHSLICE + NEWC + STOPTSTDADDRQ // should fail + 100 THROWIFNOT // check that status is -1 (for some reason STSLICEQ returns 0 for success and -1 for fail) + DROP2 // drop builder and slice +}>s "should fail to store too short slice via STOPTSTDADDRQ" run-test + +<{ + b{110011001111} PUSHSLICE + NEWC + STOPTSTDADDRQ // should fail + 100 THROWIFNOT // check that status is -1 (for some reason STSLICEQ returns 0 for success and -1 for fail) + DROP2 // drop builder and slice +}>s "should fail to store random slice via STOPTSTDADDRQ" run-test + +<{ + 100 PUSHINT + NEWC + STOPTSTDADDRQ // should fail + 100 THROWIFNOT // check that status is -1 (for some reason STSLICEQ returns 0 for success and -1 for fail) + DROP2 // drop builder and slice +}>s "should fail to store int via STOPTSTDADDRQ" run-test diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index 16e24b1464..88f9f19363 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -1533,6 +1533,20 @@ bool skip_message_addr(CellSlice& cs, int global_version) { } } +bool skip_std_message_addr(CellSlice& cs, int global_version) { + switch ((unsigned)cs.fetch_ulong(2)) { + case 2: { // addr_std$10 + return skip_maybe_anycast(cs, global_version) // anycast:(Maybe Anycast) + && cs.advance(8 + 256); // workchain_id:int8 address:bits256 = MsgAddressInt; + } + case 0: // addr_none$00 = MsgAddressExt; + case 1: // addr_extern$01 + case 3: // addr_var$11 + default: + return false; + } +} + int exec_load_message_addr(VmState* st, bool quiet) { VM_LOG(st) << "execute LDMSGADDR" << (quiet ? "Q" : ""); Stack& stack = st->get_stack(); @@ -1551,6 +1565,66 @@ int exec_load_message_addr(VmState* st, bool quiet) { return 0; } +int exec_load_std_message_addr(VmState* st, bool quiet) { + VM_LOG(st) << "execute LDSTDADDR" << (quiet ? "Q" : ""); + Stack& stack = st->get_stack(); + auto csr = stack.pop_cellslice(); + td::Ref addr{true}; + if (util::load_std_msg_addr_q(csr.write(), addr.write(), st->get_global_version(), quiet)) { + stack.push_cellslice(std::move(addr)); + stack.push_cellslice(std::move(csr)); + if (quiet) { + stack.push_bool(true); + } + } else { + stack.push_cellslice(std::move(csr)); + stack.push_bool(false); + } + return 0; +} + +int exec_load_opt_std_message_addr(VmState* st, bool quiet) { + VM_LOG(st) << "execute LDOPTSTDADDR" << (quiet ? "Q" : ""); + Stack& stack = st->get_stack(); + auto csr = stack.pop_cellslice(); + + if (!csr->have(2)) { + // No data to load the tag for the address (0b00 or 0b10) + if (quiet) { + stack.push_cellslice(std::move(csr)); + stack.push_bool(false); + return 0; + } + + throw VmError{Excno::cell_und}; + } + + auto tag = csr.write().prefetch_ulong(2); + if (tag == 0b00) { // addr_none$00 + csr.write().skip_first(2); + // addr_none -> push null + stack.push_null(); + stack.push_cellslice(std::move(csr)); + if (quiet) { + stack.push_bool(true); + } + return 0; + } + + td::Ref addr{true}; + if (util::load_std_msg_addr_q(csr.write(), addr.write(), st->get_global_version(), quiet)) { + stack.push_cellslice(std::move(addr)); + stack.push_cellslice(std::move(csr)); + if (quiet) { + stack.push_bool(true); + } + } else { + stack.push_cellslice(std::move(csr)); + stack.push_bool(false); + } + return 0; +} + bool parse_maybe_anycast(CellSlice& cs, StackEntry& res, int global_version) { res = StackEntry{}; if (cs.prefetch_ulong(1) != 1) { @@ -1722,6 +1796,118 @@ int exec_rewrite_message_addr(VmState* st, bool allow_var_addr, bool quiet) { return 0; } +bool is_valid_std_msg_addr(const Ref& cs, int global_version) { + if ((unsigned)cs->prefetch_ulong(2) != 0b10) { + // not a addr_std$10 + return false; + } + + if (global_version >= 10) { + if (cs->prefetch_ulong(3) == 0b10'1) { // 0b10 prefix and 0b1 for Maybe Anycast + // anycast addresses is disabled since TVM 11 + return false; + } + + return cs->size() == 3 + 8 + 256 && cs->size_refs() == 0; // anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; + } + + // Fallback to copy cell slice and check with anycast + Ref cloned_cs = cs; + if (!skip_std_message_addr(cloned_cs.write(), global_version)) { + return false; + } + return cloned_cs->size() == 0 && cloned_cs->size_refs() == 0; +} + +int exec_store_std_address(VmState* st, bool quiet) { + Stack& stack = st->get_stack(); + VM_LOG(st) << "execute STSTDADDR" << (quiet ? "Q" : ""); + stack.check_underflow(2); + auto builder = stack.pop_builder(); + auto cs = stack.pop_cellslice(); + bool is_std = is_valid_std_msg_addr(cs, st->get_global_version()); + + if (!builder->can_extend_by(cs->size(), cs->size_refs()) || !is_std) { + if (!quiet) { + if (!is_std) { + throw VmError{Excno::cell_ov, "not a MsgAddressInt"}; + } + throw VmError{Excno::cell_ov}; + } + stack.push_cellslice(std::move(cs)); + stack.push_builder(std::move(builder)); + stack.push_bool(true); + return 0; + } + cell_builder_add_slice(builder.write(), *cs); + stack.push_builder(std::move(builder)); + if (quiet) { + stack.push_bool(false); + } + return 0; +} + +int exec_store_opt_std_address(VmState* st, bool quiet) { + Stack& stack = st->get_stack(); + VM_LOG(st) << "execute STOPTSTDADDR" << (quiet ? "Q" : ""); + stack.check_underflow(2); + auto builder = stack.pop_builder(); + + auto cs_or_null = stack.pop(); + if (cs_or_null.is_null()) { + if (!builder->can_extend_by(2, 0)) { + // No room to store 0b00 for addr_none + if (!quiet) { + throw VmError{Excno::cell_ov}; + } + stack.push_null(); + stack.push_builder(std::move(builder)); + stack.push_bool(true); + return 0; + } + + // null is stored as addr_none$00 + builder.write().store_zeroes_bool(2); + stack.push_builder(std::move(builder)); + if (quiet) { + stack.push_bool(false); + } + return 0; + } + + auto cs = cs_or_null.as_slice(); + if (cs.is_null()) { + if (!quiet) { + throw VmError{Excno::type_chk, "not a cell slice"}; + } + stack.push_cellslice(std::move(cs)); + stack.push_builder(std::move(builder)); + stack.push_bool(true); + return 0; + } + bool is_std = is_valid_std_msg_addr(cs, st->get_global_version()); + + if (!builder->can_extend_by(cs->size(), cs->size_refs()) || !is_std) { + if (!quiet) { + if (!is_std) { + throw VmError{Excno::cell_ov, "not a MsgAddressInt"}; + } + throw VmError{Excno::cell_ov}; + } + stack.push_cellslice(std::move(cs)); + stack.push_builder(std::move(builder)); + stack.push_bool(true); + return 0; + } + + cell_builder_add_slice(builder.write(), *cs); + stack.push_builder(std::move(builder)); + if (quiet) { + stack.push_bool(false); + } + return 0; +} + void register_ton_currency_address_ops(OpcodeTable& cp0) { using namespace std::placeholders; cp0.insert(OpcodeInstr::mksimple(0xfa00, 16, "LDGRAMS", std::bind(exec_load_var_integer, _1, 4, false, false))) @@ -1743,7 +1929,15 @@ void register_ton_currency_address_ops(OpcodeTable& cp0) { .insert( OpcodeInstr::mksimple(0xfa46, 16, "REWRITEVARADDR", std::bind(exec_rewrite_message_addr, _1, true, false))) .insert( - OpcodeInstr::mksimple(0xfa47, 16, "REWRITEVARADDRQ", std::bind(exec_rewrite_message_addr, _1, true, true))); + OpcodeInstr::mksimple(0xfa47, 16, "REWRITEVARADDRQ", std::bind(exec_rewrite_message_addr, _1, true, true))) + .insert(OpcodeInstr::mksimple(0xfa48, 16, "LDSTDADDR", std::bind(exec_load_std_message_addr, _1, false))->require_version(12)) + .insert(OpcodeInstr::mksimple(0xfa49, 16, "LDSTDADDRQ", std::bind(exec_load_std_message_addr, _1, true))->require_version(12)) + .insert(OpcodeInstr::mksimple(0xfa50, 16, "LDOPTSTDADDR", std::bind(exec_load_opt_std_message_addr, _1, false))->require_version(12)) + .insert(OpcodeInstr::mksimple(0xfa51, 16, "LDOPTSTDADDRQ", std::bind(exec_load_opt_std_message_addr, _1, true))->require_version(12)) + .insert(OpcodeInstr::mksimple(0xfa52, 16, "STSTDADDR", std::bind(exec_store_std_address, _1, false))->require_version(12)) + .insert(OpcodeInstr::mksimple(0xfa53, 16, "STSTDADDRQ", std::bind(exec_store_std_address, _1, true))->require_version(12)) + .insert(OpcodeInstr::mksimple(0xfa54, 16, "STOPTSTDADDR", std::bind(exec_store_opt_std_address, _1, false))->require_version(12)) + .insert(OpcodeInstr::mksimple(0xfa55, 16, "STOPTSTDADDRQ", std::bind(exec_store_opt_std_address, _1, true))->require_version(12)); } static constexpr int output_actions_idx = 5; @@ -2170,6 +2364,18 @@ bool load_msg_addr_q(CellSlice& cs, CellSlice& res, int global_version, bool qui res.cut_tail(cs); return true; } +bool load_std_msg_addr_q(CellSlice& cs, CellSlice& res, int global_version, bool quiet) { + res = cs; + if (!skip_std_message_addr(cs, global_version)) { + cs = res; + if (quiet) { + return false; + } + throw VmError{Excno::cell_und, "cannot load a MsgAddressInt"}; + } + res.cut_tail(cs); + return true; +} bool parse_std_addr_q(CellSlice cs, ton::WorkchainId& res_wc, ton::StdSmcAddress& res_addr, int global_version, bool quiet) { // Like exec_rewrite_message_addr, but for std address case diff --git a/crypto/vm/tonops.h b/crypto/vm/tonops.h index d335050858..a5597202da 100644 --- a/crypto/vm/tonops.h +++ b/crypto/vm/tonops.h @@ -33,6 +33,7 @@ namespace util { bool load_var_integer_q(CellSlice& cs, td::RefInt256& res, int len_bits, bool sgnd, bool quiet); bool load_coins_q(CellSlice& cs, td::RefInt256& res, bool quiet); bool load_msg_addr_q(CellSlice& cs, CellSlice& res, int global_version, bool quiet); +bool load_std_msg_addr_q(CellSlice& cs, CellSlice& res, int global_version, bool quiet); bool parse_std_addr_q(CellSlice cs, ton::WorkchainId& res_wc, ton::StdSmcAddress& res_addr, int global_version, bool quiet); diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index bf62e6adc7..57387227ad 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -271,6 +271,14 @@ The bounced message has the same 0th and 1st bits in `extra_flags` as the origin ### New TVM instructions - `BTOS` (`b - s`) - same as `ENDC CTOS`, but without gas cost for cell creation and loading. Gas cost: `26`. - `HASHBU` (`b - hash`) - same as `ENDC HASHCU`, but without gas cost for cell creation. Gas cost: `26`. +- `LDSTDADDR` (`s - a s'`) - loads `addr_std$10`, if address is not `addr_std`, throws an error 9 (`cannot load a MsgAddressInt`). Gas cost: `26`. +- `LDSTDADDRQ` (`s - a s' -1 or s 0`) - quiet version of `LDSTDADDR`. Gas cost: `26`. +- `LDOPTSTDADDR` (`s - a s or null s`) - loads `addr_std$10` or `addr_none$00`, if address is `addr_none$00` pushes a Null, if address is not `addr_std` or `addr_none`, throws an error 9 (`cannot load a MsgAddressInt`). Gas cost: `26`. +- `LDOPTSTDADDRQ` (`s - (a s' -1 or null s' -1) or s 0`) - quiet version of `LDOPTSTDADDR`. Gas cost: `26`. +- `STSTDADDR` (`s b - b'`) - stores `addr_std$10`, if address is not `addr_std`, throws an error 9 (`cannot load a MsgAddressInt`). Gas cost: `26`. +- `STSTDADDRQ` (`s b - b' 0 or s b -1`) - quiet version of `STSTDADDR`. Gas cost: `26`. +- `STOPTSTDADDR` (`s b - b'`) - stores `addr_std$10` or Null. Null is stored as `addr_none$00`, if address is not `addr_std`, throws an error 9 (`cannot load a MsgAddressInt`). Gas cost: `26`. +- `STOPTSTDADDRQ` (`s b - b' 0 or s b -1`) - quiet version of `STOPTSTDADDR`. Gas cost: `26`. ### Other TVM changes - `SENDMSG` instruction treats `extra_flags` field accordingly (see above). diff --git a/test/regression-tests.ans b/test/regression-tests.ans index 16db1c9f1f..8fccc23abd 100644 --- a/test/regression-tests.ans +++ b/test/regression-tests.ans @@ -20,6 +20,7 @@ Test_Fift_test_get_extra_balance_default cc2428172b660ae66489e1b4868786654195137 Test_Fift_test_hash_ext_default 686fc5680feca5b3bb207768215b27f6872a95128762dee0d7f2c88bc492d62d Test_Fift_test_hmap_default c269246882039824bb5822e896c3e6e82ef8e1251b6b251f5af8ea9fb8d05067 Test_Fift_test_levels_default 9fba4a7c98aec9000f42846d6e5fd820343ba61d68f9139dd16c88ccda757cf3 +Test_Fift_test_load_store_std_addr_default 02a56bbe0e8db387858d4c6f111650f8470b9c22c82c0c0afcfc6c8da487e563 Test_Fift_test_namespaces_default e6419619c51332fb5e8bf22043ef415db686c47fe24f03061e5ad831014e7c6c Test_Fift_test_p256_default e1948ddd3d2686baa9f70fdf376ffcebbc2ec5f20eeb366cd856254e61fbfa31 Test_Fift_test_rist255_default f4d7558f200a656934f986145c19b1dedbe2ad029292a5a975576d6891e25fc4 From 66ddb328f38fbb589c56b5e88acc5f1d93211142 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 27 Oct 2025 15:22:50 +0300 Subject: [PATCH 419/524] Add more logs for BlockArchiver and similar things --- validator/apply-block.cpp | 2 +- validator/db/archiver.cpp | 12 +++++++++++- validator/db/archiver.hpp | 1 + validator/db/rootdb.cpp | 8 ++++---- validator/manager.cpp | 9 ++++++++- validator/validate-broadcast.cpp | 7 ++++--- 6 files changed, 29 insertions(+), 10 deletions(-) diff --git a/validator/apply-block.cpp b/validator/apply-block.cpp index 55f5071029..085302e69d 100644 --- a/validator/apply-block.cpp +++ b/validator/apply-block.cpp @@ -36,7 +36,7 @@ void ApplyBlock::abort_query(td::Status reason) { } void ApplyBlock::finish_query() { - VLOG(VALIDATOR_DEBUG) << "successfully finishing apply block query"; + VLOG(VALIDATOR_DEBUG) << "successfully finishing apply block query in " << perf_timer_.elapsed() << " s"; handle_->set_processed(); ValidatorInvariants::check_post_apply(handle_); diff --git a/validator/db/archiver.cpp b/validator/db/archiver.cpp index 93ba18ca29..7fff6d4471 100644 --- a/validator/db/archiver.cpp +++ b/validator/db/archiver.cpp @@ -30,6 +30,7 @@ BlockArchiver::BlockArchiver(BlockHandle handle, td::actor::ActorIdid().to_str(); if (handle_->id().is_masterchain()) { td::actor::send_closure(db_, &Db::get_block_state, handle_, [SelfId = actor_id(this), archive = archive_](td::Result> R) { @@ -46,6 +47,7 @@ void BlockArchiver::start_up() { } void BlockArchiver::move_handle() { + VLOG(VALIDATOR_DEBUG) << "move_handle"; if (handle_->handle_moved_to_archive()) { moved_handle(); } else { @@ -58,6 +60,7 @@ void BlockArchiver::move_handle() { } void BlockArchiver::moved_handle() { + VLOG(VALIDATOR_DEBUG) << "moved_handle"; CHECK(handle_->handle_moved_to_archive()); if (handle_->moved_to_archive()) { finish_query(); @@ -78,6 +81,7 @@ void BlockArchiver::moved_handle() { } void BlockArchiver::got_proof(td::BufferSlice data) { + VLOG(VALIDATOR_DEBUG) << "got_proof"; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); td::actor::send_closure(SelfId, &BlockArchiver::written_proof); @@ -87,6 +91,7 @@ void BlockArchiver::got_proof(td::BufferSlice data) { } void BlockArchiver::written_proof() { + VLOG(VALIDATOR_DEBUG) << "written_proof"; if (!handle_->inited_proof_link()) { written_proof_link(); return; @@ -102,6 +107,7 @@ void BlockArchiver::written_proof() { } void BlockArchiver::got_proof_link(td::BufferSlice data) { + VLOG(VALIDATOR_DEBUG) << "got_proof_link"; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); td::actor::send_closure(SelfId, &BlockArchiver::written_proof_link); @@ -111,6 +117,7 @@ void BlockArchiver::got_proof_link(td::BufferSlice data) { } void BlockArchiver::written_proof_link() { + VLOG(VALIDATOR_DEBUG) << "written_proof_link"; if (!handle_->received()) { written_block_data(); return; @@ -124,6 +131,7 @@ void BlockArchiver::written_proof_link() { } void BlockArchiver::got_block_data(td::BufferSlice data) { + VLOG(VALIDATOR_DEBUG) << "got_block_data"; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); td::actor::send_closure(SelfId, &BlockArchiver::written_block_data); @@ -133,6 +141,7 @@ void BlockArchiver::got_block_data(td::BufferSlice data) { } void BlockArchiver::written_block_data() { + VLOG(VALIDATOR_DEBUG) << "written_block_data"; handle_->set_moved_to_archive(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { @@ -143,6 +152,7 @@ void BlockArchiver::written_block_data() { } void BlockArchiver::finish_query() { + VLOG(VALIDATOR_DEBUG) << "finished archiving block in " << timer_.elapsed() << " s"; if (promise_) { promise_.set_value(td::Unit()); } @@ -150,8 +160,8 @@ void BlockArchiver::finish_query() { } void BlockArchiver::abort_query(td::Status reason) { + VLOG(VALIDATOR_WARNING) << "failed to archive block " << handle_->id() << ": " << reason; if (promise_) { - VLOG(VALIDATOR_WARNING) << "failed to archive block " << handle_->id() << ": " << reason; promise_.set_error(std::move(reason)); } stop(); diff --git a/validator/db/archiver.hpp b/validator/db/archiver.hpp index 9498977fdf..531e114c50 100644 --- a/validator/db/archiver.hpp +++ b/validator/db/archiver.hpp @@ -54,6 +54,7 @@ class BlockArchiver : public td::actor::Actor { td::actor::ActorId archive_; td::actor::ActorId db_; td::Promise promise_; + td::Timer timer_; }; } // namespace validator diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 636ca4200d..907a82f50b 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -389,8 +389,8 @@ void RootDb::try_get_static_file(FileHash file_hash, td::Promise promise) { - td::actor::create_actor("archiver", std::move(handle), archive_db_.get(), actor_id(this), - std::move(promise)) + td::actor::create_actor(PSTRING() << "archiver" << handle->id().id.to_str(), std::move(handle), + archive_db_.get(), actor_id(this), std::move(promise)) .release(); } @@ -464,8 +464,8 @@ void RootDb::start_up() { } void RootDb::archive(BlockHandle handle, td::Promise promise) { - td::actor::create_actor("archiveblock", std::move(handle), archive_db_.get(), actor_id(this), - std::move(promise)) + td::actor::create_actor(PSTRING() << "archiver" << handle->id().id.to_str(), std::move(handle), + archive_db_.get(), actor_id(this), std::move(promise)) .release(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 696e7b3691..9e7bcb62e6 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1595,6 +1595,7 @@ void ValidatorManagerImpl::new_block_cont(BlockHandle handle, td::Ref promise) { if (state->get_shard().is_masterchain() && handle->id().id.seqno > last_masterchain_seqno_) { if (handle->id().id.seqno == last_masterchain_seqno_ + 1) { + VLOG(VALIDATOR_DEBUG) << "new block " << handle->id().id.to_str() << " is the next masterchain block"; last_masterchain_seqno_ = handle->id().id.seqno; last_masterchain_state_ = td::Ref{state}; last_masterchain_block_id_ = handle->id(); @@ -1614,6 +1615,7 @@ void ValidatorManagerImpl::new_block_cont(BlockHandle handle, td::Refid(); last_masterchain_seqno_ = last_masterchain_block_id_.id.seqno; CHECK(it->first == last_masterchain_seqno_); + VLOG(VALIDATOR_DEBUG) << "processing pending masterchain block " << last_masterchain_block_id_.id.to_str(); auto l_promise = std::move(std::get<2>(it->second)); last_masterchain_block_handle_->set_processed(); @@ -1630,6 +1632,7 @@ void ValidatorManagerImpl::new_block_cont(BlockHandle handle, td::Refid().id.to_str() << " is too new masterchain block"; auto it = pending_masterchain_states_.find(handle->id().id.seqno); if (it != pending_masterchain_states_.end()) { std::get<2>(it->second).emplace_back(std::move(promise)); @@ -1642,12 +1645,14 @@ void ValidatorManagerImpl::new_block_cont(BlockHandle handle, td::Refid().id.to_str() << " is already processed"; handle->set_processed(); promise.set_value(td::Unit()); } } void ValidatorManagerImpl::new_block(BlockHandle handle, td::Ref state, td::Promise promise) { + VLOG(VALIDATOR_DEBUG) << "new block " << handle->id().id.to_str(); if (handle->is_applied()) { return new_block_cont(std::move(handle), std::move(state), std::move(promise)); } else { @@ -2296,7 +2301,9 @@ void ValidatorManagerImpl::new_masterchain_block() { td::actor::send_closure(actor, &ShardBlockRetainer::update_masterchain_state, last_masterchain_state_); } if (last_masterchain_seqno_ % 1024 == 0) { - LOG(WARNING) << "applied masterchain block " << last_masterchain_block_id_; + LOG(WARNING) << "applied masterchain block " << last_masterchain_block_id_.to_str(); + } else { + LOG(DEBUG) << "applied masterchain block " << last_masterchain_block_id_.to_str(); } } diff --git a/validator/validate-broadcast.cpp b/validator/validate-broadcast.cpp index 0450496973..189697c536 100644 --- a/validator/validate-broadcast.cpp +++ b/validator/validate-broadcast.cpp @@ -37,7 +37,8 @@ void ValidateBroadcast::abort_query(td::Status reason) { void ValidateBroadcast::finish_query() { if (promise_) { - VLOG(VALIDATOR_DEBUG) << "validated broadcast for " << broadcast_.block_id.to_str(); + VLOG(VALIDATOR_DEBUG) << "validated broadcast for " << broadcast_.block_id.to_str() << " in " + << perf_timer_.elapsed() << " s"; promise_.set_result(td::Unit()); } stop(); @@ -321,8 +322,8 @@ void ValidateBroadcast::checked_proof() { }); VLOG(VALIDATOR_DEBUG) << "apply block"; - td::actor::create_actor("applyblock", handle_->id(), data_, handle_->id(), manager_, timeout_, - std::move(P)) + td::actor::create_actor(PSTRING() << "apply" << handle_->id().id.to_str(), handle_->id(), data_, + handle_->id(), manager_, timeout_, std::move(P)) .release(); } else { finish_query(); From 7b7db09cad6a4639286d360577f24800e10b6195 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 27 Oct 2025 21:34:51 +0300 Subject: [PATCH 420/524] Add timeout to default failure signal handler (#1856) * Add timeout to default failure signal handler * Put init_alarm_signal under "if posix" --- tdutils/td/utils/port/signals.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tdutils/td/utils/port/signals.cpp b/tdutils/td/utils/port/signals.cpp index 12962f29e3..1bb485bec0 100644 --- a/tdutils/td/utils/port/signals.cpp +++ b/tdutils/td/utils/port/signals.cpp @@ -317,7 +317,26 @@ static void block_stdin() { #endif } +#if TD_PORT_POSIX +[[maybe_unused]] static void init_alarm_signal() { + struct sigaction act; + act.sa_handler = [](int) { + signal_safe_write("Signal handler timeout\n"); + block_stdin(); + _Exit(EXIT_FAILURE); + }; + sigemptyset(&act.sa_mask); + act.sa_flags = SA_RESETHAND | SA_ONSTACK; + sigaction(SIGALRM, &act, nullptr); + alarm(3); +} +#endif + [[maybe_unused]] static void default_failure_signal_handler(int sig) { +#if TD_PORT_POSIX + init_alarm_signal(); +#endif + Stacktrace::init(); signal_safe_write_signal_number(sig); @@ -325,6 +344,9 @@ static void block_stdin() { options.use_gdb = true; Stacktrace::print_to_stderr(options); +#if TD_PORT_POSIX + alarm(0); +#endif block_stdin(); _Exit(EXIT_FAILURE); } From b118ab31da8b1763a09e05c6565cb00a32979090 Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Mon, 27 Oct 2025 14:36:17 -0400 Subject: [PATCH 421/524] Moar CMake improvements (#1847) * Require OpenSSL from top-level CMakeLists * Automatically generate TLO files for TL schemas * Remove useless directory include in tddb target * Do not compile some validator sources 3 times Instead, put them into an object library and use it in 3 different validator targets. * Do not make ninja regenerate smartcont/provider-code.h on every run OUTPUT parameter is relative to the current binary directory by default, so we need to explicitly use ${CMAKE_CURRENT_SOURCE_DIR}. * Run CI in forks as well I faced some unexpected backlash from other developers because of disabling it in #1844. --- .../build-ton-linux-android-tonlib.yml | 2 -- .../build-ton-linux-arm64-appimage.yml | 2 -- .../build-ton-linux-arm64-shared.yml | 2 -- .../build-ton-linux-x86-64-appimage.yml | 2 -- .../build-ton-linux-x86-64-shared.yml | 2 -- .../build-ton-linux-x86-64-werror.yml | 2 -- .../build-ton-macos-13-x86-64-portable.yml | 2 -- .../build-ton-macos-14-arm64-portable.yml | 2 -- .../build-ton-macos-15-arm64-shared.yml | 2 -- .../build-ton-macos-arm64-shared.yml | 2 -- .../build-ton-macos-x86-64-shared.yml | 2 -- .../workflows/build-ton-wasm-emscripten.yml | 2 -- .../workflows/docker-ubuntu-branch-image.yml | 6 ------ .github/workflows/docker-ubuntu-image.yml | 6 ------ .github/workflows/ton-ccpcheck.yml | 2 -- .github/workflows/ton-x86-64-windows.yml | 1 - .gitignore | 1 + .gitmodules | 3 +++ CMakeLists.txt | 7 ++++++ catchain/CMakeLists.txt | 4 ---- create-hardfork/CMakeLists.txt | 5 ----- crypto/CMakeLists.txt | 4 ---- dht-server/CMakeLists.txt | 5 ----- dht/CMakeLists.txt | 4 ---- fec/CMakeLists.txt | 4 ---- overlay/CMakeLists.txt | 4 ---- rldp/CMakeLists.txt | 4 ---- rldp2/CMakeLists.txt | 4 ---- storage/CMakeLists.txt | 5 ----- storage/storage-daemon/CMakeLists.txt | 2 +- tddb/CMakeLists.txt | 1 - tdutils/CMakeLists.txt | 10 ++------- terminal/CMakeLists.txt | 4 ---- third-party/tl-parser | 1 + tl/generate/CMakeLists.txt | 9 +++++++- tl/generate/scheme/lite_api.tlo | Bin 20256 -> 0 bytes tl/generate/scheme/ton_api.tlo | Bin 118988 -> 0 bytes tl/generate/scheme/tonlib_api.tlo | Bin 34380 -> 0 bytes tl/generate/scheme/update-tlo.sh | 5 ----- tonlib/CMakeLists.txt | 4 ---- utils/CMakeLists.txt | 4 ---- validator-engine/CMakeLists.txt | 5 ----- validator-session/CMakeLists.txt | 4 ---- validator/CMakeLists.txt | 20 ++++++------------ validator/impl/CMakeLists.txt | 4 ---- 45 files changed, 30 insertions(+), 136 deletions(-) create mode 160000 third-party/tl-parser delete mode 100644 tl/generate/scheme/lite_api.tlo delete mode 100644 tl/generate/scheme/ton_api.tlo delete mode 100644 tl/generate/scheme/tonlib_api.tlo delete mode 100755 tl/generate/scheme/update-tlo.sh diff --git a/.github/workflows/build-ton-linux-android-tonlib.yml b/.github/workflows/build-ton-linux-android-tonlib.yml index b92ea9d409..ff47fa939e 100644 --- a/.github/workflows/build-ton-linux-android-tonlib.yml +++ b/.github/workflows/build-ton-linux-android-tonlib.yml @@ -4,8 +4,6 @@ on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: - if: github.repository == 'ton-blockchain/ton' - runs-on: ubuntu-22.04 steps: diff --git a/.github/workflows/build-ton-linux-arm64-appimage.yml b/.github/workflows/build-ton-linux-arm64-appimage.yml index a258fdbb74..cb8a49844c 100644 --- a/.github/workflows/build-ton-linux-arm64-appimage.yml +++ b/.github/workflows/build-ton-linux-arm64-appimage.yml @@ -4,8 +4,6 @@ on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: - if: github.repository == 'ton-blockchain/ton' - runs-on: ubuntu-22.04-arm steps: diff --git a/.github/workflows/build-ton-linux-arm64-shared.yml b/.github/workflows/build-ton-linux-arm64-shared.yml index 40f63509a5..ec3f10e2ae 100644 --- a/.github/workflows/build-ton-linux-arm64-shared.yml +++ b/.github/workflows/build-ton-linux-arm64-shared.yml @@ -4,8 +4,6 @@ on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: - if: github.repository == 'ton-blockchain/ton' - strategy: fail-fast: false matrix: diff --git a/.github/workflows/build-ton-linux-x86-64-appimage.yml b/.github/workflows/build-ton-linux-x86-64-appimage.yml index 36974b8044..0bebcbadc1 100644 --- a/.github/workflows/build-ton-linux-x86-64-appimage.yml +++ b/.github/workflows/build-ton-linux-x86-64-appimage.yml @@ -4,8 +4,6 @@ on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: - if: github.repository == 'ton-blockchain/ton' - runs-on: ubuntu-22.04 steps: diff --git a/.github/workflows/build-ton-linux-x86-64-shared.yml b/.github/workflows/build-ton-linux-x86-64-shared.yml index 9a61c9f42f..17e1716446 100644 --- a/.github/workflows/build-ton-linux-x86-64-shared.yml +++ b/.github/workflows/build-ton-linux-x86-64-shared.yml @@ -4,8 +4,6 @@ on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: - if: github.repository == 'ton-blockchain/ton' - strategy: fail-fast: false matrix: diff --git a/.github/workflows/build-ton-linux-x86-64-werror.yml b/.github/workflows/build-ton-linux-x86-64-werror.yml index cf783bfda6..84e76ea984 100644 --- a/.github/workflows/build-ton-linux-x86-64-werror.yml +++ b/.github/workflows/build-ton-linux-x86-64-werror.yml @@ -4,8 +4,6 @@ on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: - if: github.repository == 'ton-blockchain/ton' - runs-on: ubuntu-24.04 steps: diff --git a/.github/workflows/build-ton-macos-13-x86-64-portable.yml b/.github/workflows/build-ton-macos-13-x86-64-portable.yml index 8e4e044042..7bf760c3e3 100644 --- a/.github/workflows/build-ton-macos-13-x86-64-portable.yml +++ b/.github/workflows/build-ton-macos-13-x86-64-portable.yml @@ -4,8 +4,6 @@ on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: - if: github.repository == 'ton-blockchain/ton' - runs-on: macos-13 steps: diff --git a/.github/workflows/build-ton-macos-14-arm64-portable.yml b/.github/workflows/build-ton-macos-14-arm64-portable.yml index 3fa1a13da3..dbf06579d4 100644 --- a/.github/workflows/build-ton-macos-14-arm64-portable.yml +++ b/.github/workflows/build-ton-macos-14-arm64-portable.yml @@ -4,8 +4,6 @@ on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: - if: github.repository == 'ton-blockchain/ton' - runs-on: macos-14 steps: diff --git a/.github/workflows/build-ton-macos-15-arm64-shared.yml b/.github/workflows/build-ton-macos-15-arm64-shared.yml index dabf3e7717..77953bc4fe 100644 --- a/.github/workflows/build-ton-macos-15-arm64-shared.yml +++ b/.github/workflows/build-ton-macos-15-arm64-shared.yml @@ -4,8 +4,6 @@ on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: - if: github.repository == 'ton-blockchain/ton' - runs-on: macos-15 steps: diff --git a/.github/workflows/build-ton-macos-arm64-shared.yml b/.github/workflows/build-ton-macos-arm64-shared.yml index be0d78b735..e9c70f15d4 100644 --- a/.github/workflows/build-ton-macos-arm64-shared.yml +++ b/.github/workflows/build-ton-macos-arm64-shared.yml @@ -4,8 +4,6 @@ on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: - if: github.repository == 'ton-blockchain/ton' - runs-on: macos-14 steps: diff --git a/.github/workflows/build-ton-macos-x86-64-shared.yml b/.github/workflows/build-ton-macos-x86-64-shared.yml index 98f71ec3f7..9ac8c54460 100644 --- a/.github/workflows/build-ton-macos-x86-64-shared.yml +++ b/.github/workflows/build-ton-macos-x86-64-shared.yml @@ -4,8 +4,6 @@ on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: - if: github.repository == 'ton-blockchain/ton' - runs-on: macos-13 steps: diff --git a/.github/workflows/build-ton-wasm-emscripten.yml b/.github/workflows/build-ton-wasm-emscripten.yml index 7dcfbc9c53..ce09f530e1 100644 --- a/.github/workflows/build-ton-wasm-emscripten.yml +++ b/.github/workflows/build-ton-wasm-emscripten.yml @@ -4,8 +4,6 @@ on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: - if: github.repository == 'ton-blockchain/ton' - runs-on: ubuntu-22.04 steps: diff --git a/.github/workflows/docker-ubuntu-branch-image.yml b/.github/workflows/docker-ubuntu-branch-image.yml index f8e0f206c5..bd97ed233f 100644 --- a/.github/workflows/docker-ubuntu-branch-image.yml +++ b/.github/workflows/docker-ubuntu-branch-image.yml @@ -12,8 +12,6 @@ env: jobs: build-arm64: - if: github.repository == 'ton-blockchain/ton' - runs-on: ubuntu-22.04-arm steps: - name: Check out repository @@ -51,8 +49,6 @@ jobs: build-amd64: - if: github.repository == 'ton-blockchain/ton' - runs-on: ubuntu-22.04 steps: - name: Check out repository @@ -90,8 +86,6 @@ jobs: merge: - if: github.repository == 'ton-blockchain/ton' - runs-on: ubuntu-22.04 needs: - build-amd64 diff --git a/.github/workflows/docker-ubuntu-image.yml b/.github/workflows/docker-ubuntu-image.yml index 76cf3db625..7a5eee10ad 100644 --- a/.github/workflows/docker-ubuntu-image.yml +++ b/.github/workflows/docker-ubuntu-image.yml @@ -11,8 +11,6 @@ env: jobs: build-arm64: - if: github.repository == 'ton-blockchain/ton' - runs-on: ubuntu-22.04-arm steps: - name: Check out repository @@ -55,8 +53,6 @@ jobs: tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-arm64 build-amd64: - if: github.repository == 'ton-blockchain/ton' - runs-on: ubuntu-22.04 steps: - name: Check out repository @@ -99,8 +95,6 @@ jobs: tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-amd64 merge: - if: github.repository == 'ton-blockchain/ton' - runs-on: ubuntu-22.04 needs: - build-amd64 diff --git a/.github/workflows/ton-ccpcheck.yml b/.github/workflows/ton-ccpcheck.yml index 9b2352108d..42faac5789 100644 --- a/.github/workflows/ton-ccpcheck.yml +++ b/.github/workflows/ton-ccpcheck.yml @@ -4,8 +4,6 @@ on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: - if: github.repository == 'ton-blockchain/ton' - runs-on: ubuntu-22.04 steps: diff --git a/.github/workflows/ton-x86-64-windows.yml b/.github/workflows/ton-x86-64-windows.yml index 0f52413ed4..cbd3e13dc8 100644 --- a/.github/workflows/ton-x86-64-windows.yml +++ b/.github/workflows/ton-x86-64-windows.yml @@ -8,7 +8,6 @@ defaults: jobs: build: - if: github.repository == 'ton-blockchain/ton' runs-on: windows-2022 diff --git a/.gitignore b/.gitignore index ffcf936986..8cb80106cb 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ parsetab.py parsetab.pyc tdutils/generate/auto/ tl/generate/auto/ +tl/generate/scheme/*.tlo compile_commands.json crypto/block/block-auto.cpp crypto/block/block-auto.h diff --git a/.gitmodules b/.gitmodules index 325e3f4e0d..e4e307960e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -17,3 +17,6 @@ path = third-party/secp256k1 url = https://github.com/bitcoin-core/secp256k1 branch = v0.3.2 +[submodule "third-party/tl-parser"] + path = third-party/tl-parser + url = https://github.com/ton-blockchain/tl-parser.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 6e758c457b..fb5bc69bf4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -170,6 +170,9 @@ if (USE_LIBRAPTORQ) add_subdirectory(third-party/libraptorq EXCLUDE_FROM_ALL) endif() +message("Add tl-parser") +add_subdirectory(third-party/tl-parser EXCLUDE_FROM_ALL) + message("Add ton") set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH}) @@ -366,6 +369,10 @@ if (NOT LATEX_FOUND) endif() #END internal +if (NOT OPENSSL_FOUND) + find_package(OpenSSL REQUIRED) +endif() + function(target_link_libraries_system target) set(libs ${ARGN}) foreach(lib ${libs}) diff --git a/catchain/CMakeLists.txt b/catchain/CMakeLists.txt index 6175cb86b6..46b7c13420 100644 --- a/catchain/CMakeLists.txt +++ b/catchain/CMakeLists.txt @@ -1,7 +1,3 @@ -if (NOT OPENSSL_FOUND) - find_package(OpenSSL REQUIRED) -endif() - set(CATCHAIN_SOURCE catchain-received-block.cpp #catchain-receiver-fork.cpp diff --git a/create-hardfork/CMakeLists.txt b/create-hardfork/CMakeLists.txt index daa7f2e730..48bbc1e0fc 100644 --- a/create-hardfork/CMakeLists.txt +++ b/create-hardfork/CMakeLists.txt @@ -1,8 +1,3 @@ -if (NOT OPENSSL_FOUND) - find_package(OpenSSL REQUIRED) -endif() - - set(CREATE_HARDFORK_SOURCE create-hardfork.cpp ) diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 50f8903941..ec7b3870ad 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -1,7 +1,3 @@ -if (NOT OPENSSL_FOUND) - find_package(OpenSSL REQUIRED) -endif() - set(TON_CRYPTO_CORE_SOURCE Ed25519.cpp common/bigint.cpp diff --git a/dht-server/CMakeLists.txt b/dht-server/CMakeLists.txt index 07c93f2473..b7917fe43b 100644 --- a/dht-server/CMakeLists.txt +++ b/dht-server/CMakeLists.txt @@ -1,8 +1,3 @@ -if (NOT OPENSSL_FOUND) - find_package(OpenSSL REQUIRED) -endif() - - set (DHT_SERVER_SOURCE dht-server.cpp dht-server.hpp diff --git a/dht/CMakeLists.txt b/dht/CMakeLists.txt index f8d49a2d82..a2f63af58c 100644 --- a/dht/CMakeLists.txt +++ b/dht/CMakeLists.txt @@ -1,7 +1,3 @@ -if (NOT OPENSSL_FOUND) - find_package(OpenSSL REQUIRED) -endif() - set(DHT_SOURCE dht.cpp dht-remote-node.cpp diff --git a/fec/CMakeLists.txt b/fec/CMakeLists.txt index 8549dc0e55..721f988de7 100644 --- a/fec/CMakeLists.txt +++ b/fec/CMakeLists.txt @@ -1,7 +1,3 @@ -if (NOT OPENSSL_FOUND) - find_package(OpenSSL REQUIRED) -endif() - set(FEC_SOURCE fec.h fec.cpp diff --git a/overlay/CMakeLists.txt b/overlay/CMakeLists.txt index c2c9dc6173..e7b91d6e24 100644 --- a/overlay/CMakeLists.txt +++ b/overlay/CMakeLists.txt @@ -1,7 +1,3 @@ -if (NOT OPENSSL_FOUND) - find_package(OpenSSL REQUIRED) -endif() - set(OVERLAY_SOURCE overlay-manager.cpp overlay.cpp diff --git a/rldp/CMakeLists.txt b/rldp/CMakeLists.txt index 6884763282..82e238d3ca 100644 --- a/rldp/CMakeLists.txt +++ b/rldp/CMakeLists.txt @@ -1,7 +1,3 @@ -if (NOT OPENSSL_FOUND) - find_package(OpenSSL REQUIRED) -endif() - set(RLDP_SOURCE rldp.cpp rldp-peer.cpp diff --git a/rldp2/CMakeLists.txt b/rldp2/CMakeLists.txt index 984e7bb1a0..bf0c212c23 100644 --- a/rldp2/CMakeLists.txt +++ b/rldp2/CMakeLists.txt @@ -1,7 +1,3 @@ -if (NOT OPENSSL_FOUND) - find_package(OpenSSL REQUIRED) -endif() - if (NOT GSL_FOUND) find_package(GSL) endif() diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index 5a53e73988..9bd16356e3 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -1,8 +1,3 @@ -if (NOT OPENSSL_FOUND) - find_package(OpenSSL REQUIRED) -endif() - - set(STORAGE_SOURCE LoadSpeed.cpp MerkleTree.cpp diff --git a/storage/storage-daemon/CMakeLists.txt b/storage/storage-daemon/CMakeLists.txt index 5392d4ab90..c4f28dea5c 100644 --- a/storage/storage-daemon/CMakeLists.txt +++ b/storage/storage-daemon/CMakeLists.txt @@ -5,7 +5,7 @@ add_custom_command( COMMAND embed-provider-code smartcont/storage-provider-code.boc smartcont/provider-code.h COMMENT "Generate provider-code.h" DEPENDS embed-provider-code smartcont/storage-provider-code.boc - OUTPUT smartcont/provider-code.h + OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/smartcont/provider-code.h" ) set(STORAGE_DAEMON_SOURCE diff --git a/tddb/CMakeLists.txt b/tddb/CMakeLists.txt index 57e69403b4..ca4cbd48a5 100644 --- a/tddb/CMakeLists.txt +++ b/tddb/CMakeLists.txt @@ -58,7 +58,6 @@ if (TDDB_USE_ROCKSDB) target_sources(tddb PRIVATE ${TDDB_ROCKSDB_SOURCE}) target_compile_definitions(tddb PUBLIC -DTDDB_USE_ROCKSDB) target_link_libraries(tddb PUBLIC rocksdb) - target_include_directories(tddb PRIVATE $) endif() add_executable(io-bench test/io-bench.cpp) diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index e1d0c87b80..2450eb4a5d 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -1,5 +1,7 @@ option(TDUTILS_MIME_TYPE "Generate mime types conversion (gperf is required)" ON) +set(TD_HAVE_OPENSSL 1) + if (WIN32) if (WINGETOPT_FOUND) set(TD_HAVE_GETOPT 1) @@ -19,14 +21,6 @@ endif() if (ZLIB_FOUND) set(TD_HAVE_ZLIB 1) message(STATUS "Found ZLIB: ${ZLIB_INCLUDE_DIR} ${ZLIB_LIBRARIES}") - - # OpenSSL internally depends on zlib - if (NOT OPENSSL_FOUND) - find_package(OpenSSL) - endif() - if (OPENSSL_FOUND) - set(TD_HAVE_OPENSSL 1) - endif() endif() if (CRC32C_FOUND) diff --git a/terminal/CMakeLists.txt b/terminal/CMakeLists.txt index 2d0c503045..d3b06abb7a 100644 --- a/terminal/CMakeLists.txt +++ b/terminal/CMakeLists.txt @@ -1,7 +1,3 @@ -if (NOT OPENSSL_FOUND) - find_package(OpenSSL REQUIRED) -endif() - set(TERMINAL_SOURCE terminal.cpp diff --git a/third-party/tl-parser b/third-party/tl-parser new file mode 160000 index 0000000000..aba8925886 --- /dev/null +++ b/third-party/tl-parser @@ -0,0 +1 @@ +Subproject commit aba8925886a599b439f0a9a919abdaa363b8cb2e diff --git a/tl/generate/CMakeLists.txt b/tl/generate/CMakeLists.txt index 8ff7f79a47..4dac089e2d 100644 --- a/tl/generate/CMakeLists.txt +++ b/tl/generate/CMakeLists.txt @@ -60,7 +60,14 @@ add_executable(tonlib_generate_java_api ${TL_GENERATE_JAVA_SOURCE}) target_link_libraries(tonlib_generate_java_api PRIVATE tdtl) if (NOT CMAKE_CROSSCOMPILING) - find_program(PHP_EXECUTABLE php) + foreach(tl_scheme lite_api ton_api tonlib_api) + add_custom_command( + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/scheme + COMMAND $ -e "${tl_scheme}.tlo" "${tl_scheme}.tl" + DEPENDS tl-parser "scheme/${tl_scheme}.tl" + OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/scheme/${tl_scheme}.tlo" + ) + endforeach() set(GENERATE_COMMON_CMD generate_common) add_executable(generate_common ${TL_GENERATE_COMMON_SOURCE}) diff --git a/tl/generate/scheme/lite_api.tlo b/tl/generate/scheme/lite_api.tlo deleted file mode 100644 index c576a7f458bc48ea7b02c0adc0f6584cb12608b1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20256 zcmd^H3yfS<8J^PF_qK&XdDwzLd8t?wM8w1hZ7Ec@P`VXOg-q_w+}*wI?7h=_?{s%d zd?AonQGyZ)f{GemDi{(Fp`vNjsu55?j1UDyDUpYmpaQ<|eCOQ%&OiU&bLTEyqluHW zJ2U4$=YO34|Ihg!cb>TBgtwQH*J%bUZdn}h41m!3Vo zym_Vpn5-9QUiSSvR&<}Is0qRp7pEIA^$#AqbI_^z^;H|(D00TIo^5Y;es$9|E9+kD z4!hwtH(YgkwHh@2c&!&ToVYrEZqseL>;2Kd#@7#?xUPA??D^|!3a4$(xj&p~95j2* zaO}h`=Y?bdp5FM><$VY9IraMRxD(bQ5SQR0c>1el@mzd8r*CyVs7|bPVkc+LAs4-| zc_723=bYs_HP^_T)~CO~^TNYh@0o>X$nz%@kWKc>zjNP?*GfF+G~*4?n3nsnx7{j_ z!C&3-wPkXymZ06P-f+uh_pFd}HiwQMIoc$Zd0Tb-_y4&3@3Uy-bI+L8_DV9LzmUxd z$6k0Vr*%!>k9uQf<81rAs5JZsz6WcXVd(ntW^dB&Uw(Gteiv~5W+`zFe^0Oc{>7ZX zwz-KJ+4T9%GpD`#a3$f-W+?UNiBMg;w+xKGKQLpUa_A}&-gibhfh98GB zHX0}H+P)vRpJdJ^H)_`799-W$_U3($V6?T(d#@LdZw!NA)K2SD6Vq4oJsApuiDpAv zR7cz&J?>+dyoTZ4;6$+-R>vLB*UDqloxl9%-kT1!&r_Oer}vVf-8XWa!W^wWZQ;8% zzOroX|B$AelTSsTu)p4@mU z*M(u%uPL7c@6pWD)9|N@9y)~as2#AT)Hphu?rpzrA=?g_VX88TOxd-hE|`qAUL4BO_N zWb{(X13vW3=N?Yi3RpN_8-KT+93FZg!EbTHK;LK9*KBi6S33UiVg2Xd(G0Kd9Z@k%a=%7Q z^T_U{L$HBxP51G=@3?dqzQbp{Fz1x3X1?8rIqRHyr1rbW932OZm^cXUfdjlO-}CtE z%Gx6I@+IIyOhED{8ZgH|7#eiUR5LLX-dA6H#C-&d&m30APNS{Z37BA6dYnHyyrX-++&CT{gGc@bNjE#>I>qR#=tAql)Pdsf`PnsJf`p<9>e=Q zJiT+kgS9vw9n;lKOo#WJcYNlsI<^Z;Y=`$87N0q+j`1QB;>X|IgZcR!#^Fc<7~Xg zUdesD_5Rhgg{jSe7J{FC^vBO~80_ixmp^g)QXOYbnmBXS)}|ZIln{S{2gE&~u~&ax z@emW@1v;^Bgb*-YW@tIHKFMxN{s=$Tw9 z(;FcxFj7a{jTi7sm;nt_X_&~xz;Jy7j;?gnG9DpY(Za#!6OW+VKp1_MISfbSFfa~2 z$@aMnB@dnGljl(ueZa@*tBYL82+NgVjO#!iZAEZnJ&s%ETr}<^=Yp}Kaw$)$i-}qo zwuTq>U;4zsah=O)DL?|3B_rV03#tF$bDm%Rq38L%U(<>3XSGv>&LIy3TD#SI=+R&t zR3_Y+iiS%T0!{or0=V*M7)-Kpx!?Q5$Z}=tbAQc^V#W>D=e!ir=K-zHtwrP-q|@Lp zAydm^#r;b*@whjY$GRY#QkM9?Dv!acPC~U9PGl?P)*n$Yh{)K;AMv98p#HPTMdtRj zTtrd;Fwm9sTqvNsxpvp#+#sKUbD3VYeFkj=1HG5|M$M1*&ZQE@I3Ia5_9>?uSH_)a z+^i1#@-EOf>eby2FhzU}=;A!c{HNxD1-95>z<`$`XH8$|ZmsBadA`5t_xx%w*{Hk8 zfdQVS{V()tTEQ@edl$|${KLHq-yz??&pFyl@P~1(x1^Kc1><;O4`mK_7XK>H0?dDd z|G~ID4-J|nU!D9I@eLj@A20AN^UucgO(F?nTz89{A9?!S$&czYp@8opUO=DW#PA~X z;#HOx#~65lo&e86KbXQ?$T2HM13X__=Fe-g2&8}k!}Vit<#kYU(W?-$iV6gps5Bb7 zvupu(S8fZvZPsPxN&6yuDdPLUBbSS;4G^j4g3_cm0GKvC>Tt*3C+GfNZn;OjchW!h zPErQWP>KCTzK&!P=PagWjxt1---r^mo4Y zRofiVP#FoTiU!gH{FQ!&#|yY+&8N<4J{W7m-$8$hoSj(HU^n<>{V2ho)^IQm8p!u_hg4az?>BJJev{0x z4#z9&Pi=Tv{RwcuU*y~_qR(@sANyLw_xB@D$ZnlB-2*>d4zi9`�pT1~NeY4D{rk zQ$=JjFj+NmOOm<)tuiiIR;_!gE?ZHcB7&H0SDynKtH>(%EI>_) zbO`ghO>Cz{m$_~ubhlMnKoW-Z5VGX*m$k@AQH#VlWKXesxvl_`1+wI}#7L{{HVioF z>&u?+@pX_RCw_@m;O4Z-`lp~(*FOQnb)|?t^r!|AU64`8Kr6pfK=y-Gr{RovszTwJ z13ZC~`KeB_>4)qp%^0;0v?h`qd#`Q92enQF;%< z{%67VxzkuPy-)#$>mF#NvzqYYuUw1v5&NQs^$Yeadm$px3xRR$BY4)h;J0;`gu%gS zAsm>KpwK=;_DyJ$T#e+a>6^efx7j>Bpnb#oHXgAa$1nRYTKG->1;#o4#q7YKsts_- zv%f1noeeEnzJ;I9Bp=}8^vOOCO<&UIficiV{vl5`bG4By85*6}cMp|St?Dw3P0pZ= z^C5dr($t{>!WftFB44Sr5^g0L55nYt2OfF;_why4IyPFcKBqmB?kuIpE^@4cRQNqAgT)U@6e8ek&&};j;3kS^5FG>Fgo>;Q3Y9sO+_2sUlsU z!Z`S$yMGa1+mqxS$Y(%yS@K7mPB7&6X5D9!c#Gi3|H!@MaXZ1{t`59e@jLk9dn$W} z+T?Ijsf2N^W5i<~Z#2evqJ6-69KY;A;`q_Hf^n`_dA6G8$FvQy#fe6Kh7pax$!V1R zPHY+7?}TwKqfU8O>GO)@DTy}V;%@o#G_vt;0BYp?gu6+I0PJ zL&lyE4$R>=tbKUZRv#WU*a@Wf@hnO^_1(4jq4i@;J820dxDEe8`NySqy1AH z#7>IC26XWDG3p&7o3+V- z_>{e0i+(y)-JkWAtMykSLqUCCMc0*mU0>*a_6Ytg0Ii#}c9p2NtH}O!x(gF;vS({% zs($&A8N7*hoD%ip&|RYwzE1kqKjo;(h^cc>ya)Y;PeKo{^p~>dX6?04@8G>=Bs-I} zDU~R9Iv08eCR9^^-)ct#Ez(W~V<%L&QXok?hbNiK{OFy>EWd(3Q=`^1kNqjFrJf7g z^SzNX;g@|9hpgD|^}=r${OSQKXEG`g4Rev5&8DCS{VKWbOsX__QuTpcv(ERdvq;kZ z!o;2h{23D^yS1_vW4`+^`khhrY+XfnEzo{LKf#B*lMYU*{^BHmVr~W5_no=wzwEm> zwf4`Q{8<$#hoto)qJ9Ocd+v=sVJ|!NXZIReB^?}wcX_YXGNPUX7`=KymVE!U-W2o^ zfE@^Ym*g(@i2D=34iaulzXOLS8Ky6r%f_hh|)Fe%KpEWq%aWdmHF z2)C^J-}9MmkML(|)!wJLS=qltb$_zsJR2>d7kkTxtlO{JbH~T|vu279`50@7YRAwK zl83D6KECyahLDF?2e;KyiR!v^UU%Y|@X0#o(?>1&Ie#uy(S45ZZf-T{DO_{O_vPKa zps%)mD56j1`QCc%t_%6|iJ-5&6dRSO7D~F&Nfw%pscM?<#eQVjEX5yWg9OKY0MRXL zq8k_9FeLP07}W-S*-8#}s>jeC+H{(5$a>1rfBy11VYeCwN-A@yr_4*N<4&s|7l4F} zEP0-Q=$CbrZ?5w17I#s7*{43^qH>Y_zi-)c%Ypp)Ql&Hc_XMbK5;PJ`vN!TePo1_x z+$WeZL?y~AQ{I4bl=zGn^hiF3@M>)WdYacGNIb~=^S9sknm%FE=TYS+N!6ygTFL_z zvKjWQ55F6NPE{jJ8wav%33=EsuILk0+7mw~u-0PLAMcHNDqEz%Q#-{xxi-l_=4J=3 ze8WoqTNFwM)C0q&tA+Bj*e{wd^P-Rb@E^BuKa2~Xw%s;uX!Vg(&f8|gdM^v~{d0Yy zdzsXyE+dvyEynt_!N<%50-AWMRCrwg#&s;eC z4ze#v;Ya_~CE9hE4z``^41$-MC6-IvYq-@l(%P4JS#ih2l`as zzxs8Tw4en4%kP}<)!jM|(aP0l)wX00)I~q<_qdF74^;BYVY2_JJQCDqATad;{;c(7 v|I??x_4M7`{-+#|2SLLqo6ugsuYixd9KJr2&>gCpA61`XN(+yzv-b1fueCLS diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo deleted file mode 100644 index bf531b7af9deb192d7628c25692399ab8c7691e8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 118988 zcmeFa2b^40**`uc+ecJ5kf~=BtZB*-*fJB?=$D#Gjmf^{{Q!#&qvAL^PH!j=RBvs zcEPs0N1We2n?)k0O@F89>x&cJn~meIh5tqQNw3vzSvD<`Nl{TB`d|M-5QXFEfwP8| zPwUMma*+t3RsWM}Z1zW>o%n6#A1x;#mIfa(6}v|rXm3mMgPpqgP^w95n`LUXQi*i02+!OV;H)#4T+061@Yui8LzvkULMAHWx z$Y(Pt%}ZaR8l;2ednBs{dv!GMzKLi&q3!3e>;4CPAMww-|DtuQ_*s;pQXB7o-u(Sr zwEY~|g~<+%CoWRwAHaQ4%MDR2>%x50i48$;x9ob(?}jP97xkpm2}erSxSgM;PRQKT zr0T^o>Gott^FFCeTQp_0ee7AB zt?)~C+2K>I&jNUtkd4m%U9}fUI$Xg9KjWnzUZdNHrqhy0WLu)itPMUt^}s*(i$(LX z&S)~-JS~-p>G&EX^Mkv7d#izkunQ8oo)qGRt-rkg>33Ti3hP+g>&nI{hYxUJXC&L( z6WK&MmayUeqU;a-J@{G4Y%Z?^FnNc6!)6bCw_}{g%Yof->L+ZlW3T+xKRw)S&`a}- zXg+GwAu0PfxsdLw;~9lS)OYmqF-mo*qrdlTeqH&P~UEaCTEcotfFJ(??1C z_%~>5UViVPjRo+2?a{He1c!=bdf)F)aNT$%)PPZ$5DF{StOg zw#O>tHV>VBQ-6V1M@~D>r{8qnE)4`9W2-mMj@$fx?}7c_P_T(qimE8orq0;r<9568 z&;h`!VV5L&i^Koot{uM*-L)rE3Hp!9)#?aYP(QoP@V|TnJ{;JheC$zi-}A5^U~?L+ zRnK3(c;jUQUD#wgo>=bmJa+xPnQZDF;nwMIi|u zw#A8T4x>IBKc~EK^>xENcp6+}GrfuULQ1%)n$C}*d-;R^%)fiPS3NrW#8hYK zyl7n6jgwv27HBDl1b-p)EiCDz&x)q0L&!CI@!A!WLf}~lXD!0}m_Ajzhv7%zp=8k( zt3SVO*;Qfq$%(NSu~qPcTo+wAxYpCZqB=d5Bn1(qsRvX>ogcl@@mXcOQ$+{Bm~en8 zpEqh!lLI5vY}N1CFV)>u3wUP;XyRnr`%YPSN)_Pr@Xw0xo?qm9s~N7yyvm6moOy`C zr&9%=EKIlj{_9>E4txs|NB2C&zJyWmcaZntAlS{sNx!C>#FN+6!;42RyaAX+0R7`+bdd zG3WZu%azUu77@YbKhLD2FOG`$banMEqqw9-DBGF*+v=`7=KVx%BHfWpQ>3O@5F6hu ziRaopxOB><>+eq6^Zf<~-*hK6w!vPTIdS_TF04u|X@)G3ZI8y(c*Eow{xM|V`;Hn^ zSceAmF*SK(qqFv-w+^py>dff0&zl{4-re_BJ8=8Nv|WODSa){!osACMoMb*hog1|S zJN)FgThHk4z>^DNfIu0)9sZt~GoKk@!cV1nDrcI?Cg&@E-tP$3r%Pv1qN|&n$5v;v zsT0y%2Sa-`JK`h{rtIvO`}J)0IorUi(^7Qq;NcVY{DgG|_jl40t546pHZpXAql2Za zGpdEb7Ip4(qiM<&v*_{c{qXi5j(^>TLl4*v>Ll}}Fq3D~&xw=1Q#RJZM8+Mhkg5)y|?u9F6-w?`OOh;j=Irr zFKYF=k-}km*j_<0O!{iqtv2`Z;MUJE0<$jc4jt@y%uVA)`s&Y2(M&YC?X+aEJnEd@ zvA2)+*Ilj`E4&GN=3n|6(|`LXr%VZ;zdWCfvg)k9KIY=LpZ%7P{;X&&zmQV+xrwf} zM0R>2n@_eUX{N-!X4A8J!a{$$V?2%*IT@MF7nZ#B<%51-FmUMRZ;c@XZ(KKdj9;Ff zR0?jW)nTLG`Pr$b*z3+Eog}Ptg}qC*Jj36HW;t{hrlL#jx?7bv_bk8t_Q_^?y0rnw z&^DfHS8OwY`wcH%?3c@=Jz0Z%X5Pw&9~kP>6LlI`Pjl!6)$6=`+&TWXGC!T5AxZn9 zWS5T5n%|)M@9lox6@I!@A9i)-d%MiJ-cd|8nIE0X*x-rt0b> z+@|9HXtN3J$NP0pSwnXw7fmhBP*||G$CD@jXy#;}-PHI@4Z#=FBtd(U zVvL@;u=@EI+YUI+-_IyK^Lz4YMRy@hHCXxhaCpyjTYcdO*7}pCr@k2OmxC6RsB_8M z>1LjpefVI1y`nkHSy!&Q+22MM_QZ6`Yvpmb&X;cY^SCInT+I^L{cGK&t*>%!=fw~- zX1iUdPpHXk>yt}QS{u2Vlfh-IvGbMY8Rs^K#HGes&b&=fTaRA3@}FCW#HGBaNSi(T zCu5$TXu*}qsywcwY|;DYFYw0>p4`_sO7XEE^~f*%y3S#SCZG+`iXIi%V%dr{d;Z+7 zAB9T>wB~t*n17A=dgr%FB^@W zqo01!XuvL>Vwd-p>+jfoh=7-J*ybUw_|4=#BSf7z@w0e}_H(Qn*?9YW`Dy!0T3nCz zwpiCq?_78235^-uBQ#ITk`u-#?~6n#$%aAmvai1K^w2>*cs*HZjPIwHBY zwV%;9J&<#02zX2`+jMs4Be=)>YZx#h{yKSFpOJ3pngi8x`5TNb#2Iro7$hAX>`OHH;*!=bW%oo8b z^^&YMAF6utveVr*l*T%|7TpGac4GVO2CL9N#|vyF;(7k%ix<^pQ}J$|%qM|YSgmwBZ8~l#v4=HN|{1is6E7;(lA9q=1tiQ|GT~fA73M5^P`~z;6Y6O)D2t`qmSiGUq@P{+@TaLd7uMU?805oOe^7PQU>7!<%w&`K z-l=-mr%exMb`RLW3 zLoc`8daA2qhff~mo3y|yc>L_TAAGvGL2q7S8H?v`ZnV{(oY{KqSbKeP^k^4+)z(&b zY}xT{*KQ8a*sEoeanP=>tTc5Bc=W_0qHYrlZWoOyKI=jubA4kw%w!G{cVbAi-n0iO)RR`h8Cpe zMs2nj_SUc04L9g6>`o-&V*hjyzau}m{$|tfcm!+HYe((PgUmWkv$g7T;t#KGHTiSu zVDinXmuW*s&bQQ4$(xPG)uZou)>3b7A|JKVJLlSw)utUBK4mku*8k194}WO-D|Khe zUrUWkD$}`NS?fBj5%QWcBNj}W`{TnKUHj9%E-_CNWcRP${qAd?eiXcliLy-v(HZjQ zt}l9R*sE8p4exQwu!=!~H&*0~Hoq*~j^@2HdAEW(x*^ag&|c@n*FQMe^UF5MG*nko z)K4~h^8WEs9;c4lZEI_5l{=pI7s=Za+5}cg-;cjn^UzSC1JqG^n4N|id^Tl0y#9yP zFM08#t=X+d{Yf$Xbv^QrTYdFH=y37shaZ)G0eWHd*nZKtHcP#t?QPqpGj8|dtd>vO z!1O`Z%e00^9@v8Q={7J*NarSFwC3E|lU}kYo3PLKeSF|Go4NXt^-JU`Q?9!C{c%Bb z7v@#o#htH>?^%znIBQ>s_LX~n@Eo#WItSuT5W z>?dyjDs>21O6YO)qenb8(I!JkUG_Qt+LqdU%uV}kxoF=nk}p>QuEh9mxACv8uFX?| z=xjQ(wf}~<9zJ0J!C~yvJT)8ZOfFS9gin9XCGyn}&jV z_C2i2-&xSnWWdr`UBwmCe&{cD*yF>N!3FRbka~0EL3Or{e(TS}3hLwv1TpKlZS{kb zk8O18@whn>sUd)%xtjZqy!X?A3@;}D3EoEI%F(y%HpH#dL!QM;;`VXxDdTGv*Kr+e zj4y(}W)tB|W?D9i~NMzH|l-O%#$^n}!D*wlDSHH$`Dkd4sI;`Il zKiT41*Y>kA*$&DZDXq4`S3bDoOc#E3XO;&Fz;Dt3UkA@^uXOFVf1;NMu_i3@bHk^X zsxJS36LD1FV2a~uVn z{K4mL8*t9LEt$8r6dkeQ$!OZd(AAx#&2NdgbH2q!w`oQ7_ZqwfJ+-00X4j*a^*wxm z2fHYdN^~V?>B5HlsIQMZZHNaS)5{ThX#sW2dXTAR$^Yy%*jGm_VOZPHE>E1iVx(Kg zaRA+sIx{qWTF@ovZ8N*NZn&?mcQVDMi`3uO4fE4<>$&r?K{CEE`?c<|0W|Z|afqz+ z-m#DWc*)hCURhq9FqM&9LH*j48M_}o)JKz}uf6@ZKTx^%Hokh`v5RI-k}+3ecD*9| zR=ak2_~k)9Tsko_^~CmB`Q(<7VZJ)F?Ij&oli#*_cg*;D=h2pWEVb?2ZP{5ZkBs-# z&0#&B##-2En9z%=#H5Y>Zd0GTYm^U1o;F>os8*Es+|AFwEBr&zRkied3L-<%y=s4L zZ*Lbm^Xj8!k+q92+h>d%=kcuSX?gXoNM1p;$^4y5>Mk|;o|%p>=53lPxD?S@-Jia= ze1t)t;vaRTdh-kDnzYe|_nP(e6DD2i`a{$G-XGV@7--OSROO?`e!1B@C$1GfqIBg9 z9Z+?8G}fuMEAf1L5Z{+Qb;X6kM-&c+Y1hHbdQ3mEI{ny0gJ;zRD8_kdUT=i8>22js zV~^R=z>zTR;8){C{8qJe z-35pd%6_H}U!f0w8eMbPtv%VFSGCc770$f^xWD3shnwa=$Jyx>TUYIQ+Wq*3zwtwz zUH-fGu4>eo6-{9;L6|z~&WkpE=LP-^8sKTf8?R0-qfOsYg`MsJAbPknf%#<~P{#pL zJgECI;`)9J{5v*g^T!z$e@j?(Pe!|UPX_*$@c3K8kDL*GvcA)~FQXZPMk2UFL-Jfs zKBF#Tpo*rBm=DyEco+ZV)@S;;c$4Y;4pU0v#r+zlE=;`B&VKRD0=zr#SQ0Pp(xDUrQz0 zu~!B^*iGse<>gu0&*K1?yixy2-otM=qq~$|a6Y;}Qt^nY2{73Y_-xnjBA0Aesl`Xo_}Y7AnzN z0@kL->%MNBv!zuwCD_zF(g(5+>$jYE2_@fB+J32750t--h6=Gv7u|J3iH(vlSjGn6 zX{9rctw}n&s6m7DS;cS`Ts9rNK7ZLxql@dnl^wYoWUs@pClPN|8KH%qgcGI?)J1s- zU$N;*_7k92-;V1PTBB?;uok4N2FGvWt%@^cr69O4K3L^Yb4=}{XVd#-EB?025LfRy zi(rpuUqf4kT-C0BYh|0f+GuR`)7$TEa%tpfm3ZBZ|w8UDQFYKC`RO_{kSUT_APsOyk?U}iv?7wZe51#e2V~4uB zCfp0v4QCDo&*wIV#-CzMox@xjX20ywz+4LawhsoO^b7LonoPL-10KwDBF_DZ2AE%hQZiI-9<-qa2Fl^U5pO; z->`ImOVUyI*`@TpFZ^4|j=v?Wy4NnF@3q6fB`p4yu(;o@hE`wH9<yl)fMFZp zo2;|l1uxxL)xD&-CrvjcQ33A2lX9DFE+`jA9f@uWQU@`eyUtEWNwsIHn=4_+%;VVn4V?Ih8@E1njnwev6tM2Y; zVPQ_ve9M4+Yk|{IqKUJV+$O$fue)oTO5h_UZY-`xXWp@+;>}T_i7$)} zP5#tfiF8p3?n=bJrJv((shcxrAGETDdKtbm5qBs;-yC1f@wx0L5PvrlVm{_=9g!f9gI;J#3EP-_nko z(O9ost$JvC;C@T43tQpe9PeQ-+|O#3<8Sm09}YZo9{vrxux;=+e%C)Sjm_-FN$`R) z6kv=lVbE8+d1ZZvzBe(d@5pTKiGw)q&V&ryfA&dgmcXC175Fd10cH@PpIOef1U?O+ z{cUY*)a#Avu0cU}=i+qH&#{0TO81>UTUG0O0hw-I-RtO+H?JEPLMnhZ`u##wb=wF3NXr2!;~PHs%~eODTJV35KMJqd3Q3Kpuu!749gCgG=%n*J0Hxw zF8;e1?Sx;F0MApQN80^_L?@6w=2MXtv8? zfa!DGE=IqK_S0rt*?Y8}O#eczQffSC^7!e{eaC9Qc6FpSurO~NE5hVKChiws{pS!J zBN%r&gJ{6Gp(~s2#+EUz=NZJU{2=@`oL(=HHdTTCB) zit>4HsztR+)8A_HnS3-wGoEy&8o!{nTajsR&*3s7OZkTQm3J?)o&dq$vK{wvECzmL zjD<{epK(0fn!?pCE>DOX+&{2C0;h)1vBbM$88mYd{0*AyhvoXSi!&&G0;fp_cqAR~ zj%M6@=-ttb@1WP8fFVbDmM)jvUe`4JPL;Q(q8O0EmiPh2%n!=ivzaa`gTM1HNM|*G zYt5-kfEgBY02cQIvrM2{>9`cb;lF^BZ2)jp4jj51hD96%Y>@8E`jt+1#&sbl>PlOJ zPJixSTAd1lUb&?X&jB;;3I|TE8v(bzPc=)b(d!Egk8_~l#e^SB)9gzXAIbq+x%=dL z7*nI3nJ0}=^$G+jK*05JFJMr8*$*=;_CE%}HpFSxTJ+nc;N^TU!`3>uWh{igl+V(+ zU;IE@mUFvF{<(B!a&S+MTf$DRs^`QAgrDcHb#GHK=}>mTc&pcRG_! z;0|+%SH=(-_ZTPOCC*!Cp0&Hg*-j^d87K4t`E59rFX7xB%}D1Gl%t5)6r>}Q_WrAW zwu{a)lssx1INz(@LIX8+P~AOFSrp-VvvD``Iv?OF2L{NYB-3DacEBO#XqR-qJ>cbQW?HOM$Wl5bi5o&#(=fEAl|`~V%*`~d-Fi;tcl=n@C=^--D)@-6?JLzDE(p| zip&eDX|<1X#bmt17!&UTa6OK-peOn9-d50KaNSrDHfULq@S|L|*G-WtRjM2)+0E?$ z`~>=ug3}WlrC`lmiQB4-`Oq)TY18F~*ln3};{|Yoz6o2|9qsKRH85>%uO?iWz8HFx zdiT9e(V^d_z<1~y{DWUQ+fDq&T185%ctx&j+7@*s4ZLTe$BcBI3J7rOi~#Um`dk(J zlT$gUy;iC*vmOkXpBrcm5oi5d*}PG<}b67Xim?FxdSJ)y3QW_hEdKT2JNMZ@c|Ax>Asbr;qKAEoTTI)F@4 z2j+V(%oM&u_BsNF4L61CHOX{=(BND8v7$pTV)bc!`tC$b4$*yoSd=Z}6H(+z% z!Vfu^`L++59p3iA_X6M8$i4-3FBSi6F$6qwI|m+H3^C!FQj|BhdWLEhdn|lyR>8VX z$DqpJOxXUh4EdwFZ-v5LG_Dq|NuRL2x#z8)4$CrLjD8ehqrj%5SiQnpj78gKL7FH*A2%j9RNz43UD3Niwo z>*JfOzt48xy2yL8O4ZjFqH~*?1F+^{4b}?9Z0vHu{Bce5p;TN}Ko&nVEKNPdZ6W&D}W(J)8a= zW{z9dVV*!s>ICnI89wEn(?glGMHrt#9f?2a?J*4R@m zWmX@Mby!}|k^BJ0eA|prNAMl|@R(v_;zyYIi?A^Sd`bR;-de+`3g0FF>l-7buR!0W zVq_^9S^wZm^1t|yKDUi@+n{p)X81#MThu+rhxg&u*Y8kA(pj+kG4Y{6bnq+?0WuA4 zb3yH$@!8rtm$R%U4^Q24(jqV32w7?AmFWYAt$yHpcg{E;WwQPB%|~fQUia~)EyKhL zIWA^?Km$1OL(W$D&ppQ$zNHuASJfhTXHWWHMUL*G#l>^|0Tn)_U~8Pqk-Ukx(=w9LoT{G=!F6|Bu()K33Ov-UFo57S8c(chfUfV0m~PTOMK# zd1GY8*v)0cTgFv#$-WKp7t@bT?vvh1ythRux#`|bmYd_?zgq4Mk;VT$V?w>pi0`sZ zl*R0O@}Wnb^MilDN9DW->k&Am|M=f})N5h*E@_9+8}Hdc4B{X7koIOifS+N|7Udg~ zPDqI08+gDxqw*81<2aRo7xR0vo%N+hUQmoOys6}Xs=C$g)^Yt6Ft|g2>oZ@V$@RdO ze6Las-KY@Xz!$fru=CHjJDKT09?5S|4v2T2m>0m;xH9O>0Wt*D!#xMGUafa`v8rgR zhr5wwy-FIN;q6U)v(>>Hoh%dU5c**}&|%O!o~%oh71)^OCS~3v_bV%%(LOik$Ltq) zJ&VhO{0{UaZ~R`U7x^O*d;?D$Gd3m;7K;avXMJKn_>#O&eDmS`y!T6!>itr@X$s!4 z1`qtj`lT>!x*B<`j(G;sHp4YKaHh`<>Y}-k1l^TIw`f7U%jF>P(*bN8R!)q;036`%B9TH$F< z(fhl2zZdNSzFVwaFkZleE`G30?&*V$$NP>FFK-XoSls4&!QdC)WLzv}b91a|`Y7uY zvOtfroaF^Ql;a0^&3B1eR(uCvunqKR;x^lEvG~Wdfm0E7XrJzT<2V_|H>r<}iOHqT zXa?yA^d*nR+saBk_y!)i->6c~Qn*&+(~mJvAZZ8b*DMb@w<=*+$KXx!AMoC@k^$c# z1GkehZ`*>^cTh*--hblP$0Cm5@T=1Vak#t}Z$<+*V3{v`lXZ_?_4coa6xLN*oJF{} zKJ!*ezcw0sp0=Qg-+*@bqep`Etyr6q&YY#U-LQF~EZv0<=Ym=!!g6t&DkT^EVxa7I zO8A*g5A7>c505M4FV{`E;?!7-=5X^1(F00-aR0NZ`GY(SiFSI)B~Ld_(q>To^sle) zw#m`D!>=!x5Fm$f<(*RYe@fTy&i(0&Mw_6`I+=FnXJdW(iQj#3p3!a;{yE9?l1)XU zwtn(+cw@WWYuE0s9}Fxu6`k8_&V0>~#ZAvm8nRzRjD{ZZv-!$B)8@W`_q&A*F75@1 zcApFqzl_~B{9iu%^M?%HRqI3^XYyXE9ocAXGyE?fHFz?pxQ%bJ?~&&{Kv!$U>9aSs z&w=vX*?fCW0iKvHa*96X4;_cx4ApbZ$#guiTqi zj59ZS{|?vZ_*CN*bI%);155pwYmH^N3&kOhy0V;YV)FOpi#J~8@uy|uoC#%+kNK-| z_$xL4Z|3cFB(7R>C(+lpZari&DnquQ`2$Ip<%JB=9&aX_y15U{HM2d`8qhQv+b=;m zWG<&li0{?zGtnALkWI?n$R9jz<9(&T5L7?R*azc7t{*h+0pALM{trI8)t~098tn5s zZCR8_8UK}j2OZXWOKe5waUcW7LDZ{^(`}mKPMsishAdL<*M?77Sx0iG^392q)6TYv zkH|?s-V`?Nn~bZuBGaDicc3ZxI_J&T|IttLMMq^~iZkYm^QVpF%SOB5^DA2rcS$G8 zQ$d_*bN{`*@*H$zYFU2=r`p+I=o+%y__^hP8;*ABDqV2A!FTDG)yi)u8alIbrmo>5 zReIkBg9Ns(8U^EBVibg@Qo-m(hH}})lrO4sZ*N{utO5Vh=JD1ye(Knqd{Ik@Z_*cS z{WNIW+Po?gds%{b0UgQ9vFGRZXp}zSC>`Iy&!+Z4;Fa`ud+(mL=tJRy+FvvS_zwE) zhjjn9IzNY8nj4U5FZL_cl{79rsqxRyH?Bl-9_@?Zo0P9#ERjmZ+uU_D^cKuHDL&$D1$VNeOxmB<9&CIJ*KNQ$Y64F(zXR_)R~3Ylc?KVn z=kMJ7c^39jJY(_+dZGs{)_!Nw0n$Fj_+{LvBXRG2Z_{=dL%6t=ZfCS@3f!kmd2m1> z-}<2y#tlAfd|f(u+h;H?P*bMO(HuS8O)sA(>1tLQR#QEcOIw-yE=;|bAx1cY2iZl( znBp-6G#wPGA73+emlq(P<}u#3*g0N~@9S$9)RDMPdEx5oFs5*D({Y7trZ*9HE>F%y zP%ddy<@BwBvRww{VL|gaz$IzVzk9n^J-Iq!D*2*)%lIy77SlP~6?G);vigrt^>8DS zV6#Ckg6|S{m_1a|#!L@*B)v~B+&YIGTI&(V`&pVu1Z9$D7|; zC2?N3k3_=Y7<1)u;45f@_%R>Aqrd1e_(fL^uJ!r_rF1vWp-j>&<|~XJbtL{Dz0&bnCG8!-&#V*cksmPN^E4*s(V+P0m`FVMF_ z;kZ8z!bVUII^0GosOzA&nG_1dKY71N7(D`}LcbC3;*f2P`2=0bXXg7Ow^8jy*LBYx zJU;I*b!Q#-?WVq=M{YBW8+9b^%eLElY@^lzuF^neBYTD!QbFJD0U%+upQt7=6!y0|0~M-=JP4& z%0-7MSkoePEtygEELEA%GrzI2^(kWFm3Q)@50xgNMFe{Ac)@#jbc-z2|b z;vnj1i|gEL&h?#_D_N$=e;+ka4-NOvUDiDul^wDOqu3-)+!%-t9@bS@*&LGM{R zt8(BeXbd3r0{U8`Q7BnaCpdZ&sn45fIHF9JF5)cirTByR`^_k@pX&He9G0h37w|+m^(0 zZJrOMQ}`SH$>UVn2Bc3yjw(kEPT)*^uswlO+Vi{9_I$s=u_qsQz&A;k?kh{<`V-Ux zUX!-Du8KDbx!+~Fz$xion>lg&Aue6D4V-SwO=R1nu>{IMpZl3&au?f^4ZT!GQ>n}{ z@4b&8+YZ_HzN1Wj!Gq)%FoO#DCD)J9L~A61GO4d({MOPG3%x#vt4Gz`q0LUfDe15M z=&i$R9QqW7BlsIMIe)OxWc>qP;yd=dyYH=b@O8u@5&SLj(b&@EFGy$72bczMNE++T z?!L3pp@Aa;G-*${ZX|;5Vf9tazDyVRB;Cnxx1PZ2Ef5LYnErOsem4k^+>0KDVcG^erW9zhBR0pR*0w7lB981PsRwkEZUp zP$vDcRG%28&#S;4h{gI%ly79e$+CifDeJ>0?D+||cP|Z%MDV@9wg_PKv$YMxD%LCP z-3I~6fk%wtGUfyL6#)Go^5D7GMutu>eVy!#Ly#1A=SI`C!AU1Z2mZw5Z-N$#nEPO4O#ajzt^>Qc5H z))#0=K2MzVowBhOePy!Vo34NZkC@y94|E2zolX+c%mrPu<8m9!??46X-i6T`ZKuxO zVInBb4cGAA%^0|;mu?QFhrU!T)RA*d;AeO|nd^?`W1X$tw5vSPI;9nln|K!(1=A_4 zvqOkFTLjXfj$OSimUMa>tp_tFH|uJ8;p$c|bKS+&K~Uc!&t*a{^zK)ZMi2Sk5cB0` z-J!H38>5HQdQz!Y-GZE_r-bcH-yA^?aM8q6A-uMoim0YPvtPo$9JHz(fBE*1cSc+N zQm+aCMz-GyHGN;a?~UF+e}UO{fk)Eby7IA$xUX>gDyOeN8EE1XRn-bWJQdGI&z#&renjMUQc zv24@>0?gY#Ic18JGwhjIuA`r+&X)sD_Gz2qd^vDRUM|nSxF#S*sCNM7(`X|V z?Tr8ieDJ)Wxvpl$JaZl^NQYm)vQL3e)YH8?aq0m?__4NgQ~v{VOX$WyI(VF-izn$V zE;{4s+*;@FOF33TCaDMT6JQgm2h6cY04H@(YHVM_{5ZX}8N1;>13Z{k?rOjrV9oxo zPVFgE5*GCTvJYVy!K;*U*td__drQCHNf~p~^|rDIU?C^k33#k>78;yzWwj0T%597J z04~YL5pRyV(Vs6-QmY5J3)Q|!ssK3Y|Ha0e71*d_xp$2D06xjbq_1|}YIC1HTt3vp z<3I^Gsf)^-8i*H-ji6=f0eB?+J&(C*+(;jN9>`8p6GniM{uXweOK*nI<@7ZEb_l|k z%zlE_U};_o9jgWkf@z?jh~`8`6M9pbsGCpAwI9|mc$M;<(>wO|@qYQ1WB8Pax6nlb zG*qXtD-;P>spEe1BpdGLl-Uya3!;mqj1rCE4EWoUf$J4(Tkod&nMfHm`ijT3V05Re`79w`&sr6;BwBq()+sT+*=DW zRp7vwL&6X9Xge*lRj;y^pxd*0!a~Nwc7raY{=nO}e0BrDg(T}-CF*of(g@(8OP*&f z6%%WPnm`xw-TC-?$I2L!BaFRVl8LFy2>;TXTKQId=9RyS9`KzHz4&>VxS zDfrG%k8mpuE8S7FjXrQm`p9AJ^yiX1i7a5GFBO|p1%KwaL+Y3H0GyH!uS1Tf&EucL>JSrapPlzWkwN6?czu1cJH)*#m}n>*;;?QR_n=#B|0le{%J=@_}6 zdlC^gk-8|QE0k9(E4({X&RH;@pe^~VU9rso-&}3;QMpW7`xDBgKbdzR$hOEdP)E|3 zxANf!hWh-Lq;=jeK)Iw*B^5!4OkFS?)RS~NFCTY~KS!ZQOIm2HXA#{aqUPZu5x{|- zo1>SwW2A3aQq8l15A*KTzR{)b-EO`zni2wkW*|MiOYYv@&)eGRUPCqCF(#=vQE7|k3=a_ zJLNB0gOKNqilGNfxFH2_bfZp=Hn-7v36|f4FIQuI51*8eZ3g{o2t7WAJOMthN2AJS zh>4L1;9xgw3=5;{N*X0{whP%LFQ{La%(bQwZrbQBL5Cs0$$kuQRRl-NVib|}F9@av zc58Jm@gp$6_{Vbb7$uPEGC9XwfpGol=7rX_aIp32{2px%pq+vAteyG=(e9_7ZR;Jr5ZGB>fQwKGo{amuRr*bw)YBNc!rot`hNR z)#G=Z#B`W8@bEX}|A6u5+l92J_0Cg|SOXS3aKBP=&MHLAlzkiXDES7D?@eew-rwKQ zMw;$SE}B}L!P+!nByVBH>Xpto9QuGi1&xD&SL*BJ$v>Jo+1K}}88&rrYcUOW+mneb z4~hXV?G|P&U_t7UUz$DxJd%$uwjFSse{N0jF~28I6P`VZg?L92aFTwR`?{o-S++CQ zKk7(Y9}e%CZlfhSkO<&Fi*xBxCUNZ6`O@uv{p2ar&Dq}s%fI1TY^a(YYZGE~ATw`}u-ZNR$0bSlBSSq(H<+J4?i3#%Q zO4SA1#|b*u(EWLblfeOx*`N3K`h7%MXO>p$%FMxP+VBQSyp@(smAXuPtWW4d>J$43 znnUVSZRnjHkDw0p$uXd$KE>jMt$jzYTv_;bL&$YL$YaI@@F!&i?yW=0s92@Sr~?_x z>E9}MRRz4`THC-}PpQi#QtiC@>@X1IAGHKWo33clOdbDG{gP?VQa%`V9aN6L*ask+ z)X&*J8T0f+i+(a$wchQvCBVuyT+H@N4|OCxtb_Y=P1oRa=-D=X5eZcSEa>t0oZhLT z^V-Gj5LP$fi+u++-cA2f1Elwu^c~ zlB5|a+AedHTyK`V0SDl)*|nACYl^|~T^R5${Lo&_H=pS)%ARyQpOyQ{%~)OElg`qP zs%Tr1cASdwql9C>fV`TT9((=us-K)yOJmjMY?R(l$sSPk*3{oho9j0myrd31n)24> zdb`>(-oxTSF!dTHN2aA~5H$xT9P+5OMBOQV{^YgEr?Sm6~=@ zJ;`rGlFv9=dnWA{b@aYWACwVKv(@h!`VNgPhIM^9u$9ej(+-utQ+=?4Dc#t0}l-z zUEHb@!#8PnsV{R4V`J=F1wTuxDLKWG>CbEr@FRJK?2QJ`^ju5?-@!M>&i@qOuw_M# zHk_5%U-|j+)Anb%SXRiu-!NzRqjz$%h6q_z#@eC%8gG6;UhY$+ywb11PqB4rrY+wP z0qt+k{KNM~`)SLguMy?}zPO=uuK3O5J|hGj_~7CxdTfBZcTq0=G0ga&)P1AnJ>J9& z{LbtvZTAkcuE3MDQ|*C&IAc_R4Df=5$*ykP+5vk*4%SteIaE3RX4=wkV5g6^ng8HO z3++X;3>+s$nKt`Mts1W6TJbD7c1t7gRs&+_NeFQ4cseM4v z(6b!3s+jvPvct3iXZ&NB&kxBC3p>^6a<;{$^2dIzt=(-wN?vpRhTce~>(0=yio=!C z>4y5}_?=Jtsb4SN{UfXw*vr<&Uw!52p>Aw)<;JvNB!V&-`zr8`M~PfQ>WFCq4|u>2 z@o@X+F8=EPwx`%%}QU*>>th7P-sTlNq*V`4BW|fiU}=%zMG4530CX;yiU2A7NR*A7sN1_P^?nsV|4HK7{>e#}`jwS=j#H z6j>l|G5@g1dhib;54iD0@4qcDKK4h$b}Vlge=o)d_$;>PIjn8N`nufT4coppb^Lv! z*|y5p{rk#UX7~f^&BQb1<0BgnSMcu@!C_yO;phN$Uo54gO?7V}c#?cyb=?Jb_mg~6M~H9WoAa}c z;hS|2nnAh##>xf%k+x$xK{kV2Qucn6PhV9pWv8A3-(>p;bM6p2HT@NMB&}&fD_6N~ zQLQ~Lo`P@dOA9iXw17v_Dy!=5aDCCaClcR43vlei_$KR}zx=BM+&1Ormd$F%GRjb& z=PKp+NyZG=Bh1*Sm|rk|pegzL-n?Dbx^c(jFP~XR+n5Lt<=~OqeX+5>in~R|KgLJ%>o%`x|X&+_MAFA})i?ahiSntK+F#81X zN*?y!{F1N%Os-C>;bZKK5|IOT{n7$x$vG~sV1U)J9TOG%o;*K{2Uo)4J%^cp! z9Gz|qZ^hcjhVp)PV*Bj|)7ISOy>r|}8D2X?ne@kEbL*flWs~^=>kas%-bOz@;<&nO zD&EZ(GE-sx<%<_dni8iOw~Ni&F-_o;G?R0C&#X~2_2B@f$M&gGp9}XSnRFX4-jAkp z%Wzg7n|A4kTtTkWDtln+osJi9xf$;^g#W&WeR+AU;$Jhs{Bs`?hW`zr``8wT4ec*& zc7;y zSQtG(K8FpJH-Go#lRnix>+GqbLe>H28ew!$RzkjFyt8kcyxn_K-Cgy{M!W#v@y7nK zvAo%6r)&3ZYy|Bo+?ch4?0Ei=GPLLazxoH{lm0RD=WmQtHkhKLXqHx;SO=VEZLAFt zTdLKZW8&zv*mG>gmwM-1a-25U<(6nJgW6I*7tY&EX2P-EO?k&u-M(`_ZFkq{Sl&(T zb0+QQ$6b~g>)&$8idXwEF_{K9_#o#s8~PRy@JL(Nzi?8+r~vx8p0->pn{2~^Bw!?M z?!(z>pl3O@XFa1{0j;&SoKT;m9ff#Ov^z-)h=iNI$-c6j`teM=XJ@$v2cHQO?|@U% zUHR~*DY^sOp_@r#0ETa%3pnN%-(QJ7qInry9BXgDQ33Dcht< zUiw0aZ2pIOlFp>xH2l;?2agFd9ri`}-i6s;ZzvDSom z0Xw)pf_-x2BlsqLq;^@7Zei=`miL&*eQfZP`eo5Htvh9wr5Q@wL&CA$fnVC={imK7 zIDmCQZj()<^Vr*f@3I|)@kP{;xNrOG4o@^OZv6(qY`Ur{vfn4iehA!d|JzA71+=P0 zqUO@!OJ-YW5-_cq_V%238^VmE>fDdY^0XeL%&Z6SAbFiV_MhLT`&4wh(Y)?UT(zL( zf+^vGoynl=svB}|$dB3JeN{By3a$D;|lEDZ6Mvc8zI zGsf4sNN#TPqNf2rPs=4@T_ zVz&PVc6ma_kl@*bE6Wz{2QX#Q>w(=-y?as9GUe*0_t>?zMN`o1pE5+f})Jv(p4LC~0Ho%0j z1Ir5;q`cF2IE;HFbYj_br@79hE1M~sh(rKapbzyLi^dcb zJ2%$#|padf!>c)Vwtqj z*7|YGYt0dQN?08p6KWoO=GDq@3VgbD`yIy8VbVMjJlGt}@l> zDgh(gf3ZH4^@%!?&bc+$9cRWtrb7XMp5%!{00TOl8`on@$^DJcudnX{T#5M?<_k0= zU)yZ9`o#x=`?py@r(r>FchB?eYW-~5Tctakg6Issk0I!s1xc+M5TN^_?M61ZAk-pDMs#O0E5c$*ts^DB3jtpbyhOYq9y9PC?=_dlS~o zXKRr#IwuVo7Csw6V~J-OkN0G(!KUzpx}NTvprX- z`vHl#qa?GP08fLy@1kHN3^+Ls23)-ZCtOVF#)PAo;O@%?*&tk-ygzQI?V~AuzY^;g z|25@hABJB#DpCFN#2;SWYQ_b}hiB2nd=Zq(wn%rhs4+jw2YejgpsRA_OPT_B2#Sfw z>A4Q`1v-+iX+uZOxA3Jlm!cee0gv(thc71_EBS+HOMaO)a7o&$N8j_Th4$P8?SeY?os+xp zTm&WsT?Y6`_23M&}19c|3m2)s0D8aiuiqv$T?VWgh4Y=}dV2>UVf9b`w0w@e>` zu22U*o+J z`MpWAg05=jStFAt_ASVwrPgv+rgOjY_tSc~(o7jGMp=N7@(+yC?KItax*KkpnIbB` zFk@W<#$>H@FI8(-8*O1%JFLN<8Pmhel{C`1|6~_Mu@G}O^hWyfFtDr}$SHL*Y3`2? zZ*+a4jhDhB_zr#W9AlXNcp&E`dekId9?^awa}d=QM*s&t zI7WxfnL*EN+rTAx*yEOA6<*tJ(@Q3~+36W=kqF8p-3k?hbk6`C1jQ2>8(D7Pku>MM zGkLcHnmT`0PcEWN(yTJE$;gJWn;I=6HhpuFd-oxzdNiSKHO(Y_KE1-p=a1J8GX_+6gT~SBU`uM%E^*wF+Sxy>6rkqVkpcm8!puVIx<*J+C5A1^-d8a$mLVyEZ&f6&`SAn}! z?%3ez$OQ#&GG^;sG~Bvasr9$&5INVTS`UCuq%LPYw&Fl{{$1M631Di`(a9wM3w@$K z`vSg6e7$dj~4i5Iq*%&@x!k_7&gGD zkF^l4%YH8RLI?FlCf=W3f6B?D7_YL?3~$nRW>+`vwj1^RE!Evel0STt{GIXCrL%_G z@DL#52;y`3I}19vgJk(l{2ezw{u_E@N$bz?C47UA@;IV;Sth$A*2z~wDIYP%s^GhmR@SI=s=jy$HU8KOQsutN8=}bax@WX=86E3P$zUb{Mt+_?L3- zu*ZijgA3%uA*HCgYZ+ybmD_dSWu0mtWMEs)xz!YUzy!r(;FR>Yj(+RU!wTr zC@4e8P#8N1w#LO6h=R99RecpYt2% zDUV`n3mdAto3A*0!3fq}vNP*!PI8@yb;f;6rLo3c`QVN-*~hspoA&7#I$}@PN9hcA zBAbq;OhV+;rD#=O*7I^HT6eAesAe?FLzCtUan$5$Phs{3o3=f-^ZB~1 zg}yC%2#iJlsdl(NI{8<|t(}hVcL`1K=p4uHL!R!w0Ws{GqPrVEIAZ6`S$A~SZ*H#R z06gZ=f-cr;er{Vw>{NML9M5JVD0uB%-hrq3X(Jw(IYotf%HHeyYhQodea|eaH^)Hm!E#_7_sY???B^>U(@D*MKr&|#^)!R8t1pmm&}u`KwKGMqBLW--kH z>Ne=ibK)C#=RPk?Ttpp-_a~Qp_n!U|uiCW{iPTJpsJA%TAB*K-rOp+Z{g%v?puh zQ_}|xGG%e*5m*M;klU$gQ{+uWh2{?=7?3kL|qBK12@O`^*t+zc2a7r4VZHE;-BWo78AgWfzs4Mi09M-C+(#p0GWZF5ClBySo{*!vZK5F0k3>)=b!o;)vp=tj#l-A@&W+9a zfiQgl=t>^{IHJ#C?);sZqYxnS0?B6`Z7%Mi%{xe8SXkJ|jO$G*KXE!!0UZQe)0OMc zJFLBYC8)2I@^YJ!c?ZO$ZXzvl8O4idnC33fhL_6JrkjJahZWai=$vI2r8+;Vl^O^A<+4x>a7b4s~ zAX^2#-Mpd5xhrub$`A@EN+Z&zJUW%CoOl7!Pb|eBA@Ug*tYnm2M9L=L01jPm!v)OM!SIT&UEK=8gacyl|edvAn=0>ywW$@8Co7K7ahr?(_7f zpH##*@CrDthi|gp^*d!px%2!OBRajnOj_$f+4{=5Dc5R1U-Ej}fOFPu$+21WGwwu# zH$rOGgGj(ipV`nggBsPptIMbrD`p?hdIden>!sg+{+|u*nnrU6J-^l6w4(ZZ;ECrV zH-+znU0wEYB&;RoDQYD>4$@bzq+8>fi_zv|G(hj`*G!v;@Lq*pl1E763S%XQsd1}Diok$ zjuUBooy=>$w8B(lYMJI6mI#@Gaw>_N`3Fyu|IV-XZy)UAUtLZasaf(rdx3u*Ysh&- zdG{jY2E4@GcJ9dskMiLzczVZ2qZ&u*X+}{!-bKiuqwl^kaSJmZo3wU$;^Y-0U0Mru za;P^i9oUWKGQtHIX^%eo`Fcge^qVky`0tqU_0BA>YHQ@O{FsQ%f4~FR!FYG(@*R#F z?&CYgi+qu7W>?pNXYNmn-Q!xW-oMn?P5P=WgxF(JYz_iA>h)S5wM%Cro5+EdoIhZH z0Pj-PWxsElHr_9*yCCl_ZBiwcoBPo)aqM5Mo5pw|<}xb+Pmr#Q>CvX2F`rG^VL}-F z(9AOHhyCK4pq~wqIraB-!~E^m^?wZSB7i{~;PG8CJ8z2I9DY8ez|RYplVER|SF%09 zr}XzXX1~@w)^AT|v(S8XJ{ut%aFCCEy)GF~CDf%y&IO|c1Da-^tj^cP6Wz8x3v%?s zeGX2VBk$8R`@q2(1}kRNUE)z*uYrTMT-O2K!5-euM6!bpD#j9Y)s_-Js6F(LC)#>C zTI0!Fw2g#|w>gS3akDO=69+_{=X~(1=xts<%kf=zw;%uUlB=OVOfG@9LGFGmzaoS5 z*yzGTC{Lk*a;O;F>N;pK-49IDLWl=Sm{a<&B=H%<$6$>+n$IU_17~Zzht65j6o*nn z5HC$&x1f&ZCI~jP9{^{e@4GXMCSJA`^e=67;Wam`3Fy<*hIi!tY%k7H|Ic3j-1c8# zFSL#Rw2nb8>Fif@Hd;FVDBHNrwtfHmTYCq@G-cbxm}bwU)%@@O*_Z|{svZA(+azop z=ALu2&EC*>>@{2X`>H~_Qr^LK9diuSrP|#7z#`n1@l9Ju-xcus78g9~&o>!gJB>thos2LHzV-(y z_uj_GANd7c?USPkEH}S*r!*rG)Is0PbIoD$*hX4wSt|NfenbI&C-XAsIZRFp9;9Bn zc6s>aK|VS30UMNoUmg>i=TYT2lJ$-{69394w~P$);a3erJ#d0@(BOQxGN*^Lz!4;4 zh0eef36x77paYuUC4H#`&xJAd6z2Q{_~IYNuD6X}+BDH82m24*luS1|(G1QsEGJ>C zEM?jhW7Se;iy-@k?b~b!C~ookaD?BrRry<#hd7nN@2IwlVYWYB!VgU`*SD zJsW27y=2}QKeh9nO&mpw)Q)-nC45)Vgj!DIuR(JJCeNQ<-1Mt4R-dBbH~G{zNyooG zV9NR492bvExD71LU_NMl`K@#^2YAGW_zcG#Q@7i$`!MRy-6XHGGPpDAo5JscTNhpJ z&qGiL05qpiM+F`!Di>OvsxYzb1F0`nWH-)QYo68 zbfOg9Aw@8V$4z8!#K|f>{^zb#`JJ3U28^>W+nM`wm)QdQq%iB}v|gu~5_A0K97lOu zZ@$>v{QwsS<|FC3R|h(%-4xR$xSy?~y{@i#(KzV`I&$0uJkDKx?yc2; zG38S0UE*TvAeefrUEpkF3Ah?iQd_mWv4**(GSJ1-N>Q0s65$nE1SNwBKfr?r7cMyX zG-2zs?}#1iS}(bqpMO{8ytNvx0M}pJ+uJ1!-#eoyQjQnvqyXyZd9!y;T+6WR`|x8& z3^h-*+tlao^2cMhlU4Z@VBq_nv%qOZ+G;x$j72Ks^!mLl5q5OoFqqtQC_eYh0DZ*17 z%0uuq1aRxoo=4ybgVuIjPfybht$Jr;ER~3+d(_-vP@5WDScm2#yD=G}%oYScq!2!t z)+ftc{hRIDfajgiR(OLS%+AnmN(HXkrHQ{rx3bn$Mj!D7&$2zTFT$s!FQT1^F^A`i z@IoSoa`+&x$5&zM(&a?kz3xw6T+Zzeep;vAgA$ir5y2pz9H90Yk!5k?g_zU?Y^ili z&>5Ub#an4A9Q-&|E|`-v$2)_e#r1%1^zbV0ajwb^l%+b2+3vPXsXMA8tJ9CMuGl8f zv9!sT?K4J>b2}7e2&bRUWN{{aK_W+k&Rha;unn&dl$hXS~0)(VD)8@;b3`;sUk#fDf?cto_@l#F*RwQjR6O_c^x5) zjfiGr=YG06M;F_sT4QO;J#%OawGIqjq@k&2#k5fsO~KPO@63wwV-u!E!?=h{S{$z-r_|A9PhD}L7;`Bd zIS%YjH~u)bq|_>{JkUe7|#E!HEHA1XJ~-4SQ*y15c8{E^2yrSgIr zn)}Jq@t9;ZrAP+(MitINXK0_c!&DGVHO3lh45xtrW5&jQv>%^dlA}v?yb)LsTwQKS zvb$SC1@{y6iXP^DC|Jcky3)6~Z9|6+`>J0@t}50Xl}>&6wt8ip-i{<#=#=MG(?+|(6)EW3yrZwwInxU6 zFbMeyrKy$+Uwr zcL$PcE(*K_`kFh1rbkd_PGI7dxt9qWCeE96O#HRf)_5dh!b-n}zHmoDrYD;#1y|QW zSFq`RNo_he;my88z_-C`OnB+b;G-V+^|Gkyq)d1de|1M9-P)t4kxf_=H}^{>oY_D1 z)AMVXFH?va9(fS#Zt@`K=u5%Udxn_L2W~ZSi%?_MF>OfJ?l*D1rgg0eqaNt=26W;VahG*_e!B;B$ z{&sHi-H+b;pnjaT!xpvy^btY+Kzmn>8lG-!bsF3n`n^%+o#AEs<-P%N9`?r%V-Ji^ z#6C*ZE9kyK4}DNB``a+{4AtDN=%JC4H)w1GUd*u(=xOdEk;@z&@4fVwW9ZCAIz_j} z;{oY@-^-5$PpGT z#cEyCZ1;>CbtLZRpGZA5m~#){o{^-(`t*`9z0VX?@2k9uthQJ}| z?KtJ2Rl{|7)04VHGo8Q#Dq5%~Ub`U=nZo9wVdg7pz5PCJUKPZ@*$?v^1^74hsch+v zxOKUdU&SarF4sM13D{C^#j^dNAF0o==T82|c)LEasa|h3qe}m$Tdg_RQrgO<+utwy z;JC&D-Rd^IdHKDE7PPw(`6uH?J!!kk?mgq#%?t2n6SQcT)9rgY&6Tu8(aza!h?&ao zO3(w0%r`+3GQ+=_KJ#Sy(~>7%OV!akOU<<1F^)wSs&LOQ^1YCk>C1b&&D>%$;uk-g zj)6-<=$PJkK3Z*+Ic9*xm$nVDGN^l zZPZ~M;hV%Ubgy?FtpX114v`Z-IP(zc4`J^ZM?Hx*S-J4IYTzX&$dZ{>6x*waiGj($9 zqEp`+o=DSDVxn11!(lwrBG3<;Q33I6jyUdV@BTnvD$! z?NRm0yva`({gicQoGiy4!%Uu9*8KI5EsF5eqNlH1d$AvHEI%sF<9%rNrM_UB*V3Iv zn98Qc?&d#XY+m)+Uf1@+?O8IhGE|uZL%#4dRUT*!tlD7l(i@qq4 zrqI`-oRN89zv+|Iq0^)Te<xNkRTZd8OVBgO1b5_ee`jlk=V?#K~xynt$QTrqHdg9}Ush z+VUiu4=yhsv|@5m{#%&A{V^?X@*k#c)YHW%dV8RA2?XgO$UoPY_bHhj{wuIwc6Gmq zHhe3#eSH6?^?vFf9=_*WKE2awOu8e1D}*r&lAgnQXJ0MWH`r|b$kT2@0Qx`79nHJxlZVIo>CmRoEN5;xbT=#$>M|~Tlei9? z{o0!C{J5M15L#N8$R9+D?}-#)oSDYScIvuad%OBdxJAnMXG)l#CmdR|R zhyCG{M*ldC=x}Q%1#nvr+a5NgFHTLH-RsY7!AJ42L$N#PZ8Cr7(63JM=N3e36OSFD z&sV04MbLqF{ z2g<(*^W{UPSFQ2q%jC~)=P9!uf85xVd7G^rto@(;hWK-0$zyO%%x!N&Rz^2UAN@U z9HCE@^So>i)Rleaz9(cK^T#Dk*CQH<>_2ARQF2Z^tQ}BK(!2fY9ozhQvO`bZ)T_r7 z;C(~rNtuyj9ed=K@A&EQ8YR(ci6)VT%v<@edaoJ0vaaw=^8VDjzv%S$7c_G{HAWM; zi|Dk0-g>uN=S#{y5*NNnT(7@!$xr+_CUDV^R-=hr)N(^q>Yy*BM8E;t2IaAUmo&bd zy5J3eTO%4eccAIRxzTjABawCJ&Am1<6!d_Db%1XY=jX3Xn={epr!LM+*17g;KROs+ zyJDLGCO&+V__o~Qx`X|Bl)|Up4-SLliC;bcrQe4MhjZz(bBrW%{Df+?M_0;+Z<3!6 z{yFXKKpRt7YF|X>P#tBBIPj@fX`3*44(do+Er(rtsK3n<->OfT-;<}YMo$7a)<+^= zY&+mM(Bu9w%o&5AHAB1+idc%Dn%APeANkg0F~mCl!N1J8sTu{R4!ZgF&B%Q)>^I<- z@u3|yKKQ2v{{9cLV%tX~l6hvf?>-lNlQ^EfXYaoa4e~wMg#g!G+WM;LZ}=v0JbB+q zd-&~}>_VKKj%$e9=^|pf)PQqp)_09F2VU%aRh8Nt%MiMq%6mlB9G(uVXXmP`p{11m9H3Zk3*N%MHDdDj`(sWR z$X z?R>)F7aZ)5zx3QHj|WBkWw}_FWk>OCCH^rt>(jK`4_D_7_S;R+Q?pH;AGx$JW#lwG z#FH=I9`X*y9UH9|tIs*p@54lECY=VP{o>SB<=m^4X6Tz1F|lLHS>@cqh%)?a{iP{u z>z%)Sc59#S5Pju1YCqFr+F940Owe4L>Ye>S>I;wY&sTyj z#YUQgb2js3c?Qmuw{ipK-%dK>q#OPD2hk)6+cG&?81lM%ZhIj*E{Eab^11bA0oc&P6Qt@dBit~chc~;hY#}Or5!Ic zr#wCN1?uy7rBrUN`WWweW7R%_<_3XR^6=U%`KbZ25f4tH70|)l`S|!g;EnrHwTG21 zTmW9QOLHBjRR2?KT^4dx(@-LrO}K}o%v^@;2EL?hi}(2N!vfkq$!6;{A`$eHe$`Ap-@Y-7gtNP(4~Fx=MGVGXV`|0ruqXFnbgzdf)tEpl zDBne0N$YRt+!(D>w46PFRJh+&`>H#M*k6k2p@LpJ=*;LI3^e8ZH}ufAC+m&{%`!!| z;1vRYe-9j$o%A3;iMK);|9-*QX1@(Nq^!UAXs2xkI{7h_>h{qDbmzK5Pmnq_?-P{m zpM48;B>vqejhr?#2tVdd3Gmfdo>t>lwp*Ezd!XEu0pBEzYvb?lJ6O8qp+<=CS`V;1~i8bqG^Y)%EJ_3ACJlU(G)}yw zE?LJiIC~k(s=7PCJK$Kq_$KR3Kjo{(>bag}ucEmJD9CQ+Ud6W_{Q6v1rwZ>u$$Tdz zbJ?i77n1v|V)#{DppZh<)DJpex%&BLQy(V%?D#!akIcD`mvipcOh=S*v23ypW5)Uy zPHKRDn2%y>BZ`j_dv#gIz$84RQbC&Yf-@%KXKniLOiQ6}9tNe&h$&tG%#jic1MnzrIm)NIK zN7`qryMBJrNRyUY4gP<{U2ALK%$1kKpK^oC1^q$jG;U{=gyqz znX|Jyvu)A+u|Ia_-t*n_zK=O~b|?(5#9-_o8Bt75fpS@TwJdv`?E&jx;5*b}JJOO^ zlF?Uq_i%Z%Bv2U%=*PhAbl+*6kH#)_^Wx>tRu2?ohsyuKP!u-Ys(~OZdS@PZ`Ar!c z@HfuGjdyl7<%@n)h6i&uT~fx(92))I@RsF{bw?keVuE$OO!~T1xNy^7p$A*!&jX=R zW^D!V=36Tt3~=<&~ha~da#^M((QG~lWVhn=%~{~yP_@8r_kTQ|ZJS?PCF2>`IqZhR)!>ucvU=jb zTfU`Y$<1ns*gtMNbl#MAK{ zU(tdDw9Nc5hL*^oF;{e=b>;F$BV`^1vasZjM2B?7$(JiWv)MlS1NxvI$@&Br$Ev+6H_JVBjd&wmbF!n%Au_DDZVyD{bM^mh7t8gH}qls8xc>< zlm0q=syF&6e4g6lZ@hi4=9bCn)ZSQk+;QxBkmo~A>sr(|w&Q(o8R?nssamw0-p%gX z8aT)Mb^C5Mw&VSN8x8-SMfZ@CPUs^Wv*8fC`oN)T>r!~5TMblB2< zw*Bk#A#Z`E*8NET+|a-_oW|SFoSE9+mIk<=PT&kngd zP||_pc}gVua5{y~k0UN*4o*l%4XIOTJvZjV%9t_X-k2Jcy&l-zWn`ysQWy50-&~g` z#|3BBBCtgQD_!a&0qHg6VTp7OeRZ!2KUNrEW$co(f%BJ|uID>oniUKvUq;wy-qBgu zRPO6A|FqjH<#W_{Hr{j1y0ryv#s@n*G(Iy{!rC{#OL@`Et7#OI?i@7$Z_(G=OYUat94Tx7mX8tKMa_dX59jA3PbNG{k7*9=F5k?FejPj2-gY?LsIMGA?c1utj zl*W$zay$orxZG^UIb-Y;8Vcb?`?=H0yK@{)zQfMm?PQ4kavX!_PFpj8@dZxkLXmoP zNLNLuhqE@_L>y$}X{B(qR_#CPy`a9;cy0{0*tyCr+SZZn-reW^U+JjgVOyIX=TFAe zV?Zn%yYJ@WDP|nU9MT#Z)FI;o_k}&l@X=zsE^;L7pJU4YWcyUEXEvDB-ZH_v(SQa^ePO;(P@&Qw-VyDxLZ)lBau#{ywVq~tb!0!!rYsh$Jptd)cqI7 zwPtdiQ>T%bLnVJoHfWIvBZ#`k=_y=FH%p+JT6uui*ylP*{FHuoM%~9Ndy_3d4=}|v zD!ysjvbEin9+~oTp&QtS>xO6ayf@4^#rrpM!9#(xNHV&bq3&;?<_UvG8`X_LdqgpY$SgvpRv;(tH}}aC!cJHD26Xi zez_w&xX>D0XXe@l4=i|q7yq}M-Y>Xv+w{Q(M&a|cC%#$OEg8MG*t-im!~VH$5A~dy zkS*lM(m@kXGPW#f{FL7k$RHZV`67CM9@6{5VbRo(b2mMMQ|1wN)pAK zIHmKZqFcJw~se*wwUiRSzAx#j zch5cBJ@?%0+*40n{=pAN+|x&Y&z}E6+uIwx;T`tkx21nk{+(MI512GJk;sl@3Uiam zoR`l>b|5hPlMi55{oo_(n;mRdDqqkrfCt;`Ck9Vwj%WTKBA5kR9nwd!aBpeet)K z*DlQEvbk`+d;Y#@eaFJ}!`WQHOD@bLa-+laVTm{DXlwf5Z|e~oH#Kt|0tHME+%1vU zdnPg53U3Hl!(gAi?9BI1W7t(0PjV7g=a>6@ucKw*`a&)~eW|{ zLlcI=qZ8h&Jcl~>NXC_rh#Y<2(5I~TP z?|$m8@rKUK@JPY7LzrD2dEQlDtLw}Z4mxyBs7!AjvgL$Yf;sr0aJgT;^Ynhe9C~m# z%z~{OPXx@IL&9Mm`f|rP6U*4cOEYT+qg38q$mYDB(P8+4FxmE9y2sItUD^KbLe7g1 zMIsYJbhUN!>MQFBt|#7~M?2{U_WJz%Wd9U`S?%R|v-wnEw98xTQHKwMzp`t1R^9U6 zPkD(#C`{_smmZw3d@!{9ud})~)Gr@~FEe~;THx!>UAErT-0dYsa-Q)nVRm-rPrmhq zsD&eSq%!>^WEkw;nfF~^=b*x1?)&dcS2cEzW)jXJhqb}GjWrW)IQyx(Rl`Y)24S?P-hTbgb*rfb$a%3G_{$GypS9=fKcCr~ z&L#%)vmNQ4Ts)JHn-1BT*Fo1&M{s*={Y|v`C*=x%)jSA#=kV4FL=4cKs=QxJ5DS=cTF$X zSw2!&n(se$#2fLtQ>WN5LF$X?%-b;Wx6>U@s!nMCNlpFzH>SEe-2?Glavm!!TwmR> zU%$n?73-|XWwU*uedc4EKJjVRm#O2%+aP`2_RnoMwlYuL6a*O7eiyznxS`#_3nK}Y zd6#8(FK7NGJPkLY{6|l^;Gw2u2F7USFkwFH8;fV|+6cIXnL@72nAP!UKUld?pImm0muZUh5^DE`oIT^;18-r8Siy8_UGgvrS`mGJKHD>^ZUaj%hx; z&<52C(|7*wpT2%_I#uwxz1&(aHyehmaf3JX$A$knb(b9lY$=8CHvL@9!~f7Ql<%MI z=1Mh$FGx4ft=ZH(h2a)>ZjKX=j}tGP^z{~tmz=~MxF!{P>tp0Yh zg*78sSU8&5Ry#w$ebIKt79z8tg*=E$K(8iyf@hjD6ActJ>N@XEgaU~4I6G;V&JgoFC)|Di=SB71boXzhI(n@qiI5Sp!>`@ zb-OnLt~;GdlEF$o?8;+tw~0dU8;&~rHq5nIuH5{(58xf@Qn7h}-|VZn`cN6Z zOWX~v4E+%gaA5As`d|OZ-7n%D)~1>kaM@Z`7PB|w@X5Uyyq7c^b8h3t5giXr;F2^y zeDRXs8{gvS@*WTNeLxew89J;ADsjxhzoK^#ITO3&4+?UIq;$>n3@%%s# z4%R>DIy9zzKlZItu9)Ur+1i6=mvG5EqOXC6Mhh29VKPJy_{6$b@>!erPR?YDH zc^D6CFPZ(I&%+`AJ3Ku0)NyNeb9xv^^$(=8@#O5a2bM7NUg=}`iJ!f2EH4KI@&esf z(>7tc|I}~Z>}$omxTNJH5temZ@&D$`7ar)~F)f4!=B3rh0GM+2h4R*s3>IHrGS(YU z$1`x5<_WUZx`f@21OUo0R#&ps_1QhwwAr~;i8QP)?#~)G=6a0V%HcM99c|3n45RcP z%A1p3K9NhANh;ohzaW1Kx?B(bwBeFlJ~?$HlcL=;FR2=pdE_>Ir#|EI?{1#oWH%9( zWwXN(>`TL!fe*Gbypw#bSa-vuMq6iTo;GN>4*QmdAv9Hmm_>(D;M3Z6!K?xZ{9tHd|aP zutLyu>TH?MJJu4FaNcp)guk}fdMmB_8q2@Ni}@?-Fia=r93fsjN5FSEMn+L42Yg~Q z%tld{<$#TvvxnZ|*#o{y+_menxnza0H>fWFkECNf8g^N8uU)`WzIsJ@4;ulvk%mML&^!iFxl+Un7yx9 zCO%}`o7OwvW{l*$Sb{o7FHHyYvA%3BmhlRtSZw0aqx%NxL}{OhcAlwPV!$8{fMa_B z-v$x=2fdlI8!MAp^WnQ}Q!N+f)7l16N8&W+JZ|2D&k|p)(>LHD3-*dNKGc!;%$ZQq zo(bV|Vz)i+WqI&j!kV)o&z=q8GibA~(sMOA$AKQ}oiAhFWE(O);FR>tIg=NB&V({4 zdySe42FZ>(62CcbJ%OwdIl~@%baU5EqAVkcaq-3 zVUJENES(^nfR{MU`4{6<=U*t3xU10*>PYZ9=~ie|wq^3Sx!gk%wEkpg#6&C2WK7tNzl)(7^tHglWptnNVz`kg{&q zz-WEbSZv}6V)@efot&pZwgz5fI>mw~X;Z9w=+uE2KWolZ`-*3(_%8hzzDxMUCp`MX zbnI*k$1QUoj*WIeOoa5YlW6rGPsfJi z`8;MM#+3jrXpRHC(%0O2#9Kd(GVMk2L5!SC8*t3Cu4m#z===_GBKQnmSqF^&@!j8g z_&1mT@G*>Sx-86h0~goF92x$-mgpeX1fM|%<;)k}Nq7@?Lhy(?!DkJRcM{&jpb$J_ zQ1Ds9})(poBW`mY0YMIdIT8855V|`RLmPeR^c>pKp z8O*z;{~v!k_zCJiE+%N<&~RbYrw>G7^-PQq)kBOBKFfBfDbP#HhT%78-X(o0(*Pb< z&CFX&j1ke$F-CYV=}o4?a2tdaOWP|g|c zb-Uu7gf+2B?)(>@0nhEJeI(Y!uwgu=J;@d^P4{928_-4X(Vo(7S!cj)D$;-O7wA*X z)XzBn3EwrZcqi+aSSVY^kA=cJT@UYMJ+xt6-bPR1v(z2FOW4LEUhQtdT-Y3DEXZca zP7yPuY2cl#cjxVcb?tt=&I}!7_fvdOp6l|M-9%>@L$syG2MN%ZY=7vrfj0Y!5Myn` z=-#dz3KCG`V%=+AP2AQMoc_eH|>*l`B5k4aF=2XDc%h7 zuuaQ#u3X20KfSh=YXqhbe3Jg-d)>8ovZK#d!!+67+qG43(FEPdHVMNGe^j^D#=+q; z&>zIk|M z9_ACg-~|eBMk!&?w`FXeTQZ7Bh0Y|(Wt-3*v{_dIr?DH+?%;I-4&7auY%}Hu_zVF3 z2l_I`&n$gKj33HC8|TNs9i_fTZY*Zt9Fid0twFyP<$C=*pKoFIxo^PkZL3({C1ZhyR4g#c&_{R-sKFkf zs|=dJ&$z(p4#C5Gf;QuZkCpMqSrvbbvTFTGO>rqO#)g1Z~Nu zjGZ=Y%qzxDqfGKy&i9rZ8<;lmN!l_7yGWa6r4)mWGSG%iL*7bbTP^iM6r^ckHWWX^AIfCo}@!l*y%LbmMY%=~{qFzXmlqmbZEFWTX zwS3@N%D3@L8;@?caY?u*A&ca!!|Oy&nUsYjF7lX zBDB!f{L{Iexj!P;;4-E)jLy7MW?W*<0DK*F4<+11FioSG;+JJ;4P>c?M(a(N-m+tf zEaDqok6{5v~N=Sh+Q@1#A| zL!a(w7W%dRx{^1Hk$k_+EZch4lSU#Dya#{&ylu$FRpUDz z@Tu3Zyte{9@}qSG9YVhh1Ahh>$*a^e!{M*S#WI0crUx5bdt%)NbAJi(JuBnuaDPj- zZ%uDD$%_|_6SQd_LT8<&duT!J47`%=mkzq-g(&V#fv$C}5vI+4Tl>~(@NQW%Yx7jb z>)$K01hxnMN19i@?Bid^H{!P+e*az^^y2d$iQv6#mvUoJ;#fZU9wl&_VoX&y2(L_l*he!%QyU8((Z36O_=ttg{6fvH-M=5P$O41&e z#%w$b5BGt7ayN!`u^chuyUNFD`1}!F$zRF+Obk*~hwQ{^a44~8Q10gmW_<5Y6bNxhYuB{2Z>T$u8&nr`oJ;QSKQ65sC^&;cNF`i6Y z^3}u?$@~0t7IDzbJFgq+X~vt2Tk1i)yK+N4u$(q2?2S}wQ@E?D^{RP$^|W2rwM*W5 zD5{P&s+c!uDoieMZ`MUoM2+R3L_?iJHumMKnqBs>As}V{_LR+&K(;p#6O9E|H3J83%-vX`)7YZ_-zu~XjiFFA* zIdW#+EAAEht%!Hhwkz$w7~kS*m$TtUh6%Cb@yG!lr5tZW_un$v+LM_8p_}4PjkJjd z>?6_5AlJ=|zV-P6!#EZLVQRObuNqxZ95Y3?#Nw{u(N=oyyh9mZutSLcxQ*pqBKUz< zcX;9u=fjL{oEwPzzkckTXK}~I{hXgP`dJYlHJy_6Gr{?k?b#;c?!Acs$9LE&(`vT! zD}-+Q=Ou{4X1@wMY8CSmACP#3Jq6AGWPP?3(3Sm2-1Rp@CBB2NTBifKSE_Bc*_^4- z9jrns4V*!CP4{tw=TPvIXi?wu& zuFAN>ZpbNRVBG?*r0?fD2#B3W8F)Z^A!xkYvGLmy>ka<4cWiy7yNIA!DgK`I4qZyU z`#B@r{2t$->n5k`O6^@~ty9hVpqBcExll$lVJ~`qxUF;ldUcOOFiI5+7@m~7H@tk4FGal%6I?n%7*^zt|_i>m{(3gDT-fA1i{@b$^vl_#D@Cp5Z zmX65`y4Q_55_d_S7PqRvJ1M`5so-f_;P_T&4#91!pxuJTVdQz?wq#jBOY&2a3&tI+ zS8GCRKT6#@T&gU>);FNUzyfSW~jPFvnVdFpJ1H8oN=buTJ=AXfPDU0^O z@|+EATCVRhZQzu&pL~Ar)zJe06k&Vk6lt{UE!9 z{HaGSK9Gg@$A*QHVLAL*%Z9%rOeYcI82QXSQQlr;h?^ zlZL&pz9#Fk4#B6XME_xZ^tUa0UDJkk9CG=ETuvV39V*db#^&llUESXpH|j{-7rwpU zpG^KGAGf`=Q_$2a~wn?~)x+%w>pX1EQhVRlAwxz#F{lK}wPG0=QoLCF3 zPr9=GZfhap-!0=-z%6XO3tavapOEcmUB4xF5L%o zyzPVgtXZ?S@z?p;gj=NFHDtPF)H}z^Gp%Z39(3&c`M>J??-_yg%yvzLV|-=%>)6}x zW%_c|8#nT;>a z5OR0>sf1V9YY`li$~=d)qp_hXyK$6t_57+x`V)iI@EJhBdS z?C62_UUU!V6(+nr#pDh7$avy5t7*?#7uQ)se8XPRPC7QE*4%HfEs1j6qtX3E$7bGF z^U5`%9i-V~FlYE7{NUrZVH=29meu@dhx4ah)-r6$Wy4X1G zqwD+5r~cvKEw$$St~eGzM{N7rhqs>5a(>kA!!7gHEiu_Im~Za$)!dHO0p0ybt;NkO zx^<=Ci$q~?G7vyNGxwm}pF%MtWXzLkc;yOGw zhvi{l`L%t{x#(w~YDUbbi^t?xO!fm|d0_R~ce%ALbW|zs8ZZ+J1^&l&YhQJ zzf7^BCb#*RwsUPtyWjrK_4aDJQyV9=wAs8>3)w=@7@X6K?WY=l6s7*7y7QR+%x$Oh zf8`(Ux`Nj1m_i#{7?c02f+9W;P3H#t=$-F>q{H%{ zGJ{1TIyYD)J+quTKiFN@p1bQ5nvl#bO@$nRZDU=*{&_rp#@Kp=JWznC=6*Tz4SuW_ zv+I&Ye|hBe$wj`qaYL5#bg>_(rr$Dtosa6i$vYpgv%)`bR7`n@8K%efPz^oEUCnt+ z7+=qyc+wYJh%fOIR}r0OvX0Iq`cgJc9{PLG{)Nsdx#I9Y>>MS$^G)hlx{yNh>0F1E z=4{Nf&Ug6gg!(^E@^P<-=jexh7&q^m_NcGy%6z1to}|M(@BE`)5fhNavQ=4XYf z${%v(0pB}(x~)ekL`pM!==>fNDW3@JGfdw)r^h3yhhAy&apxnPp9DBq4xMY{smliT z>acZ6nV~w&kF2k0&b)BrrR`MTENWbb$EJ_o$E(JUnMb{!ciWS5cicUIhVDB|liR-< zn&a&|nzuFSp-ZFvBb1rL2~W!69O=wQHSw;~-$87r&Y%AE3;r$BA$eKpa1#jdi;UC_iIZHVLUAv*Wc-Uy_&Hd?K;hlb2L>cPB>{=0CP5d3gbSnuh{nVe0@fI z)|HKxcQY4wr#z2mjp619#5Z$m`Ne#q62Hf8*iaL-{Q7f@cgl0MdJf|kbB{{=PW?n; zONZrm%zT9T=6+P|dXDZ}+wD&ZuNvF(R+pHwRO0Q1Jx{ou@3QL{>3Wyrh;0e^h%`PO zAN|?<#b2E`nC#;@HP3?@N|&8u=BcpdntZ$@KY?6pqP&R`)#Oaneepa!Xs)ly(Dt8X5*0WER<1e~f>>V7t&f~il++LODk47$k6=B;2_YJfj^}UPJ z@7nW>4s%z=(mgI!7;xKu{#_jAVZ8TEg6=@b`xw~2-n{6Ft<&&RrTmLj_ORR}K<0HL z)0g`>a$gg&*Seg^Y(vO%tYz2h%99?s;FRfOXxo$QanY{gZmFjI(-+oer|@q(<#}Eg ziSXS9@D90jOo`0%#kPbxkQ)zldc;k~zA%k(yL=;_oj7h6{4XAHvSV%+bO%89(_LeGr#bGH&JcXu z9DmOCR!yI)hQ5v&f8eRlT<`86*i2uJ1E$Zi>DUwLr?~Ak?k8?fPM&Jx(YfV)`Ls<3 z&86g-Fzl}y`Lu1%UG`r;Yoq)e^!r}8z%)clzeNZq*0IL2KkL|l7HSxi6MkTjqUX`N zwtZ#XJnKNm62~t1$&FDGnJ4 zL^qhDyctd-B!t+ivnT7K%r8E_fDgDr09^pE|Dd8=pD)@u^OqlIPWwCztu=*r00861F$r zXcKu>2-|>c`mC^>0#9PJmyE17F4WviT#apud@x~p`2A_eU1a(nypVGj+Fw@FC$!I; z(SPR+GoW9Accn+a&6_#K{8|{_f8f4R4Gn#_Vd`nvco7D2=m;^2yS5cehhz+eGHXWj$-$7yo+tnbR%qG5b|agZpgQeidw@ z9p{6T|E$`4wVID#)j$7Ihvj3;J{9x9ZL+Pjo<4BtoF>FLx{+ApAMZGP>CC53N1hV+ z+4*GfyidjaR>G(EoI1PDe46nspsVNGif`)6pZ}1FyCnX)Oz5%AF`?sPqb`nB{PcIn@rrJDGs$=85_vD)3Lc1JU07FHOS9Qs1R3yb1h(HcpYoedD_aY9d_lUR-kkr z+vP}Ry9?8G&{&}J7hl`^`u&aTXhXz?pG1($8@zElO5MrWCFrI>{8kEN!B{MRBZXy? H`N97O&t;uC diff --git a/tl/generate/scheme/update-tlo.sh b/tl/generate/scheme/update-tlo.sh deleted file mode 100755 index decc73b18d..0000000000 --- a/tl/generate/scheme/update-tlo.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -cd $(dirname $0) -tl-parser -e ton_api.tlo ton_api.tl -tl-parser -e tonlib_api.tlo tonlib_api.tl -tl-parser -e lite_api.tlo lite_api.tl diff --git a/tonlib/CMakeLists.txt b/tonlib/CMakeLists.txt index f281762232..a59bd9459b 100644 --- a/tonlib/CMakeLists.txt +++ b/tonlib/CMakeLists.txt @@ -1,9 +1,5 @@ option(TONLIBJSON_STATIC "Build tonlibjson as static library" OFF) -if (NOT OPENSSL_FOUND) - find_package(OpenSSL REQUIRED) -endif() - set(TONLIB_SOURCE tonlib/Client.cpp tonlib/Config.cpp diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 0afdf7389f..284b612173 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -1,7 +1,3 @@ -if (NOT OPENSSL_FOUND) - find_package(OpenSSL REQUIRED) -endif() - add_executable(generate-random-id generate-random-id.cpp ) target_link_libraries(generate-random-id tl_api ton_crypto keys adnl git) diff --git a/validator-engine/CMakeLists.txt b/validator-engine/CMakeLists.txt index 693becdc85..c8001a11c1 100644 --- a/validator-engine/CMakeLists.txt +++ b/validator-engine/CMakeLists.txt @@ -1,8 +1,3 @@ -if (NOT OPENSSL_FOUND) - find_package(OpenSSL REQUIRED) -endif() - - set(VALIDATOR_ENGINE_SOURCE validator-engine.hpp validator-engine.cpp diff --git a/validator-session/CMakeLists.txt b/validator-session/CMakeLists.txt index 6d4b00bd61..c57fad54ea 100644 --- a/validator-session/CMakeLists.txt +++ b/validator-session/CMakeLists.txt @@ -1,7 +1,3 @@ -if (NOT OPENSSL_FOUND) - find_package(OpenSSL REQUIRED) -endif() - set(VALIDATOR_SESSION_SOURCE candidate-serializer.cpp persistent-vector.cpp diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index e33010fe14..f0af651877 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -1,7 +1,3 @@ -if (NOT OPENSSL_FOUND) - find_package(OpenSSL REQUIRED) -endif() - #add_subdirectory(dummy0) add_subdirectory(impl) @@ -110,8 +106,6 @@ set(VALIDATOR_SOURCE manager-init.cpp manager.cpp - ${VALIDATOR_DB_SOURCE} - ${VALIDATOR_HEADERS} ) @@ -131,8 +125,6 @@ set(DISK_VALIDATOR_SOURCE manager-init.cpp manager-disk.cpp - - ${VALIDATOR_DB_SOURCE} ) set(HARDFORK_VALIDATOR_SOURCE @@ -151,8 +143,6 @@ set(HARDFORK_VALIDATOR_SOURCE manager-init.cpp manager-hardfork.cpp - - ${VALIDATOR_DB_SOURCE} ) set(FULL_NODE_SOURCE @@ -188,6 +178,10 @@ set(FULL_NODE_SOURCE net/get-next-key-blocks.cpp ) +add_library(validator-db OBJECT ${VALIDATOR_DB_SOURCE}) +target_include_directories(validator-db PUBLIC $) +target_link_libraries(validator-db PUBLIC tdactor adnl rldp tl_api dht tdfec overlay catchain validatorsession ton_db) + add_library(validator STATIC ${VALIDATOR_SOURCE}) add_library(validator-disk STATIC ${DISK_VALIDATOR_SOURCE}) add_library(validator-hardfork STATIC ${HARDFORK_VALIDATOR_SOURCE}) @@ -221,10 +215,10 @@ target_include_directories(full-node PUBLIC ${OPENSSL_INCLUDE_DIR} ) -target_link_libraries(validator PRIVATE tdactor adnl rldp tl_api dht tdfec overlay catchain validatorsession ton_db) +target_link_libraries(validator PRIVATE validator-db) -target_link_libraries(validator-disk PRIVATE tdactor adnl rldp tl_api dht tdfec overlay catchain validatorsession ton_db) +target_link_libraries(validator-disk PRIVATE validator-db) -target_link_libraries(validator-hardfork PRIVATE tdactor adnl rldp tl_api dht tdfec overlay catchain validatorsession ton_db) +target_link_libraries(validator-hardfork PRIVATE validator-db) target_link_libraries(full-node PRIVATE tdactor adnl rldp rldp2 tl_api dht tdfec overlay catchain validatorsession ton_db) diff --git a/validator/impl/CMakeLists.txt b/validator/impl/CMakeLists.txt index 9ab32c47be..37ec9d4254 100644 --- a/validator/impl/CMakeLists.txt +++ b/validator/impl/CMakeLists.txt @@ -1,7 +1,3 @@ -if (NOT OPENSSL_FOUND) - find_package(OpenSSL REQUIRED) -endif() - set(TON_VALIDATOR_SOURCE accept-block.cpp block.cpp From 0f6252a03fae0580b1feeec8dfb0440e4cf43247 Mon Sep 17 00:00:00 2001 From: Vasilii Lazarev <97127102+Lapo4kaKek@users.noreply.github.com> Date: Tue, 28 Oct 2025 12:15:07 +0300 Subject: [PATCH 422/524] fix(tonlib): correct error messages for gen_utime checks in header verification (mode&4) (#1857) Co-authored-by: EmelyanenkoK --- tonlib/tonlib/TonlibClient.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 87f285a36f..8845583ae8 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -5983,10 +5983,10 @@ td::Status check_lookup_block_proof(lite_api_ptr utime) { - return td::Status::Error("prev header end_lt > lt"); + return td::Status::Error("prev header gen_utime > utime"); } if (info.gen_utime < utime) { - return td::Status::Error("header end_lt < lt"); + return td::Status::Error("header gen_utime < utime"); } } } From 1d752023a01f927689f503b58137a7b446136311 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 28 Oct 2025 15:43:56 +0300 Subject: [PATCH 423/524] Increase timeout for ValidateBroadcast --- validator/manager.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index 9e7bcb62e6..fb72d90f7a 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -230,9 +230,10 @@ void ValidatorManagerImpl::new_block_broadcast(BlockBroadcast broadcast, td::Pro promise.set_result(std::move(R)); }; BlockIdExt block_id = broadcast.block_id; - td::actor::create_actor( - PSTRING() << "broadcast" << block_id.id.to_str(), std::move(broadcast), last_masterchain_block_handle_, - last_masterchain_state_, last_known_key_block_handle_, actor_id(this), td::Timestamp::in(2.0), std::move(promise)) + td::actor::create_actor(PSTRING() << "broadcast" << block_id.id.to_str(), std::move(broadcast), + last_masterchain_block_handle_, last_masterchain_state_, + last_known_key_block_handle_, actor_id(this), td::Timestamp::in(20.0), + std::move(promise)) .release(); } From e376747d1759a062b8e4a74d41ef8b1184e4d0e2 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 28 Oct 2025 16:19:30 +0300 Subject: [PATCH 424/524] Don't create more than one BlockArchiver for a block --- validator/db/archiver.cpp | 5 +++++ validator/db/rootdb.cpp | 28 ++++++++++++++++++++-------- validator/db/rootdb.hpp | 3 ++- validator/interfaces/db.h | 1 - validator/manager-disk.cpp | 2 +- validator/manager.cpp | 2 +- 6 files changed, 29 insertions(+), 12 deletions(-) diff --git a/validator/db/archiver.cpp b/validator/db/archiver.cpp index 7fff6d4471..ffbfc9f053 100644 --- a/validator/db/archiver.cpp +++ b/validator/db/archiver.cpp @@ -31,6 +31,11 @@ BlockArchiver::BlockArchiver(BlockHandle handle, td::actor::ActorIdid().to_str(); + if (handle_->moved_to_archive()) { + VLOG(VALIDATOR_DEBUG) << "already moved"; + finish_query(); + return; + } if (handle_->id().is_masterchain()) { td::actor::send_closure(db_, &Db::get_block_state, handle_, [SelfId = actor_id(this), archive = archive_](td::Result> R) { diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 907a82f50b..596f7417f9 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -388,12 +388,6 @@ void RootDb::try_get_static_file(FileHash file_hash, td::Promise promise) { - td::actor::create_actor(PSTRING() << "archiver" << handle->id().id.to_str(), std::move(handle), - archive_db_.get(), actor_id(this), std::move(promise)) - .release(); -} - void RootDb::get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::get_block_by_lt, account, lt, std::move(promise)); } @@ -464,8 +458,26 @@ void RootDb::start_up() { } void RootDb::archive(BlockHandle handle, td::Promise promise) { - td::actor::create_actor(PSTRING() << "archiver" << handle->id().id.to_str(), std::move(handle), - archive_db_.get(), actor_id(this), std::move(promise)) + auto [it, inserted] = archive_block_waiters_.emplace(handle->id(), std::vector>{}); + it->second.push_back(std::move(promise)); + if (!inserted) { + VLOG(VALIDATOR_DEBUG) << "archive block " << handle->id().id.to_str() << " : already in progress"; + return; + } + td::actor::create_actor( + PSTRING() << "archiver" << handle->id().id.to_str(), handle, archive_db_.get(), actor_id(this), + [this, SelfId = actor_id(this), block_id = handle->id()](td::Result R) { + td::actor::send_lambda(SelfId, [this, R = std::move(R), block_id]() { + auto it2 = archive_block_waiters_.find(block_id); + if (it2 == archive_block_waiters_.end()) { + return; + } + for (auto &promise : it2->second) { + promise.set_result(R.clone()); + } + archive_block_waiters_.erase(it2); + }); + }) .release(); } diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 14c30f6eed..a39b174226 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -97,7 +97,6 @@ class RootDb : public Db { void try_get_static_file(FileHash file_hash, td::Promise promise) override; - void apply_block(BlockHandle handle, td::Promise promise) override; void get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise promise) override; void get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise promise) override; void get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, @@ -157,6 +156,8 @@ class RootDb : public Db { td::actor::ActorOwn state_db_; td::actor::ActorOwn static_files_db_; td::actor::ActorOwn archive_db_; + + std::map>> archive_block_waiters_; }; } // namespace validator diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 834bbd7299..50ff678c02 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -88,7 +88,6 @@ class Db : public td::actor::Actor { virtual void store_block_handle(BlockHandle handle, td::Promise promise) = 0; virtual void get_block_handle(BlockIdExt id, td::Promise promise) = 0; - virtual void apply_block(BlockHandle handle, td::Promise promise) = 0; virtual void get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise promise) = 0; virtual void get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index eb06dd304a..0b519c2cdb 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -856,7 +856,7 @@ void ValidatorManagerImpl::new_block(BlockHandle handle, td::Ref sta std::move(promise)); } }); - td::actor::send_closure(db_, &Db::apply_block, handle, std::move(P)); + td::actor::send_closure(db_, &Db::archive, handle, std::move(P)); } } diff --git a/validator/manager.cpp b/validator/manager.cpp index fb72d90f7a..2d1f7faa86 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1666,7 +1666,7 @@ void ValidatorManagerImpl::new_block(BlockHandle handle, td::Ref sta std::move(promise)); } }); - td::actor::send_closure(db_, &Db::apply_block, handle, std::move(P)); + td::actor::send_closure(db_, &Db::archive, handle, std::move(P)); } } From 5d27d05f36819be3492a6d38cce297f81ca2263f Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 28 Oct 2025 16:50:58 +0300 Subject: [PATCH 425/524] Optimize reading files in BlockArchiver --- validator/db/archive-manager.cpp | 24 ++++++++++++------------ validator/db/archive-manager.hpp | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index d8afa1c0bf..36890eb2a9 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -262,10 +262,11 @@ void ArchiveManager::get_key_block_proof(FileReference ref_id, td::Promise promise) { - get_file_short_cont(std::move(ref_id), get_max_temp_file_desc_idx(), std::move(promise)); + get_temp_file_short_cont(std::move(ref_id), get_max_temp_file_desc_idx(), std::move(promise)); } -void ArchiveManager::get_file_short_cont(FileReference ref_id, PackageId idx, td::Promise promise) { +void ArchiveManager::get_temp_file_short_cont(FileReference ref_id, PackageId idx, + td::Promise promise) { auto f = get_temp_file_desc_by_idx(idx); if (!f) { promise.set_error(td::Status::Error(ErrorCode::notready, "file not in db")); @@ -276,7 +277,8 @@ void ArchiveManager::get_file_short_cont(FileReference ref_id, PackageId idx, td if (R.is_ok()) { promise.set_value(R.move_as_ok()); } else { - td::actor::send_closure(SelfId, &ArchiveManager::get_file_short_cont, std::move(ref_id), idx, std::move(promise)); + td::actor::send_closure(SelfId, &ArchiveManager::get_temp_file_short_cont, std::move(ref_id), idx, + std::move(promise)); } }); td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_file, nullptr, std::move(ref_id), std::move(P)); @@ -294,20 +296,18 @@ void ArchiveManager::get_file(ConstBlockHandle handle, FileReference ref_id, td: if (handle->handle_moved_to_archive()) { auto f = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()), 0, 0, 0, false); if (f) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), ref_id, idx = get_max_temp_file_desc_idx(), - promise = std::move(promise)](td::Result R) mutable { + promise = [=, promise = std::move(promise), + file_actor = f->file_actor_id()](td::Result R) mutable { if (R.is_ok()) { promise.set_value(R.move_as_ok()); - } else { - td::actor::send_closure(SelfId, &ArchiveManager::get_file_short_cont, ref_id, idx, std::move(promise)); + return; } - }); - td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_file, std::move(handle), std::move(ref_id), - std::move(P)); - return; + td::actor::send_closure(file_actor, &ArchiveSlice::get_file, std::move(handle), std::move(ref_id), + std::move(promise)); + }; } } - get_file_short_cont(std::move(ref_id), get_max_temp_file_desc_idx(), std::move(promise)); + get_temp_file_short(std::move(ref_id), std::move(promise)); } void ArchiveManager::register_perm_state(FileReferenceShort id) { diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index 0d0af4095e..947923f4b6 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -204,7 +204,7 @@ class ArchiveManager : public td::actor::Actor { void deleted_package(PackageId seqno, td::Promise promise); void get_handle_cont(BlockIdExt block_id, PackageId id, td::Promise promise); void get_handle_finish(BlockHandle handle, td::Promise promise); - void get_file_short_cont(FileReference ref_id, PackageId idx, td::Promise promise); + void get_temp_file_short_cont(FileReference ref_id, PackageId idx, td::Promise promise); const FileDescription *get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt, bool force); From a3f948eacf34c997c06c0301f6dd7c1d9656f547 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 28 Oct 2025 17:54:46 +0300 Subject: [PATCH 426/524] Better log for "file not in db" --- validator/db/archive-manager.cpp | 2 +- validator/db/archive-slice.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 36890eb2a9..d3731b92eb 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -269,7 +269,7 @@ void ArchiveManager::get_temp_file_short_cont(FileReference ref_id, PackageId id td::Promise promise) { auto f = get_temp_file_desc_by_idx(idx); if (!f) { - promise.set_error(td::Status::Error(ErrorCode::notready, "file not in db")); + promise.set_error(td::Status::Error(ErrorCode::notready, PSTRING() << "file not in db: " << ref_id.filename())); return; } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), ref_id, idx = get_prev_temp_file_desc_idx(idx), diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index 50d705a9ff..2831778dc7 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -385,7 +385,8 @@ void ArchiveSlice::get_file(ConstBlockHandle handle, FileReference ref_id, td::P auto R = kv_->get(ref_id.hash().to_hex(), value); R.ensure(); if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) { - promise.set_error(td::Status::Error(ErrorCode::notready, "file not in archive slice")); + promise.set_error(td::Status::Error( + ErrorCode::notready, PSTRING() << "file " << ref_id.filename() << " not in archive slice " << get_name())); return; } auto offset = td::to_integer(value); From d6560cafa25ffdeecae4d5933af6ba70b8a54d4d Mon Sep 17 00:00:00 2001 From: neodix42 Date: Wed, 29 Oct 2025 11:12:42 +0400 Subject: [PATCH 427/524] Adjust the delay of the initial node sync (#1860) * upgrade abseil to August 2025, Patch 1 * Add a new argument to validator-engine that helps to speed up the initial node sync. * correct wording --- validator-engine/validator-engine.cpp | 13 ++++++++++++- validator-engine/validator-engine.hpp | 4 ++++ validator/full-node-shard.cpp | 4 ++-- validator/full-node.h | 1 + 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index c78e0acc36..b94e6cbf10 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -2240,7 +2240,8 @@ void ValidatorEngine::start_full_node() { ton::validator::fullnode::FullNodeOptions full_node_options{ .config_ = config_.full_node_config, .public_broadcast_speed_multiplier_ = broadcast_speed_multiplier_public_, - .private_broadcast_speed_multiplier_ = broadcast_speed_multiplier_private_}; + .private_broadcast_speed_multiplier_ = broadcast_speed_multiplier_private_, + .initial_sync_delay_ = initial_sync_delay_}; full_node_ = ton::validator::fullnode::FullNode::create( short_id, full_node_id_, validator_options_->zero_block_id().file_hash, full_node_options, keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(), @@ -5572,6 +5573,16 @@ int main(int argc, char *argv[]) { "don't select the best persistent state on initial sync, start on init_block from global config", [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_skip_key_sync, true); }); }); + p.add_checked_option( + '\0', "initial-sync-delay", "delay before initial sync completion (in seconds, default: 60.0)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v < 0) { + return td::Status::Error("initial-sync-delay should be non-negative"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_initial_sync_delay, v); }); + return td::Status::OK(); + }); p.add_checked_option( '\0', "sync-shards-upto", "stop syncing shards on this masterchain seqno", [&](td::Slice s) -> td::Status { TRY_RESULT(v, td::to_integer_safe(s)); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index ea6ef0da47..26b6842efc 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -260,6 +260,7 @@ class ValidatorEngine : public td::actor::Actor { td::optional sync_shards_upto_; ton::adnl::AdnlNodeIdShort shard_block_retainer_adnl_id_ = ton::adnl::AdnlNodeIdShort::zero(); bool shard_block_retainer_adnl_id_fullnode_ = false; + double initial_sync_delay_ = 60.0; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -390,6 +391,9 @@ class ValidatorEngine : public td::actor::Actor { void set_shard_block_retainer_adnl_id_fullnode() { shard_block_retainer_adnl_id_fullnode_ = true; } + void set_initial_sync_delay(double value) { + initial_sync_delay_ = value; + } void start_up() override; ValidatorEngine() { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 771b28be38..39a416ed54 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -213,7 +213,7 @@ void FullNodeShardImpl::got_next_block(td::Result R) { if (handle_->unix_time() > td::Clocks::system() - 300) { promise_.set_value(td::Unit()); } else { - sync_completed_at_ = td::Timestamp::in(60.0); + sync_completed_at_ = td::Timestamp::in(opts_.initial_sync_delay_); } } get_next_block(); @@ -1085,7 +1085,7 @@ void FullNodeShardImpl::set_handle(BlockHandle handle, td::Promise pro promise_ = std::move(promise); get_next_block(); - sync_completed_at_ = td::Timestamp::in(60.0); + sync_completed_at_ = td::Timestamp::in(opts_.initial_sync_delay_); alarm_timestamp().relax(sync_completed_at_); } diff --git a/validator/full-node.h b/validator/full-node.h index 4c11c1fbd8..2d1fd7aaf8 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -59,6 +59,7 @@ struct FullNodeOptions { FullNodeConfig config_; double public_broadcast_speed_multiplier_ = 1.0; double private_broadcast_speed_multiplier_ = 1.0; + double initial_sync_delay_ = 60.0; }; struct CustomOverlayParams { From ccdff9c94ce8172428af905eda72fff2c0a17bd7 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Wed, 29 Oct 2025 12:32:21 +0400 Subject: [PATCH 428/524] Upgrade abseil to August 2025, Patch 1, tag=20250814.1 (#1859) * upgrade abseil to August 2025, Patch 1 * upgrade abseil to August 2025, Patch 1 --- .gitmodules | 1 + third-party/abseil-cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index e4e307960e..4b1d0203da 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,6 +7,7 @@ [submodule "third-party/abseil-cpp"] path = third-party/abseil-cpp url = https://github.com/abseil/abseil-cpp.git + branch = 20250814.1 [submodule "third-party/libraptorq"] path = third-party/libraptorq url = https://github.com/ton-blockchain/libRaptorQ diff --git a/third-party/abseil-cpp b/third-party/abseil-cpp index 8c6e53ef3a..d38452e1ee 160000 --- a/third-party/abseil-cpp +++ b/third-party/abseil-cpp @@ -1 +1 @@ -Subproject commit 8c6e53ef3adb1227fffa442c50349dab134a54bc +Subproject commit d38452e1ee03523a208362186fd42248ff2609f6 From 27f96429c6b259b7bba8b5ea8afabb08be13a404 Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Thu, 30 Oct 2025 02:16:21 -0400 Subject: [PATCH 429/524] Remove outdated warning suppressions (#1861) These were needed because our abseil version was too old. --- suppression_mappings.txt | 9 --------- 1 file changed, 9 deletions(-) diff --git a/suppression_mappings.txt b/suppression_mappings.txt index 5822b3f183..2fc34a30a8 100644 --- a/suppression_mappings.txt +++ b/suppression_mappings.txt @@ -1,12 +1,3 @@ -[#warnings] -src:*/ciso646 - -[deprecated-builtins] -src:*/third-party/abseil-cpp/absl/meta/type_traits.h - -[array-parameter] -src:*/third-party/abseil-cpp/absl/hash/internal/low_level_hash.cc - [deprecated-literal-operator] src:*/tdutils/td/utils/date.h From 85d8b0178bb66d0a46a79b38e2530574ddd1c1ca Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Thu, 30 Oct 2025 02:17:17 -0400 Subject: [PATCH 430/524] Make DHT node not care about validator section of global config (#1862) --- dht-server/dht-server.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index 46a011352c..c8fc4508d2 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -433,14 +433,6 @@ td::Status DhtServer::load_global_config() { TRY_RESULT_PREFIX(dht, ton::dht::Dht::create_global_config(std::move(conf.dht_)), "bad [dht] section: "); dht_config_ = std::move(dht); - if (!conf.validator_) { - return td::Status::Error(ton::ErrorCode::error, "does not contain [validator] section"); - } - - if (!conf.validator_->zero_state_) { - return td::Status::Error(ton::ErrorCode::error, "[validator] section does not contain [zero_state]"); - } - return td::Status::OK(); } From a1ddbccb40a6cf41c032fe48e1c24f664c8ac8e6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 30 Oct 2025 18:12:37 +0300 Subject: [PATCH 431/524] Better error processing in ArchiveManager --- adnl/adnl-packet.cpp | 2 +- validator/db/archive-manager.cpp | 71 ++++++++++++++++++-------------- validator/db/archive-manager.hpp | 4 +- validator/impl/check-proof.cpp | 2 +- 4 files changed, 44 insertions(+), 35 deletions(-) diff --git a/adnl/adnl-packet.cpp b/adnl/adnl-packet.cpp index b12493e777..7159fbdde3 100644 --- a/adnl/adnl-packet.cpp +++ b/adnl/adnl-packet.cpp @@ -56,7 +56,7 @@ td::Result AdnlPacket::create(tl_object_ptraddress_))); + TRY_RESULT(addr_list, AdnlAddressList::create(std::move(packet->priority_address_))); R.priority_addr_ = std::move(addr_list); } if (R.flags_ & Flags::f_seqno) { diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index d3731b92eb..2020164576 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -70,8 +70,9 @@ void ArchiveManager::add_handle(BlockHandle handle, td::Promise promis handle->unix_time(), handle->logical_time(), handle->inited_is_key_block() && handle->is_key_block()) : get_package_id(handle->masterchain_ref_block()); - auto f = get_file_desc(handle->id().shard_full(), p, handle->id().seqno(), handle->unix_time(), - handle->logical_time(), true); + TRY_RESULT_PROMISE(promise, f, + get_file_desc(handle->id().shard_full(), p, handle->id().seqno(), handle->unix_time(), + handle->logical_time(), true)); td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::add_handle, std::move(handle), std::move(promise)); } @@ -83,16 +84,17 @@ void ArchiveManager::update_handle(BlockHandle handle, td::Promise pro promise.set_value(td::Unit()); return; } - f = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()), handle->id().seqno(), - handle->unix_time(), handle->logical_time(), true); - if (!f) { + auto F = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()), + handle->id().seqno(), handle->unix_time(), handle->logical_time(), true); + if (F.is_error()) { handle->flushed_upto(handle->version()); promise.set_value(td::Unit()); return; } + f = F.move_as_ok(); } else { - f = get_file_desc(handle->id().shard_full(), get_temp_package_id(), 0, 0, 0, true); - CHECK(f); + TRY_RESULT_PROMISE_ASSIGN(promise, f, + get_file_desc(handle->id().shard_full(), get_temp_package_id(), 0, 0, 0, true)); } td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::update_handle, std::move(handle), std::move(promise)); } @@ -110,13 +112,17 @@ void ArchiveManager::add_file(BlockHandle handle, FileReference ref_id, td::Buff auto ig = mp.init_guard(); ig.add_promise(std::move(promise)); auto f1 = get_file_desc(handle->id().shard_full(), get_temp_package_id(), 0, 0, 0, true); - td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::add_file, nullptr, std::move(ref_id), data.clone(), - ig.get_promise()); + if (f1.is_ok()) { + td::actor::send_closure(f1.ok()->file_actor_id(), &ArchiveSlice::add_file, nullptr, std::move(ref_id), + data.clone(), ig.get_promise()); + } if (copy_to_key) { auto f2 = get_file_desc(handle->id().shard_full(), get_key_package_id(handle->masterchain_ref_block()), handle->id().seqno(), handle->unix_time(), handle->logical_time(), true); - td::actor::send_closure(f2->file_actor_id(), &ArchiveSlice::add_file, nullptr, ref_id, std::move(data), - ig.get_promise()); + if (f2.is_ok()) { + td::actor::send_closure(f2.ok()->file_actor_id(), &ArchiveSlice::add_file, nullptr, ref_id, std::move(data), + ig.get_promise()); + } } return; } @@ -128,24 +134,30 @@ void ArchiveManager::add_file(BlockHandle handle, FileReference ref_id, td::Buff ig.add_promise(std::move(promise)); auto f1 = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()), handle->id().seqno(), handle->unix_time(), handle->logical_time(), true); - td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::add_file, handle, ref_id, data.clone(), ig.get_promise()); + if (f1.is_ok()) { + td::actor::send_closure(f1.ok()->file_actor_id(), &ArchiveSlice::add_file, handle, ref_id, data.clone(), + ig.get_promise()); + } if (copy_to_key) { auto f2 = get_file_desc(handle->id().shard_full(), get_key_package_id(handle->masterchain_ref_block()), handle->id().seqno(), handle->unix_time(), handle->logical_time(), true); - td::actor::send_closure(f2->file_actor_id(), &ArchiveSlice::add_file, handle, ref_id, std::move(data), - ig.get_promise()); + if (f2.is_ok()) { + td::actor::send_closure(f2.ok()->file_actor_id(), &ArchiveSlice::add_file, handle, ref_id, std::move(data), + ig.get_promise()); + } } } void ArchiveManager::add_key_block_proof(UnixTime ts, BlockSeqno seqno, LogicalTime lt, FileReference ref_id, td::BufferSlice data, td::Promise promise) { - auto f = get_file_desc(ShardIdFull{masterchainId}, get_key_package_id(seqno), seqno, ts, lt, true); + TRY_RESULT_PROMISE(promise, f, + get_file_desc(ShardIdFull{masterchainId}, get_key_package_id(seqno), seqno, ts, lt, true)); td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::add_file, nullptr, std::move(ref_id), std::move(data), std::move(promise)); } void ArchiveManager::add_temp_file_short(FileReference ref_id, td::BufferSlice data, td::Promise promise) { - auto f = get_file_desc(ref_id.shard(), get_temp_package_id(), 0, 0, 0, true); + TRY_RESULT_PROMISE(promise, f, get_file_desc(ref_id.shard(), get_temp_package_id(), 0, 0, 0, true)); td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::add_file, nullptr, std::move(ref_id), std::move(data), std::move(promise)); } @@ -287,17 +299,17 @@ void ArchiveManager::get_temp_file_short_cont(FileReference ref_id, PackageId id void ArchiveManager::get_file(ConstBlockHandle handle, FileReference ref_id, td::Promise promise) { if (handle->moved_to_archive()) { auto f = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()), 0, 0, 0, false); - if (f) { - td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_file, std::move(handle), std::move(ref_id), + if (f.is_ok()) { + td::actor::send_closure(f.ok()->file_actor_id(), &ArchiveSlice::get_file, std::move(handle), std::move(ref_id), std::move(promise)); return; } } if (handle->handle_moved_to_archive()) { auto f = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()), 0, 0, 0, false); - if (f) { + if (f.ok()) { promise = [=, promise = std::move(promise), - file_actor = f->file_actor_id()](td::Result R) mutable { + file_actor = f.ok()->file_actor_id()](td::Result R) mutable { if (R.is_ok()) { promise.set_value(R.move_as_ok()); return; @@ -656,13 +668,14 @@ void ArchiveManager::load_package(PackageId id) { update_permanent_slices(); } -const ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, - UnixTime ts, LogicalTime lt, bool force) { +td::Result ArchiveManager::get_file_desc(ShardIdFull shard, PackageId id, + BlockSeqno seqno, UnixTime ts, + LogicalTime lt, bool force) { auto &f = get_file_map(id); auto it = f.find(id); if (it != f.end()) { if (it->second.deleted) { - return nullptr; + return td::Status::Error("file is deleted"); } if (force && !id.temp) { update_desc(f, it->second, shard, seqno, ts, lt); @@ -670,7 +683,7 @@ const ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull return &it->second; } if (!force) { - return nullptr; + return td::Status::Error("file not found"); } return add_file_desc(shard, id, seqno, ts, lt); @@ -1186,13 +1199,9 @@ void ArchiveManager::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull sh void ArchiveManager::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) { auto arch = static_cast(archive_id); - auto F = get_file_desc(ShardIdFull{masterchainId}, PackageId{arch, false, false}, 0, 0, 0, false); - if (!F) { - promise.set_error(td::Status::Error(ErrorCode::notready, "archive not found")); - return; - } - - td::actor::send_closure(F->file_actor_id(), &ArchiveSlice::get_slice, archive_id, offset, limit, std::move(promise)); + TRY_RESULT_PROMISE(promise, f, + get_file_desc(ShardIdFull{masterchainId}, PackageId{arch, false, false}, 0, 0, 0, false)); + td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_slice, archive_id, offset, limit, std::move(promise)); } void ArchiveManager::commit_transaction() { diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index 947923f4b6..19a41b194b 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -206,8 +206,8 @@ class ArchiveManager : public td::actor::Actor { void get_handle_finish(BlockHandle handle, td::Promise promise); void get_temp_file_short_cont(FileReference ref_id, PackageId idx, td::Promise promise); - const FileDescription *get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt, - bool force); + td::Result get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, + LogicalTime lt, bool force); const FileDescription *add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt); void update_desc(FileMap &f, const FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts, LogicalTime lt); diff --git a/validator/impl/check-proof.cpp b/validator/impl/check-proof.cpp index 05d5c84ba1..ae0ce92fcf 100644 --- a/validator/impl/check-proof.cpp +++ b/validator/impl/check-proof.cpp @@ -68,7 +68,7 @@ void CheckProof::finish_query() { ValidatorInvariants::check_post_check_proof_link(handle_); } if (promise_) { - VLOG(VALIDATOR_DEBUG) << "checked proof for " << handle_->id(); + VLOG(VALIDATOR_DEBUG) << "checked proof for " << handle_->id().to_str(); promise_.set_result(handle_); } stop(); From fc397a9fb7f653674d9017541a766822ac64b729 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Thu, 30 Oct 2025 21:14:21 +0300 Subject: [PATCH 432/524] Add config contract version with ownable parameters --- .../smartcont/config-with-ownable-params.fc | 690 ++++++++++++++++++ 1 file changed, 690 insertions(+) create mode 100644 crypto/smartcont/config-with-ownable-params.fc diff --git a/crypto/smartcont/config-with-ownable-params.fc b/crypto/smartcont/config-with-ownable-params.fc new file mode 100644 index 0000000000..41893678e0 --- /dev/null +++ b/crypto/smartcont/config-with-ownable-params.fc @@ -0,0 +1,690 @@ +#pragma version =0.4.6; +;; Currently deployed config-contract in mainnet can be found +;; on https://verifier.ton.org/Ef9VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVbxn +;; Tests and other artifacts of this contract version are available at https://github.com/ton-blockchain/config-with-ownable-params + +(slice, int) check_exotic(cell c) asm "XCTOS"; +int equal_slice_bits (slice a, slice b) asm "SDEQ"; + +const int CUSTOM_PARAM_1 = -1024; +const int CUSTOM_PARAM_2 = -1025; +const slice ADMIN_ADDRESS_FOR_CUSTOM_PARAMS = "EQA_o6NFLu73wozeYNERTsW8lkU5OarbRbIkoNuWdy5SPDA_"a; +const int OP_SET_CUSTOM_PARAM = 0x6e43536c; + +() set_conf_param(int index, cell value) impure { + var cs = get_data().begin_parse(); + var cfg_dict = cs~load_ref(); + cfg_dict~idict_set_ref(32, index, value); + set_data(begin_cell().store_ref(cfg_dict).store_slice(cs).end_cell()); +} + +(cell, int, int, cell) load_data() inline { + var cs = get_data().begin_parse(); + var res = (cs~load_ref(), cs~load_uint(32), cs~load_uint(256), cs~load_dict()); + cs.end_parse(); + return res; +} + +() store_data(cfg_dict, stored_seqno, public_key, vote_dict) impure inline { + set_data(begin_cell() + .store_ref(cfg_dict) + .store_uint(stored_seqno, 32) + .store_uint(public_key, 256) + .store_dict(vote_dict) + .end_cell()); +} + +;; (min_tot_rounds, max_tot_rounds, min_wins, max_losses, min_store_sec, max_store_sec, bit_price, cell_price) +_ parse_vote_config(cell c) inline { + var cs = c.begin_parse(); + throw_unless(44, cs~load_uint(8) == 0x36); + var res = (cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(32), cs~load_uint(32), cs~load_uint(32), cs~load_uint(32)); + cs.end_parse(); + return res; +} + +;; cfg_vote_setup#91 normal_params:^ConfigProposalSetup critical_params:^ConfigProposalSetup = ConfigVotingSetup; +_ get_vote_config_internal(int critical?, cell cparam11) inline_ref { + var cs = cparam11.begin_parse(); + throw_unless(44, cs~load_uint(8) == 0x91); + if (critical?) { + cs~load_ref(); + } + return parse_vote_config(cs.preload_ref()); +} + +_ get_vote_config(int critical?) inline { + return get_vote_config_internal(critical?, config_param(11)); +} + +(int, int) check_validator_set(cell vset) { + var cs = vset.begin_parse(); + throw_unless(9, cs~load_uint(8) == 0x12); ;; validators_ext#12 only + int utime_since = cs~load_uint(32); + int utime_until = cs~load_uint(32); + int total = cs~load_uint(16); + int main = cs~load_uint(16); + throw_unless(9, main > 0); + throw_unless(9, total >= main); + return (utime_since, utime_until); +} + +() send_answer(addr, query_id, ans_tag, mode) impure { + ;; int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool src:MsgAddress -> 011000 + send_raw_message(begin_cell() + .store_uint(0x18, 6) + .store_slice(addr) + .store_uint(0, 5 + 4 + 4 + 64 + 32 + 1 + 1) + .store_uint(ans_tag, 32) + .store_uint(query_id, 64) + .end_cell(), mode); +} + +() send_confirmation(addr, query_id, ans_tag) impure inline { + return send_answer(addr, query_id, ans_tag, 64); +} + +() send_error(addr, query_id, ans_tag) impure inline { + return send_answer(addr, query_id, ans_tag, 64); +} + +;; forward a message to elector smart contract to make it upgrade its code +() change_elector_code(slice cs) impure { + var dest_addr = config_param(1).begin_parse().preload_uint(256); + var query_id = now(); + send_raw_message(begin_cell() + .store_uint(0xc4ff, 17) + .store_uint(dest_addr, 256) + .store_grams(1 << 30) ;; ~ 1 Gram (will be returned back) + .store_uint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) + .store_uint(0x4e436f64, 32) ;; action + .store_uint(query_id, 64) + .store_slice(cs) + .end_cell(), 0); +} + +() after_code_upgrade(slice param, cont old_code) impure method_id(1666) { +} + +_ perform_action(cfg_dict, public_key, action, cs) inline_ref { + if (action == 0x43665021) { + ;; change one configuration parameter + var param_index = cs~load_int(32); + var param_value = cs~load_ref(); + cs.end_parse(); + cfg_dict~idict_set_ref(32, param_index, param_value); + return (cfg_dict, public_key); + } elseif (action == 0x4e436f64) { + ;; change configuration smart contract code + var new_code = cs~load_ref(); + set_code(new_code); + var old_code = get_c3(); + set_c3(new_code.begin_parse().bless()); + after_code_upgrade(cs, old_code); + throw(0); + return (cfg_dict, public_key); + } elseif (action == 0x50624b21) { + ;; change configuration master public key + public_key = cs~load_uint(256); + cs.end_parse(); + return (cfg_dict, public_key); + } elseif (action == 0x4e43ef05) { + ;; change election smart contract code + change_elector_code(cs); + return (cfg_dict, public_key); + } else { + throw_if(32, action); + return (cfg_dict, public_key); + } +} + +(cell, int, cell) get_current_vset() inline_ref { + var vset = config_param(34); + var cs = begin_parse(vset); + ;; validators_ext#12 utime_since:uint32 utime_until:uint32 + ;; total:(## 16) main:(## 16) { main <= total } { main >= 1 } + ;; total_weight:uint64 + throw_unless(40, cs~load_uint(8) == 0x12); + cs~skip_bits(32 + 32 + 16 + 16); + var (total_weight, dict) = (cs~load_uint(64), cs~load_dict()); + cs.end_parse(); + return (vset, total_weight, dict); +} + +(slice, int) get_validator_descr(int idx) inline_ref { + var (vset, total_weight, dict) = get_current_vset(); + var (value, _) = dict.udict_get?(16, idx); + return (value, total_weight); +} + +(int, int) unpack_validator_descr(slice cs) inline { + ;; ed25519_pubkey#8e81278a pubkey:bits256 = SigPubKey; + ;; validator#53 public_key:SigPubKey weight:uint64 = ValidatorDescr; + ;; validator_addr#73 public_key:SigPubKey weight:uint64 adnl_addr:bits256 = ValidatorDescr; + throw_unless(41, (cs~load_uint(8) & ~ 0x20) == 0x53); + throw_unless(41, cs~load_uint(32) == 0x8e81278a); + return (cs~load_uint(256), cs~load_uint(64)); +} + +;; cfg_proposal#f3 param_id:int32 param_value:(Maybe ^Cell) if_hash_equal:(Maybe uint256) +;; c -> (param-id param-cell maybe-hash) +(int, cell, int) parse_config_proposal(cell c) inline_ref { + var cs = c.begin_parse(); + throw_unless(44, cs~load_int(8) == 0xf3 - 0x100); + var (id, val, hash) = (cs~load_int(32), cs~load_maybe_ref(), cs~load_int(1)); + if (hash) { + hash = cs~load_uint(256); + } else { + hash = -1; + } + cs.end_parse(); + return (id, val, hash); +} + +(cell, int, cell) accept_proposal(cell cfg_dict, cell proposal, int critical?) inline_ref { + var (param_id, param_val, req_hash) = parse_config_proposal(proposal); + cell cur_val = cfg_dict.idict_get_ref(32, param_id); + int cur_hash = null?(cur_val) ? 0 : cell_hash(cur_val); + if ((cur_hash != req_hash) & (req_hash >= 0)) { + ;; current value has incorrect hash, do not apply changes + return (cfg_dict, 0, null()); + } + if ((param_id == CUSTOM_PARAM_1) | (param_id == CUSTOM_PARAM_2)) { + ;; Custom parameters cannot be changed by validator voting, do not apply changes + return (cfg_dict, 0, null()); + } + cell mparams = cfg_dict.idict_get_ref(32, 9); ;; mandatory parameters + var (_, found?) = mparams.idict_get?(32, param_id); + if (found? & param_val.null?()) { + ;; cannot set a mandatory parameter to (null) + return (cfg_dict, 0, null()); + } + cell cparams = cfg_dict.idict_get_ref(32, 10); ;; critical parameters + (_, found?) = cparams.idict_get?(32, param_id); + if (found? < critical?) { + ;; trying to set a critical parameter after a non-critical voting + return (cfg_dict, 0, null()); + } + ;; CHANGE ONE CONFIGURATION PARAMETER (!) + cfg_dict~idict_set_ref(32, param_id, param_val); + return (cfg_dict, param_id, param_val); +} + +(cell, int) perform_proposed_action(cell cfg_dict, int public_key, int param_id, cell param_val) inline_ref { + if (param_id == -999) { + ;; appoint or depose dictator + return (cfg_dict, param_val.null?() ? 0 : param_val.begin_parse().preload_uint(256)); + } + if (param_val.null?()) { + return (cfg_dict, public_key); + } + if (param_id == -1000) { + ;; upgrade code + var cs = param_val.begin_parse(); + var new_code = cs~load_ref(); + set_code(new_code); + var old_code = get_c3(); + set_c3(new_code.begin_parse().bless()); + after_code_upgrade(cs, old_code); + throw(0); + return (cfg_dict, public_key); + } + if (param_id == -1001) { + ;; update elector code + var cs = param_val.begin_parse(); + change_elector_code(cs); + } + return (cfg_dict, public_key); +} + +;; cfg_proposal_status#ce expires:uint32 proposal:^ConfigProposal is_critical:Bool +;; voters:(HashmapE 16 True) remaining_weight:int64 validator_set_id:uint256 +;; rounds_remaining:uint8 wins:uint8 losses:uint8 = ConfigProposalStatus; +(int, cell, int, cell, int, int, slice) unpack_proposal_status(slice cs) inline_ref { + throw_unless(44, cs~load_int(8) == 0xce - 0x100); + return (cs~load_uint(32), cs~load_ref(), cs~load_int(1), cs~load_dict(), cs~load_int(64), cs~load_uint(256), cs); +} + +slice update_proposal_status(slice rest, int weight_remaining, int critical?) inline_ref { + var (min_tot_rounds, max_tot_rounds, min_wins, max_losses, _, _, _, _) = get_vote_config(critical?); + var (rounds_remaining, wins, losses) = (rest~load_uint(8), rest~load_uint(8), rest~load_uint(8)); + losses -= (weight_remaining >= 0); + if (losses > max_losses) { + ;; lost too many times + return null(); + } + rounds_remaining -= 1; + if (rounds_remaining < 0) { + ;; existed for too many rounds + return null(); + } + return begin_cell() + .store_uint(rounds_remaining, 8) + .store_uint(wins, 8) + .store_uint(losses, 8) + .end_cell().begin_parse(); +} + +builder begin_pack_proposal_status(int expires, cell proposal, int critical?, cell voters, int weight_remaining, int vset_id) inline { + return begin_cell() + .store_int(0xce - 0x100, 8) + .store_uint(expires, 32) + .store_ref(proposal) + .store_int(critical?, 1) + .store_dict(voters) + .store_int(weight_remaining, 64) + .store_uint(vset_id, 256); +} + +(cell, cell, int) register_vote(vote_dict, phash, idx, weight) inline_ref { + var (pstatus, found?) = vote_dict.udict_get?(256, phash); + ifnot (found?) { + ;; config proposal not found + return (vote_dict, null(), -1); + } + var (cur_vset, total_weight, _) = get_current_vset(); + int cur_vset_id = cur_vset.cell_hash(); + var (expires, proposal, critical?, voters, weight_remaining, vset_id, rest) = unpack_proposal_status(pstatus); + if (expires <= now()) { + ;; config proposal expired, delete and report not found + vote_dict~udict_delete?(256, phash); + return (vote_dict, null(), -1); + } + if (vset_id != cur_vset_id) { + ;; config proposal belongs to a previous validator set + vset_id = cur_vset_id; + rest = update_proposal_status(rest, weight_remaining, critical?); + voters = null(); + weight_remaining = muldiv(total_weight, 3, 4); + } + if (rest.null?()) { + ;; discard proposal (existed for too many rounds, or too many losses) + vote_dict~udict_delete?(256, phash); + return (vote_dict, null(), -1); + } + var (_, found?) = voters.udict_get?(16, idx); + if (found?) { + ;; already voted for this proposal, ignore vote + return (vote_dict, null(), -2); + } + ;; register vote + voters~udict_set_builder(16, idx, begin_cell().store_uint(now(), 32)); + int old_wr = weight_remaining; + weight_remaining -= weight; + if ((weight_remaining ^ old_wr) >= 0) { + ;; not enough votes, or proposal already accepted in this round + ;; simply update weight_remaining + vote_dict~udict_set_builder(256, phash, begin_pack_proposal_status(expires, proposal, critical?, voters, weight_remaining, vset_id).store_slice(rest)); + return (vote_dict, null(), 2); + } + ;; proposal wins in this round + var (min_tot_rounds, max_tot_rounds, min_wins, max_losses, _, _, _, _) = get_vote_config(critical?); + var (rounds_remaining, wins, losses) = (rest~load_uint(8), rest~load_uint(8), rest~load_uint(8)); + wins += 1; + if (wins >= min_wins) { + ;; proposal is accepted, remove and process + vote_dict~udict_delete?(256, phash); + return (vote_dict, proposal, 6 - critical?); + } + ;; update proposal info + vote_dict~udict_set_builder(256, phash, + begin_pack_proposal_status(expires, proposal, critical?, voters, weight_remaining, vset_id) + .store_uint(rounds_remaining, 8) + .store_uint(wins, 8) + .store_uint(losses, 8)); + return (vote_dict, null(), 2); +} + +int proceed_register_vote(phash, idx, weight) impure inline_ref { + var (cfg_dict, stored_seqno, public_key, vote_dict) = load_data(); + (vote_dict, var accepted_proposal, var status) = register_vote(vote_dict, phash, idx, weight); + store_data(cfg_dict, stored_seqno, public_key, vote_dict); + ifnot (accepted_proposal.null?()) { + var critical? = 6 - status; + (cfg_dict, var param_id, var param_val) = accept_proposal(cfg_dict, accepted_proposal, critical?); + store_data(cfg_dict, stored_seqno, public_key, vote_dict); + if (param_id) { + commit(); + (cfg_dict, public_key) = perform_proposed_action(cfg_dict, public_key, param_id, param_val); + store_data(cfg_dict, stored_seqno, public_key, vote_dict); + } + } + return status; +} + +(slice, int) scan_proposal(int phash, slice pstatus) inline_ref { + var (cur_vset, total_weight, _) = get_current_vset(); + int cur_vset_id = cur_vset.cell_hash(); + var (expires, proposal, critical?, voters, weight_remaining, vset_id, rest) = unpack_proposal_status(pstatus); + if (expires <= now()) { + ;; config proposal expired, delete + return (null(), true); + } + if (vset_id == cur_vset_id) { + ;; config proposal already processed or voted for in this round, change nothing + return (pstatus, false); + } + ;; config proposal belongs to a previous validator set + vset_id = cur_vset_id; + rest = update_proposal_status(rest, weight_remaining, critical?); + voters = null(); + weight_remaining = muldiv(total_weight, 3, 4); + if (rest.null?()) { + ;; discard proposal (existed for too many rounds, or too many losses) + return (null(), true); + } + ;; return updated proposal + return (begin_pack_proposal_status(expires, proposal, critical?, voters, weight_remaining, vset_id).store_slice(rest).end_cell().begin_parse(), true); +} + +cell scan_random_proposal(cell vote_dict) inline_ref { + var (phash, pstatus, found?) = vote_dict.udict_get_nexteq?(256, random()); + ifnot (found?) { + return vote_dict; + } + (pstatus, var changed?) = scan_proposal(phash, pstatus); + if (changed?) { + if (pstatus.null?()) { + vote_dict~udict_delete?(256, phash); + } else { + vote_dict~udict_set(256, phash, pstatus); + } + } + return vote_dict; +} + +int register_voting_proposal(slice cs, int msg_value) impure inline_ref { + var (expire_at, proposal, critical?) = (cs~load_uint(32), cs~load_ref(), cs~load_int(1)); + if (expire_at >> 30) { + expire_at -= now(); + } + var (param_id, param_val, hash) = parse_config_proposal(proposal); + if ((param_id == CUSTOM_PARAM_1) | (param_id == CUSTOM_PARAM_2)) { + return -0xc3726956; ;; negative hash means error and will be returned as answer tag + } + if (hash >= 0) { + cell cur_val = config_param(param_id); + int cur_hash = null?(cur_val) ? 0 : cell_hash(cur_val); + if (cur_hash != hash) { + hash = -0xe2646356; ;; bad current value + } + } else { + var m_params = config_param(9); + var (_, found?) = m_params.idict_get?(32, param_id); + if (found?) { + hash = -0xcd506e6c; ;; cannot set mandatory parameter to null + } + } + if (param_val.cell_depth() >= 256) { + hash = -0xc2616456; ;; bad value + } + if (hash < -1) { + return hash; ;; return error if any + } + ifnot (critical?) { + var crit_params = config_param(10); + var (_, found?) = crit_params.idict_get?(32, param_id); + if (found?) { + hash = -0xc3726954; ;; trying to set a critical parameter without critical flag + } + } + if (hash < -1) { + return hash; + } + ;; obtain vote proposal configuration + var vote_cfg = get_vote_config(critical?); + var (min_tot_rounds, max_tot_rounds, min_wins, max_losses, min_store_sec, max_store_sec, bit_price, cell_price) = vote_cfg; + if (expire_at < min_store_sec) { + return -0xc5787069; ;; expired + } + expire_at = min(expire_at, max_store_sec); + ;; compute price + var (_, bits, refs) = compute_data_size(param_val, 1024); + var pps = bit_price * (bits + 1024) + cell_price * (refs + 2); + var price = pps * expire_at; + expire_at += now(); + var (cfg_dict, stored_seqno, public_key, vote_dict) = load_data(); + int phash = proposal.cell_hash(); + var (pstatus, found?) = vote_dict.udict_get?(256, phash); + if (found?) { + ;; proposal already exists; we can only extend it + var (expires, r_proposal, r_critical?, voters, weight_remaining, vset_id, rest) = unpack_proposal_status(pstatus); + if (r_critical? != critical?) { + return -0xc3726955; ;; cannot upgrade critical parameter to non-critical... + } + if (expires >= expire_at) { + return -0xc16c7245; ;; proposal already exists + } + ;; recompute price + price = pps * (expire_at - expires + 16384); + if (msg_value - price < (1 << 30)) { + return -0xf0617924; ;; need more money + } + ;; update expiration time + vote_dict~udict_set_builder(256, phash, begin_pack_proposal_status(expire_at, r_proposal, r_critical?, voters, weight_remaining, vset_id).store_slice(rest)); + store_data(cfg_dict, stored_seqno, public_key, vote_dict); + return price; + } + if (msg_value - price < (1 << 30)) { + return -0xf0617924; ;; need more money + } + ;; obtain current validator set data + var (vset, total_weight, _) = get_current_vset(); + int weight_remaining = muldiv(total_weight, 3, 4); + ;; create new proposal + vote_dict~udict_set_builder(256, phash, + begin_pack_proposal_status(expire_at, proposal, critical?, null(), weight_remaining, vset.cell_hash()) + .store_uint(max_tot_rounds, 8).store_uint(0, 16)); + store_data(cfg_dict, stored_seqno, public_key, vote_dict); + return price; +} + +() recv_internal(int msg_value, cell in_msg_cell, slice in_msg) impure { + var cs = in_msg_cell.begin_parse(); + var flags = cs~load_uint(4); ;; int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool + var s_addr = cs~load_msg_addr(); + (int src_wc, int src_addr) = s_addr.parse_std_addr(); + int is_admin_of_custom_param = equal_slice_bits(s_addr, ADMIN_ADDRESS_FOR_CUSTOM_PARAMS); + int is_valid_workchain = is_admin_of_custom_param | (src_wc == -1); + if ((~ is_valid_workchain) | (flags & 1) | in_msg.slice_empty?()) { + ;; source not in masterchain, or a bounced message, or a simple transfer + return (); + } + int tag = in_msg~load_uint(32); + int query_id = in_msg~load_uint(64); + + throw_if(37, is_admin_of_custom_param & (tag != OP_SET_CUSTOM_PARAM)); + + if (tag == 0x4e565354) { + ;; set next validator set + var vset = in_msg~load_ref(); + in_msg.end_parse(); + var elector_param = config_param(1); + var elector_addr = cell_null?(elector_param) ? -1 : elector_param.begin_parse().preload_uint(256); + var ok = false; + if (src_addr == elector_addr) { + ;; message from elector smart contract + ;; set next validator set + (var t_since, var t_until) = check_validator_set(vset); + var t = now(); + ok = (t_since > t) & (t_until > t_since); + } + if (ok) { + set_conf_param(36, vset); + ;; send confirmation + return send_confirmation(s_addr, query_id, 0xee764f4b); + } else { + return send_error(s_addr, query_id, 0xee764f6f); + } + } + if (tag == OP_SET_CUSTOM_PARAM) { + cell value = in_msg~load_ref(); + int index = in_msg~load_int(32); + slice receiver_address = in_msg~load_msg_addr(); + in_msg.end_parse(); + (_, int exotic) = value.check_exotic(); + int depth = value.cell_depth(); + + (int response_wc, int response_address_hash) = receiver_address.parse_std_addr(); ;; Checking the validity of the response address + response_wc~impure_touch(); ;; Prevent compiler optimisation from removing parse_std_addr checking + + if ( + (exotic == 0) & ;; forbid exotic cells in config + (depth == 0) & ;; forbid cells with more than one ref + (((index == CUSTOM_PARAM_1) | (index == CUSTOM_PARAM_2)) & is_admin_of_custom_param) + ) { + set_conf_param(index, value); + ;; send confirmation + return send_confirmation(receiver_address, query_id, 0xef764f4b); + } else { + return send_error(receiver_address, query_id, 0xef764f6f); + } + } + if (tag == 0x6e565052) { + ;; new voting proposal + var price = register_voting_proposal(in_msg, msg_value); + int mode = 64; + int ans_tag = - price; + if (price >= 0) { + ;; ok, debit price + raw_reserve(price, 4); + ans_tag = 0xee565052; + mode = 128; + } + return send_answer(s_addr, query_id, ans_tag, mode); + } + if (tag == 0x566f7465) { + ;; vote for a configuration proposal + var signature = in_msg~load_bits(512); + var msg_body = in_msg; + var (sign_tag, idx, phash) = (in_msg~load_uint(32), in_msg~load_uint(16), in_msg~load_uint(256)); + in_msg.end_parse(); + throw_unless(37, sign_tag == 0x566f7445); + var (vdescr, total_weight) = get_validator_descr(idx); + var (val_pubkey, weight) = unpack_validator_descr(vdescr); + throw_unless(34, check_data_signature(msg_body, signature, val_pubkey)); + int res = proceed_register_vote(phash, idx, weight); + return send_confirmation(s_addr, query_id, res + 0xd6745240); + } + ;; if tag is non-zero and its higher bit is zero, throw an exception (the message is an unsupported query) + ;; to bounce message back to sender + throw_unless(37, (tag == 0) | (tag & (1 << 31))); + ;; do nothing for other internal messages +} + +() recv_external(slice in_msg) impure { + var signature = in_msg~load_bits(512); + var cs = in_msg; + int action = cs~load_uint(32); + int msg_seqno = cs~load_uint(32); + var valid_until = cs~load_uint(32); + throw_if(35, valid_until < now()); + throw_if(39, slice_depth(cs) > 128); + var (cfg_dict, stored_seqno, public_key, vote_dict) = load_data(); + throw_unless(33, msg_seqno == stored_seqno); + if (action == 0x566f7465) { + ;; vote for a configuration proposal + var (idx, phash) = (cs~load_uint(16), cs~load_uint(256)); + cs.end_parse(); + var (vdescr, total_weight) = get_validator_descr(idx); + var (val_pubkey, weight) = unpack_validator_descr(vdescr); + throw_unless(34, check_data_signature(in_msg, signature, val_pubkey)); + accept_message(); + stored_seqno = (stored_seqno + 1) % (1 << 32); + store_data(cfg_dict, stored_seqno, public_key, vote_dict); + commit(); + proceed_register_vote(phash, idx, weight); + return (); + } + {- + throw_unless(34, check_signature(slice_hash(in_msg), signature, public_key)); + accept_message(); + stored_seqno = (stored_seqno + 1) % (1 << 32); + store_data(cfg_dict, stored_seqno, public_key, vote_dict); + commit(); + (cfg_dict, public_key) = perform_action(cfg_dict, public_key, action, cs); + store_data(cfg_dict, stored_seqno, public_key, vote_dict); + -} +} + +() run_ticktock(int is_tock) impure { + var (cfg_dict, stored_seqno, public_key, vote_dict) = load_data(); + int kl = 32; + var next_vset = cfg_dict.idict_get_ref(kl, 36); + var updated? = false; + ifnot (next_vset.null?()) { + ;; check whether we have to set next_vset as the current validator set + var ds = next_vset.begin_parse(); + if (ds.slice_bits() >= 40) { + var tag = ds~load_uint(8); + var since = ds.preload_uint(32); + if ((since <= now()) & (tag == 0x12)) { + ;; next validator set becomes active! + var cur_vset = cfg_dict~idict_set_get_ref(kl, 34, next_vset); ;; next_vset -> cur_vset + cfg_dict~idict_set_get_ref(kl, 32, cur_vset); ;; cur_vset -> prev_vset + cfg_dict~idict_delete?(kl, 36); ;; (null) -> next_vset + updated? = true; + } + } + } + ifnot (updated?) { + ;; if nothing has been done so far, scan a random voting proposal instead + vote_dict = scan_random_proposal(vote_dict); + } + ;; save data and return + return store_data(cfg_dict, stored_seqno, public_key, vote_dict); +} + +int seqno() method_id { + return get_data().begin_parse().preload_uint(32); +} + +_ unpack_proposal(slice pstatus) inline_ref { + (int expires, cell proposal, int critical?, cell voters, int weight_remaining, int vset_id, slice rest) = unpack_proposal_status(pstatus); + var voters_list = null(); + var voter_id = (1 << 32); + do { + (voter_id, _, var f) = voters.udict_get_prev?(16, voter_id); + if (f) { + voters_list = cons(voter_id, voters_list); + } + } until (~ f); + var (rounds_remaining, wins, losses) = (rest~load_uint(8), rest~load_uint(8), rest~load_uint(8)); + rest.end_parse(); + var (param_id, param_val, param_hash) = parse_config_proposal(proposal); + return [expires, critical?, [param_id, param_val, param_hash], vset_id, voters_list, weight_remaining, rounds_remaining, wins, losses]; +} + +_ get_proposal(int phash) method_id { + (_, _, _, var vote_dict) = load_data(); + var (pstatus, found?) = vote_dict.udict_get?(256, phash); + ifnot (found?) { + return null(); + } + return unpack_proposal(pstatus); +} + +_ list_proposals() method_id { + (_, _, _, var vote_dict) = load_data(); + var phash = (1 << 255) + ((1 << 255) - 1); + var list = null(); + do { + (phash, var pstatus, var f) = vote_dict.udict_get_prev?(256, phash); + if (f) { + list = cons([phash, unpack_proposal(pstatus)], list); + } + } until (~ f); + return list; +} + +_ proposal_storage_price(int critical?, int seconds, int bits, int refs) method_id { + var cfg_dict = get_data().begin_parse().preload_ref(); + var cparam11 = cfg_dict.idict_get_ref(32, 11); + var (min_tot_rounds, max_tot_rounds, min_wins, max_losses, min_store_sec, max_store_sec, bit_price, cell_price) = get_vote_config_internal(critical?, cparam11); + if (seconds < min_store_sec) { + return -1; + } + seconds = min(seconds, max_store_sec); + return (bit_price * (bits + 1024) + cell_price * (refs + 2)) * seconds; +} + From 6f6db6f742649be77e69d626c394b8f6ffe87f08 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Thu, 30 Oct 2025 21:29:35 +0300 Subject: [PATCH 433/524] Update changelog --- Changelog.md | 10 ++++++++++ recent_changelog.md | 14 +++++++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Changelog.md b/Changelog.md index 8c6ed697d7..b5ae89f86a 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,13 @@ +## 2025.11 Update + +1. [TVM version v12](./doc/GlobalVersions.md) update: [forbid unused high bits in extra_flags](https://github.com/ton-blockchain/TEPs/pull/503/commits/d949d70d5a69026d273cbbc07653d12c4373117a), [bounce extra_flags equal to initial message extra_flags](https://github.com/ton-blockchain/TEPs/pull/503/commits/d33ff342d69de04f1c33d11360dcf06b63a6c21e), [new TVM opcodes](https://github.com/ton-blockchain/ton/commit/ecd8fbb833c408eb34ec1aa4516e9e4344b54a22). +2. Abseil upgrade +3. Improvements in node synchronisation +4. Fixing rare ArchiveManager issues +5. Various improvements in logging, builds, DHT node behavior, private net launching, failure handlers. + +Besides the work of the core team, this update is based on the efforts of the @Lapo4kaKek and [Vahagn x.com/vah_13](https://x.com/vah_13). + ## 2025.10 Update 1. [TVM version v12](./doc/GlobalVersions.md): full bounces, new `BTOS` and `HASHBU` instuctions, limit on contract size in masterchain. diff --git a/recent_changelog.md b/recent_changelog.md index fc4ae86c4a..422d4d8a1f 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,9 +1,9 @@ -## 2025.10 Update +## 2025.11 Update -1. [TVM version v12](./doc/GlobalVersions.md): full bounces, new `BTOS` and `HASHBU` instuctions, limit on contract size in masterchain. -2. Optimistic collation/validation: allow nodes to generate and check block candidates before previous block is fully signed (not fully activated yet). -3. Introduced custom block compression algorithm. -4. Overlay improvements: improved overlay discovery on shard configuration update, private externals in custom overlays. -5. Various improvements: session stats, telemetry in fast-sync overlay, earlier block broadcasts, limiting ttl for values in DHT, fixing search by utime in native blockexplorer, faster downloading candidates in validator session, parallelization of storing to cell_db, avoiding touching packfiles on startup. +1. [TVM version v12](./doc/GlobalVersions.md) update: [forbid unused high bits in extra_flags](https://github.com/ton-blockchain/TEPs/pull/503/commits/d949d70d5a69026d273cbbc07653d12c4373117a), [bounce extra_flags equal to initial message extra_flags](https://github.com/ton-blockchain/TEPs/pull/503/commits/d33ff342d69de04f1c33d11360dcf06b63a6c21e), [new TVM opcodes](https://github.com/ton-blockchain/ton/commit/ecd8fbb833c408eb34ec1aa4516e9e4344b54a22). +2. Abseil upgrade +3. Improvements in node synchronisation +4. Fixing rare ArchiveManager issues +5. Various improvements in logging, builds, DHT node behavior, private net launching, failure handlers. -Besides the work of the core team, this update is based on the efforts of the Tonstudio team: @hacker-volodya @Shvandre; and @mkiesel (avoiding touching packfiles on startup). +Besides the work of the core team, this update is based on the efforts of the @Lapo4kaKek and [Vahagn x.com/vah_13](https://x.com/vah_13). From af7b55483a18037e4e7b5e56b06e83289e5a83da Mon Sep 17 00:00:00 2001 From: "Kirlll A." <43433980+sonofmom@users.noreply.github.com> Date: Fri, 31 Oct 2025 13:14:33 +0100 Subject: [PATCH 434/524] Merge pull request #1866 from sonofmom/update/double_fd Double FD limit --- validator-engine/validator-engine.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index b94e6cbf10..cc7d676634 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -5228,7 +5228,7 @@ int main(int argc, char *argv[]) { td::log_interface = td::default_log_interface; }; - LOG_STATUS(td::change_maximize_rlimit(td::RlimitType::nofile, 786432)); + LOG_STATUS(td::change_maximize_rlimit(td::RlimitType::nofile, 1572864)); std::vector> acts; From a8810e74b65d0cbb1a68e595724bf1f082ff5f3a Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 3 Nov 2025 09:40:31 +0300 Subject: [PATCH 435/524] Fix error processing in finished_wait_state (#1868) --- validator/manager.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index 2d1f7faa86..4256cf0920 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1350,7 +1350,6 @@ void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Resultsecond.waiting_preliminary_) { X.promise.set_error(S.clone()); } - wait_state_.erase(it); } else if (!it->second.waiting_.empty() || !it->second.waiting_preliminary_.empty()) { auto X = it->second.get_timeout(); auto P1 = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { @@ -1364,7 +1363,9 @@ void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Resultid())) .release(); it->second.actor_ = id; + return; } + wait_state_.erase(it); } } From 4df3609b9bf42fb798fe590798d391801542c62b Mon Sep 17 00:00:00 2001 From: hustrust Date: Thu, 6 Nov 2025 21:33:33 +0800 Subject: [PATCH 436/524] chore: remove repetitive words in comments (#1873) Signed-off-by: hustrust Co-authored-by: EmelyanenkoK --- tdutils/td/utils/StealingQueue.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdutils/td/utils/StealingQueue.h b/tdutils/td/utils/StealingQueue.h index 86c2f0ef6d..1a2c74c73d 100644 --- a/tdutils/td/utils/StealingQueue.h +++ b/tdutils/td/utils/StealingQueue.h @@ -26,7 +26,7 @@ class StealingQueue { public: // tries to put a value // returns if succeeded - // only owner is alowed to to do this + // only owner is alowed to do this template void local_push(T value, F&& overflow_f) { while (true) { @@ -58,7 +58,7 @@ class StealingQueue { // tries to pop a value // returns if succeeded - // only owner is alowed to to do this + // only owner is alowed to do this bool local_pop(T& value) { auto tail = tail_.load(std::memory_order_relaxed); auto head = head_.load(); From 4369abd66894168e6e938a56bac6a2b0cd90306d Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Thu, 6 Nov 2025 08:49:10 -0500 Subject: [PATCH 437/524] Remove concept of virtualization level (#1648) * Mostly do not reference virtualization level outside of cells This is done by adding `Cell::is_virtualized` method that checks if `virtualization != 0` and providing a default value for vm::MerkleProof::virtualize virtualization parameter. Everywhere except for a single place in tests (where it doesn't matter), the parameter was set to 1. * Remove concept of virtualization level We use virtualization level 0 for non-virtualized cells and level 1 for virtualized ones. With this patch, we do not store this level anymore and instead infer it based on "effective" level (previously known as VirtualizationParameters::level) and the actual level of the cell. This is mainly done to reduce cognitive complexity of the code. However, as a nice side-effect, changes in VirtualCell give a negligible performance improvement on the verification benchmark. --- .../blockchain-explorer-http.cpp | 4 +- crypto/CMakeLists.txt | 1 - crypto/block/check-proof.cpp | 22 +++++----- crypto/test/test-db.cpp | 42 +++++++++---------- crypto/vm/boc.cpp | 4 +- crypto/vm/cells/Cell.cpp | 4 +- crypto/vm/cells/Cell.h | 9 ++-- crypto/vm/cells/CellBuilder.cpp | 2 +- crypto/vm/cells/CellSlice.cpp | 27 ++++++------ crypto/vm/cells/CellSlice.h | 10 ++--- crypto/vm/cells/DataCell.cpp | 19 ++++----- crypto/vm/cells/DataCell.h | 10 ++--- crypto/vm/cells/ExtCell.h | 7 ++-- crypto/vm/cells/MerkleProof.cpp | 12 +++--- crypto/vm/cells/MerkleProof.h | 6 +-- crypto/vm/cells/PrunnedCell.h | 4 +- crypto/vm/cells/UsageCell.h | 8 ++-- crypto/vm/cells/VirtualCell.h | 35 +++++++--------- crypto/vm/db/DynamicBagOfCellsDb.cpp | 4 +- crypto/vm/db/DynamicBagOfCellsDbV2.cpp | 4 +- crypto/vm/db/InMemoryBagOfCellsDb.cpp | 4 +- crypto/vm/db/StaticBagOfCellsDb.cpp | 8 ++-- crypto/vm/large-boc-serializer.cpp | 2 +- lite-client/lite-client.cpp | 16 +++---- storage/MerkleTree.cpp | 4 +- storage/MicrochunkTree.cpp | 2 +- storage/TorrentMeta.cpp | 2 +- tonlib/tonlib/TonlibClient.cpp | 22 +++++----- utils/prepare-ls-slice-config.cpp | 2 +- validator/downloaders/download-state.cpp | 4 +- validator/impl/accept-block.cpp | 2 +- validator/impl/check-proof.cpp | 2 +- validator/impl/collator.cpp | 6 +-- validator/impl/ihr-message.cpp | 2 +- validator/impl/out-msg-queue-proof.cpp | 2 +- validator/impl/proof.cpp | 4 +- validator/impl/top-shard-descr.cpp | 4 +- validator/impl/validate-query.cpp | 4 +- 38 files changed, 156 insertions(+), 170 deletions(-) diff --git a/blockchain-explorer/blockchain-explorer-http.cpp b/blockchain-explorer/blockchain-explorer-http.cpp index e5203cb7e7..f07e0e9f42 100644 --- a/blockchain-explorer/blockchain-explorer-http.cpp +++ b/blockchain-explorer/blockchain-explorer-http.cpp @@ -366,7 +366,7 @@ HttpAnswer& HttpAnswer::operator<<(AccountCell acc_c) { last_trans_hash.set_zero(); block::CurrencyCollection balance = block::CurrencyCollection::zero(); try { - auto state_root = vm::MerkleProof::virtualize(acc_c.q_roots[1], 1); + auto state_root = vm::MerkleProof::virtualize(acc_c.q_roots[1]); if (state_root.is_null()) { abort("account state proof is invalid"); return *this; @@ -474,7 +474,7 @@ HttpAnswer& HttpAnswer::operator<<(BlockHeaderCell head_c) { vm::CellSlice cs{vm::NoVm(), head_c.root}; auto block_id = head_c.block_id; try { - auto virt_root = vm::MerkleProof::virtualize(head_c.root, 1); + auto virt_root = vm::MerkleProof::virtualize(head_c.root); if (virt_root.is_null()) { abort("invalid merkle proof"); return *this; diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index ec7b3870ad..1051f0d18e 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -66,7 +66,6 @@ set(TON_CRYPTO_CORE_SOURCE vm/cells/PrunnedCell.h vm/cells/UsageCell.h vm/cells/VirtualCell.h - vm/cells/VirtualizationParameters.h vm/cells.h vm/cellslice.h diff --git a/crypto/block/check-proof.cpp b/crypto/block/check-proof.cpp index c3c9085da7..82eaa5b48a 100644 --- a/crypto/block/check-proof.cpp +++ b/crypto/block/check-proof.cpp @@ -73,7 +73,7 @@ td::Status check_block_header_proof(td::Ref root, ton::BlockIdExt blki td::Result check_state_proof(ton::BlockIdExt blkid, td::Slice proof) { TRY_RESULT(proof_root, vm::std_boc_deserialize(proof)); - auto virt_root = vm::MerkleProof::virtualize(std::move(proof_root), 1); + auto virt_root = vm::MerkleProof::virtualize(std::move(proof_root)); if (virt_root.is_null()) { return td::Status::Error("account state proof is invalid"); } @@ -86,7 +86,7 @@ td::Result> check_extract_state_proof(ton::BlockIdExt blkid, td::S try { TRY_RESULT(state_hash, check_state_proof(blkid, proof)); TRY_RESULT(state_root, vm::std_boc_deserialize(data)); - auto state_virt_root = vm::MerkleProof::virtualize(std::move(state_root), 1); + auto state_virt_root = vm::MerkleProof::virtualize(std::move(state_root)); if (state_virt_root.is_null()) { return td::Status::Error("account state proof is invalid"); } @@ -118,13 +118,13 @@ td::Status check_shard_proof(ton::BlockIdExt blk, ton::BlockIdExt shard_blk, td: return td::Status::Error("shard configuration proof must have exactly two roots"); } try { - auto mc_state_root = vm::MerkleProof::virtualize(std::move(P_roots[1]), 1); + auto mc_state_root = vm::MerkleProof::virtualize(std::move(P_roots[1])); if (mc_state_root.is_null()) { return td::Status::Error("shard configuration proof is invalid"); } ton::Bits256 mc_state_hash = mc_state_root->get_hash().bits(); TRY_STATUS_PREFIX( - check_block_header_proof(vm::MerkleProof::virtualize(std::move(P_roots[0]), 1), blk, &mc_state_hash, true), + check_block_header_proof(vm::MerkleProof::virtualize(std::move(P_roots[0])), blk, &mc_state_hash, true), "error in shard configuration block header proof :"); block::gen::ShardStateUnsplit::Record sstate; if (!(tlb::unpack_cell(mc_state_root, sstate))) { @@ -171,12 +171,12 @@ td::Status check_account_proof(td::Slice proof, ton::BlockIdExt shard_blk, const } try { - auto state_root = vm::MerkleProof::virtualize(std::move(Q_roots[1]), 1); + auto state_root = vm::MerkleProof::virtualize(std::move(Q_roots[1])); if (state_root.is_null()) { return td::Status::Error("account state proof is invalid"); } ton::Bits256 state_hash = state_root->get_hash().bits(); - TRY_STATUS_PREFIX(check_block_header_proof(vm::MerkleProof::virtualize(std::move(Q_roots[0]), 1), shard_blk, + TRY_STATUS_PREFIX(check_block_header_proof(vm::MerkleProof::virtualize(std::move(Q_roots[0])), shard_blk, &state_hash, true, save_utime, save_lt), "error in account shard block header proof : "); block::gen::ShardStateUnsplit::Record sstate; @@ -223,7 +223,7 @@ td::Result AccountState::validate(ton::BlockIdExt ref_blk, b Ref root; if (is_virtualized && true_root.not_null()) { - root = vm::MerkleProof::virtualize(true_root, 1); + root = vm::MerkleProof::virtualize(true_root); if (root.is_null()) { return td::Status::Error("account state proof is invalid"); } @@ -346,7 +346,7 @@ td::Result BlockTransactionList::validate(bool check if (check_proof) { try { TRY_RESULT(proof_cell, vm::std_boc_deserialize(std::move(proof_boc))); - auto virt_root = vm::MerkleProof::virtualize(proof_cell, 1); + auto virt_root = vm::MerkleProof::virtualize(proof_cell); if (blkid.root_hash != virt_root->get_hash().bits()) { return td::Status::Error("Invalid block proof root hash"); @@ -452,7 +452,7 @@ td::Status BlockProofLink::validate(td::uint32* save_utime) const { } try { // virtualize Merkle proof roots - auto vs_root = vm::MerkleProof::virtualize(proof, 1); + auto vs_root = vm::MerkleProof::virtualize(proof); if (vs_root.is_null()) { return td::Status::Error("BlockProofLink contains an invalid Merkle proof for source block "s + from.to_str()); } @@ -460,7 +460,7 @@ td::Status BlockProofLink::validate(td::uint32* save_utime) const { if (from.seqno()) { TRY_STATUS(check_block_header(vs_root, from, is_fwd ? nullptr : &state_hash)); } - auto vd_root = dest_proof.not_null() ? vm::MerkleProof::virtualize(dest_proof, 1) : Ref{}; + auto vd_root = dest_proof.not_null() ? vm::MerkleProof::virtualize(dest_proof) : Ref{}; if (vd_root.is_null() && to.seqno()) { return td::Status::Error("BlockProofLink contains an invalid Merkle proof for destination block "s + to.to_str()); } @@ -483,7 +483,7 @@ td::Status BlockProofLink::validate(td::uint32* save_utime) const { } if (!is_fwd) { // check a backward link - auto vstate_root = vm::MerkleProof::virtualize(state_proof, 1); + auto vstate_root = vm::MerkleProof::virtualize(state_proof); if (vstate_root.is_null()) { return td::Status::Error("backward BlockProofLink contains an invalid Merkle proof for source state "s + from.to_str()); diff --git a/crypto/test/test-db.cpp b/crypto/test/test-db.cpp index c70738c7b3..179767dfea 100644 --- a/crypto/test/test-db.cpp +++ b/crypto/test/test-db.cpp @@ -663,13 +663,13 @@ TEST(Cell, MerkleProof) { auto proof = MerkleProof::generate(cell, is_prunned); // CellBuilder::virtualize(proof, 1); //ASSERT_EQ(1u, proof->get_level()); - auto virtualized_proof = MerkleProof::virtualize(proof, 1); + auto virtualized_proof = MerkleProof::virtualize(proof); auto exploration3 = CellExplorer::explore(virtualized_proof, exploration.ops); ASSERT_EQ(exploration.log, exploration3.log); auto proof2 = MerkleProof::generate(cell, usage_tree.get()); CHECK(proof2->get_depth() == proof->get_depth()); - auto virtualized_proof2 = MerkleProof::virtualize(proof2, 1); + auto virtualized_proof2 = MerkleProof::virtualize(proof2); auto exploration4 = CellExplorer::explore(virtualized_proof2, exploration.ops); ASSERT_EQ(exploration.log, exploration4.log); } @@ -690,7 +690,7 @@ TEST(Cell, MerkleProofCombine) { CellExplorer::explore(usage_cell, exploration1.ops); proof1 = MerkleProof::generate(cell, usage_tree.get()); - auto virtualized_proof = MerkleProof::virtualize(proof1, 1); + auto virtualized_proof = MerkleProof::virtualize(proof1); auto exploration = CellExplorer::explore(virtualized_proof, exploration1.ops); ASSERT_EQ(exploration.log, exploration1.log); } @@ -702,7 +702,7 @@ TEST(Cell, MerkleProofCombine) { CellExplorer::explore(usage_cell, exploration2.ops); proof2 = MerkleProof::generate(cell, usage_tree.get()); - auto virtualized_proof = MerkleProof::virtualize(proof2, 1); + auto virtualized_proof = MerkleProof::virtualize(proof2); auto exploration = CellExplorer::explore(virtualized_proof, exploration2.ops); ASSERT_EQ(exploration.log, exploration2.log); } @@ -715,7 +715,7 @@ TEST(Cell, MerkleProofCombine) { CellExplorer::explore(usage_cell, exploration2.ops); proof12 = MerkleProof::generate(cell, usage_tree.get()); - auto virtualized_proof = MerkleProof::virtualize(proof12, 1); + auto virtualized_proof = MerkleProof::virtualize(proof12); auto exploration_a = CellExplorer::explore(virtualized_proof, exploration1.ops); auto exploration_b = CellExplorer::explore(virtualized_proof, exploration2.ops); ASSERT_EQ(exploration_a.log, exploration1.log); @@ -724,7 +724,7 @@ TEST(Cell, MerkleProofCombine) { { auto check = [&](auto proof_union) { - auto virtualized_proof = MerkleProof::virtualize(proof_union, 1); + auto virtualized_proof = MerkleProof::virtualize(proof_union); auto exploration_a = CellExplorer::explore(virtualized_proof, exploration1.ops); auto exploration_b = CellExplorer::explore(virtualized_proof, exploration2.ops); ASSERT_EQ(exploration_a.log, exploration1.log); @@ -738,14 +738,14 @@ TEST(Cell, MerkleProofCombine) { check(proof_union_fast); } { - cell = MerkleProof::virtualize(proof12, 1); + cell = MerkleProof::virtualize(proof12); auto usage_tree = std::make_shared(); auto usage_cell = UsageCell::create(cell, usage_tree->root_ptr()); CellExplorer::explore(usage_cell, exploration1.ops); auto proof = MerkleProof::generate(cell, usage_tree.get()); - auto virtualized_proof = MerkleProof::virtualize(proof, 2); + auto virtualized_proof = MerkleProof::virtualize(proof); auto exploration = CellExplorer::explore(virtualized_proof, exploration1.ops); ASSERT_EQ(exploration.log, exploration1.log); if (proof->get_hash() != proof1->get_hash()) { @@ -2177,28 +2177,28 @@ TEST(Cell, MerkleProofHands) { test_boc_deserializer_full(merkle_proof).ensure(); { - auto virtual_node = proof->virtualize({0, 1}); + auto virtual_node = proof->virtualize(0); ASSERT_EQ(0u, virtual_node->get_level()); - ASSERT_EQ(1u, virtual_node->get_virtualization()); + ASSERT_EQ(1u, virtual_node->is_virtualized()); CellSlice cs{NoVm(), virtual_node}; auto virtual_data = cs.fetch_ref(); ASSERT_EQ(0u, virtual_data->get_level()); - ASSERT_EQ(1u, virtual_data->get_virtualization()); + ASSERT_EQ(1u, virtual_data->is_virtualized()); ASSERT_EQ(data->get_hash(), virtual_data->get_hash()); auto virtual_node_copy = CellBuilder{}.store_bits(node->get_data(), node->get_bits()).store_ref(virtual_data).finalize(); ASSERT_EQ(0u, virtual_node_copy->get_level()); - ASSERT_EQ(1u, virtual_node_copy->get_virtualization()); + ASSERT_EQ(1u, virtual_node_copy->is_virtualized()); ASSERT_EQ(virtual_node->get_hash(), virtual_node_copy->get_hash()); { auto two_nodes = CellBuilder{}.store_ref(virtual_node).store_ref(node).finalize(); ASSERT_EQ(0u, two_nodes->get_level()); - ASSERT_EQ(1u, two_nodes->get_virtualization()); + ASSERT_EQ(1u, two_nodes->is_virtualized()); CellSlice cs2(NoVm(), two_nodes); - ASSERT_EQ(1u, cs2.prefetch_ref(0)->get_virtualization()); - ASSERT_EQ(0u, cs2.prefetch_ref(1)->get_virtualization()); + ASSERT_EQ(1u, cs2.prefetch_ref(0)->is_virtualized()); + ASSERT_EQ(0u, cs2.prefetch_ref(1)->is_virtualized()); } } LOG(ERROR) << td::NamedThreadSafeCounter::get_default(); @@ -2220,7 +2220,7 @@ TEST(Cell, MerkleProofArrayHands) { ASSERT_TRUE(proof->get_hash(1) != arr.root()->get_hash(1)); ASSERT_EQ(arr.root()->get_hash(0), arr.root()->get_hash(1)); - CompactArray new_arr(arr.size(), proof->virtualize({0, 1})); + CompactArray new_arr(arr.size(), proof->virtualize(0)); for (auto k : keys) { ASSERT_EQ(arr.get(k), new_arr.get(k)); } @@ -2247,7 +2247,7 @@ TEST(Cell, MerkleProofCombineArray) { } } - CompactArray arr2(n, vm::MerkleProof::virtualize(root, 1)); + CompactArray arr2(n, vm::MerkleProof::virtualize(root)); for (size_t i = 0; i < n; i++) { CHECK(arr.get(i) == arr2.get(i)); } @@ -2945,7 +2945,7 @@ TEST(TonDb, BocRespectsUsageCell) { auto usage_cell = vm::UsageCell::create(cell, usage_tree->root_ptr()); auto serialization = serialize_boc(usage_cell); auto proof = vm::MerkleProof::generate(cell, usage_tree.get()); - auto virtualized_proof = vm::MerkleProof::virtualize(proof, 1); + auto virtualized_proof = vm::MerkleProof::virtualize(proof); auto serialization_of_virtualized_cell = serialize_boc(virtualized_proof); ASSERT_STREQ(serialization, serialization_of_virtualized_cell); } @@ -2971,7 +2971,7 @@ TEST(UsageTree, ThreadSafe) { thread.join(); } auto proof = vm::MerkleProof::generate(cell, usage_tree.get()); - auto virtualized_proof = vm::MerkleProof::virtualize(proof, 1); + auto virtualized_proof = vm::MerkleProof::virtualize(proof); for (auto &exploration : explorations) { auto new_exploration = vm::CellExplorer::explore(virtualized_proof, exploration.ops); ASSERT_EQ(exploration.log, new_exploration.log); @@ -3051,8 +3051,8 @@ TEST(TonDb, DoNotMakeListsPrunned) { auto cell = vm::CellBuilder().store_bytes("abc").finalize(); auto is_prunned = [&](const td::Ref &cell) { return true; }; auto proof = vm::MerkleProof::generate(cell, is_prunned); - auto virtualized_proof = vm::MerkleProof::virtualize(proof, 1); - ASSERT_TRUE(virtualized_proof->get_virtualization() == 0); + auto virtualized_proof = vm::MerkleProof::virtualize(proof); + ASSERT_TRUE(!virtualized_proof->is_virtualized()); } TEST(TonDb, CellStat) { diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 29c6c50c2d..974411b6cf 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -163,7 +163,7 @@ int BagOfCells::add_root(td::Ref add_root) { if (add_root.is_null()) { return 0; } - LOG_CHECK(add_root->get_virtualization() == 0) << "TODO: support serialization of virtualized cells"; + LOG_CHECK(!add_root->is_virtualized()) << "TODO: support serialization of virtualized cells"; //const Cell::Hash& hash = add_root->get_hash(); //for (const auto& root_info : roots) { //if (root_info.cell->get_hash() == hash) { @@ -217,7 +217,7 @@ td::Result BagOfCells::import_cell(td::Ref cell, int depth) { cell_list_[pos].should_cache = true; return pos; } - if (cell->get_virtualization() != 0) { + if (cell->is_virtualized()) { return td::Status::Error( "error while importing a cell into a bag of cells: cell has non-zero virtualization level"); } diff --git a/crypto/vm/cells/Cell.cpp b/crypto/vm/cells/Cell.cpp index 1c20ba41af..9bed3481d0 100644 --- a/crypto/vm/cells/Cell.cpp +++ b/crypto/vm/cells/Cell.cpp @@ -48,8 +48,8 @@ td::Status Cell::check_equals_unloaded(const Ref& other) const { return td::Status::OK(); } -Ref Cell::virtualize(VirtualizationParameters virt) const { - return VirtualCell::create(virt, Ref(this)); +Ref Cell::virtualize(td::uint32 effective_level) const { + return VirtualCell::create(effective_level, Ref(this)); } std::ostream& operator<<(std::ostream& os, const Cell& c) { diff --git a/crypto/vm/cells/Cell.h b/crypto/vm/cells/Cell.h index 25b71fcb24..5f0d4d14d3 100644 --- a/crypto/vm/cells/Cell.h +++ b/crypto/vm/cells/Cell.h @@ -25,7 +25,6 @@ #include "vm/cells/CellTraits.h" #include "vm/cells/CellUsageTree.h" #include "vm/cells/LevelMask.h" -#include "vm/cells/VirtualizationParameters.h" #include "td/utils/Status.h" @@ -37,14 +36,13 @@ class DataCell; struct LoadedCell { Ref data_cell; - detail::VirtualizationParameters virt; + td::uint32 effective_level; CellUsageTree::NodePtr tree_node; // TODO: inline_vector? }; class Cell : public CellTraits { public: using LevelMask = detail::LevelMask; - using VirtualizationParameters = detail::VirtualizationParameters; using LoadedCell = vm::LoadedCell; using Hash = CellHash; @@ -59,8 +57,9 @@ class Cell : public CellTraits { // load interface virtual td::Status set_data_cell(Ref &&data_cell) const = 0; virtual td::Result load_cell() const = 0; - virtual Ref virtualize(VirtualizationParameters virt) const; - virtual td::uint32 get_virtualization() const = 0; + virtual Ref virtualize(td::uint32 effective_level) const; + // Cell is virtualized if its effective level is less than its actual level. + virtual bool is_virtualized() const = 0; virtual CellUsageTree::NodePtr get_tree_node() const = 0; virtual bool is_loaded() const = 0; diff --git a/crypto/vm/cells/CellBuilder.cpp b/crypto/vm/cells/CellBuilder.cpp index a9ad449e1c..f99cd86d99 100644 --- a/crypto/vm/cells/CellBuilder.cpp +++ b/crypto/vm/cells/CellBuilder.cpp @@ -100,7 +100,7 @@ Ref CellBuilder::finalize(bool special) { } Ref CellBuilder::create_pruned_branch(Ref cell, td::uint32 new_level, td::uint32 virt_level) { - if (cell->is_loaded() && cell->get_level() <= virt_level && cell->get_virtualization() == 0) { + if (cell->is_loaded() && cell->get_level() <= virt_level && !cell->is_virtualized()) { CellSlice cs(NoVm{}, cell); if (cs.size_refs() == 0) { return cell; diff --git a/crypto/vm/cells/CellSlice.cpp b/crypto/vm/cells/CellSlice.cpp index bea20f95d1..8bebb41b38 100644 --- a/crypto/vm/cells/CellSlice.cpp +++ b/crypto/vm/cells/CellSlice.cpp @@ -30,7 +30,7 @@ CellSlice::CellSlice(Ref&& ref) : cell(std::move(ref)), bits_st(0), refs_s */ CellSlice::CellSlice(VirtualCell::LoadedCell loaded_cell) - : virt(loaded_cell.virt) + : effective_level(loaded_cell.effective_level) , cell(std::move(loaded_cell.data_cell)) , tree_node(std::move(loaded_cell.tree_node)) , bits_st(0) @@ -79,7 +79,7 @@ CellSlice::CellSlice(Ref ref) : CellSlice(VirtualCell::LoadedCell{std: CellSlice::CellSlice(const CellSlice& cs) = default; bool CellSlice::load(VirtualCell::LoadedCell loaded_cell) { - virt = loaded_cell.virt; + effective_level = loaded_cell.effective_level; cell = std::move(loaded_cell.data_cell); tree_node = std::move(loaded_cell.tree_node); bits_st = 0; @@ -125,7 +125,7 @@ CellSlice::CellSlice(Ref dc_ref, unsigned _bits_en, unsigned _refs_en, */ CellSlice::CellSlice(const CellSlice& cs, unsigned _bits_en, unsigned _refs_en, unsigned _bits_st, unsigned _refs_st) - : virt(cs.virt) + : effective_level(cs.effective_level) , cell(cs.cell) , tree_node(cs.tree_node) , bits_st(cs.bits_st + _bits_st) @@ -142,7 +142,7 @@ CellSlice::CellSlice(const CellSlice& cs, unsigned _bits_en, unsigned _refs_en, } CellSlice::CellSlice(const CellSlice& cs, unsigned _bits_en, unsigned _refs_en) - : virt(cs.virt) + : effective_level(cs.effective_level) , cell(cs.cell) , tree_node(cs.tree_node) , bits_st(cs.bits_st) @@ -158,7 +158,7 @@ CellSlice::CellSlice(const CellSlice& cs, unsigned _bits_en, unsigned _refs_en) } Cell::LoadedCell CellSlice::move_as_loaded_cell() { - Cell::LoadedCell res{std::move(cell), std::move(virt), std::move(tree_node)}; + Cell::LoadedCell res{std::move(cell), effective_level, std::move(tree_node)}; clear(); return res; } @@ -203,13 +203,13 @@ void CellSlice::error() { */ unsigned CellSlice::get_cell_level() const { - return cell->get_level_mask().apply(virt.get_level()).get_level(); + return cell->get_level_mask().apply(effective_level).get_level(); } unsigned CellSlice::get_level() const { unsigned l = 0; for (unsigned i = refs_st; i < refs_en; i++) { - auto res = cell->get_ref(i)->virtualize(child_virt()); + auto res = cell->get_ref(i)->virtualize(child_effective_level()); unsigned l1 = res->get_level(); // maybe l1 = cell->get_ref(i)->get_level_mask().apply(virt.get_level()).get_level(); if (l1 > l) { @@ -223,7 +223,7 @@ Ref CellSlice::get_base_cell() const { if (cell.is_null()) { return {}; } - auto res = cell->virtualize(virt); + auto res = cell->virtualize(effective_level); if (!tree_node.empty()) { res = UsageCell::create(std::move(res), tree_node); } @@ -740,7 +740,7 @@ bool CellSlice::prefetch_bytes(td::MutableSlice slice) const { Ref CellSlice::prefetch_ref(unsigned offset) const { if (offset < size_refs()) { auto ref_id = refs_st + offset; - auto res = cell->get_ref(ref_id)->virtualize(child_virt()); + auto res = cell->get_ref(ref_id)->virtualize(child_effective_level()); if (!tree_node.empty()) { res = UsageCell::create(std::move(res), tree_node.create_child(ref_id)); } @@ -753,7 +753,7 @@ Ref CellSlice::prefetch_ref(unsigned offset) const { Ref CellSlice::fetch_ref() { if (have_refs()) { auto ref_id = refs_st++; - auto res = cell->get_ref(ref_id)->virtualize(child_virt()); + auto res = cell->get_ref(ref_id)->virtualize(child_effective_level()); if (!tree_node.empty()) { res = UsageCell::create(std::move(res), tree_node.create_child(ref_id)); } @@ -1083,9 +1083,8 @@ VirtualCell::LoadedCell load_cell_slice_impl(Ref cell, bool* can_be_specia } auto loaded_cell = r_loaded_cell.move_as_ok(); if (loaded_cell.data_cell->special_type() == DataCell::SpecialType::PrunnedBranch) { - auto virtualization = loaded_cell.virt.get_virtualization(); - if (virtualization != 0) { - throw VmVirtError{virtualization}; + if (loaded_cell.effective_level < loaded_cell.data_cell->get_level()) { + throw VmVirtError{1}; } } if (can_be_special) { @@ -1111,7 +1110,7 @@ VirtualCell::LoadedCell load_cell_slice_impl(Ref cell, bool* can_be_specia } throw VmError{Excno::cell_und, "failed to load library cell (no vm_state_interface available)"}; } else if (loaded_cell.data_cell->special_type() == DataCell::SpecialType::PrunnedBranch) { - CHECK(loaded_cell.virt.get_virtualization() == 0); + CHECK(loaded_cell.effective_level >= loaded_cell.data_cell->get_level()); throw VmError{Excno::cell_und, "trying to load prunned cell"}; } throw VmError{Excno::cell_und, "unexpected special cell"}; diff --git a/crypto/vm/cells/CellSlice.h b/crypto/vm/cells/CellSlice.h index 7525272b59..7d4a5fb650 100644 --- a/crypto/vm/cells/CellSlice.h +++ b/crypto/vm/cells/CellSlice.h @@ -32,7 +32,7 @@ struct NoVmOrd {}; struct NoVmSpec {}; class CellSlice : public td::CntObject { - Cell::VirtualizationParameters virt; + td::uint32 effective_level; Ref cell; CellUsageTree::NodePtr tree_node; unsigned bits_st, refs_st; @@ -84,9 +84,6 @@ class CellSlice : public td::CntObject { return cell->special_type(); } int child_merkle_depth(int merkle_depth) const { - if (merkle_depth == Cell::VirtualizationParameters::max_level()) { - return merkle_depth; - } if (cell->special_type() == Cell::SpecialType::MerkleProof || cell->special_type() == Cell::SpecialType::MerkleUpdate) { merkle_depth++; @@ -289,9 +286,8 @@ class CellSlice : public td::CntObject { void init_bits_refs(); void init_preload() const; void preload_at_least(unsigned req_bits) const; - Cell::VirtualizationParameters child_virt() const { - return Cell::VirtualizationParameters(static_cast(child_merkle_depth(virt.get_level())), - virt.get_virtualization()); + td::uint32 child_effective_level() const { + return child_merkle_depth(effective_level); } }; diff --git a/crypto/vm/cells/DataCell.cpp b/crypto/vm/cells/DataCell.cpp index b6ff8021f3..f4da28b8f5 100644 --- a/crypto/vm/cells/DataCell.cpp +++ b/crypto/vm/cells/DataCell.cpp @@ -71,16 +71,13 @@ class CellChecker { return td::Status::Error("Invalid special cell type"); } - // Afterwards, we do some common checks and compute virtualization level. + // Afterwards, we validate depth array and figure out if this cell should be virtualized. if (*std::max_element(depth_.begin(), depth_.end()) > CellTraits::max_depth) { return td::Status::Error("Depth is too big"); } for (int i = 0; i < refs_cnt_; ++i) { - virtualization_ = std::max(virtualization_, refs_[i]->get_virtualization()); - } - if (virtualization_ > std::numeric_limits::max()) { - return td::Status::Error("Virtualization is too big to be stored in vm::DataCell"); + virtualized_ |= refs_[i]->is_virtualized(); } // And finally, we compute cell hashes. @@ -111,8 +108,8 @@ class CellChecker { return level_mask_; } - td::uint8 virtualization() const { - return static_cast(virtualization_); + bool virtualized() const { + return virtualized_; } std::array const& depths() const { @@ -314,7 +311,7 @@ class CellChecker { int bit_length_; Cell::LevelMask level_mask_; - td::uint32 virtualization_{0}; + bool virtualized_{false}; std::array depth_{}; std::array hash_{}; }; @@ -353,7 +350,7 @@ td::Result> DataCell::create(td::Slice data, int bit_length, td::S void* storage = use_arena ? allocate_in_arena(cell_size) : ::operator new(cell_size); DataCell* allocated_cell = new (storage) - DataCell{bit_length, refs.size(), checker.type(), checker.level_mask(), use_arena, checker.virtualization()}; + DataCell{bit_length, refs.size(), checker.type(), checker.level_mask(), use_arena, checker.virtualized()}; auto& cell = *allocated_cell; auto mutable_data = cell.trailer_ + level_info_size; @@ -443,14 +440,14 @@ std::string DataCell::to_hex() const { } DataCell::DataCell(int bit_length, size_t refs_cnt, Cell::SpecialType type, LevelMask level_mask, - bool allocated_in_arena, td::uint8 virtualization) + bool allocated_in_arena, bool virtualized) : bit_length_(bit_length) , refs_cnt_(static_cast(refs_cnt)) , type_(static_cast(type)) , level_(static_cast(level_mask.get_level())) , level_mask_(level_mask.get_mask()) , allocated_in_arena_(allocated_in_arena) - , virtualization_(virtualization) { + , virtualized_(virtualized) { get_thread_safe_counter().add(1); } diff --git a/crypto/vm/cells/DataCell.h b/crypto/vm/cells/DataCell.h index 172b3390a7..9e2e52b0e3 100644 --- a/crypto/vm/cells/DataCell.h +++ b/crypto/vm/cells/DataCell.h @@ -69,13 +69,13 @@ class DataCell final : public Cell { virtual td::Result load_cell() const override { return LoadedCell{ .data_cell = Ref{this}, - .virt = {}, + .effective_level = level_, .tree_node = {}, }; } - virtual td::uint32 get_virtualization() const override { - return virtualization_; + virtual bool is_virtualized() const override { + return virtualized_; } virtual CellUsageTree::NodePtr get_tree_node() const override { @@ -169,7 +169,7 @@ class DataCell final : public Cell { } DataCell(int bit_length, size_t refs_cnt, Cell::SpecialType type, LevelMask level_mask, bool allocated_in_arena, - td::uint8 virtualization); + bool virtualized); detail::LevelInfo const* level_info() const { return reinterpret_cast(trailer_); @@ -197,7 +197,7 @@ class DataCell final : public Cell { unsigned level_ : 3; unsigned level_mask_ : 3; unsigned allocated_in_arena_ : 1; - unsigned virtualization_ : 8; + unsigned virtualized_ : 1; std::array, max_refs> refs_{}; diff --git a/crypto/vm/cells/ExtCell.h b/crypto/vm/cells/ExtCell.h index dbbd8575b7..e2188fd7a5 100644 --- a/crypto/vm/cells/ExtCell.h +++ b/crypto/vm/cells/ExtCell.h @@ -54,10 +54,11 @@ class ExtCell : public Cell { td::Result load_cell() const override { TRY_RESULT(data_cell, load_data_cell()); - return LoadedCell{std::move(data_cell), {}, {}}; + auto level = data_cell->get_level(); + return LoadedCell{std::move(data_cell), level, {}}; } - td::uint32 get_virtualization() const override { - return 0; + bool is_virtualized() const override { + return false; } CellUsageTree::NodePtr get_tree_node() const override { return {}; diff --git a/crypto/vm/cells/MerkleProof.cpp b/crypto/vm/cells/MerkleProof.cpp index 4c1fcbdb7c..58f617a27f 100644 --- a/crypto/vm/cells/MerkleProof.cpp +++ b/crypto/vm/cells/MerkleProof.cpp @@ -107,8 +107,8 @@ Ref MerkleProof::generate_raw(Ref cell, CellUsageTree *usage_tree) { return detail::MerkleProofImpl(usage_tree).create_from(cell); } -Ref MerkleProof::virtualize_raw(Ref cell, Cell::VirtualizationParameters virt) { - return cell->virtualize(virt); +Ref MerkleProof::virtualize_raw(Ref cell, td::uint32 effective_level) { + return cell->virtualize(effective_level); } Ref MerkleProof::generate(Ref cell, IsPrunnedFunction is_prunned) { @@ -145,17 +145,17 @@ td::Result> unpack_proof(Ref cell) { return cs.fetch_ref(); } -Ref MerkleProof::virtualize(Ref cell, int virtualization) { +Ref MerkleProof::virtualize(Ref cell) { auto r_raw = unpack_proof(std::move(cell)); if (r_raw.is_error()) { return {}; } - return virtualize_raw(r_raw.move_as_ok(), {0 /*level*/, static_cast(virtualization)}); + return virtualize_raw(r_raw.move_as_ok(), 0); } -td::Result> MerkleProof::try_virtualize(Ref cell, int virtualization) { +td::Result> MerkleProof::try_virtualize(Ref cell) { TRY_RESULT(unpacked_cell, unpack_proof(std::move(cell))); - return unpacked_cell->virtualize({0 /*level*/, static_cast(virtualization)}); + return unpacked_cell->virtualize(0); } class MerkleProofCombineFast { diff --git a/crypto/vm/cells/MerkleProof.h b/crypto/vm/cells/MerkleProof.h index 68f0be7b97..b9fcfcf3a7 100644 --- a/crypto/vm/cells/MerkleProof.h +++ b/crypto/vm/cells/MerkleProof.h @@ -35,8 +35,8 @@ class MerkleProof { static Ref generate(Ref cell, CellUsageTree *usage_tree); // cell must have zero level and must be a MerkleProof - static Ref virtualize(Ref cell, int virtualization); - static td::Result> try_virtualize(Ref cell, int virtualization = 1); + static Ref virtualize(Ref cell); + static td::Result> try_virtualize(Ref cell); static Ref combine(Ref a, Ref b); static td::Result> combine_status(Ref a, Ref b); @@ -47,7 +47,7 @@ class MerkleProof { // works fine with cell of non-zero level, but this is not supported (yet?) in MerkeProof special cell static Ref generate_raw(Ref cell, IsPrunnedFunction is_prunned); static Ref generate_raw(Ref cell, CellUsageTree *usage_tree); - static Ref virtualize_raw(Ref cell, Cell::VirtualizationParameters virt); + static Ref virtualize_raw(Ref cell, td::uint32 effective_level); static Ref combine_raw(Ref a, Ref b); static Ref combine_fast_raw(Ref a, Ref b); }; diff --git a/crypto/vm/cells/PrunnedCell.h b/crypto/vm/cells/PrunnedCell.h index f4d40c2853..d25e3bc012 100644 --- a/crypto/vm/cells/PrunnedCell.h +++ b/crypto/vm/cells/PrunnedCell.h @@ -129,8 +129,8 @@ class PrunnedCell final : public Cell { explicit PrunnedCell(Info info, ExtraT&& extra) : info_(info), extra_(std::move(extra)) { } - td::uint32 get_virtualization() const override { - return 0; + virtual bool is_virtualized() const override { + return false; } CellUsageTree::NodePtr get_tree_node() const override { return {}; diff --git a/crypto/vm/cells/UsageCell.h b/crypto/vm/cells/UsageCell.h index c634bc9090..824d383e6c 100644 --- a/crypto/vm/cells/UsageCell.h +++ b/crypto/vm/cells/UsageCell.h @@ -48,8 +48,8 @@ class UsageCell : public Cell { } return std::move(loaded_cell); } - Ref virtualize(VirtualizationParameters virt) const override { - auto virtualized_cell = cell_->virtualize(virt); + Ref virtualize(td::uint32 effective_level) const override { + auto virtualized_cell = cell_->virtualize(effective_level); if (tree_node_.empty()) { return virtualized_cell; } @@ -59,8 +59,8 @@ class UsageCell : public Cell { return create(std::move(virtualized_cell), tree_node_); } - td::uint32 get_virtualization() const override { - return cell_->get_virtualization(); + bool is_virtualized() const override { + return cell_->is_virtualized(); } CellUsageTree::NodePtr get_tree_node() const override { diff --git a/crypto/vm/cells/VirtualCell.h b/crypto/vm/cells/VirtualCell.h index a75bdf9de3..b89df3eb6b 100644 --- a/crypto/vm/cells/VirtualCell.h +++ b/crypto/vm/cells/VirtualCell.h @@ -25,15 +25,15 @@ class VirtualCell : public Cell { struct PrivateTag {}; public: - static Ref create(VirtualizationParameters virt, Ref cell) { - if (cell->get_level() <= virt.get_level()) { + static Ref create(td::uint32 effective_level, Ref cell) { + if (cell->get_level() <= effective_level) { return cell; } - return Ref{true, virt, std::move(cell), PrivateTag{}}; + return Ref{true, effective_level, std::move(cell), PrivateTag{}}; } - VirtualCell(VirtualizationParameters virt, Ref cell, PrivateTag) : virt_(virt), cell_(std::move(cell)) { - CHECK(cell_->get_virtualization() <= virt_.get_virtualization()); + VirtualCell(td::uint32 effective_level, Ref cell, PrivateTag) + : effective_level_(effective_level), cell_(std::move(cell)) { } // load interface @@ -42,20 +42,19 @@ class VirtualCell : public Cell { } td::Result load_cell() const override { TRY_RESULT(loaded_cell, cell_->load_cell()); - loaded_cell.virt = loaded_cell.virt.apply(virt_); + loaded_cell.effective_level = std::min(loaded_cell.effective_level, effective_level_); return std::move(loaded_cell); } - Ref virtualize(VirtualizationParameters virt) const override { - auto new_virt = virt_.apply(virt); - if (new_virt == virt_) { + Ref virtualize(td::uint32 new_effective_level) const override { + if (effective_level_ <= new_effective_level) { return Ref(this); } - return create(new_virt, cell_); + return create(new_effective_level, cell_); } - td::uint32 get_virtualization() const override { - return virt_.get_virtualization(); + bool is_virtualized() const override { + return true; } CellUsageTree::NodePtr get_tree_node() const override { @@ -68,23 +67,19 @@ class VirtualCell : public Cell { // hash and level LevelMask get_level_mask() const override { - return cell_->get_level_mask().apply(virt_.get_level()); + return cell_->get_level_mask().apply(effective_level_); } protected: const Hash do_get_hash(td::uint32 level) const override { - return cell_->get_hash(fix_level(level)); + return cell_->get_hash(std::min(effective_level_, level)); } td::uint16 do_get_depth(td::uint32 level) const override { - return cell_->get_depth(fix_level(level)); + return cell_->get_depth(std::min(effective_level_, level)); } private: - VirtualizationParameters virt_; + td::uint32 effective_level_; Ref cell_; - - int fix_level(int level) const { - return get_level_mask().apply(level).get_level(); - } }; } // namespace vm diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index 697e4d67f8..1b91bd2976 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -217,7 +217,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat if (cell.is_null()) { return; } - if (cell->get_virtualization() != 0) { + if (cell->is_virtualized()) { return; } to_inc_.push_back(cell); @@ -226,7 +226,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat if (cell.is_null()) { return; } - if (cell->get_virtualization() != 0) { + if (cell->is_virtualized()) { return; } to_dec_.push_back(cell); diff --git a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp index 3b7d7c2336..ae06beb1b5 100644 --- a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp @@ -844,7 +844,7 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { if (cell.is_null()) { return; } - if (cell->get_virtualization() != 0) { + if (cell->is_virtualized()) { return; } to_inc_.push_back(cell); @@ -853,7 +853,7 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { if (cell.is_null()) { return; } - if (cell->get_virtualization() != 0) { + if (cell->is_virtualized()) { return; } to_dec_.push_back(cell); diff --git a/crypto/vm/db/InMemoryBagOfCellsDb.cpp b/crypto/vm/db/InMemoryBagOfCellsDb.cpp index 829ed38d8b..a41edb6f48 100644 --- a/crypto/vm/db/InMemoryBagOfCellsDb.cpp +++ b/crypto/vm/db/InMemoryBagOfCellsDb.cpp @@ -860,7 +860,7 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { if (cell.is_null()) { return; } - if (cell->get_virtualization() != 0) { + if (cell->is_virtualized()) { return; } to_inc_.push_back(cell); @@ -870,7 +870,7 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { if (cell.is_null()) { return; } - if (cell->get_virtualization() != 0) { + if (cell->is_virtualized()) { return; } to_dec_.push_back(cell); diff --git a/crypto/vm/db/StaticBagOfCellsDb.cpp b/crypto/vm/db/StaticBagOfCellsDb.cpp index 215dcfa3bc..0279ac77fa 100644 --- a/crypto/vm/db/StaticBagOfCellsDb.cpp +++ b/crypto/vm/db/StaticBagOfCellsDb.cpp @@ -45,11 +45,11 @@ class RootCell : public Cell { td::Result load_cell() const override { return cell_->load_cell(); } - Ref virtualize(VirtualizationParameters virt) const override { - return cell_->virtualize(virt); + Ref virtualize(td::uint32 effective_level) const override { + return cell_->virtualize(effective_level); } - td::uint32 get_virtualization() const override { - return cell_->get_virtualization(); + bool is_virtualized() const override { + return cell_->is_virtualized(); } CellUsageTree::NodePtr get_tree_node() const override { return cell_->get_tree_node(); diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp index ec7eee298d..786fe62e94 100644 --- a/crypto/vm/large-boc-serializer.cpp +++ b/crypto/vm/large-boc-serializer.cpp @@ -152,7 +152,7 @@ td::Result LargeBocSerializer::import_cell(Hash root_hash, int root_depth) for (size_t i = 0; i < loaded_results.size(); ++i) { auto& cell = loaded_results[i]; - if (cell->get_virtualization() != 0) { + if (cell->is_virtualized()) { return td::Status::Error( "error while importing a cell into a bag of cells: cell has non-zero virtualization level"); } diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 9e260dea09..659c71c0c4 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -2416,7 +2416,7 @@ void TestNode::got_one_transaction(ton::BlockIdExt req_blkid, ton::BlockIdExt bl } auto proof_root = P.move_as_ok(); try { - auto block_root = vm::MerkleProof::virtualize(std::move(proof_root), 1); + auto block_root = vm::MerkleProof::virtualize(std::move(proof_root)); if (block_root.is_null()) { LOG(ERROR) << "transaction block proof is invalid"; return; @@ -2866,7 +2866,7 @@ void TestNode::got_config_params(ton::BlockIdExt req_blkid, int mode, std::strin block::check_extract_state_proof(blkid, f->state_proof_.as_slice(), f->config_proof_.as_slice()), PSLICE() << "masterchain state proof for " << blkid.to_str() << " is invalid :"); } else { - block = vm::MerkleProof::virtualize(config_proof, 1); + block = vm::MerkleProof::virtualize(config_proof); if (block.is_null()) { promise.set_error( td::Status::Error("cannot virtualize configuration proof constructed from key block "s + blkid.to_str())); @@ -3252,7 +3252,7 @@ void TestNode::got_block_header_raw(td::BufferSlice res, td::Promiseget_hash().bits(); - TRY_STATUS(block::check_block_header_proof(vm::MerkleProof::virtualize(state_proof, 1), blk_id, &state_hash, true, + TRY_STATUS(block::check_block_header_proof(vm::MerkleProof::virtualize(state_proof), blk_id, &state_hash, true, save_utime, save_lt)); return td::Status::OK(); } @@ -4276,7 +4276,7 @@ static bool visit(Ref cs_ref) { td::Result> TestNode::ValidatorLoadInfo::build_proof(int idx, td::Bits256* save_pubkey) const { try { - auto state_virt_root = vm::MerkleProof::virtualize(std::move(data_proof), 1); + auto state_virt_root = vm::MerkleProof::virtualize(std::move(data_proof)); if (state_virt_root.is_null()) { return td::Status::Error("account state proof is invalid"); } @@ -4607,7 +4607,7 @@ td::Status TestNode::ValidatorLoadInfo::init_check_proofs() { if (lt != end_lt) { return td::Status::Error(PSLICE() << "incorrect block logical time: declared " << end_lt << ", actual " << lt); } - auto vstate = vm::MerkleProof::virtualize(data_proof, 1); + auto vstate = vm::MerkleProof::virtualize(data_proof); if (vstate.is_null()) { return td::Status::Error(PSLICE() << "cannot virtualize state of block " << blk_id.to_str()); } diff --git a/storage/MerkleTree.cpp b/storage/MerkleTree.cpp index 2ba4aeb867..d8767ffce9 100644 --- a/storage/MerkleTree.cpp +++ b/storage/MerkleTree.cpp @@ -174,7 +174,7 @@ td::Result> MerkleTree::gen_proof(size_t l, size_t r) const { return td::Status::Error("Got no proofs yet"); } auto usage_tree = std::make_shared(); - auto root_raw = vm::MerkleProof::virtualize(root_proof_, 1); + auto root_raw = vm::MerkleProof::virtualize(root_proof_); auto usage_cell = vm::UsageCell::create(root_raw, usage_tree->root_ptr()); TRY_STATUS(TRY_VM(do_gen_proof(std::move(usage_cell), 0, n_ - 1, l, r))); auto res = vm::MerkleProof::generate(root_raw, usage_tree.get()); @@ -202,7 +202,7 @@ td::Ref MerkleTree::get_root(size_t depth_limit) const { return root_proof_; } auto usage_tree = std::make_shared(); - auto root_raw = vm::MerkleProof::virtualize(root_proof_, 1); + auto root_raw = vm::MerkleProof::virtualize(root_proof_); auto usage_cell = vm::UsageCell::create(root_raw, usage_tree->root_ptr()); do_gen_proof(std::move(usage_cell), unpack_proof(root_proof_).move_as_ok(), depth_limit); auto res = vm::MerkleProof::generate(root_raw, usage_tree.get()); diff --git a/storage/MicrochunkTree.cpp b/storage/MicrochunkTree.cpp index 1c73593363..689c7aff7e 100644 --- a/storage/MicrochunkTree.cpp +++ b/storage/MicrochunkTree.cpp @@ -91,7 +91,7 @@ void MicrochunkTree::Builder::add_microchunk(td::Slice s) { } MicrochunkTree::MicrochunkTree(td::Ref root_proof) : root_proof_(root_proof) { - td::Ref virt_root = vm::MerkleProof::virtualize(root_proof_, 1); + td::Ref virt_root = vm::MerkleProof::virtualize(root_proof_); CHECK(!virt_root.is_null()); CHECK(virt_root->get_depth() <= 50); total_size_ = MICROCHUNK_SIZE << virt_root->get_depth(); diff --git a/storage/TorrentMeta.cpp b/storage/TorrentMeta.cpp index 1a8fdd8975..3e5242b55c 100644 --- a/storage/TorrentMeta.cpp +++ b/storage/TorrentMeta.cpp @@ -42,7 +42,7 @@ td::Result TorrentMeta::deserialize(td::Slice data) { } } if (res.root_proof.not_null()) { - auto root = vm::MerkleProof::virtualize(res.root_proof, 1); + auto root = vm::MerkleProof::virtualize(res.root_proof); if (root.is_null()) { return td::Status::Error("Root proof is not a merkle proof"); } diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 8845583ae8..9eafe9076d 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -1685,7 +1685,7 @@ class GetShardBlockProof : public td::actor::Actor { abort(TonlibError::InvalidBagOfCells("proof")); return; } - auto block_root = vm::MerkleProof::virtualize(R.move_as_ok(), 1); + auto block_root = vm::MerkleProof::virtualize(R.move_as_ok()); if (cur_id.root_hash != block_root->get_hash().bits()) { abort(td::Status::Error("invalid block hash in proof")); return; @@ -1845,13 +1845,13 @@ class GetOutMsgQueueSizes : public td::actor::Actor { if (roots.size() != 2) { return td::Status::Error("expected 2 roots in proof"); } - auto state_root = vm::MerkleProof::virtualize(std::move(roots[1]), 1); + auto state_root = vm::MerkleProof::virtualize(std::move(roots[1])); if (state_root.is_null()) { return td::Status::Error("state proof is invalid"); } ton::Bits256 state_hash = state_root->get_hash().bits(); - TRY_STATUS_PREFIX(block::check_block_header_proof(vm::MerkleProof::virtualize(std::move(roots[0]), 1), - blocks_[i], &state_hash, true, nullptr, nullptr), + TRY_STATUS_PREFIX(block::check_block_header_proof(vm::MerkleProof::virtualize(std::move(roots[0])), blocks_[i], + &state_hash, true, nullptr, nullptr), "error in block header proof: "); block::gen::ShardStateUnsplit::Record sstate; @@ -1977,7 +1977,7 @@ class RunEmulator : public TonlibQueryActor { TRY_RESULT(root, vm::std_boc_deserialize(std::move(header->header_proof_))); try { - auto virt_root = vm::MerkleProof::virtualize(root, 1); + auto virt_root = vm::MerkleProof::virtualize(root); if (virt_root.is_null()) { return td::Status::Error("block header proof is not a valid Merkle proof"); } @@ -5767,7 +5767,7 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getShards& request, return data_cell.move_as_error_prefix("Couldn't deserialize shards data: "); } try { - auto virt_root = vm::MerkleProof::virtualize(proof_cell.move_as_ok(), 1); + auto virt_root = vm::MerkleProof::virtualize(proof_cell.move_as_ok()); if (virt_root.is_null()) { return td::Status::Error("Virt root is null"); } @@ -5869,7 +5869,7 @@ td::Status check_lookup_block_proof(lite_api_ptrget_hash().bits()) { return td::Status::Error("invalid block hash in proof"); } @@ -5913,7 +5913,7 @@ td::Status check_lookup_block_proof(lite_api_ptrproof_))); - auto virt_root = vm::MerkleProof::virtualize(proof_cell, 1); + auto virt_root = vm::MerkleProof::virtualize(proof_cell); if (root_hash != virt_root->get_hash().bits()) { return td::Status::Error("Invalid block proof root hash"); @@ -6216,7 +6216,7 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getBlockHeader& req } else { auto root = R.move_as_ok(); try { - auto virt_root = vm::MerkleProof::virtualize(root, 1); + auto virt_root = vm::MerkleProof::virtualize(root); if (virt_root.is_null()) { return td::Status::Error("Virt root is null"); } else { diff --git a/utils/prepare-ls-slice-config.cpp b/utils/prepare-ls-slice-config.cpp index c22ced7839..c63e4ec7c8 100644 --- a/utils/prepare-ls-slice-config.cpp +++ b/utils/prepare-ls-slice-config.cpp @@ -136,7 +136,7 @@ class PrepareLsSliceConfig : public td::actor::Actor { res->seqno_ = block_id.seqno(); auto root = vm::std_boc_deserialize(obj.header_proof_).move_as_ok(); - root = vm::MerkleProof::virtualize(root, 1); + root = vm::MerkleProof::virtualize(root); block::gen::Block::Record blk; block::gen::BlockInfo::Record info; CHECK(tlb::unpack_cell(root, blk) && tlb::unpack_cell(blk.info, info)); diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index d60811d59d..fbaa8da8f5 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -80,7 +80,7 @@ class SplitStateDeserializer { vm::CellBuilder cb; block::gen::t_ShardStateUnsplit.pack(cb, shard_state_); - if (cb.finalize()->get_virtualization() > 0) { + if (cb.finalize()->is_virtualized()) { return td::Status::Error("State headers is pruned outside of account dict"); } @@ -110,7 +110,7 @@ class SplitStateDeserializer { vm::CellBuilder cb; block::gen::t_ShardStateUnsplit.pack(cb, shard_state_); auto state_root = cb.finalize(); - CHECK(state_root->get_virtualization() == 0); + CHECK(!state_root->is_virtualized()); return state_root; } diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index 8de97b6f4d..9999c4df71 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -759,7 +759,7 @@ bool AcceptBlockQuery::unpack_proof_link(BlockIdExt id, Ref proof_lin return fatal_error("block proof link is for another block: expected "s + id.to_str() + ", found " + proof_blk_id.to_str()); } - auto virt_root = vm::MerkleProof::virtualize(proof.root, 1); + auto virt_root = vm::MerkleProof::virtualize(proof.root); if (virt_root.is_null()) { return fatal_error("block proof link for block "s + id.to_str() + " does not contain a valid Merkle proof for the block header"); diff --git a/validator/impl/check-proof.cpp b/validator/impl/check-proof.cpp index ae0ce92fcf..d223a63bba 100644 --- a/validator/impl/check-proof.cpp +++ b/validator/impl/check-proof.cpp @@ -149,7 +149,7 @@ bool CheckProof::init_parse(bool is_aux) { sig_weight_ = 0; sig_root_.clear(); } - auto virt_root = vm::MerkleProof::virtualize(proof.root, 1); + auto virt_root = vm::MerkleProof::virtualize(proof.root); if (virt_root.is_null()) { return fatal_error("block proof for block "s + proof_blk_id.to_str() + " does not contain a valid Merkle proof for the block header"); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 14c2d9aede..662bdff994 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -2928,7 +2928,7 @@ static td::Ref clean_usage_cells(td::Ref old_root, td::Refsecond; } auto loaded_cell = cell->load_cell().move_as_ok(); - CHECK(loaded_cell.virt.get_virtualization() == 0); + CHECK(loaded_cell.effective_level >= loaded_cell.data_cell->get_level()); td::Ref data_cell = std::move(loaded_cell.data_cell); td::Ref children[vm::Cell::max_refs]; bool changed = false; @@ -3024,7 +3024,7 @@ bool Collator::process_account_storage_dict(block::Account& account) { return false; } } else { - collated_data_stat.add_loaded_cell(loaded_cell.data_cell, loaded_cell.virt.get_level()); + collated_data_stat.add_loaded_cell(loaded_cell.data_cell, static_cast(loaded_cell.effective_level)); } } vm::CellSlice cs{std::move(loaded_cell.data_cell)}; @@ -6731,7 +6731,7 @@ void Collator::on_cell_loaded(const vm::LoadedCell& loaded_cell) { if (block_limit_status_) { block_limit_status_->collated_data_size_estimate -= stat->estimate_proof_size(); } - stat->add_loaded_cell(loaded_cell.data_cell, loaded_cell.virt.get_level()); + stat->add_loaded_cell(loaded_cell.data_cell, static_cast(loaded_cell.effective_level)); if (block_limit_status_) { block_limit_status_->collated_data_size_estimate += stat->estimate_proof_size(); } diff --git a/validator/impl/ihr-message.cpp b/validator/impl/ihr-message.cpp index 75cf08bc72..7c15ac566c 100644 --- a/validator/impl/ihr-message.cpp +++ b/validator/impl/ihr-message.cpp @@ -76,7 +76,7 @@ td::Result> IhrMessageQ::create_ihr_message(td::BufferSlice dat return td::Status::Error("IHR message does not contain a valid source BlockIdExt"); } try { - auto virt_root = vm::MerkleProof::virtualize(proof, 1); + auto virt_root = vm::MerkleProof::virtualize(proof); if (virt_root.is_null()) { return td::Status::Error("IHR message does not contain a valid Merkle proof"); } diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index dc56997a15..9f4b877eb8 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -192,7 +192,7 @@ td::Result>> OutMsgQueueProof::fetch(Shard block_state_proof = block_state_proofs[j++]; TRY_RESULT_ASSIGN(state_root_hash, unpack_block_state_proof(blocks[i], block_state_proof)); } - auto state_root = vm::MerkleProof::virtualize(queue_proofs[i], 1); + auto state_root = vm::MerkleProof::virtualize(queue_proofs[i]); if (state_root->get_hash().as_slice() != state_root_hash.as_slice()) { return td::Status::Error("state root hash mismatch"); } diff --git a/validator/impl/proof.cpp b/validator/impl/proof.cpp index d722221161..341029e57c 100644 --- a/validator/impl/proof.cpp +++ b/validator/impl/proof.cpp @@ -132,7 +132,7 @@ td::Result ProofLinkQ::get_virtual_root(bool lazy) if (proof_blk_id != id_) { return td::Status::Error(-668, "masterchain block proof is for another block"); } - auto virt_root = vm::MerkleProof::virtualize(proof.root, 1); + auto virt_root = vm::MerkleProof::virtualize(proof.root); if (virt_root.is_null()) { return td::Status::Error(-668, "block proof for block "s + proof_blk_id.to_str() + " does not contain a valid Merkle proof for the block header"); @@ -179,7 +179,7 @@ td::Result> create_block_state_proof(td::Ref root) { } td::Result unpack_block_state_proof(BlockIdExt block_id, td::Ref proof) { - auto virt_root = vm::MerkleProof::virtualize(proof, 1); + auto virt_root = vm::MerkleProof::virtualize(proof); if (virt_root.is_null()) { return td::Status::Error("invalid Merkle proof"); } diff --git a/validator/impl/top-shard-descr.cpp b/validator/impl/top-shard-descr.cpp index 9eadeef350..f38d0cbdc1 100644 --- a/validator/impl/top-shard-descr.cpp +++ b/validator/impl/top-shard-descr.cpp @@ -39,7 +39,7 @@ ShardTopBlockDescrQ* ShardTopBlockDescrQ::make_copy() const { } td::Status ShardTopBlockDescrQ::unpack_one_proof(BlockIdExt& cur_id, Ref proof_root, bool is_head) { - auto virt_root = vm::MerkleProof::virtualize(proof_root, 1); + auto virt_root = vm::MerkleProof::virtualize(proof_root); if (virt_root.is_null()) { return td::Status::Error(-666, "link for block "s + cur_id.to_str() + " inside ShardTopBlockDescr of " + block_id_.to_str() + @@ -508,7 +508,7 @@ Ref ShardTopBlockDescrQ::get_prev_descr(int pos, int sum_cnt (unsigned)(pos + sum_cnt) > size()) { return {}; } - auto virt_root = vm::MerkleProof::virtualize(proof_roots_.at(pos), 1); + auto virt_root = vm::MerkleProof::virtualize(proof_roots_.at(pos)); auto res = block::McShardHash::from_block(std::move(virt_root), chain_blk_ids_.at(pos).file_hash); if (res.not_null()) { auto& total_fees = res.write().fees_collected_; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index a1ab5026f2..bd889645c9 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -717,7 +717,7 @@ bool ValidateQuery::extract_collated_data_from(Ref croot, int idx) { if (cs.special_type() != vm::Cell::SpecialType::MerkleProof) { return reject_query("it is a special cell, but not a Merkle proof root"); } - auto virt_root = vm::MerkleProof::virtualize(croot, 1); + auto virt_root = vm::MerkleProof::virtualize(croot); if (virt_root.is_null()) { return reject_query("invalid Merkle proof"); } @@ -747,7 +747,7 @@ bool ValidateQuery::extract_collated_data_from(Ref croot, int idx) { } // account_storage_dict_proof#37c1e3fc proof:^Cell = AccountStorageDictProof; Ref proof = cs.prefetch_ref(); - auto virt_root = vm::MerkleProof::virtualize(proof, 1); + auto virt_root = vm::MerkleProof::virtualize(proof); if (virt_root.is_null()) { return reject_query("invalid Merkle proof in AccountStorageDictProof"); } From a401b75c91d407298ace1f60ea42d58761b6295c Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Thu, 6 Nov 2025 09:09:36 -0500 Subject: [PATCH 438/524] Fix -Werror build after #1791 (#1875) --- tdactor/benchmark/benchmark-coro.cpp | 8 ++++---- tdactor/td/actor/coro_utils.h | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tdactor/benchmark/benchmark-coro.cpp b/tdactor/benchmark/benchmark-coro.cpp index 6b06c6a388..42fe9c8bfe 100644 --- a/tdactor/benchmark/benchmark-coro.cpp +++ b/tdactor/benchmark/benchmark-coro.cpp @@ -52,7 +52,7 @@ class CoroBenchmark final : public td::actor::Actor { tasks.clear(); auto elapsed = timer.elapsed(); - auto ops_per_sec = total_ops / elapsed; + auto ops_per_sec = static_cast(total_ops) / elapsed; LOG(INFO) << name << " " << ops_per_thread << " ops: " << elapsed << "s (threads=" << thread_count << ", " << static_cast(ops_per_sec) << " ops/sec)"; co_return td::Unit(); @@ -76,7 +76,7 @@ class CoroBenchmark final : public td::actor::Actor { (void)co_await ask(db_, &BenchmarkDatabase::square, i); } auto elapsed = timer.elapsed(); - auto ops_per_sec = ops_count / elapsed; + auto ops_per_sec = static_cast(total_ops) / elapsed; LOG(INFO) << "Direct delayed " << ops_count << " ops: " << elapsed << "s (" << static_cast(ops_per_sec) << " ops/sec)"; @@ -85,7 +85,7 @@ class CoroBenchmark final : public td::actor::Actor { (void)co_await ask_immediate(db_, &BenchmarkDatabase::square, i); } elapsed = timer.elapsed(); - ops_per_sec = ops_count / elapsed; + ops_per_sec = static_cast(total_ops) / elapsed; LOG(INFO) << "Direct immediate " << ops_count << " ops: " << elapsed << "s (" << static_cast(ops_per_sec) << " ops/sec)"; @@ -95,7 +95,7 @@ class CoroBenchmark final : public td::actor::Actor { (void)co_await local_square(i); } elapsed = timer.elapsed(); - ops_per_sec = ops_count / elapsed; + ops_per_sec = static_cast(total_ops) / elapsed; LOG(INFO) << "Local coroutine " << ops_count << " ops: " << elapsed << "s (" << static_cast(ops_per_sec) << " ops/sec)"; diff --git a/tdactor/td/actor/coro_utils.h b/tdactor/td/actor/coro_utils.h index 0abb729673..e394290438 100644 --- a/tdactor/td/actor/coro_utils.h +++ b/tdactor/td/actor/coro_utils.h @@ -169,7 +169,6 @@ template auto ask_impl(TargetId&& to, MemFn mf, Args&&... args) { using Meta = unified_result; using T = Meta::type; - using Ret = detail::memfn_meta::ret; using TT = std::conditional_t, td::Unit, T>; From d7b4e738d2394e3cfbac00ad32146e78b8e6b212 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Mon, 20 Oct 2025 00:08:26 -0400 Subject: [PATCH 439/524] Allow clang-format to reorder includes --- .clang-format | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.clang-format b/.clang-format index 098799e102..886aa8df99 100644 --- a/.clang-format +++ b/.clang-format @@ -49,6 +49,16 @@ DisableFormat: false ExperimentalAutoDetectBinPacking: false FixNamespaceComments: true # ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] +IncludeBlocks: Regroup +IncludeCategories: + - Regex: '>$' + Priority: -3 + - Regex: '.h>$' + Priority: -2 + - Regex: '^\".*/.*\"$' + Priority: -1 + - Regex: '^\"[^/]*\"$' + Priority: 0 IndentCaseLabels: true IndentWidth: 2 IndentWrappedFunctionNames: false @@ -70,7 +80,7 @@ PenaltyExcessCharacter: 1000000 PenaltyReturnTypeOnItsOwnLine: 200 PointerAlignment: Left ReflowComments: false # true -SortIncludes: false # disabled, because we need case insensitive sort +SortIncludes: true SpaceAfterCStyleCast: false SpaceAfterTemplateKeyword: true SpaceBeforeAssignmentOperators: true From 26923f4228745629d10aca0a65bcb0bff13c8155 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Thu, 6 Nov 2025 09:23:01 -0500 Subject: [PATCH 440/524] Fix errors that will surface due to header reordering in the next commit --- crypto/block/block-binlog.h | 27 ++++++++++++++------------- crypto/common/AtomicRef.h | 7 ++++--- crypto/common/util.cpp | 2 +- crypto/common/util.h | 2 +- dht/utils/dht-resolve.cpp | 2 +- 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/crypto/block/block-binlog.h b/crypto/block/block-binlog.h index 904f4c7327..37d34b0293 100644 --- a/crypto/block/block-binlog.h +++ b/crypto/block/block-binlog.h @@ -17,11 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/utils/int_types.h" -#include "ton/ton-types.h" #include +#include "td/utils/as.h" +#include "ton/ton-types.h" + namespace block { namespace log { @@ -35,9 +36,9 @@ struct Start { unsigned type_field; unsigned created_at; unsigned char zerostate_root_hash[32]; - Start(const RootHash& hash, unsigned _now = 0) + Start(const ton::RootHash& hash, unsigned _now = 0) : tag_field(tag), type_field(log_type), created_at(_now ? _now : (unsigned)std::time(nullptr)) { - td::as(zerostate_root_hash) = hash; + td::as(zerostate_root_hash) = hash; } }; @@ -48,10 +49,10 @@ struct SetZeroState { long long file_size; unsigned char file_hash[32]; unsigned char root_hash[32]; - SetZeroState(const RootHash& rhash, const FileHash& fhash, unsigned long long _fsize, unsigned _flags = 0) + SetZeroState(const ton::RootHash& rhash, const ton::FileHash& fhash, unsigned long long _fsize, unsigned _flags = 0) : tag_field(tag), flags(_flags), file_size(_fsize) { - td::as(file_hash) = fhash; - td::as(root_hash) = rhash; + td::as(file_hash) = fhash; + td::as(root_hash) = rhash; } }; @@ -66,7 +67,7 @@ struct NewBlock { unsigned char file_hash[32]; unsigned char root_hash[32]; unsigned char last_bytes[8]; - NewBlock(const ton::BlockId& block, const RootHash& rhash, const FileHash& fhash, unsigned long long _fsize, + NewBlock(const ton::BlockId& block, const ton::RootHash& rhash, const ton::FileHash& fhash, unsigned long long _fsize, unsigned _flags) : tag_field(tag) , flags(_flags) @@ -74,8 +75,8 @@ struct NewBlock { , seqno(block.seqno) , shard(block.shard) , file_size(_fsize) { - td::as(file_hash) = fhash; - td::as(root_hash) = rhash; + td::as(file_hash) = fhash; + td::as(root_hash) = rhash; td::as(last_bytes) = 0; } }; @@ -91,7 +92,7 @@ struct NewState { unsigned char file_hash[32]; unsigned char root_hash[32]; unsigned char last_bytes[8]; - NewState(const ton::BlockId& state, const RootHash& rhash, const FileHash& fhash, unsigned long long _fsize, + NewState(const ton::BlockId& state, const ton::RootHash& rhash, const ton::FileHash& fhash, unsigned long long _fsize, unsigned _flags) : tag_field(tag) , flags(_flags) @@ -99,8 +100,8 @@ struct NewState { , seqno(state.seqno) , shard(state.shard) , file_size(_fsize) { - td::as(file_hash) = fhash; - td::as(root_hash) = rhash; + td::as(file_hash) = fhash; + td::as(root_hash) = rhash; td::as(last_bytes) = 0; } }; diff --git a/crypto/common/AtomicRef.h b/crypto/common/AtomicRef.h index e82bcda49c..33036fe1fa 100644 --- a/crypto/common/AtomicRef.h +++ b/crypto/common/AtomicRef.h @@ -18,11 +18,12 @@ */ #pragma once -#include "td/utils/SpinLock.h" -#include "common/refcnt.hpp" - +#include #include +#include "common/refcnt.hpp" +#include "td/utils/SpinLock.h" + namespace td { template class AtomicRefSpinlock { diff --git a/crypto/common/util.cpp b/crypto/common/util.cpp index d1b24a76bc..50b547cbbe 100644 --- a/crypto/common/util.cpp +++ b/crypto/common/util.cpp @@ -163,7 +163,7 @@ std::size_t buff_base64_decode(td::MutableSlice buffer, td::Slice encoded, bool return wptr - (unsigned char *)buffer.data(); } -td::BufferSlice base64_decode(td::Slice encoded, bool allow_base64_url) { +td::BufferSlice base64_decode_to_buffer_slice(td::Slice encoded, bool allow_base64_url) { auto s = decoded_base64_size(encoded, allow_base64_url); if (s <= 0) { return td::BufferSlice{}; diff --git a/crypto/common/util.h b/crypto/common/util.h index c56677b6ad..49f50867ff 100644 --- a/crypto/common/util.h +++ b/crypto/common/util.h @@ -34,7 +34,7 @@ bool is_valid_base64(td::Slice encoded, bool allow_base64_url = true); td::int32 decoded_base64_size(td::Slice encoded, bool allow_base64_url = true); std::size_t buff_base64_decode(td::MutableSlice buffer, td::Slice data, bool allow_base64_url = true); -td::BufferSlice base64_decode(td::Slice encoded, bool allow_base64_url = true); +td::BufferSlice base64_decode_to_buffer_slice(td::Slice encoded, bool allow_base64_url = true); std::string str_base64_decode(td::Slice encoded, bool allow_base64_url = true); //TODO: move it somewhere else diff --git a/dht/utils/dht-resolve.cpp b/dht/utils/dht-resolve.cpp index 5ab98d0158..a66846b1eb 100644 --- a/dht/utils/dht-resolve.cpp +++ b/dht/utils/dht-resolve.cpp @@ -149,7 +149,7 @@ class Resolver : public td::actor::Actor { }; td::Result parse_bits256(td::Slice s) { - td::BufferSlice str = td::base64_decode(s, true); + td::BufferSlice str = td::base64_decode_to_buffer_slice(s, true); if (str.size() != 32) { return td::Status::Error("Invalid bits256"); } From f262baeaba892f17b8f612f97b919a2b25e83bf0 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Thu, 6 Nov 2025 09:27:00 -0500 Subject: [PATCH 441/524] Reformat entire codebase (except Tolk) with clang-format 21 This was done by `clang-format -i $(git ls-files '*.h' '*.hpp' '*.cpp' ':!^example/android/third_party' ':!^tolk/')`. --- adnl/adnl-address-list.cpp | 9 +- adnl/adnl-address-list.h | 3 +- adnl/adnl-address-list.hpp | 2 +- adnl/adnl-channel.cpp | 8 +- adnl/adnl-channel.h | 4 +- adnl/adnl-channel.hpp | 3 +- adnl/adnl-db.cpp | 3 +- adnl/adnl-db.hpp | 3 +- adnl/adnl-ext-client.cpp | 2 +- adnl/adnl-ext-client.h | 3 +- adnl/adnl-ext-client.hpp | 11 +- adnl/adnl-ext-connection.hpp | 12 +- adnl/adnl-ext-server.cpp | 3 +- adnl/adnl-ext-server.hpp | 11 +- adnl/adnl-local-id.cpp | 7 +- adnl/adnl-local-id.h | 7 +- adnl/adnl-message.h | 2 +- adnl/adnl-network-manager.cpp | 7 +- adnl/adnl-network-manager.h | 5 +- adnl/adnl-network-manager.hpp | 10 +- adnl/adnl-node-id.cpp | 4 +- adnl/adnl-node-id.hpp | 2 +- adnl/adnl-node.h | 3 +- adnl/adnl-packet.cpp | 3 +- adnl/adnl-packet.h | 2 +- adnl/adnl-peer-table.cpp | 95 +- adnl/adnl-peer-table.h | 12 +- adnl/adnl-peer-table.hpp | 12 +- adnl/adnl-peer.cpp | 30 +- adnl/adnl-peer.h | 4 +- adnl/adnl-peer.hpp | 18 +- adnl/adnl-pong.cpp | 25 +- adnl/adnl-proxy-types.cpp | 9 +- adnl/adnl-proxy-types.h | 4 +- adnl/adnl-proxy-types.hpp | 4 +- adnl/adnl-proxy.cpp | 25 +- adnl/adnl-query.cpp | 3 +- adnl/adnl-query.h | 6 +- adnl/adnl-test-loopback-implementation.h | 4 +- adnl/adnl-tunnel.cpp | 2 +- adnl/adnl-tunnel.h | 7 +- adnl/adnl.h | 7 +- adnl/test/adnl-test-ping.cpp | 8 +- adnl/utils.cpp | 11 +- adnl/utils.hpp | 19 +- .../blockchain-explorer-http.cpp | 15 +- .../blockchain-explorer-http.hpp | 6 +- .../blockchain-explorer-query.cpp | 26 +- .../blockchain-explorer-query.hpp | 11 +- blockchain-explorer/blockchain-explorer.cpp | 47 +- blockchain-explorer/blockchain-explorer.hpp | 5 +- catchain/catchain-block.cpp | 6 +- catchain/catchain-received-block.cpp | 4 +- catchain/catchain-received-block.h | 13 +- catchain/catchain-receiver-interface.h | 3 +- catchain/catchain-receiver-source.cpp | 3 +- catchain/catchain-receiver-source.h | 4 +- catchain/catchain-receiver-source.hpp | 4 +- catchain/catchain-receiver.cpp | 56 +- catchain/catchain-receiver.h | 4 +- catchain/catchain-receiver.hpp | 12 +- catchain/catchain-types.h | 2 +- catchain/catchain.cpp | 29 +- catchain/catchain.h | 1 + catchain/catchain.hpp | 7 +- common/checksum.h | 2 +- common/delay.h | 4 +- common/errorlog.cpp | 10 +- common/errorlog.h | 1 + common/global-version.h | 2 +- common/int-to-string.hpp | 2 +- common/io.hpp | 14 +- create-hardfork/create-hardfork.cpp | 43 +- crypto/Ed25519.cpp | 14 +- crypto/Ed25519.h | 2 +- crypto/block/Binlog.cpp | 5 +- crypto/block/account-storage-stat.h | 9 +- crypto/block/adjust-block.cpp | 15 +- crypto/block/block-db-impl.h | 7 +- crypto/block/block-db.cpp | 13 +- crypto/block/block-db.h | 8 +- crypto/block/block-parse.cpp | 74 +- crypto/block/block-parse.h | 16 +- crypto/block/block.cpp | 11 +- crypto/block/block.h | 15 +- crypto/block/check-proof.cpp | 29 +- crypto/block/check-proof.h | 2 +- crypto/block/create-state.cpp | 48 +- crypto/block/dump-block.cpp | 15 +- crypto/block/mc-config.cpp | 26 +- crypto/block/mc-config.h | 34 +- crypto/block/output-queue-merger.cpp | 2 +- crypto/block/output-queue-merger.h | 2 +- .../PrecompiledSmartContract.cpp | 4 +- .../PrecompiledSmartContract.h | 18 +- crypto/block/precompiled-smc/common.h | 5 +- crypto/block/test-block.cpp | 10 +- crypto/block/test-weight-distr.cpp | 5 +- crypto/block/transaction.cpp | 50 +- crypto/block/transaction.h | 42 +- crypto/common/bigexp.cpp | 5 +- crypto/common/bigint.hpp | 10 +- crypto/common/bitstring.cpp | 5 +- crypto/common/bitstring.h | 11 +- crypto/common/promiseop.hpp | 3 +- crypto/common/refcnt.cpp | 4 +- crypto/common/refcnt.hpp | 4 +- crypto/common/refint.cpp | 6 +- crypto/common/refint.h | 7 +- crypto/common/util.cpp | 6 +- crypto/common/util.h | 1 + crypto/ellcurve/p256.cpp | 10 +- crypto/ellcurve/secp256k1.cpp | 9 +- crypto/fift/Continuation.cpp | 2 +- crypto/fift/Continuation.h | 1 + crypto/fift/Dictionary.h | 3 +- crypto/fift/Fift.cpp | 4 +- crypto/fift/Fift.h | 6 +- crypto/fift/HashMap.cpp | 3 +- crypto/fift/HashMap.h | 2 +- crypto/fift/IntCtx.h | 19 +- crypto/fift/SourceLookup.cpp | 7 +- crypto/fift/fift-main.cpp | 45 +- crypto/fift/utils.cpp | 34 +- crypto/fift/utils.h | 3 +- crypto/fift/words.cpp | 53 +- crypto/func/analyzer.cpp | 6 +- crypto/func/asmops.cpp | 4 +- crypto/func/builtins.cpp | 2 +- crypto/func/func-main.cpp | 15 +- crypto/func/func.cpp | 23 +- crypto/func/func.h | 80 +- crypto/func/gen-abscode.cpp | 25 +- crypto/func/keywords.cpp | 3 +- crypto/func/optimize.cpp | 2 +- crypto/func/parse-func.cpp | 80 +- crypto/funcfiftlib/funcfiftlib.cpp | 23 +- crypto/keccak/keccak.cpp | 639 +- crypto/keccak/keccak.h | 2 +- crypto/openssl/bignum.h | 6 +- crypto/openssl/digest.hpp | 1 - crypto/openssl/rand.cpp | 7 +- crypto/openssl/rand.hpp | 2 +- crypto/openssl/residue.h | 3 +- crypto/parser/lexer.cpp | 12 +- crypto/parser/lexer.h | 5 +- crypto/parser/srcread.cpp | 3 +- crypto/parser/srcread.h | 2 +- crypto/parser/symtable.cpp | 5 +- crypto/parser/symtable.h | 5 +- crypto/smc-envelope/GenericAccount.cpp | 4 +- crypto/smc-envelope/GenericAccount.h | 3 +- crypto/smc-envelope/HighloadWallet.cpp | 10 +- crypto/smc-envelope/HighloadWallet.h | 5 +- crypto/smc-envelope/HighloadWalletV2.cpp | 10 +- crypto/smc-envelope/HighloadWalletV2.h | 11 +- crypto/smc-envelope/ManualDns.cpp | 23 +- crypto/smc-envelope/ManualDns.h | 12 +- crypto/smc-envelope/MultisigWallet.cpp | 8 +- crypto/smc-envelope/MultisigWallet.h | 6 +- crypto/smc-envelope/PaymentChannel.cpp | 10 +- crypto/smc-envelope/PaymentChannel.h | 7 +- crypto/smc-envelope/SmartContract.cpp | 79 +- crypto/smc-envelope/SmartContract.h | 10 +- crypto/smc-envelope/SmartContractCode.cpp | 12 +- crypto/smc-envelope/SmartContractCode.h | 3 +- crypto/smc-envelope/TestGiver.cpp | 6 +- crypto/smc-envelope/TestGiver.h | 5 +- crypto/smc-envelope/TestWallet.cpp | 9 +- crypto/smc-envelope/TestWallet.h | 5 +- crypto/smc-envelope/Wallet.cpp | 10 +- crypto/smc-envelope/Wallet.h | 5 +- crypto/smc-envelope/WalletInterface.h | 12 +- crypto/smc-envelope/WalletV3.cpp | 10 +- crypto/smc-envelope/WalletV3.h | 5 +- crypto/smc-envelope/WalletV4.cpp | 10 +- crypto/smc-envelope/WalletV4.h | 5 +- crypto/test/Ed25519.cpp | 15 +- crypto/test/fift.cpp | 7 +- crypto/test/modbigint.cpp | 16 +- crypto/test/test-bigint.cpp | 17 +- crypto/test/test-cells.cpp | 26 +- crypto/test/test-db.cpp | 123 +- crypto/test/test-smartcont.cpp | 36 +- crypto/test/vm.cpp | 13 +- crypto/tl/tlbc-aux.h | 4 +- crypto/tl/tlbc-data.h | 1 + crypto/tl/tlbc-gen-cpp.cpp | 5 +- crypto/tl/tlbc.cpp | 29 +- crypto/tl/tlblib.hpp | 3 +- crypto/util/mintless-proof-generator.cpp | 12 +- crypto/vm/Hasher.cpp | 25 +- crypto/vm/Hasher.h | 7 +- crypto/vm/arithops.cpp | 7 +- crypto/vm/atom.h | 7 +- crypto/vm/bls.h | 3 +- crypto/vm/boc-compression.cpp | 22 +- crypto/vm/boc-compression.h | 8 +- crypto/vm/boc-writers.h | 16 +- crypto/vm/boc.cpp | 18 +- crypto/vm/boc.h | 19 +- crypto/vm/cellops.cpp | 23 +- crypto/vm/cells.h | 1 - crypto/vm/cells/Cell.cpp | 6 +- crypto/vm/cells/Cell.h | 12 +- crypto/vm/cells/CellBuilder.cpp | 9 +- crypto/vm/cells/CellBuilder.h | 9 +- crypto/vm/cells/CellHash.cpp | 5 +- crypto/vm/cells/CellHash.h | 7 +- crypto/vm/cells/CellSlice.cpp | 6 +- crypto/vm/cells/CellString.cpp | 4 +- crypto/vm/cells/CellString.h | 1 - crypto/vm/cells/CellTraits.cpp | 5 +- crypto/vm/cells/CellTraits.h | 2 +- crypto/vm/cells/CellUsageTree.cpp | 1 + crypto/vm/cells/CellUsageTree.h | 4 +- crypto/vm/cells/ExtCell.h | 10 +- crypto/vm/cells/LevelMask.h | 2 +- crypto/vm/cells/MerkleProof.cpp | 9 +- crypto/vm/cells/MerkleProof.h | 10 +- crypto/vm/cells/MerkleUpdate.cpp | 5 +- crypto/vm/cells/MerkleUpdate.h | 9 +- crypto/vm/cells/PrunnedCell.h | 2 +- crypto/vm/cells/VirtualizationParameters.h | 4 +- crypto/vm/continuation.cpp | 6 +- crypto/vm/continuation.h | 4 +- crypto/vm/contops.cpp | 99 +- crypto/vm/cp0.cpp | 10 +- crypto/vm/db/BlobView.cpp | 12 +- crypto/vm/db/CellHashTable.h | 5 +- crypto/vm/db/CellStorage.cpp | 15 +- crypto/vm/db/CellStorage.h | 8 +- crypto/vm/db/DynamicBagOfCellsDb.cpp | 63 +- crypto/vm/db/DynamicBagOfCellsDb.h | 9 +- crypto/vm/db/DynamicBagOfCellsDbV2.cpp | 44 +- crypto/vm/db/InMemoryBagOfCellsDb.cpp | 12 +- crypto/vm/db/StaticBagOfCellsDb.cpp | 21 +- crypto/vm/db/StaticBagOfCellsDb.h | 3 +- crypto/vm/db/TonDb.cpp | 5 +- crypto/vm/db/TonDb.h | 33 +- crypto/vm/debugops.cpp | 10 +- crypto/vm/dict.cpp | 23 +- crypto/vm/dict.h | 7 +- crypto/vm/dictops.cpp | 11 +- crypto/vm/dispatch.cpp | 5 +- crypto/vm/fmt.hpp | 5 +- crypto/vm/large-boc-serializer.cpp | 32 +- crypto/vm/memo.cpp | 2 +- crypto/vm/memo.h | 2 +- crypto/vm/opctable.cpp | 18 +- crypto/vm/opctable.h | 12 +- crypto/vm/stack.cpp | 12 +- crypto/vm/stack.hpp | 19 +- crypto/vm/stackops.cpp | 4 +- crypto/vm/tonops.cpp | 199 +- crypto/vm/tonops.h | 3 +- crypto/vm/tupleops.cpp | 4 +- crypto/vm/utils.h | 4 +- crypto/vm/vm.cpp | 12 +- crypto/vm/vm.h | 14 +- crypto/vm/vmstate.h | 11 +- dht-server/dht-server.cpp | 58 +- dht-server/dht-server.hpp | 13 +- dht/dht-bucket.cpp | 15 +- dht/dht-bucket.hpp | 4 +- dht/dht-in.hpp | 5 +- dht/dht-node.cpp | 3 +- dht/dht-node.hpp | 12 +- dht/dht-query.cpp | 17 +- dht/dht-query.hpp | 24 +- dht/dht-remote-node.cpp | 11 +- dht/dht-remote-node.hpp | 8 +- dht/dht-types.cpp | 10 +- dht/dht-types.h | 6 +- dht/dht.cpp | 23 +- dht/dht.h | 10 +- dht/dht.hpp | 13 +- dht/utils/dht-ping-servers.cpp | 22 +- dht/utils/dht-resolve.cpp | 20 +- emulator/StringLog.h | 3 +- emulator/emulator-emscripten.cpp | 206 +- emulator/emulator-extern.cpp | 161 +- emulator/emulator-extern.h | 31 +- emulator/test/emulator-tests.cpp | 517 +- emulator/transaction-emulator.cpp | 309 +- emulator/transaction-emulator.h | 55 +- emulator/tvm-emulator.hpp | 10 +- example/android/native-lib.cpp | 10 +- fec/fec.cpp | 5 +- fec/fec.h | 2 +- git.h | 2 +- http/http-client.cpp | 4 +- http/http-client.h | 6 +- http/http-client.hpp | 4 +- http/http-connection.h | 6 +- http/http-inbound-connection.cpp | 3 +- http/http-inbound-connection.h | 2 +- http/http-outbound-connection.cpp | 4 +- http/http-outbound-connection.h | 8 +- http/http-proxy.cpp | 15 +- http/http-server.cpp | 2 +- http/http-server.h | 4 +- http/http.cpp | 8 +- http/http.h | 10 +- keyring/keyring.cpp | 9 +- keyring/keyring.h | 2 +- keyring/keyring.hpp | 6 +- keys/encryptor.cpp | 7 +- keys/encryptor.h | 6 +- keys/encryptor.hpp | 5 +- keys/keys.cpp | 5 +- keys/keys.hpp | 12 +- lite-client/ext-client.cpp | 3 +- lite-client/ext-client.h | 3 +- lite-client/lite-client-common.cpp | 8 +- lite-client/lite-client-common.h | 3 +- lite-client/lite-client.cpp | 106 +- lite-client/lite-client.h | 13 +- lite-client/query-utils.cpp | 13 +- lite-client/query-utils.hpp | 4 +- memprof/memprof/memprof.cpp | 6 +- overlay/overlay-broadcast.cpp | 5 +- overlay/overlay-broadcast.hpp | 3 +- overlay/overlay-fec-broadcast.cpp | 3 +- overlay/overlay-fec-broadcast.hpp | 8 +- overlay/overlay-fec.cpp | 3 +- overlay/overlay-fec.hpp | 3 +- overlay/overlay-id.hpp | 9 +- overlay/overlay-manager.cpp | 16 +- overlay/overlay-manager.h | 12 +- overlay/overlay-peers.cpp | 19 +- overlay/overlay.cpp | 25 +- overlay/overlay.h | 6 +- overlay/overlay.hpp | 36 +- overlay/overlays.h | 13 +- rldp-http-proxy/DNSResolver.cpp | 30 +- rldp-http-proxy/DNSResolver.h | 4 +- rldp-http-proxy/rldp-http-proxy.cpp | 40 +- rldp/rldp-in.hpp | 16 +- rldp/rldp-peer.cpp | 7 +- rldp/rldp-peer.hpp | 7 +- rldp/rldp.cpp | 7 +- rldp/rldp.hpp | 8 +- rldp2/Bbr.cpp | 5 +- rldp2/Bbr.h | 3 +- rldp2/FecHelper.cpp | 4 +- rldp2/InboundTransfer.cpp | 4 +- rldp2/InboundTransfer.h | 5 +- rldp2/LossSender.cpp | 4 +- rldp2/LossStats.cpp | 5 +- rldp2/LossStats.h | 4 +- rldp2/OutboundTransfer.h | 5 +- rldp2/Pacer.h | 2 +- rldp2/RldpConnection.cpp | 17 +- rldp2/RldpConnection.h | 17 +- rldp2/RldpSender.cpp | 9 +- rldp2/RttStats.cpp | 3 +- rldp2/SenderPackets.cpp | 4 +- rldp2/SenderPackets.h | 1 + rldp2/rldp-in.hpp | 14 +- rldp2/rldp.cpp | 7 +- rldp2/rldp.hpp | 6 +- storage/LoadSpeed.cpp | 4 +- storage/MerkleTree.cpp | 5 +- storage/MerkleTree.h | 5 +- storage/MicrochunkTree.cpp | 5 +- storage/MicrochunkTree.h | 6 +- storage/NodeActor.cpp | 17 +- storage/NodeActor.h | 12 +- storage/PartsHelper.h | 6 +- storage/PeerActor.cpp | 51 +- storage/PeerActor.h | 9 +- storage/PeerManager.h | 3 +- storage/PeerState.h | 25 +- storage/SpeedLimiter.cpp | 3 +- storage/SpeedLimiter.h | 3 +- storage/Torrent.cpp | 6 +- storage/Torrent.h | 12 +- storage/TorrentCreator.cpp | 7 +- storage/TorrentCreator.h | 8 +- storage/TorrentHeader.cpp | 4 +- storage/TorrentHeader.h | 2 +- storage/TorrentInfo.cpp | 5 +- storage/TorrentInfo.h | 3 +- storage/TorrentMeta.cpp | 10 +- storage/TorrentMeta.h | 4 +- storage/db.h | 3 +- storage/storage-cli.cpp | 27 +- storage/storage-daemon/StorageManager.cpp | 7 +- storage/storage-daemon/StorageManager.h | 5 +- storage/storage-daemon/StorageProvider.cpp | 11 +- storage/storage-daemon/StorageProvider.h | 7 +- storage/storage-daemon/smc-util.cpp | 5 +- storage/storage-daemon/smc-util.h | 9 +- storage/storage-daemon/storage-daemon-cli.cpp | 28 +- storage/storage-daemon/storage-daemon.cpp | 35 +- storage/test/storage.cpp | 60 +- tdactor/benchmark/benchmark-coro.cpp | 67 +- tdactor/benchmark/benchmark.cpp | 33 +- tdactor/benchmark/gbench-coro-folly.cpp | 15 +- tdactor/benchmark/gbench-coro-yaclib.cpp | 15 +- tdactor/benchmark/gbench-coro.cpp | 9 +- tdactor/benchmark/third_party/FAAArrayQueue.h | 4 +- .../benchmark/third_party/HazardPointers.h | 2 +- .../benchmark/third_party/MoodyCamelQueue.h | 13 +- tdactor/example/actor-example-coroutines.cpp | 23 +- tdactor/td/actor/ActorId.h | 2 +- tdactor/td/actor/ActorOwn.h | 2 +- tdactor/td/actor/ActorShared.h | 2 +- tdactor/td/actor/ActorStats.cpp | 4 +- tdactor/td/actor/MultiPromise.cpp | 4 +- tdactor/td/actor/PromiseFuture.h | 22 +- tdactor/td/actor/actor.h | 2 +- tdactor/td/actor/common.h | 8 +- tdactor/td/actor/core/Actor.h | 1 - tdactor/td/actor/core/ActorExecuteContext.h | 7 +- tdactor/td/actor/core/ActorExecutor.cpp | 1 - tdactor/td/actor/core/ActorExecutor.h | 1 - tdactor/td/actor/core/ActorInfo.h | 5 +- tdactor/td/actor/core/ActorInfoCreator.h | 4 +- tdactor/td/actor/core/ActorLocker.h | 5 +- tdactor/td/actor/core/ActorMessage.h | 1 - tdactor/td/actor/core/ActorSignals.h | 2 +- tdactor/td/actor/core/ActorState.h | 5 +- tdactor/td/actor/core/ActorTypeStat.cpp | 13 +- tdactor/td/actor/core/ActorTypeStat.h | 7 +- tdactor/td/actor/core/CpuWorker.cpp | 7 +- tdactor/td/actor/core/CpuWorker.h | 3 +- tdactor/td/actor/core/IoWorker.cpp | 3 +- tdactor/td/actor/core/Scheduler.cpp | 10 +- tdactor/td/actor/core/Scheduler.h | 30 +- tdactor/td/actor/core/SchedulerContext.h | 9 +- tdactor/td/actor/coro.h | 8 +- tdactor/td/actor/coro_awaitables.h | 8 +- tdactor/td/actor/coro_executor.h | 8 +- tdactor/td/actor/coro_task.h | 12 +- tdactor/td/actor/coro_types.h | 5 +- tdactor/td/actor/coro_utils.h | 16 +- tdactor/test/actors_core.cpp | 28 +- tdactor/test/actors_promise.cpp | 39 +- tdactor/test/test-coro.cpp | 14 +- tddb/td/db/KeyValue.h | 19 +- tddb/td/db/KeyValueAsync.h | 3 +- tddb/td/db/MemoryKeyValue.cpp | 5 +- tddb/td/db/MemoryKeyValue.h | 10 +- tddb/td/db/RocksDb.cpp | 18 +- tddb/td/db/RocksDb.h | 14 +- tddb/td/db/binlog/Binlog.cpp | 17 +- tddb/td/db/binlog/Binlog.h | 8 +- tddb/td/db/binlog/BinlogReaderHelper.cpp | 4 +- tddb/td/db/binlog/BinlogReaderHelper.h | 2 +- tddb/td/db/utils/BlobView.cpp | 12 +- tddb/td/db/utils/BlobView.h | 2 +- tddb/td/db/utils/ChainBuffer.cpp | 6 +- tddb/td/db/utils/CyclicBuffer.cpp | 10 +- tddb/td/db/utils/CyclicBuffer.h | 4 +- tddb/td/db/utils/FileSyncState.h | 4 +- tddb/td/db/utils/FileToStreamActor.h | 4 +- tddb/td/db/utils/StreamInterface.h | 2 +- tddb/td/db/utils/StreamToFileActor.h | 7 +- tddb/test/binlog.cpp | 34 +- tddb/test/io-bench.cpp | 23 +- tddb/test/key_value.cpp | 8 +- tdfec/benchmark/benchmark.cpp | 16 +- .../fec/algebra/BeliefPropagationDecoding.h | 4 +- tdfec/td/fec/algebra/InactivationDecoding.cpp | 5 +- tdfec/td/fec/algebra/InactivationDecoding.h | 2 +- tdfec/td/fec/algebra/MatrixGF256.h | 1 - tdfec/td/fec/algebra/Octet.h | 4 +- tdfec/td/fec/algebra/Simd.h | 3 +- tdfec/td/fec/fec.cpp | 8 +- tdfec/td/fec/fec.h | 2 +- tdfec/td/fec/online/Decoder.h | 5 +- tdfec/td/fec/online/Encoder.cpp | 3 +- tdfec/td/fec/online/Encoder.h | 3 +- tdfec/td/fec/online/Rfc.h | 6 +- tdfec/td/fec/raptorq/Decoder.h | 6 +- tdfec/td/fec/raptorq/Encoder.cpp | 1 - tdfec/td/fec/raptorq/Encoder.h | 9 +- tdfec/td/fec/raptorq/RawEncoder.h | 2 +- tdfec/td/fec/raptorq/Rfc.cpp | 4 +- tdfec/td/fec/raptorq/Rfc.h | 3 +- tdfec/td/fec/raptorq/Solver.cpp | 6 +- tdfec/td/fec/raptorq/Solver.h | 5 +- tdfec/test/LibRaptorQ.cpp | 3 +- tdfec/test/LibRaptorQ.h | 2 +- tdfec/test/fec-test.cpp | 6 +- tdnet/example/tcp_ping_pong.cpp | 8 +- tdnet/example/udp_ping_pong.cpp | 6 +- tdnet/td/net/FdListener.h | 1 - tdnet/td/net/TcpListener.h | 3 +- tdnet/td/net/UdpServer.cpp | 7 +- tdnet/td/net/UdpServer.h | 1 - tdtl/td/tl/tl_config.cpp | 4 +- tdtl/td/tl/tl_config.h | 6 +- tdtl/td/tl/tl_core.cpp | 4 +- tdtl/td/tl/tl_file_outputer.cpp | 4 +- tdtl/td/tl/tl_file_outputer.h | 4 +- tdtl/td/tl/tl_file_utils.cpp | 4 +- tdtl/td/tl/tl_generate.cpp | 17 +- tdtl/td/tl/tl_generate.h | 4 +- tdtl/td/tl/tl_simple.h | 5 +- tdtl/td/tl/tl_string_outputer.h | 4 +- tdtl/td/tl/tl_writer.cpp | 7 +- tdtl/td/tl/tl_writer.h | 4 +- tdutils/td/utils/AesCtrByteFlow.h | 4 +- tdutils/td/utils/BigNum.cpp | 7 +- tdutils/td/utils/BufferedFd.h | 12 +- tdutils/td/utils/BufferedReader.h | 4 +- tdutils/td/utils/BufferedUdp.h | 12 +- tdutils/td/utils/ByteFlow.h | 2 +- tdutils/td/utils/CancellationToken.h | 1 + tdutils/td/utils/ChangesProcessor.h | 4 +- tdutils/td/utils/Closure.h | 8 +- tdutils/td/utils/ConcurrentHashTable.h | 10 +- tdutils/td/utils/Container.h | 4 +- tdutils/td/utils/DecTree.h | 6 +- tdutils/td/utils/Destructor.h | 4 +- tdutils/td/utils/Enumerator.h | 4 +- .../td/utils/EpochBasedMemoryReclamation.h | 6 +- tdutils/td/utils/FileLog.cpp | 9 +- tdutils/td/utils/FileLog.h | 4 +- tdutils/td/utils/FloodControlFast.h | 2 +- tdutils/td/utils/FloodControlStrict.h | 4 +- tdutils/td/utils/GitInfo.cpp | 3 +- tdutils/td/utils/Gzip.cpp | 5 +- tdutils/td/utils/Gzip.h | 2 +- tdutils/td/utils/GzipByteFlow.cpp | 2 +- tdutils/td/utils/GzipByteFlow.h | 4 +- tdutils/td/utils/HashMap.h | 4 +- tdutils/td/utils/HazardPointers.h | 4 +- tdutils/td/utils/Hints.cpp | 7 +- tdutils/td/utils/Hints.h | 6 +- tdutils/td/utils/HttpUrl.cpp | 3 +- tdutils/td/utils/HttpUrl.h | 2 +- tdutils/td/utils/JsonBuilder.cpp | 7 +- tdutils/td/utils/JsonBuilder.h | 12 +- tdutils/td/utils/LRUCache.h | 3 +- tdutils/td/utils/MemoryLog.h | 8 +- tdutils/td/utils/MimeType.cpp | 1 - tdutils/td/utils/MimeType.h | 2 +- tdutils/td/utils/MpmcQueue.h | 10 +- tdutils/td/utils/MpmcWaiter.h | 10 +- tdutils/td/utils/MpscLinkQueue.h | 4 +- tdutils/td/utils/MpscPollableQueue.h | 4 +- tdutils/td/utils/Named.h | 2 +- tdutils/td/utils/ObjectPool.h | 10 +- tdutils/td/utils/OptionParser.cpp | 7 +- tdutils/td/utils/OptionParser.h | 6 +- tdutils/td/utils/OptionsParser.cpp | 7 +- tdutils/td/utils/OptionsParser.h | 6 +- tdutils/td/utils/OrderedEventsProcessor.h | 4 +- tdutils/td/utils/Parser.h | 10 +- tdutils/td/utils/PathView.cpp | 1 - tdutils/td/utils/Random.cpp | 1 - tdutils/td/utils/Random.h | 2 +- tdutils/td/utils/ScopeGuard.h | 4 +- tdutils/td/utils/SharedObjectPool.h | 16 +- tdutils/td/utils/SharedSlice.cpp | 1 - tdutils/td/utils/SharedSlice.h | 4 +- tdutils/td/utils/Slice-decl.h | 4 +- tdutils/td/utils/Slice.h | 6 +- tdutils/td/utils/Span.h | 4 +- tdutils/td/utils/SpinLock.h | 4 +- tdutils/td/utils/StackAllocator.cpp | 1 - tdutils/td/utils/StackAllocator.h | 8 +- tdutils/td/utils/Status.cpp | 5 +- tdutils/td/utils/Status.h | 22 +- tdutils/td/utils/StealingQueue.h | 2 +- tdutils/td/utils/Storer.h | 9 +- tdutils/td/utils/StringBuilder.cpp | 11 +- tdutils/td/utils/StringBuilder.h | 14 +- tdutils/td/utils/ThreadLocalStorage.h | 6 +- tdutils/td/utils/ThreadSafeCounter.h | 11 +- tdutils/td/utils/Time.cpp | 6 +- tdutils/td/utils/TimedStat.h | 4 +- tdutils/td/utils/Timer.cpp | 9 +- tdutils/td/utils/Timer.h | 6 +- tdutils/td/utils/TsFileLog.cpp | 11 +- tdutils/td/utils/TsFileLog.h | 2 +- tdutils/td/utils/TsList.h | 6 +- tdutils/td/utils/UInt.h | 2 +- tdutils/td/utils/Variant.h | 6 +- tdutils/td/utils/VectorQueue.h | 8 +- tdutils/td/utils/base64.cpp | 9 +- tdutils/td/utils/base64.h | 2 +- tdutils/td/utils/benchmark.h | 10 +- tdutils/td/utils/buffer.cpp | 7 +- tdutils/td/utils/buffer.h | 8 +- tdutils/td/utils/check.cpp | 3 +- tdutils/td/utils/common.h | 6 +- tdutils/td/utils/crypto.cpp | 15 +- tdutils/td/utils/crypto.h | 5 +- tdutils/td/utils/date.h | 10417 +++++++--------- tdutils/td/utils/filesystem.cpp | 9 +- tdutils/td/utils/filesystem.h | 3 +- tdutils/td/utils/find_boundary.cpp | 4 +- tdutils/td/utils/find_boundary.h | 2 +- tdutils/td/utils/format.h | 14 +- tdutils/td/utils/int_types.h | 4 +- tdutils/td/utils/invoke.h | 24 +- tdutils/td/utils/logging.cpp | 16 +- tdutils/td/utils/logging.h | 10 +- tdutils/td/utils/lz4.cpp | 3 +- tdutils/td/utils/lz4.h | 2 +- tdutils/td/utils/misc.cpp | 7 +- tdutils/td/utils/misc.h | 12 +- tdutils/td/utils/optional.h | 12 +- tdutils/td/utils/overloaded.h | 4 +- tdutils/td/utils/port/Clocks.cpp | 4 +- tdutils/td/utils/port/CxCli.h | 32 +- tdutils/td/utils/port/EventFdBase.h | 2 +- tdutils/td/utils/port/FileFd.cpp | 17 +- tdutils/td/utils/port/FileFd.h | 13 +- tdutils/td/utils/port/IPAddress.cpp | 7 +- tdutils/td/utils/port/IPAddress.h | 5 +- tdutils/td/utils/port/MemoryMapping.cpp | 3 +- tdutils/td/utils/port/MemoryMapping.h | 4 +- tdutils/td/utils/port/Poll.h | 1 - tdutils/td/utils/port/PollBase.h | 2 +- tdutils/td/utils/port/PollFlags.h | 4 +- tdutils/td/utils/port/RwMutex.h | 5 +- tdutils/td/utils/port/ServerSocketFd.cpp | 13 +- tdutils/td/utils/port/ServerSocketFd.h | 9 +- tdutils/td/utils/port/SocketFd.cpp | 12 +- tdutils/td/utils/port/SocketFd.h | 13 +- tdutils/td/utils/port/Stat.cpp | 9 +- tdutils/td/utils/port/Stat.h | 5 +- tdutils/td/utils/port/StdStreams.cpp | 11 +- tdutils/td/utils/port/StdStreams.h | 10 +- tdutils/td/utils/port/UdpSocketFd.cpp | 12 +- tdutils/td/utils/port/UdpSocketFd.h | 13 +- tdutils/td/utils/port/detail/Epoll.cpp | 9 +- tdutils/td/utils/port/detail/Epoll.h | 10 +- tdutils/td/utils/port/detail/EventFdBsd.cpp | 13 +- tdutils/td/utils/port/detail/EventFdBsd.h | 4 +- tdutils/td/utils/port/detail/EventFdLinux.cpp | 17 +- tdutils/td/utils/port/detail/EventFdLinux.h | 4 +- tdutils/td/utils/port/detail/EventFdWindows.h | 4 +- tdutils/td/utils/port/detail/Iocp.h | 8 +- tdutils/td/utils/port/detail/KQueue.cpp | 9 +- tdutils/td/utils/port/detail/KQueue.h | 16 +- tdutils/td/utils/port/detail/NativeFd.cpp | 5 +- tdutils/td/utils/port/detail/NativeFd.h | 5 +- tdutils/td/utils/port/detail/Poll.cpp | 8 +- tdutils/td/utils/port/detail/Poll.h | 6 +- tdutils/td/utils/port/detail/PollableFd.h | 14 +- tdutils/td/utils/port/detail/Select.cpp | 4 +- tdutils/td/utils/port/detail/Select.h | 6 +- .../td/utils/port/detail/ThreadIdGuard.cpp | 7 +- .../td/utils/port/detail/ThreadPthread.cpp | 4 +- tdutils/td/utils/port/detail/ThreadPthread.h | 19 +- tdutils/td/utils/port/detail/ThreadStl.h | 14 +- tdutils/td/utils/port/detail/WineventPoll.h | 2 +- tdutils/td/utils/port/path.cpp | 10 +- tdutils/td/utils/port/path.h | 10 +- tdutils/td/utils/port/rlimit.cpp | 7 +- tdutils/td/utils/port/rlimit.h | 2 +- tdutils/td/utils/port/signals.cpp | 10 +- tdutils/td/utils/port/signals.h | 2 +- tdutils/td/utils/port/sleep.cpp | 3 +- tdutils/td/utils/port/stacktrace.cpp | 5 +- tdutils/td/utils/port/thread.h | 1 - tdutils/td/utils/port/thread_local.h | 15 +- tdutils/td/utils/port/uname.cpp | 6 +- tdutils/td/utils/port/user.cpp | 3 +- tdutils/td/utils/port/wstring_convert.cpp | 4 +- tdutils/td/utils/port/wstring_convert.h | 6 +- tdutils/td/utils/tests.cpp | 13 +- tdutils/td/utils/tests.h | 16 +- tdutils/td/utils/tl_helpers.h | 17 +- tdutils/td/utils/tl_parsers.cpp | 3 +- tdutils/td/utils/tl_parsers.h | 18 +- tdutils/td/utils/tl_storers.h | 10 +- tdutils/td/utils/translit.cpp | 9 +- tdutils/td/utils/translit.h | 2 +- tdutils/td/utils/uint128.h | 6 +- tdutils/td/utils/unicode.cpp | 7 +- tdutils/td/utils/unique_ptr.h | 2 +- tdutils/td/utils/utf8.cpp | 3 +- tdutils/td/utils/utf8.h | 2 +- tdutils/test/ConcurrentHashMap.cpp | 10 +- tdutils/test/EpochBasedMemoryReclamation.cpp | 8 +- tdutils/test/HazardPointers.cpp | 10 +- tdutils/test/List.cpp | 20 +- tdutils/test/MpmcQueue.cpp | 8 +- tdutils/test/MpmcWaiter.cpp | 6 +- tdutils/test/MpscLinkQueue.cpp | 2 +- tdutils/test/OptionParser.cpp | 4 +- tdutils/test/OrderedEventsProcessor.cpp | 8 +- tdutils/test/SharedObjectPool.cpp | 30 +- tdutils/test/SharedSlice.cpp | 2 +- tdutils/test/StealingQueue.cpp | 7 +- tdutils/test/buffer.cpp | 5 +- tdutils/test/crypto.cpp | 10 +- tdutils/test/filesystem.cpp | 2 +- tdutils/test/gzip.cpp | 8 +- tdutils/test/heap.cpp | 11 +- tdutils/test/json.cpp | 11 +- tdutils/test/log.cpp | 12 +- tdutils/test/misc.cpp | 34 +- tdutils/test/port.cpp | 11 +- tdutils/test/pq.cpp | 9 +- tdutils/test/variant.cpp | 2 +- terminal/terminal.cpp | 5 +- terminal/terminal.h | 6 +- terminal/terminal.hpp | 6 +- test/test-adnl.cpp | 18 +- test/test-catchain.cpp | 15 +- test/test-dht.cpp | 11 +- test/test-hello-world.cpp | 3 +- test/test-http.cpp | 13 +- test/test-overlay.cpp | 29 +- test/test-rldp.cpp | 11 +- test/test-rldp2.cpp | 11 +- test/test-rocksdb.cpp | 8 +- test/test-td-main.cpp | 5 +- test/test-ton-collator.cpp | 39 +- test/test-validator-session-state.cpp | 16 +- third-party/wingetopt/src/getopt.h | 55 +- tl-utils/common-utils.hpp | 11 +- tl-utils/lite-utils.cpp | 12 +- tl-utils/lite-utils.hpp | 6 +- tl-utils/tl-utils.cpp | 9 +- tl-utils/tl-utils.hpp | 6 +- tl/generate/generate_common.cpp | 16 +- tl/generate/generate_java.cpp | 6 +- tl/generate/tl_json_converter.cpp | 9 +- tl/generate/tl_json_converter.h | 4 +- tl/generate/tl_writer_cpp.cpp | 4 +- tl/generate/tl_writer_cpp.h | 4 +- tl/generate/tl_writer_h.cpp | 4 +- tl/generate/tl_writer_h.h | 4 +- tl/generate/tl_writer_hpp.cpp | 4 +- tl/generate/tl_writer_hpp.h | 4 +- tl/generate/tl_writer_java.cpp | 10 +- tl/generate/tl_writer_java.h | 4 +- tl/generate/tl_writer_jni_cpp.cpp | 22 +- tl/generate/tl_writer_jni_cpp.h | 4 +- tl/generate/tl_writer_jni_h.cpp | 10 +- tl/generate/tl_writer_jni_h.h | 4 +- tl/generate/tl_writer_td.cpp | 6 +- tl/generate/tl_writer_td.h | 4 +- tl/tl/TlObject.h | 2 +- tl/tl/tl_jni_object.cpp | 6 +- tl/tl/tl_jni_object.h | 15 +- tl/tl/tl_json.h | 12 +- tl/tl/tl_object_parse.h | 5 +- tl/tl/tl_object_store.h | 3 +- ton/lite-tl.hpp | 3 +- ton/ton-io.hpp | 3 +- ton/ton-shard.h | 5 +- ton/ton-tl.hpp | 3 +- ton/ton-types.h | 8 +- tonlib/test/offline.cpp | 30 +- tonlib/test/online.cpp | 59 +- tonlib/tonlib/Client.cpp | 9 +- tonlib/tonlib/ClientJson.cpp | 13 +- tonlib/tonlib/ClientJson.h | 7 +- tonlib/tonlib/Config.cpp | 5 +- tonlib/tonlib/Config.h | 2 +- tonlib/tonlib/ExtClient.cpp | 1 - tonlib/tonlib/ExtClient.h | 12 +- tonlib/tonlib/ExtClientOutbound.cpp | 3 +- tonlib/tonlib/ExtClientOutbound.h | 2 +- tonlib/tonlib/KeyStorage.cpp | 16 +- tonlib/tonlib/KeyStorage.h | 6 +- tonlib/tonlib/KeyValue.cpp | 17 +- tonlib/tonlib/KeyValue.h | 4 +- tonlib/tonlib/LastBlock.cpp | 10 +- tonlib/tonlib/LastBlock.h | 6 +- tonlib/tonlib/LastBlockStorage.cpp | 9 +- tonlib/tonlib/LastBlockStorage.h | 3 +- tonlib/tonlib/LastConfig.cpp | 11 +- tonlib/tonlib/LastConfig.h | 9 +- tonlib/tonlib/Logging.cpp | 15 +- tonlib/tonlib/Logging.h | 3 +- tonlib/tonlib/TonlibClient.cpp | 1004 +- tonlib/tonlib/TonlibClient.h | 30 +- tonlib/tonlib/TonlibClientWrapper.h | 2 +- tonlib/tonlib/TonlibError.h | 2 +- tonlib/tonlib/keys/DecryptedKey.cpp | 7 +- tonlib/tonlib/keys/DecryptedKey.h | 5 +- tonlib/tonlib/keys/EncryptedKey.cpp | 5 +- tonlib/tonlib/keys/EncryptedKey.h | 5 +- tonlib/tonlib/keys/Mnemonic.cpp | 17 +- tonlib/tonlib/keys/Mnemonic.h | 3 +- tonlib/tonlib/keys/SimpleEncryption.cpp | 6 +- tonlib/tonlib/keys/SimpleEncryption.h | 6 +- tonlib/tonlib/tonlib-cli.cpp | 56 +- tonlib/tonlib/tonlib_client_json.cpp | 7 +- tonlib/tonlib/utils.cpp | 2 +- tonlib/tonlib/utils.h | 6 +- utils/generate-random-id.cpp | 13 +- utils/json2tlo.cpp | 23 +- utils/opcode-timing.cpp | 14 +- utils/pack-viewer.cpp | 11 +- utils/prepare-ls-slice-config.cpp | 37 +- utils/proxy-liteserver.cpp | 29 +- .../validator-engine-console-query.cpp | 65 +- .../validator-engine-console-query.h | 9 +- .../validator-engine-console.cpp | 32 +- .../validator-engine-console.h | 7 +- validator-engine/validator-engine.cpp | 264 +- validator-engine/validator-engine.hpp | 29 +- validator-session/candidate-serializer.cpp | 59 +- validator-session/candidate-serializer.h | 2 +- validator-session/persistent-vector.cpp | 5 +- validator-session/persistent-vector.h | 5 +- .../validator-session-description.cpp | 9 +- .../validator-session-description.h | 7 +- .../validator-session-description.hpp | 10 +- .../validator-session-round-attempt-state.cpp | 8 +- .../validator-session-round-attempt-state.h | 12 +- validator-session/validator-session-state.cpp | 9 +- validator-session/validator-session-state.h | 13 +- validator-session/validator-session-types.h | 8 +- validator-session/validator-session.cpp | 87 +- validator-session/validator-session.h | 8 +- validator-session/validator-session.hpp | 13 +- validator/apply-block.cpp | 8 +- validator/apply-block.hpp | 6 +- validator/block-handle.cpp | 3 +- validator/block-handle.hpp | 7 +- validator/collation-manager.cpp | 10 +- validator/collation-manager.hpp | 3 +- .../collator-node/collator-node-session.cpp | 1 - .../collator-node/collator-node-session.hpp | 5 +- validator/collator-node/collator-node.cpp | 11 +- validator/collator-node/collator-node.hpp | 8 +- validator/collator-node/utils.cpp | 5 +- validator/collator-node/utils.hpp | 2 +- validator/db/archive-db.cpp | 7 +- validator/db/archive-db.hpp | 11 +- validator/db/archive-manager.cpp | 7 +- validator/db/archive-manager.hpp | 3 +- validator/db/archive-slice.cpp | 43 +- validator/db/archive-slice.hpp | 22 +- validator/db/archiver.cpp | 3 +- validator/db/archiver.hpp | 5 +- validator/db/celldb.cpp | 114 +- validator/db/celldb.hpp | 19 +- validator/db/db-utils.cpp | 4 +- validator/db/fileref.cpp | 5 +- validator/db/fileref.hpp | 5 +- validator/db/files-async.hpp | 9 +- validator/db/package.cpp | 3 +- validator/db/package.hpp | 2 +- .../permanent-celldb-utils.cpp | 3 +- .../permanent-celldb/permanent-celldb-utils.h | 10 +- validator/db/rootdb.cpp | 16 +- validator/db/rootdb.hpp | 12 +- validator/db/statedb.cpp | 9 +- validator/db/statedb.hpp | 1 - validator/db/staticfilesdb.cpp | 2 +- validator/db/staticfilesdb.hpp | 2 +- validator/downloaders/download-state.cpp | 9 +- validator/downloaders/download-state.hpp | 1 + .../downloaders/wait-block-data-disk.cpp | 8 +- validator/downloaders/wait-block-data.cpp | 30 +- .../downloaders/wait-block-state-merge.cpp | 3 +- .../downloaders/wait-block-state-merge.hpp | 1 - validator/downloaders/wait-block-state.cpp | 7 +- validator/fabric.h | 14 +- validator/full-node-fast-sync-overlays.cpp | 10 +- validator/full-node-fast-sync-overlays.hpp | 4 +- validator/full-node-master.cpp | 15 +- validator/full-node-master.h | 3 +- validator/full-node-private-overlay.cpp | 11 +- validator/full-node-private-overlay.hpp | 6 +- validator/full-node-serializer.cpp | 21 +- validator/full-node-serializer.hpp | 2 +- validator/full-node-shard-queries.hpp | 3 +- validator/full-node-shard.cpp | 90 +- validator/full-node-shard.h | 5 +- validator/full-node-shard.hpp | 12 +- validator/full-node.cpp | 50 +- validator/full-node.h | 21 +- validator/full-node.hpp | 15 +- validator/impl/accept-block.cpp | 50 +- validator/impl/accept-block.hpp | 9 +- validator/impl/block.hpp | 2 +- validator/impl/candidates-buffer.hpp | 4 +- validator/impl/check-proof.cpp | 17 +- validator/impl/check-proof.hpp | 14 +- validator/impl/collator-impl.h | 31 +- validator/impl/collator.cpp | 79 +- validator/impl/collator.h | 2 +- validator/impl/config.cpp | 5 +- validator/impl/external-message.cpp | 100 +- validator/impl/external-message.hpp | 13 +- validator/impl/fabric.cpp | 36 +- validator/impl/ihr-message.cpp | 7 +- validator/impl/ihr-message.hpp | 4 +- validator/impl/liteserver-cache.hpp | 3 +- validator/impl/liteserver.cpp | 332 +- validator/impl/liteserver.hpp | 41 +- validator/impl/out-msg-queue-proof.cpp | 21 +- validator/impl/out-msg-queue-proof.hpp | 14 +- validator/impl/proof.cpp | 5 +- validator/impl/proof.hpp | 5 +- validator/impl/shard.cpp | 19 +- validator/impl/shard.hpp | 3 +- validator/impl/signature-set.cpp | 7 +- validator/impl/top-shard-descr.cpp | 14 +- validator/impl/top-shard-descr.hpp | 2 +- validator/impl/validate-query.cpp | 57 +- validator/impl/validate-query.hpp | 17 +- validator/impl/validator-set.cpp | 7 +- validator/impl/validator-set.hpp | 12 +- validator/import-db-slice-local.cpp | 19 +- validator/import-db-slice-local.hpp | 2 +- validator/import-db-slice.cpp | 14 +- validator/import-db-slice.hpp | 2 +- validator/interfaces/block-handle.h | 2 +- validator/interfaces/block.h | 2 +- validator/interfaces/config.h | 5 +- validator/interfaces/db.h | 4 +- validator/interfaces/external-message.h | 2 +- validator/interfaces/ihr-message.h | 2 +- validator/interfaces/liteserver.h | 4 +- validator/interfaces/message-queue.h | 2 +- validator/interfaces/out-msg-queue-proof.h | 4 +- validator/interfaces/persistent-state.h | 2 +- validator/interfaces/proof.h | 1 + validator/interfaces/shard-block.h | 3 +- validator/interfaces/shard.h | 7 +- validator/interfaces/validator-full-id.h | 2 +- validator/interfaces/validator-manager.h | 31 +- validator/interfaces/validator-set.h | 7 +- validator/manager-disk.cpp | 16 +- validator/manager-disk.h | 2 +- validator/manager-disk.hpp | 19 +- validator/manager-hardfork.cpp | 16 +- validator/manager-hardfork.h | 2 +- validator/manager-hardfork.hpp | 25 +- validator/manager-init.cpp | 15 +- validator/manager-init.h | 3 +- validator/manager-init.hpp | 11 +- validator/manager.cpp | 52 +- validator/manager.h | 13 +- validator/manager.hpp | 52 +- validator/net/download-archive-slice.cpp | 7 +- validator/net/download-archive-slice.hpp | 5 +- validator/net/download-block-new.cpp | 10 +- validator/net/download-block-new.hpp | 4 +- validator/net/download-block.cpp | 7 +- validator/net/download-block.hpp | 4 +- validator/net/download-next-block.cpp | 7 +- validator/net/download-next-block.hpp | 6 +- validator/net/download-proof.cpp | 7 +- validator/net/download-proof.hpp | 4 +- validator/net/download-state.cpp | 7 +- validator/net/download-state.hpp | 6 +- validator/net/get-next-key-blocks.cpp | 9 +- validator/net/get-next-key-blocks.hpp | 4 +- validator/queue-size-counter.cpp | 3 +- validator/shard-block-retainer.cpp | 3 +- validator/shard-block-retainer.hpp | 3 +- validator/shard-block-verifier.cpp | 4 +- validator/shard-block-verifier.hpp | 3 +- validator/shard-client.cpp | 7 +- validator/shard-client.hpp | 3 +- validator/state-serializer.cpp | 26 +- validator/state-serializer.hpp | 6 +- validator/stats-merger.h | 2 +- validator/stats-provider.h | 5 +- validator/storage-stat-cache.cpp | 2 +- validator/storage-stat-cache.hpp | 4 +- validator/token-manager.h | 4 +- validator/validate-broadcast.cpp | 5 +- validator/validate-broadcast.hpp | 6 +- validator/validator-full-id.cpp | 2 +- validator/validator-group.cpp | 25 +- validator/validator-group.hpp | 12 +- validator/validator-options.cpp | 10 +- validator/validator-options.hpp | 5 +- validator/validator-telemetry.cpp | 7 +- validator/validator-telemetry.hpp | 5 +- validator/validator.h | 37 +- 980 files changed, 11466 insertions(+), 13067 deletions(-) diff --git a/adnl/adnl-address-list.cpp b/adnl/adnl-address-list.cpp index 912943a86a..1dcb62cc7c 100644 --- a/adnl/adnl-address-list.cpp +++ b/adnl/adnl-address-list.cpp @@ -16,12 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "adnl-address-list.hpp" -#include "adnl-peer-table.h" #include "auto/tl/ton_api.hpp" -#include "td/utils/overloaded.h" -#include "td/net/UdpServer.h" #include "keys/encryptor.h" +#include "td/net/UdpServer.h" +#include "td/utils/overloaded.h" + +#include "adnl-address-list.hpp" +#include "adnl-peer-table.h" namespace ton { diff --git a/adnl/adnl-address-list.h b/adnl/adnl-address-list.h index ebc7473a64..9c31335f06 100644 --- a/adnl/adnl-address-list.h +++ b/adnl/adnl-address-list.h @@ -18,9 +18,10 @@ */ #pragma once -#include "adnl-network-manager.h" #include "crypto/common/refcnt.hpp" +#include "adnl-network-manager.h" + namespace ton { namespace adnl { diff --git a/adnl/adnl-address-list.hpp b/adnl/adnl-address-list.hpp index 0c869378ba..86758f9f26 100644 --- a/adnl/adnl-address-list.hpp +++ b/adnl/adnl-address-list.hpp @@ -18,8 +18,8 @@ */ #pragma once -#include "adnl-node-id.hpp" #include "adnl-address-list.h" +#include "adnl-node-id.hpp" namespace ton { diff --git a/adnl/adnl-channel.cpp b/adnl/adnl-channel.cpp index 4da9d2eed5..d6cd4d5f26 100644 --- a/adnl/adnl-channel.cpp +++ b/adnl/adnl-channel.cpp @@ -16,12 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "crypto/Ed25519.h" +#include "td/utils/crypto.h" + #include "adnl-channel.hpp" -#include "adnl-peer.h" #include "adnl-peer-table.h" - -#include "td/utils/crypto.h" -#include "crypto/Ed25519.h" +#include "adnl-peer.h" namespace ton { diff --git a/adnl/adnl-channel.h b/adnl/adnl-channel.h index 94b5877d55..57d171e764 100644 --- a/adnl/adnl-channel.h +++ b/adnl/adnl-channel.h @@ -19,9 +19,9 @@ #pragma once #include "adnl-local-id.h" -#include "adnl-peer.h" -#include "adnl-peer-table.h" #include "adnl-network-manager.h" +#include "adnl-peer-table.h" +#include "adnl-peer.h" namespace ton { diff --git a/adnl/adnl-channel.hpp b/adnl/adnl-channel.hpp index 0c30fbd5b2..7c33d61f47 100644 --- a/adnl/adnl-channel.hpp +++ b/adnl/adnl-channel.hpp @@ -18,9 +18,10 @@ */ #pragma once -#include "adnl-channel.h" #include "keys/encryptor.h" +#include "adnl-channel.h" + namespace ton { namespace adnl { diff --git a/adnl/adnl-db.cpp b/adnl/adnl-db.cpp index c86f3f7e78..ed9b5d38ca 100644 --- a/adnl/adnl-db.cpp +++ b/adnl/adnl-db.cpp @@ -16,9 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "adnl-db.hpp" #include "td/db/RocksDb.h" +#include "adnl-db.hpp" + namespace ton { namespace adnl { diff --git a/adnl/adnl-db.hpp b/adnl/adnl-db.hpp index 0f49374576..d6fdd9a836 100644 --- a/adnl/adnl-db.hpp +++ b/adnl/adnl-db.hpp @@ -18,9 +18,10 @@ */ #pragma once -#include "adnl-db.h" #include "td/db/KeyValue.h" +#include "adnl-db.h" + namespace ton { namespace adnl { diff --git a/adnl/adnl-ext-client.cpp b/adnl/adnl-ext-client.cpp index 873d34b110..503765de62 100644 --- a/adnl/adnl-ext-client.cpp +++ b/adnl/adnl-ext-client.cpp @@ -16,8 +16,8 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "adnl-ext-client.hpp" #include "adnl-ext-client.h" +#include "adnl-ext-client.hpp" namespace ton { diff --git a/adnl/adnl-ext-client.h b/adnl/adnl-ext-client.h index 9e537a5a51..5e2c1a28a3 100644 --- a/adnl/adnl-ext-client.h +++ b/adnl/adnl-ext-client.h @@ -18,9 +18,10 @@ */ #pragma once -#include "adnl-node-id.hpp" #include "td/utils/port/IPAddress.h" +#include "adnl-node-id.hpp" + namespace ton { namespace adnl { diff --git a/adnl/adnl-ext-client.hpp b/adnl/adnl-ext-client.hpp index 76de83ab44..e1fe36ae23 100644 --- a/adnl/adnl-ext-client.hpp +++ b/adnl/adnl-ext-client.hpp @@ -19,12 +19,13 @@ #pragma once #include "auto/tl/lite_api.h" -#include "adnl-ext-connection.hpp" -#include "tl-utils/lite-utils.hpp" -#include "td/utils/Random.h" -#include "adnl-query.h" #include "keys/encryptor.h" +#include "td/utils/Random.h" +#include "tl-utils/lite-utils.hpp" + #include "adnl-ext-client.h" +#include "adnl-ext-connection.hpp" +#include "adnl-query.h" namespace ton { @@ -86,7 +87,7 @@ class AdnlExtClientImpl : public AdnlExtClient { if (!conn_.empty() && conn_.get() == conn) { callback_->on_stop_ready(); conn_ = {}; - for (auto& q : out_queries_) { + for (auto &q : out_queries_) { td::actor::send_closure(q.second, &AdnlQuery::set_error, td::Status::Error(ErrorCode::cancelled)); } alarm_timestamp() = next_create_at_; diff --git a/adnl/adnl-ext-connection.hpp b/adnl/adnl-ext-connection.hpp index ce3c29b3c9..484f2611c0 100644 --- a/adnl/adnl-ext-connection.hpp +++ b/adnl/adnl-ext-connection.hpp @@ -18,15 +18,15 @@ */ #pragma once +#include +#include + +#include "common/errorcode.h" #include "td/net/TcpListener.h" -#include "td/utils/crypto.h" #include "td/utils/BufferedFd.h" -#include "tl-utils/tl-utils.hpp" #include "td/utils/Random.h" -#include "common/errorcode.h" - -#include -#include +#include "td/utils/crypto.h" +#include "tl-utils/tl-utils.hpp" namespace ton { diff --git a/adnl/adnl-ext-server.cpp b/adnl/adnl-ext-server.cpp index 627a25cb51..40b2e8e861 100644 --- a/adnl/adnl-ext-server.cpp +++ b/adnl/adnl-ext-server.cpp @@ -16,8 +16,9 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "adnl-ext-server.hpp" #include "keys/encryptor.h" + +#include "adnl-ext-server.hpp" #include "utils.hpp" namespace ton { diff --git a/adnl/adnl-ext-server.hpp b/adnl/adnl-ext-server.hpp index b4604a76dd..886422fe2a 100644 --- a/adnl/adnl-ext-server.hpp +++ b/adnl/adnl-ext-server.hpp @@ -18,15 +18,16 @@ */ #pragma once -#include "adnl-peer-table.h" +#include +#include + #include "td/net/TcpListener.h" -#include "td/utils/crypto.h" #include "td/utils/BufferedFd.h" +#include "td/utils/crypto.h" + #include "adnl-ext-connection.hpp" #include "adnl-ext-server.h" - -#include -#include +#include "adnl-peer-table.h" namespace ton { diff --git a/adnl/adnl-local-id.cpp b/adnl/adnl-local-id.cpp index e0c62de76b..a26ee3d980 100644 --- a/adnl/adnl-local-id.cpp +++ b/adnl/adnl-local-id.cpp @@ -16,11 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/crypto.h" +#include "keys/encryptor.h" #include "td/utils/Random.h" +#include "td/utils/crypto.h" #include "adnl-local-id.h" -#include "keys/encryptor.h" #include "utils.hpp" namespace ton { @@ -41,7 +41,7 @@ AdnlAddressList AdnlLocalId::get_addr_list() const { } void AdnlLocalId::receive(td::IPAddress addr, td::BufferSlice data) { - InboundRateLimiter& rate_limiter = inbound_rate_limiter_[addr]; + InboundRateLimiter &rate_limiter = inbound_rate_limiter_[addr]; if (!rate_limiter.rate_limiter.take()) { VLOG(ADNL_NOTICE) << this << ": dropping IN message: rate limit exceeded"; add_dropped_packet_stats(addr); @@ -371,7 +371,6 @@ tl_object_ptr AdnlLocalId::PacketStats::tl(b return obj; } - } // namespace adnl } // namespace ton diff --git a/adnl/adnl-local-id.h b/adnl/adnl-local-id.h index fa7f7f749f..4ca4d95481 100644 --- a/adnl/adnl-local-id.h +++ b/adnl/adnl-local-id.h @@ -20,12 +20,11 @@ #include -#include "td/actor/actor.h" -#include "td/utils/BufferedUdp.h" #include "auto/tl/ton_api.h" -#include "keys/encryptor.h" -#include "adnl-peer-table.h" #include "dht/dht.h" +#include "keys/encryptor.h" +#include "td/actor/actor.h" +#include "td/utils/BufferedUdp.h" #include "adnl-peer-table.h" #include "utils.hpp" diff --git a/adnl/adnl-message.h b/adnl/adnl-message.h index 43849e9822..60f5c36375 100644 --- a/adnl/adnl-message.h +++ b/adnl/adnl-message.h @@ -18,8 +18,8 @@ */ #pragma once -#include "adnl/adnl.h" #include "adnl/adnl-query.h" +#include "adnl/adnl.h" namespace ton { diff --git a/adnl/adnl-network-manager.cpp b/adnl/adnl-network-manager.cpp index 077fb93907..fb2fce4715 100644 --- a/adnl/adnl-network-manager.cpp +++ b/adnl/adnl-network-manager.cpp @@ -16,13 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "adnl-network-manager.hpp" -#include "adnl-peer-table.h" - #include "auto/tl/ton_api.hpp" - #include "td/utils/overloaded.h" +#include "adnl-network-manager.hpp" +#include "adnl-peer-table.h" + namespace ton { namespace adnl { diff --git a/adnl/adnl-network-manager.h b/adnl/adnl-network-manager.h index 67cf602aac..642ce1081e 100644 --- a/adnl/adnl-network-manager.h +++ b/adnl/adnl-network-manager.h @@ -18,16 +18,15 @@ */ #pragma once -#include "td/actor/actor.h" +#include #include "td/actor/PromiseFuture.h" +#include "td/actor/actor.h" #include "td/utils/port/IPAddress.h" #include "adnl-node-id.hpp" #include "adnl-proxy-types.h" -#include - namespace td { class UdpServer; } diff --git a/adnl/adnl-network-manager.hpp b/adnl/adnl-network-manager.hpp index a77be19d6f..f457da65c3 100644 --- a/adnl/adnl-network-manager.hpp +++ b/adnl/adnl-network-manager.hpp @@ -18,17 +18,17 @@ */ #pragma once +#include + +#include "td/actor/PromiseFuture.h" #include "td/actor/actor.h" -#include "td/utils/BufferedUdp.h" -#include "td/net/UdpServer.h" #include "td/net/TcpListener.h" +#include "td/net/UdpServer.h" +#include "td/utils/BufferedUdp.h" -#include "td/actor/PromiseFuture.h" #include "adnl-network-manager.h" #include "adnl-received-mask.h" -#include - namespace td { class UdpServer; } diff --git a/adnl/adnl-node-id.cpp b/adnl/adnl-node-id.cpp index d3830aeadb..e383766044 100644 --- a/adnl/adnl-node-id.cpp +++ b/adnl/adnl-node-id.cpp @@ -17,10 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "adnl-node-id.hpp" - #include "common/util.h" +#include "adnl-node-id.hpp" + namespace ton { namespace adnl { td::Result AdnlNodeIdShort::parse(td::Slice id) { diff --git a/adnl/adnl-node-id.hpp b/adnl/adnl-node-id.hpp index 43e3629f4c..4c22256001 100644 --- a/adnl/adnl-node-id.hpp +++ b/adnl/adnl-node-id.hpp @@ -18,8 +18,8 @@ */ #pragma once -#include "keys/keys.hpp" #include "common/io.hpp" +#include "keys/keys.hpp" namespace ton { diff --git a/adnl/adnl-node.h b/adnl/adnl-node.h index 0fc6c2cf05..330a02dd58 100644 --- a/adnl/adnl-node.h +++ b/adnl/adnl-node.h @@ -19,8 +19,9 @@ #pragma once #include "auto/tl/ton_api.h" -#include "adnl-node-id.hpp" + #include "adnl-address-list.h" +#include "adnl-node-id.hpp" namespace ton { diff --git a/adnl/adnl-packet.cpp b/adnl/adnl-packet.cpp index 7159fbdde3..f8f476700c 100644 --- a/adnl/adnl-packet.cpp +++ b/adnl/adnl-packet.cpp @@ -16,9 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "adnl-packet.h" #include "td/utils/Random.h" +#include "adnl-packet.h" + namespace ton { namespace adnl { diff --git a/adnl/adnl-packet.h b/adnl/adnl-packet.h index 363a74c238..6e12a7df88 100644 --- a/adnl/adnl-packet.h +++ b/adnl/adnl-packet.h @@ -18,8 +18,8 @@ */ #pragma once -#include "adnl/adnl.h" #include "adnl/adnl-message.h" +#include "adnl/adnl.h" namespace ton { diff --git a/adnl/adnl-peer-table.cpp b/adnl/adnl-peer-table.cpp index 5a92ca1c11..e4d7e90c83 100644 --- a/adnl/adnl-peer-table.cpp +++ b/adnl/adnl-peer-table.cpp @@ -16,21 +16,19 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "adnl-peer-table.hpp" -#include "adnl-peer.h" -#include "adnl-channel.h" -#include "utils.hpp" - -#include "td/utils/tl_storers.h" +#include "td/db/RocksDb.h" +#include "td/utils/Random.h" #include "td/utils/crypto.h" #include "td/utils/tl_parsers.h" -#include "td/utils/Random.h" -#include "td/db/RocksDb.h" +#include "td/utils/tl_storers.h" -#include "utils.hpp" -#include "adnl-query.h" +#include "adnl-channel.h" #include "adnl-ext-client.h" +#include "adnl-peer-table.hpp" +#include "adnl-peer.h" +#include "adnl-query.h" #include "adnl-tunnel.h" +#include "utils.hpp" namespace ton { @@ -87,22 +85,22 @@ void AdnlPeerTableImpl::receive_packet(td::IPAddress addr, AdnlCategoryMask cat_ void AdnlPeerTableImpl::update_id(AdnlPeerTableImpl::PeerInfo &peer_info, AdnlNodeIdFull &&peer_id) { if (peer_info.peer_id.empty()) { peer_info.peer_id = std::move(peer_id); - for (auto &e: peer_info.peers) { + for (auto &e : peer_info.peers) { td::actor::send_closure(e.second, &AdnlPeerPair::update_peer_id, peer_info.peer_id); } } } -td::actor::ActorOwn &AdnlPeerTableImpl::get_peer_pair( - AdnlNodeIdShort peer_id, AdnlPeerTableImpl::PeerInfo &peer_info, - AdnlNodeIdShort local_id, AdnlPeerTableImpl::LocalIdInfo &local_id_info) { +td::actor::ActorOwn &AdnlPeerTableImpl::get_peer_pair(AdnlNodeIdShort peer_id, + AdnlPeerTableImpl::PeerInfo &peer_info, + AdnlNodeIdShort local_id, + AdnlPeerTableImpl::LocalIdInfo &local_id_info) { auto it = peer_info.peers.find(local_id); if (it == peer_info.peers.end()) { - it = peer_info.peers.emplace(local_id, - AdnlPeerPair::create(network_manager_, actor_id(this), - local_id_info.mode, local_id_info.local_id.get(), - dht_node_, local_id, peer_id)) - .first; + it = peer_info.peers + .emplace(local_id, AdnlPeerPair::create(network_manager_, actor_id(this), local_id_info.mode, + local_id_info.local_id.get(), dht_node_, local_id, peer_id)) + .first; if (!peer_info.peer_id.empty()) { td::actor::send_closure(it->second, &AdnlPeerPair::update_peer_id, peer_info.peer_id); } @@ -146,8 +144,8 @@ void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket update_id(it->second, packet.from()); } - td::actor::send_closure(get_peer_pair(src, it->second, dst, it2->second), - &AdnlPeerPair::receive_packet, std::move(packet), serialized_size); + td::actor::send_closure(get_peer_pair(src, it->second, dst, it2->second), &AdnlPeerPair::receive_packet, + std::move(packet), serialized_size); } void AdnlPeerTableImpl::add_peer(AdnlNodeIdShort local_id, AdnlNodeIdFull id, AdnlAddressList addr_list) { @@ -160,8 +158,8 @@ void AdnlPeerTableImpl::add_peer(AdnlNodeIdShort local_id, AdnlNodeIdFull id, Ad auto &peer_info = peers_[id_short]; update_id(peer_info, std::move(id)); if (!addr_list.empty()) { - td::actor::send_closure(get_peer_pair(id_short, peer_info, local_id, it2->second), - &AdnlPeerPair::update_addr_list, std::move(addr_list)); + td::actor::send_closure(get_peer_pair(id_short, peer_info, local_id, it2->second), &AdnlPeerPair::update_addr_list, + std::move(addr_list)); } } @@ -185,8 +183,8 @@ void AdnlPeerTableImpl::send_message_in(AdnlNodeIdShort src, AdnlNodeIdShort dst std::vector messages; messages.push_back(OutboundAdnlMessage{std::move(message), flags}); - td::actor::send_closure(get_peer_pair(dst, peer_info, src, it2->second), - &AdnlPeerPair::send_messages, std::move(messages)); + td::actor::send_closure(get_peer_pair(dst, peer_info, src, it2->second), &AdnlPeerPair::send_messages, + std::move(messages)); } void AdnlPeerTableImpl::answer_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlQueryId query_id, @@ -214,8 +212,8 @@ void AdnlPeerTableImpl::send_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, std return; } - td::actor::send_closure(get_peer_pair(dst, peer_info, src, it2->second), - &AdnlPeerPair::send_query, name, std::move(promise), timeout, std::move(data), 0); + td::actor::send_closure(get_peer_pair(dst, peer_info, src, it2->second), &AdnlPeerPair::send_query, name, + std::move(promise), timeout, std::move(data), 0); } void AdnlPeerTableImpl::add_id_ex(AdnlNodeIdFull id, AdnlAddressList addr_list, td::uint8 cat, td::uint32 mode) { @@ -411,7 +409,7 @@ void AdnlPeerTableImpl::get_conn_ip_str(AdnlNodeIdShort l_id, AdnlNodeIdShort p_ } void AdnlPeerTableImpl::get_stats_peer(AdnlNodeIdShort peer_id, AdnlPeerTableImpl::PeerInfo &peer_info, bool all, - td::Promise>> promise) { + td::Promise>> promise) { class Cb : public td::actor::Actor { public: explicit Cb(td::Promise>> promise) @@ -447,17 +445,17 @@ void AdnlPeerTableImpl::get_stats_peer(AdnlNodeIdShort peer_id, AdnlPeerTableImp for (auto &[local_id, peer_pair] : peer_info.peers) { td::actor::send_closure(callback, &Cb::inc_pending); - td::actor::send_closure(peer_pair, &AdnlPeerPair::get_stats, all, - [local_id = local_id, peer_id = peer_id, - callback](td::Result> R) { - if (R.is_error()) { - VLOG(ADNL_NOTICE) << "failed to get stats for peer pair " << peer_id << "->" << local_id - << " : " << R.move_as_error(); - td::actor::send_closure(callback, &Cb::dec_pending); - } else { - td::actor::send_closure(callback, &Cb::got_peer_pair_stats, R.move_as_ok()); - } - }); + td::actor::send_closure( + peer_pair, &AdnlPeerPair::get_stats, all, + [local_id = local_id, peer_id = peer_id, callback](td::Result> R) { + if (R.is_error()) { + VLOG(ADNL_NOTICE) << "failed to get stats for peer pair " << peer_id << "->" << local_id << " : " + << R.move_as_error(); + td::actor::send_closure(callback, &Cb::dec_pending); + } else { + td::actor::send_closure(callback, &Cb::got_peer_pair_stats, R.move_as_ok()); + } + }); } td::actor::send_closure(callback, &Cb::dec_pending); } @@ -530,16 +528,15 @@ void AdnlPeerTableImpl::get_stats(bool all, td::Promise>> R) { - if (R.is_error()) { - VLOG(ADNL_NOTICE) << "failed to get stats for peer " << id << " : " << R.move_as_error(); - td::actor::send_closure(callback, &Cb::dec_pending); - } else { - td::actor::send_closure(callback, &Cb::got_peer_stats, R.move_as_ok()); - } - }); + get_stats_peer(id, peer, all, + [id = id, callback](td::Result>> R) { + if (R.is_error()) { + VLOG(ADNL_NOTICE) << "failed to get stats for peer " << id << " : " << R.move_as_error(); + td::actor::send_closure(callback, &Cb::dec_pending); + } else { + td::actor::send_closure(callback, &Cb::got_peer_stats, R.move_as_ok()); + } + }); } td::actor::send_closure(callback, &Cb::dec_pending); } diff --git a/adnl/adnl-peer-table.h b/adnl/adnl-peer-table.h index 13b4adb537..8eb2af2bbd 100644 --- a/adnl/adnl-peer-table.h +++ b/adnl/adnl-peer-table.h @@ -18,18 +18,16 @@ */ #pragma once +#include "adnl/adnl-db.h" +#include "adnl/adnl-query.h" +#include "auto/tl/ton_api.h" +#include "common/io.hpp" #include "td/actor/actor.h" #include "td/utils/BufferedUdp.h" +#include "adnl-packet.h" #include "adnl.h" #include "utils.hpp" -#include "adnl/adnl-query.h" -#include "adnl/adnl-db.h" -#include "common/io.hpp" - -#include "adnl-packet.h" - -#include "auto/tl/ton_api.h" namespace ton { diff --git a/adnl/adnl-peer-table.hpp b/adnl/adnl-peer-table.hpp index 408f846d0d..2b02aa39be 100644 --- a/adnl/adnl-peer-table.hpp +++ b/adnl/adnl-peer-table.hpp @@ -21,15 +21,16 @@ #include #include +#include "keys/encryptor.h" + #include "adnl-peer-table.h" #include "adnl-peer.h" -#include "keys/encryptor.h" //#include "adnl-decryptor.h" +#include "adnl-address-list.h" +#include "adnl-ext-server.h" #include "adnl-local-id.h" #include "adnl-query.h" #include "utils.hpp" -#include "adnl-ext-server.h" -#include "adnl-address-list.h" namespace ton { @@ -144,9 +145,10 @@ class AdnlPeerTableImpl : public AdnlPeerTable { //td::uint64 last_query_id_ = 1; static void update_id(PeerInfo &peer_info, AdnlNodeIdFull &&peer_id); - td::actor::ActorOwn &get_peer_pair(AdnlNodeIdShort peer_id, PeerInfo &peer_info, AdnlNodeIdShort local_id, LocalIdInfo &local_id_info); + td::actor::ActorOwn &get_peer_pair(AdnlNodeIdShort peer_id, PeerInfo &peer_info, + AdnlNodeIdShort local_id, LocalIdInfo &local_id_info); static void get_stats_peer(AdnlNodeIdShort peer_id, PeerInfo &peer_info, bool all, - td::Promise>> promise); + td::Promise>> promise); }; inline td::StringBuilder &operator<<(td::StringBuilder &sb, const AdnlPeerTableImpl::PrintId &id) { diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index 4913216ee4..e89850dce7 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -16,18 +16,17 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "adnl-peer.h" -#include "adnl-peer.hpp" -#include "adnl-local-id.h" - -#include "utils.hpp" - +#include "auto/tl/ton_api.h" #include "td/actor/PromiseFuture.h" -#include "td/utils/base64.h" #include "td/utils/Random.h" -#include "auto/tl/ton_api.h" +#include "td/utils/base64.h" #include "td/utils/overloaded.h" +#include "adnl-local-id.h" +#include "adnl-peer.h" +#include "adnl-peer.hpp" +#include "utils.hpp" + namespace ton { namespace adnl { @@ -475,9 +474,8 @@ void AdnlPeerPairImpl::alarm_query(AdnlQueryId id) { AdnlPeerPairImpl::AdnlPeerPairImpl(td::actor::ActorId network_manager, td::actor::ActorId peer_table, td::uint32 local_mode, - td::actor::ActorId local_actor, - td::actor::ActorId dht_node, AdnlNodeIdShort local_id, - AdnlNodeIdShort peer_id) { + td::actor::ActorId local_actor, td::actor::ActorId dht_node, + AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id) { network_manager_ = network_manager; peer_table_ = peer_table; local_actor_ = local_actor; @@ -886,10 +884,12 @@ void AdnlPeerPairImpl::conn_change_state(AdnlConnectionIdShort id, bool ready) { } } -td::actor::ActorOwn AdnlPeerPair::create( - td::actor::ActorId network_manager, td::actor::ActorId peer_table, - td::uint32 local_mode, td::actor::ActorId local_actor, - td::actor::ActorId dht_node, AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id) { +td::actor::ActorOwn AdnlPeerPair::create(td::actor::ActorId network_manager, + td::actor::ActorId peer_table, + td::uint32 local_mode, + td::actor::ActorId local_actor, + td::actor::ActorId dht_node, AdnlNodeIdShort local_id, + AdnlNodeIdShort peer_id) { auto X = td::actor::create_actor("peerpair", network_manager, peer_table, local_mode, local_actor, dht_node, local_id, peer_id); return td::actor::ActorOwn(std::move(X)); diff --git a/adnl/adnl-peer.h b/adnl/adnl-peer.h index 4d6812a891..604cade42e 100644 --- a/adnl/adnl-peer.h +++ b/adnl/adnl-peer.h @@ -18,13 +18,13 @@ */ #pragma once +#include "auto/tl/ton_api.h" +#include "dht/dht.h" #include "td/actor/actor.h" #include "td/utils/BufferedUdp.h" -#include "dht/dht.h" #include "adnl-peer-table.h" #include "utils.hpp" -#include "auto/tl/ton_api.h" namespace ton { diff --git a/adnl/adnl-peer.hpp b/adnl/adnl-peer.hpp index b1d9835789..ddea18f1ed 100644 --- a/adnl/adnl-peer.hpp +++ b/adnl/adnl-peer.hpp @@ -18,20 +18,19 @@ */ #pragma once -#include #include #include - -#include "adnl-peer.h" -#include "adnl-peer-table.h" -#include "adnl-network-manager.h" -#include "keys/encryptor.h" -#include "adnl-channel.h" -#include "adnl-query.h" +#include #include "crypto/Ed25519.h" +#include "keys/encryptor.h" #include "td/utils/DecTree.h" +#include "adnl-channel.h" +#include "adnl-network-manager.h" +#include "adnl-peer-table.h" +#include "adnl-peer.h" +#include "adnl-query.h" #include "utils.hpp" namespace ton { @@ -60,8 +59,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair { AdnlPeerPairImpl(td::actor::ActorId network_manager, td::actor::ActorId peer_table, td::uint32 local_mode, td::actor::ActorId local_actor, - td::actor::ActorId dht_node, AdnlNodeIdShort local_id, - AdnlNodeIdShort peer_id); + td::actor::ActorId dht_node, AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id); void start_up() override; void alarm() override; diff --git a/adnl/adnl-pong.cpp b/adnl/adnl-pong.cpp index 434f0ef7b4..767013b796 100644 --- a/adnl/adnl-pong.cpp +++ b/adnl/adnl-pong.cpp @@ -25,22 +25,24 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include + +#include "adnl/adnl.h" +#include "auto/tl/ton_api_json.h" +#include "common/checksum.h" +#include "common/errorcode.h" #include "td/actor/actor.h" -#include "td/utils/buffer.h" -#include "td/utils/port/IPAddress.h" #include "td/net/UdpServer.h" -#include "td/utils/port/signals.h" -#include "td/utils/OptionParser.h" #include "td/utils/FileLog.h" +#include "td/utils/OptionParser.h" +#include "td/utils/buffer.h" +#include "td/utils/filesystem.h" +#include "td/utils/port/IPAddress.h" #include "td/utils/port/path.h" +#include "td/utils/port/signals.h" #include "td/utils/port/user.h" -#include "td/utils/filesystem.h" -#include "common/checksum.h" -#include "common/errorcode.h" #include "tl-utils/tl-utils.hpp" -#include "auto/tl/ton_api_json.h" -#include "adnl/adnl.h" -#include + #include "git.h" #if TD_DARWIN || TD_LINUX @@ -99,7 +101,8 @@ int main(int argc, char *argv[]) { SET_VERBOSITY_LEVEL(v); }); p.add_option('V', "version", "shows adnl-pong build information", [&]() { - std::cout << "adnl-pong build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "adnl-pong build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_option('h', "help", "prints_help", [&]() { diff --git a/adnl/adnl-proxy-types.cpp b/adnl/adnl-proxy-types.cpp index 8ca2def917..a39f797728 100644 --- a/adnl/adnl-proxy-types.cpp +++ b/adnl/adnl-proxy-types.cpp @@ -16,12 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "adnl-proxy-types.hpp" -#include "tl-utils/tl-utils.hpp" #include "auto/tl/ton_api.hpp" -#include "td/utils/overloaded.h" -#include "td/utils/Time.h" #include "common/errorcode.h" +#include "td/utils/Time.h" +#include "td/utils/overloaded.h" +#include "tl-utils/tl-utils.hpp" + +#include "adnl-proxy-types.hpp" namespace ton { diff --git a/adnl/adnl-proxy-types.h b/adnl/adnl-proxy-types.h index f18d2dc04d..62358f3b32 100644 --- a/adnl/adnl-proxy-types.h +++ b/adnl/adnl-proxy-types.h @@ -18,9 +18,9 @@ */ #pragma once -#include "td/utils/buffer.h" -#include "td/utils/Status.h" #include "auto/tl/ton_api.h" +#include "td/utils/Status.h" +#include "td/utils/buffer.h" namespace ton { diff --git a/adnl/adnl-proxy-types.hpp b/adnl/adnl-proxy-types.hpp index 7b2182880e..47e259a86f 100644 --- a/adnl/adnl-proxy-types.hpp +++ b/adnl/adnl-proxy-types.hpp @@ -18,10 +18,10 @@ */ #pragma once -#include "adnl-proxy-types.h" - #include "common/checksum.h" +#include "adnl-proxy-types.h" + namespace ton { namespace adnl { diff --git a/adnl/adnl-proxy.cpp b/adnl/adnl-proxy.cpp index 81ec8dcd0a..d432ad8f17 100644 --- a/adnl/adnl-proxy.cpp +++ b/adnl/adnl-proxy.cpp @@ -25,24 +25,26 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include + +#include "auto/tl/ton_api_json.h" +#include "common/checksum.h" +#include "common/errorcode.h" #include "td/actor/actor.h" -#include "td/utils/buffer.h" -#include "td/utils/port/IPAddress.h" #include "td/net/UdpServer.h" -#include "td/utils/port/signals.h" -#include "td/utils/OptionParser.h" #include "td/utils/FileLog.h" -#include "td/utils/port/path.h" -#include "td/utils/port/user.h" +#include "td/utils/OptionParser.h" +#include "td/utils/buffer.h" #include "td/utils/filesystem.h" #include "td/utils/overloaded.h" -#include "common/checksum.h" -#include "common/errorcode.h" +#include "td/utils/port/IPAddress.h" +#include "td/utils/port/path.h" +#include "td/utils/port/signals.h" +#include "td/utils/port/user.h" #include "tl-utils/tl-utils.hpp" -#include "auto/tl/ton_api_json.h" + #include "adnl-proxy-types.h" #include "adnl-received-mask.h" -#include #include "git.h" #if TD_DARWIN || TD_LINUX @@ -304,7 +306,8 @@ int main(int argc, char *argv[]) { SET_VERBOSITY_LEVEL(v); }); p.add_option('V', "version", "shows adnl-proxy build information", [&]() { - std::cout << "adnl-proxy build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "adnl-proxy build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_option('h', "help", "prints_help", [&]() { diff --git a/adnl/adnl-query.cpp b/adnl/adnl-query.cpp index 89dbbfc496..0ce210b2fa 100644 --- a/adnl/adnl-query.cpp +++ b/adnl/adnl-query.cpp @@ -16,10 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "adnl-query.h" #include "common/errorcode.h" #include "td/utils/Random.h" +#include "adnl-query.h" + namespace ton { namespace adnl { diff --git a/adnl/adnl-query.h b/adnl/adnl-query.h index 3db8c754c1..50838c8fb9 100644 --- a/adnl/adnl-query.h +++ b/adnl/adnl-query.h @@ -18,13 +18,13 @@ */ #pragma once -#include "td/actor/actor.h" +#include + #include "common/bitstring.h" #include "common/errorcode.h" +#include "td/actor/actor.h" #include "td/utils/buffer.h" -#include - namespace ton { namespace adnl { diff --git a/adnl/adnl-test-loopback-implementation.h b/adnl/adnl-test-loopback-implementation.h index 4773f05395..6cf1a59ee8 100644 --- a/adnl/adnl-test-loopback-implementation.h +++ b/adnl/adnl-test-loopback-implementation.h @@ -18,11 +18,11 @@ */ #pragma once +#include + #include "adnl/adnl.h" #include "td/utils/Random.h" -#include - namespace ton { namespace adnl { diff --git a/adnl/adnl-tunnel.cpp b/adnl/adnl-tunnel.cpp index 360196f4ea..b0d316a6c2 100644 --- a/adnl/adnl-tunnel.cpp +++ b/adnl/adnl-tunnel.cpp @@ -16,8 +16,8 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "adnl-tunnel.h" #include "adnl-peer-table.h" +#include "adnl-tunnel.h" namespace ton { diff --git a/adnl/adnl-tunnel.h b/adnl/adnl-tunnel.h index d7f6a66fa1..868e007163 100644 --- a/adnl/adnl-tunnel.h +++ b/adnl/adnl-tunnel.h @@ -18,11 +18,12 @@ */ #pragma once -#include "adnl.h" -#include "adnl-peer-table.h" +#include + #include "keys/encryptor.h" -#include +#include "adnl-peer-table.h" +#include "adnl.h" namespace ton { diff --git a/adnl/adnl.h b/adnl/adnl.h index b49581a912..ba0d948942 100644 --- a/adnl/adnl.h +++ b/adnl/adnl.h @@ -18,13 +18,14 @@ */ #pragma once -#include "td/actor/actor.h" #include "auto/tl/ton_api.h" +#include "common/errorcode.h" +#include "keyring/keyring.h" +#include "td/actor/actor.h" #include "td/utils/port/IPAddress.h" + #include "adnl-node-id.hpp" #include "adnl-node.h" -#include "common/errorcode.h" -#include "keyring/keyring.h" namespace ton { diff --git a/adnl/test/adnl-test-ping.cpp b/adnl/test/adnl-test-ping.cpp index 289aa043f8..818487abb5 100644 --- a/adnl/test/adnl-test-ping.cpp +++ b/adnl/test/adnl-test-ping.cpp @@ -25,16 +25,16 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include +#include + #include "adnl/adnl-network-manager.h" #include "adnl/adnl-peer-table.h" #include "adnl/utils.hpp" #include "keys/encryptor.h" +#include "td/utils/OptionParser.h" #include "td/utils/Time.h" #include "td/utils/format.h" -#include "td/utils/OptionParser.h" - -#include -#include template std::ostream &operator<<(std::ostream &stream, const td::UInt &x) { diff --git a/adnl/utils.cpp b/adnl/utils.cpp index cb05f39696..20aa582c98 100644 --- a/adnl/utils.cpp +++ b/adnl/utils.cpp @@ -16,12 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "utils.hpp" -#include "tl/tl_object_store.h" -#include "td/utils/format.h" -#include "td/utils/Random.h" -#include "keys/encryptor.h" #include "auto/tl/ton_api.hpp" +#include "keys/encryptor.h" +#include "td/utils/Random.h" +#include "td/utils/format.h" +#include "tl/tl_object_store.h" + #include "adnl-node-id.hpp" +#include "utils.hpp" namespace ton {} // namespace ton diff --git a/adnl/utils.hpp b/adnl/utils.hpp index 18d3f2074e..ebfcfabc98 100644 --- a/adnl/utils.hpp +++ b/adnl/utils.hpp @@ -18,19 +18,19 @@ */ #pragma once +#include "common/checksum.h" +#include "common/errorcode.h" +#include "common/status.h" +#include "td/utils/base64.h" #include "td/utils/buffer.h" -#include "td/utils/misc.h" #include "td/utils/crypto.h" #include "td/utils/format.h" -#include "td/utils/base64.h" +#include "td/utils/misc.h" #include "tl-utils/tl-utils.hpp" -#include "common/errorcode.h" -#include "common/checksum.h" +#include "adnl-address-list.hpp" #include "adnl-node-id.hpp" -#include "common/status.h" #include "adnl-node.h" -#include "adnl-address-list.hpp" namespace ton { @@ -41,8 +41,9 @@ inline bool adnl_node_is_older(AdnlNode &a, AdnlNode &b) { } class RateLimiter { -public: - explicit RateLimiter(td::uint32 capacity, double period) : capacity_(capacity), period_(period), remaining_(capacity) { + public: + explicit RateLimiter(td::uint32 capacity, double period) + : capacity_(capacity), period_(period), remaining_(capacity) { } bool take() { @@ -67,7 +68,7 @@ class RateLimiter { return increment_at_; } -private: + private: td::uint32 capacity_; double period_; td::uint32 remaining_; diff --git a/blockchain-explorer/blockchain-explorer-http.cpp b/blockchain-explorer/blockchain-explorer-http.cpp index f07e0e9f42..1ef086406c 100644 --- a/blockchain-explorer/blockchain-explorer-http.cpp +++ b/blockchain-explorer/blockchain-explorer-http.cpp @@ -25,17 +25,18 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "blockchain-explorer-http.hpp" +#include "block/block-auto.h" #include "block/block-db.h" -#include "block/block.h" #include "block/block-parse.h" -#include "block/block-auto.h" +#include "block/block.h" +#include "block/mc-config.h" +#include "td/utils/date.h" +#include "ton/ton-shard.h" #include "vm/boc.h" #include "vm/cellops.h" #include "vm/cells/MerkleProof.h" -#include "block/mc-config.h" -#include "ton/ton-shard.h" -#include "td/utils/date.h" + +#include "blockchain-explorer-http.hpp" bool local_scripts{false}; @@ -132,7 +133,7 @@ HttpAnswer& HttpAnswer::operator<<(MessageCell msg) { << "destination" << AddressCell{info.dest} << "\n" << "lt" << info.created_lt << "\n" << "time" << info.created_at << " (" << time_to_human(info.created_at) << ")\n" - << "value" << currency_collection.to_str()<< "\n"; + << "value" << currency_collection.to_str() << "\n"; break; } default: diff --git a/blockchain-explorer/blockchain-explorer-http.hpp b/blockchain-explorer/blockchain-explorer-http.hpp index e21f1b7eb1..c0f4f15874 100644 --- a/blockchain-explorer/blockchain-explorer-http.hpp +++ b/blockchain-explorer/blockchain-explorer-http.hpp @@ -27,11 +27,11 @@ */ #pragma once +#include "block/block.h" +#include "td/utils/Random.h" #include "ton/ton-types.h" #include "vm/boc.h" #include "vm/cellops.h" -#include "td/utils/Random.h" -#include "block/block.h" extern bool local_scripts; @@ -136,7 +136,7 @@ class HttpAnswer { td::Ref root; T x; template - RawData(td::Ref root, Args &&... args) : root(std::move(root)), x(std::forward(args)...) { + RawData(td::Ref root, Args &&...args) : root(std::move(root)), x(std::forward(args)...) { } }; diff --git a/blockchain-explorer/blockchain-explorer-query.cpp b/blockchain-explorer/blockchain-explorer-query.cpp index 6919c5c14c..c07f3e216b 100644 --- a/blockchain-explorer/blockchain-explorer-query.cpp +++ b/blockchain-explorer/blockchain-explorer-query.cpp @@ -26,29 +26,25 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "blockchain-explorer-query.hpp" -#include "blockchain-explorer-http.hpp" -#include "block/mc-config.h" -#include "crypto/block/check-proof.h" - #include "auto/tl/lite_api.h" - -#include "tl-utils/tl-utils.hpp" -#include "tl-utils/lite-utils.hpp" - -#include "ton/ton-tl.hpp" -#include "ton/lite-tl.hpp" - -#include "common/errorcode.h" #include "block/block-auto.h" +#include "block/mc-config.h" +#include "common/errorcode.h" +#include "crypto/block/check-proof.h" #include "crypto/vm/utils.h" #include "td/utils/crypto.h" - +#include "tl-utils/lite-utils.hpp" +#include "tl-utils/tl-utils.hpp" +#include "ton/lite-tl.hpp" +#include "ton/ton-tl.hpp" #include "vm/boc.h" #include "vm/cellops.h" #include "vm/cells/MerkleProof.h" -#include "vm/vm.h" #include "vm/cp0.h" +#include "vm/vm.h" + +#include "blockchain-explorer-http.hpp" +#include "blockchain-explorer-query.hpp" td::Result parse_block_id(std::map &opts, bool allow_empty) { if (allow_empty) { diff --git a/blockchain-explorer/blockchain-explorer-query.hpp b/blockchain-explorer/blockchain-explorer-query.hpp index 29501265bb..911d8ec298 100644 --- a/blockchain-explorer/blockchain-explorer-query.hpp +++ b/blockchain-explorer/blockchain-explorer-query.hpp @@ -27,14 +27,14 @@ */ #pragma once +#include +#include + +#include "block/block.h" #include "td/actor/actor.h" #include "ton/ton-types.h" -#include "block/block.h" -#include "blockchain-explorer.hpp" -#include - -#include +#include "blockchain-explorer.hpp" td::Result parse_block_id(std::map &opts, bool allow_empty = false); td::Result parse_account_addr(std::map &opts); @@ -334,4 +334,3 @@ class HttpQueryStatus : public HttpQueryCommon { private: CoreActorInterface::RemoteNodeStatusList results_; }; - diff --git a/blockchain-explorer/blockchain-explorer.cpp b/blockchain-explorer/blockchain-explorer.cpp index d21bc465ce..1a0d909a38 100644 --- a/blockchain-explorer/blockchain-explorer.cpp +++ b/blockchain-explorer/blockchain-explorer.cpp @@ -25,45 +25,41 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include + #include "adnl/adnl-ext-client.h" #include "adnl/utils.hpp" +#include "auto/tl/lite_api.h" #include "auto/tl/ton_api_json.h" +#include "block/block-auto.h" +#include "block/block-db.h" +#include "block/block.h" +#include "block/mc-config.h" +#include "lite-client/ext-client.h" #include "td/utils/OptionParser.h" +#include "td/utils/Random.h" #include "td/utils/Time.h" +#include "td/utils/crypto.h" #include "td/utils/filesystem.h" #include "td/utils/format.h" -#include "td/utils/Random.h" -#include "td/utils/crypto.h" +#include "td/utils/port/FileFd.h" #include "td/utils/port/signals.h" #include "td/utils/port/user.h" -#include "td/utils/port/FileFd.h" +#include "tl-utils/lite-utils.hpp" +#include "ton/lite-tl.hpp" #include "ton/ton-tl.hpp" -#include "block/block-db.h" -#include "block/block.h" -#include "block/block-auto.h" -#include "vm/boc.h" -#include "vm/cellops.h" -#include "vm/cells/MerkleProof.h" -#include "block/mc-config.h" -#include "blockchain-explorer.hpp" -#include "blockchain-explorer-http.hpp" -#include "blockchain-explorer-query.hpp" - #include "vm/boc.h" #include "vm/cellops.h" #include "vm/cells/MerkleProof.h" #include "vm/vm.h" -#include "auto/tl/lite_api.h" -#include "ton/lite-tl.hpp" -#include "tl-utils/lite-utils.hpp" -#include "lite-client/ext-client.h" - -#include +#include "blockchain-explorer-http.hpp" +#include "blockchain-explorer-query.hpp" +#include "blockchain-explorer.hpp" #if TD_DARWIN || TD_LINUX -#include #include +#include #endif #include #include @@ -265,8 +261,8 @@ class CoreActor : public CoreActorInterface { MHD_destroy_post_processor(postprocessor); } static MHD_RESULT iterate_post(void* coninfo_cls, enum MHD_ValueKind kind, const char* key, const char* filename, - const char* content_type, const char* transfer_encoding, const char* data, uint64_t off, - size_t size) { + const char* content_type, const char* transfer_encoding, const char* data, + uint64_t off, size_t size) { auto ptr = static_cast(coninfo_cls); ptr->total_size += strlen(key) + size; if (ptr->total_size > MAX_POST_SIZE) { @@ -292,8 +288,9 @@ class CoreActor : public CoreActorInterface { } } - static MHD_RESULT process_http_request(void* cls, struct MHD_Connection* connection, const char* url, const char* method, - const char* version, const char* upload_data, size_t* upload_data_size, void** ptr) { + static MHD_RESULT process_http_request(void* cls, struct MHD_Connection* connection, const char* url, + const char* method, const char* version, const char* upload_data, + size_t* upload_data_size, void** ptr) { struct MHD_Response* response = nullptr; MHD_RESULT ret; diff --git a/blockchain-explorer/blockchain-explorer.hpp b/blockchain-explorer/blockchain-explorer.hpp index abbbde9e9b..432552ed52 100644 --- a/blockchain-explorer/blockchain-explorer.hpp +++ b/blockchain-explorer/blockchain-explorer.hpp @@ -27,11 +27,12 @@ */ #pragma once +#include + #include "td/actor/actor.h" #include "td/utils/buffer.h" -#include "ton/ton-types.h" #include "td/utils/port/IPAddress.h" -#include +#include "ton/ton-types.h" #define MAX_POST_SIZE (64 << 10) diff --git a/catchain/catchain-block.cpp b/catchain/catchain-block.cpp index e12c89f9ec..bb3b033dff 100644 --- a/catchain/catchain-block.cpp +++ b/catchain/catchain-block.cpp @@ -32,9 +32,9 @@ std::unique_ptr CatChainBlock::create(td::uint32 src, td::uint32 } CatChainBlockImpl::CatChainBlockImpl(td::uint32 src, td::uint32 fork, const PublicKeyHash &src_hash, - CatChainBlockHeight height, const CatChainBlockHash &hash, - td::SharedSlice payload, CatChainBlock *prev, - std::vector deps, std::vector vt) + CatChainBlockHeight height, const CatChainBlockHash &hash, td::SharedSlice payload, + CatChainBlock *prev, std::vector deps, + std::vector vt) : src_(src) , fork_(fork) , src_hash_(src_hash) diff --git a/catchain/catchain-received-block.cpp b/catchain/catchain-received-block.cpp index 5996e9678e..13065009ba 100644 --- a/catchain/catchain-received-block.cpp +++ b/catchain/catchain-received-block.cpp @@ -18,11 +18,11 @@ */ #include +#include "auto/tl/ton_api.hpp" + #include "catchain-received-block.hpp" #include "catchain-receiver-source.h" -#include "auto/tl/ton_api.hpp" - namespace ton { namespace catchain { diff --git a/catchain/catchain-received-block.h b/catchain/catchain-received-block.h index 3c4f5d765c..9a5a65b85a 100644 --- a/catchain/catchain-received-block.h +++ b/catchain/catchain-received-block.h @@ -18,11 +18,10 @@ */ #pragma once -#include "td/utils/buffer.h" -#include "td/utils/SharedSlice.h" #include "auto/tl/ton_api.h" - #include "catchain/catchain-receiver.h" +#include "td/utils/SharedSlice.h" +#include "td/utils/buffer.h" namespace ton { @@ -86,20 +85,17 @@ class CatChainReceivedBlock { static tl_object_ptr block_id(const CatChainReceiver *chain, const tl_object_ptr &block); static CatChainBlockHash block_hash(const CatChainReceiver *chain, - const tl_object_ptr &block, - const td::Slice &payload); + const tl_object_ptr &block, const td::Slice &payload); static CatChainBlockHash block_hash(const CatChainReceiver *chain, const tl_object_ptr &block); static td::Status pre_validate_block(const CatChainReceiver *chain, - const tl_object_ptr &block, - const td::Slice &payload); + const tl_object_ptr &block, const td::Slice &payload); static td::Status pre_validate_block(const CatChainReceiver *chain, const tl_object_ptr &block); static CatChainBlockPayloadHash data_payload_hash(const CatChainReceiver *chain, const tl_object_ptr &data, const td::Slice &payload); - virtual ~CatChainReceivedBlock() = default; }; @@ -120,4 +116,3 @@ inline td::StringBuilder &operator<<(td::StringBuilder &sb, const ton::catchain: } } // namespace td - diff --git a/catchain/catchain-receiver-interface.h b/catchain/catchain-receiver-interface.h index bc02832aef..c05fb77da4 100644 --- a/catchain/catchain-receiver-interface.h +++ b/catchain/catchain-receiver-interface.h @@ -18,9 +18,10 @@ */ #pragma once -#include "td/actor/actor.h" #include "adnl/adnl.h" #include "overlay/overlays.h" +#include "td/actor/actor.h" + #include "catchain-types.h" namespace ton { diff --git a/catchain/catchain-receiver-source.cpp b/catchain/catchain-receiver-source.cpp index e758b3359d..2275f39e8e 100644 --- a/catchain/catchain-receiver-source.cpp +++ b/catchain/catchain-receiver-source.cpp @@ -16,9 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "catchain-receiver-source.hpp" #include "common/errorlog.h" +#include "catchain-receiver-source.hpp" + namespace ton { namespace catchain { diff --git a/catchain/catchain-receiver-source.h b/catchain/catchain-receiver-source.h index 136906a02c..4817cc3556 100644 --- a/catchain/catchain-receiver-source.h +++ b/catchain/catchain-receiver-source.h @@ -18,10 +18,10 @@ */ #pragma once -#include "catchain-receiver.h" - #include "keys/encryptor.h" +#include "catchain-receiver.h" + namespace ton { namespace catchain { diff --git a/catchain/catchain-receiver-source.hpp b/catchain/catchain-receiver-source.hpp index cf08c42196..b04c4b9a50 100644 --- a/catchain/catchain-receiver-source.hpp +++ b/catchain/catchain-receiver-source.hpp @@ -19,11 +19,11 @@ #pragma once #include +#include +#include "catchain-received-block.h" #include "catchain-receiver-source.h" #include "catchain-receiver.h" -#include "catchain-received-block.h" -#include namespace ton { diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp index b663cfc06f..5c2f56a6fb 100644 --- a/catchain/catchain-receiver.cpp +++ b/catchain/catchain-receiver.cpp @@ -18,17 +18,17 @@ */ #include #include + +#include "common/delay.h" #include "td/actor/PromiseFuture.h" -#include "td/utils/Random.h" #include "td/db/RocksDb.h" -#include "td/utils/port/path.h" +#include "td/utils/Random.h" +#include "td/utils/ThreadSafeCounter.h" #include "td/utils/overloaded.h" -#include "common/delay.h" +#include "td/utils/port/path.h" #include "catchain-receiver.hpp" -#include "td/utils/ThreadSafeCounter.h" - namespace ton { namespace catchain { @@ -101,8 +101,8 @@ void CatChainReceiverImpl::receive_block(adnl::AdnlNodeIdShort src, tl_object_pt td::uint64 max_block_height = get_max_block_height(opts_, sources_.size()); if ((td::uint32)block->height_ > max_block_height) { - VLOG(CATCHAIN_WARNING) << this << ": received too many blocks from " << src - << " (limit=" << max_block_height << ")"; + VLOG(CATCHAIN_WARNING) << this << ": received too many blocks from " << src << " (limit=" << max_block_height + << ")"; return; } @@ -140,16 +140,15 @@ void CatChainReceiverImpl::receive_block(adnl::AdnlNodeIdShort src, tl_object_pt create_block(std::move(block), td::SharedSlice{payload.as_slice()}); if (!opts_.debug_disable_db) { - db_.set( - id, std::move(raw_data), [](td::Unit) {}, 1.0); + db_.set(id, std::move(raw_data), [](td::Unit) {}, 1.0); } block_written_to_db(id); } void CatChainReceiverImpl::receive_block_answer(adnl::AdnlNodeIdShort src, td::BufferSlice data) { if (data.size() > opts_.max_serialized_block_size) { - VLOG(CATCHAIN_INFO) << this << ": received bad block result " << src << ": too big (size=" - << data.size() << ", limit=" << opts_.max_serialized_block_size << ")"; + VLOG(CATCHAIN_INFO) << this << ": received bad block result " << src << ": too big (size=" << data.size() + << ", limit=" << opts_.max_serialized_block_size << ")"; return; } auto F = fetch_tl_prefix(data, true); @@ -159,10 +158,9 @@ void CatChainReceiverImpl::receive_block_answer(adnl::AdnlNodeIdShort src, td::B } auto f = F.move_as_ok(); ton_api::downcast_call( - *f, - td::overloaded( - [&](ton_api::catchain_blockNotFound &r) { VLOG(CATCHAIN_INFO) << this << ": catchain block not found"; }, - [&](ton_api::catchain_blockResult &r) { receive_block(src, std::move(r.block_), std::move(data)); })); + *f, td::overloaded( + [&](ton_api::catchain_blockNotFound &r) { VLOG(CATCHAIN_INFO) << this << ": catchain block not found"; }, + [&](ton_api::catchain_blockResult &r) { receive_block(src, std::move(r.block_), std::move(data)); })); } void CatChainReceiverImpl::receive_message_from_overlay(adnl::AdnlNodeIdShort src, td::BufferSlice data) { @@ -171,8 +169,8 @@ void CatChainReceiverImpl::receive_message_from_overlay(adnl::AdnlNodeIdShort sr } if (data.size() > opts_.max_serialized_block_size) { - VLOG(CATCHAIN_WARNING) << this << ": dropping broken block from " << src << ": too big (size=" - << data.size() << ", limit=" << opts_.max_serialized_block_size << ")"; + VLOG(CATCHAIN_WARNING) << this << ": dropping broken block from " << src << ": too big (size=" << data.size() + << ", limit=" << opts_.max_serialized_block_size << ")"; return; } auto R = fetch_tl_prefix(data, true); @@ -461,17 +459,13 @@ void CatChainReceiverImpl::debug_add_fork(td::BufferSlice payload, CatChainBlock td::actor::send_closure_later(keyring_, &keyring::Keyring::sign_message, local_id_, std::move(id_s), std::move(P)); } -CatChainReceiverImpl::CatChainReceiverImpl(std::unique_ptr callback, - const CatChainOptions &opts, +CatChainReceiverImpl::CatChainReceiverImpl(std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId overlay_manager, - const std::vector &ids, - const PublicKeyHash &local_id, - const CatChainSessionId &unique_hash, - std::string db_root, - std::string db_suffix, - bool allow_unsafe_self_blocks_resync) + const std::vector &ids, const PublicKeyHash &local_id, + const CatChainSessionId &unique_hash, std::string db_root, + std::string db_suffix, bool allow_unsafe_self_blocks_resync) : callback_(std::move(callback)) , opts_(opts) , keyring_(std::move(keyring)) @@ -648,10 +642,10 @@ void CatChainReceiverImpl::read_db() { read_db_ = true; next_rotate_ = td::Timestamp::in(td::Random::fast(NEIGHBOURS_ROTATE_INTERVAL_MIN, NEIGHBOURS_ROTATE_INTERVAL_MAX)); - next_sync_ = td::Timestamp::in( - 0.001 * td::Random::fast(NEIGHBOURS_ROTATE_INTERVAL_MIN, NEIGHBOURS_ROTATE_INTERVAL_MAX)); - initial_sync_complete_at_ = td::Timestamp::in( - allow_unsafe_self_blocks_resync_ ? EXPECTED_UNSAFE_INITIAL_SYNC_DURATION : EXPECTED_INITIAL_SYNC_DURATION); + next_sync_ = + td::Timestamp::in(0.001 * td::Random::fast(NEIGHBOURS_ROTATE_INTERVAL_MIN, NEIGHBOURS_ROTATE_INTERVAL_MAX)); + initial_sync_complete_at_ = td::Timestamp::in(allow_unsafe_self_blocks_resync_ ? EXPECTED_UNSAFE_INITIAL_SYNC_DURATION + : EXPECTED_INITIAL_SYNC_DURATION); alarm_timestamp().relax(next_rotate_); alarm_timestamp().relax(next_sync_); alarm_timestamp().relax(initial_sync_complete_at_); @@ -1046,7 +1040,7 @@ void CatChainReceiverImpl::block_written_to_db(CatChainBlockHash hash) { run_scheduler(); } -static void destroy_db(const std::string& name, td::uint32 attempt) { +static void destroy_db(const std::string &name, td::uint32 attempt) { auto S = td::RocksDb::destroy(name); if (S.is_ok()) { return; @@ -1082,7 +1076,7 @@ void CatChainReceiverImpl::destroy() { stop(); } -td::uint64 get_max_block_height(const CatChainOptions& opts, size_t sources_cnt) { +td::uint64 get_max_block_height(const CatChainOptions &opts, size_t sources_cnt) { if (opts.max_block_height_coeff == 0) { return std::numeric_limits::max(); } diff --git a/catchain/catchain-receiver.h b/catchain/catchain-receiver.h index 75c8735114..4b271a16cf 100644 --- a/catchain/catchain-receiver.h +++ b/catchain/catchain-receiver.h @@ -22,8 +22,8 @@ #include "adnl/utils.hpp" #include "auto/tl/ton_api.h" -#include "catchain-types.h" #include "catchain-receiver-interface.h" +#include "catchain-types.h" namespace ton { @@ -68,7 +68,7 @@ class CatChainReceiver : public CatChainReceiverInterface { virtual ~CatChainReceiver() = default; }; -td::uint64 get_max_block_height(const CatChainOptions& opts, size_t sources_cnt); +td::uint64 get_max_block_height(const CatChainOptions &opts, size_t sources_cnt); } // namespace catchain diff --git a/catchain/catchain-receiver.hpp b/catchain/catchain-receiver.hpp index 5c4d37648a..e25329413d 100644 --- a/catchain/catchain-receiver.hpp +++ b/catchain/catchain-receiver.hpp @@ -19,16 +19,16 @@ #pragma once #include -#include #include - -#include "catchain-types.h" -#include "catchain-receiver.h" -#include "catchain-receiver-source.h" -#include "catchain-received-block.h" +#include #include "td/db/KeyValueAsync.h" +#include "catchain-received-block.h" +#include "catchain-receiver-source.h" +#include "catchain-receiver.h" +#include "catchain-types.h" + namespace ton { namespace catchain { diff --git a/catchain/catchain-types.h b/catchain/catchain-types.h index 4031acc51e..9eb208ea87 100644 --- a/catchain/catchain-types.h +++ b/catchain/catchain-types.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/utils/int_types.h" #include "adnl/adnl-node-id.hpp" +#include "td/utils/int_types.h" #include "ton/ton-types.h" namespace ton { diff --git a/catchain/catchain.cpp b/catchain/catchain.cpp index 53ae16b529..6b205d98c4 100644 --- a/catchain/catchain.cpp +++ b/catchain/catchain.cpp @@ -16,11 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "catchain-types.h" -#include "catchain.hpp" - #include + #include "catchain-receiver.h" +#include "catchain-types.h" +#include "catchain.hpp" namespace ton { @@ -208,7 +208,8 @@ void CatChainImpl::on_blame(td::uint32 src_id) { td::actor::send_closure(receiver_, &CatChainReceiverInterface::on_blame_processed, src_id); } -void CatChainImpl::on_custom_query(const PublicKeyHash &src, td::BufferSlice data, td::Promise promise) { +void CatChainImpl::on_custom_query(const PublicKeyHash &src, td::BufferSlice data, + td::Promise promise) { callback_->process_query(src, std::move(data), std::move(promise)); } @@ -229,8 +230,8 @@ void CatChainImpl::on_receiver_started() { CatChainImpl::CatChainImpl(std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId overlay_manager, std::vector ids, - const PublicKeyHash &local_id, const CatChainSessionId &unique_hash, - std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync) + const PublicKeyHash &local_id, const CatChainSessionId &unique_hash, std::string db_root, + std::string db_suffix, bool allow_unsafe_self_blocks_resync) : opts_(opts) , unique_hash_(unique_hash) , db_root_(std::move(db_root)) @@ -271,7 +272,8 @@ void CatChainImpl::start_up() { void blame(td::uint32 src_id) override { td::actor::send_closure(id_, &CatChainImpl::on_blame, src_id); } - void on_custom_query(const PublicKeyHash &src, td::BufferSlice data, td::Promise promise) override { + void on_custom_query(const PublicKeyHash &src, td::BufferSlice data, + td::Promise promise) override { td::actor::send_closure(id_, &CatChainImpl::on_custom_query, src, std::move(data), std::move(promise)); } void on_broadcast(const PublicKeyHash &src, td::BufferSlice data) override { @@ -289,9 +291,9 @@ void CatChainImpl::start_up() { auto cb = std::make_unique(actor_id(this)); - receiver_ = CatChainReceiverInterface::create( - std::move(cb), opts_, args_->keyring, args_->adnl, args_->overlay_manager, args_->ids, args_->local_id, - args_->unique_hash, db_root_, db_suffix_, allow_unsafe_self_blocks_resync_); + receiver_ = CatChainReceiverInterface::create(std::move(cb), opts_, args_->keyring, args_->adnl, + args_->overlay_manager, args_->ids, args_->local_id, args_->unique_hash, + db_root_, db_suffix_, allow_unsafe_self_blocks_resync_); args_ = nullptr; //alarm_timestamp() = td::Timestamp::in(opts_.idle_timeout); } @@ -303,10 +305,9 @@ td::actor::ActorOwn CatChain::create(std::unique_ptr callbac std::vector ids, const PublicKeyHash &local_id, const CatChainSessionId &unique_hash, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync) { - return td::actor::create_actor("catchain", std::move(callback), opts, std::move(keyring), - std::move(adnl), std::move(overlay_manager), std::move(ids), - local_id, unique_hash, std::move(db_root), std::move(db_suffix), - allow_unsafe_self_blocks_resync); + return td::actor::create_actor( + "catchain", std::move(callback), opts, std::move(keyring), std::move(adnl), std::move(overlay_manager), + std::move(ids), local_id, unique_hash, std::move(db_root), std::move(db_suffix), allow_unsafe_self_blocks_resync); } CatChainBlock *CatChainImpl::get_block(CatChainBlockHash hash) const { diff --git a/catchain/catchain.h b/catchain/catchain.h index c5c8af28d1..110939b3f0 100644 --- a/catchain/catchain.h +++ b/catchain/catchain.h @@ -21,6 +21,7 @@ #include "adnl/adnl.h" #include "adnl/utils.hpp" #include "overlay/overlays.h" + #include "catchain-types.h" namespace ton { diff --git a/catchain/catchain.hpp b/catchain/catchain.hpp index 586cf4744d..73f4f29ef0 100644 --- a/catchain/catchain.hpp +++ b/catchain/catchain.hpp @@ -20,11 +20,12 @@ #include -#include "catchain.h" -#include "catchain-types.h" -#include "catchain-receiver-interface.h" #include "td/utils/DecTree.h" +#include "catchain-receiver-interface.h" +#include "catchain-types.h" +#include "catchain.h" + namespace ton { namespace catchain { diff --git a/common/checksum.h b/common/checksum.h index ff69223951..480a3a9bed 100644 --- a/common/checksum.h +++ b/common/checksum.h @@ -18,9 +18,9 @@ */ #pragma once -#include "td/utils/int_types.h" #include "crypto/common/bitstring.h" #include "td/utils/crypto.h" +#include "td/utils/int_types.h" namespace td { diff --git a/common/delay.h b/common/delay.h index 3df8e7d864..a2aac19924 100644 --- a/common/delay.h +++ b/common/delay.h @@ -53,7 +53,7 @@ void delay_action(T promise, td::Timestamp timeout) { template class AsyncApply : public td::actor::Actor { public: - AsyncApply(PromiseT promise, ValueT value) : promise_(std::move(promise)), value_(std::move(value)){ + AsyncApply(PromiseT promise, ValueT value) : promise_(std::move(promise)), value_(std::move(value)) { } void start_up() override { @@ -61,7 +61,7 @@ class AsyncApply : public td::actor::Actor { stop(); } - static void create(td::Slice name, PromiseT promise, ValueT value ) { + static void create(td::Slice name, PromiseT promise, ValueT value) { td::actor::create_actor(PSLICE() << "async:" << name, std::move(promise), std::move(value)).release(); } diff --git a/common/errorlog.cpp b/common/errorlog.cpp index a06166986a..a871c5a0b5 100644 --- a/common/errorlog.cpp +++ b/common/errorlog.cpp @@ -16,15 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "errorlog.h" -#include "checksum.h" +#include -#include "td/utils/port/FileFd.h" +#include "td/utils/Time.h" #include "td/utils/filesystem.h" +#include "td/utils/port/FileFd.h" #include "td/utils/port/path.h" -#include "td/utils/Time.h" -#include +#include "checksum.h" +#include "errorlog.h" namespace ton { diff --git a/common/errorlog.h b/common/errorlog.h index 884bfc4d68..3fa3196ece 100644 --- a/common/errorlog.h +++ b/common/errorlog.h @@ -19,6 +19,7 @@ #pragma once #include + #include "td/utils/buffer.h" namespace ton { diff --git a/common/global-version.h b/common/global-version.h index d88a22c32a..1eddf96463 100644 --- a/common/global-version.h +++ b/common/global-version.h @@ -21,4 +21,4 @@ namespace ton { // See doc/GlobalVersions.md constexpr int SUPPORTED_VERSION = 12; -} +} // namespace ton diff --git a/common/int-to-string.hpp b/common/int-to-string.hpp index 88aecc6549..7067223f92 100644 --- a/common/int-to-string.hpp +++ b/common/int-to-string.hpp @@ -18,8 +18,8 @@ */ #pragma once -#include "td/utils/int_types.h" #include "td/utils/Slice.h" +#include "td/utils/int_types.h" namespace ton { diff --git a/common/io.hpp b/common/io.hpp index e328636f94..037d37b8b0 100644 --- a/common/io.hpp +++ b/common/io.hpp @@ -18,19 +18,17 @@ */ #pragma once +#include "common/errorcode.h" +#include "common/status.h" +#include "crypto/common/bitstring.h" +#include "keys/keys.hpp" +#include "td/utils/base64.h" #include "td/utils/buffer.h" -#include "td/utils/misc.h" #include "td/utils/crypto.h" #include "td/utils/format.h" -#include "td/utils/base64.h" +#include "td/utils/misc.h" #include "tl-utils/tl-utils.hpp" -#include "common/errorcode.h" -#include "common/status.h" -#include "keys/keys.hpp" - -#include "crypto/common/bitstring.h" - namespace td { template diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index aadf877ca1..c8840b3a7c 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -28,37 +28,34 @@ #include "adnl/adnl.h" #include "adnl/utils.hpp" #include "auto/tl/ton_api_json.h" +#include "catchain/catchain.h" +#include "common/errorlog.h" +#include "crypto/block/block-db.h" +#include "crypto/vm/vm.h" #include "dht/dht.h" #include "overlay/overlays.h" #include "td/utils/OptionParser.h" +#include "td/utils/Random.h" #include "td/utils/Time.h" #include "td/utils/filesystem.h" #include "td/utils/format.h" -#include "td/utils/Random.h" -#include "td/utils/port/signals.h" #include "td/utils/port/FileFd.h" -#include "catchain/catchain.h" -#include "validator-session/validator-session.h" -#include "validator/manager-hardfork.h" -#include "td/utils/filesystem.h" #include "td/utils/port/path.h" - -#include "ton/ton-types.h" -#include "ton/ton-tl.hpp" +#include "td/utils/port/signals.h" #include "ton/ton-io.hpp" - +#include "ton/ton-tl.hpp" +#include "ton/ton-types.h" +#include "validator-session/validator-session.h" #include "validator/fabric.h" #include "validator/impl/collator.h" -#include "crypto/vm/vm.h" -#include "crypto/block/block-db.h" - -#include "common/errorlog.h" +#include "validator/manager-hardfork.h" #if TD_DARWIN || TD_LINUX #include #endif #include #include + #include "git.h" int verbosity; @@ -151,11 +148,11 @@ class HardforkCreator : public td::actor::Actor { } td::Status create_validator_options() { - if(!global_config_.length()) { + if (!global_config_.length()) { opts_ = ton::validator::ValidatorManagerOptions::create( - ton::BlockIdExt{ton::masterchainId, ton::shardIdAll, 0, ton::RootHash::zero(), ton::FileHash::zero()}, - ton::BlockIdExt{ton::masterchainId, ton::shardIdAll, 0, ton::RootHash::zero(), ton::FileHash::zero()}); - return td::Status::OK(); + ton::BlockIdExt{ton::masterchainId, ton::shardIdAll, 0, ton::RootHash::zero(), ton::FileHash::zero()}, + ton::BlockIdExt{ton::masterchainId, ton::shardIdAll, 0, ton::RootHash::zero(), ton::FileHash::zero()}); + return td::Status::OK(); } TRY_RESULT_PREFIX(conf_data, td::read_file(global_config_), "failed to read: "); TRY_RESULT_PREFIX(conf_json, td::json_decode(conf_data.as_slice()), "failed to parse json: "); @@ -175,7 +172,7 @@ class HardforkCreator : public td::actor::Actor { std::vector h; for (auto &x : conf.validator_->hardforks_) { auto b = ton::create_block_id(x); - if (!b.is_masterchain()) { + if (!b.is_masterchain()) { return td::Status::Error(ton::ErrorCode::error, "[validator/hardforks] section contains not masterchain block id"); } @@ -327,13 +324,15 @@ int main(int argc, char *argv[]) { std::exit(2); }); p.add_option('V', "version", "shows create-hardfork build information", [&]() { - std::cout << "create-hardfork build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "create-hardfork build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_option('D', "db", "root for dbs", [&](td::Slice fname) { td::actor::send_closure(x, &HardforkCreator::set_db_root, fname.str()); }); - p.add_option('C', "config", "global config path", - [&](td::Slice fname) { td::actor::send_closure(x, &HardforkCreator::set_global_config_path, fname.str()); }); + p.add_option('C', "config", "global config path", [&](td::Slice fname) { + td::actor::send_closure(x, &HardforkCreator::set_global_config_path, fname.str()); + }); p.add_option('m', "ext-message", "binary file with serialized inbound external message", [&](td::Slice fname) { td::actor::send_closure(x, &HardforkCreator::load_ext_message, fname.str()); }); p.add_option( diff --git a/crypto/Ed25519.cpp b/crypto/Ed25519.cpp index 20fb8a3ca2..cb5414e027 100644 --- a/crypto/Ed25519.cpp +++ b/crypto/Ed25519.cpp @@ -16,20 +16,18 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "crypto/Ed25519.h" - +#include #include +#include +#include -#include "td/utils/base64.h" +#include "crypto/Ed25519.h" #include "td/utils/BigNum.h" +#include "td/utils/ScopeGuard.h" +#include "td/utils/base64.h" #include "td/utils/format.h" #include "td/utils/logging.h" #include "td/utils/misc.h" -#include "td/utils/ScopeGuard.h" - -#include -#include -#include namespace td { diff --git a/crypto/Ed25519.h b/crypto/Ed25519.h index a3340d2eca..8a801a54f1 100644 --- a/crypto/Ed25519.h +++ b/crypto/Ed25519.h @@ -18,9 +18,9 @@ */ #pragma once -#include "td/utils/common.h" #include "td/utils/SharedSlice.h" #include "td/utils/Status.h" +#include "td/utils/common.h" #if TD_HAVE_OPENSSL diff --git a/crypto/block/Binlog.cpp b/crypto/block/Binlog.cpp index b774e69314..fc5b055c2c 100644 --- a/crypto/block/Binlog.cpp +++ b/crypto/block/Binlog.cpp @@ -16,14 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "crypto/block/Binlog.h" +#include +#include "crypto/block/Binlog.h" #include "td/utils/as.h" #include "td/utils/misc.h" #include "td/utils/port/path.h" -#include - namespace block { /* * diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h index 4ada076b88..12ce4c78a2 100644 --- a/crypto/block/account-storage-stat.h +++ b/crypto/block/account-storage-stat.h @@ -15,13 +15,14 @@ along with TON Blockchain. If not, see . */ #pragma once +#include "common/bitstring.h" #include "common/refcnt.hpp" -#include "vm/dict.h" -#include "ton/ton-types.h" #include "ton/ton-shard.h" -#include "common/bitstring.h" -#include "block.h" +#include "ton/ton-types.h" #include "vm/db/CellHashTable.h" +#include "vm/dict.h" + +#include "block.h" namespace block { using td::Ref; diff --git a/crypto/block/adjust-block.cpp b/crypto/block/adjust-block.cpp index 16c77691b8..6bc270fd9f 100644 --- a/crypto/block/adjust-block.cpp +++ b/crypto/block/adjust-block.cpp @@ -25,15 +25,17 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include +#include + #include "block/block.h" +#include "td/utils/crypto.h" #include "vm/boc.h" -#include -#include "block-db.h" +#include "vm/cp0.h" + #include "block-auto.h" +#include "block-db.h" #include "block-parse.h" -#include "vm/cp0.h" -#include "td/utils/crypto.h" -#include #include "git.h" using td::Ref; @@ -158,7 +160,8 @@ int main(int argc, char* const argv[]) { new_verbosity_level = VERBOSITY_NAME(FATAL) + (verbosity = td::to_integer(td::Slice(optarg))); break; case 'V': - std::cout << "adjust-block build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "adjust-block build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); break; default: diff --git a/crypto/block/block-db-impl.h b/crypto/block/block-db-impl.h index 07869ca956..d7e99bdd49 100644 --- a/crypto/block/block-db-impl.h +++ b/crypto/block/block-db-impl.h @@ -17,12 +17,13 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include #include -#include "vm/cells.h" +#include + #include "block/Binlog.h" -#include "block/block-db.h" #include "block/block-binlog.h" +#include "block/block-db.h" +#include "vm/cells.h" namespace block { using td::Ref; diff --git a/crypto/block/block-db.cpp b/crypto/block/block-db.cpp index 21c7c0a0a9..d9eef85d40 100644 --- a/crypto/block/block-db.cpp +++ b/crypto/block/block-db.cpp @@ -16,21 +16,22 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "block-db.h" -#include "block-db-impl.h" -#include "block-binlog.h" +#include + #include "td/utils/common.h" #include "td/utils/crypto.h" +#include "td/utils/filesystem.h" #include "td/utils/format.h" #include "td/utils/misc.h" #include "td/utils/port/FileFd.h" #include "td/utils/port/path.h" -#include "td/utils/filesystem.h" -#include "vm/cellslice.h" #include "vm/boc.h" +#include "vm/cellslice.h" #include "vm/db/StaticBagOfCellsDb.h" -#include +#include "block-binlog.h" +#include "block-db-impl.h" +#include "block-db.h" namespace block { diff --git a/crypto/block/block-db.h b/crypto/block/block-db.h index cd9498fbf8..1cd609bcfe 100644 --- a/crypto/block/block-db.h +++ b/crypto/block/block-db.h @@ -17,12 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/utils/int_types.h" -#include "td/utils/buffer.h" -#include "td/actor/actor.h" -#include "ton/ton-types.h" #include "crypto/common/refcnt.hpp" #include "crypto/vm/cells.h" +#include "td/actor/actor.h" +#include "td/utils/buffer.h" +#include "td/utils/int_types.h" +#include "ton/ton-types.h" namespace block { diff --git a/crypto/block/block-parse.cpp b/crypto/block/block-parse.cpp index 9443b69f37..fdfb3172d2 100644 --- a/crypto/block/block-parse.cpp +++ b/crypto/block/block-parse.cpp @@ -16,12 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/bits.h" -#include "block/block-parse.h" #include "block/block-auto.h" -#include "ton/ton-shard.h" +#include "block/block-parse.h" #include "common/util.h" +#include "td/utils/bits.h" #include "td/utils/crypto.h" +#include "ton/ton-shard.h" namespace block { using namespace std::literals::string_literals; @@ -47,7 +47,7 @@ bool debug(int x) { } // namespace #define DBG_START int dbg = 0; -#define DBG debug(++dbg)&& +#define DBG debug(++dbg) && #define DEB_START DBG_START #define DEB DBG @@ -1353,29 +1353,29 @@ bool TrComputePhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const const TrComputePhase t_TrComputePhase; bool TrActionPhase::skip(vm::CellSlice& cs) const { - return cs.advance(3) // success:Bool valid:Bool no_funds:Bool - && t_AccStatusChange.skip(cs) // status_change:AccStatusChange - && t_Maybe_Grams.skip(cs) // total_fwd_fees:(Maybe Grams) - && t_Maybe_Grams.skip(cs) // total_action_fees:(Maybe Grams) - && cs.advance(32) // result_code:int32 - && Maybe{32}.skip(cs) // result_arg:(Maybe int32) - && cs.advance(16 * 4 + 256) // tot_actions:uint16 spec_actions:uint16 - // skipped_actions:uint16 msgs_created:uint16 - // action_list_hash:uint256 - && t_StorageUsed.skip(cs); // tot_msg_size:StorageUsed + return cs.advance(3) // success:Bool valid:Bool no_funds:Bool + && t_AccStatusChange.skip(cs) // status_change:AccStatusChange + && t_Maybe_Grams.skip(cs) // total_fwd_fees:(Maybe Grams) + && t_Maybe_Grams.skip(cs) // total_action_fees:(Maybe Grams) + && cs.advance(32) // result_code:int32 + && Maybe{32}.skip(cs) // result_arg:(Maybe int32) + && cs.advance(16 * 4 + 256) // tot_actions:uint16 spec_actions:uint16 + // skipped_actions:uint16 msgs_created:uint16 + // action_list_hash:uint256 + && t_StorageUsed.skip(cs); // tot_msg_size:StorageUsed } bool TrActionPhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { - return cs.advance(3) // success:Bool valid:Bool no_funds:Bool - && t_AccStatusChange.validate_skip(ops, cs, weak) // status_change:AccStatusChange - && t_Maybe_Grams.validate_skip(ops, cs, weak) // total_fwd_fees:(Maybe Grams) - && t_Maybe_Grams.validate_skip(ops, cs, weak) // total_action_fees:(Maybe Grams) - && cs.advance(32) // result_code:int32 - && Maybe{32}.validate_skip(ops, cs, weak) // result_arg:(Maybe int32) - && cs.advance(16 * 4 + 256) // tot_actions:uint16 spec_actions:uint16 - // skipped_actions:uint16 msgs_created:uint16 - // action_list_hash:uint256 - && t_StorageUsed.validate_skip(ops, cs, weak); // tot_msg_size:StorageUsed + return cs.advance(3) // success:Bool valid:Bool no_funds:Bool + && t_AccStatusChange.validate_skip(ops, cs, weak) // status_change:AccStatusChange + && t_Maybe_Grams.validate_skip(ops, cs, weak) // total_fwd_fees:(Maybe Grams) + && t_Maybe_Grams.validate_skip(ops, cs, weak) // total_action_fees:(Maybe Grams) + && cs.advance(32) // result_code:int32 + && Maybe{32}.validate_skip(ops, cs, weak) // result_arg:(Maybe int32) + && cs.advance(16 * 4 + 256) // tot_actions:uint16 spec_actions:uint16 + // skipped_actions:uint16 msgs_created:uint16 + // action_list_hash:uint256 + && t_StorageUsed.validate_skip(ops, cs, weak); // tot_msg_size:StorageUsed } const TrActionPhase t_TrActionPhase; @@ -1385,14 +1385,14 @@ bool TrBouncePhase::skip(vm::CellSlice& cs) const { case tr_phase_bounce_negfunds: return cs.advance(2); // tr_phase_bounce_negfunds$00 case tr_phase_bounce_nofunds: - return cs.advance(2) // tr_phase_bounce_nofunds$01 + return cs.advance(2) // tr_phase_bounce_nofunds$01 && t_StorageUsed.skip(cs) // msg_size:StorageUsed - && t_Grams.skip(cs); // req_fwd_fees:Grams + && t_Grams.skip(cs); // req_fwd_fees:Grams case tr_phase_bounce_ok: - return cs.advance(1) // tr_phase_bounce_ok$1 + return cs.advance(1) // tr_phase_bounce_ok$1 && t_StorageUsed.skip(cs) // msg_size:StorageUsed - && t_Grams.skip(cs) // msg_fees:Grams - && t_Grams.skip(cs); // fwd_fees:Grams + && t_Grams.skip(cs) // msg_fees:Grams + && t_Grams.skip(cs); // fwd_fees:Grams } return false; } @@ -1402,14 +1402,14 @@ bool TrBouncePhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const case tr_phase_bounce_negfunds: return cs.advance(2); // tr_phase_bounce_negfunds$00 case tr_phase_bounce_nofunds: - return cs.advance(2) // tr_phase_bounce_nofunds$01 + return cs.advance(2) // tr_phase_bounce_nofunds$01 && t_StorageUsed.validate_skip(ops, cs, weak) // msg_size:StorageUsed - && t_Grams.validate_skip(ops, cs, weak); // req_fwd_fees:Grams + && t_Grams.validate_skip(ops, cs, weak); // req_fwd_fees:Grams case tr_phase_bounce_ok: - return cs.advance(1) // tr_phase_bounce_ok$1 + return cs.advance(1) // tr_phase_bounce_ok$1 && t_StorageUsed.validate_skip(ops, cs, weak) // msg_size:StorageUsed - && t_Grams.validate_skip(ops, cs, weak) // msg_fees:Grams - && t_Grams.validate_skip(ops, cs, weak); // fwd_fees:Grams + && t_Grams.validate_skip(ops, cs, weak) // msg_fees:Grams + && t_Grams.validate_skip(ops, cs, weak); // fwd_fees:Grams } return false; } @@ -1843,7 +1843,7 @@ bool InMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return false; } -static td::RefInt256 get_ihr_fee(const CommonMsgInfo::Record_int_msg_info &info, int global_version) { +static td::RefInt256 get_ihr_fee(const CommonMsgInfo::Record_int_msg_info& info, int global_version) { // Legacy: extra_flags was previously ihr_fee return global_version >= 12 ? td::zero_refint() : t_Grams.as_integer(std::move(info.extra_flags)); } @@ -2441,7 +2441,7 @@ bool Aug_ShardFees::eval_leaf(vm::CellBuilder& cb, vm::CellSlice& cs) const { const Aug_ShardFees aug_ShardFees; -bool validate_message_libs(const td::Ref &cell) { +bool validate_message_libs(const td::Ref& cell) { gen::Message::Record rec; if (!type_unpack_cell(cell, gen::t_Message_Any, rec)) { return false; @@ -2457,7 +2457,7 @@ bool validate_message_libs(const td::Ref &cell) { } } -bool validate_message_relaxed_libs(const td::Ref &cell) { +bool validate_message_relaxed_libs(const td::Ref& cell) { gen::MessageRelaxed::Record rec; if (!type_unpack_cell(cell, gen::t_MessageRelaxed_Any, rec)) { return false; diff --git a/crypto/block/block-parse.h b/crypto/block/block-parse.h index 2eb8c5d778..0e49d1cf99 100644 --- a/crypto/block/block-parse.h +++ b/crypto/block/block-parse.h @@ -17,17 +17,19 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include + +#include "block/block.h" #include "common/refcnt.hpp" +#include "td/utils/StringBuilder.h" +#include "td/utils/bits.h" +#include "tl/tlblib.hpp" +#include "ton/ton-types.h" +#include "vm/boc.h" #include "vm/cells.h" #include "vm/cellslice.h" #include "vm/dict.h" -#include "vm/boc.h" -#include "block/block.h" -#include -#include "tl/tlblib.hpp" -#include "td/utils/bits.h" -#include "td/utils/StringBuilder.h" -#include "ton/ton-types.h" + #include "block-auto.h" namespace block { diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index 0893d630f9..d7b3cc1b83 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -16,18 +16,18 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/bits.h" -#include "block/block.h" #include "block/block-auto.h" #include "block/block-parse.h" +#include "block/block.h" #include "block/mc-config.h" -#include "ton/ton-shard.h" #include "common/bigexp.h" #include "common/util.h" +#include "td/utils/Random.h" +#include "td/utils/bits.h" #include "td/utils/crypto.h" -#include "td/utils/tl_storers.h" #include "td/utils/misc.h" -#include "td/utils/Random.h" +#include "td/utils/tl_storers.h" +#include "ton/ton-shard.h" #include "vm/fmt.hpp" namespace block { @@ -2560,5 +2560,4 @@ bool MsgMetadata::operator!=(const MsgMetadata& other) const { return !(*this == other); } - } // namespace block diff --git a/crypto/block/block.h b/crypto/block/block.h index c83fb67018..c8f40d11e1 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -17,18 +17,19 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include + #include "common/refcnt.hpp" +#include "td/utils/CancellationToken.h" +#include "td/utils/StringBuilder.h" +#include "td/utils/bits.h" +#include "tl/tlblib.hpp" +#include "ton/ton-types.h" +#include "vm/boc.h" #include "vm/cells.h" #include "vm/cellslice.h" #include "vm/dict.h" -#include "vm/boc.h" #include "vm/stack.hpp" -#include -#include "tl/tlblib.hpp" -#include "td/utils/bits.h" -#include "td/utils/CancellationToken.h" -#include "td/utils/StringBuilder.h" -#include "ton/ton-types.h" namespace block { diff --git a/crypto/block/check-proof.cpp b/crypto/block/check-proof.cpp index 82eaa5b48a..8153acddb9 100644 --- a/crypto/block/check-proof.cpp +++ b/crypto/block/check-proof.cpp @@ -16,17 +16,16 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "check-proof.h" -#include "block/block.h" -#include "block/block-parse.h" #include "block/block-auto.h" +#include "block/block-parse.h" +#include "block/block.h" #include "block/mc-config.h" - +#include "openssl/digest.hpp" #include "ton/ton-shard.h" - #include "vm/cells/MerkleProof.h" -#include "openssl/digest.hpp" + #include "Ed25519.h" +#include "check-proof.h" namespace block { using namespace std::literals::string_literals; @@ -321,8 +320,8 @@ td::Result BlockTransaction::validate(bool check_proof) } if (check_proof && proof->get_hash().bits().compare(root->get_hash().bits(), 256)) { return td::Status::Error(PSLICE() << "transaction hash mismatch: Merkle proof expects " - << proof->get_hash().bits().to_hex(256) - << " but received data has " << root->get_hash().bits().to_hex(256)); + << proof->get_hash().bits().to_hex(256) << " but received data has " + << root->get_hash().bits().to_hex(256)); } block::gen::Transaction::Record trans; if (!tlb::unpack_cell(root, trans)) { @@ -340,7 +339,8 @@ td::Result BlockTransaction::validate(bool check_proof) td::Result BlockTransactionList::validate(bool check_proof) const { constexpr int max_answer_transactions = 256; - TRY_RESULT_PREFIX(list, vm::std_boc_deserialize_multi(std::move(transactions_boc)), "cannot deserialize transactions boc: "); + TRY_RESULT_PREFIX(list, vm::std_boc_deserialize_multi(std::move(transactions_boc)), + "cannot deserialize transactions boc: "); std::vector> tx_proofs(list.size()); if (check_proof) { @@ -357,7 +357,7 @@ td::Result BlockTransactionList::validate(bool check return td::Status::Error("Error unpacking proof cell"); } vm::AugmentedDictionary acc_dict{vm::load_cell_slice_ref(extra.account_blocks), 256, - block::tlb::aug_ShardAccountBlocks}; + block::tlb::aug_ShardAccountBlocks}; bool eof = false; ton::LogicalTime reverse = reverse_mode ? ~0ULL : 0; @@ -367,7 +367,7 @@ td::Result BlockTransactionList::validate(bool check int count = 0; while (!eof && count < req_count && count < max_answer_transactions) { auto value = acc_dict.extract_value( - acc_dict.vm::DictionaryFixed::lookup_nearest_key(cur_addr.bits(), 256, !reverse, allow_same)); + acc_dict.vm::DictionaryFixed::lookup_nearest_key(cur_addr.bits(), 256, !reverse, allow_same)); if (value.is_null()) { eof = true; break; @@ -382,11 +382,11 @@ td::Result BlockTransactionList::validate(bool check return td::Status::Error("Error unpacking proof account block"); } vm::AugmentedDictionary trans_dict{vm::DictNonEmpty(), std::move(acc_blk.transactions), 64, - block::tlb::aug_AccountTransactions}; + block::tlb::aug_AccountTransactions}; td::BitArray<64> cur_trans{(long long)trans_lt}; while (count < req_count && count < max_answer_transactions) { auto tvalue = trans_dict.extract_value_ref( - trans_dict.vm::DictionaryFixed::lookup_nearest_key(cur_trans.bits(), 64, !reverse)); + trans_dict.vm::DictionaryFixed::lookup_nearest_key(cur_trans.bits(), 64, !reverse)); if (tvalue.is_null()) { trans_lt = reverse; break; @@ -398,7 +398,8 @@ td::Result BlockTransactionList::validate(bool check } } if (static_cast(count) != list.size()) { - return td::Status::Error(PSLICE() << "Txs count mismatch in proof (" << count << ") and response (" << list.size() << ")"); + return td::Status::Error(PSLICE() << "Txs count mismatch in proof (" << count << ") and response (" + << list.size() << ")"); } } catch (vm::VmError& err) { return err.as_status("Couldn't verify proof: "); diff --git a/crypto/block/check-proof.h b/crypto/block/check-proof.h index 497a4ebaaf..9579b94d91 100644 --- a/crypto/block/check-proof.h +++ b/crypto/block/check-proof.h @@ -18,8 +18,8 @@ */ #pragma once -#include "vm/cells.h" #include "block/block.h" +#include "vm/cells.h" namespace block { using td::Ref; diff --git a/crypto/block/create-state.cpp b/crypto/block/create-state.cpp index a32564a215..1fcdde8850 100644 --- a/crypto/block/create-state.cpp +++ b/crypto/block/create-state.cpp @@ -25,42 +25,40 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include -#include +#include +#include +#include #include -#include #include +#include +#include +#include +#include +#include +#include +#include -#include "vm/stack.hpp" -#include "vm/boc.h" - -#include "fift/Fift.h" #include "fift/Dictionary.h" -#include "fift/SourceLookup.h" +#include "fift/Fift.h" #include "fift/IntCtx.h" +#include "fift/SourceLookup.h" #include "fift/words.h" - +#include "td/utils/Parser.h" #include "td/utils/logging.h" #include "td/utils/misc.h" -#include "td/utils/Parser.h" #include "td/utils/port/path.h" #include "td/utils/port/signals.h" +#include "vm/boc.h" +#include "vm/stack.hpp" -#include "block.h" -#include "block-parse.h" #include "block-auto.h" -#include "mc-config.h" +#include "block-parse.h" +#include "block.h" #include "git.h" +#include "mc-config.h" #if defined(_INTERNAL_COMPILE) || defined(_TONLIB_COMPILE) #define WITH_TONLIB @@ -329,7 +327,7 @@ td::RefInt256 create_smartcontract(td::RefInt256 smc_addr, Ref code, R } PDO(cb.store_long_bool(ctor, 2)); // addr_std$10 or addr_var$11 if (fixed_prefix_length) { - PDO(cb.store_long_bool(1, 1) // just$1 + PDO(cb.store_long_bool(1, 1) // just$1 && cb.store_ulong_rchk_bool(fixed_prefix_length, 5) // depth:(## 5) && cb.store_bits_bool(addr.cbits(), fixed_prefix_length)); // rewrite pfx:(depth * Bit) } else { @@ -815,11 +813,11 @@ void usage(const char* progname) { void parse_include_path_set(std::string include_path_set, std::vector& res) { td::Parser parser(include_path_set); while (!parser.empty()) { - #if TD_WINDOWS +#if TD_WINDOWS auto path_separator = '@'; - #else +#else auto path_separator = ':'; - #endif +#endif auto path = parser.read_till_nofail(path_separator); if (!path.empty()) { res.push_back(path.str()); diff --git a/crypto/block/dump-block.cpp b/crypto/block/dump-block.cpp index e7a196775f..62a340bd3c 100644 --- a/crypto/block/dump-block.cpp +++ b/crypto/block/dump-block.cpp @@ -25,16 +25,18 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include +#include + #include "block/block.h" #include "vm/boc.h" -#include -#include "block-db.h" +#include "vm/cp0.h" + #include "block-auto.h" +#include "block-db.h" #include "block-parse.h" -#include "mc-config.h" -#include "vm/cp0.h" -#include #include "git.h" +#include "mc-config.h" using td::Ref; using namespace std::literals::string_literals; @@ -283,7 +285,8 @@ int main(int argc, char* const argv[]) { dump = 0; break; case 'V': - std::cout << "dump-block build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "dump-block build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); break; case 'h': diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index d9a67b0be4..8af5247a87 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -25,26 +25,28 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "mc-config.h" -#include "block/block.h" -#include "block/block-parse.h" +#include +#include +#include + #include "block/block-auto.h" +#include "block/block-parse.h" +#include "block/block.h" #include "common/bitstring.h" -#include "vm/dict.h" +#include "openssl/digest.hpp" #include "td/utils/bits.h" #include "td/utils/uint128.h" -#include "ton/ton-types.h" #include "ton/ton-shard.h" -#include "openssl/digest.hpp" -#include -#include -#include +#include "ton/ton-types.h" +#include "vm/dict.h" + +#include "mc-config.h" namespace block { using namespace std::literals::string_literals; using td::Ref; -#define DBG(__n) dbg(__n)&& +#define DBG(__n) dbg(__n) && #define DSTART int __dcnt = 0; #define DEB DBG(++__dcnt) @@ -482,7 +484,7 @@ td::Result Config::unpack_workchain_list(Ref root) { } class ValidatorSetCache { -public: + public: ValidatorSetCache() { cache_.reserve(MAX_CACHE_SIZE + 1); } @@ -517,7 +519,7 @@ class ValidatorSetCache { } } -private: + private: std::mutex mutex_; struct CacheEntry : td::ListNode { diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 5c6e224df8..378439ae8f 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -17,19 +17,20 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include +#include +#include +#include +#include + +#include "common/bitstring.h" #include "common/refcnt.hpp" +#include "ton/ton-shard.h" +#include "ton/ton-types.h" #include "vm/db/StaticBagOfCellsDb.h" #include "vm/dict.h" -#include "ton/ton-types.h" -#include "ton/ton-shard.h" -#include "common/bitstring.h" -#include "block.h" -#include -#include -#include -#include -#include +#include "block.h" namespace block { using td::Ref; @@ -340,8 +341,8 @@ struct StoragePrices { , mc_cell_price(_mc_cprice) { } static td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector& pricing, - const StorageUsed& storage_used, ton::UnixTime last_paid, - bool is_special, bool is_masterchain); + const StorageUsed& storage_used, ton::UnixTime last_paid, bool is_special, + bool is_masterchain); }; struct GasLimitsPrices { @@ -434,10 +435,10 @@ struct WorkchainInfo : public td::CntObject { int min_addr_len, max_addr_len, addr_len_step; // Default values are used when split_merge_timings is not set in config - unsigned split_merge_delay = 100; // prepare (delay) split/merge for 100 seconds - unsigned split_merge_interval = 100; // split/merge is enabled during 60 second interval - unsigned min_split_merge_interval = 30; // split/merge interval must be at least 30 seconds - unsigned max_split_merge_delay = 1000; // end of split/merge interval must be at most 1000 seconds in the future + unsigned split_merge_delay = 100; // prepare (delay) split/merge for 100 seconds + unsigned split_merge_interval = 100; // split/merge is enabled during 60 second interval + unsigned min_split_merge_interval = 30; // split/merge interval must be at least 30 seconds + unsigned max_split_merge_delay = 1000; // end of split/merge interval must be at most 1000 seconds in the future td::uint32 persistent_state_split_depth = 0; @@ -636,7 +637,8 @@ class Config { bool set_block_id_ext(const ton::BlockIdExt& block_id_ext); td::Result> get_special_smartcontracts(bool without_config = false) const; bool is_special_smartcontract(const ton::StdSmcAddress& addr) const; - static td::Result> unpack_validator_set(Ref valset_root, bool use_cache = false); + static td::Result> unpack_validator_set(Ref valset_root, + bool use_cache = false); td::Result> get_storage_prices() const; static td::Result do_get_one_storage_prices(vm::CellSlice cs); td::Result get_gas_limits_prices(bool is_masterchain = false) const; diff --git a/crypto/block/output-queue-merger.cpp b/crypto/block/output-queue-merger.cpp index 74b5845e70..43c59276fe 100644 --- a/crypto/block/output-queue-merger.cpp +++ b/crypto/block/output-queue-merger.cpp @@ -214,7 +214,7 @@ bool OutputQueueMerger::load() { } while (!heap.empty() && heap[0]->lt <= lt); std::sort(msg_list.begin() + orig_size, msg_list.end(), MsgKeyValue::less); for (size_t i = orig_size; i < msg_list.size(); ++i) { - td::int32 &remaining = src_remaining_msgs_[msg_list[i]->source]; + td::int32& remaining = src_remaining_msgs_[msg_list[i]->source]; if (remaining != -1) { if (remaining == 0) { limit_exceeded = true; diff --git a/crypto/block/output-queue-merger.h b/crypto/block/output-queue-merger.h index 26deb7ee47..98ec1677ba 100644 --- a/crypto/block/output-queue-merger.h +++ b/crypto/block/output-queue-merger.h @@ -17,9 +17,9 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "block/mc-config.h" #include "ton/ton-types.h" #include "vm/cells/CellSlice.h" -#include "block/mc-config.h" namespace block { using td::Ref; diff --git a/crypto/block/precompiled-smc/PrecompiledSmartContract.cpp b/crypto/block/precompiled-smc/PrecompiledSmartContract.cpp index 6797216df4..2c5d87d8ad 100644 --- a/crypto/block/precompiled-smc/PrecompiledSmartContract.cpp +++ b/crypto/block/precompiled-smc/PrecompiledSmartContract.cpp @@ -14,10 +14,12 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "common.h" #include + #include "vm/memo.h" +#include "common.h" + namespace block::precompiled { using namespace vm; diff --git a/crypto/block/precompiled-smc/PrecompiledSmartContract.h b/crypto/block/precompiled-smc/PrecompiledSmartContract.h index 509ebf7930..e20cf90ece 100644 --- a/crypto/block/precompiled-smc/PrecompiledSmartContract.h +++ b/crypto/block/precompiled-smc/PrecompiledSmartContract.h @@ -15,18 +15,19 @@ along with TON Blockchain Library. If not, see . */ #pragma once +#include + +#include "block/block.h" +#include "block/mc-config.h" #include "common/refcnt.hpp" #include "common/refint.h" +#include "td/utils/bits.h" +#include "tl/tlblib.hpp" +#include "ton/ton-types.h" +#include "vm/boc.h" #include "vm/cells.h" #include "vm/cellslice.h" #include "vm/dict.h" -#include "vm/boc.h" -#include -#include "tl/tlblib.hpp" -#include "td/utils/bits.h" -#include "ton/ton-types.h" -#include "block/block.h" -#include "block/mc-config.h" namespace block::precompiled { @@ -77,7 +78,8 @@ class PrecompiledSmartContract { Result run(td::Ref my_address, ton::UnixTime now, ton::LogicalTime cur_lt, CurrencyCollection balance, td::Ref c4, vm::CellSlice msg_body, td::Ref msg, CurrencyCollection msg_balance, bool is_external, std::vector> libraries, int global_version, td::uint16 max_data_depth, - td::Ref my_code, td::Ref unpacked_config, td::RefInt256 due_payment, td::uint64 precompiled_gas_usage); + td::Ref my_code, td::Ref unpacked_config, td::RefInt256 due_payment, + td::uint64 precompiled_gas_usage); td::Ref get_c4() const { return c4_; diff --git a/crypto/block/precompiled-smc/common.h b/crypto/block/precompiled-smc/common.h index 0406d7deef..174db76785 100644 --- a/crypto/block/precompiled-smc/common.h +++ b/crypto/block/precompiled-smc/common.h @@ -15,7 +15,8 @@ along with TON Blockchain Library. If not, see . */ #pragma once -#include "PrecompiledSmartContract.h" #include "vm/arithops.h" #include "vm/cellops.h" -#include "vm/tonops.h" \ No newline at end of file +#include "vm/tonops.h" + +#include "PrecompiledSmartContract.h" \ No newline at end of file diff --git a/crypto/block/test-block.cpp b/crypto/block/test-block.cpp index 6e10ebe9ac..70433ecec7 100644 --- a/crypto/block/test-block.cpp +++ b/crypto/block/test-block.cpp @@ -25,14 +25,16 @@ Copyright 2017-2019 Telegram Systems LLP */ +#include +#include + #include "block/block.h" #include "vm/boc.h" -#include -#include "block-db.h" +#include "vm/cp0.h" + #include "block-auto.h" +#include "block-db.h" #include "block-parse.h" -#include "vm/cp0.h" -#include using td::Ref; diff --git a/crypto/block/test-weight-distr.cpp b/crypto/block/test-weight-distr.cpp index a94b791cd3..9c364e8451 100644 --- a/crypto/block/test-weight-distr.cpp +++ b/crypto/block/test-weight-distr.cpp @@ -25,11 +25,12 @@ Copyright 2020 Telegram Systems LLP */ +#include #include + +#include "block/block.h" #include "td/utils/Random.h" #include "td/utils/misc.h" -#include "block/block.h" -#include const int MAX_N = 1000, MAX_K = 100, DEFAULT_K = 7; diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 0ef94b0766..c89946f552 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -16,16 +16,16 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "block/transaction.h" -#include "block/block.h" -#include "block/block-parse.h" #include "block/block-auto.h" +#include "block/block-parse.h" +#include "block/block.h" +#include "block/transaction.h" #include "crypto/openssl/rand.hpp" +#include "td/utils/Timer.h" #include "td/utils/bits.h" #include "td/utils/uint128.h" #include "ton/ton-shard.h" #include "vm/vm.h" -#include "td/utils/Timer.h" namespace { /** @@ -35,7 +35,8 @@ namespace { */ class StringLoggerTail : public td::LogInterface { public: - explicit StringLoggerTail(size_t max_size = 256) : buf(max_size, '\0') {} + explicit StringLoggerTail(size_t max_size = 256) : buf(max_size, '\0') { + } /** * Appends a slice of data to the buffer. @@ -78,7 +79,7 @@ class StringLoggerTail : public td::LogInterface { size_t pos = 0; bool truncated = false; }; -} +} // namespace namespace block { using td::Ref; @@ -179,7 +180,7 @@ bool Account::store_maybe_anycast(vm::CellBuilder& cb) const { if (!addr_rewrite_length_set || !addr_rewrite_length) { return cb.store_bool_bool(false); } - return cb.store_bool_bool(true) // just$1 + return cb.store_bool_bool(true) // just$1 && cb.store_uint_leq(30, addr_rewrite_length) // depth:(#<= 30) && cb.store_bits_bool(addr_rewrite.cbits(), addr_rewrite_length); // rewrite_pfx:(bits depth) } @@ -394,7 +395,7 @@ bool Account::recompute_tmp_addr(Ref& tmp_addr, int fixed_prefix_ if (!cb.store_bool_bool(false)) { // anycast:(Maybe Anycast) return false; } - } else if (!(cb.store_bool_bool(true) // just$1 + } else if (!(cb.store_bool_bool(true) // just$1 && cb.store_long_bool(fixed_prefix_length, 5) // depth:(#<= 30) && cb.store_bits_bool(addr.bits(), fixed_prefix_length))) { // rewrite_pfx:(bits depth) return false; @@ -1488,16 +1489,16 @@ Ref Transaction::prepare_vm_c7(const ComputePhaseConfig& cfg) const { return {}; } std::vector tuple = { - td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea - td::zero_refint(), // actions:Integer - td::zero_refint(), // msgs_sent:Integer - td::make_refint(now), // unixtime:Integer - td::make_refint(account.block_lt), // block_lt:Integer - td::make_refint(start_lt), // trans_lt:Integer - std::move(rand_seed_int), // rand_seed:Integer - balance.as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] - my_addr, // myself:MsgAddressInt - vm::StackEntry::maybe(cfg.global_config) // global_config:(Maybe Cell) ] = SmartContractInfo; + td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea + td::zero_refint(), // actions:Integer + td::zero_refint(), // msgs_sent:Integer + td::make_refint(now), // unixtime:Integer + td::make_refint(account.block_lt), // block_lt:Integer + td::make_refint(start_lt), // trans_lt:Integer + std::move(rand_seed_int), // rand_seed:Integer + balance.as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] + my_addr, // myself:MsgAddressInt + vm::StackEntry::maybe(cfg.global_config) // global_config:(Maybe Cell) ] = SmartContractInfo; }; if (cfg.global_version >= 4) { tuple.push_back(vm::StackEntry::maybe(new_code)); // code:Cell @@ -1506,7 +1507,7 @@ Ref Transaction::prepare_vm_c7(const ComputePhaseConfig& cfg) const { } else { tuple.push_back(block::CurrencyCollection::zero().as_vm_tuple()); } - tuple.push_back(storage_phase->fees_collected); // storage_fees:Integer + tuple.push_back(storage_phase->fees_collected); // storage_fees:Integer // See crypto/block/mc-config.cpp#2223 (get_prev_blocks_info) // [ wc:Integer shard:Integer seqno:Integer root_hash:Integer file_hash:Integer] = BlockId; @@ -2141,7 +2142,7 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { } ap.valid = true; for (int i = n - 1; i >= 0; --i) { - if(ap.action_list[i].is_null()) { + if (ap.action_list[i].is_null()) { continue; } ap.result_arg = n - 1 - i; @@ -2471,8 +2472,8 @@ bool Transaction::check_replace_src_addr(Ref& src_addr) const { * * @returns True if the destination address is valid, false otherwise. */ -bool Transaction::check_rewrite_dest_addr(Ref& dest_addr, const ActionPhaseConfig& cfg, - bool* is_mc, bool allow_anycast) const { +bool Transaction::check_rewrite_dest_addr(Ref& dest_addr, const ActionPhaseConfig& cfg, bool* is_mc, + bool allow_anycast) const { if (!dest_addr->prefetch_ulong(1)) { // all external addresses allowed if (is_mc) { @@ -3570,7 +3571,8 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { td::Status S = stats.replace_roots(roots); time_storage_stat += timer.elapsed_both(); if (S.is_error()) { - LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error(); + LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " + << S.move_as_error(); return false; } } @@ -3965,7 +3967,7 @@ Ref Transaction::commit(Account& acc) { if (force_remove_anycast_address) { CHECK(acc.forget_addr_rewrite_length()); } else if (orig_addr_rewrite_set && new_addr_rewrite_length >= 0 && acc.status != Account::acc_active && - acc_status == Account::acc_active) { + acc_status == Account::acc_active) { LOG(DEBUG) << "setting address rewriting info for newly-activated account " << acc.addr.to_hex() << " with addr_rewrite_length=" << new_addr_rewrite_length << ", orig_addr_rewrite=" << orig_addr_rewrite.bits().to_hex(new_addr_rewrite_length); diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 5feb7c747f..fa1e57c101 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -17,21 +17,23 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "account-storage-stat.h" +#include + +#include "block/block-auto.h" +#include "block/block.h" +#include "block/mc-config.h" #include "common/refcnt.hpp" #include "common/refint.h" +#include "precompiled-smc/PrecompiledSmartContract.h" +#include "td/utils/bits.h" +#include "tl/tlblib.hpp" +#include "ton/ton-types.h" +#include "vm/boc.h" #include "vm/cells.h" #include "vm/cellslice.h" #include "vm/dict.h" -#include "vm/boc.h" -#include -#include "tl/tlblib.hpp" -#include "td/utils/bits.h" -#include "ton/ton-types.h" -#include "block/block.h" -#include "block/mc-config.h" -#include "precompiled-smc/PrecompiledSmartContract.h" -#include "block/block-auto.h" + +#include "account-storage-stat.h" namespace block { using td::Ref; @@ -266,13 +268,16 @@ struct Account { int verbosity{3 * 0}; ton::UnixTime now_{0}; ton::WorkchainId workchain{ton::workchainInvalid}; - td::BitArray<32> addr_rewrite; // rewrite (anycast) data, addr_rewrite_length bits + td::BitArray<32> addr_rewrite; // rewrite (anycast) data, addr_rewrite_length bits bool addr_rewrite_length_set{false}; unsigned char addr_rewrite_length{0}; - ton::StdSmcAddress addr; // rewritten address (by replacing a prefix of `addr_orig` with `addr_rewrite`); it is the key in ShardAccounts - ton::StdSmcAddress addr_orig; // address indicated in smart-contract data (must coincide with hash of StateInit) - Ref my_addr; // address as stored in the smart contract (MsgAddressInt); corresponds to `addr_orig` + anycast info - Ref my_addr_exact; // exact address without anycast info; corresponds to `addr` and has no anycast (rewrite) info + ton::StdSmcAddress + addr; // rewritten address (by replacing a prefix of `addr_orig` with `addr_rewrite`); it is the key in ShardAccounts + ton::StdSmcAddress addr_orig; // address indicated in smart-contract data (must coincide with hash of StateInit) + Ref + my_addr; // address as stored in the smart contract (MsgAddressInt); corresponds to `addr_orig` + anycast info + Ref + my_addr_exact; // exact address without anycast info; corresponds to `addr` and has no anycast (rewrite) info ton::LogicalTime last_trans_end_lt_; ton::LogicalTime last_trans_lt_; ton::Bits256 last_trans_hash_; @@ -306,7 +311,8 @@ struct Account { td::Result> compute_account_storage_dict() const; td::Status init_account_storage_stat(Ref dict_root); bool deactivate(); - bool recompute_tmp_addr(Ref& tmp_addr, int fixed_prefix_length, td::ConstBitPtr orig_addr_rewrite) const; + bool recompute_tmp_addr(Ref& tmp_addr, int fixed_prefix_length, + td::ConstBitPtr orig_addr_rewrite) const; td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector& pricing) const; bool is_masterchain() const { return workchain == ton::masterchainId; @@ -443,8 +449,8 @@ struct Transaction { int try_action_send_msg(const vm::CellSlice& cs, ActionPhase& ap, const ActionPhaseConfig& cfg, int redoing = 0); int try_action_reserve_currency(vm::CellSlice& cs, ActionPhase& ap, const ActionPhaseConfig& cfg); bool check_replace_src_addr(Ref& src_addr) const; - bool check_rewrite_dest_addr(Ref& dest_addr, const ActionPhaseConfig& cfg, - bool* is_mc = nullptr, bool allow_anycast = true) const; + bool check_rewrite_dest_addr(Ref& dest_addr, const ActionPhaseConfig& cfg, bool* is_mc = nullptr, + bool allow_anycast = true) const; bool serialize_storage_phase(vm::CellBuilder& cb); bool serialize_credit_phase(vm::CellBuilder& cb); bool serialize_compute_phase(vm::CellBuilder& cb); diff --git a/crypto/common/bigexp.cpp b/crypto/common/bigexp.cpp index 365f0595ba..1d0b78e8c2 100644 --- a/crypto/common/bigexp.cpp +++ b/crypto/common/bigexp.cpp @@ -16,11 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "bigexp.h" -#include "td/utils/bits.h" #include "td/utils/as.h" +#include "td/utils/bits.h" #include "td/utils/misc.h" +#include "bigexp.h" + namespace td { bool NegExpBinTable::init() { diff --git a/crypto/common/bigint.hpp b/crypto/common/bigint.hpp index 4061f82d31..aa06f965c1 100644 --- a/crypto/common/bigint.hpp +++ b/crypto/common/bigint.hpp @@ -17,18 +17,18 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include -#include #include +#include #include #include #include #include -#include -#include "common/bitstring.h" +#include +#include -#include "td/utils/bits.h" +#include "common/bitstring.h" #include "td/utils/Span.h" +#include "td/utils/bits.h" #include "td/utils/uint128.h" /************************************** diff --git a/crypto/common/bitstring.cpp b/crypto/common/bitstring.cpp index 3a6f331191..9d6e05a41a 100644 --- a/crypto/common/bitstring.cpp +++ b/crypto/common/bitstring.cpp @@ -16,13 +16,14 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "common/bitstring.h" #include #include + +#include "common/bitstring.h" +#include "crypto/openssl/digest.hpp" #include "td/utils/as.h" #include "td/utils/bits.h" #include "td/utils/misc.h" -#include "crypto/openssl/digest.hpp" namespace td { diff --git a/crypto/common/bitstring.h b/crypto/common/bitstring.h index 12333522e9..c994df13f0 100644 --- a/crypto/common/bitstring.h +++ b/crypto/common/bitstring.h @@ -17,12 +17,13 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "common/refcnt.hpp" -#include #include -#include -#include #include +#include +#include +#include + +#include "common/refcnt.hpp" #include "td/utils/bits.h" namespace td { @@ -284,7 +285,7 @@ class BitSliceGen { ensure_throw(set_size_bool(bits)); return *this; } - BitSliceGen subslice(unsigned from, unsigned bits) const & { + BitSliceGen subslice(unsigned from, unsigned bits) const& { return BitSliceGen(*this, from, bits); } BitSliceGen subslice(unsigned from, unsigned bits) && { diff --git a/crypto/common/promiseop.hpp b/crypto/common/promiseop.hpp index 1e14bc7d78..610be97e03 100644 --- a/crypto/common/promiseop.hpp +++ b/crypto/common/promiseop.hpp @@ -17,9 +17,10 @@ Copyright 2020 Telegram Systems LLP */ #pragma once -#include "refcnt.hpp" #include "td/actor/PromiseFuture.h" +#include "refcnt.hpp" + namespace td { template diff --git a/crypto/common/refcnt.cpp b/crypto/common/refcnt.cpp index c6d1b8884d..b758cf7a42 100644 --- a/crypto/common/refcnt.cpp +++ b/crypto/common/refcnt.cpp @@ -16,10 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "refcnt.hpp" - #include "td/utils/ScopeGuard.h" +#include "refcnt.hpp" + namespace td { Ref CntObject::clone() const { diff --git a/crypto/common/refcnt.hpp b/crypto/common/refcnt.hpp index 953cc77970..f31a0faa87 100644 --- a/crypto/common/refcnt.hpp +++ b/crypto/common/refcnt.hpp @@ -18,10 +18,10 @@ */ #pragma once -#include -#include #include +#include #include +#include #include "td/utils/StringBuilder.h" #include "td/utils/logging.h" diff --git a/crypto/common/refint.cpp b/crypto/common/refint.cpp index a2ccdaa693..e33e0eb07e 100644 --- a/crypto/common/refint.cpp +++ b/crypto/common/refint.cpp @@ -16,12 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "common/refint.h" -#include #include +#include -#include "td/utils/StringBuilder.h" +#include "common/refint.h" #include "td/utils/Slice.h" +#include "td/utils/StringBuilder.h" namespace td { diff --git a/crypto/common/refint.h b/crypto/common/refint.h index e3305c2f9b..aa1d7b2c4c 100644 --- a/crypto/common/refint.h +++ b/crypto/common/refint.h @@ -18,10 +18,11 @@ */ #pragma once -#include "common/refcnt.hpp" -#include "common/bigint.hpp" -#include #include +#include + +#include "common/bigint.hpp" +#include "common/refcnt.hpp" namespace td { class StringBuilder; diff --git a/crypto/common/util.cpp b/crypto/common/util.cpp index 50b547cbbe..61d7255834 100644 --- a/crypto/common/util.cpp +++ b/crypto/common/util.cpp @@ -16,12 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "util.h" - #include -#include "td/utils/crypto.h" #include "td/utils/base64.h" +#include "td/utils/crypto.h" + +#include "util.h" namespace td { diff --git a/crypto/common/util.h b/crypto/common/util.h index 49f50867ff..3f91850a0d 100644 --- a/crypto/common/util.h +++ b/crypto/common/util.h @@ -18,6 +18,7 @@ */ #pragma once #include + #include "td/utils/Slice.h" #include "td/utils/Status.h" #include "td/utils/buffer.h" diff --git a/crypto/ellcurve/p256.cpp b/crypto/ellcurve/p256.cpp index 684ac8ca08..0aee9f7862 100644 --- a/crypto/ellcurve/p256.cpp +++ b/crypto/ellcurve/p256.cpp @@ -15,12 +15,14 @@ along with TON Blockchain Library. If not, see . */ -#include "p256.h" +#include +#include +#include + #include "td/utils/check.h" #include "td/utils/misc.h" -#include -#include -#include + +#include "p256.h" namespace td { diff --git a/crypto/ellcurve/secp256k1.cpp b/crypto/ellcurve/secp256k1.cpp index b98eea7be4..34f7df03ed 100644 --- a/crypto/ellcurve/secp256k1.cpp +++ b/crypto/ellcurve/secp256k1.cpp @@ -15,13 +15,14 @@ along with TON Blockchain Library. If not, see . */ -#include "secp256k1.h" +#include +#include +#include + #include "td/utils/check.h" #include "td/utils/logging.h" -#include -#include -#include +#include "secp256k1.h" namespace td::secp256k1 { diff --git a/crypto/fift/Continuation.cpp b/crypto/fift/Continuation.cpp index e895082fd1..11adc13e36 100644 --- a/crypto/fift/Continuation.cpp +++ b/crypto/fift/Continuation.cpp @@ -17,8 +17,8 @@ Copyright 2020 Telegram Systems LLP */ #include "Continuation.h" -#include "IntCtx.h" #include "Dictionary.h" +#include "IntCtx.h" namespace fift { diff --git a/crypto/fift/Continuation.h b/crypto/fift/Continuation.h index 6623b64271..3b2e1b6b2f 100644 --- a/crypto/fift/Continuation.h +++ b/crypto/fift/Continuation.h @@ -18,6 +18,7 @@ */ #pragma once #include + #include "common/refcnt.hpp" #include "common/refint.h" #include "vm/stack.hpp" diff --git a/crypto/fift/Dictionary.h b/crypto/fift/Dictionary.h index b24bc74269..f796a85446 100644 --- a/crypto/fift/Dictionary.h +++ b/crypto/fift/Dictionary.h @@ -17,9 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "vm/box.hpp" + #include "Continuation.h" #include "HashMap.h" -#include "vm/box.hpp" namespace fift { using td::Ref; diff --git a/crypto/fift/Fift.cpp b/crypto/fift/Fift.cpp index e0faca57cc..33403496c7 100644 --- a/crypto/fift/Fift.cpp +++ b/crypto/fift/Fift.cpp @@ -16,12 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "td/utils/PathView.h" + #include "Fift.h" #include "IntCtx.h" #include "words.h" -#include "td/utils/PathView.h" - namespace fift { Fift::Fift(Config config) : config_(std::move(config)) { diff --git a/crypto/fift/Fift.h b/crypto/fift/Fift.h index a177279934..3c4da91e4b 100644 --- a/crypto/fift/Fift.h +++ b/crypto/fift/Fift.h @@ -18,11 +18,11 @@ */ #pragma once -#include "SourceLookup.h" -#include "Dictionary.h" - #include "td/utils/Status.h" +#include "Dictionary.h" +#include "SourceLookup.h" + namespace fift { struct IntCtx; diff --git a/crypto/fift/HashMap.cpp b/crypto/fift/HashMap.cpp index a4ca0e9b35..37f84e3f88 100644 --- a/crypto/fift/HashMap.cpp +++ b/crypto/fift/HashMap.cpp @@ -14,8 +14,9 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "HashMap.h" #include "td/utils/Random.h" + +#include "HashMap.h" #include "IntCtx.h" namespace fift { diff --git a/crypto/fift/HashMap.h b/crypto/fift/HashMap.h index d3d9ffbefa..ffb9efa1aa 100644 --- a/crypto/fift/HashMap.h +++ b/crypto/fift/HashMap.h @@ -16,8 +16,8 @@ */ #pragma once #include "common/refcnt.hpp" -#include "vm/stack.hpp" #include "vm/atom.h" +#include "vm/stack.hpp" namespace fift { diff --git a/crypto/fift/IntCtx.h b/crypto/fift/IntCtx.h index 5d574ae5f0..979a6d81a3 100644 --- a/crypto/fift/IntCtx.h +++ b/crypto/fift/IntCtx.h @@ -18,20 +18,19 @@ */ #pragma once -#include "crypto/vm/stack.hpp" -#include "crypto/vm/box.hpp" -#include "crypto/common/bitstring.h" - -#include "td/utils/Status.h" - -#include "Dictionary.h" -#include "Continuation.h" - #include #include #include #include +#include "crypto/common/bitstring.h" +#include "crypto/vm/box.hpp" +#include "crypto/vm/stack.hpp" +#include "td/utils/Status.h" + +#include "Continuation.h" +#include "Dictionary.h" + namespace vm { class TonDbImpl; // from crypto/vm/db/TonDb.h using TonDb = std::unique_ptr; @@ -175,7 +174,7 @@ struct IntCtx { return parser ? parser->include_depth : -1; } - operator vm::Stack &() { + operator vm::Stack&() { return stack; } diff --git a/crypto/fift/SourceLookup.cpp b/crypto/fift/SourceLookup.cpp index cd80e31408..64058eff57 100644 --- a/crypto/fift/SourceLookup.cpp +++ b/crypto/fift/SourceLookup.cpp @@ -16,14 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "SourceLookup.h" +#include #include "td/utils/PathView.h" -#include "td/utils/PathView.h" -#include "td/utils/port/path.h" #include "td/utils/filesystem.h" +#include "td/utils/port/path.h" -#include +#include "SourceLookup.h" namespace fift { td::Result OsFileLoader::read_file(td::CSlice filename) { diff --git a/crypto/fift/fift-main.cpp b/crypto/fift/fift-main.cpp index cdc36fc077..953b3266da 100644 --- a/crypto/fift/fift-main.cpp +++ b/crypto/fift/fift-main.cpp @@ -25,33 +25,32 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/stack.hpp" -#include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include -#include +#include +#include +#include #include #include +#include +#include +#include +#include +#include +#include -#include "Fift.h" -#include "Dictionary.h" -#include "SourceLookup.h" -#include "words.h" - +#include "td/utils/Parser.h" #include "td/utils/logging.h" #include "td/utils/misc.h" -#include "td/utils/Parser.h" #include "td/utils/port/path.h" +#include "vm/stack.hpp" +#include "Dictionary.h" +#include "Fift.h" +#include "SourceLookup.h" #include "git.h" +#include "words.h" void usage(const char* progname) { std::cerr << "A simple Fift interpreter. Type `bye` to quit, or `words` to get a list of all commands\n"; @@ -60,7 +59,8 @@ void usage(const char* progname) { << " [-i] [-n] [-I ] {-L } ...\n"; std::cerr << "\t-n\tDo not preload standard preamble file `Fift.fif`\n" "\t-i\tForce interactive mode even if explicit source file names are indicated\n" - "\t-I\tSets colon-separated (unix) or at-separated (windows) library source include path. If not indicated, " + "\t-I\tSets colon-separated (unix) or at-separated (windows) library source include " + "path. If not indicated, " "$FIFTPATH is used instead.\n" "\t-L\tPre-loads a library source file\n" "\t-s\tScript mode: use first argument as a fift source file and import remaining arguments as $n)\n" @@ -72,11 +72,11 @@ void usage(const char* progname) { void parse_include_path_set(std::string include_path_set, std::vector& res) { td::Parser parser(include_path_set); while (!parser.empty()) { - #if TD_WINDOWS +#if TD_WINDOWS auto path_separator = '@'; - #else +#else auto path_separator = ':'; - #endif +#endif auto path = parser.read_till_nofail(path_separator); if (!path.empty()) { res.push_back(path.str()); @@ -118,7 +118,8 @@ int main(int argc, char* const argv[]) { new_verbosity_level = VERBOSITY_NAME(FATAL) + td::to_integer(td::Slice(optarg)); break; case 'V': - std::cout << "Fift build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "Fift build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); break; diff --git a/crypto/fift/utils.cpp b/crypto/fift/utils.cpp index 6057b2dc07..851363b3f9 100644 --- a/crypto/fift/utils.cpp +++ b/crypto/fift/utils.cpp @@ -16,14 +16,16 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "utils.h" -#include "words.h" +#include + #include "td/utils/PathView.h" #include "td/utils/filesystem.h" #include "td/utils/misc.h" #include "td/utils/port/path.h" #include "vm/boc.h" -#include + +#include "utils.h" +#include "words.h" namespace fift { namespace { @@ -163,7 +165,7 @@ td::Result create_source_lookup(std::string&& main, bool nee return std::move(res); } -td::Result run_fift(fift::SourceLookup source_lookup, std::ostream *stream, +td::Result run_fift(fift::SourceLookup source_lookup, std::ostream* stream, bool preload_fift = true, std::vector args = {}) { fift::Fift::Config config; config.source_lookup = std::move(source_lookup); @@ -173,8 +175,8 @@ td::Result run_fift(fift::SourceLookup source_lookup, std::o config.error_stream = stream; config.output_stream = stream; if (args.size() != 0) { - std::vector argv; - for (auto &arg : args) { + std::vector argv; + for (auto& arg : args) { argv.push_back(arg.c_str()); } fift::import_cmdline_args(config.dictionary, argv[0], td::narrow_cast(argv.size() - 1), argv.data() + 1); @@ -189,7 +191,8 @@ td::Result run_fift(fift::SourceLookup source_lookup, std::o } // namespace td::Result mem_run_fift(std::string source, std::vector args, std::string fift_dir) { std::stringstream ss; - TRY_RESULT(source_lookup, create_source_lookup(std::move(source), true, true, true, true, true, true, true, fift_dir)); + TRY_RESULT(source_lookup, + create_source_lookup(std::move(source), true, true, true, true, true, true, true, fift_dir)); TRY_RESULT_ASSIGN(source_lookup, run_fift(std::move(source_lookup), &ss, true, std::move(args))); FiftOutput res; res.source_lookup = std::move(source_lookup); @@ -207,8 +210,8 @@ td::Result mem_run_fift(SourceLookup source_lookup, std::vector create_mem_source_lookup(std::string main, std::string fift_dir, bool need_preamble, bool need_asm, bool need_ton_util, bool need_lisp, bool need_w3_code) { - return create_source_lookup(std::move(main), need_preamble, need_asm, need_ton_util, need_lisp, need_w3_code, false, false, - fift_dir); + return create_source_lookup(std::move(main), need_preamble, need_asm, need_ton_util, need_lisp, need_w3_code, false, + false, fift_dir); } td::Result> compile_asm(td::Slice asm_code) { @@ -229,20 +232,21 @@ td::Result compile_asm_program(std::string&& program_code std::string main_fif; main_fif.reserve(program_code.size() + 100); main_fif.append(program_code.data(), program_code.size()); - main_fif.append(R"( dup hashB B>X $>B "hex" B>file)"); // write codeHashHex to a file - main_fif.append(R"( boc>B B>base64 $>B "boc" B>file)"); // write codeBoc64 to a file + main_fif.append(R"( dup hashB B>X $>B "hex" B>file)"); // write codeHashHex to a file + main_fif.append(R"( boc>B B>base64 $>B "boc" B>file)"); // write codeBoc64 to a file std::stringstream fift_output_stream; - TRY_RESULT(source_lookup, create_source_lookup(std::move(main_fif), true, true, false, false, false, false, false, fift_dir)); + TRY_RESULT(source_lookup, + create_source_lookup(std::move(main_fif), true, true, false, false, false, false, false, fift_dir)); TRY_RESULT(res, run_fift(std::move(source_lookup), &fift_output_stream)); TRY_RESULT(boc, res.read_file("boc")); TRY_RESULT(hex, res.read_file("hex")); return CompiledProgramOutput{ - std::move(program_code), - std::move(boc.data), - std::move(hex.data), + std::move(program_code), + std::move(boc.data), + std::move(hex.data), }; } diff --git a/crypto/fift/utils.h b/crypto/fift/utils.h index fab92c5420..906e09e9c4 100644 --- a/crypto/fift/utils.h +++ b/crypto/fift/utils.h @@ -18,9 +18,10 @@ */ #pragma once -#include "Fift.h" #include +#include "Fift.h" + namespace fift { struct FiftOutput { SourceLookup source_lookup; diff --git a/crypto/fift/words.cpp b/crypto/fift/words.cpp index 394dc45055..8692f2c5d2 100644 --- a/crypto/fift/words.cpp +++ b/crypto/fift/words.cpp @@ -16,47 +16,40 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "words.h" - -#include "Dictionary.h" -#include "IntCtx.h" -#include "SourceLookup.h" -#include "HashMap.h" +#include -#include "common/refcnt.hpp" +#include "block/block.h" #include "common/bigint.hpp" -#include "common/refint.h" #include "common/bitstring.h" +#include "common/global-version.h" +#include "common/refcnt.hpp" +#include "common/refint.h" #include "common/util.h" - #include "openssl/digest.hpp" - -#include "Ed25519.h" - -#include "vm/cells.h" -#include "vm/cellslice.h" -#include "vm/vm.h" -#include "vm/cp0.h" -#include "vm/dict.h" -#include "vm/boc.h" - -#include "vm/box.hpp" -#include "vm/atom.h" - -#include "block/block.h" -#include "common/global-version.h" - +#include "td/utils/PathView.h" +#include "td/utils/Timer.h" +#include "td/utils/crypto.h" #include "td/utils/filesystem.h" #include "td/utils/misc.h" #include "td/utils/optional.h" -#include "td/utils/PathView.h" -#include "td/utils/port/thread.h" #include "td/utils/port/Stat.h" -#include "td/utils/Timer.h" +#include "td/utils/port/thread.h" #include "td/utils/tl_helpers.h" -#include "td/utils/crypto.h" +#include "vm/atom.h" +#include "vm/boc.h" +#include "vm/box.hpp" +#include "vm/cells.h" +#include "vm/cellslice.h" +#include "vm/cp0.h" +#include "vm/dict.h" +#include "vm/vm.h" -#include +#include "Dictionary.h" +#include "Ed25519.h" +#include "HashMap.h" +#include "IntCtx.h" +#include "SourceLookup.h" +#include "words.h" namespace fift { diff --git a/crypto/func/analyzer.cpp b/crypto/func/analyzer.cpp index 9d59cf9adf..8a30458185 100644 --- a/crypto/func/analyzer.cpp +++ b/crypto/func/analyzer.cpp @@ -16,9 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "func.h" #include "vm/boc.h" +#include "func.h" + namespace funC { /* @@ -892,7 +893,8 @@ bool Op::mark_noreturn() { return set_noreturn(true); case _If: case _TryCatch: - return set_noreturn((int(block0->mark_noreturn()) & int(block1 && block1->mark_noreturn())) | int(next->mark_noreturn())); + return set_noreturn((int(block0->mark_noreturn()) & int(block1 && block1->mark_noreturn())) | + int(next->mark_noreturn())); case _Again: block0->mark_noreturn(); return set_noreturn(true); diff --git a/crypto/func/asmops.cpp b/crypto/func/asmops.cpp index 71ee58f685..93fd58f8c0 100644 --- a/crypto/func/asmops.cpp +++ b/crypto/func/asmops.cpp @@ -16,9 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include + #include "parser/srcread.h" + #include "func.h" -#include namespace funC { diff --git a/crypto/func/builtins.cpp b/crypto/func/builtins.cpp index cf3adf4188..db0b9e7302 100644 --- a/crypto/func/builtins.cpp +++ b/crypto/func/builtins.cpp @@ -1004,7 +1004,7 @@ AsmOp compile_cond_throw(std::vector& res, std::vector& args AsmOp compile_throw_arg(std::vector& res, std::vector& args, const SrcLocation&) { func_assert(res.empty() && args.size() == 2); - VarDescr &x = args[1]; + VarDescr& x = args[1]; if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { x.unused(); return exec_arg_op("THROWARG", x.int_const, 1, 0); diff --git a/crypto/func/func-main.cpp b/crypto/func/func-main.cpp index c8208eeea4..f7e372ef2a 100644 --- a/crypto/func/func-main.cpp +++ b/crypto/func/func-main.cpp @@ -25,12 +25,14 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "func.h" -#include "parser/srcread.h" +#include +#include + #include "parser/lexer.h" +#include "parser/srcread.h" #include "parser/symtable.h" -#include -#include + +#include "func.h" #include "git.h" void usage(const char* progname) { @@ -96,7 +98,8 @@ int main(int argc, char* const argv[]) { break; case 'V': std::cout << "FunC semantic version: v" << funC::func_version << "\n"; - std::cout << "Build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "Build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); break; case 'h': @@ -105,7 +108,7 @@ int main(int argc, char* const argv[]) { } } - std::ostream *outs = &std::cout; + std::ostream* outs = &std::cout; std::unique_ptr fs; if (!output_filename.empty()) { diff --git a/crypto/func/func.cpp b/crypto/func/func.cpp index 39648c05a0..39fc76cf7a 100644 --- a/crypto/func/func.cpp +++ b/crypto/func/func.cpp @@ -25,14 +25,16 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "func.h" -#include "parser/srcread.h" -#include "parser/lexer.h" -#include -#include "git.h" #include +#include + +#include "parser/lexer.h" +#include "parser/srcread.h" #include "td/utils/port/path.h" +#include "func.h" +#include "git.h" + namespace funC { int verbosity, indent, opt_level = 2; @@ -70,7 +72,7 @@ td::Result fs_read_callback(ReadCallback::Kind kind, const char* qu * */ -void generate_output_func(SymDef* func_sym, std::ostream &outs, std::ostream &errs) { +void generate_output_func(SymDef* func_sym, std::ostream& outs, std::ostream& errs) { SymValCodeFunc* func_val = dynamic_cast(func_sym->value); func_assert(func_val); std::string name = sym::symbols.get_name(func_sym->sym_idx); @@ -156,7 +158,7 @@ void generate_output_func(SymDef* func_sym, std::ostream &outs, std::ostream &er } } -int generate_output(std::ostream &outs, std::ostream &errs) { +int generate_output(std::ostream& outs, std::ostream& errs) { if (asm_preamble) { outs << "\"Asm.fif\" include\n"; } @@ -186,7 +188,7 @@ int generate_output(std::ostream &outs, std::ostream &errs) { generate_output_func(func_sym, outs, errs); } catch (src::Error& err) { errs << "cannot generate code for function `" << sym::symbols.get_name(func_sym->sym_idx) << "`:\n" - << err << std::endl; + << err << std::endl; ++errors; } } @@ -199,7 +201,7 @@ int generate_output(std::ostream &outs, std::ostream &errs) { return errors; } -void output_inclusion_stack(std::ostream &errs) { +void output_inclusion_stack(std::ostream& errs) { while (!funC::inclusion_locations.empty()) { src::SrcLocation loc = funC::inclusion_locations.top(); funC::inclusion_locations.pop(); @@ -211,8 +213,7 @@ void output_inclusion_stack(std::ostream &errs) { } } - -int func_proceed(const std::vector &sources, std::ostream &outs, std::ostream &errs) { +int func_proceed(const std::vector& sources, std::ostream& outs, std::ostream& errs) { if (funC::program_envelope && !funC::indent) { funC::indent = 1; } diff --git a/crypto/func/func.h b/crypto/func/func.h index 25711db8f2..e02729de9d 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -17,19 +17,20 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include -#include +#include +#include +#include #include #include +#include #include -#include -#include -#include -#include "common/refcnt.hpp" +#include + #include "common/bigint.hpp" +#include "common/refcnt.hpp" #include "common/refint.h" -#include "parser/srcread.h" #include "parser/lexer.h" +#include "parser/srcread.h" #include "parser/symtable.h" #include "td/utils/Status.h" @@ -300,8 +301,8 @@ void unify(TypeExpr*& te1, TypeExpr*& te2); using src::Lexem; using src::SrcLocation; -using sym::SymDef; using sym::sym_idx_t; +using sym::SymDef; using sym::var_idx_t; using const_idx_t = int; @@ -313,7 +314,7 @@ struct TmpVar { sym_idx_t name; int coord; std::unique_ptr where; - std::vector> on_modification; + std::vector> on_modification; bool undefined = false; TmpVar(var_idx_t _idx, int _cls, TypeExpr* _type = 0, SymDef* sym = 0, const SrcLocation* loc = 0); void show(std::ostream& os, int omit_idx = 0) const; @@ -825,12 +826,10 @@ struct SymValConst : sym::SymValBase { td::RefInt256 intval; std::string strval; Keyword type; - SymValConst(int idx, td::RefInt256 value) - : sym::SymValBase(_Const, idx), intval(value) { + SymValConst(int idx, td::RefInt256 value) : sym::SymValBase(_Const, idx), intval(value) { type = _Int; } - SymValConst(int idx, std::string value) - : sym::SymValBase(_Const, idx), strval(value) { + SymValConst(int idx, std::string value) : sym::SymValBase(_Const, idx), strval(value) { type = _Slice; } ~SymValConst() override = default; @@ -856,27 +855,21 @@ extern std::set prohibited_var_names; */ class ReadCallback { -public: + public: /// Noncopyable. ReadCallback(ReadCallback const&) = delete; ReadCallback& operator=(ReadCallback const&) = delete; - enum class Kind - { - ReadFile, - Realpath - }; + enum class Kind { ReadFile, Realpath }; - static std::string kindString(Kind _kind) - { - switch (_kind) - { - case Kind::ReadFile: - return "source"; - case Kind::Realpath: - return "realpath"; - default: - throw ""; // todo ? + static std::string kindString(Kind _kind) { + switch (_kind) { + case Kind::ReadFile: + return "source"; + case Kind::Realpath: + return "realpath"; + default: + throw ""; // todo ? } } @@ -978,7 +971,8 @@ struct Expr { } int define_new_vars(CodeBlob& code); int predefine_vars(); - std::vector pre_compile(CodeBlob& code, std::vector>* lval_globs = nullptr) const; + std::vector pre_compile(CodeBlob& code, + std::vector>* lval_globs = nullptr) const; static std::vector pre_compile_let(CodeBlob& code, Expr* lhs, Expr* rhs, const SrcLocation& here); var_idx_t new_tmp(CodeBlob& code) const; std::vector new_tmp_vect(CodeBlob& code) const { @@ -1036,7 +1030,8 @@ struct AsmOp { void out_indent_nl(std::ostream& os, bool no_nl = false) const; std::string to_string() const; void compute_gconst() { - gconst = (is_custom() && (op == "PUSHNULL" || op == "NEWC" || op == "NEWB" || op == "TRUE" || op == "FALSE" || op == "NOW")); + gconst = (is_custom() && + (op == "PUSHNULL" || op == "NEWC" || op == "NEWB" || op == "TRUE" || op == "FALSE" || op == "NOW")); } bool is_nop() const { return t == a_none && op.empty(); @@ -1234,7 +1229,7 @@ struct AsmOpList { ip->indent = (ip == list_.begin()) ? indent_ : (ip - 1)->indent; } void indent_all() { - for (auto &op : list_) { + for (auto& op : list_) { ++op.indent; } } @@ -1333,7 +1328,7 @@ struct StackTransform { // c := a * b static bool compose(const StackTransform& a, const StackTransform& b, StackTransform& c); StackTransform& operator*=(const StackTransform& other); - StackTransform operator*(const StackTransform& b) const &; + StackTransform operator*(const StackTransform& b) const&; bool equal(const StackTransform& other, bool relaxed = false) const; bool almost_equal(const StackTransform& other) const { return equal(other, true); @@ -1569,8 +1564,14 @@ struct Stack { StackLayoutExt s; AsmOpList& o; enum { - _StkCmt = 1, _CptStkCmt = 2, _DisableOpt = 4, _DisableOut = 128, _Shown = 256, - _InlineFunc = 512, _NeedRetAlt = 1024, _InlineAny = 2048, + _StkCmt = 1, + _CptStkCmt = 2, + _DisableOpt = 4, + _DisableOut = 128, + _Shown = 256, + _InlineFunc = 512, + _NeedRetAlt = 1024, + _InlineAny = 2048, _ModeSave = _InlineFunc | _NeedRetAlt | _InlineAny, _Garbage = -0x10000 }; @@ -1646,7 +1647,7 @@ struct Stack { show(mode); } } - bool operator==(const Stack& y) const & { + bool operator==(const Stack& y) const& { return s == y.s; } void apply_wrappers(int callxargs_count) { @@ -1688,7 +1689,7 @@ inline simple_compile_func_t make_simple_compile(AsmOp op) { } inline compile_func_t make_ext_compile(std::vector ops) { - return [ops = std::move(ops)](AsmOpList & dest, std::vector & out, std::vector & in)->bool { + return [ops = std::move(ops)](AsmOpList& dest, std::vector& out, std::vector& in) -> bool { return dest.append(ops); }; } @@ -1736,7 +1737,6 @@ AsmOp push_const(td::RefInt256 x); void define_builtins(); - extern int verbosity, indent, opt_level; extern bool stack_layout_comments, op_rewrite_comments, program_envelope, asm_preamble, interactive; extern std::string generated_from, boc_output_filename; @@ -1785,8 +1785,6 @@ extern GlobalPragma pragma_allow_post_modification, pragma_compute_asm_ltr; * */ -int func_proceed(const std::vector &sources, std::ostream &outs, std::ostream &errs); +int func_proceed(const std::vector& sources, std::ostream& outs, std::ostream& errs); } // namespace funC - - diff --git a/crypto/func/gen-abscode.cpp b/crypto/func/gen-abscode.cpp index 7421dabbde..0a6998f640 100644 --- a/crypto/func/gen-abscode.cpp +++ b/crypto/func/gen-abscode.cpp @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #include + #include "func.h" using namespace std::literals::string_literals; @@ -228,7 +229,8 @@ var_idx_t Expr::new_tmp(CodeBlob& code) const { void add_set_globs(CodeBlob& code, std::vector>& globs, const SrcLocation& here) { for (const auto& p : globs) { - auto& op = code.emplace_back(here, Op::_SetGlob, std::vector{}, std::vector{ p.second }, p.first); + auto& op = + code.emplace_back(here, Op::_SetGlob, std::vector{}, std::vector{p.second}, p.first); op.flags |= Op::_Impure; } } @@ -267,8 +269,8 @@ std::vector Expr::pre_compile_let(CodeBlob& code, Expr* lhs, Expr* rh return right; } -std::vector pre_compile_tensor(const std::vector args, CodeBlob &code, - std::vector> *lval_globs, +std::vector pre_compile_tensor(const std::vector args, CodeBlob& code, + std::vector>* lval_globs, std::vector arg_order) { if (arg_order.empty()) { arg_order.resize(args.size()); @@ -288,22 +290,21 @@ std::vector pre_compile_tensor(const std::vector args, CodeBl TmpVar& var = code.vars.at(res_lists[i][j]); if (code.flags & CodeBlob::_AllowPostModification) { if (!lval_globs && (var.cls & TmpVar::_Named)) { - Op *op = &code.emplace_back(nullptr, Op::_Let, std::vector(), std::vector()); + Op* op = &code.emplace_back(nullptr, Op::_Let, std::vector(), std::vector()); op->flags |= Op::_Disabled; - var.on_modification.push_back([modified_vars, i, j, op, done = false](const SrcLocation &here) mutable { + var.on_modification.push_back([modified_vars, i, j, op, done = false](const SrcLocation& here) mutable { if (!done) { done = true; modified_vars->push_back({i, j, op}); } }); } else { - var.on_modification.push_back([](const SrcLocation &) { - }); + var.on_modification.push_back([](const SrcLocation&) {}); } } else { - var.on_modification.push_back([name = var.to_string()](const SrcLocation &here) { - throw src::ParseError{here, PSTRING() << "Modifying local variable " << name - << " after using it in the same expression"}; + var.on_modification.push_back([name = var.to_string()](const SrcLocation& here) { + throw src::ParseError{ + here, PSTRING() << "Modifying local variable " << name << " after using it in the same expression"}; }); } } @@ -314,7 +315,7 @@ std::vector pre_compile_tensor(const std::vector args, CodeBl code.vars.at(v).on_modification.pop_back(); } } - for (const ModifiedVar &m : *modified_vars) { + for (const ModifiedVar& m : *modified_vars) { var_idx_t& v = res_lists[m.i][m.j]; var_idx_t v2 = code.create_tmp_var(code.vars[v].v_type, code.vars[v].where.get()); m.op->left = {v2}; @@ -395,7 +396,7 @@ std::vector Expr::pre_compile(CodeBlob& code, std::vectorpush_back({ sym, rvect[0] }); + lval_globs->push_back({sym, rvect[0]}); return rvect; } else { code.emplace_back(here, Op::_GlobVar, rvect, std::vector{}, sym); diff --git a/crypto/func/keywords.cpp b/crypto/func/keywords.cpp index fedce9db23..cd73c9a8fc 100644 --- a/crypto/func/keywords.cpp +++ b/crypto/func/keywords.cpp @@ -130,8 +130,7 @@ void define_keywords() { .add_keyword("infixr", Kw::_Infixr) .add_keyword("const", Kw::_Const); - sym::symbols.add_keyword("#pragma", Kw::_PragmaHashtag) - .add_keyword("#include", Kw::_IncludeHashtag); + sym::symbols.add_keyword("#pragma", Kw::_PragmaHashtag).add_keyword("#include", Kw::_IncludeHashtag); } } // namespace funC diff --git a/crypto/func/optimize.cpp b/crypto/func/optimize.cpp index 74bb97ecde..258663901a 100644 --- a/crypto/func/optimize.cpp +++ b/crypto/func/optimize.cpp @@ -33,7 +33,7 @@ void Optimizer::set_code(AsmOpConsList code) { void Optimizer::unpack() { int i = 0, j = 0; - for (AsmOpCons *p = code_.get(); p && i < n; p = p->cdr.get(), ++j) { + for (AsmOpCons* p = code_.get(); p && i < n; p = p->cdr.get(), ++j) { if (p->car->is_very_custom()) { break; } diff --git a/crypto/func/parse-func.cpp b/crypto/func/parse-func.cpp index 179cc9bb41..d2f2219bf1 100644 --- a/crypto/func/parse-func.cpp +++ b/crypto/func/parse-func.cpp @@ -16,12 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "func.h" -#include "td/utils/crypto.h" +#include "block/block.h" #include "common/refint.h" #include "openssl/digest.hpp" -#include "block/block.h" +#include "td/utils/crypto.h" + #include "block-parse.h" +#include "func.h" namespace sym { @@ -270,25 +271,25 @@ void parse_const_decl(Lexer& lex) { code.flags |= CodeBlob::_ComputeAsmLtr; } // Handles processing and resolution of literals and consts - auto x = parse_expr(lex, code, false); // also does lex.next() ! + auto x = parse_expr(lex, code, false); // also does lex.next() ! if (x->flags != Expr::_IsRvalue) { lex.cur().error("expression is not strictly Rvalue"); } if ((wanted_type == Expr::_Const) && (x->cls == Expr::_Apply)) - wanted_type = Expr::_None; // Apply is additionally checked to result in an integer + wanted_type = Expr::_None; // Apply is additionally checked to result in an integer if ((wanted_type != Expr::_None) && (x->cls != wanted_type)) { lex.cur().error("expression type does not match wanted type"); } SymValConst* new_value = nullptr; - if (x->cls == Expr::_Const) { // Integer constant + if (x->cls == Expr::_Const) { // Integer constant new_value = new SymValConst{const_cnt++, x->intval}; - } else if (x->cls == Expr::_SliceConst) { // Slice constant (string) + } else if (x->cls == Expr::_SliceConst) { // Slice constant (string) new_value = new SymValConst{const_cnt++, x->strval}; } else if (x->cls == Expr::_Apply) { code.emplace_back(loc, Op::_Import, std::vector()); auto tmp_vars = x->pre_compile(code); code.emplace_back(loc, Op::_Return, std::move(tmp_vars)); - code.emplace_back(loc, Op::_Nop); // This is necessary to prevent SIGSEGV! + code.emplace_back(loc, Op::_Nop); // This is necessary to prevent SIGSEGV! // It is REQUIRED to execute "optimizations" as in func.cpp code.simplify_var_types(); code.prune_unreachable_code(); @@ -427,7 +428,8 @@ Expr* make_func_apply(Expr* fun, Expr* x) { res->flags = Expr::_IsRvalue | (fun->flags & Expr::_IsImpure); } else { res = new Expr{Expr::_VarApply, {fun, x}}; - res->flags = Expr::_IsRvalue | Expr::_IsImpure; // for `some_var()`, don't make any considerations about runtime value, it's impure + res->flags = Expr::_IsRvalue | + Expr::_IsImpure; // for `some_var()`, don't make any considerations about runtime value, it's impure } return res; } @@ -503,8 +505,7 @@ Expr* parse_expr100(Lexer& lex, CodeBlob& code, bool nv) { switch (str_type) { case 0: case 's': - case 'a': - { + case 'a': { res = new Expr{Expr::_SliceConst, lex.cur().loc}; res->e_type = TypeExpr::new_atomic(_Slice); break; @@ -512,14 +513,12 @@ Expr* parse_expr100(Lexer& lex, CodeBlob& code, bool nv) { case 'u': case 'h': case 'H': - case 'c': - { + case 'c': { res = new Expr{Expr::_Const, lex.cur().loc}; res->e_type = TypeExpr::new_atomic(_Int); break; } - default: - { + default: { res = new Expr{Expr::_Const, lex.cur().loc}; res->e_type = TypeExpr::new_atomic(_Int); lex.cur().error("invalid string type `" + std::string(1, static_cast(str_type)) + "`"); @@ -535,7 +534,8 @@ Expr* parse_expr100(Lexer& lex, CodeBlob& code, bool nv) { case 's': { res->strval = str; unsigned char buff[128]; - int bits = (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.data(), str.data() + str.size()); + int bits = + (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.data(), str.data() + str.size()); if (bits < 0) { lex.cur().error_at("Invalid hex bitstring constant `", "`"); } @@ -561,15 +561,13 @@ Expr* parse_expr100(Lexer& lex, CodeBlob& code, bool nv) { break; } case 'h': - case 'H': - { + case 'H': { unsigned char hash[32]; digest::hash_str(hash, str.data(), str.size()); res->intval = td::bits_to_refint(hash, (str_type == 'h') ? 32 : 256, false); break; } - case 'c': - { + case 'c': { res->intval = td::make_refint(td::crc32(td::Slice{str})); break; } @@ -625,12 +623,10 @@ Expr* parse_expr100(Lexer& lex, CodeBlob& code, bool nv) { if (val->type == _Int) { res->cls = Expr::_Const; res->intval = val->get_int_value(); - } - else if (val->type == _Slice) { + } else if (val->type == _Slice) { res->cls = Expr::_SliceConst; res->strval = val->get_str_value(); - } - else { + } else { lex.cur().error("Invalid symbolic constant type"); } res->e_type = TypeExpr::new_atomic(val->type); @@ -1317,7 +1313,7 @@ SymValAsmFunc* parse_asm_func_body(Lexer& lex, TypeExpr* func_type, const Formal lex.expect(')'); } while (lex.tp() == _String) { - std::string ops = lex.cur().str; // \n\n... + std::string ops = lex.cur().str; // \n\n... std::string op; for (const char& c : ops) { if (c == '\n') { @@ -1350,10 +1346,10 @@ SymValAsmFunc* parse_asm_func_body(Lexer& lex, TypeExpr* func_type, const Formal } crc_s.push_back(impure); for (const int& x : arg_order) { - crc_s += std::string((const char*) (&x), (const char*) (&x + 1)); + crc_s += std::string((const char*)(&x), (const char*)(&x + 1)); } for (const int& x : ret_order) { - crc_s += std::string((const char*) (&x), (const char*) (&x + 1)); + crc_s += std::string((const char*)(&x), (const char*)(&x + 1)); } auto res = new SymValAsmFunc{func_type, asm_ops, impure}; res->arg_order = std::move(arg_order); @@ -1586,7 +1582,8 @@ void parse_pragma(Lexer& lex) { lex.next(); if (!pragma_name.compare("version") || !pragma_name.compare("not-version")) { bool negate = !pragma_name.compare("not-version"); - char op = '='; bool eq = false; + char op = '='; + bool eq = false; int sem_ver[3] = {0, 0, 0}; char segs = 1; auto stoi = [&](const std::string& s) { @@ -1663,32 +1660,29 @@ void parse_pragma(Lexer& lex) { bool match = true; switch (op) { case '=': - if ((func_ver[0] != sem_ver[0]) || - (func_ver[1] != sem_ver[1]) || - (func_ver[2] != sem_ver[2])) { + if ((func_ver[0] != sem_ver[0]) || (func_ver[1] != sem_ver[1]) || (func_ver[2] != sem_ver[2])) { match = false; } break; case '>': - if ( ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] == sem_ver[2]) && !eq) || - ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] < sem_ver[2])) || - ((func_ver[0] == sem_ver[0]) && (func_ver[1] < sem_ver[1])) || - ((func_ver[0] < sem_ver[0])) ) { + if (((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] == sem_ver[2]) && !eq) || + ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] < sem_ver[2])) || + ((func_ver[0] == sem_ver[0]) && (func_ver[1] < sem_ver[1])) || ((func_ver[0] < sem_ver[0]))) { match = false; } break; case '<': - if ( ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] == sem_ver[2]) && !eq) || - ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] > sem_ver[2])) || - ((func_ver[0] == sem_ver[0]) && (func_ver[1] > sem_ver[1])) || - ((func_ver[0] > sem_ver[0])) ) { + if (((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] == sem_ver[2]) && !eq) || + ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] > sem_ver[2])) || + ((func_ver[0] == sem_ver[0]) && (func_ver[1] > sem_ver[1])) || ((func_ver[0] > sem_ver[0]))) { match = false; } break; case '^': - if ( ((segs == 3) && ((func_ver[0] != sem_ver[0]) || (func_ver[1] != sem_ver[1]) || (func_ver[2] < sem_ver[2]))) - || ((segs == 2) && ((func_ver[0] != sem_ver[0]) || (func_ver[1] < sem_ver[1]))) - || ((segs == 1) && ((func_ver[0] < sem_ver[0]))) ) { + if (((segs == 3) && + ((func_ver[0] != sem_ver[0]) || (func_ver[1] != sem_ver[1]) || (func_ver[2] < sem_ver[2]))) || + ((segs == 2) && ((func_ver[0] != sem_ver[0]) || (func_ver[1] < sem_ver[1]))) || + ((segs == 1) && ((func_ver[0] < sem_ver[0])))) { match = false; } break; @@ -1783,7 +1777,7 @@ bool parse_source_file(const char* filename, src::Lexem lex, bool is_main) { } return true; } - if (lex.tp) { // included + if (lex.tp) { // included funC::generated_from += std::string{"incl:"}; } funC::generated_from += std::string{"`"} + filename + "` "; diff --git a/crypto/funcfiftlib/funcfiftlib.cpp b/crypto/funcfiftlib/funcfiftlib.cpp index 403c075dd2..901d82440b 100644 --- a/crypto/funcfiftlib/funcfiftlib.cpp +++ b/crypto/funcfiftlib/funcfiftlib.cpp @@ -25,28 +25,30 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include +#include + +#include "fift/utils.h" #include "func/func.h" -#include "git.h" #include "td/utils/JsonBuilder.h" -#include "fift/utils.h" -#include "td/utils/base64.h" #include "td/utils/Status.h" -#include -#include +#include "td/utils/base64.h" #include "vm/boc.h" -td::Result compile_internal(char *config_json) { +#include "git.h" + +td::Result compile_internal(char* config_json) { TRY_RESULT(input_json, td::json_decode(td::MutableSlice(config_json))) td::JsonObject& config = input_json.get_object(); TRY_RESULT(opt_level, td::get_json_object_int_field(config, "optLevel", false)); TRY_RESULT(sources_obj, td::get_json_object_field(config, "sources", td::JsonValue::Type::Array, false)); - auto &sources_arr = sources_obj.get_array(); + auto& sources_arr = sources_obj.get_array(); std::vector sources; - for (auto &item : sources_arr) { + for (auto& item : sources_arr) { sources.push_back(item.get_string().str()); } @@ -89,8 +91,7 @@ td::Result compile_internal(char *config_json) { /// If the callback is not supported, *o_contents and *o_error must be set to NULL. typedef void (*CStyleReadFileCallback)(char const* _kind, char const* _data, char** o_contents, char** o_error); -funC::ReadCallback::Callback wrapReadCallback(CStyleReadFileCallback _readCallback) -{ +funC::ReadCallback::Callback wrapReadCallback(CStyleReadFileCallback _readCallback) { funC::ReadCallback::Callback readCallback; if (_readCallback) { readCallback = [=](funC::ReadCallback::Kind _kind, char const* _data) -> td::Result { @@ -121,7 +122,7 @@ const char* version() { return strdup(version_json.string_builder().as_cslice().c_str()); } -const char *func_compile(char *config_json, CStyleReadFileCallback callback) { +const char* func_compile(char* config_json, CStyleReadFileCallback callback) { if (callback) { funC::read_callback = wrapReadCallback(callback); } else { diff --git a/crypto/keccak/keccak.cpp b/crypto/keccak/keccak.cpp index aac48ddc49..70bce10916 100644 --- a/crypto/keccak/keccak.cpp +++ b/crypto/keccak/keccak.cpp @@ -26,241 +26,225 @@ * =================================================================== */ -#include #include #include #include +#include #define KECCAK_F1600_STATE 200 #define MIN(x, y) ((x) < (y) ? (x) : (y)) #define MAX(x, y) ((x) < (y) ? (x) : (y)) /** Standard errors common to all ciphers **/ -#define ERR_NULL 1 -#define ERR_MEMORY 2 -#define ERR_NOT_ENOUGH_DATA 3 -#define ERR_ENCRYPT 4 -#define ERR_DECRYPT 5 -#define ERR_KEY_SIZE 6 -#define ERR_NONCE_SIZE 7 -#define ERR_NR_ROUNDS 8 -#define ERR_DIGEST_SIZE 9 -#define ERR_MAX_DATA 10 -#define ERR_MAX_OFFSET 11 -#define ERR_BLOCK_SIZE 12 -#define ERR_TAG_SIZE 13 -#define ERR_VALUE 14 -#define ERR_EC_POINT 15 -#define ERR_EC_CURVE 16 -#define ERR_MODULUS 17 -#define ERR_UNKNOWN 32 - -typedef struct -{ - uint64_t state[25]; - - /* The buffer is as long as the state, +#define ERR_NULL 1 +#define ERR_MEMORY 2 +#define ERR_NOT_ENOUGH_DATA 3 +#define ERR_ENCRYPT 4 +#define ERR_DECRYPT 5 +#define ERR_KEY_SIZE 6 +#define ERR_NONCE_SIZE 7 +#define ERR_NR_ROUNDS 8 +#define ERR_DIGEST_SIZE 9 +#define ERR_MAX_DATA 10 +#define ERR_MAX_OFFSET 11 +#define ERR_BLOCK_SIZE 12 +#define ERR_TAG_SIZE 13 +#define ERR_VALUE 14 +#define ERR_EC_POINT 15 +#define ERR_EC_CURVE 16 +#define ERR_MODULUS 17 +#define ERR_UNKNOWN 32 + +typedef struct { + uint64_t state[25]; + + /* The buffer is as long as the state, * but only 'rate' bytes will be used. */ - uint8_t buf[KECCAK_F1600_STATE]; + uint8_t buf[KECCAK_F1600_STATE]; - /* When absorbing, this is the number of bytes in buf that + /* When absorbing, this is the number of bytes in buf that * are coming from the message and outstanding. * When squeezing, this is the remaining number of bytes * that can be used as digest. */ - unsigned valid_bytes; + unsigned valid_bytes; - /* All values in bytes */ - unsigned capacity; - unsigned rate; + /* All values in bytes */ + unsigned capacity; + unsigned rate; - uint8_t squeezing; - uint8_t rounds; + uint8_t squeezing; + uint8_t rounds; } keccak_state; #undef ROL64 -#define ROL64(x,y) ((((x) << (y)) | (x) >> (64-(y))) & 0xFFFFFFFFFFFFFFFFULL) +#define ROL64(x, y) ((((x) << (y)) | (x) >> (64 - (y))) & 0xFFFFFFFFFFFFFFFFULL) -static void keccak_function (uint64_t *state, unsigned rounds); +static void keccak_function(uint64_t *state, unsigned rounds); -int keccak_reset(keccak_state *state) -{ - if (NULL == state) - return ERR_NULL; +int keccak_reset(keccak_state *state) { + if (NULL == state) + return ERR_NULL; - memset(state->state, 0, sizeof(state->state)); - memset(state->buf, 0, sizeof(state->buf)); - state->valid_bytes = 0; - state->squeezing = 0; + memset(state->state, 0, sizeof(state->state)); + memset(state->buf, 0, sizeof(state->buf)); + state->valid_bytes = 0; + state->squeezing = 0; - return 0; + return 0; } -static void keccak_absorb_internal (keccak_state *self) -{ - unsigned i,j; - uint64_t d; +static void keccak_absorb_internal(keccak_state *self) { + unsigned i, j; + uint64_t d; - for (i=j=0; j < self->rate; ++i, j += 8) { - d = *(const uint64_t*)(self->buf + j); - self->state[i] ^= d; - } + for (i = j = 0; j < self->rate; ++i, j += 8) { + d = *(const uint64_t *)(self->buf + j); + self->state[i] ^= d; + } } -static void -keccak_squeeze_internal (keccak_state *self) -{ - unsigned i, j; +static void keccak_squeeze_internal(keccak_state *self) { + unsigned i, j; - for (i=j=0; j < self->rate; ++i, j += 8) { - *(uint64_t*)(self->buf+j) = self->state[i]; - } + for (i = j = 0; j < self->rate; ++i, j += 8) { + *(uint64_t *)(self->buf + j) = self->state[i]; + } } -int keccak_init (keccak_state **state, - size_t capacity_bytes, - uint8_t rounds) -{ - keccak_state *ks; +int keccak_init(keccak_state **state, size_t capacity_bytes, uint8_t rounds) { + keccak_state *ks; - if (NULL == state) { - return ERR_NULL; - } + if (NULL == state) { + return ERR_NULL; + } - *state = ks = (keccak_state*) calloc(1, sizeof(keccak_state)); - if (NULL == ks) - return ERR_MEMORY; + *state = ks = (keccak_state *)calloc(1, sizeof(keccak_state)); + if (NULL == ks) + return ERR_MEMORY; - if (capacity_bytes >= KECCAK_F1600_STATE) - return ERR_DIGEST_SIZE; + if (capacity_bytes >= KECCAK_F1600_STATE) + return ERR_DIGEST_SIZE; - if ((rounds != 12) && (rounds != 24)) - return ERR_NR_ROUNDS; + if ((rounds != 12) && (rounds != 24)) + return ERR_NR_ROUNDS; - ks->capacity = (unsigned)capacity_bytes; + ks->capacity = (unsigned)capacity_bytes; - ks->rate = KECCAK_F1600_STATE - ks->capacity; + ks->rate = KECCAK_F1600_STATE - ks->capacity; - ks->squeezing = 0; - ks->rounds = rounds; + ks->squeezing = 0; + ks->rounds = rounds; - return 0; + return 0; } -int keccak_destroy(keccak_state *state) -{ - free(state); - return 0; +int keccak_destroy(keccak_state *state) { + free(state); + return 0; } -int keccak_absorb (keccak_state *self, - const uint8_t *in, - size_t length) -{ - if (NULL==self || NULL==in) - return ERR_NULL; - - if (self->squeezing != 0) - return ERR_UNKNOWN; - - while (length > 0) { - unsigned tc; - unsigned left; - - left = self->rate - self->valid_bytes; - tc = (unsigned) MIN(length, left); - memcpy(self->buf + self->valid_bytes, in, tc); - - self->valid_bytes += tc; - in += tc; - length -= tc; - - if (self->valid_bytes == self->rate) { - keccak_absorb_internal (self); - keccak_function(self->state, self->rounds); - self->valid_bytes = 0; - } +int keccak_absorb(keccak_state *self, const uint8_t *in, size_t length) { + if (NULL == self || NULL == in) + return ERR_NULL; + + if (self->squeezing != 0) + return ERR_UNKNOWN; + + while (length > 0) { + unsigned tc; + unsigned left; + + left = self->rate - self->valid_bytes; + tc = (unsigned)MIN(length, left); + memcpy(self->buf + self->valid_bytes, in, tc); + + self->valid_bytes += tc; + in += tc; + length -= tc; + + if (self->valid_bytes == self->rate) { + keccak_absorb_internal(self); + keccak_function(self->state, self->rounds); + self->valid_bytes = 0; } + } - return 0; + return 0; } -static void keccak_finish (keccak_state *self, uint8_t padding) -{ - assert(self->squeezing == 0); - assert(self->valid_bytes < self->rate); +static void keccak_finish(keccak_state *self, uint8_t padding) { + assert(self->squeezing == 0); + assert(self->valid_bytes < self->rate); - /* Padding */ - memset(self->buf + self->valid_bytes, 0, self->rate - self->valid_bytes); - self->buf[self->valid_bytes] = padding; - self->buf[self->rate-1] |= 0x80; + /* Padding */ + memset(self->buf + self->valid_bytes, 0, self->rate - self->valid_bytes); + self->buf[self->valid_bytes] = padding; + self->buf[self->rate - 1] |= 0x80; - /* Final absorb */ - keccak_absorb_internal (self); - keccak_function (self->state, self->rounds); + /* Final absorb */ + keccak_absorb_internal(self); + keccak_function(self->state, self->rounds); - /* First squeeze */ - self->squeezing = 1; - keccak_squeeze_internal (self); - self->valid_bytes = self->rate; + /* First squeeze */ + self->squeezing = 1; + keccak_squeeze_internal(self); + self->valid_bytes = self->rate; } -int keccak_squeeze (keccak_state *self, uint8_t *out, size_t length, uint8_t padding) -{ - if ((NULL == self) || (NULL == out)) - return ERR_NULL; +int keccak_squeeze(keccak_state *self, uint8_t *out, size_t length, uint8_t padding) { + if ((NULL == self) || (NULL == out)) + return ERR_NULL; - if (self->squeezing == 0) { - keccak_finish (self, padding); - } + if (self->squeezing == 0) { + keccak_finish(self, padding); + } - assert(self->squeezing == 1); - assert(self->valid_bytes > 0); - assert(self->valid_bytes <= self->rate); + assert(self->squeezing == 1); + assert(self->valid_bytes > 0); + assert(self->valid_bytes <= self->rate); - while (length > 0) { - unsigned tc; + while (length > 0) { + unsigned tc; - tc = (unsigned)MIN(self->valid_bytes, length); - memcpy(out, self->buf + (self->rate - self->valid_bytes), tc); + tc = (unsigned)MIN(self->valid_bytes, length); + memcpy(out, self->buf + (self->rate - self->valid_bytes), tc); - self->valid_bytes -= tc; - out += tc; - length -= tc; + self->valid_bytes -= tc; + out += tc; + length -= tc; - if (self->valid_bytes == 0) { - keccak_function (self->state, self->rounds); - keccak_squeeze_internal (self); - self->valid_bytes = self->rate; - } + if (self->valid_bytes == 0) { + keccak_function(self->state, self->rounds); + keccak_squeeze_internal(self); + self->valid_bytes = self->rate; } + } - return 0; + return 0; } -int keccak_digest(keccak_state *state, uint8_t *digest, size_t len, uint8_t padding) -{ - keccak_state tmp; +int keccak_digest(keccak_state *state, uint8_t *digest, size_t len, uint8_t padding) { + keccak_state tmp; - if ((NULL==state) || (NULL==digest)) - return ERR_NULL; + if ((NULL == state) || (NULL == digest)) + return ERR_NULL; - if (2*len != state->capacity) - return ERR_UNKNOWN; + if (2 * len != state->capacity) + return ERR_UNKNOWN; - tmp = *state; - return keccak_squeeze(&tmp, digest, len, padding); + tmp = *state; + return keccak_squeeze(&tmp, digest, len, padding); } -int keccak_copy(const keccak_state *src, keccak_state *dst) -{ - if (NULL == src || NULL == dst) { - return ERR_NULL; - } +int keccak_copy(const keccak_state *src, keccak_state *dst) { + if (NULL == src || NULL == dst) { + return ERR_NULL; + } - *dst = *src; - return 0; + *dst = *src; + return 0; } /* Keccak core function */ @@ -293,181 +277,160 @@ int keccak_copy(const keccak_state *src, keccak_state *dst) #define ROT_24 14 static const uint64_t roundconstants[KECCAK_ROUNDS] = { - 0x0000000000000001ULL, - 0x0000000000008082ULL, - 0x800000000000808aULL, - 0x8000000080008000ULL, - 0x000000000000808bULL, - 0x0000000080000001ULL, - 0x8000000080008081ULL, - 0x8000000000008009ULL, - 0x000000000000008aULL, - 0x0000000000000088ULL, - 0x0000000080008009ULL, - 0x000000008000000aULL, - 0x000000008000808bULL, - 0x800000000000008bULL, - 0x8000000000008089ULL, - 0x8000000000008003ULL, - 0x8000000000008002ULL, - 0x8000000000000080ULL, - 0x000000000000800aULL, - 0x800000008000000aULL, - 0x8000000080008081ULL, - 0x8000000000008080ULL, - 0x0000000080000001ULL, - 0x8000000080008008ULL -}; - -static void keccak_function (uint64_t *state, unsigned rounds) -{ - unsigned i; - unsigned start_round; - - /* Temporary variables to avoid indexing overhead */ - uint64_t a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12; - uint64_t a13, a14, a15, a16, a17, a18, a19, a20, a21, a22, a23, a24; - - uint64_t b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12; - uint64_t b13, b14, b15, b16, b17, b18, b19, b20, b21, b22, b23, b24; - - uint64_t c0, c1, c2, c3, c4, d; - - a0 = state[0]; - a1 = state[1]; - a2 = state[2]; - a3 = state[3]; - a4 = state[4]; - a5 = state[5]; - a6 = state[6]; - a7 = state[7]; - a8 = state[8]; - a9 = state[9]; - a10 = state[10]; - a11 = state[11]; - a12 = state[12]; - a13 = state[13]; - a14 = state[14]; - a15 = state[15]; - a16 = state[16]; - a17 = state[17]; - a18 = state[18]; - a19 = state[19]; - a20 = state[20]; - a21 = state[21]; - a22 = state[22]; - a23 = state[23]; - a24 = state[24]; - - if (rounds == 24) - start_round = 0; - else /* rounds == 12 */ - start_round = 12; - - for (i = start_round; i < KECCAK_ROUNDS; ++i) { - /* + 0x0000000000000001ULL, 0x0000000000008082ULL, 0x800000000000808aULL, 0x8000000080008000ULL, 0x000000000000808bULL, + 0x0000000080000001ULL, 0x8000000080008081ULL, 0x8000000000008009ULL, 0x000000000000008aULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x000000008000000aULL, 0x000000008000808bULL, 0x800000000000008bULL, 0x8000000000008089ULL, + 0x8000000000008003ULL, 0x8000000000008002ULL, 0x8000000000000080ULL, 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, 0x0000000080000001ULL, 0x8000000080008008ULL}; + +static void keccak_function(uint64_t *state, unsigned rounds) { + unsigned i; + unsigned start_round; + + /* Temporary variables to avoid indexing overhead */ + uint64_t a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12; + uint64_t a13, a14, a15, a16, a17, a18, a19, a20, a21, a22, a23, a24; + + uint64_t b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12; + uint64_t b13, b14, b15, b16, b17, b18, b19, b20, b21, b22, b23, b24; + + uint64_t c0, c1, c2, c3, c4, d; + + a0 = state[0]; + a1 = state[1]; + a2 = state[2]; + a3 = state[3]; + a4 = state[4]; + a5 = state[5]; + a6 = state[6]; + a7 = state[7]; + a8 = state[8]; + a9 = state[9]; + a10 = state[10]; + a11 = state[11]; + a12 = state[12]; + a13 = state[13]; + a14 = state[14]; + a15 = state[15]; + a16 = state[16]; + a17 = state[17]; + a18 = state[18]; + a19 = state[19]; + a20 = state[20]; + a21 = state[21]; + a22 = state[22]; + a23 = state[23]; + a24 = state[24]; + + if (rounds == 24) + start_round = 0; + else /* rounds == 12 */ + start_round = 12; + + for (i = start_round; i < KECCAK_ROUNDS; ++i) { + /* Uses temporary variables and loop unrolling to avoid array indexing and inner loops overhead */ - /* Prepare column parity for Theta step */ - c0 = a0 ^ a5 ^ a10 ^ a15 ^ a20; - c1 = a1 ^ a6 ^ a11 ^ a16 ^ a21; - c2 = a2 ^ a7 ^ a12 ^ a17 ^ a22; - c3 = a3 ^ a8 ^ a13 ^ a18 ^ a23; - c4 = a4 ^ a9 ^ a14 ^ a19 ^ a24; - - /* Theta + Rho + Pi steps */ - d = c4 ^ ROL64(c1, 1); - b0 = d ^ a0; - b16 = ROL64(d ^ a5, ROT_01); - b7 = ROL64(d ^ a10, ROT_02); - b23 = ROL64(d ^ a15, ROT_03); - b14 = ROL64(d ^ a20, ROT_04); - - d = c0 ^ ROL64(c2, 1); - b10 = ROL64(d ^ a1, ROT_05); - b1 = ROL64(d ^ a6, ROT_06); - b17 = ROL64(d ^ a11, ROT_07); - b8 = ROL64(d ^ a16, ROT_08); - b24 = ROL64(d ^ a21, ROT_09); - - d = c1 ^ ROL64(c3, 1); - b20 = ROL64(d ^ a2, ROT_10); - b11 = ROL64(d ^ a7, ROT_11); - b2 = ROL64(d ^ a12, ROT_12); - b18 = ROL64(d ^ a17, ROT_13); - b9 = ROL64(d ^ a22, ROT_14); - - d = c2 ^ ROL64(c4, 1); - b5 = ROL64(d ^ a3, ROT_15); - b21 = ROL64(d ^ a8, ROT_16); - b12 = ROL64(d ^ a13, ROT_17); - b3 = ROL64(d ^ a18, ROT_18); - b19 = ROL64(d ^ a23, ROT_19); - - d = c3 ^ ROL64(c0, 1); - b15 = ROL64(d ^ a4, ROT_20); - b6 = ROL64(d ^ a9, ROT_21); - b22 = ROL64(d ^ a14, ROT_22); - b13 = ROL64(d ^ a19, ROT_23); - b4 = ROL64(d ^ a24, ROT_24); - - /* Chi + Iota steps */ - a0 = b0 ^ (~b1 & b2) ^ roundconstants[i]; - a1 = b1 ^ (~b2 & b3); - a2 = b2 ^ (~b3 & b4); - a3 = b3 ^ (~b4 & b0); - a4 = b4 ^ (~b0 & b1); - - a5 = b5 ^ (~b6 & b7); - a6 = b6 ^ (~b7 & b8); - a7 = b7 ^ (~b8 & b9); - a8 = b8 ^ (~b9 & b5); - a9 = b9 ^ (~b5 & b6); - - a10 = b10 ^ (~b11 & b12); - a11 = b11 ^ (~b12 & b13); - a12 = b12 ^ (~b13 & b14); - a13 = b13 ^ (~b14 & b10); - a14 = b14 ^ (~b10 & b11); - - a15 = b15 ^ (~b16 & b17); - a16 = b16 ^ (~b17 & b18); - a17 = b17 ^ (~b18 & b19); - a18 = b18 ^ (~b19 & b15); - a19 = b19 ^ (~b15 & b16); - - a20 = b20 ^ (~b21 & b22); - a21 = b21 ^ (~b22 & b23); - a22 = b22 ^ (~b23 & b24); - a23 = b23 ^ (~b24 & b20); - a24 = b24 ^ (~b20 & b21); - } - - state[0] = a0; - state[1] = a1; - state[2] = a2; - state[3] = a3; - state[4] = a4; - state[5] = a5; - state[6] = a6; - state[7] = a7; - state[8] = a8; - state[9] = a9; - state[10] = a10; - state[11] = a11; - state[12] = a12; - state[13] = a13; - state[14] = a14; - state[15] = a15; - state[16] = a16; - state[17] = a17; - state[18] = a18; - state[19] = a19; - state[20] = a20; - state[21] = a21; - state[22] = a22; - state[23] = a23; - state[24] = a24; + /* Prepare column parity for Theta step */ + c0 = a0 ^ a5 ^ a10 ^ a15 ^ a20; + c1 = a1 ^ a6 ^ a11 ^ a16 ^ a21; + c2 = a2 ^ a7 ^ a12 ^ a17 ^ a22; + c3 = a3 ^ a8 ^ a13 ^ a18 ^ a23; + c4 = a4 ^ a9 ^ a14 ^ a19 ^ a24; + + /* Theta + Rho + Pi steps */ + d = c4 ^ ROL64(c1, 1); + b0 = d ^ a0; + b16 = ROL64(d ^ a5, ROT_01); + b7 = ROL64(d ^ a10, ROT_02); + b23 = ROL64(d ^ a15, ROT_03); + b14 = ROL64(d ^ a20, ROT_04); + + d = c0 ^ ROL64(c2, 1); + b10 = ROL64(d ^ a1, ROT_05); + b1 = ROL64(d ^ a6, ROT_06); + b17 = ROL64(d ^ a11, ROT_07); + b8 = ROL64(d ^ a16, ROT_08); + b24 = ROL64(d ^ a21, ROT_09); + + d = c1 ^ ROL64(c3, 1); + b20 = ROL64(d ^ a2, ROT_10); + b11 = ROL64(d ^ a7, ROT_11); + b2 = ROL64(d ^ a12, ROT_12); + b18 = ROL64(d ^ a17, ROT_13); + b9 = ROL64(d ^ a22, ROT_14); + + d = c2 ^ ROL64(c4, 1); + b5 = ROL64(d ^ a3, ROT_15); + b21 = ROL64(d ^ a8, ROT_16); + b12 = ROL64(d ^ a13, ROT_17); + b3 = ROL64(d ^ a18, ROT_18); + b19 = ROL64(d ^ a23, ROT_19); + + d = c3 ^ ROL64(c0, 1); + b15 = ROL64(d ^ a4, ROT_20); + b6 = ROL64(d ^ a9, ROT_21); + b22 = ROL64(d ^ a14, ROT_22); + b13 = ROL64(d ^ a19, ROT_23); + b4 = ROL64(d ^ a24, ROT_24); + + /* Chi + Iota steps */ + a0 = b0 ^ (~b1 & b2) ^ roundconstants[i]; + a1 = b1 ^ (~b2 & b3); + a2 = b2 ^ (~b3 & b4); + a3 = b3 ^ (~b4 & b0); + a4 = b4 ^ (~b0 & b1); + + a5 = b5 ^ (~b6 & b7); + a6 = b6 ^ (~b7 & b8); + a7 = b7 ^ (~b8 & b9); + a8 = b8 ^ (~b9 & b5); + a9 = b9 ^ (~b5 & b6); + + a10 = b10 ^ (~b11 & b12); + a11 = b11 ^ (~b12 & b13); + a12 = b12 ^ (~b13 & b14); + a13 = b13 ^ (~b14 & b10); + a14 = b14 ^ (~b10 & b11); + + a15 = b15 ^ (~b16 & b17); + a16 = b16 ^ (~b17 & b18); + a17 = b17 ^ (~b18 & b19); + a18 = b18 ^ (~b19 & b15); + a19 = b19 ^ (~b15 & b16); + + a20 = b20 ^ (~b21 & b22); + a21 = b21 ^ (~b22 & b23); + a22 = b22 ^ (~b23 & b24); + a23 = b23 ^ (~b24 & b20); + a24 = b24 ^ (~b20 & b21); + } + + state[0] = a0; + state[1] = a1; + state[2] = a2; + state[3] = a3; + state[4] = a4; + state[5] = a5; + state[6] = a6; + state[7] = a7; + state[8] = a8; + state[9] = a9; + state[10] = a10; + state[11] = a11; + state[12] = a12; + state[13] = a13; + state[14] = a14; + state[15] = a15; + state[16] = a16; + state[17] = a17; + state[18] = a18; + state[19] = a19; + state[20] = a20; + state[21] = a21; + state[22] = a22; + state[23] = a23; + state[24] = a24; } diff --git a/crypto/keccak/keccak.h b/crypto/keccak/keccak.h index 28fa66fb1d..039f933b7e 100644 --- a/crypto/keccak/keccak.h +++ b/crypto/keccak/keccak.h @@ -27,8 +27,8 @@ */ #pragma once -#include #include +#include struct keccak_state; diff --git a/crypto/openssl/bignum.h b/crypto/openssl/bignum.h index 032dbb0250..8b866a76a3 100644 --- a/crypto/openssl/bignum.h +++ b/crypto/openssl/bignum.h @@ -18,10 +18,10 @@ */ #pragma once -#include #include - #include +#include + #include "td/utils/bits.h" #include "td/utils/misc.h" @@ -55,7 +55,7 @@ class BignumBitref { int n; public: - BignumBitref(BIGNUM* x, int _n) : ptr(x), n(_n){}; + BignumBitref(BIGNUM* x, int _n) : ptr(x), n(_n) {}; operator bool() const { return BN_is_bit_set(ptr, n); } diff --git a/crypto/openssl/digest.hpp b/crypto/openssl/digest.hpp index 3e36f7e99c..a5d7a46dbb 100644 --- a/crypto/openssl/digest.hpp +++ b/crypto/openssl/digest.hpp @@ -18,7 +18,6 @@ */ #pragma once #include - #include #include #include diff --git a/crypto/openssl/rand.cpp b/crypto/openssl/rand.cpp index 57f5c05f6d..708d3b7716 100644 --- a/crypto/openssl/rand.cpp +++ b/crypto/openssl/rand.cpp @@ -16,13 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "openssl/rand.hpp" +#include +#include +#include "openssl/rand.hpp" #include "td/utils/common.h" -#include -#include - namespace prng { int os_get_random_bytes(void *buf, int n); diff --git a/crypto/openssl/rand.hpp b/crypto/openssl/rand.hpp index 6ead2be7fc..87c7b52936 100644 --- a/crypto/openssl/rand.hpp +++ b/crypto/openssl/rand.hpp @@ -17,8 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include #include +#include namespace prng { diff --git a/crypto/openssl/residue.h b/crypto/openssl/residue.h index adea1de787..3bf8247a8b 100644 --- a/crypto/openssl/residue.h +++ b/crypto/openssl/residue.h @@ -17,9 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "bignum.h" #include "common/refcnt.hpp" +#include "bignum.h" + namespace arith { class Residue; class ResidueRing; diff --git a/crypto/parser/lexer.cpp b/crypto/parser/lexer.cpp index 117f1df5ac..5403cf1a30 100644 --- a/crypto/parser/lexer.cpp +++ b/crypto/parser/lexer.cpp @@ -16,10 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include +#include + #include "lexer.h" #include "symtable.h" -#include -#include namespace src { @@ -126,8 +127,11 @@ int Lexem::set(std::string _str, const SrcLocation& _loc, int _tp, int _val) { Lexer::Lexer(SourceReader& _src, bool init, std::string active_chars, std::string eol_cmts, std::string open_cmts, std::string close_cmts, std::string quote_chars, std::string multiline_quote) - : src(_src), eof(false), lexem("", src.here(), Lexem::Undefined), peek_lexem("", {}, Lexem::Undefined), - multiline_quote(std::move(multiline_quote)) { + : src(_src) + , eof(false) + , lexem("", src.here(), Lexem::Undefined) + , peek_lexem("", {}, Lexem::Undefined) + , multiline_quote(std::move(multiline_quote)) { std::memset(char_class, 0, sizeof(char_class)); unsigned char activity = cc::active; for (char c : active_chars) { diff --git a/crypto/parser/lexer.h b/crypto/parser/lexer.h index 686d8eacdb..933161512d 100644 --- a/crypto/parser/lexer.h +++ b/crypto/parser/lexer.h @@ -17,10 +17,11 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "srcread.h" #include -#include #include +#include + +#include "srcread.h" namespace src { diff --git a/crypto/parser/srcread.cpp b/crypto/parser/srcread.cpp index 332f15393a..ca644b9f7a 100644 --- a/crypto/parser/srcread.cpp +++ b/crypto/parser/srcread.cpp @@ -16,9 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "srcread.h" #include +#include "srcread.h" + namespace src { /* diff --git a/crypto/parser/srcread.h b/crypto/parser/srcread.h index 61128eff2c..6ae0a8776f 100644 --- a/crypto/parser/srcread.h +++ b/crypto/parser/srcread.h @@ -18,9 +18,9 @@ */ #pragma once +#include #include #include -#include namespace src { diff --git a/crypto/parser/symtable.cpp b/crypto/parser/symtable.cpp index d52d96482e..6d9551cdc2 100644 --- a/crypto/parser/symtable.cpp +++ b/crypto/parser/symtable.cpp @@ -16,9 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "symtable.h" -#include #include +#include + +#include "symtable.h" namespace sym { diff --git a/crypto/parser/symtable.h b/crypto/parser/symtable.h index 51d59dfa42..e1909d1e4c 100644 --- a/crypto/parser/symtable.h +++ b/crypto/parser/symtable.h @@ -17,10 +17,11 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "srcread.h" -#include "lexer.h" #include +#include "lexer.h" +#include "srcread.h" + namespace sym { /* diff --git a/crypto/smc-envelope/GenericAccount.cpp b/crypto/smc-envelope/GenericAccount.cpp index 4c9bd16596..f9c94a3630 100644 --- a/crypto/smc-envelope/GenericAccount.cpp +++ b/crypto/smc-envelope/GenericAccount.cpp @@ -16,10 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "GenericAccount.h" - #include "block/block-auto.h" #include "block/block-parse.h" + +#include "GenericAccount.h" namespace ton { namespace smc { diff --git a/crypto/smc-envelope/GenericAccount.h b/crypto/smc-envelope/GenericAccount.h index 93a059f999..cccd85c6d6 100644 --- a/crypto/smc-envelope/GenericAccount.h +++ b/crypto/smc-envelope/GenericAccount.h @@ -17,8 +17,9 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "vm/cells.h" #include "block/block.h" +#include "vm/cells.h" + #include "Ed25519.h" #include "SmartContract.h" diff --git a/crypto/smc-envelope/HighloadWallet.cpp b/crypto/smc-envelope/HighloadWallet.cpp index 69843fcffa..0d82f4165b 100644 --- a/crypto/smc-envelope/HighloadWallet.cpp +++ b/crypto/smc-envelope/HighloadWallet.cpp @@ -16,15 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "HighloadWallet.h" -#include "GenericAccount.h" -#include "SmartContractCode.h" +#include +#include "td/utils/base64.h" #include "vm/boc.h" #include "vm/cells/CellString.h" -#include "td/utils/base64.h" -#include +#include "GenericAccount.h" +#include "HighloadWallet.h" +#include "SmartContractCode.h" namespace ton { diff --git a/crypto/smc-envelope/HighloadWallet.h b/crypto/smc-envelope/HighloadWallet.h index db36caaeea..da17b12328 100644 --- a/crypto/smc-envelope/HighloadWallet.h +++ b/crypto/smc-envelope/HighloadWallet.h @@ -18,13 +18,14 @@ */ #pragma once +#include "block/block.h" #include "smc-envelope/SmartContract.h" #include "smc-envelope/WalletInterface.h" #include "vm/cells.h" -#include "Ed25519.h" -#include "block/block.h" #include "vm/cells/CellString.h" +#include "Ed25519.h" + namespace ton { struct HighloadWalletTraits { using InitData = WalletInterface::DefaultInitData; diff --git a/crypto/smc-envelope/HighloadWalletV2.cpp b/crypto/smc-envelope/HighloadWalletV2.cpp index e0454233e9..567bd260ea 100644 --- a/crypto/smc-envelope/HighloadWalletV2.cpp +++ b/crypto/smc-envelope/HighloadWalletV2.cpp @@ -16,15 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "HighloadWalletV2.h" -#include "GenericAccount.h" -#include "SmartContractCode.h" +#include +#include "td/utils/base64.h" #include "vm/boc.h" #include "vm/cells/CellString.h" -#include "td/utils/base64.h" -#include +#include "GenericAccount.h" +#include "HighloadWalletV2.h" +#include "SmartContractCode.h" namespace ton { diff --git a/crypto/smc-envelope/HighloadWalletV2.h b/crypto/smc-envelope/HighloadWalletV2.h index 6b5e6dba21..bc328f39bc 100644 --- a/crypto/smc-envelope/HighloadWalletV2.h +++ b/crypto/smc-envelope/HighloadWalletV2.h @@ -18,13 +18,14 @@ */ #pragma once +#include "block/block.h" #include "smc-envelope/SmartContract.h" #include "smc-envelope/WalletInterface.h" #include "vm/cells.h" -#include "Ed25519.h" -#include "block/block.h" #include "vm/cells/CellString.h" +#include "Ed25519.h" + namespace ton { struct HighloadWalletV2Traits { using InitData = WalletInterface::DefaultInitData; @@ -40,9 +41,9 @@ class HighloadWalletV2 : public WalletBase> make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 valid_until, td::Span gifts) const override; static td::Ref get_init_data(const InitData& init_data) noexcept; - td::Result> get_init_message(const td::Ed25519::PrivateKey& private_key, - td::uint32 valid_until = std::numeric_limits::max()) const - noexcept; + td::Result> get_init_message( + const td::Ed25519::PrivateKey& private_key, + td::uint32 valid_until = std::numeric_limits::max()) const noexcept; // can't use get methods for compatibility with old revisions td::Result get_wallet_id() const override; diff --git a/crypto/smc-envelope/ManualDns.cpp b/crypto/smc-envelope/ManualDns.cpp index 617ab91563..58ccd01194 100644 --- a/crypto/smc-envelope/ManualDns.cpp +++ b/crypto/smc-envelope/ManualDns.cpp @@ -16,21 +16,17 @@ Copyright 2019-2020 Telegram Systems LLP */ -#include "ManualDns.h" - +#include "block/block-auto.h" +#include "block/block-parse.h" +#include "common/util.h" #include "smc-envelope/SmartContractCode.h" - -#include "vm/dict.h" - -#include "td/utils/format.h" -#include "td/utils/overloaded.h" #include "td/utils/Parser.h" #include "td/utils/Random.h" +#include "td/utils/format.h" +#include "td/utils/overloaded.h" +#include "vm/dict.h" -#include "block/block-auto.h" -#include "block/block-parse.h" - -#include "common/util.h" +#include "ManualDns.h" namespace ton { td::StringBuilder& operator<<(td::StringBuilder& sb, const ManualDns::EntryData& data) { @@ -162,9 +158,8 @@ SmartContract::Args DnsInterface::resolve_args_raw(td::Slice encoded_name, td::B block::StdAddress address) { SmartContract::Args res; res.set_method_id("dnsresolve"); - res.set_stack( - {vm::load_cell_slice_ref(vm::CellBuilder().store_bytes(encoded_name).finalize()), - td::bits_to_refint(category.cbits(), 256, false)}); + res.set_stack({vm::load_cell_slice_ref(vm::CellBuilder().store_bytes(encoded_name).finalize()), + td::bits_to_refint(category.cbits(), 256, false)}); res.set_address(std::move(address)); return res; } diff --git a/crypto/smc-envelope/ManualDns.h b/crypto/smc-envelope/ManualDns.h index d24cd02348..16ca1b76f4 100644 --- a/crypto/smc-envelope/ManualDns.h +++ b/crypto/smc-envelope/ManualDns.h @@ -17,18 +17,17 @@ Copyright 2019-2020 Telegram Systems LLP */ #pragma once -#include "td/utils/Variant.h" +#include + +#include "common/checksum.h" +#include "smc-envelope/SmartContract.h" #include "td/utils/Status.h" +#include "td/utils/Variant.h" #include "vm/cells/Cell.h" #include "vm/cells/CellSlice.h" #include "vm/cells/CellString.h" -#include "smc-envelope/SmartContract.h" - #include "Ed25519.h" -#include "common/checksum.h" - -#include namespace ton { const td::Bits256 DNS_NEXT_RESOLVER_CATEGORY = @@ -351,6 +350,7 @@ class ManualDns : public ton::SmartContract, public DnsInterface { return res; } td::Result> create_update_query(CombinedActions& combined) const; + private: block::StdAddress address_; }; diff --git a/crypto/smc-envelope/MultisigWallet.cpp b/crypto/smc-envelope/MultisigWallet.cpp index 5a9fab6a82..2dc4e275ad 100644 --- a/crypto/smc-envelope/MultisigWallet.cpp +++ b/crypto/smc-envelope/MultisigWallet.cpp @@ -16,13 +16,11 @@ Copyright 2019-2020 Telegram Systems LLP */ -#include "MultisigWallet.h" - -#include "SmartContractCode.h" - +#include "td/utils/misc.h" #include "vm/dict.h" -#include "td/utils/misc.h" +#include "MultisigWallet.h" +#include "SmartContractCode.h" namespace ton { diff --git a/crypto/smc-envelope/MultisigWallet.h b/crypto/smc-envelope/MultisigWallet.h index 96da297a4f..c28d0d1411 100644 --- a/crypto/smc-envelope/MultisigWallet.h +++ b/crypto/smc-envelope/MultisigWallet.h @@ -17,12 +17,12 @@ Copyright 2019-2020 Telegram Systems LLP */ #pragma once +#include + #include "vm/cells.h" -#include "SmartContract.h" #include "Ed25519.h" - -#include +#include "SmartContract.h" namespace ton { class MultisigWallet : public ton::SmartContract { diff --git a/crypto/smc-envelope/PaymentChannel.cpp b/crypto/smc-envelope/PaymentChannel.cpp index ff54996c55..48758c0613 100644 --- a/crypto/smc-envelope/PaymentChannel.cpp +++ b/crypto/smc-envelope/PaymentChannel.cpp @@ -17,14 +17,14 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "PaymentChannel.h" -#include "GenericAccount.h" -#include "vm/cells.h" -#include "vm/cellslice.h" -#include "Ed25519.h" #include "block/block-auto.h" #include "block/block-parse.h" +#include "vm/cells.h" +#include "vm/cellslice.h" +#include "Ed25519.h" +#include "GenericAccount.h" +#include "PaymentChannel.h" #include "SmartContract.h" #include "SmartContractCode.h" diff --git a/crypto/smc-envelope/PaymentChannel.h b/crypto/smc-envelope/PaymentChannel.h index 0cd0220d5f..4f317ce0e6 100644 --- a/crypto/smc-envelope/PaymentChannel.h +++ b/crypto/smc-envelope/PaymentChannel.h @@ -18,14 +18,13 @@ */ #pragma once -#include "vm/cells.h" -#include "vm/cellslice.h" -#include "Ed25519.h" #include "block/block-auto.h" #include "block/block-parse.h" - #include "td/utils/Variant.h" +#include "vm/cells.h" +#include "vm/cellslice.h" +#include "Ed25519.h" #include "SmartContract.h" #include "SmartContractCode.h" diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index c067b88ff5..1b1ba7ef5c 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -16,19 +16,17 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "SmartContract.h" - -#include "GenericAccount.h" -#include "transaction.h" - -#include "block/block.h" #include "block/block-auto.h" +#include "block/block.h" +#include "td/utils/crypto.h" #include "vm/cellslice.h" #include "vm/cp0.h" #include "vm/memo.h" #include "vm/vm.h" -#include "td/utils/crypto.h" +#include "GenericAccount.h" +#include "SmartContract.h" +#include "transaction.h" namespace ton { int SmartContract::Answer::output_actions_count(td::Ref list) { @@ -46,29 +44,28 @@ td::Ref build_internal_message(td::RefInt256 amount, td::Refbit_size(false) + 7) >> 3); - b.store_long_bool(len, 4) && b.store_int256_bool(*amount, len * 8, false); // grams:Grams - b.store_zeroes(1 + 4 + 4 + 64 + 32 + 1); // extra currencies, extra_flags, fwd_fee, created_lt, created_at, init + b.store_long_bool(len, 4) && b.store_int256_bool(*amount, len * 8, false); // grams:Grams + b.store_zeroes(1 + 4 + 4 + 64 + 32 + 1); // extra currencies, extra_flags, fwd_fee, created_lt, created_at, init // body:(Either X ^X) if (b.remaining_bits() >= 1 + body->size() && b.remaining_refs() >= body->size_refs()) { - b.store_zeroes(1); - b.append_cellslice(body); + b.store_zeroes(1); + b.append_cellslice(body); } else { - b.store_ones(1); - b.store_ref(vm::CellBuilder().append_cellslice(body).finalize_novm()); + b.store_ones(1); + b.store_ref(vm::CellBuilder().append_cellslice(body).finalize_novm()); } return b.finalize_novm(); } @@ -78,41 +75,39 @@ td::Ref build_external_message(td::RefInt256 amount, td::Ref= 1 + (*body).size() && b.remaining_refs() >= (*body).size_refs()) { - b.store_zeroes(1); - b.append_cellslice(body); + b.store_zeroes(1); + b.append_cellslice(body); } else { - b.store_ones(1); - b.store_ref(vm::CellBuilder().append_cellslice(body).finalize_novm()); + b.store_ones(1); + b.store_ref(vm::CellBuilder().append_cellslice(body).finalize_novm()); } return b.finalize_novm(); } -td::Ref prepare_vm_stack(td::RefInt256 amount, td::Ref body, SmartContract::Args args, int selector) { +td::Ref prepare_vm_stack(td::RefInt256 amount, td::Ref body, SmartContract::Args args, + int selector) { td::Ref stack_ref{true}; td::RefInt256 acc_addr{true}; //CHECK(acc_addr.write().import_bits(account.addr.cbits(), 256)); vm::Stack& stack = stack_ref.write(); - if(args.balance) { + if (args.balance) { stack.push_int(td::make_refint(args.balance)); } else { stack.push_int(td::make_refint(10000000000)); } stack.push_int(amount); - if(selector == 0) { + if (selector == 0) { stack.push_cell(build_internal_message(amount, body, args)); } else { stack.push_cell(build_external_message(amount, body, args)); @@ -177,8 +172,8 @@ td::Ref prepare_vm_c7(SmartContract::Args args, td::Ref cod } if (global_version >= 6) { tuple.push_back(args.config ? args.config.value()->get_unpacked_config_tuple(now) - : vm::StackEntry{}); // unpacked_config_tuple - tuple.push_back(td::zero_refint()); // due_payment + : vm::StackEntry{}); // unpacked_config_tuple + tuple.push_back(td::zero_refint()); // due_payment // precompiled_gas_usage:(Maybe Integer) td::optional precompiled; if (code.not_null() && args.config) { @@ -220,8 +215,9 @@ std::shared_ptr try_fetch_config_from_c7(td::Ref config_addr_cs.fetch_bits_to(config_addr); } } - auto global_config = block::Config(config_cell, std::move(config_addr), - block::Config::needWorkchainInfo | block::Config::needSpecialSmc | block::Config::needCapabilities); + auto global_config = + block::Config(config_cell, std::move(config_addr), + block::Config::needWorkchainInfo | block::Config::needSpecialSmc | block::Config::needCapabilities); auto unpack_res = global_config.unpack(); if (unpack_res.is_error()) { LOG(ERROR) << "Failed to unpack config: " << unpack_res.error(); @@ -406,6 +402,7 @@ SmartContract::Answer SmartContract::send_external_message(td::Ref cel } SmartContract::Answer SmartContract::send_internal_message(td::Ref cell, Args args) { return run_method( - args.set_stack(prepare_vm_stack(td::make_refint(args.amount), vm::load_cell_slice_ref(cell), args, 0)).set_method_id(0)); + args.set_stack(prepare_vm_stack(td::make_refint(args.amount), vm::load_cell_slice_ref(cell), args, 0)) + .set_method_id(0)); } } // namespace ton diff --git a/crypto/smc-envelope/SmartContract.h b/crypto/smc-envelope/SmartContract.h index 49edb9693c..edba89477c 100644 --- a/crypto/smc-envelope/SmartContract.h +++ b/crypto/smc-envelope/SmartContract.h @@ -18,16 +18,14 @@ */ #pragma once +#include "block/block.h" +#include "block/mc-config.h" +#include "td/utils/crypto.h" +#include "td/utils/optional.h" #include "vm/cells.h" #include "vm/stack.hpp" #include "vm/vm.h" -#include "td/utils/optional.h" -#include "td/utils/crypto.h" - -#include "block/block.h" -#include "block/mc-config.h" - namespace ton { class SmartContract : public td::CntObject { static td::Ref empty_slice(); diff --git a/crypto/smc-envelope/SmartContractCode.cpp b/crypto/smc-envelope/SmartContractCode.cpp index 585450f61e..6b5594b62d 100644 --- a/crypto/smc-envelope/SmartContractCode.cpp +++ b/crypto/smc-envelope/SmartContractCode.cpp @@ -16,12 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "SmartContractCode.h" - -#include "vm/boc.h" #include #include "td/utils/base64.h" +#include "vm/boc.h" + +#include "SmartContractCode.h" namespace ton { namespace { @@ -38,13 +38,13 @@ const auto& get_map() { auto with_tvm_code = [&](auto name, td::Slice code_str) { map[name] = vm::std_boc_deserialize(td::base64_decode(code_str).move_as_ok()).move_as_ok(); }; -#include "smartcont/auto/multisig-code.cpp" -#include "smartcont/auto/wallet-code.cpp" +#include "smartcont/auto/dns-manual-code.cpp" #include "smartcont/auto/highload-wallet-code.cpp" #include "smartcont/auto/highload-wallet-v2-code.cpp" -#include "smartcont/auto/dns-manual-code.cpp" +#include "smartcont/auto/multisig-code.cpp" #include "smartcont/auto/payment-channel-code.cpp" #include "smartcont/auto/restricted-wallet3-code.cpp" +#include "smartcont/auto/wallet-code.cpp" with_tvm_code("highload-wallet-r1", "te6ccgEBBgEAhgABFP8A9KQT9KDyyAsBAgEgAgMCAUgEBQC88oMI1xgg0x/TH9Mf+CMTu/Jj7UTQ0x/TH9P/" diff --git a/crypto/smc-envelope/SmartContractCode.h b/crypto/smc-envelope/SmartContractCode.h index be50d2a158..aae06707b9 100644 --- a/crypto/smc-envelope/SmartContractCode.h +++ b/crypto/smc-envelope/SmartContractCode.h @@ -17,9 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "vm/cells.h" - #include "td/utils/Span.h" +#include "vm/cells.h" namespace ton { class SmartContractCode { diff --git a/crypto/smc-envelope/TestGiver.cpp b/crypto/smc-envelope/TestGiver.cpp index 2a1691835d..32689b4111 100644 --- a/crypto/smc-envelope/TestGiver.cpp +++ b/crypto/smc-envelope/TestGiver.cpp @@ -16,11 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "TestGiver.h" -#include "GenericAccount.h" - #include "td/utils/base64.h" +#include "GenericAccount.h" +#include "TestGiver.h" + namespace ton { const block::StdAddress& TestGiver::address() noexcept { static block::StdAddress res = diff --git a/crypto/smc-envelope/TestGiver.h b/crypto/smc-envelope/TestGiver.h index b51ac7dbe0..c3ba4f06c7 100644 --- a/crypto/smc-envelope/TestGiver.h +++ b/crypto/smc-envelope/TestGiver.h @@ -17,10 +17,11 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "SmartContract.h" -#include "smc-envelope/WalletInterface.h" #include "block/block.h" +#include "smc-envelope/WalletInterface.h" #include "vm/cells/CellString.h" + +#include "SmartContract.h" namespace ton { class TestGiver : public SmartContract, public WalletInterface { public: diff --git a/crypto/smc-envelope/TestWallet.cpp b/crypto/smc-envelope/TestWallet.cpp index 41a66b6858..69c6c6a578 100644 --- a/crypto/smc-envelope/TestWallet.cpp +++ b/crypto/smc-envelope/TestWallet.cpp @@ -15,13 +15,12 @@ along with TON Blockchain Library. If not, see . Copyright 2017-2020 Telegram Systems LLP */ -#include "TestWallet.h" -#include "GenericAccount.h" +#include "td/utils/base64.h" +#include "vm/boc.h" +#include "GenericAccount.h" #include "SmartContractCode.h" - -#include "vm/boc.h" -#include "td/utils/base64.h" +#include "TestWallet.h" namespace ton { td::Ref TestWallet::get_init_state(const td::Ed25519::PublicKey& public_key, td::int32 revision) noexcept { diff --git a/crypto/smc-envelope/TestWallet.h b/crypto/smc-envelope/TestWallet.h index 38e0f12ba1..319c0a9a26 100644 --- a/crypto/smc-envelope/TestWallet.h +++ b/crypto/smc-envelope/TestWallet.h @@ -18,13 +18,14 @@ */ #pragma once +#include "block/block.h" #include "smc-envelope/SmartContract.h" #include "smc-envelope/WalletInterface.h" #include "vm/cells.h" -#include "Ed25519.h" -#include "block/block.h" #include "vm/cells/CellString.h" +#include "Ed25519.h" + namespace ton { class TestWallet : public ton::SmartContract, public WalletInterface { public: diff --git a/crypto/smc-envelope/Wallet.cpp b/crypto/smc-envelope/Wallet.cpp index fac4f1a6ba..506713e0d5 100644 --- a/crypto/smc-envelope/Wallet.cpp +++ b/crypto/smc-envelope/Wallet.cpp @@ -16,15 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "Wallet.h" -#include "GenericAccount.h" -#include "SmartContractCode.h" +#include +#include "td/utils/base64.h" #include "vm/boc.h" #include "vm/cells/CellString.h" -#include "td/utils/base64.h" -#include +#include "GenericAccount.h" +#include "SmartContractCode.h" +#include "Wallet.h" namespace ton { td::Ref Wallet::get_init_state(const td::Ed25519::PublicKey& public_key, td::int32 revision) noexcept { diff --git a/crypto/smc-envelope/Wallet.h b/crypto/smc-envelope/Wallet.h index db074f6a11..77d873e191 100644 --- a/crypto/smc-envelope/Wallet.h +++ b/crypto/smc-envelope/Wallet.h @@ -18,13 +18,14 @@ */ #pragma once +#include "block/block.h" #include "smc-envelope/SmartContract.h" #include "smc-envelope/WalletInterface.h" #include "vm/cells.h" -#include "Ed25519.h" -#include "block/block.h" #include "vm/cells/CellString.h" +#include "Ed25519.h" + namespace ton { class Wallet : public ton::SmartContract, public WalletInterface { public: diff --git a/crypto/smc-envelope/WalletInterface.h b/crypto/smc-envelope/WalletInterface.h index e88ca0a612..8ad50c17fd 100644 --- a/crypto/smc-envelope/WalletInterface.h +++ b/crypto/smc-envelope/WalletInterface.h @@ -18,17 +18,17 @@ */ #pragma once -#include "td/utils/common.h" -#include "Ed25519.h" -#include "block/block.h" +#include + #include "block/block-parse.h" +#include "block/block.h" +#include "td/utils/common.h" #include "vm/cells/CellString.h" +#include "Ed25519.h" +#include "GenericAccount.h" #include "SmartContract.h" #include "SmartContractCode.h" -#include "GenericAccount.h" - -#include namespace ton { class WalletInterface : public SmartContract { diff --git a/crypto/smc-envelope/WalletV3.cpp b/crypto/smc-envelope/WalletV3.cpp index 35a302e64d..1bbbca3ee6 100644 --- a/crypto/smc-envelope/WalletV3.cpp +++ b/crypto/smc-envelope/WalletV3.cpp @@ -16,15 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "WalletV3.h" -#include "GenericAccount.h" -#include "SmartContractCode.h" +#include +#include "td/utils/base64.h" #include "vm/boc.h" #include "vm/cells/CellString.h" -#include "td/utils/base64.h" -#include +#include "GenericAccount.h" +#include "SmartContractCode.h" +#include "WalletV3.h" namespace ton { td::Result> WalletV3::make_a_gift_message(const td::Ed25519::PrivateKey& private_key, diff --git a/crypto/smc-envelope/WalletV3.h b/crypto/smc-envelope/WalletV3.h index 926e7ffbdb..5b47ad1e4c 100644 --- a/crypto/smc-envelope/WalletV3.h +++ b/crypto/smc-envelope/WalletV3.h @@ -18,13 +18,14 @@ */ #pragma once +#include "block/block.h" #include "smc-envelope/SmartContract.h" #include "smc-envelope/WalletInterface.h" #include "vm/cells.h" -#include "Ed25519.h" -#include "block/block.h" #include "vm/cells/CellString.h" +#include "Ed25519.h" + namespace ton { struct WalletV3Traits { diff --git a/crypto/smc-envelope/WalletV4.cpp b/crypto/smc-envelope/WalletV4.cpp index 738fa9c75a..4106039ee9 100644 --- a/crypto/smc-envelope/WalletV4.cpp +++ b/crypto/smc-envelope/WalletV4.cpp @@ -14,15 +14,15 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "WalletV4.h" -#include "GenericAccount.h" -#include "SmartContractCode.h" +#include +#include "td/utils/base64.h" #include "vm/boc.h" #include "vm/cells/CellString.h" -#include "td/utils/base64.h" -#include +#include "GenericAccount.h" +#include "SmartContractCode.h" +#include "WalletV4.h" namespace ton { td::Result> WalletV4::make_a_gift_message(const td::Ed25519::PrivateKey& private_key, diff --git a/crypto/smc-envelope/WalletV4.h b/crypto/smc-envelope/WalletV4.h index 721e810394..313647ac29 100644 --- a/crypto/smc-envelope/WalletV4.h +++ b/crypto/smc-envelope/WalletV4.h @@ -16,13 +16,14 @@ */ #pragma once +#include "block/block.h" #include "smc-envelope/SmartContract.h" #include "smc-envelope/WalletInterface.h" #include "vm/cells.h" -#include "Ed25519.h" -#include "block/block.h" #include "vm/cells/CellString.h" +#include "Ed25519.h" + namespace ton { struct WalletV4Traits { diff --git a/crypto/test/Ed25519.cpp b/crypto/test/Ed25519.cpp index 940b50796b..6ca2971f1c 100644 --- a/crypto/test/Ed25519.cpp +++ b/crypto/test/Ed25519.cpp @@ -16,20 +16,19 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "crypto/Ed25519.h" +#include +#include +#include "crypto/Ed25519.h" +#include "keys/keys.hpp" +#include "td/utils/JsonBuilder.h" +#include "td/utils/Slice.h" +#include "td/utils/benchmark.h" #include "td/utils/logging.h" #include "td/utils/misc.h" -#include "td/utils/Slice.h" #include "td/utils/tests.h" -#include "td/utils/JsonBuilder.h" #include "wycheproof.h" -#include "keys/keys.hpp" -#include "td/utils/benchmark.h" - -#include -#include unsigned char fixed_privkey[32] = "abacabadabacabaeabacabadabacaba"; unsigned char fixed_pubkey[32] = {0x6f, 0x9e, 0x5b, 0xde, 0xce, 0x87, 0x21, 0xeb, 0x57, 0x37, 0xfb, diff --git a/crypto/test/fift.cpp b/crypto/test/fift.cpp index 4d522ce738..331880f10a 100644 --- a/crypto/test/fift.cpp +++ b/crypto/test/fift.cpp @@ -16,14 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "fift/words.h" #include "fift/Fift.h" #include "fift/utils.h" - -#include "td/utils/tests.h" +#include "fift/words.h" #include "td/utils/PathView.h" -#include "td/utils/port/path.h" #include "td/utils/filesystem.h" +#include "td/utils/port/path.h" +#include "td/utils/tests.h" std::string current_dir() { return td::PathView(td::realpath(__FILE__).move_as_ok()).parent_dir().str(); diff --git a/crypto/test/modbigint.cpp b/crypto/test/modbigint.cpp index 75051fa625..0f66c53b43 100644 --- a/crypto/test/modbigint.cpp +++ b/crypto/test/modbigint.cpp @@ -14,12 +14,12 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include -#include #include -#include +#include #include #include +#include +#include namespace modint { @@ -180,7 +180,7 @@ struct MixedRadix { template const MixedRadix& as_shorter() const { - static_assert(M <= N,"error"); + static_assert(M <= N, "error"); return *reinterpret_cast*>(this); } @@ -458,7 +458,7 @@ struct ModArray { } template ModArray(const ModArray& other) { - static_assert(M >= N,"error"); + static_assert(M >= N, "error"); std::copy(other.a, other.a + N, a); } ModArray(const int* p) : a(p) { @@ -787,7 +787,7 @@ struct ModArray { ModArray& operator/=(const ModArray& other) { if (!try_divide(other)) { - assert(false); // division by zero? + assert(false); // division by zero? } return *this; } @@ -819,7 +819,7 @@ struct ModArray { template const ModArray& as_shorter() const { - static_assert(M <= N,"error"); + static_assert(M <= N, "error"); return *reinterpret_cast*>(this); } @@ -962,7 +962,7 @@ std::ostream& operator<<(std::ostream& os, MixedRadix&& x) { template struct ArrayRawDumpRef { const std::array& ref; - ArrayRawDumpRef(const std::array& _ref) : ref(_ref){}; + ArrayRawDumpRef(const std::array& _ref) : ref(_ref) {}; }; template diff --git a/crypto/test/test-bigint.cpp b/crypto/test/test-bigint.cpp index a6f6e8d671..5155bd201b 100644 --- a/crypto/test/test-bigint.cpp +++ b/crypto/test/test-bigint.cpp @@ -14,21 +14,22 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include -#include #include -#include +#include +#include #include -#include #include -#include -#include "common/refcnt.hpp" +#include +#include +#include + #include "common/bigint.hpp" +#include "common/refcnt.hpp" #include "common/refint.h" -#include "modbigint.cpp" - #include "td/utils/tests.h" +#include "modbigint.cpp" + int mkint_chk_mode = -1, res_chk_mode = 0; long long iterations = 100000, cur_iteration = -1, debug_iteration = -2; #define IFDEBUG if (cur_iteration == debug_iteration || debug_iteration == -3) diff --git a/crypto/test/test-cells.cpp b/crypto/test/test-cells.cpp index 327f73c62c..dc909924d7 100644 --- a/crypto/test/test-cells.cpp +++ b/crypto/test/test-cells.cpp @@ -17,26 +17,26 @@ Copyright 2017-2020 Telegram Systems LLP */ #include -#include -#include +#include +#include +#include #include -#include #include -#include -#include -#include -#include "common/refcnt.hpp" -#include "common/bigint.hpp" -#include "common/refint.h" +#include +#include +#include + #include "common/bigexp.h" +#include "common/bigint.hpp" #include "common/bitstring.h" +#include "common/refcnt.hpp" +#include "common/refint.h" #include "common/util.h" -#include "vm/cells.h" -#include "vm/cellslice.h" - -#include "td/utils/tests.h" #include "td/utils/crypto.h" #include "td/utils/misc.h" +#include "td/utils/tests.h" +#include "vm/cells.h" +#include "vm/cellslice.h" static std::stringstream create_ss() { std::stringstream ss; diff --git a/crypto/test/test-db.cpp b/crypto/test/test-db.cpp index 179767dfea..443287a9a6 100644 --- a/crypto/test/test-db.cpp +++ b/crypto/test/test-db.cpp @@ -16,61 +16,54 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/boc.h" -#include "vm/cells.h" -#include "common/AtomicRef.h" -#include "vm/cells/CellString.h" -#include "vm/cells/MerkleProof.h" -#include "vm/cells/MerkleUpdate.h" -#include "vm/db/CellStorage.h" -#include "vm/db/TonDb.h" -#include "vm/db/StaticBagOfCellsDb.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include -#include "td/utils/base64.h" -#include "td/utils/benchmark.h" -#include "td/utils/crypto.h" +#include "common/AtomicRef.h" +#include "openssl/digest.hpp" +#include "storage/db.h" +#include "td/actor/actor.h" +#include "td/db/MemoryKeyValue.h" +#include "td/db/RocksDb.h" +#include "td/db/utils/BlobView.h" +#include "td/db/utils/CyclicBuffer.h" #include "td/utils/Random.h" #include "td/utils/Slice.h" #include "td/utils/Span.h" #include "td/utils/Status.h" #include "td/utils/Timer.h" +#include "td/utils/VectorQueue.h" +#include "td/utils/base64.h" +#include "td/utils/benchmark.h" +#include "td/utils/crypto.h" #include "td/utils/filesystem.h" -#include "td/utils/port/path.h" #include "td/utils/format.h" #include "td/utils/misc.h" +#include "td/utils/overloaded.h" +#include "td/utils/port/path.h" #include "td/utils/tests.h" -#include "td/utils/tl_parsers.h" #include "td/utils/tl_helpers.h" - -#include "td/db/utils/BlobView.h" -#include "td/db/RocksDb.h" -#include "td/db/MemoryKeyValue.h" -#include "td/db/utils/CyclicBuffer.h" - -#include -#include -#include -#include - -#include - -#include "openssl/digest.hpp" -#include "storage/db.h" -#include "td/utils/VectorQueue.h" +#include "td/utils/tl_parsers.h" +#include "vm/boc.h" +#include "vm/cells.h" +#include "vm/cells/CellString.h" +#include "vm/cells/MerkleProof.h" +#include "vm/cells/MerkleUpdate.h" +#include "vm/db/CellStorage.h" +#include "vm/db/StaticBagOfCellsDb.h" +#include "vm/db/TonDb.h" #include "vm/dict.h" -#include -#include -#include -#include - -#include -#include -#include - -#include "td/actor/actor.h" -#include "td/utils/overloaded.h" - class ActorExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { public: ActorExecutor(size_t tn) : tn_(tn) { @@ -1692,28 +1685,29 @@ TEST(TonDb, BocDeserializerSimpleThreads) { } class RandomTree { - public: - RandomTree(size_t size, td::Random::Xorshift128plus rnd) : rnd_(rnd) { - root_ = create(size); - } - Ref root() const { - return root_; - } - private: - Ref root_; - td::Random::Xorshift128plus rnd_; - Ref create(size_t size) { - CellBuilder cb; - cb.store_long(rnd_(), rnd_() % 63 + 1); - if (size > 0) { - td::uint64 rc = (rnd_() % 4) + 1; - for (td::uint64 i = 0; i < rc; i++) { - auto ref = create(size / rc); - cb.store_ref(std::move(ref)); - } + public: + RandomTree(size_t size, td::Random::Xorshift128plus rnd) : rnd_(rnd) { + root_ = create(size); + } + Ref root() const { + return root_; + } + + private: + Ref root_; + td::Random::Xorshift128plus rnd_; + Ref create(size_t size) { + CellBuilder cb; + cb.store_long(rnd_(), rnd_() % 63 + 1); + if (size > 0) { + td::uint64 rc = (rnd_() % 4) + 1; + for (td::uint64 i = 0; i < rc; i++) { + auto ref = create(size / rc); + cb.store_ref(std::move(ref)); } - return cb.finalize(); } + return cb.finalize(); + } }; class CompactArray { @@ -3035,14 +3029,15 @@ TEST(TonDb, LargeBocSerializer) { auto a_cell = vm::deserialize_boc(td::BufferSlice(a)); for (int i = 0; i < 4; i++) { fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write) - .move_as_ok(); + .move_as_ok(); boc_serialize_to_file_large(dboc->get_cell_db_reader(), root->get_hash(), fd, 31); fd.close(); auto b = td::read_file_str(path).move_as_ok(); auto b_cell = vm::deserialize_boc(td::BufferSlice(b)); ASSERT_EQ(a_cell->get_hash(), b_cell->get_hash()); - if (i > 0) ASSERT_EQ(prev_b, b); + if (i > 0) + ASSERT_EQ(prev_b, b); prev_b = b; } } diff --git a/crypto/test/test-smartcont.cpp b/crypto/test/test-smartcont.cpp index a278407218..a63ac18fb7 100644 --- a/crypto/test/test-smartcont.cpp +++ b/crypto/test/test-smartcont.cpp @@ -16,45 +16,41 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/dict.h" -#include "common/bigint.hpp" - -#include "Ed25519.h" +#include +#include +#include #include "block/block-auto.h" -#include "block/block.h" #include "block/block-parse.h" - +#include "block/block.h" +#include "common/bigint.hpp" #include "fift/Fift.h" -#include "fift/words.h" #include "fift/utils.h" - +#include "fift/words.h" #include "smc-envelope/GenericAccount.h" +#include "smc-envelope/HighloadWallet.h" +#include "smc-envelope/HighloadWalletV2.h" #include "smc-envelope/ManualDns.h" #include "smc-envelope/MultisigWallet.h" +#include "smc-envelope/PaymentChannel.h" #include "smc-envelope/SmartContract.h" #include "smc-envelope/SmartContractCode.h" #include "smc-envelope/WalletV3.h" #include "smc-envelope/WalletV4.h" -#include "smc-envelope/HighloadWallet.h" -#include "smc-envelope/HighloadWalletV2.h" -#include "smc-envelope/PaymentChannel.h" - -#include "td/utils/base64.h" -#include "td/utils/crypto.h" +#include "td/utils/PathView.h" #include "td/utils/Random.h" -#include "td/utils/tests.h" #include "td/utils/ScopeGuard.h" #include "td/utils/StringBuilder.h" #include "td/utils/Timer.h" -#include "td/utils/PathView.h" +#include "td/utils/Variant.h" +#include "td/utils/base64.h" +#include "td/utils/crypto.h" #include "td/utils/filesystem.h" #include "td/utils/port/path.h" -#include "td/utils/Variant.h" +#include "td/utils/tests.h" +#include "vm/dict.h" -#include -#include -#include +#include "Ed25519.h" std::string current_dir() { return td::PathView(td::realpath(__FILE__).move_as_ok()).parent_dir().str(); diff --git a/crypto/test/vm.cpp b/crypto/test/vm.cpp index 0f1b044298..f6869046b3 100644 --- a/crypto/test/vm.cpp +++ b/crypto/test/vm.cpp @@ -16,16 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/vm.h" -#include "vm/cp0.h" -#include "vm/dict.h" -#include "fift/utils.h" #include "common/bigint.hpp" - -#include "td/utils/base64.h" -#include "td/utils/tests.h" +#include "fift/utils.h" #include "td/utils/ScopeGuard.h" #include "td/utils/StringBuilder.h" +#include "td/utils/base64.h" +#include "td/utils/tests.h" +#include "vm/cp0.h" +#include "vm/dict.h" +#include "vm/vm.h" std::string run_vm(td::Ref cell) { vm::init_vm().ensure(); diff --git a/crypto/tl/tlbc-aux.h b/crypto/tl/tlbc-aux.h index 97e5082e80..3b261418e5 100644 --- a/crypto/tl/tlbc-aux.h +++ b/crypto/tl/tlbc-aux.h @@ -17,10 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include -#include #include #include +#include +#include namespace tlbc { diff --git a/crypto/tl/tlbc-data.h b/crypto/tl/tlbc-data.h index 8b6f2392f4..38f24c0297 100644 --- a/crypto/tl/tlbc-data.h +++ b/crypto/tl/tlbc-data.h @@ -19,6 +19,7 @@ #pragma once #include #include + #include "common/linalloc.hpp" namespace tlbc { diff --git a/crypto/tl/tlbc-gen-cpp.cpp b/crypto/tl/tlbc-gen-cpp.cpp index 5730f1691a..f7b5067365 100644 --- a/crypto/tl/tlbc-gen-cpp.cpp +++ b/crypto/tl/tlbc-gen-cpp.cpp @@ -16,10 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tlbc-gen-cpp.h" #include "td/utils/bits.h" #include "td/utils/filesystem.h" +#include "tlbc-gen-cpp.h" + namespace tlbc { /* @@ -217,7 +218,7 @@ unsigned long long CppTypeCode::compute_selector_mask() const { struct HexConstWriter { unsigned long long mask; - explicit HexConstWriter(unsigned long long _mask) : mask(_mask){}; + explicit HexConstWriter(unsigned long long _mask) : mask(_mask) {}; void write(std::ostream& os) const; }; diff --git a/crypto/tl/tlbc.cpp b/crypto/tl/tlbc.cpp index d3a6edb5ce..294f7deaf5 100644 --- a/crypto/tl/tlbc.cpp +++ b/crypto/tl/tlbc.cpp @@ -25,31 +25,32 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include -#include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include #include +#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include -#include "common/refcnt.hpp" #include "common/bigint.hpp" +#include "common/refcnt.hpp" #include "common/refint.h" -#include "parser/srcread.h" #include "parser/lexer.h" +#include "parser/srcread.h" #include "parser/symtable.h" #include "td/utils/Slice-decl.h" -#include "td/utils/format.h" #include "td/utils/crypto.h" +#include "td/utils/format.h" + #include "tlbc-aux.h" #include "tlbc-data.h" #include "tlbc-gen-cpp.h" diff --git a/crypto/tl/tlblib.hpp b/crypto/tl/tlblib.hpp index c10049a93a..bd74a93d58 100644 --- a/crypto/tl/tlblib.hpp +++ b/crypto/tl/tlblib.hpp @@ -17,10 +17,11 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include #include #include + #include "vm/cellslice.h" -#include namespace tlb { diff --git a/crypto/util/mintless-proof-generator.cpp b/crypto/util/mintless-proof-generator.cpp index 62b5f0d660..e4dbe037df 100644 --- a/crypto/util/mintless-proof-generator.cpp +++ b/crypto/util/mintless-proof-generator.cpp @@ -15,12 +15,12 @@ along with TON Blockchain Library. If not, see . */ -#include "block-parse.h" -#include "block.h" +#include +#include +#include + #include "td/actor/core/Actor.h" #include "td/db/utils/BlobView.h" - -#include #include "td/utils/OptionParser.h" #include "td/utils/Time.h" #include "td/utils/base64.h" @@ -29,8 +29,8 @@ #include "vm/cells/MerkleProof.h" #include "vm/db/StaticBagOfCellsDb.h" -#include -#include +#include "block-parse.h" +#include "block.h" const size_t KEY_LEN = 3 + 8 + 256; diff --git a/crypto/vm/Hasher.cpp b/crypto/vm/Hasher.cpp index f70988d33d..42e0d5c12e 100644 --- a/crypto/vm/Hasher.cpp +++ b/crypto/vm/Hasher.cpp @@ -14,12 +14,13 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "vm/Hasher.h" -#include "vm/excno.hpp" -#include "vm/vm.h" #include #include + #include "keccak/keccak.h" +#include "vm/Hasher.h" +#include "vm/excno.hpp" +#include "vm/vm.h" namespace vm { @@ -27,7 +28,7 @@ using td::Ref; class HasherImplEVP : public Hasher::HasherImpl { public: - explicit HasherImplEVP(EVP_MD_CTX* ctx) : ctx_(ctx) { + explicit HasherImplEVP(EVP_MD_CTX *ctx) : ctx_(ctx) { } ~HasherImplEVP() override { @@ -74,7 +75,7 @@ class HasherImplKeccak : public Hasher::HasherImpl { td::BufferSlice finish() override { td::BufferSlice hash(hash_size_); - CHECK(keccak_digest(state_, (unsigned char*)hash.data(), hash_size_, 1) == 0); + CHECK(keccak_digest(state_, (unsigned char *)hash.data(), hash_size_, 1) == 0); return hash; } @@ -99,9 +100,15 @@ Hasher::Hasher(int hash_id) : id_(hash_id) { CHECK(ctx != nullptr); const EVP_MD *evp; switch (hash_id) { - case SHA256: evp = EVP_sha256(); break; - case SHA512: evp = EVP_sha512(); break; - case BLAKE2B: evp = EVP_blake2b512(); break; + case SHA256: + evp = EVP_sha256(); + break; + case SHA512: + evp = EVP_sha512(); + break; + case BLAKE2B: + evp = EVP_blake2b512(); + break; default: throw VmError{Excno::range_chk, "invalid hash id"}; } @@ -145,4 +152,4 @@ size_t Hasher::bytes_per_gas_unit() const { return BYTES_PER_GAS_UNIT[id_]; } -} +} // namespace vm diff --git a/crypto/vm/Hasher.h b/crypto/vm/Hasher.h index 7e4416908d..9d1f4fdb77 100644 --- a/crypto/vm/Hasher.h +++ b/crypto/vm/Hasher.h @@ -15,11 +15,12 @@ along with TON Blockchain Library. If not, see . */ #pragma once +#include + +#include "common/bitstring.h" #include "common/refcnt.hpp" #include "td/utils/buffer.h" -#include "common/bitstring.h" #include "vm/cells/Cell.h" -#include namespace vm { @@ -55,4 +56,4 @@ class Hasher { std::unique_ptr impl_; }; -} \ No newline at end of file +} // namespace vm \ No newline at end of file diff --git a/crypto/vm/arithops.cpp b/crypto/vm/arithops.cpp index fc482fc4d6..fee9353740 100644 --- a/crypto/vm/arithops.cpp +++ b/crypto/vm/arithops.cpp @@ -17,14 +17,15 @@ Copyright 2017-2020 Telegram Systems LLP */ #include + +#include "common/bigint.hpp" +#include "common/refint.h" #include "vm/arithops.h" +#include "vm/excno.hpp" #include "vm/log.h" #include "vm/opctable.h" #include "vm/stack.hpp" -#include "vm/excno.hpp" #include "vm/vm.h" -#include "common/bigint.hpp" -#include "common/refint.h" namespace vm { diff --git a/crypto/vm/atom.h b/crypto/vm/atom.h index 9946a9ff34..a3708fc252 100644 --- a/crypto/vm/atom.h +++ b/crypto/vm/atom.h @@ -18,8 +18,9 @@ */ #pragma once #include -#include "vm/stack.hpp" + #include "td/utils/Slice.h" +#include "vm/stack.hpp" namespace vm { using td::Ref; @@ -35,9 +36,9 @@ class Atom : public td::CntObject { public: Atom(const Atom&) = delete; - Atom &operator=(const Atom&) = delete; + Atom& operator=(const Atom&) = delete; Atom(Atom&&) = delete; - Atom &operator=(Atom&&) = delete; + Atom& operator=(Atom&&) = delete; ~Atom() override = default; Atom(std::string name, int index) : name_(name), index_(index) { } diff --git a/crypto/vm/bls.h b/crypto/vm/bls.h index b7ffc1368d..65c34b5ef6 100644 --- a/crypto/vm/bls.h +++ b/crypto/vm/bls.h @@ -16,9 +16,10 @@ */ #include -#include "td/utils/buffer.h" + #include "common/bitstring.h" #include "common/refint.h" +#include "td/utils/buffer.h" namespace vm { namespace bls { diff --git a/crypto/vm/boc-compression.cpp b/crypto/vm/boc-compression.cpp index d37e0380ab..41095efded 100644 --- a/crypto/vm/boc-compression.cpp +++ b/crypto/vm/boc-compression.cpp @@ -16,16 +16,17 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "boc-compression.h" - #include #include -#include "vm/boc.h" + +#include "td/utils/Slice-decl.h" +#include "td/utils/lz4.h" #include "vm/boc-writers.h" +#include "vm/boc.h" #include "vm/cells.h" #include "vm/cellslice.h" -#include "td/utils/Slice-decl.h" -#include "td/utils/lz4.h" + +#include "boc-compression.h" namespace vm { @@ -42,7 +43,8 @@ td::Result boc_compress_baseline_lz4(const std::vector>> boc_decompress_baseline_lz4(td::Slice compressed, int max_decompressed_size) { +td::Result>> boc_decompress_baseline_lz4(td::Slice compressed, + int max_decompressed_size) { // Check minimum input size for decompressed size header if (compressed.size() < kDecompressedSizeBytes) { return td::Status::Error("BOC decompression failed: input too small for header"); @@ -283,7 +285,7 @@ td::Result boc_compress_improved_structure_lz4(const std::vecto if (rank[boc_graph[node][j]] <= i + 1) continue; - int delta = rank[boc_graph[node][j]] - i - 2; // Always >= 0 because of above check + int delta = rank[boc_graph[node][j]] - i - 2; // Always >= 0 because of above check size_t required_bits = 1 + (31 ^ td::count_leading_zeroes32(node_count - i - 3)); if (required_bits < 8 - (result.size() + 1) % 8 + 1) { @@ -340,7 +342,8 @@ td::Result boc_compress_improved_structure_lz4(const std::vecto return compressed_with_size; } -td::Result>> boc_decompress_improved_structure_lz4(td::Slice compressed, int max_decompressed_size) { +td::Result>> boc_decompress_improved_structure_lz4(td::Slice compressed, + int max_decompressed_size) { constexpr size_t kMaxCellDataLengthBits = 1024; // Check minimum input size for decompressed size header @@ -392,7 +395,6 @@ td::Result>> boc_decompress_improved_structure_lz4 return td::Status::Error("BOC decompression failed: incorrect node count provided"); } - // Validate root indexes for (int i = 0; i < root_count; ++i) { if (root_indexes[i] >= node_count) { @@ -596,7 +598,7 @@ td::Result boc_compress(const std::vector>& b } else if (algo == CompressionAlgorithm::ImprovedStructureLZ4) { TRY_RESULT_ASSIGN(compressed, boc_compress_improved_structure_lz4(boc_roots)); } else { - return td::Status::Error("Unknown compression algorithm"); + return td::Status::Error("Unknown compression algorithm"); } td::BufferSlice compressed_with_algo(compressed.size() + 1); diff --git a/crypto/vm/boc-compression.h b/crypto/vm/boc-compression.h index f143edd1fb..9396a96f4a 100644 --- a/crypto/vm/boc-compression.h +++ b/crypto/vm/boc-compression.h @@ -20,8 +20,8 @@ This file is part of TON Blockchain Library. #include "td/utils/Status.h" #include "td/utils/buffer.h" -#include "vm/cells/CellSlice.h" #include "vm/cells/CellBuilder.h" +#include "vm/cells/CellSlice.h" #include "vm/excno.hpp" namespace vm { @@ -34,9 +34,11 @@ td::Result boc_compress_baseline_lz4(const std::vector>> boc_decompress_baseline_lz4(td::Slice compressed, int max_decompressed_size); td::Result boc_compress_improved_structure_lz4(const std::vector>& boc_roots); -td::Result>> boc_decompress_improved_structure_lz4(td::Slice compressed, int max_decompressed_size); +td::Result>> boc_decompress_improved_structure_lz4(td::Slice compressed, + int max_decompressed_size); -td::Result boc_compress(const std::vector>& boc_roots, CompressionAlgorithm algo = CompressionAlgorithm::BaselineLZ4); +td::Result boc_compress(const std::vector>& boc_roots, + CompressionAlgorithm algo = CompressionAlgorithm::BaselineLZ4); td::Result>> boc_decompress(td::Slice compressed, int max_decompressed_size); } // namespace vm diff --git a/crypto/vm/boc-writers.h b/crypto/vm/boc-writers.h index e33886df1f..ffd09ebcab 100644 --- a/crypto/vm/boc-writers.h +++ b/crypto/vm/boc-writers.h @@ -15,15 +15,17 @@ along with TON Blockchain Library. If not, see . */ #pragma once -#include "td/utils/port/FileFd.h" -#include "td/utils/crypto.h" #include +#include "td/utils/crypto.h" +#include "td/utils/port/FileFd.h" + namespace vm { namespace boc_writers { struct BufferWriter { BufferWriter(unsigned char* store_start, unsigned char* store_end) - : store_start(store_start), store_ptr(store_start), store_end(store_end) {} + : store_start(store_start), store_ptr(store_start), store_end(store_end) { + } size_t position() const { return store_ptr - store_start; @@ -63,8 +65,8 @@ struct BufferWriter { }; struct FileWriter { - FileWriter(td::FileFd& fd, size_t expected_size) - : fd(fd), expected_size(expected_size) {} + FileWriter(td::FileFd& fd, size_t expected_size) : fd(fd), expected_size(expected_size) { + } ~FileWriter() { flush(); @@ -142,5 +144,5 @@ struct FileWriter { BufferWriter writer = BufferWriter(buf.data(), buf.data() + buf.size()); td::Status res = td::Status::OK(); }; -} -} \ No newline at end of file +} // namespace boc_writers +} // namespace vm \ No newline at end of file diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 974411b6cf..cc59b4864c 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -16,18 +16,19 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include -#include #include -#include "vm/boc.h" -#include "vm/boc-writers.h" -#include "vm/cells.h" -#include "vm/cellslice.h" +#include +#include + +#include "td/utils/Slice-decl.h" #include "td/utils/bits.h" #include "td/utils/crypto.h" #include "td/utils/format.h" #include "td/utils/misc.h" -#include "td/utils/Slice-decl.h" +#include "vm/boc-writers.h" +#include "vm/boc.h" +#include "vm/cells.h" +#include "vm/cellslice.h" namespace vm { using td::Ref; @@ -1157,7 +1158,7 @@ td::Result CellStorageStat::add_used_storage(Ref CellStorageStat::add_used_storage(td::Span> cells, bool kill_dup, - unsigned skip_count_root) { + unsigned skip_count_root) { CellInfo result; for (const auto& cell : cells) { TRY_RESULT(info, add_used_storage(cell, kill_dup, skip_count_root)); @@ -1297,7 +1298,6 @@ void ProofStorageStat::add_loaded_cells(const ProofStorageStat& other) { } } - td::uint64 ProofStorageStat::estimate_proof_size() const { return proof_size_; } diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 5bf45c9224..28f908b3df 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -17,19 +17,19 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/utils/CancellationToken.h" - -#include #include -#include "vm/db/DynamicBagOfCellsDb.h" -#include "vm/cells.h" -#include "td/utils/Status.h" -#include "td/utils/buffer.h" +#include + +#include "td/utils/CancellationToken.h" #include "td/utils/HashMap.h" #include "td/utils/HashSet.h" +#include "td/utils/Status.h" #include "td/utils/Time.h" #include "td/utils/Timer.h" +#include "td/utils/buffer.h" #include "td/utils/port/FileFd.h" +#include "vm/cells.h" +#include "vm/db/DynamicBagOfCellsDb.h" namespace vm { using td::Ref; @@ -143,8 +143,7 @@ struct CellStorageStat { td::Result add_used_storage(const CellSlice& cs, bool kill_dup = true, unsigned skip_count_root = 0); td::Result add_used_storage(CellSlice&& cs, bool kill_dup = true, unsigned skip_count_root = 0); td::Result add_used_storage(Ref cell, bool kill_dup = true, unsigned skip_count_root = 0); - td::Result add_used_storage(td::Span> cells, bool kill_dup = true, - unsigned skip_count_root = 0); + td::Result add_used_storage(td::Span> cells, bool kill_dup = true, unsigned skip_count_root = 0); unsigned long long limit_cells = std::numeric_limits::max(); unsigned long long limit_bits = std::numeric_limits::max(); @@ -406,6 +405,6 @@ td::Result std_boc_serialize_multi(std::vector> root, td::Status std_boc_serialize_to_file(Ref root, td::FileFd& fd, int mode = 0, td::CancellationToken cancellation_token = {}); td::Status boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, - int mode = 0, td::CancellationToken cancellation_token = {}); + int mode = 0, td::CancellationToken cancellation_token = {}); } // namespace vm diff --git a/crypto/vm/cellops.cpp b/crypto/vm/cellops.cpp index b76439fac6..d7c1fc7717 100644 --- a/crypto/vm/cellops.cpp +++ b/crypto/vm/cellops.cpp @@ -17,15 +17,16 @@ Copyright 2017-2020 Telegram Systems LLP */ #include + +#include "common/bigint.hpp" +#include "common/refint.h" #include "vm/cellops.h" +#include "vm/excno.hpp" #include "vm/log.h" #include "vm/opctable.h" #include "vm/stack.hpp" -#include "vm/excno.hpp" -#include "vm/vmstate.h" #include "vm/vm.h" -#include "common/bigint.hpp" -#include "common/refint.h" +#include "vm/vmstate.h" namespace vm { @@ -1541,10 +1542,16 @@ void register_cell_deserialize_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xd765, 16, "CDEPTH", exec_cell_depth)) .insert(OpcodeInstr::mksimple(0xd766, 16, "CLEVEL", exec_cell_level)->require_version(6)) .insert(OpcodeInstr::mksimple(0xd767, 16, "CLEVELMASK", exec_cell_level_mask)->require_version(6)) - .insert(OpcodeInstr::mkfixed(0xd768 >> 2, 14, 2, instr::dump_1c_and(3, "CHASHI "), std::bind(exec_cell_hash_i, _1, _2, false))->require_version(6)) - .insert(OpcodeInstr::mkfixed(0xd76c >> 2, 14, 2, instr::dump_1c_and(3, "CDEPTHI "), std::bind(exec_cell_depth_i, _1, _2, false))->require_version(6)) - .insert(OpcodeInstr::mksimple(0xd770, 16, "CHASHIX ", std::bind(exec_cell_hash_i, _1, 0, true))->require_version(6)) - .insert(OpcodeInstr::mksimple(0xd771, 16, "CDEPTHIX ", std::bind(exec_cell_depth_i, _1, 0, true))->require_version(6)); + .insert(OpcodeInstr::mkfixed(0xd768 >> 2, 14, 2, instr::dump_1c_and(3, "CHASHI "), + std::bind(exec_cell_hash_i, _1, _2, false)) + ->require_version(6)) + .insert(OpcodeInstr::mkfixed(0xd76c >> 2, 14, 2, instr::dump_1c_and(3, "CDEPTHI "), + std::bind(exec_cell_depth_i, _1, _2, false)) + ->require_version(6)) + .insert( + OpcodeInstr::mksimple(0xd770, 16, "CHASHIX ", std::bind(exec_cell_hash_i, _1, 0, true))->require_version(6)) + .insert(OpcodeInstr::mksimple(0xd771, 16, "CDEPTHIX ", std::bind(exec_cell_depth_i, _1, 0, true)) + ->require_version(6)); } void register_cell_ops(OpcodeTable& cp0) { diff --git a/crypto/vm/cells.h b/crypto/vm/cells.h index cf917eff4a..2911e4d87e 100644 --- a/crypto/vm/cells.h +++ b/crypto/vm/cells.h @@ -60,4 +60,3 @@ // cell_i.level_mask = // cell_i.ref[j] = cell.ref[j]_(i+1) // cell_i.depth = max_j(1 + cell_i.ref[j].depth) - diff --git a/crypto/vm/cells/Cell.cpp b/crypto/vm/cells/Cell.cpp index 9bed3481d0..0288953540 100644 --- a/crypto/vm/cells/Cell.cpp +++ b/crypto/vm/cells/Cell.cpp @@ -16,11 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include + #include "vm/cells/Cell.h" -#include "vm/cells/VirtualCell.h" #include "vm/cells/DataCell.h" - -#include +#include "vm/cells/VirtualCell.h" namespace vm { td::Status Cell::check_equals_unloaded(const Ref& other) const { diff --git a/crypto/vm/cells/Cell.h b/crypto/vm/cells/Cell.h index 5f0d4d14d3..f1d5cc0150 100644 --- a/crypto/vm/cells/Cell.h +++ b/crypto/vm/cells/Cell.h @@ -17,19 +17,17 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "common/refcnt.hpp" +#include + #include "common/bitstring.h" +#include "common/refcnt.hpp" #include "td/utils/HashSet.h" - +#include "td/utils/Status.h" #include "vm/cells/CellHash.h" #include "vm/cells/CellTraits.h" #include "vm/cells/CellUsageTree.h" #include "vm/cells/LevelMask.h" -#include "td/utils/Status.h" - -#include - namespace vm { using td::Ref; class DataCell; @@ -55,7 +53,7 @@ class Cell : public CellTraits { } // load interface - virtual td::Status set_data_cell(Ref &&data_cell) const = 0; + virtual td::Status set_data_cell(Ref&& data_cell) const = 0; virtual td::Result load_cell() const = 0; virtual Ref virtualize(td::uint32 effective_level) const; // Cell is virtualized if its effective level is less than its actual level. diff --git a/crypto/vm/cells/CellBuilder.cpp b/crypto/vm/cells/CellBuilder.cpp index f99cd86d99..21d0b1dcbb 100644 --- a/crypto/vm/cells/CellBuilder.cpp +++ b/crypto/vm/cells/CellBuilder.cpp @@ -16,16 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "openssl/digest.hpp" +#include "td/utils/format.h" +#include "td/utils/misc.h" #include "vm/cells/CellBuilder.h" - #include "vm/cells/CellSlice.h" #include "vm/cells/DataCell.h" -#include "td/utils/misc.h" -#include "td/utils/format.h" - -#include "openssl/digest.hpp" - namespace vm { using td::Ref; diff --git a/crypto/vm/cells/CellBuilder.h b/crypto/vm/cells/CellBuilder.h index 4c701c7d2d..03fc2e5b88 100644 --- a/crypto/vm/cells/CellBuilder.h +++ b/crypto/vm/cells/CellBuilder.h @@ -17,12 +17,11 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "common/refint.h" +#include "td/utils/ThreadSafeCounter.h" #include "vm/cells/DataCell.h" #include "vm/cells/VirtualCell.h" #include "vm/vmstate.h" -#include "common/refint.h" - -#include "td/utils/ThreadSafeCounter.h" namespace vm { @@ -184,9 +183,9 @@ class CellBuilder : public td::CntObject { bool finalize_to(Ref& res, bool special = false) { return (res = finalize(special)).not_null(); } - CellSlice as_cellslice() const &; + CellSlice as_cellslice() const&; CellSlice as_cellslice() &&; - Ref as_cellslice_ref() const &; + Ref as_cellslice_ref() const&; Ref as_cellslice_ref() &&; static td::int64 get_total_cell_builders() { return get_thread_safe_counter().sum(); diff --git a/crypto/vm/cells/CellHash.cpp b/crypto/vm/cells/CellHash.cpp index 038e9a9a0e..94f37cdd45 100644 --- a/crypto/vm/cells/CellHash.cpp +++ b/crypto/vm/cells/CellHash.cpp @@ -16,10 +16,9 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/cells/CellHash.h" - -#include "td/utils/StringBuilder.h" #include "td/utils/Slice.h" +#include "td/utils/StringBuilder.h" +#include "vm/cells/CellHash.h" namespace vm { td::StringBuilder &operator<<(td::StringBuilder &sb, const CellHash &hash) { diff --git a/crypto/vm/cells/CellHash.h b/crypto/vm/cells/CellHash.h index 9435675f16..9bbfd4cc69 100644 --- a/crypto/vm/cells/CellHash.h +++ b/crypto/vm/cells/CellHash.h @@ -17,12 +17,11 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "vm/cells/CellTraits.h" -#include "common/bitstring.h" +#include +#include "common/bitstring.h" #include "td/utils/as.h" - -#include +#include "vm/cells/CellTraits.h" namespace td { class StringBuilder; } diff --git a/crypto/vm/cells/CellSlice.cpp b/crypto/vm/cells/CellSlice.cpp index 8bebb41b38..e8d9d58a2c 100644 --- a/crypto/vm/cells/CellSlice.cpp +++ b/crypto/vm/cells/CellSlice.cpp @@ -16,10 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/cells/CellSlice.h" -#include "vm/excno.hpp" #include "td/utils/bits.h" #include "td/utils/misc.h" +#include "vm/cells/CellSlice.h" +#include "vm/excno.hpp" namespace vm { @@ -985,7 +985,7 @@ void CellSlice::dump(std::ostream& os, int level, bool endl) const { if (level > 2) { char tmp[64]; std::snprintf(tmp, sizeof(tmp), "; ptr=data+%ld; z=%016llx", - static_cast(ptr && cell.not_null() ? ptr - cell->get_data() : -1), static_cast(z)); + static_cast(ptr && cell.not_null() ? ptr - cell->get_data() : -1), static_cast(z)); os << tmp << " (have " << size() << " bits; " << zd << " preloaded)"; } if (endl) { diff --git a/crypto/vm/cells/CellString.cpp b/crypto/vm/cells/CellString.cpp index 474bc797e0..870cd93218 100644 --- a/crypto/vm/cells/CellString.cpp +++ b/crypto/vm/cells/CellString.cpp @@ -16,11 +16,11 @@ Copyright 2019-2020 Telegram Systems LLP */ -#include "CellString.h" #include "td/utils/misc.h" - #include "vm/cells/CellSlice.h" +#include "CellString.h" + namespace vm { td::Status CellString::store(CellBuilder &cb, td::Slice slice, unsigned int top_bits) { td::uint32 size = td::narrow_cast(slice.size() * 8); diff --git a/crypto/vm/cells/CellString.h b/crypto/vm/cells/CellString.h index 10bd89aa70..57a27c7dd4 100644 --- a/crypto/vm/cells/CellString.h +++ b/crypto/vm/cells/CellString.h @@ -19,7 +19,6 @@ #pragma once #include "td/utils/Status.h" - #include "vm/cells/CellBuilder.h" namespace vm { diff --git a/crypto/vm/cells/CellTraits.cpp b/crypto/vm/cells/CellTraits.cpp index a619d86c20..c4e25aeaa1 100644 --- a/crypto/vm/cells/CellTraits.cpp +++ b/crypto/vm/cells/CellTraits.cpp @@ -16,10 +16,9 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/cells/CellTraits.h" - -#include "td/utils/StringBuilder.h" #include "td/utils/Slice.h" +#include "td/utils/StringBuilder.h" +#include "vm/cells/CellTraits.h" namespace vm { td::StringBuilder& operator<<(td::StringBuilder& sb, CellTraits::SpecialType special_type) { diff --git a/crypto/vm/cells/CellTraits.h b/crypto/vm/cells/CellTraits.h index c994e47e29..9b039b5b76 100644 --- a/crypto/vm/cells/CellTraits.h +++ b/crypto/vm/cells/CellTraits.h @@ -17,8 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/utils/int_types.h" #include "common/refcnt.hpp" +#include "td/utils/int_types.h" namespace td { class StringBuilder; diff --git a/crypto/vm/cells/CellUsageTree.cpp b/crypto/vm/cells/CellUsageTree.cpp index 05a4f0c452..28e531662d 100644 --- a/crypto/vm/cells/CellUsageTree.cpp +++ b/crypto/vm/cells/CellUsageTree.cpp @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "vm/cells/CellUsageTree.h" + #include "DataCell.h" namespace vm { diff --git a/crypto/vm/cells/CellUsageTree.h b/crypto/vm/cells/CellUsageTree.h index cd2470ddae..6a90eae428 100644 --- a/crypto/vm/cells/CellUsageTree.h +++ b/crypto/vm/cells/CellUsageTree.h @@ -18,11 +18,11 @@ */ #pragma once -#include "vm/cells/CellTraits.h" +#include #include "td/utils/int_types.h" #include "td/utils/logging.h" -#include +#include "vm/cells/CellTraits.h" namespace vm { diff --git a/crypto/vm/cells/ExtCell.h b/crypto/vm/cells/ExtCell.h index e2188fd7a5..0f0bd3be97 100644 --- a/crypto/vm/cells/ExtCell.h +++ b/crypto/vm/cells/ExtCell.h @@ -17,14 +17,14 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "vm/cells/Cell.h" -#include "vm/cells/PrunnedCell.h" -#include "common/AtomicRef.h" - #include -#include "td/utils/port/thread_local.h" + +#include "common/AtomicRef.h" #include "td/utils/HazardPointers.h" #include "td/utils/optional.h" +#include "td/utils/port/thread_local.h" +#include "vm/cells/Cell.h" +#include "vm/cells/PrunnedCell.h" namespace vm { diff --git a/crypto/vm/cells/LevelMask.h b/crypto/vm/cells/LevelMask.h index 35d03d9373..9dc3d24c04 100644 --- a/crypto/vm/cells/LevelMask.h +++ b/crypto/vm/cells/LevelMask.h @@ -18,9 +18,9 @@ */ #pragma once +#include "td/utils/StringBuilder.h" #include "td/utils/bits.h" #include "td/utils/int_types.h" -#include "td/utils/StringBuilder.h" namespace td { class StringBuilder; diff --git a/crypto/vm/cells/MerkleProof.cpp b/crypto/vm/cells/MerkleProof.cpp index 58f617a27f..6bc586866e 100644 --- a/crypto/vm/cells/MerkleProof.cpp +++ b/crypto/vm/cells/MerkleProof.cpp @@ -16,14 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/cells/MerkleProof.h" +#include "td/utils/HashMap.h" +#include "td/utils/HashSet.h" #include "td/utils/Status.h" +#include "vm/boc.h" #include "vm/cells/CellBuilder.h" #include "vm/cells/CellSlice.h" -#include "vm/boc.h" - -#include "td/utils/HashMap.h" -#include "td/utils/HashSet.h" +#include "vm/cells/MerkleProof.h" namespace vm { namespace detail { diff --git a/crypto/vm/cells/MerkleProof.h b/crypto/vm/cells/MerkleProof.h index b9fcfcf3a7..32fa133a29 100644 --- a/crypto/vm/cells/MerkleProof.h +++ b/crypto/vm/cells/MerkleProof.h @@ -17,11 +17,11 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "vm/cells/Cell.h" -#include "td/utils/buffer.h" - -#include #include +#include + +#include "td/utils/buffer.h" +#include "vm/cells/Cell.h" namespace vm { @@ -71,7 +71,7 @@ class MerkleProofBuilder { bool extract_proof_to(Ref &proof_root) const; td::Result extract_proof_boc() const; - void set_cell_load_callback(std::function f) { + void set_cell_load_callback(std::function f) { usage_tree->set_cell_load_callback(std::move(f)); } const CellUsageTree &get_usage_tree() const { diff --git a/crypto/vm/cells/MerkleUpdate.cpp b/crypto/vm/cells/MerkleUpdate.cpp index 894612bdcd..fdf4839152 100644 --- a/crypto/vm/cells/MerkleUpdate.cpp +++ b/crypto/vm/cells/MerkleUpdate.cpp @@ -16,11 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/cells/MerkleUpdate.h" -#include "vm/cells/MerkleProof.h" - #include "td/utils/HashMap.h" #include "td/utils/HashSet.h" +#include "vm/cells/MerkleProof.h" +#include "vm/cells/MerkleUpdate.h" namespace vm { namespace detail { diff --git a/crypto/vm/cells/MerkleUpdate.h b/crypto/vm/cells/MerkleUpdate.h index 41b4c138fc..487c545fbe 100644 --- a/crypto/vm/cells/MerkleUpdate.h +++ b/crypto/vm/cells/MerkleUpdate.h @@ -17,13 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "vm/cells/Cell.h" -#include "vm/cells/CellSlice.h" -#include "vm/cells/CellBuilder.h" +#include #include "td/utils/Status.h" - -#include +#include "vm/cells/Cell.h" +#include "vm/cells/CellBuilder.h" +#include "vm/cells/CellSlice.h" namespace vm { class MerkleUpdate { diff --git a/crypto/vm/cells/PrunnedCell.h b/crypto/vm/cells/PrunnedCell.h index d25e3bc012..7e3f894d27 100644 --- a/crypto/vm/cells/PrunnedCell.h +++ b/crypto/vm/cells/PrunnedCell.h @@ -148,7 +148,7 @@ class PrunnedCell final : public Cell { return info_.get_depth(trailer_)[get_level_mask().apply(level).get_hash_i()]; } - td::Status set_data_cell(Ref &&data_cell) const override { + td::Status set_data_cell(Ref&& data_cell) const override { return td::Status::OK(); } diff --git a/crypto/vm/cells/VirtualizationParameters.h b/crypto/vm/cells/VirtualizationParameters.h index b83e87d239..fb25f8fa88 100644 --- a/crypto/vm/cells/VirtualizationParameters.h +++ b/crypto/vm/cells/VirtualizationParameters.h @@ -18,11 +18,11 @@ */ #pragma once +#include + #include "td/utils/int_types.h" #include "td/utils/logging.h" -#include - namespace vm { namespace detail { class VirtualizationParameters { diff --git a/crypto/vm/continuation.cpp b/crypto/vm/continuation.cpp index bc67c6d226..b62cd9efcc 100644 --- a/crypto/vm/continuation.cpp +++ b/crypto/vm/continuation.cpp @@ -16,14 +16,14 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/dispatch.h" +#include "td/utils/misc.h" +#include "vm/boc.h" #include "vm/continuation.h" #include "vm/dict.h" +#include "vm/dispatch.h" #include "vm/log.h" #include "vm/vm.h" #include "vm/vmstate.h" -#include "vm/boc.h" -#include "td/utils/misc.h" namespace vm { diff --git a/crypto/vm/continuation.h b/crypto/vm/continuation.h index 0c758c9221..3a3b7f6894 100644 --- a/crypto/vm/continuation.h +++ b/crypto/vm/continuation.h @@ -20,9 +20,9 @@ #include "common/refcnt.hpp" #include "vm/cellslice.h" +#include "vm/log.h" #include "vm/stack.hpp" #include "vm/vmstate.h" -#include "vm/log.h" namespace vm { @@ -373,7 +373,7 @@ class OrdCont : public Continuation { Ref get_stack_ref() const { return data.stack; } - Ref copy_ord() const & { + Ref copy_ord() const& { return Ref{true, *this}; } Ref copy_ord() && { diff --git a/crypto/vm/contops.cpp b/crypto/vm/contops.cpp index e4499aa027..b2b0d04a60 100644 --- a/crypto/vm/contops.cpp +++ b/crypto/vm/contops.cpp @@ -17,13 +17,14 @@ Copyright 2017-2020 Telegram Systems LLP */ #include + +#include "vm/cellops.h" +#include "vm/continuation.h" #include "vm/contops.h" +#include "vm/excno.hpp" #include "vm/log.h" #include "vm/opctable.h" #include "vm/stack.hpp" -#include "vm/continuation.h" -#include "vm/cellops.h" -#include "vm/excno.hpp" #include "vm/vm.h" using namespace std::literals::string_literals; @@ -260,7 +261,7 @@ int exec_runvm_common(VmState* st, unsigned mode) { gas_limit = std::min(gas_limit, st->get_gas_limits().gas_remaining); vm::GasLimits gas{gas_limit, gas_max}; - VmStateInterface::Guard guard{nullptr}; // Don't consume gas for creating/loading cells during VM init + VmStateInterface::Guard guard{nullptr}; // Don't consume gas for creating/loading cells during VM init VmState new_state{ std::move(code), st->get_global_version(), std::move(new_stack), gas, (int)mode & 3, std::move(data), VmLog{}, std::vector>{}, std::move(c7)}; @@ -301,21 +302,24 @@ void register_continuation_jump_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xdb39, 16, "RETVARARGS", exec_ret_varargs)) .insert(OpcodeInstr::mksimple(0xdb3a, 16, "JMPXVARARGS", exec_jmpx_varargs)) .insert(OpcodeInstr::mksimple(0xdb3b, 16, "CALLCCVARARGS", exec_callcc_varargs)) - .insert(OpcodeInstr::mkext(0xdb3c, 16, 0, std::bind(dump_push_ref, _1, _2, _3, "CALLREF"), - std::bind(exec_do_with_ref, _1, _2, _4, - [](auto st, auto cont) { return st->call(std::move(cont)); }, "CALLREF"), - compute_len_push_ref)) - .insert(OpcodeInstr::mkext(0xdb3d, 16, 0, std::bind(dump_push_ref, _1, _2, _3, "JMPREF"), - std::bind(exec_do_with_ref, _1, _2, _4, - [](auto st, auto cont) { return st->jump(std::move(cont)); }, "JMPREF"), - compute_len_push_ref)) + .insert(OpcodeInstr::mkext( + 0xdb3c, 16, 0, std::bind(dump_push_ref, _1, _2, _3, "CALLREF"), + std::bind( + exec_do_with_ref, _1, _2, _4, [](auto st, auto cont) { return st->call(std::move(cont)); }, "CALLREF"), + compute_len_push_ref)) + .insert(OpcodeInstr::mkext( + 0xdb3d, 16, 0, std::bind(dump_push_ref, _1, _2, _3, "JMPREF"), + std::bind( + exec_do_with_ref, _1, _2, _4, [](auto st, auto cont) { return st->jump(std::move(cont)); }, "JMPREF"), + compute_len_push_ref)) .insert(OpcodeInstr::mkext(0xdb3e, 16, 0, std::bind(dump_push_ref, _1, _2, _3, "JMPREFDATA"), - std::bind(exec_do_with_ref, _1, _2, _4, - [](auto st, auto cont) { - st->push_code(); - return st->jump(std::move(cont)); - }, - "JMPREFDATA"), + std::bind( + exec_do_with_ref, _1, _2, _4, + [](auto st, auto cont) { + st->push_code(); + return st->jump(std::move(cont)); + }, + "JMPREFDATA"), compute_len_push_ref)) .insert(OpcodeInstr::mksimple(0xdb3f, 16, "RETDATA", exec_ret_data)) .insert(OpcodeInstr::mkfixed(0xdb4, 12, 12, dump_runvm, exec_runvm)->require_version(4)) @@ -565,40 +569,40 @@ void register_continuation_cond_loop_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xe1, 8, "IFNOTJMP", exec_ifnot_jmp)) .insert(OpcodeInstr::mksimple(0xe2, 8, "IFELSE", exec_if_else)) .insert(OpcodeInstr::mkext(0xe300, 16, 0, std::bind(dump_push_ref, _1, _2, _3, "IFREF"), - std::bind(exec_do_with_cell, _1, _2, _4, - [](auto st, auto cell) { - return st->get_stack().pop_bool() - ? st->call(st->ref_to_cont(std::move(cell))) - : 0; - }, - "IFREF"), + std::bind( + exec_do_with_cell, _1, _2, _4, + [](auto st, auto cell) { + return st->get_stack().pop_bool() ? st->call(st->ref_to_cont(std::move(cell))) + : 0; + }, + "IFREF"), compute_len_push_ref)) .insert(OpcodeInstr::mkext(0xe301, 16, 0, std::bind(dump_push_ref, _1, _2, _3, "IFNOTREF"), - std::bind(exec_do_with_cell, _1, _2, _4, - [](auto st, auto cell) { - return st->get_stack().pop_bool() - ? 0 - : st->call(st->ref_to_cont(std::move(cell))); - }, - "IFNOTREF"), + std::bind( + exec_do_with_cell, _1, _2, _4, + [](auto st, auto cell) { + return st->get_stack().pop_bool() ? 0 + : st->call(st->ref_to_cont(std::move(cell))); + }, + "IFNOTREF"), compute_len_push_ref)) .insert(OpcodeInstr::mkext(0xe302, 16, 0, std::bind(dump_push_ref, _1, _2, _3, "IFJMPREF"), - std::bind(exec_do_with_cell, _1, _2, _4, - [](auto st, auto cell) { - return st->get_stack().pop_bool() - ? st->jump(st->ref_to_cont(std::move(cell))) - : 0; - }, - "IFJMPREF"), + std::bind( + exec_do_with_cell, _1, _2, _4, + [](auto st, auto cell) { + return st->get_stack().pop_bool() ? st->jump(st->ref_to_cont(std::move(cell))) + : 0; + }, + "IFJMPREF"), compute_len_push_ref)) .insert(OpcodeInstr::mkext(0xe303, 16, 0, std::bind(dump_push_ref, _1, _2, _3, "IFNOTJMPREF"), - std::bind(exec_do_with_cell, _1, _2, _4, - [](auto st, auto cell) { - return st->get_stack().pop_bool() - ? 0 - : st->jump(st->ref_to_cont(std::move(cell))); - }, - "IFNOTJMPREF"), + std::bind( + exec_do_with_cell, _1, _2, _4, + [](auto st, auto cell) { + return st->get_stack().pop_bool() ? 0 + : st->jump(st->ref_to_cont(std::move(cell))); + }, + "IFNOTJMPREF"), compute_len_push_ref)) .insert(OpcodeInstr::mksimple(0xe304, 16, "CONDSEL", exec_condsel)) .insert(OpcodeInstr::mksimple(0xe305, 16, "CONDSELCHK", exec_condsel_chk)) @@ -1072,7 +1076,8 @@ void register_continuation_change_ops(OpcodeTable& cp0) { cp0.insert(OpcodeInstr::mksimple(0xede0, 16, "PUSHCTRX", exec_push_ctr_var)) .insert(OpcodeInstr::mksimple(0xede1, 16, "POPCTRX", exec_pop_ctr_var)) .insert(OpcodeInstr::mksimple(0xede2, 16, "SETCONTCTRX", exec_setcont_ctr_var)) - .insert(OpcodeInstr::mkfixed(0xede3, 16, 8, instr::dump_1c_l_add(1, "SETCONTCTRMANY "), exec_setcont_ctr_many)->require_version(9)) + .insert(OpcodeInstr::mkfixed(0xede3, 16, 8, instr::dump_1c_l_add(1, "SETCONTCTRMANY "), exec_setcont_ctr_many) + ->require_version(9)) .insert(OpcodeInstr::mksimple(0xede4, 16, "SETCONTCTRMANYX", exec_setcont_ctr_many_var)->require_version(9)) .insert(OpcodeInstr::mksimple(0xedf0, 16, "BOOLAND", std::bind(exec_compos, _1, 1, "BOOLAND"))) .insert(OpcodeInstr::mksimple(0xedf1, 16, "BOOLOR", std::bind(exec_compos, _1, 2, "BOOLOR"))) diff --git a/crypto/vm/cp0.cpp b/crypto/vm/cp0.cpp index fe5dd8ef1f..8b9c979e2d 100644 --- a/crypto/vm/cp0.cpp +++ b/crypto/vm/cp0.cpp @@ -16,16 +16,16 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "cp0.h" -#include "opctable.h" -#include "stackops.h" -#include "tupleops.h" #include "arithops.h" #include "cellops.h" #include "contops.h" -#include "dictops.h" +#include "cp0.h" #include "debugops.h" +#include "dictops.h" +#include "opctable.h" +#include "stackops.h" #include "tonops.h" +#include "tupleops.h" namespace vm { diff --git a/crypto/vm/db/BlobView.cpp b/crypto/vm/db/BlobView.cpp index 1a879d0032..4835134626 100644 --- a/crypto/vm/db/BlobView.cpp +++ b/crypto/vm/db/BlobView.cpp @@ -16,17 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/db/BlobView.h" +#include +#include -#include "td/utils/port/FileFd.h" #include "td/utils/HashMap.h" - #include "td/utils/format.h" -#include "td/utils/port/RwMutex.h" +#include "td/utils/port/FileFd.h" #include "td/utils/port/MemoryMapping.h" - -#include -#include +#include "td/utils/port/RwMutex.h" +#include "vm/db/BlobView.h" namespace vm { td::Result BlobView::view(td::MutableSlice slice, td::uint64 offset) { diff --git a/crypto/vm/db/CellHashTable.h b/crypto/vm/db/CellHashTable.h index a389806380..8d2f583b7f 100644 --- a/crypto/vm/db/CellHashTable.h +++ b/crypto/vm/db/CellHashTable.h @@ -18,10 +18,11 @@ */ #pragma once -#include "td/utils/Slice.h" -#include "td/utils/HashSet.h" #include +#include "td/utils/HashSet.h" +#include "td/utils/Slice.h" + namespace vm { template class CellHashTable { diff --git a/crypto/vm/db/CellStorage.cpp b/crypto/vm/db/CellStorage.cpp index 33c18ab3f9..518ea478da 100644 --- a/crypto/vm/db/CellStorage.cpp +++ b/crypto/vm/db/CellStorage.cpp @@ -16,16 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/db/CellStorage.h" +#include #include "td/utils/Parser.h" -#include "vm/db/DynamicBagOfCellsDb.h" -#include "vm/boc.h" #include "td/utils/base64.h" -#include "td/utils/tl_parsers.h" #include "td/utils/tl_helpers.h" - -#include +#include "td/utils/tl_parsers.h" +#include "vm/boc.h" +#include "vm/db/CellStorage.h" +#include "vm/db/DynamicBagOfCellsDb.h" namespace vm { namespace { @@ -194,8 +193,8 @@ td::Result CellLoader::load(td::Slice hash, bool need_da return res; } -td::Result> CellLoader::load_bulk(td::Span hashes, bool need_data, - ExtCellCreator &ext_cell_creator) { +td::Result> CellLoader::load_bulk(td::Span hashes, bool need_data, + ExtCellCreator &ext_cell_creator) { std::vector values; TRY_RESULT(get_statuses, reader_->get_multi(hashes, &values)); std::vector res; diff --git a/crypto/vm/db/CellStorage.h b/crypto/vm/db/CellStorage.h index e17fab5f22..90defbf614 100644 --- a/crypto/vm/db/CellStorage.h +++ b/crypto/vm/db/CellStorage.h @@ -18,11 +18,10 @@ */ #pragma once #include "td/db/KeyValue.h" -#include "vm/db/DynamicBagOfCellsDb.h" -#include "vm/cells.h" - #include "td/utils/Slice.h" #include "td/utils/Status.h" +#include "vm/cells.h" +#include "vm/db/DynamicBagOfCellsDb.h" namespace vm { using KeyValue = td::KeyValue; @@ -49,7 +48,8 @@ class CellLoader { }; CellLoader(std::shared_ptr reader, std::function on_load_callback = {}); td::Result load(td::Slice hash, bool need_data, ExtCellCreator &ext_cell_creator); - td::Result> load_bulk(td::Span hashes, bool need_data, ExtCellCreator &ext_cell_creator); + td::Result> load_bulk(td::Span hashes, bool need_data, + ExtCellCreator &ext_cell_creator); static td::Result load(td::Slice hash, td::Slice value, bool need_data, ExtCellCreator &ext_cell_creator); td::Result load_refcnt(td::Slice hash); // This only loads refcnt_, cell_ == null KeyValueReader &key_value_reader() const { diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index 1b91bd2976..f47c732469 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -16,20 +16,18 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/db/DynamicBagOfCellsDb.h" -#include "vm/db/CellStorage.h" -#include "vm/db/CellHashTable.h" - -#include "vm/cells/ExtCell.h" +#include +#include "common/delay.h" +#include "td/actor/actor.h" +#include "td/utils/ThreadSafeCounter.h" #include "td/utils/base64.h" #include "td/utils/format.h" -#include "td/utils/ThreadSafeCounter.h" - +#include "vm/cells/ExtCell.h" #include "vm/cellslice.h" -#include -#include "td/actor/actor.h" -#include "common/delay.h" +#include "vm/db/CellHashTable.h" +#include "vm/db/CellStorage.h" +#include "vm/db/DynamicBagOfCellsDb.h" namespace vm { namespace { @@ -181,29 +179,28 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return; } SimpleExtCellCreator ext_cell_creator(cell_db_reader_); - executor->execute_async( - [executor, loader = *loader_, hash = CellHash::from_slice(hash), db = this, - ext_cell_creator = std::move(ext_cell_creator), promise = std::move(promise_ptr)]() mutable { - TRY_RESULT_PROMISE((*promise), res, loader.load(hash.as_slice(), true, ext_cell_creator)); - if (res.status != CellLoader::LoadResult::Ok) { - promise->set_error(td::Status::Error("cell not found")); - return; - } - Ref cell = res.cell(); - executor->execute_sync([hash, db, res = std::move(res), - ext_cell_creator = std::move(ext_cell_creator)]() mutable { - db->hash_table_.apply(hash.as_slice(), [&](CellInfo &info) { - db->update_cell_info_loaded(info, hash.as_slice(), std::move(res)); - }); - for (auto &ext_cell : ext_cell_creator.get_created_cells()) { - auto ext_cell_hash = ext_cell->get_hash(); - db->hash_table_.apply(ext_cell_hash.as_slice(), [&](CellInfo &info) { - db->update_cell_info_created_ext(info, std::move(ext_cell)); - }); - } - }); - promise->set_result(std::move(cell)); + executor->execute_async([executor, loader = *loader_, hash = CellHash::from_slice(hash), db = this, + ext_cell_creator = std::move(ext_cell_creator), + promise = std::move(promise_ptr)]() mutable { + TRY_RESULT_PROMISE((*promise), res, loader.load(hash.as_slice(), true, ext_cell_creator)); + if (res.status != CellLoader::LoadResult::Ok) { + promise->set_error(td::Status::Error("cell not found")); + return; + } + Ref cell = res.cell(); + executor->execute_sync([hash, db, res = std::move(res), + ext_cell_creator = std::move(ext_cell_creator)]() mutable { + db->hash_table_.apply(hash.as_slice(), [&](CellInfo &info) { + db->update_cell_info_loaded(info, hash.as_slice(), std::move(res)); }); + for (auto &ext_cell : ext_cell_creator.get_created_cells()) { + auto ext_cell_hash = ext_cell->get_hash(); + db->hash_table_.apply(ext_cell_hash.as_slice(), + [&](CellInfo &info) { db->update_cell_info_created_ext(info, std::move(ext_cell)); }); + } + }); + promise->set_result(std::move(cell)); + }); } CellInfo &get_cell_info_lazy(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) { return hash_table_.apply(hash.substr(hash.size() - Cell::hash_bytes), @@ -400,7 +397,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return db_->load_bulk(hashes); } TRY_RESULT(load_result, cell_loader_->load_bulk(hashes, true, *this)); - + std::vector> res; res.reserve(load_result.size()); for (auto &load_res : load_result) { diff --git a/crypto/vm/db/DynamicBagOfCellsDb.h b/crypto/vm/db/DynamicBagOfCellsDb.h index ec0aee9c14..288e12aedc 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.h +++ b/crypto/vm/db/DynamicBagOfCellsDb.h @@ -17,14 +17,13 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/db/KeyValue.h" -#include "vm/cells.h" +#include +#include "td/actor/PromiseFuture.h" +#include "td/db/KeyValue.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" -#include "td/actor/PromiseFuture.h" - -#include +#include "vm/cells.h" namespace td { class KeyValueReader; diff --git a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp index ae06beb1b5..b618669264 100644 --- a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp @@ -1,18 +1,15 @@ -#include "vm/db/DynamicBagOfCellsDb.h" -#include "vm/db/CellStorage.h" -#include "vm/db/CellHashTable.h" - -#include "vm/cells/ExtCell.h" +#include +#include "td/utils/ThreadSafeCounter.h" #include "td/utils/base64.h" #include "td/utils/format.h" -#include "td/utils/ThreadSafeCounter.h" #include "td/utils/misc.h" #include "validator/validator.h" - +#include "vm/cells/ExtCell.h" #include "vm/cellslice.h" - -#include +#include "vm/db/CellHashTable.h" +#include "vm/db/CellStorage.h" +#include "vm/db/DynamicBagOfCellsDb.h" namespace vm { namespace { @@ -770,9 +767,9 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { std::vector> result; auto s = cell_db_reader_->key_value_reader().for_each_in_range( "desc", "desd", [&](const td::Slice &key, const td::Slice &value) { - if (result.size() >= max_count) { - return td::Status::Error("COUNT_LIMIT"); - } + if (result.size() >= max_count) { + return td::Status::Error("COUNT_LIMIT"); + } if (td::begins_with(key, "desc") && key.size() != 32) { result.emplace_back(key.str(), value.str()); } @@ -1443,9 +1440,10 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { CHECK(state.sync_with_db); auto data_cell = info->cell->load_cell().move_as_ok().data_cell; stats_.diff_full.inc(); - worker.add_result({.type = CellStorer::Diff::Set, - .key = info->cell->get_hash(), - .value = CellStorer::serialize_value(ref_cnt_diff + state.db_ref_cnt, data_cell, should_compress)}); + worker.add_result( + {.type = CellStorer::Diff::Set, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_value(ref_cnt_diff + state.db_ref_cnt, data_cell, should_compress)}); } else { stats_.diff_ref_cnt.inc(); worker.add_result({.type = CellStorer::Diff::Merge, @@ -1478,10 +1476,10 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { LOG_IF(ERROR, dbg) << "DEC REFCNT " << *info; CHECK(info->cell->is_loaded()); - worker.add_result( - {.type = CellStorer::Diff::Set, - .key = info->cell->get_hash(), - .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, should_compress)}); + worker.add_result({.type = CellStorer::Diff::Set, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_value( + new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, should_compress)}); stats_.dec_save_full.inc(); } } else { @@ -1495,10 +1493,10 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { LOG_IF(ERROR, dbg) << "INC REFCNT " << *info; } - worker.add_result( - {.type = CellStorer::Diff::Set, - .key = info->cell->get_hash(), - .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, should_compress)}); + worker.add_result({.type = CellStorer::Diff::Set, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_value( + new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, should_compress)}); stats_.inc_save_full.inc(); } } diff --git a/crypto/vm/db/InMemoryBagOfCellsDb.cpp b/crypto/vm/db/InMemoryBagOfCellsDb.cpp index a41edb6f48..b7f59134c6 100644 --- a/crypto/vm/db/InMemoryBagOfCellsDb.cpp +++ b/crypto/vm/db/InMemoryBagOfCellsDb.cpp @@ -1,5 +1,7 @@ -#include "CellStorage.h" -#include "DynamicBagOfCellsDb.h" +#include + +#include "td/utils/HashMap.h" +#include "td/utils/HashSet.h" #include "td/utils/Timer.h" #include "td/utils/base64.h" #include "td/utils/format.h" @@ -11,10 +13,8 @@ #include "vm/cells/DataCell.h" #include "vm/cells/ExtCell.h" -#include "td/utils/HashMap.h" -#include "td/utils/HashSet.h" - -#include +#include "CellStorage.h" +#include "DynamicBagOfCellsDb.h" #if TD_PORT_POSIX #include diff --git a/crypto/vm/db/StaticBagOfCellsDb.cpp b/crypto/vm/db/StaticBagOfCellsDb.cpp index 0279ac77fa..f70035f2ba 100644 --- a/crypto/vm/db/StaticBagOfCellsDb.cpp +++ b/crypto/vm/db/StaticBagOfCellsDb.cpp @@ -16,19 +16,16 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/db/StaticBagOfCellsDb.h" - -#include "vm/boc.h" - -#include "vm/cells/ExtCell.h" +#include +#include "td/utils/ConcurrentHashTable.h" #include "td/utils/crypto.h" #include "td/utils/format.h" #include "td/utils/misc.h" #include "td/utils/port/RwMutex.h" -#include "td/utils/ConcurrentHashTable.h" - -#include +#include "vm/boc.h" +#include "vm/cells/ExtCell.h" +#include "vm/db/StaticBagOfCellsDb.h" namespace vm { // @@ -39,7 +36,7 @@ class RootCell : public Cell { struct PrivateTag {}; public: - td::Status set_data_cell(Ref &&data_cell) const override { + td::Status set_data_cell(Ref&& data_cell) const override { return cell_->set_data_cell(std::move(data_cell)); } td::Result load_cell() const override { @@ -371,12 +368,14 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { } if (std::numeric_limits::max() - res.begin < info_.data_offset || std::numeric_limits::max() - res.end < info_.data_offset) { - return td::Status::Error(PSTRING() << "bag-of-cell error: invalid cell location (1) " << res.begin << ":" << res.end); + return td::Status::Error(PSTRING() << "bag-of-cell error: invalid cell location (1) " << res.begin << ":" + << res.end); } res.begin += static_cast(info_.data_offset); res.end += static_cast(info_.data_offset); if (res.begin > res.end) { - return td::Status::Error(PSTRING() << "bag-of-cell error: invalid cell location (2) " << res.begin << ":" << res.end); + return td::Status::Error(PSTRING() << "bag-of-cell error: invalid cell location (2) " << res.begin << ":" + << res.end); } return res; } diff --git a/crypto/vm/db/StaticBagOfCellsDb.h b/crypto/vm/db/StaticBagOfCellsDb.h index 03c45ee88a..4cd144653c 100644 --- a/crypto/vm/db/StaticBagOfCellsDb.h +++ b/crypto/vm/db/StaticBagOfCellsDb.h @@ -18,10 +18,9 @@ */ #pragma once -#include "vm/cells.h" #include "td/db/utils/BlobView.h" - #include "td/utils/Status.h" +#include "vm/cells.h" namespace vm { class StaticBagOfCellsDb : public std::enable_shared_from_this { diff --git a/crypto/vm/db/TonDb.cpp b/crypto/vm/db/TonDb.cpp index ac69973fd0..67970f9392 100644 --- a/crypto/vm/db/TonDb.cpp +++ b/crypto/vm/db/TonDb.cpp @@ -16,10 +16,9 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/db/TonDb.h" - -#include "td/utils/tl_helpers.h" #include "td/utils/Random.h" +#include "td/utils/tl_helpers.h" +#include "vm/db/TonDb.h" #if TDDB_USE_ROCKSDB #include "td/db/RocksDb.h" diff --git a/crypto/vm/db/TonDb.h b/crypto/vm/db/TonDb.h index 6cd8aa4fa5..eae692a0df 100644 --- a/crypto/vm/db/TonDb.h +++ b/crypto/vm/db/TonDb.h @@ -17,15 +17,14 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "vm/cellslice.h" -#include "vm/cells.h" -#include "vm/boc.h" #include "td/db/KeyValue.h" -#include "vm/db/CellStorage.h" -#include "vm/db/CellHashTable.h" - #include "td/utils/Slice.h" #include "td/utils/Status.h" +#include "vm/boc.h" +#include "vm/cells.h" +#include "vm/cellslice.h" +#include "vm/db/CellHashTable.h" +#include "vm/db/CellStorage.h" namespace vm { class SmartContractDbImpl; @@ -113,7 +112,6 @@ class TonDbTransactionImpl; using TonDbTransaction = std::unique_ptr; class TonDbTransactionImpl { public: - SmartContractDb begin_smartcontract(td::Slice hash = std::string(32, '\0')); void commit_smartcontract(SmartContractDb txn); @@ -146,16 +144,25 @@ class TonDbTransactionImpl { struct Eq { using is_transparent = void; // Pred to use - bool operator()(const SmartContractInfo &info, const SmartContractInfo &other_info) const { return info.hash == other_info.hash;} - bool operator()(const SmartContractInfo &info, td::Slice hash) const { return info.hash == hash;} - bool operator()(td::Slice hash, const SmartContractInfo &info) const { return info.hash == hash;} - + bool operator()(const SmartContractInfo &info, const SmartContractInfo &other_info) const { + return info.hash == other_info.hash; + } + bool operator()(const SmartContractInfo &info, td::Slice hash) const { + return info.hash == hash; + } + bool operator()(td::Slice hash, const SmartContractInfo &info) const { + return info.hash == hash; + } }; struct Hash { using is_transparent = void; // Pred to use using transparent_key_equal = Eq; - size_t operator()(td::Slice hash) const { return cell_hash_slice_hash(hash); } - size_t operator()(const SmartContractInfo &info) const { return cell_hash_slice_hash(info.hash);} + size_t operator()(td::Slice hash) const { + return cell_hash_slice_hash(hash); + } + size_t operator()(const SmartContractInfo &info) const { + return cell_hash_slice_hash(info.hash); + } }; }; diff --git a/crypto/vm/debugops.cpp b/crypto/vm/debugops.cpp index 3f27de2378..29a8345a61 100644 --- a/crypto/vm/debugops.cpp +++ b/crypto/vm/debugops.cpp @@ -17,11 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #include + #include "vm/debugops.h" +#include "vm/excno.hpp" #include "vm/log.h" #include "vm/opctable.h" #include "vm/stack.hpp" -#include "vm/excno.hpp" #include "vm/vm.h" namespace vm { @@ -113,7 +114,7 @@ int exec_dump_string(VmState* st) { Stack& stack = st->get_stack(); - if (stack.depth() > 0){ + if (stack.depth() > 0) { auto cs = stack[0].as_slice(); if (cs.not_null()) { // wanted t_slice @@ -127,8 +128,7 @@ int exec_dump_string(VmState* st) { std::string s{tmp, tmp + cnt}; std::cerr << "#DEBUG#: " << s << std::endl; - } - else { + } else { std::cerr << "#DEBUG#: slice contains not valid bits count" << std::endl; } @@ -151,7 +151,7 @@ void register_debug_ops(OpcodeTable& cp0) { // NB: all non-redefined opcodes in fe00..feff should be redirected to dummy debug definitions cp0.insert(OpcodeInstr::mksimple(0xfe00, 16, "DUMPSTK", exec_dump_stack)) .insert(OpcodeInstr::mkfixedrange(0xfe01, 0xfe14, 16, 8, instr::dump_1c_and(0xff, "DEBUG "), exec_dummy_debug)) - .insert(OpcodeInstr::mksimple(0xfe14, 16,"STRDUMP", exec_dump_string)) + .insert(OpcodeInstr::mksimple(0xfe14, 16, "STRDUMP", exec_dump_string)) .insert(OpcodeInstr::mkfixedrange(0xfe15, 0xfe20, 16, 8, instr::dump_1c_and(0xff, "DEBUG "), exec_dummy_debug)) .insert(OpcodeInstr::mkfixed(0xfe2, 12, 4, instr::dump_1sr("DUMP"), exec_dump_value)) .insert(OpcodeInstr::mkfixedrange(0xfe30, 0xfef0, 16, 8, instr::dump_1c_and(0xff, "DEBUG "), exec_dummy_debug)) diff --git a/crypto/vm/dict.cpp b/crypto/vm/dict.cpp index 0450932a2f..e8c0e03a51 100644 --- a/crypto/vm/dict.cpp +++ b/crypto/vm/dict.cpp @@ -16,14 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/dict.h" -#include "vm/cells.h" -#include "vm/cellslice.h" -#include "vm/stack.hpp" #include "common/bitstring.h" #include "td/utils/Random.h" - #include "td/utils/bits.h" +#include "vm/cells.h" +#include "vm/cellslice.h" +#include "vm/dict.h" +#include "vm/stack.hpp" namespace vm { @@ -180,7 +179,7 @@ bool DictionaryBase::append_dict_to_bool(CellBuilder& cb) && { return cb.store_maybe_ref(std::move(root_cell)); } -bool DictionaryBase::append_dict_to_bool(CellBuilder& cb) const & { +bool DictionaryBase::append_dict_to_bool(CellBuilder& cb) const& { return is_valid() && cb.store_maybe_ref(root_cell); } @@ -1102,8 +1101,8 @@ Ref Dictionary::lookup_set_gen(td::ConstBitPtr key, int key_len, cons } Ref Dictionary::lookup_set(td::ConstBitPtr key, int key_len, Ref value, SetMode mode) { - return lookup_set_gen(key, key_len, [value](CellBuilder& cb) { return cell_builder_add_slice_bool(cb, *value); }, - mode); + return lookup_set_gen( + key, key_len, [value](CellBuilder& cb) { return cell_builder_add_slice_bool(cb, *value); }, mode); } Ref Dictionary::lookup_set_ref(td::ConstBitPtr key, int key_len, Ref val_ref, SetMode mode) { @@ -2261,8 +2260,8 @@ bool DictionaryFixed::combine_with(DictionaryFixed& dict2) { } bool DictionaryFixed::dict_check_for_each(Ref dict, td::BitPtr key_buffer, int n, int total_key_len, - const DictionaryFixed::foreach_func_t& foreach_func, - bool invert_first, bool shuffle) const { + const DictionaryFixed::foreach_func_t& foreach_func, bool invert_first, + bool shuffle) const { if (dict.is_null()) { return true; } @@ -2282,7 +2281,7 @@ bool DictionaryFixed::dict_check_for_each(Ref dict, td::BitPtr key_buffer, if (l) { invert_first = false; } - bool invert = shuffle ? td::Random::fast(0, 1) == 1: invert_first; + bool invert = shuffle ? td::Random::fast(0, 1) == 1 : invert_first; if (invert) { std::swap(c1, c2); } @@ -2804,7 +2803,7 @@ Ref AugmentedDictionary::extract_root() && { return std::move(root); } -bool AugmentedDictionary::append_dict_to_bool(CellBuilder& cb) const & { +bool AugmentedDictionary::append_dict_to_bool(CellBuilder& cb) const& { if (!is_valid()) { return false; } diff --git a/crypto/vm/dict.h b/crypto/vm/dict.h index 4da56acd97..a0c8a70500 100644 --- a/crypto/vm/dict.h +++ b/crypto/vm/dict.h @@ -17,11 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include + #include "common/bitstring.h" #include "vm/cells.h" #include "vm/cellslice.h" #include "vm/stack.hpp" -#include namespace vm { using td::BitSlice; @@ -138,7 +139,7 @@ class DictionaryBase { return std::move(root_cell); } bool append_dict_to_bool(CellBuilder& cb) &&; - bool append_dict_to_bool(CellBuilder& cb) const &; + bool append_dict_to_bool(CellBuilder& cb) const&; int get_key_bits() const { return key_bits; } @@ -575,7 +576,7 @@ class AugmentedDictionary final : public DictionaryFixed { Ref get_wrapped_dict_root() const; Ref extract_root() &&; bool append_dict_to_bool(CellBuilder& cb) &&; - bool append_dict_to_bool(CellBuilder& cb) const &; + bool append_dict_to_bool(CellBuilder& cb) const&; Ref get_root_extra() const; Ref lookup(td::ConstBitPtr key, int key_len); Ref lookup_ref(td::ConstBitPtr key, int key_len); diff --git a/crypto/vm/dictops.cpp b/crypto/vm/dictops.cpp index 02f26fdd29..c11dfa127d 100644 --- a/crypto/vm/dictops.cpp +++ b/crypto/vm/dictops.cpp @@ -17,15 +17,16 @@ Copyright 2017-2020 Telegram Systems LLP */ #include + +#include "common/bigint.hpp" +#include "common/refint.h" +#include "vm/dict.h" +#include "vm/dictops.h" +#include "vm/excno.hpp" #include "vm/log.h" #include "vm/opctable.h" #include "vm/stack.hpp" -#include "vm/excno.hpp" #include "vm/vm.h" -#include "common/bigint.hpp" -#include "common/refint.h" -#include "vm/dictops.h" -#include "vm/dict.h" namespace vm { diff --git a/crypto/vm/dispatch.cpp b/crypto/vm/dispatch.cpp index 53999c7b84..823223b65a 100644 --- a/crypto/vm/dispatch.cpp +++ b/crypto/vm/dispatch.cpp @@ -16,12 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/dispatch.h" -#include "vm/excno.hpp" #include #include #include +#include "vm/dispatch.h" +#include "vm/excno.hpp" + namespace vm { namespace { diff --git a/crypto/vm/fmt.hpp b/crypto/vm/fmt.hpp index dff7290daa..37b2cee97d 100644 --- a/crypto/vm/fmt.hpp +++ b/crypto/vm/fmt.hpp @@ -17,11 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include + #include "common/refint.h" +#include "vm/cellparse.hpp" #include "vm/cells.h" #include "vm/cellslice.h" -#include "vm/cellparse.hpp" -#include namespace vm { diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp index 786fe62e94..62788155aa 100644 --- a/crypto/vm/large-boc-serializer.cpp +++ b/crypto/vm/large-boc-serializer.cpp @@ -14,14 +14,14 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ +#include + #include "td/utils/Time.h" #include "td/utils/Timer.h" - -#include -#include "vm/boc.h" +#include "td/utils/misc.h" #include "vm/boc-writers.h" +#include "vm/boc.h" #include "vm/cellslice.h" -#include "td/utils/misc.h" namespace vm { @@ -129,7 +129,7 @@ td::Result LargeBocSerializer::import_cell(Hash root_hash, int root_depth) if (current_depth > Cell::max_depth) { return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large"); } - + cell_list.resize(cell_list.size() + current_depth_hashes.size()); td::BTreeMap> next_depth_hashes; auto batch_start = current_depth_hashes.begin(); @@ -145,8 +145,8 @@ td::Result LargeBocSerializer::import_cell(Hash root_hash, int root_depth) ++batch_start; } - TRY_RESULT_PREFIX(loaded_results, reader->load_bulk(batch_hashes), - "error while importing a cell into a bag of cells: "); + TRY_RESULT_PREFIX(loaded_results, reader->load_bulk(batch_hashes), + "error while importing a cell into a bag of cells: "); DCHECK(loaded_results.size() == batch_hashes.size()); for (size_t i = 0; i < loaded_results.size(); ++i) { @@ -154,7 +154,7 @@ td::Result LargeBocSerializer::import_cell(Hash root_hash, int root_depth) if (cell->is_virtualized()) { return td::Status::Error( - "error while importing a cell into a bag of cells: cell has non-zero virtualization level"); + "error while importing a cell into a bag of cells: cell has non-zero virtualization level"); } const auto hash = cell->get_hash(); @@ -197,7 +197,7 @@ td::Result LargeBocSerializer::import_cell(Hash root_hash, int root_depth) dc_info.should_cache = idx_should_cache->second; dc_info.hcnt = static_cast(dc->get_level_mask().get_hashes_count()); DCHECK(dc_info.hcnt <= 4); - dc_info.wt = 0; // will be calculated after traversing + dc_info.wt = 0; // will be calculated after traversing TRY_RESULT(serialized_size, td::narrow_cast_safe(dc->get_serialized_size())); data_bytes += dc_info.serialized_size = serialized_size; cell_count++; @@ -212,7 +212,7 @@ td::Result LargeBocSerializer::import_cell(Hash root_hash, int root_depth) current_depth++; } DCHECK(next_child_idx == cell_count); - + for (int idx = cell_count - 1; idx >= start_ind; --idx) { CellInfo& cell_info = cell_list[idx]->second; @@ -481,23 +481,23 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { } for (int batch_start = 0; batch_start < cell_count; batch_start += load_batch_size) { int batch_end = std::min(batch_start + static_cast(load_batch_size), cell_count); - + std::vector batch_hashes; batch_hashes.reserve(batch_end - batch_start); for (int i = batch_start; i < batch_end; ++i) { int cell_index = cell_count - 1 - i; batch_hashes.push_back(cell_list[cell_index]->first.as_slice()); } - + TRY_RESULT(batch_cells, reader->load_bulk(std::move(batch_hashes))); - + for (int i = batch_start; i < batch_end; ++i) { int idx_in_batch = i - batch_start; int cell_index = cell_count - 1 - i; - + const auto& dc_info = cell_list[cell_index]->second; auto& dc = batch_cells[idx_in_batch]; - + bool with_hash = (mode & Mode::WithIntHashes) && !dc_info.wt; if (dc_info.is_root_cell && (mode & Mode::WithTopHash)) { with_hash = true; @@ -532,7 +532,7 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { } // namespace td::Status boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, - int mode, td::CancellationToken cancellation_token) { + int mode, td::CancellationToken cancellation_token) { td::Timer timer; CHECK(reader != nullptr) LargeBocSerializer serializer(reader); diff --git a/crypto/vm/memo.cpp b/crypto/vm/memo.cpp index 015bab6d06..97616142dc 100644 --- a/crypto/vm/memo.cpp +++ b/crypto/vm/memo.cpp @@ -16,8 +16,8 @@ Copyright 2020 Telegram Systems LLP */ -#include "vm/memo.h" #include "vm/excno.hpp" +#include "vm/memo.h" #include "vm/vm.h" namespace vm { diff --git a/crypto/vm/memo.h b/crypto/vm/memo.h index db6dc8e200..82b513a31a 100644 --- a/crypto/vm/memo.h +++ b/crypto/vm/memo.h @@ -18,9 +18,9 @@ */ #pragma once #include "common/refcnt.hpp" +#include "td/utils/optional.h" #include "vm/cells.h" #include "vm/vmstate.h" -#include "td/utils/optional.h" namespace vm { using td::Ref; diff --git a/crypto/vm/opctable.cpp b/crypto/vm/opctable.cpp index 0521a76391..e58110ea9e 100644 --- a/crypto/vm/opctable.cpp +++ b/crypto/vm/opctable.cpp @@ -17,17 +17,17 @@ Copyright 2017-2020 Telegram Systems LLP */ #include -#include -#include "vm/opctable.h" -#include "vm/cellslice.h" -#include "vm/excno.hpp" -#include "vm/vm.h" -#include +#include #include +#include +#include #include -#include #include "td/utils/format.h" +#include "vm/cellslice.h" +#include "vm/excno.hpp" +#include "vm/opctable.h" +#include "vm/vm.h" namespace vm { @@ -382,8 +382,8 @@ dump_arg_instr_func_t dump_2sr(std::string prefix, std::string suffix) { dump_arg_instr_func_t dump_2sr_adj(unsigned adj, std::string prefix, std::string suffix) { return [adj, prefix, suffix](CellSlice&, unsigned args) -> std::string { std::ostringstream os; - os << prefix << 's' << (int)((args >> 4) & 15) - (int)((adj >> 4) & 15) << ",s" << (int)(args & 15) - (int)(adj & 15) - << suffix; + os << prefix << 's' << (int)((args >> 4) & 15) - (int)((adj >> 4) & 15) << ",s" + << (int)(args & 15) - (int)(adj & 15) << suffix; return os.str(); }; } diff --git a/crypto/vm/opctable.h b/crypto/vm/opctable.h index 34d2ef0a75..56870a6ce1 100644 --- a/crypto/vm/opctable.h +++ b/crypto/vm/opctable.h @@ -17,11 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "vm/dispatch.h" #include +#include #include #include -#include + +#include "vm/dispatch.h" namespace vm { @@ -194,13 +195,16 @@ class OpcodeInstrExt : public OpcodeInstr { class OpcodeInstrWithVersion : public OpcodeInstr { public: OpcodeInstrWithVersion() = delete; - OpcodeInstrWithVersion(OpcodeInstr* instr, int required_version) : - OpcodeInstr(instr->get_opcode_min(), instr->get_opcode_max()), instr(instr), required_version(required_version) { + OpcodeInstrWithVersion(OpcodeInstr* instr, int required_version) + : OpcodeInstr(instr->get_opcode_min(), instr->get_opcode_max()) + , instr(instr) + , required_version(required_version) { } ~OpcodeInstrWithVersion() override = default; int dispatch(VmState* st, CellSlice& cs, unsigned opcode, unsigned bits) const override; std::string dump(CellSlice& cs, unsigned opcode, unsigned bits) const override; int instr_len(const CellSlice& cs, unsigned opcode, unsigned bits) const override; + private: OpcodeInstr* instr; int required_version; diff --git a/crypto/vm/stack.cpp b/crypto/vm/stack.cpp index aac1b6b532..a3dce4a156 100644 --- a/crypto/vm/stack.cpp +++ b/crypto/vm/stack.cpp @@ -16,13 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/stack.hpp" -#include "vm/continuation.h" -#include "vm/box.hpp" +#include "td/utils/misc.h" #include "vm/atom.h" -#include "vm/vmstate.h" #include "vm/boc.h" -#include "td/utils/misc.h" +#include "vm/box.hpp" +#include "vm/continuation.h" +#include "vm/stack.hpp" +#include "vm/vmstate.h" namespace td { template class td::Cnt; @@ -83,7 +83,7 @@ std::string StackEntry::to_lisp_string() const { return std::move(os).str(); } -static std::string cell_to_hex(const td::Ref &cell) { +static std::string cell_to_hex(const td::Ref& cell) { auto boc = vm::std_boc_serialize(cell); if (boc.is_ok()) { return td::buffer_to_hex(boc.move_as_ok().as_slice()); diff --git a/crypto/vm/stack.hpp b/crypto/vm/stack.hpp index 6a52e4a2b7..342e733c8d 100644 --- a/crypto/vm/stack.hpp +++ b/crypto/vm/stack.hpp @@ -18,25 +18,24 @@ */ #pragma once -#include #include -#include -#include +#include +#include #include -#include #include -#include "common/refcnt.hpp" +#include +#include +#include + #include "common/bigint.hpp" -#include "common/refint.h" #include "common/bitstring.h" +#include "common/refcnt.hpp" +#include "common/refint.h" +#include "td/utils/Span.h" #include "vm/cells.h" #include "vm/cellslice.h" #include "vm/excno.hpp" -#include "td/utils/Span.h" - -#include - namespace td { extern template class td::Cnt; extern template class td::Ref>; diff --git a/crypto/vm/stackops.cpp b/crypto/vm/stackops.cpp index c8180f1a37..b48d763e8d 100644 --- a/crypto/vm/stackops.cpp +++ b/crypto/vm/stackops.cpp @@ -16,11 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "vm/excno.hpp" #include "vm/log.h" -#include "vm/stackops.h" #include "vm/opctable.h" #include "vm/stack.hpp" -#include "vm/excno.hpp" +#include "vm/stackops.h" #include "vm/vm.h" namespace vm { diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index 88f9f19363..a98fd9f703 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -17,23 +17,24 @@ Copyright 2017-2020 Telegram Systems LLP */ #include -#include "vm/tonops.h" +#include + +#include "block/block-auto.h" +#include "block/block-parse.h" +#include "crypto/ellcurve/p256.h" +#include "crypto/ellcurve/secp256k1.h" +#include "openssl/digest.hpp" +#include "vm/Hasher.h" +#include "vm/boc.h" +#include "vm/dict.h" +#include "vm/excno.hpp" #include "vm/log.h" #include "vm/opctable.h" #include "vm/stack.hpp" -#include "vm/excno.hpp" +#include "vm/tonops.h" #include "vm/vm.h" -#include "vm/dict.h" -#include "vm/boc.h" -#include "Ed25519.h" -#include "vm/Hasher.h" -#include "block/block-auto.h" -#include "block/block-parse.h" -#include "crypto/ellcurve/secp256k1.h" -#include "crypto/ellcurve/p256.h" -#include "openssl/digest.hpp" -#include +#include "Ed25519.h" #include "bls.h" #include "mc-config.h" @@ -59,7 +60,7 @@ bool debug(int x) { } // namespace #define DBG_START int dbg = 0; -#define DBG debug(++dbg)&& +#define DBG debug(++dbg) && #define DEB_START DBG_START #define DEB DBG @@ -322,8 +323,7 @@ int exec_get_storage_fee(VmState* st) { td::int64 delta = stack.pop_long_range(std::numeric_limits::max(), 0); td::uint64 bits = stack.pop_long_range(std::numeric_limits::max(), 0); td::uint64 cells = stack.pop_long_range(std::numeric_limits::max(), 0); - td::optional maybe_prices = - util::get_storage_prices(get_unpacked_config_tuple(st)); + td::optional maybe_prices = util::get_storage_prices(get_unpacked_config_tuple(st)); stack.push_int(util::calculate_storage_fee(maybe_prices, is_masterchain, delta, bits, cells)); return 0; } @@ -467,15 +467,23 @@ void register_ton_config_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xf82a, 16, "MYCODE", std::bind(exec_get_param, _1, 10, "MYCODE"))) .insert(OpcodeInstr::mksimple(0xf82b, 16, "INCOMINGVALUE", std::bind(exec_get_param, _1, 11, "INCOMINGVALUE"))) .insert(OpcodeInstr::mksimple(0xf82c, 16, "STORAGEFEES", std::bind(exec_get_param, _1, 12, "STORAGEFEES"))) - .insert(OpcodeInstr::mksimple(0xf82d, 16, "PREVBLOCKSINFOTUPLE", std::bind(exec_get_param, _1, 13, "PREVBLOCKSINFOTUPLE"))) - .insert(OpcodeInstr::mksimple(0xf82e, 16, "UNPACKEDCONFIGTUPLE", std::bind(exec_get_param, _1, 14, "UNPACKEDCONFIGTUPLE"))) + .insert(OpcodeInstr::mksimple(0xf82d, 16, "PREVBLOCKSINFOTUPLE", + std::bind(exec_get_param, _1, 13, "PREVBLOCKSINFOTUPLE"))) + .insert(OpcodeInstr::mksimple(0xf82e, 16, "UNPACKEDCONFIGTUPLE", + std::bind(exec_get_param, _1, 14, "UNPACKEDCONFIGTUPLE"))) .insert(OpcodeInstr::mksimple(0xf82f, 16, "DUEPAYMENT", std::bind(exec_get_param, _1, 15, "DUEPAYMENT"))) .insert(OpcodeInstr::mksimple(0xf830, 16, "CONFIGDICT", exec_get_config_dict)) .insert(OpcodeInstr::mksimple(0xf832, 16, "CONFIGPARAM", std::bind(exec_get_config_param, _1, false))) .insert(OpcodeInstr::mksimple(0xf833, 16, "CONFIGOPTPARAM", std::bind(exec_get_config_param, _1, true))) - .insert(OpcodeInstr::mksimple(0xf83400, 24, "PREVMCBLOCKS", std::bind(exec_get_prev_blocks_info, _1, 0, "PREVMCBLOCKS"))->require_version(4)) - .insert(OpcodeInstr::mksimple(0xf83401, 24, "PREVKEYBLOCK", std::bind(exec_get_prev_blocks_info, _1, 1, "PREVKEYBLOCK"))->require_version(4)) - .insert(OpcodeInstr::mksimple(0xf83402, 24, "PREVMCBLOCKS_100", std::bind(exec_get_prev_blocks_info, _1, 2, "PREVMCBLOCKS_100"))->require_version(9)) + .insert(OpcodeInstr::mksimple(0xf83400, 24, "PREVMCBLOCKS", + std::bind(exec_get_prev_blocks_info, _1, 0, "PREVMCBLOCKS")) + ->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf83401, 24, "PREVKEYBLOCK", + std::bind(exec_get_prev_blocks_info, _1, 1, "PREVKEYBLOCK")) + ->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf83402, 24, "PREVMCBLOCKS_100", + std::bind(exec_get_prev_blocks_info, _1, 2, "PREVMCBLOCKS_100")) + ->require_version(9)) .insert(OpcodeInstr::mksimple(0xf835, 16, "GLOBALID", exec_get_global_id)->require_version(4)) .insert(OpcodeInstr::mksimple(0xf836, 16, "GETGASFEE", exec_get_gas_fee)->require_version(6)) .insert(OpcodeInstr::mksimple(0xf837, 16, "GETSTORAGEFEE", exec_get_storage_fee)->require_version(6)) @@ -488,21 +496,42 @@ void register_ton_config_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mkfixedrange(0xf841, 0xf860, 16, 5, instr::dump_1c_and(31, "GETGLOB "), exec_get_global)) .insert(OpcodeInstr::mksimple(0xf860, 16, "SETGLOBVAR", exec_set_global_var)) .insert(OpcodeInstr::mkfixedrange(0xf861, 0xf880, 16, 5, instr::dump_1c_and(31, "SETGLOB "), exec_set_global)) - .insert(OpcodeInstr::mksimple(0xf880, 16, "GETEXTRABALANCE", exec_get_extra_currency_balance)->require_version(10)) - .insert(OpcodeInstr::mkfixedrange(0xf88100, 0xf88111, 24, 8, instr::dump_1c_l_add(0, "GETPARAMLONG "), exec_get_var_param_long)->require_version(11)) - .insert(OpcodeInstr::mksimple(0xf88111, 24, "INMSGPARAMS", std::bind(exec_get_param, _1, 17, "INMSGPARAMS"))->require_version(11)) - .insert(OpcodeInstr::mkfixedrange(0xf88112, 0xf881ff, 24, 8, instr::dump_1c_l_add(0, "GETPARAMLONG "), exec_get_var_param_long)->require_version(11)) - .insert(OpcodeInstr::mksimple(0xf890, 16, "INMSG_BOUNCE", std::bind(exec_get_in_msg_param, _1, 0, "INMSG_BOUNCE"))->require_version(11)) - .insert(OpcodeInstr::mksimple(0xf891, 16, "INMSG_BOUNCED", std::bind(exec_get_in_msg_param, _1, 1, "INMSG_BOUNCED"))->require_version(11)) - .insert(OpcodeInstr::mksimple(0xf892, 16, "INMSG_SRC", std::bind(exec_get_in_msg_param, _1, 2, "INMSG_SRC"))->require_version(11)) - .insert(OpcodeInstr::mksimple(0xf893, 16, "INMSG_FWDFEE", std::bind(exec_get_in_msg_param, _1, 3, "INMSG_FWDFEE"))->require_version(11)) - .insert(OpcodeInstr::mksimple(0xf894, 16, "INMSG_LT", std::bind(exec_get_in_msg_param, _1, 4, "INMSG_LT"))->require_version(11)) - .insert(OpcodeInstr::mksimple(0xf895, 16, "INMSG_UTIME", std::bind(exec_get_in_msg_param, _1, 5, "INMSG_UTIME"))->require_version(11)) - .insert(OpcodeInstr::mksimple(0xf896, 16, "INMSG_ORIGVALUE", std::bind(exec_get_in_msg_param, _1, 6, "INMSG_ORIGVALUE"))->require_version(11)) - .insert(OpcodeInstr::mksimple(0xf897, 16, "INMSG_VALUE", std::bind(exec_get_in_msg_param, _1, 7, "INMSG_VALUE"))->require_version(11)) - .insert(OpcodeInstr::mksimple(0xf898, 16, "INMSG_VALUEEXTRA", std::bind(exec_get_in_msg_param, _1, 8, "INMSG_VALUEEXTRA"))->require_version(11)) - .insert(OpcodeInstr::mksimple(0xf899, 16, "INMSG_STATEINIT", std::bind(exec_get_in_msg_param, _1, 9, "INMSG_STATEINIT"))->require_version(11)) - .insert(OpcodeInstr::mkfixedrange(0xf89a, 0xf8a0, 16, 4, instr::dump_1c("INMSGPARAM "), exec_get_var_in_msg_param)->require_version(11)); + .insert( + OpcodeInstr::mksimple(0xf880, 16, "GETEXTRABALANCE", exec_get_extra_currency_balance)->require_version(10)) + .insert(OpcodeInstr::mkfixedrange(0xf88100, 0xf88111, 24, 8, instr::dump_1c_l_add(0, "GETPARAMLONG "), + exec_get_var_param_long) + ->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf88111, 24, "INMSGPARAMS", std::bind(exec_get_param, _1, 17, "INMSGPARAMS")) + ->require_version(11)) + .insert(OpcodeInstr::mkfixedrange(0xf88112, 0xf881ff, 24, 8, instr::dump_1c_l_add(0, "GETPARAMLONG "), + exec_get_var_param_long) + ->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf890, 16, "INMSG_BOUNCE", std::bind(exec_get_in_msg_param, _1, 0, "INMSG_BOUNCE")) + ->require_version(11)) + .insert( + OpcodeInstr::mksimple(0xf891, 16, "INMSG_BOUNCED", std::bind(exec_get_in_msg_param, _1, 1, "INMSG_BOUNCED")) + ->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf892, 16, "INMSG_SRC", std::bind(exec_get_in_msg_param, _1, 2, "INMSG_SRC")) + ->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf893, 16, "INMSG_FWDFEE", std::bind(exec_get_in_msg_param, _1, 3, "INMSG_FWDFEE")) + ->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf894, 16, "INMSG_LT", std::bind(exec_get_in_msg_param, _1, 4, "INMSG_LT")) + ->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf895, 16, "INMSG_UTIME", std::bind(exec_get_in_msg_param, _1, 5, "INMSG_UTIME")) + ->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf896, 16, "INMSG_ORIGVALUE", + std::bind(exec_get_in_msg_param, _1, 6, "INMSG_ORIGVALUE")) + ->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf897, 16, "INMSG_VALUE", std::bind(exec_get_in_msg_param, _1, 7, "INMSG_VALUE")) + ->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf898, 16, "INMSG_VALUEEXTRA", + std::bind(exec_get_in_msg_param, _1, 8, "INMSG_VALUEEXTRA")) + ->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf899, 16, "INMSG_STATEINIT", + std::bind(exec_get_in_msg_param, _1, 9, "INMSG_STATEINIT")) + ->require_version(11)) + .insert(OpcodeInstr::mkfixedrange(0xf89a, 0xf8a0, 16, 4, instr::dump_1c("INMSGPARAM "), exec_get_var_in_msg_param) + ->require_version(11)); } td::RefInt256 generate_randu256(VmState* st) { @@ -1364,28 +1393,43 @@ void register_ton_crypto_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xf910, 16, "CHKSIGNU", std::bind(exec_ed25519_check_signature, _1, false))) .insert(OpcodeInstr::mksimple(0xf911, 16, "CHKSIGNS", std::bind(exec_ed25519_check_signature, _1, true))) .insert(OpcodeInstr::mksimple(0xf912, 16, "ECRECOVER", exec_ecrecover)->require_version(4)) - .insert(OpcodeInstr::mksimple(0xf913, 16, "SECP256K1_XONLY_PUBKEY_TWEAK_ADD", exec_secp256k1_xonly_pubkey_tweak_add)->require_version(9)) - .insert(OpcodeInstr::mksimple(0xf914, 16, "P256_CHKSIGNU", std::bind(exec_p256_chksign, _1, false))->require_version(4)) - .insert(OpcodeInstr::mksimple(0xf915, 16, "P256_CHKSIGNS", std::bind(exec_p256_chksign, _1, true))->require_version(4)) + .insert( + OpcodeInstr::mksimple(0xf913, 16, "SECP256K1_XONLY_PUBKEY_TWEAK_ADD", exec_secp256k1_xonly_pubkey_tweak_add) + ->require_version(9)) + .insert(OpcodeInstr::mksimple(0xf914, 16, "P256_CHKSIGNU", std::bind(exec_p256_chksign, _1, false)) + ->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf915, 16, "P256_CHKSIGNS", std::bind(exec_p256_chksign, _1, true)) + ->require_version(4)) .insert(OpcodeInstr::mksimple(0xf916, 16, "HASHBU", std::bind(exec_compute_hash, _1, 2))->require_version(12)) .insert(OpcodeInstr::mksimple(0xf920, 16, "RIST255_FROMHASH", exec_ristretto255_from_hash)->require_version(4)) - .insert(OpcodeInstr::mksimple(0xf921, 16, "RIST255_VALIDATE", std::bind(exec_ristretto255_validate, _1, false))->require_version(4)) - .insert(OpcodeInstr::mksimple(0xf922, 16, "RIST255_ADD", std::bind(exec_ristretto255_add, _1, false))->require_version(4)) - .insert(OpcodeInstr::mksimple(0xf923, 16, "RIST255_SUB", std::bind(exec_ristretto255_sub, _1, false))->require_version(4)) - .insert(OpcodeInstr::mksimple(0xf924, 16, "RIST255_MUL", std::bind(exec_ristretto255_mul, _1, false))->require_version(4)) - .insert(OpcodeInstr::mksimple(0xf925, 16, "RIST255_MULBASE", std::bind(exec_ristretto255_mul_base, _1, false))->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf921, 16, "RIST255_VALIDATE", std::bind(exec_ristretto255_validate, _1, false)) + ->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf922, 16, "RIST255_ADD", std::bind(exec_ristretto255_add, _1, false)) + ->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf923, 16, "RIST255_SUB", std::bind(exec_ristretto255_sub, _1, false)) + ->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf924, 16, "RIST255_MUL", std::bind(exec_ristretto255_mul, _1, false)) + ->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf925, 16, "RIST255_MULBASE", std::bind(exec_ristretto255_mul_base, _1, false)) + ->require_version(4)) .insert(OpcodeInstr::mksimple(0xf926, 16, "RIST255_PUSHL", exec_ristretto255_push_l)->require_version(4)) - .insert(OpcodeInstr::mksimple(0xb7f921, 24, "RIST255_QVALIDATE", std::bind(exec_ristretto255_validate, _1, true))->require_version(4)) - .insert(OpcodeInstr::mksimple(0xb7f922, 24, "RIST255_QADD", std::bind(exec_ristretto255_add, _1, true))->require_version(4)) - .insert(OpcodeInstr::mksimple(0xb7f923, 24, "RIST255_QSUB", std::bind(exec_ristretto255_sub, _1, true))->require_version(4)) - .insert(OpcodeInstr::mksimple(0xb7f924, 24, "RIST255_QMUL", std::bind(exec_ristretto255_mul, _1, true))->require_version(4)) - .insert(OpcodeInstr::mksimple(0xb7f925, 24, "RIST255_QMULBASE", std::bind(exec_ristretto255_mul_base, _1, true))->require_version(4)) + .insert(OpcodeInstr::mksimple(0xb7f921, 24, "RIST255_QVALIDATE", std::bind(exec_ristretto255_validate, _1, true)) + ->require_version(4)) + .insert(OpcodeInstr::mksimple(0xb7f922, 24, "RIST255_QADD", std::bind(exec_ristretto255_add, _1, true)) + ->require_version(4)) + .insert(OpcodeInstr::mksimple(0xb7f923, 24, "RIST255_QSUB", std::bind(exec_ristretto255_sub, _1, true)) + ->require_version(4)) + .insert(OpcodeInstr::mksimple(0xb7f924, 24, "RIST255_QMUL", std::bind(exec_ristretto255_mul, _1, true)) + ->require_version(4)) + .insert(OpcodeInstr::mksimple(0xb7f925, 24, "RIST255_QMULBASE", std::bind(exec_ristretto255_mul_base, _1, true)) + ->require_version(4)) .insert(OpcodeInstr::mksimple(0xf93000, 24, "BLS_VERIFY", exec_bls_verify)->require_version(4)) .insert(OpcodeInstr::mksimple(0xf93001, 24, "BLS_AGGREGATE", exec_bls_aggregate)->require_version(4)) - .insert(OpcodeInstr::mksimple(0xf93002, 24, "BLS_FASTAGGREGATEVERIFY", exec_bls_fast_aggregate_verify)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93002, 24, "BLS_FASTAGGREGATEVERIFY", exec_bls_fast_aggregate_verify) + ->require_version(4)) .insert(OpcodeInstr::mksimple(0xf93003, 24, "BLS_AGGREGATEVERIFY", exec_bls_aggregate_verify)->require_version(4)) .insert(OpcodeInstr::mksimple(0xf93010, 24, "BLS_G1_ADD", exec_bls_g1_add)->require_version(4)) @@ -1535,13 +1579,13 @@ bool skip_message_addr(CellSlice& cs, int global_version) { bool skip_std_message_addr(CellSlice& cs, int global_version) { switch ((unsigned)cs.fetch_ulong(2)) { - case 2: { // addr_std$10 - return skip_maybe_anycast(cs, global_version) // anycast:(Maybe Anycast) - && cs.advance(8 + 256); // workchain_id:int8 address:bits256 = MsgAddressInt; + case 2: { // addr_std$10 + return skip_maybe_anycast(cs, global_version) // anycast:(Maybe Anycast) + && cs.advance(8 + 256); // workchain_id:int8 address:bits256 = MsgAddressInt; } - case 0: // addr_none$00 = MsgAddressExt; - case 1: // addr_extern$01 - case 3: // addr_var$11 + case 0: // addr_none$00 = MsgAddressExt; + case 1: // addr_extern$01 + case 3: // addr_var$11 default: return false; } @@ -1600,7 +1644,7 @@ int exec_load_opt_std_message_addr(VmState* st, bool quiet) { } auto tag = csr.write().prefetch_ulong(2); - if (tag == 0b00) { // addr_none$00 + if (tag == 0b00) { // addr_none$00 csr.write().skip_first(2); // addr_none -> push null stack.push_null(); @@ -1808,7 +1852,8 @@ bool is_valid_std_msg_addr(const Ref& cs, int global_version) { return false; } - return cs->size() == 3 + 8 + 256 && cs->size_refs() == 0; // anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; + return cs->size() == 3 + 8 + 256 && + cs->size_refs() == 0; // anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; } // Fallback to copy cell slice and check with anycast @@ -1930,14 +1975,22 @@ void register_ton_currency_address_ops(OpcodeTable& cp0) { OpcodeInstr::mksimple(0xfa46, 16, "REWRITEVARADDR", std::bind(exec_rewrite_message_addr, _1, true, false))) .insert( OpcodeInstr::mksimple(0xfa47, 16, "REWRITEVARADDRQ", std::bind(exec_rewrite_message_addr, _1, true, true))) - .insert(OpcodeInstr::mksimple(0xfa48, 16, "LDSTDADDR", std::bind(exec_load_std_message_addr, _1, false))->require_version(12)) - .insert(OpcodeInstr::mksimple(0xfa49, 16, "LDSTDADDRQ", std::bind(exec_load_std_message_addr, _1, true))->require_version(12)) - .insert(OpcodeInstr::mksimple(0xfa50, 16, "LDOPTSTDADDR", std::bind(exec_load_opt_std_message_addr, _1, false))->require_version(12)) - .insert(OpcodeInstr::mksimple(0xfa51, 16, "LDOPTSTDADDRQ", std::bind(exec_load_opt_std_message_addr, _1, true))->require_version(12)) - .insert(OpcodeInstr::mksimple(0xfa52, 16, "STSTDADDR", std::bind(exec_store_std_address, _1, false))->require_version(12)) - .insert(OpcodeInstr::mksimple(0xfa53, 16, "STSTDADDRQ", std::bind(exec_store_std_address, _1, true))->require_version(12)) - .insert(OpcodeInstr::mksimple(0xfa54, 16, "STOPTSTDADDR", std::bind(exec_store_opt_std_address, _1, false))->require_version(12)) - .insert(OpcodeInstr::mksimple(0xfa55, 16, "STOPTSTDADDRQ", std::bind(exec_store_opt_std_address, _1, true))->require_version(12)); + .insert(OpcodeInstr::mksimple(0xfa48, 16, "LDSTDADDR", std::bind(exec_load_std_message_addr, _1, false)) + ->require_version(12)) + .insert(OpcodeInstr::mksimple(0xfa49, 16, "LDSTDADDRQ", std::bind(exec_load_std_message_addr, _1, true)) + ->require_version(12)) + .insert(OpcodeInstr::mksimple(0xfa50, 16, "LDOPTSTDADDR", std::bind(exec_load_opt_std_message_addr, _1, false)) + ->require_version(12)) + .insert(OpcodeInstr::mksimple(0xfa51, 16, "LDOPTSTDADDRQ", std::bind(exec_load_opt_std_message_addr, _1, true)) + ->require_version(12)) + .insert(OpcodeInstr::mksimple(0xfa52, 16, "STSTDADDR", std::bind(exec_store_std_address, _1, false)) + ->require_version(12)) + .insert(OpcodeInstr::mksimple(0xfa53, 16, "STSTDADDRQ", std::bind(exec_store_std_address, _1, true)) + ->require_version(12)) + .insert(OpcodeInstr::mksimple(0xfa54, 16, "STOPTSTDADDR", std::bind(exec_store_opt_std_address, _1, false)) + ->require_version(12)) + .insert(OpcodeInstr::mksimple(0xfa55, 16, "STOPTSTDADDRQ", std::bind(exec_store_opt_std_address, _1, true)) + ->require_version(12)); } static constexpr int output_actions_idx = 5; @@ -1977,7 +2030,7 @@ int parse_addr_workchain(CellSlice cs) { throw VmError{Excno::range_chk, "not an internal MsgAddress"}; } bool is_var = cs.fetch_ulong(1); - if (cs.fetch_ulong(1) == 1) { // Anycast + if (cs.fetch_ulong(1) == 1) { // Anycast unsigned depth; cs.fetch_uint_leq(30, depth); cs.skip_first(depth); @@ -2019,7 +2072,7 @@ int exec_send_message(VmState* st) { unsigned extra_flags_len = 0; bool have_extra_currencies = false; bool ext_msg = msg.info->prefetch_ulong(1); - if (ext_msg) { // External message + if (ext_msg) { // External message block::gen::CommonMsgInfoRelaxed::Record_ext_out_msg_info info; if (!tlb::csr_unpack(msg.info, info)) { throw VmError{Excno::unknown, "invalid message"}; @@ -2027,7 +2080,7 @@ int exec_send_message(VmState* st) { ihr_disabled = true; dest = std::move(info.dest); value = user_fwd_fee = user_ihr_fee = td::zero_refint(); - } else { // Internal message + } else { // Internal message block::gen::CommonMsgInfoRelaxed::Record_int_msg_info info; if (!tlb::csr_unpack(msg.info, info)) { throw VmError{Excno::unknown, "invalid message"}; @@ -2130,11 +2183,11 @@ int exec_send_message(VmState* st) { td::uint64 bits = stat.bits; auto compute_fees = [&]() { td::uint64 fwd_fee_short = prices.lump_price + td::uint128(prices.bit_price) - .mult(bits) - .add(td::uint128(prices.cell_price).mult(cells)) - .add(td::uint128(0xffffu)) - .shr(16) - .lo(); + .mult(bits) + .add(td::uint128(prices.cell_price).mult(cells)) + .add(td::uint128(0xffffu)) + .shr(16) + .lo(); td::uint64 ihr_fee_short; if (ihr_disabled) { ihr_fee_short = 0; diff --git a/crypto/vm/tonops.h b/crypto/vm/tonops.h index a5597202da..b522ae3a76 100644 --- a/crypto/vm/tonops.h +++ b/crypto/vm/tonops.h @@ -17,8 +17,9 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "vm/vm.h" #include "ton/ton-types.h" +#include "vm/vm.h" + #include "mc-config.h" namespace vm { diff --git a/crypto/vm/tupleops.cpp b/crypto/vm/tupleops.cpp index f4be2c76b4..c23c1dc047 100644 --- a/crypto/vm/tupleops.cpp +++ b/crypto/vm/tupleops.cpp @@ -16,11 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "vm/excno.hpp" #include "vm/log.h" -#include "vm/stackops.h" #include "vm/opctable.h" #include "vm/stack.hpp" -#include "vm/excno.hpp" +#include "vm/stackops.h" #include "vm/vm.h" namespace vm { diff --git a/crypto/vm/utils.h b/crypto/vm/utils.h index 34d761b1d2..be56bbcba3 100644 --- a/crypto/vm/utils.h +++ b/crypto/vm/utils.h @@ -17,10 +17,10 @@ Copyright 2019-2020 Telegram Systems LLP */ #pragma once -#include "stack.hpp" - #include +#include "stack.hpp" + namespace vm { td::Result> parse_stack_entries(td::Slice str, bool prefix_only = false); diff --git a/crypto/vm/vm.cpp b/crypto/vm/vm.cpp index 356f0df223..744fa3ae58 100644 --- a/crypto/vm/vm.cpp +++ b/crypto/vm/vm.cpp @@ -16,16 +16,17 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/dispatch.h" +#include + #include "vm/continuation.h" #include "vm/dict.h" +#include "vm/dispatch.h" #include "vm/log.h" #include "vm/vm.h" + #include "cp0.h" #include "memo.h" -#include - namespace vm { VmState::VmState() : cp(-1), dispatch(&dummy_dispatch_table), quit0(true, 0), quit1(true, 1) { @@ -450,7 +451,8 @@ int VmState::step() { } ++steps; if (code->size()) { - VM_LOG_MASK(this, vm::VmLog::ExecLocation) << "code cell hash: " << code->get_base_cell()->get_hash().to_hex() << " offset: " << code->cur_pos(); + VM_LOG_MASK(this, vm::VmLog::ExecLocation) + << "code cell hash: " << code->get_base_cell()->get_hash().to_hex() << " offset: " << code->cur_pos(); return dispatch->dispatch(this, code.write()); } else if (code->size_refs()) { VM_LOG(this) << "execute implicit JMPREF"; @@ -515,7 +517,7 @@ int VmState::run() { restore_parent_vm(~res); } res = run_inner(); - } catch (VmNoGas &vmoog) { + } catch (VmNoGas& vmoog) { ++steps; VM_LOG(this) << "unhandled out-of-gas exception: gas consumed=" << gas.gas_consumed() << ", limit=" << gas.gas_limit; diff --git a/crypto/vm/vm.h b/crypto/vm/vm.h index aad67d61e1..bb1647d058 100644 --- a/crypto/vm/vm.h +++ b/crypto/vm/vm.h @@ -20,13 +20,13 @@ #include "common/refcnt.hpp" #include "td/utils/HashMap.h" +#include "td/utils/HashSet.h" +#include "td/utils/optional.h" #include "vm/cellslice.h" +#include "vm/continuation.h" +#include "vm/log.h" #include "vm/stack.hpp" #include "vm/vmstate.h" -#include "vm/log.h" -#include "vm/continuation.h" -#include "td/utils/HashSet.h" -#include "td/utils/optional.h" namespace vm { @@ -101,7 +101,7 @@ class VmState final : public VmStateInterface { bool chksig_always_succeed{false}; bool stop_on_accept_message{false}; td::optional missing_library; - td::uint16 max_data_depth = 512; // Default value + td::uint16 max_data_depth = 512; // Default value int global_version{0}; size_t chksgn_counter = 0; size_t get_extra_balance_counter = 0; @@ -170,8 +170,8 @@ class VmState final : public VmStateInterface { get_extra_balance_cheap_max_gas_price = 200 }; VmState(); - VmState(Ref _code, int global_version, Ref _stack, const GasLimits& _gas, int flags = 0, Ref _data = {}, - VmLog log = {}, std::vector> _libraries = {}, Ref init_c7 = {}); + VmState(Ref _code, int global_version, Ref _stack, const GasLimits& _gas, int flags = 0, + Ref _data = {}, VmLog log = {}, std::vector> _libraries = {}, Ref init_c7 = {}); VmState(Ref _code, int global_version, Ref _stack, const GasLimits& _gas, int flags = 0, Ref _data = {}, VmLog log = {}, std::vector> _libraries = {}, Ref init_c7 = {}) : VmState(convert_code_cell(std::move(_code), global_version, _libraries), global_version, std::move(_stack), diff --git a/crypto/vm/vmstate.h b/crypto/vm/vmstate.h index 0d6c3fdf9a..9a3ee7c8f5 100644 --- a/crypto/vm/vmstate.h +++ b/crypto/vm/vmstate.h @@ -17,11 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "common/refcnt.hpp" -#include "vm/cells.h" #include "common/global-version.h" - +#include "common/refcnt.hpp" #include "td/utils/Context.h" +#include "vm/cells.h" namespace vm { using td::Ref; @@ -33,9 +32,9 @@ class VmStateInterface : public td::Context { td::ConstBitPtr hash) { // may throw a dictionary exception; returns nullptr if library is not found return {}; } - virtual void register_cell_load(const CellHash& cell_hash){}; - virtual void register_cell_create(){}; - virtual void register_new_cell(Ref& cell){}; + virtual void register_cell_load(const CellHash& cell_hash) {}; + virtual void register_cell_create() {}; + virtual void register_new_cell(Ref& cell) {}; virtual bool register_op(int op_units = 1) { return true; }; diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index c8fc4508d2..b3f8ef86a3 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -25,29 +25,29 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "dht-server.hpp" - -#include "td/utils/filesystem.h" +#include "memprof/memprof.h" #include "td/actor/MultiPromise.h" -#include "td/utils/overloaded.h" #include "td/utils/OptionParser.h" -#include "td/utils/port/path.h" -#include "td/utils/port/user.h" -#include "td/utils/port/signals.h" +#include "td/utils/Random.h" #include "td/utils/ThreadSafeCounter.h" #include "td/utils/TsFileLog.h" -#include "td/utils/Random.h" +#include "td/utils/filesystem.h" +#include "td/utils/overloaded.h" +#include "td/utils/port/path.h" +#include "td/utils/port/signals.h" +#include "td/utils/port/user.h" -#include "memprof/memprof.h" +#include "dht-server.hpp" #if TD_DARWIN || TD_LINUX #include #endif #include -#include -#include #include +#include #include +#include + #include "git.h" Config::Config() { @@ -1225,24 +1225,24 @@ int main(int argc, char *argv[]) { td::log_interface = logger_.get(); }); td::uint32 threads = 7; - p.add_checked_option( - 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice arg) { - td::int32 v; - try { - v = std::stoi(arg.str()); - } catch (...) { - return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: not a number"); - } - if (v <= 0) { - return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be > 0"); - } - if (v > 127) { - LOG(WARNING) << "`--threads " << v << "` is too big, effective value will be 127"; - v = 127; - } - threads = v; - return td::Status::OK(); - }); + p.add_checked_option('t', "threads", PSTRING() << "number of threads (default=" << threads << ")", + [&](td::Slice arg) { + td::int32 v; + try { + v = std::stoi(arg.str()); + } catch (...) { + return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: not a number"); + } + if (v <= 0) { + return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be > 0"); + } + if (v > 127) { + LOG(WARNING) << "`--threads " << v << "` is too big, effective value will be 127"; + v = 127; + } + threads = v; + return td::Status::OK(); + }); p.add_checked_option('u', "user", "change user", [&](td::Slice user) { return td::change_user(user.str()); }); p.run(argc, argv).ensure(); diff --git a/dht-server/dht-server.hpp b/dht-server/dht-server.hpp index 7c9e561948..4853ce9ae9 100644 --- a/dht-server/dht-server.hpp +++ b/dht-server/dht-server.hpp @@ -25,19 +25,16 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include +#include + #include "adnl/adnl.h" +#include "auto/tl/ton_api.hpp" +#include "auto/tl/ton_api_json.h" #include "dht/dht.h" - #include "td/actor/MultiPromise.h" - #include "ton/ton-types.h" -#include "auto/tl/ton_api_json.h" -#include "auto/tl/ton_api.hpp" - -#include -#include - enum DhtServerPermissions : td::uint32 { vep_default = 1, vep_modify = 2, vep_unsafe = 4 }; using AdnlCategory = td::int8; diff --git a/dht/dht-bucket.cpp b/dht/dht-bucket.cpp index f60f9961c7..3e1351e8c3 100644 --- a/dht/dht-bucket.cpp +++ b/dht/dht-bucket.cpp @@ -16,19 +16,16 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/tl_storers.h" -#include "td/utils/crypto.h" -#include "td/utils/tl_parsers.h" +#include "auto/tl/ton_api.hpp" +#include "keys/encryptor.h" #include "td/utils/Random.h" - +#include "td/utils/crypto.h" #include "td/utils/format.h" +#include "td/utils/tl_parsers.h" +#include "td/utils/tl_storers.h" -#include "keys/encryptor.h" - -#include "auto/tl/ton_api.hpp" - -#include "dht-remote-node.hpp" #include "dht-bucket.hpp" +#include "dht-remote-node.hpp" #include "dht.hpp" //#include diff --git a/dht/dht-bucket.hpp b/dht/dht-bucket.hpp index e12fe6a470..5b9af8c0c2 100644 --- a/dht/dht-bucket.hpp +++ b/dht/dht-bucket.hpp @@ -18,11 +18,11 @@ */ #pragma once +#include + #include "dht-remote-node.hpp" #include "dht.hpp" -#include - namespace ton { namespace dht { diff --git a/dht/dht-in.hpp b/dht/dht-in.hpp index 2f4e70416f..742704a3bf 100644 --- a/dht/dht-in.hpp +++ b/dht/dht-in.hpp @@ -18,10 +18,11 @@ */ #pragma once -#include "dht.hpp" +#include + #include "td/db/KeyValueAsync.h" -#include +#include "dht.hpp" namespace ton { diff --git a/dht/dht-node.cpp b/dht/dht-node.cpp index 409e3f6802..a3f255789e 100644 --- a/dht/dht-node.cpp +++ b/dht/dht-node.cpp @@ -16,9 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "dht-node.hpp" #include "keys/encryptor.h" +#include "dht-node.hpp" + namespace ton { namespace dht { diff --git a/dht/dht-node.hpp b/dht/dht-node.hpp index d5860f91b1..e6be7eb077 100644 --- a/dht/dht-node.hpp +++ b/dht/dht-node.hpp @@ -18,13 +18,13 @@ */ #pragma once -#include "adnl/adnl-node-id.hpp" #include "adnl/adnl-address-list.hpp" - -#include "dht-types.h" +#include "adnl/adnl-node-id.hpp" #include "auto/tl/ton_api.hpp" #include "td/utils/overloaded.h" +#include "dht-types.h" + namespace ton { namespace dht { @@ -39,10 +39,12 @@ class DhtNode { public: DhtNode() = default; - DhtNode(adnl::AdnlNodeIdFull id, adnl::AdnlAddressList addr_list, td::int32 version, td::int32 network_id, td::BufferSlice signature) + DhtNode(adnl::AdnlNodeIdFull id, adnl::AdnlAddressList addr_list, td::int32 version, td::int32 network_id, + td::BufferSlice signature) : id_(id), addr_list_(addr_list), version_(version), network_id_(network_id), signature_(signature.as_slice()) { } - DhtNode(adnl::AdnlNodeIdFull id, adnl::AdnlAddressList addr_list, td::int32 version, td::int32 network_id, td::SharedSlice signature) + DhtNode(adnl::AdnlNodeIdFull id, adnl::AdnlAddressList addr_list, td::int32 version, td::int32 network_id, + td::SharedSlice signature) : id_(id), addr_list_(addr_list), version_(version), network_id_(network_id), signature_(std::move(signature)) { } static td::Result create(tl_object_ptr obj, td::int32 our_network_id) { diff --git a/dht/dht-query.cpp b/dht/dht-query.cpp index 3d43b10699..949c648520 100644 --- a/dht/dht-query.cpp +++ b/dht/dht-query.cpp @@ -16,18 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "dht.hpp" - -#include "td/utils/tl_storers.h" -#include "td/utils/crypto.h" +#include "auto/tl/ton_api.hpp" #include "td/utils/Random.h" -#include "td/utils/overloaded.h" - +#include "td/utils/crypto.h" #include "td/utils/format.h" - -#include "auto/tl/ton_api.hpp" +#include "td/utils/overloaded.h" +#include "td/utils/tl_storers.h" #include "dht-query.hpp" +#include "dht.hpp" namespace ton { @@ -216,9 +213,7 @@ void DhtQueryFindValue::on_result(td::Result R, adnl::AdnlNodeI need_stop = true; } }, - [&](ton_api::dht_valueNotFound &v) { - add_nodes(DhtNodesList{std::move(v.nodes_), our_network_id()}); - })); + [&](ton_api::dht_valueNotFound &v) { add_nodes(DhtNodesList{std::move(v.nodes_), our_network_id()}); })); if (need_stop) { stop(); } else if (send_get_nodes) { diff --git a/dht/dht-query.hpp b/dht/dht-query.hpp index e305f107a1..0689441330 100644 --- a/dht/dht-query.hpp +++ b/dht/dht-query.hpp @@ -18,16 +18,14 @@ */ #pragma once -#include #include - -#include "td/utils/int_types.h" -#include "td/actor/actor.h" +#include #include "adnl/adnl.h" -#include "td/actor/PromiseFuture.h" - #include "auto/tl/ton_api.hpp" +#include "td/actor/PromiseFuture.h" +#include "td/actor/actor.h" +#include "td/utils/int_types.h" #include "dht.hpp" @@ -44,8 +42,8 @@ class DhtQuery : public td::actor::Actor { bool client_only_; public: - DhtQuery(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, td::uint32 k, - td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only, td::actor::ActorId node, + DhtQuery(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, td::uint32 k, td::uint32 a, + td::int32 our_network_id, DhtNode self, bool client_only, td::actor::ActorId node, td::actor::ActorId adnl) : key_(key) , self_(std::move(self)) @@ -167,9 +165,9 @@ class DhtQueryFindValueSingle : public DhtQueryFindValue { class DhtQueryFindValueMany : public DhtQueryFindValue { public: DhtQueryFindValueMany(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list, - td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only, - td::actor::ActorId node, td::actor::ActorId adnl, - std::function callback, td::Promise promise) + td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only, + td::actor::ActorId node, td::actor::ActorId adnl, + std::function callback, td::Promise promise) : DhtQueryFindValue(key, print_id, src, std::move(list), k, a, our_network_id, std::move(self), client_only, node, adnl) , callback_(std::move(callback)) @@ -205,8 +203,8 @@ class DhtQueryStore : public td::actor::Actor { public: DhtQueryStore(DhtValue key_value, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list, - td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only, td::actor::ActorId node, - td::actor::ActorId adnl, td::Promise promise); + td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only, + td::actor::ActorId node, td::actor::ActorId adnl, td::Promise promise); void send_stores(td::Result res); void store_ready(td::Result res); void start_up() override; diff --git a/dht/dht-remote-node.cpp b/dht/dht-remote-node.cpp index 1273750c16..6c4b6e5af1 100644 --- a/dht/dht-remote-node.cpp +++ b/dht/dht-remote-node.cpp @@ -16,17 +16,14 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "dht.hpp" - -#include "td/utils/tl_storers.h" -#include "td/utils/crypto.h" +#include "auto/tl/ton_api.hpp" #include "td/utils/Random.h" - +#include "td/utils/crypto.h" #include "td/utils/format.h" - -#include "auto/tl/ton_api.hpp" +#include "td/utils/tl_storers.h" #include "dht-remote-node.hpp" +#include "dht.hpp" namespace ton { diff --git a/dht/dht-remote-node.hpp b/dht/dht-remote-node.hpp index 8e7db48939..a7df84c094 100644 --- a/dht/dht-remote-node.hpp +++ b/dht/dht-remote-node.hpp @@ -18,16 +18,14 @@ */ #pragma once -#include "td/utils/int_types.h" -#include "td/actor/actor.h" - #include "adnl/adnl.h" #include "adnl/utils.hpp" +#include "auto/tl/ton_api.hpp" +#include "td/actor/actor.h" +#include "td/utils/int_types.h" #include "dht.hpp" -#include "auto/tl/ton_api.hpp" - namespace ton { namespace dht { diff --git a/dht/dht-types.cpp b/dht/dht-types.cpp index 118df2a80a..176ab1c44b 100644 --- a/dht/dht-types.cpp +++ b/dht/dht-types.cpp @@ -16,14 +16,14 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "dht-types.h" -#include "td/utils/Random.h" -#include "td/utils/overloaded.h" -#include "keys/encryptor.h" +#include #include "auto/tl/ton_api.hpp" +#include "keys/encryptor.h" +#include "td/utils/Random.h" +#include "td/utils/overloaded.h" -#include +#include "dht-types.h" namespace ton { diff --git a/dht/dht-types.h b/dht/dht-types.h index 45657d456c..ba7e3efedb 100644 --- a/dht/dht-types.h +++ b/dht/dht-types.h @@ -18,11 +18,11 @@ */ #pragma once -#include "td/utils/int_types.h" -#include "td/utils/SharedSlice.h" #include "adnl/adnl-node-id.hpp" -#include "tl-utils/tl-utils.hpp" #include "common/io.hpp" +#include "td/utils/SharedSlice.h" +#include "td/utils/int_types.h" +#include "tl-utils/tl-utils.hpp" namespace ton { diff --git a/dht/dht.cpp b/dht/dht.cpp index da3b336010..82910c13c7 100644 --- a/dht/dht.cpp +++ b/dht/dht.cpp @@ -16,23 +16,19 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "dht.hpp" - -#include "td/utils/tl_storers.h" -#include "td/utils/crypto.h" +#include "auto/tl/ton_api.hpp" +#include "td/db/RocksDb.h" #include "td/utils/Random.h" #include "td/utils/base64.h" - +#include "td/utils/crypto.h" #include "td/utils/format.h" +#include "td/utils/tl_storers.h" -#include "td/db/RocksDb.h" - -#include "auto/tl/ton_api.hpp" - -#include "dht.h" #include "dht-bucket.hpp" -#include "dht-query.hpp" #include "dht-in.hpp" +#include "dht-query.hpp" +#include "dht.h" +#include "dht.hpp" namespace ton { @@ -469,8 +465,9 @@ void DhtMemberImpl::get_value_in(DhtKeyId key, td::Promise result) { network_id = network_id_, id = id_, client_only = client_only_](td::Result R) mutable { R.ensure(); - td::actor::create_actor("FindValueQuery", key, print_id, id, std::move(list), k, a, network_id, - R.move_as_ok(), client_only, SelfId, adnl, std::move(promise)) + td::actor::create_actor("FindValueQuery", key, print_id, id, std::move(list), k, a, + network_id, R.move_as_ok(), client_only, SelfId, adnl, + std::move(promise)) .release(); }); diff --git a/dht/dht.h b/dht/dht.h index 5abff94a10..c5b5837b18 100644 --- a/dht/dht.h +++ b/dht/dht.h @@ -18,15 +18,11 @@ */ #pragma once -#include "td/utils/int_types.h" - -#include "td/actor/PromiseFuture.h" - -#include "td/actor/actor.h" - #include "adnl/adnl.h" - #include "auto/tl/ton_api.hpp" +#include "td/actor/PromiseFuture.h" +#include "td/actor/actor.h" +#include "td/utils/int_types.h" #include "dht-types.h" diff --git a/dht/dht.hpp b/dht/dht.hpp index 9fb05e0820..ef26813e05 100644 --- a/dht/dht.hpp +++ b/dht/dht.hpp @@ -20,19 +20,16 @@ #include -#include "td/utils/int_types.h" -#include "td/actor/actor.h" -#include "td/utils/List.h" - #include "adnl/adnl.h" - #include "adnl/utils.hpp" +#include "auto/tl/ton_api.hpp" #include "keys/encryptor.h" +#include "td/actor/actor.h" +#include "td/utils/List.h" +#include "td/utils/int_types.h" -#include "dht.h" #include "dht-node.hpp" - -#include "auto/tl/ton_api.hpp" +#include "dht.h" namespace ton { diff --git a/dht/utils/dht-ping-servers.cpp b/dht/utils/dht-ping-servers.cpp index 0feafb9a7d..f01008a5a3 100644 --- a/dht/utils/dht-ping-servers.cpp +++ b/dht/utils/dht-ping-servers.cpp @@ -25,22 +25,22 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include + #include "adnl/adnl-network-manager.h" #include "adnl/adnl.h" #include "adnl/utils.hpp" -#include "keys/encryptor.h" -#include "td/utils/Time.h" -#include "td/utils/format.h" -#include "td/utils/OptionParser.h" -#include "td/utils/filesystem.h" -#include "dht/dht.hpp" #include "auto/tl/ton_api_json.h" #include "common/delay.h" +#include "dht/dht.hpp" +#include "keys/encryptor.h" +#include "td/utils/OptionParser.h" #include "td/utils/Random.h" +#include "td/utils/Time.h" +#include "td/utils/filesystem.h" +#include "td/utils/format.h" #include "terminal/terminal.h" -#include - class AdnlNode : public td::actor::Actor { private: td::actor::ActorOwn network_manager_; @@ -136,8 +136,8 @@ class AdnlNode : public td::actor::Actor { adnl_, &ton::adnl::Adnl::send_query, local_id_, id, "ping", [SelfId = actor_id(this), i, timer = td::Timer()](td::Result R) { td::actor::send_closure(SelfId, &AdnlNode::on_pong, i, timer.elapsed(), R.is_ok()); - }, td::Timestamp::in(5.0), - ton::create_serialize_tl_object(td::Random::fast_uint64())); + }, + td::Timestamp::in(5.0), ton::create_serialize_tl_object(td::Random::fast_uint64())); } if (pings_remaining_ == 0) { @@ -168,7 +168,7 @@ class AdnlNode : public td::actor::Actor { return; } td::TerminalIO::out() << "Pinged " << nodes_.size() << " nodes:\n"; - for (const auto& node : nodes_) { + for (const auto &node : nodes_) { td::TerminalIO::out() << node.id << " : " << node.received << "/" << node.sent; if (node.received > 0) { td::TerminalIO::out() << " (avg. time = " << node.sum_time / node.received << ")"; diff --git a/dht/utils/dht-resolve.cpp b/dht/utils/dht-resolve.cpp index a66846b1eb..615e1d4999 100644 --- a/dht/utils/dht-resolve.cpp +++ b/dht/utils/dht-resolve.cpp @@ -25,22 +25,22 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include + #include "adnl/adnl-network-manager.h" #include "adnl/adnl.h" #include "adnl/utils.hpp" -#include "keys/encryptor.h" -#include "td/utils/Time.h" -#include "td/utils/format.h" -#include "td/utils/OptionParser.h" -#include "td/utils/filesystem.h" -#include "dht/dht.hpp" #include "auto/tl/ton_api_json.h" #include "common/delay.h" +#include "common/util.h" +#include "dht/dht.hpp" +#include "keys/encryptor.h" +#include "td/utils/OptionParser.h" #include "td/utils/Random.h" +#include "td/utils/Time.h" +#include "td/utils/filesystem.h" +#include "td/utils/format.h" #include "terminal/terminal.h" -#include "common/util.h" - -#include class Resolver : public td::actor::Actor { private: @@ -132,7 +132,7 @@ class Resolver : public td::actor::Actor { if (!conf.dht_) { return td::Status::Error(ton::ErrorCode::error, "does not contain [dht] section"); } - ton::ton_api::dht_nodes* static_nodes = nullptr; + ton::ton_api::dht_nodes *static_nodes = nullptr; ton::ton_api::downcast_call(*conf.dht_, [&](auto &f) { static_nodes = f.static_nodes_.get(); }); auto &nodes = static_nodes->nodes_; if (server_idx_ >= 0) { diff --git a/emulator/StringLog.h b/emulator/StringLog.h index 724182c425..46c456a81c 100644 --- a/emulator/StringLog.h +++ b/emulator/StringLog.h @@ -1,9 +1,10 @@ #ifndef TON_STRINGLOG_H #define TON_STRINGLOG_H -#include "td/utils/logging.h" #include +#include "td/utils/logging.h" + class StringLog : public td::LogInterface { public: StringLog() { diff --git a/emulator/emulator-emscripten.cpp b/emulator/emulator-emscripten.cpp index 7b8e9dd1ea..a60d9411c0 100644 --- a/emulator/emulator-emscripten.cpp +++ b/emulator/emulator-emscripten.cpp @@ -1,11 +1,13 @@ -#include "emulator-extern.h" -#include "td/utils/logging.h" +#include + +#include "crypto/common/bitstring.h" #include "td/utils/JsonBuilder.h" +#include "td/utils/logging.h" #include "td/utils/misc.h" #include "td/utils/optional.h" + #include "StringLog.h" -#include -#include "crypto/common/bitstring.h" +#include "emulator-extern.h" struct TransactionEmulationParams { uint32_t utime; @@ -23,7 +25,7 @@ td::Result decode_transaction_emulation_params(const std::string json_str(json); TRY_RESULT(input_json, td::json_decode(td::MutableSlice(json_str))); - auto &obj = input_json.get_object(); + auto& obj = input_json.get_object(); TRY_RESULT(utime_field, td::get_json_object_field(obj, "utime", td::JsonValue::Type::Number, false)); TRY_RESULT(utime, td::to_integer_safe(utime_field.get_number())); @@ -83,7 +85,7 @@ td::Result decode_get_method_params(const char* json) { std::string json_str(json); TRY_RESULT(input_json, td::json_decode(td::MutableSlice(json_str))); - auto &obj = input_json.get_object(); + auto& obj = input_json.get_object(); TRY_RESULT(code, td::get_json_object_string_field(obj, "code", false)); params.code = code; @@ -123,7 +125,7 @@ td::Result decode_get_method_params(const char* json) { td::StringBuilder ec_builder; auto& ec_obj = ec_field.get_object(); bool is_first = true; - for (auto &field_value : ec_obj) { + for (auto& field_value : ec_obj) { auto currency_id = field_value.first; if (field_value.second.type() != td::JsonValue::Type::String) { return td::Status::Error(PSLICE() << "EC amount must be of type String"); @@ -171,130 +173,130 @@ class NoopLog : public td::LogInterface { extern "C" { -void* create_emulator(const char *config, int verbosity) { - NoopLog logger; +void* create_emulator(const char* config, int verbosity) { + NoopLog logger; - td::log_interface = &logger; + td::log_interface = &logger; - SET_VERBOSITY_LEVEL(verbosity_NEVER); - return transaction_emulator_create(config, verbosity); + SET_VERBOSITY_LEVEL(verbosity_NEVER); + return transaction_emulator_create(config, verbosity); } void destroy_emulator(void* em) { - NoopLog logger; + NoopLog logger; - td::log_interface = &logger; + td::log_interface = &logger; - SET_VERBOSITY_LEVEL(verbosity_NEVER); - transaction_emulator_destroy(em); + SET_VERBOSITY_LEVEL(verbosity_NEVER); + transaction_emulator_destroy(em); } -const char *emulate_with_emulator(void* em, const char* libs, const char* account, const char* message, const char* params) { - StringLog logger; +const char* emulate_with_emulator(void* em, const char* libs, const char* account, const char* message, + const char* params) { + StringLog logger; - td::log_interface = &logger; - SET_VERBOSITY_LEVEL(verbosity_DEBUG); + td::log_interface = &logger; + SET_VERBOSITY_LEVEL(verbosity_DEBUG); - auto decoded_params_res = decode_transaction_emulation_params(params); - if (decoded_params_res.is_error()) { - return strdup(R"({"fail":true,"message":"Can't decode other params"})"); - } - auto decoded_params = decoded_params_res.move_as_ok(); + auto decoded_params_res = decode_transaction_emulation_params(params); + if (decoded_params_res.is_error()) { + return strdup(R"({"fail":true,"message":"Can't decode other params"})"); + } + auto decoded_params = decoded_params_res.move_as_ok(); - bool rand_seed_set = true; - if (decoded_params.rand_seed_hex) { - rand_seed_set = transaction_emulator_set_rand_seed(em, decoded_params.rand_seed_hex.unwrap().c_str()); - } + bool rand_seed_set = true; + if (decoded_params.rand_seed_hex) { + rand_seed_set = transaction_emulator_set_rand_seed(em, decoded_params.rand_seed_hex.unwrap().c_str()); + } - bool prev_blocks_set = true; - if (decoded_params.prev_blocks_info) { - prev_blocks_set = transaction_emulator_set_prev_blocks_info(em, decoded_params.prev_blocks_info.unwrap().c_str()); - } + bool prev_blocks_set = true; + if (decoded_params.prev_blocks_info) { + prev_blocks_set = transaction_emulator_set_prev_blocks_info(em, decoded_params.prev_blocks_info.unwrap().c_str()); + } - if (!transaction_emulator_set_libs(em, libs) || - !transaction_emulator_set_lt(em, decoded_params.lt) || - !transaction_emulator_set_unixtime(em, decoded_params.utime) || - !transaction_emulator_set_ignore_chksig(em, decoded_params.ignore_chksig) || - !transaction_emulator_set_debug_enabled(em, decoded_params.debug_enabled) || - !rand_seed_set || - !prev_blocks_set) { - transaction_emulator_destroy(em); - return strdup(R"({"fail":true,"message":"Can't set params"})"); - } + if (!transaction_emulator_set_libs(em, libs) || !transaction_emulator_set_lt(em, decoded_params.lt) || + !transaction_emulator_set_unixtime(em, decoded_params.utime) || + !transaction_emulator_set_ignore_chksig(em, decoded_params.ignore_chksig) || + !transaction_emulator_set_debug_enabled(em, decoded_params.debug_enabled) || !rand_seed_set || !prev_blocks_set) { + transaction_emulator_destroy(em); + return strdup(R"({"fail":true,"message":"Can't set params"})"); + } - const char *result; - if (decoded_params.is_tick_tock) { - result = transaction_emulator_emulate_tick_tock_transaction(em, account, decoded_params.is_tock); - } else { - result = transaction_emulator_emulate_transaction(em, account, message); - } + const char* result; + if (decoded_params.is_tick_tock) { + result = transaction_emulator_emulate_tick_tock_transaction(em, account, decoded_params.is_tock); + } else { + result = transaction_emulator_emulate_transaction(em, account, message); + } - const char* output = nullptr; - { - td::JsonBuilder jb; - auto json_obj = jb.enter_object(); - json_obj("output", td::JsonRaw(td::Slice(result))); - json_obj("logs", logger.get_string()); - json_obj.leave(); - output = strdup(jb.string_builder().as_cslice().c_str()); - } - free((void*) result); + const char* output = nullptr; + { + td::JsonBuilder jb; + auto json_obj = jb.enter_object(); + json_obj("output", td::JsonRaw(td::Slice(result))); + json_obj("logs", logger.get_string()); + json_obj.leave(); + output = strdup(jb.string_builder().as_cslice().c_str()); + } + free((void*)result); - return output; + return output; } -const char *emulate(const char *config, const char* libs, int verbosity, const char* account, const char* message, const char* params) { - auto em = transaction_emulator_create(config, verbosity); - auto result = emulate_with_emulator(em, libs, account, message, params); - transaction_emulator_destroy(em); - return result; +const char* emulate(const char* config, const char* libs, int verbosity, const char* account, const char* message, + const char* params) { + auto em = transaction_emulator_create(config, verbosity); + auto result = emulate_with_emulator(em, libs, account, message, params); + transaction_emulator_destroy(em); + return result; } -const char *run_get_method(const char *params, const char* stack, const char* config) { - StringLog logger; +const char* run_get_method(const char* params, const char* stack, const char* config) { + StringLog logger; - td::log_interface = &logger; - SET_VERBOSITY_LEVEL(verbosity_DEBUG); + td::log_interface = &logger; + SET_VERBOSITY_LEVEL(verbosity_DEBUG); - auto decoded_params_res = decode_get_method_params(params); - if (decoded_params_res.is_error()) { - return strdup(R"({"fail":true,"message":"Can't decode params"})"); - } - auto decoded_params = decoded_params_res.move_as_ok(); - - auto tvm = tvm_emulator_create(decoded_params.code.c_str(), decoded_params.data.c_str(), decoded_params.verbosity); - - if ((decoded_params.libs && !tvm_emulator_set_libraries(tvm, decoded_params.libs.value().c_str())) || - !tvm_emulator_set_c7(tvm, decoded_params.address.c_str(), decoded_params.unixtime, decoded_params.balance, - decoded_params.rand_seed_hex.c_str(), config) || - (decoded_params.extra_currencies.size() > 0 && !tvm_emulator_set_extra_currencies(tvm, decoded_params.extra_currencies.c_str())) || - (decoded_params.prev_blocks_info && !tvm_emulator_set_prev_blocks_info(tvm, decoded_params.prev_blocks_info.value().c_str())) || - (decoded_params.gas_limit > 0 && !tvm_emulator_set_gas_limit(tvm, decoded_params.gas_limit)) || - !tvm_emulator_set_debug_enabled(tvm, decoded_params.debug_enabled)) { - tvm_emulator_destroy(tvm); - return strdup(R"({"fail":true,"message":"Can't set params"})"); - } + auto decoded_params_res = decode_get_method_params(params); + if (decoded_params_res.is_error()) { + return strdup(R"({"fail":true,"message":"Can't decode params"})"); + } + auto decoded_params = decoded_params_res.move_as_ok(); + + auto tvm = tvm_emulator_create(decoded_params.code.c_str(), decoded_params.data.c_str(), decoded_params.verbosity); + + if ((decoded_params.libs && !tvm_emulator_set_libraries(tvm, decoded_params.libs.value().c_str())) || + !tvm_emulator_set_c7(tvm, decoded_params.address.c_str(), decoded_params.unixtime, decoded_params.balance, + decoded_params.rand_seed_hex.c_str(), config) || + (decoded_params.extra_currencies.size() > 0 && + !tvm_emulator_set_extra_currencies(tvm, decoded_params.extra_currencies.c_str())) || + (decoded_params.prev_blocks_info && + !tvm_emulator_set_prev_blocks_info(tvm, decoded_params.prev_blocks_info.value().c_str())) || + (decoded_params.gas_limit > 0 && !tvm_emulator_set_gas_limit(tvm, decoded_params.gas_limit)) || + !tvm_emulator_set_debug_enabled(tvm, decoded_params.debug_enabled)) { + tvm_emulator_destroy(tvm); + return strdup(R"({"fail":true,"message":"Can't set params"})"); + } - auto res = tvm_emulator_run_get_method(tvm, decoded_params.method_id, stack); + auto res = tvm_emulator_run_get_method(tvm, decoded_params.method_id, stack); - tvm_emulator_destroy(tvm); + tvm_emulator_destroy(tvm); - const char* output = nullptr; - { - td::JsonBuilder jb; - auto json_obj = jb.enter_object(); - json_obj("output", td::JsonRaw(td::Slice(res))); - json_obj("logs", logger.get_string()); - json_obj.leave(); - output = strdup(jb.string_builder().as_cslice().c_str()); - } - free((void*) res); + const char* output = nullptr; + { + td::JsonBuilder jb; + auto json_obj = jb.enter_object(); + json_obj("output", td::JsonRaw(td::Slice(res))); + json_obj("logs", logger.get_string()); + json_obj.leave(); + output = strdup(jb.string_builder().as_cslice().c_str()); + } + free((void*)res); - return output; + return output; } -const char *version() { +const char* version() { return emulator_version(); } - } \ No newline at end of file diff --git a/emulator/emulator-extern.cpp b/emulator/emulator-extern.cpp index 2addd353cb..b69e14229e 100644 --- a/emulator/emulator-extern.cpp +++ b/emulator/emulator-extern.cpp @@ -1,15 +1,16 @@ -#include "emulator-extern.h" -#include "td/utils/base64.h" -#include "td/utils/Status.h" +#include "crypto/vm/memo.h" +#include "crypto/vm/stack.hpp" #include "td/utils/JsonBuilder.h" -#include "td/utils/logging.h" +#include "td/utils/Status.h" #include "td/utils/Variant.h" +#include "td/utils/base64.h" +#include "td/utils/logging.h" #include "td/utils/overloaded.h" + +#include "emulator-extern.h" +#include "git.h" #include "transaction-emulator.h" #include "tvm-emulator.hpp" -#include "crypto/vm/stack.hpp" -#include "crypto/vm/memo.h" -#include "git.h" td::Result> boc_b64_to_cell(const char *boc) { TRY_RESULT_PREFIX(boc_decoded, td::base64_decode(td::Slice(boc)), "Can't decode base64 boc: "); @@ -17,12 +18,13 @@ td::Result> boc_b64_to_cell(const char *boc) { } td::Result cell_to_boc_b64(td::Ref cell) { - TRY_RESULT_PREFIX(boc, vm::std_boc_serialize(std::move(cell), vm::BagOfCells::Mode::WithCRC32C), "Can't serialize cell: "); + TRY_RESULT_PREFIX(boc, vm::std_boc_serialize(std::move(cell), vm::BagOfCells::Mode::WithCRC32C), + "Can't serialize cell: "); return td::base64_encode(boc.as_slice()); } -const char *success_response(std::string&& transaction, std::string&& new_shard_account, std::string&& vm_log, - td::optional&& actions, double elapsed_time) { +const char *success_response(std::string &&transaction, std::string &&new_shard_account, std::string &&vm_log, + td::optional &&actions, double elapsed_time) { td::JsonBuilder jb; auto json_obj = jb.enter_object(); json_obj("success", td::JsonTrue()); @@ -39,7 +41,7 @@ const char *success_response(std::string&& transaction, std::string&& new_shard_ return strdup(jb.string_builder().as_cslice().c_str()); } -const char *error_response(std::string&& error) { +const char *error_response(std::string &&error) { td::JsonBuilder jb; auto json_obj = jb.enter_object(); json_obj("success", td::JsonFalse()); @@ -49,7 +51,7 @@ const char *error_response(std::string&& error) { return strdup(jb.string_builder().as_cslice().c_str()); } -const char *external_not_accepted_response(std::string&& vm_log, int vm_exit_code, double elapsed_time) { +const char *external_not_accepted_response(std::string &&vm_log, int vm_exit_code, double elapsed_time) { td::JsonBuilder jb; auto json_obj = jb.enter_object(); json_obj("success", td::JsonFalse()); @@ -64,7 +66,7 @@ const char *external_not_accepted_response(std::string&& vm_log, int vm_exit_cod #define ERROR_RESPONSE(error) return error_response(error) -td::Result decode_config(const char* config_boc) { +td::Result decode_config(const char *config_boc) { TRY_RESULT_PREFIX(config_params_cell, boc_b64_to_cell(config_boc), "Can't deserialize config params boc: "); auto config_dict = std::make_unique(config_params_cell, 32); auto config_addr_cell = config_dict->lookup_ref(td::BitArray<32>::zero()); @@ -77,7 +79,9 @@ td::Result decode_config(const char* config_boc) { } ton::StdSmcAddress config_addr; config_addr_cs.fetch_bits_to(config_addr); - auto global_config = block::Config(config_params_cell, std::move(config_addr), block::Config::needWorkchainInfo | block::Config::needSpecialSmc | block::Config::needCapabilities); + auto global_config = + block::Config(config_params_cell, std::move(config_addr), + block::Config::needWorkchainInfo | block::Config::needSpecialSmc | block::Config::needCapabilities); TRY_STATUS_PREFIX(global_config.unpack(), "Can't unpack config params: "); return global_config; } @@ -101,9 +105,10 @@ void *emulator_config_create(const char *config_params_boc) { return new block::Config(config.move_as_ok()); } -const char *transaction_emulator_emulate_transaction(void *transaction_emulator, const char *shard_account_boc, const char *message_boc) { +const char *transaction_emulator_emulate_transaction(void *transaction_emulator, const char *shard_account_boc, + const char *message_boc) { auto emulator = static_cast(transaction_emulator); - + auto message_cell_r = boc_b64_to_cell(message_boc); if (message_cell_r.is_error()) { ERROR_RESPONSE(PSTRING() << "Can't deserialize message boc: " << message_cell_r.move_as_error()); @@ -129,14 +134,13 @@ const char *transaction_emulator_emulate_transaction(void *transaction_emulator, if (msg_tag == block::gen::CommonMsgInfo::ext_in_msg_info) { block::gen::CommonMsgInfo::Record_ext_in_msg_info info; if (!tlb::unpack(message_cs, info)) { - ERROR_RESPONSE(PSTRING() << "Can't unpack inbound external message"); + ERROR_RESPONSE(PSTRING() << "Can't unpack inbound external message"); } addr_slice = std::move(info.dest); - } - else if (msg_tag == block::gen::CommonMsgInfo::int_msg_info) { + } else if (msg_tag == block::gen::CommonMsgInfo::int_msg_info) { block::gen::CommonMsgInfo::Record_int_msg_info info; if (!tlb::unpack(message_cs, info)) { - ERROR_RESPONSE(PSTRING() << "Can't unpack inbound internal message"); + ERROR_RESPONSE(PSTRING() << "Can't unpack inbound internal message"); } addr_slice = std::move(info.dest); } else { @@ -175,27 +179,32 @@ const char *transaction_emulator_emulate_transaction(void *transaction_emulator, account.last_trans_hash_ = shard_account.last_trans_hash; } - auto result = emulator->emulate_transaction(std::move(account), message_cell, now, 0, block::transaction::Transaction::tr_ord); + auto result = + emulator->emulate_transaction(std::move(account), message_cell, now, 0, block::transaction::Transaction::tr_ord); if (result.is_error()) { ERROR_RESPONSE(PSTRING() << "Emulate transaction failed: " << result.move_as_error()); } auto emulation_result = result.move_as_ok(); - auto external_not_accepted = dynamic_cast(emulation_result.get()); + auto external_not_accepted = + dynamic_cast(emulation_result.get()); if (external_not_accepted) { - return external_not_accepted_response(std::move(external_not_accepted->vm_log), external_not_accepted->vm_exit_code, + return external_not_accepted_response(std::move(external_not_accepted->vm_log), external_not_accepted->vm_exit_code, external_not_accepted->elapsed_time); } - auto emulation_success = std::move(dynamic_cast(*emulation_result)); + auto emulation_success = + std::move(dynamic_cast(*emulation_result)); auto trans_boc_b64 = cell_to_boc_b64(std::move(emulation_success.transaction)); if (trans_boc_b64.is_error()) { ERROR_RESPONSE(PSTRING() << "Can't serialize Transaction to boc " << trans_boc_b64.move_as_error()); } - auto new_shard_account_cell = vm::CellBuilder().store_ref(emulation_success.account.total_state) - .store_bits(emulation_success.account.last_trans_hash_.as_bitslice()) - .store_long(emulation_success.account.last_trans_lt_).finalize(); + auto new_shard_account_cell = vm::CellBuilder() + .store_ref(emulation_success.account.total_state) + .store_bits(emulation_success.account.last_trans_hash_.as_bitslice()) + .store_long(emulation_success.account.last_trans_lt_) + .finalize(); auto new_shard_account_boc_b64 = cell_to_boc_b64(std::move(new_shard_account_cell)); if (new_shard_account_boc_b64.is_error()) { ERROR_RESPONSE(PSTRING() << "Can't serialize ShardAccount to boc " << new_shard_account_boc_b64.move_as_error()); @@ -205,18 +214,21 @@ const char *transaction_emulator_emulate_transaction(void *transaction_emulator, if (emulation_success.actions.not_null()) { auto actions_boc_b64_result = cell_to_boc_b64(std::move(emulation_success.actions)); if (actions_boc_b64_result.is_error()) { - ERROR_RESPONSE(PSTRING() << "Can't serialize actions list cell to boc " << actions_boc_b64_result.move_as_error()); + ERROR_RESPONSE(PSTRING() << "Can't serialize actions list cell to boc " + << actions_boc_b64_result.move_as_error()); } actions_boc_b64 = actions_boc_b64_result.move_as_ok(); } - return success_response(trans_boc_b64.move_as_ok(), new_shard_account_boc_b64.move_as_ok(), std::move(emulation_success.vm_log), - std::move(actions_boc_b64), emulation_success.elapsed_time); + return success_response(trans_boc_b64.move_as_ok(), new_shard_account_boc_b64.move_as_ok(), + std::move(emulation_success.vm_log), std::move(actions_boc_b64), + emulation_success.elapsed_time); } -const char *transaction_emulator_emulate_tick_tock_transaction(void *transaction_emulator, const char *shard_account_boc, bool is_tock) { +const char *transaction_emulator_emulate_tick_tock_transaction(void *transaction_emulator, + const char *shard_account_boc, bool is_tock) { auto emulator = static_cast(transaction_emulator); - + auto shard_account_cell = boc_b64_to_cell(shard_account_boc); if (shard_account_cell.is_error()) { ERROR_RESPONSE(PSTRING() << "Can't deserialize shard account boc: " << shard_account_cell.move_as_error()); @@ -230,7 +242,7 @@ const char *transaction_emulator_emulate_tick_tock_transaction(void *transaction td::Ref addr_slice; auto account_slice = vm::load_cell_slice(shard_account.account); if (block::gen::t_Account.get_tag(account_slice) == block::gen::Account::account_none) { - ERROR_RESPONSE(PSTRING() << "Can't run tick/tock transaction on account_none"); + ERROR_RESPONSE(PSTRING() << "Can't run tick/tock transaction on account_none"); } block::gen::Account::Record_account account_record; if (!tlb::unpack(account_slice, account_record)) { @@ -267,9 +279,11 @@ const char *transaction_emulator_emulate_tick_tock_transaction(void *transaction ERROR_RESPONSE(PSTRING() << "Can't serialize Transaction to boc " << trans_boc_b64.move_as_error()); } - auto new_shard_account_cell = vm::CellBuilder().store_ref(emulation_success.account.total_state) - .store_bits(emulation_success.account.last_trans_hash_.as_bitslice()) - .store_long(emulation_success.account.last_trans_lt_).finalize(); + auto new_shard_account_cell = vm::CellBuilder() + .store_ref(emulation_success.account.total_state) + .store_bits(emulation_success.account.last_trans_hash_.as_bitslice()) + .store_long(emulation_success.account.last_trans_lt_) + .finalize(); auto new_shard_account_boc_b64 = cell_to_boc_b64(std::move(new_shard_account_cell)); if (new_shard_account_boc_b64.is_error()) { ERROR_RESPONSE(PSTRING() << "Can't serialize ShardAccount to boc " << new_shard_account_boc_b64.move_as_error()); @@ -279,13 +293,15 @@ const char *transaction_emulator_emulate_tick_tock_transaction(void *transaction if (emulation_success.actions.not_null()) { auto actions_boc_b64_result = cell_to_boc_b64(std::move(emulation_success.actions)); if (actions_boc_b64_result.is_error()) { - ERROR_RESPONSE(PSTRING() << "Can't serialize actions list cell to boc " << actions_boc_b64_result.move_as_error()); + ERROR_RESPONSE(PSTRING() << "Can't serialize actions list cell to boc " + << actions_boc_b64_result.move_as_error()); } actions_boc_b64 = actions_boc_b64_result.move_as_ok(); } - return success_response(trans_boc_b64.move_as_ok(), new_shard_account_boc_b64.move_as_ok(), std::move(emulation_success.vm_log), - std::move(actions_boc_b64), emulation_success.elapsed_time); + return success_response(trans_boc_b64.move_as_ok(), new_shard_account_boc_b64.move_as_ok(), + std::move(emulation_success.vm_log), std::move(actions_boc_b64), + emulation_success.elapsed_time); } bool transaction_emulator_set_unixtime(void *transaction_emulator, uint32_t unixtime) { @@ -304,7 +320,7 @@ bool transaction_emulator_set_lt(void *transaction_emulator, uint64_t lt) { return true; } -bool transaction_emulator_set_rand_seed(void *transaction_emulator, const char* rand_seed_hex) { +bool transaction_emulator_set_rand_seed(void *transaction_emulator, const char *rand_seed_hex) { auto emulator = static_cast(transaction_emulator); auto rand_seed_hex_slice = td::Slice(rand_seed_hex); @@ -332,7 +348,7 @@ bool transaction_emulator_set_ignore_chksig(void *transaction_emulator, bool ign return true; } -bool transaction_emulator_set_config(void *transaction_emulator, const char* config_boc) { +bool transaction_emulator_set_config(void *transaction_emulator, const char *config_boc) { auto emulator = static_cast(transaction_emulator); auto global_config_res = decode_config(config_boc); @@ -346,21 +362,21 @@ bool transaction_emulator_set_config(void *transaction_emulator, const char* con return true; } -void config_deleter(block::Config* ptr) { - // We do not delete the config object, since ownership management is delegated to the caller +void config_deleter(block::Config *ptr) { + // We do not delete the config object, since ownership management is delegated to the caller } -bool transaction_emulator_set_config_object(void *transaction_emulator, void* config) { +bool transaction_emulator_set_config_object(void *transaction_emulator, void *config) { auto emulator = static_cast(transaction_emulator); - + std::shared_ptr config_ptr(static_cast(config), config_deleter); - + emulator->set_config(config_ptr); return true; } -bool transaction_emulator_set_libs(void *transaction_emulator, const char* shardchain_libs_boc) { +bool transaction_emulator_set_libs(void *transaction_emulator, const char *shardchain_libs_boc) { auto emulator = static_cast(transaction_emulator); if (shardchain_libs_boc != nullptr) { @@ -383,7 +399,7 @@ bool transaction_emulator_set_debug_enabled(void *transaction_emulator, bool deb return true; } -bool transaction_emulator_set_prev_blocks_info(void *transaction_emulator, const char* info_boc) { +bool transaction_emulator_set_prev_blocks_info(void *transaction_emulator, const char *info_boc) { auto emulator = static_cast(transaction_emulator); if (info_boc != nullptr) { @@ -454,14 +470,15 @@ bool tvm_emulator_set_libraries(void *tvm_emulator, const char *libs_boc) { return true; } -bool tvm_emulator_set_c7(void *tvm_emulator, const char *address, uint32_t unixtime, uint64_t balance, const char *rand_seed_hex, const char *config_boc) { +bool tvm_emulator_set_c7(void *tvm_emulator, const char *address, uint32_t unixtime, uint64_t balance, + const char *rand_seed_hex, const char *config_boc) { auto emulator = static_cast(tvm_emulator); auto std_address = block::StdAddress::parse(td::Slice(address)); if (std_address.is_error()) { LOG(ERROR) << "Can't parse address: " << std_address.move_as_error(); return false; } - + std::shared_ptr global_config; if (config_boc != nullptr) { auto config_params_cell = boc_b64_to_cell(config_boc); @@ -492,8 +509,9 @@ bool tvm_emulator_set_c7(void *tvm_emulator, const char *address, uint32_t unixt td::BitArray<256> rand_seed; rand_seed.as_slice().copy_from(rand_seed_bytes.move_as_ok()); - emulator->set_c7(std_address.move_as_ok(), unixtime, balance, rand_seed, std::const_pointer_cast(global_config)); - + emulator->set_c7(std_address.move_as_ok(), unixtime, balance, rand_seed, + std::const_pointer_cast(global_config)); + return true; } @@ -503,8 +521,8 @@ bool tvm_emulator_set_extra_currencies(void *tvm_emulator, const char *extra_cur td::Slice extra_currencies_str{extra_currencies}; while (true) { auto next_space_pos = extra_currencies_str.find(' '); - auto currency_id_amount = next_space_pos == td::Slice::npos ? - extra_currencies_str.substr(0) : extra_currencies_str.substr(0, next_space_pos); + auto currency_id_amount = next_space_pos == td::Slice::npos ? extra_currencies_str.substr(0) + : extra_currencies_str.substr(0, next_space_pos); if (!currency_id_amount.empty()) { auto delim_pos = currency_id_amount.find('='); @@ -550,14 +568,14 @@ bool tvm_emulator_set_extra_currencies(void *tvm_emulator, const char *extra_cur return true; } -bool tvm_emulator_set_config_object(void* tvm_emulator, void* config) { +bool tvm_emulator_set_config_object(void *tvm_emulator, void *config) { auto emulator = static_cast(tvm_emulator); auto global_config = std::shared_ptr(static_cast(config), config_deleter); emulator->set_config(global_config); return true; } -bool tvm_emulator_set_prev_blocks_info(void *tvm_emulator, const char* info_boc) { +bool tvm_emulator_set_prev_blocks_info(void *tvm_emulator, const char *info_boc) { auto emulator = static_cast(tvm_emulator); if (info_boc != nullptr) { @@ -604,15 +622,15 @@ const char *tvm_emulator_run_get_method(void *tvm_emulator, int method_id, const auto stack_cs = vm::load_cell_slice(stack_cell.move_as_ok()); td::Ref stack; if (!vm::Stack::deserialize_to(stack_cs, stack)) { - ERROR_RESPONSE(PSTRING() << "Couldn't deserialize stack"); + ERROR_RESPONSE(PSTRING() << "Couldn't deserialize stack"); } auto emulator = static_cast(tvm_emulator); auto result = emulator->run_get_method(method_id, stack); - + vm::FakeVmStateLimits fstate(3500); // limit recursive (de)serialization calls vm::VmStateInterface::Guard guard(&fstate); - + vm::CellBuilder stack_cb; if (!result.stack->serialize(stack_cb)) { ERROR_RESPONSE(PSTRING() << "Couldn't serialize stack"); @@ -639,8 +657,7 @@ const char *tvm_emulator_run_get_method(void *tvm_emulator, int method_id, const return strdup(jb.string_builder().as_cslice().c_str()); } -struct TvmEulatorEmulateRunMethodResponse -{ +struct TvmEulatorEmulateRunMethodResponse { const char *response; const char *log; }; @@ -648,7 +665,7 @@ struct TvmEulatorEmulateRunMethodResponse TvmEulatorEmulateRunMethodResponse emulate_run_method(uint32_t len, const char *params_boc, int64_t gas_limit) { auto params_cell = vm::std_boc_deserialize(td::Slice(params_boc, len)); if (params_cell.is_error()) { - return { nullptr, nullptr }; + return {nullptr, nullptr}; } auto params_cs = vm::load_cell_slice(params_cell.move_as_ok()); auto code = params_cs.fetch_ref(); @@ -663,12 +680,12 @@ TvmEulatorEmulateRunMethodResponse emulate_run_method(uint32_t len, const char * td::Ref stack; if (!vm::Stack::deserialize_to(stack_cs, stack)) { - return { nullptr, nullptr }; + return {nullptr, nullptr}; } td::Ref c7; if (!vm::Stack::deserialize_to(c7_cs, c7)) { - return { nullptr, nullptr }; + return {nullptr, nullptr}; } auto emulator = new emulator::TvmEmulator(code, data); @@ -683,7 +700,7 @@ TvmEulatorEmulateRunMethodResponse emulate_run_method(uint32_t len, const char * vm::CellBuilder stack_cb; if (!result.stack->serialize(stack_cb)) { - return { nullptr, nullptr }; + return {nullptr, nullptr}; } vm::CellBuilder cb; @@ -693,16 +710,16 @@ TvmEulatorEmulateRunMethodResponse emulate_run_method(uint32_t len, const char * auto ser = vm::std_boc_serialize(cb.finalize()); if (!ser.is_ok()) { - return { nullptr, nullptr }; + return {nullptr, nullptr}; } auto sok = ser.move_as_ok(); auto sz = uint32_t(sok.size()); - char* rn = (char*)malloc(sz + 4); + char *rn = (char *)malloc(sz + 4); memcpy(rn, &sz, 4); - memcpy(rn+4, sok.data(), sz); + memcpy(rn + 4, sok.data(), sz); - return { rn, strdup(result.vm_log.data()) }; + return {rn, strdup(result.vm_log.data())}; } const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc, int64_t gas_limit) { @@ -717,8 +734,8 @@ void *tvm_emulator_emulate_run_method_detailed(uint32_t len, const char *params_ void run_method_detailed_result_destroy(void *detailed_result) { auto result = static_cast(detailed_result); - free(const_cast(result->response)); - free(const_cast(result->log)); + free(const_cast(result->response)); + free(const_cast(result->log)); delete result; } @@ -802,7 +819,7 @@ void string_destroy(const char *str) { } } -const char* emulator_version() { +const char *emulator_version() { auto version_json = td::JsonBuilder(); auto obj = version_json.enter_object(); obj("emulatorLibCommitHash", GitMetadata::CommitSHA1()); diff --git a/emulator/emulator-extern.h b/emulator/emulator-extern.h index 2da7674565..8410cb1bc5 100644 --- a/emulator/emulator-extern.h +++ b/emulator/emulator-extern.h @@ -1,6 +1,7 @@ #pragma once #include + #include "emulator_export.h" #ifdef __cplusplus @@ -45,7 +46,7 @@ EMULATOR_EXPORT bool transaction_emulator_set_lt(void *transaction_emulator, uin * @param rand_seed_hex Hex string of length 64 * @return true in case of success, false in case of error */ -EMULATOR_EXPORT bool transaction_emulator_set_rand_seed(void *transaction_emulator, const char* rand_seed_hex); +EMULATOR_EXPORT bool transaction_emulator_set_rand_seed(void *transaction_emulator, const char *rand_seed_hex); /** * @brief Set ignore_chksig flag for emulation @@ -61,7 +62,7 @@ EMULATOR_EXPORT bool transaction_emulator_set_ignore_chksig(void *transaction_em * @param config_boc Base64 encoded BoC serialized Config dictionary (Hashmap 32 ^Cell) * @return true in case of success, false in case of error */ -EMULATOR_EXPORT bool transaction_emulator_set_config(void *transaction_emulator, const char* config_boc); +EMULATOR_EXPORT bool transaction_emulator_set_config(void *transaction_emulator, const char *config_boc); /** * @brief Set config for emulation @@ -69,7 +70,7 @@ EMULATOR_EXPORT bool transaction_emulator_set_config(void *transaction_emulator, * @param config Pointer to Config object * @return true in case of success, false in case of error */ -EMULATOR_EXPORT bool transaction_emulator_set_config_object(void *transaction_emulator, void* config); +EMULATOR_EXPORT bool transaction_emulator_set_config_object(void *transaction_emulator, void *config); /** * @brief Set libraries for emulation @@ -77,7 +78,7 @@ EMULATOR_EXPORT bool transaction_emulator_set_config_object(void *transaction_em * @param libs_boc Base64 encoded BoC serialized shared libraries dictionary (HashmapE 256 ^Cell). * @return true in case of success, false in case of error */ -EMULATOR_EXPORT bool transaction_emulator_set_libs(void *transaction_emulator, const char* libs_boc); +EMULATOR_EXPORT bool transaction_emulator_set_libs(void *transaction_emulator, const char *libs_boc); /** * @brief Enable or disable TVM debug primitives @@ -93,7 +94,7 @@ EMULATOR_EXPORT bool transaction_emulator_set_debug_enabled(void *transaction_em * @param info_boc Base64 encoded BoC serialized TVM tuple (VmStackValue). * @return true in case of success, false in case of error */ -EMULATOR_EXPORT bool transaction_emulator_set_prev_blocks_info(void *transaction_emulator, const char* info_boc); +EMULATOR_EXPORT bool transaction_emulator_set_prev_blocks_info(void *transaction_emulator, const char *info_boc); /** * @brief Emulate transaction @@ -117,7 +118,9 @@ EMULATOR_EXPORT bool transaction_emulator_set_prev_blocks_info(void *transaction * "elapsed_time": 0.02 * } */ -EMULATOR_EXPORT const char *transaction_emulator_emulate_transaction(void *transaction_emulator, const char *shard_account_boc, const char *message_boc); +EMULATOR_EXPORT const char *transaction_emulator_emulate_transaction(void *transaction_emulator, + const char *shard_account_boc, + const char *message_boc); /** * @brief Emulate tick tock transaction @@ -140,7 +143,9 @@ EMULATOR_EXPORT const char *transaction_emulator_emulate_transaction(void *trans * "elapsed_time": 0.02 * } */ -EMULATOR_EXPORT const char *transaction_emulator_emulate_tick_tock_transaction(void *transaction_emulator, const char *shard_account_boc, bool is_tock); +EMULATOR_EXPORT const char *transaction_emulator_emulate_tick_tock_transaction(void *transaction_emulator, + const char *shard_account_boc, + bool is_tock); /** * @brief Destroy TransactionEmulator object @@ -180,7 +185,8 @@ EMULATOR_EXPORT bool tvm_emulator_set_libraries(void *tvm_emulator, const char * * @param config Base64 encoded BoC serialized Config dictionary (Hashmap 32 ^Cell). Optional. * @return true in case of success, false in case of error */ -EMULATOR_EXPORT bool tvm_emulator_set_c7(void *tvm_emulator, const char *address, uint32_t unixtime, uint64_t balance, const char *rand_seed_hex, const char *config); +EMULATOR_EXPORT bool tvm_emulator_set_c7(void *tvm_emulator, const char *address, uint32_t unixtime, uint64_t balance, + const char *rand_seed_hex, const char *config); /** * @brief Set extra currencies balance @@ -196,7 +202,7 @@ EMULATOR_EXPORT bool tvm_emulator_set_extra_currencies(void *tvm_emulator, const * @param config Pointer to Config object * @return true in case of success, false in case of error */ -EMULATOR_EXPORT bool tvm_emulator_set_config_object(void* tvm_emulator, void* config); +EMULATOR_EXPORT bool tvm_emulator_set_config_object(void *tvm_emulator, void *config); /** * @brief Set tuple of previous blocks (13th element of c7) @@ -204,7 +210,7 @@ EMULATOR_EXPORT bool tvm_emulator_set_config_object(void* tvm_emulator, void* co * @param info_boc Base64 encoded BoC serialized TVM tuple (VmStackValue). * @return true in case of success, false in case of error */ -EMULATOR_EXPORT bool tvm_emulator_set_prev_blocks_info(void *tvm_emulator, const char* info_boc); +EMULATOR_EXPORT bool tvm_emulator_set_prev_blocks_info(void *tvm_emulator, const char *info_boc); /** * @brief Set TVM gas limit @@ -322,7 +328,8 @@ EMULATOR_EXPORT const char *tvm_emulator_send_external_message(void *tvm_emulato * "actions": "Base64 boc decoded actions cell of type (OutList n)" * } */ -EMULATOR_EXPORT const char *tvm_emulator_send_internal_message(void *tvm_emulator, const char *message_body_boc, uint64_t amount); +EMULATOR_EXPORT const char *tvm_emulator_send_internal_message(void *tvm_emulator, const char *message_body_boc, + uint64_t amount); /** * @brief Destroy TVM emulator object @@ -348,7 +355,7 @@ EMULATOR_EXPORT void string_destroy(const char *string); /** * @brief Get git commit hash and date of the library */ -EMULATOR_EXPORT const char* emulator_version(); +EMULATOR_EXPORT const char *emulator_version(); #ifdef __cplusplus } // extern "C" diff --git a/emulator/test/emulator-tests.cpp b/emulator/test/emulator-tests.cpp index 4057f58161..b107b1aed1 100644 --- a/emulator/test/emulator-tests.cpp +++ b/emulator/test/emulator-tests.cpp @@ -1,165 +1,309 @@ -#include "td/utils/tests.h" - #include "block/block-auto.h" -#include "block/block.h" #include "block/block-parse.h" - +#include "block/block.h" #include "crypto/vm/boc.h" - +#include "emulator/emulator-extern.h" +#include "smc-envelope/WalletV3.h" +#include "td/utils/JsonBuilder.h" #include "td/utils/base64.h" #include "td/utils/crypto.h" -#include "td/utils/JsonBuilder.h" - -#include "smc-envelope/WalletV3.h" - -#include "emulator/emulator-extern.h" +#include "td/utils/tests.h" // testnet config as of 27.06.24 -const char *config_boc = "te6cckICAl8AAQAANecAAAIBIAABAAICAtgAAwAEAgL1AA0ADgIBIAAFAAYCAUgCPgI/AgEgAAcACAIBSAAJAAoCASAAHgAfAgEgAGUAZgIBSAALAAwCAWoA0gDTAQFI" - "AJIBAUgAsgEDpDMADwIBbgAQABEAQDPAueB1cC0DTaIjG28I/scJsoxoIScEE9LNtuiQoYa2AgOuIAASABMBA7LwABoBASAAFAEBIAAYAQHAABUCAWoAFgAXAIm/VzGV" - "o387z8N7BhdH91LBHMMhBLu7nv21jwo9wtTSXQIBABvI0aFLnw2QbZgjMPCLRdtRHxhUyinQudg6sdiohIwgwCAAQ79oJ47o6vzJDO5wV60LQESEyBcI3zuSSKtFQIlz" - "hk86tAMBg+mbgbrrZVY0qEWL8HxF+gYzy9t5jLO50+QkJ2DWbWFHj0Qaw5TPlNDYOnY0A2VNeAnS9bZ98W8X7FTvgVqStlmABAAZAIOgCYiOTH0TnIIa0oSKjkT3CsgH" - "NUU1Iy/5E472ortANeCAAAAAAAAAAAAAAAAROiXXYZuWf8AAi5Oy+xV/i+2JL9ABA6BgABsCASAAHAAdAFur4AAAAAAHGv1JjQAAEeDul1fav9HZ8+939/IsLGZ46E5h" - "3qjR13yIrB8mcfbBAFur/////8AHGv1JjQAAEeDul1fav9HZ8+939/IsLGZ46E5h3qjR13yIrB8mcfbBAgEgACAAIQIBIAAzADQCASAAIgAjAgEgACkAKgIBIAAkACUB" - "AUgAKAEBIAAmAQEgACcAQFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVAEAzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMwBAAQEBAQEBAQEBAQEB" - "AQEBAQEBAQEBAQEBAQEBAQEBAQECASAAKwAsAQFYAC8BASAALQEBIAAuAEDv5x0Thgr6pq6ur2NvkWhIf4DxAxsL+Nk5rknT6n99oABTAf//////////////////////" - "////////////////////gAAAAIAAAAFAAQHAADACASAAMQAyABW+AAADvLNnDcFVUAAVv////7y9GpSiABACASAANQA2AgEgADcAOAIBIABCAEMCASAATgBPAgEgADkA" - "OgIBIAA+AD8BASAAOwEBIAA9AQHAADwAt9BTLudOzwABAnAAKtiftocOhhpk4QsHt8jHSWwV/O7nxvFyZKUf75zoqiN3Bfb/JZk7D9mvTw7EDHU5BlaNBz2ml2s54kRz" - "l0iBoQAAAAAP////+AAAAAAAAAAEABMaQ7msoAEBIB9IAQEgAEABASAAQQAUa0ZVPxAEO5rKAAAgAAAcIAAACWAAAAC0AAADhAEBIABEAQEgAEUAGsQAAAAGAAAAAAAA" - "AC4CA81AAEYARwIBIABVAEgAA6igAgEgAEkASgIBIABLAEwCASAATQBdAgEgAFsAXgIBIABbAFsCAUgAYQBhAQEgAFABASAAYgIBIABRAFICAtkAUwBUAgm3///wYABf" - "AGACASAAVQBWAgFiAFwAXQIBIABgAFcCAc4AYQBhAgEgAFgAWQIBIABaAF4CASAAXgBbAAFYAgEgAGEAYQIBIABeAF4AAdQAAUgAAfwCAdQAYQBhAAEgAgKRAGMAZAAq" - "NgIGAgUAD0JAAJiWgAAAAAEAAAH0ACo2BAcDBQBMS0ABMS0AAAAAAgAAA+gCASAAZwBoAgEgAHoAewIBIABpAGoCASAAcABxAgEgAGsAbAEBSABvAQEgAG0BASAAbgAM" - "AB4AHgADADFgkYTnKgAHEcN5N+CAAGteYg9IAAAB4AAIAE3QZgAAAAAAAAAAAAAAAIAAAAAAAAD6AAAAAAAAAfQAAAAAAAPQkEACASAAcgBzAgEgAHYAdwEBIAB0AQEg" - "AHUAlNEAAAAAAAAAZAAAAAAAD0JA3gAAAAAnEAAAAAAAAAAPQkAAAAAAAhYOwAAAAAAAACcQAAAAAAAmJaAAAAAABfXhAAAAAAA7msoAAJTRAAAAAAAAAGQAAAAAAACc" - "QN4AAAAAAZAAAAAAAAAAD0JAAAAAAAAPQkAAAAAAAAAnEAAAAAAAmJaAAAAAAAX14QAAAAAAO5rKAAEBIAB4AQEgAHkAUF3DAAIAAAAIAAAAEAAAwwAATiAAAYagAAJJ" - "8MMAAAPoAAATiAAAJxAAUF3DAAIAAAAIAAAAEAAAwwAehIAAmJaAATEtAMMAAABkAAATiAAAJxACAUgAfAB9AgEgAIAAgQEBIAB+AQEgAH8AQuoAAAAAAJiWgAAAAAAn" - "EAAAAAAAD0JAAAAAAYAAVVVVVQBC6gAAAAAABhqAAAAAAAGQAAAAAAAAnEAAAAABgABVVVVVAgEgAIIAgwEBWACGAQEgAIQBASAAhQAkwgEAAAD6AAAA+gAAA+gAAAAP" - "AErZAQMAAAfQAAA+gAAAAAMAAAAIAAAABAAgAAAAIAAAAAQAACcQAQHAAIcCASAAiACJAgFIAIoAiwIBagCQAJEAA9+wAgFYAIwAjQIBIACOAI8AQb7c3f6FapnFy4B4" - "QZnAdwvqMfKODXM49zeESA3vRM2QFABBvrMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM4AEG+tWede5qpBXVOzaq9SvpqBpwzTJ067Hk01rWZxT5wQ7gAQb8a" - "Yme1MOiTF+EsYXWNG8wYLwlq/ZXmR6g2PgSXaPOEegBBvzSTEofK4j4twU1E7XMbFoxvESypy3LTYwDOK8PDTfsWASsSZn08y2Z9WOsAEAAQD/////////3AAJMCAswA" - "lACVAgEgAJYAlwIBIACkAKUCASAAmACZAgEgAJ4AnwIBIACaAJsCASAAnACdAJsc46BJ4rulpzksHMZaJjfdtBExV1HRdikp9U7VlmJllrEaW2TYAFmAXnBlZIRH4Sqp" - "CbKkE6v60jyawOEYfVWJDgHg5kDaLMWq7kWQy6AAmxzjoEniuRloX7kgG9FNmRyw/AB/KERuToZdY5v8AHv9JJ8bCIKAWYBecGVkhEt/mk7tOEXbKUWuqIz/1NliY9sm" - "KNHFQimyb79WXudTIACbHOOgSeK0/SaSD6j2aEnWfmW/B7LOQBq2QiiBlnaLIzfq+J2HM0BZgF5wZWSEWPYUSh0McOyjsLL8prcsF5RNab+7jLN/5bOme1r98c8gAJsc" - "46BJ4rT4ptGRb52wRyHzhe/A8y/IQOC/W5R5aC6/l1IM4f/EgFmAXnBlZIRmDW7+WN70SpQsfX5DetODFOpW6zjCBx7cDf6E+rEipKACASAAoAChAgEgAKIAowCbHOOg" - "SeKqqZCAjJ16vfAa2GI9Dcp/I9zBTG2CwPqbx22lq00uLoBZgF5wZWSETeqWp7jqIGPuCYnPZSlQ1fMuSS4e1gF/i9uIeD8GEkNgAJsc46BJ4rugeQAFCtwRUJhvWRbx" - "smlpXTdXCio8SJSBdH/6VPCkAFmAXnBlZIRQPeE6JpjzEwkPI2mvCM1sDTcny96f2dhZ2DcBQmmywCAAmxzjoEnimDpTGClVkh/V+/mJmKVKEpdp4MvFgP5onw6saJRD" - "QApAWYBecGVkhElWAHSIgIhlXt+lUyQjmndd50temeILBd7WJwjjWBeIIACbHOOgSeKtcjPEr2gq3gMraY11K9Ikv1SPcVaj3veDWrY1o4nxKcBZgF5wZWSEabqKQLtX" - "PIkaYDaKvupB8EOxFDWpuMaJJVqafjw4h4sgAgEgAKYApwIBIACsAK0CASAAqACpAgEgAKoAqwCbHOOgSeK8POt5lMj96a3WrXWw7peFtWWh5oi9wsZqXRsrnHM4eoBZ" - "gF5wZWSEXlJk0ILG3LG9zsmxXf+r2OTayqr9FSKLBt9LJAow+aBgAJsc46BJ4qjb23m1w/0EvFl179XCQUUMk32z0kjSh+t6V2jnnqeFwFmAXnBlZIR2KWk8cqZgC06K" - "AphhfzE3VceQWtppAGEbybk06szO9KAAmxzjoEnihVEG74vb19K1l5o8WtWa0dH/gTPfytoA1LsVXR3ztfgAWYBecGVkhEVHN0AzKnDpKLX5P7Tnay/Ogc4rxeoks/yh" - "U3aWhEnGIACbHOOgSeKNl8PpsnZjGIy1CTzi01K8MhvQAEhGlzUDwj2ACC/yFUALGRulQuFOdHw2ulDcYktF860U0mFOYFaQPC7MVNbEeSsk45C9tSPgAgEgAK4ArwIB" - "IACwALEAmxzjoEnivAzuiTw+hkcXtw4XyJGYavfPayk6ehceV8FqrxrzKbQACMou1fGNuRpwF6ilPaS03+BSsz0YID1gpIkGozQp7gRFcQsyZFvVYACbHOOgSeKsoYF9" - "T9f0ArrtFxbViCRmpw2DsDzrllY35uHzP9DEosAICQwVUUQOx01jZ84Uy8ccqQ90Ml6tj5Sw14wOK055ds2sYSPy532gAJsc46BJ4piyhqkrUrk/KUOony6llV0S+DnZ" - "xDLdccZzKJ7bV+XiAAeBJKPSjdajMGMdZwRvewwnwsyc/7uHN718Pd8cHn7VQG1i9BJSeaAAmxzjoEnihY8aTVKeJnW4JHbfVPfkJwElQXxxqG94pNWmN6n9I5jABA51" - "90xtZChBtmQcmPHlOmtU6aLeZ+HBY7/jW6AMz26cNcymYyIuIAErEmZ9WOtmfXULABAAEA/////////3wACzAgLMALQAtQIBIAC2ALcCASAAxADFAgEgALgAuQIBIAC+" - "AL8CASAAugC7AgEgALwAvQCbHOOgSeK5Nyl3TF7AOD2UwhNOh+y3h9P5e0emd2zjffbNatQR1EBS4qdSDsPAZjIVSudNcsvyCAIbiOyNPYmj/MJG5lMjVLkYt4TIEDCg" - "AJsc46BJ4q0qr9PzfnnT+A41FG5Owo+9L+LsuT6PrQkuoR7XsLMzgFLioMqMr4sLf5pO7ThF2ylFrqiM/9TZYmPbJijRxUIpsm+/Vl7nUyAAmxzjoEnisgCK09re8agW" - "Ee8S6q329jm1WbZoHBHjO9oP0q3qItiAUuKgyoyviwfhKqkJsqQTq/rSPJrA4Rh9VYkOAeDmQNosxaruRZDLoACbHOOgSeKeKPVNUBZ96hhTOP8lp1kiAm2wfuT0HIxn" - "lw/0cyISP8BS4qDKjK+LGPYUSh0McOyjsLL8prcsF5RNab+7jLN/5bOme1r98c8gAgEgAMAAwQIBIADCAMMAmxzjoEnip+PTCe8vsapzyPHm88uO5qKBwt9yvn+S6aJW" - "OlcBqeDAUuKgyoyviyYNbv5Y3vRKlCx9fkN604MU6lbrOMIHHtwN/oT6sSKkoACbHOOgSeKwOTDV9phg7jYWvy7bbTD8N773bX9y1P7lxC7vtvdbvsBS4qDKjK+LDeqW" - "p7jqIGPuCYnPZSlQ1fMuSS4e1gF/i9uIeD8GEkNgAJsc46BJ4opGGis7tEqqLAW2742I2ugw5S5lFxeYpc4D9f/qbOMhwFLioMqMr4sQPeE6JpjzEwkPI2mvCM1sDTcn" - "y96f2dhZ2DcBQmmywCAAmxzjoEniqGUvGQXdvzVXTq/g3DpDkom5aqVipETXzq2o+FZdGDfAUuKgyoyviwlWAHSIgIhlXt+lUyQjmndd50temeILBd7WJwjjWBeIIAIB" - "IADGAMcCASAAzADNAgEgAMgAyQIBIADKAMsAmxzjoEnihA6ouVC73YehzpHoNBKL8q3Gp4YbwxOBhJdxpNWePHwAUuKgyoyviym6ikC7VzyJGmA2ir7qQfBDsRQ1qbjG" - "iSVamn48OIeLIACbHOOgSeKr2ACjLl9IlajrtDqvMLD+lfOMRQvmZAaL2NVDooVPYQBS4qDKjK+LHlJk0ILG3LG9zsmxXf+r2OTayqr9FSKLBt9LJAow+aBgAJsc46BJ" - "4oohDH+XJf2EoPKNkp+gv/WG2UonjUWXV+B/IvWUldUuQFLioMqMr4s2KWk8cqZgC06KAphhfzE3VceQWtppAGEbybk06szO9KAAmxzjoEnilP2IvoMbkK7LwTeBBX8u" - "dYI608SRo4nDIg7XUWQf2CYAUuKgyoyviwVHN0AzKnDpKLX5P7Tnay/Ogc4rxeoks/yhU3aWhEnGIAIBIADOAM8CASAA0ADRAJsc46BJ4qS3beCYCuu47Ohag9xU5wk6" - "/1uLtI/5NZ+VaqSyKsGdAApHFgZLFGK0fDa6UNxiS0XzrRTSYU5gVpA8LsxU1sR5KyTjkL21I+AAmxzjoEnivJI7eg6kFGx7dvMX7Xzoog/s5cwHxrcfec5z8/aP/8kA" - "CFtq86KYH4dNY2fOFMvHHKkPdDJerY+UsNeMDitOeXbNrGEj8ud9oACbHOOgSeKlwkl68jfkl6kGCq/tElh6bM85sFBPnt7exnkRJq68iQAG+mnlyjEXYzBjHWcEb3sM" - "J8LMnP+7hze9fD3fHB5+1UBtYvQSUnmgAJsc46BJ4oYswn2e5gWf+Va6NJ+K8sfz4qIHmVG2ryktqCkE9P8hQAPDhRot06toQbZkHJjx5TprVOmi3mfhwWO/41ugDM9u" - "nDXMpmMiLiABASAA1AEBIAD6AQsAtb0+sEAA1QIBIADWANcCA8H4ANgA2QID4fgA+AD5AgEgAPwA/QIBIADaANsCASAA3ADdAgEgAbgBuQIBIAGQAZECASAA3gDfAgEg" - "AOAA4QIBIADqAOsAQb7edpH5xbuqiZNqTG9H7flTOIfNiYtDxI5AH4T6G4tcVAIBIADiAOMAQb6U4RvTn2B6e+8nmlEv/eZoRz1YKr3qyDudETjcrMFgKAIBIADkAOUC" - "ASAA5gDnAgEgAOgA6QBBvgukN4cHaqlFuawJv/TGaxhU3HU2B5iu8cZPVMOseQOgAEG+K7U1xAKEqaBEZoqjpyAnvSx8Z9jfPTeAR/anR5axvmAAQb4tEpbKJaulevOY" - "XQPqlmgiMgHDU6C6X7KRxpFyzPf0YABBvjbzLj0Z1oudyhyW/QhJ0OUxRj9zEM8Y1YUI9Py3ga6gAgFqAOwA7QIBIADuAO8AQb4JmTypqySHVMVJMHWspb3xrs2Lrdy4" - "eJ+M7QxpbS4cIABBvgOb8O+4IZEUWqtnRGQ8JpMkMBocpZyk/do3d/9MYnVgAgEgAPAA8QBBvqQeZ13QP0lszxNKt380fCWuaV94vwC/bfuqmrlg1/fIAgEgAPIA8wIB" - "IAD0APUAQb4G2ph6AS/mD/+cIv4aIYm1z5jAgCW/TTDEr72ygXOP4ABBvhBZkdUWyc1zdg9Fhp9QSsWD+LSyXChKLJOiMF3rVNqgAgEgAPYA9wBBvhsYuojZc90oYnM2" - "WQ+c6cHdiTDRBD2UgxkJlbkZa+mgAEG9wBVbqgGsx1Pog5dkmDyUl4VIe1ZME2BEDY6zMNoQYsAAQb3R4obtqmXfb1H2NxdElqeDuWD4d+Y73ozNJ7dE4jGfQAIBIAHw" - "AfECASACGAIZAQPAwAD7AFWgESjR4FjxyuEAXHMvOQot+HG+D9TtSQavwKbeV09n3G92AAAAAAAAAH0QAgEgAP4A/wIBIAEcAR0CASABAAEBAgEgAR4BHwIBIAECAQMC" - "ASABEAERAgEgAQQBBQIBIAEIAQkCAWIBBgEHAEG+tp/96j2CYcuIRGkfljl5uv/Pilfg3KwCY8xwdr1JdqgAA97wAEG99o5GkuI7pwd5/g4Lt+avHh31l5WoNTndbJgd" - "dTJBicACAUgBCgELAgEgAQwBDQBBvgIKjJdXg0pHrRIfDgYLQ20dIU6mEbDa1FxtUXy9B6rgAEG+Cev2EcR/qY3lMYZ3tIojHR5s+wWySfwNg7XZgP23waACASABDgEP" - "AEG+fZGfOd+cHGx01cd8+xQAwUjfI/VrANsfVPw1jZFJhTAAQb4y2lPdHZUPm695Z+bh0Z1dcta4xXX7fl6dlc2SXOliIABBvhfW5EoZl/I8jARohetHRk6pp1y3mrXR" - "28rFYjHHtJCgAgFqARIBEwIBIAEUARUAQb4zE+Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU+KeaYLIABBvgPcWeL0jqPxd5IiX7AAYESGqFqZ7o60BjQZJwpPQP1gAgEg" - "ARYBFwBBvofANH7PG2eeTdX5Vr2ZUebxCfwJyzBCE4oriUVRU3jIAgEgARgBGQIBIAEaARsAQb4btDCZEGRAOXaB6WwVqFzYTd1zZgyp15BIuy9n029k4ABBvimf97Kd" - "WV/siLZ3qM/+nVRE+t0X0XdLsOK51DJ6WSPgAEG+CQrglDQDcC3b6lTaIr2tVPRR4RlxVAwxYNcF+6BkvaAAQb4mML93xvUT+iBDJrOfhiRGSs3vOczEy9DJAbuCb7aU" - "4AIBIAFAAUECASABYAFhAgEgASABIQIBIAE0ATUCASABIgEjAgFYAS4BLwIBIAEkASUAQb6L1UE7T5lmGOuEiyPgykuqAW0ENCaxjsi4fdzZq2D0GAICcAEmAScCASAB" - "KAEpAD+9QolK/7nMhu3MO9bzK31P7DqSFoQkLyeYP3RWz5f3KwA/vVaiOV3iXF+2BW0R7uGwqmnXP7y0cjEHibQT6v4MssECASABKgErAgV/rWABLAEtAEG96YUi7d3r" - "hTwVGwv/pocif6dNQ6DcZ3JVzvqdhFltQ0AAQb3zT7C1dlWQlR1QmfrLfaGi5Sj94Guq/gLQXakuFmoVwAA/u8n6yK+GpbUUdG9dja4DHHLGGEu5ZXb6rUHFOFMS7kAA" - "P7v3dUiUhgaZGC+mdUGyJEzagm0IMNe3d2Q1lCRBTK5AAEG+co6LJmQv3h46OSV3KsT2gWyv6MLPKOrfIXFt86dsXVACASABMAExAEG+KQF+kzAAZybpH/1z1zYof09W" - "YAAY6MbQHDj3AO9dCGACASABMgEzAEG9xJZFhUbajV1FgRPu0X8LSHY3DIBRmI4wC6uLpNG5lkAAQb3/+UXNzozn7Eb1PsCLs8NaD2VhG+9qBBlvLJG76KkTQAIBIAE2" - "ATcCASABPgE/AgEgATgBOQIBYgE8AT0AQb5l6UC6/ZmwRTHlWwthzsJcYx+8Vj2vmom9/nu617FmkAIBIAE6ATsAQb4J64Df7Vfb8/jmlGnsZByGAdCsEWA/FfWXyVEU" - "5d6CoABBvhv0Q/VEAfHxjnYRJRxb6xtGetqoO1OgjstzC/3Ok41gAEG964EWqVOQS0JWHUcxnAz6STWs7+BsROmocJCo+xmqe0AAQb3vR9oRALXcwLQPRb70F/gP7SAV" - "WqyMgCIasOqw+b47wABBvpbvxWd5+q2vJUVqR9AlbEIfdFysLR0PXGgVlBf8x5hYAEG+j9bgcxjKxRmfMrJEC6BbHTCQ+WNXqC3H+z591gZw0AgCASABQgFDAgEgAUgB" - "SQIBSAFEAUUAQb7KkreZXaSZXSPGxbgwuJddzpWJly3MFNYwALkyQcIdDABBvnLW0BTZocy0D6h48ehPtgqA0XqNxrqB86bTTks9uvuQAgEgAUYBRwBBvjYzcOXWIfyk" - "HqSDt3m92Hacz/XRoWD5F4yy0AQ/E0ogAEG+AShOVhiiJZ6Itzjs8O75CiiF+eXloz74MSVsHpPAMiACASABSgFLAgEgAVABUQIDeuABTAFNAgFYAU4BTwA/vVuDIbt9" - "1w2Z2FpLSOsyAUPo2ovei28SxaHKDSUdRz0AP71qm4D4evL40x1qJi6AGLh6oOBtxFr5bgc8Xr8jaeWRAEG+HzK7ymUhDh5PL//pLHqwaYidq3sym7hIWC32Rqol+mAA" - "Qb41DOvSox2jnjN40ZFtUSQhSJMCyEWhBRdRERRSltibIAIBIAFSAVMCASABWAFZAgFYAVQBVQIBIAFWAVcAQb3cHJ+brtBSsROnSioWNJqFxZ+5hIGX7ta5KuhleBFn" - "wABBvf/lQA5TJrGDmv6EqacNl5j6ktTzbQOEGqpl45xcekNAAEG+Nve9GdRJhn/t0fgYe7d1pkTBxa2AfiXcWeRYqE1K3yAAQb4jrXHoxDyh1ZYGBdBoQgLaScxW6pZR" - "1hEhJC8BqF+5IAIBIAFaAVsCAVgBXgFfAEG+CdErMSfFYmEK9J9XimJDXyszQjtVELtHIXQt7AvQjKACAUgBXAFdAEC9ivFB4bA7PAP0VXnTs784TO/4CoWLb1QqRdyr" - "0orLAgBAvb5z8xm2yt/HlB1G9TB2Qna4rVgzGxI/n4z3UYr3a7gAQb3f0PQO3/nU5ypuXD5/SaZboj2RhZjd5z47o7VM8AjDwABBvfGIqWXxgi7mCltWrYf4pQa2aRZP" - "FvMA8LBV1hmpauDAAgEgAWIBYwIBIAGAAYECASABZAFlAgEgAXIBcwIBIAFmAWcCAVgBcAFxAgFIAWgBaQIBIAFqAWsAQb33dj2qlHUSOf2DkiVrVwhcqy3SkE9YbBfn" - "zU07vK+uwABBvdxiQ8Yt/Lb9BztkNe9dyXuUyTOcKJRlF9BteI2LK99AAgEgAWwBbQBBvjxAsXZAtTQoMwJV27nrzNCyFum1aU1fbygeFMFuYX9gAgFIAW4BbwBBvdro" - "odCnIayUb5VXYFh23qJGAE4Oed7iqqU/L0iFAPpAAD+9QlUpU0rFnXRmWi3ZnIsFtIIm3JDSdtVPEGqGefBt/wA/vWGl+1GrGASEj3GaAizvMOXDl69yZpcU2YUtCHfG" - "jLUAQb4d/oR88TrfAGcKrMn44T3wBnbh3TWVQWr8rVq0bYTnYABBvhpY6fA3+apwMQXdpEMu8s8uFXf+625mtfciMt0dh4LgAgEgAXQBdQIBIAF4AXkAQb5d0CvPvsyC" - "ZxuTbUe5O2PtTudCwtgc3Ou4DMuX2WizEAIBSAF2AXcAQb3BrlEdo+Hw0uZZJxCgCdxWs/njs6bTHuprY7HtqNl0QABBvcSsc0L20So00ByQZ2oo0aUWf4BlreuHcpYk" - "R/C5Av7AAgEgAXoBewIBIAF+AX8CASABfAF9AEG+ErNElODwkPB+KvEKqCtCz8CS5HCcsC8/VoJGV5f0+uAAQb3FCW/Cy20jtvAS0j4k9eQvRg9tcpaQgFnHc5cB7Fdv" - "wABBvc5nMn9h2c6FeqzonvA74SwaTxZXTgLEXOKOIFOki9BAAEG+NkNRDvICKDQNaqBlpx1LnSn5qpShA00BPg8Tfv+LHaAAQb4+0zsN9j+Lxs1EvbGG0fMwbeeqbWlx" - "TzyjV4LE+0uJYAIBIAGCAYMCAUgBigGLAgEgAYQBhQIBIAGGAYcAQb5O+6O6Y7dWb4HOnMBK4fZ7QNo9woEzBIeKd5+K08xlkABBvlwlLor18dZ5/O3AomXxI5hxYM4o" - "J1Xrrx0JChLVxHpQAgFYAYgBiQBBvn9hAM+g43TTR8vOvZfnhX3kPBCgPp3T0+YF+Ai6RFHwAEG99KmZCgwzysLzIR2TNaJdbyX4lKduOMlCmhCp4L9gJEAAQb3Ntnmm" - "W4yzmAdiAYg7sNjoD8sCiWIvgvkpuYpTXcyiQAIBZgGMAY0CAW4BjgGPAEC9hzviVxD170gIZfsWPGFKfbOB6LCP5YhH7I7fWz7wdwBAvaey9kbu3gkPDYYEraB8b3sF" - "UrCgg4ask3C+O8UJ1mkAQL2wAL6FGQaCTbDdEwGUJ82TDpVMLoNr4ZGZWxcofghZAEC9lqzgehIXoMRj58vAWaHnNAi6UXEU5Ce942dJqf4HawIBIAGSAZMCASABqAGp" - "AgEgAZQBlQIBIAGkAaUCASABlgGXAgEgAZwBnQIBagGYAZkCASABmgGbAEC9syAieemf3vF3umY0lCaQxLhwvbTFuL8eQxPYrpeZ8ABAvbl6reyIsCKH2fq2I8+oEnkS" - "4xYy3RUH/7ka152WrisAQb4CJHgAcs+wQzgf/9IPKdknw/ej0Z+Q+n3BtSEKi0hIoABBvgqovnD/owP5nsA4G62765H5klOyA1TV+7jriGf2CtjgAgFYAZ4BnwIBIAGg" - "AaEAQb3dAG8Nta3/iYiTymgGxV0CfKQlN6UlidHeNgbvtMT9wABBve7An2cFgShRoZx3xA7hUDRtwbcLae0x4dPQQlAH8o3AAEG+HDeG9ZNvkzq3wDDpGt0cb5cHHFQ0" - "itHD3s5R2YHy8eACAWIBogGjAD+9ewqjet2JVaCzHa8NXfnW3ZtLEzEASpk9eicyztCrvwA/vXDzaFNMjF1BnqMojulsIHfT2Dj1ltCTVvoe8wu+GKcCASABpgGnAEG+" - "un2oV7CbmRhYGc7tLiCXj/L40+4ZlzvlmEnZPxyuQrgAQb5ElmikSUchX0lT+0ASVhwF0OBnUB8X4TD4m4/v2Dfl0ABBvlBR7mcUQO8IfN+DkkDYHF1reSJZhv08w6k+" - "JIA6ITiwAgEgAaoBqwIBIAG0AbUCAVgBrAGtAgEgAbIBswBBvhX0m4apMW/GEDxtnd+z0ug75voHd+OibSQbA2+tUPigAgEgAa4BrwIBWAGwAbEAQb3WKikPb9a/J2ti" - "V6yOhNUW5BivimV3gM+EI3VAxst6QAA/vUeSH4ZL+7V8eQBEF/0lm/ouIJ+wQs5QTzBpsSHSXLcAP71t4YT+jYHLpx5Gv3HFoOzL5rhg0Ukud8G3adF8AYlRAEG+Zf0n" - "TrwaPPTPlLjegNsGkoz7UV5wz7oYQet9+SNmRfAAQb5m0tqyXFYp4ntucDLTwJV1gxwoh6JoJL1Y0rfwfLQhUABBvqSCHVak+jIc9ANutTAfHpZNM3YdGky7yaDzsTrg" - "0WhIAgN9eAG2AbcAP70AGCAXHtaQJNqiST0rNTs8mUZSo5H6vM7gvA+3q7+iAD+9FgzFlOZUrfRtonCQzjDSFzrRv4l/94TFs9oi+RQ6kgIBIAG6AbsCASAB1gHXAgEg" - "AbwBvQIBIAHKAcsCASABvgG/AgEgAcQBxQBBvqg93lUVxmlCEks5kL8jTFcqg8lElfAi8dSee8j2jFDIAgEgAcABwQICcwHCAcMAQb5gqEQiOqBKE6++9fJCR6LRVtNC" - "cE9MFknXFlF0leXQMAA/vWDgwPyHRVDvZl2iYgjJ3nWePRW2wjoUWAxrbgzB5a8AP71vi5ua8R9Xas7ZJOxnHw9u9q/5yyOmKiac4YXhpzZdAEG+s1A7ERdFjokIunFC" - "SgeOxki+V8FwbGaF2nFzHDuF3TgCASABxgHHAEG+VoZmB1FqSlGFLPm5r9LBLAX67F6BFQLDlwahNArjz1ACAnIByAHJAD+9QiJtY3MezTL7KB0xvFikeKH4EL/XSXL0" - "b7P1FoVCXwA/vWinW8a2SNxgyMi+e0ML00BiBRy4kZh/JQrAHMZZ3Y0CASABzAHNAgEgAdIB0wIBWAHOAc8CBX+rYAHQAdEAQb4MUGwt25IQd3/yHjI03F71G8Kp2GMa" - "MEv2TiWoTKbs4ABBvjfgYNaJyJijra4RuhLyyPeGUpRcBZhwzdStzQ2MIyDgAD+8XsswC94XkGKDsoUR3B73WxXRX2LdrWSok77uwX/c8AA/vF/xbT+aFbepxFKzgZQ9" - "HbF9uy1KEVspm2/20klhldAAQb6ORoMEHrkmcAR+9ntDkAj0Hq6gLGUT0ceglU8Tm9jfuAIBIAHUAdUAQb5A/TMaqnaKx2BBvcxafTpwUxZYRXcKXTAZj80OapRScABB" - "vm8iGJqmHDhbx34EGjoh2YHhU4mpC/HVkmnz7NBQA0LwAgEgAdgB2QIBIAHmAecCASAB2gHbAgEgAd4B3wIDeqAB3AHdAEG+rC9orZ39Jto92k4zrR5989Z4qySyANXA" - "U8TLG5+0zfgAP71bgmShTXyEATbw0sECEmtwNtuzKI+S3DHEAPCPRhvTAD+9YC74p2ZuEIcz5A4sE69a7MTFuARvrmQnzUDgc7Mo3QIBIAHgAeECA3jgAeQB5QBBvlnO" - "v0cNQ7XgFJEwo9boghCVUHzfZ+urQtJh6esRW5xQAgFqAeIB4wBAvYY1sTf2ZnuWrkRZ+aijWbaH+q5ZMHkghn/Ys+tCZhoAQL2mLfoqMZw77ln7oAn0Cna+Bkp/snNw" - "xHgR2MTl/uqVAD+9XiSecyAvpnbNK3Z28HAfLhXvbXN59PmK+A7M2VDdAwA/vVcEpETq6AblfmVHtN91B7GNEyGglVc2447ooPciTZMCAUgB6AHpAgEgAe4B7wIBIAHq" - "AesAQb5J79ZyWgm+nqrXs6x0I4wkPiKQBH28C7RWNfPTqAfu8ABBvga7i8W/V7fCfyaKf+LLs48ld6A5hMVDltkVnlrlk+IgAgFYAewB7QBAvZIZkLzw7YHDbLe+Scl6" - "3uhdXfRwOUa0JHwJvuhGG3kAQL2a+QtRGkljjF6hjiME0j7LnnMjJkDh6mYBahv3SgufAEG+q3Z1cONnEXUOq6coX7x0RaK8l2WJj/QViIJee2G6qcgAQb6p4a4p479A" - "eC04K9HUR0x8B9TDrIBoSgVyWXe7xEjGWAIBIAHyAfMCASACBAIFAgEgAfQB9QIBIAH6AfsCAUgB9gH3AEG/JvWFCk64ubdT7k9fADlAADZW2oUeE0F//hNAx5vmQ24C" - "ASAB+AH5AEG+ortA8RL/qsRfVCCcmhh9yV+abEsHsmRmSDIyM5jiKZgAQb52rnetuJmLxwetwRXlQ8SwkzMrIHn9f1t+3vxypn8ikABBvlRRrWQUSUCo75+dTtj6fP1U" - "VTmV5DEujv1TIAc3ZLZQAgFYAfwB/QIBIAH+Af8AQb6OgDPbFGfKzqixWPD2Hmgt4G6KWUdQTJBPH3A9K+TZ6ABBvoMGKypw006AeRYqimLjmY2Ufp+SHk8C0ZJBNgVB" - "lzw4AgFqAgACAQIBWAICAgMAQb4FNJ5NJO4+0QwlVAWckUZXdk+PfYDexDZ1+ju9SxhF4ABBvjxQpfN455vPpJ/T+t2rtlKCE9X6KviHFRV802gCPe5gAEG+eMP12XnW" - "n0wTl6XmbgClnjYFM2JY2UAZYhUaknKJf3AAQb5WLKPfVeykQ1NoeXCT+51aWRbOsYTKmyd3AQSzEZ39EAIBIAIGAgcCASACDAINAgFYAggCCQIBIAIKAgsAQb68pxxy" - "oAcWOvpflv3VjfgrRk9v44uazdxMziPqfc1hGABBvqK0CHqoBidcEUJHx4naV3TtgmUv1oEhGpt3DFLGnncoAEG+xnddXOiUNI6DJEK4qY1Cxoa8Hl6iQkWXMWUwTPTo" - "H6wAQb72G1Ke4q6X03mCI87z+qVMO/gd+xvXv6SSwdWpfbnvjAIBIAIOAg8AQb8B8+e/xOcnn+D3yL8SGkEf/SXAx3pRSH/Lf3UDC6zxGgIBIAIQAhEAQb7an34AE4Mg" - "4PeqZAW6F6j/JbgFl8egPBFDGYC5dIgrvABBvpMd78gzSiVsK0zz0AHtEja8x1UoB/NDZMjn+l86NQK4AgFYAhICEwIBIAIUAhUAQb4zj6RBc4mQ6p3ng7mGJ7tp7Mbz" - "ERhe7obkM9A0wnCCIABBvcdlWZEG0Xj7uGgLfagzT4G4zmtS/JDEdPQBzOA0r99AAgEgAhYCFwBAvYD00VNmocZyrS8LPuogdwJgYw9wWC7QCKaicnWos7IAQL2UR4JV" - "cHfZibOIOqdJm+OTPN6Z1z0bykKu09Up+xc/AgEgAhoCGwIBIAIoAikCASACHAIdAgEgAiYCJwIBWAIeAh8CASACJAIlAEG+pJiW3Qo4nq8pKjVzzfs3/0uJxMmWXYyD" - "sduLHtuy8ggCASACIAIhAEG+VOzUzgqzn6yjJdPd2lOP2LQqiZF7O2/LbcmLzMf+hfACAnICIgIjAD+9bmuGAYNACsk0M2FDu866cYUghqLilNK52oLflBoKXQA/vU+c" - "jkDnrb+NojfOEJpwm2m9hlmHmr3HOWwyl4LEIcEAQb7xrpmUHCzHHfaaDbiK66LDRKeKblhi4QoTVRthJ2OzbABBvu6d/bOGE/iiKiKq5AGCvcetA3Izw45ihY196+ey" - "/BbcAEG/IPVJM6fGP9OC+PczMUdiKPNfwkUrt4eslgzXXEY0qCIAQb8FwRfn4LbYMTzpLsSBuEI3vAaLitADflpdxp+M5JVWtgIBIAIqAisCASACNgI3AEG/OXz/ktGT" - "HClb8arzLt3XEjlJTw9LEYxjGvSJNff79loCASACLAItAgFIAi4CLwIBIAIwAjEAQb5bNqQnT8GAdHDnixf9NzTB5VYvmnvaYs6m53KwbxMzsABBvlGslmQWFAphVxFA" - "GGIJvfuk/oBpngdzy0sJ8WxmWNSQAgN+ugIyAjMCAW4CNAI1AD+84Hccb00HqhGM3lRQZIZ3QmOuWlRDBQ9+uXRKu1L+hAA/vOLc2o+R4+ofOAQzeQiU06F6MN1nTGWW" - "J0eurH869zQAQb36Q2nDRQfZx/XsGJ+z0zYtk4S6OXPZcUASOm420y1FQABBvd9bukINCpKmNEXeA+ve7Mnhp8WSt+MPJFDCUYjDLZ1AAgEgAjgCOQBBvzD0lLSsv1Pi" - "WQ0jVDajeXFbJ/TkSakvdy+g0TPR27KGAgFYAjoCOwIBWAI8Aj0AQb53taVCRMwrV1sky/EE45BOJoTTJ0d6vkLZIb6j4k+G0ABBvlKuPPc+sdv9ffRS/Kj+bSQKZFE7" - "fT/jbtog/5dYYCCQAEG+ZZdBcxF7VCWJS+ti78o7J2qY+aXyKipCl2P0CfXeUhAAQb5gdZIvzW7H8KDz4y1oKMiuAzlXY+TF7PGVAwUvGCn0UAIBIAJAAkEBA6DAAkwB" - "AfwCQgIBIAJDAkQBwbnpmKopRu2n8DHZCDhXCHvJdckI7xw0kBvbb0npdd7jjldXaYBVRMxJsrwBE0/IJ4amdSKh5/Ec0+nZhJr583uAAAAAAAAAAAAAAABtiv/XlkR5" - "bE7cmy0osGrcZKJHU0ACRwEB1AJFAQH0AkYBwcaYme1MOiTF+EsYXWNG8wYLwlq/ZXmR6g2PgSXaPOEeN1Z517mqkFdU7Nqr1K+moGnDNMnTrseTTWtZnFPnBDuAAAAA" - "AAAAAAAAAABtiv/XlkR5bE7cmy0osGrcZKJHU0ACRwLFAaUkEAuNdJLBIqJ50rOuJIeLHBBTEnUHFMTTlSvkBfBlTSx/ArBlJBChmMwsWi3fU4ek+WJDvjF7AhFPUcNX" - "4kaAAAAAAAAAAAAAAAAAJ37Hglt9pn14Z9Vgj9pE3L7fXbBAAkcCTgIBIAJIAkkCASACSgJLAIO/z+IwR9x5RqPSfAzguJqFxanKeUhZQgFsmKwj4GuAK2WAAAAAAAAA" - "AAAAAAB7G3oHXwv9lQmh8vd3TonVSERFqMAAgr+jPzrhTYloKgTCsGgEFNx7OdH+sJ98etJnwrIVSsFxHwAAAAAAAAAAAAAAAOsF4basDVdO8s8p/fAcwLo9j5vxAIK/" - "n8LJGSxLhg32E0QLb7fZPphHZGiLJJFDrBMD8NcM15MAAAAAAAAAAAAAAADlTNYxyXvgdnFyrRaQRoiWLQnS/gLFAbUl61s8X25tzWBr7nugeg7IMDUhKEm34FWUmcD2" - "utVNIR8VdL9iPRR4dwjF/dVl4ymiWr+kkJXphEJvGbzwSXSAAAAAAAAAAAAAAAAAWZG0lbam3LV4+pciTNFehvbNeeLAAk0CTgIBIAJPAlAAMEO5rKAEO5rKADehIAPk" - "4cBAX14QA5iWgAIBIAJRAlIAg7/T7quzPdTpPcCght7xTpoi+g9Sw7gtkYDSyaOh0qHc0AAAAAAAAAAAAAAAADavGw+/CvXTnyDIJ6fZU+llAiixQAIBIAJTAlQCASAC" - "WwJcAgEgAlUCVgCBv1wad2ywThLttxU0gcwWuSJSuLNadPm8j3J85ggRzjkGAAAAAAAAAAAAAAAB1xLrLNteGQzkOClxdvv3E/l3M5UAgb8JuDCFQxifbIdTfjd1x7Mq" - "S+Z7dzIUkHtIdVjcVeFT2AAAAAAAAAAAAAAAAiwal03Yl9B7p2fVDSCtlYsZX6m+AgEgAlcCWAIBIAJZAloAgb7jxvbib0yb3DKvQBDcHL/hdg7NjCuqjUQ09t8hgmhV" - "oAAAAAAAAAAAAAAABEGpMZGoNId5F80sBzWgnjo+AP2UAIG+sE8ccijAbmkaBJVfyfgqY5pf4QSO+c5IFGVC9WwlY/AAAAAAAAAAAAAAAAeg08QveVui23B9QhrdMd7a" - "nx/sGACBvqxwYOyAk+H0YGBc70gZFJc6oqUvcHywU+yJNBfSNh+AAAAAAAAAAAAAAAADFU5kDFbQI6mIkEJqJNGncvWjiygCASACXQJeAIG/acxhhr+dznhtppGVCg+k" - "FqjL65rOddHn1mwyRj1rYgQAAAAAAAAAAAAAAACRfpTwfZ9v81WVbRpRYN+1/m9YhwCBvw9fhTm/NqURBT4FuwJczZWe39F575hmpFtt8KVniCwIAAAAAAAAAAAAAAAB" - "DkxuMKeNKjBZpVAjNVjJ/URzwhoAgb8RuD3rFDyNUpuXtBAnWTykKVAuY7UKLrye419st2b25AAAAAAAAAAAAAAAAlUrmS7Amiwb/77tvRUhnpfLLMXeL4vIgQ=="; - +const char* config_boc = + "te6cckICAl8AAQAANecAAAIBIAABAAICAtgAAwAEAgL1AA0ADgIBIAAFAAYCAUgCPgI/" + "AgEgAAcACAIBSAAJAAoCASAAHgAfAgEgAGUAZgIBSAALAAwCAWoA0gDTAQFI" + "AJIBAUgAsgEDpDMADwIBbgAQABEAQDPAueB1cC0DTaIjG28I/" + "scJsoxoIScEE9LNtuiQoYa2AgOuIAASABMBA7LwABoBASAAFAEBIAAYAQHAABUCAWoAFgAXAIm/VzGV" + "o387z8N7BhdH91LBHMMhBLu7nv21jwo9wtTSXQIBABvI0aFLnw2QbZgjMPCLRdtRHxhUyinQudg6sdiohIwgwCAAQ79oJ47o6vzJDO5wV60LQESEyB" + "cI3zuSSKtFQIlz" + "hk86tAMBg+mbgbrrZVY0qEWL8HxF+gYzy9t5jLO50+" + "QkJ2DWbWFHj0Qaw5TPlNDYOnY0A2VNeAnS9bZ98W8X7FTvgVqStlmABAAZAIOgCYiOTH0TnIIa0oSKjkT3CsgH" + "NUU1Iy/5E472ortANeCAAAAAAAAAAAAAAAAROiXXYZuWf8AAi5Oy+xV/" + "i+2JL9ABA6BgABsCASAAHAAdAFur4AAAAAAHGv1JjQAAEeDul1fav9HZ8+939/IsLGZ46E5h" + "3qjR13yIrB8mcfbBAFur/////8AHGv1JjQAAEeDul1fav9HZ8+939/" + "IsLGZ46E5h3qjR13yIrB8mcfbBAgEgACAAIQIBIAAzADQCASAAIgAjAgEgACkAKgIBIAAkACUB" + "AUgAKAEBIAAmAQEgACcAQFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVAEAzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMwBAAQ" + "EBAQEBAQEBAQEB" + "AQEBAQEBAQEBAQEBAQEBAQEBAQECASAAKwAsAQFYAC8BASAALQEBIAAuAEDv5x0Thgr6pq6ur2NvkWhIf4DxAxsL+Nk5rknT6n99oABTAf////////" + "//////////////" + "////////////////////gAAAAIAAAAFAAQHAADACASAAMQAyABW+AAADvLNnDcFVUAAVv////" + "7y9GpSiABACASAANQA2AgEgADcAOAIBIABCAEMCASAATgBPAgEgADkA" + "OgIBIAA+AD8BASAAOwEBIAA9AQHAADwAt9BTLudOzwABAnAAKtiftocOhhpk4QsHt8jHSWwV/O7nxvFyZKUf75zoqiN3Bfb/" + "JZk7D9mvTw7EDHU5BlaNBz2ml2s54kRz" + "l0iBoQAAAAAP////" + "+AAAAAAAAAAEABMaQ7msoAEBIB9IAQEgAEABASAAQQAUa0ZVPxAEO5rKAAAgAAAcIAAACWAAAAC0AAADhAEBIABEAQEgAEUAGsQAAAAGAAAAAAAA" + "AC4CA81AAEYARwIBIABVAEgAA6igAgEgAEkASgIBIABLAEwCASAATQBdAgEgAFsAXgIBIABbAFsCAUgAYQBhAQEgAFABASAAYgIBIABRAFICAtkAUw" + "BUAgm3///wYABf" + "AGACASAAVQBWAgFiAFwAXQIBIABgAFcCAc4AYQBhAgEgAFgAWQIBIABaAF4CASAAXgBbAAFYAgEgAGEAYQIBIABeAF4AAdQAAUgAAfwCAdQAYQBhAA" + "EgAgKRAGMAZAAq" + "NgIGAgUAD0JAAJiWgAAAAAEAAAH0ACo2BAcDBQBMS0ABMS0AAAAAAgAAA+" + "gCASAAZwBoAgEgAHoAewIBIABpAGoCASAAcABxAgEgAGsAbAEBSABvAQEgAG0BASAAbgAM" + "AB4AHgADADFgkYTnKgAHEcN5N+" + "CAAGteYg9IAAAB4AAIAE3QZgAAAAAAAAAAAAAAAIAAAAAAAAD6AAAAAAAAAfQAAAAAAAPQkEACASAAcgBzAgEgAHYAdwEBIAB0AQEg" + "AHUAlNEAAAAAAAAAZAAAAAAAD0JA3gAAAAAnEAAAAAAAAAAPQkAAAAAAAhYOwAAAAAAAACcQAAAAAAAmJaAAAAAABfXhAAAAAAA7msoAAJTRAAAAAA" + "AAAGQAAAAAAACc" + "QN4AAAAAAZAAAAAAAAAAD0JAAAAAAAAPQkAAAAAAAAAnEAAAAAAAmJaAAAAAAAX14QAAAAAAO5rKAAEBIAB4AQEgAHkAUF3DAAIAAAAIAAAAEAAAww" + "AATiAAAYagAAJJ" + "8MMAAAPoAAATiAAAJxAAUF3DAAIAAAAIAAAAEAAAwwAehIAAmJaAATEtAMMAAABkAAATiAAAJxACAUgAfAB9AgEgAIAAgQEBIAB+" + "AQEgAH8AQuoAAAAAAJiWgAAAAAAn" + "EAAAAAAAD0JAAAAAAYAAVVVVVQBC6gAAAAAABhqAAAAAAAGQAAAAAAAAnEAAAAABgABVVVVVAgEgAIIAgwEBWACGAQEgAIQBASAAhQAkwgEAAAD6AA" + "AA+gAAA+gAAAAP" + "AErZAQMAAAfQAAA+gAAAAAMAAAAIAAAABAAgAAAAIAAAAAQAACcQAQHAAIcCASAAiACJAgFIAIoAiwIBagCQAJEAA9+" + "wAgFYAIwAjQIBIACOAI8AQb7c3f6FapnFy4B4" + "QZnAdwvqMfKODXM49zeESA3vRM2QFABBvrMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM4AEG+" + "tWede5qpBXVOzaq9SvpqBpwzTJ067Hk01rWZxT5wQ7gAQb8a" + "Yme1MOiTF+EsYXWNG8wYLwlq/ZXmR6g2PgSXaPOEegBBvzSTEofK4j4twU1E7XMbFoxvESypy3LTYwDOK8PDTfsWASsSZn08y2Z9WOsAEAAQD/////" + "////3AAJMCAswA" + "lACVAgEgAJYAlwIBIACkAKUCASAAmACZAgEgAJ4AnwIBIACaAJsCASAAnACdAJsc46BJ4rulpzksHMZaJjfdtBExV1HRdikp9U7VlmJllrEaW2TYAF" + "mAXnBlZIRH4Sqp" + "CbKkE6v60jyawOEYfVWJDgHg5kDaLMWq7kWQy6AAmxzjoEniuRloX7kgG9FNmRyw/AB/KERuToZdY5v8AHv9JJ8bCIKAWYBecGVkhEt/" + "mk7tOEXbKUWuqIz/1NliY9sm" + "KNHFQimyb79WXudTIACbHOOgSeK0/SaSD6j2aEnWfmW/B7LOQBq2QiiBlnaLIzfq+J2HM0BZgF5wZWSEWPYUSh0McOyjsLL8prcsF5RNab+7jLN/" + "5bOme1r98c8gAJsc" + "46BJ4rT4ptGRb52wRyHzhe/A8y/IQOC/W5R5aC6/l1IM4f/" + "EgFmAXnBlZIRmDW7+WN70SpQsfX5DetODFOpW6zjCBx7cDf6E+rEipKACASAAoAChAgEgAKIAowCbHOOg" + "SeKqqZCAjJ16vfAa2GI9Dcp/I9zBTG2CwPqbx22lq00uLoBZgF5wZWSETeqWp7jqIGPuCYnPZSlQ1fMuSS4e1gF/" + "i9uIeD8GEkNgAJsc46BJ4rugeQAFCtwRUJhvWRbx" + "smlpXTdXCio8SJSBdH/6VPCkAFmAXnBlZIRQPeE6JpjzEwkPI2mvCM1sDTcny96f2dhZ2DcBQmmywCAAmxzjoEnimDpTGClVkh/V+/" + "mJmKVKEpdp4MvFgP5onw6saJRD" + "QApAWYBecGVkhElWAHSIgIhlXt+" + "lUyQjmndd50temeILBd7WJwjjWBeIIACbHOOgSeKtcjPEr2gq3gMraY11K9Ikv1SPcVaj3veDWrY1o4nxKcBZgF5wZWSEabqKQLtX" + "PIkaYDaKvupB8EOxFDWpuMaJJVqafjw4h4sgAgEgAKYApwIBIACsAK0CASAAqACpAgEgAKoAqwCbHOOgSeK8POt5lMj96a3WrXWw7peFtWWh5oi9ws" + "ZqXRsrnHM4eoBZ" + "gF5wZWSEXlJk0ILG3LG9zsmxXf+r2OTayqr9FSKLBt9LJAow+aBgAJsc46BJ4qjb23m1w/" + "0EvFl179XCQUUMk32z0kjSh+t6V2jnnqeFwFmAXnBlZIR2KWk8cqZgC06K" + "AphhfzE3VceQWtppAGEbybk06szO9KAAmxzjoEnihVEG74vb19K1l5o8WtWa0dH/" + "gTPfytoA1LsVXR3ztfgAWYBecGVkhEVHN0AzKnDpKLX5P7Tnay/Ogc4rxeoks/yh" + "U3aWhEnGIACbHOOgSeKNl8PpsnZjGIy1CTzi01K8MhvQAEhGlzUDwj2ACC/" + "yFUALGRulQuFOdHw2ulDcYktF860U0mFOYFaQPC7MVNbEeSsk45C9tSPgAgEgAK4ArwIB" + "IACwALEAmxzjoEnivAzuiTw+hkcXtw4XyJGYavfPayk6ehceV8FqrxrzKbQACMou1fGNuRpwF6ilPaS03+" + "BSsz0YID1gpIkGozQp7gRFcQsyZFvVYACbHOOgSeKsoYF9" + "T9f0ArrtFxbViCRmpw2DsDzrllY35uHzP9DEosAICQwVUUQOx01jZ84Uy8ccqQ90Ml6tj5Sw14wOK055ds2sYSPy532gAJsc46BJ4piyhqkrUrk/" + "KUOony6llV0S+DnZ" + "xDLdccZzKJ7bV+XiAAeBJKPSjdajMGMdZwRvewwnwsyc/" + "7uHN718Pd8cHn7VQG1i9BJSeaAAmxzjoEnihY8aTVKeJnW4JHbfVPfkJwElQXxxqG94pNWmN6n9I5jABA51" + "90xtZChBtmQcmPHlOmtU6aLeZ+HBY7/jW6AMz26cNcymYyIuIAErEmZ9WOtmfXULABAAEA/////////" + "3wACzAgLMALQAtQIBIAC2ALcCASAAxADFAgEgALgAuQIBIAC+" + "AL8CASAAugC7AgEgALwAvQCbHOOgSeK5Nyl3TF7AOD2UwhNOh+y3h9P5e0emd2zjffbNatQR1EBS4qdSDsPAZjIVSudNcsvyCAIbiOyNPYmj/" + "MJG5lMjVLkYt4TIEDCg" + "AJsc46BJ4q0qr9PzfnnT+A41FG5Owo+9L+LsuT6PrQkuoR7XsLMzgFLioMqMr4sLf5pO7ThF2ylFrqiM/9TZYmPbJijRxUIpsm+/" + "Vl7nUyAAmxzjoEnisgCK09re8agW" + "Ee8S6q329jm1WbZoHBHjO9oP0q3qItiAUuKgyoyviwfhKqkJsqQTq/" + "rSPJrA4Rh9VYkOAeDmQNosxaruRZDLoACbHOOgSeKeKPVNUBZ96hhTOP8lp1kiAm2wfuT0HIxn" + "lw/0cyISP8BS4qDKjK+LGPYUSh0McOyjsLL8prcsF5RNab+7jLN/" + "5bOme1r98c8gAgEgAMAAwQIBIADCAMMAmxzjoEnip+PTCe8vsapzyPHm88uO5qKBwt9yvn+S6aJW" + "OlcBqeDAUuKgyoyviyYNbv5Y3vRKlCx9fkN604MU6lbrOMIHHtwN/" + "oT6sSKkoACbHOOgSeKwOTDV9phg7jYWvy7bbTD8N773bX9y1P7lxC7vtvdbvsBS4qDKjK+LDeqW" + "p7jqIGPuCYnPZSlQ1fMuSS4e1gF/i9uIeD8GEkNgAJsc46BJ4opGGis7tEqqLAW2742I2ugw5S5lFxeYpc4D9f/" + "qbOMhwFLioMqMr4sQPeE6JpjzEwkPI2mvCM1sDTcn" + "y96f2dhZ2DcBQmmywCAAmxzjoEniqGUvGQXdvzVXTq/" + "g3DpDkom5aqVipETXzq2o+FZdGDfAUuKgyoyviwlWAHSIgIhlXt+lUyQjmndd50temeILBd7WJwjjWBeIIAIB" + "IADGAMcCASAAzADNAgEgAMgAyQIBIADKAMsAmxzjoEnihA6ouVC73YehzpHoNBKL8q3Gp4YbwxOBhJdxpNWePHwAUuKgyoyviym6ikC7VzyJGmA2ir" + "7qQfBDsRQ1qbjG" + "iSVamn48OIeLIACbHOOgSeKr2ACjLl9IlajrtDqvMLD+lfOMRQvmZAaL2NVDooVPYQBS4qDKjK+LHlJk0ILG3LG9zsmxXf+" + "r2OTayqr9FSKLBt9LJAow+aBgAJsc46BJ" + "4oohDH+XJf2EoPKNkp+gv/WG2UonjUWXV+B/" + "IvWUldUuQFLioMqMr4s2KWk8cqZgC06KAphhfzE3VceQWtppAGEbybk06szO9KAAmxzjoEnilP2IvoMbkK7LwTeBBX8u" + "dYI608SRo4nDIg7XUWQf2CYAUuKgyoyviwVHN0AzKnDpKLX5P7Tnay/Ogc4rxeoks/" + "yhU3aWhEnGIAIBIADOAM8CASAA0ADRAJsc46BJ4qS3beCYCuu47Ohag9xU5wk6" + "/1uLtI/5NZ+VaqSyKsGdAApHFgZLFGK0fDa6UNxiS0XzrRTSYU5gVpA8LsxU1sR5KyTjkL21I+AAmxzjoEnivJI7eg6kFGx7dvMX7Xzoog/" + "s5cwHxrcfec5z8/aP/8kA" + "CFtq86KYH4dNY2fOFMvHHKkPdDJerY+UsNeMDitOeXbNrGEj8ud9oACbHOOgSeKlwkl68jfkl6kGCq/" + "tElh6bM85sFBPnt7exnkRJq68iQAG+mnlyjEXYzBjHWcEb3sM" + "J8LMnP+7hze9fD3fHB5+1UBtYvQSUnmgAJsc46BJ4oYswn2e5gWf+Va6NJ+" + "K8sfz4qIHmVG2ryktqCkE9P8hQAPDhRot06toQbZkHJjx5TprVOmi3mfhwWO/41ugDM9u" + "nDXMpmMiLiABASAA1AEBIAD6AQsAtb0+sEAA1QIBIADWANcCA8H4ANgA2QID4fgA+AD5AgEgAPwA/" + "QIBIADaANsCASAA3ADdAgEgAbgBuQIBIAGQAZECASAA3gDfAgEg" + "AOAA4QIBIADqAOsAQb7edpH5xbuqiZNqTG9H7flTOIfNiYtDxI5AH4T6G4tcVAIBIADiAOMAQb6U4RvTn2B6e+8nmlEv/" + "eZoRz1YKr3qyDudETjcrMFgKAIBIADkAOUC" + "ASAA5gDnAgEgAOgA6QBBvgukN4cHaqlFuawJv/TGaxhU3HU2B5iu8cZPVMOseQOgAEG+K7U1xAKEqaBEZoqjpyAnvSx8Z9jfPTeAR/" + "anR5axvmAAQb4tEpbKJaulevOY" + "XQPqlmgiMgHDU6C6X7KRxpFyzPf0YABBvjbzLj0Z1oudyhyW/" + "QhJ0OUxRj9zEM8Y1YUI9Py3ga6gAgFqAOwA7QIBIADuAO8AQb4JmTypqySHVMVJMHWspb3xrs2Lrdy4" + "eJ+M7QxpbS4cIABBvgOb8O+4IZEUWqtnRGQ8JpMkMBocpZyk/do3d/9MYnVgAgEgAPAA8QBBvqQeZ13QP0lszxNKt380fCWuaV94vwC/bfuqmrlg1/" + "fIAgEgAPIA8wIB" + "IAD0APUAQb4G2ph6AS/mD/+cIv4aIYm1z5jAgCW/" + "TTDEr72ygXOP4ABBvhBZkdUWyc1zdg9Fhp9QSsWD+LSyXChKLJOiMF3rVNqgAgEgAPYA9wBBvhsYuojZc90oYnM2" + "WQ+c6cHdiTDRBD2UgxkJlbkZa+mgAEG9wBVbqgGsx1Pog5dkmDyUl4VIe1ZME2BEDY6zMNoQYsAAQb3R4obtqmXfb1H2NxdElqeDuWD4d+" + "Y73ozNJ7dE4jGfQAIBIAHw" + "AfECASACGAIZAQPAwAD7AFWgESjR4FjxyuEAXHMvOQot+HG+D9TtSQavwKbeV09n3G92AAAAAAAAAH0QAgEgAP4A/" + "wIBIAEcAR0CASABAAEBAgEgAR4BHwIBIAECAQMC" + "ASABEAERAgEgAQQBBQIBIAEIAQkCAWIBBgEHAEG+tp/96j2CYcuIRGkfljl5uv/Pilfg3KwCY8xwdr1JdqgAA97wAEG99o5GkuI7pwd5/" + "g4Lt+avHh31l5WoNTndbJgd" + "dTJBicACAUgBCgELAgEgAQwBDQBBvgIKjJdXg0pHrRIfDgYLQ20dIU6mEbDa1FxtUXy9B6rgAEG+Cev2EcR/" + "qY3lMYZ3tIojHR5s+wWySfwNg7XZgP23waACASABDgEP" + "AEG+fZGfOd+cHGx01cd8+xQAwUjfI/VrANsfVPw1jZFJhTAAQb4y2lPdHZUPm695Z+bh0Z1dcta4xXX7fl6dlc2SXOliIABBvhfW5EoZl/" + "I8jARohetHRk6pp1y3mrXR" + "28rFYjHHtJCgAgFqARIBEwIBIAEUARUAQb4zE+Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU+" + "KeaYLIABBvgPcWeL0jqPxd5IiX7AAYESGqFqZ7o60BjQZJwpPQP1gAgEg" + "ARYBFwBBvofANH7PG2eeTdX5Vr2ZUebxCfwJyzBCE4oriUVRU3jIAgEgARgBGQIBIAEaARsAQb4btDCZEGRAOXaB6WwVqFzYTd1zZgyp15BIuy9n02" + "9k4ABBvimf97Kd" + "WV/siLZ3qM/" + "+nVRE+t0X0XdLsOK51DJ6WSPgAEG+CQrglDQDcC3b6lTaIr2tVPRR4RlxVAwxYNcF+6BkvaAAQb4mML93xvUT+" + "iBDJrOfhiRGSs3vOczEy9DJAbuCb7aU" + "4AIBIAFAAUECASABYAFhAgEgASABIQIBIAE0ATUCASABIgEjAgFYAS4BLwIBIAEkASUAQb6L1UE7T5lmGOuEiyPgykuqAW0ENCaxjsi4fdzZq2D0GA" + "ICcAEmAScCASAB" + "KAEpAD+9QolK/7nMhu3MO9bzK31P7DqSFoQkLyeYP3RWz5f3KwA/vVaiOV3iXF+2BW0R7uGwqmnXP7y0cjEHibQT6v4MssECASABKgErAgV/" + "rWABLAEtAEG96YUi7d3r" + "hTwVGwv/pocif6dNQ6DcZ3JVzvqdhFltQ0AAQb3zT7C1dlWQlR1QmfrLfaGi5Sj94Guq/gLQXakuFmoVwAA/" + "u8n6yK+GpbUUdG9dja4DHHLGGEu5ZXb6rUHFOFMS7kAA" + "P7v3dUiUhgaZGC+mdUGyJEzagm0IMNe3d2Q1lCRBTK5AAEG+co6LJmQv3h46OSV3KsT2gWyv6MLPKOrfIXFt86dsXVACASABMAExAEG+KQF+" + "kzAAZybpH/1z1zYof09W" + "YAAY6MbQHDj3AO9dCGACASABMgEzAEG9xJZFhUbajV1FgRPu0X8LSHY3DIBRmI4wC6uLpNG5lkAAQb3/" + "+UXNzozn7Eb1PsCLs8NaD2VhG+9qBBlvLJG76KkTQAIBIAE2" + "ATcCASABPgE/AgEgATgBOQIBYgE8AT0AQb5l6UC6/ZmwRTHlWwthzsJcYx+8Vj2vmom9/nu617FmkAIBIAE6ATsAQb4J64Df7Vfb8/" + "jmlGnsZByGAdCsEWA/FfWXyVEU" + "5d6CoABBvhv0Q/VEAfHxjnYRJRxb6xtGetqoO1OgjstzC/" + "3Ok41gAEG964EWqVOQS0JWHUcxnAz6STWs7+BsROmocJCo+xmqe0AAQb3vR9oRALXcwLQPRb70F/gP7SAV" + "WqyMgCIasOqw+b47wABBvpbvxWd5+q2vJUVqR9AlbEIfdFysLR0PXGgVlBf8x5hYAEG+j9bgcxjKxRmfMrJEC6BbHTCQ+WNXqC3H+" + "z591gZw0AgCASABQgFDAgEgAUgB" + "SQIBSAFEAUUAQb7KkreZXaSZXSPGxbgwuJddzpWJly3MFNYwALkyQcIdDABBvnLW0BTZocy0D6h48ehPtgqA0XqNxrqB86bTTks9uvuQAgEgAUYBRw" + "BBvjYzcOXWIfyk" + "HqSDt3m92Hacz/XRoWD5F4yy0AQ/" + "E0ogAEG+AShOVhiiJZ6Itzjs8O75CiiF+eXloz74MSVsHpPAMiACASABSgFLAgEgAVABUQIDeuABTAFNAgFYAU4BTwA/vVuDIbt9" + "1w2Z2FpLSOsyAUPo2ovei28SxaHKDSUdRz0AP71qm4D4evL40x1qJi6AGLh6oOBtxFr5bgc8Xr8jaeWRAEG+HzK7ymUhDh5PL//" + "pLHqwaYidq3sym7hIWC32Rqol+mAA" + "Qb41DOvSox2jnjN40ZFtUSQhSJMCyEWhBRdRERRSltibIAIBIAFSAVMCASABWAFZAgFYAVQBVQIBIAFWAVcAQb3cHJ+brtBSsROnSioWNJqFxZ+" + "5hIGX7ta5KuhleBFn" + "wABBvf/lQA5TJrGDmv6EqacNl5j6ktTzbQOEGqpl45xcekNAAEG+Nve9GdRJhn/" + "t0fgYe7d1pkTBxa2AfiXcWeRYqE1K3yAAQb4jrXHoxDyh1ZYGBdBoQgLaScxW6pZR" + "1hEhJC8BqF+5IAIBIAFaAVsCAVgBXgFfAEG+" + "CdErMSfFYmEK9J9XimJDXyszQjtVELtHIXQt7AvQjKACAUgBXAFdAEC9ivFB4bA7PAP0VXnTs784TO/4CoWLb1QqRdyr" + "0orLAgBAvb5z8xm2yt/HlB1G9TB2Qna4rVgzGxI/n4z3UYr3a7gAQb3f0PQO3/nU5ypuXD5/" + "SaZboj2RhZjd5z47o7VM8AjDwABBvfGIqWXxgi7mCltWrYf4pQa2aRZP" + "FvMA8LBV1hmpauDAAgEgAWIBYwIBIAGAAYECASABZAFlAgEgAXIBcwIBIAFmAWcCAVgBcAFxAgFIAWgBaQIBIAFqAWsAQb33dj2qlHUSOf2DkiVrVw" + "hcqy3SkE9YbBfn" + "zU07vK+uwABBvdxiQ8Yt/" + "Lb9BztkNe9dyXuUyTOcKJRlF9BteI2LK99AAgEgAWwBbQBBvjxAsXZAtTQoMwJV27nrzNCyFum1aU1fbygeFMFuYX9gAgFIAW4BbwBBvdro" + "odCnIayUb5VXYFh23qJGAE4Oed7iqqU/L0iFAPpAAD+9QlUpU0rFnXRmWi3ZnIsFtIIm3JDSdtVPEGqGefBt/wA/" + "vWGl+1GrGASEj3GaAizvMOXDl69yZpcU2YUtCHfG" + "jLUAQb4d/" + "oR88TrfAGcKrMn44T3wBnbh3TWVQWr8rVq0bYTnYABBvhpY6fA3+apwMQXdpEMu8s8uFXf+" + "625mtfciMt0dh4LgAgEgAXQBdQIBIAF4AXkAQb5d0CvPvsyC" + "ZxuTbUe5O2PtTudCwtgc3Ou4DMuX2WizEAIBSAF2AXcAQb3BrlEdo+Hw0uZZJxCgCdxWs/" + "njs6bTHuprY7HtqNl0QABBvcSsc0L20So00ByQZ2oo0aUWf4BlreuHcpYk" + "R/C5Av7AAgEgAXoBewIBIAF+AX8CASABfAF9AEG+ErNElODwkPB+KvEKqCtCz8CS5HCcsC8/VoJGV5f0+uAAQb3FCW/" + "Cy20jtvAS0j4k9eQvRg9tcpaQgFnHc5cB7Fdv" + "wABBvc5nMn9h2c6FeqzonvA74SwaTxZXTgLEXOKOIFOki9BAAEG+NkNRDvICKDQNaqBlpx1LnSn5qpShA00BPg8Tfv+LHaAAQb4+0zsN9j+" + "Lxs1EvbGG0fMwbeeqbWlx" + "TzyjV4LE+0uJYAIBIAGCAYMCAUgBigGLAgEgAYQBhQIBIAGGAYcAQb5O+6O6Y7dWb4HOnMBK4fZ7QNo9woEzBIeKd5+K08xlkABBvlwlLor18dZ5/" + "O3AomXxI5hxYM4o" + "J1Xrrx0JChLVxHpQAgFYAYgBiQBBvn9hAM+g43TTR8vOvZfnhX3kPBCgPp3T0+YF+" + "Ai6RFHwAEG99KmZCgwzysLzIR2TNaJdbyX4lKduOMlCmhCp4L9gJEAAQb3Ntnmm" + "W4yzmAdiAYg7sNjoD8sCiWIvgvkpuYpTXcyiQAIBZgGMAY0CAW4BjgGPAEC9hzviVxD170gIZfsWPGFKfbOB6LCP5YhH7I7fWz7wdwBAvaey9kbu3g" + "kPDYYEraB8b3sF" + "UrCgg4ask3C+" + "O8UJ1mkAQL2wAL6FGQaCTbDdEwGUJ82TDpVMLoNr4ZGZWxcofghZAEC9lqzgehIXoMRj58vAWaHnNAi6UXEU5Ce942dJqf4HawIBIAGSAZMCASABqA" + "Gp" + "AgEgAZQBlQIBIAGkAaUCASABlgGXAgEgAZwBnQIBagGYAZkCASABmgGbAEC9syAieemf3vF3umY0lCaQxLhwvbTFuL8eQxPYrpeZ8ABAvbl6reyIsC" + "KH2fq2I8+oEnkS" + "4xYy3RUH/7ka152WrisAQb4CJHgAcs+wQzgf/9IPKdknw/ej0Z+Q+n3BtSEKi0hIoABBvgqovnD/" + "owP5nsA4G62765H5klOyA1TV+7jriGf2CtjgAgFYAZ4BnwIBIAGg" + "AaEAQb3dAG8Nta3/" + "iYiTymgGxV0CfKQlN6UlidHeNgbvtMT9wABBve7An2cFgShRoZx3xA7hUDRtwbcLae0x4dPQQlAH8o3AAEG+HDeG9ZNvkzq3wDDpGt0cb5cHHFQ0" + "itHD3s5R2YHy8eACAWIBogGjAD+9ewqjet2JVaCzHa8NXfnW3ZtLEzEASpk9eicyztCrvwA/" + "vXDzaFNMjF1BnqMojulsIHfT2Dj1ltCTVvoe8wu+GKcCASABpgGnAEG+" + "un2oV7CbmRhYGc7tLiCXj/L40+4ZlzvlmEnZPxyuQrgAQb5ElmikSUchX0lT+0ASVhwF0OBnUB8X4TD4m4/" + "v2Dfl0ABBvlBR7mcUQO8IfN+DkkDYHF1reSJZhv08w6k+" + "JIA6ITiwAgEgAaoBqwIBIAG0AbUCAVgBrAGtAgEgAbIBswBBvhX0m4apMW/" + "GEDxtnd+z0ug75voHd+OibSQbA2+tUPigAgEgAa4BrwIBWAGwAbEAQb3WKikPb9a/J2ti" + "V6yOhNUW5BivimV3gM+EI3VAxst6QAA/vUeSH4ZL+7V8eQBEF/0lm/" + "ouIJ+wQs5QTzBpsSHSXLcAP71t4YT+jYHLpx5Gv3HFoOzL5rhg0Ukud8G3adF8AYlRAEG+Zf0n" + "TrwaPPTPlLjegNsGkoz7UV5wz7oYQet9+SNmRfAAQb5m0tqyXFYp4ntucDLTwJV1gxwoh6JoJL1Y0rfwfLQhUABBvqSCHVak+" + "jIc9ANutTAfHpZNM3YdGky7yaDzsTrg" + "0WhIAgN9eAG2AbcAP70AGCAXHtaQJNqiST0rNTs8mUZSo5H6vM7gvA+3q7+iAD+9FgzFlOZUrfRtonCQzjDSFzrRv4l/" + "94TFs9oi+RQ6kgIBIAG6AbsCASAB1gHXAgEg" + "AbwBvQIBIAHKAcsCASABvgG/" + "AgEgAcQBxQBBvqg93lUVxmlCEks5kL8jTFcqg8lElfAi8dSee8j2jFDIAgEgAcABwQICcwHCAcMAQb5gqEQiOqBKE6++9fJCR6LRVtNC" + "cE9MFknXFlF0leXQMAA/vWDgwPyHRVDvZl2iYgjJ3nWePRW2wjoUWAxrbgzB5a8AP71vi5ua8R9Xas7ZJOxnHw9u9q/" + "5yyOmKiac4YXhpzZdAEG+s1A7ERdFjokIunFC" + "SgeOxki+V8FwbGaF2nFzHDuF3TgCASABxgHHAEG+VoZmB1FqSlGFLPm5r9LBLAX67F6BFQLDlwahNArjz1ACAnIByAHJAD+" + "9QiJtY3MezTL7KB0xvFikeKH4EL/XSXL0" + "b7P1FoVCXwA/vWinW8a2SNxgyMi+e0ML00BiBRy4kZh/JQrAHMZZ3Y0CASABzAHNAgEgAdIB0wIBWAHOAc8CBX+rYAHQAdEAQb4MUGwt25IQd3/" + "yHjI03F71G8Kp2GMa" + "MEv2TiWoTKbs4ABBvjfgYNaJyJijra4RuhLyyPeGUpRcBZhwzdStzQ2MIyDgAD+8XsswC94XkGKDsoUR3B73WxXRX2LdrWSok77uwX/c8AA/vF/" + "xbT+aFbepxFKzgZQ9" + "HbF9uy1KEVspm2/20klhldAAQb6ORoMEHrkmcAR+9ntDkAj0Hq6gLGUT0ceglU8Tm9jfuAIBIAHUAdUAQb5A/" + "TMaqnaKx2BBvcxafTpwUxZYRXcKXTAZj80OapRScABB" + "vm8iGJqmHDhbx34EGjoh2YHhU4mpC/" + "HVkmnz7NBQA0LwAgEgAdgB2QIBIAHmAecCASAB2gHbAgEgAd4B3wIDeqAB3AHdAEG+rC9orZ39Jto92k4zrR5989Z4qySyANXA" + "U8TLG5+0zfgAP71bgmShTXyEATbw0sECEmtwNtuzKI+S3DHEAPCPRhvTAD+" + "9YC74p2ZuEIcz5A4sE69a7MTFuARvrmQnzUDgc7Mo3QIBIAHgAeECA3jgAeQB5QBBvlnO" + "v0cNQ7XgFJEwo9boghCVUHzfZ+urQtJh6esRW5xQAgFqAeIB4wBAvYY1sTf2ZnuWrkRZ+aijWbaH+q5ZMHkghn/" + "Ys+tCZhoAQL2mLfoqMZw77ln7oAn0Cna+Bkp/snNw" + "xHgR2MTl/uqVAD+9XiSecyAvpnbNK3Z28HAfLhXvbXN59PmK+A7M2VDdAwA/" + "vVcEpETq6AblfmVHtN91B7GNEyGglVc2447ooPciTZMCAUgB6AHpAgEgAe4B7wIBIAHq" + "AesAQb5J79ZyWgm+nqrXs6x0I4wkPiKQBH28C7RWNfPTqAfu8ABBvga7i8W/" + "V7fCfyaKf+LLs48ld6A5hMVDltkVnlrlk+IgAgFYAewB7QBAvZIZkLzw7YHDbLe+Scl6" + "3uhdXfRwOUa0JHwJvuhGG3kAQL2a+QtRGkljjF6hjiME0j7LnnMjJkDh6mYBahv3SgufAEG+q3Z1cONnEXUOq6coX7x0RaK8l2WJj/" + "QViIJee2G6qcgAQb6p4a4p479A" + "eC04K9HUR0x8B9TDrIBoSgVyWXe7xEjGWAIBIAHyAfMCASACBAIFAgEgAfQB9QIBIAH6AfsCAUgB9gH3AEG/" + "JvWFCk64ubdT7k9fADlAADZW2oUeE0F//hNAx5vmQ24C" + "ASAB+AH5AEG+ortA8RL/" + "qsRfVCCcmhh9yV+abEsHsmRmSDIyM5jiKZgAQb52rnetuJmLxwetwRXlQ8SwkzMrIHn9f1t+3vxypn8ikABBvlRRrWQUSUCo75+dTtj6fP1U" + "VTmV5DEujv1TIAc3ZLZQAgFYAfwB/" + "QIBIAH+Af8AQb6OgDPbFGfKzqixWPD2Hmgt4G6KWUdQTJBPH3A9K+TZ6ABBvoMGKypw006AeRYqimLjmY2Ufp+SHk8C0ZJBNgVB" + "lzw4AgFqAgACAQIBWAICAgMAQb4FNJ5NJO4+0QwlVAWckUZXdk+PfYDexDZ1+ju9SxhF4ABBvjxQpfN455vPpJ/" + "T+t2rtlKCE9X6KviHFRV802gCPe5gAEG+eMP12XnW" + "n0wTl6XmbgClnjYFM2JY2UAZYhUaknKJf3AAQb5WLKPfVeykQ1NoeXCT+" + "51aWRbOsYTKmyd3AQSzEZ39EAIBIAIGAgcCASACDAINAgFYAggCCQIBIAIKAgsAQb68pxxy" + "oAcWOvpflv3VjfgrRk9v44uazdxMziPqfc1hGABBvqK0CHqoBidcEUJHx4naV3TtgmUv1oEhGpt3DFLGnncoAEG+" + "xnddXOiUNI6DJEK4qY1Cxoa8Hl6iQkWXMWUwTPTo" + "H6wAQb72G1Ke4q6X03mCI87z+qVMO/gd+xvXv6SSwdWpfbnvjAIBIAIOAg8AQb8B8+e/xOcnn+D3yL8SGkEf/SXAx3pRSH/" + "Lf3UDC6zxGgIBIAIQAhEAQb7an34AE4Mg" + "4PeqZAW6F6j/JbgFl8egPBFDGYC5dIgrvABBvpMd78gzSiVsK0zz0AHtEja8x1UoB/" + "NDZMjn+l86NQK4AgFYAhICEwIBIAIUAhUAQb4zj6RBc4mQ6p3ng7mGJ7tp7Mbz" + "ERhe7obkM9A0wnCCIABBvcdlWZEG0Xj7uGgLfagzT4G4zmtS/" + "JDEdPQBzOA0r99AAgEgAhYCFwBAvYD00VNmocZyrS8LPuogdwJgYw9wWC7QCKaicnWos7IAQL2UR4JV" + "cHfZibOIOqdJm+OTPN6Z1z0bykKu09Up+xc/" + "AgEgAhoCGwIBIAIoAikCASACHAIdAgEgAiYCJwIBWAIeAh8CASACJAIlAEG+pJiW3Qo4nq8pKjVzzfs3/0uJxMmWXYyD" + "sduLHtuy8ggCASACIAIhAEG+VOzUzgqzn6yjJdPd2lOP2LQqiZF7O2/" + "LbcmLzMf+hfACAnICIgIjAD+9bmuGAYNACsk0M2FDu866cYUghqLilNK52oLflBoKXQA/vU+c" + "jkDnrb+NojfOEJpwm2m9hlmHmr3HOWwyl4LEIcEAQb7xrpmUHCzHHfaaDbiK66LDRKeKblhi4QoTVRthJ2OzbABBvu6d/bOGE/" + "iiKiKq5AGCvcetA3Izw45ihY196+ey" + "/BbcAEG/" + "IPVJM6fGP9OC+PczMUdiKPNfwkUrt4eslgzXXEY0qCIAQb8FwRfn4LbYMTzpLsSBuEI3vAaLitADflpdxp+M5JVWtgIBIAIqAisCASACNgI3AEG/" + "OXz/ktGT" + "HClb8arzLt3XEjlJTw9LEYxjGvSJNff79loCASACLAItAgFIAi4CLwIBIAIwAjEAQb5bNqQnT8GAdHDnixf9NzTB5VYvmnvaYs6m53KwbxMzsABBvl" + "GslmQWFAphVxFA" + "GGIJvfuk/oBpngdzy0sJ8WxmWNSQAgN+ugIyAjMCAW4CNAI1AD+84Hccb00HqhGM3lRQZIZ3QmOuWlRDBQ9+uXRKu1L+hAA/" + "vOLc2o+R4+ofOAQzeQiU06F6MN1nTGWW" + "J0eurH869zQAQb36Q2nDRQfZx/" + "XsGJ+z0zYtk4S6OXPZcUASOm420y1FQABBvd9bukINCpKmNEXeA+ve7Mnhp8WSt+MPJFDCUYjDLZ1AAgEgAjgCOQBBvzD0lLSsv1Pi" + "WQ0jVDajeXFbJ/TkSakvdy+g0TPR27KGAgFYAjoCOwIBWAI8Aj0AQb53taVCRMwrV1sky/" + "EE45BOJoTTJ0d6vkLZIb6j4k+G0ABBvlKuPPc+sdv9ffRS/Kj+bSQKZFE7" + "fT/jbtog/" + "5dYYCCQAEG+ZZdBcxF7VCWJS+ti78o7J2qY+aXyKipCl2P0CfXeUhAAQb5gdZIvzW7H8KDz4y1oKMiuAzlXY+" + "TF7PGVAwUvGCn0UAIBIAJAAkEBA6DAAkwB" + "AfwCQgIBIAJDAkQBwbnpmKopRu2n8DHZCDhXCHvJdckI7xw0kBvbb0npdd7jjldXaYBVRMxJsrwBE0/IJ4amdSKh5/" + "Ec0+nZhJr583uAAAAAAAAAAAAAAABtiv/XlkR5" + "bE7cmy0osGrcZKJHU0ACRwEB1AJFAQH0AkYBwcaYme1MOiTF+EsYXWNG8wYLwlq/" + "ZXmR6g2PgSXaPOEeN1Z517mqkFdU7Nqr1K+moGnDNMnTrseTTWtZnFPnBDuAAAAA" + "AAAAAAAAAABtiv/XlkR5bE7cmy0osGrcZKJHU0ACRwLFAaUkEAuNdJLBIqJ50rOuJIeLHBBTEnUHFMTTlSvkBfBlTSx/" + "ArBlJBChmMwsWi3fU4ek+WJDvjF7AhFPUcNX" + "4kaAAAAAAAAAAAAAAAAAJ37Hglt9pn14Z9Vgj9pE3L7fXbBAAkcCTgIBIAJIAkkCASACSgJLAIO/" + "z+IwR9x5RqPSfAzguJqFxanKeUhZQgFsmKwj4GuAK2WAAAAAAAAA" + "AAAAAAB7G3oHXwv9lQmh8vd3TonVSERFqMAAgr+jPzrhTYloKgTCsGgEFNx7OdH+sJ98etJnwrIVSsFxHwAAAAAAAAAAAAAAAOsF4basDVdO8s8p/" + "fAcwLo9j5vxAIK/" + "n8LJGSxLhg32E0QLb7fZPphHZGiLJJFDrBMD8NcM15MAAAAAAAAAAAAAAADlTNYxyXvgdnFyrRaQRoiWLQnS/" + "gLFAbUl61s8X25tzWBr7nugeg7IMDUhKEm34FWUmcD2" + "utVNIR8VdL9iPRR4dwjF/" + "dVl4ymiWr+kkJXphEJvGbzwSXSAAAAAAAAAAAAAAAAAWZG0lbam3LV4+pciTNFehvbNeeLAAk0CTgIBIAJPAlAAMEO5rKAEO5rKADehIAPk" + "4cBAX14QA5iWgAIBIAJRAlIAg7/T7quzPdTpPcCght7xTpoi+g9Sw7gtkYDSyaOh0qHc0AAAAAAAAAAAAAAAADavGw+/" + "CvXTnyDIJ6fZU+llAiixQAIBIAJTAlQCASAC" + "WwJcAgEgAlUCVgCBv1wad2ywThLttxU0gcwWuSJSuLNadPm8j3J85ggRzjkGAAAAAAAAAAAAAAAB1xLrLNteGQzkOClxdvv3E/" + "l3M5UAgb8JuDCFQxifbIdTfjd1x7Mq" + "S+Z7dzIUkHtIdVjcVeFT2AAAAAAAAAAAAAAAAiwal03Yl9B7p2fVDSCtlYsZX6m+AgEgAlcCWAIBIAJZAloAgb7jxvbib0yb3DKvQBDcHL/" + "hdg7NjCuqjUQ09t8hgmhV" + "oAAAAAAAAAAAAAAABEGpMZGoNId5F80sBzWgnjo+AP2UAIG+sE8ccijAbmkaBJVfyfgqY5pf4QSO+c5IFGVC9WwlY/" + "AAAAAAAAAAAAAAAAeg08QveVui23B9QhrdMd7a" + "nx/sGACBvqxwYOyAk+H0YGBc70gZFJc6oqUvcHywU+yJNBfSNh+AAAAAAAAAAAAAAAADFU5kDFbQI6mIkEJqJNGncvWjiygCASACXQJeAIG/" + "acxhhr+dznhtppGVCg+k" + "FqjL65rOddHn1mwyRj1rYgQAAAAAAAAAAAAAAACRfpTwfZ9v81WVbRpRYN+1/m9YhwCBvw9fhTm/" + "NqURBT4FuwJczZWe39F575hmpFtt8KVniCwIAAAAAAAAAAAAAAAB" + "DkxuMKeNKjBZpVAjNVjJ/URzwhoAgb8RuD3rFDyNUpuXtBAnWTykKVAuY7UKLrye419st2b25AAAAAAAAAAAAAAAAlUrmS7Amiwb/" + "77tvRUhnpfLLMXeL4vIgQ=="; constexpr td::int64 Ton = 1000000000; @@ -173,7 +317,7 @@ TEST(Emulator, wallet_int_and_ext_msg) { auto address = wallet->get_address(); - void *emulator = transaction_emulator_create(config_boc, 3); + void* emulator = transaction_emulator_create(config_boc, 3); const uint64_t lt = 42000000000; CHECK(transaction_emulator_set_lt(emulator, lt)); const uint32_t utime = 1337; @@ -185,7 +329,11 @@ TEST(Emulator, wallet_int_and_ext_msg) { { td::Ref account_root; block::gen::Account().cell_pack_account_none(account_root); - auto none_shard_account_cell = vm::CellBuilder().store_ref(account_root).store_bits(td::Bits256::zero().as_bitslice()).store_long(0).finalize(); + auto none_shard_account_cell = vm::CellBuilder() + .store_ref(account_root) + .store_bits(td::Bits256::zero().as_bitslice()) + .store_long(0) + .finalize(); auto none_shard_account_boc = td::base64_encode(std_boc_serialize(none_shard_account_cell).move_as_ok()); td::Ref int_msg; @@ -199,14 +347,15 @@ TEST(Emulator, wallet_int_and_ext_msg) { block::gen::MsgAddressInt::Record_addr_std src; src.anycast = vm::CellBuilder().store_zeroes(1).as_cellslice_ref(); src.workchain_id = 0; - src.address = td::Bits256();; + src.address = td::Bits256(); + ; tlb::csr_pack(msg_info.src, src); } { block::gen::MsgAddressInt::Record_addr_std dest; dest.anycast = vm::CellBuilder().store_zeroes(1).as_cellslice_ref(); dest.workchain_id = address.workchain; - dest.address = address.addr; + dest.address = address.addr; tlb::csr_pack(msg_info.dest, dest); } { @@ -226,11 +375,12 @@ TEST(Emulator, wallet_int_and_ext_msg) { msg_info.created_lt = 0; msg_info.created_at = static_cast(utime); tlb::csr_pack(message.info, msg_info); - message.init = vm::CellBuilder() - .store_ones(1) - .store_zeroes(1) - .append_cellslice(vm::load_cell_slice(ton::GenericAccount::get_init_state(wallet->get_state()))) - .as_cellslice_ref(); + message.init = + vm::CellBuilder() + .store_ones(1) + .store_zeroes(1) + .append_cellslice(vm::load_cell_slice(ton::GenericAccount::get_init_state(wallet->get_state()))) + .as_cellslice_ref(); message.body = vm::CellBuilder().store_zeroes(1).as_cellslice_ref(); tlb::type_pack_cell(int_msg, block::gen::t_Message_Any, message); @@ -240,7 +390,8 @@ TEST(Emulator, wallet_int_and_ext_msg) { auto int_msg_boc = td::base64_encode(std_boc_serialize(int_msg).move_as_ok()); - std::string int_emu_res = transaction_emulator_emulate_transaction(emulator, none_shard_account_boc.c_str(), int_msg_boc.c_str()); + std::string int_emu_res = + transaction_emulator_emulate_transaction(emulator, none_shard_account_boc.c_str(), int_msg_boc.c_str()); LOG(ERROR) << "int_emu_res = " << int_emu_res; auto int_result_json = td::json_decode(td::MutableSlice(int_emu_res)); @@ -253,7 +404,8 @@ TEST(Emulator, wallet_int_and_ext_msg) { auto success = success_field.move_as_ok().get_boolean(); CHECK(success); - auto transaction_field = td::get_json_object_field(int_result_obj, "transaction", td::JsonValue::Type::String, false); + auto transaction_field = + td::get_json_object_field(int_result_obj, "transaction", td::JsonValue::Type::String, false); CHECK(transaction_field.is_ok()); auto transaction_boc_b64 = transaction_field.move_as_ok().get_string(); auto transaction_boc = td::base64_decode(transaction_boc_b64); @@ -271,7 +423,8 @@ TEST(Emulator, wallet_int_and_ext_msg) { CHECK(trans.lt == lt); CHECK(trans.now == utime); - auto shard_account_field = td::get_json_object_field(int_result_obj, "shard_account", td::JsonValue::Type::String, false); + auto shard_account_field = + td::get_json_object_field(int_result_obj, "shard_account", td::JsonValue::Type::String, false); CHECK(shard_account_field.is_ok()); auto shard_account_boc_b64 = shard_account_field.move_as_ok().get_string(); shard_account_after_boc_b64 = shard_account_boc_b64.str(); @@ -281,7 +434,8 @@ TEST(Emulator, wallet_int_and_ext_msg) { CHECK(shard_account_cell.is_ok()); block::gen::ShardAccount::Record shard_account; block::gen::Account::Record_account account; - CHECK(tlb::unpack_cell(shard_account_cell.move_as_ok(), shard_account) && tlb::unpack_cell(shard_account.account, account)); + CHECK(tlb::unpack_cell(shard_account_cell.move_as_ok(), shard_account) && + tlb::unpack_cell(shard_account.account, account)); CHECK(shard_account.last_trans_hash == trans_hash); CHECK(shard_account.last_trans_lt == lt); ton::WorkchainId wc; @@ -290,26 +444,29 @@ TEST(Emulator, wallet_int_and_ext_msg) { CHECK(address.workchain == wc); CHECK(address.addr == addr); } - + // emulate external message { - auto ext_body = wallet->make_a_gift_message(priv_key, utime + 60, {ton::WalletV3::Gift{block::StdAddress(0, ton::StdSmcAddress()), 1 * Ton}}); + auto ext_body = wallet->make_a_gift_message( + priv_key, utime + 60, {ton::WalletV3::Gift{block::StdAddress(0, ton::StdSmcAddress()), 1 * Ton}}); CHECK(ext_body.is_ok()); auto ext_msg = ton::GenericAccount::create_ext_message(address, {}, ext_body.move_as_ok()); auto ext_msg_boc = td::base64_encode(std_boc_serialize(ext_msg).move_as_ok()); - std::string ext_emu_res = transaction_emulator_emulate_transaction(emulator, shard_account_after_boc_b64.c_str(), ext_msg_boc.c_str()); + std::string ext_emu_res = + transaction_emulator_emulate_transaction(emulator, shard_account_after_boc_b64.c_str(), ext_msg_boc.c_str()); LOG(ERROR) << "ext_emu_res = " << ext_emu_res; auto ext_result_json = td::json_decode(td::MutableSlice(ext_emu_res)); CHECK(ext_result_json.is_ok()); auto ext_result = ext_result_json.move_as_ok(); - auto &ext_result_obj = ext_result.get_object(); + auto& ext_result_obj = ext_result.get_object(); auto ext_success_field = td::get_json_object_field(ext_result_obj, "success", td::JsonValue::Type::Boolean, false); CHECK(ext_success_field.is_ok()); auto ext_success = ext_success_field.move_as_ok().get_boolean(); CHECK(ext_success); - auto ext_transaction_field = td::get_json_object_field(ext_result_obj, "transaction", td::JsonValue::Type::String, false); + auto ext_transaction_field = + td::get_json_object_field(ext_result_obj, "transaction", td::JsonValue::Type::String, false); CHECK(ext_transaction_field.is_ok()); auto ext_transaction_boc_b64 = ext_transaction_field.move_as_ok().get_string(); auto ext_transaction_boc = td::base64_decode(ext_transaction_boc_b64); @@ -319,13 +476,15 @@ TEST(Emulator, wallet_int_and_ext_msg) { td::Bits256 ext_trans_hash = ext_trans_cell.ok()->get_hash().bits(); block::gen::Transaction::Record ext_trans; block::gen::TransactionDescr::Record_trans_ord ext_trans_descr; - CHECK(tlb::unpack_cell(ext_trans_cell.move_as_ok(), ext_trans) && tlb::unpack_cell(ext_trans.description, ext_trans_descr)); + CHECK(tlb::unpack_cell(ext_trans_cell.move_as_ok(), ext_trans) && + tlb::unpack_cell(ext_trans.description, ext_trans_descr)); CHECK(ext_trans.outmsg_cnt == 1); CHECK(ext_trans.account_addr == wallet->get_address().addr); CHECK(ext_trans_descr.aborted == false); CHECK(ext_trans_descr.destroyed == false); - auto ext_shard_account_field = td::get_json_object_field(ext_result_obj, "shard_account", td::JsonValue::Type::String, false); + auto ext_shard_account_field = + td::get_json_object_field(ext_result_obj, "shard_account", td::JsonValue::Type::String, false); CHECK(ext_shard_account_field.is_ok()); auto ext_shard_account_boc_b64 = ext_shard_account_field.move_as_ok().get_string(); auto ext_shard_account_boc = td::base64_decode(ext_shard_account_boc_b64); @@ -334,7 +493,8 @@ TEST(Emulator, wallet_int_and_ext_msg) { CHECK(ext_shard_account_cell.is_ok()); block::gen::ShardAccount::Record ext_shard_account; block::gen::Account::Record_account ext_account; - CHECK(tlb::unpack_cell(ext_shard_account_cell.move_as_ok(), ext_shard_account) && tlb::unpack_cell(ext_shard_account.account, ext_account)); + CHECK(tlb::unpack_cell(ext_shard_account_cell.move_as_ok(), ext_shard_account) && + tlb::unpack_cell(ext_shard_account.account, ext_account)); CHECK(ext_shard_account.last_trans_hash == ext_trans_hash); CHECK(ext_shard_account.last_trans_lt == ext_trans.lt); ton::WorkchainId wc; @@ -359,7 +519,7 @@ TEST(Emulator, tvm_emulator) { auto data = ton::WalletV3::get_init_data(init_data); auto data_boc_b64 = td::base64_encode(std_boc_serialize(data).move_as_ok()); - void *tvm_emulator = tvm_emulator_create(code_boc_b64.c_str(), data_boc_b64.c_str(), 1); + void* tvm_emulator = tvm_emulator_create(code_boc_b64.c_str(), data_boc_b64.c_str(), 1); unsigned method_crc = td::crc16("seqno"); unsigned method_id = (method_crc & 0xffff) | 0x10000; auto stack = td::make_ref(); @@ -402,7 +562,8 @@ TEST(Emulator, tvm_emulator) { } TEST(Emulator, tvm_emulator_extra_currencies) { - void *tvm_emulator = tvm_emulator_create("te6cckEBBAEAHgABFP8A9KQT9LzyyAsBAgFiAgMABtBfBAAJofpP8E8XmGlj", "te6cckEBAQEAAgAAAEysuc0=", 1); + void* tvm_emulator = tvm_emulator_create("te6cckEBBAEAHgABFP8A9KQT9LzyyAsBAgFiAgMABtBfBAAJofpP8E8XmGlj", + "te6cckEBAQEAAgAAAEysuc0=", 1); std::string addr = "0:" + std::string(64, 'F'); tvm_emulator_set_c7(tvm_emulator, addr.c_str(), 1337, 1000, std::string(64, 'F').c_str(), nullptr); CHECK(tvm_emulator_set_extra_currencies(tvm_emulator, "100=20000 200=1")); diff --git a/emulator/transaction-emulator.cpp b/emulator/transaction-emulator.cpp index abdf0d2cfa..414970e96d 100644 --- a/emulator/transaction-emulator.cpp +++ b/emulator/transaction-emulator.cpp @@ -1,8 +1,10 @@ #include -#include "transaction-emulator.h" + #include "crypto/common/refcnt.hpp" -#include "vm/vm.h" #include "tdutils/td/utils/Time.h" +#include "vm/vm.h" + +#include "transaction-emulator.h" using td::Ref; using namespace std::string_literals; @@ -10,151 +12,154 @@ using namespace std::string_literals; namespace emulator { td::Result> TransactionEmulator::emulate_transaction( block::Account&& account, td::Ref msg_root, ton::UnixTime utime, ton::LogicalTime lt, int trans_type) { + td::Ref old_mparams; + std::vector storage_prices; + block::StoragePhaseConfig storage_phase_cfg{&storage_prices}; + block::ComputePhaseConfig compute_phase_cfg; + block::ActionPhaseConfig action_phase_cfg; + block::SerializeConfig serialize_config; + td::RefInt256 masterchain_create_fee, basechain_create_fee; + + if (!utime) { + utime = unixtime_; + } + if (!utime) { + utime = (unsigned)std::time(nullptr); + } - td::Ref old_mparams; - std::vector storage_prices; - block::StoragePhaseConfig storage_phase_cfg{&storage_prices}; - block::ComputePhaseConfig compute_phase_cfg; - block::ActionPhaseConfig action_phase_cfg; - block::SerializeConfig serialize_config; - td::RefInt256 masterchain_create_fee, basechain_create_fee; - - if (!utime) { - utime = unixtime_; - } - if (!utime) { - utime = (unsigned)std::time(nullptr); - } - - auto fetch_res = block::FetchConfigParams::fetch_config_params( - *config_, prev_blocks_info_, &old_mparams, &storage_prices, &storage_phase_cfg, &rand_seed_, &compute_phase_cfg, - &action_phase_cfg, &serialize_config, &masterchain_create_fee, &basechain_create_fee, account.workchain, utime); - if(fetch_res.is_error()) { - return fetch_res.move_as_error_prefix("cannot fetch config params "); - } + auto fetch_res = block::FetchConfigParams::fetch_config_params( + *config_, prev_blocks_info_, &old_mparams, &storage_prices, &storage_phase_cfg, &rand_seed_, &compute_phase_cfg, + &action_phase_cfg, &serialize_config, &masterchain_create_fee, &basechain_create_fee, account.workchain, utime); + if (fetch_res.is_error()) { + return fetch_res.move_as_error_prefix("cannot fetch config params "); + } - TRY_STATUS(vm::init_vm(debug_enabled_)); + TRY_STATUS(vm::init_vm(debug_enabled_)); - if (!lt) { - lt = lt_; - } - if (!lt) { - lt = (account.last_trans_lt_ / block::ConfigInfo::get_lt_align() + 1) * block::ConfigInfo::get_lt_align(); // next block after account_.last_trans_lt_ - } - account.block_lt = lt - lt % block::ConfigInfo::get_lt_align(); + if (!lt) { + lt = lt_; + } + if (!lt) { + lt = (account.last_trans_lt_ / block::ConfigInfo::get_lt_align() + 1) * + block::ConfigInfo::get_lt_align(); // next block after account_.last_trans_lt_ + } + account.block_lt = lt - lt % block::ConfigInfo::get_lt_align(); - compute_phase_cfg.libraries = std::make_unique(libraries_); - compute_phase_cfg.ignore_chksig = ignore_chksig_; - compute_phase_cfg.with_vm_log = true; - compute_phase_cfg.vm_log_verbosity = vm_log_verbosity_; + compute_phase_cfg.libraries = std::make_unique(libraries_); + compute_phase_cfg.ignore_chksig = ignore_chksig_; + compute_phase_cfg.with_vm_log = true; + compute_phase_cfg.vm_log_verbosity = vm_log_verbosity_; - double start_time = td::Time::now(); - auto res = create_transaction(msg_root, &account, utime, lt, trans_type, - &storage_phase_cfg, &compute_phase_cfg, - &action_phase_cfg); - double elapsed = td::Time::now() - start_time; + double start_time = td::Time::now(); + auto res = create_transaction(msg_root, &account, utime, lt, trans_type, &storage_phase_cfg, &compute_phase_cfg, + &action_phase_cfg); + double elapsed = td::Time::now() - start_time; - if(res.is_error()) { - return res.move_as_error_prefix("cannot run message on account "); - } - std::unique_ptr trans = res.move_as_ok(); + if (res.is_error()) { + return res.move_as_error_prefix("cannot run message on account "); + } + std::unique_ptr trans = res.move_as_ok(); - if (!trans->compute_phase->accepted && trans->in_msg_extern) { - auto vm_log = trans->compute_phase->vm_log; - auto vm_exit_code = trans->compute_phase->exit_code; - return std::make_unique(std::move(vm_log), vm_exit_code, elapsed); - } + if (!trans->compute_phase->accepted && trans->in_msg_extern) { + auto vm_log = trans->compute_phase->vm_log; + auto vm_exit_code = trans->compute_phase->exit_code; + return std::make_unique(std::move(vm_log), vm_exit_code, + elapsed); + } - if (!trans->serialize(serialize_config)) { - return td::Status::Error(-669,"cannot serialize new transaction for smart contract "s + trans->account.addr.to_hex()); - } + if (!trans->serialize(serialize_config)) { + return td::Status::Error(-669, + "cannot serialize new transaction for smart contract "s + trans->account.addr.to_hex()); + } - auto trans_root = trans->commit(account); - if (trans_root.is_null()) { - return td::Status::Error(PSLICE() << "cannot commit new transaction for smart contract"); - } + auto trans_root = trans->commit(account); + if (trans_root.is_null()) { + return td::Status::Error(PSLICE() << "cannot commit new transaction for smart contract"); + } - return std::make_unique(std::move(trans_root), std::move(account), - std::move(trans->compute_phase->vm_log), std::move(trans->compute_phase->actions), elapsed); + return std::make_unique(std::move(trans_root), std::move(account), + std::move(trans->compute_phase->vm_log), + std::move(trans->compute_phase->actions), elapsed); } -td::Result TransactionEmulator::emulate_transaction(block::Account&& account, td::Ref original_trans) { +td::Result TransactionEmulator::emulate_transaction( + block::Account&& account, td::Ref original_trans) { + block::gen::Transaction::Record record_trans; + if (!tlb::unpack_cell(original_trans, record_trans)) { + return td::Status::Error("Failed to unpack Transaction"); + } - block::gen::Transaction::Record record_trans; - if (!tlb::unpack_cell(original_trans, record_trans)) { - return td::Status::Error("Failed to unpack Transaction"); + ton::LogicalTime lt = record_trans.lt; + ton::UnixTime utime = record_trans.now; + account.now_ = utime; + account.block_lt = record_trans.lt - record_trans.lt % block::ConfigInfo::get_lt_align(); + td::Ref msg_root = record_trans.r1.in_msg->prefetch_ref(); + int tag = block::gen::t_TransactionDescr.get_tag(vm::load_cell_slice(record_trans.description)); + + int trans_type = block::transaction::Transaction::tr_none; + switch (tag) { + case block::gen::TransactionDescr::trans_ord: { + trans_type = block::transaction::Transaction::tr_ord; + break; } - - ton::LogicalTime lt = record_trans.lt; - ton::UnixTime utime = record_trans.now; - account.now_ = utime; - account.block_lt = record_trans.lt - record_trans.lt % block::ConfigInfo::get_lt_align(); - td::Ref msg_root = record_trans.r1.in_msg->prefetch_ref(); - int tag = block::gen::t_TransactionDescr.get_tag(vm::load_cell_slice(record_trans.description)); - - int trans_type = block::transaction::Transaction::tr_none; - switch (tag) { - case block::gen::TransactionDescr::trans_ord: { - trans_type = block::transaction::Transaction::tr_ord; - break; - } - case block::gen::TransactionDescr::trans_storage: { - trans_type = block::transaction::Transaction::tr_storage; - break; - } - case block::gen::TransactionDescr::trans_tick_tock: { - block::gen::TransactionDescr::Record_trans_tick_tock tick_tock; - if (!tlb::unpack_cell(record_trans.description, tick_tock)) { - return td::Status::Error("Failed to unpack tick tock transaction description"); - } - trans_type = tick_tock.is_tock ? block::transaction::Transaction::tr_tock : block::transaction::Transaction::tr_tick; - break; - } - case block::gen::TransactionDescr::trans_split_prepare: { - trans_type = block::transaction::Transaction::tr_split_prepare; - break; - } - case block::gen::TransactionDescr::trans_split_install: { - trans_type = block::transaction::Transaction::tr_split_install; - break; - } - case block::gen::TransactionDescr::trans_merge_prepare: { - trans_type = block::transaction::Transaction::tr_merge_prepare; - break; - } - case block::gen::TransactionDescr::trans_merge_install: { - trans_type = block::transaction::Transaction::tr_merge_install; - break; + case block::gen::TransactionDescr::trans_storage: { + trans_type = block::transaction::Transaction::tr_storage; + break; + } + case block::gen::TransactionDescr::trans_tick_tock: { + block::gen::TransactionDescr::Record_trans_tick_tock tick_tock; + if (!tlb::unpack_cell(record_trans.description, tick_tock)) { + return td::Status::Error("Failed to unpack tick tock transaction description"); } + trans_type = + tick_tock.is_tock ? block::transaction::Transaction::tr_tock : block::transaction::Transaction::tr_tick; + break; + } + case block::gen::TransactionDescr::trans_split_prepare: { + trans_type = block::transaction::Transaction::tr_split_prepare; + break; + } + case block::gen::TransactionDescr::trans_split_install: { + trans_type = block::transaction::Transaction::tr_split_install; + break; } + case block::gen::TransactionDescr::trans_merge_prepare: { + trans_type = block::transaction::Transaction::tr_merge_prepare; + break; + } + case block::gen::TransactionDescr::trans_merge_install: { + trans_type = block::transaction::Transaction::tr_merge_install; + break; + } + } - TRY_RESULT(emulation, emulate_transaction(std::move(account), msg_root, utime, lt, trans_type)); - - if (auto emulation_result_ptr = dynamic_cast(emulation.get())) { - auto& emulation_result = *emulation_result_ptr; - - if (td::Bits256(emulation_result.transaction->get_hash().bits()) != td::Bits256(original_trans->get_hash().bits())) { - return td::Status::Error("transaction hash mismatch"); - } + TRY_RESULT(emulation, emulate_transaction(std::move(account), msg_root, utime, lt, trans_type)); - if (!check_state_update(emulation_result.account, record_trans)) { - return td::Status::Error("account hash mismatch"); - } + if (auto emulation_result_ptr = dynamic_cast(emulation.get())) { + auto& emulation_result = *emulation_result_ptr; - return std::move(emulation_result); + if (td::Bits256(emulation_result.transaction->get_hash().bits()) != + td::Bits256(original_trans->get_hash().bits())) { + return td::Status::Error("transaction hash mismatch"); + } - } else if (auto emulation_not_accepted_ptr = dynamic_cast(emulation.get())) { - return td::Status::Error( PSTRING() - << "VM Log: " << emulation_not_accepted_ptr->vm_log - << ", VM Exit Code: " << emulation_not_accepted_ptr->vm_exit_code - << ", Elapsed Time: " << emulation_not_accepted_ptr->elapsed_time); - } else { - return td::Status::Error("emulation failed"); + if (!check_state_update(emulation_result.account, record_trans)) { + return td::Status::Error("account hash mismatch"); } -} -td::Result TransactionEmulator::emulate_transactions_chain(block::Account&& account, std::vector>&& original_transactions) { + return std::move(emulation_result); + + } else if (auto emulation_not_accepted_ptr = dynamic_cast(emulation.get())) { + return td::Status::Error(PSTRING() << "VM Log: " << emulation_not_accepted_ptr->vm_log + << ", VM Exit Code: " << emulation_not_accepted_ptr->vm_exit_code + << ", Elapsed Time: " << emulation_not_accepted_ptr->elapsed_time); + } else { + return td::Status::Error("emulation failed"); + } +} +td::Result TransactionEmulator::emulate_transactions_chain( + block::Account&& account, std::vector>&& original_transactions) { std::vector> emulated_transactions; for (const auto& original_trans : original_transactions) { if (original_trans.is_null()) { @@ -166,21 +171,20 @@ td::Result TransactionEmulator::emulate_tra account = std::move(emulation_result.account); } - return TransactionEmulator::EmulationChain{ std::move(emulated_transactions), std::move(account) }; + return TransactionEmulator::EmulationChain{std::move(emulated_transactions), std::move(account)}; } -bool TransactionEmulator::check_state_update(const block::Account& account, const block::gen::Transaction::Record& trans) { +bool TransactionEmulator::check_state_update(const block::Account& account, + const block::gen::Transaction::Record& trans) { block::gen::HASH_UPDATE::Record hash_update; return tlb::type_unpack_cell(trans.state_update, block::gen::t_HASH_UPDATE_Account, hash_update) && - hash_update.new_hash == account.total_state->get_hash().bits(); + hash_update.new_hash == account.total_state->get_hash().bits(); } td::Result> TransactionEmulator::create_transaction( - td::Ref msg_root, block::Account* acc, - ton::UnixTime utime, ton::LogicalTime lt, int trans_type, - block::StoragePhaseConfig* storage_phase_cfg, - block::ComputePhaseConfig* compute_phase_cfg, - block::ActionPhaseConfig* action_phase_cfg) { + td::Ref msg_root, block::Account* acc, ton::UnixTime utime, ton::LogicalTime lt, int trans_type, + block::StoragePhaseConfig* storage_phase_cfg, block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg) { bool external{false}, ihr_delivered{false}, need_credit_phase{false}; if (msg_root.not_null()) { @@ -200,47 +204,54 @@ td::Result> TransactionEmulator if (msg_root.not_null() && !trans->unpack_input_msg(ihr_delivered, action_phase_cfg)) { if (external) { // inbound external message was not accepted - return td::Status::Error(-701,"inbound external message rejected by account "s + acc->addr.to_hex() + - " before smart-contract execution"); + return td::Status::Error(-701, "inbound external message rejected by account "s + acc->addr.to_hex() + + " before smart-contract execution"); } - return td::Status::Error(-669,"cannot unpack input message for a new transaction"); + return td::Status::Error(-669, "cannot unpack input message for a new transaction"); } if (trans->bounce_enabled) { if (!trans->prepare_storage_phase(*storage_phase_cfg, true)) { - return td::Status::Error(-669,"cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + return td::Status::Error( + -669, "cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); } if (need_credit_phase && !trans->prepare_credit_phase()) { - return td::Status::Error(-669,"cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + return td::Status::Error( + -669, "cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); } } else { if (need_credit_phase && !trans->prepare_credit_phase()) { - return td::Status::Error(-669,"cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + return td::Status::Error( + -669, "cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); } if (!trans->prepare_storage_phase(*storage_phase_cfg, true, need_credit_phase)) { - return td::Status::Error(-669,"cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + return td::Status::Error( + -669, "cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); } } if (!trans->prepare_compute_phase(*compute_phase_cfg)) { - return td::Status::Error(-669,"cannot create compute phase of a new transaction for smart contract "s + acc->addr.to_hex()); + return td::Status::Error( + -669, "cannot create compute phase of a new transaction for smart contract "s + acc->addr.to_hex()); } if (!trans->compute_phase->accepted) { if (!external && trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { - return td::Status::Error(-669,"new ordinary transaction for smart contract "s + acc->addr.to_hex() + - " has not been accepted by the smart contract (?)"); + return td::Status::Error(-669, "new ordinary transaction for smart contract "s + acc->addr.to_hex() + + " has not been accepted by the smart contract (?)"); } } if (trans->compute_phase->success && !trans->prepare_action_phase(*action_phase_cfg)) { - return td::Status::Error(-669,"cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex()); + return td::Status::Error( + -669, "cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex()); } - if (trans->bounce_enabled - && (!trans->compute_phase->success || trans->action_phase->state_exceeds_limits || trans->action_phase->bounce) - && !trans->prepare_bounce_phase(*action_phase_cfg)) { - return td::Status::Error(-669,"cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex()); + if (trans->bounce_enabled && + (!trans->compute_phase->success || trans->action_phase->state_exceeds_limits || trans->action_phase->bounce) && + !trans->prepare_bounce_phase(*action_phase_cfg)) { + return td::Status::Error( + -669, "cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex()); } return trans; @@ -266,7 +277,7 @@ void TransactionEmulator::set_config(std::shared_ptr config) { config_ = std::move(config); } -void TransactionEmulator::set_libs(vm::Dictionary &&libs) { +void TransactionEmulator::set_libs(vm::Dictionary&& libs) { libraries_ = std::forward(libs); } @@ -278,4 +289,4 @@ void TransactionEmulator::set_prev_blocks_info(td::Ref prev_blocks_in prev_blocks_info_ = std::move(prev_blocks_info); } -} // namespace emulator +} // namespace emulator diff --git a/emulator/transaction-emulator.h b/emulator/transaction-emulator.h index 848548c32b..58589aee29 100644 --- a/emulator/transaction-emulator.h +++ b/emulator/transaction-emulator.h @@ -1,11 +1,11 @@ #pragma once -#include "crypto/common/refcnt.hpp" -#include "ton/ton-types.h" -#include "crypto/vm/cells.h" -#include "block/transaction.h" #include "block/block-auto.h" #include "block/block-parse.h" #include "block/mc-config.h" +#include "block/transaction.h" +#include "crypto/common/refcnt.hpp" +#include "crypto/vm/cells.h" +#include "ton/ton-types.h" namespace emulator { class TransactionEmulator { @@ -19,17 +19,24 @@ class TransactionEmulator { bool debug_enabled_; td::Ref prev_blocks_info_; -public: - TransactionEmulator(std::shared_ptr config, int vm_log_verbosity = 0) : - config_(std::move(config)), libraries_(256), vm_log_verbosity_(vm_log_verbosity), - unixtime_(0), lt_(0), rand_seed_(td::BitArray<256>::zero()), ignore_chksig_(false), debug_enabled_(false) { + public: + TransactionEmulator(std::shared_ptr config, int vm_log_verbosity = 0) + : config_(std::move(config)) + , libraries_(256) + , vm_log_verbosity_(vm_log_verbosity) + , unixtime_(0) + , lt_(0) + , rand_seed_(td::BitArray<256>::zero()) + , ignore_chksig_(false) + , debug_enabled_(false) { } struct EmulationResult { std::string vm_log; double elapsed_time; - EmulationResult(std::string vm_log_, double elapsed_time_) : vm_log(vm_log_), elapsed_time(elapsed_time_) {} + EmulationResult(std::string vm_log_, double elapsed_time_) : vm_log(vm_log_), elapsed_time(elapsed_time_) { + } virtual ~EmulationResult() = default; }; @@ -47,12 +54,12 @@ class TransactionEmulator { } }; - struct EmulationExternalNotAccepted: EmulationResult { + struct EmulationExternalNotAccepted : EmulationResult { int vm_exit_code; - EmulationExternalNotAccepted(std::string vm_log_, int vm_exit_code_, double elapsed_time_) : - EmulationResult(vm_log_, elapsed_time_), vm_exit_code(vm_exit_code_) - {} + EmulationExternalNotAccepted(std::string vm_log_, int vm_exit_code_, double elapsed_time_) + : EmulationResult(vm_log_, elapsed_time_), vm_exit_code(vm_exit_code_) { + } }; struct EmulationChain { @@ -68,29 +75,29 @@ class TransactionEmulator { return unixtime_; } - td::Result> emulate_transaction( - block::Account&& account, td::Ref msg_root, ton::UnixTime utime, ton::LogicalTime lt, int trans_type); + td::Result> emulate_transaction(block::Account&& account, td::Ref msg_root, + ton::UnixTime utime, ton::LogicalTime lt, + int trans_type); td::Result emulate_transaction(block::Account&& account, td::Ref original_trans); - td::Result emulate_transactions_chain(block::Account&& account, std::vector>&& original_transactions); + td::Result emulate_transactions_chain(block::Account&& account, + std::vector>&& original_transactions); void set_unixtime(ton::UnixTime unixtime); void set_lt(ton::LogicalTime lt); void set_rand_seed(td::BitArray<256>& rand_seed); void set_ignore_chksig(bool ignore_chksig); void set_config(std::shared_ptr config); - void set_libs(vm::Dictionary &&libs); + void set_libs(vm::Dictionary&& libs); void set_debug_enabled(bool debug_enabled); void set_prev_blocks_info(td::Ref prev_blocks_info); -private: + private: bool check_state_update(const block::Account& account, const block::gen::Transaction::Record& trans); td::Result> create_transaction( - td::Ref msg_root, block::Account* acc, - ton::UnixTime utime, ton::LogicalTime lt, int trans_type, - block::StoragePhaseConfig* storage_phase_cfg, - block::ComputePhaseConfig* compute_phase_cfg, - block::ActionPhaseConfig* action_phase_cfg); + td::Ref msg_root, block::Account* acc, ton::UnixTime utime, ton::LogicalTime lt, int trans_type, + block::StoragePhaseConfig* storage_phase_cfg, block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg); }; -} // namespace emulator +} // namespace emulator diff --git a/emulator/tvm-emulator.hpp b/emulator/tvm-emulator.hpp index acc13627c0..55c0ca6aa4 100644 --- a/emulator/tvm-emulator.hpp +++ b/emulator/tvm-emulator.hpp @@ -5,10 +5,11 @@ namespace emulator { class TvmEmulator { ton::SmartContract smc_; ton::SmartContract::Args args_; -public: + + public: using Answer = ton::SmartContract::Answer; - TvmEmulator(td::Ref code, td::Ref data): smc_({code, data}) { + TvmEmulator(td::Ref code, td::Ref data) : smc_({code, data}) { } void set_vm_verbosity_level(int vm_log_verbosity) { @@ -23,7 +24,8 @@ class TvmEmulator { args_.set_limits(vm::GasLimits(limit)); } - void set_c7(block::StdAddress address, uint32_t unixtime, uint64_t balance, td::BitArray<256> rand_seed, std::shared_ptr config) { + void set_c7(block::StdAddress address, uint32_t unixtime, uint64_t balance, td::BitArray<256> rand_seed, + std::shared_ptr config) { args_.set_address(std::move(address)); args_.set_now(unixtime); args_.set_balance(balance); @@ -67,4 +69,4 @@ class TvmEmulator { return smc_.send_internal_message(message_body, args.set_amount(amount)); } }; -} \ No newline at end of file +} // namespace emulator \ No newline at end of file diff --git a/example/android/native-lib.cpp b/example/android/native-lib.cpp index 5e079c0427..03262ee8e9 100644 --- a/example/android/native-lib.cpp +++ b/example/android/native-lib.cpp @@ -16,18 +16,16 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include -#include #include #include +#include +#include #include +#include "auto/tl/tonlib_api.h" #include "tl/tl_jni_object.h" - -#include "tonlib/tonlib_client_json.h" - #include "tonlib/Client.h" -#include "auto/tl/tonlib_api.h" +#include "tonlib/tonlib_client_json.h" extern "C" JNIEXPORT jstring JNICALL Java_drinkless_org_tonlib_MainActivity_stringFromJNI(JNIEnv *env, jobject /* this */, diff --git a/fec/fec.cpp b/fec/fec.cpp index f6379fd743..575d105619 100644 --- a/fec/fec.cpp +++ b/fec/fec.cpp @@ -16,10 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "fec.h" -#include "td/utils/overloaded.h" #include "auto/tl/ton_api.hpp" #include "td/utils/misc.h" +#include "td/utils/overloaded.h" + +#include "fec.h" namespace ton { diff --git a/fec/fec.h b/fec/fec.h index 89bc12a2fc..d8462748e9 100644 --- a/fec/fec.h +++ b/fec/fec.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/fec/fec.h" #include "auto/tl/ton_api.h" +#include "td/fec/fec.h" #include "td/utils/Variant.h" namespace ton { diff --git a/git.h b/git.h index b79c8bce63..b0253819c0 100644 --- a/git.h +++ b/git.h @@ -2,7 +2,7 @@ #include class GitMetadata { -public: + public: // Is the metadata populated? We may not have metadata if // there wasn't a .git directory (e.g. downloaded source // code without revision history). diff --git a/http/http-client.cpp b/http/http-client.cpp index 6c43e3ebcc..e33cbbcb25 100644 --- a/http/http-client.cpp +++ b/http/http-client.cpp @@ -16,10 +16,10 @@ Copyright 2019-2020 Telegram Systems LLP */ -#include "http-client.hpp" - #include "td/utils/Random.h" +#include "http-client.hpp" + namespace ton { namespace http { diff --git a/http/http-client.h b/http/http-client.h index 9ec80be469..2c2db20f1e 100644 --- a/http/http-client.h +++ b/http/http-client.h @@ -18,10 +18,10 @@ */ #pragma once -#include "http.h" - -#include "td/utils/port/IPAddress.h" #include "td/actor/actor.h" +#include "td/utils/port/IPAddress.h" + +#include "http.h" namespace ton { diff --git a/http/http-client.hpp b/http/http-client.hpp index abb48867a6..7e528b4ca6 100644 --- a/http/http-client.hpp +++ b/http/http-client.hpp @@ -16,11 +16,11 @@ Copyright 2019-2020 Telegram Systems LLP */ +#include "td/utils/Random.h" + #include "http-client.h" #include "http-outbound-connection.h" -#include "td/utils/Random.h" - namespace ton { namespace http { diff --git a/http/http-connection.h b/http/http-connection.h index 840ac8a1b1..8db50884c4 100644 --- a/http/http-connection.h +++ b/http/http-connection.h @@ -18,11 +18,11 @@ */ #pragma once +#include "common/errorcode.h" #include "td/actor/actor.h" -#include "td/utils/port/SocketFd.h" -#include "td/utils/buffer.h" #include "td/utils/BufferedFd.h" -#include "common/errorcode.h" +#include "td/utils/buffer.h" +#include "td/utils/port/SocketFd.h" #include "http.h" diff --git a/http/http-inbound-connection.cpp b/http/http-inbound-connection.cpp index 57064dddc3..9422af8736 100644 --- a/http/http-inbound-connection.cpp +++ b/http/http-inbound-connection.cpp @@ -16,9 +16,10 @@ Copyright 2019-2020 Telegram Systems LLP */ -#include "http-inbound-connection.h" #include "td/utils/misc.h" +#include "http-inbound-connection.h" + namespace ton { namespace http { diff --git a/http/http-inbound-connection.h b/http/http-inbound-connection.h index e360226756..ab06fec088 100644 --- a/http/http-inbound-connection.h +++ b/http/http-inbound-connection.h @@ -18,9 +18,9 @@ */ #pragma once -#include "http.h" #include "http-connection.h" #include "http-server.h" +#include "http.h" namespace ton { diff --git a/http/http-outbound-connection.cpp b/http/http-outbound-connection.cpp index 63621df875..4da3466e29 100644 --- a/http/http-outbound-connection.cpp +++ b/http/http-outbound-connection.cpp @@ -16,10 +16,10 @@ Copyright 2019-2020 Telegram Systems LLP */ -#include "http-outbound-connection.h" - #include "td/utils/port/StdStreams.h" +#include "http-outbound-connection.h" + namespace ton { namespace http { diff --git a/http/http-outbound-connection.h b/http/http-outbound-connection.h index a3d7525833..15d673d10a 100644 --- a/http/http-outbound-connection.h +++ b/http/http-outbound-connection.h @@ -18,12 +18,12 @@ */ #pragma once -#include "http.h" -#include "http-connection.h" -#include "http-client.h" - #include +#include "http-client.h" +#include "http-connection.h" +#include "http.h" + namespace ton { namespace http { diff --git a/http/http-proxy.cpp b/http/http-proxy.cpp index 2e027f9feb..add2408db8 100644 --- a/http/http-proxy.cpp +++ b/http/http-proxy.cpp @@ -25,15 +25,15 @@ Copyright 2019-2020 Telegram Systems LLP */ -#include "http/http-server.h" -#include "http/http-client.h" +#include +#include -#include "td/utils/port/signals.h" -#include "td/utils/OptionParser.h" +#include "http/http-client.h" +#include "http/http-server.h" #include "td/utils/FileLog.h" +#include "td/utils/OptionParser.h" +#include "td/utils/port/signals.h" -#include -#include #include "git.h" #if TD_DARWIN || TD_LINUX @@ -266,7 +266,8 @@ int main(int argc, char *argv[]) { SET_VERBOSITY_LEVEL(v); }); p.add_option('V', "version", "shows http-proxy build version", [&]() { - std::cout << "http-proxy build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "http-proxy build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_option('h', "help", "prints_help", [&]() { diff --git a/http/http-server.cpp b/http/http-server.cpp index ac4be1b3d8..e79817b61e 100644 --- a/http/http-server.cpp +++ b/http/http-server.cpp @@ -16,8 +16,8 @@ Copyright 2019-2020 Telegram Systems LLP */ -#include "http-server.h" #include "http-inbound-connection.h" +#include "http-server.h" namespace ton { diff --git a/http/http-server.h b/http/http-server.h index 257308458e..903c386a23 100644 --- a/http/http-server.h +++ b/http/http-server.h @@ -19,10 +19,10 @@ #pragma once #include "td/actor/actor.h" -#include "http.h" - #include "td/net/TcpListener.h" +#include "http.h" + namespace ton { namespace http { diff --git a/http/http.cpp b/http/http.cpp index dd8ab80931..da9d99a6a0 100644 --- a/http/http.cpp +++ b/http/http.cpp @@ -16,11 +16,11 @@ Copyright 2019-2020 Telegram Systems LLP */ -#include "http.h" +#include #include "td/utils/misc.h" -#include +#include "http.h" namespace ton { @@ -755,8 +755,8 @@ td::Result> HttpResponse::create(std::string proto return td::Status::Error(PSTRING() << "bad status code '" << code << "'"); } - return std::make_unique(std::move(proto_version), code, std::move(reason), force_no_payload, - keep_alive, is_tunnel); + return std::make_unique(std::move(proto_version), code, std::move(reason), force_no_payload, keep_alive, + is_tunnel); } td::Status HttpResponse::complete_parse_header() { diff --git a/http/http.h b/http/http.h index 8e109455e8..21e10e625e 100644 --- a/http/http.h +++ b/http/http.h @@ -18,14 +18,14 @@ */ #pragma once -#include "td/utils/buffer.h" -#include "auto/tl/ton_api.h" -#include "td/actor/PromiseFuture.h" - -#include #include +#include #include +#include "auto/tl/ton_api.h" +#include "td/actor/PromiseFuture.h" +#include "td/utils/buffer.h" + namespace ton { namespace http { diff --git a/keyring/keyring.cpp b/keyring/keyring.cpp index a31d173ecf..58659356b3 100644 --- a/keyring/keyring.cpp +++ b/keyring/keyring.cpp @@ -16,12 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "keyring.hpp" #include "common/errorcode.h" #include "common/io.hpp" -#include "td/utils/port/path.h" -#include "td/utils/filesystem.h" #include "td/utils/Random.h" +#include "td/utils/filesystem.h" +#include "td/utils/port/path.h" + +#include "keyring.hpp" namespace ton { @@ -43,7 +44,7 @@ void KeyringImpl::start_up() { } } -td::Result KeyringImpl::load_key(PublicKeyHash key_hash) { +td::Result KeyringImpl::load_key(PublicKeyHash key_hash) { auto it = map_.find(key_hash); if (it != map_.end()) { return it->second.get(); diff --git a/keyring/keyring.h b/keyring/keyring.h index 3b9064a7bf..aa4eb4fd8b 100644 --- a/keyring/keyring.h +++ b/keyring/keyring.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/actor/actor.h" #include "keys/keys.hpp" +#include "td/actor/actor.h" namespace ton { diff --git a/keyring/keyring.hpp b/keyring/keyring.hpp index eca9073a5f..7876dab647 100644 --- a/keyring/keyring.hpp +++ b/keyring/keyring.hpp @@ -18,10 +18,11 @@ */ #pragma once -#include "keyring.h" +#include + #include "keys/encryptor.h" -#include +#include "keyring.h" namespace ton { @@ -73,4 +74,3 @@ class KeyringImpl : public Keyring { } // namespace keyring } // namespace ton - diff --git a/keys/encryptor.cpp b/keys/encryptor.cpp index 8fef9a0952..880827b1e9 100644 --- a/keys/encryptor.cpp +++ b/keys/encryptor.cpp @@ -16,15 +16,14 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "auto/tl/ton_api.hpp" +#include "common/errorcode.h" +#include "common/status.h" #include "td/utils/crypto.h" #include "td/utils/overloaded.h" #include "encryptor.h" #include "encryptor.hpp" -#include "auto/tl/ton_api.hpp" - -#include "common/status.h" -#include "common/errorcode.h" #include "keys.hpp" namespace ton { diff --git a/keys/encryptor.h b/keys/encryptor.h index 818c97d630..5c80e1f580 100644 --- a/keys/encryptor.h +++ b/keys/encryptor.h @@ -18,11 +18,11 @@ */ #pragma once +#include "auto/tl/ton_api.h" +#include "td/actor/PromiseFuture.h" #include "td/actor/actor.h" -#include "td/utils/buffer.h" #include "td/utils/Status.h" -#include "td/actor/PromiseFuture.h" -#include "auto/tl/ton_api.h" +#include "td/utils/buffer.h" namespace ton { diff --git a/keys/encryptor.hpp b/keys/encryptor.hpp index bcc841dc83..18129a6514 100644 --- a/keys/encryptor.hpp +++ b/keys/encryptor.hpp @@ -18,11 +18,12 @@ */ #pragma once -#include "encryptor.h" -#include "crypto/Ed25519.h" #include "auto/tl/ton_api.h" +#include "crypto/Ed25519.h" #include "tl-utils/tl-utils.hpp" +#include "encryptor.h" + namespace ton { class EncryptorNone : public Encryptor { diff --git a/keys/keys.cpp b/keys/keys.cpp index 01afb26d2f..39801c7b99 100644 --- a/keys/keys.cpp +++ b/keys/keys.cpp @@ -16,13 +16,14 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "keys.hpp" #include "auto/tl/ton_api.hpp" +#include "crypto/Ed25519.h" #include "td/utils/overloaded.h" #include "tl-utils/tl-utils.hpp" + #include "encryptor.h" #include "encryptor.hpp" -#include "crypto/Ed25519.h" +#include "keys.hpp" namespace ton { diff --git a/keys/keys.hpp b/keys/keys.hpp index 2e517b2e45..77e93cf6ce 100644 --- a/keys/keys.hpp +++ b/keys/keys.hpp @@ -18,15 +18,15 @@ */ #pragma once -#include "td/utils/int_types.h" -#include "td/utils/buffer.h" #include "auto/tl/ton_api.h" +#include "common/errorcode.h" +#include "crypto/Ed25519.h" +#include "crypto/common/bitstring.h" +#include "td/actor/actor.h" #include "td/utils/UInt.h" #include "td/utils/Variant.h" -#include "td/actor/actor.h" -#include "crypto/common/bitstring.h" -#include "crypto/Ed25519.h" -#include "common/errorcode.h" +#include "td/utils/buffer.h" +#include "td/utils/int_types.h" namespace ton { diff --git a/lite-client/ext-client.cpp b/lite-client/ext-client.cpp index d6b7982226..948dd9f0de 100644 --- a/lite-client/ext-client.cpp +++ b/lite-client/ext-client.cpp @@ -14,10 +14,11 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "ext-client.h" #include "td/utils/Random.h" #include "ton/ton-shard.h" +#include "ext-client.h" + namespace liteclient { class ExtClientImpl : public ExtClient { diff --git a/lite-client/ext-client.h b/lite-client/ext-client.h index ef4523fd69..215276b709 100644 --- a/lite-client/ext-client.h +++ b/lite-client/ext-client.h @@ -15,9 +15,10 @@ along with TON Blockchain Library. If not, see . */ #pragma once +#include "adnl/adnl-ext-client.h" #include "td/actor/actor.h" #include "ton/ton-types.h" -#include "adnl/adnl-ext-client.h" + #include "query-utils.hpp" namespace liteclient { diff --git a/lite-client/lite-client-common.cpp b/lite-client/lite-client-common.cpp index 5bc8ca7380..58709927a6 100644 --- a/lite-client/lite-client-common.cpp +++ b/lite-client/lite-client-common.cpp @@ -16,13 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "lite-client-common.h" - #include "auto/tl/lite_api.hpp" +#include "td/utils/Random.h" +#include "td/utils/overloaded.h" #include "tl-utils/lite-utils.hpp" #include "ton/lite-tl.hpp" -#include "td/utils/overloaded.h" -#include "td/utils/Random.h" + +#include "lite-client-common.h" using namespace std::literals::string_literals; diff --git a/lite-client/lite-client-common.h b/lite-client/lite-client-common.h index a73dab91d0..a437557b9f 100644 --- a/lite-client/lite-client-common.h +++ b/lite-client/lite-client-common.h @@ -18,9 +18,8 @@ */ #pragma once -#include "crypto/block/block.h" - #include "auto/tl/lite_api.hpp" +#include "crypto/block/block.h" namespace liteclient { diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 659c71c0c4..bbc91f39c3 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -25,41 +25,41 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "lite-client.h" - -#include "lite-client-common.h" - -#include "tl-utils/lite-utils.hpp" -#include "auto/tl/ton_api_json.h" #include "auto/tl/lite_api.hpp" +#include "auto/tl/ton_api_json.h" +#include "block/block-auto.h" +#include "block/block-db.h" +#include "block/block-parse.h" +#include "block/block.h" +#include "block/check-proof.h" +#include "block/mc-config.h" +#include "common/checksum.h" +#include "crypto/common/util.h" +#include "crypto/vm/utils.h" #include "td/utils/OptionParser.h" -#include "td/utils/Time.h" -#include "td/utils/filesystem.h" #include "td/utils/Random.h" +#include "td/utils/Time.h" #include "td/utils/crypto.h" -#include "td/utils/port/signals.h" +#include "td/utils/filesystem.h" #include "td/utils/port/FileFd.h" +#include "td/utils/port/signals.h" +#include "tl-utils/lite-utils.hpp" #include "ton/lite-tl.hpp" -#include "block/block-db.h" -#include "block/block.h" -#include "block/block-parse.h" -#include "block/block-auto.h" -#include "block/mc-config.h" -#include "block/check-proof.h" #include "vm/boc.h" #include "vm/cellops.h" #include "vm/cells/MerkleProof.h" -#include "vm/vm.h" #include "vm/cp0.h" #include "vm/memo.h" -#include "crypto/vm/utils.h" -#include "crypto/common/util.h" -#include "common/checksum.h" +#include "vm/vm.h" + +#include "lite-client-common.h" +#include "lite-client.h" #if TD_DARWIN || TD_LINUX #include #endif #include + #include "git.h" using namespace std::literals::string_literals; @@ -426,7 +426,7 @@ void TestNode::got_server_mc_block_id(ton::BlockIdExt blkid, ton::ZeroStateIdExt if (time >= created) { td::TerminalIO::out() << " created at " << created << " (" << time - created << " seconds ago)\n"; } else { - td::TerminalIO::out() << " created at " << created << " (" << created - time << " seconds in the future)\n"; + td::TerminalIO::out() << " created at " << created << " (" << created - time << " seconds in the future)\n"; } } else { td::TerminalIO::out() << "\n"; @@ -3820,16 +3820,16 @@ void TestNode::continue_check_validator_load3(std::unique_ptr( block::Config::unpack_catchain_validators_config(info1->config->get_config_param(28))); - load_validator_shard_shares( - start_seqno, end_seqno, std::move(validator_set), std::move(catchain_config), - [=, this, info1 = std::move(info1), - info2 = std::move(info2)](td::Result> R) mutable { - if (R.is_error()) { - LOG(ERROR) << "failed to load validator shard shares: " << R.move_as_error(); - } else { - continue_check_validator_load4(std::move(info1), std::move(info2), mode, file_pfx, R.move_as_ok()); - } - }); + load_validator_shard_shares(start_seqno, end_seqno, std::move(validator_set), std::move(catchain_config), + [=, this, info1 = std::move(info1), + info2 = std::move(info2)](td::Result> R) mutable { + if (R.is_error()) { + LOG(ERROR) << "failed to load validator shard shares: " << R.move_as_error(); + } else { + continue_check_validator_load4(std::move(info1), std::move(info2), mode, file_pfx, + R.move_as_ok()); + } + }); } else { continue_check_validator_load4(std::move(info1), std::move(info2), mode, std::move(file_pfx), {}); } @@ -4041,7 +4041,7 @@ void TestNode::load_validator_shard_shares_cont(std::shared_ptrcatchain_config, shard, state->validator_set, cc_seqno); - for (const auto &val : val_set) { + for (const auto& val : val_set) { result[val.key.as_bits256()] += blocks_count; } }; @@ -4069,7 +4069,7 @@ void TestNode::load_validator_shard_shares_cont(std::shared_ptrpromise.set_error(e.as_status("cannot parse shard hashes: ")); return; } @@ -4106,7 +4106,7 @@ bool compute_punishment_default(int interval, bool severe, td::RefInt256& fine, fine = td::make_refint(101 * 1000000000LL); // 101 fine_part = 0; - return true; // todo: (tolya-yanot) temporary reduction of fine + return true; // todo: (tolya-yanot) temporary reduction of fine if (severe) { fine = td::make_refint(2500 * 1000000000LL); // GR$2500 @@ -4128,41 +4128,49 @@ bool compute_punishment_default(int interval, bool severe, td::RefInt256& fine, return true; } -bool compute_punishment(int interval, bool severe, td::RefInt256& fine, unsigned& fine_part, Ref punishment_params) { - if(punishment_params.is_null()) { +bool compute_punishment(int interval, bool severe, td::RefInt256& fine, unsigned& fine_part, + Ref punishment_params) { + if (punishment_params.is_null()) { return compute_punishment_default(interval, severe, fine, fine_part); } block::gen::MisbehaviourPunishmentConfig::Record rec; if (!tlb::unpack_cell(punishment_params, rec)) { - return false; + return false; } - if(interval <= rec.unpunishable_interval) { - return false; + if (interval <= rec.unpunishable_interval) { + return false; } fine = block::tlb::t_Grams.as_integer(rec.default_flat_fine); fine_part = rec.default_proportional_fine; if (severe) { - fine = fine * rec.severity_flat_mult; fine >>= 8; - fine_part = fine_part * rec.severity_proportional_mult; fine_part >>= 8; + fine = fine * rec.severity_flat_mult; + fine >>= 8; + fine_part = fine_part * rec.severity_proportional_mult; + fine_part >>= 8; } if (interval >= rec.long_interval) { - fine = fine * rec.long_flat_mult; fine >>= 8; - fine_part = fine_part * rec.long_proportional_mult; fine_part >>= 8; + fine = fine * rec.long_flat_mult; + fine >>= 8; + fine_part = fine_part * rec.long_proportional_mult; + fine_part >>= 8; return true; } if (interval >= rec.medium_interval) { - fine = fine * rec.medium_flat_mult; fine >>= 8; - fine_part = fine_part * rec.medium_proportional_mult; fine_part >>= 8; + fine = fine * rec.medium_flat_mult; + fine >>= 8; + fine_part = fine_part * rec.medium_proportional_mult; + fine_part >>= 8; return true; } return true; } -bool check_punishment(int interval, bool severe, td::RefInt256 fine, unsigned fine_part, Ref punishment_params) { +bool check_punishment(int interval, bool severe, td::RefInt256 fine, unsigned fine_part, + Ref punishment_params) { td::RefInt256 computed_fine; unsigned computed_fine_part; return compute_punishment(interval, severe, computed_fine, computed_fine_part, punishment_params) && @@ -4198,7 +4206,7 @@ td::Status TestNode::write_val_create_proof(TestNode::ValidatorLoadInfo& info1, int severity = (severe ? 2 : 1); td::RefInt256 fine = td::make_refint(101000000000); - unsigned fine_part = 0; // todo: (tolya-yanot) temporary reduction of fine // 0xffffffff / 16; // 1/16 + unsigned fine_part = 0; // todo: (tolya-yanot) temporary reduction of fine // 0xffffffff / 16; // 1/16 if (!compute_punishment(interval, severe, fine, fine_part, punishment_params)) { return td::Status::Error("cannot compute adequate punishment"); } @@ -4513,7 +4521,8 @@ td::Status TestNode::continue_check_validator_load_proof(std::unique_ptrconfig->get_config_param(40))) { + if (!check_punishment(interval, severe, suggested_fine, rec.suggested_fine_part, + info2->config->get_config_param(40))) { LOG(ERROR) << "proposed punishment (fine " << td::dec_string(suggested_fine) << ", fine_part=" << (double)rec.suggested_fine_part / (1LL << 32) << " is too harsh"; show_vote(root->get_hash().bits(), false); @@ -4662,7 +4671,8 @@ int main(int argc, char* argv[]) { return (verbosity >= 0 && verbosity <= 9) ? td::Status::OK() : td::Status::Error("verbosity must be 0..9"); }); p.add_option('V', "version", "shows lite-client build information", [&]() { - std::cout << "lite-client build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "lite-client build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 4adb803801..7353edf453 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -26,17 +26,18 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "ext-client.h" #include "adnl/adnl-ext-client.h" +#include "auto/tl/lite_api.h" +#include "block/block.h" +#include "block/mc-config.h" +#include "td/utils/filesystem.h" +#include "terminal/terminal.h" #include "tl-utils/tl-utils.hpp" #include "ton/ton-types.h" -#include "terminal/terminal.h" #include "vm/cells.h" #include "vm/stack.hpp" -#include "block/block.h" -#include "block/mc-config.h" -#include "td/utils/filesystem.h" -#include "auto/tl/lite_api.h" + +#include "ext-client.h" using td::Ref; diff --git a/lite-client/query-utils.cpp b/lite-client/query-utils.cpp index 7c9e7cfc73..2c20da6a0b 100644 --- a/lite-client/query-utils.cpp +++ b/lite-client/query-utils.cpp @@ -14,20 +14,19 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "query-utils.hpp" - -#include "block-parse.h" -#include "td/utils/overloaded.h" -#include "tl-utils/common-utils.hpp" +#include -#include "block/block-auto.h" #include "auto/tl/lite_api.hpp" +#include "block/block-auto.h" #include "overlay/overlay-broadcast.hpp" +#include "td/utils/overloaded.h" +#include "tl-utils/common-utils.hpp" #include "tl-utils/lite-utils.hpp" #include "ton/lite-tl.hpp" #include "ton/ton-shard.h" -#include +#include "block-parse.h" +#include "query-utils.hpp" namespace liteclient { diff --git a/lite-client/query-utils.hpp b/lite-client/query-utils.hpp index d214246951..8b733553db 100644 --- a/lite-client/query-utils.hpp +++ b/lite-client/query-utils.hpp @@ -15,10 +15,10 @@ along with TON Blockchain Library. If not, see . */ #pragma once -#include "ton/ton-types.h" +#include "adnl/adnl-node-id.hpp" #include "auto/tl/lite_api.h" #include "td/utils/port/IPAddress.h" -#include "adnl/adnl-node-id.hpp" +#include "ton/ton-types.h" namespace liteclient { diff --git a/memprof/memprof/memprof.cpp b/memprof/memprof/memprof.cpp index ce2554e1e8..f16908471b 100644 --- a/memprof/memprof/memprof.cpp +++ b/memprof/memprof/memprof.cpp @@ -18,7 +18,6 @@ */ #undef NDEBUG #include "memprof/memprof.h" - #include "td/utils/port/platform.h" #if (TD_DARWIN || TD_LINUX) && defined(USE_MEMPROF) @@ -29,14 +28,13 @@ #include #include #include +#include +#include #include #include #include #include -#include -#include - bool is_memprof_on() { return true; } diff --git a/overlay/overlay-broadcast.cpp b/overlay/overlay-broadcast.cpp index 615b3e7c81..308257b517 100644 --- a/overlay/overlay-broadcast.cpp +++ b/overlay/overlay-broadcast.cpp @@ -16,16 +16,17 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "overlay-broadcast.hpp" #include "adnl/adnl-node-id.hpp" #include "common/util.h" -#include "overlay.hpp" #include "keys/encryptor.h" #include "td/actor/PromiseFuture.h" #include "td/actor/actor.h" #include "td/utils/Status.h" #include "td/utils/port/Stat.h" +#include "overlay-broadcast.hpp" +#include "overlay.hpp" + namespace ton { namespace overlay { diff --git a/overlay/overlay-broadcast.hpp b/overlay/overlay-broadcast.hpp index e7b39eecc0..4a77e082e0 100644 --- a/overlay/overlay-broadcast.hpp +++ b/overlay/overlay-broadcast.hpp @@ -101,7 +101,8 @@ class BroadcastSimple : public td::ListNode { void update_overlay(OverlayImpl *overlay); void broadcast_checked(td::Result R); - static td::Status create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr broadcast); + static td::Status create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + tl_object_ptr broadcast); static td::Status create_new(td::actor::ActorId overlay, td::actor::ActorId keyring, PublicKeyHash local_id, td::BufferSlice data, td::uint32 flags); diff --git a/overlay/overlay-fec-broadcast.cpp b/overlay/overlay-fec-broadcast.cpp index 5a0ad10ddf..c67f613bb7 100644 --- a/overlay/overlay-fec-broadcast.cpp +++ b/overlay/overlay-fec-broadcast.cpp @@ -16,9 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "keys/encryptor.h" + #include "overlay-fec-broadcast.hpp" #include "overlay.hpp" -#include "keys/encryptor.h" namespace ton { diff --git a/overlay/overlay-fec-broadcast.hpp b/overlay/overlay-fec-broadcast.hpp index 85de648e3d..e56a0aa676 100644 --- a/overlay/overlay-fec-broadcast.hpp +++ b/overlay/overlay-fec-broadcast.hpp @@ -18,13 +18,13 @@ */ #pragma once +#include + #include "auto/tl/ton_api.h" +#include "common/checksum.h" +#include "fec/fec.h" #include "overlay/overlay.h" #include "td/utils/List.h" -#include "fec/fec.h" -#include "common/checksum.h" - -#include namespace ton { diff --git a/overlay/overlay-fec.cpp b/overlay/overlay-fec.cpp index 817d3b7c24..42e7cc6a85 100644 --- a/overlay/overlay-fec.cpp +++ b/overlay/overlay-fec.cpp @@ -16,9 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "adnl/utils.hpp" + #include "overlay-fec.hpp" #include "overlay.hpp" -#include "adnl/utils.hpp" namespace ton { diff --git a/overlay/overlay-fec.hpp b/overlay/overlay-fec.hpp index b72e830ea8..65169bfd11 100644 --- a/overlay/overlay-fec.hpp +++ b/overlay/overlay-fec.hpp @@ -18,8 +18,9 @@ */ #pragma once -#include "overlay-manager.h" #include "fec/fec.h" + +#include "overlay-manager.h" #include "overlay.h" namespace ton { diff --git a/overlay/overlay-id.hpp b/overlay/overlay-id.hpp index 2625773bd7..06ea5795bd 100644 --- a/overlay/overlay-id.hpp +++ b/overlay/overlay-id.hpp @@ -18,17 +18,18 @@ */ #pragma once -#include "auto/tl/ton_api.h" +#include +#include + #include "adnl/adnl-node-id.hpp" +#include "auto/tl/ton_api.h" +#include "keys/encryptor.h" #include "overlay/overlays.h" #include "td/utils/SharedSlice.h" #include "td/utils/buffer.h" #include "td/utils/overloaded.h" -#include "keys/encryptor.h" #include "td/utils/port/StdStreams.h" #include "td/utils/unique_ptr.h" -#include -#include namespace ton { diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index f24c6cbc2c..4fd649bd32 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -16,24 +16,22 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "overlay-manager.h" -#include "auto/tl/ton_api.h" -#include "auto/tl/ton_api.hpp" -#include "overlay.h" +#include #include "adnl/utils.hpp" +#include "auto/tl/ton_api.h" +#include "auto/tl/ton_api.hpp" #include "td/actor/actor.h" #include "td/actor/common.h" -#include "td/utils/Random.h" - #include "td/db/RocksDb.h" - +#include "td/utils/Random.h" #include "td/utils/Status.h" #include "td/utils/buffer.h" #include "td/utils/overloaded.h" - #include "td/utils/port/Poll.h" -#include + +#include "overlay-manager.h" +#include "overlay.h" namespace ton { diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index 68b033a3b2..667cf6e04f 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -20,14 +20,13 @@ #include -#include "td/actor/actor.h" -#include "td/db/KeyValueAsync.h" - #include "adnl/adnl.h" #include "dht/dht.h" +#include "td/actor/actor.h" +#include "td/db/KeyValueAsync.h" -#include "overlays.h" #include "overlay-id.hpp" +#include "overlays.h" namespace ton { @@ -57,9 +56,8 @@ class OverlayManager : public Overlays { OverlayOptions opts) override; void create_semiprivate_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::vector root_public_keys, - OverlayMemberCertificate certificate, - std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, - OverlayOptions opts) override; + OverlayMemberCertificate certificate, std::unique_ptr callback, + OverlayPrivacyRules rules, td::string scope, OverlayOptions opts) override; void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules, std::string scope) override; diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index 52cc40cf61..58587c667a 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -16,15 +16,17 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include +#include + #include "adnl/adnl-node-id.hpp" #include "adnl/adnl-node.h" #include "auto/tl/ton_api.h" -#include "overlay.hpp" #include "td/utils/Status.h" #include "td/utils/Time.h" #include "td/utils/port/signals.h" -#include -#include + +#include "overlay.hpp" namespace ton { @@ -398,8 +400,8 @@ void OverlayImpl::send_random_peers_v2(adnl::AdnlNodeIdShort src, td::Promise R) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), peer, timer = td::Timer(), oid = print_id()](td::Result R) { if (R.is_error()) { VLOG(OVERLAY_INFO) << oid << " ping to " << peer << " failed : " << R.move_as_error(); return; @@ -430,8 +432,8 @@ void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) { continue; } - if (overlay_type_ != OverlayType::FixedMemberList && X->get_version() <= td::Clocks::system() - - Overlays::overlay_peer_ttl()) { + if (overlay_type_ != OverlayType::FixedMemberList && + X->get_version() <= td::Clocks::system() - Overlays::overlay_peer_ttl()) { if (X->is_permanent_member()) { del_from_neighbour_list(X); } else { @@ -680,8 +682,7 @@ size_t OverlayImpl::neighbours_cnt() const { void OverlayImpl::update_root_member_list(std::vector ids, std::vector root_public_keys, OverlayMemberCertificate cert) { - auto expected_size = - (td::uint32)(ids.size() + root_public_keys.size() * opts_.max_slaves_in_semiprivate_overlay_); + auto expected_size = (td::uint32)(ids.size() + root_public_keys.size() * opts_.max_slaves_in_semiprivate_overlay_); opts_.max_peers_ = std::max(opts_.max_peers_, expected_size); std::sort(ids.begin(), ids.end()); auto old_root_public_keys = std::move(peer_list_.root_public_keys_); diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index cad18bedcb..d653f4a4d1 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -16,21 +16,20 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "auto/tl/ton_api.h" -#include "td/utils/Random.h" -#include "common/delay.h" +#include #include "adnl/utils.hpp" -#include "dht/dht.h" - -#include "overlay.hpp" +#include "auto/tl/ton_api.h" #include "auto/tl/ton_api.hpp" - +#include "common/delay.h" +#include "dht/dht.h" #include "keys/encryptor.h" +#include "td/utils/Random.h" #include "td/utils/Status.h" #include "td/utils/StringBuilder.h" #include "td/utils/port/signals.h" -#include + +#include "overlay.hpp" namespace ton { @@ -526,7 +525,7 @@ td::Status OverlayImpl::check_date(td::uint32 date) { return td::Status::OK(); } -BroadcastCheckResult OverlayImpl::check_source_eligible(const PublicKeyHash& source, const Certificate* cert, +BroadcastCheckResult OverlayImpl::check_source_eligible(const PublicKeyHash &source, const Certificate *cert, td::uint32 size, bool is_fec) { if (size == 0) { return BroadcastCheckResult::Forbidden; @@ -543,15 +542,15 @@ BroadcastCheckResult OverlayImpl::check_source_eligible(const PublicKeyHash& sou /* skip_check_signature = */ cached); if (r2 != BroadcastCheckResult::Forbidden) { if (cached_cert == checked_certificates_cache_.end()) { - cached_cert = checked_certificates_cache_.emplace( - source, std::make_unique(source, cert_hash)).first; + cached_cert = + checked_certificates_cache_.emplace(source, std::make_unique(source, cert_hash)).first; } else { cached_cert->second->cert_hash = cert_hash; cached_cert->second->remove(); } checked_certificates_cache_lru_.put(cached_cert->second.get()); while (checked_certificates_cache_.size() > max_checked_certificates_cache_size_) { - auto to_remove = (CachedCertificate*)checked_certificates_cache_lru_.get(); + auto to_remove = (CachedCertificate *)checked_certificates_cache_lru_.get(); CHECK(to_remove); to_remove->remove(); checked_certificates_cache_.erase(to_remove->source); @@ -561,7 +560,7 @@ BroadcastCheckResult OverlayImpl::check_source_eligible(const PublicKeyHash& sou return broadcast_check_result_max(r, r2); } -BroadcastCheckResult OverlayImpl::check_source_eligible(PublicKey source, const Certificate* cert, td::uint32 size, +BroadcastCheckResult OverlayImpl::check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, bool is_fec) { return check_source_eligible(source.compute_short_id(), cert, size, is_fec); } diff --git a/overlay/overlay.h b/overlay/overlay.h index bfab498775..56c76bbf32 100644 --- a/overlay/overlay.h +++ b/overlay/overlay.h @@ -18,14 +18,12 @@ */ #pragma once +#include "adnl/adnl.h" #include "auto/tl/ton_api.h" +#include "td/actor/actor.h" #include "td/utils/buffer.h" #include "td/utils/int_types.h" -#include "td/actor/actor.h" - -#include "adnl/adnl.h" - #include "overlay-manager.h" namespace ton { diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 7727f5f1fb..64c2116001 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -19,37 +19,35 @@ #pragma once #include -#include -#include #include +#include +#include #include #include -#include +#include #include "adnl/adnl-node-id.hpp" -#include "overlay.h" -#include "overlay-manager.h" -#include "overlay-fec.hpp" -#include "overlay-broadcast.hpp" -#include "overlay-fec-broadcast.hpp" -#include "overlay-id.hpp" - +#include "adnl/utils.hpp" +#include "auto/tl/ton_api.h" +#include "auto/tl/ton_api.hpp" +#include "fec/fec.h" +#include "keys/encryptor.h" #include "td/utils/DecTree.h" #include "td/utils/List.h" #include "td/utils/Status.h" #include "td/utils/Time.h" #include "td/utils/buffer.h" #include "td/utils/overloaded.h" -#include "fec/fec.h" - -#include "adnl/utils.hpp" -#include "keys/encryptor.h" - -#include "auto/tl/ton_api.h" -#include "auto/tl/ton_api.hpp" #include "td/utils/port/signals.h" #include "tl-utils/common-utils.hpp" +#include "overlay-broadcast.hpp" +#include "overlay-fec-broadcast.hpp" +#include "overlay-fec.hpp" +#include "overlay-id.hpp" +#include "overlay-manager.h" +#include "overlay.h" + namespace ton { namespace overlay { @@ -476,9 +474,7 @@ class OverlayImpl : public Overlay { OverlayOptions opts_; struct CachedCertificate : td::ListNode { - CachedCertificate(PublicKeyHash source, td::Bits256 cert_hash) - : source(source) - , cert_hash(cert_hash) { + CachedCertificate(PublicKeyHash source, td::Bits256 cert_hash) : source(source), cert_hash(cert_hash) { } PublicKeyHash source; diff --git a/overlay/overlays.h b/overlay/overlays.h index 972e48cce6..5e15520a93 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -18,19 +18,18 @@ */ #pragma once +#include + #include "adnl/adnl-node-id.hpp" #include "adnl/adnl.h" #include "auto/tl/ton_api.h" #include "dht/dht.h" - #include "td/actor/PromiseFuture.h" #include "td/actor/actor.h" #include "td/utils/Status.h" #include "td/utils/buffer.h" #include "td/utils/common.h" -#include - namespace ton { namespace overlay { @@ -171,7 +170,7 @@ class OverlayMemberCertificate { expire_at_ = std::numeric_limits::max(); } OverlayMemberCertificate(PublicKey signed_by, td::uint32 flags, td::int32 slot, td::int32 expire_at, - td::BufferSlice signature) + td::BufferSlice signature) : signed_by_(std::move(signed_by)) , flags_(flags) , slot_(slot) @@ -258,7 +257,6 @@ class OverlayMemberCertificate { td::SharedSlice signature_; }; - struct OverlayOptions { bool announce_self_ = true; bool frequent_dht_lookup_ = false; @@ -323,9 +321,8 @@ class Overlays : public td::actor::Actor { virtual void create_semiprivate_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::vector root_public_keys, - OverlayMemberCertificate certificate, - std::unique_ptr callback, OverlayPrivacyRules rules, - td::string scope, OverlayOptions opts) = 0; + OverlayMemberCertificate certificate, std::unique_ptr callback, + OverlayPrivacyRules rules, td::string scope, OverlayOptions opts) = 0; virtual void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules, std::string scope) = 0; diff --git a/rldp-http-proxy/DNSResolver.cpp b/rldp-http-proxy/DNSResolver.cpp index 850c38ccc8..f0369d0c37 100644 --- a/rldp-http-proxy/DNSResolver.cpp +++ b/rldp-http-proxy/DNSResolver.cpp @@ -23,9 +23,10 @@ exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. */ -#include "DNSResolver.h" -#include "td/utils/overloaded.h" #include "common/delay.h" +#include "td/utils/overloaded.h" + +#include "DNSResolver.h" static const double CACHE_TIMEOUT_HARD = 300.0; static const double CACHE_TIMEOUT_SOFT = 270.0; @@ -78,19 +79,18 @@ void DNSResolver::resolve(std::string host, td::Promise promise) { auto obj = R.move_as_ok(); std::string result; if (!obj->entries_.empty()) { - tonlib_api::downcast_call(*obj->entries_[0]->entry_, - td::overloaded( - [&](tonlib_api::dns_entryDataAdnlAddress &x) { - auto R = ton::adnl::AdnlNodeIdShort::parse(x.adnl_address_->adnl_address_); - if (R.is_ok()) { - ton::adnl::AdnlNodeIdShort id = R.move_as_ok(); - result = id.serialize() + ".adnl"; - } - }, - [&](tonlib_api::dns_entryDataStorageAddress &x) { - result = td::to_lower(x.bag_id_.to_hex()) + ".bag"; - }, - [&](auto &x) {})); + tonlib_api::downcast_call( + *obj->entries_[0]->entry_, + td::overloaded( + [&](tonlib_api::dns_entryDataAdnlAddress &x) { + auto R = ton::adnl::AdnlNodeIdShort::parse(x.adnl_address_->adnl_address_); + if (R.is_ok()) { + ton::adnl::AdnlNodeIdShort id = R.move_as_ok(); + result = id.serialize() + ".adnl"; + } + }, + [&](tonlib_api::dns_entryDataStorageAddress &x) { result = td::to_lower(x.bag_id_.to_hex()) + ".bag"; }, + [&](auto &x) {})); } if (result.empty()) { if (promise) { diff --git a/rldp-http-proxy/DNSResolver.h b/rldp-http-proxy/DNSResolver.h index 6cffe0bd8e..4afcc86453 100644 --- a/rldp-http-proxy/DNSResolver.h +++ b/rldp-http-proxy/DNSResolver.h @@ -24,10 +24,10 @@ from all source files in the program, then also delete it here. */ #pragma once -#include "td/actor/actor.h" -#include "tonlib/tonlib/TonlibClientWrapper.h" #include "adnl/adnl.h" #include "td/actor/PromiseFuture.h" +#include "td/actor/actor.h" +#include "tonlib/tonlib/TonlibClientWrapper.h" class DNSResolver : public td::actor::Actor { public: diff --git a/rldp-http-proxy/rldp-http-proxy.cpp b/rldp-http-proxy/rldp-http-proxy.cpp index 0d518d6d19..792da10ec1 100644 --- a/rldp-http-proxy/rldp-http-proxy.cpp +++ b/rldp-http-proxy/rldp-http-proxy.cpp @@ -25,39 +25,33 @@ Copyright 2019-2020 Telegram Systems LLP */ -#include "http/http-server.h" -#include "http/http-client.h" - -#include "td/utils/port/signals.h" -#include "td/utils/OptionParser.h" -#include "td/utils/FileLog.h" -#include "td/utils/Random.h" -#include "td/utils/filesystem.h" +#include +#include +#include +#include "adnl/adnl.h" #include "auto/tl/ton_api_json.h" #include "auto/tl/tonlib_api.hpp" - -#include "td/actor/MultiPromise.h" - +#include "common/delay.h" #include "common/errorcode.h" - -#include "tonlib/tonlib/TonlibClient.h" - -#include "adnl/adnl.h" +#include "dht/dht.h" +#include "http/http-client.h" +#include "http/http-server.h" #include "rldp/rldp.h" #include "rldp2/rldp.h" -#include "dht/dht.h" - -#include -#include -#include -#include "git.h" +#include "td/actor/MultiPromise.h" #include "td/utils/BufferedFd.h" -#include "common/delay.h" +#include "td/utils/FileLog.h" +#include "td/utils/OptionParser.h" +#include "td/utils/Random.h" +#include "td/utils/filesystem.h" #include "td/utils/port/path.h" - +#include "td/utils/port/signals.h" +#include "tonlib/tonlib/TonlibClient.h" #include "tonlib/tonlib/TonlibClientWrapper.h" + #include "DNSResolver.h" +#include "git.h" #if TD_DARWIN || TD_LINUX #include diff --git a/rldp/rldp-in.hpp b/rldp/rldp-in.hpp index 266f128a5a..f19dfb5bda 100644 --- a/rldp/rldp-in.hpp +++ b/rldp/rldp-in.hpp @@ -18,17 +18,16 @@ */ #pragma once -#include "rldp.hpp" -#include "rldp-peer.h" +#include +#include -#include "tl-utils/tl-utils.hpp" -#include "adnl/adnl-query.h" #include "adnl/adnl-peer-table.h" - +#include "adnl/adnl-query.h" #include "td/utils/List.h" +#include "tl-utils/tl-utils.hpp" -#include -#include +#include "rldp-peer.h" +#include "rldp.hpp" namespace ton { @@ -99,7 +98,8 @@ class RldpIn : public RldpImpl { void in_transfer_completed(TransferId transfer_id, bool success); void add_id(adnl::AdnlNodeIdShort local_id) override; - void get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, td::Promise promise) override; + void get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, + td::Promise promise) override; void set_default_mtu(td::uint64 mtu) override { default_mtu_ = mtu; diff --git a/rldp/rldp-peer.cpp b/rldp/rldp-peer.cpp index c599e9490b..574e90c792 100644 --- a/rldp/rldp-peer.cpp +++ b/rldp/rldp-peer.cpp @@ -16,12 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "rldp-peer.hpp" #include "adnl/utils.hpp" -#include "td/utils/Random.h" -#include "td/utils/overloaded.h" #include "auto/tl/ton_api.hpp" #include "common/io.hpp" +#include "td/utils/Random.h" +#include "td/utils/overloaded.h" + +#include "rldp-peer.hpp" namespace ton { diff --git a/rldp/rldp-peer.hpp b/rldp/rldp-peer.hpp index 8bcac5e66e..b51aa72b9f 100644 --- a/rldp/rldp-peer.hpp +++ b/rldp/rldp-peer.hpp @@ -18,11 +18,12 @@ */ #pragma once -#include "rldp-peer.h" +#include + #include "fec/fec.h" -#include "rldp.hpp" -#include +#include "rldp-peer.h" +#include "rldp.hpp" namespace ton { diff --git a/rldp/rldp.cpp b/rldp/rldp.cpp index 402740e19a..ceecde244d 100644 --- a/rldp/rldp.cpp +++ b/rldp/rldp.cpp @@ -16,11 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "rldp-in.hpp" #include "auto/tl/ton_api.h" #include "auto/tl/ton_api.hpp" -#include "td/utils/Random.h" #include "fec/fec.h" +#include "td/utils/Random.h" + +#include "rldp-in.hpp" namespace ton { @@ -260,7 +261,7 @@ void RldpIn::add_id(adnl::AdnlNodeIdShort local_id) { local_ids_.insert(local_id); } -void RldpIn::get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, td::Promise promise) { +void RldpIn::get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, td::Promise promise) { td::actor::send_closure(adnl_, &adnl::AdnlPeerTable::get_conn_ip_str, l_id, p_id, std::move(promise)); } diff --git a/rldp/rldp.hpp b/rldp/rldp.hpp index 978ae39b12..f75d0ca336 100644 --- a/rldp/rldp.hpp +++ b/rldp/rldp.hpp @@ -18,13 +18,13 @@ */ #pragma once -#include "rldp.h" -#include "rldp-peer.h" +#include -#include "tl-utils/tl-utils.hpp" #include "adnl/adnl-query.h" +#include "tl-utils/tl-utils.hpp" -#include +#include "rldp-peer.h" +#include "rldp.h" namespace ton { diff --git a/rldp2/Bbr.cpp b/rldp2/Bbr.cpp index b0f841a96b..093e7eb18a 100644 --- a/rldp2/Bbr.cpp +++ b/rldp2/Bbr.cpp @@ -17,13 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "Bbr.h" +#include "td/utils/Random.h" +#include "Bbr.h" #include "BdwStats.h" #include "RttStats.h" -#include "td/utils/Random.h" - namespace ton { namespace rldp2 { void Bbr::step(const RttStats &rtt_stats, const BdwStats &bdw_stats, td::uint64 in_flight, td::Timestamp now) { diff --git a/rldp2/Bbr.h b/rldp2/Bbr.h index 6d2acf70c5..9b977b14df 100644 --- a/rldp2/Bbr.h +++ b/rldp2/Bbr.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/utils/int_types.h" #include "td/utils/Time.h" +#include "td/utils/int_types.h" namespace ton { namespace rldp2 { @@ -45,4 +45,3 @@ struct Bbr { } // namespace rldp2 } // namespace ton - diff --git a/rldp2/FecHelper.cpp b/rldp2/FecHelper.cpp index 469ab234a6..ce06227833 100644 --- a/rldp2/FecHelper.cpp +++ b/rldp2/FecHelper.cpp @@ -17,10 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "FecHelper.h" - #include "td/utils/check.h" +#include "FecHelper.h" + namespace ton { namespace rldp2 { td::uint32 FecHelper::get_fec_symbols_count() const { diff --git a/rldp2/InboundTransfer.cpp b/rldp2/InboundTransfer.cpp index 1869d3a33d..e0b8499688 100644 --- a/rldp2/InboundTransfer.cpp +++ b/rldp2/InboundTransfer.cpp @@ -17,10 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "InboundTransfer.h" - #include "common/errorcode.h" +#include "InboundTransfer.h" + namespace ton { namespace rldp2 { size_t InboundTransfer::total_size() const { diff --git a/rldp2/InboundTransfer.h b/rldp2/InboundTransfer.h index c9fed8e4e8..ab1a94749b 100644 --- a/rldp2/InboundTransfer.h +++ b/rldp2/InboundTransfer.h @@ -19,14 +19,13 @@ #pragma once -#include "td/utils/optional.h" +#include #include "fec/fec.h" +#include "td/utils/optional.h" #include "RldpReceiver.h" -#include - namespace ton { namespace rldp2 { struct InboundTransfer { diff --git a/rldp2/LossSender.cpp b/rldp2/LossSender.cpp index 69ecde5947..48fbc3ed67 100644 --- a/rldp2/LossSender.cpp +++ b/rldp2/LossSender.cpp @@ -17,10 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "LossSender.h" - #include "td/utils/logging.h" +#include "LossSender.h" + #if TON_HAVE_GSL #include #endif diff --git a/rldp2/LossStats.cpp b/rldp2/LossStats.cpp index 62691a5187..fac63705bf 100644 --- a/rldp2/LossStats.cpp +++ b/rldp2/LossStats.cpp @@ -17,10 +17,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "LossStats.h" +#include + #include "td/utils/misc.h" -#include +#include "LossStats.h" namespace ton { namespace rldp2 { diff --git a/rldp2/LossStats.h b/rldp2/LossStats.h index a39e04de7c..e7e5deef3e 100644 --- a/rldp2/LossStats.h +++ b/rldp2/LossStats.h @@ -19,10 +19,10 @@ #pragma once -#include "LossSender.h" - #include "td/utils/int_types.h" +#include "LossSender.h" + namespace ton { namespace rldp2 { struct LossStats { diff --git a/rldp2/OutboundTransfer.h b/rldp2/OutboundTransfer.h index ab3e5e545c..613844b5d7 100644 --- a/rldp2/OutboundTransfer.h +++ b/rldp2/OutboundTransfer.h @@ -19,10 +19,11 @@ #pragma once -#include "RldpSender.h" +#include + #include "fec/fec.h" -#include +#include "RldpSender.h" namespace ton { namespace rldp2 { diff --git a/rldp2/Pacer.h b/rldp2/Pacer.h index cc50a6924d..ac39db4b2d 100644 --- a/rldp2/Pacer.h +++ b/rldp2/Pacer.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/utils/optional.h" #include "td/utils/Time.h" +#include "td/utils/optional.h" namespace ton { namespace rldp2 { diff --git a/rldp2/RldpConnection.cpp b/rldp2/RldpConnection.cpp index 47506178e4..38904d6ce8 100644 --- a/rldp2/RldpConnection.cpp +++ b/rldp2/RldpConnection.cpp @@ -17,20 +17,17 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "RldpConnection.h" -#include "rldp.hpp" - -#include "td/utils/overloaded.h" -#include "td/utils/Random.h" -#include "td/utils/tl_helpers.h" - -#include "tl-utils/tl-utils.hpp" #include "auto/tl/ton_api.h" #include "auto/tl/ton_api.hpp" - #include "common/errorcode.h" - #include "td/actor//actor.h" +#include "td/utils/Random.h" +#include "td/utils/overloaded.h" +#include "td/utils/tl_helpers.h" +#include "tl-utils/tl-utils.hpp" + +#include "RldpConnection.h" +#include "rldp.hpp" namespace ton { namespace rldp2 { diff --git a/rldp2/RldpConnection.h b/rldp2/RldpConnection.h index b9c43bcb3b..fc53cd3cb9 100644 --- a/rldp2/RldpConnection.h +++ b/rldp2/RldpConnection.h @@ -19,6 +19,13 @@ #pragma once +#include + +#include "common/bitstring.h" +#include "td/utils/Heap.h" +#include "td/utils/VectorQueue.h" +#include "td/utils/buffer.h" + #include "Bbr.h" #include "InboundTransfer.h" #include "LossStats.h" @@ -26,14 +33,6 @@ #include "Pacer.h" #include "RttStats.h" -#include "common/bitstring.h" - -#include "td/utils/buffer.h" -#include "td/utils/Heap.h" -#include "td/utils/VectorQueue.h" - -#include - namespace ton { namespace rldp2 { using TransferId = td::Bits256; @@ -119,7 +118,7 @@ class RldpConnection { const RldpSender &sender; td::uint32 before_in_flight{sender.get_inflight_symbols_count()}; - Guard(td::uint32 &in_flight_count, const RldpSender &sender) : in_flight_count(in_flight_count), sender(sender){}; + Guard(td::uint32 &in_flight_count, const RldpSender &sender) : in_flight_count(in_flight_count), sender(sender) {}; ~Guard() { in_flight_count -= before_in_flight; in_flight_count += sender.get_inflight_symbols_count(); diff --git a/rldp2/RldpSender.cpp b/rldp2/RldpSender.cpp index 00c71956c7..6e3232eb11 100644 --- a/rldp2/RldpSender.cpp +++ b/rldp2/RldpSender.cpp @@ -17,13 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "RldpSender.h" +#include "td/utils/misc.h" -#include "RttStats.h" -#include "LossStats.h" #include "BdwStats.h" - -#include "td/utils/misc.h" +#include "LossStats.h" +#include "RldpSender.h" +#include "RttStats.h" namespace ton { namespace rldp2 { diff --git a/rldp2/RttStats.cpp b/rldp2/RttStats.cpp index 4442338698..f603d527a5 100644 --- a/rldp2/RttStats.cpp +++ b/rldp2/RttStats.cpp @@ -17,9 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include + #include "RttStats.h" #include "rldp.hpp" -#include namespace ton { namespace rldp2 { diff --git a/rldp2/SenderPackets.cpp b/rldp2/SenderPackets.cpp index 268972c30e..4a940e94f5 100644 --- a/rldp2/SenderPackets.cpp +++ b/rldp2/SenderPackets.cpp @@ -17,10 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "SenderPackets.h" - #include "td/utils/bits.h" +#include "SenderPackets.h" + namespace ton { namespace rldp2 { td::uint32 SenderPackets::next_seqno() const { diff --git a/rldp2/SenderPackets.h b/rldp2/SenderPackets.h index 43f06834e6..ba2b4616b1 100644 --- a/rldp2/SenderPackets.h +++ b/rldp2/SenderPackets.h @@ -20,6 +20,7 @@ #pragma once #include "td/utils/VectorQueue.h" + #include "Ack.h" #include "BdwStats.h" diff --git a/rldp2/rldp-in.hpp b/rldp2/rldp-in.hpp index 095c14796d..be1df7171c 100644 --- a/rldp2/rldp-in.hpp +++ b/rldp2/rldp-in.hpp @@ -18,16 +18,15 @@ */ #pragma once -#include "rldp.hpp" +#include +#include -#include "tl-utils/tl-utils.hpp" -#include "adnl/adnl-query.h" #include "adnl/adnl-peer-table.h" - +#include "adnl/adnl-query.h" #include "td/utils/List.h" +#include "tl-utils/tl-utils.hpp" -#include -#include +#include "rldp.hpp" namespace ton { @@ -90,7 +89,8 @@ class RldpIn : public RldpImpl { void add_id(adnl::AdnlNodeIdShort local_id) override; - void get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, td::Promise promise) override; + void get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, + td::Promise promise) override; void set_default_mtu(td::uint64 mtu) override; diff --git a/rldp2/rldp.cpp b/rldp2/rldp.cpp index 95780b2375..86660a5faa 100644 --- a/rldp2/rldp.cpp +++ b/rldp2/rldp.cpp @@ -16,12 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "rldp-in.hpp" #include "auto/tl/ton_api.h" #include "auto/tl/ton_api.hpp" -#include "td/utils/Random.h" #include "fec/fec.h" +#include "td/utils/Random.h" + #include "RldpConnection.h" +#include "rldp-in.hpp" namespace ton { @@ -31,7 +32,7 @@ class RldpConnectionActor : public td::actor::Actor, private ConnectionCallback public: RldpConnectionActor(td::actor::ActorId rldp, adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::actor::ActorId adnl) - : rldp_(std::move(rldp)), src_(src), dst_(dst), adnl_(std::move(adnl)){}; + : rldp_(std::move(rldp)), src_(src), dst_(dst), adnl_(std::move(adnl)) {}; void send(TransferId transfer_id, td::BufferSlice query, td::Timestamp timeout = td::Timestamp::never()) { connection_.send(transfer_id, std::move(query), timeout); diff --git a/rldp2/rldp.hpp b/rldp2/rldp.hpp index 473cc95b46..79c2ef9da4 100644 --- a/rldp2/rldp.hpp +++ b/rldp2/rldp.hpp @@ -18,12 +18,12 @@ */ #pragma once -#include "rldp.h" +#include -#include "tl-utils/tl-utils.hpp" #include "adnl/adnl-query.h" +#include "tl-utils/tl-utils.hpp" -#include +#include "rldp.h" namespace ton { diff --git a/storage/LoadSpeed.cpp b/storage/LoadSpeed.cpp index 46aac2ff5a..267843486f 100644 --- a/storage/LoadSpeed.cpp +++ b/storage/LoadSpeed.cpp @@ -17,10 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "LoadSpeed.h" - #include "td/utils/format.h" +#include "LoadSpeed.h" + namespace ton { void LoadSpeed::add(td::uint64 size, td::Timestamp now) { total_size_ += size; diff --git a/storage/MerkleTree.cpp b/storage/MerkleTree.cpp index d8767ffce9..7f287826d7 100644 --- a/storage/MerkleTree.cpp +++ b/storage/MerkleTree.cpp @@ -17,16 +17,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "MerkleTree.h" - #include "common/bitstring.h" #include "td/utils/UInt.h" - #include "vm/cells/CellSlice.h" #include "vm/cells/MerkleProof.h" #include "vm/cellslice.h" #include "vm/excno.hpp" +#include "MerkleTree.h" + namespace ton { static td::Result> unpack_proof(td::Ref root) { vm::CellSlice cs(vm::NoVm(), root); diff --git a/storage/MerkleTree.h b/storage/MerkleTree.h index cf080c407f..b20fa22767 100644 --- a/storage/MerkleTree.h +++ b/storage/MerkleTree.h @@ -19,12 +19,13 @@ #pragma once -#include "td/utils/optional.h" +#include + #include "td/utils/Slice.h" +#include "td/utils/optional.h" #include "vm/cells.h" #include "Bitset.h" -#include namespace ton { // merkle_node$_ {n:#} left:^(ton::MerkleTree n) right:^(ton::MerkleTree n) = ton::MerkleTree (n + 1); diff --git a/storage/MicrochunkTree.cpp b/storage/MicrochunkTree.cpp index 689c7aff7e..06820a2c16 100644 --- a/storage/MicrochunkTree.cpp +++ b/storage/MicrochunkTree.cpp @@ -15,11 +15,12 @@ along with TON Blockchain Library. If not, see . */ -#include "MicrochunkTree.h" -#include "Torrent.h" #include "vm/cells/CellSlice.h" #include "vm/cells/MerkleProof.h" +#include "MicrochunkTree.h" +#include "Torrent.h" + namespace ton { static td::Ref prun(const td::Ref &node) { diff --git a/storage/MicrochunkTree.h b/storage/MicrochunkTree.h index a16616400c..5140b0ff5a 100644 --- a/storage/MicrochunkTree.h +++ b/storage/MicrochunkTree.h @@ -17,12 +17,13 @@ #pragma once -#include "td/utils/optional.h" +#include + #include "td/utils/Slice.h" +#include "td/utils/optional.h" #include "vm/cells.h" #include "Bitset.h" -#include namespace ton { @@ -39,6 +40,7 @@ class MicrochunkTree { MicrochunkTree finalize(); static td::Result build_for_torrent(Torrent &torrent, td::uint64 prun_size = 1 << 17); + private: td::uint64 file_size_; td::uint64 prun_size_; diff --git a/storage/NodeActor.cpp b/storage/NodeActor.cpp index 92a83968fe..294bffb566 100644 --- a/storage/NodeActor.cpp +++ b/storage/NodeActor.cpp @@ -17,18 +17,17 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "NodeActor.h" - -#include "vm/boc.h" -#include "vm/cellslice.h" - -#include "td/utils/Enumerator.h" -#include "td/utils/tests.h" -#include "td/utils/overloaded.h" -#include "tl-utils/tl-utils.hpp" #include "auto/tl/ton_api.hpp" #include "common/delay.h" #include "td/actor/MultiPromise.h" +#include "td/utils/Enumerator.h" +#include "td/utils/overloaded.h" +#include "td/utils/tests.h" +#include "tl-utils/tl-utils.hpp" +#include "vm/boc.h" +#include "vm/cellslice.h" + +#include "NodeActor.h" namespace ton { NodeActor::NodeActor(PeerId self_id, Torrent torrent, td::unique_ptr callback, diff --git a/storage/NodeActor.h b/storage/NodeActor.h index 2f0a3232ff..054f2b8cdb 100644 --- a/storage/NodeActor.h +++ b/storage/NodeActor.h @@ -19,16 +19,16 @@ #pragma once -#include "LoadSpeed.h" -#include "PartsHelper.h" -#include "PeerActor.h" -#include "Torrent.h" -#include "SpeedLimiter.h" +#include #include "td/utils/Random.h" #include "td/utils/Variant.h" -#include +#include "LoadSpeed.h" +#include "PartsHelper.h" +#include "PeerActor.h" +#include "SpeedLimiter.h" +#include "Torrent.h" #include "db.h" namespace ton { diff --git a/storage/PartsHelper.h b/storage/PartsHelper.h index 6ad4e0b39f..4ee15edff0 100644 --- a/storage/PartsHelper.h +++ b/storage/PartsHelper.h @@ -17,12 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "PeerState.h" -#include "Bitset.h" - #include "td/utils/Random.h" #include "td/utils/Status.h" +#include "Bitset.h" +#include "PeerState.h" + namespace ton { struct PartsHelper { public: diff --git a/storage/PeerActor.cpp b/storage/PeerActor.cpp index e140b4ce24..4166f49d69 100644 --- a/storage/PeerActor.cpp +++ b/storage/PeerActor.cpp @@ -17,15 +17,14 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "PeerActor.h" #include "auto/tl/ton_api.hpp" - -#include "tl-utils/tl-utils.hpp" - -#include "td/utils/overloaded.h" +#include "common/delay.h" #include "td/utils/Random.h" +#include "td/utils/overloaded.h" +#include "tl-utils/tl-utils.hpp" #include "vm/boc.h" -#include "common/delay.h" + +#include "PeerActor.h" namespace ton { @@ -492,26 +491,26 @@ void PeerActor::process_update_peer_parts(const tl_object_ptr(offset + i)); - } - } - })); + downcast_call(*update, td::overloaded( + [&](const ton::ton_api::storage_updateHavePieces &have_pieces) { + LOG(DEBUG) + << "Processing updateHavePieces query (" << have_pieces.piece_id_ << " pieces)"; + for (auto id : have_pieces.piece_id_) { + add_piece(id); + } + }, + [&](const ton::ton_api::storage_updateState &state) {}, + [&](const ton::ton_api::storage_updateInit &init) { + LOG(DEBUG) << "Processing updateInit query (offset=" << init.have_pieces_offset_ << ")"; + td::Bitset new_bitset; + new_bitset.set_raw(init.have_pieces_.as_slice().str()); + size_t offset = init.have_pieces_offset_; + for (auto size = new_bitset.size(), i = (size_t)0; i < size; i++) { + if (new_bitset.get(i)) { + add_piece(static_cast(offset + i)); + } + } + })); state_->peer_ready_parts_.add_elements(std::move(new_peer_ready_parts)); } diff --git a/storage/PeerActor.h b/storage/PeerActor.h index a4229e16cd..b00b46bffb 100644 --- a/storage/PeerActor.h +++ b/storage/PeerActor.h @@ -19,12 +19,11 @@ #pragma once -#include "Bitset.h" -#include "PeerState.h" - +#include "auto/tl/ton_api.h" #include "td/utils/optional.h" -#include "auto/tl/ton_api.h" +#include "Bitset.h" +#include "PeerState.h" namespace ton { class PeerActor : public td::actor::Actor { @@ -131,7 +130,7 @@ class PeerActor : public td::actor::Actor { void on_get_info_result(td::Result r_answer); template - td::uint64 create_and_send_query(ArgsT &&... args); + td::uint64 create_and_send_query(ArgsT &&...args); td::uint64 send_query(td::BufferSlice query); void schedule_loop(); diff --git a/storage/PeerManager.h b/storage/PeerManager.h index 38d1494b00..2147d835b9 100644 --- a/storage/PeerManager.h +++ b/storage/PeerManager.h @@ -15,9 +15,10 @@ along with TON Blockchain Library. If not, see . */ +#include "overlay/overlay.h" #include "rldp2/rldp.h" #include "td/actor/actor.h" -#include "overlay/overlay.h" + #include "NodeActor.h" namespace ton_rldp = ton::rldp2; diff --git a/storage/PeerState.h b/storage/PeerState.h index 29c080a26b..5034441154 100644 --- a/storage/PeerState.h +++ b/storage/PeerState.h @@ -18,16 +18,16 @@ */ #pragma once +#include +#include + +#include "td/actor/actor.h" #include "td/utils/buffer.h" #include "td/utils/common.h" #include "td/utils/optional.h" -#include "td/actor/actor.h" -#include "TorrentInfo.h" #include "SpeedLimiter.h" - -#include -#include +#include "TorrentInfo.h" namespace ton { using PeerId = td::uint64; @@ -79,6 +79,7 @@ class MessageBuffer { } return result; } + private: std::atomic*> ptr_{nullptr}; }; @@ -93,7 +94,7 @@ struct PeerState { auto key() const { return std::tie(will_upload, want_download); } - bool operator==(const State &other) const { + bool operator==(const State& other) const { return key() == other.key(); } }; @@ -110,13 +111,13 @@ struct PeerState { td::BufferSlice data; }; - std::set node_queries_active_; // Node only - MessageBuffer node_queries_; // Node -> Peer - MessageBuffer>> node_queries_results_; // Peer -> Node + std::set node_queries_active_; // Node only + MessageBuffer node_queries_; // Node -> Peer + MessageBuffer>> node_queries_results_; // Peer -> Node - std::set peer_queries_active_; // Peer only - MessageBuffer peer_queries_; // Peer -> Node - MessageBuffer>> peer_queries_results_; // Node -> Peer + std::set peer_queries_active_; // Peer only + MessageBuffer peer_queries_; // Peer -> Node + MessageBuffer>> peer_queries_results_; // Node -> Peer // Peer -> Node MessageBuffer peer_ready_parts_; diff --git a/storage/SpeedLimiter.cpp b/storage/SpeedLimiter.cpp index 704c7402de..1a5fafda55 100644 --- a/storage/SpeedLimiter.cpp +++ b/storage/SpeedLimiter.cpp @@ -15,9 +15,10 @@ along with TON Blockchain Library. If not, see . */ -#include "SpeedLimiter.h" #include "common/errorcode.h" +#include "SpeedLimiter.h" + namespace ton { SpeedLimiter::SpeedLimiter(double max_speed) : max_speed_(max_speed) { diff --git a/storage/SpeedLimiter.h b/storage/SpeedLimiter.h index b623073270..a8fd22cb86 100644 --- a/storage/SpeedLimiter.h +++ b/storage/SpeedLimiter.h @@ -16,9 +16,10 @@ */ #pragma once -#include "td/actor/actor.h" #include +#include "td/actor/actor.h" + namespace ton { class SpeedLimiter : public td::actor::Actor { diff --git a/storage/Torrent.cpp b/storage/Torrent.cpp index 545b6d102c..ccb3b4e67a 100644 --- a/storage/Torrent.cpp +++ b/storage/Torrent.cpp @@ -17,13 +17,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "Torrent.h" - #include "td/utils/Status.h" #include "td/utils/crypto.h" #include "td/utils/port/Stat.h" -#include "td/utils/tl_helpers.h" #include "td/utils/port/path.h" +#include "td/utils/tl_helpers.h" + +#include "Torrent.h" namespace ton { diff --git a/storage/Torrent.h b/storage/Torrent.h index 1ae30fc264..3f0a2aa32a 100644 --- a/storage/Torrent.h +++ b/storage/Torrent.h @@ -18,14 +18,14 @@ */ #pragma once -#include "MerkleTree.h" -#include "TorrentMeta.h" +#include +#include -#include "td/utils/buffer.h" #include "td/db/utils/BlobView.h" +#include "td/utils/buffer.h" -#include -#include +#include "MerkleTree.h" +#include "TorrentMeta.h" namespace ton { class Torrent { @@ -159,7 +159,7 @@ class Torrent { void load_from_files(std::string files_path); - td::Status copy_to(const std::string& new_root_dir); + td::Status copy_to(const std::string &new_root_dir); private: td::Bits256 hash_; diff --git a/storage/TorrentCreator.cpp b/storage/TorrentCreator.cpp index ab060023ed..fec373310a 100644 --- a/storage/TorrentCreator.cpp +++ b/storage/TorrentCreator.cpp @@ -17,15 +17,14 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "TorrentCreator.h" - #include "td/db/utils/CyclicBuffer.h" - -#include "td/utils/crypto.h" #include "td/utils/PathView.h" +#include "td/utils/crypto.h" #include "td/utils/port/path.h" #include "td/utils/tl_helpers.h" + #include "MicrochunkTree.h" +#include "TorrentCreator.h" #include "TorrentHeader.hpp" namespace ton { diff --git a/storage/TorrentCreator.h b/storage/TorrentCreator.h index b54157c12d..86d8d39fc7 100644 --- a/storage/TorrentCreator.h +++ b/storage/TorrentCreator.h @@ -18,11 +18,11 @@ */ #pragma once -#include "Torrent.h" - -#include "td/utils/optional.h" -#include "td/utils/SharedSlice.h" #include "td/db/utils/BlobView.h" +#include "td/utils/SharedSlice.h" +#include "td/utils/optional.h" + +#include "Torrent.h" namespace ton { class Torrent::Creator { diff --git a/storage/TorrentHeader.cpp b/storage/TorrentHeader.cpp index aec7c46e7b..0c19db41f5 100644 --- a/storage/TorrentHeader.cpp +++ b/storage/TorrentHeader.cpp @@ -17,10 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "TorrentHeader.hpp" - #include "td/utils/tl_helpers.h" +#include "TorrentHeader.hpp" + namespace ton { td::CSlice TorrentHeader::get_dir_name() const { return dir_name; diff --git a/storage/TorrentHeader.h b/storage/TorrentHeader.h index 65ea859f81..3f86c4ee43 100644 --- a/storage/TorrentHeader.h +++ b/storage/TorrentHeader.h @@ -20,8 +20,8 @@ #pragma once #include "td/utils/Slice.h" -#include "td/utils/buffer.h" #include "td/utils/Status.h" +#include "td/utils/buffer.h" namespace ton { // fec_info_none#c82a1964 = FecInfo; diff --git a/storage/TorrentInfo.cpp b/storage/TorrentInfo.cpp index 26aa092ed9..71dad3118a 100644 --- a/storage/TorrentInfo.cpp +++ b/storage/TorrentInfo.cpp @@ -17,12 +17,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "TorrentInfo.h" - +#include "td/utils/misc.h" #include "vm/cells/CellString.h" #include "vm/cellslice.h" -#include "td/utils/misc.h" +#include "TorrentInfo.h" namespace ton { bool TorrentInfo::pack(vm::CellBuilder &cb) const { diff --git a/storage/TorrentInfo.h b/storage/TorrentInfo.h index 2aee4858f4..4da57afab5 100644 --- a/storage/TorrentInfo.h +++ b/storage/TorrentInfo.h @@ -21,9 +21,8 @@ #include "td/utils/Slice.h" #include "td/utils/UInt.h" - -#include "vm/cells.h" #include "td/utils/optional.h" +#include "vm/cells.h" namespace ton { // torrent_info piece_size:uint32 file_size:uint64 root_hash:(## 256) header_size:uint64 header_hash:(## 256) diff --git a/storage/TorrentMeta.cpp b/storage/TorrentMeta.cpp index 3e5242b55c..23f291484c 100644 --- a/storage/TorrentMeta.cpp +++ b/storage/TorrentMeta.cpp @@ -17,18 +17,16 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "TorrentMeta.h" - -#include "TorrentHeader.hpp" - +#include "td/utils/UInt.h" #include "td/utils/crypto.h" #include "td/utils/tl_helpers.h" -#include "td/utils/UInt.h" - #include "vm/boc.h" #include "vm/cells/MerkleProof.h" #include "vm/cellslice.h" +#include "TorrentHeader.hpp" +#include "TorrentMeta.h" + namespace ton { td::Result TorrentMeta::deserialize(td::Slice data) { diff --git a/storage/TorrentMeta.h b/storage/TorrentMeta.h index cfa2b8b0b1..22ed5f4a86 100644 --- a/storage/TorrentMeta.h +++ b/storage/TorrentMeta.h @@ -19,11 +19,11 @@ #pragma once +#include "td/utils/optional.h" + #include "TorrentHeader.h" #include "TorrentInfo.h" -#include "td/utils/optional.h" - namespace ton { // // torrent_file#6a7181e0 flags:(## 32) info_boc_size:uint32 diff --git a/storage/db.h b/storage/db.h index 269be0d9da..d3dedbe332 100644 --- a/storage/db.h +++ b/storage/db.h @@ -24,8 +24,7 @@ namespace db { using DbType = td::KeyValueAsync; template -inline void db_get(DbType& db, td::Bits256 key, bool allow_not_found, - td::Promise> promise) { +inline void db_get(DbType& db, td::Bits256 key, bool allow_not_found, td::Promise> promise) { db.get(key, [allow_not_found, promise = std::move(promise)](td::Result R) mutable { TRY_RESULT_PROMISE(promise, r, std::move(R)); if (r.status == td::KeyValueReader::GetStatus::NotFound) { diff --git a/storage/storage-cli.cpp b/storage/storage-cli.cpp index 858433d135..de0be0eeff 100644 --- a/storage/storage-cli.cpp +++ b/storage/storage-cli.cpp @@ -17,37 +17,34 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include +#include +#include +#include + #include "adnl/adnl.h" +#include "auto/tl/ton_api_json.h" #include "common/bigint.hpp" #include "common/bitstring.h" #include "dht/dht.h" #include "keys/encryptor.h" #include "overlay/overlay.h" - +#include "td/actor/MultiPromise.h" #include "td/utils/JsonBuilder.h" -#include "td/utils/port/signals.h" -#include "td/utils/Parser.h" -#include "td/utils/overloaded.h" #include "td/utils/OptionParser.h" +#include "td/utils/Parser.h" #include "td/utils/PathView.h" #include "td/utils/Random.h" -#include "td/utils/misc.h" #include "td/utils/filesystem.h" +#include "td/utils/misc.h" +#include "td/utils/overloaded.h" #include "td/utils/port/path.h" - -#include "td/actor/MultiPromise.h" +#include "td/utils/port/signals.h" #include "terminal/terminal.h" +#include "PeerManager.h" #include "Torrent.h" #include "TorrentCreator.h" -#include "PeerManager.h" - -#include "auto/tl/ton_api_json.h" - -#include -#include -#include -#include #include "git.h" struct StorageCliOptions { diff --git a/storage/storage-daemon/StorageManager.cpp b/storage/storage-daemon/StorageManager.cpp index cf93f735f6..68b2165642 100644 --- a/storage/storage-daemon/StorageManager.cpp +++ b/storage/storage-daemon/StorageManager.cpp @@ -15,11 +15,12 @@ along with TON Blockchain Library. If not, see . */ -#include "StorageManager.h" +#include "td/actor/MultiPromise.h" +#include "td/db/RocksDb.h" #include "td/utils/filesystem.h" #include "td/utils/port/path.h" -#include "td/db/RocksDb.h" -#include "td/actor/MultiPromise.h" + +#include "StorageManager.h" namespace ton { diff --git a/storage/storage-daemon/StorageManager.h b/storage/storage-daemon/StorageManager.h index fbfd68ebc7..a7c974a342 100644 --- a/storage/storage-daemon/StorageManager.h +++ b/storage/storage-daemon/StorageManager.h @@ -16,12 +16,13 @@ */ #pragma once -#include "td/actor/actor.h" #include "adnl/adnl.h" -#include "rldp2/rldp.h" #include "overlay/overlays.h" +#include "rldp2/rldp.h" #include "storage/PeerManager.h" #include "storage/db.h" +#include "td/actor/actor.h" + #include "SpeedLimiter.h" namespace ton { diff --git a/storage/storage-daemon/StorageProvider.cpp b/storage/storage-daemon/StorageProvider.cpp index e1eb55d6f0..a6c69def9a 100644 --- a/storage/storage-daemon/StorageProvider.cpp +++ b/storage/storage-daemon/StorageProvider.cpp @@ -15,14 +15,15 @@ along with TON Blockchain Library. If not, see . */ -#include "StorageProvider.h" -#include "td/db/RocksDb.h" -#include "td/utils/JsonBuilder.h" #include "auto/tl/ton_api_json.h" -#include "td/utils/port/path.h" #include "block/block-auto.h" #include "common/delay.h" #include "td/actor/MultiPromise.h" +#include "td/db/RocksDb.h" +#include "td/utils/JsonBuilder.h" +#include "td/utils/port/path.h" + +#include "StorageProvider.h" namespace ton { @@ -262,7 +263,7 @@ void StorageProvider::process_transaction(tl_object_ptrdestination_->account_address_; } } diff --git a/storage/storage-daemon/StorageProvider.h b/storage/storage-daemon/StorageProvider.h index 3d14343800..7690d00d28 100644 --- a/storage/storage-daemon/StorageProvider.h +++ b/storage/storage-daemon/StorageProvider.h @@ -16,13 +16,14 @@ */ #pragma once -#include "td/actor/actor.h" +#include "keyring/keyring.h" +#include "storage/MicrochunkTree.h" #include "storage/db.h" +#include "td/actor/actor.h" #include "tonlib/tonlib/TonlibClientWrapper.h" + #include "StorageManager.h" -#include "keyring/keyring.h" #include "smc-util.h" -#include "storage/MicrochunkTree.h" namespace ton { diff --git a/storage/storage-daemon/smc-util.cpp b/storage/storage-daemon/smc-util.cpp index 59af7d85a2..f76a1a207c 100644 --- a/storage/storage-daemon/smc-util.cpp +++ b/storage/storage-daemon/smc-util.cpp @@ -15,10 +15,11 @@ along with TON Blockchain Library. If not, see . */ -#include "smc-util.h" -#include "td/utils/filesystem.h" #include "keys/encryptor.h" #include "smartcont/provider-code.h" +#include "td/utils/filesystem.h" + +#include "smc-util.h" namespace ton { diff --git a/storage/storage-daemon/smc-util.h b/storage/storage-daemon/smc-util.h index 86940d5ffd..63c3a11a03 100644 --- a/storage/storage-daemon/smc-util.h +++ b/storage/storage-daemon/smc-util.h @@ -16,13 +16,14 @@ */ #pragma once -#include "ton/ton-types.h" -#include "crypto/vm/cellslice.h" +#include + #include "block/block-parse.h" +#include "crypto/vm/cellslice.h" +#include "keyring/keyring.h" #include "td/actor/actor.h" +#include "ton/ton-types.h" #include "tonlib/tonlib/TonlibClientWrapper.h" -#include -#include "keyring/keyring.h" namespace ton { diff --git a/storage/storage-daemon/storage-daemon-cli.cpp b/storage/storage-daemon/storage-daemon-cli.cpp index 9cb1d4172a..94259c7f69 100644 --- a/storage/storage-daemon/storage-daemon-cli.cpp +++ b/storage/storage-daemon/storage-daemon-cli.cpp @@ -14,30 +14,28 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ +#include +#include +#include +#include + +#include "adnl/adnl-ext-client.h" +#include "auto/tl/ton_api_json.h" #include "common/bitstring.h" +#include "common/refint.h" +#include "crypto/block/block.h" #include "keys/encryptor.h" -#include "adnl/adnl-ext-client.h" - -#include "td/utils/port/signals.h" -#include "td/utils/Parser.h" +#include "td/actor/MultiPromise.h" #include "td/utils/OptionParser.h" +#include "td/utils/Parser.h" #include "td/utils/PathView.h" -#include "td/utils/misc.h" #include "td/utils/filesystem.h" +#include "td/utils/misc.h" #include "td/utils/port/path.h" - -#include "td/actor/MultiPromise.h" +#include "td/utils/port/signals.h" #include "terminal/terminal.h" -#include "auto/tl/ton_api_json.h" - -#include -#include -#include -#include #include "git.h" -#include "common/refint.h" -#include "crypto/block/block.h" namespace ton { diff --git a/storage/storage-daemon/storage-daemon.cpp b/storage/storage-daemon/storage-daemon.cpp index d1cbf37b59..54217de355 100644 --- a/storage/storage-daemon/storage-daemon.cpp +++ b/storage/storage-daemon/storage-daemon.cpp @@ -14,38 +14,37 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "td/utils/filesystem.h" -#include "td/actor/actor.h" +#include "adnl/adnl.h" +#include "auto/tl/ton_api_json.h" +#include "common/delay.h" +#include "dht/dht.h" +#include "overlay/overlays.h" +#include "rldp2/rldp.h" #include "td/actor/MultiPromise.h" +#include "td/actor/actor.h" +#include "td/utils/FileLog.h" #include "td/utils/OptionParser.h" +#include "td/utils/Random.h" +#include "td/utils/filesystem.h" +#include "td/utils/port/IPAddress.h" #include "td/utils/port/path.h" #include "td/utils/port/signals.h" #include "td/utils/port/user.h" -#include "td/utils/port/IPAddress.h" -#include "td/utils/Random.h" -#include "td/utils/FileLog.h" -#include "checksum.h" -#include "git.h" -#include "auto/tl/ton_api_json.h" -#include "common/delay.h" -#include "adnl/adnl.h" -#include "rldp2/rldp.h" -#include "dht/dht.h" -#include "overlay/overlays.h" - -#include "Torrent.h" -#include "TorrentCreator.h" #include "StorageManager.h" #include "StorageProvider.h" +#include "Torrent.h" +#include "TorrentCreator.h" +#include "checksum.h" +#include "git.h" #if TD_DARWIN || TD_LINUX #include #endif -#include "td/utils/port/rlimit.h" - #include +#include "td/utils/port/rlimit.h" + namespace ton { td::BufferSlice create_query_error(td::CSlice message) { diff --git a/storage/test/storage.cpp b/storage/test/storage.cpp index ff5a4831ad..08846d7f5e 100644 --- a/storage/test/storage.cpp +++ b/storage/test/storage.cpp @@ -17,55 +17,45 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/benchmark.h" -#include "td/utils/crypto.h" +#include "auto/tl/ton_api.h" +#include "auto/tl/ton_api.hpp" +#include "fec/fec.h" +#include "rldp2/LossSender.h" +#include "rldp2/RldpConnection.h" +#include "td/actor/actor.h" +#include "td/db/utils/CyclicBuffer.h" #include "td/utils/Container.h" -#include "td/utils/misc.h" -#include "td/utils/optional.h" -#include "td/utils/overloaded.h" -#include "td/utils/Status.h" #include "td/utils/Span.h" -#include "td/utils/tests.h" -#include "td/utils/Timer.h" +#include "td/utils/Status.h" +#include "td/utils/ThreadSafeCounter.h" #include "td/utils/Time.h" -#include "td/utils/tl_helpers.h" +#include "td/utils/Timer.h" #include "td/utils/UInt.h" #include "td/utils/VectorQueue.h" -#include "td/utils/ThreadSafeCounter.h" - +#include "td/utils/benchmark.h" +#include "td/utils/crypto.h" #include "td/utils/filesystem.h" +#include "td/utils/misc.h" +#include "td/utils/optional.h" +#include "td/utils/overloaded.h" #include "td/utils/port/path.h" - +#include "td/utils/tests.h" +#include "td/utils/tl_helpers.h" #include "tl-utils/tl-utils.hpp" - -#include "auto/tl/ton_api.h" -#include "auto/tl/ton_api.hpp" - -#include "td/actor/actor.h" - -#include "td/db/utils/CyclicBuffer.h" - #include "vm/boc.h" #include "vm/cells.h" -#include "vm/cellslice.h" -#include "vm/cells/MerkleProof.h" #include "vm/cells/CellString.h" - -#include "fec/fec.h" - -#include "rldp2/RldpConnection.h" -#include "rldp2/LossSender.h" +#include "vm/cells/MerkleProof.h" +#include "vm/cellslice.h" #include "Bitset.h" +#include "MerkleTree.h" +#include "NodeActor.h" +#include "PeerActor.h" #include "PeerState.h" #include "Torrent.h" #include "TorrentCreator.h" -#include "NodeActor.h" -#include "PeerActor.h" - -#include "MerkleTree.h" - constexpr td::uint64 Byte = 1; constexpr td::uint64 KiloByte = (1 << 10) * Byte; constexpr td::uint64 MegaByte = (1 << 10) * KiloByte; @@ -1337,8 +1327,7 @@ TEST(Torrent, Peer) { scheduler.run_in_context([&] { auto peer_manager = td::actor::create_actor("PeerManager"); guard->push_back(td::actor::create_actor( - "Node#1", 1, std::move(torrent), - td::make_unique(stop_watcher, complete_watcher), + "Node#1", 1, std::move(torrent), td::make_unique(stop_watcher, complete_watcher), td::make_unique(peer_manager.get(), 1, gen_peers(1, 2)), nullptr, ton::SpeedLimiters{})); for (size_t i = 2; i <= peers_n; i++) { ton::Torrent::Options options; @@ -1347,8 +1336,7 @@ TEST(Torrent, Peer) { auto node_actor = td::actor::create_actor( PSLICE() << "Node#" << i, i, std::move(other_torrent), td::make_unique(stop_watcher, complete_watcher), - td::make_unique(peer_manager.get(), i, gen_peers(i, 2)), - nullptr, ton::SpeedLimiters{}); + td::make_unique(peer_manager.get(), i, gen_peers(i, 2)), nullptr, ton::SpeedLimiters{}); if (i == 3) { td::actor::create_actor("StatsActor", node_actor.get()).release(); diff --git a/tdactor/benchmark/benchmark-coro.cpp b/tdactor/benchmark/benchmark-coro.cpp index 42fe9c8bfe..98af490fe0 100644 --- a/tdactor/benchmark/benchmark-coro.cpp +++ b/tdactor/benchmark/benchmark-coro.cpp @@ -1,15 +1,15 @@ -#include "td/actor/coro.h" +#include + #include "td/actor/actor.h" +#include "td/actor/coro.h" #include "td/utils/benchmark.h" -#include - using namespace td::actor; class BenchmarkDatabase final : public td::actor::Actor { public: Task square(size_t x) { - co_return x * x; + co_return x* x; } }; @@ -24,80 +24,81 @@ class CoroBenchmark final : public td::actor::Actor { LOG(INFO) << "=== Running benchmarks ==="; std::vector thread_counts = {1, 10}; size_t total_ops = 100000; - + for (size_t thread_count : thread_counts) { size_t ops_per_thread = std::max(1, total_ops / thread_count); std::vector> tasks; - + auto run_benchmark = [&](const char* name, bool immediate) -> Task { td::Timer timer; for (size_t t = 0; t < thread_count; t++) { auto task_name = PSTRING() << name << "_" << t; - tasks.push_back(spawn_actor(std::move(task_name), [](auto db, auto ops_per_thread, auto immediate) -> Task { - for (size_t i = 0; i < ops_per_thread; i++) { - if (immediate) { - (void)co_await ask_immediate(db, &BenchmarkDatabase::square, i); - } else { - (void)co_await ask(db, &BenchmarkDatabase::square, i); - } - } - co_return td::Unit(); - }(db_, ops_per_thread, immediate))); + tasks.push_back( + spawn_actor(std::move(task_name), [](auto db, auto ops_per_thread, auto immediate) -> Task { + for (size_t i = 0; i < ops_per_thread; i++) { + if (immediate) { + (void)co_await ask_immediate(db, &BenchmarkDatabase::square, i); + } else { + (void)co_await ask(db, &BenchmarkDatabase::square, i); + } + } + co_return td::Unit(); + }(db_, ops_per_thread, immediate))); } - + for (auto& task : tasks) { co_await std::move(task); } tasks.clear(); - + auto elapsed = timer.elapsed(); auto ops_per_sec = static_cast(total_ops) / elapsed; - LOG(INFO) << name << " " << ops_per_thread << " ops: " << elapsed << "s (threads=" << thread_count - << ", " << static_cast(ops_per_sec) << " ops/sec)"; + LOG(INFO) << name << " " << ops_per_thread << " ops: " << elapsed << "s (threads=" << thread_count << ", " + << static_cast(ops_per_sec) << " ops/sec)"; co_return td::Unit(); }; - + co_await run_benchmark("Immediate", true); co_await run_benchmark("Delayed", false); } - + LOG(INFO) << "=== Optimized benchmarks (direct) ==="; constexpr size_t single_thread = 1; size_t ops_count = total_ops / single_thread; - + // Warm up for (size_t i = 0; i < 1000; i++) { (void)co_await ask(db_, &BenchmarkDatabase::square, i); } - + td::Timer timer; for (size_t i = 0; i < ops_count; i++) { (void)co_await ask(db_, &BenchmarkDatabase::square, i); } auto elapsed = timer.elapsed(); auto ops_per_sec = static_cast(total_ops) / elapsed; - LOG(INFO) << "Direct delayed " << ops_count << " ops: " << elapsed << "s (" - << static_cast(ops_per_sec) << " ops/sec)"; - + LOG(INFO) << "Direct delayed " << ops_count << " ops: " << elapsed << "s (" << static_cast(ops_per_sec) + << " ops/sec)"; + timer = {}; for (size_t i = 0; i < ops_count; i++) { (void)co_await ask_immediate(db_, &BenchmarkDatabase::square, i); } elapsed = timer.elapsed(); ops_per_sec = static_cast(total_ops) / elapsed; - LOG(INFO) << "Direct immediate " << ops_count << " ops: " << elapsed << "s (" - << static_cast(ops_per_sec) << " ops/sec)"; - + LOG(INFO) << "Direct immediate " << ops_count << " ops: " << elapsed << "s (" << static_cast(ops_per_sec) + << " ops/sec)"; + timer = {}; for (size_t i = 0; i < ops_count; i++) { - auto local_square = [](size_t x) -> Task { co_return x * x; }; + auto local_square = [](size_t x) -> Task { co_return x* x; }; (void)co_await local_square(i); } elapsed = timer.elapsed(); ops_per_sec = static_cast(total_ops) / elapsed; - LOG(INFO) << "Local coroutine " << ops_count << " ops: " << elapsed << "s (" - << static_cast(ops_per_sec) << " ops/sec)"; + LOG(INFO) << "Local coroutine " << ops_count << " ops: " << elapsed << "s (" << static_cast(ops_per_sec) + << " ops/sec)"; co_return td::Unit(); } diff --git a/tdactor/benchmark/benchmark.cpp b/tdactor/benchmark/benchmark.cpp index a7d30373e4..58c28db200 100644 --- a/tdactor/benchmark/benchmark.cpp +++ b/tdactor/benchmark/benchmark.cpp @@ -42,17 +42,19 @@ extern "C" { #include #endif -#include "td/actor/core/ActorLocker.h" -#include "td/actor/actor.h" +#include +#include +#include +#include +#include +#include +#include +#include -#include "td/utils/benchmark.h" -#include "td/utils/crypto.h" -#include "td/utils/logging.h" -#include "td/utils/misc.h" +#include "td/actor/actor.h" +#include "td/actor/core/ActorLocker.h" #include "td/utils/MpmcQueue.h" #include "td/utils/MpmcWaiter.h" -#include "td/utils/port/thread.h" -#include "td/utils/queue.h" #include "td/utils/Random.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" @@ -60,15 +62,12 @@ extern "C" { #include "td/utils/ThreadSafeCounter.h" #include "td/utils/UInt.h" #include "td/utils/VectorQueue.h" - -#include -#include -#include -#include -#include -#include -#include -#include +#include "td/utils/benchmark.h" +#include "td/utils/crypto.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/port/thread.h" +#include "td/utils/queue.h" using td::int32; using td::uint32; diff --git a/tdactor/benchmark/gbench-coro-folly.cpp b/tdactor/benchmark/gbench-coro-folly.cpp index 1b5c808916..ce03b6be01 100644 --- a/tdactor/benchmark/gbench-coro-folly.cpp +++ b/tdactor/benchmark/gbench-coro-folly.cpp @@ -1,18 +1,15 @@ +#include #include - #include #include #include -#include #include #include - -#include +#include #include #include #include - namespace { using folly::coro::Task; @@ -29,7 +26,7 @@ struct DatabaseService { } Task check_auth(int user_id) { - co_return (user_id % 2) == 0; // Even IDs are authorized + co_return (user_id % 2) == 0; // Even IDs are authorized } }; @@ -37,7 +34,6 @@ struct RequestHandler { DatabaseService& db; folly::Executor::KeepAlive<> ex; - Task handle_request(int request_id) { const int user_id = request_id % 1000; @@ -51,7 +47,7 @@ struct RequestHandler { } }; -} // namespace +} // namespace // Real-world benchmark: HTTP-like request handler with database lookups // Simulates: Request -> Auth Check -> DB Query -> Response @@ -80,8 +76,7 @@ static void BM_HttpRequestHandler_Folly(benchmark::State& state) { } // Await all responses concurrently - auto responses = - folly::coro::blockingWait(folly::coro::collectAllRange(std::move(requests))); + auto responses = folly::coro::blockingWait(folly::coro::collectAllRange(std::move(requests))); for (auto& r : responses) { benchmark::DoNotOptimize(r); diff --git a/tdactor/benchmark/gbench-coro-yaclib.cpp b/tdactor/benchmark/gbench-coro-yaclib.cpp index c32e417bce..0eb05d5f29 100644 --- a/tdactor/benchmark/gbench-coro-yaclib.cpp +++ b/tdactor/benchmark/gbench-coro-yaclib.cpp @@ -1,5 +1,9 @@ +#include #include - +#include +#include +#include +#include #include #include #include @@ -11,20 +15,13 @@ #include #include #include -#include #include #include +#include #include #include - -#include -#include #include -#include -#include -#include - struct DatabaseService { DatabaseService(yaclib::IExecutorPtr executor) : strand_(executor) { } diff --git a/tdactor/benchmark/gbench-coro.cpp b/tdactor/benchmark/gbench-coro.cpp index bc029046d3..259727c535 100644 --- a/tdactor/benchmark/gbench-coro.cpp +++ b/tdactor/benchmark/gbench-coro.cpp @@ -1,12 +1,12 @@ -#include -#include "td/actor/coro.h" -#include "td/actor/actor.h" - #include +#include #include #include #include +#include "td/actor/actor.h" +#include "td/actor/coro.h" + #if defined(__APPLE__) || defined(__linux__) #include #elif defined(_WIN32) @@ -397,7 +397,6 @@ enum class AskMode { Scheduled, Immediate }; static void BM_Ask(benchmark::State& state) { coro_benchmark(state, [](auto& state) -> Task { - const auto method = static_cast(state.range(0)); const auto mode = static_cast(state.range(1)); const int num_tasks = static_cast(state.range(2)); diff --git a/tdactor/benchmark/third_party/FAAArrayQueue.h b/tdactor/benchmark/third_party/FAAArrayQueue.h index 8423826520..b8800b6da1 100644 --- a/tdactor/benchmark/third_party/FAAArrayQueue.h +++ b/tdactor/benchmark/third_party/FAAArrayQueue.h @@ -29,11 +29,11 @@ #ifndef _FAA_ARRAY_QUEUE_HP_H_ #define _FAA_ARRAY_QUEUE_HP_H_ -#include "HazardPointers.h" - #include #include +#include "HazardPointers.h" + namespace ConcurrencyFreaks { /** *

Fetch-And-Add Array Queue

diff --git a/tdactor/benchmark/third_party/HazardPointers.h b/tdactor/benchmark/third_party/HazardPointers.h index e562486a29..55b8f88349 100644 --- a/tdactor/benchmark/third_party/HazardPointers.h +++ b/tdactor/benchmark/third_party/HazardPointers.h @@ -31,8 +31,8 @@ #define _HAZARD_POINTERS_H_ #include -#include #include +#include namespace ConcurrencyFreaks { template diff --git a/tdactor/benchmark/third_party/MoodyCamelQueue.h b/tdactor/benchmark/third_party/MoodyCamelQueue.h index 1c7cf80143..cd03e3e97b 100644 --- a/tdactor/benchmark/third_party/MoodyCamelQueue.h +++ b/tdactor/benchmark/third_party/MoodyCamelQueue.h @@ -47,6 +47,7 @@ #ifdef MCDBGQ_USE_RELACY #include "relacy/relacy_std.hpp" + #include "relacy_shims.h" // We only use malloc/free anyway, and the delete macro messes up `= delete` method declarations. // We'll override the default trait malloc ourselves without a macro. @@ -58,16 +59,16 @@ #include // Requires C++11. Sorry VS2010. #include #endif +#include +#include +#include // for CHAR_BIT #include // for max_align_t #include #include -#include -#include -#include #include -#include // for CHAR_BIT -#include #include // partly for __WINPTHREADS_VERSION if on MinGW-w64 w/ POSIX threading +#include +#include // Platform-specific definitions of a numeric thread ID type and an invalid value namespace moodycamel { @@ -1711,7 +1712,7 @@ class ConcurrentQueue { , parent(parent_) { } - virtual ~ProducerBase(){}; + virtual ~ProducerBase() {}; template inline bool dequeue(U& element) { diff --git a/tdactor/example/actor-example-coroutines.cpp b/tdactor/example/actor-example-coroutines.cpp index e41603e60a..6d0cfff0da 100644 --- a/tdactor/example/actor-example-coroutines.cpp +++ b/tdactor/example/actor-example-coroutines.cpp @@ -1,16 +1,16 @@ -#include "absl/status/status.h" -#include "td/actor/coro.h" -#include "td/actor/actor.h" -#include "td/utils/SharedSlice.h" -#include "td/utils/port/sleep.h" - #include -#include #include #include #include +#include #include +#include "absl/status/status.h" +#include "td/actor/actor.h" +#include "td/actor/coro.h" +#include "td/utils/SharedSlice.h" +#include "td/utils/port/sleep.h" + using namespace td::actor; Task example_create() { @@ -83,7 +83,7 @@ Task task_error() { co_return td::Status::Error("test error"); } td::Result result_error() { - return td::Status::Error("test error"); + return td::Status::Error("test error"); } Task pass_task_error() { co_await task_error(); @@ -106,7 +106,6 @@ Task example_error_handling() { Task example_actor() { struct TaskActor : public Actor { TaskActor(td::Promise promise) : promise_(std::move(promise)) { - } void start_up() override { // it is usual actor all coroutines create FROM actor, will be executed ON actor @@ -117,9 +116,10 @@ Task example_actor() { finish(); co_return td::Unit(); } - private: + + private: td::Promise promise_; - int state_ {17}; + int state_{17}; void finish() { promise_.set_result(state_); @@ -131,7 +131,6 @@ Task example_actor() { auto task_actor = create_actor("task_actor", std::move(promise)); CHECK(19 == (co_await std::move(task))); co_return td::Unit(); - } Task example_all() { diff --git a/tdactor/td/actor/ActorId.h b/tdactor/td/actor/ActorId.h index 248a5f68cc..003fc70488 100644 --- a/tdactor/td/actor/ActorId.h +++ b/tdactor/td/actor/ActorId.h @@ -64,7 +64,7 @@ class ActorId { } template - static ActorId create(ActorOptions &options, ArgsT &&... args) { + static ActorId create(ActorOptions &options, ArgsT &&...args) { return ActorId(detail::create_actor(options, std::forward(args)...)); } diff --git a/tdactor/td/actor/ActorOwn.h b/tdactor/td/actor/ActorOwn.h index 01ceef5493..29392e00e7 100644 --- a/tdactor/td/actor/ActorOwn.h +++ b/tdactor/td/actor/ActorOwn.h @@ -17,8 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/actor/common.h" #include "td/actor/ActorId.h" +#include "td/actor/common.h" namespace td { namespace actor { diff --git a/tdactor/td/actor/ActorShared.h b/tdactor/td/actor/ActorShared.h index ffc4176bfc..48f793a2f0 100644 --- a/tdactor/td/actor/ActorShared.h +++ b/tdactor/td/actor/ActorShared.h @@ -17,8 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/actor/common.h" #include "td/actor/ActorId.h" +#include "td/actor/common.h" namespace td { namespace actor { diff --git a/tdactor/td/actor/ActorStats.cpp b/tdactor/td/actor/ActorStats.cpp index ad037204b5..79f7a6016c 100644 --- a/tdactor/td/actor/ActorStats.cpp +++ b/tdactor/td/actor/ActorStats.cpp @@ -1,6 +1,6 @@ -#include "ActorStats.h" - #include "td/utils/ThreadSafeCounter.h" + +#include "ActorStats.h" namespace td { namespace actor { void td::actor::ActorStats::start_up() { diff --git a/tdactor/td/actor/MultiPromise.cpp b/tdactor/td/actor/MultiPromise.cpp index 7ef534cdfc..53ee5a712c 100644 --- a/tdactor/td/actor/MultiPromise.cpp +++ b/tdactor/td/actor/MultiPromise.cpp @@ -16,10 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/actor/MultiPromise.h" - #include +#include "td/actor/MultiPromise.h" + namespace td { namespace detail { class MultiPromiseImpl { diff --git a/tdactor/td/actor/PromiseFuture.h b/tdactor/td/actor/PromiseFuture.h index 91cb916ccc..568e824264 100644 --- a/tdactor/td/actor/PromiseFuture.h +++ b/tdactor/td/actor/PromiseFuture.h @@ -17,16 +17,16 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include +#include +#include + #include "td/utils/Closure.h" +#include "td/utils/ScopeGuard.h" +#include "td/utils/Status.h" #include "td/utils/common.h" #include "td/utils/invoke.h" // for tuple_for_each #include "td/utils/logging.h" -#include "td/utils/ScopeGuard.h" -#include "td/utils/Status.h" - -#include -#include -#include namespace td { namespace detail { @@ -299,7 +299,7 @@ class Promise { }; } template - auto send_closure(ArgsT &&... args); + auto send_closure(ArgsT &&...args); private: std::unique_ptr> promise_; @@ -309,7 +309,7 @@ namespace detail { template class JoinPromise : public PromiseInterface { public: - explicit JoinPromise(ArgsT &&... arg) : promises_(std::forward(arg)...) { + explicit JoinPromise(ArgsT &&...arg) : promises_(std::forward(arg)...) { } void set_value(Unit &&) override { tuple_for_each(promises_, [](auto &promise) { promise.set_value(Unit()); }); @@ -337,7 +337,7 @@ class PromiseCreator { } template - static Promise<> join(ArgsT &&... args) { + static Promise<> join(ArgsT &&...args) { return Promise<>(std::make_unique>(std::forward(args)...)); } }; @@ -421,7 +421,7 @@ class PromiseMerger : public std::enable_shared_from_this @@ -431,7 +431,7 @@ class PromiseMerger : public std::enable_shared_from_this auto split() { - return call_tuple([this](auto &&... arg) { return R{this->make_promise(arg)...}; }, std::move(args_)); + return call_tuple([this](auto &&...arg) { return R{this->make_promise(arg)...}; }, std::move(args_)); } }; diff --git a/tdactor/td/actor/actor.h b/tdactor/td/actor/actor.h index 12721fca3c..9291c82f31 100644 --- a/tdactor/td/actor/actor.h +++ b/tdactor/td/actor/actor.h @@ -17,10 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/actor/common.h" #include "td/actor/ActorId.h" #include "td/actor/ActorOwn.h" #include "td/actor/ActorShared.h" +#include "td/actor/common.h" namespace td { namespace actor { diff --git a/tdactor/td/actor/common.h b/tdactor/td/actor/common.h index d81d67645b..aa0072e1a1 100644 --- a/tdactor/td/actor/common.h +++ b/tdactor/td/actor/common.h @@ -17,15 +17,13 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "td/actor/PromiseFuture.h" #include "td/actor/core/Actor.h" #include "td/actor/core/ActorSignals.h" #include "td/actor/core/ActorTypeStat.h" -#include "td/actor/core/SchedulerId.h" -#include "td/actor/core/SchedulerContext.h" #include "td/actor/core/Scheduler.h" - -#include "td/actor/PromiseFuture.h" - +#include "td/actor/core/SchedulerContext.h" +#include "td/actor/core/SchedulerId.h" #include "td/utils/Timer.h" namespace td { diff --git a/tdactor/td/actor/core/Actor.h b/tdactor/td/actor/core/Actor.h index 3d581fe5e7..0c9a00c2c9 100644 --- a/tdactor/td/actor/core/Actor.h +++ b/tdactor/td/actor/core/Actor.h @@ -18,7 +18,6 @@ */ #pragma once #include "td/actor/core/ActorInfo.h" - #include "td/utils/SharedObjectPool.h" namespace td { diff --git a/tdactor/td/actor/core/ActorExecuteContext.h b/tdactor/td/actor/core/ActorExecuteContext.h index 37d80b5922..8b15d01dea 100644 --- a/tdactor/td/actor/core/ActorExecuteContext.h +++ b/tdactor/td/actor/core/ActorExecuteContext.h @@ -18,12 +18,11 @@ */ #pragma once -#include "td/actor/core/Context.h" +#include -#include "td/utils/logging.h" +#include "td/actor/core/Context.h" #include "td/utils/Time.h" - -#include +#include "td/utils/logging.h" namespace td { namespace actor { diff --git a/tdactor/td/actor/core/ActorExecutor.cpp b/tdactor/td/actor/core/ActorExecutor.cpp index d154224245..474b5d2da1 100644 --- a/tdactor/td/actor/core/ActorExecutor.cpp +++ b/tdactor/td/actor/core/ActorExecutor.cpp @@ -17,7 +17,6 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "td/actor/core/ActorExecutor.h" - #include "td/utils/ScopeGuard.h" namespace td { diff --git a/tdactor/td/actor/core/ActorExecutor.h b/tdactor/td/actor/core/ActorExecutor.h index dd86b5ae2a..3137c8cd27 100644 --- a/tdactor/td/actor/core/ActorExecutor.h +++ b/tdactor/td/actor/core/ActorExecutor.h @@ -26,7 +26,6 @@ #include "td/actor/core/ActorState.h" #include "td/actor/core/ActorTypeStat.h" #include "td/actor/core/SchedulerContext.h" - #include "td/utils/format.h" namespace td { diff --git a/tdactor/td/actor/core/ActorInfo.h b/tdactor/td/actor/core/ActorInfo.h index 690b06d9fb..3850d67ec5 100644 --- a/tdactor/td/actor/core/ActorInfo.h +++ b/tdactor/td/actor/core/ActorInfo.h @@ -18,14 +18,13 @@ */ #pragma once +#include "td/actor/core/ActorMailbox.h" #include "td/actor/core/ActorState.h" #include "td/actor/core/ActorTypeStat.h" -#include "td/actor/core/ActorMailbox.h" - #include "td/utils/Heap.h" #include "td/utils/List.h" -#include "td/utils/Time.h" #include "td/utils/SharedObjectPool.h" +#include "td/utils/Time.h" namespace td { namespace actor { diff --git a/tdactor/td/actor/core/ActorInfoCreator.h b/tdactor/td/actor/core/ActorInfoCreator.h index d818dc6313..4e0816a476 100644 --- a/tdactor/td/actor/core/ActorInfoCreator.h +++ b/tdactor/td/actor/core/ActorInfoCreator.h @@ -17,8 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/actor/core/ActorInfo.h" #include "td/actor/core/Actor.h" +#include "td/actor/core/ActorInfo.h" namespace td { namespace actor { @@ -46,7 +46,7 @@ class ActorInfoCreator { return *this; } - Options& with_actor_stat_id(td::uint32 new_id) { + Options &with_actor_stat_id(td::uint32 new_id) { actor_stat_id = new_id; return *this; } diff --git a/tdactor/td/actor/core/ActorLocker.h b/tdactor/td/actor/core/ActorLocker.h index f0811bea95..ae66768e6f 100644 --- a/tdactor/td/actor/core/ActorLocker.h +++ b/tdactor/td/actor/core/ActorLocker.h @@ -18,13 +18,12 @@ */ #pragma once +#include + #include "td/actor/core/ActorSignals.h" #include "td/actor/core/ActorState.h" - #include "td/utils/logging.h" -#include - namespace td { namespace actor { namespace core { diff --git a/tdactor/td/actor/core/ActorMessage.h b/tdactor/td/actor/core/ActorMessage.h index e8a1b659a9..7e2880e6c6 100644 --- a/tdactor/td/actor/core/ActorMessage.h +++ b/tdactor/td/actor/core/ActorMessage.h @@ -19,7 +19,6 @@ #pragma once #include "td/actor/core/ActorExecuteContext.h" - #include "td/utils/MpscLinkQueue.h" namespace td { diff --git a/tdactor/td/actor/core/ActorSignals.h b/tdactor/td/actor/core/ActorSignals.h index c62478430f..457dfef767 100644 --- a/tdactor/td/actor/core/ActorSignals.h +++ b/tdactor/td/actor/core/ActorSignals.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/utils/common.h" #include "td/utils/bits.h" +#include "td/utils/common.h" #include "td/utils/format.h" namespace td { diff --git a/tdactor/td/actor/core/ActorState.h b/tdactor/td/actor/core/ActorState.h index 5719b816a9..1ad40789bd 100644 --- a/tdactor/td/actor/core/ActorState.h +++ b/tdactor/td/actor/core/ActorState.h @@ -18,14 +18,13 @@ */ #pragma once +#include + #include "td/actor/core/ActorSignals.h" #include "td/actor/core/SchedulerId.h" - #include "td/utils/common.h" #include "td/utils/format.h" -#include - namespace td { namespace actor { namespace core { diff --git a/tdactor/td/actor/core/ActorTypeStat.cpp b/tdactor/td/actor/core/ActorTypeStat.cpp index 4de6f10542..674c1b0794 100644 --- a/tdactor/td/actor/core/ActorTypeStat.cpp +++ b/tdactor/td/actor/core/ActorTypeStat.cpp @@ -1,13 +1,14 @@ -#include "td/actor/core/Actor.h" -#include "td/actor/core/ActorTypeStat.h" -#include "td/actor/core/Scheduler.h" -#include "td/utils/port/thread_local.h" -#include #include #include +#include +#include #include #include -#include + +#include "td/actor/core/Actor.h" +#include "td/actor/core/ActorTypeStat.h" +#include "td/actor/core/Scheduler.h" +#include "td/utils/port/thread_local.h" #ifdef __has_include #if __has_include() diff --git a/tdactor/td/actor/core/ActorTypeStat.h b/tdactor/td/actor/core/ActorTypeStat.h index 5b0bd18dc6..2b048b0d22 100644 --- a/tdactor/td/actor/core/ActorTypeStat.h +++ b/tdactor/td/actor/core/ActorTypeStat.h @@ -1,9 +1,10 @@ #pragma once -#include "td/utils/int_types.h" -#include "td/utils/port/Clocks.h" #include -#include #include +#include + +#include "td/utils/int_types.h" +#include "td/utils/port/Clocks.h" namespace td { namespace actor { diff --git a/tdactor/td/actor/core/CpuWorker.cpp b/tdactor/td/actor/core/CpuWorker.cpp index b8cf7ce412..7083b6fef4 100644 --- a/tdactor/td/actor/core/CpuWorker.cpp +++ b/tdactor/td/actor/core/CpuWorker.cpp @@ -16,13 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/actor/core/CpuWorker.h" +#include #include "td/actor/core/ActorExecutor.h" -#include "td/actor/core/SchedulerContext.h" - +#include "td/actor/core/CpuWorker.h" #include "td/actor/core/Scheduler.h" // FIXME: afer LocalQueue is in a separate file -#include +#include "td/actor/core/SchedulerContext.h" namespace td { namespace actor { diff --git a/tdactor/td/actor/core/CpuWorker.h b/tdactor/td/actor/core/CpuWorker.h index dd520241a5..18f89ee755 100644 --- a/tdactor/td/actor/core/CpuWorker.h +++ b/tdactor/td/actor/core/CpuWorker.h @@ -18,9 +18,8 @@ */ #pragma once -#include "td/actor/core/SchedulerMessage.h" #include "td/actor/core/SchedulerContext.h" - +#include "td/actor/core/SchedulerMessage.h" #include "td/utils/MpmcQueue.h" #include "td/utils/MpmcWaiter.h" #include "td/utils/Span.h" diff --git a/tdactor/td/actor/core/IoWorker.cpp b/tdactor/td/actor/core/IoWorker.cpp index f4d2dfcdd3..8238a5cc4c 100644 --- a/tdactor/td/actor/core/IoWorker.cpp +++ b/tdactor/td/actor/core/IoWorker.cpp @@ -16,9 +16,8 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/actor/core/IoWorker.h" - #include "td/actor/core/ActorExecutor.h" +#include "td/actor/core/IoWorker.h" #include "td/actor/core/Scheduler.h" namespace td { diff --git a/tdactor/td/actor/core/Scheduler.cpp b/tdactor/td/actor/core/Scheduler.cpp index dfb5706389..9412cb9232 100644 --- a/tdactor/td/actor/core/Scheduler.cpp +++ b/tdactor/td/actor/core/Scheduler.cpp @@ -16,12 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/actor/core/Scheduler.h" +#include #include "td/actor/core/CpuWorker.h" #include "td/actor/core/IoWorker.h" - -#include +#include "td/actor/core/Scheduler.h" namespace td { namespace actor { @@ -183,9 +182,8 @@ void Scheduler::ContextImpl::add_token_to_cpu_queue(SchedulerToken token, Schedu } auto &info = scheduler_group()->schedulers.at(scheduler_id.value()); if (scheduler_id == get_scheduler_id() && cpu_worker_id_.is_valid()) { - auto should_notify = info.cpu_local_queue[cpu_worker_id_.value()].push(token, [&](auto value) { - info.cpu_queue->push(value, get_thread_id()); - }); + auto should_notify = info.cpu_local_queue[cpu_worker_id_.value()].push( + token, [&](auto value) { info.cpu_queue->push(value, get_thread_id()); }); if (should_notify) { info.cpu_queue_waiter->notify(); } diff --git a/tdactor/td/actor/core/Scheduler.h b/tdactor/td/actor/core/Scheduler.h index 85458294fb..461b43dab8 100644 --- a/tdactor/td/actor/core/Scheduler.h +++ b/tdactor/td/actor/core/Scheduler.h @@ -18,9 +18,15 @@ */ #pragma once +#include +#include +#include +#include +#include + +#include "td/actor/core/Actor.h" #include "td/actor/core/ActorExecuteContext.h" #include "td/actor/core/ActorExecutor.h" -#include "td/actor/core/Actor.h" #include "td/actor/core/ActorInfo.h" #include "td/actor/core/ActorInfoCreator.h" #include "td/actor/core/ActorLocker.h" @@ -30,35 +36,28 @@ #include "td/actor/core/SchedulerContext.h" #include "td/actor/core/SchedulerId.h" #include "td/actor/core/SchedulerMessage.h" - #include "td/utils/AtomicRead.h" #include "td/utils/Closure.h" -#include "td/utils/common.h" -#include "td/utils/format.h" #include "td/utils/Heap.h" #include "td/utils/List.h" -#include "td/utils/logging.h" #include "td/utils/MpmcQueue.h" -#include "td/utils/StealingQueue.h" #include "td/utils/MpmcWaiter.h" #include "td/utils/MpscLinkQueue.h" #include "td/utils/MpscPollableQueue.h" +#include "td/utils/ScopeGuard.h" +#include "td/utils/Slice.h" +#include "td/utils/StealingQueue.h" +#include "td/utils/Time.h" +#include "td/utils/common.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" #include "td/utils/optional.h" #include "td/utils/port/Poll.h" #include "td/utils/port/detail/Iocp.h" #include "td/utils/port/thread.h" #include "td/utils/port/thread_local.h" -#include "td/utils/ScopeGuard.h" -#include "td/utils/Slice.h" -#include "td/utils/Time.h" #include "td/utils/type_traits.h" -#include -#include -#include -#include -#include - namespace td { namespace actor { namespace core { @@ -264,7 +263,6 @@ class Scheduler { } private: - ActorInfoCreator *creator_; SchedulerId scheduler_id_; CpuWorkerId cpu_worker_id_; diff --git a/tdactor/td/actor/core/SchedulerContext.h b/tdactor/td/actor/core/SchedulerContext.h index 55dc8fa45c..7138e53b5c 100644 --- a/tdactor/td/actor/core/SchedulerContext.h +++ b/tdactor/td/actor/core/SchedulerContext.h @@ -17,20 +17,19 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/actor/core/Context.h" -#include "td/actor/core/SchedulerId.h" #include "td/actor/core/ActorInfo.h" #include "td/actor/core/ActorInfoCreator.h" - -#include "td/utils/port/Poll.h" +#include "td/actor/core/Context.h" +#include "td/actor/core/SchedulerId.h" #include "td/utils/Heap.h" +#include "td/utils/port/Poll.h" namespace td { namespace actor { namespace core { // Token type for CPU queue - encodes either ActorInfo* (bit 0 = 0) or coroutine handle (bit 0 = 1) -using SchedulerToken = void*; +using SchedulerToken = void *; class SchedulerDispatcher { public: diff --git a/tdactor/td/actor/coro.h b/tdactor/td/actor/coro.h index 3013b57348..1dbeef3fde 100644 --- a/tdactor/td/actor/coro.h +++ b/tdactor/td/actor/coro.h @@ -1,11 +1,9 @@ #pragma once -#include "td/actor/coro_types.h" -#include "td/actor/coro_executor.h" #include "td/actor/coro_awaitables.h" +#include "td/actor/coro_executor.h" #include "td/actor/coro_task.h" +#include "td/actor/coro_types.h" #include "td/actor/coro_utils.h" -namespace td::actor { - -} // namespace td::actor \ No newline at end of file +namespace td::actor {} // namespace td::actor \ No newline at end of file diff --git a/tdactor/td/actor/coro_awaitables.h b/tdactor/td/actor/coro_awaitables.h index 2296d73ea4..6e5b3b33a6 100644 --- a/tdactor/td/actor/coro_awaitables.h +++ b/tdactor/td/actor/coro_awaitables.h @@ -1,14 +1,14 @@ #pragma once -#include "td/actor/coro_types.h" -#include "td/actor/coro_executor.h" -#include "td/utils/Status.h" - #include #include #include #include +#include "td/actor/coro_executor.h" +#include "td/actor/coro_types.h" +#include "td/utils/Status.h" + namespace td::actor { namespace detail { diff --git a/tdactor/td/actor/coro_executor.h b/tdactor/td/actor/coro_executor.h index 6871529e13..765b9ad14f 100644 --- a/tdactor/td/actor/coro_executor.h +++ b/tdactor/td/actor/coro_executor.h @@ -1,14 +1,14 @@ #pragma once +#include +#include +#include + #include "td/actor/actor.h" #include "td/actor/core/SchedulerContext.h" #include "td/actor/core/SchedulerId.h" #include "td/actor/coro_types.h" -#include -#include -#include - namespace td::actor { namespace detail { diff --git a/tdactor/td/actor/coro_task.h b/tdactor/td/actor/coro_task.h index accf714ca3..50a94cb6d3 100644 --- a/tdactor/td/actor/coro_task.h +++ b/tdactor/td/actor/coro_task.h @@ -1,17 +1,17 @@ #pragma once -#include "td/actor/coro_types.h" -#include "td/actor/coro_executor.h" -#include "td/actor/coro_awaitables.h" -#include "td/actor/PromiseFuture.h" -#include "td/utils/Status.h" - #include #include #include #include #include +#include "td/actor/PromiseFuture.h" +#include "td/actor/coro_awaitables.h" +#include "td/actor/coro_executor.h" +#include "td/actor/coro_types.h" +#include "td/utils/Status.h" + namespace td::actor { namespace detail { diff --git a/tdactor/td/actor/coro_types.h b/tdactor/td/actor/coro_types.h index ae65dd70ca..9a087a7ac0 100644 --- a/tdactor/td/actor/coro_types.h +++ b/tdactor/td/actor/coro_types.h @@ -1,14 +1,13 @@ #pragma once -#include #include +#include +#include #include #include -#include namespace td::actor { - template concept TDResultLike = requires(R r) { { r.is_error() } -> std::convertible_to; diff --git a/tdactor/td/actor/coro_utils.h b/tdactor/td/actor/coro_utils.h index e394290438..f6fc0bc076 100644 --- a/tdactor/td/actor/coro_utils.h +++ b/tdactor/td/actor/coro_utils.h @@ -1,16 +1,16 @@ #pragma once -#include "td/actor/coro_types.h" -#include "td/actor/coro_executor.h" -#include "td/actor/coro_task.h" -#include "td/actor/actor.h" -#include "td/utils/Status.h" -#include "td/utils/Slice.h" - #include -#include #include #include +#include + +#include "td/actor/actor.h" +#include "td/actor/coro_executor.h" +#include "td/actor/coro_task.h" +#include "td/actor/coro_types.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" namespace td::actor { diff --git a/tdactor/test/actors_core.cpp b/tdactor/test/actors_core.cpp index 43eb67dc46..3eba4d08c7 100644 --- a/tdactor/test/actors_core.cpp +++ b/tdactor/test/actors_core.cpp @@ -16,28 +16,26 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/actor/core/ActorLocker.h" -#include "td/actor/actor.h" -#include "td/actor/PromiseFuture.h" -#include "td/actor/ActorStats.h" +#include +#include +#include +#include -#include "td/utils/format.h" -#include "td/utils/logging.h" -#include "td/utils/misc.h" -#include "td/utils/port/thread.h" -#include "td/utils/port/thread.h" +#include "td/actor/ActorStats.h" +#include "td/actor/PromiseFuture.h" +#include "td/actor/actor.h" +#include "td/actor/core/ActorLocker.h" #include "td/utils/Random.h" #include "td/utils/Slice.h" #include "td/utils/StringBuilder.h" -#include "td/utils/tests.h" #include "td/utils/Time.h" #include "td/utils/TimedStat.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" #include "td/utils/port/sleep.h" - -#include -#include -#include -#include +#include "td/utils/port/thread.h" +#include "td/utils/tests.h" TEST(Actor2, signals) { using td::actor::core::ActorSignals; diff --git a/tdactor/test/actors_promise.cpp b/tdactor/test/actors_promise.cpp index 5717b394c0..b6ee511764 100644 --- a/tdactor/test/actors_promise.cpp +++ b/tdactor/test/actors_promise.cpp @@ -16,9 +16,9 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/actor/actor.h" -#include "td/actor/PromiseFuture.h" #include "td/actor/MultiPromise.h" +#include "td/actor/PromiseFuture.h" +#include "td/actor/actor.h" #include "td/utils/MovableValue.h" #include "td/utils/tests.h" @@ -93,23 +93,24 @@ TEST(Actor, promise) { CHECK(*res == 5); } - {//{ - //Promise promise; - //std::tuple &&> f(std::move(promise)); - //std::tuple> x = std::move(f); - //} - - { - //using T = Result; - //using T = std::unique_ptr; - //using T = std::function; - //using T = std::vector; - //using T = X; - ////using T = Promise; - //T f; - //std::tuple g(std::move(f)); - //std::tuple h = std::move(g); - }} + { //{ + //Promise promise; + //std::tuple &&> f(std::move(promise)); + //std::tuple> x = std::move(f); + //} + + { + //using T = Result; + //using T = std::unique_ptr; + //using T = std::function; + //using T = std::vector; + //using T = X; + ////using T = Promise; + //T f; + //std::tuple g(std::move(f)); + //std::tuple h = std::move(g); + } + } { int result = 0; diff --git a/tdactor/test/test-coro.cpp b/tdactor/test/test-coro.cpp index 27508d44cf..772ad2b233 100644 --- a/tdactor/test/test-coro.cpp +++ b/tdactor/test/test-coro.cpp @@ -1,10 +1,3 @@ -#include "absl/strings/str_format.h" -#include "td/actor/coro.h" -#include "td/actor/actor.h" -#include "td/utils/Random.h" -#include "td/utils/tests.h" -#include "td/utils/port/sleep.h" - #include #include #include @@ -13,6 +6,13 @@ #include #include +#include "absl/strings/str_format.h" +#include "td/actor/actor.h" +#include "td/actor/coro.h" +#include "td/utils/Random.h" +#include "td/utils/port/sleep.h" +#include "td/utils/tests.h" + using namespace td::actor; template diff --git a/tddb/td/db/KeyValue.h b/tddb/td/db/KeyValue.h index 6bbe7df3cf..f9c18c718b 100644 --- a/tddb/td/db/KeyValue.h +++ b/tddb/td/db/KeyValue.h @@ -17,12 +17,13 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include + +#include "td/utils/Span.h" #include "td/utils/Status.h" +#include "td/utils/ThreadSafeCounter.h" #include "td/utils/Time.h" #include "td/utils/logging.h" -#include "td/utils/Span.h" -#include "td/utils/ThreadSafeCounter.h" -#include namespace td { struct UsageStats { size_t get_count{}; @@ -60,8 +61,8 @@ class KeyValueReader { virtual ~KeyValueReader() = default; enum class GetStatus : int32 { Ok, NotFound }; - virtual Result get(Slice key, std::string &value) = 0; - virtual Result> get_multi(td::Span keys, std::vector *values) = 0; + virtual Result get(Slice key, std::string& value) = 0; + virtual Result> get_multi(td::Span keys, std::vector* values) = 0; virtual Result count(Slice prefix) = 0; virtual Status for_each(std::function f) { return Status::Error("for_each is not supported"); @@ -79,10 +80,10 @@ class PrefixedKeyValueReader : public KeyValueReader { Result get(Slice key, std::string& value) override { return reader_->get(PSLICE() << prefix_ << key, value); } - Result> get_multi(td::Span keys, std::vector *values) override { + Result> get_multi(td::Span keys, std::vector* values) override { std::vector prefixed_keys; prefixed_keys.reserve(keys.size()); - for (auto &key : keys) { + for (auto& key : keys) { prefixed_keys.push_back(PSLICE() << prefix_ << key); } return reader_->get_multi(prefixed_keys, values); @@ -135,10 +136,10 @@ class PrefixedKeyValue : public KeyValue { Result get(Slice key, std::string& value) override { return kv_->get(PSLICE() << prefix_ << key, value); } - Result> get_multi(td::Span keys, std::vector *values) override { + Result> get_multi(td::Span keys, std::vector* values) override { std::vector prefixed_keys; prefixed_keys.reserve(keys.size()); - for (auto &key : keys) { + for (auto& key : keys) { prefixed_keys.push_back(PSLICE() << prefix_ << key); } return kv_->get_multi(prefixed_keys, values); diff --git a/tddb/td/db/KeyValueAsync.h b/tddb/td/db/KeyValueAsync.h index 98d2ec3b5e..8f55486a28 100644 --- a/tddb/td/db/KeyValueAsync.h +++ b/tddb/td/db/KeyValueAsync.h @@ -18,9 +18,8 @@ */ #pragma once -#include "td/actor/actor.h" #include "td/actor/PromiseFuture.h" - +#include "td/actor/actor.h" #include "td/db/KeyValue.h" namespace td { diff --git a/tddb/td/db/MemoryKeyValue.cpp b/tddb/td/db/MemoryKeyValue.cpp index c79daccea9..b1aa29c592 100644 --- a/tddb/td/db/MemoryKeyValue.cpp +++ b/tddb/td/db/MemoryKeyValue.cpp @@ -17,9 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "td/db/MemoryKeyValue.h" - -#include "td/utils/format.h" #include "td/utils/Span.h" +#include "td/utils/format.h" namespace td { Result MemoryKeyValue::get(Slice key, std::string &value) { @@ -43,7 +42,7 @@ std::unique_ptr MemoryKeyValue:: } Result> MemoryKeyValue::get_multi(td::Span keys, - std::vector *values) { + std::vector *values) { values->resize(keys.size()); std::vector res; res.reserve(keys.size()); diff --git a/tddb/td/db/MemoryKeyValue.h b/tddb/td/db/MemoryKeyValue.h index 6a5be7866d..db941ea6a2 100644 --- a/tddb/td/db/MemoryKeyValue.h +++ b/tddb/td/db/MemoryKeyValue.h @@ -17,16 +17,16 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/db/KeyValue.h" - #include +#include "td/db/KeyValue.h" + namespace td { struct Merger { virtual ~Merger() = default; - virtual void merge_value_and_update(std::string &value, Slice update) = 0; - virtual void merge_update_and_update(std::string &left_update, Slice right_update) = 0; + virtual void merge_value_and_update(std::string& value, Slice update) = 0; + virtual void merge_update_and_update(std::string& left_update, Slice right_update) = 0; }; class MemoryKeyValue : public KeyValue { public: @@ -34,7 +34,7 @@ class MemoryKeyValue : public KeyValue { MemoryKeyValue(std::shared_ptr merger) : merger_(std::move(merger)) { } Result get(Slice key, std::string& value) override; - Result> get_multi(td::Span keys, std::vector *values) override; + Result> get_multi(td::Span keys, std::vector* values) override; Status for_each(std::function f) override; Status for_each_in_range(Slice begin, Slice end, std::function f) override; Status set(Slice key, Slice value) override; diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index 660381a31d..4f1c4bb810 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -16,15 +16,14 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/db/RocksDb.h" - #include "rocksdb/db.h" -#include "rocksdb/table.h" +#include "rocksdb/filter_policy.h" #include "rocksdb/statistics.h" -#include "rocksdb/write_batch.h" +#include "rocksdb/table.h" #include "rocksdb/utilities/optimistic_transaction_db.h" #include "rocksdb/utilities/transaction.h" -#include "rocksdb/filter_policy.h" +#include "rocksdb/write_batch.h" +#include "td/db/RocksDb.h" #include "td/utils/misc.h" namespace td { @@ -198,11 +197,14 @@ Result> RocksDb::get_multi(td::Span keys, rocksdb::ReadOptions options; if (snapshot_) { options.snapshot = snapshot_.get(); - db_->MultiGet(options, db_->DefaultColumnFamily(), keys_rocksdb.size(), keys_rocksdb.data(), values_rocksdb.data(), statuses.data()); + db_->MultiGet(options, db_->DefaultColumnFamily(), keys_rocksdb.size(), keys_rocksdb.data(), values_rocksdb.data(), + statuses.data()); } else if (transaction_) { - transaction_->MultiGet(options, db_->DefaultColumnFamily(), keys_rocksdb.size(), keys_rocksdb.data(), values_rocksdb.data(), statuses.data()); + transaction_->MultiGet(options, db_->DefaultColumnFamily(), keys_rocksdb.size(), keys_rocksdb.data(), + values_rocksdb.data(), statuses.data()); } else { - db_->MultiGet(options, db_->DefaultColumnFamily(), keys_rocksdb.size(), keys_rocksdb.data(), values_rocksdb.data(), statuses.data()); + db_->MultiGet(options, db_->DefaultColumnFamily(), keys_rocksdb.size(), keys_rocksdb.data(), values_rocksdb.data(), + statuses.data()); } std::vector res(statuses.size()); values->resize(statuses.size()); diff --git a/tddb/td/db/RocksDb.h b/tddb/td/db/RocksDb.h index c9fa93e10e..1610875716 100644 --- a/tddb/td/db/RocksDb.h +++ b/tddb/td/db/RocksDb.h @@ -22,18 +22,16 @@ #error "RocksDb is not supported" #endif -#include "td/db/KeyValue.h" -#include "td/utils/Span.h" -#include "td/utils/Status.h" -#include "td/utils/optional.h" - -#include "td/utils/Time.h" - +#include #include #include #include -#include +#include "td/db/KeyValue.h" +#include "td/utils/Span.h" +#include "td/utils/Status.h" +#include "td/utils/Time.h" +#include "td/utils/optional.h" namespace rocksdb { class DB; diff --git a/tddb/td/db/binlog/Binlog.cpp b/tddb/td/db/binlog/Binlog.cpp index 7c60b11358..4429e2b116 100644 --- a/tddb/td/db/binlog/Binlog.cpp +++ b/tddb/td/db/binlog/Binlog.cpp @@ -16,22 +16,19 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "Binlog.h" - -#include "BinlogReaderHelper.h" - -#include "td/db/utils/StreamInterface.h" +#include "td/actor/actor.h" #include "td/db/utils/ChainBuffer.h" #include "td/db/utils/CyclicBuffer.h" #include "td/db/utils/FileSyncState.h" -#include "td/db/utils/StreamToFileActor.h" #include "td/db/utils/FileToStreamActor.h" - -#include "td/actor/actor.h" - +#include "td/db/utils/StreamInterface.h" +#include "td/db/utils/StreamToFileActor.h" +#include "td/utils/VectorQueue.h" #include "td/utils/misc.h" #include "td/utils/port/path.h" -#include "td/utils/VectorQueue.h" + +#include "Binlog.h" +#include "BinlogReaderHelper.h" namespace td { namespace { diff --git a/tddb/td/db/binlog/Binlog.h b/tddb/td/db/binlog/Binlog.h index 86eb369147..00055d323a 100644 --- a/tddb/td/db/binlog/Binlog.h +++ b/tddb/td/db/binlog/Binlog.h @@ -18,16 +18,14 @@ */ #pragma once -#include "BinlogReaderInterface.h" - +#include "td/actor/actor.h" #include "td/db/utils/FileSyncState.h" #include "td/db/utils/StreamInterface.h" - -#include "td/actor/actor.h" - #include "td/utils/misc.h" #include "td/utils/port/FileFd.h" +#include "BinlogReaderInterface.h" + namespace td { class BinlogReaderInterface; class StreamToFileActor; diff --git a/tddb/td/db/binlog/BinlogReaderHelper.cpp b/tddb/td/db/binlog/BinlogReaderHelper.cpp index 5ea45f64a3..57ae737cf4 100644 --- a/tddb/td/db/binlog/BinlogReaderHelper.cpp +++ b/tddb/td/db/binlog/BinlogReaderHelper.cpp @@ -16,11 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "td/utils/misc.h" + #include "BinlogReaderHelper.h" #include "BinlogReaderInterface.h" -#include "td/utils/misc.h" - namespace td { td::Status BinlogReaderHelper::parse(BinlogReaderInterface& reader, td::Slice data) { SCOPE_EXIT { diff --git a/tddb/td/db/binlog/BinlogReaderHelper.h b/tddb/td/db/binlog/BinlogReaderHelper.h index 19f6a58676..2f59dc4797 100644 --- a/tddb/td/db/binlog/BinlogReaderHelper.h +++ b/tddb/td/db/binlog/BinlogReaderHelper.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/utils/Status.h" #include "td/utils/Slice.h" +#include "td/utils/Status.h" namespace td { diff --git a/tddb/td/db/utils/BlobView.cpp b/tddb/td/db/utils/BlobView.cpp index ebfbc6d545..195fd3d95f 100644 --- a/tddb/td/db/utils/BlobView.cpp +++ b/tddb/td/db/utils/BlobView.cpp @@ -16,17 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/db/utils/BlobView.h" +#include +#include -#include "td/utils/port/FileFd.h" +#include "td/db/utils/BlobView.h" #include "td/utils/HashMap.h" - #include "td/utils/format.h" -#include "td/utils/port/RwMutex.h" +#include "td/utils/port/FileFd.h" #include "td/utils/port/MemoryMapping.h" - -#include -#include +#include "td/utils/port/RwMutex.h" namespace td { diff --git a/tddb/td/db/utils/BlobView.h b/tddb/td/db/utils/BlobView.h index 8e2dcfc274..82605f694d 100644 --- a/tddb/td/db/utils/BlobView.h +++ b/tddb/td/db/utils/BlobView.h @@ -17,8 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/utils/buffer.h" #include "td/utils/Status.h" +#include "td/utils/buffer.h" namespace td { class BlobViewImpl; diff --git a/tddb/td/db/utils/ChainBuffer.cpp b/tddb/td/db/utils/ChainBuffer.cpp index 7478eed8c1..48cd8cf169 100644 --- a/tddb/td/db/utils/ChainBuffer.cpp +++ b/tddb/td/db/utils/ChainBuffer.cpp @@ -16,10 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "ChainBuffer.h" - -#include "td/utils/buffer.h" #include "td/db/utils/StreamInterface.h" +#include "td/utils/buffer.h" + +#include "ChainBuffer.h" namespace td { namespace detail { diff --git a/tddb/td/db/utils/CyclicBuffer.cpp b/tddb/td/db/utils/CyclicBuffer.cpp index d8f7e079b4..970fc450eb 100644 --- a/tddb/td/db/utils/CyclicBuffer.cpp +++ b/tddb/td/db/utils/CyclicBuffer.cpp @@ -16,15 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "CyclicBuffer.h" - -#include "td/utils/misc.h" -#include "td/utils/Slice.h" - #include #include #include +#include "td/utils/Slice.h" +#include "td/utils/misc.h" + +#include "CyclicBuffer.h" + namespace td { namespace detail { class CyclicBuffer : public StreamWriterInterface, public StreamReaderInterface { diff --git a/tddb/td/db/utils/CyclicBuffer.h b/tddb/td/db/utils/CyclicBuffer.h index c82a1e5be6..d454a90a2c 100644 --- a/tddb/td/db/utils/CyclicBuffer.h +++ b/tddb/td/db/utils/CyclicBuffer.h @@ -18,10 +18,10 @@ */ #pragma once -#include "StreamInterface.h" - #include +#include "StreamInterface.h" + namespace td { class CyclicBuffer { diff --git a/tddb/td/db/utils/FileSyncState.h b/tddb/td/db/utils/FileSyncState.h index fcd23514fe..26ab0b5e1a 100644 --- a/tddb/td/db/utils/FileSyncState.h +++ b/tddb/td/db/utils/FileSyncState.h @@ -17,9 +17,9 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include -#include #include +#include +#include #include "td/utils/common.h" diff --git a/tddb/td/db/utils/FileToStreamActor.h b/tddb/td/db/utils/FileToStreamActor.h index 0d09392e5d..19e3aaa246 100644 --- a/tddb/td/db/utils/FileToStreamActor.h +++ b/tddb/td/db/utils/FileToStreamActor.h @@ -18,11 +18,11 @@ */ #pragma once -#include "StreamInterface.h" - #include "td/actor/actor.h" #include "td/utils/port/FileFd.h" +#include "StreamInterface.h" + namespace td { class FileToStreamActor : public td::actor::Actor { public: diff --git a/tddb/td/db/utils/StreamInterface.h b/tddb/td/db/utils/StreamInterface.h index cf8f1d519a..7959f153b9 100644 --- a/tddb/td/db/utils/StreamInterface.h +++ b/tddb/td/db/utils/StreamInterface.h @@ -17,10 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/utils/buffer.h" #include "td/utils/Slice.h" #include "td/utils/Span.h" #include "td/utils/Status.h" +#include "td/utils/buffer.h" #include "td/utils/port/IoSlice.h" namespace td { diff --git a/tddb/td/db/utils/StreamToFileActor.h b/tddb/td/db/utils/StreamToFileActor.h index 2037cc43a0..37c1a8440d 100644 --- a/tddb/td/db/utils/StreamToFileActor.h +++ b/tddb/td/db/utils/StreamToFileActor.h @@ -17,13 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "StreamInterface.h" -#include "FileSyncState.h" - +#include "td/actor/actor.h" #include "td/utils/Time.h" #include "td/utils/port/FileFd.h" -#include "td/actor/actor.h" +#include "FileSyncState.h" +#include "StreamInterface.h" namespace td { class StreamToFileActor : public actor::Actor { diff --git a/tddb/test/binlog.cpp b/tddb/test/binlog.cpp index 8a665439f2..936bc3f9ea 100644 --- a/tddb/test/binlog.cpp +++ b/tddb/test/binlog.cpp @@ -16,36 +16,32 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/tests.h" +#include +#include "td/actor/actor.h" +#include "td/db/binlog/Binlog.h" +#include "td/db/binlog/BinlogReaderHelper.h" +#include "td/db/utils/ChainBuffer.h" +#include "td/db/utils/CyclicBuffer.h" +#include "td/db/utils/StreamInterface.h" +#include "td/utils/Slice.h" +#include "td/utils/Span.h" +#include "td/utils/UInt.h" +#include "td/utils/Variant.h" +#include "td/utils/VectorQueue.h" #include "td/utils/as.h" #include "td/utils/base64.h" #include "td/utils/benchmark.h" #include "td/utils/buffer.h" #include "td/utils/crypto.h" #include "td/utils/filesystem.h" -#include "td/utils/Slice.h" -#include "td/utils/Span.h" #include "td/utils/misc.h" -#include "td/utils/overloaded.h" #include "td/utils/optional.h" +#include "td/utils/overloaded.h" #include "td/utils/port/FileFd.h" -#include "td/utils/port/path.h" #include "td/utils/port/IoSlice.h" -#include "td/utils/UInt.h" -#include "td/utils/Variant.h" -#include "td/utils/VectorQueue.h" - -#include "td/actor/actor.h" - -#include "td/db/utils/StreamInterface.h" -#include "td/db/utils/ChainBuffer.h" -#include "td/db/utils/CyclicBuffer.h" -#include "td/db/binlog/BinlogReaderHelper.h" - -#include "td/db/binlog/Binlog.h" - -#include +#include "td/utils/port/path.h" +#include "td/utils/tests.h" // Toy Binlog Implementation using td::int64; diff --git a/tddb/test/io-bench.cpp b/tddb/test/io-bench.cpp index 833f6b2a91..365de9f5ad 100644 --- a/tddb/test/io-bench.cpp +++ b/tddb/test/io-bench.cpp @@ -25,23 +25,22 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/OptionParser.h" -#include "td/utils/filesystem.h" -#include "td/utils/port/FileFd.h" -#include "td/utils/Timer.h" -#include "td/utils/crypto.h" -#include "td/utils/BufferedReader.h" -#include "td/utils/optional.h" -#include "td/actor/actor.h" +#include -#include "td/db/utils/StreamInterface.h" +#include "td/actor/actor.h" #include "td/db/utils/ChainBuffer.h" #include "td/db/utils/CyclicBuffer.h" #include "td/db/utils/FileSyncState.h" -#include "td/db/utils/StreamToFileActor.h" #include "td/db/utils/FileToStreamActor.h" - -#include +#include "td/db/utils/StreamInterface.h" +#include "td/db/utils/StreamToFileActor.h" +#include "td/utils/BufferedReader.h" +#include "td/utils/OptionParser.h" +#include "td/utils/Timer.h" +#include "td/utils/crypto.h" +#include "td/utils/filesystem.h" +#include "td/utils/optional.h" +#include "td/utils/port/FileFd.h" namespace td { class AsyncCyclicBufferReader : public td::actor::Actor { diff --git a/tddb/test/key_value.cpp b/tddb/test/key_value.cpp index 921ad059ab..a1e6f7bae2 100644 --- a/tddb/test/key_value.cpp +++ b/tddb/test/key_value.cpp @@ -16,16 +16,14 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/tests.h" - -#include "td/db/KeyValueAsync.h" #include "td/db/KeyValue.h" +#include "td/db/KeyValueAsync.h" #include "td/db/RocksDb.h" - +#include "td/utils/UInt.h" #include "td/utils/benchmark.h" #include "td/utils/buffer.h" #include "td/utils/optional.h" -#include "td/utils/UInt.h" +#include "td/utils/tests.h" TEST(KeyValue, simple) { td::Slice db_name = "testdb"; diff --git a/tdfec/benchmark/benchmark.cpp b/tdfec/benchmark/benchmark.cpp index db7b33a1b6..7179102181 100644 --- a/tdfec/benchmark/benchmark.cpp +++ b/tdfec/benchmark/benchmark.cpp @@ -25,17 +25,17 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/benchmark.h" -#include "td/utils/format.h" -#include "td/utils/Random.h" -#include "td/utils/Time.h" -#include "td/utils/tests.h" +#include -#include "td/fec/fec.h" -#include "td/fec/algebra/Octet.h" #include "td/fec/algebra/GaussianElimination.h" +#include "td/fec/algebra/Octet.h" #include "td/fec/algebra/Simd.h" -#include +#include "td/fec/fec.h" +#include "td/utils/Random.h" +#include "td/utils/Time.h" +#include "td/utils/benchmark.h" +#include "td/utils/format.h" +#include "td/utils/tests.h" template class Simd_gf256_from_gf2 : public td::Benchmark { diff --git a/tdfec/td/fec/algebra/BeliefPropagationDecoding.h b/tdfec/td/fec/algebra/BeliefPropagationDecoding.h index b152e5d3f5..3aa18a519c 100644 --- a/tdfec/td/fec/algebra/BeliefPropagationDecoding.h +++ b/tdfec/td/fec/algebra/BeliefPropagationDecoding.h @@ -17,9 +17,9 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/utils/buffer.h" -#include "td/utils/Span.h" #include "td/fec/algebra/MatrixGF256.h" +#include "td/utils/Span.h" +#include "td/utils/buffer.h" namespace td { class BeliefPropagationDecoding { diff --git a/tdfec/td/fec/algebra/InactivationDecoding.cpp b/tdfec/td/fec/algebra/InactivationDecoding.cpp index 34adff6ef3..8cce2f11b2 100644 --- a/tdfec/td/fec/algebra/InactivationDecoding.cpp +++ b/tdfec/td/fec/algebra/InactivationDecoding.cpp @@ -16,12 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/fec/algebra/InactivationDecoding.h" +#include +#include "td/fec/algebra/InactivationDecoding.h" #include "td/utils/logging.h" -#include - namespace td { InactivationDecodingResult InactivationDecoding::run() { init(); diff --git a/tdfec/td/fec/algebra/InactivationDecoding.h b/tdfec/td/fec/algebra/InactivationDecoding.h index 06a50e0bb4..adc0e8c92a 100644 --- a/tdfec/td/fec/algebra/InactivationDecoding.h +++ b/tdfec/td/fec/algebra/InactivationDecoding.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/utils/common.h" #include "td/fec/algebra/SparseMatrixGF2.h" +#include "td/utils/common.h" namespace td { struct InactivationDecodingResult { diff --git a/tdfec/td/fec/algebra/MatrixGF256.h b/tdfec/td/fec/algebra/MatrixGF256.h index 43f20155c5..fe3ed9fd31 100644 --- a/tdfec/td/fec/algebra/MatrixGF256.h +++ b/tdfec/td/fec/algebra/MatrixGF256.h @@ -19,7 +19,6 @@ #pragma once #include "td/fec/algebra/Octet.h" #include "td/fec/algebra/Simd.h" - #include "td/utils/Span.h" #include "td/utils/format.h" diff --git a/tdfec/td/fec/algebra/Octet.h b/tdfec/td/fec/algebra/Octet.h index 405352dd67..09c11e776c 100644 --- a/tdfec/td/fec/algebra/Octet.h +++ b/tdfec/td/fec/algebra/Octet.h @@ -17,11 +17,11 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/utils/logging.h" - #include #include +#include "td/utils/logging.h" + namespace td { class Octet { diff --git a/tdfec/td/fec/algebra/Simd.h b/tdfec/td/fec/algebra/Simd.h index 58e0ac0301..bfc8e9f733 100644 --- a/tdfec/td/fec/algebra/Simd.h +++ b/tdfec/td/fec/algebra/Simd.h @@ -17,9 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/utils/misc.h" - #include "td/fec/algebra/Octet.h" +#include "td/utils/misc.h" #if __SSSE3__ #define TD_SSE3 1 diff --git a/tdfec/td/fec/fec.cpp b/tdfec/td/fec/fec.cpp index c9ac131150..9b388fff97 100644 --- a/tdfec/td/fec/fec.cpp +++ b/tdfec/td/fec/fec.cpp @@ -17,12 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "td/fec/fec.h" - -#include "td/fec/raptorq/Encoder.h" -#include "td/fec/raptorq/Decoder.h" - -#include "td/fec/online/Encoder.h" #include "td/fec/online/Decoder.h" +#include "td/fec/online/Encoder.h" +#include "td/fec/raptorq/Decoder.h" +#include "td/fec/raptorq/Encoder.h" namespace td { namespace fec { diff --git a/tdfec/td/fec/fec.h b/tdfec/td/fec/fec.h index 0cd61ccb98..ad462d6aee 100644 --- a/tdfec/td/fec/fec.h +++ b/tdfec/td/fec/fec.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/utils/buffer.h" #include "td/utils/Status.h" +#include "td/utils/buffer.h" namespace td { namespace raptorq { diff --git a/tdfec/td/fec/online/Decoder.h b/tdfec/td/fec/online/Decoder.h index bd7533d95d..4c3b6ecd4a 100644 --- a/tdfec/td/fec/online/Decoder.h +++ b/tdfec/td/fec/online/Decoder.h @@ -17,11 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/fec/online/Rfc.h" -#include "td/fec/common/SymbolRef.h" #include "td/fec/algebra/BeliefPropagationDecoding.h" - +#include "td/fec/common/SymbolRef.h" #include "td/fec/online/Encoder.h" +#include "td/fec/online/Rfc.h" namespace td { namespace online_code { diff --git a/tdfec/td/fec/online/Encoder.cpp b/tdfec/td/fec/online/Encoder.cpp index df289726f5..1640014bb1 100644 --- a/tdfec/td/fec/online/Encoder.cpp +++ b/tdfec/td/fec/online/Encoder.cpp @@ -16,9 +16,8 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/fec/online/Encoder.h" - #include "td/fec/common/SymbolsView.h" +#include "td/fec/online/Encoder.h" namespace td { namespace online_code { diff --git a/tdfec/td/fec/online/Encoder.h b/tdfec/td/fec/online/Encoder.h index d2d2343ade..89eaf14d70 100644 --- a/tdfec/td/fec/online/Encoder.h +++ b/tdfec/td/fec/online/Encoder.h @@ -17,9 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/fec/online/Rfc.h" #include "td/fec/algebra/MatrixGF256.h" - +#include "td/fec/online/Rfc.h" #include "td/utils/buffer.h" namespace td { diff --git a/tdfec/td/fec/online/Rfc.h b/tdfec/td/fec/online/Rfc.h index a254bc4e1e..f6c1579f7a 100644 --- a/tdfec/td/fec/online/Rfc.h +++ b/tdfec/td/fec/online/Rfc.h @@ -17,12 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include #include +#include -#include "td/utils/Status.h" -#include "td/utils/Span.h" #include "td/utils/Random.h" +#include "td/utils/Span.h" +#include "td/utils/Status.h" namespace td { namespace online_code { diff --git a/tdfec/td/fec/raptorq/Decoder.h b/tdfec/td/fec/raptorq/Decoder.h index 2288b3e4c6..c1a677cf39 100644 --- a/tdfec/td/fec/raptorq/Decoder.h +++ b/tdfec/td/fec/raptorq/Decoder.h @@ -17,11 +17,11 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/fec/raptorq/Solver.h" -#include "td/fec/raptorq/Encoder.h" - #include +#include "td/fec/raptorq/Encoder.h" +#include "td/fec/raptorq/Solver.h" + namespace td { namespace raptorq { diff --git a/tdfec/td/fec/raptorq/Encoder.cpp b/tdfec/td/fec/raptorq/Encoder.cpp index dde4843835..1a0bdae207 100644 --- a/tdfec/td/fec/raptorq/Encoder.cpp +++ b/tdfec/td/fec/raptorq/Encoder.cpp @@ -17,7 +17,6 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "td/fec/raptorq/Encoder.h" - #include "td/fec/raptorq/Solver.h" namespace td { diff --git a/tdfec/td/fec/raptorq/Encoder.h b/tdfec/td/fec/raptorq/Encoder.h index 64e9b4156d..3325fd86a8 100644 --- a/tdfec/td/fec/raptorq/Encoder.h +++ b/tdfec/td/fec/raptorq/Encoder.h @@ -17,13 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/fec/raptorq/RawEncoder.h" -#include "td/fec/common/SymbolsView.h" +#include -#include "td/utils/optional.h" +#include "td/fec/common/SymbolsView.h" +#include "td/fec/raptorq/RawEncoder.h" #include "td/utils/buffer.h" - -#include +#include "td/utils/optional.h" namespace td { namespace raptorq { diff --git a/tdfec/td/fec/raptorq/RawEncoder.h b/tdfec/td/fec/raptorq/RawEncoder.h index 4222a5408c..a3afad2f62 100644 --- a/tdfec/td/fec/raptorq/RawEncoder.h +++ b/tdfec/td/fec/raptorq/RawEncoder.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/fec/raptorq/Rfc.h" #include "td/fec/algebra/MatrixGF256.h" +#include "td/fec/raptorq/Rfc.h" namespace td { namespace raptorq { diff --git a/tdfec/td/fec/raptorq/Rfc.cpp b/tdfec/td/fec/raptorq/Rfc.cpp index df390e1926..93aebc255a 100644 --- a/tdfec/td/fec/raptorq/Rfc.cpp +++ b/tdfec/td/fec/raptorq/Rfc.cpp @@ -16,10 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/fec/raptorq/Rfc.h" - #include +#include "td/fec/raptorq/Rfc.h" + namespace td { namespace raptorq { diff --git a/tdfec/td/fec/raptorq/Rfc.h b/tdfec/td/fec/raptorq/Rfc.h index e3a331316e..93c5b1128d 100644 --- a/tdfec/td/fec/raptorq/Rfc.h +++ b/tdfec/td/fec/raptorq/Rfc.h @@ -17,10 +17,9 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/utils/Status.h" - #include "td/fec/algebra/MatrixGF256.h" #include "td/fec/algebra/SparseMatrixGF2.h" +#include "td/utils/Status.h" namespace td { namespace raptorq { diff --git a/tdfec/td/fec/raptorq/Solver.cpp b/tdfec/td/fec/raptorq/Solver.cpp index 772271fa7c..a4c9b4b848 100644 --- a/tdfec/td/fec/raptorq/Solver.cpp +++ b/tdfec/td/fec/raptorq/Solver.cpp @@ -16,13 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/fec/raptorq/Solver.h" +#include + #include "td/fec/algebra/GaussianElimination.h" #include "td/fec/algebra/InactivationDecoding.h" +#include "td/fec/raptorq/Solver.h" #include "td/utils/ThreadSafeCounter.h" - #include "td/utils/Timer.h" -#include namespace td { namespace raptorq { diff --git a/tdfec/td/fec/raptorq/Solver.h b/tdfec/td/fec/raptorq/Solver.h index a1b993c2f8..7cc58bcd56 100644 --- a/tdfec/td/fec/raptorq/Solver.h +++ b/tdfec/td/fec/raptorq/Solver.h @@ -17,12 +17,11 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "td/fec/common/SymbolRef.h" +#include "td/fec/raptorq/Rfc.h" #include "td/utils/Slice.h" #include "td/utils/Span.h" -#include "td/fec/raptorq/Rfc.h" -#include "td/fec/common/SymbolRef.h" - namespace td { namespace raptorq { diff --git a/tdfec/test/LibRaptorQ.cpp b/tdfec/test/LibRaptorQ.cpp index e4e926d2ee..5d530d28e9 100644 --- a/tdfec/test/LibRaptorQ.cpp +++ b/tdfec/test/LibRaptorQ.cpp @@ -16,9 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "LibRaptorQ.h" #include "RaptorQ/RaptorQ_v1_hdr.hpp" +#include "LibRaptorQ.h" + namespace RaptorQ = RaptorQ__v1; namespace td { diff --git a/tdfec/test/LibRaptorQ.h b/tdfec/test/LibRaptorQ.h index 6d48b3d029..00e4309e91 100644 --- a/tdfec/test/LibRaptorQ.h +++ b/tdfec/test/LibRaptorQ.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/utils/buffer.h" #include "td/fec/fec.h" +#include "td/utils/buffer.h" namespace td { namespace fec { diff --git a/tdfec/test/fec-test.cpp b/tdfec/test/fec-test.cpp index 16879847c7..045b96eb0f 100644 --- a/tdfec/test/fec-test.cpp +++ b/tdfec/test/fec-test.cpp @@ -19,14 +19,14 @@ //#include "fec.h" //#include "raptorq/Solver.h" #include "td/fec/fec.h" -#include "td/fec/raptorq/Encoder.h" #include "td/fec/raptorq/Decoder.h" +#include "td/fec/raptorq/Encoder.h" #if USE_LIBRAPTORQ #include "LibRaptorQ.h" #endif -#include "td/utils/tests.h" - #include + +#include "td/utils/tests.h" td::Slice get_long_string() { const size_t max_symbol_size = 200; const size_t symbols_count = 100; diff --git a/tdnet/example/tcp_ping_pong.cpp b/tdnet/example/tcp_ping_pong.cpp index 2c47924230..bece8f2655 100644 --- a/tdnet/example/tcp_ping_pong.cpp +++ b/tdnet/example/tcp_ping_pong.cpp @@ -26,14 +26,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "td/actor/actor.h" - +#include "td/net/TcpListener.h" #include "td/utils/BufferedFd.h" +#include "td/utils/Observer.h" #include "td/utils/OptionParser.h" -#include "td/utils/port/SocketFd.h" #include "td/utils/port/ServerSocketFd.h" -#include "td/utils/Observer.h" - -#include "td/net/TcpListener.h" +#include "td/utils/port/SocketFd.h" class PingClient : public td::actor::Actor, td::ObserverBase { public: diff --git a/tdnet/example/udp_ping_pong.cpp b/tdnet/example/udp_ping_pong.cpp index 78154a7753..1c1472aa3f 100644 --- a/tdnet/example/udp_ping_pong.cpp +++ b/tdnet/example/udp_ping_pong.cpp @@ -26,13 +26,11 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "td/actor/actor.h" - -#include "td/utils/OptionParser.h" +#include "td/net/UdpServer.h" #include "td/utils/Observer.h" +#include "td/utils/OptionParser.h" #include "td/utils/port/UdpSocketFd.h" -#include "td/net/UdpServer.h" - // PingPong class PingPong : public td::actor::Actor { public: diff --git a/tdnet/td/net/FdListener.h b/tdnet/td/net/FdListener.h index 261d7e37c3..ab428d4f42 100644 --- a/tdnet/td/net/FdListener.h +++ b/tdnet/td/net/FdListener.h @@ -19,7 +19,6 @@ #pragma once #include "td/actor/actor.h" - #include "td/utils/Observer.h" #include "td/utils/port/detail/PollableFd.h" diff --git a/tdnet/td/net/TcpListener.h b/tdnet/td/net/TcpListener.h index fc11f8bc64..3dae7e06a2 100644 --- a/tdnet/td/net/TcpListener.h +++ b/tdnet/td/net/TcpListener.h @@ -19,9 +19,8 @@ #pragma once #include "td/actor/actor.h" - -#include "td/utils/port/ServerSocketFd.h" #include "td/utils/Observer.h" +#include "td/utils/port/ServerSocketFd.h" namespace td { class TcpListener : public td::actor::Actor, private td::ObserverBase { diff --git a/tdnet/td/net/UdpServer.cpp b/tdnet/td/net/UdpServer.cpp index ba28c5cfc3..867a375f89 100644 --- a/tdnet/td/net/UdpServer.cpp +++ b/tdnet/td/net/UdpServer.cpp @@ -16,14 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/net/UdpServer.h" +#include + #include "td/net/FdListener.h" #include "td/net/TcpListener.h" - +#include "td/net/UdpServer.h" #include "td/utils/BufferedFd.h" -#include - namespace td { namespace { int VERBOSITY_NAME(udp_server) = VERBOSITY_NAME(DEBUG) + 10; diff --git a/tdnet/td/net/UdpServer.h b/tdnet/td/net/UdpServer.h index f6d0d268e5..0c9ad48a04 100644 --- a/tdnet/td/net/UdpServer.h +++ b/tdnet/td/net/UdpServer.h @@ -20,7 +20,6 @@ #include "td/actor/actor.h" #include "td/utils/BufferedUdp.h" - #include "td/utils/port/UdpSocketFd.h" namespace td { diff --git a/tdtl/td/tl/tl_config.cpp b/tdtl/td/tl/tl_config.cpp index cc1b0f4218..305b727948 100644 --- a/tdtl/td/tl/tl_config.cpp +++ b/tdtl/td/tl/tl_config.cpp @@ -16,13 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tl_config.h" - #include #include #include #include +#include "tl_config.h" + namespace td { namespace tl { diff --git a/tdtl/td/tl/tl_config.h b/tdtl/td/tl/tl_config.h index 2fe6469f38..aac6648618 100644 --- a/tdtl/td/tl/tl_config.h +++ b/tdtl/td/tl/tl_config.h @@ -18,15 +18,15 @@ */ #pragma once -#include "tl_core.h" -#include "tl_simple_parser.h" - #include #include #include #include #include +#include "tl_core.h" +#include "tl_simple_parser.h" + namespace td { namespace tl { diff --git a/tdtl/td/tl/tl_core.cpp b/tdtl/td/tl/tl_core.cpp index 93aaf8814c..76d23090ab 100644 --- a/tdtl/td/tl/tl_core.cpp +++ b/tdtl/td/tl/tl_core.cpp @@ -16,10 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tl_core.h" - #include +#include "tl_core.h" + namespace td { namespace tl { diff --git a/tdtl/td/tl/tl_file_outputer.cpp b/tdtl/td/tl/tl_file_outputer.cpp index 1cda2222af..9758c297d5 100644 --- a/tdtl/td/tl/tl_file_outputer.cpp +++ b/tdtl/td/tl/tl_file_outputer.cpp @@ -16,10 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tl_file_outputer.h" - #include +#include "tl_file_outputer.h" + namespace td { namespace tl { diff --git a/tdtl/td/tl/tl_file_outputer.h b/tdtl/td/tl/tl_file_outputer.h index 4569459530..6767c5ee90 100644 --- a/tdtl/td/tl/tl_file_outputer.h +++ b/tdtl/td/tl/tl_file_outputer.h @@ -18,11 +18,11 @@ */ #pragma once -#include "tl_outputer.h" - #include #include +#include "tl_outputer.h" + namespace td { namespace tl { diff --git a/tdtl/td/tl/tl_file_utils.cpp b/tdtl/td/tl/tl_file_utils.cpp index d604447e18..3a76df7a3a 100644 --- a/tdtl/td/tl/tl_file_utils.cpp +++ b/tdtl/td/tl/tl_file_utils.cpp @@ -16,11 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tl_file_utils.h" - #include #include +#include "tl_file_utils.h" + namespace td { namespace tl { diff --git a/tdtl/td/tl/tl_generate.cpp b/tdtl/td/tl/tl_generate.cpp index 9e149aaeaa..19a4ee3e36 100644 --- a/tdtl/td/tl/tl_generate.cpp +++ b/tdtl/td/tl/tl_generate.cpp @@ -16,15 +16,6 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tl_generate.h" - -#include "tl_config.h" -#include "tl_core.h" -#include "tl_file_utils.h" -#include "tl_outputer.h" -#include "tl_string_outputer.h" -#include "tl_writer.h" - #include #include #include @@ -33,6 +24,14 @@ #include #include +#include "tl_config.h" +#include "tl_core.h" +#include "tl_file_utils.h" +#include "tl_generate.h" +#include "tl_outputer.h" +#include "tl_string_outputer.h" +#include "tl_writer.h" + namespace td { namespace tl { diff --git a/tdtl/td/tl/tl_generate.h b/tdtl/td/tl/tl_generate.h index 508c104fe8..5d4031dc84 100644 --- a/tdtl/td/tl/tl_generate.h +++ b/tdtl/td/tl/tl_generate.h @@ -18,12 +18,12 @@ */ #pragma once +#include + #include "tl_config.h" #include "tl_outputer.h" #include "tl_writer.h" -#include - namespace td { namespace tl { diff --git a/tdtl/td/tl/tl_simple.h b/tdtl/td/tl/tl_simple.h index 06d6ea133d..f76f76129d 100644 --- a/tdtl/td/tl/tl_simple.h +++ b/tdtl/td/tl/tl_simple.h @@ -18,17 +18,16 @@ */ #pragma once -#include "td/tl/tl_config.h" - #include #include #include +#include #include #include #include #include -#include +#include "td/tl/tl_config.h" namespace td { namespace tl { diff --git a/tdtl/td/tl/tl_string_outputer.h b/tdtl/td/tl/tl_string_outputer.h index 9c87c257c5..965ffa40ca 100644 --- a/tdtl/td/tl/tl_string_outputer.h +++ b/tdtl/td/tl/tl_string_outputer.h @@ -18,10 +18,10 @@ */ #pragma once -#include "tl_outputer.h" - #include +#include "tl_outputer.h" + namespace td { namespace tl { diff --git a/tdtl/td/tl/tl_writer.cpp b/tdtl/td/tl/tl_writer.cpp index f4f5f27ccd..f4c9a4d259 100644 --- a/tdtl/td/tl/tl_writer.cpp +++ b/tdtl/td/tl/tl_writer.cpp @@ -16,13 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tl_writer.h" - -#include "tl_core.h" - #include #include +#include "tl_core.h" +#include "tl_writer.h" + namespace td { namespace tl { diff --git a/tdtl/td/tl/tl_writer.h b/tdtl/td/tl/tl_writer.h index acdd77aeb4..53d2091b87 100644 --- a/tdtl/td/tl/tl_writer.h +++ b/tdtl/td/tl/tl_writer.h @@ -18,12 +18,12 @@ */ #pragma once -#include "tl_core.h" - #include #include #include +#include "tl_core.h" + namespace td { namespace tl { diff --git a/tdutils/td/utils/AesCtrByteFlow.h b/tdutils/td/utils/AesCtrByteFlow.h index 0b1f2408d7..d5449458e7 100644 --- a/tdutils/td/utils/AesCtrByteFlow.h +++ b/tdutils/td/utils/AesCtrByteFlow.h @@ -19,11 +19,11 @@ #pragma once #include "td/utils/ByteFlow.h" -#include "td/utils/common.h" -#include "td/utils/crypto.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" #include "td/utils/UInt.h" +#include "td/utils/common.h" +#include "td/utils/crypto.h" namespace td { diff --git a/tdutils/td/utils/BigNum.cpp b/tdutils/td/utils/BigNum.cpp index 22f5244652..e013ae1915 100644 --- a/tdutils/td/utils/BigNum.cpp +++ b/tdutils/td/utils/BigNum.cpp @@ -22,13 +22,12 @@ char disable_linker_warning_about_empty_file_bignum_cpp TD_UNUSED; #if TD_HAVE_OPENSSL -#include "td/utils/logging.h" -#include "td/utils/misc.h" - +#include #include #include -#include +#include "td/utils/logging.h" +#include "td/utils/misc.h" namespace td { diff --git a/tdutils/td/utils/BufferedFd.h b/tdutils/td/utils/BufferedFd.h index 03ab73d8bb..29921dbcd9 100644 --- a/tdutils/td/utils/BufferedFd.h +++ b/tdutils/td/utils/BufferedFd.h @@ -18,17 +18,17 @@ */ #pragma once +#include + +#include "td/utils/Slice.h" +#include "td/utils/Span.h" +#include "td/utils/Status.h" #include "td/utils/buffer.h" #include "td/utils/common.h" #include "td/utils/format.h" #include "td/utils/logging.h" -#include "td/utils/port/detail/PollableFd.h" #include "td/utils/port/IoSlice.h" -#include "td/utils/Slice.h" -#include "td/utils/Span.h" -#include "td/utils/Status.h" - -#include +#include "td/utils/port/detail/PollableFd.h" namespace td { // just reads from given reader and writes to given writer diff --git a/tdutils/td/utils/BufferedReader.h b/tdutils/td/utils/BufferedReader.h index da639b3d7f..f37931185a 100644 --- a/tdutils/td/utils/BufferedReader.h +++ b/tdutils/td/utils/BufferedReader.h @@ -18,10 +18,10 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/port/FileFd.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" +#include "td/utils/common.h" +#include "td/utils/port/FileFd.h" namespace td { diff --git a/tdutils/td/utils/BufferedUdp.h b/tdutils/td/utils/BufferedUdp.h index 3fa93e9da3..2b1c78766c 100644 --- a/tdutils/td/utils/BufferedUdp.h +++ b/tdutils/td/utils/BufferedUdp.h @@ -18,18 +18,18 @@ */ #pragma once +#include + +#include "td/utils/Span.h" +#include "td/utils/Status.h" +#include "td/utils/VectorQueue.h" #include "td/utils/buffer.h" #include "td/utils/common.h" #include "td/utils/logging.h" #include "td/utils/optional.h" +#include "td/utils/port/UdpSocketFd.h" #include "td/utils/port/detail/PollableFd.h" #include "td/utils/port/thread_local.h" -#include "td/utils/port/UdpSocketFd.h" -#include "td/utils/Span.h" -#include "td/utils/Status.h" -#include "td/utils/VectorQueue.h" - -#include namespace td { diff --git a/tdutils/td/utils/ByteFlow.h b/tdutils/td/utils/ByteFlow.h index db30ce4dc3..f9b4d73a33 100644 --- a/tdutils/td/utils/ByteFlow.h +++ b/tdutils/td/utils/ByteFlow.h @@ -18,9 +18,9 @@ */ #pragma once +#include "td/utils/Status.h" #include "td/utils/buffer.h" #include "td/utils/common.h" -#include "td/utils/Status.h" namespace td { diff --git a/tdutils/td/utils/CancellationToken.h b/tdutils/td/utils/CancellationToken.h index 7ef3049799..3f4450e854 100644 --- a/tdutils/td/utils/CancellationToken.h +++ b/tdutils/td/utils/CancellationToken.h @@ -20,6 +20,7 @@ #include #include + #include "Status.h" namespace td { diff --git a/tdutils/td/utils/ChangesProcessor.h b/tdutils/td/utils/ChangesProcessor.h index 16c37dd3ad..41b61ffe17 100644 --- a/tdutils/td/utils/ChangesProcessor.h +++ b/tdutils/td/utils/ChangesProcessor.h @@ -18,10 +18,10 @@ */ #pragma once -#include "td/utils/common.h" - #include +#include "td/utils/common.h" + namespace td { // Process changes after they are finished in order of addition diff --git a/tdutils/td/utils/Closure.h b/tdutils/td/utils/Closure.h index 3407c75428..da3a0df581 100644 --- a/tdutils/td/utils/Closure.h +++ b/tdutils/td/utils/Closure.h @@ -18,15 +18,15 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/invoke.h" -#include "td/utils/logging.h" - #include #include #include #include +#include "td/utils/common.h" +#include "td/utils/invoke.h" +#include "td/utils/logging.h" + // // Essentially we have: // (ActorT::func, arg1, arg2, ..., argn) diff --git a/tdutils/td/utils/ConcurrentHashTable.h b/tdutils/td/utils/ConcurrentHashTable.h index 725d1278dd..e592a63bfa 100644 --- a/tdutils/td/utils/ConcurrentHashTable.h +++ b/tdutils/td/utils/ConcurrentHashTable.h @@ -18,15 +18,15 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/HazardPointers.h" -#include "td/utils/logging.h" -#include "td/utils/port/thread_local.h" - #include #include #include +#include "td/utils/HazardPointers.h" +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/port/thread_local.h" + namespace td { // AtomicHashArray diff --git a/tdutils/td/utils/Container.h b/tdutils/td/utils/Container.h index e235f21f60..7e2f8fec64 100644 --- a/tdutils/td/utils/Container.h +++ b/tdutils/td/utils/Container.h @@ -18,10 +18,10 @@ */ #pragma once -#include "td/utils/common.h" - #include +#include "td/utils/common.h" + namespace td { // 1. Allocates all objects in vector. (but vector never shrinks) diff --git a/tdutils/td/utils/DecTree.h b/tdutils/td/utils/DecTree.h index 1ebb6cd632..1a067154bc 100644 --- a/tdutils/td/utils/DecTree.h +++ b/tdutils/td/utils/DecTree.h @@ -18,12 +18,12 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/Random.h" - #include #include +#include "td/utils/Random.h" +#include "td/utils/common.h" + namespace td { template > diff --git a/tdutils/td/utils/Destructor.h b/tdutils/td/utils/Destructor.h index fadeb89b85..e0597de64d 100644 --- a/tdutils/td/utils/Destructor.h +++ b/tdutils/td/utils/Destructor.h @@ -18,11 +18,11 @@ */ #pragma once -#include "td/utils/common.h" - #include #include +#include "td/utils/common.h" + namespace td { class Destructor { diff --git a/tdutils/td/utils/Enumerator.h b/tdutils/td/utils/Enumerator.h index 6d301ae15c..6cffd9fdf8 100644 --- a/tdutils/td/utils/Enumerator.h +++ b/tdutils/td/utils/Enumerator.h @@ -18,12 +18,12 @@ */ #pragma once -#include "td/utils/common.h" - #include #include #include +#include "td/utils/common.h" + namespace td { template diff --git a/tdutils/td/utils/EpochBasedMemoryReclamation.h b/tdutils/td/utils/EpochBasedMemoryReclamation.h index becff2d722..e0096dd07d 100644 --- a/tdutils/td/utils/EpochBasedMemoryReclamation.h +++ b/tdutils/td/utils/EpochBasedMemoryReclamation.h @@ -18,13 +18,13 @@ */ #pragma once +#include +#include + #include "td/utils/common.h" #include "td/utils/logging.h" #include "td/utils/port/sleep.h" -#include -#include - namespace td { template diff --git a/tdutils/td/utils/FileLog.cpp b/tdutils/td/utils/FileLog.cpp index 036969bffa..8ca7f95b29 100644 --- a/tdutils/td/utils/FileLog.cpp +++ b/tdutils/td/utils/FileLog.cpp @@ -16,16 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/FileLog.h" +#include +#include "td/utils/FileLog.h" +#include "td/utils/Slice.h" #include "td/utils/common.h" #include "td/utils/logging.h" #include "td/utils/port/FileFd.h" -#include "td/utils/port/path.h" #include "td/utils/port/StdStreams.h" -#include "td/utils/Slice.h" - -#include +#include "td/utils/port/path.h" namespace td { diff --git a/tdutils/td/utils/FileLog.h b/tdutils/td/utils/FileLog.h index 0bdbddf018..21cce4cd87 100644 --- a/tdutils/td/utils/FileLog.h +++ b/tdutils/td/utils/FileLog.h @@ -18,11 +18,11 @@ */ #pragma once +#include "td/utils/Slice.h" +#include "td/utils/Status.h" #include "td/utils/common.h" #include "td/utils/logging.h" #include "td/utils/port/FileFd.h" -#include "td/utils/Slice.h" -#include "td/utils/Status.h" namespace td { diff --git a/tdutils/td/utils/FloodControlFast.h b/tdutils/td/utils/FloodControlFast.h index 36bae44f95..07a9e25c1f 100644 --- a/tdutils/td/utils/FloodControlFast.h +++ b/tdutils/td/utils/FloodControlFast.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/utils/common.h" #include "td/utils/TimedStat.h" +#include "td/utils/common.h" namespace td { diff --git a/tdutils/td/utils/FloodControlStrict.h b/tdutils/td/utils/FloodControlStrict.h index 2aa2d02067..fe8daf424b 100644 --- a/tdutils/td/utils/FloodControlStrict.h +++ b/tdutils/td/utils/FloodControlStrict.h @@ -18,10 +18,10 @@ */ #pragma once -#include "td/utils/common.h" - #include +#include "td/utils/common.h" + namespace td { // More strict implementaions of flood control than FloodControlFast. diff --git a/tdutils/td/utils/GitInfo.cpp b/tdutils/td/utils/GitInfo.cpp index 0811e64072..516e6c568f 100644 --- a/tdutils/td/utils/GitInfo.cpp +++ b/tdutils/td/utils/GitInfo.cpp @@ -16,9 +16,8 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/GitInfo.h" - #include "auto/git_info.h" +#include "td/utils/GitInfo.h" namespace td { diff --git a/tdutils/td/utils/Gzip.cpp b/tdutils/td/utils/Gzip.cpp index 5fa764b83f..0b4fded40f 100644 --- a/tdutils/td/utils/Gzip.cpp +++ b/tdutils/td/utils/Gzip.cpp @@ -21,14 +21,13 @@ char disable_linker_warning_about_empty_file_gzip_cpp TD_UNUSED; #if TD_HAVE_ZLIB -#include "td/utils/logging.h" - #include #include #include - #include +#include "td/utils/logging.h" + namespace td { class Gzip::Impl { diff --git a/tdutils/td/utils/Gzip.h b/tdutils/td/utils/Gzip.h index 63e871116d..99f410cc24 100644 --- a/tdutils/td/utils/Gzip.h +++ b/tdutils/td/utils/Gzip.h @@ -21,9 +21,9 @@ #include "td/utils/common.h" #if TD_HAVE_ZLIB -#include "td/utils/buffer.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" +#include "td/utils/buffer.h" namespace td { diff --git a/tdutils/td/utils/GzipByteFlow.cpp b/tdutils/td/utils/GzipByteFlow.cpp index 5cce04eda0..36d9b0c6e7 100644 --- a/tdutils/td/utils/GzipByteFlow.cpp +++ b/tdutils/td/utils/GzipByteFlow.cpp @@ -21,8 +21,8 @@ char disable_linker_warning_about_empty_file_gzipbyteflow_cpp TD_UNUSED; #if TD_HAVE_ZLIB -#include "td/utils/common.h" #include "td/utils/Status.h" +#include "td/utils/common.h" namespace td { diff --git a/tdutils/td/utils/GzipByteFlow.h b/tdutils/td/utils/GzipByteFlow.h index ba78fef46e..370d34ae45 100644 --- a/tdutils/td/utils/GzipByteFlow.h +++ b/tdutils/td/utils/GzipByteFlow.h @@ -18,11 +18,11 @@ */ #pragma once +#include + #include "td/utils/ByteFlow.h" #include "td/utils/Gzip.h" -#include - namespace td { #if TD_HAVE_ZLIB diff --git a/tdutils/td/utils/HashMap.h b/tdutils/td/utils/HashMap.h index d40006785d..7a18949f73 100644 --- a/tdutils/td/utils/HashMap.h +++ b/tdutils/td/utils/HashMap.h @@ -21,12 +21,12 @@ #include "td/utils/Hash.h" #if TD_HAVE_ABSL +#include #include #include -#include #else -#include #include +#include #endif namespace td { diff --git a/tdutils/td/utils/HazardPointers.h b/tdutils/td/utils/HazardPointers.h index 7465140a5d..fe1b7e5447 100644 --- a/tdutils/td/utils/HazardPointers.h +++ b/tdutils/td/utils/HazardPointers.h @@ -18,12 +18,12 @@ */ #pragma once -#include "td/utils/common.h" - #include #include #include +#include "td/utils/common.h" + namespace td { template > diff --git a/tdutils/td/utils/Hints.cpp b/tdutils/td/utils/Hints.cpp index 0b8987df42..e012b72424 100644 --- a/tdutils/td/utils/Hints.cpp +++ b/tdutils/td/utils/Hints.cpp @@ -16,17 +16,16 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/Hints.h" +#include +#include "td/utils/Hints.h" +#include "td/utils/Slice.h" #include "td/utils/logging.h" #include "td/utils/misc.h" -#include "td/utils/Slice.h" #include "td/utils/translit.h" #include "td/utils/unicode.h" #include "td/utils/utf8.h" -#include - namespace td { vector Hints::fix_words(vector words) { diff --git a/tdutils/td/utils/Hints.h b/tdutils/td/utils/Hints.h index 8081d778b8..1cceb12b03 100644 --- a/tdutils/td/utils/Hints.h +++ b/tdutils/td/utils/Hints.h @@ -18,13 +18,13 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/Slice.h" - #include #include #include +#include "td/utils/Slice.h" +#include "td/utils/common.h" + namespace td { // TODO template KeyT diff --git a/tdutils/td/utils/HttpUrl.cpp b/tdutils/td/utils/HttpUrl.cpp index 6e2d1c4d5d..b315795855 100644 --- a/tdutils/td/utils/HttpUrl.cpp +++ b/tdutils/td/utils/HttpUrl.cpp @@ -17,11 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "td/utils/HttpUrl.h" - +#include "td/utils/Parser.h" #include "td/utils/format.h" #include "td/utils/logging.h" #include "td/utils/misc.h" -#include "td/utils/Parser.h" #include "td/utils/port/IPAddress.h" namespace td { diff --git a/tdutils/td/utils/HttpUrl.h b/tdutils/td/utils/HttpUrl.h index f1ce84aeec..bcc9f7942b 100644 --- a/tdutils/td/utils/HttpUrl.h +++ b/tdutils/td/utils/HttpUrl.h @@ -18,10 +18,10 @@ */ #pragma once -#include "td/utils/common.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" #include "td/utils/StringBuilder.h" +#include "td/utils/common.h" namespace td { diff --git a/tdutils/td/utils/JsonBuilder.cpp b/tdutils/td/utils/JsonBuilder.cpp index b858db6632..a11ec37933 100644 --- a/tdutils/td/utils/JsonBuilder.cpp +++ b/tdutils/td/utils/JsonBuilder.cpp @@ -16,12 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/JsonBuilder.h" +#include -#include "td/utils/misc.h" +#include "td/utils/JsonBuilder.h" #include "td/utils/ScopeGuard.h" - -#include +#include "td/utils/misc.h" namespace td { diff --git a/tdutils/td/utils/JsonBuilder.h b/tdutils/td/utils/JsonBuilder.h index 0fdaceb337..55c803a7e1 100644 --- a/tdutils/td/utils/JsonBuilder.h +++ b/tdutils/td/utils/JsonBuilder.h @@ -18,17 +18,17 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/logging.h" +#include +#include +#include + #include "td/utils/Parser.h" #include "td/utils/Slice.h" #include "td/utils/StackAllocator.h" #include "td/utils/Status.h" #include "td/utils/StringBuilder.h" - -#include -#include -#include +#include "td/utils/common.h" +#include "td/utils/logging.h" namespace td { diff --git a/tdutils/td/utils/LRUCache.h b/tdutils/td/utils/LRUCache.h index d8f51525ba..b665cb3ab9 100644 --- a/tdutils/td/utils/LRUCache.h +++ b/tdutils/td/utils/LRUCache.h @@ -16,8 +16,9 @@ */ #pragma once -#include #include +#include + #include "List.h" #include "check.h" diff --git a/tdutils/td/utils/MemoryLog.h b/tdutils/td/utils/MemoryLog.h index 6e78d77502..3ac44c9bc2 100644 --- a/tdutils/td/utils/MemoryLog.h +++ b/tdutils/td/utils/MemoryLog.h @@ -18,14 +18,14 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/logging.h" -#include "td/utils/Slice.h" - #include #include #include +#include "td/utils/Slice.h" +#include "td/utils/common.h" +#include "td/utils/logging.h" + namespace td { template diff --git a/tdutils/td/utils/MimeType.cpp b/tdutils/td/utils/MimeType.cpp index ad873d1c08..d87fe06be6 100644 --- a/tdutils/td/utils/MimeType.cpp +++ b/tdutils/td/utils/MimeType.cpp @@ -17,7 +17,6 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "td/utils/MimeType.h" - #include "td/utils/logging.h" const char *extension_to_mime_type(const char *extension, size_t extension_len); // auto-generated diff --git a/tdutils/td/utils/MimeType.h b/tdutils/td/utils/MimeType.h index 583bd65998..1dc9545bf5 100644 --- a/tdutils/td/utils/MimeType.h +++ b/tdutils/td/utils/MimeType.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/utils/common.h" #include "td/utils/Slice.h" +#include "td/utils/common.h" namespace td { diff --git a/tdutils/td/utils/MpmcQueue.h b/tdutils/td/utils/MpmcQueue.h index 1a5f8fa366..ef8857bb46 100644 --- a/tdutils/td/utils/MpmcQueue.h +++ b/tdutils/td/utils/MpmcQueue.h @@ -23,14 +23,14 @@ // To close queue, one should send as much sentinel elements as there are readers. // Once there are no readers and writers, one may easily destroy queue -#include "td/utils/format.h" +#include +#include + #include "td/utils/HazardPointers.h" +#include "td/utils/ScopeGuard.h" +#include "td/utils/format.h" #include "td/utils/logging.h" #include "td/utils/port/thread.h" -#include "td/utils/ScopeGuard.h" - -#include -#include namespace td { diff --git a/tdutils/td/utils/MpmcWaiter.h b/tdutils/td/utils/MpmcWaiter.h index 8fb1a5111e..61db2f8e81 100644 --- a/tdutils/td/utils/MpmcWaiter.h +++ b/tdutils/td/utils/MpmcWaiter.h @@ -18,15 +18,15 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/logging.h" -#include "td/utils/port/thread.h" - -#include #include +#include #include #include +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/port/thread.h" + namespace td { class MpmcEagerWaiter { diff --git a/tdutils/td/utils/MpscLinkQueue.h b/tdutils/td/utils/MpscLinkQueue.h index 41c5ce8509..6957b84614 100644 --- a/tdutils/td/utils/MpscLinkQueue.h +++ b/tdutils/td/utils/MpscLinkQueue.h @@ -18,10 +18,10 @@ */ #pragma once -#include "td/utils/common.h" - #include +#include "td/utils/common.h" + namespace td { //NB: holder of the queue holds all responsibility of freeing its nodes class MpscLinkQueueImpl { diff --git a/tdutils/td/utils/MpscPollableQueue.h b/tdutils/td/utils/MpscPollableQueue.h index cdace8d12a..1ad72da0c1 100644 --- a/tdutils/td/utils/MpscPollableQueue.h +++ b/tdutils/td/utils/MpscPollableQueue.h @@ -24,10 +24,10 @@ #if !TD_EVENTFD_UNSUPPORTED -#include "td/utils/SpinLock.h" - #include +#include "td/utils/SpinLock.h" + namespace td { // interface like in PollableQueue template diff --git a/tdutils/td/utils/Named.h b/tdutils/td/utils/Named.h index 70fadc077e..77abd4ba96 100644 --- a/tdutils/td/utils/Named.h +++ b/tdutils/td/utils/Named.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/utils/common.h" #include "td/utils/Slice.h" +#include "td/utils/common.h" namespace td { diff --git a/tdutils/td/utils/ObjectPool.h b/tdutils/td/utils/ObjectPool.h index a34ef9087b..4d3fe89aa8 100644 --- a/tdutils/td/utils/ObjectPool.h +++ b/tdutils/td/utils/ObjectPool.h @@ -18,13 +18,13 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/logging.h" - #include #include #include +#include "td/utils/common.h" +#include "td/utils/logging.h" + namespace td { // It is draft object pool implementaion // @@ -168,7 +168,7 @@ class ObjectPool { }; template - OwnerPtr create(ArgsT &&... args) { + OwnerPtr create(ArgsT &&...args) { Storage *storage = get_storage(); storage->init_data(std::forward(args)...); return OwnerPtr(storage, this); @@ -213,7 +213,7 @@ class ObjectPool { std::atomic generation{1}; template - void init_data(ArgsT &&... args) { + void init_data(ArgsT &&...args) { // new (&data) DataT(std::forward(args)...); data = DataT(std::forward(args)...); } diff --git a/tdutils/td/utils/OptionParser.cpp b/tdutils/td/utils/OptionParser.cpp index 634570e1b0..9dfc31159c 100644 --- a/tdutils/td/utils/OptionParser.cpp +++ b/tdutils/td/utils/OptionParser.cpp @@ -17,13 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/OptionParser.h" - -#include "td/utils/misc.h" - #include #include +#include "td/utils/OptionParser.h" +#include "td/utils/misc.h" + namespace td { void OptionParser::set_description(string description) { diff --git a/tdutils/td/utils/OptionParser.h b/tdutils/td/utils/OptionParser.h index 92987c9202..2d2475a347 100644 --- a/tdutils/td/utils/OptionParser.h +++ b/tdutils/td/utils/OptionParser.h @@ -18,12 +18,12 @@ */ #pragma once -#include "td/utils/common.h" +#include + #include "td/utils/Slice.h" #include "td/utils/Status.h" #include "td/utils/StringBuilder.h" - -#include +#include "td/utils/common.h" namespace td { diff --git a/tdutils/td/utils/OptionsParser.cpp b/tdutils/td/utils/OptionsParser.cpp index bf8508c7f5..b4b7bce9c7 100644 --- a/tdutils/td/utils/OptionsParser.cpp +++ b/tdutils/td/utils/OptionsParser.cpp @@ -34,7 +34,7 @@ void OptionParser::set_description(std::string description) { } void OptionParser::add_option(Option::Type type, char short_key, Slice long_key, Slice description, - std::function callback) { + std::function callback) { for (auto &option : options_) { if (option.short_key == short_key || (!long_key.empty() && long_key == option.long_key)) { LOG(ERROR) << "Ignore duplicated option '" << short_key << "' '" << long_key << "'"; @@ -44,12 +44,11 @@ void OptionParser::add_option(Option::Type type, char short_key, Slice long_key, } void OptionParser::add_option(char short_key, Slice long_key, Slice description, - std::function callback) { + std::function callback) { add_option(Option::Type::Arg, short_key, long_key, description, std::move(callback)); } -void OptionParser::add_option(char short_key, Slice long_key, Slice description, - std::function callback) { +void OptionParser::add_option(char short_key, Slice long_key, Slice description, std::function callback) { // Ouch. There must be some better way add_option(Option::Type::NoArg, short_key, long_key, description, std::bind([](std::function &func, Slice) { return func(); }, std::move(callback), diff --git a/tdutils/td/utils/OptionsParser.h b/tdutils/td/utils/OptionsParser.h index a6173784ec..3be784cca8 100644 --- a/tdutils/td/utils/OptionsParser.h +++ b/tdutils/td/utils/OptionsParser.h @@ -18,12 +18,12 @@ */ #pragma once -#include "td/utils/common.h" +#include + #include "td/utils/Slice.h" #include "td/utils/Status.h" #include "td/utils/StringBuilder.h" - -#include +#include "td/utils/common.h" namespace td { diff --git a/tdutils/td/utils/OrderedEventsProcessor.h b/tdutils/td/utils/OrderedEventsProcessor.h index ac2236141c..e68f74c3b0 100644 --- a/tdutils/td/utils/OrderedEventsProcessor.h +++ b/tdutils/td/utils/OrderedEventsProcessor.h @@ -18,11 +18,11 @@ */ #pragma once +#include + #include "td/utils/common.h" #include "td/utils/logging.h" -#include - namespace td { // Process states in order defined by their Id diff --git a/tdutils/td/utils/Parser.h b/tdutils/td/utils/Parser.h index 36a3ad71a4..11ef51dbb2 100644 --- a/tdutils/td/utils/Parser.h +++ b/tdutils/td/utils/Parser.h @@ -18,14 +18,14 @@ */ #pragma once +#include +#include + +#include "td/utils/Slice.h" +#include "td/utils/Status.h" #include "td/utils/common.h" #include "td/utils/format.h" #include "td/utils/logging.h" -#include "td/utils/Slice.h" -#include "td/utils/Status.h" - -#include -#include namespace td { diff --git a/tdutils/td/utils/PathView.cpp b/tdutils/td/utils/PathView.cpp index d4a01dd832..6c47910209 100644 --- a/tdutils/td/utils/PathView.cpp +++ b/tdutils/td/utils/PathView.cpp @@ -18,7 +18,6 @@ */ #include "td/utils/PathView.h" - #include "td/utils/misc.h" namespace td { diff --git a/tdutils/td/utils/Random.cpp b/tdutils/td/utils/Random.cpp index 337662202b..3786f7a83f 100644 --- a/tdutils/td/utils/Random.cpp +++ b/tdutils/td/utils/Random.cpp @@ -17,7 +17,6 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "td/utils/Random.h" - #include "td/utils/logging.h" #include "td/utils/port/thread_local.h" diff --git a/tdutils/td/utils/Random.h b/tdutils/td/utils/Random.h index b877dbe707..f824f713fd 100644 --- a/tdutils/td/utils/Random.h +++ b/tdutils/td/utils/Random.h @@ -18,9 +18,9 @@ */ #pragma once -#include "td/utils/common.h" #include "td/utils/Slice.h" #include "td/utils/Span.h" +#include "td/utils/common.h" namespace td { diff --git a/tdutils/td/utils/ScopeGuard.h b/tdutils/td/utils/ScopeGuard.h index 1fe5069a36..7881770f27 100644 --- a/tdutils/td/utils/ScopeGuard.h +++ b/tdutils/td/utils/ScopeGuard.h @@ -18,13 +18,13 @@ */ #pragma once -#include "td/utils/common.h" - #include #include #include #include +#include "td/utils/common.h" + namespace td { class Guard { diff --git a/tdutils/td/utils/SharedObjectPool.h b/tdutils/td/utils/SharedObjectPool.h index 0a9123d317..797844687d 100644 --- a/tdutils/td/utils/SharedObjectPool.h +++ b/tdutils/td/utils/SharedObjectPool.h @@ -18,15 +18,15 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/logging.h" -#include "td/utils/MpscLinkQueue.h" - #include #include #include #include +#include "td/utils/MpscLinkQueue.h" +#include "td/utils/common.h" +#include "td/utils/logging.h" + namespace td { namespace detail { @@ -59,7 +59,7 @@ class SharedPtrRaw : public DeleterT, private MpscLinkQueueImpl::Node { CHECK(option_magic_ == Magic); } template - void init_data(ArgsT &&... args) { + void init_data(ArgsT &&...args) { new (&option_data_) DataT(std::forward(args)...); } void destroy_data() { @@ -165,13 +165,13 @@ class SharedPtr { } template - static SharedPtr create(ArgsT &&... args) { + static SharedPtr create(ArgsT &&...args) { auto raw = make_unique(DeleterT()); raw->init_data(std::forward(args)...); return SharedPtr(raw.release()); } template - static SharedPtr create_with_deleter(D &&d, ArgsT &&... args) { + static SharedPtr create_with_deleter(D &&d, ArgsT &&...args) { auto raw = make_unique(std::forward(d)); raw->init_data(std::forward(args)...); return SharedPtr(raw.release()); @@ -208,7 +208,7 @@ class SharedObjectPool { } template - Ptr alloc(ArgsT &&... args) { + Ptr alloc(ArgsT &&...args) { auto *raw = alloc_raw(); raw->init_data(std::forward(args)...); return Ptr(raw); diff --git a/tdutils/td/utils/SharedSlice.cpp b/tdutils/td/utils/SharedSlice.cpp index dcfcef7c58..791fe29769 100644 --- a/tdutils/td/utils/SharedSlice.cpp +++ b/tdutils/td/utils/SharedSlice.cpp @@ -18,7 +18,6 @@ */ #include "td/utils/SharedSlice.h" - #include "td/utils/buffer.h" namespace td { diff --git a/tdutils/td/utils/SharedSlice.h b/tdutils/td/utils/SharedSlice.h index eac7ad2fe2..e0ba5ef3b6 100644 --- a/tdutils/td/utils/SharedSlice.h +++ b/tdutils/td/utils/SharedSlice.h @@ -19,13 +19,13 @@ #pragma once -#include "td/utils/Slice.h" - #include #include #include #include +#include "td/utils/Slice.h" + namespace td { namespace detail { diff --git a/tdutils/td/utils/Slice-decl.h b/tdutils/td/utils/Slice-decl.h index 430ad7ff2a..6e75d23f0c 100644 --- a/tdutils/td/utils/Slice-decl.h +++ b/tdutils/td/utils/Slice-decl.h @@ -18,10 +18,10 @@ */ #pragma once -#include "td/utils/common.h" - #include +#include "td/utils/common.h" + namespace td { class Slice; diff --git a/tdutils/td/utils/Slice.h b/tdutils/td/utils/Slice.h index 2aa4627632..86cdaeb9cd 100644 --- a/tdutils/td/utils/Slice.h +++ b/tdutils/td/utils/Slice.h @@ -18,12 +18,12 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/Slice-decl.h" - #include #include +#include "td/utils/Slice-decl.h" +#include "td/utils/common.h" + namespace td { inline MutableSlice::MutableSlice() : s_(const_cast("")), len_(0) { diff --git a/tdutils/td/utils/Span.h b/tdutils/td/utils/Span.h index 060e89a20b..2a2fbb4a6e 100644 --- a/tdutils/td/utils/Span.h +++ b/tdutils/td/utils/Span.h @@ -18,11 +18,11 @@ */ #pragma once -#include "td/utils/common.h" - #include #include +#include "td/utils/common.h" + namespace td { namespace detail { diff --git a/tdutils/td/utils/SpinLock.h b/tdutils/td/utils/SpinLock.h index f0856f0c0a..5c26c7cebb 100644 --- a/tdutils/td/utils/SpinLock.h +++ b/tdutils/td/utils/SpinLock.h @@ -18,11 +18,11 @@ */ #pragma once -#include "td/utils/port/thread.h" - #include #include +#include "td/utils/port/thread.h" + namespace td { class SpinLock { diff --git a/tdutils/td/utils/StackAllocator.cpp b/tdutils/td/utils/StackAllocator.cpp index 76f0dbdfac..cef69c488d 100644 --- a/tdutils/td/utils/StackAllocator.cpp +++ b/tdutils/td/utils/StackAllocator.cpp @@ -17,7 +17,6 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "td/utils/StackAllocator.h" - #include "td/utils/port/thread_local.h" namespace td { diff --git a/tdutils/td/utils/StackAllocator.h b/tdutils/td/utils/StackAllocator.h index b654b1b1e7..395c664a70 100644 --- a/tdutils/td/utils/StackAllocator.h +++ b/tdutils/td/utils/StackAllocator.h @@ -18,14 +18,14 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/MovableValue.h" -#include "td/utils/Slice.h" - #include #include #include +#include "td/utils/MovableValue.h" +#include "td/utils/Slice.h" +#include "td/utils/common.h" + namespace td { class StackAllocator { diff --git a/tdutils/td/utils/Status.cpp b/tdutils/td/utils/Status.cpp index 0dc4071e13..c7767176c8 100644 --- a/tdutils/td/utils/Status.cpp +++ b/tdutils/td/utils/Status.cpp @@ -23,11 +23,10 @@ #endif #if TD_PORT_POSIX -#include "td/utils/port/thread_local.h" - +#include #include -#include +#include "td/utils/port/thread_local.h" #endif namespace td { diff --git a/tdutils/td/utils/Status.h b/tdutils/td/utils/Status.h index 27e09af59f..d43e4fd97d 100644 --- a/tdutils/td/utils/Status.h +++ b/tdutils/td/utils/Status.h @@ -18,13 +18,6 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/logging.h" -#include "td/utils/ScopeGuard.h" -#include "td/utils/Slice.h" -#include "td/utils/StackAllocator.h" -#include "td/utils/StringBuilder.h" - #include #include #include @@ -32,6 +25,13 @@ #include #include +#include "td/utils/ScopeGuard.h" +#include "td/utils/Slice.h" +#include "td/utils/StackAllocator.h" +#include "td/utils/StringBuilder.h" +#include "td/utils/common.h" +#include "td/utils/logging.h" + #define TRY_STATUS(status) \ { \ auto try_status = (status); \ @@ -460,7 +460,7 @@ class Result { } struct emplace_t {}; template - Result(emplace_t, ArgsT &&... args) : status_(), value_(std::forward(args)...) { + Result(emplace_t, ArgsT &&...args) : status_(), value_(std::forward(args)...) { } Result(Status &&status) : status_(std::move(status)) { CHECK(status_.is_error()); @@ -495,7 +495,7 @@ class Result { return *this; } template - void emplace(ArgsT &&... args) { + void emplace(ArgsT &&...args) { if (status_.is_ok()) { value_.~T(); } @@ -614,7 +614,7 @@ class Result { ResultUnwrap try_unwrap() && { return ResultUnwrap(std::move(*this)); } - + // Returns a wrapper that prevents error propagation when co_awaited ResultWrap wrap() && { return ResultWrap(std::move(*this)); @@ -627,7 +627,6 @@ class Result { }; }; - // Wrapper to prevent error propagation when co_awaiting Result template struct ResultWrap { @@ -637,7 +636,6 @@ struct ResultWrap { template struct ResultUnwrap { Result result; - }; template <> diff --git a/tdutils/td/utils/StealingQueue.h b/tdutils/td/utils/StealingQueue.h index 1a2c74c73d..e3d3559270 100644 --- a/tdutils/td/utils/StealingQueue.h +++ b/tdutils/td/utils/StealingQueue.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/utils/Status.h" #include "td/utils/Span.h" +#include "td/utils/Status.h" namespace td { template class StealingQueue { diff --git a/tdutils/td/utils/Storer.h b/tdutils/td/utils/Storer.h index be40a110c6..d0e731d34c 100644 --- a/tdutils/td/utils/Storer.h +++ b/tdutils/td/utils/Storer.h @@ -18,15 +18,14 @@ */ #pragma once -#include "td/utils/StorerBase.h" +#include +#include -#include "td/utils/common.h" #include "td/utils/Slice.h" +#include "td/utils/StorerBase.h" +#include "td/utils/common.h" #include "td/utils/tl_storers.h" -#include -#include - namespace td { class SliceStorer : public Storer { diff --git a/tdutils/td/utils/StringBuilder.cpp b/tdutils/td/utils/StringBuilder.cpp index 07fa0175c9..e8c22730ce 100644 --- a/tdutils/td/utils/StringBuilder.cpp +++ b/tdutils/td/utils/StringBuilder.cpp @@ -16,12 +16,6 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/StringBuilder.h" - -#include "td/utils/misc.h" -#include "td/utils/port/thread_local.h" -#include "td/utils/Slice.h" - #include #include #include @@ -30,6 +24,11 @@ #include #include +#include "td/utils/Slice.h" +#include "td/utils/StringBuilder.h" +#include "td/utils/misc.h" +#include "td/utils/port/thread_local.h" + namespace td { StringBuilder::StringBuilder(MutableSlice slice, bool use_buffer) diff --git a/tdutils/td/utils/StringBuilder.h b/tdutils/td/utils/StringBuilder.h index 685416fe35..8f48f864fd 100644 --- a/tdutils/td/utils/StringBuilder.h +++ b/tdutils/td/utils/StringBuilder.h @@ -18,14 +18,14 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/Slice.h" -#include "td/utils/StackAllocator.h" - #include #include #include +#include "td/utils/Slice.h" +#include "td/utils/StackAllocator.h" +#include "td/utils/common.h" + namespace td { class StringBuilder { @@ -151,16 +151,16 @@ std::enable_if_t::value, string> to_string(const T &x) { template struct LambdaPrintHelper { - SB& sb; + SB &sb; }; template -SB& operator<<(const LambdaPrintHelper& helper, F&& f) { +SB &operator<<(const LambdaPrintHelper &helper, F &&f) { f(helper.sb); return helper.sb; } struct LambdaPrint {}; -inline LambdaPrintHelper operator<<(td::StringBuilder& sb, const LambdaPrint&) { +inline LambdaPrintHelper operator<<(td::StringBuilder &sb, const LambdaPrint &) { return LambdaPrintHelper{sb}; } diff --git a/tdutils/td/utils/ThreadLocalStorage.h b/tdutils/td/utils/ThreadLocalStorage.h index a09b05ff63..9691354c20 100644 --- a/tdutils/td/utils/ThreadLocalStorage.h +++ b/tdutils/td/utils/ThreadLocalStorage.h @@ -19,12 +19,12 @@ #pragma once -#include "td/utils/common.h" -#include "td/utils/port/thread_local.h" - #include #include +#include "td/utils/common.h" +#include "td/utils/port/thread_local.h" + namespace td { template class ThreadLocalStorage { diff --git a/tdutils/td/utils/ThreadSafeCounter.h b/tdutils/td/utils/ThreadSafeCounter.h index 8601275a9e..5d585aa578 100644 --- a/tdutils/td/utils/ThreadSafeCounter.h +++ b/tdutils/td/utils/ThreadSafeCounter.h @@ -19,6 +19,11 @@ #pragma once +#include +#include +#include +#include + #include "port/thread.h" #include "td/utils/Slice.h" #include "td/utils/StringBuilder.h" @@ -27,11 +32,6 @@ #include "td/utils/logging.h" #include "td/utils/port/Clocks.h" -#include -#include -#include -#include - namespace td { template class ThreadSafeMultiCounter { @@ -231,6 +231,7 @@ struct NamedPerfCounter { friend StringBuilder &operator<<(StringBuilder &sb, const NamedPerfCounter &counter) { return sb << counter.counter_; } + private: NamedThreadSafeCounter counter_; }; diff --git a/tdutils/td/utils/Time.cpp b/tdutils/td/utils/Time.cpp index c792dd24de..56ccaf9629 100644 --- a/tdutils/td/utils/Time.cpp +++ b/tdutils/td/utils/Time.cpp @@ -16,10 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/Time.h" - -#include #include +#include + +#include "td/utils/Time.h" namespace td { diff --git a/tdutils/td/utils/TimedStat.h b/tdutils/td/utils/TimedStat.h index 6e9498827b..5a794db8d5 100644 --- a/tdutils/td/utils/TimedStat.h +++ b/tdutils/td/utils/TimedStat.h @@ -18,11 +18,11 @@ */ #pragma once +#include + #include "td/utils/common.h" #include "td/utils/optional.h" -#include - namespace td { template diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index abb69b1000..f784a4eb85 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -16,14 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/Timer.h" +#include +#include +#include "td/utils/Time.h" +#include "td/utils/Timer.h" #include "td/utils/format.h" #include "td/utils/logging.h" -#include "td/utils/Time.h" - -#include -#include namespace td { diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index de793aec07..3f183a9f84 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -18,11 +18,11 @@ */ #pragma once -#include "td/utils/StringBuilder.h" -#include "td/utils/Status.h" - #include +#include "td/utils/Status.h" +#include "td/utils/StringBuilder.h" + namespace td { class Timer { diff --git a/tdutils/td/utils/TsFileLog.cpp b/tdutils/td/utils/TsFileLog.cpp index e7bb0665f2..ccea5c6e48 100644 --- a/tdutils/td/utils/TsFileLog.cpp +++ b/tdutils/td/utils/TsFileLog.cpp @@ -17,16 +17,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/TsFileLog.h" +#include +#include -#include "td/utils/common.h" #include "td/utils/FileLog.h" +#include "td/utils/Slice.h" +#include "td/utils/TsFileLog.h" +#include "td/utils/common.h" #include "td/utils/logging.h" #include "td/utils/port/thread_local.h" -#include "td/utils/Slice.h" - -#include -#include namespace td { namespace detail { diff --git a/tdutils/td/utils/TsFileLog.h b/tdutils/td/utils/TsFileLog.h index 1cf3e8f030..0a7d4c99bd 100644 --- a/tdutils/td/utils/TsFileLog.h +++ b/tdutils/td/utils/TsFileLog.h @@ -19,9 +19,9 @@ #pragma once +#include "td/utils/Status.h" #include "td/utils/common.h" #include "td/utils/logging.h" -#include "td/utils/Status.h" namespace td { class TsFileLog { diff --git a/tdutils/td/utils/TsList.h b/tdutils/td/utils/TsList.h index bee326fc09..cc85bf3a41 100644 --- a/tdutils/td/utils/TsList.h +++ b/tdutils/td/utils/TsList.h @@ -19,11 +19,11 @@ #pragma once -#include "td/utils/common.h" -#include "td/utils/List.h" - #include +#include "td/utils/List.h" +#include "td/utils/common.h" + namespace td { template diff --git a/tdutils/td/utils/UInt.h b/tdutils/td/utils/UInt.h index 45c43e2d2a..e8374950c0 100644 --- a/tdutils/td/utils/UInt.h +++ b/tdutils/td/utils/UInt.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/utils/common.h" #include "td/utils/Slice.h" +#include "td/utils/common.h" namespace td { diff --git a/tdutils/td/utils/Variant.h b/tdutils/td/utils/Variant.h index 708fc1cb66..02f21c6774 100644 --- a/tdutils/td/utils/Variant.h +++ b/tdutils/td/utils/Variant.h @@ -18,13 +18,13 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/logging.h" - #include #include #include +#include "td/utils/common.h" +#include "td/utils/logging.h" + namespace td { namespace detail { diff --git a/tdutils/td/utils/VectorQueue.h b/tdutils/td/utils/VectorQueue.h index a55b7208e1..a20561ef75 100644 --- a/tdutils/td/utils/VectorQueue.h +++ b/tdutils/td/utils/VectorQueue.h @@ -18,11 +18,11 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/Span.h" - #include +#include "td/utils/Span.h" +#include "td/utils/common.h" + namespace td { template @@ -33,7 +33,7 @@ class VectorQueue { vector_.emplace_back(std::forward(s)); } template - void emplace(Args &&... args) { + void emplace(Args &&...args) { vector_.emplace_back(std::forward(args)...); } T pop() { diff --git a/tdutils/td/utils/base64.cpp b/tdutils/td/utils/base64.cpp index d9bb3e01c5..afa6d45385 100644 --- a/tdutils/td/utils/base64.cpp +++ b/tdutils/td/utils/base64.cpp @@ -17,14 +17,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/base64.h" +#include +#include -#include "td/utils/common.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" - -#include -#include +#include "td/utils/base64.h" +#include "td/utils/common.h" namespace td { diff --git a/tdutils/td/utils/base64.h b/tdutils/td/utils/base64.h index d785302439..4ad15e15df 100644 --- a/tdutils/td/utils/base64.h +++ b/tdutils/td/utils/base64.h @@ -19,10 +19,10 @@ #pragma once -#include "td/utils/common.h" #include "td/utils/SharedSlice.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" +#include "td/utils/common.h" namespace td { diff --git a/tdutils/td/utils/benchmark.h b/tdutils/td/utils/benchmark.h index d2cd9b3c7b..9cab964d2f 100644 --- a/tdutils/td/utils/benchmark.h +++ b/tdutils/td/utils/benchmark.h @@ -18,15 +18,15 @@ */ #pragma once -#include "td/utils/format.h" -#include "td/utils/logging.h" -#include "td/utils/port/Clocks.h" -#include "td/utils/StringBuilder.h" - #include #include #include +#include "td/utils/StringBuilder.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/port/Clocks.h" + #define BENCH(name, desc) \ class name##Bench : public ::td::Benchmark { \ public: \ diff --git a/tdutils/td/utils/buffer.cpp b/tdutils/td/utils/buffer.cpp index 07cd485783..a8b9b5924b 100644 --- a/tdutils/td/utils/buffer.cpp +++ b/tdutils/td/utils/buffer.cpp @@ -16,13 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/buffer.h" - -#include "td/utils/port/thread_local.h" - #include #include +#include "td/utils/buffer.h" +#include "td/utils/port/thread_local.h" + // fixes https://bugs.llvm.org/show_bug.cgi?id=33723 for clang >= 3.6 + c++11 + libc++ #if TD_CLANG && _LIBCPP_VERSION #define TD_OFFSETOF __builtin_offsetof diff --git a/tdutils/td/utils/buffer.h b/tdutils/td/utils/buffer.h index 848da2c5c8..35aa8fbc63 100644 --- a/tdutils/td/utils/buffer.h +++ b/tdutils/td/utils/buffer.h @@ -18,14 +18,14 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/port/thread_local.h" -#include "td/utils/Slice.h" - #include #include #include +#include "td/utils/Slice.h" +#include "td/utils/common.h" +#include "td/utils/port/thread_local.h" + namespace td { struct BufferRaw { diff --git a/tdutils/td/utils/check.cpp b/tdutils/td/utils/check.cpp index a4676d0782..01caeb7ce5 100644 --- a/tdutils/td/utils/check.cpp +++ b/tdutils/td/utils/check.cpp @@ -16,10 +16,9 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "td/utils/Slice.h" #include "td/utils/check.h" - #include "td/utils/logging.h" -#include "td/utils/Slice.h" namespace td { namespace detail { diff --git a/tdutils/td/utils/common.h b/tdutils/td/utils/common.h index f9f86c5c10..b2d74f24ff 100644 --- a/tdutils/td/utils/common.h +++ b/tdutils/td/utils/common.h @@ -54,13 +54,13 @@ #endif // clang-format on +#include +#include + #include "td/utils/check.h" #include "td/utils/int_types.h" #include "td/utils/unique_ptr.h" -#include -#include - #define TD_DEBUG #define TD_DEFINE_STR_IMPL(x) #x diff --git a/tdutils/td/utils/crypto.cpp b/tdutils/td/utils/crypto.cpp index ea1efbe794..bc39ff553f 100644 --- a/tdutils/td/utils/crypto.cpp +++ b/tdutils/td/utils/crypto.cpp @@ -16,20 +16,19 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/crypto.h" - -#include "td/utils/as.h" #include "td/utils/BigNum.h" -#include "td/utils/bits.h" -#include "td/utils/common.h" -#include "td/utils/logging.h" -#include "td/utils/misc.h" -#include "td/utils/port/RwMutex.h" #include "td/utils/Random.h" #include "td/utils/ScopeGuard.h" #include "td/utils/SharedSlice.h" #include "td/utils/StackAllocator.h" #include "td/utils/StringBuilder.h" +#include "td/utils/as.h" +#include "td/utils/bits.h" +#include "td/utils/common.h" +#include "td/utils/crypto.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/port/RwMutex.h" #if TD_HAVE_OPENSSL #include diff --git a/tdutils/td/utils/crypto.h b/tdutils/td/utils/crypto.h index 4494ef4818..904a3be1a2 100644 --- a/tdutils/td/utils/crypto.h +++ b/tdutils/td/utils/crypto.h @@ -18,12 +18,11 @@ */ #pragma once -#include "td/utils/buffer.h" -#include "td/utils/common.h" #include "td/utils/SharedSlice.h" #include "td/utils/Slice.h" -#include "td/utils/SharedSlice.h" #include "td/utils/Status.h" +#include "td/utils/buffer.h" +#include "td/utils/common.h" namespace td { diff --git a/tdutils/td/utils/date.h b/tdutils/td/utils/date.h index a632737e4f..08f53edc72 100644 --- a/tdutils/td/utils/date.h +++ b/tdutils/td/utils/date.h @@ -33,15 +33,15 @@ // We did not mean to shout. #ifndef HAS_STRING_VIEW -# if __cplusplus >= 201703 || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) -# define HAS_STRING_VIEW 1 -# else -# define HAS_STRING_VIEW 0 -# endif +#if __cplusplus >= 201703 || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) +#define HAS_STRING_VIEW 1 +#else +#define HAS_STRING_VIEW 0 +#endif #endif // HAS_STRING_VIEW -#include #include +#include #include #include #include @@ -62,98 +62,97 @@ #include #include #if HAS_STRING_VIEW -# include +#include #endif -#include #include +#include #ifdef __GNUC__ -# pragma GCC diagnostic push -# if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 7) -# pragma GCC diagnostic ignored "-Wpedantic" -# endif -# if __GNUC__ < 5 - // GCC 4.9 Bug 61489 Wrong warning with -Wmissing-field-initializers -# pragma GCC diagnostic ignored "-Wmissing-field-initializers" -# endif +#pragma GCC diagnostic push +#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 7) +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#if __GNUC__ < 5 +// GCC 4.9 Bug 61489 Wrong warning with -Wmissing-field-initializers +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" +#endif #endif #ifdef _MSC_VER -# pragma warning(push) +#pragma warning(push) // warning C4127: conditional expression is constant -# pragma warning(disable : 4127) +#pragma warning(disable : 4127) #endif -namespace date -{ +namespace date { //---------------+ // Configuration | //---------------+ #ifndef ONLY_C_LOCALE -# define ONLY_C_LOCALE 0 +#define ONLY_C_LOCALE 0 #endif #if defined(_MSC_VER) && (!defined(__clang__) || (_MSC_VER < 1910)) // MSVC -# ifndef _SILENCE_CXX17_UNCAUGHT_EXCEPTION_DEPRECATION_WARNING -# define _SILENCE_CXX17_UNCAUGHT_EXCEPTION_DEPRECATION_WARNING -# endif -# if _MSC_VER < 1910 +#ifndef _SILENCE_CXX17_UNCAUGHT_EXCEPTION_DEPRECATION_WARNING +#define _SILENCE_CXX17_UNCAUGHT_EXCEPTION_DEPRECATION_WARNING +#endif +#if _MSC_VER < 1910 // before VS2017 -# define CONSTDATA const -# define CONSTCD11 -# define CONSTCD14 -# define NOEXCEPT _NOEXCEPT -# else +#define CONSTDATA const +#define CONSTCD11 +#define CONSTCD14 +#define NOEXCEPT _NOEXCEPT +#else // VS2017 and later -# define CONSTDATA constexpr const -# define CONSTCD11 constexpr -# define CONSTCD14 constexpr -# define NOEXCEPT noexcept -# endif +#define CONSTDATA constexpr const +#define CONSTCD11 constexpr +#define CONSTCD14 constexpr +#define NOEXCEPT noexcept +#endif #elif defined(__SUNPRO_CC) && __SUNPRO_CC <= 0x5150 // Oracle Developer Studio 12.6 and earlier -# define CONSTDATA constexpr const -# define CONSTCD11 constexpr -# define CONSTCD14 -# define NOEXCEPT noexcept +#define CONSTDATA constexpr const +#define CONSTCD11 constexpr +#define CONSTCD14 +#define NOEXCEPT noexcept #elif __cplusplus >= 201402 // C++14 -# define CONSTDATA constexpr const -# define CONSTCD11 constexpr -# define CONSTCD14 constexpr -# define NOEXCEPT noexcept +#define CONSTDATA constexpr const +#define CONSTCD11 constexpr +#define CONSTCD14 constexpr +#define NOEXCEPT noexcept #else // C++11 -# define CONSTDATA constexpr const -# define CONSTCD11 constexpr -# define CONSTCD14 -# define NOEXCEPT noexcept +#define CONSTDATA constexpr const +#define CONSTCD11 constexpr +#define CONSTCD14 +#define NOEXCEPT noexcept #endif #ifndef HAS_UNCAUGHT_EXCEPTIONS -# if __cplusplus > 201703 || (defined(_MSVC_LANG) && _MSVC_LANG > 201703L) -# define HAS_UNCAUGHT_EXCEPTIONS 1 -# else -# define HAS_UNCAUGHT_EXCEPTIONS 0 -# endif +#if __cplusplus > 201703 || (defined(_MSVC_LANG) && _MSVC_LANG > 201703L) +#define HAS_UNCAUGHT_EXCEPTIONS 1 +#else +#define HAS_UNCAUGHT_EXCEPTIONS 0 +#endif #endif // HAS_UNCAUGHT_EXCEPTIONS #ifndef HAS_VOID_T -# if __cplusplus >= 201703 || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) -# define HAS_VOID_T 1 -# else -# define HAS_VOID_T 0 -# endif +#if __cplusplus >= 201703 || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) +#define HAS_VOID_T 1 +#else +#define HAS_VOID_T 0 +#endif #endif // HAS_VOID_T // Protect from Oracle sun macro #ifdef sun -# undef sun +#undef sun #endif // Work around for a NVCC compiler bug which causes it to fail @@ -173,39 +172,34 @@ using ratio_divide = decltype(std::ratio_divide{}); // durations -using days = std::chrono::duration - , std::chrono::hours::period>>; +using days = std::chrono::duration, std::chrono::hours::period>>; -using weeks = std::chrono::duration - , days::period>>; +using weeks = std::chrono::duration, days::period>>; -using years = std::chrono::duration - , days::period>>; +using years = std::chrono::duration, days::period>>; -using months = std::chrono::duration - >>; +using months = std::chrono::duration>>; // time_point template - using sys_time = std::chrono::time_point; +using sys_time = std::chrono::time_point; -using sys_days = sys_time; +using sys_days = sys_time; using sys_seconds = sys_time; struct local_t {}; template - using local_time = std::chrono::time_point; +using local_time = std::chrono::time_point; using local_seconds = local_time; -using local_days = local_time; +using local_days = local_time; // types -struct last_spec -{ - explicit last_spec() = default; +struct last_spec { + explicit last_spec() = default; }; class day; @@ -231,307 +225,288 @@ class year_month_weekday_last; // date composition operators CONSTCD11 year_month operator/(const year& y, const month& m) NOEXCEPT; -CONSTCD11 year_month operator/(const year& y, int m) NOEXCEPT; +CONSTCD11 year_month operator/(const year& y, int m) NOEXCEPT; CONSTCD11 month_day operator/(const day& d, const month& m) NOEXCEPT; -CONSTCD11 month_day operator/(const day& d, int m) NOEXCEPT; +CONSTCD11 month_day operator/(const day& d, int m) NOEXCEPT; CONSTCD11 month_day operator/(const month& m, const day& d) NOEXCEPT; -CONSTCD11 month_day operator/(const month& m, int d) NOEXCEPT; -CONSTCD11 month_day operator/(int m, const day& d) NOEXCEPT; +CONSTCD11 month_day operator/(const month& m, int d) NOEXCEPT; +CONSTCD11 month_day operator/(int m, const day& d) NOEXCEPT; CONSTCD11 month_day_last operator/(const month& m, last_spec) NOEXCEPT; -CONSTCD11 month_day_last operator/(int m, last_spec) NOEXCEPT; +CONSTCD11 month_day_last operator/(int m, last_spec) NOEXCEPT; CONSTCD11 month_day_last operator/(last_spec, const month& m) NOEXCEPT; -CONSTCD11 month_day_last operator/(last_spec, int m) NOEXCEPT; +CONSTCD11 month_day_last operator/(last_spec, int m) NOEXCEPT; CONSTCD11 month_weekday operator/(const month& m, const weekday_indexed& wdi) NOEXCEPT; -CONSTCD11 month_weekday operator/(int m, const weekday_indexed& wdi) NOEXCEPT; +CONSTCD11 month_weekday operator/(int m, const weekday_indexed& wdi) NOEXCEPT; CONSTCD11 month_weekday operator/(const weekday_indexed& wdi, const month& m) NOEXCEPT; -CONSTCD11 month_weekday operator/(const weekday_indexed& wdi, int m) NOEXCEPT; +CONSTCD11 month_weekday operator/(const weekday_indexed& wdi, int m) NOEXCEPT; CONSTCD11 month_weekday_last operator/(const month& m, const weekday_last& wdl) NOEXCEPT; -CONSTCD11 month_weekday_last operator/(int m, const weekday_last& wdl) NOEXCEPT; +CONSTCD11 month_weekday_last operator/(int m, const weekday_last& wdl) NOEXCEPT; CONSTCD11 month_weekday_last operator/(const weekday_last& wdl, const month& m) NOEXCEPT; -CONSTCD11 month_weekday_last operator/(const weekday_last& wdl, int m) NOEXCEPT; +CONSTCD11 month_weekday_last operator/(const weekday_last& wdl, int m) NOEXCEPT; CONSTCD11 year_month_day operator/(const year_month& ym, const day& d) NOEXCEPT; -CONSTCD11 year_month_day operator/(const year_month& ym, int d) NOEXCEPT; +CONSTCD11 year_month_day operator/(const year_month& ym, int d) NOEXCEPT; CONSTCD11 year_month_day operator/(const year& y, const month_day& md) NOEXCEPT; -CONSTCD11 year_month_day operator/(int y, const month_day& md) NOEXCEPT; +CONSTCD11 year_month_day operator/(int y, const month_day& md) NOEXCEPT; CONSTCD11 year_month_day operator/(const month_day& md, const year& y) NOEXCEPT; -CONSTCD11 year_month_day operator/(const month_day& md, int y) NOEXCEPT; +CONSTCD11 year_month_day operator/(const month_day& md, int y) NOEXCEPT; CONSTCD11 - year_month_day_last operator/(const year_month& ym, last_spec) NOEXCEPT; +year_month_day_last operator/(const year_month& ym, last_spec) NOEXCEPT; CONSTCD11 - year_month_day_last operator/(const year& y, const month_day_last& mdl) NOEXCEPT; +year_month_day_last operator/(const year& y, const month_day_last& mdl) NOEXCEPT; CONSTCD11 - year_month_day_last operator/(int y, const month_day_last& mdl) NOEXCEPT; +year_month_day_last operator/(int y, const month_day_last& mdl) NOEXCEPT; CONSTCD11 - year_month_day_last operator/(const month_day_last& mdl, const year& y) NOEXCEPT; +year_month_day_last operator/(const month_day_last& mdl, const year& y) NOEXCEPT; CONSTCD11 - year_month_day_last operator/(const month_day_last& mdl, int y) NOEXCEPT; +year_month_day_last operator/(const month_day_last& mdl, int y) NOEXCEPT; CONSTCD11 -year_month_weekday -operator/(const year_month& ym, const weekday_indexed& wdi) NOEXCEPT; +year_month_weekday operator/(const year_month& ym, const weekday_indexed& wdi) NOEXCEPT; CONSTCD11 -year_month_weekday -operator/(const year& y, const month_weekday& mwd) NOEXCEPT; +year_month_weekday operator/(const year& y, const month_weekday& mwd) NOEXCEPT; CONSTCD11 -year_month_weekday -operator/(int y, const month_weekday& mwd) NOEXCEPT; +year_month_weekday operator/(int y, const month_weekday& mwd) NOEXCEPT; CONSTCD11 -year_month_weekday -operator/(const month_weekday& mwd, const year& y) NOEXCEPT; +year_month_weekday operator/(const month_weekday& mwd, const year& y) NOEXCEPT; CONSTCD11 -year_month_weekday -operator/(const month_weekday& mwd, int y) NOEXCEPT; +year_month_weekday operator/(const month_weekday& mwd, int y) NOEXCEPT; CONSTCD11 -year_month_weekday_last -operator/(const year_month& ym, const weekday_last& wdl) NOEXCEPT; +year_month_weekday_last operator/(const year_month& ym, const weekday_last& wdl) NOEXCEPT; CONSTCD11 -year_month_weekday_last -operator/(const year& y, const month_weekday_last& mwdl) NOEXCEPT; +year_month_weekday_last operator/(const year& y, const month_weekday_last& mwdl) NOEXCEPT; CONSTCD11 -year_month_weekday_last -operator/(int y, const month_weekday_last& mwdl) NOEXCEPT; +year_month_weekday_last operator/(int y, const month_weekday_last& mwdl) NOEXCEPT; CONSTCD11 -year_month_weekday_last -operator/(const month_weekday_last& mwdl, const year& y) NOEXCEPT; +year_month_weekday_last operator/(const month_weekday_last& mwdl, const year& y) NOEXCEPT; CONSTCD11 -year_month_weekday_last -operator/(const month_weekday_last& mwdl, int y) NOEXCEPT; +year_month_weekday_last operator/(const month_weekday_last& mwdl, int y) NOEXCEPT; // Detailed interface // day -class day -{ - unsigned char d_; +class day { + unsigned char d_; -public: - day() = default; - explicit CONSTCD11 day(unsigned d) NOEXCEPT; + public: + day() = default; + explicit CONSTCD11 day(unsigned d) NOEXCEPT; - CONSTCD14 day& operator++() NOEXCEPT; - CONSTCD14 day operator++(int) NOEXCEPT; - CONSTCD14 day& operator--() NOEXCEPT; - CONSTCD14 day operator--(int) NOEXCEPT; + CONSTCD14 day& operator++() NOEXCEPT; + CONSTCD14 day operator++(int) NOEXCEPT; + CONSTCD14 day& operator--() NOEXCEPT; + CONSTCD14 day operator--(int) NOEXCEPT; - CONSTCD14 day& operator+=(const days& d) NOEXCEPT; - CONSTCD14 day& operator-=(const days& d) NOEXCEPT; + CONSTCD14 day& operator+=(const days& d) NOEXCEPT; + CONSTCD14 day& operator-=(const days& d) NOEXCEPT; - CONSTCD11 explicit operator unsigned() const NOEXCEPT; - CONSTCD11 bool ok() const NOEXCEPT; + CONSTCD11 explicit operator unsigned() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; }; CONSTCD11 bool operator==(const day& x, const day& y) NOEXCEPT; CONSTCD11 bool operator!=(const day& x, const day& y) NOEXCEPT; -CONSTCD11 bool operator< (const day& x, const day& y) NOEXCEPT; -CONSTCD11 bool operator> (const day& x, const day& y) NOEXCEPT; +CONSTCD11 bool operator<(const day& x, const day& y) NOEXCEPT; +CONSTCD11 bool operator>(const day& x, const day& y) NOEXCEPT; CONSTCD11 bool operator<=(const day& x, const day& y) NOEXCEPT; CONSTCD11 bool operator>=(const day& x, const day& y) NOEXCEPT; -CONSTCD11 day operator+(const day& x, const days& y) NOEXCEPT; -CONSTCD11 day operator+(const days& x, const day& y) NOEXCEPT; -CONSTCD11 day operator-(const day& x, const days& y) NOEXCEPT; -CONSTCD11 days operator-(const day& x, const day& y) NOEXCEPT; +CONSTCD11 day operator+(const day& x, const days& y) NOEXCEPT; +CONSTCD11 day operator+(const days& x, const day& y) NOEXCEPT; +CONSTCD11 day operator-(const day& x, const days& y) NOEXCEPT; +CONSTCD11 days operator-(const day& x, const day& y) NOEXCEPT; -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const day& d); +template +std::basic_ostream& operator<<(std::basic_ostream& os, const day& d); // month -class month -{ - unsigned char m_; +class month { + unsigned char m_; -public: - month() = default; - explicit CONSTCD11 month(unsigned m) NOEXCEPT; + public: + month() = default; + explicit CONSTCD11 month(unsigned m) NOEXCEPT; - CONSTCD14 month& operator++() NOEXCEPT; - CONSTCD14 month operator++(int) NOEXCEPT; - CONSTCD14 month& operator--() NOEXCEPT; - CONSTCD14 month operator--(int) NOEXCEPT; + CONSTCD14 month& operator++() NOEXCEPT; + CONSTCD14 month operator++(int) NOEXCEPT; + CONSTCD14 month& operator--() NOEXCEPT; + CONSTCD14 month operator--(int) NOEXCEPT; - CONSTCD14 month& operator+=(const months& m) NOEXCEPT; - CONSTCD14 month& operator-=(const months& m) NOEXCEPT; + CONSTCD14 month& operator+=(const months& m) NOEXCEPT; + CONSTCD14 month& operator-=(const months& m) NOEXCEPT; - CONSTCD11 explicit operator unsigned() const NOEXCEPT; - CONSTCD11 bool ok() const NOEXCEPT; + CONSTCD11 explicit operator unsigned() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; }; CONSTCD11 bool operator==(const month& x, const month& y) NOEXCEPT; CONSTCD11 bool operator!=(const month& x, const month& y) NOEXCEPT; -CONSTCD11 bool operator< (const month& x, const month& y) NOEXCEPT; -CONSTCD11 bool operator> (const month& x, const month& y) NOEXCEPT; +CONSTCD11 bool operator<(const month& x, const month& y) NOEXCEPT; +CONSTCD11 bool operator>(const month& x, const month& y) NOEXCEPT; CONSTCD11 bool operator<=(const month& x, const month& y) NOEXCEPT; CONSTCD11 bool operator>=(const month& x, const month& y) NOEXCEPT; -CONSTCD14 month operator+(const month& x, const months& y) NOEXCEPT; -CONSTCD14 month operator+(const months& x, const month& y) NOEXCEPT; -CONSTCD14 month operator-(const month& x, const months& y) NOEXCEPT; -CONSTCD14 months operator-(const month& x, const month& y) NOEXCEPT; +CONSTCD14 month operator+(const month& x, const months& y) NOEXCEPT; +CONSTCD14 month operator+(const months& x, const month& y) NOEXCEPT; +CONSTCD14 month operator-(const month& x, const months& y) NOEXCEPT; +CONSTCD14 months operator-(const month& x, const month& y) NOEXCEPT; -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const month& m); +template +std::basic_ostream& operator<<(std::basic_ostream& os, const month& m); // year -class year -{ - short y_; +class year { + short y_; -public: - year() = default; - explicit CONSTCD11 year(int y) NOEXCEPT; + public: + year() = default; + explicit CONSTCD11 year(int y) NOEXCEPT; - CONSTCD14 year& operator++() NOEXCEPT; - CONSTCD14 year operator++(int) NOEXCEPT; - CONSTCD14 year& operator--() NOEXCEPT; - CONSTCD14 year operator--(int) NOEXCEPT; + CONSTCD14 year& operator++() NOEXCEPT; + CONSTCD14 year operator++(int) NOEXCEPT; + CONSTCD14 year& operator--() NOEXCEPT; + CONSTCD14 year operator--(int) NOEXCEPT; - CONSTCD14 year& operator+=(const years& y) NOEXCEPT; - CONSTCD14 year& operator-=(const years& y) NOEXCEPT; + CONSTCD14 year& operator+=(const years& y) NOEXCEPT; + CONSTCD14 year& operator-=(const years& y) NOEXCEPT; - CONSTCD11 year operator-() const NOEXCEPT; - CONSTCD11 year operator+() const NOEXCEPT; + CONSTCD11 year operator-() const NOEXCEPT; + CONSTCD11 year operator+() const NOEXCEPT; - CONSTCD11 bool is_leap() const NOEXCEPT; + CONSTCD11 bool is_leap() const NOEXCEPT; - CONSTCD11 explicit operator int() const NOEXCEPT; - CONSTCD11 bool ok() const NOEXCEPT; + CONSTCD11 explicit operator int() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; - static CONSTCD11 year min() NOEXCEPT { return year{-32767}; } - static CONSTCD11 year max() NOEXCEPT { return year{32767}; } + static CONSTCD11 year min() NOEXCEPT { + return year{-32767}; + } + static CONSTCD11 year max() NOEXCEPT { + return year{32767}; + } }; CONSTCD11 bool operator==(const year& x, const year& y) NOEXCEPT; CONSTCD11 bool operator!=(const year& x, const year& y) NOEXCEPT; -CONSTCD11 bool operator< (const year& x, const year& y) NOEXCEPT; -CONSTCD11 bool operator> (const year& x, const year& y) NOEXCEPT; +CONSTCD11 bool operator<(const year& x, const year& y) NOEXCEPT; +CONSTCD11 bool operator>(const year& x, const year& y) NOEXCEPT; CONSTCD11 bool operator<=(const year& x, const year& y) NOEXCEPT; CONSTCD11 bool operator>=(const year& x, const year& y) NOEXCEPT; -CONSTCD11 year operator+(const year& x, const years& y) NOEXCEPT; -CONSTCD11 year operator+(const years& x, const year& y) NOEXCEPT; -CONSTCD11 year operator-(const year& x, const years& y) NOEXCEPT; -CONSTCD11 years operator-(const year& x, const year& y) NOEXCEPT; +CONSTCD11 year operator+(const year& x, const years& y) NOEXCEPT; +CONSTCD11 year operator+(const years& x, const year& y) NOEXCEPT; +CONSTCD11 year operator-(const year& x, const years& y) NOEXCEPT; +CONSTCD11 years operator-(const year& x, const year& y) NOEXCEPT; -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const year& y); +template +std::basic_ostream& operator<<(std::basic_ostream& os, const year& y); // weekday -class weekday -{ - unsigned char wd_; -public: - weekday() = default; - explicit CONSTCD11 weekday(unsigned wd) NOEXCEPT; - CONSTCD14 weekday(const sys_days& dp) NOEXCEPT; - CONSTCD14 explicit weekday(const local_days& dp) NOEXCEPT; - - CONSTCD14 weekday& operator++() NOEXCEPT; - CONSTCD14 weekday operator++(int) NOEXCEPT; - CONSTCD14 weekday& operator--() NOEXCEPT; - CONSTCD14 weekday operator--(int) NOEXCEPT; - - CONSTCD14 weekday& operator+=(const days& d) NOEXCEPT; - CONSTCD14 weekday& operator-=(const days& d) NOEXCEPT; - - CONSTCD11 bool ok() const NOEXCEPT; - - CONSTCD11 unsigned c_encoding() const NOEXCEPT; - CONSTCD11 unsigned iso_encoding() const NOEXCEPT; - - CONSTCD11 weekday_indexed operator[](unsigned index) const NOEXCEPT; - CONSTCD11 weekday_last operator[](last_spec) const NOEXCEPT; - -private: - static CONSTCD14 unsigned char weekday_from_days(int z) NOEXCEPT; - - friend CONSTCD11 bool operator==(const weekday& x, const weekday& y) NOEXCEPT; - friend CONSTCD14 days operator-(const weekday& x, const weekday& y) NOEXCEPT; - friend CONSTCD14 weekday operator+(const weekday& x, const days& y) NOEXCEPT; - template - friend std::basic_ostream& - operator<<(std::basic_ostream& os, const weekday& wd); - friend class weekday_indexed; +class weekday { + unsigned char wd_; + + public: + weekday() = default; + explicit CONSTCD11 weekday(unsigned wd) NOEXCEPT; + CONSTCD14 weekday(const sys_days& dp) NOEXCEPT; + CONSTCD14 explicit weekday(const local_days& dp) NOEXCEPT; + + CONSTCD14 weekday& operator++() NOEXCEPT; + CONSTCD14 weekday operator++(int) NOEXCEPT; + CONSTCD14 weekday& operator--() NOEXCEPT; + CONSTCD14 weekday operator--(int) NOEXCEPT; + + CONSTCD14 weekday& operator+=(const days& d) NOEXCEPT; + CONSTCD14 weekday& operator-=(const days& d) NOEXCEPT; + + CONSTCD11 bool ok() const NOEXCEPT; + + CONSTCD11 unsigned c_encoding() const NOEXCEPT; + CONSTCD11 unsigned iso_encoding() const NOEXCEPT; + + CONSTCD11 weekday_indexed operator[](unsigned index) const NOEXCEPT; + CONSTCD11 weekday_last operator[](last_spec) const NOEXCEPT; + + private: + static CONSTCD14 unsigned char weekday_from_days(int z) NOEXCEPT; + + friend CONSTCD11 bool operator==(const weekday& x, const weekday& y) NOEXCEPT; + friend CONSTCD14 days operator-(const weekday& x, const weekday& y) NOEXCEPT; + friend CONSTCD14 weekday operator+(const weekday& x, const days& y) NOEXCEPT; + template + friend std::basic_ostream& operator<<(std::basic_ostream& os, const weekday& wd); + friend class weekday_indexed; }; CONSTCD11 bool operator==(const weekday& x, const weekday& y) NOEXCEPT; CONSTCD11 bool operator!=(const weekday& x, const weekday& y) NOEXCEPT; -CONSTCD14 weekday operator+(const weekday& x, const days& y) NOEXCEPT; -CONSTCD14 weekday operator+(const days& x, const weekday& y) NOEXCEPT; -CONSTCD14 weekday operator-(const weekday& x, const days& y) NOEXCEPT; -CONSTCD14 days operator-(const weekday& x, const weekday& y) NOEXCEPT; +CONSTCD14 weekday operator+(const weekday& x, const days& y) NOEXCEPT; +CONSTCD14 weekday operator+(const days& x, const weekday& y) NOEXCEPT; +CONSTCD14 weekday operator-(const weekday& x, const days& y) NOEXCEPT; +CONSTCD14 days operator-(const weekday& x, const weekday& y) NOEXCEPT; -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const weekday& wd); +template +std::basic_ostream& operator<<(std::basic_ostream& os, const weekday& wd); // weekday_indexed -class weekday_indexed -{ - unsigned char wd_ : 4; - unsigned char index_ : 4; +class weekday_indexed { + unsigned char wd_ : 4; + unsigned char index_ : 4; -public: - weekday_indexed() = default; - CONSTCD11 weekday_indexed(const date::weekday& wd, unsigned index) NOEXCEPT; + public: + weekday_indexed() = default; + CONSTCD11 weekday_indexed(const date::weekday& wd, unsigned index) NOEXCEPT; - CONSTCD11 date::weekday weekday() const NOEXCEPT; - CONSTCD11 unsigned index() const NOEXCEPT; - CONSTCD11 bool ok() const NOEXCEPT; + CONSTCD11 date::weekday weekday() const NOEXCEPT; + CONSTCD11 unsigned index() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; }; CONSTCD11 bool operator==(const weekday_indexed& x, const weekday_indexed& y) NOEXCEPT; CONSTCD11 bool operator!=(const weekday_indexed& x, const weekday_indexed& y) NOEXCEPT; -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const weekday_indexed& wdi); +template +std::basic_ostream& operator<<(std::basic_ostream& os, const weekday_indexed& wdi); // weekday_last -class weekday_last -{ - date::weekday wd_; +class weekday_last { + date::weekday wd_; -public: - explicit CONSTCD11 weekday_last(const date::weekday& wd) NOEXCEPT; + public: + explicit CONSTCD11 weekday_last(const date::weekday& wd) NOEXCEPT; - CONSTCD11 date::weekday weekday() const NOEXCEPT; - CONSTCD11 bool ok() const NOEXCEPT; + CONSTCD11 date::weekday weekday() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; }; CONSTCD11 bool operator==(const weekday_last& x, const weekday_last& y) NOEXCEPT; CONSTCD11 bool operator!=(const weekday_last& x, const weekday_last& y) NOEXCEPT; -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const weekday_last& wdl); +template +std::basic_ostream& operator<<(std::basic_ostream& os, const weekday_last& wdl); -namespace detail -{ +namespace detail { struct unspecified_month_disambiguator {}; @@ -539,40 +514,39 @@ struct unspecified_month_disambiguator {}; // year_month -class year_month -{ - date::year y_; - date::month m_; +class year_month { + date::year y_; + date::month m_; -public: - year_month() = default; - CONSTCD11 year_month(const date::year& y, const date::month& m) NOEXCEPT; + public: + year_month() = default; + CONSTCD11 year_month(const date::year& y, const date::month& m) NOEXCEPT; - CONSTCD11 date::year year() const NOEXCEPT; - CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::year year() const NOEXCEPT; + CONSTCD11 date::month month() const NOEXCEPT; - template - CONSTCD14 year_month& operator+=(const months& dm) NOEXCEPT; - template - CONSTCD14 year_month& operator-=(const months& dm) NOEXCEPT; - CONSTCD14 year_month& operator+=(const years& dy) NOEXCEPT; - CONSTCD14 year_month& operator-=(const years& dy) NOEXCEPT; + template + CONSTCD14 year_month& operator+=(const months& dm) NOEXCEPT; + template + CONSTCD14 year_month& operator-=(const months& dm) NOEXCEPT; + CONSTCD14 year_month& operator+=(const years& dy) NOEXCEPT; + CONSTCD14 year_month& operator-=(const years& dy) NOEXCEPT; - CONSTCD11 bool ok() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; }; CONSTCD11 bool operator==(const year_month& x, const year_month& y) NOEXCEPT; CONSTCD11 bool operator!=(const year_month& x, const year_month& y) NOEXCEPT; -CONSTCD11 bool operator< (const year_month& x, const year_month& y) NOEXCEPT; -CONSTCD11 bool operator> (const year_month& x, const year_month& y) NOEXCEPT; +CONSTCD11 bool operator<(const year_month& x, const year_month& y) NOEXCEPT; +CONSTCD11 bool operator>(const year_month& x, const year_month& y) NOEXCEPT; CONSTCD11 bool operator<=(const year_month& x, const year_month& y) NOEXCEPT; CONSTCD11 bool operator>=(const year_month& x, const year_month& y) NOEXCEPT; -template +template CONSTCD14 year_month operator+(const year_month& ym, const months& dm) NOEXCEPT; -template +template CONSTCD14 year_month operator+(const months& dm, const year_month& ym) NOEXCEPT; -template +template CONSTCD14 year_month operator-(const year_month& ym, const months& dm) NOEXCEPT; CONSTCD11 months operator-(const year_month& x, const year_month& y) NOEXCEPT; @@ -580,394 +554,343 @@ CONSTCD11 year_month operator+(const year_month& ym, const years& dy) NOEXCEPT; CONSTCD11 year_month operator+(const years& dy, const year_month& ym) NOEXCEPT; CONSTCD11 year_month operator-(const year_month& ym, const years& dy) NOEXCEPT; -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const year_month& ym); +template +std::basic_ostream& operator<<(std::basic_ostream& os, const year_month& ym); // month_day -class month_day -{ - date::month m_; - date::day d_; +class month_day { + date::month m_; + date::day d_; -public: - month_day() = default; - CONSTCD11 month_day(const date::month& m, const date::day& d) NOEXCEPT; + public: + month_day() = default; + CONSTCD11 month_day(const date::month& m, const date::day& d) NOEXCEPT; - CONSTCD11 date::month month() const NOEXCEPT; - CONSTCD11 date::day day() const NOEXCEPT; + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::day day() const NOEXCEPT; - CONSTCD14 bool ok() const NOEXCEPT; + CONSTCD14 bool ok() const NOEXCEPT; }; CONSTCD11 bool operator==(const month_day& x, const month_day& y) NOEXCEPT; CONSTCD11 bool operator!=(const month_day& x, const month_day& y) NOEXCEPT; -CONSTCD11 bool operator< (const month_day& x, const month_day& y) NOEXCEPT; -CONSTCD11 bool operator> (const month_day& x, const month_day& y) NOEXCEPT; +CONSTCD11 bool operator<(const month_day& x, const month_day& y) NOEXCEPT; +CONSTCD11 bool operator>(const month_day& x, const month_day& y) NOEXCEPT; CONSTCD11 bool operator<=(const month_day& x, const month_day& y) NOEXCEPT; CONSTCD11 bool operator>=(const month_day& x, const month_day& y) NOEXCEPT; -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const month_day& md); +template +std::basic_ostream& operator<<(std::basic_ostream& os, const month_day& md); // month_day_last -class month_day_last -{ - date::month m_; +class month_day_last { + date::month m_; -public: - CONSTCD11 explicit month_day_last(const date::month& m) NOEXCEPT; + public: + CONSTCD11 explicit month_day_last(const date::month& m) NOEXCEPT; - CONSTCD11 date::month month() const NOEXCEPT; - CONSTCD11 bool ok() const NOEXCEPT; + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; }; CONSTCD11 bool operator==(const month_day_last& x, const month_day_last& y) NOEXCEPT; CONSTCD11 bool operator!=(const month_day_last& x, const month_day_last& y) NOEXCEPT; -CONSTCD11 bool operator< (const month_day_last& x, const month_day_last& y) NOEXCEPT; -CONSTCD11 bool operator> (const month_day_last& x, const month_day_last& y) NOEXCEPT; +CONSTCD11 bool operator<(const month_day_last& x, const month_day_last& y) NOEXCEPT; +CONSTCD11 bool operator>(const month_day_last& x, const month_day_last& y) NOEXCEPT; CONSTCD11 bool operator<=(const month_day_last& x, const month_day_last& y) NOEXCEPT; CONSTCD11 bool operator>=(const month_day_last& x, const month_day_last& y) NOEXCEPT; -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const month_day_last& mdl); +template +std::basic_ostream& operator<<(std::basic_ostream& os, const month_day_last& mdl); // month_weekday -class month_weekday -{ - date::month m_; - date::weekday_indexed wdi_; -public: - CONSTCD11 month_weekday(const date::month& m, - const date::weekday_indexed& wdi) NOEXCEPT; +class month_weekday { + date::month m_; + date::weekday_indexed wdi_; + + public: + CONSTCD11 month_weekday(const date::month& m, const date::weekday_indexed& wdi) NOEXCEPT; - CONSTCD11 date::month month() const NOEXCEPT; - CONSTCD11 date::weekday_indexed weekday_indexed() const NOEXCEPT; + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::weekday_indexed weekday_indexed() const NOEXCEPT; - CONSTCD11 bool ok() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; }; CONSTCD11 bool operator==(const month_weekday& x, const month_weekday& y) NOEXCEPT; CONSTCD11 bool operator!=(const month_weekday& x, const month_weekday& y) NOEXCEPT; -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const month_weekday& mwd); +template +std::basic_ostream& operator<<(std::basic_ostream& os, const month_weekday& mwd); // month_weekday_last -class month_weekday_last -{ - date::month m_; - date::weekday_last wdl_; +class month_weekday_last { + date::month m_; + date::weekday_last wdl_; -public: - CONSTCD11 month_weekday_last(const date::month& m, - const date::weekday_last& wd) NOEXCEPT; + public: + CONSTCD11 month_weekday_last(const date::month& m, const date::weekday_last& wd) NOEXCEPT; - CONSTCD11 date::month month() const NOEXCEPT; - CONSTCD11 date::weekday_last weekday_last() const NOEXCEPT; + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::weekday_last weekday_last() const NOEXCEPT; - CONSTCD11 bool ok() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; }; CONSTCD11 - bool operator==(const month_weekday_last& x, const month_weekday_last& y) NOEXCEPT; +bool operator==(const month_weekday_last& x, const month_weekday_last& y) NOEXCEPT; CONSTCD11 - bool operator!=(const month_weekday_last& x, const month_weekday_last& y) NOEXCEPT; +bool operator!=(const month_weekday_last& x, const month_weekday_last& y) NOEXCEPT; -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const month_weekday_last& mwdl); +template +std::basic_ostream& operator<<(std::basic_ostream& os, const month_weekday_last& mwdl); // class year_month_day -class year_month_day -{ - date::year y_; - date::month m_; - date::day d_; - -public: - year_month_day() = default; - CONSTCD11 year_month_day(const date::year& y, const date::month& m, - const date::day& d) NOEXCEPT; - CONSTCD14 year_month_day(const year_month_day_last& ymdl) NOEXCEPT; - - CONSTCD14 year_month_day(sys_days dp) NOEXCEPT; - CONSTCD14 explicit year_month_day(local_days dp) NOEXCEPT; - - template - CONSTCD14 year_month_day& operator+=(const months& m) NOEXCEPT; - template - CONSTCD14 year_month_day& operator-=(const months& m) NOEXCEPT; - CONSTCD14 year_month_day& operator+=(const years& y) NOEXCEPT; - CONSTCD14 year_month_day& operator-=(const years& y) NOEXCEPT; - - CONSTCD11 date::year year() const NOEXCEPT; - CONSTCD11 date::month month() const NOEXCEPT; - CONSTCD11 date::day day() const NOEXCEPT; - - CONSTCD14 operator sys_days() const NOEXCEPT; - CONSTCD14 explicit operator local_days() const NOEXCEPT; - CONSTCD14 bool ok() const NOEXCEPT; - -private: - static CONSTCD14 year_month_day from_days(days dp) NOEXCEPT; - CONSTCD14 days to_days() const NOEXCEPT; +class year_month_day { + date::year y_; + date::month m_; + date::day d_; + + public: + year_month_day() = default; + CONSTCD11 year_month_day(const date::year& y, const date::month& m, const date::day& d) NOEXCEPT; + CONSTCD14 year_month_day(const year_month_day_last& ymdl) NOEXCEPT; + + CONSTCD14 year_month_day(sys_days dp) NOEXCEPT; + CONSTCD14 explicit year_month_day(local_days dp) NOEXCEPT; + + template + CONSTCD14 year_month_day& operator+=(const months& m) NOEXCEPT; + template + CONSTCD14 year_month_day& operator-=(const months& m) NOEXCEPT; + CONSTCD14 year_month_day& operator+=(const years& y) NOEXCEPT; + CONSTCD14 year_month_day& operator-=(const years& y) NOEXCEPT; + + CONSTCD11 date::year year() const NOEXCEPT; + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::day day() const NOEXCEPT; + + CONSTCD14 operator sys_days() const NOEXCEPT; + CONSTCD14 explicit operator local_days() const NOEXCEPT; + CONSTCD14 bool ok() const NOEXCEPT; + + private: + static CONSTCD14 year_month_day from_days(days dp) NOEXCEPT; + CONSTCD14 days to_days() const NOEXCEPT; }; CONSTCD11 bool operator==(const year_month_day& x, const year_month_day& y) NOEXCEPT; CONSTCD11 bool operator!=(const year_month_day& x, const year_month_day& y) NOEXCEPT; -CONSTCD11 bool operator< (const year_month_day& x, const year_month_day& y) NOEXCEPT; -CONSTCD11 bool operator> (const year_month_day& x, const year_month_day& y) NOEXCEPT; +CONSTCD11 bool operator<(const year_month_day& x, const year_month_day& y) NOEXCEPT; +CONSTCD11 bool operator>(const year_month_day& x, const year_month_day& y) NOEXCEPT; CONSTCD11 bool operator<=(const year_month_day& x, const year_month_day& y) NOEXCEPT; CONSTCD11 bool operator>=(const year_month_day& x, const year_month_day& y) NOEXCEPT; -template +template CONSTCD14 year_month_day operator+(const year_month_day& ymd, const months& dm) NOEXCEPT; -template +template CONSTCD14 year_month_day operator+(const months& dm, const year_month_day& ymd) NOEXCEPT; -template +template CONSTCD14 year_month_day operator-(const year_month_day& ymd, const months& dm) NOEXCEPT; -CONSTCD11 year_month_day operator+(const year_month_day& ymd, const years& dy) NOEXCEPT; -CONSTCD11 year_month_day operator+(const years& dy, const year_month_day& ymd) NOEXCEPT; -CONSTCD11 year_month_day operator-(const year_month_day& ymd, const years& dy) NOEXCEPT; +CONSTCD11 year_month_day operator+(const year_month_day& ymd, const years& dy) NOEXCEPT; +CONSTCD11 year_month_day operator+(const years& dy, const year_month_day& ymd) NOEXCEPT; +CONSTCD11 year_month_day operator-(const year_month_day& ymd, const years& dy) NOEXCEPT; -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const year_month_day& ymd); +template +std::basic_ostream& operator<<(std::basic_ostream& os, const year_month_day& ymd); // year_month_day_last -class year_month_day_last -{ - date::year y_; - date::month_day_last mdl_; - -public: - CONSTCD11 year_month_day_last(const date::year& y, - const date::month_day_last& mdl) NOEXCEPT; - - template - CONSTCD14 year_month_day_last& operator+=(const months& m) NOEXCEPT; - template - CONSTCD14 year_month_day_last& operator-=(const months& m) NOEXCEPT; - CONSTCD14 year_month_day_last& operator+=(const years& y) NOEXCEPT; - CONSTCD14 year_month_day_last& operator-=(const years& y) NOEXCEPT; - - CONSTCD11 date::year year() const NOEXCEPT; - CONSTCD11 date::month month() const NOEXCEPT; - CONSTCD11 date::month_day_last month_day_last() const NOEXCEPT; - CONSTCD14 date::day day() const NOEXCEPT; - - CONSTCD14 operator sys_days() const NOEXCEPT; - CONSTCD14 explicit operator local_days() const NOEXCEPT; - CONSTCD11 bool ok() const NOEXCEPT; +class year_month_day_last { + date::year y_; + date::month_day_last mdl_; + + public: + CONSTCD11 year_month_day_last(const date::year& y, const date::month_day_last& mdl) NOEXCEPT; + + template + CONSTCD14 year_month_day_last& operator+=(const months& m) NOEXCEPT; + template + CONSTCD14 year_month_day_last& operator-=(const months& m) NOEXCEPT; + CONSTCD14 year_month_day_last& operator+=(const years& y) NOEXCEPT; + CONSTCD14 year_month_day_last& operator-=(const years& y) NOEXCEPT; + + CONSTCD11 date::year year() const NOEXCEPT; + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::month_day_last month_day_last() const NOEXCEPT; + CONSTCD14 date::day day() const NOEXCEPT; + + CONSTCD14 operator sys_days() const NOEXCEPT; + CONSTCD14 explicit operator local_days() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; }; CONSTCD11 - bool operator==(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; +bool operator==(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; CONSTCD11 - bool operator!=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; +bool operator!=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; CONSTCD11 - bool operator< (const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; +bool operator<(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; CONSTCD11 - bool operator> (const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; +bool operator>(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; CONSTCD11 - bool operator<=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; +bool operator<=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; CONSTCD11 - bool operator>=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; +bool operator>=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; -template -CONSTCD14 -year_month_day_last -operator+(const year_month_day_last& ymdl, const months& dm) NOEXCEPT; +template +CONSTCD14 year_month_day_last operator+(const year_month_day_last& ymdl, const months& dm) NOEXCEPT; -template -CONSTCD14 -year_month_day_last -operator+(const months& dm, const year_month_day_last& ymdl) NOEXCEPT; +template +CONSTCD14 year_month_day_last operator+(const months& dm, const year_month_day_last& ymdl) NOEXCEPT; CONSTCD11 -year_month_day_last -operator+(const year_month_day_last& ymdl, const years& dy) NOEXCEPT; +year_month_day_last operator+(const year_month_day_last& ymdl, const years& dy) NOEXCEPT; CONSTCD11 -year_month_day_last -operator+(const years& dy, const year_month_day_last& ymdl) NOEXCEPT; +year_month_day_last operator+(const years& dy, const year_month_day_last& ymdl) NOEXCEPT; -template -CONSTCD14 -year_month_day_last -operator-(const year_month_day_last& ymdl, const months& dm) NOEXCEPT; +template +CONSTCD14 year_month_day_last operator-(const year_month_day_last& ymdl, const months& dm) NOEXCEPT; CONSTCD11 -year_month_day_last -operator-(const year_month_day_last& ymdl, const years& dy) NOEXCEPT; +year_month_day_last operator-(const year_month_day_last& ymdl, const years& dy) NOEXCEPT; -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const year_month_day_last& ymdl); +template +std::basic_ostream& operator<<(std::basic_ostream& os, const year_month_day_last& ymdl); // year_month_weekday -class year_month_weekday -{ - date::year y_; - date::month m_; - date::weekday_indexed wdi_; - -public: - year_month_weekday() = default; - CONSTCD11 year_month_weekday(const date::year& y, const date::month& m, - const date::weekday_indexed& wdi) NOEXCEPT; - CONSTCD14 year_month_weekday(const sys_days& dp) NOEXCEPT; - CONSTCD14 explicit year_month_weekday(const local_days& dp) NOEXCEPT; - - template - CONSTCD14 year_month_weekday& operator+=(const months& m) NOEXCEPT; - template - CONSTCD14 year_month_weekday& operator-=(const months& m) NOEXCEPT; - CONSTCD14 year_month_weekday& operator+=(const years& y) NOEXCEPT; - CONSTCD14 year_month_weekday& operator-=(const years& y) NOEXCEPT; - - CONSTCD11 date::year year() const NOEXCEPT; - CONSTCD11 date::month month() const NOEXCEPT; - CONSTCD11 date::weekday weekday() const NOEXCEPT; - CONSTCD11 unsigned index() const NOEXCEPT; - CONSTCD11 date::weekday_indexed weekday_indexed() const NOEXCEPT; - - CONSTCD14 operator sys_days() const NOEXCEPT; - CONSTCD14 explicit operator local_days() const NOEXCEPT; - CONSTCD14 bool ok() const NOEXCEPT; - -private: - static CONSTCD14 year_month_weekday from_days(days dp) NOEXCEPT; - CONSTCD14 days to_days() const NOEXCEPT; +class year_month_weekday { + date::year y_; + date::month m_; + date::weekday_indexed wdi_; + + public: + year_month_weekday() = default; + CONSTCD11 year_month_weekday(const date::year& y, const date::month& m, const date::weekday_indexed& wdi) NOEXCEPT; + CONSTCD14 year_month_weekday(const sys_days& dp) NOEXCEPT; + CONSTCD14 explicit year_month_weekday(const local_days& dp) NOEXCEPT; + + template + CONSTCD14 year_month_weekday& operator+=(const months& m) NOEXCEPT; + template + CONSTCD14 year_month_weekday& operator-=(const months& m) NOEXCEPT; + CONSTCD14 year_month_weekday& operator+=(const years& y) NOEXCEPT; + CONSTCD14 year_month_weekday& operator-=(const years& y) NOEXCEPT; + + CONSTCD11 date::year year() const NOEXCEPT; + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::weekday weekday() const NOEXCEPT; + CONSTCD11 unsigned index() const NOEXCEPT; + CONSTCD11 date::weekday_indexed weekday_indexed() const NOEXCEPT; + + CONSTCD14 operator sys_days() const NOEXCEPT; + CONSTCD14 explicit operator local_days() const NOEXCEPT; + CONSTCD14 bool ok() const NOEXCEPT; + + private: + static CONSTCD14 year_month_weekday from_days(days dp) NOEXCEPT; + CONSTCD14 days to_days() const NOEXCEPT; }; CONSTCD11 - bool operator==(const year_month_weekday& x, const year_month_weekday& y) NOEXCEPT; +bool operator==(const year_month_weekday& x, const year_month_weekday& y) NOEXCEPT; CONSTCD11 - bool operator!=(const year_month_weekday& x, const year_month_weekday& y) NOEXCEPT; +bool operator!=(const year_month_weekday& x, const year_month_weekday& y) NOEXCEPT; -template -CONSTCD14 -year_month_weekday -operator+(const year_month_weekday& ymwd, const months& dm) NOEXCEPT; +template +CONSTCD14 year_month_weekday operator+(const year_month_weekday& ymwd, const months& dm) NOEXCEPT; -template -CONSTCD14 -year_month_weekday -operator+(const months& dm, const year_month_weekday& ymwd) NOEXCEPT; +template +CONSTCD14 year_month_weekday operator+(const months& dm, const year_month_weekday& ymwd) NOEXCEPT; CONSTCD11 -year_month_weekday -operator+(const year_month_weekday& ymwd, const years& dy) NOEXCEPT; +year_month_weekday operator+(const year_month_weekday& ymwd, const years& dy) NOEXCEPT; CONSTCD11 -year_month_weekday -operator+(const years& dy, const year_month_weekday& ymwd) NOEXCEPT; +year_month_weekday operator+(const years& dy, const year_month_weekday& ymwd) NOEXCEPT; -template -CONSTCD14 -year_month_weekday -operator-(const year_month_weekday& ymwd, const months& dm) NOEXCEPT; +template +CONSTCD14 year_month_weekday operator-(const year_month_weekday& ymwd, const months& dm) NOEXCEPT; CONSTCD11 -year_month_weekday -operator-(const year_month_weekday& ymwd, const years& dy) NOEXCEPT; +year_month_weekday operator-(const year_month_weekday& ymwd, const years& dy) NOEXCEPT; -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const year_month_weekday& ymwdi); +template +std::basic_ostream& operator<<(std::basic_ostream& os, const year_month_weekday& ymwdi); // year_month_weekday_last -class year_month_weekday_last -{ - date::year y_; - date::month m_; - date::weekday_last wdl_; - -public: - CONSTCD11 year_month_weekday_last(const date::year& y, const date::month& m, - const date::weekday_last& wdl) NOEXCEPT; - - template - CONSTCD14 year_month_weekday_last& operator+=(const months& m) NOEXCEPT; - template - CONSTCD14 year_month_weekday_last& operator-=(const months& m) NOEXCEPT; - CONSTCD14 year_month_weekday_last& operator+=(const years& y) NOEXCEPT; - CONSTCD14 year_month_weekday_last& operator-=(const years& y) NOEXCEPT; - - CONSTCD11 date::year year() const NOEXCEPT; - CONSTCD11 date::month month() const NOEXCEPT; - CONSTCD11 date::weekday weekday() const NOEXCEPT; - CONSTCD11 date::weekday_last weekday_last() const NOEXCEPT; - - CONSTCD14 operator sys_days() const NOEXCEPT; - CONSTCD14 explicit operator local_days() const NOEXCEPT; - CONSTCD11 bool ok() const NOEXCEPT; - -private: - CONSTCD14 days to_days() const NOEXCEPT; +class year_month_weekday_last { + date::year y_; + date::month m_; + date::weekday_last wdl_; + + public: + CONSTCD11 year_month_weekday_last(const date::year& y, const date::month& m, const date::weekday_last& wdl) NOEXCEPT; + + template + CONSTCD14 year_month_weekday_last& operator+=(const months& m) NOEXCEPT; + template + CONSTCD14 year_month_weekday_last& operator-=(const months& m) NOEXCEPT; + CONSTCD14 year_month_weekday_last& operator+=(const years& y) NOEXCEPT; + CONSTCD14 year_month_weekday_last& operator-=(const years& y) NOEXCEPT; + + CONSTCD11 date::year year() const NOEXCEPT; + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::weekday weekday() const NOEXCEPT; + CONSTCD11 date::weekday_last weekday_last() const NOEXCEPT; + + CONSTCD14 operator sys_days() const NOEXCEPT; + CONSTCD14 explicit operator local_days() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; + + private: + CONSTCD14 days to_days() const NOEXCEPT; }; CONSTCD11 -bool -operator==(const year_month_weekday_last& x, const year_month_weekday_last& y) NOEXCEPT; +bool operator==(const year_month_weekday_last& x, const year_month_weekday_last& y) NOEXCEPT; CONSTCD11 -bool -operator!=(const year_month_weekday_last& x, const year_month_weekday_last& y) NOEXCEPT; +bool operator!=(const year_month_weekday_last& x, const year_month_weekday_last& y) NOEXCEPT; -template -CONSTCD14 -year_month_weekday_last -operator+(const year_month_weekday_last& ymwdl, const months& dm) NOEXCEPT; +template +CONSTCD14 year_month_weekday_last operator+(const year_month_weekday_last& ymwdl, const months& dm) NOEXCEPT; -template -CONSTCD14 -year_month_weekday_last -operator+(const months& dm, const year_month_weekday_last& ymwdl) NOEXCEPT; +template +CONSTCD14 year_month_weekday_last operator+(const months& dm, const year_month_weekday_last& ymwdl) NOEXCEPT; CONSTCD11 -year_month_weekday_last -operator+(const year_month_weekday_last& ymwdl, const years& dy) NOEXCEPT; +year_month_weekday_last operator+(const year_month_weekday_last& ymwdl, const years& dy) NOEXCEPT; CONSTCD11 -year_month_weekday_last -operator+(const years& dy, const year_month_weekday_last& ymwdl) NOEXCEPT; +year_month_weekday_last operator+(const years& dy, const year_month_weekday_last& ymwdl) NOEXCEPT; -template -CONSTCD14 -year_month_weekday_last -operator-(const year_month_weekday_last& ymwdl, const months& dm) NOEXCEPT; +template +CONSTCD14 year_month_weekday_last operator-(const year_month_weekday_last& ymwdl, const months& dm) NOEXCEPT; CONSTCD11 -year_month_weekday_last -operator-(const year_month_weekday_last& ymwdl, const years& dy) NOEXCEPT; +year_month_weekday_last operator-(const year_month_weekday_last& ymwdl, const years& dy) NOEXCEPT; -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const year_month_weekday_last& ymwdl); +template +std::basic_ostream& operator<<(std::basic_ostream& os, + const year_month_weekday_last& ymwdl); #if !defined(_MSC_VER) || (_MSC_VER >= 1900) -inline namespace literals -{ +inline namespace literals { -CONSTCD11 date::day operator "" _d(unsigned long long d) NOEXCEPT; -CONSTCD11 date::year operator "" _y(unsigned long long y) NOEXCEPT; +CONSTCD11 date::day operator"" _d(unsigned long long d) NOEXCEPT; +CONSTCD11 date::year operator"" _y(unsigned long long y) NOEXCEPT; -} // inline namespace literals -#endif // !defined(_MSC_VER) || (_MSC_VER >= 1900) +} // namespace literals +#endif // !defined(_MSC_VER) || (_MSC_VER >= 1900) // CONSTDATA date::month January{1}; // CONSTDATA date::month February{2}; @@ -993,16 +916,11 @@ CONSTCD11 date::year operator "" _y(unsigned long long y) NOEXCEPT; #if HAS_VOID_T template > -struct is_clock - : std::false_type -{}; +struct is_clock : std::false_type {}; template -struct is_clock> - : std::true_type -{}; +struct is_clock> : std::true_type {}; #endif // HAS_VOID_T @@ -1013,915 +931,716 @@ struct is_clock> -class save_istream -{ -protected: - std::basic_ios& is_; - CharT fill_; - std::ios::fmtflags flags_; - std::streamsize width_; - std::basic_ostream* tie_; - std::locale loc_; - -public: - ~save_istream() - { - is_.fill(fill_); - is_.flags(flags_); - is_.width(width_); - is_.imbue(loc_); - is_.tie(tie_); - } - - save_istream(const save_istream&) = delete; - save_istream& operator=(const save_istream&) = delete; - - explicit save_istream(std::basic_ios& is) - : is_(is) - , fill_(is.fill()) - , flags_(is.flags()) - , width_(is.width(0)) - , tie_(is.tie(nullptr)) - , loc_(is.getloc()) - { - if (tie_ != nullptr) - tie_->flush(); - } +template > +class save_istream { + protected: + std::basic_ios& is_; + CharT fill_; + std::ios::fmtflags flags_; + std::streamsize width_; + std::basic_ostream* tie_; + std::locale loc_; + + public: + ~save_istream() { + is_.fill(fill_); + is_.flags(flags_); + is_.width(width_); + is_.imbue(loc_); + is_.tie(tie_); + } + + save_istream(const save_istream&) = delete; + save_istream& operator=(const save_istream&) = delete; + + explicit save_istream(std::basic_ios& is) + : is_(is), fill_(is.fill()), flags_(is.flags()), width_(is.width(0)), tie_(is.tie(nullptr)), loc_(is.getloc()) { + if (tie_ != nullptr) + tie_->flush(); + } }; -template> -class save_ostream - : private save_istream -{ -public: - ~save_ostream() - { - if ((this->flags_ & std::ios::unitbuf) && +template > +class save_ostream : private save_istream { + public: + ~save_ostream() { + if ((this->flags_ & std::ios::unitbuf) && #if HAS_UNCAUGHT_EXCEPTIONS - std::uncaught_exceptions() == 0 && + std::uncaught_exceptions() == 0 && #else - !std::uncaught_exception() && + !std::uncaught_exception() && #endif - this->is_.good()) - this->is_.rdbuf()->pubsync(); - } + this->is_.good()) + this->is_.rdbuf()->pubsync(); + } - save_ostream(const save_ostream&) = delete; - save_ostream& operator=(const save_ostream&) = delete; + save_ostream(const save_ostream&) = delete; + save_ostream& operator=(const save_ostream&) = delete; - explicit save_ostream(std::basic_ios& os) - : save_istream(os) - { - } + explicit save_ostream(std::basic_ios& os) : save_istream(os) { + } }; template -struct choose_trunc_type -{ - static const int digits = std::numeric_limits::digits; - using type = typename std::conditional - < - digits < 32, - std::int32_t, - typename std::conditional - < - digits < 64, - std::int64_t, +struct choose_trunc_type { + static const int digits = std::numeric_limits::digits; + using type = typename std::conditional < digits < 32, std::int32_t, + typename std::conditional::type - >::type; + >::type>::type; }; template -CONSTCD11 -inline -typename std::enable_if -< - !std::chrono::treat_as_floating_point::value, - T ->::type -trunc(T t) NOEXCEPT -{ - return t; +CONSTCD11 inline typename std::enable_if::value, T>::type trunc(T t) NOEXCEPT { + return t; } template -CONSTCD14 -inline -typename std::enable_if -< - std::chrono::treat_as_floating_point::value, - T ->::type -trunc(T t) NOEXCEPT -{ - using std::numeric_limits; - using I = typename choose_trunc_type::type; - CONSTDATA auto digits = numeric_limits::digits; - static_assert(digits < numeric_limits::digits, ""); - CONSTDATA auto max = I{1} << (digits-1); - CONSTDATA auto min = -max; - const auto negative = t < T{0}; - if (min <= t && t <= max && t != 0 && t == t) - { - t = static_cast(static_cast(t)); - if (t == 0 && negative) - t = -t; - } - return t; +CONSTCD14 inline typename std::enable_if::value, T>::type trunc(T t) NOEXCEPT { + using std::numeric_limits; + using I = typename choose_trunc_type::type; + CONSTDATA auto digits = numeric_limits::digits; + static_assert(digits < numeric_limits::digits, ""); + CONSTDATA auto max = I{1} << (digits - 1); + CONSTDATA auto min = -max; + const auto negative = t < T{0}; + if (min <= t && t <= max && t != 0 && t == t) { + t = static_cast(static_cast(t)); + if (t == 0 && negative) + t = -t; + } + return t; } template -struct static_gcd -{ - static const std::intmax_t value = static_gcd::value; +struct static_gcd { + static const std::intmax_t value = static_gcd::value; }; template -struct static_gcd -{ - static const std::intmax_t value = Xp; +struct static_gcd { + static const std::intmax_t value = Xp; }; template <> -struct static_gcd<0, 0> -{ - static const std::intmax_t value = 1; +struct static_gcd<0, 0> { + static const std::intmax_t value = 1; }; template -struct no_overflow -{ -private: - static const std::intmax_t gcd_n1_n2 = static_gcd::value; - static const std::intmax_t gcd_d1_d2 = static_gcd::value; - static const std::intmax_t n1 = R1::num / gcd_n1_n2; - static const std::intmax_t d1 = R1::den / gcd_d1_d2; - static const std::intmax_t n2 = R2::num / gcd_n1_n2; - static const std::intmax_t d2 = R2::den / gcd_d1_d2; +struct no_overflow { + private: + static const std::intmax_t gcd_n1_n2 = static_gcd::value; + static const std::intmax_t gcd_d1_d2 = static_gcd::value; + static const std::intmax_t n1 = R1::num / gcd_n1_n2; + static const std::intmax_t d1 = R1::den / gcd_d1_d2; + static const std::intmax_t n2 = R2::num / gcd_n1_n2; + static const std::intmax_t d2 = R2::den / gcd_d1_d2; #ifdef __cpp_constexpr - static const std::intmax_t max = std::numeric_limits::max(); + static const std::intmax_t max = std::numeric_limits::max(); #else - static const std::intmax_t max = LLONG_MAX; + static const std::intmax_t max = LLONG_MAX; #endif - template - struct mul // overflow == false - { - static const std::intmax_t value = Xp * Yp; - }; - - template - struct mul - { - static const std::intmax_t value = 1; - }; - -public: - static const bool value = (n1 <= max / d2) && (n2 <= max / d1); - typedef std::ratio::value, - mul::value> type; + template + struct mul // overflow == false + { + static const std::intmax_t value = Xp * Yp; + }; + + template + struct mul { + static const std::intmax_t value = 1; + }; + + public: + static const bool value = (n1 <= max / d2) && (n2 <= max / d1); + typedef std::ratio::value, mul::value> type; }; -} // detail +} // namespace detail // trunc towards zero template -CONSTCD11 -inline -typename std::enable_if -< - detail::no_overflow::value, - To ->::type -trunc(const std::chrono::duration& d) -{ - return To{detail::trunc(std::chrono::duration_cast(d).count())}; +CONSTCD11 inline typename std::enable_if::value, To>::type trunc( + const std::chrono::duration& d) { + return To{detail::trunc(std::chrono::duration_cast(d).count())}; } template -CONSTCD11 -inline -typename std::enable_if -< - !detail::no_overflow::value, - To ->::type -trunc(const std::chrono::duration& d) -{ - using std::chrono::duration_cast; - using std::chrono::duration; - using rep = typename std::common_type::type; - return To{detail::trunc(duration_cast(duration_cast>(d)).count())}; +CONSTCD11 inline typename std::enable_if::value, To>::type trunc( + const std::chrono::duration& d) { + using std::chrono::duration; + using std::chrono::duration_cast; + using rep = typename std::common_type::type; + return To{detail::trunc(duration_cast(duration_cast>(d)).count())}; } #ifndef HAS_CHRONO_ROUNDING -# if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 190023918 || (_MSC_FULL_VER >= 190000000 && defined (__clang__))) -# define HAS_CHRONO_ROUNDING 1 -# elif defined(__cpp_lib_chrono) && __cplusplus > 201402 && __cpp_lib_chrono >= 201510 -# define HAS_CHRONO_ROUNDING 1 -# elif defined(_LIBCPP_VERSION) && __cplusplus > 201402 && _LIBCPP_VERSION >= 3800 -# define HAS_CHRONO_ROUNDING 1 -# else -# define HAS_CHRONO_ROUNDING 0 -# endif +#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 190023918 || (_MSC_FULL_VER >= 190000000 && defined(__clang__))) +#define HAS_CHRONO_ROUNDING 1 +#elif defined(__cpp_lib_chrono) && __cplusplus > 201402 && __cpp_lib_chrono >= 201510 +#define HAS_CHRONO_ROUNDING 1 +#elif defined(_LIBCPP_VERSION) && __cplusplus > 201402 && _LIBCPP_VERSION >= 3800 +#define HAS_CHRONO_ROUNDING 1 +#else +#define HAS_CHRONO_ROUNDING 0 +#endif #endif // HAS_CHRONO_ROUNDING #if HAS_CHRONO_ROUNDING == 0 // round down template -CONSTCD14 -inline -typename std::enable_if -< - detail::no_overflow::value, - To ->::type -floor(const std::chrono::duration& d) -{ - auto t = trunc(d); - if (t > d) - return t - To{1}; - return t; +CONSTCD14 inline typename std::enable_if::value, To>::type floor( + const std::chrono::duration& d) { + auto t = trunc(d); + if (t > d) + return t - To{1}; + return t; } template -CONSTCD14 -inline -typename std::enable_if -< - !detail::no_overflow::value, - To ->::type -floor(const std::chrono::duration& d) -{ - using rep = typename std::common_type::type; - return floor(floor>(d)); +CONSTCD14 inline typename std::enable_if::value, To>::type floor( + const std::chrono::duration& d) { + using rep = typename std::common_type::type; + return floor(floor>(d)); } // round to nearest, to even on tie template -CONSTCD14 -inline -To -round(const std::chrono::duration& d) -{ - auto t0 = floor(d); - auto t1 = t0 + To{1}; - if (t1 == To{0} && t0 < To{0}) - t1 = -t1; - auto diff0 = d - t0; - auto diff1 = t1 - d; - if (diff0 == diff1) - { - if (t0 - trunc(t0/2)*2 == To{0}) - return t0; - return t1; - } - if (diff0 < diff1) - return t0; +CONSTCD14 inline To round(const std::chrono::duration& d) { + auto t0 = floor(d); + auto t1 = t0 + To{1}; + if (t1 == To{0} && t0 < To{0}) + t1 = -t1; + auto diff0 = d - t0; + auto diff1 = t1 - d; + if (diff0 == diff1) { + if (t0 - trunc(t0 / 2) * 2 == To{0}) + return t0; return t1; + } + if (diff0 < diff1) + return t0; + return t1; } // round up template -CONSTCD14 -inline -To -ceil(const std::chrono::duration& d) -{ - auto t = trunc(d); - if (t < d) - return t + To{1}; - return t; +CONSTCD14 inline To ceil(const std::chrono::duration& d) { + auto t = trunc(d); + if (t < d) + return t + To{1}; + return t; } -template ::is_signed - >::type> -CONSTCD11 -std::chrono::duration -abs(std::chrono::duration d) -{ - return d >= d.zero() ? d : -d; +template ::is_signed>::type> +CONSTCD11 std::chrono::duration abs(std::chrono::duration d) { + return d >= d.zero() ? d : -d; } // round down template -CONSTCD11 -inline -std::chrono::time_point -floor(const std::chrono::time_point& tp) -{ - using std::chrono::time_point; - return time_point{date::floor(tp.time_since_epoch())}; +CONSTCD11 inline std::chrono::time_point floor(const std::chrono::time_point& tp) { + using std::chrono::time_point; + return time_point{date::floor(tp.time_since_epoch())}; } // round to nearest, to even on tie template -CONSTCD11 -inline -std::chrono::time_point -round(const std::chrono::time_point& tp) -{ - using std::chrono::time_point; - return time_point{round(tp.time_since_epoch())}; +CONSTCD11 inline std::chrono::time_point round(const std::chrono::time_point& tp) { + using std::chrono::time_point; + return time_point{round(tp.time_since_epoch())}; } // round up template -CONSTCD11 -inline -std::chrono::time_point -ceil(const std::chrono::time_point& tp) -{ - using std::chrono::time_point; - return time_point{ceil(tp.time_since_epoch())}; +CONSTCD11 inline std::chrono::time_point ceil(const std::chrono::time_point& tp) { + using std::chrono::time_point; + return time_point{ceil(tp.time_since_epoch())}; } #else // HAS_CHRONO_ROUNDING == 1 -using std::chrono::floor; +using std::chrono::abs; using std::chrono::ceil; +using std::chrono::floor; using std::chrono::round; -using std::chrono::abs; #endif // HAS_CHRONO_ROUNDING -namespace detail -{ +namespace detail { template -CONSTCD14 -inline -typename std::enable_if -< - !std::chrono::treat_as_floating_point::value, - To ->::type -round_i(const std::chrono::duration& d) -{ - return round(d); +CONSTCD14 inline typename std::enable_if::value, To>::type +round_i(const std::chrono::duration& d) { + return round(d); } template -CONSTCD14 -inline -typename std::enable_if -< - std::chrono::treat_as_floating_point::value, - To ->::type -round_i(const std::chrono::duration& d) -{ - return d; +CONSTCD14 inline typename std::enable_if::value, To>::type +round_i(const std::chrono::duration& d) { + return d; } template -CONSTCD11 -inline -std::chrono::time_point -round_i(const std::chrono::time_point& tp) -{ - using std::chrono::time_point; - return time_point{round_i(tp.time_since_epoch())}; +CONSTCD11 inline std::chrono::time_point round_i(const std::chrono::time_point& tp) { + using std::chrono::time_point; + return time_point{round_i(tp.time_since_epoch())}; } -} // detail +} // namespace detail // trunc towards zero template -CONSTCD11 -inline -std::chrono::time_point -trunc(const std::chrono::time_point& tp) -{ - using std::chrono::time_point; - return time_point{trunc(tp.time_since_epoch())}; +CONSTCD11 inline std::chrono::time_point trunc(const std::chrono::time_point& tp) { + using std::chrono::time_point; + return time_point{trunc(tp.time_since_epoch())}; } // day -CONSTCD11 inline day::day(unsigned d) NOEXCEPT : d_(static_cast(d)) {} -CONSTCD14 inline day& day::operator++() NOEXCEPT {++d_; return *this;} -CONSTCD14 inline day day::operator++(int) NOEXCEPT {auto tmp(*this); ++(*this); return tmp;} -CONSTCD14 inline day& day::operator--() NOEXCEPT {--d_; return *this;} -CONSTCD14 inline day day::operator--(int) NOEXCEPT {auto tmp(*this); --(*this); return tmp;} -CONSTCD14 inline day& day::operator+=(const days& d) NOEXCEPT {*this = *this + d; return *this;} -CONSTCD14 inline day& day::operator-=(const days& d) NOEXCEPT {*this = *this - d; return *this;} -CONSTCD11 inline day::operator unsigned() const NOEXCEPT {return d_;} -CONSTCD11 inline bool day::ok() const NOEXCEPT {return 1 <= d_ && d_ <= 31;} +CONSTCD11 inline day::day(unsigned d) NOEXCEPT : d_(static_cast(d)) { +} +CONSTCD14 inline day& day::operator++() NOEXCEPT { + ++d_; + return *this; +} +CONSTCD14 inline day day::operator++(int) NOEXCEPT { + auto tmp(*this); + ++(*this); + return tmp; +} +CONSTCD14 inline day& day::operator--() NOEXCEPT { + --d_; + return *this; +} +CONSTCD14 inline day day::operator--(int) NOEXCEPT { + auto tmp(*this); + --(*this); + return tmp; +} +CONSTCD14 inline day& day::operator+=(const days& d) NOEXCEPT { + *this = *this + d; + return *this; +} +CONSTCD14 inline day& day::operator-=(const days& d) NOEXCEPT { + *this = *this - d; + return *this; +} +CONSTCD11 inline day::operator unsigned() const NOEXCEPT { + return d_; +} +CONSTCD11 inline bool day::ok() const NOEXCEPT { + return 1 <= d_ && d_ <= 31; +} CONSTCD11 -inline -bool -operator==(const day& x, const day& y) NOEXCEPT -{ - return static_cast(x) == static_cast(y); +inline bool operator==(const day& x, const day& y) NOEXCEPT { + return static_cast(x) == static_cast(y); } CONSTCD11 -inline -bool -operator!=(const day& x, const day& y) NOEXCEPT -{ - return !(x == y); +inline bool operator!=(const day& x, const day& y) NOEXCEPT { + return !(x == y); } CONSTCD11 -inline -bool -operator<(const day& x, const day& y) NOEXCEPT -{ - return static_cast(x) < static_cast(y); +inline bool operator<(const day& x, const day& y) NOEXCEPT { + return static_cast(x) < static_cast(y); } CONSTCD11 -inline -bool -operator>(const day& x, const day& y) NOEXCEPT -{ - return y < x; +inline bool operator>(const day& x, const day& y) NOEXCEPT { + return y < x; } CONSTCD11 -inline -bool -operator<=(const day& x, const day& y) NOEXCEPT -{ - return !(y < x); +inline bool operator<=(const day& x, const day& y) NOEXCEPT { + return !(y < x); } CONSTCD11 -inline -bool -operator>=(const day& x, const day& y) NOEXCEPT -{ - return !(x < y); +inline bool operator>=(const day& x, const day& y) NOEXCEPT { + return !(x < y); } CONSTCD11 -inline -days -operator-(const day& x, const day& y) NOEXCEPT -{ - return days{static_cast(static_cast(x) - - static_cast(y))}; +inline days operator-(const day& x, const day& y) NOEXCEPT { + return days{static_cast(static_cast(x) - static_cast(y))}; } CONSTCD11 -inline -day -operator+(const day& x, const days& y) NOEXCEPT -{ - return day{static_cast(x) + static_cast(y.count())}; +inline day operator+(const day& x, const days& y) NOEXCEPT { + return day{static_cast(x) + static_cast(y.count())}; } CONSTCD11 -inline -day -operator+(const days& x, const day& y) NOEXCEPT -{ - return y + x; +inline day operator+(const days& x, const day& y) NOEXCEPT { + return y + x; } CONSTCD11 -inline -day -operator-(const day& x, const days& y) NOEXCEPT -{ - return x + -y; +inline day operator-(const day& x, const days& y) NOEXCEPT { + return x + -y; } -template -inline -std::basic_ostream& -operator<<(std::basic_ostream& os, const day& d) -{ - detail::save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - os.width(2); - os << static_cast(d); - if (!d.ok()) - os << " is not a valid day"; - return os; +template +inline std::basic_ostream& operator<<(std::basic_ostream& os, const day& d) { + detail::save_ostream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(2); + os << static_cast(d); + if (!d.ok()) + os << " is not a valid day"; + return os; } // month -CONSTCD11 inline month::month(unsigned m) NOEXCEPT : m_(static_cast(m)) {} -CONSTCD14 inline month& month::operator++() NOEXCEPT {*this += months{1}; return *this;} -CONSTCD14 inline month month::operator++(int) NOEXCEPT {auto tmp(*this); ++(*this); return tmp;} -CONSTCD14 inline month& month::operator--() NOEXCEPT {*this -= months{1}; return *this;} -CONSTCD14 inline month month::operator--(int) NOEXCEPT {auto tmp(*this); --(*this); return tmp;} +CONSTCD11 inline month::month(unsigned m) NOEXCEPT : m_(static_cast(m)) { +} +CONSTCD14 inline month& month::operator++() NOEXCEPT { + *this += months{1}; + return *this; +} +CONSTCD14 inline month month::operator++(int) NOEXCEPT { + auto tmp(*this); + ++(*this); + return tmp; +} +CONSTCD14 inline month& month::operator--() NOEXCEPT { + *this -= months{1}; + return *this; +} +CONSTCD14 inline month month::operator--(int) NOEXCEPT { + auto tmp(*this); + --(*this); + return tmp; +} CONSTCD14 -inline -month& -month::operator+=(const months& m) NOEXCEPT -{ - *this = *this + m; - return *this; +inline month& month::operator+=(const months& m) NOEXCEPT { + *this = *this + m; + return *this; } CONSTCD14 -inline -month& -month::operator-=(const months& m) NOEXCEPT -{ - *this = *this - m; - return *this; +inline month& month::operator-=(const months& m) NOEXCEPT { + *this = *this - m; + return *this; } -CONSTCD11 inline month::operator unsigned() const NOEXCEPT {return m_;} -CONSTCD11 inline bool month::ok() const NOEXCEPT {return 1 <= m_ && m_ <= 12;} +CONSTCD11 inline month::operator unsigned() const NOEXCEPT { + return m_; +} +CONSTCD11 inline bool month::ok() const NOEXCEPT { + return 1 <= m_ && m_ <= 12; +} CONSTCD11 -inline -bool -operator==(const month& x, const month& y) NOEXCEPT -{ - return static_cast(x) == static_cast(y); +inline bool operator==(const month& x, const month& y) NOEXCEPT { + return static_cast(x) == static_cast(y); } CONSTCD11 -inline -bool -operator!=(const month& x, const month& y) NOEXCEPT -{ - return !(x == y); +inline bool operator!=(const month& x, const month& y) NOEXCEPT { + return !(x == y); } CONSTCD11 -inline -bool -operator<(const month& x, const month& y) NOEXCEPT -{ - return static_cast(x) < static_cast(y); +inline bool operator<(const month& x, const month& y) NOEXCEPT { + return static_cast(x) < static_cast(y); } CONSTCD11 -inline -bool -operator>(const month& x, const month& y) NOEXCEPT -{ - return y < x; +inline bool operator>(const month& x, const month& y) NOEXCEPT { + return y < x; } CONSTCD11 -inline -bool -operator<=(const month& x, const month& y) NOEXCEPT -{ - return !(y < x); +inline bool operator<=(const month& x, const month& y) NOEXCEPT { + return !(y < x); } CONSTCD11 -inline -bool -operator>=(const month& x, const month& y) NOEXCEPT -{ - return !(x < y); +inline bool operator>=(const month& x, const month& y) NOEXCEPT { + return !(x < y); } CONSTCD14 -inline -months -operator-(const month& x, const month& y) NOEXCEPT -{ - auto const d = static_cast(x) - static_cast(y); - return months(d <= 11 ? d : d + 12); +inline months operator-(const month& x, const month& y) NOEXCEPT { + auto const d = static_cast(x) - static_cast(y); + return months(d <= 11 ? d : d + 12); } CONSTCD14 -inline -month -operator+(const month& x, const months& y) NOEXCEPT -{ - auto const mu = static_cast(static_cast(x)) + y.count() - 1; - auto const yr = (mu >= 0 ? mu : mu-11) / 12; - return month{static_cast(mu - yr * 12 + 1)}; +inline month operator+(const month& x, const months& y) NOEXCEPT { + auto const mu = static_cast(static_cast(x)) + y.count() - 1; + auto const yr = (mu >= 0 ? mu : mu - 11) / 12; + return month{static_cast(mu - yr * 12 + 1)}; } CONSTCD14 -inline -month -operator+(const months& x, const month& y) NOEXCEPT -{ - return y + x; +inline month operator+(const months& x, const month& y) NOEXCEPT { + return y + x; } CONSTCD14 -inline -month -operator-(const month& x, const months& y) NOEXCEPT -{ - return x + -y; -} - -template -inline -std::basic_ostream& -operator<<(std::basic_ostream& os, const month& m) -{ - if (m.ok()) - { - CharT fmt[] = {'%', 'b', 0}; - os << format(os.getloc(), fmt, m); - } - else - os << static_cast(m) << " is not a valid month"; - return os; +inline month operator-(const month& x, const months& y) NOEXCEPT { + return x + -y; +} + +template +inline std::basic_ostream& operator<<(std::basic_ostream& os, const month& m) { + if (m.ok()) { + CharT fmt[] = {'%', 'b', 0}; + os << format(os.getloc(), fmt, m); + } else + os << static_cast(m) << " is not a valid month"; + return os; } // year -CONSTCD11 inline year::year(int y) NOEXCEPT : y_(static_cast(y)) {} -CONSTCD14 inline year& year::operator++() NOEXCEPT {++y_; return *this;} -CONSTCD14 inline year year::operator++(int) NOEXCEPT {auto tmp(*this); ++(*this); return tmp;} -CONSTCD14 inline year& year::operator--() NOEXCEPT {--y_; return *this;} -CONSTCD14 inline year year::operator--(int) NOEXCEPT {auto tmp(*this); --(*this); return tmp;} -CONSTCD14 inline year& year::operator+=(const years& y) NOEXCEPT {*this = *this + y; return *this;} -CONSTCD14 inline year& year::operator-=(const years& y) NOEXCEPT {*this = *this - y; return *this;} -CONSTCD11 inline year year::operator-() const NOEXCEPT {return year{-y_};} -CONSTCD11 inline year year::operator+() const NOEXCEPT {return *this;} +CONSTCD11 inline year::year(int y) NOEXCEPT : y_(static_cast(y)) { +} +CONSTCD14 inline year& year::operator++() NOEXCEPT { + ++y_; + return *this; +} +CONSTCD14 inline year year::operator++(int) NOEXCEPT { + auto tmp(*this); + ++(*this); + return tmp; +} +CONSTCD14 inline year& year::operator--() NOEXCEPT { + --y_; + return *this; +} +CONSTCD14 inline year year::operator--(int) NOEXCEPT { + auto tmp(*this); + --(*this); + return tmp; +} +CONSTCD14 inline year& year::operator+=(const years& y) NOEXCEPT { + *this = *this + y; + return *this; +} +CONSTCD14 inline year& year::operator-=(const years& y) NOEXCEPT { + *this = *this - y; + return *this; +} +CONSTCD11 inline year year::operator-() const NOEXCEPT { + return year{-y_}; +} +CONSTCD11 inline year year::operator+() const NOEXCEPT { + return *this; +} CONSTCD11 -inline -bool -year::is_leap() const NOEXCEPT -{ - return y_ % 4 == 0 && (y_ % 100 != 0 || y_ % 400 == 0); +inline bool year::is_leap() const NOEXCEPT { + return y_ % 4 == 0 && (y_ % 100 != 0 || y_ % 400 == 0); } -CONSTCD11 inline year::operator int() const NOEXCEPT {return y_;} +CONSTCD11 inline year::operator int() const NOEXCEPT { + return y_; +} CONSTCD11 -inline -bool -year::ok() const NOEXCEPT -{ - return y_ != std::numeric_limits::min(); +inline bool year::ok() const NOEXCEPT { + return y_ != std::numeric_limits::min(); } CONSTCD11 -inline -bool -operator==(const year& x, const year& y) NOEXCEPT -{ - return static_cast(x) == static_cast(y); +inline bool operator==(const year& x, const year& y) NOEXCEPT { + return static_cast(x) == static_cast(y); } CONSTCD11 -inline -bool -operator!=(const year& x, const year& y) NOEXCEPT -{ - return !(x == y); +inline bool operator!=(const year& x, const year& y) NOEXCEPT { + return !(x == y); } CONSTCD11 -inline -bool -operator<(const year& x, const year& y) NOEXCEPT -{ - return static_cast(x) < static_cast(y); +inline bool operator<(const year& x, const year& y) NOEXCEPT { + return static_cast(x) < static_cast(y); } CONSTCD11 -inline -bool -operator>(const year& x, const year& y) NOEXCEPT -{ - return y < x; +inline bool operator>(const year& x, const year& y) NOEXCEPT { + return y < x; } CONSTCD11 -inline -bool -operator<=(const year& x, const year& y) NOEXCEPT -{ - return !(y < x); +inline bool operator<=(const year& x, const year& y) NOEXCEPT { + return !(y < x); } CONSTCD11 -inline -bool -operator>=(const year& x, const year& y) NOEXCEPT -{ - return !(x < y); +inline bool operator>=(const year& x, const year& y) NOEXCEPT { + return !(x < y); } CONSTCD11 -inline -years -operator-(const year& x, const year& y) NOEXCEPT -{ - return years{static_cast(x) - static_cast(y)}; +inline years operator-(const year& x, const year& y) NOEXCEPT { + return years{static_cast(x) - static_cast(y)}; } CONSTCD11 -inline -year -operator+(const year& x, const years& y) NOEXCEPT -{ - return year{static_cast(x) + y.count()}; +inline year operator+(const year& x, const years& y) NOEXCEPT { + return year{static_cast(x) + y.count()}; } CONSTCD11 -inline -year -operator+(const years& x, const year& y) NOEXCEPT -{ - return y + x; +inline year operator+(const years& x, const year& y) NOEXCEPT { + return y + x; } CONSTCD11 -inline -year -operator-(const year& x, const years& y) NOEXCEPT -{ - return year{static_cast(x) - y.count()}; +inline year operator-(const year& x, const years& y) NOEXCEPT { + return year{static_cast(x) - y.count()}; } -template -inline -std::basic_ostream& -operator<<(std::basic_ostream& os, const year& y) -{ - detail::save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::internal); - os.width(4 + (y < year{0})); - os.imbue(std::locale::classic()); - os << static_cast(y); - if (!y.ok()) - os << " is not a valid year"; - return os; +template +inline std::basic_ostream& operator<<(std::basic_ostream& os, const year& y) { + detail::save_ostream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::internal); + os.width(4 + (y < year{0})); + os.imbue(std::locale::classic()); + os << static_cast(y); + if (!y.ok()) + os << " is not a valid year"; + return os; } // weekday CONSTCD14 -inline -unsigned char -weekday::weekday_from_days(int z) NOEXCEPT -{ - auto u = static_cast(z); - return static_cast(z >= -4 ? (u+4) % 7 : u % 7); +inline unsigned char weekday::weekday_from_days(int z) NOEXCEPT { + auto u = static_cast(z); + return static_cast(z >= -4 ? (u + 4) % 7 : u % 7); } CONSTCD11 -inline -weekday::weekday(unsigned wd) NOEXCEPT - : wd_(static_cast(wd != 7 ? wd : 0)) - {} +inline weekday::weekday(unsigned wd) NOEXCEPT : wd_(static_cast(wd != 7 ? wd : 0)) { +} CONSTCD14 -inline -weekday::weekday(const sys_days& dp) NOEXCEPT - : wd_(weekday_from_days(dp.time_since_epoch().count())) - {} +inline weekday::weekday(const sys_days& dp) NOEXCEPT : wd_(weekday_from_days(dp.time_since_epoch().count())) { +} CONSTCD14 -inline -weekday::weekday(const local_days& dp) NOEXCEPT - : wd_(weekday_from_days(dp.time_since_epoch().count())) - {} +inline weekday::weekday(const local_days& dp) NOEXCEPT : wd_(weekday_from_days(dp.time_since_epoch().count())) { +} -CONSTCD14 inline weekday& weekday::operator++() NOEXCEPT {*this += days{1}; return *this;} -CONSTCD14 inline weekday weekday::operator++(int) NOEXCEPT {auto tmp(*this); ++(*this); return tmp;} -CONSTCD14 inline weekday& weekday::operator--() NOEXCEPT {*this -= days{1}; return *this;} -CONSTCD14 inline weekday weekday::operator--(int) NOEXCEPT {auto tmp(*this); --(*this); return tmp;} +CONSTCD14 inline weekday& weekday::operator++() NOEXCEPT { + *this += days{1}; + return *this; +} +CONSTCD14 inline weekday weekday::operator++(int) NOEXCEPT { + auto tmp(*this); + ++(*this); + return tmp; +} +CONSTCD14 inline weekday& weekday::operator--() NOEXCEPT { + *this -= days{1}; + return *this; +} +CONSTCD14 inline weekday weekday::operator--(int) NOEXCEPT { + auto tmp(*this); + --(*this); + return tmp; +} CONSTCD14 -inline -weekday& -weekday::operator+=(const days& d) NOEXCEPT -{ - *this = *this + d; - return *this; +inline weekday& weekday::operator+=(const days& d) NOEXCEPT { + *this = *this + d; + return *this; } CONSTCD14 -inline -weekday& -weekday::operator-=(const days& d) NOEXCEPT -{ - *this = *this - d; - return *this; +inline weekday& weekday::operator-=(const days& d) NOEXCEPT { + *this = *this - d; + return *this; } -CONSTCD11 inline bool weekday::ok() const NOEXCEPT {return wd_ <= 6;} +CONSTCD11 inline bool weekday::ok() const NOEXCEPT { + return wd_ <= 6; +} CONSTCD11 -inline -unsigned weekday::c_encoding() const NOEXCEPT -{ - return unsigned{wd_}; +inline unsigned weekday::c_encoding() const NOEXCEPT { + return unsigned{wd_}; } CONSTCD11 -inline -unsigned weekday::iso_encoding() const NOEXCEPT -{ - return unsigned{((wd_ == 0u) ? 7u : wd_)}; +inline unsigned weekday::iso_encoding() const NOEXCEPT { + return unsigned{((wd_ == 0u) ? 7u : wd_)}; } CONSTCD11 -inline -bool -operator==(const weekday& x, const weekday& y) NOEXCEPT -{ - return x.wd_ == y.wd_; +inline bool operator==(const weekday& x, const weekday& y) NOEXCEPT { + return x.wd_ == y.wd_; } CONSTCD11 -inline -bool -operator!=(const weekday& x, const weekday& y) NOEXCEPT -{ - return !(x == y); +inline bool operator!=(const weekday& x, const weekday& y) NOEXCEPT { + return !(x == y); } CONSTCD14 -inline -days -operator-(const weekday& x, const weekday& y) NOEXCEPT -{ - auto const wdu = x.wd_ - y.wd_; - auto const wk = (wdu >= 0 ? wdu : wdu-6) / 7; - return days{wdu - wk * 7}; +inline days operator-(const weekday& x, const weekday& y) NOEXCEPT { + auto const wdu = x.wd_ - y.wd_; + auto const wk = (wdu >= 0 ? wdu : wdu - 6) / 7; + return days{wdu - wk * 7}; } CONSTCD14 -inline -weekday -operator+(const weekday& x, const days& y) NOEXCEPT -{ - auto const wdu = static_cast(static_cast(x.wd_)) + y.count(); - auto const wk = (wdu >= 0 ? wdu : wdu-6) / 7; - return weekday{static_cast(wdu - wk * 7)}; +inline weekday operator+(const weekday& x, const days& y) NOEXCEPT { + auto const wdu = static_cast(static_cast(x.wd_)) + y.count(); + auto const wk = (wdu >= 0 ? wdu : wdu - 6) / 7; + return weekday{static_cast(wdu - wk * 7)}; } CONSTCD14 -inline -weekday -operator+(const days& x, const weekday& y) NOEXCEPT -{ - return y + x; +inline weekday operator+(const days& x, const weekday& y) NOEXCEPT { + return y + x; } CONSTCD14 -inline -weekday -operator-(const weekday& x, const days& y) NOEXCEPT -{ - return x + -y; -} - -template -inline -std::basic_ostream& -operator<<(std::basic_ostream& os, const weekday& wd) -{ - if (wd.ok()) - { - CharT fmt[] = {'%', 'a', 0}; - os << format(fmt, wd); - } - else - os << static_cast(wd.wd_) << " is not a valid weekday"; - return os; +inline weekday operator-(const weekday& x, const days& y) NOEXCEPT { + return x + -y; +} + +template +inline std::basic_ostream& operator<<(std::basic_ostream& os, const weekday& wd) { + if (wd.ok()) { + CharT fmt[] = {'%', 'a', 0}; + os << format(fmt, wd); + } else + os << static_cast(wd.wd_) << " is not a valid weekday"; + return os; } #if !defined(_MSC_VER) || (_MSC_VER >= 1900) -inline namespace literals -{ +inline namespace literals { CONSTCD11 -inline -date::day -operator "" _d(unsigned long long d) NOEXCEPT -{ - return date::day{static_cast(d)}; +inline date::day operator"" _d(unsigned long long d) NOEXCEPT { + return date::day{static_cast(d)}; } CONSTCD11 -inline -date::year -operator "" _y(unsigned long long y) NOEXCEPT -{ - return date::year(static_cast(y)); +inline date::year operator"" _y(unsigned long long y) NOEXCEPT { + return date::year(static_cast(y)); } #endif // !defined(_MSC_VER) || (_MSC_VER >= 1900) @@ -1976,1744 +1695,1196 @@ CONSTDATA date::weekday Sunday{7}; // weekday_indexed CONSTCD11 -inline -weekday -weekday_indexed::weekday() const NOEXCEPT -{ - return date::weekday{static_cast(wd_)}; +inline weekday weekday_indexed::weekday() const NOEXCEPT { + return date::weekday{static_cast(wd_)}; } -CONSTCD11 inline unsigned weekday_indexed::index() const NOEXCEPT {return index_;} +CONSTCD11 inline unsigned weekday_indexed::index() const NOEXCEPT { + return index_; +} CONSTCD11 -inline -bool -weekday_indexed::ok() const NOEXCEPT -{ - return weekday().ok() && 1 <= index_ && index_ <= 5; +inline bool weekday_indexed::ok() const NOEXCEPT { + return weekday().ok() && 1 <= index_ && index_ <= 5; } #ifdef __GNUC__ -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wconversion" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" #endif // __GNUC__ CONSTCD11 -inline -weekday_indexed::weekday_indexed(const date::weekday& wd, unsigned index) NOEXCEPT - : wd_(static_cast(static_cast(wd.wd_))) - , index_(static_cast(index)) - {} +inline weekday_indexed::weekday_indexed(const date::weekday& wd, unsigned index) NOEXCEPT + : wd_(static_cast(static_cast(wd.wd_))), + index_(static_cast(index)) { +} #ifdef __GNUC__ -# pragma GCC diagnostic pop +#pragma GCC diagnostic pop #endif // __GNUC__ -template -inline -std::basic_ostream& -operator<<(std::basic_ostream& os, const weekday_indexed& wdi) -{ - os << wdi.weekday() << '[' << wdi.index(); - if (!(1 <= wdi.index() && wdi.index() <= 5)) - os << " is not a valid index"; - os << ']'; - return os; +template +inline std::basic_ostream& operator<<(std::basic_ostream& os, + const weekday_indexed& wdi) { + os << wdi.weekday() << '[' << wdi.index(); + if (!(1 <= wdi.index() && wdi.index() <= 5)) + os << " is not a valid index"; + os << ']'; + return os; } CONSTCD11 -inline -weekday_indexed -weekday::operator[](unsigned index) const NOEXCEPT -{ - return {*this, index}; +inline weekday_indexed weekday::operator[](unsigned index) const NOEXCEPT { + return {*this, index}; } CONSTCD11 -inline -bool -operator==(const weekday_indexed& x, const weekday_indexed& y) NOEXCEPT -{ - return x.weekday() == y.weekday() && x.index() == y.index(); +inline bool operator==(const weekday_indexed& x, const weekday_indexed& y) NOEXCEPT { + return x.weekday() == y.weekday() && x.index() == y.index(); } CONSTCD11 -inline -bool -operator!=(const weekday_indexed& x, const weekday_indexed& y) NOEXCEPT -{ - return !(x == y); +inline bool operator!=(const weekday_indexed& x, const weekday_indexed& y) NOEXCEPT { + return !(x == y); } // weekday_last -CONSTCD11 inline date::weekday weekday_last::weekday() const NOEXCEPT {return wd_;} -CONSTCD11 inline bool weekday_last::ok() const NOEXCEPT {return wd_.ok();} -CONSTCD11 inline weekday_last::weekday_last(const date::weekday& wd) NOEXCEPT : wd_(wd) {} +CONSTCD11 inline date::weekday weekday_last::weekday() const NOEXCEPT { + return wd_; +} +CONSTCD11 inline bool weekday_last::ok() const NOEXCEPT { + return wd_.ok(); +} +CONSTCD11 inline weekday_last::weekday_last(const date::weekday& wd) NOEXCEPT : wd_(wd) { +} CONSTCD11 -inline -bool -operator==(const weekday_last& x, const weekday_last& y) NOEXCEPT -{ - return x.weekday() == y.weekday(); +inline bool operator==(const weekday_last& x, const weekday_last& y) NOEXCEPT { + return x.weekday() == y.weekday(); } CONSTCD11 -inline -bool -operator!=(const weekday_last& x, const weekday_last& y) NOEXCEPT -{ - return !(x == y); +inline bool operator!=(const weekday_last& x, const weekday_last& y) NOEXCEPT { + return !(x == y); } -template -inline -std::basic_ostream& -operator<<(std::basic_ostream& os, const weekday_last& wdl) -{ - return os << wdl.weekday() << "[last]"; +template +inline std::basic_ostream& operator<<(std::basic_ostream& os, const weekday_last& wdl) { + return os << wdl.weekday() << "[last]"; } CONSTCD11 -inline -weekday_last -weekday::operator[](last_spec) const NOEXCEPT -{ - return weekday_last{*this}; +inline weekday_last weekday::operator[](last_spec) const NOEXCEPT { + return weekday_last{*this}; } // year_month CONSTCD11 -inline -year_month::year_month(const date::year& y, const date::month& m) NOEXCEPT - : y_(y) - , m_(m) - {} +inline year_month::year_month(const date::year& y, const date::month& m) NOEXCEPT : y_(y), m_(m) { +} -CONSTCD11 inline year year_month::year() const NOEXCEPT {return y_;} -CONSTCD11 inline month year_month::month() const NOEXCEPT {return m_;} -CONSTCD11 inline bool year_month::ok() const NOEXCEPT {return y_.ok() && m_.ok();} +CONSTCD11 inline year year_month::year() const NOEXCEPT { + return y_; +} +CONSTCD11 inline month year_month::month() const NOEXCEPT { + return m_; +} +CONSTCD11 inline bool year_month::ok() const NOEXCEPT { + return y_.ok() && m_.ok(); +} -template -CONSTCD14 -inline -year_month& -year_month::operator+=(const months& dm) NOEXCEPT -{ - *this = *this + dm; - return *this; +template +CONSTCD14 inline year_month& year_month::operator+=(const months& dm) NOEXCEPT { + *this = *this + dm; + return *this; } -template -CONSTCD14 -inline -year_month& -year_month::operator-=(const months& dm) NOEXCEPT -{ - *this = *this - dm; - return *this; +template +CONSTCD14 inline year_month& year_month::operator-=(const months& dm) NOEXCEPT { + *this = *this - dm; + return *this; } CONSTCD14 -inline -year_month& -year_month::operator+=(const years& dy) NOEXCEPT -{ - *this = *this + dy; - return *this; +inline year_month& year_month::operator+=(const years& dy) NOEXCEPT { + *this = *this + dy; + return *this; } CONSTCD14 -inline -year_month& -year_month::operator-=(const years& dy) NOEXCEPT -{ - *this = *this - dy; - return *this; +inline year_month& year_month::operator-=(const years& dy) NOEXCEPT { + *this = *this - dy; + return *this; } CONSTCD11 -inline -bool -operator==(const year_month& x, const year_month& y) NOEXCEPT -{ - return x.year() == y.year() && x.month() == y.month(); +inline bool operator==(const year_month& x, const year_month& y) NOEXCEPT { + return x.year() == y.year() && x.month() == y.month(); } CONSTCD11 -inline -bool -operator!=(const year_month& x, const year_month& y) NOEXCEPT -{ - return !(x == y); +inline bool operator!=(const year_month& x, const year_month& y) NOEXCEPT { + return !(x == y); } CONSTCD11 -inline -bool -operator<(const year_month& x, const year_month& y) NOEXCEPT -{ - return x.year() < y.year() ? true - : (x.year() > y.year() ? false - : (x.month() < y.month())); +inline bool operator<(const year_month& x, const year_month& y) NOEXCEPT { + return x.year() < y.year() ? true : (x.year() > y.year() ? false : (x.month() < y.month())); } CONSTCD11 -inline -bool -operator>(const year_month& x, const year_month& y) NOEXCEPT -{ - return y < x; +inline bool operator>(const year_month& x, const year_month& y) NOEXCEPT { + return y < x; } CONSTCD11 -inline -bool -operator<=(const year_month& x, const year_month& y) NOEXCEPT -{ - return !(y < x); +inline bool operator<=(const year_month& x, const year_month& y) NOEXCEPT { + return !(y < x); } CONSTCD11 -inline -bool -operator>=(const year_month& x, const year_month& y) NOEXCEPT -{ - return !(x < y); +inline bool operator>=(const year_month& x, const year_month& y) NOEXCEPT { + return !(x < y); } -template -CONSTCD14 -inline -year_month -operator+(const year_month& ym, const months& dm) NOEXCEPT -{ - auto dmi = static_cast(static_cast(ym.month())) - 1 + dm.count(); - auto dy = (dmi >= 0 ? dmi : dmi-11) / 12; - dmi = dmi - dy * 12 + 1; - return (ym.year() + years(dy)) / month(static_cast(dmi)); +template +CONSTCD14 inline year_month operator+(const year_month& ym, const months& dm) NOEXCEPT { + auto dmi = static_cast(static_cast(ym.month())) - 1 + dm.count(); + auto dy = (dmi >= 0 ? dmi : dmi - 11) / 12; + dmi = dmi - dy * 12 + 1; + return (ym.year() + years(dy)) / month(static_cast(dmi)); } -template -CONSTCD14 -inline -year_month -operator+(const months& dm, const year_month& ym) NOEXCEPT -{ - return ym + dm; +template +CONSTCD14 inline year_month operator+(const months& dm, const year_month& ym) NOEXCEPT { + return ym + dm; } -template -CONSTCD14 -inline -year_month -operator-(const year_month& ym, const months& dm) NOEXCEPT -{ - return ym + -dm; +template +CONSTCD14 inline year_month operator-(const year_month& ym, const months& dm) NOEXCEPT { + return ym + -dm; } CONSTCD11 -inline -months -operator-(const year_month& x, const year_month& y) NOEXCEPT -{ - return (x.year() - y.year()) + - months(static_cast(x.month()) - static_cast(y.month())); +inline months operator-(const year_month& x, const year_month& y) NOEXCEPT { + return (x.year() - y.year()) + months(static_cast(x.month()) - static_cast(y.month())); } CONSTCD11 -inline -year_month -operator+(const year_month& ym, const years& dy) NOEXCEPT -{ - return (ym.year() + dy) / ym.month(); +inline year_month operator+(const year_month& ym, const years& dy) NOEXCEPT { + return (ym.year() + dy) / ym.month(); } CONSTCD11 -inline -year_month -operator+(const years& dy, const year_month& ym) NOEXCEPT -{ - return ym + dy; +inline year_month operator+(const years& dy, const year_month& ym) NOEXCEPT { + return ym + dy; } CONSTCD11 -inline -year_month -operator-(const year_month& ym, const years& dy) NOEXCEPT -{ - return ym + -dy; +inline year_month operator-(const year_month& ym, const years& dy) NOEXCEPT { + return ym + -dy; } -template -inline -std::basic_ostream& -operator<<(std::basic_ostream& os, const year_month& ym) -{ - return os << ym.year() << '/' << ym.month(); +template +inline std::basic_ostream& operator<<(std::basic_ostream& os, const year_month& ym) { + return os << ym.year() << '/' << ym.month(); } // month_day CONSTCD11 -inline -month_day::month_day(const date::month& m, const date::day& d) NOEXCEPT - : m_(m) - , d_(d) - {} +inline month_day::month_day(const date::month& m, const date::day& d) NOEXCEPT : m_(m), d_(d) { +} -CONSTCD11 inline date::month month_day::month() const NOEXCEPT {return m_;} -CONSTCD11 inline date::day month_day::day() const NOEXCEPT {return d_;} +CONSTCD11 inline date::month month_day::month() const NOEXCEPT { + return m_; +} +CONSTCD11 inline date::day month_day::day() const NOEXCEPT { + return d_; +} CONSTCD14 -inline -bool -month_day::ok() const NOEXCEPT -{ - CONSTDATA date::day d[] = - { - date::day(31), date::day(29), date::day(31), - date::day(30), date::day(31), date::day(30), - date::day(31), date::day(31), date::day(30), - date::day(31), date::day(30), date::day(31) - }; - return m_.ok() && date::day{1} <= d_ && d_ <= d[static_cast(m_)-1]; +inline bool month_day::ok() const NOEXCEPT { + CONSTDATA date::day d[] = {date::day(31), date::day(29), date::day(31), date::day(30), date::day(31), date::day(30), + date::day(31), date::day(31), date::day(30), date::day(31), date::day(30), date::day(31)}; + return m_.ok() && date::day{1} <= d_ && d_ <= d[static_cast(m_) - 1]; } CONSTCD11 -inline -bool -operator==(const month_day& x, const month_day& y) NOEXCEPT -{ - return x.month() == y.month() && x.day() == y.day(); +inline bool operator==(const month_day& x, const month_day& y) NOEXCEPT { + return x.month() == y.month() && x.day() == y.day(); } CONSTCD11 -inline -bool -operator!=(const month_day& x, const month_day& y) NOEXCEPT -{ - return !(x == y); +inline bool operator!=(const month_day& x, const month_day& y) NOEXCEPT { + return !(x == y); } CONSTCD11 -inline -bool -operator<(const month_day& x, const month_day& y) NOEXCEPT -{ - return x.month() < y.month() ? true - : (x.month() > y.month() ? false - : (x.day() < y.day())); +inline bool operator<(const month_day& x, const month_day& y) NOEXCEPT { + return x.month() < y.month() ? true : (x.month() > y.month() ? false : (x.day() < y.day())); } CONSTCD11 -inline -bool -operator>(const month_day& x, const month_day& y) NOEXCEPT -{ - return y < x; +inline bool operator>(const month_day& x, const month_day& y) NOEXCEPT { + return y < x; } CONSTCD11 -inline -bool -operator<=(const month_day& x, const month_day& y) NOEXCEPT -{ - return !(y < x); +inline bool operator<=(const month_day& x, const month_day& y) NOEXCEPT { + return !(y < x); } CONSTCD11 -inline -bool -operator>=(const month_day& x, const month_day& y) NOEXCEPT -{ - return !(x < y); +inline bool operator>=(const month_day& x, const month_day& y) NOEXCEPT { + return !(x < y); } -template -inline -std::basic_ostream& -operator<<(std::basic_ostream& os, const month_day& md) -{ - return os << md.month() << '/' << md.day(); +template +inline std::basic_ostream& operator<<(std::basic_ostream& os, const month_day& md) { + return os << md.month() << '/' << md.day(); } // month_day_last -CONSTCD11 inline month month_day_last::month() const NOEXCEPT {return m_;} -CONSTCD11 inline bool month_day_last::ok() const NOEXCEPT {return m_.ok();} -CONSTCD11 inline month_day_last::month_day_last(const date::month& m) NOEXCEPT : m_(m) {} +CONSTCD11 inline month month_day_last::month() const NOEXCEPT { + return m_; +} +CONSTCD11 inline bool month_day_last::ok() const NOEXCEPT { + return m_.ok(); +} +CONSTCD11 inline month_day_last::month_day_last(const date::month& m) NOEXCEPT : m_(m) { +} CONSTCD11 -inline -bool -operator==(const month_day_last& x, const month_day_last& y) NOEXCEPT -{ - return x.month() == y.month(); +inline bool operator==(const month_day_last& x, const month_day_last& y) NOEXCEPT { + return x.month() == y.month(); } CONSTCD11 -inline -bool -operator!=(const month_day_last& x, const month_day_last& y) NOEXCEPT -{ - return !(x == y); +inline bool operator!=(const month_day_last& x, const month_day_last& y) NOEXCEPT { + return !(x == y); } CONSTCD11 -inline -bool -operator<(const month_day_last& x, const month_day_last& y) NOEXCEPT -{ - return x.month() < y.month(); +inline bool operator<(const month_day_last& x, const month_day_last& y) NOEXCEPT { + return x.month() < y.month(); } CONSTCD11 -inline -bool -operator>(const month_day_last& x, const month_day_last& y) NOEXCEPT -{ - return y < x; +inline bool operator>(const month_day_last& x, const month_day_last& y) NOEXCEPT { + return y < x; } CONSTCD11 -inline -bool -operator<=(const month_day_last& x, const month_day_last& y) NOEXCEPT -{ - return !(y < x); +inline bool operator<=(const month_day_last& x, const month_day_last& y) NOEXCEPT { + return !(y < x); } CONSTCD11 -inline -bool -operator>=(const month_day_last& x, const month_day_last& y) NOEXCEPT -{ - return !(x < y); +inline bool operator>=(const month_day_last& x, const month_day_last& y) NOEXCEPT { + return !(x < y); } -template -inline -std::basic_ostream& -operator<<(std::basic_ostream& os, const month_day_last& mdl) -{ - return os << mdl.month() << "/last"; +template +inline std::basic_ostream& operator<<(std::basic_ostream& os, const month_day_last& mdl) { + return os << mdl.month() << "/last"; } // month_weekday CONSTCD11 -inline -month_weekday::month_weekday(const date::month& m, - const date::weekday_indexed& wdi) NOEXCEPT - : m_(m) - , wdi_(wdi) - {} +inline month_weekday::month_weekday(const date::month& m, const date::weekday_indexed& wdi) NOEXCEPT : m_(m), + wdi_(wdi) { +} -CONSTCD11 inline month month_weekday::month() const NOEXCEPT {return m_;} +CONSTCD11 inline month month_weekday::month() const NOEXCEPT { + return m_; +} CONSTCD11 -inline -weekday_indexed -month_weekday::weekday_indexed() const NOEXCEPT -{ - return wdi_; +inline weekday_indexed month_weekday::weekday_indexed() const NOEXCEPT { + return wdi_; } CONSTCD11 -inline -bool -month_weekday::ok() const NOEXCEPT -{ - return m_.ok() && wdi_.ok(); +inline bool month_weekday::ok() const NOEXCEPT { + return m_.ok() && wdi_.ok(); } CONSTCD11 -inline -bool -operator==(const month_weekday& x, const month_weekday& y) NOEXCEPT -{ - return x.month() == y.month() && x.weekday_indexed() == y.weekday_indexed(); +inline bool operator==(const month_weekday& x, const month_weekday& y) NOEXCEPT { + return x.month() == y.month() && x.weekday_indexed() == y.weekday_indexed(); } CONSTCD11 -inline -bool -operator!=(const month_weekday& x, const month_weekday& y) NOEXCEPT -{ - return !(x == y); +inline bool operator!=(const month_weekday& x, const month_weekday& y) NOEXCEPT { + return !(x == y); } -template -inline -std::basic_ostream& -operator<<(std::basic_ostream& os, const month_weekday& mwd) -{ - return os << mwd.month() << '/' << mwd.weekday_indexed(); +template +inline std::basic_ostream& operator<<(std::basic_ostream& os, const month_weekday& mwd) { + return os << mwd.month() << '/' << mwd.weekday_indexed(); } // month_weekday_last CONSTCD11 -inline -month_weekday_last::month_weekday_last(const date::month& m, - const date::weekday_last& wdl) NOEXCEPT - : m_(m) - , wdl_(wdl) - {} +inline month_weekday_last::month_weekday_last(const date::month& m, const date::weekday_last& wdl) NOEXCEPT + : m_(m), + wdl_(wdl) { +} -CONSTCD11 inline month month_weekday_last::month() const NOEXCEPT {return m_;} +CONSTCD11 inline month month_weekday_last::month() const NOEXCEPT { + return m_; +} CONSTCD11 -inline -weekday_last -month_weekday_last::weekday_last() const NOEXCEPT -{ - return wdl_; +inline weekday_last month_weekday_last::weekday_last() const NOEXCEPT { + return wdl_; } CONSTCD11 -inline -bool -month_weekday_last::ok() const NOEXCEPT -{ - return m_.ok() && wdl_.ok(); +inline bool month_weekday_last::ok() const NOEXCEPT { + return m_.ok() && wdl_.ok(); } CONSTCD11 -inline -bool -operator==(const month_weekday_last& x, const month_weekday_last& y) NOEXCEPT -{ - return x.month() == y.month() && x.weekday_last() == y.weekday_last(); +inline bool operator==(const month_weekday_last& x, const month_weekday_last& y) NOEXCEPT { + return x.month() == y.month() && x.weekday_last() == y.weekday_last(); } CONSTCD11 -inline -bool -operator!=(const month_weekday_last& x, const month_weekday_last& y) NOEXCEPT -{ - return !(x == y); +inline bool operator!=(const month_weekday_last& x, const month_weekday_last& y) NOEXCEPT { + return !(x == y); } -template -inline -std::basic_ostream& -operator<<(std::basic_ostream& os, const month_weekday_last& mwdl) -{ - return os << mwdl.month() << '/' << mwdl.weekday_last(); +template +inline std::basic_ostream& operator<<(std::basic_ostream& os, + const month_weekday_last& mwdl) { + return os << mwdl.month() << '/' << mwdl.weekday_last(); } // year_month_day_last CONSTCD11 -inline -year_month_day_last::year_month_day_last(const date::year& y, - const date::month_day_last& mdl) NOEXCEPT - : y_(y) - , mdl_(mdl) - {} +inline year_month_day_last::year_month_day_last(const date::year& y, const date::month_day_last& mdl) NOEXCEPT + : y_(y), + mdl_(mdl) { +} -template -CONSTCD14 -inline -year_month_day_last& -year_month_day_last::operator+=(const months& m) NOEXCEPT -{ - *this = *this + m; - return *this; +template +CONSTCD14 inline year_month_day_last& year_month_day_last::operator+=(const months& m) NOEXCEPT { + *this = *this + m; + return *this; } -template -CONSTCD14 -inline -year_month_day_last& -year_month_day_last::operator-=(const months& m) NOEXCEPT -{ - *this = *this - m; - return *this; +template +CONSTCD14 inline year_month_day_last& year_month_day_last::operator-=(const months& m) NOEXCEPT { + *this = *this - m; + return *this; } CONSTCD14 -inline -year_month_day_last& -year_month_day_last::operator+=(const years& y) NOEXCEPT -{ - *this = *this + y; - return *this; +inline year_month_day_last& year_month_day_last::operator+=(const years& y) NOEXCEPT { + *this = *this + y; + return *this; } CONSTCD14 -inline -year_month_day_last& -year_month_day_last::operator-=(const years& y) NOEXCEPT -{ - *this = *this - y; - return *this; +inline year_month_day_last& year_month_day_last::operator-=(const years& y) NOEXCEPT { + *this = *this - y; + return *this; } -CONSTCD11 inline year year_month_day_last::year() const NOEXCEPT {return y_;} -CONSTCD11 inline month year_month_day_last::month() const NOEXCEPT {return mdl_.month();} +CONSTCD11 inline year year_month_day_last::year() const NOEXCEPT { + return y_; +} +CONSTCD11 inline month year_month_day_last::month() const NOEXCEPT { + return mdl_.month(); +} CONSTCD11 -inline -month_day_last -year_month_day_last::month_day_last() const NOEXCEPT -{ - return mdl_; +inline month_day_last year_month_day_last::month_day_last() const NOEXCEPT { + return mdl_; } CONSTCD14 -inline -day -year_month_day_last::day() const NOEXCEPT -{ - CONSTDATA date::day d[] = - { - date::day(31), date::day(28), date::day(31), - date::day(30), date::day(31), date::day(30), - date::day(31), date::day(31), date::day(30), - date::day(31), date::day(30), date::day(31) - }; - return (month() != February || !y_.is_leap()) && mdl_.ok() ? - d[static_cast(month()) - 1] : date::day{29}; +inline day year_month_day_last::day() const NOEXCEPT { + CONSTDATA date::day d[] = {date::day(31), date::day(28), date::day(31), date::day(30), date::day(31), date::day(30), + date::day(31), date::day(31), date::day(30), date::day(31), date::day(30), date::day(31)}; + return (month() != February || !y_.is_leap()) && mdl_.ok() ? d[static_cast(month()) - 1] : date::day{29}; } CONSTCD14 -inline -year_month_day_last::operator sys_days() const NOEXCEPT -{ - return sys_days(year()/month()/day()); +inline year_month_day_last::operator sys_days() const NOEXCEPT { + return sys_days(year() / month() / day()); } CONSTCD14 -inline -year_month_day_last::operator local_days() const NOEXCEPT -{ - return local_days(year()/month()/day()); +inline year_month_day_last::operator local_days() const NOEXCEPT { + return local_days(year() / month() / day()); } CONSTCD11 -inline -bool -year_month_day_last::ok() const NOEXCEPT -{ - return y_.ok() && mdl_.ok(); +inline bool year_month_day_last::ok() const NOEXCEPT { + return y_.ok() && mdl_.ok(); } CONSTCD11 -inline -bool -operator==(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT -{ - return x.year() == y.year() && x.month_day_last() == y.month_day_last(); +inline bool operator==(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT { + return x.year() == y.year() && x.month_day_last() == y.month_day_last(); } CONSTCD11 -inline -bool -operator!=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT -{ - return !(x == y); +inline bool operator!=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT { + return !(x == y); } CONSTCD11 -inline -bool -operator<(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT -{ - return x.year() < y.year() ? true - : (x.year() > y.year() ? false - : (x.month_day_last() < y.month_day_last())); +inline bool operator<(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT { + return x.year() < y.year() ? true : (x.year() > y.year() ? false : (x.month_day_last() < y.month_day_last())); } CONSTCD11 -inline -bool -operator>(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT -{ - return y < x; +inline bool operator>(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT { + return y < x; } CONSTCD11 -inline -bool -operator<=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT -{ - return !(y < x); +inline bool operator<=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT { + return !(y < x); } CONSTCD11 -inline -bool -operator>=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT -{ - return !(x < y); +inline bool operator>=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT { + return !(x < y); } -template -inline -std::basic_ostream& -operator<<(std::basic_ostream& os, const year_month_day_last& ymdl) -{ - return os << ymdl.year() << '/' << ymdl.month_day_last(); +template +inline std::basic_ostream& operator<<(std::basic_ostream& os, + const year_month_day_last& ymdl) { + return os << ymdl.year() << '/' << ymdl.month_day_last(); } -template -CONSTCD14 -inline -year_month_day_last -operator+(const year_month_day_last& ymdl, const months& dm) NOEXCEPT -{ - return (ymdl.year() / ymdl.month() + dm) / last; +template +CONSTCD14 inline year_month_day_last operator+(const year_month_day_last& ymdl, const months& dm) NOEXCEPT { + return (ymdl.year() / ymdl.month() + dm) / last; } -template -CONSTCD14 -inline -year_month_day_last -operator+(const months& dm, const year_month_day_last& ymdl) NOEXCEPT -{ - return ymdl + dm; +template +CONSTCD14 inline year_month_day_last operator+(const months& dm, const year_month_day_last& ymdl) NOEXCEPT { + return ymdl + dm; } -template -CONSTCD14 -inline -year_month_day_last -operator-(const year_month_day_last& ymdl, const months& dm) NOEXCEPT -{ - return ymdl + (-dm); +template +CONSTCD14 inline year_month_day_last operator-(const year_month_day_last& ymdl, const months& dm) NOEXCEPT { + return ymdl + (-dm); } CONSTCD11 -inline -year_month_day_last -operator+(const year_month_day_last& ymdl, const years& dy) NOEXCEPT -{ - return {ymdl.year()+dy, ymdl.month_day_last()}; +inline year_month_day_last operator+(const year_month_day_last& ymdl, const years& dy) NOEXCEPT { + return {ymdl.year() + dy, ymdl.month_day_last()}; } CONSTCD11 -inline -year_month_day_last -operator+(const years& dy, const year_month_day_last& ymdl) NOEXCEPT -{ - return ymdl + dy; +inline year_month_day_last operator+(const years& dy, const year_month_day_last& ymdl) NOEXCEPT { + return ymdl + dy; } CONSTCD11 -inline -year_month_day_last -operator-(const year_month_day_last& ymdl, const years& dy) NOEXCEPT -{ - return ymdl + (-dy); +inline year_month_day_last operator-(const year_month_day_last& ymdl, const years& dy) NOEXCEPT { + return ymdl + (-dy); } // year_month_day CONSTCD11 -inline -year_month_day::year_month_day(const date::year& y, const date::month& m, - const date::day& d) NOEXCEPT - : y_(y) - , m_(m) - , d_(d) - {} +inline year_month_day::year_month_day(const date::year& y, const date::month& m, const date::day& d) NOEXCEPT : y_(y), + m_(m), + d_(d) { +} CONSTCD14 -inline -year_month_day::year_month_day(const year_month_day_last& ymdl) NOEXCEPT - : y_(ymdl.year()) - , m_(ymdl.month()) - , d_(ymdl.day()) - {} +inline year_month_day::year_month_day(const year_month_day_last& ymdl) NOEXCEPT : y_(ymdl.year()), + m_(ymdl.month()), + d_(ymdl.day()) { +} CONSTCD14 -inline -year_month_day::year_month_day(sys_days dp) NOEXCEPT - : year_month_day(from_days(dp.time_since_epoch())) - {} +inline year_month_day::year_month_day(sys_days dp) NOEXCEPT : year_month_day(from_days(dp.time_since_epoch())) { +} CONSTCD14 -inline -year_month_day::year_month_day(local_days dp) NOEXCEPT - : year_month_day(from_days(dp.time_since_epoch())) - {} +inline year_month_day::year_month_day(local_days dp) NOEXCEPT : year_month_day(from_days(dp.time_since_epoch())) { +} -CONSTCD11 inline year year_month_day::year() const NOEXCEPT {return y_;} -CONSTCD11 inline month year_month_day::month() const NOEXCEPT {return m_;} -CONSTCD11 inline day year_month_day::day() const NOEXCEPT {return d_;} +CONSTCD11 inline year year_month_day::year() const NOEXCEPT { + return y_; +} +CONSTCD11 inline month year_month_day::month() const NOEXCEPT { + return m_; +} +CONSTCD11 inline day year_month_day::day() const NOEXCEPT { + return d_; +} -template -CONSTCD14 -inline -year_month_day& -year_month_day::operator+=(const months& m) NOEXCEPT -{ - *this = *this + m; - return *this; +template +CONSTCD14 inline year_month_day& year_month_day::operator+=(const months& m) NOEXCEPT { + *this = *this + m; + return *this; } -template -CONSTCD14 -inline -year_month_day& -year_month_day::operator-=(const months& m) NOEXCEPT -{ - *this = *this - m; - return *this; +template +CONSTCD14 inline year_month_day& year_month_day::operator-=(const months& m) NOEXCEPT { + *this = *this - m; + return *this; } CONSTCD14 -inline -year_month_day& -year_month_day::operator+=(const years& y) NOEXCEPT -{ - *this = *this + y; - return *this; +inline year_month_day& year_month_day::operator+=(const years& y) NOEXCEPT { + *this = *this + y; + return *this; } CONSTCD14 -inline -year_month_day& -year_month_day::operator-=(const years& y) NOEXCEPT -{ - *this = *this - y; - return *this; +inline year_month_day& year_month_day::operator-=(const years& y) NOEXCEPT { + *this = *this - y; + return *this; } CONSTCD14 -inline -days -year_month_day::to_days() const NOEXCEPT -{ - static_assert(std::numeric_limits::digits >= 18, - "This algorithm has not been ported to a 16 bit unsigned integer"); - static_assert(std::numeric_limits::digits >= 20, - "This algorithm has not been ported to a 16 bit signed integer"); - auto const y = static_cast(y_) - (m_ <= February); - auto const m = static_cast(m_); - auto const d = static_cast(d_); - auto const era = (y >= 0 ? y : y-399) / 400; - auto const yoe = static_cast(y - era * 400); // [0, 399] - auto const doy = (153*(m > 2 ? m-3 : m+9) + 2)/5 + d-1; // [0, 365] - auto const doe = yoe * 365 + yoe/4 - yoe/100 + doy; // [0, 146096] - return days{era * 146097 + static_cast(doe) - 719468}; +inline days year_month_day::to_days() const NOEXCEPT { + static_assert(std::numeric_limits::digits >= 18, + "This algorithm has not been ported to a 16 bit unsigned integer"); + static_assert(std::numeric_limits::digits >= 20, + "This algorithm has not been ported to a 16 bit signed integer"); + auto const y = static_cast(y_) - (m_ <= February); + auto const m = static_cast(m_); + auto const d = static_cast(d_); + auto const era = (y >= 0 ? y : y - 399) / 400; + auto const yoe = static_cast(y - era * 400); // [0, 399] + auto const doy = (153 * (m > 2 ? m - 3 : m + 9) + 2) / 5 + d - 1; // [0, 365] + auto const doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; // [0, 146096] + return days{era * 146097 + static_cast(doe) - 719468}; } CONSTCD14 -inline -year_month_day::operator sys_days() const NOEXCEPT -{ - return sys_days{to_days()}; +inline year_month_day::operator sys_days() const NOEXCEPT { + return sys_days{to_days()}; } CONSTCD14 -inline -year_month_day::operator local_days() const NOEXCEPT -{ - return local_days{to_days()}; +inline year_month_day::operator local_days() const NOEXCEPT { + return local_days{to_days()}; } CONSTCD14 -inline -bool -year_month_day::ok() const NOEXCEPT -{ - if (!(y_.ok() && m_.ok())) - return false; - return date::day{1} <= d_ && d_ <= (y_ / m_ / last).day(); +inline bool year_month_day::ok() const NOEXCEPT { + if (!(y_.ok() && m_.ok())) + return false; + return date::day{1} <= d_ && d_ <= (y_ / m_ / last).day(); } CONSTCD11 -inline -bool -operator==(const year_month_day& x, const year_month_day& y) NOEXCEPT -{ - return x.year() == y.year() && x.month() == y.month() && x.day() == y.day(); +inline bool operator==(const year_month_day& x, const year_month_day& y) NOEXCEPT { + return x.year() == y.year() && x.month() == y.month() && x.day() == y.day(); } CONSTCD11 -inline -bool -operator!=(const year_month_day& x, const year_month_day& y) NOEXCEPT -{ - return !(x == y); +inline bool operator!=(const year_month_day& x, const year_month_day& y) NOEXCEPT { + return !(x == y); } CONSTCD11 -inline -bool -operator<(const year_month_day& x, const year_month_day& y) NOEXCEPT -{ - return x.year() < y.year() ? true - : (x.year() > y.year() ? false - : (x.month() < y.month() ? true - : (x.month() > y.month() ? false - : (x.day() < y.day())))); +inline bool operator<(const year_month_day& x, const year_month_day& y) NOEXCEPT { + return x.year() < y.year() + ? true + : (x.year() > y.year() + ? false + : (x.month() < y.month() ? true : (x.month() > y.month() ? false : (x.day() < y.day())))); } CONSTCD11 -inline -bool -operator>(const year_month_day& x, const year_month_day& y) NOEXCEPT -{ - return y < x; +inline bool operator>(const year_month_day& x, const year_month_day& y) NOEXCEPT { + return y < x; } CONSTCD11 -inline -bool -operator<=(const year_month_day& x, const year_month_day& y) NOEXCEPT -{ - return !(y < x); +inline bool operator<=(const year_month_day& x, const year_month_day& y) NOEXCEPT { + return !(y < x); } CONSTCD11 -inline -bool -operator>=(const year_month_day& x, const year_month_day& y) NOEXCEPT -{ - return !(x < y); +inline bool operator>=(const year_month_day& x, const year_month_day& y) NOEXCEPT { + return !(x < y); } -template -inline -std::basic_ostream& -operator<<(std::basic_ostream& os, const year_month_day& ymd) -{ - detail::save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - os.imbue(std::locale::classic()); - os << ymd.year() << '-'; - os.width(2); - os << static_cast(ymd.month()) << '-'; - os << ymd.day(); - if (!ymd.ok()) - os << " is not a valid date"; - return os; +template +inline std::basic_ostream& operator<<(std::basic_ostream& os, const year_month_day& ymd) { + detail::save_ostream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.imbue(std::locale::classic()); + os << ymd.year() << '-'; + os.width(2); + os << static_cast(ymd.month()) << '-'; + os << ymd.day(); + if (!ymd.ok()) + os << " is not a valid date"; + return os; } CONSTCD14 -inline -year_month_day -year_month_day::from_days(days dp) NOEXCEPT -{ - static_assert(std::numeric_limits::digits >= 18, - "This algorithm has not been ported to a 16 bit unsigned integer"); - static_assert(std::numeric_limits::digits >= 20, - "This algorithm has not been ported to a 16 bit signed integer"); - auto const z = dp.count() + 719468; - auto const era = (z >= 0 ? z : z - 146096) / 146097; - auto const doe = static_cast(z - era * 146097); // [0, 146096] - auto const yoe = (doe - doe/1460 + doe/36524 - doe/146096) / 365; // [0, 399] - auto const y = static_cast(yoe) + era * 400; - auto const doy = doe - (365*yoe + yoe/4 - yoe/100); // [0, 365] - auto const mp = (5*doy + 2)/153; // [0, 11] - auto const d = doy - (153*mp+2)/5 + 1; // [1, 31] - auto const m = mp < 10 ? mp+3 : mp-9; // [1, 12] - return year_month_day{date::year{y + (m <= 2)}, date::month(m), date::day(d)}; -} - -template -CONSTCD14 -inline -year_month_day -operator+(const year_month_day& ymd, const months& dm) NOEXCEPT -{ - return (ymd.year() / ymd.month() + dm) / ymd.day(); +inline year_month_day year_month_day::from_days(days dp) NOEXCEPT { + static_assert(std::numeric_limits::digits >= 18, + "This algorithm has not been ported to a 16 bit unsigned integer"); + static_assert(std::numeric_limits::digits >= 20, + "This algorithm has not been ported to a 16 bit signed integer"); + auto const z = dp.count() + 719468; + auto const era = (z >= 0 ? z : z - 146096) / 146097; + auto const doe = static_cast(z - era * 146097); // [0, 146096] + auto const yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; // [0, 399] + auto const y = static_cast(yoe) + era * 400; + auto const doy = doe - (365 * yoe + yoe / 4 - yoe / 100); // [0, 365] + auto const mp = (5 * doy + 2) / 153; // [0, 11] + auto const d = doy - (153 * mp + 2) / 5 + 1; // [1, 31] + auto const m = mp < 10 ? mp + 3 : mp - 9; // [1, 12] + return year_month_day{date::year{y + (m <= 2)}, date::month(m), date::day(d)}; } -template -CONSTCD14 -inline -year_month_day -operator+(const months& dm, const year_month_day& ymd) NOEXCEPT -{ - return ymd + dm; +template +CONSTCD14 inline year_month_day operator+(const year_month_day& ymd, const months& dm) NOEXCEPT { + return (ymd.year() / ymd.month() + dm) / ymd.day(); } -template -CONSTCD14 -inline -year_month_day -operator-(const year_month_day& ymd, const months& dm) NOEXCEPT -{ - return ymd + (-dm); +template +CONSTCD14 inline year_month_day operator+(const months& dm, const year_month_day& ymd) NOEXCEPT { + return ymd + dm; +} + +template +CONSTCD14 inline year_month_day operator-(const year_month_day& ymd, const months& dm) NOEXCEPT { + return ymd + (-dm); } CONSTCD11 -inline -year_month_day -operator+(const year_month_day& ymd, const years& dy) NOEXCEPT -{ - return (ymd.year() + dy) / ymd.month() / ymd.day(); +inline year_month_day operator+(const year_month_day& ymd, const years& dy) NOEXCEPT { + return (ymd.year() + dy) / ymd.month() / ymd.day(); } CONSTCD11 -inline -year_month_day -operator+(const years& dy, const year_month_day& ymd) NOEXCEPT -{ - return ymd + dy; +inline year_month_day operator+(const years& dy, const year_month_day& ymd) NOEXCEPT { + return ymd + dy; } CONSTCD11 -inline -year_month_day -operator-(const year_month_day& ymd, const years& dy) NOEXCEPT -{ - return ymd + (-dy); +inline year_month_day operator-(const year_month_day& ymd, const years& dy) NOEXCEPT { + return ymd + (-dy); } // year_month_weekday CONSTCD11 -inline -year_month_weekday::year_month_weekday(const date::year& y, const date::month& m, - const date::weekday_indexed& wdi) - NOEXCEPT - : y_(y) - , m_(m) - , wdi_(wdi) - {} +inline year_month_weekday::year_month_weekday(const date::year& y, const date::month& m, + const date::weekday_indexed& wdi) NOEXCEPT : y_(y), + m_(m), + wdi_(wdi) { +} CONSTCD14 -inline -year_month_weekday::year_month_weekday(const sys_days& dp) NOEXCEPT - : year_month_weekday(from_days(dp.time_since_epoch())) - {} +inline year_month_weekday::year_month_weekday(const sys_days& dp) NOEXCEPT + : year_month_weekday(from_days(dp.time_since_epoch())) { +} CONSTCD14 -inline -year_month_weekday::year_month_weekday(const local_days& dp) NOEXCEPT - : year_month_weekday(from_days(dp.time_since_epoch())) - {} +inline year_month_weekday::year_month_weekday(const local_days& dp) NOEXCEPT + : year_month_weekday(from_days(dp.time_since_epoch())) { +} -template -CONSTCD14 -inline -year_month_weekday& -year_month_weekday::operator+=(const months& m) NOEXCEPT -{ - *this = *this + m; - return *this; +template +CONSTCD14 inline year_month_weekday& year_month_weekday::operator+=(const months& m) NOEXCEPT { + *this = *this + m; + return *this; } -template -CONSTCD14 -inline -year_month_weekday& -year_month_weekday::operator-=(const months& m) NOEXCEPT -{ - *this = *this - m; - return *this; +template +CONSTCD14 inline year_month_weekday& year_month_weekday::operator-=(const months& m) NOEXCEPT { + *this = *this - m; + return *this; } CONSTCD14 -inline -year_month_weekday& -year_month_weekday::operator+=(const years& y) NOEXCEPT -{ - *this = *this + y; - return *this; +inline year_month_weekday& year_month_weekday::operator+=(const years& y) NOEXCEPT { + *this = *this + y; + return *this; } CONSTCD14 -inline -year_month_weekday& -year_month_weekday::operator-=(const years& y) NOEXCEPT -{ - *this = *this - y; - return *this; +inline year_month_weekday& year_month_weekday::operator-=(const years& y) NOEXCEPT { + *this = *this - y; + return *this; } -CONSTCD11 inline year year_month_weekday::year() const NOEXCEPT {return y_;} -CONSTCD11 inline month year_month_weekday::month() const NOEXCEPT {return m_;} +CONSTCD11 inline year year_month_weekday::year() const NOEXCEPT { + return y_; +} +CONSTCD11 inline month year_month_weekday::month() const NOEXCEPT { + return m_; +} CONSTCD11 -inline -weekday -year_month_weekday::weekday() const NOEXCEPT -{ - return wdi_.weekday(); +inline weekday year_month_weekday::weekday() const NOEXCEPT { + return wdi_.weekday(); } CONSTCD11 -inline -unsigned -year_month_weekday::index() const NOEXCEPT -{ - return wdi_.index(); +inline unsigned year_month_weekday::index() const NOEXCEPT { + return wdi_.index(); } CONSTCD11 -inline -weekday_indexed -year_month_weekday::weekday_indexed() const NOEXCEPT -{ - return wdi_; +inline weekday_indexed year_month_weekday::weekday_indexed() const NOEXCEPT { + return wdi_; } CONSTCD14 -inline -year_month_weekday::operator sys_days() const NOEXCEPT -{ - return sys_days{to_days()}; +inline year_month_weekday::operator sys_days() const NOEXCEPT { + return sys_days{to_days()}; } CONSTCD14 -inline -year_month_weekday::operator local_days() const NOEXCEPT -{ - return local_days{to_days()}; +inline year_month_weekday::operator local_days() const NOEXCEPT { + return local_days{to_days()}; } CONSTCD14 -inline -bool -year_month_weekday::ok() const NOEXCEPT -{ - if (!y_.ok() || !m_.ok() || !wdi_.weekday().ok() || wdi_.index() < 1) - return false; - if (wdi_.index() <= 4) - return true; - auto d2 = wdi_.weekday() - date::weekday(static_cast(y_/m_/1)) + - days((wdi_.index()-1)*7 + 1); - return static_cast(d2.count()) <= static_cast((y_/m_/last).day()); +inline bool year_month_weekday::ok() const NOEXCEPT { + if (!y_.ok() || !m_.ok() || !wdi_.weekday().ok() || wdi_.index() < 1) + return false; + if (wdi_.index() <= 4) + return true; + auto d2 = wdi_.weekday() - date::weekday(static_cast(y_ / m_ / 1)) + days((wdi_.index() - 1) * 7 + 1); + return static_cast(d2.count()) <= static_cast((y_ / m_ / last).day()); } CONSTCD14 -inline -year_month_weekday -year_month_weekday::from_days(days d) NOEXCEPT -{ - sys_days dp{d}; - auto const wd = date::weekday(dp); - auto const ymd = year_month_day(dp); - return {ymd.year(), ymd.month(), wd[(static_cast(ymd.day())-1)/7+1]}; +inline year_month_weekday year_month_weekday::from_days(days d) NOEXCEPT { + sys_days dp{d}; + auto const wd = date::weekday(dp); + auto const ymd = year_month_day(dp); + return {ymd.year(), ymd.month(), wd[(static_cast(ymd.day()) - 1) / 7 + 1]}; } CONSTCD14 -inline -days -year_month_weekday::to_days() const NOEXCEPT -{ - auto d = sys_days(y_/m_/1); - return (d + (wdi_.weekday() - date::weekday(d) + days{(wdi_.index()-1)*7}) - ).time_since_epoch(); +inline days year_month_weekday::to_days() const NOEXCEPT { + auto d = sys_days(y_ / m_ / 1); + return (d + (wdi_.weekday() - date::weekday(d) + days{(wdi_.index() - 1) * 7})).time_since_epoch(); } CONSTCD11 -inline -bool -operator==(const year_month_weekday& x, const year_month_weekday& y) NOEXCEPT -{ - return x.year() == y.year() && x.month() == y.month() && - x.weekday_indexed() == y.weekday_indexed(); +inline bool operator==(const year_month_weekday& x, const year_month_weekday& y) NOEXCEPT { + return x.year() == y.year() && x.month() == y.month() && x.weekday_indexed() == y.weekday_indexed(); } CONSTCD11 -inline -bool -operator!=(const year_month_weekday& x, const year_month_weekday& y) NOEXCEPT -{ - return !(x == y); +inline bool operator!=(const year_month_weekday& x, const year_month_weekday& y) NOEXCEPT { + return !(x == y); } -template -inline -std::basic_ostream& -operator<<(std::basic_ostream& os, const year_month_weekday& ymwdi) -{ - return os << ymwdi.year() << '/' << ymwdi.month() - << '/' << ymwdi.weekday_indexed(); +template +inline std::basic_ostream& operator<<(std::basic_ostream& os, + const year_month_weekday& ymwdi) { + return os << ymwdi.year() << '/' << ymwdi.month() << '/' << ymwdi.weekday_indexed(); } -template -CONSTCD14 -inline -year_month_weekday -operator+(const year_month_weekday& ymwd, const months& dm) NOEXCEPT -{ - return (ymwd.year() / ymwd.month() + dm) / ymwd.weekday_indexed(); +template +CONSTCD14 inline year_month_weekday operator+(const year_month_weekday& ymwd, const months& dm) NOEXCEPT { + return (ymwd.year() / ymwd.month() + dm) / ymwd.weekday_indexed(); } -template -CONSTCD14 -inline -year_month_weekday -operator+(const months& dm, const year_month_weekday& ymwd) NOEXCEPT -{ - return ymwd + dm; +template +CONSTCD14 inline year_month_weekday operator+(const months& dm, const year_month_weekday& ymwd) NOEXCEPT { + return ymwd + dm; } -template -CONSTCD14 -inline -year_month_weekday -operator-(const year_month_weekday& ymwd, const months& dm) NOEXCEPT -{ - return ymwd + (-dm); +template +CONSTCD14 inline year_month_weekday operator-(const year_month_weekday& ymwd, const months& dm) NOEXCEPT { + return ymwd + (-dm); } CONSTCD11 -inline -year_month_weekday -operator+(const year_month_weekday& ymwd, const years& dy) NOEXCEPT -{ - return {ymwd.year()+dy, ymwd.month(), ymwd.weekday_indexed()}; +inline year_month_weekday operator+(const year_month_weekday& ymwd, const years& dy) NOEXCEPT { + return {ymwd.year() + dy, ymwd.month(), ymwd.weekday_indexed()}; } CONSTCD11 -inline -year_month_weekday -operator+(const years& dy, const year_month_weekday& ymwd) NOEXCEPT -{ - return ymwd + dy; +inline year_month_weekday operator+(const years& dy, const year_month_weekday& ymwd) NOEXCEPT { + return ymwd + dy; } CONSTCD11 -inline -year_month_weekday -operator-(const year_month_weekday& ymwd, const years& dy) NOEXCEPT -{ - return ymwd + (-dy); +inline year_month_weekday operator-(const year_month_weekday& ymwd, const years& dy) NOEXCEPT { + return ymwd + (-dy); } // year_month_weekday_last CONSTCD11 -inline -year_month_weekday_last::year_month_weekday_last(const date::year& y, - const date::month& m, - const date::weekday_last& wdl) NOEXCEPT - : y_(y) - , m_(m) - , wdl_(wdl) - {} +inline year_month_weekday_last::year_month_weekday_last(const date::year& y, const date::month& m, + const date::weekday_last& wdl) NOEXCEPT : y_(y), + m_(m), + wdl_(wdl) { +} -template -CONSTCD14 -inline -year_month_weekday_last& -year_month_weekday_last::operator+=(const months& m) NOEXCEPT -{ - *this = *this + m; - return *this; +template +CONSTCD14 inline year_month_weekday_last& year_month_weekday_last::operator+=(const months& m) NOEXCEPT { + *this = *this + m; + return *this; } -template -CONSTCD14 -inline -year_month_weekday_last& -year_month_weekday_last::operator-=(const months& m) NOEXCEPT -{ - *this = *this - m; - return *this; +template +CONSTCD14 inline year_month_weekday_last& year_month_weekday_last::operator-=(const months& m) NOEXCEPT { + *this = *this - m; + return *this; } CONSTCD14 -inline -year_month_weekday_last& -year_month_weekday_last::operator+=(const years& y) NOEXCEPT -{ - *this = *this + y; - return *this; +inline year_month_weekday_last& year_month_weekday_last::operator+=(const years& y) NOEXCEPT { + *this = *this + y; + return *this; } CONSTCD14 -inline -year_month_weekday_last& -year_month_weekday_last::operator-=(const years& y) NOEXCEPT -{ - *this = *this - y; - return *this; +inline year_month_weekday_last& year_month_weekday_last::operator-=(const years& y) NOEXCEPT { + *this = *this - y; + return *this; } -CONSTCD11 inline year year_month_weekday_last::year() const NOEXCEPT {return y_;} -CONSTCD11 inline month year_month_weekday_last::month() const NOEXCEPT {return m_;} +CONSTCD11 inline year year_month_weekday_last::year() const NOEXCEPT { + return y_; +} +CONSTCD11 inline month year_month_weekday_last::month() const NOEXCEPT { + return m_; +} CONSTCD11 -inline -weekday -year_month_weekday_last::weekday() const NOEXCEPT -{ - return wdl_.weekday(); +inline weekday year_month_weekday_last::weekday() const NOEXCEPT { + return wdl_.weekday(); } CONSTCD11 -inline -weekday_last -year_month_weekday_last::weekday_last() const NOEXCEPT -{ - return wdl_; +inline weekday_last year_month_weekday_last::weekday_last() const NOEXCEPT { + return wdl_; } CONSTCD14 -inline -year_month_weekday_last::operator sys_days() const NOEXCEPT -{ - return sys_days{to_days()}; +inline year_month_weekday_last::operator sys_days() const NOEXCEPT { + return sys_days{to_days()}; } CONSTCD14 -inline -year_month_weekday_last::operator local_days() const NOEXCEPT -{ - return local_days{to_days()}; +inline year_month_weekday_last::operator local_days() const NOEXCEPT { + return local_days{to_days()}; } CONSTCD11 -inline -bool -year_month_weekday_last::ok() const NOEXCEPT -{ - return y_.ok() && m_.ok() && wdl_.ok(); +inline bool year_month_weekday_last::ok() const NOEXCEPT { + return y_.ok() && m_.ok() && wdl_.ok(); } CONSTCD14 -inline -days -year_month_weekday_last::to_days() const NOEXCEPT -{ - auto const d = sys_days(y_/m_/last); - return (d - (date::weekday{d} - wdl_.weekday())).time_since_epoch(); +inline days year_month_weekday_last::to_days() const NOEXCEPT { + auto const d = sys_days(y_ / m_ / last); + return (d - (date::weekday{d} - wdl_.weekday())).time_since_epoch(); } CONSTCD11 -inline -bool -operator==(const year_month_weekday_last& x, const year_month_weekday_last& y) NOEXCEPT -{ - return x.year() == y.year() && x.month() == y.month() && - x.weekday_last() == y.weekday_last(); +inline bool operator==(const year_month_weekday_last& x, const year_month_weekday_last& y) NOEXCEPT { + return x.year() == y.year() && x.month() == y.month() && x.weekday_last() == y.weekday_last(); } CONSTCD11 -inline -bool -operator!=(const year_month_weekday_last& x, const year_month_weekday_last& y) NOEXCEPT -{ - return !(x == y); +inline bool operator!=(const year_month_weekday_last& x, const year_month_weekday_last& y) NOEXCEPT { + return !(x == y); } -template -inline -std::basic_ostream& -operator<<(std::basic_ostream& os, const year_month_weekday_last& ymwdl) -{ - return os << ymwdl.year() << '/' << ymwdl.month() << '/' << ymwdl.weekday_last(); +template +inline std::basic_ostream& operator<<(std::basic_ostream& os, + const year_month_weekday_last& ymwdl) { + return os << ymwdl.year() << '/' << ymwdl.month() << '/' << ymwdl.weekday_last(); } -template -CONSTCD14 -inline -year_month_weekday_last -operator+(const year_month_weekday_last& ymwdl, const months& dm) NOEXCEPT -{ - return (ymwdl.year() / ymwdl.month() + dm) / ymwdl.weekday_last(); +template +CONSTCD14 inline year_month_weekday_last operator+(const year_month_weekday_last& ymwdl, const months& dm) NOEXCEPT { + return (ymwdl.year() / ymwdl.month() + dm) / ymwdl.weekday_last(); } -template -CONSTCD14 -inline -year_month_weekday_last -operator+(const months& dm, const year_month_weekday_last& ymwdl) NOEXCEPT -{ - return ymwdl + dm; +template +CONSTCD14 inline year_month_weekday_last operator+(const months& dm, const year_month_weekday_last& ymwdl) NOEXCEPT { + return ymwdl + dm; } -template -CONSTCD14 -inline -year_month_weekday_last -operator-(const year_month_weekday_last& ymwdl, const months& dm) NOEXCEPT -{ - return ymwdl + (-dm); +template +CONSTCD14 inline year_month_weekday_last operator-(const year_month_weekday_last& ymwdl, const months& dm) NOEXCEPT { + return ymwdl + (-dm); } CONSTCD11 -inline -year_month_weekday_last -operator+(const year_month_weekday_last& ymwdl, const years& dy) NOEXCEPT -{ - return {ymwdl.year()+dy, ymwdl.month(), ymwdl.weekday_last()}; +inline year_month_weekday_last operator+(const year_month_weekday_last& ymwdl, const years& dy) NOEXCEPT { + return {ymwdl.year() + dy, ymwdl.month(), ymwdl.weekday_last()}; } CONSTCD11 -inline -year_month_weekday_last -operator+(const years& dy, const year_month_weekday_last& ymwdl) NOEXCEPT -{ - return ymwdl + dy; +inline year_month_weekday_last operator+(const years& dy, const year_month_weekday_last& ymwdl) NOEXCEPT { + return ymwdl + dy; } CONSTCD11 -inline -year_month_weekday_last -operator-(const year_month_weekday_last& ymwdl, const years& dy) NOEXCEPT -{ - return ymwdl + (-dy); +inline year_month_weekday_last operator-(const year_month_weekday_last& ymwdl, const years& dy) NOEXCEPT { + return ymwdl + (-dy); } // year_month from operator/() CONSTCD11 -inline -year_month -operator/(const year& y, const month& m) NOEXCEPT -{ - return {y, m}; +inline year_month operator/(const year& y, const month& m) NOEXCEPT { + return {y, m}; } CONSTCD11 -inline -year_month -operator/(const year& y, int m) NOEXCEPT -{ - return y / month(static_cast(m)); +inline year_month operator/(const year& y, int m) NOEXCEPT { + return y / month(static_cast(m)); } // month_day from operator/() CONSTCD11 -inline -month_day -operator/(const month& m, const day& d) NOEXCEPT -{ - return {m, d}; +inline month_day operator/(const month& m, const day& d) NOEXCEPT { + return {m, d}; } CONSTCD11 -inline -month_day -operator/(const day& d, const month& m) NOEXCEPT -{ - return m / d; +inline month_day operator/(const day& d, const month& m) NOEXCEPT { + return m / d; } CONSTCD11 -inline -month_day -operator/(const month& m, int d) NOEXCEPT -{ - return m / day(static_cast(d)); +inline month_day operator/(const month& m, int d) NOEXCEPT { + return m / day(static_cast(d)); } CONSTCD11 -inline -month_day -operator/(int m, const day& d) NOEXCEPT -{ - return month(static_cast(m)) / d; +inline month_day operator/(int m, const day& d) NOEXCEPT { + return month(static_cast(m)) / d; } -CONSTCD11 inline month_day operator/(const day& d, int m) NOEXCEPT {return m / d;} +CONSTCD11 inline month_day operator/(const day& d, int m) NOEXCEPT { + return m / d; +} // month_day_last from operator/() CONSTCD11 -inline -month_day_last -operator/(const month& m, last_spec) NOEXCEPT -{ - return month_day_last{m}; +inline month_day_last operator/(const month& m, last_spec) NOEXCEPT { + return month_day_last{m}; } CONSTCD11 -inline -month_day_last -operator/(last_spec, const month& m) NOEXCEPT -{ - return m/last; +inline month_day_last operator/(last_spec, const month& m) NOEXCEPT { + return m / last; } CONSTCD11 -inline -month_day_last -operator/(int m, last_spec) NOEXCEPT -{ - return month(static_cast(m))/last; +inline month_day_last operator/(int m, last_spec) NOEXCEPT { + return month(static_cast(m)) / last; } CONSTCD11 -inline -month_day_last -operator/(last_spec, int m) NOEXCEPT -{ - return m/last; +inline month_day_last operator/(last_spec, int m) NOEXCEPT { + return m / last; } // month_weekday from operator/() CONSTCD11 -inline -month_weekday -operator/(const month& m, const weekday_indexed& wdi) NOEXCEPT -{ - return {m, wdi}; +inline month_weekday operator/(const month& m, const weekday_indexed& wdi) NOEXCEPT { + return {m, wdi}; } CONSTCD11 -inline -month_weekday -operator/(const weekday_indexed& wdi, const month& m) NOEXCEPT -{ - return m / wdi; +inline month_weekday operator/(const weekday_indexed& wdi, const month& m) NOEXCEPT { + return m / wdi; } CONSTCD11 -inline -month_weekday -operator/(int m, const weekday_indexed& wdi) NOEXCEPT -{ - return month(static_cast(m)) / wdi; +inline month_weekday operator/(int m, const weekday_indexed& wdi) NOEXCEPT { + return month(static_cast(m)) / wdi; } CONSTCD11 -inline -month_weekday -operator/(const weekday_indexed& wdi, int m) NOEXCEPT -{ - return m / wdi; +inline month_weekday operator/(const weekday_indexed& wdi, int m) NOEXCEPT { + return m / wdi; } // month_weekday_last from operator/() CONSTCD11 -inline -month_weekday_last -operator/(const month& m, const weekday_last& wdl) NOEXCEPT -{ - return {m, wdl}; +inline month_weekday_last operator/(const month& m, const weekday_last& wdl) NOEXCEPT { + return {m, wdl}; } CONSTCD11 -inline -month_weekday_last -operator/(const weekday_last& wdl, const month& m) NOEXCEPT -{ - return m / wdl; +inline month_weekday_last operator/(const weekday_last& wdl, const month& m) NOEXCEPT { + return m / wdl; } CONSTCD11 -inline -month_weekday_last -operator/(int m, const weekday_last& wdl) NOEXCEPT -{ - return month(static_cast(m)) / wdl; +inline month_weekday_last operator/(int m, const weekday_last& wdl) NOEXCEPT { + return month(static_cast(m)) / wdl; } CONSTCD11 -inline -month_weekday_last -operator/(const weekday_last& wdl, int m) NOEXCEPT -{ - return m / wdl; +inline month_weekday_last operator/(const weekday_last& wdl, int m) NOEXCEPT { + return m / wdl; } // year_month_day from operator/() CONSTCD11 -inline -year_month_day -operator/(const year_month& ym, const day& d) NOEXCEPT -{ - return {ym.year(), ym.month(), d}; +inline year_month_day operator/(const year_month& ym, const day& d) NOEXCEPT { + return {ym.year(), ym.month(), d}; } CONSTCD11 -inline -year_month_day -operator/(const year_month& ym, int d) NOEXCEPT -{ - return ym / day(static_cast(d)); +inline year_month_day operator/(const year_month& ym, int d) NOEXCEPT { + return ym / day(static_cast(d)); } CONSTCD11 -inline -year_month_day -operator/(const year& y, const month_day& md) NOEXCEPT -{ - return y / md.month() / md.day(); +inline year_month_day operator/(const year& y, const month_day& md) NOEXCEPT { + return y / md.month() / md.day(); } CONSTCD11 -inline -year_month_day -operator/(int y, const month_day& md) NOEXCEPT -{ - return year(y) / md; +inline year_month_day operator/(int y, const month_day& md) NOEXCEPT { + return year(y) / md; } CONSTCD11 -inline -year_month_day -operator/(const month_day& md, const year& y) NOEXCEPT -{ - return y / md; +inline year_month_day operator/(const month_day& md, const year& y) NOEXCEPT { + return y / md; } CONSTCD11 -inline -year_month_day -operator/(const month_day& md, int y) NOEXCEPT -{ - return year(y) / md; +inline year_month_day operator/(const month_day& md, int y) NOEXCEPT { + return year(y) / md; } // year_month_day_last from operator/() CONSTCD11 -inline -year_month_day_last -operator/(const year_month& ym, last_spec) NOEXCEPT -{ - return {ym.year(), month_day_last{ym.month()}}; +inline year_month_day_last operator/(const year_month& ym, last_spec) NOEXCEPT { + return {ym.year(), month_day_last{ym.month()}}; } CONSTCD11 -inline -year_month_day_last -operator/(const year& y, const month_day_last& mdl) NOEXCEPT -{ - return {y, mdl}; +inline year_month_day_last operator/(const year& y, const month_day_last& mdl) NOEXCEPT { + return {y, mdl}; } CONSTCD11 -inline -year_month_day_last -operator/(int y, const month_day_last& mdl) NOEXCEPT -{ - return year(y) / mdl; +inline year_month_day_last operator/(int y, const month_day_last& mdl) NOEXCEPT { + return year(y) / mdl; } CONSTCD11 -inline -year_month_day_last -operator/(const month_day_last& mdl, const year& y) NOEXCEPT -{ - return y / mdl; +inline year_month_day_last operator/(const month_day_last& mdl, const year& y) NOEXCEPT { + return y / mdl; } CONSTCD11 -inline -year_month_day_last -operator/(const month_day_last& mdl, int y) NOEXCEPT -{ - return year(y) / mdl; +inline year_month_day_last operator/(const month_day_last& mdl, int y) NOEXCEPT { + return year(y) / mdl; } // year_month_weekday from operator/() CONSTCD11 -inline -year_month_weekday -operator/(const year_month& ym, const weekday_indexed& wdi) NOEXCEPT -{ - return {ym.year(), ym.month(), wdi}; +inline year_month_weekday operator/(const year_month& ym, const weekday_indexed& wdi) NOEXCEPT { + return {ym.year(), ym.month(), wdi}; } CONSTCD11 -inline -year_month_weekday -operator/(const year& y, const month_weekday& mwd) NOEXCEPT -{ - return {y, mwd.month(), mwd.weekday_indexed()}; +inline year_month_weekday operator/(const year& y, const month_weekday& mwd) NOEXCEPT { + return {y, mwd.month(), mwd.weekday_indexed()}; } CONSTCD11 -inline -year_month_weekday -operator/(int y, const month_weekday& mwd) NOEXCEPT -{ - return year(y) / mwd; +inline year_month_weekday operator/(int y, const month_weekday& mwd) NOEXCEPT { + return year(y) / mwd; } CONSTCD11 -inline -year_month_weekday -operator/(const month_weekday& mwd, const year& y) NOEXCEPT -{ - return y / mwd; +inline year_month_weekday operator/(const month_weekday& mwd, const year& y) NOEXCEPT { + return y / mwd; } CONSTCD11 -inline -year_month_weekday -operator/(const month_weekday& mwd, int y) NOEXCEPT -{ - return year(y) / mwd; +inline year_month_weekday operator/(const month_weekday& mwd, int y) NOEXCEPT { + return year(y) / mwd; } // year_month_weekday_last from operator/() CONSTCD11 -inline -year_month_weekday_last -operator/(const year_month& ym, const weekday_last& wdl) NOEXCEPT -{ - return {ym.year(), ym.month(), wdl}; +inline year_month_weekday_last operator/(const year_month& ym, const weekday_last& wdl) NOEXCEPT { + return {ym.year(), ym.month(), wdl}; } CONSTCD11 -inline -year_month_weekday_last -operator/(const year& y, const month_weekday_last& mwdl) NOEXCEPT -{ - return {y, mwdl.month(), mwdl.weekday_last()}; +inline year_month_weekday_last operator/(const year& y, const month_weekday_last& mwdl) NOEXCEPT { + return {y, mwdl.month(), mwdl.weekday_last()}; } CONSTCD11 -inline -year_month_weekday_last -operator/(int y, const month_weekday_last& mwdl) NOEXCEPT -{ - return year(y) / mwdl; +inline year_month_weekday_last operator/(int y, const month_weekday_last& mwdl) NOEXCEPT { + return year(y) / mwdl; } CONSTCD11 -inline -year_month_weekday_last -operator/(const month_weekday_last& mwdl, const year& y) NOEXCEPT -{ - return y / mwdl; +inline year_month_weekday_last operator/(const month_weekday_last& mwdl, const year& y) NOEXCEPT { + return y / mwdl; } CONSTCD11 -inline -year_month_weekday_last -operator/(const month_weekday_last& mwdl, int y) NOEXCEPT -{ - return year(y) / mwdl; +inline year_month_weekday_last operator/(const month_weekday_last& mwdl, int y) NOEXCEPT { + return year(y) / mwdl; } template struct fields; template -std::basic_ostream& -to_stream(std::basic_ostream& os, const CharT* fmt, - const fields& fds, const std::string* abbrev = nullptr, - const std::chrono::seconds* offset_sec = nullptr); +std::basic_ostream& to_stream(std::basic_ostream& os, const CharT* fmt, + const fields& fds, const std::string* abbrev = nullptr, + const std::chrono::seconds* offset_sec = nullptr); template -std::basic_istream& -from_stream(std::basic_istream& is, const CharT* fmt, - fields& fds, std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr); +std::basic_istream& from_stream(std::basic_istream& is, const CharT* fmt, + fields& fds, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr); // hh_mm_ss -namespace detail -{ +namespace detail { -struct undocumented {explicit undocumented() = default;}; +struct undocumented { + explicit undocumented() = default; +}; // width::value is the number of fractional decimal digits in 1/n // width<0>::value and width<1>::value are defined to be 0 @@ -3724,863 +2895,601 @@ struct undocumented {explicit undocumented() = default;}; // Example: width<4>::value == 2 // Example: width<10>::value == 1 // Example: width<1000>::value == 3 -template -struct width -{ - static_assert(d > 0, "width called with zero denominator"); - static CONSTDATA unsigned value = 1 + width::value; +template +struct width { + static_assert(d > 0, "width called with zero denominator"); + static CONSTDATA unsigned value = 1 + width::value; }; template -struct width -{ - static CONSTDATA unsigned value = 0; +struct width { + static CONSTDATA unsigned value = 0; }; template -struct static_pow10 -{ -private: - static CONSTDATA std::uint64_t h = static_pow10::value; -public: - static CONSTDATA std::uint64_t value = h * h * (exp % 2 ? 10 : 1); +struct static_pow10 { + private: + static CONSTDATA std::uint64_t h = static_pow10::value; + + public: + static CONSTDATA std::uint64_t value = h * h * (exp % 2 ? 10 : 1); }; template <> -struct static_pow10<0> -{ - static CONSTDATA std::uint64_t value = 1; +struct static_pow10<0> { + static CONSTDATA std::uint64_t value = 1; }; template -class decimal_format_seconds -{ - using CT = typename std::common_type::type; - using rep = typename CT::rep; - static unsigned CONSTDATA trial_width = - detail::width::value; -public: - static unsigned CONSTDATA width = trial_width < 19 ? trial_width : 6u; - using precision = std::chrono::duration::value>>; - -private: - std::chrono::seconds s_; - precision sub_s_; - -public: - CONSTCD11 decimal_format_seconds() - : s_() - , sub_s_() - {} - - CONSTCD11 explicit decimal_format_seconds(const Duration& d) NOEXCEPT - : s_(std::chrono::duration_cast(d)) - , sub_s_(std::chrono::duration_cast(d - s_)) - {} - - CONSTCD14 std::chrono::seconds& seconds() NOEXCEPT {return s_;} - CONSTCD11 std::chrono::seconds seconds() const NOEXCEPT {return s_;} - CONSTCD11 precision subseconds() const NOEXCEPT {return sub_s_;} - - CONSTCD14 precision to_duration() const NOEXCEPT - { - return s_ + sub_s_; - } - - CONSTCD11 bool in_conventional_range() const NOEXCEPT - { - return sub_s_ < std::chrono::seconds{1} && s_ < std::chrono::minutes{1}; - } - - template - friend - std::basic_ostream& - operator<<(std::basic_ostream& os, const decimal_format_seconds& x) - { - return x.print(os, std::chrono::treat_as_floating_point{}); - } - - template - std::basic_ostream& - print(std::basic_ostream& os, std::true_type) const - { - date::detail::save_ostream _(os); - std::chrono::duration d = s_ + sub_s_; - if (d < std::chrono::seconds{10}) - os << '0'; - os << std::fixed << d.count(); - return os; - } +class decimal_format_seconds { + using CT = typename std::common_type::type; + using rep = typename CT::rep; + static unsigned CONSTDATA trial_width = detail::width::value; + + public: + static unsigned CONSTDATA width = trial_width < 19 ? trial_width : 6u; + using precision = std::chrono::duration::value>>; + + private: + std::chrono::seconds s_; + precision sub_s_; + + public: + CONSTCD11 decimal_format_seconds() : s_(), sub_s_() { + } + + CONSTCD11 explicit decimal_format_seconds(const Duration& d) NOEXCEPT + : s_(std::chrono::duration_cast(d)), + sub_s_(std::chrono::duration_cast(d - s_)) { + } + + CONSTCD14 std::chrono::seconds& seconds() NOEXCEPT { + return s_; + } + CONSTCD11 std::chrono::seconds seconds() const NOEXCEPT { + return s_; + } + CONSTCD11 precision subseconds() const NOEXCEPT { + return sub_s_; + } + + CONSTCD14 precision to_duration() const NOEXCEPT { + return s_ + sub_s_; + } + + CONSTCD11 bool in_conventional_range() const NOEXCEPT { + return sub_s_ < std::chrono::seconds{1} && s_ < std::chrono::minutes{1}; + } + + template + friend std::basic_ostream& operator<<(std::basic_ostream& os, + const decimal_format_seconds& x) { + return x.print(os, std::chrono::treat_as_floating_point{}); + } + + template + std::basic_ostream& print(std::basic_ostream& os, std::true_type) const { + date::detail::save_ostream _(os); + std::chrono::duration d = s_ + sub_s_; + if (d < std::chrono::seconds{10}) + os << '0'; + os << std::fixed << d.count(); + return os; + } - template - std::basic_ostream& - print(std::basic_ostream& os, std::false_type) const - { - date::detail::save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - os.width(2); - os << s_.count(); - if (width > 0) - { + template + std::basic_ostream& print(std::basic_ostream& os, std::false_type) const { + date::detail::save_ostream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(2); + os << s_.count(); + if (width > 0) { #if !ONLY_C_LOCALE - os << std::use_facet>(os.getloc()).decimal_point(); + os << std::use_facet>(os.getloc()).decimal_point(); #else - os << '.'; + os << '.'; #endif - date::detail::save_ostream _s(os); - os.imbue(std::locale::classic()); - os.width(width); - os << sub_s_.count(); - } - return os; + date::detail::save_ostream _s(os); + os.imbue(std::locale::classic()); + os.width(width); + os << sub_s_.count(); } + return os; + } }; template -inline -CONSTCD11 -typename std::enable_if - < - std::numeric_limits::is_signed, - std::chrono::duration - >::type -abs(std::chrono::duration d) -{ - return d >= d.zero() ? +d : -d; +inline CONSTCD11 typename std::enable_if::is_signed, std::chrono::duration>::type +abs(std::chrono::duration d) { + return d >= d.zero() ? +d : -d; } template -inline -CONSTCD11 -typename std::enable_if - < - !std::numeric_limits::is_signed, - std::chrono::duration - >::type -abs(std::chrono::duration d) -{ - return d; +inline CONSTCD11 typename std::enable_if::is_signed, std::chrono::duration>::type +abs(std::chrono::duration d) { + return d; } } // namespace detail template -class hh_mm_ss -{ - using dfs = detail::decimal_format_seconds::type>; - - std::chrono::hours h_; - std::chrono::minutes m_; - dfs s_; - bool neg_; - -public: - static unsigned CONSTDATA fractional_width = dfs::width; - using precision = typename dfs::precision; - - CONSTCD11 hh_mm_ss() NOEXCEPT - : hh_mm_ss(Duration::zero()) - {} - - CONSTCD11 explicit hh_mm_ss(Duration d) NOEXCEPT - : h_(std::chrono::duration_cast(detail::abs(d))) - , m_(std::chrono::duration_cast(detail::abs(d)) - h_) - , s_(detail::abs(d) - h_ - m_) - , neg_(d < Duration::zero()) - {} - - CONSTCD11 std::chrono::hours hours() const NOEXCEPT {return h_;} - CONSTCD11 std::chrono::minutes minutes() const NOEXCEPT {return m_;} - CONSTCD11 std::chrono::seconds seconds() const NOEXCEPT {return s_.seconds();} - CONSTCD14 std::chrono::seconds& - seconds(detail::undocumented) NOEXCEPT {return s_.seconds();} - CONSTCD11 precision subseconds() const NOEXCEPT {return s_.subseconds();} - CONSTCD11 bool is_negative() const NOEXCEPT {return neg_;} - - CONSTCD11 explicit operator precision() const NOEXCEPT {return to_duration();} - CONSTCD11 precision to_duration() const NOEXCEPT - {return (s_.to_duration() + m_ + h_) * (1-2*neg_);} - - CONSTCD11 bool in_conventional_range() const NOEXCEPT - { - return !neg_ && h_ < days{1} && m_ < std::chrono::hours{1} && - s_.in_conventional_range(); - } - -private: - - template - friend - std::basic_ostream& - operator<<(std::basic_ostream& os, hh_mm_ss const& tod) - { - if (tod.is_negative()) - os << '-'; - if (tod.h_ < std::chrono::hours{10}) - os << '0'; - os << tod.h_.count() << ':'; - if (tod.m_ < std::chrono::minutes{10}) - os << '0'; - os << tod.m_.count() << ':' << tod.s_; - return os; - } - - template - friend - std::basic_ostream& - date::to_stream(std::basic_ostream& os, const CharT* fmt, - const fields& fds, const std::string* abbrev, - const std::chrono::seconds* offset_sec); - - template - friend - std::basic_istream& - date::from_stream(std::basic_istream& is, const CharT* fmt, - fields& fds, - std::basic_string* abbrev, std::chrono::minutes* offset); +class hh_mm_ss { + using dfs = detail::decimal_format_seconds::type>; + + std::chrono::hours h_; + std::chrono::minutes m_; + dfs s_; + bool neg_; + + public: + static unsigned CONSTDATA fractional_width = dfs::width; + using precision = typename dfs::precision; + + CONSTCD11 hh_mm_ss() NOEXCEPT : hh_mm_ss(Duration::zero()) { + } + + CONSTCD11 explicit hh_mm_ss(Duration d) NOEXCEPT + : h_(std::chrono::duration_cast(detail::abs(d))), + m_(std::chrono::duration_cast(detail::abs(d)) - h_), + s_(detail::abs(d) - h_ - m_), + neg_(d < Duration::zero()) { + } + + CONSTCD11 std::chrono::hours hours() const NOEXCEPT { + return h_; + } + CONSTCD11 std::chrono::minutes minutes() const NOEXCEPT { + return m_; + } + CONSTCD11 std::chrono::seconds seconds() const NOEXCEPT { + return s_.seconds(); + } + CONSTCD14 std::chrono::seconds& seconds(detail::undocumented) NOEXCEPT { + return s_.seconds(); + } + CONSTCD11 precision subseconds() const NOEXCEPT { + return s_.subseconds(); + } + CONSTCD11 bool is_negative() const NOEXCEPT { + return neg_; + } + + CONSTCD11 explicit operator precision() const NOEXCEPT { + return to_duration(); + } + CONSTCD11 precision to_duration() const NOEXCEPT { + return (s_.to_duration() + m_ + h_) * (1 - 2 * neg_); + } + + CONSTCD11 bool in_conventional_range() const NOEXCEPT { + return !neg_ && h_ < days{1} && m_ < std::chrono::hours{1} && s_.in_conventional_range(); + } + + private: + template + friend std::basic_ostream& operator<<(std::basic_ostream& os, hh_mm_ss const& tod) { + if (tod.is_negative()) + os << '-'; + if (tod.h_ < std::chrono::hours{10}) + os << '0'; + os << tod.h_.count() << ':'; + if (tod.m_ < std::chrono::minutes{10}) + os << '0'; + os << tod.m_.count() << ':' << tod.s_; + return os; + } + + template + friend std::basic_ostream& date::to_stream(std::basic_ostream& os, const CharT* fmt, + const fields& fds, const std::string* abbrev, + const std::chrono::seconds* offset_sec); + + template + friend std::basic_istream& date::from_stream(std::basic_istream& is, const CharT* fmt, + fields& fds, + std::basic_string* abbrev, + std::chrono::minutes* offset); }; -inline -CONSTCD14 -bool -is_am(std::chrono::hours const& h) NOEXCEPT -{ - using std::chrono::hours; - return hours{0} <= h && h < hours{12}; +inline CONSTCD14 bool is_am(std::chrono::hours const& h) NOEXCEPT { + using std::chrono::hours; + return hours{0} <= h && h < hours{12}; } -inline -CONSTCD14 -bool -is_pm(std::chrono::hours const& h) NOEXCEPT -{ - using std::chrono::hours; - return hours{12} <= h && h < hours{24}; +inline CONSTCD14 bool is_pm(std::chrono::hours const& h) NOEXCEPT { + using std::chrono::hours; + return hours{12} <= h && h < hours{24}; } -inline -CONSTCD14 -std::chrono::hours -make12(std::chrono::hours h) NOEXCEPT -{ - using std::chrono::hours; - if (h < hours{12}) - { - if (h == hours{0}) - h = hours{12}; - } - else - { - if (h != hours{12}) - h = h - hours{12}; - } - return h; +inline CONSTCD14 std::chrono::hours make12(std::chrono::hours h) NOEXCEPT { + using std::chrono::hours; + if (h < hours{12}) { + if (h == hours{0}) + h = hours{12}; + } else { + if (h != hours{12}) + h = h - hours{12}; + } + return h; } -inline -CONSTCD14 -std::chrono::hours -make24(std::chrono::hours h, bool is_pm) NOEXCEPT -{ - using std::chrono::hours; - if (is_pm) - { - if (h != hours{12}) - h = h + hours{12}; - } - else if (h == hours{12}) - h = hours{0}; - return h; +inline CONSTCD14 std::chrono::hours make24(std::chrono::hours h, bool is_pm) NOEXCEPT { + using std::chrono::hours; + if (is_pm) { + if (h != hours{12}) + h = h + hours{12}; + } else if (h == hours{12}) + h = hours{0}; + return h; } template using time_of_day = hh_mm_ss; template ::value>::type> -CONSTCD11 -inline -hh_mm_ss> -make_time(const std::chrono::duration& d) -{ - return hh_mm_ss>(d); + class = typename std::enable_if::value>::type> +CONSTCD11 inline hh_mm_ss> make_time(const std::chrono::duration& d) { + return hh_mm_ss>(d); } template -inline -typename std::enable_if -< - !std::chrono::treat_as_floating_point::value && - std::ratio_less::value - , std::basic_ostream& ->::type -operator<<(std::basic_ostream& os, const sys_time& tp) -{ - auto const dp = date::floor(tp); - return os << year_month_day(dp) << ' ' << make_time(tp-dp); +inline typename std::enable_if::value && + std::ratio_less::value, + std::basic_ostream&>::type +operator<<(std::basic_ostream& os, const sys_time& tp) { + auto const dp = date::floor(tp); + return os << year_month_day(dp) << ' ' << make_time(tp - dp); } template -inline -std::basic_ostream& -operator<<(std::basic_ostream& os, const sys_days& dp) -{ - return os << year_month_day(dp); +inline std::basic_ostream& operator<<(std::basic_ostream& os, const sys_days& dp) { + return os << year_month_day(dp); } template -inline -std::basic_ostream& -operator<<(std::basic_ostream& os, const local_time& ut) -{ - return (os << sys_time{ut.time_since_epoch()}); +inline std::basic_ostream& operator<<(std::basic_ostream& os, + const local_time& ut) { + return (os << sys_time{ut.time_since_epoch()}); } -namespace detail -{ +namespace detail { template class string_literal; template -inline -CONSTCD14 -string_literal::type, - N1 + N2 - 1> -operator+(const string_literal& x, const string_literal& y) NOEXCEPT; +inline CONSTCD14 + string_literal::type, N1 + N2 - 1> + operator+(const string_literal& x, const string_literal& y) NOEXCEPT; template -class string_literal -{ - CharT p_[N]; - - CONSTCD11 string_literal() NOEXCEPT - : p_{} - {} - -public: - using const_iterator = const CharT*; - - string_literal(string_literal const&) = default; - string_literal& operator=(string_literal const&) = delete; - - template ::type> - CONSTCD11 string_literal(CharT c) NOEXCEPT - : p_{c} - { - } - - template ::type> - CONSTCD11 string_literal(CharT c1, CharT c2) NOEXCEPT - : p_{c1, c2} - { - } - - template ::type> - CONSTCD11 string_literal(CharT c1, CharT c2, CharT c3) NOEXCEPT - : p_{c1, c2, c3} - { - } - - CONSTCD14 string_literal(const CharT(&a)[N]) NOEXCEPT - : p_{} - { - for (std::size_t i = 0; i < N; ++i) - p_[i] = a[i]; - } - - template ::type> - CONSTCD14 string_literal(const char(&a)[N]) NOEXCEPT - : p_{} - { - for (std::size_t i = 0; i < N; ++i) - p_[i] = a[i]; - } - - template ::value>::type> - CONSTCD14 string_literal(string_literal const& a) NOEXCEPT - : p_{} - { - for (std::size_t i = 0; i < N; ++i) - p_[i] = a[i]; - } - - CONSTCD11 const CharT* data() const NOEXCEPT {return p_;} - CONSTCD11 std::size_t size() const NOEXCEPT {return N-1;} - - CONSTCD11 const_iterator begin() const NOEXCEPT {return p_;} - CONSTCD11 const_iterator end() const NOEXCEPT {return p_ + N-1;} - - CONSTCD11 CharT const& operator[](std::size_t n) const NOEXCEPT - { - return p_[n]; - } - - template - friend - std::basic_ostream& - operator<<(std::basic_ostream& os, const string_literal& s) - { - return os << s.p_; - } - - template - friend - CONSTCD14 - string_literal::type, - N1 + N2 - 1> - operator+(const string_literal& x, const string_literal& y) NOEXCEPT; +class string_literal { + CharT p_[N]; + + CONSTCD11 string_literal() NOEXCEPT : p_{} { + } + + public: + using const_iterator = const CharT*; + + string_literal(string_literal const&) = default; + string_literal& operator=(string_literal const&) = delete; + + template ::type> + CONSTCD11 string_literal(CharT c) NOEXCEPT : p_{c} { + } + + template ::type> + CONSTCD11 string_literal(CharT c1, CharT c2) NOEXCEPT : p_{c1, c2} { + } + + template ::type> + CONSTCD11 string_literal(CharT c1, CharT c2, CharT c3) NOEXCEPT : p_{c1, c2, c3} { + } + + CONSTCD14 string_literal(const CharT (&a)[N]) NOEXCEPT : p_{} { + for (std::size_t i = 0; i < N; ++i) + p_[i] = a[i]; + } + + template ::type> + CONSTCD14 string_literal(const char (&a)[N]) NOEXCEPT : p_{} { + for (std::size_t i = 0; i < N; ++i) + p_[i] = a[i]; + } + + template ::value>::type> + CONSTCD14 string_literal(string_literal const& a) NOEXCEPT : p_{} { + for (std::size_t i = 0; i < N; ++i) + p_[i] = a[i]; + } + + CONSTCD11 const CharT* data() const NOEXCEPT { + return p_; + } + CONSTCD11 std::size_t size() const NOEXCEPT { + return N - 1; + } + + CONSTCD11 const_iterator begin() const NOEXCEPT { + return p_; + } + CONSTCD11 const_iterator end() const NOEXCEPT { + return p_ + N - 1; + } + + CONSTCD11 CharT const& operator[](std::size_t n) const NOEXCEPT { + return p_[n]; + } + + template + friend std::basic_ostream& operator<<(std::basic_ostream& os, const string_literal& s) { + return os << s.p_; + } + + template + friend CONSTCD14 + string_literal::type, N1 + N2 - 1> + operator+(const string_literal& x, const string_literal& y) NOEXCEPT; }; template -CONSTCD11 -inline -string_literal -operator+(const string_literal& x, const string_literal& y) NOEXCEPT -{ +CONSTCD11 inline string_literal operator+(const string_literal& x, + const string_literal& y) NOEXCEPT { return string_literal(x[0], y[0]); } template -CONSTCD11 -inline -string_literal -operator+(const string_literal& x, const string_literal& y) NOEXCEPT -{ +CONSTCD11 inline string_literal operator+(const string_literal& x, + const string_literal& y) NOEXCEPT { return string_literal(x[0], x[1], y[0]); } template -CONSTCD14 -inline -string_literal::type, - N1 + N2 - 1> -operator+(const string_literal& x, const string_literal& y) NOEXCEPT -{ - using CT = typename std::conditional::type; +CONSTCD14 inline string_literal::type, + N1 + N2 - 1> +operator+(const string_literal& x, const string_literal& y) NOEXCEPT { + using CT = typename std::conditional::type; - string_literal r; - std::size_t i = 0; - for (; i < N1-1; ++i) - r.p_[i] = CT(x.p_[i]); - for (std::size_t j = 0; j < N2; ++j, ++i) - r.p_[i] = CT(y.p_[j]); + string_literal r; + std::size_t i = 0; + for (; i < N1 - 1; ++i) + r.p_[i] = CT(x.p_[i]); + for (std::size_t j = 0; j < N2; ++j, ++i) + r.p_[i] = CT(y.p_[j]); - return r; + return r; } - template -inline -std::basic_string -operator+(std::basic_string x, const string_literal& y) -{ - x.append(y.data(), y.size()); - return x; +inline std::basic_string operator+(std::basic_string x, + const string_literal& y) { + x.append(y.data(), y.size()); + return x; } -#if __cplusplus >= 201402 && (!defined(__EDG_VERSION__) || __EDG_VERSION__ > 411) \ - && (!defined(__SUNPRO_CC) || __SUNPRO_CC > 0x5150) +#if __cplusplus >= 201402 && (!defined(__EDG_VERSION__) || __EDG_VERSION__ > 411) && \ + (!defined(__SUNPRO_CC) || __SUNPRO_CC > 0x5150) template ::value || - std::is_same::value || - std::is_same::value || - std::is_same::value>> -CONSTCD14 -inline -string_literal -msl(CharT c) NOEXCEPT -{ - return string_literal{c}; + class = std::enable_if_t::value || std::is_same::value || + std::is_same::value || std::is_same::value>> +CONSTCD14 inline string_literal msl(CharT c) NOEXCEPT { + return string_literal{c}; } CONSTCD14 -inline -std::size_t -to_string_len(std::intmax_t i) -{ - std::size_t r = 0; - do - { - i /= 10; - ++r; - } while (i > 0); - return r; +inline std::size_t to_string_len(std::intmax_t i) { + std::size_t r = 0; + do { + i /= 10; + ++r; + } while (i > 0); + return r; } template -CONSTCD14 -inline -std::enable_if_t -< - N < 10, - string_literal -> -msl() NOEXCEPT -{ - return msl(char(N % 10 + '0')); + CONSTCD14 inline std::enable_if_t < N<10, string_literal> msl() NOEXCEPT { + return msl(char(N % 10 + '0')); } template -CONSTCD14 -inline -std::enable_if_t -< - 10 <= N, - string_literal -> -msl() NOEXCEPT -{ - return msl() + msl(char(N % 10 + '0')); +CONSTCD14 inline std::enable_if_t<10 <= N, string_literal> msl() NOEXCEPT { + return msl() + msl(char(N % 10 + '0')); } template -CONSTCD14 -inline -std::enable_if_t -< +CONSTCD14 inline std::enable_if_t< std::ratio::type::den != 1, - string_literal::type::num) + - to_string_len(std::ratio::type::den) + 4> -> -msl(std::ratio) NOEXCEPT -{ - using R = typename std::ratio::type; - return msl(CharT{'['}) + msl() + msl(CharT{'/'}) + - msl() + msl(CharT{']'}); + string_literal::type::num) + to_string_len(std::ratio::type::den) + 4>> +msl(std::ratio) NOEXCEPT { + using R = typename std::ratio::type; + return msl(CharT{'['}) + msl() + msl(CharT{'/'}) + msl() + msl(CharT{']'}); } template -CONSTCD14 -inline -std::enable_if_t -< - std::ratio::type::den == 1, - string_literal::type::num) + 3> -> -msl(std::ratio) NOEXCEPT -{ - using R = typename std::ratio::type; - return msl(CharT{'['}) + msl() + msl(CharT{']'}); +CONSTCD14 inline std::enable_if_t::type::den == 1, + string_literal::type::num) + 3>> +msl(std::ratio) NOEXCEPT { + using R = typename std::ratio::type; + return msl(CharT{'['}) + msl() + msl(CharT{']'}); } - #else // __cplusplus < 201402 || (defined(__EDG_VERSION__) && __EDG_VERSION__ <= 411) -inline -std::string -to_string(std::uint64_t x) -{ - return std::to_string(x); +inline std::string to_string(std::uint64_t x) { + return std::to_string(x); } template -inline -std::basic_string -to_string(std::uint64_t x) -{ - auto y = std::to_string(x); - return std::basic_string(y.begin(), y.end()); +inline std::basic_string to_string(std::uint64_t x) { + auto y = std::to_string(x); + return std::basic_string(y.begin(), y.end()); } template -inline -typename std::enable_if -< - std::ratio::type::den != 1, - std::basic_string ->::type -msl(std::ratio) -{ - using R = typename std::ratio::type; - return std::basic_string(1, '[') + to_string(R::num) + CharT{'/'} + - to_string(R::den) + CharT{']'}; +inline typename std::enable_if::type::den != 1, std::basic_string>::type msl(std::ratio) { + using R = typename std::ratio::type; + return std::basic_string(1, '[') + to_string(R::num) + CharT{'/'} + to_string(R::den) + + CharT{']'}; } template -inline -typename std::enable_if -< - std::ratio::type::den == 1, - std::basic_string ->::type -msl(std::ratio) -{ - using R = typename std::ratio::type; - return std::basic_string(1, '[') + to_string(R::num) + CharT{']'}; +inline typename std::enable_if::type::den == 1, std::basic_string>::type msl(std::ratio) { + using R = typename std::ratio::type; + return std::basic_string(1, '[') + to_string(R::num) + CharT{']'}; } #endif // __cplusplus < 201402 || (defined(__EDG_VERSION__) && __EDG_VERSION__ <= 411) template -CONSTCD11 -inline -string_literal -msl(std::atto) NOEXCEPT -{ - return string_literal{'a'}; +CONSTCD11 inline string_literal msl(std::atto) NOEXCEPT { + return string_literal{'a'}; } template -CONSTCD11 -inline -string_literal -msl(std::femto) NOEXCEPT -{ - return string_literal{'f'}; +CONSTCD11 inline string_literal msl(std::femto) NOEXCEPT { + return string_literal{'f'}; } template -CONSTCD11 -inline -string_literal -msl(std::pico) NOEXCEPT -{ - return string_literal{'p'}; +CONSTCD11 inline string_literal msl(std::pico) NOEXCEPT { + return string_literal{'p'}; } template -CONSTCD11 -inline -string_literal -msl(std::nano) NOEXCEPT -{ - return string_literal{'n'}; +CONSTCD11 inline string_literal msl(std::nano) NOEXCEPT { + return string_literal{'n'}; } template -CONSTCD11 -inline -typename std::enable_if -< - std::is_same::value, - string_literal ->::type -msl(std::micro) NOEXCEPT -{ - return string_literal{'\xC2', '\xB5'}; +CONSTCD11 inline typename std::enable_if::value, string_literal>::type msl( + std::micro) NOEXCEPT { + return string_literal{'\xC2', '\xB5'}; } template -CONSTCD11 -inline -typename std::enable_if -< - !std::is_same::value, - string_literal ->::type -msl(std::micro) NOEXCEPT -{ - return string_literal{CharT{static_cast('\xB5')}}; +CONSTCD11 inline typename std::enable_if::value, string_literal>::type msl( + std::micro) NOEXCEPT { + return string_literal{CharT{static_cast('\xB5')}}; } template -CONSTCD11 -inline -string_literal -msl(std::milli) NOEXCEPT -{ - return string_literal{'m'}; +CONSTCD11 inline string_literal msl(std::milli) NOEXCEPT { + return string_literal{'m'}; } template -CONSTCD11 -inline -string_literal -msl(std::centi) NOEXCEPT -{ - return string_literal{'c'}; +CONSTCD11 inline string_literal msl(std::centi) NOEXCEPT { + return string_literal{'c'}; } template -CONSTCD11 -inline -string_literal -msl(std::deca) NOEXCEPT -{ - return string_literal{'d', 'a'}; +CONSTCD11 inline string_literal msl(std::deca) NOEXCEPT { + return string_literal{'d', 'a'}; } template -CONSTCD11 -inline -string_literal -msl(std::deci) NOEXCEPT -{ - return string_literal{'d'}; +CONSTCD11 inline string_literal msl(std::deci) NOEXCEPT { + return string_literal{'d'}; } template -CONSTCD11 -inline -string_literal -msl(std::hecto) NOEXCEPT -{ - return string_literal{'h'}; +CONSTCD11 inline string_literal msl(std::hecto) NOEXCEPT { + return string_literal{'h'}; } template -CONSTCD11 -inline -string_literal -msl(std::kilo) NOEXCEPT -{ - return string_literal{'k'}; +CONSTCD11 inline string_literal msl(std::kilo) NOEXCEPT { + return string_literal{'k'}; } template -CONSTCD11 -inline -string_literal -msl(std::mega) NOEXCEPT -{ - return string_literal{'M'}; +CONSTCD11 inline string_literal msl(std::mega) NOEXCEPT { + return string_literal{'M'}; } template -CONSTCD11 -inline -string_literal -msl(std::giga) NOEXCEPT -{ - return string_literal{'G'}; +CONSTCD11 inline string_literal msl(std::giga) NOEXCEPT { + return string_literal{'G'}; } template -CONSTCD11 -inline -string_literal -msl(std::tera) NOEXCEPT -{ - return string_literal{'T'}; +CONSTCD11 inline string_literal msl(std::tera) NOEXCEPT { + return string_literal{'T'}; } template -CONSTCD11 -inline -string_literal -msl(std::peta) NOEXCEPT -{ - return string_literal{'P'}; +CONSTCD11 inline string_literal msl(std::peta) NOEXCEPT { + return string_literal{'P'}; } template -CONSTCD11 -inline -string_literal -msl(std::exa) NOEXCEPT -{ - return string_literal{'E'}; +CONSTCD11 inline string_literal msl(std::exa) NOEXCEPT { + return string_literal{'E'}; } template -CONSTCD11 -inline -auto -get_units(Period p) - -> decltype(msl(p) + string_literal{'s'}) -{ - return msl(p) + string_literal{'s'}; +CONSTCD11 inline auto get_units(Period p) -> decltype(msl(p) + string_literal{'s'}) { + return msl(p) + string_literal{'s'}; } template -CONSTCD11 -inline -string_literal -get_units(std::ratio<1>) -{ - return string_literal{'s'}; +CONSTCD11 inline string_literal get_units(std::ratio<1>) { + return string_literal{'s'}; } template -CONSTCD11 -inline -string_literal -get_units(std::ratio<3600>) -{ - return string_literal{'h'}; +CONSTCD11 inline string_literal get_units(std::ratio<3600>) { + return string_literal{'h'}; } template -CONSTCD11 -inline -string_literal -get_units(std::ratio<60>) -{ - return string_literal{'m', 'i', 'n'}; +CONSTCD11 inline string_literal get_units(std::ratio<60>) { + return string_literal{'m', 'i', 'n'}; } template -CONSTCD11 -inline -string_literal -get_units(std::ratio<86400>) -{ - return string_literal{'d'}; +CONSTCD11 inline string_literal get_units(std::ratio<86400>) { + return string_literal{'d'}; } template > struct make_string; template <> -struct make_string -{ - template - static - std::string - from(Rep n) - { - return std::to_string(n); - } +struct make_string { + template + static std::string from(Rep n) { + return std::to_string(n); + } }; template -struct make_string -{ - template - static - std::basic_string - from(Rep n) - { - auto s = std::to_string(n); - return std::basic_string(s.begin(), s.end()); - } +struct make_string { + template + static std::basic_string from(Rep n) { + auto s = std::to_string(n); + return std::basic_string(s.begin(), s.end()); + } }; template <> -struct make_string -{ - template - static - std::wstring - from(Rep n) - { - return std::to_wstring(n); - } +struct make_string { + template + static std::wstring from(Rep n) { + return std::to_wstring(n); + } }; template -struct make_string -{ - template - static - std::basic_string - from(Rep n) - { - auto s = std::to_wstring(n); - return std::basic_string(s.begin(), s.end()); - } +struct make_string { + template + static std::basic_string from(Rep n) { + auto s = std::to_wstring(n); + return std::basic_string(s.begin(), s.end()); + } }; } // namespace detail @@ -4590,254 +3499,174 @@ struct make_string CONSTDATA year nanyear{-32768}; template -struct fields -{ - year_month_day ymd{nanyear/0/0}; - weekday wd{8u}; - hh_mm_ss tod{}; - bool has_tod = false; - - fields() = default; - - fields(year_month_day ymd_) : ymd(ymd_) {} - fields(weekday wd_) : wd(wd_) {} - fields(hh_mm_ss tod_) : tod(tod_), has_tod(true) {} - - fields(year_month_day ymd_, weekday wd_) : ymd(ymd_), wd(wd_) {} - fields(year_month_day ymd_, hh_mm_ss tod_) : ymd(ymd_), tod(tod_), - has_tod(true) {} - - fields(weekday wd_, hh_mm_ss tod_) : wd(wd_), tod(tod_), has_tod(true) {} - - fields(year_month_day ymd_, weekday wd_, hh_mm_ss tod_) - : ymd(ymd_) - , wd(wd_) - , tod(tod_) - , has_tod(true) - {} +struct fields { + year_month_day ymd{nanyear / 0 / 0}; + weekday wd{8u}; + hh_mm_ss tod{}; + bool has_tod = false; + + fields() = default; + + fields(year_month_day ymd_) : ymd(ymd_) { + } + fields(weekday wd_) : wd(wd_) { + } + fields(hh_mm_ss tod_) : tod(tod_), has_tod(true) { + } + + fields(year_month_day ymd_, weekday wd_) : ymd(ymd_), wd(wd_) { + } + fields(year_month_day ymd_, hh_mm_ss tod_) : ymd(ymd_), tod(tod_), has_tod(true) { + } + + fields(weekday wd_, hh_mm_ss tod_) : wd(wd_), tod(tod_), has_tod(true) { + } + + fields(year_month_day ymd_, weekday wd_, hh_mm_ss tod_) : ymd(ymd_), wd(wd_), tod(tod_), has_tod(true) { + } }; -namespace detail -{ +namespace detail { template -unsigned -extract_weekday(std::basic_ostream& os, const fields& fds) -{ - if (!fds.ymd.ok() && !fds.wd.ok()) - { - // fds does not contain a valid weekday - os.setstate(std::ios::failbit); - return 8; - } - weekday wd; - if (fds.ymd.ok()) - { - wd = weekday{sys_days(fds.ymd)}; - if (fds.wd.ok() && wd != fds.wd) - { - // fds.ymd and fds.wd are inconsistent - os.setstate(std::ios::failbit); - return 8; - } +unsigned extract_weekday(std::basic_ostream& os, const fields& fds) { + if (!fds.ymd.ok() && !fds.wd.ok()) { + // fds does not contain a valid weekday + os.setstate(std::ios::failbit); + return 8; + } + weekday wd; + if (fds.ymd.ok()) { + wd = weekday{sys_days(fds.ymd)}; + if (fds.wd.ok() && wd != fds.wd) { + // fds.ymd and fds.wd are inconsistent + os.setstate(std::ios::failbit); + return 8; } - else - wd = fds.wd; - return static_cast((wd - Sunday).count()); + } else + wd = fds.wd; + return static_cast((wd - Sunday).count()); } template -unsigned -extract_month(std::basic_ostream& os, const fields& fds) -{ - if (!fds.ymd.month().ok()) - { - // fds does not contain a valid month - os.setstate(std::ios::failbit); - return 0; - } - return static_cast(fds.ymd.month()); +unsigned extract_month(std::basic_ostream& os, const fields& fds) { + if (!fds.ymd.month().ok()) { + // fds does not contain a valid month + os.setstate(std::ios::failbit); + return 0; + } + return static_cast(fds.ymd.month()); } } // namespace detail #if ONLY_C_LOCALE -namespace detail -{ - -inline -std::pair -weekday_names() -{ - static const std::string nm[] = - { - "Sunday", - "Monday", - "Tuesday", - "Wednesday", - "Thursday", - "Friday", - "Saturday", - "Sun", - "Mon", - "Tue", - "Wed", - "Thu", - "Fri", - "Sat" - }; - return std::make_pair(nm, nm+sizeof(nm)/sizeof(nm[0])); -} - -inline -std::pair -month_names() -{ - static const std::string nm[] = - { - "January", - "February", - "March", - "April", - "May", - "June", - "July", - "August", - "September", - "October", - "November", - "December", - "Jan", - "Feb", - "Mar", - "Apr", - "May", - "Jun", - "Jul", - "Aug", - "Sep", - "Oct", - "Nov", - "Dec" - }; - return std::make_pair(nm, nm+sizeof(nm)/sizeof(nm[0])); -} - -inline -std::pair -ampm_names() -{ - static const std::string nm[] = - { - "AM", - "PM" - }; - return std::make_pair(nm, nm+sizeof(nm)/sizeof(nm[0])); +namespace detail { + +inline std::pair weekday_names() { + static const std::string nm[] = {"Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", + "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}; + return std::make_pair(nm, nm + sizeof(nm) / sizeof(nm[0])); +} + +inline std::pair month_names() { + static const std::string nm[] = {"January", "February", "March", "April", "May", "June", "July", "August", + "September", "October", "November", "December", "Jan", "Feb", "Mar", "Apr", + "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; + return std::make_pair(nm, nm + sizeof(nm) / sizeof(nm[0])); +} + +inline std::pair ampm_names() { + static const std::string nm[] = {"AM", "PM"}; + return std::make_pair(nm, nm + sizeof(nm) / sizeof(nm[0])); } template -FwdIter -scan_keyword(std::basic_istream& is, FwdIter kb, FwdIter ke) -{ - size_t nkw = static_cast(std::distance(kb, ke)); - const unsigned char doesnt_match = '\0'; - const unsigned char might_match = '\1'; - const unsigned char does_match = '\2'; - unsigned char statbuf[100]; - unsigned char* status = statbuf; - std::unique_ptr stat_hold(0, free); - if (nkw > sizeof(statbuf)) - { - status = (unsigned char*)std::malloc(nkw); - if (status == nullptr) - throw std::bad_alloc(); - stat_hold.reset(status); +FwdIter scan_keyword(std::basic_istream& is, FwdIter kb, FwdIter ke) { + size_t nkw = static_cast(std::distance(kb, ke)); + const unsigned char doesnt_match = '\0'; + const unsigned char might_match = '\1'; + const unsigned char does_match = '\2'; + unsigned char statbuf[100]; + unsigned char* status = statbuf; + std::unique_ptr stat_hold(0, free); + if (nkw > sizeof(statbuf)) { + status = (unsigned char*)std::malloc(nkw); + if (status == nullptr) + throw std::bad_alloc(); + stat_hold.reset(status); + } + size_t n_might_match = nkw; // At this point, any keyword might match + size_t n_does_match = 0; // but none of them definitely do + // Initialize all statuses to might_match, except for "" keywords are does_match + unsigned char* st = status; + for (auto ky = kb; ky != ke; ++ky, ++st) { + if (!ky->empty()) + *st = might_match; + else { + *st = does_match; + --n_might_match; + ++n_does_match; } - size_t n_might_match = nkw; // At this point, any keyword might match - size_t n_does_match = 0; // but none of them definitely do - // Initialize all statuses to might_match, except for "" keywords are does_match - unsigned char* st = status; - for (auto ky = kb; ky != ke; ++ky, ++st) - { - if (!ky->empty()) - *st = might_match; - else - { + } + // While there might be a match, test keywords against the next CharT + for (size_t indx = 0; is && n_might_match > 0; ++indx) { + // Peek at the next CharT but don't consume it + auto ic = is.peek(); + if (ic == EOF) { + is.setstate(std::ios::eofbit); + break; + } + auto c = static_cast(toupper(ic)); + bool consume = false; + // For each keyword which might match, see if the indx character is c + // If a match if found, consume c + // If a match is found, and that is the last character in the keyword, + // then that keyword matches. + // If the keyword doesn't match this character, then change the keyword + // to doesn't match + st = status; + for (auto ky = kb; ky != ke; ++ky, ++st) { + if (*st == might_match) { + if (c == static_cast(toupper((*ky)[indx]))) { + consume = true; + if (ky->size() == indx + 1) { *st = does_match; --n_might_match; ++n_does_match; + } + } else { + *st = doesnt_match; + --n_might_match; } + } } - // While there might be a match, test keywords against the next CharT - for (size_t indx = 0; is && n_might_match > 0; ++indx) - { - // Peek at the next CharT but don't consume it - auto ic = is.peek(); - if (ic == EOF) - { - is.setstate(std::ios::eofbit); - break; - } - auto c = static_cast(toupper(ic)); - bool consume = false; - // For each keyword which might match, see if the indx character is c - // If a match if found, consume c - // If a match is found, and that is the last character in the keyword, - // then that keyword matches. - // If the keyword doesn't match this character, then change the keyword - // to doesn't match + // consume if we matched a character + if (consume) { + (void)is.get(); + // If we consumed a character and there might be a matched keyword that + // was marked matched on a previous iteration, then such keywords + // are now marked as not matching. + if (n_might_match + n_does_match > 1) { st = status; - for (auto ky = kb; ky != ke; ++ky, ++st) - { - if (*st == might_match) - { - if (c == static_cast(toupper((*ky)[indx]))) - { - consume = true; - if (ky->size() == indx+1) - { - *st = does_match; - --n_might_match; - ++n_does_match; - } - } - else - { - *st = doesnt_match; - --n_might_match; - } - } - } - // consume if we matched a character - if (consume) - { - (void)is.get(); - // If we consumed a character and there might be a matched keyword that - // was marked matched on a previous iteration, then such keywords - // are now marked as not matching. - if (n_might_match + n_does_match > 1) - { - st = status; - for (auto ky = kb; ky != ke; ++ky, ++st) - { - if (*st == does_match && ky->size() != indx+1) - { - *st = doesnt_match; - --n_does_match; - } - } - } + for (auto ky = kb; ky != ke; ++ky, ++st) { + if (*st == does_match && ky->size() != indx + 1) { + *st = doesnt_match; + --n_does_match; + } } + } } - // We've exited the loop because we hit eof and/or we have no more "might matches". - // Return the first matching result - for (st = status; kb != ke; ++kb, ++st) - if (*st == does_match) - break; - if (kb == ke) - is.setstate(std::ios::failbit); - return kb; + } + // We've exited the loop because we hit eof and/or we have no more "might matches". + // Return the first matching result + for (st = status; kb != ke; ++kb, ++st) + if (*st == does_match) + break; + if (kb == ke) + is.setstate(std::ios::failbit); + return kb; } } // namespace detail @@ -4845,3163 +3674,2633 @@ scan_keyword(std::basic_istream& is, FwdIter kb, FwdIter ke) #endif // ONLY_C_LOCALE template -std::basic_ostream& -to_stream(std::basic_ostream& os, const CharT* fmt, - const fields& fds, const std::string* abbrev, - const std::chrono::seconds* offset_sec) -{ +std::basic_ostream& to_stream(std::basic_ostream& os, const CharT* fmt, + const fields& fds, const std::string* abbrev, + const std::chrono::seconds* offset_sec) { #if ONLY_C_LOCALE - using detail::weekday_names; - using detail::month_names; - using detail::ampm_names; + using detail::ampm_names; + using detail::month_names; + using detail::weekday_names; #endif - using detail::save_ostream; - using detail::get_units; - using detail::extract_weekday; - using detail::extract_month; - using std::ios; - using std::chrono::duration_cast; - using std::chrono::seconds; - using std::chrono::minutes; - using std::chrono::hours; - date::detail::save_ostream ss(os); - os.fill(' '); - os.flags(std::ios::skipws | std::ios::dec); - os.width(0); - tm tm{}; - bool insert_negative = fds.has_tod && fds.tod.to_duration() < Duration::zero(); + using detail::extract_month; + using detail::extract_weekday; + using detail::get_units; + using detail::save_ostream; + using std::ios; + using std::chrono::duration_cast; + using std::chrono::hours; + using std::chrono::minutes; + using std::chrono::seconds; + date::detail::save_ostream ss(os); + os.fill(' '); + os.flags(std::ios::skipws | std::ios::dec); + os.width(0); + tm tm{}; + bool insert_negative = fds.has_tod && fds.tod.to_duration() < Duration::zero(); #if !ONLY_C_LOCALE - auto& facet = std::use_facet>(os.getloc()); + auto& facet = std::use_facet>(os.getloc()); #endif - const CharT* command = nullptr; - CharT modified = CharT{}; - for (; *fmt; ++fmt) - { - switch (*fmt) - { - case 'a': - case 'A': - if (command) - { - if (modified == CharT{}) - { - tm.tm_wday = static_cast(extract_weekday(os, fds)); - if (os.fail()) - return os; + const CharT* command = nullptr; + CharT modified = CharT{}; + for (; *fmt; ++fmt) { + switch (*fmt) { + case 'a': + case 'A': + if (command) { + if (modified == CharT{}) { + tm.tm_wday = static_cast(extract_weekday(os, fds)); + if (os.fail()) + return os; #if !ONLY_C_LOCALE - const CharT f[] = {'%', *fmt}; - facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); -#else // ONLY_C_LOCALE - os << weekday_names().first[tm.tm_wday+7*(*fmt == 'a')]; + const CharT f[] = {'%', *fmt}; + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); +#else // ONLY_C_LOCALE + os << weekday_names().first[tm.tm_wday + 7 * (*fmt == 'a')]; #endif // ONLY_C_LOCALE - } - else - { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; - } - else - os << *fmt; - break; - case 'b': - case 'B': - case 'h': - if (command) - { - if (modified == CharT{}) - { - tm.tm_mon = static_cast(extract_month(os, fds)) - 1; + } else { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } else + os << *fmt; + break; + case 'b': + case 'B': + case 'h': + if (command) { + if (modified == CharT{}) { + tm.tm_mon = static_cast(extract_month(os, fds)) - 1; #if !ONLY_C_LOCALE - const CharT f[] = {'%', *fmt}; - facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); -#else // ONLY_C_LOCALE - os << month_names().first[tm.tm_mon+12*(*fmt != 'B')]; + const CharT f[] = {'%', *fmt}; + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); +#else // ONLY_C_LOCALE + os << month_names().first[tm.tm_mon + 12 * (*fmt != 'B')]; #endif // ONLY_C_LOCALE - } - else - { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; - } - else - os << *fmt; - break; - case 'c': - case 'x': - if (command) - { - if (modified == CharT{'O'}) - os << CharT{'%'} << modified << *fmt; - else - { - if (!fds.ymd.ok()) - os.setstate(std::ios::failbit); - if (*fmt == 'c' && !fds.has_tod) - os.setstate(std::ios::failbit); + } else { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } else + os << *fmt; + break; + case 'c': + case 'x': + if (command) { + if (modified == CharT{'O'}) + os << CharT{'%'} << modified << *fmt; + else { + if (!fds.ymd.ok()) + os.setstate(std::ios::failbit); + if (*fmt == 'c' && !fds.has_tod) + os.setstate(std::ios::failbit); #if !ONLY_C_LOCALE - tm = std::tm{}; - auto const& ymd = fds.ymd; - auto ld = local_days(ymd); - if (*fmt == 'c') - { - tm.tm_sec = static_cast(fds.tod.seconds().count()); - tm.tm_min = static_cast(fds.tod.minutes().count()); - tm.tm_hour = static_cast(fds.tod.hours().count()); - } - tm.tm_mday = static_cast(static_cast(ymd.day())); - tm.tm_mon = static_cast(extract_month(os, fds) - 1); - tm.tm_year = static_cast(ymd.year()) - 1900; - tm.tm_wday = static_cast(extract_weekday(os, fds)); - if (os.fail()) - return os; - tm.tm_yday = static_cast((ld - local_days(ymd.year()/1/1)).count()); - CharT f[3] = {'%'}; - auto fe = std::begin(f) + 1; - if (modified == CharT{'E'}) - *fe++ = modified; - *fe++ = *fmt; - facet.put(os, os, os.fill(), &tm, std::begin(f), fe); -#else // ONLY_C_LOCALE - if (*fmt == 'c') - { - auto wd = static_cast(extract_weekday(os, fds)); - os << weekday_names().first[static_cast(wd)+7] - << ' '; - os << month_names().first[extract_month(os, fds)-1+12] << ' '; - auto d = static_cast(static_cast(fds.ymd.day())); - if (d < 10) - os << ' '; - os << d << ' ' - << make_time(duration_cast(fds.tod.to_duration())) - << ' ' << fds.ymd.year(); - - } - else // *fmt == 'x' - { - auto const& ymd = fds.ymd; - save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - os.width(2); - os << static_cast(ymd.month()) << CharT{'/'}; - os.width(2); - os << static_cast(ymd.day()) << CharT{'/'}; - os.width(2); - os << static_cast(ymd.year()) % 100; - } -#endif // ONLY_C_LOCALE - } - command = nullptr; - modified = CharT{}; + tm = std::tm{}; + auto const& ymd = fds.ymd; + auto ld = local_days(ymd); + if (*fmt == 'c') { + tm.tm_sec = static_cast(fds.tod.seconds().count()); + tm.tm_min = static_cast(fds.tod.minutes().count()); + tm.tm_hour = static_cast(fds.tod.hours().count()); } - else - os << *fmt; - break; - case 'C': - if (command) + tm.tm_mday = static_cast(static_cast(ymd.day())); + tm.tm_mon = static_cast(extract_month(os, fds) - 1); + tm.tm_year = static_cast(ymd.year()) - 1900; + tm.tm_wday = static_cast(extract_weekday(os, fds)); + if (os.fail()) + return os; + tm.tm_yday = static_cast((ld - local_days(ymd.year() / 1 / 1)).count()); + CharT f[3] = {'%'}; + auto fe = std::begin(f) + 1; + if (modified == CharT{'E'}) + *fe++ = modified; + *fe++ = *fmt; + facet.put(os, os, os.fill(), &tm, std::begin(f), fe); +#else // ONLY_C_LOCALE + if (*fmt == 'c') { + auto wd = static_cast(extract_weekday(os, fds)); + os << weekday_names().first[static_cast(wd) + 7] << ' '; + os << month_names().first[extract_month(os, fds) - 1 + 12] << ' '; + auto d = static_cast(static_cast(fds.ymd.day())); + if (d < 10) + os << ' '; + os << d << ' ' << make_time(duration_cast(fds.tod.to_duration())) << ' ' << fds.ymd.year(); + + } else // *fmt == 'x' { - if (modified == CharT{'O'}) - os << CharT{'%'} << modified << *fmt; - else - { - if (!fds.ymd.year().ok()) - os.setstate(std::ios::failbit); - auto y = static_cast(fds.ymd.year()); -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#endif - { - save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - if (y >= 0) - { - os.width(2); - os << y/100; - } - else - { - os << CharT{'-'}; - os.width(2); - os << -(y-99)/100; - } - } + auto const& ymd = fds.ymd; + save_ostream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(2); + os << static_cast(ymd.month()) << CharT{'/'}; + os.width(2); + os << static_cast(ymd.day()) << CharT{'/'}; + os.width(2); + os << static_cast(ymd.year()) % 100; + } +#endif // ONLY_C_LOCALE + } + command = nullptr; + modified = CharT{}; + } else + os << *fmt; + break; + case 'C': + if (command) { + if (modified == CharT{'O'}) + os << CharT{'%'} << modified << *fmt; + else { + if (!fds.ymd.year().ok()) + os.setstate(std::ios::failbit); + auto y = static_cast(fds.ymd.year()); #if !ONLY_C_LOCALE - else if (modified == CharT{'E'}) - { - tm.tm_year = y - 1900; - CharT f[3] = {'%', 'E', 'C'}; - facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); - } + if (modified == CharT{}) #endif - } - command = nullptr; - modified = CharT{}; - } - else - os << *fmt; - break; - case 'd': - case 'e': - if (command) { - if (modified == CharT{'E'}) - os << CharT{'%'} << modified << *fmt; - else - { - if (!fds.ymd.day().ok()) - os.setstate(std::ios::failbit); - auto d = static_cast(static_cast(fds.ymd.day())); + save_ostream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + if (y >= 0) { + os.width(2); + os << y / 100; + } else { + os << CharT{'-'}; + os.width(2); + os << -(y - 99) / 100; + } + } #if !ONLY_C_LOCALE - if (modified == CharT{}) + else if (modified == CharT{'E'}) { + tm.tm_year = y - 1900; + CharT f[3] = {'%', 'E', 'C'}; + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } #endif - { - save_ostream _(os); - if (*fmt == CharT{'d'}) - os.fill('0'); - else - os.fill(' '); - os.flags(std::ios::dec | std::ios::right); - os.width(2); - os << d; - } + } + command = nullptr; + modified = CharT{}; + } else + os << *fmt; + break; + case 'd': + case 'e': + if (command) { + if (modified == CharT{'E'}) + os << CharT{'%'} << modified << *fmt; + else { + if (!fds.ymd.day().ok()) + os.setstate(std::ios::failbit); + auto d = static_cast(static_cast(fds.ymd.day())); #if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) - { - tm.tm_mday = d; - CharT f[3] = {'%', 'O', *fmt}; - facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); - } + if (modified == CharT{}) #endif - } - command = nullptr; - modified = CharT{}; - } - else - os << *fmt; - break; - case 'D': - if (command) - { - if (modified == CharT{}) - { - if (!fds.ymd.ok()) - os.setstate(std::ios::failbit); - auto const& ymd = fds.ymd; - save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - os.width(2); - os << static_cast(ymd.month()) << CharT{'/'}; - os.width(2); - os << static_cast(ymd.day()) << CharT{'/'}; - os.width(2); - os << static_cast(ymd.year()) % 100; - } - else - { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; - } - else - os << *fmt; - break; - case 'F': - if (command) - { - if (modified == CharT{}) - { - if (!fds.ymd.ok()) - os.setstate(std::ios::failbit); - auto const& ymd = fds.ymd; - save_ostream _(os); - os.imbue(std::locale::classic()); - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - os.width(4); - os << static_cast(ymd.year()) << CharT{'-'}; - os.width(2); - os << static_cast(ymd.month()) << CharT{'-'}; - os.width(2); - os << static_cast(ymd.day()); - } - else - { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; - } - else - os << *fmt; - break; - case 'g': - case 'G': - if (command) { - if (modified == CharT{}) - { - if (!fds.ymd.ok()) - os.setstate(std::ios::failbit); - auto ld = local_days(fds.ymd); - auto y = year_month_day{ld + days{3}}.year(); - auto start = local_days((y-years{1})/December/Thursday[last]) + - (Monday-Thursday); - if (ld < start) - --y; - if (*fmt == CharT{'G'}) - os << y; - else - { - save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - os.width(2); - os << std::abs(static_cast(y)) % 100; - } - } - else - { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; + save_ostream _(os); + if (*fmt == CharT{'d'}) + os.fill('0'); + else + os.fill(' '); + os.flags(std::ios::dec | std::ios::right); + os.width(2); + os << d; } - else - os << *fmt; - break; - case 'H': - case 'I': - if (command) - { - if (modified == CharT{'E'}) - os << CharT{'%'} << modified << *fmt; - else - { - if (!fds.has_tod) - os.setstate(std::ios::failbit); - if (insert_negative) - { - os << '-'; - insert_negative = false; - } - auto hms = fds.tod; #if !ONLY_C_LOCALE - if (modified == CharT{}) + else if (modified == CharT{'O'}) { + tm.tm_mday = d; + CharT f[3] = {'%', 'O', *fmt}; + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } #endif - { - auto h = *fmt == CharT{'I'} ? date::make12(hms.hours()) : hms.hours(); - if (h < hours{10}) - os << CharT{'0'}; - os << h.count(); - } + } + command = nullptr; + modified = CharT{}; + } else + os << *fmt; + break; + case 'D': + if (command) { + if (modified == CharT{}) { + if (!fds.ymd.ok()) + os.setstate(std::ios::failbit); + auto const& ymd = fds.ymd; + save_ostream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(2); + os << static_cast(ymd.month()) << CharT{'/'}; + os.width(2); + os << static_cast(ymd.day()) << CharT{'/'}; + os.width(2); + os << static_cast(ymd.year()) % 100; + } else { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } else + os << *fmt; + break; + case 'F': + if (command) { + if (modified == CharT{}) { + if (!fds.ymd.ok()) + os.setstate(std::ios::failbit); + auto const& ymd = fds.ymd; + save_ostream _(os); + os.imbue(std::locale::classic()); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(4); + os << static_cast(ymd.year()) << CharT{'-'}; + os.width(2); + os << static_cast(ymd.month()) << CharT{'-'}; + os.width(2); + os << static_cast(ymd.day()); + } else { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } else + os << *fmt; + break; + case 'g': + case 'G': + if (command) { + if (modified == CharT{}) { + if (!fds.ymd.ok()) + os.setstate(std::ios::failbit); + auto ld = local_days(fds.ymd); + auto y = year_month_day{ld + days{3}}.year(); + auto start = local_days((y - years{1}) / December / Thursday[last]) + (Monday - Thursday); + if (ld < start) + --y; + if (*fmt == CharT{'G'}) + os << y; + else { + save_ostream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(2); + os << std::abs(static_cast(y)) % 100; + } + } else { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } else + os << *fmt; + break; + case 'H': + case 'I': + if (command) { + if (modified == CharT{'E'}) + os << CharT{'%'} << modified << *fmt; + else { + if (!fds.has_tod) + os.setstate(std::ios::failbit); + if (insert_negative) { + os << '-'; + insert_negative = false; + } + auto hms = fds.tod; #if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) - { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_hour = static_cast(hms.hours().count()); - facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); - } + if (modified == CharT{}) #endif - } - modified = CharT{}; - command = nullptr; - } - else - os << *fmt; - break; - case 'j': - if (command) { - if (modified == CharT{}) - { - if (fds.ymd.ok() || fds.has_tod) - { - days doy; - if (fds.ymd.ok()) - { - auto ld = local_days(fds.ymd); - auto y = fds.ymd.year(); - doy = ld - local_days(y/January/1) + days{1}; - } - else - { - doy = duration_cast(fds.tod.to_duration()); - } - save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - os.width(3); - os << doy.count(); - } - else - { - os.setstate(std::ios::failbit); - } - } - else - { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; + auto h = *fmt == CharT{'I'} ? date::make12(hms.hours()) : hms.hours(); + if (h < hours{10}) + os << CharT{'0'}; + os << h.count(); } - else - os << *fmt; - break; - case 'm': - if (command) - { - if (modified == CharT{'E'}) - os << CharT{'%'} << modified << *fmt; - else - { - if (!fds.ymd.month().ok()) - os.setstate(std::ios::failbit); - auto m = static_cast(fds.ymd.month()); #if !ONLY_C_LOCALE - if (modified == CharT{}) + else if (modified == CharT{'O'}) { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_hour = static_cast(hms.hours().count()); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } #endif - { - if (m < 10) - os << CharT{'0'}; - os << m; - } + } + modified = CharT{}; + command = nullptr; + } else + os << *fmt; + break; + case 'j': + if (command) { + if (modified == CharT{}) { + if (fds.ymd.ok() || fds.has_tod) { + days doy; + if (fds.ymd.ok()) { + auto ld = local_days(fds.ymd); + auto y = fds.ymd.year(); + doy = ld - local_days(y / January / 1) + days{1}; + } else { + doy = duration_cast(fds.tod.to_duration()); + } + save_ostream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(3); + os << doy.count(); + } else { + os.setstate(std::ios::failbit); + } + } else { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } else + os << *fmt; + break; + case 'm': + if (command) { + if (modified == CharT{'E'}) + os << CharT{'%'} << modified << *fmt; + else { + if (!fds.ymd.month().ok()) + os.setstate(std::ios::failbit); + auto m = static_cast(fds.ymd.month()); #if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) - { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_mon = static_cast(m-1); - facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); - } + if (modified == CharT{}) #endif - } - modified = CharT{}; - command = nullptr; - } - else - os << *fmt; - break; - case 'M': - if (command) { - if (modified == CharT{'E'}) - os << CharT{'%'} << modified << *fmt; - else - { - if (!fds.has_tod) - os.setstate(std::ios::failbit); - if (insert_negative) - { - os << '-'; - insert_negative = false; - } + if (m < 10) + os << CharT{'0'}; + os << m; + } #if !ONLY_C_LOCALE - if (modified == CharT{}) + else if (modified == CharT{'O'}) { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_mon = static_cast(m - 1); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } #endif - { - if (fds.tod.minutes() < minutes{10}) - os << CharT{'0'}; - os << fds.tod.minutes().count(); - } + } + modified = CharT{}; + command = nullptr; + } else + os << *fmt; + break; + case 'M': + if (command) { + if (modified == CharT{'E'}) + os << CharT{'%'} << modified << *fmt; + else { + if (!fds.has_tod) + os.setstate(std::ios::failbit); + if (insert_negative) { + os << '-'; + insert_negative = false; + } #if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) - { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_min = static_cast(fds.tod.minutes().count()); - facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); - } + if (modified == CharT{}) #endif - } - modified = CharT{}; - command = nullptr; - } - else - os << *fmt; - break; - case 'n': - if (command) { - if (modified == CharT{}) - os << CharT{'\n'}; - else - { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; + if (fds.tod.minutes() < minutes{10}) + os << CharT{'0'}; + os << fds.tod.minutes().count(); } - else - os << *fmt; - break; - case 'p': - if (command) - { - if (modified == CharT{}) - { - if (!fds.has_tod) - os.setstate(std::ios::failbit); #if !ONLY_C_LOCALE - const CharT f[] = {'%', *fmt}; - tm.tm_hour = static_cast(fds.tod.hours().count()); - facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); -#else - if (date::is_am(fds.tod.hours())) - os << ampm_names().first[0]; - else - os << ampm_names().first[1]; -#endif - } - else - { - os << CharT{'%'} << modified << *fmt; - } - modified = CharT{}; - command = nullptr; + else if (modified == CharT{'O'}) { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_min = static_cast(fds.tod.minutes().count()); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); } +#endif + } + modified = CharT{}; + command = nullptr; + } else + os << *fmt; + break; + case 'n': + if (command) { + if (modified == CharT{}) + os << CharT{'\n'}; + else { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } else + os << *fmt; + break; + case 'p': + if (command) { + if (modified == CharT{}) { + if (!fds.has_tod) + os.setstate(std::ios::failbit); +#if !ONLY_C_LOCALE + const CharT f[] = {'%', *fmt}; + tm.tm_hour = static_cast(fds.tod.hours().count()); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); +#else + if (date::is_am(fds.tod.hours())) + os << ampm_names().first[0]; else - os << *fmt; - break; - case 'Q': - case 'q': - if (command) - { - if (modified == CharT{}) - { - if (!fds.has_tod) - os.setstate(std::ios::failbit); - auto d = fds.tod.to_duration(); - if (*fmt == 'q') - os << get_units(typename decltype(d)::period::type{}); - else - os << d.count(); - } - else - { - os << CharT{'%'} << modified << *fmt; - } - modified = CharT{}; - command = nullptr; - } + os << ampm_names().first[1]; +#endif + } else { + os << CharT{'%'} << modified << *fmt; + } + modified = CharT{}; + command = nullptr; + } else + os << *fmt; + break; + case 'Q': + case 'q': + if (command) { + if (modified == CharT{}) { + if (!fds.has_tod) + os.setstate(std::ios::failbit); + auto d = fds.tod.to_duration(); + if (*fmt == 'q') + os << get_units(typename decltype(d)::period::type{}); else - os << *fmt; - break; - case 'r': - if (command) - { - if (modified == CharT{}) - { - if (!fds.has_tod) - os.setstate(std::ios::failbit); + os << d.count(); + } else { + os << CharT{'%'} << modified << *fmt; + } + modified = CharT{}; + command = nullptr; + } else + os << *fmt; + break; + case 'r': + if (command) { + if (modified == CharT{}) { + if (!fds.has_tod) + os.setstate(std::ios::failbit); #if !ONLY_C_LOCALE - const CharT f[] = {'%', *fmt}; - tm.tm_hour = static_cast(fds.tod.hours().count()); - tm.tm_min = static_cast(fds.tod.minutes().count()); - tm.tm_sec = static_cast(fds.tod.seconds().count()); - facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + const CharT f[] = {'%', *fmt}; + tm.tm_hour = static_cast(fds.tod.hours().count()); + tm.tm_min = static_cast(fds.tod.minutes().count()); + tm.tm_sec = static_cast(fds.tod.seconds().count()); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); #else - hh_mm_ss tod(duration_cast(fds.tod.to_duration())); - save_ostream _(os); - os.fill('0'); - os.width(2); - os << date::make12(tod.hours()).count() << CharT{':'}; - os.width(2); - os << tod.minutes().count() << CharT{':'}; - os.width(2); - os << tod.seconds().count() << CharT{' '}; - if (date::is_am(tod.hours())) - os << ampm_names().first[0]; - else - os << ampm_names().first[1]; + hh_mm_ss tod(duration_cast(fds.tod.to_duration())); + save_ostream _(os); + os.fill('0'); + os.width(2); + os << date::make12(tod.hours()).count() << CharT{':'}; + os.width(2); + os << tod.minutes().count() << CharT{':'}; + os.width(2); + os << tod.seconds().count() << CharT{' '}; + if (date::is_am(tod.hours())) + os << ampm_names().first[0]; + else + os << ampm_names().first[1]; #endif - } - else - { - os << CharT{'%'} << modified << *fmt; - } - modified = CharT{}; - command = nullptr; + } else { + os << CharT{'%'} << modified << *fmt; + } + modified = CharT{}; + command = nullptr; + } else + os << *fmt; + break; + case 'R': + if (command) { + if (modified == CharT{}) { + if (!fds.has_tod) + os.setstate(std::ios::failbit); + if (fds.tod.hours() < hours{10}) + os << CharT{'0'}; + os << fds.tod.hours().count() << CharT{':'}; + if (fds.tod.minutes() < minutes{10}) + os << CharT{'0'}; + os << fds.tod.minutes().count(); + } else { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } else + os << *fmt; + break; + case 'S': + if (command) { + if (modified == CharT{'E'}) + os << CharT{'%'} << modified << *fmt; + else { + if (!fds.has_tod) + os.setstate(std::ios::failbit); + if (insert_negative) { + os << '-'; + insert_negative = false; } - else - os << *fmt; - break; - case 'R': - if (command) +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#endif { - if (modified == CharT{}) - { - if (!fds.has_tod) - os.setstate(std::ios::failbit); - if (fds.tod.hours() < hours{10}) - os << CharT{'0'}; - os << fds.tod.hours().count() << CharT{':'}; - if (fds.tod.minutes() < minutes{10}) - os << CharT{'0'}; - os << fds.tod.minutes().count(); - } - else - { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; + os << fds.tod.s_; } - else - os << *fmt; - break; - case 'S': - if (command) - { - if (modified == CharT{'E'}) - os << CharT{'%'} << modified << *fmt; - else - { - if (!fds.has_tod) - os.setstate(std::ios::failbit); - if (insert_negative) - { - os << '-'; - insert_negative = false; - } #if !ONLY_C_LOCALE - if (modified == CharT{}) + else if (modified == CharT{'O'}) { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_sec = static_cast(fds.tod.s_.seconds().count()); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } #endif - { - os << fds.tod.s_; - } + } + modified = CharT{}; + command = nullptr; + } else + os << *fmt; + break; + case 't': + if (command) { + if (modified == CharT{}) + os << CharT{'\t'}; + else { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } else + os << *fmt; + break; + case 'T': + if (command) { + if (modified == CharT{}) { + if (!fds.has_tod) + os.setstate(std::ios::failbit); + os << fds.tod; + } else { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } else + os << *fmt; + break; + case 'u': + if (command) { + if (modified == CharT{'E'}) + os << CharT{'%'} << modified << *fmt; + else { + auto wd = extract_weekday(os, fds); #if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) - { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_sec = static_cast(fds.tod.s_.seconds().count()); - facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); - } + if (modified == CharT{}) #endif - } - modified = CharT{}; - command = nullptr; - } - else - os << *fmt; - break; - case 't': - if (command) { - if (modified == CharT{}) - os << CharT{'\t'}; - else - { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; + os << (wd != 0 ? wd : 7u); } - else - os << *fmt; - break; - case 'T': - if (command) - { - if (modified == CharT{}) - { - if (!fds.has_tod) - os.setstate(std::ios::failbit); - os << fds.tod; - } - else - { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; - } - else - os << *fmt; - break; - case 'u': - if (command) - { - if (modified == CharT{'E'}) - os << CharT{'%'} << modified << *fmt; - else - { - auto wd = extract_weekday(os, fds); #if !ONLY_C_LOCALE - if (modified == CharT{}) + else if (modified == CharT{'O'}) { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_wday = static_cast(wd); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } #endif - { - os << (wd != 0 ? wd : 7u); - } + } + modified = CharT{}; + command = nullptr; + } else + os << *fmt; + break; + case 'U': + if (command) { + if (modified == CharT{'E'}) + os << CharT{'%'} << modified << *fmt; + else { + auto const& ymd = fds.ymd; + if (!ymd.ok()) + os.setstate(std::ios::failbit); + auto ld = local_days(ymd); #if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) - { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_wday = static_cast(wd); - facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); - } + if (modified == CharT{}) #endif - } - modified = CharT{}; - command = nullptr; - } - else - os << *fmt; - break; - case 'U': - if (command) { - if (modified == CharT{'E'}) - os << CharT{'%'} << modified << *fmt; - else - { - auto const& ymd = fds.ymd; - if (!ymd.ok()) - os.setstate(std::ios::failbit); - auto ld = local_days(ymd); -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#endif - { - auto st = local_days(Sunday[1]/January/ymd.year()); - if (ld < st) - os << CharT{'0'} << CharT{'0'}; - else - { - auto wn = duration_cast(ld - st).count() + 1; - if (wn < 10) - os << CharT{'0'}; - os << wn; - } - } - #if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) - { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_year = static_cast(ymd.year()) - 1900; - tm.tm_wday = static_cast(extract_weekday(os, fds)); - if (os.fail()) - return os; - tm.tm_yday = static_cast((ld - local_days(ymd.year()/1/1)).count()); - facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); - } -#endif - } - modified = CharT{}; - command = nullptr; + auto st = local_days(Sunday[1] / January / ymd.year()); + if (ld < st) + os << CharT{'0'} << CharT{'0'}; + else { + auto wn = duration_cast(ld - st).count() + 1; + if (wn < 10) + os << CharT{'0'}; + os << wn; + } } - else - os << *fmt; - break; - case 'V': - if (command) - { - if (modified == CharT{'E'}) - os << CharT{'%'} << modified << *fmt; - else - { - if (!fds.ymd.ok()) - os.setstate(std::ios::failbit); - auto ld = local_days(fds.ymd); #if !ONLY_C_LOCALE - if (modified == CharT{}) + else if (modified == CharT{'O'}) { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_year = static_cast(ymd.year()) - 1900; + tm.tm_wday = static_cast(extract_weekday(os, fds)); + if (os.fail()) + return os; + tm.tm_yday = static_cast((ld - local_days(ymd.year() / 1 / 1)).count()); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } #endif - { - auto y = year_month_day{ld + days{3}}.year(); - auto st = local_days((y-years{1})/12/Thursday[last]) + - (Monday-Thursday); - if (ld < st) - { - --y; - st = local_days((y - years{1})/12/Thursday[last]) + - (Monday-Thursday); - } - auto wn = duration_cast(ld - st).count() + 1; - if (wn < 10) - os << CharT{'0'}; - os << wn; - } + } + modified = CharT{}; + command = nullptr; + } else + os << *fmt; + break; + case 'V': + if (command) { + if (modified == CharT{'E'}) + os << CharT{'%'} << modified << *fmt; + else { + if (!fds.ymd.ok()) + os.setstate(std::ios::failbit); + auto ld = local_days(fds.ymd); #if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) - { - const CharT f[] = {'%', modified, *fmt}; - auto const& ymd = fds.ymd; - tm.tm_year = static_cast(ymd.year()) - 1900; - tm.tm_wday = static_cast(extract_weekday(os, fds)); - if (os.fail()) - return os; - tm.tm_yday = static_cast((ld - local_days(ymd.year()/1/1)).count()); - facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); - } + if (modified == CharT{}) #endif - } - modified = CharT{}; - command = nullptr; - } - else - os << *fmt; - break; - case 'w': - if (command) { - auto wd = extract_weekday(os, fds); - if (os.fail()) - return os; + auto y = year_month_day{ld + days{3}}.year(); + auto st = local_days((y - years{1}) / 12 / Thursday[last]) + (Monday - Thursday); + if (ld < st) { + --y; + st = local_days((y - years{1}) / 12 / Thursday[last]) + (Monday - Thursday); + } + auto wn = duration_cast(ld - st).count() + 1; + if (wn < 10) + os << CharT{'0'}; + os << wn; + } #if !ONLY_C_LOCALE - if (modified == CharT{}) -#else - if (modified != CharT{'E'}) + else if (modified == CharT{'O'}) { + const CharT f[] = {'%', modified, *fmt}; + auto const& ymd = fds.ymd; + tm.tm_year = static_cast(ymd.year()) - 1900; + tm.tm_wday = static_cast(extract_weekday(os, fds)); + if (os.fail()) + return os; + tm.tm_yday = static_cast((ld - local_days(ymd.year() / 1 / 1)).count()); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } #endif - { - os << wd; - } + } + modified = CharT{}; + command = nullptr; + } else + os << *fmt; + break; + case 'w': + if (command) { + auto wd = extract_weekday(os, fds); + if (os.fail()) + return os; #if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) - { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_wday = static_cast(wd); - facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); - } + if (modified == CharT{}) +#else + if (modified != CharT{'E'}) #endif - else - { - os << CharT{'%'} << modified << *fmt; - } - modified = CharT{}; - command = nullptr; - } - else - os << *fmt; - break; - case 'W': - if (command) - { - if (modified == CharT{'E'}) - os << CharT{'%'} << modified << *fmt; - else - { - auto const& ymd = fds.ymd; - if (!ymd.ok()) - os.setstate(std::ios::failbit); - auto ld = local_days(ymd); + { + os << wd; + } #if !ONLY_C_LOCALE - if (modified == CharT{}) + else if (modified == CharT{'O'}) { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_wday = static_cast(wd); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } #endif - { - auto st = local_days(Monday[1]/January/ymd.year()); - if (ld < st) - os << CharT{'0'} << CharT{'0'}; - else - { - auto wn = duration_cast(ld - st).count() + 1; - if (wn < 10) - os << CharT{'0'}; - os << wn; - } - } + else { + os << CharT{'%'} << modified << *fmt; + } + modified = CharT{}; + command = nullptr; + } else + os << *fmt; + break; + case 'W': + if (command) { + if (modified == CharT{'E'}) + os << CharT{'%'} << modified << *fmt; + else { + auto const& ymd = fds.ymd; + if (!ymd.ok()) + os.setstate(std::ios::failbit); + auto ld = local_days(ymd); #if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) - { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_year = static_cast(ymd.year()) - 1900; - tm.tm_wday = static_cast(extract_weekday(os, fds)); - if (os.fail()) - return os; - tm.tm_yday = static_cast((ld - local_days(ymd.year()/1/1)).count()); - facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); - } + if (modified == CharT{}) #endif - } - modified = CharT{}; - command = nullptr; - } - else - os << *fmt; - break; - case 'X': - if (command) { - if (modified == CharT{'O'}) - os << CharT{'%'} << modified << *fmt; - else - { - if (!fds.has_tod) - os.setstate(std::ios::failbit); + auto st = local_days(Monday[1] / January / ymd.year()); + if (ld < st) + os << CharT{'0'} << CharT{'0'}; + else { + auto wn = duration_cast(ld - st).count() + 1; + if (wn < 10) + os << CharT{'0'}; + os << wn; + } + } #if !ONLY_C_LOCALE - tm = std::tm{}; - tm.tm_sec = static_cast(fds.tod.seconds().count()); - tm.tm_min = static_cast(fds.tod.minutes().count()); - tm.tm_hour = static_cast(fds.tod.hours().count()); - CharT f[3] = {'%'}; - auto fe = std::begin(f) + 1; - if (modified == CharT{'E'}) - *fe++ = modified; - *fe++ = *fmt; - facet.put(os, os, os.fill(), &tm, std::begin(f), fe); -#else - os << fds.tod; -#endif - } - command = nullptr; - modified = CharT{}; + else if (modified == CharT{'O'}) { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_year = static_cast(ymd.year()) - 1900; + tm.tm_wday = static_cast(extract_weekday(os, fds)); + if (os.fail()) + return os; + tm.tm_yday = static_cast((ld - local_days(ymd.year() / 1 / 1)).count()); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); } - else - os << *fmt; - break; - case 'y': - if (command) - { - if (!fds.ymd.year().ok()) - os.setstate(std::ios::failbit); - auto y = static_cast(fds.ymd.year()); +#endif + } + modified = CharT{}; + command = nullptr; + } else + os << *fmt; + break; + case 'X': + if (command) { + if (modified == CharT{'O'}) + os << CharT{'%'} << modified << *fmt; + else { + if (!fds.has_tod) + os.setstate(std::ios::failbit); #if !ONLY_C_LOCALE - if (modified == CharT{}) - { + tm = std::tm{}; + tm.tm_sec = static_cast(fds.tod.seconds().count()); + tm.tm_min = static_cast(fds.tod.minutes().count()); + tm.tm_hour = static_cast(fds.tod.hours().count()); + CharT f[3] = {'%'}; + auto fe = std::begin(f) + 1; + if (modified == CharT{'E'}) + *fe++ = modified; + *fe++ = *fmt; + facet.put(os, os, os.fill(), &tm, std::begin(f), fe); +#else + os << fds.tod; #endif - y = std::abs(y) % 100; - if (y < 10) - os << CharT{'0'}; - os << y; + } + command = nullptr; + modified = CharT{}; + } else + os << *fmt; + break; + case 'y': + if (command) { + if (!fds.ymd.year().ok()) + os.setstate(std::ios::failbit); + auto y = static_cast(fds.ymd.year()); #if !ONLY_C_LOCALE - } - else - { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_year = y - 1900; - facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); - } + if (modified == CharT{}) { #endif - modified = CharT{}; - command = nullptr; - } - else - os << *fmt; - break; - case 'Y': - if (command) - { - if (modified == CharT{'O'}) - os << CharT{'%'} << modified << *fmt; - else - { - if (!fds.ymd.year().ok()) - os.setstate(std::ios::failbit); - auto y = fds.ymd.year(); + y = std::abs(y) % 100; + if (y < 10) + os << CharT{'0'}; + os << y; #if !ONLY_C_LOCALE - if (modified == CharT{}) + } else { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_year = y - 1900; + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } #endif - { - save_ostream _(os); - os.imbue(std::locale::classic()); - os << y; - } + modified = CharT{}; + command = nullptr; + } else + os << *fmt; + break; + case 'Y': + if (command) { + if (modified == CharT{'O'}) + os << CharT{'%'} << modified << *fmt; + else { + if (!fds.ymd.year().ok()) + os.setstate(std::ios::failbit); + auto y = fds.ymd.year(); #if !ONLY_C_LOCALE - else if (modified == CharT{'E'}) - { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_year = static_cast(y) - 1900; - facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); - } + if (modified == CharT{}) #endif - } - modified = CharT{}; - command = nullptr; - } - else - os << *fmt; - break; - case 'z': - if (command) - { - if (offset_sec == nullptr) - { - // Can not format %z with unknown offset - os.setstate(ios::failbit); - return os; - } - auto m = duration_cast(*offset_sec); - auto neg = m < minutes{0}; - m = date::abs(m); - auto h = duration_cast(m); - m -= h; - if (neg) - os << CharT{'-'}; - else - os << CharT{'+'}; - if (h < hours{10}) - os << CharT{'0'}; - os << h.count(); - if (modified != CharT{}) - os << CharT{':'}; - if (m < minutes{10}) - os << CharT{'0'}; - os << m.count(); - command = nullptr; - modified = CharT{}; - } - else - os << *fmt; - break; - case 'Z': - if (command) { - if (modified == CharT{}) - { - if (abbrev == nullptr) - { - // Can not format %Z with unknown time_zone - os.setstate(ios::failbit); - return os; - } - for (auto c : *abbrev) - os << CharT(c); - } - else - { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; - } - else - os << *fmt; - break; - case 'E': - case 'O': - if (command) - { - if (modified == CharT{}) - { - modified = *fmt; - } - else - { - os << CharT{'%'} << modified << *fmt; - command = nullptr; - modified = CharT{}; - } + save_ostream _(os); + os.imbue(std::locale::classic()); + os << y; } - else - os << *fmt; - break; - case '%': - if (command) - { - if (modified == CharT{}) - { - os << CharT{'%'}; - command = nullptr; - } - else - { - os << CharT{'%'} << modified << CharT{'%'}; - command = nullptr; - modified = CharT{}; - } - } - else - command = fmt; - break; - default: - if (command) - { - os << CharT{'%'}; - command = nullptr; +#if !ONLY_C_LOCALE + else if (modified == CharT{'E'}) { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_year = static_cast(y) - 1900; + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); } - if (modified != CharT{}) - { - os << modified; - modified = CharT{}; +#endif + } + modified = CharT{}; + command = nullptr; + } else + os << *fmt; + break; + case 'z': + if (command) { + if (offset_sec == nullptr) { + // Can not format %z with unknown offset + os.setstate(ios::failbit); + return os; + } + auto m = duration_cast(*offset_sec); + auto neg = m < minutes{0}; + m = date::abs(m); + auto h = duration_cast(m); + m -= h; + if (neg) + os << CharT{'-'}; + else + os << CharT{'+'}; + if (h < hours{10}) + os << CharT{'0'}; + os << h.count(); + if (modified != CharT{}) + os << CharT{':'}; + if (m < minutes{10}) + os << CharT{'0'}; + os << m.count(); + command = nullptr; + modified = CharT{}; + } else + os << *fmt; + break; + case 'Z': + if (command) { + if (modified == CharT{}) { + if (abbrev == nullptr) { + // Can not format %Z with unknown time_zone + os.setstate(ios::failbit); + return os; } - os << *fmt; - break; + for (auto c : *abbrev) + os << CharT(c); + } else { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } else + os << *fmt; + break; + case 'E': + case 'O': + if (command) { + if (modified == CharT{}) { + modified = *fmt; + } else { + os << CharT{'%'} << modified << *fmt; + command = nullptr; + modified = CharT{}; + } + } else + os << *fmt; + break; + case '%': + if (command) { + if (modified == CharT{}) { + os << CharT{'%'}; + command = nullptr; + } else { + os << CharT{'%'} << modified << CharT{'%'}; + command = nullptr; + modified = CharT{}; + } + } else + command = fmt; + break; + default: + if (command) { + os << CharT{'%'}; + command = nullptr; } + if (modified != CharT{}) { + os << modified; + modified = CharT{}; + } + os << *fmt; + break; } - if (command) - os << CharT{'%'}; - if (modified != CharT{}) - os << modified; - return os; + } + if (command) + os << CharT{'%'}; + if (modified != CharT{}) + os << modified; + return os; } template -inline -std::basic_ostream& -to_stream(std::basic_ostream& os, const CharT* fmt, const year& y) -{ - using CT = std::chrono::seconds; - fields fds{y/0/0}; - return to_stream(os, fmt, fds); +inline std::basic_ostream& to_stream(std::basic_ostream& os, const CharT* fmt, + const year& y) { + using CT = std::chrono::seconds; + fields fds{y / 0 / 0}; + return to_stream(os, fmt, fds); } template -inline -std::basic_ostream& -to_stream(std::basic_ostream& os, const CharT* fmt, const month& m) -{ - using CT = std::chrono::seconds; - fields fds{m/0/nanyear}; - return to_stream(os, fmt, fds); +inline std::basic_ostream& to_stream(std::basic_ostream& os, const CharT* fmt, + const month& m) { + using CT = std::chrono::seconds; + fields fds{m / 0 / nanyear}; + return to_stream(os, fmt, fds); } template -inline -std::basic_ostream& -to_stream(std::basic_ostream& os, const CharT* fmt, const day& d) -{ - using CT = std::chrono::seconds; - fields fds{d/0/nanyear}; - return to_stream(os, fmt, fds); +inline std::basic_ostream& to_stream(std::basic_ostream& os, const CharT* fmt, + const day& d) { + using CT = std::chrono::seconds; + fields fds{d / 0 / nanyear}; + return to_stream(os, fmt, fds); } template -inline -std::basic_ostream& -to_stream(std::basic_ostream& os, const CharT* fmt, const weekday& wd) -{ - using CT = std::chrono::seconds; - fields fds{wd}; - return to_stream(os, fmt, fds); +inline std::basic_ostream& to_stream(std::basic_ostream& os, const CharT* fmt, + const weekday& wd) { + using CT = std::chrono::seconds; + fields fds{wd}; + return to_stream(os, fmt, fds); } template -inline -std::basic_ostream& -to_stream(std::basic_ostream& os, const CharT* fmt, const year_month& ym) -{ - using CT = std::chrono::seconds; - fields fds{ym/0}; - return to_stream(os, fmt, fds); +inline std::basic_ostream& to_stream(std::basic_ostream& os, const CharT* fmt, + const year_month& ym) { + using CT = std::chrono::seconds; + fields fds{ym / 0}; + return to_stream(os, fmt, fds); } template -inline -std::basic_ostream& -to_stream(std::basic_ostream& os, const CharT* fmt, const month_day& md) -{ - using CT = std::chrono::seconds; - fields fds{md/nanyear}; - return to_stream(os, fmt, fds); +inline std::basic_ostream& to_stream(std::basic_ostream& os, const CharT* fmt, + const month_day& md) { + using CT = std::chrono::seconds; + fields fds{md / nanyear}; + return to_stream(os, fmt, fds); } template -inline -std::basic_ostream& -to_stream(std::basic_ostream& os, const CharT* fmt, - const year_month_day& ymd) -{ - using CT = std::chrono::seconds; - fields fds{ymd}; - return to_stream(os, fmt, fds); +inline std::basic_ostream& to_stream(std::basic_ostream& os, const CharT* fmt, + const year_month_day& ymd) { + using CT = std::chrono::seconds; + fields fds{ymd}; + return to_stream(os, fmt, fds); } template -inline -std::basic_ostream& -to_stream(std::basic_ostream& os, const CharT* fmt, - const std::chrono::duration& d) -{ - using Duration = std::chrono::duration; - using CT = typename std::common_type::type; - fields fds{hh_mm_ss{d}}; - return to_stream(os, fmt, fds); +inline std::basic_ostream& to_stream(std::basic_ostream& os, const CharT* fmt, + const std::chrono::duration& d) { + using Duration = std::chrono::duration; + using CT = typename std::common_type::type; + fields fds{hh_mm_ss{d}}; + return to_stream(os, fmt, fds); } template -std::basic_ostream& -to_stream(std::basic_ostream& os, const CharT* fmt, - const local_time& tp, const std::string* abbrev = nullptr, - const std::chrono::seconds* offset_sec = nullptr) -{ - using CT = typename std::common_type::type; - auto ld = floor(tp); - fields fds{year_month_day{ld}, hh_mm_ss{tp-local_seconds{ld}}}; - return to_stream(os, fmt, fds, abbrev, offset_sec); +std::basic_ostream& to_stream(std::basic_ostream& os, const CharT* fmt, + const local_time& tp, const std::string* abbrev = nullptr, + const std::chrono::seconds* offset_sec = nullptr) { + using CT = typename std::common_type::type; + auto ld = floor(tp); + fields fds{year_month_day{ld}, hh_mm_ss{tp - local_seconds{ld}}}; + return to_stream(os, fmt, fds, abbrev, offset_sec); } template -std::basic_ostream& -to_stream(std::basic_ostream& os, const CharT* fmt, - const sys_time& tp) -{ - using std::chrono::seconds; - using CT = typename std::common_type::type; - const std::string abbrev("UTC"); - CONSTDATA seconds offset{0}; - auto sd = floor(tp); - fields fds{year_month_day{sd}, hh_mm_ss{tp-sys_seconds{sd}}}; - return to_stream(os, fmt, fds, &abbrev, &offset); +std::basic_ostream& to_stream(std::basic_ostream& os, const CharT* fmt, + const sys_time& tp) { + using std::chrono::seconds; + using CT = typename std::common_type::type; + const std::string abbrev("UTC"); + CONSTDATA seconds offset{0}; + auto sd = floor(tp); + fields fds{year_month_day{sd}, hh_mm_ss{tp - sys_seconds{sd}}}; + return to_stream(os, fmt, fds, &abbrev, &offset); } // format template -auto -format(const std::locale& loc, const CharT* fmt, const Streamable& tp) - -> decltype(to_stream(std::declval&>(), fmt, tp), - std::basic_string{}) -{ - std::basic_ostringstream os; - os.exceptions(std::ios::failbit | std::ios::badbit); - os.imbue(loc); - to_stream(os, fmt, tp); - return os.str(); +auto format(const std::locale& loc, const CharT* fmt, const Streamable& tp) + -> decltype(to_stream(std::declval&>(), fmt, tp), std::basic_string{}) { + std::basic_ostringstream os; + os.exceptions(std::ios::failbit | std::ios::badbit); + os.imbue(loc); + to_stream(os, fmt, tp); + return os.str(); } template -auto -format(const CharT* fmt, const Streamable& tp) - -> decltype(to_stream(std::declval&>(), fmt, tp), - std::basic_string{}) -{ - std::basic_ostringstream os; - os.exceptions(std::ios::failbit | std::ios::badbit); - to_stream(os, fmt, tp); - return os.str(); +auto format(const CharT* fmt, const Streamable& tp) + -> decltype(to_stream(std::declval&>(), fmt, tp), std::basic_string{}) { + std::basic_ostringstream os; + os.exceptions(std::ios::failbit | std::ios::badbit); + to_stream(os, fmt, tp); + return os.str(); } template -auto -format(const std::locale& loc, const std::basic_string& fmt, - const Streamable& tp) +auto format(const std::locale& loc, const std::basic_string& fmt, const Streamable& tp) -> decltype(to_stream(std::declval&>(), fmt.c_str(), tp), - std::basic_string{}) -{ - std::basic_ostringstream os; - os.exceptions(std::ios::failbit | std::ios::badbit); - os.imbue(loc); - to_stream(os, fmt.c_str(), tp); - return os.str(); + std::basic_string{}) { + std::basic_ostringstream os; + os.exceptions(std::ios::failbit | std::ios::badbit); + os.imbue(loc); + to_stream(os, fmt.c_str(), tp); + return os.str(); } template -auto -format(const std::basic_string& fmt, const Streamable& tp) +auto format(const std::basic_string& fmt, const Streamable& tp) -> decltype(to_stream(std::declval&>(), fmt.c_str(), tp), - std::basic_string{}) -{ - std::basic_ostringstream os; - os.exceptions(std::ios::failbit | std::ios::badbit); - to_stream(os, fmt.c_str(), tp); - return os.str(); + std::basic_string{}) { + std::basic_ostringstream os; + os.exceptions(std::ios::failbit | std::ios::badbit); + to_stream(os, fmt.c_str(), tp); + return os.str(); } // parse -namespace detail -{ +namespace detail { template -bool -read_char(std::basic_istream& is, CharT fmt, std::ios::iostate& err) -{ - auto ic = is.get(); - if (Traits::eq_int_type(ic, Traits::eof()) || - !Traits::eq(Traits::to_char_type(ic), fmt)) - { - err |= std::ios::failbit; - is.setstate(std::ios::failbit); - return false; - } - return true; +bool read_char(std::basic_istream& is, CharT fmt, std::ios::iostate& err) { + auto ic = is.get(); + if (Traits::eq_int_type(ic, Traits::eof()) || !Traits::eq(Traits::to_char_type(ic), fmt)) { + err |= std::ios::failbit; + is.setstate(std::ios::failbit); + return false; + } + return true; } template -unsigned -read_unsigned(std::basic_istream& is, unsigned m = 1, unsigned M = 10) -{ - unsigned x = 0; - unsigned count = 0; - while (true) - { - auto ic = is.peek(); - if (Traits::eq_int_type(ic, Traits::eof())) - break; - auto c = static_cast(Traits::to_char_type(ic)); - if (!('0' <= c && c <= '9')) - break; - (void)is.get(); - ++count; - x = 10*x + static_cast(c - '0'); - if (count == M) - break; - } - if (count < m) - is.setstate(std::ios::failbit); - return x; +unsigned read_unsigned(std::basic_istream& is, unsigned m = 1, unsigned M = 10) { + unsigned x = 0; + unsigned count = 0; + while (true) { + auto ic = is.peek(); + if (Traits::eq_int_type(ic, Traits::eof())) + break; + auto c = static_cast(Traits::to_char_type(ic)); + if (!('0' <= c && c <= '9')) + break; + (void)is.get(); + ++count; + x = 10 * x + static_cast(c - '0'); + if (count == M) + break; + } + if (count < m) + is.setstate(std::ios::failbit); + return x; } template -int -read_signed(std::basic_istream& is, unsigned m = 1, unsigned M = 10) -{ - auto ic = is.peek(); - if (!Traits::eq_int_type(ic, Traits::eof())) - { - auto c = static_cast(Traits::to_char_type(ic)); - if (('0' <= c && c <= '9') || c == '-' || c == '+') - { - if (c == '-' || c == '+') - (void)is.get(); - auto x = static_cast(read_unsigned(is, std::max(m, 1u), M)); - if (!is.fail()) - { - if (c == '-') - x = -x; - return x; - } - } +int read_signed(std::basic_istream& is, unsigned m = 1, unsigned M = 10) { + auto ic = is.peek(); + if (!Traits::eq_int_type(ic, Traits::eof())) { + auto c = static_cast(Traits::to_char_type(ic)); + if (('0' <= c && c <= '9') || c == '-' || c == '+') { + if (c == '-' || c == '+') + (void)is.get(); + auto x = static_cast(read_unsigned(is, std::max(m, 1u), M)); + if (!is.fail()) { + if (c == '-') + x = -x; + return x; + } } - if (m > 0) - is.setstate(std::ios::failbit); - return 0; + } + if (m > 0) + is.setstate(std::ios::failbit); + return 0; } template -long double -read_long_double(std::basic_istream& is, unsigned m = 1, unsigned M = 10) -{ - unsigned count = 0; - unsigned fcount = 0; - unsigned long long i = 0; - unsigned long long f = 0; - bool parsing_fraction = false; +long double read_long_double(std::basic_istream& is, unsigned m = 1, unsigned M = 10) { + unsigned count = 0; + unsigned fcount = 0; + unsigned long long i = 0; + unsigned long long f = 0; + bool parsing_fraction = false; #if ONLY_C_LOCALE - typename Traits::int_type decimal_point = '.'; + typename Traits::int_type decimal_point = '.'; #else - auto decimal_point = Traits::to_int_type( - std::use_facet>(is.getloc()).decimal_point()); + auto decimal_point = Traits::to_int_type(std::use_facet>(is.getloc()).decimal_point()); #endif - while (true) - { - auto ic = is.peek(); - if (Traits::eq_int_type(ic, Traits::eof())) - break; - if (Traits::eq_int_type(ic, decimal_point)) - { - decimal_point = Traits::eof(); - parsing_fraction = true; - } - else - { - auto c = static_cast(Traits::to_char_type(ic)); - if (!('0' <= c && c <= '9')) - break; - if (!parsing_fraction) - { - i = 10*i + static_cast(c - '0'); - } - else - { - f = 10*f + static_cast(c - '0'); - ++fcount; - } - } - (void)is.get(); - if (++count == M) - break; - } - if (count < m) - { - is.setstate(std::ios::failbit); - return 0; + while (true) { + auto ic = is.peek(); + if (Traits::eq_int_type(ic, Traits::eof())) + break; + if (Traits::eq_int_type(ic, decimal_point)) { + decimal_point = Traits::eof(); + parsing_fraction = true; + } else { + auto c = static_cast(Traits::to_char_type(ic)); + if (!('0' <= c && c <= '9')) + break; + if (!parsing_fraction) { + i = 10 * i + static_cast(c - '0'); + } else { + f = 10 * f + static_cast(c - '0'); + ++fcount; + } } - return i + f/std::pow(10.L, fcount); + (void)is.get(); + if (++count == M) + break; + } + if (count < m) { + is.setstate(std::ios::failbit); + return 0; + } + return i + f / std::pow(10.L, fcount); } -struct rs -{ - int& i; - unsigned m; - unsigned M; +struct rs { + int& i; + unsigned m; + unsigned M; }; -struct ru -{ - int& i; - unsigned m; - unsigned M; +struct ru { + int& i; + unsigned m; + unsigned M; }; -struct rld -{ - long double& i; - unsigned m; - unsigned M; +struct rld { + long double& i; + unsigned m; + unsigned M; }; template -void -read(std::basic_istream&) -{ -} - -template -void -read(std::basic_istream& is, CharT a0, Args&& ...args); - -template -void -read(std::basic_istream& is, rs a0, Args&& ...args); - -template -void -read(std::basic_istream& is, ru a0, Args&& ...args); - -template -void -read(std::basic_istream& is, int a0, Args&& ...args); - -template -void -read(std::basic_istream& is, rld a0, Args&& ...args); - -template -void -read(std::basic_istream& is, CharT a0, Args&& ...args) -{ - // No-op if a0 == CharT{} - if (a0 != CharT{}) - { - auto ic = is.peek(); - if (Traits::eq_int_type(ic, Traits::eof())) - { - is.setstate(std::ios::failbit | std::ios::eofbit); - return; - } - if (!Traits::eq(Traits::to_char_type(ic), a0)) - { - is.setstate(std::ios::failbit); - return; - } - (void)is.get(); - } - read(is, std::forward(args)...); +void read(std::basic_istream&) { } -template -void -read(std::basic_istream& is, rs a0, Args&& ...args) -{ - auto x = read_signed(is, a0.m, a0.M); - if (is.fail()) - return; - a0.i = x; - read(is, std::forward(args)...); -} +template +void read(std::basic_istream& is, CharT a0, Args&&... args); -template -void -read(std::basic_istream& is, ru a0, Args&& ...args) -{ - auto x = read_unsigned(is, a0.m, a0.M); - if (is.fail()) - return; - a0.i = static_cast(x); - read(is, std::forward(args)...); -} +template +void read(std::basic_istream& is, rs a0, Args&&... args); -template -void -read(std::basic_istream& is, int a0, Args&& ...args) -{ - if (a0 != -1) - { - auto u = static_cast(a0); - CharT buf[std::numeric_limits::digits10+2u] = {}; - auto e = buf; - do - { - *e++ = static_cast(CharT(u % 10) + CharT{'0'}); - u /= 10; - } while (u > 0); - std::reverse(buf, e); - for (auto p = buf; p != e && is.rdstate() == std::ios::goodbit; ++p) - read(is, *p); +template +void read(std::basic_istream& is, ru a0, Args&&... args); + +template +void read(std::basic_istream& is, int a0, Args&&... args); + +template +void read(std::basic_istream& is, rld a0, Args&&... args); + +template +void read(std::basic_istream& is, CharT a0, Args&&... args) { + // No-op if a0 == CharT{} + if (a0 != CharT{}) { + auto ic = is.peek(); + if (Traits::eq_int_type(ic, Traits::eof())) { + is.setstate(std::ios::failbit | std::ios::eofbit); + return; } - if (is.rdstate() == std::ios::goodbit) - read(is, std::forward(args)...); -} - -template -void -read(std::basic_istream& is, rld a0, Args&& ...args) -{ - auto x = read_long_double(is, a0.m, a0.M); - if (is.fail()) - return; - a0.i = x; + if (!Traits::eq(Traits::to_char_type(ic), a0)) { + is.setstate(std::ios::failbit); + return; + } + (void)is.get(); + } + read(is, std::forward(args)...); +} + +template +void read(std::basic_istream& is, rs a0, Args&&... args) { + auto x = read_signed(is, a0.m, a0.M); + if (is.fail()) + return; + a0.i = x; + read(is, std::forward(args)...); +} + +template +void read(std::basic_istream& is, ru a0, Args&&... args) { + auto x = read_unsigned(is, a0.m, a0.M); + if (is.fail()) + return; + a0.i = static_cast(x); + read(is, std::forward(args)...); +} + +template +void read(std::basic_istream& is, int a0, Args&&... args) { + if (a0 != -1) { + auto u = static_cast(a0); + CharT buf[std::numeric_limits::digits10 + 2u] = {}; + auto e = buf; + do { + *e++ = static_cast(CharT(u % 10) + CharT{'0'}); + u /= 10; + } while (u > 0); + std::reverse(buf, e); + for (auto p = buf; p != e && is.rdstate() == std::ios::goodbit; ++p) + read(is, *p); + } + if (is.rdstate() == std::ios::goodbit) read(is, std::forward(args)...); } +template +void read(std::basic_istream& is, rld a0, Args&&... args) { + auto x = read_long_double(is, a0.m, a0.M); + if (is.fail()) + return; + a0.i = x; + read(is, std::forward(args)...); +} + template -inline -void -checked_set(T& value, T from, T not_a_value, std::basic_ios& is) -{ - if (!is.fail()) - { - if (value == not_a_value) - value = std::move(from); - else if (value != from) - is.setstate(std::ios::failbit); - } +inline void checked_set(T& value, T from, T not_a_value, std::basic_ios& is) { + if (!is.fail()) { + if (value == not_a_value) + value = std::move(from); + else if (value != from) + is.setstate(std::ios::failbit); + } } -} // namespace detail; +} // namespace detail template > -std::basic_istream& -from_stream(std::basic_istream& is, const CharT* fmt, - fields& fds, std::basic_string* abbrev, - std::chrono::minutes* offset) -{ - using std::numeric_limits; - using std::ios; - using std::chrono::duration; - using std::chrono::duration_cast; - using std::chrono::seconds; - using std::chrono::minutes; - using std::chrono::hours; - using detail::round_i; - typename std::basic_istream::sentry ok{is, true}; - if (ok) - { - date::detail::save_istream ss(is); - is.fill(' '); - is.flags(std::ios::skipws | std::ios::dec); - is.width(0); +std::basic_istream& from_stream(std::basic_istream& is, const CharT* fmt, + fields& fds, std::basic_string* abbrev, + std::chrono::minutes* offset) { + using detail::round_i; + using std::ios; + using std::numeric_limits; + using std::chrono::duration; + using std::chrono::duration_cast; + using std::chrono::hours; + using std::chrono::minutes; + using std::chrono::seconds; + typename std::basic_istream::sentry ok{is, true}; + if (ok) { + date::detail::save_istream ss(is); + is.fill(' '); + is.flags(std::ios::skipws | std::ios::dec); + is.width(0); #if !ONLY_C_LOCALE - auto& f = std::use_facet>(is.getloc()); - std::tm tm{}; + auto& f = std::use_facet>(is.getloc()); + std::tm tm{}; #endif - const CharT* command = nullptr; - auto modified = CharT{}; - auto width = -1; - - CONSTDATA int not_a_year = numeric_limits::min(); - CONSTDATA int not_a_2digit_year = 100; - CONSTDATA int not_a_century = not_a_year / 100; - CONSTDATA int not_a_month = 0; - CONSTDATA int not_a_day = 0; - CONSTDATA int not_a_hour = numeric_limits::min(); - CONSTDATA int not_a_hour_12_value = 0; - CONSTDATA int not_a_minute = not_a_hour; - CONSTDATA Duration not_a_second = Duration::min(); - CONSTDATA int not_a_doy = -1; - CONSTDATA int not_a_weekday = 8; - CONSTDATA int not_a_week_num = 100; - CONSTDATA int not_a_ampm = -1; - CONSTDATA minutes not_a_offset = minutes::min(); - - int Y = not_a_year; // c, F, Y * - int y = not_a_2digit_year; // D, x, y * - int g = not_a_2digit_year; // g * - int G = not_a_year; // G * - int C = not_a_century; // C * - int m = not_a_month; // b, B, h, m, c, D, F, x * - int d = not_a_day; // c, d, D, e, F, x * - int j = not_a_doy; // j * - int wd = not_a_weekday; // a, A, u, w * - int H = not_a_hour; // c, H, R, T, X * - int I = not_a_hour_12_value; // I, r * - int p = not_a_ampm; // p, r * - int M = not_a_minute; // c, M, r, R, T, X * - Duration s = not_a_second; // c, r, S, T, X * - int U = not_a_week_num; // U * - int V = not_a_week_num; // V * - int W = not_a_week_num; // W * - std::basic_string temp_abbrev; // Z * - minutes temp_offset = not_a_offset; // z * - - using detail::read; - using detail::rs; - using detail::ru; - using detail::rld; - using detail::checked_set; - for (; *fmt != CharT{} && !is.fail(); ++fmt) - { - switch (*fmt) - { - case 'a': - case 'A': - case 'u': - case 'w': // wd: a, A, u, w - if (command) - { - int trial_wd = not_a_weekday; - if (*fmt == 'a' || *fmt == 'A') - { - if (modified == CharT{}) - { + const CharT* command = nullptr; + auto modified = CharT{}; + auto width = -1; + + CONSTDATA int not_a_year = numeric_limits::min(); + CONSTDATA int not_a_2digit_year = 100; + CONSTDATA int not_a_century = not_a_year / 100; + CONSTDATA int not_a_month = 0; + CONSTDATA int not_a_day = 0; + CONSTDATA int not_a_hour = numeric_limits::min(); + CONSTDATA int not_a_hour_12_value = 0; + CONSTDATA int not_a_minute = not_a_hour; + CONSTDATA Duration not_a_second = Duration::min(); + CONSTDATA int not_a_doy = -1; + CONSTDATA int not_a_weekday = 8; + CONSTDATA int not_a_week_num = 100; + CONSTDATA int not_a_ampm = -1; + CONSTDATA minutes not_a_offset = minutes::min(); + + int Y = not_a_year; // c, F, Y * + int y = not_a_2digit_year; // D, x, y * + int g = not_a_2digit_year; // g * + int G = not_a_year; // G * + int C = not_a_century; // C * + int m = not_a_month; // b, B, h, m, c, D, F, x * + int d = not_a_day; // c, d, D, e, F, x * + int j = not_a_doy; // j * + int wd = not_a_weekday; // a, A, u, w * + int H = not_a_hour; // c, H, R, T, X * + int I = not_a_hour_12_value; // I, r * + int p = not_a_ampm; // p, r * + int M = not_a_minute; // c, M, r, R, T, X * + Duration s = not_a_second; // c, r, S, T, X * + int U = not_a_week_num; // U * + int V = not_a_week_num; // V * + int W = not_a_week_num; // W * + std::basic_string temp_abbrev; // Z * + minutes temp_offset = not_a_offset; // z * + + using detail::checked_set; + using detail::read; + using detail::rld; + using detail::rs; + using detail::ru; + for (; *fmt != CharT{} && !is.fail(); ++fmt) { + switch (*fmt) { + case 'a': + case 'A': + case 'u': + case 'w': // wd: a, A, u, w + if (command) { + int trial_wd = not_a_weekday; + if (*fmt == 'a' || *fmt == 'A') { + if (modified == CharT{}) { #if !ONLY_C_LOCALE - ios::iostate err = ios::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt+1); - is.setstate(err); - if (!is.fail()) - trial_wd = tm.tm_wday; + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt + 1); + is.setstate(err); + if (!is.fail()) + trial_wd = tm.tm_wday; #else - auto nm = detail::weekday_names(); - auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; - if (!is.fail()) - trial_wd = i % 7; + auto nm = detail::weekday_names(); + auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; + if (!is.fail()) + trial_wd = i % 7; #endif - } - else - read(is, CharT{'%'}, width, modified, *fmt); - } - else // *fmt == 'u' || *fmt == 'w' - { + } else + read(is, CharT{'%'}, width, modified, *fmt); + } else // *fmt == 'u' || *fmt == 'w' + { #if !ONLY_C_LOCALE - if (modified == CharT{}) + if (modified == CharT{}) #else - if (modified != CharT{'E'}) + if (modified != CharT{'E'}) #endif - { - read(is, ru{trial_wd, 1, width == -1 ? - 1u : static_cast(width)}); - if (!is.fail()) - { - if (*fmt == 'u') - { - if (!(1 <= trial_wd && trial_wd <= 7)) - { - trial_wd = not_a_weekday; - is.setstate(ios::failbit); - } - else if (trial_wd == 7) - trial_wd = 0; - } - else // *fmt == 'w' - { - if (!(0 <= trial_wd && trial_wd <= 6)) - { - trial_wd = not_a_weekday; - is.setstate(ios::failbit); - } - } - } - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) - { - ios::iostate err = ios::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt+1); - is.setstate(err); - if (!is.fail()) - trial_wd = tm.tm_wday; - } -#endif - else - read(is, CharT{'%'}, width, modified, *fmt); + { + read(is, ru{trial_wd, 1, width == -1 ? 1u : static_cast(width)}); + if (!is.fail()) { + if (*fmt == 'u') { + if (!(1 <= trial_wd && trial_wd <= 7)) { + trial_wd = not_a_weekday; + is.setstate(ios::failbit); + } else if (trial_wd == 7) + trial_wd = 0; + } else // *fmt == 'w' + { + if (!(0 <= trial_wd && trial_wd <= 6)) { + trial_wd = not_a_weekday; + is.setstate(ios::failbit); } - if (trial_wd != not_a_weekday) - checked_set(wd, trial_wd, not_a_weekday, is); + } } - else // !command - read(is, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - break; - case 'b': - case 'B': - case 'h': - if (command) - { - if (modified == CharT{}) - { - int ttm = not_a_month; + } #if !ONLY_C_LOCALE - ios::iostate err = ios::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt+1); - if ((err & ios::failbit) == 0) - ttm = tm.tm_mon + 1; - is.setstate(err); + else if (modified == CharT{'O'}) { + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt + 1); + is.setstate(err); + if (!is.fail()) + trial_wd = tm.tm_wday; + } +#endif + else + read(is, CharT{'%'}, width, modified, *fmt); + } + if (trial_wd != not_a_weekday) + checked_set(wd, trial_wd, not_a_weekday, is); + } else // !command + read(is, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + break; + case 'b': + case 'B': + case 'h': + if (command) { + if (modified == CharT{}) { + int ttm = not_a_month; +#if !ONLY_C_LOCALE + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt + 1); + if ((err & ios::failbit) == 0) + ttm = tm.tm_mon + 1; + is.setstate(err); #else - auto nm = detail::month_names(); - auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; - if (!is.fail()) - ttm = i % 12 + 1; + auto nm = detail::month_names(); + auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; + if (!is.fail()) + ttm = i % 12 + 1; #endif - checked_set(m, ttm, not_a_month, is); - } - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'c': - if (command) - { - if (modified != CharT{'O'}) - { + checked_set(m, ttm, not_a_month, is); + } else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'c': + if (command) { + if (modified != CharT{'O'}) { #if !ONLY_C_LOCALE - ios::iostate err = ios::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt+1); - if ((err & ios::failbit) == 0) - { - checked_set(Y, tm.tm_year + 1900, not_a_year, is); - checked_set(m, tm.tm_mon + 1, not_a_month, is); - checked_set(d, tm.tm_mday, not_a_day, is); - checked_set(H, tm.tm_hour, not_a_hour, is); - checked_set(M, tm.tm_min, not_a_minute, is); - checked_set(s, duration_cast(seconds{tm.tm_sec}), - not_a_second, is); - } - is.setstate(err); + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt + 1); + if ((err & ios::failbit) == 0) { + checked_set(Y, tm.tm_year + 1900, not_a_year, is); + checked_set(m, tm.tm_mon + 1, not_a_month, is); + checked_set(d, tm.tm_mday, not_a_day, is); + checked_set(H, tm.tm_hour, not_a_hour, is); + checked_set(M, tm.tm_min, not_a_minute, is); + checked_set(s, duration_cast(seconds{tm.tm_sec}), not_a_second, is); + } + is.setstate(err); #else - // "%a %b %e %T %Y" - auto nm = detail::weekday_names(); - auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; - checked_set(wd, static_cast(i % 7), not_a_weekday, is); - ws(is); - nm = detail::month_names(); - i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; - checked_set(m, static_cast(i % 12 + 1), not_a_month, is); - ws(is); - int td = not_a_day; - read(is, rs{td, 1, 2}); - checked_set(d, td, not_a_day, is); - ws(is); - using dfs = detail::decimal_format_seconds; - CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; - int tH; - int tM; - long double S; - read(is, ru{tH, 1, 2}, CharT{':'}, ru{tM, 1, 2}, - CharT{':'}, rld{S, 1, w}); - checked_set(H, tH, not_a_hour, is); - checked_set(M, tM, not_a_minute, is); - checked_set(s, round_i(duration{S}), - not_a_second, is); - ws(is); - int tY = not_a_year; - read(is, rs{tY, 1, 4u}); - checked_set(Y, tY, not_a_year, is); + // "%a %b %e %T %Y" + auto nm = detail::weekday_names(); + auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; + checked_set(wd, static_cast(i % 7), not_a_weekday, is); + ws(is); + nm = detail::month_names(); + i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; + checked_set(m, static_cast(i % 12 + 1), not_a_month, is); + ws(is); + int td = not_a_day; + read(is, rs{td, 1, 2}); + checked_set(d, td, not_a_day, is); + ws(is); + using dfs = detail::decimal_format_seconds; + CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; + int tH; + int tM; + long double S; + read(is, ru{tH, 1, 2}, CharT{':'}, ru{tM, 1, 2}, CharT{':'}, rld{S, 1, w}); + checked_set(H, tH, not_a_hour, is); + checked_set(M, tM, not_a_minute, is); + checked_set(s, round_i(duration{S}), not_a_second, is); + ws(is); + int tY = not_a_year; + read(is, rs{tY, 1, 4u}); + checked_set(Y, tY, not_a_year, is); #endif - } - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'x': - if (command) - { - if (modified != CharT{'O'}) - { + } else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'x': + if (command) { + if (modified != CharT{'O'}) { #if !ONLY_C_LOCALE - ios::iostate err = ios::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt+1); - if ((err & ios::failbit) == 0) - { - checked_set(Y, tm.tm_year + 1900, not_a_year, is); - checked_set(m, tm.tm_mon + 1, not_a_month, is); - checked_set(d, tm.tm_mday, not_a_day, is); - } - is.setstate(err); + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt + 1); + if ((err & ios::failbit) == 0) { + checked_set(Y, tm.tm_year + 1900, not_a_year, is); + checked_set(m, tm.tm_mon + 1, not_a_month, is); + checked_set(d, tm.tm_mday, not_a_day, is); + } + is.setstate(err); #else - // "%m/%d/%y" - int ty = not_a_2digit_year; - int tm = not_a_month; - int td = not_a_day; - read(is, ru{tm, 1, 2}, CharT{'/'}, ru{td, 1, 2}, CharT{'/'}, - rs{ty, 1, 2}); - checked_set(y, ty, not_a_2digit_year, is); - checked_set(m, tm, not_a_month, is); - checked_set(d, td, not_a_day, is); + // "%m/%d/%y" + int ty = not_a_2digit_year; + int tm = not_a_month; + int td = not_a_day; + read(is, ru{tm, 1, 2}, CharT{'/'}, ru{td, 1, 2}, CharT{'/'}, rs{ty, 1, 2}); + checked_set(y, ty, not_a_2digit_year, is); + checked_set(m, tm, not_a_month, is); + checked_set(d, td, not_a_day, is); #endif - } - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'X': - if (command) - { - if (modified != CharT{'O'}) - { + } else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'X': + if (command) { + if (modified != CharT{'O'}) { #if !ONLY_C_LOCALE - ios::iostate err = ios::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt+1); - if ((err & ios::failbit) == 0) - { - checked_set(H, tm.tm_hour, not_a_hour, is); - checked_set(M, tm.tm_min, not_a_minute, is); - checked_set(s, duration_cast(seconds{tm.tm_sec}), - not_a_second, is); - } - is.setstate(err); + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt + 1); + if ((err & ios::failbit) == 0) { + checked_set(H, tm.tm_hour, not_a_hour, is); + checked_set(M, tm.tm_min, not_a_minute, is); + checked_set(s, duration_cast(seconds{tm.tm_sec}), not_a_second, is); + } + is.setstate(err); #else - // "%T" - using dfs = detail::decimal_format_seconds; - CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; - int tH = not_a_hour; - int tM = not_a_minute; - long double S; - read(is, ru{tH, 1, 2}, CharT{':'}, ru{tM, 1, 2}, - CharT{':'}, rld{S, 1, w}); - checked_set(H, tH, not_a_hour, is); - checked_set(M, tM, not_a_minute, is); - checked_set(s, round_i(duration{S}), - not_a_second, is); + // "%T" + using dfs = detail::decimal_format_seconds; + CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; + int tH = not_a_hour; + int tM = not_a_minute; + long double S; + read(is, ru{tH, 1, 2}, CharT{':'}, ru{tM, 1, 2}, CharT{':'}, rld{S, 1, w}); + checked_set(H, tH, not_a_hour, is); + checked_set(M, tM, not_a_minute, is); + checked_set(s, round_i(duration{S}), not_a_second, is); #endif - } - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'C': - if (command) - { - int tC = not_a_century; + } else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'C': + if (command) { + int tC = not_a_century; #if !ONLY_C_LOCALE - if (modified == CharT{}) - { + if (modified == CharT{}) { #endif - read(is, rs{tC, 1, width == -1 ? 2u : static_cast(width)}); + read(is, rs{tC, 1, width == -1 ? 2u : static_cast(width)}); #if !ONLY_C_LOCALE - } - else - { - ios::iostate err = ios::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt+1); - if ((err & ios::failbit) == 0) - { - auto tY = tm.tm_year + 1900; - tC = (tY >= 0 ? tY : tY-99) / 100; - } - is.setstate(err); - } + } else { + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt + 1); + if ((err & ios::failbit) == 0) { + auto tY = tm.tm_year + 1900; + tC = (tY >= 0 ? tY : tY - 99) / 100; + } + is.setstate(err); + } #endif - checked_set(C, tC, not_a_century, is); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'D': - if (command) - { - if (modified == CharT{}) - { - int tn = not_a_month; - int td = not_a_day; - int ty = not_a_2digit_year; - read(is, ru{tn, 1, 2}, CharT{'\0'}, CharT{'/'}, CharT{'\0'}, - ru{td, 1, 2}, CharT{'\0'}, CharT{'/'}, CharT{'\0'}, - rs{ty, 1, 2}); - checked_set(y, ty, not_a_2digit_year, is); - checked_set(m, tn, not_a_month, is); - checked_set(d, td, not_a_day, is); - } - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'F': - if (command) - { - if (modified == CharT{}) - { - int tY = not_a_year; - int tn = not_a_month; - int td = not_a_day; - read(is, rs{tY, 1, width == -1 ? 4u : static_cast(width)}, - CharT{'-'}, ru{tn, 1, 2}, CharT{'-'}, ru{td, 1, 2}); - checked_set(Y, tY, not_a_year, is); - checked_set(m, tn, not_a_month, is); - checked_set(d, td, not_a_day, is); - } - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'd': - case 'e': - if (command) - { + checked_set(C, tC, not_a_century, is); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'D': + if (command) { + if (modified == CharT{}) { + int tn = not_a_month; + int td = not_a_day; + int ty = not_a_2digit_year; + read(is, ru{tn, 1, 2}, CharT{'\0'}, CharT{'/'}, CharT{'\0'}, ru{td, 1, 2}, CharT{'\0'}, CharT{'/'}, + CharT{'\0'}, rs{ty, 1, 2}); + checked_set(y, ty, not_a_2digit_year, is); + checked_set(m, tn, not_a_month, is); + checked_set(d, td, not_a_day, is); + } else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'F': + if (command) { + if (modified == CharT{}) { + int tY = not_a_year; + int tn = not_a_month; + int td = not_a_day; + read(is, rs{tY, 1, width == -1 ? 4u : static_cast(width)}, CharT{'-'}, ru{tn, 1, 2}, CharT{'-'}, + ru{td, 1, 2}); + checked_set(Y, tY, not_a_year, is); + checked_set(m, tn, not_a_month, is); + checked_set(d, td, not_a_day, is); + } else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'd': + case 'e': + if (command) { #if !ONLY_C_LOCALE - if (modified == CharT{}) + if (modified == CharT{}) #else - if (modified != CharT{'E'}) + if (modified != CharT{'E'}) #endif - { - int td = not_a_day; - read(is, rs{td, 1, width == -1 ? 2u : static_cast(width)}); - checked_set(d, td, not_a_day, is); - } + { + int td = not_a_day; + read(is, rs{td, 1, width == -1 ? 2u : static_cast(width)}); + checked_set(d, td, not_a_day, is); + } #if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) - { - ios::iostate err = ios::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt+1); - command = nullptr; - width = -1; - modified = CharT{}; - if ((err & ios::failbit) == 0) - checked_set(d, tm.tm_mday, not_a_day, is); - is.setstate(err); - } + else if (modified == CharT{'O'}) { + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt + 1); + command = nullptr; + width = -1; + modified = CharT{}; + if ((err & ios::failbit) == 0) + checked_set(d, tm.tm_mday, not_a_day, is); + is.setstate(err); + } #endif - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'H': - if (command) - { + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'H': + if (command) { #if !ONLY_C_LOCALE - if (modified == CharT{}) + if (modified == CharT{}) #else - if (modified != CharT{'E'}) + if (modified != CharT{'E'}) #endif - { - int tH = not_a_hour; - read(is, ru{tH, 1, width == -1 ? 2u : static_cast(width)}); - checked_set(H, tH, not_a_hour, is); - } + { + int tH = not_a_hour; + read(is, ru{tH, 1, width == -1 ? 2u : static_cast(width)}); + checked_set(H, tH, not_a_hour, is); + } #if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) - { - ios::iostate err = ios::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt+1); - if ((err & ios::failbit) == 0) - checked_set(H, tm.tm_hour, not_a_hour, is); - is.setstate(err); - } + else if (modified == CharT{'O'}) { + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt + 1); + if ((err & ios::failbit) == 0) + checked_set(H, tm.tm_hour, not_a_hour, is); + is.setstate(err); + } #endif - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'I': - if (command) - { - if (modified == CharT{}) - { - int tI = not_a_hour_12_value; - // reads in an hour into I, but most be in [1, 12] - read(is, rs{tI, 1, width == -1 ? 2u : static_cast(width)}); - if (!(1 <= tI && tI <= 12)) - is.setstate(ios::failbit); - checked_set(I, tI, not_a_hour_12_value, is); - } - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'j': - if (command) - { - if (modified == CharT{}) - { - int tj = not_a_doy; - read(is, ru{tj, 1, width == -1 ? 3u : static_cast(width)}); - checked_set(j, tj, not_a_doy, is); - } - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'M': - if (command) - { + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'I': + if (command) { + if (modified == CharT{}) { + int tI = not_a_hour_12_value; + // reads in an hour into I, but most be in [1, 12] + read(is, rs{tI, 1, width == -1 ? 2u : static_cast(width)}); + if (!(1 <= tI && tI <= 12)) + is.setstate(ios::failbit); + checked_set(I, tI, not_a_hour_12_value, is); + } else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'j': + if (command) { + if (modified == CharT{}) { + int tj = not_a_doy; + read(is, ru{tj, 1, width == -1 ? 3u : static_cast(width)}); + checked_set(j, tj, not_a_doy, is); + } else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'M': + if (command) { #if !ONLY_C_LOCALE - if (modified == CharT{}) + if (modified == CharT{}) #else - if (modified != CharT{'E'}) + if (modified != CharT{'E'}) #endif - { - int tM = not_a_minute; - read(is, ru{tM, 1, width == -1 ? 2u : static_cast(width)}); - checked_set(M, tM, not_a_minute, is); - } + { + int tM = not_a_minute; + read(is, ru{tM, 1, width == -1 ? 2u : static_cast(width)}); + checked_set(M, tM, not_a_minute, is); + } #if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) - { - ios::iostate err = ios::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt+1); - if ((err & ios::failbit) == 0) - checked_set(M, tm.tm_min, not_a_minute, is); - is.setstate(err); - } + else if (modified == CharT{'O'}) { + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt + 1); + if ((err & ios::failbit) == 0) + checked_set(M, tm.tm_min, not_a_minute, is); + is.setstate(err); + } #endif - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'm': - if (command) - { + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'm': + if (command) { #if !ONLY_C_LOCALE - if (modified == CharT{}) + if (modified == CharT{}) #else - if (modified != CharT{'E'}) + if (modified != CharT{'E'}) #endif - { - int tn = not_a_month; - read(is, rs{tn, 1, width == -1 ? 2u : static_cast(width)}); - checked_set(m, tn, not_a_month, is); - } + { + int tn = not_a_month; + read(is, rs{tn, 1, width == -1 ? 2u : static_cast(width)}); + checked_set(m, tn, not_a_month, is); + } #if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) - { - ios::iostate err = ios::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt+1); - if ((err & ios::failbit) == 0) - checked_set(m, tm.tm_mon + 1, not_a_month, is); - is.setstate(err); - } + else if (modified == CharT{'O'}) { + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt + 1); + if ((err & ios::failbit) == 0) + checked_set(m, tm.tm_mon + 1, not_a_month, is); + is.setstate(err); + } #endif - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'n': - case 't': - if (command) - { - if (modified == CharT{}) - { - // %n matches a single white space character - // %t matches 0 or 1 white space characters - auto ic = is.peek(); - if (Traits::eq_int_type(ic, Traits::eof())) - { - ios::iostate err = ios::eofbit; - if (*fmt == 'n') - err |= ios::failbit; - is.setstate(err); - break; - } - if (isspace(ic)) - { - (void)is.get(); - } - else if (*fmt == 'n') - is.setstate(ios::failbit); - } - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'n': + case 't': + if (command) { + if (modified == CharT{}) { + // %n matches a single white space character + // %t matches 0 or 1 white space characters + auto ic = is.peek(); + if (Traits::eq_int_type(ic, Traits::eof())) { + ios::iostate err = ios::eofbit; + if (*fmt == 'n') + err |= ios::failbit; + is.setstate(err); break; - case 'p': - if (command) - { - if (modified == CharT{}) - { - int tp = not_a_ampm; + } + if (isspace(ic)) { + (void)is.get(); + } else if (*fmt == 'n') + is.setstate(ios::failbit); + } else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'p': + if (command) { + if (modified == CharT{}) { + int tp = not_a_ampm; #if !ONLY_C_LOCALE - tm = std::tm{}; - tm.tm_hour = 1; - ios::iostate err = ios::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt+1); - is.setstate(err); - if (tm.tm_hour == 1) - tp = 0; - else if (tm.tm_hour == 13) - tp = 1; - else - is.setstate(err); + tm = std::tm{}; + tm.tm_hour = 1; + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt + 1); + is.setstate(err); + if (tm.tm_hour == 1) + tp = 0; + else if (tm.tm_hour == 13) + tp = 1; + else + is.setstate(err); #else - auto nm = detail::ampm_names(); - auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; - tp = static_cast(i); + auto nm = detail::ampm_names(); + auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; + tp = static_cast(i); #endif - checked_set(p, tp, not_a_ampm, is); - } - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - - break; - case 'r': - if (command) - { - if (modified == CharT{}) - { + checked_set(p, tp, not_a_ampm, is); + } else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + + break; + case 'r': + if (command) { + if (modified == CharT{}) { #if !ONLY_C_LOCALE - ios::iostate err = ios::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt+1); - if ((err & ios::failbit) == 0) - { - checked_set(H, tm.tm_hour, not_a_hour, is); - checked_set(M, tm.tm_min, not_a_hour, is); - checked_set(s, duration_cast(seconds{tm.tm_sec}), - not_a_second, is); - } - is.setstate(err); + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt + 1); + if ((err & ios::failbit) == 0) { + checked_set(H, tm.tm_hour, not_a_hour, is); + checked_set(M, tm.tm_min, not_a_hour, is); + checked_set(s, duration_cast(seconds{tm.tm_sec}), not_a_second, is); + } + is.setstate(err); #else - // "%I:%M:%S %p" - using dfs = detail::decimal_format_seconds; - CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; - long double S; - int tI = not_a_hour_12_value; - int tM = not_a_minute; - read(is, ru{tI, 1, 2}, CharT{':'}, ru{tM, 1, 2}, - CharT{':'}, rld{S, 1, w}); - checked_set(I, tI, not_a_hour_12_value, is); - checked_set(M, tM, not_a_minute, is); - checked_set(s, round_i(duration{S}), - not_a_second, is); - ws(is); - auto nm = detail::ampm_names(); - auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; - checked_set(p, static_cast(i), not_a_ampm, is); + // "%I:%M:%S %p" + using dfs = detail::decimal_format_seconds; + CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; + long double S; + int tI = not_a_hour_12_value; + int tM = not_a_minute; + read(is, ru{tI, 1, 2}, CharT{':'}, ru{tM, 1, 2}, CharT{':'}, rld{S, 1, w}); + checked_set(I, tI, not_a_hour_12_value, is); + checked_set(M, tM, not_a_minute, is); + checked_set(s, round_i(duration{S}), not_a_second, is); + ws(is); + auto nm = detail::ampm_names(); + auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; + checked_set(p, static_cast(i), not_a_ampm, is); #endif - } - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'R': - if (command) - { - if (modified == CharT{}) - { - int tH = not_a_hour; - int tM = not_a_minute; - read(is, ru{tH, 1, 2}, CharT{'\0'}, CharT{':'}, CharT{'\0'}, - ru{tM, 1, 2}, CharT{'\0'}); - checked_set(H, tH, not_a_hour, is); - checked_set(M, tM, not_a_minute, is); - } - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'S': - if (command) - { - #if !ONLY_C_LOCALE - if (modified == CharT{}) + } else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'R': + if (command) { + if (modified == CharT{}) { + int tH = not_a_hour; + int tM = not_a_minute; + read(is, ru{tH, 1, 2}, CharT{'\0'}, CharT{':'}, CharT{'\0'}, ru{tM, 1, 2}, CharT{'\0'}); + checked_set(H, tH, not_a_hour, is); + checked_set(M, tM, not_a_minute, is); + } else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'S': + if (command) { +#if !ONLY_C_LOCALE + if (modified == CharT{}) #else - if (modified != CharT{'E'}) + if (modified != CharT{'E'}) #endif - { - using dfs = detail::decimal_format_seconds; - CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; - long double S; - read(is, rld{S, 1, width == -1 ? w : static_cast(width)}); - checked_set(s, round_i(duration{S}), - not_a_second, is); - } + { + using dfs = detail::decimal_format_seconds; + CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; + long double S; + read(is, rld{S, 1, width == -1 ? w : static_cast(width)}); + checked_set(s, round_i(duration{S}), not_a_second, is); + } #if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) - { - ios::iostate err = ios::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt+1); - if ((err & ios::failbit) == 0) - checked_set(s, duration_cast(seconds{tm.tm_sec}), - not_a_second, is); - is.setstate(err); - } + else if (modified == CharT{'O'}) { + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt + 1); + if ((err & ios::failbit) == 0) + checked_set(s, duration_cast(seconds{tm.tm_sec}), not_a_second, is); + is.setstate(err); + } #endif - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'T': - if (command) - { - if (modified == CharT{}) - { - using dfs = detail::decimal_format_seconds; - CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; - int tH = not_a_hour; - int tM = not_a_minute; - long double S; - read(is, ru{tH, 1, 2}, CharT{':'}, ru{tM, 1, 2}, - CharT{':'}, rld{S, 1, w}); - checked_set(H, tH, not_a_hour, is); - checked_set(M, tM, not_a_minute, is); - checked_set(s, round_i(duration{S}), - not_a_second, is); - } - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'Y': - if (command) - { + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'T': + if (command) { + if (modified == CharT{}) { + using dfs = detail::decimal_format_seconds; + CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; + int tH = not_a_hour; + int tM = not_a_minute; + long double S; + read(is, ru{tH, 1, 2}, CharT{':'}, ru{tM, 1, 2}, CharT{':'}, rld{S, 1, w}); + checked_set(H, tH, not_a_hour, is); + checked_set(M, tM, not_a_minute, is); + checked_set(s, round_i(duration{S}), not_a_second, is); + } else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'Y': + if (command) { #if !ONLY_C_LOCALE - if (modified == CharT{}) + if (modified == CharT{}) #else - if (modified != CharT{'O'}) + if (modified != CharT{'O'}) #endif - { - int tY = not_a_year; - read(is, rs{tY, 1, width == -1 ? 4u : static_cast(width)}); - checked_set(Y, tY, not_a_year, is); - } + { + int tY = not_a_year; + read(is, rs{tY, 1, width == -1 ? 4u : static_cast(width)}); + checked_set(Y, tY, not_a_year, is); + } #if !ONLY_C_LOCALE - else if (modified == CharT{'E'}) - { - ios::iostate err = ios::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt+1); - if ((err & ios::failbit) == 0) - checked_set(Y, tm.tm_year + 1900, not_a_year, is); - is.setstate(err); - } + else if (modified == CharT{'E'}) { + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt + 1); + if ((err & ios::failbit) == 0) + checked_set(Y, tm.tm_year + 1900, not_a_year, is); + is.setstate(err); + } #endif - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'y': - if (command) - { + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'y': + if (command) { #if !ONLY_C_LOCALE - if (modified == CharT{}) + if (modified == CharT{}) #endif - { - int ty = not_a_2digit_year; - read(is, ru{ty, 1, width == -1 ? 2u : static_cast(width)}); - checked_set(y, ty, not_a_2digit_year, is); - } + { + int ty = not_a_2digit_year; + read(is, ru{ty, 1, width == -1 ? 2u : static_cast(width)}); + checked_set(y, ty, not_a_2digit_year, is); + } #if !ONLY_C_LOCALE - else - { - ios::iostate err = ios::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt+1); - if ((err & ios::failbit) == 0) - checked_set(Y, tm.tm_year + 1900, not_a_year, is); - is.setstate(err); - } + else { + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt + 1); + if ((err & ios::failbit) == 0) + checked_set(Y, tm.tm_year + 1900, not_a_year, is); + is.setstate(err); + } #endif - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'g': - if (command) - { - if (modified == CharT{}) - { - int tg = not_a_2digit_year; - read(is, ru{tg, 1, width == -1 ? 2u : static_cast(width)}); - checked_set(g, tg, not_a_2digit_year, is); - } - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'G': - if (command) - { - if (modified == CharT{}) - { - int tG = not_a_year; - read(is, rs{tG, 1, width == -1 ? 4u : static_cast(width)}); - checked_set(G, tG, not_a_year, is); - } - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'U': - if (command) - { - if (modified == CharT{}) - { - int tU = not_a_week_num; - read(is, ru{tU, 1, width == -1 ? 2u : static_cast(width)}); - checked_set(U, tU, not_a_week_num, is); - } - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'V': - if (command) - { - if (modified == CharT{}) - { - int tV = not_a_week_num; - read(is, ru{tV, 1, width == -1 ? 2u : static_cast(width)}); - checked_set(V, tV, not_a_week_num, is); - } - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'W': - if (command) - { - if (modified == CharT{}) - { - int tW = not_a_week_num; - read(is, ru{tW, 1, width == -1 ? 2u : static_cast(width)}); - checked_set(W, tW, not_a_week_num, is); - } - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'E': - case 'O': - if (command) - { - if (modified == CharT{}) - { - modified = *fmt; - } - else - { - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - } - else - read(is, *fmt); - break; - case '%': - if (command) - { - if (modified == CharT{}) - read(is, *fmt); - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - command = fmt; - break; - case 'z': - if (command) - { - int tH, tM; - minutes toff = not_a_offset; - bool neg = false; - auto ic = is.peek(); - if (!Traits::eq_int_type(ic, Traits::eof())) - { - auto c = static_cast(Traits::to_char_type(ic)); - if (c == '-') - neg = true; - } - if (modified == CharT{}) - { - read(is, rs{tH, 2, 2}); - if (!is.fail()) - toff = hours{std::abs(tH)}; - if (is.good()) - { - ic = is.peek(); - if (!Traits::eq_int_type(ic, Traits::eof())) - { - auto c = static_cast(Traits::to_char_type(ic)); - if ('0' <= c && c <= '9') - { - read(is, ru{tM, 2, 2}); - if (!is.fail()) - toff += minutes{tM}; - } - } - } - } - else - { - read(is, rs{tH, 1, 2}); - if (!is.fail()) - toff = hours{std::abs(tH)}; - if (is.good()) - { - ic = is.peek(); - if (!Traits::eq_int_type(ic, Traits::eof())) - { - auto c = static_cast(Traits::to_char_type(ic)); - if (c == ':') - { - (void)is.get(); - read(is, ru{tM, 2, 2}); - if (!is.fail()) - toff += minutes{tM}; - } - } - } - } - if (neg) - toff = -toff; - checked_set(temp_offset, toff, not_a_offset, is); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - case 'Z': - if (command) - { - if (modified == CharT{}) - { - std::basic_string buf; - while (is.rdstate() == std::ios::goodbit) - { - auto i = is.rdbuf()->sgetc(); - if (Traits::eq_int_type(i, Traits::eof())) - { - is.setstate(ios::eofbit); - break; - } - auto wc = Traits::to_char_type(i); - auto c = static_cast(wc); - // is c a valid time zone name or abbreviation character? - if (!(CharT{1} < wc && wc < CharT{127}) || !(isalnum(c) || - c == '_' || c == '/' || c == '-' || c == '+')) - break; - buf.push_back(c); - is.rdbuf()->sbumpc(); - } - if (buf.empty()) - is.setstate(ios::failbit); - checked_set(temp_abbrev, buf, {}, is); - } - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - else - read(is, *fmt); - break; - default: - if (command) - { - if (width == -1 && modified == CharT{} && '0' <= *fmt && *fmt <= '9') - { - width = static_cast(*fmt) - '0'; - while ('0' <= fmt[1] && fmt[1] <= '9') - width = 10*width + static_cast(*++fmt) - '0'; - } - else - { - if (modified == CharT{}) - read(is, CharT{'%'}, width, *fmt); - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - } - else // !command - { - if (isspace(static_cast(*fmt))) - { - // space matches 0 or more white space characters - if (is.good()) - ws(is); - } - else - read(is, *fmt); - } - break; + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'g': + if (command) { + if (modified == CharT{}) { + int tg = not_a_2digit_year; + read(is, ru{tg, 1, width == -1 ? 2u : static_cast(width)}); + checked_set(g, tg, not_a_2digit_year, is); + } else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'G': + if (command) { + if (modified == CharT{}) { + int tG = not_a_year; + read(is, rs{tG, 1, width == -1 ? 4u : static_cast(width)}); + checked_set(G, tG, not_a_year, is); + } else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'U': + if (command) { + if (modified == CharT{}) { + int tU = not_a_week_num; + read(is, ru{tU, 1, width == -1 ? 2u : static_cast(width)}); + checked_set(U, tU, not_a_week_num, is); + } else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'V': + if (command) { + if (modified == CharT{}) { + int tV = not_a_week_num; + read(is, ru{tV, 1, width == -1 ? 2u : static_cast(width)}); + checked_set(V, tV, not_a_week_num, is); + } else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'W': + if (command) { + if (modified == CharT{}) { + int tW = not_a_week_num; + read(is, ru{tW, 1, width == -1 ? 2u : static_cast(width)}); + checked_set(W, tW, not_a_week_num, is); + } else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'E': + case 'O': + if (command) { + if (modified == CharT{}) { + modified = *fmt; + } else { + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; } - } - // is.fail() || *fmt == CharT{} - if (is.rdstate() == ios::goodbit && command) - { + } else + read(is, *fmt); + break; + case '%': + if (command) { if (modified == CharT{}) - read(is, CharT{'%'}, width); + read(is, *fmt); else - read(is, CharT{'%'}, width, modified); - } - if (!is.fail()) - { - if (y != not_a_2digit_year) - { - // Convert y and an optional C to Y - if (!(0 <= y && y <= 99)) - goto broken; - if (C == not_a_century) - { - if (Y == not_a_year) - { - if (y >= 69) - C = 19; - else - C = 20; - } - else - { - C = (Y >= 0 ? Y : Y-100) / 100; - } - } - int tY; - if (C >= 0) - tY = 100*C + y; - else - tY = 100*(C+1) - (y == 0 ? 100 : y); - if (Y != not_a_year && Y != tY) - goto broken; - Y = tY; + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + command = fmt; + break; + case 'z': + if (command) { + int tH, tM; + minutes toff = not_a_offset; + bool neg = false; + auto ic = is.peek(); + if (!Traits::eq_int_type(ic, Traits::eof())) { + auto c = static_cast(Traits::to_char_type(ic)); + if (c == '-') + neg = true; } - if (g != not_a_2digit_year) - { - // Convert g and an optional C to G - if (!(0 <= g && g <= 99)) - goto broken; - if (C == not_a_century) - { - if (G == not_a_year) - { - if (g >= 69) - C = 19; - else - C = 20; - } - else - { - C = (G >= 0 ? G : G-100) / 100; - } + if (modified == CharT{}) { + read(is, rs{tH, 2, 2}); + if (!is.fail()) + toff = hours{std::abs(tH)}; + if (is.good()) { + ic = is.peek(); + if (!Traits::eq_int_type(ic, Traits::eof())) { + auto c = static_cast(Traits::to_char_type(ic)); + if ('0' <= c && c <= '9') { + read(is, ru{tM, 2, 2}); + if (!is.fail()) + toff += minutes{tM}; + } } - int tG; - if (C >= 0) - tG = 100*C + g; - else - tG = 100*(C+1) - (g == 0 ? 100 : g); - if (G != not_a_year && G != tG) - goto broken; - G = tG; - } - if (Y < static_cast(year::min()) || Y > static_cast(year::max())) - Y = not_a_year; - bool computed = false; - if (G != not_a_year && V != not_a_week_num && wd != not_a_weekday) - { - year_month_day ymd_trial = sys_days(year{G-1}/December/Thursday[last]) + - (Monday-Thursday) + weeks{V-1} + - (weekday{static_cast(wd)}-Monday); - if (Y == not_a_year) - Y = static_cast(ymd_trial.year()); - else if (year{Y} != ymd_trial.year()) - goto broken; - if (m == not_a_month) - m = static_cast(static_cast(ymd_trial.month())); - else if (month(static_cast(m)) != ymd_trial.month()) - goto broken; - if (d == not_a_day) - d = static_cast(static_cast(ymd_trial.day())); - else if (day(static_cast(d)) != ymd_trial.day()) - goto broken; - computed = true; - } - if (Y != not_a_year && U != not_a_week_num && wd != not_a_weekday) - { - year_month_day ymd_trial = sys_days(year{Y}/January/Sunday[1]) + - weeks{U-1} + - (weekday{static_cast(wd)} - Sunday); - if (Y == not_a_year) - Y = static_cast(ymd_trial.year()); - else if (year{Y} != ymd_trial.year()) - goto broken; - if (m == not_a_month) - m = static_cast(static_cast(ymd_trial.month())); - else if (month(static_cast(m)) != ymd_trial.month()) - goto broken; - if (d == not_a_day) - d = static_cast(static_cast(ymd_trial.day())); - else if (day(static_cast(d)) != ymd_trial.day()) - goto broken; - computed = true; - } - if (Y != not_a_year && W != not_a_week_num && wd != not_a_weekday) - { - year_month_day ymd_trial = sys_days(year{Y}/January/Monday[1]) + - weeks{W-1} + - (weekday{static_cast(wd)} - Monday); - if (Y == not_a_year) - Y = static_cast(ymd_trial.year()); - else if (year{Y} != ymd_trial.year()) - goto broken; - if (m == not_a_month) - m = static_cast(static_cast(ymd_trial.month())); - else if (month(static_cast(m)) != ymd_trial.month()) - goto broken; - if (d == not_a_day) - d = static_cast(static_cast(ymd_trial.day())); - else if (day(static_cast(d)) != ymd_trial.day()) - goto broken; - computed = true; - } - if (j != not_a_doy && Y != not_a_year) - { - auto ymd_trial = year_month_day{local_days(year{Y}/1/1) + days{j-1}}; - if (m == 0) - m = static_cast(static_cast(ymd_trial.month())); - else if (month(static_cast(m)) != ymd_trial.month()) - goto broken; - if (d == 0) - d = static_cast(static_cast(ymd_trial.day())); - else if (day(static_cast(d)) != ymd_trial.day()) - goto broken; - j = not_a_doy; - } - auto ymd = year{Y}/m/d; - if (ymd.ok()) - { - if (wd == not_a_weekday) - wd = static_cast((weekday(sys_days(ymd)) - Sunday).count()); - else if (wd != static_cast((weekday(sys_days(ymd)) - Sunday).count())) - goto broken; - if (!computed) - { - if (G != not_a_year || V != not_a_week_num) - { - sys_days sd = ymd; - auto G_trial = year_month_day{sd + days{3}}.year(); - auto start = sys_days((G_trial - years{1})/December/Thursday[last]) + - (Monday - Thursday); - if (sd < start) - { - --G_trial; - if (V != not_a_week_num) - start = sys_days((G_trial - years{1})/December/Thursday[last]) - + (Monday - Thursday); - } - if (G != not_a_year && G != static_cast(G_trial)) - goto broken; - if (V != not_a_week_num) - { - auto V_trial = duration_cast(sd - start).count() + 1; - if (V != V_trial) - goto broken; - } - } - if (U != not_a_week_num) - { - auto start = sys_days(Sunday[1]/January/ymd.year()); - auto U_trial = floor(sys_days(ymd) - start).count() + 1; - if (U != U_trial) - goto broken; - } - if (W != not_a_week_num) - { - auto start = sys_days(Monday[1]/January/ymd.year()); - auto W_trial = floor(sys_days(ymd) - start).count() + 1; - if (W != W_trial) - goto broken; - } + } + } else { + read(is, rs{tH, 1, 2}); + if (!is.fail()) + toff = hours{std::abs(tH)}; + if (is.good()) { + ic = is.peek(); + if (!Traits::eq_int_type(ic, Traits::eof())) { + auto c = static_cast(Traits::to_char_type(ic)); + if (c == ':') { + (void)is.get(); + read(is, ru{tM, 2, 2}); + if (!is.fail()) + toff += minutes{tM}; + } } + } } - fds.ymd = ymd; - if (I != not_a_hour_12_value) - { - if (!(1 <= I && I <= 12)) - goto broken; - if (p != not_a_ampm) - { - // p is in [0, 1] == [AM, PM] - // Store trial H in I - if (I == 12) - --p; - I += p*12; - // Either set H from I or make sure H and I are consistent - if (H == not_a_hour) - H = I; - else if (I != H) - goto broken; - } - else // p == not_a_ampm - { - // if H, make sure H and I could be consistent - if (H != not_a_hour) - { - if (I == 12) - { - if (H != 0 && H != 12) - goto broken; - } - else if (!(I == H || I == H+12)) - { - goto broken; - } - } + if (neg) + toff = -toff; + checked_set(temp_offset, toff, not_a_offset, is); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + case 'Z': + if (command) { + if (modified == CharT{}) { + std::basic_string buf; + while (is.rdstate() == std::ios::goodbit) { + auto i = is.rdbuf()->sgetc(); + if (Traits::eq_int_type(i, Traits::eof())) { + is.setstate(ios::eofbit); + break; } + auto wc = Traits::to_char_type(i); + auto c = static_cast(wc); + // is c a valid time zone name or abbreviation character? + if (!(CharT{1} < wc && wc < CharT{127}) || + !(isalnum(c) || c == '_' || c == '/' || c == '-' || c == '+')) + break; + buf.push_back(c); + is.rdbuf()->sbumpc(); + } + if (buf.empty()) + is.setstate(ios::failbit); + checked_set(temp_abbrev, buf, {}, is); + } else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } else + read(is, *fmt); + break; + default: + if (command) { + if (width == -1 && modified == CharT{} && '0' <= *fmt && *fmt <= '9') { + width = static_cast(*fmt) - '0'; + while ('0' <= fmt[1] && fmt[1] <= '9') + width = 10 * width + static_cast(*++fmt) - '0'; + } else { + if (modified == CharT{}) + read(is, CharT{'%'}, width, *fmt); + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; } - if (H != not_a_hour) - { - fds.has_tod = true; - fds.tod = hh_mm_ss{hours{H}}; - } - if (M != not_a_minute) - { - fds.has_tod = true; - fds.tod.m_ = minutes{M}; + } else // !command + { + if (isspace(static_cast(*fmt))) { + // space matches 0 or more white space characters + if (is.good()) + ws(is); + } else + read(is, *fmt); + } + break; + } + } + // is.fail() || *fmt == CharT{} + if (is.rdstate() == ios::goodbit && command) { + if (modified == CharT{}) + read(is, CharT{'%'}, width); + else + read(is, CharT{'%'}, width, modified); + } + if (!is.fail()) { + if (y != not_a_2digit_year) { + // Convert y and an optional C to Y + if (!(0 <= y && y <= 99)) + goto broken; + if (C == not_a_century) { + if (Y == not_a_year) { + if (y >= 69) + C = 19; + else + C = 20; + } else { + C = (Y >= 0 ? Y : Y - 100) / 100; + } + } + int tY; + if (C >= 0) + tY = 100 * C + y; + else + tY = 100 * (C + 1) - (y == 0 ? 100 : y); + if (Y != not_a_year && Y != tY) + goto broken; + Y = tY; + } + if (g != not_a_2digit_year) { + // Convert g and an optional C to G + if (!(0 <= g && g <= 99)) + goto broken; + if (C == not_a_century) { + if (G == not_a_year) { + if (g >= 69) + C = 19; + else + C = 20; + } else { + C = (G >= 0 ? G : G - 100) / 100; + } + } + int tG; + if (C >= 0) + tG = 100 * C + g; + else + tG = 100 * (C + 1) - (g == 0 ? 100 : g); + if (G != not_a_year && G != tG) + goto broken; + G = tG; + } + if (Y < static_cast(year::min()) || Y > static_cast(year::max())) + Y = not_a_year; + bool computed = false; + if (G != not_a_year && V != not_a_week_num && wd != not_a_weekday) { + year_month_day ymd_trial = sys_days(year{G - 1} / December / Thursday[last]) + (Monday - Thursday) + + weeks{V - 1} + (weekday{static_cast(wd)} - Monday); + if (Y == not_a_year) + Y = static_cast(ymd_trial.year()); + else if (year{Y} != ymd_trial.year()) + goto broken; + if (m == not_a_month) + m = static_cast(static_cast(ymd_trial.month())); + else if (month(static_cast(m)) != ymd_trial.month()) + goto broken; + if (d == not_a_day) + d = static_cast(static_cast(ymd_trial.day())); + else if (day(static_cast(d)) != ymd_trial.day()) + goto broken; + computed = true; + } + if (Y != not_a_year && U != not_a_week_num && wd != not_a_weekday) { + year_month_day ymd_trial = + sys_days(year{Y} / January / Sunday[1]) + weeks{U - 1} + (weekday{static_cast(wd)} - Sunday); + if (Y == not_a_year) + Y = static_cast(ymd_trial.year()); + else if (year{Y} != ymd_trial.year()) + goto broken; + if (m == not_a_month) + m = static_cast(static_cast(ymd_trial.month())); + else if (month(static_cast(m)) != ymd_trial.month()) + goto broken; + if (d == not_a_day) + d = static_cast(static_cast(ymd_trial.day())); + else if (day(static_cast(d)) != ymd_trial.day()) + goto broken; + computed = true; + } + if (Y != not_a_year && W != not_a_week_num && wd != not_a_weekday) { + year_month_day ymd_trial = + sys_days(year{Y} / January / Monday[1]) + weeks{W - 1} + (weekday{static_cast(wd)} - Monday); + if (Y == not_a_year) + Y = static_cast(ymd_trial.year()); + else if (year{Y} != ymd_trial.year()) + goto broken; + if (m == not_a_month) + m = static_cast(static_cast(ymd_trial.month())); + else if (month(static_cast(m)) != ymd_trial.month()) + goto broken; + if (d == not_a_day) + d = static_cast(static_cast(ymd_trial.day())); + else if (day(static_cast(d)) != ymd_trial.day()) + goto broken; + computed = true; + } + if (j != not_a_doy && Y != not_a_year) { + auto ymd_trial = year_month_day{local_days(year{Y} / 1 / 1) + days{j - 1}}; + if (m == 0) + m = static_cast(static_cast(ymd_trial.month())); + else if (month(static_cast(m)) != ymd_trial.month()) + goto broken; + if (d == 0) + d = static_cast(static_cast(ymd_trial.day())); + else if (day(static_cast(d)) != ymd_trial.day()) + goto broken; + j = not_a_doy; + } + auto ymd = year{Y} / m / d; + if (ymd.ok()) { + if (wd == not_a_weekday) + wd = static_cast((weekday(sys_days(ymd)) - Sunday).count()); + else if (wd != static_cast((weekday(sys_days(ymd)) - Sunday).count())) + goto broken; + if (!computed) { + if (G != not_a_year || V != not_a_week_num) { + sys_days sd = ymd; + auto G_trial = year_month_day{sd + days{3}}.year(); + auto start = sys_days((G_trial - years{1}) / December / Thursday[last]) + (Monday - Thursday); + if (sd < start) { + --G_trial; + if (V != not_a_week_num) + start = sys_days((G_trial - years{1}) / December / Thursday[last]) + (Monday - Thursday); } - if (s != not_a_second) - { - fds.has_tod = true; - fds.tod.s_ = detail::decimal_format_seconds{s}; + if (G != not_a_year && G != static_cast(G_trial)) + goto broken; + if (V != not_a_week_num) { + auto V_trial = duration_cast(sd - start).count() + 1; + if (V != V_trial) + goto broken; } - if (j != not_a_doy) - { - fds.has_tod = true; - fds.tod.h_ += hours{days{j}}; + } + if (U != not_a_week_num) { + auto start = sys_days(Sunday[1] / January / ymd.year()); + auto U_trial = floor(sys_days(ymd) - start).count() + 1; + if (U != U_trial) + goto broken; + } + if (W != not_a_week_num) { + auto start = sys_days(Monday[1] / January / ymd.year()); + auto W_trial = floor(sys_days(ymd) - start).count() + 1; + if (W != W_trial) + goto broken; + } + } + } + fds.ymd = ymd; + if (I != not_a_hour_12_value) { + if (!(1 <= I && I <= 12)) + goto broken; + if (p != not_a_ampm) { + // p is in [0, 1] == [AM, PM] + // Store trial H in I + if (I == 12) + --p; + I += p * 12; + // Either set H from I or make sure H and I are consistent + if (H == not_a_hour) + H = I; + else if (I != H) + goto broken; + } else // p == not_a_ampm + { + // if H, make sure H and I could be consistent + if (H != not_a_hour) { + if (I == 12) { + if (H != 0 && H != 12) + goto broken; + } else if (!(I == H || I == H + 12)) { + goto broken; } - if (wd != not_a_weekday) - fds.wd = weekday{static_cast(wd)}; - if (abbrev != nullptr) - *abbrev = std::move(temp_abbrev); - if (offset != nullptr && temp_offset != not_a_offset) - *offset = temp_offset; + } } - return is; + } + if (H != not_a_hour) { + fds.has_tod = true; + fds.tod = hh_mm_ss{hours{H}}; + } + if (M != not_a_minute) { + fds.has_tod = true; + fds.tod.m_ = minutes{M}; + } + if (s != not_a_second) { + fds.has_tod = true; + fds.tod.s_ = detail::decimal_format_seconds{s}; + } + if (j != not_a_doy) { + fds.has_tod = true; + fds.tod.h_ += hours{days{j}}; + } + if (wd != not_a_weekday) + fds.wd = weekday{static_cast(wd)}; + if (abbrev != nullptr) + *abbrev = std::move(temp_abbrev); + if (offset != nullptr && temp_offset != not_a_offset) + *offset = temp_offset; } -broken: - is.setstate(ios::failbit); return is; + } +broken: + is.setstate(ios::failbit); + return is; } template > -std::basic_istream& -from_stream(std::basic_istream& is, const CharT* fmt, year& y, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) -{ - using CT = std::chrono::seconds; - fields fds{}; - from_stream(is, fmt, fds, abbrev, offset); - if (!fds.ymd.year().ok()) - is.setstate(std::ios::failbit); - if (!is.fail()) - y = fds.ymd.year(); - return is; +std::basic_istream& from_stream(std::basic_istream& is, const CharT* fmt, year& y, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) { + using CT = std::chrono::seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.year().ok()) + is.setstate(std::ios::failbit); + if (!is.fail()) + y = fds.ymd.year(); + return is; } template > -std::basic_istream& -from_stream(std::basic_istream& is, const CharT* fmt, month& m, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) -{ - using CT = std::chrono::seconds; - fields fds{}; - from_stream(is, fmt, fds, abbrev, offset); - if (!fds.ymd.month().ok()) - is.setstate(std::ios::failbit); - if (!is.fail()) - m = fds.ymd.month(); - return is; +std::basic_istream& from_stream(std::basic_istream& is, const CharT* fmt, month& m, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) { + using CT = std::chrono::seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.month().ok()) + is.setstate(std::ios::failbit); + if (!is.fail()) + m = fds.ymd.month(); + return is; } template > -std::basic_istream& -from_stream(std::basic_istream& is, const CharT* fmt, day& d, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) -{ - using CT = std::chrono::seconds; - fields fds{}; - from_stream(is, fmt, fds, abbrev, offset); - if (!fds.ymd.day().ok()) - is.setstate(std::ios::failbit); - if (!is.fail()) - d = fds.ymd.day(); - return is; +std::basic_istream& from_stream(std::basic_istream& is, const CharT* fmt, day& d, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) { + using CT = std::chrono::seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.day().ok()) + is.setstate(std::ios::failbit); + if (!is.fail()) + d = fds.ymd.day(); + return is; } template > -std::basic_istream& -from_stream(std::basic_istream& is, const CharT* fmt, weekday& wd, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) -{ - using CT = std::chrono::seconds; - fields fds{}; - from_stream(is, fmt, fds, abbrev, offset); - if (!fds.wd.ok()) - is.setstate(std::ios::failbit); - if (!is.fail()) - wd = fds.wd; - return is; +std::basic_istream& from_stream(std::basic_istream& is, const CharT* fmt, weekday& wd, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) { + using CT = std::chrono::seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.wd.ok()) + is.setstate(std::ios::failbit); + if (!is.fail()) + wd = fds.wd; + return is; } template > -std::basic_istream& -from_stream(std::basic_istream& is, const CharT* fmt, year_month& ym, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) -{ - using CT = std::chrono::seconds; - fields fds{}; - from_stream(is, fmt, fds, abbrev, offset); - if (!fds.ymd.month().ok()) - is.setstate(std::ios::failbit); - if (!is.fail()) - ym = fds.ymd.year()/fds.ymd.month(); - return is; +std::basic_istream& from_stream(std::basic_istream& is, const CharT* fmt, year_month& ym, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) { + using CT = std::chrono::seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.month().ok()) + is.setstate(std::ios::failbit); + if (!is.fail()) + ym = fds.ymd.year() / fds.ymd.month(); + return is; } template > -std::basic_istream& -from_stream(std::basic_istream& is, const CharT* fmt, month_day& md, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) -{ - using CT = std::chrono::seconds; - fields fds{}; - from_stream(is, fmt, fds, abbrev, offset); - if (!fds.ymd.month().ok() || !fds.ymd.day().ok()) - is.setstate(std::ios::failbit); - if (!is.fail()) - md = fds.ymd.month()/fds.ymd.day(); - return is; +std::basic_istream& from_stream(std::basic_istream& is, const CharT* fmt, month_day& md, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) { + using CT = std::chrono::seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.month().ok() || !fds.ymd.day().ok()) + is.setstate(std::ios::failbit); + if (!is.fail()) + md = fds.ymd.month() / fds.ymd.day(); + return is; } template > -std::basic_istream& -from_stream(std::basic_istream& is, const CharT* fmt, - year_month_day& ymd, std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) -{ - using CT = std::chrono::seconds; - fields fds{}; - from_stream(is, fmt, fds, abbrev, offset); - if (!fds.ymd.ok()) - is.setstate(std::ios::failbit); - if (!is.fail()) - ymd = fds.ymd; - return is; +std::basic_istream& from_stream(std::basic_istream& is, const CharT* fmt, + year_month_day& ymd, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) { + using CT = std::chrono::seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.ok()) + is.setstate(std::ios::failbit); + if (!is.fail()) + ymd = fds.ymd; + return is; } template > -std::basic_istream& -from_stream(std::basic_istream& is, const CharT* fmt, - sys_time& tp, std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) -{ - using CT = typename std::common_type::type; - using detail::round_i; - std::chrono::minutes offset_local{}; - auto offptr = offset ? offset : &offset_local; - fields fds{}; - fds.has_tod = true; - from_stream(is, fmt, fds, abbrev, offptr); - if (!fds.ymd.ok() || !fds.tod.in_conventional_range()) - is.setstate(std::ios::failbit); - if (!is.fail()) - tp = round_i(sys_days(fds.ymd) - *offptr + fds.tod.to_duration()); - return is; +std::basic_istream& from_stream(std::basic_istream& is, const CharT* fmt, + sys_time& tp, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) { + using CT = typename std::common_type::type; + using detail::round_i; + std::chrono::minutes offset_local{}; + auto offptr = offset ? offset : &offset_local; + fields fds{}; + fds.has_tod = true; + from_stream(is, fmt, fds, abbrev, offptr); + if (!fds.ymd.ok() || !fds.tod.in_conventional_range()) + is.setstate(std::ios::failbit); + if (!is.fail()) + tp = round_i(sys_days(fds.ymd) - *offptr + fds.tod.to_duration()); + return is; } template > -std::basic_istream& -from_stream(std::basic_istream& is, const CharT* fmt, - local_time& tp, std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) -{ - using CT = typename std::common_type::type; - using detail::round_i; - fields fds{}; - fds.has_tod = true; - from_stream(is, fmt, fds, abbrev, offset); - if (!fds.ymd.ok() || !fds.tod.in_conventional_range()) - is.setstate(std::ios::failbit); - if (!is.fail()) - tp = round_i(local_seconds{local_days(fds.ymd)} + fds.tod.to_duration()); - return is; +std::basic_istream& from_stream(std::basic_istream& is, const CharT* fmt, + local_time& tp, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) { + using CT = typename std::common_type::type; + using detail::round_i; + fields fds{}; + fds.has_tod = true; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.ok() || !fds.tod.in_conventional_range()) + is.setstate(std::ios::failbit); + if (!is.fail()) + tp = round_i(local_seconds{local_days(fds.ymd)} + fds.tod.to_duration()); + return is; } template > -std::basic_istream& -from_stream(std::basic_istream& is, const CharT* fmt, - std::chrono::duration& d, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) -{ - using Duration = std::chrono::duration; - using CT = typename std::common_type::type; - fields fds{}; - from_stream(is, fmt, fds, abbrev, offset); - if (!fds.has_tod) - is.setstate(std::ios::failbit); - if (!is.fail()) - d = std::chrono::duration_cast(fds.tod.to_duration()); - return is; -} - -template , - class Alloc = std::allocator> -struct parse_manip -{ - const std::basic_string format_; - Parsable& tp_; - std::basic_string* abbrev_; - std::chrono::minutes* offset_; - -public: - parse_manip(std::basic_string format, Parsable& tp, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) - : format_(std::move(format)) - , tp_(tp) - , abbrev_(abbrev) - , offset_(offset) - {} - +std::basic_istream& from_stream(std::basic_istream& is, const CharT* fmt, + std::chrono::duration& d, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) { + using Duration = std::chrono::duration; + using CT = typename std::common_type::type; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.has_tod) + is.setstate(std::ios::failbit); + if (!is.fail()) + d = std::chrono::duration_cast(fds.tod.to_duration()); + return is; +} + +template , class Alloc = std::allocator> +struct parse_manip { + const std::basic_string format_; + Parsable& tp_; + std::basic_string* abbrev_; + std::chrono::minutes* offset_; + + public: + parse_manip(std::basic_string format, Parsable& tp, + std::basic_string* abbrev = nullptr, std::chrono::minutes* offset = nullptr) + : format_(std::move(format)), tp_(tp), abbrev_(abbrev), offset_(offset) { + } }; template -std::basic_istream& -operator>>(std::basic_istream& is, - const parse_manip& x) -{ - return from_stream(is, x.format_.c_str(), x.tp_, x.abbrev_, x.offset_); +std::basic_istream& operator>>(std::basic_istream& is, + const parse_manip& x) { + return from_stream(is, x.format_.c_str(), x.tp_, x.abbrev_, x.offset_); } template -inline -auto -parse(const std::basic_string& format, Parsable& tp) - -> decltype(from_stream(std::declval&>(), - format.c_str(), tp), - parse_manip{format, tp}) -{ - return {format, tp}; +inline auto parse(const std::basic_string& format, Parsable& tp) + -> decltype(from_stream(std::declval&>(), format.c_str(), tp), + parse_manip{format, tp}) { + return {format, tp}; } template -inline -auto -parse(const std::basic_string& format, Parsable& tp, - std::basic_string& abbrev) - -> decltype(from_stream(std::declval&>(), - format.c_str(), tp, &abbrev), - parse_manip{format, tp, &abbrev}) -{ - return {format, tp, &abbrev}; +inline auto parse(const std::basic_string& format, Parsable& tp, + std::basic_string& abbrev) + -> decltype(from_stream(std::declval&>(), format.c_str(), tp, &abbrev), + parse_manip{format, tp, &abbrev}) { + return {format, tp, &abbrev}; } template -inline -auto -parse(const std::basic_string& format, Parsable& tp, - std::chrono::minutes& offset) - -> decltype(from_stream(std::declval&>(), - format.c_str(), tp, - std::declval*>(), - &offset), - parse_manip{format, tp, nullptr, &offset}) -{ - return {format, tp, nullptr, &offset}; +inline auto parse(const std::basic_string& format, Parsable& tp, std::chrono::minutes& offset) + -> decltype(from_stream(std::declval&>(), format.c_str(), tp, + std::declval*>(), &offset), + parse_manip{format, tp, nullptr, &offset}) { + return {format, tp, nullptr, &offset}; } template -inline -auto -parse(const std::basic_string& format, Parsable& tp, - std::basic_string& abbrev, std::chrono::minutes& offset) - -> decltype(from_stream(std::declval&>(), - format.c_str(), tp, &abbrev, &offset), - parse_manip{format, tp, &abbrev, &offset}) -{ - return {format, tp, &abbrev, &offset}; +inline auto parse(const std::basic_string& format, Parsable& tp, + std::basic_string& abbrev, std::chrono::minutes& offset) + -> decltype(from_stream(std::declval&>(), format.c_str(), tp, &abbrev, &offset), + parse_manip{format, tp, &abbrev, &offset}) { + return {format, tp, &abbrev, &offset}; } // const CharT* formats template -inline -auto -parse(const CharT* format, Parsable& tp) +inline auto parse(const CharT* format, Parsable& tp) -> decltype(from_stream(std::declval&>(), format, tp), - parse_manip{format, tp}) -{ - return {format, tp}; + parse_manip{format, tp}) { + return {format, tp}; } template -inline -auto -parse(const CharT* format, Parsable& tp, std::basic_string& abbrev) - -> decltype(from_stream(std::declval&>(), format, - tp, &abbrev), - parse_manip{format, tp, &abbrev}) -{ - return {format, tp, &abbrev}; +inline auto parse(const CharT* format, Parsable& tp, std::basic_string& abbrev) + -> decltype(from_stream(std::declval&>(), format, tp, &abbrev), + parse_manip{format, tp, &abbrev}) { + return {format, tp, &abbrev}; } template -inline -auto -parse(const CharT* format, Parsable& tp, std::chrono::minutes& offset) - -> decltype(from_stream(std::declval&>(), format, - tp, std::declval*>(), &offset), - parse_manip{format, tp, nullptr, &offset}) -{ - return {format, tp, nullptr, &offset}; +inline auto parse(const CharT* format, Parsable& tp, std::chrono::minutes& offset) + -> decltype(from_stream(std::declval&>(), format, tp, + std::declval*>(), &offset), + parse_manip{format, tp, nullptr, &offset}) { + return {format, tp, nullptr, &offset}; } template -inline -auto -parse(const CharT* format, Parsable& tp, - std::basic_string& abbrev, std::chrono::minutes& offset) - -> decltype(from_stream(std::declval&>(), format, - tp, &abbrev, &offset), - parse_manip{format, tp, &abbrev, &offset}) -{ - return {format, tp, &abbrev, &offset}; +inline auto parse(const CharT* format, Parsable& tp, std::basic_string& abbrev, + std::chrono::minutes& offset) + -> decltype(from_stream(std::declval&>(), format, tp, &abbrev, &offset), + parse_manip{format, tp, &abbrev, &offset}) { + return {format, tp, &abbrev, &offset}; } // duration streaming template -inline -std::basic_ostream& -operator<<(std::basic_ostream& os, - const std::chrono::duration& d) -{ - return os << detail::make_string::from(d.count()) + - detail::get_units(typename Period::type{}); +inline std::basic_ostream& operator<<(std::basic_ostream& os, + const std::chrono::duration& d) { + return os << detail::make_string::from(d.count()) + detail::get_units(typename Period::type{}); } } // namespace date #ifdef _MSC_VER -# pragma warning(pop) +#pragma warning(pop) #endif #ifdef __GNUC__ -# pragma GCC diagnostic pop +#pragma GCC diagnostic pop #endif #endif // DATE_H diff --git a/tdutils/td/utils/filesystem.cpp b/tdutils/td/utils/filesystem.cpp index b84b6b3f8a..586dc39287 100644 --- a/tdutils/td/utils/filesystem.cpp +++ b/tdutils/td/utils/filesystem.cpp @@ -16,16 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/filesystem.h" - +#include "td/utils/PathView.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" #include "td/utils/buffer.h" +#include "td/utils/filesystem.h" #include "td/utils/logging.h" #include "td/utils/misc.h" -#include "td/utils/PathView.h" #include "td/utils/port/FileFd.h" #include "td/utils/port/path.h" -#include "td/utils/Slice.h" -#include "td/utils/Status.h" #include "td/utils/unicode.h" #include "td/utils/utf8.h" diff --git a/tdutils/td/utils/filesystem.h b/tdutils/td/utils/filesystem.h index f8b6c3188f..d43fd49594 100644 --- a/tdutils/td/utils/filesystem.h +++ b/tdutils/td/utils/filesystem.h @@ -18,11 +18,10 @@ */ #pragma once -#include "td/utils/buffer.h" #include "td/utils/SharedSlice.h" #include "td/utils/Slice.h" -#include "td/utils/SharedSlice.h" #include "td/utils/Status.h" +#include "td/utils/buffer.h" namespace td { diff --git a/tdutils/td/utils/find_boundary.cpp b/tdutils/td/utils/find_boundary.cpp index d1d21f75f9..6da2815a98 100644 --- a/tdutils/td/utils/find_boundary.cpp +++ b/tdutils/td/utils/find_boundary.cpp @@ -16,10 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/find_boundary.h" - #include +#include "td/utils/find_boundary.h" + namespace td { bool find_boundary(ChainBufferReader range, Slice boundary, size_t &already_read) { diff --git a/tdutils/td/utils/find_boundary.h b/tdutils/td/utils/find_boundary.h index 35a02630e8..6a4c861afe 100644 --- a/tdutils/td/utils/find_boundary.h +++ b/tdutils/td/utils/find_boundary.h @@ -18,9 +18,9 @@ */ #pragma once +#include "td/utils/Slice.h" #include "td/utils/buffer.h" #include "td/utils/common.h" -#include "td/utils/Slice.h" namespace td { diff --git a/tdutils/td/utils/format.h b/tdutils/td/utils/format.h index cf4447186a..4960a2da80 100644 --- a/tdutils/td/utils/format.h +++ b/tdutils/td/utils/format.h @@ -18,14 +18,14 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/invoke.h" // for tuple_for_each -#include "td/utils/Slice.h" -#include "td/utils/StringBuilder.h" - +#include #include #include -#include + +#include "td/utils/Slice.h" +#include "td/utils/StringBuilder.h" +#include "td/utils/common.h" +#include "td/utils/invoke.h" // for tuple_for_each namespace td { namespace format { @@ -315,7 +315,7 @@ StringBuilder &operator<<(StringBuilder &sb, const Concat &concat) { } template -auto concat(const ArgsT &... args) { +auto concat(const ArgsT &...args) { return Concat{std::tie(args...)}; } diff --git a/tdutils/td/utils/int_types.h b/tdutils/td/utils/int_types.h index 9a6c49e6f5..a6a3c06992 100644 --- a/tdutils/td/utils/int_types.h +++ b/tdutils/td/utils/int_types.h @@ -18,11 +18,11 @@ */ #pragma once -#include "td/utils/port/platform.h" - #include #include +#include "td/utils/port/platform.h" + namespace td { #if !TD_WINDOWS diff --git a/tdutils/td/utils/invoke.h b/tdutils/td/utils/invoke.h index 3eb79da8e3..04801859a5 100644 --- a/tdutils/td/utils/invoke.h +++ b/tdutils/td/utils/invoke.h @@ -18,13 +18,13 @@ */ #pragma once -#include "td/utils/common.h" - #include #include #include #include +#include "td/utils/common.h" + namespace td { namespace detail { @@ -66,7 +66,7 @@ struct is_reference_wrapper> : std::true_type {}; template auto invoke_impl(T Base::*pmf, Derived &&ref, - Args &&... args) noexcept(noexcept((std::forward(ref).*pmf)(std::forward(args)...))) + Args &&...args) noexcept(noexcept((std::forward(ref).*pmf)(std::forward(args)...))) -> std::enable_if_t::value && std::is_base_of>::value, decltype((std::forward(ref).*pmf)(std::forward(args)...))> { return (std::forward(ref).*pmf)(std::forward(args)...); @@ -74,7 +74,7 @@ auto invoke_impl(T Base::*pmf, Derived &&ref, template auto invoke_impl(T Base::*pmf, RefWrap &&ref, - Args &&... args) noexcept(noexcept((ref.get().*pmf)(std::forward(args)...))) + Args &&...args) noexcept(noexcept((ref.get().*pmf)(std::forward(args)...))) -> std::enable_if_t::value && is_reference_wrapper>::value, decltype((ref.get().*pmf)(std::forward(args)...))> @@ -84,7 +84,7 @@ auto invoke_impl(T Base::*pmf, RefWrap &&ref, template auto invoke_impl(T Base::*pmf, Pointer &&ptr, - Args &&... args) noexcept(noexcept(((*std::forward(ptr)).*pmf)(std::forward(args)...))) + Args &&...args) noexcept(noexcept(((*std::forward(ptr)).*pmf)(std::forward(args)...))) -> std::enable_if_t::value && !is_reference_wrapper>::value && !std::is_base_of>::value, decltype(((*std::forward(ptr)).*pmf)(std::forward(args)...))> { @@ -114,7 +114,7 @@ auto invoke_impl(T Base::*pmd, Pointer &&ptr) noexcept(noexcept((*std::forward

-auto invoke_impl(F &&f, Args &&... args) noexcept(noexcept(std::forward(f)(std::forward(args)...))) +auto invoke_impl(F &&f, Args &&...args) noexcept(noexcept(std::forward(f)(std::forward(args)...))) -> std::enable_if_t>::value, decltype(std::forward(f)(std::forward(args)...))> { return std::forward(f)(std::forward(args)...); @@ -122,7 +122,7 @@ auto invoke_impl(F &&f, Args &&... args) noexcept(noexcept(std::forward(f)(st template auto invoke(F &&f, - ArgTypes &&... args) noexcept(noexcept(invoke_impl(std::forward(f), std::forward(args)...))) + ArgTypes &&...args) noexcept(noexcept(invoke_impl(std::forward(f), std::forward(args)...))) -> decltype(invoke_impl(std::forward(f), std::forward(args)...)) { return invoke_impl(std::forward(f), std::forward(args)...); } @@ -188,29 +188,29 @@ void tuple_for_each(const std::tuple &tuple, const F &func) { } template = 0> -auto &&get_nth_argument(Arg &&arg, Args &&... args) { +auto &&get_nth_argument(Arg &&arg, Args &&...args) { return std::forward(arg); } template = 0> -auto &&get_nth_argument(Arg &&arg, Args &&... args) { +auto &&get_nth_argument(Arg &&arg, Args &&...args) { return get_nth_argument(std::forward(args)...); } template -auto &&get_last_argument(Args &&... args) { +auto &&get_last_argument(Args &&...args) { return get_nth_argument(std::forward(args)...); } namespace detail { template -auto call_n_arguments_impl(IntSeq, F &&f, Args &&... args) { +auto call_n_arguments_impl(IntSeq, F &&f, Args &&...args) { return f(get_nth_argument(std::forward(args)...)...); } } // namespace detail template -auto call_n_arguments(F &&f, Args &&... args) { +auto call_n_arguments(F &&f, Args &&...args) { return detail::call_n_arguments_impl(detail::IntRange(), f, std::forward(args)...); } diff --git a/tdutils/td/utils/logging.cpp b/tdutils/td/utils/logging.cpp index 03d32ee2c7..1781380a01 100644 --- a/tdutils/td/utils/logging.cpp +++ b/tdutils/td/utils/logging.cpp @@ -16,19 +16,19 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/logging.h" +#include +#include +#include -#include "ThreadSafeCounter.h" -#include "td/utils/port/Clocks.h" -#include "td/utils/port/StdStreams.h" -#include "td/utils/port/thread_local.h" #include "td/utils/Slice.h" #include "td/utils/Time.h" #include "td/utils/date.h" +#include "td/utils/logging.h" +#include "td/utils/port/Clocks.h" +#include "td/utils/port/StdStreams.h" +#include "td/utils/port/thread_local.h" -#include -#include -#include +#include "ThreadSafeCounter.h" #if TD_ANDROID #include diff --git a/tdutils/td/utils/logging.h b/tdutils/td/utils/logging.h index bb28f6df8f..f8580d44f6 100644 --- a/tdutils/td/utils/logging.h +++ b/tdutils/td/utils/logging.h @@ -35,15 +35,15 @@ * CHECK(condition) <===> LOG_IF(FATAL, !(condition)) */ -#include "td/utils/common.h" -#include "td/utils/port/thread_local.h" +#include +#include + #include "td/utils/Slice.h" #include "td/utils/StackAllocator.h" #include "td/utils/StringBuilder.h" +#include "td/utils/common.h" #include "td/utils/port/Clocks.h" - -#include -#include +#include "td/utils/port/thread_local.h" #define PSTR_IMPL() ::td::Logger(::td::NullLog().ref(), ::td::LogOptions::plain(), 0) #define PSLICE() ::td::detail::Slicify() & PSTR_IMPL() diff --git a/tdutils/td/utils/lz4.cpp b/tdutils/td/utils/lz4.cpp index ebf456aaa9..901b23db7b 100644 --- a/tdutils/td/utils/lz4.cpp +++ b/tdutils/td/utils/lz4.cpp @@ -14,9 +14,10 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ +#include + #include "td/utils/buffer.h" #include "td/utils/misc.h" -#include namespace td { diff --git a/tdutils/td/utils/lz4.h b/tdutils/td/utils/lz4.h index fbbc470fad..964c10a02d 100644 --- a/tdutils/td/utils/lz4.h +++ b/tdutils/td/utils/lz4.h @@ -16,8 +16,8 @@ */ #pragma once -#include "td/utils/buffer.h" #include "td/utils/Status.h" +#include "td/utils/buffer.h" namespace td { diff --git a/tdutils/td/utils/misc.cpp b/tdutils/td/utils/misc.cpp index caff44e39c..774e673d96 100644 --- a/tdutils/td/utils/misc.cpp +++ b/tdutils/td/utils/misc.cpp @@ -16,15 +16,14 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/misc.h" - -#include "td/utils/port/thread_local.h" - #include #include #include #include +#include "td/utils/misc.h" +#include "td/utils/port/thread_local.h" + namespace td { char *str_dup(Slice str) { diff --git a/tdutils/td/utils/misc.h b/tdutils/td/utils/misc.h index 7d19b6ece4..969cfcb4b2 100644 --- a/tdutils/td/utils/misc.h +++ b/tdutils/td/utils/misc.h @@ -18,17 +18,17 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/logging.h" -#include "td/utils/Slice.h" -#include "td/utils/Status.h" -#include "td/utils/StringBuilder.h" - #include #include #include #include +#include "td/utils/Slice.h" +#include "td/utils/Status.h" +#include "td/utils/StringBuilder.h" +#include "td/utils/common.h" +#include "td/utils/logging.h" + namespace td { char *str_dup(Slice str); diff --git a/tdutils/td/utils/optional.h b/tdutils/td/utils/optional.h index 7723d2c311..2c96dc3ff7 100644 --- a/tdutils/td/utils/optional.h +++ b/tdutils/td/utils/optional.h @@ -18,12 +18,12 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/Status.h" - #include #include +#include "td/utils/Status.h" +#include "td/utils/common.h" + namespace td { template ::value> @@ -90,15 +90,15 @@ class optional { } template - void emplace(ArgsT &&... args) { + void emplace(ArgsT &&...args) { impl_.emplace(std::forward(args)...); } - bool operator==(const optional& other) const { + bool operator==(const optional &other) const { return (bool)*this == (bool)other && (!(bool)*this || value() == other.value()); } - bool operator!=(const optional& other) const { + bool operator!=(const optional &other) const { return !(*this == other); } diff --git a/tdutils/td/utils/overloaded.h b/tdutils/td/utils/overloaded.h index 492dbc3e30..16cf31e9f5 100644 --- a/tdutils/td/utils/overloaded.h +++ b/tdutils/td/utils/overloaded.h @@ -32,9 +32,7 @@ struct overload : public F { }; template -struct overload - : public overload - , overload { +struct overload : public overload, overload { overload(F f, Fs... fs) : overload(f), overload(fs...) { } using overload::operator(); diff --git a/tdutils/td/utils/port/Clocks.cpp b/tdutils/td/utils/port/Clocks.cpp index 7e27d51a98..65796a9ea2 100644 --- a/tdutils/td/utils/port/Clocks.cpp +++ b/tdutils/td/utils/port/Clocks.cpp @@ -16,11 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/port/Clocks.h" - #include #include +#include "td/utils/port/Clocks.h" + namespace td { int64 Clocks::monotonic_nano() { auto duration = std::chrono::steady_clock::now().time_since_epoch(); diff --git a/tdutils/td/utils/port/CxCli.h b/tdutils/td/utils/port/CxCli.h index 8c8433d499..c0ad3b490f 100644 --- a/tdutils/td/utils/port/CxCli.h +++ b/tdutils/td/utils/port/CxCli.h @@ -42,8 +42,8 @@ #define REF_NEW ref new #define CLRCALL -#define DEPRECATED_ATTRIBUTE(message) ::Windows::Foundation::Metadata::Deprecated(message,\ - ::Windows::Foundation::Metadata::DeprecationType::Deprecate, 0x0) +#define DEPRECATED_ATTRIBUTE(message) \ + ::Windows::Foundation::Metadata::Deprecated(message, ::Windows::Foundation::Metadata::DeprecationType::Deprecate, 0x0) namespace CxCli { @@ -59,8 +59,9 @@ using Platform::String; using Platform::NullReferenceException; -template class ConcurrentDictionary { -public: +template +class ConcurrentDictionary { + public: bool TryGetValue(Key key, Value &value) { std::lock_guard guard(mutex_); auto it = impl_.find(key); @@ -80,11 +81,12 @@ template class ConcurrentDictionary { impl_.erase(it); return true; } - Value &operator [] (Key key) { + Value &operator[](Key key) { std::lock_guard guard(mutex_); return impl_[key]; } -private: + + private: std::mutex mutex_; std::map impl_; }; @@ -93,19 +95,19 @@ inline std::int64_t Increment(volatile std::int64_t &value) { return InterlockedIncrement64(&value); } -inline std::string string_to_unmanaged(String^ str) { +inline std::string string_to_unmanaged(String ^ str) { if (!str) { return std::string(); } return td::from_wstring(str->Data(), str->Length()).ok(); } -inline String^ string_from_unmanaged(const std::string &from) { +inline String ^ string_from_unmanaged(const std::string &from) { auto tmp = td::to_wstring(from).ok(); return REF_NEW String(tmp.c_str(), static_cast(tmp.size())); } -} // namespace CxCli +} // namespace CxCli #elif TD_CLI @@ -135,27 +137,27 @@ using System::NullReferenceException; using System::Collections::Concurrent::ConcurrentDictionary; -inline std::int64_t Increment(std::int64_t %value) { +inline std::int64_t Increment(std::int64_t % value) { return System::Threading::Interlocked::Increment(value); } -inline std::string string_to_unmanaged(String^ str) { +inline std::string string_to_unmanaged(String ^ str) { if (!str || str->Length == 0) { return std::string(); } - Array^ bytes = System::Text::Encoding::UTF8->GetBytes(str); + Array ^ bytes = System::Text::Encoding::UTF8->GetBytes(str); cli::pin_ptr pinned_ptr = &bytes[0]; std::string result(reinterpret_cast(&pinned_ptr[0]), bytes->Length); return result; } -inline String^ string_from_unmanaged(const std::string &from) { +inline String ^ string_from_unmanaged(const std::string &from) { if (from.empty()) { return String::Empty; } - Array^ bytes = REF_NEW Vector(static_cast(from.size())); + Array ^ bytes = REF_NEW Vector(static_cast(from.size())); cli::pin_ptr pinned_ptr = &bytes[0]; for (size_t i = 0; i < from.size(); ++i) { pinned_ptr[i] = from[i]; @@ -163,6 +165,6 @@ inline String^ string_from_unmanaged(const std::string &from) { return System::Text::Encoding::UTF8->GetString(bytes); } -} // namespace CxCli +} // namespace CxCli #endif diff --git a/tdutils/td/utils/port/EventFdBase.h b/tdutils/td/utils/port/EventFdBase.h index 86bdfe6e41..35ad66b280 100644 --- a/tdutils/td/utils/port/EventFdBase.h +++ b/tdutils/td/utils/port/EventFdBase.h @@ -18,9 +18,9 @@ */ #pragma once +#include "td/utils/Status.h" #include "td/utils/common.h" #include "td/utils/port/detail/PollableFd.h" -#include "td/utils/Status.h" namespace td { class EventFdBase { diff --git a/tdutils/td/utils/port/FileFd.cpp b/tdutils/td/utils/port/FileFd.cpp index 9a34c1b188..bd420154c5 100644 --- a/tdutils/td/utils/port/FileFd.cpp +++ b/tdutils/td/utils/port/FileFd.cpp @@ -23,24 +23,23 @@ #include "td/utils/port/wstring_convert.h" #endif +#include +#include +#include +#include + +#include "td/utils/ScopeGuard.h" +#include "td/utils/StringBuilder.h" #include "td/utils/common.h" #include "td/utils/logging.h" #include "td/utils/misc.h" +#include "td/utils/port/PollFlags.h" #include "td/utils/port/detail/PollableFd.h" #include "td/utils/port/detail/skip_eintr.h" -#include "td/utils/port/PollFlags.h" #include "td/utils/port/sleep.h" -#include "td/utils/ScopeGuard.h" -#include "td/utils/StringBuilder.h" - -#include -#include -#include -#include #if TD_PORT_POSIX #include - #include #include #include diff --git a/tdutils/td/utils/port/FileFd.h b/tdutils/td/utils/port/FileFd.h index 4fcad0982c..0cc7151d5b 100644 --- a/tdutils/td/utils/port/FileFd.h +++ b/tdutils/td/utils/port/FileFd.h @@ -18,16 +18,15 @@ */ #pragma once -#include "td/utils/port/config.h" - -#include "td/utils/common.h" -#include "td/utils/port/detail/NativeFd.h" -#include "td/utils/port/detail/PollableFd.h" -#include "td/utils/port/IoSlice.h" -#include "td/utils/port/Stat.h" #include "td/utils/Slice.h" #include "td/utils/Span.h" #include "td/utils/Status.h" +#include "td/utils/common.h" +#include "td/utils/port/IoSlice.h" +#include "td/utils/port/Stat.h" +#include "td/utils/port/config.h" +#include "td/utils/port/detail/NativeFd.h" +#include "td/utils/port/detail/PollableFd.h" namespace td { namespace detail { diff --git a/tdutils/td/utils/port/IPAddress.cpp b/tdutils/td/utils/port/IPAddress.cpp index c3dd58308f..9565589792 100644 --- a/tdutils/td/utils/port/IPAddress.cpp +++ b/tdutils/td/utils/port/IPAddress.cpp @@ -18,15 +18,14 @@ */ #define _WINSOCK_DEPRECATED_NO_WARNINGS // we need to use inet_addr instead of inet_pton -#include "td/utils/port/IPAddress.h" - +#include "td/utils/ScopeGuard.h" +#include "td/utils/Slice.h" #include "td/utils/format.h" #include "td/utils/logging.h" #include "td/utils/misc.h" +#include "td/utils/port/IPAddress.h" #include "td/utils/port/SocketFd.h" #include "td/utils/port/thread_local.h" -#include "td/utils/ScopeGuard.h" -#include "td/utils/Slice.h" #include "td/utils/utf8.h" #if TD_WINDOWS diff --git a/tdutils/td/utils/port/IPAddress.h b/tdutils/td/utils/port/IPAddress.h index 4062da8d14..3d4877c78e 100644 --- a/tdutils/td/utils/port/IPAddress.h +++ b/tdutils/td/utils/port/IPAddress.h @@ -18,12 +18,11 @@ */ #pragma once -#include "td/utils/port/config.h" - -#include "td/utils/common.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" #include "td/utils/StringBuilder.h" +#include "td/utils/common.h" +#include "td/utils/port/config.h" #if !TD_WINDOWS #include diff --git a/tdutils/td/utils/port/MemoryMapping.cpp b/tdutils/td/utils/port/MemoryMapping.cpp index 8a1edb545f..8cd008b485 100644 --- a/tdutils/td/utils/port/MemoryMapping.cpp +++ b/tdutils/td/utils/port/MemoryMapping.cpp @@ -16,10 +16,9 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/port/MemoryMapping.h" - #include "td/utils/logging.h" #include "td/utils/misc.h" +#include "td/utils/port/MemoryMapping.h" // TODO: // windows, diff --git a/tdutils/td/utils/port/MemoryMapping.h b/tdutils/td/utils/port/MemoryMapping.h index 0b4afc6183..cfd7e42b05 100644 --- a/tdutils/td/utils/port/MemoryMapping.h +++ b/tdutils/td/utils/port/MemoryMapping.h @@ -18,10 +18,10 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/port/FileFd.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" +#include "td/utils/common.h" +#include "td/utils/port/FileFd.h" namespace td { diff --git a/tdutils/td/utils/port/Poll.h b/tdutils/td/utils/port/Poll.h index 339ae67828..3bdef71e1a 100644 --- a/tdutils/td/utils/port/Poll.h +++ b/tdutils/td/utils/port/Poll.h @@ -19,7 +19,6 @@ #pragma once #include "td/utils/port/config.h" - #include "td/utils/port/detail/Epoll.h" #include "td/utils/port/detail/KQueue.h" #include "td/utils/port/detail/Poll.h" diff --git a/tdutils/td/utils/port/PollBase.h b/tdutils/td/utils/port/PollBase.h index b57a768070..6ef2a6ccfe 100644 --- a/tdutils/td/utils/port/PollBase.h +++ b/tdutils/td/utils/port/PollBase.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/utils/port/detail/PollableFd.h" #include "td/utils/port/PollFlags.h" +#include "td/utils/port/detail/PollableFd.h" namespace td { class PollBase { diff --git a/tdutils/td/utils/port/PollFlags.h b/tdutils/td/utils/port/PollFlags.h index 685a74a49b..a9645c3cce 100644 --- a/tdutils/td/utils/port/PollFlags.h +++ b/tdutils/td/utils/port/PollFlags.h @@ -18,10 +18,10 @@ */ #pragma once -#include "td/utils/StringBuilder.h" - #include +#include "td/utils/StringBuilder.h" + namespace td { class PollFlags { diff --git a/tdutils/td/utils/port/RwMutex.h b/tdutils/td/utils/port/RwMutex.h index 93ea82b53a..e167b22c12 100644 --- a/tdutils/td/utils/port/RwMutex.h +++ b/tdutils/td/utils/port/RwMutex.h @@ -18,10 +18,9 @@ */ #pragma once -#include "td/utils/port/config.h" - -#include "td/utils/common.h" #include "td/utils/Status.h" +#include "td/utils/common.h" +#include "td/utils/port/config.h" #if TD_PORT_POSIX #include diff --git a/tdutils/td/utils/port/ServerSocketFd.cpp b/tdutils/td/utils/port/ServerSocketFd.cpp index d92dd8e2f4..46408366f5 100644 --- a/tdutils/td/utils/port/ServerSocketFd.cpp +++ b/tdutils/td/utils/port/ServerSocketFd.cpp @@ -16,20 +16,17 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/port/ServerSocketFd.h" - -#include "td/utils/port/config.h" - #include "td/utils/common.h" #include "td/utils/logging.h" -#include "td/utils/port/detail/skip_eintr.h" #include "td/utils/port/IPAddress.h" #include "td/utils/port/PollFlags.h" +#include "td/utils/port/ServerSocketFd.h" +#include "td/utils/port/config.h" +#include "td/utils/port/detail/skip_eintr.h" #if TD_PORT_POSIX -#include - #include +#include #include #include #include @@ -39,9 +36,9 @@ #endif #if TD_PORT_WINDOWS -#include "td/utils/port/detail/Iocp.h" #include "td/utils/SpinLock.h" #include "td/utils/VectorQueue.h" +#include "td/utils/port/detail/Iocp.h" #endif #include diff --git a/tdutils/td/utils/port/ServerSocketFd.h b/tdutils/td/utils/port/ServerSocketFd.h index 1ed389fbcc..ec50fcd093 100644 --- a/tdutils/td/utils/port/ServerSocketFd.h +++ b/tdutils/td/utils/port/ServerSocketFd.h @@ -18,14 +18,13 @@ */ #pragma once -#include "td/utils/port/detail/NativeFd.h" -#include "td/utils/port/detail/PollableFd.h" -#include "td/utils/port/SocketFd.h" +#include #include "td/utils/Slice.h" #include "td/utils/Status.h" - -#include +#include "td/utils/port/SocketFd.h" +#include "td/utils/port/detail/NativeFd.h" +#include "td/utils/port/detail/PollableFd.h" namespace td { namespace detail { diff --git a/tdutils/td/utils/port/SocketFd.cpp b/tdutils/td/utils/port/SocketFd.cpp index 50ac249f5a..9c7a5e1ce0 100644 --- a/tdutils/td/utils/port/SocketFd.cpp +++ b/tdutils/td/utils/port/SocketFd.cpp @@ -16,26 +16,24 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/port/SocketFd.h" - #include "td/utils/common.h" #include "td/utils/format.h" #include "td/utils/logging.h" #include "td/utils/misc.h" -#include "td/utils/port/detail/skip_eintr.h" #include "td/utils/port/PollFlags.h" +#include "td/utils/port/SocketFd.h" +#include "td/utils/port/detail/skip_eintr.h" #if TD_PORT_WINDOWS -#include "td/utils/buffer.h" -#include "td/utils/port/detail/Iocp.h" #include "td/utils/SpinLock.h" #include "td/utils/VectorQueue.h" +#include "td/utils/buffer.h" +#include "td/utils/port/detail/Iocp.h" #endif #if TD_PORT_POSIX -#include - #include +#include #include #include #include diff --git a/tdutils/td/utils/port/SocketFd.h b/tdutils/td/utils/port/SocketFd.h index 8dc8a5ea39..1ef195a9c3 100644 --- a/tdutils/td/utils/port/SocketFd.h +++ b/tdutils/td/utils/port/SocketFd.h @@ -18,17 +18,16 @@ */ #pragma once -#include "td/utils/port/config.h" +#include -#include "td/utils/port/detail/NativeFd.h" -#include "td/utils/port/detail/PollableFd.h" -#include "td/utils/port/IoSlice.h" -#include "td/utils/port/IPAddress.h" #include "td/utils/Slice.h" #include "td/utils/Span.h" #include "td/utils/Status.h" - -#include +#include "td/utils/port/IPAddress.h" +#include "td/utils/port/IoSlice.h" +#include "td/utils/port/config.h" +#include "td/utils/port/detail/NativeFd.h" +#include "td/utils/port/detail/PollableFd.h" namespace td { diff --git a/tdutils/td/utils/port/Stat.cpp b/tdutils/td/utils/port/Stat.cpp index 73b006084e..e70ba7ed8c 100644 --- a/tdutils/td/utils/port/Stat.cpp +++ b/tdutils/td/utils/port/Stat.cpp @@ -16,20 +16,19 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/port/Stat.h" - #include "td/utils/port/FileFd.h" +#include "td/utils/port/Stat.h" #if TD_PORT_POSIX +#include + +#include "td/utils/ScopeGuard.h" #include "td/utils/format.h" #include "td/utils/logging.h" #include "td/utils/misc.h" #include "td/utils/port/Clocks.h" #include "td/utils/port/detail/skip_eintr.h" -#include "td/utils/ScopeGuard.h" - -#include #if TD_DARWIN #include diff --git a/tdutils/td/utils/port/Stat.h b/tdutils/td/utils/port/Stat.h index 82e1832a0a..fe8e3fa6cf 100644 --- a/tdutils/td/utils/port/Stat.h +++ b/tdutils/td/utils/port/Stat.h @@ -18,11 +18,10 @@ */ #pragma once -#include "td/utils/port/config.h" - -#include "td/utils/common.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" +#include "td/utils/common.h" +#include "td/utils/port/config.h" namespace td { diff --git a/tdutils/td/utils/port/StdStreams.cpp b/tdutils/td/utils/port/StdStreams.cpp index 1d20bffbf1..f344e010ec 100644 --- a/tdutils/td/utils/port/StdStreams.cpp +++ b/tdutils/td/utils/port/StdStreams.cpp @@ -16,18 +16,17 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/port/StdStreams.h" +#include +#include +#include "td/utils/Slice.h" #include "td/utils/logging.h" #include "td/utils/misc.h" +#include "td/utils/port/PollFlags.h" +#include "td/utils/port/StdStreams.h" #include "td/utils/port/detail/Iocp.h" #include "td/utils/port/detail/NativeFd.h" -#include "td/utils/port/PollFlags.h" #include "td/utils/port/thread.h" -#include "td/utils/Slice.h" - -#include -#include namespace td { diff --git a/tdutils/td/utils/port/StdStreams.h b/tdutils/td/utils/port/StdStreams.h index 68e48b0f06..bf86f582fb 100644 --- a/tdutils/td/utils/port/StdStreams.h +++ b/tdutils/td/utils/port/StdStreams.h @@ -18,14 +18,14 @@ */ #pragma once +#include +#include + +#include "td/utils/Status.h" #include "td/utils/buffer.h" #include "td/utils/common.h" -#include "td/utils/port/detail/PollableFd.h" #include "td/utils/port/FileFd.h" -#include "td/utils/Status.h" - -#include -#include +#include "td/utils/port/detail/PollableFd.h" namespace td { diff --git a/tdutils/td/utils/port/UdpSocketFd.cpp b/tdutils/td/utils/port/UdpSocketFd.cpp index 49f09829b0..2b27afb9b1 100644 --- a/tdutils/td/utils/port/UdpSocketFd.cpp +++ b/tdutils/td/utils/port/UdpSocketFd.cpp @@ -16,26 +16,24 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/port/UdpSocketFd.h" - +#include "td/utils/VectorQueue.h" #include "td/utils/common.h" #include "td/utils/format.h" #include "td/utils/logging.h" #include "td/utils/misc.h" -#include "td/utils/port/detail/skip_eintr.h" #include "td/utils/port/PollFlags.h" #include "td/utils/port/SocketFd.h" -#include "td/utils/VectorQueue.h" +#include "td/utils/port/UdpSocketFd.h" +#include "td/utils/port/detail/skip_eintr.h" #if TD_PORT_WINDOWS -#include "td/utils/port/detail/Iocp.h" #include "td/utils/SpinLock.h" +#include "td/utils/port/detail/Iocp.h" #endif #if TD_PORT_POSIX -#include - #include +#include #include #include #include diff --git a/tdutils/td/utils/port/UdpSocketFd.h b/tdutils/td/utils/port/UdpSocketFd.h index ae88d202f0..5bd9e46404 100644 --- a/tdutils/td/utils/port/UdpSocketFd.h +++ b/tdutils/td/utils/port/UdpSocketFd.h @@ -18,18 +18,17 @@ */ #pragma once -#include "td/utils/port/config.h" +#include +#include "td/utils/Slice.h" +#include "td/utils/Span.h" +#include "td/utils/Status.h" #include "td/utils/buffer.h" #include "td/utils/optional.h" +#include "td/utils/port/IPAddress.h" +#include "td/utils/port/config.h" #include "td/utils/port/detail/NativeFd.h" #include "td/utils/port/detail/PollableFd.h" -#include "td/utils/port/IPAddress.h" -#include "td/utils/Slice.h" -#include "td/utils/Span.h" -#include "td/utils/Status.h" - -#include namespace td { // Udp and errors diff --git a/tdutils/td/utils/port/detail/Epoll.cpp b/tdutils/td/utils/port/detail/Epoll.cpp index 638f78d7df..f74a11ff1b 100644 --- a/tdutils/td/utils/port/detail/Epoll.cpp +++ b/tdutils/td/utils/port/detail/Epoll.cpp @@ -22,14 +22,13 @@ char disable_linker_warning_about_empty_file_epoll_cpp TD_UNUSED; #ifdef TD_POLL_EPOLL -#include "td/utils/format.h" -#include "td/utils/logging.h" -#include "td/utils/Status.h" - #include - #include +#include "td/utils/Status.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" + namespace td { namespace detail { void Epoll::init() { diff --git a/tdutils/td/utils/port/detail/Epoll.h b/tdutils/td/utils/port/detail/Epoll.h index a8e25e3a90..9100c0271a 100644 --- a/tdutils/td/utils/port/detail/Epoll.h +++ b/tdutils/td/utils/port/detail/Epoll.h @@ -22,14 +22,14 @@ #ifdef TD_POLL_EPOLL -#include "td/utils/common.h" +#include + #include "td/utils/List.h" -#include "td/utils/port/detail/NativeFd.h" -#include "td/utils/port/detail/PollableFd.h" +#include "td/utils/common.h" #include "td/utils/port/PollBase.h" #include "td/utils/port/PollFlags.h" - -#include +#include "td/utils/port/detail/NativeFd.h" +#include "td/utils/port/detail/PollableFd.h" namespace td { namespace detail { diff --git a/tdutils/td/utils/port/detail/EventFdBsd.cpp b/tdutils/td/utils/port/detail/EventFdBsd.cpp index bf8cf14e84..3d808bc5ab 100644 --- a/tdutils/td/utils/port/detail/EventFdBsd.cpp +++ b/tdutils/td/utils/port/detail/EventFdBsd.cpp @@ -22,19 +22,18 @@ char disable_linker_warning_about_empty_file_event_fd_bsd_cpp TD_UNUSED; #ifdef TD_EVENTFD_BSD -#include "td/utils/logging.h" -#include "td/utils/port/detail/NativeFd.h" -#include "td/utils/port/PollFlags.h" -#include "td/utils/port/SocketFd.h" -#include "td/utils/Slice.h" - #include - #include #include #include #include +#include "td/utils/Slice.h" +#include "td/utils/logging.h" +#include "td/utils/port/PollFlags.h" +#include "td/utils/port/SocketFd.h" +#include "td/utils/port/detail/NativeFd.h" + namespace td { namespace detail { diff --git a/tdutils/td/utils/port/detail/EventFdBsd.h b/tdutils/td/utils/port/detail/EventFdBsd.h index 18a3a2608f..f252c3a767 100644 --- a/tdutils/td/utils/port/detail/EventFdBsd.h +++ b/tdutils/td/utils/port/detail/EventFdBsd.h @@ -22,11 +22,11 @@ #ifdef TD_EVENTFD_BSD +#include "td/utils/Status.h" #include "td/utils/common.h" -#include "td/utils/port/detail/PollableFd.h" #include "td/utils/port/EventFdBase.h" #include "td/utils/port/SocketFd.h" -#include "td/utils/Status.h" +#include "td/utils/port/detail/PollableFd.h" namespace td { namespace detail { diff --git a/tdutils/td/utils/port/detail/EventFdLinux.cpp b/tdutils/td/utils/port/detail/EventFdLinux.cpp index 49e8a52204..1dff5e83cd 100644 --- a/tdutils/td/utils/port/detail/EventFdLinux.cpp +++ b/tdutils/td/utils/port/detail/EventFdLinux.cpp @@ -22,20 +22,19 @@ char disable_linker_warning_about_empty_file_event_fd_linux_cpp TD_UNUSED; #ifdef TD_EVENTFD_LINUX -#include "td/utils/logging.h" -#include "td/utils/misc.h" -#include "td/utils/port/detail/NativeFd.h" -#include "td/utils/port/detail/skip_eintr.h" -#include "td/utils/port/PollFlags.h" -#include "td/utils/ScopeGuard.h" -#include "td/utils/Slice.h" - #include - #include #include #include +#include "td/utils/ScopeGuard.h" +#include "td/utils/Slice.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/port/PollFlags.h" +#include "td/utils/port/detail/NativeFd.h" +#include "td/utils/port/detail/skip_eintr.h" + namespace td { namespace detail { class EventFdLinuxImpl { diff --git a/tdutils/td/utils/port/detail/EventFdLinux.h b/tdutils/td/utils/port/detail/EventFdLinux.h index 6fa748d9fc..284c555b97 100644 --- a/tdutils/td/utils/port/detail/EventFdLinux.h +++ b/tdutils/td/utils/port/detail/EventFdLinux.h @@ -22,10 +22,10 @@ #ifdef TD_EVENTFD_LINUX +#include "td/utils/Status.h" #include "td/utils/common.h" -#include "td/utils/port/detail/PollableFd.h" #include "td/utils/port/EventFdBase.h" -#include "td/utils/Status.h" +#include "td/utils/port/detail/PollableFd.h" namespace td { namespace detail { diff --git a/tdutils/td/utils/port/detail/EventFdWindows.h b/tdutils/td/utils/port/detail/EventFdWindows.h index 442020bd68..d952c3427d 100644 --- a/tdutils/td/utils/port/detail/EventFdWindows.h +++ b/tdutils/td/utils/port/detail/EventFdWindows.h @@ -22,11 +22,11 @@ #ifdef TD_EVENTFD_WINDOWS +#include "td/utils/Status.h" #include "td/utils/common.h" +#include "td/utils/port/EventFdBase.h" #include "td/utils/port/detail/NativeFd.h" #include "td/utils/port/detail/PollableFd.h" -#include "td/utils/port/EventFdBase.h" -#include "td/utils/Status.h" namespace td { namespace detail { diff --git a/tdutils/td/utils/port/detail/Iocp.h b/tdutils/td/utils/port/detail/Iocp.h index 6947d85e1d..ac0f203316 100644 --- a/tdutils/td/utils/port/detail/Iocp.h +++ b/tdutils/td/utils/port/detail/Iocp.h @@ -22,12 +22,12 @@ #ifdef TD_PORT_WINDOWS -#include "td/utils/common.h" +#include + #include "td/utils/Context.h" -#include "td/utils/port/detail/NativeFd.h" #include "td/utils/Status.h" - -#include +#include "td/utils/common.h" +#include "td/utils/port/detail/NativeFd.h" namespace td { namespace detail { diff --git a/tdutils/td/utils/port/detail/KQueue.cpp b/tdutils/td/utils/port/detail/KQueue.cpp index f25a1cef21..04f5b6a047 100644 --- a/tdutils/td/utils/port/detail/KQueue.cpp +++ b/tdutils/td/utils/port/detail/KQueue.cpp @@ -22,14 +22,13 @@ char disable_linker_warning_about_empty_file_kqueue_cpp TD_UNUSED; #ifdef TD_POLL_KQUEUE -#include "td/utils/logging.h" -#include "td/utils/Status.h" - #include -#include - #include #include +#include + +#include "td/utils/Status.h" +#include "td/utils/logging.h" namespace td { namespace detail { diff --git a/tdutils/td/utils/port/detail/KQueue.h b/tdutils/td/utils/port/detail/KQueue.h index f5a8c69348..1c88062c8d 100644 --- a/tdutils/td/utils/port/detail/KQueue.h +++ b/tdutils/td/utils/port/detail/KQueue.h @@ -22,18 +22,16 @@ #ifdef TD_POLL_KQUEUE -#include "td/utils/common.h" -#include "td/utils/List.h" -#include "td/utils/port/detail/NativeFd.h" -#include "td/utils/port/detail/PollableFd.h" -#include "td/utils/port/PollBase.h" -#include "td/utils/port/PollFlags.h" - #include - +#include #include // must be included before sys/event.h, which depends on sys/types.h on FreeBSD -#include +#include "td/utils/List.h" +#include "td/utils/common.h" +#include "td/utils/port/PollBase.h" +#include "td/utils/port/PollFlags.h" +#include "td/utils/port/detail/NativeFd.h" +#include "td/utils/port/detail/PollableFd.h" namespace td { namespace detail { diff --git a/tdutils/td/utils/port/detail/NativeFd.cpp b/tdutils/td/utils/port/detail/NativeFd.cpp index f1a080cbec..157a914ddb 100644 --- a/tdutils/td/utils/port/detail/NativeFd.cpp +++ b/tdutils/td/utils/port/detail/NativeFd.cpp @@ -16,11 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/port/detail/NativeFd.h" - +#include "td/utils/Status.h" #include "td/utils/format.h" #include "td/utils/logging.h" -#include "td/utils/Status.h" +#include "td/utils/port/detail/NativeFd.h" #if TD_PORT_POSIX #include diff --git a/tdutils/td/utils/port/detail/NativeFd.h b/tdutils/td/utils/port/detail/NativeFd.h index 8ce114f1f7..1a799bb11f 100644 --- a/tdutils/td/utils/port/detail/NativeFd.h +++ b/tdutils/td/utils/port/detail/NativeFd.h @@ -18,11 +18,10 @@ */ #pragma once -#include "td/utils/port/config.h" - -#include "td/utils/common.h" #include "td/utils/Status.h" #include "td/utils/StringBuilder.h" +#include "td/utils/common.h" +#include "td/utils/port/config.h" namespace td { diff --git a/tdutils/td/utils/port/detail/Poll.cpp b/tdutils/td/utils/port/detail/Poll.cpp index 07abe46271..a5d8f76599 100644 --- a/tdutils/td/utils/port/detail/Poll.cpp +++ b/tdutils/td/utils/port/detail/Poll.cpp @@ -22,13 +22,13 @@ char disable_linker_warning_about_empty_file_poll_cpp TD_UNUSED; #ifdef TD_POLL_POLL +#include + +#include "td/utils/ScopeGuard.h" +#include "td/utils/Status.h" #include "td/utils/format.h" #include "td/utils/logging.h" #include "td/utils/misc.h" -#include "td/utils/ScopeGuard.h" -#include "td/utils/Status.h" - -#include namespace td { namespace detail { diff --git a/tdutils/td/utils/port/detail/Poll.h b/tdutils/td/utils/port/detail/Poll.h index 81ffd6eb9d..3e2e6ff2c2 100644 --- a/tdutils/td/utils/port/detail/Poll.h +++ b/tdutils/td/utils/port/detail/Poll.h @@ -22,12 +22,12 @@ #ifdef TD_POLL_POLL +#include + #include "td/utils/common.h" -#include "td/utils/port/detail/PollableFd.h" #include "td/utils/port/PollBase.h" #include "td/utils/port/PollFlags.h" - -#include +#include "td/utils/port/detail/PollableFd.h" namespace td { namespace detail { diff --git a/tdutils/td/utils/port/detail/PollableFd.h b/tdutils/td/utils/port/detail/PollableFd.h index 6cb621e828..b0eb818959 100644 --- a/tdutils/td/utils/port/detail/PollableFd.h +++ b/tdutils/td/utils/port/detail/PollableFd.h @@ -18,17 +18,17 @@ */ #pragma once +#include +#include + +#include "td/utils/List.h" +#include "td/utils/Observer.h" +#include "td/utils/SpinLock.h" #include "td/utils/common.h" #include "td/utils/format.h" -#include "td/utils/List.h" #include "td/utils/logging.h" -#include "td/utils/Observer.h" -#include "td/utils/port/detail/NativeFd.h" #include "td/utils/port/PollFlags.h" -#include "td/utils/SpinLock.h" - -#include -#include +#include "td/utils/port/detail/NativeFd.h" namespace td { diff --git a/tdutils/td/utils/port/detail/Select.cpp b/tdutils/td/utils/port/detail/Select.cpp index eab58e7ee9..b80d67a945 100644 --- a/tdutils/td/utils/port/detail/Select.cpp +++ b/tdutils/td/utils/port/detail/Select.cpp @@ -22,10 +22,10 @@ char disable_linker_warning_about_empty_file_select_cpp TD_UNUSED; #ifdef TD_POLL_SELECT -#include "td/utils/logging.h" - #include +#include "td/utils/logging.h" + namespace td { namespace detail { diff --git a/tdutils/td/utils/port/detail/Select.h b/tdutils/td/utils/port/detail/Select.h index a50c26e931..e132aa8746 100644 --- a/tdutils/td/utils/port/detail/Select.h +++ b/tdutils/td/utils/port/detail/Select.h @@ -22,12 +22,12 @@ #ifdef TD_POLL_SELECT +#include + #include "td/utils/common.h" -#include "td/utils/port/detail/PollableFd.h" #include "td/utils/port/PollBase.h" #include "td/utils/port/PollFlags.h" - -#include +#include "td/utils/port/detail/PollableFd.h" namespace td { namespace detail { diff --git a/tdutils/td/utils/port/detail/ThreadIdGuard.cpp b/tdutils/td/utils/port/detail/ThreadIdGuard.cpp index 8e4b89edcb..b0bd85c17a 100644 --- a/tdutils/td/utils/port/detail/ThreadIdGuard.cpp +++ b/tdutils/td/utils/port/detail/ThreadIdGuard.cpp @@ -16,14 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/port/detail/ThreadIdGuard.h" +#include +#include #include "td/utils/common.h" +#include "td/utils/port/detail/ThreadIdGuard.h" #include "td/utils/port/thread_local.h" -#include -#include - namespace td { namespace detail { class ThreadIdManager { diff --git a/tdutils/td/utils/port/detail/ThreadPthread.cpp b/tdutils/td/utils/port/detail/ThreadPthread.cpp index 8000445765..28da9f83c9 100644 --- a/tdutils/td/utils/port/detail/ThreadPthread.cpp +++ b/tdutils/td/utils/port/detail/ThreadPthread.cpp @@ -23,10 +23,10 @@ char disable_linker_warning_about_empty_file_thread_pthread_cpp TD_UNUSED; #if TD_THREAD_PTHREAD -#include "td/utils/misc.h" - #include #include + +#include "td/utils/misc.h" #if TD_FREEBSD || TD_OPENBSD || TD_NETBSD #include #endif diff --git a/tdutils/td/utils/port/detail/ThreadPthread.h b/tdutils/td/utils/port/detail/ThreadPthread.h index b28f6fac24..7595695286 100644 --- a/tdutils/td/utils/port/detail/ThreadPthread.h +++ b/tdutils/td/utils/port/detail/ThreadPthread.h @@ -22,19 +22,18 @@ #ifdef TD_THREAD_PTHREAD -#include "td/utils/common.h" -#include "td/utils/Destructor.h" -#include "td/utils/invoke.h" -#include "td/utils/MovableValue.h" -#include "td/utils/port/detail/ThreadIdGuard.h" -#include "td/utils/port/thread_local.h" -#include "td/utils/Slice.h" - +#include #include #include #include -#include +#include "td/utils/Destructor.h" +#include "td/utils/MovableValue.h" +#include "td/utils/Slice.h" +#include "td/utils/common.h" +#include "td/utils/invoke.h" +#include "td/utils/port/detail/ThreadIdGuard.h" +#include "td/utils/port/thread_local.h" namespace td { namespace detail { @@ -52,7 +51,7 @@ class ThreadPthread { return *this; } template - explicit ThreadPthread(Function &&f, Args &&... args) { + explicit ThreadPthread(Function &&f, Args &&...args) { auto func = create_destructor([args = std::make_tuple(decay_copy(std::forward(f)), decay_copy(std::forward(args))...)]() mutable { invoke_tuple(std::move(args)); diff --git a/tdutils/td/utils/port/detail/ThreadStl.h b/tdutils/td/utils/port/detail/ThreadStl.h index d7928f1747..521c564c66 100644 --- a/tdutils/td/utils/port/detail/ThreadStl.h +++ b/tdutils/td/utils/port/detail/ThreadStl.h @@ -22,17 +22,17 @@ #ifdef TD_THREAD_STL -#include "td/utils/common.h" -#include "td/utils/invoke.h" -#include "td/utils/port/detail/ThreadIdGuard.h" -#include "td/utils/port/thread_local.h" -#include "td/utils/Slice.h" - #include #include #include #include +#include "td/utils/Slice.h" +#include "td/utils/common.h" +#include "td/utils/invoke.h" +#include "td/utils/port/detail/ThreadIdGuard.h" +#include "td/utils/port/thread_local.h" + namespace td { namespace detail { class ThreadStl { @@ -46,7 +46,7 @@ class ThreadStl { join(); } template - explicit ThreadStl(Function &&f, Args &&... args) { + explicit ThreadStl(Function &&f, Args &&...args) { thread_ = std::thread([args = std::make_tuple(decay_copy(std::forward(f)), decay_copy(std::forward(args))...)]() mutable { ThreadIdGuard thread_id_guard; diff --git a/tdutils/td/utils/port/detail/WineventPoll.h b/tdutils/td/utils/port/detail/WineventPoll.h index 7bca8d47d2..5ff3cd9832 100644 --- a/tdutils/td/utils/port/detail/WineventPoll.h +++ b/tdutils/td/utils/port/detail/WineventPoll.h @@ -23,9 +23,9 @@ #ifdef TD_POLL_WINEVENT #include "td/utils/common.h" -#include "td/utils/port/detail/PollableFd.h" #include "td/utils/port/PollBase.h" #include "td/utils/port/PollFlags.h" +#include "td/utils/port/detail/PollableFd.h" namespace td { namespace detail { diff --git a/tdutils/td/utils/port/path.cpp b/tdutils/td/utils/port/path.cpp index ebd9d96a92..8feff83b46 100644 --- a/tdutils/td/utils/port/path.cpp +++ b/tdutils/td/utils/port/path.cpp @@ -16,18 +16,16 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/port/path.h" - -#include "td/utils/port/config.h" - +#include "td/utils/ScopeGuard.h" #include "td/utils/format.h" #include "td/utils/logging.h" +#include "td/utils/port/config.h" #include "td/utils/port/detail/skip_eintr.h" -#include "td/utils/ScopeGuard.h" +#include "td/utils/port/path.h" #if TD_PORT_WINDOWS -#include "td/utils/port/wstring_convert.h" #include "td/utils/Random.h" +#include "td/utils/port/wstring_convert.h" #endif #if TD_PORT_POSIX diff --git a/tdutils/td/utils/port/path.h b/tdutils/td/utils/port/path.h index 99daa3681a..c00df224bf 100644 --- a/tdutils/td/utils/port/path.h +++ b/tdutils/td/utils/port/path.h @@ -18,15 +18,15 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/port/FileFd.h" -#include "td/utils/Slice.h" -#include "td/utils/Status.h" - #include #include #include +#include "td/utils/Slice.h" +#include "td/utils/Status.h" +#include "td/utils/common.h" +#include "td/utils/port/FileFd.h" + namespace td { Status mkdir(CSlice dir, int32 mode = 0700) TD_WARN_UNUSED_RESULT; diff --git a/tdutils/td/utils/port/rlimit.cpp b/tdutils/td/utils/port/rlimit.cpp index b3b54acd04..df1deb4001 100644 --- a/tdutils/td/utils/port/rlimit.cpp +++ b/tdutils/td/utils/port/rlimit.cpp @@ -17,11 +17,9 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/port/rlimit.h" - -#include "td/utils/port/config.h" - #include "td/utils/misc.h" +#include "td/utils/port/config.h" +#include "td/utils/port/rlimit.h" #if TD_PORT_POSIX #include @@ -104,4 +102,3 @@ td::Status change_maximize_rlimit(RlimitType rlim, td::uint64 value) { #endif } // namespace td - diff --git a/tdutils/td/utils/port/rlimit.h b/tdutils/td/utils/port/rlimit.h index 5891b0796f..77acb99e6e 100644 --- a/tdutils/td/utils/port/rlimit.h +++ b/tdutils/td/utils/port/rlimit.h @@ -19,8 +19,8 @@ #pragma once -#include "td/utils/common.h" #include "td/utils/Status.h" +#include "td/utils/common.h" namespace td { diff --git a/tdutils/td/utils/port/signals.cpp b/tdutils/td/utils/port/signals.cpp index 1bb485bec0..fd14a8bfa2 100644 --- a/tdutils/td/utils/port/signals.cpp +++ b/tdutils/td/utils/port/signals.cpp @@ -16,14 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/port/signals.h" - -#include "td/utils/port/config.h" -#include "td/utils/port/stacktrace.h" -#include "td/utils/port/StdStreams.h" - #include "td/utils/common.h" #include "td/utils/format.h" +#include "td/utils/port/StdStreams.h" +#include "td/utils/port/config.h" +#include "td/utils/port/signals.h" +#include "td/utils/port/stacktrace.h" #if TD_PORT_POSIX #include diff --git a/tdutils/td/utils/port/signals.h b/tdutils/td/utils/port/signals.h index 50d59bb4bf..e20740b969 100644 --- a/tdutils/td/utils/port/signals.h +++ b/tdutils/td/utils/port/signals.h @@ -18,9 +18,9 @@ */ #pragma once -#include "td/utils/common.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" +#include "td/utils/common.h" namespace td { diff --git a/tdutils/td/utils/port/sleep.cpp b/tdutils/td/utils/port/sleep.cpp index 7351e608ca..fc8420614d 100644 --- a/tdutils/td/utils/port/sleep.cpp +++ b/tdutils/td/utils/port/sleep.cpp @@ -16,9 +16,8 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/port/sleep.h" - #include "td/utils/port/config.h" +#include "td/utils/port/sleep.h" #if TD_PORT_POSIX #if _POSIX_C_SOURCE >= 199309L diff --git a/tdutils/td/utils/port/stacktrace.cpp b/tdutils/td/utils/port/stacktrace.cpp index e89daec671..2c993536ae 100644 --- a/tdutils/td/utils/port/stacktrace.cpp +++ b/tdutils/td/utils/port/stacktrace.cpp @@ -16,9 +16,8 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/port/stacktrace.h" - #include "td/utils/port/signals.h" +#include "td/utils/port/stacktrace.h" #if TD_WINDOWS #include @@ -58,7 +57,7 @@ void print_backtrace(void) { SymFromAddr(process, (DWORD64)(stack[i]), nullptr, symbol); // Don't use sprintf here because it is not signal-safe char buf[256 + 32]; - char* buf_ptr = buf; + char *buf_ptr = buf; if (frames - i - 1 < 10) { strcpy(buf_ptr, " "); buf_ptr += strlen(buf_ptr); diff --git a/tdutils/td/utils/port/thread.h b/tdutils/td/utils/port/thread.h index d7955c5539..003235ce70 100644 --- a/tdutils/td/utils/port/thread.h +++ b/tdutils/td/utils/port/thread.h @@ -19,7 +19,6 @@ #pragma once #include "td/utils/port/config.h" - #include "td/utils/port/detail/ThreadPthread.h" #include "td/utils/port/detail/ThreadStl.h" diff --git a/tdutils/td/utils/port/thread_local.h b/tdutils/td/utils/port/thread_local.h index eb5ca77d8d..8a2c1085f3 100644 --- a/tdutils/td/utils/port/thread_local.h +++ b/tdutils/td/utils/port/thread_local.h @@ -18,14 +18,13 @@ */ #pragma once -#include "td/utils/port/config.h" - -#include "td/utils/common.h" -#include "td/utils/Destructor.h" - #include #include +#include "td/utils/Destructor.h" +#include "td/utils/common.h" +#include "td/utils/port/config.h" + namespace td { // clang-format off @@ -41,7 +40,7 @@ namespace td { // If raw_ptr is not nullptr, allocate T as in std::make_unique(args...) and store pointer into raw_ptr template -bool init_thread_local(P &raw_ptr, ArgsT &&... args); +bool init_thread_local(P &raw_ptr, ArgsT &&...args); // Destroy all thread locals, and store nullptr into corresponding pointers void clear_thread_locals(); @@ -54,7 +53,7 @@ namespace detail { void add_thread_local_destructor(unique_ptr destructor); template -void do_init_thread_local(P &raw_ptr, ArgsT &&... args) { +void do_init_thread_local(P &raw_ptr, ArgsT &&...args) { auto ptr = std::make_unique(std::forward(args)...); raw_ptr = ptr.get(); @@ -66,7 +65,7 @@ void do_init_thread_local(P &raw_ptr, ArgsT &&... args) { } // namespace detail template -bool init_thread_local(P &raw_ptr, ArgsT &&... args) { +bool init_thread_local(P &raw_ptr, ArgsT &&...args) { if (likely(raw_ptr != nullptr)) { return false; } diff --git a/tdutils/td/utils/port/uname.cpp b/tdutils/td/utils/port/uname.cpp index 1f94de881c..09338c7522 100644 --- a/tdutils/td/utils/port/uname.cpp +++ b/tdutils/td/utils/port/uname.cpp @@ -16,15 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/port/uname.h" - -#include "td/utils/port/config.h" - #include "td/utils/common.h" #include "td/utils/filesystem.h" #include "td/utils/logging.h" #include "td/utils/misc.h" #include "td/utils/port/Stat.h" +#include "td/utils/port/config.h" +#include "td/utils/port/uname.h" #if TD_PORT_POSIX diff --git a/tdutils/td/utils/port/user.cpp b/tdutils/td/utils/port/user.cpp index 17df4b646f..49ad42fc60 100644 --- a/tdutils/td/utils/port/user.cpp +++ b/tdutils/td/utils/port/user.cpp @@ -17,9 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/port/user.h" - #include "td/utils/port/config.h" +#include "td/utils/port/user.h" #if TD_PORT_POSIX #include diff --git a/tdutils/td/utils/port/wstring_convert.cpp b/tdutils/td/utils/port/wstring_convert.cpp index e61da781ea..9dae331818 100644 --- a/tdutils/td/utils/port/wstring_convert.cpp +++ b/tdutils/td/utils/port/wstring_convert.cpp @@ -22,10 +22,10 @@ char disable_linker_warning_about_empty_file_wstring_convert_cpp TD_UNUSED; #if TD_PORT_WINDOWS -#include "td/utils/utf8.h" - #include +#include "td/utils/utf8.h" + namespace td { Result to_wstring(CSlice slice) { diff --git a/tdutils/td/utils/port/wstring_convert.h b/tdutils/td/utils/port/wstring_convert.h index 18b008c7cb..ea07d7c90a 100644 --- a/tdutils/td/utils/port/wstring_convert.h +++ b/tdutils/td/utils/port/wstring_convert.h @@ -22,11 +22,11 @@ #if TD_PORT_WINDOWS -#include "td/utils/common.h" +#include + #include "td/utils/Slice.h" #include "td/utils/Status.h" - -#include +#include "td/utils/common.h" namespace td { diff --git a/tdutils/td/utils/tests.cpp b/tdutils/td/utils/tests.cpp index 7ce7caf2fd..3f5d673679 100644 --- a/tdutils/td/utils/tests.cpp +++ b/tdutils/td/utils/tests.cpp @@ -16,19 +16,18 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/tests.h" +#include -#include "td/utils/crypto.h" -#include "td/utils/filesystem.h" #include "td/utils/Parser.h" #include "td/utils/PathView.h" -#include "td/utils/port/path.h" -#include "td/utils/port/Stat.h" #include "td/utils/ScopeGuard.h" #include "td/utils/StringBuilder.h" #include "td/utils/Time.h" - -#include +#include "td/utils/crypto.h" +#include "td/utils/filesystem.h" +#include "td/utils/port/Stat.h" +#include "td/utils/port/path.h" +#include "td/utils/tests.h" namespace td { diff --git a/tdutils/td/utils/tests.h b/tdutils/td/utils/tests.h index 9938cca7ea..efd8223dae 100644 --- a/tdutils/td/utils/tests.h +++ b/tdutils/td/utils/tests.h @@ -18,19 +18,19 @@ */ #pragma once -#include "td/utils/common.h" +#include +#include +#include + #include "td/utils/Context.h" -#include "td/utils/format.h" -#include "td/utils/logging.h" -#include "td/utils/port/thread.h" #include "td/utils/Random.h" #include "td/utils/Slice.h" #include "td/utils/Span.h" #include "td/utils/Status.h" - -#include -#include -#include +#include "td/utils/common.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/port/thread.h" #define REGISTER_TESTS(x) \ void TD_CONCAT(register_tests_, x)() { \ diff --git a/tdutils/td/utils/tl_helpers.h b/tdutils/td/utils/tl_helpers.h index 340da01ad4..42c2d03732 100644 --- a/tdutils/td/utils/tl_helpers.h +++ b/tdutils/td/utils/tl_helpers.h @@ -18,21 +18,20 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/logging.h" -#include "td/utils/misc.h" +#include +#include +#include + #include "td/utils/SharedSlice.h" #include "td/utils/Slice.h" -#include "td/utils/SharedSlice.h" #include "td/utils/StackAllocator.h" #include "td/utils/Status.h" +#include "td/utils/Variant.h" +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" #include "td/utils/tl_parsers.h" #include "td/utils/tl_storers.h" -#include "td/utils/Variant.h" - -#include -#include -#include #define BEGIN_STORE_FLAGS() \ do { \ diff --git a/tdutils/td/utils/tl_parsers.cpp b/tdutils/td/utils/tl_parsers.cpp index c41cfeb95d..449dda96b0 100644 --- a/tdutils/td/utils/tl_parsers.cpp +++ b/tdutils/td/utils/tl_parsers.cpp @@ -16,9 +16,8 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/tl_parsers.h" - #include "td/utils/misc.h" +#include "td/utils/tl_parsers.h" namespace td { diff --git a/tdutils/td/utils/tl_parsers.h b/tdutils/td/utils/tl_parsers.h index 6b5ae32b90..5ab13505d6 100644 --- a/tdutils/td/utils/tl_parsers.h +++ b/tdutils/td/utils/tl_parsers.h @@ -18,21 +18,21 @@ */ #pragma once -#include "td/utils/buffer.h" -#include "td/utils/common.h" -#include "td/utils/format.h" -#include "td/utils/logging.h" -#include "td/utils/Slice.h" -#include "td/utils/Status.h" -#include "td/utils/UInt.h" -#include "td/utils/utf8.h" - #include #include #include #include #include +#include "td/utils/Slice.h" +#include "td/utils/Status.h" +#include "td/utils/UInt.h" +#include "td/utils/buffer.h" +#include "td/utils/common.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/utf8.h" + namespace td { class TlParser { diff --git a/tdutils/td/utils/tl_storers.h b/tdutils/td/utils/tl_storers.h index 8f8c352a98..1b547bb4ea 100644 --- a/tdutils/td/utils/tl_storers.h +++ b/tdutils/td/utils/tl_storers.h @@ -18,18 +18,18 @@ */ #pragma once -#include "td/utils/common.h" -#include "td/utils/logging.h" #include "td/utils/Slice.h" #include "td/utils/StorerBase.h" #include "td/utils/UInt.h" +#include "td/utils/common.h" +#include "td/utils/logging.h" //FIXME -#include "td/utils/SharedSlice.h" -#include "crypto/common/bitstring.h" - #include +#include "crypto/common/bitstring.h" +#include "td/utils/SharedSlice.h" + namespace td { class TlStorerUnsafe { diff --git a/tdutils/td/utils/translit.cpp b/tdutils/td/utils/translit.cpp index 38ef212fb8..1cd4b86ba9 100644 --- a/tdutils/td/utils/translit.cpp +++ b/tdutils/td/utils/translit.cpp @@ -16,15 +16,14 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/translit.h" - -#include "td/utils/misc.h" -#include "td/utils/utf8.h" - #include #include #include +#include "td/utils/misc.h" +#include "td/utils/translit.h" +#include "td/utils/utf8.h" + namespace td { static const std::unordered_map &get_en_to_ru_simple_rules() { diff --git a/tdutils/td/utils/translit.h b/tdutils/td/utils/translit.h index dd8a1f7234..92cf3261eb 100644 --- a/tdutils/td/utils/translit.h +++ b/tdutils/td/utils/translit.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/utils/common.h" #include "td/utils/Slice.h" +#include "td/utils/common.h" namespace td { diff --git a/tdutils/td/utils/uint128.h b/tdutils/td/utils/uint128.h index 8e17493a0f..5360e90f85 100644 --- a/tdutils/td/utils/uint128.h +++ b/tdutils/td/utils/uint128.h @@ -18,12 +18,12 @@ */ #pragma once -#include "td/utils/bits.h" -#include "td/utils/common.h" - #include #include +#include "td/utils/bits.h" +#include "td/utils/common.h" + namespace td { class uint128_emulated { diff --git a/tdutils/td/utils/unicode.cpp b/tdutils/td/utils/unicode.cpp index 062582317c..87dd4259de 100644 --- a/tdutils/td/utils/unicode.cpp +++ b/tdutils/td/utils/unicode.cpp @@ -16,13 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/unicode.h" - -#include "td/utils/logging.h" - #include #include +#include "td/utils/logging.h" +#include "td/utils/unicode.h" + namespace td { // list of [(range_begin << 5) + range_type] diff --git a/tdutils/td/utils/unique_ptr.h b/tdutils/td/utils/unique_ptr.h index 1ec629a83b..06061379af 100644 --- a/tdutils/td/utils/unique_ptr.h +++ b/tdutils/td/utils/unique_ptr.h @@ -110,7 +110,7 @@ bool operator!=(const unique_ptr &p, std::nullptr_t) { } template -unique_ptr make_unique(Args &&... args) { +unique_ptr make_unique(Args &&...args) { return unique_ptr(new Type(std::forward(args)...)); } diff --git a/tdutils/td/utils/utf8.cpp b/tdutils/td/utils/utf8.cpp index 9872143ef4..42c1aadb6d 100644 --- a/tdutils/td/utils/utf8.cpp +++ b/tdutils/td/utils/utf8.cpp @@ -16,10 +16,9 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/utf8.h" - #include "td/utils/logging.h" // for UNREACHABLE #include "td/utils/unicode.h" +#include "td/utils/utf8.h" namespace td { diff --git a/tdutils/td/utils/utf8.h b/tdutils/td/utils/utf8.h index 0259204ce6..e370ea8a39 100644 --- a/tdutils/td/utils/utf8.h +++ b/tdutils/td/utils/utf8.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/utils/common.h" #include "td/utils/Slice.h" +#include "td/utils/common.h" namespace td { diff --git a/tdutils/test/ConcurrentHashMap.cpp b/tdutils/test/ConcurrentHashMap.cpp index 63703082a7..eece8cbc99 100644 --- a/tdutils/test/ConcurrentHashMap.cpp +++ b/tdutils/test/ConcurrentHashMap.cpp @@ -16,16 +16,16 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/benchmark.h" +#include +#include + #include "td/utils/ConcurrentHashTable.h" +#include "td/utils/SpinLock.h" +#include "td/utils/benchmark.h" #include "td/utils/misc.h" #include "td/utils/port/thread.h" -#include "td/utils/SpinLock.h" #include "td/utils/tests.h" -#include -#include - #if !TD_THREAD_UNSUPPORTED #if TD_HAVE_ABSL diff --git a/tdutils/test/EpochBasedMemoryReclamation.cpp b/tdutils/test/EpochBasedMemoryReclamation.cpp index 5e8e2986f0..ac1537d083 100644 --- a/tdutils/test/EpochBasedMemoryReclamation.cpp +++ b/tdutils/test/EpochBasedMemoryReclamation.cpp @@ -16,15 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/common.h" +#include + #include "td/utils/EpochBasedMemoryReclamation.h" +#include "td/utils/Random.h" +#include "td/utils/common.h" #include "td/utils/logging.h" #include "td/utils/port/thread.h" -#include "td/utils/Random.h" #include "td/utils/tests.h" -#include - #if !TD_THREAD_UNSUPPORTED TEST(EpochBaseMemoryReclamation, stress) { struct Node { diff --git a/tdutils/test/HazardPointers.cpp b/tdutils/test/HazardPointers.cpp index 49e2881917..e20d397333 100644 --- a/tdutils/test/HazardPointers.cpp +++ b/tdutils/test/HazardPointers.cpp @@ -16,16 +16,16 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/common.h" +#include + #include "td/utils/HazardPointers.h" -#include "td/utils/logging.h" -#include "td/utils/port/thread.h" #include "td/utils/Random.h" #include "td/utils/Slice.h" +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/port/thread.h" #include "td/utils/tests.h" -#include - #if !TD_THREAD_UNSUPPORTED TEST(HazardPointers, stress) { struct Node { diff --git a/tdutils/test/List.cpp b/tdutils/test/List.cpp index 74fe9c21c6..705c516109 100644 --- a/tdutils/test/List.cpp +++ b/tdutils/test/List.cpp @@ -17,19 +17,19 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/common.h" -#include "td/utils/List.h" -#include "td/utils/MovableValue.h" -#include "td/utils/port/thread.h" -#include "td/utils/Random.h" -#include "td/utils/tests.h" -#include "td/utils/TsList.h" - #include #include #include #include +#include "td/utils/List.h" +#include "td/utils/MovableValue.h" +#include "td/utils/Random.h" +#include "td/utils/TsList.h" +#include "td/utils/common.h" +#include "td/utils/port/thread.h" +#include "td/utils/tests.h" + struct ListData { td::MovableValue value; td::MovableValue in_list; @@ -69,9 +69,7 @@ static void do_run_list_test(ListRootT &root, std::atomic &id) { td::Random::Xorshift128plus rnd(123); - auto next_id = [&] { - return ++id; - }; + auto next_id = [&] { return ++id; }; auto add_node = [&] { if (nodes.size() >= 20) { return; diff --git a/tdutils/test/MpmcQueue.cpp b/tdutils/test/MpmcQueue.cpp index 5ede9af508..3d1980ca37 100644 --- a/tdutils/test/MpmcQueue.cpp +++ b/tdutils/test/MpmcQueue.cpp @@ -16,15 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include +#include + +#include "td/utils/MpmcQueue.h" #include "td/utils/common.h" #include "td/utils/logging.h" -#include "td/utils/MpmcQueue.h" #include "td/utils/port/thread.h" #include "td/utils/tests.h" -#include -#include - TEST(OneValue, simple) { { std::string x{"hello"}; diff --git a/tdutils/test/MpmcWaiter.cpp b/tdutils/test/MpmcWaiter.cpp index d0a9fc84e7..49c6aa3d86 100644 --- a/tdutils/test/MpmcWaiter.cpp +++ b/tdutils/test/MpmcWaiter.cpp @@ -16,14 +16,14 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include + #include "td/utils/MpmcWaiter.h" +#include "td/utils/Random.h" #include "td/utils/port/sleep.h" #include "td/utils/port/thread.h" -#include "td/utils/Random.h" #include "td/utils/tests.h" -#include - #if !TD_THREAD_UNSUPPORTED template void test_waiter_stress_one_one() { diff --git a/tdutils/test/MpscLinkQueue.cpp b/tdutils/test/MpscLinkQueue.cpp index e8f7880259..d30853eac7 100644 --- a/tdutils/test/MpscLinkQueue.cpp +++ b/tdutils/test/MpscLinkQueue.cpp @@ -16,10 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "td/utils/MpscLinkQueue.h" #include "td/utils/common.h" #include "td/utils/format.h" #include "td/utils/logging.h" -#include "td/utils/MpscLinkQueue.h" #include "td/utils/port/thread.h" #include "td/utils/tests.h" diff --git a/tdutils/test/OptionParser.cpp b/tdutils/test/OptionParser.cpp index 95cc9d527c..7dcd6fa7fe 100644 --- a/tdutils/test/OptionParser.cpp +++ b/tdutils/test/OptionParser.cpp @@ -17,11 +17,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/common.h" -#include "td/utils/misc.h" #include "td/utils/OptionParser.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" +#include "td/utils/common.h" +#include "td/utils/misc.h" #include "td/utils/tests.h" TEST(OptionParser, run) { diff --git a/tdutils/test/OrderedEventsProcessor.cpp b/tdutils/test/OrderedEventsProcessor.cpp index 434fcabed5..51301f77ec 100644 --- a/tdutils/test/OrderedEventsProcessor.cpp +++ b/tdutils/test/OrderedEventsProcessor.cpp @@ -16,14 +16,14 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/OrderedEventsProcessor.h" -#include "td/utils/Random.h" -#include "td/utils/tests.h" - #include #include #include +#include "td/utils/OrderedEventsProcessor.h" +#include "td/utils/Random.h" +#include "td/utils/tests.h" + TEST(OrderedEventsProcessor, random) { int d = 5001; int n = 1000000; diff --git a/tdutils/test/SharedObjectPool.cpp b/tdutils/test/SharedObjectPool.cpp index 868438fbda..8204a41679 100644 --- a/tdutils/test/SharedObjectPool.cpp +++ b/tdutils/test/SharedObjectPool.cpp @@ -16,12 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/common.h" +#include + #include "td/utils/SharedObjectPool.h" +#include "td/utils/common.h" #include "td/utils/tests.h" -#include - TEST(AtomicRefCnt, simple) { td::detail::AtomicRefCnt cnt{0}; cnt.inc(); @@ -101,15 +101,27 @@ TEST(SharedObjectPool, simple) { }; { td::SharedObjectPool pool; - { auto ptr1 = pool.alloc(); } - { auto ptr2 = pool.alloc(); } - { auto ptr3 = pool.alloc(); } - { auto ptr4 = pool.alloc(); } - { auto ptr5 = pool.alloc(); } + { + auto ptr1 = pool.alloc(); + } + { + auto ptr2 = pool.alloc(); + } + { + auto ptr3 = pool.alloc(); + } + { + auto ptr4 = pool.alloc(); + } + { + auto ptr5 = pool.alloc(); + } CHECK(Node::cnt() == 0); CHECK(pool.total_size() == 1); CHECK(pool.calc_free_size() == 1); - { auto ptr6 = pool.alloc(), ptr7 = pool.alloc(), ptr8 = pool.alloc(); } + { + auto ptr6 = pool.alloc(), ptr7 = pool.alloc(), ptr8 = pool.alloc(); + } CHECK(pool.total_size() == 3); CHECK(pool.calc_free_size() == 3); } diff --git a/tdutils/test/SharedSlice.cpp b/tdutils/test/SharedSlice.cpp index 265564d4bc..f438875872 100644 --- a/tdutils/test/SharedSlice.cpp +++ b/tdutils/test/SharedSlice.cpp @@ -17,9 +17,9 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "td/utils/SharedSlice.h" #include "td/utils/common.h" #include "td/utils/port/thread.h" -#include "td/utils/SharedSlice.h" #include "td/utils/tests.h" char disable_linker_warning_about_empty_file_tdutils_test_shared_slice_cpp TD_UNUSED; diff --git a/tdutils/test/StealingQueue.cpp b/tdutils/test/StealingQueue.cpp index e5439842b1..58a3640030 100644 --- a/tdutils/test/StealingQueue.cpp +++ b/tdutils/test/StealingQueue.cpp @@ -17,12 +17,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/tests.h" -#include "td/utils/benchmark.h" - #include "td/utils/AtomicRead.h" -#include "td/utils/StealingQueue.h" #include "td/utils/MpmcQueue.h" +#include "td/utils/StealingQueue.h" +#include "td/utils/benchmark.h" +#include "td/utils/tests.h" namespace td { TEST(StealingQueue, very_simple) { diff --git a/tdutils/test/buffer.cpp b/tdutils/test/buffer.cpp index 0d691da342..77487c3c40 100644 --- a/tdutils/test/buffer.cpp +++ b/tdutils/test/buffer.cpp @@ -16,10 +16,9 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/tests.h" - -#include "td/utils/buffer.h" #include "td/utils/Random.h" +#include "td/utils/buffer.h" +#include "td/utils/tests.h" using namespace td; diff --git a/tdutils/test/crypto.cpp b/tdutils/test/crypto.cpp index 39eb9e1878..2ad959e1c9 100644 --- a/tdutils/test/crypto.cpp +++ b/tdutils/test/crypto.cpp @@ -16,17 +16,17 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include + +#include "td/utils/Random.h" +#include "td/utils/Slice.h" +#include "td/utils/UInt.h" #include "td/utils/base64.h" #include "td/utils/benchmark.h" #include "td/utils/common.h" #include "td/utils/crypto.h" #include "td/utils/logging.h" -#include "td/utils/Random.h" -#include "td/utils/Slice.h" #include "td/utils/tests.h" -#include "td/utils/UInt.h" - -#include static td::vector strings{"", "1", "short test string", td::string(1000000, 'a')}; diff --git a/tdutils/test/filesystem.cpp b/tdutils/test/filesystem.cpp index e531c9663e..64eb25c986 100644 --- a/tdutils/test/filesystem.cpp +++ b/tdutils/test/filesystem.cpp @@ -16,8 +16,8 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/filesystem.h" #include "td/utils/Slice.h" +#include "td/utils/filesystem.h" #include "td/utils/tests.h" static void test_clean_filename(td::CSlice name, td::Slice result) { diff --git a/tdutils/test/gzip.cpp b/tdutils/test/gzip.cpp index 7cfb0bbaa8..d872d8ccd1 100644 --- a/tdutils/test/gzip.cpp +++ b/tdutils/test/gzip.cpp @@ -16,15 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/buffer.h" #include "td/utils/ByteFlow.h" -#include "td/utils/common.h" #include "td/utils/Gzip.h" #include "td/utils/GzipByteFlow.h" -#include "td/utils/logging.h" #include "td/utils/Status.h" -#include "td/utils/tests.h" #include "td/utils/Time.h" +#include "td/utils/buffer.h" +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/tests.h" static void encode_decode(td::string s) { auto r = td::gzencode(s, 2); diff --git a/tdutils/test/heap.cpp b/tdutils/test/heap.cpp index 20464c2e0e..764f9b55ad 100644 --- a/tdutils/test/heap.cpp +++ b/tdutils/test/heap.cpp @@ -16,16 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/tests.h" - -#include "td/utils/common.h" -#include "td/utils/Heap.h" -#include "td/utils/Random.h" - #include #include #include +#include "td/utils/Heap.h" +#include "td/utils/Random.h" +#include "td/utils/common.h" +#include "td/utils/tests.h" + REGISTER_TESTS(heap) TEST(Heap, sort_random_perm) { diff --git a/tdutils/test/json.cpp b/tdutils/test/json.cpp index f20f332eaa..b97561d75d 100644 --- a/tdutils/test/json.cpp +++ b/tdutils/test/json.cpp @@ -16,14 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/tests.h" +#include #include "td/utils/JsonBuilder.h" -#include "td/utils/logging.h" #include "td/utils/Slice.h" #include "td/utils/StringBuilder.h" - -#include +#include "td/utils/logging.h" +#include "td/utils/tests.h" REGISTER_TESTS(json) @@ -75,7 +74,9 @@ TEST(JSON, nested) { { auto a = jb.enter_array(); a << 1; - { a.enter_value().enter_array() << 2; } + { + a.enter_value().enter_array() << 2; + } a << 3; } ASSERT_EQ(jb.string_builder().is_error(), false); diff --git a/tdutils/test/log.cpp b/tdutils/test/log.cpp index 133ed4a8a7..fdf660f195 100644 --- a/tdutils/test/log.cpp +++ b/tdutils/test/log.cpp @@ -17,18 +17,18 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/benchmark.h" +#include +#include + #include "td/utils/FileLog.h" +#include "td/utils/Slice.h" +#include "td/utils/TsFileLog.h" +#include "td/utils/benchmark.h" #include "td/utils/format.h" #include "td/utils/logging.h" #include "td/utils/port/path.h" #include "td/utils/port/thread.h" -#include "td/utils/Slice.h" #include "td/utils/tests.h" -#include "td/utils/TsFileLog.h" - -#include -#include char disable_linker_warning_about_empty_file_tdutils_test_log_cpp TD_UNUSED; diff --git a/tdutils/test/misc.cpp b/tdutils/test/misc.cpp index f42e5fcdbe..bec7491fc7 100644 --- a/tdutils/test/misc.cpp +++ b/tdutils/test/misc.cpp @@ -16,46 +16,46 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/as.h" -#include "td/utils/base64.h" +#include +#include +#include +#include +#include +#include + #include "td/utils/BigNum.h" -#include "td/utils/bits.h" #include "td/utils/CancellationToken.h" -#include "td/utils/common.h" #include "td/utils/Hash.h" #include "td/utils/HashMap.h" #include "td/utils/HashSet.h" #include "td/utils/HttpUrl.h" +#include "td/utils/Random.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" +#include "td/utils/StringBuilder.h" +#include "td/utils/Time.h" +#include "td/utils/as.h" +#include "td/utils/base64.h" +#include "td/utils/bits.h" +#include "td/utils/common.h" #include "td/utils/invoke.h" #include "td/utils/logging.h" #include "td/utils/misc.h" #include "td/utils/port/EventFd.h" #include "td/utils/port/FileFd.h" #include "td/utils/port/IPAddress.h" +#include "td/utils/port/Stat.h" #include "td/utils/port/path.h" #include "td/utils/port/sleep.h" -#include "td/utils/port/Stat.h" #include "td/utils/port/thread.h" #include "td/utils/port/uname.h" #include "td/utils/port/wstring_convert.h" -#include "td/utils/Random.h" -#include "td/utils/Slice.h" -#include "td/utils/Status.h" -#include "td/utils/StringBuilder.h" #include "td/utils/tests.h" -#include "td/utils/Time.h" #include "td/utils/translit.h" #include "td/utils/uint128.h" #include "td/utils/unicode.h" #include "td/utils/utf8.h" -#include -#include -#include -#include -#include -#include - #if TD_HAVE_ABSL #include #endif diff --git a/tdutils/test/port.cpp b/tdutils/test/port.cpp index 0260228e2e..12d277a7eb 100644 --- a/tdutils/test/port.cpp +++ b/tdutils/test/port.cpp @@ -16,6 +16,9 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include + +#include "td/utils/Slice.h" #include "td/utils/common.h" #include "td/utils/logging.h" #include "td/utils/misc.h" @@ -25,11 +28,8 @@ #include "td/utils/port/signals.h" #include "td/utils/port/thread.h" #include "td/utils/port/thread_local.h" -#include "td/utils/Slice.h" #include "td/utils/tests.h" -#include - using namespace td; TEST(Port, files) { @@ -147,13 +147,12 @@ TEST(Port, Writev) { } #if TD_PORT_POSIX && !TD_THREAD_UNSUPPORTED +#include +#include #include #include #include -#include -#include - static std::mutex m; static std::vector ptrs; static std::vector addrs; diff --git a/tdutils/test/pq.cpp b/tdutils/test/pq.cpp index e9009e2eab..7060023bf9 100644 --- a/tdutils/test/pq.cpp +++ b/tdutils/test/pq.cpp @@ -16,7 +16,9 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/tests.h" +#include +#include +#include #include "td/utils/BigNum.h" #include "td/utils/common.h" @@ -24,10 +26,7 @@ #include "td/utils/format.h" #include "td/utils/logging.h" #include "td/utils/misc.h" - -#include -#include -#include +#include "td/utils/tests.h" REGISTER_TESTS(pq) diff --git a/tdutils/test/variant.cpp b/tdutils/test/variant.cpp index 5fbbde1691..e45b56a4c3 100644 --- a/tdutils/test/variant.cpp +++ b/tdutils/test/variant.cpp @@ -18,8 +18,8 @@ */ #include "td/utils/Slice.h" #include "td/utils/StringBuilder.h" -#include "td/utils/tests.h" #include "td/utils/Variant.h" +#include "td/utils/tests.h" REGISTER_TESTS(variant); diff --git a/terminal/terminal.cpp b/terminal/terminal.cpp index cbe8c74c8e..c4a383dbe5 100644 --- a/terminal/terminal.cpp +++ b/terminal/terminal.cpp @@ -16,12 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "terminal.hpp" #include "td/utils/port/StdStreams.h" +#include "terminal.hpp" + #ifdef USE_READLINE -#include #include +#include #endif #include "td/utils/find_boundary.h" diff --git a/terminal/terminal.h b/terminal/terminal.h index e716e6cf63..625e3c90b7 100644 --- a/terminal/terminal.h +++ b/terminal/terminal.h @@ -18,12 +18,12 @@ */ #pragma once -#include "td/actor/actor.h" -#include "td/utils/buffer.h" - #include #include +#include "td/actor/actor.h" +#include "td/utils/buffer.h" + namespace td { class TerminalIOOutputter { diff --git a/terminal/terminal.hpp b/terminal/terminal.hpp index 00b6415b77..244c9a2931 100644 --- a/terminal/terminal.hpp +++ b/terminal/terminal.hpp @@ -18,11 +18,13 @@ */ #pragma once -#include "terminal.h" #include +#include + #include "td/utils/port/FileFd.h" #include "td/utils/port/StdStreams.h" -#include + +#include "terminal.h" namespace td { diff --git a/test/test-adnl.cpp b/test/test-adnl.cpp index 45011fbc0b..772d1ac62e 100644 --- a/test/test-adnl.cpp +++ b/test/test-adnl.cpp @@ -25,20 +25,18 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include +#include +#include +#include + #include "adnl/adnl-network-manager.h" -#include "adnl/adnl.h" #include "adnl/adnl-test-loopback-implementation.h" - +#include "adnl/adnl.h" #include "keys/encryptor.h" - -#include "td/utils/port/signals.h" -#include "td/utils/port/path.h" #include "td/utils/Random.h" - -#include -#include -#include -#include +#include "td/utils/port/path.h" +#include "td/utils/port/signals.h" int main() { SET_VERBOSITY_LEVEL(verbosity_INFO); diff --git a/test/test-catchain.cpp b/test/test-catchain.cpp index 3131c2b95c..74616e0683 100644 --- a/test/test-catchain.cpp +++ b/test/test-catchain.cpp @@ -25,30 +25,29 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "adnl/adnl-test-loopback-implementation.h" #include "adnl/adnl.h" #include "adnl/utils.hpp" -#include "adnl/adnl-test-loopback-implementation.h" +#include "catchain/catchain.h" +#include "common/errorlog.h" #include "dht/dht.h" #include "overlay/overlays.h" #include "td/utils/OptionParser.h" +#include "td/utils/Random.h" #include "td/utils/Time.h" #include "td/utils/filesystem.h" #include "td/utils/format.h" +#include "td/utils/overloaded.h" +#include "td/utils/port/FileFd.h" #include "td/utils/port/path.h" -#include "td/utils/Random.h" #include "td/utils/port/signals.h" -#include "td/utils/port/FileFd.h" -#include "td/utils/overloaded.h" -#include "catchain/catchain.h" -#include "common/errorlog.h" #if TD_DARWIN || TD_LINUX #include #endif #include -#include - #include +#include struct Node { ton::PublicKeyHash id; diff --git a/test/test-dht.cpp b/test/test-dht.cpp index 8d814f6e05..ff3b613348 100644 --- a/test/test-dht.cpp +++ b/test/test-dht.cpp @@ -25,18 +25,17 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include +#include + #include "adnl/adnl-network-manager.h" #include "adnl/adnl-test-loopback-implementation.h" #include "adnl/adnl.h" #include "dht/dht.h" #include "dht/dht.hpp" - -#include "td/utils/port/signals.h" -#include "td/utils/port/path.h" #include "td/utils/Random.h" - -#include -#include +#include "td/utils/port/path.h" +#include "td/utils/port/signals.h" int main() { SET_VERBOSITY_LEVEL(verbosity_INFO); diff --git a/test/test-hello-world.cpp b/test/test-hello-world.cpp index aa6438e33a..f2faaab537 100644 --- a/test/test-hello-world.cpp +++ b/test/test-hello-world.cpp @@ -29,9 +29,8 @@ #include "auto/tl/ton_api.h" #include "auto/tl/ton_api_json.h" - -#include "tl/tl_json.h" #include "td/utils/Random.h" +#include "tl/tl_json.h" namespace { std::string config = R"json( diff --git a/test/test-http.cpp b/test/test-http.cpp index 3369692f66..8d07b79eed 100644 --- a/test/test-http.cpp +++ b/test/test-http.cpp @@ -25,25 +25,24 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "common/errorlog.h" +#include "http/http.h" #include "td/utils/OptionParser.h" +#include "td/utils/Random.h" #include "td/utils/Time.h" #include "td/utils/filesystem.h" #include "td/utils/format.h" +#include "td/utils/overloaded.h" +#include "td/utils/port/FileFd.h" #include "td/utils/port/path.h" -#include "td/utils/Random.h" #include "td/utils/port/signals.h" -#include "td/utils/port/FileFd.h" -#include "td/utils/overloaded.h" -#include "common/errorlog.h" -#include "http/http.h" #if TD_DARWIN || TD_LINUX #include #endif #include -#include - #include +#include void dump_reader(td::ChainBufferReader &reader) { auto b = reader.move_as_buffer_slice(); diff --git a/test/test-overlay.cpp b/test/test-overlay.cpp index 31db3e780f..39659c7af3 100644 --- a/test/test-overlay.cpp +++ b/test/test-overlay.cpp @@ -25,21 +25,22 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include +#include + #include "adnl/adnl-node-id.hpp" +#include "adnl/adnl-test-loopback-implementation.h" #include "adnl/adnl.h" #include "adnl/utils.hpp" -#include "adnl/adnl-test-loopback-implementation.h" #include "auto/tl/ton_api.h" -#include "checksum.h" #include "common/bitstring.h" +#include "common/errorlog.h" #include "dht/dht.h" #include "keys/keys.hpp" -#include "overlay-manager.h" -#include "overlay.h" -#include "overlay-id.hpp" #include "overlay/overlays.h" #include "td/actor/actor.h" #include "td/utils/OptionParser.h" +#include "td/utils/Random.h" #include "td/utils/Status.h" #include "td/utils/Time.h" #include "td/utils/UInt.h" @@ -47,24 +48,24 @@ #include "td/utils/crypto.h" #include "td/utils/filesystem.h" #include "td/utils/format.h" +#include "td/utils/overloaded.h" +#include "td/utils/port/FileFd.h" #include "td/utils/port/path.h" -#include "td/utils/Random.h" #include "td/utils/port/signals.h" -#include "td/utils/port/FileFd.h" -#include "td/utils/overloaded.h" -#include "common/errorlog.h" #include "tl-utils/common-utils.hpp" #include "tl/TlObject.h" -#include -#include + +#include "checksum.h" +#include "overlay-id.hpp" +#include "overlay-manager.h" +#include "overlay.h" #if TD_DARWIN || TD_LINUX #include #endif #include -#include - #include +#include struct Node { ton::PrivateKey pk; @@ -238,7 +239,7 @@ int main(int argc, char *argv[]) { (n1.can_receive ? 0 : ton::overlay::OverlayMemberFlags::DoNotReceiveBroadcasts); ton::overlay::OverlayMemberCertificate cert(root_nodes[i / node_slaves_cnt].id_full, 0, i % node_slaves_cnt, - 2000000000, td::BufferSlice()); + 2000000000, td::BufferSlice()); auto buf = cert.to_sign_data(n1.adnl_id); auto dec = root_nodes[i / node_slaves_cnt].pk.create_decryptor().move_as_ok(); auto signature = dec->sign(buf.as_slice()).move_as_ok(); diff --git a/test/test-rldp.cpp b/test/test-rldp.cpp index b07f5f7dc4..ce3bbf2c1a 100644 --- a/test/test-rldp.cpp +++ b/test/test-rldp.cpp @@ -25,17 +25,16 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include +#include + #include "adnl/adnl-network-manager.h" #include "adnl/adnl-test-loopback-implementation.h" #include "adnl/adnl.h" #include "rldp/rldp.h" - -#include "td/utils/port/signals.h" -#include "td/utils/port/path.h" #include "td/utils/Random.h" - -#include -#include +#include "td/utils/port/path.h" +#include "td/utils/port/signals.h" int main() { SET_VERBOSITY_LEVEL(verbosity_INFO); diff --git a/test/test-rldp2.cpp b/test/test-rldp2.cpp index 646b27d5da..f7acb01d4e 100644 --- a/test/test-rldp2.cpp +++ b/test/test-rldp2.cpp @@ -25,17 +25,16 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include +#include + #include "adnl/adnl-network-manager.h" #include "adnl/adnl-test-loopback-implementation.h" #include "adnl/adnl.h" #include "rldp2/rldp.h" - -#include "td/utils/port/signals.h" -#include "td/utils/port/path.h" #include "td/utils/Random.h" - -#include -#include +#include "td/utils/port/path.h" +#include "td/utils/port/signals.h" int main() { SET_VERBOSITY_LEVEL(verbosity_INFO); diff --git a/test/test-rocksdb.cpp b/test/test-rocksdb.cpp index 8733277760..6f8dbb3aa1 100644 --- a/test/test-rocksdb.cpp +++ b/test/test-rocksdb.cpp @@ -25,13 +25,13 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include + #include "td/db/RocksDb.h" #include "td/utils/OptionParser.h" -#include "td/utils/port/signals.h" -#include "td/utils/port/path.h" #include "td/utils/StringBuilder.h" - -#include +#include "td/utils/port/path.h" +#include "td/utils/port/signals.h" std::string dir = "stress-db"; int db_n = 20; diff --git a/test/test-td-main.cpp b/test/test-td-main.cpp index 8fda2657a8..c653489a6a 100644 --- a/test/test-td-main.cpp +++ b/test/test-td-main.cpp @@ -25,12 +25,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "td/utils/tests.h" -#include "td/utils/misc.h" - #include +#include "td/utils/misc.h" #include "td/utils/port/signals.h" +#include "td/utils/tests.h" int main(int argc, char **argv) { td::set_default_failure_signal_handler().ensure(); diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 3ee723ec88..2002f80b55 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -28,32 +28,27 @@ #include "adnl/adnl.h" #include "adnl/utils.hpp" #include "auto/tl/ton_api_json.h" +#include "catchain/catchain.h" +#include "common/errorlog.h" +#include "crypto/block/block-db.h" +#include "crypto/vm/vm.h" #include "dht/dht.h" #include "overlay/overlays.h" #include "td/utils/OptionParser.h" +#include "td/utils/Random.h" #include "td/utils/Time.h" #include "td/utils/filesystem.h" #include "td/utils/format.h" -#include "td/utils/Random.h" -#include "td/utils/port/signals.h" #include "td/utils/port/FileFd.h" -#include "catchain/catchain.h" -#include "validator-session/validator-session.h" -#include "validator/manager-disk.h" -#include "td/utils/filesystem.h" #include "td/utils/port/path.h" - -#include "ton/ton-types.h" -#include "ton/ton-tl.hpp" +#include "td/utils/port/signals.h" #include "ton/ton-io.hpp" - - +#include "ton/ton-tl.hpp" +#include "ton/ton-types.h" +#include "validator-session/validator-session.h" #include "validator/fabric.h" #include "validator/impl/collator.h" -#include "crypto/vm/vm.h" -#include "crypto/block/block-db.h" - -#include "common/errorlog.h" +#include "validator/manager-disk.h" #if TD_DARWIN || TD_LINUX #include @@ -231,12 +226,12 @@ class TestNode : public td::actor::Actor { } td::Status create_validator_options() { - if(!global_config_.length()) { + if (!global_config_.length()) { LOG(INFO) << "no global config file passed. Using zero-init config"; opts_ = ton::validator::ValidatorManagerOptions::create( - ton::BlockIdExt{ton::masterchainId, ton::shardIdAll, 0, ton::RootHash::zero(), ton::FileHash::zero()}, - ton::BlockIdExt{ton::masterchainId, ton::shardIdAll, 0, ton::RootHash::zero(), ton::FileHash::zero()}); - return td::Status::OK(); + ton::BlockIdExt{ton::masterchainId, ton::shardIdAll, 0, ton::RootHash::zero(), ton::FileHash::zero()}, + ton::BlockIdExt{ton::masterchainId, ton::shardIdAll, 0, ton::RootHash::zero(), ton::FileHash::zero()}); + return td::Status::OK(); } TRY_RESULT_PREFIX(conf_data, td::read_file(global_config_), "failed to read: "); TRY_RESULT_PREFIX(conf_json, td::json_decode(conf_data.as_slice()), "failed to parse json: "); @@ -256,7 +251,7 @@ class TestNode : public td::actor::Actor { std::vector h; for (auto &x : conf.validator_->hardforks_) { auto b = ton::create_block_id(x); - if (!b.is_masterchain()) { + if (!b.is_masterchain()) { return td::Status::Error(ton::ErrorCode::error, "[validator/hardforks] section contains not masterchain block id"); } @@ -272,12 +267,10 @@ class TestNode : public td::actor::Actor { } opts_.write().set_hardforks(std::move(h)); - - LOG(INFO) << "Hardforks num in config: "<< opts_->get_hardforks().size(); + LOG(INFO) << "Hardforks num in config: " << opts_->get_hardforks().size(); return td::Status::OK(); } - void run() { zero_id_.workchain = ton::masterchainId; td::mkdir(db_root_).ensure(); diff --git a/test/test-validator-session-state.cpp b/test/test-validator-session-state.cpp index a675070ce2..c854e4bf75 100644 --- a/test/test-validator-session-state.cpp +++ b/test/test-validator-session-state.cpp @@ -25,20 +25,18 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "adnl/adnl.h" +#include +#include +#include +#include "adnl/adnl.h" +#include "td/utils/Random.h" #include "td/utils/misc.h" -#include "td/utils/port/signals.h" #include "td/utils/port/path.h" -#include "td/utils/Random.h" - +#include "td/utils/port/signals.h" #include "validator-session/validator-session-description.h" -#include "validator-session/validator-session-state.h" #include "validator-session/validator-session-description.hpp" - -#include -#include -#include +#include "validator-session/validator-session-state.h" class Description : public ton::validatorsession::ValidatorSessionDescription { public: diff --git a/third-party/wingetopt/src/getopt.h b/third-party/wingetopt/src/getopt.h index f3f864bda6..1426b69ce3 100644 --- a/third-party/wingetopt/src/getopt.h +++ b/third-party/wingetopt/src/getopt.h @@ -15,28 +15,28 @@ /* All the headers include this file. */ #include -#if defined( WINGETOPT_SHARED_LIB ) -# if defined( BUILDING_WINGETOPT_DLL ) -# define WINGETOPT_API __declspec(dllexport) -# else -# define WINGETOPT_API __declspec(dllimport) -# endif +#if defined(WINGETOPT_SHARED_LIB) +#if defined(BUILDING_WINGETOPT_DLL) +#define WINGETOPT_API __declspec(dllexport) #else -# define WINGETOPT_API +#define WINGETOPT_API __declspec(dllimport) +#endif +#else +#define WINGETOPT_API #endif #ifdef __cplusplus extern "C" { #endif -WINGETOPT_API extern int optind; /* index of first non-option in argv */ -WINGETOPT_API extern int optopt; /* single option character, as parsed */ -WINGETOPT_API extern int opterr; /* flag to enable built-in diagnostics... */ - /* (user may set to zero, to suppress) */ +WINGETOPT_API extern int optind; /* index of first non-option in argv */ +WINGETOPT_API extern int optopt; /* single option character, as parsed */ +WINGETOPT_API extern int opterr; /* flag to enable built-in diagnostics... */ + /* (user may set to zero, to suppress) */ -WINGETOPT_API extern char *optarg; /* pointer to argument of current option */ +WINGETOPT_API extern char *optarg; /* pointer to argument of current option */ -extern int getopt(int nargc, char * const *nargv, const char *options); +extern int getopt(int nargc, char *const *nargv, const char *options); #ifdef _BSD_SOURCE /* @@ -45,7 +45,7 @@ extern int getopt(int nargc, char * const *nargv, const char *options); * proclaim their BSD heritage, before including this header; however, * to maintain portability, developers are advised to avoid it. */ -# define optreset __mingw_optreset +#define optreset __mingw_optreset extern int optreset; #endif #ifdef __cplusplus @@ -69,25 +69,24 @@ extern int optreset; extern "C" { #endif -struct option /* specification for a long form option... */ +struct option /* specification for a long form option... */ { - const char *name; /* option name, without leading hyphens */ - int has_arg; /* does it take an argument? */ - int *flag; /* where to save its status, or NULL */ - int val; /* its associated status value */ + const char *name; /* option name, without leading hyphens */ + int has_arg; /* does it take an argument? */ + int *flag; /* where to save its status, or NULL */ + int val; /* its associated status value */ }; -enum /* permitted values for its `has_arg' field... */ +enum /* permitted values for its `has_arg' field... */ { - no_argument = 0, /* option never takes an argument */ - required_argument, /* option always requires an argument */ - optional_argument /* option may take an argument */ + no_argument = 0, /* option never takes an argument */ + required_argument, /* option always requires an argument */ + optional_argument /* option may take an argument */ }; -extern int getopt_long(int nargc, char * const *nargv, const char *options, - const struct option *long_options, int *idx); -extern int getopt_long_only(int nargc, char * const *nargv, const char *options, - const struct option *long_options, int *idx); +extern int getopt_long(int nargc, char *const *nargv, const char *options, const struct option *long_options, int *idx); +extern int getopt_long_only(int nargc, char *const *nargv, const char *options, const struct option *long_options, + int *idx); /* * Previous MinGW implementation had... */ @@ -95,7 +94,7 @@ extern int getopt_long_only(int nargc, char * const *nargv, const char *options, /* * ...for the long form API only; keep this for compatibility. */ -# define HAVE_DECL_GETOPT 1 +#define HAVE_DECL_GETOPT 1 #endif #ifdef __cplusplus diff --git a/tl-utils/common-utils.hpp b/tl-utils/common-utils.hpp index 05f8a825fe..5486e4e119 100644 --- a/tl-utils/common-utils.hpp +++ b/tl-utils/common-utils.hpp @@ -17,10 +17,9 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "tl/tl_object_parse.h" -#include "td/utils/tl_parsers.h" - #include "crypto/common/bitstring.h" +#include "td/utils/tl_parsers.h" +#include "tl/tl_object_parse.h" namespace ton { @@ -224,19 +223,19 @@ td::Result fetch_result(const td::BufferSlice &message, } template -td::BufferSlice create_serialize_tl_object(Args &&... args) { +td::BufferSlice create_serialize_tl_object(Args &&...args) { Type object(std::forward(args)...); return serialize_tl_object(&object, true); } template -td::BufferSlice create_serialize_tl_object_suffix(td::Slice suffix, Args &&... args) { +td::BufferSlice create_serialize_tl_object_suffix(td::Slice suffix, Args &&...args) { Type object(std::forward(args)...); return serialize_tl_object(&object, true, suffix); } template -auto create_hash_tl_object(Args &&... args) { +auto create_hash_tl_object(Args &&...args) { Type object(std::forward(args)...); return get_tl_object_sha_bits256(&object); } diff --git a/tl-utils/lite-utils.cpp b/tl-utils/lite-utils.cpp index 387474cfad..21ce247b54 100644 --- a/tl-utils/lite-utils.cpp +++ b/tl-utils/lite-utils.cpp @@ -16,13 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tl-utils.hpp" -#include "tl/tl_object_store.h" +#include + #include "auto/tl/lite_api.hpp" -#include "td/utils/tl_storers.h" -#include "td/utils/crypto.h" #include "crypto/common/bitstring.h" -#include +#include "td/utils/crypto.h" +#include "td/utils/tl_storers.h" +#include "tl/tl_object_store.h" + +#include "tl-utils.hpp" namespace ton { diff --git a/tl-utils/lite-utils.hpp b/tl-utils/lite-utils.hpp index fb3c1dc881..da17bbb32a 100644 --- a/tl-utils/lite-utils.hpp +++ b/tl-utils/lite-utils.hpp @@ -19,11 +19,9 @@ #pragma once #include "auto/tl/lite_api.h" - -#include "tl/tl_object_parse.h" -#include "td/utils/tl_parsers.h" - #include "crypto/common/bitstring.h" +#include "td/utils/tl_parsers.h" +#include "tl/tl_object_parse.h" #include "common-utils.hpp" diff --git a/tl-utils/tl-utils.cpp b/tl-utils/tl-utils.cpp index dc2c2ec057..598a05dd1a 100644 --- a/tl-utils/tl-utils.cpp +++ b/tl-utils/tl-utils.cpp @@ -16,12 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tl-utils.hpp" -#include "tl/tl_object_store.h" #include "auto/tl/ton_api.hpp" -#include "td/utils/tl_storers.h" -#include "td/utils/crypto.h" #include "crypto/common/bitstring.h" +#include "td/utils/crypto.h" +#include "td/utils/tl_storers.h" +#include "tl/tl_object_store.h" + +#include "tl-utils.hpp" namespace ton { diff --git a/tl-utils/tl-utils.hpp b/tl-utils/tl-utils.hpp index e46d6f9602..3e4430937d 100644 --- a/tl-utils/tl-utils.hpp +++ b/tl-utils/tl-utils.hpp @@ -19,11 +19,9 @@ #pragma once #include "auto/tl/ton_api.h" - -#include "tl/tl_object_parse.h" -#include "td/utils/tl_parsers.h" - #include "crypto/common/bitstring.h" +#include "td/utils/tl_parsers.h" +#include "tl/tl_object_parse.h" #include "common-utils.hpp" diff --git a/tl/generate/generate_common.cpp b/tl/generate/generate_common.cpp index 0068ccbd4c..28f0f5eee2 100644 --- a/tl/generate/generate_common.cpp +++ b/tl/generate/generate_common.cpp @@ -25,19 +25,19 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include +#include + +#include "td/tl/tl_config.h" +#include "td/tl/tl_generate.h" #include "td/tl/tl_writer.h" + +#include "tl_json_converter.h" #include "tl_writer_cpp.h" #include "tl_writer_h.h" #include "tl_writer_hpp.h" -#include "tl_writer_jni_h.h" #include "tl_writer_jni_cpp.h" -#include "tl_json_converter.h" - -#include "td/tl/tl_config.h" -#include "td/tl/tl_generate.h" - -#include -#include +#include "tl_writer_jni_h.h" template diff --git a/tl/generate/generate_java.cpp b/tl/generate/generate_java.cpp index 8f2bc31b46..7e837ca34f 100644 --- a/tl/generate/generate_java.cpp +++ b/tl/generate/generate_java.cpp @@ -25,13 +25,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tl_writer_java.h" +#include +#include #include "td/tl/tl_config.h" #include "td/tl/tl_generate.h" -#include -#include +#include "tl_writer_java.h" int main(int argc, char *argv[]) { if (argc < 5) { diff --git a/tl/generate/tl_json_converter.cpp b/tl/generate/tl_json_converter.cpp index d1a0488fb8..5170820c00 100644 --- a/tl/generate/tl_json_converter.cpp +++ b/tl/generate/tl_json_converter.cpp @@ -16,18 +16,17 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tl_json_converter.h" +#include #include "td/tl/tl_simple.h" - +#include "td/utils/Slice.h" +#include "td/utils/StringBuilder.h" #include "td/utils/buffer.h" #include "td/utils/common.h" #include "td/utils/filesystem.h" #include "td/utils/logging.h" -#include "td/utils/Slice.h" -#include "td/utils/StringBuilder.h" -#include +#include "tl_json_converter.h" namespace td { diff --git a/tl/generate/tl_json_converter.h b/tl/generate/tl_json_converter.h index 3cb96dafe1..9bc0078436 100644 --- a/tl/generate/tl_json_converter.h +++ b/tl/generate/tl_json_converter.h @@ -27,11 +27,11 @@ */ #pragma once +#include + #include "td/tl/tl_config.h" #include "td/tl/tl_writer.h" -#include - namespace td { void gen_json_converter(const tl::tl_config &config, const std::string &file_name, const std::string &tl_name, diff --git a/tl/generate/tl_writer_cpp.cpp b/tl/generate/tl_writer_cpp.cpp index a06cf124bc..056f3cf71e 100644 --- a/tl/generate/tl_writer_cpp.cpp +++ b/tl/generate/tl_writer_cpp.cpp @@ -16,10 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tl_writer_cpp.h" - #include +#include "tl_writer_cpp.h" + namespace td { std::string TD_TL_writer_cpp::gen_output_begin() const { diff --git a/tl/generate/tl_writer_cpp.h b/tl/generate/tl_writer_cpp.h index 2698161b56..230a9093cf 100644 --- a/tl/generate/tl_writer_cpp.h +++ b/tl/generate/tl_writer_cpp.h @@ -18,12 +18,12 @@ */ #pragma once -#include "tl_writer_td.h" - #include #include #include +#include "tl_writer_td.h" + namespace td { class TD_TL_writer_cpp : public TD_TL_writer { diff --git a/tl/generate/tl_writer_h.cpp b/tl/generate/tl_writer_h.cpp index cc3f4e485e..e14cfebe4a 100644 --- a/tl/generate/tl_writer_h.cpp +++ b/tl/generate/tl_writer_h.cpp @@ -16,11 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tl_writer_h.h" - #include #include +#include "tl_writer_h.h" + namespace td { std::string TD_TL_writer_h::forward_declaration(std::string type) { diff --git a/tl/generate/tl_writer_h.h b/tl/generate/tl_writer_h.h index 00c9a79de4..4ab306ec3f 100644 --- a/tl/generate/tl_writer_h.h +++ b/tl/generate/tl_writer_h.h @@ -18,12 +18,12 @@ */ #pragma once -#include "tl_writer_td.h" - #include #include #include +#include "tl_writer_td.h" + namespace td { class TD_TL_writer_h : public TD_TL_writer { diff --git a/tl/generate/tl_writer_hpp.cpp b/tl/generate/tl_writer_hpp.cpp index cdcf26c4dd..be59f0fa4a 100644 --- a/tl/generate/tl_writer_hpp.cpp +++ b/tl/generate/tl_writer_hpp.cpp @@ -16,10 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tl_writer_hpp.h" - #include +#include "tl_writer_hpp.h" + namespace td { bool TD_TL_writer_hpp::is_documentation_generated() const { diff --git a/tl/generate/tl_writer_hpp.h b/tl/generate/tl_writer_hpp.h index f892e4280f..84f8ebc542 100644 --- a/tl/generate/tl_writer_hpp.h +++ b/tl/generate/tl_writer_hpp.h @@ -18,12 +18,12 @@ */ #pragma once -#include "tl_writer_td.h" - #include #include #include +#include "tl_writer_td.h" + namespace td { class TD_TL_writer_hpp : public TD_TL_writer { diff --git a/tl/generate/tl_writer_java.cpp b/tl/generate/tl_writer_java.cpp index d371c2f8e3..eb0986122a 100644 --- a/tl/generate/tl_writer_java.cpp +++ b/tl/generate/tl_writer_java.cpp @@ -16,10 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tl_writer_java.h" - #include +#include "tl_writer_java.h" + namespace td { const int TD_TL_writer_java::MAX_ARITY; @@ -33,9 +33,9 @@ int TD_TL_writer_java::get_max_arity() const { } bool TD_TL_writer_java::is_built_in_simple_type(const std::string &name) const { - return name == "Bool" || name == "Int32" || name == "Int53" || name == "Int64" || name == "Int128" || name == "Int256" || name == "Double" || - name == "String" || name == "Bytes" || name == "SecureString" || name == "SecureBytes" || name == "Object" || - name == "Function"; + return name == "Bool" || name == "Int32" || name == "Int53" || name == "Int64" || name == "Int128" || + name == "Int256" || name == "Double" || name == "String" || name == "Bytes" || name == "SecureString" || + name == "SecureBytes" || name == "Object" || name == "Function"; } bool TD_TL_writer_java::is_built_in_complex_type(const std::string &name) const { diff --git a/tl/generate/tl_writer_java.h b/tl/generate/tl_writer_java.h index e1f11c3168..9c3d943aa3 100644 --- a/tl/generate/tl_writer_java.h +++ b/tl/generate/tl_writer_java.h @@ -18,12 +18,12 @@ */ #pragma once -#include "td/tl/tl_writer.h" - #include #include #include +#include "td/tl/tl_writer.h" + namespace td { class TD_TL_writer_java : public tl::TL_writer { diff --git a/tl/generate/tl_writer_jni_cpp.cpp b/tl/generate/tl_writer_jni_cpp.cpp index 20afaa894d..3403ddd8d3 100644 --- a/tl/generate/tl_writer_jni_cpp.cpp +++ b/tl/generate/tl_writer_jni_cpp.cpp @@ -16,18 +16,18 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tl_writer_jni_cpp.h" - #include #include +#include "tl_writer_jni_cpp.h" + namespace td { bool TD_TL_writer_jni_cpp::is_built_in_simple_type(const std::string &name) const { assert(name != "function"); - return name == "Bool" || name == "Int32" || name == "Int53" || name == "Int64" || name == "Int128" || name == "Int256" || name == "Double" || - name == "String" || name == "Bytes" || name == "SecureString" || name == "SecureBytes" || name == "Function" || - name == "Object"; + return name == "Bool" || name == "Int32" || name == "Int53" || name == "Int64" || name == "Int128" || + name == "Int256" || name == "Double" || name == "String" || name == "Bytes" || name == "SecureString" || + name == "SecureBytes" || name == "Function" || name == "Object"; } bool TD_TL_writer_jni_cpp::is_built_in_complex_type(const std::string &name) const { @@ -312,8 +312,9 @@ std::string TD_TL_writer_jni_cpp::gen_vector_store(const std::string &field_name assert(false); // TODO } if (vector_type == "std::int32_t" || vector_type == "std::int64_t" || vector_type == "double" || - vector_type == string_type || vector_type == secure_string_type || vector_type == "td::Bits128" || vector_type == "td::Bits256" || - vector_type.compare(0, 11, "std::vector") == 0 || vector_type.compare(0, 10, "object_ptr") == 0) { + vector_type == string_type || vector_type == secure_string_type || vector_type == "td::Bits128" || + vector_type == "td::Bits256" || vector_type.compare(0, 11, "std::vector") == 0 || + vector_type.compare(0, 10, "object_ptr") == 0) { return "{ " "auto arr_tmp_ = td::jni::store_vector(env, " + field_name + @@ -355,8 +356,8 @@ std::string TD_TL_writer_jni_cpp::gen_type_store(const std::string &field_name, } std::string res; - if (name == "Int32" || name == "Int53" || name == "Int64" || name == "Int128" || name == "Int256" || name == "Double" || name == "Bool" || name == "String" || - name == "SecureString" || name == "#") { + if (name == "Int32" || name == "Int53" || name == "Int64" || name == "Int128" || name == "Int256" || + name == "Double" || name == "Bool" || name == "String" || name == "SecureString" || name == "#") { if (storer_type == 1) { res = "s.store_field(\"" + get_pretty_field_name(field_name) + "\", " + field_name + ");"; } else if (name == "Bool") { @@ -384,7 +385,8 @@ std::string TD_TL_writer_jni_cpp::gen_type_store(const std::string &field_name, "); if (nextBytes) { env->SetObjectField(s, " + field_name + "fieldID, nextBytes); env->DeleteLocalRef(nextBytes); } }"; } else if (name == "#") { - res = "env->SetIntField(s, " + TD_TL_writer_cpp::get_pretty_field_name(field_name) + "_fieldID, " + field_name + ");"; + res = "env->SetIntField(s, " + TD_TL_writer_cpp::get_pretty_field_name(field_name) + "_fieldID, " + field_name + + ");"; } else { assert(false); } diff --git a/tl/generate/tl_writer_jni_cpp.h b/tl/generate/tl_writer_jni_cpp.h index f9cc8fed89..f9061c08d6 100644 --- a/tl/generate/tl_writer_jni_cpp.h +++ b/tl/generate/tl_writer_jni_cpp.h @@ -18,13 +18,13 @@ */ #pragma once -#include "tl_writer_cpp.h" - #include #include #include #include +#include "tl_writer_cpp.h" + namespace td { class TD_TL_writer_jni_cpp : public TD_TL_writer_cpp { diff --git a/tl/generate/tl_writer_jni_h.cpp b/tl/generate/tl_writer_jni_h.cpp index dc2551fdff..6f64ed48cb 100644 --- a/tl/generate/tl_writer_jni_h.cpp +++ b/tl/generate/tl_writer_jni_h.cpp @@ -16,16 +16,16 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tl_writer_jni_h.h" - #include +#include "tl_writer_jni_h.h" + namespace td { bool TD_TL_writer_jni_h::is_built_in_simple_type(const std::string &name) const { - return name == "Bool" || name == "Int32" || name == "Int53" || name == "Int64" || name == "Int128" || name == "Int256" || name == "Double" || - name == "String" || name == "Bytes" || name == "SecureString" || name == "SecureBytes" || name == "Function" || - name == "Object"; + return name == "Bool" || name == "Int32" || name == "Int53" || name == "Int64" || name == "Int128" || + name == "Int256" || name == "Double" || name == "String" || name == "Bytes" || name == "SecureString" || + name == "SecureBytes" || name == "Function" || name == "Object"; } bool TD_TL_writer_jni_h::is_built_in_complex_type(const std::string &name) const { diff --git a/tl/generate/tl_writer_jni_h.h b/tl/generate/tl_writer_jni_h.h index 0d0f7f85df..13c8aab3cc 100644 --- a/tl/generate/tl_writer_jni_h.h +++ b/tl/generate/tl_writer_jni_h.h @@ -18,11 +18,11 @@ */ #pragma once -#include "tl_writer_h.h" - #include #include +#include "tl_writer_h.h" + namespace td { class TD_TL_writer_jni_h : public TD_TL_writer_h { diff --git a/tl/generate/tl_writer_td.cpp b/tl/generate/tl_writer_td.cpp index dbcc025d50..b1d99f8157 100644 --- a/tl/generate/tl_writer_td.cpp +++ b/tl/generate/tl_writer_td.cpp @@ -16,10 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tl_writer_td.h" - -#include #include +#include + +#include "tl_writer_td.h" namespace td { diff --git a/tl/generate/tl_writer_td.h b/tl/generate/tl_writer_td.h index 53a297b156..48d91d667f 100644 --- a/tl/generate/tl_writer_td.h +++ b/tl/generate/tl_writer_td.h @@ -18,11 +18,11 @@ */ #pragma once -#include "td/tl/tl_writer.h" - #include #include +#include "td/tl/tl_writer.h" + namespace td { class TD_TL_writer : public tl::TL_writer { diff --git a/tl/tl/TlObject.h b/tl/tl/TlObject.h index a781187a23..411ff2f4be 100644 --- a/tl/tl/TlObject.h +++ b/tl/tl/TlObject.h @@ -118,7 +118,7 @@ using tl_object_ptr = std::unique_ptr; * \return Wrapped pointer to the created TL-object. */ template -tl_object_ptr create_tl_object(Args &&... args) { +tl_object_ptr create_tl_object(Args &&...args) { return tl_object_ptr(new Type(std::forward(args)...)); } diff --git a/tl/tl/tl_jni_object.cpp b/tl/tl/tl_jni_object.cpp index 26a94d6273..9a6383b661 100644 --- a/tl/tl/tl_jni_object.cpp +++ b/tl/tl/tl_jni_object.cpp @@ -16,14 +16,14 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tl_jni_object.h" +#include +#include "td/utils/Slice.h" #include "td/utils/common.h" #include "td/utils/logging.h" #include "td/utils/misc.h" -#include "td/utils/Slice.h" -#include +#include "tl_jni_object.h" namespace td { namespace jni { diff --git a/tl/tl/tl_jni_object.h b/tl/tl/tl_jni_object.h index e39a063d9b..aafcc16b9c 100644 --- a/tl/tl/tl_jni_object.h +++ b/tl/tl/tl_jni_object.h @@ -18,16 +18,15 @@ */ #pragma once -#include - #include +#include #include #include #include -#include "td/utils/Slice.h" -#include "td/utils/SharedSlice.h" #include "common/bitstring.h" +#include "td/utils/SharedSlice.h" +#include "td/utils/Slice.h" namespace td { namespace jni { @@ -113,7 +112,7 @@ SecureString from_bytes_secure(JNIEnv *env, jbyteArray arr); jbyteArray to_bytes(JNIEnv *env, Slice b); jbyteArray to_bytes_secure(JNIEnv *env, Slice b); -template +template td::BitArray from_bits(JNIEnv *env, jbyteArray arr) { td::BitArray b; if (arr != nullptr) { @@ -125,7 +124,7 @@ td::BitArray from_bits(JNIEnv *env, jbyteArray arr) { return b; } -template +template jbyteArray to_bits(JNIEnv *env, td::BitArray b) { assert(n % 8 == 0); jsize length = n / 8; @@ -148,7 +147,7 @@ jobjectArray store_vector(JNIEnv *env, const std::vector &v); jobjectArray store_vector(JNIEnv *env, const std::vector &v); -template +template jobjectArray store_vector(JNIEnv *env, const std::vector> &v) { jint length = static_cast(v.size()); jobjectArray arr = env->NewObjectArray(length, ObjectClass, jobject()); @@ -276,7 +275,7 @@ struct FetchVector { } }; -template +template struct FetchVector> { static std::vector> fetch(JNIEnv *env, jobjectArray arr) { std::vector> result; diff --git a/tl/tl/tl_json.h b/tl/tl/tl_json.h index 8eee3aad58..bf91d06370 100644 --- a/tl/tl/tl_json.h +++ b/tl/tl/tl_json.h @@ -18,18 +18,18 @@ */ #pragma once +#include + +#include "td/utils/JsonBuilder.h" +#include "td/utils/SharedSlice.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" #include "td/utils/base64.h" #include "td/utils/buffer.h" #include "td/utils/format.h" -#include "td/utils/JsonBuilder.h" #include "td/utils/misc.h" -#include "td/utils/Slice.h" -#include "td/utils/SharedSlice.h" -#include "td/utils/Status.h" #include "td/utils/tl_storers.h" -#include - namespace td { struct JsonInt64 { int64 value; diff --git a/tl/tl/tl_object_parse.h b/tl/tl/tl_object_parse.h index aa621d89d7..c683d50c07 100644 --- a/tl/tl/tl_object_parse.h +++ b/tl/tl/tl_object_parse.h @@ -23,10 +23,9 @@ #include #include -#include "tl/TlObject.h" - -#include "td/utils/int_types.h" #include "crypto/common/bitstring.h" +#include "td/utils/int_types.h" +#include "tl/TlObject.h" namespace ton { diff --git a/tl/tl/tl_object_store.h b/tl/tl/tl_object_store.h index fef7eec5a5..2e34be4c3e 100644 --- a/tl/tl/tl_object_store.h +++ b/tl/tl/tl_object_store.h @@ -23,9 +23,8 @@ #include #include -#include "tl/TlObject.h" - #include "td/utils/misc.h" +#include "tl/TlObject.h" namespace ton { diff --git a/ton/lite-tl.hpp b/ton/lite-tl.hpp index 024748fd83..0f1556294e 100644 --- a/ton/lite-tl.hpp +++ b/ton/lite-tl.hpp @@ -18,9 +18,10 @@ */ #pragma once -#include "ton-types.h" #include "auto/tl/lite_api.h" +#include "ton-types.h" + namespace ton { inline tl_object_ptr create_tl_lite_block_id(const BlockIdExt &block_id) { diff --git a/ton/ton-io.hpp b/ton/ton-io.hpp index 1905dc5fd0..bb3210a26d 100644 --- a/ton/ton-io.hpp +++ b/ton/ton-io.hpp @@ -18,9 +18,10 @@ */ #pragma once -#include "ton-types.h" #include "td/utils/base64.h" +#include "ton-types.h" + namespace td { inline td::StringBuilder &operator<<(td::StringBuilder &stream, const ton::Bits256 &x) { diff --git a/ton/ton-shard.h b/ton/ton-shard.h index b68d3ec003..8767d3b99c 100644 --- a/ton/ton-shard.h +++ b/ton/ton-shard.h @@ -18,9 +18,10 @@ */ #pragma once -#include "ton-types.h" -#include "td/utils/bits.h" #include "crypto/common/bitstring.h" +#include "td/utils/bits.h" + +#include "ton-types.h" namespace ton { diff --git a/ton/ton-tl.hpp b/ton/ton-tl.hpp index 1bc9a28c81..61dc79362a 100644 --- a/ton/ton-tl.hpp +++ b/ton/ton-tl.hpp @@ -18,10 +18,11 @@ */ #pragma once -#include "ton-types.h" #include "auto/tl/ton_api.hpp" #include "td/utils/overloaded.h" +#include "ton-types.h" + namespace ton { inline tl_object_ptr create_tl_block_id(const BlockIdExt &block_id) { diff --git a/ton/ton-types.h b/ton/ton-types.h index 87947d027f..010106a5a4 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -18,16 +18,16 @@ */ #pragma once +#include + #include "crypto/common/bitstring.h" -#include "td/utils/buffer.h" -#include "td/utils/bits.h" #include "td/utils/Slice.h" #include "td/utils/UInt.h" +#include "td/utils/bits.h" +#include "td/utils/buffer.h" #include "td/utils/misc.h" #include "td/utils/optional.h" -#include - namespace ton { using WorkchainId = td::int32; diff --git a/tonlib/test/offline.cpp b/tonlib/test/offline.cpp index 47d5d6a268..0933ba2a4d 100644 --- a/tonlib/test/offline.cpp +++ b/tonlib/test/offline.cpp @@ -17,35 +17,31 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "block/block.h" -#include "block/block-auto.h" -#include "block/mc-config.h" - -#include "vm/cells.h" -#include "vm/boc.h" -#include "vm/cells/CellString.h" - -#include "tonlib/utils.h" -#include "tonlib/TonlibClient.h" -#include "tonlib/Client.h" - #include "auto/tl/ton_api_json.h" #include "auto/tl/tonlib_api_json.h" - +#include "block/block-auto.h" +#include "block/block.h" +#include "block/mc-config.h" +#include "td/utils/PathView.h" #include "td/utils/benchmark.h" #include "td/utils/filesystem.h" #include "td/utils/optional.h" #include "td/utils/overloaded.h" #include "td/utils/port/path.h" -#include "td/utils/PathView.h" #include "td/utils/tests.h" +#include "tonlib/Client.h" +#include "tonlib/TonlibClient.h" +#include "tonlib/utils.h" +#include "vm/boc.h" +#include "vm/cells.h" +#include "vm/cells/CellString.h" // KeyManager -#include "tonlib/keys/bip39.h" #include "tonlib/keys/DecryptedKey.h" #include "tonlib/keys/EncryptedKey.h" #include "tonlib/keys/Mnemonic.h" #include "tonlib/keys/SimpleEncryption.h" +#include "tonlib/keys/bip39.h" TEST(Tonlib, CellString) { for (unsigned size : @@ -689,7 +685,7 @@ TEST(Tonlib, ConfigCache) { sync_send(client, make_object( make_object(testnet, "testnet", true, false))) .ensure(); - sync_send(client, make_object( - make_object(custom, "testnet", true, false))) + sync_send(client, + make_object(make_object(custom, "testnet", true, false))) .ensure_error(); } diff --git a/tonlib/test/online.cpp b/tonlib/test/online.cpp index cf892c292d..96a0e0c021 100644 --- a/tonlib/test/online.cpp +++ b/tonlib/test/online.cpp @@ -26,41 +26,37 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "adnl/adnl-ext-client.h" -#include "tl-utils/tl-utils.hpp" #include "auto/tl/ton_api_json.h" #include "auto/tl/tonlib_api_json.h" -#include "tl/tl_json.h" -#include "ton/ton-types.h" -#include "ton/ton-tl.hpp" -#include "block/block.h" #include "block/block-auto.h" -#include "Ed25519.h" - +#include "block/block.h" #include "smc-envelope/GenericAccount.h" #include "smc-envelope/ManualDns.h" #include "smc-envelope/MultisigWallet.h" -#include "tonlib/LastBlock.h" -#include "tonlib/ExtClient.h" -#include "tonlib/utils.h" - -#include "tonlib/TonlibCallback.h" -#include "tonlib/Client.h" - -#include "vm/cells.h" -#include "vm/boc.h" -#include "vm/cells/MerkleProof.h" - #include "td/utils/Container.h" +#include "td/utils/MpscPollableQueue.h" #include "td/utils/OptionParser.h" #include "td/utils/Random.h" #include "td/utils/filesystem.h" -#include "td/utils/tests.h" #include "td/utils/optional.h" #include "td/utils/overloaded.h" -#include "td/utils/MpscPollableQueue.h" #include "td/utils/port/path.h" - #include "td/utils/port/signals.h" +#include "td/utils/tests.h" +#include "tl-utils/tl-utils.hpp" +#include "tl/tl_json.h" +#include "ton/ton-tl.hpp" +#include "ton/ton-types.h" +#include "tonlib/Client.h" +#include "tonlib/ExtClient.h" +#include "tonlib/LastBlock.h" +#include "tonlib/TonlibCallback.h" +#include "tonlib/utils.h" +#include "vm/boc.h" +#include "vm/cells.h" +#include "vm/cells/MerkleProof.h" + +#include "Ed25519.h" using namespace tonlib; @@ -594,9 +590,9 @@ void test_multisig(Client& client, const Wallet& giver_wallet) { void dns_resolve(Client& client, const Wallet& dns, std::string name) { using namespace ton::tonlib_api; auto address = dns.get_address(); - auto resolved = - sync_send(client, make_object<::ton::tonlib_api::dns_resolve>( - std::move(address), name, td::sha256_bits256(td::Slice("cat", 3)), 4)).move_as_ok(); + auto resolved = sync_send(client, make_object<::ton::tonlib_api::dns_resolve>( + std::move(address), name, td::sha256_bits256(td::Slice("cat", 3)), 4)) + .move_as_ok(); CHECK(resolved->entries_.size() == 1); LOG(INFO) << to_string(resolved); LOG(INFO) << "OK"; @@ -748,17 +744,14 @@ void test_dns(Client& client, const Wallet& giver_wallet) { using namespace ton::tonlib_api; std::vector> actions; - actions.push_back(make_object( - make_object("A", ton::DNS_NEXT_RESOLVER_CATEGORY, - make_object(A_B.get_address())))); + actions.push_back(make_object(make_object( + "A", ton::DNS_NEXT_RESOLVER_CATEGORY, make_object(A_B.get_address())))); auto init_A = create_update_dns_query(client, A, std::move(actions)).move_as_ok(); - actions.push_back(make_object( - make_object("B", ton::DNS_NEXT_RESOLVER_CATEGORY, - make_object(A_B_C.get_address())))); + actions.push_back(make_object(make_object( + "B", ton::DNS_NEXT_RESOLVER_CATEGORY, make_object(A_B_C.get_address())))); auto init_A_B = create_update_dns_query(client, A_B, std::move(actions)).move_as_ok(); - actions.push_back( - make_object(make_object("C", td::sha256_bits256(td::Slice("cat", 3)), - make_object("Hello dns")))); + actions.push_back(make_object(make_object("C", td::sha256_bits256(td::Slice("cat", 3)), + make_object("Hello dns")))); auto init_A_B_C = create_update_dns_query(client, A_B_C, std::move(actions)).move_as_ok(); LOG(INFO) << "Send dns init queries"; diff --git a/tonlib/tonlib/Client.cpp b/tonlib/tonlib/Client.cpp index 056aa0dbb0..58aea5829a 100644 --- a/tonlib/tonlib/Client.cpp +++ b/tonlib/tonlib/Client.cpp @@ -16,13 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "Client.h" - -#include "tonlib/TonlibClient.h" -#include "tonlib/TonlibCallback.h" - #include "td/actor/actor.h" #include "td/utils/MpscPollableQueue.h" +#include "tonlib/TonlibCallback.h" +#include "tonlib/TonlibClient.h" + +#include "Client.h" int VERBOSITY_NAME(tonlib_requests) = VERBOSITY_NAME(DEBUG); diff --git a/tonlib/tonlib/ClientJson.cpp b/tonlib/tonlib/ClientJson.cpp index 11a7f4f60d..81b9e7cf90 100644 --- a/tonlib/tonlib/ClientJson.cpp +++ b/tonlib/tonlib/ClientJson.cpp @@ -16,20 +16,17 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tonlib/ClientJson.h" +#include #include "auto/tl/tonlib_api_json.h" - -#include "tl/tl_json.h" - +#include "td/utils/JsonBuilder.h" +#include "td/utils/Status.h" #include "td/utils/common.h" #include "td/utils/format.h" -#include "td/utils/JsonBuilder.h" #include "td/utils/logging.h" #include "td/utils/port/thread_local.h" -#include "td/utils/Status.h" - -#include +#include "tl/tl_json.h" +#include "tonlib/ClientJson.h" namespace tonlib { diff --git a/tonlib/tonlib/ClientJson.h b/tonlib/tonlib/ClientJson.h index fa640372c9..a3cf92221b 100644 --- a/tonlib/tonlib/ClientJson.h +++ b/tonlib/tonlib/ClientJson.h @@ -18,16 +18,15 @@ */ #pragma once -#include "tonlib/Client.h" - -#include "td/utils/Slice.h" - #include #include #include #include #include +#include "td/utils/Slice.h" +#include "tonlib/Client.h" + namespace tonlib { class ClientJson final { diff --git a/tonlib/tonlib/Config.cpp b/tonlib/tonlib/Config.cpp index 063b34bbdf..f8d5d0139c 100644 --- a/tonlib/tonlib/Config.cpp +++ b/tonlib/tonlib/Config.cpp @@ -16,12 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "Config.h" #include "adnl/adnl-node-id.hpp" -#include "td/utils/JsonBuilder.h" #include "auto/tl/ton_api_json.h" +#include "td/utils/JsonBuilder.h" #include "ton/ton-tl.hpp" +#include "Config.h" + namespace tonlib { td::Result parse_block_id_ext(td::JsonObject &obj) { ton::WorkchainId zero_workchain_id; diff --git a/tonlib/tonlib/Config.h b/tonlib/tonlib/Config.h index 28f23881ba..8c6a81bb1d 100644 --- a/tonlib/tonlib/Config.h +++ b/tonlib/tonlib/Config.h @@ -18,9 +18,9 @@ */ #pragma once #include "adnl/adnl-node-id.hpp" +#include "lite-client/ext-client.h" #include "td/utils/port/IPAddress.h" #include "ton/ton-types.h" -#include "lite-client/ext-client.h" namespace tonlib { struct Config { diff --git a/tonlib/tonlib/ExtClient.cpp b/tonlib/tonlib/ExtClient.cpp index b66ca25c1d..18976d24e4 100644 --- a/tonlib/tonlib/ExtClient.cpp +++ b/tonlib/tonlib/ExtClient.cpp @@ -17,7 +17,6 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "tonlib/ExtClient.h" - #include "tonlib/LastBlock.h" #include "tonlib/LastConfig.h" diff --git a/tonlib/tonlib/ExtClient.h b/tonlib/tonlib/ExtClient.h index b0194c18fd..78019f63b8 100644 --- a/tonlib/tonlib/ExtClient.h +++ b/tonlib/tonlib/ExtClient.h @@ -18,17 +18,14 @@ */ #pragma once #include "adnl/adnl-ext-client.h" -#include "tl-utils/lite-utils.hpp" - #include "auto/tl/lite_api.h" - -#include "ton/ton-types.h" - +#include "lite-client/ext-client.h" #include "td/actor/actor.h" #include "td/utils/Container.h" #include "td/utils/Random.h" +#include "tl-utils/lite-utils.hpp" +#include "ton/ton-types.h" -#include "lite-client/ext-client.h" #include "TonlibError.h" #include "utils.h" @@ -86,8 +83,7 @@ class ExtClient { return TonlibError::LiteServer(f->code_, f->message_); } return ton::fetch_result(std::move(data)); - } - (); + }(); VLOG_IF(lite_server, res.is_ok()) << "got result from liteserver: " << tag << " " << td::Slice(to_string(res.ok())).truncate(1 << 12); VLOG_IF(lite_server, res.is_error()) << "got error from liteserver: " << tag << " " << res.error(); diff --git a/tonlib/tonlib/ExtClientOutbound.cpp b/tonlib/tonlib/ExtClientOutbound.cpp index e5fac8b477..e97c3ab5ae 100644 --- a/tonlib/tonlib/ExtClientOutbound.cpp +++ b/tonlib/tonlib/ExtClientOutbound.cpp @@ -17,9 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include + #include "ExtClientOutbound.h" #include "TonlibError.h" -#include namespace tonlib { diff --git a/tonlib/tonlib/ExtClientOutbound.h b/tonlib/tonlib/ExtClientOutbound.h index bf52c8c260..6218d72a93 100644 --- a/tonlib/tonlib/ExtClientOutbound.h +++ b/tonlib/tonlib/ExtClientOutbound.h @@ -17,8 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/actor/actor.h" #include "lite-client/ext-client.h" +#include "td/actor/actor.h" namespace tonlib { class ExtClientOutbound : public liteclient::ExtClient { diff --git a/tonlib/tonlib/KeyStorage.cpp b/tonlib/tonlib/KeyStorage.cpp index cd2a73bc01..c34f047906 100644 --- a/tonlib/tonlib/KeyStorage.cpp +++ b/tonlib/tonlib/KeyStorage.cpp @@ -16,18 +16,16 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "KeyStorage.h" - -#include "tonlib/keys/Mnemonic.h" +#include "td/utils/PathView.h" +#include "td/utils/crypto.h" +#include "td/utils/filesystem.h" +#include "td/utils/port/path.h" +#include "tonlib/TonlibError.h" #include "tonlib/keys/DecryptedKey.h" #include "tonlib/keys/EncryptedKey.h" +#include "tonlib/keys/Mnemonic.h" -#include "tonlib/TonlibError.h" - -#include "td/utils/filesystem.h" -#include "td/utils/port/path.h" -#include "td/utils/crypto.h" -#include "td/utils/PathView.h" +#include "KeyStorage.h" namespace tonlib { namespace { diff --git a/tonlib/tonlib/KeyStorage.h b/tonlib/tonlib/KeyStorage.h index 9d012ccdd5..5652e13663 100644 --- a/tonlib/tonlib/KeyStorage.h +++ b/tonlib/tonlib/KeyStorage.h @@ -18,13 +18,13 @@ */ #pragma once -#include "td/utils/Status.h" +#include + #include "td/utils/SharedSlice.h" +#include "td/utils/Status.h" #include "KeyValue.h" -#include - namespace tonlib { struct DecryptedKey; class KeyStorage { diff --git a/tonlib/tonlib/KeyValue.cpp b/tonlib/tonlib/KeyValue.cpp index b62ce7c018..fb24d41773 100644 --- a/tonlib/tonlib/KeyValue.cpp +++ b/tonlib/tonlib/KeyValue.cpp @@ -16,15 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "KeyValue.h" +#include +#include +#include +#include "td/utils/PathView.h" #include "td/utils/filesystem.h" #include "td/utils/port/path.h" -#include "td/utils/PathView.h" -#include -#include -#include +#include "KeyValue.h" namespace tonlib { namespace detail { @@ -110,9 +110,8 @@ class KeyValueDir : public KeyValue { return false; } - return std::all_of(key.begin(), key.end(), [](char c) { - return std::isalnum(c) || c == '_' || c == '-' || c == '.'; - }); + return std::all_of(key.begin(), key.end(), + [](char c) { return std::isalnum(c) || c == '_' || c == '-' || c == '.'; }); } }; @@ -149,7 +148,7 @@ class KeyValueInmemory : public KeyValue { return td::Status::OK(); } void foreach_key(std::function f) override { - for (auto &it : map_) { + for (auto& it : map_) { f(it.first); } } diff --git a/tonlib/tonlib/KeyValue.h b/tonlib/tonlib/KeyValue.h index 33aef44e7f..77ae5c78e1 100644 --- a/tonlib/tonlib/KeyValue.h +++ b/tonlib/tonlib/KeyValue.h @@ -17,12 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include + #include "td/utils/SharedSlice.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" -#include - namespace tonlib { class KeyValue { public: diff --git a/tonlib/tonlib/LastBlock.cpp b/tonlib/tonlib/LastBlock.cpp index 10a4d10fba..97b7cde018 100644 --- a/tonlib/tonlib/LastBlock.cpp +++ b/tonlib/tonlib/LastBlock.cpp @@ -16,17 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "lite-client/lite-client-common.h" +#include "td/utils/JsonBuilder.h" +#include "ton/lite-tl.hpp" #include "tonlib/LastBlock.h" #include "tonlib/LastConfig.h" - #include "tonlib/utils.h" -#include "ton/lite-tl.hpp" - -#include "lite-client/lite-client-common.h" - -#include "td/utils/JsonBuilder.h" - namespace tonlib { // init_state <-> last_key_block diff --git a/tonlib/tonlib/LastBlock.h b/tonlib/tonlib/LastBlock.h index df6e192cb7..d8ca5be604 100644 --- a/tonlib/tonlib/LastBlock.h +++ b/tonlib/tonlib/LastBlock.h @@ -18,12 +18,10 @@ */ #pragma once #include "td/actor/actor.h" - -#include "tonlib/Config.h" -#include "tonlib/ExtClient.h" - #include "td/utils/CancellationToken.h" #include "td/utils/tl_helpers.h" +#include "tonlib/Config.h" +#include "tonlib/ExtClient.h" namespace block { struct BlockProofChain; diff --git a/tonlib/tonlib/LastBlockStorage.cpp b/tonlib/tonlib/LastBlockStorage.cpp index 6e5b3f0f16..c206b628db 100644 --- a/tonlib/tonlib/LastBlockStorage.cpp +++ b/tonlib/tonlib/LastBlockStorage.cpp @@ -16,14 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "LastBlockStorage.h" - -#include "tonlib/utils.h" - #include "td/utils/as.h" #include "td/utils/filesystem.h" #include "td/utils/port/path.h" #include "td/utils/tl_helpers.h" +#include "tonlib/utils.h" + +#include "LastBlockStorage.h" namespace tonlib { @@ -53,7 +52,7 @@ std::string get_file_name(td::Slice name) { } // namespace td::Result LastBlockStorage::get_state(td::Slice name) { - // This migration addresses an issue in the old version of Tonlib, where the td::buffer_to_hex + // This migration addresses an issue in the old version of Tonlib, where the td::buffer_to_hex // incorrectly reversed the order of nibbles in hex representation. auto data_r = kv_->get(get_file_name(name)); if (data_r.is_error()) { diff --git a/tonlib/tonlib/LastBlockStorage.h b/tonlib/tonlib/LastBlockStorage.h index b87d9634d7..39d3a51afb 100644 --- a/tonlib/tonlib/LastBlockStorage.h +++ b/tonlib/tonlib/LastBlockStorage.h @@ -18,9 +18,8 @@ */ #pragma once -#include "tonlib/LastBlock.h" - #include "tonlib/KeyValue.h" +#include "tonlib/LastBlock.h" namespace tonlib { class LastBlockStorage { diff --git a/tonlib/tonlib/LastConfig.cpp b/tonlib/tonlib/LastConfig.cpp index 45d6c9ebb3..f5d026059f 100644 --- a/tonlib/tonlib/LastConfig.cpp +++ b/tonlib/tonlib/LastConfig.cpp @@ -16,16 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tonlib/LastConfig.h" - -#include "tonlib/utils.h" - -#include "ton/lite-tl.hpp" +#include "block/block-auto.h" #include "block/check-proof.h" #include "block/mc-config.h" -#include "block/block-auto.h" - #include "lite-client/lite-client-common.h" +#include "ton/lite-tl.hpp" +#include "tonlib/LastConfig.h" +#include "tonlib/utils.h" #include "LastBlock.h" diff --git a/tonlib/tonlib/LastConfig.h b/tonlib/tonlib/LastConfig.h index 901733dcc9..223ecad552 100644 --- a/tonlib/tonlib/LastConfig.h +++ b/tonlib/tonlib/LastConfig.h @@ -17,15 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "block/mc-config.h" #include "td/actor/actor.h" - -#include "tonlib/Config.h" -#include "tonlib/ExtClient.h" - #include "td/utils/CancellationToken.h" #include "td/utils/tl_helpers.h" - -#include "block/mc-config.h" +#include "tonlib/Config.h" +#include "tonlib/ExtClient.h" namespace tonlib { struct LastConfigState { diff --git a/tonlib/tonlib/Logging.cpp b/tonlib/tonlib/Logging.cpp index 4a52c88f67..9fb7345013 100644 --- a/tonlib/tonlib/Logging.cpp +++ b/tonlib/tonlib/Logging.cpp @@ -16,19 +16,17 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "Logging.h" -#include "utils.h" +#include +#include +#include #include "auto/tl/tonlib_api.h" - #include "td/utils/FileLog.h" #include "td/utils/logging.h" #include "td/utils/misc.h" -#include "td/utils/misc.h" -#include -#include -#include +#include "Logging.h" +#include "utils.h" namespace tonlib { @@ -44,8 +42,7 @@ auto &log_data() { return data; } -#define ADD_TAG(tag) \ - { #tag, &VERBOSITY_NAME(tag) } +#define ADD_TAG(tag) {#tag, &VERBOSITY_NAME(tag)} static const std::map log_tags{ADD_TAG(tonlib_query), ADD_TAG(last_block), ADD_TAG(last_config), ADD_TAG(lite_server)}; #undef ADD_TAG diff --git a/tonlib/tonlib/Logging.h b/tonlib/tonlib/Logging.h index 708836b928..d02fd7e49c 100644 --- a/tonlib/tonlib/Logging.h +++ b/tonlib/tonlib/Logging.h @@ -19,10 +19,9 @@ #pragma once #include "auto/tl/tonlib_api.h" - -#include "td/utils/common.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" +#include "td/utils/common.h" namespace tonlib { namespace tonlib_api = ton::tonlib_api; diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 9eafe9076d..b39f04d726 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -16,52 +16,45 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "TonlibClient.h" - -#include "tonlib/ExtClientOutbound.h" -#include "tonlib/LastBlock.h" -#include "tonlib/LastConfig.h" -#include "tonlib/Logging.h" -#include "tonlib/utils.h" -#include "tonlib/keys/Mnemonic.h" -#include "tonlib/keys/SimpleEncryption.h" -#include "tonlib/TonlibError.h" - +#include "auto/tl/tonlib_api.hpp" +#include "block/block-auto.h" +#include "block/check-proof.h" +#include "common/util.h" +#include "emulator/transaction-emulator.h" +#include "lite-client/lite-client-common.h" #include "smc-envelope/GenericAccount.h" -#include "smc-envelope/ManualDns.h" -#include "smc-envelope/WalletV3.h" -#include "smc-envelope/WalletV4.h" #include "smc-envelope/HighloadWallet.h" #include "smc-envelope/HighloadWalletV2.h" +#include "smc-envelope/ManualDns.h" #include "smc-envelope/PaymentChannel.h" #include "smc-envelope/SmartContractCode.h" - -#include "emulator/transaction-emulator.h" - -#include "auto/tl/tonlib_api.hpp" -#include "block/block-auto.h" -#include "block/check-proof.h" +#include "smc-envelope/WalletV3.h" +#include "smc-envelope/WalletV4.h" +#include "td/actor/MultiPromise.h" +#include "td/utils/Random.h" +#include "td/utils/as.h" +#include "td/utils/optional.h" +#include "td/utils/overloaded.h" +#include "td/utils/port/path.h" +#include "td/utils/tests.h" #include "ton/lite-tl.hpp" #include "ton/ton-shard.h" -#include "lite-client/lite-client-common.h" - +#include "tonlib/ExtClientOutbound.h" +#include "tonlib/LastBlock.h" +#include "tonlib/LastConfig.h" +#include "tonlib/Logging.h" +#include "tonlib/TonlibError.h" +#include "tonlib/keys/Mnemonic.h" +#include "tonlib/keys/SimpleEncryption.h" +#include "tonlib/utils.h" #include "vm/boc.h" #include "vm/cellops.h" #include "vm/cells/MerkleProof.h" -#include "vm/vm.h" #include "vm/cp0.h" #include "vm/memo.h" +#include "vm/vm.h" -#include "td/utils/as.h" -#include "td/utils/Random.h" -#include "td/utils/optional.h" -#include "td/utils/overloaded.h" - -#include "td/utils/tests.h" -#include "td/utils/port/path.h" - -#include "common/util.h" -#include "td/actor/MultiPromise.h" +#include "TonlibClient.h" template using lite_api_ptr = ton::lite_api::object_ptr; @@ -237,7 +230,7 @@ td::Result>> parse td::Result> to_extra_currenctes_dict( const std::vector>& extra_currencies) { vm::Dictionary dict{32}; - for (const auto &f : extra_currencies) { + for (const auto& f : extra_currencies) { if (f->amount_ == 0) { continue; } @@ -253,7 +246,7 @@ td::Result> to_extra_currenctes_dict( return std::move(dict).extract_root_cell(); } -td::Status check_enough_extra_currencies(const td::Ref &balance, const td::Ref &amount) { +td::Status check_enough_extra_currencies(const td::Ref& balance, const td::Ref& amount) { block::CurrencyCollection c1{td::zero_refint(), balance}; block::CurrencyCollection c2{td::zero_refint(), amount}; auto res = TRY_VM(td::Result{c1 >= c2}); @@ -264,7 +257,7 @@ td::Status check_enough_extra_currencies(const td::Ref &balance, const return td::Status::OK(); } -td::Result> add_extra_currencies(const td::Ref &e1, const td::Ref &e2) { +td::Result> add_extra_currencies(const td::Ref& e1, const td::Ref& e2) { block::CurrencyCollection c1{td::zero_refint(), e1}; block::CurrencyCollection c2{td::zero_refint(), e2}; TRY_RESULT_ASSIGN(c1, TRY_VM(td::Result{c1 + c2})); @@ -528,7 +521,11 @@ class AccountState { if (account_root.is_null()) { block::gen::Account().cell_pack_account_none(account_root); } - auto cell = vm::CellBuilder().store_ref(account_root).store_bits(raw_.info.last_trans_hash.as_bitslice()).store_long(raw_.info.last_trans_lt).finalize(); + auto cell = vm::CellBuilder() + .store_ref(account_root) + .store_bits(raw_.info.last_trans_hash.as_bitslice()) + .store_long(raw_.info.last_trans_lt) + .finalize(); return tonlib_api::make_object(to_bytes(cell)); } @@ -537,7 +534,11 @@ class AccountState { if (account_root.is_null()) { block::gen::Account().cell_pack_account_none(account_root); } - return vm::CellBuilder().store_ref(account_root).store_bits(raw_.info.last_trans_hash.as_bitslice()).store_long(raw_.info.last_trans_lt).as_cellslice_ref(); + return vm::CellBuilder() + .store_ref(account_root) + .store_bits(raw_.info.last_trans_hash.as_bitslice()) + .store_long(raw_.info.last_trans_lt) + .as_cellslice_ref(); } //NB: Order is important! Used during guessAccountRevision @@ -1130,8 +1131,9 @@ td::Result to_balance(td::Ref balance_ref) { class GetTransactionHistory : public td::actor::Actor { public: - GetTransactionHistory(ExtClientRef ext_client_ref, block::StdAddress address, ton::LogicalTime lt, ton::Bits256 hash, td::int32 count, - td::actor::ActorShared<> parent, td::Promise promise) + GetTransactionHistory(ExtClientRef ext_client_ref, block::StdAddress address, ton::LogicalTime lt, ton::Bits256 hash, + td::int32 count, td::actor::ActorShared<> parent, + td::Promise promise) : address_(std::move(address)) , lt_(std::move(lt)) , hash_(std::move(hash)) @@ -1803,9 +1805,8 @@ class GetShardBlockProof : public td::actor::Actor { class GetOutMsgQueueSizes : public td::actor::Actor { public: - GetOutMsgQueueSizes(ExtClientRef ext_client_ref, std::vector blocks, - td::actor::ActorShared<> parent, - td::Promise>&& promise) + GetOutMsgQueueSizes(ExtClientRef ext_client_ref, std::vector blocks, td::actor::ActorShared<> parent, + td::Promise>&& promise) : blocks_(std::move(blocks)), parent_(std::move(parent)), promise_(std::move(promise)) { client_.set_client(ext_client_ref); } @@ -1930,14 +1931,15 @@ class GetOutMsgQueueSizes : public td::actor::Actor { auto to_lite_api(const tonlib_api::ton_blockIdExt& blk) -> td::Result>; auto to_tonlib_api(const ton::lite_api::liteServer_transactionId& txid) -> tonlib_api_ptr; -td::Status check_block_transactions_proof(lite_api_ptr& bTxes, int32_t mode, - ton::LogicalTime start_lt, td::Bits256 start_addr, td::Bits256 root_hash, int req_count); +td::Status check_block_transactions_proof(lite_api_ptr& bTxes, + int32_t mode, ton::LogicalTime start_lt, td::Bits256 start_addr, + td::Bits256 root_hash, int req_count); class RunEmulator : public TonlibQueryActor { public: RunEmulator(ExtClientRef ext_client_ref, int_api::GetAccountStateByTransaction request, - td::actor::ActorShared parent, td::Promise>&& promise) - : TonlibQueryActor(std::move(parent)), request_(std::move(request)), promise_(std::move(promise)) { + td::actor::ActorShared parent, td::Promise>&& promise) + : TonlibQueryActor(std::move(parent)), request_(std::move(request)), promise_(std::move(promise)) { client_.set_client(ext_client_ref); } @@ -1957,10 +1959,10 @@ class RunEmulator : public TonlibQueryActor { td::int64 actor_id_{1}; FullBlockId block_id_; - td::Ref mc_state_root_; // ^ShardStateUnsplit + td::Ref mc_state_root_; // ^ShardStateUnsplit td::unique_ptr account_state_; vm::Dictionary global_libraries_{256}; - std::vector> transactions_; // std::vector<^Transaction> + std::vector> transactions_; // std::vector<^Transaction> size_t count_{0}; size_t count_transactions_{0}; @@ -1969,72 +1971,77 @@ class RunEmulator : public TonlibQueryActor { void get_block_id(td::Promise&& promise) { auto shard_id = ton::shard_prefix(request_.address.addr, 60); - auto query = ton::lite_api::liteServer_lookupBlock(0b111111010, ton::create_tl_lite_block_id_simple({request_.address.workchain, shard_id, 0}), request_.lt, 0); - client_.send_query(std::move(query), promise.wrap([shard_id](td::Result> header_r) -> td::Result { - - TRY_RESULT(header, std::move(header_r)); - ton::BlockIdExt block_id = ton::create_block_id(header->id_); - TRY_RESULT(root, vm::std_boc_deserialize(std::move(header->header_proof_))); - - try { - auto virt_root = vm::MerkleProof::virtualize(root); - if (virt_root.is_null()) { - return td::Status::Error("block header proof is not a valid Merkle proof"); - } + auto query = ton::lite_api::liteServer_lookupBlock( + 0b111111010, ton::create_tl_lite_block_id_simple({request_.address.workchain, shard_id, 0}), request_.lt, 0); + client_.send_query( + std::move(query), + promise.wrap([shard_id](td::Result> header_r) + -> td::Result { + TRY_RESULT(header, std::move(header_r)); + ton::BlockIdExt block_id = ton::create_block_id(header->id_); + TRY_RESULT(root, vm::std_boc_deserialize(std::move(header->header_proof_))); + + try { + auto virt_root = vm::MerkleProof::virtualize(root); + if (virt_root.is_null()) { + return td::Status::Error("block header proof is not a valid Merkle proof"); + } - if (ton::RootHash{virt_root->get_hash().bits()} != block_id.root_hash) { - return td::Status::Error("block header has incorrect root hash"); - } + if (ton::RootHash{virt_root->get_hash().bits()} != block_id.root_hash) { + return td::Status::Error("block header has incorrect root hash"); + } - std::vector prev_blocks; - ton::BlockIdExt mc_block_id; - bool after_split; - td::Status status = block::unpack_block_prev_blk_ext(virt_root, block_id, prev_blocks, mc_block_id, after_split); - if (status.is_error()) { - return status.move_as_error(); - } + std::vector prev_blocks; + ton::BlockIdExt mc_block_id; + bool after_split; + td::Status status = + block::unpack_block_prev_blk_ext(virt_root, block_id, prev_blocks, mc_block_id, after_split); + if (status.is_error()) { + return status.move_as_error(); + } - ton::BlockIdExt prev_block; - if (prev_blocks.size() == 1 || ton::shard_is_ancestor(prev_blocks[0].id.shard, shard_id)) { - prev_block = std::move(prev_blocks[0]); - } else { - prev_block = std::move(prev_blocks[1]); - } + ton::BlockIdExt prev_block; + if (prev_blocks.size() == 1 || ton::shard_is_ancestor(prev_blocks[0].id.shard, shard_id)) { + prev_block = std::move(prev_blocks[0]); + } else { + prev_block = std::move(prev_blocks[1]); + } - block::gen::Block::Record block; - block::gen::BlockExtra::Record extra; - if (!tlb::unpack_cell(virt_root, block) || !tlb::unpack_cell(block.extra, extra)) { - return td::Status::Error("cannot unpack block header"); - } + block::gen::Block::Record block; + block::gen::BlockExtra::Record extra; + if (!tlb::unpack_cell(virt_root, block) || !tlb::unpack_cell(block.extra, extra)) { + return td::Status::Error("cannot unpack block header"); + } - return FullBlockId{std::move(block_id), std::move(mc_block_id), std::move(prev_block), std::move(extra.rand_seed)}; - } catch (vm::VmError& err) { - return err.as_status("error processing header"); - } catch (vm::VmVirtError& err) { - return err.as_status("error processing header"); - } - })); + return FullBlockId{std::move(block_id), std::move(mc_block_id), std::move(prev_block), + std::move(extra.rand_seed)}; + } catch (vm::VmError& err) { + return err.as_status("error processing header"); + } catch (vm::VmVirtError& err) { + return err.as_status("error processing header"); + } + })); } void get_mc_state_root(td::Promise>&& promise) { TRY_RESULT_PROMISE(promise, lite_block, to_lite_api(*to_tonlib_api(block_id_.mc))); auto block = ton::create_block_id(lite_block); - client_.send_query(ton::lite_api::liteServer_getConfigAll(0b11'11111111, std::move(lite_block)), promise.wrap([block](auto r_config) -> td::Result> { + client_.send_query(ton::lite_api::liteServer_getConfigAll(0b11'11111111, std::move(lite_block)), + promise.wrap([block](auto r_config) -> td::Result> { + TRY_RESULT(state, block::check_extract_state_proof(block, r_config->state_proof_.as_slice(), + r_config->config_proof_.as_slice())); - TRY_RESULT(state, block::check_extract_state_proof(block, r_config->state_proof_.as_slice(), r_config->config_proof_.as_slice())); - - return std::move(state); - })); + return std::move(state); + })); } void get_account_state(td::Promise>&& promise) { auto actor_id = actor_id_++; actors_[actor_id] = td::actor::create_actor( - "GetAccountState", client_.get_client(), request_.address, block_id_.prev, - actor_shared(this, actor_id), - promise.wrap([address = request_.address](auto&& state) { - return td::make_unique(std::move(address), std::move(state), 0); - })); + "GetAccountState", client_.get_client(), request_.address, block_id_.prev, actor_shared(this, actor_id), + promise.wrap([address = request_.address](auto&& state) { + return td::make_unique(std::move(address), std::move(state), 0); + })); } td::Status get_transactions(std::int64_t lt) { @@ -2042,9 +2049,11 @@ class RunEmulator : public TonlibQueryActor { auto after = ton::lite_api::make_object(request_.address.addr, lt); auto mode = 0b10100111; constexpr int req_count = 256; - auto query = ton::lite_api::liteServer_listBlockTransactions(std::move(lite_block), mode, req_count, std::move(after), false, true); + auto query = ton::lite_api::liteServer_listBlockTransactions(std::move(lite_block), mode, req_count, + std::move(after), false, true); - client_.send_query(std::move(query), [self = this, mode, lt, root_hash = block_id_.id.root_hash](lite_api_ptr&& bTxes) { + client_.send_query(std::move(query), [self = this, mode, lt, root_hash = block_id_.id.root_hash]( + lite_api_ptr&& bTxes) { if (!bTxes) { self->check(td::Status::Error("liteServer.blockTransactions is null")); return; @@ -2064,7 +2073,9 @@ class RunEmulator : public TonlibQueryActor { } self->transactions_.push_back({}); - self->get_transaction(id->lt_, id->hash_, [self, i = self->transactions_.size() - 1](auto transaction) { self->set_transaction(i, std::move(transaction)); }); + self->get_transaction(id->lt_, id->hash_, [self, i = self->transactions_.size() - 1](auto transaction) { + self->set_transaction(i, std::move(transaction)); + }); if (!self->incomplete_) { return; @@ -2084,9 +2095,8 @@ class RunEmulator : public TonlibQueryActor { auto actor_id = actor_id_++; actors_[actor_id] = td::actor::create_actor( "GetTransactionHistory", client_.get_client(), request_.address, lt, hash, 1, actor_shared(this, actor_id), - promise.wrap([](auto&& transactions) mutable { - return std::move(transactions.transactions.front().transaction); - })); + promise.wrap( + [](auto&& transactions) mutable { return std::move(transactions.transactions.front().transaction); })); } void start_up() override { @@ -2660,10 +2670,10 @@ td::Status TonlibClient::do_request(tonlib_api::guessAccount& request, request.public_key_, wallet_id_ + ton::basechainId), ton::basechainId}); sources.push_back(Source{tonlib_api::make_object( - request.public_key_, wallet_id_ + ton::masterchainId), + request.public_key_, wallet_id_ + ton::masterchainId), ton::masterchainId}); sources.push_back(Source{tonlib_api::make_object( - request.public_key_, wallet_id_ + ton::basechainId), + request.public_key_, wallet_id_ + ton::basechainId), ton::basechainId}); for (Source& source : sources) { auto o_type = get_wallet_type(*source.init_state); @@ -3040,8 +3050,7 @@ td::Result TonlibClient::validate_config(tonlib_api::o state.vert_seqno = vert_seqno; bool user_defined_init_block = false; - if (new_config.init_block_id.is_valid() && - state.last_key_block_id.id.seqno < new_config.init_block_id.id.seqno) { + if (new_config.init_block_id.is_valid() && state.last_key_block_id.id.seqno < new_config.init_block_id.id.seqno) { state.last_key_block_id = new_config.init_block_id; user_defined_init_block = true; LOG(INFO) << "Use init block from USER config: " << new_config.init_block_id.to_str(); @@ -3137,8 +3146,7 @@ td::Result to_std_address(td::Ref cs) { } struct ToRawTransactions { explicit ToRawTransactions(td::optional private_key, bool try_decode_messages = true) - : private_key_(std::move(private_key)) - , try_decode_messages_(try_decode_messages) { + : private_key_(std::move(private_key)), try_decode_messages_(try_decode_messages) { } td::optional private_key_; @@ -3204,7 +3212,8 @@ struct ToRawTransactions { } } if (!data) { - data = tonlib_api::make_object(to_bytes(std::move(body_cell)), to_bytes(std::move(init_state_cell))); + data = tonlib_api::make_object(to_bytes(std::move(body_cell)), + to_bytes(std::move(init_state_cell))); } return data; }; @@ -3239,8 +3248,7 @@ struct ToRawTransactions { } TRY_RESULT(dest, to_std_address(msg_info.dest)); return tonlib_api::make_object( - msg_hash, - tonlib_api::make_object(), + msg_hash, tonlib_api::make_object(), tonlib_api::make_object(std::move(dest)), 0, std::vector>{}, 0, 0, 0, std::move(body_hash), get_data("")); @@ -3253,8 +3261,7 @@ struct ToRawTransactions { TRY_RESULT(src, to_std_address(msg_info.src)); auto created_lt = static_cast(msg_info.created_lt); return tonlib_api::make_object( - msg_hash, - tonlib_api::make_object(src), + msg_hash, tonlib_api::make_object(src), tonlib_api::make_object(), 0, std::vector>{}, 0, 0, created_lt, std::move(body_hash), get_data(src)); @@ -3314,8 +3321,7 @@ struct ToRawTransactions { address = std_address.rserialize(true); } return tonlib_api::make_object( - tonlib_api::make_object(std::move(address)), - info.now, data, + tonlib_api::make_object(std::move(address)), info.now, data, tonlib_api::make_object(info.prev_trans_lt, info.prev_trans_hash.as_slice().str()), fees, storage_fee, fees - storage_fee, std::move(in_msg), std::move(out_msgs)); @@ -3388,14 +3394,13 @@ struct ToRawTransactions { address = std_address.rserialize(true); } return tonlib_api::make_object( - tonlib_api::make_object(std::move(address)), - info.now, data, - tonlib_api::make_object(info.lt, - info.hash.as_slice().str()), - fees, storage_fee, fees - storage_fee, std::move(in_msg), std::move(out_msgs)); + tonlib_api::make_object(std::move(address)), info.now, data, + tonlib_api::make_object(info.lt, info.hash.as_slice().str()), fees, + storage_fee, fees - storage_fee, std::move(in_msg), std::move(out_msgs)); } - td::Result> to_raw_transaction(block::BlockTransaction::Info&& info) { + td::Result> to_raw_transaction( + block::BlockTransaction::Info&& info) { return TRY_VM(to_raw_transaction_or_throw(std::move(info))); } @@ -3453,14 +3458,13 @@ td::Result get_ext_in_msg_hash_norm(td::Ref ext_in_msg_ce } auto cb = vm::CellBuilder(); - bool status = - cb.store_long_bool(2, 2) && // message$_ -> info:CommonMsgInfo -> ext_in_msg_info$10 - cb.store_long_bool(0, 2) && // message$_ -> info:CommonMsgInfo -> src:MsgAddressExt -> addr_none$00 - cb.append_cellslice_bool(msg_info.dest) && // message$_ -> info:CommonMsgInfo -> dest:MsgAddressInt - cb.store_long_bool(0, 4) && // message$_ -> info:CommonMsgInfo -> import_fee:Grams -> 0 - cb.store_long_bool(0, 1) && // message$_ -> init:(Maybe (Either StateInit ^StateInit)) -> nothing$0 - cb.store_long_bool(1, 1) && // message$_ -> body:(Either X ^X) -> right$1 - cb.store_ref_bool(body); + bool status = cb.store_long_bool(2, 2) && // message$_ -> info:CommonMsgInfo -> ext_in_msg_info$10 + cb.store_long_bool(0, 2) && // message$_ -> info:CommonMsgInfo -> src:MsgAddressExt -> addr_none$00 + cb.append_cellslice_bool(msg_info.dest) && // message$_ -> info:CommonMsgInfo -> dest:MsgAddressInt + cb.store_long_bool(0, 4) && // message$_ -> info:CommonMsgInfo -> import_fee:Grams -> 0 + cb.store_long_bool(0, 1) && // message$_ -> init:(Maybe (Either StateInit ^StateInit)) -> nothing$0 + cb.store_long_bool(1, 1) && // message$_ -> body:(Either X ^X) -> right$1 + cb.store_ref_bool(body); if (!status) { return td::Status::Error("Failed to build normalized message"); @@ -3474,10 +3478,10 @@ td::Status TonlibClient::do_request(const tonlib_api::raw_sendMessageReturnHash& auto hash = body->get_hash().as_slice().str(); TRY_RESULT(hash_norm, get_ext_in_msg_hash_norm(body)); - make_request(int_api::SendMessage{std::move(body)}, - promise.wrap([hash = std::move(hash), hash_norm = std::move(hash_norm)](auto res) { - return tonlib_api::make_object(std::move(hash), hash_norm.as_slice().str()); - })); + make_request(int_api::SendMessage{std::move(body)}, promise.wrap([hash = std::move(hash), + hash_norm = std::move(hash_norm)](auto res) { + return tonlib_api::make_object(std::move(hash), hash_norm.as_slice().str()); + })); return td::Status::OK(); } @@ -3588,7 +3592,7 @@ td::Status TonlibClient::do_request(tonlib_api::raw_getTransactions& request, } td::Status TonlibClient::do_request(tonlib_api::raw_getTransactionsV2& request, - td::Promise>&& promise) { + td::Promise>&& promise) { if (!request.account_address_) { return TonlibError::EmptyField("account_address"); } @@ -3624,9 +3628,10 @@ td::Status TonlibClient::do_request(tonlib_api::raw_getTransactionsV2& request, auto actor_id = actor_id_++; actors_[actor_id] = td::actor::create_actor( "GetTransactionHistory", client_.get_client(), account_address, lt, hash, count, actor_shared(this, actor_id), - promise.wrap([private_key = std::move(private_key), try_decode_messages = request.try_decode_messages_](auto&& x) mutable { - return ToRawTransactions(std::move(private_key), try_decode_messages).to_raw_transactions(std::move(x)); - })); + promise.wrap( + [private_key = std::move(private_key), try_decode_messages = request.try_decode_messages_](auto&& x) mutable { + return ToRawTransactions(std::move(private_key), try_decode_messages).to_raw_transactions(std::move(x)); + })); return td::Status::OK(); } @@ -3848,26 +3853,25 @@ class GenericCreateSendGrams : public TonlibQueryActor { td::Result to_dns_action(tonlib_api::dns_Action& action) { using R = td::Result; - return downcast_call2(action, - td::overloaded( - [&](tonlib_api::dns_actionDeleteAll& del_all) -> R { - return ton::ManualDns::Action{"", td::Bits256::zero(), {}}; - }, - [&](tonlib_api::dns_actionDelete& del) -> R { - return ton::ManualDns::Action{del.name_, del.category_, {}}; - }, - [&](tonlib_api::dns_actionSet& set) -> R { - if (!set.entry_) { - return TonlibError::EmptyField("entry"); - } - if (!set.entry_->entry_) { - return TonlibError::EmptyField("entry.entry"); - } - TRY_RESULT(entry_data, to_dns_entry_data(*set.entry_->entry_)); - TRY_RESULT(data_cell, entry_data.as_cell()); - return ton::ManualDns::Action{set.entry_->name_, set.entry_->category_, - std::move(data_cell)}; - })); + return downcast_call2(action, td::overloaded( + [&](tonlib_api::dns_actionDeleteAll& del_all) -> R { + return ton::ManualDns::Action{"", td::Bits256::zero(), {}}; + }, + [&](tonlib_api::dns_actionDelete& del) -> R { + return ton::ManualDns::Action{del.name_, del.category_, {}}; + }, + [&](tonlib_api::dns_actionSet& set) -> R { + if (!set.entry_) { + return TonlibError::EmptyField("entry"); + } + if (!set.entry_->entry_) { + return TonlibError::EmptyField("entry.entry"); + } + TRY_RESULT(entry_data, to_dns_entry_data(*set.entry_->entry_)); + TRY_RESULT(data_cell, entry_data.as_cell()); + return ton::ManualDns::Action{set.entry_->name_, set.entry_->category_, + std::move(data_cell)}; + })); } td::Status parse_action(tonlib_api::Action& action) { @@ -4629,7 +4633,8 @@ bool is_list(vm::StackEntry entry) { entry = entry.as_tuple()->at(1); } }; -auto to_tonlib_api(const vm::StackEntry& entry, int& limit) -> td::Result> { +auto to_tonlib_api(const vm::StackEntry& entry, int& limit) + -> td::Result> { if (limit <= 0) { return td::Status::Error(PSLICE() << "TVM stack size exceeds limit"); } @@ -4671,10 +4676,11 @@ auto to_tonlib_api(const vm::StackEntry& entry, int& limit) -> td::Result& stack) -> td::Result>> { +auto to_tonlib_api(const td::Ref& stack) + -> td::Result>> { int stack_limit = 8000; std::vector> tl_stack; - for (auto& entry: stack->as_span()) { + for (auto& entry : stack->as_span()) { TRY_RESULT(tl_entry, to_tonlib_api(entry, --stack_limit)); tl_stack.push_back(std::move(tl_entry)); } @@ -4723,8 +4729,8 @@ td::Result from_tonlib_api(tonlib_api::tvm_StackEntry& entry) { })); } -void deep_library_search(std::set& set, std::set& visited, - vm::Dictionary& libs, td::Ref cell, int depth, size_t max_libs = 16) { +void deep_library_search(std::set& set, std::set& visited, vm::Dictionary& libs, + td::Ref cell, int depth, size_t max_libs = 16) { if (depth <= 0 || set.size() >= max_libs || visited.size() >= 256) { return; } @@ -4750,7 +4756,7 @@ void deep_library_search(std::set& set, std::set& v } return; } - for (unsigned int i=0; iget_refs_cnt(); i++) { + for (unsigned int i = 0; i < loaded_cell.data_cell->get_refs_cnt(); i++) { deep_library_search(set, visited, libs, loaded_cell.data_cell->get_ref(i), depth - 1, max_libs); } } @@ -4763,7 +4769,8 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_getLibraries& request, if (query_context_.block_id) { get_libraries(query_context_.block_id.value(), request.library_list_, std::move(promise)); } else { - client_.with_last_block([this, promise = std::move(promise), library_list = request.library_list_](td::Result r_last_block) mutable { + client_.with_last_block([this, promise = std::move(promise), + library_list = request.library_list_](td::Result r_last_block) mutable { if (r_last_block.is_error()) { promise.set_error(r_last_block.move_as_error_prefix(TonlibError::Internal("get last block failed "))); } else { @@ -4774,7 +4781,8 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_getLibraries& request, return td::Status::OK(); } -void TonlibClient::get_libraries(ton::BlockIdExt blkid, std::vector library_list, td::Promise>&& promise) { +void TonlibClient::get_libraries(ton::BlockIdExt blkid, std::vector library_list, + td::Promise>&& promise) { sort(library_list.begin(), library_list.end()); library_list.erase(unique(library_list.begin(), library_list.end()), library_list.end()); @@ -4798,77 +4806,86 @@ void TonlibClient::get_libraries(ton::BlockIdExt blkid, std::vector } auto missed_lib_ids = not_cached_hashes; - client_.send_query(ton::lite_api::liteServer_getLibrariesWithProof(ton::create_tl_lite_block_id(blkid), 1, std::move(missed_lib_ids)), - promise.wrap([self=this, blkid, result_entries = std::move(result_entries), not_cached_hashes] - (td::Result> r_libraries) mutable - -> td::Result> { - if (r_libraries.is_error()) { - LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); - return r_libraries.move_as_error(); - } + client_.send_query( + ton::lite_api::liteServer_getLibrariesWithProof(ton::create_tl_lite_block_id(blkid), 1, + std::move(missed_lib_ids)), + promise.wrap([self = this, blkid, result_entries = std::move(result_entries), not_cached_hashes]( + td::Result> + r_libraries) mutable -> td::Result> { + if (r_libraries.is_error()) { + LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); + return r_libraries.move_as_error(); + } - auto libraries = r_libraries.move_as_ok(); - auto state = block::check_extract_state_proof(blkid, libraries->state_proof_.as_slice(), - libraries->data_proof_.as_slice()); - if (state.is_error()) { - LOG(WARNING) << "cannot check state proof: " << state.move_as_error().to_string(); - return state.move_as_error(); - } - auto state_root = state.move_as_ok(); + auto libraries = r_libraries.move_as_ok(); + auto state = block::check_extract_state_proof(blkid, libraries->state_proof_.as_slice(), + libraries->data_proof_.as_slice()); + if (state.is_error()) { + LOG(WARNING) << "cannot check state proof: " << state.move_as_error().to_string(); + return state.move_as_error(); + } + auto state_root = state.move_as_ok(); - try { - block::gen::ShardStateUnsplit::Record state_record; - if (!tlb::unpack_cell(state_root, state_record)) { - return td::Status::Error("cannot unpack shardchain state"); - } - auto libraries_dict = vm::Dictionary(state_record.r1.libraries->prefetch_ref(), 256); - - for (auto& hash : not_cached_hashes) { - auto csr = libraries_dict.lookup(hash.bits(), 256); - if (csr.is_null()) { - LOG(WARNING) << "library " << hash.to_hex() << " not found in config"; - if (std::any_of(libraries->result_.begin(), libraries->result_.end(), - [&hash](const auto& lib) { return lib->hash_.bits().equals(hash.cbits(), 256); })) { - return TonlibError::Internal("library is included in response but it's not found in proof"); + try { + block::gen::ShardStateUnsplit::Record state_record; + if (!tlb::unpack_cell(state_root, state_record)) { + return td::Status::Error("cannot unpack shardchain state"); } - continue; - } - block::gen::LibDescr::Record libdescr; - if (!tlb::csr_unpack(csr, libdescr)) { - return TonlibError::Internal("cannot unpack LibDescr record"); - } + auto libraries_dict = vm::Dictionary(state_record.r1.libraries->prefetch_ref(), 256); + + for (auto& hash : not_cached_hashes) { + auto csr = libraries_dict.lookup(hash.bits(), 256); + if (csr.is_null()) { + LOG(WARNING) << "library " << hash.to_hex() << " not found in config"; + if (std::any_of(libraries->result_.begin(), libraries->result_.end(), + [&hash](const auto& lib) { return lib->hash_.bits().equals(hash.cbits(), 256); })) { + return TonlibError::Internal("library is included in response but it's not found in proof"); + } + continue; + } + block::gen::LibDescr::Record libdescr; + if (!tlb::csr_unpack(csr, libdescr)) { + return TonlibError::Internal("cannot unpack LibDescr record"); + } - auto lib_it = std::find_if(libraries->result_.begin(), libraries->result_.end(), - [&hash](const auto& lib) { return lib->hash_.bits().equals(hash.cbits(), 256); }); - if (lib_it == libraries->result_.end()) { - return TonlibError::Internal("library is found in proof but not in response"); - } - auto& lib = *lib_it; - auto contents = vm::std_boc_deserialize(lib->data_); - if (!contents.is_ok() || contents.ok().is_null()) { - return TonlibError::Internal(PSLICE() << "cannot deserialize library cell " << lib->hash_.to_hex()); - } + auto lib_it = std::find_if(libraries->result_.begin(), libraries->result_.end(), [&hash](const auto& lib) { + return lib->hash_.bits().equals(hash.cbits(), 256); + }); + if (lib_it == libraries->result_.end()) { + return TonlibError::Internal("library is found in proof but not in response"); + } + auto& lib = *lib_it; + auto contents = vm::std_boc_deserialize(lib->data_); + if (!contents.is_ok() || contents.ok().is_null()) { + return TonlibError::Internal(PSLICE() << "cannot deserialize library cell " << lib->hash_.to_hex()); + } - if (!contents.ok()->get_hash().bits().equals(hash.cbits(), 256)) { - return TonlibError::Internal(PSLICE() << "library hash mismatch data " << contents.ok()->get_hash().to_hex() << " != requested " << hash.to_hex()); - } + if (!contents.ok()->get_hash().bits().equals(hash.cbits(), 256)) { + return TonlibError::Internal(PSLICE() + << "library hash mismatch data " << contents.ok()->get_hash().to_hex() + << " != requested " << hash.to_hex()); + } - if (contents.ok()->get_hash() != libdescr.lib->get_hash()) { - return TonlibError::Internal(PSLICE() << "library hash mismatch data " << lib->hash_.to_hex() << " != proof " << libdescr.lib->get_hash().to_hex()); - } + if (contents.ok()->get_hash() != libdescr.lib->get_hash()) { + return TonlibError::Internal(PSLICE() << "library hash mismatch data " << lib->hash_.to_hex() + << " != proof " << libdescr.lib->get_hash().to_hex()); + } - result_entries.push_back(tonlib_api::make_object(lib->hash_, lib->data_.as_slice().str())); - self->libraries.set_ref(lib->hash_, contents.move_as_ok()); - LOG(DEBUG) << "registered library " << lib->hash_.to_hex(); - } - self->store_libs_to_disk(); - return tonlib_api::make_object(std::move(result_entries)); - } catch (vm::VmError& err) { - return TonlibError::Internal(PSLICE() << "error while checking getLibrariesWithProof proof: " << err.get_msg()); - } catch (vm::VmVirtError& err) { - return TonlibError::Internal(PSLICE() << "virtualization error while checking getLibrariesWithProof proof: " << err.get_msg()); - } - })); + result_entries.push_back( + tonlib_api::make_object(lib->hash_, lib->data_.as_slice().str())); + self->libraries.set_ref(lib->hash_, contents.move_as_ok()); + LOG(DEBUG) << "registered library " << lib->hash_.to_hex(); + } + self->store_libs_to_disk(); + return tonlib_api::make_object(std::move(result_entries)); + } catch (vm::VmError& err) { + return TonlibError::Internal(PSLICE() + << "error while checking getLibrariesWithProof proof: " << err.get_msg()); + } catch (vm::VmVirtError& err) { + return TonlibError::Internal(PSLICE() << "virtualization error while checking getLibrariesWithProof proof: " + << err.get_msg()); + } + })); } td::Status TonlibClient::do_request(const tonlib_api::smc_getLibrariesExt& request, @@ -4961,8 +4978,8 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, args.set_now(it->second->get_sync_time()); args.set_address(it->second->get_address()); - client_.with_last_config([self = this, smc = std::move(smc), args = std::move(args), promise = std::move(promise) - ](td::Result r_state) mutable { + client_.with_last_config([self = this, smc = std::move(smc), args = std::move(args), + promise = std::move(promise)](td::Result r_state) mutable { TRY_RESULT_PROMISE(promise, state, std::move(r_state)); args.set_config(state.config); args.set_prev_blocks_info(state.prev_blocks_info); @@ -4985,8 +5002,7 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, } else { self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); } - } - else { + } else { self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); } }); @@ -5022,7 +5038,6 @@ void TonlibClient::process_new_libraries( void TonlibClient::perform_smc_execution(td::Ref smc, ton::SmartContract::Args args, td::Promise>&& promise) { - args.set_libraries(libraries); auto res = smc->run_get_method(args); @@ -5039,45 +5054,46 @@ void TonlibClient::perform_smc_execution(td::Ref smc, ton::S td::Bits256 hash = res.missing_library.value(); LOG(DEBUG) << "Requesting missing library: " << hash.to_hex(); std::vector req = {hash}; - client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(req)), - [self = this, res = std::move(res), res_stack = std::move(res_stack), hash, - smc = std::move(smc), args = std::move(args), promise = std::move(promise)] - (td::Result> r_libraries) mutable - { - if (r_libraries.is_error()) { - LOG(WARNING) << "cannot obtain missing library: " << r_libraries.move_as_error().to_string(); - promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); - return; - } - bool found = false, updated = false; - auto libraries = r_libraries.move_as_ok(); - for (auto& lr : libraries->result_) { - auto contents = vm::std_boc_deserialize(lr->data_); - if (contents.is_ok() && contents.ok().not_null()) { - if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { - LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); - continue; + client_.send_query( + ton::lite_api::liteServer_getLibraries(std::move(req)), + [self = this, res = std::move(res), res_stack = std::move(res_stack), hash, smc = std::move(smc), + args = std::move(args), promise = std::move(promise)]( + td::Result> r_libraries) mutable { + if (r_libraries.is_error()) { + LOG(WARNING) << "cannot obtain missing library: " << r_libraries.move_as_error().to_string(); + promise.set_value( + tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); + return; } - found |= (lr->hash_ == hash); - updated = true; - self->libraries.set_ref(lr->hash_, contents.move_as_ok()); - LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); - } else { - LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); - } - } - if (updated) { - self->store_libs_to_disk(); - } - if (!found) { - LOG(WARNING) << "cannot obtain library " << hash.to_hex() << ", it may not exist"; - promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); - } else { - self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); - } - }); - } - else { + bool found = false, updated = false; + auto libraries = r_libraries.move_as_ok(); + for (auto& lr : libraries->result_) { + auto contents = vm::std_boc_deserialize(lr->data_); + if (contents.is_ok() && contents.ok().not_null()) { + if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { + LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); + continue; + } + found |= (lr->hash_ == hash); + updated = true; + self->libraries.set_ref(lr->hash_, contents.move_as_ok()); + LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); + } else { + LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); + } + } + if (updated) { + self->store_libs_to_disk(); + } + if (!found) { + LOG(WARNING) << "cannot obtain library " << hash.to_hex() << ", it may not exist"; + promise.set_value( + tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); + } else { + self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); + } + }); + } else { promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); } } @@ -5153,9 +5169,8 @@ void TonlibClient::do_dns_request(std::string name, td::Bits256 category, td::in td::optional block_id, block::StdAddress address, td::Promise>&& promise) { auto block_id_copy = block_id.copy(); - td::Promise new_promise = - promise.send_closure(actor_id(this), &TonlibClient::finish_dns_resolve, name, category, ttl, std::move(block_id), - address); + td::Promise new_promise = promise.send_closure(actor_id(this), &TonlibClient::finish_dns_resolve, name, + category, ttl, std::move(block_id), address); if (0) { make_request(int_api::GetAccountState{address, std::move(block_id_copy), {}}, @@ -5193,8 +5208,7 @@ td::Status TonlibClient::do_request(const tonlib_api::dns_resolve& request, name += '.'; } TRY_RESULT(account_address, get_account_address(request.account_address_->account_address_)); - do_dns_request(name, request.category_, request.ttl_, std::move(block_id), account_address, - std::move(promise)); + do_dns_request(name, request.category_, request.ttl_, std::move(block_id), account_address, std::move(promise)); return td::Status::OK(); } @@ -5568,11 +5582,9 @@ td::Status TonlibClient::do_request(int_api::GetAccountState request, td::Status TonlibClient::do_request(int_api::GetAccountStateByTransaction request, td::Promise>&& promise) { auto actor_id = actor_id_++; - actors_[actor_id] = td::actor::create_actor( - "RunEmulator", client_.get_client(), request, actor_shared(this, actor_id), - promise.wrap([](auto&& state) { - return std::move(state); - })); + actors_[actor_id] = + td::actor::create_actor("RunEmulator", client_.get_client(), request, actor_shared(this, actor_id), + promise.wrap([](auto&& state) { return std::move(state); })); return td::Status::OK(); } @@ -5645,15 +5657,15 @@ auto to_tonlib_api(const ton::lite_api::tonNode_blockIdExt& blk) -> tonlib_api_p auto to_tonlib_api(const ton::lite_api::tonNode_zeroStateIdExt& zeroStateId) -> tonlib_api_ptr { - return tonlib_api::make_object( //TODO check wether shard indeed 0??? + return tonlib_api::make_object( //TODO check wether shard indeed 0??? zeroStateId.workchain_, 0, 0, zeroStateId.root_hash_.as_slice().str(), zeroStateId.file_hash_.as_slice().str()); } auto to_lite_api(const tonlib_api::ton_blockIdExt& blk) -> td::Result> { TRY_RESULT(root_hash, to_bits256(blk.root_hash_, "blk.root_hash")) TRY_RESULT(file_hash, to_bits256(blk.file_hash_, "blk.file_hash")) - return ton::lite_api::make_object( - blk.workchain_, blk.shard_, blk.seqno_, root_hash, file_hash); + return ton::lite_api::make_object(blk.workchain_, blk.shard_, blk.seqno_, + root_hash, file_hash); } td::Result to_block_id(const tonlib_api::ton_blockIdExt& blk) { @@ -5662,31 +5674,35 @@ td::Result to_block_id(const tonlib_api::ton_blockIdExt& blk) { return ton::BlockIdExt(blk.workchain_, blk.shard_, blk.seqno_, root_hash, file_hash); } -void TonlibClient::get_config_param(int32_t param, int32_t mode, ton::BlockIdExt block, td::Promise>&& promise) { - std::vector params = { param }; - client_.send_query(ton::lite_api::liteServer_getConfigParams(mode, ton::create_tl_lite_block_id(block), std::move(params)), - promise.wrap([param, block](auto r_config) -> td::Result> { - auto state = block::check_extract_state_proof(block, r_config->state_proof_.as_slice(), - r_config->config_proof_.as_slice()); - if (state.is_error()) { - return state.move_as_error_prefix(TonlibError::ValidateConfig()); - } - auto config = block::Config::extract_from_state(std::move(state.move_as_ok()), 0); - if (config.is_error()) { - return config.move_as_error_prefix(TonlibError::ValidateConfig()); - } - tonlib_api::configInfo config_result; - config_result.config_ = tonlib_api::make_object(to_bytes(config.move_as_ok()->get_config_param(param))); - return tonlib_api::make_object(std::move(config_result)); - })); +void TonlibClient::get_config_param(int32_t param, int32_t mode, ton::BlockIdExt block, + td::Promise>&& promise) { + std::vector params = {param}; + client_.send_query( + ton::lite_api::liteServer_getConfigParams(mode, ton::create_tl_lite_block_id(block), std::move(params)), + promise.wrap([param, block](auto r_config) -> td::Result> { + auto state = block::check_extract_state_proof(block, r_config->state_proof_.as_slice(), + r_config->config_proof_.as_slice()); + if (state.is_error()) { + return state.move_as_error_prefix(TonlibError::ValidateConfig()); + } + auto config = block::Config::extract_from_state(std::move(state.move_as_ok()), 0); + if (config.is_error()) { + return config.move_as_error_prefix(TonlibError::ValidateConfig()); + } + tonlib_api::configInfo config_result; + config_result.config_ = + tonlib_api::make_object(to_bytes(config.move_as_ok()->get_config_param(param))); + return tonlib_api::make_object(std::move(config_result)); + })); } td::Status TonlibClient::do_request(const tonlib_api::getConfigParam& request, - td::Promise>&& promise) { + td::Promise>&& promise) { if (query_context_.block_id) { get_config_param(request.param_, request.mode_, query_context_.block_id.value(), std::move(promise)); } else { - client_.with_last_block([this, promise = std::move(promise), param = request.param_, mode = request.mode_](td::Result r_last_block) mutable { + client_.with_last_block([this, promise = std::move(promise), param = request.param_, + mode = request.mode_](td::Result r_last_block) mutable { if (r_last_block.is_error()) { promise.set_error(r_last_block.move_as_error_prefix(TonlibError::Internal("get last block failed "))); } else { @@ -5697,42 +5713,45 @@ td::Status TonlibClient::do_request(const tonlib_api::getConfigParam& request, return td::Status::OK(); } -void TonlibClient::get_config_all(int32_t mode, ton::BlockIdExt block, td::Promise>&& promise) { +void TonlibClient::get_config_all(int32_t mode, ton::BlockIdExt block, + td::Promise>&& promise) { client_.send_query(ton::lite_api::liteServer_getConfigAll(mode, ton::create_tl_lite_block_id(block)), - promise.wrap([block](auto r_config) -> td::Result> { - auto state = block::check_extract_state_proof(block, r_config->state_proof_.as_slice(), - r_config->config_proof_.as_slice()); - if (state.is_error()) { - return state.move_as_error_prefix(TonlibError::ValidateConfig()); - } - auto config = block::Config::extract_from_state(std::move(state.move_as_ok()), 0); - if (config.is_error()) { - return config.move_as_error_prefix(TonlibError::ValidateConfig()); - } - tonlib_api::configInfo config_result; - config_result.config_ = tonlib_api::make_object(to_bytes(config.move_as_ok()->get_root_cell())); - return tonlib_api::make_object(std::move(config_result)); - })); + promise.wrap([block](auto r_config) -> td::Result> { + auto state = block::check_extract_state_proof(block, r_config->state_proof_.as_slice(), + r_config->config_proof_.as_slice()); + if (state.is_error()) { + return state.move_as_error_prefix(TonlibError::ValidateConfig()); + } + auto config = block::Config::extract_from_state(std::move(state.move_as_ok()), 0); + if (config.is_error()) { + return config.move_as_error_prefix(TonlibError::ValidateConfig()); + } + tonlib_api::configInfo config_result; + config_result.config_ = tonlib_api::make_object( + to_bytes(config.move_as_ok()->get_root_cell())); + return tonlib_api::make_object(std::move(config_result)); + })); } td::Status TonlibClient::do_request(const tonlib_api::getConfigAll& request, - td::Promise>&& promise) { + td::Promise>&& promise) { if (query_context_.block_id) { get_config_all(request.mode_, query_context_.block_id.value(), std::move(promise)); } else { - client_.with_last_block([this, promise = std::move(promise), mode = request.mode_](td::Result r_last_block) mutable { - if (r_last_block.is_error()) { - promise.set_error(r_last_block.move_as_error_prefix(TonlibError::Internal("get last block failed "))); - } else { - this->get_config_all(mode, r_last_block.move_as_ok().last_block_id, std::move(promise)); - } - }); + client_.with_last_block( + [this, promise = std::move(promise), mode = request.mode_](td::Result r_last_block) mutable { + if (r_last_block.is_error()) { + promise.set_error(r_last_block.move_as_error_prefix(TonlibError::Internal("get last block failed "))); + } else { + this->get_config_all(mode, r_last_block.move_as_ok().last_block_id, std::move(promise)); + } + }); } return td::Status::OK(); } td::Status TonlibClient::do_request(const tonlib_api::blocks_getMasterchainInfo& masterchain_info, - td::Promise>&& promise) { + td::Promise>&& promise) { client_.send_query(ton::lite_api::liteServer_getMasterchainInfo(), promise.wrap([](lite_api_ptr&& masterchain_info) { return tonlib_api::make_object( @@ -5746,68 +5765,69 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getShards& request, td::Promise>&& promise) { TRY_RESULT(block, to_lite_api(*request.id_)) TRY_RESULT(req_blk_id, to_block_id(*request.id_)); - client_.send_query(ton::lite_api::liteServer_getAllShardsInfo(std::move(block)), - promise.wrap([req_blk_id](lite_api_ptr&& all_shards_info) - -> td::Result> { - auto blk_id = ton::create_block_id(all_shards_info->id_); - if (blk_id != req_blk_id) { - return td::Status::Error("Liteserver responded with wrong block"); - } - td::BufferSlice proof = std::move((*all_shards_info).proof_); - td::BufferSlice data = std::move((*all_shards_info).data_); - if (data.empty() || proof.empty()) { - return td::Status::Error("Shard configuration or proof is empty"); - } - auto proof_cell = vm::std_boc_deserialize(std::move(proof)); - if (proof_cell.is_error()) { - return proof_cell.move_as_error_prefix("Couldn't deserialize shards proof: "); - } - auto data_cell = vm::std_boc_deserialize(std::move(data)); - if (data_cell.is_error()) { - return data_cell.move_as_error_prefix("Couldn't deserialize shards data: "); - } - try { - auto virt_root = vm::MerkleProof::virtualize(proof_cell.move_as_ok()); - if (virt_root.is_null()) { - return td::Status::Error("Virt root is null"); - } - if (ton::RootHash{virt_root->get_hash().bits()} != blk_id.root_hash) { - return td::Status::Error("Block shards merkle proof has incorrect root hash"); - } - - block::gen::Block::Record blk; - block::gen::BlockExtra::Record extra; - block::gen::McBlockExtra::Record mc_extra; - if (!tlb::unpack_cell(virt_root, blk) || !tlb::unpack_cell(blk.extra, extra) || !extra.custom->have_refs() || - !tlb::unpack_cell(extra.custom->prefetch_ref(), mc_extra)) { - return td::Status::Error("cannot unpack block extra of block " + blk_id.to_str()); - } - auto data_csr = vm::load_cell_slice_ref(data_cell.move_as_ok()); - if (data_csr->prefetch_ref()->get_hash() != mc_extra.shard_hashes->prefetch_ref()->get_hash()) { - return td::Status::Error("Block shards data and proof hashes don't match"); - } - - block::ShardConfig sh_conf; - if (!sh_conf.unpack(data_csr)) { - return td::Status::Error("cannot extract shard block list from shard configuration"); - } - auto ids = sh_conf.get_shard_hash_ids(true); - tonlib_api::blocks_shards shards; - for (auto& id : ids) { - auto ref = sh_conf.get_shard_hash(ton::ShardIdFull(id)); - if (ref.not_null()) { - shards.shards_.push_back(to_tonlib_api(ref->top_block_id())); - } - } - return tonlib_api::make_object(std::move(shards)); - } catch (vm::VmError& err) { - return err.as_status("Couldn't verify proof: "); - } catch (vm::VmVirtError& err) { - return err.as_status("Couldn't verify proof: "); - } catch (...) { - return td::Status::Error("Unknown exception raised while verifying proof"); - } - })); + client_.send_query( + ton::lite_api::liteServer_getAllShardsInfo(std::move(block)), + promise.wrap([req_blk_id](lite_api_ptr&& all_shards_info) + -> td::Result> { + auto blk_id = ton::create_block_id(all_shards_info->id_); + if (blk_id != req_blk_id) { + return td::Status::Error("Liteserver responded with wrong block"); + } + td::BufferSlice proof = std::move((*all_shards_info).proof_); + td::BufferSlice data = std::move((*all_shards_info).data_); + if (data.empty() || proof.empty()) { + return td::Status::Error("Shard configuration or proof is empty"); + } + auto proof_cell = vm::std_boc_deserialize(std::move(proof)); + if (proof_cell.is_error()) { + return proof_cell.move_as_error_prefix("Couldn't deserialize shards proof: "); + } + auto data_cell = vm::std_boc_deserialize(std::move(data)); + if (data_cell.is_error()) { + return data_cell.move_as_error_prefix("Couldn't deserialize shards data: "); + } + try { + auto virt_root = vm::MerkleProof::virtualize(proof_cell.move_as_ok()); + if (virt_root.is_null()) { + return td::Status::Error("Virt root is null"); + } + if (ton::RootHash{virt_root->get_hash().bits()} != blk_id.root_hash) { + return td::Status::Error("Block shards merkle proof has incorrect root hash"); + } + + block::gen::Block::Record blk; + block::gen::BlockExtra::Record extra; + block::gen::McBlockExtra::Record mc_extra; + if (!tlb::unpack_cell(virt_root, blk) || !tlb::unpack_cell(blk.extra, extra) || !extra.custom->have_refs() || + !tlb::unpack_cell(extra.custom->prefetch_ref(), mc_extra)) { + return td::Status::Error("cannot unpack block extra of block " + blk_id.to_str()); + } + auto data_csr = vm::load_cell_slice_ref(data_cell.move_as_ok()); + if (data_csr->prefetch_ref()->get_hash() != mc_extra.shard_hashes->prefetch_ref()->get_hash()) { + return td::Status::Error("Block shards data and proof hashes don't match"); + } + + block::ShardConfig sh_conf; + if (!sh_conf.unpack(data_csr)) { + return td::Status::Error("cannot extract shard block list from shard configuration"); + } + auto ids = sh_conf.get_shard_hash_ids(true); + tonlib_api::blocks_shards shards; + for (auto& id : ids) { + auto ref = sh_conf.get_shard_hash(ton::ShardIdFull(id)); + if (ref.not_null()) { + shards.shards_.push_back(to_tonlib_api(ref->top_block_id())); + } + } + return tonlib_api::make_object(std::move(shards)); + } catch (vm::VmError& err) { + return err.as_status("Couldn't verify proof: "); + } catch (vm::VmVirtError& err) { + return err.as_status("Couldn't verify proof: "); + } catch (...) { + return td::Status::Error("Unknown exception raised while verifying proof"); + } + })); return td::Status::OK(); } @@ -5816,29 +5836,34 @@ td::Status check_lookup_block_proof(lite_api_ptr>&& promise) { - auto lite_block = ton::lite_api::make_object((*request.id_).workchain_, (*request.id_).shard_, (*request.id_).seqno_); + td::Promise>&& promise) { + auto lite_block = ton::lite_api::make_object( + (*request.id_).workchain_, (*request.id_).shard_, (*request.id_).seqno_); auto blkid = ton::BlockId(request.id_->workchain_, request.id_->shard_, request.id_->seqno_); - client_.with_last_block( - [self = this, blkid, lite_block = std::move(lite_block), mode = request.mode_, lt = (td::uint64)request.lt_, - utime = (td::uint32)request.utime_, promise = std::move(promise)](td::Result r_last_block) mutable { - if (r_last_block.is_error()) { - promise.set_error(r_last_block.move_as_error_prefix(TonlibError::Internal("get last block failed "))); - return; - } + client_.with_last_block([self = this, blkid, lite_block = std::move(lite_block), mode = request.mode_, + lt = (td::uint64)request.lt_, utime = (td::uint32)request.utime_, + promise = std::move(promise)](td::Result r_last_block) mutable { + if (r_last_block.is_error()) { + promise.set_error(r_last_block.move_as_error_prefix(TonlibError::Internal("get last block failed "))); + return; + } - self->client_.send_query(ton::lite_api::liteServer_lookupBlockWithProof(mode, std::move(lite_block), ton::create_tl_lite_block_id(r_last_block.ok().last_block_id), lt, utime), - promise.wrap([blkid, mode, utime, lt, last_block = r_last_block.ok().last_block_id](lite_api_ptr&& result) - -> td::Result> { + self->client_.send_query( + ton::lite_api::liteServer_lookupBlockWithProof( + mode, std::move(lite_block), ton::create_tl_lite_block_id(r_last_block.ok().last_block_id), lt, utime), + promise.wrap([blkid, mode, utime, lt, last_block = r_last_block.ok().last_block_id]( + lite_api_ptr&& result) + -> td::Result> { TRY_STATUS(check_lookup_block_proof(result, mode, blkid, last_block, lt, utime)); return to_tonlib_api(*result->id_); - }) - ); + })); }); return td::Status::OK(); } -td::Status check_lookup_block_proof(lite_api_ptr& result, int mode, ton::BlockId blkid, ton::BlockIdExt client_mc_blkid, td::uint64 lt, td::uint32 utime) { +td::Status check_lookup_block_proof(lite_api_ptr& result, int mode, + ton::BlockId blkid, ton::BlockIdExt client_mc_blkid, td::uint64 lt, + td::uint32 utime) { try { ton::BlockIdExt cur_id = ton::create_block_id(result->mc_block_id_); if (!cur_id.is_masterchain_ext()) { @@ -5846,7 +5871,7 @@ td::Status check_lookup_block_proof(lite_api_ptrclient_mc_state_proof_.as_slice(), - result->mc_block_proof_.as_slice()); + result->mc_block_proof_.as_slice()); if (state.is_error()) { LOG(WARNING) << "cannot check state proof: " << state.move_as_error().to_string(); return state.move_as_error(); @@ -6001,12 +6026,13 @@ td::Status check_lookup_block_proof(lite_api_ptr tonlib_api_ptr { - return tonlib_api::make_object( - txid.mode_, txid.account_.as_slice().str(), txid.lt_, txid.hash_.as_slice().str()); + return tonlib_api::make_object(txid.mode_, txid.account_.as_slice().str(), txid.lt_, + txid.hash_.as_slice().str()); } -td::Status check_block_transactions_proof(lite_api_ptr& bTxes, int32_t mode, - ton::LogicalTime start_lt, td::Bits256 start_addr, td::Bits256 root_hash, int req_count) { +td::Status check_block_transactions_proof(lite_api_ptr& bTxes, + int32_t mode, ton::LogicalTime start_lt, td::Bits256 start_addr, + td::Bits256 root_hash, int req_count) { if ((mode & ton::lite_api::liteServer_listBlockTransactions::WANT_PROOF_MASK) == 0) { return td::Status::OK(); } @@ -6026,7 +6052,7 @@ td::Status check_block_transactions_proof(lite_api_ptr cur_trans{(long long)trans_lt}; while (count < req_count && count < max_answer_transactions) { auto tvalue = trans_dict.extract_value_ref( - trans_dict.vm::DictionaryFixed::lookup_nearest_key(cur_trans.bits(), 64, !reverse)); + trans_dict.vm::DictionaryFixed::lookup_nearest_key(cur_trans.bits(), 64, !reverse)); if (tvalue.is_null()) { trans_lt = reverse; break; @@ -6075,7 +6101,8 @@ td::Status check_block_transactions_proof(lite_api_ptr(count) != bTxes->ids_.size()) { - return td::Status::Error(PSLICE() << "Txs count mismatch in proof (" << count << ") and response (" << bTxes->ids_.size() << ")"); + return td::Status::Error(PSLICE() << "Txs count mismatch in proof (" << count << ") and response (" + << bTxes->ids_.size() << ")"); } } catch (vm::VmError& err) { return err.as_status("Couldn't verify proof: "); @@ -6088,7 +6115,7 @@ td::Status check_block_transactions_proof(lite_api_ptr>&& promise) { + td::Promise>&& promise) { TRY_RESULT(block, to_lite_api(*request.id_)) auto root_hash = block->root_hash_; bool check_proof = request.mode_ & ton::lite_api::liteServer_listBlockTransactions::WANT_PROOF_MASK; @@ -6111,36 +6138,33 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getTransactions& re after = nullptr; } - client_.send_query(ton::lite_api::liteServer_listBlockTransactions( - std::move(block), - request.mode_, - request.count_, - std::move(after), - reverse_mode, - check_proof), - promise.wrap([root_hash, req_count = request.count_, start_addr, start_lt, mode = request.mode_] - (lite_api_ptr&& bTxes) -> td::Result> { - TRY_STATUS(check_block_transactions_proof(bTxes, mode, start_lt, start_addr, root_hash, req_count)); - - tonlib_api::blocks_transactions r; - r.id_ = to_tonlib_api(*bTxes->id_); - r.req_count_ = bTxes->req_count_; - r.incomplete_ = bTxes->incomplete_; - for (auto& id: bTxes->ids_) { - r.transactions_.push_back(to_tonlib_api(*id)); - } - return tonlib_api::make_object(std::move(r)); - })); + client_.send_query( + ton::lite_api::liteServer_listBlockTransactions(std::move(block), request.mode_, request.count_, std::move(after), + reverse_mode, check_proof), + promise.wrap([root_hash, req_count = request.count_, start_addr, start_lt, + mode = request.mode_](lite_api_ptr&& bTxes) + -> td::Result> { + TRY_STATUS(check_block_transactions_proof(bTxes, mode, start_lt, start_addr, root_hash, req_count)); + + tonlib_api::blocks_transactions r; + r.id_ = to_tonlib_api(*bTxes->id_); + r.req_count_ = bTxes->req_count_; + r.incomplete_ = bTxes->incomplete_; + for (auto& id : bTxes->ids_) { + r.transactions_.push_back(to_tonlib_api(*id)); + } + return tonlib_api::make_object(std::move(r)); + })); return td::Status::OK(); } td::Status TonlibClient::do_request(const tonlib_api::blocks_getTransactionsExt& request, - td::Promise>&& promise) { + td::Promise>&& promise) { TRY_RESULT(block, to_lite_api(*request.id_)) bool check_proof = request.mode_ & ton::lite_api::liteServer_listBlockTransactionsExt::WANT_PROOF_MASK; bool reverse_mode = request.mode_ & ton::lite_api::liteServer_listBlockTransactionsExt::REVERSE_ORDER_MASK; bool has_starting_tx = request.mode_ & ton::lite_api::liteServer_listBlockTransactionsExt::AFTER_MASK; - + td::Bits256 start_addr; ton::LogicalTime start_lt; ton::lite_api::object_ptr after; @@ -6148,7 +6172,7 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getTransactionsExt& if (!request.after_) { return td::Status::Error("Missing field `after`"); } - TRY_RESULT_ASSIGN(start_addr, to_bits256(request.after_->account_, "account")); + TRY_RESULT_ASSIGN(start_addr, to_bits256(request.after_->account_, "account")); start_lt = request.after_->lt_; after = ton::lite_api::make_object(start_addr, start_lt); } else { @@ -6157,55 +6181,52 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getTransactionsExt& after = nullptr; } auto block_id = ton::create_block_id(block); - client_.send_query(ton::lite_api::liteServer_listBlockTransactionsExt( - std::move(block), - request.mode_, - request.count_, - std::move(after), - reverse_mode, - check_proof), - promise.wrap([block_id, check_proof, reverse_mode, start_addr, start_lt, req_count = request.count_] - (lite_api_ptr&& bTxes) -> td::Result> { - if (block_id != create_block_id(bTxes->id_)) { - return td::Status::Error("Liteserver responded with wrong block"); - } - - block::BlockTransactionList list; - list.blkid = block_id; - list.transactions_boc = std::move(bTxes->transactions_); - list.proof_boc = std::move(bTxes->proof_); - list.reverse_mode = reverse_mode; - list.start_lt = start_lt; - list.start_addr = start_addr; - list.req_count = req_count; - auto info = list.validate(check_proof); - if (info.is_error()) { - return info.move_as_error_prefix("Validation of block::BlockTransactionList failed: "); - } - - auto raw_transactions = ToRawTransactions(td::optional()).to_raw_transactions(info.move_as_ok()); - if (raw_transactions.is_error()) { - return raw_transactions.move_as_error_prefix("Error occured while creating tonlib_api::raw_transaction: "); - } - - tonlib_api::blocks_transactionsExt r; - r.id_ = to_tonlib_api(*bTxes->id_); - r.req_count_ = bTxes->req_count_; - r.incomplete_ = bTxes->incomplete_; - r.transactions_ = raw_transactions.move_as_ok(); - return tonlib_api::make_object(std::move(r)); - })); + client_.send_query( + ton::lite_api::liteServer_listBlockTransactionsExt(std::move(block), request.mode_, request.count_, + std::move(after), reverse_mode, check_proof), + promise.wrap([block_id, check_proof, reverse_mode, start_addr, start_lt, + req_count = request.count_](lite_api_ptr&& bTxes) + -> td::Result> { + if (block_id != create_block_id(bTxes->id_)) { + return td::Status::Error("Liteserver responded with wrong block"); + } + + block::BlockTransactionList list; + list.blkid = block_id; + list.transactions_boc = std::move(bTxes->transactions_); + list.proof_boc = std::move(bTxes->proof_); + list.reverse_mode = reverse_mode; + list.start_lt = start_lt; + list.start_addr = start_addr; + list.req_count = req_count; + auto info = list.validate(check_proof); + if (info.is_error()) { + return info.move_as_error_prefix("Validation of block::BlockTransactionList failed: "); + } + + auto raw_transactions = + ToRawTransactions(td::optional()).to_raw_transactions(info.move_as_ok()); + if (raw_transactions.is_error()) { + return raw_transactions.move_as_error_prefix("Error occured while creating tonlib_api::raw_transaction: "); + } + + tonlib_api::blocks_transactionsExt r; + r.id_ = to_tonlib_api(*bTxes->id_); + r.req_count_ = bTxes->req_count_; + r.incomplete_ = bTxes->incomplete_; + r.transactions_ = raw_transactions.move_as_ok(); + return tonlib_api::make_object(std::move(r)); + })); return td::Status::OK(); } td::Status TonlibClient::do_request(const tonlib_api::blocks_getBlockHeader& request, - td::Promise>&& promise) { + td::Promise>&& promise) { TRY_RESULT(lite_block, to_lite_api(*request.id_)) TRY_RESULT(req_blk_id, to_block_id(*request.id_)); - client_.send_query(ton::lite_api::liteServer_getBlockHeader( - std::move(lite_block), - 0xffff), - promise.wrap([req_blk_id](lite_api_ptr&& hdr) -> td::Result> { + client_.send_query(ton::lite_api::liteServer_getBlockHeader(std::move(lite_block), 0xffff), + promise.wrap([req_blk_id](lite_api_ptr&& hdr) + -> td::Result> { auto blk_id = ton::create_block_id(hdr->id_); if (blk_id != req_blk_id) { return td::Status::Error("Liteserver responded with wrong block"); @@ -6341,20 +6362,23 @@ void TonlibClient::load_libs_from_disk() { if (r_dict.is_error()) { return; } - libraries = vm::Dictionary(vm::load_cell_slice(vm::CellBuilder().append_cellslice(vm::load_cell_slice( - r_dict.move_as_ok())).finalize()), 256); + libraries = vm::Dictionary( + vm::load_cell_slice(vm::CellBuilder().append_cellslice(vm::load_cell_slice(r_dict.move_as_ok())).finalize()), + 256); LOG(DEBUG) << "loaded libraries from disk cache"; } void TonlibClient::store_libs_to_disk() { // NB: Dictionary.get_root_cell does not compute_root, and it is protected - kv_->set("tonlib.libcache", vm::std_boc_serialize(vm::CellBuilder().append_cellslice(libraries.get_root()) - .finalize()).move_as_ok().as_slice()); + kv_->set("tonlib.libcache", vm::std_boc_serialize(vm::CellBuilder().append_cellslice(libraries.get_root()).finalize()) + .move_as_ok() + .as_slice()); LOG(DEBUG) << "stored libraries to disk cache"; } -td::Status TonlibClient::do_request(const int_api::ScanAndLoadGlobalLibs& request, td::Promise promise) { +td::Status TonlibClient::do_request(const int_api::ScanAndLoadGlobalLibs& request, + td::Promise promise) { if (request.root.is_null()) { promise.set_value(vm::Dictionary{256}); return td::Status::OK(); diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index ed19be65e7..3403fae973 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -18,8 +18,13 @@ */ #pragma once -#include "TonlibCallback.h" +#include +#include "lite-client/ext-client.h" +#include "smc-envelope/ManualDns.h" +#include "td/actor/actor.h" +#include "td/utils/CancellationToken.h" +#include "td/utils/optional.h" #include "tonlib/Config.h" #include "tonlib/ExtClient.h" #include "tonlib/ExtClientOutbound.h" @@ -27,15 +32,7 @@ #include "tonlib/KeyValue.h" #include "tonlib/LastBlockStorage.h" -#include "td/actor/actor.h" - -#include "td/utils/CancellationToken.h" -#include "td/utils/optional.h" - -#include "smc-envelope/ManualDns.h" -#include "lite-client/ext-client.h" - -#include +#include "TonlibCallback.h" namespace tonlib { namespace int_api { @@ -253,7 +250,7 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(const tonlib_api::getShardAccountCell& request, td::Promise>&& promise); td::Status do_request(const tonlib_api::getShardAccountCellByTransaction& request, - td::Promise>&& promise); + td::Promise>&& promise); td::Status do_request(tonlib_api::guessAccountRevision& request, td::Promise>&& promise); td::Status do_request(tonlib_api::guessAccount& request, @@ -318,7 +315,8 @@ class TonlibClient : public td::actor::Actor { td::Result> get_smc_info(td::int64 id); void finish_load_smc(td::unique_ptr query, td::Promise>&& promise); td::Status do_request(const tonlib_api::smc_load& request, td::Promise>&& promise); - td::Status do_request(const tonlib_api::smc_loadByTransaction& request, td::Promise>&& promise); + td::Status do_request(const tonlib_api::smc_loadByTransaction& request, + td::Promise>&& promise); td::Status do_request(const tonlib_api::smc_forget& request, td::Promise>&& promise); td::Status do_request(const tonlib_api::smc_getCode& request, td::Promise>&& promise); @@ -334,7 +332,8 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(const tonlib_api::smc_getLibraries& request, td::Promise>&& promise); - void get_libraries(ton::BlockIdExt blkid, std::vector library_list_, td::Promise>&& promise); + void get_libraries(ton::BlockIdExt blkid, std::vector library_list_, + td::Promise>&& promise); td::Status do_request(const tonlib_api::smc_getLibrariesExt& request, td::Promise>&& promise); @@ -397,12 +396,11 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(const tonlib_api::blocks_getOutMsgQueueSizes& request, td::Promise>&& promise); - void get_config_param(int32_t param, int32_t mode, ton::BlockIdExt block, + void get_config_param(int32_t param, int32_t mode, ton::BlockIdExt block, td::Promise>&& promise); td::Status do_request(const tonlib_api::getConfigParam& request, td::Promise>&& promise); - void get_config_all(int32_t mode, ton::BlockIdExt block, - td::Promise>&& promise); + void get_config_all(int32_t mode, ton::BlockIdExt block, td::Promise>&& promise); td::Status do_request(const tonlib_api::getConfigAll& request, td::Promise>&& promise); diff --git a/tonlib/tonlib/TonlibClientWrapper.h b/tonlib/tonlib/TonlibClientWrapper.h index 041b25de41..2c0af82dc8 100644 --- a/tonlib/tonlib/TonlibClientWrapper.h +++ b/tonlib/tonlib/TonlibClientWrapper.h @@ -24,8 +24,8 @@ from all source files in the program, then also delete it here. */ #pragma once -#include "td/actor/actor.h" #include "auto/tl/tonlib_api.hpp" +#include "td/actor/actor.h" #include "tonlib/tonlib/TonlibClient.h" namespace tonlib { diff --git a/tonlib/tonlib/TonlibError.h b/tonlib/tonlib/TonlibError.h index c151849a47..495c9e05a6 100644 --- a/tonlib/tonlib/TonlibError.h +++ b/tonlib/tonlib/TonlibError.h @@ -19,8 +19,8 @@ #pragma once -#include "td/utils/Status.h" #include "common/errorcode.h" +#include "td/utils/Status.h" // NEED_MNEMONIC_PASSWORD // KEY_UNKNOWN // KEY_DECRYPT diff --git a/tonlib/tonlib/keys/DecryptedKey.cpp b/tonlib/tonlib/keys/DecryptedKey.cpp index 105a38dae4..029da8e7f3 100644 --- a/tonlib/tonlib/keys/DecryptedKey.cpp +++ b/tonlib/tonlib/keys/DecryptedKey.cpp @@ -16,13 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "DecryptedKey.h" - +#include "td/utils/Random.h" +#include "td/utils/crypto.h" #include "tonlib/keys/EncryptedKey.h" #include "tonlib/keys/SimpleEncryption.h" -#include "td/utils/Random.h" -#include "td/utils/crypto.h" +#include "DecryptedKey.h" namespace tonlib { DecryptedKey::DecryptedKey(const Mnemonic &mnemonic) diff --git a/tonlib/tonlib/keys/DecryptedKey.h b/tonlib/tonlib/keys/DecryptedKey.h index 101fb6b3c3..47aada6697 100644 --- a/tonlib/tonlib/keys/DecryptedKey.h +++ b/tonlib/tonlib/keys/DecryptedKey.h @@ -17,12 +17,11 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include #include +#include -#include "td/utils/tl_helpers.h" #include "td/utils/SharedSlice.h" - +#include "td/utils/tl_helpers.h" #include "tonlib/keys/Mnemonic.h" namespace tonlib { diff --git a/tonlib/tonlib/keys/EncryptedKey.cpp b/tonlib/tonlib/keys/EncryptedKey.cpp index e43840766b..ce6821f333 100644 --- a/tonlib/tonlib/keys/EncryptedKey.cpp +++ b/tonlib/tonlib/keys/EncryptedKey.cpp @@ -16,12 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "EncryptedKey.h" - +#include "td/utils/crypto.h" #include "tonlib/keys/DecryptedKey.h" #include "tonlib/keys/SimpleEncryption.h" -#include "td/utils/crypto.h" +#include "EncryptedKey.h" namespace tonlib { td::Result EncryptedKey::decrypt(td::Slice local_password, bool check_public_key, bool old) const { diff --git a/tonlib/tonlib/keys/EncryptedKey.h b/tonlib/tonlib/keys/EncryptedKey.h index 1148b92736..45e5b826fa 100644 --- a/tonlib/tonlib/keys/EncryptedKey.h +++ b/tonlib/tonlib/keys/EncryptedKey.h @@ -18,11 +18,10 @@ */ #pragma once -#include "td/utils/Status.h" +#include #include "crypto/Ed25519.h" - -#include +#include "td/utils/Status.h" namespace tonlib { struct DecryptedKey; diff --git a/tonlib/tonlib/keys/Mnemonic.cpp b/tonlib/tonlib/keys/Mnemonic.cpp index cc026d6cc2..462bd1f3de 100644 --- a/tonlib/tonlib/keys/Mnemonic.cpp +++ b/tonlib/tonlib/keys/Mnemonic.cpp @@ -16,23 +16,20 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "Mnemonic.h" - -#include "tonlib/keys/bip39.h" - +#include #include -#include "td/utils/crypto.h" -#include "td/utils/format.h" +#include "crypto/Ed25519.h" #include "td/utils/Random.h" #include "td/utils/Span.h" +#include "td/utils/Timer.h" +#include "td/utils/crypto.h" +#include "td/utils/format.h" #include "td/utils/misc.h" #include "td/utils/optional.h" -#include "td/utils/Timer.h" - -#include "crypto/Ed25519.h" +#include "tonlib/keys/bip39.h" -#include +#include "Mnemonic.h" namespace tonlib { td::Result Mnemonic::create(td::SecureString words, td::SecureString password) { diff --git a/tonlib/tonlib/keys/Mnemonic.h b/tonlib/tonlib/keys/Mnemonic.h index 99805b89d6..3e7ddf7f25 100644 --- a/tonlib/tonlib/keys/Mnemonic.h +++ b/tonlib/tonlib/keys/Mnemonic.h @@ -19,11 +19,10 @@ #pragma once #include "crypto/Ed25519.h" - -#include "td/utils/int_types.h" #include "td/utils/Span.h" #include "td/utils/Status.h" #include "td/utils/UInt.h" +#include "td/utils/int_types.h" namespace tonlib { class Mnemonic { diff --git a/tonlib/tonlib/keys/SimpleEncryption.cpp b/tonlib/tonlib/keys/SimpleEncryption.cpp index 8ffe9d4ac8..a914be3d8a 100644 --- a/tonlib/tonlib/keys/SimpleEncryption.cpp +++ b/tonlib/tonlib/keys/SimpleEncryption.cpp @@ -16,11 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "SimpleEncryption.h" - -#include "td/utils/misc.h" #include "td/utils/Random.h" #include "td/utils/SharedSlice.h" +#include "td/utils/misc.h" + +#include "SimpleEncryption.h" namespace tonlib { td::AesCbcState SimpleEncryption::calc_aes_cbc_state_hash(td::Slice hash) { diff --git a/tonlib/tonlib/keys/SimpleEncryption.h b/tonlib/tonlib/keys/SimpleEncryption.h index bb613d7fed..62cf1d43d2 100644 --- a/tonlib/tonlib/keys/SimpleEncryption.h +++ b/tonlib/tonlib/keys/SimpleEncryption.h @@ -18,10 +18,10 @@ */ #pragma once -#include "td/utils/crypto.h" -#include "td/utils/Slice.h" -#include "td/utils/SharedSlice.h" #include "crypto/Ed25519.h" +#include "td/utils/SharedSlice.h" +#include "td/utils/Slice.h" +#include "td/utils/crypto.h" namespace tonlib { class SimpleEncryption { diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 3a7d72e523..6c9961f0e9 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -25,40 +25,35 @@ Copyright 2019-2020 Telegram Systems LLP */ +#include +#include +#include + #include "auto/tl/tonlib_api.h" +#include "auto/tl/tonlib_api.hpp" #include "block/block.h" #include "common/bigint.hpp" #include "common/refint.h" +#include "lite-client/ext-client.h" +#include "smc-envelope/ManualDns.h" +#include "smc-envelope/PaymentChannel.h" #include "td/actor/actor.h" - #include "td/actor/common.h" -#include "td/utils/Time.h" -#include "td/utils/filesystem.h" #include "td/utils/OptionParser.h" -#include "td/utils/overloaded.h" #include "td/utils/Parser.h" -#include "td/utils/port/signals.h" -#include "td/utils/port/path.h" #include "td/utils/Random.h" +#include "td/utils/Time.h" #include "td/utils/as.h" - +#include "td/utils/filesystem.h" +#include "td/utils/overloaded.h" +#include "td/utils/port/path.h" +#include "td/utils/port/signals.h" #include "terminal/terminal.h" - -#include "tonlib/TonlibClient.h" #include "tonlib/TonlibCallback.h" - -#include "smc-envelope/ManualDns.h" -#include "smc-envelope/PaymentChannel.h" - -#include "auto/tl/tonlib_api.hpp" - +#include "tonlib/TonlibClient.h" #include "vm/boc.h" #include "vm/cells/CellBuilder.h" -#include "lite-client/ext-client.h" -#include -#include -#include #include "git.h" using tonlib_api::make_object; @@ -230,14 +225,14 @@ class TonlibCli : public td::actor::Actor { td::actor::ActorShared<> parent_; }; ref_cnt_++; - raw_client_ = liteclient::ExtClient::create(config.lite_servers, - td::make_unique(td::actor::actor_shared())); + raw_client_ = + liteclient::ExtClient::create(config.lite_servers, td::make_unique(td::actor::actor_shared())); } auto config = !options_.config.empty() - ? make_object(options_.config, options_.name, - options_.use_callbacks_for_network, options_.ignore_cache) - : nullptr; + ? make_object(options_.config, options_.name, + options_.use_callbacks_for_network, options_.ignore_cache) + : nullptr; tonlib_api::object_ptr ks_type; if (options_.in_memory) { @@ -378,8 +373,10 @@ class TonlibCli : public td::actor::Actor { td::TerminalIO::out() << "runmethod ...\tRuns GET method of account " " with specified parameters\n"; td::TerminalIO::out() << "getstate \tget state of wallet with requested key\n"; - td::TerminalIO::out() << "getstatebytransaction \tget state of wallet with requested key after transaction with local time and hash (base64url)\n"; - td::TerminalIO::out() << "getconfig \tshow specified configuration parameter from the latest masterchain state\n"; + td::TerminalIO::out() << "getstatebytransaction \tget state of wallet with requested key " + "after transaction with local time and hash (base64url)\n"; + td::TerminalIO::out() + << "getconfig \tshow specified configuration parameter from the latest masterchain state\n"; td::TerminalIO::out() << "guessrevision \tsearch of existing accounts corresponding to the given key\n"; td::TerminalIO::out() << "guessaccount \tsearch of existing accounts corresponding to the given key\n"; td::TerminalIO::out() << "getaddress \tget address of wallet with requested key\n"; @@ -1028,8 +1025,8 @@ class TonlibCli : public td::actor::Actor { TRY_RESULT_PROMISE(promise, data, tonlib::to_tonlib_api(action.data.value())); sb << action.data.value(); TRY_STATUS_PROMISE(promise, std::move(error)); - td::TerminalIO::out() << "Set dns entry: " << action.name << ":" << action.category << " " - << sb.as_cslice() << "\n"; + td::TerminalIO::out() << "Set dns entry: " << action.name << ":" << action.category << " " << sb.as_cslice() + << "\n"; actions.push_back(make_object( make_object(action.name, action.category, std::move(data)))); } @@ -2233,7 +2230,8 @@ int main(int argc, char* argv[]) { return (verbosity >= 0 && verbosity <= 20) ? td::Status::OK() : td::Status::Error("verbosity must be 0..20"); }); p.add_option('V', "version", "show tonlib-cli build information", [&]() { - std::cout << "tonlib-cli build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "tonlib-cli build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_checked_option('C', "config-force", "set lite server config, drop config related blockchain cache", diff --git a/tonlib/tonlib/tonlib_client_json.cpp b/tonlib/tonlib/tonlib_client_json.cpp index b6e2569988..c9fc1516fe 100644 --- a/tonlib/tonlib/tonlib_client_json.cpp +++ b/tonlib/tonlib/tonlib_client_json.cpp @@ -16,13 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tonlib/tonlib_client_json.h" - -#include "tonlib/ClientJson.h" - #include "td/utils/Slice.h" - +#include "tonlib/ClientJson.h" #include "tonlib/Logging.h" +#include "tonlib/tonlib_client_json.h" extern "C" int tonlib_client_json_square(int x, const char *str) { return x * x; diff --git a/tonlib/tonlib/utils.cpp b/tonlib/tonlib/utils.cpp index 1ca5022773..fabfcc1bf1 100644 --- a/tonlib/tonlib/utils.cpp +++ b/tonlib/tonlib/utils.cpp @@ -16,8 +16,8 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "tonlib/utils.h" #include "td/utils/misc.h" +#include "tonlib/utils.h" #include "vm/cellslice.h" namespace tonlib { int VERBOSITY_NAME(tonlib_query) = VERBOSITY_NAME(DEBUG); diff --git a/tonlib/tonlib/utils.h b/tonlib/tonlib/utils.h index a6bb184f9b..68fdf36a1d 100644 --- a/tonlib/tonlib/utils.h +++ b/tonlib/tonlib/utils.h @@ -17,10 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "vm/cells.h" -#include "ton/ton-types.h" -#include "block/block.h" #include "block/block-parse.h" +#include "block/block.h" +#include "ton/ton-types.h" +#include "vm/cells.h" namespace tonlib { extern int VERBOSITY_NAME(tonlib_query); diff --git a/utils/generate-random-id.cpp b/utils/generate-random-id.cpp index a487ac1727..cc11b9a7dc 100644 --- a/utils/generate-random-id.cpp +++ b/utils/generate-random-id.cpp @@ -25,18 +25,20 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include #include +#include #include + #include "adnl/utils.hpp" #include "auto/tl/ton_api.h" #include "auto/tl/ton_api_json.h" -#include "tl/tl_json.h" +#include "dht/dht-node.hpp" +#include "keys/encryptor.h" #include "td/utils/OptionParser.h" #include "td/utils/filesystem.h" -#include "keys/encryptor.h" +#include "tl/tl_json.h" + #include "git.h" -#include "dht/dht-node.hpp" int main(int argc, char *argv[]) { ton::PrivateKey pk; @@ -59,7 +61,8 @@ int main(int argc, char *argv[]) { std::exit(2); }); p.add_option('V', "version", "shows generate-random-id build information", [&]() { - std::cout << "generate-random-id build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "generate-random-id build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_option('n', "name", "path to save private keys to", [&](td::Slice arg) { name = arg.str(); }); diff --git a/utils/json2tlo.cpp b/utils/json2tlo.cpp index f4c4fdbfbc..3cd3473fbe 100644 --- a/utils/json2tlo.cpp +++ b/utils/json2tlo.cpp @@ -25,20 +25,21 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include +#include +#include #include +#include #include -#include -#include -#include "td/utils/OptionParser.h" -#include "keys/encryptor.h" + +#include "auto/tl/ton_api.h" #include "auto/tl/ton_api_json.h" -#include "td/utils/filesystem.h" -#include "common/io.hpp" #include "common/checksum.h" +#include "common/io.hpp" +#include "keys/encryptor.h" +#include "td/utils/OptionParser.h" +#include "td/utils/filesystem.h" #include "tl/tl_json.h" -#include "auto/tl/ton_api.h" -#include "auto/tl/ton_api_json.h" + #include "git.h" int main(int argc, char *argv[]) { @@ -53,7 +54,8 @@ int main(int argc, char *argv[]) { p.add_option('o', "out", "output", [&](td::Slice key) { out_f = key.str(); }); p.add_option('r', "reverse", "read tlo, print json", [&]() { reverse_ = !reverse_; }); p.add_option('V', "version", "shows json2tlo build information", [&]() { - std::cout << "json2tlo build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "json2tlo build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_option('h', "help", "prints_help", [&]() { @@ -110,4 +112,3 @@ int main(int argc, char *argv[]) { return 0; } - diff --git a/utils/opcode-timing.cpp b/utils/opcode-timing.cpp index 47171eec09..5e7411035c 100644 --- a/utils/opcode-timing.cpp +++ b/utils/opcode-timing.cpp @@ -1,18 +1,18 @@ #include #include -#include "vm/vm.h" -#include "vm/cp0.h" -#include "vm/dict.h" -#include "fift/utils.h" #include "common/bigint.hpp" - -#include "td/utils/base64.h" +#include "fift/utils.h" #include "td/utils/ScopeGuard.h" #include "td/utils/StringBuilder.h" #include "td/utils/Timer.h" -#include "block.h" +#include "td/utils/base64.h" #include "td/utils/filesystem.h" +#include "vm/cp0.h" +#include "vm/dict.h" +#include "vm/vm.h" + +#include "block.h" #include "mc-config.h" td::Ref c7; diff --git a/utils/pack-viewer.cpp b/utils/pack-viewer.cpp index 4e903b0e65..1280c4303b 100644 --- a/utils/pack-viewer.cpp +++ b/utils/pack-viewer.cpp @@ -25,15 +25,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include +#include +#include #include +#include #include -#include -#include -#include "td/utils/OptionParser.h" -#include "validator/db/package.hpp" +#include "td/utils/OptionParser.h" #include "validator/db/fileref.hpp" +#include "validator/db/package.hpp" void run(std::string filename) { auto R = ton::Package::open(filename, true, false); @@ -58,4 +58,3 @@ int main(int argc, char **argv) { run(argv[1]); return 0; } - diff --git a/utils/prepare-ls-slice-config.cpp b/utils/prepare-ls-slice-config.cpp index c63e4ec7c8..d05a3a8f77 100644 --- a/utils/prepare-ls-slice-config.cpp +++ b/utils/prepare-ls-slice-config.cpp @@ -14,34 +14,33 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . */ -#include "td/utils/filesystem.h" -#include "td/actor/actor.h" +#include +#include + +#include "adnl/adnl.h" +#include "auto/tl/lite_api.h" +#include "auto/tl/ton_api.h" +#include "auto/tl/ton_api_json.h" +#include "block/block-auto.h" +#include "block/mc-config.h" +#include "lite-client/ext-client.h" #include "td/actor/MultiPromise.h" +#include "td/actor/actor.h" +#include "td/utils/FileLog.h" #include "td/utils/OptionParser.h" +#include "td/utils/Random.h" +#include "td/utils/filesystem.h" +#include "td/utils/overloaded.h" +#include "td/utils/port/IPAddress.h" #include "td/utils/port/path.h" #include "td/utils/port/signals.h" -#include "td/utils/port/IPAddress.h" -#include "td/utils/Random.h" -#include "td/utils/FileLog.h" -#include "git.h" -#include "auto/tl/ton_api.h" -#include "auto/tl/lite_api.h" +#include "td/utils/tl_storers.h" #include "tl-utils/lite-utils.hpp" -#include "auto/tl/ton_api_json.h" -#include "adnl/adnl.h" -#include "lite-client/ext-client.h" #include "ton/lite-tl.hpp" - -#include "td/utils/overloaded.h" - -#include -#include "td/utils/tl_storers.h" #include "vm/boc.h" #include "vm/cells/MerkleProof.h" -#include -#include "block/block-auto.h" -#include "block/mc-config.h" +#include "git.h" using namespace ton; diff --git a/utils/proxy-liteserver.cpp b/utils/proxy-liteserver.cpp index e89cb1bbf9..db25ebca1a 100644 --- a/utils/proxy-liteserver.cpp +++ b/utils/proxy-liteserver.cpp @@ -23,31 +23,32 @@ exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. */ -#include "td/utils/filesystem.h" -#include "td/actor/actor.h" +#include "adnl/adnl.h" +#include "auto/tl/lite_api.h" +#include "auto/tl/ton_api.h" +#include "auto/tl/ton_api_json.h" +#include "lite-client/ext-client.h" #include "td/actor/MultiPromise.h" +#include "td/actor/actor.h" +#include "td/utils/FileLog.h" #include "td/utils/OptionParser.h" +#include "td/utils/Random.h" +#include "td/utils/filesystem.h" +#include "td/utils/port/IPAddress.h" #include "td/utils/port/path.h" #include "td/utils/port/signals.h" -#include "td/utils/port/IPAddress.h" -#include "td/utils/Random.h" -#include "td/utils/FileLog.h" -#include "git.h" -#include "auto/tl/ton_api.h" -#include "auto/tl/lite_api.h" #include "tl-utils/lite-utils.hpp" -#include "auto/tl/ton_api_json.h" -#include "adnl/adnl.h" -#include "lite-client/ext-client.h" + +#include "git.h" #if TD_DARWIN || TD_LINUX #include #endif -#include "td/utils/overloaded.h" - +#include #include #include -#include + +#include "td/utils/overloaded.h" #include "td/utils/tl_storers.h" using namespace ton; diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 1a9ca052e2..59b4efbd8f 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -25,22 +25,23 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "validator-engine-console-query.h" +#include +#include + #include "auto/tl/ton_api.h" -#include "td/utils/StringBuilder.h" -#include "validator-engine-console.h" -#include "terminal/terminal.h" -#include "td/utils/filesystem.h" -#include "overlay/overlays.h" -#include "ton/ton-tl.hpp" -#include "td/utils/JsonBuilder.h" #include "auto/tl/ton_api_json.h" #include "keys/encryptor.h" +#include "overlay/overlays.h" +#include "td/utils/JsonBuilder.h" +#include "td/utils/StringBuilder.h" +#include "td/utils/filesystem.h" #include "td/utils/port/path.h" +#include "terminal/terminal.h" #include "tl/tl_json.h" +#include "ton/ton-tl.hpp" -#include -#include +#include "validator-engine-console-query.h" +#include "validator-engine-console.h" Tokenizer::Tokenizer(td::BufferSlice data) : data_(std::move(data)) { remaining_ = data_.as_slice(); @@ -1043,7 +1044,7 @@ td::Status ImportCertificateQuery::receive(td::BufferSlice data) { } td::Status SignShardOverlayCertificateQuery::run() { - TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(max_size_, tokenizer_.get_token()); @@ -1098,11 +1099,11 @@ td::Status ImportShardOverlayCertificateQuery::receive(td::BufferSlice data) { return td::Status::OK(); } td::Status GetActorStatsQuery::run() { - auto r_file_name = tokenizer_.get_token(); - if (r_file_name.is_ok()) { + auto r_file_name = tokenizer_.get_token(); + if (r_file_name.is_ok()) { file_name_ = r_file_name.move_as_ok(); - } - return td::Status::OK(); + } + return td::Status::OK(); } td::Status GetActorStatsQuery::send() { auto b = ton::create_serialize_tl_object(); @@ -1371,8 +1372,7 @@ td::Status GetCollatorOptionsJsonQuery::run() { } td::Status GetCollatorOptionsJsonQuery::send() { - auto b = - ton::create_serialize_tl_object(); + auto b = ton::create_serialize_tl_object(); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1400,8 +1400,7 @@ td::Status GetAdnlStatsJsonQuery::run() { } td::Status GetAdnlStatsJsonQuery::send() { - auto b = - ton::create_serialize_tl_object(all_); + auto b = ton::create_serialize_tl_object(all_); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1429,8 +1428,7 @@ td::Status GetAdnlStatsQuery::run() { } td::Status GetAdnlStatsQuery::send() { - auto b = - ton::create_serialize_tl_object(all_); + auto b = ton::create_serialize_tl_object(all_); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1450,11 +1448,10 @@ td::Status GetAdnlStatsQuery::receive(td::BufferSlice data) { } sb << "LOCAL ID " << local_id->short_id_ << "\n"; if (!local_id->current_decrypt_.empty()) { - std::sort(local_id->current_decrypt_.begin(), local_id->current_decrypt_.end(), - [](const ton::tl_object_ptr &a, - const ton::tl_object_ptr &b) { - return a->packets_ > b->packets_; - }); + std::sort( + local_id->current_decrypt_.begin(), local_id->current_decrypt_.end(), + [](const ton::tl_object_ptr &a, + const ton::tl_object_ptr &b) { return a->packets_ > b->packets_; }); td::uint64 total = 0; for (auto &x : local_id->current_decrypt_) { total += x->packets_; @@ -1470,11 +1467,10 @@ td::Status GetAdnlStatsQuery::receive(td::BufferSlice data) { if (vec.empty()) { return; } - std::sort(vec.begin(), vec.end(), - [](const ton::tl_object_ptr &a, - const ton::tl_object_ptr &b) { - return a->packets_ > b->packets_; - }); + std::sort( + vec.begin(), vec.end(), + [](const ton::tl_object_ptr &a, + const ton::tl_object_ptr &b) { return a->packets_ > b->packets_; }); td::uint64 total = 0; for (auto &x : vec) { total += x->packets_; @@ -1820,12 +1816,13 @@ td::Status GetCollationManagerStatsQuery::receive(td::BufferSlice data) { "received incorrect answer: "); if (list->local_ids_.empty()) { td::TerminalIO::out() << "No stats\n"; - return td::Status::OK();; + return td::Status::OK(); + ; } for (auto &stats : list->local_ids_) { td::TerminalIO::out() << "VALIDATOR ADNL ID = " << stats->adnl_id_ << "\n"; - std::map collators; - for (auto &collator: stats->collators_) { + std::map collators; + for (auto &collator : stats->collators_) { collators[collator->adnl_id_] = collator.get(); } for (auto &shard : stats->shards_) { diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index c06924d9a1..13aeba5d2a 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -27,17 +27,16 @@ */ #pragma once +#include "keys/keys.hpp" +#include "td/actor/actor.h" +#include "td/utils/SharedSlice.h" #include "td/utils/Status.h" +#include "td/utils/base64.h" #include "td/utils/buffer.h" #include "td/utils/misc.h" -#include "td/utils/SharedSlice.h" #include "td/utils/port/IPAddress.h" -#include "td/actor/actor.h" #include "ton/ton-types.h" -#include "keys/keys.hpp" -#include "td/utils/base64.h" - class ValidatorEngineConsole; class Tokenizer { diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index a747a648f1..67c0d86076 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -25,40 +25,42 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "validator-engine-console.h" #include "adnl/adnl-ext-client.h" -#include "tl-utils/lite-utils.hpp" -#include "auto/tl/ton_api_json.h" #include "auto/tl/lite_api.h" +#include "auto/tl/ton_api_json.h" +#include "block/block-auto.h" +#include "block/block-db.h" +#include "block/block-parse.h" +#include "block/block.h" +#include "block/check-proof.h" +#include "block/mc-config.h" #include "td/utils/OptionParser.h" +#include "td/utils/Random.h" #include "td/utils/Time.h" +#include "td/utils/crypto.h" #include "td/utils/filesystem.h" #include "td/utils/format.h" -#include "td/utils/Random.h" -#include "td/utils/crypto.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/port/StdStreams.h" #include "td/utils/port/signals.h" #include "td/utils/port/stacktrace.h" -#include "td/utils/port/StdStreams.h" -#include "td/utils/port/FileFd.h" #include "terminal/terminal.h" +#include "tl-utils/lite-utils.hpp" #include "ton/lite-tl.hpp" -#include "block/block-db.h" -#include "block/block.h" -#include "block/block-parse.h" -#include "block/block-auto.h" -#include "block/mc-config.h" -#include "block/check-proof.h" +#include "ton/ton-shard.h" #include "vm/boc.h" #include "vm/cellops.h" #include "vm/cells/MerkleProof.h" -#include "ton/ton-shard.h" + +#include "validator-engine-console.h" #if TD_DARWIN || TD_LINUX -#include #include +#include #endif #include #include + #include "git.h" int verbosity; diff --git a/validator-engine-console/validator-engine-console.h b/validator-engine-console/validator-engine-console.h index c802794efe..3cf3587345 100644 --- a/validator-engine-console/validator-engine-console.h +++ b/validator-engine-console/validator-engine-console.h @@ -26,14 +26,15 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include + #include "adnl/adnl-ext-client.h" +#include "terminal/terminal.h" #include "tl-utils/tl-utils.hpp" #include "ton/ton-types.h" -#include "terminal/terminal.h" #include "vm/cells.h" -#include "validator-engine-console-query.h" -#include +#include "validator-engine-console-query.h" class ValidatorEngineConsole : public td::actor::Actor { private: diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index cc7d676634..daa6b844bd 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -25,68 +25,65 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "validator-engine.hpp" +#include +#include #include "adnl/adnl-node-id.hpp" +#include "auto/tl/lite_api.h" #include "auto/tl/ton_api.h" -#include "errorcode.h" +#include "common/errorlog.h" +#include "crypto/fift/utils.h" +#include "crypto/vm/vm.h" +#include "dht/dht.hpp" #include "keys/keys.hpp" -#include "overlay-manager.h" -#include "overlays.h" +#include "memprof/memprof.h" +#include "td/actor/MultiPromise.h" #include "td/actor/PromiseFuture.h" #include "td/actor/actor.h" +#include "td/utils/OptionParser.h" +#include "td/utils/Random.h" #include "td/utils/Status.h" +#include "td/utils/ThreadSafeCounter.h" #include "td/utils/Time.h" +#include "td/utils/TsFileLog.h" #include "td/utils/buffer.h" -#include "tl-utils/tl-utils.hpp" -#include "tl/TlObject.h" -#include "ton/ton-types.h" -#include "ton/ton-tl.hpp" -#include "ton/ton-io.hpp" - -#include "common/errorlog.h" - -#include "crypto/vm/vm.h" -#include "crypto/fift/utils.h" - #include "td/utils/filesystem.h" -#include "td/actor/MultiPromise.h" #include "td/utils/overloaded.h" -#include "td/utils/OptionParser.h" #include "td/utils/port/path.h" +#include "td/utils/port/rlimit.h" #include "td/utils/port/signals.h" #include "td/utils/port/user.h" -#include "td/utils/port/rlimit.h" -#include "td/utils/ThreadSafeCounter.h" -#include "td/utils/TsFileLog.h" -#include "td/utils/Random.h" - -#include "auto/tl/lite_api.h" +#include "tl-utils/tl-utils.hpp" +#include "tl/TlObject.h" #include "tl/tl_json.h" +#include "ton/ton-io.hpp" +#include "ton/ton-tl.hpp" +#include "ton/ton-types.h" -#include "memprof/memprof.h" - -#include "dht/dht.hpp" -#include -#include +#include "errorcode.h" +#include "overlay-manager.h" +#include "overlays.h" +#include "validator-engine.hpp" #if TD_DARWIN || TD_LINUX #include #endif #include -#include +#include #include +#include #include #include -#include -#include "git.h" -#include "block-auto.h" -#include "block-parse.h" -#include "common/delay.h" + #include "block/precompiled-smc/PrecompiledSmartContract.h" +#include "common/delay.h" #include "interfaces/validator-manager.h" #include "tl-utils/lite-utils.hpp" +#include "block-auto.h" +#include "block-parse.h" +#include "git.h" + #if TON_USE_JEMALLOC #include #endif @@ -188,7 +185,7 @@ Config::Config(const ton::ton_api::engine_validator_config &config) { } if (config.extraconfig_->collator_node_whitelist_) { collator_node_whiltelist_enabled = config.extraconfig_->collator_node_whitelist_->enabled_; - for (const auto& id : config.extraconfig_->collator_node_whitelist_->adnl_ids_) { + for (const auto &id : config.extraconfig_->collator_node_whitelist_->adnl_ids_) { collator_node_whitelist.emplace(id); } } @@ -290,7 +287,7 @@ ton::tl_object_ptr Config::tl() const { if (collator_node_whiltelist_enabled || !collator_node_whitelist.empty()) { collator_node_whitelist_obj = ton::create_tl_object(); collator_node_whitelist_obj->enabled_ = collator_node_whiltelist_enabled; - for (const auto& id : collator_node_whitelist) { + for (const auto &id : collator_node_whitelist) { collator_node_whitelist_obj->adnl_ids_.push_back(id.bits256_value()); } } @@ -491,7 +488,7 @@ td::Result Config::config_add_collator(ton::adnl::AdnlNodeIdShort addr, to if (!shard.is_valid_ext()) { return td::Status::Error(PSTRING() << "invalid shard: " << shard.to_str()); } - auto& shards = collators[addr]; + auto &shards = collators[addr]; if (std::find(shards.begin(), shards.end(), shard) != shards.end()) { return false; } @@ -503,7 +500,7 @@ td::Result Config::config_del_collator(ton::adnl::AdnlNodeIdShort addr, to if (!shard.is_valid_ext()) { return td::Status::Error(PSTRING() << "invalid shard: " << shard.to_str()); } - auto& shards = collators[addr]; + auto &shards = collators[addr]; auto it = std::find(shards.begin(), shards.end(), shard); if (it == shards.end()) { return false; @@ -1646,7 +1643,7 @@ td::Status ValidatorEngine::load_global_config() { validator_options_.write().set_fast_state_serializer_enabled(fast_state_serializer_enabled_); validator_options_.write().set_catchain_broadcast_speed_multiplier(broadcast_speed_multiplier_catchain_); - for (auto& id : config_.collator_node_whitelist) { + for (auto &id : config_.collator_node_whitelist) { validator_options_.write().set_collator_node_whitelisted_validator(id, true); } validator_options_.write().set_collator_node_whitelist_enabled(config_.collator_node_whiltelist_enabled); @@ -2243,8 +2240,8 @@ void ValidatorEngine::start_full_node() { .private_broadcast_speed_multiplier_ = broadcast_speed_multiplier_private_, .initial_sync_delay_ = initial_sync_delay_}; full_node_ = ton::validator::fullnode::FullNode::create( - short_id, full_node_id_, validator_options_->zero_block_id().file_hash, - full_node_options, keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(), + short_id, full_node_id_, validator_options_->zero_block_id().file_hash, full_node_options, keyring_.get(), + adnl_.get(), rldp_.get(), rldp2_.get(), default_dht_node_.is_zero() ? td::actor::ActorId{} : dht_nodes_[default_dht_node_].get(), overlay_manager_.get(), validator_manager_.get(), full_node_client_.get(), db_root_, std::move(P)); for (auto &v : config_.validators) { @@ -2294,8 +2291,8 @@ void ValidatorEngine::started_lite_server() { } void ValidatorEngine::start_collator() { - for (auto& [id, shards] : config_.collators) { - for (auto& shard : shards) { + for (auto &[id, shards] : config_.collators) { + for (auto &shard : shards) { td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, id, shard); } } @@ -2504,9 +2501,8 @@ void ValidatorEngine::try_add_full_node_adnl_addr(ton::PublicKeyHash id, td::Pro } if (!full_node_.empty() && id != full_node_id_.pubkey_hash()) { - td::actor::send_closure( - adnl_.get(), &ton::adnl::Adnl::unsubscribe, full_node_id_, - ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_newFastSyncMemberCertificate::ID)); + td::actor::send_closure(adnl_.get(), &ton::adnl::Adnl::unsubscribe, full_node_id_, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_newFastSyncMemberCertificate::ID)); td::actor::send_closure( adnl_.get(), &ton::adnl::Adnl::unsubscribe, full_node_id_, ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_requestFastSyncOverlayMemberCertificate::ID)); @@ -2781,8 +2777,7 @@ void ValidatorEngine::register_fast_sync_certificate_callback() { } void receive_message(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { - auto R = ton::fetch_tl_object( - std::move(data), true); + auto R = ton::fetch_tl_object(std::move(data), true); if (R.is_error()) { return; } @@ -2825,10 +2820,9 @@ void ValidatorEngine::register_fast_sync_certificate_callback() { private: td::actor::ActorId validator_engine_; }; - td::actor::send_closure( - adnl_.get(), &ton::adnl::Adnl::subscribe, full_node_id_, - ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_newFastSyncMemberCertificate::ID), - std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_.get(), &ton::adnl::Adnl::subscribe, full_node_id_, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_newFastSyncMemberCertificate::ID), + std::make_unique(actor_id(this))); td::actor::send_closure( adnl_.get(), &ton::adnl::Adnl::subscribe, full_node_id_, ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_requestFastSyncOverlayMemberCertificate::ID), @@ -2977,11 +2971,11 @@ ton::PublicKeyHash ValidatorEngine::find_local_validator_for_cert_issuing() { if (state_.is_null()) { return ton::PublicKeyHash{}; } - for (auto& val_set : {validator_set_, validator_set_next_, validator_set_prev_}) { + for (auto &val_set : {validator_set_, validator_set_next_, validator_set_prev_}) { if (val_set.is_null()) { continue; } - for (auto &[val_id, _]: config_.validators) { + for (auto &[val_id, _] : config_.validators) { if (val_set->is_validator(ton::NodeIdShort{val_id.bits256_value()})) { return val_id; } @@ -3089,11 +3083,11 @@ static td::Result> parse_collator_optio } else { opts.dispatch_phase_3_max_per_initiator = {}; } - for (const std::string& s : f.whitelist_) { + for (const std::string &s : f.whitelist_) { TRY_RESULT(addr, block::StdAddress::parse(s)); opts.whitelist.emplace(addr.workchain, addr.addr); } - for (const std::string& s : f.prioritylist_) { + for (const std::string &s : f.prioritylist_) { TRY_RESULT(addr, block::StdAddress::parse(s)); opts.prioritylist.emplace(addr.workchain, addr.addr); } @@ -4631,9 +4625,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getAdnlSt }); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addShard &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addShard &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -4664,9 +4657,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addShard }); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delShard &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delShard &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -4796,9 +4788,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getCollat }); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollator &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollator &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -4838,9 +4829,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollat }); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCollator &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCollator &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -5302,15 +5292,14 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); }); return td::Status::OK(); }); - p.add_checked_option('b', "block-ttl", "deprecated", - [&](td::Slice fname) { - auto v = td::to_double(fname); - if (v <= 0) { - return td::Status::Error("block-ttl should be positive"); - } - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_block_ttl, v); }); - return td::Status::OK(); - }); + p.add_checked_option('b', "block-ttl", "deprecated", [&](td::Slice fname) { + auto v = td::to_double(fname); + if (v <= 0) { + return td::Status::Error("block-ttl should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_block_ttl, v); }); + return td::Status::OK(); + }); p.add_checked_option( 'A', "archive-ttl", "ttl for archived blocks (in seconds) default=7*86400. Note: archived blocks are gc'd after state-ttl + " @@ -5323,16 +5312,14 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_archive_ttl, v); }); return td::Status::OK(); }); - p.add_checked_option( - 'K', "key-proof-ttl", "deprecated", - [&](td::Slice fname) { - auto v = td::to_double(fname); - if (v <= 0) { - return td::Status::Error("key-proof-ttl should be positive"); - } - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_key_proof_ttl, v); }); - return td::Status::OK(); - }); + p.add_checked_option('K', "key-proof-ttl", "deprecated", [&](td::Slice fname) { + auto v = td::to_double(fname); + if (v <= 0) { + return td::Status::Error("key-proof-ttl should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_key_proof_ttl, v); }); + return td::Status::OK(); + }); p.add_checked_option('S', "sync-before", "in initial sync download all blocks for last given seconds default=3600", [&](td::Slice fname) { auto v = td::to_double(fname); @@ -5370,9 +5357,8 @@ int main(int argc, char *argv[]) { }); return td::Status::OK(); }); - p.add_option('M', "not-all-shards", "monitor only a necessary set of shards instead of all", [&]() { - acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_not_all_shards); }); - }); + p.add_option('M', "not-all-shards", "monitor only a necessary set of shards instead of all", + [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_not_all_shards); }); }); p.add_checked_option( '\0', "add-shard", "add shard to monitor (same as addshard in validator console), format: 0:8000000000000000", [&](td::Slice arg) -> td::Status { @@ -5388,24 +5374,24 @@ int main(int argc, char *argv[]) { return td::Status::OK(); }); td::uint32 threads = 7; - p.add_checked_option( - 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice arg) { - td::int32 v; - try { - v = std::stoi(arg.str()); - } catch (...) { - return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: not a number"); - } - if (v <= 0) { - return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be > 0"); - } - if (v > 127) { - LOG(WARNING) << "`--threads " << v << "` is too big, effective value will be 127"; - v = 127; - } - threads = v; - return td::Status::OK(); - }); + p.add_checked_option('t', "threads", PSTRING() << "number of threads (default=" << threads << ")", + [&](td::Slice arg) { + td::int32 v; + try { + v = std::stoi(arg.str()); + } catch (...) { + return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: not a number"); + } + if (v <= 0) { + return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be > 0"); + } + if (v > 127) { + LOG(WARNING) << "`--threads " << v << "` is too big, effective value will be 127"; + v = 127; + } + threads = v; + return td::Status::OK(); + }); p.add_checked_option('u', "user", "change user", [&](td::Slice user) { return td::change_user(user.str()); }); p.add_checked_option('\0', "shutdown-at", "stop validator at the given time (unix timestamp)", [&](td::Slice arg) { TRY_RESULT(at, td::to_integer_safe(arg)); @@ -5472,18 +5458,13 @@ int main(int argc, char *argv[]) { p.add_option( '\0', "celldb-in-memory", "store all cells in-memory, much faster but requires a lot of RAM. RocksDb is still used as persistent storage", - [&]() { - acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_in_memory, true); }); - }); - p.add_option( - '\0', "celldb-v2", - "use new version off celldb", - [&]() { - acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_v2, true); }); - }); + [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_in_memory, true); }); }); + p.add_option('\0', "celldb-v2", "use new version off celldb", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_v2, true); }); + }); p.add_option( '\0', "celldb-disable-bloom-filter", - "disable using bloom filter in CellDb. Enabled bloom filter reduces read latency, but increases memory usage", + "disable using bloom filter in CellDb. Enabled bloom filter reduces read latency, but increases memory usage", [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_disable_bloom_filter, true); }); }); @@ -5497,32 +5478,27 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_catchain_max_block_delay, v); }); return td::Status::OK(); }); - p.add_checked_option( - '\0', "catchain-max-block-delay-slow", "max extended catchain block delay (for too long rounds), (default: 1.0)", - [&](td::Slice s) -> td::Status { - auto v = td::to_double(s); - if (v < 0) { - return td::Status::Error("catchain-max-block-delay-slow should be non-negative"); - } - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_catchain_max_block_delay_slow, v); }); - return td::Status::OK(); - }); - p.add_option( - '\0', "fast-state-serializer", - "faster persistent state serializer, but requires more RAM", - [&]() { - acts.push_back( - [&x]() { td::actor::send_closure(x, &ValidatorEngine::set_fast_state_serializer_enabled, true); }); - }); - p.add_option( - '\0', "collect-validator-telemetry", - "store validator telemetry from fast sync overlay to a given file (json format)", - [&](td::Slice s) { - acts.push_back( - [&x, s = s.str()]() { - td::actor::send_closure(x, &ValidatorEngine::set_validator_telemetry_filename, s); - }); - }); + p.add_checked_option('\0', "catchain-max-block-delay-slow", + "max extended catchain block delay (for too long rounds), (default: 1.0)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v < 0) { + return td::Status::Error("catchain-max-block-delay-slow should be non-negative"); + } + acts.push_back([&x, v]() { + td::actor::send_closure(x, &ValidatorEngine::set_catchain_max_block_delay_slow, v); + }); + return td::Status::OK(); + }); + p.add_option('\0', "fast-state-serializer", "faster persistent state serializer, but requires more RAM", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_fast_state_serializer_enabled, true); }); + }); + p.add_option('\0', "collect-validator-telemetry", + "store validator telemetry from fast sync overlay to a given file (json format)", [&](td::Slice s) { + acts.push_back([&x, s = s.str()]() { + td::actor::send_closure(x, &ValidatorEngine::set_validator_telemetry_filename, s); + }); + }); p.add_option( '\0', "disable-state-serializer", "disable persistent state serializer (similar to set-state-serializer-enabled 0 in validator console)", [&]() { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 26b6842efc..11c503d65a 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -27,24 +27,23 @@ */ #pragma once +#include "adnl/adnl-ext-client.h" #include "adnl/adnl-node-id.hpp" #include "adnl/adnl.h" #include "auto/tl/ton_api.h" -#include "overlays.h" +#include "auto/tl/ton_api.hpp" +#include "auto/tl/ton_api_json.h" +#include "dht/dht.h" #include "rldp/rldp.h" #include "rldp2/rldp.h" -#include "dht/dht.h" +#include "td/actor/MultiPromise.h" #include "td/actor/PromiseFuture.h" +#include "validator/full-node-master.h" +#include "validator/full-node.h" #include "validator/manager.h" #include "validator/validator.h" -#include "validator/full-node.h" -#include "validator/full-node-master.h" -#include "adnl/adnl-ext-client.h" - -#include "td/actor/MultiPromise.h" -#include "auto/tl/ton_api_json.h" -#include "auto/tl/ton_api.hpp" +#include "overlays.h" enum ValidatorEnginePermissions : td::uint32 { vep_default = 1, vep_modify = 2, vep_unsafe = 4 }; @@ -492,8 +491,8 @@ class ValidatorEngine : public td::actor::Actor { td::uint32 flags, td::int32 slot, td::int32 expire_at, td::Promise promise); void process_fast_sync_overlay_certificate_request(ton::PublicKeyHash issue_by, ton::adnl::AdnlNodeIdShort issue_to, - td::uint32 flags, td::int32 slot, td::int32 expire_at, - td::Promise promise); + td::uint32 flags, td::int32 slot, td::int32 expire_at, + td::Promise promise); ton::PublicKeyHash find_local_validator_for_cert_issuing(); std::string custom_overlays_config_file() const { @@ -592,10 +591,10 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getActorTextStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); - void run_control_query(ton::ton_api::engine_validator_addShard &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); - void run_control_query(ton::ton_api::engine_validator_delShard &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_addShard &query, td::BufferSlice data, ton::PublicKeyHash src, + td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_delShard &query, td::BufferSlice data, ton::PublicKeyHash src, + td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_addCollator &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_delCollator &query, td::BufferSlice data, diff --git a/validator-session/candidate-serializer.cpp b/validator-session/candidate-serializer.cpp index 9aa376e2a2..94dfe20480 100644 --- a/validator-session/candidate-serializer.cpp +++ b/validator-session/candidate-serializer.cpp @@ -14,11 +14,12 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "candidate-serializer.h" -#include "tl-utils/tl-utils.hpp" -#include "vm/boc.h" #include "td/utils/lz4.h" +#include "tl-utils/tl-utils.hpp" #include "vm/boc-compression.h" +#include "vm/boc.h" + +#include "candidate-serializer.h" #include "validator-session-types.h" namespace ton::validatorsession { @@ -43,32 +44,32 @@ td::Result> deserialize_candi } TRY_RESULT(f, fetch_tl_object(data, true)); td::Result> res; - ton_api::downcast_call(*f, td::overloaded( - [&](ton_api::validatorSession_candidate& c) { - res = td::Status::Error("Received decompressed tl object, while compression_enabled=true"); - }, - [&](ton_api::validatorSession_compressedCandidate& c) { - res = [&]() -> td::Result> { - if (c.decompressed_size_ > max_decompressed_data_size) { - return td::Status::Error("decompressed size is too big"); - } - TRY_RESULT(p, decompress_candidate_data(c.data_, false, c.decompressed_size_, - max_decompressed_data_size, proto_version)); - return create_tl_object(c.src_, c.round_, c.root_hash_, std::move(p.first), - std::move(p.second)); - }(); - }, - [&](ton_api::validatorSession_compressedCandidateV2& c) { - res = [&]() -> td::Result> { - if (c.data_.size() > max_decompressed_data_size) { - return td::Status::Error("Compressed data is too big"); - } - TRY_RESULT(p, decompress_candidate_data(c.data_, true, 0, - max_decompressed_data_size, proto_version)); - return create_tl_object(c.src_, c.round_, c.root_hash_, std::move(p.first), - std::move(p.second)); - }(); - })); + ton_api::downcast_call( + *f, td::overloaded( + [&](ton_api::validatorSession_candidate& c) { + res = td::Status::Error("Received decompressed tl object, while compression_enabled=true"); + }, + [&](ton_api::validatorSession_compressedCandidate& c) { + res = [&]() -> td::Result> { + if (c.decompressed_size_ > max_decompressed_data_size) { + return td::Status::Error("decompressed size is too big"); + } + TRY_RESULT(p, decompress_candidate_data(c.data_, false, c.decompressed_size_, + max_decompressed_data_size, proto_version)); + return create_tl_object(c.src_, c.round_, c.root_hash_, + std::move(p.first), std::move(p.second)); + }(); + }, + [&](ton_api::validatorSession_compressedCandidateV2& c) { + res = [&]() -> td::Result> { + if (c.data_.size() > max_decompressed_data_size) { + return td::Status::Error("Compressed data is too big"); + } + TRY_RESULT(p, decompress_candidate_data(c.data_, true, 0, max_decompressed_data_size, proto_version)); + return create_tl_object(c.src_, c.round_, c.root_hash_, + std::move(p.first), std::move(p.second)); + }(); + })); return res; } diff --git a/validator-session/candidate-serializer.h b/validator-session/candidate-serializer.h index 9ab53cc89c..cf688a6921 100644 --- a/validator-session/candidate-serializer.h +++ b/validator-session/candidate-serializer.h @@ -15,8 +15,8 @@ along with TON Blockchain Library. If not, see . */ #pragma once -#include "ton/ton-types.h" #include "auto/tl/ton_api.h" +#include "ton/ton-types.h" namespace ton::validatorsession { diff --git a/validator-session/persistent-vector.cpp b/validator-session/persistent-vector.cpp index b3a40e2cef..2c73f08101 100644 --- a/validator-session/persistent-vector.cpp +++ b/validator-session/persistent-vector.cpp @@ -16,11 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "persistent-vector.h" - +#include "adnl/utils.hpp" #include "auto/tl/ton_api.h" -#include "adnl/utils.hpp" +#include "persistent-vector.h" namespace ton { diff --git a/validator-session/persistent-vector.h b/validator-session/persistent-vector.h index 3491164ca1..62a704527f 100644 --- a/validator-session/persistent-vector.h +++ b/validator-session/persistent-vector.h @@ -20,10 +20,9 @@ #include -#include "td/utils/int_types.h" -#include "td/utils/buffer.h" - #include "adnl/utils.hpp" +#include "td/utils/buffer.h" +#include "td/utils/int_types.h" #include "validator-session-description.h" diff --git a/validator-session/validator-session-description.cpp b/validator-session/validator-session-description.cpp index c747e26e70..1b6f6f05f9 100644 --- a/validator-session/validator-session-description.cpp +++ b/validator-session/validator-session-description.cpp @@ -16,10 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "validator-session.hpp" #include "td/utils/Random.h" #include "td/utils/crypto.h" + #include "validator-session-description.hpp" +#include "validator-session.hpp" namespace ton { @@ -184,9 +185,7 @@ ValidatorSessionDescriptionImpl::MemPool::~MemPool() { void *ValidatorSessionDescriptionImpl::MemPool::alloc(size_t size, size_t align) { CHECK(align && !(align & (align - 1))); // align should be a power of 2 CHECK(size + align <= chunk_size_); - auto get_padding = [&](const uint8_t* ptr) { - return (-(size_t)ptr) & (align - 1); - }; + auto get_padding = [&](const uint8_t *ptr) { return (-(size_t)ptr) & (align - 1); }; while (true) { size_t idx = ptr_ / chunk_size_; if (idx < data_.size()) { @@ -212,7 +211,7 @@ void ValidatorSessionDescriptionImpl::MemPool::clear() { ptr_ = 0; } -bool ValidatorSessionDescriptionImpl::MemPool::contains(const void* ptr) const { +bool ValidatorSessionDescriptionImpl::MemPool::contains(const void *ptr) const { if (ptr == nullptr) { return true; } diff --git a/validator-session/validator-session-description.h b/validator-session/validator-session-description.h index ac50431258..03d9d77f6c 100644 --- a/validator-session/validator-session-description.h +++ b/validator-session/validator-session-description.h @@ -19,16 +19,15 @@ #pragma once #include + +#include "adnl/utils.hpp" +#include "catchain/catchain.h" #include "crypto/common/refcnt.hpp" #include "crypto/common/refint.h" - #include "td/utils/int_types.h" - -#include "adnl/utils.hpp" #include "ton/ton-types.h" #include "validator-session-types.h" -#include "catchain/catchain.h" namespace ton { diff --git a/validator-session/validator-session-description.hpp b/validator-session/validator-session-description.hpp index 0fbb4edebe..af379d0098 100644 --- a/validator-session/validator-session-description.hpp +++ b/validator-session/validator-session-description.hpp @@ -18,14 +18,14 @@ */ #pragma once -#include #include - -#include "validator-session.h" -#include "validator-session-state.h" +#include #include "keys/encryptor.h" +#include "validator-session-state.h" +#include "validator-session.h" + namespace ton { namespace validatorsession { @@ -65,7 +65,7 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription { ~MemPool(); void *alloc(size_t size, size_t align); void clear(); - bool contains(const void* ptr) const; + bool contains(const void *ptr) const; private: size_t chunk_size_; diff --git a/validator-session/validator-session-round-attempt-state.cpp b/validator-session/validator-session-round-attempt-state.cpp index 33c3d8047b..afecb79ab5 100644 --- a/validator-session/validator-session-round-attempt-state.cpp +++ b/validator-session/validator-session-round-attempt-state.cpp @@ -16,11 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "validator-session-state.h" -#include "td/utils/Random.h" +#include + #include "auto/tl/ton_api.hpp" +#include "td/utils/Random.h" -#include +#include "validator-session-state.h" namespace ton { @@ -480,7 +481,6 @@ void ValidatorSessionRoundAttemptState::dump(ValidatorSessionDescription& desc, sb << "<<<<\n"; } - } // namespace validatorsession } // namespace ton diff --git a/validator-session/validator-session-round-attempt-state.h b/validator-session/validator-session-round-attempt-state.h index a4a6b68bdf..5dede6297e 100644 --- a/validator-session/validator-session-round-attempt-state.h +++ b/validator-session/validator-session-round-attempt-state.h @@ -17,20 +17,17 @@ #pragma once -#include "td/utils/int_types.h" -#include "td/utils/buffer.h" +#include + #include "adnl/utils.hpp" #include "common/io.hpp" +#include "td/utils/buffer.h" +#include "td/utils/int_types.h" #include "persistent-vector.h" - #include "validator-session-description.h" - #include "validator-session-types.h" -#include - - namespace ton { namespace validatorsession { @@ -614,7 +611,6 @@ class ValidatorSessionRoundAttemptState : public ValidatorSessionDescription::Ro using AttemptVector = CntSortedVector; - } // namespace validatorsession } // namespace ton diff --git a/validator-session/validator-session-state.cpp b/validator-session/validator-session-state.cpp index 97b9189549..97e5566904 100644 --- a/validator-session/validator-session-state.cpp +++ b/validator-session/validator-session-state.cpp @@ -16,12 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "validator-session-state.h" -#include "td/utils/overloaded.h" -#include "td/utils/Random.h" +#include + #include "auto/tl/ton_api.hpp" +#include "td/utils/Random.h" +#include "td/utils/overloaded.h" -#include +#include "validator-session-state.h" namespace td { diff --git a/validator-session/validator-session-state.h b/validator-session/validator-session-state.h index 4efaf77ff8..c2db1bb09e 100644 --- a/validator-session/validator-session-state.h +++ b/validator-session/validator-session-state.h @@ -18,24 +18,24 @@ */ #pragma once -#include "td/utils/int_types.h" -#include "td/utils/buffer.h" +#include + #include "adnl/utils.hpp" #include "common/io.hpp" +#include "td/utils/buffer.h" +#include "td/utils/int_types.h" #include "persistent-vector.h" #include "validator-session-description.h" -#include "validator-session-types.h" #include "validator-session-round-attempt-state.h" - -#include +#include "validator-session-types.h" namespace td { td::StringBuilder& operator<<(td::StringBuilder& sb, const ton::ton_api::validatorSession_round_Message& message); td::StringBuilder& operator<<(td::StringBuilder& sb, const ton::ton_api::validatorSession_round_Message* message); -} +} // namespace td namespace ton { @@ -178,7 +178,6 @@ class ValidatorSessionOldRoundState : public ValidatorSessionDescription::RootOb const HashType hash_; }; - class ValidatorSessionRoundState : public ValidatorSessionDescription::RootObject { public: static HashType create_hash(ValidatorSessionDescription& desc, const SentBlock* precommitted_block, td::uint32 seqno, diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index 982d2642b6..a5740ff888 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -18,13 +18,13 @@ */ #pragma once -#include "td/utils/int_types.h" -#include "crypto/common/bitstring.h" +#include + #include "adnl/adnl-node-id.hpp" +#include "crypto/common/bitstring.h" +#include "td/utils/int_types.h" #include "ton/ton-types.h" -#include - namespace ton { namespace validatorsession { diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 27c105929b..6f2499adeb 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -16,12 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "validator-session.hpp" #include "td/utils/Random.h" -#include "candidate-serializer.h" #include "td/utils/overloaded.h" #include "ton/ton-tl.hpp" +#include "candidate-serializer.h" +#include "validator-session.hpp" + namespace ton { namespace validatorsession { @@ -208,7 +209,7 @@ void ValidatorSessionImpl::preprocess_block(catchain::CatChainBlock *block) { } bool ValidatorSessionImpl::ensure_candidate_unique(td::uint32 src_idx, td::uint32 round, - ValidatorSessionCandidateId block_id) { + ValidatorSessionCandidateId block_id) { auto it = src_round_candidate_[src_idx].find(round); if (it != src_round_candidate_[src_idx].end() && it->second != block_id) { VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << description_->get_source_adnl_id(src_idx) << "][candidate " @@ -770,8 +771,8 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { td::actor::send_closure(SelfId, &ValidatorSessionImpl::candidate_decision_ok, round, hash, root_hash, file_hash, src, R.ok_from(), timer.elapsed(), R.is_cached()); } else { - td::actor::send_closure(SelfId, &ValidatorSessionImpl::candidate_decision_fail, round, hash, R.reason(), - src, R.proof(), timer.elapsed(), R.is_cached()); + td::actor::send_closure(SelfId, &ValidatorSessionImpl::candidate_decision_fail, round, hash, R.reason(), src, + R.proof(), timer.elapsed(), R.is_cached()); } }); pending_approve_.insert(block_id); @@ -1166,9 +1167,9 @@ void ValidatorSessionImpl::start() { auto w = description().export_catchain_nodes(); - catchain_ = catchain::CatChain::create( - make_catchain_callback(), description().opts().catchain_opts, keyring_, adnl_, overlay_manager_, std::move(w), - local_id(), unique_hash_, db_root_, db_suffix_, allow_unsafe_self_blocks_resync_); + catchain_ = catchain::CatChain::create(make_catchain_callback(), description().opts().catchain_opts, keyring_, adnl_, + overlay_manager_, std::move(w), local_id(), unique_hash_, db_root_, db_suffix_, + allow_unsafe_self_blocks_resync_); check_all(); } @@ -1280,14 +1281,14 @@ void ValidatorSessionImpl::stats_init() { cur_stats_.total_weight = description().get_total_weight(); cur_stats_.self = description().get_source_id(local_idx()); - for (auto it = stats_pending_approve_.begin(); it != stats_pending_approve_.end(); ) { + for (auto it = stats_pending_approve_.begin(); it != stats_pending_approve_.end();) { if (it->first.first < cur_round_) { it = stats_pending_approve_.erase(it); } else { ++it; } } - for (auto it = stats_pending_sign_.begin(); it != stats_pending_sign_.end(); ) { + for (auto it = stats_pending_sign_.begin(); it != stats_pending_sign_.end();) { if (it->first.first < cur_round_) { it = stats_pending_sign_.erase(it); } else { @@ -1305,7 +1306,7 @@ void ValidatorSessionImpl::stats_init() { void ValidatorSessionImpl::stats_add_round() { td::uint32 round = cur_stats_.first_round + cur_stats_.rounds.size(); cur_stats_.rounds.emplace_back(); - auto& stat = cur_stats_.rounds.back(); + auto &stat = cur_stats_.rounds.back(); stat.producers.resize(description().get_max_priority() + 1); for (td::uint32 i = 0; i < description().get_total_nodes(); i++) { td::int32 priority = description().get_node_priority(i, round); @@ -1374,38 +1375,38 @@ ValidatorSessionStats::Producer *ValidatorSessionImpl::stats_get_candidate_stat_ } void ValidatorSessionImpl::stats_process_action(td::uint32 node_id, ton_api::validatorSession_round_Message &action) { - ton_api::downcast_call(action, td::overloaded( - [&](const ton_api::validatorSession_message_submittedBlock &obj) { - auto candidate_id = description().candidate_id( - node_id, obj.root_hash_, obj.file_hash_, obj.collated_data_file_hash_); - auto stat = stats_get_candidate_stat( - obj.round_, description().get_source_id(node_id), candidate_id); - if (stat && stat->got_submit_at <= 0.0) { - stat->got_submit_at = td::Clocks::system(); - stat->block_id.root_hash = obj.root_hash_; - stat->block_id.file_hash = obj.file_hash_; - stat->collated_data_hash = obj.collated_data_file_hash_; - } - }, - [&](const ton_api::validatorSession_message_approvedBlock &obj) { - if (obj.candidate_ == skip_round_candidate_id()) { - return; - } - process_approve(node_id, obj.round_, obj.candidate_); - }, - [&](const ton_api::validatorSession_message_commit &obj) { - if (obj.candidate_ == skip_round_candidate_id()) { - return; - } - auto stat = stats_get_candidate_stat_by_id(obj.round_, obj.candidate_); - if (stat) { - stat->set_signed_by(node_id, description().get_node_weight(node_id), - description().get_total_weight()); - } else { - stats_pending_sign_[{obj.round_, obj.candidate_}].push_back(node_id); - } - }, - [](const auto &) {})); + ton_api::downcast_call( + action, + td::overloaded( + [&](const ton_api::validatorSession_message_submittedBlock &obj) { + auto candidate_id = + description().candidate_id(node_id, obj.root_hash_, obj.file_hash_, obj.collated_data_file_hash_); + auto stat = stats_get_candidate_stat(obj.round_, description().get_source_id(node_id), candidate_id); + if (stat && stat->got_submit_at <= 0.0) { + stat->got_submit_at = td::Clocks::system(); + stat->block_id.root_hash = obj.root_hash_; + stat->block_id.file_hash = obj.file_hash_; + stat->collated_data_hash = obj.collated_data_file_hash_; + } + }, + [&](const ton_api::validatorSession_message_approvedBlock &obj) { + if (obj.candidate_ == skip_round_candidate_id()) { + return; + } + process_approve(node_id, obj.round_, obj.candidate_); + }, + [&](const ton_api::validatorSession_message_commit &obj) { + if (obj.candidate_ == skip_round_candidate_id()) { + return; + } + auto stat = stats_get_candidate_stat_by_id(obj.round_, obj.candidate_); + if (stat) { + stat->set_signed_by(node_id, description().get_node_weight(node_id), description().get_total_weight()); + } else { + stats_pending_sign_[{obj.round_, obj.candidate_}].push_back(node_id); + } + }, + [](const auto &) {})); } void ValidatorSessionImpl::process_approve(td::uint32 node_id, td::uint32 round, diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index f8e2d62e72..f72c826390 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -20,15 +20,13 @@ #include "adnl/adnl.h" #include "adnl/utils.hpp" +#include "auto/tl/lite_api.h" +#include "catchain/catchain-types.h" +#include "overlay/overlays.h" #include "rldp2/rldp.h" - #include "ton/ton-types.h" -#include "overlay/overlays.h" -#include "catchain/catchain-types.h" - #include "validator-session-types.h" -#include "auto/tl/lite_api.h" namespace ton { diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 4861c52e17..73ee245028 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -18,17 +18,16 @@ */ #pragma once -#include -#include -#include #include +#include +#include +#include -#include "validator-session.h" -#include "validator-session-state.h" - +#include "catchain/catchain.h" #include "keys/encryptor.h" -#include "catchain/catchain.h" +#include "validator-session-state.h" +#include "validator-session.h" namespace ton { diff --git a/validator/apply-block.cpp b/validator/apply-block.cpp index 085302e69d..309d76619e 100644 --- a/validator/apply-block.cpp +++ b/validator/apply-block.cpp @@ -16,12 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "apply-block.hpp" #include "adnl/utils.hpp" -#include "ton/ton-io.hpp" -#include "validator/invariants.hpp" #include "td/actor/MultiPromise.h" +#include "ton/ton-io.hpp" #include "validator/fabric.h" +#include "validator/invariants.hpp" + +#include "apply-block.hpp" namespace ton { @@ -264,7 +265,6 @@ void ApplyBlock::applied_prev() { } }); td::actor::send_closure(manager_, &ValidatorManager::new_block, handle_, state_, std::move(P)); - } void ApplyBlock::applied_set() { diff --git a/validator/apply-block.hpp b/validator/apply-block.hpp index 95deb025a5..8285f2cd03 100644 --- a/validator/apply-block.hpp +++ b/validator/apply-block.hpp @@ -18,8 +18,8 @@ */ #pragma once -#include "td/actor/actor.h" #include "interfaces/validator-manager.h" +#include "td/actor/actor.h" namespace ton { @@ -48,8 +48,8 @@ class ApplyBlock : public td::actor::Actor { , timeout_(timeout) , promise_(std::move(promise)) , perf_timer_("applyblock", 0.1, [manager](double duration) { - send_closure(manager, &ValidatorManager::add_perf_timer_stat, "applyblock", duration); - }) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "applyblock", duration); + }) { } static constexpr td::uint32 apply_block_priority() { diff --git a/validator/block-handle.cpp b/validator/block-handle.cpp index 7fc770cc3b..fe9b400b35 100644 --- a/validator/block-handle.cpp +++ b/validator/block-handle.cpp @@ -16,10 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "block-handle.hpp" #include "adnl/utils.hpp" #include "ton/ton-tl.hpp" +#include "block-handle.hpp" + namespace ton { namespace validator { diff --git a/validator/block-handle.hpp b/validator/block-handle.hpp index 372c6140a1..b448468f76 100644 --- a/validator/block-handle.hpp +++ b/validator/block-handle.hpp @@ -19,12 +19,11 @@ #pragma once #include "interfaces/block-handle.h" -#include "ton/ton-shard.h" -#include "td/actor/actor.h" #include "interfaces/validator-manager.h" -#include "ton/ton-io.hpp" - +#include "td/actor/actor.h" #include "td/utils/ThreadSafeCounter.h" +#include "ton/ton-io.hpp" +#include "ton/ton-shard.h" namespace ton { diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index 7d694af985..a2127ab6a1 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -14,16 +14,16 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "collation-manager.hpp" +#include +#include +#include #include "collator-node/collator-node.hpp" #include "collator-node/utils.hpp" -#include "fabric.h" #include "td/utils/Random.h" -#include -#include -#include +#include "collation-manager.hpp" +#include "fabric.h" namespace ton::validator { diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp index 6691a387ea..387df2d206 100644 --- a/validator/collation-manager.hpp +++ b/validator/collation-manager.hpp @@ -16,9 +16,10 @@ */ #pragma once +#include + #include "interfaces/validator-manager.h" #include "rldp2/rldp.h" -#include namespace ton::validator { diff --git a/validator/collator-node/collator-node-session.cpp b/validator/collator-node/collator-node-session.cpp index 8e131f3c0c..11838de9df 100644 --- a/validator/collator-node/collator-node-session.cpp +++ b/validator/collator-node/collator-node-session.cpp @@ -16,7 +16,6 @@ */ #include "collator-node-session.hpp" - #include "collator-node.hpp" #include "fabric.h" #include "utils.hpp" diff --git a/validator/collator-node/collator-node-session.hpp b/validator/collator-node/collator-node-session.hpp index 0f9f03d30f..11d2c7cf4e 100644 --- a/validator/collator-node/collator-node-session.hpp +++ b/validator/collator-node/collator-node-session.hpp @@ -16,11 +16,12 @@ */ #pragma once +#include +#include + #include "interfaces/validator-manager.h" #include "rldp/rldp.h" #include "rldp2/rldp.h" -#include -#include namespace ton::validator { diff --git a/validator/collator-node/collator-node.cpp b/validator/collator-node/collator-node.cpp index 9b03d9dc0b..75dc3321af 100644 --- a/validator/collator-node/collator-node.cpp +++ b/validator/collator-node/collator-node.cpp @@ -14,15 +14,16 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "collator-node.hpp" +#include "impl/collator-impl.h" +#include "impl/shard.hpp" +#include "td/utils/lz4.h" #include "ton/ton-tl.hpp" -#include "fabric.h" + #include "block-auto.h" #include "block-db.h" -#include "td/utils/lz4.h" #include "checksum.h" -#include "impl/collator-impl.h" -#include "impl/shard.hpp" +#include "collator-node.hpp" +#include "fabric.h" #include "utils.hpp" namespace ton::validator { diff --git a/validator/collator-node/collator-node.hpp b/validator/collator-node/collator-node.hpp index cb9cec47c5..bf04e57d91 100644 --- a/validator/collator-node/collator-node.hpp +++ b/validator/collator-node/collator-node.hpp @@ -16,12 +16,14 @@ */ #pragma once -#include "collator-node-session.hpp" +#include +#include + #include "interfaces/validator-manager.h" #include "rldp/rldp.h" #include "rldp2/rldp.h" -#include -#include + +#include "collator-node-session.hpp" namespace ton::validator { diff --git a/validator/collator-node/utils.cpp b/validator/collator-node/utils.cpp index 23e2ec40f9..41b6dc4329 100644 --- a/validator/collator-node/utils.cpp +++ b/validator/collator-node/utils.cpp @@ -14,12 +14,13 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "utils.hpp" -#include "checksum.h" #include "keys/keys.hpp" #include "ton/ton-tl.hpp" #include "validator-session/candidate-serializer.h" +#include "checksum.h" +#include "utils.hpp" + namespace ton::validator { tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress) { diff --git a/validator/collator-node/utils.hpp b/validator/collator-node/utils.hpp index 26b67d5a97..8ceba01f22 100644 --- a/validator/collator-node/utils.hpp +++ b/validator/collator-node/utils.hpp @@ -16,8 +16,8 @@ */ #pragma once -#include "ton/ton-types.h" #include "tl/generate/auto/tl/ton_api.h" +#include "ton/ton-types.h" namespace ton::validator { diff --git a/validator/db/archive-db.cpp b/validator/db/archive-db.cpp index cf0dbed1c1..1dd7c03b0e 100644 --- a/validator/db/archive-db.cpp +++ b/validator/db/archive-db.cpp @@ -1,12 +1,11 @@ -#include "archive-db.hpp" #include "common/errorcode.h" - #include "common/int-to-string.hpp" -#include "files-async.hpp" - #include "td/db/RocksDb.h" #include "validator/fabric.h" +#include "archive-db.hpp" +#include "files-async.hpp" + namespace ton { namespace validator { diff --git a/validator/db/archive-db.hpp b/validator/db/archive-db.hpp index d842d9f47d..2305f8e159 100644 --- a/validator/db/archive-db.hpp +++ b/validator/db/archive-db.hpp @@ -1,15 +1,16 @@ #pragma once +#include +#include + #include "td/actor/actor.h" #include "td/utils/buffer.h" -#include "ton/ton-types.h" #include "td/utils/port/FileFd.h" -#include "package.hpp" -#include "filedb.hpp" +#include "ton/ton-types.h" #include "validator/interfaces/block-handle.h" -#include -#include +#include "filedb.hpp" +#include "package.hpp" namespace ton { diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 2020164576..e63b1599f7 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -16,12 +16,13 @@ Copyright 2019-2020 Telegram Systems LLP */ -#include "archive-manager.hpp" +#include "common/delay.h" #include "td/actor/MultiPromise.h" +#include "td/db/RocksDb.h" #include "td/utils/overloaded.h" + +#include "archive-manager.hpp" #include "files-async.hpp" -#include "td/db/RocksDb.h" -#include "common/delay.h" namespace ton { diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index 19a41b194b..247e0a99b3 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -18,9 +18,10 @@ */ #pragma once -#include "archive-slice.hpp" #include "interfaces/persistent-state.h" +#include "archive-slice.hpp" + namespace ton { namespace validator { diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index 2831778dc7..1cdbef1363 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -16,22 +16,22 @@ Copyright 2019-2020 Telegram Systems LLP */ -#include "archive-slice.hpp" - +#include "common/delay.h" #include "td/actor/MultiPromise.h" -#include "validator/fabric.h" #include "td/db/RocksDb.h" #include "td/utils/port/path.h" -#include "common/delay.h" -#include "files-async.hpp" +#include "validator/fabric.h" + +#include "archive-slice.hpp" #include "db-utils.h" +#include "files-async.hpp" namespace ton { namespace validator { class PackageStatistics { - public: + public: void record_open(uint64_t count = 1) { open_count.fetch_add(count, std::memory_order_relaxed); } @@ -82,7 +82,7 @@ class PackageStatistics { return ss.str(); } - private: + private: std::atomic_uint64_t open_count{0}; std::atomic_uint64_t close_count{0}; PercentileStats read_time; @@ -131,8 +131,12 @@ void PackageWriter::append(std::string filename, td::BufferSlice data, class PackageReader : public td::actor::Actor { public: PackageReader(std::shared_ptr package, td::uint64 offset, - td::Promise> promise, std::shared_ptr statistics) - : package_(std::move(package)), offset_(offset), promise_(std::move(promise)), statistics_(std::move(statistics)) { + td::Promise> promise, + std::shared_ptr statistics) + : package_(std::move(package)) + , offset_(offset) + , promise_(std::move(promise)) + , statistics_(std::move(statistics)) { } void start_up() override { auto start = td::Timestamp::now(); @@ -404,7 +408,8 @@ void ArchiveSlice::get_file(ConstBlockHandle handle, FileReference ref_id, td::P promise.set_value(std::move(R.move_as_ok().second)); } }); - td::actor::create_actor("reader", p->package, offset, std::move(P), statistics_.pack_statistics).release(); + td::actor::create_actor("reader", p->package, offset, std::move(P), statistics_.pack_statistics) + .release(); } void ArchiveSlice::get_block_common(AccountIdPrefixFull account_id, @@ -509,14 +514,14 @@ void ArchiveSlice::get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno return get_block_common( account_id, [seqno](ton_api::db_lt_desc_value &w) { - return seqno > static_cast(w.last_seqno_) - ? 1 - : seqno == static_cast(w.last_seqno_) ? 0 : -1; + return seqno > static_cast(w.last_seqno_) ? 1 + : seqno == static_cast(w.last_seqno_) ? 0 + : -1; }, [seqno](ton_api::db_lt_el_value &w) { - return seqno > static_cast(w.id_->seqno_) - ? 1 - : seqno == static_cast(w.id_->seqno_) ? 0 : -1; + return seqno > static_cast(w.id_->seqno_) ? 1 + : seqno == static_cast(w.id_->seqno_) ? 0 + : -1; }, true, std::move(promise)); } @@ -721,7 +726,7 @@ void ArchiveSlice::do_close() { id_to_package_.clear(); } -template +template td::Promise ArchiveSlice::begin_async_query(td::Promise promise) { ++active_queries_; return [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { @@ -1030,7 +1035,7 @@ void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle, td:: return; } - std::map old_packages; + std::map old_packages; std::map> new_packages; std::string value; @@ -1071,7 +1076,7 @@ void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle, td:: truncate_shard(masterchain_seqno, shard, package->seqno, new_packages[package->shard_prefix].get()); } - for (auto& [shard_prefix, package] : old_packages) { + for (auto &[shard_prefix, package] : old_packages) { auto new_package = new_packages[shard_prefix]; CHECK(new_package); package->package = new_package; diff --git a/validator/db/archive-slice.hpp b/validator/db/archive-slice.hpp index 47232252ab..4fb0bab92b 100644 --- a/validator/db/archive-slice.hpp +++ b/validator/db/archive-slice.hpp @@ -18,11 +18,13 @@ */ #pragma once +#include + +#include "td/db/RocksDb.h" #include "validator/interfaces/db.h" -#include "package.hpp" + #include "fileref.hpp" -#include "td/db/RocksDb.h" -#include +#include "package.hpp" namespace rocksdb { class Statistics; @@ -70,7 +72,8 @@ struct DbStatistics { class PackageWriter : public td::actor::Actor { public: - PackageWriter(std::weak_ptr package, bool async_mode = false, std::shared_ptr statistics = nullptr) + PackageWriter(std::weak_ptr package, bool async_mode = false, + std::shared_ptr statistics = nullptr) : package_(std::move(package)), async_mode_(async_mode), statistics_(std::move(statistics)) { } @@ -132,7 +135,7 @@ class ArchiveSlice : public td::actor::Actor { private: void before_query(); void do_close(); - template + template td::Promise begin_async_query(td::Promise promise); void end_async_query(); @@ -164,9 +167,7 @@ class ArchiveSlice : public td::actor::Actor { bool shard_separated_{false}; td::uint32 shard_split_depth_ = 0; - enum Status { - st_closed, st_open, st_want_close - } status_ = st_closed; + enum Status { st_closed, st_open, st_want_close } status_ = st_closed; size_t active_queries_ = 0; std::string db_root_; @@ -175,8 +176,8 @@ class ArchiveSlice : public td::actor::Actor { std::unique_ptr kv_; struct PackageInfo { - PackageInfo(std::shared_ptr package, td::actor::ActorOwn writer, BlockSeqno seqno, ShardIdFull shard_prefix, - std::string path, td::uint32 idx, td::uint32 version) + PackageInfo(std::shared_ptr package, td::actor::ActorOwn writer, BlockSeqno seqno, + ShardIdFull shard_prefix, std::string path, td::uint32 idx, td::uint32 version) : package(std::move(package)) , writer(std ::move(writer)) , seqno(seqno) @@ -222,6 +223,7 @@ class ArchiveLru : public td::actor::Actor { } void on_query(td::actor::ActorId slice, PackageId id, size_t files_count); void set_permanent_slices(std::vector ids); + private: size_t current_idx_ = 1; struct SliceInfo { diff --git a/validator/db/archiver.cpp b/validator/db/archiver.cpp index ffbfc9f053..d0630d0cb8 100644 --- a/validator/db/archiver.cpp +++ b/validator/db/archiver.cpp @@ -16,9 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "ton/ton-tl.hpp" + #include "archiver.hpp" #include "rootdb.hpp" -#include "ton/ton-tl.hpp" namespace ton { diff --git a/validator/db/archiver.hpp b/validator/db/archiver.hpp index 531e114c50..e70075fcce 100644 --- a/validator/db/archiver.hpp +++ b/validator/db/archiver.hpp @@ -18,10 +18,11 @@ */ #pragma once -#include "ton/ton-types.h" #include "td/actor/actor.h" -#include "validator/interfaces/block-handle.h" #include "ton/ton-io.hpp" +#include "ton/ton-types.h" +#include "validator/interfaces/block-handle.h" + #include "archive-manager.hpp" namespace ton { diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index f4a3a084bf..1072fd28e1 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -16,26 +16,21 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "celldb.hpp" - -#include "files-async.hpp" -#include "rootdb.hpp" - -#include "td/db/RocksDb.h" -#include "rocksdb/utilities/optimistic_transaction_db.h" +#include +#include -#include "ton/ton-tl.hpp" -#include "ton/ton-io.hpp" -#include "common/delay.h" #include "block/block-auto.h" +#include "common/delay.h" #include "permanent-celldb/permanent-celldb-utils.h" +#include "rocksdb/utilities/optimistic_transaction_db.h" #include "td/actor/MultiPromise.h" +#include "td/db/RocksDb.h" +#include "ton/ton-io.hpp" +#include "ton/ton-tl.hpp" -#include -#include - -#include -#include +#include "celldb.hpp" +#include "files-async.hpp" +#include "rootdb.hpp" namespace ton { @@ -234,8 +229,8 @@ void CellDbIn::start_up() { } db_options.enable_bloom_filter = !opts_->get_celldb_disable_bloom_filter(); - db_options.two_level_index_and_filter = db_options.enable_bloom_filter - && opts_->state_ttl() >= 60 * 60 * 24 * 30; // 30 days + db_options.two_level_index_and_filter = + db_options.enable_bloom_filter && opts_->state_ttl() >= 60 * 60 * 24 * 30; // 30 days if (db_options.two_level_index_and_filter && !opts_->get_celldb_in_memory()) { o_celldb_cache_size = std::max(o_celldb_cache_size ? o_celldb_cache_size.value() : 0UL, 16UL << 30); } @@ -799,55 +794,56 @@ void CellDbIn::gc_cont2(BlockHandle handle) { P = std::move(P), N = std::move(N), cell = std::move(cell), timer = std::move(timer), timer_all = std::move(timer_all), handle](td::Result R) mutable { R.ensure(); - td::actor::send_lambda_later(SelfId, [this, timer_boc = std::move(timer_boc), F = std::move(F), key_hash, - P = std::move(P), N = std::move(N), cell = std::move(cell), - timer = std::move(timer), timer_all = std::move(timer_all), handle]() mutable { - TD_PERF_COUNTER(celldb_gc_cell); - vm::CellStorer stor{*cell_db_}; - timer_boc.reset(); + td::actor::send_lambda_later( + SelfId, + [this, timer_boc = std::move(timer_boc), F = std::move(F), key_hash, P = std::move(P), N = std::move(N), + cell = std::move(cell), timer = std::move(timer), timer_all = std::move(timer_all), handle]() mutable { + TD_PERF_COUNTER(celldb_gc_cell); + vm::CellStorer stor{*cell_db_}; + timer_boc.reset(); - td::PerfWarningTimer timer_write_batch{"gccell_write_batch", 0.05}; - cell_db_->begin_write_batch().ensure(); + td::PerfWarningTimer timer_write_batch{"gccell_write_batch", 0.05}; + cell_db_->begin_write_batch().ensure(); - boc_->meta_erase(get_key(key_hash)).ensure(); - set_block(F.prev, std::move(P)); - set_block(F.next, std::move(N)); - if (handle->id().is_masterchain()) { - last_deleted_mc_state_ = handle->id().seqno(); - std::string key = "stats.last_deleted_mc_seqno", value = td::to_string(last_deleted_mc_state_); - boc_->meta_set(td::as_slice(key), td::as_slice(value)); - } + boc_->meta_erase(get_key(key_hash)).ensure(); + set_block(F.prev, std::move(P)); + set_block(F.next, std::move(N)); + if (handle->id().is_masterchain()) { + last_deleted_mc_state_ = handle->id().seqno(); + std::string key = "stats.last_deleted_mc_seqno", value = td::to_string(last_deleted_mc_state_); + boc_->meta_set(td::as_slice(key), td::as_slice(value)); + } - boc_->commit(stor).ensure(); - cell_db_->commit_write_batch().ensure(); + boc_->commit(stor).ensure(); + cell_db_->commit_write_batch().ensure(); - alarm_timestamp() = td::Timestamp::now(); - timer_write_batch.reset(); + alarm_timestamp() = td::Timestamp::now(); + timer_write_batch.reset(); - td::PerfWarningTimer timer_free_cells{"gccell_free_cells", 0.05}; - auto before = td::ref_get_delete_count(); - cell = {}; - auto after = td::ref_get_delete_count(); - if (timer_free_cells.elapsed() > 0.04) { - LOG(ERROR) << "deleted " << after - before << " cells"; - } - timer_free_cells.reset(); + td::PerfWarningTimer timer_free_cells{"gccell_free_cells", 0.05}; + auto before = td::ref_get_delete_count(); + cell = {}; + auto after = td::ref_get_delete_count(); + if (timer_free_cells.elapsed() > 0.04) { + LOG(ERROR) << "deleted " << after - before << " cells"; + } + timer_free_cells.reset(); - td::PerfWarningTimer timer_finish{"gccell_finish", 0.05}; - if (!opts_->get_celldb_in_memory()) { - boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); - td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); - } + td::PerfWarningTimer timer_finish{"gccell_finish", 0.05}; + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + } - DCHECK(get_block(key_hash).is_error()); - if (!opts_->get_disable_rocksdb_stats()) { - cell_db_statistics_.gc_cell_time_.insert(timer.elapsed() * 1e6); - } - LOG(DEBUG) << "Deleted state " << handle->id().to_str(); - timer_finish.reset(); - timer_all.reset(); - release_db(); - }); + DCHECK(get_block(key_hash).is_error()); + if (!opts_->get_disable_rocksdb_stats()) { + cell_db_statistics_.gc_cell_time_.insert(timer.elapsed() * 1e6); + } + LOG(DEBUG) << "Deleted state " << handle->id().to_str(); + timer_finish.reset(); + timer_all.reset(); + release_db(); + }); }); } diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index f70f6aef82..d83f68eec6 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -18,19 +18,20 @@ */ #pragma once -#include "td/actor/actor.h" -#include "crypto/vm/db/DynamicBagOfCellsDb.h" +#include +#include + +#include "auto/tl/ton_api.h" #include "crypto/vm/db/CellStorage.h" -#include "td/db/KeyValue.h" -#include "ton/ton-types.h" +#include "crypto/vm/db/DynamicBagOfCellsDb.h" #include "interfaces/block-handle.h" -#include "auto/tl/ton_api.h" -#include "validator.h" -#include "db-utils.h" +#include "td/actor/actor.h" +#include "td/db/KeyValue.h" #include "td/db/RocksDb.h" +#include "ton/ton-types.h" -#include -#include +#include "db-utils.h" +#include "validator.h" namespace rocksdb { class Statistics; diff --git a/validator/db/db-utils.cpp b/validator/db/db-utils.cpp index 9375f97e60..b7f2f849e5 100644 --- a/validator/db/db-utils.cpp +++ b/validator/db/db-utils.cpp @@ -14,11 +14,11 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "db-utils.h" +#include #include "td/utils/logging.h" -#include +#include "db-utils.h" namespace ton::validator { diff --git a/validator/db/fileref.cpp b/validator/db/fileref.cpp index 4e87327dc6..bfdf391dc6 100644 --- a/validator/db/fileref.cpp +++ b/validator/db/fileref.cpp @@ -16,10 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "fileref.hpp" #include "auto/tl/ton_api.hpp" -#include "td/utils/overloaded.h" #include "td/utils/misc.h" +#include "td/utils/overloaded.h" + +#include "fileref.hpp" namespace ton { diff --git a/validator/db/fileref.hpp b/validator/db/fileref.hpp index df368f759c..d6e678a44a 100644 --- a/validator/db/fileref.hpp +++ b/validator/db/fileref.hpp @@ -18,11 +18,11 @@ */ #pragma once -#include "ton/ton-types.h" #include "td/actor/actor.h" -#include "validator/interfaces/shard.h" #include "td/db/KeyValue.h" #include "ton/ton-tl.hpp" +#include "ton/ton-types.h" +#include "validator/interfaces/shard.h" namespace ton { @@ -473,4 +473,3 @@ class FileReference { } // namespace validator } // namespace ton - diff --git a/validator/db/files-async.hpp b/validator/db/files-async.hpp index c850899636..177f685763 100644 --- a/validator/db/files-async.hpp +++ b/validator/db/files-async.hpp @@ -18,12 +18,11 @@ */ #pragma once -#include "td/utils/port/path.h" -#include "td/utils/filesystem.h" +#include "common/errorcode.h" #include "td/actor/actor.h" #include "td/utils/buffer.h" - -#include "common/errorcode.h" +#include "td/utils/filesystem.h" +#include "td/utils/port/path.h" namespace ton { @@ -80,7 +79,7 @@ class WriteFile : public td::actor::Actor { } WriteFile(std::string tmp_dir, std::string new_name, td::BufferSlice data, td::Promise promise) : tmp_dir_(tmp_dir), new_name_(new_name), promise_(std::move(promise)) { - write_data_ = [data_ptr = std::make_shared(std::move(data))] (td::FileFd& fd) { + write_data_ = [data_ptr = std::make_shared(std::move(data))](td::FileFd& fd) { auto data = std::move(*data_ptr); while (data.size() > 0) { auto piece_size = std::min(data.size(), 1 << 30); diff --git a/validator/db/package.cpp b/validator/db/package.cpp index be96164229..f3db291748 100644 --- a/validator/db/package.cpp +++ b/validator/db/package.cpp @@ -16,9 +16,10 @@ Copyright 2019-2020 Telegram Systems LLP */ -#include "package.hpp" #include "common/errorcode.h" +#include "package.hpp" + namespace ton { namespace { diff --git a/validator/db/package.hpp b/validator/db/package.hpp index b2dd4e9fc3..f67c395d68 100644 --- a/validator/db/package.hpp +++ b/validator/db/package.hpp @@ -19,8 +19,8 @@ #pragma once #include "td/actor/actor.h" -#include "td/utils/port/FileFd.h" #include "td/utils/buffer.h" +#include "td/utils/port/FileFd.h" namespace ton { diff --git a/validator/db/permanent-celldb/permanent-celldb-utils.cpp b/validator/db/permanent-celldb/permanent-celldb-utils.cpp index b1d19ceb4b..04e33d9ab3 100644 --- a/validator/db/permanent-celldb/permanent-celldb-utils.cpp +++ b/validator/db/permanent-celldb/permanent-celldb-utils.cpp @@ -14,12 +14,13 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "permanent-celldb-utils.h" #include "block/block-auto.h" #include "td/actor/MultiPromise.h" #include "td/utils/HashMap.h" #include "vm/db/CellStorage.h" +#include "permanent-celldb-utils.h" + namespace ton::validator { void calculate_permanent_celldb_update(const std::map>& blocks, diff --git a/validator/db/permanent-celldb/permanent-celldb-utils.h b/validator/db/permanent-celldb/permanent-celldb-utils.h index fad2ddfcdd..64a9afd64d 100644 --- a/validator/db/permanent-celldb/permanent-celldb-utils.h +++ b/validator/db/permanent-celldb/permanent-celldb-utils.h @@ -15,14 +15,14 @@ along with TON Blockchain Library. If not, see . */ #pragma once -#include "vm/cells.h" -#include "ton/ton-types.h" +#include +#include + #include "interfaces/block.h" +#include "ton/ton-types.h" +#include "vm/cells.h" #include "vm/db/DynamicBagOfCellsDb.h" -#include -#include - namespace ton::validator { struct PermanentCellDbUpdate { diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 596f7417f9..accc111c4c 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -16,16 +16,16 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "rootdb.hpp" -#include "validator/fabric.h" -#include "archiver.hpp" - +#include "common/checksum.h" +#include "td/actor/MultiPromise.h" #include "td/db/RocksDb.h" -#include "ton/ton-tl.hpp" #include "td/utils/overloaded.h" -#include "common/checksum.h" +#include "ton/ton-tl.hpp" +#include "validator/fabric.h" #include "validator/stats-merger.h" -#include "td/actor/MultiPromise.h" + +#include "archiver.hpp" +#include "rootdb.hpp" namespace ton { @@ -333,7 +333,7 @@ void RootDb::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterc void RootDb::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, - std::function write_data, + std::function write_data, td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state_gen, block_id, masterchain_block_id, type, std::move(write_data), std::move(promise)); diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index a39b174226..5a0007eb9a 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -22,10 +22,10 @@ #include "td/db/KeyValueAsync.h" #include "ton/ton-types.h" +#include "archive-manager.hpp" #include "celldb.hpp" #include "statedb.hpp" #include "staticfilesdb.hpp" -#include "archive-manager.hpp" #include "validator.h" namespace ton { @@ -65,7 +65,7 @@ class RootDb : public Db { void store_block_state_from_data(BlockHandle handle, td::Ref block, td::Promise> promise) override; void store_block_state_from_data_preliminary(std::vector> blocks, - td::Promise promise) override; + td::Promise promise) override; void get_block_state(ConstBlockHandle handle, td::Promise> promise) override; void store_block_state_part(BlockId effective_block, td::Ref cell, td::Promise> promise) override; @@ -80,7 +80,7 @@ class RootDb : public Db { void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::BufferSlice state, td::Promise promise) override; void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, - std::function write_data, + std::function write_data, td::Promise promise) override; void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override; @@ -142,8 +142,10 @@ class RootDb : public Db { void set_async_mode(bool mode, td::Promise promise) override; void run_gc(UnixTime mc_ts, UnixTime gc_ts, double archive_ttl) override; - void add_persistent_state_description(td::Ref desc, td::Promise promise) override; - void get_persistent_state_descriptions(td::Promise>> promise) override; + void add_persistent_state_description(td::Ref desc, + td::Promise promise) override; + void get_persistent_state_descriptions( + td::Promise>> promise) override; void iterate_temp_block_handles(std::function f) override; diff --git a/validator/db/statedb.cpp b/validator/db/statedb.cpp index 753ed7e646..6c5d87ce42 100644 --- a/validator/db/statedb.cpp +++ b/validator/db/statedb.cpp @@ -16,11 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "statedb.hpp" -#include "ton/ton-tl.hpp" #include "adnl/utils.hpp" #include "td/db/RocksDb.h" #include "ton/ton-shard.h" +#include "ton/ton-tl.hpp" + +#include "statedb.hpp" namespace ton { @@ -184,7 +185,7 @@ void StateDb::update_hardforks(std::vector blocks, td::Promise> vec; - for (auto &e : blocks) { + for (auto& e : blocks) { vec.push_back(create_tl_block_id(e)); } @@ -210,7 +211,7 @@ void StateDb::get_hardforks(td::Promise> promise) { auto f = F.move_as_ok(); std::vector vec; - for (auto &e : f->blocks_) { + for (auto& e : f->blocks_) { vec.push_back(create_block_id(e)); } diff --git a/validator/db/statedb.hpp b/validator/db/statedb.hpp index fe23898f4f..858b66493c 100644 --- a/validator/db/statedb.hpp +++ b/validator/db/statedb.hpp @@ -21,7 +21,6 @@ #include "td/actor/actor.h" #include "td/db/KeyValueAsync.h" #include "ton/ton-types.h" - #include "validator/interfaces/db.h" namespace ton { diff --git a/validator/db/staticfilesdb.cpp b/validator/db/staticfilesdb.cpp index 9485f15401..f4ccba088e 100644 --- a/validator/db/staticfilesdb.cpp +++ b/validator/db/staticfilesdb.cpp @@ -16,8 +16,8 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "staticfilesdb.hpp" #include "files-async.hpp" +#include "staticfilesdb.hpp" namespace ton { diff --git a/validator/db/staticfilesdb.hpp b/validator/db/staticfilesdb.hpp index c56deb9d0d..557810861b 100644 --- a/validator/db/staticfilesdb.hpp +++ b/validator/db/staticfilesdb.hpp @@ -18,8 +18,8 @@ */ #pragma once -#include "ton/ton-types.h" #include "td/actor/actor.h" +#include "ton/ton-types.h" namespace ton { diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index fbaa8da8f5..866899c37f 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -16,14 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "download-state.hpp" -#include "validator/fabric.h" #include "common/checksum.h" #include "common/delay.h" -#include "ton/ton-io.hpp" -#include "vm/cells/MerkleProof.h" #include "crypto/block/block-auto.h" #include "crypto/block/block-parse.h" +#include "ton/ton-io.hpp" +#include "validator/fabric.h" +#include "vm/cells/MerkleProof.h" + +#include "download-state.hpp" namespace ton { diff --git a/validator/downloaders/download-state.hpp b/validator/downloaders/download-state.hpp index 1c1be04c47..239b8b45de 100644 --- a/validator/downloaders/download-state.hpp +++ b/validator/downloaders/download-state.hpp @@ -19,6 +19,7 @@ #pragma once #include "validator/interfaces/validator-manager.h" + #include "stats-provider.h" namespace ton { diff --git a/validator/downloaders/wait-block-data-disk.cpp b/validator/downloaders/wait-block-data-disk.cpp index 719420cccd..d88265fe3d 100644 --- a/validator/downloaders/wait-block-data-disk.cpp +++ b/validator/downloaders/wait-block-data-disk.cpp @@ -16,11 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "wait-block-data-disk.hpp" -#include "fabric.h" #include "adnl/utils.hpp" #include "ton/ton-io.hpp" +#include "fabric.h" +#include "wait-block-data-disk.hpp" + namespace ton { namespace validator { @@ -54,7 +55,8 @@ void WaitBlockDataDisk::start_up() { } void WaitBlockDataDisk::start() { - if (handle_->received() && (handle_->id().is_masterchain() ? handle_->inited_proof() : handle_->inited_proof_link())) { + if (handle_->received() && + (handle_->id().is_masterchain() ? handle_->inited_proof() : handle_->inited_proof_link())) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &WaitBlockDataDisk::abort_query, R.move_as_error_prefix("db error: ")); diff --git a/validator/downloaders/wait-block-data.cpp b/validator/downloaders/wait-block-data.cpp index 86e8f4b8c1..eb476d199e 100644 --- a/validator/downloaders/wait-block-data.cpp +++ b/validator/downloaders/wait-block-data.cpp @@ -16,16 +16,16 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "wait-block-data.hpp" - -#include "block-parse.h" -#include "block-auto.h" -#include "fabric.h" #include "adnl/utils.hpp" -#include "ton/ton-io.hpp" #include "common/delay.h" +#include "ton/ton-io.hpp" #include "vm/cells/MerkleProof.h" +#include "block-auto.h" +#include "block-parse.h" +#include "fabric.h" +#include "wait-block-data.hpp" + namespace ton { namespace validator { @@ -108,15 +108,15 @@ void WaitBlockData::start() { td::actor::send_closure(manager_, &ValidatorManager::try_get_static_file, handle_->id().file_hash, std::move(P)); } else if (try_get_candidate_ && !handle_->id().is_masterchain()) { try_get_candidate_ = false; - td::actor::send_closure( - manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, handle_->id(), - [SelfId = actor_id(this), id = handle_->id()](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &WaitBlockData::start); - } else { - td::actor::send_closure(SelfId, &WaitBlockData::loaded_data, ReceivedBlock{id, R.move_as_ok()}); - } - }); + td::actor::send_closure(manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, handle_->id(), + [SelfId = actor_id(this), id = handle_->id()](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &WaitBlockData::start); + } else { + td::actor::send_closure(SelfId, &WaitBlockData::loaded_data, + ReceivedBlock{id, R.move_as_ok()}); + } + }); } else { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { diff --git a/validator/downloaders/wait-block-state-merge.cpp b/validator/downloaders/wait-block-state-merge.cpp index c5f1cfa7f8..2c8fb0c368 100644 --- a/validator/downloaders/wait-block-state-merge.cpp +++ b/validator/downloaders/wait-block-state-merge.cpp @@ -16,9 +16,10 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "ton/ton-io.hpp" + #include "wait-block-state-merge.hpp" #include "wait-block-state.hpp" -#include "ton/ton-io.hpp" namespace ton { diff --git a/validator/downloaders/wait-block-state-merge.hpp b/validator/downloaders/wait-block-state-merge.hpp index 6695a80fc2..1472be9480 100644 --- a/validator/downloaders/wait-block-state-merge.hpp +++ b/validator/downloaders/wait-block-state-merge.hpp @@ -67,4 +67,3 @@ class WaitBlockStateMerge : public td::actor::Actor { } // namespace validator } // namespace ton - diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index a69e2b1308..2631f00730 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -16,12 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "wait-block-state.hpp" -#include "validator/fabric.h" -#include "ton/ton-io.hpp" #include "common/checksum.h" #include "common/delay.h" +#include "ton/ton-io.hpp" #include "validator/downloaders/download-state.hpp" +#include "validator/fabric.h" + +#include "wait-block-state.hpp" namespace ton { diff --git a/validator/fabric.h b/validator/fabric.h index 8a3f6193fe..a1a48779a5 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -18,8 +18,9 @@ */ #pragma once -#include "interfaces/validator-manager.h" #include "interfaces/db.h" +#include "interfaces/validator-manager.h" + #include "validator.h" namespace ton { @@ -47,7 +48,8 @@ struct ValidateParams { BlockIdExt min_masterchain_block_id; std::vector prev; td::Ref validator_set = {}; - PublicKeyHash local_validator_id = PublicKeyHash::zero();; + PublicKeyHash local_validator_id = PublicKeyHash::zero(); + ; bool is_fake = false; // Optional - used for validation of optimistic candidates @@ -70,8 +72,7 @@ td::Result create_block_handle(td::BufferSlice data); td::Result create_block_handle(td::Slice data); td::Result create_temp_block_handle(td::BufferSlice data); BlockHandle create_empty_block_handle(BlockIdExt id); -td::Result> create_ext_message(td::BufferSlice data, - block::SizeLimitsConfig::ExtMsgLimits limits); +td::Result> create_ext_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits); td::Result> create_ihr_message(td::BufferSlice data); td::Result>> create_new_shard_block_descriptions(td::BufferSlice data); @@ -112,8 +113,9 @@ void run_collate_query(CollateParams params, td::actor::ActorId promise); void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise); -void run_fetch_account_state(WorkchainId wc, StdSmcAddress addr, td::actor::ActorId manager, - td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise); +void run_fetch_account_state( + WorkchainId wc, StdSmcAddress addr, td::actor::ActorId manager, + td::Promise, UnixTime, LogicalTime, std::unique_ptr>> promise); void run_validate_shard_block_description(td::BufferSlice data, BlockHandle masterchain_block, td::Ref masterchain_state, td::actor::ActorId manager, td::Timestamp timeout, diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index e9927518a5..e25f636233 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -15,14 +15,14 @@ along with TON Blockchain Library. If not, see . */ -#include "full-node-fast-sync-overlays.hpp" - -#include "checksum.h" -#include "ton/ton-tl.hpp" +#include "auto/tl/ton_api_json.h" #include "common/delay.h" #include "td/utils/JsonBuilder.h" #include "tl/tl_json.h" -#include "auto/tl/ton_api_json.h" +#include "ton/ton-tl.hpp" + +#include "checksum.h" +#include "full-node-fast-sync-overlays.hpp" #include "full-node-serializer.hpp" namespace ton::validator::fullnode { diff --git a/validator/full-node-fast-sync-overlays.hpp b/validator/full-node-fast-sync-overlays.hpp index 3505ae5eba..2a886cb108 100644 --- a/validator/full-node-fast-sync-overlays.hpp +++ b/validator/full-node-fast-sync-overlays.hpp @@ -16,11 +16,11 @@ */ #pragma once +#include + #include "full-node.h" #include "validator-telemetry.hpp" -#include - namespace ton::validator::fullnode { class FullNodeFastSyncOverlay : public td::actor::Actor { diff --git a/validator/full-node-master.cpp b/validator/full-node-master.cpp index 7037f83963..da54688318 100644 --- a/validator/full-node-master.cpp +++ b/validator/full-node-master.cpp @@ -16,19 +16,16 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "adnl/utils.hpp" +#include "auto/tl/lite_api.h" +#include "common/delay.h" #include "td/utils/SharedSlice.h" -#include "full-node-master.hpp" -#include "full-node-shard-queries.hpp" - +#include "tl-utils/lite-utils.hpp" #include "ton/ton-shard.h" #include "ton/ton-tl.hpp" -#include "adnl/utils.hpp" - -#include "common/delay.h" - -#include "auto/tl/lite_api.h" -#include "tl-utils/lite-utils.hpp" +#include "full-node-master.hpp" +#include "full-node-shard-queries.hpp" namespace ton { diff --git a/validator/full-node-master.h b/validator/full-node-master.h index 04c66accfe..a44e464acf 100644 --- a/validator/full-node-master.h +++ b/validator/full-node-master.h @@ -18,9 +18,10 @@ */ #pragma once -#include "full-node.h" #include "validator/interfaces/block-handle.h" +#include "full-node.h" + namespace ton { namespace validator { diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index 7ea813a466..b62a7c190c 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -14,14 +14,15 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "full-node-private-overlay.hpp" -#include "ton/ton-tl.hpp" -#include "common/delay.h" -#include "common/checksum.h" -#include "full-node-serializer.hpp" #include "auto/tl/ton_api_json.h" +#include "common/checksum.h" +#include "common/delay.h" #include "td/utils/JsonBuilder.h" #include "tl/tl_json.h" +#include "ton/ton-tl.hpp" + +#include "full-node-private-overlay.hpp" +#include "full-node-serializer.hpp" namespace ton::validator::fullnode { diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index 2cbe25f686..5c620ec6fe 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -16,11 +16,11 @@ */ #pragma once +#include + #include "full-node.h" #include "validator-telemetry.hpp" -#include - namespace ton::validator::fullnode { class FullNodePrivateBlockOverlay : public td::actor::Actor { @@ -37,7 +37,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); - void process_telemetry_broadcast(PublicKeyHash src, const tl_object_ptr& telemetry); + void process_telemetry_broadcast(PublicKeyHash src, const tl_object_ptr &telemetry); template void process_broadcast(PublicKeyHash, T &) { diff --git a/validator/full-node-serializer.cpp b/validator/full-node-serializer.cpp index 5665cc2339..adce0fe22a 100644 --- a/validator/full-node-serializer.cpp +++ b/validator/full-node-serializer.cpp @@ -14,16 +14,17 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "full-node-serializer.hpp" -#include "ton/ton-tl.hpp" -#include "tl-utils/common-utils.hpp" #include "auto/tl/ton_api.hpp" +#include "td/utils/lz4.h" +#include "td/utils/overloaded.h" +#include "tl-utils/common-utils.hpp" #include "tl-utils/tl-utils.hpp" -#include "vm/boc.h" +#include "ton/ton-tl.hpp" #include "vm/boc-compression.h" -#include "td/utils/lz4.h" +#include "vm/boc.h" + +#include "full-node-serializer.hpp" #include "full-node.h" -#include "td/utils/overloaded.h" namespace ton::validator::fullnode { @@ -165,15 +166,17 @@ static td::Status deserialize_block_full(ton_api::tonNode_dataFullCompressed& f, return td::Status::OK(); } -static td::Status deserialize_block_full(ton_api::tonNode_dataFullCompressedV2& f, BlockIdExt& id, td::BufferSlice& proof, - td::BufferSlice& data, bool& is_proof_link, int max_decompressed_size) { +static td::Status deserialize_block_full(ton_api::tonNode_dataFullCompressedV2& f, BlockIdExt& id, + td::BufferSlice& proof, td::BufferSlice& data, bool& is_proof_link, + int max_decompressed_size) { TRY_RESULT(roots, vm::boc_decompress(f.compressed_, max_decompressed_size)); if (roots.size() != 2) { return td::Status::Error("expected 2 roots in boc"); } TRY_RESULT_ASSIGN(proof, vm::std_boc_serialize(roots[0], 0)); TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(roots[1], 31)); - VLOG(FULL_NODE_DEBUG) << "Decompressing block full V2: " << f.compressed_.size() << " -> " << data.size() + proof.size(); + VLOG(FULL_NODE_DEBUG) << "Decompressing block full V2: " << f.compressed_.size() << " -> " + << data.size() + proof.size(); id = create_block_id(f.id_); is_proof_link = f.is_link_; return td::Status::OK(); diff --git a/validator/full-node-serializer.hpp b/validator/full-node-serializer.hpp index f6751689c7..d28495f691 100644 --- a/validator/full-node-serializer.hpp +++ b/validator/full-node-serializer.hpp @@ -15,8 +15,8 @@ along with TON Blockchain Library. If not, see . */ #pragma once -#include "ton/ton-types.h" #include "auto/tl/ton_api.h" +#include "ton/ton-types.h" namespace ton::validator::fullnode { diff --git a/validator/full-node-shard-queries.hpp b/validator/full-node-shard-queries.hpp index 170682297f..349cc521cb 100644 --- a/validator/full-node-shard-queries.hpp +++ b/validator/full-node-shard-queries.hpp @@ -18,8 +18,9 @@ */ #pragma once -#include "validator/validator.h" #include "ton/ton-tl.hpp" +#include "validator/validator.h" + #include "full-node-serializer.hpp" namespace ton { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 39a416ed54..98fc39344c 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -16,35 +16,32 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "auto/tl/ton_api.h" -#include "checksum.h" -#include "overlays.h" -#include "td/utils/SharedSlice.h" -#include "td/utils/overloaded.h" -#include "full-node-shard.hpp" -#include "full-node-shard-queries.hpp" -#include "full-node-serializer.hpp" - -#include "td/utils/buffer.h" -#include "ton/ton-shard.h" -#include "ton/ton-tl.hpp" - #include "adnl/utils.hpp" +#include "auto/tl/ton_api.h" +#include "auto/tl/ton_api_json.h" +#include "common/delay.h" +#include "impl/out-msg-queue-proof.hpp" +#include "net/download-archive-slice.hpp" #include "net/download-block-new.hpp" #include "net/download-block.hpp" #include "net/download-next-block.hpp" -#include "net/download-state.hpp" #include "net/download-proof.hpp" +#include "net/download-state.hpp" #include "net/get-next-key-blocks.hpp" -#include "net/download-archive-slice.hpp" -#include "impl/out-msg-queue-proof.hpp" - -#include "td/utils/Random.h" - -#include "common/delay.h" #include "td/utils/JsonBuilder.h" +#include "td/utils/Random.h" +#include "td/utils/SharedSlice.h" +#include "td/utils/buffer.h" +#include "td/utils/overloaded.h" #include "tl/tl_json.h" -#include "auto/tl/ton_api_json.h" +#include "ton/ton-shard.h" +#include "ton/ton-tl.hpp" + +#include "checksum.h" +#include "full-node-serializer.hpp" +#include "full-node-shard-queries.hpp" +#include "full-node-shard.hpp" +#include "overlays.h" namespace ton { @@ -79,8 +76,7 @@ void Neighbour::update_roundtrip(double t) { void FullNodeShardImpl::create_overlay() { class Callback : public overlay::Overlays::Callback { public: - void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, - td::BufferSlice data) override { + void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { td::actor::send_closure(node_, &FullNodeShardImpl::receive_message, src, std::move(data)); } void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, @@ -147,8 +143,7 @@ void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broad return; } td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_external_message, - std::move(q->message_->data_), - promise.wrap([](td::Ref) { return td::Unit(); })); + std::move(q->message_->data_), promise.wrap([](td::Ref) { return td::Unit(); })); } void FullNodeShardImpl::remove_neighbour(adnl::AdnlNodeIdShort id) { @@ -508,8 +503,8 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise) { - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); @@ -1023,8 +1018,8 @@ void FullNodeShardImpl::download_block_proof_link(BlockIdExt block_id, td::uint3 void FullNodeShardImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) { auto &b = choose_neighbour(); - td::actor::create_actor("next", block_id, 16, adnl_id_, overlay_id_, b.adnl_id, - 1, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, + td::actor::create_actor("next", block_id, 16, adnl_id_, overlay_id_, b.adnl_id, 1, timeout, + validator_manager_, rldp_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) .release(); } @@ -1179,25 +1174,28 @@ void FullNodeShardImpl::signed_new_certificate(ton::overlay::Certificate cert) { td::actor::send_closure(overlays_, &overlay::Overlays::update_certificate, adnl_id_, overlay_id_, local_id_, cert_); } -void FullNodeShardImpl::sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expire_at, td::uint32 max_size, td::Promise promise) { +void FullNodeShardImpl::sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expire_at, td::uint32 max_size, + td::Promise promise) { auto sign_by = sign_cert_by_; if (sign_by.is_zero()) { promise.set_error(td::Status::Error("Node has no key with signing authority")); return; } - ton::overlay::Certificate cert{ - sign_by, static_cast(expire_at), max_size, - overlay::CertificateFlags::Trusted | overlay::CertificateFlags::AllowFec, td::BufferSlice{}}; + ton::overlay::Certificate cert{sign_by, static_cast(expire_at), max_size, + overlay::CertificateFlags::Trusted | overlay::CertificateFlags::AllowFec, + td::BufferSlice{}}; auto to_sign = cert.to_sign(overlay_id_, signed_key); auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), expire_at = expire_at, max_size = max_size, promise = std::move(promise)](td::Result> R) mutable { + [SelfId = actor_id(this), expire_at = expire_at, max_size = max_size, + promise = std::move(promise)](td::Result> R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error_prefix("failed to create certificate: failed to sign: ")); } else { auto p = R.move_as_ok(); - auto c = ton::create_serialize_tl_object(p.second.tl(), static_cast(expire_at), max_size, std::move(p.first)); + auto c = ton::create_serialize_tl_object( + p.second.tl(), static_cast(expire_at), max_size, std::move(p.first)); promise.set_value(std::move(c)); } }); @@ -1205,10 +1203,12 @@ void FullNodeShardImpl::sign_overlay_certificate(PublicKeyHash signed_key, td::u std::move(P)); } -void FullNodeShardImpl::import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) { - td::actor::send_closure(overlays_, &ton::overlay::Overlays::update_certificate, - adnl_id_, overlay_id_, signed_key, cert); - promise.set_value( td::Unit() ); +void FullNodeShardImpl::import_overlay_certificate(PublicKeyHash signed_key, + std::shared_ptr cert, + td::Promise promise) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::update_certificate, adnl_id_, overlay_id_, signed_key, + cert); + promise.set_value(td::Unit()); } void FullNodeShardImpl::update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) { @@ -1226,7 +1226,8 @@ void FullNodeShardImpl::update_validators(std::vector public_key_ authorized_keys.emplace(key, overlay::Overlays::max_fec_broadcast_size()); } - rules_ = overlay::OverlayPrivacyRules{overlay::Overlays::max_fec_broadcast_size(), overlay::CertificateFlags::AllowFec, std::move(authorized_keys)}; + rules_ = overlay::OverlayPrivacyRules{overlay::Overlays::max_fec_broadcast_size(), + overlay::CertificateFlags::AllowFec, std::move(authorized_keys)}; td::actor::send_closure(overlays_, &overlay::Overlays::set_privacy_rules, adnl_id_, overlay_id_, rules_); if (update_cert) { @@ -1294,11 +1295,10 @@ const Neighbour &FullNodeShardImpl::choose_neighbour(td::uint32 required_version if (neighbours_.size() == 0) { return Neighbour::zero; } - auto is_eligible = - [&](const Neighbour &n) { - return n.version_major > required_version_major || - (n.version_major == required_version_major && n.version_minor >= required_version_minor); - }; + auto is_eligible = [&](const Neighbour &n) { + return n.version_major > required_version_major || + (n.version_major == required_version_major && n.version_minor >= required_version_minor); + }; double min_unreliability = 1e9; for (auto &[_, x] : neighbours_) { diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 0c6aa441b8..59df18c24b 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -18,9 +18,10 @@ */ #pragma once -#include "full-node.h" -#include "validator/interfaces/block-handle.h" #include "adnl/adnl-ext-client.h" +#include "validator/interfaces/block-handle.h" + +#include "full-node.h" namespace ton { diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index a90460adb5..c28bad9b8f 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -18,11 +18,13 @@ */ #pragma once +#include + #include "auto/tl/ton_api.h" -#include "full-node-shard.h" #include "td/actor/PromiseFuture.h" #include "td/utils/port/Poll.h" -#include + +#include "full-node-shard.h" namespace ton { @@ -207,8 +209,10 @@ class FullNodeShardImpl : public FullNodeShard { void update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) override; - void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) override; - void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) override; + void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, + td::Promise promise) override; + void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, + td::Promise promise) override; void sign_new_certificate(PublicKeyHash sign_by); void signed_new_certificate(ton::overlay::Certificate cert); diff --git a/validator/full-node.cpp b/validator/full-node.cpp index d648fc821b..0c83e30b9a 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -16,15 +16,16 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "full-node.hpp" -#include "ton/ton-io.hpp" -#include "td/actor/MultiPromise.h" -#include "full-node.h" #include "common/delay.h" #include "impl/out-msg-queue-proof.hpp" +#include "td/actor/MultiPromise.h" #include "td/utils/Random.h" +#include "ton/ton-io.hpp" #include "ton/ton-tl.hpp" +#include "full-node.h" +#include "full-node.hpp" + namespace ton { namespace validator { @@ -109,7 +110,7 @@ void FullNodeImpl::del_collator_adnl_id(adnl::AdnlNodeIdShort id) { void FullNodeImpl::sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) { auto it = shards_.find(shard_id); - if(it == shards_.end() || it->second.actor.empty()) { + if (it == shards_.end() || it->second.actor.empty()) { promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); return; } @@ -121,7 +122,7 @@ void FullNodeImpl::import_shard_overlay_certificate(ShardIdFull shard_id, Public std::shared_ptr cert, td::Promise promise) { auto it = shards_.find(shard_id); - if(it == shards_.end() || it->second.actor.empty()) { + if (it == shards_.end() || it->second.actor.empty()) { promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); return; } @@ -150,15 +151,15 @@ void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise state, std } } - for (auto it = shards_.begin(); it != shards_.end(); ) { + for (auto it = shards_.begin(); it != shards_.end();) { if (all_shards.contains(it->first)) { ++it; } else { @@ -367,8 +368,9 @@ void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_se } if (mode & broadcast_mode_private_block) { if (!private_block_overlays_.empty()) { - td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_block_candidate, - block_id, cc_seqno, validator_set_hash, data.clone()); + td::actor::send_closure(private_block_overlays_.begin()->second, + &FullNodePrivateBlockOverlay::send_block_candidate, block_id, cc_seqno, + validator_set_hash, data.clone()); } auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(block_id.shard_full()).first; if (!fast_sync_overlay.empty()) { @@ -388,11 +390,11 @@ void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_se } void FullNodeImpl::send_out_msg_queue_proof_broadcast(td::Ref broadcast) { - auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(broadcast->dst_shard).first; - if (!fast_sync_overlay.empty()) { - td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_out_msg_queue_proof_broadcast, - std::move(broadcast)); - } + auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(broadcast->dst_shard).first; + if (!fast_sync_overlay.empty()) { + td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_out_msg_queue_proof_broadcast, + std::move(broadcast)); + } } void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, int mode) { @@ -489,7 +491,7 @@ void FullNodeImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeou } void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, - td::Timestamp timeout, td::Promise promise) { + td::Timestamp timeout, td::Promise promise) { auto shard = get_shard(shard_prefix, /* historical = */ true); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping download archive query to unknown shard"; @@ -722,7 +724,7 @@ void FullNodeImpl::start_up() { std::move(data), mode); } void send_out_msg_queue_proof_broadcast(td::Ref broadcast) override { - td::actor::send_closure(id_, &FullNodeImpl::send_out_msg_queue_proof_broadcast, std::move(broadcast)); + td::actor::send_closure(id_, &FullNodeImpl::send_out_msg_queue_proof_broadcast, std::move(broadcast)); } void send_broadcast(BlockBroadcast broadcast, int mode) override { td::actor::send_closure(id_, &FullNodeImpl::send_broadcast, std::move(broadcast), mode); @@ -826,8 +828,8 @@ void FullNodeImpl::update_custom_overlay(CustomOverlayInfo &overlay) { old_actors.erase(it); } else { overlay.actors_[local_id] = td::actor::create_actor( - "CustomOverlay", local_id, params, zero_state_file_hash_, opts_, keyring_, adnl_, rldp_, rldp2_, - overlays_, validator_manager_, actor_id(this)); + "CustomOverlay", local_id, params, zero_state_file_hash_, opts_, keyring_, adnl_, rldp_, rldp2_, overlays_, + validator_manager_, actor_id(this)); } } }; @@ -914,8 +916,8 @@ td::actor::ActorOwn FullNode::create( td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, std::string db_root, td::Promise started_promise) { - return td::actor::create_actor("fullnode", local_id, adnl_id, zero_state_file_hash, opts, keyring, - adnl, rldp, rldp2, dht, overlays, validator_manager, client, db_root, + return td::actor::create_actor("fullnode", local_id, adnl_id, zero_state_file_hash, opts, keyring, adnl, + rldp, rldp2, dht, overlays, validator_manager, client, db_root, std::move(started_promise)); } @@ -939,7 +941,7 @@ bool CustomOverlayParams::send_shard(const ShardIdFull &shard) const { [&](const ShardIdFull &our_shard) { return shard_intersects(shard, our_shard); }); } -CustomOverlayParams CustomOverlayParams::fetch(const ton_api::engine_validator_customOverlay& f) { +CustomOverlayParams CustomOverlayParams::fetch(const ton_api::engine_validator_customOverlay &f) { CustomOverlayParams c; c.name_ = f.name_; for (const auto &node : f.nodes_) { diff --git a/validator/full-node.h b/validator/full-node.h index 2d1fd7aaf8..9ab9c39a56 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -18,20 +18,18 @@ */ #pragma once -#include #include +#include -#include "ton/ton-types.h" - -#include "td/actor/actor.h" - +#include "adnl/adnl-ext-client.h" #include "adnl/adnl.h" -#include "rldp/rldp.h" -#include "rldp2/rldp.h" #include "dht/dht.h" #include "overlay/overlays.h" +#include "rldp/rldp.h" +#include "rldp2/rldp.h" +#include "td/actor/actor.h" +#include "ton/ton-types.h" #include "validator/validator.h" -#include "adnl/adnl-ext-client.h" namespace ton { @@ -85,9 +83,8 @@ class FullNode : public td::actor::Actor { virtual void add_collator_adnl_id(adnl::AdnlNodeIdShort id) = 0; virtual void del_collator_adnl_id(adnl::AdnlNodeIdShort id) = 0; - virtual void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, - td::uint32 expiry_at, td::uint32 max_size, - td::Promise promise) = 0; + virtual void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, + td::uint32 max_size, td::Promise promise) = 0; virtual void import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) = 0; @@ -106,7 +103,7 @@ class FullNode : public td::actor::Actor { virtual void set_validator_telemetry_filename(std::string value) = 0; virtual void import_fast_sync_member_certificate(adnl::AdnlNodeIdShort local_id, - overlay::OverlayMemberCertificate cert) = 0; + overlay::OverlayMemberCertificate cert) = 0; static constexpr td::uint32 max_block_size() { return 4 << 20; diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 482a916bbe..11ffabd94a 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -18,19 +18,20 @@ */ #pragma once -#include "full-node.h" #include "full-node-shard.h" +#include "full-node.h" //#include "ton-node-slave.h" -#include "interfaces/proof.h" -#include "interfaces/shard.h" -#include "full-node-private-overlay.hpp" -#include "full-node-fast-sync-overlays.hpp" - #include -#include #include +#include #include +#include "interfaces/proof.h" +#include "interfaces/shard.h" + +#include "full-node-fast-sync-overlays.hpp" +#include "full-node-private-overlay.hpp" + namespace ton { namespace validator { diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index 9999c4df71..310f687123 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -16,23 +16,21 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "accept-block.hpp" #include "adnl/utils.hpp" +#include "block/block-auto.h" +#include "block/block-parse.h" +#include "block/block.h" #include "interfaces/validator-manager.h" -#include "ton/ton-tl.hpp" #include "ton/ton-io.hpp" - -#include "fabric.h" -#include "top-shard-descr.hpp" - +#include "ton/ton-tl.hpp" +#include "validator/invariants.hpp" +#include "vm/boc.h" #include "vm/cells.h" #include "vm/cells/MerkleProof.h" -#include "vm/boc.h" -#include "block/block.h" -#include "block/block-parse.h" -#include "block/block-auto.h" -#include "validator/invariants.hpp" +#include "accept-block.hpp" +#include "fabric.h" +#include "top-shard-descr.hpp" namespace ton { @@ -56,8 +54,8 @@ AcceptBlockQuery::AcceptBlockQuery(BlockIdExt id, td::Ref data, std:: , manager_(manager) , promise_(std::move(promise)) , perf_timer_("acceptblock", 0.1, [manager](double duration) { - send_closure(manager, &ValidatorManager::add_perf_timer_stat, "acceptblock", duration); - }) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "acceptblock", duration); + }) { state_keep_old_hash_.clear(); state_old_hash_.clear(); state_hash_.clear(); @@ -76,8 +74,8 @@ AcceptBlockQuery::AcceptBlockQuery(AcceptBlockQuery::IsFake fake, BlockIdExt id, , manager_(manager) , promise_(std::move(promise)) , perf_timer_("acceptblock", 0.1, [manager](double duration) { - send_closure(manager, &ValidatorManager::add_perf_timer_stat, "acceptblock", duration); - }) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "acceptblock", duration); + }) { state_keep_old_hash_.clear(); state_old_hash_.clear(); state_hash_.clear(); @@ -93,8 +91,8 @@ AcceptBlockQuery::AcceptBlockQuery(ForceFork ffork, BlockIdExt id, td::Refinited_proof_link())) { finish_query(); return; - } + } if (data_.is_null()) { - td::actor::send_closure(manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, id_, [SelfId = actor_id(this)](td::Result R) { - if (R.is_ok()) { - td::actor::send_closure(SelfId, &AcceptBlockQuery::got_block_candidate_data, R.move_as_ok()); - } else { - td::actor::send_closure(SelfId, &AcceptBlockQuery::got_block_handle_cont); - } - }); + td::actor::send_closure(manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, id_, + [SelfId = actor_id(this)](td::Result R) { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &AcceptBlockQuery::got_block_candidate_data, + R.move_as_ok()); + } else { + td::actor::send_closure(SelfId, &AcceptBlockQuery::got_block_handle_cont); + } + }); } else { got_block_handle_cont(); } diff --git a/validator/impl/accept-block.hpp b/validator/impl/accept-block.hpp index e582650c18..61aa323989 100644 --- a/validator/impl/accept-block.hpp +++ b/validator/impl/accept-block.hpp @@ -18,13 +18,14 @@ */ #pragma once +#include "interfaces/validator-manager.h" #include "td/actor/actor.h" -#include "ton/ton-types.h" #include "ton/ton-shard.h" -#include "interfaces/validator-manager.h" -#include "validator-set.hpp" -#include "signature-set.hpp" +#include "ton/ton-types.h" + #include "shard.hpp" +#include "signature-set.hpp" +#include "validator-set.hpp" namespace ton { diff --git a/validator/impl/block.hpp b/validator/impl/block.hpp index 108c871d9c..4a06a98086 100644 --- a/validator/impl/block.hpp +++ b/validator/impl/block.hpp @@ -17,8 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "ton/ton-types.h" #include "interfaces/block.h" +#include "ton/ton-types.h" #include "vm/db/StaticBagOfCellsDb.h" namespace ton { diff --git a/validator/impl/candidates-buffer.hpp b/validator/impl/candidates-buffer.hpp index 5db1e51601..90914cff33 100644 --- a/validator/impl/candidates-buffer.hpp +++ b/validator/impl/candidates-buffer.hpp @@ -15,9 +15,9 @@ along with TON Blockchain Library. If not, see . */ #pragma once -#include "ton/ton-types.h" -#include "td/actor/actor.h" #include "interfaces/validator-manager.h" +#include "td/actor/actor.h" +#include "ton/ton-types.h" namespace ton::validator { diff --git a/validator/impl/check-proof.cpp b/validator/impl/check-proof.cpp index d223a63bba..077c4245bb 100644 --- a/validator/impl/check-proof.cpp +++ b/validator/impl/check-proof.cpp @@ -16,21 +16,20 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "check-proof.hpp" #include "adnl/utils.hpp" +#include "block/block-auto.h" +#include "block/block-parse.h" #include "ton/ton-io.hpp" #include "ton/ton-tl.hpp" -#include "fabric.h" -#include "signature-set.hpp" -#include "validator-set.hpp" -#include "shard.hpp" - -#include "block/block-parse.h" -#include "block/block-auto.h" +#include "validator/invariants.hpp" #include "vm/boc.h" #include "vm/cells/MerkleProof.h" -#include "validator/invariants.hpp" +#include "check-proof.hpp" +#include "fabric.h" +#include "shard.hpp" +#include "signature-set.hpp" +#include "validator-set.hpp" namespace ton { diff --git a/validator/impl/check-proof.hpp b/validator/impl/check-proof.hpp index 5c9a29b54b..b81c76387a 100644 --- a/validator/impl/check-proof.hpp +++ b/validator/impl/check-proof.hpp @@ -18,9 +18,9 @@ */ #pragma once -#include "td/actor/actor.h" #include "interfaces/block-handle.h" #include "interfaces/validator-manager.h" +#include "td/actor/actor.h" namespace ton { @@ -49,8 +49,8 @@ class CheckProof : public td::actor::Actor { , promise_(std::move(promise)) , skip_check_signatures_(skip_check_signatures) , perf_timer_("checkproof", 0.1, [manager](double duration) { - send_closure(manager, &ValidatorManager::add_perf_timer_stat, "checkproof", duration); - }) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "checkproof", duration); + }) { } CheckProof(BlockIdExt id, td::Ref proof, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, bool skip_check_signatures, td::Ref known_state) @@ -63,8 +63,8 @@ class CheckProof : public td::actor::Actor { , state_(std::move(known_state)) , skip_check_signatures_(skip_check_signatures) , perf_timer_("checkproof", 0.1, [manager](double duration) { - send_closure(manager, &ValidatorManager::add_perf_timer_stat, "checkproof", duration); - }) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "checkproof", duration); + }) { } CheckProof(BlockIdExt id, td::Ref proof_link, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise) @@ -75,8 +75,8 @@ class CheckProof : public td::actor::Actor { , timeout_(timeout) , promise_(std::move(promise)) , perf_timer_("checkproof", 0.1, [manager](double duration) { - send_closure(manager, &ValidatorManager::add_perf_timer_stat, "checkproof", duration); - }) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "checkproof", duration); + }) { } private: diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 83ae4370aa..31b9a1dd21 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -17,24 +17,26 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "block-parse.h" -#include "interfaces/validator-manager.h" -#include "shard.hpp" -#include "top-shard-descr.hpp" -#include "common/refcnt.hpp" -#include "vm/cells.h" -#include "vm/dict.h" -#include "block/mc-config.h" -#include "block/block.h" -#include "block/transaction.h" +#include +#include + #include "block/block-db.h" +#include "block/block.h" +#include "block/mc-config.h" #include "block/output-queue-merger.h" +#include "block/transaction.h" +#include "common/global-version.h" +#include "common/refcnt.hpp" +#include "interfaces/validator-manager.h" +#include "vm/cells.h" #include "vm/cells/MerkleProof.h" #include "vm/cells/MerkleUpdate.h" -#include -#include -#include "common/global-version.h" +#include "vm/dict.h" + +#include "block-parse.h" #include "fabric.h" +#include "shard.hpp" +#include "top-shard-descr.hpp" namespace ton { @@ -343,7 +345,8 @@ class Collator final : public td::actor::Actor { bool is_our_address(Ref addr_ref) const; bool is_our_address(ton::AccountIdPrefixFull addr_prefix) const; bool is_our_address(const ton::StdSmcAddress& addr) const; - void after_get_external_messages(td::Result, int>>> res, td::PerfLogAction token); + void after_get_external_messages(td::Result, int>>> res, + td::PerfLogAction token); td::Result register_external_message_cell(Ref ext_msg, const ExtMessage::Hash& ext_hash, int priority); // td::Result register_external_message(td::Slice ext_msg_boc); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 662bdff994..00f9cb5d10 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -16,27 +16,29 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "candidate-serializer.h" -#include "collator-impl.h" -#include "vm/boc.h" -#include "td/db/utils/BlobView.h" -#include "vm/db/StaticBagOfCellsDb.h" -#include "block/mc-config.h" -#include "block/block.h" -#include "block/block-parse.h" +#include +#include +#include + +#include "adnl/utils.hpp" #include "block/block-auto.h" -#include "vm/dict.h" +#include "block/block-parse.h" +#include "block/block.h" +#include "block/mc-config.h" #include "crypto/openssl/rand.hpp" +#include "td/db/utils/BlobView.h" +#include "td/utils/Random.h" #include "ton/ton-shard.h" -#include "adnl/utils.hpp" -#include -#include +#include "vm/boc.h" +#include "vm/db/StaticBagOfCellsDb.h" +#include "vm/dict.h" + +#include "candidate-serializer.h" +#include "collator-impl.h" #include "fabric.h" #include "storage-stat-cache.hpp" -#include "validator-set.hpp" #include "top-shard-descr.hpp" -#include -#include "td/utils/Random.h" +#include "validator-set.hpp" namespace ton { @@ -65,7 +67,7 @@ static constexpr int MAX_ATTEMPTS = 5; * @param promise The promise to return the result. */ Collator::Collator(CollateParams params, td::actor::ActorId manager, td::Timestamp timeout, - td::CancellationToken cancellation_token, td::Promise promise) + td::CancellationToken cancellation_token, td::Promise promise) : shard_(params.shard) , is_hardfork_(params.is_hardfork) , min_mc_block_id{params.min_masterchain_block_id} @@ -1067,7 +1069,7 @@ void Collator::got_neighbor_msg_queue(unsigned i, Ref res) { block_state_proofs_.emplace(block_id.root_hash, res->block_state_proof_); } - auto &neighbor_stats = stats_.neighbors.at(i); + auto& neighbor_stats = stats_.neighbors.at(i); neighbor_stats.shard = block_id.shard_full(); neighbor_stats.is_trivial = shard_intersects(block_id.shard_full(), shard_); neighbor_stats.is_local = res->is_local_; @@ -1080,9 +1082,7 @@ void Collator::got_neighbor_msg_queue(unsigned i, Ref res) { neighbor_proof_builders_.push_back(vm::MerkleProofBuilder{res->state_root_}); state_root = neighbor_proof_builders_.back().root(); if (full_collated_data_ && !block_id.is_masterchain()) { - neighbor_proof_builders_.back().set_cell_load_callback([&](const vm::LoadedCell& cell) { - on_cell_loaded(cell); - }); + neighbor_proof_builders_.back().set_cell_load_callback([&](const vm::LoadedCell& cell) { on_cell_loaded(cell); }); } } auto state = ShardStateQ::fetch(block_id, {}, state_root); @@ -1188,9 +1188,7 @@ bool Collator::unpack_merge_last_state() { // 1. prepare for creating a MerkleUpdate based on previous state state_usage_tree_ = std::make_shared(); if (full_collated_data_ && !is_masterchain()) { - state_usage_tree_->set_cell_load_callback([&](const vm::LoadedCell& cell) { - on_cell_loaded(cell); - }); + state_usage_tree_->set_cell_load_callback([&](const vm::LoadedCell& cell) { on_cell_loaded(cell); }); } prev_state_root_ = vm::UsageCell::create(prev_state_root_pure_, state_usage_tree_->root_ptr()); // 2. extract back slightly virtualized roots of the two original states @@ -1237,9 +1235,7 @@ bool Collator::unpack_last_state() { // prepare for creating a MerkleUpdate based on previous state state_usage_tree_ = std::make_shared(); if (full_collated_data_ && !is_masterchain()) { - state_usage_tree_->set_cell_load_callback([&](const vm::LoadedCell& cell) { - on_cell_loaded(cell); - }); + state_usage_tree_->set_cell_load_callback([&](const vm::LoadedCell& cell) { on_cell_loaded(cell); }); } prev_state_root_ = vm::UsageCell::create(prev_state_root_pure_, state_usage_tree_->root_ptr()); // unpack previous state @@ -2829,9 +2825,7 @@ bool Collator::init_account_storage_dict(block::Account& account) { } } dict.mpb = vm::MerkleProofBuilder(std::move(dict_root)); - dict.mpb.set_cell_load_callback([&](const vm::LoadedCell& cell) { - on_cell_loaded(cell); - }); + dict.mpb.set_cell_load_callback([&](const vm::LoadedCell& cell) { on_cell_loaded(cell); }); } auto S = account.init_account_storage_stat(dict.mpb.root()); if (S.is_error()) { @@ -2841,7 +2835,6 @@ bool Collator::init_account_storage_dict(block::Account& account) { return true; } - /** * Looks up an account in the Collator's account map. * @@ -3012,11 +3005,11 @@ bool Collator::process_account_storage_dict(block::Account& account) { switch (collated_data_stat.get_cell_status(cell->get_hash())) { case vm::ProofStorageStat::c_none: proof_size_diff += vm::ProofStorageStat::estimate_serialized_size(loaded_cell.data_cell); - break; + break; case vm::ProofStorageStat::c_prunned: proof_size_diff -= vm::ProofStorageStat::estimate_prunned_size(); - proof_size_diff += vm::ProofStorageStat::estimate_serialized_size(loaded_cell.data_cell); - break; + proof_size_diff += vm::ProofStorageStat::estimate_serialized_size(loaded_cell.data_cell); + break; case vm::ProofStorageStat::c_loaded: break; } @@ -3729,7 +3722,7 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R defer = true; } } else { - auto &x = unprocessed_deferred_messages_[src_addr]; + auto& x = unprocessed_deferred_messages_[src_addr]; CHECK(x > 0); if (--x == 0) { unprocessed_deferred_messages_.erase(src_addr); @@ -4105,7 +4098,7 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT << " enqueued_lt=" << enq_msg_descr.enqueued_lt_ << " has been already processed by us before, skipping"; // should we dequeue the message if it is ours (after a merge?) // (it should have been dequeued by out_msg_queue_cleanup() before) - auto &neighbor_stats = stats_.neighbors.at(src_nb_idx); + auto& neighbor_stats = stats_.neighbors.at(src_nb_idx); ++neighbor_stats.skipped_msgs; return true; } @@ -4200,7 +4193,7 @@ bool Collator::process_inbound_internal_messages() { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); auto kv = nb_out_msgs_->extract_cur(); CHECK(kv && kv->msg.not_null()); - auto &neighbor_stats = stats_.neighbors.at(kv->source); + auto& neighbor_stats = stats_.neighbors.at(kv->source); if (kv->limit_exceeded) { LOG(INFO) << "limit for imported messages is reached, stop processing inbound internal messages"; neighbor_stats.limit_reached = true; @@ -4249,7 +4242,7 @@ bool Collator::process_inbound_internal_messages() { if (verbosity > 1) { FLOG(INFO) { sb << "invalid inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() - << " msg="; + << " msg="; block::gen::t_EnqueuedMsg.print(sb, kv->msg); }; } @@ -6317,9 +6310,8 @@ bool Collator::create_collated_data() { } state_usage_tree_->set_use_mark_for_is_loaded(false); - Ref state_proof = vm::MerkleProof::generate(prev_state_root_, [&](const Ref& c) { - return !collated_data_stat.is_loaded(c->get_hash()); - }); + Ref state_proof = vm::MerkleProof::generate( + prev_state_root_, [&](const Ref& c) { return !collated_data_stat.is_loaded(c->get_hash()); }); if (state_proof.is_null()) { return fatal_error("cannot generate Merkle proof for previous state"); } @@ -6338,10 +6330,9 @@ bool Collator::create_collated_data() { } } // 4. Proofs for message queues - for (vm::MerkleProofBuilder &mpb : neighbor_proof_builders_) { - Ref proof = vm::MerkleProof::generate(mpb.original_root(), [&](const Ref& c) { - return !collated_data_stat.is_loaded(c->get_hash()); - }); + for (vm::MerkleProofBuilder& mpb : neighbor_proof_builders_) { + Ref proof = vm::MerkleProof::generate( + mpb.original_root(), [&](const Ref& c) { return !collated_data_stat.is_loaded(c->get_hash()); }); if (proof.is_null()) { return fatal_error("cannot generate Merkle proof for neighbor"); } diff --git a/validator/impl/collator.h b/validator/impl/collator.h index 5acc20e8d8..463e6d2674 100644 --- a/validator/impl/collator.h +++ b/validator/impl/collator.h @@ -17,10 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "block/block-db.h" #include "td/actor/actor.h" #include "ton/ton-types.h" #include "validator/validator.h" -#include "block/block-db.h" #include "vm/cells.h" namespace ton { diff --git a/validator/impl/config.cpp b/validator/impl/config.cpp index 1afac3156b..d355df669f 100644 --- a/validator/impl/config.cpp +++ b/validator/impl/config.cpp @@ -17,11 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "config.hpp" -#include "block/block-parse.h" #include "block/block-auto.h" +#include "block/block-parse.h" #include "vm/boc.h" #include "vm/cells/MerkleProof.h" + +#include "config.hpp" #include "validator-set.hpp" namespace ton { diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index 54acb105e8..0ab839b443 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -17,23 +17,25 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "external-message.hpp" -#include "collator-impl.h" -#include "vm/boc.h" -#include "block/block-parse.h" #include "block/block-auto.h" #include "block/block-db.h" -#include "fabric.h" +#include "block/block-parse.h" +#include "crypto/openssl/rand.hpp" #include "td/actor/actor.h" #include "td/utils/Random.h" -#include "crypto/openssl/rand.hpp" +#include "vm/boc.h" + +#include "collator-impl.h" +#include "external-message.hpp" +#include "fabric.h" namespace ton { namespace validator { using td::Ref; -ExtMessageQ::ExtMessageQ(td::BufferSlice data, td::Ref root, AccountIdPrefixFull addr_prefix, ton::WorkchainId wc, ton::StdSmcAddress addr) +ExtMessageQ::ExtMessageQ(td::BufferSlice data, td::Ref root, AccountIdPrefixFull addr_prefix, + ton::WorkchainId wc, ton::StdSmcAddress addr) : root_(std::move(root)), addr_prefix_(addr_prefix), data_(std::move(data)), wc_(wc), addr_(addr) { hash_ = block::compute_file_hash(data_); } @@ -79,7 +81,7 @@ td::Result> ExtMessageQ::create_ext_message(td::BufferSlice dat } ton::StdSmcAddress addr; ton::WorkchainId wc; - if(!block::tlb::t_MsgAddressInt.extract_std_address(info.dest, wc, addr)) { + if (!block::tlb::t_MsgAddressInt.extract_std_address(info.dest, wc, addr)) { return td::Status::Error(PSLICE() << "Can't parse destination address"); } @@ -125,49 +127,45 @@ void ExtMessageQ::run_message(td::Ref message, td::actor::ActorId msg_root, - std::unique_ptr config) { - - Ref old_mparams; - std::vector storage_prices_; - block::StoragePhaseConfig storage_phase_cfg_{&storage_prices_}; - td::BitArray<256> rand_seed_; - block::ComputePhaseConfig compute_phase_cfg_; - block::ActionPhaseConfig action_phase_cfg_; - block::SerializeConfig serialize_config_; - td::RefInt256 masterchain_create_fee, basechain_create_fee; - - auto fetch_res = block::FetchConfigParams::fetch_config_params( - *config, &old_mparams, &storage_prices_, &storage_phase_cfg_, &rand_seed_, &compute_phase_cfg_, - &action_phase_cfg_, &serialize_config_, &masterchain_create_fee, &basechain_create_fee, wc, utime); - if(fetch_res.is_error()) { - auto error = fetch_res.move_as_error(); - LOG(DEBUG) << "Cannot fetch config params: " << error.message(); - return error.move_as_error_prefix("Cannot fetch config params: "); - } - compute_phase_cfg_.libraries = std::make_unique(config->get_libraries_root(), 256); - compute_phase_cfg_.with_vm_log = true; - compute_phase_cfg_.stop_on_accept_message = true; - - auto res = - Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt, &storage_phase_cfg_, &compute_phase_cfg_, - &action_phase_cfg_, &serialize_config_, true, lt); - if(res.is_error()) { - auto error = res.move_as_error(); - LOG(DEBUG) << "Cannot run message on account: " << error.message(); - return error.move_as_error_prefix("Cannot run message on account: "); - } - std::unique_ptr trans = res.move_as_ok(); - - auto trans_root = trans->commit(*acc); - if (trans_root.is_null()) { - LOG(DEBUG) << "Cannot commit new transaction for smart contract"; - return td::Status::Error("Cannot commit new transaction for smart contract"); - } - return td::Status::OK(); +td::Status ExtMessageQ::run_message_on_account(ton::WorkchainId wc, block::Account* acc, UnixTime utime, LogicalTime lt, + td::Ref msg_root, std::unique_ptr config) { + Ref old_mparams; + std::vector storage_prices_; + block::StoragePhaseConfig storage_phase_cfg_{&storage_prices_}; + td::BitArray<256> rand_seed_; + block::ComputePhaseConfig compute_phase_cfg_; + block::ActionPhaseConfig action_phase_cfg_; + block::SerializeConfig serialize_config_; + td::RefInt256 masterchain_create_fee, basechain_create_fee; + + auto fetch_res = block::FetchConfigParams::fetch_config_params( + *config, &old_mparams, &storage_prices_, &storage_phase_cfg_, &rand_seed_, &compute_phase_cfg_, + &action_phase_cfg_, &serialize_config_, &masterchain_create_fee, &basechain_create_fee, wc, utime); + if (fetch_res.is_error()) { + auto error = fetch_res.move_as_error(); + LOG(DEBUG) << "Cannot fetch config params: " << error.message(); + return error.move_as_error_prefix("Cannot fetch config params: "); + } + compute_phase_cfg_.libraries = std::make_unique(config->get_libraries_root(), 256); + compute_phase_cfg_.with_vm_log = true; + compute_phase_cfg_.stop_on_accept_message = true; + + auto res = + Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt, &storage_phase_cfg_, &compute_phase_cfg_, + &action_phase_cfg_, &serialize_config_, true, lt); + if (res.is_error()) { + auto error = res.move_as_error(); + LOG(DEBUG) << "Cannot run message on account: " << error.message(); + return error.move_as_error_prefix("Cannot run message on account: "); + } + std::unique_ptr trans = res.move_as_ok(); + + auto trans_root = trans->commit(*acc); + if (trans_root.is_null()) { + LOG(DEBUG) << "Cannot commit new transaction for smart contract"; + return td::Status::Error("Cannot commit new transaction for smart contract"); + } + return td::Status::OK(); } } // namespace validator diff --git a/validator/impl/external-message.hpp b/validator/impl/external-message.hpp index ad7ecc74ea..fee2cef977 100644 --- a/validator/impl/external-message.hpp +++ b/validator/impl/external-message.hpp @@ -18,11 +18,11 @@ */ #pragma once -#include "interfaces/validator-manager.h" -#include "validator/interfaces/external-message.h" -#include "auto/tl/ton_api.h" #include "adnl/utils.hpp" +#include "auto/tl/ton_api.h" #include "block/transaction.h" +#include "interfaces/validator-manager.h" +#include "validator/interfaces/external-message.h" namespace ton { @@ -63,11 +63,8 @@ class ExtMessageQ : public ExtMessage { block::SizeLimitsConfig::ExtMsgLimits limits); static void run_message(td::Ref message, td::actor::ActorId manager, td::Promise> promise); - static td::Status run_message_on_account(ton::WorkchainId wc, - block::Account* acc, - UnixTime utime, LogicalTime lt, - td::Ref msg_root, - std::unique_ptr config); + static td::Status run_message_on_account(ton::WorkchainId wc, block::Account* acc, UnixTime utime, LogicalTime lt, + td::Ref msg_root, std::unique_ptr config); }; } // namespace validator diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index af171c3c9e..7b5b439f4b 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -16,25 +16,26 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "fabric.h" -#include "collator-impl.h" -#include "validator/db/rootdb.hpp" +#include "ton/ton-io.hpp" #include "validator/block-handle.hpp" -#include "apply-block.hpp" +#include "validator/db/rootdb.hpp" +#include "validator/fabric.h" + #include "accept-block.hpp" -#include "shard.hpp" +#include "apply-block.hpp" #include "block.hpp" -#include "proof.hpp" -#include "signature-set.hpp" +#include "check-proof.hpp" +#include "collator-impl.h" #include "external-message.hpp" +#include "fabric.h" #include "ihr-message.hpp" -#include "validate-query.hpp" -#include "check-proof.hpp" -#include "top-shard-descr.hpp" -#include "ton/ton-io.hpp" -#include "liteserver.hpp" -#include "validator/fabric.h" #include "liteserver-cache.hpp" +#include "liteserver.hpp" +#include "proof.hpp" +#include "shard.hpp" +#include "signature-set.hpp" +#include "top-shard-descr.hpp" +#include "validate-query.hpp" namespace ton { @@ -113,8 +114,7 @@ td::Ref create_signature_set(std::vector sig_ return td::Ref{true, std::move(sig_set)}; } -td::Result> create_ext_message(td::BufferSlice data, - block::SizeLimitsConfig::ExtMsgLimits limits) { +td::Result> create_ext_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits) { TRY_RESULT(res, ExtMessageQ::create_ext_message(std::move(data), limits)); return std::move(res); } @@ -230,8 +230,10 @@ void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, - td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise) { +void run_fetch_account_state( + WorkchainId wc, StdSmcAddress addr, td::actor::ActorId manager, + td::Promise, UnixTime, LogicalTime, std::unique_ptr>> + promise) { LiteQuery::fetch_account_state(wc, addr, std::move(manager), std::move(promise)); } diff --git a/validator/impl/ihr-message.cpp b/validator/impl/ihr-message.cpp index 7c15ac566c..f97f61cabb 100644 --- a/validator/impl/ihr-message.cpp +++ b/validator/impl/ihr-message.cpp @@ -16,13 +16,14 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "ihr-message.hpp" -#include "vm/boc.h" -#include "block/block-parse.h" #include "block/block-auto.h" #include "block/block-db.h" +#include "block/block-parse.h" +#include "vm/boc.h" #include "vm/cells/MerkleProof.h" +#include "ihr-message.hpp" + namespace ton { namespace validator { diff --git a/validator/impl/ihr-message.hpp b/validator/impl/ihr-message.hpp index 0ec6060037..d1b8b7fb3e 100644 --- a/validator/impl/ihr-message.hpp +++ b/validator/impl/ihr-message.hpp @@ -18,9 +18,9 @@ */ #pragma once -#include "validator/interfaces/ihr-message.h" -#include "auto/tl/ton_api.h" #include "adnl/utils.hpp" +#include "auto/tl/ton_api.h" +#include "validator/interfaces/ihr-message.h" namespace ton { diff --git a/validator/impl/liteserver-cache.hpp b/validator/impl/liteserver-cache.hpp index 20beca9189..07d78bef5c 100644 --- a/validator/impl/liteserver-cache.hpp +++ b/validator/impl/liteserver-cache.hpp @@ -16,9 +16,10 @@ */ #pragma once -#include "interfaces/liteserver.h" #include +#include "interfaces/liteserver.h" + namespace ton::validator { class LiteServerCacheImpl : public LiteServerCache { diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 14e0414642..8850f2a3b5 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -16,34 +16,36 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "liteserver.hpp" +#include + +#include "adnl/utils.hpp" +#include "auto/tl/lite_api.h" +#include "auto/tl/lite_api.hpp" +#include "block/block-auto.h" +#include "block/block-parse.h" +#include "block/block.h" +#include "block/check-proof.h" +#include "td/actor/MultiPromise.h" +#include "td/utils/Random.h" #include "td/utils/Slice.h" #include "td/utils/common.h" #include "td/utils/crypto.h" #include "td/utils/overloaded.h" -#include "auto/tl/lite_api.h" -#include "auto/tl/lite_api.hpp" -#include "adnl/utils.hpp" -#include "ton/lite-tl.hpp" #include "tl-utils/lite-utils.hpp" -#include "td/utils/Random.h" -#include "vm/boc.h" #include "tl/tlblib.hpp" -#include "block/block.h" -#include "block/block-parse.h" -#include "block/block-auto.h" -#include "block/check-proof.h" -#include "vm/dict.h" +#include "ton/lite-tl.hpp" +#include "vm/boc.h" #include "vm/cells/MerkleProof.h" -#include "vm/vm.h" +#include "vm/dict.h" #include "vm/memo.h" +#include "vm/vm.h" + +#include "collator-impl.h" +#include "fabric.h" +#include "liteserver.hpp" #include "shard.hpp" -#include "validator-set.hpp" #include "signature-set.hpp" -#include "fabric.h" -#include -#include "td/actor/MultiPromise.h" -#include "collator-impl.h" +#include "validator-set.hpp" namespace ton { @@ -56,15 +58,16 @@ td::int32 get_tl_tag(td::Slice slice) { } void LiteQuery::run_query(td::BufferSlice data, td::actor::ActorId manager, - td::actor::ActorId cache, - td::Promise promise) { + td::actor::ActorId cache, td::Promise promise) { td::actor::create_actor("litequery", std::move(data), std::move(manager), std::move(cache), std::move(promise)) .release(); } -void LiteQuery::fetch_account_state(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, - td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise) { +void LiteQuery::fetch_account_state( + WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise, UnixTime, LogicalTime, std::unique_ptr>> + promise) { td::actor::create_actor("litequery", wc, acc_addr, std::move(manager), std::move(promise)).release(); } @@ -74,8 +77,9 @@ LiteQuery::LiteQuery(td::BufferSlice data, td::actor::ActorId timeout_ = td::Timestamp::in(default_timeout_msec * 0.001); } -LiteQuery::LiteQuery(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, - td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise) +LiteQuery::LiteQuery( + WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise, UnixTime, LogicalTime, std::unique_ptr>> promise) : manager_(std::move(manager)), acc_state_promise_(std::move(promise)), acc_workchain_(wc), acc_addr_(acc_addr) { timeout_ = td::Timestamp::in(default_timeout_msec * 0.001); } @@ -157,32 +161,30 @@ void LiteQuery::start_up() { use_cache_ = use_cache(); if (use_cache_) { cache_key_ = td::sha256_bits256(query_); - td::actor::send_closure( - cache_, &LiteServerCache::lookup, cache_key_, [SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &LiteQuery::perform); - } else { - td::actor::send_closure(SelfId, &LiteQuery::finish_query, R.move_as_ok(), true); - } - }); + td::actor::send_closure(cache_, &LiteServerCache::lookup, cache_key_, + [SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &LiteQuery::perform); + } else { + td::actor::send_closure(SelfId, &LiteQuery::finish_query, R.move_as_ok(), true); + } + }); } else { perform(); } } -bool LiteQuery::use_cache() { +bool LiteQuery::use_cache() { if (cache_.empty()) { return false; } bool use = false; - lite_api::downcast_call( - *query_obj_, - td::overloaded( - [&](lite_api::liteServer_runSmcMethod& q) { - // wc=-1, seqno=-1 means "use latest mc block" - use = q.id_->workchain_ != masterchainId || q.id_->seqno_ != -1; - }, - [&](auto& obj) { use = false; })); + lite_api::downcast_call(*query_obj_, td::overloaded( + [&](lite_api::liteServer_runSmcMethod& q) { + // wc=-1, seqno=-1 means "use latest mc block" + use = q.id_->workchain_ != masterchainId || q.id_->seqno_ != -1; + }, + [&](auto& obj) { use = false; })); return use; } @@ -230,7 +232,8 @@ void LiteQuery::perform() { this->perform_lookupBlock(ton::create_block_id_simple(q.id_), q.mode_, q.lt_, q.utime_); }, [&](lite_api::liteServer_lookupBlockWithProof& q) { - this->perform_lookupBlockWithProof(ton::create_block_id_simple(q.id_), ton::create_block_id(q.mc_block_id_), q.mode_, q.lt_, q.utime_); + this->perform_lookupBlockWithProof(ton::create_block_id_simple(q.id_), ton::create_block_id(q.mc_block_id_), + q.mode_, q.lt_, q.utime_); }, [&](lite_api::liteServer_listBlockTransactions& q) { this->perform_listBlockTransactions(ton::create_block_id(q.id_), q.mode_, q.count_, @@ -239,8 +242,8 @@ void LiteQuery::perform() { }, [&](lite_api::liteServer_listBlockTransactionsExt& q) { this->perform_listBlockTransactionsExt(ton::create_block_id(q.id_), q.mode_, q.count_, - (q.mode_ & 128) ? q.after_->account_ : td::Bits256::zero(), - static_cast((q.mode_ & 128) ? (q.after_->lt_) : 0)); + (q.mode_ & 128) ? q.after_->account_ : td::Bits256::zero(), + static_cast((q.mode_ & 128) ? (q.after_->lt_) : 0)); }, [&](lite_api::liteServer_getConfigParams& q) { this->perform_getConfigParams(ton::create_block_id(q.id_), (q.mode_ & 0xffff) | 0x10000, q.param_list_); @@ -262,15 +265,11 @@ void LiteQuery::perform() { this->perform_runSmcMethod(ton::create_block_id(q.id_), static_cast(q.account_->workchain_), q.account_->id_, q.mode_, q.method_id_, std::move(q.params_)); }, - [&](lite_api::liteServer_getLibraries& q) { - this->perform_getLibraries(q.library_list_); - }, + [&](lite_api::liteServer_getLibraries& q) { this->perform_getLibraries(q.library_list_); }, [&](lite_api::liteServer_getLibrariesWithProof& q) { this->perform_getLibrariesWithProof(ton::create_block_id(q.id_), q.mode_, q.library_list_); }, - [&](lite_api::liteServer_getShardBlockProof& q) { - this->perform_getShardBlockProof(create_block_id(q.id_)); - }, + [&](lite_api::liteServer_getShardBlockProof& q) { this->perform_getShardBlockProof(create_block_id(q.id_)); }, [&](lite_api::liteServer_nonfinal_getCandidate& q) { this->perform_nonfinal_getCandidate(q.id_->creator_, create_block_id(q.id_->block_id_), q.id_->collated_data_hash_); @@ -314,22 +313,22 @@ void LiteQuery::perform_getMasterchainInfo(int mode) { fatal_error("unsupported getMasterchainInfo mode"); return; } - td::actor::send_closure_later( - manager_, &ton::validator::ValidatorManager::get_last_liteserver_state_block, - [Self = actor_id(this), return_state = bool(acc_state_promise_), mode](td::Result, BlockIdExt>> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - } else { - auto pair = res.move_as_ok(); - auto func = return_state ? &LiteQuery::gotMasterchainInfoForAccountState : &LiteQuery::continue_getMasterchainInfo; - td::actor::send_closure_later(Self, func, std::move(pair.first), - pair.second, mode); - } - }); + td::actor::send_closure_later(manager_, &ton::validator::ValidatorManager::get_last_liteserver_state_block, + [Self = actor_id(this), return_state = bool(acc_state_promise_), + mode](td::Result, BlockIdExt>> res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + auto pair = res.move_as_ok(); + auto func = return_state ? &LiteQuery::gotMasterchainInfoForAccountState + : &LiteQuery::continue_getMasterchainInfo; + td::actor::send_closure_later(Self, func, std::move(pair.first), pair.second, mode); + } + }); } void LiteQuery::gotMasterchainInfoForAccountState(Ref mc_state, BlockIdExt blkid, - int mode) { + int mode) { perform_getAccountState(blkid, acc_workchain_, acc_addr_, 0x80000000); } @@ -775,8 +774,7 @@ bool LiteQuery::request_zero_state(BlockIdExt blkid) { return; } td::actor::send_closure_later( - manager, &ValidatorManager::get_zero_state, blkid, - [=](td::Result res) { + manager, &ValidatorManager::get_zero_state, blkid, [=](td::Result res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load zerostate of "s + blkid.to_str() + " : ")); @@ -895,22 +893,24 @@ void LiteQuery::perform_getLibraries(std::vector library_list) { LOG(INFO) << "too many libraries requested, returning only first 16"; library_list.resize(16); } - sort( library_list.begin(), library_list.end() ); - library_list.erase( unique( library_list.begin(), library_list.end() ), library_list.end() ); + sort(library_list.begin(), library_list.end()); + library_list.erase(unique(library_list.begin(), library_list.end()), library_list.end()); td::actor::send_closure_later( manager_, &ton::validator::ValidatorManager::get_last_liteserver_state_block, - [Self = actor_id(this), library_list](td::Result, BlockIdExt>> res) -> void { + [Self = actor_id(this), + library_list](td::Result, BlockIdExt>> res) -> void { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { auto pair = res.move_as_ok(); - td::actor::send_closure_later(Self, &LiteQuery::continue_getLibraries, std::move(pair.first), - pair.second, library_list); + td::actor::send_closure_later(Self, &LiteQuery::continue_getLibraries, std::move(pair.first), pair.second, + library_list); } }); } -void LiteQuery::continue_getLibraries(Ref mc_state, BlockIdExt blkid, std::vector library_list) { +void LiteQuery::continue_getLibraries(Ref mc_state, BlockIdExt blkid, + std::vector library_list) { LOG(INFO) << "obtained last masterchain block = " << blkid.to_str(); base_blk_id_ = blkid; CHECK(mc_state.not_null()); @@ -931,7 +931,7 @@ void LiteQuery::continue_getLibraries(Ref mc_s LOG(INFO) << "\n" << os.str(); auto lib_dict = std::make_unique(config->get_libraries_root(), 256); - for (auto k: *lib_dict) { + for (auto k : *lib_dict) { std::ostringstream oss; k.second->print_rec(oss); LOG(INFO) << "library " << k.first.to_hex(256) << ": \n" << oss.str(); @@ -963,8 +963,8 @@ void LiteQuery::perform_getLibrariesWithProof(BlockIdExt blkid, int mode, std::v LOG(INFO) << "too many libraries requested, returning only first 16"; library_list.resize(16); } - sort( library_list.begin(), library_list.end() ); - library_list.erase( unique( library_list.begin(), library_list.end() ), library_list.end() ); + sort(library_list.begin(), library_list.end()); + library_list.erase(unique(library_list.begin(), library_list.end()), library_list.end()); set_continuation([this, library_list, mode]() -> void { continue_getLibrariesWithProof(library_list, mode); }); request_mc_block_data_state(blkid); @@ -1004,8 +1004,9 @@ void LiteQuery::continue_getLibrariesWithProof(std::vector library_ // include first 16 publishers in the proof auto publishers_dict = vm::Dictionary{vm::DictNonEmpty(), libdescr.publishers, 256}; auto iter = publishers_dict.begin(); - constexpr int max_publishers = 15; // set to 15 because publishers_dict.begin() counts as the first visit - for (int i = 0; i < max_publishers && iter != publishers_dict.end(); ++i, ++iter) {} + constexpr int max_publishers = 15; // set to 15 because publishers_dict.begin() counts as the first visit + for (int i = 0; i < max_publishers && iter != publishers_dict.end(); ++i, ++iter) { + } } result_hashes.push_back(hash); @@ -1031,7 +1032,7 @@ void LiteQuery::continue_getLibrariesWithProof(std::vector library_ } if (!libdescr.lib->get_hash().bits().equals(hash.bits(), 256)) { LOG(ERROR) << "public library hash mismatch: expected " << hash.to_hex() << " , found " - << libdescr.lib->get_hash().to_hex(); + << libdescr.lib->get_hash().to_hex(); continue; } td::BufferSlice libdata; @@ -1046,8 +1047,9 @@ void LiteQuery::continue_getLibrariesWithProof(std::vector library_ result.push_back(ton::create_tl_object(hash, std::move(libdata))); } - auto b = ton::create_serialize_tl_object(ton::create_tl_lite_block_id(base_blk_id_), mode, std::move(result), - state_proof_boc.move_as_ok(), data_proof_boc.move_as_ok()); + auto b = ton::create_serialize_tl_object( + ton::create_tl_lite_block_id(base_blk_id_), mode, std::move(result), state_proof_boc.move_as_ok(), + data_proof_boc.move_as_ok()); finish_query(std::move(b)); } @@ -1252,7 +1254,8 @@ bool LiteQuery::make_shard_info_proof(Ref& proof, BlockIdExt& blkid, A return true; } -bool LiteQuery::make_ancestor_block_proof(Ref& proof, Ref mc_state, const BlockIdExt& old_blkid) { +bool LiteQuery::make_ancestor_block_proof(Ref& proof, Ref mc_state, + const BlockIdExt& old_blkid) { vm::MerkleProofBuilder mpb{mc_state->root_cell()}; auto rconfig = block::ConfigInfo::extract_config(mpb.root(), mc_state->get_block_id(), block::ConfigInfo::needPrevBlocks); @@ -1327,9 +1330,8 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { return; } auto rconfig = config.move_as_ok(); - acc_state_promise_.set_value(std::make_tuple( - std::move(acc_csr), sstate.gen_utime, sstate.gen_lt, std::move(rconfig) - )); + acc_state_promise_.set_value( + std::make_tuple(std::move(acc_csr), sstate.gen_utime, sstate.gen_lt, std::move(rconfig))); return; } @@ -2035,7 +2037,8 @@ void LiteQuery::continue_getAllShardsInfo() { finish_query(std::move(b)); } -void LiteQuery::perform_lookupBlockWithProof(BlockId blkid, BlockIdExt mc_blkid, int mode, LogicalTime lt, UnixTime utime) { +void LiteQuery::perform_lookupBlockWithProof(BlockId blkid, BlockIdExt mc_blkid, int mode, LogicalTime lt, + UnixTime utime) { if (!((1 << (mode & 7)) & 0x16)) { fatal_error("exactly one of mode.0, mode.1 and mode.2 bits must be set"); return; @@ -2059,30 +2062,33 @@ void LiteQuery::perform_lookupBlockWithProof(BlockId blkid, BlockIdExt mc_blkid, << lt << ", " << utime << ") liteserver query"; ton::AccountIdPrefixFull pfx{blkid.workchain, blkid.shard}; - auto P = td::PromiseCreator::lambda( - [Self = actor_id(this), mc_blkid, manager = manager_, pfx](td::Result res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - return; - } - auto handle = res.move_as_ok(); - if (!handle->inited_masterchain_ref_block()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, td::Status::Error("block doesn't have masterchain ref")); - return; - } - if (handle->masterchain_ref_block() > mc_blkid.seqno()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, td::Status::Error("specified mc block is older than block's masterchain ref")); - return; - } - LOG(DEBUG) << "requesting data for block " << handle->id().to_str(); - td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle, - [Self, mc_ref_blkid = handle->masterchain_ref_block(), pfx](td::Result> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - } else { - td::actor::send_closure_later(Self, &LiteQuery::continue_lookupBlockWithProof_getHeaderProof, res.move_as_ok(), pfx, mc_ref_blkid); - } - }); + auto P = td::PromiseCreator::lambda([Self = actor_id(this), mc_blkid, manager = manager_, + pfx](td::Result res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + return; + } + auto handle = res.move_as_ok(); + if (!handle->inited_masterchain_ref_block()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, td::Status::Error("block doesn't have masterchain ref")); + return; + } + if (handle->masterchain_ref_block() > mc_blkid.seqno()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, + td::Status::Error("specified mc block is older than block's masterchain ref")); + return; + } + LOG(DEBUG) << "requesting data for block " << handle->id().to_str(); + td::actor::send_closure_later( + manager, &ValidatorManager::get_block_data_from_db, handle, + [Self, mc_ref_blkid = handle->masterchain_ref_block(), pfx](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_lookupBlockWithProof_getHeaderProof, + res.move_as_ok(), pfx, mc_ref_blkid); + } + }); }); if (mode & 2) { @@ -2096,7 +2102,9 @@ void LiteQuery::perform_lookupBlockWithProof(BlockId blkid, BlockIdExt mc_blkid, } } -void LiteQuery::continue_lookupBlockWithProof_getHeaderProof(Ref block, AccountIdPrefixFull req_prefix, BlockSeqno masterchain_ref_seqno) { +void LiteQuery::continue_lookupBlockWithProof_getHeaderProof(Ref block, + AccountIdPrefixFull req_prefix, + BlockSeqno masterchain_ref_seqno) { blk_id_ = block->block_id(); LOG(INFO) << "obtained data for getBlockHeader(" << blk_id_.to_str() << ", " << mode_ << ")"; CHECK(block.not_null()); @@ -2131,26 +2139,30 @@ void LiteQuery::continue_lookupBlockWithProof_getHeaderProof(Ref R) mutable { + get_block_handle_checked(prev_blkid, [Self = actor_id(this), masterchain_ref_seqno, + manager = manager_](td::Result R) mutable { if (R.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); return; } - td::actor::send_closure(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), + td::actor::send_closure(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), [Self, masterchain_ref_seqno](td::Result> res) mutable { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - return; - } - td::actor::send_closure(Self, &LiteQuery::continue_lookupBlockWithProof_gotPrevBlockData, res.move_as_ok(), masterchain_ref_seqno); - }); + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + return; + } + td::actor::send_closure(Self, + &LiteQuery::continue_lookupBlockWithProof_gotPrevBlockData, + res.move_as_ok(), masterchain_ref_seqno); + }); }); } else { continue_lookupBlockWithProof_gotPrevBlockData(Ref(), masterchain_ref_seqno); } } -void LiteQuery::continue_lookupBlockWithProof_gotPrevBlockData(Ref prev_block, BlockSeqno masterchain_ref_seqno) { +void LiteQuery::continue_lookupBlockWithProof_gotPrevBlockData(Ref prev_block, + BlockSeqno masterchain_ref_seqno) { if (prev_block.not_null()) { CHECK(prev_block.not_null()); if (prev_block->root_cell().is_null()) { @@ -2174,29 +2186,33 @@ void LiteQuery::continue_lookupBlockWithProof_gotPrevBlockData(Ref pr if (!blk_id_.is_masterchain()) { ton::AccountIdPrefixFull pfx{ton::masterchainId, ton::shardIdAll}; - td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_seqno_from_db, pfx, masterchain_ref_seqno, + td::actor::send_closure_later( + manager_, &ValidatorManager::get_block_by_seqno_from_db, pfx, masterchain_ref_seqno, [manager = manager_, Self = actor_id(this)](td::Result R) mutable { - if (R.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); - return; - } - td::actor::send_closure(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), - [Self](td::Result> res) mutable { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - return; - } - td::actor::send_closure(Self, &LiteQuery::continue_lookupBlockWithProof_buildProofLinks, res.move_as_ok(), std::vector>>()); - }); - }); + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), + [Self](td::Result> res) mutable { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + return; + } + td::actor::send_closure( + Self, &LiteQuery::continue_lookupBlockWithProof_buildProofLinks, + res.move_as_ok(), std::vector>>()); + }); + }); } else { base_blk_id_alt_ = blk_id_; - td::actor::send_closure(actor_id(this), &LiteQuery::continue_lookupBlockWithProof_getClientMcBlockDataState, std::vector>>()); + td::actor::send_closure(actor_id(this), &LiteQuery::continue_lookupBlockWithProof_getClientMcBlockDataState, + std::vector>>()); } } -void LiteQuery::continue_lookupBlockWithProof_buildProofLinks(td::Ref cur_block, - std::vector>> result) { +void LiteQuery::continue_lookupBlockWithProof_buildProofLinks( + td::Ref cur_block, std::vector>> result) { BlockIdExt cur_id = cur_block->block_id(); BlockIdExt prev_id; vm::MerkleProofBuilder mpb{cur_block->root_cell()}; @@ -2272,19 +2288,20 @@ void LiteQuery::continue_lookupBlockWithProof_buildProofLinks(td::Ref td::actor::send_closure_later(Self, &LiteQuery::continue_lookupBlockWithProof_buildProofLinks, R.move_as_ok(), std::move(result)); } - }); + }); } -void LiteQuery::continue_lookupBlockWithProof_getClientMcBlockDataState(std::vector>> links) { - set_continuation([this, links = std::move(links)]() -> void { - continue_lookupBlockWithProof_getMcBlockPrev(std::move(links)); - }); +void LiteQuery::continue_lookupBlockWithProof_getClientMcBlockDataState( + std::vector>> links) { + set_continuation( + [this, links = std::move(links)]() -> void { continue_lookupBlockWithProof_getMcBlockPrev(std::move(links)); }); request_mc_block_data_state(base_blk_id_); } -void LiteQuery::continue_lookupBlockWithProof_getMcBlockPrev(std::vector>> links) { +void LiteQuery::continue_lookupBlockWithProof_getMcBlockPrev( + std::vector>> links) { td::BufferSlice mc_state_proof_buf, client_mc_blk_proof_buf; - + if (base_blk_id_alt_ != base_blk_id_) { vm::MerkleProofBuilder mpb{mc_state_->root_cell()}; auto prev_blocks_dict = block::get_prev_blocks_dict(mpb.root()); @@ -2323,17 +2340,17 @@ void LiteQuery::continue_lookupBlockWithProof_getMcBlockPrev(std::vector(create_tl_lite_block_id(p.first), prev_block_proof.move_as_ok())); + links_res.push_back(create_tl_object(create_tl_lite_block_id(p.first), + prev_block_proof.move_as_ok())); } - auto b = ton::create_serialize_tl_object(ton::create_tl_lite_block_id(blk_id_), - mode_, ton::create_tl_lite_block_id(base_blk_id_alt_), std::move(mc_state_proof_buf), std::move(client_mc_blk_proof_buf), - std::move(links_res), std::move(lookup_header_proof_), std::move(lookup_prev_header_proof_)); + auto b = ton::create_serialize_tl_object( + ton::create_tl_lite_block_id(blk_id_), mode_, ton::create_tl_lite_block_id(base_blk_id_alt_), + std::move(mc_state_proof_buf), std::move(client_mc_blk_proof_buf), std::move(links_res), + std::move(lookup_header_proof_), std::move(lookup_prev_header_proof_)); finish_query(std::move(b)); } - void LiteQuery::perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, UnixTime utime) { if (!((1 << (mode & 7)) & 0x16)) { fatal_error("exactly one of mode.0, mode.1 and mode.2 bits must be set"); @@ -2368,14 +2385,13 @@ void LiteQuery::perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, Uni ton::AccountIdPrefixFull pfx{blkid.workchain, blkid.shard}; if (mode & 2) { - td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_lt_for_litequery, pfx, lt, - std::move(P)); + td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_lt_for_litequery, pfx, lt, std::move(P)); } else if (mode & 4) { td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_unix_time_for_litequery, pfx, utime, std::move(P)); } else { - td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_seqno_for_litequery, pfx, - blkid.seqno, std::move(P)); + td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_seqno_for_litequery, pfx, blkid.seqno, + std::move(P)); } } @@ -2417,7 +2433,8 @@ static td::Result> get_i } block::tlb::MsgEnvelope::Record_std env; if (!block::tlb::unpack_cell(std::move(msg_env), env)) { - return td::Status::Error(PSTRING() << "failed to unpack MsgEnvelope for message with hash " << in_msg_hash.to_hex()); + return td::Status::Error(PSTRING() << "failed to unpack MsgEnvelope for message with hash " + << in_msg_hash.to_hex()); } if (!env.metadata) { return nullptr; @@ -2539,7 +2556,8 @@ void LiteQuery::finish_listBlockTransactions(int mode, int req_count) { finish_query(std::move(b)); } -void LiteQuery::perform_listBlockTransactionsExt(BlockIdExt blkid, int mode, int count, Bits256 account, LogicalTime lt) { +void LiteQuery::perform_listBlockTransactionsExt(BlockIdExt blkid, int mode, int count, Bits256 account, + LogicalTime lt) { LOG(INFO) << "started a listBlockTransactionsExt(" << blkid.to_str() << ", " << mode << ", " << count << ", " << account.to_hex() << ", " << lt << ") liteserver query"; base_blk_id_ = blkid; @@ -2686,7 +2704,7 @@ void LiteQuery::finish_listBlockTransactionsExt(int mode, int req_count) { fatal_error(res.move_as_error()); return; } - + auto b = ton::create_serialize_tl_object( ton::create_tl_lite_block_id(base_blk_id_), req_count, !eof, res.move_as_ok(), std::move(proof_data)); LOG(INFO) << "listBlockTransactionsExt() query completed"; @@ -3690,8 +3708,8 @@ void LiteQuery::finish_getDispatchQueueMessages(StdSmcAddress addr, LogicalTime void LiteQuery::perform_nonfinal_getCandidate(td::Bits256 source, BlockIdExt blkid, td::Bits256 collated_data_hash) { LOG(INFO) << "started a nonfinal.getCandidate liteserver query"; td::actor::send_closure_later( - manager_, &ValidatorManager::get_block_candidate_for_litequery, PublicKey{pubkeys::Ed25519{source}}, blkid, collated_data_hash, - [Self = actor_id(this)](td::Result R) { + manager_, &ValidatorManager::get_block_candidate_for_litequery, PublicKey{pubkeys::Ed25519{source}}, blkid, + collated_data_hash, [Self = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); } else { diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index afeabb9c01..05b7900b3a 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -17,17 +17,18 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "ton/ton-types.h" -#include "td/actor/actor.h" -#include "td/utils/Time.h" +#include "auto/tl/lite_api.h" +#include "block/block-auto.h" #include "interfaces/block-handle.h" -#include "interfaces/validator-manager.h" #include "interfaces/shard.h" +#include "interfaces/validator-manager.h" +#include "td/actor/actor.h" +#include "td/utils/Time.h" +#include "ton/ton-types.h" + #include "block.hpp" -#include "shard.hpp" #include "proof.hpp" -#include "block/block-auto.h" -#include "auto/tl/lite_api.h" +#include "shard.hpp" namespace ton { @@ -41,7 +42,8 @@ class LiteQuery : public td::actor::Actor { td::Timestamp timeout_; td::Promise promise_; - td::Promise,UnixTime,LogicalTime,std::unique_ptr>> acc_state_promise_; + td::Promise, UnixTime, LogicalTime, std::unique_ptr>> + acc_state_promise_; tl_object_ptr query_obj_; bool use_cache_{false}; @@ -84,13 +86,16 @@ class LiteQuery : public td::actor::Actor { }; // version 1.1; +1 = build block proof chains, +2 = masterchainInfoExt, +4 = runSmcMethod LiteQuery(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise); - LiteQuery(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, - td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise); + LiteQuery(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise, UnixTime, LogicalTime, std::unique_ptr>> + promise); static void run_query(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise); - static void fetch_account_state(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, - td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise); + static void fetch_account_state( + WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise, UnixTime, LogicalTime, std::unique_ptr>> + promise); private: bool fatal_error(td::Status error); @@ -142,11 +147,15 @@ class LiteQuery : public td::actor::Actor { void perform_getConfigParams(BlockIdExt blkid, int mode, std::vector param_list = {}); void continue_getConfigParams(int mode, std::vector param_list); void perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, UnixTime utime); - void perform_lookupBlockWithProof(BlockId blkid, BlockIdExt client_mc_blkid, int mode, LogicalTime lt, UnixTime utime); - void continue_lookupBlockWithProof_getHeaderProof(Ref block, AccountIdPrefixFull req_prefix, BlockSeqno masterchain_ref_seqno); + void perform_lookupBlockWithProof(BlockId blkid, BlockIdExt client_mc_blkid, int mode, LogicalTime lt, + UnixTime utime); + void continue_lookupBlockWithProof_getHeaderProof(Ref block, + AccountIdPrefixFull req_prefix, BlockSeqno masterchain_ref_seqno); void continue_lookupBlockWithProof_gotPrevBlockData(Ref prev_block, BlockSeqno masterchain_ref_seqno); - void continue_lookupBlockWithProof_buildProofLinks(td::Ref cur_block, std::vector>> result); - void continue_lookupBlockWithProof_getClientMcBlockDataState(std::vector>> links); + void continue_lookupBlockWithProof_buildProofLinks(td::Ref cur_block, + std::vector>> result); + void continue_lookupBlockWithProof_getClientMcBlockDataState( + std::vector>> links); void continue_lookupBlockWithProof_getMcBlockPrev(std::vector>> links); void perform_listBlockTransactions(BlockIdExt blkid, int mode, int count, Bits256 account, LogicalTime lt); void finish_listBlockTransactions(int mode, int count); diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 9f4b877eb8..4d761dc138 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -14,15 +14,16 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "out-msg-queue-proof.hpp" -#include "interfaces/proof.h" -#include "shard.hpp" -#include "vm/cells/MerkleProof.h" +#include "block/block-auto.h" +#include "block/block-parse.h" #include "common/delay.h" +#include "interfaces/proof.h" #include "interfaces/validator-manager.h" -#include "block/block-parse.h" -#include "block/block-auto.h" +#include "vm/cells/MerkleProof.h" + +#include "out-msg-queue-proof.hpp" #include "output-queue-merger.h" +#include "shard.hpp" namespace ton { @@ -381,7 +382,7 @@ void OutMsgQueueImporter::get_proof_import(std::shared_ptr entry, st if (R.is_error()) { FLOG(DEBUG) { sb << "Failed to get out msg queue for " << dst_shard.to_str() << " from"; - for (const BlockIdExt &block : blocks) { + for (const BlockIdExt& block : blocks) { sb << " " << block.id.to_str(); } sb << ": " << R.move_as_error(); @@ -432,7 +433,7 @@ void OutMsgQueueImporter::got_proof(std::shared_ptr entry, std::vect void OutMsgQueueImporter::finish_query(std::shared_ptr entry) { FLOG(INFO) { sb << "Done importing neighbor msg queues for shard " << entry->dst_shard.to_str() << " from"; - for (const BlockIdExt &block : entry->blocks) { + for (const BlockIdExt& block : entry->blocks) { sb << " " << block.id.to_str(); } sb << " in " << entry->timer.elapsed() << "s"; @@ -472,7 +473,7 @@ bool OutMsgQueueImporter::check_timeout(std::shared_ptr entry) { if (entry->timeout.is_in_past()) { FLOG(DEBUG) { sb << "Aborting importing neighbor msg queues for shard " << entry->dst_shard.to_str() << " from"; - for (const BlockIdExt &block : entry->blocks) { + for (const BlockIdExt& block : entry->blocks) { sb << " " << block.id.to_str(); } sb << ": timeout"; @@ -497,7 +498,7 @@ void OutMsgQueueImporter::alarm() { if (!it->second->done) { FLOG(DEBUG) { sb << "Aborting importing neighbor msg queues for shard " << it->second->dst_shard.to_str() << " from"; - for (const BlockIdExt &block : it->second->blocks) { + for (const BlockIdExt& block : it->second->blocks) { sb << " " << block.id.to_str(); } sb << ": timeout"; diff --git a/validator/impl/out-msg-queue-proof.hpp b/validator/impl/out-msg-queue-proof.hpp index b537f31a91..da09af8b59 100644 --- a/validator/impl/out-msg-queue-proof.hpp +++ b/validator/impl/out-msg-queue-proof.hpp @@ -15,12 +15,13 @@ along with TON Blockchain Library. If not, see . */ #pragma once -#include "vm/cells.h" -#include "ton/ton-types.h" #include "auto/tl/ton_api.h" #include "interfaces/out-msg-queue-proof.h" -#include "td/actor/actor.h" #include "interfaces/shard.h" +#include "td/actor/actor.h" +#include "ton/ton-types.h" +#include "vm/cells.h" + #include "validator.h" namespace ton { @@ -81,10 +82,9 @@ class OutMsgQueueImporter : public td::actor::Actor { void get_proof_local(std::shared_ptr entry, BlockIdExt block); void get_proof_import(std::shared_ptr entry, std::vector blocks, block::ImportedMsgQueueLimits limits); - enum class ProofSource { - SmallCache, Broadcast, Query, Local - }; - void got_proof(std::shared_ptr entry, std::vector> proofs, ProofSource proof_source); + enum class ProofSource { SmallCache, Broadcast, Query, Local }; + void got_proof(std::shared_ptr entry, std::vector> proofs, + ProofSource proof_source); void finish_query(std::shared_ptr entry); bool check_timeout(std::shared_ptr entry); diff --git a/validator/impl/proof.cpp b/validator/impl/proof.cpp index 341029e57c..49fa8a33fc 100644 --- a/validator/impl/proof.cpp +++ b/validator/impl/proof.cpp @@ -16,11 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "proof.hpp" -#include "block/block-parse.h" #include "block/block-auto.h" +#include "block/block-parse.h" #include "vm/boc.h" #include "vm/cells/MerkleProof.h" + +#include "proof.hpp" #include "validator-set.hpp" namespace ton { diff --git a/validator/impl/proof.hpp b/validator/impl/proof.hpp index b6a41be58b..38a3a81957 100644 --- a/validator/impl/proof.hpp +++ b/validator/impl/proof.hpp @@ -17,11 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "interfaces/proof.h" -#include "interfaces/config.h" #include "block/block-db.h" #include "block/mc-config.h" +#include "interfaces/config.h" +#include "interfaces/proof.h" #include "vm/db/StaticBagOfCellsDb.h" + #include "config.hpp" namespace ton { diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 5803eac89f..d0d858a6b7 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -16,17 +16,18 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "shard.hpp" -#include "message-queue.hpp" -#include "validator-set.hpp" -#include "vm/boc.h" -#include "td/db/utils/BlobView.h" -#include "vm/db/StaticBagOfCellsDb.h" -#include "vm/cellslice.h" -#include "vm/cells/MerkleUpdate.h" -#include "block/block-parse.h" #include "block/block-auto.h" +#include "block/block-parse.h" +#include "td/db/utils/BlobView.h" #include "td/utils/filesystem.h" +#include "vm/boc.h" +#include "vm/cells/MerkleUpdate.h" +#include "vm/cellslice.h" +#include "vm/db/StaticBagOfCellsDb.h" + +#include "message-queue.hpp" +#include "shard.hpp" +#include "validator-set.hpp" #define LAZY_STATE_DESERIALIZE 1 diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index 7949de1293..ac6ade5d26 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -17,9 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "block/mc-config.h" #include "interfaces/shard.h" #include "vm/db/StaticBagOfCellsDb.h" -#include "block/mc-config.h" + #include "config.hpp" namespace ton { diff --git a/validator/impl/signature-set.cpp b/validator/impl/signature-set.cpp index 0078a115b6..4fdd450729 100644 --- a/validator/impl/signature-set.cpp +++ b/validator/impl/signature-set.cpp @@ -16,11 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "signature-set.hpp" -#include "auto/tl/ton_api.hpp" #include "adnl/utils.hpp" -#include "vm/dict.h" +#include "auto/tl/ton_api.hpp" #include "vm/boc.h" +#include "vm/dict.h" + +#include "signature-set.hpp" namespace ton { diff --git a/validator/impl/top-shard-descr.cpp b/validator/impl/top-shard-descr.cpp index f38d0cbdc1..9c7cb735b9 100644 --- a/validator/impl/top-shard-descr.cpp +++ b/validator/impl/top-shard-descr.cpp @@ -16,18 +16,18 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "top-shard-descr.hpp" +#include "block/block-auto.h" +#include "block/block-parse.h" #include "common/errorcode.h" +#include "vm/boc.h" +#include "vm/cells.h" +#include "vm/cells/MerkleProof.h" + #include "shard.hpp" #include "signature-set.hpp" +#include "top-shard-descr.hpp" #include "validator-set.hpp" -#include "vm/cells.h" -#include "vm/cells/MerkleProof.h" -#include "vm/boc.h" -#include "block/block-parse.h" -#include "block/block-auto.h" - namespace ton { namespace validator { diff --git a/validator/impl/top-shard-descr.hpp b/validator/impl/top-shard-descr.hpp index e1fd5778f0..48a0b2fb75 100644 --- a/validator/impl/top-shard-descr.hpp +++ b/validator/impl/top-shard-descr.hpp @@ -18,9 +18,9 @@ */ #pragma once +#include "block/mc-config.h" #include "interfaces/shard-block.h" #include "interfaces/validator-manager.h" -#include "block/mc-config.h" namespace ton { diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index bd889645c9..9d8380d718 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -16,25 +16,26 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "validate-query.hpp" -#include "top-shard-descr.hpp" -#include "validator-set.hpp" +#include + #include "adnl/utils.hpp" -#include "ton/ton-tl.hpp" -#include "ton/ton-io.hpp" -#include "vm/boc.h" +#include "block/block-auto.h" #include "block/block-db.h" -#include "block/block.h" #include "block/block-parse.h" -#include "block/block-auto.h" +#include "block/block.h" #include "block/output-queue-merger.h" +#include "common/errorlog.h" +#include "ton/ton-io.hpp" +#include "ton/ton-tl.hpp" +#include "vm/boc.h" #include "vm/cells/MerkleProof.h" #include "vm/cells/MerkleUpdate.h" -#include "common/errorlog.h" + #include "fabric.h" #include "storage-stat-cache.hpp" - -#include +#include "top-shard-descr.hpp" +#include "validate-query.hpp" +#include "validator-set.hpp" namespace ton { @@ -1139,8 +1140,8 @@ bool ValidateQuery::fetch_config_params() { if (compute_phase_cfg_.global_version >= 4) { auto prev_blocks_info = config_->get_prev_blocks_info(); if (prev_blocks_info.is_error()) { - return fatal_error(prev_blocks_info.move_as_error_prefix( - "cannot fetch prev blocks info from masterchain configuration: ")); + return fatal_error( + prev_blocks_info.move_as_error_prefix("cannot fetch prev blocks info from masterchain configuration: ")); } compute_phase_cfg_.prev_blocks_info = prev_blocks_info.move_as_ok(); } @@ -3062,14 +3063,14 @@ bool ValidateQuery::precheck_one_account_update(td::ConstBitPtr acc_id, Ref= 3 * 0) { FLOG(INFO) { sb << "state of account " << workchain() << ":" << acc_id.to_hex(256) - << " in the old shardchain state:" << "\n"; + << " in the old shardchain state:" << "\n"; if (old_value.not_null()) { block::gen::t_ShardAccount.print(sb, old_value); } else { sb << "" << "\n"; } sb << "state of account " << workchain() << ":" << acc_id.to_hex(256) - << " in the new shardchain state:" << "\n"; + << " in the new shardchain state:" << "\n"; if (new_value.not_null()) { block::gen::t_ShardAccount.print(sb, new_value); } else { @@ -3452,7 +3453,8 @@ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id } // mode for msg_export_{ext,new,imm,tr,deq_imm,???,deq/deq_short,tr_req,new_defer,deferred_tr} static const int tag_mode[10] = {0, 2, 0, 2, 1, 0, 1, 3, 0, 2}; - static const char* tag_str[10] = {"ext", "new", "imm", "tr", "deq_imm", "???", "deq", "tr_req", "new_defer", "deferred_tr"}; + static const char* tag_str[10] = {"ext", "new", "imm", "tr", "deq_imm", + "???", "deq", "tr_req", "new_defer", "deferred_tr"}; if (tag < 0 || tag >= 10 || !(tag_mode[tag] & mode)) { return reject_query(PSTRING() << "OutMsgDescr corresponding to " << m_str[mode] << "queued message with key " << out_msg_id.to_hex(352) << " has invalid tag " << tag << "(" << tag_str[tag & 7] @@ -4077,9 +4079,8 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) dest_prefix.to_str() + "... yet"); } if (from_dispatch_queue && next_prefix != cur_prefix) { - return reject_query( - "next hop address "s + next_prefix.to_str() + "... of deferred internal message with hash " + key.to_hex(256) + - " must coincide with its current prefix "s + cur_prefix.to_str() + "..."s); + return reject_query("next hop address "s + next_prefix.to_str() + "... of deferred internal message with hash " + + key.to_hex(256) + " must coincide with its current prefix "s + cur_prefix.to_str() + "..."s); } // if a message is processed by a transaction, it must have destination inside the current shard if (transaction.not_null() && !ton::shard_contains(shard_, dest_prefix)) { @@ -4275,8 +4276,8 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) } if (cur_prefix != next_prefix && tag == block::gen::InMsg::msg_import_deferred_tr) { return reject_query("internal message from DispatchQueue with hash "s + key.to_hex(256) + - " is a msg_import_deferred_tr$00101, but its current address " + - cur_prefix.to_str() + " is not equal to next address"); + " is a msg_import_deferred_tr$00101, but its current address " + cur_prefix.to_str() + + " is not equal to next address"); } CHECK(transaction.is_null()); auto out_msg_cs = out_msg_dict_->lookup(key, 256); @@ -4657,8 +4658,9 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms } // check that next hop is nearer to the destination than the current address if (count_matching_bits(dest_prefix, next_prefix) < count_matching_bits(dest_prefix, cur_prefix)) { - return reject_query("next hop address "s + next_prefix.to_str() + "... of outbound internal message with hash " + - key.to_hex(256) + " is further from its destination " + dest_prefix.to_str() + + return reject_query("next hop address "s + next_prefix.to_str() + + "... of outbound internal message with hash " + key.to_hex(256) + + " is further from its destination " + dest_prefix.to_str() + "... than its current address " + cur_prefix.to_str() + "..."); } // current address must belong to this shard (otherwise we should never had exported this message) @@ -5525,7 +5527,7 @@ std::unique_ptr ValidateQuery::unpack_account(td::ConstBitPtr ad * * @returns IHR fee */ -static td::RefInt256 get_ihr_fee(const block::gen::CommonMsgInfo::Record_int_msg_info &info, int global_version) { +static td::RefInt256 get_ihr_fee(const block::gen::CommonMsgInfo::Record_int_msg_info& info, int global_version) { // Legacy: extra_flags was previously ihr_fee return global_version >= 12 ? td::zero_refint() : block::tlb::t_Grams.as_integer(std::move(info.extra_flags)); } @@ -7153,13 +7155,12 @@ bool ValidateQuery::postcheck_value_flow() { << import_fees_ << ", the total transaction fees are " << transaction_fees_.to_str() << ", creation fee for this block is " << value_flow_.created.to_str() << ", the total imported fees from shards are " << value_flow_.fees_imported.to_str() - << " and the burned fees are " << fees_burned_.to_str() - << " with a total of " << expected_fees.to_str()); + << " and the burned fees are " << fees_burned_.to_str() << " with a total of " + << expected_fees.to_str()); } if (total_burned_ != value_flow_.burned) { return reject_query(PSTRING() << "invalid burned in value flow: " << id_.to_str() << " declared " - << value_flow_.burned.to_str() << ", correct value is " - << total_burned_.to_str()); + << value_flow_.burned.to_str() << ", correct value is " << total_burned_.to_str()); } return true; } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index f338f975c6..35bd706a9d 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -18,20 +18,21 @@ */ #pragma once +#include +#include +#include -#include "block-parse.h" -#include "fabric.h" +#include "block/mc-config.h" +#include "block/transaction.h" +#include "common/global-version.h" #include "interfaces/validator-manager.h" #include "vm/cells.h" #include "vm/dict.h" -#include "block/mc-config.h" -#include "block/transaction.h" + +#include "block-parse.h" +#include "fabric.h" #include "shard.hpp" #include "signature-set.hpp" -#include -#include -#include -#include "common/global-version.h" namespace ton { diff --git a/validator/impl/validator-set.cpp b/validator/impl/validator-set.cpp index d4b0d647b6..3eb77502e4 100644 --- a/validator/impl/validator-set.cpp +++ b/validator/impl/validator-set.cpp @@ -16,13 +16,14 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "validator-set.hpp" #include "auto/tl/ton_api.h" -// #include "adnl/utils.hpp" -#include "block/block.h" +#include "validator-set.hpp" +// #include "adnl/utils.hpp" #include +#include "block/block.h" + namespace ton { namespace validator { diff --git a/validator/impl/validator-set.hpp b/validator/impl/validator-set.hpp index 81c0bd5183..7b09bbe9c5 100644 --- a/validator/impl/validator-set.hpp +++ b/validator/impl/validator-set.hpp @@ -18,14 +18,14 @@ */ #pragma once -#include "validator/interfaces/validator-set.h" -#include "validator/interfaces/signature-set.h" -#include "ton/ton-types.h" -#include "keys/encryptor.h" -#include "block/mc-config.h" - #include +#include "block/mc-config.h" +#include "keys/encryptor.h" +#include "ton/ton-types.h" +#include "validator/interfaces/signature-set.h" +#include "validator/interfaces/validator-set.h" + namespace ton { namespace validator { diff --git a/validator/import-db-slice-local.cpp b/validator/import-db-slice-local.cpp index 945f41a6b5..6ed0620965 100644 --- a/validator/import-db-slice-local.cpp +++ b/validator/import-db-slice-local.cpp @@ -14,19 +14,19 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "import-db-slice-local.hpp" +#include -#include "validator/db/fileref.hpp" -#include "td/utils/overloaded.h" -#include "validator/fabric.h" -#include "td/actor/MultiPromise.h" +#include "block/block-auto.h" #include "common/checksum.h" +#include "downloaders/download-state.hpp" +#include "td/actor/MultiPromise.h" +#include "td/utils/overloaded.h" #include "td/utils/port/path.h" #include "ton/ton-io.hpp" -#include "downloaders/download-state.hpp" -#include "block/block-auto.h" +#include "validator/db/fileref.hpp" +#include "validator/fabric.h" -#include +#include "import-db-slice-local.hpp" namespace ton { @@ -155,8 +155,7 @@ void ArchiveImporterLocal::process_masterchain_blocks() { BlockSeqno expected_seqno = last_masterchain_state_->get_seqno() + 1; for (auto &[seqno, _] : masterchain_blocks_) { if (seqno != expected_seqno) { - abort_query( - td::Status::Error(ErrorCode::protoviolation, "non-consecutive masterchain blocks in the archive")); + abort_query(td::Status::Error(ErrorCode::protoviolation, "non-consecutive masterchain blocks in the archive")); return; } ++expected_seqno; diff --git a/validator/import-db-slice-local.hpp b/validator/import-db-slice-local.hpp index e9e6efdb1e..335e2a609c 100644 --- a/validator/import-db-slice-local.hpp +++ b/validator/import-db-slice-local.hpp @@ -18,8 +18,8 @@ #include "td/actor/actor.h" #include "td/utils/port/path.h" -#include "validator/interfaces/validator-manager.h" #include "validator/db/package.hpp" +#include "validator/interfaces/validator-manager.h" namespace ton { diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index aafd3c22f0..cd3efd6e20 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -16,18 +16,18 @@ Copyright 2019-2020 Telegram Systems LLP */ -#include "import-db-slice.hpp" +#include -#include "validator/db/fileref.hpp" -#include "td/utils/overloaded.h" -#include "validator/fabric.h" -#include "td/actor/MultiPromise.h" #include "common/checksum.h" +#include "downloaders/download-state.hpp" +#include "td/actor/MultiPromise.h" +#include "td/utils/overloaded.h" #include "td/utils/port/path.h" #include "ton/ton-io.hpp" -#include "downloaders/download-state.hpp" +#include "validator/db/fileref.hpp" +#include "validator/fabric.h" -#include +#include "import-db-slice.hpp" namespace ton { diff --git a/validator/import-db-slice.hpp b/validator/import-db-slice.hpp index fc295216ae..1401a4922d 100644 --- a/validator/import-db-slice.hpp +++ b/validator/import-db-slice.hpp @@ -20,8 +20,8 @@ #include "td/actor/actor.h" #include "td/utils/port/path.h" -#include "validator/interfaces/validator-manager.h" #include "validator/db/package.hpp" +#include "validator/interfaces/validator-manager.h" namespace ton { diff --git a/validator/interfaces/block-handle.h b/validator/interfaces/block-handle.h index ee1a225837..0a63f07b8c 100644 --- a/validator/interfaces/block-handle.h +++ b/validator/interfaces/block-handle.h @@ -18,9 +18,9 @@ */ #pragma once -#include "ton/ton-types.h" #include "td/actor/PromiseFuture.h" #include "td/actor/actor.h" +#include "ton/ton-types.h" namespace ton { diff --git a/validator/interfaces/block.h b/validator/interfaces/block.h index 45903ea842..bba8a733fc 100644 --- a/validator/interfaces/block.h +++ b/validator/interfaces/block.h @@ -17,8 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "vm/cells.h" #include "ton/ton-types.h" +#include "vm/cells.h" namespace ton { diff --git a/validator/interfaces/config.h b/validator/interfaces/config.h index fa458d0697..4e3169e2b0 100644 --- a/validator/interfaces/config.h +++ b/validator/interfaces/config.h @@ -18,10 +18,11 @@ */ #pragma once -#include "ton/ton-types.h" +#include "crypto/block/mc-config.h" #include "crypto/common/refcnt.hpp" +#include "ton/ton-types.h" + #include "validator-set.h" -#include "crypto/block/mc-config.h" namespace ton { diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 50ff678c02..cd3de75de3 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -54,7 +54,7 @@ class Db : public td::actor::Actor { virtual void store_block_state_from_data(BlockHandle handle, td::Ref block, td::Promise> promise) = 0; virtual void store_block_state_from_data_preliminary(std::vector> blocks, - td::Promise promise) = 0; + td::Promise promise) = 0; virtual void get_block_state(ConstBlockHandle handle, td::Promise> promise) = 0; virtual void store_block_state_part(BlockId effective_block, td::Ref cell, td::Promise> promise) = 0; @@ -65,7 +65,7 @@ class Db : public td::actor::Actor { td::Promise promise) = 0; virtual void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, - std::function write_data, + std::function write_data, td::Promise promise) = 0; virtual void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) = 0; diff --git a/validator/interfaces/external-message.h b/validator/interfaces/external-message.h index 871c624a4f..529d2bf334 100644 --- a/validator/interfaces/external-message.h +++ b/validator/interfaces/external-message.h @@ -19,8 +19,8 @@ #pragma once #include "crypto/common/refcnt.hpp" -#include "ton/ton-types.h" #include "crypto/vm/cells.h" +#include "ton/ton-types.h" namespace ton { diff --git a/validator/interfaces/ihr-message.h b/validator/interfaces/ihr-message.h index aa234b3aef..47ad499a0c 100644 --- a/validator/interfaces/ihr-message.h +++ b/validator/interfaces/ihr-message.h @@ -19,8 +19,8 @@ #pragma once #include "crypto/common/refcnt.hpp" -#include "ton/ton-types.h" #include "crypto/vm/cells.h" +#include "ton/ton-types.h" namespace ton { diff --git a/validator/interfaces/liteserver.h b/validator/interfaces/liteserver.h index f2f78d41fe..cbf4cf85ff 100644 --- a/validator/interfaces/liteserver.h +++ b/validator/interfaces/liteserver.h @@ -18,9 +18,9 @@ */ #pragma once +#include "common/bitstring.h" #include "td/actor/actor.h" #include "td/utils/buffer.h" -#include "common/bitstring.h" namespace ton::validator { @@ -35,4 +35,4 @@ class LiteServerCache : public td::actor::Actor { virtual void drop_send_message_from_cache(td::Bits256 key) = 0; }; -} // namespace ton::validator \ No newline at end of file +} // namespace ton::validator \ No newline at end of file diff --git a/validator/interfaces/message-queue.h b/validator/interfaces/message-queue.h index 816ce06683..1007ae2784 100644 --- a/validator/interfaces/message-queue.h +++ b/validator/interfaces/message-queue.h @@ -18,9 +18,9 @@ */ #pragma once -#include "ton/ton-types.h" #include "crypto/common/refcnt.hpp" #include "crypto/vm/cells.h" +#include "ton/ton-types.h" namespace ton { diff --git a/validator/interfaces/out-msg-queue-proof.h b/validator/interfaces/out-msg-queue-proof.h index 564b040cfa..ec601a7268 100644 --- a/validator/interfaces/out-msg-queue-proof.h +++ b/validator/interfaces/out-msg-queue-proof.h @@ -15,10 +15,10 @@ along with TON Blockchain Library. If not, see . */ #pragma once -#include "vm/cells.h" -#include "ton/ton-types.h" #include "auto/tl/ton_api.h" #include "block/block.h" +#include "ton/ton-types.h" +#include "vm/cells.h" namespace ton { diff --git a/validator/interfaces/persistent-state.h b/validator/interfaces/persistent-state.h index 36d9ef0bd4..74b5100965 100644 --- a/validator/interfaces/persistent-state.h +++ b/validator/interfaces/persistent-state.h @@ -19,8 +19,8 @@ #include "auto/tl/ton_api.h" #include "td/utils/Variant.h" #include "td/utils/overloaded.h" -#include "ton/ton-types.h" #include "ton/ton-shard.h" +#include "ton/ton-types.h" namespace ton { diff --git a/validator/interfaces/proof.h b/validator/interfaces/proof.h index 6665ad0801..eaf79a98b0 100644 --- a/validator/interfaces/proof.h +++ b/validator/interfaces/proof.h @@ -19,6 +19,7 @@ #pragma once #include "ton/ton-types.h" + #include "config.h" namespace ton { diff --git a/validator/interfaces/shard-block.h b/validator/interfaces/shard-block.h index 6e31cc136c..58bf04eb40 100644 --- a/validator/interfaces/shard-block.h +++ b/validator/interfaces/shard-block.h @@ -18,9 +18,10 @@ */ #pragma once +#include "interfaces/block-handle.h" #include "ton/ton-types.h" + #include "shard.h" -#include "interfaces/block-handle.h" namespace ton { diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 9a3a5aab09..07524f66f0 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -20,12 +20,13 @@ #include "ton/ton-shard.h" #include "ton/ton-types.h" -#include "validator-set.h" -#include "config.h" +#include "vm/cells.h" + #include "block.h" +#include "config.h" #include "message-queue.h" -#include "vm/cells.h" #include "proof.h" +#include "validator-set.h" namespace ton { diff --git a/validator/interfaces/validator-full-id.h b/validator/interfaces/validator-full-id.h index 0e1fa7392c..7840964162 100644 --- a/validator/interfaces/validator-full-id.h +++ b/validator/interfaces/validator-full-id.h @@ -18,9 +18,9 @@ */ #pragma once -#include "ton/ton-types.h" #include "auto/tl/ton_api.h" #include "keys/keys.hpp" +#include "ton/ton-types.h" namespace ton { diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 90e5247566..bfc35c3615 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -18,21 +18,22 @@ */ #pragma once -#include "shard.h" +#include + +#include "auto/tl/lite_api.h" +#include "crypto/vm/db/DynamicBagOfCellsDb.h" +#include "impl/out-msg-queue-proof.hpp" +#include "validator-session/validator-session-types.h" +#include "validator/validator.h" + #include "block.h" -#include "proof.h" #include "external-message.h" #include "ihr-message.h" -#include "shard-block.h" -#include "message-queue.h" -#include "validator/validator.h" #include "liteserver.h" -#include "crypto/vm/db/DynamicBagOfCellsDb.h" -#include "validator-session/validator-session-types.h" -#include "auto/tl/lite_api.h" -#include "impl/out-msg-queue-proof.hpp" - -#include +#include "message-queue.h" +#include "proof.h" +#include "shard-block.h" +#include "shard.h" namespace ton { @@ -62,7 +63,7 @@ struct StorageStatCacheStats { tl_object_ptr tl() const { return create_tl_object(small_cnt, small_cells, hit_cnt, hit_cells, - miss_cnt, miss_cells); + miss_cnt, miss_cells); } }; @@ -199,8 +200,8 @@ struct ValidationStats { tl_object_ptr tl() const { return create_tl_object( create_tl_block_id(block_id), collated_data_hash, validated_at, self.bits256_value(), valid, comment, - actual_bytes, actual_collated_data_bytes, total_time, work_time.total.real, work_time.total.cpu, - time_stats, work_time.to_str(false), work_time.to_str(true), storage_stat_cache.tl()); + actual_bytes, actual_collated_data_bytes, total_time, work_time.total.real, work_time.total.cpu, time_stats, + work_time.to_str(false), work_time.to_str(true), storage_stat_cache.tl()); } }; @@ -392,7 +393,7 @@ class ValidatorManager : public ValidatorManagerInterface { promise.set_result(td::Unit()); } - virtual void iterate_temp_block_handles(std::function f) { + virtual void iterate_temp_block_handles(std::function f) { } static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { diff --git a/validator/interfaces/validator-set.h b/validator/interfaces/validator-set.h index ad7fb9b552..3185fcc54d 100644 --- a/validator/interfaces/validator-set.h +++ b/validator/interfaces/validator-set.h @@ -18,10 +18,11 @@ */ #pragma once -#include "validator-full-id.h" -#include "signature-set.h" -#include "ton/ton-types.h" #include "adnl/adnl-node-id.hpp" +#include "ton/ton-types.h" + +#include "signature-set.h" +#include "validator-full-id.h" namespace ton { diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 0b519c2cdb..0bba54df6e 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -16,17 +16,17 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "manager-disk.hpp" -#include "validator-group.hpp" #include "adnl/utils.hpp" -#include "downloaders/wait-block-state.hpp" -#include "downloaders/wait-block-state-merge.hpp" #include "downloaders/wait-block-data-disk.hpp" -#include "validator-group.hpp" +#include "downloaders/wait-block-state-merge.hpp" +#include "downloaders/wait-block-state.hpp" +#include "td/utils/overloaded.h" +#include "ton/ton-io.hpp" + #include "fabric.h" +#include "manager-disk.hpp" #include "manager.h" -#include "ton/ton-io.hpp" -#include "td/utils/overloaded.h" +#include "validator-group.hpp" namespace ton { @@ -522,7 +522,7 @@ void ValidatorManagerImpl::wait_block_message_queue_short(BlockIdExt block_id, t void ValidatorManagerImpl::get_external_messages( ShardIdFull shard, td::Promise, int>>> promise) { std::vector, int>> res; - for (const auto& x : ext_messages_) { + for (const auto &x : ext_messages_) { res.emplace_back(x, 0); } promise.set_result(std::move(res)); diff --git a/validator/manager-disk.h b/validator/manager-disk.h index 2a1740c047..ef72cb8f87 100644 --- a/validator/manager-disk.h +++ b/validator/manager-disk.h @@ -17,8 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "validator/validator.h" #include "adnl/adnl.h" +#include "validator/validator.h" namespace ton { diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 1c4d00d720..0bef96ee9f 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -18,15 +18,16 @@ */ #pragma once -#include "interfaces/validator-manager.h" +#include +#include + #include "interfaces/db.h" -#include "validator-group.hpp" -#include "manager-init.h" +#include "interfaces/validator-manager.h" + #include "manager-disk.h" +#include "manager-init.h" #include "queue-size-counter.hpp" - -#include -#include +#include "validator-group.hpp" namespace ton { @@ -401,15 +402,15 @@ class ValidatorManagerImpl : public ValidatorManager { get_shard_state_from_db_short(block_id, std::move(promise)); } void get_block_by_lt_for_litequery(AccountIdPrefixFull account, LogicalTime lt, - td::Promise promise) override { + td::Promise promise) override { get_block_by_lt_from_db(account, lt, std::move(promise)); } void get_block_by_unix_time_for_litequery(AccountIdPrefixFull account, UnixTime ts, - td::Promise promise) override { + td::Promise promise) override { get_block_by_unix_time_from_db(account, ts, std::move(promise)); } void get_block_by_seqno_for_litequery(AccountIdPrefixFull account, BlockSeqno seqno, - td::Promise promise) override { + td::Promise promise) override { get_block_by_seqno_from_db(account, seqno, std::move(promise)); } void get_block_candidate_for_litequery(PublicKey source, BlockIdExt block_id, FileHash collated_data_hash, diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index 717ab3aa55..950e595238 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -16,18 +16,18 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "manager-hardfork.hpp" -#include "validator-group.hpp" #include "adnl/utils.hpp" -#include "downloaders/wait-block-state.hpp" -#include "downloaders/wait-block-state-merge.hpp" #include "downloaders/wait-block-data-disk.hpp" -#include "validator-group.hpp" +#include "downloaders/wait-block-state-merge.hpp" +#include "downloaders/wait-block-state.hpp" +#include "td/utils/filesystem.h" +#include "td/utils/overloaded.h" +#include "ton/ton-io.hpp" + #include "fabric.h" +#include "manager-hardfork.hpp" #include "manager.h" -#include "ton/ton-io.hpp" -#include "td/utils/overloaded.h" -#include "td/utils/filesystem.h" +#include "validator-group.hpp" namespace ton { diff --git a/validator/manager-hardfork.h b/validator/manager-hardfork.h index 824d27f788..36cf60e0dd 100644 --- a/validator/manager-hardfork.h +++ b/validator/manager-hardfork.h @@ -17,8 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "validator/validator.h" #include "adnl/adnl.h" +#include "validator/validator.h" namespace ton { diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 37b5d6dd39..77cef2a1df 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -18,16 +18,17 @@ */ #pragma once -#include "interfaces/validator-manager.h" +#include +#include + #include "interfaces/db.h" +#include "interfaces/validator-manager.h" #include "ton/ton-types.h" -#include "validator-group.hpp" -#include "manager-init.h" + #include "manager-hardfork.h" +#include "manager-init.h" #include "queue-size-counter.hpp" - -#include -#include +#include "validator-group.hpp" namespace ton { @@ -264,7 +265,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise, int>>> promise) override; void get_ihr_messages(ShardIdFull shard, td::Promise>> promise) override; void get_shard_blocks_for_collator(BlockIdExt masterchain_block_id, - td::Promise>> promise) override; + td::Promise>> promise) override; void complete_external_messages(std::vector to_delay, std::vector to_delete) override { } @@ -439,9 +440,9 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } - void prepare_actor_stats(td::Promise promise) override { + void prepare_actor_stats(td::Promise promise) override { UNREACHABLE(); - } + } void prepare_perf_timer_stats(td::Promise> promise) override { UNREACHABLE(); @@ -473,15 +474,15 @@ class ValidatorManagerImpl : public ValidatorManager { get_shard_state_from_db_short(block_id, std::move(promise)); } void get_block_by_lt_for_litequery(AccountIdPrefixFull account, LogicalTime lt, - td::Promise promise) override { + td::Promise promise) override { get_block_by_lt_from_db(account, lt, std::move(promise)); } void get_block_by_unix_time_for_litequery(AccountIdPrefixFull account, UnixTime ts, - td::Promise promise) override { + td::Promise promise) override { get_block_by_unix_time_from_db(account, ts, std::move(promise)); } void get_block_by_seqno_for_litequery(AccountIdPrefixFull account, BlockSeqno seqno, - td::Promise promise) override { + td::Promise promise) override { get_block_by_seqno_from_db(account, seqno, std::move(promise)); } void get_block_candidate_for_litequery(PublicKey source, BlockIdExt block_id, FileHash collated_data_hash, diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index 886a71544e..d7a27dcea7 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -16,16 +16,17 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "manager-init.hpp" -#include "td/utils/filesystem.h" -#include "fabric.h" -#include "downloaders/wait-block-state.hpp" -#include "ton/ton-io.hpp" -#include "common/checksum.h" #include "adnl/utils.hpp" -#include "validator/downloaders/download-state.hpp" +#include "common/checksum.h" #include "common/delay.h" +#include "downloaders/wait-block-state.hpp" #include "td/actor/MultiPromise.h" +#include "td/utils/filesystem.h" +#include "ton/ton-io.hpp" +#include "validator/downloaders/download-state.hpp" + +#include "fabric.h" +#include "manager-init.hpp" namespace ton { diff --git a/validator/manager-init.h b/validator/manager-init.h index d60efb31d9..1069fc1fa1 100644 --- a/validator/manager-init.h +++ b/validator/manager-init.h @@ -18,8 +18,9 @@ */ #pragma once -#include "interfaces/validator-manager.h" #include "interfaces/db.h" +#include "interfaces/validator-manager.h" + #include "shard-client.hpp" namespace ton { diff --git a/validator/manager-init.hpp b/validator/manager-init.hpp index 90dc5ccc97..2f41673cdd 100644 --- a/validator/manager-init.hpp +++ b/validator/manager-init.hpp @@ -18,16 +18,15 @@ */ #pragma once -#include "td/actor/actor.h" -#include "ton/ton-types.h" +#include -#include "interfaces/validator-manager.h" #include "interfaces/db.h" -#include "shard-client.hpp" +#include "interfaces/validator-manager.h" +#include "td/actor/actor.h" +#include "ton/ton-types.h" #include "manager-init.h" - -#include +#include "shard-client.hpp" namespace ton { diff --git a/validator/manager.cpp b/validator/manager.cpp index 4256cf0920..0cd8ed431d 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -16,40 +16,37 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "manager.hpp" -#include "checksum.h" -#include "td/utils/buffer.h" -#include "validator-group.hpp" -#include "downloaders/wait-block-state.hpp" -#include "downloaders/wait-block-state-merge.hpp" -#include "downloaders/wait-block-data.hpp" -#include "fabric.h" -#include "manager.h" -#include "validate-broadcast.hpp" -#include "ton/ton-tl.hpp" -#include "ton/ton-io.hpp" -#include "state-serializer.hpp" -#include "get-next-key-blocks.h" -#include "import-db-slice.hpp" -#include "import-db-slice-local.hpp" +#include #include "auto/tl/lite_api.h" -#include "tl-utils/lite-utils.hpp" #include "auto/tl/ton_api_json.h" -#include "tl/tl_json.h" - -#include "td/utils/Random.h" -#include "td/utils/port/path.h" -#include "td/utils/JsonBuilder.h" - #include "common/delay.h" #include "db/fileref.hpp" +#include "downloaders/wait-block-data.hpp" +#include "downloaders/wait-block-state-merge.hpp" +#include "downloaders/wait-block-state.hpp" #include "td/actor/MultiPromise.h" +#include "td/utils/JsonBuilder.h" +#include "td/utils/Random.h" +#include "td/utils/buffer.h" #include "td/utils/filesystem.h" - +#include "td/utils/port/path.h" +#include "tl-utils/lite-utils.hpp" +#include "tl/tl_json.h" +#include "ton/ton-io.hpp" +#include "ton/ton-tl.hpp" #include "validator/stats-merger.h" -#include +#include "checksum.h" +#include "fabric.h" +#include "get-next-key-blocks.h" +#include "import-db-slice-local.hpp" +#include "import-db-slice.hpp" +#include "manager.h" +#include "manager.hpp" +#include "state-serializer.hpp" +#include "validate-broadcast.hpp" +#include "validator-group.hpp" namespace ton { @@ -664,7 +661,7 @@ void ValidatorManagerImpl::add_cached_block_data(BlockIdExt block_id, td::Buffer if (block_id.is_masterchain()) { return; } - td::BufferSlice& block_data = cached_block_data_.get(block_id); + td::BufferSlice &block_data = cached_block_data_.get(block_id); if (!block_data.empty()) { return; } @@ -1831,8 +1828,7 @@ void ValidatorManagerImpl::send_get_block_proof_link_request(BlockIdExt block_id // Proof link can be created from the cached block data LOG(DEBUG) << "send_get_block_proof_link_request: creating proof link from cached block data for " << block_id.to_str(); - TRY_RESULT_PROMISE_PREFIX(promise, block_root, vm::std_boc_deserialize(*cached), - "failed to create proof link: "); + TRY_RESULT_PROMISE_PREFIX(promise, block_root, vm::std_boc_deserialize(*cached), "failed to create proof link: "); TRY_RESULT_PROMISE_PREFIX(promise, proof_link, WaitBlockData::generate_proof_link(block_id, block_root), "failed to create proof link: "); promise.set_result(std::move(proof_link)); diff --git a/validator/manager.h b/validator/manager.h index 652b775885..1265659bf0 100644 --- a/validator/manager.h +++ b/validator/manager.h @@ -17,10 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "validator/validator.h" #include "adnl/adnl.h" #include "rldp/rldp.h" #include "rldp2/rldp.h" +#include "validator/validator.h" namespace ton { @@ -28,13 +28,10 @@ namespace validator { class ValidatorManagerFactory { public: - static td::actor::ActorOwn create(td::Ref opts, - std::string db_root, - td::actor::ActorId keyring, - td::actor::ActorId adnl, - td::actor::ActorId rldp, - td::actor::ActorId rldp2, - td::actor::ActorId overlays); + static td::actor::ActorOwn create( + td::Ref opts, std::string db_root, td::actor::ActorId keyring, + td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, + td::actor::ActorId overlays); }; } // namespace validator diff --git a/validator/manager.hpp b/validator/manager.hpp index b3fdbc344c..f5a1668711 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -18,34 +18,35 @@ */ #pragma once +#include +#include +#include +#include + +#include "collator-node/collator-node.hpp" #include "common/refcnt.hpp" -#include "interfaces/validator-manager.h" +#include "impl/candidates-buffer.hpp" #include "interfaces/db.h" +#include "interfaces/validator-manager.h" +#include "rldp/rldp.h" +#include "rldp2/rldp.h" #include "td/actor/ActorStats.h" #include "td/actor/PromiseFuture.h" +#include "td/utils/LRUCache.h" #include "td/utils/SharedSlice.h" #include "td/utils/buffer.h" #include "td/utils/port/Poll.h" #include "td/utils/port/StdStreams.h" -#include "validator-group.hpp" -#include "shard-client.hpp" + #include "manager-init.h" -#include "state-serializer.hpp" -#include "rldp/rldp.h" -#include "rldp2/rldp.h" -#include "token-manager.h" #include "queue-size-counter.hpp" -#include "storage-stat-cache.hpp" -#include "impl/candidates-buffer.hpp" -#include "collator-node/collator-node.hpp" -#include "shard-block-verifier.hpp" #include "shard-block-retainer.hpp" -#include "td/utils/LRUCache.h" - -#include -#include -#include -#include +#include "shard-block-verifier.hpp" +#include "shard-client.hpp" +#include "state-serializer.hpp" +#include "storage-stat-cache.hpp" +#include "token-manager.h" +#include "validator-group.hpp" namespace ton { @@ -275,14 +276,14 @@ class ValidatorManagerImpl : public ValidatorManager { std::map, std::unique_ptr>> ext_messages_; std::map, std::map>> ext_addr_messages_; - void erase(const MessageId& id) { + void erase(const MessageId &id) { auto it = ext_messages_.find(id); CHECK(it != ext_messages_.end()); ext_addr_messages_[it->second->address()].erase(id.hash); ext_messages_.erase(it); } }; - std::map ext_msgs_; // priority -> messages + std::map ext_msgs_; // priority -> messages std::map>> ext_messages_hashes_; // hash -> priority td::Timestamp cleanup_mempool_at_; // IHR ? @@ -448,7 +449,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void set_block_state_from_data(BlockHandle handle, td::Ref block, td::Promise> promise) override; - void set_block_state_from_data_preliminary(std::vector> blocks, td::Promise promise) override; + void set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) override; void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::BufferSlice state, td::Promise promise) override; @@ -681,11 +683,11 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) override; void get_block_state_for_litequery(BlockIdExt block_id, td::Promise> promise) override; void get_block_by_lt_for_litequery(AccountIdPrefixFull account, LogicalTime lt, - td::Promise promise) override; + td::Promise promise) override; void get_block_by_unix_time_for_litequery(AccountIdPrefixFull account, UnixTime ts, - td::Promise promise) override; + td::Promise promise) override; void get_block_by_seqno_for_litequery(AccountIdPrefixFull account, BlockSeqno seqno, - td::Promise promise) override; + td::Promise promise) override; void process_block_handle_for_litequery_error(BlockIdExt block_id, td::Result r_handle, td::Promise promise); void process_lookup_block_for_litequery_error(AccountIdPrefixFull account, int type, td::uint64 value, @@ -702,7 +704,7 @@ class ValidatorManagerImpl : public ValidatorManager { ++(success ? total_ls_queries_ok_ : total_ls_queries_error_)[lite_query_id]; } - void get_storage_stat_cache(td::Promise(const td::Bits256&)>> promise) override { + void get_storage_stat_cache(td::Promise(const td::Bits256 &)>> promise) override { td::actor::send_closure(storage_stat_cache_, &StorageStatCache::get_cache, std::move(promise)); } void update_storage_stat_cache(std::vector, td::uint32>> data) override { @@ -837,7 +839,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::actor::ActorOwn storage_stat_cache_; - template + template void write_session_stats(const T &obj); td::actor::ActorOwn shard_block_verifier_; diff --git a/validator/net/download-archive-slice.cpp b/validator/net/download-archive-slice.cpp index c2f8eceac6..586505200b 100644 --- a/validator/net/download-archive-slice.cpp +++ b/validator/net/download-archive-slice.cpp @@ -16,11 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "download-archive-slice.hpp" -#include "td/utils/port/path.h" +#include + #include "td/utils/overloaded.h" +#include "td/utils/port/path.h" -#include +#include "download-archive-slice.hpp" namespace ton { diff --git a/validator/net/download-archive-slice.hpp b/validator/net/download-archive-slice.hpp index 42fd715f77..ae150ab75e 100644 --- a/validator/net/download-archive-slice.hpp +++ b/validator/net/download-archive-slice.hpp @@ -18,11 +18,11 @@ */ #pragma once +#include "adnl/adnl-ext-client.h" #include "overlay/overlays.h" +#include "td/utils/port/FileFd.h" #include "ton/ton-types.h" #include "validator/validator.h" -#include "adnl/adnl-ext-client.h" -#include "td/utils/port/FileFd.h" namespace ton { @@ -84,4 +84,3 @@ class DownloadArchiveSlice : public td::actor::Actor { } // namespace validator } // namespace ton - diff --git a/validator/net/download-block-new.cpp b/validator/net/download-block-new.cpp index 37580cef14..893d16ceba 100644 --- a/validator/net/download-block-new.cpp +++ b/validator/net/download-block-new.cpp @@ -16,13 +16,14 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "download-block-new.hpp" -#include "ton/ton-tl.hpp" #include "adnl/utils.hpp" -#include "ton/ton-shard.h" #include "td/utils/overloaded.h" #include "ton/ton-io.hpp" +#include "ton/ton-shard.h" +#include "ton/ton-tl.hpp" #include "validator/full-node.h" + +#include "download-block-new.hpp" #include "full-node-serializer.hpp" namespace ton { @@ -227,7 +228,8 @@ void DownloadBlockNew::got_data(td::BufferSlice data) { BlockIdExt id; td::BufferSlice proof, block_data; bool is_link; - td::Status S = deserialize_block_full(*f, id, proof, block_data, is_link, overlay::Overlays::max_fec_broadcast_size()); + td::Status S = + deserialize_block_full(*f, id, proof, block_data, is_link, overlay::Overlays::max_fec_broadcast_size()); if (S.is_error()) { abort_query(S.move_as_error_prefix("cannot deserialize block: ")); return; diff --git a/validator/net/download-block-new.hpp b/validator/net/download-block-new.hpp index ecd062ee3c..8984d2ccf4 100644 --- a/validator/net/download-block-new.hpp +++ b/validator/net/download-block-new.hpp @@ -18,11 +18,11 @@ */ #pragma once +#include "adnl/adnl-ext-client.h" #include "overlay/overlays.h" +#include "rldp/rldp.h" #include "ton/ton-types.h" #include "validator/validator.h" -#include "rldp/rldp.h" -#include "adnl/adnl-ext-client.h" namespace ton { diff --git a/validator/net/download-block.cpp b/validator/net/download-block.cpp index 75dc57fc44..30e8221463 100644 --- a/validator/net/download-block.cpp +++ b/validator/net/download-block.cpp @@ -16,14 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "download-block.hpp" -#include "ton/ton-tl.hpp" #include "adnl/utils.hpp" -#include "ton/ton-shard.h" #include "td/utils/overloaded.h" #include "ton/ton-io.hpp" +#include "ton/ton-shard.h" +#include "ton/ton-tl.hpp" #include "validator/full-node.h" +#include "download-block.hpp" + namespace ton { namespace validator { diff --git a/validator/net/download-block.hpp b/validator/net/download-block.hpp index 2e2a715bf6..49d3f671c0 100644 --- a/validator/net/download-block.hpp +++ b/validator/net/download-block.hpp @@ -18,11 +18,11 @@ */ #pragma once +#include "adnl/adnl-ext-client.h" #include "overlay/overlays.h" +#include "rldp/rldp.h" #include "ton/ton-types.h" #include "validator/validator.h" -#include "rldp/rldp.h" -#include "adnl/adnl-ext-client.h" namespace ton { diff --git a/validator/net/download-next-block.cpp b/validator/net/download-next-block.cpp index 18472d4baa..1e0b63bab1 100644 --- a/validator/net/download-next-block.cpp +++ b/validator/net/download-next-block.cpp @@ -16,12 +16,13 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "download-next-block.hpp" -#include "ton/ton-tl.hpp" #include "adnl/utils.hpp" -#include "download-block.hpp" +#include "ton/ton-tl.hpp" #include "validator/full-node.h" +#include "download-block.hpp" +#include "download-next-block.hpp" + namespace ton { namespace validator { diff --git a/validator/net/download-next-block.hpp b/validator/net/download-next-block.hpp index 840bb12166..71c71970c9 100644 --- a/validator/net/download-next-block.hpp +++ b/validator/net/download-next-block.hpp @@ -18,12 +18,12 @@ */ #pragma once +#include "adnl/adnl-ext-client.h" #include "overlay/overlays.h" -#include "ton/ton-types.h" -#include "validator/validator.h" #include "rldp/rldp.h" #include "ton/ton-io.hpp" -#include "adnl/adnl-ext-client.h" +#include "ton/ton-types.h" +#include "validator/validator.h" namespace ton { diff --git a/validator/net/download-proof.cpp b/validator/net/download-proof.cpp index 784ecac209..8de10bc630 100644 --- a/validator/net/download-proof.cpp +++ b/validator/net/download-proof.cpp @@ -16,14 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "download-proof.hpp" -#include "ton/ton-tl.hpp" #include "adnl/utils.hpp" -#include "ton/ton-shard.h" #include "td/utils/overloaded.h" #include "ton/ton-io.hpp" +#include "ton/ton-shard.h" +#include "ton/ton-tl.hpp" #include "validator/full-node.h" +#include "download-proof.hpp" + namespace ton { namespace validator { diff --git a/validator/net/download-proof.hpp b/validator/net/download-proof.hpp index 9caf9c3a12..89516afe9a 100644 --- a/validator/net/download-proof.hpp +++ b/validator/net/download-proof.hpp @@ -18,11 +18,11 @@ */ #pragma once +#include "adnl/adnl-ext-client.h" #include "overlay/overlays.h" +#include "rldp/rldp.h" #include "ton/ton-types.h" #include "validator/validator.h" -#include "rldp/rldp.h" -#include "adnl/adnl-ext-client.h" namespace ton { diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index da6363f0d6..c27f52bb9f 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -16,10 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "download-state.hpp" -#include "ton/ton-tl.hpp" -#include "ton/ton-io.hpp" #include "td/utils/overloaded.h" +#include "ton/ton-io.hpp" +#include "ton/ton-tl.hpp" + +#include "download-state.hpp" #include "full-node.h" namespace ton { diff --git a/validator/net/download-state.hpp b/validator/net/download-state.hpp index ebf119bd37..1752ab29ea 100644 --- a/validator/net/download-state.hpp +++ b/validator/net/download-state.hpp @@ -18,12 +18,12 @@ */ #pragma once +#include + +#include "adnl/adnl-ext-client.h" #include "overlay/overlays.h" #include "ton/ton-types.h" #include "validator/validator.h" -#include "adnl/adnl-ext-client.h" - -#include namespace ton { diff --git a/validator/net/get-next-key-blocks.cpp b/validator/net/get-next-key-blocks.cpp index 2c12e495ba..b7e3f11704 100644 --- a/validator/net/get-next-key-blocks.cpp +++ b/validator/net/get-next-key-blocks.cpp @@ -16,14 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "get-next-key-blocks.hpp" -#include "download-proof.hpp" -#include "ton/ton-tl.hpp" #include "adnl/utils.hpp" -#include "ton/ton-shard.h" #include "td/utils/overloaded.h" #include "ton/ton-io.hpp" +#include "ton/ton-shard.h" +#include "ton/ton-tl.hpp" + +#include "download-proof.hpp" #include "full-node.h" +#include "get-next-key-blocks.hpp" namespace ton { diff --git a/validator/net/get-next-key-blocks.hpp b/validator/net/get-next-key-blocks.hpp index 14c040bd20..2c99e3f916 100644 --- a/validator/net/get-next-key-blocks.hpp +++ b/validator/net/get-next-key-blocks.hpp @@ -18,11 +18,11 @@ */ #pragma once +#include "adnl/adnl-ext-client.h" #include "overlay/overlays.h" +#include "rldp/rldp.h" #include "ton/ton-types.h" #include "validator/validator.h" -#include "rldp/rldp.h" -#include "adnl/adnl-ext-client.h" namespace ton { diff --git a/validator/queue-size-counter.cpp b/validator/queue-size-counter.cpp index 487bdefc80..59d3a13039 100644 --- a/validator/queue-size-counter.cpp +++ b/validator/queue-size-counter.cpp @@ -14,13 +14,14 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "queue-size-counter.hpp" #include "block/block-auto.h" #include "block/block-parse.h" #include "common/delay.h" #include "td/actor/MultiPromise.h" #include "td/utils/Random.h" +#include "queue-size-counter.hpp" + namespace ton::validator { static td::Result calc_queue_size(const td::Ref &state) { diff --git a/validator/shard-block-retainer.cpp b/validator/shard-block-retainer.cpp index 8ef4d8f21e..4ff0ba042b 100644 --- a/validator/shard-block-retainer.cpp +++ b/validator/shard-block-retainer.cpp @@ -14,9 +14,10 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "shard-block-retainer.hpp" #include +#include "shard-block-retainer.hpp" + namespace ton::validator { void ShardBlockRetainer::start_up() { diff --git a/validator/shard-block-retainer.hpp b/validator/shard-block-retainer.hpp index 366a6d506e..d4471294b8 100644 --- a/validator/shard-block-retainer.hpp +++ b/validator/shard-block-retainer.hpp @@ -15,9 +15,10 @@ along with TON Blockchain Library. If not, see . */ #pragma once +#include + #include "interfaces/validator-manager.h" #include "rldp2/rldp.h" -#include namespace ton::validator { diff --git a/validator/shard-block-verifier.cpp b/validator/shard-block-verifier.cpp index 0403ff22aa..bab0273709 100644 --- a/validator/shard-block-verifier.cpp +++ b/validator/shard-block-verifier.cpp @@ -14,10 +14,10 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "shard-block-verifier.hpp" - #include "td/actor/MultiPromise.h" +#include "shard-block-verifier.hpp" + namespace ton::validator { void ShardBlockVerifier::start_up() { diff --git a/validator/shard-block-verifier.hpp b/validator/shard-block-verifier.hpp index 025996681e..3237dc48a6 100644 --- a/validator/shard-block-verifier.hpp +++ b/validator/shard-block-verifier.hpp @@ -15,9 +15,10 @@ along with TON Blockchain Library. If not, see . */ #pragma once +#include + #include "interfaces/validator-manager.h" #include "rldp2/rldp.h" -#include namespace ton::validator { diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index 49ee7b6e8a..3a1b0c9cb8 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -16,11 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "shard-client.hpp" -#include "ton/ton-io.hpp" -#include "validator/fabric.h" #include "td/actor/MultiPromise.h" +#include "ton/ton-io.hpp" #include "validator/downloaders/download-state.hpp" +#include "validator/fabric.h" + +#include "shard-client.hpp" namespace ton { diff --git a/validator/shard-client.hpp b/validator/shard-client.hpp index c13b625aa8..c4ccccda1c 100644 --- a/validator/shard-client.hpp +++ b/validator/shard-client.hpp @@ -18,9 +18,10 @@ */ #pragma once -#include "interfaces/validator-manager.h" #include +#include "interfaces/validator-manager.h" + namespace ton { namespace validator { diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index b024dc1009..de1a66a6d7 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -16,17 +16,18 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "state-serializer.hpp" +#include "common/delay.h" #include "crypto/block/block-auto.h" #include "crypto/block/block-parse.h" +#include "td/utils/HashSet.h" #include "td/utils/Random.h" +#include "td/utils/filesystem.h" #include "td/utils/overloaded.h" #include "ton/ton-io.hpp" -#include "common/delay.h" -#include "td/utils/filesystem.h" -#include "td/utils/HashSet.h" #include "vm/cells/MerkleProof.h" +#include "state-serializer.hpp" + namespace ton { namespace validator { @@ -250,7 +251,7 @@ void AsyncStateSerializer::store_persistent_state_description(td::Refget_block_id(); desc.start_time = state->get_unix_time(); desc.end_time = ValidatorManager::persistent_state_ttl(desc.start_time); - for (const auto &v : state->get_shards()) { + for (const auto& v : state->get_shards()) { desc.shard_blocks.push_back({ .block = v->top_block_id(), .split_depth = state->persistent_state_split_depth(v->shard().workchain), @@ -272,8 +273,7 @@ void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) { class CachedCellDbReader : public vm::CellDbReader { public: - CachedCellDbReader(std::shared_ptr parent, - std::shared_ptr cache) + CachedCellDbReader(std::shared_ptr parent, std::shared_ptr cache) : parent_(std::move(parent)), cache_(std::move(cache)) { } td::Result> load_cell(td::Slice hash) override { @@ -319,9 +319,10 @@ class CachedCellDbReader : public vm::CellDbReader { return res; }; void print_stats() const { - LOG(WARNING) << "CachedCellDbReader stats : " << total_reqs_ << " reads, " << cached_reqs_ << " cached, " + LOG(WARNING) << "CachedCellDbReader stats : " << total_reqs_ << " reads, " << cached_reqs_ << " cached, " << bulk_reqs_ << " bulk reqs"; } + private: std::shared_ptr parent_; std::shared_ptr cache_; @@ -355,8 +356,8 @@ void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard, return; } td::Timer timer; - LOG(WARNING) << "Preloading previous persistent state for shard " << shard.to_str() << " (" - << cur_shards.size() << " files)"; + LOG(WARNING) << "Preloading previous persistent state for shard " << shard.to_str() << " (" << cur_shards.size() + << " files)"; vm::CellHashSet cells; std::function)> dfs = [&](td::Ref cell) { if (!cells.insert(cell).second) { @@ -421,7 +422,7 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state CHECK(shards_.size() == 0); auto vec = state->get_shards(); - for (auto &v : vec) { + for (auto& v : vec) { if (opts_->need_monitor(v->shard(), state)) { shards_.push_back({ .block_id = v->top_block_id(), @@ -441,7 +442,8 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state previous_state_cache->prepare_cache(shard, UnsplitStateType{}); } auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache->cache); - auto res = vm::boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); + auto res = + vm::boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); new_cell_db_reader->print_stats(); return res; }; diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index baa9268db0..407b0c6777 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -18,11 +18,11 @@ */ #pragma once -#include "interfaces/validator-manager.h" -#include "interfaces/shard.h" - #include +#include "interfaces/shard.h" +#include "interfaces/validator-manager.h" + namespace ton { namespace validator { diff --git a/validator/stats-merger.h b/validator/stats-merger.h index bc5b7085e6..30323169dc 100644 --- a/validator/stats-merger.h +++ b/validator/stats-merger.h @@ -18,8 +18,8 @@ */ #pragma once -#include "td/utils/int_types.h" #include "td/actor/actor.h" +#include "td/utils/int_types.h" namespace ton { diff --git a/validator/stats-provider.h b/validator/stats-provider.h index e0a7f5650b..2c45ff6546 100644 --- a/validator/stats-provider.h +++ b/validator/stats-provider.h @@ -16,10 +16,11 @@ */ #pragma once -#include "validator.h" +#include + #include "common/AtomicRef.h" -#include +#include "validator.h" namespace ton { diff --git a/validator/storage-stat-cache.cpp b/validator/storage-stat-cache.cpp index a81d958f5a..19b475781c 100644 --- a/validator/storage-stat-cache.cpp +++ b/validator/storage-stat-cache.cpp @@ -25,7 +25,7 @@ void StorageStatCache::get_cache(td::Promise(con } void StorageStatCache::update(std::vector, td::uint32>> data) { - for (auto &[cell, size] : data) { + for (auto& [cell, size] : data) { if (size < MIN_ACCOUNT_CELLS) { continue; } diff --git a/validator/storage-stat-cache.hpp b/validator/storage-stat-cache.hpp index 194a0b79ab..fdf4778da5 100644 --- a/validator/storage-stat-cache.hpp +++ b/validator/storage-stat-cache.hpp @@ -15,12 +15,12 @@ along with TON Blockchain Library. If not, see . */ #pragma once +#include + #include "interfaces/validator-manager.h" #include "td/utils/ConcurrentHashTable.h" #include "td/utils/LRUCache.h" -#include - namespace ton::validator { class StorageStatCache : public td::actor::Actor { diff --git a/validator/token-manager.h b/validator/token-manager.h index 0fd0126a35..9779711a3c 100644 --- a/validator/token-manager.h +++ b/validator/token-manager.h @@ -18,11 +18,11 @@ */ #pragma once +#include + #include "td/actor/actor.h" #include "validator/interfaces/validator-manager.h" -#include - namespace ton { namespace validator { diff --git a/validator/validate-broadcast.cpp b/validator/validate-broadcast.cpp index 189697c536..e7a4ec43b8 100644 --- a/validator/validate-broadcast.cpp +++ b/validator/validate-broadcast.cpp @@ -16,11 +16,12 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "validate-broadcast.hpp" -#include "fabric.h" #include "adnl/utils.hpp" #include "ton/ton-io.hpp" + #include "apply-block.hpp" +#include "fabric.h" +#include "validate-broadcast.hpp" namespace ton { diff --git a/validator/validate-broadcast.hpp b/validator/validate-broadcast.hpp index 74496fa3f2..c284dd043a 100644 --- a/validator/validate-broadcast.hpp +++ b/validator/validate-broadcast.hpp @@ -18,8 +18,8 @@ */ #pragma once -#include "validator/interfaces/validator-manager.h" #include "validator/interfaces/shard.h" +#include "validator/interfaces/validator-manager.h" namespace ton { @@ -62,8 +62,8 @@ class ValidateBroadcast : public td::actor::Actor { , timeout_(timeout) , promise_(std::move(promise)) , perf_timer_("validatebroadcast", 0.1, [manager](double duration) { - send_closure(manager, &ValidatorManager::add_perf_timer_stat, "validatebroadcast", duration); - }) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "validatebroadcast", duration); + }) { } void start_up() override; diff --git a/validator/validator-full-id.cpp b/validator/validator-full-id.cpp index 5cca11c20b..81ff76259d 100644 --- a/validator/validator-full-id.cpp +++ b/validator/validator-full-id.cpp @@ -16,8 +16,8 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "interfaces/validator-full-id.h" #include "adnl/utils.hpp" +#include "interfaces/validator-full-id.h" namespace ton { diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index c79dbda4ac..aef4e0cc97 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -16,15 +16,16 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "validator-group.hpp" -#include "fabric.h" -#include "full-node-master.hpp" -#include "ton/ton-io.hpp" -#include "td/utils/overloaded.h" +#include "collator-node/collator-node.hpp" #include "common/delay.h" -#include "ton/lite-tl.hpp" #include "td/utils/Random.h" -#include "collator-node/collator-node.hpp" +#include "td/utils/overloaded.h" +#include "ton/lite-tl.hpp" +#include "ton/ton-io.hpp" + +#include "fabric.h" +#include "full-node-master.hpp" +#include "validator-group.hpp" namespace ton { @@ -388,11 +389,11 @@ void ValidatorGroup::generate_block_optimistic(validatorsession::BlockSourceInfo }; LOG(WARNING) << "Optimistically generating next block after " << block_id.to_str(); td::uint64 max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; - td::actor::send_closure(collation_manager_, &CollationManager::collate_block_optimistic, shard_, min_masterchain_block_id_, - block_id, std::move(prev_block), Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, - source_info.priority, validator_set_, max_answer_size, - optimistic_generation_->cancellation_token_source.get_cancellation_token(), std::move(P), - config_.proto_version); + td::actor::send_closure(collation_manager_, &CollationManager::collate_block_optimistic, shard_, + min_masterchain_block_id_, block_id, std::move(prev_block), + Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, source_info.priority, validator_set_, + max_answer_size, optimistic_generation_->cancellation_token_source.get_cancellation_token(), + std::move(P), config_.proto_version); } void ValidatorGroup::generated_block_optimistic(validatorsession::BlockSourceInfo source_info, diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 3df456efa3..33200471db 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -18,15 +18,14 @@ */ #pragma once -#include "collation-manager.hpp" -#include "interfaces/validator-manager.h" - -#include "validator-session/validator-session.h" +#include +#include "interfaces/validator-manager.h" #include "rldp/rldp.h" #include "rldp2/rldp.h" +#include "validator-session/validator-session.h" -#include +#include "collation-manager.hpp" namespace ton { @@ -56,7 +55,8 @@ class ValidatorGroup : public td::actor::Actor { BlockId create_next_block_id_simple() const; void generate_block_optimistic(validatorsession::BlockSourceInfo source_info, td::BufferSlice prev_block, - RootHash prev_root_hash, FileHash prev_file_hash, td::Promise promise); + RootHash prev_root_hash, FileHash prev_file_hash, + td::Promise promise); void generated_block_optimistic(validatorsession::BlockSourceInfo source_info, td::Result R); void start(std::vector prev, BlockIdExt min_masterchain_block_id); diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index a4a61373c0..2508b6d2c5 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -16,11 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "validator-options.hpp" +#include #include "ton/ton-shard.h" -#include +#include "validator-options.hpp" namespace ton { @@ -86,9 +86,9 @@ td::Status ShardBlockVerifierConfig::unpack(const ton_api::engine_validator_shar shard.trusted_nodes.push_back(node_id); } if (shard_obj->required_confirms_ < 0 || shard_obj->required_confirms_ > (int)shard.trusted_nodes.size()) { - return td::Status::Error(PSTRING() - << "invalid required_confirms " << shard_obj->required_confirms_ << " for shard " - << shard.shard_id.to_str() << " (nodes: " << shard.trusted_nodes.size() << ")"); + return td::Status::Error(PSTRING() << "invalid required_confirms " << shard_obj->required_confirms_ + << " for shard " << shard.shard_id.to_str() + << " (nodes: " << shard.trusted_nodes.size() << ")"); } shard.required_confirms = shard_obj->required_confirms_; shards.push_back(std::move(shard)); diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 8c7401625c..be5a03acea 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -217,7 +217,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { unsafe_catchains_.insert(seqno); } void add_unsafe_catchain_rotate(BlockSeqno seqno, CatchainSeqno cc_seqno, td::uint32 value) override { - VLOG(INFO) << "Add unsafe catchain rotation: Master block seqno " << seqno<<" Catchain seqno " << cc_seqno << " New value "<< value; + VLOG(INFO) << "Add unsafe catchain rotation: Master block seqno " << seqno << " Catchain seqno " << cc_seqno + << " New value " << value; unsafe_catchain_rotates_[cc_seqno] = std::make_pair(seqno, value); } void truncate_db(BlockSeqno seqno) override { @@ -300,7 +301,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { shard_block_verifier_config_ = std::move(config); } - ValidatorManagerOptionsImpl *make_copy() const override { + ValidatorManagerOptionsImpl* make_copy() const override { return new ValidatorManagerOptionsImpl(*this); } diff --git a/validator/validator-telemetry.cpp b/validator/validator-telemetry.cpp index b870e8b84d..38c7c18d2c 100644 --- a/validator/validator-telemetry.cpp +++ b/validator/validator-telemetry.cpp @@ -23,11 +23,12 @@ exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. */ -#include "validator-telemetry.hpp" -#include "git.h" +#include "interfaces/validator-manager.h" #include "td/utils/Random.h" #include "td/utils/port/uname.h" -#include "interfaces/validator-manager.h" + +#include "git.h" +#include "validator-telemetry.hpp" namespace ton::validator { diff --git a/validator/validator-telemetry.hpp b/validator/validator-telemetry.hpp index d83641dfe4..01394f2ecd 100644 --- a/validator/validator-telemetry.hpp +++ b/validator/validator-telemetry.hpp @@ -24,10 +24,11 @@ from all source files in the program, then also delete it here. */ #pragma once -#include "overlay.h" -#include "td/actor/actor.h" #include "adnl/adnl.h" #include "interfaces/shard.h" +#include "td/actor/actor.h" + +#include "overlay.h" namespace ton::validator { class ValidatorManager; diff --git a/validator/validator.h b/validator/validator.h index 826fc8f8f7..b395708487 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -18,27 +18,24 @@ */ #pragma once -#include #include #include - -#include "interfaces/persistent-state.h" -#include "td/actor/actor.h" - -#include "ton/ton-types.h" +#include #include "adnl/adnl.h" +#include "catchain/catchain-types.h" #include "dht/dht.h" -#include "overlay/overlays.h" - #include "interfaces/block-handle.h" -#include "interfaces/validator-set.h" #include "interfaces/block.h" +#include "interfaces/external-message.h" +#include "interfaces/out-msg-queue-proof.h" +#include "interfaces/persistent-state.h" #include "interfaces/proof.h" #include "interfaces/shard.h" -#include "catchain/catchain-types.h" -#include "interfaces/out-msg-queue-proof.h" -#include "interfaces/external-message.h" +#include "interfaces/validator-set.h" +#include "overlay/overlays.h" +#include "td/actor/actor.h" +#include "ton/ton-types.h" namespace ton { @@ -51,7 +48,7 @@ class ActionToken { struct PerfTimerStats { std::string name; - std::deque> stats; // + std::deque> stats; // }; struct CollatorOptions : public td::CntObject { @@ -80,9 +77,7 @@ struct CollatorOptions : public td::CntObject { }; struct CollatorsList : public td::CntObject { - enum SelectMode { - mode_random, mode_ordered, mode_round_robin - }; + enum SelectMode { mode_random, mode_ordered, mode_round_robin }; struct Shard { ShardIdFull shard_id; SelectMode select_mode = mode_random; @@ -196,11 +191,11 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_collator_node_whitelist_enabled(bool enabled) = 0; virtual void set_shard_block_verifier_config(td::Ref config) = 0; - static td::Ref create( - BlockIdExt zero_block_id, BlockIdExt init_block_id, - bool allow_blockchain_init = false, double sync_blocks_before = 3600, double block_ttl = 86400, - double state_ttl = 86400, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650, - double max_mempool_num = 999999, bool initial_sync_disabled = false); + static td::Ref create(BlockIdExt zero_block_id, BlockIdExt init_block_id, + bool allow_blockchain_init = false, double sync_blocks_before = 3600, + double block_ttl = 86400, double state_ttl = 86400, + double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650, + double max_mempool_num = 999999, bool initial_sync_disabled = false); }; class ValidatorManagerInterface : public td::actor::Actor { From 716c23d1afc3b9bde000b1542e4a17b32f0fc2ab Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Thu, 6 Nov 2025 09:38:46 -0500 Subject: [PATCH 442/524] Check C++ formatting in CI --- .github/workflows/build-ton-linux-x86-64-werror.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-ton-linux-x86-64-werror.yml b/.github/workflows/build-ton-linux-x86-64-werror.yml index 84e76ea984..e045d0e7f5 100644 --- a/.github/workflows/build-ton-linux-x86-64-werror.yml +++ b/.github/workflows/build-ton-linux-x86-64-werror.yml @@ -1,9 +1,9 @@ -name: Ubuntu TON -Werror build (shared, x86-64) +name: Lint on: [push, pull_request, workflow_dispatch, workflow_call] jobs: - build: + lint: runs-on: ubuntu-24.04 steps: @@ -27,7 +27,7 @@ jobs: run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 21 clang + sudo ./llvm.sh 21 all # TODO: Preserve ccache cache between runs once we get rid of warning suppression mappings. @@ -41,3 +41,9 @@ jobs: -DTON_USE_LLD=On \ -DTON_WERROR_BUILD=On ninja -C build + + - name: Check C++ formatting + run: | + # TODO: Run clang-format only on changed files. + clang-format-21 -i $(git ls-files '*.h' '*.hpp' '*.cpp' ':!^example/android/third_party' ':!^tolk/') + git diff --exit-code From 1abeaef5bff79942b7ff92c7d5c79f7643c835d4 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Thu, 6 Nov 2025 09:53:16 -0500 Subject: [PATCH 443/524] Add custom Git merge driver to merge over formatting changes To use it, run ``` git config merge.clang-format-driver.driver "assembly/clang-format-driver %O %A %B %P" ``` --- .gitattributes | 4 ++++ assembly/clang-format-driver | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100755 assembly/clang-format-driver diff --git a/.gitattributes b/.gitattributes index d69dd2b15c..8aa2d7d9f3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -34,3 +34,7 @@ third_party/** linguist-vendored *.a binary *.jar binary *.tlo binary + +*.h merge=clang-format-driver +*.cpp merge=clang-format-driver +*.hpp merge=clang-format-driver diff --git a/assembly/clang-format-driver b/assembly/clang-format-driver new file mode 100755 index 0000000000..a54669ae42 --- /dev/null +++ b/assembly/clang-format-driver @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -eo pipefail + +BASE="$1" +OURS="$2" +THEIRS="$3" +FILE="$4" + +TMP=$(mktemp) +TMP_BASE=$(mktemp) + +cp "$THEIRS" "$TMP" +cp "$BASE" "$TMP_BASE" +clang-format -i --assume-filename="$FILE" --style=file:.clang-format "$TMP" +clang-format -i --assume-filename="$FILE" --style=file:.clang-format "$TMP_BASE" + +git merge-file "$OURS" "$TMP_BASE" "$TMP" + +rm "$TMP" +rm "$TMP_BASE" From f5a43807cb6d02178ddc8a06231aef1889bee030 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Thu, 6 Nov 2025 22:12:19 +0400 Subject: [PATCH 444/524] Automate Github Actions cache clean up (#1870) * upgrade abseil to August 2025, Patch 1 * add gh action to clear all gh actions cache * add gh action to clear all gh actions cache --- .github/workflows/clear-gh-cache.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/workflows/clear-gh-cache.yml diff --git a/.github/workflows/clear-gh-cache.yml b/.github/workflows/clear-gh-cache.yml new file mode 100644 index 0000000000..3e166be656 --- /dev/null +++ b/.github/workflows/clear-gh-cache.yml @@ -0,0 +1,13 @@ +name: Clear all GH cache + +on: + workflow_dispatch + +jobs: + clear-cache: + env: + GH_TOKEN: ${{ github.token }} + runs-on: ubuntu-latest + steps: + - name: Clear all cache + run: gh cache delete --all From 431aa80ce6c7e7236a8f7f23617c35438395a0ab Mon Sep 17 00:00:00 2001 From: neodix42 Date: Thu, 6 Nov 2025 22:12:39 +0400 Subject: [PATCH 445/524] Depricate macOS-13 gh actions (#1869) * upgrade abseil to August 2025, Patch 1 * migrate gh action macos-13 to macos-15 x86. --- ...le.yml => build-ton-macos-15-x86-64-portable.yml} | 12 ++++++------ .github/workflows/create-release.yml | 4 ++-- .github/workflows/create-tolk-release.yml | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) rename .github/workflows/{build-ton-macos-13-x86-64-portable.yml => build-ton-macos-15-x86-64-portable.yml} (80%) diff --git a/.github/workflows/build-ton-macos-13-x86-64-portable.yml b/.github/workflows/build-ton-macos-15-x86-64-portable.yml similarity index 80% rename from .github/workflows/build-ton-macos-13-x86-64-portable.yml rename to .github/workflows/build-ton-macos-15-x86-64-portable.yml index 7bf760c3e3..3d01c98545 100644 --- a/.github/workflows/build-ton-macos-13-x86-64-portable.yml +++ b/.github/workflows/build-ton-macos-15-x86-64-portable.yml @@ -1,10 +1,10 @@ -name: MacOS-13 TON build (portable, x86-64) +name: MacOS-15 TON build (portable, x86-64) on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: - runs-on: macos-13 + runs-on: macos-15-intel steps: - name: Check out repository @@ -28,15 +28,15 @@ jobs: uses: actions/cache@v4 with: path: 3pp - key: ${{ runner.os }}-${{ runner.arch }}-13-3pp-${{ hashFiles('**/assembly/native/build-macos-portable.sh') }} + key: ${{ runner.os }}-${{ runner.arch }}-15-3pp-${{ hashFiles('**/assembly/native/build-macos-portable.sh') }} - name: Cache TON test id: cache-ton uses: actions/cache@v4 with: path: ~/.ccache - key: ${{ runner.os }}-${{ runner.arch }}-13-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} - restore-keys: ${{ runner.os }}-${{ runner.arch }}-13-portable-ccache + key: ${{ runner.os }}-${{ runner.arch }}-15-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-15-portable-ccache - name: Build TON run: | @@ -44,7 +44,7 @@ jobs: git submodule update cp assembly/native/build-macos-portable.sh . chmod +x build-macos-portable.sh - ./build-macos-portable.sh -t -a -c -o 13.0 + ./build-macos-portable.sh -t -a -c -o 15.0 ccache -sp - name: Run Tests diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 09e1c632d6..a09f0fc9bf 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -53,7 +53,7 @@ jobs: - name: Download Mac x86-64 artifacts uses: dawidd6/action-download-artifact@v6 with: - workflow: build-ton-macos-13-x86-64-portable.yml + workflow: build-ton-macos-15-x86-64-portable.yml path: artifacts workflow_conclusion: success branch: master @@ -71,7 +71,7 @@ jobs: - name: Download and unzip Mac x86-64 artifacts uses: dawidd6/action-download-artifact@v6 with: - workflow: build-ton-macos-13-x86-64-portable.yml + workflow: build-ton-macos-15-x86-64-portable.yml path: artifacts workflow_conclusion: success branch: master diff --git a/.github/workflows/create-tolk-release.yml b/.github/workflows/create-tolk-release.yml index 963768444a..fbbbe23056 100644 --- a/.github/workflows/create-tolk-release.yml +++ b/.github/workflows/create-tolk-release.yml @@ -40,7 +40,7 @@ jobs: - name: Download and unzip Mac x86-64 artifacts uses: dawidd6/action-download-artifact@v6 with: - workflow: build-ton-macos-13-x86-64-portable.yml + workflow: build-ton-macos-15-x86-64-portable.yml path: artifacts workflow_conclusion: success branch: master From a420b83719a9ac2e895bef08781b4a897a5c3c74 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 7 Nov 2025 15:36:16 +0300 Subject: [PATCH 446/524] Remove FullNodePrivateBlockOverlay --- validator/CMakeLists.txt | 4 +- validator/full-node-custom-overlays.cpp | 250 +++++++++ ...rlay.hpp => full-node-custom-overlays.hpp} | 84 --- validator/full-node-private-overlay.cpp | 529 ------------------ validator/full-node.cpp | 116 +--- validator/full-node.h | 2 +- validator/full-node.hpp | 9 +- validator/validator-group.cpp | 12 +- 8 files changed, 289 insertions(+), 717 deletions(-) create mode 100644 validator/full-node-custom-overlays.cpp rename validator/{full-node-private-overlay.hpp => full-node-custom-overlays.hpp} (54%) delete mode 100644 validator/full-node-private-overlay.cpp diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index f0af651877..dd4b84d1c6 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -155,8 +155,8 @@ set(FULL_NODE_SOURCE full-node-master.h full-node-master.hpp full-node-master.cpp - full-node-private-overlay.hpp - full-node-private-overlay.cpp + full-node-custom-overlays.hpp + full-node-custom-overlays.cpp full-node-serializer.hpp full-node-serializer.cpp full-node-fast-sync-overlays.hpp diff --git a/validator/full-node-custom-overlays.cpp b/validator/full-node-custom-overlays.cpp new file mode 100644 index 0000000000..7cd4182eb7 --- /dev/null +++ b/validator/full-node-custom-overlays.cpp @@ -0,0 +1,250 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "auto/tl/ton_api_json.h" +#include "common/checksum.h" +#include "common/delay.h" +#include "td/utils/JsonBuilder.h" +#include "tl/tl_json.h" +#include "ton/ton-tl.hpp" + +#include "full-node-custom-overlays.hpp" +#include "full-node-serializer.hpp" + +namespace ton::validator::fullnode { + +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { + process_block_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) { + process_block_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query) { + process_block_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { + VLOG(FULL_NODE_DEBUG) << "Dropping block broadcast in private overlay \"" << name_ << "\" from unauthorized sender " + << src; + return; + } + auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); + if (B.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Received block broadcast in custom overlay \"" << name_ << "\" from " << src << ": " + << B.ok().block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); +} + +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query) { + auto it = msg_senders_.find(adnl::AdnlNodeIdShort{src}); + if (it == msg_senders_.end()) { + VLOG(FULL_NODE_DEBUG) << "Dropping external message broadcast in custom overlay \"" << name_ + << "\" from unauthorized sender " << src; + return; + } + VLOG(FULL_NODE_DEBUG) << "Got external message in custom overlay \"" << name_ << "\" from " << src + << " (priority=" << it->second << ")"; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_external_message, + std::move(query.message_->data_), it->second); +} + +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { + VLOG(FULL_NODE_DEBUG) << "Dropping block candidate broadcast in private overlay \"" << name_ + << "\" from unauthorized sender " << src; + return; + } + BlockIdExt block_id; + CatchainSeqno cc_seqno; + td::uint32 validator_set_hash; + td::BufferSlice data; + auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, + overlay::Overlays::max_fec_broadcast_size()); + if (S.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << S; + return; + } + if (data.size() > FullNode::max_block_size()) { + VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; + return; + } + if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { + VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; + return; + } + VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in custom overlay \"" << name_ << "\" from " << src << ": " + << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, + validator_set_hash, std::move(data)); +} + +void FullNodeCustomOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { + if (adnl::AdnlNodeIdShort{src} == local_id_) { + return; + } + auto B = fetch_tl_object(std::move(broadcast), true); + if (B.is_error()) { + return; + } + ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); }); +} + +void FullNodeCustomOverlay::send_external_message(td::BufferSlice data) { + if (!inited_ || opts_.config_.ext_messages_broadcast_disabled_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending external message to custom overlay \"" << name_ << "\""; + auto B = create_serialize_tl_object( + create_tl_object(std::move(data))); + if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(B)); + } else { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(B)); + } +} + +void FullNodeCustomOverlay::send_broadcast(BlockBroadcast broadcast) { + if (!inited_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending block broadcast to custom overlay \"" << name_ + << "\": " << broadcast.block_id.to_str(); + auto B = serialize_block_broadcast(broadcast, true); // compression_enabled = true + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); + return; + } + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); +} + +void FullNodeCustomOverlay::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + if (!inited_) { + return; + } + auto B = + serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in custom overlay \"" << name_ << "\": " << block_id.to_str(); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); +} + +void FullNodeCustomOverlay::start_up() { + std::sort(nodes_.begin(), nodes_.end()); + nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end()); + std::vector nodes; + for (const adnl::AdnlNodeIdShort &id : nodes_) { + nodes.push_back(id.bits256_value()); + } + auto X = create_hash_tl_object(zero_state_file_hash_, name_, std::move(nodes)); + td::BufferSlice b{32}; + b.as_slice().copy_from(as_slice(X)); + overlay_id_full_ = overlay::OverlayIdFull{std::move(b)}; + overlay_id_ = overlay_id_full_.compute_short_id(); + try_init(); +} + +void FullNodeCustomOverlay::try_init() { + // Sometimes adnl id is added to validator engine later (or not at all) + td::actor::send_closure( + adnl_, &adnl::Adnl::check_id_exists, local_id_, [SelfId = actor_id(this)](td::Result R) { + if (R.is_ok() && R.ok()) { + td::actor::send_closure(SelfId, &FullNodeCustomOverlay::init); + } else { + delay_action([SelfId]() { td::actor::send_closure(SelfId, &FullNodeCustomOverlay::try_init); }, + td::Timestamp::in(30.0)); + } + }); +} + +void FullNodeCustomOverlay::init() { + LOG(FULL_NODE_WARNING) << "Creating custom overlay \"" << name_ << "\" for adnl id " << local_id_ << " : " + << nodes_.size() << " nodes, " << msg_senders_.size() << " msg senders, " + << block_senders_.size() << " block senders, overlay_id=" << overlay_id_; + class Callback : public overlay::Overlays::Callback { + public: + void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + } + void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + td::actor::send_closure(node_, &FullNodeCustomOverlay::receive_broadcast, src, std::move(data)); + } + void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + } + Callback(td::actor::ActorId node) : node_(node) { + } + + private: + td::actor::ActorId node_; + }; + + std::map authorized_keys; + for (const auto &sender : msg_senders_) { + authorized_keys[sender.first.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size(); + } + for (const auto &sender : block_senders_) { + authorized_keys[sender.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size(); + } + overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), 0, std::move(authorized_keys)}; + overlay::OverlayOptions overlay_options; + overlay_options.broadcast_speed_multiplier_ = opts_.private_broadcast_speed_multiplier_; + td::actor::send_closure( + overlays_, &overlay::Overlays::create_private_overlay_ex, local_id_, overlay_id_full_.clone(), nodes_, + std::make_unique(actor_id(this)), rules, + PSTRING() << R"({ "type": "custom-overlay", "name": ")" << td::format::Escaped{name_} << R"(" })", + overlay_options); + + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); + td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); + inited_ = true; +} + +void FullNodeCustomOverlay::tear_down() { + LOG(FULL_NODE_WARNING) << "Destroying custom overlay \"" << name_ << "\" for adnl id " << local_id_; + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_); +} + +} // namespace ton::validator::fullnode diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-custom-overlays.hpp similarity index 54% rename from validator/full-node-private-overlay.hpp rename to validator/full-node-custom-overlays.hpp index 5c620ec6fe..90c8d403b7 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-custom-overlays.hpp @@ -23,90 +23,6 @@ namespace ton::validator::fullnode { -class FullNodePrivateBlockOverlay : public td::actor::Actor { - public: - void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); - void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); - void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query); - void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); - - void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); - - void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); - void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); - void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query); - void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); - - void process_telemetry_broadcast(PublicKeyHash src, const tl_object_ptr &telemetry); - - template - void process_broadcast(PublicKeyHash, T &) { - VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; - } - void receive_broadcast(PublicKeyHash src, td::BufferSlice query); - - void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data); - void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data); - void send_broadcast(BlockBroadcast broadcast); - void send_validator_telemetry(tl_object_ptr telemetry); - - void collect_validator_telemetry(std::string filename); - - void set_config(FullNodeConfig config) { - opts_.config_ = std::move(config); - } - - void start_up() override; - void tear_down() override; - - FullNodePrivateBlockOverlay(adnl::AdnlNodeIdShort local_id, std::vector nodes, - FileHash zero_state_file_hash, FullNodeOptions opts, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId rldp2, - td::actor::ActorId overlays, - td::actor::ActorId validator_manager, - td::actor::ActorId full_node) - : local_id_(local_id) - , nodes_(std::move(nodes)) - , zero_state_file_hash_(zero_state_file_hash) - , opts_(opts) - , keyring_(keyring) - , adnl_(adnl) - , rldp_(rldp) - , rldp2_(rldp2) - , overlays_(overlays) - , validator_manager_(validator_manager) - , full_node_(full_node) { - } - - private: - adnl::AdnlNodeIdShort local_id_; - std::vector nodes_; - FileHash zero_state_file_hash_; - FullNodeOptions opts_; - bool enable_compression_ = true; - - td::actor::ActorId keyring_; - td::actor::ActorId adnl_; - td::actor::ActorId rldp_; - td::actor::ActorId rldp2_; - td::actor::ActorId overlays_; - td::actor::ActorId validator_manager_; - td::actor::ActorId full_node_; - - bool inited_ = false; - overlay::OverlayIdFull overlay_id_full_; - overlay::OverlayIdShort overlay_id_; - - void try_init(); - void init(); - - td::actor::ActorOwn telemetry_sender_; - bool collect_telemetry_ = false; - std::ofstream telemetry_file_; -}; - class FullNodeCustomOverlay : public td::actor::Actor { public: void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp deleted file mode 100644 index b62a7c190c..0000000000 --- a/validator/full-node-private-overlay.cpp +++ /dev/null @@ -1,529 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . -*/ -#include "auto/tl/ton_api_json.h" -#include "common/checksum.h" -#include "common/delay.h" -#include "td/utils/JsonBuilder.h" -#include "tl/tl_json.h" -#include "ton/ton-tl.hpp" - -#include "full-node-private-overlay.hpp" -#include "full-node-serializer.hpp" - -namespace ton::validator::fullnode { - -void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { - process_block_broadcast(src, query); -} - -void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, - ton_api::tonNode_blockBroadcastCompressed &query) { - process_block_broadcast(src, query); -} - -void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, - ton_api::tonNode_blockBroadcastCompressedV2 &query) { - process_block_broadcast(src, query); -} - -void FullNodePrivateBlockOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { - auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); - if (B.is_error()) { - LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); - return; - } - VLOG(FULL_NODE_DEBUG) << "Received block broadcast in private overlay from " << src << ": " - << B.ok().block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); -} - -void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { - BlockIdExt block_id = create_block_id(query.block_->block_); - VLOG(FULL_NODE_DEBUG) << "Received newShardBlockBroadcast in private overlay from " << src << ": " - << block_id.to_str(); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block_description_broadcast, - block_id, query.block_->cc_seqno_, std::move(query.block_->data_)); -} - -void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, - ton_api::tonNode_newBlockCandidateBroadcast &query) { - process_block_candidate_broadcast(src, query); -} - -void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, - ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { - process_block_candidate_broadcast(src, query); -} - -void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, - ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query) { - process_block_candidate_broadcast(src, query); -} - -void FullNodePrivateBlockOverlay::process_block_candidate_broadcast(PublicKeyHash src, - ton_api::tonNode_Broadcast &query) { - BlockIdExt block_id; - CatchainSeqno cc_seqno; - td::uint32 validator_set_hash; - td::BufferSlice data; - auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, - overlay::Overlays::max_fec_broadcast_size()); - if (S.is_error()) { - LOG(DEBUG) << "dropped broadcast: " << S; - return; - } - if (data.size() > FullNode::max_block_size()) { - VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; - return; - } - if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { - VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; - return; - } - VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in private overlay from " << src << ": " << block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, - validator_set_hash, std::move(data)); -} - -void FullNodePrivateBlockOverlay::process_telemetry_broadcast( - PublicKeyHash src, const tl_object_ptr &telemetry) { - if (telemetry->adnl_id_ != src.bits256_value()) { - VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": adnl_id mismatch"; - return; - } - auto now = (td::int32)td::Clocks::system(); - if (telemetry->timestamp_ < now - 60) { - VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": too old (" - << now - telemetry->timestamp_ << "s ago)"; - return; - } - if (telemetry->timestamp_ > now + 60) { - VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": too new (" - << telemetry->timestamp_ - now << "s in the future)"; - return; - } - VLOG(FULL_NODE_DEBUG) << "Got telemetry broadcast from " << src; - auto s = td::json_encode(td::ToJson(*telemetry), false); - std::erase_if(s, [](char c) { return c == '\n' || c == '\r'; }); - telemetry_file_ << s << "\n"; - telemetry_file_.flush(); - if (telemetry_file_.fail()) { - VLOG(FULL_NODE_WARNING) << "Failed to write telemetry to file"; - } -} - -void FullNodePrivateBlockOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { - if (adnl::AdnlNodeIdShort{src} == local_id_) { - return; - } - auto B = fetch_tl_object(std::move(broadcast), true); - if (B.is_error()) { - if (collect_telemetry_ && src != local_id_.pubkey_hash()) { - auto R = fetch_tl_prefix(broadcast, true); - if (R.is_ok()) { - process_telemetry_broadcast(src, R.ok()); - } - } - return; - } - ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); }); -} - -void FullNodePrivateBlockOverlay::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, - td::BufferSlice data) { - if (!inited_) { - return; - } - VLOG(FULL_NODE_DEBUG) << "Sending newShardBlockBroadcast in private overlay: " << block_id.to_str(); - auto B = create_serialize_tl_object( - create_tl_object(create_tl_block_id(block_id), cc_seqno, std::move(data))); - if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) { - td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), 0, std::move(B)); - } else { - td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); - } -} - -void FullNodePrivateBlockOverlay::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, - td::uint32 validator_set_hash, td::BufferSlice data) { - if (!inited_) { - return; - } - auto B = - serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled - if (B.is_error()) { - VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); - return; - } - VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in private overlay: " << block_id.to_str(); - td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); -} - -void FullNodePrivateBlockOverlay::send_broadcast(BlockBroadcast broadcast) { - if (!inited_) { - return; - } - VLOG(FULL_NODE_DEBUG) << "Sending block broadcast in private overlay" - << (enable_compression_ ? " (with compression)" : "") << ": " << broadcast.block_id.to_str(); - auto B = serialize_block_broadcast(broadcast, enable_compression_); - if (B.is_error()) { - VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); - return; - } - td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); -} - -void FullNodePrivateBlockOverlay::send_validator_telemetry(tl_object_ptr telemetry) { - if (collect_telemetry_) { - process_telemetry_broadcast(local_id_.pubkey_hash(), telemetry); - } - auto data = serialize_tl_object(telemetry, true); - if (data.size() <= overlay::Overlays::max_simple_broadcast_size()) { - td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), 0, std::move(data)); - } else { - td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), 0, std::move(data)); - } -} - -void FullNodePrivateBlockOverlay::collect_validator_telemetry(std::string filename) { - if (collect_telemetry_) { - telemetry_file_.close(); - } - collect_telemetry_ = true; - LOG(FULL_NODE_WARNING) << "Collecting validator telemetry to " << filename << " (local id: " << local_id_ << ")"; - telemetry_file_.open(filename, std::ios_base::app); - if (!telemetry_file_.is_open()) { - LOG(WARNING) << "Cannot open file " << filename << " for validator telemetry"; - } -} - -void FullNodePrivateBlockOverlay::start_up() { - std::sort(nodes_.begin(), nodes_.end()); - nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end()); - - std::vector nodes; - for (const adnl::AdnlNodeIdShort &id : nodes_) { - nodes.push_back(id.bits256_value()); - } - auto X = create_hash_tl_object(zero_state_file_hash_, std::move(nodes)); - td::BufferSlice b{32}; - b.as_slice().copy_from(as_slice(X)); - overlay_id_full_ = overlay::OverlayIdFull{std::move(b)}; - overlay_id_ = overlay_id_full_.compute_short_id(); - - try_init(); -} - -void FullNodePrivateBlockOverlay::try_init() { - // Sometimes adnl id is added to validator engine later (or not at all) - td::actor::send_closure( - adnl_, &adnl::Adnl::check_id_exists, local_id_, [SelfId = actor_id(this)](td::Result R) { - if (R.is_ok() && R.ok()) { - td::actor::send_closure(SelfId, &FullNodePrivateBlockOverlay::init); - } else { - delay_action([SelfId]() { td::actor::send_closure(SelfId, &FullNodePrivateBlockOverlay::try_init); }, - td::Timestamp::in(30.0)); - } - }); -} - -void FullNodePrivateBlockOverlay::init() { - LOG(FULL_NODE_WARNING) << "Creating private block overlay for adnl id " << local_id_ << " : " << nodes_.size() - << " nodes, overlay_id=" << overlay_id_; - class Callback : public overlay::Overlays::Callback { - public: - void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { - } - void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, - td::Promise promise) override { - } - void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { - td::actor::send_closure(node_, &FullNodePrivateBlockOverlay::receive_broadcast, src, std::move(data)); - } - void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, - td::Promise promise) override { - } - Callback(td::actor::ActorId node) : node_(node) { - } - - private: - td::actor::ActorId node_; - }; - - overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), - overlay::CertificateFlags::AllowFec | overlay::CertificateFlags::Trusted, - {}}; - overlay::OverlayOptions overlay_options; - overlay_options.broadcast_speed_multiplier_ = opts_.private_broadcast_speed_multiplier_; - overlay_options.private_ping_peers_ = true; - td::actor::send_closure(overlays_, &overlay::Overlays::create_private_overlay_ex, local_id_, overlay_id_full_.clone(), - nodes_, std::make_unique(actor_id(this)), rules, R"({ "type": "private-blocks" })", - overlay_options); - - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); - td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); - inited_ = true; - - class TelemetryCallback : public ValidatorTelemetry::Callback { - public: - explicit TelemetryCallback(td::actor::ActorId id) : id_(id) { - } - void send_telemetry(tl_object_ptr telemetry) override { - td::actor::send_closure(id_, &FullNodePrivateBlockOverlay::send_validator_telemetry, std::move(telemetry)); - } - - private: - td::actor::ActorId id_; - }; - telemetry_sender_ = td::actor::create_actor("telemetry", local_id_, - std::make_unique(actor_id(this))); -} - -void FullNodePrivateBlockOverlay::tear_down() { - if (inited_) { - td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_); - } -} - -void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { - process_block_broadcast(src, query); -} - -void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) { - process_block_broadcast(src, query); -} - -void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query) { - process_block_broadcast(src, query); -} - -void FullNodeCustomOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { - if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { - VLOG(FULL_NODE_DEBUG) << "Dropping block broadcast in private overlay \"" << name_ << "\" from unauthorized sender " - << src; - return; - } - auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); - if (B.is_error()) { - LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); - return; - } - VLOG(FULL_NODE_DEBUG) << "Received block broadcast in custom overlay \"" << name_ << "\" from " << src << ": " - << B.ok().block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); -} - -void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query) { - auto it = msg_senders_.find(adnl::AdnlNodeIdShort{src}); - if (it == msg_senders_.end()) { - VLOG(FULL_NODE_DEBUG) << "Dropping external message broadcast in custom overlay \"" << name_ - << "\" from unauthorized sender " << src; - return; - } - VLOG(FULL_NODE_DEBUG) << "Got external message in custom overlay \"" << name_ << "\" from " << src - << " (priority=" << it->second << ")"; - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_external_message, - std::move(query.message_->data_), it->second); -} - -void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { - process_block_candidate_broadcast(src, query); -} - -void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, - ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { - process_block_candidate_broadcast(src, query); -} - -void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, - ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query) { - process_block_candidate_broadcast(src, query); -} - -void FullNodeCustomOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { - if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { - VLOG(FULL_NODE_DEBUG) << "Dropping block candidate broadcast in private overlay \"" << name_ - << "\" from unauthorized sender " << src; - return; - } - BlockIdExt block_id; - CatchainSeqno cc_seqno; - td::uint32 validator_set_hash; - td::BufferSlice data; - auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, - overlay::Overlays::max_fec_broadcast_size()); - if (S.is_error()) { - LOG(DEBUG) << "dropped broadcast: " << S; - return; - } - if (data.size() > FullNode::max_block_size()) { - VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; - return; - } - if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { - VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; - return; - } - VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in custom overlay \"" << name_ << "\" from " << src << ": " - << block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, - validator_set_hash, std::move(data)); -} - -void FullNodeCustomOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { - if (adnl::AdnlNodeIdShort{src} == local_id_) { - return; - } - auto B = fetch_tl_object(std::move(broadcast), true); - if (B.is_error()) { - return; - } - ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); }); -} - -void FullNodeCustomOverlay::send_external_message(td::BufferSlice data) { - if (!inited_ || opts_.config_.ext_messages_broadcast_disabled_) { - return; - } - VLOG(FULL_NODE_DEBUG) << "Sending external message to custom overlay \"" << name_ << "\""; - auto B = create_serialize_tl_object( - create_tl_object(std::move(data))); - if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) { - td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), 0, std::move(B)); - } else { - td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), 0, std::move(B)); - } -} - -void FullNodeCustomOverlay::send_broadcast(BlockBroadcast broadcast) { - if (!inited_) { - return; - } - VLOG(FULL_NODE_DEBUG) << "Sending block broadcast to custom overlay \"" << name_ - << "\": " << broadcast.block_id.to_str(); - auto B = serialize_block_broadcast(broadcast, true); // compression_enabled = true - if (B.is_error()) { - VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); - return; - } - td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); -} - -void FullNodeCustomOverlay::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, - td::uint32 validator_set_hash, td::BufferSlice data) { - if (!inited_) { - return; - } - auto B = - serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled - if (B.is_error()) { - VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); - return; - } - VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in custom overlay \"" << name_ << "\": " << block_id.to_str(); - td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); -} - -void FullNodeCustomOverlay::start_up() { - std::sort(nodes_.begin(), nodes_.end()); - nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end()); - std::vector nodes; - for (const adnl::AdnlNodeIdShort &id : nodes_) { - nodes.push_back(id.bits256_value()); - } - auto X = create_hash_tl_object(zero_state_file_hash_, name_, std::move(nodes)); - td::BufferSlice b{32}; - b.as_slice().copy_from(as_slice(X)); - overlay_id_full_ = overlay::OverlayIdFull{std::move(b)}; - overlay_id_ = overlay_id_full_.compute_short_id(); - try_init(); -} - -void FullNodeCustomOverlay::try_init() { - // Sometimes adnl id is added to validator engine later (or not at all) - td::actor::send_closure( - adnl_, &adnl::Adnl::check_id_exists, local_id_, [SelfId = actor_id(this)](td::Result R) { - if (R.is_ok() && R.ok()) { - td::actor::send_closure(SelfId, &FullNodeCustomOverlay::init); - } else { - delay_action([SelfId]() { td::actor::send_closure(SelfId, &FullNodeCustomOverlay::try_init); }, - td::Timestamp::in(30.0)); - } - }); -} - -void FullNodeCustomOverlay::init() { - LOG(FULL_NODE_WARNING) << "Creating custom overlay \"" << name_ << "\" for adnl id " << local_id_ << " : " - << nodes_.size() << " nodes, " << msg_senders_.size() << " msg senders, " - << block_senders_.size() << " block senders, overlay_id=" << overlay_id_; - class Callback : public overlay::Overlays::Callback { - public: - void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { - } - void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, - td::Promise promise) override { - } - void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { - td::actor::send_closure(node_, &FullNodeCustomOverlay::receive_broadcast, src, std::move(data)); - } - void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, - td::Promise promise) override { - } - Callback(td::actor::ActorId node) : node_(node) { - } - - private: - td::actor::ActorId node_; - }; - - std::map authorized_keys; - for (const auto &sender : msg_senders_) { - authorized_keys[sender.first.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size(); - } - for (const auto &sender : block_senders_) { - authorized_keys[sender.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size(); - } - overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), 0, std::move(authorized_keys)}; - overlay::OverlayOptions overlay_options; - overlay_options.broadcast_speed_multiplier_ = opts_.private_broadcast_speed_multiplier_; - td::actor::send_closure( - overlays_, &overlay::Overlays::create_private_overlay_ex, local_id_, overlay_id_full_.clone(), nodes_, - std::make_unique(actor_id(this)), rules, - PSTRING() << R"({ "type": "custom-overlay", "name": ")" << td::format::Escaped{name_} << R"(" })", - overlay_options); - - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); - td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); - inited_ = true; -} - -void FullNodeCustomOverlay::tear_down() { - LOG(FULL_NODE_WARNING) << "Destroying custom overlay \"" << name_ << "\" for adnl id " << local_id_; - td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_); -} - -} // namespace ton::validator::fullnode diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 0c83e30b9a..a27555f9ad 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -41,7 +41,6 @@ void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise pr } local_keys_.insert(key); - create_private_block_overlay(key); for (auto &p : custom_overlays_) { update_custom_overlay(p.second); } @@ -71,7 +70,6 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr return; } local_keys_.erase(key); - private_block_overlays_.erase(key); update_validator_telemetry_collector(); for (auto &p : custom_overlays_) { update_custom_overlay(p.second); @@ -156,9 +154,6 @@ void FullNodeImpl::set_config(FullNodeConfig config) { td::actor::send_closure(s.second.actor, &FullNodeShard::set_config, config); } } - for (auto &overlay : private_block_overlays_) { - td::actor::send_closure(overlay.second, &FullNodePrivateBlockOverlay::set_config, config); - } for (auto &overlay : custom_overlays_) { for (auto &actor : overlay.second.actors_) { td::actor::send_closure(actor.second, &FullNodeCustomOverlay::set_config, config); @@ -263,31 +258,25 @@ void FullNodeImpl::on_new_masterchain_block(td::Ref state, std } } - int proto_version = state->get_consensus_config().proto_version; - use_old_private_overlays_ = (proto_version < 5); - - if (!use_old_private_overlays_) { - private_block_overlays_.clear(); - std::set my_adnl_ids; - my_adnl_ids.insert(adnl_id_); - for (const auto &[adnl_id, _] : local_collator_nodes_) { - my_adnl_ids.insert(adnl_id); - } - for (auto key : local_keys_) { - auto it = current_validators_.find(key); - if (it != current_validators_.end()) { - my_adnl_ids.insert(it->second); - } - } - std::set monitoring_shards; - for (ShardIdFull shard : shards_to_monitor) { - monitoring_shards.insert(cut_shard(shard)); + std::set my_adnl_ids; + my_adnl_ids.insert(adnl_id_); + for (const auto &[adnl_id, _] : local_collator_nodes_) { + my_adnl_ids.insert(adnl_id); + } + for (auto key : local_keys_) { + auto it = current_validators_.find(key); + if (it != current_validators_.end()) { + my_adnl_ids.insert(it->second); } - fast_sync_overlays_.update_overlays(state, std::move(my_adnl_ids), std::move(monitoring_shards), - zero_state_file_hash_, keyring_, adnl_, overlays_, validator_manager_, - actor_id(this)); - update_validator_telemetry_collector(); } + std::set monitoring_shards; + for (ShardIdFull shard : shards_to_monitor) { + monitoring_shards.insert(cut_shard(shard)); + } + fast_sync_overlays_.update_overlays(state, std::move(my_adnl_ids), std::move(monitoring_shards), + zero_state_file_hash_, keyring_, adnl_, overlays_, validator_manager_, + actor_id(this)); + update_validator_telemetry_collector(); } void FullNodeImpl::update_shard_actor(ShardIdFull shard, bool active) { @@ -349,10 +338,6 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; return; } - if (!private_block_overlays_.empty()) { - td::actor::send_closure(private_block_overlays_.begin()->second, - &FullNodePrivateBlockOverlay::send_shard_block_info, block_id, cc_seqno, data.clone()); - } auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(ShardIdFull(masterchainId)).first; if (!fast_sync_overlay.empty()) { td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_shard_block_info, block_id, cc_seqno, @@ -366,12 +351,7 @@ void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_se if (mode & broadcast_mode_custom) { send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); } - if (mode & broadcast_mode_private_block) { - if (!private_block_overlays_.empty()) { - td::actor::send_closure(private_block_overlays_.begin()->second, - &FullNodePrivateBlockOverlay::send_block_candidate, block_id, cc_seqno, - validator_set_hash, data.clone()); - } + if (mode & broadcast_mode_fast_sync) { auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(block_id.shard_full()).first; if (!fast_sync_overlay.empty()) { td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_block_candidate, block_id, cc_seqno, @@ -401,11 +381,7 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, int mode) { if (mode & broadcast_mode_custom) { send_block_broadcast_to_custom_overlays(broadcast); } - if (mode & broadcast_mode_private_block) { - if (!private_block_overlays_.empty()) { - td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, - broadcast.clone()); - } + if (mode & broadcast_mode_fast_sync) { auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(broadcast.block_id.shard_full()).first; if (!fast_sync_overlay.empty()) { td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_broadcast, broadcast.clone()); @@ -660,31 +636,18 @@ void FullNodeImpl::set_validator_telemetry_filename(std::string value) { } void FullNodeImpl::update_validator_telemetry_collector() { - if (use_old_private_overlays_) { - if (validator_telemetry_filename_.empty() || private_block_overlays_.empty()) { - validator_telemetry_collector_key_ = PublicKeyHash::zero(); - return; - } - if (!private_block_overlays_.contains(validator_telemetry_collector_key_)) { - auto it = private_block_overlays_.begin(); - validator_telemetry_collector_key_ = it->first; - td::actor::send_closure(it->second, &FullNodePrivateBlockOverlay::collect_validator_telemetry, + if (validator_telemetry_filename_.empty()) { + validator_telemetry_collector_key_ = PublicKeyHash::zero(); + return; + } + if (fast_sync_overlays_.get_masterchain_overlay_for(adnl::AdnlNodeIdShort{validator_telemetry_collector_key_}) + .empty()) { + auto [actor, adnl_id] = fast_sync_overlays_.choose_overlay(ShardIdFull{masterchainId}); + validator_telemetry_collector_key_ = adnl_id.pubkey_hash(); + if (!actor.empty()) { + td::actor::send_closure(actor, &FullNodeFastSyncOverlay::collect_validator_telemetry, validator_telemetry_filename_); } - } else { - if (validator_telemetry_filename_.empty()) { - validator_telemetry_collector_key_ = PublicKeyHash::zero(); - return; - } - if (fast_sync_overlays_.get_masterchain_overlay_for(adnl::AdnlNodeIdShort{validator_telemetry_collector_key_}) - .empty()) { - auto [actor, adnl_id] = fast_sync_overlays_.choose_overlay(ShardIdFull{masterchainId}); - validator_telemetry_collector_key_ = adnl_id.pubkey_hash(); - if (!actor.empty()) { - td::actor::send_closure(actor, &FullNodeFastSyncOverlay::collect_validator_telemetry, - validator_telemetry_filename_); - } - } } } @@ -789,31 +752,10 @@ void FullNodeImpl::update_private_overlays() { update_custom_overlay(p.second); } - private_block_overlays_.clear(); update_validator_telemetry_collector(); if (local_keys_.empty()) { return; } - for (const auto &key : local_keys_) { - create_private_block_overlay(key); - } -} - -void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { - if (!use_old_private_overlays_) { - return; - } - CHECK(local_keys_.count(key)); - if (current_validators_.count(key)) { - std::vector nodes; - for (const auto &p : current_validators_) { - nodes.push_back(p.second); - } - private_block_overlays_[key] = td::actor::create_actor( - "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, opts_, keyring_, - adnl_, rldp_, rldp2_, overlays_, validator_manager_, actor_id(this)); - update_validator_telemetry_collector(); - } } void FullNodeImpl::update_custom_overlay(CustomOverlayInfo &overlay) { diff --git a/validator/full-node.h b/validator/full-node.h index 9ab9c39a56..a69a75ba04 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -114,7 +114,7 @@ class FullNode : public td::actor::Actor { static constexpr td::uint64 max_state_size() { return 4ull << 30; } - enum { broadcast_mode_public = 1, broadcast_mode_private_block = 2, broadcast_mode_custom = 4 }; + enum { broadcast_mode_public = 1, broadcast_mode_fast_sync = 2, broadcast_mode_custom = 4 }; static constexpr td::int32 MAX_FAST_SYNC_OVERLAY_CLIENTS = 5; diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 11ffabd94a..ecc446da37 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -29,8 +29,8 @@ #include "interfaces/proof.h" #include "interfaces/shard.h" +#include "full-node-custom-overlays.hpp" #include "full-node-fast-sync-overlays.hpp" -#include "full-node-private-overlay.hpp" namespace ton { @@ -156,12 +156,6 @@ class FullNodeImpl : public FullNode { td::Promise started_promise_; FullNodeOptions opts_; - // Private overlays: - // Old overlays - one private overlay for all validators - // New overlays (fast sync overlays) - semiprivate overlay per shard (monitor_min_split depth) - // for validators and authorized nodes - bool use_old_private_overlays_ = true; - std::map> private_block_overlays_; FullNodeFastSyncOverlays fast_sync_overlays_; struct CustomOverlayInfo { @@ -173,7 +167,6 @@ class FullNodeImpl : public FullNode { std::queue custom_overlays_sent_broadcasts_lru_; void update_private_overlays(); - void create_private_block_overlay(PublicKeyHash key); void update_custom_overlay(CustomOverlayInfo& overlay); void send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast); void send_block_candidate_broadcast_to_custom_overlays(const BlockIdExt& block_id, CatchainSeqno cc_seqno, diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index aef4e0cc97..be91ba8617 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -291,7 +291,7 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so if (source_info.source.compute_short_id() == local_id_) { send_broadcast_mode |= fullnode::FullNode::broadcast_mode_public; if (!sent_candidate) { - send_broadcast_mode |= fullnode::FullNode::broadcast_mode_private_block; + send_broadcast_mode |= fullnode::FullNode::broadcast_mode_fast_sync; } } if (!sent_candidate) { @@ -307,7 +307,7 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so if (!shard_.is_masterchain() || source_info.source.compute_short_id() == local_id_) { send_broadcast_mode |= fullnode::FullNode::broadcast_mode_public; if (!sent_candidate) { - send_broadcast_mode |= fullnode::FullNode::broadcast_mode_private_block; + send_broadcast_mode |= fullnode::FullNode::broadcast_mode_fast_sync; } } if (!sent_candidate) { @@ -726,10 +726,10 @@ void ValidatorGroup::get_validator_group_info_for_litequery_cont( void ValidatorGroup::send_block_candidate_broadcast(BlockIdExt id, td::BufferSlice data) { if (sent_candidate_broadcasts_.insert(id).second) { - td::actor::send_closure( - manager_, &ValidatorManager::send_block_candidate_broadcast, id, validator_set_->get_catchain_seqno(), - validator_set_->get_validator_set_hash(), std::move(data), - fullnode::FullNode::broadcast_mode_private_block | fullnode::FullNode::broadcast_mode_custom); + td::actor::send_closure(manager_, &ValidatorManager::send_block_candidate_broadcast, id, + validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), + std::move(data), + fullnode::FullNode::broadcast_mode_fast_sync | fullnode::FullNode::broadcast_mode_custom); } } From 460200d01c018b8f77874ed90a35ac5ed1c49c42 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Sat, 8 Nov 2025 12:38:19 +0400 Subject: [PATCH 447/524] Merge pull request #1879 from neodix42/fix-android-build-1 Fix android tonlib GH build by upgrading Android NDK r25b->r27d --- assembly/android/build-android-tonlib.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/assembly/android/build-android-tonlib.sh b/assembly/android/build-android-tonlib.sh index 29dbffc88f..dda74c1664 100644 --- a/assembly/android/build-android-tonlib.sh +++ b/assembly/android/build-android-tonlib.sh @@ -12,12 +12,12 @@ export CC=$(which clang-16) export CXX=$(which clang++-16) export CCACHE_DISABLE=1 -if [ ! -d android-ndk-r25b ]; then - rm android-ndk-r25b-linux.zip - echo "Downloading https://dl.google.com/android/repository/android-ndk-r25b-linux.zip" - wget -q https://dl.google.com/android/repository/android-ndk-r25b-linux.zip - unzip -q android-ndk-r25b-linux.zip - test $? -eq 0 || { echo "Can't unzip android-ndk-r25b-linux.zip"; exit 1; } +if [ ! -d android-ndk-r27d ]; then + rm android-ndk-r27d-linux.zip + echo "Downloading https://dl.google.com/android/repository/android-ndk-r27d-linux.zip" + wget -q https://dl.google.com/android/repository/android-ndk-r27d-linux.zip + unzip -q android-ndk-r27d-linux.zip + test $? -eq 0 || { echo "Can't unzip android-ndk-r27d-linux.zip"; exit 1; } echo "Android NDK extracted" else echo "Using extracted Android NDK" @@ -29,13 +29,13 @@ export JAVA_INCLUDE_PATH=${JAVA_HOME}/include export JAVA_AWT_INCLUDE_PATH=${JAVA_HOME}/include export JAVA_INCLUDE_PATH2=${JAVA_HOME}/include/linux -export ANDROID_NDK_ROOT=$(pwd)/android-ndk-r25b +export ANDROID_NDK_ROOT=$(pwd)/android-ndk-r27d export NDK_PLATFORM="android-21" export ANDROID_PLATFORM="android-21" export OPENSSL_DIR=$(pwd)/example/android/third_party/crypto rm -rf example/android/src/drinkless/org/ton/TonApi.java -cd example/android/ +cd example/android/ || exit rm CMakeCache.txt .ninja_* cmake -GNinja -DTON_ONLY_TONLIB=ON . From 90267652ad79ab1492c87864ee8096da42c4c716 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Tue, 2 Sep 2025 16:43:55 +0300 Subject: [PATCH 448/524] [Tolk] Rich bounces for createMessage() from TVM 12 TVM 12 introduces "rich bounces" (returning full body instead of only 256 bits). Here we update stdlib and createMessage() to support sending and parsing them. --- crypto/smartcont/tolk-stdlib/common.tolk | 81 ++++++++++++++++++++++-- tolk-tester/tests/send-msg-1.tolk | 45 ++++++++++--- tolk-tester/tests/send-msg-2.tolk | 24 ++++--- tolk/send-message-api.cpp | 66 +++++++++++++++++-- 4 files changed, 186 insertions(+), 30 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index e0a81bfb9c..d02803f675 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -1410,11 +1410,40 @@ fun AutoDeployAddress.buildAddress(self): builder fun AutoDeployAddress.addressMatches(self, addr: address): bool builtin +/// `BounceMode` represents behavior of an outgoing message when a receiver failed to handle it. +/// If you specify to bounce a message, don't forget about the `onBouncedMessage` entrypoint. +/// ``` +/// val outMsg = createMessage({ +/// bounce: BounceMode.Only256BitsOfBody, +/// ... +/// }); +/// +/// fun onBouncedMessage(in: InMessageBounced) { +/// // in.bouncedBody is populated depending on BounceMode +/// // for instance, "Only256BitsOfBody" means +/// // "bounced prefix 0xFFFFFFFF" and 256 bits of data +/// in.bouncedBody.skipBouncedPrefix(); +/// val originalOpcode = in.bouncedBody.loadUint(32); +/// ... +/// } +/// ``` +enum BounceMode { + NoBounce // a message will just disappear on error + Only256BitsOfBody // `in.bouncedBody` will be "0xFFFFFFFF" and the first 256 bits of outgoing body (most cheap) + RichBounce // `in.bouncedBody` will be struct RichBounceBody (most expensive, but allows accessing all data sent) + RichBounceOnlyRootCell // `in.bouncedBody` will be struct RichBounceBody without refs in `originalBody` +} + +/// Before TVM-12, `bounce` was only true or false. +/// Now, use `enum BounceMode` instead. "true" behaves like `Only256BitsOfBody`. +@deprecated("use `BounceMode.NoBounce` instead of `false`") +type OldBounceMode = bool + /// Options for creating an outgoing message. /// Consider [createMessage] for examples. struct CreateMessageOptions { - /// whether a message will bounce back on error - bounce: bool + /// whether a message will bounce back on error, and what piece of outgoing `body` it will contain + bounce: BounceMode | OldBounceMode /// message value: attached tons (or tons + extra currencies) value: coins | (coins, ExtraCurrenciesMap) /// destination is either a provided address, or is auto-calculated by stateInit @@ -1430,7 +1459,7 @@ struct CreateMessageOptions { /// Typically, you just send it. In advanced scenarios, you can estimate fees or even postpone sending. /// ``` /// val reply = createMessage({ -/// bounce: false, +/// bounce: BounceMode.NoBounce, /// value: ton("0.05"), /// dest: senderAddress, /// body: RequestedInfo { ... } // note: no toCell! just pass an object @@ -1675,10 +1704,12 @@ struct InMessage { /// `InMessageBounced` is an input for `onBouncedMessage`. /// Very similar to a non-bounced input [InMessage]. -/// Note, that `bouncedBody` is currently 256 bits: 0xFFFFFFFF + 224 bits from the originally sent message. -/// Parse it with care! +/// Note that `bouncedBody` is populated depending on [BounceMode] specified for [createMessage]. /// ``` /// fun onBouncedMessage(in: InMessageBounced) { +/// // in.bouncedBody is populated depending on BounceMode +/// // for instance, "Only256BitsOfBody" means +/// // "bounced prefix 0xFFFFFFFF" and 256 bits of data /// in.bouncedBody.skipBouncedPrefix(); /// val originalOpcode = in.bouncedBody.loadUint(32); /// ``` @@ -1689,10 +1720,48 @@ struct InMessageBounced { originalForwardFee: coins // comission that the sender has payed to send this message createdLt: uint64 // logical time when a message was created (and bounced) createdAt: uint32 // unixtime when a message was created (and bounced) - bouncedBody: slice // currently 256 bits: 0xFFFFFFFF + 224 bits sent originally + bouncedBody: slice // populated depending on `BounceMode` for `createMessage` +} + +/// When you use [createMessage] with "rich bounce mode", and a message is bounced, +/// then `in.bouncedBody` inside `onBouncedMessage` has this format. +/// ``` +/// fun onBouncedMessage(in: InMessageBounced) { +/// // suppose you ALWAYS use BounceMode.RichBounce in createMessage +/// // so you parse ALL bounces as follows +/// val rich = lazy RichBounceBody.fromSlice(in.bouncedBody); +/// // handle rich.originalBody, probably with lazy match +/// // use rich.xxx to get exitCode, gasUsed, and so on +/// ``` +struct (0xfffffffe) RichBounceBody { + originalBody: cell // a cell with original body, but for `RichBounceOnlyRootCell` without refs (cheaper) + originalInfo: Cell // some fields of an original (outgoing) message + bouncedByPhase: uint8 // see docs + exitCode: int32 // exception code from `throw` or TVM internals + computePhase: RichBounceComputePhaseInfo? // filled if compute phase executed +} + +/// Describes compute phase details of a bounced message. +struct RichBounceComputePhaseInfo { + gasUsed: uint32 + vmSteps: uint32 +} + +/// Describes original outgoing message fields (besides body) of a bounced message. +/// ``` +/// createMessage({ +/// bounce: BounceMode.RichBounce, +/// value: ton("0.05") // it will be `valueCoins` if this message is bounced +/// ``` +struct RichBounceOriginalMsgInfo { + valueCoins: coins + valueExtra: ExtraCurrenciesMap + createdLt: uint64 + createdAt: uint32 } /// Skip 0xFFFFFFFF prefix (when a message is bounced). +/// Used when parsing `in.bounceBody` for "not rich" `BounceMode.Only256BitsOfBody` bounces. @pure fun slice.skipBouncedPrefix(mutate self): self asm "32 LDU" "NIP" diff --git a/tolk-tester/tests/send-msg-1.tolk b/tolk-tester/tests/send-msg-1.tolk index c15b2d356d..6f7646ce0a 100644 --- a/tolk-tester/tests/send-msg-1.tolk +++ b/tolk-tester/tests/send-msg-1.tolk @@ -1,5 +1,4 @@ import "@stdlib/tvm-dicts" -import "@stdlib/tvm-lowlevel" /* int_msg_info$0 1 @@ -9,7 +8,7 @@ int_msg_info$0 1 src:MsgAddress // always 00 on send 2 dest:MsgAddressInt // parameter 267 value:CurrencyCollection // parameter 124 + 1 - ihr_fee:Grams // always 0, not implemented 4 + extra_flags:Grams // if new bounces enabled 12 fwd_fee:Grams // always 0 on send 4 created_lt:uint64 // always 0 on send 64 created_at:uint32 // always 0 on send 32 @@ -21,7 +20,7 @@ _ split_depth:(Maybe (## 5)) 1 + library:(Maybe ^Cell) 1 = StateInit; message$_ {X:Type} - info:CommonMsgInfoRelaxed 502 + info:CommonMsgInfoRelaxed 510 init:(Maybe (Either StateInit ^StateInit)) 12 body:(Either X ^X) // body is either embedded or stored as ref = MessageRelaxed X; @@ -79,7 +78,7 @@ fun test1() { val body: MyBody = { queryId: 800 }; var b = createMessage({ body: body, - bounce: true, + bounce: BounceMode.Only256BitsOfBody, dest: address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), value: 123, }); @@ -105,7 +104,7 @@ fun test2() { val body: MyBody = { queryId: 127493264572 }; var b = createMessage({ body, - bounce: false, + bounce: BounceMode.NoBounce, dest: address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), value: 90, }); @@ -191,7 +190,7 @@ fun test5() { val body: MyBody = { queryId: 800 }; var b = createMessage({ body, - bounce: true, + bounce: BounceMode.Only256BitsOfBody, dest: address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), value: (123, ec_dict), }); @@ -348,7 +347,7 @@ fun test10_manual(bd: Body500Bits) { @method_id(110) fun test10(bd: Body500Bits) { var b = createMessage({ - bounce: false, + bounce: BounceMode.NoBounce, body: bd, dest: address("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"), value: 1 << 118, @@ -374,7 +373,7 @@ fun test11_manual(bd: Body750Bits) { @method_id(111) fun test11(bd: Body750Bits) { var b = createMessage({ - bounce: false, + bounce: BounceMode.NoBounce, body: bd, dest: address("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"), value: 1 << 110, @@ -724,6 +723,35 @@ fun test24() { return b.hash(); } +@noinline +fun test25_manual() { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA")) + .storeCoins(123) // value.grams + .storeUint(0, 1) // no extra currencies + .storeCoins(3) // extra_flags for bounces + .storeUint(0, 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + // MyBody: op + queryId + .storeUint(0x12345678, 32) + .storeUint(800, 64) + .endCell(); +} + +@method_id(125) +fun test25() { + val body: MyBody = { queryId: 800 }; + var b = createMessage({ + body: body, + bounce: BounceMode.RichBounce, + dest: address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), + value: 123, + }); + assert(b.hash() == test25_manual().hash(), 101); + return b.hash(); +} + + fun main() { } @@ -752,4 +780,5 @@ fun main() { @testcase | 122 | 88 12892 | 104095783372529353117379287284649790470404208891518326338200063605256169961669 @testcase | 123 | 999 | 93691386126134034953606952841435771108648354412078750046821446829279390964584 @testcase | 124 | | 21886688052816798288463190773103865772534937765373272039597499398551023701577 +@testcase | 125 | | 12745496241890009334837209777890803918215266832624965751220009451486906344811 */ diff --git a/tolk-tester/tests/send-msg-2.tolk b/tolk-tester/tests/send-msg-2.tolk index 050cd84322..76580e5220 100644 --- a/tolk-tester/tests/send-msg-2.tolk +++ b/tolk-tester/tests/send-msg-2.tolk @@ -6,7 +6,7 @@ int_msg_info$0 1 src:MsgAddress // always 00 on send 2 dest:MsgAddressInt // parameter 267 value:CurrencyCollection // parameter 124 + 1 - ihr_fee:Grams // always 0, not implemented 4 + extra_flags:Grams // if new bounces enabled 12 fwd_fee:Grams // always 0 on send 4 created_lt:uint64 // always 0 on send 64 created_at:uint32 // always 0 on send 32 @@ -18,7 +18,7 @@ _ split_depth:(Maybe (## 5)) 1 + library:(Maybe ^Cell) 1 = StateInit; message$_ {X:Type} - info:CommonMsgInfoRelaxed 502 + info:CommonMsgInfoRelaxed 510 init:(Maybe (Either StateInit ^StateInit)) 12 body:(Either X ^X) // body is either embedded or stored as ref = MessageRelaxed X; @@ -132,7 +132,7 @@ fun test1() { val jettonWalletCode = beginCell().storeInt(0x273849723892, 94).endCell(); var b = createMessage({ - bounce: true, + bounce: BounceMode.Only256BitsOfBody, dest: { stateInit: { code: jettonWalletCode, @@ -157,10 +157,12 @@ fun test2_manual(toAddress: address, masterMsg: cell, jettonWalletCode: cell) { var toWalletAddress = calculateJettonWalletAddress(shardPrefix, stateInit); return beginCell() - .storeUint(0x10, 6) // not bounceable + .storeUint(0x18, 6) // bounceable .storeAddress(toWalletAddress) // dest .storeCoins(ton("0.05")) - .storeUint(0b111, 1 + 4 + 4 + 64 + 32 + 1 + 1 + 1) // 1 state init exists + 1 state init ref + 1 body ref + .storeUint(0, 1) // no extra currencies + .storeCoins(1) // extra_flags for bounces + .storeUint(0b111, 4 + 64 + 32 + 1 + 1 + 1) // 1 state init exists + 1 state init ref + 1 body ref .storeRef(stateInit) .storeRef(masterMsg) .endCell(); @@ -174,7 +176,7 @@ fun test2() { var stateInitCell = calculateJettonWalletStateInitWithShard(toAddress, getMyAddressDev(), jettonWalletCode); var b = createMessage({ - bounce: false, + bounce: BounceMode.RichBounceOnlyRootCell, dest: { workchain: BASECHAIN, stateInit: stateInitCell, @@ -202,7 +204,9 @@ fun test3_manual(myCode: cell, myData: cell, msgBody: Test3Body) { .storeUint(0x18, 6) // bounceable .storeBuilder(addrInShard) // dest .storeCoins(ton("0.001")) - .storeUint(0b10, 1 + 4 + 4 + 64 + 32 + 1 + 1) // 1 state init exists + 1 state init inline + .storeUint(0, 1) // no extra currencies + .storeCoins(3) // extra_flags for bounces + .storeUint(0b10, 4 + 64 + 32 + 1 + 1) // 1 state init exists + 1 state init inline .storeUint(1, 1) .storeUint(20, 5) // shard depth .storeUint(0b0110, 4) // code + data exist @@ -221,7 +225,7 @@ fun test3() { val body: Test3Body = { bigData: builder800.endCell().beginParse() as bits800 }; var b = createMessage({ - bounce: true, + bounce: BounceMode.RichBounce, dest: { stateInit: { code: myCode, @@ -331,8 +335,8 @@ fun main() {} /** @testcase | 101 | | 114692697425779687591180352274328586758984823673144676907237399062035787598876 -@testcase | 102 | | 19141316219057792078125032524380927286847294424548966584483011017258286475706 -@testcase | 103 | | 33184074158306258754479533493449661031201063538320283251640171344508567641003 +@testcase | 102 | | 92833870282045340164651591875742409601508873505925232125500574773472682371231 +@testcase | 103 | | 56762665750429421964960025520433779418560428561961050303426782106579738725679 @testcase | 104 | | 52060388210984960140400107426752119783720473036191582802152450095687953630545 @testcase | 105 | | 18248582963085351579261363895401647616194364201057112283341027605548475235397 @testcase | 106 | 2 | 73367500160949078368695742399322272058735198813159066885436981754975296689213 diff --git a/tolk/send-message-api.cpp b/tolk/send-message-api.cpp index b5ef61aa7a..cec95e6794 100644 --- a/tolk/send-message-api.cpp +++ b/tolk/send-message-api.cpp @@ -91,10 +91,10 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co StructPtr s_Options = lookup_global_symbol("CreateMessageOptions")->try_as(); StructPtr s_AutoDeployAddress = lookup_global_symbol("AutoDeployAddress")->try_as(); - const TypeDataBool* t_bounce = s_Options->find_field("bounce")->declared_type->try_as(); + const TypeDataUnion* t_bounce = s_Options->find_field("bounce")->declared_type->try_as(); const TypeDataUnion* t_dest = s_Options->find_field("dest")->declared_type->try_as(); const TypeDataUnion* t_value = s_Options->find_field("value")->declared_type->try_as(); - tolk_assert(t_bounce); + tolk_assert(t_bounce && t_bounce->get_width_on_stack() == (1+1) && t_bounce->size() == 2); tolk_assert(t_dest && t_dest->get_width_on_stack() == (1+3+3+1) && t_dest->size() == 4); tolk_assert(t_value && t_value->get_width_on_stack() == (2+1) && t_value->size() == 2); @@ -112,6 +112,9 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co std::vector ir_body = next_slice(bodyT->get_width_on_stack()); tolk_assert(offset == static_cast(rvect.size())); + // since TVM 12, field `bounce` is a union: `bounce: BounceMode | bool` + std::vector ir_bounce_is_bool = pre_compile_is_type(code, t_bounce, TypeDataBool::create(), ir_bounce, loc, "(bounce-is-bool)"); + // field `dest` is `dest: address | AutoDeployAddress | (int8, uint256) | builder`; // struct AutoDeployAddress { workchain: int8; stateInit: ContractState | cell; toShard: AddressShardingOptions?; } // struct ContractState { code: cell; data: cell; } @@ -132,7 +135,7 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co bool body_already_ref = bodyT == TypeDataCell::create() || is_type_cellT(bodyT); // if `body` is `UnsafeBodyNoRef` bool body_force_no_ref = is_type_UnsafeBodyNoRef_T(bodyT); - // max size of all fields before body = 514 (502 CommonMsgInfoRelaxed + 12 StateInit), so 500 bits will fit + // max size of all fields before body = 522 (510 CommonMsgInfoRelaxed + 12 StateInit), so 500 bits will fit bool body_100p_fits_no_ref = body_size.max_bits <= 500 && body_size.max_refs < 2; // final decision: 1 (^X) or 0 (X) bool body_store_as_ref = body_already_ref || (!body_100p_fits_no_ref && !body_force_no_ref); @@ -160,7 +163,21 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co // fill `ihr_disabled:Bool` always 1 ctx.storeUint(ir_one, 1); // fill `bounce:Bool` from p.bounce (if it's constant (most likely), it will be concatenated with prev and next) - ctx.storeBool(ir_bounce[0]); + Op& if_oldBounceFormat = code.emplace_back(loc, Op::_If, ir_bounce_is_bool); + { + // input is `bounce: true` or false (old bounce mode), so 0-th slot is 0 or -1 + code.push_set_cur(if_oldBounceFormat.block0); + ctx.storeBool(ir_bounce[0]); + code.close_pop_cur(loc); + } + { + // input is `bounce: BounceMode.*` (enum), then write 0 for `NoBounce=0` or 1 otherwise + code.push_set_cur(if_oldBounceFormat.block1); + std::vector ir_not_NoBounce = code.create_tmp_var(TypeDataInt::create(), loc, "(not-eq-NoBounce)"); + code.emplace_back(loc, Op::_Call, ir_not_NoBounce, std::vector{ir_bounce[0], ir_zero}, lookup_function("_!=_")); + ctx.storeBool(ir_not_NoBounce[0]); + code.close_pop_cur(loc); + } // fill `bounced:Bool` + `src:MsgAddress` 00 ctx.storeUint(ir_zero, 1 + 2); @@ -287,8 +304,45 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co code.close_pop_cur(loc); } - // tail of CommonMsgInfoRelaxed: 4*0 ihr_fee + 4*0 fwd_fee + 64*0 created_lt + 32*0 created_at - ctx.storeUint(ir_zero, 4 + 4 + 64 + 32); + // fill `extra_flags:Grams` (formerly `ihr_fee` always 0, now renamed and used for "new bounce format" only) + Op& if_oldBounceFormat2 = code.emplace_back(loc, Op::_If, ir_bounce_is_bool); + { + code.push_set_cur(if_oldBounceFormat2.block0); + ctx.storeCoins(ir_zero); // extra_flags = 0 + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_oldBounceFormat2.block1); + std::vector ir_eq_RichBounce = code.create_tmp_var(TypeDataInt::create(), loc, "(eq-RichBounce)"); + code.emplace_back(loc, Op::_Call, ir_eq_RichBounce, std::vector{ir_bounce[0], code.create_int(loc, 2, "(enum-RichBounce)")}, lookup_function("_==_")); + Op& if_RichBounce = code.emplace_back(loc, Op::_If, ir_eq_RichBounce); + { + code.push_set_cur(if_RichBounce.block0); + ctx.storeCoins(code.create_int(loc, 3, "(extra-flags-3)")); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_RichBounce.block1); + std::vector ir_eq_RichBounceRoot = code.create_tmp_var(TypeDataInt::create(), loc, "(eq-ir_eq_RichBounceRoot)"); + code.emplace_back(loc, Op::_Call, ir_eq_RichBounceRoot, std::vector{ir_bounce[0], code.create_int(loc, 3, "(enum-RichBounceRoot)")}, lookup_function("_==_")); + Op& if_RichBounceRoot = code.emplace_back(loc, Op::_If, ir_eq_RichBounceRoot); + { + code.push_set_cur(if_RichBounceRoot.block0); + ctx.storeCoins(code.create_int(loc, 1, "(extra-flags-1)")); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_RichBounceRoot.block1); + ctx.storeCoins(ir_zero); // extra_flags = 0 + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + + // tail of CommonMsgInfoRelaxed: 4*0 fwd_fee + 64*0 created_lt + 32*0 created_at + ctx.storeUint(ir_zero, 4 + 64 + 32); // fill `init: (Maybe (Either StateInit ^StateInit))` // it's present only if p.dest contains StateInit From 03847bac0bb58aa7971f637ab62c148d7bb61294 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Tue, 2 Sep 2025 18:17:23 +0300 Subject: [PATCH 449/524] [Tolk] Simplify address computation and StateInit with TVM 12 `BTOS` TVM 12 has a new "builder-to-slice" instruction (without expensive intermediate cell creation). Hence, "endCell().beginParse()" can not be cheap. As a result, - addresses (slices under the hood) can be used instead of builders - StateInit.hash() becomes much more lightweight Old methods for "builders-with-addresses" are deprecated, new ones "returning addresses directly" introduced. --- crypto/smartcont/tolk-stdlib/common.tolk | 147 +++++++++++------------ tolk-tester/tests/build-addr-tests.tolk | 29 ++--- tolk-tester/tests/cells-slices.tolk | 72 +++++++++-- tolk-tester/tests/constants-tests.tolk | 2 +- tolk-tester/tests/generics-4.tolk | 3 +- tolk-tester/tests/lazy-load-tests.tolk | 5 +- tolk-tester/tests/pack-unpack-1.tolk | 5 +- tolk-tester/tests/pack-unpack-6.tolk | 6 +- tolk-tester/tests/pack-unpack-8.tolk | 3 +- tolk-tester/tests/parse-address.tolk | 1 + tolk-tester/tests/send-msg-4.tolk | 2 +- tolk-tester/tests/strings-tests.tolk | 2 +- tolk/builtins.cpp | 11 ++ tolk/maps-kv-api.cpp | 4 +- tolk/optimize.cpp | 114 ++++++++++++++++++ tolk/pack-unpack-serializers.cpp | 10 +- tolk/send-message-api.cpp | 16 +++ tolk/tolk.h | 6 + 18 files changed, 304 insertions(+), 134 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index d02803f675..d4385e3e9d 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -563,6 +563,12 @@ fun cell.hash(self): uint256 fun slice.hash(self): uint256 asm "HASHSU" +/// Computes the hash of a builder and returns it as a 256-bit unsigned integer. +/// The same as "transform builder to cell + calc hash of that cell", but without cell creation. +@pure +fun builder.hash(self): uint256 + asm "HASHBU" + /// Computes sha256 of the data bits of a slice (without refs). /// If the bit length is not divisible by eight, throws a cell underflow exception. @pure @@ -1114,6 +1120,12 @@ fun beginCell(): builder fun builder.endCell(self): cell asm "ENDC" +/// Converts a builder into a slice. +/// The same as `b.endCell().beginParse()` +@pure +fun builder.toSlice(self): slice + asm "BTOS" + /// Stores a reference to a cell into a builder. @pure fun builder.storeRef(mutate self, c: cell): self @@ -1357,7 +1369,7 @@ struct ContractState { } /// `AddressShardingOptions` provides settings to calculate an address in another shard. -/// Consider [createMessage] and [address.buildSameAddressInAnotherShard] for usage. +/// Consider [createMessage] and [address.calculateSameAddressInAnotherShard] for usage. struct AddressShardingOptions { fixedPrefixLength: uint5 // shard depth, formerly splitDepth closeTo: address @@ -1379,24 +1391,31 @@ struct AddressShardingOptions { /// You just provide code+data, and the compiler automatically calculates the destination address, /// because in TON, the address of a contract, by definition, is a hash of its initial state. /// You can also use it without sending a message. -/// See [AutoDeployAddress.buildAddress] and [AutoDeployAddress.addressMatches]. +/// See [AutoDeployAddress.calculateAddress]. struct AutoDeployAddress { workchain: int8 = BASECHAIN stateInit: ContractState | cell toShard: AddressShardingOptions? = null } +/// Deprecated, use [AutoDeployAddress.calculateAddress] instead. +/// Since late 2025 (TVM 12 has a cheap "builder-to-slice" instruction), +/// there is no need to use this function. +/// Historically its purpose was to avoid creating a cell and to return `builder` with a valid address. +@pure +@deprecated("use calculateAddress instead") +fun AutoDeployAddress.buildAddress(self): builder + builtin + /// Constructs an address that a deployed contract will have. /// For example, from a jetton minter, you want to calculate an address of a jetton wallet: /// ``` /// val jwDeployed = calcDeployedJettonWallet(...); -/// val jwAddrBuilt = jwDeployed.buildAddress(); +/// val jwAddress = jwDeployed.calculateAddress(); /// ``` /// Just instead of `dest` for [createMessage], you use it is to calculate a jetton/nft address. -/// Note: returns `builder`, not `address`! It's cheap. -/// If you really need `address`, use `address.fromValidBuilder(res)`. @pure -fun AutoDeployAddress.buildAddress(self): builder +fun AutoDeployAddress.calculateAddress(self): address builtin /// Checks that an address matches a deployed contract. @@ -1406,7 +1425,9 @@ fun AutoDeployAddress.buildAddress(self): builder /// val fromJW = jwDeployed.addressMatches(senderAddress); /// ``` /// Just instead of `dest` for [createMessage], you use it is to check whether a sender is a jetton/nft. +/// Deprecated since late 2025 (TVM 12 with a cheap "builder-to-slice" instruction), to be removed later. @pure +@deprecated("Use `a == jwDeployed.calculateAddress()` instead of `jwDeployed.addressMatches(a)") fun AutoDeployAddress.addressMatches(self, addr: address): bool builtin @@ -1579,56 +1600,33 @@ struct StateInit { /// ... /// ``` @pure -fun StateInit.calcHashCodeData(code: cell, data: cell): uint256 asm -""" // code data - DUP2 // code data code data - HASHCU - SWAP - HASHCU // code data dataHash codeHash - SWAP2 // dataHash codeHash code data - CDEPTH - SWAP - CDEPTH // dataHash codeHash dataDepth codeDepth - - NEWC - x{020134} STSLICECONST // store refs_descriptor | bits_descriptor | data - 16 STU // store codeDepth - 16 STU // store dataDepth - 256 STU // store codeHash - 256 STU // store dataHash - - ONE HASHEXT_SHA256 -""" +fun StateInit.calcHashCodeData(code: cell, data: cell): uint256 asm (data code) + "NEWC" + "b{00110} STSLICECONST" + "STREF" + "STREF" + "HASHBU" /// Calculates a hash of StateInit if fixedPrefixLength+code+data are set. @pure -fun StateInit.calcHashPrefixCodeData(fixedPrefixLength: uint5, code: cell, data: cell): uint256 asm -""" // pDepth code data - DUP2 // pDepth code data code data - HASHCU - SWAP - HASHCU // pDepth code data dataHash codeHash - SWAP2 // pDepth dataHash codeHash code data - CDEPTH - SWAP - CDEPTH // pDepth dataHash codeHash dataDepth codeDepth - - s4 PUSH // pDepth dataHash codeHash dataDepth codeDepth pDepth - 10 LSHIFT# - 0x020381A0 PUSHINT - ADD // calc refs_descriptor | bits_descriptor | data - - NEWC - 32 STU - 16 STU - 16 STU - 256 STU - 256 STU - - ONE HASHEXT_SHA256 // pDepth hash - NIP -""" +fun StateInit.calcHashPrefixCodeData(fixedPrefixLength: uint5, code: cell, data: cell): uint256 asm (data code fixedPrefixLength) + "NEWC" + "b{1} STSLICECONST" + "5 STU" + "b{0110} STSLICECONST" + "STREF" + "STREF" + "HASHBU" + +/// Deprecated, use [address.calculateSameAddressInAnotherShard] instead. +/// Since late 2025 (TVM 12 has a cheap "builder-to-slice" instruction), +/// there is no need to use this function. +/// Historically its purpose was to avoid creating a cell and to return `builder` with a valid address. +@pure +@deprecated("use calculateSameAddressInAnotherShard instead") +fun address.buildSameAddressInAnotherShard(self, options: AddressShardingOptions): builder + builtin /// Given an internal address A="aaaa...a" returns "bbaa...a" (D bits from address B, 256-D from A). /// @@ -1640,38 +1638,33 @@ fun StateInit.calcHashPrefixCodeData(fixedPrefixLength: uint5, code: cell, data: /// ``` /// More precisely, self (input) is 267 bits: '100' (std addr no anycast) + workchainA + "aaaa...a". /// The result is also 267 bits: '100' + workchainB + "bb" (D bits) + "aa...a" (256-D bits). -/// -/// Note: returns `builder`, not `address`! Because builder is cheap, and you can send a message to it: -/// ``` -/// createMessage({ -/// dest: resBuilder, // a builder containing a valid address can be passed to createMessage() -/// ... -/// ``` -/// If you really need `address`, use `address.fromValidBuilder(resBuilder)`. @pure -fun address.buildSameAddressInAnotherShard(self, options: AddressShardingOptions): builder +fun address.calculateSameAddressInAnotherShard(self, options: AddressShardingOptions): address builtin -/// Converts a builder containing a valid address (internal/external/none) to `address` (slice under the hood). -/// Gas-expensive! Because or a cell creation: essentially, it's `b.endCell().beginParse()`. -/// Example: you've manually built a 267-bit address: '100' (std addr no anycast) + workchain + hash -/// and you want to return `address` (TVM slice) from a contract getter: -/// ``` -/// get calcWalletAddress(ownerAddress: address): address { -/// val b = calculate(...); // manually calculate into builder -/// return address.fromValidBuilder(b) // but really need address -/// } -/// ``` -/// Hint: if you want just to send a message, don't call this function, use a builder directly: -/// ``` -/// createMessage({ -/// dest: someBuilder, // a builder containing a valid address can be passed to createMessage() -/// ... // so, don't call this expensive function in this case -/// ``` +/// Converts a builder containing a valid address to `address` (slice under the hood). +/// Deprecated! Since late 2025 (TVM 12 has a cheap "builder-to-slice" instruction), +/// all compiler built-ins return `address` directly, not `builder`. @pure +@deprecated("prefer using `address` everywhere, avoid using `builder` and manual address construction") fun address.fromValidBuilder(b: builder): address asm "ENDC" "CTOS" +/// Constructs an internal address (267 bits) from workchain and hash. +/// Hint: if you just want to send a message, don't call this function, pass workchain+hash directly: +/// ``` +/// createMessage({ +/// dest: (workchain, hash), // a pair can be passed instead of `address` +/// ... // so, don't call this function in such a case +/// ``` +fun address.fromWorkchainAndHash(workchain: int8, hash: uint256): address + asm (hash workchain) + "NEWC" + "b{100} STSLICECONST" // addr_std$10 + 0 (no anycast) + "8 STI" + "256 STU" + "BTOS" + /// Sends a raw message — a correctly serialized TL object `Message X`. /// In practice, you'll use a high-level wrapper [createMessage]. diff --git a/tolk-tester/tests/build-addr-tests.tolk b/tolk-tester/tests/build-addr-tests.tolk index 788450a556..9c1704a23a 100644 --- a/tolk-tester/tests/build-addr-tests.tolk +++ b/tolk-tester/tests/build-addr-tests.tolk @@ -51,12 +51,11 @@ fun test2() { val a = address("0:1234aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); val b = address("0:FFFFbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); val dd1 = buildAddrInShard_manual(a, {closeTo: b, fixedPrefixLength: 16}); - val dd2 = a.buildSameAddressInAnotherShard({fixedPrefixLength: 16, closeTo: b}); + val dd2 = a.calculateSameAddressInAnotherShard({fixedPrefixLength: 16, closeTo: b}); assert((a as slice).remainingBitsCount() == 267, 267); assert((b as slice).remainingBitsCount() == 267, 267); - assert(dd1.endCell().hash() == dd2.endCell().hash(), 400); - return address.fromValidBuilder(dd2) - == address("0:FFFFaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + assert(dd1.endCell().hash() == (dd2 as slice).hash(), 400); + return dd2 == address("0:FFFFaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); } @method_id(103) @@ -68,12 +67,12 @@ fun test3() { var t3_a = address("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"); var t3_b = address("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"); return ( - t1_a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: t1_b}).endCell().hash() & 0xFFFF, - t1_a.buildSameAddressInAnotherShard({fixedPrefixLength: 4, closeTo: t1_b}).endCell().hash() & 0xFFFF, - t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 2, closeTo: t2_b}).endCell().hash() & 0xFFFF, - t3_a.buildSameAddressInAnotherShard({fixedPrefixLength: 30, closeTo: t2_b}).endCell().hash() & 0xFFFF, - t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 1, closeTo: t3_b}).endCell().hash() & 0xFFFF, - t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 0, closeTo: t3_b}).endCell().hash() & 0xFFFF, + (t1_a.calculateSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: t1_b}) as slice).hash() & 0xFFFF, + (t1_a.calculateSameAddressInAnotherShard({fixedPrefixLength: 4, closeTo: t1_b}) as slice).hash() & 0xFFFF, + (t2_a.calculateSameAddressInAnotherShard({fixedPrefixLength: 2, closeTo: t2_b}) as slice).hash() & 0xFFFF, + (t3_a.calculateSameAddressInAnotherShard({fixedPrefixLength: 30, closeTo: t2_b}) as slice).hash() & 0xFFFF, + (t2_a.calculateSameAddressInAnotherShard({fixedPrefixLength: 1, closeTo: t3_b}) as slice).hash() & 0xFFFF, + (t2_a.calculateSameAddressInAnotherShard({fixedPrefixLength: 0, closeTo: t3_b}) as slice).hash() & 0xFFFF, ) } @@ -98,7 +97,7 @@ fun test5() { fun test6() { val a = address("0:00000000000000000000000000000000000000000000000000000000000000FF"); val b = address("1:1100000000000000000000000000000000000000000000000000000000000000"); - return address.fromValidBuilder(a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: b})) == + return a.calculateSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: b}) == address("1:11000000000000000000000000000000000000000000000000000000000000FF"); } @@ -219,8 +218,7 @@ fun test12() { stateInit: { code: jwCode, data: createEmptyCell() }, toShard: { fixedPrefixLength: i, closeTo: ownerAddress } }; - val addrBuilt = params.buildAddress(); - val addr = address.fromValidBuilder(addrBuilt); + val addr = params.calculateAddress(); assert (params.addressMatches(addr)) throw 123; params.workchain = 0; assert (!params.addressMatches(addr)) throw 123; @@ -276,15 +274,14 @@ fun test14() { stateInit: stateInitCell, toShard: { fixedPrefixLength: 10, closeTo: ownerAddress } }; - val addrBuilt = si.buildAddress(); - val addr = addrBuilt.endCell().beginParse() as address; + val addr = si.calculateAddress(); val stateInitHash = stateInitCell.hash(); val (wc, hash) = addr.getWorkchainAndHash(); val manual_isAddrOfContract = ((stateInitHash & ((1 << 246) - 1)) == (hash & ((1 << 246) - 1))) & (9 == wc); return ( - manualBuilt.endCell().hash() == addrBuilt.endCell().hash(), + manualBuilt.endCell().hash() == (addr as slice).hash(), manual_isAddrOfContract, si.addressMatches(addr), ) diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index e1c2ac9c39..0b94b19c4e 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -50,7 +50,7 @@ fun test2(): [int,int,int] { // operator ! here and below is used just for testing purposes, it doesn't affect the result b!.myStoreInt(3, 32); - var cs: slice = b.endCell().beginParse(); + var cs: slice = b.toSlice(); var one: int = cs.myLoadInt(32); var (two: int, three: int) = (cs.myLoadInt(32), cs.myLoadInt(32)); @@ -59,7 +59,7 @@ fun test2(): [int,int,int] { @method_id(103) fun test3(ret: int): int { - val same: int = beginCell()!.storeUint(ret,32).endCell().beginParse().loadUint(32); + val same: int = beginCell()!.storeUint(ret,32).toSlice().loadUint(32); return same; } @@ -255,7 +255,7 @@ fun test18() { if (true) { b.storeInt(x + 2, 8).storeUint(x = match (x) { 24 => 5, else => 0 }, 4); } - var s = b.endCell().beginParse(); + var s = [b.endCell()].0.beginParse(); return (s.loadUint(14), s.loadInt(8), s.loadUint(4)); } @@ -274,7 +274,7 @@ fun test20() { var b = true ? beginCell() : null; b.storeBool(true).storeBool(x); b = b.storeBool(true).storeUint(0, n *= 2).storeBool(!!true).storeCoins(0); - var s = b.endCell().beginParse(); + var s = b.toSlice(); return (s.loadBool(), s.loadBool(), s.loadBool(), s.loadUint(8), s.loadBool(), s.loadCoins()); } @@ -469,6 +469,26 @@ fun test36() { return beginCell().storeInt(0, 0) } +@method_id(137) +fun test37() { + return beginCell().storeInt(12, 32).endCell().hash() & 0xFFFF +} + +@method_id(138) +fun test38() { + return beginCell().endCell().beginParse().remainingBitsCount() +} + +@method_id(139) +fun test39() { + return beginCell().endCell() +} + +@method_id(140) +fun test40() { + return beginCell().toSlice().remainingBitsAndRefsCount() +} + fun main(): int { return 0; } @@ -512,22 +532,22 @@ fun main(): int { @testcase | 135 | 7 | 5 @testcase | 135 | 8 | 5 @testcase | 136 | | BC{0000} +@testcase | 137 | | 39065 +@testcase | 138 | | 0 +@testcase | 140 | | 0 0 We test that consequtive storeInt/storeUint with constants are joined into a single number @fif_codegen """ test6() PROC:<{ - NEWC - x{000000010000000200000003} STSLICECONST - ENDC // '11 + x{000000010000000200000003} PUSHSLICE """ @fif_codegen """ test17() PROC:<{ - NEWC - x{107b} STSLICECONST + x{107b} PUSHSLICE """ @fif_codegen @@ -552,8 +572,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ test20() PROC:<{ - NEWC - x{a010} STSLICECONST + x{a010} PUSHSLICE """ @fif_codegen @@ -701,4 +720,35 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin }> """ +@fif_codegen +""" + test37() PROC:<{ + NEWC + x{0000000c} STSLICECONST + HASHBU +""" + +@fif_codegen +""" + test38() PROC:<{ + x{} PUSHSLICE + SBITS + }> +""" + +@fif_codegen +""" + test39() PROC:<{ + PUSHREF + }> +""" + +@fif_codegen +""" + test40() PROC:<{ + x{} PUSHSLICE + SBITREFS + }> +""" + */ diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index 5a386cd4b2..42cace8b20 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -126,5 +126,5 @@ fun main() { @testcase | 105 | | -1 0 7 48 @testcase | 106 | | -1 0 0 -1 -@code_hash 32362412747322136329528616455651783746542516198110452861733590440068294458753 +@code_hash 109852777684517203308276021763918179390784357880680494742516881492720986364812 */ diff --git a/tolk-tester/tests/generics-4.tolk b/tolk-tester/tests/generics-4.tolk index 6833d3867e..b44ec0492a 100644 --- a/tolk-tester/tests/generics-4.tolk +++ b/tolk-tester/tests/generics-4.tolk @@ -191,8 +191,7 @@ fun main() { @fif_codegen """ test6() PROC:<{ // - NEWC // '8 - ENDC // p.init.USlot2 + PUSHREF // p.init.USlot2 -1 PUSHINT // p.init.USlot2 '11=-1 FALSE // p.init.USlot2 '11=-1 '12 TRUE // p.init.USlot2 '11=-1 '12 '14 diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index 51ec9148d9..0954efc241 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -1898,8 +1898,7 @@ fun main() { IFJMP:<{ // '14 NEWC // '14 b STSLICE // b - ENDC // '21 - CTOS // s + BTOS // s """ @fif_codegen @@ -1928,7 +1927,7 @@ fun main() { NEWC 8 STI 8 STI - ENDC + BTOS """ @fif_codegen diff --git a/tolk-tester/tests/pack-unpack-1.tolk b/tolk-tester/tests/pack-unpack-1.tolk index 8239403190..519eb4ea9e 100644 --- a/tolk-tester/tests/pack-unpack-1.tolk +++ b/tolk-tester/tests/pack-unpack-1.tolk @@ -122,10 +122,7 @@ fun main(c: cell) { @fif_codegen """ test3() PROC:<{ - NEWC - x{0000000a00000014} STSLICECONST // b - ENDC // c - CTOS // s + x{0000000a00000014} PUSHSLICE // s 64 LDU NIP // s SBITS // '17 diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk index 4a261869c5..ae3ec403f0 100644 --- a/tolk-tester/tests/pack-unpack-6.tolk +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -132,8 +132,7 @@ RETALT x{0109ab} PUSHSLICE NEWC STSLICE - ENDC - CTOS + BTOS x{01} SDBEGINSQ 63 THROWIFNOT 8 PLDI @@ -174,8 +173,7 @@ IF:<{ x{11} PUSHSLICE NEWC STSLICE - ENDC - HASHCU + HASHBU """ @fif_codegen diff --git a/tolk-tester/tests/pack-unpack-8.tolk b/tolk-tester/tests/pack-unpack-8.tolk index a146996a13..1cfbce98da 100644 --- a/tolk-tester/tests/pack-unpack-8.tolk +++ b/tolk-tester/tests/pack-unpack-8.tolk @@ -191,8 +191,7 @@ fun main() { test4() PROC:<{ // v NEWC // v b 3 STU // b - ENDC // '4 - CTOS // s + BTOS // s 3 LDU // '9 s ENDS // '9 }> diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index b4613655ca..b753f5838e 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -13,6 +13,7 @@ fun verifyAddr(addr: address, workchain: int, number: int) { assert (wc == workchain) throw 111; assert (h == number) throw 111; assert (addr.getWorkchain() == workchain) throw 111; + assert (address.fromWorkchainAndHash(workchain, number) == addr) throw 113; var s = addr as slice; assert (s.remainingBitsCount() == 3 + 8 + 256) throw 112; diff --git a/tolk-tester/tests/send-msg-4.tolk b/tolk-tester/tests/send-msg-4.tolk index 84706fd71e..454966774c 100644 --- a/tolk-tester/tests/send-msg-4.tolk +++ b/tolk-tester/tests/send-msg-4.tolk @@ -53,7 +53,7 @@ fun main() { /** @testcase | 0 | | 0 -@fif_codegen ONE HASHEXT_SHA256 +@fif_codegen HASHBU @fif_codegen }>ELSE<{ @fif_codegen HASHCU @fif_codegen 256 STU diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index 99a4221610..c6d0f22b4f 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -78,5 +78,5 @@ fun test1() { @testcase | 0 | | 0 @testcase | 101 | | [ 65 66 67 68 ] -@code_hash 52991558142486159683575375539594116045551936710713942466413832119934760926753 +@code_hash 5620054157808655551766377324451164816605429927637692949572895262349774660488 */ diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index fd48cc033c..256acd3fff 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1323,7 +1323,9 @@ GenerateOpsImpl generate_T_estimatePackSize; GenerateOpsImpl generate_createMessage; GenerateOpsImpl generate_createExternalLogMessage; GenerateOpsImpl generate_address_buildInAnotherShard; +GenerateOpsImpl generate_address_calculateInAnotherShard; GenerateOpsImpl generate_AutoDeployAddress_buildAddress; +GenerateOpsImpl generate_AutoDeployAddress_calculateAddress; GenerateOpsImpl generate_AutoDeployAddress_addressMatches; GenerateOpsImpl generate_mapKV_exists; @@ -1620,6 +1622,9 @@ void define_builtins() { define_builtin_method("address.buildSameAddressInAnotherShard", Address, {Address, AddressShardingOptions}, Builder, nullptr, generate_address_buildInAnotherShard, FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); + define_builtin_method("address.calculateSameAddressInAnotherShard", Address, {Address, AddressShardingOptions}, Address, nullptr, + generate_address_calculateInAnotherShard, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); define_builtin_method("debug.print", debug, {typeT}, Unit, declGenericT, compile_debug_print_to_string, FunctionData::flagAllowAnyWidthT); @@ -1678,6 +1683,9 @@ void define_builtins() { define_builtin_method("AutoDeployAddress.buildAddress", AutoDeployAddress, {AutoDeployAddress}, Builder, nullptr, generate_AutoDeployAddress_buildAddress, FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); + define_builtin_method("AutoDeployAddress.calculateAddress", AutoDeployAddress, {AutoDeployAddress}, Address, nullptr, + generate_AutoDeployAddress_calculateAddress, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); define_builtin_method("AutoDeployAddress.addressMatches", AutoDeployAddress, {AutoDeployAddress, Address}, Bool, nullptr, generate_AutoDeployAddress_addressMatches, FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); @@ -1833,8 +1841,11 @@ void patch_builtins_after_stdlib_loaded() { TypePtr AutoDeployAddress = TypeDataStruct::create(struct_ref_AutoDeployAddress); lookup_function("address.buildSameAddressInAnotherShard")->mutate()->parameters[1].declared_type = AddressShardingOptions; + lookup_function("address.calculateSameAddressInAnotherShard")->mutate()->parameters[1].declared_type = AddressShardingOptions; lookup_function("AutoDeployAddress.buildAddress")->mutate()->receiver_type = AutoDeployAddress; lookup_function("AutoDeployAddress.buildAddress")->mutate()->parameters[0].declared_type = AutoDeployAddress; + lookup_function("AutoDeployAddress.calculateAddress")->mutate()->receiver_type = AutoDeployAddress; + lookup_function("AutoDeployAddress.calculateAddress")->mutate()->parameters[0].declared_type = AutoDeployAddress; lookup_function("AutoDeployAddress.addressMatches")->mutate()->receiver_type = AutoDeployAddress; lookup_function("AutoDeployAddress.addressMatches")->mutate()->parameters[0].declared_type = AutoDeployAddress; diff --git a/tolk/maps-kv-api.cpp b/tolk/maps-kv-api.cpp index 770450ae3b..6fc0112db5 100644 --- a/tolk/maps-kv-api.cpp +++ b/tolk/maps-kv-api.cpp @@ -230,9 +230,7 @@ class DictKeyValue { PackContext ctx(code, loc, ir_builder, create_default_PackOptions(code, loc)); ctx.generate_pack_any(TKey, std::vector(*exact_key)); std::vector ir_slice = code.create_tmp_var(TypeDataSlice::create(), loc, "(map-key)"); - // todo BTOS - code.emplace_back(loc, Op::_Call, ir_slice, ir_builder, lookup_function("builder.endCell")); - code.emplace_back(loc, Op::_Call, ir_slice, ir_slice, lookup_function("cell.beginParse")); + code.emplace_back(loc, Op::_Call, ir_slice, ir_builder, lookup_function("builder.toSlice")); key_irv = ir_slice[0]; } } diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index a3a0e29162..f30bd15975 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -458,6 +458,117 @@ bool Optimizer::detect_rewrite_DICTSETB_DICTSET() { return false; } +// pattern `ENDC` + `CTOS` -> `BTOS` (a new TVM 12 instruction "builder to slice") +bool Optimizer::detect_rewrite_ENDC_CTOS() { + bool first_endc = op_[0]->is_custom() && op_[0]->op == "ENDC"; + if (!first_endc || pb_ < 2) { + return false; + } + + bool next_ctos = op_[1]->is_custom() && op_[1]->op == "CTOS"; + if (!next_ctos) { + return false; + } + + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, "BTOS", 1, 1)); + return true; +} + +// pattern `ENDC` + `HASHCU` -> `HASHBU` (a new TVM 12 instruction "hash of a builder") +bool Optimizer::detect_rewrite_ENDC_HASHCU() { + bool first_endc = op_[0]->is_custom() && op_[0]->op == "ENDC"; + if (!first_endc || pb_ < 2) { + return false; + } + + bool next_hashcu = op_[1]->is_custom() && op_[1]->op == "HASHCU"; + if (!next_hashcu) { + return false; + } + + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, "HASHBU", 1, 1)); + return true; +} + +// pattern `NEWC` + `BTOS` -> `x{} PUSHSLICE` +bool Optimizer::detect_rewrite_NEWC_BTOS() { + bool first_newc = op_[0]->is_custom() && op_[0]->op == "NEWC"; + if (!first_newc || pb_ < 2) { + return false; + } + + bool next_btos = op_[1]->is_custom() && op_[1]->op == "BTOS"; + if (!next_btos) { + return false; + } + + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, "x{} PUSHSLICE", 0, 1)); + return true; +} + +// pattern `NEWC` + `x{...} STSLICECONST` + `BTOS` -> `x{...} PUSHSLICE` +bool Optimizer::detect_rewrite_NEWC_STSLICECONST_BTOS() { + bool first_newc = op_[0]->is_custom() && op_[0]->op == "NEWC"; + if (!first_newc || pb_ < 3) { + return false; + } + + bool next_stsliceconst = op_[1]->is_custom() && op_[1]->op.ends_with(" STSLICECONST"); + bool next_btos = op_[2]->is_custom() && op_[2]->op == "BTOS"; + if (!next_stsliceconst || !next_btos) { + return false; + } + + std::string op_pushslice = op_[1]->op.substr(0, op_[1]->op.rfind(' ')) + " PUSHSLICE"; + p_ = 3; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, op_pushslice, 0, 1)); + return true; +} + + +// pattern `NEWC` + `ENDC` + `CTOS` -> `x{} PUSHSLICE` +bool Optimizer::detect_rewrite_NEWC_ENDC_CTOS() { + bool first_newc = op_[0]->is_custom() && op_[0]->op == "NEWC"; + if (!first_newc || pb_ < 3) { + return false; + } + + bool next_endc = op_[1]->is_custom() && op_[1]->op == "ENDC"; + bool next_ctos = op_[2]->is_custom() && op_[2]->op == "CTOS"; + if (!next_endc || !next_ctos) { + return false; + } + + p_ = 3; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, "x{} PUSHSLICE", 0, 1)); + return true; +} + +// pattern `NEWC` + `ENDC` -> ` PUSHREF` +bool Optimizer::detect_rewrite_NEWC_ENDC() { + bool first_newc = op_[0]->is_custom() && op_[0]->op == "NEWC"; + if (!first_newc || pb_ < 2) { + return false; + } + + bool next_endc = op_[1]->is_custom() && op_[1]->op == "ENDC"; + if (!next_endc) { + return false; + } + + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, " PUSHREF", 0, 1)); + return true; +} bool Optimizer::is_push_const(int* i, int* c) const { return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_push_const(i, c); @@ -882,6 +993,9 @@ bool Optimizer::find_at_least(int pb) { detect_rewrite_LDxx_DROP() || detect_rewrite_SWAP_symmetric() || detect_rewrite_SWAP_PUSH_STUR() || detect_rewrite_SWAP_STxxxR() || detect_rewrite_NOT_THROWIF() || detect_rewrite_DICTSETB_DICTSET() || + detect_rewrite_ENDC_CTOS() || detect_rewrite_ENDC_HASHCU() || + detect_rewrite_NEWC_BTOS() || detect_rewrite_NEWC_STSLICECONST_BTOS() || + detect_rewrite_NEWC_ENDC_CTOS() || detect_rewrite_NEWC_ENDC() || (!(mode_ & 1) && ((is_rot() && rewrite(AsmOp::Custom(loc, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(loc, "-ROT", 3, 3))) || (is_2dup() && rewrite(AsmOp::Custom(loc, "2DUP", 2, 4))) || diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index d000bfb6fe..e5cce47078 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -505,15 +505,7 @@ struct S_RemainingBitsAndRefs final : ISerializer { } void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { - FunctionPtr f_beginCell = lookup_function("beginCell"); - FunctionPtr f_endCell = lookup_function("builder.endCell"); - FunctionPtr f_beginParse = lookup_function("cell.beginParse"); - - std::vector ir_builder = code.create_tmp_var(TypeDataBuilder::create(), loc, "(tmp-builder)"); - std::vector ir_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(tmp-cell)"); - code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, f_beginCell); - code.emplace_back(loc, Op::_Call, ir_cell, ir_builder, f_endCell); - code.emplace_back(loc, Op::_Call, ctx->ir_slice, ir_cell, f_beginParse); + unpack(ctx, code, loc); } PackSize estimate(const EstimateContext* ctx) override { diff --git a/tolk/send-message-api.cpp b/tolk/send-message-api.cpp index cec95e6794..67ed082405 100644 --- a/tolk/send-message-api.cpp +++ b/tolk/send-message-api.cpp @@ -579,6 +579,14 @@ std::vector generate_address_buildInAnotherShard(FunctionPtr called_f return ir_builder; } +// fun address.calculateSameAddressInAnotherShard(self, options: AddressShardingOptions): address +std::vector generate_address_calculateInAnotherShard(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + // it's "build address" + BTOS (until a deprecated "build" function removed) + std::vector ir_builder = generate_address_buildInAnotherShard(called_f, code, loc, args); + code.emplace_back(loc, Op::_Call, ir_builder, ir_builder, lookup_function("builder.toSlice")); + return ir_builder; +} + // fun AutoDeployAddress.buildAddress(self): builder std::vector generate_AutoDeployAddress_buildAddress(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& ir_options) { IR_AutoDeployAddress ir_self(code, loc, ir_options[0]); @@ -658,6 +666,14 @@ std::vector generate_AutoDeployAddress_buildAddress(FunctionPtr calle return ir_builder; } +// fun AutoDeployAddress.calculateAddress(self): address +std::vector generate_AutoDeployAddress_calculateAddress(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& ir_options) { + // it's "build address" + BTOS (until a deprecated "build" function removed) + std::vector ir_builder = generate_AutoDeployAddress_buildAddress(called_f, code, loc, ir_options); + code.emplace_back(loc, Op::_Call, ir_builder, ir_builder, lookup_function("builder.toSlice")); + return ir_builder; +} + // fun AutoDeployAddress.addressMatches(self, addr: address): bool std::vector generate_AutoDeployAddress_addressMatches(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& ir_self_and_addr) { IR_AutoDeployAddress ir_self(code, loc, ir_self_and_addr[0]); diff --git a/tolk/tolk.h b/tolk/tolk.h index 13a9acefe3..aa1fa0fd6e 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -908,6 +908,12 @@ struct Optimizer { bool detect_rewrite_SWAP_STxxxR(); bool detect_rewrite_NOT_THROWIF(); bool detect_rewrite_DICTSETB_DICTSET(); + bool detect_rewrite_ENDC_CTOS(); + bool detect_rewrite_ENDC_HASHCU(); + bool detect_rewrite_NEWC_BTOS(); + bool detect_rewrite_NEWC_STSLICECONST_BTOS(); + bool detect_rewrite_NEWC_ENDC_CTOS(); + bool detect_rewrite_NEWC_ENDC(); AsmOpConsList extract_code(); }; From 5059e015d8bb35cc972fda66896369efc35d11a9 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sat, 18 Oct 2025 11:39:19 +0300 Subject: [PATCH 450/524] [Tolk] SrcRange instead of SrcLocation in all AST vertices Now, every AST vertex (both expressions and statements) have a full range pointing to original Tolk sources. It will allow prettifying compilation errors (by underlining an expression from beginning to end) and better source mapping in the future. --- .../tests/invalid-semantics/err-4629.tolk | 2 +- .../tests/invalid-symbol/err-2980.tolk | 3 +- tolk/CMakeLists.txt | 1 + tolk/abscode.cpp | 95 ++- tolk/analyzer.cpp | 22 +- tolk/asmops.cpp | 108 +-- tolk/ast-aux-data.h | 11 +- tolk/ast-from-tokens.cpp | 742 +++++++++++------- tolk/ast-replicator.h | 128 +-- tolk/ast-stringifier.h | 4 +- tolk/ast.cpp | 8 +- tolk/ast.h | 392 ++++----- tolk/builtins.cpp | 417 +++++----- tolk/codegen.cpp | 242 +++--- tolk/compilation-errors.cpp | 112 +++ tolk/compilation-errors.h | 83 ++ tolk/constant-evaluator.cpp | 50 +- tolk/gen-entrypoints.cpp | 30 +- tolk/gen-entrypoints.h | 4 +- tolk/generics-helpers.cpp | 23 +- tolk/generics-helpers.h | 4 +- tolk/lazy-helpers.cpp | 3 +- tolk/lexer.cpp | 23 +- tolk/lexer.h | 12 +- tolk/maps-kv-api.cpp | 296 +++---- tolk/maps-kv-api.h | 2 +- tolk/optimize.cpp | 116 +-- tolk/pack-unpack-api.cpp | 86 +- tolk/pack-unpack-api.h | 8 +- tolk/pack-unpack-serializers.cpp | 561 ++++++------- tolk/pack-unpack-serializers.h | 14 +- tolk/pipe-ast-to-legacy.cpp | 671 ++++++++-------- tolk/pipe-calc-rvalue-lvalue.cpp | 2 +- tolk/pipe-check-inferred-types.cpp | 183 +++-- tolk/pipe-check-private-fields.cpp | 11 +- tolk/pipe-check-pure-impure.cpp | 16 +- tolk/pipe-check-rvalue-lvalue.cpp | 70 +- tolk/pipe-check-serialized-fields.cpp | 33 +- tolk/pipe-constant-folding.cpp | 30 +- tolk/pipe-detect-inline-in-place.cpp | 9 +- tolk/pipe-discover-parse-sources.cpp | 8 +- tolk/pipe-generate-fif-output.cpp | 2 +- tolk/pipe-infer-types-and-calls.cpp | 156 ++-- tolk/pipe-lazy-load-insertions.cpp | 47 +- tolk/pipe-optimize-boolean-expr.cpp | 29 +- tolk/pipe-refine-lvalue-for-mutate.cpp | 12 +- tolk/pipe-register-symbols.cpp | 65 +- tolk/pipe-resolve-identifiers.cpp | 85 +- tolk/pipe-resolve-types.cpp | 78 +- tolk/pipe-transform-on-message.cpp | 14 +- tolk/send-message-api.cpp | 428 +++++----- tolk/smart-casts-cfg.cpp | 2 +- tolk/src-file.cpp | 196 ++--- tolk/src-file.h | 119 ++- tolk/symtable.cpp | 34 +- tolk/symtable.h | 58 +- tolk/tolk.cpp | 26 +- tolk/tolk.h | 284 ++++--- tolk/type-system.cpp | 2 +- tolk/type-system.h | 2 + 60 files changed, 3289 insertions(+), 2985 deletions(-) create mode 100644 tolk/compilation-errors.cpp create mode 100644 tolk/compilation-errors.h diff --git a/tolk-tester/tests/invalid-semantics/err-4629.tolk b/tolk-tester/tests/invalid-semantics/err-4629.tolk index dc69a83be5..ff63cf8c6e 100644 --- a/tolk-tester/tests/invalid-semantics/err-4629.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4629.tolk @@ -12,5 +12,5 @@ fun main() { /** @compilation_should_fail @stderr field `item` missed in initialization of struct `Wrapper` -@stderr = Wrapper { +@stderr }; */ diff --git a/tolk-tester/tests/invalid-symbol/err-2980.tolk b/tolk-tester/tests/invalid-symbol/err-2980.tolk index 65e87ff4a7..1509420eab 100644 --- a/tolk-tester/tests/invalid-symbol/err-2980.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2980.tolk @@ -6,6 +6,7 @@ @compilation_should_fail On Linux/Mac, `realpath()` returns an error, and the error message is "cannot find file" On Windows, it fails after, on reading, with a message "cannot open file" -@stderr err-2980.tolk:2:7: error: Failed to import: +@stderr err-2980.tolk:2:14: error: Failed to import: @stderr import "unexisting.tolk"; +@stderr ^^^^^^^^^^^^^^^^^ */ diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index 6657be95df..1085cac846 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -2,6 +2,7 @@ set(TOLK_SOURCE src-file.cpp lexer.cpp symtable.cpp + compilation-errors.cpp compiler-state.cpp ast.cpp ast-from-tokens.cpp diff --git a/tolk/abscode.cpp b/tolk/abscode.cpp index 462aad7b2d..5fa7706a7b 100644 --- a/tolk/abscode.cpp +++ b/tolk/abscode.cpp @@ -44,8 +44,8 @@ void TmpVar::show(std::ostream& os) const { os << '_' << name; } #ifdef TOLK_DEBUG - if (desc) { - os << ' ' << desc; // "origin" of implicitly created tmp var, like `'15 (binary-op) '16 (glob-var)` + if (purpose) { + os << ' ' << purpose; // "purpose" of implicitly created tmp var, like `'15 (binary-op) '16 (glob-var)` } #endif } @@ -175,9 +175,9 @@ void VarDescrList::show(std::ostream& os) const { os << " ]\n"; } -void Op::show(std::ostream& os, const std::vector& vars, std::string pfx, int mode) const { +void Op::show(std::ostream& os, const std::vector& vars, const std::string& indent, int mode) const { if (mode & 2) { - os << pfx << " ["; + os << indent << " ["; for (const auto& v : var_info.list) { os << ' '; if (v.flags & VarDescr::_Last) { @@ -203,10 +203,10 @@ void Op::show(std::ostream& os, const std::vector& vars, std::string pfx } switch (cl) { case _Nop: - os << pfx << dis << "NOP\n"; + os << indent << dis << "NOP\n"; break; case _Call: - os << pfx << dis << "CALL: "; + os << indent << dis << "CALL: "; show_var_list(os, left, vars); os << " := " << (f_sym ? f_sym->name : "(null)") << " "; if ((mode & 4) && args.size() == right.size()) { @@ -217,105 +217,105 @@ void Op::show(std::ostream& os, const std::vector& vars, std::string pfx os << std::endl; break; case _CallInd: - os << pfx << dis << "CALLIND: "; + os << indent << dis << "CALLIND: "; show_var_list(os, left, vars); os << " := EXEC "; show_var_list(os, right, vars); os << std::endl; break; case _Let: - os << pfx << dis << "LET "; + os << indent << dis << "LET "; show_var_list(os, left, vars); os << " := "; show_var_list(os, right, vars); os << std::endl; break; case _Tuple: - os << pfx << dis << "MKTUPLE "; + os << indent << dis << "MKTUPLE "; show_var_list(os, left, vars); os << " := "; show_var_list(os, right, vars); os << std::endl; break; case _UnTuple: - os << pfx << dis << "UNTUPLE "; + os << indent << dis << "UNTUPLE "; show_var_list(os, left, vars); os << " := "; show_var_list(os, right, vars); os << std::endl; break; case _IntConst: - os << pfx << dis << "CONST "; + os << indent << dis << "CONST "; show_var_list(os, left, vars); os << " := " << int_const << std::endl; break; case _SliceConst: - os << pfx << dis << "SCONST "; + os << indent << dis << "SCONST "; show_var_list(os, left, vars); os << " := " << str_const << std::endl; break; case _Import: - os << pfx << dis << "IMPORT "; + os << indent << dis << "IMPORT "; show_var_list(os, left, vars); os << std::endl; break; case _Return: - os << pfx << dis << "RETURN "; + os << indent << dis << "RETURN "; show_var_list(os, left, vars); os << std::endl; break; case _GlobVar: - os << pfx << dis << "GLOBVAR "; + os << indent << dis << "GLOBVAR "; show_var_list(os, left, vars); os << " := " << (g_sym ? g_sym->name : "(null)") << std::endl; break; case _SetGlob: - os << pfx << dis << "SETGLOB "; + os << indent << dis << "SETGLOB "; os << (g_sym ? g_sym->name : "(null)") << " := "; show_var_list(os, right, vars); os << std::endl; break; case _Repeat: - os << pfx << dis << "REPEAT "; + os << indent << dis << "REPEAT "; show_var_list(os, left, vars); os << ' '; - show_block(os, block0.get(), vars, pfx, mode); + show_block(os, block0.get(), vars, indent, mode); os << std::endl; break; case _If: - os << pfx << dis << "IF "; + os << indent << dis << "IF "; show_var_list(os, left, vars); os << ' '; - show_block(os, block0.get(), vars, pfx, mode); + show_block(os, block0.get(), vars, indent, mode); os << " ELSE "; - show_block(os, block1.get(), vars, pfx, mode); + show_block(os, block1.get(), vars, indent, mode); os << std::endl; break; case _While: - os << pfx << dis << "WHILE "; + os << indent << dis << "WHILE "; show_var_list(os, left, vars); os << ' '; - show_block(os, block0.get(), vars, pfx, mode); + show_block(os, block0.get(), vars, indent, mode); os << " DO "; - show_block(os, block1.get(), vars, pfx, mode); + show_block(os, block1.get(), vars, indent, mode); os << std::endl; break; case _Until: - os << pfx << dis << "UNTIL "; + os << indent << dis << "UNTIL "; show_var_list(os, left, vars); os << ' '; - show_block(os, block0.get(), vars, pfx, mode); + show_block(os, block0.get(), vars, indent, mode); os << std::endl; break; case _Again: - os << pfx << dis << "AGAIN "; + os << indent << dis << "AGAIN "; show_var_list(os, left, vars); os << ' '; - show_block(os, block0.get(), vars, pfx, mode); + show_block(os, block0.get(), vars, indent, mode); os << std::endl; break; default: - os << pfx << dis << " "; + os << indent << dis << " "; show_var_list(os, left, vars); os << " -- "; show_var_list(os, right, vars); @@ -359,13 +359,13 @@ void Op::show_var_list(std::ostream& os, const std::vector& list, cons } } -void Op::show_block(std::ostream& os, const Op* block, const std::vector& vars, std::string pfx, int mode) { +void Op::show_block(std::ostream& os, const Op* block, const std::vector& vars, const std::string& indent, int mode) { os << "{" << std::endl; - std::string pfx2 = pfx + " "; + std::string sub_indent = indent + " "; for (const Op& op : block) { - op.show(os, vars, pfx2, mode); + op.show(os, vars, sub_indent, mode); } - os << pfx << "}"; + os << indent << "}"; } std::ostream& operator<<(std::ostream& os, const CodeBlob& code) { @@ -380,11 +380,6 @@ void CodeBlob::print(std::ostream& os, int flags) const { for (const auto& var : vars) { var.show(os); os << " : " << var.v_type << std::endl; - if (var.loc.is_defined() && (flags & 1) != 0) { - var.loc.show(os); - os << " defined here:\n"; - var.loc.show_context(os); - } } } os << "------- BEGIN --------\n"; @@ -394,7 +389,7 @@ void CodeBlob::print(std::ostream& os, int flags) const { os << "-------- END ---------\n\n"; } -std::vector CodeBlob::create_var(TypePtr var_type, SrcLocation loc, std::string name) { +std::vector CodeBlob::create_var(TypePtr var_type, AnyV origin, std::string name) { std::vector ir_idx; int stack_w = var_type->get_width_on_stack(); ir_idx.reserve(stack_w); @@ -402,33 +397,33 @@ std::vector CodeBlob::create_var(TypePtr var_type, SrcLocation loc, s for (int i = 0; i < t_struct->struct_ref->get_num_fields(); ++i) { StructFieldPtr field_ref = t_struct->struct_ref->get_field(i); std::string sub_name = name.empty() || t_struct->struct_ref->get_num_fields() == 1 ? name : name + "." + field_ref->name; - std::vector nested = create_var(field_ref->declared_type, loc, std::move(sub_name)); + std::vector nested = create_var(field_ref->declared_type, origin, std::move(sub_name)); ir_idx.insert(ir_idx.end(), nested.begin(), nested.end()); } } else if (const TypeDataTensor* t_tensor = var_type->try_as()) { for (int i = 0; i < t_tensor->size(); ++i) { std::string sub_name = name.empty() ? name : name + "." + std::to_string(i); - std::vector nested = create_var(t_tensor->items[i], loc, std::move(sub_name)); + std::vector nested = create_var(t_tensor->items[i], origin, std::move(sub_name)); ir_idx.insert(ir_idx.end(), nested.begin(), nested.end()); } } else if (const TypeDataAlias* t_alias = var_type->try_as()) { - ir_idx = create_var(t_alias->underlying_type, loc, std::move(name)); + ir_idx = create_var(t_alias->underlying_type, origin, std::move(name)); } else if (const TypeDataUnion* t_union = var_type->try_as(); t_union && stack_w != 1) { std::string utag_name = name.empty() ? "'UTag" : name + ".UTag"; if (t_union->or_null) { // in stack comments, `a:(int,int)?` will be "a.0 a.1 a.UTag" - ir_idx = create_var(t_union->or_null, loc, std::move(name)); + ir_idx = create_var(t_union->or_null, origin, std::move(name)); } else { // in stack comments, `a:int|slice` will be "a.USlot1 a.UTag" for (int i = 0; i < stack_w - 1; ++i) { std::string slot_name = name.empty() ? "'USlot" + std::to_string(i + 1) : name + ".USlot" + std::to_string(i + 1); - ir_idx.emplace_back(create_var(TypeDataUnknown::create(), loc, std::move(slot_name))[0]); + ir_idx.emplace_back(create_var(TypeDataUnknown::create(), origin, std::move(slot_name))[0]); } } - ir_idx.emplace_back(create_var(TypeDataInt::create(), loc, std::move(utag_name))[0]); + ir_idx.emplace_back(create_var(TypeDataInt::create(), origin, std::move(utag_name))[0]); } else if (var_type != TypeDataVoid::create() && var_type != TypeDataNever::create()) { #ifdef TOLK_DEBUG tolk_assert(stack_w == 1); #endif - vars.emplace_back(var_cnt, var_type, std::move(name), loc); + vars.emplace_back(var_cnt, var_type, std::move(name)); ir_idx.emplace_back(var_cnt); var_cnt++; } @@ -436,14 +431,14 @@ std::vector CodeBlob::create_var(TypePtr var_type, SrcLocation loc, s return ir_idx; } -var_idx_t CodeBlob::create_int(SrcLocation loc, int64_t value, const char* desc) { - vars.emplace_back(var_cnt, TypeDataInt::create(), std::string{}, loc); +var_idx_t CodeBlob::create_int(AnyV origin, int64_t value, const char* purpose) { + vars.emplace_back(var_cnt, TypeDataInt::create(), std::string{}); #ifdef TOLK_DEBUG - vars.back().desc = desc; + vars.back().purpose = purpose; #endif var_idx_t ir_int = var_cnt; var_cnt++; - emplace_back(loc, Op::_IntConst, std::vector{ir_int}, td::make_refint(value)); + emplace_back(origin, Op::_IntConst, std::vector{ir_int}, td::make_refint(value)); return ir_int; } diff --git a/tolk/analyzer.cpp b/tolk/analyzer.cpp index 5d70b80649..fdefc13b7c 100644 --- a/tolk/analyzer.cpp +++ b/tolk/analyzer.cpp @@ -14,7 +14,9 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ +#include "ast.h" #include "tolk.h" +#include "compilation-errors.h" #include "compiler-state.h" #include "type-system.h" @@ -500,8 +502,7 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) { return set_var_info(std::move(merge_info)); } default: - std::cerr << "fatal: unknown operation in compute_used_vars()\n"; - throw ParseError(loc, "unknown operation"); + fire(origin, "unknown operation in compute_used_vars()"); } } @@ -583,7 +584,7 @@ bool prune_unreachable(std::unique_ptr& ops) { op.cl = Op::_If; std::unique_ptr new_op = std::move(op.block0); op.block0 = std::move(op.block1); - op.block1 = std::make_unique(op.next->loc, Op::_Nop); + op.block1 = std::make_unique(op.next->origin, Op::_Nop); new_op->last().next = std::move(ops); ops = std::move(new_op); } @@ -628,8 +629,7 @@ bool prune_unreachable(std::unique_ptr& ops) { break; } default: - std::cerr << "fatal: unknown operation \n"; - throw ParseError(op.loc, "unknown operation in prune_unreachable()"); + fire(op.origin, "unknown operation in prune_unreachable()"); } if (reach) { return prune_unreachable(op.next); @@ -643,7 +643,7 @@ bool prune_unreachable(std::unique_ptr& ops) { void CodeBlob::prune_unreachable_code() { if (prune_unreachable(ops)) { - throw ParseError(fun_ref->loc, "control reaches end of function"); + fire(fun_ref, fun_ref->ident_anchor, "control reaches end of function (stack is malformed, a compiler bug)"); } } @@ -718,7 +718,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { if (arg_order_already_equals_asm()) { maybe_swap_builtin_args_to_compile(); } - std::get(f_sym->body)->compile(tmp, res, args, loc); + std::get(f_sym->body)->compile(tmp, res, args, origin); if (arg_order_already_equals_asm()) { maybe_swap_builtin_args_to_compile(); } @@ -835,8 +835,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { break; } default: - std::cerr << "fatal: unknown operation \n"; - throw ParseError(loc, "unknown operation in fwd_analyze()"); + fire(origin, "unknown operation in fwd_analyze()"); } if (next) { return next->fwd_analyze(std::move(values)); @@ -910,7 +909,7 @@ bool Op::mark_noreturn() { } last_in_block1->next = std::move(next); next = std::move(block1); - block1 = std::make_unique(loc, Op::_Nop); + block1 = std::make_unique(origin, Op::_Nop); block1->var_info = std::move(block1_var_info); } else { block1->mark_noreturn(); @@ -933,8 +932,7 @@ bool Op::mark_noreturn() { block0->mark_noreturn(); return set_noreturn(next->mark_noreturn()); default: - std::cerr << "fatal: unknown operation \n"; - throw ParseError(loc, "unknown operation in mark_noreturn()"); + fire(origin, "unknown operation in mark_noreturn()"); } } diff --git a/tolk/asmops.cpp b/tolk/asmops.cpp index 3dc39239b4..01d5973973 100644 --- a/tolk/asmops.cpp +++ b/tolk/asmops.cpp @@ -14,6 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ +#include "ast.h" #include "tolk.h" #include @@ -72,30 +73,30 @@ int AsmOp::SReg::calc_out_strlen() const { } -AsmOp AsmOp::Const(SrcLocation loc, int arg, const std::string& push_op) { +AsmOp AsmOp::Const(AnyV origin, int arg, const std::string& push_op) { std::ostringstream os; os << arg << ' ' << push_op; - return AsmOp::Const(loc, os.str()); + return AsmOp::Const(origin, os.str()); } -AsmOp AsmOp::make_stk2(SrcLocation loc, int a, int b, const char* str, int delta) { +AsmOp AsmOp::make_stk2(AnyV origin, int a, int b, const char* str, int delta) { std::ostringstream os; os << SReg(a) << ' ' << SReg(b) << ' ' << str; int c = std::max(a, b) + 1; - return AsmOp::Custom(loc, os.str(), c, c + delta); + return AsmOp::Custom(origin, os.str(), c, c + delta); } -AsmOp AsmOp::make_stk3(SrcLocation loc, int a, int b, int c, const char* str, int delta) { +AsmOp AsmOp::make_stk3(AnyV origin, int a, int b, int c, const char* str, int delta) { std::ostringstream os; os << SReg(a) << ' ' << SReg(b) << ' ' << SReg(c) << ' ' << str; int m = std::max(a, std::max(b, c)) + 1; - return AsmOp::Custom(loc, os.str(), m, m + delta); + return AsmOp::Custom(origin, os.str(), m, m + delta); } -AsmOp AsmOp::BlkSwap(SrcLocation loc, int a, int b) { +AsmOp AsmOp::BlkSwap(AnyV origin, int a, int b) { std::ostringstream os; if (a == 1 && b == 1) { - return AsmOp::Xchg(loc, 0, 1); + return AsmOp::Xchg(origin, 0, 1); } else if (a == 1) { if (b == 2) { os << "ROT"; @@ -111,127 +112,127 @@ AsmOp AsmOp::BlkSwap(SrcLocation loc, int a, int b) { } else { os << a << " " << b << " BLKSWAP"; } - return AsmOp::Custom(loc, os.str(), a + b, a + b); + return AsmOp::Custom(origin, os.str(), a + b, a + b); } -AsmOp AsmOp::BlkPush(SrcLocation loc, int a, int b) { +AsmOp AsmOp::BlkPush(AnyV origin, int a, int b) { std::ostringstream os; if (a == 1) { - return AsmOp::Push(loc, b); + return AsmOp::Push(origin, b); } else if (a == 2 && b == 1) { os << "2DUP"; } else { os << a << " " << b << " BLKPUSH"; } - return AsmOp::Custom(loc, os.str(), b + 1, a + b + 1); + return AsmOp::Custom(origin, os.str(), b + 1, a + b + 1); } -AsmOp AsmOp::BlkDrop(SrcLocation loc, int a) { +AsmOp AsmOp::BlkDrop(AnyV origin, int a) { std::ostringstream os; if (a == 1) { - return AsmOp::Pop(loc, 0); + return AsmOp::Pop(origin, 0); } else if (a == 2) { os << "2DROP"; } else { os << a << " BLKDROP"; } - return AsmOp::Custom(loc, os.str(), a, 0); + return AsmOp::Custom(origin, os.str(), a, 0); } -AsmOp AsmOp::BlkDrop2(SrcLocation loc, int a, int b) { +AsmOp AsmOp::BlkDrop2(AnyV origin, int a, int b) { if (!b) { - return BlkDrop(loc, a); + return BlkDrop(origin, a); } std::ostringstream os; os << a << " " << b << " BLKDROP2"; - return AsmOp::Custom(loc, os.str(), a + b, b); + return AsmOp::Custom(origin, os.str(), a + b, b); } -AsmOp AsmOp::BlkReverse(SrcLocation loc, int a, int b) { +AsmOp AsmOp::BlkReverse(AnyV origin, int a, int b) { std::ostringstream os; os << a << " " << b << " REVERSE"; - return AsmOp::Custom(loc, os.str(), a + b, a + b); + return AsmOp::Custom(origin, os.str(), a + b, a + b); } -AsmOp AsmOp::Tuple(SrcLocation loc, int a) { +AsmOp AsmOp::Tuple(AnyV origin, int a) { switch (a) { case 1: - return AsmOp::Custom(loc, "SINGLE", 1, 1); + return AsmOp::Custom(origin, "SINGLE", 1, 1); case 2: - return AsmOp::Custom(loc, "PAIR", 2, 1); + return AsmOp::Custom(origin, "PAIR", 2, 1); case 3: - return AsmOp::Custom(loc, "TRIPLE", 3, 1); + return AsmOp::Custom(origin, "TRIPLE", 3, 1); } std::ostringstream os; os << a << " TUPLE"; - return AsmOp::Custom(loc, os.str(), a, 1); + return AsmOp::Custom(origin, os.str(), a, 1); } -AsmOp AsmOp::UnTuple(SrcLocation loc, int a) { +AsmOp AsmOp::UnTuple(AnyV origin, int a) { switch (a) { case 1: - return AsmOp::Custom(loc, "UNSINGLE", 1, 1); + return AsmOp::Custom(origin, "UNSINGLE", 1, 1); case 2: - return AsmOp::Custom(loc, "UNPAIR", 1, 2); + return AsmOp::Custom(origin, "UNPAIR", 1, 2); case 3: - return AsmOp::Custom(loc, "UNTRIPLE", 1, 3); + return AsmOp::Custom(origin, "UNTRIPLE", 1, 3); } std::ostringstream os; os << a << " UNTUPLE"; - return AsmOp::Custom(loc, os.str(), 1, a); + return AsmOp::Custom(origin, os.str(), 1, a); } -AsmOp AsmOp::IntConst(SrcLocation loc, const td::RefInt256& x) { +AsmOp AsmOp::IntConst(AnyV origin, const td::RefInt256& x) { if (x->signed_fits_bits(8)) { - return AsmOp::Const(loc, dec_string(x) + " PUSHINT"); + return AsmOp::Const(origin, dec_string(x) + " PUSHINT"); } if (!x->is_valid()) { - return AsmOp::Const(loc, "PUSHNAN"); + return AsmOp::Const(origin, "PUSHNAN"); } int k = is_pos_pow2(x); if (k >= 0) { - return AsmOp::Const(loc, k, "PUSHPOW2"); + return AsmOp::Const(origin, k, "PUSHPOW2"); } k = is_pos_pow2(x + 1); if (k >= 0) { - return AsmOp::Const(loc, k, "PUSHPOW2DEC"); + return AsmOp::Const(origin, k, "PUSHPOW2DEC"); } k = is_pos_pow2(-x); if (k >= 0) { - return AsmOp::Const(loc, k, "PUSHNEGPOW2"); + return AsmOp::Const(origin, k, "PUSHNEGPOW2"); } if (!x->mod_pow2_short(23)) { - return AsmOp::Const(loc, dec_string(x) + " PUSHINTX"); + return AsmOp::Const(origin, dec_string(x) + " PUSHINTX"); } - return AsmOp::Const(loc, dec_string(x) + " PUSHINT"); + return AsmOp::Const(origin, dec_string(x) + " PUSHINT"); } -AsmOp AsmOp::BoolConst(SrcLocation loc, bool f) { - return AsmOp::Const(loc, f ? "TRUE" : "FALSE"); +AsmOp AsmOp::BoolConst(AnyV origin, bool f) { + return AsmOp::Const(origin, f ? "TRUE" : "FALSE"); } -AsmOp AsmOp::Parse(SrcLocation loc, const std::string& custom_op) { +AsmOp AsmOp::Parse(AnyV origin, const std::string& custom_op) { if (custom_op == "NOP") { - return AsmOp::Nop(loc); + return AsmOp::Nop(origin); } else if (custom_op == "SWAP") { - return AsmOp::Xchg(loc, 1); + return AsmOp::Xchg(origin, 1); } else if (custom_op == "DROP") { - return AsmOp::Pop(loc, 0); + return AsmOp::Pop(origin, 0); } else if (custom_op == "NIP") { - return AsmOp::Pop(loc, 1); + return AsmOp::Pop(origin, 1); } else if (custom_op == "DUP") { - return AsmOp::Push(loc, 0); + return AsmOp::Push(origin, 0); } else if (custom_op == "OVER") { - return AsmOp::Push(loc, 1); + return AsmOp::Push(origin, 1); } else if (custom_op.ends_with(" PUSHINT") && custom_op[0] >= '1' && custom_op[0] <= '9' && custom_op.find(' ') == custom_op.rfind(' ')) { - return AsmOp::IntConst(loc, td::string_to_int256(custom_op.substr(0, custom_op.find(' ')))); + return AsmOp::IntConst(origin, td::string_to_int256(custom_op.substr(0, custom_op.find(' ')))); } else { - return AsmOp::Custom(loc, custom_op); + return AsmOp::Custom(origin, custom_op); } } -AsmOp AsmOp::Parse(SrcLocation loc, std::string custom_op, int args, int retv) { - auto res = Parse(loc, custom_op); +AsmOp AsmOp::Parse(AnyV origin, std::string custom_op, int args, int retv) { + auto res = Parse(origin, custom_op); if (res.is_custom()) { res.a = args; res.b = retv; @@ -275,9 +276,8 @@ int AsmOp::out(std::ostream& os) const { } int AsmOp::out_indented(std::ostream& os, bool print_src_line_above) const { - static int last_line_no = -1; - if (loc.is_defined() && print_src_line_above) { - loc.show_line_to_fif_output(os, indent, &last_line_no); + if (origin && origin->range.is_valid() && print_src_line_above) { + origin->range.output_first_line_to_fif(os, indent); } for (int i = 0; i < indent * 2; i++) { os << ' '; diff --git a/tolk/ast-aux-data.h b/tolk/ast-aux-data.h index 36a806a091..1f97771281 100644 --- a/tolk/ast-aux-data.h +++ b/tolk/ast-aux-data.h @@ -18,7 +18,6 @@ #include "ast.h" #include "lazy-helpers.h" -#include "tolk.h" /* * This file contains a schema of aux_data inside ast_artificial_aux_vertex @@ -27,13 +26,15 @@ namespace tolk { +struct CodeBlob; + // AuxData_ForceFiftLocation is created when transforming AST to IR; // it wraps constants to force codegen location point to usage, not to init_val AST nodes struct AuxData_ForceFiftLocation final : ASTAuxData { - SrcLocation forced_loc; + AnyV forced_origin; - explicit AuxData_ForceFiftLocation(SrcLocation forced_loc) - : forced_loc(forced_loc) { + explicit AuxData_ForceFiftLocation(AnyV forced_origin) + : forced_origin(forced_origin) { } }; @@ -69,7 +70,7 @@ struct AuxData_OnInternalMessage_getField final : ASTAuxData { , field_name(field_name) { } - std::vector generate_get_InMessage_field(CodeBlob& code, SrcLocation loc) const; + std::vector generate_get_InMessage_field(CodeBlob& code, AnyV origin) const; }; } // namespace tolk diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 03016a5fe8..2cc42b101f 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -15,6 +15,7 @@ along with TON Blockchain Library. If not, see . */ #include "ast.h" +#include "compilation-errors.h" #include "type-system.h" #include "platform-utils.h" #include "tolk-version.h" @@ -52,31 +53,31 @@ static bool is_add_or_sub_binary_op(TokenType tok) { // fire an error for a case "flags & 0xFF != 0" (equivalent to "flags & 1", probably unexpected) // it would better be a warning, but we decided to make it a strict error GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_lower_precedence(SrcLocation loc, std::string_view op_lower, std::string_view op_higher) { +static void fire_lower_precedence(SrcRange range, std::string_view op_lower, std::string_view op_higher) { std::string name_lower = static_cast(op_lower); std::string name_higher = static_cast(op_higher); - throw ParseError(loc, name_lower + " has lower precedence than " + name_higher + - ", probably this code won't work as you expected. " - "Use parenthesis: either (... " + name_lower + " ...) to evaluate it first, or (... " + name_higher + " ...) to suppress this error."); + fire(range, name_lower + " has lower precedence than " + name_higher + + ", probably this code won't work as you expected. " + "Use parenthesis: either (... " + name_lower + " ...) to evaluate it first, or (... " + name_higher + " ...) to suppress this error."); } // fire an error for a case "arg1 & arg2 | arg3" GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_mix_and_or_no_parenthesis(SrcLocation loc, std::string_view op1, std::string_view op2) { +static void fire_mix_and_or_no_parenthesis(SrcRange range, std::string_view op1, std::string_view op2) { std::string name1 = static_cast(op1); std::string name2 = static_cast(op2); - throw ParseError(loc, "mixing " + name1 + " with " + name2 + " without parenthesis may lead to accidental errors. " - "Use parenthesis to emphasize operator precedence."); + fire(range, "mixing " + name1 + " with " + name2 + " without parenthesis may lead to accidental errors. " + "Use parenthesis to emphasize operator precedence."); } // fire an error "Tolk does not have ++i operator" GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_no_increment_operator(SrcLocation loc, bool is_increment) { +static void fire_no_increment_operator(SrcRange range, bool is_increment) { std::string op_name = is_increment ? "increment" : "decrement"; - std::string op_wrong = is_increment ? "++" : "--"; + std::string op_wrong = is_increment ? "++" : "--"; std::string op_right = is_increment ? "+=" : "-="; - throw ParseError(loc, std::string("Tolk has no ") + op_name + " operator\n" + - "hint: use `i " + op_right + " 1`, not `i" + op_wrong + "`"); + fire(range, std::string("Tolk has no ") + op_name + " operator\n" + + "hint: use `i " + op_right + " 1`, not `i" + op_wrong + "`"); } // diagnose when bitwise operators are used in a probably wrong way due to tricky precedence @@ -84,53 +85,55 @@ static void fire_error_no_increment_operator(SrcLocation loc, bool is_increment) // the only way to suppress this error for the programmer is to use parenthesis // (how do we detect presence of parenthesis? simple: (0!=1) is ast_parenthesized_expr{ast_binary_operator}, // that's why if rhs->type == ast_binary_operator, it's not surrounded by parenthesis) -static void diagnose_bitwise_precedence(SrcLocation loc, std::string_view operator_name, AnyExprV lhs, AnyExprV rhs) { +static void diagnose_bitwise_precedence(SrcRange range, std::string_view operator_name, AnyExprV lhs, AnyExprV rhs) { // handle "flags & 0xFF != 0" (rhs = "0xFF != 0") if (rhs->kind == ast_binary_operator && is_comparison_binary_op(rhs->as()->tok)) { - fire_error_lower_precedence(loc, operator_name, rhs->as()->operator_name); + fire_lower_precedence(range, operator_name, rhs->as()->operator_name); } // handle "0 != flags & 0xFF" (lhs = "0 != flags") if (lhs->kind == ast_binary_operator && is_comparison_binary_op(lhs->as()->tok)) { - fire_error_lower_precedence(loc, operator_name, lhs->as()->operator_name); + fire_lower_precedence(range, operator_name, lhs->as()->operator_name); } } // similar to above, but detect potentially invalid usage of && and || // since anyway, using parenthesis when both && and || occur in the same expression, // && and || have equal operator precedence in Tolk -static void diagnose_and_or_precedence(SrcLocation loc, AnyExprV lhs, TokenType rhs_tok, std::string_view rhs_operator_name) { +static void diagnose_and_or_precedence(SrcRange range, AnyExprV lhs, TokenType rhs_tok, std::string_view rhs_operator_name) { if (auto lhs_op = lhs->try_as()) { // handle "arg1 & arg2 | arg3" (lhs = "arg1 & arg2") if (is_bitwise_binary_op(lhs_op->tok) && is_bitwise_binary_op(rhs_tok) && lhs_op->tok != rhs_tok) { - fire_error_mix_and_or_no_parenthesis(loc, lhs_op->operator_name, rhs_operator_name); + fire_mix_and_or_no_parenthesis(range, lhs_op->operator_name, rhs_operator_name); } // handle "arg1 && arg2 || arg3" (lhs = "arg1 && arg2") if (is_logical_binary_op(lhs_op->tok) && is_logical_binary_op(rhs_tok) && lhs_op->tok != rhs_tok) { - fire_error_mix_and_or_no_parenthesis(loc, lhs_op->operator_name, rhs_operator_name); + fire_mix_and_or_no_parenthesis(range, lhs_op->operator_name, rhs_operator_name); } } } // diagnose "a << 8 + 1" (equivalent to "a << 9", probably unexpected) -static void diagnose_addition_in_bitshift(SrcLocation loc, std::string_view bitshift_operator_name, AnyExprV rhs) { +static void diagnose_addition_in_bitshift(SrcRange range, std::string_view bitshift_operator_name, AnyExprV rhs) { if (rhs->kind == ast_binary_operator && is_add_or_sub_binary_op(rhs->as()->tok)) { - fire_error_lower_precedence(loc, bitshift_operator_name, rhs->as()->operator_name); + fire_lower_precedence(range, bitshift_operator_name, rhs->as()->operator_name); } } // replace (a == null) and similar to ast_is_type_operator(a, null) (as if `a is null` was written) static AnyExprV maybe_replace_eq_null_with_isNull_check(V v) { - bool has_null = v->get_lhs()->kind == ast_null_keyword || v->get_rhs()->kind == ast_null_keyword; - bool replace = has_null && (v->tok == tok_eq || v->tok == tok_neq); + bool lhs_is_null = v->get_lhs()->kind == ast_null_keyword; + bool rhs_is_null = v->get_rhs()->kind == ast_null_keyword; + bool replace = (lhs_is_null || rhs_is_null) && (v->tok == tok_eq || v->tok == tok_neq); if (!replace) { return v; } - AnyExprV v_nullable = v->get_lhs()->kind == ast_null_keyword ? v->get_rhs() : v->get_lhs(); - AnyTypeV rhs_null_type = createV(v->loc, "null"); - return createV(v->loc, v_nullable, rhs_null_type, v->tok == tok_neq); + AnyExprV v_null_kw = lhs_is_null ? v->get_lhs() : v->get_rhs(); + AnyExprV v_nullable = lhs_is_null ? v->get_rhs() : v->get_lhs(); + AnyTypeV rhs_null_type = createV(v_null_kw->range, "null"); + return createV(v->range, v_nullable, rhs_null_type, v->tok == tok_neq); } // parse `123` / `0xFF` / `0b10001` to td::RefInt256 @@ -165,11 +168,12 @@ static td::RefInt256 parse_tok_int_const(std::string_view text) { static AnyTypeV parse_type_expression(Lexer& lex); -static std::vector parse_nested_type_list(Lexer& lex, TokenType tok_op, const char* s_op, TokenType tok_cl, const char* s_cl) { +static std::vector parse_nested_type_list(Lexer& lex, TokenType tok_op, const char* s_op, TokenType tok_cl, const char* s_cl, SrcRange& out_range) { lex.expect(tok_op, s_op); std::vector sub_types; while (true) { if (lex.tok() == tok_cl) { // empty lists allowed + out_range.end(lex.cur_range()); lex.next(); break; } @@ -182,6 +186,7 @@ static std::vector parse_nested_type_list(Lexer& lex, TokenType tok_op // treat token `>>` like two `>`; consume one here doing nothing (break) and leave the second `>` in a lexer if (tok_cl == tok_gt && lex.tok() == tok_rshift) { lex.hack_replace_rshift_with_one_triangle(); + out_range.end(lex.cur_range()); break; } lex.unexpected(s_cl); @@ -190,25 +195,25 @@ static std::vector parse_nested_type_list(Lexer& lex, TokenType tok_op return sub_types; } -static std::vector parse_nested_type_list_in_triangles(Lexer& lex) { - return parse_nested_type_list(lex, tok_lt, "`<`", tok_gt, "`>` or `,`"); -} - static AnyTypeV parse_simple_type(Lexer& lex) { - SrcLocation loc = lex.cur_location(); switch (lex.tok()) { case tok_self: case tok_identifier: case tok_null: { + SrcRange range = lex.cur_range(); std::string_view text = lex.cur_str(); lex.next(); - return createV(loc, text); + return createV(range, text); } case tok_oppar: { - return createV(loc, parse_nested_type_list(lex, tok_oppar, "`(`", tok_clpar, "`)` or `,`")); + SrcRange range = lex.range_start(); + std::vector tensor_items = parse_nested_type_list(lex, tok_oppar, "`(`", tok_clpar, "`)` or `,`", range); + return createV(range, std::move(tensor_items)); } case tok_opbracket: { - return createV(loc, parse_nested_type_list(lex, tok_opbracket, "`[`", tok_clbracket, "`]` or `,`")); + SrcRange range = lex.range_start(); + std::vector tuple_items = parse_nested_type_list(lex, tok_opbracket, "`[`", tok_clbracket, "`]` or `,`", range); + return createV(range, std::move(tuple_items)); } default: lex.unexpected(""); @@ -218,18 +223,20 @@ static AnyTypeV parse_simple_type(Lexer& lex) { static AnyTypeV parse_type_nullable(Lexer& lex) { AnyTypeV result = parse_simple_type(lex); - if (lex.tok() == tok_lt) { - std::vector args = parse_nested_type_list_in_triangles(lex); + if (lex.tok() == tok_lt) { // T1 + SrcRange range = SrcRange::empty_at_start(result->range); + auto args = parse_nested_type_list(lex, tok_lt, "`<`", tok_gt, "`>` or `,`", range); std::vector outer_and_args; outer_and_args.reserve(1 + args.size()); outer_and_args.push_back(result); outer_and_args.insert(outer_and_args.end(), args.begin(), args.end()); - result = createV(result->loc, std::move(outer_and_args)); + result = createV(range, std::move(outer_and_args)); } if (lex.tok() == tok_question) { + SrcRange range = SrcRange::overlap(result->range, lex.cur_range()); lex.next(); - result = createV(result->loc, result); + result = createV(range, result); } return result; @@ -248,7 +255,8 @@ static AnyTypeV parse_type_expression(Lexer& lex) { lex.next(); items.emplace_back(parse_type_nullable(lex)); } - result = createV(result->loc, std::move(items)); + SrcRange range = SrcRange::overlap(items.front()->range, items.back()->range); + result = createV(range, std::move(items)); } if (lex.tok() == tok_arrow) { // `int -> int`, `(cell, slice) -> void`, `int -> int -> int`, `int | cell -> void` @@ -262,7 +270,8 @@ static AnyTypeV parse_type_expression(Lexer& lex) { params_and_return.push_back(result); } params_and_return.push_back(parse_type_expression(lex)); - result = createV(result->loc, std::move(params_and_return)); + SrcRange range = SrcRange::overlap(params_and_return.front()->range, params_and_return.back()->range); + result = createV(range, std::move(params_and_return)); } return result; @@ -282,32 +291,44 @@ static AnyTypeV parse_type_from_tokens(Lexer& lex) { AnyExprV parse_expr(Lexer& lex); AnyV parse_statement(Lexer& lex); +static V parse_identifier(Lexer& lex, const char* str_expected) { + lex.check(tok_identifier, str_expected); + SrcRange range = lex.cur_range(); + std::string_view name = lex.cur_str(); + lex.next(); + return createV(range, name); +} + static V parse_genericsT_list(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); std::vector genericsT_items; lex.expect(tok_lt, "`<`"); while (true) { lex.check(tok_identifier, "T"); - SrcLocation locT = lex.cur_location(); + SrcRange rangeT = lex.cur_range(); std::string_view nameT = lex.cur_str(); lex.next(); AnyTypeV default_type = nullptr; if (lex.tok() == tok_assign) { // lex.next(); default_type = parse_type_expression(lex); + rangeT.end(default_type->range); } - genericsT_items.emplace_back(createV(locT, nameT, default_type)); + genericsT_items.emplace_back(createV(rangeT, nameT, default_type)); if (lex.tok() != tok_comma) { break; } lex.next(); } - lex.expect(tok_gt, "`>`"); - return createV{loc, std::move(genericsT_items)}; + + lex.check(tok_gt, "`>`"); + range.end(lex.cur_range()); + lex.next(); + return createV(range, std::move(genericsT_items)); } static AnyV parse_parameter(Lexer& lex, AnyTypeV self_type) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); // optional keyword `mutate` meaning that a function will mutate a passed argument (like passed by reference) bool declared_as_mutate = false; @@ -317,26 +338,31 @@ static AnyV parse_parameter(Lexer& lex, AnyTypeV self_type) { } // parameter name (or underscore for an unnamed parameter) - std::string_view param_name; + V v_ident = nullptr; bool is_self = false; if (lex.tok() == tok_identifier) { - param_name = lex.cur_str(); + v_ident = parse_identifier(lex, "parameter name"); } else if (lex.tok() == tok_self) { if (!self_type) { lex.error("`self` can only be the first parameter of a method"); } is_self = true; - param_name = "self"; - } else if (lex.tok() != tok_underscore) { + v_ident = createV(lex.cur_range(), "self"); + lex.next(); + } else if (lex.tok() == tok_underscore) { + v_ident = createV(lex.cur_range(), ""); + lex.next(); + } else { lex.unexpected("parameter name"); } - lex.next(); + range.end(v_ident->range); // parameter type after colon is mandatory AnyTypeV param_type = self_type; if (!is_self) { lex.expect(tok_colon, "`: `"); param_type = parse_type_from_tokens(lex); + range.end(param_type->range); } else if (lex.tok() == tok_colon) { lex.error("`self` parameter should not have a type"); } @@ -349,19 +375,19 @@ static AnyV parse_parameter(Lexer& lex, AnyTypeV self_type) { } lex.next(); default_value = parse_expr(lex); + range.end(default_value->range); } - return createV(loc, param_name, param_type, default_value, declared_as_mutate); + return createV(range, v_ident, param_type, default_value, declared_as_mutate); } static AnyV parse_global_var_declaration(Lexer& lex, const std::vector>& annotations) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.expect(tok_global, "`global`"); - lex.check(tok_identifier, "global variable name"); - auto v_ident = createV(lex.cur_location(), lex.cur_str()); - lex.next(); + auto v_ident = parse_identifier(lex, "global variable name"); lex.expect(tok_colon, "`:`"); AnyTypeV declared_type = parse_type_from_tokens(lex); + range.end(declared_type->range); if (lex.tok() == tok_comma) { lex.error("multiple declarations are not allowed, split globals on separate lines"); } @@ -375,19 +401,17 @@ static AnyV parse_global_var_declaration(Lexer& lex, const std::vectorerror("this annotation is not applicable to global"); + fire(v_annotation, "this annotation is not applicable to global"); } } - return createV(loc, v_ident, declared_type); + return createV(range, v_ident, declared_type); } static AnyV parse_constant_declaration(Lexer& lex, const std::vector>& annotations) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.expect(tok_const, "`const`"); - lex.check(tok_identifier, "constant name"); - auto v_ident = createV(lex.cur_location(), lex.cur_str()); - lex.next(); + auto v_ident = parse_identifier(lex, "constant name"); AnyTypeV declared_type = nullptr; if (lex.tok() == tok_colon) { lex.next(); @@ -405,19 +429,18 @@ static AnyV parse_constant_declaration(Lexer& lex, const std::vectorerror("this annotation is not applicable to constant"); + fire(v_annotation, "this annotation is not applicable to constant"); } } - return createV(loc, v_ident, declared_type, init_value); + range.end(init_value->range); + return createV(range, v_ident, declared_type, init_value); } static AnyV parse_type_alias_declaration(Lexer& lex, const std::vector>& annotations) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.expect(tok_type, "`type`"); - lex.check(tok_identifier, "type name"); - auto v_ident = createV(lex.cur_location(), lex.cur_str()); - lex.next(); + auto v_ident = parse_identifier(lex, "type name"); V genericsT_list = nullptr; if (lex.tok() == tok_lt) { // 'type Response' @@ -426,11 +449,13 @@ static AnyV parse_type_alias_declaration(Lexer& lex, const std::vector = builtin + range.end(lex.cur_range()); lex.next(); - return createV(loc); + return createV(range); } AnyTypeV underlying_type = parse_type_from_tokens(lex); + range.end(underlying_type->range); for (auto v_annotation : annotations) { switch (v_annotation->kind) { @@ -438,16 +463,16 @@ static AnyV parse_type_alias_declaration(Lexer& lex, const std::vectorerror("this annotation is not applicable to type alias"); + fire(v_annotation, "this annotation is not applicable to type alias"); } } - return createV(loc, v_ident, genericsT_list, underlying_type); + return createV(range, v_ident, genericsT_list, underlying_type); } static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable, bool allow_lateinit) { - SrcLocation loc = lex.cur_location(); if (lex.tok() == tok_oppar) { + SrcRange range = lex.range_start(); lex.next(); AnyExprV first = parse_var_declaration_lhs(lex, is_immutable, false); if (lex.tok() == tok_clpar) { @@ -462,10 +487,13 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable, bool al } args.push_back(parse_var_declaration_lhs(lex, is_immutable, false)); } - lex.expect(tok_clpar, "`)`"); - return createV(loc, std::move(args)); + lex.check(tok_clpar, "`)`"); + range.end(lex.cur_range()); + lex.next(); + return createV(range, std::move(args)); } if (lex.tok() == tok_opbracket) { + SrcRange range = lex.range_start(); lex.next(); std::vector args(1, parse_var_declaration_lhs(lex, is_immutable, false)); while (lex.tok() == tok_comma) { @@ -475,18 +503,22 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable, bool al } args.push_back(parse_var_declaration_lhs(lex, is_immutable, false)); } - lex.expect(tok_clbracket, "`]`"); - return createV(loc, std::move(args)); + lex.check(tok_clbracket, "`]`"); + range.end(lex.cur_range()); + lex.next(); + return createV(range, std::move(args)); } if (lex.tok() == tok_identifier) { - auto v_ident = createV(loc, lex.cur_str()); + SrcRange range = lex.range_start(); + auto v_ident = parse_identifier(lex, "variable name"); + range.end(v_ident->range); AnyTypeV declared_type = nullptr; bool marked_as_redef = false; bool is_lateinit = false; - lex.next(); if (lex.tok() == tok_colon) { lex.next(); declared_type = parse_type_from_tokens(lex); + range.end(declared_type->range); } else if (lex.tok() == tok_redef) { lex.next(); marked_as_redef = true; @@ -497,22 +529,24 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable, bool al } is_lateinit = true; } - return createV(loc, v_ident, declared_type, is_immutable, is_lateinit, marked_as_redef); + return createV(range, v_ident, declared_type, is_immutable, is_lateinit, marked_as_redef); } if (lex.tok() == tok_underscore) { + SrcRange range = lex.cur_range(); AnyTypeV declared_type = nullptr; lex.next(); if (lex.tok() == tok_colon) { lex.next(); declared_type = parse_type_from_tokens(lex); + range.end(declared_type->range); } - return createV(loc, createV(loc, ""), declared_type, true, false, false); + return createV(range, createV(range, ""), declared_type, true, false, false); } lex.unexpected("variable name"); } static AnyExprV parse_local_vars_declaration(Lexer& lex, bool allow_lateinit) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); bool is_immutable = lex.tok() == tok_val; lex.next(); @@ -529,13 +563,14 @@ static AnyExprV parse_local_vars_declaration(Lexer& lex, bool allow_lateinit) { if (lex.tok() == tok_comma) { lex.error("multiple declarations are not allowed, split variables on separate lines"); } - return createV(loc, createV(loc, lhs), rhs); + range.end(rhs->range); + return createV(range, createV(range, lhs), rhs); } // "parameters" are at function declaration: `fun f(param1: int, mutate param2: slice)` // for methods like `fun builder.storeUint(self, i: int)`, receiver_type = builder (type of self) static V parse_parameter_list(Lexer& lex, AnyTypeV receiver_type) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); std::vector params; lex.expect(tok_oppar, "parameter list"); if (lex.tok() != tok_clpar) { @@ -548,13 +583,15 @@ static V parse_parameter_list(Lexer& lex, AnyTypeV receiver_ params.push_back(parse_parameter(lex, nullptr)); } } - lex.expect(tok_clpar, "`)`"); - return createV(loc, std::move(params)); + lex.check(tok_clpar, "`)`"); + range.end(lex.cur_range()); + lex.next(); + return createV(range, std::move(params)); } // "arguments" are at function call: `f(arg1, mutate arg2)` static AnyExprV parse_argument(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); // keyword `mutate` is necessary when a parameter is declared `mutate` (to make mutation obvious for the reader) bool passed_as_mutate = false; @@ -564,11 +601,12 @@ static AnyExprV parse_argument(Lexer& lex) { } AnyExprV expr = parse_expr(lex); - return createV(loc, expr, passed_as_mutate); + range.end(expr->range); + return createV(range, expr, passed_as_mutate); } static V parse_argument_list(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); std::vector args; lex.expect(tok_oppar, "`(`"); if (lex.tok() != tok_clpar) { @@ -581,24 +619,31 @@ static V parse_argument_list(Lexer& lex) { args.push_back(parse_argument(lex)); } } - lex.expect(tok_clpar, "`)`"); - return createV(loc, std::move(args)); + + lex.check(tok_clpar, "`)`"); + range.end(lex.cur_range()); + lex.next(); + return createV(range, std::move(args)); } static V parse_maybe_instantiationTs_after_identifier(Lexer& lex) { lex.check(tok_lt, "`<`"); Lexer::SavedPositionForLookahead backup = lex.save_parsing_position(); try { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.next(); std::vector instantiationTs; - instantiationTs.push_back(createV(lex.cur_location(), parse_type_from_tokens(lex))); + AnyTypeV type_node = parse_type_from_tokens(lex); + instantiationTs.push_back(createV(type_node->range, type_node)); while (lex.tok() == tok_comma) { lex.next(); - instantiationTs.push_back(createV(lex.cur_location(), parse_type_from_tokens(lex))); + type_node = parse_type_from_tokens(lex); + instantiationTs.push_back(createV(type_node->range, type_node)); } - lex.expect(tok_gt, "`>`"); - return createV(loc, std::move(instantiationTs)); + lex.check(tok_gt, "`>`"); + range.end(lex.cur_range()); + lex.next(); + return createV(range, std::move(instantiationTs)); } catch (const ParseError&) { lex.restore_position(backup); return nullptr; @@ -606,23 +651,24 @@ static V parse_maybe_instantiationTs_after_identifier(L } static V parse_object_field(Lexer& lex) { - SrcLocation loc = lex.cur_location(); - lex.check(tok_identifier, "field name"); - auto v_ident = createV(lex.cur_location(), lex.cur_str()); - lex.next(); + SrcRange range = lex.range_start(); + auto v_ident = parse_identifier(lex, "field name"); + range.end(v_ident->range); if (lex.tok() == tok_comma || lex.tok() == tok_clbrace) { - auto v_same_ident = createV(v_ident->loc, v_ident->name); - auto v_same_expr = createV(v_ident->loc, v_same_ident, nullptr); - return createV(loc, v_ident, v_same_expr); + auto v_same_ident = createV(v_ident->range, v_ident->name); + auto v_same_expr = createV(v_ident->range, v_same_ident, nullptr); + return createV(range, v_ident, v_same_expr); } lex.expect(tok_colon, "`:`"); - return createV(loc, v_ident, parse_expr(lex)); + AnyExprV init_val = parse_expr(lex); + range.end(init_val->range); + return createV(range, v_ident, init_val); } static V parse_object_body(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.expect(tok_opbrace, "`{`"); std::vector fields; @@ -634,16 +680,18 @@ static V parse_object_body(Lexer& lex) { lex.unexpected("`,`"); } } - lex.expect(tok_clbrace, "`}`"); - return createV(loc, std::move(fields)); + lex.check(tok_clbrace, "`}`"); + range.end(lex.cur_range()); + lex.next(); + return createV(range, std::move(fields)); } // `throw code` / `throw (code)` / `throw (code, arg)` // it's technically a statement (can't occur "in any place of expression"), // but inside `match` arm it can appear without semicolon: `pattern => throw 123` static AnyV parse_throw_expression(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.expect(tok_throw, "`throw`"); AnyExprV thrown_code, thrown_arg; @@ -654,20 +702,23 @@ static AnyV parse_throw_expression(Lexer& lex) { lex.next(); thrown_arg = parse_expr(lex); } else { - thrown_arg = createV(loc); + thrown_arg = createV(SrcRange::empty_at_start(range)); } - lex.expect(tok_clpar, "`)`"); + lex.check(tok_clpar, "`)`"); + range.end(lex.cur_range()); + lex.next(); } else { // throw code thrown_code = parse_expr(lex); - thrown_arg = createV(loc); + range.end(thrown_code->range); + thrown_arg = createV(SrcRange::empty_at_start(range)); } - return createV(loc, thrown_code, thrown_arg); + return createV(range, thrown_code, thrown_arg); } // `pattern => body` inside `match` static V parse_match_arm(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); MatchArmKind pattern_kind = static_cast(-1); AnyTypeV exact_type = nullptr; AnyExprV pattern_expr = nullptr; @@ -694,37 +745,41 @@ static V parse_match_arm(Lexer& lex) { if (pattern_kind == static_cast(-1)) { lex.restore_position(backup); - throw ParseError(loc, "expected or in `match` before `=>`"); + lex.error("expected or in `match` before `=>`"); } lex.expect(tok_double_arrow, "`=>`"); V body = nullptr; if (lex.tok() == tok_opbrace) { // pattern => { ... } AnyV v_block = parse_statement(lex); - body = createV(v_block->loc, v_block); + body = createV(v_block->range, v_block); } else if (lex.tok() == tok_throw) { // pattern => throw 123 (allow without braces) AnyV v_throw = parse_throw_expression(lex); - AnyV v_block = createV(v_throw->loc, v_throw->loc, {v_throw}); - body = createV(v_block->loc, v_block); + AnyV v_block = createV(v_throw->range, {v_throw}); + body = createV(v_block->range, v_block); } else if (lex.tok() == tok_return) { // pattern => return 123 (allow without braces, like throw) + SrcRange block_range = lex.range_start(); lex.next(); - AnyV v_return = createV(lex.cur_location(), parse_expr(lex)); - AnyV v_block = createV(v_return->loc, v_return->loc, {v_return}); - body = createV(v_block->loc, v_block); + AnyExprV return_value = parse_expr(lex); + block_range.end(return_value->range); + AnyV v_return = createV(block_range, return_value); + AnyV v_block = createV(v_return->range, {v_return}); + body = createV(v_block->range, v_block); } else { AnyExprV unbraced_expr = parse_expr(lex); - AnyV v_block = createV(unbraced_expr->loc, unbraced_expr->loc, {createV(unbraced_expr->loc, unbraced_expr)}); - body = createV(unbraced_expr->loc, v_block); + AnyV v_block = createV(unbraced_expr->range, {createV(unbraced_expr->range, unbraced_expr)}); + body = createV(unbraced_expr->range, v_block); } + range.end(body->range); if (pattern_expr == nullptr) { // for match by type / default case, empty vertex, not nullptr - pattern_expr = createV(loc); + pattern_expr = createV(SrcRange::empty_at_start(range)); } - return createV(loc, pattern_kind, exact_type, pattern_expr, body); + return createV(range, pattern_kind, exact_type, pattern_expr, body); } static V parse_match_expression(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.expect(tok_match, "`match`"); lex.expect(tok_oppar, "`(`"); @@ -752,32 +807,38 @@ static V parse_match_expression(Lexer& lex) { lex.unexpected("`,`"); } } - lex.expect(tok_clbrace, "`}`"); - return createV(loc, std::move(subject_and_arms)); + + lex.check(tok_clbrace, "`}`"); + range.end(lex.cur_range()); + lex.next(); + return createV(range, std::move(subject_and_arms)); } static V parse_lazy_operator(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.expect(tok_lazy, "`lazy`"); AnyExprV expr = parse_expr(lex); - return createV(loc, expr); + range.end(expr->range); + return createV(range, expr); } // parse (expr) / [expr] / identifier / number static AnyExprV parse_expr100(Lexer& lex) { - SrcLocation loc = lex.cur_location(); switch (lex.tok()) { case tok_oppar: { + SrcRange range = lex.range_start(); lex.next(); if (lex.tok() == tok_clpar) { + range.end(lex.cur_range()); lex.next(); - return createV(loc, {}); + return createV(range, {}); } AnyExprV first = parse_expr(lex); if (lex.tok() == tok_clpar) { + range.end(lex.cur_range()); lex.next(); - return createV(loc, first); + return createV(range, first); } std::vector items(1, first); while (lex.tok() == tok_comma) { @@ -787,17 +848,21 @@ static AnyExprV parse_expr100(Lexer& lex) { } items.emplace_back(parse_expr(lex)); } - lex.expect(tok_clpar, "`)`"); + lex.check(tok_clpar, "`)`"); + range.end(lex.cur_range()); + lex.next(); if (items.size() == 1) { // we can reach here for 1 element with trailing comma: `(item, )` return items[0]; // then just return item, not a 1-element tensor, } // since 1-element tensors won't be type compatible with item's type - return createV(loc, std::move(items)); + return createV(range, std::move(items)); } case tok_opbracket: { + SrcRange range = lex.range_start(); lex.next(); if (lex.tok() == tok_clbracket) { + range.end(lex.cur_range()); lex.next(); - return createV(loc, {}); + return createV(range, {}); } std::vector items(1, parse_expr(lex)); while (lex.tok() == tok_comma) { @@ -807,68 +872,90 @@ static AnyExprV parse_expr100(Lexer& lex) { } items.emplace_back(parse_expr(lex)); } - lex.expect(tok_clbracket, "`]`"); - return createV(loc, std::move(items)); + lex.check(tok_clbracket, "`]`"); + range.end(lex.cur_range()); + lex.next(); + return createV(range, std::move(items)); } case tok_int_const: { + SrcRange range = lex.cur_range(); std::string_view orig_str = lex.cur_str(); td::RefInt256 intval = parse_tok_int_const(orig_str); if (intval.is_null() || !intval->signed_fits_bits(257)) { lex.error("invalid integer constant"); } lex.next(); - return createV(loc, std::move(intval), orig_str); + return createV(range, std::move(intval), orig_str); } case tok_string_const: { - std::string_view str_val = lex.cur_str(); + SrcRange range = lex.cur_range(); + std::string_view str_val = lex.cur_str(); // with surrounding quotes, remove them + if (str_val.starts_with(R"(""")")) { // multi-line literal: begins-ends with 3 quotes + str_val = str_val.substr(3, str_val.size() - 6); + } else { + str_val = str_val.substr(1, str_val.size() - 2); + } lex.next(); - return createV(loc, str_val); + return createV(range, str_val); } case tok_underscore: { + SrcRange range = lex.cur_range(); lex.next(); - return createV(loc); + return createV(range); } case tok_true: { + SrcRange range = lex.cur_range(); lex.next(); - return createV(loc, true); + return createV(range, true); } case tok_false: { + SrcRange range = lex.cur_range(); lex.next(); - return createV(loc, false); + return createV(range, false); } case tok_null: { + SrcRange range = lex.cur_range(); lex.next(); - return createV(loc); + return createV(range); } case tok_self: { + SrcRange range = lex.cur_range(); lex.next(); - auto v_ident = createV(loc, "self"); - return createV(loc, v_ident, nullptr); + auto v_ident = createV(range, "self"); + return createV(range, v_ident, nullptr); } case tok_identifier: { - auto v_ident = createV(loc, lex.cur_str()); + auto v_ident = parse_identifier(lex, "identifier"); + SrcRange range = v_ident->range; V v_instantiationTs = nullptr; - lex.next(); if (lex.tok() == tok_lt) { v_instantiationTs = parse_maybe_instantiationTs_after_identifier(lex); + if (v_instantiationTs) { + range.end(v_instantiationTs->range); + } } if (lex.tok() == tok_opbrace) { - AnyTypeV type_node = createV(v_ident->loc, v_ident->name); // `Pair { ... }` - if (v_instantiationTs) { // `Pair { ... }` + AnyTypeV type_node = createV(v_ident->range, v_ident->name); // `Pair { ... }` + if (v_instantiationTs) { // `Pair { ... }` std::vector ident_and_args; ident_and_args.reserve(1 + v_instantiationTs->size()); ident_and_args.push_back(type_node); for (int i = 0; i < v_instantiationTs->size(); ++i) { ident_and_args.push_back(v_instantiationTs->get_item(i)->type_node); } - type_node = createV(v_ident->loc, std::move(ident_and_args)); + SrcRange tri_range = SrcRange::overlap(v_ident->range, v_instantiationTs->range); + type_node = createV(tri_range, std::move(ident_and_args)); } - return createV(loc, type_node, parse_object_body(lex)); + auto body = parse_object_body(lex); + range.end(body->range); + return createV(range, type_node, body); } - return createV(loc, v_ident, v_instantiationTs); + return createV(range, v_ident, v_instantiationTs); + } + case tok_opbrace: { + auto body = parse_object_body(lex); + return createV(body->range, nullptr, body); } - case tok_opbrace: - return createV(loc, nullptr, parse_object_body(lex)); case tok_match: return parse_match_expression(lex); case tok_lazy: @@ -882,12 +969,15 @@ static AnyExprV parse_expr100(Lexer& lex) { static AnyExprV parse_fun_call_postfix(Lexer& lex, AnyExprV lhs) { while (true) { if (lex.tok() == tok_oppar) { - lhs = createV(lhs->loc, lhs, parse_argument_list(lex)); + auto argument_list = parse_argument_list(lex); + SrcRange range = SrcRange::overlap(lhs->range, argument_list->range); + lhs = createV(range, lhs, argument_list); } else if (lex.tok() == tok_logical_not) { + SrcRange range = SrcRange::overlap(lhs->range, lex.cur_range()); lex.next(); - lhs = createV(lhs->loc, lhs); + lhs = createV(range, lhs); } else if (lex.tok() == tok_double_plus || lex.tok() == tok_double_minus) { - fire_error_no_increment_operator(lex.cur_location(), lex.tok() == tok_double_plus); + fire_no_increment_operator(lex.cur_range(), lex.tok() == tok_double_plus); } else { break; } @@ -908,23 +998,29 @@ static AnyExprV parse_expr90(Lexer& lex) { static AnyExprV parse_expr80(Lexer& lex) { AnyExprV lhs = parse_expr90(lex); while (lex.tok() == tok_dot) { - SrcLocation loc = lex.cur_location(); + SrcRange range(lhs->range); lex.next(); V v_ident = nullptr; V v_instantiationTs = nullptr; if (lex.tok() == tok_identifier) { // obj.field / obj.method - v_ident = createV(lex.cur_location(), lex.cur_str()); - lex.next(); - if (lex.tok() == tok_lt) { + v_ident = parse_identifier(lex, "field name"); + range.end(v_ident->range); + if (lex.tok() == tok_lt) { // obj.method v_instantiationTs = parse_maybe_instantiationTs_after_identifier(lex); + if (v_instantiationTs) { + range.end(v_instantiationTs->range); + } } } else if (lex.tok() == tok_int_const) { // obj.0 (indexed access) - v_ident = createV(lex.cur_location(), lex.cur_str()); + SrcRange idx_range = lex.cur_range(); + std::string_view idx_name = lex.cur_str(); + v_ident = createV(idx_range, idx_name); + range.end(v_ident->range); lex.next(); } else { lex.unexpected("method name"); } - lhs = createV(loc, lhs, v_ident, v_instantiationTs); + lhs = createV(range, lhs, v_ident, v_instantiationTs); if (lex.tok() == tok_oppar || lex.tok() == tok_logical_not || lex.tok() == tok_double_plus || lex.tok() == tok_double_minus) { lhs = parse_fun_call_postfix(lex, lhs); } @@ -936,17 +1032,19 @@ static AnyExprV parse_expr80(Lexer& lex) { static AnyExprV parse_expr75(Lexer& lex) { TokenType t = lex.tok(); if (t == tok_logical_not || t == tok_bitwise_not || t == tok_minus || t == tok_plus) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); + SrcRange operator_range = lex.cur_range(); std::string_view operator_name = lex.cur_str(); lex.next(); AnyExprV rhs = parse_expr75(lex); - return createV(loc, operator_name, t, rhs); + range.end(rhs->range); + return createV(range, operator_range, operator_name, t, rhs); } if (t == tok_double_minus || t == tok_double_plus) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.cur_range(); lex.next(); parse_expr75(lex); - fire_error_no_increment_operator(loc, t == tok_double_plus); + fire_no_increment_operator(range, t == tok_double_plus); } return parse_expr80(lex); } @@ -956,17 +1054,17 @@ static AnyExprV parse_expr40(Lexer& lex) { AnyExprV lhs = parse_expr75(lex); TokenType t = lex.tok(); while (t == tok_as || t == tok_is) { - SrcLocation loc = lex.cur_location(); lex.next(); AnyTypeV rhs_type = parse_type_from_tokens(lex); + SrcRange range = SrcRange::overlap(lhs->range, rhs_type->range); if (t == tok_as) { - lhs = createV(loc, lhs, rhs_type); + lhs = createV(range, lhs, rhs_type); } else { bool is_negated = lhs->kind == ast_not_null_operator; // `a !is ...`, now lhs = `a!` if (is_negated) { lhs = lhs->as()->get_expr(); } - lhs = createV(loc, lhs, rhs_type, is_negated); + lhs = createV(range, lhs, rhs_type, is_negated); } t = lex.tok(); } @@ -978,11 +1076,12 @@ static AnyExprV parse_expr30(Lexer& lex) { AnyExprV lhs = parse_expr40(lex); TokenType t = lex.tok(); while (t == tok_mul || t == tok_div || t == tok_mod || t == tok_divC || t == tok_divR) { - SrcLocation loc = lex.cur_location(); + SrcRange operator_range = lex.cur_range(); std::string_view operator_name = lex.cur_str(); lex.next(); AnyExprV rhs = parse_expr40(lex); - lhs = createV(loc, operator_name, t, lhs, rhs); + SrcRange range = SrcRange::overlap(lhs->range, rhs->range); + lhs = createV(range, operator_range, operator_name, t, lhs, rhs); t = lex.tok(); } return lhs; @@ -993,11 +1092,12 @@ static AnyExprV parse_expr20(Lexer& lex) { AnyExprV lhs = parse_expr30(lex); TokenType t = lex.tok(); while (t == tok_minus || t == tok_plus) { - SrcLocation loc = lex.cur_location(); + SrcRange operator_range = lex.cur_range(); std::string_view operator_name = lex.cur_str(); lex.next(); AnyExprV rhs = parse_expr30(lex); - lhs = createV(loc, operator_name, t, lhs, rhs); + SrcRange range = SrcRange::overlap(lhs->range, rhs->range); + lhs = createV(range, operator_range, operator_name, t, lhs, rhs); t = lex.tok(); } return lhs; @@ -1008,12 +1108,13 @@ static AnyExprV parse_expr17(Lexer& lex) { AnyExprV lhs = parse_expr20(lex); TokenType t = lex.tok(); while (t == tok_lshift || t == tok_rshift || t == tok_rshiftC || t == tok_rshiftR) { - SrcLocation loc = lex.cur_location(); + SrcRange operator_range = lex.cur_range(); std::string_view operator_name = lex.cur_str(); lex.next(); AnyExprV rhs = parse_expr20(lex); - diagnose_addition_in_bitshift(loc, operator_name, rhs); - lhs = createV(loc, operator_name, t, lhs, rhs); + SrcRange range = SrcRange::overlap(lhs->range, rhs->range); + diagnose_addition_in_bitshift(range, operator_name, rhs); + lhs = createV(range, operator_range, operator_name, t, lhs, rhs); t = lex.tok(); } return lhs; @@ -1024,11 +1125,12 @@ static AnyExprV parse_expr15(Lexer& lex) { AnyExprV lhs = parse_expr17(lex); TokenType t = lex.tok(); if (t == tok_eq || t == tok_lt || t == tok_gt || t == tok_leq || t == tok_geq || t == tok_neq || t == tok_spaceship) { - SrcLocation loc = lex.cur_location(); + SrcRange operator_range = lex.cur_range(); std::string_view operator_name = lex.cur_str(); lex.next(); AnyExprV rhs = parse_expr17(lex); - lhs = createV(loc, operator_name, t, lhs, rhs); + SrcRange range = SrcRange::overlap(lhs->range, rhs->range); + lhs = createV(range, operator_range, operator_name, t, lhs, rhs); if (t == tok_eq || t == tok_neq) { lhs = maybe_replace_eq_null_with_isNull_check(lhs->as()); } @@ -1041,13 +1143,14 @@ static AnyExprV parse_expr14(Lexer& lex) { AnyExprV lhs = parse_expr15(lex); TokenType t = lex.tok(); while (t == tok_bitwise_and || t == tok_bitwise_or || t == tok_bitwise_xor) { - SrcLocation loc = lex.cur_location(); + SrcRange operator_range = lex.cur_range(); std::string_view operator_name = lex.cur_str(); lex.next(); AnyExprV rhs = parse_expr15(lex); - diagnose_bitwise_precedence(loc, operator_name, lhs, rhs); - diagnose_and_or_precedence(loc, lhs, t, operator_name); - lhs = createV(loc, operator_name, t, lhs, rhs); + SrcRange range = SrcRange::overlap(lhs->range, rhs->range); + diagnose_bitwise_precedence(range, operator_name, lhs, rhs); + diagnose_and_or_precedence(range, lhs, t, operator_name); + lhs = createV(range, operator_range, operator_name, t, lhs, rhs); t = lex.tok(); } return lhs; @@ -1058,12 +1161,13 @@ static AnyExprV parse_expr13(Lexer& lex) { AnyExprV lhs = parse_expr14(lex); TokenType t = lex.tok(); while (t == tok_logical_and || t == tok_logical_or) { - SrcLocation loc = lex.cur_location(); + SrcRange operator_range = lex.cur_range(); std::string_view operator_name = lex.cur_str(); lex.next(); AnyExprV rhs = parse_expr14(lex); - diagnose_and_or_precedence(loc, lhs, t, operator_name); - lhs = createV(loc, operator_name, t, lhs, rhs); + SrcRange range = SrcRange::overlap(lhs->range, rhs->range); + diagnose_and_or_precedence(range, lhs, t, operator_name); + lhs = createV(range, operator_range, operator_name, t, lhs, rhs); t = lex.tok(); } return lhs; @@ -1074,27 +1178,28 @@ static AnyExprV parse_expr10(Lexer& lex) { AnyExprV lhs = parse_expr13(lex); TokenType t = lex.tok(); if (t == tok_assign) { - SrcLocation loc = lex.cur_location(); lex.next(); AnyExprV rhs = parse_expr10(lex); - return createV(loc, lhs, rhs); + SrcRange range = SrcRange::overlap(lhs->range, rhs->range); + return createV(range, lhs, rhs); } if (t == tok_set_plus || t == tok_set_minus || t == tok_set_mul || t == tok_set_div || t == tok_set_mod || t == tok_set_lshift || t == tok_set_rshift || t == tok_set_bitwise_and || t == tok_set_bitwise_or || t == tok_set_bitwise_xor) { - SrcLocation loc = lex.cur_location(); + SrcRange operator_range = lex.cur_range(); std::string_view operator_name = lex.cur_str().substr(0, lex.cur_str().size() - 1); // "+" for += lex.next(); AnyExprV rhs = parse_expr10(lex); - return createV(loc, operator_name, t, lhs, rhs); + SrcRange range = SrcRange::overlap(lhs->range, rhs->range); + return createV(range, operator_range, operator_name, t, lhs, rhs); } if (t == tok_question) { - SrcLocation loc = lex.cur_location(); lex.next(); AnyExprV when_true = parse_expr10(lex); lex.expect(tok_colon, "`:`"); AnyExprV when_false = parse_expr10(lex); - return createV(loc, lhs, when_true, when_false); + SrcRange range = SrcRange::overlap(lhs->range, when_false->range); + return createV(range, lhs, when_true, when_false); } return lhs; } @@ -1104,7 +1209,7 @@ AnyExprV parse_expr(Lexer& lex) { } static V parse_block_statement(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.expect(tok_opbrace, "`{`"); std::vector items; while (lex.tok() != tok_clbrace) { @@ -1120,22 +1225,29 @@ static V parse_block_statement(Lexer& lex) { lex.expect(tok_semicolon, "`;`"); } } - SrcLocation loc_end = lex.cur_location(); - lex.expect(tok_clbrace, "`}`"); - return createV(loc, loc_end, std::move(items)); + lex.check(tok_clbrace, "`}`"); + range.end(lex.cur_range()); + lex.next(); + return createV(range, std::move(items)); } static AnyV parse_return_statement(Lexer& lex) { - SrcLocation loc = lex.cur_location(); - lex.expect(tok_return, "`return`"); - AnyExprV child = lex.tok() == tok_semicolon || lex.tok() == tok_clbrace - ? createV(lex.cur_location()) - : parse_expr(lex); - return createV(loc, child); + lex.check(tok_return, "`return`"); + SrcRange range = lex.cur_range(); + lex.next(); + + AnyExprV child = nullptr; + if (lex.tok() == tok_semicolon || lex.tok() == tok_clbrace) { + child = createV(SrcRange::empty_at_end(range)); + } else { + child = parse_expr(lex); + range.end(child->range); + } + return createV(range, child); } static AnyV parse_if_statement(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.expect(tok_if, "`if`"); lex.expect(tok_oppar, "`(`"); @@ -1148,70 +1260,73 @@ static AnyV parse_if_statement(Lexer& lex) { lex.next(); if (lex.tok() == tok_if) { AnyV v_inner_if = parse_if_statement(lex); - else_body = createV(v_inner_if->loc, lex.cur_location(), {v_inner_if}); + else_body = createV(v_inner_if->range, {v_inner_if}); } else { else_body = parse_block_statement(lex); } } else { // no 'else', create empty block - else_body = createV(lex.cur_location(), lex.cur_location(), {}); + else_body = createV(SrcRange::empty_at_end(if_body->range), {}); } - return createV(loc, false, cond, if_body, else_body); + range.end(else_body->range); + return createV(range, false, cond, if_body, else_body); } static AnyV parse_repeat_statement(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.expect(tok_repeat, "`repeat`"); lex.expect(tok_oppar, "`(`"); AnyExprV cond = parse_expr(lex); lex.expect(tok_clpar, "`)`"); V body = parse_block_statement(lex); - return createV(loc, cond, body); + range.end(body->range); + return createV(range, cond, body); } static AnyV parse_while_statement(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.expect(tok_while, "`while`"); lex.expect(tok_oppar, "`(`"); AnyExprV cond = parse_expr(lex); lex.expect(tok_clpar, "`)`"); V body = parse_block_statement(lex); - return createV(loc, cond, body); + range.end(body->range); + return createV(range, cond, body); } static AnyV parse_do_while_statement(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.expect(tok_do, "`do`"); V body = parse_block_statement(lex); lex.expect(tok_while, "`while`"); lex.expect(tok_oppar, "`(`"); AnyExprV cond = parse_expr(lex); - lex.expect(tok_clpar, "`)`"); - return createV(loc, body, cond); + + lex.check(tok_clpar, "`)`"); + range.end(lex.cur_range()); + lex.next(); + return createV(range, body, cond); } static AnyExprV parse_catch_variable(Lexer& lex) { - SrcLocation loc = lex.cur_location(); if (lex.tok() == tok_identifier) { - std::string_view var_name = lex.cur_str(); - lex.next(); - auto v_ident = createV(loc, var_name); - return createV(loc, v_ident, nullptr); + auto v_ident = parse_identifier(lex, "catch variable"); + return createV(v_ident->range, v_ident, nullptr); } if (lex.tok() == tok_underscore) { + auto v_ident = createV(lex.cur_range(), ""); lex.next(); - auto v_ident = createV(loc, ""); - return createV(loc, v_ident, nullptr); + return createV(v_ident->range, v_ident, nullptr); } lex.unexpected("identifier"); } static AnyExprV create_catch_underscore_variable(const Lexer& lex) { - auto v_ident = createV(lex.cur_location(), ""); - return createV(lex.cur_location(), v_ident, nullptr); + auto v_ident = createV(SrcRange::empty_at_start(lex.cur_range()), ""); + return createV(v_ident->range, v_ident, nullptr); } static AnyV parse_assert_statement(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.expect(tok_assert, "`assert`"); lex.expect(tok_oppar, "`(`"); @@ -1220,24 +1335,27 @@ static AnyV parse_assert_statement(Lexer& lex) { if (lex.tok() == tok_comma) { // assert(cond, code) lex.next(); thrown_code = parse_expr(lex); - lex.expect(tok_clpar, "`)`"); + lex.check(tok_clpar, "`)`"); + range.end(lex.cur_range()); + lex.next(); } else { // assert(cond) throw code lex.expect(tok_clpar, "`)`"); lex.expect(tok_throw, "`throw excNo` after assert"); thrown_code = parse_expr(lex); + range.end(thrown_code->range); } - return createV(loc, cond, thrown_code); + return createV(range, cond, thrown_code); } static AnyV parse_try_catch_statement(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.expect(tok_try, "`try`"); V try_body = parse_block_statement(lex); std::vector catch_args; lex.expect(tok_catch, "`catch`"); - SrcLocation catch_loc = lex.cur_location(); + SrcRange catch_range = lex.range_start(); if (lex.tok() == tok_oppar) { lex.next(); catch_args.push_back(parse_catch_variable(lex)); @@ -1247,15 +1365,19 @@ static AnyV parse_try_catch_statement(Lexer& lex) { } else { // catch (excNo) -> catch (excNo, _) catch_args.push_back(create_catch_underscore_variable(lex)); } - lex.expect(tok_clpar, "`)`"); + lex.check(tok_clpar, "`)`"); + catch_range.end(lex.cur_range()); + lex.next(); } else { // catch -> catch (_, _) catch_args.push_back(create_catch_underscore_variable(lex)); catch_args.push_back(create_catch_underscore_variable(lex)); + catch_range = SrcRange::empty_at_start(lex.cur_range()); } - V catch_expr = createV(catch_loc, std::move(catch_args)); + V catch_expr = createV(catch_range, std::move(catch_args)); V catch_body = parse_block_statement(lex); - return createV(loc, try_body, catch_expr, catch_body); + range.end(catch_body->range); + return createV(range, try_body, catch_expr, catch_body); } AnyV parse_statement(Lexer& lex) { @@ -1282,7 +1404,7 @@ AnyV parse_statement(Lexer& lex) { case tok_try: return parse_try_catch_statement(lex); case tok_semicolon: - return createV(lex.cur_location()); + return createV(lex.cur_range()); case tok_break: case tok_continue: lex.error("break/continue from loops are not supported yet"); @@ -1301,12 +1423,12 @@ static AnyV parse_func_body(Lexer& lex) { return parse_block_statement(lex); } -static AnyV parse_asm_func_body(Lexer& lex, V param_list) { - SrcLocation loc = lex.cur_location(); +static AnyV parse_asm_func_body(Lexer& lex, V name_ident, V param_list) { + SrcRange range = lex.range_start(); lex.expect(tok_asm, "`asm`"); size_t n_params = param_list->size(); if (n_params > 16) { - throw ParseError(loc, "assembler built-in function can have at most 16 arguments"); + fire(name_ident, "assembler built-in function can have at most 16 arguments"); } std::vector arg_order, ret_order; if (lex.tok() == tok_oppar) { @@ -1332,15 +1454,14 @@ static AnyV parse_asm_func_body(Lexer& lex, V param_list) { std::vector asm_commands; lex.check(tok_string_const, "\"ASM COMMAND\""); while (lex.tok() == tok_string_const) { - std::string_view asm_command = lex.cur_str(); - asm_commands.push_back(createV(lex.cur_location(), asm_command)); - lex.next(); + asm_commands.push_back(parse_expr(lex)); } - return createV(loc, std::move(arg_order), std::move(ret_order), std::move(asm_commands)); + range.end(asm_commands.back()->range); + return createV(range, std::move(arg_order), std::move(ret_order), std::move(asm_commands)); } static V parse_annotation(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.cur_range(); lex.check(tok_annotation_at, "`@`"); std::string_view name = lex.cur_str(); AnnotationKind kind = Vertex::parse_kind(name); @@ -1348,7 +1469,7 @@ static V parse_annotation(Lexer& lex) { V v_arg = nullptr; if (lex.tok() == tok_oppar) { - SrcLocation loc_args = lex.cur_location(); + SrcRange range_args = lex.range_start(); lex.next(); std::vector args; args.push_back(parse_expr(lex)); @@ -1359,21 +1480,23 @@ static V parse_annotation(Lexer& lex) { } args.push_back(parse_expr(lex)); } - lex.expect(tok_clpar, "`)`"); - v_arg = createV(loc_args, std::move(args)); + lex.check(tok_clpar, "`)`"); + range_args.end(lex.cur_range()); + v_arg = createV(range_args, std::move(args)); + lex.next(); } switch (kind) { case AnnotationKind::unknown: - throw ParseError(loc, "unknown annotation " + static_cast(name)); + fire(range, "unknown annotation " + static_cast(name)); case AnnotationKind::inline_simple: case AnnotationKind::inline_ref: case AnnotationKind::noinline: case AnnotationKind::pure: if (v_arg) { - throw ParseError(v_arg->loc, "arguments aren't allowed for " + static_cast(name)); + fire(range, "arguments aren't allowed for " + static_cast(name)); } - v_arg = createV(loc, {}); + v_arg = createV(range, {}); break; case AnnotationKind::deprecated: case AnnotationKind::custom: @@ -1381,23 +1504,27 @@ static V parse_annotation(Lexer& lex) { break; case AnnotationKind::method_id: if (!v_arg || v_arg->size() != 1 || v_arg->get_item(0)->kind != ast_int_const) { - throw ParseError(loc, "expecting `(number)` after " + static_cast(name)); + fire(range, "expecting `(number)` after " + static_cast(name)); } break; case AnnotationKind::overflow1023_policy: case AnnotationKind::on_bounced_policy: { if (!v_arg || v_arg->size() != 1 || v_arg->get_item(0)->kind != ast_string_const) { - throw ParseError(loc, "expecting `(\"policy_name\")` after " + static_cast(name)); + fire(range, "expecting `(\"policy_name\")` after " + static_cast(name)); } break; } } - return createV(loc, name, kind, v_arg); + if (v_arg == nullptr) { + v_arg = createV(SrcRange::empty_at_end(range), {}); + } + range.end(v_arg->range); + return createV(range, name, kind, v_arg); } static AnyV parse_function_declaration(Lexer& lex, const std::vector>& annotations, bool is_contract_getter) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.expect(tok_fun, "`fun`"); AnyTypeV receiver_type = nullptr; @@ -1424,7 +1551,7 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(lex.cur_location(), f_name); + auto v_ident = createV(lex.cur_range(), f_name); lex.next(); V genericsT_list = nullptr; @@ -1433,7 +1560,7 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector v_param_list = parse_parameter_list(lex, receiver_type)->as(); - bool accepts_self = !v_param_list->empty() && v_param_list->get_param(0)->param_name == "self"; + bool accepts_self = !v_param_list->empty() && v_param_list->get_param(0)->get_identifier()->name == "self"; int n_mutate_params = v_param_list->get_mutate_params_count(); AnyTypeV ret_type = nullptr; @@ -1445,7 +1572,7 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(lex.cur_location(), "void"); + ret_type = createV(lex.cur_range(), "void"); lex.next(); } else { ret_type = parse_type_from_tokens(lex); @@ -1453,16 +1580,16 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(lex.cur_location()); + v_body = createV(lex.cur_range()); lex.next(); } else if (lex.tok() == tok_opbrace) { v_body = parse_func_body(lex); @@ -1470,7 +1597,7 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vectorerror("@method_id can be specified only for regular functions"); + fire(v_annotation, "@method_id can be specified only for regular functions"); } auto v_int = v_annotation->get_arg()->get_item(0)->as(); if (v_int->intval.is_null() || !v_int->intval->signed_fits_bits(32)) { - v_int->error("invalid integer constant"); + fire(v_int, "invalid integer constant"); } tvm_method_id = v_int->intval; break; @@ -1521,10 +1648,10 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vectorerror("incorrect value for " + static_cast(v_annotation->name)); + fire(v_annotation, "incorrect value for " + static_cast(v_annotation->name)); } if (f_name != "onInternalMessage") { - v_annotation->error("this annotation is applicable only to onInternalMessage()"); + fire(v_annotation, "this annotation is applicable only to onInternalMessage()"); } break; } @@ -1533,15 +1660,16 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vectorerror("this annotation is not applicable to functions"); + fire(v_annotation, "this annotation is not applicable to functions"); } } - return createV(loc, v_ident, v_param_list, v_body, receiver_type, ret_type, genericsT_list, std::move(tvm_method_id), flags, inline_mode); + range.end(v_body->range); + return createV(range, v_ident, v_param_list, v_body, receiver_type, ret_type, genericsT_list, std::move(tvm_method_id), flags, inline_mode); } static AnyV parse_struct_field(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); bool is_private = false; if (lex.tok() == tok_private) { @@ -1555,23 +1683,23 @@ static AnyV parse_struct_field(Lexer& lex) { is_readonly = true; } - lex.check(tok_identifier, "field name"); - auto v_ident = createV(lex.cur_location(), lex.cur_str()); - lex.next(); + auto v_ident = parse_identifier(lex, "field name"); lex.expect(tok_colon, "`: `"); AnyTypeV declared_type = parse_type_from_tokens(lex); + range.end(declared_type->range); AnyExprV default_value = nullptr; if (lex.tok() == tok_assign) { // `id: int = 3` lex.next(); default_value = parse_expr(lex); + range.end(default_value->range); } - return createV(loc, v_ident, is_private, is_readonly, default_value, declared_type); + return createV(range, v_ident, is_private, is_readonly, default_value, declared_type); } -static V parse_struct_body(Lexer& lex) { - SrcLocation loc = lex.cur_location(); +static V parse_struct_body(Lexer& lex, V name_ident) { + SrcRange range = lex.range_start(); std::vector fields; if (lex.tok() == tok_opbrace) { // `struct A` equal to `struct A {}` @@ -1582,14 +1710,18 @@ static V parse_struct_body(Lexer& lex) { lex.next(); } } - lex.expect(tok_clbrace, "`}`"); + lex.check(tok_clbrace, "`}`"); + range.end(lex.cur_range()); + lex.next(); + } else { + range = name_ident->range; } - return createV(loc, std::move(fields)); + return createV(range, std::move(fields)); } static AnyV parse_struct_declaration(Lexer& lex, const std::vector>& annotations) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.expect(tok_struct, "`struct`"); AnyExprV opcode = nullptr; @@ -1600,16 +1732,15 @@ static AnyV parse_struct_declaration(Lexer& lex, const std::vector(lex.cur_location(), parse_tok_int_const(opcode_str), opcode_str); + SrcRange opcode_range = lex.cur_range(); + opcode = createV(opcode_range, parse_tok_int_const(opcode_str), opcode_str); lex.next(); lex.expect(tok_clpar, "`)`"); } else { - opcode = createV(lex.cur_location()); + opcode = createV(SrcRange::empty_at_start(range)); } - lex.check(tok_identifier, "identifier"); - auto v_ident = createV(lex.cur_location(), lex.cur_str()); - lex.next(); + auto v_ident = parse_identifier(lex, "identifier"); V genericsT_list = nullptr; if (lex.tok() == tok_lt) { // 'struct Wrapper' @@ -1627,35 +1758,37 @@ static AnyV parse_struct_declaration(Lexer& lex, const std::vectorerror("incorrect value for " + static_cast(v_annotation->name)); + fire(v_annotation, "incorrect value for " + static_cast(v_annotation->name)); } break; } default: - v_annotation->error("this annotation is not applicable to struct"); + fire(v_annotation, "this annotation is not applicable to struct"); } } - return createV(loc, v_ident, genericsT_list, overflow1023_policy, opcode, parse_struct_body(lex)); + auto body = parse_struct_body(lex, v_ident); + range.end(body->range); + return createV(range, v_ident, genericsT_list, overflow1023_policy, opcode, body); } static AnyV parse_enum_member(Lexer& lex) { - SrcLocation loc = lex.cur_location(); - lex.check(tok_identifier, "member name"); - auto v_ident = createV(lex.cur_location(), lex.cur_str()); - lex.next(); + SrcRange range = lex.range_start(); + auto v_ident = parse_identifier(lex, "member name"); + range.end(v_ident->range); AnyExprV init_value = nullptr; if (lex.tok() == tok_assign) { // `Red = 1` lex.next(); init_value = parse_expr(lex); + range.end(init_value->range); } - return createV(loc, v_ident, init_value); + return createV(range, v_ident, init_value); } static V parse_enum_body(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.expect(tok_opbrace, "`{`"); std::vector members; @@ -1665,18 +1798,18 @@ static V parse_enum_body(Lexer& lex) { lex.next(); } } - lex.expect(tok_clbrace, "`}`"); - return createV(loc, std::move(members)); + lex.check(tok_clbrace, "`}`"); + range.end(lex.cur_range()); + lex.next(); + return createV(range, std::move(members)); } static AnyV parse_enum_declaration(Lexer& lex, const std::vector>& annotations) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.expect(tok_enum, "`enum`"); - lex.check(tok_identifier, "identifier"); - auto v_ident = createV(lex.cur_location(), lex.cur_str()); - lex.next(); + auto v_ident = parse_identifier(lex, "identifier"); AnyTypeV colon_type = nullptr; if (lex.tok() == tok_colon) { // enum Role: int8 @@ -1690,38 +1823,41 @@ static AnyV parse_enum_declaration(Lexer& lex, const std::vectorerror("this annotation is not applicable to enum"); + fire(v_annotation, "this annotation is not applicable to enum"); } } - return createV(loc, v_ident, colon_type, parse_enum_body(lex)); + auto body = parse_enum_body(lex); + range.end(body->range); + return createV(range, v_ident, colon_type, body); } static AnyV parse_tolk_required_version(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.next_special(tok_semver, "semver"); // syntax: "tolk 0.6" std::string semver = static_cast(lex.cur_str()); + range.end(lex.cur_range()); lex.next(); // for simplicity, there is no syntax ">= version" and so on, just strict compare if (TOLK_VERSION != semver && TOLK_VERSION != semver + ".0") { // 0.6 = 0.6.0 - loc.show_warning("the contract is written in Tolk v" + semver + ", but you use Tolk compiler v" + TOLK_VERSION + "; probably, it will lead to compilation errors or hash changes"); + compilation_warning(range, "the contract is written in Tolk v" + semver + ", but you use Tolk compiler v" + TOLK_VERSION + "; probably, it will lead to compilation errors or hash changes"); } - return createV(loc, semver); // semicolon is not necessary + return createV(range, semver); // semicolon is not necessary } static AnyV parse_import_directive(Lexer& lex) { - SrcLocation loc = lex.cur_location(); + SrcRange range = lex.range_start(); lex.expect(tok_import, "`import`"); lex.check(tok_string_const, "source file name"); - std::string_view rel_filename = lex.cur_str(); + auto v_str = parse_expr(lex)->as(); + std::string_view rel_filename = v_str->str_val; if (rel_filename.empty()) { lex.error("imported file name is an empty string"); } - auto v_str = createV(lex.cur_location(), rel_filename); - lex.next(); - return createV(loc, v_str); // semicolon is not necessary + range.end(v_str->range); + return createV(range, v_str); } @@ -1734,6 +1870,7 @@ AnyV parse_src_file_to_ast(const SrcFile* file) { std::vector toplevel_declarations; std::vector> annotations; Lexer lex(file); + SrcRange range = lex.range_start(); while (!lex.is_eof()) { switch (lex.tok()) { @@ -1802,7 +1939,8 @@ AnyV parse_src_file_to_ast(const SrcFile* file) { } } - return createV(file, std::move(toplevel_declarations)); + range.end(toplevel_declarations.empty() ? lex.cur_range() : toplevel_declarations.back()->range); + return createV(file, range, std::move(toplevel_declarations)); } } // namespace tolk diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 1f41dc3d14..2b03e4c939 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -52,193 +52,193 @@ class ASTReplicator final { // types static V clone(V v) { - return createV(v->loc, v->text); + return createV(v->range, v->text); } static V clone(V v) { - return createV(v->loc, clone(v->get_inner())); + return createV(v->range, clone(v->get_inner())); } static V clone(V v) { - return createV(v->loc, clone(v->get_items())); + return createV(v->range, clone(v->get_items())); } static V clone(V v) { - return createV(v->loc, clone(v->get_items())); + return createV(v->range, clone(v->get_items())); } static V clone(V v) { - return createV(v->loc, clone(v->get_params_and_return())); + return createV(v->range, clone(v->get_params_and_return())); } static V clone(V v) { - return createV(v->loc, clone(v->get_variants())); + return createV(v->range, clone(v->get_variants())); } static V clone(V v) { - return createV(v->loc, clone(v->get_inner_and_args())); + return createV(v->range, clone(v->get_inner_and_args())); } // expressions static V clone(V v) { - return createV(v->loc); + return createV(v->range); } static V clone(V v) { - return createV(v->loc, clone(v->get_expr())); + return createV(v->range, clone(v->get_expr())); } static V clone(V v) { - return createV(v->loc, clone(v->get_block_statement())); + return createV(v->range, clone(v->get_block_statement())); } static V clone(V v) { - return createV(v->loc, clone(v->get_expr())); + return createV(v->range, clone(v->get_expr())); } static V clone(V v) { - return createV(v->loc, clone(v->get_wrapped_expr()), v->aux_data, v->inferred_type); + return createV(clone(v->get_wrapped_expr()), v->aux_data, v->inferred_type); } static V clone(V v) { - return createV(v->loc, clone(v->get_items())); + return createV(v->range, clone(v->get_items())); } static V clone(V v) { - return createV(v->loc, clone(v->get_items())); + return createV(v->range, clone(v->get_items())); } static V clone(V v) { - return createV(v->loc, clone(v->get_identifier()), v->has_instantiationTs() ? clone(v->get_instantiationTs()) : nullptr); + return createV(v->range, clone(v->get_identifier()), v->has_instantiationTs() ? clone(v->get_instantiationTs()) : nullptr); } static V clone(V v) { - return createV(v->loc, clone(v->get_identifier()), clone(v->type_node), v->is_immutable, v->is_lateinit, v->marked_as_redef); + return createV(v->range, clone(v->get_identifier()), clone(v->type_node), v->is_immutable, v->is_lateinit, v->marked_as_redef); } static V clone(V v) { - return createV(v->loc, clone(v->get_expr())); + return createV(v->range, clone(v->get_expr())); } static V clone(V v) { - return createV(v->loc, v->intval, v->orig_str); + return createV(v->range, v->intval, v->orig_str); } static V clone(V v) { - return createV(v->loc, v->str_val); + return createV(v->range, v->str_val); } static V clone(V v) { - return createV(v->loc, v->bool_val); + return createV(v->range, v->bool_val); } static V clone(V v) { - return createV(v->loc); + return createV(v->range); } static V clone(V v) { - return createV(v->loc, clone(v->get_expr()), v->passed_as_mutate); + return createV(v->range, clone(v->get_expr()), v->passed_as_mutate); } static V clone(V v) { - return createV(v->loc, clone(v->get_arguments())); + return createV(v->range, clone(v->get_arguments())); } static V clone(V v) { - return createV(v->loc, clone(v->get_obj()), clone(v->get_identifier()), v->has_instantiationTs() ? clone(v->get_instantiationTs()) : nullptr); + return createV(v->range, clone(v->get_obj()), clone(v->get_identifier()), v->has_instantiationTs() ? clone(v->get_instantiationTs()) : nullptr); } static V clone(V v) { - return createV(v->loc, clone(v->get_callee()), clone(v->get_arg_list())); + return createV(v->range, clone(v->get_callee()), clone(v->get_arg_list())); } static V clone(V v) { - return createV(v->loc); + return createV(v->range); } static V clone(V v) { - return createV(v->loc, clone(v->get_lhs()), clone(v->get_rhs())); + return createV(v->range, clone(v->get_lhs()), clone(v->get_rhs())); } static V clone(V v) { - return createV(v->loc, v->operator_name, v->tok, clone(v->get_lhs()), clone(v->get_rhs())); + return createV(v->range, v->operator_range, v->operator_name, v->tok, clone(v->get_lhs()), clone(v->get_rhs())); } static V clone(V v) { - return createV(v->loc, v->operator_name, v->tok, clone(v->get_rhs())); + return createV(v->range, v->operator_range, v->operator_name, v->tok, clone(v->get_rhs())); } static V clone(V v) { - return createV(v->loc, v->operator_name, v->tok, clone(v->get_lhs()), clone(v->get_rhs())); + return createV(v->range, v->operator_range, v->operator_name, v->tok, clone(v->get_lhs()), clone(v->get_rhs())); } static V clone(V v) { - return createV(v->loc, clone(v->get_cond()), clone(v->get_when_true()), clone(v->get_when_false())); + return createV(v->range, clone(v->get_cond()), clone(v->get_when_true()), clone(v->get_when_false())); } static V clone(V v) { - return createV(v->loc, clone(v->get_expr()), clone(v->type_node)); + return createV(v->range, clone(v->get_expr()), clone(v->type_node)); } static V clone(V v) { - return createV(v->loc, clone(v->get_expr()), clone(v->type_node), v->is_negated); + return createV(v->range, clone(v->get_expr()), clone(v->type_node), v->is_negated); } static V clone(V v) { - return createV(v->loc, clone(v->get_expr())); + return createV(v->range, clone(v->get_expr())); } static V clone(V v) { - return createV(v->loc, clone(v->get_expr())); + return createV(v->range, clone(v->get_expr())); } static V clone(V v) { - return createV(v->loc, clone(v->get_all_children())); + return createV(v->range, clone(v->get_all_children())); } static V clone(V v) { - return createV(v->loc, v->pattern_kind, clone(v->pattern_type_node), clone(v->get_pattern_expr()), clone(v->get_body())); + return createV(v->range, v->pattern_kind, clone(v->pattern_type_node), clone(v->get_pattern_expr()), clone(v->get_body())); } static V clone(V v) { - return createV(v->loc, clone(v->get_field_identifier()), clone(v->get_init_val())); + return createV(v->range, clone(v->get_field_identifier()), clone(v->get_init_val())); } static V clone(V v) { - return createV(v->loc, clone(v->get_all_fields())); + return createV(v->range, clone(v->get_all_fields())); } static V clone(V v) { - return createV(v->loc, clone(v->type_node), clone(v->get_body())); + return createV(v->range, clone(v->type_node), clone(v->get_body())); } // statements static V clone(V v) { - return createV(v->loc); + return createV(v->range); } static V clone(V v) { - return createV(v->loc, v->loc_end, clone(v->get_items())); + return createV(v->range, clone(v->get_items())); } static V clone(V v) { - return createV(v->loc, clone(v->get_return_value())); + return createV(v->range, clone(v->get_return_value())); } static V clone(V v) { - return createV(v->loc, v->is_ifnot, clone(v->get_cond()), clone(v->get_if_body()), clone(v->get_else_body())); + return createV(v->range, v->is_ifnot, clone(v->get_cond()), clone(v->get_if_body()), clone(v->get_else_body())); } static V clone(V v) { - return createV(v->loc, clone(v->get_cond()), clone(v->get_body())); + return createV(v->range, clone(v->get_cond()), clone(v->get_body())); } static V clone(V v) { - return createV(v->loc, clone(v->get_cond()), clone(v->get_body())); + return createV(v->range, clone(v->get_cond()), clone(v->get_body())); } static V clone(V v) { - return createV(v->loc, clone(v->get_body()), clone(v->get_cond())); + return createV(v->range, clone(v->get_body()), clone(v->get_cond())); } static V clone(V v) { - return createV(v->loc, clone(v->get_thrown_code()), clone(v->get_thrown_arg())); + return createV(v->range, clone(v->get_thrown_code()), clone(v->get_thrown_arg())); } static V clone(V v) { - return createV(v->loc, clone(v->get_cond()), clone(v->get_thrown_code())); + return createV(v->range, clone(v->get_cond()), clone(v->get_thrown_code())); } static V clone(V v) { - return createV(v->loc, clone(v->get_try_body()), clone(v->get_catch_expr()), clone(v->get_catch_body())); + return createV(v->range, clone(v->get_try_body()), clone(v->get_catch_expr()), clone(v->get_catch_body())); } static V clone(V v) { - return createV(v->loc, v->arg_order, v->ret_order, clone(v->get_asm_commands())); + return createV(v->range, v->arg_order, v->ret_order, clone(v->get_asm_commands())); } // other (common) static V clone(V v) { - return createV(v->loc, v->name); + return createV(v->range, v->name); } static V clone(V v) { - return createV(v->loc, v->nameT, clone(v->default_type_node)); + return createV(v->range, v->nameT, clone(v->default_type_node)); } static V clone(V v) { - return createV(v->loc, clone(v->get_items())); + return createV(v->range, clone(v->get_items())); } static V clone(V v) { - return createV(v->loc, clone(v->type_node)); + return createV(v->range, clone(v->type_node)); } static V clone(V v) { - return createV(v->loc, clone(v->get_items())); + return createV(v->range, clone(v->get_items())); } static V clone(V v) { - return createV(v->loc, v->param_name, clone(v->type_node), v->default_value ? clone(v->default_value) : nullptr, v->declared_as_mutate); + return createV(v->range, clone(v->get_identifier()), clone(v->type_node), v->default_value ? clone(v->default_value) : nullptr, v->declared_as_mutate); } static V clone(V v) { - return createV(v->loc, clone(v->get_params())); + return createV(v->range, clone(v->get_params())); } static V clone(V v) { - return createV(v->loc, clone(v->get_identifier()), v->is_private, v->is_readonly, v->default_value ? clone(v->default_value) : nullptr, clone(v->type_node)); + return createV(v->range, clone(v->get_identifier()), v->is_private, v->is_readonly, v->default_value ? clone(v->default_value) : nullptr, clone(v->type_node)); } static V clone(V v) { - return createV(v->loc, clone(v->get_all_fields())); + return createV(v->range, clone(v->get_all_fields())); } @@ -335,7 +335,7 @@ class ASTReplicator final { // the cloned function becomes a deep copy, all AST nodes are copied, no previous pointers left static V clone_function_ast(V v_orig) { return createV( - v_orig->loc, + v_orig->range, clone(v_orig->get_identifier()), clone(v_orig->get_param_list()), clone(v_orig->get_body()), @@ -351,11 +351,11 @@ class ASTReplicator final { // the cloned struct becomes a deep copy, all AST nodes are copied, no previous pointers left static V clone_struct_ast(V v_orig, V new_name_ident) { return createV( - v_orig->loc, + v_orig->range, new_name_ident, clone(v_orig->genericsT_list), v_orig->overflow1023_policy, - v_orig->has_opcode() ? static_cast(clone(v_orig->get_opcode())) : createV(v_orig->loc), + v_orig->has_opcode() ? static_cast(clone(v_orig->get_opcode())) : createV(v_orig->range), clone(v_orig->get_struct_body()) ); } @@ -363,7 +363,7 @@ class ASTReplicator final { // the cloned type alias becomes a deep copy, all AST nodes are copied, no previous pointers left static V clone_type_alias_ast(V v_orig, V new_name_ident) { return createV( - v_orig->loc, + v_orig->range, new_name_ident, clone(v_orig->genericsT_list), clone(v_orig->underlying_type_node) diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index 570ab048c7..693a24e82d 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -204,13 +204,13 @@ class ASTStringifier final : public ASTVisitor { case ast_annotation: return static_cast(v->as()->name); case ast_parameter: - return static_cast(v->as()->param_name) + ": " + ast_type_node_to_string(v->as()->type_node); + return static_cast(v->as()->get_identifier()->name) + ": " + ast_type_node_to_string(v->as()->type_node); case ast_function_declaration: { std::string param_names; for (int i = 0; i < v->as()->get_num_params(); i++) { if (!param_names.empty()) param_names += ","; - param_names += v->as()->get_param(i)->param_name; + param_names += v->as()->get_param(i)->get_identifier()->name; } std::string decl = "fun "; if (auto receiver_node = v->as()->receiver_type_node) { diff --git a/tolk/ast.cpp b/tolk/ast.cpp index b13d90beb1..0dc8d14533 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -21,7 +21,7 @@ namespace tolk { -static_assert(sizeof(ASTNodeBase) == 12); +static_assert(sizeof(ASTNodeBase) == 16); #ifdef TOLK_DEBUG @@ -46,10 +46,6 @@ UnexpectedASTNodeKind::UnexpectedASTNodeKind(AnyV v_unexpected, const char* plac message += place_where; } -void ASTNodeBase::error(const std::string& err_msg) const { - throw ParseError(loc, err_msg); -} - AnnotationKind Vertex::parse_kind(std::string_view name) { if (name == "@pure") { return AnnotationKind::pure; @@ -92,7 +88,7 @@ int Vertex::lookup_idx(std::string_view nameT) const { int Vertex::lookup_idx(std::string_view param_name) const { for (size_t idx = 0; idx < children.size(); ++idx) { - if (children[idx] && children[idx]->as()->param_name == param_name) { + if (children[idx] && children[idx]->as()->get_identifier()->name == param_name) { return static_cast(idx); } } diff --git a/tolk/ast.h b/tolk/ast.h index 1b748a58bd..457115496d 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -18,7 +18,7 @@ #include #include "fwd-declarations.h" -#include "platform-utils.h" +#include "compilation-errors.h" #include "src-file.h" #include "lexer.h" #include "symtable.h" @@ -171,24 +171,19 @@ using V = const Vertex*; #define createV new Vertex -struct UnexpectedASTNodeKind final : std::exception { - AnyV v_unexpected; - std::string message; - - explicit UnexpectedASTNodeKind(AnyV v_unexpected, const char* place_where); - - const char* what() const noexcept override { - return message.c_str(); - } -}; - // --------------------------------------------------------- struct ASTNodeBase { const ASTNodeKind kind; - const SrcLocation loc; + const SrcRange range; - ASTNodeBase(ASTNodeKind kind, SrcLocation loc) : kind(kind), loc(loc) {} + ASTNodeBase(ASTNodeKind kind, SrcRange range) : kind(kind), range(range) { +#ifdef TOLK_DEBUG + if (range.is_defined() && !range.is_valid()) { + throw Fatal("forgot to end() range in ast"); + } +#endif + } ASTNodeBase(const ASTNodeBase&) = delete; template @@ -211,9 +206,6 @@ struct ASTNodeBase { std::string to_debug_string(bool colored) const; void debug_print() const; #endif - - GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD - void error(const std::string& err_msg) const; }; struct ASTNodeDeclaredTypeBase : ASTNodeBase { @@ -222,7 +214,7 @@ struct ASTNodeDeclaredTypeBase : ASTNodeBase { ASTNodeDeclaredTypeBase* mutate() const { return const_cast(this); } void assign_resolved_type(TypePtr resolved_type); - ASTNodeDeclaredTypeBase(ASTNodeKind kind, SrcLocation loc) : ASTNodeBase(kind, loc) {} + ASTNodeDeclaredTypeBase(ASTNodeKind kind, SrcRange range) : ASTNodeBase(kind, range) {} }; struct ASTNodeExpressionBase : ASTNodeBase { @@ -238,17 +230,17 @@ struct ASTNodeExpressionBase : ASTNodeBase { void assign_lvalue_true(); void assign_always_true_or_false(int flow_true_false_state); - ASTNodeExpressionBase(ASTNodeKind kind, SrcLocation loc) : ASTNodeBase(kind, loc) {} + ASTNodeExpressionBase(ASTNodeKind kind, SrcRange range) : ASTNodeBase(kind, range) {} }; struct ASTNodeStatementBase : ASTNodeBase { - ASTNodeStatementBase(ASTNodeKind kind, SrcLocation loc) : ASTNodeBase(kind, loc) {} + ASTNodeStatementBase(ASTNodeKind kind, SrcRange range) : ASTNodeBase(kind, range) {} }; struct ASTTypeLeaf : ASTNodeDeclaredTypeBase { protected: - ASTTypeLeaf(ASTNodeKind kind, SrcLocation loc) - : ASTNodeDeclaredTypeBase(kind, loc) {} + ASTTypeLeaf(ASTNodeKind kind, SrcRange range) + : ASTNodeDeclaredTypeBase(kind, range) {} }; struct ASTTypeVararg : ASTNodeDeclaredTypeBase { @@ -257,8 +249,8 @@ struct ASTTypeVararg : ASTNodeDeclaredTypeBase { protected: std::vector children; - ASTTypeVararg(ASTNodeKind kind, SrcLocation loc, std::vector&& children) - : ASTNodeDeclaredTypeBase(kind, loc), children(std::move(children)) {} + ASTTypeVararg(ASTNodeKind kind, SrcRange range, std::vector&& children) + : ASTNodeDeclaredTypeBase(kind, range), children(std::move(children)) {} }; struct ASTExprLeaf : ASTNodeExpressionBase { @@ -266,8 +258,8 @@ struct ASTExprLeaf : ASTNodeExpressionBase { friend class ASTReplacer; protected: - ASTExprLeaf(ASTNodeKind kind, SrcLocation loc) - : ASTNodeExpressionBase(kind, loc) {} + ASTExprLeaf(ASTNodeKind kind, SrcRange range) + : ASTNodeExpressionBase(kind, range) {} }; struct ASTExprUnary : ASTNodeExpressionBase { @@ -277,8 +269,8 @@ struct ASTExprUnary : ASTNodeExpressionBase { protected: AnyExprV child; - ASTExprUnary(ASTNodeKind kind, SrcLocation loc, AnyExprV child) - : ASTNodeExpressionBase(kind, loc), child(child) {} + ASTExprUnary(ASTNodeKind kind, SrcRange range, AnyExprV child) + : ASTNodeExpressionBase(kind, range), child(child) {} }; struct ASTExprBinary : ASTNodeExpressionBase { @@ -289,8 +281,8 @@ struct ASTExprBinary : ASTNodeExpressionBase { AnyExprV lhs; AnyExprV rhs; - ASTExprBinary(ASTNodeKind kind, SrcLocation loc, AnyExprV lhs, AnyExprV rhs) - : ASTNodeExpressionBase(kind, loc), lhs(lhs), rhs(rhs) {} + ASTExprBinary(ASTNodeKind kind, SrcRange range, AnyExprV lhs, AnyExprV rhs) + : ASTNodeExpressionBase(kind, range), lhs(lhs), rhs(rhs) {} }; struct ASTExprVararg : ASTNodeExpressionBase { @@ -302,8 +294,8 @@ struct ASTExprVararg : ASTNodeExpressionBase { AnyExprV child(int i) const { return children.at(i); } - ASTExprVararg(ASTNodeKind kind, SrcLocation loc, std::vector&& children) - : ASTNodeExpressionBase(kind, loc), children(std::move(children)) {} + ASTExprVararg(ASTNodeKind kind, SrcRange range, std::vector&& children) + : ASTNodeExpressionBase(kind, range), children(std::move(children)) {} public: int size() const { return static_cast(children.size()); } @@ -317,8 +309,8 @@ struct ASTExprBlockOfStatements : ASTNodeExpressionBase { protected: AnyV child_block_statement; - ASTExprBlockOfStatements(ASTNodeKind kind, SrcLocation loc, AnyV child_block_statement) - : ASTNodeExpressionBase(kind, loc), child_block_statement(child_block_statement) {} + ASTExprBlockOfStatements(ASTNodeKind kind, SrcRange range, AnyV child_block_statement) + : ASTNodeExpressionBase(kind, range), child_block_statement(child_block_statement) {} }; struct ASTStatementUnary : ASTNodeStatementBase { @@ -330,8 +322,8 @@ struct ASTStatementUnary : ASTNodeStatementBase { AnyExprV child_as_expr() const { return reinterpret_cast(child); } - ASTStatementUnary(ASTNodeKind kind, SrcLocation loc, AnyV child) - : ASTNodeStatementBase(kind, loc), child(child) {} + ASTStatementUnary(ASTNodeKind kind, SrcRange range, AnyV child) + : ASTNodeStatementBase(kind, range), child(child) {} }; struct ASTStatementVararg : ASTNodeStatementBase { @@ -343,8 +335,8 @@ struct ASTStatementVararg : ASTNodeStatementBase { AnyExprV child_as_expr(int i) const { return reinterpret_cast(children.at(i)); } - ASTStatementVararg(ASTNodeKind kind, SrcLocation loc, std::vector&& children) - : ASTNodeStatementBase(kind, loc), children(std::move(children)) {} + ASTStatementVararg(ASTNodeKind kind, SrcRange range, std::vector&& children) + : ASTNodeStatementBase(kind, range), children(std::move(children)) {} public: int size() const { return static_cast(children.size()); } @@ -356,8 +348,8 @@ struct ASTOtherLeaf : ASTNodeBase { friend class ASTReplacer; protected: - ASTOtherLeaf(ASTNodeKind kind, SrcLocation loc) - : ASTNodeBase(kind, loc) {} + ASTOtherLeaf(ASTNodeKind kind, SrcRange range) + : ASTNodeBase(kind, range) {} }; struct ASTOtherVararg : ASTNodeBase { @@ -369,8 +361,8 @@ struct ASTOtherVararg : ASTNodeBase { AnyExprV child_as_expr(int i) const { return reinterpret_cast(children.at(i)); } - ASTOtherVararg(ASTNodeKind kind, SrcLocation loc, std::vector&& children) - : ASTNodeBase(kind, loc), children(std::move(children)) {} + ASTOtherVararg(ASTNodeKind kind, SrcRange range, std::vector&& children) + : ASTNodeBase(kind, range), children(std::move(children)) {} public: int size() const { return static_cast(children.size()); } @@ -389,8 +381,8 @@ template<> struct Vertex final : ASTOtherLeaf { std::string_view name; // empty for underscore - Vertex(SrcLocation loc, std::string_view name) - : ASTOtherLeaf(ast_identifier, loc) + Vertex(SrcRange range, std::string_view name) + : ASTOtherLeaf(ast_identifier, range) , name(name) {} }; @@ -407,8 +399,8 @@ template<> struct Vertex final : ASTTypeLeaf { std::string_view text; - Vertex(SrcLocation loc, std::string_view text) - : ASTTypeLeaf(ast_type_leaf_text, loc) + Vertex(SrcRange range, std::string_view text) + : ASTTypeLeaf(ast_type_leaf_text, range) , text(text) {} }; @@ -418,8 +410,8 @@ template<> struct Vertex final : ASTTypeVararg { AnyTypeV get_inner() const { return children.at(0); } - Vertex(SrcLocation loc, AnyTypeV inner) - : ASTTypeVararg(ast_type_question_nullable, loc, {inner}) {} + Vertex(SrcRange range, AnyTypeV inner) + : ASTTypeVararg(ast_type_question_nullable, range, {inner}) {} }; template<> @@ -428,8 +420,8 @@ template<> struct Vertex final : ASTTypeVararg { const std::vector& get_items() const { return children; } - Vertex(SrcLocation loc, std::vector&& items) - : ASTTypeVararg(ast_type_parenthesis_tensor, loc, std::move(items)) {} + Vertex(SrcRange range, std::vector&& items) + : ASTTypeVararg(ast_type_parenthesis_tensor, range, std::move(items)) {} }; template<> @@ -438,8 +430,8 @@ template<> struct Vertex final : ASTTypeVararg { const std::vector& get_items() const { return children; } - Vertex(SrcLocation loc, std::vector&& items) - : ASTTypeVararg(ast_type_bracket_tuple, loc, std::move(items)) {} + Vertex(SrcRange range, std::vector&& items) + : ASTTypeVararg(ast_type_bracket_tuple, range, std::move(items)) {} }; template<> @@ -448,8 +440,8 @@ template<> struct Vertex final : ASTTypeVararg { const std::vector& get_params_and_return() const { return children; } - Vertex(SrcLocation loc, std::vector&& params_and_return) - : ASTTypeVararg(ast_type_arrow_callable, loc, std::move(params_and_return)) {} + Vertex(SrcRange range, std::vector&& params_and_return) + : ASTTypeVararg(ast_type_arrow_callable, range, std::move(params_and_return)) {} }; template<> @@ -458,8 +450,8 @@ template<> struct Vertex final : ASTTypeVararg { const std::vector& get_variants() const { return children; } - Vertex(SrcLocation loc, std::vector&& variants) - : ASTTypeVararg(ast_type_vertical_bar_union, loc, std::move(variants)) {} + Vertex(SrcRange range, std::vector&& variants) + : ASTTypeVararg(ast_type_vertical_bar_union, range, std::move(variants)) {} }; @@ -470,8 +462,8 @@ template<> struct Vertex final : ASTTypeVararg { const std::vector& get_inner_and_args() const { return children; } - Vertex(SrcLocation loc, std::vector&& inner_and_args) - : ASTTypeVararg(ast_type_triangle_args, loc, std::move(inner_and_args)) {} + Vertex(SrcRange range, std::vector&& inner_and_args) + : ASTTypeVararg(ast_type_triangle_args, range, std::move(inner_and_args)) {} }; @@ -485,8 +477,8 @@ template<> // ast_empty_expression is "nothing" in context of expression, it has "unknown" type // example: `throw 123;` then "throw arg" is empty expression (opposed to `throw (123, arg)`) struct Vertex final : ASTExprLeaf { - explicit Vertex(SrcLocation loc) - : ASTExprLeaf(ast_empty_expression, loc) {} + explicit Vertex(SrcRange range) + : ASTExprLeaf(ast_empty_expression, range) {} }; @@ -496,8 +488,8 @@ template<> struct Vertex final : ASTExprUnary { AnyExprV get_expr() const { return child; } - Vertex(SrcLocation loc, AnyExprV expr) - : ASTExprUnary(ast_parenthesized_expression, loc, expr) {} + Vertex(SrcRange range, AnyExprV expr) + : ASTExprUnary(ast_parenthesized_expression, range, expr) {} }; template<> @@ -509,8 +501,8 @@ template<> struct Vertex final : ASTExprBlockOfStatements { auto get_block_statement() const { return child_block_statement->as(); } - Vertex(SrcLocation loc, AnyV child_block_statement) - : ASTExprBlockOfStatements(ast_braced_expression, loc, child_block_statement) {} + Vertex(SrcRange range, AnyV child_block_statement) + : ASTExprBlockOfStatements(ast_braced_expression, range, child_block_statement) {} }; template<> @@ -519,8 +511,8 @@ template<> struct Vertex final : ASTExprUnary { AnyExprV get_expr() const { return child; } - Vertex(SrcLocation loc, AnyExprV expr) - : ASTExprUnary(ast_braced_yield_result, loc, expr) {} + Vertex(SrcRange range, AnyExprV expr) + : ASTExprUnary(ast_braced_yield_result, range, expr) {} }; template<> @@ -532,8 +524,8 @@ struct Vertex final : ASTExprUnary { AnyExprV get_wrapped_expr() const { return child; } - Vertex(SrcLocation loc, AnyExprV wrapped_expr, const ASTAuxData* aux_data, TypePtr inferred_type) - : ASTExprUnary(ast_artificial_aux_vertex, loc, wrapped_expr) + Vertex(AnyExprV wrapped_expr, const ASTAuxData* aux_data, TypePtr inferred_type) + : ASTExprUnary(ast_artificial_aux_vertex, wrapped_expr->range, wrapped_expr) , aux_data(aux_data) { assign_inferred_type(inferred_type); } @@ -549,8 +541,8 @@ struct Vertex final : ASTExprVararg { const std::vector& get_items() const { return children; } AnyExprV get_item(int i) const { return child(i); } - Vertex(SrcLocation loc, std::vector items) - : ASTExprVararg(ast_tensor, loc, std::move(items)) {} + Vertex(SrcRange range, std::vector items) + : ASTExprVararg(ast_tensor, range, std::move(items)) {} }; template<> @@ -562,8 +554,8 @@ struct Vertex final : ASTExprVararg { const std::vector& get_items() const { return children; } AnyExprV get_item(int i) const { return child(i); } - Vertex(SrcLocation loc, std::vector items) - : ASTExprVararg(ast_bracket_tuple, loc, std::move(items)) {} + Vertex(SrcRange range, std::vector items) + : ASTExprVararg(ast_bracket_tuple, range, std::move(items)) {} }; template<> @@ -587,8 +579,8 @@ struct Vertex final : ASTExprLeaf { Vertex* mutate() const { return const_cast(this); } void assign_sym(const Symbol* sym); - Vertex(SrcLocation loc, V name_identifier, V instantiationTs) - : ASTExprLeaf(ast_reference, loc) + Vertex(SrcRange range, V name_identifier, V instantiationTs) + : ASTExprLeaf(ast_reference, range) , identifier(name_identifier), instantiationTs(instantiationTs) {} }; @@ -614,8 +606,8 @@ struct Vertex final : ASTExprLeaf { Vertex* mutate() const { return const_cast(this); } void assign_var_ref(LocalVarPtr var_ref); - Vertex(SrcLocation loc, V identifier, AnyTypeV type_node, bool is_immutable, bool is_lateinit, bool marked_as_redef) - : ASTExprLeaf(ast_local_var_lhs, loc) + Vertex(SrcRange range, V identifier, AnyTypeV type_node, bool is_immutable, bool is_lateinit, bool marked_as_redef) + : ASTExprLeaf(ast_local_var_lhs, range) , identifier(identifier), type_node(type_node), is_immutable(is_immutable), is_lateinit(is_lateinit), marked_as_redef(marked_as_redef) {} }; @@ -627,8 +619,8 @@ template<> struct Vertex final : ASTExprUnary { AnyExprV get_expr() const { return child; } // ast_local_var_lhs / ast_tensor / ast_bracket_tuple - Vertex(SrcLocation loc, AnyExprV expr) - : ASTExprUnary(ast_local_vars_declaration, loc, expr) {} + Vertex(SrcRange range, AnyExprV expr) + : ASTExprUnary(ast_local_vars_declaration, range, expr) {} }; template<> @@ -639,8 +631,8 @@ struct Vertex final : ASTExprLeaf { td::RefInt256 intval; // parsed value, 255 for "0xFF" std::string_view orig_str; // original "0xFF"; empty for nodes generated by compiler (e.g. in constant folding) - Vertex(SrcLocation loc, td::RefInt256 intval, std::string_view orig_str) - : ASTExprLeaf(ast_int_const, loc) + Vertex(SrcRange range, td::RefInt256 intval, std::string_view orig_str) + : ASTExprLeaf(ast_int_const, range) , intval(std::move(intval)) , orig_str(orig_str) {} }; @@ -656,8 +648,8 @@ struct Vertex final : ASTExprLeaf { Vertex* mutate() const { return const_cast(this); } void assign_literal_value(std::string&& literal_value); - Vertex(SrcLocation loc, std::string_view str_val) - : ASTExprLeaf(ast_string_const, loc) + Vertex(SrcRange range, std::string_view str_val) + : ASTExprLeaf(ast_string_const, range) , str_val(str_val) {} }; @@ -666,8 +658,8 @@ template<> struct Vertex final : ASTExprLeaf { bool bool_val; - Vertex(SrcLocation loc, bool bool_val) - : ASTExprLeaf(ast_bool_const, loc) + Vertex(SrcRange range, bool bool_val) + : ASTExprLeaf(ast_bool_const, range) , bool_val(bool_val) {} }; @@ -675,8 +667,8 @@ template<> // ast_null_keyword is the `null` literal // it should be handled with care; for instance, `null` takes special place in the type system struct Vertex final : ASTExprLeaf { - explicit Vertex(SrcLocation loc) - : ASTExprLeaf(ast_null_keyword, loc) {} + explicit Vertex(SrcRange range) + : ASTExprLeaf(ast_null_keyword, range) {} }; template<> @@ -689,8 +681,8 @@ struct Vertex final : ASTExprUnary { AnyExprV get_expr() const { return child; } - Vertex(SrcLocation loc, AnyExprV expr, bool passed_as_mutate) - : ASTExprUnary(ast_argument, loc, expr) + Vertex(SrcRange range, AnyExprV expr, bool passed_as_mutate) + : ASTExprUnary(ast_argument, range, expr) , passed_as_mutate(passed_as_mutate) {} }; @@ -700,8 +692,8 @@ struct Vertex final : ASTExprVararg { const std::vector& get_arguments() const { return children; } auto get_arg(int i) const { return child(i)->as(); } - Vertex(SrcLocation loc, std::vector arguments) - : ASTExprVararg(ast_argument_list, loc, std::move(arguments)) {} + Vertex(SrcRange range, std::vector arguments) + : ASTExprVararg(ast_argument_list, range, std::move(arguments)) {} }; template<> @@ -738,8 +730,8 @@ struct Vertex final : ASTExprUnary { Vertex* mutate() const { return const_cast(this); } void assign_target(const DotTarget& target); - Vertex(SrcLocation loc, AnyExprV obj, V identifier, V instantiationTs) - : ASTExprUnary(ast_dot_access, loc, obj) + Vertex(SrcRange range, AnyExprV obj, V identifier, V instantiationTs) + : ASTExprUnary(ast_dot_access, range, obj) , identifier(identifier), instantiationTs(instantiationTs) {} }; @@ -764,8 +756,8 @@ struct Vertex final : ASTExprBinary { Vertex* mutate() const { return const_cast(this); } void assign_fun_ref(FunctionPtr fun_ref, bool dot_obj_is_self); - Vertex(SrcLocation loc, AnyExprV lhs_f, V arguments) - : ASTExprBinary(ast_function_call, loc, lhs_f, arguments) {} + Vertex(SrcRange range, AnyExprV lhs_f, V arguments) + : ASTExprBinary(ast_function_call, range, lhs_f, arguments) {} }; template<> @@ -775,8 +767,8 @@ template<> // note, that for declaration `var _ = 1` underscore is a regular local var declared (with empty name) // but for `_ = 1` (not declaration) it's underscore; it's because `var _:int` is also correct struct Vertex final : ASTExprLeaf { - explicit Vertex(SrcLocation loc) - : ASTExprLeaf(ast_underscore, loc) {} + explicit Vertex(SrcRange range) + : ASTExprLeaf(ast_underscore, range) {} }; template<> @@ -787,8 +779,8 @@ struct Vertex final : ASTExprBinary { AnyExprV get_lhs() const { return lhs; } AnyExprV get_rhs() const { return rhs; } - explicit Vertex(SrcLocation loc, AnyExprV lhs, AnyExprV rhs) - : ASTExprBinary(ast_assign, loc, lhs, rhs) {} + explicit Vertex(SrcRange range, AnyExprV lhs, AnyExprV rhs) + : ASTExprBinary(ast_assign, range, lhs, rhs) {} }; template<> @@ -796,6 +788,7 @@ template<> // examples: `a += 4` / `b <<= c` struct Vertex final : ASTExprBinary { FunctionPtr fun_ref = nullptr; // filled at type inferring, points to `_+_` built-in for += + SrcRange operator_range; // range of `+=`, not of all the expression (for error messages) std::string_view operator_name; // without equal sign, "+" for operator += TokenType tok; // tok_set_* @@ -805,9 +798,9 @@ struct Vertex final : ASTExprBinary { Vertex* mutate() const { return const_cast(this); } void assign_fun_ref(FunctionPtr fun_ref); - Vertex(SrcLocation loc, std::string_view operator_name, TokenType tok, AnyExprV lhs, AnyExprV rhs) - : ASTExprBinary(ast_set_assign, loc, lhs, rhs) - , operator_name(operator_name), tok(tok) {} + Vertex(SrcRange range, SrcRange operator_range, std::string_view operator_name, TokenType tok, AnyExprV lhs, AnyExprV rhs) + : ASTExprBinary(ast_set_assign, range, lhs, rhs) + , operator_range(operator_range), operator_name(operator_name), tok(tok) {} }; template<> @@ -815,6 +808,7 @@ template<> // examples: `-1` / `~found` struct Vertex final : ASTExprUnary { FunctionPtr fun_ref = nullptr; // filled at type inferring, points to some built-in function + SrcRange operator_range; // range of `-`, not all the expression (for error messages) std::string_view operator_name; TokenType tok; @@ -823,9 +817,9 @@ struct Vertex final : ASTExprUnary { Vertex* mutate() const { return const_cast(this); } void assign_fun_ref(FunctionPtr fun_ref); - Vertex(SrcLocation loc, std::string_view operator_name, TokenType tok, AnyExprV rhs) - : ASTExprUnary(ast_unary_operator, loc, rhs) - , operator_name(operator_name), tok(tok) {} + Vertex(SrcRange range, SrcRange operator_range, std::string_view operator_name, TokenType tok, AnyExprV rhs) + : ASTExprUnary(ast_unary_operator, range, rhs) + , operator_range(operator_range), operator_name(operator_name), tok(tok) {} }; template<> @@ -834,6 +828,7 @@ template<> // note, that `a = b` is NOT a binary operator, it's ast_assign, also `a += b`, it's ast_set_assign struct Vertex final : ASTExprBinary { FunctionPtr fun_ref = nullptr; // filled at type inferring, points to some built-in function + SrcRange operator_range; // range of `+=`, not all the expression (for error messages) std::string_view operator_name; TokenType tok; @@ -843,9 +838,9 @@ struct Vertex final : ASTExprBinary { Vertex* mutate() const { return const_cast(this); } void assign_fun_ref(FunctionPtr fun_ref); - Vertex(SrcLocation loc, std::string_view operator_name, TokenType tok, AnyExprV lhs, AnyExprV rhs) - : ASTExprBinary(ast_binary_operator, loc, lhs, rhs) - , operator_name(operator_name), tok(tok) {} + Vertex(SrcRange range, SrcRange operator_range, std::string_view operator_name, TokenType tok, AnyExprV lhs, AnyExprV rhs) + : ASTExprBinary(ast_binary_operator, range, lhs, rhs) + , operator_range(operator_range), operator_name(operator_name), tok(tok) {} }; template<> @@ -856,8 +851,8 @@ struct Vertex final : ASTExprVararg { AnyExprV get_when_true() const { return child(1); } AnyExprV get_when_false() const { return child(2); } - Vertex(SrcLocation loc, AnyExprV cond, AnyExprV when_true, AnyExprV when_false) - : ASTExprVararg(ast_ternary_operator, loc, {cond, when_true, when_false}) {} + Vertex(SrcRange range, AnyExprV cond, AnyExprV when_true, AnyExprV when_false) + : ASTExprVararg(ast_ternary_operator, range, {cond, when_true, when_false}) {} }; template<> @@ -868,8 +863,8 @@ struct Vertex final : ASTExprUnary { AnyExprV get_expr() const { return child; } - Vertex(SrcLocation loc, AnyExprV expr, AnyTypeV type_node) - : ASTExprUnary(ast_cast_as_operator, loc, expr) + Vertex(SrcRange range, AnyExprV expr, AnyTypeV type_node) + : ASTExprUnary(ast_cast_as_operator, range, expr) , type_node(type_node) {} }; @@ -885,8 +880,8 @@ struct Vertex final : ASTExprUnary { Vertex* mutate() const { return const_cast(this); } void assign_is_negated(bool is_negated); - Vertex(SrcLocation loc, AnyExprV expr, AnyTypeV type_node, bool is_negated) - : ASTExprUnary(ast_is_type_operator, loc, expr) + Vertex(SrcRange range, AnyExprV expr, AnyTypeV type_node, bool is_negated) + : ASTExprUnary(ast_is_type_operator, range, expr) , type_node(type_node), is_negated(is_negated) {} }; @@ -896,8 +891,8 @@ template<> struct Vertex final : ASTExprUnary { AnyExprV get_expr() const { return child; } - Vertex(SrcLocation loc, AnyExprV expr) - : ASTExprUnary(ast_not_null_operator, loc, expr) {} + Vertex(SrcRange range, AnyExprV expr) + : ASTExprUnary(ast_not_null_operator, range, expr) {} }; template<> @@ -907,12 +902,13 @@ struct Vertex final : ASTExprUnary { LocalVarPtr dest_var_ref = nullptr; // `var st = lazy Storage.load()` AnyExprV get_expr() const { return child; } + SrcRange keyword_range() const { return SrcRange::span(range, 4); } Vertex* mutate() const { return const_cast(this); } void assign_dest_var_ref(LocalVarPtr dest_var_ref); - Vertex(SrcLocation loc, AnyExprV expr) - : ASTExprUnary(ast_lazy_operator, loc, expr) {} + Vertex(SrcRange range, AnyExprV expr) + : ASTExprUnary(ast_lazy_operator, range, expr) {} }; template<> @@ -924,11 +920,12 @@ struct Vertex final : ASTExprVararg { int get_arms_count() const { return size() - 1; } auto get_arm(int i) const { return child(i + 1)->as(); } const std::vector& get_all_children() const { return children; } + SrcRange keyword_range() const { return SrcRange::span(range, 5); } bool is_statement() const { return !is_rvalue && !is_lvalue; } - Vertex(SrcLocation loc, std::vector&& subject_and_arms) - : ASTExprVararg(ast_match_expression, loc, std::move(subject_and_arms)) {} + Vertex(SrcRange range, std::vector&& subject_and_arms) + : ASTExprVararg(ast_match_expression, range, std::move(subject_and_arms)) {} }; template<> @@ -947,8 +944,8 @@ struct Vertex final : ASTExprBinary { Vertex* mutate() const { return const_cast(this); } void assign_resolved_pattern(MatchArmKind pattern_kind, AnyExprV pattern_expr); - Vertex(SrcLocation loc, MatchArmKind pattern_kind, AnyTypeV pattern_type_node, AnyExprV pattern_expr, V body) - : ASTExprBinary(ast_match_arm, loc, pattern_expr, body) + Vertex(SrcRange range, MatchArmKind pattern_kind, AnyTypeV pattern_type_node, AnyExprV pattern_expr, V body) + : ASTExprBinary(ast_match_arm, range, pattern_expr, body) , pattern_kind(pattern_kind), pattern_type_node(pattern_type_node) {} }; @@ -969,8 +966,8 @@ struct Vertex final : ASTExprUnary { Vertex* mutate() const { return const_cast(this); } void assign_field_ref(StructFieldPtr field_ref); - Vertex(SrcLocation loc, V name_identifier, AnyExprV init_val) - : ASTExprUnary(ast_object_field, loc, init_val) + Vertex(SrcRange range, V name_identifier, AnyExprV init_val) + : ASTExprUnary(ast_object_field, range, init_val) , identifier(name_identifier) {} }; @@ -982,8 +979,8 @@ struct Vertex final : ASTExprVararg { auto get_field(int i) const { return child(i)->as(); } std::vector get_all_fields() const { return children; } - Vertex(SrcLocation loc, std::vector&& fields) - : ASTExprVararg(ast_object_body, loc, std::move(fields)) {} + Vertex(SrcRange range, std::vector&& fields) + : ASTExprVararg(ast_object_body, range, std::move(fields)) {} }; template<> @@ -1000,8 +997,8 @@ struct Vertex final : ASTExprUnary { Vertex* mutate() const { return const_cast(this); } void assign_struct_ref(StructPtr struct_ref); - Vertex(SrcLocation loc, AnyTypeV type_node, V body) - : ASTExprUnary(ast_object_literal, loc, body) + Vertex(SrcRange range, AnyTypeV type_node, V body) + : ASTExprUnary(ast_object_literal, range, body) , type_node(type_node) {} }; @@ -1017,8 +1014,8 @@ template<> // example: `;` (just semicolon) // example: body of `builtin` function is empty statement (not a zero sequence) struct Vertex final : ASTStatementVararg { - explicit Vertex(SrcLocation loc) - : ASTStatementVararg(ast_empty_statement, loc, {}) {} + explicit Vertex(SrcRange range) + : ASTStatementVararg(ast_empty_statement, range, {}) {} }; template<> @@ -1026,7 +1023,6 @@ template<> // example: function body is a block // example: do while body is a block struct Vertex final : ASTStatementVararg { - SrcLocation loc_end; AnyV first_unreachable = nullptr; const std::vector& get_items() const { return children; } @@ -1036,9 +1032,8 @@ struct Vertex final : ASTStatementVararg { void assign_first_unreachable(AnyV first_unreachable); void assign_new_children(std::vector&& children); - Vertex(SrcLocation loc, SrcLocation loc_end, std::vector&& items) - : ASTStatementVararg(ast_block_statement, loc, std::move(items)) - , loc_end(loc_end) {} + Vertex(SrcRange range, std::vector&& items) + : ASTStatementVararg(ast_block_statement, range, std::move(items)) {} }; template<> @@ -1048,9 +1043,10 @@ template<> struct Vertex : ASTStatementUnary { AnyExprV get_return_value() const { return child_as_expr(); } bool has_return_value() const { return child->kind != ast_empty_expression; } + SrcRange keyword_range() const { return SrcRange::span(range, 6); } - Vertex(SrcLocation loc, AnyExprV child) - : ASTStatementUnary(ast_return_statement, loc, child) {} + Vertex(SrcRange range, AnyExprV child) + : ASTStatementUnary(ast_return_statement, range, child) {} }; template<> @@ -1064,9 +1060,10 @@ struct Vertex final : ASTStatementVararg { AnyExprV get_cond() const { return child_as_expr(0); } auto get_if_body() const { return children.at(1)->as(); } auto get_else_body() const { return children.at(2)->as(); } // always exists (when else omitted, it's empty) + SrcRange keyword_range() const { return SrcRange::span(range, 2); } - Vertex(SrcLocation loc, bool is_ifnot, AnyExprV cond, V if_body, V else_body) - : ASTStatementVararg(ast_if_statement, loc, {cond, if_body, else_body}) + Vertex(SrcRange range, bool is_ifnot, AnyExprV cond, V if_body, V else_body) + : ASTStatementVararg(ast_if_statement, range, {cond, if_body, else_body}) , is_ifnot(is_ifnot) {} }; @@ -1076,9 +1073,10 @@ template<> struct Vertex final : ASTStatementVararg { AnyExprV get_cond() const { return child_as_expr(0); } auto get_body() const { return children.at(1)->as(); } + SrcRange keyword_range() const { return SrcRange::span(range, 6); } - Vertex(SrcLocation loc, AnyExprV cond, V body) - : ASTStatementVararg(ast_repeat_statement, loc, {cond, body}) {} + Vertex(SrcRange range, AnyExprV cond, V body) + : ASTStatementVararg(ast_repeat_statement, range, {cond, body}) {} }; template<> @@ -1087,9 +1085,10 @@ template<> struct Vertex final : ASTStatementVararg { AnyExprV get_cond() const { return child_as_expr(0); } auto get_body() const { return children.at(1)->as(); } + SrcRange keyword_range() const { return SrcRange::span(range, 5); } - Vertex(SrcLocation loc, AnyExprV cond, V body) - : ASTStatementVararg(ast_while_statement, loc, {cond, body}) {} + Vertex(SrcRange range, AnyExprV cond, V body) + : ASTStatementVararg(ast_while_statement, range, {cond, body}) {} }; template<> @@ -1098,9 +1097,10 @@ template<> struct Vertex final : ASTStatementVararg { auto get_body() const { return children.at(0)->as(); } AnyExprV get_cond() const { return child_as_expr(1); } + SrcRange keyword_range() const { return SrcRange::span(range, 2); } - Vertex(SrcLocation loc, V body, AnyExprV cond) - : ASTStatementVararg(ast_do_while_statement, loc, {body, cond}) {} + Vertex(SrcRange range, V body, AnyExprV cond) + : ASTStatementVararg(ast_do_while_statement, range, {body, cond}) {} }; template<> @@ -1111,9 +1111,10 @@ struct Vertex final : ASTStatementVararg { AnyExprV get_thrown_code() const { return child_as_expr(0); } bool has_thrown_arg() const { return child_as_expr(1)->kind != ast_empty_expression; } AnyExprV get_thrown_arg() const { return child_as_expr(1); } + SrcRange keyword_range() const { return SrcRange::span(range, 5); } - Vertex(SrcLocation loc, AnyExprV thrown_code, AnyExprV thrown_arg) - : ASTStatementVararg(ast_throw_statement, loc, {thrown_code, thrown_arg}) {} + Vertex(SrcRange range, AnyExprV thrown_code, AnyExprV thrown_arg) + : ASTStatementVararg(ast_throw_statement, range, {thrown_code, thrown_arg}) {} }; template<> @@ -1122,9 +1123,10 @@ template<> struct Vertex final : ASTStatementVararg { AnyExprV get_cond() const { return child_as_expr(0); } AnyExprV get_thrown_code() const { return child_as_expr(1); } + SrcRange keyword_range() const { return SrcRange::span(range, 6); } - Vertex(SrcLocation loc, AnyExprV cond, AnyExprV thrown_code) - : ASTStatementVararg(ast_assert_statement, loc, {cond, thrown_code}) {} + Vertex(SrcRange range, AnyExprV cond, AnyExprV thrown_code) + : ASTStatementVararg(ast_assert_statement, range, {cond, thrown_code}) {} }; template<> @@ -1136,9 +1138,10 @@ struct Vertex final : ASTStatementVararg { auto get_try_body() const { return children.at(0)->as(); } auto get_catch_expr() const { return children.at(1)->as(); } // (excNo, arg), always len 2 auto get_catch_body() const { return children.at(2)->as(); } + SrcRange keyword_range() const { return SrcRange::span(range, 3); } - Vertex(SrcLocation loc, V try_body, V catch_expr, V catch_body) - : ASTStatementVararg(ast_try_catch_statement, loc, {try_body, catch_expr, catch_body}) {} + Vertex(SrcRange range, V try_body, V catch_expr, V catch_body) + : ASTStatementVararg(ast_try_catch_statement, range, {try_body, catch_expr, catch_body}) {} }; template<> @@ -1152,8 +1155,8 @@ struct Vertex final : ASTStatementVararg { const std::vector& get_asm_commands() const { return children; } // ast_string_const[] - Vertex(SrcLocation loc, std::vector arg_order, std::vector ret_order, std::vector asm_commands) - : ASTStatementVararg(ast_asm_body, loc, std::move(asm_commands)) + Vertex(SrcRange range, std::vector arg_order, std::vector ret_order, std::vector asm_commands) + : ASTStatementVararg(ast_asm_body, range, std::move(asm_commands)) , arg_order(std::move(arg_order)), ret_order(std::move(ret_order)) {} }; @@ -1172,8 +1175,8 @@ struct Vertex final : ASTOtherLeaf { std::string_view nameT; AnyTypeV default_type_node; // exists for ``, nullptr otherwise - Vertex(SrcLocation loc, std::string_view nameT, AnyTypeV default_type_node) - : ASTOtherLeaf(ast_genericsT_item, loc) + Vertex(SrcRange range, std::string_view nameT, AnyTypeV default_type_node) + : ASTOtherLeaf(ast_genericsT_item, range) , nameT(nameT), default_type_node(default_type_node) {} }; @@ -1184,8 +1187,8 @@ struct Vertex final : ASTOtherVararg { std::vector get_items() const { return children; } auto get_item(int i) const { return children.at(i)->as(); } - Vertex(SrcLocation loc, std::vector genericsT_items) - : ASTOtherVararg(ast_genericsT_list, loc, std::move(genericsT_items)) {} + Vertex(SrcRange range, std::vector genericsT_items) + : ASTOtherVararg(ast_genericsT_list, range, std::move(genericsT_items)) {} int lookup_idx(std::string_view nameT) const; }; @@ -1197,8 +1200,8 @@ template<> struct Vertex final : ASTOtherLeaf { AnyTypeV type_node; - Vertex(SrcLocation loc, AnyTypeV type_node) - : ASTOtherLeaf(ast_instantiationT_item, loc) + Vertex(SrcRange range, AnyTypeV type_node) + : ASTOtherLeaf(ast_instantiationT_item, range) , type_node(type_node) {} }; @@ -1209,25 +1212,24 @@ struct Vertex final : ASTOtherVararg { std::vector get_items() const { return children; } auto get_item(int i) const { return children.at(i)->as(); } - Vertex(SrcLocation loc, std::vector&& instantiationTs) - : ASTOtherVararg(ast_instantiationT_list, loc, std::move(instantiationTs)) {} + Vertex(SrcRange range, std::vector&& instantiationTs) + : ASTOtherVararg(ast_instantiationT_list, range, std::move(instantiationTs)) {} }; template<> // ast_parameter is a parameter of a function in its declaration // example: `fun f(a: int, mutate b: slice)` has 2 parameters // example: `fun f(a: int = 0)` has 1 parameter with default value -struct Vertex final : ASTOtherLeaf { - std::string_view param_name; +struct Vertex final : ASTOtherVararg { AnyTypeV type_node; // always exists, typing parameters is mandatory AnyExprV default_value; // default value of the parameter or nullptr bool declared_as_mutate; // declared as `mutate param_name` - bool is_underscore() const { return param_name.empty(); } + auto get_identifier() const { return children.at(0)->as(); } - Vertex(SrcLocation loc, std::string_view param_name, AnyTypeV type_node, AnyExprV default_value, bool declared_as_mutate) - : ASTOtherLeaf(ast_parameter, loc) - , param_name(param_name), type_node(type_node), default_value(default_value), declared_as_mutate(declared_as_mutate) {} + Vertex(SrcRange range, V name_identifier, AnyTypeV type_node, AnyExprV default_value, bool declared_as_mutate) + : ASTOtherVararg(ast_parameter, range, {name_identifier}) + , type_node(type_node), default_value(default_value), declared_as_mutate(declared_as_mutate) {} }; template<> @@ -1237,8 +1239,8 @@ struct Vertex final : ASTOtherVararg { const std::vector& get_params() const { return children; } auto get_param(int i) const { return children.at(i)->as(); } - Vertex(SrcLocation loc, std::vector params) - : ASTOtherVararg(ast_parameter_list, loc, std::move(params)) {} + Vertex(SrcRange range, std::vector params) + : ASTOtherVararg(ast_parameter_list, range, std::move(params)) {} int lookup_idx(std::string_view param_name) const; int get_mutate_params_count() const; @@ -1256,8 +1258,8 @@ struct Vertex final : ASTOtherVararg { static AnnotationKind parse_kind(std::string_view name); - Vertex(SrcLocation loc, std::string_view name, AnnotationKind kind, V arg_probably_empty) - : ASTOtherVararg(ast_annotation, loc, {arg_probably_empty}) + Vertex(SrcRange range, std::string_view name, AnnotationKind kind, V arg_probably_empty) + : ASTOtherVararg(ast_annotation, range, {arg_probably_empty}) , name(name), kind(kind) {} }; @@ -1289,8 +1291,8 @@ struct Vertex final : ASTOtherVararg { Vertex* mutate() const { return const_cast(this); } void assign_fun_ref(FunctionPtr fun_ref); - Vertex(SrcLocation loc, V name_identifier, V parameters, AnyV body, AnyTypeV receiver_type_node, AnyTypeV return_type_node, V genericsT_list, td::RefInt256 tvm_method_id, int flags, FunctionInlineMode inline_mode) - : ASTOtherVararg(ast_function_declaration, loc, {name_identifier, parameters, body}) + Vertex(SrcRange range, V name_identifier, V parameters, AnyV body, AnyTypeV receiver_type_node, AnyTypeV return_type_node, V genericsT_list, td::RefInt256 tvm_method_id, int flags, FunctionInlineMode inline_mode) + : ASTOtherVararg(ast_function_declaration, range, {name_identifier, parameters, body}) , receiver_type_node(receiver_type_node), return_type_node(return_type_node), genericsT_list(genericsT_list), tvm_method_id(std::move(tvm_method_id)), flags(flags), inline_mode(inline_mode) {} }; @@ -1307,8 +1309,8 @@ struct Vertex final : ASTOtherVararg { Vertex* mutate() const { return const_cast(this); } void assign_glob_ref(GlobalVarPtr glob_ref); - Vertex(SrcLocation loc, V name_identifier, AnyTypeV type_node) - : ASTOtherVararg(ast_global_var_declaration, loc, {name_identifier}) + Vertex(SrcRange range, V name_identifier, AnyTypeV type_node) + : ASTOtherVararg(ast_global_var_declaration, range, {name_identifier}) , type_node(type_node) {} }; @@ -1325,8 +1327,8 @@ struct Vertex final : ASTOtherVararg { Vertex* mutate() const { return const_cast(this); } void assign_const_ref(GlobalConstPtr const_ref); - Vertex(SrcLocation loc, V name_identifier, AnyTypeV type_node, AnyExprV init_value) - : ASTOtherVararg(ast_constant_declaration, loc, {name_identifier, init_value}) + Vertex(SrcRange range, V name_identifier, AnyTypeV type_node, AnyExprV init_value) + : ASTOtherVararg(ast_constant_declaration, range, {name_identifier, init_value}) , type_node(type_node) {} }; @@ -1344,8 +1346,8 @@ struct Vertex final : ASTOtherVararg { Vertex* mutate() const { return const_cast(this); } void assign_alias_ref(AliasDefPtr alias_ref); - Vertex(SrcLocation loc, V name_identifier, V genericsT_list, AnyTypeV underlying_type_node) - : ASTOtherVararg(ast_type_alias_declaration, loc, {name_identifier}) + Vertex(SrcRange range, V name_identifier, V genericsT_list, AnyTypeV underlying_type_node) + : ASTOtherVararg(ast_type_alias_declaration, range, {name_identifier}) , genericsT_list(genericsT_list), underlying_type_node(underlying_type_node) {} }; @@ -1360,8 +1362,8 @@ struct Vertex final : ASTOtherVararg { auto get_identifier() const { return children.at(0)->as(); } - Vertex(SrcLocation loc, V name_identifier, bool is_private, bool is_readonly, AnyExprV default_value, AnyTypeV type_node) - : ASTOtherVararg(ast_struct_field, loc, {name_identifier}) + Vertex(SrcRange range, V name_identifier, bool is_private, bool is_readonly, AnyExprV default_value, AnyTypeV type_node) + : ASTOtherVararg(ast_struct_field, range, {name_identifier}) , is_private(is_private), is_readonly(is_readonly), type_node(type_node), default_value(default_value) {} }; @@ -1373,8 +1375,8 @@ struct Vertex final : ASTOtherVararg { auto get_field(int i) const { return children.at(i)->as(); } const std::vector& get_all_fields() const { return children; } - Vertex(SrcLocation loc, std::vector&& members) - : ASTOtherVararg(ast_struct_body, loc, std::move(members)) {} + Vertex(SrcRange range, std::vector&& members) + : ASTOtherVararg(ast_struct_body, range, std::move(members)) {} }; template<> @@ -1395,8 +1397,8 @@ struct Vertex final : ASTOtherVararg { Vertex* mutate() const { return const_cast(this); } void assign_struct_ref(StructPtr struct_ref); - Vertex(SrcLocation loc, V name_identifier, V genericsT_list, StructData::Overflow1023Policy overflow1023_policy, AnyExprV opcode, V struct_body) - : ASTOtherVararg(ast_struct_declaration, loc, {name_identifier, opcode, struct_body}) + Vertex(SrcRange range, V name_identifier, V genericsT_list, StructData::Overflow1023Policy overflow1023_policy, AnyExprV opcode, V struct_body) + : ASTOtherVararg(ast_struct_declaration, range, {name_identifier, opcode, struct_body}) , genericsT_list(genericsT_list), overflow1023_policy(overflow1023_policy) {} }; @@ -1408,8 +1410,8 @@ struct Vertex final : ASTOtherVararg { auto get_identifier() const { return children.at(0)->as(); } - Vertex(SrcLocation loc, V name_identifier, AnyExprV init_value) - : ASTOtherVararg(ast_enum_member, loc, {name_identifier}) + Vertex(SrcRange range, V name_identifier, AnyExprV init_value) + : ASTOtherVararg(ast_enum_member, range, {name_identifier}) , init_value(init_value) {} }; @@ -1421,8 +1423,8 @@ struct Vertex final : ASTOtherVararg { auto get_member(int i) const { return children.at(i)->as(); } const std::vector& get_all_members() const { return children; } - Vertex(SrcLocation loc, std::vector&& members) - : ASTOtherVararg(ast_enum_body, loc, std::move(members)) {} + Vertex(SrcRange range, std::vector&& members) + : ASTOtherVararg(ast_enum_body, range, std::move(members)) {} }; template<> @@ -1439,8 +1441,8 @@ struct Vertex final : ASTOtherVararg { Vertex* mutate() const { return const_cast(this); } void assign_enum_ref(EnumDefPtr enum_ref); - Vertex(SrcLocation loc, V name_identifier, AnyTypeV colon_type, V enum_body) - : ASTOtherVararg(ast_enum_declaration, loc, {name_identifier, enum_body}) + Vertex(SrcRange range, V name_identifier, AnyTypeV colon_type, V enum_body) + : ASTOtherVararg(ast_enum_declaration, range, {name_identifier, enum_body}) , colon_type(colon_type) {} }; @@ -1451,8 +1453,8 @@ template<> struct Vertex final : ASTOtherLeaf { std::string_view semver; - Vertex(SrcLocation loc, std::string_view semver) - : ASTOtherLeaf(ast_tolk_required_version, loc) + Vertex(SrcRange range, std::string_view semver) + : ASTOtherLeaf(ast_tolk_required_version, range) , semver(semver) {} }; @@ -1469,8 +1471,8 @@ struct Vertex final : ASTOtherVararg { Vertex* mutate() const { return const_cast(this); } void assign_src_file(const SrcFile* file); - Vertex(SrcLocation loc, V file_name) - : ASTOtherVararg(ast_import_directive, loc, {file_name}) {} + Vertex(SrcRange range, V file_name) + : ASTOtherVararg(ast_import_directive, range, {file_name}) {} }; template<> @@ -1483,8 +1485,8 @@ struct Vertex final : ASTOtherVararg { const std::vector& get_toplevel_declarations() const { return children; } - Vertex(const SrcFile* file, std::vector toplevel_declarations) - : ASTOtherVararg(ast_tolk_file, SrcLocation(file), std::move(toplevel_declarations)) + Vertex(const SrcFile* file, SrcRange range, std::vector toplevel_declarations) + : ASTOtherVararg(ast_tolk_file, range, std::move(toplevel_declarations)) , file(file) {} }; diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 256acd3fff..7c7bb51ff5 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -15,6 +15,7 @@ along with TON Blockchain Library. If not, see . */ #include "tolk.h" +#include "compilation-errors.h" #include "compiler-state.h" #include "type-system.h" #include "generics-helpers.h" @@ -72,14 +73,14 @@ void define_builtin_method(const std::string& name, TypePtr receiver_type, const } void FunctionBodyBuiltinAsmOp::compile(AsmOpList& dest, std::vector& out, std::vector& in, - SrcLocation loc) const { - dest << simple_compile(out, in, loc); + AnyV origin) const { + dest << simple_compile(out, in, origin); } -void FunctionBodyAsm::compile(AsmOpList& dest, SrcLocation loc) const { +void FunctionBodyAsm::compile(AsmOpList& dest, AnyV origin) const { for (const AsmOp& op : ops) { AsmOp copy = op; - copy.loc = loc; + copy.origin = origin; dest << std::move(copy); } } @@ -349,89 +350,89 @@ bool VarDescr::always_neq(const VarDescr& other) const { (always_odd() && other.always_even()); } -AsmOp exec_op(SrcLocation loc, std::string op) { - return AsmOp::Custom(loc, op); +AsmOp exec_op(AnyV origin, std::string op) { + return AsmOp::Custom(origin, op); } -AsmOp exec_op(SrcLocation loc, std::string op, int args, int retv = 1) { - return AsmOp::Custom(loc, op, args, retv); +AsmOp exec_op(AnyV origin, std::string op, int args, int retv = 1) { + return AsmOp::Custom(origin, op, args, retv); } -AsmOp exec_arg_op(SrcLocation loc, std::string op, long long arg, int args, int retv) { +AsmOp exec_arg_op(AnyV origin, std::string op, long long arg, int args, int retv) { std::ostringstream os; os << arg << ' ' << op; - return AsmOp::Custom(loc, os.str(), args, retv); + return AsmOp::Custom(origin, os.str(), args, retv); } -AsmOp exec_arg_op(SrcLocation loc, std::string op, td::RefInt256 arg, int args, int retv) { +AsmOp exec_arg_op(AnyV origin, std::string op, td::RefInt256 arg, int args, int retv) { std::ostringstream os; os << arg << ' ' << op; - return AsmOp::Custom(loc, os.str(), args, retv); + return AsmOp::Custom(origin, os.str(), args, retv); } -AsmOp exec_arg2_op(SrcLocation loc, std::string op, long long imm1, long long imm2, int args, int retv) { +AsmOp exec_arg2_op(AnyV origin, std::string op, long long imm1, long long imm2, int args, int retv) { std::ostringstream os; os << imm1 << ' ' << imm2 << ' ' << op; - return AsmOp::Custom(loc, os.str(), args, retv); + return AsmOp::Custom(origin, os.str(), args, retv); } -AsmOp push_const(SrcLocation loc, td::RefInt256 x) { - return AsmOp::IntConst(loc, std::move(x)); +AsmOp push_const(AnyV origin, td::RefInt256 x) { + return AsmOp::IntConst(origin, std::move(x)); } -static AsmOp compile_add(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_add(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const + y.int_const); if (!r.int_const->is_valid()) { - throw ParseError(loc, "integer overflow"); + fire(origin, "integer overflow"); } x.unused(); y.unused(); - return push_const(loc, r.int_const); + return push_const(origin, r.int_const); } r.val = emulate_add(x.val, y.val); if (y.is_int_const() && y.int_const->signed_fits_bits(8)) { y.unused(); if (y.always_zero()) { - return AsmOp::Nop(loc); + return AsmOp::Nop(origin); } if (*y.int_const == 1) { - return exec_op(loc, "INC", 1); + return exec_op(origin, "INC", 1); } if (*y.int_const == -1) { - return exec_op(loc, "DEC", 1); + return exec_op(origin, "DEC", 1); } - return exec_arg_op(loc, "ADDCONST", y.int_const, 1); + return exec_arg_op(origin, "ADDCONST", y.int_const, 1); } if (x.is_int_const() && x.int_const->signed_fits_bits(8)) { x.unused(); if (x.always_zero()) { - return AsmOp::Nop(loc); + return AsmOp::Nop(origin); } if (*x.int_const == 1) { - return exec_op(loc, "INC", 1); + return exec_op(origin, "INC", 1); } if (*x.int_const == -1) { - return exec_op(loc, "DEC", 1); + return exec_op(origin, "DEC", 1); } - return exec_arg_op(loc, "ADDCONST", x.int_const, 1); + return exec_arg_op(origin, "ADDCONST", x.int_const, 1); } - return exec_op(loc, "ADD", 2); + return exec_op(origin, "ADD", 2); } -static AsmOp compile_sub(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_sub(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const - y.int_const); if (!r.int_const->is_valid()) { - throw ParseError(loc, "integer overflow"); + fire(origin, "integer overflow"); } x.unused(); y.unused(); - return push_const(loc, r.int_const); + return push_const(origin, r.int_const); } r.val = emulate_sub(x.val, y.val); if (y.is_int_const() && (-y.int_const)->signed_fits_bits(8)) { @@ -440,121 +441,121 @@ static AsmOp compile_sub(std::vector& res, std::vector& args return {}; } if (*y.int_const == 1) { - return exec_op(loc, "DEC", 1); + return exec_op(origin, "DEC", 1); } if (*y.int_const == -1) { - return exec_op(loc, "INC", 1); + return exec_op(origin, "INC", 1); } - return exec_arg_op(loc, "ADDCONST", -y.int_const, 1); + return exec_arg_op(origin, "ADDCONST", -y.int_const, 1); } if (x.always_zero()) { x.unused(); - return exec_op(loc, "NEGATE", 1); + return exec_op(origin, "NEGATE", 1); } - return exec_op(loc, "SUB", 2); + return exec_op(origin, "SUB", 2); } -static AsmOp compile_unary_minus(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_unary_minus(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(-x.int_const); if (!r.int_const->is_valid()) { - throw ParseError(loc, "integer overflow"); + fire(origin, "integer overflow"); } x.unused(); - return push_const(loc, r.int_const); + return push_const(origin, r.int_const); } r.val = emulate_negate(x.val); - return exec_op(loc, "NEGATE", 1); + return exec_op(origin, "NEGATE", 1); } -static AsmOp compile_unary_plus(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_unary_plus(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(x.int_const); x.unused(); - return push_const(loc, r.int_const); + return push_const(origin, r.int_const); } r.val = x.val; - return AsmOp::Nop(loc); + return AsmOp::Nop(origin); } -static AsmOp compile_logical_not(std::vector& res, std::vector& args, SrcLocation loc, bool for_int_arg) { +static AsmOp compile_logical_not(std::vector& res, std::vector& args, AnyV origin, bool for_int_arg) { tolk_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(x.int_const == 0 ? -1 : 0); x.unused(); - return push_const(loc, r.int_const); + return push_const(origin, r.int_const); } r.val = VarDescr::ValBool; // for integers, `!var` is `var != 0` // for booleans, `!var` can be shortened to `~var` (works the same for 0/-1 but consumes less) - return for_int_arg ? exec_op(loc, "0 EQINT", 1) : exec_op(loc, "NOT", 1); + return for_int_arg ? exec_op(origin, "0 EQINT", 1) : exec_op(origin, "NOT", 1); } -static AsmOp compile_bitwise_and(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_bitwise_and(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const & y.int_const); x.unused(); y.unused(); - return push_const(loc, r.int_const); + return push_const(origin, r.int_const); } r.val = emulate_bitwise_and(x.val, y.val); - return exec_op(loc, "AND", 2); + return exec_op(origin, "AND", 2); } -static AsmOp compile_bitwise_or(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_bitwise_or(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const | y.int_const); x.unused(); y.unused(); - return push_const(loc, r.int_const); + return push_const(origin, r.int_const); } r.val = emulate_bitwise_or(x.val, y.val); - return exec_op(loc, "OR", 2); + return exec_op(origin, "OR", 2); } -static AsmOp compile_bitwise_xor(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_bitwise_xor(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const ^ y.int_const); x.unused(); y.unused(); - return push_const(loc, r.int_const); + return push_const(origin, r.int_const); } r.val = emulate_bitwise_xor(x.val, y.val); - return exec_op(loc, "XOR", 2); + return exec_op(origin, "XOR", 2); } -static AsmOp compile_bitwise_not(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_bitwise_not(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(~x.int_const); x.unused(); - return push_const(loc, r.int_const); + return push_const(origin, r.int_const); } r.val = emulate_bitwise_not(x.val); - return exec_op(loc, "NOT", 1); + return exec_op(origin, "NOT", 1); } -static AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation loc) { +static AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, AnyV origin) { if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const * y.int_const); if (!r.int_const->is_valid()) { - throw ParseError(loc, "integer overflow"); + fire(origin, "integer overflow"); } x.unused(); y.unused(); - return push_const(loc, r.int_const); + return push_const(origin, r.int_const); } r.val = emulate_mul(x.val, y.val); if (y.is_int_const()) { @@ -565,23 +566,23 @@ static AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLoca // dubious optimization: NaN * 0 = ? r.set_const(y.int_const); x.unused(); - return push_const(loc, r.int_const); + return push_const(origin, r.int_const); } if (*y.int_const == 1 && x.always_finite()) { - return AsmOp::Nop(loc); + return AsmOp::Nop(origin); } if (*y.int_const == -1) { - return exec_op(loc, "NEGATE", 1); + return exec_op(origin, "NEGATE", 1); } - return exec_arg_op(loc, "MULCONST", y.int_const, 1); + return exec_arg_op(origin, "MULCONST", y.int_const, 1); } if (k > 0) { y.unused(); - return exec_arg_op(loc, "LSHIFT#", k, 1); + return exec_arg_op(origin, "LSHIFT#", k, 1); } if (k == 0) { y.unused(); - return AsmOp::Nop(loc); + return AsmOp::Nop(origin); } } if (x.is_int_const()) { @@ -592,48 +593,48 @@ static AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLoca // dubious optimization: NaN * 0 = ? r.set_const(x.int_const); y.unused(); - return push_const(loc, r.int_const); + return push_const(origin, r.int_const); } if (*x.int_const == 1 && y.always_finite()) { - return AsmOp::Nop(loc); + return AsmOp::Nop(origin); } if (*x.int_const == -1) { - return exec_op(loc, "NEGATE", 1); + return exec_op(origin, "NEGATE", 1); } - return exec_arg_op(loc, "MULCONST", x.int_const, 1); + return exec_arg_op(origin, "MULCONST", x.int_const, 1); } if (k > 0) { x.unused(); - return exec_arg_op(loc, "LSHIFT#", k, 1); + return exec_arg_op(origin, "LSHIFT#", k, 1); } if (k == 0) { x.unused(); - return AsmOp::Nop(loc); + return AsmOp::Nop(origin); } } - return exec_op(loc, "MUL", 2); + return exec_op(origin, "MUL", 2); } -static AsmOp compile_mul(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_mul(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 1 && args.size() == 2); - return compile_mul_internal(res[0], args[0], args[1], loc); + return compile_mul_internal(res[0], args[0], args[1], origin); } -static AsmOp compile_lshift(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_lshift(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (y.is_int_const()) { auto yv = y.int_const->to_long(); if (yv < 0 || yv > 256) { - throw ParseError(loc, "lshift argument is out of range"); + fire(origin, "lshift argument is out of range"); } else if (x.is_int_const()) { r.set_const(x.int_const << (int)yv); if (!r.int_const->is_valid()) { - throw ParseError(loc, "integer overflow"); + fire(origin, "integer overflow"); } x.unused(); y.unused(); - return push_const(loc, r.int_const); + return push_const(origin, r.int_const); } } r.val = emulate_lshift(x.val, y.val); @@ -642,38 +643,38 @@ static AsmOp compile_lshift(std::vector& res, std::vector& a if (!k /* && x.always_finite() */) { // dubious optimization: what if x=NaN ? y.unused(); - return AsmOp::Nop(loc); + return AsmOp::Nop(origin); } y.unused(); - return exec_arg_op(loc, "LSHIFT#", k, 1); + return exec_arg_op(origin, "LSHIFT#", k, 1); } if (x.is_int_const()) { auto xv = x.int_const->to_long(); if (xv == 1) { x.unused(); - return exec_op(loc, "POW2", 1); + return exec_op(origin, "POW2", 1); } if (xv == -1) { x.unused(); - return exec_op(loc, "-1 PUSHINT SWAP LSHIFT", 1); + return exec_op(origin, "-1 PUSHINT SWAP LSHIFT", 1); } } - return exec_op(loc, "LSHIFT", 2); + return exec_op(origin, "LSHIFT", 2); } -static AsmOp compile_rshift(std::vector& res, std::vector& args, SrcLocation loc, +static AsmOp compile_rshift(std::vector& res, std::vector& args, AnyV origin, int round_mode) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (y.is_int_const()) { auto yv = y.int_const->to_long(); if (yv < 0 || yv > 256) { - throw ParseError(loc, "rshift argument is out of range"); + fire(origin, "rshift argument is out of range"); } else if (x.is_int_const()) { r.set_const(td::rshift(x.int_const, (int)yv, round_mode)); x.unused(); y.unused(); - return push_const(loc, r.int_const); + return push_const(origin, r.int_const); } } r.val = emulate_rshift(x.val, y.val); @@ -683,36 +684,36 @@ static AsmOp compile_rshift(std::vector& res, std::vector& a if (!k /* && x.always_finite() */) { // dubious optimization: what if x=NaN ? y.unused(); - return AsmOp::Nop(loc); + return AsmOp::Nop(origin); } y.unused(); - return exec_arg_op(loc, rshift + "#", k, 1); + return exec_arg_op(origin, rshift + "#", k, 1); } - return exec_op(loc, rshift, 2); + return exec_op(origin, rshift, 2); } -static AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation loc, int round_mode) { +static AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, AnyV origin, int round_mode) { if (x.is_int_const() && y.is_int_const()) { r.set_const(div(x.int_const, y.int_const, round_mode)); if (!r.int_const->is_valid()) { - throw ParseError(loc, *y.int_const == 0 ? "division by zero" : "integer overflow"); + fire(origin, *y.int_const == 0 ? "division by zero" : "integer overflow"); } x.unused(); y.unused(); - return push_const(loc, r.int_const); + return push_const(origin, r.int_const); } r.val = emulate_div(x.val, y.val); if (y.is_int_const()) { if (*y.int_const == 0) { - throw ParseError(loc, "division by zero"); + fire(origin, "division by zero"); } if (*y.int_const == 1 && x.always_finite()) { y.unused(); - return AsmOp::Nop(loc); + return AsmOp::Nop(origin); } if (*y.int_const == -1) { y.unused(); - return exec_op(loc, "NEGATE", 1); + return exec_op(origin, "NEGATE", 1); } int k = is_pos_pow2(y.int_const); if (k > 0) { @@ -721,44 +722,44 @@ static AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLoca if (round_mode >= 0) { op += (round_mode > 0 ? 'C' : 'R'); } - return exec_arg_op(loc, op + '#', k, 1); + return exec_arg_op(origin, op + '#', k, 1); } } std::string op = "DIV"; if (round_mode >= 0) { op += (round_mode > 0 ? 'C' : 'R'); } - return exec_op(loc, op, 2); + return exec_op(origin, op, 2); } -static AsmOp compile_div(std::vector& res, std::vector& args, SrcLocation loc, int round_mode) { +static AsmOp compile_div(std::vector& res, std::vector& args, AnyV origin, int round_mode) { tolk_assert(res.size() == 1 && args.size() == 2); - return compile_div_internal(res[0], args[0], args[1], loc, round_mode); + return compile_div_internal(res[0], args[0], args[1], origin, round_mode); } -static AsmOp compile_mod(std::vector& res, std::vector& args, SrcLocation loc, +static AsmOp compile_mod(std::vector& res, std::vector& args, AnyV origin, int round_mode) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(mod(x.int_const, y.int_const, round_mode)); if (!r.int_const->is_valid()) { - throw ParseError(loc, *y.int_const == 0 ? "division by zero" : "integer overflow"); + fire(origin, *y.int_const == 0 ? "division by zero" : "integer overflow"); } x.unused(); y.unused(); - return push_const(loc, r.int_const); + return push_const(origin, r.int_const); } r.val = emulate_mod(x.val, y.val); if (y.is_int_const()) { if (*y.int_const == 0) { - throw ParseError(loc, "division by zero"); + fire(origin, "division by zero"); } if ((*y.int_const == 1 || *y.int_const == -1) && x.always_finite()) { x.unused(); y.unused(); r.set_const(td::zero_refint()); - return push_const(loc, r.int_const); + return push_const(origin, r.int_const); } int k = is_pos_pow2(y.int_const); if (k > 0) { @@ -767,29 +768,29 @@ static AsmOp compile_mod(std::vector& res, std::vector& args if (round_mode >= 0) { op += (round_mode > 0 ? 'C' : 'R'); } - return exec_arg_op(loc, op + '#', k, 1); + return exec_arg_op(origin, op + '#', k, 1); } } std::string op = "MOD"; if (round_mode >= 0) { op += (round_mode > 0 ? 'C' : 'R'); } - return exec_op(loc, op, 2); + return exec_op(origin, op, 2); } -static AsmOp compile_muldiv(std::vector& res, std::vector& args, SrcLocation loc, +static AsmOp compile_muldiv(std::vector& res, std::vector& args, AnyV origin, int round_mode) { tolk_assert(res.size() == 1 && args.size() == 3); VarDescr &r = res[0], &x = args[0], &y = args[1], &z = args[2]; if (x.is_int_const() && y.is_int_const() && z.is_int_const()) { r.set_const(muldiv(x.int_const, y.int_const, z.int_const, round_mode)); if (!r.int_const->is_valid()) { - throw ParseError(loc, *z.int_const == 0 ? "division by zero" : "integer overflow"); + fire(origin, *z.int_const == 0 ? "division by zero" : "integer overflow"); } x.unused(); y.unused(); z.unused(); - return push_const(loc, r.int_const); + return push_const(origin, r.int_const); } if (x.always_zero() || y.always_zero()) { // dubious optimization for z=0... @@ -797,26 +798,26 @@ static AsmOp compile_muldiv(std::vector& res, std::vector& a y.unused(); z.unused(); r.set_const(td::make_refint(0)); - return push_const(loc, r.int_const); + return push_const(origin, r.int_const); } char c = (round_mode < 0) ? 0 : (round_mode > 0 ? 'C' : 'R'); r.val = emulate_div(emulate_mul(x.val, y.val), z.val); if (z.is_int_const()) { if (*z.int_const == 0) { - throw ParseError(loc, "division by zero"); + fire(origin, "division by zero"); } if (*z.int_const == 1) { z.unused(); - return compile_mul_internal(r, x, y, loc); + return compile_mul_internal(r, x, y, origin); } } if (y.is_int_const() && *y.int_const == 1) { y.unused(); - return compile_div_internal(r, x, z, loc, round_mode); + return compile_div_internal(r, x, z, origin, round_mode); } if (x.is_int_const() && *x.int_const == 1) { x.unused(); - return compile_div_internal(r, y, z, loc, round_mode); + return compile_div_internal(r, y, z, origin, round_mode); } if (z.is_int_const()) { int k = is_pos_pow2(z.int_const); @@ -826,7 +827,7 @@ static AsmOp compile_muldiv(std::vector& res, std::vector& a if (c) { op += c; } - return exec_arg_op(loc, op + '#', k, 2); + return exec_arg_op(origin, op + '#', k, 2); } } if (y.is_int_const()) { @@ -837,7 +838,7 @@ static AsmOp compile_muldiv(std::vector& res, std::vector& a if (c) { op += c; } - return exec_arg_op(loc, op, k, 2); + return exec_arg_op(origin, op, k, 2); } } if (x.is_int_const()) { @@ -848,19 +849,19 @@ static AsmOp compile_muldiv(std::vector& res, std::vector& a if (c) { op += c; } - return exec_arg_op(loc, op, k, 2); + return exec_arg_op(origin, op, k, 2); } } std::string op = "MULDIV"; if (c) { op += c; } - return exec_op(loc, op, 3); + return exec_op(origin, op, 3); } // fun mulDivMod(x: int, y: int, z: int): (int, int) asm "MULDIVMOD"; -static AsmOp compile_muldivmod(std::vector&, std::vector&, SrcLocation loc) { - return AsmOp::Custom(loc, "MULDIVMOD", 3, 2); +static AsmOp compile_muldivmod(std::vector&, std::vector&, AnyV origin) { + return AsmOp::Custom(origin, "MULDIVMOD", 3, 2); } static int compute_compare(td::RefInt256 x, td::RefInt256 y, int mode) { @@ -904,7 +905,7 @@ static int compute_compare(const VarDescr& x, const VarDescr& y, int mode) { } } -static AsmOp compile_cmp_int(std::vector& res, std::vector& args, SrcLocation loc, int mode) { +static AsmOp compile_cmp_int(std::vector& res, std::vector& args, AnyV origin, int mode) { tolk_assert(mode >= 1 && mode <= 7); tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; @@ -913,7 +914,7 @@ static AsmOp compile_cmp_int(std::vector& res, std::vector& r.set_const(v); x.unused(); y.unused(); - return mode == 7 ? push_const(loc, r.int_const) : AsmOp::BoolConst(loc, v != 0); + return mode == 7 ? push_const(origin, r.int_const) : AsmOp::BoolConst(origin, v != 0); } int v = compute_compare(x, y, mode); // std::cerr << "compute_compare(" << x << ", " << y << ", " << mode << ") = " << v << std::endl; @@ -922,7 +923,7 @@ static AsmOp compile_cmp_int(std::vector& res, std::vector& r.set_const(v - (v >> 2) - 2); x.unused(); y.unused(); - return mode == 7 ? push_const(loc, r.int_const) : AsmOp::BoolConst(loc, v & 1); + return mode == 7 ? push_const(origin, r.int_const) : AsmOp::BoolConst(origin, v & 1); } r.val = ~0; if (v & 1) { @@ -941,31 +942,31 @@ static AsmOp compile_cmp_int(std::vector& res, std::vector& if (mode != 7) { if (y.is_int_const() && y.int_const >= -128 && y.int_const <= 127) { y.unused(); - return exec_arg_op(loc, cmp_int_names[mode], y.int_const + cmp_int_delta[mode], 1); + return exec_arg_op(origin, cmp_int_names[mode], y.int_const + cmp_int_delta[mode], 1); } if (x.is_int_const() && x.int_const >= -128 && x.int_const <= 127) { x.unused(); mode = ((mode & 4) >> 2) | (mode & 2) | ((mode & 1) << 2); - return exec_arg_op(loc, cmp_int_names[mode], x.int_const + cmp_int_delta[mode], 1); + return exec_arg_op(origin, cmp_int_names[mode], x.int_const + cmp_int_delta[mode], 1); } } - return exec_op(loc, cmp_names[mode], 2); + return exec_op(origin, cmp_names[mode], 2); } -static AsmOp compile_throw(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_throw(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.empty() && args.size() == 1); VarDescr& x = args[0]; if (x.is_int_const() && x.int_const >= 0) { // in Fift assembler, "N THROW" is valid if N < 2048; for big N (particularly, widely used 0xFFFF) // we now still generate "N THROW", and later, in optimizer, transform it to "PUSHINT" + "THROWANY" x.unused(); - return exec_arg_op(loc, "THROW", x.int_const, 0, 0); + return exec_arg_op(origin, "THROW", x.int_const, 0, 0); } else { - return exec_op(loc, "THROWANY", 1, 0); + return exec_op(origin, "THROWANY", 1, 0); } } -static AsmOp compile_throw_if_unless(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_throw_if_unless(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.empty() && args.size() == 3); VarDescr &x = args[0], &y = args[1], &z = args[2]; if (!z.always_true() && !z.always_false()) { @@ -980,22 +981,22 @@ static AsmOp compile_throw_if_unless(std::vector& res, std::vectorunsigned_fits_bits(11)) { x.unused(); - return skip_cond ? exec_arg_op(loc, "THROW", x.int_const, 0, 0) : exec_arg_op(loc, "THROW"s + suff, x.int_const, 1, 0); + return skip_cond ? exec_arg_op(origin, "THROW", x.int_const, 0, 0) : exec_arg_op(origin, "THROW"s + suff, x.int_const, 1, 0); } else { - return skip_cond ? exec_op(loc, "THROWANY", 1, 0) : exec_op(loc, "THROWANY"s + suff, 2, 0); + return skip_cond ? exec_op(origin, "THROWANY", 1, 0) : exec_op(origin, "THROWANY"s + suff, 2, 0); } } -static AsmOp compile_calc_InMessage_originalForwardFee(std::vector& res, std::vector& args, SrcLocation loc) { - return exec_op(loc, "GETORIGINALFWDFEE", 2); +static AsmOp compile_calc_InMessage_originalForwardFee(std::vector& res, std::vector& args, AnyV origin) { + return exec_op(origin, "GETORIGINALFWDFEE", 2); } -static AsmOp compile_calc_InMessage_getInMsgParam(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_calc_InMessage_getInMsgParam(std::vector& res, std::vector& args, AnyV origin) { // instead of "0 INMSGPARAM", generate "INMSG_BOUNCE", etc. — these are aliases in Asm.fif static const char* aliases[] = { "INMSG_BOUNCE", "INMSG_BOUNCED", "INMSG_SRC", "INMSG_FWDFEE", "INMSG_LT", "INMSG_UTIME", "INMSG_ORIGVALUE", "INMSG_VALUE", "INMSG_VALUEEXTRA", "INMSG_STATEINIT", @@ -1004,51 +1005,51 @@ static AsmOp compile_calc_InMessage_getInMsgParam(std::vector& res, st args[0].unused(); size_t idx = args[0].int_const->to_long(); if (idx < std::size(aliases)) { - return exec_op(loc, aliases[idx], 0); + return exec_op(origin, aliases[idx], 0); } - return exec_arg_op(loc, "INMSGPARAM", args[0].int_const, 1); + return exec_arg_op(origin, "INMSGPARAM", args[0].int_const, 1); } -static AsmOp compile_throw_arg(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_throw_arg(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.empty() && args.size() == 2); VarDescr &x = args[1]; if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { x.unused(); - return exec_arg_op(loc, "THROWARG", x.int_const, 1, 0); + return exec_arg_op(origin, "THROWARG", x.int_const, 1, 0); } else { - return exec_op(loc, "THROWARGANY", 2, 0); + return exec_op(origin, "THROWARGANY", 2, 0); } } // `x ? y : z` can be compiled as `CONDSEL` asm instruction if y and z are don't require evaluation -static AsmOp compile_ternary_as_condsel(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_ternary_as_condsel(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 1 && args.size() == 3); VarDescr& cond = args[0]; // args = [ cond, when_true, when_false ] if (cond.always_true()) { cond.unused(); args[2].unused(); - return AsmOp::Nop(loc); + return AsmOp::Nop(origin); } if (cond.always_false()) { cond.unused(); args[1].unused(); - return AsmOp::Nop(loc); + return AsmOp::Nop(origin); } - return exec_op(loc, "CONDSEL", 3, 1); + return exec_op(origin, "CONDSEL", 3, 1); } -static AsmOp compile_bool_const(std::vector& res, std::vector& args, SrcLocation loc, bool val) { +static AsmOp compile_bool_const(std::vector& res, std::vector& args, AnyV origin, bool val) { tolk_assert(res.size() == 1 && args.empty()); VarDescr& r = res[0]; r.set_const(val ? -1 : 0); - return AsmOp::Const(loc, val ? "TRUE" : "FALSE"); + return AsmOp::Const(origin, val ? "TRUE" : "FALSE"); } // fun slice.loadInt (mutate self, len: int): int asm(s len -> 1 0) "LDIX"; // fun slice.loadUint (mutate self, len: int): int asm( -> 1 0) "LDUX"; // fun slice.preloadInt (self, len: int): int asm "PLDIX"; // fun slice.preloadUint(self, len: int): int asm "PLDUX"; -static AsmOp compile_fetch_int(std::vector& res, std::vector& args, SrcLocation loc, bool fetch, bool sgnd) { +static AsmOp compile_fetch_int(std::vector& res, std::vector& args, AnyV origin, bool fetch, bool sgnd) { tolk_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); auto &y = args[1], &r = res.back(); r.val = (sgnd ? VarDescr::FiniteInt : VarDescr::FiniteUInt); @@ -1063,14 +1064,14 @@ static AsmOp compile_fetch_int(std::vector& res, std::vector } if (v > 0) { y.unused(); - return exec_arg_op(loc, (fetch ? "LD"s : "PLD"s) + (sgnd ? 'I' : 'U'), v, 1, 1 + (unsigned)fetch); + return exec_arg_op(origin, (fetch ? "LD"s : "PLD"s) + (sgnd ? 'I' : 'U'), v, 1, 1 + (unsigned)fetch); } } - return exec_op(loc, (fetch ? "LD"s : "PLD"s) + (sgnd ? "IX" : "UX"), 2, 1 + (unsigned)fetch); + return exec_op(origin, (fetch ? "LD"s : "PLD"s) + (sgnd ? "IX" : "UX"), 2, 1 + (unsigned)fetch); } // fun slice.__loadVarInt(mutate self, bits: int, unsigned: bool): int -static AsmOp compile_fetch_varint(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_fetch_varint(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(args.size() == 3 && res.size() == 2); // it's a hidden function for auto-serialization (not exposed to stdlib), to bits/unsigned are not dynamic tolk_assert(args[1].is_int_const() && args[2].is_int_const()); @@ -1080,17 +1081,17 @@ static AsmOp compile_fetch_varint(std::vector& res, std::vector& res, std::vector& args, SrcLocation loc, bool sgnd) { +static AsmOp compile_store_int(std::vector& res, std::vector& args, AnyV origin, bool sgnd) { tolk_assert(args.size() == 3 && res.size() == 1); auto& x = args[1]; auto& z = args[2]; @@ -1103,18 +1104,18 @@ static AsmOp compile_store_int(std::vector& res, std::vector if (x.int_const->fits_bits(len, sgnd)) { z.unused(); x.unused(); - return AsmOp::Custom(loc, "MY_store_int"s + (sgnd ? "I " : "U ") + x.int_const->to_dec_string() + " " + z.int_const->to_dec_string(), 1); + return AsmOp::Custom(origin, "MY_store_int"s + (sgnd ? "I " : "U ") + x.int_const->to_dec_string() + " " + z.int_const->to_dec_string(), 1); } } if (z.is_int_const() && z.int_const > 0 && z.int_const <= 256) { z.unused(); - return exec_arg_op(loc, sgnd? "STI" : "STU", z.int_const, 2, 1); + return exec_arg_op(origin, sgnd? "STI" : "STU", z.int_const, 2, 1); } - return exec_op(loc, sgnd ? "STIX" : "STUX", 3, 1); + return exec_op(origin, sgnd ? "STIX" : "STUX", 3, 1); } // fun builder.__storeVarInt (mutate self, x: int, bits: int, unsigned: bool): self -static AsmOp compile_store_varint(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_store_varint(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(args.size() == 4 && res.size() == 1); // it's a hidden function for auto-serialization (not exposed to stdlib), to bits/unsigned are not dynamic tolk_assert(args[2].is_int_const() && args[3].is_int_const()); @@ -1124,47 +1125,47 @@ static AsmOp compile_store_varint(std::vector& res, std::vector 1 0) "1 STI"; -static AsmOp compile_store_bool(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_store_bool(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(args.size() == 2 && res.size() == 1); auto& v = args[1]; // same purpose as for storeInt/storeUint above // (particularly, `b.storeUint(const_int,32).storeBool(const_bool)` will be joined) if (v.is_int_const() && v.int_const == 0 && G.settings.optimization_level >= 2) { v.unused(); - return AsmOp::Custom(loc, "MY_store_intU 0 1", 1); + return AsmOp::Custom(origin, "MY_store_intU 0 1", 1); } if (v.is_int_const() && v.int_const == -1 && G.settings.optimization_level >= 2) { v.unused(); - return AsmOp::Custom(loc, "MY_store_intU 1 1", 1); + return AsmOp::Custom(origin, "MY_store_intU 1 1", 1); } - return exec_op(loc, "1 STI", 2, 1); + return exec_op(origin, "1 STI", 2, 1); } // fun builder.storeCoins(mutate self, value: coins): self asm "STGRAMS"; -static AsmOp compile_store_coins(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_store_coins(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(args.size() == 2 && res.size() == 1); auto& v = args[1]; // same purpose as for storeInt/storeUint above // (particularly, `b.storeUint(const_int,32).storeCoins(const_zero)` will be joined) if (v.is_int_const() && v.int_const == 0 && G.settings.optimization_level >= 2) { v.unused(); - return AsmOp::Custom(loc, "MY_store_intU 0 4", 1); + return AsmOp::Custom(origin, "MY_store_intU 0 4", 1); } - return exec_op(loc, "STGRAMS", 2, 1); + return exec_op(origin, "STGRAMS", 2, 1); } // fun slice.loadBits (mutate self, len: int): self asm(s len -> 1 0) "LDSLICEX" // fun slice.preloadBits(self, len: int): slice asm(s len -> 1 0) "PLDSLICEX" -static AsmOp compile_fetch_slice(std::vector& res, std::vector& args, SrcLocation loc, bool fetch) { +static AsmOp compile_fetch_slice(std::vector& res, std::vector& args, AnyV origin, bool fetch) { tolk_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); auto& y = args[1]; int v = -1; @@ -1172,15 +1173,15 @@ static AsmOp compile_fetch_slice(std::vector& res, std::vectorto_long(); if (v > 0) { y.unused(); - return exec_arg_op(loc, fetch ? "LDSLICE" : "PLDSLICE", v, 1, 1 + (unsigned)fetch); + return exec_arg_op(origin, fetch ? "LDSLICE" : "PLDSLICE", v, 1, 1 + (unsigned)fetch); } } - return exec_op(loc, fetch ? "LDSLICEX" : "PLDSLICEX", 2, 1 + (unsigned)fetch); + return exec_op(origin, fetch ? "LDSLICEX" : "PLDSLICEX", 2, 1 + (unsigned)fetch); } // fun slice.tryStripPrefix(mutate self, prefix: int, prefixLen: int): bool // constructs "x{...} SDBEGINSQ" for constant arguments -static AsmOp compile_slice_sdbeginsq(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_slice_sdbeginsq(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(args.size() == 3 && res.size() == 2); auto& prefix = args[1]; auto& prefix_len = args[2]; @@ -1189,13 +1190,13 @@ static AsmOp compile_slice_sdbeginsq(std::vector& res, std::vectorto_long(), static_cast(prefix_len.int_const->to_long())); - return AsmOp::Custom(loc, opcode.format_as_slice() + " SDBEGINSQ", 0, 1); + return AsmOp::Custom(origin, opcode.format_as_slice() + " SDBEGINSQ", 0, 1); } - throw ParseError(loc, "slice.tryStripPrefix can be used only with constant arguments"); + fire(origin, "slice.tryStripPrefix can be used only with constant arguments"); } // fun slice.skipBits(mutate self, len: int): self "SDSKIPFIRST" -static AsmOp compile_skip_bits_in_slice(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_skip_bits_in_slice(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(args.size() == 2 && res.size() == 1); auto& len = args[1]; // same technique as for storeUint: @@ -1203,52 +1204,52 @@ static AsmOp compile_skip_bits_in_slice(std::vector& res, std::vector< // to track this, represent it as a separate fake instruction to be detected by optimizer later if (len.is_int_const() && len.int_const >= 0 && G.settings.optimization_level >= 2) { len.unused(); - return AsmOp::Custom(loc, "MY_skip_bits " + len.int_const->to_dec_string(), 1); + return AsmOp::Custom(origin, "MY_skip_bits " + len.int_const->to_dec_string(), 1); } - return exec_op(loc, "SDSKIPFIRST", 2, 1); + return exec_op(origin, "SDSKIPFIRST", 2, 1); } // fun tuple.get(t: tuple, index: int): X asm "INDEXVAR"; -static AsmOp compile_tuple_get(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_tuple_get(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(args.size() == 2 && res.size() == 1); auto& y = args[1]; if (y.is_int_const() && y.int_const >= 0 && y.int_const < 16) { y.unused(); - return exec_arg_op(loc, "INDEX", y.int_const, 1, 1); + return exec_arg_op(origin, "INDEX", y.int_const, 1, 1); } - return exec_op(loc, "INDEXVAR", 2, 1); + return exec_op(origin, "INDEXVAR", 2, 1); } // fun tuple.set(mutate self: tuple, value: X, index: int): void asm "SETINDEXVAR"; -static AsmOp compile_tuple_set_at(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_tuple_set_at(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(args.size() == 3 && res.size() == 1); auto& y = args[2]; if (y.is_int_const() && y.int_const >= 0 && y.int_const < 16) { y.unused(); - return exec_arg_op(loc, "SETINDEX", y.int_const, 1, 1); + return exec_arg_op(origin, "SETINDEX", y.int_const, 1, 1); } - return exec_op(loc, "SETINDEXVAR", 2, 1); + return exec_op(origin, "SETINDEXVAR", 2, 1); } // fun debug.dumpStack(): void asm "DUMPSTK"; -static AsmOp compile_dumpstk(std::vector&, std::vector&, SrcLocation loc) { - return AsmOp::Custom(loc, "DUMPSTK", 0, 0); +static AsmOp compile_dumpstk(std::vector&, std::vector&, AnyV origin) { + return AsmOp::Custom(origin, "DUMPSTK", 0, 0); } // fun debug.printString(x: T): void asm "STRDUMP"; -static AsmOp compile_strdump(std::vector&, std::vector&, SrcLocation loc) { - return AsmOp::Custom(loc, "STRDUMP DROP", 1, 1); +static AsmOp compile_strdump(std::vector&, std::vector&, AnyV origin) { + return AsmOp::Custom(origin, "STRDUMP DROP", 1, 1); } // fun debug.print(x: T): void; -static AsmOp compile_debug_print_to_string(std::vector&, std::vector& args, SrcLocation loc) { +static AsmOp compile_debug_print_to_string(std::vector&, std::vector& args, AnyV origin) { int n = static_cast(args.size()); if (n == 1) { // most common case - return AsmOp::Custom(loc, "s0 DUMP DROP", 1, 1); + return AsmOp::Custom(origin, "s0 DUMP DROP", 1, 1); } if (n > 15) { - throw ParseError(loc, "call overflow, exceeds 15 elements"); + fire(origin, "call overflow, exceeds 15 elements"); } std::string cmd; for (int i = n - 1; i >= 0; --i) { @@ -1256,54 +1257,54 @@ static AsmOp compile_debug_print_to_string(std::vector&, std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_any_object_to_tuple(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 1); int n = static_cast(args.size()); if (n > 15) { - throw ParseError(loc, "call overflow, exceeds 15 elements"); + fire(origin, "call overflow, exceeds 15 elements"); } - return exec_op(loc, std::to_string(args.size()) + " TUPLE", n, 1); + return exec_op(origin, std::to_string(args.size()) + " TUPLE", n, 1); } // fun sizeof(anything: T): int; // (returns the number of stack elements) -static AsmOp compile_any_object_sizeof(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_any_object_sizeof(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 1); int n = static_cast(args.size()); res[0].set_const(n); for (int i = 0; i < n; ++i) { args[i].unused(); } - return AsmOp::IntConst(loc, td::make_refint(n)); + return AsmOp::IntConst(origin, td::make_refint(n)); } // fun ton(amount: slice): coins; ton("0.05") replaced by 50000000 at compile-time // same for stringCrc32(constString: slice) and others -static AsmOp compile_time_only_function(std::vector&, std::vector&, SrcLocation loc) { +static AsmOp compile_time_only_function(std::vector&, std::vector&, AnyV origin) { // all ton() invocations are constants, replaced by integers; no dynamic values allowed, no work at runtime tolk_assert(false); - return AsmOp::Nop(loc); + return AsmOp::Nop(origin); } // `null` literal is under the hood transformed to PUSHNULL -static AsmOp compile_push_null(std::vector&, std::vector&, SrcLocation loc) { - return AsmOp::Const(loc, "PUSHNULL"); +static AsmOp compile_push_null(std::vector&, std::vector&, AnyV origin) { + return AsmOp::Const(origin, "PUSHNULL"); } // fun __isNull(X arg): bool -static AsmOp compile_is_null(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_is_null(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(args.size() == 1 && res.size() == 1); res[0].val = VarDescr::ValBool; - return exec_op(loc, "ISNULL", 1, 1); + return exec_op(origin, "ISNULL", 1, 1); } // fun __expect_type(, ""): void; -static AsmOp compile_expect_type(std::vector&, std::vector&, SrcLocation loc) { +static AsmOp compile_expect_type(std::vector&, std::vector&, AnyV origin) { // handled by type checker, does nothing at runtime - return AsmOp::Nop(loc); + return AsmOp::Nop(origin); } // implemented in dedicated files @@ -1858,10 +1859,10 @@ void patch_builtins_after_stdlib_loaded() { // in stdlib, there is a default parameter `options = {}`; since default parameters are evaluated with AST, // emulate its presence in built-in functions; it looks ugly, but currently I don't have a better solution - auto v_empty_PackOptions = createV({}, nullptr, createV({}, {})); + auto v_empty_PackOptions = createV(SrcRange::undefined(), nullptr, createV(SrcRange::undefined(), {})); v_empty_PackOptions->assign_struct_ref(struct_ref_PackOptions); v_empty_PackOptions->assign_inferred_type(PackOptions); - auto v_empty_UnpackOptions = createV({}, nullptr, createV({}, {})); + auto v_empty_UnpackOptions = createV(SrcRange::undefined(), nullptr, createV(SrcRange::undefined(), {})); v_empty_UnpackOptions->assign_struct_ref(struct_ref_UnpackOptions); v_empty_UnpackOptions->assign_inferred_type(UnpackOptions); @@ -1909,7 +1910,7 @@ void patch_builtins_after_stdlib_loaded() { lookup_function("map.addOrGetExisting")->mutate()->declared_return_type = LookupResultT; lookup_function("map.deleteAndGetDeleted")->mutate()->declared_return_type = LookupResultT; - auto v_def_throwCode = createV({}, td::make_refint(9), "9"); + auto v_def_throwCode = createV(SrcRange::undefined(), td::make_refint(9), "9"); v_def_throwCode->assign_inferred_type(TypeDataInt::create()); lookup_function("map.mustGet")->mutate()->parameters[2].assign_default_value(v_def_throwCode); diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp index 8be3688105..9c944ab034 100644 --- a/tolk/codegen.cpp +++ b/tolk/codegen.cpp @@ -15,6 +15,7 @@ along with TON Blockchain Library. If not, see . */ #include "tolk.h" +#include "compilation-errors.h" #include "compiler-state.h" #include "type-system.h" @@ -82,10 +83,10 @@ void Stack::forget_const() { } } -void Stack::issue_pop(SrcLocation loc, int i) { +void Stack::issue_pop(AnyV origin, int i) { validate(i); if (output_enabled()) { - o << AsmOp::Pop(loc, i); + o << AsmOp::Pop(origin, i); } at(i) = get(0); s.pop_back(); @@ -93,22 +94,22 @@ void Stack::issue_pop(SrcLocation loc, int i) { opt_show(); } -void Stack::issue_push(SrcLocation loc, int i) { +void Stack::issue_push(AnyV origin, int i) { validate(i); if (output_enabled()) { - o << AsmOp::Push(loc, i); + o << AsmOp::Push(origin, i); } s.push_back(get(i)); modified(); opt_show(); } -void Stack::issue_xchg(SrcLocation loc, int i, int j) { +void Stack::issue_xchg(AnyV origin, int i, int j) { validate(i); validate(j); if (i != j && get(i) != get(j)) { if (output_enabled()) { - o << AsmOp::Xchg(loc, i, j); + o << AsmOp::Xchg(origin, i, j); } std::swap(at(i), at(j)); modified(); @@ -116,7 +117,7 @@ void Stack::issue_xchg(SrcLocation loc, int i, int j) { } } -int Stack::drop_vars_except(SrcLocation loc, const VarDescrList& var_info, int excl_var) { +int Stack::drop_vars_except(AnyV origin, const VarDescrList& var_info, int excl_var) { int dropped = 0, changes; do { changes = 0; @@ -125,7 +126,7 @@ int Stack::drop_vars_except(SrcLocation loc, const VarDescrList& var_info, int e var_idx_t idx = at(i).first; if (((!var_info[idx] || var_info[idx]->is_unused()) && idx != excl_var) || find(idx, 0, i - 1) >= 0) { // unneeded - issue_pop(loc, i); + issue_pop(origin, i); changes = 1; break; } @@ -141,7 +142,7 @@ void Stack::show() { os << ' '; o.show_var_ext(os, i); } - o << AsmOp::Comment({}, os.str()); + o << AsmOp::Comment(nullptr, os.str()); mode |= _Shown; } @@ -175,17 +176,17 @@ void Stack::assign_var(var_idx_t new_idx, var_idx_t old_idx) { } } -void Stack::do_copy_var(SrcLocation loc, var_idx_t new_idx, var_idx_t old_idx) { +void Stack::do_copy_var(AnyV origin, var_idx_t new_idx, var_idx_t old_idx) { int i = find(old_idx); tolk_assert(i >= 0 && "variable not found in stack"); if (find(old_idx, i + 1) < 0) { - issue_push(loc, i); + issue_push(origin, i); tolk_assert(at(0).first == old_idx); } assign_var(new_idx, old_idx); } -void Stack::enforce_state(SrcLocation loc, const StackLayout& req_stack) { +void Stack::enforce_state(AnyV origin, const StackLayout& req_stack) { int k = (int)req_stack.size(); for (int i = 0; i < k; i++) { var_idx_t x = req_stack[i]; @@ -194,18 +195,18 @@ void Stack::enforce_state(SrcLocation loc, const StackLayout& req_stack) { } while (depth() > 0 && std::find(req_stack.cbegin(), req_stack.cend(), get(0).first) == req_stack.cend()) { // current TOS entry is unused in req_stack, drop it - issue_pop(loc, 0); + issue_pop(origin, 0); } int j = find(x); if (j >= depth() - i) { - issue_push(loc, j); + issue_push(origin, j); j = 0; } - issue_xchg(loc, j, depth() - i - 1); + issue_xchg(origin, j, depth() - i - 1); tolk_assert(s[i].first == x); } while (depth() > k) { - issue_pop(loc, 0); + issue_pop(origin, 0); } tolk_assert(depth() == k); for (int i = 0; i < k; i++) { @@ -223,12 +224,12 @@ void Stack::merge_const(const Stack& req_stack) { } } -void Stack::merge_state(SrcLocation loc, const Stack& req_stack) { - enforce_state(loc, req_stack.vars()); +void Stack::merge_state(AnyV origin, const Stack& req_stack) { + enforce_state(origin, req_stack.vars()); merge_const(req_stack); } -void Stack::rearrange_top(SrcLocation loc, const StackLayout& top, std::vector last) { +void Stack::rearrange_top(AnyV origin, const StackLayout& top, std::vector last) { while (last.size() < top.size()) { last.push_back(false); } @@ -253,24 +254,24 @@ void Stack::rearrange_top(SrcLocation loc, const StackLayout& top, std::vectoris_builtin_function() && f_sym->name == "__throw") - || (cl == _IntConst && next->cl == _Call && next->f_sym->is_builtin_function() && next->f_sym->name == "__throw"); + bool will_now_immediate_throw = (cl == _Call && f_sym->is_builtin() && f_sym->name == "__throw") + || (cl == _IntConst && next->cl == _Call && next->f_sym->is_builtin() && next->f_sym->name == "__throw"); if (!will_now_immediate_throw) { - stack.drop_vars_except(loc, var_info); + stack.drop_vars_except(origin, var_info); stack.opt_show(); } @@ -293,9 +294,9 @@ bool Op::generate_code_step(Stack& stack) { case _Import: return true; case _Return: { - stack.enforce_state(loc, left); + stack.enforce_state(origin, left); if (stack.o.retalt_ && (stack.mode & Stack::_NeedRetAlt)) { - stack.o << AsmOp::Custom(loc, "RETALT"); + stack.o << AsmOp::Custom(origin, "RETALT"); stack.o.retalt_inserted_ = true; } stack.opt_show(); @@ -309,11 +310,11 @@ bool Op::generate_code_step(Stack& stack) { auto cidx = stack.o.register_const(int_const); int i = stack.find_const(cidx); if (i < 0) { - stack.o << push_const(loc, int_const); + stack.o << push_const(origin, int_const); stack.push_new_const(left[0], cidx); } else { tolk_assert(stack.at(i).second == cidx); - stack.do_copy_var(loc, left[0], stack[i]); + stack.do_copy_var(origin, left[0], stack[i]); } return true; } @@ -322,7 +323,7 @@ bool Op::generate_code_step(Stack& stack) { if (!p || p->is_unused()) { return true; } - stack.o << AsmOp::Const(loc, "x{" + str_const + "} PUSHSLICE"); + stack.o << AsmOp::Const(origin, "x{" + str_const + "} PUSHSLICE"); stack.push_new_var(left[0]); return true; } @@ -338,10 +339,10 @@ bool Op::generate_code_step(Stack& stack) { if (!used || disabled()) { return true; } - stack.o << AsmOp::Custom(loc, "$" + g_sym->name + " GETGLOB", 0, 1); + stack.o << AsmOp::Custom(origin, "$" + g_sym->name + " GETGLOB", 0, 1); if (left.size() != 1) { tolk_assert(left.size() <= 15); - stack.o << AsmOp::UnTuple(loc, (int)left.size()); + stack.o << AsmOp::UnTuple(origin, (int)left.size()); } for (auto i : left) { stack.push_new_var(i); @@ -353,9 +354,9 @@ bool Op::generate_code_step(Stack& stack) { if (!p || p->is_unused() || disabled()) { return true; } - stack.o << AsmOp::Custom(loc, "CONT:<{"); + stack.o << AsmOp::Custom(origin, "CONT:<{"); stack.o.indent(); - if (f_sym->is_asm_function() || f_sym->is_builtin_function()) { + if (f_sym->is_asm_function() || f_sym->is_builtin()) { // TODO: create and compile a true lambda instead of this (so that arg_order and ret_order would work correctly) std::vector args0, res; int w_arg = 0; @@ -371,15 +372,15 @@ bool Op::generate_code_step(Stack& stack) { args0.emplace_back(0); } if (f_sym->is_asm_function()) { - std::get(f_sym->body)->compile(stack.o, loc); // compile res := f (args0) + std::get(f_sym->body)->compile(stack.o, origin); // compile res := f (args0) } else { - std::get(f_sym->body)->compile(stack.o, res, args0, loc); // compile res := f (args0) + std::get(f_sym->body)->compile(stack.o, res, args0, origin); // compile res := f (args0) } } else { - stack.o << AsmOp::Custom(loc, f_sym->name + "() CALLDICT", (int)right.size(), (int)left.size()); + stack.o << AsmOp::Custom(origin, f_sym->name + "() CALLDICT", (int)right.size(), (int)left.size()); } stack.o.undent(); - stack.o << AsmOp::Custom({}, "}>"); + stack.o << AsmOp::Custom(nullptr, "}>"); stack.push_new_var(left.at(0)); return true; } @@ -411,7 +412,7 @@ bool Op::generate_code_step(Stack& stack) { if (is_last) { stack.assign_var(--i, x); } else { - stack.do_copy_var(loc, --i, x); + stack.do_copy_var(origin, --i, x); } } i = 0; @@ -431,15 +432,15 @@ bool Op::generate_code_step(Stack& stack) { for (var_idx_t x : right) { last.push_back(var_info[x] && var_info[x]->is_last()); } - stack.rearrange_top(loc, right, std::move(last)); + stack.rearrange_top(origin, right, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right.size(); tolk_assert(k >= 0); if (cl == _Tuple) { - stack.o << AsmOp::Tuple(loc, (int)right.size()); + stack.o << AsmOp::Tuple(origin, (int)right.size()); tolk_assert(left.size() == 1); } else { - stack.o << AsmOp::UnTuple(loc, (int)left.size()); + stack.o << AsmOp::UnTuple(origin, (int)left.size()); tolk_assert(right.size() == 1); } stack.s.resize(k); @@ -480,7 +481,7 @@ bool Op::generate_code_step(Stack& stack) { for (var_idx_t x : right1) { last.push_back(var_info[x] && var_info[x]->is_last()); } - stack.rearrange_top(loc, right1, std::move(last)); + stack.rearrange_top(origin, right1, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right1.size(); tolk_assert(k >= 0); @@ -489,12 +490,12 @@ bool Op::generate_code_step(Stack& stack) { } auto exec_callxargs = [&](int args, int ret) { if (args <= 15 && ret <= 15) { - stack.o << exec_arg2_op(loc, "CALLXARGS", args, ret, args + 1, ret); + stack.o << exec_arg2_op(origin, "CALLXARGS", args, ret, args + 1, ret); } else { tolk_assert(args <= 254 && ret <= 254); - stack.o << AsmOp::Const(loc, PSTRING() << args << " PUSHINT"); - stack.o << AsmOp::Const(loc, PSTRING() << ret << " PUSHINT"); - stack.o << AsmOp::Custom(loc, "CALLXVARARGS", args + 3, ret); + stack.o << AsmOp::Const(origin, PSTRING() << args << " PUSHINT"); + stack.o << AsmOp::Const(origin, PSTRING() << ret << " PUSHINT"); + stack.o << AsmOp::Custom(origin, "CALLXVARARGS", args + 3, ret); } }; if (cl == _CallInd) { @@ -506,24 +507,24 @@ bool Op::generate_code_step(Stack& stack) { res.emplace_back(i); } if (f_sym->is_asm_function()) { - std::get(f_sym->body)->compile(stack.o, loc); // compile res := f (args) + std::get(f_sym->body)->compile(stack.o, origin); // compile res := f (args) } else { if (arg_order_already_equals_asm()) { maybe_swap_builtin_args_to_compile(); } - std::get(f_sym->body)->compile(stack.o, res, args, loc); // compile res := f (args) + std::get(f_sym->body)->compile(stack.o, res, args, origin); // compile res := f (args) if (arg_order_already_equals_asm()) { maybe_swap_builtin_args_to_compile(); } } } else { if (f_sym->inline_mode == FunctionInlineMode::inlineViaFif || f_sym->inline_mode == FunctionInlineMode::inlineRef) { - stack.o << AsmOp::Custom(loc, f_sym->name + "() INLINECALLDICT", (int)right.size(), (int)left.size()); + stack.o << AsmOp::Custom(origin, f_sym->name + "() INLINECALLDICT", (int)right.size(), (int)left.size()); } else if (f_sym->is_code_function() && std::get(f_sym->body)->code->require_callxargs) { - stack.o << AsmOp::Custom(loc, f_sym->name + ("() PREPAREDICT"), 0, 2); + stack.o << AsmOp::Custom(origin, f_sym->name + ("() PREPAREDICT"), 0, 2); exec_callxargs((int)right.size() + 1, (int)left.size()); } else { - stack.o << AsmOp::Custom(loc, f_sym->name + "() CALLDICT", (int)right.size(), (int)left.size()); + stack.o << AsmOp::Custom(origin, f_sym->name + "() CALLDICT", (int)right.size(), (int)left.size()); } } stack.modified(); @@ -540,7 +541,7 @@ bool Op::generate_code_step(Stack& stack) { for (var_idx_t x : right) { last.push_back(var_info[x] && var_info[x]->is_last()); } - stack.rearrange_top(loc, right, std::move(last)); + stack.rearrange_top(origin, right, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right.size(); tolk_assert(k >= 0); @@ -548,10 +549,10 @@ bool Op::generate_code_step(Stack& stack) { tolk_assert(stack.s[k + i].first == right[i]); } if (right.size() > 1) { - stack.o << AsmOp::Tuple(loc, (int)right.size()); + stack.o << AsmOp::Tuple(origin, (int)right.size()); } if (!right.empty()) { - stack.o << AsmOp::Custom(loc, "$" + g_sym->name + " SETGLOB", 1, 0); + stack.o << AsmOp::Custom(origin, "$" + g_sym->name + " SETGLOB", 1, 0); stack.modified(); } stack.s.resize(k); @@ -565,7 +566,7 @@ bool Op::generate_code_step(Stack& stack) { stack.o.retalt_ = true; } var_idx_t x = left[0]; - stack.rearrange_top(loc, x, var_info[x] && var_info[x]->is_last()); + stack.rearrange_top(origin, x, var_info[x] && var_info[x]->is_last()); tolk_assert(stack[0] == x); stack.opt_show(); stack.s.pop_back(); @@ -575,19 +576,19 @@ bool Op::generate_code_step(Stack& stack) { Op* block_noreturn = is0 ? block0.get() : block1.get(); Op* block_other = is0 ? block1.get() : block0.get(); stack.mode &= ~Stack::_InlineFunc; - stack.o << AsmOp::Custom(loc, is0 ? "IF:<{" : "IFNOT:<{"); + stack.o << AsmOp::Custom(origin, is0 ? "IF:<{" : "IFNOT:<{"); stack.o.indent(); Stack stack_copy{stack}; block_noreturn->generate_code_all(stack_copy); stack.o.undent(); - stack.o << AsmOp::Custom({}, "}>ELSE<{"); + stack.o << AsmOp::Custom(nullptr, "}>ELSE<{"); stack.o.indent(); block_other->generate_code_all(stack); if (!block_other->noreturn()) { next->generate_code_all(stack); } stack.o.undent(); - stack.o << AsmOp::Custom({}, "}>"); + stack.o << AsmOp::Custom(nullptr, "}>"); return false; } if (block1->is_empty() || block0->is_empty()) { @@ -596,88 +597,88 @@ bool Op::generate_code_step(Stack& stack) { // if (left) block0; ... // if (!left) block1; ... if (block->noreturn()) { - stack.o << AsmOp::Custom(loc, is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); + stack.o << AsmOp::Custom(origin, is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); stack.o.indent(); Stack stack_copy{stack}; stack_copy.mode &= ~Stack::_InlineFunc; stack_copy.mode |= next->noreturn() ? 0 : Stack::_NeedRetAlt; block->generate_code_all(stack_copy); stack.o.undent(); - stack.o << AsmOp::Custom({}, "}>"); + stack.o << AsmOp::Custom(nullptr, "}>"); return true; } - stack.o << AsmOp::Custom(loc, is0 ? "IF:<{" : "IFNOT:<{"); + stack.o << AsmOp::Custom(origin, is0 ? "IF:<{" : "IFNOT:<{"); stack.o.indent(); Stack stack_copy{stack}, stack_target{stack}; stack_target.disable_output(); - stack_target.drop_vars_except(loc, next->var_info); + stack_target.drop_vars_except(origin, next->var_info); stack_copy.mode &= ~Stack::_InlineFunc; block->generate_code_all(stack_copy); - stack_copy.drop_vars_except(loc, var_info); + stack_copy.drop_vars_except(origin, var_info); stack_copy.opt_show(); if ((is0 && stack_copy == stack) || (!is0 && stack_copy.vars() == stack.vars())) { stack.o.undent(); - stack.o << AsmOp::Custom({}, "}>"); + stack.o << AsmOp::Custom(nullptr, "}>"); if (!is0) { stack.merge_const(stack_copy); } return true; } // stack_copy.drop_vars_except(next->var_info); - stack_copy.enforce_state(loc, stack_target.vars()); + stack_copy.enforce_state(origin, stack_target.vars()); stack_copy.opt_show(); if (stack_copy.vars() == stack.vars()) { stack.o.undent(); - stack.o << AsmOp::Custom({}, "}>"); + stack.o << AsmOp::Custom(nullptr, "}>"); stack.merge_const(stack_copy); return true; } stack.o.undent(); - stack.o << AsmOp::Custom({}, "}>ELSE<{"); + stack.o << AsmOp::Custom(nullptr, "}>ELSE<{"); stack.o.indent(); - stack.merge_state(loc, stack_copy); + stack.merge_state(origin, stack_copy); stack.opt_show(); stack.o.undent(); - stack.o << AsmOp::Custom({}, "}>"); + stack.o << AsmOp::Custom(nullptr, "}>"); return true; } if (block0->noreturn() || block1->noreturn()) { bool is0 = block0->noreturn(); Op* block_noreturn = is0 ? block0.get() : block1.get(); Op* block_other = is0 ? block1.get() : block0.get(); - stack.o << AsmOp::Custom(loc, is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); + stack.o << AsmOp::Custom(origin, is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); stack.o.indent(); Stack stack_copy{stack}; stack_copy.mode &= ~Stack::_InlineFunc; stack_copy.mode |= (block_other->noreturn() || next->noreturn()) ? 0 : Stack::_NeedRetAlt; block_noreturn->generate_code_all(stack_copy); stack.o.undent(); - stack.o << AsmOp::Custom({}, "}>"); + stack.o << AsmOp::Custom(nullptr, "}>"); block_other->generate_code_all(stack); return !block_other->noreturn(); } - stack.o << AsmOp::Custom(loc, "IF:<{"); + stack.o << AsmOp::Custom(origin, "IF:<{"); stack.o.indent(); Stack stack_copy{stack}; stack_copy.mode &= ~Stack::_InlineFunc; block0->generate_code_all(stack_copy); - stack_copy.drop_vars_except(loc, next->var_info); + stack_copy.drop_vars_except(origin, next->var_info); stack_copy.opt_show(); stack.o.undent(); - stack.o << AsmOp::Custom({}, "}>ELSE<{"); + stack.o << AsmOp::Custom(nullptr, "}>ELSE<{"); stack.o.indent(); stack.mode &= ~Stack::_InlineFunc; block1->generate_code_all(stack); - stack.merge_state(loc, stack_copy); + stack.merge_state(origin, stack_copy); stack.opt_show(); stack.o.undent(); - stack.o << AsmOp::Custom({}, "}>"); + stack.o << AsmOp::Custom(nullptr, "}>"); return true; } case _Repeat: { var_idx_t x = left[0]; //stack.drop_vars_except(block0->var_info, x); - stack.rearrange_top(loc, x, var_info[x] && var_info[x]->is_last()); + stack.rearrange_top(origin, x, var_info[x] && var_info[x]->is_last()); tolk_assert(stack[0] == x); stack.opt_show(); stack.s.pop_back(); @@ -686,7 +687,7 @@ bool Op::generate_code_step(Stack& stack) { stack.o.retalt_ = true; } if (true || !next->is_empty()) { - stack.o << AsmOp::Custom(loc, "REPEAT:<{"); + stack.o << AsmOp::Custom(origin, "REPEAT:<{"); stack.o.indent(); stack.forget_const(); if (block0->noreturn()) { @@ -700,47 +701,47 @@ bool Op::generate_code_step(Stack& stack) { stack.mode &= ~Stack::_InlineFunc; stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); - stack.enforce_state(loc, std::move(layout1)); + stack.enforce_state(origin, std::move(layout1)); stack.opt_show(); } stack.o.undent(); - stack.o << AsmOp::Custom({}, "}>"); + stack.o << AsmOp::Custom(nullptr, "}>"); return true; } else { - stack.o << AsmOp::Custom(loc, "REPEATEND"); + stack.o << AsmOp::Custom(origin, "REPEATEND"); stack.forget_const(); StackLayout layout1 = stack.vars(); block0->generate_code_all(stack); - stack.enforce_state(loc, std::move(layout1)); + stack.enforce_state(origin, std::move(layout1)); stack.opt_show(); return false; } } case _Again: { - stack.drop_vars_except(loc, block0->var_info); + stack.drop_vars_except(origin, block0->var_info); stack.opt_show(); if (block0->noreturn()) { stack.o.retalt_ = true; } if (!next->is_empty() || inline_func) { - stack.o << AsmOp::Custom(loc, "AGAIN:<{"); + stack.o << AsmOp::Custom(origin, "AGAIN:<{"); stack.o.indent(); stack.forget_const(); StackLayout layout1 = stack.vars(); stack.mode &= ~Stack::_InlineFunc; stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); - stack.enforce_state(loc, std::move(layout1)); + stack.enforce_state(origin, std::move(layout1)); stack.opt_show(); stack.o.undent(); - stack.o << AsmOp::Custom({}, "}>"); + stack.o << AsmOp::Custom(nullptr, "}>"); return true; } else { - stack.o << AsmOp::Custom(loc, "AGAINEND"); + stack.o << AsmOp::Custom(origin, "AGAINEND"); stack.forget_const(); StackLayout layout1 = stack.vars(); block0->generate_code_all(stack); - stack.enforce_state(loc, std::move(layout1)); + stack.enforce_state(origin, std::move(layout1)); stack.opt_show(); return false; } @@ -752,7 +753,7 @@ bool Op::generate_code_step(Stack& stack) { stack.o.retalt_ = true; } if (true || !next->is_empty()) { - stack.o << AsmOp::Custom(loc, "UNTIL:<{"); + stack.o << AsmOp::Custom(origin, "UNTIL:<{"); stack.o.indent(); stack.forget_const(); auto layout1 = stack.vars(); @@ -760,20 +761,20 @@ bool Op::generate_code_step(Stack& stack) { stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); layout1.push_back(left[0]); - stack.enforce_state(loc, std::move(layout1)); + stack.enforce_state(origin, std::move(layout1)); stack.opt_show(); stack.o.undent(); - stack.o << AsmOp::Custom({}, "}>"); + stack.o << AsmOp::Custom(nullptr, "}>"); stack.s.pop_back(); stack.modified(); return true; } else { - stack.o << AsmOp::Custom(loc, "UNTILEND"); + stack.o << AsmOp::Custom(origin, "UNTILEND"); stack.forget_const(); StackLayout layout1 = stack.vars(); block0->generate_code_all(stack); layout1.push_back(left[0]); - stack.enforce_state(loc, std::move(layout1)); + stack.enforce_state(origin, std::move(layout1)); stack.opt_show(); return false; } @@ -781,36 +782,36 @@ bool Op::generate_code_step(Stack& stack) { case _While: { // while (block0 | left) block1; ...next var_idx_t x = left[0]; - stack.drop_vars_except(loc, block0->var_info); + stack.drop_vars_except(origin, block0->var_info); stack.opt_show(); StackLayout layout1 = stack.vars(); bool next_empty = false && next->is_empty(); if (block0->noreturn()) { stack.o.retalt_ = true; } - stack.o << AsmOp::Custom(loc, "WHILE:<{"); + stack.o << AsmOp::Custom(origin, "WHILE:<{"); stack.o.indent(); stack.forget_const(); stack.mode &= ~Stack::_InlineFunc; stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); - stack.rearrange_top(loc, x, !next->var_info[x] && !block1->var_info[x]); + stack.rearrange_top(origin, x, !next->var_info[x] && !block1->var_info[x]); stack.opt_show(); stack.s.pop_back(); stack.modified(); stack.o.undent(); Stack stack_copy{stack}; - stack.o << AsmOp::Custom(loc, next_empty ? "}>DO:" : "}>DO<{"); + stack.o << AsmOp::Custom(origin, next_empty ? "}>DO:" : "}>DO<{"); if (!next_empty) { stack.o.indent(); } stack_copy.opt_show(); block1->generate_code_all(stack_copy); - stack_copy.enforce_state(loc, std::move(layout1)); + stack_copy.enforce_state(origin, std::move(layout1)); stack_copy.opt_show(); if (!next_empty) { stack.o.undent(); - stack.o << AsmOp::Custom({}, "}>"); + stack.o << AsmOp::Custom(nullptr, "}>"); return true; } else { return false; @@ -841,28 +842,28 @@ bool Op::generate_code_step(Stack& stack) { } catch_stack.push_new_var(left[0]); catch_stack.push_new_var(left[1]); - stack.rearrange_top(loc, catch_vars, catch_last); + stack.rearrange_top(origin, catch_vars, catch_last); stack.opt_show(); - stack.o << AsmOp::Custom(loc, "<{"); + stack.o << AsmOp::Custom(origin, "<{"); stack.o.indent(); if (block1->noreturn()) { catch_stack.mode |= Stack::_NeedRetAlt; } block1->generate_code_all(catch_stack); - catch_stack.drop_vars_except(loc, next->var_info); + catch_stack.drop_vars_except(origin, next->var_info); catch_stack.opt_show(); stack.o.undent(); - stack.o << AsmOp::Custom({}, "}>CONT"); - stack.o << AsmOp::Custom(loc, "0b10111010 SETCONTMANY"); + stack.o << AsmOp::Custom(nullptr, "}>CONT"); + stack.o << AsmOp::Custom(origin, "0b10111010 SETCONTMANY"); for (size_t begin = catch_vars.size(), end = begin; end > 0; end = begin) { begin = end >= block_size ? end - block_size : 0; - stack.o << AsmOp::Custom(loc, std::to_string(end - begin) + " PUSHINT"); - stack.o << AsmOp::Custom(loc, "-1 PUSHINT"); - stack.o << AsmOp::Custom(loc, "SETCONTVARARGS"); + stack.o << AsmOp::Custom(origin, std::to_string(end - begin) + " PUSHINT"); + stack.o << AsmOp::Custom(origin, "-1 PUSHINT"); + stack.o << AsmOp::Custom(origin, "SETCONTVARARGS"); } stack.s.erase(stack.s.end() - catch_vars.size(), stack.s.end()); stack.modified(); - stack.o << AsmOp::Custom(loc, "<{"); + stack.o << AsmOp::Custom(origin, "<{"); stack.o.indent(); if (block0->noreturn()) { stack.mode |= Stack::_NeedRetAlt; @@ -871,20 +872,19 @@ bool Op::generate_code_step(Stack& stack) { if (block0->noreturn()) { stack.s = std::move(catch_stack.s); } else if (!block1->noreturn()) { - stack.merge_state(loc, catch_stack); + stack.merge_state(origin, catch_stack); } stack.opt_show(); stack.o.undent(); - stack.o << AsmOp::Custom({}, "}>CONT"); - stack.o << AsmOp::Custom(loc, "c1 PUSH"); - stack.o << AsmOp::Custom(loc, "COMPOSALT"); - stack.o << AsmOp::Custom(loc, "SWAP"); - stack.o << AsmOp::Custom(loc, "TRY"); + stack.o << AsmOp::Custom(nullptr, "}>CONT"); + stack.o << AsmOp::Custom(origin, "c1 PUSH"); + stack.o << AsmOp::Custom(origin, "COMPOSALT"); + stack.o << AsmOp::Custom(origin, "SWAP"); + stack.o << AsmOp::Custom(origin, "TRY"); return true; } default: - std::cerr << "fatal: unknown operation \n"; - throw ParseError(loc, "unknown operation in generate_code()"); + fire(origin, "unknown operation in generate_code()"); } } @@ -906,7 +906,7 @@ void CodeBlob::generate_code(std::ostream& os, int mode, int indent) const { stack.push_new_var(x); } ops->generate_code_all(stack); - stack.apply_wrappers(fun_ref->loc, require_callxargs && (mode & Stack::_InlineAny) ? n_import_width : -1); + stack.apply_wrappers(fun_ref->ident_anchor, require_callxargs && (mode & Stack::_InlineAny) ? n_import_width : -1); if (G.settings.optimization_level >= 2) { optimize_code(out_list); } diff --git a/tolk/compilation-errors.cpp b/tolk/compilation-errors.cpp new file mode 100644 index 0000000000..070df57943 --- /dev/null +++ b/tolk/compilation-errors.cpp @@ -0,0 +1,112 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "compilation-errors.h" +#include "ast.h" + +namespace tolk { + +void on_assertion_failed(const char *description, const char *file_name, int line_number) { + std::string message = static_cast("Assertion failed at ") + file_name + ":" + std::to_string(line_number) + ": " + description; +#ifdef TOLK_DEBUG +#ifdef __arm64__ + // when developing, it's handy when the debugger stops on assertion failure (stacktraces and watches are available) + std::cerr << message << std::endl; + __builtin_debugtrap(); +#endif +#endif + throw Fatal(std::move(message)); +} + +GNU_ATTRIBUTE_NOINLINE +static void output_compiler_message( + std::ostream& os, + bool is_warning, + std::string_view in_function, + SrcRange range, + std::string_view message + ) { + std::string loc_text = range.stringify_start_location(true); + os << loc_text << ": " << (is_warning ? "warning: " : "error: "); + + if (message.find('\n') == std::string::npos) { + // just print a single-line message after "error:" + os << message << std::endl; + } else { + // print "location: line1 \n (spaces) line2 \n ..." + std::string loc_spaces(std::min(static_cast(loc_text.size()), 9), ' '); + size_t start = 0, end; + while ((end = message.find('\n', start)) != std::string::npos) { + if (start > 0) { + os << loc_spaces << " "; + } + os << message.substr(start, end - start) << std::endl; + start = end + 1; + } + if (start < message.size()) { + os << loc_spaces << " " << message.substr(start) << std::endl; + } + } + if (!in_function.empty()) { + os << std::endl << " // in function `" << in_function << "`" << std::endl; + } + range.output_underlined(os); +} + + +void fire(AnyV at, std::string message) { + throw ParseError(at->range, std::move(message)); +} + +void fire(SrcRange range, std::string message) { + throw ParseError(range, std::move(message)); +} + +void fire(FunctionPtr in_function, AnyV at, std::string message) { + std::string f = in_function ? in_function->as_human_readable() : ""; + throw ParseError(f, at->range, std::move(message)); +} + +void fire(FunctionPtr in_function, SrcRange range, std::string message) { + std::string f = in_function ? in_function->as_human_readable() : ""; + throw ParseError(f, range, std::move(message)); +} + +void ParseError::output_compilation_error(std::ostream& os) const { + output_compiler_message(os, false, in_function, range, message); +} + + +void compilation_warning(AnyV at, const std::string& message) { + output_compiler_message(std::cerr, true, "", at->range, message); +} + +void compilation_warning(SrcRange range, const std::string& message) { + output_compiler_message(std::cerr, true, "", range, message); +} + +void compilation_warning(FunctionPtr in_function, AnyV at, const std::string& message) { + std::string f = in_function ? in_function->as_human_readable() : ""; + output_compiler_message(std::cerr, true, f, at->range, message); +} + +void compilation_warning(FunctionPtr in_function, SrcRange range, const std::string& message) { + std::string f = in_function ? in_function->as_human_readable() : ""; + output_compiler_message(std::cerr, true, f, range, message); +} + + +} // namespace tolk diff --git a/tolk/compilation-errors.h b/tolk/compilation-errors.h new file mode 100644 index 0000000000..6c9406940e --- /dev/null +++ b/tolk/compilation-errors.h @@ -0,0 +1,83 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "fwd-declarations.h" +#include "src-file.h" +#include "platform-utils.h" + +namespace tolk { + +#define tolk_assert(expr) if(UNLIKELY(!(expr))) on_assertion_failed(#expr, __FILE__, __LINE__); + +GNU_ATTRIBUTE_COLD GNU_ATTRIBUTE_NORETURN +void on_assertion_failed(const char *description, const char *file_name, int line_number); + +// fire a general non-recoverable error, just a wrapper over `throw` +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +void fire(AnyV at, std::string message); +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +void fire(SrcRange range, std::string message); +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +void fire(FunctionPtr in_function, AnyV at, std::string message); +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +void fire(FunctionPtr in_function, SrcRange range, std::string message); + +void compilation_warning(AnyV at, const std::string& message); +void compilation_warning(SrcRange range, const std::string& message); +void compilation_warning(FunctionPtr function, AnyV at, const std::string& message); +void compilation_warning(FunctionPtr function, SrcRange range, const std::string& message); + +struct Fatal final : std::exception { + std::string message; + + explicit Fatal(std::string message) + : message(std::move(message)) {} + + const char* what() const noexcept override { + return message.c_str(); + } +}; + +struct ParseError final : std::exception { + std::string in_function; + SrcRange range; + std::string message; + + ParseError(SrcRange range, std::string message) + : range(range), message(std::move(message)) {} + ParseError(std::string in_function, SrcRange range, std::string message) + : in_function(std::move(in_function)), range(range), message(std::move(message)) {} + + const char* what() const noexcept override { + return message.c_str(); + } + void output_compilation_error(std::ostream& os) const; +}; + +struct UnexpectedASTNodeKind final : std::exception { + AnyV v_unexpected; + std::string message; + + explicit UnexpectedASTNodeKind(AnyV v_unexpected, const char* place_where); + + const char* what() const noexcept override { + return message.c_str(); + } +}; + +} // namespace tolk diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index 5952b8c060..e6ad6a21dc 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -16,7 +16,7 @@ */ #include "constant-evaluator.h" #include "ast.h" -#include "tolk.h" +#include "compilation-errors.h" #include "type-system.h" #include "openssl/digest.hpp" #include "crypto/common/util.h" @@ -26,10 +26,10 @@ namespace tolk { GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_const_string_required(SrcLocation loc, std::string_view f_name, std::string_view example_arg) { +static void fire_const_string_required(SrcRange range, std::string_view f_name, std::string_view example_arg) { std::string name = static_cast(f_name); std::string example = static_cast(example_arg); - throw ParseError(loc, "function `" + name + "` requires a constant string, like `" + name + "(\"" + example + "\")`"); + fire(range, "function `" + name + "` requires a constant string, like `" + name + "(\"" + example + "\")`"); } @@ -91,16 +91,16 @@ static bool parse_raw_address(std::string_view acc_string, int& workchain, ton:: return true; } -static void parse_any_std_address(std::string_view str, SrcLocation loc, unsigned char (*data)[3 + 8 + 256]) { +static void parse_any_std_address(std::string_view str, SrcRange range, unsigned char (*data)[3 + 8 + 256]) { ton::WorkchainId workchain; ton::StdSmcAddress addr; bool correct = (str.size() == 48 && parse_friendly_address(str.data(), workchain, addr)) || (str.size() != 48 && parse_raw_address(str, workchain, addr)); if (!correct) { - throw ParseError(loc, "invalid standard address"); + fire(range, "invalid standard address"); } if (workchain < -128 || workchain >= 128) { - throw ParseError(loc, "anycast addresses not supported"); + fire(range, "anycast addresses not supported"); } td::bitstring::bits_store_long_top(*data, 0, static_cast(4) << (64 - 3), 3); @@ -109,7 +109,7 @@ static void parse_any_std_address(std::string_view str, SrcLocation loc, unsigne } // internal helper: for `ton("0.05")`, parse string literal "0.05" to 50000000 -static td::RefInt256 parse_nanotons_as_floating_string(SrcLocation loc, std::string_view str) { +static td::RefInt256 parse_nanotons_as_floating_string(SrcRange range, std::string_view str) { bool is_negative = false; size_t i = 0; @@ -132,21 +132,21 @@ static td::RefInt256 parse_nanotons_as_floating_string(SrcLocation loc, std::str char c = str[i]; if (c == '.') { if (seen_dot) { - throw ParseError(loc, "argument is not a valid number like \"0.05\""); + fire(range, "argument is not a valid number like \"0.05\""); } seen_dot = true; } else if (c >= '0' && c <= '9') { if (!seen_dot) { integer_part = integer_part * 10 + (c - '0'); if (++integer_digits > 9) { - throw ParseError(loc, "argument is too big and leads to overflow"); + fire(range, "argument is too big and leads to overflow"); } } else if (fractional_digits < 9) { fractional_part = fractional_part * 10 + (c - '0'); fractional_digits++; } } else { - throw ParseError(loc, "argument is not a valid number like \"0.05\""); + fire(range, "argument is not a valid number like \"0.05\""); } } @@ -171,7 +171,7 @@ static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(Vtry_as(); if (!t_struct || !t_struct->struct_ref->opcode.exists()) { - throw ParseError(v->loc, "type `" + receiver->as_human_readable() + "` does not have a serialization prefix"); + fire(v, "type `" + receiver->as_human_readable() + "` does not have a serialization prefix"); } return td::make_refint(f_name.ends_with('x') ? t_struct->struct_ref->opcode.pack_prefix : t_struct->struct_ref->opcode.prefix_len); } @@ -189,16 +189,16 @@ static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(Vloc, f_name, f_name == "ton" ? "0.05" : "some_str"); + fire_const_string_required(v->range, f_name, f_name == "ton" ? "0.05" : "some_str"); } if (f_name == "ton") { - return parse_nanotons_as_floating_string(v_arg->loc, str); + return parse_nanotons_as_floating_string(v_arg->range, str); } if (f_name == "address") { // previously, postfix "..."a, but it returned `slice` (now returns `address`) unsigned char data[3 + 8 + 256]; // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; - parse_any_std_address(str, v_arg->loc, &data); + parse_any_std_address(str, v_arg->range, &data); return td::BitSlice{data, sizeof(data)}.to_hex(); } @@ -226,7 +226,7 @@ static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(Verror("invalid hex bitstring constant"); + fire(v_arg, "invalid hex bitstring constant"); } return static_cast(str); } @@ -234,10 +234,10 @@ static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(Verror("empty integer ascii-constant"); + fire(v_arg, "empty integer ascii-constant"); } if (intval.is_null()) { - v_arg->error("too long integer ascii-constant"); + fire(v_arg, "too long integer ascii-constant"); } return std::move(intval); } @@ -260,7 +260,7 @@ struct ConstantExpressionChecker { static void handle_reference(V v) { GlobalConstPtr const_ref = v->sym->try_as(); if (!const_ref) { - v->error("symbol `" + static_cast(v->get_name()) + "` is not a constant"); + fire(v, "symbol `" + static_cast(v->get_name()) + "` is not a constant"); } } @@ -270,7 +270,7 @@ struct ConstantExpressionChecker { // `ton(local_var)` is denied; it's validated not here, but when replacing its value with a calculated one return; } - v->error("not a constant expression"); + fire(v, "not a constant expression"); } // `const a = (1, 2)`, why not; or it's a default value of a field @@ -323,7 +323,7 @@ struct ConstantExpressionChecker { return; } } - v->error("not a constant expression"); + fire(v, "not a constant expression"); } // check that `2 + 3` is constant @@ -344,7 +344,7 @@ struct EnumMemberEvaluator { case tok_bitwise_not: return ~rhs; case tok_plus: return rhs; default: - fire(nullptr, v->loc, "unsupported operator in `enum` member value"); + fire(v, "unsupported operator in `enum` member value"); } } @@ -366,7 +366,7 @@ struct EnumMemberEvaluator { case tok_lshift: return lhs << static_cast(rhs->to_long()); case tok_rshift: return lhs >> static_cast(rhs->to_long()); default: - fire(nullptr, v->loc, "unsupported operator in `enum` member value"); + fire(v, "unsupported operator in `enum` member value"); } } @@ -374,7 +374,7 @@ struct EnumMemberEvaluator { static td::RefInt256 handle_reference(V v) { GlobalConstPtr const_ref = v->sym->try_as(); if (!const_ref) { - v->error("symbol `" + static_cast(v->get_name()) + "` is not a constant"); + fire(v, "symbol `" + static_cast(v->get_name()) + "` is not a constant"); } return visit(const_ref->init_value); } @@ -399,10 +399,10 @@ struct EnumMemberEvaluator { // `enum E { V = AnotherEnum.A }` not allowed (can be allowed in the future, // but remember to construct an initialization tree, since A may be auto-computed, and prevent recursion) if (v_dot->is_target_enum_member()) { - fire(nullptr, v->loc, "referencing other members in initializers is not allowed yet"); + fire(v, "referencing other members in initializers is not allowed yet"); } } - fire(nullptr, v->loc, "unsupported operation in `enum` member value"); + fire(v, "unsupported operation in `enum` member value"); } }; diff --git a/tolk/gen-entrypoints.cpp b/tolk/gen-entrypoints.cpp index 196b3dd162..a2a897a36b 100644 --- a/tolk/gen-entrypoints.cpp +++ b/tolk/gen-entrypoints.cpp @@ -35,7 +35,7 @@ namespace tolk { // implemented in ast-from-legacy.cpp -std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, SrcLocation loc, FunctionPtr f_inlined, AnyExprV self_obj, bool is_before_immediate_return, const std::vector>& vars_per_arg); +std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, AnyV origin, FunctionPtr f_inlined, AnyExprV self_obj, bool is_before_immediate_return, const std::vector>& vars_per_arg); // check for "modern" `fun onInternalMessage(in: InMessage)`, @@ -45,7 +45,7 @@ static bool is_modern_onInternalMessage(FunctionPtr f_onInternalMessage) { return f_onInternalMessage->get_num_params() == 1 && f_onInternalMessage->get_param(0).name == "in.body"; } -std::vector AuxData_OnInternalMessage_getField::generate_get_InMessage_field(CodeBlob& code, SrcLocation loc) const { +std::vector AuxData_OnInternalMessage_getField::generate_get_InMessage_field(CodeBlob& code, AnyV origin) const { if (field_name == "body") { // take `in.body` from a stack return f_onInternalMessage->find_param("in.body")->ir_idx; } @@ -63,17 +63,17 @@ std::vector AuxData_OnInternalMessage_getField::generate_get_InMessag else if (field_name == "valueExtra") idx = 8; tolk_assert(idx != -1); - std::vector ir_msgparam = code.create_tmp_var(TypeDataInt::create(), loc, field_name.data()); - code.emplace_back(loc, Op::_Call, ir_msgparam, std::vector{code.create_int(loc, idx, "(param-idx)")}, lookup_function("__InMessage.getInMsgParam")); + std::vector ir_msgparam = code.create_tmp_var(TypeDataInt::create(), origin, field_name.data()); + code.emplace_back(origin, Op::_Call, ir_msgparam, std::vector{code.create_int(origin, idx, "(param-idx)")}, lookup_function("__InMessage.getInMsgParam")); if (field_name == "originalForwardFee") { - code.emplace_back(loc, Op::_Call, ir_msgparam, std::vector{ir_msgparam[0], code.create_int(loc, 0, "(basechain)")}, lookup_function("__InMessage.originalForwardFee")); + code.emplace_back(origin, Op::_Call, ir_msgparam, std::vector{ir_msgparam[0], code.create_int(origin, 0, "(basechain)")}, lookup_function("__InMessage.originalForwardFee")); } return ir_msgparam; } -void handle_onInternalMessage_codegen_start(FunctionPtr f_onInternalMessage, const std::vector& rvect_params, CodeBlob& code, SrcLocation loc) { +void handle_onInternalMessage_codegen_start(FunctionPtr f_onInternalMessage, const std::vector& rvect_params, CodeBlob& code, AnyV origin) { // ignore FunC-style `onInternalMessage(msgCell, msgBody)` if (!is_modern_onInternalMessage(f_onInternalMessage)) { return; @@ -87,32 +87,32 @@ void handle_onInternalMessage_codegen_start(FunctionPtr f_onInternalMessage, con FunctionPtr f_onBouncedMessage = sym ? sym->try_as() : nullptr; AuxData_OnInternalMessage_getField get_isBounced(f_onInternalMessage, "isBounced"); - std::vector ir_isBounced = get_isBounced.generate_get_InMessage_field(code, loc); + std::vector ir_isBounced = get_isBounced.generate_get_InMessage_field(code, origin); if (f_onBouncedMessage) { // generate: `if (isBounced) { onBouncedMessage(); return; } tolk_assert(f_onBouncedMessage->inferred_return_type->get_width_on_stack() == 0); - Op& if_isBounced = code.emplace_back(loc, Op::_If, ir_isBounced); + Op& if_isBounced = code.emplace_back(origin, Op::_If, ir_isBounced); { code.push_set_cur(if_isBounced.block0); std::vector ir_bodySlice(rvect_params.end() - 1, rvect_params.end()); if (f_onBouncedMessage->is_inlined_in_place()) { - gen_inline_fun_call_in_place(code, TypeDataVoid::create(), loc, f_onBouncedMessage, nullptr, true, {ir_bodySlice}); + gen_inline_fun_call_in_place(code, TypeDataVoid::create(), origin, f_onBouncedMessage, nullptr, true, {ir_bodySlice}); } else { - Op& op_call = code.emplace_back(loc, Op::_Call, std::vector{}, ir_bodySlice, f_onBouncedMessage); + Op& op_call = code.emplace_back(origin, Op::_Call, std::vector{}, ir_bodySlice, f_onBouncedMessage); op_call.set_impure_flag(); } - code.emplace_back(loc, Op::_Return, std::vector{}); - code.close_pop_cur(loc); + code.emplace_back(origin, Op::_Return, std::vector{}); + code.close_pop_cur(origin); } { code.push_set_cur(if_isBounced.block1); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } } else { // generate: `assert (!isBounced) throw 0` - std::vector args_throw0if = { code.create_int(loc, 0, "(exit-0)"), ir_isBounced[0], code.create_int(loc, 1, "") }; - Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_throw0if), lookup_function("__throw_if_unless")); + std::vector args_throw0if = { code.create_int(origin, 0, "(exit-0)"), ir_isBounced[0], code.create_int(origin, 1, "") }; + Op& op_assert = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_throw0if), lookup_function("__throw_if_unless")); op_assert.set_impure_flag(); } } diff --git a/tolk/gen-entrypoints.h b/tolk/gen-entrypoints.h index 90f62b9bc2..5a47346066 100644 --- a/tolk/gen-entrypoints.h +++ b/tolk/gen-entrypoints.h @@ -22,7 +22,7 @@ namespace tolk { -void handle_onInternalMessage_codegen_start(FunctionPtr f_onInternalMessage, const std::vector& rvect_params, CodeBlob& code, SrcLocation loc); -std::vector generate_get_requested_field_parsing_on_demand(const AuxData_OnInternalMessage_getField* aux_data, CodeBlob& code, SrcLocation loc); +void handle_onInternalMessage_codegen_start(FunctionPtr f_onInternalMessage, const std::vector& rvect_params, CodeBlob& code, AnyV origin); +std::vector generate_get_requested_field_parsing_on_demand(const AuxData_OnInternalMessage_getField* aux_data, CodeBlob& code, AnyV origin); } // namespace tolk diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index 4f76bc86c4..74928a2e14 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -15,9 +15,9 @@ along with TON Blockchain Library. If not, see . */ #include "generics-helpers.h" -#include "tolk.h" #include "ast.h" #include "ast-replicator.h" +#include "compilation-errors.h" #include "type-system.h" #include "compiler-state.h" #include "pipeline.h" @@ -292,12 +292,12 @@ TypePtr GenericSubstitutionsDeducing::auto_deduce_from_argument(TypePtr param_ty return replace_genericT_with_deduced(param_type, &deducedTs); } -TypePtr GenericSubstitutionsDeducing::auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type) { +TypePtr GenericSubstitutionsDeducing::auto_deduce_from_argument(FunctionPtr cur_f, SrcRange range, TypePtr param_type, TypePtr arg_type) { std::string_view unknown_nameT; consider_next_condition(param_type, arg_type); param_type = replace_genericT_with_deduced(param_type, &deducedTs, true, &unknown_nameT); if (param_type->has_genericT_inside()) { - fire_error_can_not_deduce(cur_f, loc, unknown_nameT); + fire_can_not_deduce(cur_f, range, unknown_nameT); } return param_type; } @@ -315,11 +315,12 @@ void GenericSubstitutionsDeducing::apply_defaults_from_declaration() { deducedTs.rewrite_missing_with_defaults(); } -void GenericSubstitutionsDeducing::fire_error_can_not_deduce(FunctionPtr cur_f, SrcLocation loc, std::string_view nameT) const { +// todo pass at, not ra +void GenericSubstitutionsDeducing::fire_can_not_deduce(FunctionPtr cur_f, SrcRange range, std::string_view nameT) const { if (fun_ref) { - fire(cur_f, loc, "can not deduce " + static_cast(nameT) + " for generic function `" + fun_ref->as_human_readable() + "`; instantiate it manually with `" + fun_ref->name + "<...>()`"); + fire(cur_f, range, "can not deduce " + static_cast(nameT) + " for generic function `" + fun_ref->as_human_readable() + "`; instantiate it manually with `" + fun_ref->name + "<...>()`"); } else { - fire(cur_f, loc, "can not deduce " + static_cast(nameT) + " for generic struct `" + struct_ref->as_human_readable() + "`; instantiate it manually with `" + struct_ref->name + "<...>`"); + fire(cur_f, range, "can not deduce " + static_cast(nameT) + " for generic struct `" + struct_ref->as_human_readable() + "`; instantiate it manually with `" + struct_ref->name + "<...>`"); } } @@ -417,16 +418,16 @@ FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstituti // built-in functions don't have AST to clone, types of parameters don't exist in AST, etc. // nevertheless, for outer code to follow a single algorithm, // when calling `debugPrint(x)`, we clone it as "debugPrint", replace types, and insert into symtable - if (fun_ref->is_builtin_function()) { + if (fun_ref->is_builtin()) { std::vector new_parameters; new_parameters.reserve(fun_ref->get_num_params()); for (const LocalVarData& orig_p : fun_ref->parameters) { TypePtr new_param_type = replace_genericT_with_deduced(orig_p.declared_type, allocatedTs); - new_parameters.emplace_back(orig_p.name, orig_p.loc, new_param_type, orig_p.default_value, orig_p.flags, orig_p.param_idx); + new_parameters.emplace_back(orig_p.name, nullptr, new_param_type, orig_p.default_value, orig_p.flags, orig_p.param_idx); } TypePtr new_return_type = replace_genericT_with_deduced(fun_ref->declared_return_type, allocatedTs); TypePtr new_receiver_type = replace_genericT_with_deduced(fun_ref->receiver_type, allocatedTs); - FunctionData* new_fun_ref = new FunctionData(new_name, fun_ref->loc, fun_ref->method_name, new_receiver_type, new_return_type, std::move(new_parameters), fun_ref->flags, fun_ref->inline_mode, nullptr, allocatedTs, fun_ref->body, fun_ref->ast_root); + FunctionData* new_fun_ref = new FunctionData(new_name, nullptr, fun_ref->method_name, new_receiver_type, new_return_type, std::move(new_parameters), fun_ref->flags, fun_ref->inline_mode, nullptr, allocatedTs, fun_ref->body, fun_ref->ast_root); new_fun_ref->arg_order = fun_ref->arg_order; new_fun_ref->ret_order = fun_ref->ret_order; new_fun_ref->base_fun_ref = fun_ref; @@ -466,7 +467,7 @@ StructPtr instantiate_generic_struct(StructPtr struct_ref, GenericsSubstitutions const GenericsSubstitutions* allocatedTs = new GenericsSubstitutions(std::move(substitutedTs)); V orig_root = struct_ref->ast_root->as(); - V new_name_ident = createV(orig_root->get_identifier()->loc, new_name); + V new_name_ident = createV(orig_root->get_identifier()->range, new_name); V new_root = ASTReplicator::clone_struct_ast(orig_root, new_name_ident); StructPtr new_struct_ref = pipeline_register_instantiated_generic_struct(struct_ref, new_root, std::move(new_name), allocatedTs); @@ -489,7 +490,7 @@ AliasDefPtr instantiate_generic_alias(AliasDefPtr alias_ref, GenericsSubstitutio const GenericsSubstitutions* allocatedTs = new GenericsSubstitutions(std::move(substitutedTs)); V orig_root = alias_ref->ast_root->as(); - V new_name_ident = createV(orig_root->get_identifier()->loc, new_name); + V new_name_ident = createV(orig_root->get_identifier()->range, new_name); V new_root = ASTReplicator::clone_type_alias_ast(orig_root, new_name_ident); AliasDefPtr new_alias_ref = pipeline_register_instantiated_generic_alias(alias_ref, new_root, std::move(new_name), allocatedTs); diff --git a/tolk/generics-helpers.h b/tolk/generics-helpers.h index 4cec94cd28..06bec2511b 100644 --- a/tolk/generics-helpers.h +++ b/tolk/generics-helpers.h @@ -101,10 +101,10 @@ class GenericSubstitutionsDeducing { TypePtr replace_Ts_with_currently_deduced(TypePtr orig) const; TypePtr auto_deduce_from_argument(TypePtr param_type, TypePtr arg_type); - TypePtr auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type); + TypePtr auto_deduce_from_argument(FunctionPtr cur_f, SrcRange range, TypePtr param_type, TypePtr arg_type); std::string_view get_first_not_deduced_nameT() const; void apply_defaults_from_declaration(); - void fire_error_can_not_deduce(FunctionPtr cur_f, SrcLocation loc, std::string_view nameT) const; + void fire_can_not_deduce(FunctionPtr cur_f, SrcRange range, std::string_view nameT) const; GenericsSubstitutions&& flush() { return std::move(deducedTs); diff --git a/tolk/lazy-helpers.cpp b/tolk/lazy-helpers.cpp index 22ef5fff27..f87c7bf1ba 100644 --- a/tolk/lazy-helpers.cpp +++ b/tolk/lazy-helpers.cpp @@ -15,8 +15,9 @@ along with TON Blockchain Library. If not, see . */ #include "lazy-helpers.h" -#include "tolk.h" #include "type-system.h" +#include "symtable.h" +#include "compilation-errors.h" namespace tolk { diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index c349a91ae3..14d0440a14 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -15,6 +15,7 @@ along with TON Blockchain Library. If not, see . */ #include "lexer.h" +#include "compilation-errors.h" #include #include #include @@ -181,9 +182,9 @@ struct ChunkString final : ChunkLexerBase { if (lex->char_at() != '"') { lex->error("string extends past end of line"); } - - std::string_view str_val(str_begin + 1, lex->c_str() - str_begin - 1); lex->skip_chars(1); + + std::string_view str_val(str_begin, lex->c_str() - str_begin); // with surrounding quotes lex->add_token(tok_string_const, str_val); return true; @@ -205,9 +206,9 @@ struct ChunkMultilineString final : ChunkLexerBase { if (lex->is_eof()) { lex->error("string extends past end of file"); } - - std::string_view str_val(str_begin + 3, lex->c_str() - str_begin - 3); lex->skip_chars(3); + + std::string_view str_val(str_begin, lex->c_str() - str_begin); // with surrounding quotes lex->add_token(tok_string_const, str_val); return true; } @@ -553,11 +554,10 @@ LexingTrie TolkLanguageGrammar::trie; // Lexer::Lexer(const SrcFile* file) - : file(file) + : file_id(file->file_id) , p_start(file->text.data()) , p_end(p_start + file->text.size()) - , p_next(p_start) - , location(file) { + , p_next(p_start) { next(); } @@ -585,28 +585,29 @@ void Lexer::next_special(TokenType parse_next_as, const char* str_expected) { } Lexer::SavedPositionForLookahead Lexer::save_parsing_position() const { - return {p_next, cur_token_idx, cur_token, location}; + return {p_next, cur_token_idx, cur_token_offset, cur_token}; } void Lexer::restore_position(SavedPositionForLookahead saved) { p_next = saved.p_next; cur_token_idx = last_token_idx = saved.cur_token_idx; + cur_token_offset = saved.cur_token_offset; cur_token = saved.cur_token; - location = saved.loc; } void Lexer::hack_replace_rshift_with_one_triangle() { // overcome the `>>` problem when parsing generics, leave only `>` here, see comments at usage assert(cur_token.type == tok_rshift); cur_token = Token(tok_gt, ">"); + cur_token_offset++; } void Lexer::error(const std::string& err_msg) const { - throw ParseError(cur_location(), err_msg); + fire(cur_range(), err_msg); } void Lexer::unexpected(const char* str_expected) const { - throw ParseError(cur_location(), "expected " + std::string(str_expected) + ", got `" + std::string(cur_str()) + "`"); + fire(cur_range(), "expected " + std::string(str_expected) + ", got `" + std::string(cur_str()) + "`"); } void lexer_init() { diff --git a/tolk/lexer.h b/tolk/lexer.h index b92b2a1c29..8656da6095 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -154,12 +154,12 @@ class Lexer { int cur_token_idx = -1; Token cur_token; // = tokens_circularbuf[cur_token_idx & 7] - const SrcFile* file; + int file_id; const char *p_start, *p_end, *p_next; - SrcLocation location; + int cur_token_offset = 0; void update_location() { - location.char_offset = static_cast(p_next - p_start); + cur_token_offset = static_cast(p_next - p_start); } public: @@ -167,8 +167,8 @@ class Lexer { struct SavedPositionForLookahead { const char* p_next = nullptr; int cur_token_idx = 0; + int cur_token_offset = 0; Token cur_token; - SrcLocation loc; }; explicit Lexer(const SrcFile* file); @@ -208,8 +208,8 @@ class Lexer { TokenType tok() const { return cur_token.type; } std::string_view cur_str() const { return cur_token.str_val; } - SrcLocation cur_location() const { return location; } - const SrcFile* cur_file() const { return file; } + SrcRange cur_range() const { return SrcRange::span(file_id, cur_token_offset, static_cast(cur_token.str_val.size())); } + SrcRange range_start() const { return SrcRange::unclosed_range(file_id, cur_token_offset); } void next(); void next_special(TokenType parse_next_as, const char* str_expected); diff --git a/tolk/maps-kv-api.cpp b/tolk/maps-kv-api.cpp index 6fc0112db5..950290edee 100644 --- a/tolk/maps-kv-api.cpp +++ b/tolk/maps-kv-api.cpp @@ -161,7 +161,7 @@ bool check_mapKV_TValue_is_valid(TypePtr TValue, std::string& because_msg) { // an internal helper, having TKey and TValue, generate IR variables passed to __dict.* built-in functions class DictKeyValue { CodeBlob& code; - SrcLocation loc; + AnyV origin; DictKeyKind key_kind; int key_len; @@ -173,15 +173,15 @@ class DictKeyValue { public: var_idx_t ir_key_kind() const { - return code.create_int(loc, static_cast(key_kind), "(key-kind)"); + return code.create_int(origin, static_cast(key_kind), "(key-kind)"); } var_idx_t ir_value_kind() const { - return code.create_int(loc, static_cast(value_kind), "(value-kind)"); + return code.create_int(origin, static_cast(value_kind), "(value-kind)"); } var_idx_t ir_key_len() const { - return code.create_int(loc, key_len, "(key-len)"); + return code.create_int(origin, key_len, "(key-len)"); } var_idx_t ir_key_val() const { @@ -194,9 +194,9 @@ class DictKeyValue { return value_irv; } - DictKeyValue(CodeBlob& code, SrcLocation loc, TypePtr TKey, const std::vector* exact_key, TypePtr TValue, const std::vector* exact_value, bool allow_REF_TValue = false) + DictKeyValue(CodeBlob& code, AnyV origin, TypePtr TKey, const std::vector* exact_key, TypePtr TValue, const std::vector* exact_value, bool allow_REF_TValue = false) : code(code) - , loc(loc) { + , origin(origin) { if (int i_bits = is_TKey_TVM_int(TKey)) { key_kind = DictKeyKind::IntKey; key_len = i_bits; @@ -225,12 +225,12 @@ class DictKeyValue { key_len = pack_size.max_bits; if (exact_key != nullptr) { - std::vector ir_builder = code.create_tmp_var(TypeDataBuilder::create(), loc, "(map-keyB)"); - code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); - PackContext ctx(code, loc, ir_builder, create_default_PackOptions(code, loc)); + std::vector ir_builder = code.create_tmp_var(TypeDataBuilder::create(), origin, "(map-keyB)"); + code.emplace_back(origin, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); + PackContext ctx(code, origin, ir_builder, create_default_PackOptions(code, origin)); ctx.generate_pack_any(TKey, std::vector(*exact_key)); - std::vector ir_slice = code.create_tmp_var(TypeDataSlice::create(), loc, "(map-key)"); - code.emplace_back(loc, Op::_Call, ir_slice, ir_builder, lookup_function("builder.toSlice")); + std::vector ir_slice = code.create_tmp_var(TypeDataSlice::create(), origin, "(map-key)"); + code.emplace_back(origin, Op::_Call, ir_slice, ir_builder, lookup_function("builder.toSlice")); key_irv = ir_slice[0]; } } @@ -253,9 +253,9 @@ class DictKeyValue { value_kind = DictValueKind::BuilderValue; if (exact_value != nullptr) { - std::vector ir_builder = code.create_tmp_var(TypeDataBuilder::create(), loc, "(valueB)"); - code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); - PackContext ctx(code, loc, ir_builder, create_default_PackOptions(code, loc)); + std::vector ir_builder = code.create_tmp_var(TypeDataBuilder::create(), origin, "(valueB)"); + code.emplace_back(origin, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); + PackContext ctx(code, origin, ir_builder, create_default_PackOptions(code, origin)); ctx.generate_pack_any(TValue, std::vector(*exact_value)); value_irv = ir_builder[0]; } @@ -269,24 +269,24 @@ class DictKeyValue { // so, in practice, we don't need any transformations from a TVM result, // but when K is complex (like struct Point), TVM instructions return a slice, which is needed to be unpacked to K GNU_ATTRIBUTE_NOINLINE -static std::vector construct_MapEntry_with_non_trivial_key(CodeBlob& code, SrcLocation loc, std::vector&& ir_entry, TypePtr TKey) { +static std::vector construct_MapEntry_with_non_trivial_key(CodeBlob& code, AnyV origin, std::vector&& ir_entry, TypePtr TKey) { tolk_assert(ir_entry.size() == 3); // slice value, slice key, isFound - std::vector ir_key = code.create_tmp_var(TKey, loc, "(entry-key)"); - Op& if_found = code.emplace_back(loc, Op::_If, std::vector{ir_entry[2]}); + std::vector ir_key = code.create_tmp_var(TKey, origin, "(entry-key)"); + Op& if_found = code.emplace_back(origin, Op::_If, std::vector{ir_entry[2]}); { code.push_set_cur(if_found.block0); - UnpackContext ctx(code, loc, std::vector(ir_entry.begin() + 1, ir_entry.begin() + 2), create_default_UnpackOptions(code, loc)); + UnpackContext ctx(code, origin, std::vector(ir_entry.begin() + 1, ir_entry.begin() + 2), create_default_UnpackOptions(code, origin)); std::vector ir_unpacked_key = ctx.generate_unpack_any(TKey); - code.emplace_back(loc, Op::_Let, ir_key, std::move(ir_unpacked_key)); - code.close_pop_cur(loc); + code.emplace_back(origin, Op::_Let, ir_key, std::move(ir_unpacked_key)); + code.close_pop_cur(origin); } { code.push_set_cur(if_found.block1); for (var_idx_t ith_null : ir_key) { - code.emplace_back(loc, Op::_Let, std::vector{ith_null}, std::vector{ir_entry[1]}); + code.emplace_back(origin, Op::_Let, std::vector{ith_null}, std::vector{ir_entry[1]}); } - code.close_pop_cur(loc); + code.close_pop_cur(origin); } std::vector ir_result; @@ -297,14 +297,14 @@ static std::vector construct_MapEntry_with_non_trivial_key(CodeBlob& return ir_result; } -static std::vector create_ir_MapEntry(CodeBlob& code, SrcLocation loc) { - return code.create_tmp_var(TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create(), TypeDataInt::create()}), loc, "(entry)"); +static std::vector create_ir_MapEntry(CodeBlob& code, AnyV origin) { + return code.create_tmp_var(TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create(), TypeDataInt::create()}), origin, "(entry)"); } // see a comment above construct_MapEntry_with_non_trivial_key() -static std::vector finalize_ir_MapEntry(CodeBlob& code, SrcLocation loc, std::vector&& ir_entry, TypePtr TKey) { +static std::vector finalize_ir_MapEntry(CodeBlob& code, AnyV origin, std::vector&& ir_entry, TypePtr TKey) { if (!is_TKey_TVM_int(TKey) && !is_TKey_TVM_uint(TKey) && !is_TKey_TVM_slice(TKey)) { - ir_entry = construct_MapEntry_with_non_trivial_key(code, loc, std::move(ir_entry), TKey); + ir_entry = construct_MapEntry_with_non_trivial_key(code, origin, std::move(ir_entry), TKey); } return ir_entry; } @@ -335,151 +335,151 @@ static std::string choose_dict_op(std::string_view op_slice, VarDescr& var_with_ } // "empty map" is just NULL in TVM; it's extracted as a built-in to check for K/V correctness in advance -AsmOp compile_createEmptyMap(std::vector& res, std::vector& args, SrcLocation loc) { +AsmOp compile_createEmptyMap(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 1 && args.empty()); - return AsmOp::Custom(loc, "NEWDICT", 0, 1); + return AsmOp::Custom(origin, "NEWDICT", 0, 1); } // "convert dict to map" is just NOP; it's extracted as a built-in to allow non-1 width K/V -AsmOp compile_createMapFromLowLevelDict(std::vector& res, std::vector& args, SrcLocation loc) { +AsmOp compile_createMapFromLowLevelDict(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 1 && args.size() == 1); - return AsmOp::Parse(loc, "NOP"); + return AsmOp::Parse(origin, "NOP"); } // DICTGET: k D n => (x −1) OR (0); + NULLSWAPIFNOT => (x -1) OR (null 0) -AsmOp compile_dict_get(std::vector& res, std::vector& args, SrcLocation loc) { +AsmOp compile_dict_get(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 2 && args.size() == 2+3); std::string op = choose_dict_op("DICTGET", args[0], args[1]); - return AsmOp::Custom(loc, op + " NULLSWAPIFNOT", 3, 2); + return AsmOp::Custom(origin, op + " NULLSWAPIFNOT", 3, 2); } // DICTGET: k D n => (x −1) OR (0) // but since it's MUST get, it's wrapped to assert(found), so assume x (value) will exist on a stack -AsmOp compile_dict_mustGet(std::vector& res, std::vector& args, SrcLocation loc) { +AsmOp compile_dict_mustGet(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 2 && args.size() == 2+3); std::string op = choose_dict_op("DICTGET", args[0], args[1]); - return AsmOp::Custom(loc, op, 3, 2); + return AsmOp::Custom(origin, op, 3, 2); } // DICTMIN: D n => (x k −1) OR (0); + NULLSWAPIFNOT2 => (x k -1) OR (null null 0) -AsmOp compile_dict_getMin(std::vector& res, std::vector& args, SrcLocation loc) { +AsmOp compile_dict_getMin(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 3 && args.size() == 2+2); std::string op = choose_dict_op("DICTMIN", args[0], args[1]); - return AsmOp::Custom(loc, op + " NULLSWAPIFNOT2", 2, 3); + return AsmOp::Custom(origin, op + " NULLSWAPIFNOT2", 2, 3); } // DICTMAX: D n => (x k −1) OR (0); + NULLSWAPIFNOT2 => (x k -1) OR (null null 0) -AsmOp compile_dict_getMax(std::vector& res, std::vector& args, SrcLocation loc) { +AsmOp compile_dict_getMax(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 3 && args.size() == 2+2); std::string op = choose_dict_op("DICTMAX", args[0], args[1]); - return AsmOp::Custom(loc, op + " NULLSWAPIFNOT2", 2, 3); + return AsmOp::Custom(origin, op + " NULLSWAPIFNOT2", 2, 3); } // DICTGETNEXT: k D n => (x k −1) OR (0); + NULLSWAPIFNOT2 => (x k -1) OR (null null 0) -AsmOp compile_dict_getNext(std::vector& res, std::vector& args, SrcLocation loc) { +AsmOp compile_dict_getNext(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 3 && args.size() == 2+3); std::string op = choose_dict_op("DICTGETNEXT", args[0], args[1]); - return AsmOp::Custom(loc, op + " NULLSWAPIFNOT2", 3, 3); + return AsmOp::Custom(origin, op + " NULLSWAPIFNOT2", 3, 3); } // DICTGETNEXTEQ: k D n => (x k −1) OR (0); + NULLSWAPIFNOT2 => (x k -1) OR (null null 0) -AsmOp compile_dict_getNextEq(std::vector& res, std::vector& args, SrcLocation loc) { +AsmOp compile_dict_getNextEq(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 3 && args.size() == 2+3); std::string op = choose_dict_op("DICTGETNEXTEQ", args[0], args[1]); - return AsmOp::Custom(loc, op + " NULLSWAPIFNOT2", 3, 3); + return AsmOp::Custom(origin, op + " NULLSWAPIFNOT2", 3, 3); } // DICTGETPREV: k D n => (x k −1) OR (0); + NULLSWAPIFNOT2 => (x k -1) OR (null null 0) -AsmOp compile_dict_getPrev(std::vector& res, std::vector& args, SrcLocation loc) { +AsmOp compile_dict_getPrev(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 3 && args.size() == 2+3); std::string op = choose_dict_op("DICTGETPREV", args[0], args[1]); - return AsmOp::Custom(loc, op + " NULLSWAPIFNOT2", 3, 3); + return AsmOp::Custom(origin, op + " NULLSWAPIFNOT2", 3, 3); } // DICTGETPREVEQ: k D n => (x k −1) OR (0); + NULLSWAPIFNOT2 => (x k -1) OR (null null 0) -AsmOp compile_dict_getPrevEq(std::vector& res, std::vector& args, SrcLocation loc) { +AsmOp compile_dict_getPrevEq(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 3 && args.size() == 2+3); std::string op = choose_dict_op("DICTGETPREVEQ", args[0], args[1]); - return AsmOp::Custom(loc, op + " NULLSWAPIFNOT2", 3, 3); + return AsmOp::Custom(origin, op + " NULLSWAPIFNOT2", 3, 3); } // DICTSET: x k D n => D' -AsmOp compile_dict_set(std::vector& res, std::vector& args, SrcLocation loc) { +AsmOp compile_dict_set(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 1 && args.size() == 2+4); std::string op = choose_dict_op("DICTSET", args[0], args[1]); - return AsmOp::Custom(loc, op, 4, 1); + return AsmOp::Custom(origin, op, 4, 1); } // DICTSETGET: x k D n => (D' y −1) or (D' 0); + NULLSWAPIFNOT => (D' y -1) OR (D' null 0) -AsmOp compile_dict_setGet(std::vector& res, std::vector& args, SrcLocation loc) { +AsmOp compile_dict_setGet(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 3 && args.size() == 2+4); std::string op = choose_dict_op("DICTSETGET", args[0], args[1]); - return AsmOp::Custom(loc, op + " NULLSWAPIFNOT", 4, 3); + return AsmOp::Custom(origin, op + " NULLSWAPIFNOT", 4, 3); } // DICTREPLACE: x k D n => (D' -1) OR (D 0) -AsmOp compile_dict_replace(std::vector& res, std::vector& args, SrcLocation loc) { +AsmOp compile_dict_replace(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 2 && args.size() == 2+4); std::string op = choose_dict_op("DICTREPLACE", args[0], args[1]); - return AsmOp::Custom(loc, op, 4, 2); + return AsmOp::Custom(origin, op, 4, 2); } // DICTREPLACEGET: x k D n => (D' y -1) OR (D 0); + NULLSWAPIFNOT => (D' y -1) OR (D null 0) -AsmOp compile_dict_replaceGet(std::vector& res, std::vector& args, SrcLocation loc) { +AsmOp compile_dict_replaceGet(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 3 && args.size() == 2+4); std::string op = choose_dict_op("DICTREPLACEGET", args[0], args[1]); - return AsmOp::Custom(loc, op + " NULLSWAPIFNOT", 4, 3); + return AsmOp::Custom(origin, op + " NULLSWAPIFNOT", 4, 3); } // DICTADD: x k D n => (D' -1) OR (D 0) -AsmOp compile_dict_add(std::vector& res, std::vector& args, SrcLocation loc) { +AsmOp compile_dict_add(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 2 && args.size() == 2+4); std::string op = choose_dict_op("DICTADD", args[0], args[1]); - return AsmOp::Custom(loc, op, 4, 2); + return AsmOp::Custom(origin, op, 4, 2); } // DICTADDGET: x k D n => (D' -1) OR (D y 0); + NULLSWAPIF + NOT => (D' null 0) OR (D y -1) (from "isAdded" to "isFound") -AsmOp compile_dict_addGet(std::vector& res, std::vector& args, SrcLocation loc) { +AsmOp compile_dict_addGet(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 3 && args.size() == 2+4); std::string op = choose_dict_op("DICTADDGET", args[0], args[1]); - return AsmOp::Custom(loc, op + " NULLSWAPIF" + " NOT", 4, 3); + return AsmOp::Custom(origin, op + " NULLSWAPIF" + " NOT", 4, 3); } // DICTDEL: k D n => (D' -1) OR (D 0) -AsmOp compile_dict_del(std::vector& res, std::vector& args, SrcLocation loc) { +AsmOp compile_dict_del(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 2 && args.size() == 2+3); std::string op = choose_dict_op("DICTDEL", args[0], args[1]); - return AsmOp::Custom(loc, op, 3, 2); + return AsmOp::Custom(origin, op, 3, 2); } // DICTDELGET: k D n => (D' x -1) OR (D 0); + NULLSWAPIFNOT => (D' x -1) OR (D null 0) -AsmOp compile_dict_delGet(std::vector& res, std::vector& args, SrcLocation loc) { +AsmOp compile_dict_delGet(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 3 && args.size() == 2+3); std::string op = choose_dict_op("DICTDELGET", args[0], args[1]); - return AsmOp::Custom(loc, op + " NULLSWAPIFNOT", 3, 3); + return AsmOp::Custom(origin, op + " NULLSWAPIFNOT", 3, 3); } // fun map.exists(self, key: K): bool -std::vector generate_mapKV_exists(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_mapKV_exists(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); - DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + DictKeyValue kv(code, origin, TKey, &args[1], TypeDataSlice::create(), nullptr); - std::vector ir_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create()}), loc, "(lookup)"); + std::vector ir_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create()}), origin, "(lookup)"); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; - code.emplace_back(loc, Op::_Call, ir_lookup, std::move(dict_args), lookup_function("__dict.get")); + code.emplace_back(origin, Op::_Call, ir_lookup, std::move(dict_args), lookup_function("__dict.get")); return {ir_lookup[1]}; // isFound from (sliceOrNull isFound) } // fun map.get(self, key: K): MapLookupResult -std::vector generate_mapKV_get(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_mapKV_get(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); - DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + DictKeyValue kv(code, origin, TKey, &args[1], TypeDataSlice::create(), nullptr); - std::vector ir_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create()}), loc, "(lookup)"); + std::vector ir_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create()}), origin, "(lookup)"); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; - code.emplace_back(loc, Op::_Call, ir_lookup, std::move(dict_args), lookup_function("__dict.get")); + code.emplace_back(origin, Op::_Call, ir_lookup, std::move(dict_args), lookup_function("__dict.get")); // in all functions where we return MapLookupResult: // on a stack we have (slice, found) - exactly the shape of MapLookupResult; @@ -488,19 +488,19 @@ std::vector generate_mapKV_get(FunctionPtr called_f, CodeBlob& code, } // fun map.mustGet(self, key: K, throwIfNotFound: int = 9): V -std::vector generate_mapKV_mustGet(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_mapKV_mustGet(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); TypePtr TValue = called_f->substitutedTs->typeT_at(1); // since we don't return MapLookupResult, for `map>` we can use DICTGETREF bool use_DICTGETREF = is_TValue_cell_or_CellT(TValue); - DictKeyValue kv(code, loc, TKey, &args[1], use_DICTGETREF ? TValue : TypeDataSlice::create(), nullptr, true); + DictKeyValue kv(code, origin, TKey, &args[1], use_DICTGETREF ? TValue : TypeDataSlice::create(), nullptr, true); - std::vector ir_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create()}), loc, "(lookup)"); + std::vector ir_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create()}), origin, "(lookup)"); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; - code.emplace_back(loc, Op::_Call, ir_lookup, std::move(dict_args), lookup_function("__dict.mustGet")); + code.emplace_back(origin, Op::_Call, ir_lookup, std::move(dict_args), lookup_function("__dict.mustGet")); - std::vector args_assert = { args[2][0], ir_lookup[1], code.create_int(loc, 0, "") }; - Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); + std::vector args_assert = { args[2][0], ir_lookup[1], code.create_int(origin, 0, "") }; + Op& op_assert = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); op_assert.set_impure_flag(); std::vector ir_slice(ir_lookup.begin(), ir_lookup.begin() + 1); @@ -512,210 +512,210 @@ std::vector generate_mapKV_mustGet(FunctionPtr called_f, CodeBlob& co } // load TValue and check for assertEnd (it's the default behavior) - UnpackContext ctx(code, loc, std::move(ir_slice), create_default_UnpackOptions(code, loc)); + UnpackContext ctx(code, origin, std::move(ir_slice), create_default_UnpackOptions(code, origin)); std::vector ir_value = ctx.generate_unpack_any(TValue); ctx.assertEndIfOption(); return ir_value; } // fun map.set(mutate self, key: K, value: V): self -std::vector generate_mapKV_set(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_mapKV_set(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); TypePtr TValue = called_f->substitutedTs->typeT_at(1); - DictKeyValue kv(code, loc, TKey, &args[1], TValue, &args[2], true); + DictKeyValue kv(code, origin, TKey, &args[1], TValue, &args[2], true); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_value_val(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; - code.emplace_back(loc, Op::_Call, args[0], std::move(dict_args), lookup_function("__dict.set")); + code.emplace_back(origin, Op::_Call, args[0], std::move(dict_args), lookup_function("__dict.set")); return args[0]; // return mutated map } // fun map.setAndGetPrevious(mutate self, key: K, value: V): MapLookupResult -std::vector generate_mapKV_setGet(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_mapKV_setGet(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); TypePtr TValue = called_f->substitutedTs->typeT_at(1); - DictKeyValue kv(code, loc, TKey, &args[1], TValue, &args[2]); + DictKeyValue kv(code, origin, TKey, &args[1], TValue, &args[2]); - std::vector ir_map_and_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataSlice::create(), TypeDataInt::create()}), loc, "(map-and-lookup)"); + std::vector ir_map_and_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataSlice::create(), TypeDataInt::create()}), origin, "(map-and-lookup)"); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_value_val(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; - code.emplace_back(loc, Op::_Call, ir_map_and_lookup, std::move(dict_args), lookup_function("__dict.setGet")); + code.emplace_back(origin, Op::_Call, ir_map_and_lookup, std::move(dict_args), lookup_function("__dict.setGet")); return ir_map_and_lookup; } // fun map.replaceIfExists(mutate self, key: K, value: V): bool -std::vector generate_mapKV_replace(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_mapKV_replace(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); TypePtr TValue = called_f->substitutedTs->typeT_at(1); - DictKeyValue kv(code, loc, TKey, &args[1], TValue, &args[2], true); + DictKeyValue kv(code, origin, TKey, &args[1], TValue, &args[2], true); - std::vector ir_map_and_was_added = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataBool::create()}), loc, "(map-and-was-added)"); + std::vector ir_map_and_was_added = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataBool::create()}), origin, "(map-and-was-added)"); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_value_val(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; - code.emplace_back(loc, Op::_Call, ir_map_and_was_added, std::move(dict_args), lookup_function("__dict.replace")); + code.emplace_back(origin, Op::_Call, ir_map_and_was_added, std::move(dict_args), lookup_function("__dict.replace")); return ir_map_and_was_added; } // fun map.replaceAndGetPrevious(mutate self, key: K, value: V): MapLookupResult -std::vector generate_mapKV_replaceGet(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_mapKV_replaceGet(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); TypePtr TValue = called_f->substitutedTs->typeT_at(1); - DictKeyValue kv(code, loc, TKey, &args[1], TValue, &args[2]); + DictKeyValue kv(code, origin, TKey, &args[1], TValue, &args[2]); - std::vector ir_map_and_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataSlice::create(), TypeDataInt::create()}), loc, "(map-and-lookup)"); + std::vector ir_map_and_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataSlice::create(), TypeDataInt::create()}), origin, "(map-and-lookup)"); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_value_val(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; - code.emplace_back(loc, Op::_Call, ir_map_and_lookup, std::move(dict_args), lookup_function("__dict.replaceGet")); + code.emplace_back(origin, Op::_Call, ir_map_and_lookup, std::move(dict_args), lookup_function("__dict.replaceGet")); return ir_map_and_lookup; } // fun map.addIfNotExists(mutate self, key: K, value: V): bool -std::vector generate_mapKV_add(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_mapKV_add(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); TypePtr TValue = called_f->substitutedTs->typeT_at(1); - DictKeyValue kv(code, loc, TKey, &args[1], TValue, &args[2], true); + DictKeyValue kv(code, origin, TKey, &args[1], TValue, &args[2], true); - std::vector ir_map_and_was_added = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataBool::create()}), loc, "(map-and-was-added)"); + std::vector ir_map_and_was_added = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataBool::create()}), origin, "(map-and-was-added)"); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_value_val(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; - code.emplace_back(loc, Op::_Call, ir_map_and_was_added, std::move(dict_args), lookup_function("__dict.add")); + code.emplace_back(origin, Op::_Call, ir_map_and_was_added, std::move(dict_args), lookup_function("__dict.add")); return ir_map_and_was_added; } // fun map.addOrGetExisting(mutate self, key: K, value: V): MapLookupResult -std::vector generate_mapKV_addGet(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_mapKV_addGet(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); TypePtr TValue = called_f->substitutedTs->typeT_at(1); - DictKeyValue kv(code, loc, TKey, &args[1], TValue, &args[2]); + DictKeyValue kv(code, origin, TKey, &args[1], TValue, &args[2]); - std::vector ir_map_and_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataSlice::create(), TypeDataInt::create()}), loc, "(lookup)"); + std::vector ir_map_and_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataSlice::create(), TypeDataInt::create()}), origin, "(lookup)"); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_value_val(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; - code.emplace_back(loc, Op::_Call, ir_map_and_lookup, std::move(dict_args), lookup_function("__dict.addGet")); + code.emplace_back(origin, Op::_Call, ir_map_and_lookup, std::move(dict_args), lookup_function("__dict.addGet")); return ir_map_and_lookup; } // fun map.delete(mutate self, key: K): bool -std::vector generate_mapKV_del(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_mapKV_del(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); - DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + DictKeyValue kv(code, origin, TKey, &args[1], TypeDataSlice::create(), nullptr); - std::vector ir_map_and_was_deleted = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataBool::create()}), loc, "(map-and-was-added)"); + std::vector ir_map_and_was_deleted = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataBool::create()}), origin, "(map-and-was-added)"); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; - code.emplace_back(loc, Op::_Call, ir_map_and_was_deleted, std::move(dict_args), lookup_function("__dict.del")); + code.emplace_back(origin, Op::_Call, ir_map_and_was_deleted, std::move(dict_args), lookup_function("__dict.del")); return ir_map_and_was_deleted; } // fun map.deleteAndGetDeleted(mutate self, key: K): MapLookupResult -std::vector generate_mapKV_delGet(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_mapKV_delGet(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); - DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + DictKeyValue kv(code, origin, TKey, &args[1], TypeDataSlice::create(), nullptr); - std::vector ir_map_and_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataSlice::create(), TypeDataInt::create()}), loc, "(lookup)"); + std::vector ir_map_and_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataSlice::create(), TypeDataInt::create()}), origin, "(lookup)"); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; - code.emplace_back(loc, Op::_Call, ir_map_and_lookup, std::move(dict_args), lookup_function("__dict.delGet")); + code.emplace_back(origin, Op::_Call, ir_map_and_lookup, std::move(dict_args), lookup_function("__dict.delGet")); return ir_map_and_lookup; } // fun map.findFirst(): MapEntry -std::vector generate_mapKV_findFirst(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_mapKV_findFirst(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); - DictKeyValue kv(code, loc, TKey, nullptr, TypeDataSlice::create(), nullptr); + DictKeyValue kv(code, origin, TKey, nullptr, TypeDataSlice::create(), nullptr); // on a stack (ir_entry) we will have: either (x k −1) or (null null 0) - std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector ir_entry = create_ir_MapEntry(code, origin); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), args[0][0], kv.ir_key_len() }; - code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getMin")); + code.emplace_back(origin, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getMin")); - return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); + return finalize_ir_MapEntry(code, origin, std::move(ir_entry), TKey); } // fun map.findLast(): MapEntry -std::vector generate_mapKV_findLast(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_mapKV_findLast(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); - DictKeyValue kv(code, loc, TKey, nullptr, TypeDataSlice::create(), nullptr); + DictKeyValue kv(code, origin, TKey, nullptr, TypeDataSlice::create(), nullptr); - std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector ir_entry = create_ir_MapEntry(code, origin); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), args[0][0], kv.ir_key_len() }; - code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getMax")); + code.emplace_back(origin, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getMax")); - return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); + return finalize_ir_MapEntry(code, origin, std::move(ir_entry), TKey); } // fun map.findKeyGreater(pivotKey: K): MapEntry -std::vector generate_mapKV_findKeyGreater(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_mapKV_findKeyGreater(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); - DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + DictKeyValue kv(code, origin, TKey, &args[1], TypeDataSlice::create(), nullptr); - std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector ir_entry = create_ir_MapEntry(code, origin); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; - code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getNext")); + code.emplace_back(origin, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getNext")); - return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); + return finalize_ir_MapEntry(code, origin, std::move(ir_entry), TKey); } // fun map.findKeyGreaterOrEqual(pivotKey: K): MapEntry -std::vector generate_mapKV_findKeyGreaterOrEqual(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_mapKV_findKeyGreaterOrEqual(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); - DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + DictKeyValue kv(code, origin, TKey, &args[1], TypeDataSlice::create(), nullptr); - std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector ir_entry = create_ir_MapEntry(code, origin); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; - code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getNextEq")); + code.emplace_back(origin, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getNextEq")); - return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); + return finalize_ir_MapEntry(code, origin, std::move(ir_entry), TKey); } // fun map.findKeyLess(pivotKey: K): MapEntry -std::vector generate_mapKV_findKeyLess(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_mapKV_findKeyLess(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); - DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + DictKeyValue kv(code, origin, TKey, &args[1], TypeDataSlice::create(), nullptr); - std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector ir_entry = create_ir_MapEntry(code, origin); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; - code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getPrev")); + code.emplace_back(origin, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getPrev")); - return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); + return finalize_ir_MapEntry(code, origin, std::move(ir_entry), TKey); } // fun map.findKeyLessOrEqual(pivotKey: K): MapEntry -std::vector generate_mapKV_findKeyLessOrEqual(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_mapKV_findKeyLessOrEqual(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); - DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + DictKeyValue kv(code, origin, TKey, &args[1], TypeDataSlice::create(), nullptr); - std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector ir_entry = create_ir_MapEntry(code, origin); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; - code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getPrevEq")); + code.emplace_back(origin, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getPrevEq")); - return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); + return finalize_ir_MapEntry(code, origin, std::move(ir_entry), TKey); } // fun map.iterateNext(current: MapEntry): MapEntry -std::vector generate_mapKV_iterateNext(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_mapKV_iterateNext(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { std::vector ir_pivot_key(args[1].begin() + 1, args[1].end() - 1); TypePtr TKey = called_f->substitutedTs->typeT_at(0); - DictKeyValue kv(code, loc, TKey, &ir_pivot_key, TypeDataSlice::create(), nullptr); + DictKeyValue kv(code, origin, TKey, &ir_pivot_key, TypeDataSlice::create(), nullptr); - std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector ir_entry = create_ir_MapEntry(code, origin); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; - code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getNext")); + code.emplace_back(origin, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getNext")); - return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); + return finalize_ir_MapEntry(code, origin, std::move(ir_entry), TKey); } // fun map.iteratePrev(current: MapEntry): MapEntry -std::vector generate_mapKV_iteratePrev(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_mapKV_iteratePrev(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { std::vector ir_pivot_key(args[1].begin() + 1, args[1].end() - 1); TypePtr TKey = called_f->substitutedTs->typeT_at(0); - DictKeyValue kv(code, loc, TKey, &ir_pivot_key, TypeDataSlice::create(), nullptr); + DictKeyValue kv(code, origin, TKey, &ir_pivot_key, TypeDataSlice::create(), nullptr); - std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector ir_entry = create_ir_MapEntry(code, origin); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; - code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getPrev")); + code.emplace_back(origin, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getPrev")); - return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); + return finalize_ir_MapEntry(code, origin, std::move(ir_entry), TKey); } } // namespace tolk diff --git a/tolk/maps-kv-api.h b/tolk/maps-kv-api.h index d6d64adc41..5ba61f25dc 100644 --- a/tolk/maps-kv-api.h +++ b/tolk/maps-kv-api.h @@ -17,7 +17,7 @@ #pragma once #include "fwd-declarations.h" -#include "tolk.h" +#include namespace tolk { diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index f30bd15975..3282a201db 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -159,8 +159,8 @@ bool Optimizer::detect_rewrite_big_THROW() { p_ = 1; q_ = 2; - oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, td::make_refint(excno))); - oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, "THROWANY", 1, 0)); + oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->origin, td::make_refint(excno))); + oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->origin, "THROWANY", 1, 0)); return true; } @@ -206,8 +206,8 @@ bool Optimizer::detect_rewrite_MY_store_int() { if (!use_stsliceconst) { p_ = n_merged; q_ = 2; - oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, total_number)); - oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, std::to_string(total_len) + (first_unsigned ? " STUR" : " STIR"), 1, 1)); + oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->origin, total_number)); + oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->origin, std::to_string(total_len) + (first_unsigned ? " STUR" : " STIR"), 1, 1)); return true; } @@ -229,7 +229,7 @@ bool Optimizer::detect_rewrite_MY_store_int() { } result += " STSLICECONST"; - oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, result, 0, 1)); + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, result, 0, 1)); return true; } @@ -256,11 +256,11 @@ bool Optimizer::detect_rewrite_MY_skip_bits() { p_ = n_merged; q_ = 2; if (total_skip_bits <= 256) { - oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, std::to_string(total_skip_bits) + " LDU")); - oq_[1] = std::make_unique(AsmOp::Pop(op_[0]->loc, 1)); + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, std::to_string(total_skip_bits) + " LDU")); + oq_[1] = std::make_unique(AsmOp::Pop(op_[0]->origin, 1)); } else { - oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, td::make_refint(total_skip_bits))); - oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, "SDSKIPFIRST")); + oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->origin, td::make_refint(total_skip_bits))); + oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->origin, "SDSKIPFIRST")); } return true; } @@ -284,7 +284,7 @@ bool Optimizer::detect_rewrite_NEWC_PUSH_STUR() { q_ = 3; oq_[0] = std::move(op_[1]); oq_[1] = std::move(op_[0]); - oq_[2] = std::make_unique(AsmOp::Custom(oq_[0]->loc, op_[2]->op.substr(0, op_[2]->op.size() - 1), 1, 1)); + oq_[2] = std::make_unique(AsmOp::Custom(oq_[0]->origin, op_[2]->op.substr(0, op_[2]->op.size() - 1), 1, 1)); return true; } @@ -306,7 +306,7 @@ bool Optimizer::detect_rewrite_LDxx_DROP() { if (f.ends_with(ends_with[i])) { p_ = 2; q_ = 1; - oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, op_[0]->op.substr(0, f.rfind(' ')) + repl_with[i], 0, 1)); + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, op_[0]->op.substr(0, f.rfind(' ')) + repl_with[i], 0, 1)); return true; } } @@ -314,7 +314,7 @@ bool Optimizer::detect_rewrite_LDxx_DROP() { if (f == equl_to[i]) { p_ = 2; q_ = 1; - oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, repl_to[i], 0, 1)); + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, repl_to[i], 0, 1)); return true; } } @@ -360,8 +360,8 @@ bool Optimizer::detect_rewrite_SWAP_PUSH_STUR() { p_ = 3; q_ = 3; oq_[0] = std::move(op_[1]); - oq_[1] = std::make_unique(AsmOp::BlkSwap(oq_[0]->loc, 1, 2)); // ROT - oq_[2] = std::make_unique(AsmOp::Custom(oq_[0]->loc, op_[2]->op.substr(0, op_[2]->op.size() - 1), 1, 1)); + oq_[1] = std::make_unique(AsmOp::BlkSwap(oq_[0]->origin, 1, 2)); // ROT + oq_[2] = std::make_unique(AsmOp::Custom(oq_[0]->origin, op_[2]->op.substr(0, op_[2]->op.size() - 1), 1, 1)); return true; } @@ -383,7 +383,7 @@ bool Optimizer::detect_rewrite_SWAP_STxxxR() { if (f.ends_with(ends_with[i])) { p_ = 2; q_ = 1; - oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, op_[1]->op.substr(0, f.rfind(' ')) + repl_with[i], 1, 1)); + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, op_[1]->op.substr(0, f.rfind(' ')) + repl_with[i], 1, 1)); return true; } } @@ -391,7 +391,7 @@ bool Optimizer::detect_rewrite_SWAP_STxxxR() { if (f == equl_to[i]) { p_ = 2; q_ = 1; - oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, repl_to[i], 0, 1)); + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, repl_to[i], 0, 1)); return true; } } @@ -414,7 +414,7 @@ bool Optimizer::detect_rewrite_NOT_THROWIF() { if (f.ends_with(ends_with[i])) { p_ = 2; q_ = 1; - oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, op_[1]->op.substr(0, f.rfind(' ')) + repl_with[i], 1, 0)); + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, op_[1]->op.substr(0, f.rfind(' ')) + repl_with[i], 1, 0)); return true; } } @@ -447,10 +447,10 @@ bool Optimizer::detect_rewrite_DICTSETB_DICTSET() { new_op.replace(pos, std::strlen(contains_b[i]), repl_with[i]); p_ = 5; q_ = 4; - oq_[0] = std::make_unique(AsmOp::Custom(op_[1]->loc, op_[1]->op.substr(0, op_[1]->op.rfind(' ')) + " PUSHSLICE", 0, 1)); + oq_[0] = std::make_unique(AsmOp::Custom(op_[1]->origin, op_[1]->op.substr(0, op_[1]->op.rfind(' ')) + " PUSHSLICE", 0, 1)); oq_[1] = std::move(op_[2]); oq_[2] = std::move(op_[3]); - oq_[3] = std::make_unique(AsmOp::Custom(op_[4]->loc, new_op)); + oq_[3] = std::make_unique(AsmOp::Custom(op_[4]->origin, new_op)); return true; } } @@ -472,7 +472,7 @@ bool Optimizer::detect_rewrite_ENDC_CTOS() { p_ = 2; q_ = 1; - oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, "BTOS", 1, 1)); + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, "BTOS", 1, 1)); return true; } @@ -490,7 +490,7 @@ bool Optimizer::detect_rewrite_ENDC_HASHCU() { p_ = 2; q_ = 1; - oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, "HASHBU", 1, 1)); + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, "HASHBU", 1, 1)); return true; } @@ -508,7 +508,7 @@ bool Optimizer::detect_rewrite_NEWC_BTOS() { p_ = 2; q_ = 1; - oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, "x{} PUSHSLICE", 0, 1)); + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, "x{} PUSHSLICE", 0, 1)); return true; } @@ -528,7 +528,7 @@ bool Optimizer::detect_rewrite_NEWC_STSLICECONST_BTOS() { std::string op_pushslice = op_[1]->op.substr(0, op_[1]->op.rfind(' ')) + " PUSHSLICE"; p_ = 3; q_ = 1; - oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, op_pushslice, 0, 1)); + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, op_pushslice, 0, 1)); return true; } @@ -548,7 +548,7 @@ bool Optimizer::detect_rewrite_NEWC_ENDC_CTOS() { p_ = 3; q_ = 1; - oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, "x{} PUSHSLICE", 0, 1)); + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, "x{} PUSHSLICE", 0, 1)); return true; } @@ -566,7 +566,7 @@ bool Optimizer::detect_rewrite_NEWC_ENDC() { p_ = 2; q_ = 1; - oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, " PUSHREF", 0, 1)); + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, " PUSHREF", 0, 1)); return true; } @@ -585,7 +585,7 @@ bool Optimizer::rewrite_push_const(int i, int c) { show_left(); oq_[1] = std::move(op_[idx]); oq_[0] = std::move(op_[!idx]); - *oq_[0] = AsmOp::Push(oq_[0]->loc, i); + *oq_[0] = AsmOp::Push(oq_[0]->origin, i); show_right(); return true; } @@ -604,7 +604,7 @@ bool Optimizer::rewrite_const_rot(int c) { show_left(); oq_[0] = std::move(op_[idx]); oq_[1] = std::move(op_[!idx]); - *oq_[1] = AsmOp::Custom(oq_[0]->loc, "ROT", 3, 3); + *oq_[1] = AsmOp::Custom(oq_[0]->origin, "ROT", 3, 3); show_right(); return true; } @@ -623,7 +623,7 @@ bool Optimizer::rewrite_const_pop(int c, int i) { show_left(); oq_[0] = std::move(op_[idx]); oq_[1] = std::move(op_[!idx]); - *oq_[1] = AsmOp::Pop(oq_[0]->loc, i); + *oq_[1] = AsmOp::Pop(oq_[0]->origin, i); show_right(); return true; } @@ -981,13 +981,13 @@ bool Optimizer::find_at_least(int pb) { pb_ = pb; // show_stack_transforms(); int i, j, k, l, c; - SrcLocation loc; // for asm ops inserted by optimizer, leave location empty (in fift output, it'll be attached to above) + AnyV origin = nullptr; // for asm ops inserted by optimizer, leave location empty (in fift output, it'll be attached to above) return (is_push_const(&i, &c) && rewrite_push_const(i, c)) || (is_nop() && rewrite_nop()) || (!(mode_ & 1) && is_const_rot(&c) && rewrite_const_rot(c)) || (is_const_push_xchgs() && rewrite_const_push_xchgs()) || (is_const_pop(&c, &i) && rewrite_const_pop(c, i)) || - (is_xchg(&i, &j) && rewrite(AsmOp::Xchg(loc, i, j))) || (is_push(&i) && rewrite(AsmOp::Push(loc, i))) || - (is_pop(&i) && rewrite(AsmOp::Pop(loc, i))) || (is_pop_pop(&i, &j) && rewrite(AsmOp::Pop(loc, i), AsmOp::Pop(loc, j))) || - (is_xchg_xchg(&i, &j, &k, &l) && rewrite(AsmOp::Xchg(loc, i, j), AsmOp::Xchg(loc, k, l))) || + (is_xchg(&i, &j) && rewrite(AsmOp::Xchg(origin, i, j))) || (is_push(&i) && rewrite(AsmOp::Push(origin, i))) || + (is_pop(&i) && rewrite(AsmOp::Pop(origin, i))) || (is_pop_pop(&i, &j) && rewrite(AsmOp::Pop(origin, i), AsmOp::Pop(origin, j))) || + (is_xchg_xchg(&i, &j, &k, &l) && rewrite(AsmOp::Xchg(origin, i, j), AsmOp::Xchg(origin, k, l))) || detect_rewrite_big_THROW() || detect_rewrite_MY_store_int() || detect_rewrite_MY_skip_bits() || detect_rewrite_NEWC_PUSH_STUR() || detect_rewrite_LDxx_DROP() || @@ -997,33 +997,33 @@ bool Optimizer::find_at_least(int pb) { detect_rewrite_NEWC_BTOS() || detect_rewrite_NEWC_STSLICECONST_BTOS() || detect_rewrite_NEWC_ENDC_CTOS() || detect_rewrite_NEWC_ENDC() || (!(mode_ & 1) && - ((is_rot() && rewrite(AsmOp::Custom(loc, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(loc, "-ROT", 3, 3))) || - (is_2dup() && rewrite(AsmOp::Custom(loc, "2DUP", 2, 4))) || - (is_2swap() && rewrite(AsmOp::Custom(loc, "2SWAP", 2, 4))) || - (is_2over() && rewrite(AsmOp::Custom(loc, "2OVER", 2, 4))) || - (is_tuck() && rewrite(AsmOp::Custom(loc, "TUCK", 2, 3))) || - (is_2drop() && rewrite(AsmOp::Custom(loc, "2DROP", 2, 0))) || (is_xchg2(&i, &j) && rewrite(AsmOp::Xchg2(loc, i, j))) || - (is_xcpu(&i, &j) && rewrite(AsmOp::XcPu(loc, i, j))) || (is_puxc(&i, &j) && rewrite(AsmOp::PuXc(loc, i, j))) || - (is_push2(&i, &j) && rewrite(AsmOp::Push2(loc, i, j))) || (is_blkswap(&i, &j) && rewrite(AsmOp::BlkSwap(loc, i, j))) || - (is_blkpush(&i, &j) && rewrite(AsmOp::BlkPush(loc, i, j))) || (is_blkdrop(&i) && rewrite(AsmOp::BlkDrop(loc, i))) || - (is_push_rot(&i) && rewrite(AsmOp::Push(loc, i), AsmOp::Custom(loc, "ROT"))) || - (is_push_rotrev(&i) && rewrite(AsmOp::Push(loc, i), AsmOp::Custom(loc, "-ROT"))) || - (is_push_xchg(&i, &j, &k) && rewrite(AsmOp::Push(loc, i), AsmOp::Xchg(loc, j, k))) || - (is_reverse(&i, &j) && rewrite(AsmOp::BlkReverse(loc, i, j))) || - (is_blkdrop2(&i, &j) && rewrite(AsmOp::BlkDrop2(loc, i, j))) || - (is_nip_seq(&i, &j) && rewrite(AsmOp::Xchg(loc, i, j), AsmOp::BlkDrop(loc, i))) || - (is_pop_blkdrop(&i, &k) && rewrite(AsmOp::Pop(loc, i), AsmOp::BlkDrop(loc, k))) || + ((is_rot() && rewrite(AsmOp::Custom(origin, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(origin, "-ROT", 3, 3))) || + (is_2dup() && rewrite(AsmOp::Custom(origin, "2DUP", 2, 4))) || + (is_2swap() && rewrite(AsmOp::Custom(origin, "2SWAP", 2, 4))) || + (is_2over() && rewrite(AsmOp::Custom(origin, "2OVER", 2, 4))) || + (is_tuck() && rewrite(AsmOp::Custom(origin, "TUCK", 2, 3))) || + (is_2drop() && rewrite(AsmOp::Custom(origin, "2DROP", 2, 0))) || (is_xchg2(&i, &j) && rewrite(AsmOp::Xchg2(origin, i, j))) || + (is_xcpu(&i, &j) && rewrite(AsmOp::XcPu(origin, i, j))) || (is_puxc(&i, &j) && rewrite(AsmOp::PuXc(origin, i, j))) || + (is_push2(&i, &j) && rewrite(AsmOp::Push2(origin, i, j))) || (is_blkswap(&i, &j) && rewrite(AsmOp::BlkSwap(origin, i, j))) || + (is_blkpush(&i, &j) && rewrite(AsmOp::BlkPush(origin, i, j))) || (is_blkdrop(&i) && rewrite(AsmOp::BlkDrop(origin, i))) || + (is_push_rot(&i) && rewrite(AsmOp::Push(origin, i), AsmOp::Custom(origin, "ROT"))) || + (is_push_rotrev(&i) && rewrite(AsmOp::Push(origin, i), AsmOp::Custom(origin, "-ROT"))) || + (is_push_xchg(&i, &j, &k) && rewrite(AsmOp::Push(origin, i), AsmOp::Xchg(origin, j, k))) || + (is_reverse(&i, &j) && rewrite(AsmOp::BlkReverse(origin, i, j))) || + (is_blkdrop2(&i, &j) && rewrite(AsmOp::BlkDrop2(origin, i, j))) || + (is_nip_seq(&i, &j) && rewrite(AsmOp::Xchg(origin, i, j), AsmOp::BlkDrop(origin, i))) || + (is_pop_blkdrop(&i, &k) && rewrite(AsmOp::Pop(origin, i), AsmOp::BlkDrop(origin, k))) || (is_2pop_blkdrop(&i, &j, &k) && (k >= 3 && k <= 13 && i != j + 1 && i <= 15 && j <= 14 - ? rewrite(AsmOp::Xchg2(loc, j + 1, i), AsmOp::BlkDrop(loc, k + 2)) - : rewrite(AsmOp::Pop(loc, i), AsmOp::Pop(loc, j), AsmOp::BlkDrop(loc, k)))) || - (is_xchg3(&i, &j, &k) && rewrite(AsmOp::Xchg3(loc, i, j, k))) || - (is_xc2pu(&i, &j, &k) && rewrite(AsmOp::Xc2Pu(loc, i, j, k))) || - (is_xcpuxc(&i, &j, &k) && rewrite(AsmOp::XcPuXc(loc, i, j, k))) || - (is_xcpu2(&i, &j, &k) && rewrite(AsmOp::XcPu2(loc, i, j, k))) || - (is_puxc2(&i, &j, &k) && rewrite(AsmOp::PuXc2(loc, i, j, k))) || - (is_puxcpu(&i, &j, &k) && rewrite(AsmOp::PuXcPu(loc, i, j, k))) || - (is_pu2xc(&i, &j, &k) && rewrite(AsmOp::Pu2Xc(loc, i, j, k))) || - (is_push3(&i, &j, &k) && rewrite(AsmOp::Push3(loc, i, j, k))))); + ? rewrite(AsmOp::Xchg2(origin, j + 1, i), AsmOp::BlkDrop(origin, k + 2)) + : rewrite(AsmOp::Pop(origin, i), AsmOp::Pop(origin, j), AsmOp::BlkDrop(origin, k)))) || + (is_xchg3(&i, &j, &k) && rewrite(AsmOp::Xchg3(origin, i, j, k))) || + (is_xc2pu(&i, &j, &k) && rewrite(AsmOp::Xc2Pu(origin, i, j, k))) || + (is_xcpuxc(&i, &j, &k) && rewrite(AsmOp::XcPuXc(origin, i, j, k))) || + (is_xcpu2(&i, &j, &k) && rewrite(AsmOp::XcPu2(origin, i, j, k))) || + (is_puxc2(&i, &j, &k) && rewrite(AsmOp::PuXc2(origin, i, j, k))) || + (is_puxcpu(&i, &j, &k) && rewrite(AsmOp::PuXcPu(origin, i, j, k))) || + (is_pu2xc(&i, &j, &k) && rewrite(AsmOp::Pu2Xc(origin, i, j, k))) || + (is_push3(&i, &j, &k) && rewrite(AsmOp::Push3(origin, i, j, k))))); } bool Optimizer::find() { diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index f710d2e30a..8d68529182 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -248,38 +248,38 @@ static int calc_offset_on_stack(StructPtr struct_ref, int field_idx) { // fun T.toCell(self, options: PackOptions): Cell -std::vector generate_T_toCell(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_T_toCell(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr typeT = called_f->substitutedTs->typeT_at(0); FunctionPtr f_beginCell = lookup_function("beginCell"); FunctionPtr f_endCell = lookup_function("builder.endCell"); - std::vector rvect_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); - code.emplace_back(loc, Op::_Call, rvect_builder, std::vector{}, f_beginCell); + std::vector rvect_builder = code.create_var(TypeDataBuilder::create(), origin, "b"); + code.emplace_back(origin, Op::_Call, rvect_builder, std::vector{}, f_beginCell); - PackContext ctx(code, loc, rvect_builder, args[1]); + PackContext ctx(code, origin, rvect_builder, args[1]); ctx.generate_pack_any(typeT, std::vector(args[0])); - std::vector rvect_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(cell)"); - code.emplace_back(loc, Op::_Call, rvect_cell, std::move(rvect_builder), f_endCell); + std::vector rvect_cell = code.create_tmp_var(TypeDataCell::create(), origin, "(cell)"); + code.emplace_back(origin, Op::_Call, rvect_cell, std::move(rvect_builder), f_endCell); return rvect_cell; } // fun builder.storeAny(mutate self, v: T, options: PackOptions = {}): self -std::vector generate_builder_storeAny(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_builder_storeAny(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr typeT = called_f->substitutedTs->typeT_at(0); - PackContext ctx(code, loc, args[0], args[2]); // mutate this builder + PackContext ctx(code, origin, args[0], args[2]); // mutate this builder ctx.generate_pack_any(typeT, std::vector(args[1])); return args[0]; // return mutated builder } // fun T.fromSlice(rawSlice: slice, options: UnpackOptions): T -std::vector generate_T_fromSlice(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { - std::vector slice_copy = code.create_var(TypeDataSlice::create(), loc, "s"); - code.emplace_back(loc, Op::_Let, slice_copy, args[0]); +std::vector generate_T_fromSlice(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { + std::vector slice_copy = code.create_var(TypeDataSlice::create(), origin, "s"); + code.emplace_back(origin, Op::_Let, slice_copy, args[0]); TypePtr typeT = called_f->substitutedTs->typeT_at(0); - UnpackContext ctx(code, loc, std::move(slice_copy), args[1]); + UnpackContext ctx(code, origin, std::move(slice_copy), args[1]); std::vector rvect_struct = ctx.generate_unpack_any(typeT); tolk_assert(typeT->get_width_on_stack() == static_cast(rvect_struct.size())); @@ -290,9 +290,9 @@ std::vector generate_T_fromSlice(FunctionPtr called_f, CodeBlob& code } // fun slice.loadAny(mutate self, options: UnpackOptions): T -std::vector generate_slice_loadAny(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_slice_loadAny(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr typeT = called_f->substitutedTs->typeT_at(0); - UnpackContext ctx(code, loc, args[0], args[1]); + UnpackContext ctx(code, origin, args[0], args[1]); std::vector rvect_struct = ctx.generate_unpack_any(typeT); tolk_assert(typeT->get_width_on_stack() == static_cast(rvect_struct.size())); @@ -304,13 +304,13 @@ std::vector generate_slice_loadAny(FunctionPtr called_f, CodeBlob& co // fun T.fromCell(packedCell: cell, options: UnpackOptions): T // fun Cell.load(self, options: UnpackOptions): T -std::vector generate_T_fromCell(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_T_fromCell(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr typeT = called_f->substitutedTs->typeT_at(0); FunctionPtr f_beginParse = lookup_function("cell.beginParse"); - std::vector ir_slice = code.create_var(TypeDataSlice::create(), loc, "s"); - code.emplace_back(loc, Op::_Call, ir_slice, args[0], f_beginParse); + std::vector ir_slice = code.create_var(TypeDataSlice::create(), origin, "s"); + code.emplace_back(origin, Op::_Call, ir_slice, args[0], f_beginParse); - UnpackContext ctx(code, loc, std::move(ir_slice), args[1]); + UnpackContext ctx(code, origin, std::move(ir_slice), args[1]); std::vector rvect_struct = ctx.generate_unpack_any(typeT); tolk_assert(typeT->get_width_on_stack() == static_cast(rvect_struct.size())); @@ -322,15 +322,15 @@ std::vector generate_T_fromCell(FunctionPtr called_f, CodeBlob& code, } // fun slice.skipAny(mutate self, options: UnpackOptions): self -std::vector generate_slice_skipAny(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_slice_skipAny(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr typeT = called_f->substitutedTs->typeT_at(0); - UnpackContext ctx(code, loc, args[0], args[1]); // mutate this slice + UnpackContext ctx(code, origin, args[0], args[1]); // mutate this slice ctx.generate_skip_any(typeT); return args[0]; // return mutated slice } -void generate_lazy_struct_from_slice(CodeBlob& code, SrcLocation loc, const LazyVariableLoadedState* lazy_variable, const LazyStructLoadInfo& load_info, const std::vector& ir_obj) { +void generate_lazy_struct_from_slice(CodeBlob& code, AnyV origin, const LazyVariableLoadedState* lazy_variable, const LazyStructLoadInfo& load_info, const std::vector& ir_obj) { StructPtr original_struct = load_info.original_struct; StructPtr hidden_struct = load_info.hidden_struct; tolk_assert(hidden_struct->fields.size() == load_info.ith_field_action.size()); @@ -339,7 +339,7 @@ void generate_lazy_struct_from_slice(CodeBlob& code, SrcLocation loc, const Lazy tolk_assert(loaded_state && !loaded_state->was_loaded_once()); loaded_state->mutate()->on_started_loading(hidden_struct); - UnpackContext ctx(code, loc, lazy_variable->ir_slice, lazy_variable->ir_options); + UnpackContext ctx(code, origin, lazy_variable->ir_slice, lazy_variable->ir_options); if (hidden_struct->opcode.exists()) { ctx.loadAndCheckOpcode(hidden_struct->opcode); @@ -357,7 +357,7 @@ void generate_lazy_struct_from_slice(CodeBlob& code, SrcLocation loc, const Lazy std::vector ir_field = ctx.generate_unpack_any(hidden_field->declared_type); int stack_offset = calc_offset_on_stack(original_struct, original_field->field_idx); int stack_width = hidden_field->declared_type->get_width_on_stack(); - code.emplace_back(loc, Op::_Let, std::vector(ir_obj.begin() + stack_offset, ir_obj.begin() + stack_offset + stack_width), std::move(ir_field)); + code.emplace_back(origin, Op::_Let, std::vector(ir_obj.begin() + stack_offset, ir_obj.begin() + stack_offset + stack_width), std::move(ir_field)); loaded_state->mutate()->on_original_field_loaded(hidden_field); } else { tolk_assert(hidden_field->name == "(gap)"); @@ -377,8 +377,8 @@ void generate_lazy_struct_from_slice(CodeBlob& code, SrcLocation loc, const Lazy break; } case LazyStructLoadInfo::SaveImmutableTail: { - std::vector ir_immutable_tail = code.create_tmp_var(TypeDataSlice::create(), loc, "(lazy-tail-slice)"); - code.emplace_back(loc, Op::_Let, ir_immutable_tail, lazy_variable->ir_slice); + std::vector ir_immutable_tail = code.create_tmp_var(TypeDataSlice::create(), origin, "(lazy-tail-slice)"); + code.emplace_back(origin, Op::_Let, ir_immutable_tail, lazy_variable->ir_slice); loaded_state->mutate()->on_aside_field_loaded(hidden_field, std::move(ir_immutable_tail)); break; } @@ -388,17 +388,17 @@ void generate_lazy_struct_from_slice(CodeBlob& code, SrcLocation loc, const Lazy // options.assertEndAfterReading is ignored by `lazy`, because tail fields may be skipped, it's okay } -std::vector generate_lazy_struct_to_cell(CodeBlob& code, SrcLocation loc, const LazyStructLoadedState* loaded_state, std::vector&& ir_obj, const std::vector& ir_options) { +std::vector generate_lazy_struct_to_cell(CodeBlob& code, AnyV origin, const LazyStructLoadedState* loaded_state, std::vector&& ir_obj, const std::vector& ir_options) { StructPtr original_struct = loaded_state->original_struct; StructPtr hidden_struct = loaded_state->hidden_struct; - std::vector rvect_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); - code.emplace_back(loc, Op::_Call, rvect_builder, std::vector{}, lookup_function("beginCell")); + std::vector rvect_builder = code.create_var(TypeDataBuilder::create(), origin, "b"); + code.emplace_back(origin, Op::_Call, rvect_builder, std::vector{}, lookup_function("beginCell")); - PackContext ctx(code, loc, rvect_builder, ir_options); + PackContext ctx(code, origin, rvect_builder, ir_options); if (hidden_struct->opcode.exists()) { - ctx.storeUint(code.create_int(loc, hidden_struct->opcode.pack_prefix, "(struct-prefix)"), hidden_struct->opcode.prefix_len); + ctx.storeUint(code.create_int(origin, hidden_struct->opcode.pack_prefix, "(struct-prefix)"), hidden_struct->opcode.prefix_len); } for (int field_idx = 0; field_idx < hidden_struct->get_num_fields(); ++field_idx) { @@ -423,37 +423,37 @@ std::vector generate_lazy_struct_to_cell(CodeBlob& code, SrcLocation } } - std::vector rvect_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(cell)"); - code.emplace_back(loc, Op::_Call, rvect_cell, std::move(rvect_builder), lookup_function("builder.endCell")); + std::vector rvect_cell = code.create_tmp_var(TypeDataCell::create(), origin, "(cell)"); + code.emplace_back(origin, Op::_Call, rvect_cell, std::move(rvect_builder), lookup_function("builder.endCell")); return rvect_cell; } -std::vector generate_lazy_match_for_union(CodeBlob& code, SrcLocation loc, TypePtr union_type, const LazyVariableLoadedState* lazy_variable, const LazyMatchOptions& options) { +std::vector generate_lazy_match_for_union(CodeBlob& code, AnyV origin, TypePtr union_type, const LazyVariableLoadedState* lazy_variable, const LazyMatchOptions& options) { tolk_assert(lazy_variable->ir_options.size() == 2); if (options.match_blocks.empty()) { // empty `match` statement, no arms return {}; } - UnpackContext ctx(code, loc, lazy_variable->ir_slice, lazy_variable->ir_options); + UnpackContext ctx(code, origin, lazy_variable->ir_slice, lazy_variable->ir_options); std::vector rvect_match = ctx.generate_lazy_match_any(union_type, options); return rvect_match; } -std::vector generate_lazy_object_finish_loading(CodeBlob& code, SrcLocation loc, const LazyVariableLoadedState* lazy_variable, std::vector&& ir_obj) { +std::vector generate_lazy_object_finish_loading(CodeBlob& code, AnyV origin, const LazyVariableLoadedState* lazy_variable, std::vector&& ir_obj) { tolk_assert(lazy_variable->ir_slice.size() == 1); // the call to `lazy_var.forceLoadLazyObject()` does not do anything: at the moment of analyzing, // it had marked all the object as "used", all fields where loaded, and the slice points after the last field; // so, just return the held slice static_cast(code); - static_cast(loc); + static_cast(origin); static_cast(ir_obj); return lazy_variable->ir_slice; } -std::vector generate_T_forceLoadLazyObject(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_T_forceLoadLazyObject(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { // a call to `T.forceLoadLazyObject()` was handled separately tolk_assert(false); } @@ -463,15 +463,15 @@ PackSize estimate_serialization_size(TypePtr any_type) { return ctx.estimate_any(any_type); } -std::vector generate_T_estimatePackSize(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_T_estimatePackSize(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr typeT = called_f->substitutedTs->typeT_at(0); PackSize pack_size = estimate_serialization_size(typeT); - std::vector ir_tensor = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create()}), loc, "(result-tensor)"); - code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[0]}, td::make_refint(pack_size.min_bits)); - code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[1]}, td::make_refint(pack_size.max_bits)); - code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[2]}, td::make_refint(pack_size.min_refs)); - code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[3]}, td::make_refint(pack_size.max_refs)); + std::vector ir_tensor = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create()}), origin, "(result-tensor)"); + code.emplace_back(origin, Op::_IntConst, std::vector{ir_tensor[0]}, td::make_refint(pack_size.min_bits)); + code.emplace_back(origin, Op::_IntConst, std::vector{ir_tensor[1]}, td::make_refint(pack_size.max_bits)); + code.emplace_back(origin, Op::_IntConst, std::vector{ir_tensor[2]}, td::make_refint(pack_size.min_refs)); + code.emplace_back(origin, Op::_IntConst, std::vector{ir_tensor[3]}, td::make_refint(pack_size.max_refs)); return ir_tensor; } diff --git a/tolk/pack-unpack-api.h b/tolk/pack-unpack-api.h index 45da429e56..0f2169f60a 100644 --- a/tolk/pack-unpack-api.h +++ b/tolk/pack-unpack-api.h @@ -31,9 +31,9 @@ struct LazyStructLoadInfo; struct LazyStructLoadedState; struct LazyVariableLoadedState; -void generate_lazy_struct_from_slice(CodeBlob& code, SrcLocation loc, const LazyVariableLoadedState* lazy_variable, const LazyStructLoadInfo& load_info, const std::vector& ir_obj); -std::vector generate_lazy_struct_to_cell(CodeBlob& code, SrcLocation loc, const LazyStructLoadedState* loaded_state, std::vector&& ir_obj, const std::vector& ir_options); -std::vector generate_lazy_match_for_union(CodeBlob& code, SrcLocation loc, TypePtr union_type, const LazyVariableLoadedState* lazy_variable, const LazyMatchOptions& options); -std::vector generate_lazy_object_finish_loading(CodeBlob& code, SrcLocation loc, const LazyVariableLoadedState* lazy_variable, std::vector&& ir_obj); +void generate_lazy_struct_from_slice(CodeBlob& code, AnyV origin, const LazyVariableLoadedState* lazy_variable, const LazyStructLoadInfo& load_info, const std::vector& ir_obj); +std::vector generate_lazy_struct_to_cell(CodeBlob& code, AnyV origin, const LazyStructLoadedState* loaded_state, std::vector&& ir_obj, const std::vector& ir_options); +std::vector generate_lazy_match_for_union(CodeBlob& code, AnyV origin, TypePtr union_type, const LazyVariableLoadedState* lazy_variable, const LazyMatchOptions& options); +std::vector generate_lazy_object_finish_loading(CodeBlob& code, AnyV origin, const LazyVariableLoadedState* lazy_variable, std::vector&& ir_obj); } // namespace tolk diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index e5cce47078..3f04bb935f 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -15,7 +15,8 @@ along with TON Blockchain Library. If not, see . */ #include "pack-unpack-serializers.h" -#include "tolk.h" +#include "ast.h" +#include "compilation-errors.h" #include "type-system.h" #include "td/utils/crypto.h" @@ -38,9 +39,9 @@ namespace tolk { class LValContext; std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr target_type = nullptr, LValContext* lval_ctx = nullptr); -std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc); -std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc); -std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, SrcLocation loc, FunctionPtr f_inlined, AnyExprV self_obj, bool is_before_immediate_return, const std::vector>& vars_per_arg); +std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, AnyV origin, const char* debug_desc); +std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, AnyV origin); +std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, AnyV origin, FunctionPtr f_inlined, AnyExprV self_obj, bool is_before_immediate_return, const std::vector>& vars_per_arg); bool is_type_cellT(TypePtr any_type) { if (const TypeDataStruct* t_struct = any_type->try_as()) { @@ -77,9 +78,9 @@ FunctionPtr get_custom_pack_unpack_function(TypePtr receiver_type, bool is_pack) // -PackContext::PackContext(CodeBlob& code, SrcLocation loc, std::vector ir_builder, const std::vector& ir_options) +PackContext::PackContext(CodeBlob& code, AnyV origin, std::vector ir_builder, const std::vector& ir_options) : code(code) - , loc(loc) + , origin(origin) , f_storeInt(lookup_function("builder.storeInt")) , f_storeUint(lookup_function("builder.storeUint")) , ir_builder(std::move(ir_builder)) @@ -88,64 +89,64 @@ PackContext::PackContext(CodeBlob& code, SrcLocation loc, std::vector } void PackContext::storeInt(var_idx_t ir_idx, int len) const { - std::vector args = { ir_builder0, ir_idx, code.create_int(loc, len, "(storeW)") }; - code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), f_storeInt); + std::vector args = { ir_builder0, ir_idx, code.create_int(origin, len, "(storeW)") }; + code.emplace_back(origin, Op::_Call, ir_builder, std::move(args), f_storeInt); } void PackContext::storeUint(var_idx_t ir_idx, int len) const { - std::vector args = { ir_builder0, ir_idx, code.create_int(loc, len, "(storeW)") }; - code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), f_storeUint); + std::vector args = { ir_builder0, ir_idx, code.create_int(origin, len, "(storeW)") }; + code.emplace_back(origin, Op::_Call, ir_builder, std::move(args), f_storeUint); } void PackContext::storeUint_var(var_idx_t ir_idx, var_idx_t ir_len) const { std::vector args = { ir_builder0, ir_idx, ir_len }; - code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), f_storeUint); + code.emplace_back(origin, Op::_Call, ir_builder, std::move(args), f_storeUint); } void PackContext::storeBool(var_idx_t ir_idx) const { std::vector args = { ir_builder0, ir_idx }; - code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeBool")); + code.emplace_back(origin, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeBool")); } void PackContext::storeCoins(var_idx_t ir_idx) const { std::vector args = { ir_builder0, ir_idx }; - code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeCoins")); + code.emplace_back(origin, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeCoins")); } void PackContext::storeRef(var_idx_t ir_idx) const { std::vector args = { ir_builder0, ir_idx }; - code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeRef")); + code.emplace_back(origin, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeRef")); } void PackContext::storeMaybeRef(var_idx_t ir_idx) const { std::vector args = { ir_builder0, ir_idx }; - code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeMaybeRef")); + code.emplace_back(origin, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeMaybeRef")); } void PackContext::storeAddress(var_idx_t ir_idx) const { std::vector args = { ir_builder0, ir_idx }; - code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeAddress")); + code.emplace_back(origin, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeAddress")); } void PackContext::storeBuilder(var_idx_t ir_idx) const { std::vector args = { ir_builder0, ir_idx }; - code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeBuilder")); + code.emplace_back(origin, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeBuilder")); } void PackContext::storeSlice(var_idx_t ir_idx) const { std::vector args = { ir_builder0, ir_idx }; - code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeSlice")); + code.emplace_back(origin, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeSlice")); } void PackContext::storeOpcode(PackOpcode opcode) const { - std::vector args = { ir_builder0, code.create_int(loc, opcode.pack_prefix, "(struct-prefix)"), code.create_int(loc, opcode.prefix_len, "(storeW)") }; - code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), f_storeUint); + std::vector args = { ir_builder0, code.create_int(origin, opcode.pack_prefix, "(struct-prefix)"), code.create_int(origin, opcode.prefix_len, "(storeW)") }; + code.emplace_back(origin, Op::_Call, ir_builder, std::move(args), f_storeUint); } -UnpackContext::UnpackContext(CodeBlob& code, SrcLocation loc, std::vector ir_slice, const std::vector& ir_options) +UnpackContext::UnpackContext(CodeBlob& code, AnyV origin, std::vector ir_slice, const std::vector& ir_options) : code(code) - , loc(loc) + , origin(origin) , f_loadInt(lookup_function("slice.loadInt")) , f_loadUint(lookup_function("slice.loadUint")) , f_skipBits(lookup_function("slice.skipBits")) @@ -156,55 +157,55 @@ UnpackContext::UnpackContext(CodeBlob& code, SrcLocation loc, std::vector UnpackContext::loadInt(int len, const char* debug_desc) const { - std::vector args = { ir_slice0, code.create_int(loc, len, "(loadW)") }; - std::vector result = code.create_tmp_var(TypeDataInt::create(), loc, debug_desc); - code.emplace_back(loc, Op::_Call, std::vector{ir_slice0, result[0]}, std::move(args), f_loadInt); + std::vector args = { ir_slice0, code.create_int(origin, len, "(loadW)") }; + std::vector result = code.create_tmp_var(TypeDataInt::create(), origin, debug_desc); + code.emplace_back(origin, Op::_Call, std::vector{ir_slice0, result[0]}, std::move(args), f_loadInt); return result; } std::vector UnpackContext::loadUint(int len, const char* debug_desc) const { - std::vector args = { ir_slice0, code.create_int(loc, len, "(loadW)") }; - std::vector result = code.create_tmp_var(TypeDataInt::create(), loc, debug_desc); - code.emplace_back(loc, Op::_Call, std::vector{ir_slice0, result[0]}, std::move(args), f_loadUint); + std::vector args = { ir_slice0, code.create_int(origin, len, "(loadW)") }; + std::vector result = code.create_tmp_var(TypeDataInt::create(), origin, debug_desc); + code.emplace_back(origin, Op::_Call, std::vector{ir_slice0, result[0]}, std::move(args), f_loadUint); return result; } void UnpackContext::loadAndCheckOpcode(PackOpcode opcode) const { - std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); - std::vector args = { ir_slice0, code.create_int(loc, opcode.pack_prefix, "(pack-prefix)"), code.create_int(loc, opcode.prefix_len, "(prefix-len)") }; - code.emplace_back(loc, Op::_Call, std::vector{ir_slice0, ir_prefix_eq[0]}, std::move(args), lookup_function("slice.tryStripPrefix")); - std::vector args_assert = { option_throwIfOpcodeDoesNotMatch, ir_prefix_eq[0], code.create_int(loc, 0, "") }; - Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), origin, "(prefix-eq)"); + std::vector args = { ir_slice0, code.create_int(origin, opcode.pack_prefix, "(pack-prefix)"), code.create_int(origin, opcode.prefix_len, "(prefix-len)") }; + code.emplace_back(origin, Op::_Call, std::vector{ir_slice0, ir_prefix_eq[0]}, std::move(args), lookup_function("slice.tryStripPrefix")); + std::vector args_assert = { option_throwIfOpcodeDoesNotMatch, ir_prefix_eq[0], code.create_int(origin, 0, "") }; + Op& op_assert = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); op_assert.set_impure_flag(); } void UnpackContext::skipBits(int len) const { - std::vector args = { ir_slice0, code.create_int(loc, len, "(skipW)") }; - code.emplace_back(loc, Op::_Call, ir_slice, std::move(args), f_skipBits); + std::vector args = { ir_slice0, code.create_int(origin, len, "(skipW)") }; + code.emplace_back(origin, Op::_Call, ir_slice, std::move(args), f_skipBits); } void UnpackContext::skipBits_var(var_idx_t ir_len) const { std::vector args = { ir_slice0, ir_len }; - code.emplace_back(loc, Op::_Call, ir_slice, std::move(args), f_skipBits); + code.emplace_back(origin, Op::_Call, ir_slice, std::move(args), f_skipBits); } void UnpackContext::assertEndIfOption() const { - Op& if_assertEnd = code.emplace_back(loc, Op::_If, std::vector{option_assertEndAfterReading}); + Op& if_assertEnd = code.emplace_back(origin, Op::_If, std::vector{option_assertEndAfterReading}); { code.push_set_cur(if_assertEnd.block0); - Op& op_ends = code.emplace_back(loc, Op::_Call, std::vector{}, ir_slice, lookup_function("slice.assertEnd")); + Op& op_ends = code.emplace_back(origin, Op::_Call, std::vector{}, ir_slice, lookup_function("slice.assertEnd")); op_ends.set_impure_flag(); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { code.push_set_cur(if_assertEnd.block1); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } } void UnpackContext::throwInvalidOpcode() const { std::vector args_throw = { option_throwIfOpcodeDoesNotMatch }; - Op& op_throw = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_throw), lookup_function("__throw")); + Op& op_throw = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_throw), lookup_function("__throw")); op_throw.set_impure_flag(); } @@ -217,15 +218,15 @@ const LazyMatchOptions::MatchBlock* LazyMatchOptions::find_match_block(TypePtr v tolk_assert(false); } -void LazyMatchOptions::save_match_result_on_arm_end(CodeBlob& code, SrcLocation loc, const MatchBlock* arm_block, std::vector&& ir_arm_result, const std::vector& ir_match_expr_result) const { +void LazyMatchOptions::save_match_result_on_arm_end(CodeBlob& code, AnyV origin, const MatchBlock* arm_block, std::vector&& ir_arm_result, const std::vector& ir_match_expr_result) const { if (!is_statement) { // if it's `match` expression (not statement), then every arm has a result, assigned to a whole `match` result - ir_arm_result = transition_to_target_type(std::move(ir_arm_result), code, arm_block->block_expr_type, match_expr_type, loc); - code.emplace_back(loc, Op::_Let, ir_match_expr_result, std::move(ir_arm_result)); + ir_arm_result = transition_to_target_type(std::move(ir_arm_result), code, arm_block->block_expr_type, match_expr_type, origin); + code.emplace_back(origin, Op::_Let, ir_match_expr_result, std::move(ir_arm_result)); } else if (add_return_to_all_arms) { // if it's `match` statement, even if an arm is an expression, it's void, actually // moreover, if it's the last statement in a function, add implicit "return" to all match cases to produce IFJMP - code.emplace_back(loc, Op::_Return); + code.emplace_back(origin, Op::_Return); } } @@ -244,10 +245,10 @@ void LazyMatchOptions::save_match_result_on_arm_end(CodeBlob& code, SrcLocation struct ISerializer { virtual ~ISerializer() = default; - virtual void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) = 0; - virtual std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) = 0; + virtual void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) = 0; + virtual std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) = 0; - virtual void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) = 0; + virtual void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) = 0; virtual PackSize estimate(const EstimateContext* ctx) = 0; }; @@ -258,7 +259,7 @@ struct S_IntN final : ISerializer { explicit S_IntN(int n_bits, bool is_unsigned) : n_bits(n_bits), is_unsigned(is_unsigned) {} - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { if (is_unsigned) { ctx->storeUint(rvect[0], n_bits); } else { @@ -266,7 +267,7 @@ struct S_IntN final : ISerializer { } } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { if (is_unsigned) { return ctx->loadUint(n_bits, "(loaded-uint)"); } else { @@ -274,7 +275,7 @@ struct S_IntN final : ISerializer { } } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { ctx->skipBits(n_bits); } @@ -290,23 +291,23 @@ struct S_VariadicIntN final : ISerializer { explicit S_VariadicIntN(int n_bits, bool is_unsigned) : n_bits(n_bits), is_unsigned(is_unsigned) {} - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { FunctionPtr f_storeVarInt = lookup_function("builder.__storeVarInt"); - std::vector args = { ctx->ir_builder0, rvect[0], code.create_int(loc, n_bits, "(n-bits)"), code.create_int(loc, is_unsigned, "(is-unsigned)") }; - code.emplace_back(loc, Op::_Call, ctx->ir_builder, std::move(args), f_storeVarInt); + std::vector args = { ctx->ir_builder0, rvect[0], code.create_int(origin, n_bits, "(n-bits)"), code.create_int(origin, is_unsigned, "(is-unsigned)") }; + code.emplace_back(origin, Op::_Call, ctx->ir_builder, std::move(args), f_storeVarInt); } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { FunctionPtr f_loadVarInt = lookup_function("slice.__loadVarInt"); - std::vector args = { ctx->ir_slice0, code.create_int(loc, n_bits, "(n-bits)"), code.create_int(loc, is_unsigned, "(is-unsigned)") }; - std::vector result = code.create_tmp_var(TypeDataInt::create(), loc, "(loaded-varint)"); - code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, result[0]}, std::move(args), f_loadVarInt); + std::vector args = { ctx->ir_slice0, code.create_int(origin, n_bits, "(n-bits)"), code.create_int(origin, is_unsigned, "(is-unsigned)") }; + std::vector result = code.create_tmp_var(TypeDataInt::create(), origin, "(loaded-varint)"); + code.emplace_back(origin, Op::_Call, std::vector{ctx->ir_slice0, result[0]}, std::move(args), f_loadVarInt); return result; } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { // no TVM instruction to skip, just load but don't use the result - unpack(ctx, code, loc); + unpack(ctx, code, origin); } PackSize estimate(const EstimateContext* ctx) override { @@ -324,44 +325,44 @@ struct S_BitsN final : ISerializer { explicit S_BitsN(int n_width, bool is_bits) : n_bits(is_bits ? n_width : n_width * 8) {} - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { tolk_assert(rvect.size() == 1); - Op& if_disabled_by_user = code.emplace_back(loc, Op::_If, std::vector{ctx->option_skipBitsNValidation}); + Op& if_disabled_by_user = code.emplace_back(origin, Op::_If, std::vector{ctx->option_skipBitsNValidation}); { code.push_set_cur(if_disabled_by_user.block0); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { code.push_set_cur(if_disabled_by_user.block1); FunctionPtr f_assert = lookup_function("__throw_if_unless"); constexpr int EXCNO = 9; - std::vector ir_counts = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create()}), loc, "(slice-size)"); - code.emplace_back(loc, Op::_Call, ir_counts, rvect, lookup_function("slice.remainingBitsAndRefsCount")); - std::vector args_assert0 = { code.create_int(loc, EXCNO, "(excno)"), ir_counts[1], code.create_int(loc, 1, "") }; - Op& op_assert0 = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert0), f_assert); + std::vector ir_counts = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create()}), origin, "(slice-size)"); + code.emplace_back(origin, Op::_Call, ir_counts, rvect, lookup_function("slice.remainingBitsAndRefsCount")); + std::vector args_assert0 = { code.create_int(origin, EXCNO, "(excno)"), ir_counts[1], code.create_int(origin, 1, "") }; + Op& op_assert0 = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_assert0), f_assert); op_assert0.set_impure_flag(); - std::vector ir_eq_n = code.create_tmp_var(TypeDataInt::create(), loc, "(eq-n)"); - code.emplace_back(loc, Op::_Call, ir_eq_n, std::vector{ir_counts[0], code.create_int(loc, n_bits, "(n-bits)")}, lookup_function("_==_")); - std::vector args_assertN = { code.create_int(loc, EXCNO, "(excno)"), ir_eq_n[0], code.create_int(loc, 0, "") }; - Op& op_assertN = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assertN), f_assert); + std::vector ir_eq_n = code.create_tmp_var(TypeDataInt::create(), origin, "(eq-n)"); + code.emplace_back(origin, Op::_Call, ir_eq_n, std::vector{ir_counts[0], code.create_int(origin, n_bits, "(n-bits)")}, lookup_function("_==_")); + std::vector args_assertN = { code.create_int(origin, EXCNO, "(excno)"), ir_eq_n[0], code.create_int(origin, 0, "") }; + Op& op_assertN = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_assertN), f_assert); op_assertN.set_impure_flag(); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } ctx->storeSlice(rvect[0]); } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { FunctionPtr f_loadBits = lookup_function("slice.loadBits"); - std::vector args = { ctx->ir_slice0, code.create_int(loc, n_bits, "(loadW)") }; - std::vector ir_result = code.create_tmp_var(TypeDataSlice::create(), loc, "(loaded-slice)"); - code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadBits); + std::vector args = { ctx->ir_slice0, code.create_int(origin, n_bits, "(loadW)") }; + std::vector ir_result = code.create_tmp_var(TypeDataSlice::create(), origin, "(loaded-slice)"); + code.emplace_back(origin, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadBits); return ir_result; } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { ctx->skipBits(n_bits); } @@ -371,16 +372,16 @@ struct S_BitsN final : ISerializer { }; struct S_Bool final : ISerializer { - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { tolk_assert(rvect.size() == 1); ctx->storeBool(rvect[0]); } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { return ctx->loadInt(1, "(loaded-bool)"); } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { ctx->skipBits(1); } @@ -390,24 +391,24 @@ struct S_Bool final : ISerializer { }; struct S_RawTVMcell final : ISerializer { - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { tolk_assert(rvect.size() == 1); ctx->storeRef(rvect[0]); } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { FunctionPtr f_loadRef = lookup_function("slice.loadRef"); std::vector args = ctx->ir_slice; - std::vector ir_result = code.create_tmp_var(TypeDataCell::create(), loc, "(loaded-cell)"); - code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadRef); + std::vector ir_result = code.create_tmp_var(TypeDataCell::create(), origin, "(loaded-cell)"); + code.emplace_back(origin, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadRef); return ir_result; } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { FunctionPtr f_loadRef = lookup_function("slice.loadRef"); std::vector args = ctx->ir_slice; - std::vector dummy_loaded = code.create_tmp_var(TypeDataCell::create(), loc, "(loaded-cell)"); - code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, dummy_loaded[0]}, std::move(args), f_loadRef); + std::vector dummy_loaded = code.create_tmp_var(TypeDataCell::create(), origin, "(loaded-cell)"); + code.emplace_back(origin, Op::_Call, std::vector{ctx->ir_slice0, dummy_loaded[0]}, std::move(args), f_loadRef); } PackSize estimate(const EstimateContext* ctx) override { @@ -416,22 +417,22 @@ struct S_RawTVMcell final : ISerializer { }; struct S_RawTVMcellOrNull final : ISerializer { - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { tolk_assert(rvect.size() == 1); ctx->storeMaybeRef(rvect[0]); } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { FunctionPtr f_loadMaybeRef = lookup_function("slice.loadMaybeRef"); std::vector args = ctx->ir_slice; - std::vector ir_result = code.create_tmp_var(TypeDataCell::create(), loc, "(loaded-cell)"); - code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadMaybeRef); + std::vector ir_result = code.create_tmp_var(TypeDataCell::create(), origin, "(loaded-cell)"); + code.emplace_back(origin, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadMaybeRef); return ir_result; } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { FunctionPtr f_skipMaybeRef = lookup_function("slice.skipMaybeRef"); - code.emplace_back(loc, Op::_Call, ctx->ir_slice, ctx->ir_slice, f_skipMaybeRef); + code.emplace_back(origin, Op::_Call, ctx->ir_slice, ctx->ir_slice, f_skipMaybeRef); } PackSize estimate(const EstimateContext* ctx) override { @@ -440,22 +441,22 @@ struct S_RawTVMcellOrNull final : ISerializer { }; struct S_Coins final : ISerializer { - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { tolk_assert(rvect.size() == 1); ctx->storeCoins(rvect[0]); } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { FunctionPtr f_loadCoins = lookup_function("slice.loadCoins"); std::vector args = ctx->ir_slice; - std::vector ir_result = code.create_tmp_var(TypeDataInt::create(), loc, "(loaded-coins)"); - code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadCoins); + std::vector ir_result = code.create_tmp_var(TypeDataInt::create(), origin, "(loaded-coins)"); + code.emplace_back(origin, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadCoins); return ir_result; } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { // no TVM instruction to skip, just load but don't use the result - unpack(ctx, code, loc); + unpack(ctx, code, origin); } PackSize estimate(const EstimateContext* ctx) override { @@ -464,23 +465,23 @@ struct S_Coins final : ISerializer { }; struct S_Address final : ISerializer { - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { tolk_assert(rvect.size() == 1); ctx->storeAddress(rvect[0]); } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { FunctionPtr f_loadAddress = lookup_function("slice.loadAddress"); - std::vector ir_address = code.create_tmp_var(TypeDataSlice::create(), loc, "(loaded-addr)"); - code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_address[0]}, ctx->ir_slice, f_loadAddress); + std::vector ir_address = code.create_tmp_var(TypeDataSlice::create(), origin, "(loaded-addr)"); + code.emplace_back(origin, Op::_Call, std::vector{ctx->ir_slice0, ir_address[0]}, ctx->ir_slice, f_loadAddress); return ir_address; } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { // we can't do just // ctx->skipBits(2 + 1 + 8 + 256); // because it may be addr_none or addr_extern; there is no "skip address" in TVM, so just load it - unpack(ctx, code, loc); + unpack(ctx, code, origin); } PackSize estimate(const EstimateContext* ctx) override { @@ -492,20 +493,20 @@ struct S_Address final : ISerializer { }; struct S_RemainingBitsAndRefs final : ISerializer { - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { tolk_assert(rvect.size() == 1); ctx->storeSlice(rvect[0]); } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { - std::vector ir_rem_slice = code.create_tmp_var(TypeDataSlice::create(), loc, "(remainder)"); - code.emplace_back(loc, Op::_Let, ir_rem_slice, ctx->ir_slice); - code.emplace_back(loc, Op::_Call, ctx->ir_slice, std::vector{}, lookup_function("createEmptySlice")); + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { + std::vector ir_rem_slice = code.create_tmp_var(TypeDataSlice::create(), origin, "(remainder)"); + code.emplace_back(origin, Op::_Let, ir_rem_slice, ctx->ir_slice); + code.emplace_back(origin, Op::_Call, ctx->ir_slice, std::vector{}, lookup_function("createEmptySlice")); return ir_rem_slice; } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { - unpack(ctx, code, loc); + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { + unpack(ctx, code, origin); } PackSize estimate(const EstimateContext* ctx) override { @@ -514,17 +515,17 @@ struct S_RemainingBitsAndRefs final : ISerializer { }; struct S_Builder final : ISerializer { - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { tolk_assert(rvect.size() == 1); std::vector args = { ctx->ir_builder0, rvect[0] }; - code.emplace_back(loc, Op::_Call, ctx->ir_builder, std::move(args), lookup_function("builder.storeBuilder")); + code.emplace_back(origin, Op::_Call, ctx->ir_builder, std::move(args), lookup_function("builder.storeBuilder")); } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { tolk_assert(false); // `builder` can only be used for writing, checked earlier } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { tolk_assert(false); // `builder` can only be used for writing, checked earlier } @@ -534,16 +535,16 @@ struct S_Builder final : ISerializer { }; struct S_Slice final : ISerializer { - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { tolk_assert(rvect.size() == 1); ctx->storeSlice(rvect[0]); } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { tolk_assert(false); // `slice` can only be used for writing, checked earlier } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { tolk_assert(false); // `slice` can only be used for writing, checked earlier } @@ -553,19 +554,19 @@ struct S_Slice final : ISerializer { }; struct S_Null final : ISerializer { - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { // while `null` itself is not serializable, it may be contained inside a union: // `int32 | int64 | null`, for example; // then the compiler generates prefixes for every variant, and `null` variant does nothing } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { - std::vector ir_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null)"); - code.emplace_back(loc, Op::_Call, ir_null, std::vector{}, lookup_function("__null")); + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { + std::vector ir_null = code.create_tmp_var(TypeDataNullLiteral::create(), origin, "(null)"); + code.emplace_back(origin, Op::_Call, ir_null, std::vector{}, lookup_function("__null")); return ir_null; } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { } PackSize estimate(const EstimateContext* ctx) override { @@ -574,15 +575,15 @@ struct S_Null final : ISerializer { }; struct S_Never final : ISerializer { - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { tolk_assert(rvect.empty()); } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { return {}; } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { } PackSize estimate(const EstimateContext* ctx) override { @@ -599,56 +600,56 @@ struct S_Maybe final : ISerializer { , or_null(t_union->or_null) { } - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { - std::vector ir_is_null = pre_compile_is_type(code, t_union, TypeDataNullLiteral::create(), rvect, loc, "(is-null)"); - Op& if_op = code.emplace_back(loc, Op::_If, ir_is_null); + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { + std::vector ir_is_null = pre_compile_is_type(code, t_union, TypeDataNullLiteral::create(), rvect, origin, "(is-null)"); + Op& if_op = code.emplace_back(origin, Op::_If, ir_is_null); { code.push_set_cur(if_op.block0); - ctx->storeUint(code.create_int(loc, 0, "(maybeBit)"), 1); - code.close_pop_cur(loc); + ctx->storeUint(code.create_int(origin, 0, "(maybeBit)"), 1); + code.close_pop_cur(origin); } { code.push_set_cur(if_op.block1); - ctx->storeUint(code.create_int(loc, 1, "(maybeBit)"), 1); - rvect = transition_to_target_type(std::move(rvect), code, t_union, t_union->or_null, loc); + ctx->storeUint(code.create_int(origin, 1, "(maybeBit)"), 1); + rvect = transition_to_target_type(std::move(rvect), code, t_union, t_union->or_null, origin); ctx->generate_pack_any(t_union->or_null, std::move(rvect)); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { - std::vector ir_result = code.create_tmp_var(t_union, loc, "(loaded-maybe)"); + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { + std::vector ir_result = code.create_tmp_var(t_union, origin, "(loaded-maybe)"); std::vector ir_not_null = { ctx->loadUint(1, "(maybeBit)") }; - Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_not_null)); + Op& if_op = code.emplace_back(origin, Op::_If, std::move(ir_not_null)); { code.push_set_cur(if_op.block0); std::vector rvect_maybe = ctx->generate_unpack_any(t_union->or_null); - rvect_maybe = transition_to_target_type(std::move(rvect_maybe), code, t_union->or_null, t_union, loc); - code.emplace_back(loc, Op::_Let, ir_result, std::move(rvect_maybe)); - code.close_pop_cur(loc); + rvect_maybe = transition_to_target_type(std::move(rvect_maybe), code, t_union->or_null, t_union, origin); + code.emplace_back(origin, Op::_Let, ir_result, std::move(rvect_maybe)); + code.close_pop_cur(origin); } { code.push_set_cur(if_op.block1); - std::vector rvect_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(maybe-null)"); - code.emplace_back(loc, Op::_Call, rvect_null, std::vector{}, lookup_function("__null")); - rvect_null = transition_to_target_type(std::move(rvect_null), code, TypeDataNullLiteral::create(), t_union, loc); - code.emplace_back(loc, Op::_Let, ir_result, std::move(rvect_null)); - code.close_pop_cur(loc); + std::vector rvect_null = code.create_tmp_var(TypeDataNullLiteral::create(), origin, "(maybe-null)"); + code.emplace_back(origin, Op::_Call, rvect_null, std::vector{}, lookup_function("__null")); + rvect_null = transition_to_target_type(std::move(rvect_null), code, TypeDataNullLiteral::create(), t_union, origin); + code.emplace_back(origin, Op::_Let, ir_result, std::move(rvect_null)); + code.close_pop_cur(origin); } return ir_result; } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { std::vector ir_not_null = { ctx->loadUint(1, "(maybeBit)") }; - Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_not_null)); + Op& if_op = code.emplace_back(origin, Op::_If, std::move(ir_not_null)); { code.push_set_cur(if_op.block0); ctx->generate_skip_any(t_union->or_null); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { code.push_set_cur(if_op.block1); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } } @@ -669,86 +670,86 @@ struct S_Either final : ISerializer { , t_right(t_union->variants[1]) { } - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { - std::vector ir_is_right = pre_compile_is_type(code, t_union, t_right, rvect, loc, "(is-right)"); - Op& if_op = code.emplace_back(loc, Op::_If, ir_is_right); + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { + std::vector ir_is_right = pre_compile_is_type(code, t_union, t_right, rvect, origin, "(is-right)"); + Op& if_op = code.emplace_back(origin, Op::_If, ir_is_right); { code.push_set_cur(if_op.block0); - ctx->storeUint(code.create_int(loc, 1, "(eitherBit)"), 1); - std::vector rvect_right = transition_to_target_type(std::vector(rvect), code, t_union, t_right, loc); + ctx->storeUint(code.create_int(origin, 1, "(eitherBit)"), 1); + std::vector rvect_right = transition_to_target_type(std::vector(rvect), code, t_union, t_right, origin); ctx->generate_pack_any(t_right, std::move(rvect_right)); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { code.push_set_cur(if_op.block1); - ctx->storeUint(code.create_int(loc, 0, "(eitherBit)"), 1); - std::vector rvect_left = transition_to_target_type(std::move(rvect), code, t_union, t_left, loc); + ctx->storeUint(code.create_int(origin, 0, "(eitherBit)"), 1); + std::vector rvect_left = transition_to_target_type(std::move(rvect), code, t_union, t_left, origin); ctx->generate_pack_any(t_left, std::move(rvect_left)); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { - std::vector ir_result = code.create_tmp_var(t_union, loc, "(loaded-either)"); + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { + std::vector ir_result = code.create_tmp_var(t_union, origin, "(loaded-either)"); std::vector ir_is_right = ctx->loadUint(1, "(eitherBit)"); - Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_is_right)); + Op& if_op = code.emplace_back(origin, Op::_If, std::move(ir_is_right)); { code.push_set_cur(if_op.block0); std::vector rvect_right = ctx->generate_unpack_any(t_right); - rvect_right = transition_to_target_type(std::move(rvect_right), code, t_right, t_union, loc); - code.emplace_back(loc, Op::_Let, ir_result, std::move(rvect_right)); - code.close_pop_cur(loc); + rvect_right = transition_to_target_type(std::move(rvect_right), code, t_right, t_union, origin); + code.emplace_back(origin, Op::_Let, ir_result, std::move(rvect_right)); + code.close_pop_cur(origin); } { code.push_set_cur(if_op.block1); std::vector rvect_left = ctx->generate_unpack_any(t_left); - rvect_left = transition_to_target_type(std::move(rvect_left), code, t_left, t_union, loc); - code.emplace_back(loc, Op::_Let, ir_result, std::move(rvect_left)); - code.close_pop_cur(loc); + rvect_left = transition_to_target_type(std::move(rvect_left), code, t_left, t_union, origin); + code.emplace_back(origin, Op::_Let, ir_result, std::move(rvect_left)); + code.close_pop_cur(origin); } return ir_result; } - std::vector lazy_match(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc, const LazyMatchOptions& options) const { + std::vector lazy_match(const UnpackContext* ctx, CodeBlob& code, AnyV origin, const LazyMatchOptions& options) const { for (const LazyMatchOptions::MatchBlock& m : options.match_blocks) { if (m.arm_variant == nullptr) { // `else => ...` not allowed for Either // it's not the best place to fire an error, but let it be - throw ParseError(loc, "`else` is unreachable, because this `match` has only two options (0/1 prefixes)"); + fire(SrcRange::empty_at_start(m.v_body->range), "`else` is unreachable, because this `match` has only two options (0/1 prefixes)"); } } tolk_assert(options.match_blocks.size() == 2); - std::vector ir_result = code.create_tmp_var(options.match_expr_type, loc, "(match-expression)"); + std::vector ir_result = code.create_tmp_var(options.match_expr_type, origin, "(match-expression)"); std::vector ir_is_right = ctx->loadUint(1, "(eitherBit)"); - Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_is_right)); + Op& if_op = code.emplace_back(origin, Op::_If, std::move(ir_is_right)); { code.push_set_cur(if_op.block0); const LazyMatchOptions::MatchBlock* m_block = options.find_match_block(t_right); std::vector ith_result = pre_compile_expr(m_block->v_body, code); - options.save_match_result_on_arm_end(code, loc, m_block, std::move(ith_result), ir_result); - code.close_pop_cur(loc); + options.save_match_result_on_arm_end(code, origin, m_block, std::move(ith_result), ir_result); + code.close_pop_cur(origin); } { code.push_set_cur(if_op.block1); const LazyMatchOptions::MatchBlock* m_block = options.find_match_block(t_left); std::vector ith_result = pre_compile_expr(m_block->v_body, code); - options.save_match_result_on_arm_end(code, loc, m_block, std::move(ith_result), ir_result); - code.close_pop_cur(loc); + options.save_match_result_on_arm_end(code, origin, m_block, std::move(ith_result), ir_result); + code.close_pop_cur(origin); } return ir_result; } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { std::vector ir_is_right = ctx->loadUint(1, "(eitherBit)"); - Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_is_right)); + Op& if_op = code.emplace_back(origin, Op::_If, std::move(ir_is_right)); { code.push_set_cur(if_op.block0); ctx->generate_skip_any(t_right); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { code.push_set_cur(if_op.block1); ctx->generate_skip_any(t_left); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } } @@ -768,59 +769,59 @@ struct S_MultipleConstructors final : ISerializer { tolk_assert(this->opcodes.size() == t_union->variants.size()); } - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { for (int i = 0; i < t_union->size() - 1; ++i) { TypePtr variant = t_union->variants[i]; - std::vector ir_eq_ith = pre_compile_is_type(code, t_union, variant, rvect, loc, "(arm-cond-eq)"); - Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_eq_ith)); + std::vector ir_eq_ith = pre_compile_is_type(code, t_union, variant, rvect, origin, "(arm-cond-eq)"); + Op& if_op = code.emplace_back(origin, Op::_If, std::move(ir_eq_ith)); code.push_set_cur(if_op.block0); - std::vector ith_rvect = transition_to_target_type(std::vector(rvect), code, t_union, variant, loc); - ctx->storeUint(code.create_int(loc, opcodes[i].pack_prefix, "(ith-prefix)"), opcodes[i].prefix_len); + std::vector ith_rvect = transition_to_target_type(std::vector(rvect), code, t_union, variant, origin); + ctx->storeUint(code.create_int(origin, opcodes[i].pack_prefix, "(ith-prefix)"), opcodes[i].prefix_len); ctx->generate_pack_any(variant, std::move(ith_rvect), PrefixWriteMode::DoNothingAlreadyWritten); - code.close_pop_cur(loc); + code.close_pop_cur(origin); code.push_set_cur(if_op.block1); // open ELSE } // we're inside the last ELSE TypePtr last_variant = t_union->variants.back(); - std::vector last_rvect = transition_to_target_type(std::move(rvect), code, t_union, last_variant, loc); - ctx->storeUint(code.create_int(loc, opcodes.back().pack_prefix, "(ith-prefix)"), opcodes.back().prefix_len); + std::vector last_rvect = transition_to_target_type(std::move(rvect), code, t_union, last_variant, origin); + ctx->storeUint(code.create_int(origin, opcodes.back().pack_prefix, "(ith-prefix)"), opcodes.back().prefix_len); ctx->generate_pack_any(last_variant, std::move(last_rvect), PrefixWriteMode::DoNothingAlreadyWritten); for (int i = 0; i < t_union->size() - 1; ++i) { - code.close_pop_cur(loc); // close all outer IFs + code.close_pop_cur(origin); // close all outer IFs } } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { // assume that opcodes (either automatically generated or manually specified) // form a valid prefix tree, and the order of reading does not matter; we'll definitely match the one; FunctionPtr f_tryStripPrefix = lookup_function("slice.tryStripPrefix"); - std::vector ir_result = code.create_tmp_var(t_union, loc, "(loaded-union)"); - std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); + std::vector ir_result = code.create_tmp_var(t_union, origin, "(loaded-union)"); + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), origin, "(prefix-eq)"); for (int i = 0; i < t_union->size(); ++i) { TypePtr variant = t_union->variants[i]; - std::vector args = { ctx->ir_slice0, code.create_int(loc, opcodes[i].pack_prefix, "(pack-prefix)"), code.create_int(loc, opcodes[i].prefix_len, "(prefix-len)") }; - code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), f_tryStripPrefix); - Op& if_prefix_eq = code.emplace_back(loc, Op::_If, ir_prefix_eq); + std::vector args = { ctx->ir_slice0, code.create_int(origin, opcodes[i].pack_prefix, "(pack-prefix)"), code.create_int(origin, opcodes[i].prefix_len, "(prefix-len)") }; + code.emplace_back(origin, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), f_tryStripPrefix); + Op& if_prefix_eq = code.emplace_back(origin, Op::_If, ir_prefix_eq); code.push_set_cur(if_prefix_eq.block0); std::vector ith_rvect = ctx->generate_unpack_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); - ith_rvect = transition_to_target_type(std::move(ith_rvect), code, variant, t_union, loc); - code.emplace_back(loc, Op::_Let, ir_result, std::move(ith_rvect)); - code.close_pop_cur(loc); + ith_rvect = transition_to_target_type(std::move(ith_rvect), code, variant, t_union, origin); + code.emplace_back(origin, Op::_Let, ir_result, std::move(ith_rvect)); + code.close_pop_cur(origin); code.push_set_cur(if_prefix_eq.block1); // open ELSE } // we're inside last ELSE ctx->throwInvalidOpcode(); for (int j = 0; j < t_union->size(); ++j) { - code.close_pop_cur(loc); // close all outer IFs + code.close_pop_cur(origin); // close all outer IFs } return ir_result; } - std::vector lazy_match(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc, const LazyMatchOptions& options) const { + std::vector lazy_match(const UnpackContext* ctx, CodeBlob& code, AnyV origin, const LazyMatchOptions& options) const { std::vector opcodes_order_mapping(t_union->size(), -1); const LazyMatchOptions::MatchBlock* else_block = nullptr; for (int i = 0; i < static_cast(options.match_blocks.size()); ++i) { @@ -836,53 +837,53 @@ struct S_MultipleConstructors final : ISerializer { FunctionPtr f_tryStripPrefix = lookup_function("slice.tryStripPrefix"); - std::vector ir_result = code.create_tmp_var(options.match_expr_type, loc, "(match-expression)"); - std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); + std::vector ir_result = code.create_tmp_var(options.match_expr_type, origin, "(match-expression)"); + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), origin, "(prefix-eq)"); for (int i = 0; i < t_union->size(); ++i) { StructData::PackOpcode opcode = opcodes[opcodes_order_mapping[i]]; - std::vector args = { ctx->ir_slice0, code.create_int(loc, opcode.pack_prefix, "(pack-prefix)"), code.create_int(loc, opcode.prefix_len, "(prefix-len)") }; - code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), f_tryStripPrefix); - Op& if_op = code.emplace_back(loc, Op::_If, ir_prefix_eq); + std::vector args = { ctx->ir_slice0, code.create_int(origin, opcode.pack_prefix, "(pack-prefix)"), code.create_int(origin, opcode.prefix_len, "(prefix-len)") }; + code.emplace_back(origin, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), f_tryStripPrefix); + Op& if_op = code.emplace_back(origin, Op::_If, ir_prefix_eq); code.push_set_cur(if_op.block0); std::vector ith_result = pre_compile_expr(options.match_blocks[i].v_body, code); - options.save_match_result_on_arm_end(code, loc, &options.match_blocks[i], std::move(ith_result), ir_result); - code.close_pop_cur(loc); + options.save_match_result_on_arm_end(code, origin, &options.match_blocks[i], std::move(ith_result), ir_result); + code.close_pop_cur(origin); code.push_set_cur(if_op.block1); // open ELSE } if (else_block) { std::vector else_result = pre_compile_expr(else_block->v_body, code); - options.save_match_result_on_arm_end(code, loc, else_block, std::move(else_result), ir_result); + options.save_match_result_on_arm_end(code, origin, else_block, std::move(else_result), ir_result); } else { ctx->throwInvalidOpcode(); } for (int j = 0; j < t_union->size(); ++j) { - code.close_pop_cur(loc); // close all outer IFs + code.close_pop_cur(origin); // close all outer IFs } return ir_result; } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { FunctionPtr f_tryStripPrefix = lookup_function("slice.tryStripPrefix"); - std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), origin, "(prefix-eq)"); for (int i = 0; i < t_union->size(); ++i) { TypePtr variant = t_union->variants[i]; - std::vector args = { ctx->ir_slice0, code.create_int(loc, opcodes[i].pack_prefix, "(pack-prefix)"), code.create_int(loc, opcodes[i].prefix_len, "(prefix-len)") }; - code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), f_tryStripPrefix); - Op& if_prefix_eq = code.emplace_back(loc, Op::_If, ir_prefix_eq); + std::vector args = { ctx->ir_slice0, code.create_int(origin, opcodes[i].pack_prefix, "(pack-prefix)"), code.create_int(origin, opcodes[i].prefix_len, "(prefix-len)") }; + code.emplace_back(origin, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), f_tryStripPrefix); + Op& if_prefix_eq = code.emplace_back(origin, Op::_If, ir_prefix_eq); code.push_set_cur(if_prefix_eq.block0); ctx->generate_skip_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); - code.close_pop_cur(loc); + code.close_pop_cur(origin); code.push_set_cur(if_prefix_eq.block1); // open ELSE } // we're inside last ELSE ctx->throwInvalidOpcode(); for (int j = 0; j < t_union->size(); ++j) { - code.close_pop_cur(loc); // close all outer IFs + code.close_pop_cur(origin); // close all outer IFs } } @@ -906,7 +907,7 @@ struct S_Tensor final : ISerializer { : t_tensor(t_tensor) { } - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { int stack_offset = 0; for (TypePtr item : t_tensor->items) { int stack_width = item->get_width_on_stack(); @@ -917,7 +918,7 @@ struct S_Tensor final : ISerializer { tolk_assert(stack_offset == t_tensor->get_width_on_stack()); } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { std::vector tensor_vars; tensor_vars.reserve(t_tensor->get_width_on_stack()); for (TypePtr item : t_tensor->items) { @@ -928,7 +929,7 @@ struct S_Tensor final : ISerializer { return tensor_vars; } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { for (TypePtr item : t_tensor->items) { ctx->generate_skip_any(item); } @@ -950,7 +951,7 @@ struct S_CustomStruct final : ISerializer { : struct_ref(struct_ref) { } - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { if (struct_ref->opcode.exists() && ctx->get_prefix_mode() == PrefixWriteMode::WritePrefixOfStruct) { ctx->storeOpcode(struct_ref->opcode); } @@ -965,7 +966,7 @@ struct S_CustomStruct final : ISerializer { tolk_assert(stack_offset == TypeDataStruct::create(struct_ref)->get_width_on_stack()); } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { if (struct_ref->opcode.exists() && ctx->get_prefix_mode() == PrefixReadMode::LoadAndCheck) { ctx->loadAndCheckOpcode(struct_ref->opcode); } @@ -981,7 +982,7 @@ struct S_CustomStruct final : ISerializer { return ir_struct; } - std::vector lazy_match(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc, const LazyMatchOptions& options) const { + std::vector lazy_match(const UnpackContext* ctx, CodeBlob& code, AnyV origin, const LazyMatchOptions& options) const { const LazyMatchOptions::MatchBlock* when_block = nullptr; // Point => ... const LazyMatchOptions::MatchBlock* else_block = nullptr; // else => ... for (const LazyMatchOptions::MatchBlock& match_block : options.match_blocks) { @@ -993,38 +994,38 @@ struct S_CustomStruct final : ISerializer { } } - std::vector ir_result = code.create_tmp_var(options.match_expr_type, loc, "(match-expression)"); - std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); + std::vector ir_result = code.create_tmp_var(options.match_expr_type, origin, "(match-expression)"); + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), origin, "(prefix-eq)"); StructData::PackOpcode opcode = struct_ref->opcode; if (opcode.exists()) { // it's `match` over a struct (makes sense for a struct with prefix and `else` branch) - std::vector args = { ctx->ir_slice0, code.create_int(loc, opcode.pack_prefix, "(pack-prefix)"), code.create_int(loc, opcode.prefix_len, "(prefix-len)") }; - code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), lookup_function("slice.tryStripPrefix")); + std::vector args = { ctx->ir_slice0, code.create_int(origin, opcode.pack_prefix, "(pack-prefix)"), code.create_int(origin, opcode.prefix_len, "(prefix-len)") }; + code.emplace_back(origin, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), lookup_function("slice.tryStripPrefix")); } else { - code.emplace_back(loc, Op::_Let, ir_prefix_eq, std::vector{code.create_int(loc, -1, "(true)")}); + code.emplace_back(origin, Op::_Let, ir_prefix_eq, std::vector{code.create_int(origin, -1, "(true)")}); } - Op& if_op = code.emplace_back(loc, Op::_If, ir_prefix_eq); + Op& if_op = code.emplace_back(origin, Op::_If, ir_prefix_eq); { code.push_set_cur(if_op.block0); std::vector when_result = pre_compile_expr(when_block->v_body, code); - options.save_match_result_on_arm_end(code, loc, when_block, std::move(when_result), ir_result); - code.close_pop_cur(loc); + options.save_match_result_on_arm_end(code, origin, when_block, std::move(when_result), ir_result); + code.close_pop_cur(origin); } { code.push_set_cur(if_op.block1); if (else_block) { std::vector else_result = pre_compile_expr(else_block->v_body, code); - options.save_match_result_on_arm_end(code, loc, else_block, std::move(else_result), ir_result); + options.save_match_result_on_arm_end(code, origin, else_block, std::move(else_result), ir_result); } else { ctx->throwInvalidOpcode(); } - code.close_pop_cur(loc); + code.close_pop_cur(origin); } return ir_result; } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { if (struct_ref->opcode.exists() && ctx->get_prefix_mode() == PrefixReadMode::LoadAndCheck) { ctx->loadAndCheckOpcode(struct_ref->opcode); } @@ -1055,12 +1056,12 @@ struct S_IntegerEnum final : ISerializer { : enum_ref(enum_ref) { } - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { TypePtr intN = calculate_intN_to_serialize_enum(enum_ref); return ctx->generate_pack_any(intN, std::move(rvect)); } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { const TypeDataIntN* intN = calculate_intN_to_serialize_enum(enum_ref)->try_as(); std::vector ir_num = ctx->generate_unpack_any(intN); @@ -1079,44 +1080,44 @@ struct S_IntegerEnum final : ISerializer { td::RefInt256 min_value = enum_ref->members.front()->computed_value; bool dont_check_min = intN->is_unsigned && min_value == 0; if (!dont_check_min) { // LDU can't load < 0 - std::vector ir_min_value = code.create_tmp_var(TypeDataInt::create(), loc, "(enum-min)"); - code.emplace_back(loc, Op::_IntConst, ir_min_value, min_value); - std::vector ir_lt_min = code.create_tmp_var(TypeDataInt::create(), loc, "(enum-lt-min)"); - code.emplace_back(loc, Op::_Call, ir_lt_min, std::vector{ir_num[0], ir_min_value[0]}, lookup_function("_<_")); - std::vector args_assert1 = { code.create_int(loc, 5, "(excno)"), ir_lt_min[0], code.create_int(loc, 1, "") }; - Op& op_assert1 = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert1), lookup_function("__throw_if_unless")); + std::vector ir_min_value = code.create_tmp_var(TypeDataInt::create(), origin, "(enum-min)"); + code.emplace_back(origin, Op::_IntConst, ir_min_value, min_value); + std::vector ir_lt_min = code.create_tmp_var(TypeDataInt::create(), origin, "(enum-lt-min)"); + code.emplace_back(origin, Op::_Call, ir_lt_min, std::vector{ir_num[0], ir_min_value[0]}, lookup_function("_<_")); + std::vector args_assert1 = { code.create_int(origin, 5, "(excno)"), ir_lt_min[0], code.create_int(origin, 1, "") }; + Op& op_assert1 = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_assert1), lookup_function("__throw_if_unless")); op_assert1.set_impure_flag(); } td::RefInt256 max_value = enum_ref->members.back()->computed_value; bool dont_check_max = intN->is_unsigned && max_value == (1ULL << intN->n_bits) - 1; if (!dont_check_max) { // LDU can't load >= 1<_")); - std::vector args_assert2 = { code.create_int(loc, 5, "(excno)"), ir_gt_max[0], code.create_int(loc, 1, "") }; - Op& op_assert2 = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert2), lookup_function("__throw_if_unless")); + std::vector ir_max_value = code.create_tmp_var(TypeDataInt::create(), origin, "(enum-max)"); + code.emplace_back(origin, Op::_IntConst, ir_max_value, max_value); + std::vector ir_gt_max = code.create_tmp_var(TypeDataInt::create(), origin, "(enum-gt-ax)"); + code.emplace_back(origin, Op::_Call, ir_gt_max, std::vector{ir_num[0], ir_max_value[0]}, lookup_function("_>_")); + std::vector args_assert2 = { code.create_int(origin, 5, "(excno)"), ir_gt_max[0], code.create_int(origin, 1, "") }; + Op& op_assert2 = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_assert2), lookup_function("__throw_if_unless")); op_assert2.set_impure_flag(); } } else { // okay, enum is not a sequence, just a set of values; // then validation is: "throw if v is not contained in V", check v==V_i and combine with OR - var_idx_t ir_any_of = code.create_int(loc, 0, "(any-of-equals)"); + var_idx_t ir_any_of = code.create_int(origin, 0, "(any-of-equals)"); for (EnumMemberPtr member_ref : enum_ref->members) { - std::vector ir_ith_value = code.create_tmp_var(TypeDataInt::create(), loc, "(enum-ith)"); - code.emplace_back(loc, Op::_IntConst, ir_ith_value, member_ref->computed_value); - std::vector ir_ith_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(enum-ith-eq)"); - code.emplace_back(loc, Op::_Call, ir_ith_eq, std::vector{ir_num[0], ir_ith_value[0]}, lookup_function("_==_")); - code.emplace_back(loc, Op::_Call, std::vector{ir_any_of}, std::vector{ir_any_of, ir_ith_eq[0]}, lookup_function("_|_")); + std::vector ir_ith_value = code.create_tmp_var(TypeDataInt::create(), origin, "(enum-ith)"); + code.emplace_back(origin, Op::_IntConst, ir_ith_value, member_ref->computed_value); + std::vector ir_ith_eq = code.create_tmp_var(TypeDataInt::create(), origin, "(enum-ith-eq)"); + code.emplace_back(origin, Op::_Call, ir_ith_eq, std::vector{ir_num[0], ir_ith_value[0]}, lookup_function("_==_")); + code.emplace_back(origin, Op::_Call, std::vector{ir_any_of}, std::vector{ir_any_of, ir_ith_eq[0]}, lookup_function("_|_")); } - std::vector args_assert = { code.create_int(loc, 5, "(excno)"), ir_any_of, code.create_int(loc, 0, "") }; - Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); + std::vector args_assert = { code.create_int(origin, 5, "(excno)"), ir_any_of, code.create_int(origin, 0, "") }; + Op& op_assert = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); op_assert.set_impure_flag(); } return ir_num; } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { TypePtr intN = calculate_intN_to_serialize_enum(enum_ref); return ctx->generate_skip_any(intN); } @@ -1133,26 +1134,26 @@ struct S_CustomReceiverForPackUnpack final : ISerializer { explicit S_CustomReceiverForPackUnpack(TypePtr receiver_type) : receiver_type(receiver_type) {} - void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { FunctionPtr f_pack = get_custom_pack_unpack_function(receiver_type, true); tolk_assert(f_pack && f_pack->does_accept_self() && f_pack->inferred_return_type->get_width_on_stack() == 0); std::vector vars_per_arg = { std::move(rvect), ctx->ir_builder }; - std::vector ir_mutated_builder = gen_inline_fun_call_in_place(code, TypeDataBuilder::create(), loc, f_pack, nullptr, false, vars_per_arg); - code.emplace_back(loc, Op::_Let, ctx->ir_builder, std::move(ir_mutated_builder)); + std::vector ir_mutated_builder = gen_inline_fun_call_in_place(code, TypeDataBuilder::create(), origin, f_pack, nullptr, false, vars_per_arg); + code.emplace_back(origin, Op::_Let, ctx->ir_builder, std::move(ir_mutated_builder)); } - std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { FunctionPtr f_unpack = get_custom_pack_unpack_function(receiver_type, false); tolk_assert(f_unpack && f_unpack->inferred_return_type->get_width_on_stack() == receiver_type->get_width_on_stack()); TypePtr ret_type = TypeDataTensor::create({TypeDataSlice::create(), receiver_type}); - std::vector ir_slice_and_res = gen_inline_fun_call_in_place(code, ret_type, loc, f_unpack, nullptr, false, {ctx->ir_slice}); - code.emplace_back(loc, Op::_Let, ctx->ir_slice, std::vector{ir_slice_and_res.front()}); + std::vector ir_slice_and_res = gen_inline_fun_call_in_place(code, ret_type, origin, f_unpack, nullptr, false, {ctx->ir_slice}); + code.emplace_back(origin, Op::_Let, ctx->ir_slice, std::vector{ir_slice_and_res.front()}); return std::vector(ir_slice_and_res.begin() + 1, ir_slice_and_res.end()); } - void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { // just load and ignore the result - unpack(ctx, code, loc); + unpack(ctx, code, origin); } PackSize estimate(const EstimateContext* ctx) override { @@ -1264,29 +1265,29 @@ TypePtr calculate_intN_to_serialize_enum(EnumDefPtr enum_ref) { } // there is no way to pass custom pack options to createMessage / map.set / etc., using hardcoded ones -std::vector create_default_PackOptions(CodeBlob& code, SrcLocation loc) { +std::vector create_default_PackOptions(CodeBlob& code, AnyV origin) { StructPtr s_PackOptions = lookup_global_symbol("PackOptions")->try_as(); - std::vector ir_options = code.create_tmp_var(TypeDataStruct::create(s_PackOptions), loc, "(pack-options)"); + std::vector ir_options = code.create_tmp_var(TypeDataStruct::create(s_PackOptions), origin, "(pack-options)"); tolk_assert(ir_options.size() == 1); std::vector ir_defaults = { - code.create_int(loc, 0, "(zero)"), // skipBitsNFieldsValidation + code.create_int(origin, 0, "(zero)"), // skipBitsNFieldsValidation }; - code.emplace_back(loc, Op::_Let, ir_options, std::move(ir_defaults)); + code.emplace_back(origin, Op::_Let, ir_options, std::move(ir_defaults)); return ir_options; } // there is no way to pass custom unpack options to map.get / etc., using hardcoded ones -std::vector create_default_UnpackOptions(CodeBlob& code, SrcLocation loc) { +std::vector create_default_UnpackOptions(CodeBlob& code, AnyV origin) { StructPtr s_UnpackOptions = lookup_global_symbol("UnpackOptions")->try_as(); - std::vector ir_options = code.create_tmp_var(TypeDataStruct::create(s_UnpackOptions), loc, "(unpack-options)"); + std::vector ir_options = code.create_tmp_var(TypeDataStruct::create(s_UnpackOptions), origin, "(unpack-options)"); tolk_assert(ir_options.size() == 2); std::vector ir_defaults = { - code.create_int(loc, -1, "(true)"), // assertEndAfterReading - code.create_int(loc, 63, "(excno)"), // throwIfOpcodeDoesNotMatch + code.create_int(origin, -1, "(true)"), // assertEndAfterReading + code.create_int(origin, 63, "(excno)"), // throwIfOpcodeDoesNotMatch }; - code.emplace_back(loc, Op::_Let, ir_options, std::move(ir_defaults)); + code.emplace_back(origin, Op::_Let, ir_options, std::move(ir_defaults)); return ir_options; } @@ -1394,14 +1395,14 @@ static std::unique_ptr get_serializer_for_type(TypePtr any_type) { void PackContext::generate_pack_any(TypePtr any_type, std::vector&& rvect, PrefixWriteMode prefix_mode) const { PrefixWriteMode backup = this->prefix_mode; this->prefix_mode = prefix_mode; - get_serializer_for_type(any_type)->pack(this, code, loc, std::move(rvect)); + get_serializer_for_type(any_type)->pack(this, code, origin, std::move(rvect)); this->prefix_mode = backup; } std::vector UnpackContext::generate_unpack_any(TypePtr any_type, PrefixReadMode prefix_mode) const { PrefixReadMode backup = this->prefix_mode; this->prefix_mode = prefix_mode; - std::vector result = get_serializer_for_type(any_type)->unpack(this, code, loc); + std::vector result = get_serializer_for_type(any_type)->unpack(this, code, origin); this->prefix_mode = backup; return result; } @@ -1409,22 +1410,22 @@ std::vector UnpackContext::generate_unpack_any(TypePtr any_type, Pref void UnpackContext::generate_skip_any(TypePtr any_type, PrefixReadMode prefix_mode) const { PrefixReadMode backup = this->prefix_mode; this->prefix_mode = prefix_mode; - get_serializer_for_type(any_type)->skip(this, code, loc); + get_serializer_for_type(any_type)->skip(this, code, origin); this->prefix_mode = backup; } std::vector UnpackContext::generate_lazy_match_any(TypePtr any_type, const LazyMatchOptions& options) const { std::unique_ptr serializer = get_serializer_for_type(any_type); if (auto* s = dynamic_cast(serializer.get())) { - return s->lazy_match(this, code, loc, options); + return s->lazy_match(this, code, origin, options); } if (auto* s = dynamic_cast(serializer.get())) { - return s->lazy_match(this, code, loc, options); + return s->lazy_match(this, code, origin, options); } if (auto* s = dynamic_cast(serializer.get())) { - return s->lazy_match(this, code, loc, options); + return s->lazy_match(this, code, origin, options); } - tolk_assert(false); + tolk_assert(false); } PackSize EstimateContext::estimate_any(TypePtr any_type, PrefixEstimateMode prefix_mode) const { diff --git a/tolk/pack-unpack-serializers.h b/tolk/pack-unpack-serializers.h index a511f751e6..7642447c7e 100644 --- a/tolk/pack-unpack-serializers.h +++ b/tolk/pack-unpack-serializers.h @@ -56,7 +56,7 @@ enum class PrefixWriteMode { class PackContext { CodeBlob& code; - SrcLocation loc; + AnyV origin; const FunctionPtr f_storeInt; const FunctionPtr f_storeUint; mutable PrefixWriteMode prefix_mode = PrefixWriteMode::WritePrefixOfStruct; @@ -66,7 +66,7 @@ class PackContext { const var_idx_t ir_builder0; const var_idx_t option_skipBitsNValidation; - PackContext(CodeBlob& code, SrcLocation loc, std::vector ir_builder, const std::vector& ir_options); + PackContext(CodeBlob& code, AnyV origin, std::vector ir_builder, const std::vector& ir_options); PrefixWriteMode get_prefix_mode() const { return prefix_mode; } @@ -104,12 +104,12 @@ struct LazyMatchOptions { std::vector match_blocks; const MatchBlock* find_match_block(TypePtr variant) const; - void save_match_result_on_arm_end(CodeBlob& code, SrcLocation loc, const MatchBlock* arm_block, std::vector&& ir_arm_result, const std::vector& ir_match_expr_result) const; + void save_match_result_on_arm_end(CodeBlob& code, AnyV origin, const MatchBlock* arm_block, std::vector&& ir_arm_result, const std::vector& ir_match_expr_result) const; }; class UnpackContext { CodeBlob& code; - SrcLocation loc; + AnyV origin; const FunctionPtr f_loadInt; const FunctionPtr f_loadUint; const FunctionPtr f_skipBits; @@ -121,7 +121,7 @@ class UnpackContext { const var_idx_t option_assertEndAfterReading; const var_idx_t option_throwIfOpcodeDoesNotMatch; - UnpackContext(CodeBlob& code, SrcLocation loc, std::vector ir_slice, const std::vector& ir_options); + UnpackContext(CodeBlob& code, AnyV origin, std::vector ir_slice, const std::vector& ir_options); PrefixReadMode get_prefix_mode() const { return prefix_mode; } @@ -167,7 +167,7 @@ FunctionPtr get_custom_pack_unpack_function(TypePtr receiver_type, bool is_pack) std::vector auto_generate_opcodes_for_union(TypePtr union_type, std::string& because_msg); TypePtr calculate_intN_to_serialize_enum(EnumDefPtr enum_ref); -std::vector create_default_PackOptions(CodeBlob& code, SrcLocation loc); -std::vector create_default_UnpackOptions(CodeBlob& code, SrcLocation loc); +std::vector create_default_PackOptions(CodeBlob& code, AnyV origin); +std::vector create_default_UnpackOptions(CodeBlob& code, AnyV origin); } // namespace tolk diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 568f4868d8..bc37e47715 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -15,11 +15,12 @@ along with TON Blockchain Library. If not, see . */ #include "tolk.h" -#include "src-file.h" #include "ast.h" #include "ast-aux-data.h" #include "ast-visitor.h" +#include "compilation-errors.h" #include "type-system.h" +#include "src-file.h" #include "common/refint.h" #include "smart-casts-cfg.h" #include "pack-unpack-api.h" @@ -50,7 +51,7 @@ namespace tolk { class LValContext; std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr target_type = nullptr, LValContext* lval_ctx = nullptr); -std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, CodeBlob& code, LValContext* lval_ctx); +std::vector pre_compile_symbol(const Symbol* sym, CodeBlob& code, AnyV origin, LValContext* lval_ctx); void process_any_statement(AnyV v, CodeBlob& code); static AnyV stmt_before_immediate_return = nullptr; @@ -62,9 +63,9 @@ static AnyV stmt_before_immediate_return = nullptr; class VarsModificationWatcher { struct WatchedVar { var_idx_t ir_idx; - std::function on_modification_callback; + std::function on_modification_callback; - WatchedVar(var_idx_t ir_idx, std::function on_modification_callback) + WatchedVar(var_idx_t ir_idx, std::function on_modification_callback) : ir_idx(ir_idx), on_modification_callback(std::move(on_modification_callback)) {} }; @@ -74,7 +75,7 @@ class VarsModificationWatcher { bool empty() const { return all_callbacks.empty(); } - void push_callback(var_idx_t ir_idx, std::function callback) { + void push_callback(var_idx_t ir_idx, std::function callback) { all_callbacks.emplace_back(ir_idx, std::move(callback)); } @@ -88,11 +89,11 @@ class VarsModificationWatcher { tolk_assert(false); } - void trigger_callbacks(const std::vector& left_lval_indices, SrcLocation loc) const { + void trigger_callbacks(const std::vector& left_lval_indices, AnyV origin) const { for (const WatchedVar& w : all_callbacks) { for (var_idx_t changed_var : left_lval_indices) { if (w.ir_idx == changed_var) { - w.on_modification_callback(loc, w.ir_idx); + w.on_modification_callback(origin, w.ir_idx); } } } @@ -136,24 +137,24 @@ class LValContext { // for 1-slot globals int/cell/slice, assigning to them is just SETGLOB // same for tensors, if they are fully rewritten in an expression: `gTensor = (5,6)` - void apply_fully_rewrite(CodeBlob& code, SrcLocation loc) const { - Op& op = code.emplace_back(loc, Op::_SetGlob, std::vector{}, lval_ir_idx, glob_ref); + void apply_fully_rewrite(CodeBlob& code, AnyV origin) const { + Op& op = code.emplace_back(origin, Op::_SetGlob, std::vector{}, lval_ir_idx, glob_ref); op.set_impure_flag(); } // for N-slot globals tensor/struct/union, assigning to their parts, like `gTensor.1 = 6` // we need to read gTensor as a whole (0-th and 1-th component), rewrite 1-th component, and SETGLOB a whole back - void apply_partially_rewrite(CodeBlob& code, SrcLocation loc, std::vector&& was_modified_by_let) const { + void apply_partially_rewrite(CodeBlob& code, AnyV origin, std::vector&& was_modified_by_let) const { LValContext local_lval; local_lval.enter_rval_inside_lval(); - std::vector local_ir_idx = pre_compile_symbol(loc, glob_ref, code, &local_lval); + std::vector local_ir_idx = pre_compile_symbol(glob_ref, code, origin, &local_lval); for (size_t i = 0; i < local_ir_idx.size(); ++i) { if (was_modified_by_let[i]) { - code.emplace_back(loc, Op::_Let, std::vector{local_ir_idx[i]}, std::vector{lval_ir_idx[i]}); + code.emplace_back(origin, Op::_Let, std::vector{local_ir_idx[i]}, std::vector{lval_ir_idx[i]}); } } - Op& op = code.emplace_back(loc, Op::_SetGlob, std::vector{}, local_ir_idx, glob_ref); + Op& op = code.emplace_back(origin, Op::_SetGlob, std::vector{}, local_ir_idx, glob_ref); op.set_impure_flag(); } }; @@ -166,10 +167,10 @@ class LValContext { int index_at; // for tensors, it's index_at; for structs, it's field_idx std::vector lval_ir_idx; - void apply(CodeBlob& code, SrcLocation loc) const { + void apply(CodeBlob& code, AnyV origin) const { LValContext local_lval; local_lval.enter_rval_inside_lval(); - std::vector obj_ir_idx = pre_compile_expr(tensor_obj, code, nullptr, &local_lval); + std::vector obj_ir_idx = pre_compile_expr(tensor_obj, code, nullptr, &local_lval); int stack_width, stack_offset; TypePtr obj_type = tensor_obj->inferred_type->unwrap_alias(); @@ -182,12 +183,12 @@ class LValContext { } else { tolk_assert(false); } - std::vector field_ir_idx = {obj_ir_idx.begin() + stack_offset, obj_ir_idx.begin() + stack_offset + stack_width}; + std::vector field_ir_idx(obj_ir_idx.begin() + stack_offset, obj_ir_idx.begin() + stack_offset + stack_width); tolk_assert(field_ir_idx.size() == lval_ir_idx.size()); - vars_modification_watcher.trigger_callbacks(field_ir_idx, loc); - code.emplace_back(loc, Op::_Let, field_ir_idx, lval_ir_idx); - local_lval.after_let(std::move(field_ir_idx), code, loc); + vars_modification_watcher.trigger_callbacks(field_ir_idx, origin); + code.emplace_back(origin, Op::_Let, field_ir_idx, lval_ir_idx); + local_lval.after_let(std::move(field_ir_idx), code, origin); } }; @@ -199,17 +200,17 @@ class LValContext { int index_at; std::vector lval_ir_idx; - void apply(CodeBlob& code, SrcLocation loc) const { + void apply(CodeBlob& code, AnyV origin) const { LValContext local_lval; local_lval.enter_rval_inside_lval(); - std::vector tuple_ir_idx = pre_compile_expr(tuple_obj, code, nullptr, &local_lval); - std::vector index_ir_idx = code.create_tmp_var(TypeDataInt::create(), loc, "(tuple-idx)"); - code.emplace_back(loc, Op::_IntConst, index_ir_idx, td::make_refint(index_at)); + std::vector tuple_ir_idx = pre_compile_expr(tuple_obj, code, nullptr, &local_lval); + std::vector index_ir_idx = code.create_tmp_var(TypeDataInt::create(), origin, "(tuple-idx)"); + code.emplace_back(origin, Op::_IntConst, index_ir_idx, td::make_refint(index_at)); - vars_modification_watcher.trigger_callbacks(tuple_ir_idx, loc); + vars_modification_watcher.trigger_callbacks(tuple_ir_idx, origin); FunctionPtr builtin_sym = lookup_function("tuple.set"); - code.emplace_back(loc, Op::_Call, std::vector{tuple_ir_idx}, std::vector{tuple_ir_idx[0], lval_ir_idx[0], index_ir_idx[0]}, builtin_sym); - local_lval.after_let(std::move(tuple_ir_idx), code, loc); + code.emplace_back(origin, Op::_Call, tuple_ir_idx, std::vector{tuple_ir_idx[0], lval_ir_idx[0], index_ir_idx[0]}, builtin_sym); + local_lval.after_let(std::move(tuple_ir_idx), code, origin); } }; @@ -242,7 +243,7 @@ class LValContext { modifications.emplace_back(ModifiedTupleIndex{tuple_obj, index_at, std::move(lval_ir_idx)}); } - void after_let(std::vector&& let_left_vars, CodeBlob& code, SrcLocation loc) const { + void after_let(std::vector&& let_left_vars, CodeBlob& code, AnyV origin) const { for (const auto& modification : modifications) { if (const auto* m_glob = std::get_if(&modification)) { int n_modified_by_let = 0; @@ -255,9 +256,9 @@ class LValContext { } } if (n_modified_by_let == static_cast(m_glob->lval_ir_idx.size())) { - m_glob->apply_fully_rewrite(code, loc); + m_glob->apply_fully_rewrite(code, origin); } else if (n_modified_by_let > 0) { - m_glob->apply_partially_rewrite(code, loc, std::move(was_modified_by_let)); + m_glob->apply_partially_rewrite(code, origin, std::move(was_modified_by_let)); } } else if (const auto* m_tup = std::get_if(&modification)) { bool was_tuple_index_modified = false; @@ -265,7 +266,7 @@ class LValContext { was_tuple_index_modified |= vector_contains(let_left_vars, field_ir_idx); } if (was_tuple_index_modified) { - m_tup->apply(code, loc); + m_tup->apply(code, origin); } } else if (const auto* m_tens = std::get_if(&modification)) { bool was_tensor_index_modified = false; @@ -273,7 +274,7 @@ class LValContext { was_tensor_index_modified |= vector_contains(let_left_vars, field_ir_idx); } if (was_tensor_index_modified) { - m_tens->apply(code, loc); + m_tens->apply(code, origin); } } } @@ -472,20 +473,20 @@ static std::vector> pre_compile_tensor_inner(CodeBlob& co for (var_idx_t ir_idx : vars_of_ith_arg) { if (!code.vars[ir_idx].name.empty() && !is_watched(ir_idx)) { watched_vars.emplace_back(ir_idx); - vars_modification_watcher.push_callback(ir_idx, [this, &code](SrcLocation loc, var_idx_t ir_idx) { - on_var_modified(ir_idx, loc, code); + vars_modification_watcher.push_callback(ir_idx, [this, &code](AnyV origin, var_idx_t ir_idx) { + on_var_modified(ir_idx, origin, code); }); } } res_lists.emplace_back(std::move(vars_of_ith_arg)); } - void on_var_modified(var_idx_t ir_idx, SrcLocation loc, CodeBlob& code) { + void on_var_modified(var_idx_t ir_idx, AnyV origin, CodeBlob& code) { tolk_assert(is_watched(ir_idx)); - std::vector tmp_idx_arr = code.create_tmp_var(code.vars[ir_idx].v_type, loc, "(pre-modified)"); + std::vector tmp_idx_arr = code.create_tmp_var(code.vars[ir_idx].v_type, origin, "(pre-modified)"); tolk_assert(tmp_idx_arr.size() == 1); var_idx_t tmp_idx = tmp_idx_arr[0]; - code.emplace_back(loc, Op::_Let, std::vector{tmp_idx}, std::vector{ir_idx}); + code.emplace_back(origin, Op::_Let, std::vector{tmp_idx}, std::vector{ir_idx}); for (std::vector& prev_vars : res_lists) { std::replace(prev_vars.begin(), prev_vars.end(), ir_idx, tmp_idx); } @@ -503,7 +504,7 @@ static std::vector> pre_compile_tensor_inner(CodeBlob& co WatchingVarList watched_vars(n); for (int arg_idx = 0; arg_idx < n; ++arg_idx) { TypePtr child_target_type = tensor_target_type ? tensor_target_type->items[arg_idx] : nullptr; - std::vector vars_of_ith_arg = pre_compile_expr(args[arg_idx], code, child_target_type, lval_ctx); + std::vector vars_of_ith_arg = pre_compile_expr(args[arg_idx], code, child_target_type, lval_ctx); watched_vars.add_and_watch_modifications(std::move(vars_of_ith_arg), code); } return watched_vars.clear_and_stop_watching(); @@ -527,90 +528,90 @@ static std::vector pre_compile_tensor(CodeBlob& code, const std::vect return res; } -static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyExprV rhs, SrcLocation loc) { +static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyExprV rhs) { // [lhs] = [rhs]; since type checking is ok, it's the same as "lhs = rhs" if (lhs->kind == ast_bracket_tuple && rhs->kind == ast_bracket_tuple) { // note: there are no type transitions (adding nullability flag, etc.), since only 1-slot elements allowed in tuples LValContext local_lval; - std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); - vars_modification_watcher.trigger_callbacks(left, loc); - std::vector rvect = pre_compile_tensor(code, rhs->as()->get_items()); - code.emplace_back(loc, Op::_Let, left, rvect); - local_lval.after_let(std::move(left), code, loc); - std::vector right = code.create_tmp_var(TypeDataTuple::create(), loc, "(tuple)"); - code.emplace_back(lhs->loc, Op::_Tuple, right, std::move(rvect)); - return right; + std::vector ir_left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); + vars_modification_watcher.trigger_callbacks(ir_left, lhs); + std::vector rvect = pre_compile_tensor(code, rhs->as()->get_items()); + code.emplace_back(lhs, Op::_Let, ir_left, rvect); + local_lval.after_let(std::move(ir_left), code, lhs); + std::vector ir_right = code.create_tmp_var(TypeDataTuple::create(), rhs, "(tuple)"); + code.emplace_back(lhs, Op::_Tuple, ir_right, std::move(rvect)); + return ir_right; } // [lhs] = rhs; it's un-tuple to N left vars if (lhs->kind == ast_bracket_tuple) { LValContext local_lval; - std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); - vars_modification_watcher.trigger_callbacks(left, loc); - std::vector right = pre_compile_expr(rhs, code, nullptr); + std::vector ir_left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); + vars_modification_watcher.trigger_callbacks(ir_left, lhs); + std::vector ir_right = pre_compile_expr(rhs, code, nullptr); const TypeDataBrackets* inferred_tuple = rhs->inferred_type->unwrap_alias()->try_as(); std::vector types_list = inferred_tuple->items; - std::vector rvect = code.create_tmp_var(TypeDataTensor::create(std::move(types_list)), rhs->loc, "(unpack-tuple)"); - code.emplace_back(lhs->loc, Op::_UnTuple, rvect, std::move(right)); - code.emplace_back(loc, Op::_Let, left, rvect); - local_lval.after_let(std::move(left), code, loc); - return right; + std::vector rvect = code.create_tmp_var(TypeDataTensor::create(std::move(types_list)), rhs, "(unpack-tuple)"); + code.emplace_back(lhs, Op::_UnTuple, rvect, std::move(ir_right)); + code.emplace_back(lhs, Op::_Let, ir_left, rvect); + local_lval.after_let(std::move(ir_left), code, lhs); + return ir_right; } // small optimization: `var x = rhs` or `local_var = rhs` (90% cases), LValContext not needed actually if (lhs->kind == ast_local_var_lhs || (lhs->kind == ast_reference && lhs->as()->sym->try_as())) { - std::vector left = pre_compile_expr(lhs, code, nullptr); // effectively, local_var->ir_idx - vars_modification_watcher.trigger_callbacks(left, loc); - std::vector right = pre_compile_expr(rhs, code, lhs->inferred_type); - code.emplace_back(loc, Op::_Let, std::move(left), right); - return right; + std::vector ir_left = pre_compile_expr(lhs, code, nullptr); // effectively, local_var->ir_idx + vars_modification_watcher.trigger_callbacks(ir_left, lhs); + std::vector ir_right = pre_compile_expr(rhs, code, lhs->inferred_type); + code.emplace_back(lhs, Op::_Let, std::move(ir_left), ir_right); + return ir_right; } // lhs = rhs LValContext local_lval; - std::vector left = pre_compile_expr(lhs, code, nullptr, &local_lval); - vars_modification_watcher.trigger_callbacks(left, loc); - std::vector right = pre_compile_expr(rhs, code, lhs->inferred_type); - code.emplace_back(loc, Op::_Let, left, right); - local_lval.after_let(std::move(left), code, loc); - return right; + std::vector ir_left = pre_compile_expr(lhs, code, nullptr, &local_lval); + vars_modification_watcher.trigger_callbacks(ir_left, lhs); + std::vector ir_right = pre_compile_expr(rhs, code, lhs->inferred_type); + code.emplace_back(lhs, Op::_Let, ir_left, ir_right); + local_lval.after_let(std::move(ir_left), code, lhs); + return ir_right; } -std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc) { +std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, AnyV origin, const char* debug_desc) { FunctionPtr eq_sym = lookup_function("_==_"); FunctionPtr isnull_sym = lookup_function("__isNull"); FunctionPtr not_sym = lookup_function("!b_"); - std::vector result_ir_idx = code.create_tmp_var(TypeDataBool::create(), loc, debug_desc); + std::vector ir_result = code.create_tmp_var(TypeDataBool::create(), origin, debug_desc); const TypeDataUnion* lhs_union = expr_type->unwrap_alias()->try_as(); if (!lhs_union) { // `int` is `int` / `int` is `builder`, it's compile-time, either 0, or -1 bool types_eq = expr_type->get_type_id() == cmp_type->get_type_id(); - code.emplace_back(loc, Op::_IntConst, result_ir_idx, td::make_refint(types_eq ? -1 : 0)); + code.emplace_back(origin, Op::_IntConst, ir_result, td::make_refint(types_eq ? -1 : 0)); } else if (lhs_union->is_primitive_nullable() && cmp_type == TypeDataNullLiteral::create()) { // `int?` is `null` for primitive 1-slot nullables, they hold either value of TVM NULL, no extra union tag slot - code.emplace_back(loc, Op::_Call, result_ir_idx, expr_ir_idx, isnull_sym); + code.emplace_back(origin, Op::_Call, ir_result, expr_ir_idx, isnull_sym); } else if (lhs_union->is_primitive_nullable()) { // `int?` is `int` (check for null actually) / `int?` is `builder` (compile-time false actually) bool cant_happen = lhs_union->or_null->get_type_id() != cmp_type->get_type_id(); if (cant_happen) { - code.emplace_back(loc, Op::_IntConst, result_ir_idx, td::make_refint(0)); + code.emplace_back(origin, Op::_IntConst, ir_result, td::make_refint(0)); } else { - code.emplace_back(loc, Op::_Call, result_ir_idx, expr_ir_idx, isnull_sym); - code.emplace_back(loc, Op::_Call, result_ir_idx, result_ir_idx, not_sym); + code.emplace_back(origin, Op::_Call, ir_result, expr_ir_idx, isnull_sym); + code.emplace_back(origin, Op::_Call, ir_result, ir_result, not_sym); } } else { // `int | slice` is `int`, check type id - std::vector typeid_ir_idx = code.create_tmp_var(TypeDataInt::create(), loc, "(type-id)"); - code.emplace_back(loc, Op::_IntConst, typeid_ir_idx, td::make_refint(cmp_type->get_type_id())); - code.emplace_back(loc, Op::_Call, result_ir_idx, std::vector{typeid_ir_idx[0], expr_ir_idx.back()}, eq_sym); + std::vector ir_typeid = code.create_tmp_var(TypeDataInt::create(), origin, "(type-id)"); + code.emplace_back(origin, Op::_IntConst, ir_typeid, td::make_refint(cmp_type->get_type_id())); + code.emplace_back(origin, Op::_Call, ir_result, std::vector{ir_typeid[0], expr_ir_idx.back()}, eq_sym); } - return result_ir_idx; + return ir_result; } -static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcLocation loc, +static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, AnyV origin, std::vector&& args_vars, FunctionPtr fun_ref, const char* debug_desc, bool arg_order_already_equals_asm = false) { - std::vector rvect = code.create_tmp_var(ret_type, loc, debug_desc); - Op& op = code.emplace_back(loc, Op::_Call, rvect, std::move(args_vars), fun_ref); + std::vector rvect = code.create_tmp_var(ret_type, origin, debug_desc); + Op& op = code.emplace_back(origin, Op::_Call, rvect, std::move(args_vars), fun_ref); if (!fun_ref->is_marked_as_pure()) { op.set_impure_flag(); } @@ -621,7 +622,6 @@ static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcL } static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob& code, V v_call, const std::vector>& vars_per_arg) { - SrcLocation loc = v_call->loc; FunctionPtr called_f = v_call->fun_maybe; if (called_f->is_method() && called_f->is_instantiation_of_generic_function()) { @@ -631,24 +631,24 @@ static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob if (f_name == "T.toCell" && lazy_variable && lazy_variable->is_struct()) { // in: object Lazy (partially loaded), out: Cell std::vector ir_obj = vars_per_arg[0]; // = lazy_var_ref->ir_idx - return generate_lazy_struct_to_cell(code, loc, &lazy_variable->loaded_state, std::move(ir_obj), vars_per_arg[1]); + return generate_lazy_struct_to_cell(code, v_call, &lazy_variable->loaded_state, std::move(ir_obj), vars_per_arg[1]); } if (f_name == "T.forceLoadLazyObject") { // in: object T, out: slice (same slice that a lazy variable holds, after loading/skipping all its fields) if (!lazy_variable) { - fire(code.fun_ref, v_call->loc, "this method is applicable to lazy variables only"); + fire(code.fun_ref, v_call->get_callee(), "this method is applicable to lazy variables only"); } std::vector ir_obj = vars_per_arg[0]; - return generate_lazy_object_finish_loading(code, loc, lazy_variable, std::move(ir_obj)); + return generate_lazy_object_finish_loading(code, v_call, lazy_variable, std::move(ir_obj)); } } auto gen = std::get_if(&called_f->body); tolk_assert(gen); - return (*gen)->generate_ops(called_f, code, loc, vars_per_arg); + return (*gen)->generate_ops(called_f, code, v_call, vars_per_arg); } -std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, SrcLocation loc, FunctionPtr f_inlined, AnyExprV self_obj, bool is_before_immediate_return, const std::vector>& vars_per_arg) { +std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, AnyV origin, FunctionPtr f_inlined, AnyExprV self_obj, bool is_before_immediate_return, const std::vector>& vars_per_arg) { tolk_assert(vars_per_arg.size() == f_inlined->parameters.size()); for (int i = 0; i < f_inlined->get_num_params(); ++i) { const LocalVarData& param_i = f_inlined->get_param(i); @@ -658,13 +658,13 @@ std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_ // (being honest, it's quite strange that copy+LET may lead to more stack permutations) param_i.mutate()->assign_ir_idx(std::vector(vars_per_arg[i])); } else { - std::vector ith_param = code.create_var(param_i.declared_type, loc, param_i.name); - code.emplace_back(loc, Op::_Let, ith_param, vars_per_arg[i]); - param_i.mutate()->assign_ir_idx(std::move(ith_param)); + std::vector ir_ith_param = code.create_var(param_i.declared_type, origin, param_i.name); + code.emplace_back(origin, Op::_Let, ir_ith_param, vars_per_arg[i]); + param_i.mutate()->assign_ir_idx(std::move(ir_ith_param)); } } - std::vector rvect_call = code.create_tmp_var(ret_type, loc, "(inlined-return)"); + std::vector rvect_call = code.create_tmp_var(ret_type, origin, "(inlined-return)"); std::vector* backup_outer_inline = code.inline_rvect_out; FunctionPtr backup_cur_fun = code.fun_ref; bool backup_inline_before_return = code.inlining_before_immediate_return; @@ -691,7 +691,7 @@ std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_ mutated_vars.insert(mutated_vars.end(), p_sym.ir_idx.begin(), p_sym.ir_idx.end()); } } - code.emplace_back(loc, Op::_Let, rvect_call, std::move(mutated_vars)); + code.emplace_back(origin, Op::_Let, rvect_call, std::move(mutated_vars)); } ClearStateAfterInlineInPlace visitor; @@ -715,7 +715,7 @@ std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_ // The same mechanism works for union types, but there is a union tag (UTag) slot instead of null flag. // Another example: `var i: int|slice = 5;`. This "5" is represented as "5 1" (5 for value, 1 is type_id of `int`). GNU_ATTRIBUTE_NOINLINE -static std::vector transition_expr_to_runtime_type_impl(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc) { +static std::vector transition_expr_to_runtime_type_impl(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, AnyV origin) { #ifdef TOLK_DEBUG tolk_assert(static_cast(rvect.size()) == original_type->get_width_on_stack()); #endif @@ -784,12 +784,12 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); - code.emplace_back(loc, Op::_Call, ith_null, std::vector{}, null_sym); + std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), origin, "(null-literal)"); + code.emplace_back(origin, Op::_Call, ith_null, std::vector{}, null_sym); rvect.push_back(ith_null[0]); } - std::vector last_null = code.create_tmp_var(TypeDataInt::create(), loc, "(UTag)"); - code.emplace_back(loc, Op::_IntConst, last_null, td::make_refint(0)); + std::vector last_null = code.create_tmp_var(TypeDataInt::create(), origin, "(UTag)"); + code.emplace_back(origin, Op::_IntConst, last_null, td::make_refint(0)); rvect.push_back(last_null[0]); return rvect; } @@ -801,8 +801,8 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectorhas_null() && target_w == 1); - std::vector new_rvect = code.create_tmp_var(TypeDataInt::create(), loc, "(UTag)"); - code.emplace_back(loc, Op::_IntConst, new_rvect, td::make_refint(0)); + std::vector new_rvect = code.create_tmp_var(TypeDataInt::create(), origin, "(UTag)"); + code.emplace_back(origin, Op::_IntConst, new_rvect, td::make_refint(0)); return new_rvect; } @@ -824,8 +824,8 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectorhas_null()); FunctionPtr null_sym = lookup_function("__null"); - std::vector new_rvect = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); - code.emplace_back(loc, Op::_Call, new_rvect, std::vector{}, null_sym); + std::vector new_rvect = code.create_tmp_var(TypeDataNullLiteral::create(), origin, "(null-literal)"); + code.emplace_back(origin, Op::_Call, new_rvect, std::vector{}, null_sym); return new_rvect; } @@ -844,23 +844,23 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector new_rvect; new_rvect.resize(target_w); for (int i = 0; i < target_w - 2; ++i) { // N-1 nulls - std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); - code.emplace_back(loc, Op::_Call, ith_null, std::vector{}, null_sym); + std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), origin, "(null-literal)"); + code.emplace_back(origin, Op::_Call, ith_null, std::vector{}, null_sym); new_rvect[i] = ith_null[0]; } new_rvect[target_w - 2] = rvect[0]; // value - new_rvect[target_w - 1] = code.create_tmp_var(TypeDataInt::create(), loc, "(UTag)")[0]; + new_rvect[target_w - 1] = code.create_tmp_var(TypeDataInt::create(), origin, "(UTag)")[0]; - std::vector eq_null_cond = code.create_tmp_var(TypeDataBool::create(), loc, "(value-is-null)"); + std::vector ir_eq_null = code.create_tmp_var(TypeDataBool::create(), origin, "(value-is-null)"); FunctionPtr isnull_sym = lookup_function("__isNull"); - code.emplace_back(loc, Op::_Call, eq_null_cond, rvect, isnull_sym); - Op& if_op = code.emplace_back(loc, Op::_If, eq_null_cond); + code.emplace_back(origin, Op::_Call, ir_eq_null, rvect, isnull_sym); + Op& if_op = code.emplace_back(origin, Op::_If, ir_eq_null); code.push_set_cur(if_op.block0); - code.emplace_back(loc, Op::_IntConst, std::vector{new_rvect[target_w - 1]}, td::make_refint(0)); - code.close_pop_cur(loc); + code.emplace_back(origin, Op::_IntConst, std::vector{new_rvect[target_w - 1]}, td::make_refint(0)); + code.close_pop_cur(origin); code.push_set_cur(if_op.block1); - code.emplace_back(loc, Op::_IntConst, std::vector{new_rvect[target_w - 1]}, td::make_refint(o_union->or_null->get_type_id())); - code.close_pop_cur(loc); + code.emplace_back(origin, Op::_IntConst, std::vector{new_rvect[target_w - 1]}, td::make_refint(o_union->or_null->get_type_id())); + code.close_pop_cur(origin); return new_rvect; } @@ -874,20 +874,20 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectorcalculate_exact_variant_to_fit_rhs(original_type); tolk_assert(t_subtype && target_w > t_subtype->get_width_on_stack()); - rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, t_subtype, loc); + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, t_subtype, origin); std::vector prepend_nulls; prepend_nulls.reserve(target_w - t_subtype->get_width_on_stack() - 1); for (int i = 0; i < target_w - t_subtype->get_width_on_stack() - 1; ++i) { FunctionPtr null_sym = lookup_function("__null"); - std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(UVar.null)"); + std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), origin, "(UVar.null)"); prepend_nulls.push_back(ith_null[0]); - code.emplace_back(loc, Op::_Call, std::move(ith_null), std::vector{}, null_sym); + code.emplace_back(origin, Op::_Call, std::move(ith_null), std::vector{}, null_sym); } rvect.insert(rvect.begin(), prepend_nulls.begin(), prepend_nulls.end()); - std::vector last_utag = code.create_tmp_var(TypeDataInt::create(), loc, "(UTag)"); - code.emplace_back(loc, Op::_IntConst, last_utag, td::make_refint(t_subtype->get_type_id())); - rvect.push_back(last_utag[0]); + std::vector ir_last_utag = code.create_tmp_var(TypeDataInt::create(), origin, "(UTag)"); + code.emplace_back(origin, Op::_IntConst, ir_last_utag, td::make_refint(t_subtype->get_type_id())); + rvect.push_back(ir_last_utag[0]); return rvect; } @@ -900,7 +900,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectorcalculate_exact_variant_to_fit_rhs(target_type); tolk_assert(o_subtype && orig_w > o_subtype->get_width_on_stack()); rvect = std::vector(rvect.begin() + orig_w - o_subtype->get_width_on_stack() - 1, rvect.end() - 1); - rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, o_subtype, target_type, loc); + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, o_subtype, target_type, origin); return rvect; } @@ -914,9 +914,9 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector ith_null_ir_idx = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(UVar.null)"); - prepend_nulls.push_back(ith_null_ir_idx[0]); - code.emplace_back(loc, Op::_Call, std::move(ith_null_ir_idx), std::vector{}, null_sym); + std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), origin, "(UVar.null)"); + prepend_nulls.push_back(ith_null[0]); + code.emplace_back(origin, Op::_Call, std::move(ith_null), std::vector{}, null_sym); } rvect.insert(rvect.begin(), prepend_nulls.begin(), prepend_nulls.end()); return rvect; @@ -1050,8 +1050,8 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectorsize(); ++i) { int ith_w = inferred_tensor->items[i]->get_width_on_stack(); - std::vector rvect_i{rvect.begin() + stack_offset, rvect.begin() + stack_offset + ith_w}; - std::vector result_i = transition_expr_to_runtime_type_impl(std::move(rvect_i), code, inferred_tensor->items[i], target_tensor->items[i], loc); + std::vector rvect_i(rvect.begin() + stack_offset, rvect.begin() + stack_offset + ith_w); + std::vector result_i = transition_expr_to_runtime_type_impl(std::move(rvect_i), code, inferred_tensor->items[i], target_tensor->items[i], origin); result_rvect.insert(result_rvect.end(), result_i.begin(), result_i.end()); stack_offset += ith_w; } @@ -1124,7 +1124,7 @@ GNU_ATTRIBUTE_ALWAYS_INLINE #endif static std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr target_type, AnyExprV v) { if (target_type != nullptr && target_type != v->inferred_type) { - rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, v->inferred_type, target_type, v->loc); + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, v->inferred_type, target_type, v); } return rvect; } @@ -1133,15 +1133,15 @@ static std::vector transition_to_target_type(std::vector&& #ifndef TOLK_DEBUG GNU_ATTRIBUTE_ALWAYS_INLINE #endif -std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc) { +std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, AnyV origin) { if (target_type != original_type) { - rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, target_type, loc); + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, target_type, origin); } return rvect; } -std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, CodeBlob& code, LValContext* lval_ctx) { +std::vector pre_compile_symbol(const Symbol* sym, CodeBlob& code, AnyV origin, LValContext* lval_ctx) { // referencing a local variable (not its declaration, but its usage) if (LocalVarPtr var_ref = sym->try_as()) { #ifdef TOLK_DEBUG @@ -1158,16 +1158,16 @@ std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, Co if (const CachedConstValueAtCodegen* cached = code.get_cached_const(const_ref)) { return cached->ir_idx; } - std::vector ir_sub_const = pre_compile_expr(const_ref->init_value, code, const_ref->declared_type); + std::vector ir_sub_const = pre_compile_expr(const_ref->init_value, code, const_ref->declared_type); code.cached_consts.emplace_back(CachedConstValueAtCodegen{const_ref, ir_sub_const}); return ir_sub_const; } // just referencing `a1` in a function body // (note that `a1` occurred 2 times has 2 different ir_idx, caching above is done only within another const) - ASTAuxData* aux_data = new AuxData_ForceFiftLocation(loc); - auto v_force_loc = createV(loc, const_ref->init_value, aux_data, const_ref->inferred_type); + ASTAuxData* aux_data = new AuxData_ForceFiftLocation(origin); + auto v_force_loc = createV(const_ref->init_value, aux_data, const_ref->inferred_type); code.inside_evaluating_constant = true; - std::vector ir_const = pre_compile_expr(v_force_loc, code, const_ref->declared_type); + std::vector ir_const = pre_compile_expr(v_force_loc, code, const_ref->declared_type); code.inside_evaluating_constant = false; code.cached_consts.clear(); return ir_const; @@ -1177,14 +1177,14 @@ std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, Co if (GlobalVarPtr glob_ref = sym->try_as()) { // handle `globalVar = rhs` / `mutate globalVar` if (lval_ctx && !lval_ctx->is_rval_inside_lval()) { - std::vector lval_ir_idx = code.create_tmp_var(glob_ref->declared_type, loc, "(lval-glob)"); - lval_ctx->capture_global_modification(glob_ref, lval_ir_idx); - return lval_ir_idx; + std::vector ir_lval = code.create_tmp_var(glob_ref->declared_type, origin, "(lval-glob)"); + lval_ctx->capture_global_modification(glob_ref, ir_lval); + return ir_lval; } // `globalVar` is used for reading, just create local IR var to represent its value, Op GlobVar will fill it // note, that global tensors are stored as a tuple an unpacked to N vars on read, N determined by declared_type - std::vector local_ir_idx = code.create_var(glob_ref->declared_type, loc, "g_" + glob_ref->name); - code.emplace_back(loc, Op::_GlobVar, local_ir_idx, std::vector{}, glob_ref); + std::vector local_ir_idx = code.create_var(glob_ref->declared_type, origin, "g_" + glob_ref->name); + code.emplace_back(origin, Op::_GlobVar, local_ir_idx, std::vector{}, glob_ref); if (lval_ctx) { // `globalVar.0 = rhs`, globalVar is rval inside lval lval_ctx->capture_global_modification(glob_ref, local_ir_idx); } @@ -1193,8 +1193,8 @@ std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, Co // referencing a function (not calling it! using as a callback, works similar to a global var) if (FunctionPtr fun_ref = sym->try_as()) { - std::vector rvect = code.create_tmp_var(fun_ref->inferred_full_type, loc, "(glob-var-fun)"); - code.emplace_back(loc, Op::_GlobVar, rvect, std::vector{}, fun_ref); + std::vector rvect = code.create_tmp_var(fun_ref->inferred_full_type, origin, "(glob-var-fun)"); + code.emplace_back(origin, Op::_GlobVar, rvect, std::vector{}, fun_ref); return rvect; } @@ -1202,7 +1202,7 @@ std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, Co } static std::vector process_reference(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { - std::vector rvect = pre_compile_symbol(v->loc, v->sym, code, lval_ctx); + std::vector rvect = pre_compile_symbol(v->sym, code, v, lval_ctx); // a local variable might be smart cast at this point, for example we're in `if (v != null)` // it means that we must drop the null flag (if it's a tensor), or maybe perform other stack transformations @@ -1210,7 +1210,7 @@ static std::vector process_reference(V v, CodeBlob& co if (LocalVarPtr var_ref = v->sym->try_as()) { // note, inside `if (v != null)` when `v` is used for writing, v->inferred_type is an original (declared_type) // (smart casts apply only for rvalue, not for lvalue, we don't check it here, it's a property of inferring) - rvect = transition_to_target_type(std::move(rvect), code, var_ref->declared_type, v->inferred_type, v->loc); + rvect = transition_to_target_type(std::move(rvect), code, var_ref->declared_type, v->inferred_type, v); } return transition_to_target_type(std::move(rvect), code, target_type, v); @@ -1221,15 +1221,15 @@ static std::vector process_assignment(V v, CodeBlob& code AnyExprV rhs = v->get_rhs(); if (auto lhs_decl = lhs->try_as()) { - std::vector rvect = pre_compile_let(code, lhs_decl->get_expr(), rhs, v->loc); + std::vector rvect = pre_compile_let(code, lhs_decl->get_expr(), rhs); return transition_to_target_type(std::move(rvect), code, target_type, v); } else { - std::vector rvect = pre_compile_let(code, lhs, rhs, v->loc); + std::vector rvect = pre_compile_let(code, lhs, rhs); // now rvect contains rhs IR vars constructed to fit lhs (for correct assignment, lhs type was target_type for rhs) // but the type of `lhs = rhs` is RHS (see type inferring), so rvect now should fit rhs->inferred_type (= v->inferred_type) // example: `t1 = t2 = null`, we're at `t2 = null`, earlier declared t1: `int?`, t2: `(int,int)?` // currently "null" matches t2 (3 null slots), but type of this assignment is "plain null" (1 slot) assigned later to t1 - rvect = transition_to_target_type(std::move(rvect), code, lhs->inferred_type, v->inferred_type, v->loc); + rvect = transition_to_target_type(std::move(rvect), code, lhs->inferred_type, v->inferred_type, rhs); return transition_to_target_type(std::move(rvect), code, target_type, v); } } @@ -1238,11 +1238,11 @@ static std::vector process_set_assign(V v, CodeBlob& // for "a += b", emulate "a = a + b" // seems not beautiful, but it works; probably, this transformation should be done at AST level in advance std::string_view calc_operator = v->operator_name; // "+" for operator += - auto v_apply = createV(v->loc, calc_operator, static_cast(v->tok - 1), v->get_lhs(), v->get_rhs()); + auto v_apply = createV(v->range, v->operator_range, calc_operator, static_cast(v->tok - 1), v->get_lhs(), v->get_rhs()); v_apply->assign_inferred_type(v->inferred_type); v_apply->assign_fun_ref(v->fun_ref); - std::vector rvect = pre_compile_let(code, v->get_lhs(), v_apply, v->loc); + std::vector rvect = pre_compile_let(code, v->get_lhs(), v_apply); return transition_to_target_type(std::move(rvect), code, target_type, v); } @@ -1250,43 +1250,43 @@ static std::vector process_binary_operator(V v, TokenType t = v->tok; if (v->fun_ref) { // almost all operators, fun_ref was assigned at type inferring - std::vector args_vars = pre_compile_tensor(code, {v->get_lhs(), v->get_rhs()}); - std::vector rvect = gen_op_call(code, v->inferred_type, v->loc, std::move(args_vars), v->fun_ref, "(binary-op)"); + std::vector args_vars = pre_compile_tensor(code, {v->get_lhs(), v->get_rhs()}); + std::vector rvect = gen_op_call(code, v->inferred_type, v, std::move(args_vars), v->fun_ref, "(binary-op)"); return transition_to_target_type(std::move(rvect), code, target_type, v); } if (t == tok_logical_and || t == tok_logical_or) { // do the following transformations: // a && b -> a ? (b != 0) : 0 // a || b -> a ? 1 : (b != 0) - AnyExprV v_0 = createV(v->loc, td::make_refint(0), "0"); + AnyExprV v_0 = createV(v->range, td::make_refint(0), "0"); v_0->mutate()->assign_inferred_type(TypeDataInt::create()); - AnyExprV v_1 = createV(v->loc, td::make_refint(-1), "-1"); + AnyExprV v_1 = createV(v->range, td::make_refint(-1), "-1"); v_1->mutate()->assign_inferred_type(TypeDataInt::create()); - auto v_b_ne_0 = createV(v->loc, "!=", tok_neq, v->get_rhs(), v_0); + auto v_b_ne_0 = createV(v->range, v->operator_range, "!=", tok_neq, v->get_rhs(), v_0); v_b_ne_0->mutate()->assign_inferred_type(TypeDataInt::create()); v_b_ne_0->mutate()->assign_fun_ref(lookup_function("_!=_")); - std::vector cond = pre_compile_expr(v->get_lhs(), code, nullptr); - tolk_assert(cond.size() == 1); - std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(ternary)"); - Op& if_op = code.emplace_back(v->loc, Op::_If, cond); + std::vector ir_cond = pre_compile_expr(v->get_lhs(), code, nullptr); + tolk_assert(ir_cond.size() == 1); + std::vector rvect = code.create_tmp_var(v->inferred_type, v, "(ternary)"); + Op& if_op = code.emplace_back(v, Op::_If, ir_cond); code.push_set_cur(if_op.block0); - code.emplace_back(v->loc, Op::_Let, rvect, pre_compile_expr(t == tok_logical_and ? v_b_ne_0 : v_1, code, nullptr)); - code.close_pop_cur(v->loc); + code.emplace_back(v, Op::_Let, rvect, pre_compile_expr(t == tok_logical_and ? v_b_ne_0 : v_1, code, nullptr)); + code.close_pop_cur(v); code.push_set_cur(if_op.block1); - code.emplace_back(v->loc, Op::_Let, rvect, pre_compile_expr(t == tok_logical_and ? v_0 : v_b_ne_0, code, nullptr)); - code.close_pop_cur(v->loc); + code.emplace_back(v, Op::_Let, rvect, pre_compile_expr(t == tok_logical_and ? v_0 : v_b_ne_0, code, nullptr)); + code.close_pop_cur(v); return transition_to_target_type(std::move(rvect), code, target_type, v); } if (t == tok_eq || t == tok_neq) { if (v->get_lhs()->inferred_type->unwrap_alias() == TypeDataAddress::create() && v->get_rhs()->inferred_type->unwrap_alias() == TypeDataAddress::create()) { FunctionPtr f_sliceEq = lookup_function("slice.bitsEqual"); - std::vector ir_lhs_slice = pre_compile_expr(v->get_lhs(), code); - std::vector ir_rhs_slice = pre_compile_expr(v->get_rhs(), code); - std::vector rvect = code.create_tmp_var(TypeDataBool::create(), v->loc, "(addr-eq)"); - code.emplace_back(v->loc, Op::_Call, rvect, std::vector{ir_lhs_slice[0], ir_rhs_slice[0]}, f_sliceEq); + std::vector ir_lhs_slice = pre_compile_expr(v->get_lhs(), code); + std::vector ir_rhs_slice = pre_compile_expr(v->get_rhs(), code); + std::vector rvect = code.create_tmp_var(TypeDataBool::create(), v, "(addr-eq)"); + code.emplace_back(v, Op::_Call, rvect, std::vector{ir_lhs_slice[0], ir_rhs_slice[0]}, f_sliceEq); if (t == tok_neq) { FunctionPtr not_sym = lookup_function("!b_"); - code.emplace_back(v->loc, Op::_Call, rvect, rvect, not_sym); + code.emplace_back(v, Op::_Call, rvect, rvect, not_sym); } return transition_to_target_type(std::move(rvect), code, target_type, v); } @@ -1296,33 +1296,33 @@ static std::vector process_binary_operator(V v, } static std::vector process_unary_operator(V v, CodeBlob& code, TypePtr target_type) { - std::vector rhs_vars = pre_compile_expr(v->get_rhs(), code, nullptr); - std::vector rvect = gen_op_call(code, v->inferred_type, v->loc, std::move(rhs_vars), v->fun_ref, "(unary-op)"); + std::vector rhs_vars = pre_compile_expr(v->get_rhs(), code, nullptr); + std::vector rvect = gen_op_call(code, v->inferred_type, v, std::move(rhs_vars), v->fun_ref, "(unary-op)"); return transition_to_target_type(std::move(rvect), code, target_type, v); } static std::vector process_ternary_operator(V v, CodeBlob& code, TypePtr target_type) { - std::vector cond = pre_compile_expr(v->get_cond(), code, nullptr); - tolk_assert(cond.size() == 1); - std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(cond)"); + std::vector ir_cond = pre_compile_expr(v->get_cond(), code, nullptr); + tolk_assert(ir_cond.size() == 1); + std::vector rvect = code.create_tmp_var(v->inferred_type, v, "(cond)"); if (v->get_cond()->is_always_true) { - code.emplace_back(v->get_when_true()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_true(), code, v->inferred_type)); + code.emplace_back(v->get_when_true(), Op::_Let, rvect, pre_compile_expr(v->get_when_true(), code, v->inferred_type)); } else if (v->get_cond()->is_always_false) { - code.emplace_back(v->get_when_false()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_false(), code, v->inferred_type)); + code.emplace_back(v->get_when_false(), Op::_Let, rvect, pre_compile_expr(v->get_when_false(), code, v->inferred_type)); } else if (v->inferred_type->get_width_on_stack() == 1 && is_ternary_arg_trivial_for_condsel(v->get_when_true()) && is_ternary_arg_trivial_for_condsel(v->get_when_false())) { - std::vector ir_true = pre_compile_expr(v->get_when_true(), code, v->inferred_type); - std::vector ir_false = pre_compile_expr(v->get_when_false(), code, v->inferred_type); - std::vector condsel_args = { cond[0], ir_true[0], ir_false[0] }; - code.emplace_back(v->loc, Op::_Call, rvect, std::move(condsel_args), lookup_function("__condsel")); + std::vector ir_true = pre_compile_expr(v->get_when_true(), code, v->inferred_type); + std::vector ir_false = pre_compile_expr(v->get_when_false(), code, v->inferred_type); + std::vector condsel_args = { ir_cond[0], ir_true[0], ir_false[0] }; + code.emplace_back(v, Op::_Call, rvect, std::move(condsel_args), lookup_function("__condsel")); } else { - Op& if_op = code.emplace_back(v->loc, Op::_If, cond); + Op& if_op = code.emplace_back(v, Op::_If, ir_cond); code.push_set_cur(if_op.block0); - code.emplace_back(v->get_when_true()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_true(), code, v->inferred_type)); - code.close_pop_cur(v->get_when_true()->loc); + code.emplace_back(v->get_when_true(), Op::_Let, rvect, pre_compile_expr(v->get_when_true(), code, v->inferred_type)); + code.close_pop_cur(v->get_when_true()); code.push_set_cur(if_op.block1); - code.emplace_back(v->get_when_false()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_false(), code, v->inferred_type)); - code.close_pop_cur(v->get_when_false()->loc); + code.emplace_back(v->get_when_false(), Op::_Let, rvect, pre_compile_expr(v->get_when_false(), code, v->inferred_type)); + code.close_pop_cur(v->get_when_false()); } return transition_to_target_type(std::move(rvect), code, target_type, v); @@ -1330,7 +1330,7 @@ static std::vector process_ternary_operator(V v static std::vector process_cast_as_operator(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { TypePtr child_target_type = v->type_node->resolved_type; - std::vector rvect = pre_compile_expr(v->get_expr(), code, child_target_type, lval_ctx); + std::vector rvect = pre_compile_expr(v->get_expr(), code, child_target_type, lval_ctx); return transition_to_target_type(std::move(rvect), code, target_type, v); } @@ -1340,14 +1340,14 @@ static std::vector process_is_type_operator(V v bool is_null_check = cmp_type == TypeDataNullLiteral::create(); // `v == null`, not `v is T` tolk_assert(!cmp_type->try_as()); // `v is int|slice` is a type checker error - std::vector expr_ir_idx = pre_compile_expr(v->get_expr(), code, nullptr); - std::vector result_ir_idx = pre_compile_is_type(code, lhs_type, cmp_type, expr_ir_idx, v->loc, is_null_check ? "(is-null)" : "(is-type)"); + std::vector ir_expr = pre_compile_expr(v->get_expr(), code, nullptr); + std::vector ir_result = pre_compile_is_type(code, lhs_type, cmp_type, ir_expr, v, is_null_check ? "(is-null)" : "(is-type)"); if (v->is_negated) { FunctionPtr not_sym = lookup_function("!b_"); - code.emplace_back(v->loc, Op::_Call, result_ir_idx, result_ir_idx, not_sym); + code.emplace_back(v, Op::_Call, ir_result, ir_result, not_sym); } - return transition_to_target_type(std::move(result_ir_idx), code, target_type, v); + return transition_to_target_type(std::move(ir_result), code, target_type, v); } static std::vector process_not_null_operator(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { @@ -1355,7 +1355,7 @@ static std::vector process_not_null_operator(V TypePtr without_null_type = calculate_type_subtract_rhs_type(expr_type, TypeDataNullLiteral::create()); TypePtr child_target_type = without_null_type != TypeDataNever::create() ? without_null_type : expr_type; - std::vector rvect = pre_compile_expr(v->get_expr(), code, child_target_type, lval_ctx); + std::vector rvect = pre_compile_expr(v->get_expr(), code, child_target_type, lval_ctx); return transition_to_target_type(std::move(rvect), code, target_type, v); } @@ -1371,26 +1371,26 @@ static std::vector process_lazy_operator(V v, Code tolk_assert(f_body->size() == 1 && f_body->get_item(0)->kind == ast_return_statement); auto f_returns = f_body->get_item(0)->as(); v_call = f_returns->get_return_value()->try_as(); - tolk_assert(v_call && v_call->fun_maybe && v_call->fun_maybe->is_builtin_function()); + tolk_assert(v_call && v_call->fun_maybe && v_call->fun_maybe->is_builtin()); called_f = v_call->fun_maybe; } // only predefined built-in functions are allowed for lazy loading - tolk_assert(called_f->is_builtin_function() && called_f->is_instantiation_of_generic_function()); + tolk_assert(called_f->is_builtin() && called_f->is_instantiation_of_generic_function()); std::string_view f_name = called_f->base_fun_ref->name; - std::vector ir_slice = code.create_var(TypeDataSlice::create(), v->loc, "lazyS"); + std::vector ir_slice = code.create_var(TypeDataSlice::create(), v, "lazyS"); bool has_passed_options = false; if (f_name == "T.fromSlice") { std::vector passed_slice = pre_compile_expr(v_call->get_arg(0)->get_expr(), code); - code.emplace_back(v->loc, Op::_Let, ir_slice, std::move(passed_slice)); + code.emplace_back(v, Op::_Let, ir_slice, std::move(passed_slice)); has_passed_options = v_call->get_num_args() == 2; } else if (f_name == "T.fromCell") { std::vector ir_cell = pre_compile_expr(v_call->get_arg(0)->get_expr(), code); - code.emplace_back(v->loc, Op::_Call, ir_slice, ir_cell, lookup_function("cell.beginParse")); + code.emplace_back(v, Op::_Call, ir_slice, ir_cell, lookup_function("cell.beginParse")); has_passed_options = v_call->get_num_args() == 2; } else if (f_name == "Cell.load") { std::vector ir_cell = pre_compile_expr(v_call->get_callee()->try_as()->get_obj(), code); - code.emplace_back(v->loc, Op::_Call, ir_slice, ir_cell, lookup_function("cell.beginParse")); + code.emplace_back(v, Op::_Call, ir_slice, ir_cell, lookup_function("cell.beginParse")); has_passed_options = v_call->get_num_args() == 1; } else { tolk_assert(false); @@ -1406,7 +1406,7 @@ static std::vector process_lazy_operator(V v, Code // they will be loaded by separate AST aux vertices; // same for unions: `val msg = lazy MyMsgUnion`, msg is N+1 nulls, but next lazy `match` will transition slots, // which will be filled by loads - std::vector ir_null = gen_op_call(code, TypeDataNullLiteral::create(), v->loc, {}, lookup_function("__null"), "(init-null)"); + std::vector ir_null = gen_op_call(code, TypeDataNullLiteral::create(), v, {}, lookup_function("__null"), "(init-null)"); std::vector ir_initial_nulls(v->dest_var_ref->ir_idx.size(), ir_null[0]); return transition_to_target_type(std::move(ir_initial_nulls), code, target_type, v); } @@ -1416,8 +1416,8 @@ static std::vector process_match_expression(V v const TypeDataEnum* subject_enum = subject_type->unwrap_alias()->try_as(); int n_arms = v->get_arms_count(); - std::vector subj_ir_idx = pre_compile_expr(v->get_subject(), code, nullptr); - std::vector result_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(match-expression)"); + std::vector ir_subj = pre_compile_expr(v->get_subject(), code, nullptr); + std::vector ir_result = code.create_tmp_var(v->inferred_type, v, "(match-expression)"); if (!n_arms) { // `match (subject) {}` tolk_assert(v->is_statement()); @@ -1446,7 +1446,7 @@ static std::vector process_match_expression(V v // because it might turned out to be a lazy match, where `else` is allowed; // if we are here, it's not a lazy match, it's a regular one (the lazy one is handled specially, in aux vertex) if (is_match_by_type && has_else_arm) { - throw ParseError(v->get_arm(n_arms - 1)->loc, "`else` is not allowed in `match` by type; you should cover all possible types"); + fire(v->get_arm(n_arms - 1)->get_pattern_expr(), "`else` is not allowed in `match` by type; you should cover all possible types"); } // example 1 (exhaustive): `match (v) { int => ... slice => ... builder => ... }` @@ -1462,25 +1462,25 @@ static std::vector process_match_expression(V v if (is_match_by_type) { TypePtr cmp_type = v_ith_arm->pattern_type_node->resolved_type; tolk_assert(!cmp_type->unwrap_alias()->try_as()); // `match` over `int|slice` is a type checker error - eq_ith_ir_idx = pre_compile_is_type(code, subject_type, cmp_type, subj_ir_idx, v_ith_arm->loc, "(arm-cond-eq)"); + eq_ith_ir_idx = pre_compile_is_type(code, subject_type, cmp_type, ir_subj, v_ith_arm, "(arm-cond-eq)"); } else { - std::vector ith_ir_idx = pre_compile_expr(v_ith_arm->get_pattern_expr(), code); - tolk_assert(subj_ir_idx.size() == 1 && ith_ir_idx.size() == 1); - eq_ith_ir_idx = code.create_tmp_var(TypeDataBool::create(), v_ith_arm->loc, "(arm-cond-eq)"); - code.emplace_back(v_ith_arm->loc, Op::_Call, eq_ith_ir_idx, std::vector{subj_ir_idx[0], ith_ir_idx[0]}, eq_sym); + std::vector ith_ir_idx = pre_compile_expr(v_ith_arm->get_pattern_expr(), code); + tolk_assert(ir_subj.size() == 1 && ith_ir_idx.size() == 1); + eq_ith_ir_idx = code.create_tmp_var(TypeDataBool::create(), v_ith_arm, "(arm-cond-eq)"); + code.emplace_back(v_ith_arm, Op::_Call, eq_ith_ir_idx, std::vector{ir_subj[0], ith_ir_idx[0]}, eq_sym); } - Op& if_op = code.emplace_back(v_ith_arm->loc, Op::_If, std::move(eq_ith_ir_idx)); + Op& if_op = code.emplace_back(v_ith_arm, Op::_If, std::move(eq_ith_ir_idx)); code.push_set_cur(if_op.block0); if (v->is_statement()) { pre_compile_expr(v_ith_arm->get_body(), code); if (v == stmt_before_immediate_return) { - code.emplace_back(v_ith_arm->loc, Op::_Return); + code.emplace_back(v_ith_arm, Op::_Return); } } else { - std::vector arm_ir_idx = pre_compile_expr(v_ith_arm->get_body(), code, v->inferred_type); - code.emplace_back(v->loc, Op::_Let, result_ir_idx, std::move(arm_ir_idx)); + std::vector arm_ir_idx = pre_compile_expr(v_ith_arm->get_body(), code, v->inferred_type); + code.emplace_back(v, Op::_Let, ir_result, std::move(arm_ir_idx)); } - code.close_pop_cur(v->loc); + code.close_pop_cur(v); code.push_set_cur(if_op.block1); // open ELSE } @@ -1490,19 +1490,19 @@ static std::vector process_match_expression(V v if (v->is_statement()) { pre_compile_expr(v_last_arm->get_body(), code); if (v == stmt_before_immediate_return) { - code.emplace_back(v_last_arm->loc, Op::_Return); + code.emplace_back(v_last_arm, Op::_Return); } } else { - std::vector arm_ir_idx = pre_compile_expr(v_last_arm->get_body(), code, v->inferred_type); - code.emplace_back(v->loc, Op::_Let, result_ir_idx, std::move(arm_ir_idx)); + std::vector arm_ir_idx = pre_compile_expr(v_last_arm->get_body(), code, v->inferred_type); + code.emplace_back(v, Op::_Let, ir_result, std::move(arm_ir_idx)); } } // close all outer IFs for (int i = 0; i < n_arms - is_exhaustive; ++i) { - code.close_pop_cur(v->loc); + code.close_pop_cur(v); } - return transition_to_target_type(std::move(result_ir_idx), code, target_type, v); + return transition_to_target_type(std::move(ir_result), code, target_type, v); } static std::vector process_dot_access(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { @@ -1516,9 +1516,9 @@ static std::vector process_dot_access(V v, CodeBlob& // handle `globalObj.field = rhs`, special case, then the global will be read on demand if (lval_ctx && !lval_ctx->is_rval_inside_lval()) { if (auto sink = calc_sink_leftmost_obj(v); sink && sink->sym->try_as()) { - std::vector lval_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(lval-global-field)"); - lval_ctx->capture_field_of_global_modification(v->get_obj(), field_ref->field_idx, lval_ir_idx); - return lval_ir_idx; + std::vector ir_lval = code.create_tmp_var(v->inferred_type, sink, "(lval-global-field)"); + lval_ctx->capture_field_of_global_modification(v->get_obj(), field_ref->field_idx, ir_lval); + return ir_lval; } } // handle `lazyPoint.x`, assert that slot for "x" is loaded (ensure lazy-loading correctness); @@ -1526,14 +1526,14 @@ static std::vector process_dot_access(V v, CodeBlob& if (const LazyVariableLoadedState* lazy_variable = code.get_lazy_variable(v->get_obj())) { lazy_variable->assert_field_loaded(t_struct->struct_ref, field_ref); } - std::vector lhs_vars = pre_compile_expr(v->get_obj(), code, nullptr, lval_ctx); + std::vector lhs_vars = pre_compile_expr(v->get_obj(), code, nullptr, lval_ctx); int stack_width = field_ref->declared_type->get_width_on_stack(); int stack_offset = calc_offset_on_stack(t_struct->struct_ref, field_ref->field_idx); - std::vector rvect{lhs_vars.begin() + stack_offset, lhs_vars.begin() + stack_offset + stack_width}; + std::vector rvect(lhs_vars.begin() + stack_offset, lhs_vars.begin() + stack_offset + stack_width); // an object field might be smart cast at this point, for example we're in `if (user.t != null)` // it means that we must drop the null flag (if `user.t` is a tensor), or maybe perform other stack transformations // (from original rvect = (vars of user.t) to fit smart cast) - rvect = transition_to_target_type(std::move(rvect), code, field_ref->declared_type, v->inferred_type, v->loc); + rvect = transition_to_target_type(std::move(rvect), code, field_ref->declared_type, v->inferred_type, v->get_obj()); return transition_to_target_type(std::move(rvect), code, target_type, v); } // `tensorVar.0` @@ -1542,20 +1542,20 @@ static std::vector process_dot_access(V v, CodeBlob& // handle `globalTensorVar.0 = rhs`, special case, then the global will be read on demand if (lval_ctx && !lval_ctx->is_rval_inside_lval()) { if (auto sink = calc_sink_leftmost_obj(v); sink && sink->sym->try_as()) { - std::vector lval_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(lval-global-tensor)"); - lval_ctx->capture_field_of_global_modification(v->get_obj(), index_at, lval_ir_idx); - return lval_ir_idx; + std::vector ir_lval = code.create_tmp_var(v->inferred_type, sink, "(lval-global-tensor)"); + lval_ctx->capture_field_of_global_modification(v->get_obj(), index_at, ir_lval); + return ir_lval; } } // since a tensor of N elems are N vars on a stack actually, calculate offset - std::vector lhs_vars = pre_compile_expr(v->get_obj(), code, nullptr, lval_ctx); + std::vector lhs_vars = pre_compile_expr(v->get_obj(), code, nullptr, lval_ctx); int stack_width = t_tensor->items[index_at]->get_width_on_stack(); int stack_offset = calc_offset_on_stack(t_tensor, index_at); - std::vector rvect{lhs_vars.begin() + stack_offset, lhs_vars.begin() + stack_offset + stack_width}; + std::vector rvect(lhs_vars.begin() + stack_offset, lhs_vars.begin() + stack_offset + stack_width); // a tensor index might be smart cast at this point, for example we're in `if (t.1 != null)` // it means that we must drop the null flag (if `t.1` is a tensor), or maybe perform other stack transformations // (from original rvect = (vars of t.1) to fit smart cast) - rvect = transition_to_target_type(std::move(rvect), code, t_tensor->items[index_at], v->inferred_type, v->loc); + rvect = transition_to_target_type(std::move(rvect), code, t_tensor->items[index_at], v->inferred_type, v->get_obj()); return transition_to_target_type(std::move(rvect), code, target_type, v); } // `tupleVar.0` @@ -1563,18 +1563,18 @@ static std::vector process_dot_access(V v, CodeBlob& int index_at = std::get(v->target); // handle `tupleVar.0 = rhs`, "0 SETINDEX" will be called when this was is modified if (lval_ctx && !lval_ctx->is_rval_inside_lval() && calc_sink_leftmost_obj(v)) { - std::vector lval_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(lval-tuple-field)"); - lval_ctx->capture_tuple_index_modification(v->get_obj(), index_at, lval_ir_idx); - return lval_ir_idx; + std::vector ir_lval = code.create_tmp_var(v->inferred_type, v->get_obj(), "(lval-tuple-field)"); + lval_ctx->capture_tuple_index_modification(v->get_obj(), index_at, ir_lval); + return ir_lval; } // `tupleVar.0` as rvalue: the same as "tuple.get(tupleVar, 0)" written in terms of IR vars - std::vector tuple_ir_idx = pre_compile_expr(v->get_obj(), code); - std::vector index_ir_idx = code.create_tmp_var(TypeDataInt::create(), v->get_identifier()->loc, "(tuple-idx)"); - code.emplace_back(v->loc, Op::_IntConst, index_ir_idx, td::make_refint(index_at)); - std::vector field_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(tuple-field)"); + std::vector tuple_ir_idx = pre_compile_expr(v->get_obj(), code); + std::vector index_ir_idx = code.create_tmp_var(TypeDataInt::create(), v->get_identifier(), "(tuple-idx)"); + code.emplace_back(v, Op::_IntConst, index_ir_idx, td::make_refint(index_at)); + std::vector field_ir_idx = code.create_tmp_var(v->inferred_type, v, "(tuple-field)"); tolk_assert(tuple_ir_idx.size() == 1 && field_ir_idx.size() == 1); // tuples contain only 1-slot values FunctionPtr builtin_sym = lookup_function("tuple.get"); - code.emplace_back(v->loc, Op::_Call, field_ir_idx, std::vector{tuple_ir_idx[0], index_ir_idx[0]}, builtin_sym); + code.emplace_back(v, Op::_Call, field_ir_idx, std::vector{tuple_ir_idx[0], index_ir_idx[0]}, builtin_sym); if (lval_ctx && calc_sink_leftmost_obj(v)) { // `tupleVar.0.1 = rhs`, then `tupleVar.0` is rval inside lval lval_ctx->capture_tuple_index_modification(v->get_obj(), index_at, field_ir_idx); } @@ -1589,8 +1589,8 @@ static std::vector process_dot_access(V v, CodeBlob& // all enums are integers, and their integer values have already been assigned or auto-calculated EnumMemberPtr member_ref = std::get(v->target); tolk_assert(!member_ref->computed_value.is_null()); - std::vector enum_ir_idx = code.create_tmp_var(TypeDataInt::create(), v->get_identifier()->loc, "(enum-member)"); - code.emplace_back(v->loc, Op::_IntConst, enum_ir_idx, member_ref->computed_value); + std::vector enum_ir_idx = code.create_tmp_var(TypeDataInt::create(), v->get_identifier(), "(enum-member)"); + code.emplace_back(v, Op::_IntConst, enum_ir_idx, member_ref->computed_value); return transition_to_target_type(std::move(enum_ir_idx), code, target_type, v); } @@ -1598,7 +1598,7 @@ static std::vector process_dot_access(V v, CodeBlob& // (currently, nothing except a global function can be referenced, no object-scope methods exist) FunctionPtr fun_ref = std::get(v->target); tolk_assert(fun_ref); - std::vector rvect = pre_compile_symbol(v->loc, fun_ref, code, lval_ctx); + std::vector rvect = pre_compile_symbol(fun_ref, code, v, lval_ctx); return transition_to_target_type(std::move(rvect), code, target_type, v); } @@ -1616,16 +1616,16 @@ static std::vector process_function_call(V v, Code } std::vector params_types = v->get_callee()->inferred_type->unwrap_alias()->try_as()->params_types; const TypeDataTensor* tensor_tt = TypeDataTensor::create(std::move(params_types))->try_as(); - std::vector> vars_per_arg = pre_compile_tensor_inner(code, args, tensor_tt, nullptr); + std::vector vars_per_arg = pre_compile_tensor_inner(code, args, tensor_tt, nullptr); std::vector args_vars; for (const std::vector& list : vars_per_arg) { args_vars.insert(args_vars.end(), list.cbegin(), list.cend()); } - std::vector tfunc = pre_compile_expr(v->get_callee(), code, nullptr); + std::vector tfunc = pre_compile_expr(v->get_callee(), code, nullptr); tolk_assert(tfunc.size() == 1); args_vars.push_back(tfunc[0]); - std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(call-ind)"); - Op& op = code.emplace_back(v->loc, Op::_CallInd, rvect, std::move(args_vars)); + std::vector rvect = code.create_tmp_var(v->inferred_type, v, "(call-ind)"); + Op& op = code.emplace_back(v, Op::_CallInd, rvect, std::move(args_vars)); op.set_impure_flag(); return transition_to_target_type(std::move(rvect), code, target_type, v); } @@ -1644,12 +1644,13 @@ static std::vector process_function_call(V v, Code for (int i = 0; i < v->get_num_args(); ++i) { args.push_back(v->get_arg(i)->get_expr()); } + // default values (forcing location to be here, not taken from declaraion) for (int i = delta_self + v->get_num_args(); i < fun_ref->get_num_params(); ++i) { LocalVarPtr param_ref = &fun_ref->get_param(i); tolk_assert(param_ref->has_default_value()); - SrcLocation last_loc = args.empty() ? v->loc : args.back()->loc; - ASTAuxData *aux_data = new AuxData_ForceFiftLocation(last_loc); - auto v_force_loc = createV(last_loc, param_ref->default_value, aux_data, param_ref->declared_type); + AnyV last_arg = args.empty() ? v : args.back(); + ASTAuxData *aux_data = new AuxData_ForceFiftLocation(last_arg); + auto v_force_loc = createV(param_ref->default_value, aux_data, param_ref->declared_type); args.push_back(v_force_loc); } @@ -1678,7 +1679,7 @@ static std::vector process_function_call(V v, Code } const TypeDataTensor* tensor_tt = TypeDataTensor::create(std::move(params_types))->try_as(); - std::vector> vars_per_arg = pre_compile_tensor_inner(code, args, tensor_tt, nullptr); + std::vector vars_per_arg = pre_compile_tensor_inner(code, args, tensor_tt, nullptr); TypePtr op_call_type = v->inferred_type; TypePtr real_ret_type = v->inferred_type; @@ -1707,9 +1708,9 @@ static std::vector process_function_call(V v, Code if (fun_ref->is_compile_time_special_gen()) { rvect_call = gen_compile_time_code_instead_of_fun_call(code, v, vars_per_arg); } else if (fun_ref->is_inlined_in_place() && fun_ref->is_code_function()) { - rvect_call = gen_inline_fun_call_in_place(code, op_call_type, v->loc, v->fun_maybe, v->get_self_obj(), v == stmt_before_immediate_return, vars_per_arg); + rvect_call = gen_inline_fun_call_in_place(code, op_call_type, v, v->fun_maybe, v->get_self_obj(), v == stmt_before_immediate_return, vars_per_arg); } else { - rvect_call = gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)", arg_order_already_equals_asm); + rvect_call = gen_op_call(code, op_call_type, v, std::move(args_vars), fun_ref, "(fun-call)", arg_order_already_equals_asm); } if (fun_ref->has_mutate_params()) { @@ -1721,18 +1722,18 @@ static std::vector process_function_call(V v, Code AnyExprV arg_i = obj_leftmost && i == 0 ? obj_leftmost : args[real_i]; tolk_assert(arg_i->is_lvalue || i == 0); if (arg_i->is_lvalue) { - std::vector ith_var_idx = pre_compile_expr(arg_i, code, nullptr, &local_lval); + std::vector ith_var_idx = pre_compile_expr(arg_i, code, nullptr, &local_lval); left.insert(left.end(), ith_var_idx.begin(), ith_var_idx.end()); } else { left.insert(left.end(), vars_per_arg[asm_self_idx].begin(), vars_per_arg[asm_self_idx].end()); } } } - std::vector rvect = code.create_tmp_var(real_ret_type, v->loc, "(fun-call)"); + std::vector rvect = code.create_tmp_var(real_ret_type, v, "(fun-call)"); left.insert(left.end(), rvect.begin(), rvect.end()); - vars_modification_watcher.trigger_callbacks(left, v->loc); - code.emplace_back(v->loc, Op::_Let, left, rvect_call); - local_lval.after_let(std::move(left), code, v->loc); + vars_modification_watcher.trigger_callbacks(left, v); + code.emplace_back(v, Op::_Let, left, rvect_call); + local_lval.after_let(std::move(left), code, v); rvect_call = rvect; } @@ -1765,18 +1766,18 @@ static std::vector process_braced_expression(V static std::vector process_tensor(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { // tensor is compiled "as is", for example `(1, null)` occupies 2 slots // and if assigned/passed to something other, like `(int, (int,int)?)`, a whole tensor is transitioned, it works - std::vector rvect = pre_compile_tensor(code, v->get_items(), lval_ctx); + std::vector rvect = pre_compile_tensor(code, v->get_items(), lval_ctx); return transition_to_target_type(std::move(rvect), code, target_type, v); } static std::vector process_typed_tuple(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { if (lval_ctx) { // todo some time, make "var (a, [b,c]) = (1, [2,3])" work - v->error("[...] can not be used as lvalue here"); + fire(v, "[...] can not be used as lvalue here"); } - std::vector left = code.create_tmp_var(v->inferred_type, v->loc, "(pack-tuple)"); - std::vector right = pre_compile_tensor(code, v->get_items(), lval_ctx); - code.emplace_back(v->loc, Op::_Tuple, left, std::move(right)); - return transition_to_target_type(std::move(left), code, target_type, v); + std::vector ir_left = code.create_tmp_var(v->inferred_type, v, "(pack-tuple)"); + std::vector ir_right = pre_compile_tensor(code, v->get_items(), lval_ctx); + code.emplace_back(v, Op::_Tuple, ir_left, std::move(ir_right)); + return transition_to_target_type(std::move(ir_left), code, target_type, v); } static std::vector process_object_literal_shuffled(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { @@ -1794,9 +1795,9 @@ static std::vector process_object_literal_shuffled(Vdeclared_type); } const auto* tensor_target_type = TypeDataTensor::create(std::move(target_types))->try_as(); - std::vector literal_rvect = pre_compile_tensor(code, tensor_items, lval_ctx, tensor_target_type); + std::vector literal_rvect = pre_compile_tensor(code, tensor_items, lval_ctx, tensor_target_type); - std::vector rvect = code.create_tmp_var(TypeDataStruct::create(v->struct_ref), v->loc, "(object)"); + std::vector rvect = code.create_tmp_var(TypeDataStruct::create(v->struct_ref), v, "(object)"); int stack_offset = 0; for (StructFieldPtr field_ref : v->struct_ref->fields) { int stack_width = field_ref->declared_type->get_width_on_stack(); @@ -1811,7 +1812,7 @@ static std::vector process_object_literal_shuffled(Vget_field_name() == field_ref->name) { exists_in_literal = true; std::vector literal_field_rvect(literal_rvect.begin() + tensor_offset, literal_rvect.begin() + tensor_offset + tensor_item_width); - code.emplace_back(v->loc, Op::_Let, std::move(field_rvect), std::move(literal_field_rvect)); + code.emplace_back(v, Op::_Let, std::move(field_rvect), std::move(literal_field_rvect)); break; } tensor_offset += tensor_item_width; @@ -1821,11 +1822,11 @@ static std::vector process_object_literal_shuffled(Vhas_default_value()); - SrcLocation last_loc = v->get_body()->empty() ? v->loc : v->get_body()->get_all_fields().back()->loc; - ASTAuxData *aux_data = new AuxData_ForceFiftLocation(last_loc); - auto v_force_loc = createV(v->loc, field_ref->default_value, aux_data, field_ref->declared_type); + AnyV last_field = v->get_body()->empty() ? v : v->get_body()->get_all_fields().back(); + ASTAuxData *aux_data = new AuxData_ForceFiftLocation(last_field); + auto v_force_loc = createV(field_ref->default_value, aux_data, field_ref->declared_type); std::vector def_rvect = pre_compile_expr(v_force_loc, code, field_ref->declared_type); - code.emplace_back(v->loc, Op::_Let, std::move(field_rvect), std::move(def_rvect)); + code.emplace_back(v, Op::_Let, std::move(field_rvect), std::move(def_rvect)); } return transition_to_target_type(std::move(rvect), code, target_type, v); @@ -1851,7 +1852,7 @@ static std::vector process_object_literal(V v, Co return process_object_literal_shuffled(v, code, target_type, lval_ctx); } - SrcLocation prev_loc = v->loc; + AnyV prev_init_val = v; std::vector tensor_items; std::vector target_types; tensor_items.reserve(v->struct_ref->get_num_fields()); @@ -1870,54 +1871,54 @@ static std::vector process_object_literal(V v, Co continue; // field of `never` type can be missed out of object literal (useful in generics defaults) } // (it occupies 0 slots, nothing is assignable to it — like this field is missing from a struct) tolk_assert(field_ref->has_default_value()); - ASTAuxData *aux_data = new AuxData_ForceFiftLocation(prev_loc); - auto v_force_loc = createV(v->loc, field_ref->default_value, aux_data, field_ref->declared_type); + ASTAuxData *aux_data = new AuxData_ForceFiftLocation(prev_init_val); + auto v_force_loc = createV(field_ref->default_value, aux_data, field_ref->declared_type); v_init_val = v_force_loc; // it may be a complex expression, but it's a constant, checked earlier } tensor_items.push_back(v_init_val); target_types.push_back(field_ref->declared_type); - prev_loc = v_init_val->loc; + prev_init_val = v_init_val; } const auto* tensor_target_type = TypeDataTensor::create(std::move(target_types))->try_as(); - std::vector rvect = pre_compile_tensor(code, tensor_items, lval_ctx, tensor_target_type); + std::vector rvect = pre_compile_tensor(code, tensor_items, lval_ctx, tensor_target_type); return transition_to_target_type(std::move(rvect), code, target_type, v); } static std::vector process_int_const(V v, CodeBlob& code, TypePtr target_type) { - std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(int-const)"); - code.emplace_back(v->loc, Op::_IntConst, rvect, v->intval); + std::vector rvect = code.create_tmp_var(v->inferred_type, v, "(int-const)"); + code.emplace_back(v, Op::_IntConst, rvect, v->intval); // here, like everywhere, even for just `int`, there might be a potential transition due to union types // example: passing `1` to `int | slice` puts actually "1 5" on a stack (1 for value, 5 for UTag = type_id of `int`) return transition_to_target_type(std::move(rvect), code, target_type, v); } static std::vector process_string_const(V v, CodeBlob& code, TypePtr target_type) { - std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(str-const)"); - code.emplace_back(v->loc, Op::_SliceConst, rvect, v->literal_value); + std::vector rvect = code.create_tmp_var(v->inferred_type, v, "(str-const)"); + code.emplace_back(v, Op::_SliceConst, rvect, v->literal_value); return transition_to_target_type(std::move(rvect), code, target_type, v); } static std::vector process_bool_const(V v, CodeBlob& code, TypePtr target_type) { FunctionPtr builtin_sym = lookup_function(v->bool_val ? "__true" : "__false"); - std::vector rvect = gen_op_call(code, v->inferred_type, v->loc, {}, builtin_sym, "(bool-const)"); + std::vector rvect = gen_op_call(code, v->inferred_type, v, {}, builtin_sym, "(bool-const)"); return transition_to_target_type(std::move(rvect), code, target_type, v); } static std::vector process_null_keyword(V v, CodeBlob& code, TypePtr target_type) { FunctionPtr builtin_sym = lookup_function("__null"); - std::vector rvect = gen_op_call(code, v->inferred_type, v->loc, {}, builtin_sym, "(null-literal)"); + std::vector rvect = gen_op_call(code, v->inferred_type, v, {}, builtin_sym, "(null-literal)"); return transition_to_target_type(std::move(rvect), code, target_type, v); } static std::vector process_local_var(V v, CodeBlob& code, TypePtr target_type) { if (v->marked_as_redef) { - std::vector rvect = pre_compile_symbol(v->loc, v->var_ref, code, nullptr); + std::vector rvect = pre_compile_symbol(v->var_ref, code, v, nullptr); return transition_to_target_type(std::move(rvect), code, target_type, v); } tolk_assert(v->var_ref->ir_idx.empty()); - v->var_ref->mutate()->assign_ir_idx(code.create_var(v->inferred_type, v->loc, v->var_ref->name)); - std::vector rvect = v->var_ref->ir_idx; + v->var_ref->mutate()->assign_ir_idx(code.create_var(v->inferred_type, v, v->var_ref->name)); + std::vector rvect = v->var_ref->ir_idx; return transition_to_target_type(std::move(rvect), code, target_type, v); } @@ -1930,7 +1931,7 @@ static std::vector process_local_vars_declaration(V process_underscore(V v, CodeBlob& code) { // when _ is used as left side of assignment, like `(cs, _) = cs.loadAndReturn()` - return code.create_tmp_var(v->inferred_type, v->loc, "(underscore)"); + return code.create_tmp_var(v->inferred_type, v, "(underscore)"); } static std::vector process_empty_expression(V v, CodeBlob& code, TypePtr target_type) { @@ -1939,11 +1940,13 @@ static std::vector process_empty_expression(V v } static std::vector process_artificial_aux_vertex(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + AnyExprV wrapped = v->get_wrapped_expr(); + if (const auto* data = dynamic_cast(v->aux_data)) { - SrcLocation backup = code.forced_loc; - code.forced_loc = data->forced_loc; - std::vector rvect = pre_compile_expr(v->get_wrapped_expr(), code, target_type, lval_ctx); - code.forced_loc = backup; + AnyV backup = code.forced_origin; + code.forced_origin = data->forced_origin; + std::vector rvect = pre_compile_expr(wrapped, code, target_type, lval_ctx); + code.forced_origin = backup; return rvect; } @@ -1964,18 +1967,18 @@ static std::vector process_artificial_aux_vertex(Vunion_variant) { // extract a variant from a union (a union variable or a union field of a struct) - ir_obj = transition_to_target_type(std::move(ir_obj), code, t_orig, data->union_variant, v->loc); + ir_obj = transition_to_target_type(std::move(ir_obj), code, t_orig, data->union_variant, wrapped); } // `load_info` contains instructions to skip, load, save tail, etc.; // it generates LETs to ir_obj, so stack slots of lazy_variable will contain loaded data - generate_lazy_struct_from_slice(code, v->loc, lazy_variable, data->load_info, ir_obj); - return transition_to_target_type({}, code, target_type, v); + generate_lazy_struct_from_slice(code, wrapped, lazy_variable, data->load_info, ir_obj); + return transition_to_target_type({}, code, target_type, wrapped); } // aux "match(lazyUnion)" / aux "match(obj.lastUnionField)" if (const auto* data = dynamic_cast(v->aux_data)) { - V v_match = v->get_wrapped_expr()->as(); + V v_match = wrapped->as(); pre_compile_expr(v_match->get_subject(), code, nullptr); const LazyVariableLoadedState* lazy_variable = code.get_lazy_variable(data->var_ref); @@ -2004,13 +2007,13 @@ static std::vector process_artificial_aux_vertex(Vloc, t_union, lazy_variable, options); - return transition_to_target_type(std::move(ir_match), code, target_type, v); + std::vector ir_match = generate_lazy_match_for_union(code, v_match, t_union, lazy_variable, options); + return transition_to_target_type(std::move(ir_match), code, target_type, wrapped); } if (const auto* data = dynamic_cast(v->aux_data)) { - std::vector rvect = data->generate_get_InMessage_field(code, v->loc); - return transition_to_target_type(std::move(rvect), code, target_type, v); + std::vector rvect = data->generate_get_InMessage_field(code, wrapped); + return transition_to_target_type(std::move(rvect), code, target_type, wrapped); } tolk_assert(false); @@ -2111,25 +2114,25 @@ static void process_assert_statement(V v, CodeBlob& code) std::vector ir_cond = pre_compile_expr(v->get_cond(), code); tolk_assert(ir_cond.size() == 1 && ir_thrown_code.size() == 1); - std::vector args_vars = { ir_thrown_code[0], ir_cond[0], code.create_int(v->loc, 0, "(assert-0)") }; + std::vector args_vars = { ir_thrown_code[0], ir_cond[0], code.create_int(v, 0, "(assert-0)") }; FunctionPtr builtin_sym = lookup_function("__throw_if_unless"); - gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); + gen_op_call(code, TypeDataVoid::create(), v, std::move(args_vars), builtin_sym, "(throw-call)"); } static void process_catch_variable(AnyExprV v_catch_var, CodeBlob& code) { if (auto v_ref = v_catch_var->try_as(); v_ref && v_ref->sym) { // not underscore LocalVarPtr var_ref = v_ref->sym->try_as(); tolk_assert(var_ref->ir_idx.empty()); - var_ref->mutate()->assign_ir_idx(code.create_var(v_catch_var->inferred_type, v_catch_var->loc, var_ref->name)); + var_ref->mutate()->assign_ir_idx(code.create_var(v_catch_var->inferred_type, v_catch_var, var_ref->name)); } } static void process_try_catch_statement(V v, CodeBlob& code) { code.require_callxargs = true; - Op& try_catch_op = code.emplace_back(v->loc, Op::_TryCatch); + Op& try_catch_op = code.emplace_back(v, Op::_TryCatch); code.push_set_cur(try_catch_op.block0); process_any_statement(v->get_try_body(), code); - code.close_pop_cur(v->get_try_body()->loc_end); + code.close_pop_cur(v->get_try_body()); code.push_set_cur(try_catch_op.block1); // transform catch (excNo, arg) into TVM-catch (arg, excNo), where arg is untyped and thus almost useless now @@ -2139,20 +2142,20 @@ static void process_try_catch_statement(V v, CodeBlob& process_catch_variable(catch_vars[1], code); try_catch_op.left = pre_compile_tensor(code, {catch_vars[1], catch_vars[0]}); process_any_statement(v->get_catch_body(), code); - code.close_pop_cur(v->get_catch_body()->loc_end); + code.close_pop_cur(v->get_catch_body()); } static void process_repeat_statement(V v, CodeBlob& code) { - std::vector tmp_vars = pre_compile_expr(v->get_cond(), code, nullptr); - Op& repeat_op = code.emplace_back(v->loc, Op::_Repeat, tmp_vars); + std::vector tmp_vars = pre_compile_expr(v->get_cond(), code, nullptr); + Op& repeat_op = code.emplace_back(v, Op::_Repeat, tmp_vars); code.push_set_cur(repeat_op.block0); process_any_statement(v->get_body(), code); - code.close_pop_cur(v->get_body()->loc_end); + code.close_pop_cur(v->get_body()); } static void process_if_statement(V v, CodeBlob& code) { - std::vector cond = pre_compile_expr(v->get_cond(), code, nullptr); - tolk_assert(cond.size() == 1); + std::vector ir_cond = pre_compile_expr(v->get_cond(), code, nullptr); + tolk_assert(ir_cond.size() == 1); if (v->get_cond()->is_always_true) { process_any_statement(v->get_if_body(), code); // v->is_ifnot does not matter here @@ -2163,26 +2166,26 @@ static void process_if_statement(V v, CodeBlob& code) { return; } - Op& if_op = code.emplace_back(v->loc, Op::_If, std::move(cond)); + Op& if_op = code.emplace_back(v, Op::_If, std::move(ir_cond)); code.push_set_cur(if_op.block0); process_any_statement(v->get_if_body(), code); if (v == stmt_before_immediate_return) { - code.emplace_back(v->get_if_body()->loc_end, Op::_Return); + code.emplace_back(v->get_if_body(), Op::_Return); } - code.close_pop_cur(v->get_if_body()->loc_end); + code.close_pop_cur(v->get_if_body()); code.push_set_cur(if_op.block1); process_any_statement(v->get_else_body(), code); if (v == stmt_before_immediate_return) { - code.emplace_back(v->get_else_body()->loc_end, Op::_Return); + code.emplace_back(v->get_else_body(), Op::_Return); } - code.close_pop_cur(v->get_else_body()->loc_end); + code.close_pop_cur(v->get_else_body()); if (v->is_ifnot) { std::swap(if_op.block0, if_op.block1); } } static void process_do_while_statement(V v, CodeBlob& code) { - Op& until_op = code.emplace_back(v->loc, Op::_Until); + Op& until_op = code.emplace_back(v, Op::_Until); code.push_set_cur(until_op.block0); process_any_statement(v->get_body(), code); @@ -2195,21 +2198,21 @@ static void process_do_while_statement(V v, CodeBlob& co if (auto v_not = cond->try_as(); v_not && v_not->tok == tok_logical_not) { until_cond = v_not->get_rhs(); } else if (auto v_eq = cond->try_as(); v_eq && v_eq->tok == tok_eq) { - until_cond = createV(cond->loc, "!=", tok_neq, v_eq->get_lhs(), v_eq->get_rhs()); + until_cond = createV(cond->range, v_eq->operator_range, "!=", tok_neq, v_eq->get_lhs(), v_eq->get_rhs()); } else if (auto v_neq = cond->try_as(); v_neq && v_neq->tok == tok_neq) { - until_cond = createV(cond->loc, "==", tok_eq, v_neq->get_lhs(), v_neq->get_rhs()); + until_cond = createV(cond->range, v_neq->operator_range, "==", tok_eq, v_neq->get_lhs(), v_neq->get_rhs()); } else if (auto v_leq = cond->try_as(); v_leq && v_leq->tok == tok_leq) { - until_cond = createV(cond->loc, ">", tok_gt, v_leq->get_lhs(), v_leq->get_rhs()); + until_cond = createV(cond->range, v_leq->operator_range, ">", tok_gt, v_leq->get_lhs(), v_leq->get_rhs()); } else if (auto v_lt = cond->try_as(); v_lt && v_lt->tok == tok_lt) { - until_cond = createV(cond->loc, ">=", tok_geq, v_lt->get_lhs(), v_lt->get_rhs()); + until_cond = createV(cond->range, v_lt->operator_range, ">=", tok_geq, v_lt->get_lhs(), v_lt->get_rhs()); } else if (auto v_geq = cond->try_as(); v_geq && v_geq->tok == tok_geq) { - until_cond = createV(cond->loc, "<", tok_lt, v_geq->get_lhs(), v_geq->get_rhs()); + until_cond = createV(cond->range, v_geq->operator_range, "<", tok_lt, v_geq->get_lhs(), v_geq->get_rhs()); } else if (auto v_gt = cond->try_as(); v_gt && v_gt->tok == tok_gt) { - until_cond = createV(cond->loc, "<=", tok_geq, v_gt->get_lhs(), v_gt->get_rhs()); + until_cond = createV(cond->range, v_gt->operator_range, "<=", tok_geq, v_gt->get_lhs(), v_gt->get_rhs()); } else if (cond->inferred_type->unwrap_alias() == TypeDataBool::create()) { - until_cond = createV(cond->loc, "!b", tok_logical_not, cond); + until_cond = createV(cond->range, cond->range, "!b", tok_logical_not, cond); } else { - until_cond = createV(cond->loc, "!", tok_logical_not, cond); + until_cond = createV(cond->range, cond->range, "!", tok_logical_not, cond); } until_cond->mutate()->assign_inferred_type(TypeDataInt::create()); if (auto v_bin = until_cond->try_as(); v_bin && !v_bin->fun_ref) { @@ -2220,29 +2223,29 @@ static void process_do_while_statement(V v, CodeBlob& co until_op.left = pre_compile_expr(until_cond, code, nullptr); tolk_assert(until_op.left.size() == 1); - code.close_pop_cur(v->get_body()->loc_end); + code.close_pop_cur(v->get_body()); } static void process_while_statement(V v, CodeBlob& code) { - Op& while_op = code.emplace_back(v->loc, Op::_While); + Op& while_op = code.emplace_back(v, Op::_While); code.push_set_cur(while_op.block0); while_op.left = pre_compile_expr(v->get_cond(), code, nullptr); tolk_assert(while_op.left.size() == 1); - code.close_pop_cur(v->get_body()->loc); + code.close_pop_cur(v->get_body()); code.push_set_cur(while_op.block1); process_any_statement(v->get_body(), code); - code.close_pop_cur(v->get_body()->loc_end); + code.close_pop_cur(v->get_body()); } static void process_throw_statement(V v, CodeBlob& code) { if (v->has_thrown_arg()) { FunctionPtr builtin_sym = lookup_function("__throw_arg"); - std::vector args_vars = pre_compile_tensor(code, {v->get_thrown_arg(), v->get_thrown_code()}); - gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); + std::vector args_vars = pre_compile_tensor(code, {v->get_thrown_arg(), v->get_thrown_code()}); + gen_op_call(code, TypeDataVoid::create(), v, std::move(args_vars), builtin_sym, "(throw-call)"); } else { FunctionPtr builtin_sym = lookup_function("__throw"); - std::vector args_vars = pre_compile_tensor(code, {v->get_thrown_code()}); - gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); + std::vector args_vars = pre_compile_tensor(code, {v->get_thrown_code()}); + gen_op_call(code, TypeDataVoid::create(), v, std::move(args_vars), builtin_sym, "(throw-call)"); } } @@ -2255,7 +2258,7 @@ static void process_return_statement(V v, CodeBlob& code) if (fun_ref->does_return_self()) { child_target_type = fun_ref->parameters[0].declared_type; } - std::vector return_vars = pre_compile_expr(v->get_return_value(), code, child_target_type); + std::vector return_vars = pre_compile_expr(v->get_return_value(), code, child_target_type); if (fun_ref->does_return_self()) { return_vars = {}; @@ -2272,9 +2275,9 @@ static void process_return_statement(V v, CodeBlob& code) // if fun_ref is called and inlined into a parent, assign a result instead of generating a return statement if (code.inline_rvect_out) { - code.emplace_back(v->loc, Op::_Let, *code.inline_rvect_out, std::move(return_vars)); + code.emplace_back(v, Op::_Let, *code.inline_rvect_out, std::move(return_vars)); } else { - code.emplace_back(v->loc, Op::_Return, std::move(return_vars)); + code.emplace_back(v, Op::_Return, std::move(return_vars)); } } @@ -2282,7 +2285,7 @@ static void process_return_statement(V v, CodeBlob& code) // if it's not reachable, it will be dropped // (IR cfg reachability may differ from FlowContext in case of "never" types, so there may be situations, // when IR will consider this "return" reachable and leave it, but actually execution will never reach it) -static void append_implicit_return_statement(SrcLocation loc_end, CodeBlob& code) { +static void append_implicit_return_statement(CodeBlob& code) { std::vector mutated_vars; if (code.fun_ref->has_mutate_params()) { for (const LocalVarData& p_sym: code.fun_ref->parameters) { @@ -2291,7 +2294,7 @@ static void append_implicit_return_statement(SrcLocation loc_end, CodeBlob& code } } } - code.emplace_back(loc_end, Op::_Return, std::move(mutated_vars)); + code.emplace_back(code.fun_ref->ident_anchor, Op::_Return, std::move(mutated_vars)); } @@ -2335,22 +2338,22 @@ static void convert_function_body_to_CodeBlob(FunctionPtr fun_ref, FunctionBodyC for (int i = 0; i < fun_ref->get_num_params(); ++i) { const LocalVarData& param_i = fun_ref->parameters[i]; - std::vector ir_idx = blob->create_var(param_i.declared_type, param_i.loc, param_i.name); - rvect_import.insert(rvect_import.end(), ir_idx.begin(), ir_idx.end()); - param_i.mutate()->assign_ir_idx(std::move(ir_idx)); + std::vector ir_param = blob->create_var(param_i.declared_type, param_i.ident_anchor, param_i.name); + rvect_import.insert(rvect_import.end(), ir_param.begin(), ir_param.end()); + param_i.mutate()->assign_ir_idx(std::move(ir_param)); } - blob->emplace_back(fun_ref->loc, Op::_Import, rvect_import); + blob->emplace_back(fun_ref->ident_anchor, Op::_Import, rvect_import); blob->in_var_cnt = blob->var_cnt; tolk_assert(blob->var_cnt == total_arg_width); if (fun_ref->name == "onInternalMessage") { - handle_onInternalMessage_codegen_start(fun_ref, rvect_import, *blob, fun_ref->loc); + handle_onInternalMessage_codegen_start(fun_ref, rvect_import, *blob, fun_ref->ident_anchor); } process_block_statement(v_body, *blob); - append_implicit_return_statement(v_body->loc_end, *blob); + append_implicit_return_statement(*blob); - blob->close_blk(v_body->loc_end); + blob->close_blk(v_body); code_body->set_code(blob); tolk_assert(vars_modification_watcher.empty()); } @@ -2365,7 +2368,7 @@ static void convert_asm_body_to_AsmOp(FunctionPtr fun_ref, FunctionBodyAsm* asm_ for (char c : ops) { if (c == '\n' || c == '\r') { if (!op.empty()) { - asm_ops.push_back(AsmOp::Parse({}, op, cnt, width)); + asm_ops.push_back(AsmOp::Parse(nullptr, op, cnt, width)); if (asm_ops.back().is_custom()) { cnt = width; } @@ -2376,7 +2379,7 @@ static void convert_asm_body_to_AsmOp(FunctionPtr fun_ref, FunctionBodyAsm* asm_ } } if (!op.empty()) { - asm_ops.push_back(AsmOp::Parse({}, op, cnt, width)); + asm_ops.push_back(AsmOp::Parse(nullptr, op, cnt, width)); if (asm_ops.back().is_custom()) { cnt = width; } @@ -2429,7 +2432,7 @@ class UpdateArgRetOrderConsideringStackWidth final { if (!fun_ref->ret_order.empty()) { size_t expected_width = fun_ref->inferred_return_type->get_width_on_stack() + total_arg_mutate_width; if (expected_width != fun_ref->ret_order.size()) { - v_function->get_body()->error("ret_order (after ->) expected to contain " + std::to_string(expected_width) + " numbers"); + fire(v_function->get_body(), "ret_order (after ->) expected to contain " + std::to_string(expected_width) + " numbers"); } } } diff --git a/tolk/pipe-calc-rvalue-lvalue.cpp b/tolk/pipe-calc-rvalue-lvalue.cpp index dc269a342e..cdb111c1b3 100644 --- a/tolk/pipe-calc-rvalue-lvalue.cpp +++ b/tolk/pipe-calc-rvalue-lvalue.cpp @@ -14,9 +14,9 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . */ -#include "tolk.h" #include "ast.h" #include "ast-visitor.h" +#include "compilation-errors.h" /* * This pipe assigns lvalue/rvalue flags for AST expressions. diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index a7bbe549f1..afd181fde3 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -14,9 +14,9 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "tolk.h" #include "ast.h" #include "ast-visitor.h" +#include "compilation-errors.h" #include "type-system.h" namespace tolk { @@ -52,7 +52,8 @@ static std::string expression_as_string(AnyExprV v) { // fire a general error on type mismatch; for example, "can not assign `cell` to `slice`"; // for instance, if `as` operator is applicable, compiler will suggest it GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_type_mismatch(FunctionPtr cur_f, SrcLocation loc, const char* text_tpl, TypePtr src, TypePtr dst) { +// todo not range but at +static void fire_type_mismatch(FunctionPtr cur_f, SrcRange range, const char* text_tpl, TypePtr src, TypePtr dst) { #ifdef TOLK_DEBUG tolk_assert(!dst->can_rhs_be_assigned(src)); #endif @@ -78,39 +79,39 @@ static void fire_error_type_mismatch(FunctionPtr cur_f, SrcLocation loc, const c message += "\nhint: alternatively, use `!` operator to bypass nullability checks: `!`"; } } - fire(cur_f, loc, message); + fire(cur_f, range, message); } // fire an error on `!cell` / `+slice` GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_cannot_apply_operator(FunctionPtr cur_f, SrcLocation loc, std::string_view operator_name, AnyExprV unary_expr) { +static void fire_cannot_apply_operator(FunctionPtr cur_f, SrcRange operator_range, std::string_view operator_name, AnyExprV unary_expr) { std::string op = static_cast(operator_name); - fire(cur_f, loc, "can not apply operator `" + op + "` to " + to_string(unary_expr->inferred_type)); + fire(cur_f, operator_range, "can not apply operator `" + op + "` to " + to_string(unary_expr->inferred_type)); } // fire an error on `int + cell` / `slice & int` GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_cannot_apply_operator(FunctionPtr cur_f, SrcLocation loc, std::string_view operator_name, AnyExprV lhs, AnyExprV rhs) { +static void fire_cannot_apply_operator(FunctionPtr cur_f, SrcRange operator_range, std::string_view operator_name, AnyExprV lhs, AnyExprV rhs) { std::string op = static_cast(operator_name); - fire(cur_f, loc, "can not apply operator `" + op + "` to " + to_string(lhs->inferred_type) + " and " + to_string(rhs->inferred_type)); + fire(cur_f, operator_range, "can not apply operator `" + op + "` to " + to_string(lhs->inferred_type) + " and " + to_string(rhs->inferred_type)); } GNU_ATTRIBUTE_NOINLINE -static void warning_condition_always_true_or_false(FunctionPtr cur_f, SrcLocation loc, AnyExprV cond, const char* operator_name) { +static void warning_condition_always_true_or_false(FunctionPtr cur_f, SrcRange keyword_range, AnyExprV cond, const char* operator_name) { bool no_warning = cond->kind == ast_bool_const || cond->kind == ast_int_const; if (no_warning) { // allow `while(true)` without a warning return; } - loc.show_warning("condition of " + static_cast(operator_name) + " is always " + (cond->is_always_true ? "true" : "false")); + compilation_warning(cur_f, keyword_range, "condition of " + static_cast(operator_name) + " is always " + (cond->is_always_true ? "true" : "false")); } // given `f(x: int)` and a call `f(expr)`, check that expr_type is assignable to `int` static void check_function_argument_passed(FunctionPtr cur_f, TypePtr param_type, AnyExprV ith_arg, bool is_obj_of_dot_call) { if (!param_type->can_rhs_be_assigned(ith_arg->inferred_type)) { if (is_obj_of_dot_call) { - fire(cur_f, ith_arg->loc, "can not call method for " + to_string(param_type) + " with object of type " + to_string(ith_arg)); + fire(cur_f, ith_arg, "can not call method for " + to_string(param_type) + " with object of type " + to_string(ith_arg)); } else { - fire_error_type_mismatch(cur_f, ith_arg->loc, "can not pass {src} to {dst}", ith_arg->inferred_type, param_type); + fire_type_mismatch(cur_f, ith_arg->range, "can not pass {src} to {dst}", ith_arg->inferred_type, param_type); } } } @@ -120,9 +121,9 @@ static void check_function_argument_passed(FunctionPtr cur_f, TypePtr param_type static void check_function_argument_mutate_back(FunctionPtr cur_f, TypePtr param_type, AnyExprV ith_arg, bool is_obj_of_dot_call) { if (!ith_arg->inferred_type->can_rhs_be_assigned(param_type)) { if (is_obj_of_dot_call) { - fire(cur_f, ith_arg->loc,"can not call method for mutate " + to_string(param_type) + " with object of type " + to_string(ith_arg) + ", because mutation is not type compatible"); + fire(cur_f, ith_arg,"can not call method for mutate " + to_string(param_type) + " with object of type " + to_string(ith_arg) + ", because mutation is not type compatible"); } else { - fire(cur_f, ith_arg->loc,"can not pass " + to_string(ith_arg) + " to mutate " + to_string(param_type) + ", because mutation is not type compatible"); + fire(cur_f, ith_arg,"can not pass " + to_string(ith_arg) + " to mutate " + to_string(param_type) + ", because mutation is not type compatible"); } } } @@ -132,15 +133,17 @@ static void check_function_argument_mutate_back(FunctionPtr cur_f, TypePtr param // so, it's better to see an error on assignment, that later, on `n` usage and types mismatch // (most common is situation above, but generally, `var (x,n) = xn` where xn is a tensor with 2-nd always-null, can be) GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_assign_always_null_to_variable(FunctionPtr cur_f, SrcLocation loc, LocalVarPtr assigned_var, bool is_assigned_null_literal) { +// todo pass not range +static void fire_assign_always_null_to_variable(FunctionPtr cur_f, SrcRange range, LocalVarPtr assigned_var, bool is_assigned_null_literal) { std::string var_name = assigned_var->name; - fire(cur_f, loc, "can not infer type of `" + var_name + "`, it's always null\nspecify its type with `" + var_name + ": `" + (is_assigned_null_literal ? " or use `null as `" : "")); + fire(cur_f, range, "can not infer type of `" + var_name + "`, it's always null\nspecify its type with `" + var_name + ": `" + (is_assigned_null_literal ? " or use `null as `" : "")); } // fire an error on `untypedTupleVar.0` when inferred as (int,int), or `[int, (int,int)]`, or other non-1 width in a tuple GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_cannot_put_non1_stack_width_arg_to_tuple(FunctionPtr cur_f, SrcLocation loc, TypePtr inferred_type) { - fire(cur_f, loc, "a tuple can not have " + to_string(inferred_type) + " inside, because it occupies " + std::to_string(inferred_type->get_width_on_stack()) + " stack slots in TVM, not 1"); +// todo pass not ra +static void fire_cannot_put_non1_stack_width_arg_to_tuple(FunctionPtr cur_f, SrcRange range, TypePtr inferred_type) { + fire(cur_f, range, "a tuple can not have " + to_string(inferred_type) + " inside, because it occupies " + std::to_string(inferred_type->get_width_on_stack()) + " stack slots in TVM, not 1"); } // handle __expect_type(expr, "type") call @@ -148,14 +151,14 @@ static void fire_error_cannot_put_non1_stack_width_arg_to_tuple(FunctionPtr cur_ GNU_ATTRIBUTE_NOINLINE GNU_ATTRIBUTE_COLD static void handle_possible_compiler_internal_call(FunctionPtr cur_f, V v) { FunctionPtr fun_ref = v->fun_maybe; - tolk_assert(fun_ref && fun_ref->is_builtin_function()); + tolk_assert(fun_ref && fun_ref->is_builtin()); if (fun_ref->name == "__expect_type") { tolk_assert(v->get_num_args() == 2); std::string_view expected_type_str = v->get_arg(1)->get_expr()->as()->str_val; TypePtr expr_type = v->get_arg(0)->inferred_type; if (expected_type_str != expr_type->as_human_readable()) { - fire(cur_f, v->loc, "__expect_type failed: expected `" + to_string(expected_type_str) + "`, got " + to_string(expr_type)); + fire(cur_f, v, "__expect_type failed: expected `" + to_string(expected_type_str) + "`, got " + to_string(expr_type)); } } } @@ -235,7 +238,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } // using += for other types (e.g. `tensorVar += tensorVar`) is not allowed if (!types_ok) { - fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); + fire_cannot_apply_operator(cur_f, v->operator_range, v->operator_name, lhs, rhs); } } @@ -246,12 +249,12 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { switch (v->tok) { case tok_logical_not: if (!expect_integer(rhs) && !expect_boolean(rhs)) { - fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, rhs); + fire_cannot_apply_operator(cur_f, v->operator_range, v->operator_name, rhs); } break; default: if (!expect_integer(rhs)) { - fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, rhs); + fire_cannot_apply_operator(cur_f, v->operator_range, v->operator_name, rhs); } } } @@ -272,9 +275,9 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { bool not_integer_comparison = false; if (!check_eq_neq_operator(lhs->inferred_type, rhs->inferred_type, not_integer_comparison)) { if (lhs->inferred_type->equal_to(rhs->inferred_type)) { // compare slice with slice, int? with int? - fire(cur_f, v->loc, "type " + to_string(lhs) + " can not be compared with `== !=`"); + fire(cur_f, v->operator_range, "type " + to_string(lhs) + " can not be compared with `== !=`"); } else { - fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); + fire_cannot_apply_operator(cur_f, v->operator_range, v->operator_name, lhs, rhs); } } if (not_integer_comparison) { // special handling at IR generation like for `address` @@ -289,7 +292,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { case tok_geq: case tok_spaceship: if (!expect_integer(lhs) || !expect_integer(rhs)) { - fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); + fire_cannot_apply_operator(cur_f, v->operator_range, v->operator_name, lhs, rhs); } break; // & | ^ are "overloaded" both for integers and booleans, (int & bool) is NOT allowed @@ -300,7 +303,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { bool both_int = expect_integer(lhs) && expect_integer(rhs); bool both_bool = expect_boolean(lhs) && expect_boolean(rhs); if (!both_int && !both_bool) { - fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); + fire_cannot_apply_operator(cur_f, v->operator_range, v->operator_name, lhs, rhs); } break; } @@ -310,7 +313,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { bool lhs_ok = expect_integer(lhs) || expect_boolean(lhs); bool rhs_ok = expect_integer(rhs) || expect_boolean(rhs); if (!lhs_ok || !rhs_ok) { - fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); + fire_cannot_apply_operator(cur_f, v->operator_range, v->operator_name, lhs, rhs); } break; } @@ -318,7 +321,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // they are allowed for intN (int16 + int32 is ok) and always "fall back" to general int default: if (!expect_integer(lhs) || !expect_integer(rhs)) { - fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); + fire_cannot_apply_operator(cur_f, v->operator_range, v->operator_name, lhs, rhs); } } } @@ -327,7 +330,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { parent::visit(v->get_expr()); if (!v->get_expr()->inferred_type->can_be_casted_with_as_operator(v->type_node->resolved_type)) { - fire(cur_f, v->loc, "type " + to_string(v->get_expr()) + " can not be cast to " + to_string(v->type_node->resolved_type)); + fire(cur_f, v, "type " + to_string(v->get_expr()) + " can not be cast to " + to_string(v->type_node->resolved_type)); } } @@ -336,14 +339,14 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { TypePtr rhs_type = v->type_node->resolved_type; if (rhs_type->unwrap_alias()->try_as()) { // `v is T1 | T2` / `v is T?` is disallowed - fire(cur_f, v->loc, "union types are not allowed, use concrete types in `is`"); + fire(cur_f, v, "union types are not allowed, use concrete types in `is`"); } if ((v->is_always_true && !v->is_negated) || (v->is_always_false && v->is_negated)) { - v->loc.show_warning(expression_as_string(v->get_expr()) + " is always " + to_string(rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); + compilation_warning(cur_f, v, expression_as_string(v->get_expr()) + " is always " + to_string(rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); } if ((v->is_always_false && !v->is_negated) || (v->is_always_true && v->is_negated)) { - v->loc.show_warning(expression_as_string(v->get_expr()) + " of type " + to_string(v->get_expr()) + " can never be " + to_string(rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); + compilation_warning(cur_f, v, expression_as_string(v->get_expr()) + " of type " + to_string(v->get_expr()) + " can never be " + to_string(rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); } } @@ -352,7 +355,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (v->get_expr()->inferred_type == TypeDataNullLiteral::create()) { // operator `!` used for always-null (proven by smart casts, for example), it's an error - fire(cur_f, v->loc, "operator `!` used for always null expression"); + fire(cur_f, v, "operator `!` used for always null expression"); } // if operator `!` used for non-nullable, probably a warning should be printed } @@ -363,7 +366,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { for (int i = 0; i < v->size(); ++i) { AnyExprV item = v->get_item(i); if (item->inferred_type->get_width_on_stack() != 1) { - fire_error_cannot_put_non1_stack_width_arg_to_tuple(cur_f, v->get_item(i)->loc, item->inferred_type); + fire_cannot_put_non1_stack_width_arg_to_tuple(cur_f, v->get_item(i)->range, item->inferred_type); } } } @@ -374,7 +377,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (v->is_target_indexed_access()) { TypePtr obj_type = v->get_obj()->inferred_type->unwrap_alias(); if (v->inferred_type->get_width_on_stack() != 1 && (obj_type->try_as() || obj_type->try_as())) { - fire_error_cannot_put_non1_stack_width_arg_to_tuple(cur_f, v->loc, v->inferred_type); + fire_cannot_put_non1_stack_width_arg_to_tuple(cur_f, v->range, v->inferred_type); } } } @@ -391,7 +394,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { auto arg_i = v->get_arg(i)->get_expr(); TypePtr param_type = f_callable->params_types[i]; if (!param_type->can_rhs_be_assigned(arg_i->inferred_type)) { - fire_error_type_mismatch(cur_f, arg_i->loc, "can not pass {src} to {dst}", arg_i->inferred_type, param_type); + fire_type_mismatch(cur_f, arg_i->range, "can not pass {src} to {dst}", arg_i->inferred_type, param_type); } } return; @@ -419,7 +422,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } - if (fun_ref->is_builtin_function() && fun_ref->name[0] == '_') { + if (fun_ref->is_builtin() && fun_ref->name[0] == '_') { handle_possible_compiler_internal_call(cur_f, v); } } @@ -453,11 +456,11 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } if (declared_type) { if (!declared_type->can_rhs_be_assigned(rhs_type)) { - fire_error_type_mismatch(cur_f, err_loc->loc, "can not assign {src} to variable of type {dst}", rhs_type, declared_type); + fire_type_mismatch(cur_f, err_loc->range, "can not assign {src} to variable of type {dst}", rhs_type, declared_type); } } else { if (rhs_type == TypeDataNullLiteral::create()) { - fire_error_assign_always_null_to_variable(cur_f, err_loc->loc, lhs_var->var_ref->try_as(), corresponding_maybe_rhs && corresponding_maybe_rhs->kind == ast_null_keyword); + fire_assign_always_null_to_variable(cur_f, err_loc->range, lhs_var->var_ref->try_as(), corresponding_maybe_rhs && corresponding_maybe_rhs->kind == ast_null_keyword); } } return; @@ -468,10 +471,10 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (auto lhs_tensor = lhs->try_as()) { const TypeDataTensor* rhs_type_tensor = rhs_type->unwrap_alias()->try_as(); if (!rhs_type_tensor) { - fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to a tensor"); + fire(cur_f, err_loc, "can not assign " + to_string(rhs_type) + " to a tensor"); } if (lhs_tensor->size() != rhs_type_tensor->size()) { - fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + ", sizes mismatch"); + fire(cur_f, err_loc, "can not assign " + to_string(rhs_type) + ", sizes mismatch"); } V rhs_tensor_maybe = corresponding_maybe_rhs ? corresponding_maybe_rhs->try_as() : nullptr; for (int i = 0; i < lhs_tensor->size(); ++i) { @@ -485,10 +488,10 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (auto lhs_tuple = lhs->try_as()) { const TypeDataBrackets* rhs_type_tuple = rhs_type->unwrap_alias()->try_as(); if (!rhs_type_tuple) { - fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to a tuple"); + fire(cur_f, err_loc, "can not assign " + to_string(rhs_type) + " to a tuple"); } if (lhs_tuple->size() != rhs_type_tuple->size()) { - fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + ", sizes mismatch"); + fire(cur_f, err_loc, "can not assign " + to_string(rhs_type) + ", sizes mismatch"); } V rhs_tuple_maybe = corresponding_maybe_rhs ? corresponding_maybe_rhs->try_as() : nullptr; for (int i = 0; i < lhs_tuple->size(); ++i) { @@ -501,7 +504,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (auto lhs_dot = lhs->try_as()) { if (lhs_dot->is_target_indexed_access() && lhs_dot->get_obj()->inferred_type->unwrap_alias() == TypeDataTuple::create()) { if (rhs_type->get_width_on_stack() != 1) { - fire_error_cannot_put_non1_stack_width_arg_to_tuple(cur_f, err_loc->loc, rhs_type); + fire_cannot_put_non1_stack_width_arg_to_tuple(cur_f, err_loc->range, rhs_type); } } } @@ -511,11 +514,11 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // for strange lhs like `f() = rhs` type checking will pass, but will fail lvalue check later if (!lhs->inferred_type->can_rhs_be_assigned(rhs_type)) { if (lhs->try_as()) { - fire_error_type_mismatch(cur_f, err_loc->loc, "can not assign {src} to variable of type {dst}", rhs_type, lhs->inferred_type); + fire_type_mismatch(cur_f, err_loc->range, "can not assign {src} to variable of type {dst}", rhs_type, lhs->inferred_type); } else if (lhs->try_as()) { - fire_error_type_mismatch(cur_f, err_loc->loc, "can not assign {src} to field of type {dst}", rhs_type, lhs->inferred_type); + fire_type_mismatch(cur_f, err_loc->range, "can not assign {src} to field of type {dst}", rhs_type, lhs->inferred_type); } else { - fire_error_type_mismatch(cur_f, err_loc->loc, "can not assign {src} to {dst}", rhs_type, lhs->inferred_type); + fire_type_mismatch(cur_f, err_loc->range, "can not assign {src} to {dst}", rhs_type, lhs->inferred_type); } } } @@ -525,14 +528,14 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (cur_f->does_return_self()) { if (!is_expr_valid_as_return_self(v->get_return_value())) { - fire(cur_f, v->loc, "invalid return from `self` function"); + fire(cur_f, v, "invalid return from `self` function"); } return; } TypePtr expr_type = v->get_return_value()->inferred_type; if (!cur_f->inferred_return_type->can_rhs_be_assigned(expr_type)) { - fire_error_type_mismatch(cur_f, v->get_return_value()->loc, "can not convert type {src} to return type {dst}", expr_type, cur_f->inferred_return_type); + fire_type_mismatch(cur_f, v->get_return_value()->range, "can not convert type {src} to return type {dst}", expr_type, cur_f->inferred_return_type); } } @@ -557,11 +560,11 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { AnyExprV cond = v->get_cond(); if (!expect_integer(cond) && !expect_boolean(cond)) { - fire(cur_f, cond->loc, "can not use " + to_string(cond) + " as a boolean condition"); + fire(cur_f, cond, "can not use " + to_string(cond) + " as a boolean condition"); } if (cond->is_always_true || cond->is_always_false) { - warning_condition_always_true_or_false(cur_f, v->loc, cond, "ternary operator"); + warning_condition_always_true_or_false(cur_f, cond->range, cond, "ternary operator"); } } @@ -584,56 +587,57 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { switch (v_arm->pattern_kind) { case MatchArmKind::exact_type: { if (has_expr_arm) { - fire(cur_f, v_arm->loc, "can not mix type and expression patterns in `match`"); + // todo here and below: not v_arm as range, but before => + fire(cur_f, v_arm, "can not mix type and expression patterns in `match`"); } if (has_else_arm) { - fire(cur_f, v_arm->loc, "`else` branch should be the last"); + fire(cur_f, v_arm, "`else` branch should be the last"); } has_type_arm = true; TypePtr lhs_type = v_arm->pattern_type_node->resolved_type; // `lhs_type => ...` if (lhs_type->unwrap_alias()->try_as()) { - fire(cur_f, v_arm->loc, "wrong pattern matching: union types are not allowed, use concrete types in `match`"); + fire(cur_f, v_arm, "wrong pattern matching: union types are not allowed, use concrete types in `match`"); } bool can_happen = (subject_union && subject_union->has_variant_equal_to(lhs_type)) || (!subject_union && subject_type->equal_to(lhs_type)); if (!can_happen) { - fire(cur_f, v_arm->loc, "wrong pattern matching: " + to_string(lhs_type) + " is not a variant of " + to_string(subject_type)); + fire(cur_f, v_arm, "wrong pattern matching: " + to_string(lhs_type) + " is not a variant of " + to_string(subject_type)); } auto it_mentioned = std::find_if(covered_types.begin(), covered_types.end(), [lhs_type](TypePtr existing) { return existing->equal_to(lhs_type); }); if (it_mentioned != covered_types.end()) { - fire(cur_f, v_arm->loc, "wrong pattern matching: duplicated " + to_string(lhs_type)); + fire(cur_f, v_arm, "wrong pattern matching: duplicated " + to_string(lhs_type)); } covered_types.push_back(lhs_type); break; } case MatchArmKind::const_expression: { if (has_type_arm) { - fire(cur_f, v_arm->loc, "can not mix type and expression patterns in `match`"); + fire(cur_f, v_arm, "can not mix type and expression patterns in `match`"); } if (has_else_arm) { - fire(cur_f, v_arm->loc, "`else` branch should be the last"); + fire(cur_f, v_arm, "`else` branch should be the last"); } has_expr_arm = true; TypePtr pattern_type = v_arm->get_pattern_expr()->inferred_type; bool not_integer_comparison = false; if (!check_eq_neq_operator(pattern_type, subject_type, not_integer_comparison) || not_integer_comparison) { if (pattern_type->equal_to(subject_type)) { // `match` over `slice` etc., where operator `==` can't be applied - fire(cur_f, v_arm->loc, "wrong pattern matching: can not compare type " + to_string(subject_type) + " in `match`"); + fire(cur_f, v_arm, "wrong pattern matching: can not compare type " + to_string(subject_type) + " in `match`"); } else { - fire(cur_f, v_arm->loc, "wrong pattern matching: can not compare type " + to_string(v_arm->get_pattern_expr()) + " with match subject of type " + to_string(v_subject)); + fire(cur_f, v_arm, "wrong pattern matching: can not compare type " + to_string(v_arm->get_pattern_expr()) + " with match subject of type " + to_string(v_subject)); } } if (subject_enum) { auto l_dot = v_arm->get_pattern_expr()->try_as(); if (!l_dot || !l_dot->is_target_enum_member()) { // match (someColor) { anotherColor => } - fire(cur_f, v_arm->loc, "wrong pattern matching: `match` should contain members of a enum"); + fire(cur_f, v_arm, "wrong pattern matching: `match` should contain members of a enum"); } EnumMemberPtr member_ref = std::get(l_dot->target); if (std::find(covered_enum.begin(), covered_enum.end(), member_ref) != covered_enum.end()) { - fire(cur_f, v_arm->loc, "wrong pattern matching: duplicated enum member in `match`"); + fire(cur_f, v_arm, "wrong pattern matching: duplicated enum member in `match`"); } covered_enum.push_back(member_ref); } @@ -641,7 +645,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } default: if (has_else_arm) { - fire(cur_f, v_arm->loc, "duplicated `else` branch"); + fire(cur_f, v_arm, "duplicated `else` branch"); } if (has_type_arm) { // `else` is not allowed in `match` by type, but we don't fire an error here, @@ -654,7 +658,8 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // only `else` branch if (has_else_arm && !has_type_arm && !has_expr_arm) { - fire(cur_f, v->loc, "`match` contains only `else`, but no variants"); + // todo not v range, but keyword, write te + fire(cur_f, v, "`match` contains only `else`, but no variants"); } // fire if `match` by type is not exhaustive @@ -671,7 +676,8 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { missing += to_string(variant); } } - fire(cur_f, v->loc, "`match` does not cover all possible types; missing types are: " + missing); + // todo not v->range but keyword + fire(cur_f, v, "`match` does not cover all possible types; missing types are: " + missing); } // fire if `match` by enum is not exhaustive if (has_expr_arm && subject_enum && !has_else_arm && subject_enum->enum_ref->members.size() != covered_enum.size()) { @@ -684,14 +690,16 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { missing += member_ref->name; } } - fire(cur_f, v->loc, "`match` does not cover all possible enum members; missing members are: " + missing); + // todo not v->range but ke + fire(cur_f, v, "`match` does not cover all possible enum members; missing members are: " + missing); } // fire if `match` by enum covers all cases, but contains `else` // (note that `else` for types could exist for a lazy match; for non-lazy, it's fired later) if (has_expr_arm && subject_enum && has_else_arm && subject_enum->enum_ref->members.size() == covered_enum.size()) { for (int i = 0; i < v->get_arms_count(); ++i) { if (auto v_arm = v->get_arm(i); v_arm->pattern_kind == MatchArmKind::else_branch) { - fire(cur_f, v_arm->loc, "`match` already covers all possible enum members, `else` is invalid"); + // todo not v_arm->range but before => + fire(cur_f, v_arm, "`match` already covers all possible enum members, `else` is invalid"); } } } @@ -704,7 +712,8 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { needs_else_branch = !(arm0 && arm1 && arm0->bool_val != arm1->bool_val); } if (needs_else_branch) { - fire(cur_f, v->loc, "`match` expression should have `else` branch"); + // todo not v->range + fire(cur_f, v, "`match` expression should have `else` branch"); } } } @@ -713,7 +722,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { parent::visit(v->get_init_val()); if (!v->field_ref->declared_type->can_rhs_be_assigned(v->get_init_val()->inferred_type)) { - fire_error_type_mismatch(cur_f, v->get_init_val()->loc, "can not assign {src} to field of type {dst}", v->get_init_val()->inferred_type, v->field_ref->declared_type); + fire_type_mismatch(cur_f, v->get_init_val()->range, "can not assign {src} to field of type {dst}", v->get_init_val()->inferred_type, v->field_ref->declared_type); } } @@ -722,11 +731,11 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { AnyExprV cond = v->get_cond(); if (!expect_integer(cond) && !expect_boolean(cond)) { - fire(cur_f, cond->loc, "can not use " + to_string(cond) + " as a boolean condition"); + fire(cur_f, cond, "can not use " + to_string(cond) + " as a boolean condition"); } if (cond->is_always_true || cond->is_always_false) { - warning_condition_always_true_or_false(cur_f, v->loc, cond, "`if`"); + warning_condition_always_true_or_false(cur_f, v->keyword_range(), cond, "`if`"); } } @@ -735,7 +744,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { AnyExprV cond = v->get_cond(); if (!expect_integer(cond)) { - fire(cur_f, cond->loc, "condition of `repeat` must be an integer, got " + to_string(cond)); + fire(cur_f, cond, "condition of `repeat` must be an integer, got " + to_string(cond)); } } @@ -744,11 +753,11 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { AnyExprV cond = v->get_cond(); if (!expect_integer(cond) && !expect_boolean(cond)) { - fire(cur_f, cond->loc, "can not use " + to_string(cond) + " as a boolean condition"); + fire(cur_f, cond, "can not use " + to_string(cond) + " as a boolean condition"); } if (cond->is_always_true || cond->is_always_false) { - warning_condition_always_true_or_false(cur_f, v->loc, cond, "`while`"); + warning_condition_always_true_or_false(cur_f, v->keyword_range(), cond, "`while`"); } } @@ -757,11 +766,11 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { AnyExprV cond = v->get_cond(); if (!expect_integer(cond) && !expect_boolean(cond)) { - fire(cur_f, cond->loc, "can not use " + to_string(cond) + " as a boolean condition"); + fire(cur_f, cond, "can not use " + to_string(cond) + " as a boolean condition"); } if (cond->is_always_true || cond->is_always_false) { - warning_condition_always_true_or_false(cur_f, v->loc, cond, "`do while`"); + warning_condition_always_true_or_false(cur_f, v->keyword_range(), cond, "`do while`"); } } @@ -769,10 +778,10 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { parent::visit(v); if (!expect_thrown_code(v->get_thrown_code()->inferred_type)) { - fire(cur_f, v->get_thrown_code()->loc, "excNo of `throw` must be an integer, got " + to_string(v->get_thrown_code())); + fire(cur_f, v->get_thrown_code(), "excNo of `throw` must be an integer, got " + to_string(v->get_thrown_code())); } if (v->has_thrown_arg() && v->get_thrown_arg()->inferred_type->get_width_on_stack() != 1) { - fire(cur_f, v->get_thrown_arg()->loc, "can not throw " + to_string(v->get_thrown_arg()) + ", exception arg must occupy exactly 1 stack slot"); + fire(cur_f, v->get_thrown_arg(), "can not throw " + to_string(v->get_thrown_arg()) + ", exception arg must occupy exactly 1 stack slot"); } } @@ -781,14 +790,14 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { AnyExprV cond = v->get_cond(); if (!expect_integer(cond) && !expect_boolean(cond)) { - fire(cur_f, cond->loc, "can not use " + to_string(cond) + " as a boolean condition"); + fire(cur_f, cond, "can not use " + to_string(cond) + " as a boolean condition"); } if (!expect_thrown_code(v->get_thrown_code()->inferred_type)) { - fire(cur_f, v->get_thrown_code()->loc, "thrown excNo of `assert` must be an integer, got " + to_string(v->get_thrown_code())); + fire(cur_f, v->get_thrown_code(), "thrown excNo of `assert` must be an integer, got " + to_string(v->get_thrown_code())); } if (cond->is_always_true || cond->is_always_false) { - warning_condition_always_true_or_false(cur_f, v->loc, cond, "`assert`"); + warning_condition_always_true_or_false(cur_f, v->keyword_range(), cond, "`assert`"); } } @@ -800,7 +809,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // (printing it while inferring might be a false positive if types are incorrect, due to smart casts for example) // a more correct approach would be to access cfg here somehow, but since cfg is now available only while inferring, // a special v->first_unreachable was set specifically for this warning (again, which is correct if types match) - v->first_unreachable->loc.show_warning("unreachable code"); + compilation_warning(cur_f, v->first_unreachable, "unreachable code"); } } @@ -816,7 +825,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (fun_ref->is_implicit_return() && fun_ref->declared_return_type) { if (!fun_ref->declared_return_type->can_rhs_be_assigned(TypeDataVoid::create()) || fun_ref->does_return_self()) { - fire(fun_ref, v_function->get_body()->as()->loc_end, "missing return"); + fire(fun_ref, SrcRange::empty_at_end(v_function->get_body()->range), "missing return"); } } @@ -827,7 +836,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { TypePtr inferred_type = param_ref->default_value->inferred_type; if (!param_ref->declared_type->can_rhs_be_assigned(inferred_type)) { - fire_error_type_mismatch(fun_ref, param_ref->loc, "can not assign {src} to {dst}", inferred_type, param_ref->declared_type); + fire_type_mismatch(fun_ref, param_ref->default_value->range, "can not assign {src} to {dst}", inferred_type, param_ref->declared_type); } } } @@ -843,7 +852,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (const_ref->declared_type) { // `const a: int = ...` TypePtr inferred_type = const_ref->init_value->inferred_type; if (!const_ref->declared_type->can_rhs_be_assigned(inferred_type)) { - fire_error_type_mismatch(nullptr, const_ref->loc, "can not assign {src} to {dst}", inferred_type, const_ref->declared_type); + fire_type_mismatch(nullptr, const_ref->init_value->range, "can not assign {src} to {dst}", inferred_type, const_ref->declared_type); } } } @@ -854,7 +863,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { TypePtr inferred_type = field_ref->default_value->inferred_type; if (!field_ref->declared_type->can_rhs_be_assigned(inferred_type)) { - fire_error_type_mismatch(nullptr, field_ref->loc, "can not assign {src} to {dst}", inferred_type, field_ref->declared_type); + fire_type_mismatch(nullptr, field_ref->default_value->range, "can not assign {src} to {dst}", inferred_type, field_ref->declared_type); } } @@ -864,7 +873,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { TypePtr inferred_type = member_ref->init_value->inferred_type; if (!inferred_type->equal_to(TypeDataInt::create()) && !inferred_type->equal_to(TypeDataEnum::create(enum_ref))) { - fire(nullptr, member_ref->loc, "enum member is " + to_string(inferred_type) + ", not `int`\nhint: all enums must be integers"); + fire(member_ref->init_value, "enum member is " + to_string(inferred_type) + ", not `int`\nhint: all enums must be integers"); } } }; diff --git a/tolk/pipe-check-private-fields.cpp b/tolk/pipe-check-private-fields.cpp index 46373c7acf..ca77ae3097 100644 --- a/tolk/pipe-check-private-fields.cpp +++ b/tolk/pipe-check-private-fields.cpp @@ -14,17 +14,18 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . */ -#include "tolk.h" #include "ast.h" #include "ast-visitor.h" +#include "compilation-errors.h" #include "platform-utils.h" #include "type-system.h" namespace tolk { GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_private_field_used_outside_method(FunctionPtr cur_f, SrcLocation loc, StructPtr struct_ref, StructFieldPtr field_ref) { - throw ParseError(cur_f, loc, "field `" + struct_ref->as_human_readable() + "." + field_ref->name + "` is private"); +// todo pass not range; consider all fire_xxx functions +static void fire_private_field_used_outside_method(FunctionPtr cur_f, SrcRange range, StructPtr struct_ref, StructFieldPtr field_ref) { + fire(cur_f, range, "field `" + struct_ref->as_human_readable() + "." + field_ref->name + "` is private"); } static bool is_private_field_usage_allowed(FunctionPtr cur_f, StructPtr struct_ref) { @@ -58,7 +59,7 @@ class CheckPrivateFieldsUsageVisitor final : public ASTVisitorFunctionBody { const TypeDataStruct* obj_type = v->get_obj()->inferred_type->unwrap_alias()->try_as(); tolk_assert(obj_type); if (field_ref->is_private && !is_private_field_usage_allowed(cur_f, obj_type->struct_ref)) { - fire_error_private_field_used_outside_method(cur_f, v->loc, obj_type->struct_ref, field_ref); + fire_private_field_used_outside_method(cur_f, v->range, obj_type->struct_ref, field_ref); } } } @@ -71,7 +72,7 @@ class CheckPrivateFieldsUsageVisitor final : public ASTVisitorFunctionBody { auto v_field = v->get_body()->get_field(i); StructFieldPtr field_ref = v_field->field_ref; if (field_ref->is_private && !is_private_field_usage_allowed(cur_f, v->struct_ref)) { - fire_error_private_field_used_outside_method(cur_f, v_field->loc, v->struct_ref, field_ref); + fire_private_field_used_outside_method(cur_f, v_field->range, v->struct_ref, field_ref); } } } diff --git a/tolk/pipe-check-pure-impure.cpp b/tolk/pipe-check-pure-impure.cpp index 1a91438750..57cfa9853d 100644 --- a/tolk/pipe-check-pure-impure.cpp +++ b/tolk/pipe-check-pure-impure.cpp @@ -14,9 +14,9 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . */ -#include "tolk.h" #include "ast.h" #include "ast-visitor.h" +#include "compilation-errors.h" #include "platform-utils.h" /* @@ -27,8 +27,8 @@ namespace tolk { GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_impure_operation_inside_pure_function(FunctionPtr cur_f, SrcLocation loc) { - fire(cur_f, loc, "an impure operation in a pure function"); +static void fire_impure_operation_inside_pure_function(FunctionPtr cur_f, SrcRange range) { + fire(cur_f, range, "an impure operation in a pure function"); } class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFunctionBody { @@ -37,7 +37,7 @@ class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFuncti void fire_if_global_var(AnyExprV v) const { if (auto v_ident = v->try_as()) { if (v_ident->sym->try_as()) { - fire_error_impure_operation_inside_pure_function(cur_f, v->loc); + fire_impure_operation_inside_pure_function(cur_f, v_ident->range); } } } @@ -56,11 +56,11 @@ class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFuncti // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` if (!v->fun_maybe) { // `local_var(args)` is always impure, no considerations about what's there at runtime - fire_error_impure_operation_inside_pure_function(cur_f, v->loc); + fire_impure_operation_inside_pure_function(cur_f, v->range); } if (!v->fun_maybe->is_marked_as_pure()) { - fire_error_impure_operation_inside_pure_function(cur_f, v->loc); + fire_impure_operation_inside_pure_function(cur_f, v->range); } parent::visit(v); @@ -75,11 +75,11 @@ class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFuncti } void visit(V v) override { - fire_error_impure_operation_inside_pure_function(cur_f, v->loc); + fire_impure_operation_inside_pure_function(cur_f, v->range); } void visit(V v) override { - fire_error_impure_operation_inside_pure_function(cur_f, v->loc); + fire_impure_operation_inside_pure_function(cur_f, v->range); } public: diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp index 69251f3486..fabe391d4a 100644 --- a/tolk/pipe-check-rvalue-lvalue.cpp +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -14,9 +14,9 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . */ -#include "tolk.h" #include "ast.h" #include "ast-visitor.h" +#include "compilation-errors.h" #include "platform-utils.h" #include "type-system.h" @@ -31,85 +31,85 @@ namespace tolk { GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_cannot_be_used_as_lvalue(FunctionPtr cur_f, AnyV v, const std::string& details) { +static void fire_cannot_be_used_as_lvalue(FunctionPtr cur_f, AnyV at, const std::string& details) { // example: `f() = 32` // example: `loadUint(c.beginParse(), 32)` (since `loadUint()` mutates the first argument) - throw ParseError(cur_f, v->loc, details + " can not be used as lvalue"); + fire(cur_f, at, details + " can not be used as lvalue"); } GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_modifying_immutable_variable(FunctionPtr cur_f, SrcLocation loc, LocalVarPtr var_ref) { +static void fire_modifying_immutable_variable(FunctionPtr cur_f, SrcRange range, LocalVarPtr var_ref) { if (var_ref->param_idx == 0 && var_ref->name == "self") { - throw ParseError(cur_f, loc, "modifying `self`, which is immutable by default; probably, you want to declare `mutate self`"); + fire(cur_f, range, "modifying `self`, which is immutable by default; probably, you want to declare `mutate self`"); } else { - throw ParseError(cur_f, loc, "modifying immutable variable `" + var_ref->name + "`"); + fire(cur_f, range, "modifying immutable variable `" + var_ref->name + "`"); } } GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_modifying_readonly_field(FunctionPtr cur_f, SrcLocation loc, StructPtr struct_ref, StructFieldPtr field_ref) { - throw ParseError(cur_f, loc, "modifying readonly field `" + struct_ref->as_human_readable() + "." + field_ref->name + "`"); +static void fire_modifying_readonly_field(FunctionPtr cur_f, SrcRange range, StructPtr struct_ref, StructFieldPtr field_ref) { + fire(cur_f, range, "modifying readonly field `" + struct_ref->as_human_readable() + "." + field_ref->name + "`"); } // validate a function used as rvalue, like `var cb = f` // it's not a generic function (ensured earlier at type inferring) and has some more restrictions static void validate_function_used_as_noncall(FunctionPtr cur_f, AnyExprV v, FunctionPtr fun_ref) { if (!fun_ref->arg_order.empty() || !fun_ref->ret_order.empty()) { - fire(cur_f, v->loc, "saving `" + fun_ref->name + "` into a variable will most likely lead to invalid usage, since it changes the order of variables on the stack"); + fire(cur_f, v, "saving `" + fun_ref->name + "` into a variable will most likely lead to invalid usage, since it changes the order of variables on the stack"); } if (fun_ref->has_mutate_params()) { - fire(cur_f, v->loc, "saving `" + fun_ref->name + "` into a variable is impossible, since it has `mutate` parameters and thus can only be called directly"); + fire(cur_f, v, "saving `" + fun_ref->name + "` into a variable is impossible, since it has `mutate` parameters and thus can only be called directly"); } } class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { FunctionPtr cur_f = nullptr; - void on_var_used_as_lvalue(SrcLocation loc, LocalVarPtr var_ref) const { + void on_var_used_as_lvalue(SrcRange range, LocalVarPtr var_ref) const { if (var_ref->is_immutable()) { - fire_error_modifying_immutable_variable(cur_f, loc, var_ref); + fire_modifying_immutable_variable(cur_f, range, var_ref); } var_ref->mutate()->assign_used_as_lval(); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(cur_f, v, "braced expression"); + fire_cannot_be_used_as_lvalue(cur_f, v, "braced expression"); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(cur_f, v, "assignment"); + fire_cannot_be_used_as_lvalue(cur_f, v, "assignment"); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(cur_f, v, "assignment"); + fire_cannot_be_used_as_lvalue(cur_f, v, "assignment"); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(cur_f, v, "operator " + static_cast(v->operator_name)); + fire_cannot_be_used_as_lvalue(cur_f, v, "operator " + static_cast(v->operator_name)); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(cur_f, v, "operator " + static_cast(v->operator_name)); + fire_cannot_be_used_as_lvalue(cur_f, v, "operator " + static_cast(v->operator_name)); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(cur_f, v, "operator ?:"); + fire_cannot_be_used_as_lvalue(cur_f, v, "operator ?:"); } parent::visit(v); } @@ -121,7 +121,7 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(cur_f, v, v->is_negated ? "operator !is" : "operator is"); + fire_cannot_be_used_as_lvalue(cur_f, v, v->is_negated ? "operator !is" : "operator is"); } parent::visit(v->get_expr()); } @@ -133,32 +133,32 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(cur_f, v, "lazy expression"); + fire_cannot_be_used_as_lvalue(cur_f, v, "lazy expression"); } parent::visit(v->get_expr()); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(cur_f, v, "literal"); + fire_cannot_be_used_as_lvalue(cur_f, v, "literal"); } } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(cur_f, v, "literal"); + fire_cannot_be_used_as_lvalue(cur_f, v, "literal"); } } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(cur_f, v, "literal"); + fire_cannot_be_used_as_lvalue(cur_f, v, "literal"); } } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(cur_f, v, "literal"); + fire_cannot_be_used_as_lvalue(cur_f, v, "literal"); } } @@ -175,7 +175,7 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { const TypeDataStruct* obj_type = as_dot->get_obj()->inferred_type->unwrap_alias()->try_as(); tolk_assert(obj_type); if (field_ref->is_readonly) { - fire_error_modifying_readonly_field(cur_f, as_dot->loc, obj_type->struct_ref, field_ref); + fire_modifying_readonly_field(cur_f, as_dot->range, obj_type->struct_ref, field_ref); } } @@ -193,13 +193,13 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { if (auto as_ref = leftmost_obj->try_as()) { if (LocalVarPtr var_ref = as_ref->sym->try_as()) { - on_var_used_as_lvalue(leftmost_obj->loc, var_ref); + on_var_used_as_lvalue(leftmost_obj->range, var_ref); } if (as_ref->sym->try_as()) { // `Point.create = f` if (v->is_target_enum_member()) { - fire(cur_f, v->loc, "modifying immutable constant"); + fire(cur_f, v, "modifying immutable constant"); } - fire(cur_f, v->loc, "invalid left side of assignment"); + fire(cur_f, v, "invalid left side of assignment"); } } } @@ -214,7 +214,7 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(cur_f, v, "function call"); + fire_cannot_be_used_as_lvalue(cur_f, v, "function call"); } if (!v->fun_maybe) { parent::visit(v->get_callee()); @@ -232,7 +232,7 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(cur_f, v, "`match` expression"); + fire_cannot_be_used_as_lvalue(cur_f, v, "`match` expression"); } parent::visit(v); } @@ -241,7 +241,7 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { if (v->marked_as_redef) { tolk_assert(v->var_ref); if (v->var_ref->is_immutable()) { - fire(cur_f, v->loc, "`redef` for immutable variable"); + fire(cur_f, v, "`redef` for immutable variable"); } } } @@ -250,11 +250,11 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { if (v->is_lvalue) { tolk_assert(v->sym); if (LocalVarPtr var_ref = v->sym->try_as()) { - on_var_used_as_lvalue(v->loc, var_ref); + on_var_used_as_lvalue(v->range, var_ref); } else if (v->sym->try_as()) { - fire(cur_f, v->loc, "modifying immutable constant"); + fire(cur_f, v, "modifying immutable constant"); } else if (v->sym->try_as()) { - fire(cur_f, v->loc, "function can't be used as lvalue"); + fire(cur_f, v, "function can't be used as lvalue"); } } @@ -266,7 +266,7 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { if (v->is_rvalue) { - fire(cur_f, v->loc, "`_` can't be used as a value; it's a placeholder for a left side of assignment"); + fire(cur_f, v, "`_` can't be used as a value; it's a placeholder for a left side of assignment"); } } diff --git a/tolk/pipe-check-serialized-fields.cpp b/tolk/pipe-check-serialized-fields.cpp index 792a73d293..6df6509d66 100644 --- a/tolk/pipe-check-serialized-fields.cpp +++ b/tolk/pipe-check-serialized-fields.cpp @@ -14,9 +14,9 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . */ -#include "tolk.h" #include "ast.h" #include "ast-visitor.h" +#include "compilation-errors.h" #include "pack-unpack-api.h" #include "maps-kv-api.h" #include "generics-helpers.h" @@ -26,8 +26,8 @@ namespace tolk { // fire an error on overflow 1023 bits GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_theoretical_overflow_1023(StructPtr struct_ref, PackSize size) { - throw ParseError(struct_ref->ast_root->loc, +static void fire_theoretical_overflow_1023(StructPtr struct_ref, PackSize size) { + fire(struct_ref->ident_anchor, "struct `" + struct_ref->as_human_readable() + "` can exceed 1023 bits in serialization (estimated size: " + std::to_string(size.min_bits) + ".." + std::to_string(size.max_bits) + " bits)\n\n" "1) either suppress it by adding an annotation:\n" "> @overflow1023_policy(\"suppress\")\n" @@ -44,24 +44,24 @@ static void fire_error_theoretical_overflow_1023(StructPtr struct_ref, PackSize } GNU_ATTRIBUTE_NOINLINE -static void check_map_TKey_TValue(SrcLocation loc, TypePtr TKey, TypePtr TValue) { +static void check_map_TKey_TValue(SrcRange range, TypePtr TKey, TypePtr TValue) { std::string because_msg; if (!check_mapKV_TKey_is_valid(TKey, because_msg)) { - fire(nullptr, loc, "invalid `map`: type `" + TKey->as_human_readable() + "` can not be used as a key\n" + because_msg); + fire(range, "invalid `map`: type `" + TKey->as_human_readable() + "` can not be used as a key\n" + because_msg); } if (!check_mapKV_TValue_is_valid(TValue, because_msg)) { - fire(nullptr, loc, "invalid `map`: type `" + TValue->as_human_readable() + "` can not be used as a value\n" + because_msg); + fire(range, "invalid `map`: type `" + TValue->as_human_readable() + "` can not be used as a value\n" + because_msg); } } GNU_ATTRIBUTE_NOINLINE -static void check_mapKV_inside_type(SrcLocation loc, TypePtr any_type) { - any_type->replace_children_custom([loc](TypePtr child) { +static void check_mapKV_inside_type(SrcRange range, TypePtr any_type) { + any_type->replace_children_custom([range](TypePtr child) { if (const TypeDataMapKV* t_map = child->try_as()) { - check_map_TKey_TValue(loc, t_map->TKey, t_map->TValue); + check_map_TKey_TValue(range, t_map->TKey, t_map->TValue); } if (const TypeDataAlias* t_alias = child->try_as()) { - check_mapKV_inside_type(loc, t_alias->underlying_type); + check_mapKV_inside_type(range, t_alias->underlying_type); } return child; }); @@ -69,14 +69,15 @@ static void check_mapKV_inside_type(SrcLocation loc, TypePtr any_type) { static void check_mapKV_inside_type(AnyTypeV type_node) { if (type_node && type_node->resolved_type->has_mapKV_inside()) { - check_mapKV_inside_type(type_node->loc, type_node->resolved_type); + check_mapKV_inside_type(type_node->range, type_node->resolved_type); } } // given `enum Role: int8` check colon type (not struct/slice etc.) static void check_enum_colon_type_to_be_intN(AnyTypeV colon_type_node) { if (!colon_type_node->resolved_type->try_as()) { - fire(nullptr, colon_type_node->loc, "serialization type of `enum` must be intN: `int8` / `uint32` / etc."); + // todo write test for underline + fire(colon_type_node, "serialization type of `enum` must be intN: `int8` / `uint32` / etc."); } } @@ -104,7 +105,7 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody PackSize size = estimate_serialization_size(t_struct); if (size.max_bits > 1023 && !size.is_unpredictable_infinity()) { if (struct_ref->overflow1023_policy == StructData::Overflow1023Policy::not_specified) { - fire_error_theoretical_overflow_1023(struct_ref, size); + fire_theoretical_overflow_1023(struct_ref, size); } } // don't check Cell fields for overflow of T: it would be checked on load() or other interaction with T @@ -114,13 +115,13 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody parent::visit(v); FunctionPtr fun_ref = v->fun_maybe; - if (!fun_ref || !fun_ref->is_builtin_function() || !fun_ref->is_instantiation_of_generic_function()) { + if (!fun_ref || !fun_ref->is_builtin() || !fun_ref->is_instantiation_of_generic_function()) { return; } std::string_view f_name = fun_ref->base_fun_ref->name; if (f_name == "createEmptyMap") { - check_map_TKey_TValue(v->loc, fun_ref->substitutedTs->typeT_at(0), fun_ref->substitutedTs->typeT_at(1)); + check_map_TKey_TValue(v->range, fun_ref->substitutedTs->typeT_at(0), fun_ref->substitutedTs->typeT_at(1)); return; } @@ -138,7 +139,7 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody std::string because_msg; if (!check_struct_can_be_packed_or_unpacked(serialized_type, is_pack, because_msg)) { std::string via_name = fun_ref->is_method() ? fun_ref->method_name : fun_ref->base_fun_ref->name; - fire(cur_f, v->loc, "auto-serialization via " + via_name + "() is not available for type `" + serialized_type->as_human_readable() + "`\n" + because_msg); + fire(cur_f, v, "auto-serialization via " + via_name + "() is not available for type `" + serialized_type->as_human_readable() + "`\n" + because_msg); } check_type_fits_cell_or_has_policy(serialized_type); diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index 27cecd3c60..8b2b6fc1c6 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -14,9 +14,9 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . */ -#include "tolk.h" #include "ast.h" #include "ast-replacer.h" +#include "compilation-errors.h" #include "type-system.h" #include "constant-evaluator.h" @@ -41,7 +41,7 @@ static void assign_enum_members_values(EnumDefPtr enum_ref) { for (EnumMemberPtr member_ref : enum_ref->members) { td::RefInt256 cur_value = member_ref->init_value ? eval_enum_member_init_value(member_ref->init_value) : prev_value + 1; if (!cur_value->is_valid() || !cur_value->signed_fits_bits(257)) { - fire(nullptr, member_ref->loc, "integer overflow"); + fire(member_ref->ident_anchor, "integer overflow"); } member_ref->mutate()->assign_computed_value(cur_value); @@ -51,15 +51,15 @@ static void assign_enum_members_values(EnumDefPtr enum_ref) { class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { - static V create_int_const(SrcLocation loc, td::RefInt256&& intval) { - auto v_int = createV(loc, std::move(intval), {}); + static V create_int_const(SrcRange range, td::RefInt256&& intval) { + auto v_int = createV(range, std::move(intval), {}); v_int->assign_inferred_type(TypeDataInt::create()); v_int->assign_rvalue_true(); return v_int; } - static V create_bool_const(SrcLocation loc, bool bool_val) { - auto v_bool = createV(loc, bool_val); + static V create_bool_const(SrcRange range, bool bool_val) { + auto v_bool = createV(range, bool_val); v_bool->assign_inferred_type(TypeDataBool::create()); v_bool->assign_rvalue_true(); return v_bool; @@ -73,8 +73,8 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { return inner; } - static V create_string_const(SrcLocation loc, std::string&& literal_value, TypePtr inferred_type) { - auto v_string = createV(loc, literal_value); + static V create_string_const(SrcRange range, std::string&& literal_value, TypePtr inferred_type) { + auto v_string = createV(range, literal_value); v_string->assign_inferred_type(inferred_type); v_string->assign_literal_value(std::move(literal_value)); v_string->assign_rvalue_true(); @@ -91,9 +91,9 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { tolk_assert(!intval.is_null()); intval = -intval; if (intval.is_null() || !intval->signed_fits_bits(257)) { - v->error("integer overflow"); + fire(v, "integer overflow"); } - return create_int_const(v->loc, std::move(intval)); + return create_int_const(v->range, std::move(intval)); } // same for "+1" if (t == tok_plus && v->get_rhs()->kind == ast_int_const) { @@ -102,11 +102,11 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { // `!true` / `!false` if (t == tok_logical_not && v->get_rhs()->kind == ast_bool_const) { - return create_bool_const(v->loc, !v->get_rhs()->as()->bool_val); + return create_bool_const(v->range, !v->get_rhs()->as()->bool_val); } // `!0` if (t == tok_logical_not && v->get_rhs()->kind == ast_int_const) { - return create_bool_const(v->loc, v->get_rhs()->as()->intval == 0); + return create_bool_const(v->range, v->get_rhs()->as()->intval == 0); } return v; @@ -117,7 +117,7 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { // `null == null` / `null != null` if (v->get_expr()->kind == ast_null_keyword && v->type_node->resolved_type == TypeDataNullLiteral::create()) { - return create_bool_const(v->loc, !v->is_negated); + return create_bool_const(v->range, !v->is_negated); } return v; @@ -130,9 +130,9 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { if (v->fun_maybe && v->fun_maybe->is_compile_time_const_val()) { CompileTimeFunctionResult value = eval_call_to_compile_time_function(v); if (std::holds_alternative(value)) { - return create_int_const(v->loc, std::move(std::get(value))); + return create_int_const(v->range, std::move(std::get(value))); } else { - return create_string_const(v->loc, std::move(std::get(value)), v->fun_maybe->declared_return_type); + return create_string_const(v->range, std::move(std::get(value)), v->fun_maybe->declared_return_type); } } diff --git a/tolk/pipe-detect-inline-in-place.cpp b/tolk/pipe-detect-inline-in-place.cpp index 7e892b687f..711bab3d5b 100644 --- a/tolk/pipe-detect-inline-in-place.cpp +++ b/tolk/pipe-detect-inline-in-place.cpp @@ -14,9 +14,10 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . */ -#include "tolk.h" #include "ast.h" #include "ast-visitor.h" +#include "compilation-errors.h" +#include #include /* @@ -117,7 +118,7 @@ class DetectIfToInlineFunctionInPlaceVisitor final : ASTVisitorFunctionBody { protected: void visit(V v) override { - if (v->fun_maybe && v->fun_maybe->is_builtin_function() && v->fun_maybe->name == "__expect_inline") { + if (v->fun_maybe && v->fun_maybe->is_builtin() && v->fun_maybe->name == "__expect_inline") { collected_expect_inline.push_back(v); } else { cur_state.n_function_calls++; @@ -200,7 +201,7 @@ class DetectIfToInlineFunctionInPlaceVisitor final : ASTVisitorFunctionBody { public: bool should_visit_function(FunctionPtr fun_ref) override { // unsupported or no-sense cases - if (fun_ref->is_builtin_function() || fun_ref->is_asm_function() || fun_ref->is_generic_function() || + if (fun_ref->is_builtin() || fun_ref->is_asm_function() || fun_ref->is_generic_function() || fun_ref->has_tvm_method_id() || !fun_ref->arg_order.empty() || !fun_ref->ret_order.empty() || fun_ref->is_used_as_noncall()) { return false; @@ -235,7 +236,7 @@ class DetectIfToInlineFunctionInPlaceVisitor final : ASTVisitorFunctionBody { tolk_assert(v_expect->get_num_args() == 1 && v_expect->get_arg(0)->get_expr()->kind == ast_bool_const); bool expected = v_expect->get_arg(0)->get_expr()->as()->bool_val; if (expected != will_inline) { - fire(fun_ref, v_expect->loc, "__expect_inline failed"); + fire(fun_ref, v_expect, "__expect_inline failed"); } } diff --git a/tolk/pipe-discover-parse-sources.cpp b/tolk/pipe-discover-parse-sources.cpp index da4a7850f4..2c25371acb 100644 --- a/tolk/pipe-discover-parse-sources.cpp +++ b/tolk/pipe-discover-parse-sources.cpp @@ -23,8 +23,8 @@ exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. */ -#include "tolk.h" #include "ast.h" +#include "compilation-errors.h" #include "compiler-state.h" /* @@ -40,8 +40,8 @@ namespace tolk { AnyV parse_src_file_to_ast(const SrcFile* file); void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, const std::string& entrypoint_filename) { - G.all_src_files.locate_and_register_source_file(stdlib_filename, {}); - G.all_src_files.locate_and_register_source_file(entrypoint_filename, {}); + G.all_src_files.locate_and_register_source_file(stdlib_filename, nullptr); + G.all_src_files.locate_and_register_source_file(entrypoint_filename, nullptr); while (SrcFile* file = G.all_src_files.get_next_unparsed_file()) { tolk_assert(!file->ast); @@ -56,7 +56,7 @@ void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, con ? std::move(imported_str) : file->extract_dirname() + imported_str; - const SrcFile* imported = G.all_src_files.locate_and_register_source_file(rel_filename, v_import->loc); + const SrcFile* imported = G.all_src_files.locate_and_register_source_file(rel_filename, v_import->get_file_leaf()); file->imports.push_back(SrcFile::ImportDirective{imported}); v_import->mutate()->assign_src_file(imported); } diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp index 8f55737d9d..51c5bc28b8 100644 --- a/tolk/pipe-generate-fif-output.cpp +++ b/tolk/pipe-generate-fif-output.cpp @@ -85,7 +85,7 @@ static void generate_output_func(FunctionPtr fun_ref) { modifier = "REF"; } if (G.settings.tolk_src_as_line_comments) { - std::cout << " // " << fun_ref->loc << std::endl; + std::cout << " // " << fun_ref->ident_anchor->range.stringify_start_location(false) << std::endl; } std::cout << " " << fun_ref->name << "() PROC" << modifier << ":<{"; int mode = 0; diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 152e90666b..eef62b1d3b 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -14,10 +14,10 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "tolk.h" -#include "src-file.h" #include "ast.h" #include "ast-visitor.h" +#include "compilation-errors.h" +#include "src-file.h" #include "generics-helpers.h" #include "overload-resolution.h" #include "type-system.h" @@ -141,39 +141,39 @@ static std::string to_string(std::string_view string_view) { // asm functions generally can't handle it, they expect T to be a TVM primitive // (in FunC, `forall` type just couldn't be unified with non-primitives; in Tolk, generic T is expectedly inferred) GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_calling_asm_function_with_non1_stack_width_arg(FunctionPtr cur_f, SrcLocation loc, FunctionPtr fun_ref, const GenericsSubstitutions& substitutions, int arg_idx) { - fire(cur_f, loc, "can not call `" + fun_ref->as_human_readable() + "` with " + to_string(substitutions.nameT_at(arg_idx)) + "=" + substitutions.typeT_at(arg_idx)->as_human_readable() + ", because it occupies " + std::to_string(substitutions.typeT_at(arg_idx)->get_width_on_stack()) + " stack slots in TVM, not 1"); +static void fire_calling_asm_function_with_non1_stack_width_arg(FunctionPtr cur_f, SrcRange range, FunctionPtr fun_ref, const GenericsSubstitutions& substitutions, int arg_idx) { + fire(cur_f, range, "can not call `" + fun_ref->as_human_readable() + "` with " + to_string(substitutions.nameT_at(arg_idx)) + "=" + substitutions.typeT_at(arg_idx)->as_human_readable() + ", because it occupies " + std::to_string(substitutions.typeT_at(arg_idx)->get_width_on_stack()) + " stack slots in TVM, not 1"); } // fire an error on `untypedTupleVar.0` when used without a hint GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_cannot_deduce_untyped_tuple_access(FunctionPtr cur_f, SrcLocation loc, int index) { +static void fire_cannot_deduce_untyped_tuple_access(FunctionPtr cur_f, SrcRange range, int index) { std::string idx_access = "." + std::to_string(index); - fire(cur_f, loc, "can not deduce type of `" + idx_access + "`\neither assign it to variable like `var c: int = " + idx_access + "` or cast the result like `" + idx_access + " as int`"); + fire(cur_f, range, "can not deduce type of `" + idx_access + "`\neither assign it to variable like `var c: int = " + idx_access + "` or cast the result like `" + idx_access + " as int`"); } // fire an error on using lateinit variable before definite assignment GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_using_lateinit_variable_uninitialized(FunctionPtr cur_f, SrcLocation loc, std::string_view name) { - fire(cur_f, loc, "using variable `" + static_cast(name) + "` before it's definitely assigned"); +static void fire_using_lateinit_variable_uninitialized(FunctionPtr cur_f, SrcRange range, std::string_view name) { + fire(cur_f, range, "using variable `" + static_cast(name) + "` before it's definitely assigned"); } // fire an error when `obj.f()`, method `f` not found, try to locate a method for another type GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_method_or_field_not_found(FunctionPtr cur_f, SrcLocation loc, TypePtr receiver_type, std::string_view field_name, bool called_as_method, bool is_static_dot) { +static void fire_method_or_field_not_found(FunctionPtr cur_f, SrcRange range, TypePtr receiver_type, std::string_view field_name, bool called_as_method, bool is_static_dot) { if (!called_as_method && is_static_dot && receiver_type->unwrap_alias()->try_as()) { - fire(cur_f, loc, "member `" + to_string(field_name) + "` does not exist in enum " + to_string(receiver_type)); + fire(cur_f, range, "member `" + to_string(field_name) + "` does not exist in enum " + to_string(receiver_type)); } if (!called_as_method && !is_static_dot) { - fire(cur_f, loc, "field `" + to_string(field_name) + "` doesn't exist in type " + to_string(receiver_type)); + fire(cur_f, range, "field `" + to_string(field_name) + "` doesn't exist in type " + to_string(receiver_type)); } if (std::vector other = lookup_methods_with_name(field_name); !other.empty()) { - fire(cur_f, loc, "method `" + to_string(field_name) + "` not found for type " + to_string(receiver_type) + "\n(but it exists for type " + to_string(other.front()->receiver_type) + ")"); + fire(cur_f, range, "method `" + to_string(field_name) + "` not found for type " + to_string(receiver_type) + "\n(but it exists for type " + to_string(other.front()->receiver_type) + ")"); } if (const Symbol* sym = lookup_global_symbol(field_name); sym && sym->try_as()) { - fire(cur_f, loc, "method `" + to_string(field_name) + "` not found, but there is a global function named `" + to_string(field_name) + "`\n(a function should be called `foo(arg)`, not `arg.foo()`)"); + fire(cur_f, range, "method `" + to_string(field_name) + "` not found, but there is a global function named `" + to_string(field_name) + "`\n(a function should be called `foo(arg)`, not `arg.foo()`)"); } - fire(cur_f, loc, "method `" + to_string(field_name) + "` not found"); + fire(cur_f, range, "method `" + to_string(field_name) + "` not found"); } // safe version of std::stoi that does not crash on long numbers @@ -230,7 +230,7 @@ static TypePtr try_pick_instantiated_generic_from_hint(TypePtr hint, AliasDefPtr // given `p.create` (called_receiver = Point, called_name = "create") // look up a corresponding method (it may be `Point.create` / `Point?.create` / `T.create`) -static std::pair choose_only_method_to_call(FunctionPtr cur_f, SrcLocation loc, TypePtr called_receiver, std::string_view called_name) { +static std::pair choose_only_method_to_call(FunctionPtr cur_f, SrcRange range, TypePtr called_receiver, std::string_view called_name) { std::vector candidates = resolve_methods_for_call(called_receiver, called_name); if (candidates.size() == 1) { return {candidates[0].method_ref, candidates[0].substitutedTs}; @@ -247,18 +247,18 @@ static std::pair choose_only_method_to_call( if (method_ref->is_generic_function()) { msg << " with " << candidate.substitutedTs.as_human_readable(false); } - if (method_ref->loc.is_defined()) { - msg << " (declared at " << method_ref->loc << ")\n"; - } else if (method_ref->is_builtin_function()) { + if (method_ref->ident_anchor) { + msg << " (declared at " << method_ref->ident_anchor->range.stringify_start_location(false) << ")\n"; + } else if (method_ref->is_builtin()) { msg << " (builtin)\n"; } } - fire(cur_f, loc, msg.str()); + fire(cur_f, range, msg.str()); } static void check_no_unexpected_type_arguments(FunctionPtr cur_f, V v_instantiationTs) { if (v_instantiationTs != nullptr) { - fire(cur_f, v_instantiationTs->loc, "type arguments not expected here"); + fire(cur_f, v_instantiationTs, "type arguments not expected here"); } } @@ -274,13 +274,13 @@ void check_arguments_count_at_fun_call(FunctionPtr cur_f, V v } if (!called_f->does_accept_self() && self_obj) { // static method `Point.create(...)` called as `p.create()` - fire(cur_f, v->loc, "method " + to_string(called_f) + " can not be called via dot\n(it's a static method, it does not accept `self`)"); + fire(cur_f, v->get_callee(), "method " + to_string(called_f) + " can not be called via dot\n(it's a static method, it does not accept `self`)"); } if (n_max_params < n_arguments) { - fire(cur_f, v->loc, "too many arguments in call to " + to_string(called_f) + ", expected " + std::to_string(n_max_params - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); + fire(cur_f, v->get_arg_list(), "too many arguments in call to " + to_string(called_f) + ", expected " + std::to_string(n_max_params - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); } if (n_arguments < n_min_params) { - fire(cur_f, v->loc, "too few arguments in call to " + to_string(called_f) + ", expected " + std::to_string(n_min_params - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); + fire(cur_f, v->get_arg_list(), "too few arguments in call to " + to_string(called_f) + ", expected " + std::to_string(n_min_params - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); } } @@ -748,7 +748,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // example: `var v = ternary`, show an inference error // do NOT show an error for `var v: T = ternary` (T is hint); it will be checked by type checker later if (hint == nullptr || hint == TypeDataUnknown::create() || hint->has_genericT_inside()) { - fire(cur_f, v->loc, "types of ternary branches are incompatible: " + to_string(v->get_when_true()) + " and " + to_string(v->get_when_false()) + "\nhint: maybe, you should use ` as ` to make them identical"); + fire(cur_f, v, "types of ternary branches are incompatible: " + to_string(v->get_when_true()) + " and " + to_string(v->get_when_false()) + "\nhint: maybe, you should use ` as ` to make them identical"); } } assign_inferred_type(v, branches_unifier.get_result()); @@ -780,7 +780,7 @@ class InferTypesAndCallsAndFieldsVisitor final { rhs_type = inst_rhs_type; v->type_node->mutate()->assign_resolved_type(rhs_type); } else { - fire(cur_f, v->type_node->loc, "can not deduce type arguments for " + to_string(t_struct->struct_ref) + ", provide them manually"); + fire(cur_f, v->type_node, "can not deduce type arguments for " + to_string(t_struct->struct_ref) + ", provide them manually"); } } if (const auto* t_alias = rhs_type->try_as(); t_alias && t_alias->alias_ref->is_generic_alias()) { @@ -789,7 +789,7 @@ class InferTypesAndCallsAndFieldsVisitor final { rhs_type = inst_rhs_type; v->type_node->mutate()->assign_resolved_type(rhs_type); } else { - fire(cur_f, v->type_node->loc, "can not deduce type arguments for " + to_string(t_alias->alias_ref) + ", provide them manually"); + fire(cur_f, v->type_node, "can not deduce type arguments for " + to_string(t_alias->alias_ref) + ", provide them manually"); } } @@ -874,11 +874,11 @@ class InferTypesAndCallsAndFieldsVisitor final { // given `genericF` / `t.tupleFirst` (the user manually specified instantiation Ts), // validate and collect them // returns: [int, slice] / [cell] - std::vector collect_type_arguments_for_fun(SrcLocation loc, const GenericsDeclaration* genericTs, V instantiationT_list) const { + std::vector collect_type_arguments_for_fun(const GenericsDeclaration* genericTs, V instantiationT_list) const { // for `genericF` user should provide two Ts // for `Container.wrap` — one U (and one T is implicitly from receiver) if (instantiationT_list->size() != genericTs->size() - genericTs->n_from_receiver) { - fire(cur_f, loc, "expected " + std::to_string(genericTs->size() - genericTs->n_from_receiver) + " type arguments, got " + std::to_string(instantiationT_list->size())); + fire(cur_f, instantiationT_list, "expected " + std::to_string(genericTs->size() - genericTs->n_from_receiver) + " type arguments, got " + std::to_string(instantiationT_list->size())); } std::vector type_arguments; @@ -895,12 +895,12 @@ class InferTypesAndCallsAndFieldsVisitor final { // example: was `t.push(2)`, collected , instantiate `tuple.push` (will later fail type check) // example: was `var cb = t.first;` (used as reference, as non-call), instantiate `tuple.first` // returns fun_ref to instantiated function - FunctionPtr check_and_instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs) const { + FunctionPtr check_and_instantiate_generic_function(SrcRange range, FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs) const { // T for asm function must be a TVM primitive (width 1), otherwise, asm would act incorrectly - if (fun_ref->is_asm_function() || fun_ref->is_builtin_function()) { + if (fun_ref->is_asm_function() || fun_ref->is_builtin()) { for (int i = 0; i < substitutedTs.size(); ++i) { if (substitutedTs.typeT_at(i)->get_width_on_stack() != 1 && !is_allowed_asm_generic_function_with_non1_width_T(fun_ref, i)) { - fire_error_calling_asm_function_with_non1_stack_width_arg(cur_f, loc, fun_ref, substitutedTs, i); + fire_calling_asm_function_with_non1_stack_width_arg(cur_f, range, fun_ref, substitutedTs, i); } } } @@ -919,7 +919,7 @@ class InferTypesAndCallsAndFieldsVisitor final { TypePtr declared_or_smart_casted = flow.smart_cast_or_original(SinkExpression(var_ref), var_ref->declared_type); assign_inferred_type(v, declared_or_smart_casted); if (var_ref->is_lateinit() && declared_or_smart_casted == TypeDataUnknown::create() && v->is_rvalue) { - fire_error_using_lateinit_variable_uninitialized(cur_f, v->loc, v->get_name()); + fire_using_lateinit_variable_uninitialized(cur_f, v->range, v->get_name()); } // it might be `local_var()` also, don't fill out_f_called, we have no fun_ref, it's a call of arbitrary expression @@ -940,14 +940,14 @@ class InferTypesAndCallsAndFieldsVisitor final { if (fun_ref->is_generic_function() && !v_instantiationTs && !out_f_called) { // `genericFn` is invalid as non-call, can't be used without - fire(cur_f, v->loc, "can not use a generic function " + to_string(fun_ref) + " as non-call"); + fire(cur_f, v, "can not use a generic function " + to_string(fun_ref) + " as non-call"); } else if (fun_ref->is_generic_function() && v_instantiationTs) { // `genericFn` is valid as non-call, it's a reference to instantiation // `genericFn()` is also ok, we'll assign an instantiated fun_ref to out_f_called GenericsSubstitutions substitutedTs(fun_ref->genericTs); - substitutedTs.provide_type_arguments(collect_type_arguments_for_fun(v->loc, fun_ref->genericTs, v_instantiationTs)); - fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); + substitutedTs.provide_type_arguments(collect_type_arguments_for_fun(fun_ref->genericTs, v_instantiationTs)); + fun_ref = check_and_instantiate_generic_function(v->range, fun_ref, std::move(substitutedTs)); v->mutate()->assign_sym(fun_ref); } else { @@ -959,10 +959,10 @@ class InferTypesAndCallsAndFieldsVisitor final { *out_f_called = fun_ref; // (it's still may be a generic one, then Ts will be deduced from arguments) } else { // so, it's `globalF` / `genericFn` as a reference if (fun_ref->is_compile_time_const_val() || fun_ref->is_compile_time_special_gen()) { - fire(cur_f, v->loc, "can not get reference to this function, it's compile-time only"); + fire(cur_f, v, "can not get reference to this function, it's compile-time only"); } if (fun_ref->is_entrypoint()) { - fire(cur_f, v->loc, "can not get reference to this function, it's a special entrypoint"); + fire(cur_f, v, "can not get reference to this function, it's a special entrypoint"); } fun_ref->mutate()->assign_is_used_as_noncall(); get_or_infer_return_type(fun_ref); @@ -1004,9 +1004,9 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } } - std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, v_ident->loc, receiver_type, field_name); + std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, v_ident->range, receiver_type, field_name); if (!fun_ref) { - fire_error_method_or_field_not_found(cur_f, v_ident->loc, receiver_type, field_name, out_f_called != nullptr, true); + fire_method_or_field_not_found(cur_f, v_ident->range, receiver_type, field_name, out_f_called != nullptr, true); } } } @@ -1038,11 +1038,11 @@ class InferTypesAndCallsAndFieldsVisitor final { if (!fun_ref && field_name[0] >= '0' && field_name[0] <= '9') { int index_at; if (!try_parse_string_to_int(field_name, index_at)) { - fire(cur_f, v_ident->loc, "invalid numeric index"); + fire(cur_f, v_ident, "invalid numeric index"); } if (const auto* t_tensor = obj_type->try_as()) { if (index_at >= t_tensor->size()) { - fire(cur_f, v_ident->loc, "invalid tensor index, expected 0.." + std::to_string(t_tensor->items.size() - 1)); + fire(cur_f, v_ident, "invalid tensor index, expected 0.." + std::to_string(t_tensor->items.size() - 1)); } v->mutate()->assign_target(index_at); TypePtr inferred_type = t_tensor->items[index_at]; @@ -1054,7 +1054,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } if (const auto* t_tuple = obj_type->try_as()) { if (index_at >= t_tuple->size()) { - fire(cur_f, v_ident->loc, "invalid tuple index, expected 0.." + std::to_string(t_tuple->items.size() - 1)); + fire(cur_f, v_ident, "invalid tuple index, expected 0.." + std::to_string(t_tuple->items.size() - 1)); } v->mutate()->assign_target(index_at); TypePtr inferred_type = t_tuple->items[index_at]; @@ -1070,7 +1070,7 @@ class InferTypesAndCallsAndFieldsVisitor final { item_type = TypeDataUnknown::create(); } else { if (hint == nullptr || hint == TypeDataUnknown::create() || hint->has_genericT_inside()) { - fire_error_cannot_deduce_untyped_tuple_access(cur_f, v->loc, index_at); + fire_cannot_deduce_untyped_tuple_access(cur_f, v_ident->range, index_at); } item_type = hint; } @@ -1083,7 +1083,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // check for method (`t.size` / `user.getId`); even `i.0()` can be here if `fun int.0(self)` exists // for `T.copy` / `Container.create`, substitution for T is also returned if (!fun_ref) { - std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, dot_obj->loc, dot_obj->inferred_type, field_name); + std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, dot_obj->range, dot_obj->inferred_type, field_name); } // not a field, not a method — fire an error @@ -1091,27 +1091,27 @@ class InferTypesAndCallsAndFieldsVisitor final { // as a special case, handle accessing fields of nullable objects, to show a more precise error if (const TypeDataUnion* as_union = obj_type->try_as(); as_union && as_union->or_null) { if (const TypeDataStruct* n_struct = as_union->or_null->try_as(); n_struct && n_struct->struct_ref->find_field(field_name)) { - fire(cur_f, v_ident->loc, "can not access field `" + static_cast(field_name) + "` of a possibly nullable object " + to_string(dot_obj) + "\nhint: check it via `obj != null` or use non-null assertion `obj!` operator"); + fire(cur_f, v_ident, "can not access field `" + static_cast(field_name) + "` of a possibly nullable object " + to_string(dot_obj) + "\nhint: check it via `obj != null` or use non-null assertion `obj!` operator"); } } - fire_error_method_or_field_not_found(cur_f, v_ident->loc, dot_obj->inferred_type, field_name, out_f_called != nullptr, false); + fire_method_or_field_not_found(cur_f, v_ident->range, dot_obj->inferred_type, field_name, out_f_called != nullptr, false); } // if `fun T.copy(self)` and reference `int.copy` — all generic parameters are determined by the receiver, we know it if (fun_ref->is_generic_function() && fun_ref->genericTs->size() == fun_ref->genericTs->n_from_receiver) { tolk_assert(substitutedTs.typeT_at(0) != nullptr); - fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); + fun_ref = check_and_instantiate_generic_function(v->range, fun_ref, std::move(substitutedTs)); } if (fun_ref->is_generic_function() && !v_instantiationTs && !out_f_called) { // `t.tupleAt` is invalid as non-call, can't be used without - fire(cur_f, v->loc, "can not use a generic function " + to_string(fun_ref) + " as non-call"); + fire(cur_f, v_ident, "can not use a generic function " + to_string(fun_ref) + " as non-call"); } else if (fun_ref->is_generic_function() && v_instantiationTs) { // `t.tupleAt` is valid as non-call, it's a reference to instantiation // `t.tupleAt()` is also ok, we'll assign an instantiated fun_ref to out_f_called - substitutedTs.provide_type_arguments(collect_type_arguments_for_fun(v->loc, fun_ref->genericTs, v_instantiationTs)); - fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); + substitutedTs.provide_type_arguments(collect_type_arguments_for_fun(fun_ref->genericTs, v_instantiationTs)); + fun_ref = check_and_instantiate_generic_function(v->range, fun_ref, std::move(substitutedTs)); } else { // non-generic method referenced like `var cb = c.hash;` @@ -1123,7 +1123,7 @@ class InferTypesAndCallsAndFieldsVisitor final { *out_dot_obj = dot_obj; } else { // so, it's `user.method` / `t.tupleAt` as a reference if (fun_ref->is_compile_time_const_val() || fun_ref->is_compile_time_special_gen()) { - fire(cur_f, v->get_identifier()->loc, "can not get reference to this method, it's compile-time only"); + fire(cur_f, v_ident, "can not get reference to this method, it's compile-time only"); } fun_ref->mutate()->assign_is_used_as_noncall(); v->mutate()->assign_target(fun_ref); @@ -1153,11 +1153,11 @@ class InferTypesAndCallsAndFieldsVisitor final { // callee must have "callable" inferred type const TypeDataFunCallable* f_callable = callee->inferred_type->unwrap_alias()->try_as(); if (!f_callable) { // `5()` / `SOME_CONST()` / `null()` - fire(cur_f, v->loc, "calling a non-function " + to_string(callee->inferred_type)); + fire(cur_f, v->get_callee(), "calling a non-function " + to_string(callee->inferred_type)); } // check arguments count (their types will be checked in a later pipe) if (v->get_num_args() != static_cast(f_callable->params_types.size())) { - fire(cur_f, v->loc, "expected " + std::to_string(f_callable->params_types.size()) + " arguments, got " + std::to_string(v->get_arg_list()->size())); + fire(cur_f, v->get_callee(), "expected " + std::to_string(f_callable->params_types.size()) + " arguments, got " + std::to_string(v->get_arg_list()->size())); } for (int i = 0; i < v->get_num_args(); ++i) { auto arg_i = v->get_arg(i)->get_expr(); @@ -1170,7 +1170,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // prevent calling `onBouncedMessage()` and other special functions directly if (fun_ref->is_entrypoint()) { - fire(cur_f, v->loc, fun_ref->name + " is a special entrypoint, it can not be called as a regular function"); + fire(cur_f, v->get_callee(), fun_ref->name + " is a special entrypoint, it can not be called as a regular function"); } // so, we have a call `f(args)` or `obj.f(args)`, f is fun_ref (function / method) (code / asm / builtin) @@ -1189,7 +1189,7 @@ class InferTypesAndCallsAndFieldsVisitor final { const LocalVarData& param_0 = fun_ref->parameters[0]; TypePtr param_type = param_0.declared_type; if (param_type->has_genericT_inside()) { - param_type = deducingTs.auto_deduce_from_argument(cur_f, self_obj->loc, param_type, self_obj->inferred_type); + param_type = deducingTs.auto_deduce_from_argument(cur_f, self_obj->range, param_type, self_obj->inferred_type); } if (param_0.is_mutate_parameter() && !self_obj->inferred_type->equal_to(param_type)) { if (SinkExpression s_expr = extract_sink_expression_from_vertex(self_obj)) { @@ -1202,7 +1202,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (!self_obj && fun_ref->receiver_type && fun_ref->receiver_type->has_genericT_inside() && callee->kind == ast_dot_access && callee->as()->get_obj()->kind == ast_reference) { const auto* t_static = callee->as()->get_obj()->as()->sym->try_as(); tolk_assert(t_static); - deducingTs.auto_deduce_from_argument(cur_f, v->loc, fun_ref->receiver_type, t_static->resolved_type); + deducingTs.auto_deduce_from_argument(cur_f, v->get_callee()->range, fun_ref->receiver_type, t_static->resolved_type); } // loop over every argument, one by one, like control flow goes @@ -1215,7 +1215,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } flow = infer_any_expr(arg_i, std::move(flow), false, param_type).out_flow; if (param_type->has_genericT_inside()) { // `f(a)` where f is generic: use `a` to infer param type - param_type = deducingTs.auto_deduce_from_argument(cur_f, arg_i->loc, param_type, arg_i->inferred_type); + param_type = deducingTs.auto_deduce_from_argument(cur_f, arg_i->range, param_type, arg_i->inferred_type); } assign_inferred_type(v->get_arg(i), arg_i); // arg itself is an expression if (param_i.is_mutate_parameter() && !arg_i->inferred_type->equal_to(param_type)) { @@ -1235,7 +1235,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (!nameT_unknown.empty() && hint && !hint->has_genericT_inside() && fun_ref->declared_return_type) { // example: `t.tupleFirst()`, T doesn't depend on arguments, but is determined by return type // if used like `var x: int = t.tupleFirst()` / `t.tupleFirst() as int` / etc., use hint - deducingTs.auto_deduce_from_argument(cur_f, v->loc, fun_ref->declared_return_type, hint); + deducingTs.auto_deduce_from_argument(cur_f, v->get_callee()->range, fun_ref->declared_return_type, hint); nameT_unknown = deducingTs.get_first_not_deduced_nameT(); } if (!nameT_unknown.empty()) { @@ -1243,9 +1243,9 @@ class InferTypesAndCallsAndFieldsVisitor final { nameT_unknown = deducingTs.get_first_not_deduced_nameT(); } if (!nameT_unknown.empty()) { - deducingTs.fire_error_can_not_deduce(cur_f, v->get_arg_list()->loc, nameT_unknown); + deducingTs.fire_can_not_deduce(cur_f, v->get_callee()->range, nameT_unknown); } - fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, deducingTs.flush()); + fun_ref = check_and_instantiate_generic_function(v->get_callee()->range, fun_ref, deducingTs.flush()); } // get return type either from user-specified declaration or infer here on demand traversing its body @@ -1312,7 +1312,7 @@ class InferTypesAndCallsAndFieldsVisitor final { exact_type = inst_exact_type; v_arm->pattern_type_node->mutate()->assign_resolved_type(exact_type); } else { - fire(cur_f, v_arm->loc, "can not deduce type arguments for " + to_string(t_struct->struct_ref) + ", provide them manually"); + fire(cur_f, v_arm->pattern_type_node, "can not deduce type arguments for " + to_string(t_struct->struct_ref) + ", provide them manually"); } } if (const auto* t_alias = exact_type->try_as(); t_alias && t_alias->alias_ref->is_generic_alias()) { @@ -1321,7 +1321,7 @@ class InferTypesAndCallsAndFieldsVisitor final { exact_type = inst_exact_type; v_arm->pattern_type_node->mutate()->assign_resolved_type(exact_type); } else { - fire(cur_f, v_arm->loc, "can not deduce type arguments for " + to_string(t_alias->alias_ref) + ", provide them manually"); + fire(cur_f, v_arm->pattern_type_node, "can not deduce type arguments for " + to_string(t_alias->alias_ref) + ", provide them manually"); } } arm_flow.register_known_type(s_expr, exact_type); @@ -1340,7 +1340,7 @@ class InferTypesAndCallsAndFieldsVisitor final { match_out_flow = std::move(arms_entry_facts); } if (has_expr_arm && !has_else_arm) { - FlowContext else_flow = process_any_statement(createV(v->loc), std::move(arms_entry_facts)); + FlowContext else_flow = process_any_statement(createV(SrcRange::empty_at_end(v->range)), std::move(arms_entry_facts)); match_out_flow = FlowContext::merge_flow(std::move(match_out_flow), std::move(else_flow)); } @@ -1348,12 +1348,12 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, TypeDataVoid::create()); } else { if (v->get_arms_count() == 0) { // still allow empty `match` statements, for probable codegen - fire(cur_f, v->loc, "empty `match` can't be used as expression"); + fire(cur_f, v->keyword_range(), "empty `match` can't be used as expression"); } if (branches_unifier.is_union_of_different_types()) { // same as in ternary: `match (...) { t1 => someSlice, t2 => someInt }` is `int|slice`, probably unexpected if (hint == nullptr || hint == TypeDataUnknown::create() || hint->has_genericT_inside()) { - fire(cur_f, v->loc, "type of `match` was inferred as " + to_string(branches_unifier.get_result()) + "; probably, it's not what you expected\nassign it to a variable `var v: = match (...) { ... }` manually"); + fire(cur_f, v->keyword_range(), "type of `match` was inferred as " + to_string(branches_unifier.get_result()) + "; probably, it's not what you expected\nassign it to a variable `var v: = match (...) { ... }` manually"); } } assign_inferred_type(v, branches_unifier.get_result()); @@ -1376,7 +1376,7 @@ class InferTypesAndCallsAndFieldsVisitor final { struct_ref = hint_instTs->struct_ref; // if `type WAlias = Wrapper`, here `Wrapper` (generic struct) } if (!struct_ref) { - fire(cur_f, v->type_node->loc, to_string(v->type_node->resolved_type) + " does not name a struct"); + fire(cur_f, v->type_node, to_string(v->type_node->resolved_type) + " does not name a struct"); } // example: `var v: Ok = Ok { ... }`, now struct_ref is "Ok", take "Ok" from hint if (struct_ref->is_generic_struct() && hint) { @@ -1411,7 +1411,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } } if (!struct_ref) { - fire(cur_f, v->loc, "can not detect struct name\nuse either `var v: StructName = { ... }` or `var v = StructName { ... }`"); + fire(cur_f, v, "can not detect struct name\nuse either `var v: StructName = { ... }` or `var v = StructName { ... }`"); } // so, we have struct_ref, so we can check field names and infer values @@ -1424,12 +1424,12 @@ class InferTypesAndCallsAndFieldsVisitor final { std::string_view field_name = field_i->get_field_name(); StructFieldPtr field_ref = struct_ref->find_field(field_name); if (!field_ref) { - fire(cur_f, field_i->loc, "field `" + to_string(field_name) + "` not found in struct " + to_string(struct_ref)); + fire(cur_f, field_i->get_field_identifier(), "field `" + to_string(field_name) + "` not found in struct " + to_string(struct_ref)); } field_i->mutate()->assign_field_ref(field_ref); if (occurred_mask & (1ULL << field_ref->field_idx)) { - fire(cur_f, field_i->loc, "duplicate field initialization"); + fire(cur_f, field_i->get_field_identifier(), "duplicate field initialization"); } occurred_mask |= 1ULL << field_ref->field_idx; @@ -1440,7 +1440,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } flow = infer_any_expr(val_i, std::move(flow), false, field_type).out_flow; if (field_type->has_genericT_inside()) { - deducingTs.auto_deduce_from_argument(cur_f, field_i->loc, field_type, val_i->inferred_type); + deducingTs.auto_deduce_from_argument(cur_f, field_i->get_init_val()->range, field_type, val_i->inferred_type); } assign_inferred_type(field_i, val_i); } @@ -1453,7 +1453,7 @@ class InferTypesAndCallsAndFieldsVisitor final { nameT_unknown = deducingTs.get_first_not_deduced_nameT(); } if (!nameT_unknown.empty()) { - deducingTs.fire_error_can_not_deduce(cur_f, v->loc, nameT_unknown); + deducingTs.fire_can_not_deduce(cur_f, v->range, nameT_unknown); } struct_ref = instantiate_generic_struct(struct_ref, deducingTs.flush()); // re-assign field_ref (it was earlier assigned into a field of a generic struct) @@ -1468,7 +1468,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (!(occurred_mask & (1ULL << field_ref->field_idx))) { bool allow_missing = field_ref->has_default_value() || field_ref->declared_type == TypeDataNever::create(); if (!allow_missing) { - fire(cur_f, v->get_body()->loc, "field `" + field_ref->name + "` missed in initialization of struct " + to_string(struct_ref)); + fire(cur_f, SrcRange::empty_at_end(v->range), "field `" + field_ref->name + "` missed in initialization of struct " + to_string(struct_ref)); } } } @@ -1641,7 +1641,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (!body_end.is_unreachable()) { fun_ref->mutate()->assign_is_implicit_return(); if (fun_ref->declared_return_type == TypeDataNever::create()) { - fire(fun_ref, v_function->get_body()->as()->loc_end, "a function returning `never` can not have a reachable endpoint"); + fire(fun_ref, SrcRange::empty_at_end(v_function->range), "a function returning `never` can not have a reachable endpoint"); } } @@ -1664,18 +1664,18 @@ class InferTypesAndCallsAndFieldsVisitor final { } if (!body_end.is_unreachable() && inferred_return_type != TypeDataVoid::create()) { - fire(fun_ref, v_function->get_body()->as()->loc_end, "missing return"); + fire(fun_ref, SrcRange::empty_at_end(v_function->range), "missing return"); } if (has_void_returns && has_non_void_returns) { for (AnyExprV return_expr : return_statements) { if (return_expr->inferred_type == TypeDataVoid::create()) { - fire(fun_ref, return_expr->loc, "mixing void and non-void returns in function " + to_string(fun_ref)); + fire(fun_ref, return_expr, "mixing void and non-void returns in function " + to_string(fun_ref)); } } } if (return_unifier.is_union_of_different_types()) { // `return intVar` + `return sliceVar` results in `int | slice`, probably unexpected - fire(fun_ref, v_function->get_body()->loc, "function " + to_string(fun_ref) + " calculated return type is " + to_string(inferred_return_type) + "; probably, it's not what you expected\ndeclare `fun (...): ` manually"); + fire(fun_ref, v_function->get_identifier(), "function " + to_string(fun_ref) + " calculated return type is " + to_string(inferred_return_type) + "; probably, it's not what you expected\ndeclare `fun (...): ` manually"); } } @@ -1750,7 +1750,7 @@ static void infer_and_save_return_type_of_function(FunctionPtr fun_ref) { // prevent recursion of untyped functions, like `fun f() { return g(); } fun g() { return f(); }` bool contains = std::find(called_stack.begin(), called_stack.end(), fun_ref) != called_stack.end(); if (contains) { - fire(fun_ref, fun_ref->loc, "could not infer return type of " + to_string(fun_ref) + ", because it appears in a recursive call chain\ndeclare `fun (...): ` manually"); + fire(fun_ref, fun_ref->ident_anchor, "could not infer return type of " + to_string(fun_ref) + ", because it appears in a recursive call chain\ndeclare `fun (...): ` manually"); } // dig into g's body; it's safe, since the compiler is single-threaded @@ -1770,7 +1770,7 @@ static void infer_and_save_type_of_constant(GlobalConstPtr const_ref) { // prevent recursion like `const a = b; const b = a` bool contains = std::find(called_stack.begin(), called_stack.end(), const_ref) != called_stack.end(); if (contains) { - throw ParseError(const_ref->loc, "const `" + const_ref->name + "` appears, directly or indirectly, in its own initializer"); + fire(const_ref->ident_anchor, "const `" + const_ref->name + "` appears, directly or indirectly, in its own initializer"); } called_stack.push_back(const_ref); diff --git a/tolk/pipe-lazy-load-insertions.cpp b/tolk/pipe-lazy-load-insertions.cpp index 8eb0d76d07..fe4a9f7a8f 100644 --- a/tolk/pipe-lazy-load-insertions.cpp +++ b/tolk/pipe-lazy-load-insertions.cpp @@ -14,12 +14,12 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . */ -#include "tolk.h" #include "lazy-helpers.h" #include "ast.h" #include "ast-aux-data.h" #include "ast-visitor.h" #include "ast-replacer.h" +#include "compilation-errors.h" #include "type-system.h" #include "smart-casts-cfg.h" #include "pack-unpack-api.h" @@ -150,7 +150,7 @@ static bool does_function_satisfy_for_lazy_operator(FunctionPtr fun_ref, bool al return false; } // allow `lazy SomeStruct.fromSlice(s)`; these functions are also handled while transforming AST to Ops - if (fun_ref->is_builtin_function() && fun_ref->is_instantiation_of_generic_function()) { + if (fun_ref->is_builtin() && fun_ref->is_instantiation_of_generic_function()) { std::string_view f_name = fun_ref->base_fun_ref->name; return f_name == "T.fromSlice" || f_name == "T.fromCell" || f_name == "Cell.load"; } @@ -188,7 +188,7 @@ static void check_lazy_operator_used_correctly(FunctionPtr cur_f, Vget_expr()->kind == ast_function_call && does_function_satisfy_for_lazy_operator(v->get_expr()->as()->fun_maybe); if (!is_ok_call) { - fire(cur_f, v->loc, "`lazy` operator can only be used with built-in functions like fromCell/fromSlice or simple wrappers over them"); + fire(cur_f, v->keyword_range(), "`lazy` operator can only be used with built-in functions like fromCell/fromSlice or simple wrappers over them"); } // it should be either a struct or a union of structs @@ -198,18 +198,18 @@ static void check_lazy_operator_used_correctly(FunctionPtr cur_f, Vunwrap_alias()->try_as()) { if (TypePtr wrong_variant = is_union_type_prevented_from_lazy_loading(expr_union)) { - fire(cur_f, v->loc, "`lazy` union should contain only structures, but it contains `" + wrong_variant->as_human_readable() + "`"); + fire(cur_f, v->keyword_range(), "`lazy` union should contain only structures, but it contains `" + wrong_variant->as_human_readable() + "`"); } return; } - fire(cur_f, v->loc, "`lazy` is applicable to structs, not to `" + expr_type->as_human_readable() + "`"); + fire(cur_f, v->keyword_range(), "`lazy` is applicable to structs, not to `" + expr_type->as_human_readable() + "`"); } // Given `storage.save()` for a lazy `storage` variable, check if `self` inside should gain laziness. // If yes, the body of the method is also traversed to detect usages. // If no, it's assumed that all fields of `storage` are used (an object used "as a whole"). static bool can_method_be_inlined_preserving_lazy(FunctionPtr method_ref) { - if (method_ref->is_builtin_function() && method_ref->is_instantiation_of_generic_function()) { + if (method_ref->is_builtin() && method_ref->is_instantiation_of_generic_function()) { std::string_view f_name = method_ref->base_fun_ref->name; return f_name == "T.toCell" || f_name == "T.forceLoadLazyObject"; } @@ -362,12 +362,12 @@ struct ExprUsagesWhileCollecting { } } - LazyStructLoadInfo generate_hidden_struct_load_all(SrcLocation loc, bool is_variant_of_union) const { + LazyStructLoadInfo generate_hidden_struct_load_all(bool is_variant_of_union) const { tolk_assert(struct_ref); StructPtr hidden_struct = new StructData( "(lazy)" + struct_ref->name, - loc, + struct_ref->ident_anchor, std::vector(struct_ref->fields), is_variant_of_union ? StructData::PackOpcode(0, 0) : struct_ref->opcode, struct_ref->overflow1023_policy, @@ -381,7 +381,7 @@ struct ExprUsagesWhileCollecting { } // for every field of a struct, after calculating all usages, determine: which fields to load, and which to skip - LazyStructLoadInfo calculate_hidden_struct(SrcLocation loc, bool is_variant_of_union) const { + LazyStructLoadInfo calculate_hidden_struct(bool is_variant_of_union) const { tolk_assert(struct_ref); struct FutureField { @@ -489,7 +489,8 @@ struct ExprUsagesWhileCollecting { ith_field_action.reserve(future_fields.size()); for (int field_idx = 0; field_idx < static_cast(future_fields.size()); ++field_idx) { FutureField f = future_fields[field_idx]; - StructFieldPtr created = new StructFieldData(static_cast(f.field_name), {}, field_idx, false, false, nullptr, nullptr); + AnyV v_ident = createV(SrcRange::undefined(), ""); + StructFieldPtr created = new StructFieldData(static_cast(f.field_name), v_ident, field_idx, false, false, nullptr, nullptr); created->mutate()->assign_resolved_type(f.field_type); hidden_fields.push_back(created); ith_field_action.push_back(f.action); @@ -497,7 +498,7 @@ struct ExprUsagesWhileCollecting { StructPtr hidden_struct = new StructData( "(lazy)" + struct_ref->name, - loc, + struct_ref->ident_anchor, std::move(hidden_fields), is_variant_of_union ? StructData::PackOpcode(0, 0) : struct_ref->opcode, struct_ref->overflow1023_policy, @@ -555,7 +556,7 @@ struct LazyVarInFunction { v_lazy_match_var_itself = var_usages.used_as_match_subj.front(); load_points.reserve(var_usages.variants.size()); for (ExprUsagesWhileCollecting& variant_usages : var_usages.variants) { - LazyStructLoadInfo load_info = variant_usages.calculate_hidden_struct(created_by_lazy_op->loc, true); + LazyStructLoadInfo load_info = variant_usages.calculate_hidden_struct(true); load_points.emplace_back(std::move(variant_usages.needed_above_stmt), variant_usages.expr_type, nullptr, std::move(load_info)); } return; @@ -566,11 +567,11 @@ struct LazyVarInFunction { const TypeDataStruct* t_struct = var_ref->declared_type->unwrap_alias()->try_as(); bool is_union = t_struct == nullptr; if (is_union) { - fire(cur_f, created_by_lazy_op->loc, "`lazy` will not work here, because variable `" + var_ref->name + "` it's used in a non-lazy manner\nhint: lazy union may be used only in `match` statement"); + fire(cur_f, created_by_lazy_op->keyword_range(), "`lazy` will not work here, because variable `" + var_ref->name + "` it's used in a non-lazy manner\nhint: lazy union may be used only in `match` statement"); } // so, it's just a struct, `lazy Point`; we've already calculated all statements where its fields are used - LazyStructLoadInfo load_info = var_usages.calculate_hidden_struct(created_by_lazy_op->loc, false); + LazyStructLoadInfo load_info = var_usages.calculate_hidden_struct(false); bool is_lazy_match_last_field = !load_info.ith_field_action.empty() && load_info.ith_field_action.back() == LazyStructLoadInfo::LazyMatchField; load_points.emplace_back(std::move(var_usages.needed_above_stmt), nullptr, nullptr, std::move(load_info)); @@ -587,7 +588,7 @@ struct LazyVarInFunction { if (!v_arm_body->empty()) { const TypeDataUnion* t_union = field_ref->declared_type->unwrap_alias()->try_as(); int variant_idx = t_union->get_variant_idx(union_variant); - LazyStructLoadInfo load_all = last_field_usages.variants[variant_idx].generate_hidden_struct_load_all(created_by_lazy_op->loc, true); + LazyStructLoadInfo load_all = last_field_usages.variants[variant_idx].generate_hidden_struct_load_all(true); load_points.emplace_back(std::vector{v_arm_body->get_item(0)}, union_variant, field_ref, std::move(load_all)); } } @@ -642,11 +643,11 @@ class CollectUsagesInStatementVisitor final : ASTVisitorFunctionBody { AnyExprV dot_obj = v->get_callee()->as()->get_obj(); if (extract_sink_expression_from_vertex(dot_obj) == s_expr) { // handle built-in functions specially - if (fun_ref->is_builtin_function() && fun_ref->base_fun_ref->name == "T.toCell") { + if (fun_ref->is_builtin() && fun_ref->base_fun_ref->name == "T.toCell") { lazy_expr->on_used_toCell(); return; } - if (fun_ref->is_builtin_function() && fun_ref->base_fun_ref->name == "T.forceLoadLazyObject") { + if (fun_ref->is_builtin() && fun_ref->base_fun_ref->name == "T.forceLoadLazyObject") { lazy_expr->on_used_rw(false); // just use the object as a whole, slice will point to its end return; } @@ -862,7 +863,7 @@ class CollectAllLazyObjectsAndFieldsVisitor final : public ASTVisitorFunctionBod } // for `return lazy ...` and other cases except allowed - fire(cur_f, v->loc, "incorrect `lazy` operator usage, it's not directly assigned to a variable\nhint: use `lazy` like this:\n> var st = lazy MyStorage.fromSlice(...)"); + fire(cur_f, v->keyword_range(), "incorrect `lazy` operator usage, it's not directly assigned to a variable\nhint: use `lazy` like this:\n> var st = lazy MyStorage.fromSlice(...)"); } public: @@ -907,7 +908,7 @@ class LazyLoadInsertionsReplacer final : public ASTReplacerInFunctionBody { for (const OneLoadingInsertionPoint& ins : lazy_var.load_points) { if (!ins.was_inserted_to_ast && ins.is_mentioned_in_stmt(stmt)) { ASTAuxData* aux_data = new AuxData_LazyObjectLoadFields(lazy_var.var_ref, ins.union_variant, ins.field_ref, ins.load_info); - new_children.push_back(createV(stmt->loc, createV(stmt->loc), aux_data, TypeDataVoid::create())); + new_children.push_back(createV(createV(stmt->range), aux_data, TypeDataVoid::create())); ins.mark_inserted_to_ast(); } } @@ -925,14 +926,14 @@ class LazyLoadInsertionsReplacer final : public ASTReplacerInFunctionBody { bool is_lazy_match_for_union = lazy_var.v_lazy_match_var_itself == v; if (is_lazy_match_for_union) { ASTAuxData* aux_data = new AuxData_LazyMatchForUnion(lazy_var.var_ref, nullptr); - return createV(v->loc, parent::replace(v), aux_data, v->inferred_type); + return createV(parent::replace(v), aux_data, v->inferred_type); } bool is_lazy_match_for_last_field = lazy_var.v_lazy_match_last_field == v; if (is_lazy_match_for_last_field) { StructPtr struct_ref = lazy_var.var_ref->declared_type->unwrap_alias()->try_as()->struct_ref; ASTAuxData* aux_data = new AuxData_LazyMatchForUnion(lazy_var.var_ref, struct_ref->fields.back()); - return createV(v->loc, parent::replace(v), aux_data, v->inferred_type); + return createV(parent::replace(v), aux_data, v->inferred_type); } } @@ -989,7 +990,7 @@ class CheckExpectLazyAssertionsVisitor final : public ASTVisitorFunctionBody { for (int i = 0; i < v->size(); ++i) { AnyV cur_stmt = v->get_item(i); if (auto v_call = cur_stmt->try_as()) { - if (v_call->fun_maybe && v_call->fun_maybe->is_builtin_function() && v_call->fun_maybe->name == "__expect_lazy") { + if (v_call->fun_maybe && v_call->fun_maybe->is_builtin() && v_call->fun_maybe->name == "__expect_lazy") { AnyV next_stmt = v->get_item(i + 1); std::string_view expected = v_call->get_arg(0)->get_expr()->as()->str_val; std::string actual; @@ -1003,7 +1004,7 @@ class CheckExpectLazyAssertionsVisitor final : public ASTVisitorFunctionBody { } if (actual != expected) { - throw ParseError(cur_stmt->loc, "__expect_lazy failed: actual \"" + actual + "\""); + fire(cur_stmt, "__expect_lazy failed: actual \"" + actual + "\""); } } } diff --git a/tolk/pipe-optimize-boolean-expr.cpp b/tolk/pipe-optimize-boolean-expr.cpp index 6b053915f7..e25c3201c9 100644 --- a/tolk/pipe-optimize-boolean-expr.cpp +++ b/tolk/pipe-optimize-boolean-expr.cpp @@ -14,7 +14,6 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . */ -#include "tolk.h" #include "ast.h" #include "ast-replacer.h" #include "type-system.h" @@ -35,22 +34,22 @@ namespace tolk { struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { - static V create_int_const(SrcLocation loc, td::RefInt256&& intval) { - auto v_int = createV(loc, std::move(intval), {}); + static V create_int_const(SrcRange range, td::RefInt256&& intval) { + auto v_int = createV(range, std::move(intval), {}); v_int->assign_inferred_type(TypeDataInt::create()); v_int->assign_rvalue_true(); return v_int; } - static V create_bool_const(SrcLocation loc, bool bool_val) { - auto v_bool = createV(loc, bool_val); + static V create_bool_const(SrcRange range, bool bool_val) { + auto v_bool = createV(range, bool_val); v_bool->assign_inferred_type(TypeDataInt::create()); v_bool->assign_rvalue_true(); return v_bool; } - static V create_logical_not_for_bool(SrcLocation loc, AnyExprV rhs) { - auto v_not = createV(loc, "!", tok_logical_not, rhs); + static V create_logical_not_for_bool(SrcRange range, SrcRange operator_range, AnyExprV rhs) { + auto v_not = createV(range, operator_range, "!", tok_logical_not, rhs); v_not->assign_inferred_type(TypeDataBool::create()); v_not->assign_rvalue_true(); v_not->assign_fun_ref(lookup_function("!b_")); @@ -94,8 +93,8 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { } // `!!intVar` => `intVar != 0` if (expect_integer(cond_not_not->inferred_type)) { - auto v_zero = create_int_const(v->loc, td::make_refint(0)); - auto v_neq = createV(v->loc, "!=", tok_neq, cond_not_not, v_zero); + auto v_zero = create_int_const(v->range, td::make_refint(0)); + auto v_neq = createV(v->range, v->operator_range, "!=", tok_neq, cond_not_not, v_zero); v_neq->mutate()->assign_rvalue_true(); v_neq->mutate()->assign_inferred_type(TypeDataBool::create()); v_neq->mutate()->assign_fun_ref(lookup_function("_!=_")); @@ -104,7 +103,7 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { } if (auto inner_bool = v->get_rhs()->try_as()) { // `!true` / `!false` - return create_bool_const(v->loc, !inner_bool->bool_val); + return create_bool_const(v->range, !inner_bool->bool_val); } } @@ -123,7 +122,7 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { return lhs; } // `boolVar != true` / `boolVar == false` - return create_logical_not_for_bool(v->loc, lhs); + return create_logical_not_for_bool(v->range, v->operator_range, lhs); } } @@ -138,20 +137,20 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { if (v_cond_unary->tok != tok_logical_not) { break; } - v = createV(v->loc, !v->is_ifnot, v_cond_unary->get_rhs(), v->get_if_body(), v->get_else_body()); + v = createV(v->range, !v->is_ifnot, v_cond_unary->get_rhs(), v->get_if_body(), v->get_else_body()); } // `if (x != null)` -> ifnot(x == null) if (auto v_cond_istype = v->get_cond()->try_as(); v_cond_istype && v_cond_istype->is_negated) { v_cond_istype->mutate()->assign_is_negated(!v_cond_istype->is_negated); - v = createV(v->loc, !v->is_ifnot, v_cond_istype, v->get_if_body(), v->get_else_body()); + v = createV(v->range, !v->is_ifnot, v_cond_istype, v->get_if_body(), v->get_else_body()); } // `if (addr1 != addr2)` -> ifnot(addr1 == addr2) if (auto v_cond_neq = v->get_cond()->try_as()) { if (v_cond_neq->tok == tok_neq && v_cond_neq->get_lhs()->inferred_type->unwrap_alias() == TypeDataAddress::create() && v_cond_neq->get_rhs()->inferred_type->unwrap_alias() == TypeDataAddress::create()) { - auto v_cond_eq = createV(v_cond_neq->loc, "==", tok_eq, v_cond_neq->get_lhs(), v_cond_neq->get_rhs()); + auto v_cond_eq = createV(v_cond_neq->range, v_cond_neq->operator_range, "==", tok_eq, v_cond_neq->get_lhs(), v_cond_neq->get_rhs()); v_cond_eq->mutate()->assign_inferred_type(v_cond_neq->inferred_type); v_cond_eq->mutate()->assign_rvalue_true(); - v = createV(v->loc, !v->is_ifnot, v_cond_eq, v->get_if_body(), v->get_else_body()); + v = createV(v->range, !v->is_ifnot, v_cond_eq, v->get_if_body(), v->get_else_body()); } } diff --git a/tolk/pipe-refine-lvalue-for-mutate.cpp b/tolk/pipe-refine-lvalue-for-mutate.cpp index 3e02cf7738..26526c5654 100644 --- a/tolk/pipe-refine-lvalue-for-mutate.cpp +++ b/tolk/pipe-refine-lvalue-for-mutate.cpp @@ -14,9 +14,9 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . */ -#include "tolk.h" #include "ast.h" #include "ast-visitor.h" +#include "compilation-errors.h" /* * This pipe refines rvalue/lvalue and checks `mutate` arguments validity. @@ -34,15 +34,15 @@ namespace tolk { GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_invalid_mutate_arg_passed(FunctionPtr cur_f, SrcLocation loc, FunctionPtr fun_ref, const LocalVarData& p_sym, bool arg_passed_as_mutate, AnyV arg_expr) { +static void fire_invalid_mutate_arg_passed(FunctionPtr cur_f, SrcRange range, FunctionPtr fun_ref, const LocalVarData& p_sym, bool arg_passed_as_mutate, AnyV arg_expr) { std::string arg_str(arg_expr->kind == ast_reference ? arg_expr->as()->get_name() : "obj"); if (p_sym.is_mutate_parameter() && !arg_passed_as_mutate) { // called `mutating_function(arg)`; suggest: `mutate arg` - fire(cur_f, loc, "function `" + fun_ref->as_human_readable() + "` mutates parameter `" + p_sym.name + "`\nyou need to specify `mutate` when passing an argument, like `mutate " + arg_str + "`"); + fire(cur_f, range, "function `" + fun_ref->as_human_readable() + "` mutates parameter `" + p_sym.name + "`\nyou need to specify `mutate` when passing an argument, like `mutate " + arg_str + "`"); } else { // called `usual_function(mutate arg)` - fire(cur_f, loc, "incorrect `mutate`, since `" + fun_ref->as_human_readable() + "` does not mutate this parameter"); + fire(cur_f, range, "incorrect `mutate`, since `" + fun_ref->as_human_readable() + "` does not mutate this parameter"); } } @@ -58,7 +58,7 @@ class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBod for (int i = 0; i < v->get_num_args(); ++i) { auto v_arg = v->get_arg(i); if (v_arg->passed_as_mutate) { - v_arg->error("`mutate` used for non-mutate parameter"); + fire(v_arg, "`mutate` used for non-mutate parameter"); } } return; @@ -94,7 +94,7 @@ class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBod const LocalVarData& p_sym = fun_ref->parameters[delta_self + i]; auto arg_i = v->get_arg(i); if (p_sym.is_mutate_parameter() != arg_i->passed_as_mutate) { - fire_error_invalid_mutate_arg_passed(cur_f, arg_i->loc, fun_ref, p_sym, arg_i->passed_as_mutate, arg_i->get_expr()); + fire_invalid_mutate_arg_passed(cur_f, arg_i->range, fun_ref, p_sym, arg_i->passed_as_mutate, arg_i->get_expr()); } parent::visit(arg_i); } diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 7f684259a0..cddb912297 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -14,10 +14,10 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . */ -#include "tolk.h" +#include "ast.h" #include "platform-utils.h" #include "src-file.h" -#include "ast.h" +#include "compilation-errors.h" #include "compiler-state.h" #include "generics-helpers.h" #include "pack-unpack-serializers.h" @@ -65,18 +65,18 @@ static int calculate_tvm_method_id_by_func_name(std::string_view func_name) { static void validate_arg_ret_order_of_asm_function(V v_body, int n_params) { if (n_params > 16) { - v_body->error("asm function can have at most 16 parameters"); + fire(v_body, "asm function can have at most 16 parameters"); } // asm(param1 ... paramN), param names were previously mapped into indices if (!v_body->arg_order.empty()) { if (static_cast(v_body->arg_order.size()) != n_params) { - v_body->error("arg_order of asm function must specify all parameters"); + fire(v_body, "arg_order of asm function must specify all parameters"); } std::vector visited(v_body->arg_order.size(), false); for (int j : v_body->arg_order) { if (visited[j]) { - v_body->error("arg_order of asm function contains duplicates"); + fire(v_body, "arg_order of asm function contains duplicates"); } visited[j] = true; } @@ -88,7 +88,7 @@ static void validate_arg_ret_order_of_asm_function(V v_body, int n std::vector visited(v_body->ret_order.size(), false); for (int j : v_body->ret_order) { if (j < 0 || j >= static_cast(v_body->ret_order.size()) || visited[j]) { - v_body->error("ret_order contains invalid integer, not in range 0 .. N"); + fire(v_body, "ret_order contains invalid integer, not in range 0 .. N"); } visited[j] = true; } @@ -96,7 +96,8 @@ static void validate_arg_ret_order_of_asm_function(V v_body, int n } static GlobalConstPtr register_constant(V v) { - GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->type_node, v->get_init_value()); + V v_ident = v->get_identifier(); + GlobalConstData* c_sym = new GlobalConstData(static_cast(v_ident->name), v_ident, v->type_node, v->get_init_value()); G.symtable.add_global_symbol(c_sym); G.all_constants.push_back(c_sym); @@ -105,7 +106,8 @@ static GlobalConstPtr register_constant(V v) { } static GlobalVarPtr register_global_var(V v) { - GlobalVarData* g_sym = new GlobalVarData(static_cast(v->get_identifier()->name), v->loc, v->type_node); + V v_ident = v->get_identifier(); + GlobalVarData* g_sym = new GlobalVarData(static_cast(v_ident->name), v_ident, v->type_node); G.symtable.add_global_symbol(g_sym); G.all_global_vars.push_back(g_sym); @@ -114,12 +116,13 @@ static GlobalVarPtr register_global_var(V v) { } static AliasDefPtr register_type_alias(V v, AliasDefPtr base_alias_ref = nullptr, std::string override_name = {}, const GenericsSubstitutions* substitutedTs = nullptr) { + V v_ident = v->get_identifier(); std::string name = std::move(override_name); if (name.empty()) { - name = v->get_identifier()->name; + name = v_ident->name; } const GenericsDeclaration* genericTs = nullptr; // at registering it's null; will be assigned after types resolving - AliasDefData* a_sym = new AliasDefData(std::move(name), v->loc, v->underlying_type_node, genericTs, substitutedTs, v); + AliasDefData* a_sym = new AliasDefData(std::move(name), v_ident, v->underlying_type_node, genericTs, substitutedTs, v); a_sym->base_alias_ref = base_alias_ref; // for `Response`, here is `Response` G.symtable.add_global_symbol(a_sym); @@ -134,17 +137,19 @@ static EnumDefPtr register_enum(V v) { members.reserve(v_body->get_num_members()); for (int i = 0; i < v_body->get_num_members(); ++i) { auto v_member = v_body->get_member(i); - std::string member_name = static_cast(v_member->get_identifier()->name); + V v_ident = v_member->get_identifier(); + std::string member_name = static_cast(v_ident->name); for (EnumMemberPtr prev : members) { if (UNLIKELY(prev->name == member_name)) { - v_member->error("redeclaration of member `" + member_name + "`"); + fire(v_member, "redeclaration of member `" + member_name + "`"); } } - members.emplace_back(new EnumMemberData(member_name, v_member->loc, v_member->init_value)); + members.emplace_back(new EnumMemberData(member_name, v_ident, v_member->init_value)); } - EnumDefData* e_sym = new EnumDefData(static_cast(v->get_identifier()->name), v->loc, v->colon_type, std::move(members)); + V v_ident = v->get_identifier(); + EnumDefData* e_sym = new EnumDefData(static_cast(v_ident->name), v_ident, v->colon_type, std::move(members)); G.symtable.add_global_symbol(e_sym); G.all_enums.push_back(e_sym); @@ -159,21 +164,22 @@ static StructPtr register_struct(V v, StructPtr base_str fields.reserve(v_body->get_num_fields()); for (int i = 0; i < v_body->get_num_fields(); ++i) { auto v_field = v_body->get_field(i); - std::string field_name = static_cast(v_field->get_identifier()->name); + V v_ident = v_field->get_identifier(); + std::string field_name = static_cast(v_ident->name); for (StructFieldPtr prev : fields) { if (UNLIKELY(prev->name == field_name)) { - v_field->error("redeclaration of field `" + field_name + "`"); + fire(v_field, "redeclaration of field `" + field_name + "`"); } } - fields.emplace_back(new StructFieldData(field_name, v_field->loc, i, v_field->is_private, v_field->is_readonly, v_field->type_node, v_field->default_value)); + fields.emplace_back(new StructFieldData(field_name, v_ident, i, v_field->is_private, v_field->is_readonly, v_field->type_node, v_field->default_value)); } PackOpcode opcode(0, 0); if (v->has_opcode()) { auto v_opcode = v->get_opcode()->as(); if (v_opcode->intval < 0 || v_opcode->intval > (1ULL << 48)) { - v->error("opcode must not exceed 2^48"); + fire(v, "opcode must not exceed 2^48"); } opcode.pack_prefix = v_opcode->intval->to_long(); @@ -187,12 +193,13 @@ static StructPtr register_struct(V v, StructPtr base_str } } + V v_ident = v->get_identifier(); std::string name = std::move(override_name); if (name.empty()) { - name = v->get_identifier()->name; + name = v_ident->name; } const GenericsDeclaration* genericTs = nullptr; // at registering it's null; will be assigned after types resolving - StructData* s_sym = new StructData(std::move(name), v->loc, std::move(fields), opcode, v->overflow1023_policy, genericTs, substitutedTs, v); + StructData* s_sym = new StructData(std::move(name), v_ident, std::move(fields), opcode, v->overflow1023_policy, genericTs, substitutedTs, v); s_sym->base_struct_ref = base_struct_ref; // for `Container`, here is `Container` G.symtable.add_global_symbol(s_sym); @@ -202,18 +209,19 @@ static StructPtr register_struct(V v, StructPtr base_str } static LocalVarData register_parameter(V v, int idx) { - if (v->is_underscore()) { - return LocalVarData{"", v->loc, v->type_node, v->default_value, 0, idx}; + V v_ident = v->get_identifier(); + if (v_ident->name == "_") { + return LocalVarData("", v, v->type_node, v->default_value, 0, idx); } int flags = 0; if (v->declared_as_mutate) { flags |= LocalVarData::flagMutateParameter; } - if (!v->declared_as_mutate && idx == 0 && v->param_name == "self") { + if (!v->declared_as_mutate && idx == 0 && v_ident->name == "self") { flags |= LocalVarData::flagImmutable; } - return LocalVarData(static_cast(v->param_name), v->loc, v->type_node, v->default_value, flags, idx); + return LocalVarData(static_cast(v_ident->name), v_ident, v->type_node, v->default_value, flags, idx); } static FunctionPtr register_function(V v, FunctionPtr base_fun_ref = nullptr, std::string override_name = {}, const GenericsSubstitutions* substitutedTs = nullptr) { @@ -221,7 +229,8 @@ static FunctionPtr register_function(V v, FunctionPtr return nullptr; } - std::string_view f_identifier = v->get_identifier()->name; // function or method name + V v_ident = v->get_identifier(); + std::string_view f_identifier = v_ident->name; // function or method name std::vector parameters; int n_mutate_params = 0; @@ -243,12 +252,12 @@ static FunctionPtr register_function(V v, FunctionPtr const GenericsDeclaration* genericTs = nullptr; // at registering it's null; will be assigned after types resolving FunctionBody f_body = v->get_body()->kind == ast_block_statement ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); - FunctionData* f_sym = new FunctionData(std::move(name), v->loc, std::move(method_name), v->receiver_type_node, v->return_type_node, std::move(parameters), 0, v->inline_mode, genericTs, substitutedTs, f_body, v); + FunctionData* f_sym = new FunctionData(std::move(name), v_ident, std::move(method_name), v->receiver_type_node, v->return_type_node, std::move(parameters), 0, v->inline_mode, genericTs, substitutedTs, f_body, v); f_sym->base_fun_ref = base_fun_ref; // for `f`, here is `f` if (auto v_asm = v->get_body()->try_as()) { if (!v->return_type_node) { - v_asm->error("asm function must declare return type (before asm instructions)"); + fire(v_asm, "asm function must declare return type (before asm instructions)"); } validate_arg_ret_order_of_asm_function(v_asm, v->get_num_params()); f_sym->arg_order = v_asm->arg_order; @@ -261,7 +270,7 @@ static FunctionPtr register_function(V v, FunctionPtr f_sym->tvm_method_id = calculate_tvm_method_id_by_func_name(f_identifier); for (FunctionPtr other : G.all_contract_getters) { if (other->tvm_method_id == f_sym->tvm_method_id) { - v->error(PSTRING() << "GET methods hash collision: `" << other->name << "` and `" << f_sym->name << "` produce the same hash. Consider renaming one of these functions."); + fire(v, "GET methods hash collision: `" + other->name + "` and `" + f_sym->name + "` produce the same hash. Consider renaming one of these functions."); } } } else if (v->flags & FunctionData::flagIsEntrypoint) { diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 320ebf70c5..e863b78747 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -14,13 +14,12 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "tolk.h" +#include "ast.h" +#include "ast-visitor.h" +#include "compilation-errors.h" #include "platform-utils.h" #include "compiler-state.h" #include "src-file.h" -#include "ast.h" -#include "ast-visitor.h" -#include /* * This pipe resolves identifiers (local variables, globals, constants, etc.) in all functions bodies. @@ -47,29 +46,29 @@ namespace tolk { GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_undefined_symbol(FunctionPtr cur_f, V v) { +static void fire_undefined_symbol(FunctionPtr cur_f, V v) { if (v->name == "self") { - fire(cur_f, v->loc, "using `self` in a non-member function (it does not accept the first `self` parameter)"); + fire(cur_f, v, "using `self` in a non-member function (it does not accept the first `self` parameter)"); } else { - fire(cur_f, v->loc, "undefined symbol `" + static_cast(v->name) + "`"); + fire(cur_f, v, "undefined symbol `" + static_cast(v->name) + "`"); } } GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_type_used_as_symbol(FunctionPtr cur_f, V v) { +static void fire_type_used_as_symbol(FunctionPtr cur_f, V v) { if (v->name == "random") { // calling `random()`, but it's a struct, correct is `random.uint256()` - fire(cur_f, v->loc, "`random` is not a function, you probably want `random.uint256()`"); + fire(cur_f, v, "`random` is not a function, you probably want `random.uint256()`"); } else { - fire(cur_f, v->loc, "`" + static_cast(v->name) + "` only refers to a type, but is being used as a value here"); + fire(cur_f, v, "`" + static_cast(v->name) + "` only refers to a type, but is being used as a value here"); } } GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_using_self_not_in_method(FunctionPtr cur_f, SrcLocation loc) { +static void fire_using_self_not_in_method(FunctionPtr cur_f, SrcRange range) { if (cur_f->is_static_method()) { - fire(cur_f, loc, "using `self` in a static method"); + fire(cur_f, range, "using `self` in a static method"); } else { - fire(cur_f, loc, "using `self` in a regular function (not a method)"); + fire(cur_f, range, "using `self` in a regular function (not a method)"); } } @@ -80,12 +79,11 @@ struct NameAndScopeResolver { return std::hash{}(name_key); } - void open_scope([[maybe_unused]] SrcLocation loc) { - // std::cerr << "open_scope " << scopes.size() + 1 << " at " << loc << std::endl; + void open_scope() { scopes.emplace_back(); } - void close_scope([[maybe_unused]] SrcLocation loc) { + void close_scope() { // std::cerr << "close_scope " << scopes.size() << " at " << loc << std::endl; if (UNLIKELY(scopes.empty())) { throw Fatal("cannot close the outer scope"); @@ -115,7 +113,7 @@ struct NameAndScopeResolver { uint64_t key = key_hash(v_sym->name); const auto& [_, inserted] = scopes.rbegin()->emplace(key, v_sym); if (UNLIKELY(!inserted)) { - throw ParseError(v_sym->loc, "redeclaration of local variable `" + v_sym->name + "`"); + fire(v_sym->ident_anchor, "redeclaration of local variable `" + v_sym->name + "`"); } } }; @@ -125,15 +123,15 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { static NameAndScopeResolver current_scope; static FunctionPtr cur_f; - static LocalVarPtr create_local_var_sym(std::string_view name, SrcLocation loc, AnyTypeV declared_type_node, bool immutable, bool lateinit) { - LocalVarData* v_sym = new LocalVarData(static_cast(name), loc, declared_type_node, nullptr, immutable * LocalVarData::flagImmutable + lateinit * LocalVarData::flagLateInit, -1); + static LocalVarPtr create_local_var_sym(std::string_view name, AnyV ident_anchor, AnyTypeV declared_type_node, bool immutable, bool lateinit) { + LocalVarData* v_sym = new LocalVarData(static_cast(name), ident_anchor, declared_type_node, nullptr, immutable * LocalVarData::flagImmutable + lateinit * LocalVarData::flagLateInit, -1); current_scope.add_local_var(v_sym); return v_sym; } static void process_catch_variable(AnyExprV catch_var) { if (auto v_ref = catch_var->try_as()) { - LocalVarPtr var_ref = create_local_var_sym(v_ref->get_name(), catch_var->loc, nullptr, true, false); + LocalVarPtr var_ref = create_local_var_sym(v_ref->get_name(), catch_var, nullptr, true, false); v_ref->mutate()->assign_sym(var_ref); } } @@ -143,15 +141,15 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { if (v->marked_as_redef) { const Symbol* sym = current_scope.lookup_symbol(v->get_name()); if (sym == nullptr) { - throw ParseError(cur_f, v->loc, "`redef` for unknown variable"); + fire(cur_f, v, "`redef` for unknown variable"); } LocalVarPtr var_ref = sym->try_as(); if (!var_ref) { - throw ParseError(cur_f, v->loc, "`redef` for unknown variable"); + fire(cur_f, v, "`redef` for unknown variable"); } v->mutate()->assign_var_ref(var_ref); } else { - LocalVarPtr var_ref = create_local_var_sym(v->get_name(), v->loc, v->type_node, v->is_immutable, v->is_lateinit); + LocalVarPtr var_ref = create_local_var_sym(v->get_name(), v, v->type_node, v->is_immutable, v->is_lateinit); v->mutate()->assign_var_ref(var_ref); } } @@ -164,17 +162,18 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { const Symbol* sym = current_scope.lookup_symbol(v->get_name()); if (!sym) { - fire_error_undefined_symbol(cur_f, v->get_identifier()); + fire_undefined_symbol(cur_f, v->get_identifier()); } if (sym->try_as() || sym->try_as() || sym->try_as()) { - fire_error_type_used_as_symbol(cur_f, v->get_identifier()); + fire_type_used_as_symbol(cur_f, v->get_identifier()); } v->mutate()->assign_sym(sym); // for global functions, global vars and constants, `import` must exist if (!sym->try_as()) { - if (!v->loc.is_symbol_from_same_or_builtin_file(sym->loc)) { - sym->check_import_exists_when_used_from(cur_f, v->loc); + bool allow_no_import = sym->is_builtin() || sym->ident_anchor->range.is_file_id_same_or_stdlib_common(v->range); + if (!allow_no_import) { + sym->check_import_exists_when_used_from(cur_f, v); } } } @@ -187,7 +186,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { // for `Point.create` / `int.zero` / `Color.Red`, "undefined symbol" is fired for Point/int/Color // suppress this exception till a later pipe, it will be tried to be resolved as a type if (v_type_name->get_identifier()->name == "self") { - fire_error_using_self_not_in_method(cur_f, v_type_name->loc); + fire_using_self_not_in_method(cur_f, v_type_name->range); } return; } @@ -196,15 +195,15 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { } void visit(V v) override { - current_scope.open_scope(v->loc); + current_scope.open_scope(); parent::visit(v->get_block_statement()); - current_scope.close_scope(v->loc); + current_scope.close_scope(); } void visit(V v) override { - current_scope.open_scope(v->loc); // `match (var a = init_val) { ... }` - parent::visit(v); // then `a` exists only inside `match` arms - current_scope.close_scope(v->loc); + current_scope.open_scope(); // `match (var a = init_val) { ... }` + parent::visit(v); // then `a` exists only inside `match` arms + current_scope.close_scope(); } void visit(V v) override { @@ -216,8 +215,8 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { case MatchArmKind::exact_type: { if (auto maybe_ident = v->pattern_type_node->try_as()) { if (const Symbol* sym = current_scope.lookup_symbol(maybe_ident->text); sym && sym->try_as()) { - auto v_ident = createV(v->loc, sym->name); - AnyExprV pattern_expr = createV(v->loc, v_ident, nullptr); + auto v_ident = createV(maybe_ident->range, sym->name); + AnyExprV pattern_expr = createV(v_ident->range, v_ident, nullptr); parent::visit(pattern_expr); v->mutate()->assign_resolved_pattern(MatchArmKind::const_expression, pattern_expr); } @@ -238,27 +237,27 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { if (v->empty()) { return; } - current_scope.open_scope(v->loc); + current_scope.open_scope(); parent::visit(v); - current_scope.close_scope(v->loc_end); + current_scope.close_scope(); } void visit(V v) override { - current_scope.open_scope(v->loc); + current_scope.open_scope(); parent::visit(v->get_body()); parent::visit(v->get_cond()); // in 'while' condition it's ok to use variables declared inside do - current_scope.close_scope(v->get_body()->loc_end); + current_scope.close_scope(); } void visit(V v) override { visit(v->get_try_body()); - current_scope.open_scope(v->get_catch_body()->loc); + current_scope.open_scope(); const std::vector& catch_items = v->get_catch_expr()->get_items(); tolk_assert(catch_items.size() == 2); process_catch_variable(catch_items[1]); process_catch_variable(catch_items[0]); parent::visit(v->get_catch_body()); - current_scope.close_scope(v->get_catch_body()->loc_end); + current_scope.close_scope(); } public: @@ -270,7 +269,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { cur_f = fun_ref; auto v_block = v->get_body()->as(); - current_scope.open_scope(v->loc); + current_scope.open_scope(); for (int i = 0; i < fun_ref->get_num_params(); ++i) { LocalVarPtr param_ref = &fun_ref->parameters[i]; current_scope.add_local_var(param_ref); @@ -279,7 +278,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { } } parent::visit(v_block); - current_scope.close_scope(v_block->loc_end); + current_scope.close_scope(); tolk_assert(current_scope.scopes.empty()); cur_f = nullptr; diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 8913403b13..9108adb1b2 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -14,17 +14,19 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "tolk.h" -#include "platform-utils.h" -#include "compiler-state.h" #include "ast.h" #include "ast-visitor.h" +#include "compilation-errors.h" +#include "platform-utils.h" +#include "compiler-state.h" #include "generics-helpers.h" #include "type-system.h" #include namespace tolk { +void patch_builtins_after_stdlib_loaded(); + /* * This pipe transforms AST of types into TypePtr. * It happens after all global symbols were registered, and all local references were bound. @@ -51,24 +53,24 @@ static std::unordered_map visited_structs; static std::unordered_map visited_aliases; GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_unknown_type_name(FunctionPtr cur_f, SrcLocation loc, std::string_view text) { +static void fire_unknown_type_name(FunctionPtr cur_f, SrcRange range, std::string_view text) { if (text == "auto") { - fire(cur_f, loc, "`auto` type does not exist; just omit a type for local variable (will be inferred from assignment); parameters should always be typed"); + fire(cur_f, range, "`auto` type does not exist; just omit a type for local variable (will be inferred from assignment); parameters should always be typed"); } if (text == "self") { - fire(cur_f, loc, "`self` type can be used only as a return type of a function (enforcing it to be chainable)"); + fire(cur_f, range, "`self` type can be used only as a return type of a function (enforcing it to be chainable)"); } - fire(cur_f, loc, "unknown type name `" + static_cast(text) + "`"); + fire(cur_f, range, "unknown type name `" + static_cast(text) + "`"); } GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_void_type_not_allowed_inside_union(FunctionPtr cur_f, SrcLocation loc, TypePtr disallowed_variant) { - fire(cur_f, loc, "type `" + disallowed_variant->as_human_readable() + "` is not allowed inside a union"); +static void fire_void_type_not_allowed_inside_union(FunctionPtr cur_f, SrcRange range, TypePtr disallowed_variant) { + fire(cur_f, range, "type `" + disallowed_variant->as_human_readable() + "` is not allowed inside a union"); } GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_generic_type_used_without_T(FunctionPtr cur_f, SrcLocation loc, const std::string& type_name_with_Ts) { - fire(cur_f, loc, "type `" + type_name_with_Ts + "` is generic, you should provide type arguments"); +static void fire_generic_type_used_without_T(FunctionPtr cur_f, SrcRange range, const std::string& type_name_with_Ts) { + fire(cur_f, range, "type `" + type_name_with_Ts + "` is generic, you should provide type arguments"); } static TypePtr parse_intN_uintN(std::string_view strN, bool is_unsigned) { @@ -161,7 +163,6 @@ class TypeNodesVisitorResolver { switch (v->kind) { case ast_type_leaf_text: { std::string_view text = v->as()->text; - SrcLocation loc = v->as()->loc; if (genericTs && genericTs->find_nameT(text) != -1) { // if we're inside `f`, replace "T" with TypeDataGenericT return TypeDataGenericT::create(static_cast(text)); @@ -172,14 +173,15 @@ class TypeNodesVisitorResolver { } if (text == "map") { if (!allow_without_type_arguments) { - fire_error_generic_type_used_without_T(cur_f, loc, "map"); + fire_generic_type_used_without_T(cur_f, v->range, "map"); } return TypeDataMapKV::create(TypeDataGenericT::create("K"), TypeDataGenericT::create("V")); } if (const Symbol* sym = lookup_global_symbol(text)) { - if (TypePtr custom_type = try_resolve_user_defined_type(cur_f, loc, sym, allow_without_type_arguments)) { - if (!v->loc.is_symbol_from_same_or_builtin_file(sym->loc)) { - sym->check_import_exists_when_used_from(cur_f, v->loc); + if (TypePtr custom_type = try_resolve_user_defined_type(cur_f, v->range, sym, allow_without_type_arguments)) { + bool allow_no_import = sym->is_builtin() || sym->ident_anchor->range.is_file_id_same_or_stdlib_common(v->range); + if (!allow_no_import) { + sym->check_import_exists_when_used_from(cur_f, v); } return custom_type; } @@ -190,14 +192,14 @@ class TypeNodesVisitorResolver { if (treat_unresolved_as_genericT) { return TypeDataGenericT::create(static_cast(text)); } - fire_error_unknown_type_name(cur_f, loc, text); + fire_unknown_type_name(cur_f, v->range, text); } case ast_type_question_nullable: { TypePtr inner = finalize_type_node(v->as()->get_inner()); TypePtr result = TypeDataUnion::create_nullable(inner); if (const TypeDataUnion* t_union = result->try_as()) { - validate_resulting_union_type(t_union, cur_f, v->loc); + validate_resulting_union_type(t_union, cur_f, v->range); } return result; } @@ -226,7 +228,7 @@ class TypeNodesVisitorResolver { std::vector variants = finalize_type_node(v->as()->get_variants()); TypePtr result = TypeDataUnion::create(std::move(variants)); if (const TypeDataUnion* t_union = result->try_as()) { - validate_resulting_union_type(t_union, cur_f, v->loc); + validate_resulting_union_type(t_union, cur_f, v->range); } return result; } @@ -239,7 +241,7 @@ class TypeNodesVisitorResolver { for (size_t i = 1; i < inner_and_args.size(); ++i) { type_arguments.push_back(finalize_type_node(inner_and_args[i])); } - return instantiate_generic_type_or_fire(cur_f, v->loc, inner, std::move(type_arguments)); + return instantiate_generic_type_or_fire(cur_f, inner_and_args.front()->range, inner, std::move(type_arguments)); } default: @@ -252,10 +254,10 @@ class TypeNodesVisitorResolver { // example: `var w: Wrapper = ...`, here will be an error of generic usage without T // example: `w is Wrapper`, here not, it's allowed (instantiated at type inferring later) // example: `var w: KKK`, nullptr will be returned - static TypePtr try_resolve_user_defined_type(FunctionPtr cur_f, SrcLocation loc, const Symbol* sym, bool allow_without_type_arguments) { + static TypePtr try_resolve_user_defined_type(FunctionPtr cur_f, SrcRange range, const Symbol* sym, bool allow_without_type_arguments) { if (AliasDefPtr alias_ref = sym->try_as()) { if (alias_ref->is_generic_alias() && !allow_without_type_arguments) { - fire_error_generic_type_used_without_T(cur_f, loc, alias_ref->as_human_readable()); + fire_generic_type_used_without_T(cur_f, range, alias_ref->as_human_readable()); } if (!visited_aliases.contains(alias_ref)) { visit_symbol(alias_ref); @@ -264,7 +266,7 @@ class TypeNodesVisitorResolver { } if (StructPtr struct_ref = sym->try_as()) { if (struct_ref->is_generic_struct() && !allow_without_type_arguments) { - fire_error_generic_type_used_without_T(cur_f, loc, struct_ref->as_human_readable()); + fire_generic_type_used_without_T(cur_f, range, struct_ref->as_human_readable()); } if (!visited_structs.contains(struct_ref)) { visit_symbol(struct_ref); @@ -279,7 +281,7 @@ class TypeNodesVisitorResolver { // given `Wrapper` / `Pair` / `Response`, instantiate a generic struct/alias // an error for invalid usage `Pair` / `cell` is also here - static TypePtr instantiate_generic_type_or_fire(FunctionPtr cur_f, SrcLocation loc, TypePtr type_to_instantiate, std::vector&& type_arguments) { + static TypePtr instantiate_generic_type_or_fire(FunctionPtr cur_f, SrcRange range, TypePtr type_to_instantiate, std::vector&& type_arguments) { // example: `type WrapperAlias = Wrapper`, we are at `Wrapper`, type_arguments = `` // they contain generics, so the struct is not ready to be instantiated yet bool is_still_generic = false; @@ -290,7 +292,7 @@ class TypeNodesVisitorResolver { if (const TypeDataStruct* t_struct = type_to_instantiate->try_as(); t_struct && t_struct->struct_ref->is_generic_struct()) { StructPtr struct_ref = t_struct->struct_ref; if (struct_ref->genericTs->size() != static_cast(type_arguments.size())) { - fire(cur_f, loc, "struct `" + struct_ref->as_human_readable() + "` expects " + std::to_string(struct_ref->genericTs->size()) + " type arguments, but " + std::to_string(type_arguments.size()) + " provided"); + fire(cur_f, range, "struct `" + struct_ref->as_human_readable() + "` expects " + std::to_string(struct_ref->genericTs->size()) + " type arguments, but " + std::to_string(type_arguments.size()) + " provided"); } if (is_still_generic) { return TypeDataGenericTypeWithTs::create(struct_ref, nullptr, std::move(type_arguments)); @@ -300,7 +302,7 @@ class TypeNodesVisitorResolver { if (const TypeDataAlias* t_alias = type_to_instantiate->try_as(); t_alias && t_alias->alias_ref->is_generic_alias()) { AliasDefPtr alias_ref = t_alias->alias_ref; if (alias_ref->genericTs->size() != static_cast(type_arguments.size())) { - fire(cur_f, loc, "type `" + alias_ref->as_human_readable() + "` expects " + std::to_string(alias_ref->genericTs->size()) + " type arguments, but " + std::to_string(type_arguments.size()) + " provided"); + fire(cur_f, range, "type `" + alias_ref->as_human_readable() + "` expects " + std::to_string(alias_ref->genericTs->size()) + " type arguments, but " + std::to_string(type_arguments.size()) + " provided"); } if (is_still_generic) { return TypeDataGenericTypeWithTs::create(nullptr, alias_ref, std::move(type_arguments)); @@ -309,21 +311,21 @@ class TypeNodesVisitorResolver { } if (const TypeDataMapKV* t_map = type_to_instantiate->try_as(); t_map && t_map->TKey->try_as()) { if (type_arguments.size() != 2) { - fire(cur_f, loc, "type `map` expects 2 type arguments, but " + std::to_string(type_arguments.size()) + " provided"); + fire(cur_f, range, "type `map` expects 2 type arguments, but " + std::to_string(type_arguments.size()) + " provided"); } return TypeDataMapKV::create(type_arguments[0], type_arguments[1]); } if (const TypeDataGenericT* asT = type_to_instantiate->try_as()) { - fire_error_unknown_type_name(cur_f, loc, asT->nameT); + fire_unknown_type_name(cur_f, range, asT->nameT); } // `User` / `cell` - fire(cur_f, loc, "type `" + type_to_instantiate->as_human_readable() + "` is not generic"); + fire(cur_f, range, "type `" + type_to_instantiate->as_human_readable() + "` is not generic"); } - static void validate_resulting_union_type(const TypeDataUnion* t_union, FunctionPtr cur_f, SrcLocation loc) { + static void validate_resulting_union_type(const TypeDataUnion* t_union, FunctionPtr cur_f, SrcRange range) { for (TypePtr variant : t_union->variants) { if (variant == TypeDataVoid::create() || variant == TypeDataNever::create()) { - fire_void_type_not_allowed_inside_union(cur_f, loc, variant); + fire_void_type_not_allowed_inside_union(cur_f, range, variant); } } } @@ -376,7 +378,7 @@ class TypeNodesVisitorResolver { // prevent recursion like `type A = B; type B = A` (we can't create TypeDataAlias without a resolved underlying type) bool contains = std::find(called_stack.begin(), called_stack.end(), alias_ref) != called_stack.end(); if (contains) { - throw ParseError(alias_ref->loc, "type `" + alias_ref->name + "` circularly references itself"); + fire(alias_ref->ident_anchor, "type `" + alias_ref->name + "` circularly references itself"); } if (auto v_genericsT_list = alias_ref->ast_root->as()->genericsT_list) { @@ -433,7 +435,7 @@ class TypeNodesVisitorResolver { return prevT.nameT == v_item->nameT; }); if (it_existing != itemsT.end()) { - v_item->error("duplicate generic parameter `" + static_cast(v_item->nameT) + "`"); + fire(v_item, "duplicate generic parameter `" + static_cast(v_item->nameT) + "`"); } TypePtr default_type = nullptr; if (v_list->get_item(i)->default_type_node) { @@ -492,7 +494,7 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { // also, `someFn.prop` doesn't make any sense, show "unknown type"; it also forces `address.staticMethod()` to work if (obj_ref->sym == nullptr || obj_ref->sym->try_as()) { std::string_view obj_type_name = obj_ref->get_identifier()->name; - AnyTypeV obj_type_node = createV(obj_ref->loc, obj_type_name); + AnyTypeV obj_type_node = createV(obj_ref->get_identifier()->range, obj_type_name); if (obj_ref->has_instantiationTs()) { // Container.create std::vector inner_and_args; inner_and_args.reserve(1 + obj_ref->get_instantiationTs()->size()); @@ -500,10 +502,10 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { for (int i = 0; i < obj_ref->get_instantiationTs()->size(); ++i) { inner_and_args.push_back(obj_ref->get_instantiationTs()->get_item(i)->type_node); } - obj_type_node = createV(obj_ref->loc, std::move(inner_and_args)); + obj_type_node = createV(obj_ref->range, std::move(inner_and_args)); } TypePtr type_as_reference = finalize_type_node(obj_type_node); - const Symbol* type_as_symbol = new TypeReferenceUsedAsSymbol(static_cast(obj_type_name), obj_ref->loc, type_as_reference); + const Symbol* type_as_symbol = new TypeReferenceUsedAsSymbol(static_cast(obj_type_name), obj_ref->get_identifier(), type_as_reference); obj_ref->mutate()->assign_sym(type_as_symbol); } } @@ -537,7 +539,7 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { public: bool should_visit_function(FunctionPtr fun_ref) override { - return !fun_ref->is_builtin_function(); + return !fun_ref->is_builtin(); } void start_visiting_function(FunctionPtr fun_ref, V v) override { @@ -636,7 +638,7 @@ class InfiniteStructSizeDetector { bool contains = std::find(called_stack.begin(), called_stack.end(), struct_ref) != called_stack.end(); if (contains) { - throw ParseError(struct_ref->loc, "struct `" + struct_ref->name + "` size is infinity due to recursive fields"); + fire(struct_ref->ident_anchor, "struct `" + struct_ref->name + "` size is infinity due to recursive fields"); } called_stack.push_back(struct_ref); diff --git a/tolk/pipe-transform-on-message.cpp b/tolk/pipe-transform-on-message.cpp index 68aff5060e..4ce2ecbc92 100644 --- a/tolk/pipe-transform-on-message.cpp +++ b/tolk/pipe-transform-on-message.cpp @@ -57,14 +57,14 @@ struct TransformOnInternalMessageReplacer final : ASTReplacerInFunctionBody { static void validate_onBouncedMessage(FunctionPtr f) { if (f->inferred_return_type != TypeDataVoid::create() && f->inferred_return_type != TypeDataNever::create()) { - fire(f, f->loc, "`onBouncedMessage` should return `void`"); + fire(f, f->ident_anchor, "`onBouncedMessage` should return `void`"); } if (f->get_num_params() != 1) { - fire(f, f->loc, "`onBouncedMessage` should have one parameter `InMessageBounced`"); + fire(f, f->ident_anchor, "`onBouncedMessage` should have one parameter `InMessageBounced`"); } const auto* t_struct = f->get_param(0).declared_type->try_as(); if (!t_struct || t_struct->struct_ref->name != "InMessageBounced") { - fire(f, f->loc, "`onBouncedMessage` should have one parameter `InMessageBounced`"); + fire(f, f->ident_anchor, "`onBouncedMessage` should have one parameter `InMessageBounced`"); } } @@ -72,7 +72,7 @@ struct TransformOnInternalMessageReplacer final : ASTReplacerInFunctionBody { AnyExprV replace(V v) override { // don't allow `var v = in` or passing `in` to another function (only `in.someField` is allowed) if (v->sym == param_ref) { - fire(cur_f, v->loc, "using `" + param_ref->name + "` as an object is prohibited, because `InMessage` is a built-in struct, its fields are mapped to TVM instructions\nhint: use `" + param_ref->name + ".senderAddress` and other fields directly"); + fire(cur_f, v, "using `" + param_ref->name + "` as an object is prohibited, because `InMessage` is a built-in struct, its fields are mapped to TVM instructions\nhint: use `" + param_ref->name + ".senderAddress` and other fields directly"); } return parent::replace(v); } @@ -81,11 +81,11 @@ struct TransformOnInternalMessageReplacer final : ASTReplacerInFunctionBody { // replace `in.senderAddress` / `in.valueCoins` with an aux vertex if (v->get_obj()->kind == ast_reference && v->get_obj()->as()->sym == param_ref) { if (v->is_lvalue && v->get_field_name() != "body" && v->get_field_name() != "bouncedBody") { - fire(cur_f, v->loc, "modifying an immutable variable\nhint: fields of InMessage can be used for reading only"); + fire(cur_f, v, "modifying an immutable variable\nhint: fields of InMessage can be used for reading only"); } ASTAuxData* aux_getField = new AuxData_OnInternalMessage_getField(cur_f, v->get_field_name()); - return createV(v->loc, v, aux_getField, v->inferred_type); + return createV(v, aux_getField, v->inferred_type); } return parent::replace(v); @@ -107,7 +107,7 @@ struct TransformOnInternalMessageReplacer final : ASTReplacerInFunctionBody { parent::replace(v_function->get_body()); std::vector new_parameters; - new_parameters.emplace_back("in.body", fun_ref->loc, TypeDataSlice::create(), nullptr, 0, 0); + new_parameters.emplace_back("in.body", fun_ref->ident_anchor, TypeDataSlice::create(), nullptr, 0, 0); fun_ref->mutate()->parameters = std::move(new_parameters); } }; diff --git a/tolk/send-message-api.cpp b/tolk/send-message-api.cpp index 67ed082405..92c8ba404b 100644 --- a/tolk/send-message-api.cpp +++ b/tolk/send-message-api.cpp @@ -20,8 +20,8 @@ namespace tolk { -std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc); -std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc); +std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, AnyV origin, const char* debug_desc); +std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, AnyV origin); static bool is_type_UnsafeBodyNoRef_T(TypePtr bodyT) { if (const TypeDataStruct* t_struct = bodyT->unwrap_alias()->try_as()) { @@ -33,13 +33,13 @@ static bool is_type_UnsafeBodyNoRef_T(TypePtr bodyT) { } // calculate `addrHash &= mask` where mask = `(1 << (256 - SHARD_DEPTH)) - 1` -static void append_bitwise_and_shard_mask(CodeBlob& code, SrcLocation loc, var_idx_t ir_addr_hash, var_idx_t ir_shard_depth) { - var_idx_t ir_one = code.create_int(loc, 1, "(one)"); - std::vector ir_mask = code.create_tmp_var(TypeDataInt::create(), loc, "(mask)"); - code.emplace_back(loc, Op::_Call, ir_mask, std::vector{code.create_int(loc, 256, ""), ir_shard_depth}, lookup_function("_-_")); - code.emplace_back(loc, Op::_Call, ir_mask, std::vector{ir_one, ir_mask[0]}, lookup_function("_<<_")); - code.emplace_back(loc, Op::_Call, ir_mask, std::vector{ir_mask[0], ir_one}, lookup_function("_-_")); - code.emplace_back(loc, Op::_Call, std::vector{ir_addr_hash}, std::vector{ir_addr_hash, ir_mask[0]}, lookup_function("_&_")); +static void append_bitwise_and_shard_mask(CodeBlob& code, AnyV origin, var_idx_t ir_addr_hash, var_idx_t ir_shard_depth) { + var_idx_t ir_one = code.create_int(origin, 1, "(one)"); + std::vector ir_mask = code.create_tmp_var(TypeDataInt::create(), origin, "(mask)"); + code.emplace_back(origin, Op::_Call, ir_mask, std::vector{code.create_int(origin, 256, ""), ir_shard_depth}, lookup_function("_-_")); + code.emplace_back(origin, Op::_Call, ir_mask, std::vector{ir_one, ir_mask[0]}, lookup_function("_<<_")); + code.emplace_back(origin, Op::_Call, ir_mask, std::vector{ir_mask[0], ir_one}, lookup_function("_-_")); + code.emplace_back(origin, Op::_Call, std::vector{ir_addr_hash}, std::vector{ir_addr_hash, ir_mask[0]}, lookup_function("_&_")); } // struct AutoDeployAddress { workchain: int8; stateInit: ContractState | cell; toShard: AddressShardingOptions?; } @@ -52,7 +52,7 @@ struct IR_AutoDeployAddress { stateInitCode, stateInitData, stateInitCell, // stateInit ir_shardDepth, ir_closeTo; // toShard - IR_AutoDeployAddress(CodeBlob& code, SrcLocation loc, const std::vector& ir_vars) { + IR_AutoDeployAddress(CodeBlob& code, AnyV origin, const std::vector& ir_vars) { StructPtr s_AutoDeployAddress = lookup_global_symbol("AutoDeployAddress")->try_as(); t_stateInit = s_AutoDeployAddress->find_field("stateInit")->declared_type->try_as(); t_toShard = s_AutoDeployAddress->find_field("toShard")->declared_type->try_as(); @@ -63,30 +63,30 @@ struct IR_AutoDeployAddress { workchain = ir_vars[0]; ir_stateInitField = std::vector(ir_vars.begin() + 1, ir_vars.begin() + 1 + 3); - std::vector ir_ContractState = transition_to_target_type(std::vector(ir_stateInitField), code, t_stateInit, t_stateInit->variants[0], loc); + std::vector ir_ContractState = transition_to_target_type(std::vector(ir_stateInitField), code, t_stateInit, t_stateInit->variants[0], origin); stateInitCode = ir_ContractState[0]; stateInitData = ir_ContractState[1]; - stateInitCell = transition_to_target_type(std::vector(ir_stateInitField), code, t_stateInit, t_stateInit->variants[1], loc)[0]; + stateInitCell = transition_to_target_type(std::vector(ir_stateInitField), code, t_stateInit, t_stateInit->variants[1], origin)[0]; ir_toShardField = std::vector(ir_vars.begin() + 1 + 3, ir_vars.begin() + 1 + 3 + 3); - std::vector ir_AddressSharding = transition_to_target_type(std::vector(ir_toShardField), code, t_toShard, t_toShard->or_null, loc); + std::vector ir_AddressSharding = transition_to_target_type(std::vector(ir_toShardField), code, t_toShard, t_toShard->or_null, origin); ir_shardDepth = ir_AddressSharding[0]; ir_closeTo = ir_AddressSharding[1]; } // generate IR vars "stateInit is ContractState" - std::vector is_ContractState(CodeBlob& code, SrcLocation loc) const { - return pre_compile_is_type(code, t_stateInit, t_stateInit->variants[0], ir_stateInitField, loc, "(is-ContractState)"); + std::vector is_ContractState(CodeBlob& code, AnyV origin) const { + return pre_compile_is_type(code, t_stateInit, t_stateInit->variants[0], ir_stateInitField, origin, "(is-ContractState)"); } // generate IR vars "toShard is not null" - std::vector is_AddressSharding(CodeBlob& code, SrcLocation loc) const { - return pre_compile_is_type(code, t_toShard, t_toShard->or_null, ir_toShardField, loc, "(is-AddressSharding)"); + std::vector is_AddressSharding(CodeBlob& code, AnyV origin) const { + return pre_compile_is_type(code, t_toShard, t_toShard->or_null, ir_toShardField, origin, "(is-AddressSharding)"); } }; // fun createMessage(options: CreateMessageOptions): OutMessage -std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& ir_options) { +std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& ir_options) { TypePtr bodyT = called_f->substitutedTs->typeT_at(0); StructPtr s_Options = lookup_global_symbol("CreateMessageOptions")->try_as(); StructPtr s_AutoDeployAddress = lookup_global_symbol("AutoDeployAddress")->try_as(); @@ -113,17 +113,17 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co tolk_assert(offset == static_cast(rvect.size())); // since TVM 12, field `bounce` is a union: `bounce: BounceMode | bool` - std::vector ir_bounce_is_bool = pre_compile_is_type(code, t_bounce, TypeDataBool::create(), ir_bounce, loc, "(bounce-is-bool)"); + std::vector ir_bounce_is_bool = pre_compile_is_type(code, t_bounce, TypeDataBool::create(), ir_bounce, origin, "(bounce-is-bool)"); // field `dest` is `dest: address | AutoDeployAddress | (int8, uint256) | builder`; // struct AutoDeployAddress { workchain: int8; stateInit: ContractState | cell; toShard: AddressShardingOptions?; } // struct ContractState { code: cell; data: cell; } // struct AddressShardingOptions { fixedPrefixLength: uint5; closeTo: address; } - std::vector ir_dest_is_address = pre_compile_is_type(code, t_dest, TypeDataAddress::create(), ir_dest, loc, "(is-address)"); - std::vector ir_dest_is_AutoDeploy = pre_compile_is_type(code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), ir_dest, loc, "(is-auto)"); - std::vector ir_dest_is_builder = pre_compile_is_type(code, t_dest, TypeDataBuilder::create(), ir_dest, loc, "(is-builder)"); - std::vector ir_dest_AutoDeployAddress = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), loc); - IR_AutoDeployAddress ir_dest_ad(code, loc, ir_dest_AutoDeployAddress); + std::vector ir_dest_is_address = pre_compile_is_type(code, t_dest, TypeDataAddress::create(), ir_dest, origin, "(is-address)"); + std::vector ir_dest_is_AutoDeploy = pre_compile_is_type(code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), ir_dest, origin, "(is-auto)"); + std::vector ir_dest_is_builder = pre_compile_is_type(code, t_dest, TypeDataBuilder::create(), ir_dest, origin, "(is-builder)"); + std::vector ir_dest_AutoDeployAddress = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), origin); + IR_AutoDeployAddress ir_dest_ad(code, origin, ir_dest_AutoDeployAddress); FunctionPtr f_beginCell = lookup_function("beginCell"); FunctionPtr f_endCell = lookup_function("builder.endCell"); @@ -143,95 +143,95 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co // if we need to store body ref, convert it to a cell here, before creating a builder for the message; // it's more optimal, since the `body` field is the topmost at the stack if (body_store_as_ref && !body_already_ref) { - std::vector ir_ref_builder = code.create_var(TypeDataBuilder::create(), loc, "refb"); - code.emplace_back(loc, Op::_Call, ir_ref_builder, std::vector{}, f_beginCell); - PackContext ref_ctx(code, loc, ir_ref_builder, create_default_PackOptions(code, loc)); + std::vector ir_ref_builder = code.create_var(TypeDataBuilder::create(), origin, "refb"); + code.emplace_back(origin, Op::_Call, ir_ref_builder, std::vector{}, f_beginCell); + PackContext ref_ctx(code, origin, ir_ref_builder, create_default_PackOptions(code, origin)); ref_ctx.generate_pack_any(bodyT, std::move(ir_body)); - std::vector ir_ref_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(ref-cell)"); - code.emplace_back(loc, Op::_Call, ir_ref_cell, std::move(ir_ref_builder), f_endCell); + std::vector ir_ref_cell = code.create_tmp_var(TypeDataCell::create(), origin, "(ref-cell)"); + code.emplace_back(origin, Op::_Call, ir_ref_cell, std::move(ir_ref_builder), f_endCell); ir_body = std::move(ir_ref_cell); } - std::vector ir_builder = code.create_var(TypeDataSlice::create(), loc, "b"); - code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, f_beginCell); - PackContext ctx(code, loc, ir_builder, create_default_PackOptions(code, loc)); - var_idx_t ir_zero = code.create_int(loc, 0, "(zero)"); - var_idx_t ir_one = code.create_int(loc, 1, "(one)"); + std::vector ir_builder = code.create_var(TypeDataSlice::create(), origin, "b"); + code.emplace_back(origin, Op::_Call, ir_builder, std::vector{}, f_beginCell); + PackContext ctx(code, origin, ir_builder, create_default_PackOptions(code, origin)); + var_idx_t ir_zero = code.create_int(origin, 0, "(zero)"); + var_idx_t ir_one = code.create_int(origin, 1, "(one)"); // '0' prefix int_msg_info ctx.storeUint(ir_zero, 1); // fill `ihr_disabled:Bool` always 1 ctx.storeUint(ir_one, 1); // fill `bounce:Bool` from p.bounce (if it's constant (most likely), it will be concatenated with prev and next) - Op& if_oldBounceFormat = code.emplace_back(loc, Op::_If, ir_bounce_is_bool); + Op& if_oldBounceFormat = code.emplace_back(origin, Op::_If, ir_bounce_is_bool); { // input is `bounce: true` or false (old bounce mode), so 0-th slot is 0 or -1 code.push_set_cur(if_oldBounceFormat.block0); ctx.storeBool(ir_bounce[0]); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { // input is `bounce: BounceMode.*` (enum), then write 0 for `NoBounce=0` or 1 otherwise code.push_set_cur(if_oldBounceFormat.block1); - std::vector ir_not_NoBounce = code.create_tmp_var(TypeDataInt::create(), loc, "(not-eq-NoBounce)"); - code.emplace_back(loc, Op::_Call, ir_not_NoBounce, std::vector{ir_bounce[0], ir_zero}, lookup_function("_!=_")); + std::vector ir_not_NoBounce = code.create_tmp_var(TypeDataInt::create(), origin, "(not-eq-NoBounce)"); + code.emplace_back(origin, Op::_Call, ir_not_NoBounce, std::vector{ir_bounce[0], ir_zero}, lookup_function("_!=_")); ctx.storeBool(ir_not_NoBounce[0]); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } // fill `bounced:Bool` + `src:MsgAddress` 00 ctx.storeUint(ir_zero, 1 + 2); // fill `dest:MsgAddressInt` from p.dest (complex union) - Op& if_address = code.emplace_back(loc, Op::_If, ir_dest_is_address); + Op& if_address = code.emplace_back(origin, Op::_If, ir_dest_is_address); { // input is `dest: someAddress` code.push_set_cur(if_address.block0); - std::vector ir_dest_address = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataAddress::create(), loc); + std::vector ir_dest_address = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataAddress::create(), origin); ctx.storeAddress(ir_dest_address[0]); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { code.push_set_cur(if_address.block1); - Op& if_AutoDeploy = code.emplace_back(loc, Op::_If, ir_dest_is_AutoDeploy); + Op& if_AutoDeploy = code.emplace_back(origin, Op::_If, ir_dest_is_AutoDeploy); { // input is `dest: { workchain, stateInit, [toShard] }`; // then calculate hash equal to StateInit cell would be and fill "addr_std$10 + 0 anycast + workchain + hash"; // and, if toShard, take first D bits from dest.toShard.closeTo and mix with 256-D bits of hash code.push_set_cur(if_AutoDeploy.block0); - ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); // addr_std$10 + 0 anycast + ctx.storeUint(code.create_int(origin, 0b100, "(addr-prefix)"), 3); // addr_std$10 + 0 anycast ctx.storeInt(ir_dest_ad.workchain, 8); - std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); - Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_dest_ad.is_ContractState(code, loc)); + std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), origin, "(addr-hash)"); + Op& if_ContractState = code.emplace_back(origin, Op::_If, ir_dest_ad.is_ContractState(code, origin)); { // input is `dest: { ... stateInit: { code, data } }` code.push_set_cur(if_ContractState.block0); - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding(code, loc)); + Op& if_sharded = code.emplace_back(origin, Op::_If, ir_dest_ad.is_AddressSharding(code, origin)); { // input is `dest: { ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } }; // then stateInitHash = (hash of StateInit = 0b1(depth)0110 (prefix + code + data)) code.push_set_cur(if_sharded.block0); std::vector args = { ir_dest_ad.ir_shardDepth, ir_dest_ad.stateInitCode, ir_dest_ad.stateInitData }; - code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashPrefixCodeData")); - code.close_pop_cur(loc); + code.emplace_back(origin, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashPrefixCodeData")); + code.close_pop_cur(origin); } { // input is: `dest: { ... stateInit: { code, data } }` (toShard is null); // then hash = (hash of StateInit = 0b00110 (only code + data)) code.push_set_cur(if_sharded.block1); std::vector args = { ir_dest_ad.stateInitCode, ir_dest_ad.stateInitData }; - code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashCodeData")); - code.close_pop_cur(loc); + code.emplace_back(origin, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashCodeData")); + code.close_pop_cur(origin); } - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { // input is `dest: { ... stateInit: cell }` code.push_set_cur(if_ContractState.block1); std::vector args = { ir_dest_ad.stateInitCell }; - code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("cell.hash")); - code.close_pop_cur(loc); + code.emplace_back(origin, Op::_Call, ir_hash, std::move(args), lookup_function("cell.hash")); + code.close_pop_cur(origin); } - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding(code, loc)); + Op& if_sharded = code.emplace_back(origin, Op::_If, ir_dest_ad.is_AddressSharding(code, origin)); { // input is `dest: { ... toShard: { fixedPrefixLength, closeTo } }` // we already calculated stateInitHash (ir_hash): either cell.hash() or based on prefix+code+data; @@ -243,102 +243,102 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co // | hash (result) | 01010101...yyy | // remember, that closeTo is addr_std$10 + 0 + workchain + xxx...xxx, so skip 11 bits and read 8 code.push_set_cur(if_sharded.block0); - append_bitwise_and_shard_mask(code, loc, ir_hash[0], ir_dest_ad.ir_shardDepth); - std::vector ir_lowerD = code.create_tmp_var(TypeDataInt::create(), loc, "(lowerD)"); - code.emplace_back(loc, Op::_Call, ir_lowerD, std::vector{code.create_int(loc, 256, ""), ir_dest_ad.ir_shardDepth}, lookup_function("_-_")); - std::vector ir_shardPrefix = code.create_tmp_var(TypeDataSlice::create(), loc, "(shardPrefix)"); - std::vector args_subslice = { ir_dest_ad.ir_closeTo, code.create_int(loc, 3+8, ""), ir_dest_ad.ir_shardDepth }; - code.emplace_back(loc, Op::_Call, ir_shardPrefix, std::move(args_subslice), lookup_function("slice.getMiddleBits")); + append_bitwise_and_shard_mask(code, origin, ir_hash[0], ir_dest_ad.ir_shardDepth); + std::vector ir_lowerD = code.create_tmp_var(TypeDataInt::create(), origin, "(lowerD)"); + code.emplace_back(origin, Op::_Call, ir_lowerD, std::vector{code.create_int(origin, 256, ""), ir_dest_ad.ir_shardDepth}, lookup_function("_-_")); + std::vector ir_shardPrefix = code.create_tmp_var(TypeDataSlice::create(), origin, "(shardPrefix)"); + std::vector args_subslice = { ir_dest_ad.ir_closeTo, code.create_int(origin, 3+8, ""), ir_dest_ad.ir_shardDepth }; + code.emplace_back(origin, Op::_Call, ir_shardPrefix, std::move(args_subslice), lookup_function("slice.getMiddleBits")); ctx.storeSlice(ir_shardPrefix[0]); // first 8 bits of closeTo hash ctx.storeUint_var(ir_hash[0], ir_lowerD[0]); // 248 STU (stateInitHash & mask) - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { // input is `dest: { workchain, stateInit }` (toShard is null); // we already calculated stateInitHash: either cell.hash() or based on code+data code.push_set_cur(if_sharded.block1); ctx.storeUint(ir_hash[0], 256); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { code.push_set_cur(if_AutoDeploy.block1); - Op& if_builder = code.emplace_back(loc, Op::_If, ir_dest_is_builder); + Op& if_builder = code.emplace_back(origin, Op::_If, ir_dest_is_builder); { // input is `dest: someBuilder` code.push_set_cur(if_builder.block0); - std::vector ir_dest_builder = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataBuilder::create(), loc); + std::vector ir_dest_builder = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataBuilder::create(), origin); ctx.storeBuilder(ir_dest_builder[0]); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { // input is `dest: (workchain, hash)` code.push_set_cur(if_builder.block1); - std::vector ir_dest_wc_hash = transition_to_target_type(std::vector(ir_dest), code, t_dest, t_dest->variants[2], loc); - ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); + std::vector ir_dest_wc_hash = transition_to_target_type(std::vector(ir_dest), code, t_dest, t_dest->variants[2], origin); + ctx.storeUint(code.create_int(origin, 0b100, "(addr-prefix)"), 3); ctx.storeInt(ir_dest_wc_hash[0], 8); // most likely, it's 0 (basechain), will be merged with above ctx.storeUint(ir_dest_wc_hash[1], 256); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } - code.close_pop_cur(loc); + code.close_pop_cur(origin); } - code.close_pop_cur(loc); + code.close_pop_cur(origin); } // fill `value:CurrencyCollection` from p.value `coins | (coins, dict)` - std::vector ir_is_coins = pre_compile_is_type(code, t_value, TypeDataCoins::create(), ir_value, loc, "(is-coins)"); - Op& if_coins = code.emplace_back(loc, Op::_If, ir_is_coins); + std::vector ir_is_coins = pre_compile_is_type(code, t_value, TypeDataCoins::create(), ir_value, origin, "(is-coins)"); + Op& if_coins = code.emplace_back(origin, Op::_If, ir_is_coins); { code.push_set_cur(if_coins.block0); - std::vector ir_coins = transition_to_target_type(std::vector(ir_value), code, t_value, TypeDataCoins::create(), loc); + std::vector ir_coins = transition_to_target_type(std::vector(ir_value), code, t_value, TypeDataCoins::create(), origin); ctx.storeCoins(ir_coins[0]); ctx.storeUint(ir_zero, 1); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { code.push_set_cur(if_coins.block1); - std::vector ir_coins_dict = transition_to_target_type(std::move(ir_value), code, t_value, t_value->variants[1], loc); + std::vector ir_coins_dict = transition_to_target_type(std::move(ir_value), code, t_value, t_value->variants[1], origin); ctx.storeCoins(ir_coins_dict[0]); ctx.storeMaybeRef(ir_coins_dict[1]); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } // fill `extra_flags:Grams` (formerly `ihr_fee` always 0, now renamed and used for "new bounce format" only) - Op& if_oldBounceFormat2 = code.emplace_back(loc, Op::_If, ir_bounce_is_bool); + Op& if_oldBounceFormat2 = code.emplace_back(origin, Op::_If, ir_bounce_is_bool); { code.push_set_cur(if_oldBounceFormat2.block0); ctx.storeCoins(ir_zero); // extra_flags = 0 - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { code.push_set_cur(if_oldBounceFormat2.block1); - std::vector ir_eq_RichBounce = code.create_tmp_var(TypeDataInt::create(), loc, "(eq-RichBounce)"); - code.emplace_back(loc, Op::_Call, ir_eq_RichBounce, std::vector{ir_bounce[0], code.create_int(loc, 2, "(enum-RichBounce)")}, lookup_function("_==_")); - Op& if_RichBounce = code.emplace_back(loc, Op::_If, ir_eq_RichBounce); + std::vector ir_eq_RichBounce = code.create_tmp_var(TypeDataInt::create(), origin, "(eq-RichBounce)"); + code.emplace_back(origin, Op::_Call, ir_eq_RichBounce, std::vector{ir_bounce[0], code.create_int(origin, 2, "(enum-RichBounce)")}, lookup_function("_==_")); + Op& if_RichBounce = code.emplace_back(origin, Op::_If, ir_eq_RichBounce); { code.push_set_cur(if_RichBounce.block0); - ctx.storeCoins(code.create_int(loc, 3, "(extra-flags-3)")); - code.close_pop_cur(loc); + ctx.storeCoins(code.create_int(origin, 3, "(extra-flags-3)")); + code.close_pop_cur(origin); } { code.push_set_cur(if_RichBounce.block1); - std::vector ir_eq_RichBounceRoot = code.create_tmp_var(TypeDataInt::create(), loc, "(eq-ir_eq_RichBounceRoot)"); - code.emplace_back(loc, Op::_Call, ir_eq_RichBounceRoot, std::vector{ir_bounce[0], code.create_int(loc, 3, "(enum-RichBounceRoot)")}, lookup_function("_==_")); - Op& if_RichBounceRoot = code.emplace_back(loc, Op::_If, ir_eq_RichBounceRoot); + std::vector ir_eq_RichBounceRoot = code.create_tmp_var(TypeDataInt::create(), origin, "(eq-ir_eq_RichBounceRoot)"); + code.emplace_back(origin, Op::_Call, ir_eq_RichBounceRoot, std::vector{ir_bounce[0], code.create_int(origin, 3, "(enum-RichBounceRoot)")}, lookup_function("_==_")); + Op& if_RichBounceRoot = code.emplace_back(origin, Op::_If, ir_eq_RichBounceRoot); { code.push_set_cur(if_RichBounceRoot.block0); - ctx.storeCoins(code.create_int(loc, 1, "(extra-flags-1)")); - code.close_pop_cur(loc); + ctx.storeCoins(code.create_int(origin, 1, "(extra-flags-1)")); + code.close_pop_cur(origin); } { code.push_set_cur(if_RichBounceRoot.block1); ctx.storeCoins(ir_zero); // extra_flags = 0 - code.close_pop_cur(loc); + code.close_pop_cur(origin); } - code.close_pop_cur(loc); + code.close_pop_cur(origin); } - code.close_pop_cur(loc); + code.close_pop_cur(origin); } // tail of CommonMsgInfoRelaxed: 4*0 fwd_fee + 64*0 created_lt + 32*0 created_at @@ -347,51 +347,51 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co // fill `init: (Maybe (Either StateInit ^StateInit))` // it's present only if p.dest contains StateInit // also fill the either bit of `body: (Either X ^X)` - Op& if_no_init = code.emplace_back(loc, Op::_If, ir_dest_is_AutoDeploy); + Op& if_no_init = code.emplace_back(origin, Op::_If, ir_dest_is_AutoDeploy); { // when it's known at compile-time (always in practice), this `if` is simplified, and bits join with above code.push_set_cur(if_no_init.block1); ctx.storeUint(body_store_as_ref ? ir_one : ir_zero, 1 + 1); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { code.push_set_cur(if_no_init.block0); - Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_dest_ad.is_ContractState(code, loc)); + Op& if_ContractState = code.emplace_back(origin, Op::_If, ir_dest_ad.is_ContractState(code, origin)); { // input is `dest: { ... stateInit: { code, data } }` and need to compose TL/B StateInit; // it's either just code+data OR (if `toShard: { ... }` is set) fixedPrefixLength+code+data code.push_set_cur(if_ContractState.block0); - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding(code, loc)); + Op& if_sharded = code.emplace_back(origin, Op::_If, ir_dest_ad.is_AddressSharding(code, origin)); { // 1 (maybe true) + 0 (either left) + 1 (maybe true of StateInit) + fixedPrefixLength + 0110 + body ref or not code.push_set_cur(if_sharded.block0); - ctx.storeUint(code.create_int(loc, 0b101, ""), 1 + 1 + 1); + ctx.storeUint(code.create_int(origin, 0b101, ""), 1 + 1 + 1); ctx.storeUint(ir_dest_ad.ir_shardDepth, 5); // fixedPrefixLength (shard depth) - ctx.storeUint(code.create_int(loc, 0b01100 + body_store_as_ref, ""), 4 + 1); - code.close_pop_cur(loc); + ctx.storeUint(code.create_int(origin, 0b01100 + body_store_as_ref, ""), 4 + 1); + code.close_pop_cur(origin); // also, we used dest.toShard to fill CommonMsgInfoRelaxed.dest.address (with a mask for stateInitHash, see above) } { // 1 (maybe true) + 0 (either left) + 00110 (only code and data from StateInit) + body ref or not code.push_set_cur(if_sharded.block1); - var_idx_t ir_rest_bits = code.create_int(loc, 0b10001100 + body_store_as_ref, "(rest-bits)"); + var_idx_t ir_rest_bits = code.create_int(origin, 0b10001100 + body_store_as_ref, "(rest-bits)"); ctx.storeUint(ir_rest_bits, 1 + 1 + 5 + 1); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } ctx.storeRef(ir_dest_ad.stateInitCode); ctx.storeRef(ir_dest_ad.stateInitData); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { // so, we have `dest: { stateInit: someCell }`, store it as ref // 1 (maybe true) + 1 (either right) + body ref or not code.push_set_cur(if_ContractState.block1); - var_idx_t ir_rest_bits = code.create_int(loc, 0b110 + body_store_as_ref, "(rest-bits)"); + var_idx_t ir_rest_bits = code.create_int(origin, 0b110 + body_store_as_ref, "(rest-bits)"); ctx.storeUint(ir_rest_bits, 1 + 1 + 1); ctx.storeRef(ir_dest_ad.stateInitCell); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } - code.close_pop_cur(loc); + code.close_pop_cur(origin); } // store body; previously, we've calculated whether to store is as a ref or not @@ -404,13 +404,13 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co ctx.generate_pack_any(bodyT, std::move(ir_body)); } - std::vector ir_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(msg-cell)"); - code.emplace_back(loc, Op::_Call, ir_cell, std::move(ir_builder), f_endCell); + std::vector ir_cell = code.create_tmp_var(TypeDataCell::create(), origin, "(msg-cell)"); + code.emplace_back(origin, Op::_Call, ir_cell, std::move(ir_builder), f_endCell); return ir_cell; } // fun createExternalLogMessage(options: CreateExternalLogMessageOptions): OutMessage -std::vector generate_createExternalLogMessage(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_createExternalLogMessage(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr bodyT = called_f->substitutedTs->typeT_at(0); StructPtr s_Options = lookup_global_symbol("CreateExternalLogMessageOptions")->try_as(); StructPtr s_ExtOutLogBucket = lookup_global_symbol("ExtOutLogBucket")->try_as(); @@ -434,13 +434,13 @@ std::vector generate_createExternalLogMessage(FunctionPtr called_f, C // field `dest` is `dest: address | builder | ExtOutLogBucket`; // struct ExtOutLogBucket { topic: uint248 | bits248; } - std::vector ir_dest_is_address = pre_compile_is_type(code, t_dest, TypeDataAddress::create(), ir_dest, loc, "(is-address)"); - std::vector ir_dest_is_builder = pre_compile_is_type(code, t_dest, TypeDataBuilder::create(), ir_dest, loc, "(is-builder)"); - std::vector ir_dest_ExtOutLogBucket = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataStruct::create(s_ExtOutLogBucket), loc); + std::vector ir_dest_is_address = pre_compile_is_type(code, t_dest, TypeDataAddress::create(), ir_dest, origin, "(is-address)"); + std::vector ir_dest_is_builder = pre_compile_is_type(code, t_dest, TypeDataBuilder::create(), ir_dest, origin, "(is-builder)"); + std::vector ir_dest_ExtOutLogBucket = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataStruct::create(s_ExtOutLogBucket), origin); // dest.topic (it's the only field in a struct) std::vector ir_dest_topic = std::move(ir_dest_ExtOutLogBucket); - std::vector ir_options = create_default_PackOptions(code, loc); + std::vector ir_options = create_default_PackOptions(code, origin); FunctionPtr f_beginCell = lookup_function("beginCell"); FunctionPtr f_endCell = lookup_function("builder.endCell"); @@ -459,68 +459,68 @@ std::vector generate_createExternalLogMessage(FunctionPtr called_f, C // same as for createMessage: `body` field is the topmost at the stack, convert it to a cell before creating a builder if (body_store_as_ref && !body_already_ref) { - std::vector ir_ref_builder = code.create_var(TypeDataBuilder::create(), loc, "refb"); - code.emplace_back(loc, Op::_Call, ir_ref_builder, std::vector{}, f_beginCell); - PackContext ref_ctx(code, loc, ir_ref_builder, ir_options); + std::vector ir_ref_builder = code.create_var(TypeDataBuilder::create(), origin, "refb"); + code.emplace_back(origin, Op::_Call, ir_ref_builder, std::vector{}, f_beginCell); + PackContext ref_ctx(code, origin, ir_ref_builder, ir_options); ref_ctx.generate_pack_any(bodyT, std::move(ir_body)); - std::vector ir_ref_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(ref-cell)"); - code.emplace_back(loc, Op::_Call, ir_ref_cell, std::move(ir_ref_builder), f_endCell); + std::vector ir_ref_cell = code.create_tmp_var(TypeDataCell::create(), origin, "(ref-cell)"); + code.emplace_back(origin, Op::_Call, ir_ref_cell, std::move(ir_ref_builder), f_endCell); ir_body = std::move(ir_ref_cell); } - std::vector ir_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); - code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, f_beginCell); - PackContext ctx(code, loc, ir_builder, ir_options); - var_idx_t ir_zero = code.create_int(loc, 0, "(zero)"); - var_idx_t ir_one = code.create_int(loc, 1, "(one)"); + std::vector ir_builder = code.create_var(TypeDataBuilder::create(), origin, "b"); + code.emplace_back(origin, Op::_Call, ir_builder, std::vector{}, f_beginCell); + PackContext ctx(code, origin, ir_builder, ir_options); + var_idx_t ir_zero = code.create_int(origin, 0, "(zero)"); + var_idx_t ir_one = code.create_int(origin, 1, "(one)"); // '11' prefix ext_out_msg_info + '00' src - ctx.storeUint(code.create_int(loc, 0b1100, "(out-prefix)"), 4); + ctx.storeUint(code.create_int(origin, 0b1100, "(out-prefix)"), 4); // fill `dest:MsgAddressExt` from p.dest (complex union) - Op& if_address = code.emplace_back(loc, Op::_If, ir_dest_is_address); + Op& if_address = code.emplace_back(origin, Op::_If, ir_dest_is_address); { // input is `dest: someAddress` code.push_set_cur(if_address.block0); - std::vector ir_dest_address = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataAddress::create(), loc); + std::vector ir_dest_address = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataAddress::create(), origin); ctx.storeAddress(ir_dest_address[0]); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { code.push_set_cur(if_address.block1); - Op& if_builder = code.emplace_back(loc, Op::_If, ir_dest_is_builder); + Op& if_builder = code.emplace_back(origin, Op::_If, ir_dest_is_builder); { // input is `dest: someBuilder` code.push_set_cur(if_builder.block0); - std::vector ir_dest_builder = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataBuilder::create(), loc); + std::vector ir_dest_builder = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataBuilder::create(), origin); ctx.storeBuilder(ir_dest_builder[0]); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { // input is `dest: ExtOutLogBucket`; // fill addr_extern$01 + 256 (len 9 bit) + 0x00 (prefix) + 248 bits code.push_set_cur(if_builder.block1); ctx.storeUint(ir_one, 2); // addr_extern$01 - ctx.storeUint(code.create_int(loc, 256, "(addr-len)"), 9); // len:(## 9) = 256 + ctx.storeUint(code.create_int(origin, 256, "(addr-len)"), 9); // len:(## 9) = 256 ctx.storeOpcode(s_ExtOutLogBucket->opcode); - std::vector ir_if_topic_uint = pre_compile_is_type(code, t_topic, t_topic->variants[0], ir_dest_topic, loc, "(topic-is-uint)"); - Op& if_topic_uint = code.emplace_back(loc, Op::_If, ir_if_topic_uint); + std::vector ir_if_topic_uint = pre_compile_is_type(code, t_topic, t_topic->variants[0], ir_dest_topic, origin, "(topic-is-uint)"); + Op& if_topic_uint = code.emplace_back(origin, Op::_If, ir_if_topic_uint); { // input is `dest: ExtOutLogBucket { topic: uint248 }` code.push_set_cur(if_topic_uint.block0); ctx.storeUint(ir_dest_topic[0], 248); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { // input is `dest: ExtOutLogBucket { topic: bits248 }` // for this field, generate runtime check to ensure its length code.push_set_cur(if_topic_uint.block1); ctx.generate_pack_any(t_topic->variants[1], std::vector{ir_dest_topic[0]}); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } - code.close_pop_cur(loc); + code.close_pop_cur(origin); } - code.close_pop_cur(loc); + code.close_pop_cur(origin); } // tail of CommonMsgInfoRelaxed: 64*0 created_lt + 32*0 created_at @@ -541,13 +541,13 @@ std::vector generate_createExternalLogMessage(FunctionPtr called_f, C ctx.generate_pack_any(bodyT, std::move(ir_body)); } - std::vector ir_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(msg-cell)"); - code.emplace_back(loc, Op::_Call, ir_cell, std::move(ir_builder), f_endCell); + std::vector ir_cell = code.create_tmp_var(TypeDataCell::create(), origin, "(msg-cell)"); + code.emplace_back(origin, Op::_Call, ir_cell, std::move(ir_builder), f_endCell); return ir_cell; } // fun address.buildSameAddressInAnotherShard(self, options: AddressShardingOptions): builder -std::vector generate_address_buildInAnotherShard(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_address_buildInAnotherShard(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { std::vector ir_shard_options = args[1]; tolk_assert(ir_shard_options.size() == 2); @@ -561,190 +561,190 @@ std::vector generate_address_buildInAnotherShard(FunctionPtr called_f // - take last 256-D bits of A: we'll have "aa...a" // - concatenate: we'll result in '100' + workchainB + "bbaa...a" - std::vector ir_offsetB = {code.create_int(loc, 3 + 8, "(offset-addrB)")}; - code.emplace_back(loc, Op::_Call, ir_offsetB, std::vector{ir_offsetB[0], ir_shard_options[0]}, lookup_function("_+_")); - std::vector ir_headB = code.create_tmp_var(TypeDataSlice::create(), loc, "(headB)"); - code.emplace_back(loc, Op::_Call, ir_headB, std::vector{ir_shard_options[1], ir_offsetB[0]}, lookup_function("slice.getFirstBits")); + std::vector ir_offsetB = {code.create_int(origin, 3 + 8, "(offset-addrB)")}; + code.emplace_back(origin, Op::_Call, ir_offsetB, std::vector{ir_offsetB[0], ir_shard_options[0]}, lookup_function("_+_")); + std::vector ir_headB = code.create_tmp_var(TypeDataSlice::create(), origin, "(headB)"); + code.emplace_back(origin, Op::_Call, ir_headB, std::vector{ir_shard_options[1], ir_offsetB[0]}, lookup_function("slice.getFirstBits")); - std::vector ir_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); - code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); - code.emplace_back(loc, Op::_Call, ir_builder, std::vector{ir_builder[0], ir_headB[0]}, lookup_function("builder.storeSlice")); + std::vector ir_builder = code.create_var(TypeDataBuilder::create(), origin, "b"); + code.emplace_back(origin, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); + code.emplace_back(origin, Op::_Call, ir_builder, std::vector{ir_builder[0], ir_headB[0]}, lookup_function("builder.storeSlice")); - std::vector ir_restLenA = {code.create_int(loc, 256, "(last-addrA)")}; - code.emplace_back(loc, Op::_Call, ir_restLenA, std::vector{ir_restLenA[0], ir_shard_options[0]}, lookup_function("_-_")); - std::vector ir_tailA = code.create_tmp_var(TypeDataSlice::create(), loc, "(tailA)"); - code.emplace_back(loc, Op::_Call, ir_tailA, std::vector{args[0][0], ir_restLenA[0]}, lookup_function("slice.getLastBits")); - code.emplace_back(loc, Op::_Call, ir_builder, std::vector{ir_builder[0], ir_tailA[0]}, lookup_function("builder.storeSlice")); + std::vector ir_restLenA = {code.create_int(origin, 256, "(last-addrA)")}; + code.emplace_back(origin, Op::_Call, ir_restLenA, std::vector{ir_restLenA[0], ir_shard_options[0]}, lookup_function("_-_")); + std::vector ir_tailA = code.create_tmp_var(TypeDataSlice::create(), origin, "(tailA)"); + code.emplace_back(origin, Op::_Call, ir_tailA, std::vector{args[0][0], ir_restLenA[0]}, lookup_function("slice.getLastBits")); + code.emplace_back(origin, Op::_Call, ir_builder, std::vector{ir_builder[0], ir_tailA[0]}, lookup_function("builder.storeSlice")); return ir_builder; } // fun address.calculateSameAddressInAnotherShard(self, options: AddressShardingOptions): address -std::vector generate_address_calculateInAnotherShard(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { +std::vector generate_address_calculateInAnotherShard(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { // it's "build address" + BTOS (until a deprecated "build" function removed) - std::vector ir_builder = generate_address_buildInAnotherShard(called_f, code, loc, args); - code.emplace_back(loc, Op::_Call, ir_builder, ir_builder, lookup_function("builder.toSlice")); + std::vector ir_builder = generate_address_buildInAnotherShard(called_f, code, origin, args); + code.emplace_back(origin, Op::_Call, ir_builder, ir_builder, lookup_function("builder.toSlice")); return ir_builder; } // fun AutoDeployAddress.buildAddress(self): builder -std::vector generate_AutoDeployAddress_buildAddress(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& ir_options) { - IR_AutoDeployAddress ir_self(code, loc, ir_options[0]); +std::vector generate_AutoDeployAddress_buildAddress(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& ir_options) { + IR_AutoDeployAddress ir_self(code, origin, ir_options[0]); - std::vector ir_builder = code.create_tmp_var(TypeDataSlice::create(), loc, "(addr-b)"); + std::vector ir_builder = code.create_tmp_var(TypeDataSlice::create(), origin, "(addr-b)"); // important! unlike `createMessage()`, we calculate hash and shard prefix BEFORE creating a cell // (for fewer stack manipulations) // calculate stateInitHash = (hash of StateInit cell would be, but without constructing a cell) - std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); - Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_self.is_ContractState(code, loc)); + std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), origin, "(addr-hash)"); + Op& if_ContractState = code.emplace_back(origin, Op::_If, ir_self.is_ContractState(code, origin)); { // called `{ ... stateInit: { code, data } }` code.push_set_cur(if_ContractState.block0); - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding(code, loc)); + Op& if_sharded = code.emplace_back(origin, Op::_If, ir_self.is_AddressSharding(code, origin)); { // called `{ ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } } code.push_set_cur(if_sharded.block0); std::vector args = { ir_self.ir_shardDepth, ir_self.stateInitCode, ir_self.stateInitData }; - code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashPrefixCodeData")); - code.close_pop_cur(loc); + code.emplace_back(origin, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashPrefixCodeData")); + code.close_pop_cur(origin); } { // called `{ ... stateInit: { code, data } }` (toShard is null) code.push_set_cur(if_sharded.block1); std::vector args = { ir_self.stateInitCode, ir_self.stateInitData }; - code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashCodeData")); - code.close_pop_cur(loc); + code.emplace_back(origin, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashCodeData")); + code.close_pop_cur(origin); } - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { // called `{ ... stateInit: cell }` code.push_set_cur(if_ContractState.block1); std::vector args = { ir_self.stateInitCell }; - code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("cell.hash")); - code.close_pop_cur(loc); + code.emplace_back(origin, Op::_Call, ir_hash, std::move(args), lookup_function("cell.hash")); + code.close_pop_cur(origin); } // now, if toShard, perform bitwise calculations with hashes (order on a stack matters) - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding(code, loc)); + Op& if_sharded = code.emplace_back(origin, Op::_If, ir_self.is_AddressSharding(code, origin)); { // called `{ ... toShard: { fixedPrefixLength, closeTo } }` // we already calculated stateInitHash (ir_hash): either cell.hash() or based on prefix+code+data; // keep hash = (last 256-D bits from stateInitHash) = `hash & mask` code.push_set_cur(if_sharded.block0); - append_bitwise_and_shard_mask(code, loc, ir_hash[0], ir_self.ir_shardDepth); - std::vector ir_lowerD = code.create_tmp_var(TypeDataInt::create(), loc, "(lowerD)"); - code.emplace_back(loc, Op::_Call, ir_lowerD, std::vector{code.create_int(loc, 256, ""), ir_self.ir_shardDepth}, lookup_function("_-_")); + append_bitwise_and_shard_mask(code, origin, ir_hash[0], ir_self.ir_shardDepth); + std::vector ir_lowerD = code.create_tmp_var(TypeDataInt::create(), origin, "(lowerD)"); + code.emplace_back(origin, Op::_Call, ir_lowerD, std::vector{code.create_int(origin, 256, ""), ir_self.ir_shardDepth}, lookup_function("_-_")); // calculate shard_prefix = (first D bits from dest.toShard.closeTo) - std::vector ir_shardPrefix = code.create_tmp_var(TypeDataSlice::create(), loc, "(shardPrefix)"); - std::vector args_subslice = { ir_self.ir_closeTo, code.create_int(loc, 3+8, ""), ir_self.ir_shardDepth }; - code.emplace_back(loc, Op::_Call, ir_shardPrefix, std::move(args_subslice), lookup_function("slice.getMiddleBits")); + std::vector ir_shardPrefix = code.create_tmp_var(TypeDataSlice::create(), origin, "(shardPrefix)"); + std::vector args_subslice = { ir_self.ir_closeTo, code.create_int(origin, 3+8, ""), ir_self.ir_shardDepth }; + code.emplace_back(origin, Op::_Call, ir_shardPrefix, std::move(args_subslice), lookup_function("slice.getMiddleBits")); // on a stack: stateInitHash & mask; shard prefix; create a cell and store all - code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); - PackContext ctx(code, loc, ir_builder, create_default_PackOptions(code, loc)); - ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); // addr_std$10 + 0 anycast + code.emplace_back(origin, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); + PackContext ctx(code, origin, ir_builder, create_default_PackOptions(code, origin)); + ctx.storeUint(code.create_int(origin, 0b100, "(addr-prefix)"), 3); // addr_std$10 + 0 anycast ctx.storeInt(ir_self.workchain, 8); ctx.storeSlice(ir_shardPrefix[0]); // first 8 bits of closeTo hash ctx.storeUint_var(ir_hash[0], ir_lowerD[0]); // 248 STU (stateInitHash & mask) - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { // called `{ workchain, stateInit }` (toShard is null); // on a stack: hash (already calculated); create a cell and store all code.push_set_cur(if_sharded.block1); - code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); - PackContext ctx(code, loc, ir_builder, create_default_PackOptions(code, loc)); - ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); // addr_std$10 + 0 anycast + code.emplace_back(origin, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); + PackContext ctx(code, origin, ir_builder, create_default_PackOptions(code, origin)); + ctx.storeUint(code.create_int(origin, 0b100, "(addr-prefix)"), 3); // addr_std$10 + 0 anycast ctx.storeInt(ir_self.workchain, 8); ctx.storeUint(ir_hash[0], 256); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } return ir_builder; } // fun AutoDeployAddress.calculateAddress(self): address -std::vector generate_AutoDeployAddress_calculateAddress(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& ir_options) { +std::vector generate_AutoDeployAddress_calculateAddress(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& ir_options) { // it's "build address" + BTOS (until a deprecated "build" function removed) - std::vector ir_builder = generate_AutoDeployAddress_buildAddress(called_f, code, loc, ir_options); - code.emplace_back(loc, Op::_Call, ir_builder, ir_builder, lookup_function("builder.toSlice")); + std::vector ir_builder = generate_AutoDeployAddress_buildAddress(called_f, code, origin, ir_options); + code.emplace_back(origin, Op::_Call, ir_builder, ir_builder, lookup_function("builder.toSlice")); return ir_builder; } // fun AutoDeployAddress.addressMatches(self, addr: address): bool -std::vector generate_AutoDeployAddress_addressMatches(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& ir_self_and_addr) { - IR_AutoDeployAddress ir_self(code, loc, ir_self_and_addr[0]); +std::vector generate_AutoDeployAddress_addressMatches(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& ir_self_and_addr) { + IR_AutoDeployAddress ir_self(code, origin, ir_self_and_addr[0]); // at first, calculate stateInitHash = (hash of StateInit cell would be, but without constructing a cell) - std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); - Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_self.is_ContractState(code, loc)); + std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), origin, "(addr-hash)"); + Op& if_ContractState = code.emplace_back(origin, Op::_If, ir_self.is_ContractState(code, origin)); { // called `{ ... stateInit: { code, data } }` code.push_set_cur(if_ContractState.block0); - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding(code, loc)); + Op& if_sharded = code.emplace_back(origin, Op::_If, ir_self.is_AddressSharding(code, origin)); { // called `{ ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } } code.push_set_cur(if_sharded.block0); std::vector args = { ir_self.ir_shardDepth, ir_self.stateInitCode, ir_self.stateInitData }; - code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashPrefixCodeData")); - code.close_pop_cur(loc); + code.emplace_back(origin, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashPrefixCodeData")); + code.close_pop_cur(origin); } { // called `{ ... stateInit: { code, data } }` (toShard is null) code.push_set_cur(if_sharded.block1); std::vector args = { ir_self.stateInitCode, ir_self.stateInitData }; - code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashCodeData")); - code.close_pop_cur(loc); + code.emplace_back(origin, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashCodeData")); + code.close_pop_cur(origin); } - code.close_pop_cur(loc); + code.close_pop_cur(origin); } { // called `{ ... stateInit: cell }` code.push_set_cur(if_ContractState.block1); std::vector args = { ir_self.stateInitCell }; - code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("cell.hash")); - code.close_pop_cur(loc); + code.emplace_back(origin, Op::_Call, ir_hash, std::move(args), lookup_function("cell.hash")); + code.close_pop_cur(origin); } // now calculate `stateInitHash &= mask` where mask = `(1 << (256 - SHARD_DEPTH)) - 1` - Op& if_sharded1 = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding(code, loc)); + Op& if_sharded1 = code.emplace_back(origin, Op::_If, ir_self.is_AddressSharding(code, origin)); { code.push_set_cur(if_sharded1.block0); - append_bitwise_and_shard_mask(code, loc, ir_hash[0], ir_self.ir_shardDepth); - code.close_pop_cur(loc); + append_bitwise_and_shard_mask(code, origin, ir_hash[0], ir_self.ir_shardDepth); + code.close_pop_cur(origin); } { code.push_set_cur(if_sharded1.block1); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } // now do `(wc, hash) = addr.getWorkchainAndHash()` - std::vector ir_addr_wc_hash = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create()}), loc, "(self-wc-hash)"); - code.emplace_back(loc, Op::_Call, ir_addr_wc_hash, ir_self_and_addr[1], lookup_function("address.getWorkchainAndHash")); + std::vector ir_addr_wc_hash = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create()}), origin, "(self-wc-hash)"); + code.emplace_back(origin, Op::_Call, ir_addr_wc_hash, ir_self_and_addr[1], lookup_function("address.getWorkchainAndHash")); // now calculate `hash &= mask` (the same as we did earlier for stateInitHash) - Op& if_sharded2 = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding(code, loc)); + Op& if_sharded2 = code.emplace_back(origin, Op::_If, ir_self.is_AddressSharding(code, origin)); { code.push_set_cur(if_sharded2.block0); - append_bitwise_and_shard_mask(code, loc, ir_addr_wc_hash[1], ir_self.ir_shardDepth); - code.close_pop_cur(loc); + append_bitwise_and_shard_mask(code, origin, ir_addr_wc_hash[1], ir_self.ir_shardDepth); + code.close_pop_cur(origin); } { code.push_set_cur(if_sharded2.block1); - code.close_pop_cur(loc); + code.close_pop_cur(origin); } // finally, eval `(hash == stateInitHash) & (wc == workchain)` - std::vector ir_eq_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(eq-hash)"); - code.emplace_back(loc, Op::_Call, ir_eq_hash, std::vector{ir_addr_wc_hash[1], ir_hash[0]}, lookup_function("_==_")); - std::vector ir_eq_wc = code.create_tmp_var(TypeDataInt::create(), loc, "(eq-wc)"); - code.emplace_back(loc, Op::_Call, ir_eq_wc, std::vector{ir_addr_wc_hash[0], ir_self.workchain}, lookup_function("_==_")); + std::vector ir_eq_hash = code.create_tmp_var(TypeDataInt::create(), origin, "(eq-hash)"); + code.emplace_back(origin, Op::_Call, ir_eq_hash, std::vector{ir_addr_wc_hash[1], ir_hash[0]}, lookup_function("_==_")); + std::vector ir_eq_wc = code.create_tmp_var(TypeDataInt::create(), origin, "(eq-wc)"); + code.emplace_back(origin, Op::_Call, ir_eq_wc, std::vector{ir_addr_wc_hash[0], ir_self.workchain}, lookup_function("_==_")); - std::vector ir_bool_result = code.create_tmp_var(TypeDataBool::create(), loc, "(is-addr-result)"); - code.emplace_back(loc, Op::_Call, ir_bool_result, std::vector{ir_eq_hash[0], ir_eq_wc[0]}, lookup_function("_&_")); + std::vector ir_bool_result = code.create_tmp_var(TypeDataBool::create(), origin, "(is-addr-result)"); + code.emplace_back(origin, Op::_Call, ir_bool_result, std::vector{ir_eq_hash[0], ir_eq_wc[0]}, lookup_function("_&_")); return ir_bool_result; } diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index 1bcbe546b2..f713585a6f 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -16,7 +16,7 @@ */ #include "smart-casts-cfg.h" #include "ast.h" -#include "tolk.h" +#include "compilation-errors.h" /* * This file represents internals of AST-level control flow and data flow analysis. diff --git a/tolk/src-file.cpp b/tolk/src-file.cpp index 07db0eb88b..4b394e23cc 100644 --- a/tolk/src-file.cpp +++ b/tolk/src-file.cpp @@ -15,6 +15,7 @@ along with TON Blockchain Library. If not, see . */ #include "src-file.h" +#include "compilation-errors.h" #include "compiler-state.h" #include #include @@ -22,7 +23,7 @@ namespace tolk { -static_assert(sizeof(SrcLocation) == 8); +static_assert(sizeof(SrcRange) == 12); const SrcFile* AllRegisteredSrcFiles::find_file(const std::string& realpath) const { // files with the same realpath are considered equal @@ -34,13 +35,13 @@ const SrcFile* AllRegisteredSrcFiles::find_file(const std::string& realpath) con return nullptr; } -const SrcFile* AllRegisteredSrcFiles::locate_and_register_source_file(const std::string& filename, SrcLocation included_from) { +const SrcFile* AllRegisteredSrcFiles::locate_and_register_source_file(const std::string& filename, AnyV v_import_filename) { bool is_stdlib = filename.size() > 8 && filename.starts_with("@stdlib/"); td::Result path = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::Realpath, filename.c_str()); if (path.is_error()) { - if (included_from.is_defined()) { - throw ParseError(included_from, "Failed to import: " + path.move_as_error().message().str()); + if (v_import_filename) { + fire(v_import_filename, "Failed to import: " + path.move_as_error().message().str()); } throw Fatal("Failed to locate " + filename + ": " + path.move_as_error().message().str()); } @@ -52,8 +53,8 @@ const SrcFile* AllRegisteredSrcFiles::locate_and_register_source_file(const std: td::Result text = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::ReadFile, realpath.c_str()); if (text.is_error()) { - if (included_from.is_defined()) { - throw ParseError(included_from, "Failed to import: " + text.move_as_error().message().str()); + if (v_import_filename) { + fire(v_import_filename, "Failed to import: " + text.move_as_error().message().str()); } throw Fatal("Failed to read " + realpath + ": " + text.move_as_error().message().str()); } @@ -81,7 +82,7 @@ bool SrcFile::is_offset_valid(int offset) const { SrcFile::SrcPosition SrcFile::convert_offset(int offset) const { if (!is_offset_valid(offset)) { - return SrcPosition{offset, -1, -1, "invalid offset"}; + return SrcPosition{-1, -1, offset, "invalid offset"}; } // currently, converting offset to line number is O(N): just read file contents char by char and detect lines @@ -111,7 +112,7 @@ SrcFile::SrcPosition SrcFile::convert_offset(int offset) const { } std::string_view line_str(text.data() + line_offset, line_len); - return SrcPosition{offset, line_idx + 1, char_idx + 1, line_str}; + return SrcPosition{line_idx + 1, char_idx + 1, line_offset, line_str}; } std::string SrcFile::extract_short_name() const { @@ -135,60 +136,18 @@ std::string SrcFile::extract_dirname() const { return realpath.substr(0, last_slash + 1); } - -std::ostream& operator<<(std::ostream& os, const SrcFile* src_file) { - return os << (src_file ? src_file->realpath : "unknown-location"); -} - -std::ostream& operator<<(std::ostream& os, const Fatal& fatal) { - return os << fatal.what(); -} - -const SrcFile* SrcLocation::get_src_file() const { +const SrcFile* SrcRange::get_src_file() const { return G.all_src_files.get_file(file_id); } -void SrcLocation::show(std::ostream& os) const { - const SrcFile* src_file = get_src_file(); - os << src_file; - if (src_file && src_file->is_offset_valid(char_offset)) { - SrcFile::SrcPosition pos = src_file->convert_offset(char_offset); - os << ':' << pos.line_no; - if (pos.char_no != 1) { - os << ':' << pos.char_no; - } - } -} -void SrcLocation::show_context(std::ostream& os) const { - const SrcFile* src_file = get_src_file(); - if (!src_file || !src_file->is_offset_valid(char_offset)) { - return; - } - SrcFile::SrcPosition pos = src_file->convert_offset(char_offset); - os << std::right << std::setw(4) << pos.line_no << " | "; - os << pos.line_str << "\n"; - - os << " " << " | "; - for (int i = 1; i < pos.char_no; ++i) { - os << ' '; - } - os << '^' << "\n"; -} - -// when generating Fift output, every block of asm instructions originated from the same Tolk line, -// is preceded by an original line as a comment -void SrcLocation::show_line_to_fif_output(std::ostream& os, int indent, int* last_line_no) const { - SrcFile::SrcPosition pos = G.all_src_files.get_file(file_id)->convert_offset(char_offset); - - // avoid duplicating one line multiple times in fift output - if (pos.line_no == *last_line_no) { - return; - } - *last_line_no = pos.line_no; +// -------------------------------------------- +// stringify ranges and locations, +// output to Fift as comment +static void output_line_to_fif(std::ostream& os, int indent, std::string_view line_str, int line_no, bool dots = false) { // trim some characters from start and end to see `else if (x)` not `} else if (x) {` - std::string_view s = pos.line_str; + std::string_view s = line_str; int b = 0, e = static_cast(s.size() - 1); while (std::isspace(s[b]) || s[b] == '}') { if (b < e) b++; @@ -203,73 +162,90 @@ void SrcLocation::show_line_to_fif_output(std::ostream& os, int indent, int* las for (int i = 0; i < indent * 2; ++i) { os << ' '; } - os << "// " << pos.line_no << ": " << s.substr(b, e - b + 1) << std::endl; - } + os << "// " << (dots ? "..." : "") << line_no << ": " << s.substr(b, e - b + 1) << std::endl; + } } -std::string SrcLocation::to_string() const { - std::ostringstream os; - show(os); - return os.str(); -} - -std::ostream& operator<<(std::ostream& os, SrcLocation loc) { - loc.show(os); - return os; +// when generating Fift output, every block of asm instructions originated from the same Tolk line, +// is preceded by an original line as a comment +void SrcRange::output_first_line_to_fif(std::ostream& os, int indent) const { + // avoid duplicating one line multiple times in fift output + static int last_start_offset = 0, last_start_line_no = 0; + static int last_end_offset = 0, last_end_line_no = 0; + + bool just_printed_start_line = false; + if (start_offset != last_start_offset) { + SrcFile::SrcPosition pos = G.all_src_files.get_file(file_id)->convert_offset(start_offset); + if (pos.line_no != last_start_line_no) { + output_line_to_fif(os, indent, pos.line_str, pos.line_no); + just_printed_start_line = true; + } + last_start_offset = last_end_offset = start_offset; + last_start_line_no = last_end_line_no = pos.line_no; + } + if (end_offset > last_end_offset) { + SrcFile::SrcPosition pos = G.all_src_files.get_file(file_id)->convert_offset(end_offset); + if (pos.line_no > last_end_line_no) { + std::string_view line_str = pos.line_str.substr(0, end_offset - pos.line_offset); + output_line_to_fif(os, indent, line_str, pos.line_no, just_printed_start_line); + } + last_end_offset = end_offset; + last_end_line_no = pos.line_no; + } } -void SrcLocation::show_general_error(std::ostream& os, const std::string& message, const std::string& err_type) const { - show(os); - if (!err_type.empty()) { - os << ": " << err_type; +std::string SrcRange::stringify_start_location(bool output_char_no) const { + const SrcFile* src_file = get_src_file(); + if (!src_file || !src_file->is_offset_valid(start_offset)) { + return "unknown-location"; } - os << ": " << message << std::endl; - show_context(os); -} -void SrcLocation::show_note(const std::string& err_msg) const { - show_general_error(std::cerr, err_msg, "note"); + SrcFile::SrcPosition pos = src_file->convert_offset(start_offset); + std::string s = src_file->realpath; + s += ':'; + s += std::to_string(pos.line_no); + if (output_char_no && pos.char_no != 1) { + s += ':'; + s += std::to_string(pos.char_no); + } + return s; } -void SrcLocation::show_warning(const std::string& err_msg) const { - show_general_error(std::cerr, err_msg, "warning"); -} -void SrcLocation::show_error(const std::string& err_msg) const { - show_general_error(std::cerr, err_msg, "error"); -} +void SrcRange::output_underlined(std::ostream& os) const { + const SrcFile* src_file = get_src_file(); + if (!src_file || !src_file->is_offset_valid(end_offset) || !is_valid()) { + return; + } + SrcFile::SrcPosition start = src_file->convert_offset(start_offset); + SrcFile::SrcPosition end = src_file->convert_offset(end_offset); -std::ostream& operator<<(std::ostream& os, const ParseError& error) { - error.show(os); - return os; -} + os << std::right << std::setw(4) << start.line_no << " | " << start.line_str << "\n"; + os << " " << " | "; + for (int i = 1; i < start.char_no; ++i) { + os << ' '; + } + int end_char_no_first_line = start.line_no == end.line_no ? end.char_no : static_cast(start.line_str.size()); + for (int i = start.char_no; i < end_char_no_first_line; ++i) { + os << '^'; + } + os << "\n"; -void ParseError::show(std::ostream& os) const { - if (message.find('\n') == std::string::npos) { - // just print a single-line message - os << loc << ": error: " << message << std::endl; - } else { - // print "location: line1 \n (spaces) line2 \n ..." - std::string_view message = this->message; - std::string loc_text = loc.to_string(); - std::string loc_spaces(std::min(static_cast(loc_text.size()), 9), ' '); - size_t start = 0, end; - os << loc_text << ": error: "; - while ((end = message.find('\n', start)) != std::string::npos) { - if (start > 0) { - os << loc_spaces << " "; - } - os << message.substr(start, end - start) << std::endl; - start = end + 1; - } - if (start < message.size()) { - os << loc_spaces << " " << message.substr(start) << std::endl; - } + if (end.line_no > start.line_no + 1) { + os << " ..." << " ..."; + os << "\n"; } - if (current_function) { - os << std::endl << " // in function `" << current_function->as_human_readable() << "`" << std::endl; + if (end.line_no > start.line_no) { + os << std::right << std::setw(4) << end.line_no << " | " << end.line_str << "\n"; + os << " " << " | "; + bool was_non_space = false; + for (int i = 1; i < end.char_no; ++i) { + was_non_space |= !std::isspace(end.line_str[i - 1]); + os << (was_non_space ? '^' : ' '); + } + os << "\n"; } - loc.show_context(os); } + } // namespace tolk diff --git a/tolk/src-file.h b/tolk/src-file.h index 9b39a0accc..e119b21130 100644 --- a/tolk/src-file.h +++ b/tolk/src-file.h @@ -19,14 +19,15 @@ #include #include #include "fwd-declarations.h" +#include "symtable.h" namespace tolk { struct SrcFile { struct SrcPosition { - int offset; int line_no; int char_no; + int line_offset; std::string_view line_str; }; @@ -57,46 +58,6 @@ struct SrcFile { std::string extract_dirname() const; }; - -// SrcLocation points to a location (line, column) in some loaded .tolk source SrcFile. -// Note, that instead of storing src_file, line_no, etc., only 2 ints are stored. -// The purpose is: sizeof(SrcLocation) == 8, so it's just passed/stored without pointers/refs, just like int64_t. -// When decoding SrcLocation into human-readable format, it's converted to SrcFile::SrcPosition via offset. -class SrcLocation { - friend class Lexer; - - int file_id = -1; // = SrcFile::file_id (note, that get_src_file() does linear search) - int char_offset = -1; // offset from SrcFile::text - -public: - - SrcLocation() = default; - explicit SrcLocation(const SrcFile* src_file) : file_id(src_file->file_id) { - } - - bool is_defined() const { return file_id != -1; } - bool is_stdlib() const { return file_id == 0; } - const SrcFile* get_src_file() const; - - // similar to `this->get_src_file() == symbol->get_src_file() || symbol->get_src_file()->is_stdlib()` - // (but effectively, avoiding linear search) - bool is_symbol_from_same_or_builtin_file(SrcLocation symbol_loc) const { - return file_id == symbol_loc.file_id || symbol_loc.file_id < 1; - } - - void show(std::ostream& os) const; - void show_context(std::ostream& os) const; - void show_line_to_fif_output(std::ostream& os, int indent, int* last_line_no) const; - std::string to_string() const; - - void show_general_error(std::ostream& os, const std::string& message, const std::string& err_type) const; - void show_note(const std::string& err_msg) const; - void show_warning(const std::string& err_msg) const; - void show_error(const std::string& err_msg) const; -}; - -std::ostream& operator<<(std::ostream& os, SrcLocation loc); - class AllRegisteredSrcFiles { std::vector all_src_files; int last_parsed_file_id = -1; @@ -105,41 +66,75 @@ class AllRegisteredSrcFiles { const SrcFile* get_file(int file_id) const { return all_src_files.at(file_id); } const SrcFile* find_file(const std::string& realpath) const; - const SrcFile* locate_and_register_source_file(const std::string& filename, SrcLocation included_from); + const SrcFile* locate_and_register_source_file(const std::string& filename, AnyV v_import_filename); SrcFile* get_next_unparsed_file(); auto begin() const { return all_src_files.begin(); } auto end() const { return all_src_files.end(); } }; -struct Fatal final : std::exception { - std::string message; +// SrcRange is a "substring" in some loaded .tolk source SrcFile. +// Note, that instead of storing src_file, line_no, etc., only 3 ints are stored. +// When decoding SrcRange into human-readable format, it's converted to SrcFile::SrcPosition via offset. +class SrcRange { + int file_id; // SrcFile::file_id (note, that get_src_file() does linear search) + int start_offset; // offset from SrcFile::text + int end_offset; // >= start_offset, otherwise "invalid range" + + SrcRange(int file_id, int start_offset, int end_offset) + : file_id(file_id) + , start_offset(start_offset) + , end_offset(end_offset) { + } + +public: + static SrcRange undefined() { + return SrcRange(-1, 0, -1); + } - explicit Fatal(std::string _msg) : message(std::move(_msg)) { + static SrcRange overlap(SrcRange start, SrcRange end) { + return SrcRange(start.file_id, start.start_offset, end.end_offset); } - const char* what() const noexcept override { - return message.c_str(); + + static SrcRange empty_at_start(SrcRange range) { + return SrcRange(range.file_id, range.start_offset, range.start_offset); + } + + static SrcRange empty_at_end(SrcRange range) { + return SrcRange(range.file_id, range.end_offset, range.end_offset); } -}; -std::ostream& operator<<(std::ostream& os, const Fatal& fatal); + static SrcRange span(SrcRange start, int len) { + return SrcRange(start.file_id, start.start_offset, start.start_offset + len); + } -struct ParseError : std::exception { - FunctionPtr current_function; - SrcLocation loc; - std::string message; + static SrcRange span(int file_id, int start_offset, int len) { + return SrcRange(file_id, start_offset, start_offset + len); + } - ParseError(SrcLocation loc, std::string message) - : current_function(nullptr), loc(loc), message(std::move(message)) {} - ParseError(FunctionPtr current_function, SrcLocation loc, std::string message) - : current_function(current_function), loc(loc), message(std::move(message)) {} + static SrcRange unclosed_range(int file_id, int start_offset) { + return SrcRange(file_id, start_offset, 0); // in dev mode, there is `assert` that range is closed before used + } - const char* what() const noexcept override { - return message.c_str(); + void end(SrcRange end) { + end_offset = end.end_offset; } - void show(std::ostream& os) const; -}; -std::ostream& operator<<(std::ostream& os, const ParseError& error); + bool is_defined() const { + return file_id != -1; + } + bool is_valid() const { + return end_offset >= start_offset; + } + bool is_file_id_same_or_stdlib_common(SrcRange other) const { + return file_id == 0 || file_id == other.file_id; + } + + const SrcFile* get_src_file() const; + std::string stringify_start_location(bool output_char_no) const; + + void output_first_line_to_fif(std::ostream& os, int indent) const; + void output_underlined(std::ostream& os) const; +}; } // namespace tolk diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 20b9617b44..153ce3ebfc 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -15,22 +15,25 @@ along with TON Blockchain Library. If not, see . */ #include "symtable.h" +#include "ast.h" +#include "compilation-errors.h" #include "compiler-state.h" #include "platform-utils.h" #include "generics-helpers.h" namespace tolk { -void Symbol::check_import_exists_when_used_from(FunctionPtr cur_f, SrcLocation used_loc) const { - const SrcFile* declared_in = loc.get_src_file(); +void Symbol::check_import_exists_when_used_from(FunctionPtr cur_f, AnyV usage) const { +#ifdef TOLK_DEBUG + assert(ident_anchor != nullptr); +#endif + const SrcFile* declared_in = ident_anchor->range.get_src_file(); bool has_import = false; - for (const SrcFile::ImportDirective& import : used_loc.get_src_file()->imports) { - if (import.imported_file == declared_in) { - has_import = true; - } + for (const SrcFile::ImportDirective& import : usage->range.get_src_file()->imports) { + has_import |= import.imported_file == declared_in; } if (!has_import) { - throw ParseError(cur_f, used_loc, "Using a non-imported symbol `" + name + "`\nhint: forgot to import \"" + declared_in->extract_short_name() + "\"?"); + fire(cur_f, usage, "Using a non-imported symbol `" + name + "`\nhint: forgot to import \"" + declared_in->extract_short_name() + "\"?"); } } @@ -75,7 +78,7 @@ bool FunctionData::does_need_codegen() const { } // functions with asm body don't need code generation // (even if used as non-call: `var a = beginCell;` inserts TVM continuation inline) - if (is_asm_function() || is_builtin_function()) { + if (is_asm_function() || is_builtin()) { return false; } // when a function is referenced like `var a = some_fn;` (or in some other non-call way), its continuation should exist @@ -256,22 +259,21 @@ EnumMemberPtr EnumDefData::find_member(std::string_view member_name) const { } GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_redefinition_of_symbol(SrcLocation loc, const Symbol* previous) { - SrcLocation prev_loc = previous->loc; - if (prev_loc.is_stdlib()) { - throw ParseError(loc, "redefinition of a symbol from stdlib"); +static void fire_redefinition_of_symbol(SrcRange range, const Symbol* previous) { + if (previous->is_builtin()) { + fire(range, "redefinition of built-in symbol"); } - if (prev_loc.is_defined()) { - throw ParseError(loc, "redefinition of symbol, previous was at: " + prev_loc.to_string()); + if (previous->ident_anchor->range.get_src_file()->is_stdlib_file) { + fire(range, "redefinition of a symbol from stdlib"); } - throw ParseError(loc, "redefinition of built-in symbol"); + fire(range, "redefinition of symbol, previous was at: " + previous->ident_anchor->range.stringify_start_location(false)); } void GlobalSymbolTable::add_global_symbol(const Symbol* sym) { auto key = key_hash(sym->name); auto [it, inserted] = entries.emplace(key, sym); if (!inserted) { - fire_error_redefinition_of_symbol(sym->loc, it->second); + fire_redefinition_of_symbol(sym->ident_anchor->range, it->second); } } diff --git a/tolk/symtable.h b/tolk/symtable.h index 09235eb353..5216144df2 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -27,11 +27,11 @@ namespace tolk { struct Symbol { std::string name; - SrcLocation loc; + AnyV ident_anchor; // "identifier node", e.g. for `struct Demo { ... }` will be `Demo`; nullptr for builtin - Symbol(std::string name, SrcLocation loc) + Symbol(std::string name, AnyV ident_anchor) : name(std::move(name)) - , loc(loc) { + , ident_anchor(ident_anchor) { } virtual ~Symbol() = default; @@ -44,7 +44,8 @@ struct Symbol { return dynamic_cast(this); } - void check_import_exists_when_used_from(FunctionPtr cur_f, SrcLocation used_loc) const; + bool is_builtin() const { return ident_anchor == nullptr; } + void check_import_exists_when_used_from(FunctionPtr cur_f, AnyV usage) const; }; struct LocalVarData final : Symbol { @@ -62,15 +63,15 @@ struct LocalVarData final : Symbol { int param_idx; // 0...N for function parameters, -1 for local vars std::vector ir_idx; - LocalVarData(std::string name, SrcLocation loc, AnyTypeV type_node, AnyExprV default_value, int flags, int param_idx) - : Symbol(std::move(name), loc) + LocalVarData(std::string name, AnyV ident_anchor, AnyTypeV type_node, AnyExprV default_value, int flags, int param_idx) + : Symbol(std::move(name), ident_anchor) , type_node(type_node) , default_value(default_value) , flags(flags) , param_idx(param_idx) { } - LocalVarData(std::string name, SrcLocation loc, TypePtr declared_type, AnyExprV default_value, int flags, int param_idx) - : Symbol(std::move(name), loc) + LocalVarData(std::string name, AnyV ident_anchor, TypePtr declared_type, AnyExprV default_value, int flags, int param_idx) + : Symbol(std::move(name), ident_anchor) , type_node(nullptr) // for built-in functions (their parameters) , declared_type(declared_type) , default_value(default_value) @@ -148,8 +149,8 @@ struct FunctionData final : Symbol { FunctionBody body; AnyV ast_root; // V for user-defined (not builtin) - FunctionData(std::string name, SrcLocation loc, std::string method_name, AnyTypeV receiver_type_node, AnyTypeV return_type_node, std::vector parameters, int initial_flags, FunctionInlineMode inline_mode, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) - : Symbol(std::move(name), loc) + FunctionData(std::string name, AnyV ident_anchor, std::string method_name, AnyTypeV receiver_type_node, AnyTypeV return_type_node, std::vector parameters, int initial_flags, FunctionInlineMode inline_mode, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) + : Symbol(std::move(name), ident_anchor) , flags(initial_flags) , inline_mode(inline_mode) , method_name(std::move(method_name)) @@ -161,8 +162,8 @@ struct FunctionData final : Symbol { , body(body) , ast_root(ast_root) { } - FunctionData(std::string name, SrcLocation loc, std::string method_name, TypePtr receiver_type, TypePtr declared_return_type, std::vector parameters, int initial_flags, FunctionInlineMode inline_mode, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) - : Symbol(std::move(name), loc) + FunctionData(std::string name, AnyV ident_anchor, std::string method_name, TypePtr receiver_type, TypePtr declared_return_type, std::vector parameters, int initial_flags, FunctionInlineMode inline_mode, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) + : Symbol(std::move(name), ident_anchor) , flags(initial_flags) , inline_mode(inline_mode) , method_name(std::move(method_name)) @@ -192,7 +193,6 @@ struct FunctionData final : Symbol { bool is_code_function() const { return std::holds_alternative(body); } bool is_asm_function() const { return std::holds_alternative(body); } - bool is_builtin_function() const { return ast_root == nullptr; } bool is_method() const { return !method_name.empty(); } bool is_static_method() const { return is_method() && !does_accept_self(); } @@ -241,8 +241,8 @@ struct GlobalVarData final : Symbol { TypePtr declared_type = nullptr; // = resolved type_node int flags = 0; - GlobalVarData(std::string name, SrcLocation loc, AnyTypeV type_node) - : Symbol(std::move(name), loc) + GlobalVarData(std::string name, AnyV ident_anchor, AnyTypeV type_node) + : Symbol(std::move(name), ident_anchor) , type_node(type_node) { } @@ -259,8 +259,8 @@ struct GlobalConstData final : Symbol { TypePtr inferred_type = nullptr; AnyExprV init_value; - GlobalConstData(std::string name, SrcLocation loc, AnyTypeV type_node, AnyExprV init_value) - : Symbol(std::move(name), loc) + GlobalConstData(std::string name, AnyV ident_anchor, AnyTypeV type_node, AnyExprV init_value) + : Symbol(std::move(name), ident_anchor) , type_node(type_node) , init_value(init_value) { } @@ -280,8 +280,8 @@ struct AliasDefData final : Symbol { AliasDefPtr base_alias_ref = nullptr; // for `Response`, here is `Response` AnyV ast_root; // V - AliasDefData(std::string name, SrcLocation loc, AnyTypeV underlying_type_node, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, AnyV ast_root) - : Symbol(std::move(name), loc) + AliasDefData(std::string name, AnyV ident_anchor, AnyTypeV underlying_type_node, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, AnyV ast_root) + : Symbol(std::move(name), ident_anchor) , underlying_type_node(underlying_type_node) , genericTs(genericTs) , substitutedTs(substitutedTs) @@ -312,8 +312,8 @@ struct StructFieldData final : Symbol { void assign_resolved_type(TypePtr declared_type); void assign_default_value(AnyExprV default_value); - StructFieldData(std::string name, SrcLocation loc, int field_idx, bool is_private, bool is_readonly, AnyTypeV type_node, AnyExprV default_value) - : Symbol(std::move(name), loc) + StructFieldData(std::string name, AnyV ident_anchor, int field_idx, bool is_private, bool is_readonly, AnyTypeV type_node, AnyExprV default_value) + : Symbol(std::move(name), ident_anchor) , field_idx(field_idx) , is_private(is_private) , is_readonly(is_readonly) @@ -359,8 +359,8 @@ struct StructData final : Symbol { StructData* mutate() const { return const_cast(this); } void assign_resolved_genericTs(const GenericsDeclaration* genericTs); - StructData(std::string name, SrcLocation loc, std::vector&& fields, PackOpcode opcode, Overflow1023Policy overflow1023_policy, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, AnyV ast_root) - : Symbol(std::move(name), loc) + StructData(std::string name, AnyV ident_anchor, std::vector&& fields, PackOpcode opcode, Overflow1023Policy overflow1023_policy, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, AnyV ast_root) + : Symbol(std::move(name), ident_anchor) , fields(std::move(fields)) , opcode(opcode) , overflow1023_policy(overflow1023_policy) @@ -378,8 +378,8 @@ struct EnumMemberData final : Symbol { bool has_init_value() const { return init_value != nullptr; } - EnumMemberData(std::string name, SrcLocation loc, AnyExprV init_value) - : Symbol(std::move(name), loc) + EnumMemberData(std::string name, AnyV ident_anchor, AnyExprV init_value) + : Symbol(std::move(name), ident_anchor) , init_value(init_value) { } @@ -395,8 +395,8 @@ struct EnumDefData final : Symbol { EnumMemberPtr find_member(std::string_view member_name) const; - EnumDefData(std::string name, SrcLocation loc, AnyTypeV colon_type_node, std::vector&& members) - : Symbol(std::move(name), loc) + EnumDefData(std::string name, AnyV ident_anchor, AnyTypeV colon_type_node, std::vector&& members) + : Symbol(std::move(name), ident_anchor) , colon_type_node(colon_type_node) , members(std::move(members)) { } @@ -410,8 +410,8 @@ struct EnumDefData final : Symbol { struct TypeReferenceUsedAsSymbol final : Symbol { TypePtr resolved_type; - TypeReferenceUsedAsSymbol(std::string name, SrcLocation loc, TypePtr resolved_type) - : Symbol(std::move(name), loc) + TypeReferenceUsedAsSymbol(std::string name, AnyV ident_anchor, TypePtr resolved_type) + : Symbol(std::move(name), ident_anchor) , resolved_type(resolved_type) { } }; diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp index cef2365ecc..437ab63d54 100644 --- a/tolk/tolk.cpp +++ b/tolk/tolk.cpp @@ -23,7 +23,6 @@ exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. */ -#include "tolk.h" #include "pipeline.h" #include "compiler-state.h" #include "lexer.h" @@ -32,18 +31,7 @@ namespace tolk { - -void on_assertion_failed(const char *description, const char *file_name, int line_number) { - std::string message = static_cast("Assertion failed at ") + file_name + ":" + std::to_string(line_number) + ": " + description; -#ifdef TOLK_DEBUG -#ifdef __arm64__ - // when developing, it's handy when the debugger stops on assertion failure (stacktraces and watches are available) - std::cerr << message << std::endl; - __builtin_debugtrap(); -#endif -#endif - throw Fatal(std::move(message)); -} +void define_builtins(); int tolk_proceed(const std::string &entrypoint_filename) { type_system_init(); @@ -77,14 +65,14 @@ int tolk_proceed(const std::string &entrypoint_filename) { pipeline_generate_fif_output_to_std_cout(); return 0; - } catch (Fatal& fatal) { - std::cerr << "fatal: " << fatal << std::endl; + } catch (const Fatal& fatal) { + std::cerr << "fatal: " << fatal.message << std::endl; return 2; - } catch (ParseError& error) { - std::cerr << error << std::endl; + } catch (const ParseError& error) { + error.output_compilation_error(std::cerr); return 2; - } catch (UnexpectedASTNodeKind& error) { - std::cerr << "fatal: " << error.what() << std::endl; + } catch (const UnexpectedASTNodeKind& error) { + std::cerr << "fatal: " << error.message << std::endl; std::cerr << "It's a compiler bug, please report to developers" << std::endl; return 2; } diff --git a/tolk/tolk.h b/tolk/tolk.h index aa1fa0fd6e..057dc5ad2e 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -16,29 +16,21 @@ */ #pragma once -#include "platform-utils.h" -#include "src-file.h" -#include "symtable.h" +#include "compilation-errors.h" #include "crypto/common/refint.h" -#include "td/utils/Status.h" #include #include #include #include -#include -#define tolk_assert(expr) if(UNLIKELY(!(expr))) on_assertion_failed(#expr, __FILE__, __LINE__); +/* + This file is "inherited from FunC" (keep in mind, that Tolk is a fork of FunC compiler, gradually improved) + and contains "uncategorized" entities — almost unchanged from FunC. + Some day, when intermediate representation (IR, aka Ops) is fully rewritten, it will be removed. + */ namespace tolk { -GNU_ATTRIBUTE_COLD GNU_ATTRIBUTE_NORETURN -void on_assertion_failed(const char *description, const char *file_name, int line_number); - -// fire a general error, just a wrapper over `throw` -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -inline void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { - throw ParseError(cur_f, loc, message); -} /* * @@ -52,16 +44,14 @@ struct TmpVar { var_idx_t ir_idx; // every var in IR represents 1 stack slot TypePtr v_type; // get_width_on_stack() is 1 std::string name; // "x" for vars originated from user sources; "x.0" for tensor components; empty for implicitly created tmp vars - SrcLocation loc; // location of var declaration in sources or where a tmp var was originated #ifdef TOLK_DEBUG - const char* desc = nullptr; // "origin" of tmp var, for debug output like `'15 (binary-op) '16 (glob-var)` + const char* purpose = nullptr; // "purpose" of tmp var, for debug output like `'15 (binary-op) '16 (glob-var)` #endif - TmpVar(var_idx_t ir_idx, TypePtr v_type, std::string name, SrcLocation loc) + TmpVar(var_idx_t ir_idx, TypePtr v_type, std::string name) : ir_idx(ir_idx) , v_type(v_type) - , name(std::move(name)) - , loc(loc) { + , name(std::move(name)) { } void show_as_stack_comment(std::ostream& os) const; @@ -291,48 +281,48 @@ struct Op { std::unique_ptr next; FunctionPtr f_sym = nullptr; GlobalVarPtr g_sym = nullptr; - SrcLocation loc; + AnyV origin; VarDescrList var_info; std::vector args; std::vector left, right; std::unique_ptr block0, block1; td::RefInt256 int_const; std::string str_const; - Op(SrcLocation loc, OpKind cl) : cl(cl), flags(0), loc(loc) { + Op(AnyV origin, OpKind cl) : cl(cl), flags(0), origin(origin) { } - Op(SrcLocation loc, OpKind cl, const std::vector& left) - : cl(cl), flags(0), loc(loc), left(left) { + Op(AnyV origin, OpKind cl, const std::vector& left) + : cl(cl), flags(0), origin(origin), left(left) { } - Op(SrcLocation loc, OpKind cl, std::vector&& left) - : cl(cl), flags(0), loc(loc), left(std::move(left)) { + Op(AnyV origin, OpKind cl, std::vector&& left) + : cl(cl), flags(0), origin(origin), left(std::move(left)) { } - Op(SrcLocation loc, OpKind cl, const std::vector& left, td::RefInt256 int_const) - : cl(cl), flags(0), loc(loc), left(left), int_const(std::move(int_const)) { + Op(AnyV origin, OpKind cl, const std::vector& left, td::RefInt256 int_const) + : cl(cl), flags(0), origin(origin), left(left), int_const(std::move(int_const)) { } - Op(SrcLocation loc, OpKind cl, const std::vector& left, std::string str_const) - : cl(cl), flags(0), loc(loc), left(left), str_const(std::move(str_const)) { + Op(AnyV origin, OpKind cl, const std::vector& left, std::string str_const) + : cl(cl), flags(0), origin(origin), left(left), str_const(std::move(str_const)) { } - Op(SrcLocation loc, OpKind cl, const std::vector& left, const std::vector& right) - : cl(cl), flags(0), loc(loc), left(left), right(right) { + Op(AnyV origin, OpKind cl, const std::vector& left, const std::vector& right) + : cl(cl), flags(0), origin(origin), left(left), right(right) { } - Op(SrcLocation loc, OpKind cl, std::vector&& left, std::vector&& right) - : cl(cl), flags(0), loc(loc), left(std::move(left)), right(std::move(right)) { + Op(AnyV origin, OpKind cl, std::vector&& left, std::vector&& right) + : cl(cl), flags(0), origin(origin), left(std::move(left)), right(std::move(right)) { } - Op(SrcLocation loc, OpKind cl, const std::vector& left, const std::vector& right, + Op(AnyV origin, OpKind cl, const std::vector& left, const std::vector& right, FunctionPtr _fun) - : cl(cl), flags(0), f_sym(_fun), loc(loc), left(left), right(right) { + : cl(cl), flags(0), f_sym(_fun), origin(origin), left(left), right(right) { } - Op(SrcLocation loc, OpKind cl, std::vector&& left, std::vector&& right, + Op(AnyV origin, OpKind cl, std::vector&& left, std::vector&& right, FunctionPtr fun_ref) - : cl(cl), flags(0), f_sym(fun_ref), loc(loc), left(std::move(left)), right(std::move(right)) { + : cl(cl), flags(0), f_sym(fun_ref), origin(origin), left(std::move(left)), right(std::move(right)) { } - Op(SrcLocation loc, OpKind cl, const std::vector& left, const std::vector& right, + Op(AnyV origin, OpKind cl, const std::vector& left, const std::vector& right, GlobalVarPtr glob_ref) - : cl(cl), flags(0), g_sym(glob_ref), loc(loc), left(left), right(right) { + : cl(cl), flags(0), g_sym(glob_ref), origin(origin), left(left), right(right) { } - Op(SrcLocation loc, OpKind cl, std::vector&& left, std::vector&& right, + Op(AnyV origin, OpKind cl, std::vector&& left, std::vector&& right, GlobalVarPtr _gvar) - : cl(cl), flags(0), g_sym(_gvar), loc(loc), left(std::move(left)), right(std::move(right)) { + : cl(cl), flags(0), g_sym(_gvar), origin(origin), left(std::move(left)), right(std::move(right)) { } bool disabled() const { return flags & _Disabled; } @@ -349,11 +339,10 @@ struct Op { bool arg_order_already_equals_asm() const { return flags & _ArgOrderAlreadyEqualsAsm; } void set_arg_order_already_equals_asm_flag(); - void show(std::ostream& os, const std::vector& vars, std::string pfx = "", int mode = 0) const; + void show(std::ostream& os, const std::vector& vars, const std::string& indent, int mode = 0) const; void show_var_list(std::ostream& os, const std::vector& idx_list, const std::vector& vars) const; void show_var_list(std::ostream& os, const std::vector& list, const std::vector& vars) const; - static void show_block(std::ostream& os, const Op* block, const std::vector& vars, std::string pfx = "", - int mode = 0); + static void show_block(std::ostream& os, const Op* block, const std::vector& vars, const std::string& indent, int mode = 0); bool compute_used_vars(const CodeBlob& code, bool edit); bool std_compute_used_vars(bool disabled = false); bool set_var_info(const VarDescrList& new_var_info); @@ -414,7 +403,7 @@ constexpr const_idx_t not_const = -1; struct AsmOp { enum Type { a_nop, a_comment, a_xchg, a_push, a_pop, a_const, a_custom }; Type t; - SrcLocation loc; + AnyV origin; int indent{0}; int a, b; bool gconst{false}; @@ -426,17 +415,17 @@ struct AsmOp { int calc_out_strlen() const; }; AsmOp() = default; - AsmOp(Type t, SrcLocation loc) : t(t), loc(loc) { + AsmOp(Type t, AnyV origin) : t(t), origin(origin) { } - AsmOp(Type t, SrcLocation loc, std::string _op) : t(t), loc(loc), op(std::move(_op)) { + AsmOp(Type t, AnyV origin, std::string _op) : t(t), origin(origin), op(std::move(_op)) { } - AsmOp(Type t, SrcLocation loc, int a) : t(t), loc(loc), a(a) { + AsmOp(Type t, AnyV origin, int a) : t(t), origin(origin), a(a) { } - AsmOp(Type t, SrcLocation loc, int a, std::string _op) : t(t), loc(loc), a(a), op(std::move(_op)) { + AsmOp(Type t, AnyV origin, int a, std::string _op) : t(t), origin(origin), a(a), op(std::move(_op)) { } - AsmOp(Type t, SrcLocation loc, int a, int b) : t(t), loc(loc), a(a), b(b) { + AsmOp(Type t, AnyV origin, int a, int b) : t(t), origin(origin), a(a), b(b) { } - AsmOp(Type t, SrcLocation loc, int a, int b, std::string op) : t(t), loc(loc), a(a), b(b), op(std::move(op)) { + AsmOp(Type t, AnyV origin, int a, int b, std::string op) : t(t), origin(origin), a(a), b(b), op(std::move(op)) { compute_gconst(); } int out(std::ostream& os) const; @@ -496,80 +485,80 @@ struct AsmOp { bool is_gconst() const { return !a && b == 1 && (t == a_const || gconst); } - static AsmOp Nop(SrcLocation loc) { - return AsmOp(a_nop, loc); + static AsmOp Nop(AnyV origin) { + return AsmOp(a_nop, origin); } - static AsmOp Xchg(SrcLocation loc, int a, int b = 0) { - return a == b ? AsmOp(a_nop, loc) : (a < b ? AsmOp(a_xchg, loc, a, b) : AsmOp(a_xchg, loc, b, a)); + static AsmOp Xchg(AnyV origin, int a, int b = 0) { + return a == b ? AsmOp(a_nop, origin) : (a < b ? AsmOp(a_xchg, origin, a, b) : AsmOp(a_xchg, origin, b, a)); } - static AsmOp Push(SrcLocation loc, int a) { - return AsmOp(a_push, loc, a); + static AsmOp Push(AnyV origin, int a) { + return AsmOp(a_push, origin, a); } - static AsmOp Pop(SrcLocation loc, int a) { - return AsmOp(a_pop, loc, a); + static AsmOp Pop(AnyV origin, int a) { + return AsmOp(a_pop, origin, a); } - static AsmOp Xchg2(SrcLocation loc, int a, int b) { - return make_stk2(loc, a, b, "XCHG2", 0); + static AsmOp Xchg2(AnyV origin, int a, int b) { + return make_stk2(origin, a, b, "XCHG2", 0); } - static AsmOp XcPu(SrcLocation loc, int a, int b) { - return make_stk2(loc, a, b, "XCPU", 1); + static AsmOp XcPu(AnyV origin, int a, int b) { + return make_stk2(origin, a, b, "XCPU", 1); } - static AsmOp PuXc(SrcLocation loc, int a, int b) { - return make_stk2(loc, a, b, "PUXC", 1); + static AsmOp PuXc(AnyV origin, int a, int b) { + return make_stk2(origin, a, b, "PUXC", 1); } - static AsmOp Push2(SrcLocation loc, int a, int b) { - return make_stk2(loc, a, b, "PUSH2", 2); + static AsmOp Push2(AnyV origin, int a, int b) { + return make_stk2(origin, a, b, "PUSH2", 2); } - static AsmOp Xchg3(SrcLocation loc, int a, int b, int c) { - return make_stk3(loc, a, b, c, "XCHG3", 0); + static AsmOp Xchg3(AnyV origin, int a, int b, int c) { + return make_stk3(origin, a, b, c, "XCHG3", 0); } - static AsmOp Xc2Pu(SrcLocation loc, int a, int b, int c) { - return make_stk3(loc, a, b, c, "XC2PU", 1); + static AsmOp Xc2Pu(AnyV origin, int a, int b, int c) { + return make_stk3(origin, a, b, c, "XC2PU", 1); } - static AsmOp XcPuXc(SrcLocation loc, int a, int b, int c) { - return make_stk3(loc, a, b, c, "XCPUXC", 1); + static AsmOp XcPuXc(AnyV origin, int a, int b, int c) { + return make_stk3(origin, a, b, c, "XCPUXC", 1); } - static AsmOp XcPu2(SrcLocation loc, int a, int b, int c) { - return make_stk3(loc, a, b, c, "XCPU2", 3); + static AsmOp XcPu2(AnyV origin, int a, int b, int c) { + return make_stk3(origin, a, b, c, "XCPU2", 3); } - static AsmOp PuXc2(SrcLocation loc, int a, int b, int c) { - return make_stk3(loc, a, b, c, "PUXC2", 3); + static AsmOp PuXc2(AnyV origin, int a, int b, int c) { + return make_stk3(origin, a, b, c, "PUXC2", 3); } - static AsmOp PuXcPu(SrcLocation loc, int a, int b, int c) { - return make_stk3(loc, a, b, c, "PUXCPU", 3); + static AsmOp PuXcPu(AnyV origin, int a, int b, int c) { + return make_stk3(origin, a, b, c, "PUXCPU", 3); } - static AsmOp Pu2Xc(SrcLocation loc, int a, int b, int c) { - return make_stk3(loc, a, b, c, "PU2XC", 3); + static AsmOp Pu2Xc(AnyV origin, int a, int b, int c) { + return make_stk3(origin, a, b, c, "PU2XC", 3); } - static AsmOp Push3(SrcLocation loc, int a, int b, int c) { - return make_stk3(loc, a, b, c, "PUSH3", 3); + static AsmOp Push3(AnyV origin, int a, int b, int c) { + return make_stk3(origin, a, b, c, "PUSH3", 3); } - static AsmOp BlkSwap(SrcLocation loc, int a, int b); - static AsmOp BlkPush(SrcLocation loc, int a, int b); - static AsmOp BlkDrop(SrcLocation loc, int a); - static AsmOp BlkDrop2(SrcLocation loc, int a, int b); - static AsmOp BlkReverse(SrcLocation loc, int a, int b); - static AsmOp make_stk2(SrcLocation loc, int a, int b, const char* str, int delta); - static AsmOp make_stk3(SrcLocation loc, int a, int b, int c, const char* str, int delta); - static AsmOp IntConst(SrcLocation loc, const td::RefInt256& x); - static AsmOp BoolConst(SrcLocation loc, bool f); - static AsmOp Const(SrcLocation loc, std::string push_op) { - return AsmOp(a_const, loc, 0, 1, std::move(push_op)); + static AsmOp BlkSwap(AnyV origin, int a, int b); + static AsmOp BlkPush(AnyV origin, int a, int b); + static AsmOp BlkDrop(AnyV origin, int a); + static AsmOp BlkDrop2(AnyV origin, int a, int b); + static AsmOp BlkReverse(AnyV origin, int a, int b); + static AsmOp make_stk2(AnyV origin, int a, int b, const char* str, int delta); + static AsmOp make_stk3(AnyV origin, int a, int b, int c, const char* str, int delta); + static AsmOp IntConst(AnyV origin, const td::RefInt256& x); + static AsmOp BoolConst(AnyV origin, bool f); + static AsmOp Const(AnyV origin, std::string push_op) { + return AsmOp(a_const, origin, 0, 1, std::move(push_op)); } - static AsmOp Const(SrcLocation loc, int arg, const std::string& push_op); - static AsmOp Comment(SrcLocation loc, const std::string& comment) { - return AsmOp(a_comment, loc, std::string{"// "} + comment); + static AsmOp Const(AnyV origin, int arg, const std::string& push_op); + static AsmOp Comment(AnyV origin, const std::string& comment) { + return AsmOp(a_comment, origin, std::string{"// "} + comment); } - static AsmOp Custom(SrcLocation loc, const std::string& custom_op) { - return AsmOp(a_custom, loc, 255, 255, custom_op); + static AsmOp Custom(AnyV origin, const std::string& custom_op) { + return AsmOp(a_custom, origin, 255, 255, custom_op); } - static AsmOp Parse(SrcLocation loc, const std::string& custom_op); - static AsmOp Custom(SrcLocation loc, const std::string& custom_op, int args, int retv = 1) { - return AsmOp(a_custom, loc, args, retv, custom_op); + static AsmOp Parse(AnyV origin, const std::string& custom_op); + static AsmOp Custom(AnyV origin, const std::string& custom_op, int args, int retv = 1) { + return AsmOp(a_custom, origin, args, retv, custom_op); } - static AsmOp Parse(SrcLocation loc, std::string custom_op, int args, int retv = 1); - static AsmOp Tuple(SrcLocation loc, int a); - static AsmOp UnTuple(SrcLocation loc, int a); + static AsmOp Parse(AnyV origin, std::string custom_op, int args, int retv = 1); + static AsmOp Tuple(AnyV origin, int a); + static AsmOp UnTuple(AnyV origin, int a); }; inline std::ostream& operator<<(std::ostream& os, const AsmOp& op) { @@ -578,7 +567,6 @@ inline std::ostream& operator<<(std::ostream& os, const AsmOp& op) { } std::ostream& operator<<(std::ostream& os, AsmOp::SReg stack_reg); -std::ostream& operator<<(std::ostream& os, TypePtr type_data); struct AsmOpList { std::vector list_; @@ -611,8 +599,8 @@ struct AsmOpList { void undent() { --indent_; } - void insert(size_t pos, SrcLocation loc, std::string str) { - insert(pos, AsmOp(AsmOp::a_custom, loc, 255, 255, std::move(str))); + void insert(size_t pos, AnyV origin, std::string str) { + insert(pos, AsmOp(AsmOp::a_custom, origin, 255, 255, std::move(str))); } void insert(size_t pos, const AsmOp& op) { auto ip = list_.begin() + pos; @@ -980,20 +968,20 @@ struct Stack { void modified() { mode &= ~_Shown; } - void issue_pop(SrcLocation loc, int i); - void issue_push(SrcLocation loc, int i); - void issue_xchg(SrcLocation loc, int i, int j); - int drop_vars_except(SrcLocation loc, const VarDescrList& var_info, int excl_var = 0x80000000); + void issue_pop(AnyV origin, int i); + void issue_push(AnyV origin, int i); + void issue_xchg(AnyV origin, int i, int j); + int drop_vars_except(AnyV origin, const VarDescrList& var_info, int excl_var = 0x80000000); void forget_var(var_idx_t idx); void push_new_var(var_idx_t idx); void push_new_const(var_idx_t idx, const_idx_t cidx); void assign_var(var_idx_t new_idx, var_idx_t old_idx); - void do_copy_var(SrcLocation loc, var_idx_t new_idx, var_idx_t old_idx); - void enforce_state(SrcLocation loc, const StackLayout& req_stack); - void rearrange_top(SrcLocation loc, const StackLayout& top, std::vector last); - void rearrange_top(SrcLocation loc, var_idx_t top, bool last); + void do_copy_var(AnyV origin, var_idx_t new_idx, var_idx_t old_idx); + void enforce_state(AnyV origin, const StackLayout& req_stack); + void rearrange_top(AnyV origin, const StackLayout& top, std::vector last); + void rearrange_top(AnyV origin, var_idx_t top, bool last); void merge_const(const Stack& req_stack); - void merge_state(SrcLocation loc, const Stack& req_stack); + void merge_state(AnyV origin, const Stack& req_stack); void show(); void opt_show() { if ((mode & (_StackComments | _Shown)) == _StackComments) { @@ -1003,26 +991,26 @@ struct Stack { bool operator==(const Stack& y) const & { return s == y.s; } - void apply_wrappers(SrcLocation loc, int callxargs_count) { + void apply_wrappers(AnyV origin, int callxargs_count) { int pos0 = (mode & _StackComments && !o.list_.empty() && o.list_[0].is_comment()) ? 1 : 0; bool is_inline = mode & _InlineFunc; if (o.retalt_inserted_) { - o.insert(pos0, loc, "SAMEALTSAVE"); - o.insert(pos0, loc, "c2 SAVE"); + o.insert(pos0, origin, "SAMEALTSAVE"); + o.insert(pos0, origin, "c2 SAVE"); } if (callxargs_count != -1 || (is_inline && o.retalt_)) { o.indent_all(); - o.insert(pos0, loc, "CONT:<{"); - o << AsmOp::Custom(loc, "}>"); + o.insert(pos0, origin, "CONT:<{"); + o << AsmOp::Custom(origin, "}>"); if (callxargs_count != -1) { if (callxargs_count <= 15) { - o << AsmOp::Custom(loc, PSTRING() << callxargs_count << " -1 CALLXARGS"); + o << AsmOp::Custom(origin, PSTRING() << callxargs_count << " -1 CALLXARGS"); } else { tolk_assert(callxargs_count <= 254); - o << AsmOp::Custom(loc, PSTRING() << callxargs_count << " PUSHINT -1 PUSHINT CALLXVARARGS"); + o << AsmOp::Custom(origin, PSTRING() << callxargs_count << " PUSHINT -1 PUSHINT CALLXVARARGS"); } } else { - o << AsmOp::Custom(loc, "EXECUTE"); + o << AsmOp::Custom(origin, "EXECUTE"); } } } @@ -1036,18 +1024,18 @@ struct Stack { */ struct FunctionBodyBuiltinAsmOp { - using CompileToAsmOpImpl = AsmOp(std::vector&, std::vector&, SrcLocation); + using CompileToAsmOpImpl = AsmOp(std::vector&, std::vector&, AnyV origin); std::function simple_compile; explicit FunctionBodyBuiltinAsmOp(std::function compile) : simple_compile(std::move(compile)) {} - void compile(AsmOpList& dest, std::vector& out, std::vector& in, SrcLocation loc) const; + void compile(AsmOpList& dest, std::vector& out, std::vector& in, AnyV origin) const; }; struct FunctionBodyBuiltinGenerateOps { - using GenerateOpsImpl = std::vector(FunctionPtr, CodeBlob&, SrcLocation, const std::vector>&); + using GenerateOpsImpl = std::vector(FunctionPtr, CodeBlob&, AnyV origin, const std::vector>&); std::function generate_ops; @@ -1059,7 +1047,7 @@ struct FunctionBodyAsm { std::vector ops; void set_code(std::vector&& code); - void compile(AsmOpList& dest, SrcLocation loc) const; + void compile(AsmOpList& dest, AnyV origin) const; }; struct LazyVariableLoadedState; @@ -1084,8 +1072,7 @@ struct CachedConstValueAtCodegen { struct CodeBlob { int var_cnt, in_var_cnt; FunctionPtr fun_ref; - std::string name; - SrcLocation forced_loc; + AnyV forced_origin = nullptr; std::vector vars; std::vector lazy_variables; std::vector cached_consts; @@ -1100,13 +1087,13 @@ struct CodeBlob { std::stack*> cur_ops_stack; bool require_callxargs = false; explicit CodeBlob(FunctionPtr fun_ref) - : var_cnt(0), in_var_cnt(0), fun_ref(fun_ref), name(fun_ref->name), cur_ops(&ops) { + : var_cnt(0), in_var_cnt(0), fun_ref(fun_ref), cur_ops(&ops) { } template Op& emplace_back(Args&&... args) { Op& res = *(*cur_ops = std::make_unique(args...)); - if (forced_loc.is_defined()) { - res.loc = forced_loc; + if (forced_origin) { + res.origin = forced_origin; } cur_ops = &(res.next); #ifdef TOLK_DEBUG @@ -1114,17 +1101,17 @@ struct CodeBlob { #endif return res; } - std::vector create_var(TypePtr var_type, SrcLocation loc, std::string name); - std::vector create_tmp_var(TypePtr var_type, SrcLocation loc, const char* desc) { - std::vector ir_idx = create_var(var_type, loc, {}); + std::vector create_var(TypePtr var_type, AnyV origin, std::string name); + std::vector create_tmp_var(TypePtr var_type, AnyV origin, const char* purpose) { + std::vector ir_idx = create_var(var_type, origin, {}); #ifdef TOLK_DEBUG for (var_idx_t v : ir_idx) { - vars[v].desc = desc; + vars[v].purpose = purpose; } #endif return ir_idx; } - var_idx_t create_int(SrcLocation loc, int64_t value, const char* desc); + var_idx_t create_int(AnyV origin, int64_t value, const char* purpose); bool compute_used_code_vars(); bool compute_used_code_vars(std::unique_ptr& ops, const VarDescrList& var_info, bool edit) const; void print(std::ostream& os, int flags = 0) const; @@ -1132,15 +1119,15 @@ struct CodeBlob { cur_ops_stack.push(cur_ops); cur_ops = &new_cur_ops; } - void close_blk(SrcLocation location) { - *cur_ops = std::make_unique(location, Op::_Nop); + void close_blk(AnyV origin) { + *cur_ops = std::make_unique(origin, Op::_Nop); } void pop_cur() { cur_ops = cur_ops_stack.top(); cur_ops_stack.pop(); } - void close_pop_cur(SrcLocation location) { - close_blk(location); + void close_pop_cur(AnyV origin) { + close_blk(origin); pop_cur(); } const LazyVariableLoadedState* get_lazy_variable(LocalVarPtr var_ref) const; @@ -1153,13 +1140,10 @@ struct CodeBlob { }; // defined in builtins.cpp -AsmOp exec_arg_op(SrcLocation loc, std::string op, long long arg, int args, int retv = 1); -AsmOp exec_arg_op(SrcLocation loc, std::string op, td::RefInt256 arg, int args, int retv = 1); -AsmOp exec_arg2_op(SrcLocation loc, std::string op, long long imm1, long long imm2, int args, int retv = 1); -AsmOp push_const(SrcLocation loc, td::RefInt256 x); - -void define_builtins(); -void patch_builtins_after_stdlib_loaded(); +AsmOp exec_arg_op(AnyV origin, std::string op, long long arg, int args, int retv = 1); +AsmOp exec_arg_op(AnyV origin, std::string op, td::RefInt256 arg, int args, int retv = 1); +AsmOp exec_arg2_op(AnyV origin, std::string op, long long imm1, long long imm2, int args, int retv = 1); +AsmOp push_const(AnyV origin, td::RefInt256 x); diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index ba796defd6..e0ad443ab1 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -15,7 +15,7 @@ along with TON Blockchain Library. If not, see . */ #include "type-system.h" -#include "platform-utils.h" +#include "compilation-errors.h" #include "generics-helpers.h" #include "compiler-state.h" #include diff --git a/tolk/type-system.h b/tolk/type-system.h index be043a643e..28ddbbf202 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -703,4 +703,6 @@ class TypeDataVoid final : public TypeData { void type_system_init(); +std::ostream& operator<<(std::ostream& os, TypePtr type_data); + } // namespace tolk From a2ae9464f2a93c8901730f727905330cbb4098c6 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 20 Oct 2025 16:18:03 +0300 Subject: [PATCH 451/524] [Tolk] Revise compilation errors and formatting Instead of `throw ParseError`, all errors/warnings are created using the same entrypoint: > err("format {}", arg).fire(at, current_fun) --- .../tests/invalid-declaration/err-1480.tolk | 3 +- tolk/abscode.cpp | 2 +- tolk/analyzer.cpp | 10 +- tolk/asmops.cpp | 10 - tolk/ast-from-tokens.cpp | 103 ++++---- tolk/ast-replicator.h | 1 - tolk/builtins.cpp | 32 +-- tolk/codegen.cpp | 2 +- tolk/compilation-errors.cpp | 106 +++++++-- tolk/compilation-errors.h | 81 +++++-- tolk/constant-evaluator.cpp | 43 ++-- tolk/generics-helpers.cpp | 9 +- tolk/generics-helpers.h | 4 +- tolk/lexer.cpp | 13 +- tolk/maps-kv-api.cpp | 2 +- tolk/overload-resolution.cpp | 2 +- tolk/pack-unpack-api.cpp | 16 +- tolk/pack-unpack-serializers.cpp | 6 +- tolk/pipe-ast-to-legacy.cpp | 8 +- tolk/pipe-check-inferred-types.cpp | 219 ++++++++---------- tolk/pipe-check-private-fields.cpp | 11 +- tolk/pipe-check-pure-impure.cpp | 16 +- tolk/pipe-check-rvalue-lvalue.cpp | 66 +++--- tolk/pipe-check-serialized-fields.cpp | 27 +-- tolk/pipe-constant-folding.cpp | 4 +- tolk/pipe-detect-inline-in-place.cpp | 2 +- tolk/pipe-generate-fif-output.cpp | 3 +- tolk/pipe-infer-types-and-calls.cpp | 157 +++++-------- tolk/pipe-lazy-load-insertions.cpp | 12 +- tolk/pipe-refine-lvalue-for-mutate.cpp | 11 +- tolk/pipe-register-symbols.cpp | 27 ++- tolk/pipe-resolve-identifiers.cpp | 46 ++-- tolk/pipe-resolve-types.cpp | 46 ++-- tolk/pipe-transform-on-message.cpp | 10 +- tolk/smart-casts-cfg.cpp | 2 +- tolk/src-file.cpp | 4 +- tolk/symtable.cpp | 16 +- tolk/symtable.h | 1 - tolk/tolk.cpp | 2 +- tolk/tolk.h | 1 - tolk/type-system.cpp | 38 ++- tolk/type-system.h | 2 - 42 files changed, 577 insertions(+), 599 deletions(-) diff --git a/tolk-tester/tests/invalid-declaration/err-1480.tolk b/tolk-tester/tests/invalid-declaration/err-1480.tolk index 667c5af2b9..709a653e77 100644 --- a/tolk-tester/tests/invalid-declaration/err-1480.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1480.tolk @@ -1,8 +1,9 @@ -enum C3 : int { +enum C3 : slice { } /** @compilation_should_fail @stderr serialization type of `enum` must be intN: `int8` / `uint32` / etc. +@stderr ^^^^^ */ diff --git a/tolk/abscode.cpp b/tolk/abscode.cpp index 5fa7706a7b..6fd0565196 100644 --- a/tolk/abscode.cpp +++ b/tolk/abscode.cpp @@ -379,7 +379,7 @@ void CodeBlob::print(std::ostream& os, int flags) const { if ((flags & 8) != 0) { for (const auto& var : vars) { var.show(os); - os << " : " << var.v_type << std::endl; + os << " : " << var.v_type->as_human_readable() << std::endl; } } os << "------- BEGIN --------\n"; diff --git a/tolk/analyzer.cpp b/tolk/analyzer.cpp index fdefc13b7c..356d5b017a 100644 --- a/tolk/analyzer.cpp +++ b/tolk/analyzer.cpp @@ -502,7 +502,7 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) { return set_var_info(std::move(merge_info)); } default: - fire(origin, "unknown operation in compute_used_vars()"); + err("unknown operation in compute_used_vars()").fire(origin); } } @@ -629,7 +629,7 @@ bool prune_unreachable(std::unique_ptr& ops) { break; } default: - fire(op.origin, "unknown operation in prune_unreachable()"); + err("unknown operation in prune_unreachable()").fire(op.origin); } if (reach) { return prune_unreachable(op.next); @@ -643,7 +643,7 @@ bool prune_unreachable(std::unique_ptr& ops) { void CodeBlob::prune_unreachable_code() { if (prune_unreachable(ops)) { - fire(fun_ref, fun_ref->ident_anchor, "control reaches end of function (stack is malformed, a compiler bug)"); + err("control reaches end of function (stack is malformed, a compiler bug)").fire(fun_ref->ident_anchor, fun_ref); } } @@ -835,7 +835,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { break; } default: - fire(origin, "unknown operation in fwd_analyze()"); + err("unknown operation in fwd_analyze()").fire(origin); } if (next) { return next->fwd_analyze(std::move(values)); @@ -932,7 +932,7 @@ bool Op::mark_noreturn() { block0->mark_noreturn(); return set_noreturn(next->mark_noreturn()); default: - fire(origin, "unknown operation in mark_noreturn()"); + err("unknown operation in mark_noreturn()").fire(origin); } } diff --git a/tolk/asmops.cpp b/tolk/asmops.cpp index 01d5973973..ac1c244047 100644 --- a/tolk/asmops.cpp +++ b/tolk/asmops.cpp @@ -285,16 +285,6 @@ int AsmOp::out_indented(std::ostream& os, bool print_src_line_above) const { return out(os) + indent * 2; } -std::string AsmOp::to_string() const { - if (!op.empty()) { - return op; - } else { - std::ostringstream os; - out(os); - return os.str(); - } -} - const_idx_t AsmOpList::register_const(td::RefInt256 new_const) { if (new_const.is_null()) { return not_const; diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 2cc42b101f..5e14fc76d7 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -17,7 +17,6 @@ #include "ast.h" #include "compilation-errors.h" #include "type-system.h" -#include "platform-utils.h" #include "tolk-version.h" /* @@ -50,34 +49,30 @@ static bool is_add_or_sub_binary_op(TokenType tok) { return tok == tok_plus || tok == tok_minus; } -// fire an error for a case "flags & 0xFF != 0" (equivalent to "flags & 1", probably unexpected) +// make an error for a case "flags & 0xFF != 0" (equivalent to "flags & 1", probably unexpected) // it would better be a warning, but we decided to make it a strict error -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_lower_precedence(SrcRange range, std::string_view op_lower, std::string_view op_higher) { - std::string name_lower = static_cast(op_lower); - std::string name_higher = static_cast(op_higher); - fire(range, name_lower + " has lower precedence than " + name_higher + +static Error err_lower_precedence(std::string_view op_lower, std::string_view op_higher) { + return err("{} has lower precedence than {}" ", probably this code won't work as you expected. " - "Use parenthesis: either (... " + name_lower + " ...) to evaluate it first, or (... " + name_higher + " ...) to suppress this error."); + "Use parenthesis: either (... {} ...) to evaluate it first, or (... {} ...) to suppress this error.", + op_lower, op_higher, op_lower, op_higher); } -// fire an error for a case "arg1 & arg2 | arg3" -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_mix_and_or_no_parenthesis(SrcRange range, std::string_view op1, std::string_view op2) { - std::string name1 = static_cast(op1); - std::string name2 = static_cast(op2); - fire(range, "mixing " + name1 + " with " + name2 + " without parenthesis may lead to accidental errors. " - "Use parenthesis to emphasize operator precedence."); +// make an error for a case "arg1 & arg2 | arg3" +static Error err_mix_and_or_no_parenthesis(std::string_view op1, std::string_view op2) { + return err("mixing {} with {} without parenthesis may lead to accidental errors. " + "Use parenthesis to emphasize operator precedence.", + op1, op2); } -// fire an error "Tolk does not have ++i operator" -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_no_increment_operator(SrcRange range, bool is_increment) { - std::string op_name = is_increment ? "increment" : "decrement"; - std::string op_wrong = is_increment ? "++" : "--"; - std::string op_right = is_increment ? "+=" : "-="; - fire(range, std::string("Tolk has no ") + op_name + " operator\n" + - "hint: use `i " + op_right + " 1`, not `i" + op_wrong + "`"); +// make an error "Tolk does not have ++i operator" +static Error err_no_increment_operator() { + return err("Tolk has no increment operator\n""hint: use `i += 1`, not `i++`"); +} + +// make an error "Tolk does not have --i operator" +static Error err_no_decrement_operator() { + return err("Tolk has no decrement operator\n""hint: use `i -= 1`, not `i--`"); } // diagnose when bitwise operators are used in a probably wrong way due to tricky precedence @@ -88,12 +83,12 @@ static void fire_no_increment_operator(SrcRange range, bool is_increment) { static void diagnose_bitwise_precedence(SrcRange range, std::string_view operator_name, AnyExprV lhs, AnyExprV rhs) { // handle "flags & 0xFF != 0" (rhs = "0xFF != 0") if (rhs->kind == ast_binary_operator && is_comparison_binary_op(rhs->as()->tok)) { - fire_lower_precedence(range, operator_name, rhs->as()->operator_name); + err_lower_precedence(operator_name, rhs->as()->operator_name).fire(range); } // handle "0 != flags & 0xFF" (lhs = "0 != flags") if (lhs->kind == ast_binary_operator && is_comparison_binary_op(lhs->as()->tok)) { - fire_lower_precedence(range, operator_name, lhs->as()->operator_name); + err_lower_precedence(operator_name, lhs->as()->operator_name).fire(range); } } @@ -104,12 +99,12 @@ static void diagnose_and_or_precedence(SrcRange range, AnyExprV lhs, TokenType r if (auto lhs_op = lhs->try_as()) { // handle "arg1 & arg2 | arg3" (lhs = "arg1 & arg2") if (is_bitwise_binary_op(lhs_op->tok) && is_bitwise_binary_op(rhs_tok) && lhs_op->tok != rhs_tok) { - fire_mix_and_or_no_parenthesis(range, lhs_op->operator_name, rhs_operator_name); + err_mix_and_or_no_parenthesis(lhs_op->operator_name, rhs_operator_name).fire(range); } // handle "arg1 && arg2 || arg3" (lhs = "arg1 && arg2") if (is_logical_binary_op(lhs_op->tok) && is_logical_binary_op(rhs_tok) && lhs_op->tok != rhs_tok) { - fire_mix_and_or_no_parenthesis(range, lhs_op->operator_name, rhs_operator_name); + err_mix_and_or_no_parenthesis(lhs_op->operator_name, rhs_operator_name).fire(range); } } } @@ -117,7 +112,7 @@ static void diagnose_and_or_precedence(SrcRange range, AnyExprV lhs, TokenType r // diagnose "a << 8 + 1" (equivalent to "a << 9", probably unexpected) static void diagnose_addition_in_bitshift(SrcRange range, std::string_view bitshift_operator_name, AnyExprV rhs) { if (rhs->kind == ast_binary_operator && is_add_or_sub_binary_op(rhs->as()->tok)) { - fire_lower_precedence(range, bitshift_operator_name, rhs->as()->operator_name); + err_lower_precedence(bitshift_operator_name, rhs->as()->operator_name).fire(range); } } @@ -401,7 +396,7 @@ static AnyV parse_global_var_declaration(Lexer& lex, const std::vector parse_maybe_instantiationTs_after_identifier(L range.end(lex.cur_range()); lex.next(); return createV(range, std::move(instantiationTs)); - } catch (const ParseError&) { + } catch (const ThrownParseError&) { lex.restore_position(backup); return nullptr; } @@ -727,7 +722,7 @@ static V parse_match_arm(Lexer& lex) { try { exact_type = parse_type_from_tokens(lex); pattern_kind = MatchArmKind::exact_type; - } catch (const ParseError&) { + } catch (const ThrownParseError&) { } if (!exact_type || lex.tok() != tok_double_arrow) { exact_type = nullptr; @@ -735,7 +730,7 @@ static V parse_match_arm(Lexer& lex) { try { pattern_expr = parse_expr(lex); pattern_kind = MatchArmKind::const_expression; // any expr at parsing, should result in const int/bool - } catch (const ParseError&) { + } catch (const ThrownParseError&) { } } if (!exact_type && !pattern_expr && lex.tok() == tok_else) { @@ -977,7 +972,7 @@ static AnyExprV parse_fun_call_postfix(Lexer& lex, AnyExprV lhs) { lex.next(); lhs = createV(range, lhs); } else if (lex.tok() == tok_double_plus || lex.tok() == tok_double_minus) { - fire_no_increment_operator(lex.cur_range(), lex.tok() == tok_double_plus); + lex.tok() == tok_double_plus ? err_no_increment_operator().fire(lex.cur_range()) : err_no_decrement_operator().fire(lex.cur_range()); } else { break; } @@ -1044,7 +1039,7 @@ static AnyExprV parse_expr75(Lexer& lex) { SrcRange range = lex.cur_range(); lex.next(); parse_expr75(lex); - fire_no_increment_operator(range, t == tok_double_plus); + t == tok_double_plus ? err_no_increment_operator().fire(range) : err_no_decrement_operator().fire(range); } return parse_expr80(lex); } @@ -1428,7 +1423,7 @@ static AnyV parse_asm_func_body(Lexer& lex, V name_ident, Vsize(); if (n_params > 16) { - fire(name_ident, "assembler built-in function can have at most 16 arguments"); + err("assembler built-in function can have at most 16 arguments").fire(name_ident); } std::vector arg_order, ret_order; if (lex.tok() == tok_oppar) { @@ -1488,13 +1483,13 @@ static V parse_annotation(Lexer& lex) { switch (kind) { case AnnotationKind::unknown: - fire(range, "unknown annotation " + static_cast(name)); + err("unknown annotation {}", name).fire(range); case AnnotationKind::inline_simple: case AnnotationKind::inline_ref: case AnnotationKind::noinline: case AnnotationKind::pure: if (v_arg) { - fire(range, "arguments aren't allowed for " + static_cast(name)); + err("arguments aren't allowed for {}", name).fire(range); } v_arg = createV(range, {}); break; @@ -1504,13 +1499,13 @@ static V parse_annotation(Lexer& lex) { break; case AnnotationKind::method_id: if (!v_arg || v_arg->size() != 1 || v_arg->get_item(0)->kind != ast_int_const) { - fire(range, "expecting `(number)` after " + static_cast(name)); + err("expecting `(number)` after {}", name).fire(range); } break; case AnnotationKind::overflow1023_policy: case AnnotationKind::on_bounced_policy: { if (!v_arg || v_arg->size() != 1 || v_arg->get_item(0)->kind != ast_string_const) { - fire(range, "expecting `(\"policy_name\")` after " + static_cast(name)); + err("expecting `(\"policy_name\")` after {}", name).fire(range); } break; } @@ -1532,7 +1527,7 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vectorget_arg()->get_item(0)->as(); if (v_int->intval.is_null() || !v_int->intval->signed_fits_bits(32)) { - fire(v_int, "invalid integer constant"); + err("invalid integer constant").fire(v_int); } tvm_method_id = v_int->intval; break; @@ -1648,10 +1643,10 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(v_annotation->name)); + err("incorrect value for {}", v_annotation->name).fire(v_annotation); } if (f_name != "onInternalMessage") { - fire(v_annotation, "this annotation is applicable only to onInternalMessage()"); + err("this annotation is applicable only to onInternalMessage()").fire(v_annotation); } break; } @@ -1660,7 +1655,7 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(v_annotation->name)); + err("incorrect value for {}", v_annotation->name).fire(v_annotation); } break; } default: - fire(v_annotation, "this annotation is not applicable to struct"); + err("this annotation is not applicable to struct").fire(v_annotation); } } @@ -1823,7 +1818,7 @@ static AnyV parse_enum_declaration(Lexer& lex, const std::vector= version" and so on, just strict compare if (TOLK_VERSION != semver && TOLK_VERSION != semver + ".0") { // 0.6 = 0.6.0 - compilation_warning(range, "the contract is written in Tolk v" + semver + ", but you use Tolk compiler v" + TOLK_VERSION + "; probably, it will lead to compilation errors or hash changes"); + err("the contract is written in Tolk v{}, but you use Tolk compiler v{}; probably, it will lead to compilation errors or hash changes", semver, TOLK_VERSION).warning(range, nullptr); } return createV(range, semver); // semicolon is not necessary @@ -1924,7 +1919,7 @@ AnyV parse_src_file_to_ast(const SrcFile* file) { case tok_export: case tok_operator: case tok_infix: - lex.error("`" + static_cast(lex.cur_str()) +"` is not supported yet"); + err("`{}` is not supported yet", lex.cur_str()).fire(lex.cur_range()); case tok_identifier: if (lex.cur_str() == "get") { // top-level "get", contract getter diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 2b03e4c939..2b7b003a8d 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -17,7 +17,6 @@ #pragma once #include "ast.h" -#include "platform-utils.h" namespace tolk { diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 7c7bb51ff5..bfc0d14d62 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -386,7 +386,7 @@ static AsmOp compile_add(std::vector& res, std::vector& args if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const + y.int_const); if (!r.int_const->is_valid()) { - fire(origin, "integer overflow"); + err("integer overflow").fire(origin); } x.unused(); y.unused(); @@ -428,7 +428,7 @@ static AsmOp compile_sub(std::vector& res, std::vector& args if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const - y.int_const); if (!r.int_const->is_valid()) { - fire(origin, "integer overflow"); + err("integer overflow").fire(origin); } x.unused(); y.unused(); @@ -461,7 +461,7 @@ static AsmOp compile_unary_minus(std::vector& res, std::vectoris_valid()) { - fire(origin, "integer overflow"); + err("integer overflow").fire(origin); } x.unused(); return push_const(origin, r.int_const); @@ -551,7 +551,7 @@ static AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, AnyV or if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const * y.int_const); if (!r.int_const->is_valid()) { - fire(origin, "integer overflow"); + err("integer overflow").fire(origin); } x.unused(); y.unused(); @@ -626,11 +626,11 @@ static AsmOp compile_lshift(std::vector& res, std::vector& a if (y.is_int_const()) { auto yv = y.int_const->to_long(); if (yv < 0 || yv > 256) { - fire(origin, "lshift argument is out of range"); + err("lshift argument is out of range").fire(origin); } else if (x.is_int_const()) { r.set_const(x.int_const << (int)yv); if (!r.int_const->is_valid()) { - fire(origin, "integer overflow"); + err("integer overflow").fire(origin); } x.unused(); y.unused(); @@ -669,7 +669,7 @@ static AsmOp compile_rshift(std::vector& res, std::vector& a if (y.is_int_const()) { auto yv = y.int_const->to_long(); if (yv < 0 || yv > 256) { - fire(origin, "rshift argument is out of range"); + err("rshift argument is out of range").fire(origin); } else if (x.is_int_const()) { r.set_const(td::rshift(x.int_const, (int)yv, round_mode)); x.unused(); @@ -696,7 +696,7 @@ static AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, AnyV or if (x.is_int_const() && y.is_int_const()) { r.set_const(div(x.int_const, y.int_const, round_mode)); if (!r.int_const->is_valid()) { - fire(origin, *y.int_const == 0 ? "division by zero" : "integer overflow"); + err(*y.int_const == 0 ? "division by zero" : "integer overflow").fire(origin); } x.unused(); y.unused(); @@ -705,7 +705,7 @@ static AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, AnyV or r.val = emulate_div(x.val, y.val); if (y.is_int_const()) { if (*y.int_const == 0) { - fire(origin, "division by zero"); + err("division by zero").fire(origin); } if (*y.int_const == 1 && x.always_finite()) { y.unused(); @@ -744,7 +744,7 @@ static AsmOp compile_mod(std::vector& res, std::vector& args if (x.is_int_const() && y.is_int_const()) { r.set_const(mod(x.int_const, y.int_const, round_mode)); if (!r.int_const->is_valid()) { - fire(origin, *y.int_const == 0 ? "division by zero" : "integer overflow"); + err(*y.int_const == 0 ? "division by zero" : "integer overflow").fire(origin); } x.unused(); y.unused(); @@ -753,7 +753,7 @@ static AsmOp compile_mod(std::vector& res, std::vector& args r.val = emulate_mod(x.val, y.val); if (y.is_int_const()) { if (*y.int_const == 0) { - fire(origin, "division by zero"); + err("division by zero").fire(origin); } if ((*y.int_const == 1 || *y.int_const == -1) && x.always_finite()) { x.unused(); @@ -785,7 +785,7 @@ static AsmOp compile_muldiv(std::vector& res, std::vector& a if (x.is_int_const() && y.is_int_const() && z.is_int_const()) { r.set_const(muldiv(x.int_const, y.int_const, z.int_const, round_mode)); if (!r.int_const->is_valid()) { - fire(origin, *z.int_const == 0 ? "division by zero" : "integer overflow"); + err(*z.int_const == 0 ? "division by zero" : "integer overflow").fire(origin); } x.unused(); y.unused(); @@ -804,7 +804,7 @@ static AsmOp compile_muldiv(std::vector& res, std::vector& a r.val = emulate_div(emulate_mul(x.val, y.val), z.val); if (z.is_int_const()) { if (*z.int_const == 0) { - fire(origin, "division by zero"); + err("division by zero").fire(origin); } if (*z.int_const == 1) { z.unused(); @@ -1192,7 +1192,7 @@ static AsmOp compile_slice_sdbeginsq(std::vector& res, std::vectorto_long(), static_cast(prefix_len.int_const->to_long())); return AsmOp::Custom(origin, opcode.format_as_slice() + " SDBEGINSQ", 0, 1); } - fire(origin, "slice.tryStripPrefix can be used only with constant arguments"); + err("slice.tryStripPrefix can be used only with constant arguments").fire(origin); } // fun slice.skipBits(mutate self, len: int): self "SDSKIPFIRST" @@ -1249,7 +1249,7 @@ static AsmOp compile_debug_print_to_string(std::vector&, std::vector 15) { - fire(origin, "call overflow, exceeds 15 elements"); + err("call overflow, exceeds 15 elements").fire(origin); } std::string cmd; for (int i = n - 1; i >= 0; --i) { @@ -1265,7 +1265,7 @@ static AsmOp compile_any_object_to_tuple(std::vector& res, std::vector tolk_assert(res.size() == 1); int n = static_cast(args.size()); if (n > 15) { - fire(origin, "call overflow, exceeds 15 elements"); + err("call overflow, exceeds 15 elements").fire(origin); } return exec_op(origin, std::to_string(args.size()) + " TUPLE", n, 1); } diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp index 9c944ab034..95f7e12eac 100644 --- a/tolk/codegen.cpp +++ b/tolk/codegen.cpp @@ -884,7 +884,7 @@ bool Op::generate_code_step(Stack& stack) { return true; } default: - fire(origin, "unknown operation in generate_code()"); + err("unknown operation in generate_code()").fire(origin); } } diff --git a/tolk/compilation-errors.cpp b/tolk/compilation-errors.cpp index 070df57943..68cffff70f 100644 --- a/tolk/compilation-errors.cpp +++ b/tolk/compilation-errors.cpp @@ -16,9 +16,14 @@ */ #include "compilation-errors.h" #include "ast.h" +#include "type-system.h" namespace tolk { +static std::string to_string(FunctionPtr f) { + return f ? f->as_human_readable() : ""; +} + void on_assertion_failed(const char *description, const char *file_name, int line_number) { std::string message = static_cast("Assertion failed at ") + file_name + ":" + std::to_string(line_number) + ": " + description; #ifdef TOLK_DEBUG @@ -66,47 +71,104 @@ static void output_compiler_message( range.output_underlined(os); } +void ErrorBuilder::push(const char* v) { + add_arg(v); +} -void fire(AnyV at, std::string message) { - throw ParseError(at->range, std::move(message)); +void ErrorBuilder::push(const std::string& v) { + add_arg(v); } -void fire(SrcRange range, std::string message) { - throw ParseError(range, std::move(message)); +void ErrorBuilder::push(std::string_view v) { + add_arg(static_cast(v)); } -void fire(FunctionPtr in_function, AnyV at, std::string message) { - std::string f = in_function ? in_function->as_human_readable() : ""; - throw ParseError(f, at->range, std::move(message)); +void ErrorBuilder::push(TypePtr v) { + add_arg(v->as_human_readable()); } -void fire(FunctionPtr in_function, SrcRange range, std::string message) { - std::string f = in_function ? in_function->as_human_readable() : ""; - throw ParseError(f, range, std::move(message)); +void ErrorBuilder::push(FunctionPtr v) { + add_arg(v->as_human_readable()); } -void ParseError::output_compilation_error(std::ostream& os) const { - output_compiler_message(os, false, in_function, range, message); +void ErrorBuilder::push(StructPtr v) { + add_arg(v->as_human_readable()); +} + +void ErrorBuilder::push(StructFieldPtr v) { + add_arg(v->name); +} + +void ErrorBuilder::push(AliasDefPtr v) { + add_arg(v->as_human_readable()); +} + +void ErrorBuilder::push(EnumDefPtr v) { + add_arg(v->as_human_readable()); +} + +void ErrorBuilder::push(EnumMemberPtr v) { + add_arg(v->name); +} + +void ErrorBuilder::push(GlobalConstPtr v) { + add_arg(v->name); +} + +void ErrorBuilder::push(LocalVarPtr v) { + add_arg(v->name); } +void ErrorBuilder::push(int v) { + add_arg(std::to_string(v)); +} + +void ErrorBuilder::push(size_t v) { + add_arg(std::to_string(v)); +} + +void ErrorBuilder::push(bool v) { + add_arg(v ? "true" : "false"); +} + +Error ErrorBuilder::build() const { + std::string replaced = tpl; + size_t arg_i = 0, pos; + while ((pos = replaced.find("{}")) != std::string::npos) { + if (arg_i >= args.size()) { +#ifdef TOLK_DEBUG + throw Fatal(std::string("mismatch err() tpl: ") + tpl); +#endif + break; + } + replaced.replace(pos, 2, args[arg_i++]); + } +#ifdef TOLK_DEBUG + if (arg_i != args.size()) { + throw Fatal(std::string("mismatch err() tpl: ") + tpl); + } +#endif + return Error(std::move(replaced)); +} -void compilation_warning(AnyV at, const std::string& message) { - output_compiler_message(std::cerr, true, "", at->range, message); +void Error::fire(AnyV at, FunctionPtr in_function) const { + throw ThrownParseError(to_string(in_function), at->range, message); } -void compilation_warning(SrcRange range, const std::string& message) { - output_compiler_message(std::cerr, true, "", range, message); +void Error::fire(SrcRange range, FunctionPtr in_function) const { + throw ThrownParseError(to_string(in_function), range, message); } -void compilation_warning(FunctionPtr in_function, AnyV at, const std::string& message) { - std::string f = in_function ? in_function->as_human_readable() : ""; - output_compiler_message(std::cerr, true, f, at->range, message); +void Error::warning(AnyV at, FunctionPtr in_function) const { + output_compiler_message(std::cerr, true, to_string(in_function), at->range, message); } -void compilation_warning(FunctionPtr in_function, SrcRange range, const std::string& message) { - std::string f = in_function ? in_function->as_human_readable() : ""; - output_compiler_message(std::cerr, true, f, range, message); +void Error::warning(SrcRange range, FunctionPtr in_function) const { + output_compiler_message(std::cerr, true, to_string(in_function), range, message); } +void ThrownParseError::output_compilation_error(std::ostream& os) const { + output_compiler_message(os, false, in_function, range, message); +} } // namespace tolk diff --git a/tolk/compilation-errors.h b/tolk/compilation-errors.h index 6c9406940e..b52f0797f4 100644 --- a/tolk/compilation-errors.h +++ b/tolk/compilation-errors.h @@ -19,6 +19,8 @@ #include "fwd-declarations.h" #include "src-file.h" #include "platform-utils.h" +#include +#include namespace tolk { @@ -27,20 +29,65 @@ namespace tolk { GNU_ATTRIBUTE_COLD GNU_ATTRIBUTE_NORETURN void on_assertion_failed(const char *description, const char *file_name, int line_number); -// fire a general non-recoverable error, just a wrapper over `throw` -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -void fire(AnyV at, std::string message); -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -void fire(SrcRange range, std::string message); -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -void fire(FunctionPtr in_function, AnyV at, std::string message); -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -void fire(FunctionPtr in_function, SrcRange range, std::string message); - -void compilation_warning(AnyV at, const std::string& message); -void compilation_warning(SrcRange range, const std::string& message); -void compilation_warning(FunctionPtr function, AnyV at, const std::string& message); -void compilation_warning(FunctionPtr function, SrcRange range, const std::string& message); +class [[nodiscard]] Error { + std::string message; + +public: + explicit Error(std::string message) + : message(std::move(message)) {} + + GNU_ATTRIBUTE_NORETURN + void fire(AnyV at, FunctionPtr in_function = nullptr) const; + GNU_ATTRIBUTE_NORETURN + void fire(SrcRange range, FunctionPtr in_function = nullptr) const; + + void warning(AnyV at, FunctionPtr in_function = nullptr) const; + void warning(SrcRange range, FunctionPtr in_function = nullptr) const; +}; + +class ErrorBuilder { + const char* tpl; + std::vector args; + + void add_arg(std::string v) { + args.push_back(std::move(v)); + } + +public: + explicit ErrorBuilder(const char* tpl) + : tpl(tpl) {} + + void push(const char* v); + void push(const std::string& v); + void push(std::string_view v); + void push(TypePtr v); + void push(FunctionPtr v); + void push(StructPtr v); + void push(StructFieldPtr v); + void push(AliasDefPtr v); + void push(EnumDefPtr v); + void push(EnumMemberPtr v); + void push(GlobalConstPtr v); + void push(LocalVarPtr v); + void push(int v); + void push(size_t v); + void push(bool v); + void push(void*) = delete; + void push(const void*) = delete; + void push(std::nullptr_t) = delete; + + [[nodiscard]] Error build() const; +}; + + +template +GNU_ATTRIBUTE_COLD +inline Error err(const char* tpl, Args&&... args) { + ErrorBuilder b(tpl); + (b.push(std::forward(args)), ...); + return b.build(); +} + struct Fatal final : std::exception { std::string message; @@ -53,14 +100,12 @@ struct Fatal final : std::exception { } }; -struct ParseError final : std::exception { +struct ThrownParseError final : std::exception { std::string in_function; SrcRange range; std::string message; - ParseError(SrcRange range, std::string message) - : range(range), message(std::move(message)) {} - ParseError(std::string in_function, SrcRange range, std::string message) + ThrownParseError(std::string in_function, SrcRange range, std::string message) : in_function(std::move(in_function)), range(range), message(std::move(message)) {} const char* what() const noexcept override { diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index e6ad6a21dc..924760a6f8 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -25,11 +25,8 @@ namespace tolk { -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_const_string_required(SrcRange range, std::string_view f_name, std::string_view example_arg) { - std::string name = static_cast(f_name); - std::string example = static_cast(example_arg); - fire(range, "function `" + name + "` requires a constant string, like `" + name + "(\"" + example + "\")`"); +static Error err_const_string_required(std::string_view f_name, std::string_view example_arg) { + return err("function `{}` requires a constant string, like `{}(\"{}\")`", f_name, f_name, example_arg); } @@ -97,10 +94,10 @@ static void parse_any_std_address(std::string_view str, SrcRange range, unsigned bool correct = (str.size() == 48 && parse_friendly_address(str.data(), workchain, addr)) || (str.size() != 48 && parse_raw_address(str, workchain, addr)); if (!correct) { - fire(range, "invalid standard address"); + err("invalid standard address").fire(range); } if (workchain < -128 || workchain >= 128) { - fire(range, "anycast addresses not supported"); + err("anycast addresses not supported").fire(range); } td::bitstring::bits_store_long_top(*data, 0, static_cast(4) << (64 - 3), 3); @@ -132,21 +129,21 @@ static td::RefInt256 parse_nanotons_as_floating_string(SrcRange range, std::stri char c = str[i]; if (c == '.') { if (seen_dot) { - fire(range, "argument is not a valid number like \"0.05\""); + err("argument is not a valid number like \"0.05\"").fire(range); } seen_dot = true; } else if (c >= '0' && c <= '9') { if (!seen_dot) { integer_part = integer_part * 10 + (c - '0'); if (++integer_digits > 9) { - fire(range, "argument is too big and leads to overflow"); + err("argument is too big and leads to overflow").fire(range); } } else if (fractional_digits < 9) { fractional_part = fractional_part * 10 + (c - '0'); fractional_digits++; } } else { - fire(range, "argument is not a valid number like \"0.05\""); + err("argument is not a valid number like \"0.05\"").fire(range); } } @@ -171,7 +168,7 @@ static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(Vtry_as(); if (!t_struct || !t_struct->struct_ref->opcode.exists()) { - fire(v, "type `" + receiver->as_human_readable() + "` does not have a serialization prefix"); + err("type `{}` does not have a serialization prefix", receiver).fire(v); } return td::make_refint(f_name.ends_with('x') ? t_struct->struct_ref->opcode.pack_prefix : t_struct->struct_ref->opcode.prefix_len); } @@ -189,7 +186,7 @@ static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(Vrange, f_name, f_name == "ton" ? "0.05" : "some_str"); + err_const_string_required(f_name, f_name == "ton" ? "0.05" : "some_str").fire(v); } if (f_name == "ton") { @@ -226,7 +223,7 @@ static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(V(str); } @@ -234,10 +231,10 @@ static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(V v) { GlobalConstPtr const_ref = v->sym->try_as(); if (!const_ref) { - fire(v, "symbol `" + static_cast(v->get_name()) + "` is not a constant"); + err("symbol `{}` is not a constant", v->get_name()).fire(v); } } @@ -270,7 +267,7 @@ struct ConstantExpressionChecker { // `ton(local_var)` is denied; it's validated not here, but when replacing its value with a calculated one return; } - fire(v, "not a constant expression"); + err("not a constant expression").fire(v); } // `const a = (1, 2)`, why not; or it's a default value of a field @@ -323,7 +320,7 @@ struct ConstantExpressionChecker { return; } } - fire(v, "not a constant expression"); + err("not a constant expression").fire(v); } // check that `2 + 3` is constant @@ -344,7 +341,7 @@ struct EnumMemberEvaluator { case tok_bitwise_not: return ~rhs; case tok_plus: return rhs; default: - fire(v, "unsupported operator in `enum` member value"); + err("unsupported operator in `enum` member value").fire(v); } } @@ -366,7 +363,7 @@ struct EnumMemberEvaluator { case tok_lshift: return lhs << static_cast(rhs->to_long()); case tok_rshift: return lhs >> static_cast(rhs->to_long()); default: - fire(v, "unsupported operator in `enum` member value"); + err("unsupported operator in `enum` member value").fire(v); } } @@ -374,7 +371,7 @@ struct EnumMemberEvaluator { static td::RefInt256 handle_reference(V v) { GlobalConstPtr const_ref = v->sym->try_as(); if (!const_ref) { - fire(v, "symbol `" + static_cast(v->get_name()) + "` is not a constant"); + err("symbol `{}` is not a constant", v->get_name()).fire(v); } return visit(const_ref->init_value); } @@ -399,10 +396,10 @@ struct EnumMemberEvaluator { // `enum E { V = AnotherEnum.A }` not allowed (can be allowed in the future, // but remember to construct an initialization tree, since A may be auto-computed, and prevent recursion) if (v_dot->is_target_enum_member()) { - fire(v, "referencing other members in initializers is not allowed yet"); + err("referencing other members in initializers is not allowed yet").fire(v); } } - fire(v, "unsupported operation in `enum` member value"); + err("unsupported operation in `enum` member value").fire(v); } }; diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index 74928a2e14..de2a8444e3 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -297,7 +297,7 @@ TypePtr GenericSubstitutionsDeducing::auto_deduce_from_argument(FunctionPtr cur_ consider_next_condition(param_type, arg_type); param_type = replace_genericT_with_deduced(param_type, &deducedTs, true, &unknown_nameT); if (param_type->has_genericT_inside()) { - fire_can_not_deduce(cur_f, range, unknown_nameT); + err_can_not_deduce(unknown_nameT).fire(range, cur_f); } return param_type; } @@ -315,12 +315,11 @@ void GenericSubstitutionsDeducing::apply_defaults_from_declaration() { deducedTs.rewrite_missing_with_defaults(); } -// todo pass at, not ra -void GenericSubstitutionsDeducing::fire_can_not_deduce(FunctionPtr cur_f, SrcRange range, std::string_view nameT) const { +Error GenericSubstitutionsDeducing::err_can_not_deduce(std::string_view nameT) const { if (fun_ref) { - fire(cur_f, range, "can not deduce " + static_cast(nameT) + " for generic function `" + fun_ref->as_human_readable() + "`; instantiate it manually with `" + fun_ref->name + "<...>()`"); + return err("can not deduce {} for generic function `{}`; instantiate it manually with `{}<...>()`", nameT, fun_ref, fun_ref->name); } else { - fire(cur_f, range, "can not deduce " + static_cast(nameT) + " for generic struct `" + struct_ref->as_human_readable() + "`; instantiate it manually with `" + struct_ref->name + "<...>`"); + return err("can not deduce {} for generic struct `{}`; instantiate it manually with `{}<...>`", nameT, struct_ref, struct_ref->name); } } diff --git a/tolk/generics-helpers.h b/tolk/generics-helpers.h index 06bec2511b..df1cb516e7 100644 --- a/tolk/generics-helpers.h +++ b/tolk/generics-helpers.h @@ -18,7 +18,7 @@ #include "src-file.h" #include "fwd-declarations.h" -#include "td/utils/Status.h" +#include "compilation-errors.h" #include namespace tolk { @@ -104,7 +104,7 @@ class GenericSubstitutionsDeducing { TypePtr auto_deduce_from_argument(FunctionPtr cur_f, SrcRange range, TypePtr param_type, TypePtr arg_type); std::string_view get_first_not_deduced_nameT() const; void apply_defaults_from_declaration(); - void fire_can_not_deduce(FunctionPtr cur_f, SrcRange range, std::string_view nameT) const; + Error err_can_not_deduce(std::string_view nameT) const; GenericsSubstitutions&& flush() { return std::move(deducedTs); diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index 14d0440a14..02ecc2a2e2 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -16,7 +16,6 @@ */ #include "lexer.h" #include "compilation-errors.h" -#include #include #include @@ -75,7 +74,7 @@ class LexingTrie { } #ifdef TOLK_DEBUG - assert(!cur->val); + tolk_assert(!cur->val); #endif cur->val = val; } @@ -459,7 +458,7 @@ struct TolkLanguageGrammar { case tok_semver: return ChunkSpecialParsing::parse_semver(lex); default: - assert(false); + tolk_assert(false); return false; } } @@ -575,7 +574,7 @@ void Lexer::next() { } void Lexer::next_special(TokenType parse_next_as, const char* str_expected) { - assert(cur_token_idx == last_token_idx); + tolk_assert(cur_token_idx == last_token_idx); skip_spaces(); update_location(); if (!TolkLanguageGrammar::parse_next_chunk_special(this, parse_next_as)) { @@ -597,17 +596,17 @@ void Lexer::restore_position(SavedPositionForLookahead saved) { void Lexer::hack_replace_rshift_with_one_triangle() { // overcome the `>>` problem when parsing generics, leave only `>` here, see comments at usage - assert(cur_token.type == tok_rshift); + tolk_assert(cur_token.type == tok_rshift); cur_token = Token(tok_gt, ">"); cur_token_offset++; } void Lexer::error(const std::string& err_msg) const { - fire(cur_range(), err_msg); + err("{}", err_msg).fire(cur_range()); } void Lexer::unexpected(const char* str_expected) const { - fire(cur_range(), "expected " + std::string(str_expected) + ", got `" + std::string(cur_str()) + "`"); + err("expected {}, got `{}`", str_expected, cur_str()).fire(cur_range()); } void lexer_init() { diff --git a/tolk/maps-kv-api.cpp b/tolk/maps-kv-api.cpp index 950290edee..ce8ed12716 100644 --- a/tolk/maps-kv-api.cpp +++ b/tolk/maps-kv-api.cpp @@ -117,7 +117,7 @@ bool check_mapKV_TKey_is_valid(TypePtr TKey, std::string& because_msg) { // okay, not a trivial key — it must be a serializable struct of a constant size if (TKey->unwrap_alias() == TypeDataSlice::create()) { // a dedicated error message for `map` - because_msg = "because it does not specify keyLen for a dictionary\nhint: use `address` if a key is an internal address\nhint: use `bits128` and similar if a key represents fixed-width data"; + because_msg = "because it does not specify keyLen for a dictionary\n""hint: use `address` if a key is an internal address\n""hint: use `bits128` and similar if a key represents fixed-width data"; return false; } if (!check_struct_can_be_packed_or_unpacked(TKey, false, because_msg)) { diff --git a/tolk/overload-resolution.cpp b/tolk/overload-resolution.cpp index ed98bf250c..9bb05c50e9 100644 --- a/tolk/overload-resolution.cpp +++ b/tolk/overload-resolution.cpp @@ -222,7 +222,7 @@ std::vector resolve_methods_for_call(TypePtr provided_recei } } if (dominates_all) { - assert(!dominator); + tolk_assert(!dominator); dominator = &candidate; } } diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index 8d68529182..4208df7b29 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -170,20 +170,20 @@ class PackUnpackAvailabilityChecker { if (FunctionPtr f_pack = get_custom_pack_unpack_function(t_alias, true)) { std::string receiver_name = t_alias->alias_ref->as_human_readable(); if (!check_declared_packToBuilder(t_alias, f_pack)) { - return CantSerializeBecause("because `" + receiver_name + ".packToBuilder()` is declared incorrectly\nhint: it must accept 2 parameters and return nothing:\n> fun " + receiver_name + ".packToBuilder(self, mutate b: builder)"); + return CantSerializeBecause("because `" + receiver_name + ".packToBuilder()` is declared incorrectly\n""hint: it must accept 2 parameters and return nothing:\n> fun " + receiver_name + ".packToBuilder(self, mutate b: builder)"); } if (!f_pack->is_inlined_in_place()) { return CantSerializeBecause("because `" + receiver_name + ".packToBuilder()` can't be inlined; probably, it contains `return` in the middle"); } if (FunctionPtr f_unpack = get_custom_pack_unpack_function(t_alias, false)) { if (!check_declared_unpackFromSlice(t_alias, f_unpack)) { - return CantSerializeBecause("because `" + receiver_name + ".unpackFromSlice()` is declared incorrectly\nhint: it must accept 1 parameter and return an object:\n> fun " + receiver_name + ".unpackFromSlice(mutate s: slice): " + receiver_name); + return CantSerializeBecause("because `" + receiver_name + ".unpackFromSlice()` is declared incorrectly\n""hint: it must accept 1 parameter and return an object:\n> fun " + receiver_name + ".unpackFromSlice(mutate s: slice): " + receiver_name); } if (!f_unpack->is_inlined_in_place()) { return CantSerializeBecause("because `" + receiver_name + ".unpackFromSlice()` can't be inlined; probably, it contains `return` in the middle"); } } else if (!is_pack) { - return CantSerializeBecause("because type `" + receiver_name + "` defines a custom pack function, but does not define unpack\nhint: declare unpacker like this:\n> fun " + receiver_name + ".unpackFromSlice(mutate s: slice): " + receiver_name); + return CantSerializeBecause("because type `" + receiver_name + "` defines a custom pack function, but does not define unpack\n""hint: declare unpacker like this:\n> fun " + receiver_name + ".unpackFromSlice(mutate s: slice): " + receiver_name); } return {}; } @@ -198,26 +198,26 @@ class PackUnpackAvailabilityChecker { if (is_pack) { return {}; } - return CantSerializeBecause("because type `builder` can not be used for reading, only for writing\nhint: use `bitsN` or `RemainingBitsAndRefs` for reading\nhint: using generics, you can substitute `builder` for writing and something other for reading"); + return CantSerializeBecause("because type `builder` can not be used for reading, only for writing\n""hint: use `bitsN` or `RemainingBitsAndRefs` for reading\n""hint: using generics, you can substitute `builder` for writing and something other for reading"); } if (any_type == TypeDataSlice::create()) { if (is_pack) { return {}; } - return CantSerializeBecause("because type `slice` can not be used for reading, it doesn't define binary width\nhint: replace `slice` with `address` if it's an address, actually\nhint: replace `slice` with `bits128` and similar if it represents fixed-width data without refs"); + return CantSerializeBecause("because type `slice` can not be used for reading, it doesn't define binary width\n""hint: replace `slice` with `address` if it's an address, actually\n""hint: replace `slice` with `bits128` and similar if it represents fixed-width data without refs"); } // serialization not available // for common types, make a detailed explanation with a hint how to fix if (any_type == TypeDataInt::create()) { - return CantSerializeBecause("because type `int` is not serializable, it doesn't define binary width\nhint: replace `int` with `int32` / `uint64` / `coins` / etc."); + return CantSerializeBecause("because type `int` is not serializable, it doesn't define binary width\n""hint: replace `int` with `int32` / `uint64` / `coins` / etc."); } if (any_type == TypeDataNullLiteral::create()) { - return CantSerializeBecause("because type `null` is not serializable\nhint: `int32?` and other nullable types will work"); + return CantSerializeBecause("because type `null` is not serializable\n""hint: `int32?` and other nullable types will work"); } if (any_type == TypeDataTuple::create() || any_type->try_as()) { - return CantSerializeBecause("because tuples are not serializable\nhint: use tensors instead of tuples, they will work"); + return CantSerializeBecause("because tuples are not serializable\n""hint: use tensors instead of tuples, they will work"); } return CantSerializeBecause("because type `" + any_type->as_human_readable() + "` is not serializable"); diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index 3f04bb935f..3a8dc338ff 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -714,7 +714,7 @@ struct S_Either final : ISerializer { for (const LazyMatchOptions::MatchBlock& m : options.match_blocks) { if (m.arm_variant == nullptr) { // `else => ...` not allowed for Either // it's not the best place to fire an error, but let it be - fire(SrcRange::empty_at_start(m.v_body->range), "`else` is unreachable, because this `match` has only two options (0/1 prefixes)"); + err("`else` is unreachable, because this `match` has only two options (0/1 prefixes)").fire(SrcRange::empty_at_start(m.v_body->range)); } } tolk_assert(options.match_blocks.size() == 2); @@ -1209,9 +1209,9 @@ std::vector auto_generate_opcodes_for_union(TypePtr union_type, std: // example: `int32 | int64 | A` if A has opcode; if (n_have_opcode) { if (last_struct_with_opcode && last_struct_no_opcode) { - because_msg = "because struct `" + last_struct_with_opcode->as_human_readable() + "` has opcode, but `" + last_struct_no_opcode->as_human_readable() + "` does not\nhint: manually specify opcodes to all structures"; + because_msg = "because struct `" + last_struct_with_opcode->as_human_readable() + "` has opcode, but `" + last_struct_no_opcode->as_human_readable() + "` does not\n""hint: manually specify opcodes to all structures"; } else { - because_msg = "because of mixing primitives and struct `" + last_struct_with_opcode->as_human_readable() + "` with serialization prefix\nhint: extract primitives to single-field structs and provide prefixes"; + because_msg = "because of mixing primitives and struct `" + last_struct_with_opcode->as_human_readable() + "` with serialization prefix\n""hint: extract primitives to single-field structs and provide prefixes"; } return result; } diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index bc37e47715..4e5efd03b3 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -636,7 +636,7 @@ static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob if (f_name == "T.forceLoadLazyObject") { // in: object T, out: slice (same slice that a lazy variable holds, after loading/skipping all its fields) if (!lazy_variable) { - fire(code.fun_ref, v_call->get_callee(), "this method is applicable to lazy variables only"); + err("this method is applicable to lazy variables only").fire(v_call->get_callee(), code.fun_ref); } std::vector ir_obj = vars_per_arg[0]; return generate_lazy_object_finish_loading(code, v_call, lazy_variable, std::move(ir_obj)); @@ -1446,7 +1446,7 @@ static std::vector process_match_expression(V v // because it might turned out to be a lazy match, where `else` is allowed; // if we are here, it's not a lazy match, it's a regular one (the lazy one is handled specially, in aux vertex) if (is_match_by_type && has_else_arm) { - fire(v->get_arm(n_arms - 1)->get_pattern_expr(), "`else` is not allowed in `match` by type; you should cover all possible types"); + err("`else` is not allowed in `match` by type; you should cover all possible types").fire(v->get_arm(n_arms - 1)->get_pattern_expr()); } // example 1 (exhaustive): `match (v) { int => ... slice => ... builder => ... }` @@ -1772,7 +1772,7 @@ static std::vector process_tensor(V v, CodeBlob& code, Ty static std::vector process_typed_tuple(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { if (lval_ctx) { // todo some time, make "var (a, [b,c]) = (1, [2,3])" work - fire(v, "[...] can not be used as lvalue here"); + err("[...] can not be used as lvalue here").fire(v); } std::vector ir_left = code.create_tmp_var(v->inferred_type, v, "(pack-tuple)"); std::vector ir_right = pre_compile_tensor(code, v->get_items(), lval_ctx); @@ -2432,7 +2432,7 @@ class UpdateArgRetOrderConsideringStackWidth final { if (!fun_ref->ret_order.empty()) { size_t expected_width = fun_ref->inferred_return_type->get_width_on_stack() + total_arg_mutate_width; if (expected_width != fun_ref->ret_order.size()) { - fire(v_function->get_body(), "ret_order (after ->) expected to contain " + std::to_string(expected_width) + " numbers"); + err("ret_order (after ->) expected to contain {} numbers", expected_width).fire(v_function->get_body()); } } } diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index afd181fde3..081e333a4b 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -21,22 +21,6 @@ namespace tolk { -GNU_ATTRIBUTE_NOINLINE -static std::string to_string(TypePtr type) { - return "`" + type->as_human_readable() + "`"; -} - -GNU_ATTRIBUTE_NOINLINE -static std::string to_string(AnyExprV v_with_type) { - return "`" + v_with_type->inferred_type->as_human_readable() + "`"; -} - -GNU_ATTRIBUTE_NOINLINE -static std::string to_string(std::string_view string_view) { - return static_cast(string_view); -} - -GNU_ATTRIBUTE_NOINLINE static std::string expression_as_string(AnyExprV v) { if (auto v_ref = v->try_as()) { if (v_ref->sym->try_as() || v_ref->sym->try_as()) { @@ -49,51 +33,45 @@ static std::string expression_as_string(AnyExprV v) { return "expression"; } -// fire a general error on type mismatch; for example, "can not assign `cell` to `slice`"; +// make a general error on type mismatch; for example, "can not assign `cell` to `slice`"; // for instance, if `as` operator is applicable, compiler will suggest it -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -// todo not range but at -static void fire_type_mismatch(FunctionPtr cur_f, SrcRange range, const char* text_tpl, TypePtr src, TypePtr dst) { +static Error err_type_mismatch(const char* text_tpl, TypePtr src, TypePtr dst) { #ifdef TOLK_DEBUG tolk_assert(!dst->can_rhs_be_assigned(src)); #endif std::string message = text_tpl; - message.replace(message.find("{src}"), 5, to_string(src)); - message.replace(message.find("{dst}"), 5, to_string(dst)); + message.replace(message.find("{src}"), 5, "`" + src->as_human_readable() + "`"); + message.replace(message.find("{dst}"), 5, "`" + dst->as_human_readable() + "`"); if (src->can_be_casted_with_as_operator(dst)) { bool suggest_as = !dst->try_as() && !dst->try_as(); if ((src == TypeDataAddress::create() || src == TypeDataSlice::create()) && (dst == TypeDataAddress::create() || dst == TypeDataSlice::create())) { - message += "\nhint: unlike FunC, Tolk has a special type `address` (which is slice at the TVM level);"; - message += "\n most likely, you just need `address` everywhere"; - message += "\nhint: alternatively, use `as` operator for unsafe casting: ` as " + dst->as_human_readable() + "`"; + message += "\n""hint: unlike FunC, Tolk has a special type `address` (which is slice at the TVM level);"; + message += "\n"" most likely, you just need `address` everywhere"; + message += "\n""hint: alternatively, use `as` operator for unsafe casting: ` as " + dst->as_human_readable() + "`"; } else if (suggest_as) { - message += "\nhint: use `as` operator for unsafe casting: ` as " + dst->as_human_readable() + "`"; + message += "\n""hint: use `as` operator for unsafe casting: ` as " + dst->as_human_readable() + "`"; } if (src == TypeDataBool::create() && dst == TypeDataInt::create()) { - message += "\ncaution! in TVM, bool TRUE is -1, not 1"; + message += "\n""caution! in TVM, bool TRUE is -1, not 1"; } } if (const TypeDataUnion* src_nullable = src->try_as(); src_nullable && src_nullable->or_null) { if (dst->can_rhs_be_assigned(src_nullable->or_null)) { - message += "\nhint: probably, you should check on null"; - message += "\nhint: alternatively, use `!` operator to bypass nullability checks: `!`"; + message += "\n""hint: probably, you should check on null"; + message += "\n""hint: alternatively, use `!` operator to bypass nullability checks: `!`"; } } - fire(cur_f, range, message); + return err("{}", message); } -// fire an error on `!cell` / `+slice` -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_cannot_apply_operator(FunctionPtr cur_f, SrcRange operator_range, std::string_view operator_name, AnyExprV unary_expr) { - std::string op = static_cast(operator_name); - fire(cur_f, operator_range, "can not apply operator `" + op + "` to " + to_string(unary_expr->inferred_type)); +// make an error on `!cell` / `+slice` +static Error err_cannot_apply_operator(std::string_view operator_name, AnyExprV unary_expr) { + return err("can not apply operator `{}` to `{}`", operator_name, unary_expr->inferred_type); } -// fire an error on `int + cell` / `slice & int` -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_cannot_apply_operator(FunctionPtr cur_f, SrcRange operator_range, std::string_view operator_name, AnyExprV lhs, AnyExprV rhs) { - std::string op = static_cast(operator_name); - fire(cur_f, operator_range, "can not apply operator `" + op + "` to " + to_string(lhs->inferred_type) + " and " + to_string(rhs->inferred_type)); +// make an error on `int + cell` / `slice & int` +static Error err_cannot_apply_operator(std::string_view operator_name, AnyExprV lhs, AnyExprV rhs) { + return err("can not apply operator `{}` to `{}` and `{}`", operator_name, lhs->inferred_type, rhs->inferred_type); } GNU_ATTRIBUTE_NOINLINE @@ -102,16 +80,16 @@ static void warning_condition_always_true_or_false(FunctionPtr cur_f, SrcRange k if (no_warning) { // allow `while(true)` without a warning return; } - compilation_warning(cur_f, keyword_range, "condition of " + static_cast(operator_name) + " is always " + (cond->is_always_true ? "true" : "false")); + err("condition of {} is always {}", operator_name, cond->is_always_true).warning(keyword_range, cur_f); } // given `f(x: int)` and a call `f(expr)`, check that expr_type is assignable to `int` static void check_function_argument_passed(FunctionPtr cur_f, TypePtr param_type, AnyExprV ith_arg, bool is_obj_of_dot_call) { if (!param_type->can_rhs_be_assigned(ith_arg->inferred_type)) { if (is_obj_of_dot_call) { - fire(cur_f, ith_arg, "can not call method for " + to_string(param_type) + " with object of type " + to_string(ith_arg)); + err("can not call method for `{}` with object of type `{}`", param_type, ith_arg->inferred_type).fire(ith_arg, cur_f); } else { - fire_type_mismatch(cur_f, ith_arg->range, "can not pass {src} to {dst}", ith_arg->inferred_type, param_type); + err_type_mismatch("can not pass {src} to {dst}", ith_arg->inferred_type, param_type).fire(ith_arg, cur_f); } } } @@ -121,29 +99,24 @@ static void check_function_argument_passed(FunctionPtr cur_f, TypePtr param_type static void check_function_argument_mutate_back(FunctionPtr cur_f, TypePtr param_type, AnyExprV ith_arg, bool is_obj_of_dot_call) { if (!ith_arg->inferred_type->can_rhs_be_assigned(param_type)) { if (is_obj_of_dot_call) { - fire(cur_f, ith_arg,"can not call method for mutate " + to_string(param_type) + " with object of type " + to_string(ith_arg) + ", because mutation is not type compatible"); + err("can not call method for mutate `{}` with object of type `{}`, because mutation is not type compatible", param_type, ith_arg->inferred_type).fire(ith_arg, cur_f); } else { - fire(cur_f, ith_arg,"can not pass " + to_string(ith_arg) + " to mutate " + to_string(param_type) + ", because mutation is not type compatible"); + err("can not pass `{}` to mutate `{}`, because mutation is not type compatible", ith_arg->inferred_type, param_type).fire(ith_arg, cur_f); } } } -// fire an error on `var n = null` +// make an error on `var n = null` // technically it's correct, type of `n` is TypeDataNullLiteral, but it's not what the user wanted // so, it's better to see an error on assignment, that later, on `n` usage and types mismatch // (most common is situation above, but generally, `var (x,n) = xn` where xn is a tensor with 2-nd always-null, can be) -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -// todo pass not range -static void fire_assign_always_null_to_variable(FunctionPtr cur_f, SrcRange range, LocalVarPtr assigned_var, bool is_assigned_null_literal) { - std::string var_name = assigned_var->name; - fire(cur_f, range, "can not infer type of `" + var_name + "`, it's always null\nspecify its type with `" + var_name + ": `" + (is_assigned_null_literal ? " or use `null as `" : "")); +static Error err_assign_always_null_to_variable(LocalVarPtr assigned_var, bool is_assigned_null_literal) { + return err("can not infer type of `{}`, it's always null\nspecify its type with `{}: `{}", assigned_var, assigned_var, (is_assigned_null_literal ? " or use `null as `" : "")); } -// fire an error on `untypedTupleVar.0` when inferred as (int,int), or `[int, (int,int)]`, or other non-1 width in a tuple -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -// todo pass not ra -static void fire_cannot_put_non1_stack_width_arg_to_tuple(FunctionPtr cur_f, SrcRange range, TypePtr inferred_type) { - fire(cur_f, range, "a tuple can not have " + to_string(inferred_type) + " inside, because it occupies " + std::to_string(inferred_type->get_width_on_stack()) + " stack slots in TVM, not 1"); +// make an error on `untypedTupleVar.0` when inferred as (int,int), or `[int, (int,int)]`, or other non-1 width in a tuple +static Error err_cannot_put_non1_stack_width_arg_to_tuple(TypePtr inferred_type) { + return err("a tuple can not have `{}` inside, because it occupies {} stack slots in TVM, not 1", inferred_type, inferred_type->get_width_on_stack()); } // handle __expect_type(expr, "type") call @@ -158,7 +131,7 @@ static void handle_possible_compiler_internal_call(FunctionPtr cur_f, Vget_arg(1)->get_expr()->as()->str_val; TypePtr expr_type = v->get_arg(0)->inferred_type; if (expected_type_str != expr_type->as_human_readable()) { - fire(cur_f, v, "__expect_type failed: expected `" + to_string(expected_type_str) + "`, got " + to_string(expr_type)); + err("__expect_type failed: expected `{}`, got `{}`", expected_type_str, expr_type).fire(v, cur_f); } } } @@ -238,7 +211,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } // using += for other types (e.g. `tensorVar += tensorVar`) is not allowed if (!types_ok) { - fire_cannot_apply_operator(cur_f, v->operator_range, v->operator_name, lhs, rhs); + err_cannot_apply_operator(v->operator_name, lhs, rhs).fire(v->operator_range, cur_f); } } @@ -249,12 +222,12 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { switch (v->tok) { case tok_logical_not: if (!expect_integer(rhs) && !expect_boolean(rhs)) { - fire_cannot_apply_operator(cur_f, v->operator_range, v->operator_name, rhs); + err_cannot_apply_operator(v->operator_name, rhs).fire(v->operator_range, cur_f); } break; default: if (!expect_integer(rhs)) { - fire_cannot_apply_operator(cur_f, v->operator_range, v->operator_name, rhs); + err_cannot_apply_operator(v->operator_name, rhs).fire(v->operator_range, cur_f); } } } @@ -275,9 +248,9 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { bool not_integer_comparison = false; if (!check_eq_neq_operator(lhs->inferred_type, rhs->inferred_type, not_integer_comparison)) { if (lhs->inferred_type->equal_to(rhs->inferred_type)) { // compare slice with slice, int? with int? - fire(cur_f, v->operator_range, "type " + to_string(lhs) + " can not be compared with `== !=`"); + err("type `{}` can not be compared with `== !=`", lhs->inferred_type).fire(v->operator_range, cur_f); } else { - fire_cannot_apply_operator(cur_f, v->operator_range, v->operator_name, lhs, rhs); + err_cannot_apply_operator(v->operator_name, lhs, rhs).fire(v->operator_range, cur_f); } } if (not_integer_comparison) { // special handling at IR generation like for `address` @@ -292,7 +265,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { case tok_geq: case tok_spaceship: if (!expect_integer(lhs) || !expect_integer(rhs)) { - fire_cannot_apply_operator(cur_f, v->operator_range, v->operator_name, lhs, rhs); + err_cannot_apply_operator(v->operator_name, lhs, rhs).fire(v->operator_range, cur_f); } break; // & | ^ are "overloaded" both for integers and booleans, (int & bool) is NOT allowed @@ -303,7 +276,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { bool both_int = expect_integer(lhs) && expect_integer(rhs); bool both_bool = expect_boolean(lhs) && expect_boolean(rhs); if (!both_int && !both_bool) { - fire_cannot_apply_operator(cur_f, v->operator_range, v->operator_name, lhs, rhs); + err_cannot_apply_operator(v->operator_name, lhs, rhs).fire(v->operator_range, cur_f); } break; } @@ -313,7 +286,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { bool lhs_ok = expect_integer(lhs) || expect_boolean(lhs); bool rhs_ok = expect_integer(rhs) || expect_boolean(rhs); if (!lhs_ok || !rhs_ok) { - fire_cannot_apply_operator(cur_f, v->operator_range, v->operator_name, lhs, rhs); + err_cannot_apply_operator(v->operator_name, lhs, rhs).fire(v->operator_range, cur_f); } break; } @@ -321,7 +294,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // they are allowed for intN (int16 + int32 is ok) and always "fall back" to general int default: if (!expect_integer(lhs) || !expect_integer(rhs)) { - fire_cannot_apply_operator(cur_f, v->operator_range, v->operator_name, lhs, rhs); + err_cannot_apply_operator(v->operator_name, lhs, rhs).fire(v->operator_range, cur_f); } } } @@ -330,7 +303,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { parent::visit(v->get_expr()); if (!v->get_expr()->inferred_type->can_be_casted_with_as_operator(v->type_node->resolved_type)) { - fire(cur_f, v, "type " + to_string(v->get_expr()) + " can not be cast to " + to_string(v->type_node->resolved_type)); + err("type `{}` can not be cast to `{}`", v->get_expr()->inferred_type, v->type_node->resolved_type).fire(v, cur_f); } } @@ -339,14 +312,14 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { TypePtr rhs_type = v->type_node->resolved_type; if (rhs_type->unwrap_alias()->try_as()) { // `v is T1 | T2` / `v is T?` is disallowed - fire(cur_f, v, "union types are not allowed, use concrete types in `is`"); + err("union types are not allowed, use concrete types in `is`").fire(v, cur_f); } if ((v->is_always_true && !v->is_negated) || (v->is_always_false && v->is_negated)) { - compilation_warning(cur_f, v, expression_as_string(v->get_expr()) + " is always " + to_string(rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); + err("{} is always `{}`, this condition is always {}", expression_as_string(v->get_expr()), rhs_type, v->is_always_true).warning(v, cur_f); } if ((v->is_always_false && !v->is_negated) || (v->is_always_true && v->is_negated)) { - compilation_warning(cur_f, v, expression_as_string(v->get_expr()) + " of type " + to_string(v->get_expr()) + " can never be " + to_string(rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); + err("{} of type `{}` can never be `{}`, this condition is always {}", expression_as_string(v->get_expr()), v->get_expr()->inferred_type, rhs_type, v->is_always_true).warning(v, cur_f); } } @@ -355,7 +328,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (v->get_expr()->inferred_type == TypeDataNullLiteral::create()) { // operator `!` used for always-null (proven by smart casts, for example), it's an error - fire(cur_f, v, "operator `!` used for always null expression"); + err("operator `!` used for always null expression").fire(v, cur_f); } // if operator `!` used for non-nullable, probably a warning should be printed } @@ -366,7 +339,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { for (int i = 0; i < v->size(); ++i) { AnyExprV item = v->get_item(i); if (item->inferred_type->get_width_on_stack() != 1) { - fire_cannot_put_non1_stack_width_arg_to_tuple(cur_f, v->get_item(i)->range, item->inferred_type); + err_cannot_put_non1_stack_width_arg_to_tuple(item->inferred_type).fire(v->get_item(i), cur_f); } } } @@ -377,7 +350,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (v->is_target_indexed_access()) { TypePtr obj_type = v->get_obj()->inferred_type->unwrap_alias(); if (v->inferred_type->get_width_on_stack() != 1 && (obj_type->try_as() || obj_type->try_as())) { - fire_cannot_put_non1_stack_width_arg_to_tuple(cur_f, v->range, v->inferred_type); + err_cannot_put_non1_stack_width_arg_to_tuple(v->inferred_type).fire(v, cur_f); } } } @@ -394,7 +367,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { auto arg_i = v->get_arg(i)->get_expr(); TypePtr param_type = f_callable->params_types[i]; if (!param_type->can_rhs_be_assigned(arg_i->inferred_type)) { - fire_type_mismatch(cur_f, arg_i->range, "can not pass {src} to {dst}", arg_i->inferred_type, param_type); + err_type_mismatch("can not pass {src} to {dst}", arg_i->inferred_type, param_type).fire(arg_i, cur_f); } } return; @@ -456,11 +429,11 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } if (declared_type) { if (!declared_type->can_rhs_be_assigned(rhs_type)) { - fire_type_mismatch(cur_f, err_loc->range, "can not assign {src} to variable of type {dst}", rhs_type, declared_type); + err_type_mismatch("can not assign {src} to variable of type {dst}", rhs_type, declared_type).fire(err_loc, cur_f); } } else { if (rhs_type == TypeDataNullLiteral::create()) { - fire_assign_always_null_to_variable(cur_f, err_loc->range, lhs_var->var_ref->try_as(), corresponding_maybe_rhs && corresponding_maybe_rhs->kind == ast_null_keyword); + err_assign_always_null_to_variable(lhs_var->var_ref->try_as(), corresponding_maybe_rhs && corresponding_maybe_rhs->kind == ast_null_keyword).fire(err_loc, cur_f); } } return; @@ -471,10 +444,10 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (auto lhs_tensor = lhs->try_as()) { const TypeDataTensor* rhs_type_tensor = rhs_type->unwrap_alias()->try_as(); if (!rhs_type_tensor) { - fire(cur_f, err_loc, "can not assign " + to_string(rhs_type) + " to a tensor"); + err("can not assign `{}` to a tensor", rhs_type).fire(err_loc, cur_f); } if (lhs_tensor->size() != rhs_type_tensor->size()) { - fire(cur_f, err_loc, "can not assign " + to_string(rhs_type) + ", sizes mismatch"); + err("can not assign `{}`, sizes mismatch", rhs_type).fire(err_loc, cur_f); } V rhs_tensor_maybe = corresponding_maybe_rhs ? corresponding_maybe_rhs->try_as() : nullptr; for (int i = 0; i < lhs_tensor->size(); ++i) { @@ -488,10 +461,10 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (auto lhs_tuple = lhs->try_as()) { const TypeDataBrackets* rhs_type_tuple = rhs_type->unwrap_alias()->try_as(); if (!rhs_type_tuple) { - fire(cur_f, err_loc, "can not assign " + to_string(rhs_type) + " to a tuple"); + err("can not assign `{}` to a tuple", rhs_type).fire(err_loc, cur_f); } if (lhs_tuple->size() != rhs_type_tuple->size()) { - fire(cur_f, err_loc, "can not assign " + to_string(rhs_type) + ", sizes mismatch"); + err("can not assign `{}`, sizes mismatch", rhs_type).fire(err_loc, cur_f); } V rhs_tuple_maybe = corresponding_maybe_rhs ? corresponding_maybe_rhs->try_as() : nullptr; for (int i = 0; i < lhs_tuple->size(); ++i) { @@ -504,7 +477,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (auto lhs_dot = lhs->try_as()) { if (lhs_dot->is_target_indexed_access() && lhs_dot->get_obj()->inferred_type->unwrap_alias() == TypeDataTuple::create()) { if (rhs_type->get_width_on_stack() != 1) { - fire_cannot_put_non1_stack_width_arg_to_tuple(cur_f, err_loc->range, rhs_type); + err_cannot_put_non1_stack_width_arg_to_tuple(rhs_type).fire(err_loc, cur_f); } } } @@ -514,11 +487,11 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // for strange lhs like `f() = rhs` type checking will pass, but will fail lvalue check later if (!lhs->inferred_type->can_rhs_be_assigned(rhs_type)) { if (lhs->try_as()) { - fire_type_mismatch(cur_f, err_loc->range, "can not assign {src} to variable of type {dst}", rhs_type, lhs->inferred_type); + err_type_mismatch("can not assign {src} to variable of type {dst}", rhs_type, lhs->inferred_type).fire(err_loc, cur_f); } else if (lhs->try_as()) { - fire_type_mismatch(cur_f, err_loc->range, "can not assign {src} to field of type {dst}", rhs_type, lhs->inferred_type); + err_type_mismatch("can not assign {src} to field of type {dst}", rhs_type, lhs->inferred_type).fire(err_loc, cur_f); } else { - fire_type_mismatch(cur_f, err_loc->range, "can not assign {src} to {dst}", rhs_type, lhs->inferred_type); + err_type_mismatch("can not assign {src} to {dst}", rhs_type, lhs->inferred_type).fire(err_loc, cur_f); } } } @@ -528,14 +501,14 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (cur_f->does_return_self()) { if (!is_expr_valid_as_return_self(v->get_return_value())) { - fire(cur_f, v, "invalid return from `self` function"); + err("invalid return from `self` function").fire(v, cur_f); } return; } TypePtr expr_type = v->get_return_value()->inferred_type; if (!cur_f->inferred_return_type->can_rhs_be_assigned(expr_type)) { - fire_type_mismatch(cur_f, v->get_return_value()->range, "can not convert type {src} to return type {dst}", expr_type, cur_f->inferred_return_type); + err_type_mismatch("can not convert type {src} to return type {dst}", expr_type, cur_f->inferred_return_type).fire(v->get_return_value(), cur_f); } } @@ -560,7 +533,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { AnyExprV cond = v->get_cond(); if (!expect_integer(cond) && !expect_boolean(cond)) { - fire(cur_f, cond, "can not use " + to_string(cond) + " as a boolean condition"); + err("can not use `{}` as a boolean condition", cond->inferred_type).fire(cond, cur_f); } if (cond->is_always_true || cond->is_always_false) { @@ -587,57 +560,56 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { switch (v_arm->pattern_kind) { case MatchArmKind::exact_type: { if (has_expr_arm) { - // todo here and below: not v_arm as range, but before => - fire(cur_f, v_arm, "can not mix type and expression patterns in `match`"); + err("can not mix type and expression patterns in `match`").fire(v_arm->get_pattern_expr(), cur_f); } if (has_else_arm) { - fire(cur_f, v_arm, "`else` branch should be the last"); + err("`else` branch should be the last").fire(v_arm->get_pattern_expr(), cur_f); } has_type_arm = true; TypePtr lhs_type = v_arm->pattern_type_node->resolved_type; // `lhs_type => ...` if (lhs_type->unwrap_alias()->try_as()) { - fire(cur_f, v_arm, "wrong pattern matching: union types are not allowed, use concrete types in `match`"); + err("wrong pattern matching: union types are not allowed, use concrete types in `match`").fire(v_arm->get_pattern_expr(), cur_f); } bool can_happen = (subject_union && subject_union->has_variant_equal_to(lhs_type)) || (!subject_union && subject_type->equal_to(lhs_type)); if (!can_happen) { - fire(cur_f, v_arm, "wrong pattern matching: " + to_string(lhs_type) + " is not a variant of " + to_string(subject_type)); + err("wrong pattern matching: `{}` is not a variant of `{}`", lhs_type, subject_type).fire(v_arm->get_pattern_expr(), cur_f); } auto it_mentioned = std::find_if(covered_types.begin(), covered_types.end(), [lhs_type](TypePtr existing) { return existing->equal_to(lhs_type); }); if (it_mentioned != covered_types.end()) { - fire(cur_f, v_arm, "wrong pattern matching: duplicated " + to_string(lhs_type)); + err("wrong pattern matching: duplicated `{}`", lhs_type).fire(v_arm->get_pattern_expr(), cur_f); } covered_types.push_back(lhs_type); break; } case MatchArmKind::const_expression: { if (has_type_arm) { - fire(cur_f, v_arm, "can not mix type and expression patterns in `match`"); + err("can not mix type and expression patterns in `match`").fire(v_arm->get_pattern_expr(), cur_f); } if (has_else_arm) { - fire(cur_f, v_arm, "`else` branch should be the last"); + err("`else` branch should be the last").fire(v_arm->get_pattern_expr(), cur_f); } has_expr_arm = true; TypePtr pattern_type = v_arm->get_pattern_expr()->inferred_type; bool not_integer_comparison = false; if (!check_eq_neq_operator(pattern_type, subject_type, not_integer_comparison) || not_integer_comparison) { if (pattern_type->equal_to(subject_type)) { // `match` over `slice` etc., where operator `==` can't be applied - fire(cur_f, v_arm, "wrong pattern matching: can not compare type " + to_string(subject_type) + " in `match`"); + err("wrong pattern matching: can not compare type `{}` in `match`", subject_type).fire(v_arm->get_pattern_expr(), cur_f); } else { - fire(cur_f, v_arm, "wrong pattern matching: can not compare type " + to_string(v_arm->get_pattern_expr()) + " with match subject of type " + to_string(v_subject)); + err("wrong pattern matching: can not compare type `{}` with match subject of type `{}`", v_arm->get_pattern_expr()->inferred_type, v_subject->inferred_type).fire(v_arm->get_pattern_expr(), cur_f); } } if (subject_enum) { auto l_dot = v_arm->get_pattern_expr()->try_as(); if (!l_dot || !l_dot->is_target_enum_member()) { // match (someColor) { anotherColor => } - fire(cur_f, v_arm, "wrong pattern matching: `match` should contain members of a enum"); + err("wrong pattern matching: `match` should contain members of a enum").fire(v_arm->get_pattern_expr(), cur_f); } EnumMemberPtr member_ref = std::get(l_dot->target); if (std::find(covered_enum.begin(), covered_enum.end(), member_ref) != covered_enum.end()) { - fire(cur_f, v_arm, "wrong pattern matching: duplicated enum member in `match`"); + err("wrong pattern matching: duplicated enum member in `match`").fire(v_arm->get_pattern_expr(), cur_f); } covered_enum.push_back(member_ref); } @@ -645,7 +617,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } default: if (has_else_arm) { - fire(cur_f, v_arm, "duplicated `else` branch"); + err("duplicated `else` branch").fire(v_arm->get_pattern_expr(), cur_f); } if (has_type_arm) { // `else` is not allowed in `match` by type, but we don't fire an error here, @@ -658,8 +630,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // only `else` branch if (has_else_arm && !has_type_arm && !has_expr_arm) { - // todo not v range, but keyword, write te - fire(cur_f, v, "`match` contains only `else`, but no variants"); + err("`match` contains only `else`, but no variants").fire(v->keyword_range(), cur_f); } // fire if `match` by type is not exhaustive @@ -673,11 +644,10 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (!missing.empty()) { missing += ", "; } - missing += to_string(variant); + missing += "`" + variant->as_human_readable() + "`"; } } - // todo not v->range but keyword - fire(cur_f, v, "`match` does not cover all possible types; missing types are: " + missing); + err("`match` does not cover all possible types; missing types are: {}", missing).fire(v->keyword_range(), cur_f); } // fire if `match` by enum is not exhaustive if (has_expr_arm && subject_enum && !has_else_arm && subject_enum->enum_ref->members.size() != covered_enum.size()) { @@ -690,16 +660,14 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { missing += member_ref->name; } } - // todo not v->range but ke - fire(cur_f, v, "`match` does not cover all possible enum members; missing members are: " + missing); + err("`match` does not cover all possible enum members; missing members are: {}", missing).fire(v->keyword_range(), cur_f); } // fire if `match` by enum covers all cases, but contains `else` // (note that `else` for types could exist for a lazy match; for non-lazy, it's fired later) if (has_expr_arm && subject_enum && has_else_arm && subject_enum->enum_ref->members.size() == covered_enum.size()) { for (int i = 0; i < v->get_arms_count(); ++i) { if (auto v_arm = v->get_arm(i); v_arm->pattern_kind == MatchArmKind::else_branch) { - // todo not v_arm->range but before => - fire(cur_f, v_arm, "`match` already covers all possible enum members, `else` is invalid"); + err("`match` already covers all possible enum members, `else` is invalid").fire(v_arm->get_pattern_expr(), cur_f); } } } @@ -712,8 +680,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { needs_else_branch = !(arm0 && arm1 && arm0->bool_val != arm1->bool_val); } if (needs_else_branch) { - // todo not v->range - fire(cur_f, v, "`match` expression should have `else` branch"); + err("`match` expression should have `else` branch").fire(v->keyword_range(), cur_f); } } } @@ -722,7 +689,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { parent::visit(v->get_init_val()); if (!v->field_ref->declared_type->can_rhs_be_assigned(v->get_init_val()->inferred_type)) { - fire_type_mismatch(cur_f, v->get_init_val()->range, "can not assign {src} to field of type {dst}", v->get_init_val()->inferred_type, v->field_ref->declared_type); + err_type_mismatch("can not assign {src} to field of type {dst}", v->get_init_val()->inferred_type, v->field_ref->declared_type).fire(v->get_init_val(), cur_f); } } @@ -731,7 +698,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { AnyExprV cond = v->get_cond(); if (!expect_integer(cond) && !expect_boolean(cond)) { - fire(cur_f, cond, "can not use " + to_string(cond) + " as a boolean condition"); + err("can not use `{}` as a boolean condition", cond->inferred_type).fire(cond, cur_f); } if (cond->is_always_true || cond->is_always_false) { @@ -744,7 +711,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { AnyExprV cond = v->get_cond(); if (!expect_integer(cond)) { - fire(cur_f, cond, "condition of `repeat` must be an integer, got " + to_string(cond)); + err("condition of `repeat` must be an integer, got `{}`", cond->inferred_type).fire(cond, cur_f); } } @@ -753,7 +720,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { AnyExprV cond = v->get_cond(); if (!expect_integer(cond) && !expect_boolean(cond)) { - fire(cur_f, cond, "can not use " + to_string(cond) + " as a boolean condition"); + err("can not use `{}` as a boolean condition", cond->inferred_type).fire(cond, cur_f); } if (cond->is_always_true || cond->is_always_false) { @@ -766,7 +733,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { AnyExprV cond = v->get_cond(); if (!expect_integer(cond) && !expect_boolean(cond)) { - fire(cur_f, cond, "can not use " + to_string(cond) + " as a boolean condition"); + err("can not use `{}` as a boolean condition", cond->inferred_type).fire(cond, cur_f); } if (cond->is_always_true || cond->is_always_false) { @@ -778,10 +745,10 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { parent::visit(v); if (!expect_thrown_code(v->get_thrown_code()->inferred_type)) { - fire(cur_f, v->get_thrown_code(), "excNo of `throw` must be an integer, got " + to_string(v->get_thrown_code())); + err("excNo of `throw` must be an integer, got `{}`", v->get_thrown_code()->inferred_type).fire(v->get_thrown_code(), cur_f); } if (v->has_thrown_arg() && v->get_thrown_arg()->inferred_type->get_width_on_stack() != 1) { - fire(cur_f, v->get_thrown_arg(), "can not throw " + to_string(v->get_thrown_arg()) + ", exception arg must occupy exactly 1 stack slot"); + err("can not throw `{}`, exception arg must occupy exactly 1 stack slot", v->get_thrown_arg()->inferred_type).fire(v->get_thrown_arg(), cur_f); } } @@ -790,10 +757,10 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { AnyExprV cond = v->get_cond(); if (!expect_integer(cond) && !expect_boolean(cond)) { - fire(cur_f, cond, "can not use " + to_string(cond) + " as a boolean condition"); + err("can not use `{}` as a boolean condition", cond->inferred_type).fire(cond, cur_f); } if (!expect_thrown_code(v->get_thrown_code()->inferred_type)) { - fire(cur_f, v->get_thrown_code(), "thrown excNo of `assert` must be an integer, got " + to_string(v->get_thrown_code())); + err("thrown excNo of `assert` must be an integer, got `{}`", v->get_thrown_code()->inferred_type).fire(v->get_thrown_code(), cur_f); } if (cond->is_always_true || cond->is_always_false) { @@ -809,7 +776,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // (printing it while inferring might be a false positive if types are incorrect, due to smart casts for example) // a more correct approach would be to access cfg here somehow, but since cfg is now available only while inferring, // a special v->first_unreachable was set specifically for this warning (again, which is correct if types match) - compilation_warning(cur_f, v->first_unreachable, "unreachable code"); + err("unreachable code").warning(v->first_unreachable, cur_f); } } @@ -825,7 +792,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (fun_ref->is_implicit_return() && fun_ref->declared_return_type) { if (!fun_ref->declared_return_type->can_rhs_be_assigned(TypeDataVoid::create()) || fun_ref->does_return_self()) { - fire(fun_ref, SrcRange::empty_at_end(v_function->get_body()->range), "missing return"); + err("missing return").fire(SrcRange::empty_at_end(v_function->get_body()->range), fun_ref); } } @@ -836,7 +803,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { TypePtr inferred_type = param_ref->default_value->inferred_type; if (!param_ref->declared_type->can_rhs_be_assigned(inferred_type)) { - fire_type_mismatch(fun_ref, param_ref->default_value->range, "can not assign {src} to {dst}", inferred_type, param_ref->declared_type); + err_type_mismatch("can not assign {src} to {dst}", inferred_type, param_ref->declared_type).fire(param_ref->default_value, fun_ref); } } } @@ -852,7 +819,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (const_ref->declared_type) { // `const a: int = ...` TypePtr inferred_type = const_ref->init_value->inferred_type; if (!const_ref->declared_type->can_rhs_be_assigned(inferred_type)) { - fire_type_mismatch(nullptr, const_ref->init_value->range, "can not assign {src} to {dst}", inferred_type, const_ref->declared_type); + err_type_mismatch("can not assign {src} to {dst}", inferred_type, const_ref->declared_type).fire(const_ref->init_value); } } } @@ -863,7 +830,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { TypePtr inferred_type = field_ref->default_value->inferred_type; if (!field_ref->declared_type->can_rhs_be_assigned(inferred_type)) { - fire_type_mismatch(nullptr, field_ref->default_value->range, "can not assign {src} to {dst}", inferred_type, field_ref->declared_type); + err_type_mismatch("can not assign {src} to {dst}", inferred_type, field_ref->declared_type).fire(field_ref->default_value); } } @@ -873,7 +840,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { TypePtr inferred_type = member_ref->init_value->inferred_type; if (!inferred_type->equal_to(TypeDataInt::create()) && !inferred_type->equal_to(TypeDataEnum::create(enum_ref))) { - fire(member_ref->init_value, "enum member is " + to_string(inferred_type) + ", not `int`\nhint: all enums must be integers"); + err("enum member is `{}`, not `int`\n""hint: all enums must be integers", inferred_type).fire(member_ref->init_value); } } }; diff --git a/tolk/pipe-check-private-fields.cpp b/tolk/pipe-check-private-fields.cpp index ca77ae3097..d6b9898227 100644 --- a/tolk/pipe-check-private-fields.cpp +++ b/tolk/pipe-check-private-fields.cpp @@ -17,15 +17,12 @@ #include "ast.h" #include "ast-visitor.h" #include "compilation-errors.h" -#include "platform-utils.h" #include "type-system.h" namespace tolk { -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -// todo pass not range; consider all fire_xxx functions -static void fire_private_field_used_outside_method(FunctionPtr cur_f, SrcRange range, StructPtr struct_ref, StructFieldPtr field_ref) { - fire(cur_f, range, "field `" + struct_ref->as_human_readable() + "." + field_ref->name + "` is private"); +static Error err_private_field_used_outside_method(StructPtr struct_ref, StructFieldPtr field_ref) { + return err("field `{}.{}` is private", struct_ref, field_ref); } static bool is_private_field_usage_allowed(FunctionPtr cur_f, StructPtr struct_ref) { @@ -59,7 +56,7 @@ class CheckPrivateFieldsUsageVisitor final : public ASTVisitorFunctionBody { const TypeDataStruct* obj_type = v->get_obj()->inferred_type->unwrap_alias()->try_as(); tolk_assert(obj_type); if (field_ref->is_private && !is_private_field_usage_allowed(cur_f, obj_type->struct_ref)) { - fire_private_field_used_outside_method(cur_f, v->range, obj_type->struct_ref, field_ref); + err_private_field_used_outside_method(obj_type->struct_ref, field_ref).fire(v, cur_f); } } } @@ -72,7 +69,7 @@ class CheckPrivateFieldsUsageVisitor final : public ASTVisitorFunctionBody { auto v_field = v->get_body()->get_field(i); StructFieldPtr field_ref = v_field->field_ref; if (field_ref->is_private && !is_private_field_usage_allowed(cur_f, v->struct_ref)) { - fire_private_field_used_outside_method(cur_f, v_field->range, v->struct_ref, field_ref); + err_private_field_used_outside_method(v->struct_ref, field_ref).fire(v_field, cur_f); } } } diff --git a/tolk/pipe-check-pure-impure.cpp b/tolk/pipe-check-pure-impure.cpp index 57cfa9853d..8e0e1db0ba 100644 --- a/tolk/pipe-check-pure-impure.cpp +++ b/tolk/pipe-check-pure-impure.cpp @@ -17,7 +17,6 @@ #include "ast.h" #include "ast-visitor.h" #include "compilation-errors.h" -#include "platform-utils.h" /* * This pipe checks for impure operations inside pure functions. @@ -26,9 +25,8 @@ namespace tolk { -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_impure_operation_inside_pure_function(FunctionPtr cur_f, SrcRange range) { - fire(cur_f, range, "an impure operation in a pure function"); +static Error err_impure_operation_inside_pure_function() { + return err("an impure operation in a pure function"); } class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFunctionBody { @@ -37,7 +35,7 @@ class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFuncti void fire_if_global_var(AnyExprV v) const { if (auto v_ident = v->try_as()) { if (v_ident->sym->try_as()) { - fire_impure_operation_inside_pure_function(cur_f, v_ident->range); + err_impure_operation_inside_pure_function().fire(v_ident, cur_f); } } } @@ -56,11 +54,11 @@ class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFuncti // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` if (!v->fun_maybe) { // `local_var(args)` is always impure, no considerations about what's there at runtime - fire_impure_operation_inside_pure_function(cur_f, v->range); + err_impure_operation_inside_pure_function().fire(v, cur_f); } if (!v->fun_maybe->is_marked_as_pure()) { - fire_impure_operation_inside_pure_function(cur_f, v->range); + err_impure_operation_inside_pure_function().fire(v, cur_f); } parent::visit(v); @@ -75,11 +73,11 @@ class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFuncti } void visit(V v) override { - fire_impure_operation_inside_pure_function(cur_f, v->range); + err_impure_operation_inside_pure_function().fire(v, cur_f); } void visit(V v) override { - fire_impure_operation_inside_pure_function(cur_f, v->range); + err_impure_operation_inside_pure_function().fire(v, cur_f); } public: diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp index fabe391d4a..f2bf79ea6e 100644 --- a/tolk/pipe-check-rvalue-lvalue.cpp +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -17,7 +17,6 @@ #include "ast.h" #include "ast-visitor.h" #include "compilation-errors.h" -#include "platform-utils.h" #include "type-system.h" /* @@ -30,35 +29,32 @@ namespace tolk { -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_cannot_be_used_as_lvalue(FunctionPtr cur_f, AnyV at, const std::string& details) { +static Error err_cannot_be_used_as_lvalue(const std::string& details) { // example: `f() = 32` // example: `loadUint(c.beginParse(), 32)` (since `loadUint()` mutates the first argument) - fire(cur_f, at, details + " can not be used as lvalue"); + return err("{} can not be used as lvalue", details); } -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_modifying_immutable_variable(FunctionPtr cur_f, SrcRange range, LocalVarPtr var_ref) { +static Error err_modifying_immutable_variable(LocalVarPtr var_ref) { if (var_ref->param_idx == 0 && var_ref->name == "self") { - fire(cur_f, range, "modifying `self`, which is immutable by default; probably, you want to declare `mutate self`"); + return err("modifying `self`, which is immutable by default; probably, you want to declare `mutate self`"); } else { - fire(cur_f, range, "modifying immutable variable `" + var_ref->name + "`"); + return err("modifying immutable variable `{}`", var_ref); } } -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_modifying_readonly_field(FunctionPtr cur_f, SrcRange range, StructPtr struct_ref, StructFieldPtr field_ref) { - fire(cur_f, range, "modifying readonly field `" + struct_ref->as_human_readable() + "." + field_ref->name + "`"); +static Error err_modifying_readonly_field(StructPtr struct_ref, StructFieldPtr field_ref) { + return err("modifying readonly field `{}.{}`", struct_ref, field_ref); } // validate a function used as rvalue, like `var cb = f` // it's not a generic function (ensured earlier at type inferring) and has some more restrictions static void validate_function_used_as_noncall(FunctionPtr cur_f, AnyExprV v, FunctionPtr fun_ref) { if (!fun_ref->arg_order.empty() || !fun_ref->ret_order.empty()) { - fire(cur_f, v, "saving `" + fun_ref->name + "` into a variable will most likely lead to invalid usage, since it changes the order of variables on the stack"); + err("saving `{}` into a variable will most likely lead to invalid usage, since it changes the order of variables on the stack", fun_ref).fire(v, cur_f); } if (fun_ref->has_mutate_params()) { - fire(cur_f, v, "saving `" + fun_ref->name + "` into a variable is impossible, since it has `mutate` parameters and thus can only be called directly"); + err("saving `{}` into a variable is impossible, since it has `mutate` parameters and thus can only be called directly", fun_ref).fire(v, cur_f); } } @@ -67,49 +63,49 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { void on_var_used_as_lvalue(SrcRange range, LocalVarPtr var_ref) const { if (var_ref->is_immutable()) { - fire_modifying_immutable_variable(cur_f, range, var_ref); + err_modifying_immutable_variable(var_ref).fire(range, cur_f); } var_ref->mutate()->assign_used_as_lval(); } void visit(V v) override { if (v->is_lvalue) { - fire_cannot_be_used_as_lvalue(cur_f, v, "braced expression"); + err_cannot_be_used_as_lvalue("braced expression").fire(v, cur_f); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_cannot_be_used_as_lvalue(cur_f, v, "assignment"); + err_cannot_be_used_as_lvalue("assignment").fire(v, cur_f); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_cannot_be_used_as_lvalue(cur_f, v, "assignment"); + err_cannot_be_used_as_lvalue("assignment").fire(v, cur_f); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_cannot_be_used_as_lvalue(cur_f, v, "operator " + static_cast(v->operator_name)); + err_cannot_be_used_as_lvalue("operator " + static_cast(v->operator_name)).fire(v, cur_f); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_cannot_be_used_as_lvalue(cur_f, v, "operator " + static_cast(v->operator_name)); + err_cannot_be_used_as_lvalue("operator " + static_cast(v->operator_name)).fire(v, cur_f); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_cannot_be_used_as_lvalue(cur_f, v, "operator ?:"); + err_cannot_be_used_as_lvalue("operator ?:").fire(v, cur_f); } parent::visit(v); } @@ -121,7 +117,7 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { if (v->is_lvalue) { - fire_cannot_be_used_as_lvalue(cur_f, v, v->is_negated ? "operator !is" : "operator is"); + err_cannot_be_used_as_lvalue(v->is_negated ? "operator !is" : "operator is").fire(v, cur_f); } parent::visit(v->get_expr()); } @@ -133,32 +129,32 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { if (v->is_lvalue) { - fire_cannot_be_used_as_lvalue(cur_f, v, "lazy expression"); + err_cannot_be_used_as_lvalue("lazy expression").fire(v, cur_f); } parent::visit(v->get_expr()); } void visit(V v) override { if (v->is_lvalue) { - fire_cannot_be_used_as_lvalue(cur_f, v, "literal"); + err_cannot_be_used_as_lvalue("literal").fire(v, cur_f); } } void visit(V v) override { if (v->is_lvalue) { - fire_cannot_be_used_as_lvalue(cur_f, v, "literal"); + err_cannot_be_used_as_lvalue("literal").fire(v, cur_f); } } void visit(V v) override { if (v->is_lvalue) { - fire_cannot_be_used_as_lvalue(cur_f, v, "literal"); + err_cannot_be_used_as_lvalue("literal").fire(v, cur_f); } } void visit(V v) override { if (v->is_lvalue) { - fire_cannot_be_used_as_lvalue(cur_f, v, "literal"); + err_cannot_be_used_as_lvalue("literal").fire(v, cur_f); } } @@ -175,7 +171,7 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { const TypeDataStruct* obj_type = as_dot->get_obj()->inferred_type->unwrap_alias()->try_as(); tolk_assert(obj_type); if (field_ref->is_readonly) { - fire_modifying_readonly_field(cur_f, as_dot->range, obj_type->struct_ref, field_ref); + err_modifying_readonly_field(obj_type->struct_ref, field_ref).fire(as_dot, cur_f); } } @@ -197,9 +193,9 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { } if (as_ref->sym->try_as()) { // `Point.create = f` if (v->is_target_enum_member()) { - fire(cur_f, v, "modifying immutable constant"); + err("modifying immutable constant").fire(v, cur_f); } - fire(cur_f, v, "invalid left side of assignment"); + err("invalid left side of assignment").fire(v, cur_f); } } } @@ -214,7 +210,7 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { if (v->is_lvalue) { - fire_cannot_be_used_as_lvalue(cur_f, v, "function call"); + err_cannot_be_used_as_lvalue("function call").fire(v, cur_f); } if (!v->fun_maybe) { parent::visit(v->get_callee()); @@ -232,7 +228,7 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { if (v->is_lvalue) { - fire_cannot_be_used_as_lvalue(cur_f, v, "`match` expression"); + err_cannot_be_used_as_lvalue("`match` expression").fire(v, cur_f); } parent::visit(v); } @@ -241,7 +237,7 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { if (v->marked_as_redef) { tolk_assert(v->var_ref); if (v->var_ref->is_immutable()) { - fire(cur_f, v, "`redef` for immutable variable"); + err("`redef` for immutable variable").fire(v, cur_f); } } } @@ -252,9 +248,9 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { if (LocalVarPtr var_ref = v->sym->try_as()) { on_var_used_as_lvalue(v->range, var_ref); } else if (v->sym->try_as()) { - fire(cur_f, v, "modifying immutable constant"); + err("modifying immutable constant").fire(v, cur_f); } else if (v->sym->try_as()) { - fire(cur_f, v, "function can't be used as lvalue"); + err("function can't be used as lvalue").fire(v, cur_f); } } @@ -266,7 +262,7 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { if (v->is_rvalue) { - fire(cur_f, v, "`_` can't be used as a value; it's a placeholder for a left side of assignment"); + err("`_` can't be used as a value; it's a placeholder for a left side of assignment").fire(v, cur_f); } } diff --git a/tolk/pipe-check-serialized-fields.cpp b/tolk/pipe-check-serialized-fields.cpp index 6df6509d66..e3f45a779c 100644 --- a/tolk/pipe-check-serialized-fields.cpp +++ b/tolk/pipe-check-serialized-fields.cpp @@ -24,33 +24,31 @@ namespace tolk { -// fire an error on overflow 1023 bits -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_theoretical_overflow_1023(StructPtr struct_ref, PackSize size) { - fire(struct_ref->ident_anchor, - "struct `" + struct_ref->as_human_readable() + "` can exceed 1023 bits in serialization (estimated size: " + std::to_string(size.min_bits) + ".." + std::to_string(size.max_bits) + " bits)\n\n" +// make an error on overflow 1023 bits +static Error err_theoretical_overflow_1023(StructPtr struct_ref, PackSize size) { + return err("struct `{}` can exceed 1023 bits in serialization (estimated size: {}..{} bits)\n\n" "1) either suppress it by adding an annotation:\n" "> @overflow1023_policy(\"suppress\")\n" - "> struct " + struct_ref->name + " {\n" + "> struct {} {\n" "> ...\n" "> }\n" " then, if limit exceeds, it will fail at runtime: you've manually agreed to ignore this\n\n" "2) or place some fields into a separate struct (e.g. ExtraFields), and create a ref:\n" - "> struct " + struct_ref->name + " {\n" + "> struct {} {\n" "> ...\n" "> more: Cell;\n" - "> }\n" - ); + "> }\n", + struct_ref, size.min_bits, size.max_bits, struct_ref->name, struct_ref->name); } GNU_ATTRIBUTE_NOINLINE static void check_map_TKey_TValue(SrcRange range, TypePtr TKey, TypePtr TValue) { std::string because_msg; if (!check_mapKV_TKey_is_valid(TKey, because_msg)) { - fire(range, "invalid `map`: type `" + TKey->as_human_readable() + "` can not be used as a key\n" + because_msg); + err("invalid `map`: type `{}` can not be used as a key\n{}", TKey, because_msg).fire(range); } if (!check_mapKV_TValue_is_valid(TValue, because_msg)) { - fire(range, "invalid `map`: type `" + TValue->as_human_readable() + "` can not be used as a value\n" + because_msg); + err("invalid `map`: type `{}` can not be used as a value\n{}", TValue, because_msg).fire(range); } } @@ -76,8 +74,7 @@ static void check_mapKV_inside_type(AnyTypeV type_node) { // given `enum Role: int8` check colon type (not struct/slice etc.) static void check_enum_colon_type_to_be_intN(AnyTypeV colon_type_node) { if (!colon_type_node->resolved_type->try_as()) { - // todo write test for underline - fire(colon_type_node, "serialization type of `enum` must be intN: `int8` / `uint32` / etc."); + err("serialization type of `enum` must be intN: `int8` / `uint32` / etc.").fire(colon_type_node); } } @@ -105,7 +102,7 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody PackSize size = estimate_serialization_size(t_struct); if (size.max_bits > 1023 && !size.is_unpredictable_infinity()) { if (struct_ref->overflow1023_policy == StructData::Overflow1023Policy::not_specified) { - fire_theoretical_overflow_1023(struct_ref, size); + err_theoretical_overflow_1023(struct_ref, size).fire(struct_ref->ident_anchor); } } // don't check Cell fields for overflow of T: it would be checked on load() or other interaction with T @@ -139,7 +136,7 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody std::string because_msg; if (!check_struct_can_be_packed_or_unpacked(serialized_type, is_pack, because_msg)) { std::string via_name = fun_ref->is_method() ? fun_ref->method_name : fun_ref->base_fun_ref->name; - fire(cur_f, v, "auto-serialization via " + via_name + "() is not available for type `" + serialized_type->as_human_readable() + "`\n" + because_msg); + err("auto-serialization via {}() is not available for type `{}`\n{}", via_name, serialized_type, because_msg).fire(v, cur_f); } check_type_fits_cell_or_has_policy(serialized_type); diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index 8b2b6fc1c6..9ebf3a535e 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -41,7 +41,7 @@ static void assign_enum_members_values(EnumDefPtr enum_ref) { for (EnumMemberPtr member_ref : enum_ref->members) { td::RefInt256 cur_value = member_ref->init_value ? eval_enum_member_init_value(member_ref->init_value) : prev_value + 1; if (!cur_value->is_valid() || !cur_value->signed_fits_bits(257)) { - fire(member_ref->ident_anchor, "integer overflow"); + err("integer overflow").fire(member_ref->ident_anchor); } member_ref->mutate()->assign_computed_value(cur_value); @@ -91,7 +91,7 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { tolk_assert(!intval.is_null()); intval = -intval; if (intval.is_null() || !intval->signed_fits_bits(257)) { - fire(v, "integer overflow"); + err("integer overflow").fire(v); } return create_int_const(v->range, std::move(intval)); } diff --git a/tolk/pipe-detect-inline-in-place.cpp b/tolk/pipe-detect-inline-in-place.cpp index 711bab3d5b..00bc4b618b 100644 --- a/tolk/pipe-detect-inline-in-place.cpp +++ b/tolk/pipe-detect-inline-in-place.cpp @@ -236,7 +236,7 @@ class DetectIfToInlineFunctionInPlaceVisitor final : ASTVisitorFunctionBody { tolk_assert(v_expect->get_num_args() == 1 && v_expect->get_arg(0)->get_expr()->kind == ast_bool_const); bool expected = v_expect->get_arg(0)->get_expr()->as()->bool_val; if (expected != will_inline) { - fire(fun_ref, v_expect, "__expect_inline failed"); + err("__expect_inline failed").fire(v_expect, fun_ref); } } diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp index 51c5bc28b8..3d059c2e2c 100644 --- a/tolk/pipe-generate-fif-output.cpp +++ b/tolk/pipe-generate-fif-output.cpp @@ -27,6 +27,7 @@ #include "src-file.h" #include "ast.h" #include "compiler-state.h" +#include "type-system.h" namespace tolk { @@ -42,7 +43,7 @@ void FunctionBodyAsm::set_code(std::vector&& code) { static void generate_output_func(FunctionPtr fun_ref) { tolk_assert(fun_ref->is_code_function()); if (G.is_verbosity(2)) { - std::cerr << "\n\n=========================\nfunction " << fun_ref->name << " : " << fun_ref->inferred_return_type << std::endl; + std::cerr << "\n\n=========================\nfunction " << fun_ref->name << " : " << fun_ref->inferred_return_type->as_human_readable() << std::endl; } CodeBlob* code = std::get(fun_ref->body)->code; diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index eef62b1d3b..a633ed52d4 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -107,73 +107,40 @@ static TypePtr get_or_infer_return_type(FunctionPtr fun_ref) { return fun_ref->inferred_return_type; } -GNU_ATTRIBUTE_NOINLINE -static std::string to_string(TypePtr type) { - return "`" + type->as_human_readable() + "`"; -} - -GNU_ATTRIBUTE_NOINLINE -static std::string to_string(AnyExprV v_with_type) { - return "`" + v_with_type->inferred_type->as_human_readable() + "`"; -} - -GNU_ATTRIBUTE_NOINLINE -static std::string to_string(FunctionPtr fun_ref) { - return "`" + fun_ref->as_human_readable() + "`"; -} - -GNU_ATTRIBUTE_NOINLINE -static std::string to_string(StructPtr struct_ref) { - return "`" + struct_ref->as_human_readable() + "`"; -} - -GNU_ATTRIBUTE_NOINLINE -static std::string to_string(AliasDefPtr alias_ref) { - return "`" + alias_ref->as_human_readable() + "`"; -} - -GNU_ATTRIBUTE_NOINLINE -static std::string to_string(std::string_view string_view) { - return static_cast(string_view); -} - -// fire an error when `fun f(...) asm ...` is called with T=(int,int) or other non-1 width on stack +// make an error when `fun f(...) asm ...` is called with T=(int,int) or other non-1 width on stack // asm functions generally can't handle it, they expect T to be a TVM primitive // (in FunC, `forall` type just couldn't be unified with non-primitives; in Tolk, generic T is expectedly inferred) -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_calling_asm_function_with_non1_stack_width_arg(FunctionPtr cur_f, SrcRange range, FunctionPtr fun_ref, const GenericsSubstitutions& substitutions, int arg_idx) { - fire(cur_f, range, "can not call `" + fun_ref->as_human_readable() + "` with " + to_string(substitutions.nameT_at(arg_idx)) + "=" + substitutions.typeT_at(arg_idx)->as_human_readable() + ", because it occupies " + std::to_string(substitutions.typeT_at(arg_idx)->get_width_on_stack()) + " stack slots in TVM, not 1"); +static Error err_calling_asm_function_with_non1_stack_width_arg(FunctionPtr fun_ref, const GenericsSubstitutions& substitutions, int arg_idx) { + return err("can not call `{}` with {}={}, because it occupies {} stack slots in TVM, not 1", + fun_ref, substitutions.nameT_at(arg_idx), substitutions.typeT_at(arg_idx), substitutions.typeT_at(arg_idx)->get_width_on_stack()); } -// fire an error on `untypedTupleVar.0` when used without a hint -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_cannot_deduce_untyped_tuple_access(FunctionPtr cur_f, SrcRange range, int index) { - std::string idx_access = "." + std::to_string(index); - fire(cur_f, range, "can not deduce type of `" + idx_access + "`\neither assign it to variable like `var c: int = " + idx_access + "` or cast the result like `" + idx_access + " as int`"); +// make an error on `untypedTupleVar.0` when used without a hint +static Error err_cannot_deduce_untyped_tuple_access(int index) { + return err("can not deduce type of `.{}`\neither assign it to variable like `var c: int = .{}` or cast the result like `.{} as int`", + index, index, index); } -// fire an error on using lateinit variable before definite assignment -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_using_lateinit_variable_uninitialized(FunctionPtr cur_f, SrcRange range, std::string_view name) { - fire(cur_f, range, "using variable `" + static_cast(name) + "` before it's definitely assigned"); +// make an error on using lateinit variable before definite assignment +static Error err_using_lateinit_variable_uninitialized(std::string_view name) { + return err("using variable `{}` before it's definitely assigned", name); } -// fire an error when `obj.f()`, method `f` not found, try to locate a method for another type -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_method_or_field_not_found(FunctionPtr cur_f, SrcRange range, TypePtr receiver_type, std::string_view field_name, bool called_as_method, bool is_static_dot) { +// make an error when `obj.f()`, method `f` not found, try to locate a method for another type +static Error err_method_or_field_not_found(TypePtr receiver_type, std::string_view field_name, bool called_as_method, bool is_static_dot) { if (!called_as_method && is_static_dot && receiver_type->unwrap_alias()->try_as()) { - fire(cur_f, range, "member `" + to_string(field_name) + "` does not exist in enum " + to_string(receiver_type)); + return err("member `{}` does not exist in enum `{}`", field_name, receiver_type); } if (!called_as_method && !is_static_dot) { - fire(cur_f, range, "field `" + to_string(field_name) + "` doesn't exist in type " + to_string(receiver_type)); + return err("field `{}` doesn't exist in type `{}`", field_name, receiver_type); } if (std::vector other = lookup_methods_with_name(field_name); !other.empty()) { - fire(cur_f, range, "method `" + to_string(field_name) + "` not found for type " + to_string(receiver_type) + "\n(but it exists for type " + to_string(other.front()->receiver_type) + ")"); + return err("method `{}` not found for type `{}`\n(but it exists for type `{}`)", field_name, receiver_type, other.front()->receiver_type); } if (const Symbol* sym = lookup_global_symbol(field_name); sym && sym->try_as()) { - fire(cur_f, range, "method `" + to_string(field_name) + "` not found, but there is a global function named `" + to_string(field_name) + "`\n(a function should be called `foo(arg)`, not `arg.foo()`)"); + return err("method `{}` not found, but there is a global function named `{}`\n(a function should be called `foo(arg)`, not `arg.foo()`)", field_name, field_name); } - fire(cur_f, range, "method `" + to_string(field_name) + "` not found"); + return err("method `{}` not found", field_name); } // safe version of std::stoi that does not crash on long numbers @@ -240,7 +207,7 @@ static std::pair choose_only_method_to_call( } std::ostringstream msg; - msg << "call to method `" << called_name << "` for type `" << called_receiver << "` is ambiguous\n"; + msg << "call to method `" << called_name << "` for type `" << called_receiver->as_human_readable() << "` is ambiguous\n"; for (const MethodCallCandidate& candidate : candidates) { FunctionPtr method_ref = candidate.method_ref; msg << "candidate function: `" << method_ref->as_human_readable() << "`"; @@ -253,12 +220,12 @@ static std::pair choose_only_method_to_call( msg << " (builtin)\n"; } } - fire(cur_f, range, msg.str()); + err("{}", msg.str()).fire(range, cur_f); } static void check_no_unexpected_type_arguments(FunctionPtr cur_f, V v_instantiationTs) { if (v_instantiationTs != nullptr) { - fire(cur_f, v_instantiationTs, "type arguments not expected here"); + err("type arguments not expected here").fire(v_instantiationTs, cur_f); } } @@ -274,13 +241,13 @@ void check_arguments_count_at_fun_call(FunctionPtr cur_f, V v } if (!called_f->does_accept_self() && self_obj) { // static method `Point.create(...)` called as `p.create()` - fire(cur_f, v->get_callee(), "method " + to_string(called_f) + " can not be called via dot\n(it's a static method, it does not accept `self`)"); + err("method `{}` can not be called via dot\n(it's a static method, it does not accept `self`)", called_f).fire(v->get_callee(), cur_f); } if (n_max_params < n_arguments) { - fire(cur_f, v->get_arg_list(), "too many arguments in call to " + to_string(called_f) + ", expected " + std::to_string(n_max_params - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); + err("too many arguments in call to `{}`, expected {}, have {}", called_f, n_max_params - delta_self, n_arguments - delta_self).fire(v->get_arg_list(), cur_f); } if (n_arguments < n_min_params) { - fire(cur_f, v->get_arg_list(), "too few arguments in call to " + to_string(called_f) + ", expected " + std::to_string(n_min_params - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); + err("too few arguments in call to `{}`, expected {}, have {}", called_f, n_min_params - delta_self, n_arguments - delta_self).fire(v->get_arg_list(), cur_f); } } @@ -748,7 +715,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // example: `var v = ternary`, show an inference error // do NOT show an error for `var v: T = ternary` (T is hint); it will be checked by type checker later if (hint == nullptr || hint == TypeDataUnknown::create() || hint->has_genericT_inside()) { - fire(cur_f, v, "types of ternary branches are incompatible: " + to_string(v->get_when_true()) + " and " + to_string(v->get_when_false()) + "\nhint: maybe, you should use ` as ` to make them identical"); + err("types of ternary branches are incompatible: `{}` and `{}`\n""hint: maybe, you should use ` as ` to make them identical", v->get_when_true()->inferred_type, v->get_when_false()->inferred_type).fire(v, cur_f); } } assign_inferred_type(v, branches_unifier.get_result()); @@ -780,7 +747,7 @@ class InferTypesAndCallsAndFieldsVisitor final { rhs_type = inst_rhs_type; v->type_node->mutate()->assign_resolved_type(rhs_type); } else { - fire(cur_f, v->type_node, "can not deduce type arguments for " + to_string(t_struct->struct_ref) + ", provide them manually"); + err("can not deduce type arguments for `{}`, provide them manually", t_struct->struct_ref).fire(v->type_node, cur_f); } } if (const auto* t_alias = rhs_type->try_as(); t_alias && t_alias->alias_ref->is_generic_alias()) { @@ -789,7 +756,7 @@ class InferTypesAndCallsAndFieldsVisitor final { rhs_type = inst_rhs_type; v->type_node->mutate()->assign_resolved_type(rhs_type); } else { - fire(cur_f, v->type_node, "can not deduce type arguments for " + to_string(t_alias->alias_ref) + ", provide them manually"); + err("can not deduce type arguments for `{}`, provide them manually", t_alias->alias_ref).fire(v->type_node, cur_f); } } @@ -878,7 +845,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // for `genericF` user should provide two Ts // for `Container.wrap` — one U (and one T is implicitly from receiver) if (instantiationT_list->size() != genericTs->size() - genericTs->n_from_receiver) { - fire(cur_f, instantiationT_list, "expected " + std::to_string(genericTs->size() - genericTs->n_from_receiver) + " type arguments, got " + std::to_string(instantiationT_list->size())); + err("expected {} type arguments, got {}", genericTs->size() - genericTs->n_from_receiver, instantiationT_list->size()).fire(instantiationT_list, cur_f); } std::vector type_arguments; @@ -900,7 +867,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (fun_ref->is_asm_function() || fun_ref->is_builtin()) { for (int i = 0; i < substitutedTs.size(); ++i) { if (substitutedTs.typeT_at(i)->get_width_on_stack() != 1 && !is_allowed_asm_generic_function_with_non1_width_T(fun_ref, i)) { - fire_calling_asm_function_with_non1_stack_width_arg(cur_f, range, fun_ref, substitutedTs, i); + err_calling_asm_function_with_non1_stack_width_arg(fun_ref, substitutedTs, i).fire(range, cur_f); } } } @@ -919,7 +886,7 @@ class InferTypesAndCallsAndFieldsVisitor final { TypePtr declared_or_smart_casted = flow.smart_cast_or_original(SinkExpression(var_ref), var_ref->declared_type); assign_inferred_type(v, declared_or_smart_casted); if (var_ref->is_lateinit() && declared_or_smart_casted == TypeDataUnknown::create() && v->is_rvalue) { - fire_using_lateinit_variable_uninitialized(cur_f, v->range, v->get_name()); + err_using_lateinit_variable_uninitialized(v->get_name()).fire(v, cur_f); } // it might be `local_var()` also, don't fill out_f_called, we have no fun_ref, it's a call of arbitrary expression @@ -940,7 +907,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (fun_ref->is_generic_function() && !v_instantiationTs && !out_f_called) { // `genericFn` is invalid as non-call, can't be used without - fire(cur_f, v, "can not use a generic function " + to_string(fun_ref) + " as non-call"); + err("can not use a generic function `{}` as non-call", fun_ref).fire(v, cur_f); } else if (fun_ref->is_generic_function() && v_instantiationTs) { // `genericFn` is valid as non-call, it's a reference to instantiation @@ -959,10 +926,10 @@ class InferTypesAndCallsAndFieldsVisitor final { *out_f_called = fun_ref; // (it's still may be a generic one, then Ts will be deduced from arguments) } else { // so, it's `globalF` / `genericFn` as a reference if (fun_ref->is_compile_time_const_val() || fun_ref->is_compile_time_special_gen()) { - fire(cur_f, v, "can not get reference to this function, it's compile-time only"); + err("can not get reference to this function, it's compile-time only").fire(v, cur_f); } if (fun_ref->is_entrypoint()) { - fire(cur_f, v, "can not get reference to this function, it's a special entrypoint"); + err("can not get reference to this function, it's a special entrypoint").fire(v, cur_f); } fun_ref->mutate()->assign_is_used_as_noncall(); get_or_infer_return_type(fun_ref); @@ -1006,7 +973,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, v_ident->range, receiver_type, field_name); if (!fun_ref) { - fire_method_or_field_not_found(cur_f, v_ident->range, receiver_type, field_name, out_f_called != nullptr, true); + err_method_or_field_not_found(receiver_type, field_name, out_f_called != nullptr, true).fire(v_ident, cur_f); } } } @@ -1038,11 +1005,11 @@ class InferTypesAndCallsAndFieldsVisitor final { if (!fun_ref && field_name[0] >= '0' && field_name[0] <= '9') { int index_at; if (!try_parse_string_to_int(field_name, index_at)) { - fire(cur_f, v_ident, "invalid numeric index"); + err("invalid numeric index").fire(v_ident, cur_f); } if (const auto* t_tensor = obj_type->try_as()) { if (index_at >= t_tensor->size()) { - fire(cur_f, v_ident, "invalid tensor index, expected 0.." + std::to_string(t_tensor->items.size() - 1)); + err("invalid tensor index, expected 0..{}", t_tensor->items.size() - 1).fire(v_ident, cur_f); } v->mutate()->assign_target(index_at); TypePtr inferred_type = t_tensor->items[index_at]; @@ -1054,7 +1021,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } if (const auto* t_tuple = obj_type->try_as()) { if (index_at >= t_tuple->size()) { - fire(cur_f, v_ident, "invalid tuple index, expected 0.." + std::to_string(t_tuple->items.size() - 1)); + err("invalid tuple index, expected 0..{}", t_tuple->items.size() - 1).fire(v_ident, cur_f); } v->mutate()->assign_target(index_at); TypePtr inferred_type = t_tuple->items[index_at]; @@ -1070,7 +1037,7 @@ class InferTypesAndCallsAndFieldsVisitor final { item_type = TypeDataUnknown::create(); } else { if (hint == nullptr || hint == TypeDataUnknown::create() || hint->has_genericT_inside()) { - fire_cannot_deduce_untyped_tuple_access(cur_f, v_ident->range, index_at); + err_cannot_deduce_untyped_tuple_access(index_at).fire(v_ident, cur_f); } item_type = hint; } @@ -1091,10 +1058,10 @@ class InferTypesAndCallsAndFieldsVisitor final { // as a special case, handle accessing fields of nullable objects, to show a more precise error if (const TypeDataUnion* as_union = obj_type->try_as(); as_union && as_union->or_null) { if (const TypeDataStruct* n_struct = as_union->or_null->try_as(); n_struct && n_struct->struct_ref->find_field(field_name)) { - fire(cur_f, v_ident, "can not access field `" + static_cast(field_name) + "` of a possibly nullable object " + to_string(dot_obj) + "\nhint: check it via `obj != null` or use non-null assertion `obj!` operator"); + err("can not access field `{}` of a possibly nullable object `{}`\n""hint: check it via `obj != null` or use non-null assertion `obj!` operator", field_name, dot_obj->inferred_type).fire(v_ident, cur_f); } } - fire_method_or_field_not_found(cur_f, v_ident->range, dot_obj->inferred_type, field_name, out_f_called != nullptr, false); + err_method_or_field_not_found(dot_obj->inferred_type, field_name, out_f_called != nullptr, false).fire(v_ident, cur_f); } // if `fun T.copy(self)` and reference `int.copy` — all generic parameters are determined by the receiver, we know it @@ -1105,7 +1072,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (fun_ref->is_generic_function() && !v_instantiationTs && !out_f_called) { // `t.tupleAt` is invalid as non-call, can't be used without - fire(cur_f, v_ident, "can not use a generic function " + to_string(fun_ref) + " as non-call"); + err("can not use a generic function `{}` as non-call", fun_ref).fire(v_ident, cur_f); } else if (fun_ref->is_generic_function() && v_instantiationTs) { // `t.tupleAt` is valid as non-call, it's a reference to instantiation @@ -1123,7 +1090,7 @@ class InferTypesAndCallsAndFieldsVisitor final { *out_dot_obj = dot_obj; } else { // so, it's `user.method` / `t.tupleAt` as a reference if (fun_ref->is_compile_time_const_val() || fun_ref->is_compile_time_special_gen()) { - fire(cur_f, v_ident, "can not get reference to this method, it's compile-time only"); + err("can not get reference to this method, it's compile-time only").fire(v_ident, cur_f); } fun_ref->mutate()->assign_is_used_as_noncall(); v->mutate()->assign_target(fun_ref); @@ -1153,11 +1120,11 @@ class InferTypesAndCallsAndFieldsVisitor final { // callee must have "callable" inferred type const TypeDataFunCallable* f_callable = callee->inferred_type->unwrap_alias()->try_as(); if (!f_callable) { // `5()` / `SOME_CONST()` / `null()` - fire(cur_f, v->get_callee(), "calling a non-function " + to_string(callee->inferred_type)); + err("calling a non-function `{}`", callee->inferred_type).fire(v->get_callee(), cur_f); } // check arguments count (their types will be checked in a later pipe) if (v->get_num_args() != static_cast(f_callable->params_types.size())) { - fire(cur_f, v->get_callee(), "expected " + std::to_string(f_callable->params_types.size()) + " arguments, got " + std::to_string(v->get_arg_list()->size())); + err("expected {} arguments, got {}", f_callable->params_types.size(), v->get_arg_list()->size()).fire(v->get_callee(), cur_f); } for (int i = 0; i < v->get_num_args(); ++i) { auto arg_i = v->get_arg(i)->get_expr(); @@ -1170,7 +1137,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // prevent calling `onBouncedMessage()` and other special functions directly if (fun_ref->is_entrypoint()) { - fire(cur_f, v->get_callee(), fun_ref->name + " is a special entrypoint, it can not be called as a regular function"); + err("{} is a special entrypoint, it can not be called as a regular function", fun_ref).fire(v->get_callee(), cur_f); } // so, we have a call `f(args)` or `obj.f(args)`, f is fun_ref (function / method) (code / asm / builtin) @@ -1243,7 +1210,7 @@ class InferTypesAndCallsAndFieldsVisitor final { nameT_unknown = deducingTs.get_first_not_deduced_nameT(); } if (!nameT_unknown.empty()) { - deducingTs.fire_can_not_deduce(cur_f, v->get_callee()->range, nameT_unknown); + deducingTs.err_can_not_deduce(nameT_unknown).fire(v->get_callee(), cur_f); } fun_ref = check_and_instantiate_generic_function(v->get_callee()->range, fun_ref, deducingTs.flush()); } @@ -1312,7 +1279,7 @@ class InferTypesAndCallsAndFieldsVisitor final { exact_type = inst_exact_type; v_arm->pattern_type_node->mutate()->assign_resolved_type(exact_type); } else { - fire(cur_f, v_arm->pattern_type_node, "can not deduce type arguments for " + to_string(t_struct->struct_ref) + ", provide them manually"); + err("can not deduce type arguments for `{}`, provide them manually", t_struct->struct_ref).fire(v_arm->pattern_type_node, cur_f); } } if (const auto* t_alias = exact_type->try_as(); t_alias && t_alias->alias_ref->is_generic_alias()) { @@ -1321,7 +1288,7 @@ class InferTypesAndCallsAndFieldsVisitor final { exact_type = inst_exact_type; v_arm->pattern_type_node->mutate()->assign_resolved_type(exact_type); } else { - fire(cur_f, v_arm->pattern_type_node, "can not deduce type arguments for " + to_string(t_alias->alias_ref) + ", provide them manually"); + err("can not deduce type arguments for `{}`, provide them manually", t_alias->alias_ref).fire(v_arm->pattern_type_node, cur_f); } } arm_flow.register_known_type(s_expr, exact_type); @@ -1348,12 +1315,12 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, TypeDataVoid::create()); } else { if (v->get_arms_count() == 0) { // still allow empty `match` statements, for probable codegen - fire(cur_f, v->keyword_range(), "empty `match` can't be used as expression"); + err("empty `match` can't be used as expression").fire(v->keyword_range(), cur_f); } if (branches_unifier.is_union_of_different_types()) { // same as in ternary: `match (...) { t1 => someSlice, t2 => someInt }` is `int|slice`, probably unexpected if (hint == nullptr || hint == TypeDataUnknown::create() || hint->has_genericT_inside()) { - fire(cur_f, v->keyword_range(), "type of `match` was inferred as " + to_string(branches_unifier.get_result()) + "; probably, it's not what you expected\nassign it to a variable `var v: = match (...) { ... }` manually"); + err("type of `match` was inferred as `{}`; probably, it's not what you expected\nassign it to a variable `var v: = match (...) { ... }` manually", branches_unifier.get_result()).fire(v->keyword_range(), cur_f); } } assign_inferred_type(v, branches_unifier.get_result()); @@ -1376,7 +1343,7 @@ class InferTypesAndCallsAndFieldsVisitor final { struct_ref = hint_instTs->struct_ref; // if `type WAlias = Wrapper`, here `Wrapper` (generic struct) } if (!struct_ref) { - fire(cur_f, v->type_node, to_string(v->type_node->resolved_type) + " does not name a struct"); + err("`{}` does not name a struct", v->type_node->resolved_type).fire(v->type_node, cur_f); } // example: `var v: Ok = Ok { ... }`, now struct_ref is "Ok", take "Ok" from hint if (struct_ref->is_generic_struct() && hint) { @@ -1411,7 +1378,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } } if (!struct_ref) { - fire(cur_f, v, "can not detect struct name\nuse either `var v: StructName = { ... }` or `var v = StructName { ... }`"); + err("can not detect struct name\nuse either `var v: StructName = { ... }` or `var v = StructName { ... }`").fire(v, cur_f); } // so, we have struct_ref, so we can check field names and infer values @@ -1424,12 +1391,12 @@ class InferTypesAndCallsAndFieldsVisitor final { std::string_view field_name = field_i->get_field_name(); StructFieldPtr field_ref = struct_ref->find_field(field_name); if (!field_ref) { - fire(cur_f, field_i->get_field_identifier(), "field `" + to_string(field_name) + "` not found in struct " + to_string(struct_ref)); + err("field `{}` not found in struct `{}`", field_name, struct_ref).fire(field_i->get_field_identifier(), cur_f); } field_i->mutate()->assign_field_ref(field_ref); if (occurred_mask & (1ULL << field_ref->field_idx)) { - fire(cur_f, field_i->get_field_identifier(), "duplicate field initialization"); + err("duplicate field initialization").fire(field_i->get_field_identifier(), cur_f); } occurred_mask |= 1ULL << field_ref->field_idx; @@ -1453,7 +1420,7 @@ class InferTypesAndCallsAndFieldsVisitor final { nameT_unknown = deducingTs.get_first_not_deduced_nameT(); } if (!nameT_unknown.empty()) { - deducingTs.fire_can_not_deduce(cur_f, v->range, nameT_unknown); + deducingTs.err_can_not_deduce(nameT_unknown).fire(v, cur_f); } struct_ref = instantiate_generic_struct(struct_ref, deducingTs.flush()); // re-assign field_ref (it was earlier assigned into a field of a generic struct) @@ -1468,7 +1435,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (!(occurred_mask & (1ULL << field_ref->field_idx))) { bool allow_missing = field_ref->has_default_value() || field_ref->declared_type == TypeDataNever::create(); if (!allow_missing) { - fire(cur_f, SrcRange::empty_at_end(v->range), "field `" + field_ref->name + "` missed in initialization of struct " + to_string(struct_ref)); + err("field `{}` missed in initialization of struct `{}`", field_ref, struct_ref).fire(SrcRange::empty_at_end(v->range), cur_f); } } } @@ -1641,7 +1608,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (!body_end.is_unreachable()) { fun_ref->mutate()->assign_is_implicit_return(); if (fun_ref->declared_return_type == TypeDataNever::create()) { - fire(fun_ref, SrcRange::empty_at_end(v_function->range), "a function returning `never` can not have a reachable endpoint"); + err("a function returning `never` can not have a reachable endpoint").fire(SrcRange::empty_at_end(v_function->range), fun_ref); } } @@ -1664,18 +1631,18 @@ class InferTypesAndCallsAndFieldsVisitor final { } if (!body_end.is_unreachable() && inferred_return_type != TypeDataVoid::create()) { - fire(fun_ref, SrcRange::empty_at_end(v_function->range), "missing return"); + err("missing return").fire(SrcRange::empty_at_end(v_function->range), fun_ref); } if (has_void_returns && has_non_void_returns) { for (AnyExprV return_expr : return_statements) { if (return_expr->inferred_type == TypeDataVoid::create()) { - fire(fun_ref, return_expr, "mixing void and non-void returns in function " + to_string(fun_ref)); + err("mixing void and non-void returns in function `{}`", fun_ref).fire(return_expr, fun_ref); } } } if (return_unifier.is_union_of_different_types()) { // `return intVar` + `return sliceVar` results in `int | slice`, probably unexpected - fire(fun_ref, v_function->get_identifier(), "function " + to_string(fun_ref) + " calculated return type is " + to_string(inferred_return_type) + "; probably, it's not what you expected\ndeclare `fun (...): ` manually"); + err("function `{}` calculated return type is `{}`; probably, it's not what you expected\ndeclare `fun (...): ` manually", fun_ref, inferred_return_type).fire(v_function->get_identifier(), fun_ref); } } @@ -1750,7 +1717,7 @@ static void infer_and_save_return_type_of_function(FunctionPtr fun_ref) { // prevent recursion of untyped functions, like `fun f() { return g(); } fun g() { return f(); }` bool contains = std::find(called_stack.begin(), called_stack.end(), fun_ref) != called_stack.end(); if (contains) { - fire(fun_ref, fun_ref->ident_anchor, "could not infer return type of " + to_string(fun_ref) + ", because it appears in a recursive call chain\ndeclare `fun (...): ` manually"); + err("could not infer return type of `{}`, because it appears in a recursive call chain\ndeclare `fun (...): ` manually", fun_ref).fire(fun_ref->ident_anchor, fun_ref); } // dig into g's body; it's safe, since the compiler is single-threaded @@ -1770,7 +1737,7 @@ static void infer_and_save_type_of_constant(GlobalConstPtr const_ref) { // prevent recursion like `const a = b; const b = a` bool contains = std::find(called_stack.begin(), called_stack.end(), const_ref) != called_stack.end(); if (contains) { - fire(const_ref->ident_anchor, "const `" + const_ref->name + "` appears, directly or indirectly, in its own initializer"); + err("const `{}` appears, directly or indirectly, in its own initializer", const_ref).fire(const_ref->ident_anchor); } called_stack.push_back(const_ref); diff --git a/tolk/pipe-lazy-load-insertions.cpp b/tolk/pipe-lazy-load-insertions.cpp index fe4a9f7a8f..3ed7a10bdf 100644 --- a/tolk/pipe-lazy-load-insertions.cpp +++ b/tolk/pipe-lazy-load-insertions.cpp @@ -188,7 +188,7 @@ static void check_lazy_operator_used_correctly(FunctionPtr cur_f, Vget_expr()->kind == ast_function_call && does_function_satisfy_for_lazy_operator(v->get_expr()->as()->fun_maybe); if (!is_ok_call) { - fire(cur_f, v->keyword_range(), "`lazy` operator can only be used with built-in functions like fromCell/fromSlice or simple wrappers over them"); + err("`lazy` operator can only be used with built-in functions like fromCell/fromSlice or simple wrappers over them").fire(v->keyword_range(), cur_f); } // it should be either a struct or a union of structs @@ -198,11 +198,11 @@ static void check_lazy_operator_used_correctly(FunctionPtr cur_f, Vunwrap_alias()->try_as()) { if (TypePtr wrong_variant = is_union_type_prevented_from_lazy_loading(expr_union)) { - fire(cur_f, v->keyword_range(), "`lazy` union should contain only structures, but it contains `" + wrong_variant->as_human_readable() + "`"); + err("`lazy` union should contain only structures, but it contains `{}`", wrong_variant).fire(v->keyword_range(), cur_f); } return; } - fire(cur_f, v->keyword_range(), "`lazy` is applicable to structs, not to `" + expr_type->as_human_readable() + "`"); + err("`lazy` is applicable to structs, not to `{}`", expr_type).fire(v->keyword_range(), cur_f); } // Given `storage.save()` for a lazy `storage` variable, check if `self` inside should gain laziness. @@ -567,7 +567,7 @@ struct LazyVarInFunction { const TypeDataStruct* t_struct = var_ref->declared_type->unwrap_alias()->try_as(); bool is_union = t_struct == nullptr; if (is_union) { - fire(cur_f, created_by_lazy_op->keyword_range(), "`lazy` will not work here, because variable `" + var_ref->name + "` it's used in a non-lazy manner\nhint: lazy union may be used only in `match` statement"); + err("`lazy` will not work here, because variable `{}` it's used in a non-lazy manner\n""hint: lazy union may be used only in `match` statement", var_ref).fire(created_by_lazy_op->keyword_range(), cur_f); } // so, it's just a struct, `lazy Point`; we've already calculated all statements where its fields are used @@ -863,7 +863,7 @@ class CollectAllLazyObjectsAndFieldsVisitor final : public ASTVisitorFunctionBod } // for `return lazy ...` and other cases except allowed - fire(cur_f, v->keyword_range(), "incorrect `lazy` operator usage, it's not directly assigned to a variable\nhint: use `lazy` like this:\n> var st = lazy MyStorage.fromSlice(...)"); + err("incorrect `lazy` operator usage, it's not directly assigned to a variable\n""hint: use `lazy` like this:\n> var st = lazy MyStorage.fromSlice(...)").fire(v->keyword_range(), cur_f); } public: @@ -1004,7 +1004,7 @@ class CheckExpectLazyAssertionsVisitor final : public ASTVisitorFunctionBody { } if (actual != expected) { - fire(cur_stmt, "__expect_lazy failed: actual \"" + actual + "\""); + err("__expect_lazy failed: actual \"{}\"", actual).fire(SrcRange::span(cur_stmt->range, 13)); } } } diff --git a/tolk/pipe-refine-lvalue-for-mutate.cpp b/tolk/pipe-refine-lvalue-for-mutate.cpp index 26526c5654..3c4fbc0732 100644 --- a/tolk/pipe-refine-lvalue-for-mutate.cpp +++ b/tolk/pipe-refine-lvalue-for-mutate.cpp @@ -33,16 +33,15 @@ namespace tolk { -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_invalid_mutate_arg_passed(FunctionPtr cur_f, SrcRange range, FunctionPtr fun_ref, const LocalVarData& p_sym, bool arg_passed_as_mutate, AnyV arg_expr) { +static Error err_invalid_mutate_arg_passed(FunctionPtr fun_ref, const LocalVarData& p_sym, bool arg_passed_as_mutate, AnyV arg_expr) { std::string arg_str(arg_expr->kind == ast_reference ? arg_expr->as()->get_name() : "obj"); if (p_sym.is_mutate_parameter() && !arg_passed_as_mutate) { // called `mutating_function(arg)`; suggest: `mutate arg` - fire(cur_f, range, "function `" + fun_ref->as_human_readable() + "` mutates parameter `" + p_sym.name + "`\nyou need to specify `mutate` when passing an argument, like `mutate " + arg_str + "`"); + return err("function `{}` mutates parameter `{}`\nyou need to specify `mutate` when passing an argument, like `mutate {}`", fun_ref, p_sym.name, arg_str); } else { // called `usual_function(mutate arg)` - fire(cur_f, range, "incorrect `mutate`, since `" + fun_ref->as_human_readable() + "` does not mutate this parameter"); + return err("incorrect `mutate`, since `{}` does not mutate this parameter", fun_ref); } } @@ -58,7 +57,7 @@ class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBod for (int i = 0; i < v->get_num_args(); ++i) { auto v_arg = v->get_arg(i); if (v_arg->passed_as_mutate) { - fire(v_arg, "`mutate` used for non-mutate parameter"); + err("`mutate` used for non-mutate parameter").fire(v_arg); } } return; @@ -94,7 +93,7 @@ class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBod const LocalVarData& p_sym = fun_ref->parameters[delta_self + i]; auto arg_i = v->get_arg(i); if (p_sym.is_mutate_parameter() != arg_i->passed_as_mutate) { - fire_invalid_mutate_arg_passed(cur_f, arg_i->range, fun_ref, p_sym, arg_i->passed_as_mutate, arg_i->get_expr()); + err_invalid_mutate_arg_passed(fun_ref, p_sym, arg_i->passed_as_mutate, arg_i->get_expr()).fire(arg_i, cur_f); } parent::visit(arg_i); } diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index cddb912297..7cf2afecf8 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -15,7 +15,6 @@ along with TON Blockchain. If not, see . */ #include "ast.h" -#include "platform-utils.h" #include "src-file.h" #include "compilation-errors.h" #include "compiler-state.h" @@ -65,18 +64,18 @@ static int calculate_tvm_method_id_by_func_name(std::string_view func_name) { static void validate_arg_ret_order_of_asm_function(V v_body, int n_params) { if (n_params > 16) { - fire(v_body, "asm function can have at most 16 parameters"); + err("asm function can have at most 16 parameters").fire(v_body); } // asm(param1 ... paramN), param names were previously mapped into indices if (!v_body->arg_order.empty()) { if (static_cast(v_body->arg_order.size()) != n_params) { - fire(v_body, "arg_order of asm function must specify all parameters"); + err("arg_order of asm function must specify all parameters").fire(v_body); } std::vector visited(v_body->arg_order.size(), false); for (int j : v_body->arg_order) { if (visited[j]) { - fire(v_body, "arg_order of asm function contains duplicates"); + err("arg_order of asm function contains duplicates").fire(v_body); } visited[j] = true; } @@ -88,7 +87,7 @@ static void validate_arg_ret_order_of_asm_function(V v_body, int n std::vector visited(v_body->ret_order.size(), false); for (int j : v_body->ret_order) { if (j < 0 || j >= static_cast(v_body->ret_order.size()) || visited[j]) { - fire(v_body, "ret_order contains invalid integer, not in range 0 .. N"); + err("ret_order contains invalid integer, not in range 0 .. N").fire(v_body); } visited[j] = true; } @@ -141,11 +140,11 @@ static EnumDefPtr register_enum(V v) { std::string member_name = static_cast(v_ident->name); for (EnumMemberPtr prev : members) { - if (UNLIKELY(prev->name == member_name)) { - fire(v_member, "redeclaration of member `" + member_name + "`"); + if (prev->name == member_name) { + err("redeclaration of member `{}`", member_name).fire(v_member); } } - members.emplace_back(new EnumMemberData(member_name, v_ident, v_member->init_value)); + members.emplace_back(new EnumMemberData(std::move(member_name), v_ident, v_member->init_value)); } V v_ident = v->get_identifier(); @@ -168,18 +167,18 @@ static StructPtr register_struct(V v, StructPtr base_str std::string field_name = static_cast(v_ident->name); for (StructFieldPtr prev : fields) { - if (UNLIKELY(prev->name == field_name)) { - fire(v_field, "redeclaration of field `" + field_name + "`"); + if (prev->name == field_name) { + err("redeclaration of field `{}`", field_name).fire(v_field); } } - fields.emplace_back(new StructFieldData(field_name, v_ident, i, v_field->is_private, v_field->is_readonly, v_field->type_node, v_field->default_value)); + fields.emplace_back(new StructFieldData(std::move(field_name), v_ident, i, v_field->is_private, v_field->is_readonly, v_field->type_node, v_field->default_value)); } PackOpcode opcode(0, 0); if (v->has_opcode()) { auto v_opcode = v->get_opcode()->as(); if (v_opcode->intval < 0 || v_opcode->intval > (1ULL << 48)) { - fire(v, "opcode must not exceed 2^48"); + err("opcode must not exceed 2^48").fire(v); } opcode.pack_prefix = v_opcode->intval->to_long(); @@ -257,7 +256,7 @@ static FunctionPtr register_function(V v, FunctionPtr if (auto v_asm = v->get_body()->try_as()) { if (!v->return_type_node) { - fire(v_asm, "asm function must declare return type (before asm instructions)"); + err("asm function must declare return type (before asm instructions)").fire(v_asm); } validate_arg_ret_order_of_asm_function(v_asm, v->get_num_params()); f_sym->arg_order = v_asm->arg_order; @@ -270,7 +269,7 @@ static FunctionPtr register_function(V v, FunctionPtr f_sym->tvm_method_id = calculate_tvm_method_id_by_func_name(f_identifier); for (FunctionPtr other : G.all_contract_getters) { if (other->tvm_method_id == f_sym->tvm_method_id) { - fire(v, "GET methods hash collision: `" + other->name + "` and `" + f_sym->name + "` produce the same hash. Consider renaming one of these functions."); + err("GET methods hash collision: `{}` and `{}` produce the same hash. Consider renaming one of these functions.", other, f_sym).fire(v); } } } else if (v->flags & FunctionData::flagIsEntrypoint) { diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index e863b78747..7d8fab0dcc 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -17,7 +17,6 @@ #include "ast.h" #include "ast-visitor.h" #include "compilation-errors.h" -#include "platform-utils.h" #include "compiler-state.h" #include "src-file.h" @@ -45,30 +44,27 @@ namespace tolk { -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_undefined_symbol(FunctionPtr cur_f, V v) { +static Error err_undefined_symbol(V v) { if (v->name == "self") { - fire(cur_f, v, "using `self` in a non-member function (it does not accept the first `self` parameter)"); + return err("using `self` in a non-member function (it does not accept the first `self` parameter)"); } else { - fire(cur_f, v, "undefined symbol `" + static_cast(v->name) + "`"); + return err("undefined symbol `{}`", v->name); } } -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_type_used_as_symbol(FunctionPtr cur_f, V v) { +static Error err_type_used_as_symbol(V v) { if (v->name == "random") { // calling `random()`, but it's a struct, correct is `random.uint256()` - fire(cur_f, v, "`random` is not a function, you probably want `random.uint256()`"); + return err("`random` is not a function, you probably want `random.uint256()`"); } else { - fire(cur_f, v, "`" + static_cast(v->name) + "` only refers to a type, but is being used as a value here"); + return err("`{}` only refers to a type, but is being used as a value here", v->name); } } -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_using_self_not_in_method(FunctionPtr cur_f, SrcRange range) { +static Error err_using_self_not_in_method(FunctionPtr cur_f) { if (cur_f->is_static_method()) { - fire(cur_f, range, "using `self` in a static method"); + return err("using `self` in a static method"); } else { - fire(cur_f, range, "using `self` in a regular function (not a method)"); + return err("using `self` in a regular function (not a method)"); } } @@ -85,9 +81,7 @@ struct NameAndScopeResolver { void close_scope() { // std::cerr << "close_scope " << scopes.size() << " at " << loc << std::endl; - if (UNLIKELY(scopes.empty())) { - throw Fatal("cannot close the outer scope"); - } + tolk_assert(!scopes.empty()); scopes.pop_back(); } @@ -103,17 +97,15 @@ struct NameAndScopeResolver { } void add_local_var(LocalVarPtr v_sym) { - if (UNLIKELY(scopes.empty())) { - throw Fatal("unexpected scope_level = 0"); - } + tolk_assert(!scopes.empty()); if (v_sym->name.empty()) { // underscore return; } uint64_t key = key_hash(v_sym->name); const auto& [_, inserted] = scopes.rbegin()->emplace(key, v_sym); - if (UNLIKELY(!inserted)) { - fire(v_sym->ident_anchor, "redeclaration of local variable `" + v_sym->name + "`"); + if (!inserted) { + err("redeclaration of local variable `{}`", v_sym).fire(v_sym->ident_anchor); } } }; @@ -141,11 +133,11 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { if (v->marked_as_redef) { const Symbol* sym = current_scope.lookup_symbol(v->get_name()); if (sym == nullptr) { - fire(cur_f, v, "`redef` for unknown variable"); + err("`redef` for unknown variable").fire(v, cur_f); } LocalVarPtr var_ref = sym->try_as(); if (!var_ref) { - fire(cur_f, v, "`redef` for unknown variable"); + err("`redef` for unknown variable").fire(v, cur_f); } v->mutate()->assign_var_ref(var_ref); } else { @@ -162,10 +154,10 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { const Symbol* sym = current_scope.lookup_symbol(v->get_name()); if (!sym) { - fire_undefined_symbol(cur_f, v->get_identifier()); + err_undefined_symbol(v->get_identifier()).fire(v->get_identifier(), cur_f); } if (sym->try_as() || sym->try_as() || sym->try_as()) { - fire_type_used_as_symbol(cur_f, v->get_identifier()); + err_type_used_as_symbol(v->get_identifier()).fire(v->get_identifier(), cur_f); } v->mutate()->assign_sym(sym); @@ -181,12 +173,12 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { try { parent::visit(v->get_obj()); - } catch (const ParseError&) { + } catch (const ThrownParseError&) { if (auto v_type_name = v->get_obj()->try_as()) { // for `Point.create` / `int.zero` / `Color.Red`, "undefined symbol" is fired for Point/int/Color // suppress this exception till a later pipe, it will be tried to be resolved as a type if (v_type_name->get_identifier()->name == "self") { - fire_using_self_not_in_method(cur_f, v_type_name->range); + err_using_self_not_in_method(cur_f).fire(v_type_name, cur_f); } return; } diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 9108adb1b2..24cdbdb4e9 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -17,7 +17,6 @@ #include "ast.h" #include "ast-visitor.h" #include "compilation-errors.h" -#include "platform-utils.h" #include "compiler-state.h" #include "generics-helpers.h" #include "type-system.h" @@ -52,25 +51,22 @@ void patch_builtins_after_stdlib_loaded(); static std::unordered_map visited_structs; static std::unordered_map visited_aliases; -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_unknown_type_name(FunctionPtr cur_f, SrcRange range, std::string_view text) { +static Error err_unknown_type_name(std::string_view text) { if (text == "auto") { - fire(cur_f, range, "`auto` type does not exist; just omit a type for local variable (will be inferred from assignment); parameters should always be typed"); + return err("`auto` type does not exist; just omit a type for local variable (will be inferred from assignment); parameters should always be typed"); } if (text == "self") { - fire(cur_f, range, "`self` type can be used only as a return type of a function (enforcing it to be chainable)"); + return err("`self` type can be used only as a return type of a function (enforcing it to be chainable)"); } - fire(cur_f, range, "unknown type name `" + static_cast(text) + "`"); + return err("unknown type name `{}`", text); } -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_void_type_not_allowed_inside_union(FunctionPtr cur_f, SrcRange range, TypePtr disallowed_variant) { - fire(cur_f, range, "type `" + disallowed_variant->as_human_readable() + "` is not allowed inside a union"); +static Error err_void_type_not_allowed_inside_union(TypePtr disallowed_variant) { + return err("type `{}` is not allowed inside a union", disallowed_variant); } -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_generic_type_used_without_T(FunctionPtr cur_f, SrcRange range, const std::string& type_name_with_Ts) { - fire(cur_f, range, "type `" + type_name_with_Ts + "` is generic, you should provide type arguments"); +static Error err_generic_type_used_without_T(const std::string& type_name_with_Ts) { + return err("type `{}` is generic, you should provide type arguments", type_name_with_Ts); } static TypePtr parse_intN_uintN(std::string_view strN, bool is_unsigned) { @@ -173,7 +169,7 @@ class TypeNodesVisitorResolver { } if (text == "map") { if (!allow_without_type_arguments) { - fire_generic_type_used_without_T(cur_f, v->range, "map"); + err_generic_type_used_without_T("map").fire(v, cur_f); } return TypeDataMapKV::create(TypeDataGenericT::create("K"), TypeDataGenericT::create("V")); } @@ -192,7 +188,7 @@ class TypeNodesVisitorResolver { if (treat_unresolved_as_genericT) { return TypeDataGenericT::create(static_cast(text)); } - fire_unknown_type_name(cur_f, v->range, text); + err_unknown_type_name(text).fire(v, cur_f); } case ast_type_question_nullable: { @@ -257,7 +253,7 @@ class TypeNodesVisitorResolver { static TypePtr try_resolve_user_defined_type(FunctionPtr cur_f, SrcRange range, const Symbol* sym, bool allow_without_type_arguments) { if (AliasDefPtr alias_ref = sym->try_as()) { if (alias_ref->is_generic_alias() && !allow_without_type_arguments) { - fire_generic_type_used_without_T(cur_f, range, alias_ref->as_human_readable()); + err_generic_type_used_without_T(alias_ref->as_human_readable()).fire(range, cur_f); } if (!visited_aliases.contains(alias_ref)) { visit_symbol(alias_ref); @@ -266,7 +262,7 @@ class TypeNodesVisitorResolver { } if (StructPtr struct_ref = sym->try_as()) { if (struct_ref->is_generic_struct() && !allow_without_type_arguments) { - fire_generic_type_used_without_T(cur_f, range, struct_ref->as_human_readable()); + err_generic_type_used_without_T(struct_ref->as_human_readable()).fire(range, cur_f); } if (!visited_structs.contains(struct_ref)) { visit_symbol(struct_ref); @@ -292,7 +288,7 @@ class TypeNodesVisitorResolver { if (const TypeDataStruct* t_struct = type_to_instantiate->try_as(); t_struct && t_struct->struct_ref->is_generic_struct()) { StructPtr struct_ref = t_struct->struct_ref; if (struct_ref->genericTs->size() != static_cast(type_arguments.size())) { - fire(cur_f, range, "struct `" + struct_ref->as_human_readable() + "` expects " + std::to_string(struct_ref->genericTs->size()) + " type arguments, but " + std::to_string(type_arguments.size()) + " provided"); + err("struct `{}` expects {} type arguments, but {} provided", struct_ref, struct_ref->genericTs->size(), type_arguments.size()).fire(range, cur_f); } if (is_still_generic) { return TypeDataGenericTypeWithTs::create(struct_ref, nullptr, std::move(type_arguments)); @@ -302,7 +298,7 @@ class TypeNodesVisitorResolver { if (const TypeDataAlias* t_alias = type_to_instantiate->try_as(); t_alias && t_alias->alias_ref->is_generic_alias()) { AliasDefPtr alias_ref = t_alias->alias_ref; if (alias_ref->genericTs->size() != static_cast(type_arguments.size())) { - fire(cur_f, range, "type `" + alias_ref->as_human_readable() + "` expects " + std::to_string(alias_ref->genericTs->size()) + " type arguments, but " + std::to_string(type_arguments.size()) + " provided"); + err("type `{}` expects {} type arguments, but {} provided", alias_ref, alias_ref->genericTs->size(), type_arguments.size()).fire(range, cur_f); } if (is_still_generic) { return TypeDataGenericTypeWithTs::create(nullptr, alias_ref, std::move(type_arguments)); @@ -311,21 +307,21 @@ class TypeNodesVisitorResolver { } if (const TypeDataMapKV* t_map = type_to_instantiate->try_as(); t_map && t_map->TKey->try_as()) { if (type_arguments.size() != 2) { - fire(cur_f, range, "type `map` expects 2 type arguments, but " + std::to_string(type_arguments.size()) + " provided"); + err("type `map` expects 2 type arguments, but {} provided", type_arguments.size()).fire(range, cur_f); } return TypeDataMapKV::create(type_arguments[0], type_arguments[1]); } if (const TypeDataGenericT* asT = type_to_instantiate->try_as()) { - fire_unknown_type_name(cur_f, range, asT->nameT); + err_unknown_type_name(asT->nameT).fire(range, cur_f); } // `User` / `cell` - fire(cur_f, range, "type `" + type_to_instantiate->as_human_readable() + "` is not generic"); + err("type `{}` is not generic", type_to_instantiate).fire(range, cur_f); } static void validate_resulting_union_type(const TypeDataUnion* t_union, FunctionPtr cur_f, SrcRange range) { for (TypePtr variant : t_union->variants) { if (variant == TypeDataVoid::create() || variant == TypeDataNever::create()) { - fire_void_type_not_allowed_inside_union(cur_f, range, variant); + err_void_type_not_allowed_inside_union(variant).fire(range, cur_f); } } } @@ -378,7 +374,7 @@ class TypeNodesVisitorResolver { // prevent recursion like `type A = B; type B = A` (we can't create TypeDataAlias without a resolved underlying type) bool contains = std::find(called_stack.begin(), called_stack.end(), alias_ref) != called_stack.end(); if (contains) { - fire(alias_ref->ident_anchor, "type `" + alias_ref->name + "` circularly references itself"); + err("type `{}` circularly references itself", alias_ref).fire(alias_ref->ident_anchor); } if (auto v_genericsT_list = alias_ref->ast_root->as()->genericsT_list) { @@ -435,7 +431,7 @@ class TypeNodesVisitorResolver { return prevT.nameT == v_item->nameT; }); if (it_existing != itemsT.end()) { - fire(v_item, "duplicate generic parameter `" + static_cast(v_item->nameT) + "`"); + err("duplicate generic parameter `{}`", v_item->nameT).fire(v_item); } TypePtr default_type = nullptr; if (v_list->get_item(i)->default_type_node) { @@ -638,7 +634,7 @@ class InfiniteStructSizeDetector { bool contains = std::find(called_stack.begin(), called_stack.end(), struct_ref) != called_stack.end(); if (contains) { - fire(struct_ref->ident_anchor, "struct `" + struct_ref->name + "` size is infinity due to recursive fields"); + err("struct `{}` size is infinity due to recursive fields", struct_ref).fire(struct_ref->ident_anchor); } called_stack.push_back(struct_ref); diff --git a/tolk/pipe-transform-on-message.cpp b/tolk/pipe-transform-on-message.cpp index 4ce2ecbc92..52e5762be1 100644 --- a/tolk/pipe-transform-on-message.cpp +++ b/tolk/pipe-transform-on-message.cpp @@ -57,14 +57,14 @@ struct TransformOnInternalMessageReplacer final : ASTReplacerInFunctionBody { static void validate_onBouncedMessage(FunctionPtr f) { if (f->inferred_return_type != TypeDataVoid::create() && f->inferred_return_type != TypeDataNever::create()) { - fire(f, f->ident_anchor, "`onBouncedMessage` should return `void`"); + err("`onBouncedMessage` should return `void`").fire(f->ident_anchor, f); } if (f->get_num_params() != 1) { - fire(f, f->ident_anchor, "`onBouncedMessage` should have one parameter `InMessageBounced`"); + err("`onBouncedMessage` should have one parameter `InMessageBounced`").fire(f->ident_anchor, f); } const auto* t_struct = f->get_param(0).declared_type->try_as(); if (!t_struct || t_struct->struct_ref->name != "InMessageBounced") { - fire(f, f->ident_anchor, "`onBouncedMessage` should have one parameter `InMessageBounced`"); + err("`onBouncedMessage` should have one parameter `InMessageBounced`").fire(f->ident_anchor, f); } } @@ -72,7 +72,7 @@ struct TransformOnInternalMessageReplacer final : ASTReplacerInFunctionBody { AnyExprV replace(V v) override { // don't allow `var v = in` or passing `in` to another function (only `in.someField` is allowed) if (v->sym == param_ref) { - fire(cur_f, v, "using `" + param_ref->name + "` as an object is prohibited, because `InMessage` is a built-in struct, its fields are mapped to TVM instructions\nhint: use `" + param_ref->name + ".senderAddress` and other fields directly"); + err("using `{}` as an object is prohibited, because `InMessage` is a built-in struct, its fields are mapped to TVM instructions\n""hint: use `{}.senderAddress` and other fields directly", param_ref->name, param_ref->name).fire(v, cur_f); } return parent::replace(v); } @@ -81,7 +81,7 @@ struct TransformOnInternalMessageReplacer final : ASTReplacerInFunctionBody { // replace `in.senderAddress` / `in.valueCoins` with an aux vertex if (v->get_obj()->kind == ast_reference && v->get_obj()->as()->sym == param_ref) { if (v->is_lvalue && v->get_field_name() != "body" && v->get_field_name() != "bouncedBody") { - fire(cur_f, v, "modifying an immutable variable\nhint: fields of InMessage can be used for reading only"); + err("modifying an immutable variable\n""hint: fields of InMessage can be used for reading only").fire(v, cur_f); } ASTAuxData* aux_getField = new AuxData_OnInternalMessage_getField(cur_f, v->get_field_name()); diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index f713585a6f..967ff77ea4 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -558,7 +558,7 @@ std::ostream& operator<<(std::ostream& os, const FlowContext& flow) { } std::ostream& operator<<(std::ostream& os, const FactsAboutExpr& facts) { - os << facts.expr_type; + os << (facts.expr_type == nullptr ? "(nullptr-type)" : facts.expr_type->as_human_readable()); if (facts.sign_state != SignState::Unknown) { os << " " << to_string(facts.sign_state); } diff --git a/tolk/src-file.cpp b/tolk/src-file.cpp index 4b394e23cc..5ebb8d5994 100644 --- a/tolk/src-file.cpp +++ b/tolk/src-file.cpp @@ -41,7 +41,7 @@ const SrcFile* AllRegisteredSrcFiles::locate_and_register_source_file(const std: td::Result path = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::Realpath, filename.c_str()); if (path.is_error()) { if (v_import_filename) { - fire(v_import_filename, "Failed to import: " + path.move_as_error().message().str()); + err("Failed to import: {}", path.move_as_error().message().str()).fire(v_import_filename); } throw Fatal("Failed to locate " + filename + ": " + path.move_as_error().message().str()); } @@ -54,7 +54,7 @@ const SrcFile* AllRegisteredSrcFiles::locate_and_register_source_file(const std: td::Result text = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::ReadFile, realpath.c_str()); if (text.is_error()) { if (v_import_filename) { - fire(v_import_filename, "Failed to import: " + text.move_as_error().message().str()); + err("Failed to import: {}", text.move_as_error().message().str()).fire(v_import_filename); } throw Fatal("Failed to read " + realpath + ": " + text.move_as_error().message().str()); } diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 153ce3ebfc..83f81d01ce 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -18,14 +18,13 @@ #include "ast.h" #include "compilation-errors.h" #include "compiler-state.h" -#include "platform-utils.h" #include "generics-helpers.h" namespace tolk { void Symbol::check_import_exists_when_used_from(FunctionPtr cur_f, AnyV usage) const { #ifdef TOLK_DEBUG - assert(ident_anchor != nullptr); + tolk_assert(ident_anchor != nullptr); #endif const SrcFile* declared_in = ident_anchor->range.get_src_file(); bool has_import = false; @@ -33,7 +32,7 @@ void Symbol::check_import_exists_when_used_from(FunctionPtr cur_f, AnyV usage) c has_import |= import.imported_file == declared_in; } if (!has_import) { - fire(cur_f, usage, "Using a non-imported symbol `" + name + "`\nhint: forgot to import \"" + declared_in->extract_short_name() + "\"?"); + err("Using a non-imported symbol `{}`\n""hint: forgot to import \"{}\"?", name, declared_in->extract_short_name()).fire(usage, cur_f); } } @@ -258,22 +257,21 @@ EnumMemberPtr EnumDefData::find_member(std::string_view member_name) const { return nullptr; } -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_redefinition_of_symbol(SrcRange range, const Symbol* previous) { +static Error err_redefinition_of_symbol(const Symbol* previous) { if (previous->is_builtin()) { - fire(range, "redefinition of built-in symbol"); + return err("redefinition of built-in symbol"); } if (previous->ident_anchor->range.get_src_file()->is_stdlib_file) { - fire(range, "redefinition of a symbol from stdlib"); + return err("redefinition of a symbol from stdlib"); } - fire(range, "redefinition of symbol, previous was at: " + previous->ident_anchor->range.stringify_start_location(false)); + return err("redefinition of symbol, previous was at: {}", previous->ident_anchor->range.stringify_start_location(false)); } void GlobalSymbolTable::add_global_symbol(const Symbol* sym) { auto key = key_hash(sym->name); auto [it, inserted] = entries.emplace(key, sym); if (!inserted) { - fire_redefinition_of_symbol(sym->ident_anchor->range, it->second); + err_redefinition_of_symbol(it->second).fire(sym->ident_anchor); } } diff --git a/tolk/symtable.h b/tolk/symtable.h index 5216144df2..ecd49df99c 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -16,7 +16,6 @@ */ #pragma once -#include "src-file.h" #include "fwd-declarations.h" #include "crypto/common/refint.h" #include diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp index 437ab63d54..c569a3724f 100644 --- a/tolk/tolk.cpp +++ b/tolk/tolk.cpp @@ -68,7 +68,7 @@ int tolk_proceed(const std::string &entrypoint_filename) { } catch (const Fatal& fatal) { std::cerr << "fatal: " << fatal.message << std::endl; return 2; - } catch (const ParseError& error) { + } catch (const ThrownParseError& error) { error.output_compilation_error(std::cerr); return 2; } catch (const UnexpectedASTNodeKind& error) { diff --git a/tolk/tolk.h b/tolk/tolk.h index 057dc5ad2e..4964525deb 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -430,7 +430,6 @@ struct AsmOp { } int out(std::ostream& os) const; int out_indented(std::ostream& os, bool print_src_line_above) const; - std::string to_string() const; void compute_gconst() { gconst = (is_custom() && (op == "PUSHNULL" || op == "NEWC" || op == "NEWB" || op == "TRUE" || op == "FALSE" || op == "NOW")); } diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index e0ad443ab1..d25d3ee66c 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -170,9 +170,9 @@ TypePtr TypeDataGenericT::create(std::string&& nameT) { TypePtr TypeDataGenericTypeWithTs::create(StructPtr struct_ref, AliasDefPtr alias_ref, std::vector&& type_arguments) { if (struct_ref) { - assert(alias_ref == nullptr && struct_ref->is_generic_struct()); + tolk_assert(alias_ref == nullptr && struct_ref->is_generic_struct()); } else { - assert(struct_ref == nullptr && alias_ref->is_generic_alias()); + tolk_assert(struct_ref == nullptr && alias_ref->is_generic_alias()); } CalcChildrenFlags reg; @@ -262,13 +262,11 @@ int TypeDataAlias::get_width_on_stack() const { } int TypeDataGenericT::get_width_on_stack() const { - assert(false); - return -99999; + tolk_assert(false); } int TypeDataGenericTypeWithTs::get_width_on_stack() const { - assert(false); - return -99999; + tolk_assert(false); } int TypeDataStruct::get_width_on_stack() const { @@ -319,7 +317,7 @@ int TypeDataVoid::get_width_on_stack() const { // int TypeDataAlias::get_type_id() const { - assert(!alias_ref->is_generic_alias()); + tolk_assert(!alias_ref->is_generic_alias()); return underlying_type->get_type_id(); } @@ -328,17 +326,15 @@ int TypeDataFunCallable::get_type_id() const { } int TypeDataGenericT::get_type_id() const { - assert(false); // generics must have been instantiated in advance - throw Fatal("unexpected get_type_id() call"); + tolk_assert(false); // generics must have been instantiated in advance } int TypeDataGenericTypeWithTs::get_type_id() const { - assert(false); // `Wrapper` has to be resolved in advance - throw Fatal("unexpected get_type_id() call"); + tolk_assert(false); // `Wrapper` has to be resolved in advance } int TypeDataStruct::get_type_id() const { - assert(!struct_ref->is_generic_struct()); + tolk_assert(!struct_ref->is_generic_struct()); return TypeIdCalculation::assign_type_id(this); } @@ -347,12 +343,12 @@ int TypeDataEnum::get_type_id() const { } int TypeDataTensor::get_type_id() const { - assert(!has_genericT_inside()); + tolk_assert(!has_genericT_inside()); return TypeIdCalculation::assign_type_id(this); } int TypeDataBrackets::get_type_id() const { - assert(!has_genericT_inside()); + tolk_assert(!has_genericT_inside()); return TypeIdCalculation::assign_type_id(this); } @@ -373,18 +369,16 @@ int TypeDataBitsN::get_type_id() const { } int TypeDataUnion::get_type_id() const { - assert(false); // a union can not be inside a union - throw Fatal("unexpected get_type_id() call"); + tolk_assert(false); // a union can not be inside a union } int TypeDataMapKV::get_type_id() const { - assert(!has_genericT_inside()); + tolk_assert(!has_genericT_inside()); return TypeIdCalculation::assign_type_id(this); } int TypeDataUnknown::get_type_id() const { - assert(false); // unknown can not be inside a union - throw Fatal("unexpected get_type_id() call"); + tolk_assert(false); // unknown can not be inside a union } @@ -1444,10 +1438,4 @@ TypePtr TypeDataUnion::calculate_exact_variant_to_fit_rhs(TypePtr rhs_type) cons return first_covering; } - - -std::ostream& operator<<(std::ostream& os, TypePtr type_data) { - return os << (type_data ? type_data->as_human_readable() : "(nullptr-type)"); -} - } // namespace tolk diff --git a/tolk/type-system.h b/tolk/type-system.h index 28ddbbf202..be043a643e 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -703,6 +703,4 @@ class TypeDataVoid final : public TypeData { void type_system_init(); -std::ostream& operator<<(std::ostream& os, TypePtr type_data); - } // namespace tolk From 5bcda95f1d6386a43cebddfba0f97477e432221a Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 26 Oct 2025 18:07:40 +0300 Subject: [PATCH 452/524] [Tolk] Breaking change: separate `address` and `any_address` Before: 'address' meant internal/external/none. Now: 'address' is internal only, 'any_address' is internal/external/none. `address?` (nullable) is slice/null on a stack, but serialized as internal/none (267 or 2 bits). With new TVM 12 instructions (LDSTDADDR and others), it's now possible to load and check whether an address is internal without extra gas cost. That's the reason why it was not done before. --- crypto/smartcont/tolk-stdlib/common.tolk | 112 ++++++++++---- tolk-tester/tests/build-addr-tests.tolk | 80 ++++++++++ tolk-tester/tests/cells-slices.tolk | 52 ++++++- tolk-tester/tests/constants-tests.tolk | 8 +- tolk-tester/tests/handle-msg-1.tolk | 2 +- tolk-tester/tests/inline-tests.tolk | 2 +- .../tests/invalid-serialization/err-7604.tolk | 2 +- .../tests/invalid-serialization/err-7605.tolk | 4 +- .../tests/invalid-symbol/err-2110.tolk | 10 ++ .../tests/invalid-typing/err-6013.tolk | 1 + .../tests/invalid-typing/err-6135.tolk | 2 +- .../tests/invalid-typing/err-6149.tolk | 9 +- .../tests/invalid-typing/err-6418.tolk | 9 ++ .../tests/invalid-typing/err-6419.tolk | 10 ++ .../tests/invalid-typing/err-6509.tolk | 1 + .../tests/invalid-typing/err-6595.tolk | 2 +- .../tests/invalid-typing/err-6603.tolk | 2 +- .../tests/invalid-typing/err-6633.tolk | 2 +- .../tests/invalid-typing/err-6710.tolk | 2 +- .../tests/invalid-typing/err-6716.tolk | 2 +- .../tests/invalid-typing/err-6737.tolk | 2 +- .../tests/invalid-typing/err-6803.tolk | 2 +- .../tests/invalid-typing/err-6834.tolk | 2 +- .../tests/invalid-typing/err-6937.tolk | 2 +- .../tests/invalid-typing/err-6992.tolk | 1 + tolk-tester/tests/lazy-algo-tests.tolk | 20 ++- tolk-tester/tests/lazy-load-tests.tolk | 29 +++- tolk-tester/tests/maps-tests.tolk | 33 +++-- tolk-tester/tests/pack-unpack-2.tolk | 140 +++++++++++++++++- tolk-tester/tests/pack-unpack-3.tolk | 12 +- tolk-tester/tests/pack-unpack-5.tolk | 23 ++- tolk-tester/tests/parse-address.tolk | 5 +- tolk-tester/tests/send-msg-1.tolk | 6 +- tolk-tester/tests/send-msg-3.tolk | 16 +- tolk/builtins.cpp | 4 +- tolk/maps-kv-api.cpp | 4 +- tolk/pack-unpack-api.cpp | 6 +- tolk/pack-unpack-serializers.cpp | 80 ++++++++-- tolk/pack-unpack-serializers.h | 12 +- tolk/pipe-ast-to-legacy.cpp | 15 +- tolk/pipe-check-inferred-types.cpp | 18 ++- tolk/pipe-lazy-load-insertions.cpp | 4 +- tolk/pipe-optimize-boolean-expr.cpp | 2 +- tolk/pipe-resolve-types.cpp | 5 +- tolk/send-message-api.cpp | 12 +- tolk/type-system.cpp | 49 ++++-- tolk/type-system.h | 66 ++++++--- 47 files changed, 707 insertions(+), 177 deletions(-) create mode 100644 tolk-tester/tests/invalid-symbol/err-2110.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6418.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6419.tolk diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index d4385e3e9d..c2c6b12897 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -41,12 +41,24 @@ type continuation = builtin /// A tuple occupies one stack slot regardless of its size. type tuple = builtin -/// `address` represents an internal/external/none address. -/// Most likely, you'll use it for internal addresses — "an address of a smart contract". -/// It's `slice` under the hood. `someAddress as slice` is also possible. +/// `address` represents a standard (internal) address: workchain + hash. +/// Addresses are comparable with `==` operator, they have methods like [address.getWorkchain]. +/// In binary, it's 267 bits: '10' (addr_std) + '0' (no anycast) + workchain (int8) + hash (uint256). +/// If you wish to describe "a potentially missing address", use `address?` (nullable). +/// When set to `null`, `address?` will be serialized as '00' ("none address"): +/// '00' (two zero bits) is a standard representation of a "missing" ("none") address. +/// At the TVM level, addresses are slices, so it's possible to `someAddress as slice`. /// [See docs](https://docs.ton.org/learn/overviews/addresses#address-of-smart-contract) type address = builtin +/// `any_address` represents an internal/external/none address. +/// You can test it with [any_address.isExternal] and similar methods. +/// If you wish to describe "internal or none address", use `address?` (nullable) +/// instead of "any". That's why `any_address` is used in practice extremely rarely. +/// At the TVM level, addresses are slices, so it's possible to `anyAddress as slice`. +/// [See docs](https://docs.ton.org/learn/overviews/addresses#address-of-smart-contract) +type any_address = builtin + /// `never` is a special type that represents computations that never complete normally. /// A function that always throws an exception returns `never`, and the compiler knows /// that any code after it is unreachable. @@ -1146,10 +1158,26 @@ fun builder.storeUint(mutate self, x: int, len: int): self fun builder.storeSlice(mutate self, s: slice): self asm(s self) "STSLICE" -/// Stores an address into a builder. +/// Stores an internal address into a builder. @pure fun builder.storeAddress(mutate self, addr: address): self - asm(addr self) "STSLICE" + asm(addr self) "STSTDADDR" + +/// Stores an internal address or '00' if the parameter is `null`. +/// '00' (two zero bits) is a standard representation of a "missing" ("none") address. +@pure +fun builder.storeAddressOpt(mutate self, addrOrNull: address?): self + asm(addrOrNull self) "STOPTSTDADDR" + +/// Stores "any address" (internal/external/none) into a builder. +@pure +fun builder.storeAddressAny(mutate self, addrAny: any_address): self + asm(addrAny self) "STSLICE" + +/// Stores a "none address": '00' (two zero bits) is TL addr_none$00. +@pure +fun builder.storeAddressNone(mutate self): self + asm "b{00} STSLICECONST" /// Stores the amount of nanotoncoins into a builder. @pure @@ -1178,11 +1206,6 @@ fun builder.storeMaybeRef(mutate self, c: cell?): self fun builder.storeBuilder(mutate self, from: builder): self asm(from self) "STB" -/// Stores a slice representing "none address" (TL addr_none$00 — two zero bits). -@pure -fun builder.storeAddressNone(mutate self): self - asm "b{00} STSLICECONST" - /** Slice size primitives. @@ -1243,48 +1266,67 @@ fun builder.bitsCount(self): int /// address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5") /// address("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8") /// ``` -/// Returns `address`, which can be stored in a builder, compared with `==`, etc. +/// Returns `address`, which can be stored in a struct, compared with `==`, etc. @pure fun address(stdAddress: slice): address builtin -/// Creates a slice representing "none address" (TL addr_none$00 — two zero bits). +/// Extracts workchain and hash from a standard (internal) address. +@pure +fun address.getWorkchainAndHash(self): (int8, uint256) + asm "REWRITESTDADDR" + +/// Extracts workchain from a standard (internal) address. +@pure +fun address.getWorkchain(self): int8 + asm "REWRITESTDADDR" "DROP" + +/// Creates a slice representing "none external address" (TL addr_none$00 — two zero bits). +/// Note! Use this in combination with EXTERNAL addresses (e.g. for [createExternalLogMessage]). +/// To represent "internal or none" address (which you most likely need to), +/// use `address?` (nullable) and `null` accordingly. @pure -fun createAddressNone(): address +fun createAddressNone(): any_address asm "b{00} PUSHSLICE" -/// Returns if it's an empty address (= "none address"). -/// Don't confuse it with null! An empty address is a slice "00" (two zero bits). +/// Returns if "any address" is "none". +/// In TL/B, it's addr_none$00. @pure -fun address.isNone(self): bool +fun any_address.isNone(self): bool asm "b{00} SDBEGINSQ" "NIP" -/// Returns if it's a standard (internal) address. Such addresses contain workchain (8 bits) and hash (256 bits). -/// All contract addresses are internal, so it's the most practical use case. -/// In TL/B it's addr_std$10. -/// For internal addresses, you can call [address.getWorkchain] and [address.getWorkchainAndHash]. +/// Returns if "any address" is a standard (internal) address: workchain + hash. +/// Then it can be safely cast to `address`. +/// In TL/B, it's addr_std$10. @pure -fun address.isInternal(self): bool +fun any_address.isInternal(self): bool asm "b{10} SDBEGINSQ" "NIP" -/// Returns if it's an external address, used to communication with the outside world. -/// In TL/B it's addr_extern$01. +/// Returns if "any address" is an external address, used to communication with the outside world. +/// In TL/B, it's addr_extern$01. @pure -fun address.isExternal(self): bool +fun any_address.isExternal(self): bool asm "b{01} SDBEGINSQ" "NIP" /// Extracts workchain and hash from a standard (internal) address. /// If the address is not internal, throws a cell deserialization exception. @pure -fun address.getWorkchainAndHash(self): (int8, uint256) +fun any_address.getWorkchainAndHash(self): (int8, uint256) asm "REWRITESTDADDR" /// Extracts workchain from a standard (internal) address. /// If the address is not internal, throws a cell deserialization exception. @pure -fun address.getWorkchain(self): int8 +fun any_address.getWorkchain(self): int8 asm "REWRITESTDADDR" "DROP" +/// Casts `any_address` to `address` checking that "any" is actually "internal". +/// E.g., `addr = myAny.castToInternal()` does the check and returns `address`. +/// To skip the runtime check, use unsafe type casting instead: `addr = myAny as address`. +@pure +fun any_address.castToInternal(self): address + asm "DUP" "b{10} SDBEGINSQ" "NIP" "9 THROWIFNOT" + /// Checks whether two addresses are equal. Equivalent to `a == b`. /// Deprecated! Left for smoother transition from FunC, where you used `slice` everywhere. /// Use just `a == b` and `a != b` to compare addresses, don't use bitsEqual. @@ -1293,9 +1335,23 @@ fun address.getWorkchain(self): int8 fun address.bitsEqual(self, b: address): bool asm "SDEQ" -/// Loads a valid address (internal/external/none). In TL/B, it's `MsgAddress`. +/// Loads a standard (internal) address (containing workchain + hash). In TL/B, it's `MsgAddressInt`. +/// If an address is incorrect or is not internal, throws a deserialization exception (code 9). @pure fun slice.loadAddress(mutate self): address + asm( -> 1 0) "LDSTDADDR" + +/// Loads a standard (internal) address or returns `null` if the input is "none address". +/// If an address is incorrect or is not internal/none, throws a deserialization exception (code 9). +/// '00' (two zero bits) is a standard representation of a "missing" ("none") address. +/// As a result, you get either `address` (for internal) or `null` (for none). +@pure +fun slice.loadAddressOpt(mutate self): address? + asm( -> 1 0) "LDOPTSTDADDR" + +/// Loads a valid "any address" (internal/external/none). In TL/B, it's `MsgAddress`. +@pure +fun slice.loadAddressAny(mutate self): any_address asm( -> 1 0) "LDMSGADDR" @@ -1514,7 +1570,7 @@ struct (0x00) ExtOutLogBucket { /// Consider [createExternalLogMessage] for examples. struct CreateExternalLogMessageOptions { /// destination is either an external address or a pattern to calculate it - dest: address // either some valid external/none address (not internal!) + dest: any_address // either some valid external/none address (not internal!) | builder // ... or a manually constructed builder with a valid external address | ExtOutLogBucket // ... or encode topic/eventID in destination /// body is any serializable object (or just miss this field for empty body) diff --git a/tolk-tester/tests/build-addr-tests.tolk b/tolk-tester/tests/build-addr-tests.tolk index 9c1704a23a..49af0a6455 100644 --- a/tolk-tester/tests/build-addr-tests.tolk +++ b/tolk-tester/tests/build-addr-tests.tolk @@ -287,6 +287,81 @@ fun test14() { ) } +@method_id(115) +fun test15() { + var any = createAddressNone(); + var addr = address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); + + any as any_address; + any as address; + (any as slice) as any_address; + addr as any_address; + addr as address; + (addr as slice) as address; + + return ( + any == addr, any != addr, + (any as address) == addr, any == (addr as any_address), + (any as slice).bitsEqual(addr as slice), + (any as bits123 as any_address as slice).remainingBitsCount(), + (addr as bytes1 as address as slice).remainingBitsCount(), + ) +} + +@method_id(116) +fun test16() { + val any = address("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf") as any_address; + val addr = any.castToInternal(); + return ( + addr == any, (any as slice).remainingBitsCount(), (addr as slice).remainingBitsCount(), + addr.getWorkchain() == any.getWorkchain(), + addr.getWorkchainAndHash().1 == any.getWorkchainAndHash().1, + ); +} + +@method_id(117) +fun test17() { + val ex = beginCell().storeUint(0b01, 2).storeUint(6, 9).storeUint(0, 6).endCell().beginParse() as any_address; + val in = address("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf") as any_address; + val no = createAddressNone(); + __expect_type(no, "any_address"); + + assert (ex.isExternal()) throw 123; + assert (in.isInternal()) throw 123; + assert (no.isNone()) throw 123; + return (ex.isInternal(), ex.isNone(), in.isExternal(), in.isNone(), no.isInternal(), no.isExternal()); +} + +@method_id(118) +fun test18(): (int, int8) { + try { + val any = createAddressNone(); + return (0, any.castToInternal().getWorkchain()); + } catch (ex) { + try { + return (ex, createAddressNone().getWorkchain()) + } catch (ex2) { + return (ex, ex2); + } + } +} + +@method_id(119) +fun test19() { + val a = address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); + + val c1 = a.toCell(); + val c2 = (a as address?).toCell(); + val c3 = (a as any_address).toCell(); + assert (c1.hash() == c2.hash()) throw 123; + assert (c1.hash() == c3.hash()) throw 123; + + val e1 = (null as address?).toCell(); + val e2 = createAddressNone().toCell(); + assert (e1.hash() == e2.hash()) throw 123; + + return e1.beginParse().remainingBitsCount(); +} fun main() {} @@ -304,4 +379,9 @@ fun main() {} @testcase | 112 | | 10 @testcase | 113 | | -1 0 @testcase | 114 | | -1 -1 -1 +@testcase | 115 | | 0 -1 0 0 0 2 267 +@testcase | 116 | | -1 267 267 -1 -1 +@testcase | 117 | | 0 0 0 0 0 0 +@testcase | 118 | | 9 9 +@testcase | 119 | | 2 */ diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 0b94b19c4e..c5bd45125e 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -193,7 +193,7 @@ fun test111() { .endCell().beginParse(); var op1 = s.loadUint(32); var q1 = s.loadUint(64); - if ((s as address).isNone()) { + if ((s as any_address).isNone()) { s.skipBits(2); } if (s.loadBool() == false) { @@ -333,7 +333,7 @@ fun test24(uns: bool) { fun builder.my_storeRef(mutate self, value: cell): self asm(self value) "STREFR"; -fun builder.my_storeAddress(mutate self, value: address): self +fun builder.my_storeAddress(mutate self, value: any_address): self asm(value self) "STSLICE"; fun builder.my_storeBuilder(mutate self, value: builder): self @@ -368,7 +368,7 @@ fun demo33(value: builder, dest: builder) { } @noinline -fun demo34(dest: builder, value: address) { +fun demo34(dest: builder, value: any_address) { // SWAP + STSLICE => STSLICER return dest.my_storeAddress(value); } @@ -489,6 +489,51 @@ fun test40() { return beginCell().toSlice().remainingBitsAndRefsCount() } +@method_id(141) +fun test41() { + var empty = ""; + var incorrect = stringHexToSlice("8"); + var incomplete = (address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF") as slice).getFirstBits(266); + + var data = createEmptyTuple(); + var b = beginCell(); + + try { data.push(empty.loadAddress()) } + catch (ex) { data.push(ex) } + try { data.push(incorrect.loadAddress()) } + catch (ex) { data.push(ex) } + try { data.push(incomplete.loadAddress()) } + catch (ex) { data.push(ex) } + + try { data.push(empty.loadAddressOpt()) } + catch (ex) { data.push(ex) } + try { data.push(incorrect.loadAddressOpt()) } + catch (ex) { data.push(ex) } + try { data.push(incomplete.loadAddressOpt()) } + catch (ex) { data.push(ex) } + + try { data.push(empty.loadAddressAny()) } + catch (ex) { data.push(ex) } + try { data.push(incorrect.loadAddressAny()) } + catch (ex) { data.push(ex) } + try { data.push(incomplete.loadAddressAny()) } + catch (ex) { data.push(ex) } + + try { b.storeAddress(createAddressNone() as address) } + catch (ex) { data.push(ex) } + try { b.storeAddressOpt(createAddressNone() as address) } + catch (ex) { data.push(ex) } + try { b.storeAny("123" as address) } + catch (ex) { data.push(ex) } + try { b.storeAny("123" as address?) } + catch (ex) { data.push(ex) } + + try { b.storeAddressOpt(null); data.push(b.bitsCount()) } + catch (ex) { data.push(ex) } + + return data; +} + fun main(): int { return 0; } @@ -535,6 +580,7 @@ fun main(): int { @testcase | 137 | | 39065 @testcase | 138 | | 0 @testcase | 140 | | 0 0 +@testcase | 141 | | [ 9 9 9 9 9 9 9 9 9 8 8 8 8 2 ] We test that consequtive storeInt/storeUint with constants are joined into a single number diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index 42cace8b20..6f7c993b7d 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -23,6 +23,7 @@ const strange_zero = (!10 as int); const strange_minus_1: MInt = (!0 as int); const addr1 = address("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"); +const addr2: any_address = address("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF") as any_address; // since `;` is not mandatory, this is correct from a syntax point of view const true1 = true const true2 = !!true const true3 = true1 && true2 @@ -92,7 +93,8 @@ fun test5() { @method_id(106) fun test6() { __expect_type(addr1, "address"); - return (addr1 == addr1, addr1 != addr1, addr1 == createAddressNone(), addr1.getWorkchain()); + __expect_type(addr2, "any_address"); + return (addr1 == addr1, addr1 != addr1, addr1 == createAddressNone(), addr1.getWorkchain(), addr1 == addr2); } fun main() { @@ -124,7 +126,7 @@ fun main() { @testcase | 103 | | 0 0 @testcase | 104 | | 1 1 2 @testcase | 105 | | -1 0 7 48 -@testcase | 106 | | -1 0 0 -1 +@testcase | 106 | | -1 0 0 -1 -1 -@code_hash 109852777684517203308276021763918179390784357880680494742516881492720986364812 +@code_hash 50461262039225352661292504472979593396591696467874442618206926482074675176667 */ diff --git a/tolk-tester/tests/handle-msg-1.tolk b/tolk-tester/tests/handle-msg-1.tolk index 1e08082d55..79c41ebb0a 100644 --- a/tolk-tester/tests/handle-msg-1.tolk +++ b/tolk-tester/tests/handle-msg-1.tolk @@ -41,7 +41,7 @@ fun entrypoint1() { @method_id(102) fun handle2(in: InMessage) { - return (in.valueCoins, in.senderAddress.isInternal()); + return (in.valueCoins, (in.senderAddress as any_address).isInternal()); } @method_id(202) diff --git a/tolk-tester/tests/inline-tests.tolk b/tolk-tester/tests/inline-tests.tolk index d27e79bb8c..c1f48506c0 100644 --- a/tolk-tester/tests/inline-tests.tolk +++ b/tolk-tester/tests/inline-tests.tolk @@ -453,7 +453,7 @@ fun usedIn10ButDeclaredBelow(x: int) { test5() PROC:<{ // c4 PUSH // '7 CTOS // s - LDMSGADDR // '11 s + LDSTDADDR // '11 s """ @fif_codegen diff --git a/tolk-tester/tests/invalid-serialization/err-7604.tolk b/tolk-tester/tests/invalid-serialization/err-7604.tolk index ea6edfe7ae..3c4e614177 100644 --- a/tolk-tester/tests/invalid-serialization/err-7604.tolk +++ b/tolk-tester/tests/invalid-serialization/err-7604.tolk @@ -5,7 +5,7 @@ struct CCC { fun main() { var options = CreateMessageOptions { bounce: true, - dest: createAddressNone(), + dest: address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"), value: 0, body: CCC { v: 10 } }; diff --git a/tolk-tester/tests/invalid-serialization/err-7605.tolk b/tolk-tester/tests/invalid-serialization/err-7605.tolk index 087e56457b..7c6f0cf8f7 100644 --- a/tolk-tester/tests/invalid-serialization/err-7605.tolk +++ b/tolk-tester/tests/invalid-serialization/err-7605.tolk @@ -2,8 +2,8 @@ struct BigBodyMessage { v1: int256; v2: int256; v3: int256; - a1: address; - a2: address; + a1: address?; + a2: address?; } fun testFail(body: BigBodyMessage) { diff --git a/tolk-tester/tests/invalid-symbol/err-2110.tolk b/tolk-tester/tests/invalid-symbol/err-2110.tolk new file mode 100644 index 0000000000..37e9ed5780 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2110.tolk @@ -0,0 +1,10 @@ +fun main() { + var a = address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e"); + return a.isInternal(); +} + +/** +@compilation_should_fail +@stderr method `isInternal` not found for type `address` +@stderr (but it exists for type `any_address`) + */ diff --git a/tolk-tester/tests/invalid-typing/err-6013.tolk b/tolk-tester/tests/invalid-typing/err-6013.tolk index c5ec6783d7..b9ebf2e450 100644 --- a/tolk-tester/tests/invalid-typing/err-6013.tolk +++ b/tolk-tester/tests/invalid-typing/err-6013.tolk @@ -15,4 +15,5 @@ fun main(p: Pair) { @compilation_should_fail @stderr can not apply operator `==` to `int` and `int?` @stderr in function `Pair.demoInvalid` +@stderr check on `null` first, or use unsafe operator `!` */ diff --git a/tolk-tester/tests/invalid-typing/err-6135.tolk b/tolk-tester/tests/invalid-typing/err-6135.tolk index 7bf5808719..4d378ab48b 100644 --- a/tolk-tester/tests/invalid-typing/err-6135.tolk +++ b/tolk-tester/tests/invalid-typing/err-6135.tolk @@ -8,5 +8,5 @@ fun cantAssignIntNToCoins(n: int8, c: coins) { @compilation_should_fail @stderr can not assign `coins` to variable of type `int8` @stderr n = c; -@stderr hint: use `as` operator for unsafe casting: ` as int8` +@stderr hint: use `as` operator for UNSAFE casting: ` as int8` */ diff --git a/tolk-tester/tests/invalid-typing/err-6149.tolk b/tolk-tester/tests/invalid-typing/err-6149.tolk index 7fd34b6462..1db2ad2369 100644 --- a/tolk-tester/tests/invalid-typing/err-6149.tolk +++ b/tolk-tester/tests/invalid-typing/err-6149.tolk @@ -7,12 +7,13 @@ fun Pair.compareWith(self, f: U, s: V) { return self.first == f && self.second == s; } -fun main(p: Pair) { - p.compareWith(null, null); +fun main(p: Pair) { + p.compareWith(null, null); } /** @compilation_should_fail -@stderr can not apply operator `==` to `int` and `int?` -@stderr in function `Pair.compareWith` +@stderr can not apply operator `==` to `address` and `address?` +@stderr in function `Pair.compareWith` +@stderr check on `null` first, or use unsafe operator `!` */ diff --git a/tolk-tester/tests/invalid-typing/err-6418.tolk b/tolk-tester/tests/invalid-typing/err-6418.tolk new file mode 100644 index 0000000000..09e4ac6f18 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6418.tolk @@ -0,0 +1,9 @@ +fun main(any: any_address, usual: address) { + any = usual; +} + +/** +@compilation_should_fail +@stderr can not assign `address` to variable of type `any_address` +@stderr hint: use `as` operator for conversion: ` as any_address` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6419.tolk b/tolk-tester/tests/invalid-typing/err-6419.tolk new file mode 100644 index 0000000000..75cb3b11ef --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6419.tolk @@ -0,0 +1,10 @@ +fun main(any: any_address, usual: address) { + usual = any; +} + +/** +@compilation_should_fail +@stderr can not assign `any_address` to variable of type `address` +@stderr hint: use `any_addr.castToInternal()` to check and get `address` +@stderr hint: alternatively, use `as` operator for UNSAFE casting: ` as address` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6509.tolk b/tolk-tester/tests/invalid-typing/err-6509.tolk index d10e21743d..9482f6d0aa 100644 --- a/tolk-tester/tests/invalid-typing/err-6509.tolk +++ b/tolk-tester/tests/invalid-typing/err-6509.tolk @@ -8,4 +8,5 @@ fun cantApplyPlusOnNullable() { @compilation_should_fail @stderr in function `calcSum` @stderr can not apply operator `+` to `int?` and `int?` +@stderr check on `null` first, or use unsafe operator `!` */ diff --git a/tolk-tester/tests/invalid-typing/err-6595.tolk b/tolk-tester/tests/invalid-typing/err-6595.tolk index c016ae187d..4e0cfe0382 100644 --- a/tolk-tester/tests/invalid-typing/err-6595.tolk +++ b/tolk-tester/tests/invalid-typing/err-6595.tolk @@ -5,6 +5,6 @@ fun main() { /** @compilation_should_fail @stderr can not pass `bool` to `int` -@stderr hint: use `as` operator for unsafe casting: ` as int` +@stderr hint: use `as` operator for UNSAFE casting: ` as int` @stderr caution! in TVM, bool TRUE is -1, not 1 */ diff --git a/tolk-tester/tests/invalid-typing/err-6603.tolk b/tolk-tester/tests/invalid-typing/err-6603.tolk index 5f6275c350..1deeb7c216 100644 --- a/tolk-tester/tests/invalid-typing/err-6603.tolk +++ b/tolk-tester/tests/invalid-typing/err-6603.tolk @@ -10,5 +10,5 @@ fun main(a: A) { /** @compilation_should_fail @stderr can not assign `[int, int]` to field of type `tuple` -@stderr hint: use `as` operator for unsafe casting: ` as tuple` +@stderr hint: use `as` operator for UNSAFE casting: ` as tuple` */ diff --git a/tolk-tester/tests/invalid-typing/err-6633.tolk b/tolk-tester/tests/invalid-typing/err-6633.tolk index d8ed9c2980..5985c97aeb 100644 --- a/tolk-tester/tests/invalid-typing/err-6633.tolk +++ b/tolk-tester/tests/invalid-typing/err-6633.tolk @@ -7,5 +7,5 @@ fun main(c: cell) { @stderr can not assign `address` to variable of type `slice` @stderr hint: unlike FunC, Tolk has a special type `address` (which is slice at the TVM level); @stderr most likely, you just need `address` everywhere -@stderr hint: alternatively, use `as` operator for unsafe casting: ` as slice` +@stderr hint: alternatively, use `as` operator for UNSAFE casting: ` as slice` */ diff --git a/tolk-tester/tests/invalid-typing/err-6710.tolk b/tolk-tester/tests/invalid-typing/err-6710.tolk index d0fae1d56d..9132d19191 100644 --- a/tolk-tester/tests/invalid-typing/err-6710.tolk +++ b/tolk-tester/tests/invalid-typing/err-6710.tolk @@ -10,6 +10,6 @@ fun testCantPassAddressToSlice() { /** @compilation_should_fail -@stderr can not pass `address` to `slice` +@stderr can not pass `any_address` to `slice` @stderr f(a); */ diff --git a/tolk-tester/tests/invalid-typing/err-6716.tolk b/tolk-tester/tests/invalid-typing/err-6716.tolk index 134fb9d14f..ce6fed39ec 100644 --- a/tolk-tester/tests/invalid-typing/err-6716.tolk +++ b/tolk-tester/tests/invalid-typing/err-6716.tolk @@ -3,5 +3,5 @@ fun f(d: bits8 = "") {} /** @compilation_should_fail @stderr can not assign `slice` to `bits8` -@stderr hint: use `as` operator for unsafe casting: ` as bits8` +@stderr hint: use `as` operator for UNSAFE casting: ` as bits8` */ diff --git a/tolk-tester/tests/invalid-typing/err-6737.tolk b/tolk-tester/tests/invalid-typing/err-6737.tolk index 86934f3c8f..ce408bd7b6 100644 --- a/tolk-tester/tests/invalid-typing/err-6737.tolk +++ b/tolk-tester/tests/invalid-typing/err-6737.tolk @@ -9,5 +9,5 @@ fun testAssignBetweenDifferentIntN(op: int32, qid: uint64) { @compilation_should_fail @stderr can not assign `uint64` to variable of type `int32` @stderr op = qid; -@stderr hint: use `as` operator for unsafe casting: ` as int32` +@stderr hint: use `as` operator for UNSAFE casting: ` as int32` */ diff --git a/tolk-tester/tests/invalid-typing/err-6803.tolk b/tolk-tester/tests/invalid-typing/err-6803.tolk index 1e9592a68f..c3f802ecba 100644 --- a/tolk-tester/tests/invalid-typing/err-6803.tolk +++ b/tolk-tester/tests/invalid-typing/err-6803.tolk @@ -9,5 +9,5 @@ fun cantPassBytesNToSlice(c: bits16) { @compilation_should_fail @stderr can not pass `bits16` to `slice` @stderr takeAnySlice(c); -@stderr hint: use `as` operator for unsafe casting: ` as slice` +@stderr hint: use `as` operator for UNSAFE casting: ` as slice` */ diff --git a/tolk-tester/tests/invalid-typing/err-6834.tolk b/tolk-tester/tests/invalid-typing/err-6834.tolk index 9936e96fa5..069e14576d 100644 --- a/tolk-tester/tests/invalid-typing/err-6834.tolk +++ b/tolk-tester/tests/invalid-typing/err-6834.tolk @@ -10,5 +10,5 @@ fun m(a: SnakedCell, b: SnakedCell) { /** @compilation_should_fail @stderr can not assign `SnakedCell` to variable of type `SnakedCell` -@stderr hint: use `as` operator for unsafe casting: ` as SnakedCell` +@stderr hint: use `as` operator for UNSAFE casting: ` as SnakedCell` */ diff --git a/tolk-tester/tests/invalid-typing/err-6937.tolk b/tolk-tester/tests/invalid-typing/err-6937.tolk index 6038ea656c..b37e3cd722 100644 --- a/tolk-tester/tests/invalid-typing/err-6937.tolk +++ b/tolk-tester/tests/invalid-typing/err-6937.tolk @@ -9,5 +9,5 @@ fun cantAssignSliceToBytesN() { @compilation_should_fail @stderr can not assign `slice` to variable of type `bytes16` @stderr var c2: bytes16 = getAnySlice(); -@stderr hint: use `as` operator for unsafe casting: ` as bytes16` +@stderr hint: use `as` operator for UNSAFE casting: ` as bytes16` */ diff --git a/tolk-tester/tests/invalid-typing/err-6992.tolk b/tolk-tester/tests/invalid-typing/err-6992.tolk index 53dfc5cace..bbf2b0ec12 100644 --- a/tolk-tester/tests/invalid-typing/err-6992.tolk +++ b/tolk-tester/tests/invalid-typing/err-6992.tolk @@ -12,4 +12,5 @@ fun testDoWhileCondition() { /** @compilation_should_fail @stderr can not apply operator `*` to `int` and `int?` +@stderr check on `null` first, or use unsafe operator `!` */ diff --git a/tolk-tester/tests/lazy-algo-tests.tolk b/tolk-tester/tests/lazy-algo-tests.tolk index ac8e9fcd53..6dd429122c 100644 --- a/tolk-tester/tests/lazy-algo-tests.tolk +++ b/tolk-tester/tests/lazy-algo-tests.tolk @@ -148,7 +148,7 @@ struct NotFixedWidthStorage { f4: bool; f5: cell; f6: int8 | bool; - f7: address; + f7: address?; f8: int8; f9: bits200; f10: (bits400, ()); @@ -197,6 +197,12 @@ struct HasIncrementEnd { inc: CounterIncrement; } +struct HasUnusedAddressInTensor { + f1: (int8, bits267, int8) // can be safely skipped as bits283 + f2: (int8, address, int8) // can NOT (even fixed-size) since `address` needs validation + f3: int8 +} + fun loadWalletStorage() { return WalletStorage.fromSlice(stringHexToSlice("8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_")); @@ -663,7 +669,7 @@ fun algo49(forceLoad: bool) { fun algo50() { var st = lazy NotFixedWidthStorage.fromSlice(""); - __expect_lazy("[st] skip (bits8) (int8?) (bits9) (int8 | bool) (address) (bits608), load f11"); + __expect_lazy("[st] skip (bits8) (int8?) (bits9) (int8 | bool) (address?) (bits608), load f11"); st.f11; } @@ -1463,7 +1469,7 @@ fun algo311() { fun algo312() { var st = lazy NotFixedWidthStorage.fromSlice(""); - __expect_lazy("[st] load (bits8) (int8?) (bits9) (cell) (int8 | bool) (address) f8, save immutable (tail)"); + __expect_lazy("[st] load (bits8) (int8?) (bits9) (cell) (int8 | bool) (address?) f8, save immutable (tail)"); if (rand()) { st.f8 += 1; } @@ -1473,13 +1479,19 @@ fun algo312() { fun algo313() { var st = lazy NotFixedWidthStorage.fromSlice(""); if (rand()) { - __expect_lazy("[st] load (bits8) (int8?) (bits9) f5, save immutable (tail), skip (int8 | bool) (address) (bits8), load f9"); + __expect_lazy("[st] load (bits8) (int8?) (bits9) f5, save immutable (tail), skip (int8 | bool) (address?) (bits8), load f9"); st.f5 = createEmptyCell(); st.f9; st.toCell(); } } +fun algo314() { + var l = lazy HasUnusedAddressInTensor.fromSlice(""); + __expect_lazy("[l] skip (bits283) ((int8, address, int8)), load f3"); + l.f3; +} + fun main() { } diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index 0954efc241..f6e78c7806 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -137,7 +137,7 @@ struct NotFixedWidthStorage { f4: bool; f5: cell; f6: int8 | bool; - f7: address; + f7: any_address; f8: int8; f9: bits200; f10: (bits400, ()); @@ -222,6 +222,12 @@ struct WithVariadicInts { i2: varuint32; } +struct WithInternalAddress { + f1: int8; + f2: address; + f3: int8; +} + global gModByCustom: int; type MagicGlobalModifier = (); @@ -1353,7 +1359,7 @@ fun demo310() { var st = lazy cell.load(); var sm = lazy cell.load(); var sk = lazy cell.load(); - __expect_lazy("[st] load (bits8) (int8?) (bits9) (cell) f6, save immutable (tail), skip (address), load f8"); + __expect_lazy("[st] load (bits8) (int8?) (bits9) (cell) f6, save immutable (tail), skip (any_address), load f8"); match (st.f6) { int8 => st.f6 += 90, bool => throw 123 }; __expect_lazy("[sm] load f5"); sm.f5; // previous (data-only) fields are not needed to reach a ref @@ -1371,6 +1377,24 @@ fun demo311() { return st.toCell(); } +@method_id(312) +fun demo312() { + val addr = address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); + val correct = beginCell().storeInt(0, 8).storeAddress(addr).storeInt(-1, 8).endCell(); + var st = lazy WithInternalAddress.fromCell(correct); + __expect_lazy("[st] skip (bits8) (address), load f3"); + val f3 = st.f3; + + val wrong = beginCell().storeInt(0, 8).storeAddressNone().storeAddress(addr).storeInt(-1, 8).endCell(); + var st2 = lazy WithInternalAddress.fromCell(wrong); + // check that `address` (st2.f2) is validated even though unused + try { + return (f3, 0, st2.f3); + } catch (ex) { + return (f3, ex, 0 as int8); + } +} + @method_id(400) fun demo400() { @@ -1740,6 +1764,7 @@ fun main() { @testcase | 309 | | C{EE7C06B69DDCEFEBED507FE35F603A9019D4AE79A13F670FE4E08928AF484838} @testcase | 310 | | 3332695883 9 @testcase | 311 | | C{16A4757FA412AD5E89418F1263904CD57FA4DF0E73EE8A53DC60F862C72BCD9D} +@testcase | 312 | | -1 9 0 @testcase | 400 | | 5 @testcase | 401 | x{0110} -1 | 50 diff --git a/tolk-tester/tests/maps-tests.tolk b/tolk-tester/tests/maps-tests.tolk index e80d9c01b5..8bb7b95044 100644 --- a/tolk-tester/tests/maps-tests.tolk +++ b/tolk-tester/tests/maps-tests.tolk @@ -26,7 +26,7 @@ struct Point3d { } struct DataWithCell { - a: address + a: address? v: coins p: Cell? payload: cell? = null @@ -135,10 +135,15 @@ fun test7() { @method_id(108) fun test8() { - var (a, m: map) = (1, createEmptyMap()); - val prev1 = m.setAndGetPrevious(a, createAddressNone()); + var (a, m: map, m2: map) = (1, createEmptyMap(), createEmptyMap()); + val prev1 = m.setAndGetPrevious(a, null); val prev2 = m.setAndGetPrevious(a, address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5")); - return (prev1.isFound, prev2.isFound, prev2.loadValue().isNone(), prev2.loadValue().isNone(), m.get(a).loadValue().getWorkchain()); + val prev3 = m2.setAndGetPrevious(a, createAddressNone()); + val prev4 = m2.setAndGetPrevious(a, address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5") as any_address); + return ( + prev1.isFound, prev2.isFound, prev2.loadValue() == null, prev2.loadValue() == null, m.get(a).loadValue()!.getWorkchain(), + prev3.isFound, prev4.isFound, prev4.loadValue().isNone(), prev4.loadValue().isNone(), m2.get(a).loadValue()!.getWorkchain(), + ); } @method_id(109) @@ -280,18 +285,18 @@ fun test22() { @method_id(123) fun test23() { var m: map = createEmptyMap(); - val p1 = m.setAndGetPrevious({x:1,y:1}, {a: createAddressNone(), v: ton("0.05"), p: Point{x:10,y:10}.toCell(), payload: null}); - val p2 = m.setAndGetPrevious({x:1,y:1}, {a: createAddressNone(), v: ton("0.1"), p: null, payload: createEmptyCell()}); - return (p1.isFound, p2.isFound, p2.loadValue().p!.load(), m.mustGet({x:1,y:1}).a.isNone(), m.get({x:1,y:1}).loadValue().payload.depth()); + val p1 = m.setAndGetPrevious({x:1,y:1}, {a: null, v: ton("0.05"), p: Point{x:10,y:10}.toCell(), payload: null}); + val p2 = m.setAndGetPrevious({x:1,y:1}, {a: null, v: ton("0.1"), p: null, payload: createEmptyCell()}); + return (p1.isFound, p2.isFound, p2.loadValue().p!.load(), m.mustGet({x:1,y:1}).a == null, m.get({x:1,y:1}).loadValue().payload.depth()); } @method_id(124) fun test24() { var m = createEmptyMap(); val p1 = m.deleteAndGetDeleted({x: 1, y: 1}); - m.set({x: 3, y: 3}, {a: createAddressNone(), v: ton("0.05"), p: Point{x:30,y:30}.toCell(), payload: createEmptyCell()}); - m.set({x: 1, y: 1}, {a: createAddressNone(), v: ton("0.05"), p: Point{x:10,y:10}.toCell(), payload: null}); - m.set({x: 2, y: 2}, {a: createAddressNone(), v: ton("0.05"), p: null, payload: null}); + m.set({x: 3, y: 3}, {a: null, v: ton("0.05"), p: Point{x:30,y:30}.toCell(), payload: createEmptyCell()}); + m.set({x: 1, y: 1}, {a: null, v: ton("0.05"), p: Point{x:10,y:10}.toCell(), payload: null}); + m.set({x: 2, y: 2}, {a: null, v: ton("0.05"), p: null, payload: null}); val k2 = m.findKeyGreaterOrEqual({x: 2, y: 2}).getKey(); var r = m.findFirst(); var t = createEmptyTuple(); @@ -350,9 +355,9 @@ fun test28() { @method_id(129) fun test29() { var m = createEmptyMap(); - val a1 = m.addIfNotExists(1, DataWithCell{a: createAddressNone(), v: 0, p: null, payload: null}.toCell().beginParse()); - val a2 = m.addIfNotExists(2, DataWithCell{a: createAddressNone(), v: 0, p: Point{x:20,y:20}.toCell(), payload: null}.toCell().beginParse()); - val a3 = m.addIfNotExists(3, DataWithCell{a: createAddressNone(), v: 0, p: Point{x:30,y:30}.toCell(), payload: createEmptyCell()}.toCell().beginParse()); + val a1 = m.addIfNotExists(1, DataWithCell{a: null, v: 0, p: null, payload: null}.toCell().beginParse()); + val a2 = m.addIfNotExists(2, DataWithCell{a: null, v: 0, p: Point{x:20,y:20}.toCell(), payload: null}.toCell().beginParse()); + val a3 = m.addIfNotExists(3, DataWithCell{a: null, v: 0, p: Point{x:30,y:30}.toCell(), payload: createEmptyCell()}.toCell().beginParse()); val r4 = m.replaceAndGetPrevious(4, ""); return ( a1, a2, a3, r4, @@ -534,7 +539,7 @@ fun main() { @testcase | 105 | | [ 1 10 2 20 3 30 30 30 20 10 ] @testcase | 106 | | 2 1 2 3 20 30 -1 0 @testcase | 107 | | -1 0 0 -@testcase | 108 | | 0 -1 -1 -1 0 +@testcase | 108 | | 0 -1 -1 -1 0 0 -1 -1 -1 0 @testcase | 109 | | -1 50000000 (null) 0 10000000 @testcase | 110 | | -1 0 1024 0 -1 20 @testcase | 111 | | -1 -1 (null) 1 -1 (null) (null) 0 diff --git a/tolk-tester/tests/pack-unpack-2.tolk b/tolk-tester/tests/pack-unpack-2.tolk index 7bf0f247ca..ca1fa9ca0e 100644 --- a/tolk-tester/tests/pack-unpack-2.tolk +++ b/tolk-tester/tests/pack-unpack-2.tolk @@ -8,8 +8,8 @@ struct EitherRight { value: T } type Either = EitherLeft | EitherRight; @inline -fun makeExternalAddress(hash: int, len: int): address { - return beginCell().storeUint(0b01, 2).storeUint(len, 9).storeUint(hash, len).endCell().beginParse() as address; +fun makeExternalAddress(hash: int, len: int): any_address { + return beginCell().storeUint(0b01, 2).storeUint(len, 9).storeUint(hash, len).endCell().beginParse() as any_address; } @@ -138,7 +138,7 @@ struct JustAddress { struct TwoInts32And64SepByAddress { op: int32; - addr_e: address; + addr_e: any_address; query_id: uint64; } @@ -303,7 +303,7 @@ struct DifferentIntsWithMaybe { struct DifferentMix1 { ja1: JustAddress; ja2m: JustAddress?; - ext_nn: address; + ext_nn: any_address; imm: IntAndMaybeMaybe8; tis: TwoInts32And64SepByAddress; } @@ -427,6 +427,39 @@ struct WithEnums { rem: uint7 = 0 } +type MyInt32 = int32 +type MaybeJustOfAddress = MaybeJust

+type AddressAlias = address + +@overflow1023_policy("suppress") +struct WithMoreTrickyAddresses1 { + a1: AddressAlias? + a2: address | MyInt32 | MaybeNothing + a3: TwoInts32And64 | MaybeJustOfAddress + a4: Cell? + a5: address | Cell +} + +@overflow1023_policy("suppress") +struct WithMoreTrickyAddresses2 { + a1: Maybe
+ a2: Maybe + a3: JustAddress? + a4: address | int64 | null + a5: address | Cell
+} + +@overflow1023_policy("suppress") +struct WithAnyAddress { + a1: any_address + a2: any_address? + a3: any_address | int64 + a4: address | any_address + a5: Maybe + a6: Cell + a7: Cell? +} + // --------------------------------------------- @@ -488,6 +521,7 @@ fun test_JustAddress() { @method_id(208) fun test_TwoInts32And64SepByAddress() { + run({ op: 123, addr_e: address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e") as any_address, query_id: 256 } , stringHexToSlice("0000007B80194DC6438F99D3D9DBE151944925D90B2492954BF6B9C070FBFF2DDED5F30547C0000000000000201_")); run({ op: 123, addr_e: makeExternalAddress(70, 10), query_id: 255 } , stringHexToSlice("0000007B41423000000000000007FC_")); run({ op: 123, addr_e: makeExternalAddress(666, 20), query_id: 254 } , stringHexToSlice("0000007B4280053400000000000001FD_")); run({ op: 123, addr_e: createAddressNone(), query_id: 253 } , stringHexToSlice("0000007B000000000000003F6_")); @@ -674,7 +708,7 @@ fun test_RestIsBuilderOrRemaining() { .storeUint(5, 32) .storeAddress(address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8")) .storeMaybeRef(JustInt32 { value: 123 }.toCell()) - .storeRef(JustAddress { addr: createAddressNone() }.toCell()); + .storeRef(JustAddress { addr: address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8") }.toCell()); var w: ReadWriteRest = { f1: 60, f2: ton("0.05"), @@ -683,7 +717,7 @@ fun test_RestIsBuilderOrRemaining() { var r: ReadRest_Remaining = w.toCell().beginParse().loadAny(); __expect_type(r.rest, "RemainingBitsAndRefs"); var rest = Tail224.fromSlice(r.rest); - return (r.f1, r.f2, rest.ji.value, rest.addr.getWorkchain(), rest.ref1!.load(), rest.ref2.load().addr.isNone()); + return (r.f1, r.f2, rest.ji.value, rest.addr.getWorkchain(), rest.ref1!.load(), (rest.ref2.load().addr as any_address).isNone()); } @method_id(225) @@ -789,6 +823,95 @@ fun test_SimpleEnums() { return -1; } +@method_id(233) +fun test_WithMoreTrickyAddresses1() { + run(WithMoreTrickyAddresses1{ + a1: address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), + a2: MaybeNothing {}, + a3: MaybeJust { value: address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e") }, + a4: (null as address?).toCell(), + a5: address("0:0000000000000000000000000000000000000000000000000000000000000000"), + }, stringHexToSlice("80194dc6438f99d3d9dbe151944925d90b2492954bf6b9c070fbff2dded5f30547d6006537190e3e674f676f8546512497642c924a552fdae701c3effcb77b57cc151f50000000000000000000000000000000000000000000000000000000000000000002_").appendRef(stringHexToSlice("2_"))); + run(WithMoreTrickyAddresses1{ + a1: null, + a2: address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e"), + a3: TwoInts32And64{ op: 123, query_id: 456 }, + a4: null, + a5: createAddressNone().toCell(), + }, stringHexToSlice("080194dc6438f99d3d9dbe151944925d90b2492954bf6b9c070fbff2dded5f30547c0000007b00000000000001c86_").appendRef(stringHexToSlice("2_"))); + + return true; +} + +@method_id(234) +fun test_WithMoreTrickyAddresses2() { + run(WithMoreTrickyAddresses2{ + a1: MaybeJust { value: address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e") }, + a2: MaybeJust { value: address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e") }, + a3: null, + a4: 123, + a5: address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e"), + }, stringHexToSlice("c00ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3ec00ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e600000000000000f680194dc6438f99d3d9dbe151944925d90b2492954bf6b9c070fbff2dded5f30547d_")); + run(WithMoreTrickyAddresses2{ + a1: MaybeNothing {}, + a2: MaybeJust { value: null }, + a3: { addr: address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e") }, + a4: null, + a5: address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e").toCell(), + }, stringHexToSlice("4c00ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e6_").appendRef(stringHexToSlice("80194dc6438f99d3d9dbe151944925d90b2492954bf6b9c070fbff2dded5f30547d_"))); + run(WithMoreTrickyAddresses2{ + a1: MaybeNothing {}, + a2: MaybeNothing {}, + a3: null, + a4: address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e"), + a5: address("EQAxL0oF1-zNgimPKthbDnYS4xj94rHtfNRN7_Pd1r2LNNv3").toCell(), + }, stringHexToSlice("1400ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3ec_").appendRef(stringHexToSlice("800625e940bafd99b04531e55b0b61cec25c631fbc563daf9a89bdfe7bbad7b1669_"))); + + return true; +} + +@method_id(235) +fun test_WithAnyAddress() { + run(WithAnyAddress{ + a1: createAddressNone(), + a2: makeExternalAddress(800, 10), + a3: createAddressNone(), + a4: address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e"), + a5: MaybeJust { value: address("EQAxL0oF1-zNgimPKthbDnYS4xj94rHtfNRN7_Pd1r2LNNv3") as any_address }, + a6: createAddressNone().toCell(), + a7: (null as any_address?).toCell(), + }, stringHexToSlice("282b20080194dc6438f99d3d9dbe151944925d90b2492954bf6b9c070fbff2dded5f30547d800625e940bafd99b04531e55b0b61cec25c631fbc563daf9a89bdfe7bbad7b1669").appendRef(stringHexToSlice("2_")).appendRef(stringHexToSlice("4_"))); + run(WithAnyAddress{ + a1: makeExternalAddress(4, 4), + a2: null, + a3: makeExternalAddress(4, 10), + a4: address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e") as any_address, + a5: MaybeJust { value: makeExternalAddress(8, 8) }, + a6: (address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e") as any_address).toCell(), + a7: (createAddressNone() as any_address?).toCell(), + }, stringHexToSlice("408820a01300329b8c871f33a7b3b7c2a328924bb21649252a97ed7380e1f7fe5bbdabe60a8fa82023_").appendRef(stringHexToSlice("80194dc6438f99d3d9dbe151944925d90b2492954bf6b9c070fbff2dded5f30547d_")).appendRef(stringHexToSlice("9_"))); + run(WithAnyAddress{ + a1: address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e") as any_address, + a2: address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e") as any_address, + a3: 123, + a4: createAddressNone(), + a5: MaybeNothing {}, + a6: makeExternalAddress(800, 10).toCell(), + a7: (makeExternalAddress(800, 10) as any_address?).toCell(), + }, stringHexToSlice("80194dc6438f99d3d9dbe151944925d90b2492954bf6b9c070fbff2dded5f30547d80194dc6438f99d3d9dbe151944925d90b2492954bf6b9c070fbff2dded5f30547d000000000000007b8c_").appendRef(stringHexToSlice("415904_")).appendRef(stringHexToSlice("a0ac82_"))); + run(WithAnyAddress{ + a1: createAddressNone(), + a2: createAddressNone(), + a3: createAddressNone(), + a4: createAddressNone(), + a5: MaybeNothing {}, + a6: createAddressNone().toCell(), + a7: null, + }, stringHexToSlice("2084_").appendRef(stringHexToSlice("2_"))); + + return true; +} + fun main() { var t: JustInt32 = { value: 10 }; @@ -824,7 +947,7 @@ fun main() { @testcase | 221 | | -1 @testcase | 222 | | 510 567 9392843922 typeid-13 81923 81923 typeid-14 777 0 -1 (null) (null) 0 100000 100000 typeid-14 @testcase | 223 | | 10 55 (null) -@testcase | 224 | | 60 50000000 5 9 123 -1 +@testcase | 224 | | 60 50000000 5 9 123 0 @testcase | 225 | | 5 40 65535 8 50000000 @testcase | 226 | | 1 16 0 @testcase | 227 | | 16 @@ -833,4 +956,7 @@ fun main() { @testcase | 230 | | 0 -1 -1 3 30 8 1 @testcase | 231 | | 0 -1 0 0 -1 0 @testcase | 232 | | -1 +@testcase | 233 | | -1 +@testcase | 234 | | -1 +@testcase | 235 | | -1 */ diff --git a/tolk-tester/tests/pack-unpack-3.tolk b/tolk-tester/tests/pack-unpack-3.tolk index 71331464f6..c83790f0a7 100644 --- a/tolk-tester/tests/pack-unpack-3.tolk +++ b/tolk-tester/tests/pack-unpack-3.tolk @@ -4,8 +4,8 @@ struct EitherRight { value: T } type Either = EitherLeft | EitherRight; @inline -fun makeExternalAddress(hash: int, len: int): address { - return beginCell().storeUint(0b01, 2).storeUint(len, 9).storeUint(hash, len).endCell().beginParse() as address; +fun makeExternalAddress(hash: int, len: int): any_address { + return beginCell().storeUint(0b01, 2).storeUint(len, 9).storeUint(hash, len).endCell().beginParse() as any_address; } @@ -184,7 +184,7 @@ transferParams1#794 struct(0x794) TransferParams1 { dest_int: address; amount: coins; - dest_ext: address; + dest_ext: any_address; } /* @@ -245,9 +245,9 @@ fun test_MsgCounter1() { @method_id(204) fun test_MsgExternal1() { - run(SayHiAndGoodbye{ dest_addr: null, body: BodyPayload2 { master_id: 10, owner_address: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c") } }, stringHexToSlice("892150000000000000000000000000000000000000000000000000000000000000000002_")); - run(SayHiAndGoodbye{ dest_addr: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), body: BodyPayload1 { should_forward: false, n_times: 85, content: generateSlice_44_with_ref45() } }, stringHexToSlice("89C0000000000000000000000000000000000000000000000000000000000000000002000000550000002C").appendRef(stringHexToSlice("0000002D"))); - run(SayHiAndGoodbye{ dest_addr: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), body: BodyPayload2 { master_id: -5, owner_address: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi") } }, stringHexToSlice("89CFF28E8033DB1467CACEA8B158F0F61E682DE06E8A5947504C904F1F703D2BE4D9E7EE7F9474019ED8A33E5675458AC787B0F3416F037452CA3A82648278FB81E95F26CF4_")); + run(SayHiAndGoodbye{ dest_addr: null, body: BodyPayload2 { master_id: 10, owner_address: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c") } }, stringHexToSlice("8910A8000000000000000000000000000000000000000000000000000000000000000001_")); + run(SayHiAndGoodbye{ dest_addr: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), body: BodyPayload1 { should_forward: false, n_times: 85, content: generateSlice_44_with_ref45() } }, stringHexToSlice("8980000000000000000000000000000000000000000000000000000000000000000004000000AA00000059_").appendRef(stringHexToSlice("0000002D"))); + run(SayHiAndGoodbye{ dest_addr: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), body: BodyPayload2 { master_id: -5, owner_address: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi") } }, stringHexToSlice("899FE51D0067B628CF959D5162B1E1EC3CD05BC0DD14B28EA099209E3EE07A57C9B3CFDCFF28E8033DB1467CACEA8B158F0F61E682DE06E8A5947504C904F1F703D2BE4D9E")); run(SayStoreInChain{ in_masterchain: true, contents: BodyPayload1{ should_forward: true, n_times: 20, content: generateSlice_44_with_ref45() }.toCell() as cell as Cell }, stringHexToSlice("0013C_").appendRef(stringHexToSlice("3000000140000002C").appendRef(stringHexToSlice("0000002D")))); run(SayStoreInChain{ in_masterchain: false, contents: BodyPayload2{ master_id: 37, owner_address: address(("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c")) }.toCell() as cell as Cell }, stringHexToSlice("00134_").appendRef(stringHexToSlice("4960000000000000000000000000000000000000000000000000000000000000000004_"))); diff --git a/tolk-tester/tests/pack-unpack-5.tolk b/tolk-tester/tests/pack-unpack-5.tolk index 05a9ee1e6c..3b37aeec15 100644 --- a/tolk-tester/tests/pack-unpack-5.tolk +++ b/tolk-tester/tests/pack-unpack-5.tolk @@ -136,7 +136,7 @@ fun test7() { } struct(0x10) Inner8_1 { - ea: address; + ea: any_address; } struct(0b1) CellInner8_1 { ref: Cell; @@ -222,6 +222,22 @@ fun test21() { ) } +struct JustAddress { + a: address +} +type Nullable = T | null +type OwnerAddress = address + +@method_id(122) +fun test22() { + return ( + address.estimate(), Nullable
.estimate(), + JustAddress.estimate(), Nullable.estimate(), + OwnerAddress.estimate(), Nullable.estimate(), + any_address.estimate(), Nullable.estimate(), + ) +} + fun main() { __expect_type(int8.estimatePackSize(), "(int, int, int, int)"); } @@ -230,14 +246,15 @@ fun main() { @testcase | 101 | | [ 32 32 0 0 ] [ 64 64 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 0 9999 0 4 ] [ 4 124 0 0 ] [ 6 6 0 0 ] [ 64 64 0 0 ] [ 5 253 0 0 ] @testcase | 102 | | [ 32 32 0 0 ] [ 1 33 0 0 ] [ 1 33 0 0 ] [ 17 33 0 0 ] @testcase | 103 | | [ 67 199 0 0 ] [ 69 599 0 0 ] -@testcase | 104 | | [ 4 534 0 0 ] [ 2 536 0 0 ] [ 6 1070 0 0 ] +@testcase | 104 | | [ 534 534 0 0 ] [ 4 534 0 0 ] [ 538 1068 0 0 ] @testcase | 105 | | [ 2 2 2 4 ] [ 5 9999 0 9 ] [ 4 152 1 4 ] @testcase | 106 | | [ 64 64 0 0 ] [ 5 37 1 1 ] [ 5 65 0 1 ] @testcase | 107 | | [ 16 16 0 0 ] [ 16 16 0 0 ] -@testcase | 108 | | [ 10 275 0 0 ] [ 34 158 0 0 ] [ 47 1143 0 1 ] +@testcase | 108 | | [ 10 531 0 0 ] [ 34 158 0 0 ] [ 47 1911 0 1 ] @testcase | 109 | | [ 34 130 0 0 ] [ 33 159 0 0 ] [ 4 8 0 0 ] [ 1 7 0 0 ] @testcase | 110 | | [ 32 9999 0 4 ] [ 33 9999 0 8 ] @testcase | 111 | | [ 1023 1023 0 1 ] [ 0 0 2 2 ] @testcase | 120 | | 16 4128 1 1 @testcase | 121 | | [ 2 2 0 0 ] [ 9 9 0 0 ] [ 6 6 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 123 123 0 0 ] [ 123 123 0 0 ] +@testcase | 122 | | [ 267 267 0 0 ] [ 2 267 0 0 ] [ 267 267 0 0 ] [ 1 268 0 0 ] [ 267 267 0 0 ] [ 2 267 0 0 ] [ 2 523 0 0 ] [ 1 524 0 0 ] */ diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index b753f5838e..fb6b8a0a17 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -4,9 +4,8 @@ const cc2 = address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); fun verifyAddr(addr: address, workchain: int, number: int) { assert (addr == addr) throw 111; assert (!(addr != addr)) throw 111; - assert (addr.isInternal(), 111); - assert (!addr.isExternal(), 111); - assert (!addr.isNone(), 111); + assert (addr == (addr as any_address), 111); + assert ((addr as any_address).isInternal(), 111); assert (addr.bitsEqual(addr), 111); var (wc, h) = addr.getWorkchainAndHash(); diff --git a/tolk-tester/tests/send-msg-1.tolk b/tolk-tester/tests/send-msg-1.tolk index 6f7646ce0a..e84e9498ee 100644 --- a/tolk-tester/tests/send-msg-1.tolk +++ b/tolk-tester/tests/send-msg-1.tolk @@ -650,7 +650,7 @@ fun test22(op: uint32, queryId: uint64) { } struct Body23Unlimited { - any1: (address, builder), + any1: (address?, builder), any2: RemainingBitsAndRefs; } @@ -669,7 +669,7 @@ fun test23_manual(state32: uint32, body: Body23Unlimited) { .storeUint(0b10001100, 1 + 4 + 4 + 64 + 32 + (1 + 1 + 5 + 1)) .storeRef(init.code) .storeRef(init.data) - .storeAddress(body.any1.0) + .storeAddressOpt(body.any1.0) .storeBuilder(body.any1.1) .storeSlice(body.any2) .endCell(); @@ -682,7 +682,7 @@ fun test23(state32: uint32) { code: beginCell().endCell(), }; var bd: Body23Unlimited = { - any1: (createAddressNone(), beginCell().storeRef(init.code)), + any1: (null, beginCell().storeRef(init.code)), any2: beginCell().storeUint(80, 80).endCell().beginParse(), }; diff --git a/tolk-tester/tests/send-msg-3.tolk b/tolk-tester/tests/send-msg-3.tolk index 8db47589d1..c2965e5007 100644 --- a/tolk-tester/tests/send-msg-3.tolk +++ b/tolk-tester/tests/send-msg-3.tolk @@ -67,27 +67,29 @@ fun test2(bd: Body750Bits) { } struct Body3 { - bodyAddress: address; + bodyAddress: any_address; amount: coins; someRef: cell; } @method_id(103) fun test3(eventId: int) { - val bodyAddress = address("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"); + val bodyAddress = address("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf") as any_address; val someRef = MyBody { queryId: 800 }.toCell(); var b = createExternalLogMessage({ - body: Body3 { bodyAddress, someRef, amount: ton("0.05") }, + body: UnsafeBodyNoRef { + forceInline: Body3 { bodyAddress, someRef, amount: ton("0.05") } + }, dest: { topic: eventId }, }); return b.hash(); } @noinline -fun test4_manual(dest: address, body: cell) { +fun test4_manual(dest: any_address, body: cell) { return beginCell() .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () - .storeAddress(dest) // dest (manually created external address) + .storeAddressAny(dest) // dest (manually created external address) .storeUint(1, 64 + 32 + 2) // created_lt, created_at, init:Maybe (0), body:Either right (ref) .storeRef(body) .endCell(); @@ -97,7 +99,7 @@ fun test4_manual(dest: address, body: cell) { fun test4() { val body: cell = Body3 { bodyAddress: createAddressNone(), amount: 0, someRef: createEmptyCell() }.toCell(); val dest = beginCell().storeUint(0b01, 2).storeUint(12, 9).storeUint(88, 12).endCell() - .beginParse() as address; + .beginParse() as any_address; var b = createExternalLogMessage({ body, dest, @@ -117,7 +119,7 @@ fun test5(topic: int) { @method_id(106) fun test6() { val dest = beginCell().storeUint(0b01, 2).storeUint(12, 9).storeUint(88, 12).endCell() - .beginParse() as address; + .beginParse() as any_address; var b = createExternalLogMessage({ dest, }); diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index bfc0d14d62..85df50cbf1 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1376,7 +1376,7 @@ void define_builtins() { TypePtr Bool = TypeDataBool::create(); TypePtr Slice = TypeDataSlice::create(); TypePtr Builder = TypeDataBuilder::create(); - TypePtr Address = TypeDataAddress::create(); + TypePtr Address = TypeDataAddress::internal(); TypePtr Tuple = TypeDataTuple::create(); TypePtr Never = TypeDataNever::create(); @@ -1554,7 +1554,7 @@ void define_builtins() { define_builtin_func("stringHexToSlice", {TypeDataUnknown::create()}, TypeDataSlice::create(), nullptr, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); - define_builtin_func("address", {TypeDataUnknown::create()}, TypeDataAddress::create(), nullptr, + define_builtin_func("address", {TypeDataUnknown::create()}, TypeDataAddress::internal(), nullptr, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); diff --git a/tolk/maps-kv-api.cpp b/tolk/maps-kv-api.cpp index ce8ed12716..47ffb435c8 100644 --- a/tolk/maps-kv-api.cpp +++ b/tolk/maps-kv-api.cpp @@ -85,8 +85,8 @@ static int is_TKey_TVM_uint(TypePtr TKey) { // map should use DICT* instructions // note that map is forbidden, since a raw slice doesn't define binary width static int is_TKey_TVM_slice(TypePtr TKey) { - if (TKey == TypeDataAddress::create()) { - return 3 + 8 + 256; // "internal address" assumed + if (const TypeDataAddress* t_address = TKey->try_as()) { + return t_address->is_internal() ? 3 + 8 + 256 : 0; } if (const TypeDataBitsN* t_bitsN = TKey->try_as()) { return t_bitsN->is_bits ? t_bitsN->n_width : t_bitsN->n_width * 8; diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index 4208df7b29..077b809b5f 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -90,15 +90,15 @@ class PackUnpackAvailabilityChecker { if (any_type == TypeDataCell::create()) { return {}; } - if (any_type == TypeDataAddress::create()) { - return {}; - } if (any_type == TypeDataNever::create()) { return {}; } if (any_type->try_as()) { return {}; } + if (any_type->try_as()) { + return {}; + } if (const auto* t_struct = any_type->try_as()) { StructPtr struct_ref = t_struct->struct_ref; diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index 3a8dc338ff..6a99f6d150 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -123,11 +123,16 @@ void PackContext::storeMaybeRef(var_idx_t ir_idx) const { code.emplace_back(origin, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeMaybeRef")); } -void PackContext::storeAddress(var_idx_t ir_idx) const { +void PackContext::storeAddressInt(var_idx_t ir_idx) const { std::vector args = { ir_builder0, ir_idx }; code.emplace_back(origin, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeAddress")); } +void PackContext::storeAddressAny(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(origin, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeAddressAny")); +} + void PackContext::storeBuilder(var_idx_t ir_idx) const { std::vector args = { ir_builder0, ir_idx }; code.emplace_back(origin, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeBuilder")); @@ -464,10 +469,10 @@ struct S_Coins final : ISerializer { } }; -struct S_Address final : ISerializer { +struct S_AddressInt final : ISerializer { void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { tolk_assert(rvect.size() == 1); - ctx->storeAddress(rvect[0]); + ctx->storeAddressInt(rvect[0]); } std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { @@ -478,17 +483,60 @@ struct S_Address final : ISerializer { } void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { - // we can't do just - // ctx->skipBits(2 + 1 + 8 + 256); - // because it may be addr_none or addr_extern; there is no "skip address" in TVM, so just load it + // not just skip 267 bits: do real unpack, so that an address is validated to be internal (not none, etc.) + unpack(ctx, code, origin); + } + + PackSize estimate(const EstimateContext* ctx) override { + PackSize size = PackSize(3 + 8 + 256); // '10' + 0 (anycast disabled) + workchain + hash + size.skipping_is_dangerous = true; // can't just "skip bits267" when unused in lazy + return size; + } +}; + +struct S_AddressIntOrNull final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { + // `address?`, when null, is stored as '00' (addr_none), so `address?` is not TL/B (Maybe MsgAddressInt) + tolk_assert(rvect.size() == 1); + std::vector args = { ctx->ir_builder0, rvect[0] }; + code.emplace_back(origin, Op::_Call, ctx->ir_builder, std::move(args), lookup_function("builder.storeAddressOpt")); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { + FunctionPtr f_loadAddressOpt = lookup_function("slice.loadAddressOpt"); + std::vector ir_address_orN = code.create_tmp_var(TypeDataSlice::create(), origin, "(loaded-addr)"); + code.emplace_back(origin, Op::_Call, std::vector{ctx->ir_slice0, ir_address_orN[0]}, ctx->ir_slice, f_loadAddressOpt); + return ir_address_orN; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { unpack(ctx, code, origin); } PackSize estimate(const EstimateContext* ctx) override { - // we can't do just - // return PackSize(2 + 1 + 8 + 256); - // because it may be addr_none or addr_extern; but since addr_extern is very-very uncommon, don't consider it - return PackSize(2, 2 + 1 + 8 + 256); + return PackSize(2, 3 + 8 + 256); + } +}; + +struct S_AddressAny final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeAddressAny(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { + FunctionPtr f_loadAddressAny = lookup_function("slice.loadAddressAny"); + std::vector ir_address_any = code.create_tmp_var(TypeDataSlice::create(), origin, "(loaded-addr)"); + code.emplace_back(origin, Op::_Call, std::vector{ctx->ir_slice0, ir_address_any[0]}, ctx->ir_slice, f_loadAddressAny); + return ir_address_any; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { + unpack(ctx, code, origin); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(2, 2 + 9 + 512); // an extern address could be really large } }; @@ -1319,9 +1367,6 @@ static std::unique_ptr get_serializer_for_type(TypePtr any_type) { if (any_type == TypeDataCell::create() || is_type_cellT(any_type)) { return std::make_unique(); } - if (any_type == TypeDataAddress::create()) { - return std::make_unique(); - } if (any_type == TypeDataBuilder::create()) { return std::make_unique(); } @@ -1338,6 +1383,12 @@ static std::unique_ptr get_serializer_for_type(TypePtr any_type) { if (any_type->try_as()) { return std::make_unique(); } + if (const auto* t_address = any_type->try_as()) { + if (t_address->is_internal()) { + return std::make_unique(); + } + return std::make_unique(); + } if (const auto* t_struct = any_type->try_as()) { return std::make_unique(t_struct->struct_ref); } @@ -1352,6 +1403,9 @@ static std::unique_ptr get_serializer_for_type(TypePtr any_type) { if (or_null == TypeDataCell::create() || is_type_cellT(or_null)) { return std::make_unique(); } + if (or_null->try_as() && or_null->try_as()->is_internal()) { + return std::make_unique(); // `address?` is stored as '00' (none) for null + } return std::make_unique(t_union); } diff --git a/tolk/pack-unpack-serializers.h b/tolk/pack-unpack-serializers.h index 7642447c7e..c2f211059e 100644 --- a/tolk/pack-unpack-serializers.h +++ b/tolk/pack-unpack-serializers.h @@ -28,6 +28,7 @@ struct PackSize { int max_bits; int min_refs; int max_refs; + bool skipping_is_dangerous = false; bool is_unpredictable_infinity() const { return max_bits >= 9999; @@ -39,8 +40,8 @@ struct PackSize { PackSize(int min_bits, int max_bits) : min_bits(min_bits), max_bits(max_bits), min_refs(0), max_refs() { } - PackSize(int min_bits, int max_bits, int min_refs, int max_refs) - : min_bits(min_bits), max_bits(max_bits), min_refs(min_refs), max_refs(max_refs) { + PackSize(int min_bits, int max_bits, int min_refs, int max_refs, bool skipping_is_dangerous = false) + : min_bits(min_bits), max_bits(max_bits), min_refs(min_refs), max_refs(max_refs), skipping_is_dangerous(skipping_is_dangerous) { } static PackSize unpredictable_infinity() { @@ -77,7 +78,8 @@ class PackContext { void storeCoins(var_idx_t ir_idx) const; void storeRef(var_idx_t ir_idx) const; void storeMaybeRef(var_idx_t ir_idx) const; - void storeAddress(var_idx_t ir_idx) const; + void storeAddressInt(var_idx_t ir_idx) const; + void storeAddressAny(var_idx_t ir_idx) const; void storeBuilder(var_idx_t ir_idx) const; void storeSlice(var_idx_t ir_idx) const; void storeOpcode(PackOpcode opcode) const; @@ -152,10 +154,10 @@ class EstimateContext { PrefixEstimateMode get_prefix_mode() const { return prefix_mode; } static PackSize minmax(PackSize a, PackSize b) { - return PackSize(std::min(a.min_bits, b.min_bits), std::max(a.max_bits, b.max_bits), std::min(a.min_refs, b.min_refs), std::max(a.max_refs, b.max_refs)); + return PackSize(std::min(a.min_bits, b.min_bits), std::max(a.max_bits, b.max_bits), std::min(a.min_refs, b.min_refs), std::max(a.max_refs, b.max_refs), a.skipping_is_dangerous || b.skipping_is_dangerous); } static PackSize sum(PackSize a, PackSize b) { - return PackSize(a.min_bits + b.min_bits, std::min(9999, a.max_bits + b.max_bits), a.min_refs + b.min_refs, a.max_refs + b.max_refs); + return PackSize(a.min_bits + b.min_bits, std::min(9999, a.max_bits + b.max_bits), a.min_refs + b.min_refs, a.max_refs + b.max_refs, a.skipping_is_dangerous || b.skipping_is_dangerous); } PackSize estimate_any(TypePtr any_type, PrefixEstimateMode prefix_mode = PrefixEstimateMode::IncludePrefixOfStruct) const; diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 4e5efd03b3..29a0b99bd2 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1009,13 +1009,18 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as() && target_type->try_as()) { + return rvect; + } + // pass `bits267` to `address` - if (target_type == TypeDataAddress::create() && original_type->try_as()) { + if (target_type->try_as() && original_type->try_as()) { return rvect; } // pass `address` to `bits267` - if (original_type == TypeDataAddress::create() && target_type->try_as()) { + if (original_type->try_as() && target_type->try_as()) { return rvect; } @@ -1081,13 +1086,13 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as()) { return rvect; } // pass address to slice // same, no changes in rvect - if (original_type == TypeDataAddress::create() && target_type == TypeDataSlice::create()) { + if (original_type->try_as() && target_type == TypeDataSlice::create()) { return rvect; } @@ -1278,7 +1283,7 @@ static std::vector process_binary_operator(V v, return transition_to_target_type(std::move(rvect), code, target_type, v); } if (t == tok_eq || t == tok_neq) { - if (v->get_lhs()->inferred_type->unwrap_alias() == TypeDataAddress::create() && v->get_rhs()->inferred_type->unwrap_alias() == TypeDataAddress::create()) { + if (v->get_lhs()->inferred_type->unwrap_alias()->try_as() && v->get_rhs()->inferred_type->unwrap_alias()->try_as()) { FunctionPtr f_sliceEq = lookup_function("slice.bitsEqual"); std::vector ir_lhs_slice = pre_compile_expr(v->get_lhs(), code); std::vector ir_rhs_slice = pre_compile_expr(v->get_rhs(), code); diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 081e333a4b..33ba7db686 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -44,12 +44,17 @@ static Error err_type_mismatch(const char* text_tpl, TypePtr src, TypePtr dst) { message.replace(message.find("{dst}"), 5, "`" + dst->as_human_readable() + "`"); if (src->can_be_casted_with_as_operator(dst)) { bool suggest_as = !dst->try_as() && !dst->try_as(); - if ((src == TypeDataAddress::create() || src == TypeDataSlice::create()) && (dst == TypeDataAddress::create() || dst == TypeDataSlice::create())) { + if ((src == TypeDataSlice::create() || dst == TypeDataSlice::create()) && (src->try_as() || dst->try_as())) { message += "\n""hint: unlike FunC, Tolk has a special type `address` (which is slice at the TVM level);"; message += "\n"" most likely, you just need `address` everywhere"; - message += "\n""hint: alternatively, use `as` operator for unsafe casting: ` as " + dst->as_human_readable() + "`"; + message += "\n""hint: alternatively, use `as` operator for UNSAFE casting: ` as " + dst->as_human_readable() + "`"; + } else if (src == TypeDataAddress::any() && dst == TypeDataAddress::internal()) { + message += "\n""hint: use `any_addr.castToInternal()` to check and get `address`"; + message += "\n""hint: alternatively, use `as` operator for UNSAFE casting: ` as " + dst->as_human_readable() + "`"; + } else if (src == TypeDataAddress::internal() && dst == TypeDataAddress::any()) { + message += "\n""hint: use `as` operator for conversion: ` as any_address`"; } else if (suggest_as) { - message += "\n""hint: use `as` operator for unsafe casting: ` as " + dst->as_human_readable() + "`"; + message += "\n""hint: use `as` operator for UNSAFE casting: ` as " + dst->as_human_readable() + "`"; } if (src == TypeDataBool::create() && dst == TypeDataInt::create()) { message += "\n""caution! in TVM, bool TRUE is -1, not 1"; @@ -71,7 +76,10 @@ static Error err_cannot_apply_operator(std::string_view operator_name, AnyExprV // make an error on `int + cell` / `slice & int` static Error err_cannot_apply_operator(std::string_view operator_name, AnyExprV lhs, AnyExprV rhs) { - return err("can not apply operator `{}` to `{}` and `{}`", operator_name, lhs->inferred_type, rhs->inferred_type); + const TypeDataUnion* lhs_nullable = lhs->inferred_type->unwrap_alias()->try_as(); + const TypeDataUnion* rhs_nullable = rhs->inferred_type->unwrap_alias()->try_as(); + std::string hint = lhs_nullable || rhs_nullable ? "\n""hint: check on `null` first, or use unsafe operator `!`" : ""; + return err("can not apply operator `{}` to `{}` and `{}`{}", operator_name, lhs->inferred_type, rhs->inferred_type, hint); } GNU_ATTRIBUTE_NOINLINE @@ -178,7 +186,7 @@ static bool check_eq_neq_operator(TypePtr lhs_type, TypePtr rhs_type, bool& not_ if (expect_boolean(lhs_type) && expect_boolean(rhs_type)) { return true; } - if (lhs_type->equal_to(TypeDataAddress::create()) && rhs_type->equal_to(TypeDataAddress::create())) { + if (lhs_type->unwrap_alias()->try_as() && rhs_type->unwrap_alias()->try_as()) { not_integer_comparison = true; // `address` can be compared with ==, but it's handled specially return true; } diff --git a/tolk/pipe-lazy-load-insertions.cpp b/tolk/pipe-lazy-load-insertions.cpp index 3ed7a10bdf..e1db8ddbd2 100644 --- a/tolk/pipe-lazy-load-insertions.cpp +++ b/tolk/pipe-lazy-load-insertions.cpp @@ -435,7 +435,7 @@ struct ExprUsagesWhileCollecting { if (!future_fields.empty() && future_fields.back().action == LazyStructLoadInfo::SkipField) { if (const TypeDataBitsN* last_bitsN = future_fields.back().field_type->try_as()) { PackSize cur_size = estimate_serialization_size(field_type); - if (cur_size.min_bits == cur_size.max_bits && cur_size.max_refs == 0) { + if (cur_size.min_bits == cur_size.max_bits && cur_size.max_refs == 0 && !cur_size.skipping_is_dangerous) { TypePtr total_bitsN = TypeDataBitsN::create(last_bitsN->n_width + cur_size.max_bits, true); future_fields.back().field_type = total_bitsN; continue; @@ -446,7 +446,7 @@ struct ExprUsagesWhileCollecting { // generate "skip 8 bits" instead of "skip int8" (it's more effective, and it can be merged with next) TypePtr skip_type = field_type; PackSize skip_size = estimate_serialization_size(field_type); - if (skip_size.min_bits == skip_size.max_bits && skip_size.max_refs == 0) { + if (skip_size.min_bits == skip_size.max_bits && skip_size.max_refs == 0 && !skip_size.skipping_is_dangerous) { skip_type = TypeDataBitsN::create(skip_size.max_bits, true); } future_fields.emplace_back(LazyStructLoadInfo::SkipField, "(gap)", skip_type); diff --git a/tolk/pipe-optimize-boolean-expr.cpp b/tolk/pipe-optimize-boolean-expr.cpp index e25c3201c9..62538527b4 100644 --- a/tolk/pipe-optimize-boolean-expr.cpp +++ b/tolk/pipe-optimize-boolean-expr.cpp @@ -146,7 +146,7 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { } // `if (addr1 != addr2)` -> ifnot(addr1 == addr2) if (auto v_cond_neq = v->get_cond()->try_as()) { - if (v_cond_neq->tok == tok_neq && v_cond_neq->get_lhs()->inferred_type->unwrap_alias() == TypeDataAddress::create() && v_cond_neq->get_rhs()->inferred_type->unwrap_alias() == TypeDataAddress::create()) { + if (v_cond_neq->tok == tok_neq && v_cond_neq->get_lhs()->inferred_type->unwrap_alias()->try_as() && v_cond_neq->get_rhs()->inferred_type->unwrap_alias()->try_as()) { auto v_cond_eq = createV(v_cond_neq->range, v_cond_neq->operator_range, "==", tok_eq, v_cond_neq->get_lhs(), v_cond_neq->get_rhs()); v_cond_eq->mutate()->assign_inferred_type(v_cond_neq->inferred_type); v_cond_eq->mutate()->assign_rvalue_true(); diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 24cdbdb4e9..1a675ca655 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -108,7 +108,7 @@ static TypePtr try_parse_predefined_type(std::string_view str) { break; case 7: if (str == "builder") return TypeDataBuilder::create(); - if (str == "address") return TypeDataAddress::create(); + if (str == "address") return TypeDataAddress::internal(); break; case 8: if (str == "varint16") return TypeDataIntN::create(16, false, true); @@ -118,6 +118,9 @@ static TypePtr try_parse_predefined_type(std::string_view str) { if (str == "varuint16") return TypeDataIntN::create(16, true, true); if (str == "varuint32") return TypeDataIntN::create(32, true, true); break; + case 11: + if (str == "any_address") return TypeDataAddress::any(); + break; case 12: if (str == "continuation") return TypeDataContinuation::create(); break; diff --git a/tolk/send-message-api.cpp b/tolk/send-message-api.cpp index 92c8ba404b..effaf1e532 100644 --- a/tolk/send-message-api.cpp +++ b/tolk/send-message-api.cpp @@ -119,7 +119,7 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co // struct AutoDeployAddress { workchain: int8; stateInit: ContractState | cell; toShard: AddressShardingOptions?; } // struct ContractState { code: cell; data: cell; } // struct AddressShardingOptions { fixedPrefixLength: uint5; closeTo: address; } - std::vector ir_dest_is_address = pre_compile_is_type(code, t_dest, TypeDataAddress::create(), ir_dest, origin, "(is-address)"); + std::vector ir_dest_is_address = pre_compile_is_type(code, t_dest, TypeDataAddress::internal(), ir_dest, origin, "(is-address)"); std::vector ir_dest_is_AutoDeploy = pre_compile_is_type(code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), ir_dest, origin, "(is-auto)"); std::vector ir_dest_is_builder = pre_compile_is_type(code, t_dest, TypeDataBuilder::create(), ir_dest, origin, "(is-builder)"); std::vector ir_dest_AutoDeployAddress = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), origin); @@ -186,8 +186,8 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co { // input is `dest: someAddress` code.push_set_cur(if_address.block0); - std::vector ir_dest_address = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataAddress::create(), origin); - ctx.storeAddress(ir_dest_address[0]); + std::vector ir_dest_address = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataAddress::internal(), origin); + ctx.storeAddressInt(ir_dest_address[0]); code.close_pop_cur(origin); } { @@ -434,7 +434,7 @@ std::vector generate_createExternalLogMessage(FunctionPtr called_f, C // field `dest` is `dest: address | builder | ExtOutLogBucket`; // struct ExtOutLogBucket { topic: uint248 | bits248; } - std::vector ir_dest_is_address = pre_compile_is_type(code, t_dest, TypeDataAddress::create(), ir_dest, origin, "(is-address)"); + std::vector ir_dest_is_address = pre_compile_is_type(code, t_dest, TypeDataAddress::any(), ir_dest, origin, "(is-address)"); std::vector ir_dest_is_builder = pre_compile_is_type(code, t_dest, TypeDataBuilder::create(), ir_dest, origin, "(is-builder)"); std::vector ir_dest_ExtOutLogBucket = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataStruct::create(s_ExtOutLogBucket), origin); // dest.topic (it's the only field in a struct) @@ -482,8 +482,8 @@ std::vector generate_createExternalLogMessage(FunctionPtr called_f, C { // input is `dest: someAddress` code.push_set_cur(if_address.block0); - std::vector ir_dest_address = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataAddress::create(), origin); - ctx.storeAddress(ir_dest_address[0]); + std::vector ir_dest_address = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataAddress::any(), origin); + ctx.storeAddressAny(ir_dest_address[0]); code.close_pop_cur(origin); } { diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index d25d3ee66c..63930228e2 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -84,7 +84,8 @@ TypePtr TypeDataSlice::singleton; TypePtr TypeDataBuilder::singleton; TypePtr TypeDataTuple::singleton; TypePtr TypeDataContinuation::singleton; -TypePtr TypeDataAddress::singleton; +TypePtr TypeDataAddress::singleton_internal; +TypePtr TypeDataAddress::singleton_any; TypePtr TypeDataNullLiteral::singleton; TypePtr TypeDataCoins::singleton; TypePtr TypeDataUnknown::singleton; @@ -99,7 +100,8 @@ void type_system_init() { TypeDataBuilder::singleton = new TypeDataBuilder; TypeDataTuple::singleton = new TypeDataTuple; TypeDataContinuation::singleton = new TypeDataContinuation; - TypeDataAddress::singleton = new TypeDataAddress; + TypeDataAddress::singleton_internal = new TypeDataAddress(0); + TypeDataAddress::singleton_any = new TypeDataAddress(1); TypeDataNullLiteral::singleton = new TypeDataNullLiteral; TypeDataCoins::singleton = new TypeDataCoins; TypeDataUnknown::singleton = new TypeDataUnknown; @@ -321,6 +323,13 @@ int TypeDataAlias::get_type_id() const { return underlying_type->get_type_id(); } +int TypeDataAddress::get_type_id() const { + if (is_internal()) { + return type_id_address_int; + } + return type_id_address_any; +} + int TypeDataFunCallable::get_type_id() const { return TypeIdCalculation::assign_type_id(this); } @@ -354,12 +363,12 @@ int TypeDataBrackets::get_type_id() const { int TypeDataIntN::get_type_id() const { switch (n_bits) { - case 8: return 42 + is_unsigned; // for common intN, use predefined small numbers - case 16: return 44 + is_unsigned; - case 32: return 46 + is_unsigned; - case 64: return 48 + is_unsigned; - case 128: return 50 + is_unsigned; - case 256: return 52 + is_unsigned; + case 8: return type_id_int8 + is_unsigned; // for common intN, use predefined small numbers + case 16: return type_id_int16 + is_unsigned; + case 32: return type_id_int32 + is_unsigned; + case 64: return type_id_int64 + is_unsigned; + case 128: return type_id_int128 + is_unsigned; + case 256: return type_id_int256 + is_unsigned; default: return TypeIdCalculation::assign_type_id(this); } } @@ -393,6 +402,13 @@ std::string TypeDataAlias::as_human_readable() const { return alias_ref->name; } +std::string TypeDataAddress::as_human_readable() const { + if (is_internal()) { + return "address"; + } + return "any_address"; +} + std::string TypeDataFunCallable::as_human_readable() const { std::string result = "("; for (TypePtr param : params_types) { @@ -652,8 +668,9 @@ bool TypeDataContinuation::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataAddress::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == singleton) { - return true; + if (const TypeDataAddress* rhs_address = rhs->try_as()) { + // note that not `address` to `any_address` also requires manual `as` + return kind == rhs_address->kind; } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { return can_rhs_be_assigned(rhs_alias->underlying_type); @@ -906,7 +923,7 @@ bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to->try_as()) { // `slice` to `bytes32` / `slice` to `bits8` return true; } - if (cast_to == TypeDataAddress::create()) { + if (cast_to->try_as()) { // `slice` to `address` return true; } if (const TypeDataUnion* to_union = cast_to->try_as()) { @@ -952,13 +969,16 @@ bool TypeDataAddress::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to == TypeDataSlice::create() || cast_to->try_as()) { return true; } + if (cast_to->try_as()) { + return true; // `any_address` as `address` and any other casts are ok + } if (const TypeDataUnion* to_union = cast_to->try_as()) { return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == singleton; + return false; } bool TypeDataNullLiteral::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1092,13 +1112,16 @@ bool TypeDataBitsN::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to->try_as()) { // `bytes256` as `bytes512`, `bits1` as `bytes8` return true; } + if (cast_to->try_as()) { // `bytes267` as `address` + return true; + } if (const TypeDataUnion* to_union = cast_to->try_as()) { // `bytes8` as `slice?` return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == TypeDataSlice::create() || cast_to == TypeDataAddress::create(); + return cast_to == TypeDataSlice::create(); } bool TypeDataCoins::can_be_casted_with_as_operator(TypePtr cast_to) const { diff --git a/tolk/type-system.h b/tolk/type-system.h index be043a643e..12a2848010 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -45,6 +45,27 @@ class TypeData { friend class CalcChildrenFlags; protected: + enum builtin_type_id { + type_id_int = 1, + type_id_bool = 2, + type_id_cell = 3, + type_id_slice = 4, + type_id_builder = 5, + type_id_tuple = 6, + type_id_continuation = 7, + type_id_address_int = 8, + type_id_address_any = 9, + type_id_void = 10, + type_id_coins = 17, + type_id_never = 19, + type_id_int8 = 42, + type_id_int16 = 44, + type_id_int32 = 46, + type_id_int64 = 48, + type_id_int128 = 50, + type_id_int256 = 52, + }; + enum flag_mask { flag_contains_unknown_inside = 1 << 1, flag_contains_genericT_inside = 1 << 2, @@ -141,7 +162,7 @@ class TypeDataInt final : public TypeData { public: static TypePtr create() { return singleton; } - int get_type_id() const override { return 1; } + int get_type_id() const override { return type_id_int; } std::string as_human_readable() const override { return "int"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -160,7 +181,7 @@ class TypeDataBool final : public TypeData { public: static TypePtr create() { return singleton; } - int get_type_id() const override { return 2; } + int get_type_id() const override { return type_id_bool; } std::string as_human_readable() const override { return "bool"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -178,7 +199,7 @@ class TypeDataCell final : public TypeData { public: static TypePtr create() { return singleton; } - int get_type_id() const override { return 3; } + int get_type_id() const override { return type_id_cell; } std::string as_human_readable() const override { return "cell"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -196,7 +217,7 @@ class TypeDataSlice final : public TypeData { public: static TypePtr create() { return singleton; } - int get_type_id() const override { return 4; } + int get_type_id() const override { return type_id_slice; } std::string as_human_readable() const override { return "slice"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -214,7 +235,7 @@ class TypeDataBuilder final : public TypeData { public: static TypePtr create() { return singleton; } - int get_type_id() const override { return 5; } + int get_type_id() const override { return type_id_builder; } std::string as_human_readable() const override { return "builder"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -234,7 +255,7 @@ class TypeDataTuple final : public TypeData { public: static TypePtr create() { return singleton; } - int get_type_id() const override { return 6; } + int get_type_id() const override { return type_id_tuple; } std::string as_human_readable() const override { return "tuple"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -253,27 +274,38 @@ class TypeDataContinuation final : public TypeData { public: static TypePtr create() { return singleton; } - int get_type_id() const override { return 7; } + int get_type_id() const override { return type_id_continuation; } std::string as_human_readable() const override { return "continuation"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; /* - * `address` is TypeDataAddress — TVM slice under the hood, but since it's a very common use case, - * it's extracted as a separate type (not as a struct with slice field, but just a dedicated type). + * `address` and `any_address` is TypeDataAddress — TVM slice under the hood. + * `address` means "internal" (267 bits: 100 + workchain + hash). When deserialized, automatically checked by TVM. + * `any_address` means internal/external/none. + * `address?` (nullable) is serialized NOT as Maybe, BUT as internal/none (267 or 2 bits). */ class TypeDataAddress final : public TypeData { - TypeDataAddress() : TypeData(0) {} + explicit TypeDataAddress(int kind) + : TypeData(0) + , kind(kind) {} - static TypePtr singleton; + static TypePtr singleton_internal; // address + static TypePtr singleton_any; // any_address friend void type_system_init(); + int kind; + public: - static TypePtr create() { return singleton; } + static TypePtr internal() { return singleton_internal; } + static TypePtr any() { return singleton_any; } + + bool is_internal() const { return kind == 0; } + bool is_any() const { return kind == 1; } - int get_type_id() const override { return 8; } - std::string as_human_readable() const override { return "address"; } + int get_type_id() const override; + std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; @@ -521,7 +553,7 @@ class TypeDataCoins final : public TypeData { public: static TypePtr create() { return singleton; } - int get_type_id() const override { return 17; } + int get_type_id() const override { return type_id_coins; } std::string as_human_readable() const override { return "coins"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -669,7 +701,7 @@ class TypeDataNever final : public TypeData { static TypePtr create() { return singleton; } int get_width_on_stack() const override; - int get_type_id() const override { return 19; } + int get_type_id() const override { return type_id_never; } std::string as_human_readable() const override { return "never"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -691,7 +723,7 @@ class TypeDataVoid final : public TypeData { static TypePtr create() { return singleton; } int get_width_on_stack() const override; - int get_type_id() const override { return 10; } + int get_type_id() const override { return type_id_void; } std::string as_human_readable() const override { return "void"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; From 1e86f393f1bbc3d2c76446d49a7420ddc9529705 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Tue, 28 Oct 2025 18:48:00 +0300 Subject: [PATCH 453/524] [Tolk] Mini borrow checker to prevent UB at concurrent mutations Prevent code like this: > f(mutate x, mutate x) > items.add(items.remove(), true) > x += ([x] = [0]).0 This borrow checker supports not only variables, but also fields of structs/tensors. It also analyzes cross-mutations (e.g., `point.x` is assigned while `point` is mutated). --- .../tests/allow-post-modification.tolk | 6 +- .../tests/invalid-semantics/err-4311.tolk | 19 ++ .../tests/invalid-semantics/err-4312.tolk | 31 +++ .../tests/invalid-semantics/err-4313.tolk | 16 ++ .../tests/invalid-semantics/err-4314.tolk | 37 ++++ .../tests/invalid-semantics/err-4315.tolk | 36 +++ .../tests/invalid-semantics/err-4316.tolk | 34 +++ .../tests/invalid-semantics/err-4370.tolk | 21 ++ .../tests/invalid-semantics/err-4371.tolk | 21 ++ .../tests/invalid-semantics/err-4372.tolk | 21 ++ tolk-tester/tests/mutate-methods.tolk | 51 +++++ tolk/CMakeLists.txt | 1 + tolk/pipe-mini-borrow-checker.cpp | 206 ++++++++++++++++++ tolk/pipeline.h | 1 + tolk/smart-casts-cfg.cpp | 11 + tolk/smart-casts-cfg.h | 1 + tolk/tolk.cpp | 1 + 17 files changed, 511 insertions(+), 3 deletions(-) create mode 100644 tolk-tester/tests/invalid-semantics/err-4311.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4312.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4313.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4314.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4315.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4316.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4370.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4371.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4372.tolk create mode 100644 tolk/pipe-mini-borrow-checker.cpp diff --git a/tolk-tester/tests/allow-post-modification.tolk b/tolk-tester/tests/allow-post-modification.tolk index f825d12f81..3e737cfb76 100644 --- a/tolk-tester/tests/allow-post-modification.tolk +++ b/tolk-tester/tests/allow-post-modification.tolk @@ -97,13 +97,13 @@ fun test_if_else(x: int): (int, int, int, int, int) { @method_id(21) fun test_assign_with_inner(x: int) { - var result = (x, x += 10, [x, x += 20, eq(x -= 50), x], eq2((x, x *= eq(x /= 2)))); + var result = (x, x += 10, [x, x += 20, eq(x -= 50), x], eq2((x, x *= eq(x! /= 2)))); return result; } @method_id(22) fun test_assign_with_mutate(x: int) { - var (result, _) = ((x, mul2(mutate x, x += 5), x.`~inc`(mul2(mutate x, x)), x), 0); + var (result, _) = ((x, mul2(mutate x, (x as int) += 5), x.`~inc`(mul2(mutate x!, x)), x), 0); return result; } @@ -126,7 +126,7 @@ struct Point1D { @method_id(25) fun test_with_single_slot_struct(x: Point1D) { - var result = (x, x.c += 10, [x, x.c += 20, eq(x.c -= 50), x.c], eq2((x.c, x.c *= eq(x.c /= 2)))); + var result = (x, x.c += 10, [x, x.c += 20, eq(x.c -= 50), x.c], eq2((x.c, x.c *= eq(x.c! /= 2)))); return result; } diff --git a/tolk-tester/tests/invalid-semantics/err-4311.tolk b/tolk-tester/tests/invalid-semantics/err-4311.tolk new file mode 100644 index 0000000000..81902366cf --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4311.tolk @@ -0,0 +1,19 @@ +fun int.add(mutate self, dx: int) { + self += dx +} + +fun int.del(mutate self): int { + self = 0; + return self +} + +fun main() { + var x = 10; + x.add(x.del()); +} + +/** +@compilation_should_fail +@stderr can not borrow `x` for mutation once again, it is already being mutated by `int.add` +@stderr hint: split a complex expression into several simple ones + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4312.tolk b/tolk-tester/tests/invalid-semantics/err-4312.tolk new file mode 100644 index 0000000000..8c9b3f4dee --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4312.tolk @@ -0,0 +1,31 @@ +struct Nested { + n1: int + n2: int +} + +struct Demo { + a: int + b: int + c: int + n1: Nested + n2: Nested +} + +fun Nested.mut(mutate self, x: int): int { + return x +} + +fun int.del(mutate self): int { + self = 0; + return self +} + +fun f(d: Demo) { + d.n1.mut(d.n1!.n2.del()) +} + +/** +@compilation_should_fail +@stderr can not borrow `d.n1.n2` for mutation, because `d.n1` is already being mutated by `Nested.mut` +@stderr ^^^^^^^ + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4313.tolk b/tolk-tester/tests/invalid-semantics/err-4313.tolk new file mode 100644 index 0000000000..42214f352c --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4313.tolk @@ -0,0 +1,16 @@ +fun int.incSelfWithOther(mutate self, mutate other: int) { + self += 1; + other += 1; +} + +fun m() { + var x = 0; + var y = 0; + x.incSelfWithOther(mutate y); // ok + x.incSelfWithOther(mutate x); +} + +/** +@compilation_should_fail +@stderr can not borrow `x` for mutation once again, it is already being mutated by `int.incSelfWithOther` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4314.tolk b/tolk-tester/tests/invalid-semantics/err-4314.tolk new file mode 100644 index 0000000000..5197202881 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4314.tolk @@ -0,0 +1,37 @@ +struct Nested { + n1: int + n2: int +} + +struct Demo { + a: int + b: int + c: int + n1: Nested + n2: Nested +} + +fun Demo.assignB(mutate self, newB: int): Demo { + self.b = newB; + return self; +} + +fun mutateDemoAndGetC(mutate d: Demo): int { + return d.c +} + +fun m(d1: Demo, d2: Demo) { + // ok + d1.assignB(10); + d1.assignB(mutateDemoAndGetC(mutate d2)); + d1.assignB(d2.a += d1.b); + d1.assignB(d2.c = d1.n1.n1); + + // err + d1.assignB(mutateDemoAndGetC(mutate d1)); +} + +/** +@compilation_should_fail +@stderr can not borrow `d1` for mutation once again, it is already being mutated by `Demo.assignB` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4315.tolk b/tolk-tester/tests/invalid-semantics/err-4315.tolk new file mode 100644 index 0000000000..06a2dc9167 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4315.tolk @@ -0,0 +1,36 @@ +struct Nested { + n1: int + n2: int +} + +struct Demo { + a: int + b: int + c: int + n1: Nested + n2: Nested +} + +fun Nested.assign(mutate self, n: Nested) { + self = n +} + +fun m(d: Demo) { + // ok + d.a += (d.b += (d.c = 10)); + // ok + d.n1.assign({ + n1: d.n1.n1, + n2: d.n1.n2, + }); + // err + d.n1.assign({ + n1: ((_, d.n1.n2) = (0, 1)).0, + n2: 10, + }); +} + +/** +@compilation_should_fail +@stderr can not borrow `d.n1.n2` for mutation, because `d.n1` is already being mutated by `Nested.assign` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4316.tolk b/tolk-tester/tests/invalid-semantics/err-4316.tolk new file mode 100644 index 0000000000..dfeec69b65 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4316.tolk @@ -0,0 +1,34 @@ +struct Items { + mapX: map; + mapY: map; +} + +fun Items.addX(mutate self, k: uint32, v: uint64) { + self.mapX.set(k, v); +} + +fun Items.addY(mutate self, k: uint64, v: bool) { + self.mapY.set(k, v); +} + +fun Items.removeX(mutate self, i: uint32) { + val lookup = self.mapX.deleteAndGetDeleted(i); + assert(lookup.isFound) throw 444; + return lookup.loadValue(); +} + +get fun test_dict() { + var items = Items{ + mapX: createEmptyMap(), + mapY: createEmptyMap(), + }; + + items.addX(1, 2); + items.addY(items.removeX(1), true); + assert(!items.mapX.findFirst().isFound) throw 501; +} + +/** +@compilation_should_fail +@stderr can not borrow `items` for mutation once again, it is already being mutated by `Items.addY` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4370.tolk b/tolk-tester/tests/invalid-semantics/err-4370.tolk new file mode 100644 index 0000000000..82f867f66d --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4370.tolk @@ -0,0 +1,21 @@ +fun eq(x: T) { return x } + +fun int.incrementBy(mutate self, dx: int) { self += dx } + +fun test(x: int, y: int) { + // ok + x = (x += 10); + x += (y = x); + x.incrementBy(([y] = [x]).0); + x.incrementBy(([y] = [x]).0); + x.incrementBy(([eq([x].0 = x + 10)].0)); + + // error + x.incrementBy(([y, x] = [0, 0]).0); +} + +/** +@compilation_should_fail +@stderr can not borrow `x` for mutation once again, it is already being mutated by `int.incrementBy` +@stderr ([y, x] = [0, 0]).0 + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4371.tolk b/tolk-tester/tests/invalid-semantics/err-4371.tolk new file mode 100644 index 0000000000..dc0d483393 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4371.tolk @@ -0,0 +1,21 @@ +fun eq(x: T) { return x } + +fun int.incrementBy(mutate self, dx: int) { self += dx } + +fun test(xy: (int, int)) { + // ok + xy.0 = (xy.0 += 10); + xy.0 += (xy.1 = xy.0); + xy.0.incrementBy(([xy.1] = [xy.0]).0); + xy.0.incrementBy(([xy.1] = [xy.0]).0); + xy.0.incrementBy(([eq([xy.0].0 = xy.0 + 10)].0)); + + // error + var eqVoid = eq; + eqVoid(xy.0.incrementBy(([xy.1, (xy.0.incrementBy(1), xy.0).1] = [0, 0]).0)); +} + +/** +@compilation_should_fail +@stderr can not borrow `xy.0` for mutation once again, it is already being mutated by `int.incrementBy` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4372.tolk b/tolk-tester/tests/invalid-semantics/err-4372.tolk new file mode 100644 index 0000000000..6012771adc --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4372.tolk @@ -0,0 +1,21 @@ +struct Point { + x: int + y: int +} + +fun Point.copy(self) { return self } +fun Point.mut(mutate self) { return self } + +fun test(p: Point) { + // ok + p.x += (p.copy().x = 5); + p.x += (p.copy().x += 5); + + // err + p.x += (p.mut().x = 5); +} + +/** +@compilation_should_fail +@stderr can not borrow `p` for mutation, because `p.x` is already being mutated by `assignment operator` + */ diff --git a/tolk-tester/tests/mutate-methods.tolk b/tolk-tester/tests/mutate-methods.tolk index b31e91ac89..838e7171f0 100644 --- a/tolk-tester/tests/mutate-methods.tolk +++ b/tolk-tester/tests/mutate-methods.tolk @@ -22,6 +22,21 @@ fun int.incrementTwoInPlace(mutate self, mutate y: int, byValue: int): int { return self + y; } +struct A { + f1: int + f2: int +} + +fun A.assign(mutate self, obj: A) { + self = obj; +} + +fun A.assignWith(mutate self, mutate obj: A) { + self = obj; + obj.f2 = 0; +} + + @method_id(101) fun testIncrement1() { var x = 50; @@ -306,6 +321,29 @@ fun testExplicitReturn() { ); } +fun int.add(mutate self, dx: int) { self += dx } +fun int.del(mutate self) { self = 0; return self } + +@method_id(116) +fun testUB() { + var x = 100; + var y = 100; + // x.add(x.del()) is disallowed by the borrow checker, but `!` operator allows bypassing + x.add(x!.del()); + y.add(int.del(mutate y!)); + return (x, y); +} + +@method_id(117) +fun testReceiverAndNot() { + var (a1: A, a2: A) = ({ f1: 10, f2: 20 }, { f1: 30, f2: 40 }); + a1.assign(a1); + A.assign(mutate a1, a1); + a1.assignWith(mutate a2); + A.assignWith(mutate a2, mutate a1); + return (a1, a2); +} + fun main(){ // mark used to codegen them @@ -328,6 +366,8 @@ fun main(){ @testcase | 113 | | 320 @testcase | 114 | | 1 2 4 @testcase | 115 | | 13 2 +@testcase | 116 | | 100 100 +@testcase | 117 | | 30 0 30 40 @fif_codegen """ @@ -373,4 +413,15 @@ fun main(){ }> """ +@fif_codegen +""" + testReceiverAndNot() PROC:<{ // + 30 PUSHINT // '6=30 + 40 PUSHINT // obj.f1=30 obj.f2=40 + OVER + 0 PUSHINT // a1.f1=30 a2.f2=40 a2.f1=30 a1.f2=0 + s0 s2 XCHG // a1.f1=30 a1.f2=0 a2.f1=30 a2.f2=40 + }> +""" + */ diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index 1085cac846..a2404eaec0 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -23,6 +23,7 @@ set(TOLK_SOURCE pipe-check-rvalue-lvalue.cpp pipe-check-private-fields.cpp pipe-check-pure-impure.cpp + pipe-mini-borrow-checker.cpp pipe-constant-folding.cpp pipe-optimize-boolean-expr.cpp pipe-detect-inline-in-place.cpp diff --git a/tolk/pipe-mini-borrow-checker.cpp b/tolk/pipe-mini-borrow-checker.cpp new file mode 100644 index 0000000000..d91995f8b6 --- /dev/null +++ b/tolk/pipe-mini-borrow-checker.cpp @@ -0,0 +1,206 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "ast.h" +#include "ast-visitor.h" +#include "smart-casts-cfg.h" +#include "compilation-errors.h" +#include + +/* + * This pipe prevents concurrent mutations of one and the same object at a time, because it's UB (undefined behavior). + * Example 1: + * > f(mutate x, mutate x) + * Example 2: + * > items.add(items.remove(), true) (both methods are mutating) + * Example 3: + * > x += ([x] = [0]).0 + * In all cases above, an error "can not borrow XXX for mutation once again" is printed. + * + * We analyze and prevent mutating not only for a single variable, but also for fields of an object/tensor. + * Independent fields can be safely mutated: + * > obj.x = (obj.y = ...) // ok, independent fields + * But for example, mutating `d.nested.field` while mutating `d.nested` or `d` itself, is an error: + * > d.mutatingMethod(d.nested.field = 10) // error, can not borrow `d.nested.field`, because `d` already borrowed + * > p.x += (p.mut().x = 5) // error, can not borrow `p`, because `p.x` already borrowed + * To support fields and independency check, we use SinkExpression — the same struct that is used for smart casts. + * Both variables `v` and fields `obj.f1.f2` are sink expressions. They can be extracted from vertices and compared. + * Note, that operators `as` and `!` are not valid sink expressions. As a consequence, `!` can be used to overcome + * compiler checks here: `items.add(items!.remove())` becomes okay. + * + * In order to prevent `x += ([x] = rhs).0`, we need to carefully dig into lhs of assignment. Traversing top-down, + * we can't just mark "we are inside lhs of assignment" and treat all references as mutated there, because + * `getObj(x).field` is a valid lhs, where `x` is not mutated. + */ + +namespace tolk { + +struct BorrowedVarOrField { + SinkExpression s_expr; + FunctionPtr by_function; + + std::string stringify_by_function() const { + if (by_function != nullptr) { + return by_function->as_human_readable(); + } + // only operators `+=` and similar may borrow a variable except a mutating function + return "assignment operator"; + } +}; + +// A context holding currently borrowed expressions. +// When entering a function or lhs of assignment, mutated expressions are added here. +// Therefore, while traversing top-down, other attempts to add the same expression will result in an error. +class BorrowedForWriteCtx { + std::forward_list expressions; + std::forward_list::iterator> frame_heads; + +public: + bool empty() const { + return expressions.empty(); + } + + void push_frame() { + frame_heads.emplace_front(expressions.begin()); + } + + void pop_frame() { + std::forward_list::iterator target = frame_heads.front(); + frame_heads.pop_front(); + while (expressions.begin() != target) { + expressions.pop_front(); + } + } + + void borrow_or_fire_if_twice(FunctionPtr cur_f, SinkExpression s_expr, AnyExprV where, FunctionPtr by_function) { + for (const BorrowedVarOrField& existing : expressions) { + if (existing.s_expr == s_expr) { + err("can not borrow `{}` for mutation once again, it is already being mutated by `{}`\n""hint: split a complex expression into several simple ones", s_expr.to_string(), existing.stringify_by_function()).fire(where, cur_f); + } + if (existing.s_expr.is_child_of(s_expr) || s_expr.is_child_of(existing.s_expr)) { + err("can not borrow `{}` for mutation, because `{}` is already being mutated by `{}`\n""hint: split a complex expression into several simple ones", s_expr.to_string(), existing.s_expr.to_string(), existing.stringify_by_function()).fire(where, cur_f); + } + } + expressions.emplace_front(BorrowedVarOrField{s_expr, by_function}); + } +}; + +class CheckMutationNotHappensTwiceVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; + BorrowedForWriteCtx borrow_ctx; + +protected: + void visit(V v) override { + FunctionPtr fun_ref = v->fun_maybe; + if (!fun_ref) { // a "call" to a variable, it can't be mutating + parent::visit(v); + return; + } + + borrow_ctx.push_frame(); + + int delta_self = v->get_self_obj() != nullptr; + + // obj.mutatingMethod() — borrow obj while calculating all arguments + if (v->dot_obj_is_self) { + AnyExprV self_obj = v->get_self_obj(); + parent::visit(self_obj); + if (fun_ref->does_mutate_self()) { + if (SinkExpression s_expr = extract_sink_expression_from_vertex(self_obj)) { + borrow_ctx.borrow_or_fire_if_twice(cur_f, s_expr, self_obj, fun_ref); + } + } + } + // f(mutate x) — borrow x while calculating rest arguments + for (int i = 0; i < v->get_num_args(); ++i) { + AnyExprV ith_arg = v->get_arg(i)->get_expr(); + parent::visit(ith_arg); + if (fun_ref->parameters[delta_self + i].is_mutate_parameter()) { + if (SinkExpression s_expr = extract_sink_expression_from_vertex(ith_arg)) { + borrow_ctx.borrow_or_fire_if_twice(cur_f, s_expr, ith_arg, fun_ref); + } + } + } + + borrow_ctx.pop_frame(); + } + + void visit(V v) override { + // recursively analyze assignment lhs to find not only `x = rhs`, but also `(([_, x], _)) = rhs` + // note, that rhs CAN mutate x, because assignment is happening only after evaluating it + // (unlike `x += rhs`, which can't mutate x) + process_assignment_lhs(v->get_lhs()); + parent::visit(v->get_rhs()); + } + + void visit(V v) override { + // unlike assignment `x = rhs`, operators `+=` and similar don't allow tensors and similar + borrow_ctx.push_frame(); + if (SinkExpression lhs_s_expr = extract_sink_expression_from_vertex(v->get_lhs())) { + borrow_ctx.borrow_or_fire_if_twice(cur_f, lhs_s_expr, v->get_lhs(), nullptr); + } + + // borrow lhs while calculating rhs, because `x += rhs` is actually `x = x + rhs` + // (rhs can't mutate x, it's copied before evaluating rhs) + parent::visit(v); + borrow_ctx.pop_frame(); + } + + void process_assignment_lhs(AnyExprV lhs) { + // we are not interested in `var x = rhs`, only in assigning to existing `x = rhs` + if (auto lhs_tensor = lhs->try_as()) { + for (int i = 0; i < lhs_tensor->size(); ++i) { + process_assignment_lhs(lhs_tensor->get_item(i)); + } + return; + } + if (auto lhs_tuple = lhs->try_as()) { + for (int i = 0; i < lhs_tuple->size(); ++i) { + process_assignment_lhs(lhs_tuple->get_item(i)); + } + return; + } + + // note, that for `x = rhs` we ALLOW rhs to mutate x, because assignment happens after evaluating rhs; + // for example, `b = b.storeInt()` is common and correct; + // what we do here is checking that assignment is allowed in this exact place, it's not already borrowed: + // `point.mutate(..., point.x = 10)` // can't borrow `point.x`, because `point` is already being mutated + if (SinkExpression lhs_s_expr = extract_sink_expression_from_vertex(lhs)) { + borrow_ctx.push_frame(); + borrow_ctx.borrow_or_fire_if_twice(cur_f, lhs_s_expr, lhs, nullptr); + borrow_ctx.pop_frame(); + } + + parent::visit(lhs); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + tolk_assert(borrow_ctx.empty()); + } +}; + +void pipeline_mini_borrow_checker_for_mutate() { + visit_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipeline.h b/tolk/pipeline.h index d433afcfca..f7b977f787 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -42,6 +42,7 @@ void pipeline_refine_lvalue_for_mutate_arguments(); void pipeline_check_rvalue_lvalue(); void pipeline_check_private_fields_usage(); void pipeline_check_pure_impure_operations(); +void pipeline_mini_borrow_checker_for_mutate(); void pipeline_constant_folding(); void pipeline_optimize_boolean_expressions(); void pipeline_detect_inline_in_place(); diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index 967ff77ea4..34b577c457 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -113,6 +113,17 @@ std::string SinkExpression::to_string() const { return result; } +bool SinkExpression::is_child_of(SinkExpression rhs) const { + // `c.1.2` (index_path = 3<<8 + 2) is a child of `c.1` (index_path = 2) and `c` (index_path = 0) + uint64_t mask = 0; + uint64_t rhs_path = rhs.index_path; + while (rhs_path != 0) { + mask = (mask << 8) + 0xFF; + rhs_path >>= 8; + } + return var_ref == rhs.var_ref && index_path != rhs.index_path && (index_path & mask) == rhs.index_path; +} + SinkExpression SinkExpression::get_child_s_expr(int field_idx) const { uint64_t new_index_path = index_path; // if we have c.1 (index_path = 2) and construct c.1.N, calc (N<<8 + 2) for (int empty_byte = 0; empty_byte < 8; ++empty_byte) { diff --git a/tolk/smart-casts-cfg.h b/tolk/smart-casts-cfg.h index 1fb491d8f3..6f737f896a 100644 --- a/tolk/smart-casts-cfg.h +++ b/tolk/smart-casts-cfg.h @@ -71,6 +71,7 @@ struct SinkExpression { explicit operator bool() const { return var_ref != nullptr; } std::string to_string() const; + bool is_child_of(SinkExpression rhs) const; SinkExpression get_child_s_expr(int field_idx) const; }; diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp index c569a3724f..79eb986915 100644 --- a/tolk/tolk.cpp +++ b/tolk/tolk.cpp @@ -53,6 +53,7 @@ int tolk_proceed(const std::string &entrypoint_filename) { pipeline_check_rvalue_lvalue(); pipeline_check_private_fields_usage(); pipeline_check_pure_impure_operations(); + pipeline_mini_borrow_checker_for_mutate(); pipeline_constant_folding(); pipeline_optimize_boolean_expressions(); pipeline_detect_inline_in_place(); From 39ffde5f93790b0a9623ef4cc56deee20c95ed78 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Wed, 29 Oct 2025 12:23:36 +0300 Subject: [PATCH 454/524] [Tolk] Peephole: conditions, integer and boolean negations, and THROWIF - removed incorrect optimization `NOT + N THROWIF` -> `N THROWIFNOT` (it's incorrect because NOT is bitwise, whereas THROWIFNOT is logical) - introduce a fake BOOLNOT instruction to perform correct optimizations - `0 EQINT` + `N THROWIF` -> `N THROWIFNOT` - `0 NEQINT` + `N THROWIF` -> `N THROWIF` --- tolk-tester/tests/logical-operators.tolk | 88 +++++++++++++++++++++++- tolk-tester/tests/maps-tests.tolk | 18 +++++ tolk-tester/tests/pack-unpack-6.tolk | 4 +- tolk-tester/tests/parse-address.tolk | 3 +- tolk/builtins.cpp | 51 +++++++------- tolk/gen-entrypoints.cpp | 8 +-- tolk/maps-kv-api.cpp | 15 ++-- tolk/optimize.cpp | 85 +++++++++++++++++++++-- tolk/pack-unpack-serializers.cpp | 29 ++++---- tolk/pipe-ast-to-legacy.cpp | 5 +- tolk/tolk.h | 5 +- 11 files changed, 240 insertions(+), 71 deletions(-) diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk index 0fcd3f25c0..4fb5007c89 100644 --- a/tolk-tester/tests/logical-operators.tolk +++ b/tolk-tester/tests/logical-operators.tolk @@ -174,6 +174,61 @@ fun test115() { return (tru, fals); } +@method_id(116) +fun testCodegenOptimizeThrowIF(b: bool, i: int) { + assert (!b) throw 100; + assert (!!b) throw 100; + assert (b == false) throw 100; + assert (b != false) throw 100; + + assert (!i) throw 101; + assert (!!i) throw 101; + assert (i == 0) throw 101; + assert (i != 0) throw 101; +} + +@method_id(117) +fun test117(x: int) { + var t = createEmptyTuple(); + + try { assert (x == 0) throw 100; } + catch (ex) { t.push(ex); } + try { assert (x != 0) throw 101; } + catch (ex) { t.push(ex); } + try { assert (!(x == 10)) throw 102; } + catch (ex) { t.push(ex); } + try { assert (!(x != 100)) throw 103; } + catch (ex) { t.push(ex); } + try { assert (!!(x != 100)) throw 104; } + catch (ex) { t.push(ex); } + try { assert (x) throw 105; } + catch (ex) { t.push(ex); } + try { assert (!x) throw 106; } + catch (ex) { t.push(ex); } + try { assert (~x) throw 107; } + catch (ex) { t.push(ex); } + try { assert (~(x + 1)) throw 108; } + catch (ex) { t.push(ex); } + + return t; +} + +@method_id(118) +fun test118(y: int?) { + var t = createEmptyTuple(); + + try { assert (y == null) throw 200; } + catch (ex) { t.push(ex); } + try { assert (!(y == null)) throw 201; } + catch (ex) { t.push(ex); } + try { assert (y != null) throw 202; } + catch (ex) { t.push(ex); } + try { assert (!(y != null)) throw 203; } + catch (ex) { t.push(ex); } + + return t; +} + fun main() { @@ -216,6 +271,12 @@ fun main() { @testcase | 113 | 0 | 1 @testcase | 114 | | 0 @testcase | 115 | | -1 0 +@testcase | 117 | -1 | [ 100 103 106 107 ] +@testcase | 117 | -2 | [ 100 103 106 108 ] +@testcase | 117 | 10 | [ 100 102 103 106 ] +@testcase | 117 | 100 | [ 100 104 106 ] +@testcase | 118 | null | [ 201 202 ] +@testcase | 118 | 5 | [ 200 203 ] @fif_codegen """ @@ -353,15 +414,36 @@ These are moments of future optimizations. For now, it's more than enough. DUP // x x IFNOTJMP:<{ // x DROP // - 1 PUSHINT // '7=1 + 1 PUSHINT // '5=1 }> // x DUP // x x IFNOTJMP:<{ // x DROP // - 1 PUSHINT // '8=1 + 1 PUSHINT // '6=1 }> // x 100 THROWIFNOT - -4 PUSHINT // '11=-4 + -4 PUSHINT // '8=-4 + }> +""" + +@fif_codegen +""" + testCodegenOptimizeThrowIF() PROC:<{ + OVER + 100 THROWIF + OVER + 100 THROWIFNOT + OVER + 100 THROWIF + SWAP + 100 THROWIFNOT + DUP + 101 THROWIF + DUP + 101 THROWIFNOT + DUP + 101 THROWIF + 101 THROWIFNOT }> """ diff --git a/tolk-tester/tests/maps-tests.tolk b/tolk-tester/tests/maps-tests.tolk index 8bb7b95044..85f1e282a1 100644 --- a/tolk-tester/tests/maps-tests.tolk +++ b/tolk-tester/tests/maps-tests.tolk @@ -525,6 +525,23 @@ fun test41() { ); } +@method_id(142) +fun test42(dyn: int) { + var m = createEmptyMap(); + var t = createEmptyTuple(); + + try { m.mustGet(1, 123); } + catch (ex) { t.push(ex) } + try { m.mustGet(1, 12345); } + catch (ex) { t.push(ex) } + try { m.mustGet(1, dyn); } + catch (ex) { t.push(ex) } + try { m.mustGet(1, dyn * 10); } + catch (ex) { t.push(ex) } + + return t; +} + fun main() { __expect_type(createEmptyMap, "() -> map"); @@ -573,6 +590,7 @@ fun main() { @testcase | 139 | | 0 -1 2 @testcase | 140 | | 0 0 -1 777 -1 0 777 (null) typeid-1 (null) 0 @testcase | 141 | | 0 -1 -1 0 -1 +@testcase | 142 | 80 | [ 123 12345 80 800 ] @fif_codegen """ diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk index ae3ec403f0..2a8407bc68 100644 --- a/tolk-tester/tests/pack-unpack-6.tolk +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -185,8 +185,8 @@ IF:<{ x{01} SDBEGINSQ // '2=65535 s '4 s1 s2 XCHG // s '2=65535 '4 THROWANYIFNOT // s - 8 LDI // '9 s - ENDS // '9 + 8 LDI // '8 s + ENDS // '8 }> """ diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index fb6b8a0a17..8ca3744192 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -156,8 +156,7 @@ fun main() { check0() PROC:<{ // addr REWRITESTDADDR DROP // '2 - 0 EQINT // '4 - 111 THROWIFNOT // + 111 THROWIF // }> """ diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 85df50cbf1..0f734f6478 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -492,8 +492,11 @@ static AsmOp compile_logical_not(std::vector& res, std::vector `N THROWIFNOT`, but for `NOT` (generally) it's incorrect; + // un-optimized `BOOLNOT` are later replaced with a regular `NOT` + return for_int_arg ? exec_op(origin, "0 EQINT", 1) : exec_op(origin, "BOOLNOT", 1); } static AsmOp compile_bitwise_and(std::vector& res, std::vector& args, AnyV origin) { @@ -966,29 +969,29 @@ static AsmOp compile_throw(std::vector& res, std::vector& ar } } -static AsmOp compile_throw_if_unless(std::vector& res, std::vector& args, AnyV origin) { - tolk_assert(res.empty() && args.size() == 3); - VarDescr &x = args[0], &y = args[1], &z = args[2]; - if (!z.always_true() && !z.always_false()) { - throw Fatal("invalid usage of built-in symbol"); +static AsmOp compile_throw_if_ifnot(std::vector& res, std::vector& args, AnyV origin, bool is_ifnot) { + tolk_assert(res.empty() && args.size() == 2); + VarDescr &x = args[0], &y = args[1]; + + bool skip_all = is_ifnot ? y.always_true() : y.always_false(); // __throw_if(ex, false): do nothing + if (skip_all) { + x.unused(); + y.unused(); + return AsmOp::Nop(origin); } - bool mode = z.always_true(); - z.unused(); - std::string suff = (mode ? "IF" : "IFNOT"); - bool skip_cond = false; - if (y.always_true() || y.always_false()) { + + bool skip_cond = y.always_true() || y.always_false(); + if (skip_cond) { y.unused(); - skip_cond = true; - if (y.always_true() != mode) { - x.unused(); - return AsmOp::Nop(origin); - } } + if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { x.unused(); - return skip_cond ? exec_arg_op(origin, "THROW", x.int_const, 0, 0) : exec_arg_op(origin, "THROW"s + suff, x.int_const, 1, 0); + std::string cond_asm = is_ifnot ? "THROWIFNOT" : "THROWIF"; + return skip_cond ? exec_arg_op(origin, "THROW", x.int_const, 0, 0) : exec_arg_op(origin, cond_asm, x.int_const, 1, 0); } else { - return skip_cond ? exec_op(origin, "THROWANY", 1, 0) : exec_op(origin, "THROWANY"s + suff, 2, 0); + std::string cond_asm = is_ifnot ? "THROWANYIFNOT" : "THROWANYIF"; + return skip_cond ? exec_op(origin, "THROWANY", 1, 0) : exec_op(origin, cond_asm, 2, 0); } } @@ -1352,7 +1355,6 @@ GenerateOpsImpl generate_mapKV_iteratePrev; CompileToAsmOpImpl compile_createEmptyMap; CompileToAsmOpImpl compile_createMapFromLowLevelDict; CompileToAsmOpImpl compile_dict_get; -CompileToAsmOpImpl compile_dict_mustGet; CompileToAsmOpImpl compile_dict_getMin; CompileToAsmOpImpl compile_dict_getMax; CompileToAsmOpImpl compile_dict_getNext; @@ -1510,8 +1512,11 @@ void define_builtins() { define_builtin_func("__throw_arg", {TypeDataUnknown::create(), Int}, Never, nullptr, compile_throw_arg, 0); - define_builtin_func("__throw_if_unless", ParamsInt3, Unit, nullptr, - compile_throw_if_unless, + define_builtin_func("__throw_if", ParamsInt2, Unit, nullptr, + std::bind(compile_throw_if_ifnot, _1, _2, _3, false), + 0); + define_builtin_func("__throw_ifnot", ParamsInt2, Unit, nullptr, + std::bind(compile_throw_if_ifnot, _1, _2, _3, true), 0); define_builtin_func("__InMessage.originalForwardFee", ParamsInt2, Int, nullptr, compile_calc_InMessage_originalForwardFee, @@ -1791,8 +1796,6 @@ void define_builtins() { define_builtin_func("__dict.get", {KeySliceOrInt, PlainDict, TypeDataInt::create()}, LookupSliceFound, nullptr, compile_dict_get, 0); - define_builtin_func("__dict.mustGet", {KeySliceOrInt, PlainDict, TypeDataInt::create()}, LookupSliceFound, nullptr, - compile_dict_mustGet, 0); define_builtin_func("__dict.getMin", {PlainDict}, TypeDataTensor::create({ValueSlice, KeySliceOrInt, ValueFound}), nullptr, compile_dict_getMin, 0); define_builtin_func("__dict.getMax", {PlainDict}, TypeDataTensor::create({ValueSlice, KeySliceOrInt, ValueFound}), nullptr, diff --git a/tolk/gen-entrypoints.cpp b/tolk/gen-entrypoints.cpp index a2a897a36b..a2d2cbf37a 100644 --- a/tolk/gen-entrypoints.cpp +++ b/tolk/gen-entrypoints.cpp @@ -110,10 +110,10 @@ void handle_onInternalMessage_codegen_start(FunctionPtr f_onInternalMessage, con code.close_pop_cur(origin); } } else { - // generate: `assert (!isBounced) throw 0` - std::vector args_throw0if = { code.create_int(origin, 0, "(exit-0)"), ir_isBounced[0], code.create_int(origin, 1, "") }; - Op& op_assert = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_throw0if), lookup_function("__throw_if_unless")); - op_assert.set_impure_flag(); + // generate: `if (isBounced) throw 0` + std::vector args = { code.create_int(origin, 0, "(exit-0)"), ir_isBounced[0] }; + Op& op_throw0if = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args), lookup_function("__throw_if")); + op_throw0if.set_impure_flag(); } } diff --git a/tolk/maps-kv-api.cpp b/tolk/maps-kv-api.cpp index 47ffb435c8..0c7e674aac 100644 --- a/tolk/maps-kv-api.cpp +++ b/tolk/maps-kv-api.cpp @@ -353,14 +353,6 @@ AsmOp compile_dict_get(std::vector& res, std::vector& args, return AsmOp::Custom(origin, op + " NULLSWAPIFNOT", 3, 2); } -// DICTGET: k D n => (x −1) OR (0) -// but since it's MUST get, it's wrapped to assert(found), so assume x (value) will exist on a stack -AsmOp compile_dict_mustGet(std::vector& res, std::vector& args, AnyV origin) { - tolk_assert(res.size() == 2 && args.size() == 2+3); - std::string op = choose_dict_op("DICTGET", args[0], args[1]); - return AsmOp::Custom(origin, op, 3, 2); -} - // DICTMIN: D n => (x k −1) OR (0); + NULLSWAPIFNOT2 => (x k -1) OR (null null 0) AsmOp compile_dict_getMin(std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 3 && args.size() == 2+2); @@ -497,11 +489,12 @@ std::vector generate_mapKV_mustGet(FunctionPtr called_f, CodeBlob& co std::vector ir_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create()}), origin, "(lookup)"); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; - code.emplace_back(origin, Op::_Call, ir_lookup, std::move(dict_args), lookup_function("__dict.mustGet")); + code.emplace_back(origin, Op::_Call, ir_lookup, std::move(dict_args), lookup_function("__dict.get")); - std::vector args_assert = { args[2][0], ir_lookup[1], code.create_int(origin, 0, "") }; - Op& op_assert = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); + std::vector args_throwifnot = { args[2][0], ir_lookup[1] }; + Op& op_assert = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_throwifnot), lookup_function("__throw_ifnot")); op_assert.set_impure_flag(); + // later on, preceding `NULLSWAPIFNOT` will be removed if possible by a peephole optimization std::vector ir_slice(ir_lookup.begin(), ir_lookup.begin() + 1); if (is_TValue_raw_slice(TValue)) { diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index 3282a201db..7c55baf17c 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -399,10 +399,12 @@ bool Optimizer::detect_rewrite_SWAP_STxxxR() { return false; } -// pattern `NOT` + `123 THROWIFNOT` -> `123 THROWIF` (and THROWIF -> THROWIFNOT) -bool Optimizer::detect_rewrite_NOT_THROWIF() { - bool first_not = op_[0]->is_custom() && op_[0]->op == "NOT"; - if (!first_not || pb_ < 2 || !op_[1]->is_custom()) { +// pattern `BOOLNOT` + `123 THROWIFNOT` -> `123 THROWIF` and vice versa; +// generally, it's incorrect (`NOT` is bitwise, `THROWIFNOT` is logical), but for bools (-1/0) it's correct; +// for logical negation `!boolVar`, a special fake `BOOLNOT` instruction was inserted +bool Optimizer::detect_rewrite_BOOLNOT_THROWIF() { + bool first_bool_not = op_[0]->is_custom() && op_[0]->op == "BOOLNOT"; + if (!first_bool_not || pb_ < 2 || !op_[1]->is_custom()) { return false; } @@ -414,7 +416,8 @@ bool Optimizer::detect_rewrite_NOT_THROWIF() { if (f.ends_with(ends_with[i])) { p_ = 2; q_ = 1; - oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, op_[1]->op.substr(0, f.rfind(' ')) + repl_with[i], 1, 0)); + std::string new_op = op_[1]->op.substr(0, f.rfind(' ')) + repl_with[i]; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, new_op, 1, 0)); return true; } } @@ -422,6 +425,37 @@ bool Optimizer::detect_rewrite_NOT_THROWIF() { return false; } +// pattern `0 EQINT` + `N THROWIF` -> `N THROWIFNOT` and vice versa; +// or remove condition if negated: `0 NEQINT` + `N THROWIF` -> `N THROWIF`; +// particularly, this helps to optimize code like `assert (v == 0, N)` with just one `N THROWIF` +bool Optimizer::detect_rewrite_0EQINT_THROWIF() { + bool first_0eqint = op_[0]->is_custom() && (op_[0]->op == "0 EQINT" || op_[0]->op == "0 NEQINT"); + if (!first_0eqint || pb_ < 2 || !op_[1]->is_custom()) { + return false; + } + + static const char* ends_with[] = {" THROWIF", " THROWIFNOT"}; + static const char* repl_with[] = {" THROWIFNOT", " THROWIF"}; + + std::string_view f = op_[1]->op; + for (size_t i = 0; i < std::size(ends_with); ++i) { + if (f.ends_with(ends_with[i])) { + bool drop_cond = op_[0]->op == "0 NEQINT"; + p_ = 2; + q_ = 1; + if (drop_cond) { + oq_[0] = std::move(op_[1]); + } else { + std::string new_op = op_[1]->op.substr(0, f.rfind(' ')) + repl_with[i]; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, new_op, 1, 0)); + } + return true; + } + } + + return false; +} + // pattern `NEWC` + store const slice + XCHG + keyLen + DICTSETB -> push const slice + XCHG + keyLen + DICTSET // (useful for `someMap.set(k, constVal)` where constVal is represented as a const slice) bool Optimizer::detect_rewrite_DICTSETB_DICTSET() { @@ -458,6 +492,28 @@ bool Optimizer::detect_rewrite_DICTSETB_DICTSET() { return false; } +// pattern `DICTGET NULLSWAPIFNOT` + `N THROWIFNOT` -> `DICTGET` + `N THROWIFNOT` (remove nullswap); +// especially useful for `dict.mustGet()` method with a small constant errno if a key not exists +// (for large or dynamic excno, it's XCHGed from a stack, we need to keep stack aligned, don't remove nullswap) +bool Optimizer::detect_rewrite_DICTGET_NULLSWAPIFNOT_THROWIFNOT() { + bool second_nullswap = pb_ >= 2 && op_[0]->is_custom() && op_[0]->op.ends_with(" NULLSWAPIFNOT"); + if (!second_nullswap || !op_[1]->op.ends_with(" THROWIFNOT")) { + return false; + } + + std::string op0 = op_[0]->op; + if (!op0.starts_with("DICT")) { + return false; + } + + p_ = 2; + q_ = 2; + std::string new_op = op0.substr(0, op0.rfind(' ')); + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, new_op, 3, 2)); + oq_[1] = std::move(op_[1]); + return true; +} + // pattern `ENDC` + `CTOS` -> `BTOS` (a new TVM 12 instruction "builder to slice") bool Optimizer::detect_rewrite_ENDC_CTOS() { bool first_endc = op_[0]->is_custom() && op_[0]->op == "ENDC"; @@ -570,6 +626,21 @@ bool Optimizer::detect_rewrite_NEWC_ENDC() { return true; } +// for `!x`, when `x` is boolean, a fake asm instruction `BOOLNOT` was inserted (see builtins.cpp); +// it was used for peephole optimizations, because `NOT + ...` is not correct, since `NOT` is bitwise; +// here we replace instructions left after optimizations with a simple `NOT` (-1 => 0, 0 => -1) +bool Optimizer::replace_BOOLNOT_to_NOT() { + bool first_bool_not = op_[0]->is_custom() && op_[0]->op == "BOOLNOT"; + if (!first_bool_not) { + return false; + } + + p_ = 1; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, "NOT", 1, 1)); + return true; +} + bool Optimizer::is_push_const(int* i, int* c) const { return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_push_const(i, c); } @@ -992,10 +1063,12 @@ bool Optimizer::find_at_least(int pb) { detect_rewrite_MY_store_int() || detect_rewrite_MY_skip_bits() || detect_rewrite_NEWC_PUSH_STUR() || detect_rewrite_LDxx_DROP() || detect_rewrite_SWAP_symmetric() || detect_rewrite_SWAP_PUSH_STUR() || detect_rewrite_SWAP_STxxxR() || - detect_rewrite_NOT_THROWIF() || detect_rewrite_DICTSETB_DICTSET() || + detect_rewrite_BOOLNOT_THROWIF() || detect_rewrite_0EQINT_THROWIF() || + detect_rewrite_DICTSETB_DICTSET() || detect_rewrite_DICTGET_NULLSWAPIFNOT_THROWIFNOT() || detect_rewrite_ENDC_CTOS() || detect_rewrite_ENDC_HASHCU() || detect_rewrite_NEWC_BTOS() || detect_rewrite_NEWC_STSLICECONST_BTOS() || detect_rewrite_NEWC_ENDC_CTOS() || detect_rewrite_NEWC_ENDC() || + (!(mode_ & 1) && replace_BOOLNOT_to_NOT()) || (!(mode_ & 1) && ((is_rot() && rewrite(AsmOp::Custom(origin, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(origin, "-ROT", 3, 3))) || (is_2dup() && rewrite(AsmOp::Custom(origin, "2DUP", 2, 4))) || diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index 6a99f6d150..add44e726e 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -179,8 +179,8 @@ void UnpackContext::loadAndCheckOpcode(PackOpcode opcode) const { std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), origin, "(prefix-eq)"); std::vector args = { ir_slice0, code.create_int(origin, opcode.pack_prefix, "(pack-prefix)"), code.create_int(origin, opcode.prefix_len, "(prefix-len)") }; code.emplace_back(origin, Op::_Call, std::vector{ir_slice0, ir_prefix_eq[0]}, std::move(args), lookup_function("slice.tryStripPrefix")); - std::vector args_assert = { option_throwIfOpcodeDoesNotMatch, ir_prefix_eq[0], code.create_int(origin, 0, "") }; - Op& op_assert = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); + std::vector args_throwifnot = { option_throwIfOpcodeDoesNotMatch, ir_prefix_eq[0] }; + Op& op_assert = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_throwifnot), lookup_function("__throw_ifnot")); op_assert.set_impure_flag(); } @@ -340,18 +340,17 @@ struct S_BitsN final : ISerializer { } { code.push_set_cur(if_disabled_by_user.block1); - FunctionPtr f_assert = lookup_function("__throw_if_unless"); constexpr int EXCNO = 9; std::vector ir_counts = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create()}), origin, "(slice-size)"); code.emplace_back(origin, Op::_Call, ir_counts, rvect, lookup_function("slice.remainingBitsAndRefsCount")); - std::vector args_assert0 = { code.create_int(origin, EXCNO, "(excno)"), ir_counts[1], code.create_int(origin, 1, "") }; - Op& op_assert0 = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_assert0), f_assert); + std::vector args_throwif = { code.create_int(origin, EXCNO, "(excno)"), ir_counts[1] }; + Op& op_assert0 = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_throwif), lookup_function("__throw_if")); op_assert0.set_impure_flag(); std::vector ir_eq_n = code.create_tmp_var(TypeDataInt::create(), origin, "(eq-n)"); code.emplace_back(origin, Op::_Call, ir_eq_n, std::vector{ir_counts[0], code.create_int(origin, n_bits, "(n-bits)")}, lookup_function("_==_")); - std::vector args_assertN = { code.create_int(origin, EXCNO, "(excno)"), ir_eq_n[0], code.create_int(origin, 0, "") }; - Op& op_assertN = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_assertN), f_assert); + std::vector args_throwifnot = { code.create_int(origin, EXCNO, "(excno)"), ir_eq_n[0] }; + Op& op_assertN = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_throwifnot), lookup_function("__throw_ifnot")); op_assertN.set_impure_flag(); code.close_pop_cur(origin); } @@ -1132,9 +1131,9 @@ struct S_IntegerEnum final : ISerializer { code.emplace_back(origin, Op::_IntConst, ir_min_value, min_value); std::vector ir_lt_min = code.create_tmp_var(TypeDataInt::create(), origin, "(enum-lt-min)"); code.emplace_back(origin, Op::_Call, ir_lt_min, std::vector{ir_num[0], ir_min_value[0]}, lookup_function("_<_")); - std::vector args_assert1 = { code.create_int(origin, 5, "(excno)"), ir_lt_min[0], code.create_int(origin, 1, "") }; - Op& op_assert1 = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_assert1), lookup_function("__throw_if_unless")); - op_assert1.set_impure_flag(); + std::vector args_throwif = { code.create_int(origin, 5, "(excno)"), ir_lt_min[0] }; + Op& op_assert = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_throwif), lookup_function("__throw_if")); + op_assert.set_impure_flag(); } td::RefInt256 max_value = enum_ref->members.back()->computed_value; bool dont_check_max = intN->is_unsigned && max_value == (1ULL << intN->n_bits) - 1; @@ -1143,9 +1142,9 @@ struct S_IntegerEnum final : ISerializer { code.emplace_back(origin, Op::_IntConst, ir_max_value, max_value); std::vector ir_gt_max = code.create_tmp_var(TypeDataInt::create(), origin, "(enum-gt-ax)"); code.emplace_back(origin, Op::_Call, ir_gt_max, std::vector{ir_num[0], ir_max_value[0]}, lookup_function("_>_")); - std::vector args_assert2 = { code.create_int(origin, 5, "(excno)"), ir_gt_max[0], code.create_int(origin, 1, "") }; - Op& op_assert2 = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_assert2), lookup_function("__throw_if_unless")); - op_assert2.set_impure_flag(); + std::vector args_throwif = { code.create_int(origin, 5, "(excno)"), ir_gt_max[0] }; + Op& op_assert = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_throwif), lookup_function("__throw_if")); + op_assert.set_impure_flag(); } } else { // okay, enum is not a sequence, just a set of values; @@ -1158,8 +1157,8 @@ struct S_IntegerEnum final : ISerializer { code.emplace_back(origin, Op::_Call, ir_ith_eq, std::vector{ir_num[0], ir_ith_value[0]}, lookup_function("_==_")); code.emplace_back(origin, Op::_Call, std::vector{ir_any_of}, std::vector{ir_any_of, ir_ith_eq[0]}, lookup_function("_|_")); } - std::vector args_assert = { code.create_int(origin, 5, "(excno)"), ir_any_of, code.create_int(origin, 0, "") }; - Op& op_assert = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); + std::vector args_throwifnot = { code.create_int(origin, 5, "(excno)"), ir_any_of }; + Op& op_assert = code.emplace_back(origin, Op::_Call, std::vector{}, std::move(args_throwifnot), lookup_function("__throw_ifnot")); op_assert.set_impure_flag(); } return ir_num; diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 29a0b99bd2..c158c13733 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -2119,9 +2119,8 @@ static void process_assert_statement(V v, CodeBlob& code) std::vector ir_cond = pre_compile_expr(v->get_cond(), code); tolk_assert(ir_cond.size() == 1 && ir_thrown_code.size() == 1); - std::vector args_vars = { ir_thrown_code[0], ir_cond[0], code.create_int(v, 0, "(assert-0)") }; - FunctionPtr builtin_sym = lookup_function("__throw_if_unless"); - gen_op_call(code, TypeDataVoid::create(), v, std::move(args_vars), builtin_sym, "(throw-call)"); + std::vector args_throwifnot = { ir_thrown_code[0], ir_cond[0] }; + gen_op_call(code, TypeDataVoid::create(), v, std::move(args_throwifnot), lookup_function("__throw_ifnot"), "(throw-call)"); } static void process_catch_variable(AnyExprV v_catch_var, CodeBlob& code) { diff --git a/tolk/tolk.h b/tolk/tolk.h index 4964525deb..cef4baa5ea 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -893,14 +893,17 @@ struct Optimizer { bool detect_rewrite_SWAP_symmetric(); bool detect_rewrite_SWAP_PUSH_STUR(); bool detect_rewrite_SWAP_STxxxR(); - bool detect_rewrite_NOT_THROWIF(); + bool detect_rewrite_BOOLNOT_THROWIF(); + bool detect_rewrite_0EQINT_THROWIF(); bool detect_rewrite_DICTSETB_DICTSET(); + bool detect_rewrite_DICTGET_NULLSWAPIFNOT_THROWIFNOT(); bool detect_rewrite_ENDC_CTOS(); bool detect_rewrite_ENDC_HASHCU(); bool detect_rewrite_NEWC_BTOS(); bool detect_rewrite_NEWC_STSLICECONST_BTOS(); bool detect_rewrite_NEWC_ENDC_CTOS(); bool detect_rewrite_NEWC_ENDC(); + bool replace_BOOLNOT_to_NOT(); AsmOpConsList extract_code(); }; From 7080a8ef70c406f4bf44df93ff66acf16a54db85 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Wed, 29 Oct 2025 17:46:00 +0300 Subject: [PATCH 455/524] [Tolk] Peephole: 0 EQINT + NOT -> 0 NEQINT and other xxx + NOT inversions --- tolk-tester/tests/null-keyword.tolk | 12 +++++ tolk-tester/tests/some-tests-1.tolk | 44 +++++++++++++++ tolk-tester/tests/unbalanced-ret-loops.tolk | 2 +- tolk/optimize.cpp | 60 +++++++++++++++++++++ tolk/pipe-ast-to-legacy.cpp | 42 ++++----------- tolk/pipe-optimize-boolean-expr.cpp | 6 +++ tolk/tolk.h | 1 + 7 files changed, 133 insertions(+), 34 deletions(-) diff --git a/tolk-tester/tests/null-keyword.tolk b/tolk-tester/tests/null-keyword.tolk index c21da1d0e8..51e0539ee9 100644 --- a/tolk-tester/tests/null-keyword.tolk +++ b/tolk-tester/tests/null-keyword.tolk @@ -111,6 +111,18 @@ fun main() { CAR // h numbers '13 """ +@fif_codegen +""" + UNTIL:<{ + s0 s2 XCHG // t h numbers + UNCONS // t h num numbers + s3 s3 XCHG2 // numbers h t num + TPUSH // numbers h t + s2 PUSH // numbers h t numbers + ISNULL // numbers h t '28 + }> // numbers h t +""" + @fif_codegen """ main() PROC:<{ diff --git a/tolk-tester/tests/some-tests-1.tolk b/tolk-tester/tests/some-tests-1.tolk index 42bdc773a1..471e668708 100644 --- a/tolk-tester/tests/some-tests-1.tolk +++ b/tolk-tester/tests/some-tests-1.tolk @@ -94,6 +94,42 @@ fun test9(s: int) { return ~ t; } +@method_id(110) +fun test10(x: int) { + var t = createEmptyTuple(); + + do { + t.push(x); + x += 1; + } while (x < -4); + + var y = x; + do { + t.push(y); + y -= 1; + } while (y > -8); + + y = 804; + do { + y -= 1; + t.push(y); + } while (y > 800); + + y = -804; + do { + y += 1; + t.push(y); + } while (y < -800); + + y = 130; + do { + y -= 1; + t.push(y); + } while (y > 127); + + return t; +} + /** method_id | in | out @@ -124,4 +160,12 @@ fun test9(s: int) { @testcase | 108 | -5 | 2 @testcase | 109 | 1 | -2 @testcase | 109 | 5 | -6 +@testcase | 110 | -6 | [ -6 -5 -4 -5 -6 -7 803 802 801 800 -803 -802 -801 -800 129 128 127 ] + +Check that it's not replaced to `128 LESSINT` +@fif_codegen +""" + 127 GTINT + NOT +""" */ diff --git a/tolk-tester/tests/unbalanced-ret-loops.tolk b/tolk-tester/tests/unbalanced-ret-loops.tolk index 292b48dac4..0225225f43 100644 --- a/tolk-tester/tests/unbalanced-ret-loops.tolk +++ b/tolk-tester/tests/unbalanced-ret-loops.tolk @@ -64,5 +64,5 @@ fun test4(x: int): (int, bool) { @testcase | 3 | -5 | -1 @testcase | 4 | 18 | 9 -1 -@code_hash 12359153928622198176298534554187062238616102949658930329300859312625793323482 +@code_hash 85513835024916823013161688704471708147394737211040277463082469481721767680515 */ diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index 7c55baf17c..a8bdc2549c 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -626,6 +626,65 @@ bool Optimizer::detect_rewrite_NEWC_ENDC() { return true; } +// pattern `0 EQINT` + `NOT` -> `0 NEQINT` and other (mathematical operations + NOT), like `!(a >= 4)` -> `a < 4`; +// since the first is boolean (-1 or 0), NOT will invert it, there are no occasions with bitwise integers; +// it's especially helpful to invert condition of `do while` for TVM `UNTIL` +bool Optimizer::detect_rewrite_xxx_NOT() { + bool second_not = pb_ >= 2 && op_[1]->is_custom() && op_[1]->op == "NOT"; + if (!second_not || !op_[0]->is_custom()) { + return false; + } + + static const char* ends_with[] = {" EQINT", " NEQINT"}; + static const char* repl_with[] = {" NEQINT", " EQINT"}; + static const char* equl_to[] = {"NEQ", "EQUAL", "LESS", "GEQ", "GREATER", "LEQ"}; + static const char* repl_to[] = {"EQUAL", "NEQ", "GEQ", "LESS", "LEQ", "GREATER"}; + + std::string_view f = op_[0]->op; + for (size_t i = 0; i < std::size(ends_with); ++i) { + if (f.ends_with(ends_with[i])) { + p_ = 2; + q_ = 1; + std::string new_op = op_[0]->op.substr(0, f.rfind(' ')) + repl_with[i]; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, new_op, 0, 1)); + return true; + } + } + for (size_t i = 0; i < std::size(equl_to); ++i) { + if (f == equl_to[i]) { + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, repl_to[i], 2, 1)); + return true; + } + } + // `!(a > 7)` -> `a <= 7` -> `a < 8` (but `GTINT` instead of `GREATER` for small numbers) + // `7 GTINT` + `NOT` -> `8 LESSINT` (there is no `LEINT` instruction) + // `8 LESSINT` + `NOT` -> `7 GTINT` + if (f.ends_with(" GTINT") || f.ends_with(" LESSINT")) { + bool is_gtint = f.ends_with(" GTINT"); + std::string s_number(f.substr(0, f.rfind(' '))); + td::RefInt256 number = td::string_to_int256(s_number); + number += is_gtint ? 1 : -1; + + if (number > -127 && number < 127) { + p_ = 2; + q_ = 1; + std::string new_op = number->to_dec_string() + (is_gtint ? " LESSINT" : " GTINT"); + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->origin, new_op, 2, 1)); + return true; + } + } + // `NOT` + `NOT` -> nothing (it's valid for integers also) + if (f == "NOT") { + p_ = 2; + q_ = 0; + return true; + } + + return false; +} + // for `!x`, when `x` is boolean, a fake asm instruction `BOOLNOT` was inserted (see builtins.cpp); // it was used for peephole optimizations, because `NOT + ...` is not correct, since `NOT` is bitwise; // here we replace instructions left after optimizations with a simple `NOT` (-1 => 0, 0 => -1) @@ -1068,6 +1127,7 @@ bool Optimizer::find_at_least(int pb) { detect_rewrite_ENDC_CTOS() || detect_rewrite_ENDC_HASHCU() || detect_rewrite_NEWC_BTOS() || detect_rewrite_NEWC_STSLICECONST_BTOS() || detect_rewrite_NEWC_ENDC_CTOS() || detect_rewrite_NEWC_ENDC() || + detect_rewrite_xxx_NOT() || (!(mode_ & 1) && replace_BOOLNOT_to_NOT()) || (!(mode_ & 1) && ((is_rot() && rewrite(AsmOp::Custom(origin, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(origin, "-ROT", 3, 3))) || diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index c158c13733..89aa58c509 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -2183,7 +2183,7 @@ static void process_if_statement(V v, CodeBlob& code) { code.emplace_back(v->get_else_body(), Op::_Return); } code.close_pop_cur(v->get_else_body()); - if (v->is_ifnot) { + if (v->is_ifnot) { // pre-optimized to generate IFNOT instead of IF std::swap(if_op.block0, if_op.block1); } } @@ -2193,40 +2193,16 @@ static void process_do_while_statement(V v, CodeBlob& co code.push_set_cur(until_op.block0); process_any_statement(v->get_body(), code); - // in TVM, there is only "do until", but in Tolk, we want "do while" - // here we negate condition to pass it forward to legacy to Op::_Until - // also, handle common situations as a hardcoded "optimization": replace (a<0) with (a>=0) and so on - // todo these hardcoded conditions should be removed from this place in the future AnyExprV cond = v->get_cond(); - AnyExprV until_cond; - if (auto v_not = cond->try_as(); v_not && v_not->tok == tok_logical_not) { - until_cond = v_not->get_rhs(); - } else if (auto v_eq = cond->try_as(); v_eq && v_eq->tok == tok_eq) { - until_cond = createV(cond->range, v_eq->operator_range, "!=", tok_neq, v_eq->get_lhs(), v_eq->get_rhs()); - } else if (auto v_neq = cond->try_as(); v_neq && v_neq->tok == tok_neq) { - until_cond = createV(cond->range, v_neq->operator_range, "==", tok_eq, v_neq->get_lhs(), v_neq->get_rhs()); - } else if (auto v_leq = cond->try_as(); v_leq && v_leq->tok == tok_leq) { - until_cond = createV(cond->range, v_leq->operator_range, ">", tok_gt, v_leq->get_lhs(), v_leq->get_rhs()); - } else if (auto v_lt = cond->try_as(); v_lt && v_lt->tok == tok_lt) { - until_cond = createV(cond->range, v_lt->operator_range, ">=", tok_geq, v_lt->get_lhs(), v_lt->get_rhs()); - } else if (auto v_geq = cond->try_as(); v_geq && v_geq->tok == tok_geq) { - until_cond = createV(cond->range, v_geq->operator_range, "<", tok_lt, v_geq->get_lhs(), v_geq->get_rhs()); - } else if (auto v_gt = cond->try_as(); v_gt && v_gt->tok == tok_gt) { - until_cond = createV(cond->range, v_gt->operator_range, "<=", tok_geq, v_gt->get_lhs(), v_gt->get_rhs()); - } else if (cond->inferred_type->unwrap_alias() == TypeDataBool::create()) { - until_cond = createV(cond->range, cond->range, "!b", tok_logical_not, cond); - } else { - until_cond = createV(cond->range, cond->range, "!", tok_logical_not, cond); - } - until_cond->mutate()->assign_inferred_type(TypeDataInt::create()); - if (auto v_bin = until_cond->try_as(); v_bin && !v_bin->fun_ref) { - v_bin->mutate()->assign_fun_ref(lookup_function("_" + static_cast(v_bin->operator_name) + "_")); - } else if (auto v_un = until_cond->try_as(); v_un && !v_un->fun_ref) { - v_un->mutate()->assign_fun_ref(lookup_function(static_cast(v_un->operator_name) + "_")); - } + std::vector ir_cond = pre_compile_expr(cond, code, nullptr); + tolk_assert(ir_cond.size() == 1); - until_op.left = pre_compile_expr(until_cond, code, nullptr); - tolk_assert(until_op.left.size() == 1); + // in TVM, there is only "do until", but in Tolk, we want "do while"; so, negate the condition; + // optimizations like `while (a > 0)` -> `until (!(a > 0))` -> `until (a < 1)` are implemented as peephole + FunctionPtr f_not = cond->inferred_type->equal_to(TypeDataBool::create()) ? lookup_function("!b_") : lookup_function("!_"); + std::vector ir_until = code.create_tmp_var(TypeDataInt::create(), cond, "(until-cond)"); + code.emplace_back(cond, Op::_Call, ir_until, std::move(ir_cond), f_not); + until_op.left = std::move(ir_until); code.close_pop_cur(v->get_body()); } diff --git a/tolk/pipe-optimize-boolean-expr.cpp b/tolk/pipe-optimize-boolean-expr.cpp index 62538527b4..323b3e9dff 100644 --- a/tolk/pipe-optimize-boolean-expr.cpp +++ b/tolk/pipe-optimize-boolean-expr.cpp @@ -132,6 +132,12 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { AnyV replace(V v) override { parent::replace(v); + // here we optimize common conditions to generate IFNOT instead of IF; + // obviously, this should be done later, around peephole optimizer, + // but with current implementation of stack transformations (inherited from FunC) we have no chance, + // so the best for now is to do some AST-based condition rewrites; + // in the future, I'll fully rewrite optimizer, and this part will be removed + // `if (!x)` -> ifnot(x) while (auto v_cond_unary = v->get_cond()->try_as()) { if (v_cond_unary->tok != tok_logical_not) { diff --git a/tolk/tolk.h b/tolk/tolk.h index cef4baa5ea..5474397d5c 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -903,6 +903,7 @@ struct Optimizer { bool detect_rewrite_NEWC_STSLICECONST_BTOS(); bool detect_rewrite_NEWC_ENDC_CTOS(); bool detect_rewrite_NEWC_ENDC(); + bool detect_rewrite_xxx_NOT(); bool replace_BOOLNOT_to_NOT(); AsmOpConsList extract_code(); From e376af643f8ece06edb5471e94143eddd5f1b03c Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 30 Oct 2025 17:56:51 +0300 Subject: [PATCH 456/524] [Tolk] Compile-time constant expressions for initializers and enum members Do a lot of refactor about evaluating constants. Now it's done at compile time. `2 + 3` replaced with `5` not by an IR optimizer, but via AST calculation. Every const/enum/param default can now be evaluated into ConstValExpression and later exported into ABI. As a consequence, enum members can have defaults of any complexity (earlier they were restricted to plain integers and a couple of operators). For instance, stringCrc32() and ton(), as well as `coins` serialization, is now supported in enums. Evaluated constants are cached, so kludges around exponential explosion, as well as forcing Fift location, were completely removed. --- tolk-tester/tests/comments-tests.tolk | 61 +++ tolk-tester/tests/constants-tests.tolk | 130 +++++- tolk-tester/tests/enums-tests.tolk | 14 + .../tests/invalid-declaration/err-1039.tolk | 2 +- .../tests/invalid-declaration/err-1697.tolk | 7 - tolk-tester/tests/match-by-expr-tests.tolk | 4 +- tolk-tester/tests/pack-unpack-3.tolk | 2 +- tolk-tester/tests/pack-unpack-8.tolk | 42 ++ tolk-tester/tests/some-tests-4.tolk | 4 +- tolk-tester/tests/use-before-declare.tolk | 2 +- tolk/CMakeLists.txt | 2 +- tolk/ast-aux-data.h | 10 - tolk/ast-from-tokens.cpp | 10 + tolk/ast.cpp | 4 - tolk/ast.h | 6 +- tolk/constant-evaluator.cpp | 403 ++++++++++++------ tolk/constant-evaluator.h | 63 ++- tolk/pack-unpack-serializers.cpp | 23 +- tolk/pipe-ast-to-legacy.cpp | 149 ++++--- tolk/pipe-check-const-expr.cpp | 97 +++++ tolk/pipe-check-inferred-types.cpp | 10 +- tolk/pipe-check-serialized-fields.cpp | 2 +- tolk/pipe-constant-folding.cpp | 216 ---------- tolk/pipe-optimize-boolean-expr.cpp | 17 +- tolk/pipe-register-symbols.cpp | 2 +- tolk/pipeline.h | 2 +- tolk/symtable.h | 4 +- tolk/tolk.cpp | 2 +- tolk/tolk.h | 13 - tolk/type-system.cpp | 2 +- 30 files changed, 817 insertions(+), 488 deletions(-) delete mode 100644 tolk-tester/tests/invalid-declaration/err-1697.tolk create mode 100644 tolk/pipe-check-const-expr.cpp delete mode 100644 tolk/pipe-constant-folding.cpp diff --git a/tolk-tester/tests/comments-tests.tolk b/tolk-tester/tests/comments-tests.tolk index 8287808063..0c577674fd 100644 --- a/tolk-tester/tests/comments-tests.tolk +++ b/tolk-tester/tests/comments-tests.tolk @@ -17,13 +17,46 @@ fun demo_fields_def(x: int) { return (w.f4 + 5, w.f3, w.f2, w.f1); } +@noinline +fun add(x: int, y: int = 0) { + return x + y +} + +fun demo_call_def(x: int) { + var x2 = x * x; + return add(x2); +} + +struct MyOptions { + calcAbs: bool = true + coeff: int +} + +@noinline +fun some_calc( + x: int, + options: MyOptions = { coeff: 2 }, +) { + if (options.calcAbs) { + x = abs(x); + } + return x * options.coeff; +} + +@method_id(101) +fun call_some_calc() { + return some_calc(-10); +} + fun main() { var cb = demo_fields_def; + demo_call_def; return 1; } /** @testcase | 0 | | 1 +@testcase | 101 | | 20 @fif_codegen_enable_comments @fif_codegen @@ -31,6 +64,7 @@ fun main() { demo_fields_def() PROC:<{ // x // 13: f1: x + C_20 20 ADDCONST // '6 + // 4: f2: int = 10 10 PUSHINT // '6 '7=10 // 14: f3: 100 100 PUSHINT // w.f1 w.f2=10 w.f3=100 @@ -40,4 +74,31 @@ fun main() { }> """ +@fif_codegen +""" + demo_call_def() PROC:<{ // x + // 26: var x2 = x * x + DUP // x x + MUL // x2 + // 21: fun add(x: int, y: int = 0) + 0 PUSHINT // x2 '3=0 + // 27: return add(x2) + add() CALLDICT // '4 + }> +""" + +@fif_codegen +""" + call_some_calc() PROC:<{ // + // 48: return some_calc(-10) + -10 PUSHINT // '0=-10 + // 31: calcAbs: bool = true + TRUE // '0=-10 '1 + // 38: options: MyOptions = { coeff: 2 } + 2 PUSHINT // '0=-10 '1 '2=2 + // 48: return some_calc(-10) + some_calc() CALLDICT // '3 + }> +""" + */ \ No newline at end of file diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index 6f7c993b7d..52e6255a24 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -19,8 +19,8 @@ const str2int = 0xAABBCC; const nibbles: int = 4; -const strange_zero = (!10 as int); -const strange_minus_1: MInt = (!0 as int); +const strange_zero = 1 >> 10; +const strange_minus_1: MInt = -1 as Color as int; const addr1 = address("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"); const addr2: any_address = address("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF") as any_address; @@ -49,6 +49,57 @@ fun sget2r(): slice { return str2r; } const int240: int = ((int1+int2)*10)<<3; +enum Color { + red, + green, + blue, +} + +const boolOpsWithInts = (10 < 3, 10 <= 3, 10 > 3, 10 >= 3, 10 == 3, 10 != 3, 10 <= 10, 10 >= (9 + 1), ) +const boolOpsWithBools = (true == true, true != true, false == false, !!false != !!false) +const boolBitwise = (true & true, true & false, false & true, false & false, true | false, true ^ true, true ^ false) +const intBitwise = (2 & 3, 2 ^ 3, 2 | 3, 1 | 5, 0xFF & 0x80, !0, !100) +const alsoBlue = Color.blue +const also2 = Color.blue as int + +const nullable5: int? = 5 +const nullableTensor: (int, int)? = null +const nullableTricky: (UnpackOptions?, int?, (int, slice)?) = (null, null, null) +const unionTricky: (address | int, address | int, int | int8 | null, int8 | int16 | null) = (address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"), 100, 50, null); + +struct SubParams { + sub1: bool = true + sub2: int8 = 8 + sub3: coins +} + +struct MyOptions { + allowPenalty: bool = false + closeTo: (int8, int8)? = null + sn: never + sub: SubParams = { sub3: ton("0.05") } + coeff: int16 = 2 +} + +const opts1 = MyOptions { allowPenalty: true } +const opts2 = MyOptions { coeff: 3, sub: { sub1: false, sub3: 0 } } +const opts3: MyOptions = { coeff: 0, closeTo: null, allowPenalty: true } +const opts4: MyOptions = { sub: { sub3: ton("1") }, closeTo: (3, 4), coeff: 55 } + +@noinline +fun MyOptions.equals(self, rhs: MyOptions = {}) { + return self.toCell().hash() == rhs.toCell().hash() +} + +struct Wrapper { + item: T +} + +const wOneTon = Wrapper { item: ton("1") } +const wNoTon: Wrapper = { item: 0 } +const wNoSubParams: Wrapper? = null + + fun iget240(): MInt { return int240; } @pure @@ -97,6 +148,67 @@ fun test6() { return (addr1 == addr1, addr1 != addr1, addr1 == createAddressNone(), addr1.getWorkchain(), addr1 == addr2); } +@method_id(107) +fun test7() { + return boolOpsWithInts +} + +@method_id(108) +fun test8() { + return boolOpsWithBools +} + +@method_id(109) +fun test9() { + return boolBitwise +} + +@method_id(110) +fun test10() { + return intBitwise +} + +@method_id(111) +fun test11() { + __expect_type(alsoBlue, "Color"); + __expect_type(also2, "int"); + return (alsoBlue, also2) +} + +@method_id(112) +fun test12() { + return nullableTensor +} + +@method_id(113) +fun test13() { + return nullableTricky +} + +@method_id(114) +fun test14() { + return (unionTricky.0 is address, unionTricky.1 is address, unionTricky.1 is int, unionTricky.1 is slice, unionTricky.1, 777, unionTricky.2, unionTricky.3) +} + +@method_id(115) +fun test15() { + return ( + opts1.equals(), opts1.equals(opts1), opts1.equals(opts2), opts1.equals(opts3), opts1.equals(opts4), + opts4.equals(), opts4.equals(opts1), opts4.equals(opts2), opts4.equals(opts3), opts4.equals(opts4), + 777, + opts1.equals({ closeTo: null, allowPenalty: true }), + opts2.equals({ coeff: 3, sub: { sub1: false, sub3: 0 } }), + opts3.equals({ closeTo: null, coeff: 0, allowPenalty: true }), + opts4.equals({ sub: { sub3: ton("1") }, coeff: 55, closeTo: (3, 4) }) + ) +} + +@method_id(116) +fun test16() { + __expect_type(wOneTon, "Wrapper"); + return (wOneTon, wNoTon, wNoSubParams) +} + fun main() { var i1: int = iget1(); var i2: int = iget2(); @@ -127,6 +239,16 @@ fun main() { @testcase | 104 | | 1 1 2 @testcase | 105 | | -1 0 7 48 @testcase | 106 | | -1 0 0 -1 -1 - -@code_hash 50461262039225352661292504472979593396591696467874442618206926482074675176667 +@testcase | 107 | | 0 0 -1 -1 0 -1 -1 -1 +@testcase | 108 | | -1 0 -1 0 +@testcase | 109 | | -1 0 0 0 -1 0 -1 +@testcase | 110 | | 2 1 3 5 128 -1 0 +@testcase | 111 | | 2 2 +@testcase | 112 | | (null) (null) 0 +@testcase | 113 | | (null) (null) 0 (null) (null) (null) 0 +@testcase | 114 | | -1 0 -1 0 100 1 777 50 1 (null) 0 +@testcase | 115 | | 0 -1 0 0 0 0 0 0 0 -1 777 -1 -1 -1 -1 +@testcase | 116 | | 1000000000 0 (null) (null) (null) 0 + +@code_hash 62183191459178893174419204771807638858032941060155787749136667785942705596518 */ diff --git a/tolk-tester/tests/enums-tests.tolk b/tolk-tester/tests/enums-tests.tolk index c61faada2b..e3d8903ce2 100644 --- a/tolk-tester/tests/enums-tests.tolk +++ b/tolk-tester/tests/enums-tests.tolk @@ -8,6 +8,13 @@ enum Color { Blue } +enum InvertedColor { + Red = Color.Blue, + Green = Color.Green, + Blue = Color.Red, + OneTon = ton("1"), +} + type ColorAlias = Color struct WithColor { @@ -242,6 +249,12 @@ fun test13(x: int) { } } +@method_id(114) +fun test14(x: InvertedColor) { + __expect_type(InvertedColor.Red, "InvertedColor"); + return (InvertedColor.Red, InvertedColor.Green, InvertedColor.Blue, InvertedColor.OneTon, x == InvertedColor.Red); +} + fun main(c: Color) { __expect_type(c, "Color"); @@ -276,6 +289,7 @@ fun main(c: Color) { @testcase | 113 | 111 | 257 0 @testcase | 113 | 10 | 258 0 @testcase | 113 | 50 | 0 0 +@testcase | 114 | 2 | 2 1 0 1000000000 -1 @fif_codegen """ diff --git a/tolk-tester/tests/invalid-declaration/err-1039.tolk b/tolk-tester/tests/invalid-declaration/err-1039.tolk index 52143ab754..c3f5462ede 100644 --- a/tolk-tester/tests/invalid-declaration/err-1039.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1039.tolk @@ -3,5 +3,5 @@ enum Color { Red = Color.Red } /** @compilation_should_fail -@stderr referencing other members in initializers is not allowed yet +@stderr enum `Color` initializers circularly references itself */ diff --git a/tolk-tester/tests/invalid-declaration/err-1697.tolk b/tolk-tester/tests/invalid-declaration/err-1697.tolk deleted file mode 100644 index cc8a904327..0000000000 --- a/tolk-tester/tests/invalid-declaration/err-1697.tolk +++ /dev/null @@ -1,7 +0,0 @@ - -enum Color { Red = ton("0.05") as int } - -/** -@compilation_should_fail -@stderr unsupported operation in `enum` member value - */ diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk index 588632051d..de748cfc26 100644 --- a/tolk-tester/tests/match-by-expr-tests.tolk +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -189,12 +189,12 @@ fun main() { 100 PUSHINT // '1=100 }>ELSE<{ // x DUP // x x - 2 EQINT // x '8 + 2 EQINT // x '6 IF:<{ // x DROP // 200 PUSHINT // '1=200 }>ELSE<{ // x - 3 EQINT // '13 + 3 EQINT // '11 IF:<{ // 300 PUSHINT // '1=300 }>ELSE<{ // diff --git a/tolk-tester/tests/pack-unpack-3.tolk b/tolk-tester/tests/pack-unpack-3.tolk index c83790f0a7..ac6f633d8c 100644 --- a/tolk-tester/tests/pack-unpack-3.tolk +++ b/tolk-tester/tests/pack-unpack-3.tolk @@ -249,7 +249,7 @@ fun test_MsgExternal1() { run(SayHiAndGoodbye{ dest_addr: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), body: BodyPayload1 { should_forward: false, n_times: 85, content: generateSlice_44_with_ref45() } }, stringHexToSlice("8980000000000000000000000000000000000000000000000000000000000000000004000000AA00000059_").appendRef(stringHexToSlice("0000002D"))); run(SayHiAndGoodbye{ dest_addr: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), body: BodyPayload2 { master_id: -5, owner_address: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi") } }, stringHexToSlice("899FE51D0067B628CF959D5162B1E1EC3CD05BC0DD14B28EA099209E3EE07A57C9B3CFDCFF28E8033DB1467CACEA8B158F0F61E682DE06E8A5947504C904F1F703D2BE4D9E")); run(SayStoreInChain{ in_masterchain: true, contents: BodyPayload1{ should_forward: true, n_times: 20, content: generateSlice_44_with_ref45() }.toCell() as cell as Cell }, stringHexToSlice("0013C_").appendRef(stringHexToSlice("3000000140000002C").appendRef(stringHexToSlice("0000002D")))); - run(SayStoreInChain{ in_masterchain: false, contents: BodyPayload2{ master_id: 37, owner_address: address(("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c")) }.toCell() as cell as Cell }, stringHexToSlice("00134_").appendRef(stringHexToSlice("4960000000000000000000000000000000000000000000000000000000000000000004_"))); + run(SayStoreInChain{ in_masterchain: false, contents: BodyPayload2{ master_id: 37, owner_address: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c") }.toCell() as cell as Cell }, stringHexToSlice("00134_").appendRef(stringHexToSlice("4960000000000000000000000000000000000000000000000000000000000000000004_"))); val o = MsgExternal1.fromSlice(stringHexToSlice("00134_").appendRef(stringHexToSlice("4960000000000000000000000000000000000000000000000000000000000000000004_"))); assert(o is SayStoreInChain, 400); diff --git a/tolk-tester/tests/pack-unpack-8.tolk b/tolk-tester/tests/pack-unpack-8.tolk index 1cfbce98da..16b24318c0 100644 --- a/tolk-tester/tests/pack-unpack-8.tolk +++ b/tolk-tester/tests/pack-unpack-8.tolk @@ -54,6 +54,20 @@ enum EncodedVari : varint16 { MANY = 1<<100, } +enum EOfCrc32: uint32 { + hello = stringCrc32("hello") + world = stringCrc32("world") +} + +enum Amounts: coins { + zero = ton("0"), + nanoton = 1, + small = ton("0.0001"), + more = ton("0.05"), + more1, + one = ton("1"), +} + @method_id(101) fun test1() { @@ -149,6 +163,32 @@ fun test12() { return (c3_one, c3_one == EncodedVari.MANY, c3_many == EncodedVari.MANY); } +@method_id(113) +fun test13() { + return ( + EOfCrc32.world.toCell().beginParse().loadUint(32), + EOfCrc32.fromSlice(stringHexToSlice("3610a686")) == EOfCrc32.hello + ) +} + +@method_id(114) +fun test14() { + var ex = 0; + try { ex = Amounts.fromSlice(stringHexToSlice("32dc6c08_")) as int } + catch (no) { ex = no } + return ( + Amounts.one.toCell().hash() == 101872335586569371892273454115677983665588664548247484580448471612407076042677, + Amounts.more.toCell().hash() == 82789494441528276360388239783934598298305390626975708015812230018243378632521, + beginCell().storeAny(Amounts.zero).bitsCount(), + beginCell().storeAny(Amounts.more).bitsCount(), + Amounts.fromSlice(stringHexToSlice("402faf080")) as int64, + Amounts.fromSlice(stringHexToSlice("402faf080")) as coins == ton("0.05"), + Amounts.fromSlice(stringHexToSlice("402faf080")) == Amounts.more, + Amounts.more1, + ex, + ) +} + fun main() { } @@ -174,6 +214,8 @@ fun main() { @testcase | 110 | | -1 -1 -1 -1 0 @testcase | 111 | | 0 0 0 1 -1 0 0 -1 @testcase | 112 | | 1 0 -1 +@testcase | 113 | | 980881731 -1 +@testcase | 114 | | -1 -1 4 36 50000000 -1 -1 50000001 5 @fif_codegen """ diff --git a/tolk-tester/tests/some-tests-4.tolk b/tolk-tester/tests/some-tests-4.tolk index 74645e15e4..e6f6a070c9 100644 --- a/tolk-tester/tests/some-tests-4.tolk +++ b/tolk-tester/tests/some-tests-4.tolk @@ -102,9 +102,9 @@ fun main() { """ test2() PROC:<{ IFJMP:<{ - 305482242253527345283975278648594775678 PUSHINT // '62 + 305482242253527345283975278648594775678 PUSHINT // '1=305482242253527345283975278648594775678 }> - 305482242253527345283975278648594775678 PUSHINT // '124 + 305482242253527345283975278648594775678 PUSHINT // '2=305482242253527345283975278648594775678 }> """ */ diff --git a/tolk-tester/tests/use-before-declare.tolk b/tolk-tester/tests/use-before-declare.tolk index d5ecd40701..0fed08b942 100644 --- a/tolk-tester/tests/use-before-declare.tolk +++ b/tolk-tester/tests/use-before-declare.tolk @@ -58,7 +58,7 @@ const first = 1; @fif_codegen """ test2() PROC:<{ - 2 PUSHINT // '2 + 2 PUSHINT // '0=2 }> """ */ diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index a2404eaec0..a9fcc3d809 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -23,8 +23,8 @@ set(TOLK_SOURCE pipe-check-rvalue-lvalue.cpp pipe-check-private-fields.cpp pipe-check-pure-impure.cpp + pipe-check-const-expr.cpp pipe-mini-borrow-checker.cpp - pipe-constant-folding.cpp pipe-optimize-boolean-expr.cpp pipe-detect-inline-in-place.cpp pipe-check-serialized-fields.cpp diff --git a/tolk/ast-aux-data.h b/tolk/ast-aux-data.h index 1f97771281..8d7626b86e 100644 --- a/tolk/ast-aux-data.h +++ b/tolk/ast-aux-data.h @@ -28,16 +28,6 @@ namespace tolk { struct CodeBlob; -// AuxData_ForceFiftLocation is created when transforming AST to IR; -// it wraps constants to force codegen location point to usage, not to init_val AST nodes -struct AuxData_ForceFiftLocation final : ASTAuxData { - AnyV forced_origin; - - explicit AuxData_ForceFiftLocation(AnyV forced_origin) - : forced_origin(forced_origin) { - } -}; - // AuxData_LazyObjectLoadFields is a special auto-inserted vertex to load fields of a lazy struct; // example: `var p = lazy Point.fromSlice(s); aux "load x"; return p.x` struct AuxData_LazyObjectLoadFields final : ASTAuxData { diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 5e14fc76d7..2f57573677 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -1033,6 +1033,16 @@ static AnyExprV parse_expr75(Lexer& lex) { lex.next(); AnyExprV rhs = parse_expr75(lex); range.end(rhs->range); + + // convert `-1` to `int(-1)`, not to a tree `unary(-) > int(1)` right here + if (auto rhs_int = rhs->try_as(); rhs_int && (t == tok_minus || t == tok_plus)) { + td::RefInt256 intval = rhs_int->intval; + tolk_assert(!intval.is_null()); + if (t == tok_minus) { + intval = -intval; // negation (and multiple consecutive negations) always fits 257 bits if originally fits + } + return createV(range, std::move(intval), rhs_int->orig_str); + } return createV(range, operator_range, operator_name, t, rhs); } if (t == tok_double_minus || t == tok_double_plus) { diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 0dc8d14533..c74039e186 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -138,10 +138,6 @@ void Vertex::assign_sym(const Symbol* sym) { this->sym = sym; } -void Vertex::assign_literal_value(std::string&& literal_value) { - this->literal_value = std::move(literal_value); -} - void Vertex::assign_fun_ref(FunctionPtr fun_ref, bool dot_obj_is_self) { this->fun_maybe = fun_ref; this->dot_obj_is_self = dot_obj_is_self; diff --git a/tolk/ast.h b/tolk/ast.h index 457115496d..e143291118 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -517,7 +517,7 @@ struct Vertex final : ASTExprUnary { template<> // ast_artificial_aux_vertex is a compiler-inserted vertex that can't occur in source code -// example: special vertex to force location in Fift output at constant usage +// example: implicitly inserted loads after `lazy` operator // example: `msg.isBounced` / `msg.xxx` in onInternalMessage are handled specially struct Vertex final : ASTExprUnary { const ASTAuxData* aux_data; // custom payload, see ast-aux-data.h @@ -629,7 +629,7 @@ template<> // note, that `-1` is unary minus of `1` int const struct Vertex final : ASTExprLeaf { td::RefInt256 intval; // parsed value, 255 for "0xFF" - std::string_view orig_str; // original "0xFF"; empty for nodes generated by compiler (e.g. in constant folding) + std::string_view orig_str; // original "0xFF"; empty for nodes generated by compiler Vertex(SrcRange range, td::RefInt256 intval, std::string_view orig_str) : ASTExprLeaf(ast_int_const, range) @@ -643,10 +643,8 @@ template<> // note, that TVM doesn't have strings, it has only slices, so "hello" has type slice struct Vertex final : ASTExprLeaf { std::string_view str_val; - std::string literal_value; // when "some_str" is a standalone string, value of type `slice`, for x{...} Fift output Vertex* mutate() const { return const_cast(this); } - void assign_literal_value(std::string&& literal_value); Vertex(SrcRange range, std::string_view str_val) : ASTExprLeaf(ast_string_const, range) diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index 924760a6f8..4ee50a080c 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -22,6 +22,7 @@ #include "crypto/common/util.h" #include "td/utils/crypto.h" #include "ton/ton-types.h" +#include namespace tolk { @@ -29,6 +30,12 @@ static Error err_const_string_required(std::string_view f_name, std::string_view return err("function `{}` requires a constant string, like `{}(\"{}\")`", f_name, f_name, example_arg); } +static Error err_not_a_constant_expression() { + return err("not a constant expression"); +} + +static std::unordered_map computed_constants_cache; + // parse address like "EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5" // based on unpack_std_smc_addr() from block.cpp @@ -88,7 +95,7 @@ static bool parse_raw_address(std::string_view acc_string, int& workchain, ton:: return true; } -static void parse_any_std_address(std::string_view str, SrcRange range, unsigned char (*data)[3 + 8 + 256]) { +static void parse_any_std_address(std::string_view str, SrcRange range, unsigned char (*data)[267]) { ton::WorkchainId workchain; ton::StdSmcAddress addr; bool correct = (str.size() == 48 && parse_friendly_address(str.data(), workchain, addr)) || @@ -159,7 +166,7 @@ static td::RefInt256 parse_nanotons_as_floating_string(SrcRange range, std::stri // given `ton("0.05")` evaluate it to 50000000 // given `stringCrc32("some_str")` evaluate it // etc. -static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(V v, std::string_view f_name) { +static ConstValExpression parse_vertex_call_to_compile_time_function(V v, std::string_view f_name) { // most functions accept 1 argument, but static compile-time methods like `MyStruct.getDeclaredPackPrefix()` have 0 args if (v->get_num_args() == 0) { TypePtr receiver = v->fun_maybe->receiver_type; @@ -170,7 +177,8 @@ static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(Vstruct_ref->opcode.exists()) { err("type `{}` does not have a serialization prefix", receiver).fire(v); } - return td::make_refint(f_name.ends_with('x') ? t_struct->struct_ref->opcode.pack_prefix : t_struct->struct_ref->opcode.prefix_len); + uint64_t val = f_name.ends_with('x') ? t_struct->struct_ref->opcode.pack_prefix : t_struct->struct_ref->opcode.prefix_len; + return ConstValInt{td::make_refint(val)}; } } @@ -190,33 +198,33 @@ static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(Vrange, str); + return ConstValInt{parse_nanotons_as_floating_string(v_arg->range, str)}; } - if (f_name == "address") { // previously, postfix "..."a, but it returned `slice` (now returns `address`) - unsigned char data[3 + 8 + 256]; // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; + if (f_name == "address") { // previously, postfix "..."a + unsigned char data[267]; parse_any_std_address(str, v_arg->range, &data); - return td::BitSlice{data, sizeof(data)}.to_hex(); + return ConstValAddress{td::BitSlice{data, sizeof(data)}.to_hex()}; } if (f_name == "stringCrc32") { // previously, postfix "..."c - return td::make_refint(td::crc32(td::Slice{str.data(), str.size()})); + return ConstValInt{td::make_refint(td::crc32(td::Slice{str.data(), str.size()}))}; } if (f_name == "stringCrc16") { // previously, there was no postfix in FunC, no way to calc at compile-time - return td::make_refint(td::crc16(td::Slice{str.data(), str.size()})); + return ConstValInt{td::make_refint(td::crc16(td::Slice{str.data(), str.size()}))}; } if (f_name == "stringSha256") { // previously, postfix "..."H unsigned char hash[32]; digest::hash_str(hash, str.data(), str.size()); - return td::bits_to_refint(hash, 256, false); + return ConstValInt{td::bits_to_refint(hash, 256, false)}; } if (f_name == "stringSha256_32") { // previously, postfix "..."h unsigned char hash[32]; digest::hash_str(hash, str.data(), str.size()); - return td::bits_to_refint(hash, 32, false); + return ConstValInt{td::bits_to_refint(hash, 32, false)}; } if (f_name == "stringHexToSlice") { // previously, postfix "..."s @@ -225,7 +233,7 @@ static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(V(str); + return ConstValSlice{static_cast(str)}; // the same string, we've just checked it } if (f_name == "stringToBase256") { // previously, postfix "..."u @@ -236,149 +244,190 @@ static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(V v) { - visit(v->get_rhs()); - } - - static void handle_binary_operator(V v) { - visit(v->get_lhs()); - visit(v->get_rhs()); - } +// ConstExpressionEvaluator is a class with static methods (visitors) +// handling supported operations inside constant expressions +// (in `const name = ...`, field/param defaults, enum members, etc.) +class ConstExpressionEvaluator { + // `-5` => int(-5), `!true` => false + static ConstValExpression handle_unary_operator(V v) { + ConstValExpression expr = eval_any_v_or_fire(v->get_rhs()); - // `const a = 1 + b`, we met `b` - static void handle_reference(V v) { - GlobalConstPtr const_ref = v->sym->try_as(); - if (!const_ref) { - err("symbol `{}` is not a constant", v->get_name()).fire(v); + switch (v->tok) { + case tok_minus: + if (const ConstValInt* i = std::get_if(&expr)) { + return ConstValInt{-i->int_val}; + } + break; + case tok_bitwise_not: + if (const ConstValInt* i = std::get_if(&expr)) { + return ConstValInt{~i->int_val}; + } + break; + case tok_plus: + if (const ConstValInt* i = std::get_if(&expr)) { + return ConstValInt{i->int_val}; + } + break; + case tok_logical_not: + if (const ConstValInt* i = std::get_if(&expr)) { + return ConstValBool{i->int_val == 0}; + } + if (const ConstValBool* b = std::get_if(&expr)) { + return ConstValBool{!b->bool_val}; + } + break; + default: + break; } + err_not_a_constant_expression().fire(v); } - // `const a = ton("0.05")`, we met `ton("0.05")` - static void handle_function_call(V v) { - if (v->fun_maybe && v->fun_maybe->is_compile_time_const_val()) { - // `ton(local_var)` is denied; it's validated not here, but when replacing its value with a calculated one - return; + // `2 + 3` => int(5), `10 > 3` => true, `true & false` => 0 + static ConstValExpression handle_binary_operator(V v) { + ConstValExpression expr_lhs = eval_any_v_or_fire(v->get_lhs()); + ConstValExpression expr_rhs = eval_any_v_or_fire(v->get_rhs()); + + td::RefInt256 lhs; + td::RefInt256 rhs; + + if (const ConstValInt* i_lhs = std::get_if(&expr_lhs)) { + lhs = i_lhs->int_val; + } else if (const ConstValBool* b_lhs = std::get_if(&expr_lhs)) { + lhs = td::make_refint(b_lhs->bool_val ? -1 : 0); + } + + if (const ConstValInt* i_rhs = std::get_if(&expr_rhs)) { + rhs = i_rhs->int_val; + } else if (const ConstValBool* b_rhs = std::get_if(&expr_rhs)) { + rhs = td::make_refint(b_rhs->bool_val ? -1 : 0); } - err("not a constant expression").fire(v); - } - // `const a = (1, 2)`, why not; or it's a default value of a field - static void handle_tensor(V v) { - for (int i = 0; i < v->size(); ++i) { - visit(v->get_item(i)); + if (lhs.is_null() || rhs.is_null()) { + err("operator `{}` is used incorrectly in a constant expression", v->operator_name).fire(v); } - } - // `a: Options = {}`, object literal may occur as a default value for parameters - static void handle_object_literal(V v) { - for (int i = 0; i < v->get_num_fields(); ++i) { - visit(v->get_field(i)->get_init_val()); + switch (v->tok) { + case tok_plus: return ConstValInt{lhs + rhs}; + case tok_minus: return ConstValInt{lhs - rhs}; + case tok_mul: return ConstValInt{lhs * rhs}; + case tok_div: return ConstValInt{lhs / rhs}; + case tok_mod: return ConstValInt{lhs % rhs}; + case tok_bitwise_and: return ConstValInt{lhs & rhs}; + case tok_bitwise_or: return ConstValInt{lhs | rhs}; + case tok_bitwise_xor: return ConstValInt{lhs ^ rhs}; + case tok_lshift: return ConstValInt{lhs << static_cast(rhs->to_long())}; + case tok_rshift: return ConstValInt{lhs >> static_cast(rhs->to_long())}; + case tok_logical_and: return ConstValBool{lhs != 0 && rhs != 0}; + case tok_logical_or: return ConstValBool{lhs != 0 || rhs != 0}; + case tok_gt: return ConstValBool{td::cmp(lhs, rhs) > 0}; + case tok_geq: return ConstValBool{td::cmp(lhs, rhs) >= 0}; + case tok_lt: return ConstValBool{td::cmp(lhs, rhs) < 0}; + case tok_leq: return ConstValBool{td::cmp(lhs, rhs) <= 0}; + case tok_eq: return ConstValBool{td::cmp(lhs, rhs) == 0}; + case tok_neq: return ConstValBool{td::cmp(lhs, rhs) != 0}; + default: + err("operator `{}` is not allowed in a constant expression", v->operator_name).fire(v); } } - static void visit(AnyExprV v) { - if (v->try_as() || v->try_as() || v->try_as() || v->try_as()) { - return; - } - if (auto v_unop = v->try_as()) { - return handle_unary_operator(v_unop); - } - if (auto v_binop = v->try_as()) { - return handle_binary_operator(v_binop); - } - if (auto v_ref = v->try_as()) { - return handle_reference(v_ref); - } - if (auto v_call = v->try_as()) { - return handle_function_call(v_call); - } - if (auto v_tensor = v->try_as()) { - return handle_tensor(v_tensor); - } - if (auto v_obj = v->try_as()) { - return handle_object_literal(v_obj->get_body()); - } - if (auto v_par = v->try_as()) { - return visit(v_par->get_expr()); + // `lhs as `; we allow `as` operator inside constants, but it's restricted not to change value's shape; + // e.g., `5 as int8` or `Color.Red as int` is okay, but `5 as int|slice` is not + static ConstValExpression handle_cast_as_operator(V v) { + ConstValExpression val = eval_any_v_or_fire(v->get_expr()); + + TypePtr l = v->get_expr()->inferred_type->unwrap_alias(); + TypePtr r = v->inferred_type->unwrap_alias(); + if (l->equal_to(r)) { + return val; } - if (auto v_cast_to = v->try_as()) { - return visit(v_cast_to->get_expr()); + + bool lhs_is_int = l == TypeDataInt::create() || l == TypeDataCoins::create() || l->try_as() || l->try_as(); + bool rhs_is_int = r == TypeDataInt::create() || r == TypeDataCoins::create() || r->try_as() || r->try_as(); + if (lhs_is_int && rhs_is_int && std::holds_alternative(val)) { + return val; } - if (auto v_dot = v->try_as()) { - if (v_dot->is_target_indexed_access()) { // anotherConst.0 - return visit(v_dot->get_obj()); - } - if (v_dot->is_target_enum_member()) { // Color.Red - return; - } + + bool lhs_is_slice = l->try_as() || l->try_as(); + bool rhs_is_slice = r->try_as() || r->try_as(); + if (lhs_is_slice && rhs_is_slice && std::holds_alternative(val)) { + return val; } - err("not a constant expression").fire(v); - } - // check that `2 + 3` is constant - // type inferring has already passed, so types are correct, `1 + ""` can not occur - // if v is not a constant expression like `foo()`, an exception is thrown - static void check_expression_expected_to_be_constant(AnyExprV v) { - visit(v); - } -}; + bool lhs_is_address = l->try_as(); + bool rhs_is_address = r->try_as(); + if (lhs_is_address && rhs_is_address && std::holds_alternative(val)) { + return val; + } -struct EnumMemberEvaluator { - static td::RefInt256 handle_unary_operator(V v) { - td::RefInt256 rhs = visit(v->get_rhs()); - if (!rhs->is_valid()) return rhs; - - switch (v->tok) { - case tok_minus: return -rhs; - case tok_bitwise_not: return ~rhs; - case tok_plus: return rhs; - default: - err("unsupported operator in `enum` member value").fire(v); + bool rhs_is_nullable = r->try_as() && r->try_as()->has_null(); + if (l == TypeDataNullLiteral::create() && rhs_is_nullable && std::holds_alternative(val)) { + return val; } - } - static td::RefInt256 handle_binary_operator(V v) { - td::RefInt256 lhs = visit(v->get_lhs()); - td::RefInt256 rhs = visit(v->get_rhs()); - if (!lhs->is_valid()) return lhs; - if (!rhs->is_valid()) return rhs; + err("operator `as` to `{}` from `{}` can not be used in a constant expression", r, l).fire(v); + } - switch (v->tok) { - case tok_plus: return lhs + rhs; - case tok_minus: return lhs - rhs; - case tok_mul: return lhs * rhs; - case tok_div: return lhs / rhs; - case tok_mod: return lhs % rhs; - case tok_bitwise_and: return lhs & rhs; - case tok_bitwise_or: return lhs | rhs; - case tok_bitwise_xor: return lhs ^ rhs; - case tok_lshift: return lhs << static_cast(rhs->to_long()); - case tok_rshift: return lhs >> static_cast(rhs->to_long()); - default: - err("unsupported operator in `enum` member value").fire(v); + // `ton("0.05")` and other compile-time functions + static ConstValExpression handle_function_call(V v) { + FunctionPtr fun_ref = v->fun_maybe; + if (!fun_ref || !fun_ref->is_compile_time_const_val()) { + err_not_a_constant_expression().fire(v); } + + return parse_vertex_call_to_compile_time_function(v, fun_ref->name); } - // `enum { Red = SIX }`, we met `SIX` - static td::RefInt256 handle_reference(V v) { + // `const a = ANOTHER`, or in field default, enum member, etc. + static ConstValExpression handle_reference(V v) { GlobalConstPtr const_ref = v->sym->try_as(); if (!const_ref) { err("symbol `{}` is not a constant", v->get_name()).fire(v); } - return visit(const_ref->init_value); + return eval_and_cache_const_init_val(const_ref); } - static td::RefInt256 visit(AnyExprV v) { + // `anotherConst.0` or `Color.Red` + static ConstValExpression handle_dot_access(V v) { + if (v->is_target_indexed_access()) { // anotherConst.0 + int index_at = std::get(v->target); + ConstValExpression lhs = eval_any_v_or_fire(v->get_obj()); + ConstValTensor* lhs_tensor = std::get_if(&lhs); + if (!lhs_tensor || index_at < 0 || index_at >= static_cast(lhs_tensor->items.size())) { + err_not_a_constant_expression().fire(v); + } + return eval_any_v_or_fire(lhs_tensor->items[index_at]); + } + if (v->is_target_enum_member()) { // Color.Red + EnumDefPtr enum_ref = v->inferred_type->unwrap_alias()->try_as()->enum_ref; + std::vector enum_values = calculate_enum_members_with_values(enum_ref); + td::RefInt256 member_value = enum_values[std::get(v->target)->member_idx]; + return ConstValInt{std::move(member_value)}; + } + err_not_a_constant_expression().fire(v); + } + +public: + // this function either returns or fires "not a constant expression" or something more meaningful + static ConstValExpression eval_any_v_or_fire(AnyExprV v) { if (auto v_int = v->try_as()) { - return v_int->intval; + return ConstValInt{v_int->intval}; + } + if (auto v_bool = v->try_as()) { + return ConstValBool{v_bool->bool_val}; + } + if (auto v_string = v->try_as()) { + td::Slice str_slice = td::Slice(v_string->str_val.data(), v_string->str_val.size()); + return ConstValSlice{td::hex_encode(str_slice)}; + } + if (auto v_par = v->try_as()) { + return eval_any_v_or_fire(v_par->get_expr()); } if (auto v_un = v->try_as()) { return handle_unary_operator(v_un); @@ -386,28 +435,113 @@ struct EnumMemberEvaluator { if (auto v_bin = v->try_as()) { return handle_binary_operator(v_bin); } + if (auto v_as = v->try_as()) { + return handle_cast_as_operator(v_as); + } if (auto v_ref = v->try_as()) { return handle_reference(v_ref); } - if (auto v_par = v->try_as()) { - return visit(v_par->get_expr()); - } if (auto v_dot = v->try_as()) { - // `enum E { V = AnotherEnum.A }` not allowed (can be allowed in the future, - // but remember to construct an initialization tree, since A may be auto-computed, and prevent recursion) - if (v_dot->is_target_enum_member()) { - err("referencing other members in initializers is not allowed yet").fire(v); + return handle_dot_access(v_dot); + } + if (auto v_call = v->try_as()) { + return handle_function_call(v_call); + } + if (auto v_tensor = v->try_as()) { + std::vector items; + items.reserve(v_tensor->size()); + for (int i = 0; i < v_tensor->size(); ++i) { + AnyExprV v_ith = v_tensor->get_item(i); + check_expression_is_constant_or_fire(v_ith); + items.emplace_back(v_ith); + } + return ConstValTensor{std::move(items)}; + } + if (auto v_object = v->try_as()) { + V v_body = v_object->get_body(); + std::vector> fields; + fields.reserve(v_body->size()); + for (int i = 0; i < v_body->size(); ++i) { + AnyExprV field_init_val = v_body->get_field(i)->get_init_val(); + check_expression_is_constant_or_fire(field_init_val); + fields.emplace_back(v_body->get_field(i)->field_ref, field_init_val); } + return ConstValObject{v_object->struct_ref, std::move(fields)}; + } + if (v->try_as()) { + return ConstValNullLiteral{}; } - err("unsupported operation in `enum` member value").fire(v); + err_not_a_constant_expression().fire(v); } }; +void check_expression_is_constant_or_fire(AnyExprV v_expr) { + // inline the most popular case + if (v_expr->kind == ast_int_const) { + return; + } + ConstExpressionEvaluator::eval_any_v_or_fire(v_expr); +} + +ConstValExpression eval_constant_expression_or_fire(AnyExprV v_expr) { + // inline the most popular case + if (auto v_int = v_expr->try_as()) { + return ConstValInt{v_int->intval}; + } + return ConstExpressionEvaluator::eval_any_v_or_fire(v_expr); +} + +ConstValExpression eval_and_cache_const_init_val(GlobalConstPtr const_ref) { + auto it = computed_constants_cache.find(const_ref); + if (it != computed_constants_cache.end()) { + return it->second; + } + + // constants initializers are not recursive (checked at inferring), so no stack guards here + ConstValExpression v = ConstExpressionEvaluator::eval_any_v_or_fire(const_ref->init_value); + computed_constants_cache[const_ref] = v; + return v; +} + +std::vector calculate_enum_members_with_values(EnumDefPtr enum_ref) { + static std::vector called_stack; -void check_expression_is_constant(AnyExprV v_expr) { - ConstantExpressionChecker::check_expression_expected_to_be_constant(v_expr); + // prevent recursion like `enum Color { v = Another.item } enum Another { item = Color.v }` + // (unlike constants, enums initializers were not checked earlier for recursion) + bool contains = std::find(called_stack.begin(), called_stack.end(), enum_ref) != called_stack.end(); + if (contains) { + err("enum `{}` initializers circularly references itself", enum_ref).fire(enum_ref->ident_anchor); + } + + std::vector values; + values.reserve(enum_ref->members.size()); + called_stack.push_back(enum_ref); + + td::RefInt256 prev_value = td::make_refint(-1); + for (EnumMemberPtr member_ref : enum_ref->members) { + td::RefInt256 cur_value = prev_value + 1; + if (member_ref->has_init_value()) { + ConstValExpression assigned = ConstExpressionEvaluator::eval_any_v_or_fire(member_ref->init_value); + ConstValInt* assigned_int = std::get_if(&assigned); + if (!assigned_int) { + err("invalid enum member initializer, not an integer").fire(member_ref->ident_anchor); + } + cur_value = assigned_int->int_val; + } + if (!cur_value->is_valid() || !cur_value->signed_fits_bits(257)) { + err("integer overflow").fire(member_ref->ident_anchor); + } + + values.push_back(cur_value); + prev_value = std::move(cur_value); + } + + called_stack.pop_back(); + return values; } +// for just a string literal "asdf" in Tolk code, it's hex-encoded "61626364", +// and surrounded as `x{61626364}` into Fift output std::string eval_string_const_standalone(AnyExprV v_string) { auto v = v_string->try_as(); tolk_assert(v); @@ -415,15 +549,12 @@ std::string eval_string_const_standalone(AnyExprV v_string) { return td::hex_encode(str_slice); } -CompileTimeFunctionResult eval_call_to_compile_time_function(AnyExprV v_call) { +// for `ton("0.05")` and similar compile-time only functions, we evaluate them in-place +// and push already evaluated expression to IR vars +ConstValExpression eval_call_to_compile_time_function(AnyExprV v_call) { auto v = v_call->try_as(); tolk_assert(v && v->fun_maybe->is_compile_time_const_val()); return parse_vertex_call_to_compile_time_function(v, v->fun_maybe->name); } -td::RefInt256 eval_enum_member_init_value(AnyExprV init_value) { - return EnumMemberEvaluator::visit(init_value); -} - - } // namespace tolk diff --git a/tolk/constant-evaluator.h b/tolk/constant-evaluator.h index 4f488951c6..44062be773 100644 --- a/tolk/constant-evaluator.h +++ b/tolk/constant-evaluator.h @@ -22,11 +22,66 @@ namespace tolk { -typedef std::variant CompileTimeFunctionResult; +struct ConstValInt; +struct ConstValBool; +struct ConstValSlice; +struct ConstValAddress; +struct ConstValTensor; +struct ConstValObject; +struct ConstValNullLiteral; -void check_expression_is_constant(AnyExprV v_expr); +// `const a = 2 + 3` is okay, but `const a = foo()` is not; +// "okay" means "a constant expression", which can be evaluated at compile-time; +// default values of struct fields and enum members are also required to be constant; +// `field: (int, Obj) = (2, {v: true})` is also okay, `(2, {v: true})` is a valid constant expression; +// +// so, every const/enum/param default can be evaluated into ConstValExpression +// and later exported into ABI +typedef std::variant< + ConstValInt, + ConstValBool, + ConstValSlice, + ConstValAddress, + ConstValTensor, + ConstValObject, + ConstValNullLiteral +> ConstValExpression; + +struct ConstValInt { + td::RefInt256 int_val; +}; + +struct ConstValBool { + bool bool_val; +}; + +struct ConstValSlice { + std::string str_hex; +}; + +struct ConstValAddress { + std::string std_addr_hex; +}; + +struct ConstValTensor { + std::vector items; +}; + +struct ConstValObject { + StructPtr struct_ref; + std::vector> fields; +}; + +struct ConstValNullLiteral { +}; + +ConstValExpression eval_constant_expression_or_fire(AnyExprV v_expr); +ConstValExpression eval_and_cache_const_init_val(GlobalConstPtr const_ref); +ConstValExpression eval_call_to_compile_time_function(AnyExprV v_call); + +std::vector calculate_enum_members_with_values(EnumDefPtr enum_ref); + +void check_expression_is_constant_or_fire(AnyExprV v_expr); std::string eval_string_const_standalone(AnyExprV v_string); -CompileTimeFunctionResult eval_call_to_compile_time_function(AnyExprV v_call); -td::RefInt256 eval_enum_member_init_value(AnyExprV init_value); } // namespace tolk diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index add44e726e..95e253e412 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -1104,13 +1104,14 @@ struct S_IntegerEnum final : ISerializer { } void pack(const PackContext* ctx, CodeBlob& code, AnyV origin, std::vector&& rvect) override { - TypePtr intN = calculate_intN_to_serialize_enum(enum_ref); - return ctx->generate_pack_any(intN, std::move(rvect)); + TypePtr pack_type = calculate_intN_to_serialize_enum(enum_ref); + return ctx->generate_pack_any(pack_type, std::move(rvect)); } std::vector unpack(const UnpackContext* ctx, CodeBlob& code, AnyV origin) override { - const TypeDataIntN* intN = calculate_intN_to_serialize_enum(enum_ref)->try_as(); - std::vector ir_num = ctx->generate_unpack_any(intN); + TypePtr pack_type = calculate_intN_to_serialize_enum(enum_ref); + std::vector ir_num = ctx->generate_unpack_any(pack_type); + const TypeDataIntN* intN = pack_type->try_as(); // when reading an integer value, we need to validate that it's a valid enum member; // at first, detect whether it's a sequence (A, A+1, ..., A+N) @@ -1125,7 +1126,7 @@ struct S_IntegerEnum final : ISerializer { // enum's members are A...B one by one (probably, 0...M); // then validation is: "throw if vB", but "LESSINT + THROWIF" 2 times is more generalized td::RefInt256 min_value = enum_ref->members.front()->computed_value; - bool dont_check_min = intN->is_unsigned && min_value == 0; + bool dont_check_min = intN != nullptr && intN->is_unsigned && min_value == 0; if (!dont_check_min) { // LDU can't load < 0 std::vector ir_min_value = code.create_tmp_var(TypeDataInt::create(), origin, "(enum-min)"); code.emplace_back(origin, Op::_IntConst, ir_min_value, min_value); @@ -1136,7 +1137,7 @@ struct S_IntegerEnum final : ISerializer { op_assert.set_impure_flag(); } td::RefInt256 max_value = enum_ref->members.back()->computed_value; - bool dont_check_max = intN->is_unsigned && max_value == (1ULL << intN->n_bits) - 1; + bool dont_check_max = intN != nullptr && intN->is_unsigned && max_value == (1ULL << intN->n_bits) - 1; if (!dont_check_max) { // LDU can't load >= 1<generate_skip_any(intN); + TypePtr pack_type = calculate_intN_to_serialize_enum(enum_ref); + return ctx->generate_skip_any(pack_type); } PackSize estimate(const EstimateContext* ctx) override { - TypePtr intN = calculate_intN_to_serialize_enum(enum_ref); - return ctx->estimate_any(intN); + TypePtr pack_type = calculate_intN_to_serialize_enum(enum_ref); + return ctx->estimate_any(pack_type); } }; @@ -1286,7 +1287,7 @@ std::vector auto_generate_opcodes_for_union(TypePtr union_type, std: // example: `enum Color { Red, Green, Blue }` is 00/01/10 - uint2 // example: `enum Role: int8 { ... }` — manually specified TypePtr calculate_intN_to_serialize_enum(EnumDefPtr enum_ref) { - if (enum_ref->colon_type) { + if (enum_ref->colon_type) { // intN / coins return enum_ref->colon_type; } diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 89aa58c509..3426860449 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -19,8 +19,8 @@ #include "ast-aux-data.h" #include "ast-visitor.h" #include "compilation-errors.h" +#include "constant-evaluator.h" #include "type-system.h" -#include "src-file.h" #include "common/refint.h" #include "smart-casts-cfg.h" #include "pack-unpack-api.h" @@ -384,15 +384,6 @@ const LazyVariableLoadedState* CodeBlob::get_lazy_variable(AnyExprV v) const { return nullptr; } -const CachedConstValueAtCodegen* CodeBlob::get_cached_const(GlobalConstPtr const_ref) const { - for (const CachedConstValueAtCodegen& c : cached_consts) { - if (c.const_ref == const_ref) { - return &c; - } - } - return nullptr; -} - // given `{some_expr}!`, return some_expr static AnyExprV unwrap_not_null_operator(AnyExprV v) { @@ -436,6 +427,9 @@ static bool is_ternary_arg_trivial_for_condsel(AnyExprV v, bool require_1slot = if (auto v_cast = v->try_as()) { return is_ternary_arg_trivial_for_condsel(v_cast->get_expr(), require_1slot); } + if (auto v_call = v->try_as()) { + return v_call->fun_maybe && v_call->fun_maybe->is_compile_time_const_val(); + } return false; } @@ -446,10 +440,9 @@ static std::vector> pre_compile_tensor_inner(CodeBlob& co if (n == 0) { // just `()` return {}; } - tolk_assert(!tensor_target_type || tensor_target_type->size() == n); + tolk_assert(tensor_target_type && tensor_target_type->size() == n); if (n == 1) { // just `(x)`: even if x is modified (e.g. `f(x=x+2)`), there are no next arguments - TypePtr child_target_type = tensor_target_type ? tensor_target_type->items[0] : nullptr; - return {pre_compile_expr(args[0], code, child_target_type, lval_ctx)}; + return {pre_compile_expr(args[0], code, tensor_target_type->items[0], lval_ctx)}; } // the purpose is to handle such cases: `return (x, x += y, x)` @@ -503,8 +496,7 @@ static std::vector> pre_compile_tensor_inner(CodeBlob& co WatchingVarList watched_vars(n); for (int arg_idx = 0; arg_idx < n; ++arg_idx) { - TypePtr child_target_type = tensor_target_type ? tensor_target_type->items[arg_idx] : nullptr; - std::vector vars_of_ith_arg = pre_compile_expr(args[arg_idx], code, child_target_type, lval_ctx); + std::vector vars_of_ith_arg = pre_compile_expr(args[arg_idx], code, tensor_target_type->items[arg_idx], lval_ctx); watched_vars.add_and_watch_modifications(std::move(vars_of_ith_arg), code); } return watched_vars.clear_and_stop_watching(); @@ -1108,11 +1100,15 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as()) { - return rvect; + if (original_type->try_as()) { + if (target_type == TypeDataInt::create() || target_type == TypeDataCoins::create() || target_type->try_as()) { + return rvect; + } } - if (original_type == TypeDataInt::create() && target_type->try_as()) { - return rvect; + if (target_type->try_as()) { + if (original_type == TypeDataInt::create() || original_type == TypeDataCoins::create() || original_type->try_as()) { + return rvect; + } } // `Color.Red` as `BounceMode` (all enums are integers, they can be cast one to another) if (original_type->try_as() && target_type->try_as()) { @@ -1145,6 +1141,61 @@ std::vector transition_to_target_type(std::vector&& rvect, return rvect; } +// convert a constant value (calculated by a "constant-evaluator") to IR vars; +// every init_val of `const XXX = ...` is calculated once (into ConstValExpression) and cached +static std::vector pre_compile_constant_expression(const ConstValExpression& value, CodeBlob& code, AnyV origin) { + if (const ConstValInt* val = std::get_if(&value)) { + std::vector rvect = code.create_tmp_var(TypeDataInt::create(), origin, "(int-const)"); + code.emplace_back(origin, Op::_IntConst, rvect, val->int_val); + return rvect; + } + if (const ConstValBool* val = std::get_if(&value)) { + FunctionPtr builtin_sym = lookup_function(val->bool_val ? "__true" : "__false"); + std::vector rvect = gen_op_call(code, TypeDataBool::create(), origin, {}, builtin_sym, "(bool-const)"); + return rvect; + } + if (const ConstValSlice* val = std::get_if(&value)) { + std::vector rvect = code.create_tmp_var(TypeDataSlice::create(), origin, "(str-const)"); + code.emplace_back(origin, Op::_SliceConst, rvect, val->str_hex); + return rvect; + } + if (const ConstValAddress* val = std::get_if(&value)) { + std::vector rvect = code.create_tmp_var(TypeDataSlice::create(), origin, "(addr-const)"); + code.emplace_back(origin, Op::_SliceConst, rvect, val->std_addr_hex); + return rvect; + } + if (const ConstValTensor* val = std::get_if(&value)) { + std::vector rvect; + for (AnyExprV v_item : val->items) { + std::vector ir_item = pre_compile_expr(v_item, code); + rvect.insert(rvect.end(), ir_item.begin(), ir_item.end()); + } + return rvect; + } + if (const ConstValObject* val = std::get_if(&value)) { + std::vector rvect; + for (StructFieldPtr field_ref : val->struct_ref->fields) { + std::vector ir_field; + auto it = std::find_if(val->fields.begin(), val->fields.end(), + [field_ref](const auto& p) { return p.first == field_ref; }); + if (it != val->fields.end()) { + ir_field = pre_compile_expr(it->second, code, field_ref->declared_type); + } else if (field_ref->declared_type != TypeDataNever::create()) { + tolk_assert(field_ref->has_default_value()); + ir_field = pre_compile_expr(field_ref->default_value, code, field_ref->declared_type); + } + rvect.insert(rvect.end(), ir_field.begin(), ir_field.end()); + } + return rvect; + } + if (std::get_if(&value)) { + FunctionPtr builtin_sym = lookup_function("__null"); + std::vector rvect = gen_op_call(code, TypeDataNullLiteral::create(), origin, {}, builtin_sym, "(null-literal)"); + return rvect; + } + tolk_assert(false); +} + std::vector pre_compile_symbol(const Symbol* sym, CodeBlob& code, AnyV origin, LValContext* lval_ctx) { // referencing a local variable (not its declaration, but its usage) @@ -1155,27 +1206,14 @@ std::vector pre_compile_symbol(const Symbol* sym, CodeBlob& code, Any return var_ref->ir_idx; } - // referencing a global constant, embed its init_value directly (so, it will be evaluated to a const val later) + // referencing a global constant, embed its init_value directly if (GlobalConstPtr const_ref = sym->try_as()) { tolk_assert(lval_ctx == nullptr); - // when evaluating `a1` in `const a2 = a1 + a1`, cache a1's IR vars to prevent exponential explosion - if (code.inside_evaluating_constant) { - if (const CachedConstValueAtCodegen* cached = code.get_cached_const(const_ref)) { - return cached->ir_idx; - } - std::vector ir_sub_const = pre_compile_expr(const_ref->init_value, code, const_ref->declared_type); - code.cached_consts.emplace_back(CachedConstValueAtCodegen{const_ref, ir_sub_const}); - return ir_sub_const; - } - // just referencing `a1` in a function body - // (note that `a1` occurred 2 times has 2 different ir_idx, caching above is done only within another const) - ASTAuxData* aux_data = new AuxData_ForceFiftLocation(origin); - auto v_force_loc = createV(const_ref->init_value, aux_data, const_ref->inferred_type); - code.inside_evaluating_constant = true; - std::vector ir_const = pre_compile_expr(v_force_loc, code, const_ref->declared_type); - code.inside_evaluating_constant = false; - code.cached_consts.clear(); - return ir_const; + ConstValExpression value = eval_and_cache_const_init_val(const_ref); + std::vector ir_init = pre_compile_constant_expression(value, code, origin); + tolk_assert(static_cast(ir_init.size()) == const_ref->init_value->inferred_type->get_width_on_stack()); + // handle `const a: int|slice = 1`, ir_init is int(1), transition to union + return transition_to_target_type(std::move(ir_init), code, const_ref->init_value->inferred_type, const_ref->inferred_type, origin); } // referencing a global variable, copy it to a local tmp var @@ -1634,6 +1672,12 @@ static std::vector process_function_call(V v, Code op.set_impure_flag(); return transition_to_target_type(std::move(rvect), code, target_type, v); } + // `ton("0.05")` and others, we even don't need to calculate ir_idx for arguments, just replace with constexpr + if (fun_ref->is_compile_time_const_val()) { + ConstValExpression value = eval_call_to_compile_time_function(v); + std::vector rvect = pre_compile_constant_expression(value, code, v); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } // fill args for evaluation: dot object + passed arguments + parameters defaults if not all passed AnyExprV obj_leftmost = v->get_self_obj(); @@ -1649,14 +1693,11 @@ static std::vector process_function_call(V v, Code for (int i = 0; i < v->get_num_args(); ++i) { args.push_back(v->get_arg(i)->get_expr()); } - // default values (forcing location to be here, not taken from declaraion) + // default values for (int i = delta_self + v->get_num_args(); i < fun_ref->get_num_params(); ++i) { LocalVarPtr param_ref = &fun_ref->get_param(i); tolk_assert(param_ref->has_default_value()); - AnyV last_arg = args.empty() ? v : args.back(); - ASTAuxData *aux_data = new AuxData_ForceFiftLocation(last_arg); - auto v_force_loc = createV(param_ref->default_value, aux_data, param_ref->declared_type); - args.push_back(v_force_loc); + args.push_back(param_ref->default_value); } // the purpose of tensor_tt ("tensor target type") is to transition `null` to `(int, int)?` and so on @@ -1827,10 +1868,7 @@ static std::vector process_object_literal_shuffled(Vhas_default_value()); - AnyV last_field = v->get_body()->empty() ? v : v->get_body()->get_all_fields().back(); - ASTAuxData *aux_data = new AuxData_ForceFiftLocation(last_field); - auto v_force_loc = createV(field_ref->default_value, aux_data, field_ref->declared_type); - std::vector def_rvect = pre_compile_expr(v_force_loc, code, field_ref->declared_type); + std::vector def_rvect = pre_compile_expr(field_ref->default_value, code, field_ref->declared_type); code.emplace_back(v, Op::_Let, std::move(field_rvect), std::move(def_rvect)); } @@ -1857,7 +1895,6 @@ static std::vector process_object_literal(V v, Co return process_object_literal_shuffled(v, code, target_type, lval_ctx); } - AnyV prev_init_val = v; std::vector tensor_items; std::vector target_types; tensor_items.reserve(v->struct_ref->get_num_fields()); @@ -1876,13 +1913,10 @@ static std::vector process_object_literal(V v, Co continue; // field of `never` type can be missed out of object literal (useful in generics defaults) } // (it occupies 0 slots, nothing is assignable to it — like this field is missing from a struct) tolk_assert(field_ref->has_default_value()); - ASTAuxData *aux_data = new AuxData_ForceFiftLocation(prev_init_val); - auto v_force_loc = createV(field_ref->default_value, aux_data, field_ref->declared_type); - v_init_val = v_force_loc; // it may be a complex expression, but it's a constant, checked earlier + v_init_val = field_ref->default_value; } tensor_items.push_back(v_init_val); target_types.push_back(field_ref->declared_type); - prev_init_val = v_init_val; } const auto* tensor_target_type = TypeDataTensor::create(std::move(target_types))->try_as(); std::vector rvect = pre_compile_tensor(code, tensor_items, lval_ctx, tensor_target_type); @@ -1898,8 +1932,9 @@ static std::vector process_int_const(V v, CodeBlob& co } static std::vector process_string_const(V v, CodeBlob& code, TypePtr target_type) { + std::string literal_value = eval_string_const_standalone(v); std::vector rvect = code.create_tmp_var(v->inferred_type, v, "(str-const)"); - code.emplace_back(v, Op::_SliceConst, rvect, v->literal_value); + code.emplace_back(v, Op::_SliceConst, rvect, std::move(literal_value)); return transition_to_target_type(std::move(rvect), code, target_type, v); } @@ -1947,14 +1982,6 @@ static std::vector process_empty_expression(V v static std::vector process_artificial_aux_vertex(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { AnyExprV wrapped = v->get_wrapped_expr(); - if (const auto* data = dynamic_cast(v->aux_data)) { - AnyV backup = code.forced_origin; - code.forced_origin = data->forced_origin; - std::vector rvect = pre_compile_expr(wrapped, code, target_type, lval_ctx); - code.forced_origin = backup; - return rvect; - } - // aux "load x"; example: `var p = lazy Point.fromSlice(s); aux "load x"; return p.x` if (const auto* data = dynamic_cast(v->aux_data)) { const LazyVariableLoadedState* lazy_variable = code.get_lazy_variable(data->var_ref); diff --git a/tolk/pipe-check-const-expr.cpp b/tolk/pipe-check-const-expr.cpp new file mode 100644 index 0000000000..86c221411a --- /dev/null +++ b/tolk/pipe-check-const-expr.cpp @@ -0,0 +1,97 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "ast.h" +#include "ast-visitor.h" +#include "type-system.h" +#include "constant-evaluator.h" + +/* + * This pipe checks that expressions expected to be constant, are actually constant. + * For example, `const a = 2 + 3` is okay, but `const a = foo()` is not. + * For example, field defaults and parameters defaults are also required to be constant. + * + * Also, this pipe calculates and assigns values for every `enum` members. + */ + +namespace tolk { + +class ConstantExpressionsChecker final : public ASTVisitorFunctionBody { +protected: + void visit(V v) override { + // check `ton("0.05")` and others for correctness (not `ton(local_var)`, etc.) + if (v->fun_maybe && v->fun_maybe->is_compile_time_const_val()) { + // on invalid usage, this call will fire + eval_call_to_compile_time_function(v); + // note that in AST tree, it's still left as `ton("0.05")`, `stringCrc32("...")`, etc. + // later, when transforming to IR, such compile-time functions are handled specially + } + + parent::visit(v); + } + + void visit(V v) override { + // check `2 + 3 => ...` (before =>) + // non-constant expressions like `foo() => ...` fire an error here + if (v->pattern_kind == MatchArmKind::const_expression) { + check_expression_is_constant_or_fire(v->get_pattern_expr()); + } + + parent::visit(v); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } +}; + +void pipeline_check_constant_expressions() { + // here (after type inferring) check that `const a = 2 + 3` is a valid constant expression + // non-constant expressions like `const a = foo()` fire an error here + for (GlobalConstPtr const_ref : get_all_declared_constants()) { + eval_and_cache_const_init_val(const_ref); + } + // do the same for default values of struct fields, they must be constant expressions + for (StructPtr struct_ref : get_all_declared_structs()) { + for (StructFieldPtr field_ref : struct_ref->fields) { + if (field_ref->has_default_value() && !struct_ref->is_generic_struct()) { + check_expression_is_constant_or_fire(field_ref->default_value); + } + } + } + // and for default values of parameters + for (FunctionPtr fun_ref : get_all_not_builtin_functions()) { + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + LocalVarPtr param_ref = &fun_ref->get_param(i); + if (param_ref->has_default_value() && !fun_ref->is_generic_function()) { + check_expression_is_constant_or_fire(param_ref->default_value); + } + } + } + + // assign `enum` members values (either auto-compute sequentially or use manual initializers) + for (EnumDefPtr enum_ref : get_all_declared_enums()) { + std::vector values = calculate_enum_members_with_values(enum_ref); + for (EnumMemberPtr member_ref : enum_ref->members) { + member_ref->mutate()->assign_computed_value(values[member_ref->member_idx]); + } + } + + visit_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 33ba7db686..eeb7bbf9e6 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -846,9 +846,13 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { void start_visiting_enum_member(EnumDefPtr enum_ref, EnumMemberPtr member_ref) { parent::visit(member_ref->init_value); - TypePtr inferred_type = member_ref->init_value->inferred_type; - if (!inferred_type->equal_to(TypeDataInt::create()) && !inferred_type->equal_to(TypeDataEnum::create(enum_ref))) { - err("enum member is `{}`, not `int`\n""hint: all enums must be integers", inferred_type).fire(member_ref->init_value); + TypePtr m_type = member_ref->init_value->inferred_type; + bool is_integer = m_type->equal_to(TypeDataInt::create()) + || m_type->equal_to(TypeDataCoins::create()) + || m_type->try_as() + || m_type->try_as(); + if (!is_integer) { + err("enum member is `{}`, not `int`\n""hint: all enums must be integers", m_type).fire(member_ref->init_value); } } }; diff --git a/tolk/pipe-check-serialized-fields.cpp b/tolk/pipe-check-serialized-fields.cpp index e3f45a779c..3ee0f0582c 100644 --- a/tolk/pipe-check-serialized-fields.cpp +++ b/tolk/pipe-check-serialized-fields.cpp @@ -73,7 +73,7 @@ static void check_mapKV_inside_type(AnyTypeV type_node) { // given `enum Role: int8` check colon type (not struct/slice etc.) static void check_enum_colon_type_to_be_intN(AnyTypeV colon_type_node) { - if (!colon_type_node->resolved_type->try_as()) { + if (!colon_type_node->resolved_type->try_as() && !colon_type_node->resolved_type->try_as()) { err("serialization type of `enum` must be intN: `int8` / `uint32` / etc.").fire(colon_type_node); } } diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp deleted file mode 100644 index 9ebf3a535e..0000000000 --- a/tolk/pipe-constant-folding.cpp +++ /dev/null @@ -1,216 +0,0 @@ -/* - This file is part of TON Blockchain source code. - - TON Blockchain is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - TON Blockchain is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with TON Blockchain. If not, see . -*/ -#include "ast.h" -#include "ast-replacer.h" -#include "compilation-errors.h" -#include "type-system.h" -#include "constant-evaluator.h" - -/* - * This pipe is supposed to do constant folding, like replacing `2 + 3` with `5`. - * It happens after type inferring and validity checks, one of the last ones. - * - * Currently, it just replaces `-1` (ast_unary_operator ast_int_const) with a number -1 - * and `!true` with false. - * Also, all parenthesized `((expr))` are replaced with `expr`, it's a constant transformation. - * (not to handle parenthesized in optimization passes, like `((x)) == true`) - * More rich constant folding should be done some day, but even without this, IR optimizations - * (operating low-level stack variables) pretty manage to do all related optimizations. - * Constant folding in the future, done at AST level, just would slightly reduce amount of work for optimizer. - */ - -namespace tolk { - -// assign `enum` members values (either auto-compute sequentially or use manual initializers) -static void assign_enum_members_values(EnumDefPtr enum_ref) { - td::RefInt256 prev_value = td::make_refint(-1); - for (EnumMemberPtr member_ref : enum_ref->members) { - td::RefInt256 cur_value = member_ref->init_value ? eval_enum_member_init_value(member_ref->init_value) : prev_value + 1; - if (!cur_value->is_valid() || !cur_value->signed_fits_bits(257)) { - err("integer overflow").fire(member_ref->ident_anchor); - } - - member_ref->mutate()->assign_computed_value(cur_value); - prev_value = std::move(cur_value); - } -} - - -class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { - static V create_int_const(SrcRange range, td::RefInt256&& intval) { - auto v_int = createV(range, std::move(intval), {}); - v_int->assign_inferred_type(TypeDataInt::create()); - v_int->assign_rvalue_true(); - return v_int; - } - - static V create_bool_const(SrcRange range, bool bool_val) { - auto v_bool = createV(range, bool_val); - v_bool->assign_inferred_type(TypeDataBool::create()); - v_bool->assign_rvalue_true(); - return v_bool; - } - - AnyExprV replace(V v) override { - AnyExprV inner = parent::replace(v->get_expr()); - if (v->is_lvalue) { - inner->mutate()->assign_lvalue_true(); - } - return inner; - } - - static V create_string_const(SrcRange range, std::string&& literal_value, TypePtr inferred_type) { - auto v_string = createV(range, literal_value); - v_string->assign_inferred_type(inferred_type); - v_string->assign_literal_value(std::move(literal_value)); - v_string->assign_rvalue_true(); - return v_string; - } - - AnyExprV replace(V v) override { - parent::replace(v); - - TokenType t = v->tok; - // convert "-1" (tok_minus tok_int_const) to a const -1 - if (t == tok_minus && v->get_rhs()->kind == ast_int_const) { - td::RefInt256 intval = v->get_rhs()->as()->intval; - tolk_assert(!intval.is_null()); - intval = -intval; - if (intval.is_null() || !intval->signed_fits_bits(257)) { - err("integer overflow").fire(v); - } - return create_int_const(v->range, std::move(intval)); - } - // same for "+1" - if (t == tok_plus && v->get_rhs()->kind == ast_int_const) { - return v->get_rhs(); - } - - // `!true` / `!false` - if (t == tok_logical_not && v->get_rhs()->kind == ast_bool_const) { - return create_bool_const(v->range, !v->get_rhs()->as()->bool_val); - } - // `!0` - if (t == tok_logical_not && v->get_rhs()->kind == ast_int_const) { - return create_bool_const(v->range, v->get_rhs()->as()->intval == 0); - } - - return v; - } - - AnyExprV replace(V v) override { - parent::replace(v); - - // `null == null` / `null != null` - if (v->get_expr()->kind == ast_null_keyword && v->type_node->resolved_type == TypeDataNullLiteral::create()) { - return create_bool_const(v->range, !v->is_negated); - } - - return v; - } - - AnyExprV replace(V v) override { - parent::replace(v); - - // replace `ton("0.05")` with 50000000 / `stringCrc32("some_str")` with calculated value / etc. - if (v->fun_maybe && v->fun_maybe->is_compile_time_const_val()) { - CompileTimeFunctionResult value = eval_call_to_compile_time_function(v); - if (std::holds_alternative(value)) { - return create_int_const(v->range, std::move(std::get(value))); - } else { - return create_string_const(v->range, std::move(std::get(value)), v->fun_maybe->declared_return_type); - } - } - - return v; - } - - AnyExprV replace(V v) override { - // when "some_str" occurs as a standalone constant (not inside `stringCrc32("some_str")`, - // it's actually a slice - std::string literal_value = eval_string_const_standalone(v); - v->mutate()->assign_literal_value(std::move(literal_value)); - - return v; - } - - AnyExprV replace(V v) override { - parent::replace(v); - - // replace `2 + 3 => ...` with `5 => ...` - // non-constant expressions like `foo() => ...` fire an error here - if (v->pattern_kind == MatchArmKind::const_expression && v->get_pattern_expr()->kind != ast_int_const) { - check_expression_is_constant(v->get_pattern_expr()); - } - - return v; - } - -public: - bool should_visit_function(FunctionPtr fun_ref) override { - return fun_ref->is_code_function() && !fun_ref->is_generic_function(); - } - - void start_replacing_in_function(FunctionPtr fun_ref, V v_function) override { - // visit default values of parameters - for (int i = 0; i < fun_ref->get_num_params(); ++i) { - if (LocalVarPtr param_ref = &fun_ref->get_param(i); param_ref->has_default_value()) { - check_expression_is_constant(param_ref->default_value); - AnyExprV replaced = replace_in_expression(param_ref->default_value); - param_ref->mutate()->assign_default_value(replaced); - } - } - - parent::replace(v_function->get_body()); - } - - // used to replace `ton("0.05")` and other compile-time functions inside fields defaults, etc. - AnyExprV replace_in_expression(AnyExprV init_value) { - return parent::replace(init_value); - } -}; - -void pipeline_constant_folding() { - ConstantFoldingReplacer replacer; - - // here (after type inferring) check that `const a = 2 + 3` is a valid constant expression - // non-constant expressions like `const a = foo()` fire an error here - // also, replace `const a = ton("0.05")` with `const a = 50000000` - for (GlobalConstPtr const_ref : get_all_declared_constants()) { - check_expression_is_constant(const_ref->init_value); - AnyExprV replaced = replacer.replace_in_expression(const_ref->init_value); - const_ref->mutate()->assign_init_value(replaced); - } - // do the same for default values of struct fields, they must be constant expressions - for (StructPtr struct_ref : get_all_declared_structs()) { - for (StructFieldPtr field_ref : struct_ref->fields) { - if (field_ref->has_default_value() && !struct_ref->is_generic_struct()) { - check_expression_is_constant(field_ref->default_value); - AnyExprV replaced = replacer.replace_in_expression(field_ref->default_value); - field_ref->mutate()->assign_default_value(replaced); - } - } - } - // assign `enum` members values (either auto-compute sequentially or use manual initializers) - for (EnumDefPtr enum_ref : get_all_declared_enums()) { - assign_enum_members_values(enum_ref); - } - - replace_ast_of_all_functions(); -} - -} // namespace tolk diff --git a/tolk/pipe-optimize-boolean-expr.cpp b/tolk/pipe-optimize-boolean-expr.cpp index 323b3e9dff..f4a0930259 100644 --- a/tolk/pipe-optimize-boolean-expr.cpp +++ b/tolk/pipe-optimize-boolean-expr.cpp @@ -29,6 +29,9 @@ * It's tricky to implement whether replacing is safe. * For example, safe: `a > 0 && a < 10` / `a != 3 && a != 5` * For example, unsafe: `cached && calc()` / `a > 0 && log(a)` / `b != 0 && a / b > 1` / `i >= 0 && arr[idx]` / `f != null && close(f)` + * + * Also, all parenthesized `((expr))` are replaced with `expr`, it's a constant transformation. + * (not to handle parenthesized later in tricky assignments, etc.) */ namespace tolk { @@ -43,7 +46,7 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { static V create_bool_const(SrcRange range, bool bool_val) { auto v_bool = createV(range, bool_val); - v_bool->assign_inferred_type(TypeDataInt::create()); + v_bool->assign_inferred_type(TypeDataBool::create()); v_bool->assign_rvalue_true(); return v_bool; } @@ -105,6 +108,10 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { // `!true` / `!false` return create_bool_const(v->range, !inner_bool->bool_val); } + if (auto inner_int = v->get_rhs()->try_as()) { + // `!0` / `!123` + return create_bool_const(v->range, inner_int->intval == 0); + } } return v; @@ -129,6 +136,14 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { return v; } + AnyExprV replace(V v) override { + AnyExprV inner = parent::replace(v->get_expr()); + if (v->is_lvalue) { + inner->mutate()->assign_lvalue_true(); + } + return inner; + } + AnyV replace(V v) override { parent::replace(v); diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 7cf2afecf8..d4f3eedd00 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -144,7 +144,7 @@ static EnumDefPtr register_enum(V v) { err("redeclaration of member `{}`", member_name).fire(v_member); } } - members.emplace_back(new EnumMemberData(std::move(member_name), v_ident, v_member->init_value)); + members.emplace_back(new EnumMemberData(std::move(member_name), v_ident, i, v_member->init_value)); } V v_ident = v->get_identifier(); diff --git a/tolk/pipeline.h b/tolk/pipeline.h index f7b977f787..4197c2964f 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -42,8 +42,8 @@ void pipeline_refine_lvalue_for_mutate_arguments(); void pipeline_check_rvalue_lvalue(); void pipeline_check_private_fields_usage(); void pipeline_check_pure_impure_operations(); +void pipeline_check_constant_expressions(); void pipeline_mini_borrow_checker_for_mutate(); -void pipeline_constant_folding(); void pipeline_optimize_boolean_expressions(); void pipeline_detect_inline_in_place(); void pipeline_check_serialized_fields(); diff --git a/tolk/symtable.h b/tolk/symtable.h index ecd49df99c..7d4e82eb5b 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -372,13 +372,15 @@ struct StructData final : Symbol { }; struct EnumMemberData final : Symbol { + int member_idx; AnyExprV init_value; // nullptr if no init (`Red`, not `Red = 1`) td::RefInt256 computed_value; // auto-calculated or assigned from init if integer bool has_init_value() const { return init_value != nullptr; } - EnumMemberData(std::string name, AnyV ident_anchor, AnyExprV init_value) + EnumMemberData(std::string name, AnyV ident_anchor, int member_idx, AnyExprV init_value) : Symbol(std::move(name), ident_anchor) + , member_idx(member_idx) , init_value(init_value) { } diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp index 79eb986915..75c11bf54a 100644 --- a/tolk/tolk.cpp +++ b/tolk/tolk.cpp @@ -53,8 +53,8 @@ int tolk_proceed(const std::string &entrypoint_filename) { pipeline_check_rvalue_lvalue(); pipeline_check_private_fields_usage(); pipeline_check_pure_impure_operations(); + pipeline_check_constant_expressions(); pipeline_mini_borrow_checker_for_mutate(); - pipeline_constant_folding(); pipeline_optimize_boolean_expressions(); pipeline_detect_inline_in_place(); pipeline_check_serialized_fields(); diff --git a/tolk/tolk.h b/tolk/tolk.h index 5474397d5c..021445cc99 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -1066,21 +1066,12 @@ struct LazyVarRefAtCodegen { : var_ref(var_ref), var_state(var_state) {} }; -// CachedConstValueAtCodegen is used for a map [some_const => '5] -struct CachedConstValueAtCodegen { - GlobalConstPtr const_ref; - std::vector ir_idx; -}; - struct CodeBlob { int var_cnt, in_var_cnt; FunctionPtr fun_ref; - AnyV forced_origin = nullptr; std::vector vars; std::vector lazy_variables; - std::vector cached_consts; std::vector* inline_rvect_out = nullptr; - bool inside_evaluating_constant = false; bool inlining_before_immediate_return = false; std::unique_ptr ops; std::unique_ptr* cur_ops; @@ -1095,9 +1086,6 @@ struct CodeBlob { template Op& emplace_back(Args&&... args) { Op& res = *(*cur_ops = std::make_unique(args...)); - if (forced_origin) { - res.origin = forced_origin; - } cur_ops = &(res.next); #ifdef TOLK_DEBUG _vector_of_ops.push_back(&res); @@ -1135,7 +1123,6 @@ struct CodeBlob { } const LazyVariableLoadedState* get_lazy_variable(LocalVarPtr var_ref) const; const LazyVariableLoadedState* get_lazy_variable(AnyExprV v) const; - const CachedConstValueAtCodegen* get_cached_const(GlobalConstPtr const_ref) const; void prune_unreachable_code(); void fwd_analyze(); void mark_noreturn(); diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index 63930228e2..c3d24878ba 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -1041,7 +1041,7 @@ bool TypeDataStruct::can_be_casted_with_as_operator(TypePtr cast_to) const { } bool TypeDataEnum::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (cast_to == TypeDataInt::create()) { + if (cast_to == TypeDataInt::create() || cast_to == TypeDataCoins::create() || cast_to->try_as()) { return true; } if (cast_to->try_as()) { From 10b2b57a8e872cb92c3392a59fad9a71124d95f6 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 30 Oct 2025 22:56:01 +0300 Subject: [PATCH 457/524] [Tolk] Refactor: all AST visitors and replacers have current_function --- tolk/ast-replacer.h | 11 ++++++- tolk/ast-visitor.h | 15 +++++++-- tolk/pipe-ast-to-legacy.cpp | 12 +++----- tolk/pipe-check-const-expr.cpp | 2 +- tolk/pipe-check-inferred-types.cpp | 22 +++++--------- tolk/pipe-check-private-fields.cpp | 10 +----- tolk/pipe-check-pure-impure.cpp | 6 ---- tolk/pipe-check-rvalue-lvalue.cpp | 7 ----- tolk/pipe-check-serialized-fields.cpp | 7 +---- tolk/pipe-detect-inline-in-place.cpp | 30 ++++++++---------- tolk/pipe-lazy-load-insertions.cpp | 25 +-------------- tolk/pipe-mini-borrow-checker.cpp | 6 +--- tolk/pipe-optimize-boolean-expr.cpp | 5 ++- tolk/pipe-refine-lvalue-for-mutate.cpp | 6 ---- tolk/pipe-resolve-identifiers.cpp | 42 +++++++++----------------- tolk/pipe-resolve-types.cpp | 36 +++++++++------------- tolk/pipe-transform-on-message.cpp | 22 ++++++-------- 17 files changed, 94 insertions(+), 170 deletions(-) diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 6b4a08f054..f3872678ce 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -91,6 +91,8 @@ class ASTReplacerInFunctionBody : public ASTReplacer { protected: using parent = ASTReplacerInFunctionBody; + FunctionPtr cur_f = nullptr; + // expressions virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } @@ -204,8 +206,15 @@ class ASTReplacerInFunctionBody : public ASTReplacer { public: virtual bool should_visit_function(FunctionPtr fun_ref) = 0; - virtual void start_replacing_in_function(FunctionPtr fun_ref, V v_function) { + virtual void on_enter_function(V v_function) {} + virtual void on_exit_function(V v_function) {} + + void start_replacing_in_function(FunctionPtr fun_ref, V v_function) { + cur_f = fun_ref; + on_enter_function(v_function); replace(v_function->get_body()); + on_exit_function(v_function); + cur_f = nullptr; } }; diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index dc18496b8d..d750d6ba28 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -100,6 +100,8 @@ class ASTVisitorFunctionBody : public ASTVisitor { protected: using parent = ASTVisitorFunctionBody; + FunctionPtr cur_f = nullptr; + // expressions virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } @@ -205,8 +207,17 @@ class ASTVisitorFunctionBody : public ASTVisitor { public: virtual bool should_visit_function(FunctionPtr fun_ref) = 0; - virtual void start_visiting_function(FunctionPtr fun_ref, V v_function) { - visit(v_function->get_body()); + virtual void on_enter_function(V v_function) {} + virtual void on_exit_function(V v_function) {} + + void start_visiting_function(FunctionPtr fun_ref, V v_function) { + cur_f = fun_ref; + on_enter_function(v_function); + if (V v_body_block = v_function->get_body()->try_as()) { + visit(v_body_block); + } + on_exit_function(v_function); + cur_f = nullptr; } }; diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 3426860449..108a8fcf70 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -290,7 +290,6 @@ class LValContext { class CheckReorderingForAsmArgOrderIsSafeVisitor final : public ASTVisitorFunctionBody { bool has_side_effects = false; -protected: void visit(V v) override { has_side_effects |= v->fun_maybe == nullptr || !v->fun_maybe->is_marked_as_pure() || v->fun_maybe->has_mutate_params(); parent::visit(v); @@ -341,6 +340,7 @@ class CheckReorderingForAsmArgOrderIsSafeVisitor final : public ASTVisitorFuncti // that should be cleared upon next inlining; // for instance, ir_idx of local variables point to caller (where f was inlined) class ClearStateAfterInlineInPlace final : public ASTVisitorFunctionBody { + void visit(V v) override { if (!v->marked_as_redef) { v->var_ref->mutate()->assign_ir_idx({}); @@ -352,14 +352,12 @@ class ClearStateAfterInlineInPlace final : public ASTVisitorFunctionBody { tolk_assert(false); } - void start_visiting_function(FunctionPtr fun_ref, V v_function) override { - tolk_assert(fun_ref->is_inlined_in_place()); + void on_enter_function(V v_function) override { + tolk_assert(cur_f->is_inlined_in_place()); - for (int i = 0; i < fun_ref->get_num_params(); ++i) { - fun_ref->get_param(i).mutate()->assign_ir_idx({}); + for (int i = 0; i < cur_f->get_num_params(); ++i) { + cur_f->get_param(i).mutate()->assign_ir_idx({}); } - - parent::visit(v_function->get_body()); } }; diff --git a/tolk/pipe-check-const-expr.cpp b/tolk/pipe-check-const-expr.cpp index 86c221411a..e1028dea4f 100644 --- a/tolk/pipe-check-const-expr.cpp +++ b/tolk/pipe-check-const-expr.cpp @@ -30,7 +30,7 @@ namespace tolk { class ConstantExpressionsChecker final : public ASTVisitorFunctionBody { -protected: + void visit(V v) override { // check `ton("0.05")` and others for correctness (not `ton(local_var)`, etc.) if (v->fun_maybe && v->fun_maybe->is_compile_time_const_val()) { diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index eeb7bbf9e6..c69eff3cfc 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -201,9 +201,7 @@ static bool check_eq_neq_operator(TypePtr lhs_type, TypePtr rhs_type, bool& not_ } class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { - FunctionPtr cur_f = nullptr; // may be nullptr if checking `const a = ...` init_value -protected: void visit(V v) override { AnyExprV lhs = v->get_lhs(); AnyExprV rhs = v->get_rhs(); @@ -788,30 +786,26 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } - public: +public: bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } - void start_visiting_function(FunctionPtr fun_ref, V v_function) override { - cur_f = fun_ref; - parent::visit(v_function->get_body()); - cur_f = nullptr; - - if (fun_ref->is_implicit_return() && fun_ref->declared_return_type) { - if (!fun_ref->declared_return_type->can_rhs_be_assigned(TypeDataVoid::create()) || fun_ref->does_return_self()) { - err("missing return").fire(SrcRange::empty_at_end(v_function->get_body()->range), fun_ref); + void on_exit_function(V v_function) override { + if (cur_f->is_implicit_return() && cur_f->declared_return_type) { + if (!cur_f->declared_return_type->can_rhs_be_assigned(TypeDataVoid::create()) || cur_f->does_return_self()) { + err("missing return").fire(SrcRange::empty_at_end(v_function->get_body()->range), cur_f); } } // visit default values of parameters - for (int i = 0; i < fun_ref->get_num_params(); ++i) { - if (LocalVarPtr param_ref = &fun_ref->get_param(i); param_ref->has_default_value()) { + for (int i = 0; i < cur_f->get_num_params(); ++i) { + if (LocalVarPtr param_ref = &cur_f->get_param(i); param_ref->has_default_value()) { parent::visit(param_ref->default_value); TypePtr inferred_type = param_ref->default_value->inferred_type; if (!param_ref->declared_type->can_rhs_be_assigned(inferred_type)) { - err_type_mismatch("can not assign {src} to {dst}", inferred_type, param_ref->declared_type).fire(param_ref->default_value, fun_ref); + err_type_mismatch("can not assign {src} to {dst}", inferred_type, param_ref->declared_type).fire(param_ref->default_value, cur_f); } } } diff --git a/tolk/pipe-check-private-fields.cpp b/tolk/pipe-check-private-fields.cpp index d6b9898227..e9468500bb 100644 --- a/tolk/pipe-check-private-fields.cpp +++ b/tolk/pipe-check-private-fields.cpp @@ -45,9 +45,7 @@ static bool is_private_field_usage_allowed(FunctionPtr cur_f, StructPtr struct_r } class CheckPrivateFieldsUsageVisitor final : public ASTVisitorFunctionBody { - FunctionPtr cur_f = nullptr; -protected: void visit(V v) override { parent::visit(v); @@ -74,16 +72,10 @@ class CheckPrivateFieldsUsageVisitor final : public ASTVisitorFunctionBody { } } - public: +public: bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } - - void start_visiting_function(FunctionPtr fun_ref, V v_function) override { - cur_f = fun_ref; - parent::visit(v_function->get_body()); - cur_f = nullptr; - } }; void pipeline_check_private_fields_usage() { diff --git a/tolk/pipe-check-pure-impure.cpp b/tolk/pipe-check-pure-impure.cpp index 8e0e1db0ba..002928ea5c 100644 --- a/tolk/pipe-check-pure-impure.cpp +++ b/tolk/pipe-check-pure-impure.cpp @@ -30,7 +30,6 @@ static Error err_impure_operation_inside_pure_function() { } class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFunctionBody { - FunctionPtr cur_f = nullptr; void fire_if_global_var(AnyExprV v) const { if (auto v_ident = v->try_as()) { @@ -84,11 +83,6 @@ class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFuncti bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function() && fun_ref->is_marked_as_pure(); } - - void start_visiting_function(FunctionPtr fun_ref, V v_function) override { - cur_f = fun_ref; - parent::visit(v_function->get_body()); - } }; void pipeline_check_pure_impure_operations() { diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp index f2bf79ea6e..45909a2ec8 100644 --- a/tolk/pipe-check-rvalue-lvalue.cpp +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -59,7 +59,6 @@ static void validate_function_used_as_noncall(FunctionPtr cur_f, AnyExprV v, Fun } class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { - FunctionPtr cur_f = nullptr; void on_var_used_as_lvalue(SrcRange range, LocalVarPtr var_ref) const { if (var_ref->is_immutable()) { @@ -276,12 +275,6 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } - - void start_visiting_function(FunctionPtr fun_ref, V v_function) override { - cur_f = fun_ref; - parent::visit(v_function->get_body()); - cur_f = nullptr; - } }; void pipeline_check_rvalue_lvalue() { diff --git a/tolk/pipe-check-serialized-fields.cpp b/tolk/pipe-check-serialized-fields.cpp index 3ee0f0582c..d49cc91350 100644 --- a/tolk/pipe-check-serialized-fields.cpp +++ b/tolk/pipe-check-serialized-fields.cpp @@ -80,7 +80,6 @@ static void check_enum_colon_type_to_be_intN(AnyTypeV colon_type_node) { class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody { - FunctionPtr cur_f = nullptr; static void check_type_fits_cell_or_has_policy(TypePtr serialized_type) { if (const TypeDataStruct* s_struct = serialized_type->unwrap_alias()->try_as()) { @@ -151,14 +150,10 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } - void start_visiting_function(FunctionPtr fun_ref, V v_function) override { - cur_f = fun_ref; - + void on_enter_function(V v_function) override { for (int i = 0; i < cur_f->get_num_params(); ++i) { check_mapKV_inside_type(cur_f->get_param(i).type_node); } - - parent::visit(v_function->get_body()); } }; diff --git a/tolk/pipe-detect-inline-in-place.cpp b/tolk/pipe-detect-inline-in-place.cpp index 00bc4b618b..cd7b88f94e 100644 --- a/tolk/pipe-detect-inline-in-place.cpp +++ b/tolk/pipe-detect-inline-in-place.cpp @@ -111,12 +111,11 @@ struct StateWhileTraversingFunction { }; // traverse the AST, collect metrics, and in the end, probably set the inline flag -class DetectIfToInlineFunctionInPlaceVisitor final : ASTVisitorFunctionBody { +class DetectIfToInlineFunctionInPlaceVisitor final : public ASTVisitorFunctionBody { StateWhileTraversingFunction cur_state{nullptr}; int block_depth = 0; std::vector> collected_expect_inline; // `__expect_inline()` compiler assertions -protected: void visit(V v) override { if (v->fun_maybe && v->fun_maybe->is_builtin() && v->fun_maybe->name == "__expect_inline") { collected_expect_inline.push_back(v); @@ -198,7 +197,7 @@ class DetectIfToInlineFunctionInPlaceVisitor final : ASTVisitorFunctionBody { parent::visit(v); } - public: +public: bool should_visit_function(FunctionPtr fun_ref) override { // unsupported or no-sense cases if (fun_ref->is_builtin() || fun_ref->is_asm_function() || fun_ref->is_generic_function() || @@ -215,14 +214,15 @@ class DetectIfToInlineFunctionInPlaceVisitor final : ASTVisitorFunctionBody { return true; } - void start_visiting_function(FunctionPtr fun_ref, V v_function) override { - cur_state = StateWhileTraversingFunction(fun_ref); + void on_enter_function(V v_function) override { + cur_state = StateWhileTraversingFunction(cur_f); collected_expect_inline.clear(); - parent::visit(v_function->get_body()); + } + void on_exit_function(V v_function) override { bool prevented_anyway = cur_state.is_inlining_prevented_even_if_annotated(); bool will_inline = false; - if (fun_ref->inline_mode == FunctionInlineMode::inlineViaFif) { + if (cur_f->inline_mode == FunctionInlineMode::inlineViaFif) { // if a function is marked `@inline`, so the user requested in to be inlined; // if it's possible, do it; otherwise, leave it as `PROCINLINE` to Fift will_inline = !prevented_anyway; @@ -236,14 +236,14 @@ class DetectIfToInlineFunctionInPlaceVisitor final : ASTVisitorFunctionBody { tolk_assert(v_expect->get_num_args() == 1 && v_expect->get_arg(0)->get_expr()->kind == ast_bool_const); bool expected = v_expect->get_arg(0)->get_expr()->as()->bool_val; if (expected != will_inline) { - err("__expect_inline failed").fire(v_expect, fun_ref); + err("__expect_inline failed").fire(v_expect, cur_f); } } // okay, this function will be inlined, mark the flag - bool is_called = fun_ref->n_times_called || is_called_implicitly_by_compiler(fun_ref); + bool is_called = cur_f->n_times_called || is_called_implicitly_by_compiler(cur_f); if (will_inline && is_called) { - fun_ref->mutate()->assign_inline_mode_in_place(); + cur_f->mutate()->assign_inline_mode_in_place(); } } }; @@ -253,10 +253,8 @@ class DetectIfToInlineFunctionInPlaceVisitor final : ASTVisitorFunctionBody { // 2) increments n_times_called // as a result of applying it to every function, we get a full call graph and how many times each function was called; // we'll use this call graph to detect recursive components (functions within recursions can not be inlined) -class CallGraphBuilderVisitor final : ASTVisitorFunctionBody { - FunctionPtr cur_f{nullptr}; +class CallGraphBuilderVisitor final : public ASTVisitorFunctionBody { -protected: void visit(V v) override { if (FunctionPtr called_f = v->fun_maybe) { if (called_f->is_code_function()) { @@ -267,16 +265,14 @@ class CallGraphBuilderVisitor final : ASTVisitorFunctionBody { parent::visit(v); } - public: +public: bool should_visit_function(FunctionPtr fun_ref) override { // don't include asm functions, we don't need them in calculations return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } - void start_visiting_function(FunctionPtr fun_ref, V v_function) override { - cur_f = fun_ref; + void on_enter_function(V v_function) override { call_graph[cur_f] = std::vector{}; - parent::visit(v_function->get_body()); } }; diff --git a/tolk/pipe-lazy-load-insertions.cpp b/tolk/pipe-lazy-load-insertions.cpp index e1db8ddbd2..0e824692f3 100644 --- a/tolk/pipe-lazy-load-insertions.cpp +++ b/tolk/pipe-lazy-load-insertions.cpp @@ -605,7 +605,7 @@ static ExprUsagesWhileCollecting collect_expr_usages_in_block(std::string name_f // This visitor finds usages of "v" / "v.field" / etc. in ONE statement or expression and populates lazy_expr data. // For every struct, all its fields are also populated; for a union — all its variants. // Since AST vertices don't have "parent_node", we need to remember some details while traversing top-down. -class CollectUsagesInStatementVisitor final : ASTVisitorFunctionBody { +class CollectUsagesInStatementVisitor final : public ASTVisitorFunctionBody { AnyV cur_stmt; SinkExpression s_expr; ExprUsagesWhileCollecting* lazy_expr; @@ -614,7 +614,6 @@ class CollectUsagesInStatementVisitor final : ASTVisitorFunctionBody { CollectUsagesInStatementVisitor(AnyV cur_stmt, SinkExpression s_expr, ExprUsagesWhileCollecting* lazy_expr) : cur_stmt(cur_stmt), s_expr(s_expr), lazy_expr(lazy_expr) {} -protected: void visit(V v) override { if (extract_sink_expression_from_vertex(v) == s_expr) { bool is_subj_of_dot = parent_dot && parent_dot->is_target_struct_field() && parent_dot->get_obj() == v; @@ -822,10 +821,8 @@ static ExprUsagesWhileCollecting collect_expr_usages_in_block(std::string name_f // This visitor finds `var st = lazy expr`, launches finding usages for `st`, // and adds `st` as LazyVarInFunction to a global list. class CollectAllLazyObjectsAndFieldsVisitor final : public ASTVisitorFunctionBody { - FunctionPtr cur_f = nullptr; V parent_block = nullptr; -protected: void visit(V v) override { auto backup = parent_block; parent_block = v; @@ -870,11 +867,6 @@ class CollectAllLazyObjectsAndFieldsVisitor final : public ASTVisitorFunctionBod bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } - - void start_visiting_function(FunctionPtr fun_ref, V v_function) override { - cur_f = fun_ref; - parent::visit(v_function->get_body()); - } }; // Step 2: @@ -882,10 +874,7 @@ class CollectAllLazyObjectsAndFieldsVisitor final : public ASTVisitorFunctionBod // inserting (already calculated) load vertices. They are auxiliary vertices holding special data. // They are handled later when transforming AST to Ops. class LazyLoadInsertionsReplacer final : public ASTReplacerInFunctionBody { - FunctionPtr cur_f = nullptr; - V f_body = nullptr; -protected: // `var st = lazy expr` -> save "st" (it will be used in codegen to assert "st.x" that "x" is loaded) AnyExprV replace(V v) override { for (const LazyVarInFunction& lazy_var : functions_with_lazy_vars[cur_f]) { @@ -944,19 +933,12 @@ class LazyLoadInsertionsReplacer final : public ASTReplacerInFunctionBody { bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && functions_with_lazy_vars.contains(fun_ref); } - - void start_replacing_in_function(FunctionPtr fun_ref, V v_function) override { - cur_f = fun_ref; - f_body = v_function->get_body()->as(); - parent::replace(v_function->get_body()); - } }; // Step 3: // After modifying AST (inserting loads, lazy match, etc.), // check __expect_lazy() calls, used in compiler tests as assertions. class CheckExpectLazyAssertionsVisitor final : public ASTVisitorFunctionBody { - FunctionPtr cur_f = nullptr; static std::string stringify_lazy_load_above_stmt(const AuxData_LazyObjectLoadFields* aux_load) { static const char* action_to_str[] = {"load", "skip", "lazy match", "save immutable"}; @@ -1016,11 +998,6 @@ class CheckExpectLazyAssertionsVisitor final : public ASTVisitorFunctionBody { bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && functions_with_lazy_vars.contains(fun_ref); } - - void start_visiting_function(FunctionPtr fun_ref, V v_function) override { - cur_f = fun_ref; - parent::visit(v_function->get_body()); - } }; void pipeline_lazy_load_insertions() { diff --git a/tolk/pipe-mini-borrow-checker.cpp b/tolk/pipe-mini-borrow-checker.cpp index d91995f8b6..195b4bd882 100644 --- a/tolk/pipe-mini-borrow-checker.cpp +++ b/tolk/pipe-mini-borrow-checker.cpp @@ -99,10 +99,8 @@ class BorrowedForWriteCtx { }; class CheckMutationNotHappensTwiceVisitor final : public ASTVisitorFunctionBody { - FunctionPtr cur_f = nullptr; BorrowedForWriteCtx borrow_ctx; -protected: void visit(V v) override { FunctionPtr fun_ref = v->fun_maybe; if (!fun_ref) { // a "call" to a variable, it can't be mutating @@ -192,9 +190,7 @@ class CheckMutationNotHappensTwiceVisitor final : public ASTVisitorFunctionBody return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } - void start_visiting_function(FunctionPtr fun_ref, V v_function) override { - cur_f = fun_ref; - parent::visit(v_function->get_body()); + void on_exit_function(V v_function) override { tolk_assert(borrow_ctx.empty()); } }; diff --git a/tolk/pipe-optimize-boolean-expr.cpp b/tolk/pipe-optimize-boolean-expr.cpp index f4a0930259..8cb0050f9c 100644 --- a/tolk/pipe-optimize-boolean-expr.cpp +++ b/tolk/pipe-optimize-boolean-expr.cpp @@ -36,7 +36,8 @@ namespace tolk { -struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { +class OptimizerBooleanExpressionsReplacer final : public ASTReplacerInFunctionBody { + static V create_int_const(SrcRange range, td::RefInt256&& intval) { auto v_int = createV(range, std::move(intval), {}); v_int->assign_inferred_type(TypeDataInt::create()); @@ -82,8 +83,6 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { return false; } -protected: - AnyExprV replace(V v) override { parent::replace(v); diff --git a/tolk/pipe-refine-lvalue-for-mutate.cpp b/tolk/pipe-refine-lvalue-for-mutate.cpp index 3c4fbc0732..f178b2e131 100644 --- a/tolk/pipe-refine-lvalue-for-mutate.cpp +++ b/tolk/pipe-refine-lvalue-for-mutate.cpp @@ -47,7 +47,6 @@ static Error err_invalid_mutate_arg_passed(FunctionPtr fun_ref, const LocalVarDa class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBody { - FunctionPtr cur_f = nullptr; void visit(V v) override { // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` @@ -104,11 +103,6 @@ class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBod bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } - - void start_visiting_function(FunctionPtr fun_ref, V v_function) override { - cur_f = fun_ref; - parent::visit(v_function->get_body()); - } }; void pipeline_refine_lvalue_for_mutate_arguments() { diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 7d8fab0dcc..9ad216b47c 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -111,24 +111,21 @@ struct NameAndScopeResolver { }; class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { - // more correctly this field shouldn't be static, but currently there is no need to make it a part of state - static NameAndScopeResolver current_scope; - static FunctionPtr cur_f; + NameAndScopeResolver current_scope; - static LocalVarPtr create_local_var_sym(std::string_view name, AnyV ident_anchor, AnyTypeV declared_type_node, bool immutable, bool lateinit) { + LocalVarPtr create_local_var_sym(std::string_view name, AnyV ident_anchor, AnyTypeV declared_type_node, bool immutable, bool lateinit) { LocalVarData* v_sym = new LocalVarData(static_cast(name), ident_anchor, declared_type_node, nullptr, immutable * LocalVarData::flagImmutable + lateinit * LocalVarData::flagLateInit, -1); current_scope.add_local_var(v_sym); return v_sym; } - static void process_catch_variable(AnyExprV catch_var) { + void process_catch_variable(AnyExprV catch_var) { if (auto v_ref = catch_var->try_as()) { LocalVarPtr var_ref = create_local_var_sym(v_ref->get_name(), catch_var, nullptr, true, false); v_ref->mutate()->assign_sym(var_ref); } } -protected: void visit(V v) override { if (v->marked_as_redef) { const Symbol* sym = current_scope.lookup_symbol(v->get_name()); @@ -226,10 +223,17 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { } void visit(V v) override { - if (v->empty()) { - return; - } current_scope.open_scope(); + if (v == cur_f->ast_root->as()->get_body()) { + for (int i = 0; i < cur_f->get_num_params(); ++i) { + LocalVarPtr param_ref = &cur_f->parameters[i]; + current_scope.add_local_var(param_ref); + if (param_ref->has_default_value()) { + parent::visit(param_ref->default_value); + } + } + } + parent::visit(v); current_scope.close_scope(); } @@ -257,23 +261,8 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { return fun_ref->is_code_function(); } - void start_visiting_function(FunctionPtr fun_ref, V v) override { - cur_f = fun_ref; - - auto v_block = v->get_body()->as(); - current_scope.open_scope(); - for (int i = 0; i < fun_ref->get_num_params(); ++i) { - LocalVarPtr param_ref = &fun_ref->parameters[i]; - current_scope.add_local_var(param_ref); - if (param_ref->has_default_value()) { - parent::visit(param_ref->default_value); - } - } - parent::visit(v_block); - current_scope.close_scope(); + void on_exit_function(V v_function) override { tolk_assert(current_scope.scopes.empty()); - - cur_f = nullptr; } void start_visiting_constant(GlobalConstPtr const_ref) { @@ -300,9 +289,6 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { } }; -NameAndScopeResolver AssignSymInsideFunctionVisitor::current_scope; -FunctionPtr AssignSymInsideFunctionVisitor::cur_f = nullptr; - void pipeline_resolve_identifiers_and_assign_symbols() { AssignSymInsideFunctionVisitor visitor; for (const SrcFile* file : G.all_src_files) { diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 1a675ca655..f3bf0f67e4 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -455,8 +455,6 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { return type_nodes_visitor.finalize_type_node(type_node, allow_without_type_arguments); } -protected: - void visit(V v) override { if (v->type_node) { TypePtr declared_type = finalize_type_node(v->type_node); @@ -541,40 +539,36 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { return !fun_ref->is_builtin(); } - void start_visiting_function(FunctionPtr fun_ref, V v) override { - if (fun_ref->receiver_type_node) { - TypeNodesVisitorResolver receiver_visitor(fun_ref, fun_ref->genericTs, fun_ref->substitutedTs, true); - TypePtr receiver_type = receiver_visitor.finalize_type_node(fun_ref->receiver_type_node); + void on_enter_function(V v) override { + if (cur_f->receiver_type_node) { + TypeNodesVisitorResolver receiver_visitor(cur_f, cur_f->genericTs, cur_f->substitutedTs, true); + TypePtr receiver_type = receiver_visitor.finalize_type_node(cur_f->receiver_type_node); std::string name_prefix = receiver_type->as_human_readable(); bool embrace = receiver_type->try_as() && !receiver_type->try_as()->or_null; if (embrace) { name_prefix = "(" + name_prefix + ")"; } - fun_ref->mutate()->assign_resolved_receiver_type(receiver_type, std::move(name_prefix)); - G.symtable.add_function(fun_ref); + cur_f->mutate()->assign_resolved_receiver_type(receiver_type, std::move(name_prefix)); + G.symtable.add_function(cur_f); } - if (v->genericsT_list || (fun_ref->receiver_type && fun_ref->receiver_type->has_genericT_inside())) { - const GenericsDeclaration* genericTs = TypeNodesVisitorResolver::construct_genericTs(fun_ref->receiver_type, v->genericsT_list); - fun_ref->mutate()->assign_resolved_genericTs(genericTs); + if (v->genericsT_list || (cur_f->receiver_type && cur_f->receiver_type->has_genericT_inside())) { + const GenericsDeclaration* genericTs = TypeNodesVisitorResolver::construct_genericTs(cur_f->receiver_type, v->genericsT_list); + cur_f->mutate()->assign_resolved_genericTs(genericTs); } - type_nodes_visitor = TypeNodesVisitorResolver(fun_ref, fun_ref->genericTs, fun_ref->substitutedTs, false); + type_nodes_visitor = TypeNodesVisitorResolver(cur_f, cur_f->genericTs, cur_f->substitutedTs, false); - for (int i = 0; i < fun_ref->get_num_params(); ++i) { - LocalVarPtr param_ref = &fun_ref->parameters[i]; + for (int i = 0; i < cur_f->get_num_params(); ++i) { + LocalVarPtr param_ref = &cur_f->parameters[i]; TypePtr declared_type = finalize_type_node(param_ref->type_node); param_ref->mutate()->assign_resolved_type(declared_type); if (param_ref->has_default_value()) { parent::visit(param_ref->default_value); } } - if (fun_ref->return_type_node) { - TypePtr declared_return_type = finalize_type_node(fun_ref->return_type_node); - fun_ref->mutate()->assign_resolved_type(declared_return_type); - } - - if (fun_ref->is_code_function()) { - parent::visit(v->get_body()->as()); + if (cur_f->return_type_node) { + TypePtr declared_return_type = finalize_type_node(cur_f->return_type_node); + cur_f->mutate()->assign_resolved_type(declared_return_type); } } diff --git a/tolk/pipe-transform-on-message.cpp b/tolk/pipe-transform-on-message.cpp index 52e5762be1..a713039ca1 100644 --- a/tolk/pipe-transform-on-message.cpp +++ b/tolk/pipe-transform-on-message.cpp @@ -51,8 +51,7 @@ static bool is_onBouncedMessage(FunctionPtr fun_ref) { } -struct TransformOnInternalMessageReplacer final : ASTReplacerInFunctionBody { - FunctionPtr cur_f = nullptr; +class TransformOnInternalMessageReplacer final : public ASTReplacerInFunctionBody { LocalVarPtr param_ref = nullptr; // `in` for `fun onInternalMessage(in: InMessage)` static void validate_onBouncedMessage(FunctionPtr f) { @@ -68,7 +67,6 @@ struct TransformOnInternalMessageReplacer final : ASTReplacerInFunctionBody { } } -protected: AnyExprV replace(V v) override { // don't allow `var v = in` or passing `in` to another function (only `in.someField` is allowed) if (v->sym == param_ref) { @@ -96,19 +94,17 @@ struct TransformOnInternalMessageReplacer final : ASTReplacerInFunctionBody { return is_onInternalMessage(fun_ref) || is_onBouncedMessage(fun_ref); } - void start_replacing_in_function(FunctionPtr fun_ref, V v_function) override { - if (fun_ref->name == "onBouncedMessage") { - validate_onBouncedMessage(fun_ref); + void on_enter_function(V v_function) override { + if (cur_f->name == "onBouncedMessage") { + validate_onBouncedMessage(cur_f); } + param_ref = &cur_f->parameters[0]; + } - cur_f = fun_ref; - param_ref = &fun_ref->parameters[0]; - - parent::replace(v_function->get_body()); - + void on_exit_function(V v_function) override { std::vector new_parameters; - new_parameters.emplace_back("in.body", fun_ref->ident_anchor, TypeDataSlice::create(), nullptr, 0, 0); - fun_ref->mutate()->parameters = std::move(new_parameters); + new_parameters.emplace_back("in.body", cur_f->ident_anchor, TypeDataSlice::create(), nullptr, 0, 0); + cur_f->mutate()->parameters = std::move(new_parameters); } }; From 3aa9dd5ee1296c696fa8bb65d379ed4b95d5c651 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sat, 1 Nov 2025 13:40:34 +0300 Subject: [PATCH 458/524] [Tolk] Better deal with exhaustive `match`, enums, and malformed stack Before, `match (enum)` covering all enum members, was just compiled as IF ELSE, the last ELSE was a fallback for invalid (malformed) enum members also. In practice, this can occur only if a programmer intentionally disables compiler checks or corrupts the stack via asm functions. Anyway, I decided to add an extra IF, so that if a stack contains an invalid value, excno 5 was thrown. --- tolk-tester/tests/enums-tests.tolk | 24 ++-- tolk-tester/tests/match-by-expr-tests.tolk | 89 +++++++++++++++ .../unreachable-5.tolk | 0 .../warnings-not-errors/unreachable-6.tolk | 40 +++++++ tolk/ast.cpp | 4 + tolk/ast.h | 5 + tolk/pipe-ast-to-legacy.cpp | 103 ++++++++++-------- tolk/pipe-check-inferred-types.cpp | 16 +-- tolk/pipe-infer-types-and-calls.cpp | 27 ++++- 9 files changed, 234 insertions(+), 74 deletions(-) rename tolk-tester/tests/{ => warnings-not-errors}/unreachable-5.tolk (100%) create mode 100644 tolk-tester/tests/warnings-not-errors/unreachable-6.tolk diff --git a/tolk-tester/tests/enums-tests.tolk b/tolk-tester/tests/enums-tests.tolk index e3d8903ce2..c88a724c56 100644 --- a/tolk-tester/tests/enums-tests.tolk +++ b/tolk-tester/tests/enums-tests.tolk @@ -70,11 +70,12 @@ fun checkIsColor(obj: Color | T) { return obj is Color } @inline_ref fun Color.mapToNumber(self) { - // match over enum is exhaustive, it's like "if Red {} else if Green {} else {}" + // match over enum is exhaustive, it's like "if Red {} else if Green {} else if Blue {} else throw" return match (self) { Color.Red => 100, Color.Green => 200, - Color.Blue => 300, // this works like `else` and handles invalid values at runtime also + Color.Blue => 300, + // there is a hidden "else" in case stack is malformed } } @@ -157,10 +158,13 @@ fun test8() { @method_id(109) fun test9() { + var invalidEx = 0; + try { invalidEx = (99 as Color).mapToNumber() } + catch (ex) { invalidEx = ex } + return ( Color.Red.mapToNumber(), Color.Green.mapToNumber(), Color.Blue.mapToNumber(), - // this is UB, the compiler may do any optimizations - (100 as Color).mapToNumber(), (-99 as Color).mapToNumber(), + invalidEx ); } @@ -224,7 +228,7 @@ fun test12(c1: Color, c2: Color) { Color.Blue => match (c1) { Color.Green => t.push(210), Color.Red => t.push(110), - Color.Blue => t.push(310), + else => t.push(310), }, Color.Green => match (c1) { Color.Green => t.push(220), @@ -233,7 +237,8 @@ fun test12(c1: Color, c2: Color) { } } - t.push((999 as Color).mapToNumber()); + try { t.push(123); t.push((999 as Color).mapToNumber()); } + catch {} return t; } @@ -277,14 +282,13 @@ fun main(c: Color) { @testcase | 106 | 1 | 1 1 1 1 100 -100 @testcase | 107 | | 0 1 -100 2 @testcase | 108 | | -1 0 -1 0 0 -1 0 typeid-1 -200 -@testcase | 109 | | 100 200 300 300 300 +@testcase | 109 | | 100 200 300 5 @testcase | 110 | 0 | [ 1 1 -100 88 -100 1 ] @testcase | 110 | 1 | [ -100 2 -100 88 2 2 ] @testcase | 110 | 2 | [ -100 3 -100 88 3 3 ] -@testcase | 110 | 44 | [ -100 1 -100 88 -100 2 ] @testcase | 111 | 2 | 0 -1 -1 0 -@testcase | 112 | 0 1 | [ 210 120 300 ] -@testcase | 112 | 2 4 | [ 330 320 300 ] +@testcase | 112 | 0 1 | [ 210 120 ] +@testcase | 112 | 2 2 | [ 330 310 ] @testcase | 113 | -8 | 256 -1 @testcase | 113 | 111 | 257 0 @testcase | 113 | 10 | 258 0 diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk index de748cfc26..df09d51bc2 100644 --- a/tolk-tester/tests/match-by-expr-tests.tolk +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -1,6 +1,13 @@ const C1 = 1; const C2 = C1 + 1; +const zero_addr = address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"); + +enum Role { + User, + Admin, +} + @method_id(101) fun test1(x: int) { return match (x) { @@ -137,6 +144,74 @@ fun test12(x: int) { return g12 } +fun invalidAsmRole(): Role asm "99 PUSHINT" +fun invalidAsmBool(): bool asm "99 PUSHINT" + +@noinline +fun mapRole(r: Role): int { + return match (r) { + Role.User => 100, + Role.Admin => 200, + } + // no "missing return", match is exhaustive +} + +@noinline +fun mapBool(b: bool): int { + return match (b) { + true => 100, + false => 200, + } + // no "missing return", match is exhaustive +} + +@method_id(113) +fun test13() { + var ex1 = 0; + var ex2 = 0; + + // will throw (the compiler generates "else" implicitly for enums) + try { ex1 = mapRole(invalidAsmRole()); } + catch (ex) { ex1 = ex } + + // will not throw (the compiler compiles "match" for bool as true => else =>) + try { ex2 = mapBool(invalidAsmBool()); } + catch (ex) { ex2 = ex } + + return (ex1, ex2); +} + +@method_id(114) +fun test14(case: int) { + return match (match (case) { + 1 => address("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"), + 2 => address("Ef89xh-uy860-mCcvS8zcAUs8bApmxLGygDLEKjUk5RL-311"), + else => zero_addr, + }) { + address("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf") => 10, + address("Ef89xh-uy860-mCcvS8zcAUs8bApmxLGygDLEKjUk5RL-311") => 20, + zero_addr => 30, + else => throw 123, + } +} + +@method_id(115) +fun test15(useNone: bool) { + var addr = (address("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf") as any_address).toCell().load(); + if (useNone) { + addr = createAddressNone(); + } + match (addr) { + address("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf") => + return match (addr) { + address("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf") => 1, + else => 2 + } + } + return createAddressNone() == addr ? null : 100; +} + + type asdf = int; @@ -178,6 +253,11 @@ fun main() { @testcase | 111 | -1 | 0 @testcase | 112 | 2 | 2 @testcase | 112 | 20 | 20 +@testcase | 113 | | 5 200 +@testcase | 114 | 2 | 20 +@testcase | 114 | 8 | 30 +@testcase | 115 | 0 | 1 +@testcase | 115 | -1 | (null) @fif_codegen """ @@ -261,4 +341,13 @@ fun main() { }> """ +@fif_codegen +""" + 1 EQINT // '6 + IFNOTJMP:<{ // + 5 THROW + }> // + 200 PUSHINT // '1=200 +""" + */ diff --git a/tolk-tester/tests/unreachable-5.tolk b/tolk-tester/tests/warnings-not-errors/unreachable-5.tolk similarity index 100% rename from tolk-tester/tests/unreachable-5.tolk rename to tolk-tester/tests/warnings-not-errors/unreachable-5.tolk diff --git a/tolk-tester/tests/warnings-not-errors/unreachable-6.tolk b/tolk-tester/tests/warnings-not-errors/unreachable-6.tolk new file mode 100644 index 0000000000..9471a88161 --- /dev/null +++ b/tolk-tester/tests/warnings-not-errors/unreachable-6.tolk @@ -0,0 +1,40 @@ +//unreachable-6 +enum Role { + User, + Admin, +} + +@method_id(101) +fun checkRole(r: Role) { + match (r) { + Role.User => { + return 1; + } + Role.Admin => { + return 100; + } + } + throw 123; +} + +@method_id(102) +fun checkBool(b: bool) { + match (b) { + true => return 1, + false => return 100, + } + throw 456; +} + +fun main() {} + +/** +@testcase | 101 | 1 | 100 +@testcase | 102 | -1 | 1 +@testcase | 102 | 0 | 100 +@testcase | 102 | 1 | 100 + +@stderr warning: unreachable code +@stderr throw 123; +@stderr throw 456; + */ diff --git a/tolk/ast.cpp b/tolk/ast.cpp index c74039e186..8905bfc6b1 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -151,6 +151,10 @@ void Vertex::assign_dest_var_ref(LocalVarPtr dest_var_ref) { this->dest_var_ref = dest_var_ref; } +void Vertex::assign_is_exhaustive(bool is_exhaustive) { + this->is_exhaustive = is_exhaustive; +} + void Vertex::assign_resolved_pattern(MatchArmKind pattern_kind, AnyExprV pattern_expr) { this->pattern_type_node = nullptr; this->pattern_kind = pattern_kind; diff --git a/tolk/ast.h b/tolk/ast.h index e143291118..f1d2deec07 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -914,6 +914,8 @@ template<> // example: `match (intOrSliceVar) { int => 1, slice => 2 }` // example: `match (var c = getIntOrSlice()) { int => return 0, slice => throw 123 }` struct Vertex final : ASTExprVararg { + bool is_exhaustive = false; // if it has `else` or covers all cases without `else`; can be used as expression + AnyExprV get_subject() const { return child(0); } int get_arms_count() const { return size() - 1; } auto get_arm(int i) const { return child(i + 1)->as(); } @@ -922,6 +924,9 @@ struct Vertex final : ASTExprVararg { bool is_statement() const { return !is_rvalue && !is_lvalue; } + Vertex* mutate() const { return const_cast(this); } + void assign_is_exhaustive(bool is_exhaustive); + Vertex(SrcRange range, std::vector&& subject_and_arms) : ASTExprVararg(ast_match_expression, range, std::move(subject_and_arms)) {} }; diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 108a8fcf70..e6c94eadb0 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1454,7 +1454,6 @@ static std::vector process_lazy_operator(V v, Code static std::vector process_match_expression(V v, CodeBlob& code, TypePtr target_type) { TypePtr subject_type = v->get_subject()->inferred_type; - const TypeDataEnum* subject_enum = subject_type->unwrap_alias()->try_as(); int n_arms = v->get_arms_count(); std::vector ir_subj = pre_compile_expr(v->get_subject(), code, nullptr); @@ -1465,9 +1464,9 @@ static std::vector process_match_expression(V v return {}; } - bool has_type_arm = false; - bool has_expr_arm = false; - bool has_else_arm = false; + bool has_type_arm = false; // it's either `match` by type (all arms are types covering all cases) + bool has_expr_arm = false; // or `match` by expression, patterns can't be mixed, checked earlier + bool has_else_arm = false; // if `else` exists, it's the last for (int i = 0; i < n_arms; ++i) { auto v_arm = v->get_arm(i); has_type_arm |= v_arm->pattern_kind == MatchArmKind::exact_type; @@ -1475,43 +1474,56 @@ static std::vector process_match_expression(V v has_else_arm |= v_arm->pattern_kind == MatchArmKind::else_branch; } - // it's either `match` by type (all arms patterns are types) or `match` by expression - bool is_match_by_type = has_type_arm; - // detect whether `match` is exhaustive - bool is_exhaustive = is_match_by_type // match by type covers all cases, checked earlier - || !v->is_statement() // match by expression is exhaustive, checked earlier - || (has_expr_arm && subject_enum) // match by enum (either covers all cases or contains else, checked earlier) - || has_else_arm; - // `else` is not allowed in `match` by type; this was not fired at type checking, - // because it might turned out to be a lazy match, where `else` is allowed; + // because it might have turned out to be a lazy match, where `else` is allowed; // if we are here, it's not a lazy match, it's a regular one (the lazy one is handled specially, in aux vertex) - if (is_match_by_type && has_else_arm) { + if (has_type_arm && has_else_arm) { err("`else` is not allowed in `match` by type; you should cover all possible types").fire(v->get_arm(n_arms - 1)->get_pattern_expr()); } + // in some cases, if `else` does not exist, we implicitly add it and "throw unreachable" there; + int implicit_else_unreachable_throw = 0; + // even though `match (enum)` covers all cases, if a stack is malformed (holds a wrong integer), we throw + if (v->is_exhaustive && has_expr_arm && !has_else_arm && subject_type->unwrap_alias()->try_as()) { + implicit_else_unreachable_throw = 5; // "integer is out of range" + } + + // how to compare subject and branches for `match` expression, similar to operator `==` which can handle non-integers + FunctionPtr eq_fn = lookup_function("_==_"); // for int/bool/enum + if (subject_type->unwrap_alias()->try_as()) { + eq_fn = lookup_function("slice.bitsEqual"); + } + // example 1 (exhaustive): `match (v) { int => ... slice => ... builder => ... }` - // construct nested IFs: IF is int { ... } ELSE { IF is slice { ... } ELSE { ... } } + // IF is int { ... } ELSE { IF is slice { ... } ELSE { ... } } // example 2 (exhaustive): `match (v) { -1 => ... 0 => ... else => ... }` - // construct nested IFs: IF is int { ... } ELSE { IF is slice { ... } ELSE { ... } } + // IF == -1 { ... } ELSE { IF == 0 { ... } ELSE { ... } } // example 3 (not exhaustive): `match (v) { -1 => ... 0 => ... 1 => ... }` - // construct nested IFs: IF == -1 { ... } ELSE { IF == 0 { ... } ELSE { IF == 1 { ... } } } - FunctionPtr eq_sym = lookup_function("_==_"); - for (int i = 0; i < n_arms - is_exhaustive; ++i) { + // IF == -1 { ... } ELSE { IF == 0 { ... } ELSE { IF == 1 { ... } } } + // example 4 (with implicit else): `match (role) { User => ... Admin => ... }` + // IF == 0 { ... } ELSE { IF == 1 { ... } ELSE { 5 THROW } } + for (int i = 0; i < n_arms; ++i) { auto v_ith_arm = v->get_arm(i); - std::vector eq_ith_ir_idx; - if (is_match_by_type) { - TypePtr cmp_type = v_ith_arm->pattern_type_node->resolved_type; - tolk_assert(!cmp_type->unwrap_alias()->try_as()); // `match` over `int|slice` is a type checker error - eq_ith_ir_idx = pre_compile_is_type(code, subject_type, cmp_type, ir_subj, v_ith_arm, "(arm-cond-eq)"); - } else { - std::vector ith_ir_idx = pre_compile_expr(v_ith_arm->get_pattern_expr(), code); - tolk_assert(ir_subj.size() == 1 && ith_ir_idx.size() == 1); - eq_ith_ir_idx = code.create_tmp_var(TypeDataBool::create(), v_ith_arm, "(arm-cond-eq)"); - code.emplace_back(v_ith_arm, Op::_Call, eq_ith_ir_idx, std::vector{ir_subj[0], ith_ir_idx[0]}, eq_sym); + // if we're inside `else` of inside `builder` (example 1), no more checks, we're inside last else + Op* if_op = nullptr; + bool inside_last_branch = i == n_arms - 1 && v->is_exhaustive && implicit_else_unreachable_throw == 0; + if (!inside_last_branch) { + // construct "IF enter_ith_branch" + std::vector eq_ith_ir_idx; + if (has_type_arm) { // `v is int`, `v is slice`, etc. (type before =>) + TypePtr cmp_type = v_ith_arm->pattern_type_node->resolved_type; + tolk_assert(!cmp_type->unwrap_alias()->try_as()); // `match` over `int|slice` is a type checker error + eq_ith_ir_idx = pre_compile_is_type(code, subject_type, cmp_type, ir_subj, v_ith_arm, "(arm-cond-eq)"); + } else { // `v == 0`, `v == Role.User`, etc. (expr before =>) + std::vector ith_ir_idx = pre_compile_expr(v_ith_arm->get_pattern_expr(), code); + tolk_assert(ir_subj.size() == 1 && ith_ir_idx.size() == 1); + eq_ith_ir_idx = code.create_tmp_var(TypeDataBool::create(), v_ith_arm, "(arm-cond-eq)"); + code.emplace_back(v_ith_arm, Op::_Call, eq_ith_ir_idx, std::vector{ir_subj[0], ith_ir_idx[0]}, eq_fn); + } + if_op = &code.emplace_back(v_ith_arm, Op::_If, std::move(eq_ith_ir_idx)); + code.push_set_cur(if_op->block0); } - Op& if_op = code.emplace_back(v_ith_arm, Op::_If, std::move(eq_ith_ir_idx)); - code.push_set_cur(if_op.block0); + if (v->is_statement()) { pre_compile_expr(v_ith_arm->get_body(), code); if (v == stmt_before_immediate_return) { @@ -1521,25 +1533,24 @@ static std::vector process_match_expression(V v std::vector arm_ir_idx = pre_compile_expr(v_ith_arm->get_body(), code, v->inferred_type); code.emplace_back(v, Op::_Let, ir_result, std::move(arm_ir_idx)); } - code.close_pop_cur(v); - code.push_set_cur(if_op.block1); // open ELSE - } - if (is_exhaustive) { - // we're inside the last ELSE - auto v_last_arm = v->get_arm(n_arms - 1); - if (v->is_statement()) { - pre_compile_expr(v_last_arm->get_body(), code); - if (v == stmt_before_immediate_return) { - code.emplace_back(v_last_arm, Op::_Return); - } - } else { - std::vector arm_ir_idx = pre_compile_expr(v_last_arm->get_body(), code, v->inferred_type); - code.emplace_back(v, Op::_Let, ir_result, std::move(arm_ir_idx)); + if (!inside_last_branch) { + code.close_pop_cur(v); + code.push_set_cur(if_op->block1); // open ELSE } } + + // we are inside last ELSE + // if it was user-defined, we've inserted its body already + // if it's auto-generated "unreachable", insert "N THROW" + if (implicit_else_unreachable_throw) { + Op& op_throw = code.emplace_back(v, Op::_Call, std::vector{}, std::vector{code.create_int(v, implicit_else_unreachable_throw, "(throw-else)")}, lookup_function("__throw")); + op_throw.set_impure_flag(); + } + // close all outer IFs - for (int i = 0; i < n_arms - is_exhaustive; ++i) { + int depth = n_arms - v->is_exhaustive + (implicit_else_unreachable_throw != 0); + for (int i = 0; i < depth; ++i) { code.close_pop_cur(v); } diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index c69eff3cfc..69f85549e3 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -601,7 +601,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { has_expr_arm = true; TypePtr pattern_type = v_arm->get_pattern_expr()->inferred_type; bool not_integer_comparison = false; - if (!check_eq_neq_operator(pattern_type, subject_type, not_integer_comparison) || not_integer_comparison) { + if (!check_eq_neq_operator(pattern_type, subject_type, not_integer_comparison)) { if (pattern_type->equal_to(subject_type)) { // `match` over `slice` etc., where operator `==` can't be applied err("wrong pattern matching: can not compare type `{}` in `match`", subject_type).fire(v_arm->get_pattern_expr(), cur_f); } else { @@ -677,17 +677,9 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } } - // `match` by expression, if it's not a statement, should have `else` (unless exhaustive) - if (has_expr_arm && !has_else_arm && !v->is_statement() && !subject_enum) { - bool needs_else_branch = true; - if (expect_boolean(subject_type) && v->get_arms_count() == 2) { - auto arm0 = v->get_arm(0)->get_pattern_expr()->try_as(); - auto arm1 = v->get_arm(1)->get_pattern_expr()->try_as(); - needs_else_branch = !(arm0 && arm1 && arm0->bool_val != arm1->bool_val); - } - if (needs_else_branch) { - err("`match` expression should have `else` branch").fire(v->keyword_range(), cur_f); - } + // `match` by expression, if it's not a statement, should have `else` or cover all values + if (!v->is_statement() && !v->is_exhaustive) { + err("`match` expression should have `else` branch").fire(v->keyword_range(), cur_f); } } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index a633ed52d4..7d435e9e59 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -1262,9 +1262,11 @@ class InferTypesAndCallsAndFieldsVisitor final { ExprFlow infer_match_expression(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { flow = infer_any_expr(v->get_subject(), std::move(flow), false).out_flow; SinkExpression s_expr = extract_sink_expression_from_vertex(v->get_subject()); + TypePtr subject_type = v->get_subject()->inferred_type; TypeInferringUnifyStrategy branches_unifier; FlowContext arms_entry_facts = flow.clone(); FlowContext match_out_flow; + bool has_type_arm = false; bool has_expr_arm = false; bool has_else_arm = false; @@ -1292,13 +1294,12 @@ class InferTypesAndCallsAndFieldsVisitor final { } } arm_flow.register_known_type(s_expr, exact_type); - - } else if (v_arm->pattern_kind == MatchArmKind::const_expression) { - has_expr_arm = true; - } else if (v_arm->pattern_kind == MatchArmKind::else_branch) { - has_else_arm = true; } + has_type_arm |= v_arm->pattern_kind == MatchArmKind::exact_type; + has_expr_arm |= v_arm->pattern_kind == MatchArmKind::const_expression; + has_else_arm |= v_arm->pattern_kind == MatchArmKind::else_branch; + arm_flow = infer_any_expr(v_arm->get_body(), std::move(arm_flow), false, hint).out_flow; match_out_flow = i == 0 ? std::move(arm_flow) : FlowContext::merge_flow(std::move(match_out_flow), std::move(arm_flow)); branches_unifier.unify_with(v_arm->get_body()->inferred_type, hint); @@ -1306,7 +1307,21 @@ class InferTypesAndCallsAndFieldsVisitor final { if (v->get_arms_count() == 0) { match_out_flow = std::move(arms_entry_facts); } - if (has_expr_arm && !has_else_arm) { + + // calculate if `match` is exhaustive if used correctly (correctness is checked later) + bool is_exhaustive = has_else_arm // user-defined `else` branch + || has_type_arm // all possible types must be covered, checked in a later pipe + || subject_type->unwrap_alias()->try_as(); // all enum members must be covered + // special case: `match (boolVar) { true => false => }` + if (subject_type == TypeDataBool::create() && v->get_arms_count() == 2 && has_expr_arm) { + auto arm0 = v->get_arm(0)->get_pattern_expr()->try_as(); + auto arm1 = v->get_arm(1)->get_pattern_expr()->try_as(); + is_exhaustive |= arm0 && arm1 && arm0->bool_val != arm1->bool_val; + } + + v->mutate()->assign_is_exhaustive(is_exhaustive); + if (!is_exhaustive && v->get_arms_count()) { + // add implicit `else => {}` branch for control flow FlowContext else_flow = process_any_statement(createV(SrcRange::empty_at_end(v->range)), std::move(arms_entry_facts)); match_out_flow = FlowContext::merge_flow(std::move(match_out_flow), std::move(else_flow)); } From a5f7a80801ae88a118bb97f6e2504b5489cd87bd Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sat, 1 Nov 2025 16:48:03 +0300 Subject: [PATCH 459/524] [Tolk] Some fixes of corner cases found by fuzzing Most tickets can never meet in practice: only if the user intentionally writes apriori invalid code. For example, asm `1()2 THROW` (parsing a number failed), etc. --- .../tests/invalid-semantics/err-4818.tolk | 8 +++ .../tests/invalid-serialization/err-7300.tolk | 2 +- .../tests/invalid-serialization/err-7304.tolk | 2 +- .../tests/invalid-serialization/err-7305.tolk | 2 +- .../tests/invalid-serialization/err-7306.tolk | 2 +- .../tests/invalid-serialization/err-7309.tolk | 40 ++++++++++++ .../tests/invalid-serialization/err-7310.tolk | 42 +++++++++++++ .../tests/invalid-serialization/err-7311.tolk | 30 +++++++++ .../tests/invalid-serialization/err-7312.tolk | 34 ++++++++++ .../tests/invalid-syntax/err-3319.tolk | 6 ++ .../tests/invalid-syntax/err-3490.tolk | 18 ++++++ .../tests/invalid-syntax/err-3981.tolk | 7 +++ tolk-tester/tests/mutate-methods.tolk | 4 +- tolk-tester/tests/never-type-tests.tolk | 11 ++++ .../tests/warnings-not-errors/warnings-5.tolk | 14 +++++ tolk/asmops.cpp | 2 - tolk/ast-from-tokens.cpp | 21 ++++--- tolk/codegen.cpp | 1 - tolk/optimize.cpp | 14 +++-- tolk/pipe-check-inferred-types.cpp | 40 ++++++++++-- tolk/pipe-infer-types-and-calls.cpp | 2 +- tolk/pipe-lazy-load-insertions.cpp | 62 ++++++++++++------- tolk/pipe-transform-on-message.cpp | 2 +- 23 files changed, 313 insertions(+), 53 deletions(-) create mode 100644 tolk-tester/tests/invalid-semantics/err-4818.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7309.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7310.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7311.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7312.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3319.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3490.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3981.tolk create mode 100644 tolk-tester/tests/warnings-not-errors/warnings-5.tolk diff --git a/tolk-tester/tests/invalid-semantics/err-4818.tolk b/tolk-tester/tests/invalid-semantics/err-4818.tolk new file mode 100644 index 0000000000..0de9a5caea --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4818.tolk @@ -0,0 +1,8 @@ +fun main() { + var test = builder.storeSlice(1, 5); +} + +/** +@compilation_should_fail +@stderr can not pass `int` to `builder` + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7300.tolk b/tolk-tester/tests/invalid-serialization/err-7300.tolk index 31930736e2..2186087616 100644 --- a/tolk-tester/tests/invalid-serialization/err-7300.tolk +++ b/tolk-tester/tests/invalid-serialization/err-7300.tolk @@ -10,6 +10,6 @@ fun f() { /** @compilation_should_fail -@stderr `lazy` will not work here, because variable `o` it's used in a non-lazy manner +@stderr `lazy` will not work here, because variable `o` is used in a non-lazy manner @stderr hint: lazy union may be used only in `match` statement */ diff --git a/tolk-tester/tests/invalid-serialization/err-7304.tolk b/tolk-tester/tests/invalid-serialization/err-7304.tolk index 4659caf80b..dace2daa9c 100644 --- a/tolk-tester/tests/invalid-serialization/err-7304.tolk +++ b/tolk-tester/tests/invalid-serialization/err-7304.tolk @@ -18,5 +18,5 @@ fun f(c: cell) { /** @compilation_should_fail -@stderr `lazy` will not work here, because variable `msg` it's used in a non-lazy manner +@stderr `lazy` will not work here, because variable `msg` is used in a non-lazy manner */ diff --git a/tolk-tester/tests/invalid-serialization/err-7305.tolk b/tolk-tester/tests/invalid-serialization/err-7305.tolk index 59fb5ba50b..043b1781fb 100644 --- a/tolk-tester/tests/invalid-serialization/err-7305.tolk +++ b/tolk-tester/tests/invalid-serialization/err-7305.tolk @@ -14,6 +14,6 @@ fun f() { /** @compilation_should_fail -@stderr `lazy` will not work here, because variable `msg` it's used in a non-lazy manner +@stderr `lazy` will not work here, because variable `msg` is used in a non-lazy manner @stderr lazy MsgEitherCounter */ diff --git a/tolk-tester/tests/invalid-serialization/err-7306.tolk b/tolk-tester/tests/invalid-serialization/err-7306.tolk index d515c7d494..31e3d12c5c 100644 --- a/tolk-tester/tests/invalid-serialization/err-7306.tolk +++ b/tolk-tester/tests/invalid-serialization/err-7306.tolk @@ -13,5 +13,5 @@ fun f() { /** @compilation_should_fail -@stderr `lazy` will not work here, because variable `msg` it's used in a non-lazy manner +@stderr `lazy` will not work here, because variable `msg` is used in a non-lazy manner */ diff --git a/tolk-tester/tests/invalid-serialization/err-7309.tolk b/tolk-tester/tests/invalid-serialization/err-7309.tolk new file mode 100644 index 0000000000..642e6d3b5b --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7309.tolk @@ -0,0 +1,40 @@ +struct (0x01) AskToTransfer { + jettonAmount: coins + another: uint8 +} + +struct (0x02) TransferNotificationForRecipient { + jettonAmount: coins + another: uint8 +} + +type AllowedMessageToVault = + | AskToTransfer + | TransferNotificationForRecipient + +@inline +fun AllowedMessageToVault.getJettonAmount(self, delta: coins) { + return match (self) { + AskToTransfer => self.jettonAmount + delta, + TransferNotificationForRecipient => self.jettonAmount + delta, + } +} + +fun main() { + var m = lazy AllowedMessageToVault.fromSlice(""); + match (m) { + AskToTransfer => { + return m.getJettonAmount(123); + } + TransferNotificationForRecipient => { + return m.getJettonAmount(456); + } + } +} + +/** +@compilation_should_fail +@stderr `lazy` will not work here, because variable `m` changes its type inside `match` +@stderr ^^^^ +@stderr hint: probably, it's reassigned, or called a method with a different receiver + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7310.tolk b/tolk-tester/tests/invalid-serialization/err-7310.tolk new file mode 100644 index 0000000000..5074dc1a3a --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7310.tolk @@ -0,0 +1,42 @@ +struct (0x01) AskToTransfer { + jettonAmount: coins + another: uint8 +} + +struct (0x02) TransferNotificationForRecipient { + jettonAmount: coins + another: uint8 +} + +type AllowedMessageToVault = + | AskToTransfer + | TransferNotificationForRecipient + +@inline +fun AllowedMessageToVault.setJettonAmount(mutate self, newValue: coins) { + match (self) { + AskToTransfer => { + self.jettonAmount = newValue; + } + TransferNotificationForRecipient => { + self.jettonAmount = newValue; + } + } +} + +fun main() { + var m = lazy AllowedMessageToVault.fromSlice(""); + match (m) { + AskToTransfer => { + return m.setJettonAmount(123); + } + TransferNotificationForRecipient => { + return m.setJettonAmount(456); + } + } +} + +/** +@compilation_should_fail +@stderr `lazy` will not work here, because variable `m` changes its type inside `match` + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7311.tolk b/tolk-tester/tests/invalid-serialization/err-7311.tolk new file mode 100644 index 0000000000..ab03943a2c --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7311.tolk @@ -0,0 +1,30 @@ +struct (0x01) AskToTransfer { + jettonAmount: coins + another: uint8 +} + +struct (0x02) TransferNotificationForRecipient { + jettonAmount: coins + another: uint8 +} + +type AllowedMessageToVault = + | AskToTransfer + | TransferNotificationForRecipient + +fun main() { + var m = lazy AllowedMessageToVault.fromSlice(""); + match (m) { + AskToTransfer => { + } + TransferNotificationForRecipient => { + m = AskToTransfer{jettonAmount: ton("0.1"), another: 0}; + m.another = 1; + } + } +} + +/** +@compilation_should_fail +@stderr `lazy` will not work here, because variable `m` changes its type inside `match` + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7312.tolk b/tolk-tester/tests/invalid-serialization/err-7312.tolk new file mode 100644 index 0000000000..666001f752 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7312.tolk @@ -0,0 +1,34 @@ +struct A { a1: int7; a2: int8; a3: int8; } +struct B { b1: int7; b2: int8; b3: int8; } +type AOrB = A | B; + +fun main() { + var m = lazy AOrB.fromSlice(""); + match (m) { + A => { + __expect_lazy("[m] skip (int7) load a2"); + match (m) { + A => { + m.a2 + 100; + } + } + } + B => { + __expect_lazy("[m] load b1 b2 b3"); + m = m as AOrB; + match (m) { + A => { + m.a1 + 100; + } + B => { + m.b2 - 100; + } + } + } + } +} + +/** +@compilation_should_fail +@stderr `lazy` will not work here, because variable `m` changes its type inside `match` + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3319.tolk b/tolk-tester/tests/invalid-syntax/err-3319.tolk new file mode 100644 index 0000000000..82c4b773bd --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3319.tolk @@ -0,0 +1,6 @@ +fun t(): (int, int) asm (-> 1222222222222222222222222222222223 0) "NOP" + +/** +@compilation_should_fail +@stderr expected `)`, got `1222222222222222222222222222222223` + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3490.tolk b/tolk-tester/tests/invalid-syntax/err-3490.tolk new file mode 100644 index 0000000000..a08c3cc236 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3490.tolk @@ -0,0 +1,18 @@ +struct A { } +struct B { } +type C = A | B; + +@noinline +fun C.onInternalMessage(self) { + return match (self) { + A{ => 1, + B => 2, + }; + } + + fun main() {} + +/** +@compilation_should_fail +@stderr expected or in `match` before `=>` + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3981.tolk b/tolk-tester/tests/invalid-syntax/err-3981.tolk new file mode 100644 index 0000000000..8158063ff7 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3981.tolk @@ -0,0 +1,7 @@ +struct(0x2345666666666666666666666666666666666666666666666666666666666666666666666666789) S {} + +/** +@compilation_should_fail +@stderr invalid integer constant +@stderr ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + */ diff --git a/tolk-tester/tests/mutate-methods.tolk b/tolk-tester/tests/mutate-methods.tolk index 838e7171f0..cff11ebd53 100644 --- a/tolk-tester/tests/mutate-methods.tolk +++ b/tolk-tester/tests/mutate-methods.tolk @@ -204,7 +204,7 @@ fun testStoreChaining() { b.storeUint(4, 32); b.myStoreUint(5, 32).storeUint(6, 32); storeUint(mutate b, 7, 32); - b = b.storeUint(8, 32); + builder.storeUint(mutate b, 8, 32); b = b.storeUint(9, 32).storeUint(10, 32); return b.bitsCount(); @@ -216,7 +216,7 @@ fun testStoreChainingCustom() { b.myStoreUint(4, 32); b.myStoreUint(5, 32).myStoreUint(6, 32); myStoreUint(mutate b, 7, 32); - b = b.myStoreUint(8, 32); + builder.myStoreUint(mutate b, 8, 32); b = b.myStoreUint(9, 32).myStoreUint(10, 32); val sum1 = getSumOfNumbersInCell(b.endCell()); diff --git a/tolk-tester/tests/never-type-tests.tolk b/tolk-tester/tests/never-type-tests.tolk index b19d88caf3..45043ff0e6 100644 --- a/tolk-tester/tests/never-type-tests.tolk +++ b/tolk-tester/tests/never-type-tests.tolk @@ -28,12 +28,23 @@ fun test1(x: int?) { return 123; } +fun bigThrow() { throw 1222222222222222222222222222222223; } + fun main() { __expect_type(test1, "(int?) -> int"); __expect_type(alwaysThrows(), "never"); __expect_type(alwaysThrowsButReturns(), "int"); + __expect_type(bigThrow, "() -> never"); } /** @testcase | 101 | null | 123 + +@fif_codegen +""" + bigThrow() PROC:<{ + 1222222222222222222222222222222223 PUSHINT + THROWANY + }> +""" */ diff --git a/tolk-tester/tests/warnings-not-errors/warnings-5.tolk b/tolk-tester/tests/warnings-not-errors/warnings-5.tolk new file mode 100644 index 0000000000..009464c4da --- /dev/null +++ b/tolk-tester/tests/warnings-not-errors/warnings-5.tolk @@ -0,0 +1,14 @@ +fun main(x: int, y: int) { + if (x = 1) { + y = 10; + } + assert ((y = 20)) throw 123; + return x + y; +} + +/** +@testcase | 0 | 0 0 | 21 +@stderr assignment inside condition, probably it's a misprint +@stderr if (x = 1) +@stderr assert ((y = 20)) + */ diff --git a/tolk/asmops.cpp b/tolk/asmops.cpp index ac1c244047..df0c5ef778 100644 --- a/tolk/asmops.cpp +++ b/tolk/asmops.cpp @@ -224,8 +224,6 @@ AsmOp AsmOp::Parse(AnyV origin, const std::string& custom_op) { return AsmOp::Push(origin, 0); } else if (custom_op == "OVER") { return AsmOp::Push(origin, 1); - } else if (custom_op.ends_with(" PUSHINT") && custom_op[0] >= '1' && custom_op[0] <= '9' && custom_op.find(' ') == custom_op.rfind(' ')) { - return AsmOp::IntConst(origin, td::string_to_int256(custom_op.substr(0, custom_op.find(' ')))); } else { return AsmOp::Custom(origin, custom_op); } diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 2f57573677..22e9cd5a26 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -132,11 +132,15 @@ static AnyExprV maybe_replace_eq_null_with_isNull_check(V v } // parse `123` / `0xFF` / `0b10001` to td::RefInt256 -static td::RefInt256 parse_tok_int_const(std::string_view text) { +static td::RefInt256 parse_tok_int_const(std::string_view text, SrcRange cur_range) { bool bin = text[0] == '0' && text[1] == 'b'; if (!bin) { // this function parses decimal and hex numbers - return td::string_to_int256(static_cast(text)); + td::RefInt256 intval = td::string_to_int256(static_cast(text)); + if (intval.is_null() || !intval->signed_fits_bits(257)) { + err("invalid integer constant").fire(cur_range); + } + return intval; } // parse a binary number; to make it simpler, don't allow too long numbers, it's impractical if (text.size() > 64 + 2) { @@ -726,6 +730,7 @@ static V parse_match_arm(Lexer& lex) { } if (!exact_type || lex.tok() != tok_double_arrow) { exact_type = nullptr; + pattern_kind = static_cast(-1); lex.restore_position(backup); try { pattern_expr = parse_expr(lex); @@ -875,10 +880,7 @@ static AnyExprV parse_expr100(Lexer& lex) { case tok_int_const: { SrcRange range = lex.cur_range(); std::string_view orig_str = lex.cur_str(); - td::RefInt256 intval = parse_tok_int_const(orig_str); - if (intval.is_null() || !intval->signed_fits_bits(257)) { - lex.error("invalid integer constant"); - } + td::RefInt256 intval = parse_tok_int_const(orig_str, lex.cur_range()); lex.next(); return createV(range, std::move(intval), orig_str); } @@ -1448,8 +1450,8 @@ static AnyV parse_asm_func_body(Lexer& lex, V name_ident, V(lex.cur_str()).c_str()); + while (lex.tok() == tok_int_const && lex.cur_str().size() < 6) { + int ret_idx = std::stoi(static_cast(lex.cur_str())); ret_order.push_back(ret_idx); lex.next(); } @@ -1738,7 +1740,8 @@ static AnyV parse_struct_declaration(Lexer& lex, const std::vector(opcode_range, parse_tok_int_const(opcode_str), opcode_str); + td::RefInt256 intval = parse_tok_int_const(opcode_str, opcode_range); + opcode = createV(opcode_range, std::move(intval), opcode_str); lex.next(); lex.expect(tok_clpar, "`)`"); } else { diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp index 95f7e12eac..869638b833 100644 --- a/tolk/codegen.cpp +++ b/tolk/codegen.cpp @@ -357,7 +357,6 @@ bool Op::generate_code_step(Stack& stack) { stack.o << AsmOp::Custom(origin, "CONT:<{"); stack.o.indent(); if (f_sym->is_asm_function() || f_sym->is_builtin()) { - // TODO: create and compile a true lambda instead of this (so that arg_order and ret_order would work correctly) std::vector args0, res; int w_arg = 0; for (const LocalVarData& param : f_sym->parameters) { diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index a8bdc2549c..371cb8620b 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -152,14 +152,16 @@ bool Optimizer::detect_rewrite_big_THROW() { } std::string s_number(s_num_throw.substr(0, sp)); - uint64_t excno = std::stoul(s_number); - if (excno < 2048) { // "9 THROW" left as is, but "N THROW" where N>=2^11 is invalid for Fift + td::RefInt256 excno = td::string_to_int256(s_number); + // "9 THROW" left as is, but "N THROW" where N>=2^11 is invalid for Fift + // `is_null()` can be when the user intentionally corrupts asm instructions, let Fift fail + if (excno.is_null() || (excno >= 0 && excno < 2048)) { return false; } p_ = 1; q_ = 2; - oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->origin, td::make_refint(excno))); + oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->origin, excno)); oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->origin, "THROWANY", 1, 0)); return true; } @@ -665,9 +667,11 @@ bool Optimizer::detect_rewrite_xxx_NOT() { bool is_gtint = f.ends_with(" GTINT"); std::string s_number(f.substr(0, f.rfind(' '))); td::RefInt256 number = td::string_to_int256(s_number); - number += is_gtint ? 1 : -1; - if (number > -127 && number < 127) { + if (!number.is_null()) { + number += is_gtint ? 1 : -1; + } + if (!number.is_null() && number > -127 && number < 127) { p_ = 2; q_ = 1; std::string new_op = number->to_dec_string() + (is_gtint ? " LESSINT" : " GTINT"); diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 69f85549e3..1d64a2b18c 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -134,16 +134,31 @@ static void handle_possible_compiler_internal_call(FunctionPtr cur_f, Vfun_maybe; tolk_assert(fun_ref && fun_ref->is_builtin()); - if (fun_ref->name == "__expect_type") { - tolk_assert(v->get_num_args() == 2); - std::string_view expected_type_str = v->get_arg(1)->get_expr()->as()->str_val; + if (fun_ref->name == "__expect_type" && v->get_num_args() == 2) { + // __expect_type(expr, "...") is a compiler built-in for testing, it's not indented to be called by users + auto v_expected_str = v->get_arg(1)->get_expr()->try_as(); + tolk_assert(v_expected_str && "invalid __expect_type"); TypePtr expr_type = v->get_arg(0)->inferred_type; - if (expected_type_str != expr_type->as_human_readable()) { - err("__expect_type failed: expected `{}`, got `{}`", expected_type_str, expr_type).fire(v, cur_f); + if (v_expected_str->str_val != expr_type->as_human_readable()) { + err("__expect_type failed: expected `{}`, got `{}`", v_expected_str->str_val, expr_type).fire(v, cur_f); } } } +// detect `if (x = 1)` having its condition to fire a warning; +// note that `if ((x = f()) == null)` and other usages of assignment is rvalue is okay +static bool is_assignment_inside_condition(AnyExprV cond) { + while (auto v_par = cond->try_as()) { + cond = v_par->get_expr(); + } + return cond->kind == ast_assign || cond->kind == ast_set_assign; +} + +// make an error for `if (x = 1)` +static Error err_assignment_inside_condition() { + return err("assignment inside condition, probably it's a misprint\n""hint: if it's intentional, extract assignment as a separate statement for clarity"); +} + static bool expect_integer(TypePtr inferred_type) { if (inferred_type == TypeDataInt::create()) { return true; @@ -545,6 +560,9 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (cond->is_always_true || cond->is_always_false) { warning_condition_always_true_or_false(cur_f, cond->range, cond, "ternary operator"); } + if (is_assignment_inside_condition(cond)) { + err_assignment_inside_condition().warning(cond, cur_f); + } } void visit(V v) override { @@ -702,6 +720,9 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (cond->is_always_true || cond->is_always_false) { warning_condition_always_true_or_false(cur_f, v->keyword_range(), cond, "`if`"); } + if (is_assignment_inside_condition(cond)) { + err_assignment_inside_condition().warning(cond, cur_f); + } } void visit(V v) override { @@ -724,6 +745,9 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (cond->is_always_true || cond->is_always_false) { warning_condition_always_true_or_false(cur_f, v->keyword_range(), cond, "`while`"); } + if (is_assignment_inside_condition(cond)) { + err_assignment_inside_condition().warning(cond, cur_f); + } } void visit(V v) override { @@ -737,6 +761,9 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (cond->is_always_true || cond->is_always_false) { warning_condition_always_true_or_false(cur_f, v->keyword_range(), cond, "`do while`"); } + if (is_assignment_inside_condition(cond)) { + err_assignment_inside_condition().warning(cond, cur_f); + } } void visit(V v) override { @@ -764,6 +791,9 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (cond->is_always_true || cond->is_always_false) { warning_condition_always_true_or_false(cur_f, v->keyword_range(), cond, "`assert`"); } + if (is_assignment_inside_condition(cond)) { + err_assignment_inside_condition().warning(cond, cur_f); + } } void visit(V v) override { diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 7d435e9e59..86d9aa91a9 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -1218,7 +1218,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // get return type either from user-specified declaration or infer here on demand traversing its body v->mutate()->assign_fun_ref(fun_ref, self_obj != nullptr); get_or_infer_return_type(fun_ref); - TypePtr inferred_type = fun_ref->does_return_self() ? self_obj->inferred_type : fun_ref->inferred_return_type; + TypePtr inferred_type = fun_ref->does_return_self() && self_obj ? self_obj->inferred_type : fun_ref->inferred_return_type; assign_inferred_type(v, inferred_type); assign_inferred_type(callee, fun_ref->inferred_full_type); if (inferred_type == TypeDataNever::create()) { diff --git a/tolk/pipe-lazy-load-insertions.cpp b/tolk/pipe-lazy-load-insertions.cpp index 0e824692f3..fafc060a49 100644 --- a/tolk/pipe-lazy-load-insertions.cpp +++ b/tolk/pipe-lazy-load-insertions.cpp @@ -209,11 +209,6 @@ static void check_lazy_operator_used_correctly(FunctionPtr cur_f, Vis_builtin() && method_ref->is_instantiation_of_generic_function()) { - std::string_view f_name = method_ref->base_fun_ref->name; - return f_name == "T.toCell" || f_name == "T.forceLoadLazyObject"; - } - return method_ref->is_inlined_in_place() && // only AST-inlined methods can be lazy !method_ref->has_mutate_params() && !method_ref->does_return_self(); @@ -231,6 +226,7 @@ struct ExprUsagesWhileCollecting { int used_for_writing = 0; int used_for_matching = 0; int used_for_toCell = 0; + int used_reassigned_type = 0; int total_usages_with_fields = 0; std::vector needed_above_stmt; std::vector> used_as_match_subj; @@ -272,6 +268,7 @@ struct ExprUsagesWhileCollecting { used_for_writing += rhs.used_for_writing; used_for_matching += rhs.used_for_matching; used_for_toCell += rhs.used_for_toCell; + used_reassigned_type += rhs.used_reassigned_type; total_usages_with_fields += rhs.total_usages_with_fields; needed_above_stmt.insert(needed_above_stmt.end(), rhs.needed_above_stmt.begin(), rhs.needed_above_stmt.end()); used_as_match_subj.insert(used_as_match_subj.end(), rhs.used_as_match_subj.begin(), rhs.used_as_match_subj.end()); @@ -299,6 +296,10 @@ struct ExprUsagesWhileCollecting { total_usages_with_fields++; } + void on_used_reassigned_type() { + used_reassigned_type++; + } + bool is_self_or_field_used_for_reading() const { if (used_for_reading) { return true; @@ -551,8 +552,9 @@ struct LazyVarInFunction { // handle if `msg` is used only in `match (msg) { ... }` // (it may even be not a union, just a struct with opcode, and `match` with `else`) - bool used_only_as_match = var_usages.used_for_matching == 1 && !var_usages.used_for_reading && !var_usages.used_for_toCell && !var_usages.used_for_writing; - if (used_only_as_match) { + bool used_only_as_match = var_usages.used_for_matching == 1 && !var_usages.used_for_reading && !var_usages.used_for_toCell && !var_usages.used_for_writing && !var_usages.used_reassigned_type; + bool variants_not_reassigned = std::all_of(var_usages.variants.begin(), var_usages.variants.end(), [](const ExprUsagesWhileCollecting& variant_usages) { return !variant_usages.used_reassigned_type; }); + if (used_only_as_match && variants_not_reassigned) { v_lazy_match_var_itself = var_usages.used_as_match_subj.front(); load_points.reserve(var_usages.variants.size()); for (ExprUsagesWhileCollecting& variant_usages : var_usages.variants) { @@ -566,8 +568,11 @@ struct LazyVarInFunction { // prohibit this to a union: lazy union may only be matched, nothing more (`msg is A` etc. don't work) const TypeDataStruct* t_struct = var_ref->declared_type->unwrap_alias()->try_as(); bool is_union = t_struct == nullptr; + if (is_union && used_only_as_match) { + err("`lazy` will not work here, because variable `{}` changes its type inside `match`\n""hint: probably, it's reassigned, or called a method with a different receiver", var_ref).fire(created_by_lazy_op->keyword_range(), cur_f); + } if (is_union) { - err("`lazy` will not work here, because variable `{}` it's used in a non-lazy manner\n""hint: lazy union may be used only in `match` statement", var_ref).fire(created_by_lazy_op->keyword_range(), cur_f); + err("`lazy` will not work here, because variable `{}` is used in a non-lazy manner\n""hint: lazy union may be used only in `match` statement, exactly once", var_ref).fire(created_by_lazy_op->keyword_range(), cur_f); } // so, it's just a struct, `lazy Point`; we've already calculated all statements where its fields are used @@ -620,6 +625,9 @@ class CollectUsagesInStatementVisitor final : public ASTVisitorFunctionBody { if (!is_subj_of_dot) { lazy_expr->on_used_rw(v->is_lvalue); } + if (!v->is_lvalue && !lazy_expr->expr_type->equal_to(v->inferred_type)) { + lazy_expr->on_used_reassigned_type(); // e.g. in `A => ...` variable was reassigned and now is `B` + } } } @@ -638,7 +646,7 @@ class CollectUsagesInStatementVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { FunctionPtr fun_ref = v->fun_maybe; - if (fun_ref && fun_ref->does_accept_self() && can_method_be_inlined_preserving_lazy(fun_ref)) { + if (fun_ref && fun_ref->does_accept_self() && v->dot_obj_is_self) { AnyExprV dot_obj = v->get_callee()->as()->get_obj(); if (extract_sink_expression_from_vertex(dot_obj) == s_expr) { // handle built-in functions specially @@ -646,22 +654,24 @@ class CollectUsagesInStatementVisitor final : public ASTVisitorFunctionBody { lazy_expr->on_used_toCell(); return; } - if (fun_ref->is_builtin() && fun_ref->base_fun_ref->name == "T.forceLoadLazyObject") { - lazy_expr->on_used_rw(false); // just use the object as a whole, slice will point to its end - return; + + // if receiver is another type, e.g. `fun (A|B).method(self)`, called from `match (v) { A => v.method() }` + if (!fun_ref->parameters[0].declared_type->equal_to(lazy_expr->expr_type)) { + lazy_expr->on_used_reassigned_type(); } - if (s_expr.index_path) { // obj.f.method(), mark obj.f as used anyway + // for `obj.f.method()`, mark lazy_expr=obj.f "used" anyway + if (s_expr.index_path) { lazy_expr->on_used_rw(false); } - tolk_assert(fun_ref->is_code_function()); - - // okay, we have `st.save()` / `p.getX()`, which will be inlined when transforming to IR; + // if we have `st.save()` / `p.getX()` / `obj.f.method()`, which will be inlined when transforming to IR, // dig into that method's body to fetch used fields `self.x` etc. - auto v_body_block = fun_ref->ast_root->try_as()->get_body()->try_as(); - ExprUsagesWhileCollecting inner_usages = collect_expr_usages_in_block(lazy_expr->name_str + "(=self)", SinkExpression(&fun_ref->parameters[0]), lazy_expr->expr_type, v_body_block); - inner_usages.treat_match_like_read(); // nested lazy match in inlined functions doesn't work, it's not wrapped into aux vertex - lazy_expr->merge_with_sub_block(inner_usages); - return; + if (can_method_be_inlined_preserving_lazy(fun_ref)) { + auto v_body_block = fun_ref->ast_root->try_as()->get_body()->try_as(); + ExprUsagesWhileCollecting inner_usages = collect_expr_usages_in_block(lazy_expr->name_str + "(=self)", SinkExpression(&fun_ref->parameters[0]), lazy_expr->expr_type, v_body_block); + inner_usages.treat_match_like_read(); // nested lazy match in inlined functions doesn't work, it's not wrapped into aux vertex + lazy_expr->merge_with_sub_block(inner_usages); + return; + } } } @@ -686,6 +696,10 @@ class CollectUsagesInStatementVisitor final : public ASTVisitorFunctionBody { is_safe = v_set_assign->get_rhs() == v; } + if (!lazy_expr->expr_type->equal_to(subj->inferred_type)) { + is_safe = false; // `v = v as Union; match (v)` or inside a method with a different receiver + } + if (is_match_by_cur && is_safe) { lazy_expr->on_used_as_match_subj(v); const TypeDataUnion* expr_as_union = lazy_expr->expr_type->unwrap_alias()->try_as(); @@ -973,8 +987,10 @@ class CheckExpectLazyAssertionsVisitor final : public ASTVisitorFunctionBody { AnyV cur_stmt = v->get_item(i); if (auto v_call = cur_stmt->try_as()) { if (v_call->fun_maybe && v_call->fun_maybe->is_builtin() && v_call->fun_maybe->name == "__expect_lazy") { + // __expect_lazy("...") is a compiler built-in for testing, it's not indented to be called by users + auto v_expected_str = v_call->get_arg(0)->get_expr()->try_as(); + tolk_assert(i + 1 < v->size() && v_expected_str && "invalid __expect_lazy"); AnyV next_stmt = v->get_item(i + 1); - std::string_view expected = v_call->get_arg(0)->get_expr()->as()->str_val; std::string actual; if (auto next_aux = next_stmt->try_as()) { if (const auto* aux_load = dynamic_cast(next_aux->aux_data)) { @@ -985,7 +1001,7 @@ class CheckExpectLazyAssertionsVisitor final : public ASTVisitorFunctionBody { } } - if (actual != expected) { + if (actual != v_expected_str->str_val) { err("__expect_lazy failed: actual \"{}\"", actual).fire(SrcRange::span(cur_stmt->range, 13)); } } diff --git a/tolk/pipe-transform-on-message.cpp b/tolk/pipe-transform-on-message.cpp index a713039ca1..6770458a2d 100644 --- a/tolk/pipe-transform-on-message.cpp +++ b/tolk/pipe-transform-on-message.cpp @@ -77,7 +77,7 @@ class TransformOnInternalMessageReplacer final : public ASTReplacerInFunctionBod AnyExprV replace(V v) override { // replace `in.senderAddress` / `in.valueCoins` with an aux vertex - if (v->get_obj()->kind == ast_reference && v->get_obj()->as()->sym == param_ref) { + if (v->get_obj()->kind == ast_reference && v->get_obj()->as()->sym == param_ref && v->is_target_struct_field()) { if (v->is_lvalue && v->get_field_name() != "body" && v->get_field_name() != "bouncedBody") { err("modifying an immutable variable\n""hint: fields of InMessage can be used for reading only").fire(v, cur_f); } From 3f54f72b29bea96f802a35ec919d954b9b12d3c5 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sat, 1 Nov 2025 19:42:25 +0300 Subject: [PATCH 460/524] [Tolk] Enhance stdlib with low-level built-ins T.typeName, T.toTuple, T.fromTuple --- crypto/smartcont/tolk-stdlib/common.tolk | 39 ++++++++++++++++ .../smartcont/tolk-stdlib/gas-payments.tolk | 4 ++ tolk-tester/tests/constants-tests.tolk | 11 ++++- tolk-tester/tests/lazy-load-tests.tolk | 2 +- tolk-tester/tests/strings-tests.tolk | 11 ++++- tolk-tester/tests/struct-tests.tolk | 41 +++++++++++++++++ tolk-tester/tests/union-types-tests.tolk | 15 +++++++ tolk/builtins.cpp | 45 ++++++++++++------- tolk/constant-evaluator.cpp | 5 +++ tolk/lexer.cpp | 14 ------ tolk/lexer.h | 3 -- tolk/pipe-discover-parse-sources.cpp | 3 -- 12 files changed, 155 insertions(+), 38 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index c2c6b12897..c3248abde0 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -138,6 +138,23 @@ struct Cell { private readonly tvmCell: cell } +/// Get human-readable type name of any T for logging or low-level purposes. +/// Example: +/// ``` +/// debug.printString(int.typeName()); // "int" +/// ``` +@pure +fun T.typeName(): slice + builtin + +/// Get human-readable type name of any object for logging or low-level purposes. +/// Example: +/// ``` +/// debug.printString(42.typeNameOfObject()); // "int" +/// ``` +@pure +fun T.typeNameOfObject(self): slice + builtin /** Tuple manipulation primitives. @@ -190,6 +207,28 @@ fun tuple.last(self): T fun tuple.pop(mutate self): T asm "TPOP" +/// Packs any object from a stack to a tuple. +/// An object occupies N slots on a stack — the tuple will be of size N. +/// Example: +/// ``` +/// struct Point { x: int, y: int } +/// +/// var p: Point = { x: 1, y: 2 }; +/// var t = p.toTuple(); // [ 1 2 ] +/// p = Point.fromTuple(t); // back +/// t.get(0); // 1 +/// ``` +/// It can be used for logging or low-level purposes like running a child VM or custom "equals". +/// See [T.fromTuple] counterpart. +@pure +fun T.toTuple(self): tuple + builtin + +/// Unpacks a tuple back to an object on a stack. +/// See [T.toTuple] for explanation and examples. +@pure +fun T.fromTuple(packedObject: tuple): T + builtin /** Mathematical primitives. diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index dd343508ff..e09a94e31c 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -33,10 +33,12 @@ fun setGasLimit(limit: int): void asm "SETGASLIMIT" /// Calculates fee (amount in nanotoncoins to be paid) for a transaction which consumed `gasUsed` gas units. +@pure fun calculateGasFee(workchain: int8, gasUsed: int): coins asm(gasUsed workchain) "GETGASFEE" /// Same as [calculateGasFee], but without flat price. [Docs about fees](https://docs.ton.org/develop/howto/fees-low-level) +@pure fun calculateGasFeeWithoutFlatPrice(workchain: int8, gasUsed: coins): coins asm(gasUsed workchain) "GETGASFEESIMPLE" @@ -46,10 +48,12 @@ fun calculateStorageFee(workchain: int8, seconds: int, bits: int, cells: int): c asm(cells bits seconds workchain) "GETSTORAGEFEE" /// Calculates amount of nanotoncoins you should pay to send a message of a specified size. +@pure fun calculateForwardFee(workchain: int8, bits: int, cells: int): coins asm(cells bits workchain) "GETFORWARDFEE" /// Same as [calculateForwardFee], but without lump price. [Docs about fees](https://docs.ton.org/develop/howto/fees-low-level) +@pure fun calculateForwardFeeWithoutLumpPrice(workchain: int8, bits: int, cells: int): coins asm(cells bits workchain) "GETFORWARDFEESIMPLE" diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index 52e6255a24..ba5a32aaf5 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -37,6 +37,8 @@ const tens2: (int, int, (int8, int16)) = (tens1.1, tens1.0 << 2, tens1); const intOrN: int? = null; const int32Or64: int32 | int64 = 7 as int64; +const nameOfAddress = address.typeName() + fun iget1(): int { return int1; } fun iget2(): int { return int2; } fun iget3(): int { return int1+int2; } @@ -209,6 +211,12 @@ fun test16() { return (wOneTon, wNoTon, wNoSubParams) } +@method_id(117) +fun test17() { + __expect_type(nameOfAddress, "slice"); + return (nameOfAddress.bitsEqual("address"), nameOfAddress.typeNameOfObject().bitsEqual("slice")) +} + fun main() { var i1: int = iget1(); var i2: int = iget2(); @@ -249,6 +257,7 @@ fun main() { @testcase | 114 | | -1 0 -1 0 100 1 777 50 1 (null) 0 @testcase | 115 | | 0 -1 0 0 0 0 0 0 0 -1 777 -1 -1 -1 -1 @testcase | 116 | | 1000000000 0 (null) (null) (null) 0 +@testcase | 117 | | -1 -1 -@code_hash 62183191459178893174419204771807638858032941060155787749136667785942705596518 +@code_hash 75101380693091438363237198341416144164642759114287088786910200935773618931276 */ diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index f6e78c7806..43e5738e7b 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -940,7 +940,7 @@ fun demo153() { global gTup: tuple; @noinline -fun pushToGlobalTup(v: T) { gTup.push(v.__toTuple()); } +fun pushToGlobalTup(v: T) { gTup.push(v.toTuple()); } @method_id(154) fun demo154() { diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index c6d0f22b4f..3e489d9524 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -48,6 +48,10 @@ fun calcSliceBytes(slice: slice): tuple { return t; } +fun getName() { return T.typeName() } + +type EmptyTensor = () + fun main() { var s_ascii: slice = ascii_slice(); var s_raw: slice = raw_slice(); @@ -66,6 +70,11 @@ fun main() { assert(i_maxi == 0x7a62e8a8ebac41bd6de16c65e7be363bc2d2cbc6a0873778dead4795c13db979) throw 106; assert(i_crc32 == 2235694568 && i_crc32 == 0x8541fde8) throw 107; assert(i_crc16 == 52550 && i_crc16 == 0xCD46) throw 108; + assert(getName().bitsEqual("int8")) throw 109; + assert(getName().bitsEqual(int8.typeName())) throw 109; + assert(EmptyTensor.typeName().loadUint(88) == 83932693928210635055918962) throw 110; + assert(int176.fromSlice(getName<(int, map)>()) == 15119865017903399349236740405018280815773870593949225) throw 111; + assert((176 as int176, 176).typeNameOfObject().bitsEqual("(int176, int)")) throw 112; return 0; } @@ -78,5 +87,5 @@ fun test1() { @testcase | 0 | | 0 @testcase | 101 | | [ 65 66 67 68 ] -@code_hash 5620054157808655551766377324451164816605429927637692949572895262349774660488 +@code_hash 28689458953739962699797226940965106691932724643823208129506677010881613457948 */ diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index eb33de0e1c..590563aa04 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -610,6 +610,44 @@ fun test43() { return (obj, PrivWithDefaults.createA(20)); } +@method_id(144) +fun test44() { + var t = PrivPoint.create(10, 20).toTuple(); + assert (t.get(1) == 20) throw 123; + assert (PrivPoint.fromTuple(t).z == 0) throw 124; + return (t, (10 as MInt).toTuple(), PrivWithDefaults{}.toTuple(), PrivWithDefaults.fromTuple(([1, true] as tuple))); +} + +@method_id(145) +fun test45() { + val inner: WithTensorInside = { + coords: (10, 20), + tup: [-1, -2], + otherCoords: null, + otherTup: [] as tuple, + }; + return WithTensorInside{ + coords: (1, 2), + tup: [3, 4], + otherCoords: (5, 6), + otherTup: WithTensorInside.fromTuple(inner.toTuple()).toTuple() + }.toTuple(); +} + +struct BigOnStackA { + n1: (int, int, int, int, int) = (1,2,3,4,5) + n2: (int, int, int, int, int) = (6,7,8,9,10) + n3: (int, int, int, int, int) = (11,12,13,14,15) + n4: (int, int, int, int, int) = (16,17,18,19,20) +} + +@method_id(146) +fun test46() { + var t = BigOnStackA{}.toTuple(); + return (t.size(), BigOnStackA.fromTuple(t).n3.toTuple()); +} + + fun main(x: int8, y: MInt) { __expect_type(PointAlias{x,y}, "Point"); __expect_type(Point{x,y} as Point, "Point"); @@ -671,6 +709,9 @@ type PointAlias = Point; @testcase | 141 | 0 | 3 4 1 2 @testcase | 142 | | 10 80 5 95 @testcase | 143 | | 0 0 20 0 +@testcase | 144 | | [ 10 20 0 ] [ 10 ] [ 0 0 ] 1 -1 +@testcase | 145 | | [ 1 2 [ 3 4 ] 5 6 typeid-8 [ 10 20 [ -1 -2 ] (null) (null) 0 [] ] ] +@testcase | 146 | | 20 [ 11 12 13 14 15 ] @fif_codegen """ diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 0f649f75ef..1427f598d6 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -776,6 +776,20 @@ fun test61(i: int32 | int64 | int) { } } +fun myFromTuple(t: tuple): T { + return T.fromTuple(t) +} + +@method_id(162) +fun test62() { + return ( + ((5 as int32) as int32 | int64).toTuple(), + ((5 as int64) as int32 | int64).toTuple(), + myFromTuple([10, 48] as tuple) is int64, + myFromTuple([10, 48] as tuple).typeNameOfObject().bitsEqual("int32 | int64"), + ) +} + fun main() { @@ -850,6 +864,7 @@ fun main() { @testcase | 159 | null 0 | 123 @testcase | 160 | | 10 0 @testcase | 161 | 100 48 | 20 +@testcase | 162 | | [ 5 46 ] [ 5 48 ] -1 -1 @fif_codegen diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 0f734f6478..9f324aa46a 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1006,7 +1006,7 @@ static AsmOp compile_calc_InMessage_getInMsgParam(std::vector& res, st }; tolk_assert(res.size() == 1 && args.size() == 1 && args[0].is_int_const()); args[0].unused(); - size_t idx = args[0].int_const->to_long(); + uint64_t idx = static_cast(args[0].int_const->to_long()); if (idx < std::size(aliases)) { return exec_op(origin, aliases[idx], 0); } @@ -1078,8 +1078,8 @@ static AsmOp compile_fetch_varint(std::vector& res, std::vector. +class BusListeningActor : public td::actor::Actor { + private: + friend class Runtime; + + template + void dispatch_event(BusHandle bus, ArmedEvent event) { + log_received_event(bus._node(td::Badge{}), typeid(E)); + if constexpr (!std::same_as) { + // When we install listeners, we guarantee that actual bus type is at most B. + static_cast(this)->template handle(bus.template unsafe_static_downcast_to(), event.disarm()); + } else { + static_cast(this)->template handle(std::move(bus), event.disarm()); + } + } + + void stop_with_bus() { + this->td::actor::Actor::stop(); + } + + virtual void install_listeners_at(BusTypeId new_bus_type, std::shared_ptr const& node, + td::actor::ActorId const& actor_id) = 0; + + td::BufferSlice name_; +}; + +// Base class for all user-created bus listeners. Template argument specifies which bus the actor +// spawns with. Note that you have to register actor using Runtime::register_actor for an actor to +// actually be created with the bus. +template +class SpawnsWith : public BusListeningActor { + public: + using SpawnWithBus = B; + + ~SpawnsWith() { + LOG_CHECK(owning_bus_._node(td::Badge>{}).is_stopping.load(std::memory_order_relaxed)) + << "Bus actors must not call stop() manually"; + } + + protected: + BusHandle const& owning_bus() { + CHECK(owning_bus_); + return owning_bus_; + } + + private: + friend class Runtime; + + BusHandle owning_bus_; +}; + +// Bus listeners must also inherit ConnectsTo to specify a list of buses they subscribe to. +template +struct ConnectsTo { + using ConnectToBuses = TypeList; +}; + +template +concept ActorType = std::derived_from> && + IsSpecializationOf; + +template +concept CanActorHandleEvent = + requires(A& actor, BusHandle bus, std::shared_ptr event) { actor.template handle(bus, event); }; + +class Runtime { + public: + template + void register_actor(std::string_view name) { + LOG_CHECK(!started_) << "Actors can only be registered before starting runtime"; + + using B = typename A::SpawnWithBus; + BusTypeId spawn_bus_id = get_bus_id(); + register_bus_parents(spawn_bus_id); + + // Add actor to actors_to_spawn_for_ for bus to know which actors to spawn on creation. + CreateInstanceFn create_instance_fn = &Runtime::create_actor_instance; + actors_to_spawn_for_[spawn_bus_id].push_back({ + .create_instance_fn = create_instance_fn, + .name = std::string(name), + }); + } + + template + void set_root_bus(std::shared_ptr bus) { + LOG_CHECK(!started_) << "Root bus can only be set before starting runtime"; + + root_bus_ = std::make_shared(*this, "", std::move(bus)); + } + + void start() { + LOG_CHECK(!started_) << "Runtime::start must not be called twice"; + LOG_CHECK(root_bus_) << "Root bus is not set"; + started_ = true; + + CHECK(root_bus_->bus_impl); + wire_bus(std::move(root_bus_)); + } + + template + void _wire_bus(td::Badge>, std::shared_ptr node) { + wire_bus(node); + } + + template + void _stop_bus(td::Badge>, std::shared_ptr node) { + node->is_stopping.store(true, std::memory_order_relaxed); + for (auto const& owned_actor : node->owned_actors) { + td::actor::send_closure(owned_actor, &BusListeningActor::stop_with_bus); + } + } + + private: + bool started_ = false; + + std::shared_ptr root_bus_; + + // ===== Bus inheritance ===== + template + void register_bus_parents(BusTypeId id) { + if (bus_parents_.contains(id)) { + return; + } + + if constexpr (requires { typename B::Parent; }) { + auto parent_id = get_bus_id(); + bus_parents_[id] = parent_id; + register_bus_parents(parent_id); + } else { + bus_parents_[id] = std::nullopt; + } + } + + std::map> bus_parents_; + + // ===== Actor creation for a particular bus ===== + template + class BusListeningActorImpl final : public A { + public: + struct Params { + BusListeningActorImpl& self; + BusTypeId new_bus_type; + std::shared_ptr const& node; + td::actor::ActorId const& actor_id; + }; + + virtual void install_listeners_at(BusTypeId new_bus_type, std::shared_ptr const& node, + td::actor::ActorId const& actor_id) override { + // What we are doing with ActorRegistrar is effectively: + // for (auto B : A::ConnectToBuses) { + // for (auto (BOrigin, E) : B::Events) { + // register_event_listener(node); + // } + // } + Registrar{}({*this, new_bus_type, node, actor_id}); + } + + template + struct Registrar {}; + + template + struct Registrar> { + void operator()(Params params) { + (EventRegistrar{}(params), ...); + } + }; + + template + struct EventRegistrar {}; + + template + struct EventRegistrar> { + void operator()(Params params) { + (params.self.template register_event_listener(params), ...); + if constexpr (BusWithParent) { + using Parent = BOrigin::Parent; + EventRegistrar{}(params); + } + } + }; + + template + void register_event_listener(Params params) { + if constexpr (CanActorHandleEvent) { + // handle accepts (BusHandle, std::shared_ptr) while we treat newly installed + // bus (of type BNew) as having type new_bus_type (one of types in its inheritance chain). + if (get_bus_id() == params.new_bus_type) { + // node->bus_impl is BusImpl. Since BNew <= B == new_bus_type, it is safe to + // downcast node->bus_impl to BusImpl. + auto bus_impl = + static_cast*>(static_cast*>(params.node->bus_impl.get())); + + bus_impl->dispatchers.push_back({ + .actor = params.actor_id, + .dispatcher_fn = &BusListeningActor::dispatch_event, + }); + } + } + } + }; + + using CreateInstanceFn = std::unique_ptr (Runtime::*)(std::shared_ptr node); + + struct ActorSpawnInfo { + CreateInstanceFn create_instance_fn; + std::string name; + }; + + template + std::unique_ptr create_actor_instance(std::shared_ptr node) { + auto instance = std::make_unique>(); + auto bus = std::static_pointer_cast(node->bus); + auto bus_impl = std::static_pointer_cast>(node->bus_impl); + instance->owning_bus_ = BusHandle({}, std::move(node), bus, bus_impl); + return instance; + } + + std::map> actors_to_spawn_for_; + + // ===== Event wiring ===== + void wire_bus(std::shared_ptr node) { + CHECK(node); + + // First, we create all actors that spawn on the added bus ... + std::vector> spawned_actors; + std::vector bus_inheritance_chain; + + for (std::optional bus_type = node->type_id; bus_type; bus_type = bus_parents_[*bus_type]) { + bus_inheritance_chain.push_back(*bus_type); + + for (auto const& [create_instance_fn, base_name] : actors_to_spawn_for_[*bus_type]) { + auto instance = (this->*create_instance_fn)(node); + std::string name = node->actor_name_prefix + base_name; + instance->name_ = td::BufferSlice(name.data(), name.size()); + auto actor_info = + td::actor::detail::create_actor_info(td::actor::ActorOptions{}.with_name(name), std::move(instance)); + node->owned_actors.push_back(td::actor::ActorId::unsafe_create_from_info(actor_info)); + } + } + + // Then, we wire events from the bus to all actors that are subscribed to it. Note that we have + // not returned BusHandle of a newly created bus to outside world yet, so the outside world + // cannot publish events on the bus, and actors that have the current bus injected as a owning + // bus are not started yet. + auto it = node; + while (it) { + LOG_CHECK(!it->is_stopping.load(std::memory_order_relaxed)) + << "Race between child and parent buses detected. THIS IS A BEST-EFFORT WARNING, YOU HAVE A RACE."; + + for (auto const& owned_actor : it->owned_actors) { + for (auto new_bus_type : bus_inheritance_chain) { + owned_actor.get_actor_unsafe().install_listeners_at(new_bus_type, node, owned_actor); + } + } + it = it->parent; + } + + // And lastly, we start all newly created actors + for (auto const& actor : node->owned_actors) { + td::actor::detail::register_actor_info_ptr(actor.actor_info_ptr()); + } + } +}; + +template +void BusHandle::wire_bus(std::shared_ptr node) { + node->runtime._wire_bus({}, std::move(node)); +} + +template +void BusHandle::stop_bus(std::shared_ptr node) { + node->runtime._stop_bus({}, std::move(node)); +} + +} // namespace detail + +using detail::Bus; +using detail::BusHandle; +using detail::BusType; +using detail::ConnectsTo; +using detail::Runtime; +using detail::SpawnsWith; + +#define TON_RUNTIME_DEFINE_EVENT_HANDLER() \ + template <::ton::runtime::In B, typename E> \ + requires ::ton::runtime::detail::ValidEventFor \ + constexpr void handle(::ton::runtime::BusHandle bus, ::std::shared_ptr event) = delete; + +} // namespace ton::runtime \ No newline at end of file diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index e63b1599f7..d234397faf 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -928,7 +928,7 @@ void ArchiveManager::start_up() { } td::WalkPath::run(db_root_ + "/archive/states/", [&](td::CSlice fname, td::WalkPath::Type t) -> void { - if (t == td::WalkPath::Type::NotDir) { + if (t == td::WalkPath::Type::RegularFile) { LOG(ERROR) << "checking file " << fname; auto pos = fname.rfind(TD_DIR_SLASH); if (pos != td::Slice::npos) { diff --git a/validator/manager.cpp b/validator/manager.cpp index 0cd8ed431d..101c871c7b 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1971,7 +1971,7 @@ void ValidatorManagerImpl::start_up() { auto to_import_dir = db_root_ + "/import"; auto S = td::WalkPath::run(to_import_dir, [&](td::CSlice cfname, td::WalkPath::Type t) -> void { auto fname = td::Slice(cfname); - if (t == td::WalkPath::Type::NotDir) { + if (t == td::WalkPath::Type::RegularFile) { auto d = fname.rfind(TD_DIR_SLASH); if (d != td::Slice::npos) { fname = fname.substr(d + 1); diff --git a/validator/validator.h b/validator/validator.h index 122f608a12..ef17cc43c8 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -236,7 +236,6 @@ class ValidatorManagerInterface : public td::actor::Actor { } virtual void download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { - } virtual void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) { From aa06130aeb638e8b48954354f00eeaea3c6ed600 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Sat, 15 Nov 2025 09:38:18 +0300 Subject: [PATCH 477/524] Fix type cast from UnixTime to CatchainSeqno (both are td::uint32) (#1887) --- validator/full-node-serializer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validator/full-node-serializer.cpp b/validator/full-node-serializer.cpp index adce0fe22a..4181d69b64 100644 --- a/validator/full-node-serializer.cpp +++ b/validator/full-node-serializer.cpp @@ -60,7 +60,7 @@ static td::Result deserialize_block_broadcast(ton_api::tonNode_b } return BlockBroadcast{create_block_id(f.id_), std::move(signatures), - static_cast(f.catchain_seqno_), + static_cast(f.catchain_seqno_), static_cast(f.validator_set_hash_), std::move(f.data_), std::move(f.proof_)}; @@ -84,7 +84,7 @@ static td::Result deserialize_block_broadcast(ton_api::tonNode_b << data.size() + proof.size() + signatures.size() * 96; return BlockBroadcast{create_block_id(f.id_), std::move(signatures), - static_cast(f.catchain_seqno_), + static_cast(f.catchain_seqno_), static_cast(f.validator_set_hash_), std::move(data), std::move(proof)}; @@ -106,7 +106,7 @@ static td::Result deserialize_block_broadcast(ton_api::tonNode_b << data.size() + proof.size() + signatures.size() * 96; return BlockBroadcast{create_block_id(f.id_), std::move(signatures), - static_cast(f.catchain_seqno_), + static_cast(f.catchain_seqno_), static_cast(f.validator_set_hash_), std::move(data), std::move(proof)}; From 244789a709b59997c9484cc55f4850812351e168 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Sat, 15 Nov 2025 09:38:45 +0300 Subject: [PATCH 478/524] Update Certificate::tl to handle default flags (#1888) * Update Certificate::tl to handle default flags * Fix formatting in overlay-manager.cpp --- overlay/overlay-manager.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index 4fd649bd32..35b59f107f 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -598,8 +598,12 @@ BroadcastCheckResult Certificate::check(PublicKeyHash node, OverlayIdShort overl } tl_object_ptr Certificate::tl() const { - return create_tl_object(issued_by_.get().tl(), expire_at_, max_size_, - signature_.clone_as_buffer_slice()); + if (flags_ == cert_default_flags(max_size_)) { + return create_tl_object(issued_by_.get().tl(), expire_at_, max_size_, + signature_.clone_as_buffer_slice()); + } + return create_tl_object(issued_by_.get().tl(), expire_at_, max_size_, + flags_, signature_.clone_as_buffer_slice()); } tl_object_ptr Certificate::empty_tl() { From 06d4d268dfda6b9835f1ccb45439eeec9271e4a6 Mon Sep 17 00:00:00 2001 From: Vadim Averin Date: Sat, 15 Nov 2025 09:50:16 +0300 Subject: [PATCH 479/524] Implement parallelism in validator engine (#1812) * Apply saved patches for parallel checking of accounts' transactions * Cleanup unneeded code * Fix account_dict update logic * Implement thread-safe scan_account_libraries * Implement thread-safe deferring messages * Implement thread-safe timer * Implement thread-safe storage stat cache * Implement thread-safe work time stat * Cleanup threadpool * Add option to control parallel validation of accounts * Final touches * Integrate with td::actor system * Implement synchronous `CheckAccountTxs` execution * Implement parallel `CheckAccountTxs` execution * Cleanup unneeded code * Remove unneeded lock-free structure * Final polishing * Fix ValidateQuery actor dying before all CheckAccountTxs actors finish * Catch VM errors inside CheckAccountTxs actor * Prevent ValidateQuery actor dying by timeout before CheckAccountTxs * Better naming for CheckAccountTxs actors * Fixes after merge * Fix timer measurements * More accurate validation stats * More accurate error handling * Remove unneeded alarm in CheckAccountTxs actor * Fix incorrect error reporting * Final fix of error handling * Fix error handling (again) * Rename option * Add fixes from review * Codestyle * Remove debug code --- tl/generate/scheme/ton_api.tl | 5 +- validator-engine/validator-engine.cpp | 4 + validator-engine/validator-engine.hpp | 4 + validator/fabric.h | 2 + validator/impl/validate-query.cpp | 546 +++++++++++++++-------- validator/impl/validate-query.hpp | 72 ++- validator/interfaces/validator-manager.h | 27 +- validator/validator-group.cpp | 3 +- validator/validator-options.hpp | 8 + validator/validator.h | 2 + 10 files changed, 466 insertions(+), 207 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index e9ea3e3ba3..c0e57cb813 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -980,10 +980,11 @@ validatorStats.validatedBlock self:int256 valid:Bool comment:string bytes:int collated_data_bytes:int - total_time:double work_time:double cpu_work_time:double time_stats:string + total_time:double work_time:double actual_time:double cpu_work_time:double time_stats:string work_time_real_stats:string work_time_cpu_stats:string - storage_stat_cache:validatorStats.storageStatCacheStats = validatorStats.ValidatedBlock; + storage_stat_cache:validatorStats.storageStatCacheStats + parallel_accounts_validation:Bool = validatorStats.ValidatedBlock; validatorStats.newValidatorGroup.node id:int256 pubkey:PublicKey adnl_id:int256 weight:long = validatorStats.newValidatorGroup.Node; validatorStats.newValidatorGroup session_id:int256 shard:tonNode.shardId cc_seqno:int diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 63f381581c..2e5532dec1 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1638,6 +1638,7 @@ td::Status ValidatorEngine::load_global_config() { } validator_options_.write().set_hardforks(std::move(h)); validator_options_.write().set_catchain_broadcast_speed_multiplier(broadcast_speed_multiplier_catchain_); + validator_options_.write().set_parallel_validation(parallel_validation_); for (auto &id : config_.collator_node_whitelist) { validator_options_.write().set_collator_node_whitelisted_validator(id, true); @@ -5576,6 +5577,9 @@ int main(int argc, char *argv[]) { } return td::Status::OK(); }); + p.add_option('\0', "parallel-validation", "parallel validation over different accounts", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_parallel_validation, true); }); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 267009e97c..1a72869631 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -257,6 +257,7 @@ class ValidatorEngine : public td::actor::Actor { td::optional sync_shards_upto_; ton::adnl::AdnlNodeIdShort shard_block_retainer_adnl_id_ = ton::adnl::AdnlNodeIdShort::zero(); bool shard_block_retainer_adnl_id_fullnode_ = false; + bool parallel_validation_ = false; double initial_sync_delay_ = 60.0; std::set unsafe_catchains_; @@ -382,6 +383,9 @@ class ValidatorEngine : public td::actor::Actor { void set_shard_block_retainer_adnl_id_fullnode() { shard_block_retainer_adnl_id_fullnode_ = true; } + void set_parallel_validation(bool value) { + parallel_validation_ = value; + } void set_initial_sync_delay(double value) { initial_sync_delay_ = value; } diff --git a/validator/fabric.h b/validator/fabric.h index a1a48779a5..a2e7c6c040 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -54,6 +54,8 @@ struct ValidateParams { // Optional - used for validation of optimistic candidates Ref optimistic_prev_block = {}; + + bool parallel_validation = false; }; td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_, diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 0d9d61fb91..3e7d4fe68d 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -80,6 +80,7 @@ ValidateQuery::ValidateQuery(BlockCandidate candidate, ValidateParams params, , timeout(timeout) , main_promise(std::move(promise)) , is_fake_(params.is_fake) + , parallel_accounts_validation_(params.parallel_validation) , shard_pfx_(shard_.shard) , shard_pfx_len_(ton::shard_prefix_length(shard_)) , optimistic_prev_block_(std::move(params.optimistic_prev_block)) @@ -92,7 +93,9 @@ ValidateQuery::ValidateQuery(BlockCandidate candidate, ValidateParams params, * Raises an error when timeout is reached. */ void ValidateQuery::alarm() { - abort_query(td::Status::Error(ErrorCode::timeout, "timeout")); + if (!parallel_accounts_validation_pending_) { + abort_query(td::Status::Error(ErrorCode::timeout, "timeout")); + } } /** @@ -1069,7 +1072,6 @@ bool ValidateQuery::try_unpack_mc_state() { } block_limits_ = limits.move_as_ok(); block_limits_->start_lt = start_lt_; - block_limit_status_ = std::make_unique(*block_limits_); if (!fetch_config_params()) { return false; } @@ -5443,45 +5445,47 @@ bool ValidateQuery::check_delivered_dequeued() { * * @returns A unique pointer to the created Account object, or nullptr if the creation failed. */ -std::unique_ptr ValidateQuery::make_account_from(td::ConstBitPtr addr, Ref account) { - auto ptr = std::make_unique(workchain(), addr); +std::unique_ptr ValidateQuery::CheckAccountTxs::make_account_from(td::ConstBitPtr addr, + Ref account) { + auto ptr = std::make_unique(vq_.workchain(), addr); if (account.is_null()) { - if (!ptr->init_new(now_)) { + if (!ptr->init_new(vq_.now_)) { return nullptr; } - } else if (!ptr->unpack(std::move(account), now_, is_masterchain() && config_->is_special_smartcontract(addr))) { + } else if (!ptr->unpack(std::move(account), vq_.now_, + vq_.is_masterchain() && vq_.config_->is_special_smartcontract(addr))) { return nullptr; } - ptr->block_lt = start_lt_; + ptr->block_lt = vq_.start_lt_; return ptr; } /** * Retreives an Account object from the data in the shard state. * Accounts are cached in the ValidatorQuery's map. - * Similar to Collator::make_account() + * Similar to `Collator::make_account()`. * * @param addr The 256-bit address of the account. * * @returns Pointer to the account if found or created successfully. * Returns nullptr if an error occured. */ -std::unique_ptr ValidateQuery::unpack_account(td::ConstBitPtr addr) { - auto dict_entry = ps_.account_dict_->lookup_extra(addr, 256); +std::unique_ptr ValidateQuery::CheckAccountTxs::unpack_account(td::ConstBitPtr addr) { + auto dict_entry = vq_.ps_.account_dict_->lookup_extra(addr, 256); auto new_acc = make_account_from(addr, std::move(dict_entry.first)); if (!new_acc) { reject_query("cannot load state of account "s + addr.to_hex(256) + " from previous shardchain state"); return {}; } - if (!new_acc->belongs_to_shard(shard_)) { + if (!new_acc->belongs_to_shard(vq_.shard_)) { reject_query(PSTRING() << "old state of account " << addr.to_hex(256) << " does not really belong to current shard"); return {}; } if (new_acc->storage_dict_hash) { - if (full_collated_data_ && !is_masterchain()) { - auto it = virt_account_storage_dicts_.find(new_acc->storage_dict_hash.value()); - if (it != virt_account_storage_dicts_.end()) { + if (vq_.full_collated_data_ && !vq_.is_masterchain()) { + auto it = vq_.virt_account_storage_dicts_.find(new_acc->storage_dict_hash.value()); + if (it != vq_.virt_account_storage_dicts_.end()) { LOG(DEBUG) << "Using account storage dict proof for account " << addr.to_hex(256) << ", hash=" << it->second->get_hash().to_hex(); auto S = new_acc->init_account_storage_stat(it->second); @@ -5491,8 +5495,8 @@ std::unique_ptr ValidateQuery::unpack_account(td::ConstBitPtr ad return {}; } } - } else if (storage_stat_cache_ && new_acc->storage_dict_hash) { - auto dict_root = storage_stat_cache_(new_acc->storage_dict_hash.value()); + } else if (vq_.storage_stat_cache_ && new_acc->storage_dict_hash) { + auto dict_root = vq_.storage_stat_cache_(new_acc->storage_dict_hash.value()); if (dict_root.not_null()) { auto S = new_acc->init_account_storage_stat(dict_root); if (S.is_error()) { @@ -5502,15 +5506,16 @@ std::unique_ptr ValidateQuery::unpack_account(td::ConstBitPtr ad } LOG(DEBUG) << "Inited storage stat from cache for account " << addr.to_hex(256) << " (" << new_acc->storage_used.cells << " cells)"; - storage_stat_cache_update_.emplace_back(dict_root, new_acc->storage_used.cells); - stats_.storage_stat_cache.hit_cnt++; - stats_.storage_stat_cache.hit_cells += new_acc->storage_used.cells; + + ctx_.storage_stat_cache_update.emplace_back(dict_root, new_acc->storage_used.cells); + vq_.stats_.storage_stat_cache.hit_cnt.fetch_add(1); + vq_.stats_.storage_stat_cache.hit_cells.fetch_add(new_acc->storage_used.cells); } else if (new_acc->storage_used.cells >= StorageStatCache::MIN_ACCOUNT_CELLS) { - stats_.storage_stat_cache.miss_cnt++; - stats_.storage_stat_cache.miss_cells += new_acc->storage_used.cells; + vq_.stats_.storage_stat_cache.miss_cnt.fetch_add(1); + vq_.stats_.storage_stat_cache.miss_cells.fetch_add(new_acc->storage_used.cells); } else { - stats_.storage_stat_cache.small_cnt++; - stats_.storage_stat_cache.small_cells += new_acc->storage_used.cells; + vq_.stats_.storage_stat_cache.small_cnt.fetch_add(1); + vq_.stats_.storage_stat_cache.small_cells.fetch_add(new_acc->storage_used.cells); } } } @@ -5545,9 +5550,10 @@ static td::RefInt256 get_ihr_fee(const block::gen::CommonMsgInfo::Record_int_msg * * @returns True if the transaction is valid, false otherwise. */ -bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalTime lt, Ref trans_root, - bool is_first, bool is_last) { - if (!check_timeout()) { +bool ValidateQuery::CheckAccountTxs::check_one_transaction(block::Account& account, ton::LogicalTime lt, + Ref trans_root, bool is_first, bool is_last) { + if (vq_.timeout && vq_.timeout.is_in_past()) { + abort_query(td::Status::Error(ErrorCode::timeout, "timeout")); return false; } LOG(DEBUG) << "checking transaction " << lt << " of account " << account.addr.to_hex(); @@ -5566,7 +5572,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT CHECK(tag >= 0); // we have already validated the serialization of all Transactions td::optional in_msg_metadata; if (in_msg_root.not_null()) { - auto in_descr_cs = in_msg_dict_->lookup(in_msg_root->get_hash().as_bitslice()); + auto in_descr_cs = vq_.in_msg_dict_->lookup(in_msg_root->get_hash().as_bitslice()); if (in_descr_cs.is_null()) { return reject_query(PSTRING() << "inbound message with hash " << in_msg_root->get_hash().to_hex() << " of transaction " << lt << " of account " << addr.to_hex() @@ -5581,7 +5587,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT << " has an invalid InMsg record (not one of msg_import_ext, msg_import_fin, " "msg_import_imm, msg_import_ihr or msg_import_deferred_fin)"); } - is_special_tx = is_special_in_msg(*in_descr_cs); + is_special_tx = vq_.is_special_in_msg(*in_descr_cs); // once we know there is a InMsg with correct hash, we already know that it contains a message with this hash (by the verification of InMsg), so it is our message // have still to check its destination address and imported value // and that it refers to this transaction @@ -5613,21 +5619,21 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT emitted_lt = msg_env.emitted_lt.value(); } } - if (info.created_lt != start_lt_ || !is_special_tx) { - msg_proc_lt_.emplace_back(addr, lt, emitted_lt); + if (info.created_lt != vq_.start_lt_ || !is_special_tx) { + ctx_.msg_proc_lt.emplace_back(addr, lt, emitted_lt); } dest = std::move(info.dest); CHECK(money_imported.validate_unpack(info.value)); ihr_delivered = (in_msg_tag == block::gen::InMsg::msg_import_ihr); if (!ihr_delivered) { - money_imported += get_ihr_fee(info, global_version_); + money_imported += get_ihr_fee(info, vq_.global_version_); } CHECK(money_imported.is_valid()); } WorkchainId d_wc; StdSmcAddress d_addr; CHECK(block::tlb::t_MsgAddressInt.extract_std_address(dest, d_wc, d_addr)); - if (d_wc != workchain() || d_addr != addr) { + if (d_wc != vq_.workchain() || d_addr != addr) { return reject_query(PSTRING() << "inbound message of transaction " << lt << " of account " << addr.to_hex() << " has a different destination address " << d_wc << ":" << d_addr.to_hex()); } @@ -5641,7 +5647,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT } // check output messages td::optional new_msg_metadata; - if (msg_metadata_enabled_) { + if (vq_.msg_metadata_enabled_) { if (external || is_special_tx || tag != block::gen::TransactionDescr::trans_ord) { new_msg_metadata = block::MsgMetadata{0, account.workchain, account.addr, (LogicalTime)trans.lt}; } else if (in_msg_metadata) { @@ -5653,7 +5659,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT for (int i = 0; i < trans.outmsg_cnt; i++) { auto out_msg_root = out_dict.lookup_ref(td::BitArray<15>{i}); CHECK(out_msg_root.not_null()); // we have pre-checked this - auto out_descr_cs = out_msg_dict_->lookup(out_msg_root->get_hash().as_bitslice()); + auto out_descr_cs = vq_.out_msg_dict_->lookup(out_msg_root->get_hash().as_bitslice()); if (out_descr_cs.is_null()) { return reject_query(PSTRING() << "outbound message #" << i + 1 << " with hash " << out_msg_root->get_hash().to_hex() << " of transaction " << lt << " of account " @@ -5689,7 +5695,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT // unpack exported message value (from this transaction) block::CurrencyCollection msg_export_value; CHECK(msg_export_value.unpack(info.value)); - msg_export_value += get_ihr_fee(info, global_version_); + msg_export_value += get_ihr_fee(info, vq_.global_version_); msg_export_value += msg_env.fwd_fee_remaining; CHECK(msg_export_value.is_valid()); money_exported += msg_export_value; @@ -5704,7 +5710,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT WorkchainId s_wc; StdSmcAddress ss_addr; // s_addr is some macros in Windows CHECK(block::tlb::t_MsgAddressInt.extract_std_address(src, s_wc, ss_addr)); - if (s_wc != workchain() || ss_addr != addr) { + if (s_wc != vq_.workchain() || ss_addr != addr) { return reject_query(PSTRING() << "outbound message #" << i + 1 << " of transaction " << lt << " of account " << addr.to_hex() << " has a different source address " << s_wc << ":" << ss_addr.to_hex()); @@ -5718,24 +5724,25 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT } if (tag != block::gen::OutMsg::msg_export_ext) { bool is_deferred = tag == block::gen::OutMsg::msg_export_new_defer; - if (account_expected_defer_all_messages_.count(ss_addr) && !is_deferred) { + if (ctx_.defer_all_messages && !is_deferred) { return reject_query( - PSTRING() << "outbound message #" << i + 1 << " on account " << workchain() << ":" << ss_addr.to_hex() + PSTRING() << "outbound message #" << i + 1 << " on account " << vq_.workchain() << ":" << ss_addr.to_hex() << " must be deferred because this account has earlier messages in DispatchQueue"); } if (is_deferred) { - LOG(INFO) << "message from account " << workchain() << ":" << ss_addr.to_hex() << " with lt " << message_lt + LOG(INFO) << "message from account " << vq_.workchain() << ":" << ss_addr.to_hex() << " with lt " << message_lt << " was deferred"; - if (!deferring_messages_enabled_ && !account_expected_defer_all_messages_.count(ss_addr)) { - return reject_query(PSTRING() << "outbound message #" << i + 1 << " on account " << workchain() << ":" + if (!vq_.deferring_messages_enabled_ && !ctx_.defer_all_messages) { + return reject_query(PSTRING() << "outbound message #" << i + 1 << " on account " << vq_.workchain() << ":" << ss_addr.to_hex() << " is deferred, but deferring messages is disabled"); } - if (i == 0 && !account_expected_defer_all_messages_.count(ss_addr)) { - return reject_query(PSTRING() << "outbound message #1 on account " << workchain() << ":" << ss_addr.to_hex() + if (i == 0 && !ctx_.defer_all_messages) { + return reject_query(PSTRING() << "outbound message #1 on account " << vq_.workchain() << ":" + << ss_addr.to_hex() << " must not be deferred (the first message cannot be deferred unless some " "prevoius messages are deferred)"); } - account_expected_defer_all_messages_.insert(ss_addr); + ctx_.defer_all_messages = true; } } } @@ -5746,14 +5753,14 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT tag == block::gen::TransactionDescr::trans_merge_install || tag == block::gen::TransactionDescr::trans_split_prepare || tag == block::gen::TransactionDescr::trans_split_install) { - if (is_masterchain()) { + if (vq_.is_masterchain()) { return reject_query( PSTRING() << "transaction " << lt << " of account " << addr.to_hex() << " is a split/merge prepare/install transaction, which is impossible in a masterchain block"); } bool split = (tag == block::gen::TransactionDescr::trans_split_prepare || tag == block::gen::TransactionDescr::trans_split_install); - if (split && !before_split_) { + if (split && !vq_.before_split_) { return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex() << " is a split prepare/install transaction, but this block is not before a split"); } @@ -5762,7 +5769,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT << " is a split prepare/install transaction, but it is not the last transaction " "for this account in this block"); } - if (!split && !after_merge_) { + if (!split && !vq_.after_merge_) { return reject_query( PSTRING() << "transaction " << lt << " of account " << addr.to_hex() << " is a merge prepare/install transaction, but this block is not immediately after a merge"); @@ -5778,7 +5785,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT << " is a split/merge prepare/install transaction, which are globally disabled"); } if (tag == block::gen::TransactionDescr::trans_tick_tock) { - if (!is_masterchain()) { + if (!vq_.is_masterchain()) { return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex() << " is a tick-tock transaction, which is impossible outside a masterchain block"); } @@ -5793,11 +5800,11 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT PSTRING() << "transaction " << lt << " of account " << addr.to_hex() << " is a tick transaction, but this is not the first transaction of this account"); } - if (lt != start_lt_ + 1) { + if (lt != vq_.start_lt_ + 1) { return reject_query( PSTRING() << "transaction " << lt << " of account " << addr.to_hex() << " is a tick transaction, but its logical start time differs from block's start time " - << start_lt_ << " by more than one"); + << vq_.start_lt_ << " by more than one"); } if (!account.tick) { return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex() @@ -5815,14 +5822,14 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT } } } - if (is_first && is_masterchain() && account.is_special && account.tick && + if (is_first && vq_.is_masterchain() && account.is_special && account.tick && (tag != block::gen::TransactionDescr::trans_tick_tock || (td_cs.prefetch_ulong(4) & 1)) && account.orig_status == block::Account::acc_active) { return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex() << " is the first transaction for this special tick account in this block, but the " "transaction is not a tick transaction"); } - if (is_last && is_masterchain() && account.is_special && account.tock && + if (is_last && vq_.is_masterchain() && account.is_special && account.tock && (tag != block::gen::TransactionDescr::trans_tick_tock || !(td_cs.prefetch_ulong(4) & 1)) && trans.end_status == block::gen::AccountStatus::acc_state_active) { return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex() @@ -5937,15 +5944,15 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT // similar to Collator::create_ordinary_transaction() and Collator::create_ticktock_transaction() // .... std::unique_ptr trs = - std::make_unique(account, trans_type, lt, now_, in_msg_root); + std::make_unique(account, trans_type, lt, vq_.now_, in_msg_root); td::RealCpuTimer timer; SCOPE_EXIT { - stats_.work_time.trx_tvm += trs->time_tvm; - stats_.work_time.trx_storage_stat += trs->time_storage_stat; - stats_.work_time.trx_other += timer.elapsed_both() - trs->time_tvm - trs->time_storage_stat; + ctx_.work_time.trx_tvm += trs->time_tvm; + ctx_.work_time.trx_storage_stat += trs->time_storage_stat; + ctx_.work_time.trx_other += timer.elapsed_both() - trs->time_tvm - trs->time_storage_stat; }; if (in_msg_root.not_null()) { - if (!trs->unpack_input_msg(ihr_delivered, &action_phase_cfg_)) { + if (!trs->unpack_input_msg(ihr_delivered, &vq_.action_phase_cfg_)) { // inbound external message was not accepted return reject_query(PSTRING() << "could not unpack inbound " << (external ? "external" : "internal") << " message processed by ordinary transaction " << lt << " of account " @@ -5953,7 +5960,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT } } if (trs->bounce_enabled) { - if (!trs->prepare_storage_phase(storage_phase_cfg_, true)) { + if (!trs->prepare_storage_phase(vq_.storage_phase_cfg_, true)) { return reject_query(PSTRING() << "cannot re-create storage phase of transaction " << lt << " for smart contract " << addr.to_hex()); } @@ -5966,12 +5973,12 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT return reject_query(PSTRING() << "cannot re-create credit phase of transaction " << lt << " for smart contract " << addr.to_hex()); } - if (!trs->prepare_storage_phase(storage_phase_cfg_, true, need_credit_phase)) { + if (!trs->prepare_storage_phase(vq_.storage_phase_cfg_, true, need_credit_phase)) { return reject_query(PSTRING() << "cannot re-create storage phase of transaction " << lt << " for smart contract " << addr.to_hex()); } } - if (!trs->prepare_compute_phase(compute_phase_cfg_)) { + if (!trs->prepare_compute_phase(vq_.compute_phase_cfg_)) { return reject_query(PSTRING() << "cannot re-create compute phase of transaction " << lt << " for smart contract " << addr.to_hex()); } @@ -5985,43 +5992,39 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT << " of account " << addr.to_hex() << " was not processed without any reason"); } } - if (trs->compute_phase->success && !trs->prepare_action_phase(action_phase_cfg_)) { + if (trs->compute_phase->success && !trs->prepare_action_phase(vq_.action_phase_cfg_)) { return reject_query(PSTRING() << "cannot re-create action phase of transaction " << lt << " for smart contract " << addr.to_hex()); } if (trs->bounce_enabled && (!trs->compute_phase->success || trs->action_phase->state_exceeds_limits || trs->action_phase->bounce) && - !trs->prepare_bounce_phase(action_phase_cfg_)) { + !trs->prepare_bounce_phase(vq_.action_phase_cfg_)) { return reject_query(PSTRING() << "cannot re-create bounce phase of transaction " << lt << " for smart contract " << addr.to_hex()); } - if (!trs->serialize(serialize_cfg_)) { + if (!trs->serialize(vq_.serialize_cfg_)) { return reject_query(PSTRING() << "cannot re-create the serialization of transaction " << lt << " for smart contract " << addr.to_hex()); } - if (!trs->update_limits(*block_limit_status_, /* with_gas = */ false, /* with_size = */ false)) { - return fatal_error(PSTRING() << "cannot update block limit status to include transaction " << lt << " of account " - << addr.to_hex()); - } // Collator should stop if total gas usage exceeds limits, including transactions on special accounts, but without // ticktocks and mint/recover. // Here Validator checks a weaker condition if (!is_special_tx && !trs->gas_limit_overridden && trans_type == block::transaction::Transaction::tr_ord) { - (account.is_special ? total_special_gas_used_ : total_gas_used_) += trs->gas_used(); + (account.is_special ? vq_.total_special_gas_used_ : vq_.total_gas_used_).fetch_add(trs->gas_used()); } - if (total_gas_used_ > block_limits_->gas.hard() + compute_phase_cfg_.gas_limit) { + if (vq_.total_gas_used_.load() > vq_.block_limits_->gas.hard() + vq_.compute_phase_cfg_.gas_limit) { return reject_query(PSTRING() << "gas block limits are exceeded: total_gas_used > gas_limit_hard + trx_gas_limit (" - << "total_gas_used=" << total_gas_used_ - << ", gas_limit_hard=" << block_limits_->gas.hard() - << ", trx_gas_limit=" << compute_phase_cfg_.gas_limit << ")"); + << "total_gas_used=" << vq_.total_gas_used_.load() + << ", gas_limit_hard=" << vq_.block_limits_->gas.hard() + << ", trx_gas_limit=" << vq_.compute_phase_cfg_.gas_limit << ")"); } - if (total_special_gas_used_ > block_limits_->gas.hard() + compute_phase_cfg_.special_gas_limit) { + if (vq_.total_special_gas_used_.load() > vq_.block_limits_->gas.hard() + vq_.compute_phase_cfg_.special_gas_limit) { return reject_query( PSTRING() << "gas block limits are exceeded: total_special_gas_used > gas_limit_hard + special_gas_limit (" - << "total_special_gas_used=" << total_special_gas_used_ - << ", gas_limit_hard=" << block_limits_->gas.hard() - << ", special_gas_limit=" << compute_phase_cfg_.special_gas_limit << ")"); + << "total_special_gas_used=" << vq_.total_special_gas_used_.load() + << ", gas_limit_hard=" << vq_.block_limits_->gas.hard() + << ", special_gas_limit=" << vq_.compute_phase_cfg_.special_gas_limit << ")"); } auto trans_root2 = trs->commit(account); @@ -6031,7 +6034,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT } // now compare the re-created transaction with the one we have if (trans_root2->get_hash() != trans_root->get_hash()) { - if (verbosity >= 3 * 0) { + if (vq_.verbosity >= 3 * 0) { FLOG(INFO) { sb << "original transaction " << lt << " of " << addr.to_hex() << ": "; block::gen::t_Transaction.print_ref(sb, trans_root); @@ -6071,7 +6074,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT << "transaction " << lt << " of " << addr.to_hex() << " is invalid: it has produced a set of outbound messages different from that listed in the transaction"); } - total_burned_ += trs->blackhole_burned; + ctx_.total_burned += trs->blackhole_burned; // check new balance and value flow auto new_balance = account.get_balance(); block::CurrencyCollection total_fees; @@ -6091,50 +6094,176 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT return true; } +ValidateQuery::CheckAccountTxs::CheckAccountTxs(const ValidateQuery& vq, td::actor::ActorId vq_id, + StdSmcAddress address, Ref acc_tr, Context ctx) + : vq_(vq) + , vq_id_(std::move(vq_id)) + , address_(std::move(address)) + , acc_tr_(std::move(acc_tr)) + , ctx_(std::move(ctx)) { +} + /** * Checks the validity of transactions for a given account block. - * NB: may be run in parallel for different accounts * - * @param acc_addr The address of the account. - * @param acc_blk_root The root of the AccountBlock. * * @returns True if the account transactions are valid, false otherwise. */ -bool ValidateQuery::check_account_transactions(const StdSmcAddress& acc_addr, Ref acc_blk_root) { - block::gen::AccountBlock::Record acc_blk; - CHECK(tlb::csr_unpack(std::move(acc_blk_root), acc_blk) && acc_blk.account_addr == acc_addr); - auto account_p = unpack_account(acc_addr.cbits()); - if (!account_p) { - return reject_query("cannot unpack old state of account "s + acc_addr.to_hex()); - } - auto& account = *account_p; - CHECK(account.addr == acc_addr); - vm::AugmentedDictionary trans_dict{vm::DictNonEmpty(), std::move(acc_blk.transactions), 64, - block::tlb::aug_AccountTransactions}; - td::BitArray<64> min_trans, max_trans; - CHECK(trans_dict.get_minmax_key(min_trans).not_null() && trans_dict.get_minmax_key(max_trans, true).not_null()); - ton::LogicalTime min_trans_lt = min_trans.to_ulong(), max_trans_lt = max_trans.to_ulong(); - if (!trans_dict.check_for_each_extra([this, &account, min_trans_lt, max_trans_lt](Ref value, - Ref extra, - td::ConstBitPtr key, int key_len) { - CHECK(key_len == 64); - ton::LogicalTime lt = key.get_uint(64); - extra.clear(); - return check_one_transaction(account, lt, value->prefetch_ref(), lt == min_trans_lt, lt == max_trans_lt); - })) { - return reject_query("at least one Transaction of account "s + acc_addr.to_hex() + " is invalid"); +bool ValidateQuery::CheckAccountTxs::try_check() { + try { + block::gen::AccountBlock::Record acc_blk; + CHECK(tlb::csr_unpack(std::move(acc_tr_), acc_blk) && acc_blk.account_addr == address_); + auto account_p = unpack_account(address_.cbits()); + if (!account_p) { + return reject_query("cannot unpack old state of account "s + address_.to_hex()); + } + auto& account = *account_p; + CHECK(account.addr == address_); + vm::AugmentedDictionary trans_dict{vm::DictNonEmpty(), std::move(acc_blk.transactions), 64, + block::tlb::aug_AccountTransactions}; + td::BitArray<64> min_trans, max_trans; + CHECK(trans_dict.get_minmax_key(min_trans).not_null() && trans_dict.get_minmax_key(max_trans, true).not_null()); + ton::LogicalTime min_trans_lt = min_trans.to_ulong(), max_trans_lt = max_trans.to_ulong(); + if (!trans_dict.check_for_each_extra( + [this, &account, min_trans_lt, max_trans_lt](Ref value, Ref extra, + td::ConstBitPtr key, int key_len) { + CHECK(key_len == 64); + ton::LogicalTime lt = key.get_uint(64); + extra.clear(); + return check_one_transaction(account, lt, value->prefetch_ref(), lt == min_trans_lt, lt == max_trans_lt); + })) { + return reject_query("at least one Transaction of account "s + address_.to_hex() + " is invalid"); + } + if ((!vq_.full_collated_data_ || vq_.is_masterchain()) && account.storage_dict_hash && + account.account_storage_stat && account.account_storage_stat.value().is_dict_ready() && + account.storage_used.cells >= StorageStatCache::MIN_ACCOUNT_CELLS) { + ctx_.storage_stat_cache_update.emplace_back(account.account_storage_stat.value().get_dict_root().move_as_ok(), + account.storage_used.cells); + } + if (vq_.is_masterchain() && account.libraries_changed()) { + return scan_account_libraries(account.orig_library, account.library, address_); + } else { + return true; + } + } catch (vm::VmError& err) { + return fatal_error(err.get_msg(), -666); + } catch (vm::VmVirtError& err) { + return reject_query(err.get_msg()); + } +} + +ValidateQuery::CheckAccountTxs::Context ValidateQuery::CheckAccountTxs::extract_context() { + return std::move(ctx_); +} + +void ValidateQuery::CheckAccountTxs::start_up() { + try_check(); + td::actor::send_closure(vq_id_, &ValidateQuery::after_check_account_finished, address_, extract_context()); + stop(); +} + +void ValidateQuery::CheckAccountTxs::abort_query(td::Status error) { + (void)fatal_error(std::move(error)); +} + +bool ValidateQuery::CheckAccountTxs::reject_query(std::string error, td::BufferSlice reason) { + if (!ctx_.reject_error.has_value() && !ctx_.fatal_error.has_value()) { + ctx_.reject_error = std::move(error); + ctx_.reject_reason = std::move(reason); } - if ((!full_collated_data_ || is_masterchain()) && account.storage_dict_hash && account.account_storage_stat && - account.account_storage_stat.value().is_dict_ready() && - account.storage_used.cells >= StorageStatCache::MIN_ACCOUNT_CELLS) { - storage_stat_cache_update_.emplace_back(account.account_storage_stat.value().get_dict_root().move_as_ok(), - account.storage_used.cells); + return false; +} + +bool ValidateQuery::CheckAccountTxs::reject_query(std::string err_msg, td::Status error, td::BufferSlice reason) { + return reject_query(err_msg + " : " + error.to_string(), std::move(reason)); +} + +bool ValidateQuery::CheckAccountTxs::fatal_error(td::Status error) { + if (!ctx_.fatal_error.has_value() && !ctx_.reject_error.has_value()) { + ctx_.fatal_error = std::move(error); } - if (is_masterchain() && account.libraries_changed()) { - return scan_account_libraries(account.orig_library, account.library, acc_addr); - } else { - return true; + return false; +} + +bool ValidateQuery::CheckAccountTxs::fatal_error(std::string err_msg, int err_code) { + return fatal_error(td::Status::Error(err_code, vq_.error_ctx() + err_msg)); +} + +ValidateQuery::CheckAccountTxs::Context ValidateQuery::load_check_account_transactions_context( + const StdSmcAddress& address) { + CheckAccountTxs::Context ctx{}; + if (account_expected_defer_all_messages_.count(address)) { + ctx.defer_all_messages = true; + } + return ctx; +} + +void ValidateQuery::save_account_transactions_context(const StdSmcAddress& address, CheckAccountTxs::Context ctx) { + if (ctx.fatal_error.has_value()) { + if (!check_account_failed_) { + check_account_failed_ = true; + check_account_fatal_error_ = std::move(ctx.fatal_error); + } + return; + } + + if (ctx.reject_reason.has_value()) { + if (!check_account_failed_) { + check_account_failed_ = true; + check_account_reject_error_ = std::move(ctx.reject_error); + check_account_reject_reason_ = std::move(ctx.reject_reason); + } + return; + } + + for (auto& e : ctx.msg_proc_lt) { + msg_proc_lt_.emplace_back(std::move(e)); + } + + for (auto& e : ctx.lib_publishers) { + lib_publishers_.push_back(e); + } + + if (ctx.defer_all_messages) { + account_expected_defer_all_messages_.insert(address); } + + for (auto& e : ctx.storage_stat_cache_update) { + storage_stat_cache_update_.push_back(e); + } + + stats_.work_time.trx_tvm += ctx.work_time.trx_tvm; + stats_.work_time.trx_storage_stat += ctx.work_time.trx_storage_stat; + stats_.work_time.trx_other += ctx.work_time.trx_other; + stats_.work_time.total += ctx.work_time.total; + + total_burned_ += ctx.total_burned; +} + +void ValidateQuery::after_check_account_finished(StdSmcAddress address, CheckAccountTxs::Context context) { + --pending; + save_account_transactions_context(address, std::move(context)); + if (!pending) { + parallel_accounts_validation_pending_ = false; + parallel_work_timer_.pause(); + stats_.work_time.total += parallel_work_timer_.elapsed_both(); + if (!check_account_failures()) { + reject_query("some accounts failed parallel validation"); + return; + } + try_validate(); + } +} + +bool ValidateQuery::check_account_failures() { + if (check_account_fatal_error_.has_value()) { + return fatal_error(std::move(check_account_fatal_error_.value())); + } + if (check_account_reject_error_.has_value()) { + return reject_query(std::move(check_account_reject_error_.value()), + std::move(check_account_reject_reason_.value())); + } + return true; } /** @@ -6144,17 +6273,47 @@ bool ValidateQuery::check_account_transactions(const StdSmcAddress& acc_addr, Re */ bool ValidateQuery::check_transactions() { LOG(INFO) << "checking all transactions"; - return account_blocks_dict_->check_for_each_extra( - [this](Ref value, Ref extra, td::ConstBitPtr key, int key_len) { + size_t accounts_count = 0; + bool result = account_blocks_dict_->check_for_each_extra( + [this, &accounts_count](Ref value, Ref extra, td::ConstBitPtr key, int key_len) { CHECK(key_len == 256); - return check_account_transactions(key, std::move(value)); + accounts_count++; + StdSmcAddress address = key; + if (parallel_accounts_validation_) { + pending++; + LOG(INFO) << "starting actor #" << accounts_count << " for account " << address.to_hex(); + td::actor::create_actor(PSTRING() << get_name() << ":#" << accounts_count, *this, + actor_id(this), address, std::move(value), + load_check_account_transactions_context(address)) + .release(); + return true; + } else { + CheckAccountTxs checker(*this, actor_id(this), address, std::move(value), + load_check_account_transactions_context(address)); + bool result = checker.try_check(); + save_account_transactions_context(address, checker.extract_context()); + return result; + } }); + if (accounts_count == 0 && parallel_accounts_validation_) { + parallel_accounts_validation_ = false; + } + if (parallel_accounts_validation_) { + stats_.parallel_accounts_validation = true; + parallel_accounts_validation_pending_ = true; + parallel_work_timer_.resume(); + } else { + if (!check_account_failures()) { + return reject_query("some accounts failed to be validated"); + } + } + return result; } /** * Processes changes in libraries of an account. * Used in masterchain validation. - * Similar to Collator::update_account_public_libraries() + * Similar to `Collator::update_account_public_libraries()`. * * @param orig_libs The original libraries of the account. * @param final_libs The final libraries of the account. @@ -6162,7 +6321,8 @@ bool ValidateQuery::check_transactions() { * * @returns True if the update was successful, false otherwise. */ -bool ValidateQuery::scan_account_libraries(Ref orig_libs, Ref final_libs, const td::Bits256& addr) { +bool ValidateQuery::CheckAccountTxs::scan_account_libraries(Ref orig_libs, Ref final_libs, + const td::Bits256& addr) { vm::Dictionary dict1{std::move(orig_libs), 256}, dict2{std::move(final_libs), 256}; return dict1.scan_diff( dict2, @@ -6171,7 +6331,7 @@ bool ValidateQuery::scan_account_libraries(Ref orig_libs, Ref=9.0.1", + "tontester", +] + +[dependency-groups] +dev = [ + "basedpyright>=1.32.1", + "ruff>=0.14.2", +] + +[tool.uv.sources] +tontester = { workspace = true } + +[tool.uv.workspace] +members = [ + "test/tontester", +] + +[tool.ruff] +line-length = 100 +include = ["test/integration/**/*.py", "test/tontester/**/*.py"] + +[tool.ruff.lint] +# Default rules with isort +select = [ + # pycodestyle + "E4", "E7", "E9", + # Pyflakes + "F", + # isort + "I", +] + +ignore = [ + # This disallows to use `l` and `O` as variable names because "they can be confused with `1` and + # `0`". My response will be use a proper font that makes them distinguishable. + "E741", +] + +[tool.pyright] +include = ["test/integration", "test/tontester"] + +[tool.pytest] +testpaths = ["test/tontester/tests"] diff --git a/test/integration/test_basic.py b/test/integration/test_basic.py new file mode 100644 index 0000000000..e50ba7699e --- /dev/null +++ b/test/integration/test_basic.py @@ -0,0 +1,42 @@ +import asyncio +import logging +import shutil +from pathlib import Path + +from tontester.install import Install +from tontester.network import Network + + +async def main(): + repo_root = Path(__file__).resolve().parents[2] + working_dir = repo_root / "test/integration/.network" + shutil.rmtree(working_dir, ignore_errors=True) + working_dir.mkdir(exist_ok=True) + + install = Install(repo_root / "build", repo_root) + + logging.basicConfig( + level=logging.INFO, + format="[%(levelname)s][%(asctime)s][%(name)s] %(message)s", + datefmt="%Y-%m-%d %H-%M-%S", + ) + + async with Network(install, working_dir) as network: + dht = network.create_dht_node() + + nodes: list[Network.Node] = [] + for _ in range(2): + node = network.create_full_node() + node.make_initial_validator() + node.announce_to(dht) + nodes.append(node) + + await dht.run() + for node in nodes: + await node.run() + + await network.wait_mc_block(seqno=1) + + +if __name__ == "__main__": + asyncio.run(asyncio.wait_for(main(), 5 * 60)) diff --git a/test/tontester/generate_tl.py b/test/tontester/generate_tl.py new file mode 100644 index 0000000000..fbbaab6169 --- /dev/null +++ b/test/tontester/generate_tl.py @@ -0,0 +1,22 @@ +from pathlib import Path + +import tl + +if __name__ == "__main__": + repo_root = Path(__file__).resolve().parents[2] + schemas_root = repo_root / "tl/generate/scheme" + + schemas = [ + schemas_root / "lite_api.tl", + schemas_root / "ton_api.tl", + schemas_root / "tonlib_api.tl", + ] + out_directory = repo_root / "test/tontester/src/tontester/tl" + + for schema in schemas: + tl.generate(schema, out_directory) + + tl.generate( + repo_root / "test/tontester/tests/tl/test_schema.tl", + repo_root / "test/tontester/tests/tl/generated", + ) diff --git a/test/tontester/pyproject.toml b/test/tontester/pyproject.toml new file mode 100644 index 0000000000..dbcb28d1cc --- /dev/null +++ b/test/tontester/pyproject.toml @@ -0,0 +1,8 @@ +[project] +name = "tontester" +version = "0.1.0" +requires-python = ">=3.13" +dependencies = [ + "pytonlib>=0.0.68", + "setuptools<81", # FIXME: Get rid of pytonlib instead. +] diff --git a/test/tontester/src/tl/__init__.py b/test/tontester/src/tl/__init__.py new file mode 100644 index 0000000000..c9624967b0 --- /dev/null +++ b/test/tontester/src/tl/__init__.py @@ -0,0 +1,47 @@ +from .gen import generate +from .tlobject import ( + JSONSerializable, + ModelError, + TLObject, + TLRequest, + deserialize_Bool, + deserialize_bytes, + deserialize_double, + deserialize_int, + deserialize_int32, + deserialize_int53, + deserialize_int64, + deserialize_int128, + deserialize_int256, + deserialize_list, + deserialize_long, + deserialize_object, + deserialize_secureBytes, + deserialize_secureString, + deserialize_string, + deserialize_true, +) + +__all__ = [ + "generate", + "JSONSerializable", + "ModelError", + "TLObject", + "TLRequest", + "deserialize_Bool", + "deserialize_bytes", + "deserialize_double", + "deserialize_int", + "deserialize_int128", + "deserialize_int256", + "deserialize_int32", + "deserialize_int53", + "deserialize_int64", + "deserialize_list", + "deserialize_long", + "deserialize_object", + "deserialize_secureBytes", + "deserialize_secureString", + "deserialize_string", + "deserialize_true", +] diff --git a/test/tontester/src/tl/gen.py b/test/tontester/src/tl/gen.py new file mode 100644 index 0000000000..3653206503 --- /dev/null +++ b/test/tontester/src/tl/gen.py @@ -0,0 +1,10 @@ +from pathlib import Path + + +def generate(schema: Path, out_directory: Path): + from .generator.generators import generate_tlobjects + from .generator.parsers import parse_tl + + objects = parse_tl(schema) + filename = schema.name.split(".")[0] + generate_tlobjects(objects, filename, out_directory) diff --git a/test/tontester/src/tl/generator/generators/__init__.py b/test/tontester/src/tl/generator/generators/__init__.py new file mode 100644 index 0000000000..446c6a4b72 --- /dev/null +++ b/test/tontester/src/tl/generator/generators/__init__.py @@ -0,0 +1,5 @@ +from .tlobject import generate_tlobjects + +__all__ = [ + "generate_tlobjects", +] diff --git a/test/tontester/src/tl/generator/generators/tlobject.py b/test/tontester/src/tl/generator/generators/tlobject.py new file mode 100644 index 0000000000..7b9dd4ec52 --- /dev/null +++ b/test/tontester/src/tl/generator/generators/tlobject.py @@ -0,0 +1,266 @@ +import io +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path + +from ..parsers.tlobject import ParsedTLObject +from ..sourcebuilder import SourceBuilder +from ..utils import get_class_name + +BASE_TYPES = { + "string", + "bytes", + "int", + "long", + "int128", + "int256", + "int64", + "int32", + "int53", + "double", + "Bool", + "true", + "secureString", + "secureBytes", +} + +BASE64_ENCODED_TYPES = { + "bytes", + "int128", + "int256", + "secureBytes", +} + + +@dataclass +class ImportContext: + literal_needed: bool = False + base64_needed: bool = False + + +def _write_modules( + out_file: Path, + tlobjects: list[ParsedTLObject], + type_constructors: dict[str, list[ParsedTLObject]], +): + out_file.parent.mkdir(parents=True, exist_ok=True) + source = io.StringIO() + ctx = ImportContext() + with SourceBuilder(source) as builder: + type_names: set[str] = set() + type_defs: list[str] = [] + + for t in tlobjects: + if t.name in BASE_TYPES: + continue + if not t.is_function: + type_name = get_class_name(t.result) + type_names.add(type_name) + constructors = type_constructors[t.result] + if not constructors: + pass + elif len(constructors) == 1: + type_defs.append( + "type Type{} = {}".format(type_name, constructors[0].class_name) + ) + else: + type_defs.append( + f"type Type{type_name} = {' | '.join(c.class_name for c in constructors)}" + ) + + # Generate the class for every TLObject + for t in tlobjects: + if t.name in BASE_TYPES: + continue + _write_source_code(t, t.is_function, builder, type_constructors, ctx) + builder.current_indent = 0 + + # Write the type definitions generated earlier. + builder.writeln() + for line in set(type_defs): + builder.writeln(line) + + with out_file.open("w") as f, SourceBuilder(f) as builder: + if ctx.literal_needed: + builder.writeln("from typing import override, Self, Literal") + else: + builder.writeln("from typing import override, Self") + if ctx.base64_needed: + builder.writeln("import base64") + builder.writeln("import tl") + builder.writeln() + builder.writeln(source.getvalue()) + + +def _write_source_code( + tlobject: ParsedTLObject, + is_function: bool, + builder: SourceBuilder, + type_constructors: dict[str, list[ParsedTLObject]], + ctx: ImportContext, +): + """ + Writes the source code corresponding to the given TLObject + by making use of the ``builder`` `SourceBuilder`. + + Additional information such as file path depth and + the ``Type: [Constructors]`` must be given for proper + importing and documentation strings. + """ + _write_class_init(tlobject, is_function, builder, ctx) + _write_to_dict(tlobject, builder, ctx) + _write_from_dict(tlobject, type_constructors, builder) + + +def _write_class_init( + tlobject: ParsedTLObject, + is_function: bool, + builder: SourceBuilder, + ctx: ImportContext, +): + builder.writeln() + builder.writeln() + builder.writeln( + "class {}({}):", tlobject.class_name, "tl.TLRequest" if is_function else "tl.TLObject" + ) + + args = [a.get_init_arg() for a in tlobject.args] + + # Write the __init__ function if it has any argument + if not tlobject.args: + return + + builder.writeln(f"def __init__(self, {', '.join(args)}):") + + # Set the arguments + for arg in tlobject.args: + if arg.type == "true": + ctx.literal_needed = True + builder.writeln(arg.get_init_assignment()) + + builder.end_block() + + +def _write_to_dict(tlobject: ParsedTLObject, builder: SourceBuilder, ctx: ImportContext): + builder.writeln("@override") + builder.writeln("def to_dict(self) -> dict[str, tl.JSONSerializable]:") + builder.writeln("return {") + builder.current_indent += 1 + + builder.write("'@type': '{}'", tlobject.fullname) + for arg in tlobject.args: + builder.writeln(",") + if arg.type in BASE_TYPES: + builder.write(f"'{arg.name}': ") + if arg.type in BASE64_ENCODED_TYPES: + ctx.base64_needed = True + if arg.is_vector: + builder.write(f"[base64.b64encode(x).decode() for x in self.{arg.name}]") + else: + builder.write(f"base64.b64encode(self.{arg.name}).decode()") + else: + if arg.is_vector: + builder.write("self.{0}[:]", arg.name) + else: + builder.write("self.{}", arg.name) + else: + if arg.is_vector: + builder.write(f"'{arg.name}': ") + builder.write(f"[x.to_dict() for x in self.{arg.name}]") + else: + builder.write( + f"**({{'{arg.name}': self.{arg.name}.to_dict()}} " + + f"if self.{arg.name} is not None else {{}})" + ) # do not write object in json if it's None + + builder.writeln() + builder.current_indent -= 1 + builder.writeln("}") + + builder.end_block() + + +def _write_from_dict( + tlobject: ParsedTLObject, + type_constructors: dict[str, list[ParsedTLObject]], + builder: SourceBuilder, +): + builder.writeln("@classmethod") + builder.writeln("@override") + builder.writeln("def from_dict(cls, d: tl.JSONSerializable) -> Self:") + + builder.writeln("if not isinstance(d, dict):") + builder.writeln('raise tl.ModelError(f"Expected dict, got {type(d)}")') + builder.current_indent -= 1 + + builder.writeln(f'if d.get("@type", "{tlobject.fullname}") != "{tlobject.fullname}":') + builder.writeln( + f'raise tl.ModelError(f"Expected @type to be {tlobject.fullname}, got {{d.get("@type")}}")' + ) + builder.current_indent -= 1 + + for arg in tlobject.args: + if arg.type in BASE_TYPES: + deserializer = f"tl.deserialize_{arg.type}" + elif arg.skip_constructor_id: + name = arg.get_type_class_name() + if not arg.is_vector: + deserializer = f"tl.deserialize_object({name}.from_dict)" + else: + deserializer = f"{name}.from_dict" + else: + constructors = type_constructors.get(arg.type, []) + builder.writeln( + f"def deserialize_{arg.name}(value: tl.JSONSerializable) -> Type{arg.get_type_class_name()}:" + ) + builder.writeln("if not isinstance(value, dict):") + builder.writeln('raise tl.ModelError(f"Expected dict, got {type(value)}")') + builder.current_indent -= 1 + for c in constructors: + builder.writeln( + f'if value.get("@type") == "{c.fullname}": return {c.class_name}.from_dict(value)' + ) + builder.writeln( + f'raise tl.ModelError(f"Unknown constructor for {arg.type}: {{value.get("@type")}}")' + ) + builder.current_indent -= 1 + if not arg.is_vector: + deserializer = f"tl.deserialize_object(deserialize_{arg.name})" + else: + deserializer = f"deserialize_{arg.name}" + + if not arg.is_vector: + builder.writeln( + f"_{arg.name} = " + + f"{deserializer}(d.get('{arg.name}', {arg.default_value_for_from_dict()}))" + ) + else: + builder.writeln( + f"_{arg.name} = tl.deserialize_list(d.get('{arg.name}', []), {deserializer})" + ) + + builder.writeln( + "return cls({})", + ", ".join(f"{arg.name}=_{arg.name}" for arg in tlobject.args), + ) + + builder.end_block() + + +def sort_tlobjects(tlobjects: list[ParsedTLObject]): + types: list[ParsedTLObject] = [] + type_constructors: dict[str, list[ParsedTLObject]] = defaultdict(list) + for tlobject in tlobjects: + types.append(tlobject) + if not tlobject.is_function: + type_constructors[tlobject.result].append(tlobject) + return types, type_constructors + + +def generate_tlobjects(tlobjects: list[ParsedTLObject], fn: str, output_dir: Path): + output_dir.mkdir(parents=True, exist_ok=True) + py_typed_marker = output_dir / "py.typed" + py_typed_marker.touch() + + types, type_constructors = sort_tlobjects(tlobjects) + _write_modules(output_dir / (fn + ".py"), types, type_constructors) diff --git a/test/tontester/src/tl/generator/parsers/__init__.py b/test/tontester/src/tl/generator/parsers/__init__.py new file mode 100644 index 0000000000..3a9c03930b --- /dev/null +++ b/test/tontester/src/tl/generator/parsers/__init__.py @@ -0,0 +1,6 @@ +from .tlobject import ParsedTLObject, parse_tl + +__all__ = [ + "ParsedTLObject", + "parse_tl", +] diff --git a/test/tontester/src/tl/generator/parsers/tlobject/__init__.py b/test/tontester/src/tl/generator/parsers/tlobject/__init__.py new file mode 100644 index 0000000000..af3b29ad5b --- /dev/null +++ b/test/tontester/src/tl/generator/parsers/tlobject/__init__.py @@ -0,0 +1,9 @@ +from .parsedtlobject import ParsedTLObject +from .parser import parse_tl +from .tlarg import TLArg + +__all__ = [ + "TLArg", + "ParsedTLObject", + "parse_tl", +] diff --git a/test/tontester/src/tl/generator/parsers/tlobject/parsedtlobject.py b/test/tontester/src/tl/generator/parsers/tlobject/parsedtlobject.py new file mode 100644 index 0000000000..c848ccc3cf --- /dev/null +++ b/test/tontester/src/tl/generator/parsers/tlobject/parsedtlobject.py @@ -0,0 +1,95 @@ +import re +import zlib +from typing import override + +from ...utils import get_class_name +from .tlarg import TLArg + + +class ParsedTLObject: + def __init__( + self, + fullname: str, + object_id: str | None, + args: list[TLArg], + result: str, + is_function: bool, + ): + """ + Initializes a new TLObject, given its properties. + + :param fullname: The fullname of the TL object (namespace.name) + The namespace can be omitted. + :param object_id: The hexadecimal string representing the object ID + :param args: The arguments, if any, of the TL object + :param result: The result type of the TL object + :param is_function: Is the object a function or a type? + """ + # The name can or not have a namespace + self.fullname: str = fullname + self.namespace: str + self.name: str + if "." in fullname: + # self.namespace, self.name = fullname.split('.', maxsplit=1) + self.namespace, self.name = fullname.split(".")[0], fullname + else: + self.namespace, self.name = "", fullname + + self.name = self.name.replace(".", "_") + + self.args: list[TLArg] = args + self.result: str = result + self.is_function: bool = is_function + self.id: int + if object_id is None: + self.id = self.infer_id() + else: + self.id = int(object_id, base=16) + + # self.class_name: str = snake_to_camel_case(self.name, suffix='Request' if self.is_function else '') + self.class_name: str = get_class_name( + self.name, suffix="Request" if self.is_function else "" + ) + + self.real_args: list[TLArg] = list( + a + for a in self.sorted_args() + if + # (a.flag_indicator or a.generic_definition)) + not a.generic_definition + ) + + def sorted_args(self): + """Returns the arguments properly sorted and ready to plug-in + into a Python's method header (i.e., flags and those which + can be inferred will go last so they can default =None) + """ + return sorted(self.args, key=lambda x: bool(x.flag)) + + @override + def __repr__(self, ignore_id: bool = False): + if ignore_id: + hex_id = "" + else: + hex_id = "#{:08x}".format(self.id) + + if self.args: + args = " " + " ".join([repr(arg) for arg in self.args]) + else: + args = "" + + return "{}{}{} = {}".format(self.fullname, hex_id, args, self.result) + + def infer_id(self): + representation = self.__repr__(ignore_id=True) + representation = ( + representation.replace("<", " ") + .replace(">", "") + .replace("{", "") + .replace("}", "") + .replace("Vector", "vector") + ) + + # Remove optional empty values (special-cased to the true type) + representation = re.sub(r" \w+:\w+\.\d+\?true", r"", representation) + return zlib.crc32(representation.encode("ascii")) diff --git a/test/tontester/src/tl/generator/parsers/tlobject/parser.py b/test/tontester/src/tl/generator/parsers/tlobject/parser.py new file mode 100644 index 0000000000..fdb82624b0 --- /dev/null +++ b/test/tontester/src/tl/generator/parsers/tlobject/parser.py @@ -0,0 +1,150 @@ +import collections +import re +from pathlib import Path + +from .parsedtlobject import ParsedTLObject +from .tlarg import TLArg + +CORE_TYPES = { + 0xBC799737, # boolFalse#bc799737 = Bool; + 0x997275B5, # boolTrue#997275b5 = Bool; + 0x3FEDD339, # true#3fedd339 = True; + 0xC4B9F9BB, # error#c4b9f9bb code:int text:string = Error; + 0x56730BCC, # null#56730bcc = Null; +} + + +def _from_line(line: str, is_function: bool) -> ParsedTLObject: + match = re.match( + r"^([\w.]+)" # 'name' + + r"(?:#([0-9a-fA-F]+))?" # '#optionalcode' + + + # r'(?:\s{?\w+:(?:\([^)]*\)|[\w\d<>#.?!()]+)}?)*' + r".*?" + + r"\s=\s" # ' = ' + + r"([\w\d<>#.?]+);$", # ';' + line, + ) + if match is None: + raise ValueError("Cannot parse TLObject {}".format(line)) + + args_match: list[tuple[str, str, str]] = re.findall( + r"({)?" + + r"(\w+)" + + r":" + + + # r'([\w\d<>#.?!]+)', + r"(" + + + # r'(?:\([^)]*\)|[\w\d<>#.?!]+)' + r"(?:[\w\d<>#.?!]+(?:\([^)]*\))?|\([^)]*\))" + + r")", + line, + ) + + for i, (brace, name, arg_type) in enumerate(args_match): + if "vector" in arg_type: + arg_type = arg_type.replace("(", "").replace(")", ">").replace("vector ", "Vector<") + if "(" in arg_type and ")" in arg_type: + arg_type = arg_type.replace("(", "").replace(")", "") + args_match[i] = (brace, name, arg_type) + + name = match.group(1) + + return ParsedTLObject( + fullname=name, + object_id=match.group(2), + result=match.group(3), + is_function=is_function, + args=[TLArg(name, arg_type, brace != "") for brace, name, arg_type in args_match], + ) + + +def parse_tl(file_path: Path, ignored_ids: set[int] = CORE_TYPES) -> list[ParsedTLObject]: + """ + This method yields TLObjects from a given .tl file. + + Note that the file is parsed completely before the function yields + because references to other objects may appear later in the file. + """ + obj_all: list[ParsedTLObject] = [] + obj_by_name = {} + obj_by_type: dict[str, list[ParsedTLObject]] = collections.defaultdict(list) + with file_path.open() as file: + incomplete = "" + is_function = False + for line in file: + comment_index = line.find("//") + if comment_index != -1: + line = line[:comment_index] + + line = line.strip() + if not line: + continue + + match = re.match(r"---(\w+)---", line) + if match: + following_types = match.group(1) + is_function = following_types == "functions" + continue + + if ";" not in line: + incomplete += line + " " + continue + elif incomplete: + line = incomplete + line + incomplete = "" + + try: + result = _from_line(line, is_function) + + if result.id in ignored_ids: + continue + + obj_all.append(result) + if not result.is_function: + obj_by_name[result.fullname] = result + obj_by_type[result.result].append(result) + except ValueError as e: + if all( + [ + x not in str(e) + for x in ( + "int ? = Int;", + "long ? = Long", + "double ? = Double;", + "string ? = String;", + "object ? = Object;", + "function ? = Function;", + "bytes data:string = Bytes;", + "true = True;", + "boolTrue = Bool;", + "boolFalse = Bool;", + "vector#1cb5c415 {t:Type} # [ t ] = Vector t;", + "vector {t:Type} # [ t ] = Vector t;", + "int128 4*[ int ] = Int128;", + "int256 8*[ int ] = Int256;", + ) + ] + ): + raise + + # Once all objects have been parsed, replace the + # string type from the arguments with references + # for obj in obj_all: + # for arg in obj.args: + # arg.cls = obj_by_type.get(arg.type) or ( + # [obj_by_name[arg.type]] if arg.type in obj_by_name else [] + # ) + + return obj_all + + +def find_layer(file_path: Path): + """Finds the layer used on the specified scheme.tl file.""" + layer_regex = re.compile(r"^//\s*LAYER\s*(\d+)$") + with file_path.open("r") as file: + for line in file: + match = layer_regex.match(line) + if match: + return int(match.group(1)) diff --git a/test/tontester/src/tl/generator/parsers/tlobject/tlarg.py b/test/tontester/src/tl/generator/parsers/tlobject/tlarg.py new file mode 100644 index 0000000000..7a95840481 --- /dev/null +++ b/test/tontester/src/tl/generator/parsers/tlobject/tlarg.py @@ -0,0 +1,162 @@ +import re + +from ....generator.utils import get_class_name + + +class TLArg: + def __init__(self, name: str, arg_type: str, generic_definition: bool): + """ + Initializes a new .tl argument + :param name: The name of the .tl argument + :param arg_type: The type of the .tl argument + :param generic_definition: Is the argument a generic definition? + (i.e. {X:Type}) + """ + MANGLING = { + "list": "list_", + "self": "self_", + "from": "from_", + "bytes": "bytes_", + } + + self.name: str = MANGLING.get(name, name) + + # Default values + self.is_vector: bool = False + self.flag: str | None = None # name of the flag to check if self is present + self.skip_constructor_id: bool = False + self.flag_index: int = -1 # bit index of the flag to check if self is present + # self.cls: list[ParsedTLObject] | None = None + + # The type can be an indicator that other arguments will be flags + self.flag_indicator: bool + self.type: str + self.is_generic: bool + if arg_type == "#": + self.flag_indicator = True + self.type = "int" + self.is_generic = False + else: + self.flag_indicator = False + self.is_generic = arg_type.startswith("!") + # Strip the exclamation mark always to have only the name + self.type = arg_type.lstrip("!") + + # The type may be a flag (FLAGS.IDX?REAL_TYPE) + # FLAGS can be any name, but it should have appeared previously. + flag_match = re.match(r"(\w+).(\d+)\?([\w<>.]+)", self.type) + if flag_match: + self.flag = flag_match.group(1) + self.flag_index = int(flag_match.group(2)) + # Update the type to match the exact type, not the "flagged" one + self.type = flag_match.group(3) + + # Then check if the type is a Vector + vector_match = re.match(r"[Vv]ector<([\w\d.]+)>", self.type) + if vector_match: + self.is_vector = True + + # If the type's first letter is not uppercase, then + # it is a constructor and we use (read/write) its ID + self.use_vector_id: bool = self.type[0] == "V" + + # Update the type to match the one inside the vector + self.type = vector_match.group(1) + + # See use_vector_id. An example of such case is ipPort in + # help.configSpecial + if self.type.split(".")[-1][0].islower(): + self.skip_constructor_id = True + # if self.type in ('Int128' or 'Int256'): + # self.type = 'bytes' + + self.generic_definition: bool = generic_definition + + def get_type_class_name(self): + return get_class_name(self.type) + + def type_hint(self): + cls = self.type + # if '.' in cls: + # cls = snake_to_camel_case('_'.join(cls.split('.'))) + result = { + "int": "int", + "long": "int", + "int64": "int", + "int32": "int", + "int53": "int", + "int128": "bytes", + "int256": "bytes", + "double": "float", + "string": "str", + "bytes": "bytes", + "Bool": "bool", + "true": "Literal[True]", + "secureString": "str", + "secureBytes": "bytes", + }.get(cls) + + is_primitive = result is not None + if not is_primitive: + result = self.get_type_class_name() + if not self.skip_constructor_id: + result = f"Type{result}" + + if self.is_vector: + result = f"list[{result}]" + elif not is_primitive: + result = f"{result} | None" + + return result + + def get_init_arg(self): + if self.is_vector: + return f"{self.name}: '{self.type_hint()} | None' = None" + else: + return f"{self.name}: '{self.type_hint()}' = {self.default_value()}" + + def get_init_assignment(self) -> str: + if self.is_vector: + return f"self.{self.name}: {self.type_hint()} = {self.name} if {self.name} is not None else []" + else: + return f"self.{self.name}: {self.type_hint()} = {self.name}" + + def default_value(self): + if self.is_vector: + return "[]" + elif self.type in ("int", "long", "int64", "int32", "int53"): + return "0" + elif self.type == "bytes": + return "b''" + elif self.type == "int128": + return "b'\\x00' * 16" + elif self.type == "int256": + return "b'\\x00' * 32" + elif self.type == "string": + return "''" + elif self.type == "double": + return "0.0" + elif self.type == "Bool": + return "False" + elif self.type == "true": + return "True" + elif self.type == "secureString": + return "''" + elif self.type == "secureBytes": + return "b''" + else: + return "None" + + def default_value_for_from_dict(self): + if self.is_vector: + return "[]" + elif self.type == "bytes": + return "''" + elif self.type == "int128": + return "'AAAAAAAAAAAAAAAAAAAAAA=='" + elif self.type == "int256": + return "'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='" + elif self.type == "secureBytes": + return "''" + else: + return self.default_value() diff --git a/test/tontester/src/tl/generator/sourcebuilder.py b/test/tontester/src/tl/generator/sourcebuilder.py new file mode 100644 index 0000000000..43812b93d3 --- /dev/null +++ b/test/tontester/src/tl/generator/sourcebuilder.py @@ -0,0 +1,70 @@ +import typing +from typing import override + + +class SourceBuilder: + """This class should be used to build .py source files""" + + def __init__(self, out_stream: typing.TextIO, indent_size: int = 4): + self.current_indent: int = 0 + self.on_new_line: bool = False + self.indent_size: int = indent_size + self.out_stream: typing.TextIO = out_stream + + # Was a new line added automatically before? If so, avoid it + self.auto_added_line: bool = False + + def indent(self): + """Indents the current source code line + by the current indentation level + """ + self.write(" " * (self.current_indent * self.indent_size)) + + def write(self, string: str, *args: str | tuple[str], **kwargs: dict[str, str]): + """Writes a string into the source code, + applying indentation if required + """ + if self.on_new_line: + self.on_new_line = False # We're not on a new line anymore + # If the string was not empty, indent; Else probably a new line + if string.strip(): + self.indent() + + if args or kwargs: + _ = self.out_stream.write(string.format(*args, **kwargs)) + else: + _ = self.out_stream.write(string) + + def writeln(self, string: str = "", *args: str | tuple[str], **kwargs: dict[str, str]) -> None: + """Writes a string into the source code _and_ appends a new line, + applying indentation if required + """ + self.write(string + "\n", *args, **kwargs) + self.on_new_line = True + + # If we're writing a block, increment indent for the next time + if string and string[-1] == ":": + self.current_indent += 1 + + # Clear state after the user adds a new line + self.auto_added_line = False + + def end_block(self): + """Ends an indentation block, leaving an empty line afterwards""" + self.current_indent -= 1 + + # If we did not add a new line automatically yet, now it's the time! + if not self.auto_added_line: + self.writeln() + self.auto_added_line = True + + @override + def __str__(self): + _ = self.out_stream.seek(0) + return self.out_stream.read() + + def __enter__(self): + return self + + def __exit__(self, exc_type: str, exc_val: str, exc_tb: str): + pass diff --git a/test/tontester/src/tl/generator/utils.py b/test/tontester/src/tl/generator/utils.py new file mode 100644 index 0000000000..27cc73418d --- /dev/null +++ b/test/tontester/src/tl/generator/utils.py @@ -0,0 +1,15 @@ +import re + + +def snake_to_camel_case(name: str, suffix: str = "") -> str: + # Courtesy of http://stackoverflow.com/a/31531797/4759433 + result = re.sub(r"_([a-z])", lambda m: m.group(1).upper(), name) + result = result[:1].upper() + result[1:].replace("_", "") + return result + suffix if suffix else result + + +def get_class_name(name: str, suffix: str = ""): + if "." in name: + name = name.replace(".", "_") + # return snake_to_camel_case(name, suffix) + return name[0].upper() + name[1:] + suffix diff --git a/test/tontester/src/tl/py.typed b/test/tontester/src/tl/py.typed new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/tontester/src/tl/tlobject.py b/test/tontester/src/tl/tlobject.py new file mode 100644 index 0000000000..04c050a8a5 --- /dev/null +++ b/test/tontester/src/tl/tlobject.py @@ -0,0 +1,218 @@ +import base64 +import json +from abc import ABC, abstractmethod +from collections.abc import Mapping, Sequence +from typing import Callable, Self, TextIO, cast, overload, override + +type JSONSerializable = ( + None + | bool + | int + | float + | str + | bytes + | Sequence["JSONSerializable"] + | Mapping[str, "JSONSerializable"] +) + + +class TLObject(ABC): + @staticmethod + def serialize_bytes(data: bytes | str) -> bytes: + """Write bytes by using Telegram guidelines""" + if not isinstance(data, bytes): + data = data.encode("utf-8") + + r: list[bytes] = [] + if len(data) < 254: + padding = (len(data) + 1) % 4 + if padding != 0: + padding = 4 - padding + + r.append(bytes([len(data)])) + r.append(data) + + else: + padding = len(data) % 4 + if padding != 0: + padding = 4 - padding + + r.append(bytes([254, len(data) % 256, (len(data) >> 8) % 256, (len(data) >> 16) % 256])) + r.append(data) + + r.append(bytes(padding)) + return b"".join(r) + + @override + def __eq__(self, o: object): + return isinstance(o, type(self)) and self.to_dict() == o.to_dict() + + @override + def __ne__(self, o: object): + return not isinstance(o, type(self)) or self.to_dict() != o.to_dict() + + @override + def __str__(self): + return self.__class__.__name__ + f"<{str(self.to_dict())}>" + + @abstractmethod + def to_dict(self) -> dict[str, JSONSerializable]: + raise NotImplementedError + + @overload + def to_json(self, fp: TextIO) -> None: ... + + @overload + def to_json(self, fp: None = None) -> str: ... + + def to_json(self, fp: TextIO | None = None): + """ + Represent the current `TLObject` as JSON. + + If ``fp`` is given, the JSON will be dumped to said + file pointer, otherwise a JSON string will be returned. + + Note that bytes cannot be represented + in JSON, so if those are found, they will be base64 + encoded and ISO-formatted, respectively, by default. + """ + + def encoder(value: JSONSerializable) -> JSONSerializable: + if isinstance(value, bytes): + return base64.b64encode(value).decode("ascii") + else: + return value + + d = self.to_dict() + if fp: + return json.dump(d, fp, default=encoder) + else: + return json.dumps(d, default=encoder) + + @classmethod + @abstractmethod + def from_dict(cls, d: JSONSerializable) -> Self: + raise NotImplementedError + + @classmethod + def from_json(cls, source: str) -> Self: + return cls.from_dict(cast(JSONSerializable, json.loads(source))) + + +class TLRequest(TLObject, ABC): + pass + + +class ModelError(Exception): + pass + + +def _deserialize_signed_int(value: JSONSerializable, bits: int): + if not isinstance(value, int): + if bits > 53 and isinstance(value, str): + try: + value = int(value) + except ValueError as e: + raise ModelError(f"Expected int, got {type(value)}") from e + else: + raise ModelError(f"Expected int, got {type(value)}") + min_value = -(1 << (bits - 1)) + max_value = (1 << (bits - 1)) - 1 + if min_value <= value <= max_value: + return value + raise ModelError(f"Integer value {value} out of bounds [{min_value}, {max_value}]") + + +def _deserialize_byte_string(value: JSONSerializable, length: int = -1): + if not isinstance(value, str): + raise ModelError(f"Expected base64-encoded string, got {type(value)}") + try: + decoded = base64.b64decode(value) + if length != -1 and len(decoded) != length: + raise ModelError(f"Expected byte string of length {length}, got {len(decoded)}") + return decoded + except Exception as e: + raise ModelError(f"Invalid base64-encoded string: {e}") from e + + +def deserialize_int(value: JSONSerializable): + return _deserialize_signed_int(value, 32) + + +def deserialize_long(value: JSONSerializable): + return _deserialize_signed_int(value, 64) + + +def deserialize_int128(value: JSONSerializable): + return _deserialize_byte_string(value, 16) + + +def deserialize_int256(value: JSONSerializable): + return _deserialize_byte_string(value, 32) + + +def deserialize_int64(value: JSONSerializable): + return _deserialize_signed_int(value, 64) + + +def deserialize_int32(value: JSONSerializable): + return _deserialize_signed_int(value, 32) + + +def deserialize_int53(value: JSONSerializable): + return _deserialize_signed_int(value, 53) + + +def deserialize_double(value: JSONSerializable): + if not isinstance(value, (float, int)): + raise ModelError(f"Expected float, got {type(value)}") + return float(value) + + +def deserialize_string(value: JSONSerializable): + if not isinstance(value, str): + raise ModelError(f"Expected string, got {type(value)}") + return value + + +def deserialize_bytes(value: JSONSerializable): + return _deserialize_byte_string(value) + + +def deserialize_Bool(value: JSONSerializable): + if not isinstance(value, bool): + raise ModelError(f"Expected bool, got {type(value)}") + return value + + +def deserialize_true(value: JSONSerializable): + if value is not True: + raise ModelError(f"Expected true, got {value}") + return True + + +def deserialize_secureString(value: JSONSerializable): + return deserialize_string(value) + + +def deserialize_secureBytes(value: JSONSerializable): + return _deserialize_byte_string(value) + + +def deserialize_object[T]( + cls: Callable[[JSONSerializable], T], +) -> Callable[[JSONSerializable | None], T | None]: + def deserializer(value: JSONSerializable | None) -> T | None: + if value is None: + return None + return cls(value) + + return deserializer + + +def deserialize_list[T]( + value: JSONSerializable, item_deserializer: Callable[[JSONSerializable], T] +) -> list[T]: + if not isinstance(value, list): + raise ModelError(f"Expected list, got {type(value)}") + return [item_deserializer(item) for item in value] diff --git a/test/tontester/src/tontester/__init__.py b/test/tontester/src/tontester/__init__.py new file mode 100644 index 0000000000..eb93153abd --- /dev/null +++ b/test/tontester/src/tontester/__init__.py @@ -0,0 +1,3 @@ +from . import install, key, network, tl, zerostate + +__all__ = ["install", "key", "network", "tl", "zerostate"] diff --git a/test/tontester/src/tontester/install.py b/test/tontester/src/tontester/install.py new file mode 100644 index 0000000000..03dc2a97e0 --- /dev/null +++ b/test/tontester/src/tontester/install.py @@ -0,0 +1,60 @@ +import os +import subprocess +from pathlib import Path +from typing import final + + +@final +class Install: + def __init__(self, build_dir: Path, source_dir: Path): + self._build_dir = build_dir.absolute() + self._source_dir = source_dir.absolute() + + @property + def build_dir(self): + return self._build_dir + + @property + def source_dir(self): + return self._source_dir + + @property + def fift_exe(self): + return self.build_dir / "crypto/create-state" + + @property + def fift_include_dirs(self): + return [ + self.source_dir / "crypto/fift/lib", + self.source_dir / "crypto/smartcont", + ] + + @property + def key_helper_exe(self): + return self.build_dir / "utils/generate-random-id" + + @property + def validator_engine_exe(self): + return self.build_dir / "validator-engine/validator-engine" + + @property + def dht_server_exe(self): + return self.build_dir / "dht-server/dht-server" + + @property + def tonlibjson(self): + return self.build_dir / "tonlib/libtonlibjson.so" + + +def run_fift(install: Install, code: str, working_dir: Path): + script_file = working_dir / "script.fif" + _ = script_file.write_text(code) + + args = [install.fift_exe] + for include_dir in install.fift_include_dirs: + args += ["-I", include_dir] + args += ["-s", "script.fif"] + + _ = subprocess.run(args, cwd=working_dir, check=True) + + os.remove(script_file) diff --git a/test/tontester/src/tontester/key.py b/test/tontester/src/tontester/key.py new file mode 100644 index 0000000000..0a289cea6a --- /dev/null +++ b/test/tontester/src/tontester/key.py @@ -0,0 +1,39 @@ +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from .install import Install +from .tl import ton_api + + +@dataclass +class Key: + private_key: ton_api.Pk_ed25519 + public_key: ton_api.Pub_ed25519 + short_key: ton_api.Adnl_id_short + + @staticmethod + def new(install: Install) -> "Key": + result = subprocess.run( + (install.key_helper_exe, "-m", "id"), + check=True, + stdout=subprocess.PIPE, + ).stdout + + parts = result.splitlines() + if len(parts) != 3: + raise ValueError("Invalid key generation output") + + return Key( + private_key=ton_api.Pk_ed25519.from_json(parts[0].decode()), + public_key=ton_api.Pub_ed25519.from_json(parts[1].decode()), + short_key=ton_api.Adnl_id_short.from_json(parts[2].decode()), + ) + + def add_to_keyring(self, keyring: Path) -> Path: + file = keyring / self.short_key.id.hex().upper() + _ = file.write_bytes(b"\x17\x23\x68\x49" + self.private_key.key) + return file + + def id(self): + return self.short_key.id diff --git a/test/tontester/src/tontester/log_streamer.py b/test/tontester/src/tontester/log_streamer.py new file mode 100644 index 0000000000..34ced526cd --- /dev/null +++ b/test/tontester/src/tontester/log_streamer.py @@ -0,0 +1,166 @@ +import asyncio +import io +import logging +import re +import sys +from dataclasses import dataclass +from typing import final + +_IS_TERMINAL_INTERACTIVE = sys.stderr.isatty() + + +@dataclass +class _LogEntry: + level: int + thread_id: int + timestamp: str + filename: str + line_number: int + label: str | None + message: bytearray + + def format(self): + COLORS = { + 0: "\x1b[1;31m", + 1: "\x1b[1;31m", + 2: "\x1b[1;33m", + 3: "\x1b[1;36m", + } + message = self.message.decode(errors="replace") + label = f"[{self.label}]" if self.label is not None else "" + + if message.endswith("\x1b[0m\n"): + slice_len = 5 + elif message.endswith("\n"): + slice_len = 1 + else: + slice_len = 0 + + line = f"[{self.level}][t {self.thread_id}][{self.filename}:{self.line_number}]{label} {message[:-slice_len]}" + + if _IS_TERMINAL_INTERACTIVE and self.level in COLORS: + line = f"{COLORS[self.level]}{line}\x1b[0m" + return line + + +@final +class LogStreamer: + def __init__(self, file: io.BufferedWriter, name: str, stream: asyncio.StreamReader): + self._file = file + self._name = name + self._stream = stream + self._task = asyncio.create_task(self._stream_log()) + self._logger = logging.getLogger(self._name) + self._current_entry: _LogEntry | None = None + + async def aclose(self): + await self._task + + async def _stream_log(self): + leftover = bytearray() + + while not self._stream.at_eof(): + chunk = await self._stream.read(131072) + _ = self._file.write(chunk) + + if leftover and not chunk: + self._process_line(bytes(leftover)) + break + + i = 0 + while True: + j = chunk.find(b"\n", i) + if j < 0: + break + if leftover: + line = leftover + chunk[i : j + 1] + self._process_line(bytes(line)) + leftover.clear() + else: + self._process_line(chunk[i : j + 1]) + i = j + 1 + + leftover += chunk[i:] + + self._flush_entry() + + _log_pattern = re.compile( + rb""" + (?: + \x1b\[\d+;\d+m + )? + \[ + \ *(\d+) + \] + \[ + t\ +(\d+) + \] + \[ + (\d{4}-\d{2}-\d{2}\ \d{2}:\d{2}:\d{2}\.\d{9}) + \] + \[ + ([^\]]+):(\d+) + \] + (?: + \[ + ([^\]]+) + \] + )? + \t + (.*) + """, + re.VERBOSE | re.ASCII | re.DOTALL, + ) + + def _process_line(self, line: bytes): + if self._current_entry is None: + match = self._log_pattern.match(line) + if match is None: + self._log_malformed(line) + return + + can_be_multiline = line.startswith(b"\x1b") + if len(match.groups()) == 7: + level_str, thread_id_str, timestamp, filename, line_number_str, label, message = ( + match.groups() + ) + label = label.decode() + else: + level_str, thread_id_str, timestamp, filename, line_number_str, message = ( + match.groups() + ) + label = None + + try: + level = int(level_str) + thread_id = int(thread_id_str) + line_number = int(line_number_str) + + self._current_entry = _LogEntry( + level=level, + thread_id=thread_id, + timestamp=timestamp.decode(), + filename=filename.decode(), + line_number=line_number, + label=label, + message=bytearray(message), + ) + except Exception: + self._log_malformed(line) + return + else: + can_be_multiline = True + self._current_entry.message.extend(line) + + if not can_be_multiline or self._current_entry.message.endswith(b"\x1b[0m\n"): + self._flush_entry() + + def _flush_entry(self): + if self._current_entry is not None: + self._logger.info(self._current_entry.format()) + self._current_entry = None + + def _log_malformed(self, data: bytes): + if data.endswith(b"\n"): + data = data[:-1] + self._logger.info(data.decode(errors="replace")) diff --git a/test/tontester/src/tontester/network.py b/test/tontester/src/tontester/network.py new file mode 100644 index 0000000000..90f1a6fa13 --- /dev/null +++ b/test/tontester/src/tontester/network.py @@ -0,0 +1,450 @@ +import asyncio +import logging +import signal +import subprocess +import types +from abc import ABC, abstractmethod +from dataclasses import dataclass +from enum import IntEnum, auto +from ipaddress import IPv4Address +from pathlib import Path +from typing import cast, final, override + +from pytonlib import TonlibClient, TonlibError # pyright: ignore[reportMissingTypeStubs] + +from tl import JSONSerializable, TLObject + +from .install import Install +from .key import Key +from .log_streamer import LogStreamer +from .tl import ton_api, tonlib_api +from .zerostate import NetworkConfig, Zerostate, create_zerostate + +l = logging.getLogger(__name__) + + +@dataclass +class _IPv4AddressAndPort: + ip: IPv4Address + port: int + + +class _Status(IntEnum): + INITED = auto() + ZEROSTATE_GENERATED = auto() + CLOSED = auto() + + +def _write_model(file: Path, model: TLObject): + _ = file.write_text(model.to_json()) + + +@final +class Network: + class Node(ABC): + def __init__(self, network: "Network", name: str): + self._network: Network = network + self.name: str = name + + self._directory: Path = self._network._directory / ( + "node" + str(self._network._node_idx) + ) + self._network._node_idx += 1 + + self._keyring: Path = self._directory / "keyring" + self._keyring.mkdir(parents=True) + + self._static_nodes: list["DHTNode"] = [] + + self.__process: asyncio.subprocess.Process | None = None + self.__process_watcher: asyncio.Task[None] | None = None + self.__log_streamer: LogStreamer | None = None + + @property + def _install(self): + return self._network._install + + def _new_network_address(self) -> _IPv4AddressAndPort: + self._network._port += 1 + return _IPv4AddressAndPort( + IPv4Address("127.0.42.239"), + self._network._port, + ) + + def _new_key(self) -> tuple[Key, Path]: + key = Key.new(self._install) + pk_file = key.add_to_keyring(self._keyring) + return (key, pk_file) + + def _ensure_no_zerostate_yet(self): + assert self._network._status < _Status.ZEROSTATE_GENERATED + + def _get_or_generate_zerostate(self): + return self._network._get_or_generate_zerostate() + + async def _run( + self, + executable: Path, + local_config: ton_api.Engine_validator_config, + validator_config: ton_api.Validator_config_global | None, + additional_args: list[str], + ): + async def process_watcher(): + assert self.__process is not None + return_code = await self.__process.wait() + if return_code < 0: + signal_name = signal.Signals(-return_code).name + l.info(f"Node '{self.name}' terminated by signal {signal_name}") + else: + l.info(f"Node '{self.name}' exited with code {return_code}") + + assert self._network._status < _Status.CLOSED + + global_config_file = self._directory / "config.global.json" + _write_model( + global_config_file, + ton_api.Config_global( + dht=ton_api.Dht_config_global( + static_nodes=ton_api.Dht_nodes( + nodes=[node.signed_address for node in self._static_nodes] + ), + k=3, + a=3, + ), + validator=validator_config, + ), + ) + + local_config_file = self._directory / "config.json" + _write_model(local_config_file, local_config) + + log_path = self._directory / "log" + l.info(f"Running {self.name} and saving its raw log to {log_path}") + + self.__process = await asyncio.create_subprocess_exec( + executable, + "--global-config", + global_config_file, + "--local-config", + local_config_file, + "--db", + ".", + *additional_args, + cwd=self._directory, + stderr=asyncio.subprocess.PIPE, + ) + assert self.__process.stderr is not None # to placate pyright + self.__process_watcher = asyncio.create_task(process_watcher()) + + self.__log_streamer = LogStreamer( + open(log_path, "wb"), + self.name, + self.__process.stderr, + ) + + def announce_to(self, dht: "DHTNode"): + self._static_nodes.append(dht) + + @abstractmethod + async def run(self): + pass + + async def stop(self): + if self.__process: + # No exception can occur between self.__process and self._log_streamer creation + assert self.__log_streamer is not None + assert self.__process_watcher is not None + + if not self.__process_watcher.done(): + l.info(f"Killing node '{self.name}'") + try: + self.__process.terminate() + except ProcessLookupError: + # Terminate might still fail if Python internally has already finished + # waiting for the child process but didn't yet resume the watcher. + pass + + await self.__process_watcher + await self.__log_streamer.aclose() + + def __init__(self, install: Install, directory: Path): + self._install = install + self._directory = directory.absolute() + self._port = 2000 + self._node_idx = 0 + self._status = _Status.INITED + + self.__nodes: list[Network.Node] = [] + self.__full_nodes: list[FullNode] = [] + self.__network_config: NetworkConfig = NetworkConfig() + self.__zerostate: Zerostate | None = None + + @property + def config(self): + assert self._status < _Status.ZEROSTATE_GENERATED + return self.__network_config + + def create_dht_node(self) -> "DHTNode": + assert self._status < _Status.CLOSED + + node = DHTNode(self, f"dht-{len(self.__nodes)}") + self.__nodes.append(node) + return node + + def create_full_node(self) -> "FullNode": + assert self._status < _Status.CLOSED + + node = FullNode(self, f"node-{len(self.__nodes)}") + self.__nodes.append(node) + self.__full_nodes.append(node) + return node + + def _get_or_generate_zerostate(self) -> Zerostate: + if self.__zerostate is not None: + return self.__zerostate + + assert self._status == _Status.INITED + + state_dir = self._directory / "state" + state_dir.mkdir() + + self.__zerostate = create_zerostate( + self._install, + state_dir, + self.__network_config, + [node.validator_key for node in self.__full_nodes if node.is_initial_validator], + ) + self._status = _Status.ZEROSTATE_GENERATED + return self.__zerostate + + async def aclose(self): + assert self._status < _Status.CLOSED + self._status = _Status.CLOSED + + for node in self.__nodes: + await node.stop() + + async def __aenter__(self): + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: types.TracebackType | None, + ) -> bool | None: + await asyncio.shield(self.aclose()) + + async def wait_mc_block(self, seqno: int): + client = await self.__full_nodes[0].tonlib_client() + + while True: + try: + raw_result = cast(JSONSerializable, await client.get_masterchain_info()) # pyright: ignore[reportUnknownMemberType] + except TonlibError as e: + # FIXME: We should really let node notify us that it is ready. + try: + if ( + e.result["code"] == 500 + and ( + e.result["message"] + == "LITE_SERVER_NETWORKtimeout for adnl query query" # node is not synced yet + or e.result["message"] + == "LITE_SERVER_NETWORK" # node is not listening the socket + ) + ): + await asyncio.sleep(0.2) + continue + except Exception: + pass + raise + + mc_info = tonlib_api.Blocks_masterchainInfo.from_dict(raw_result) + assert mc_info.last is not None + + if mc_info.last.seqno >= seqno: + break + else: + await asyncio.sleep(0.2) + + +def _ip_to_tl(ip: IPv4Address) -> int: + result = int(ip) + if result >= 2**31: + result -= 2**32 + return result + + +@final +class DHTNode(Network.Node): + def __init__(self, network: "Network", name: str): + super().__init__(network, name) + + self._addr = self._new_network_address() + + key, pk_file = self._new_key() + + address_list_to_sign = ton_api.Adnl_addressList( + addrs=[ + ton_api.Adnl_address_udp(ip=_ip_to_tl(self._addr.ip), port=self._addr.port), + ] + ) + signed_address = subprocess.run( + ( + self._install.key_helper_exe, + "-m", + "dht", + "-k", + pk_file, + "-a", + address_list_to_sign.to_json(), + ), + check=True, + stdout=subprocess.PIPE, + ).stdout + self._signed_address = ton_api.Dht_node.from_json(signed_address.decode()) + + self._local_config = ton_api.Engine_validator_config( + addrs=[ + ton_api.Engine_addr( + ip=_ip_to_tl(self._addr.ip), + port=self._addr.port, + categories=[0], + ) + ], + adnl=[ton_api.Engine_adnl(id=key.id(), category=0)], + dht=[ton_api.Engine_dht(id=key.id())], + ) + + @property + def signed_address(self): + return self._signed_address + + @override + async def run(self): + await self._run(self._install.dht_server_exe, self._local_config, None, []) + + +@final +class FullNode(Network.Node): + def __init__(self, network: "Network", name: str): + super().__init__(network, name) + + KEY_EXPIRATION = (1 << 31) - 1 + + self._addr = self._new_network_address() + self._liteserver_addr = self._new_network_address() + + self._fullnode_key, _ = self._new_key() + self._validator_key, _ = self._new_key() + self._liteserver_key, _ = self._new_key() + + self._local_config = ton_api.Engine_validator_config( + addrs=[ + ton_api.Engine_addr( + ip=_ip_to_tl(self._addr.ip), + port=self._addr.port, + categories=[0], + ) + ], + adnl=[ + ton_api.Engine_adnl(id=self._fullnode_key.id(), category=0), + ton_api.Engine_adnl(id=self._validator_key.id(), category=0), + ], + dht=[ + ton_api.Engine_dht(id=self._fullnode_key.id()), + ], + validators=[ + ton_api.Engine_validator( + id=self._validator_key.id(), + temp_keys=[ + ton_api.Engine_validatorTempKey( + key=self._validator_key.id(), + expire_at=KEY_EXPIRATION, + ) + ], + adnl_addrs=[ + ton_api.Engine_validatorAdnlAddress( + id=self._validator_key.id(), + expire_at=KEY_EXPIRATION, + ) + ], + expire_at=KEY_EXPIRATION, + ) + ], + fullnode=self._fullnode_key.id(), + liteservers=[ + ton_api.Engine_liteServer( + id=self._liteserver_key.id(), + # FIXME: IP? + port=self._liteserver_addr.port, + ) + ], + ) + + self._is_initial_validator = False + + self._client: TonlibClient | None = None + + def make_initial_validator(self): + self._ensure_no_zerostate_yet() + self._is_initial_validator = True + + @property + def is_initial_validator(self): + return self._is_initial_validator + + @property + def validator_key(self): + return self._validator_key + + @override + async def run(self): + zerostate = self._get_or_generate_zerostate() + + static_dir = self._directory / "static" + static_dir.mkdir() + for state in (zerostate.masterchain, zerostate.shardchain): + (static_dir / state.file_hash.hex().upper()).symlink_to(state.file) + + await self._run( + self._install.validator_engine_exe, + self._local_config, + zerostate.as_validator_config(), + ["--initial-sync-delay", "5"], + ) + + async def tonlib_client(self) -> TonlibClient: + if self._client: + return self._client + + config = ton_api.Liteclient_config_global( + liteservers=[ + ton_api.Liteserver_desc( + id=self._liteserver_key.public_key, + ip=_ip_to_tl(self._liteserver_addr.ip), + port=self._liteserver_addr.port, + ), + ], + validator=self._get_or_generate_zerostate().as_validator_config(), + ) + + keystore_dir = self._directory / "lc-keystore" + keystore_dir.mkdir() + + self._client = TonlibClient( + ls_index=0, + config=config.to_dict(), + keystore=str(keystore_dir), + cdll_path=str(self._install.tonlibjson), + verbosity_level=3, + ) + await self._client.init() + + return self._client + + @override + async def stop(self): + if self._client: + await self._client.close() + await super().stop() diff --git a/test/tontester/src/tontester/py.typed b/test/tontester/src/tontester/py.typed new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/tontester/src/tontester/zerostate.py b/test/tontester/src/tontester/zerostate.py new file mode 100644 index 0000000000..761adc61a2 --- /dev/null +++ b/test/tontester/src/tontester/zerostate.py @@ -0,0 +1,328 @@ +from dataclasses import dataclass +from pathlib import Path + +from .install import Install, run_fift +from .key import Key +from .tl import ton_api + + +def _shard_json_repr(shard: int): + if shard >= (1 << 63): + return shard - (1 << 64) + return shard + + +@dataclass +class NetworkConfig: + monitor_min_split: int = 0 + split: int = 0 + global_version: int = 11 + shard_validators: int = 1 + block_limit_mul: int = 1 + mc_catchain_lifetime: int = 250 + shard_catchain_lifetime: int = 250 + + +@dataclass +class WorkchainState: + file: Path + file_hash: bytes + root_hash: bytes + + +@dataclass +class Zerostate: + masterchain: WorkchainState + shardchain: WorkchainState + + def as_block(self): + return ton_api.TonNode_blockIdExt( + workchain=-1, + shard=_shard_json_repr(0x8000_0000_0000_0000), + seqno=0, + root_hash=self.masterchain.root_hash, + file_hash=self.masterchain.file_hash, + ) + + def as_validator_config(self): + return ton_api.Validator_config_global(zero_state=self.as_block()) + + +_TEMPLATE = """ +"TonUtil.fif" include +"Asm.fif" include +"Lists.fif" include +"FiftExt.fif" include + +256 1<<1- 15 / constant AllOnes + +wc_master setworkchain +-777 setglobalid // negative value means a test instance of the blockchain + +// Initial state of Workchain 0 (Basic workchain) + +0 mkemptyShardState + +{{ + dup isWorkchainDescr? not abort"invalid WorkchainDescr created" + B dup "basestate0.boc" B>file +Bhashu dup =: basestate0_fhash 256 u>B "basestate0.fhash" B>file +hashu dup =: basestate0_rhash 256 u>B "basestate0.rhash" B>file +basestate0_rhash basestate0_fhash now {monitor_min_split} {split} dup 0 add-std-workchain-v2 +config.workchains! + +// SmartContract #1 (Simple wallet) + +<{{ SETCP0 DUP IFNOTRET // return if recv_internal + DUP 85143 INT EQUAL IFJMP:<{{ // "seqno" get-method + DROP c4 PUSHCTR CTOS 32 PLDU // cnt + }}> + INC 32 THROWIF // fail unless recv_external + 512 INT LDSLICEX DUP 32 PLDU // sign cs cnt + c4 PUSHCTR CTOS 32 LDU 256 LDU ENDS // sign cs cnt cnt' pubk + s1 s2 XCPU // sign cs cnt pubk cnt' cnt + EQUAL 33 THROWIFNOT // ( seqno mismatch? ) + s2 PUSH HASHSU // sign cs cnt pubk hash + s0 s4 s4 XC2PU // pubk cs cnt hash sign pubk + CHKSIGNU // pubk cs cnt ? + 34 THROWIFNOT // signature mismatch + ACCEPT + SWAP 32 LDU NIP 8 LDU LDREF ENDS // pubk cnt mode msg + SWAP SENDRAWMSG // pubk cnt ; ( message sent ) + INC NEWC 32 STU 256 STU ENDC c4 POPCTR +}}>c +// code + // data +Libs{{ + x{{ABACABADABACABA}} s>c public_lib + x{{1234}} x{{5678}} |_ s>c private_lib +}}Libs // libraries +GR$4999990000 // balance +0 // split_depth +0 // ticktock +AllOnes 0 * // address +6 // mode: create+setaddr +register_smc +dup make_special dup constant smc1_addr +Masterchain over +2dup ."wallet address = " .addr cr 2dup 6 .Addr cr +"main-wallet.addr" save-address-verbose + +// SmartContract #3 +PROGRAM{{ + recv_internal x{{}} PROC + run_ticktock PROC:<{{ + c4 PUSHCTR CTOS 32 LDU 256 LDU ENDS + NEWC ROT INC 32 STUR OVER 256 STUR ENDC + c4 POPCTR + // first 32 bits of persistent data have been increased + // remaining 256 bits with an address have been fetched + // create new empty message with 0.1 Grams to that address + NEWC b{{00100010011111111}} STSLICECONST TUCK 256 STU + 100000000 INT STGRAMS // store 0.1 Grams + 1 4 + 4 + 64 + 32 + 1+ 1+ INT STZEROES ENDC + // send raw message from Cell + ZERO SENDRAWMSG + -17 INT 256 STIR 130000000 INT STGRAMS + 107 INT STZEROES ENDC + ZERO // another message with 0.13 Grams to account -17 + NEWC b{{11000100100000}} "test" $>s |+ STSLICECONST + 123456789 INT STGRAMS + 107 INT STZEROES "Hello, world!" $>s STSLICECONST ENDC + ZERO SENDRAWMSG SENDRAWMSG // external message to address "test" + }}> +}}END>c +// code + // data +// empty_cell // libraries +Libs{{ + x{{ABACABADABACABA}} s>c public_lib + x{{1234}} x{{5678}} |_ s>c public_lib +}}Libs // libraries +GR$1 // balance +0 // split_depth +3 // ticktock: tick +2 // mode: create +register_smc +dup make_special dup constant smc3_addr +."address = " 64x. cr + + +/* + * + * SmartContract #4 (elector) + * + */ +"auto/elector-code.fif" include // code in separate source file + // data: dict dict dict grams uint32 uint256 +empty_cell // libraries +GR$10 // balance: 10 grams +0 // split_depth +2 // ticktock: tick +AllOnes 3 * // address: -1:333...333 +6 // mode: create + setaddr +register_smc +dup make_special dup constant smc4_addr dup constant elector_addr +Masterchain swap +."elector smart contract address = " 2dup .addr cr 2dup 7 .Addr cr +"elector" +".addr" save-address-verbose + +/* + * + * Configuration Parameters + * + */ +// version capabilities +{global_version} capCreateStats capBounceMsgBody or capReportVersion or capShortDequeue or 64 or 128 or config.version! +// max-validators max-main-validators min-validators +1000 1000 1000 config.validator_num! +// min-stake max-stake min-total-stake max-factor +GR$10000 GR$100000 GR$10000 sg~10 config.validator_stake_limits! +// elected-for elect-start-before elect-end-before stakes-frozen-for +// 400000 200000 4000 400000 config.election_params! +// 200 190 10 10 config.election_params! // DEBUG +2400 800 60 300 config.election_params! +// config-addr = -1:5555...5555 +AllOnes 5 * constant config_addr +config_addr config.config_smc! +// elector-addr +elector_addr config.elector_smc! + +// 1 sg* 100 sg* 1000 sg* 1000000 sg* config.storage_prices! // old values (too high) +1 500 1000 500000 config.storage_prices! +config.special! + +// gas_price gas_limit special_gas_limit gas_credit block_gas_limit freeze_due_limit delete_due_limit flat_gas_limit flat_gas_price -- +10 sg* 1 *M dup 10000 1000 *M GR$0.1 GR$1.0 100 1000 config.gas_prices! +10 sg* 1 *M 20 *M 10000 1000 *M GR$0.1 GR$1.0 100 1000 config.mc_gas_prices! +// lump_price bit_price cell_price ihr_factor first_frac next_frac +100 10 sg* 10 sg* 3/2 sg*/ 1/3 sg*/ 1/3 sg*/ config.fwd_prices! +100 10 sg* 10 sg* 3/2 sg*/ 1/3 sg*/ 1/3 sg*/ config.mc_fwd_prices! +// mc-cc-lifetime sh-cc-lifetime sh-val-lifetime sh-val-num mc-shuffle +{mc_catchain_lifetime} {shard_catchain_lifetime} 1000 {shard_val} true config.catchain_params! + +// round-candidates next-cand-delay-ms consensus-timeout-ms fast-attempts attempt-duration cc-max-deps max-block-size max-collated-size new-cc-ids +// proto-version catchain-max-blocks-coeff + +29 config! +// 3 2000 16000 3 8 4 960 *Mi 960 *Mi true config.consensus_params! + +{{ +}} : make-block-limits-v2 + +128 *Ki 512 *Ki {block_limit_mul} * 1 *Mi {block_limit_mul} * triple // [ underload soft hard ] : block bytes limit +2000000 100000000 100000000 triple // gas limits +1000 500000 1000000 triple // lt limits +triple dup +// untriple make-block-limits-v2 22 config! +// untriple make-block-limits-v2 23 config! +untriple make-block-limits 22 config! +untriple make-block-limits 23 config! + +GR$1.7 GR$1 config.block_create_fees! +// smc1_addr config.collector_smc! +smc1_addr config.minter_smc! + +1000000000000 -17 of-cc 666666666666 239 of-cc cc+ config.to_mint! + +( 0 1 9 10 12 14 15 16 17 18 20 21 22 23 24 25 28 34 ) config.mandatory_params! +( -999 -1000 -1001 0 1 9 10 12 14 15 16 17 32 34 36 ) config.critical_params! + +// [ min_tot_rounds max_tot_rounds min_wins max_losses min_store_sec max_store_sec bit_pps cell_pps ] +// first for ordinary proposals, then for critical proposals +_( 2 3 2 2 1000000 10000000 1 500 ) +_( 4 7 4 2 5000000 20000000 2 1000 ) +config.param_proposals_setup! + +// deposit bit_pps cell_pps +GR$100 1 500 config.complaint_prices! + +{validators} +now dup 3600 + {mc_validators} config.validators! + +{{ + =: data + s ]] 0 runvmx abort"dict-insert failed" }} : dict-insert + +/* + * + * SmartContract #5 (Configuration smart contract) + * + */ +"auto/config-code.fif" include // code in separate source file + // data +empty_cell // libraries +GR$10 // balance +0 1 config_addr 6 register_smc // tock +dup set_config_smc +Masterchain swap +."config smart contract address = " 2dup .addr cr 2dup 7 .Addr cr +"config-master" +".addr" save-address-verbose +// Other data + +/* + * + * Create state + * + */ + +create_state +dup 31 boc+>B dup "zerostate.boc" B>file +Bhashu dup =: zerostate_fhash 256 u>B "zerostate.fhash" B>file +hashu dup =: zerostate_rhash 256 u>B "zerostate.rhash" B>file +""" + + +def create_zerostate( + install: Install, state_dir: Path, config: NetworkConfig, validator_keys: list[Key] +) -> Zerostate: + keys: list[str] = [] + for key in validator_keys: + keys.append(f"B{{{key.public_key.key.hex()}}} 17 add-validator") + + run_fift( + install, + _TEMPLATE.format( + monitor_min_split=config.monitor_min_split, + split=config.split, + global_version=config.global_version, + shard_val=config.shard_validators, + block_limit_mul=config.block_limit_mul, + validators="\n".join(keys), + mc_validators=len(keys), + mc_catchain_lifetime=config.mc_catchain_lifetime, + shard_catchain_lifetime=config.shard_catchain_lifetime, + ), + state_dir, + ) + + return Zerostate( + masterchain=WorkchainState( + file=state_dir / "zerostate.boc", + file_hash=(state_dir / "zerostate.fhash").read_bytes(), + root_hash=(state_dir / "zerostate.rhash").read_bytes(), + ), + shardchain=WorkchainState( + file=state_dir / "basestate0.boc", + file_hash=(state_dir / "basestate0.fhash").read_bytes(), + root_hash=(state_dir / "basestate0.rhash").read_bytes(), + ), + ) diff --git a/test/tontester/tests/tl/test_codegen.py b/test/tontester/tests/tl/test_codegen.py new file mode 100644 index 0000000000..c2a68537eb --- /dev/null +++ b/test/tontester/tests/tl/test_codegen.py @@ -0,0 +1,506 @@ +import base64 + +from generated.test_schema import ( + AllPrimitives, + NestedObject, + ObjectQueryRequest, + OptionalFields, + QueryRequest, + SimpleObject, + TrueField, + VariantA, + VariantB, + VariantC, + VectorObjects, + VectorPrimitives, +) + + +def test_all_primitives_serialize(): + obj = AllPrimitives( + field_int=42, + field_long=9223372036854775807, + field_int32=-2147483648, + field_int53=4503599627370495, + field_int64=-9223372036854775808, + field_int128=b"\x01" * 16, + field_int256=b"\x02" * 32, + field_double=3.14159, + field_string="test string", + field_bytes=b"test bytes", + field_Bool=True, + field_secureString="secure", + field_secureBytes=b"secure bytes", + ) + + expected = { + "@type": "allPrimitives", + "field_int": 42, + "field_long": 9223372036854775807, + "field_int32": -2147483648, + "field_int53": 4503599627370495, + "field_int64": -9223372036854775808, + "field_int128": base64.b64encode(b"\x01" * 16).decode(), + "field_int256": base64.b64encode(b"\x02" * 32).decode(), + "field_double": 3.14159, + "field_string": "test string", + "field_bytes": base64.b64encode(b"test bytes").decode(), + "field_Bool": True, + "field_secureString": "secure", + "field_secureBytes": base64.b64encode(b"secure bytes").decode(), + } + + assert obj.to_dict() == expected + + +def test_all_primitives_deserialize(): + d = { + "@type": "allPrimitives", + "field_int": 42, + "field_long": 9223372036854775807, + "field_int32": -2147483648, + "field_int53": 4503599627370495, + "field_int64": -9223372036854775808, + "field_int128": base64.b64encode(b"\x01" * 16).decode(), + "field_int256": base64.b64encode(b"\x02" * 32).decode(), + "field_double": 3.14159, + "field_string": "test string", + "field_bytes": base64.b64encode(b"test bytes").decode(), + "field_Bool": True, + "field_secureString": "secure", + "field_secureBytes": base64.b64encode(b"secure bytes").decode(), + } + + obj = AllPrimitives.from_dict(d) + + assert obj.field_int == 42 + assert obj.field_long == 9223372036854775807 + assert obj.field_int32 == -2147483648 + assert obj.field_int53 == 4503599627370495 + assert obj.field_int64 == -9223372036854775808 + assert obj.field_int128 == b"\x01" * 16 + assert obj.field_int256 == b"\x02" * 32 + assert obj.field_double == 3.14159 + assert obj.field_string == "test string" + assert obj.field_bytes == b"test bytes" + assert obj.field_Bool is True + assert obj.field_secureString == "secure" + assert obj.field_secureBytes == b"secure bytes" + + +def test_all_primitives_roundtrip(): + obj = AllPrimitives( + field_int=0, + field_long=0, + field_int32=0, + field_int53=0, + field_int64=0, + field_int128=b"\x00" * 16, + field_int256=b"\xff" * 32, + field_double=0.0, + field_string="", + field_bytes=b"", + field_Bool=False, + field_secureString="", + field_secureBytes=b"", + ) + + obj2 = AllPrimitives.from_dict(obj.to_dict()) + assert obj == obj2 + + +def test_all_primitives_defaults(): + d = {"@type": "allPrimitives"} + + obj = AllPrimitives.from_dict(d) + + assert obj.field_int == 0 + assert obj.field_long == 0 + assert obj.field_int32 == 0 + assert obj.field_int53 == 0 + assert obj.field_int64 == 0 + assert obj.field_int128 == b"\x00" * 16 + assert obj.field_int256 == b"\x00" * 32 + assert obj.field_double == 0.0 + assert obj.field_string == "" + assert obj.field_bytes == b"" + assert obj.field_Bool is False + assert obj.field_secureString == "" + assert obj.field_secureBytes == b"" + + +def test_vector_primitives_serialize(): + obj = VectorPrimitives( + vec_int=[1, 2, 3], + vec_long=[100, 200, 300], + vec_int32=[-1, -2, -3], + vec_int53=[1000, 2000, 3000], + vec_int64=[10000, 20000, 30000], + vec_int128=[b"\x01" * 16, b"\x02" * 16], + vec_int256=[b"\x03" * 32, b"\x04" * 32], + vec_double=[1.1, 2.2, 3.3], + vec_string=["a", "b", "c"], + vec_bytes=[b"x", b"y", b"z"], + vec_Bool=[True, False, True], + vec_secureString=["s1", "s2"], + vec_secureBytes=[b"sb1", b"sb2"], + ) + + expected = { + "@type": "vectorPrimitives", + "vec_int": [1, 2, 3], + "vec_long": [100, 200, 300], + "vec_int32": [-1, -2, -3], + "vec_int53": [1000, 2000, 3000], + "vec_int64": [10000, 20000, 30000], + "vec_int128": [ + base64.b64encode(b"\x01" * 16).decode(), + base64.b64encode(b"\x02" * 16).decode(), + ], + "vec_int256": [ + base64.b64encode(b"\x03" * 32).decode(), + base64.b64encode(b"\x04" * 32).decode(), + ], + "vec_double": [1.1, 2.2, 3.3], + "vec_string": ["a", "b", "c"], + "vec_bytes": [ + base64.b64encode(b"x").decode(), + base64.b64encode(b"y").decode(), + base64.b64encode(b"z").decode(), + ], + "vec_Bool": [True, False, True], + "vec_secureString": ["s1", "s2"], + "vec_secureBytes": [base64.b64encode(b"sb1").decode(), base64.b64encode(b"sb2").decode()], + } + + assert obj.to_dict() == expected + + +def test_vector_primitives_deserialize(): + d = { + "@type": "vectorPrimitives", + "vec_int": [1, 2, 3], + "vec_long": [100, 200, 300], + "vec_int32": [-1, -2, -3], + "vec_int53": [1000, 2000, 3000], + "vec_int64": [10000, 20000, 30000], + "vec_int128": [ + base64.b64encode(b"\x01" * 16).decode(), + base64.b64encode(b"\x02" * 16).decode(), + ], + "vec_int256": [ + base64.b64encode(b"\x03" * 32).decode(), + base64.b64encode(b"\x04" * 32).decode(), + ], + "vec_double": [1.1, 2.2, 3.3], + "vec_string": ["a", "b", "c"], + "vec_bytes": [ + base64.b64encode(b"x").decode(), + base64.b64encode(b"y").decode(), + base64.b64encode(b"z").decode(), + ], + "vec_Bool": [True, False, True], + "vec_secureString": ["s1", "s2"], + "vec_secureBytes": [base64.b64encode(b"sb1").decode(), base64.b64encode(b"sb2").decode()], + } + + obj = VectorPrimitives.from_dict(d) + + assert obj.vec_int == [1, 2, 3] + assert obj.vec_long == [100, 200, 300] + assert obj.vec_int32 == [-1, -2, -3] + assert obj.vec_int53 == [1000, 2000, 3000] + assert obj.vec_int64 == [10000, 20000, 30000] + assert obj.vec_int128 == [b"\x01" * 16, b"\x02" * 16] + assert obj.vec_int256 == [b"\x03" * 32, b"\x04" * 32] + assert obj.vec_double == [1.1, 2.2, 3.3] + assert obj.vec_string == ["a", "b", "c"] + assert obj.vec_bytes == [b"x", b"y", b"z"] + assert obj.vec_Bool == [True, False, True] + assert obj.vec_secureString == ["s1", "s2"] + assert obj.vec_secureBytes == [b"sb1", b"sb2"] + + +def test_vector_primitives_empty(): + obj = VectorPrimitives( + vec_int=[], + vec_long=[], + vec_int32=[], + vec_int53=[], + vec_int64=[], + vec_int128=[], + vec_int256=[], + vec_double=[], + vec_string=[], + vec_bytes=[], + vec_Bool=[], + vec_secureString=[], + vec_secureBytes=[], + ) + + obj2 = VectorPrimitives.from_dict(obj.to_dict()) + assert obj == obj2 + + +def test_vector_primitives_defaults(): + d = {"@type": "vectorPrimitives"} + + obj = VectorPrimitives.from_dict(d) + + assert obj.vec_int == [] + assert obj.vec_long == [] + assert obj.vec_int32 == [] + assert obj.vec_int53 == [] + assert obj.vec_int64 == [] + assert obj.vec_int128 == [] + assert obj.vec_int256 == [] + assert obj.vec_double == [] + assert obj.vec_string == [] + assert obj.vec_bytes == [] + assert obj.vec_Bool == [] + assert obj.vec_secureString == [] + assert obj.vec_secureBytes == [] + + +def test_simple_object(): + obj = SimpleObject(id=123, name="test") + + expected = {"@type": "simpleObject", "id": 123, "name": "test"} + + assert obj.to_dict() == expected + + obj2 = SimpleObject.from_dict(expected) + assert obj == obj2 + + +def test_simple_object_defaults(): + d = {"@type": "simpleObject"} + + obj = SimpleObject.from_dict(d) + + assert obj.id == 0 + assert obj.name == "" + + +def test_variant_a(): + obj = VariantA(value=42) + + expected = {"@type": "variantA", "value": 42} + + assert obj.to_dict() == expected + + obj2 = VariantA.from_dict(expected) + assert obj == obj2 + + +def test_variant_b(): + obj = VariantB(text="hello") + + expected = {"@type": "variantB", "text": "hello"} + + assert obj.to_dict() == expected + + obj2 = VariantB.from_dict(expected) + assert obj == obj2 + + +def test_variant_c(): + obj = VariantC(flag=False) + + expected = {"@type": "variantC", "flag": False} + + assert obj.to_dict() == expected + + obj2 = VariantC.from_dict(expected) + assert obj == obj2 + + +def test_nested_object_with_variant_a(): + simple = SimpleObject(id=1, name="obj1") + variant = VariantA(value=100) + obj = NestedObject(simple=simple, variant=variant) + + expected = { + "@type": "nestedObject", + "simple": {"@type": "simpleObject", "id": 1, "name": "obj1"}, + "variant": {"@type": "variantA", "value": 100}, + } + + assert obj.to_dict() == expected + + obj2 = NestedObject.from_dict(expected) + assert obj == obj2 + + +def test_nested_object_with_variant_b(): + simple = SimpleObject(id=2, name="obj2") + variant = VariantB(text="variant b text") + obj = NestedObject(simple=simple, variant=variant) + + obj2 = NestedObject.from_dict(obj.to_dict()) + assert obj == obj2 + + +def test_nested_object_defaults(): + d = {"@type": "nestedObject"} + + obj = NestedObject.from_dict(d) + + assert obj.simple is None + assert obj.variant is None + + +def test_vector_objects(): + obj = VectorObjects( + simples=[ + SimpleObject(id=1, name="first"), + SimpleObject(id=2, name="second"), + ], + variants=[ + VariantA(value=10), + VariantB(text="text"), + VariantC(flag=True), + ], + ) + + expected = { + "@type": "vectorObjects", + "simples": [ + {"@type": "simpleObject", "id": 1, "name": "first"}, + {"@type": "simpleObject", "id": 2, "name": "second"}, + ], + "variants": [ + {"@type": "variantA", "value": 10}, + {"@type": "variantB", "text": "text"}, + {"@type": "variantC", "flag": True}, + ], + } + + assert obj.to_dict() == expected + + obj2 = VectorObjects.from_dict(expected) + assert obj == obj2 + + +def test_vector_objects_defaults(): + d = {"@type": "vectorObjects"} + + obj = VectorObjects.from_dict(d) + + assert obj.simples == [] + assert obj.variants == [] + + +def test_optional_fields_all_present(): + obj = OptionalFields( + flags=15, + opt_int=123, + opt_string="present", + opt_object=SimpleObject(id=1, name="obj"), + opt_vec=[10, 20, 30], + ) + + expected = { + "@type": "optionalFields", + "flags": 15, + "opt_int": 123, + "opt_string": "present", + "opt_object": {"@type": "simpleObject", "id": 1, "name": "obj"}, + "opt_vec": [10, 20, 30], + } + + assert obj.to_dict() == expected + + obj2 = OptionalFields.from_dict(expected) + assert obj == obj2 + + +def test_optional_fields_only_flags(): + obj = OptionalFields(flags=7, opt_int=0, opt_string="", opt_object=None, opt_vec=None) + + d = obj.to_dict() + obj2 = OptionalFields.from_dict(d) + assert obj == obj2 + + +def test_optional_fields_flags_mismatch(): + d = { + "@type": "optionalFields", + "flags": 0, + "opt_int": 999, + "opt_string": "present even though flag is 0", + } + + obj = OptionalFields.from_dict(d) + + assert obj.flags == 0 + assert obj.opt_int == 999 + assert obj.opt_string == "present even though flag is 0" + assert obj.opt_object is None + assert obj.opt_vec == [] + + +def test_optional_fields_defaults(): + d = {"@type": "optionalFields"} + + obj = OptionalFields.from_dict(d) + + assert obj.flags == 0 + assert obj.opt_int == 0 + assert obj.opt_string == "" + assert obj.opt_object is None + assert obj.opt_vec == [] + + +def test_true_field(): + obj = TrueField(has_feature=True, other=456) + + expected = {"@type": "trueField", "has_feature": True, "other": 456} + + assert obj.to_dict() == expected + + obj2 = TrueField.from_dict(expected) + assert obj == obj2 + + +def test_true_field_defaults(): + d = {"@type": "trueField"} + + obj = TrueField.from_dict(d) + + assert obj.has_feature is True + assert obj.other == 0 + + +def test_request(): + req = QueryRequest(id=789, data="request data") + + expected = {"@type": "query", "id": 789, "data": "request data"} + + assert req.to_dict() == expected + + req2 = QueryRequest.from_dict(expected) + assert req == req2 + + +def test_object_request(): + obj = SimpleObject(id=999, name="in request") + req = ObjectQueryRequest(obj=obj) + + expected = { + "@type": "objectQuery", + "obj": {"@type": "simpleObject", "id": 999, "name": "in request"}, + } + + assert req.to_dict() == expected + + req2 = ObjectQueryRequest.from_dict(expected) + assert req == req2 + + +def test_object_request_defaults(): + d = {"@type": "objectQuery"} + + req = ObjectQueryRequest.from_dict(d) + + assert req.obj is None diff --git a/test/tontester/tests/tl/test_schema.tl b/test/tontester/tests/tl/test_schema.tl new file mode 100644 index 0000000000..b1952c490c --- /dev/null +++ b/test/tontester/tests/tl/test_schema.tl @@ -0,0 +1,63 @@ +simpleObject id:int name:string = SimpleObject; + +variantA value:int = TestVariant; +variantB text:string = TestVariant; +variantC flag:Bool = TestVariant; + +allPrimitives + field_int:int + field_long:long + field_int32:int32 + field_int53:int53 + field_int64:int64 + field_int128:int128 + field_int256:int256 + field_double:double + field_string:string + field_bytes:bytes + field_Bool:Bool + field_secureString:secureString + field_secureBytes:secureBytes + = AllPrimitives; + +vectorPrimitives + vec_int:vector + vec_long:vector + vec_int32:vector + vec_int53:vector + vec_int64:vector + vec_int128:vector + vec_int256:vector + vec_double:vector + vec_string:vector + vec_bytes:vector + vec_Bool:vector + vec_secureString:vector + vec_secureBytes:vector + = VectorPrimitives; + +nestedObject + simple:SimpleObject + variant:TestVariant + = NestedObject; + +vectorObjects + simples:vector + variants:vector + = VectorObjects; + +optionalFields + flags:# + opt_int:flags.0?int + opt_string:flags.1?string + opt_object:flags.2?SimpleObject + opt_vec:flags.3?vector + = OptionalFields; + +trueField has_feature:true other:int = TrueField; + +---functions--- + +query id:int data:string = AllPrimitives; + +objectQuery obj:SimpleObject = NestedObject; diff --git a/test/tontester/tests/tontester/test_tl.py b/test/tontester/tests/tontester/test_tl.py new file mode 100644 index 0000000000..2a2c5ab02b --- /dev/null +++ b/test/tontester/tests/tontester/test_tl.py @@ -0,0 +1,190 @@ +import base64 +import json + +import pytest +from tontester.tl.ton_api import ( + Adnl_packetContents, + Catchain_config_global, + Engine_validator_config, + Id_config_local, + Pk_aes, + Pub_unenc, + TonNode_blockIdExt, +) + + +@pytest.fixture() +def pubk(): + return Pub_unenc(data=b"\x12" * 32) + + +@pytest.fixture() +def block(): + return TonNode_blockIdExt( + workchain=-1, shard=0, seqno=123456, root_hash=b"\x01" * 32, file_hash=b"\x02" * 32 + ) + + +def test_reading_engine_validator_config(): + config = """ + { + "@type" : "engine.validator.config", + "out_port" : 3278, + "addrs" : [ + { + "@type" : "engine.addr", + "ip" : -1795443737, + "port" : 57227, + "categories" : [ + 0, + 1, + 2, + 3 + ], + "priority_categories" : [ + ] + } + ], + "adnl" : [ + { + "@type" : "engine.adnl", + "id" : "Ube8wLWXM3VOCsIaR9e2vojDMTxjpVRo6De1SJQoKM4=", + "category" : 0 + }, + { + "@type" : "engine.adnl", + "id" : "0XjyWTqN0+Q7CxboL/ag9gMfvGLOiZOFFA8+nQSv7IM=", + "category" : 0 + }, + { + "@type" : "engine.adnl", + "id" : "4VuLuYl0uqw87kQH+549L/JQwiOOWCOBbwMayqhX2Jo=", + "category" : 1 + } + ], + "dht" : [ + { + "@type" : "engine.dht", + "id" : "0XjyWTqN0+Q7CxboL/ag9gMfvGLOiZOFFA8+nQSv7IM=" + } + ], + "validators" : [ + ], + "collators" : [ + ], + "fullnode" : "4VuLuYl0uqw87kQH+549L/JQwiOOWCOBbwMayqhX2Jo=", + "fullnodeslaves" : [ + ], + "fullnodemasters" : [ + ], + "liteservers" : [ + { + "@type" : "engine.liteServer", + "id" : "LOQ9rpOR3RSYGCiRRKcOvlN3n31kIW6qv8M7XP07eZM=", + "port" : 10374 + } + ], + "control" : [ + { + "@type" : "engine.controlInterface", + "id" : "mhwOm4a0goKT8wVlhVyNQGHkqsg2FzTX1YK3Zp9NV08=", + "port" : 20223, + "allowed" : [ + { + "@type" : "engine.controlProcess", + "id" : "NhRlwaHf4OdLtuoOPm6p1yZmAIH8o+2/z7XqRmVnwv4=", + "permissions" : 15 + } + ] + } + ], + "shards_to_monitor" : [ + ], + "gc" : { + "@type" : "engine.gc", + "ids" : [ + ] + } + } + """ + + vc = Engine_validator_config.from_json(config) + + # bytes should be deserialized from base64 string + assert vc.adnl is not None + adnl_entry = vc.adnl[0] + assert adnl_entry is not None + adnl_id = adnl_entry.id + assert adnl_id is not None + assert adnl_id == base64.b64decode("Ube8wLWXM3VOCsIaR9e2vojDMTxjpVRo6De1SJQoKM4=") + + # serialization back to json should be the same + assert vc.to_json() == json.dumps(json.loads(config)) + + +def test_to_dict(pubk: Pub_unenc, block: TonNode_blockIdExt): + assert block.to_dict() == { + "@type": "tonNode.blockIdExt", + "workchain": -1, + "shard": 0, + "seqno": 123456, + "root_hash": "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=", + "file_hash": "AgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgI=", + } + + cg = Catchain_config_global(tag=b"\04" * 32, nodes=[pubk, pubk]) + assert cg.to_dict() == { + "@type": "catchain.config.global", + "tag": "BAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQ=", + "nodes": [ + {"@type": "pub.unenc", "data": "EhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhI="}, + {"@type": "pub.unenc", "data": "EhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhI="}, + ], + } + + pc = Adnl_packetContents(rand1=b"\x01\x01", flags=1, rand2=b"\x01\x01", from_=pubk) + assert pc.to_dict() == { + "@type": "adnl.packetContents", + "rand1": "AQE=", + "flags": 1, + "from_": {"@type": "pub.unenc", "data": "EhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhI="}, + "messages": [], + "seqno": 0, + "confirm_seqno": 0, + "recv_addr_list_version": 0, + "recv_priority_addr_list_version": 0, + "reinit_date": 0, + "dst_reinit_date": 0, + "signature": "", + "rand2": "AQE=", + } + pc2 = Adnl_packetContents.from_dict(pc.to_dict()) + assert pc == pc2 + + +def test_from_dict(pubk: Pub_unenc): + key = b"\x01" * 32 + c = Id_config_local(id=Pk_aes(key=key)) + + c2 = Id_config_local.from_dict( + { + "@type": "id.config.local", + "id": { + "@type": "pk.aes", + "key": base64.b64encode(key).decode(), + }, + } + ) + assert c == c2 + + pc_decoded = Adnl_packetContents.from_dict( + { + "@type": "adnl.packetContents", + "rand1": "AQE=", + "flags": 1, + "from_": {"@type": "pub.unenc", "data": "EhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhI="}, + "rand2": "AQE=", + } + ) + pc_expected = Adnl_packetContents(rand1=b"\x01\x01", flags=1, rand2=b"\x01\x01", from_=pubk) + assert pc_decoded == pc_expected diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000000..4a3ecc729e --- /dev/null +++ b/uv.lock @@ -0,0 +1,382 @@ +version = 1 +revision = 3 +requires-python = ">=3.13" + +[manifest] +members = [ + "ton", + "tontester", +] + +[[package]] +name = "basedpyright" +version = "1.33.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nodejs-wheel-binaries" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/62/c1/d62811ef22ec66bd3112ee2a8c18ff39ea87f3d8040cf3a93574024b17ea/basedpyright-1.33.0.tar.gz", hash = "sha256:a7e4d6d2285b93d8c0c91c75490d5bc00adbe7ef96ba1f1d4c010d9a731e243a", size = 22786957, upload-time = "2025-11-07T11:49:32.008Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/87/d95da7ef033c26d71a7153623e7b14fd9cc2c8676e03cf4e9199320cbaf5/basedpyright-1.33.0-py3-none-any.whl", hash = "sha256:1ab813683e28184aa60fd4000731f6fe9dbd083c9bd489420bb20f3e5dbb6ab8", size = 11859149, upload-time = "2025-11-07T11:49:28.694Z" }, +] + +[[package]] +name = "bitarray" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/06/92fdc84448d324ab8434b78e65caf4fb4c6c90b4f8ad9bdd4c8021bfaf1e/bitarray-3.8.0.tar.gz", hash = "sha256:3eae38daffd77c9621ae80c16932eea3fb3a4af141fb7cc724d4ad93eff9210d", size = 151991, upload-time = "2025-11-02T21:41:15.117Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/35/480364d4baf1e34c79076750914664373f561c58abb5c31c35b3fae613ff/bitarray-3.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:18214bac86341f1cc413772e66447d6cca10981e2880b70ecaf4e826c04f95e9", size = 148582, upload-time = "2025-11-02T21:39:42.268Z" }, + { url = "https://files.pythonhosted.org/packages/5e/a8/718b95524c803937f4edbaaf6480f39c80f6ed189d61357b345e8361ffb6/bitarray-3.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:01c5f0dc080b0ebb432f7a68ee1e88a76bd34f6d89c9568fcec65fb16ed71f0e", size = 145433, upload-time = "2025-11-02T21:39:43.552Z" }, + { url = "https://files.pythonhosted.org/packages/03/66/4a10f30dc9e2e01e3b4ecd44a511219f98e63c86b0e0f704c90fac24059b/bitarray-3.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:86685fa04067f7175f9718489ae755f6acde03593a1a9ca89305554af40e14fd", size = 332986, upload-time = "2025-11-02T21:39:44.656Z" }, + { url = "https://files.pythonhosted.org/packages/53/25/4c08774d847f80a1166e4c704b4e0f1c417c0afe6306eae0bc5e70d35faa/bitarray-3.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56896ceeffe25946c4010320629e2d858ca763cd8ded273c81672a5edbcb1e0a", size = 360634, upload-time = "2025-11-02T21:39:45.798Z" }, + { url = "https://files.pythonhosted.org/packages/a5/8f/bf8ad26169ebd0b2746d5c7564db734453ca467f8aab87e9d43b0a794383/bitarray-3.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9858dcbc23ba7eaadcd319786b982278a1a2b2020720b19db43e309579ff76fb", size = 371992, upload-time = "2025-11-02T21:39:46.968Z" }, + { url = "https://files.pythonhosted.org/packages/a9/16/ce166754e7c9d10650e02914552fa637cf3b2591f7ed16632bbf6b783312/bitarray-3.8.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aa7dec53c25f1949513457ef8b0ea1fb40e76c672cc4d2daa8ad3c8d6b73491a", size = 340315, upload-time = "2025-11-02T21:39:48.182Z" }, + { url = "https://files.pythonhosted.org/packages/de/2a/fbba3a106ddd260e84b9a624f730257c32ba51a8a029565248dfedfdf6f2/bitarray-3.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:15a2eff91f54d2b1f573cca8ca6fb58763ce8fea80e7899ab028f3987ef71cd5", size = 330473, upload-time = "2025-11-02T21:39:49.705Z" }, + { url = "https://files.pythonhosted.org/packages/68/97/56cf3c70196e7307ad32318a9d6ed969dbdc6a4534bbe429112fa7dfe42e/bitarray-3.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b1572ee0eb1967e71787af636bb7d1eb9c6735d5337762c450650e7f51844594", size = 358129, upload-time = "2025-11-02T21:39:51.189Z" }, + { url = "https://files.pythonhosted.org/packages/fd/be/afd391a5c0896d3339613321b2f94af853f29afc8bd3fbc327431244c642/bitarray-3.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5bfac7f236ba1a4d402644bdce47fb9db02a7cf3214a1f637d3a88390f9e5428", size = 356005, upload-time = "2025-11-02T21:39:52.355Z" }, + { url = "https://files.pythonhosted.org/packages/ae/08/a8e1a371babba29bad3378bb3a2cdca2b012170711e7fe1f22031a6b7b95/bitarray-3.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f0a55cf02d2cdd739b40ce10c09bbdd520e141217696add7a48b56e67bdfdfe6", size = 336862, upload-time = "2025-11-02T21:39:54.345Z" }, + { url = "https://files.pythonhosted.org/packages/ee/8a/6dc1d0fdc06991c8dc3b1fcfe1ae49fbaced42064cd1b5f24278e73fe05f/bitarray-3.8.0-cp313-cp313-win32.whl", hash = "sha256:a2ba92f59e30ce915e9e79af37649432e3a212ddddf416d4d686b1b4825bcdb2", size = 143018, upload-time = "2025-11-02T21:39:56.361Z" }, + { url = "https://files.pythonhosted.org/packages/2e/72/76e13f5cd23b8b9071747909663ce3b02da24a5e7e22c35146338625db35/bitarray-3.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:1c8f2a5d8006db5a555e06f9437e76bf52537d3dfd130cb8ae2b30866aca32c9", size = 149977, upload-time = "2025-11-02T21:39:57.718Z" }, + { url = "https://files.pythonhosted.org/packages/01/37/60f336c32336cc3ec03b0c61076f16ea2f05d5371c8a56e802161d218b77/bitarray-3.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:50ddbe3a7b4b6ab96812f5a4d570f401a2cdb95642fd04c062f98939610bbeee", size = 146930, upload-time = "2025-11-02T21:39:59.308Z" }, + { url = "https://files.pythonhosted.org/packages/1b/b0/411327a6c7f6b2bead64bb06fe60b92e0344957ec1ab0645d5ccc25fdafe/bitarray-3.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8cbd4bfc933b33b85c43ef4c1f4d5e3e9d91975ea6368acf5fbac02bac06ea89", size = 148563, upload-time = "2025-11-02T21:40:01.006Z" }, + { url = "https://files.pythonhosted.org/packages/2a/bc/ff80d97c627d774f879da0ea93223adb1267feab7e07d5c17580ffe6d632/bitarray-3.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9d35d8f8a1c9ed4e2b08187b513f8a3c71958600129db3aa26d85ea3abfd1310", size = 145422, upload-time = "2025-11-02T21:40:02.535Z" }, + { url = "https://files.pythonhosted.org/packages/66/e7/b4cb6c5689aacd0a32f3aa8a507155eaa33528c63de2f182b60843fbf700/bitarray-3.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:99f55e14e7c56f4fafe1343480c32b110ef03836c21ff7c48bae7add6818f77c", size = 332852, upload-time = "2025-11-02T21:40:03.645Z" }, + { url = "https://files.pythonhosted.org/packages/e7/91/fbd1b047e3e2f4b65590f289c8151df1d203d75b005f5aae4e072fe77d76/bitarray-3.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:dfbe2aa45b273f49e715c5345d94874cb65a28482bf231af408891c260601b8d", size = 360801, upload-time = "2025-11-02T21:40:04.827Z" }, + { url = "https://files.pythonhosted.org/packages/ef/4a/63064c593627bac8754fdafcb5343999c93ab2aeb27bcd9d270a010abea5/bitarray-3.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:64af877116edf051375b45f0bda648143176a017b13803ec7b3a3111dc05f4c5", size = 371408, upload-time = "2025-11-02T21:40:05.985Z" }, + { url = "https://files.pythonhosted.org/packages/46/97/ddc07723767bdafd170f2ff6e173c940fa874192783ee464aa3c1dedf07d/bitarray-3.8.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cdfbb27f2c46bb5bbdcee147530cbc5ca8ab858d7693924e88e30ada21b2c5e2", size = 340033, upload-time = "2025-11-02T21:40:07.189Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1e/e1ea9f1146fd4af032817069ff118918d73e5de519854ce3860e2ed560ff/bitarray-3.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4d73d4948dcc5591d880db8933004e01f1dd2296df9de815354d53469beb26fe", size = 330774, upload-time = "2025-11-02T21:40:08.496Z" }, + { url = "https://files.pythonhosted.org/packages/cf/9f/8242296c124a48d1eab471fd0838aeb7ea9c6fd720302d99ab7855d3e6d3/bitarray-3.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:28a85b056c0eb7f5d864c0ceef07034117e8ebfca756f50648c71950a568ba11", size = 358337, upload-time = "2025-11-02T21:40:10.035Z" }, + { url = "https://files.pythonhosted.org/packages/b5/6b/9095d75264c67d479f298c80802422464ce18c3cdd893252eeccf4997611/bitarray-3.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:79ec4498a545733ecace48d780d22407411b07403a2e08b9a4d7596c0b97ebd7", size = 355639, upload-time = "2025-11-02T21:40:11.485Z" }, + { url = "https://files.pythonhosted.org/packages/a0/af/c93c0ae5ef824136e90ac7ddf6cceccb1232f34240b2f55a922f874da9b4/bitarray-3.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:33af25c4ff7723363cb8404dfc2eefeab4110b654f6c98d26aba8a08c745d860", size = 336999, upload-time = "2025-11-02T21:40:12.709Z" }, + { url = "https://files.pythonhosted.org/packages/81/0f/72c951f5997b2876355d5e671f78dd2362493254876675cf22dbd24389ae/bitarray-3.8.0-cp314-cp314-win32.whl", hash = "sha256:2c3bb96b6026643ce24677650889b09073f60b9860a71765f843c99f9ab38b25", size = 142169, upload-time = "2025-11-02T21:40:14.031Z" }, + { url = "https://files.pythonhosted.org/packages/8a/55/ef1b4de8107bf13823da8756c20e1fbc9452228b4e837f46f6d9ddba3eb3/bitarray-3.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:847c7f61964225fc489fe1d49eda7e0e0d253e98862c012cecf845f9ad45cdf4", size = 148737, upload-time = "2025-11-02T21:40:15.436Z" }, + { url = "https://files.pythonhosted.org/packages/5f/26/bc0784136775024ac56cc67c0d6f9aa77a7770de7f82c3a7c9be11c217cd/bitarray-3.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:a2cb35a6efaa0e3623d8272471371a12c7e07b51a33e5efce9b58f655d864b4e", size = 146083, upload-time = "2025-11-02T21:40:17.135Z" }, + { url = "https://files.pythonhosted.org/packages/6e/64/57984e64264bf43d93a1809e645972771566a2d0345f4896b041ce20b000/bitarray-3.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:15e8d0597cc6e8496de6f4dea2a6880c57e1251502a7072f5631108a1aa28521", size = 149455, upload-time = "2025-11-02T21:40:18.558Z" }, + { url = "https://files.pythonhosted.org/packages/81/c0/0d5f2eaef1867f462f764bdb07d1e116c33a1bf052ea21889aefe4282f5b/bitarray-3.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8ffe660e963ae711cb9e2b8d8461c9b1ad6167823837fc17d59d5e539fb898fa", size = 146491, upload-time = "2025-11-02T21:40:19.665Z" }, + { url = "https://files.pythonhosted.org/packages/65/c6/bc1261f7a8862c0c59220a484464739e52235fd1e2afcb24d7f7d3fb5702/bitarray-3.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4779f356083c62e29b4198d290b7b17a39a69702d150678b7efff0fdddf494a8", size = 339721, upload-time = "2025-11-02T21:40:21.277Z" }, + { url = "https://files.pythonhosted.org/packages/81/d8/289ca55dd2939ea17b1108dc53bffc0fdc5160ba44f77502dfaae35d08c6/bitarray-3.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:025d133bf4ca8cf75f904eeb8ea946228d7c043231866143f31946a6f4dd0bf3", size = 367823, upload-time = "2025-11-02T21:40:22.463Z" }, + { url = "https://files.pythonhosted.org/packages/91/a2/61e7461ca9ac0fcb70f327a2e84b006996d2a840898e69037a39c87c6d06/bitarray-3.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:451f9958850ea98440d542278368c8d1e1ea821e2494b204570ba34a340759df", size = 377341, upload-time = "2025-11-02T21:40:23.789Z" }, + { url = "https://files.pythonhosted.org/packages/6c/87/4a0c9c8bdb13916d443e04d8f8542eef9190f31425da3c17c3478c40173f/bitarray-3.8.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6d79f659965290af60d6acc8e2716341865fe74609a7ede2a33c2f86ad893b8f", size = 344985, upload-time = "2025-11-02T21:40:25.261Z" }, + { url = "https://files.pythonhosted.org/packages/17/4c/ff9259b916efe53695b631772e5213699c738efc2471b5ffe273f4000994/bitarray-3.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fbf05678c2ae0064fb1b8de7e9e8f0fc30621b73c8477786dd0fb3868044a8c8", size = 336796, upload-time = "2025-11-02T21:40:26.942Z" }, + { url = "https://files.pythonhosted.org/packages/0f/4b/51b2468bbddbade5e2f3b8d5db08282c5b309e8687b0f02f75a8b5ff559c/bitarray-3.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:c396358023b876cff547ce87f4e8ff8a2280598873a137e8cc69e115262260b8", size = 365085, upload-time = "2025-11-02T21:40:28.224Z" }, + { url = "https://files.pythonhosted.org/packages/bf/79/53473bfc2e052c6dbb628cdc1b156be621c77aaeb715918358b01574be55/bitarray-3.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed3493a369fe849cce98542d7405c88030b355e4d2e113887cb7ecc86c205773", size = 361012, upload-time = "2025-11-02T21:40:29.635Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b1/242bf2e44bfc69e73fa2b954b425d761a8e632f78ea31008f1c3cfad0854/bitarray-3.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c764fb167411d5afaef88138542a4bfa28bd5e5ded5e8e42df87cef965efd6e9", size = 340644, upload-time = "2025-11-02T21:40:31.089Z" }, + { url = "https://files.pythonhosted.org/packages/cf/01/12e5ecf30a5de28a32485f226cad4b8a546845f65f755ce0365057ab1e92/bitarray-3.8.0-cp314-cp314t-win32.whl", hash = "sha256:e12769d3adcc419e65860de946df8d2ed274932177ac1cdb05186e498aaa9149", size = 143630, upload-time = "2025-11-02T21:40:32.351Z" }, + { url = "https://files.pythonhosted.org/packages/b6/92/6b6ade587b08024a8a890b07724775d29da9cf7497be5c3cbe226185e463/bitarray-3.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0ca70ccf789446a6dfde40b482ec21d28067172cd1f8efd50d5548159fccad9e", size = 150250, upload-time = "2025-11-02T21:40:33.596Z" }, + { url = "https://files.pythonhosted.org/packages/ed/40/be3858ffed004e47e48a2cefecdbf9b950d41098b780f9dc3aa609a88351/bitarray-3.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2a3d1b05ffdd3e95687942ae7b13c63689f85d3f15c39b33329e3cb9ce6c015f", size = 147015, upload-time = "2025-11-02T21:40:35.064Z" }, +] + +[[package]] +name = "certifi" +version = "2025.10.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/5b/b6ce21586237c77ce67d01dc5507039d444b630dd76611bbca2d8e5dcd91/certifi-2025.10.5.tar.gz", hash = "sha256:47c09d31ccf2acf0be3f701ea53595ee7e0b8fa08801c6624be771df09ae7b43", size = 164519, upload-time = "2025-10-05T04:12:15.808Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/37/af0d2ef3967ac0d6113837b44a4f0bfe1328c2b9763bd5b1744520e5cfed/certifi-2025.10.5-py3-none-any.whl", hash = "sha256:0f212c2744a9bb6de0c56639a6f68afe01ecd92d91f14ae897c4fe7bbeeef0de", size = 163286, upload-time = "2025-10-05T04:12:14.03Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/13/69/33ddede1939fdd074bce5434295f38fae7136463422fe4fd3e0e89b98062/charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a", size = 129418, upload-time = "2025-10-14T04:42:32.879Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/45/4b3a1239bbacd321068ea6e7ac28875b03ab8bc0aa0966452db17cd36714/charset_normalizer-3.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794", size = 208091, upload-time = "2025-10-14T04:41:13.346Z" }, + { url = "https://files.pythonhosted.org/packages/7d/62/73a6d7450829655a35bb88a88fca7d736f9882a27eacdca2c6d505b57e2e/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed", size = 147936, upload-time = "2025-10-14T04:41:14.461Z" }, + { url = "https://files.pythonhosted.org/packages/89/c5/adb8c8b3d6625bef6d88b251bbb0d95f8205831b987631ab0c8bb5d937c2/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72", size = 144180, upload-time = "2025-10-14T04:41:15.588Z" }, + { url = "https://files.pythonhosted.org/packages/91/ed/9706e4070682d1cc219050b6048bfd293ccf67b3d4f5a4f39207453d4b99/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328", size = 161346, upload-time = "2025-10-14T04:41:16.738Z" }, + { url = "https://files.pythonhosted.org/packages/d5/0d/031f0d95e4972901a2f6f09ef055751805ff541511dc1252ba3ca1f80cf5/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede", size = 158874, upload-time = "2025-10-14T04:41:17.923Z" }, + { url = "https://files.pythonhosted.org/packages/f5/83/6ab5883f57c9c801ce5e5677242328aa45592be8a00644310a008d04f922/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894", size = 153076, upload-time = "2025-10-14T04:41:19.106Z" }, + { url = "https://files.pythonhosted.org/packages/75/1e/5ff781ddf5260e387d6419959ee89ef13878229732732ee73cdae01800f2/charset_normalizer-3.4.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1", size = 150601, upload-time = "2025-10-14T04:41:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/d7/57/71be810965493d3510a6ca79b90c19e48696fb1ff964da319334b12677f0/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490", size = 150376, upload-time = "2025-10-14T04:41:21.398Z" }, + { url = "https://files.pythonhosted.org/packages/e5/d5/c3d057a78c181d007014feb7e9f2e65905a6c4ef182c0ddf0de2924edd65/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44", size = 144825, upload-time = "2025-10-14T04:41:22.583Z" }, + { url = "https://files.pythonhosted.org/packages/e6/8c/d0406294828d4976f275ffbe66f00266c4b3136b7506941d87c00cab5272/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133", size = 162583, upload-time = "2025-10-14T04:41:23.754Z" }, + { url = "https://files.pythonhosted.org/packages/d7/24/e2aa1f18c8f15c4c0e932d9287b8609dd30ad56dbe41d926bd846e22fb8d/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3", size = 150366, upload-time = "2025-10-14T04:41:25.27Z" }, + { url = "https://files.pythonhosted.org/packages/e4/5b/1e6160c7739aad1e2df054300cc618b06bf784a7a164b0f238360721ab86/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e", size = 160300, upload-time = "2025-10-14T04:41:26.725Z" }, + { url = "https://files.pythonhosted.org/packages/7a/10/f882167cd207fbdd743e55534d5d9620e095089d176d55cb22d5322f2afd/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc", size = 154465, upload-time = "2025-10-14T04:41:28.322Z" }, + { url = "https://files.pythonhosted.org/packages/89/66/c7a9e1b7429be72123441bfdbaf2bc13faab3f90b933f664db506dea5915/charset_normalizer-3.4.4-cp313-cp313-win32.whl", hash = "sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac", size = 99404, upload-time = "2025-10-14T04:41:29.95Z" }, + { url = "https://files.pythonhosted.org/packages/c4/26/b9924fa27db384bdcd97ab83b4f0a8058d96ad9626ead570674d5e737d90/charset_normalizer-3.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14", size = 107092, upload-time = "2025-10-14T04:41:31.188Z" }, + { url = "https://files.pythonhosted.org/packages/af/8f/3ed4bfa0c0c72a7ca17f0380cd9e4dd842b09f664e780c13cff1dcf2ef1b/charset_normalizer-3.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2", size = 100408, upload-time = "2025-10-14T04:41:32.624Z" }, + { url = "https://files.pythonhosted.org/packages/2a/35/7051599bd493e62411d6ede36fd5af83a38f37c4767b92884df7301db25d/charset_normalizer-3.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd", size = 207746, upload-time = "2025-10-14T04:41:33.773Z" }, + { url = "https://files.pythonhosted.org/packages/10/9a/97c8d48ef10d6cd4fcead2415523221624bf58bcf68a802721a6bc807c8f/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb", size = 147889, upload-time = "2025-10-14T04:41:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/979224a919a1b606c82bd2c5fa49b5c6d5727aa47b4312bb27b1734f53cd/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e", size = 143641, upload-time = "2025-10-14T04:41:36.116Z" }, + { url = "https://files.pythonhosted.org/packages/ba/33/0ad65587441fc730dc7bd90e9716b30b4702dc7b617e6ba4997dc8651495/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14", size = 160779, upload-time = "2025-10-14T04:41:37.229Z" }, + { url = "https://files.pythonhosted.org/packages/67/ed/331d6b249259ee71ddea93f6f2f0a56cfebd46938bde6fcc6f7b9a3d0e09/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191", size = 159035, upload-time = "2025-10-14T04:41:38.368Z" }, + { url = "https://files.pythonhosted.org/packages/67/ff/f6b948ca32e4f2a4576aa129d8bed61f2e0543bf9f5f2b7fc3758ed005c9/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838", size = 152542, upload-time = "2025-10-14T04:41:39.862Z" }, + { url = "https://files.pythonhosted.org/packages/16/85/276033dcbcc369eb176594de22728541a925b2632f9716428c851b149e83/charset_normalizer-3.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6", size = 149524, upload-time = "2025-10-14T04:41:41.319Z" }, + { url = "https://files.pythonhosted.org/packages/9e/f2/6a2a1f722b6aba37050e626530a46a68f74e63683947a8acff92569f979a/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e", size = 150395, upload-time = "2025-10-14T04:41:42.539Z" }, + { url = "https://files.pythonhosted.org/packages/60/bb/2186cb2f2bbaea6338cad15ce23a67f9b0672929744381e28b0592676824/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c", size = 143680, upload-time = "2025-10-14T04:41:43.661Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a5/bf6f13b772fbb2a90360eb620d52ed8f796f3c5caee8398c3b2eb7b1c60d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090", size = 162045, upload-time = "2025-10-14T04:41:44.821Z" }, + { url = "https://files.pythonhosted.org/packages/df/c5/d1be898bf0dc3ef9030c3825e5d3b83f2c528d207d246cbabe245966808d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152", size = 149687, upload-time = "2025-10-14T04:41:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/a5/42/90c1f7b9341eef50c8a1cb3f098ac43b0508413f33affd762855f67a410e/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828", size = 160014, upload-time = "2025-10-14T04:41:47.631Z" }, + { url = "https://files.pythonhosted.org/packages/76/be/4d3ee471e8145d12795ab655ece37baed0929462a86e72372fd25859047c/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec", size = 154044, upload-time = "2025-10-14T04:41:48.81Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6f/8f7af07237c34a1defe7defc565a9bc1807762f672c0fde711a4b22bf9c0/charset_normalizer-3.4.4-cp314-cp314-win32.whl", hash = "sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9", size = 99940, upload-time = "2025-10-14T04:41:49.946Z" }, + { url = "https://files.pythonhosted.org/packages/4b/51/8ade005e5ca5b0d80fb4aff72a3775b325bdc3d27408c8113811a7cbe640/charset_normalizer-3.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c", size = 107104, upload-time = "2025-10-14T04:41:51.051Z" }, + { url = "https://files.pythonhosted.org/packages/da/5f/6b8f83a55bb8278772c5ae54a577f3099025f9ade59d0136ac24a0df4bde/charset_normalizer-3.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2", size = 100743, upload-time = "2025-10-14T04:41:52.122Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "crc16" +version = "0.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/e0/70a44c4385f2b33df82e518005aae16b5c1feaf082c73c0acebe3426fc0a/crc16-0.1.1.tar.gz", hash = "sha256:c1f86aa0390f4baf07d2631b16b979580eae1d9a973a826ce45353a22ee8d396", size = 9631, upload-time = "2011-06-04T23:20:40.62Z" } + +[[package]] +name = "crc32c" +version = "2.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/66/7e97aa77af7cf6afbff26e3651b564fe41932599bc2d3dce0b2f73d4829a/crc32c-2.8.tar.gz", hash = "sha256:578728964e59c47c356aeeedee6220e021e124b9d3e8631d95d9a5e5f06e261c", size = 48179, upload-time = "2025-10-17T06:20:13.61Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/d8/3ae227890b3be40955a7144106ef4dd97d6123a82c2a5310cdab58ca49d8/crc32c-2.8-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:36f1e03ee9e9c6938e67d3bcb60e36f260170aa5f37da1185e04ef37b56af395", size = 66380, upload-time = "2025-10-17T06:19:18.009Z" }, + { url = "https://files.pythonhosted.org/packages/bd/8b/178d3f987cd0e049b484615512d3f91f3d2caeeb8ff336bb5896ae317438/crc32c-2.8-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b2f3226b94b85a8dd9b3533601d7a63e9e3e8edf03a8a169830ee8303a199aeb", size = 63048, upload-time = "2025-10-17T06:19:18.853Z" }, + { url = "https://files.pythonhosted.org/packages/f2/a1/48145ae2545ebc0169d3283ebe882da580ea4606bfb67cf4ca922ac3cfc3/crc32c-2.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6e08628bc72d5b6bc8e0730e8f142194b610e780a98c58cb6698e665cb885a5b", size = 61530, upload-time = "2025-10-17T06:19:19.974Z" }, + { url = "https://files.pythonhosted.org/packages/06/4b/cf05ed9d934cc30e5ae22f97c8272face420a476090e736615d9a6b53de0/crc32c-2.8-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:086f64793c5ec856d1ab31a026d52ad2b895ac83d7a38fce557d74eb857f0a82", size = 80001, upload-time = "2025-10-17T06:19:20.784Z" }, + { url = "https://files.pythonhosted.org/packages/15/ab/4b04801739faf36345f6ba1920be5b1c70282fec52f8280afd3613fb13e2/crc32c-2.8-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcf72ee7e0135b3d941c34bb2c26c3fc6bc207106b49fd89aaafaeae223ae209", size = 81543, upload-time = "2025-10-17T06:19:21.557Z" }, + { url = "https://files.pythonhosted.org/packages/a9/1b/6e38dde5bfd2ea69b7f2ab6ec229fcd972a53d39e2db4efe75c0ac0382ce/crc32c-2.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8a717dd9c3fd777d9bc6603717eae172887d402c4ab589d124ebd0184a83f89e", size = 80644, upload-time = "2025-10-17T06:19:22.325Z" }, + { url = "https://files.pythonhosted.org/packages/ce/45/012176ffee90059ae8ec7131019c71724ea472aa63e72c0c8edbd1fad1d7/crc32c-2.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0450bb845b3c3c7b9bdc0b4e95620ec9a40824abdc8c86d6285c919a90743c1a", size = 79919, upload-time = "2025-10-17T06:19:23.101Z" }, + { url = "https://files.pythonhosted.org/packages/f0/2b/f557629842f9dec2b3461cb3a0d854bb586ec45b814cea58b082c32f0dde/crc32c-2.8-cp313-cp313-win32.whl", hash = "sha256:765d220bfcbcffa6598ac11eb1e10af0ee4802b49fe126aa6bf79f8ddb9931d1", size = 64896, upload-time = "2025-10-17T06:19:23.88Z" }, + { url = "https://files.pythonhosted.org/packages/d0/db/fd0f698c15d1e21d47c64181a98290665a08fcbb3940cd559e9c15bda57e/crc32c-2.8-cp313-cp313-win_amd64.whl", hash = "sha256:171ff0260d112c62abcce29332986950a57bddee514e0a2418bfde493ea06bb3", size = 66646, upload-time = "2025-10-17T06:19:24.702Z" }, + { url = "https://files.pythonhosted.org/packages/db/b9/8e5d7054fe8e7eecab10fd0c8e7ffb01439417bdb6de1d66a81c38fc4a20/crc32c-2.8-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b977a32a3708d6f51703c8557008f190aaa434d7347431efb0e86fcbe78c2a50", size = 66203, upload-time = "2025-10-17T06:19:25.872Z" }, + { url = "https://files.pythonhosted.org/packages/55/5f/cc926c70057a63cc0c98a3c8a896eb15fc7e74d3034eadd53c94917c6cc3/crc32c-2.8-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7399b01db4adaf41da2fb36fe2408e75a8d82a179a9564ed7619412e427b26d6", size = 62956, upload-time = "2025-10-17T06:19:26.652Z" }, + { url = "https://files.pythonhosted.org/packages/a1/8a/0660c44a2dd2cb6ccbb529eb363b9280f5c766f1017bc8355ed8d695bd94/crc32c-2.8-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4379f73f9cdad31958a673d11a332ec725ca71572401ca865867229f5f15e853", size = 61442, upload-time = "2025-10-17T06:19:27.74Z" }, + { url = "https://files.pythonhosted.org/packages/f5/5a/6108d2dfc0fe33522ce83ba07aed4b22014911b387afa228808a278e27cd/crc32c-2.8-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e68264555fab19bab08331550dab58573e351a63ed79c869d455edd3b0aa417", size = 79109, upload-time = "2025-10-17T06:19:28.535Z" }, + { url = "https://files.pythonhosted.org/packages/84/1e/c054f9e390090c197abf3d2936f4f9effaf0c6ee14569ae03d6ddf86958a/crc32c-2.8-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b48f2486727b8d0e7ccbae4a34cb0300498433d2a9d6b49cb13cb57c2e3f19cb", size = 80987, upload-time = "2025-10-17T06:19:29.305Z" }, + { url = "https://files.pythonhosted.org/packages/c8/ad/1650e5c3341e4a485f800ea83116d72965030c5d48ccc168fcc685756e4d/crc32c-2.8-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ecf123348934a086df8c8fde7f9f2d716d523ca0707c5a1367b8bb00d8134823", size = 79994, upload-time = "2025-10-17T06:19:30.109Z" }, + { url = "https://files.pythonhosted.org/packages/d7/3b/f2ed924b177729cbb2ab30ca2902abff653c31d48c95e7b66717a9ca9fcc/crc32c-2.8-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e636ac60f76de538f7a2c0d0f3abf43104ee83a8f5e516f6345dc283ed1a4df7", size = 79046, upload-time = "2025-10-17T06:19:30.894Z" }, + { url = "https://files.pythonhosted.org/packages/4b/80/413b05ee6ace613208b31b3670c3135ee1cf451f0e72a9c839b4946acc04/crc32c-2.8-cp313-cp313t-win32.whl", hash = "sha256:8dd4a19505e0253892e1b2f1425cc3bd47f79ae5a04cb8800315d00aad7197f2", size = 64837, upload-time = "2025-10-17T06:19:32.03Z" }, + { url = "https://files.pythonhosted.org/packages/3b/1b/85eddb6ac5b38496c4e35c20298aae627970c88c3c624a22ab33e84f16c7/crc32c-2.8-cp313-cp313t-win_amd64.whl", hash = "sha256:4bb18e4bd98fb266596523ffc6be9c5b2387b2fa4e505ec56ca36336f49cb639", size = 66574, upload-time = "2025-10-17T06:19:33.143Z" }, + { url = "https://files.pythonhosted.org/packages/aa/df/50e9079b532ff53dbfc0e66eed781374bd455af02ed5df8b56ad538de4ff/crc32c-2.8-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3a3b2e4bcf7b3ee333050e7d3ff38e2ba46ea205f1d73d8949b248aaffe937ac", size = 66399, upload-time = "2025-10-17T06:19:34.279Z" }, + { url = "https://files.pythonhosted.org/packages/5a/2e/67e3b0bc3d30e46ea5d16365cc81203286387671e22f2307eb41f19abb9c/crc32c-2.8-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:445e559e66dff16be54f8a4ef95aa6b01db799a639956d995c5498ba513fccc2", size = 63044, upload-time = "2025-10-17T06:19:35.062Z" }, + { url = "https://files.pythonhosted.org/packages/36/ea/1723b17437e4344ed8d067456382ecb1f5b535d83fdc5aaebab676c6d273/crc32c-2.8-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bf3040919e17afa5782e01b1875d6a05f44b8f19c05f211d8b9f8a1deb8bbd9c", size = 61541, upload-time = "2025-10-17T06:19:36.204Z" }, + { url = "https://files.pythonhosted.org/packages/4c/6a/cbec8a235c5b46a01f319939b538958662159aec0ed3a74944e3a6de21f1/crc32c-2.8-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5607ab8221e1ffd411f64aa40dbb6850cf06dd2908c9debd05d371e1acf62ff3", size = 80139, upload-time = "2025-10-17T06:19:37.351Z" }, + { url = "https://files.pythonhosted.org/packages/21/31/d096722fe74b692d6e8206c27da1ea5f6b2a12ff92c54a62a6ba2f376254/crc32c-2.8-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f5db4f16816926986d3c94253314920689706ae13a9bf4888b47336c6735ce", size = 81736, upload-time = "2025-10-17T06:19:38.16Z" }, + { url = "https://files.pythonhosted.org/packages/f6/a2/f75ef716ff7e3c22f385ba6ef30c5de80c19a21ebe699dc90824a1903275/crc32c-2.8-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:70b0153c4d418b673309d3529334d117e1074c4a3b2d7f676e430d72c14de67b", size = 80795, upload-time = "2025-10-17T06:19:38.948Z" }, + { url = "https://files.pythonhosted.org/packages/d8/94/6d647a12d96ab087d9b8eacee3da073f981987827d57c7072f89ffc7b6cd/crc32c-2.8-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c8933531442042438753755a5c8a9034e4d88b01da9eb796f7e151b31a7256c", size = 80042, upload-time = "2025-10-17T06:19:39.725Z" }, + { url = "https://files.pythonhosted.org/packages/cd/dc/32b8896b40a0afee7a3c040536d0da5a73e68df2be9fadd21770fd158e16/crc32c-2.8-cp314-cp314-win32.whl", hash = "sha256:cdc83a3fe6c4e5df9457294cfd643de7d95bd4e9382c1dd6ed1e0f0f9169172c", size = 64914, upload-time = "2025-10-17T06:19:40.527Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b4/4308b27d307e8ecaf8dd1dcc63bbb0e47ae1826d93faa3e62d1ee00ee2d5/crc32c-2.8-cp314-cp314-win_amd64.whl", hash = "sha256:509e10035106df66770fe24b9eb8d9e32b6fb967df17744402fb67772d8b2bc7", size = 66723, upload-time = "2025-10-17T06:19:42.449Z" }, + { url = "https://files.pythonhosted.org/packages/90/d5/a19d2489fa997a143bfbbf971a5c9a43f8b1ba9e775b1fb362d8fb15260c/crc32c-2.8-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:864359a39777a07b09b28eb31337c0cc603d5c1bf0fc328c3af736a8da624ec0", size = 66201, upload-time = "2025-10-17T06:19:43.273Z" }, + { url = "https://files.pythonhosted.org/packages/98/c2/5f82f22d2c1242cb6f6fe92aa9a42991ebea86de994b8f9974d9c1d128e2/crc32c-2.8-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:14511d7cfc5d9f5e1a6c6b64caa6225c2bdc1ed00d725e9a374a3e84073ce180", size = 62956, upload-time = "2025-10-17T06:19:44.099Z" }, + { url = "https://files.pythonhosted.org/packages/9b/61/3d43d33489cf974fb78bfb3500845770e139ae6d1d83473b660bd8f79a6c/crc32c-2.8-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:918b7999b52b5dcbcea34081e9a02d46917d571921a3f209956a9a429b2e06e5", size = 61443, upload-time = "2025-10-17T06:19:44.89Z" }, + { url = "https://files.pythonhosted.org/packages/52/6d/f306ce64a352a3002f76b0fc88a1373f4541f9d34fad3668688610bab14b/crc32c-2.8-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cc445da03fc012a5a03b71da1df1b40139729e6a5571fd4215ab40bfb39689c7", size = 79106, upload-time = "2025-10-17T06:19:45.688Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b7/1f74965dd7ea762954a69d172dfb3a706049c84ffa45d31401d010a4a126/crc32c-2.8-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e3dde2ec59a8a830511d72a086ead95c0b0b7f0d418f93ea106244c5e77e350", size = 80983, upload-time = "2025-10-17T06:19:46.792Z" }, + { url = "https://files.pythonhosted.org/packages/1b/50/af93f0d91ccd61833ce77374ebfbd16f5805f5c17d18c6470976d9866d76/crc32c-2.8-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:61d51681a08b6a2a2e771b7f0cd1947fb87cb28f38ed55a01cb7c40b2ac4cdd8", size = 80009, upload-time = "2025-10-17T06:19:47.619Z" }, + { url = "https://files.pythonhosted.org/packages/ee/fa/94f394beb68a88258af694dab2f1284f55a406b615d7900bdd6235283bc4/crc32c-2.8-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:67c0716c3b1a02d5235be649487b637eed21f2d070f2b3f63f709dcd2fefb4c7", size = 79066, upload-time = "2025-10-17T06:19:48.409Z" }, + { url = "https://files.pythonhosted.org/packages/91/c6/a6050e0c64fd73c67a97da96cb59f08b05111e00b958fb87ecdce99f17ac/crc32c-2.8-cp314-cp314t-win32.whl", hash = "sha256:2e8fe863fbbd8bdb6b414a2090f1b0f52106e76e9a9c96a413495dbe5ebe492a", size = 64869, upload-time = "2025-10-17T06:19:49.197Z" }, + { url = "https://files.pythonhosted.org/packages/08/1f/c7735034e401cb1ea14f996a224518e3a3fa9987cb13680e707328a7d779/crc32c-2.8-cp314-cp314t-win_amd64.whl", hash = "sha256:20a9cfb897693eb6da19e52e2a7be2026fd4d9fc8ae318f086c0d71d5dd2d8e0", size = 66633, upload-time = "2025-10-17T06:19:50.003Z" }, +] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "nodejs-wheel-binaries" +version = "22.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/54/02f58c8119e2f1984e2572cc77a7b469dbaf4f8d171ad376e305749ef48e/nodejs_wheel_binaries-22.20.0.tar.gz", hash = "sha256:a62d47c9fd9c32191dff65bbe60261504f26992a0a19fe8b4d523256a84bd351", size = 8058, upload-time = "2025-09-26T09:48:00.906Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/6d/333e5458422f12318e3c3e6e7f194353aa68b0d633217c7e89833427ca01/nodejs_wheel_binaries-22.20.0-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:455add5ac4f01c9c830ab6771dbfad0fdf373f9b040d3aabe8cca9b6c56654fb", size = 53246314, upload-time = "2025-09-26T09:47:32.536Z" }, + { url = "https://files.pythonhosted.org/packages/56/30/dcd6879d286a35b3c4c8f9e5e0e1bcf4f9e25fe35310fc77ecf97f915a23/nodejs_wheel_binaries-22.20.0-py2.py3-none-macosx_11_0_x86_64.whl", hash = "sha256:5d8c12f97eea7028b34a84446eb5ca81829d0c428dfb4e647e09ac617f4e21fa", size = 53644391, upload-time = "2025-09-26T09:47:36.093Z" }, + { url = "https://files.pythonhosted.org/packages/58/be/c7b2e7aa3bb281d380a1c531f84d0ccfe225832dfc3bed1ca171753b9630/nodejs_wheel_binaries-22.20.0-py2.py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a2b0989194148f66e9295d8f11bc463bde02cbe276517f4d20a310fb84780ae", size = 60282516, upload-time = "2025-09-26T09:47:39.88Z" }, + { url = "https://files.pythonhosted.org/packages/3e/c5/8befacf4190e03babbae54cb0809fb1a76e1600ec3967ab8ee9f8fc85b65/nodejs_wheel_binaries-22.20.0-py2.py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5c500aa4dc046333ecb0a80f183e069e5c30ce637f1c1a37166b2c0b642dc21", size = 60347290, upload-time = "2025-09-26T09:47:43.712Z" }, + { url = "https://files.pythonhosted.org/packages/c0/bd/cfffd1e334277afa0714962c6ec432b5fe339340a6bca2e5fa8e678e7590/nodejs_wheel_binaries-22.20.0-py2.py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3279eb1b99521f0d20a850bbfc0159a658e0e85b843b3cf31b090d7da9f10dfc", size = 62178798, upload-time = "2025-09-26T09:47:47.752Z" }, + { url = "https://files.pythonhosted.org/packages/08/14/10b83a9c02faac985b3e9f5e65d63a34fc0f46b48d8a2c3e4caa3e1e7318/nodejs_wheel_binaries-22.20.0-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:d29705797b33bade62d79d8f106c2453c8a26442a9b2a5576610c0f7e7c351ed", size = 62772957, upload-time = "2025-09-26T09:47:51.266Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a9/c6a480259aa0d6b270aac2c6ba73a97444b9267adde983a5b7e34f17e45a/nodejs_wheel_binaries-22.20.0-py2.py3-none-win_amd64.whl", hash = "sha256:4bd658962f24958503541963e5a6f2cc512a8cb301e48a69dc03c879f40a28ae", size = 40120431, upload-time = "2025-09-26T09:47:54.363Z" }, + { url = "https://files.pythonhosted.org/packages/42/b1/6a4eb2c6e9efa028074b0001b61008c9d202b6b46caee9e5d1b18c088216/nodejs_wheel_binaries-22.20.0-py2.py3-none-win_arm64.whl", hash = "sha256:1fccac931faa210d22b6962bcdbc99269d16221d831b9a118bbb80fe434a60b8", size = 38844133, upload-time = "2025-09-26T09:47:57.357Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/56/f013048ac4bc4c1d9be45afd4ab209ea62822fb1598f40687e6bf45dcea4/pytest-9.0.1.tar.gz", hash = "sha256:3e9c069ea73583e255c3b21cf46b8d3c56f6e3a1a8f6da94ccb0fcf57b9d73c8", size = 1564125, upload-time = "2025-11-12T13:05:09.333Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/8b/6300fb80f858cda1c51ffa17075df5d846757081d11ab4aa35cef9e6258b/pytest-9.0.1-py3-none-any.whl", hash = "sha256:67be0030d194df2dfa7b556f2e56fb3c3315bd5c8822c6951162b92b32ce7dad", size = 373668, upload-time = "2025-11-12T13:05:07.379Z" }, +] + +[[package]] +name = "pytonlib" +version = "0.0.68" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "crc16" }, + { name = "requests" }, + { name = "tvm-valuetypes" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/17/9b/9a3d40841dc3a460e308b7a284152b00f4378c57fcfd57ef5b09ca21e8d8/pytonlib-0.0.68.tar.gz", hash = "sha256:a2414cd2692388edf9114a00dc81cbae38c7197a67042844f886275297b82279", size = 27425030, upload-time = "2025-11-03T08:05:30.977Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/c1/dc76ccabe5818dccbb131ef6e069400b3fea39f14f5a5f3ff07761f6731d/pytonlib-0.0.68-py3-none-any.whl", hash = "sha256:53ced393bbac68c6a294c73e74b8f060c0a4a2349d8e74622ea521655645d010", size = 27531020, upload-time = "2025-11-03T08:05:28.243Z" }, +] + +[[package]] +name = "requests" +version = "2.32.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, +] + +[[package]] +name = "ruff" +version = "0.14.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/55/cccfca45157a2031dcbb5a462a67f7cf27f8b37d4b3b1cd7438f0f5c1df6/ruff-0.14.4.tar.gz", hash = "sha256:f459a49fe1085a749f15414ca76f61595f1a2cc8778ed7c279b6ca2e1fd19df3", size = 5587844, upload-time = "2025-11-06T22:07:45.033Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/b9/67240254166ae1eaa38dec32265e9153ac53645a6c6670ed36ad00722af8/ruff-0.14.4-py3-none-linux_armv6l.whl", hash = "sha256:e6604613ffbcf2297cd5dcba0e0ac9bd0c11dc026442dfbb614504e87c349518", size = 12606781, upload-time = "2025-11-06T22:07:01.841Z" }, + { url = "https://files.pythonhosted.org/packages/46/c8/09b3ab245d8652eafe5256ab59718641429f68681ee713ff06c5c549f156/ruff-0.14.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:d99c0b52b6f0598acede45ee78288e5e9b4409d1ce7f661f0fa36d4cbeadf9a4", size = 12946765, upload-time = "2025-11-06T22:07:05.858Z" }, + { url = "https://files.pythonhosted.org/packages/14/bb/1564b000219144bf5eed2359edc94c3590dd49d510751dad26202c18a17d/ruff-0.14.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:9358d490ec030f1b51d048a7fd6ead418ed0826daf6149e95e30aa67c168af33", size = 11928120, upload-time = "2025-11-06T22:07:08.023Z" }, + { url = "https://files.pythonhosted.org/packages/a3/92/d5f1770e9988cc0742fefaa351e840d9aef04ec24ae1be36f333f96d5704/ruff-0.14.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81b40d27924f1f02dfa827b9c0712a13c0e4b108421665322218fc38caf615c2", size = 12370877, upload-time = "2025-11-06T22:07:10.015Z" }, + { url = "https://files.pythonhosted.org/packages/e2/29/e9282efa55f1973d109faf839a63235575519c8ad278cc87a182a366810e/ruff-0.14.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f5e649052a294fe00818650712083cddc6cc02744afaf37202c65df9ea52efa5", size = 12408538, upload-time = "2025-11-06T22:07:13.085Z" }, + { url = "https://files.pythonhosted.org/packages/8e/01/930ed6ecfce130144b32d77d8d69f5c610e6d23e6857927150adf5d7379a/ruff-0.14.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa082a8f878deeba955531f975881828fd6afd90dfa757c2b0808aadb437136e", size = 13141942, upload-time = "2025-11-06T22:07:15.386Z" }, + { url = "https://files.pythonhosted.org/packages/6a/46/a9c89b42b231a9f487233f17a89cbef9d5acd538d9488687a02ad288fa6b/ruff-0.14.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:1043c6811c2419e39011890f14d0a30470f19d47d197c4858b2787dfa698f6c8", size = 14544306, upload-time = "2025-11-06T22:07:17.631Z" }, + { url = "https://files.pythonhosted.org/packages/78/96/9c6cf86491f2a6d52758b830b89b78c2ae61e8ca66b86bf5a20af73d20e6/ruff-0.14.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a9f3a936ac27fb7c2a93e4f4b943a662775879ac579a433291a6f69428722649", size = 14210427, upload-time = "2025-11-06T22:07:19.832Z" }, + { url = "https://files.pythonhosted.org/packages/71/f4/0666fe7769a54f63e66404e8ff698de1dcde733e12e2fd1c9c6efb689cb5/ruff-0.14.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:95643ffd209ce78bc113266b88fba3d39e0461f0cbc8b55fb92505030fb4a850", size = 13658488, upload-time = "2025-11-06T22:07:22.32Z" }, + { url = "https://files.pythonhosted.org/packages/ee/79/6ad4dda2cfd55e41ac9ed6d73ef9ab9475b1eef69f3a85957210c74ba12c/ruff-0.14.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:456daa2fa1021bc86ca857f43fe29d5d8b3f0e55e9f90c58c317c1dcc2afc7b5", size = 13354908, upload-time = "2025-11-06T22:07:24.347Z" }, + { url = "https://files.pythonhosted.org/packages/b5/60/f0b6990f740bb15c1588601d19d21bcc1bd5de4330a07222041678a8e04f/ruff-0.14.4-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:f911bba769e4a9f51af6e70037bb72b70b45a16db5ce73e1f72aefe6f6d62132", size = 13587803, upload-time = "2025-11-06T22:07:26.327Z" }, + { url = "https://files.pythonhosted.org/packages/c9/da/eaaada586f80068728338e0ef7f29ab3e4a08a692f92eb901a4f06bbff24/ruff-0.14.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:76158a7369b3979fa878612c623a7e5430c18b2fd1c73b214945c2d06337db67", size = 12279654, upload-time = "2025-11-06T22:07:28.46Z" }, + { url = "https://files.pythonhosted.org/packages/66/d4/b1d0e82cf9bf8aed10a6d45be47b3f402730aa2c438164424783ac88c0ed/ruff-0.14.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:f3b8f3b442d2b14c246e7aeca2e75915159e06a3540e2f4bed9f50d062d24469", size = 12357520, upload-time = "2025-11-06T22:07:31.468Z" }, + { url = "https://files.pythonhosted.org/packages/04/f4/53e2b42cc82804617e5c7950b7079d79996c27e99c4652131c6a1100657f/ruff-0.14.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c62da9a06779deecf4d17ed04939ae8b31b517643b26370c3be1d26f3ef7dbde", size = 12719431, upload-time = "2025-11-06T22:07:33.831Z" }, + { url = "https://files.pythonhosted.org/packages/a2/94/80e3d74ed9a72d64e94a7b7706b1c1ebaa315ef2076fd33581f6a1cd2f95/ruff-0.14.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:5a443a83a1506c684e98acb8cb55abaf3ef725078be40237463dae4463366349", size = 13464394, upload-time = "2025-11-06T22:07:35.905Z" }, + { url = "https://files.pythonhosted.org/packages/54/1a/a49f071f04c42345c793d22f6cf5e0920095e286119ee53a64a3a3004825/ruff-0.14.4-py3-none-win32.whl", hash = "sha256:643b69cb63cd996f1fc7229da726d07ac307eae442dd8974dbc7cf22c1e18fff", size = 12493429, upload-time = "2025-11-06T22:07:38.43Z" }, + { url = "https://files.pythonhosted.org/packages/bc/22/e58c43e641145a2b670328fb98bc384e20679b5774258b1e540207580266/ruff-0.14.4-py3-none-win_amd64.whl", hash = "sha256:26673da283b96fe35fa0c939bf8411abec47111644aa9f7cfbd3c573fb125d2c", size = 13635380, upload-time = "2025-11-06T22:07:40.496Z" }, + { url = "https://files.pythonhosted.org/packages/30/bd/4168a751ddbbf43e86544b4de8b5c3b7be8d7167a2a5cb977d274e04f0a1/ruff-0.14.4-py3-none-win_arm64.whl", hash = "sha256:dd09c292479596b0e6fec8cd95c65c3a6dc68e9ad17b8f2382130f87ff6a75bb", size = 12663065, upload-time = "2025-11-06T22:07:42.603Z" }, +] + +[[package]] +name = "setuptools" +version = "80.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" }, +] + +[[package]] +name = "ton" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "pytest" }, + { name = "tontester" }, +] + +[package.dev-dependencies] +dev = [ + { name = "basedpyright" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "pytest", specifier = ">=9.0.1" }, + { name = "tontester", editable = "test/tontester" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "basedpyright", specifier = ">=1.32.1" }, + { name = "ruff", specifier = ">=0.14.2" }, +] + +[[package]] +name = "tontester" +version = "0.1.0" +source = { editable = "test/tontester" } +dependencies = [ + { name = "pytonlib" }, + { name = "setuptools" }, +] + +[package.metadata] +requires-dist = [ + { name = "pytonlib", specifier = ">=0.0.68" }, + { name = "setuptools", specifier = "<81" }, +] + +[[package]] +name = "tvm-valuetypes" +version = "0.0.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "bitarray" }, + { name = "crc32c" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/8c/6e856cf86ab94363333d3e3e1a3754f0c56694fab7a7c8b328493eaef00d/tvm_valuetypes-0.0.12.tar.gz", hash = "sha256:1f9304126be99e30ecb827e68e6e36e1620b8dc6a781dca1b0fdcc1e31d70983", size = 9044, upload-time = "2023-07-18T19:28:37.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/58/d98de1ffd588dcf5cda8c9d105b5949aa66c69cb3af7d0a25bae679536c1/tvm_valuetypes-0.0.12-py3-none-any.whl", hash = "sha256:e4af62baebdcc8864a1f4aac40082f3221f1e4232f79b8dfb7411d7c78856880", size = 9471, upload-time = "2023-07-18T19:28:36.304Z" }, +] + +[[package]] +name = "urllib3" +version = "2.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, +] From b008305db5d85e55118de231293d331024bf936d Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Tue, 18 Nov 2025 01:16:15 -0500 Subject: [PATCH 481/524] Update tl-parser (#1894) Fixes #1882. --- third-party/tl-parser | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third-party/tl-parser b/third-party/tl-parser index aba8925886..f50db8ea01 160000 --- a/third-party/tl-parser +++ b/third-party/tl-parser @@ -1 +1 @@ -Subproject commit aba8925886a599b439f0a9a919abdaa363b8cb2e +Subproject commit f50db8ea016ebaf1654b995f8f02e54b673fbae2 From b083624bd0121fd710b305d294120b655c9bb572 Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Tue, 18 Nov 2025 01:20:28 -0500 Subject: [PATCH 482/524] Some standalone fixes from new consensus dev branch (#1895) * Compile libraries in third-party/ with sanitizers as well This is not a complete solution (with sanitizers, you ideally need a build-the-world-with-sanitizers approach) but is good enough for abseil to stop producing memory errors. * First reset td::Promise and then execute callback Callback can override promise inplace, so doing it the other way can reset a newly created PromiseInterface. * [tontester] Allow to run node under `rr record` --- CMakeLists.txt | 27 ++++++++-------- tdactor/td/actor/PromiseFuture.h | 16 ++++----- test/tontester/src/tontester/network.py | 43 +++++++++++++++++++------ 3 files changed, 55 insertions(+), 31 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 556c8c3562..86a58b0fa0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,6 +99,20 @@ if (TON_WERROR_BUILD) add_compile_options(-Werror) endif() +if (TON_USE_ASAN) + add_compile_options(-fsanitize=address) + add_link_options(-fsanitize=address) + add_definitions(-DTD_USE_ASAN=1) +endif() +if (TON_USE_TSAN) + add_compile_options(-fsanitize=thread) + add_link_options(-fsanitize=thread) +endif() +if (TON_USE_UBSAN) + add_compile_options(-fsanitize=undefined) + add_link_options(-fsanitize=undefined) +endif() + if (TON_USE_ABSEIL) message("Add abseil-cpp") function(abseil_scope) @@ -338,19 +352,6 @@ if (GCC OR CLANG) endif() endif() -if (TON_USE_ASAN) - add_compile_options(-fsanitize=address) - add_link_options(-fsanitize=address) - add_definitions(-DTD_USE_ASAN=1) -endif() -if (TON_USE_TSAN) - add_compile_options(-fsanitize=thread) - add_link_options(-fsanitize=thread) -endif() -if (TON_USE_UBSAN) - add_compile_options(-fsanitize=undefined) - add_link_options(-fsanitize=undefined) -endif() if (TON_USE_COVERAGE) add_cxx_compiler_flag("-fprofile-arcs") add_cxx_compiler_flag("-ftest-coverage") diff --git a/tdactor/td/actor/PromiseFuture.h b/tdactor/td/actor/PromiseFuture.h index 568e824264..3624985f5c 100644 --- a/tdactor/td/actor/PromiseFuture.h +++ b/tdactor/td/actor/PromiseFuture.h @@ -232,31 +232,31 @@ class Promise { if (!promise_) { return; } - promise_->set_value(std::move(value)); - promise_.reset(); + auto promise = std::exchange(promise_, nullptr); + promise->set_value(std::move(value)); } void set_error(Status &&error) { if (!promise_) { return; } - promise_->set_error(std::move(error)); - promise_.reset(); + auto promise = std::exchange(promise_, nullptr); + promise->set_error(std::move(error)); } void set_result(Result &&result) { if (!promise_) { return; } - promise_->set_result(std::move(result)); - promise_.reset(); + auto promise = std::exchange(promise_, nullptr); + promise->set_result(std::move(result)); } template void operator()(S &&result) { if (!promise_) { return; } - promise_->operator()(std::forward(result)); - promise_.reset(); + auto promise = std::exchange(promise_, nullptr); + promise->operator()(std::forward(result)); } void reset() { promise_.reset(); diff --git a/test/tontester/src/tontester/network.py b/test/tontester/src/tontester/network.py index 90f1a6fa13..6295c5bd6c 100644 --- a/test/tontester/src/tontester/network.py +++ b/test/tontester/src/tontester/network.py @@ -8,7 +8,7 @@ from enum import IntEnum, auto from ipaddress import IPv4Address from pathlib import Path -from typing import cast, final, override +from typing import Literal, cast, final, override from pytonlib import TonlibClient, TonlibError # pyright: ignore[reportMissingTypeStubs] @@ -39,6 +39,9 @@ def _write_model(file: Path, model: TLObject): _ = file.write_text(model.to_json()) +type DebugType = None | Literal["rr"] + + @final class Network: class Node(ABC): @@ -88,6 +91,8 @@ async def _run( local_config: ton_api.Engine_validator_config, validator_config: ton_api.Validator_config_global | None, additional_args: list[str], + *, + debug: DebugType = None, ): async def process_watcher(): assert self.__process is not None @@ -121,8 +126,7 @@ async def process_watcher(): log_path = self._directory / "log" l.info(f"Running {self.name} and saving its raw log to {log_path}") - self.__process = await asyncio.create_subprocess_exec( - executable, + cmd_flags = [ "--global-config", global_config_file, "--local-config", @@ -130,9 +134,27 @@ async def process_watcher(): "--db", ".", *additional_args, - cwd=self._directory, - stderr=asyncio.subprocess.PIPE, - ) + ] + + match debug: + case None: + self.__process = await asyncio.create_subprocess_exec( + executable, + *cmd_flags, + cwd=self._directory, + stderr=asyncio.subprocess.PIPE, + ) + case "rr": + l.info(f"Recording {self.name} with rr") + self.__process = await asyncio.create_subprocess_exec( + "rr", + "record", + executable, + *cmd_flags, + cwd=self._directory, + stderr=asyncio.subprocess.PIPE, + ) + assert self.__process.stderr is not None # to placate pyright self.__process_watcher = asyncio.create_task(process_watcher()) @@ -146,7 +168,7 @@ def announce_to(self, dht: "DHTNode"): self._static_nodes.append(dht) @abstractmethod - async def run(self): + async def run(self, *, debug: DebugType = None): pass async def stop(self): @@ -321,8 +343,8 @@ def signed_address(self): return self._signed_address @override - async def run(self): - await self._run(self._install.dht_server_exe, self._local_config, None, []) + async def run(self, *, debug: DebugType = None): + await self._run(self._install.dht_server_exe, self._local_config, None, [], debug=debug) @final @@ -399,7 +421,7 @@ def validator_key(self): return self._validator_key @override - async def run(self): + async def run(self, *, debug: DebugType = None): zerostate = self._get_or_generate_zerostate() static_dir = self._directory / "static" @@ -412,6 +434,7 @@ async def run(self): self._local_config, zerostate.as_validator_config(), ["--initial-sync-delay", "5"], + debug=debug, ) async def tonlib_client(self) -> TonlibClient: From d0139023b0a84a9d6ea95f38411fd00440a97170 Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Tue, 18 Nov 2025 01:21:32 -0500 Subject: [PATCH 483/524] Remove file accidentally committed in #1877 (#1893) --- validator-engine/Runtime.h | 542 ------------------------------------- 1 file changed, 542 deletions(-) delete mode 100644 validator-engine/Runtime.h diff --git a/validator-engine/Runtime.h b/validator-engine/Runtime.h deleted file mode 100644 index ecdee70e67..0000000000 --- a/validator-engine/Runtime.h +++ /dev/null @@ -1,542 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -#include "td/actor/actor.h" -#include "td/utils/Badge.h" -#include "td/utils/buffer.h" -#include "td/utils/logging.h" - -#include "TypeRegistry.h" -#include "TypeUtils.h" - -namespace ton::runtime { - -namespace detail { - -class BusListeningActor; -class Runtime; - -struct Bus { - virtual ~Bus() = default; -}; - -template -concept BusType = std::derived_from && IsSpecializationOf; - -template -concept BusWithParent = requires { typename B::Parent; }; - -template -struct ValidEventHelper { - static constexpr bool value = In; -}; - -template -struct ValidEventHelper { - static constexpr bool value = ValidEventHelper::value || In; -}; - -template -concept ValidEventFor = ValidEventHelper::value; - -struct BusIdTag {}; -using BusTypeId = ton::runtime::IdType; - -template -BusTypeId get_bus_id() { - return get_type_id(); -} - -template -class BusHandle; - -template -class SpawnsWith; - -template -class ArmedEvent { - public: - explicit ArmedEvent(std::shared_ptr event) : event_(std::move(event)) { - } - - ArmedEvent(ArmedEvent&&) = default; - ArmedEvent& operator=(ArmedEvent&&) = default; - - ~ArmedEvent() { - LOG_CHECK(!event_) << "Event was lost because actor stopped too early. YOU HAVE A RACE."; - } - - std::shared_ptr disarm() { - return std::move(event_); - } - - private: - std::shared_ptr event_; -}; - -template -class BusEventPublishImpl { - public: - size_t publish(std::shared_ptr event, BusHandle handle) { - for (auto const& [actor, dispatcher_fn] : dispatchers) { - td::actor::send_closure(actor, dispatcher_fn, handle, ArmedEvent(event)); - } - return dispatchers.size(); - } - - private: - friend class Runtime; - - struct EventDispatcher { - using EventDispatcherFn = void (BusListeningActor::*)(BusHandle bus, ArmedEvent event); - - td::actor::ActorId actor; - EventDispatcherFn dispatcher_fn; - }; - - // Can only have actors that are owned by (non-strict) ancestor of the current bus. - std::vector dispatchers; -}; - -template -struct BusPublishImpl; - -template -using BusImpl = BusPublishImpl; - -template -struct BusPublishImpl> : BusEventPublishImpl... { - using BusEventPublishImpl::publish...; -}; - -template -struct BusPublishImpl> : BusImpl, BusEventPublishImpl... { - using BusImpl::publish; - using BusEventPublishImpl::publish...; -}; - -struct BusTreeNode { - template - BusTreeNode(Runtime& runtime, std::string actor_name_prefix, std::shared_ptr bus) - : runtime(runtime) - , actor_name_prefix(actor_name_prefix) - , type_id(get_bus_id()) - , bus(std::move(bus)) - , bus_impl(std::make_shared>()) { - } - - Runtime& runtime; - std::string actor_name_prefix; - BusTypeId type_id; - std::shared_ptr bus; - std::shared_ptr bus_impl; - - std::atomic is_stopping = false; - - std::shared_ptr parent; - std::vector> owned_actors; -}; - -void log_published_event(BusTreeNode const& bus, std::type_info const& event_name); -void log_received_event(BusTreeNode const& bus, std::type_info const& event_name); - -// A ref-counted nullable pointer of bus B. -template -class BusHandle { - public: - BusHandle() = default; - BusHandle(BusHandle const&) = default; - BusHandle(BusHandle&&) = default; - BusHandle& operator=(BusHandle const&) = default; - BusHandle& operator=(BusHandle&&) = default; - - BusHandle(std::nullptr_t) : BusHandle() { - } - - template - requires std::derived_from - BusHandle(BusHandle const& handle) - : node_(handle.node_) - , bus_(std::static_pointer_cast(handle.bus_)) - , impl_(std::static_pointer_cast>(handle.impl_)) { - } - - // This is technically not constant but we give BusHandle const& to user code. - template E> - size_t publish(std::shared_ptr event) const { - CHECK(*this); - - log_published_event(*node_, typeid(E)); - LOG_CHECK(!node_->is_stopping.load(std::memory_order_relaxed)) - << "Event was published to a degraded bus. THIS IS A BEST-EFFORT WARNING, YOU HAVE A RACE."; - - return impl_->publish(std::move(event), *this); - } - - template - BusHandle create_child(std::string_view name, std::shared_ptr bus) const { - CHECK(*this); - - auto name_prefix = node_->actor_name_prefix + std::string(name) + "."; - auto child = std::make_shared(node_->runtime, name_prefix, std::move(bus)); - child->parent = node_; - child->type_id = get_bus_id(); - - // Now, we just call `node_->runtime._wire_bus({}, child);` but since Runtime is not defined - // yet, we need a forward-declared method to do this for us. - wire_bus(child); - - BusHandle child_handle(child, std::static_pointer_cast(child->bus), - std::static_pointer_cast>(child->bus_impl)); - return child_handle; - } - - void stop() const { - CHECK(*this); - // We let runtime do the work with (again, forward-declared) `node_->runtime._stop_bus(node_)`. - stop_bus(node_); - } - - template - requires std::derived_from - BusHandle unsafe_static_downcast_to() const { - CHECK(*this); - - return BusHandle(node_, std::static_pointer_cast(bus_), std::static_pointer_cast>(impl_)); - } - - B const* operator->() const& { - CHECK(*this); - - return bus_.get(); - } - - explicit operator bool() const { - return static_cast(bus_); - } - - BusHandle(td::Badge, std::shared_ptr node, std::shared_ptr bus, - std::shared_ptr> impl) - : node_(std::move(node)), bus_(std::move(bus)), impl_(std::move(impl)) { - } - - template > T> - BusTreeNode const& _node(td::Badge) const { - return *node_; - } - - private: - template - friend class BusHandle; - - BusHandle(std::shared_ptr node, std::shared_ptr bus, std::shared_ptr> impl) - : node_(std::move(node)), bus_(std::move(bus)), impl_(std::move(impl)) { - } - - static void wire_bus(std::shared_ptr node); - static void stop_bus(std::shared_ptr node); - - std::shared_ptr node_; - std::shared_ptr bus_; - std::shared_ptr> impl_; -}; - -// Base class for all actors that handle bus events. -// -// Actor-type-erased `dispatch_event' method dispatches event to an actual handler. It is called -// from BusEventPublishImpl::publish and -// -// install_listeners_at implementation is provided by Runtime::BusListeningActorImpl. -class BusListeningActor : public td::actor::Actor { - private: - friend class Runtime; - - template - void dispatch_event(BusHandle bus, ArmedEvent event) { - log_received_event(bus._node(td::Badge{}), typeid(E)); - if constexpr (!std::same_as) { - // When we install listeners, we guarantee that actual bus type is at most B. - static_cast(this)->template handle(bus.template unsafe_static_downcast_to(), event.disarm()); - } else { - static_cast(this)->template handle(std::move(bus), event.disarm()); - } - } - - void stop_with_bus() { - this->td::actor::Actor::stop(); - } - - virtual void install_listeners_at(BusTypeId new_bus_type, std::shared_ptr const& node, - td::actor::ActorId const& actor_id) = 0; - - td::BufferSlice name_; -}; - -// Base class for all user-created bus listeners. Template argument specifies which bus the actor -// spawns with. Note that you have to register actor using Runtime::register_actor for an actor to -// actually be created with the bus. -template -class SpawnsWith : public BusListeningActor { - public: - using SpawnWithBus = B; - - ~SpawnsWith() { - LOG_CHECK(owning_bus_._node(td::Badge>{}).is_stopping.load(std::memory_order_relaxed)) - << "Bus actors must not call stop() manually"; - } - - protected: - BusHandle const& owning_bus() { - CHECK(owning_bus_); - return owning_bus_; - } - - private: - friend class Runtime; - - BusHandle owning_bus_; -}; - -// Bus listeners must also inherit ConnectsTo to specify a list of buses they subscribe to. -template -struct ConnectsTo { - using ConnectToBuses = TypeList; -}; - -template -concept ActorType = std::derived_from> && - IsSpecializationOf; - -template -concept CanActorHandleEvent = - requires(A& actor, BusHandle bus, std::shared_ptr event) { actor.template handle(bus, event); }; - -class Runtime { - public: - template - void register_actor(std::string_view name) { - LOG_CHECK(!started_) << "Actors can only be registered before starting runtime"; - - using B = typename A::SpawnWithBus; - BusTypeId spawn_bus_id = get_bus_id(); - register_bus_parents(spawn_bus_id); - - // Add actor to actors_to_spawn_for_ for bus to know which actors to spawn on creation. - CreateInstanceFn create_instance_fn = &Runtime::create_actor_instance; - actors_to_spawn_for_[spawn_bus_id].push_back({ - .create_instance_fn = create_instance_fn, - .name = std::string(name), - }); - } - - template - void set_root_bus(std::shared_ptr bus) { - LOG_CHECK(!started_) << "Root bus can only be set before starting runtime"; - - root_bus_ = std::make_shared(*this, "", std::move(bus)); - } - - void start() { - LOG_CHECK(!started_) << "Runtime::start must not be called twice"; - LOG_CHECK(root_bus_) << "Root bus is not set"; - started_ = true; - - CHECK(root_bus_->bus_impl); - wire_bus(std::move(root_bus_)); - } - - template - void _wire_bus(td::Badge>, std::shared_ptr node) { - wire_bus(node); - } - - template - void _stop_bus(td::Badge>, std::shared_ptr node) { - node->is_stopping.store(true, std::memory_order_relaxed); - for (auto const& owned_actor : node->owned_actors) { - td::actor::send_closure(owned_actor, &BusListeningActor::stop_with_bus); - } - } - - private: - bool started_ = false; - - std::shared_ptr root_bus_; - - // ===== Bus inheritance ===== - template - void register_bus_parents(BusTypeId id) { - if (bus_parents_.contains(id)) { - return; - } - - if constexpr (requires { typename B::Parent; }) { - auto parent_id = get_bus_id(); - bus_parents_[id] = parent_id; - register_bus_parents(parent_id); - } else { - bus_parents_[id] = std::nullopt; - } - } - - std::map> bus_parents_; - - // ===== Actor creation for a particular bus ===== - template - class BusListeningActorImpl final : public A { - public: - struct Params { - BusListeningActorImpl& self; - BusTypeId new_bus_type; - std::shared_ptr const& node; - td::actor::ActorId const& actor_id; - }; - - virtual void install_listeners_at(BusTypeId new_bus_type, std::shared_ptr const& node, - td::actor::ActorId const& actor_id) override { - // What we are doing with ActorRegistrar is effectively: - // for (auto B : A::ConnectToBuses) { - // for (auto (BOrigin, E) : B::Events) { - // register_event_listener(node); - // } - // } - Registrar{}({*this, new_bus_type, node, actor_id}); - } - - template - struct Registrar {}; - - template - struct Registrar> { - void operator()(Params params) { - (EventRegistrar{}(params), ...); - } - }; - - template - struct EventRegistrar {}; - - template - struct EventRegistrar> { - void operator()(Params params) { - (params.self.template register_event_listener(params), ...); - if constexpr (BusWithParent) { - using Parent = BOrigin::Parent; - EventRegistrar{}(params); - } - } - }; - - template - void register_event_listener(Params params) { - if constexpr (CanActorHandleEvent) { - // handle accepts (BusHandle, std::shared_ptr) while we treat newly installed - // bus (of type BNew) as having type new_bus_type (one of types in its inheritance chain). - if (get_bus_id() == params.new_bus_type) { - // node->bus_impl is BusImpl. Since BNew <= B == new_bus_type, it is safe to - // downcast node->bus_impl to BusImpl. - auto bus_impl = - static_cast*>(static_cast*>(params.node->bus_impl.get())); - - bus_impl->dispatchers.push_back({ - .actor = params.actor_id, - .dispatcher_fn = &BusListeningActor::dispatch_event, - }); - } - } - } - }; - - using CreateInstanceFn = std::unique_ptr (Runtime::*)(std::shared_ptr node); - - struct ActorSpawnInfo { - CreateInstanceFn create_instance_fn; - std::string name; - }; - - template - std::unique_ptr create_actor_instance(std::shared_ptr node) { - auto instance = std::make_unique>(); - auto bus = std::static_pointer_cast(node->bus); - auto bus_impl = std::static_pointer_cast>(node->bus_impl); - instance->owning_bus_ = BusHandle({}, std::move(node), bus, bus_impl); - return instance; - } - - std::map> actors_to_spawn_for_; - - // ===== Event wiring ===== - void wire_bus(std::shared_ptr node) { - CHECK(node); - - // First, we create all actors that spawn on the added bus ... - std::vector> spawned_actors; - std::vector bus_inheritance_chain; - - for (std::optional bus_type = node->type_id; bus_type; bus_type = bus_parents_[*bus_type]) { - bus_inheritance_chain.push_back(*bus_type); - - for (auto const& [create_instance_fn, base_name] : actors_to_spawn_for_[*bus_type]) { - auto instance = (this->*create_instance_fn)(node); - std::string name = node->actor_name_prefix + base_name; - instance->name_ = td::BufferSlice(name.data(), name.size()); - auto actor_info = - td::actor::detail::create_actor_info(td::actor::ActorOptions{}.with_name(name), std::move(instance)); - node->owned_actors.push_back(td::actor::ActorId::unsafe_create_from_info(actor_info)); - } - } - - // Then, we wire events from the bus to all actors that are subscribed to it. Note that we have - // not returned BusHandle of a newly created bus to outside world yet, so the outside world - // cannot publish events on the bus, and actors that have the current bus injected as a owning - // bus are not started yet. - auto it = node; - while (it) { - LOG_CHECK(!it->is_stopping.load(std::memory_order_relaxed)) - << "Race between child and parent buses detected. THIS IS A BEST-EFFORT WARNING, YOU HAVE A RACE."; - - for (auto const& owned_actor : it->owned_actors) { - for (auto new_bus_type : bus_inheritance_chain) { - owned_actor.get_actor_unsafe().install_listeners_at(new_bus_type, node, owned_actor); - } - } - it = it->parent; - } - - // And lastly, we start all newly created actors - for (auto const& actor : node->owned_actors) { - td::actor::detail::register_actor_info_ptr(actor.actor_info_ptr()); - } - } -}; - -template -void BusHandle::wire_bus(std::shared_ptr node) { - node->runtime._wire_bus({}, std::move(node)); -} - -template -void BusHandle::stop_bus(std::shared_ptr node) { - node->runtime._stop_bus({}, std::move(node)); -} - -} // namespace detail - -using detail::Bus; -using detail::BusHandle; -using detail::BusType; -using detail::ConnectsTo; -using detail::Runtime; -using detail::SpawnsWith; - -#define TON_RUNTIME_DEFINE_EVENT_HANDLER() \ - template <::ton::runtime::In B, typename E> \ - requires ::ton::runtime::detail::ValidEventFor \ - constexpr void handle(::ton::runtime::BusHandle bus, ::std::shared_ptr event) = delete; - -} // namespace ton::runtime \ No newline at end of file From 5cfc47cd55ca8a89fbe8aceb8d06c64c6968e7cd Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Wed, 19 Nov 2025 01:06:30 -0500 Subject: [PATCH 484/524] Add enough LSAN suppressions for project to compile with TON_USE_ASAN=1 (#1897) Note that most tests don't work under UBSAN+ASAN for various reasons. --- crypto/block/create-state.cpp | 4 ++++ crypto/fift/fift-main.cpp | 4 ++++ crypto/func/func-main.cpp | 4 ++++ tl/generate/CMakeLists.txt | 4 +++- 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/crypto/block/create-state.cpp b/crypto/block/create-state.cpp index 1fcdde8850..fe691eb8a8 100644 --- a/crypto/block/create-state.cpp +++ b/crypto/block/create-state.cpp @@ -80,6 +80,10 @@ using td::Ref; +extern "C" const char* __asan_default_options() { + return "detect_leaks=0"; +} + int verbosity; enum { wc_master = -1, wc_base = 0 }; diff --git a/crypto/fift/fift-main.cpp b/crypto/fift/fift-main.cpp index 953b3266da..7cbcab3bce 100644 --- a/crypto/fift/fift-main.cpp +++ b/crypto/fift/fift-main.cpp @@ -52,6 +52,10 @@ #include "git.h" #include "words.h" +extern "C" const char* __asan_default_options() { + return "detect_leaks=0"; +} + void usage(const char* progname) { std::cerr << "A simple Fift interpreter. Type `bye` to quit, or `words` to get a list of all commands\n"; std::cerr diff --git a/crypto/func/func-main.cpp b/crypto/func/func-main.cpp index f7e372ef2a..1ee9247775 100644 --- a/crypto/func/func-main.cpp +++ b/crypto/func/func-main.cpp @@ -35,6 +35,10 @@ #include "func.h" #include "git.h" +extern "C" const char* __asan_default_options() { + return "detect_leaks=0"; +} + void usage(const char* progname) { std::cerr << "usage: " << progname diff --git a/tl/generate/CMakeLists.txt b/tl/generate/CMakeLists.txt index 4dac089e2d..515c32be54 100644 --- a/tl/generate/CMakeLists.txt +++ b/tl/generate/CMakeLists.txt @@ -63,7 +63,9 @@ if (NOT CMAKE_CROSSCOMPILING) foreach(tl_scheme lite_api ton_api tonlib_api) add_custom_command( WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/scheme - COMMAND $ -e "${tl_scheme}.tlo" "${tl_scheme}.tl" + COMMAND + ${CMAKE_COMMAND} -E env ASAN_OPTIONS=detect_leaks=0 + $ -e "${tl_scheme}.tlo" "${tl_scheme}.tl" DEPENDS tl-parser "scheme/${tl_scheme}.tl" OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/scheme/${tl_scheme}.tlo" ) From 9e2c34337dbfb69a32de3f3662c1c8161e5f8cc0 Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Wed, 19 Nov 2025 01:07:02 -0500 Subject: [PATCH 485/524] Switch last remaining MacOS 13 pipeline to MacOS 15 (#1898) This causes CI failures otherwise: https://github.com/ton-blockchain/ton/actions/runs/19470205113/job/55715079514?pr=1897 --- ...s-x86-64-shared.yml => build-ton-macos-15-x86-64-shared.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{build-ton-macos-x86-64-shared.yml => build-ton-macos-15-x86-64-shared.yml} (97%) diff --git a/.github/workflows/build-ton-macos-x86-64-shared.yml b/.github/workflows/build-ton-macos-15-x86-64-shared.yml similarity index 97% rename from .github/workflows/build-ton-macos-x86-64-shared.yml rename to .github/workflows/build-ton-macos-15-x86-64-shared.yml index 9ac8c54460..506b356e85 100644 --- a/.github/workflows/build-ton-macos-x86-64-shared.yml +++ b/.github/workflows/build-ton-macos-15-x86-64-shared.yml @@ -4,7 +4,7 @@ on: [push, pull_request, workflow_dispatch, workflow_call] jobs: build: - runs-on: macos-13 + runs-on: macos-15-intel steps: - name: Check out repository From 2fa29b638e55c42820eec46fe8c8f880181faaea Mon Sep 17 00:00:00 2001 From: neodix42 Date: Thu, 6 Nov 2025 22:12:19 +0400 Subject: [PATCH 486/524] Automate Github Actions cache clean up (#1870) * upgrade abseil to August 2025, Patch 1 * add gh action to clear all gh actions cache * add gh action to clear all gh actions cache --- .github/workflows/clear-gh-cache.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/workflows/clear-gh-cache.yml diff --git a/.github/workflows/clear-gh-cache.yml b/.github/workflows/clear-gh-cache.yml new file mode 100644 index 0000000000..3e166be656 --- /dev/null +++ b/.github/workflows/clear-gh-cache.yml @@ -0,0 +1,13 @@ +name: Clear all GH cache + +on: + workflow_dispatch + +jobs: + clear-cache: + env: + GH_TOKEN: ${{ github.token }} + runs-on: ubuntu-latest + steps: + - name: Clear all cache + run: gh cache delete --all From 281d3fc035bdf3f35c5440af7426af39bee54bc2 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Fri, 21 Nov 2025 20:26:04 +0300 Subject: [PATCH 487/524] Add codex auto review (#13) * upgrade abseil to August 2025, Patch 1 * add gh action to clear all gh actions cache * add gh action to clear all gh actions cache * migrate gh action macos-13 to macos-15 x86. * need repo * automatically add ""@codex review" comment to PRs * add more permissions * add more permissions to codex review job * trigger automatic codex review only on specified gh users * trigger automatic codex review when the review is submitted * add more permissions to codex review job * trigger automatic codex review only on specified gh users * trigger automatic codex review when the review is submitted * trigger automatic codex review when the review is submitted * rework codex auto review * rework codex auto review * rework codex auto review for PRs from forks * rework codex auto review for PRs from forks * fix based on a review * fix based on a review * restore codex auto review without openai api key, remove handling of pull_request_review events * adjust to work on fork PRs too --------- Co-authored-by: neodix --- .github/workflows/clear-gh-cache.yml | 2 ++ .github/workflows/codex-review.yml | 45 ++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 .github/workflows/codex-review.yml diff --git a/.github/workflows/clear-gh-cache.yml b/.github/workflows/clear-gh-cache.yml index 3e166be656..90140ecdf4 100644 --- a/.github/workflows/clear-gh-cache.yml +++ b/.github/workflows/clear-gh-cache.yml @@ -9,5 +9,7 @@ jobs: GH_TOKEN: ${{ github.token }} runs-on: ubuntu-latest steps: + - name: Check out repository + uses: actions/checkout@v3 - name: Clear all cache run: gh cache delete --all diff --git a/.github/workflows/codex-review.yml b/.github/workflows/codex-review.yml new file mode 100644 index 0000000000..1806566336 --- /dev/null +++ b/.github/workflows/codex-review.yml @@ -0,0 +1,45 @@ +name: Codex auto review + +on: + pull_request_target: + types: [opened, reopened, synchronize] + +permissions: + contents: read + issues: write + pull-requests: write + +jobs: + request-codex-review: + if: | + github.event.pull_request.user.login == 'EmelyanenkoK' || + github.event.pull_request.user.login == 'tolya-yanot' || + github.event.pull_request.user.login == 'SpyCheese' || + github.event.pull_request.user.login == 'neodix42' || + github.event.pull_request.user.login == 'dungeon-master-666' || + github.event.pull_request.user.login == 'igroman787' || + github.event.pull_request.user.login == 'kdimentionaltree' || + github.event.pull_request.user.login == 'sonofmom' || + github.event.pull_request.user.login == 'Trinketer22' || + github.event.pull_request.user.login == 'xssnick' || + github.event.pull_request.user.login == 'tolk-vm' || + github.event.pull_request.user.login == 'DanShaders' || + github.event.pull_request.user.login == 'birydrad' || + github.event.pull_request.user.login == 'abacabadabacaba' || + github.event.pull_request.user.login == 'Mustang98' || + github.event.pull_request.user.login == 'avevad' || + github.event.pull_request.user.login == 'tvorogme' || + github.event.pull_request.user.login == 'krigga' + runs-on: ubuntu-latest + steps: + - name: Comment to trigger Codex + uses: actions/github-script@v7 + with: + github-token: ${{ github.token }} + script: | + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: "@codex review" + }) From 64b8e65b49668dfda78a787145848a1c084e1bf2 Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Mon, 24 Nov 2025 12:31:29 -0500 Subject: [PATCH 488/524] Test infra improvements (#1915) * Do not manually enumerate tests in each and every CI build script * Refactor test expectation macros to output failing line in logs Previously, logs first recorded a random source line and then message contained the line where the expectation failed. * Add non-asserting test expectation macros These fail test instead of killing the process. * Print test results in the nicer format by default Let's reserve LOG(ERROR) for errors. Unfortunately, we cannot use this everywhere as test runner is also used in TonlibClient, where custom log redirections can be set up. --- CMakeLists.txt | 50 +++++++------ assembly/native/build-macos-portable.sh | 5 +- assembly/native/build-macos-shared.sh | 5 +- assembly/native/build-ubuntu-appimages.sh | 5 +- assembly/native/build-ubuntu-portable.sh | 5 +- assembly/native/build-ubuntu-shared.sh | 5 +- assembly/native/build-windows-2019.bat | 4 +- assembly/native/build-windows-2022.bat | 4 +- assembly/native/build-windows.bat | 4 +- tdactor/test/CMakeLists.txt | 2 + tdutils/td/utils/tests.cpp | 55 +++++++++++++- tdutils/td/utils/tests.h | 90 ++++++++++++++++++++--- test/test-td-main.cpp | 3 +- 13 files changed, 170 insertions(+), 67 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 86a58b0fa0..fb4b193d89 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -401,6 +401,13 @@ function(target_link_libraries_system target) endforeach(lib) endfunction(target_link_libraries_system) +add_custom_target(all-tests) + +function(ton_test TEST_NAME) + add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME} ${ARGN}) + add_dependencies(all-tests ${TEST_NAME}) +endfunction() + set(TDUTILS_MIME_TYPE OFF CACHE BOOL "Generate mime type conversion") add_subdirectory(tdutils) add_subdirectory(memprof) @@ -575,20 +582,18 @@ endif() enable_testing() set(TEST_OPTIONS "--regression ${CMAKE_CURRENT_SOURCE_DIR}/test/regression-tests.ans --filter -Bench") separate_arguments(TEST_OPTIONS) -add_test(test-ed25519 test-ed25519) -add_test(test-bigint test-bigint) -add_test(test-vm test-vm ${TEST_OPTIONS}) -add_test(test-fift test-fift ${TEST_OPTIONS}) -add_test(test-cells test-cells ${TEST_OPTIONS}) -add_test(test-smartcont test-smartcont) -add_test(test-net test-net) -add_test(test-actors test-tdactor) -add_test(test-actors-coro tdactor/test/test-coro) -add_test(test-emulator test-emulator) +ton_test(test-ed25519) +ton_test(test-bigint) +ton_test(test-vm ${TEST_OPTIONS}) +ton_test(test-fift ${TEST_OPTIONS}) +ton_test(test-cells ${TEST_OPTIONS}) +ton_test(test-smartcont) +ton_test(test-net) +ton_test(test-tdactor) #BEGIN tonlib -add_test(test-tdutils test-tdutils) -add_test(test-tonlib-offline test-tonlib-offline) +ton_test(test-tdutils) +ton_test(test-tonlib-offline) #END tonlib # FunC tests @@ -657,15 +662,16 @@ endif() #BEGIN internal if (NOT TON_ONLY_TONLIB) -add_test(test-adnl test-adnl) -add_test(test-dht test-dht) -add_test(test-rldp test-rldp) -add_test(test-rldp2 test-rldp2) -add_test(test-validator-session-state test-validator-session-state) -add_test(test-catchain test-catchain) - -add_test(test-fec test-fec) -add_test(test-tddb test-tddb ${TEST_OPTIONS}) -add_test(test-db test-db ${TEST_OPTIONS}) +ton_test(test-adnl) +ton_test(test-dht) +ton_test(test-rldp) +ton_test(test-rldp2) +ton_test(test-validator-session-state) +ton_test(test-catchain) +ton_test(test-emulator) + +ton_test(test-fec) +ton_test(test-tddb ${TEST_OPTIONS}) +ton_test(test-db ${TEST_OPTIONS}) endif() #END internal diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index 39faa46af7..870ae73849 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -153,10 +153,7 @@ if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \ - http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ - test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont \ - test-net test-tdactor test-coro test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ - test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver + http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver all-tests test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli blockchain-explorer \ diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index dcba872cfa..13e6208345 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -91,10 +91,7 @@ if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \ - http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ - test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont \ - test-net test-tdactor test-coro test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ - test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver + http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver all-tests test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli blockchain-explorer \ diff --git a/assembly/native/build-ubuntu-appimages.sh b/assembly/native/build-ubuntu-appimages.sh index 0384714684..ef54f06321 100644 --- a/assembly/native/build-ubuntu-appimages.sh +++ b/assembly/native/build-ubuntu-appimages.sh @@ -62,10 +62,7 @@ if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy dht-ping-servers dht-resolve \ - adnl-proxy create-state emulator test-ed25519 test-bigint \ - test-vm test-fift test-cells test-smartcont test-net test-tdactor test-coro test-tdutils \ - test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ - test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver + adnl-proxy create-state emulator proxy-liteserver all-tests test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh index c384c9c421..49b2863c30 100644 --- a/assembly/native/build-ubuntu-portable.sh +++ b/assembly/native/build-ubuntu-portable.sh @@ -136,10 +136,7 @@ if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy dht-ping-servers dht-resolve \ - adnl-proxy create-state emulator test-ed25519 test-bigint \ - test-vm test-fift test-cells test-smartcont test-net test-tdactor test-coro test-tdutils \ - test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ - test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver + adnl-proxy create-state emulator proxy-liteserver all-tests test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index c866168cc1..22756b68d7 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -65,10 +65,7 @@ if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy dht-ping-servers dht-resolve \ - adnl-proxy create-state emulator test-ed25519 test-bigint \ - test-vm test-fift test-cells test-smartcont test-net test-tdactor test-coro test-tdutils \ - test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ - test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver + adnl-proxy create-state emulator proxy-liteserver all-tests test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ diff --git a/assembly/native/build-windows-2019.bat b/assembly/native/build-windows-2019.bat index 1bc8bc68d4..c8dedbf200 100644 --- a/assembly/native/build-windows-2019.bat +++ b/assembly/native/build-windows-2019.bat @@ -147,9 +147,7 @@ IF "%1"=="-t" ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ -test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^ -test-tdactor test-coro test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ -test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver dht-ping-servers dht-resolve +proxy-liteserver dht-ping-servers dht-resolve all-tests IF %errorlevel% NEQ 0 ( echo Can't compile TON exit /b %errorlevel% diff --git a/assembly/native/build-windows-2022.bat b/assembly/native/build-windows-2022.bat index 88236c7789..bfcf3089e5 100644 --- a/assembly/native/build-windows-2022.bat +++ b/assembly/native/build-windows-2022.bat @@ -147,9 +147,7 @@ IF "%1"=="-t" ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ -test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^ -test-tdactor test-coro test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ -test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver +proxy-liteserver all-tests IF %errorlevel% NEQ 0 ( echo Can't compile TON exit /b %errorlevel% diff --git a/assembly/native/build-windows.bat b/assembly/native/build-windows.bat index 0f5db3019d..4612affe32 100644 --- a/assembly/native/build-windows.bat +++ b/assembly/native/build-windows.bat @@ -147,9 +147,7 @@ IF "%1"=="-t" ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ -test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^ -test-tdactor test-coro test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ -test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver dht-ping-servers dht-resolve +proxy-liteserver dht-ping-servers dht-resolve all-tests IF %errorlevel% NEQ 0 ( echo Can't compile TON exit /b %errorlevel% diff --git a/tdactor/test/CMakeLists.txt b/tdactor/test/CMakeLists.txt index 17a71612c6..6e1fafa29f 100644 --- a/tdactor/test/CMakeLists.txt +++ b/tdactor/test/CMakeLists.txt @@ -1,3 +1,5 @@ add_executable(test-coro test-coro.cpp) target_link_libraries(test-coro tdactor tdutils) target_include_directories(test-coro PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../) + +ton_test(test-coro) diff --git a/tdutils/td/utils/tests.cpp b/tdutils/td/utils/tests.cpp index 3f5d673679..d3522f319d 100644 --- a/tdutils/td/utils/tests.cpp +++ b/tdutils/td/utils/tests.cpp @@ -16,6 +16,7 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include #include #include "td/utils/Parser.h" @@ -173,6 +174,14 @@ void TestsRunner::set_stress_flag(bool flag) { stress_flag_ = flag; } +void TestsRunner::set_pretty_output(bool flag) { + pretty_output_ = flag; +} + +bool TestsRunner::use_pretty_output() const { + return pretty_output_; +} + void TestsRunner::run_all() { while (run_all_step()) { } @@ -201,7 +210,11 @@ bool TestsRunner::run_all_step() { ++state_.it; continue; } - LOG(ERROR) << "Run test " << tag("name", name); + if (pretty_output_) { + std::cerr << "Running test " << name << "..." << std::endl; + } else { + LOG(ERROR) << "Run test " << tag("name", name); + } state_.start = Time::now(); state_.start_unadjusted = Time::now_unadjusted(); state_.is_running = true; @@ -213,21 +226,46 @@ bool TestsRunner::run_all_step() { auto passed = Time::now() - state_.start; auto real_passed = Time::now_unadjusted() - state_.start_unadjusted; - if (real_passed + 1e-9 > passed) { - LOG(ERROR) << format::as_time(passed); + if (test_failed_) { + if (pretty_output_) { + std::cerr << "FAIL" << std::endl; + failed_tests_.push_back(name); + } else { + LOG(ERROR) << "FAILED in " << format::as_time(passed); + } + any_test_failed_ = true; } else { - LOG(ERROR) << format::as_time(passed) << " real[" << format::as_time(real_passed) << "]"; + if (pretty_output_) { + std::cerr << "PASS in " << (PSTRING() << format::as_time(passed)) << std::endl; + ++passed_tests_; + } else if (real_passed + 1e-9 > passed) { + LOG(ERROR) << format::as_time(passed); + } else { + LOG(ERROR) << format::as_time(passed) << " real[" << format::as_time(real_passed) << "]"; + } } if (regression_tester_) { regression_tester_->save_db(); } state_.is_running = false; + test_failed_ = false; ++state_.it; } auto ret = state_.it != state_.end; if (!ret) { + if (pretty_output_) { + if (failed_tests_.empty()) { + std::cerr << passed_tests_ << " test(s) passed" << std::endl; + } else { + std::cerr << failed_tests_.size() << " test(s) failed:" << std::endl; + for (auto &failed_name : failed_tests_) { + std::cerr << " - " << failed_name << std::endl; + } + } + } state_ = State(); + test_failed_ = false; } return ret || stress_flag_; } @@ -246,4 +284,13 @@ Status TestsRunner::verify(Slice data) { return regression_tester_->verify_test(PSLICE() << name() << "_default", data); } +void TestsRunner::register_test_failure() { + CHECK(state_.is_running); + test_failed_ = true; +} + +bool TestsRunner::any_test_failed() const { + return any_test_failed_; +} + } // namespace td diff --git a/tdutils/td/utils/tests.h b/tdutils/td/utils/tests.h index efd8223dae..de93f370cf 100644 --- a/tdutils/td/utils/tests.h +++ b/tdutils/td/utils/tests.h @@ -20,6 +20,8 @@ #include #include +#include +#include #include #include "td/utils/Context.h" @@ -101,6 +103,7 @@ class TestContext : public Context { virtual ~TestContext() = default; virtual Slice name() = 0; virtual Status verify(Slice data) = 0; + virtual void register_test_failure() = 0; }; class TestsRunner : public TestContext { @@ -110,9 +113,12 @@ class TestsRunner : public TestContext { void add_test(string name, unique_ptr test); void add_substr_filter(string str); void set_stress_flag(bool flag); + void set_pretty_output(bool flag); void run_all(); bool run_all_step(); void set_regression_tester(unique_ptr regression_tester); + bool any_test_failed() const; + bool use_pretty_output() const; private: struct State { @@ -123,13 +129,19 @@ class TestsRunner : public TestContext { size_t end{0}; }; bool stress_flag_{false}; + bool pretty_output_{false}; vector substr_filters_; vector>> tests_; + vector failed_tests_; + size_t passed_tests_{0}; State state_; + std::atomic test_failed_ = false; unique_ptr regression_tester_; + bool any_test_failed_{false}; Slice name() override; Status verify(Slice data) override; + void register_test_failure() override; }; template @@ -177,24 +189,80 @@ inline vector rand_split(Slice str) { return res; } -template -void assert_eq_impl(const T1 &expected, const T2 &got, const char *file, int line) { - LOG_CHECK(expected == got) << tag("expected", expected) << tag("got", got) << " in " << file << " at line " << line; +namespace detail { + +std::optional stringify(auto const &value) { + if constexpr (requires(std::ostringstream builder) { builder << value; }) { + std::ostringstream builder; + builder << value; + return builder.str(); + } else if constexpr (requires { PSTRING() << value; }) { + return PSTRING() << value; + } + return std::nullopt; } -template -void assert_true_impl(const T &got, const char *file, int line) { - LOG_CHECK(got) << "Expected true in " << file << " at line " << line; +inline std::optional check(bool condition, char const *msg) { + if (condition) { + return std::nullopt; + } + + return PSTRING() << "Expectation failed: " << msg << "!"; } -} // namespace td +std::optional check_eq(auto const &a_value, auto const &b_value, char const *a_expr, char const *b_expr) { + if (a_value == b_value) { + return std::nullopt; + } -#define ASSERT_EQ(expected, got) ::td::assert_eq_impl((expected), (got), __FILE__, __LINE__) + std::ostringstream builder; + builder << "Expectation failed: " << a_expr << " is not equal to " << b_expr; + if (auto a_str = stringify(a_value), b_str = stringify(b_value); a_str.has_value() && b_str.has_value()) { + builder << " (" << *a_str << " != " << *b_str << ")"; + } + return builder.str(); +} -#define ASSERT_TRUE(got) ::td::assert_true_impl((got), __FILE__, __LINE__) +} // namespace detail + +} // namespace td -#define ASSERT_STREQ(expected, got) \ - ::td::assert_eq_impl(::td::Slice((expected)), ::td::Slice((got)), __FILE__, __LINE__) +#define ASSERT_EQ(a, b) \ + do { \ + if (auto error_message = ::td::detail::check_eq(a, b, #a, #b)) { \ + LOG(FATAL) << *error_message; \ + } \ + } while (0) + +#define ASSERT_TRUE(cond) \ + do { \ + if (auto error_message = ::td::detail::check(static_cast(cond), #cond)) { \ + LOG(FATAL) << *error_message; \ + } \ + } while (0) + +#define ASSERT_STREQ(a, b) \ + do { \ + if (auto error_message = ::td::detail::check_eq(::td::Slice((a)), ::td::Slice((b)), #a, #b)) { \ + LOG(FATAL) << *error_message; \ + } \ + } while (0) + +#define EXPECT(cond) \ + do { \ + if (auto error_message = ::td::detail::check(static_cast(cond), #cond)) { \ + LOG(ERROR) << *error_message; \ + ::td::TestContext::get()->register_test_failure(); \ + } \ + } while (0) + +#define EXPECT_EQ(a, b) \ + do { \ + if (auto error_message = ::td::detail::check_eq(a, b, #a, #b)) { \ + LOG(ERROR) << *error_message; \ + ::td::TestContext::get()->register_test_failure(); \ + } \ + } while (0) #define REGRESSION_VERIFY(data) ::td::TestContext::get()->verify(data).ensure() diff --git a/test/test-td-main.cpp b/test/test-td-main.cpp index c653489a6a..2c7f81ad6a 100644 --- a/test/test-td-main.cpp +++ b/test/test-td-main.cpp @@ -35,6 +35,7 @@ int main(int argc, char **argv) { td::set_default_failure_signal_handler().ensure(); // TODO port OptionParser to Windows auto &runner = td::TestsRunner::get_default(); + runner.set_pretty_output(true); for (int i = 1; i < argc; i++) { if (!std::strcmp(argv[i], "--filter")) { CHECK(i + 1 < argc); @@ -50,5 +51,5 @@ int main(int argc, char **argv) { } } runner.run_all(); - return 0; + return runner.any_test_failed() ? 1 : 0; } From b1a6b7bf90312b32e0731a2e31f6a61a6d59d122 Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Mon, 24 Nov 2025 12:38:52 -0500 Subject: [PATCH 489/524] [tontester] Replace pytonlib with a statically typed tonlibjson wrapper (#1906) * [tontester] Use 127.0.0.1 to bind nodes instead of 127.0.42.239 The later causes errors on MacOS. * [tontester] Use correct tonlibjson filename on MacOS * [tontester] Add TLRequest.parse_result * [tontester] Replace pytonlib with a statically typed tonlibjson wrapper Co-authored-by: Dan Klishch --------- Co-authored-by: yungwine --- pyproject.toml | 1 + test/tontester/pyproject.toml | 5 +- .../src/tl/generator/generators/tlobject.py | 39 ++- .../tl/generator/parsers/tlobject/parser.py | 2 +- test/tontester/src/tl/tlobject.py | 5 +- test/tontester/src/tonlib/__init__.py | 7 + test/tontester/src/tonlib/client.py | 129 +++++++++ test/tontester/src/tonlib/py.typed | 0 test/tontester/src/tonlib/tonlibjson.py | 195 ++++++++++++++ test/tontester/src/tontester/install.py | 10 +- test/tontester/src/tontester/network.py | 30 +-- test/tontester/tests/tonlib/test_tonlib.py | 248 ++++++++++++++++++ uv.lock | 226 +--------------- 13 files changed, 651 insertions(+), 246 deletions(-) create mode 100644 test/tontester/src/tonlib/__init__.py create mode 100644 test/tontester/src/tonlib/client.py create mode 100644 test/tontester/src/tonlib/py.typed create mode 100644 test/tontester/src/tonlib/tonlibjson.py create mode 100644 test/tontester/tests/tonlib/test_tonlib.py diff --git a/pyproject.toml b/pyproject.toml index 3924c314d1..700bb8565c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,7 @@ version = "0.1.0" requires-python = ">=3.13" dependencies = [ "pytest>=9.0.1", + "pytest-asyncio", "tontester", ] diff --git a/test/tontester/pyproject.toml b/test/tontester/pyproject.toml index dbcb28d1cc..8693c95af5 100644 --- a/test/tontester/pyproject.toml +++ b/test/tontester/pyproject.toml @@ -2,7 +2,4 @@ name = "tontester" version = "0.1.0" requires-python = ">=3.13" -dependencies = [ - "pytonlib>=0.0.68", - "setuptools<81", # FIXME: Get rid of pytonlib instead. -] +dependencies = [] diff --git a/test/tontester/src/tl/generator/generators/tlobject.py b/test/tontester/src/tl/generator/generators/tlobject.py index 7b9dd4ec52..f715300f8e 100644 --- a/test/tontester/src/tl/generator/generators/tlobject.py +++ b/test/tontester/src/tl/generator/generators/tlobject.py @@ -70,8 +70,6 @@ def _write_modules( # Generate the class for every TLObject for t in tlobjects: - if t.name in BASE_TYPES: - continue _write_source_code(t, t.is_function, builder, type_constructors, ctx) builder.current_indent = 0 @@ -110,6 +108,7 @@ def _write_source_code( _write_class_init(tlobject, is_function, builder, ctx) _write_to_dict(tlobject, builder, ctx) _write_from_dict(tlobject, type_constructors, builder) + _write_parse_result(tlobject, type_constructors, builder) def _write_class_init( @@ -247,10 +246,46 @@ def _write_from_dict( builder.end_block() +def _write_parse_result( + tlobject: ParsedTLObject, + type_constructors: dict[str, list[ParsedTLObject]], + builder: SourceBuilder, +): + if not tlobject.is_function: + return + result_defined = tlobject.result.split(".")[-1][0].islower() + + return_result = ("Type" if not result_defined else "") + get_class_name(tlobject.result) + builder.writeln("@staticmethod") + builder.writeln("@override") + builder.writeln(f"def parse_result(d: tl.JSONSerializable) -> '{return_result}':") + + builder.writeln("if not isinstance(d, dict):") + builder.writeln('raise tl.ModelError(f"Expected dict, got {type(d)}")') + builder.current_indent -= 1 + + if result_defined: + name = get_class_name(tlobject.result) + builder.writeln(f"{name}.from_dict(d)") + else: + constructors = type_constructors.get(tlobject.result, []) + for c in constructors: + builder.writeln( + f'if d.get("@type") == "{c.fullname}": return {c.class_name}.from_dict(d)' + ) + builder.writeln( + f'raise tl.ModelError(f"Unknown constructor for {tlobject.result}: {{d.get("@type")}}")' + ) + builder.current_indent -= 1 + builder.end_block() + + def sort_tlobjects(tlobjects: list[ParsedTLObject]): types: list[ParsedTLObject] = [] type_constructors: dict[str, list[ParsedTLObject]] = defaultdict(list) for tlobject in tlobjects: + if tlobject.name in BASE_TYPES: + tlobject.class_name = f"Type{tlobject.class_name}" types.append(tlobject) if not tlobject.is_function: type_constructors[tlobject.result].append(tlobject) diff --git a/test/tontester/src/tl/generator/parsers/tlobject/parser.py b/test/tontester/src/tl/generator/parsers/tlobject/parser.py index fdb82624b0..52d7a74436 100644 --- a/test/tontester/src/tl/generator/parsers/tlobject/parser.py +++ b/test/tontester/src/tl/generator/parsers/tlobject/parser.py @@ -8,7 +8,7 @@ CORE_TYPES = { 0xBC799737, # boolFalse#bc799737 = Bool; 0x997275B5, # boolTrue#997275b5 = Bool; - 0x3FEDD339, # true#3fedd339 = True; + # 0x3FEDD339, # true#3fedd339 = True; 0xC4B9F9BB, # error#c4b9f9bb code:int text:string = Error; 0x56730BCC, # null#56730bcc = Null; } diff --git a/test/tontester/src/tl/tlobject.py b/test/tontester/src/tl/tlobject.py index 04c050a8a5..ceb146f7de 100644 --- a/test/tontester/src/tl/tlobject.py +++ b/test/tontester/src/tl/tlobject.py @@ -100,7 +100,10 @@ def from_json(cls, source: str) -> Self: class TLRequest(TLObject, ABC): - pass + @staticmethod + @abstractmethod + def parse_result(d: JSONSerializable) -> TLObject: + pass class ModelError(Exception): diff --git a/test/tontester/src/tonlib/__init__.py b/test/tontester/src/tonlib/__init__.py new file mode 100644 index 0000000000..29a6dc6195 --- /dev/null +++ b/test/tontester/src/tonlib/__init__.py @@ -0,0 +1,7 @@ +from .client import TonlibClient +from .tonlibjson import TonlibError + +__all__ = [ + "TonlibClient", + "TonlibError", +] diff --git a/test/tontester/src/tonlib/client.py b/test/tontester/src/tonlib/client.py new file mode 100644 index 0000000000..1ab27e4709 --- /dev/null +++ b/test/tontester/src/tonlib/client.py @@ -0,0 +1,129 @@ +import asyncio +import logging +import traceback +from pathlib import Path + +from tontester.tl import ton_api, tonlib_api + +from .tonlibjson import TonLib + +logger = logging.getLogger(__name__) + + +class TonlibClient: + def __init__( + self, + ls_index: int, + config: ton_api.Liteclient_config_global, + cdll_path: Path, + loop: asyncio.AbstractEventLoop | None = None, + verbosity_level: int = 0, + tonlib_cancel_check_interval: int = 1, + ): + self.ls_index: int = ls_index + self._config: ton_api.Liteclient_config_global = config + self._cdll_path: Path = cdll_path + self._loop: asyncio.AbstractEventLoop | None = loop + self._verbosity_level: int = verbosity_level + self._tonlib_wrapper: TonLib | None = None + self._tonlib_cancel_check_interval: int = tonlib_cancel_check_interval + + @property + def local_config(self) -> ton_api.Liteclient_config_global: + local = ton_api.Liteclient_config_global.from_json(self._config.to_json()) + local.liteservers = [local.liteservers[self.ls_index]] + return local + + async def init(self) -> None: + if self._tonlib_wrapper: + logger.warning("init is already done") + return + event_loop = self._loop or asyncio.get_running_loop() + self._tonlib_wrapper = TonLib( + event_loop, + self.ls_index, + self._cdll_path, + self._tonlib_cancel_check_interval, + self._verbosity_level, + ) + + request = tonlib_api.InitRequest( + options=tonlib_api.Options( + config=tonlib_api.Config( + config=self.local_config.to_json(), + blockchain_name="", + use_callbacks_for_network=False, + ignore_cache=False, + ), + keystore_type=tonlib_api.KeyStoreTypeInMemory(), + ) + ) + + _ = await self._tonlib_wrapper.execute(request) + + logger.info(f"TonLib #{self.ls_index:03d} inited successfully") + + async def aclose(self): + if self._tonlib_wrapper is not None: + await self._tonlib_wrapper.aclose() + self._tonlib_wrapper = None + + async def __aenter__(self): + await self.init() + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: traceback.TracebackException | None, + ): + await self.aclose() + + def __await__(self): + return self.init().__await__() + + async def sync_tonlib(self) -> tonlib_api.Ton_blockIdExt: + assert self._tonlib_wrapper is not None + request = tonlib_api.SyncRequest() + return request.parse_result(await self._tonlib_wrapper.execute(request)) + + async def get_masterchain_info(self) -> tonlib_api.Blocks_masterchainInfo: + assert self._tonlib_wrapper is not None + request = tonlib_api.Blocks_getMasterchainInfoRequest() + return request.parse_result(await self._tonlib_wrapper.execute(request)) + + async def raw_send_message(self, serialized_boc: bytes) -> tonlib_api.TypeOk: + assert self._tonlib_wrapper is not None + request = tonlib_api.Raw_sendMessageRequest(body=serialized_boc) + return request.parse_result(await self._tonlib_wrapper.execute(request)) + + async def get_libraries(self, library_list: list[bytes]) -> tonlib_api.Smc_libraryResult: + assert self._tonlib_wrapper is not None + request = tonlib_api.Smc_getLibrariesRequest(library_list) + return request.parse_result(await self._tonlib_wrapper.execute(request)) + + async def raw_get_transactions( + self, account_address: str, from_transaction_lt: int, from_transaction_hash: bytes + ) -> tonlib_api.Raw_transactions: + assert self._tonlib_wrapper is not None + # FIXME: Replace these with proper classes for TransactionId and Address. + assert len(account_address) == 48, "account address must be serialized" + assert len(from_transaction_hash) == 32 + request = tonlib_api.Raw_getTransactionsRequest( + account_address=tonlib_api.AccountAddress(account_address), + from_transaction_id=tonlib_api.Internal_transactionId( + lt=from_transaction_lt, + hash=from_transaction_hash, + ), + ) + return request.parse_result(await self._tonlib_wrapper.execute(request)) + + async def raw_get_account_state(self, account_address: str) -> tonlib_api.Raw_fullAccountState: + assert self._tonlib_wrapper is not None + # FIXME: Replace these with proper class for Address. + assert len(account_address) == 48, "account address must be serialized" + request = tonlib_api.Raw_getAccountStateRequest( + account_address=tonlib_api.AccountAddress(account_address) + ) + return request.parse_result(await self._tonlib_wrapper.execute(request)) diff --git a/test/tontester/src/tonlib/py.typed b/test/tontester/src/tonlib/py.typed new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/tontester/src/tonlib/tonlibjson.py b/test/tontester/src/tonlib/tonlibjson.py new file mode 100644 index 0000000000..914e03fd9e --- /dev/null +++ b/test/tontester/src/tonlib/tonlibjson.py @@ -0,0 +1,195 @@ +import asyncio +import ctypes +import json +import logging +import typing +from dataclasses import dataclass +from enum import Enum, auto +from pathlib import Path +from typing import override + +from tontester.tl import tonlib_api + +from tl import JSONSerializable, TLObject + +logger = logging.getLogger(__name__) + + +class TonlibError(Exception): + def __init__(self, result: tonlib_api.Error): + super().__init__() + self.result: tonlib_api.Error = result + + @property + def code(self): + return self.result.code + + @override + def __str__(self): + return self.result.message + + +def _parse_tonlib_error(result: dict[str, JSONSerializable]): + if result.get("@type") == "error": + er = tonlib_api.Error.from_dict(result) + return TonlibError(er) + return None + + +@dataclass +class _Payload[T]: + value: T + + +class _Status(Enum): + NONE = auto() + FINISHED = auto() + CRASHED = auto() + + +class TonLib: + def __init__( + self, + loop: asyncio.AbstractEventLoop, + ls_index: int, + cdll_path: Path, + tonlib_cancel_check_interval: int, + verbosity_level: int = 0, + ): + tonlib = ctypes.CDLL(cdll_path) + + tonlib_client_set_verbosity_level = tonlib.tonlib_client_set_verbosity_level + tonlib_client_set_verbosity_level.restype = None + tonlib_client_set_verbosity_level.argtypes = [ctypes.c_int] + + tonlib_client_set_verbosity_level(verbosity_level) + + tonlib_json_client_create = tonlib.tonlib_client_json_create + tonlib_json_client_create.restype = ctypes.c_void_p + tonlib_json_client_create.argtypes = [] + self._client: int = tonlib_json_client_create() + + tonlib_json_client_receive = tonlib.tonlib_client_json_receive + tonlib_json_client_receive.restype = ctypes.c_char_p + tonlib_json_client_receive.argtypes = [ctypes.c_void_p, ctypes.c_double] + self._tonlib_json_client_receive: typing.Callable[[int, int], bytes | None] = ( + tonlib_json_client_receive + ) + + tonlib_json_client_send = tonlib.tonlib_client_json_send + tonlib_json_client_send.restype = None + tonlib_json_client_send.argtypes = [ctypes.c_void_p, ctypes.c_char_p] + self._tonlib_json_client_send: typing.Callable[[int, bytes], None] = tonlib_json_client_send + + tonlib_json_client_destroy = tonlib.tonlib_client_json_destroy + tonlib_json_client_destroy.restype = None + tonlib_json_client_destroy.argtypes = [ctypes.c_void_p] + self._tonlib_json_client_destroy: typing.Callable[[int], None] = tonlib_json_client_destroy + + # FIXME: This only exists because we cannot cancel running `tonlib_json_client_receive`. + self._tonlib_cancel_check_interval: int = tonlib_cancel_check_interval + + self._request_id: int = 0 + self._futures: dict[str, asyncio.Future[JSONSerializable]] = {} + self._loop: asyncio.AbstractEventLoop = loop + self._ls_index: int = ls_index + self._state: _Status = _Status.NONE + + self._work_notification: asyncio.Event = asyncio.Event() + self._read_results_task: asyncio.Task[None] = self._loop.create_task(self._read_results()) + + def __del__(self): + assert self._client == 0, ( + "TonLib client not destroyed. Call 'aclose' before destroying the object." + ) + + def _send(self, query: JSONSerializable): + assert self._is_working + q = json.dumps(query).encode("utf-8") + self._tonlib_json_client_send(self._client, q) + + def _next_request_id(self): + result = str(self._request_id) + self._request_id += 1 + return result + + async def execute(self, query: TLObject) -> JSONSerializable: + assert self._is_working, f"TonLib failed with state: {self._state}" + + request_id = self._next_request_id() + query_d = query.to_dict() + query_d["@extra"] = request_id + + future: asyncio.Future[JSONSerializable] = self._loop.create_future() + future.add_done_callback(lambda _: self._futures.pop(request_id, None)) + self._futures[request_id] = future + + self._send(query_d) + self._work_notification.set() + + result = await future + return result + + @property + def _is_working(self): + return self._state == _Status.NONE + + def _receive(self) -> _Payload[JSONSerializable] | None: + result = self._tonlib_json_client_receive(self._client, self._tonlib_cancel_check_interval) + if result is None: + return None + return _Payload(typing.cast(JSONSerializable, json.loads(result.decode("utf-8")))) + + async def aclose(self): + try: + self._state = _Status.FINISHED + self._work_notification.set() + await self._read_results_task + for f in self._futures.values(): + if not f.done(): + f.set_exception(asyncio.CancelledError()) + finally: + self._tonlib_json_client_destroy(self._client) + self._client = 0 + + async def _read_results(self): + try: + while True: + # If there are no requests in flight, wait for new work. + if not len(self._futures): + _ = await self._work_notification.wait() + self._work_notification.clear() + + # We might be woken up because we are cancelled. + if not self._is_working: + break + + result = await self._loop.run_in_executor(None, self._receive) + # Read has timed out. Recheck the state. + if result is None: + continue + + result = result.value + assert isinstance(result, dict) + assert isinstance(result["@extra"], str) + request_id = result["@extra"] + future = self._futures.get(request_id, None) + + if future is None or future.done(): + continue + + try: + tonlib_error = _parse_tonlib_error(result) + if tonlib_error is not None: + future.set_exception(tonlib_error) + else: + future.set_result(result) + _ = self._futures.pop(request_id, None) + except Exception as e: + future.set_exception(e) + except Exception as e: + logger.error("TonLib background task failed", exc_info=True) + self._state = _Status.CRASHED + for f in self._futures.values(): + if not f.done(): + f.set_exception(e) diff --git a/test/tontester/src/tontester/install.py b/test/tontester/src/tontester/install.py index 03dc2a97e0..dfca51dc50 100644 --- a/test/tontester/src/tontester/install.py +++ b/test/tontester/src/tontester/install.py @@ -1,5 +1,6 @@ import os import subprocess +import sys from pathlib import Path from typing import final @@ -43,7 +44,14 @@ def dht_server_exe(self): @property def tonlibjson(self): - return self.build_dir / "tonlib/libtonlibjson.so" + if sys.platform.startswith("linux"): + name = "tonlib/libtonlibjson.so" + elif sys.platform == "darwin": + name = "tonlib/libtonlibjson.dylib" + else: + raise RuntimeError(f"Unsupported platform: {sys.platform}") + + return self.build_dir / name def run_fift(install: Install, code: str, working_dir: Path): diff --git a/test/tontester/src/tontester/network.py b/test/tontester/src/tontester/network.py index 6295c5bd6c..8a4c049de5 100644 --- a/test/tontester/src/tontester/network.py +++ b/test/tontester/src/tontester/network.py @@ -8,16 +8,15 @@ from enum import IntEnum, auto from ipaddress import IPv4Address from pathlib import Path -from typing import Literal, cast, final, override +from typing import Literal, final, override -from pytonlib import TonlibClient, TonlibError # pyright: ignore[reportMissingTypeStubs] - -from tl import JSONSerializable, TLObject +from tl import TLObject +from tonlib import TonlibClient, TonlibError from .install import Install from .key import Key from .log_streamer import LogStreamer -from .tl import ton_api, tonlib_api +from .tl import ton_api from .zerostate import NetworkConfig, Zerostate, create_zerostate l = logging.getLogger(__name__) @@ -70,7 +69,7 @@ def _install(self): def _new_network_address(self) -> _IPv4AddressAndPort: self._network._port += 1 return _IPv4AddressAndPort( - IPv4Address("127.0.42.239"), + IPv4Address("127.0.0.1"), self._network._port, ) @@ -262,16 +261,16 @@ async def wait_mc_block(self, seqno: int): while True: try: - raw_result = cast(JSONSerializable, await client.get_masterchain_info()) # pyright: ignore[reportUnknownMemberType] + mc_info = await client.get_masterchain_info() except TonlibError as e: # FIXME: We should really let node notify us that it is ready. try: if ( - e.result["code"] == 500 + e.result.code == 500 and ( - e.result["message"] + e.result.message == "LITE_SERVER_NETWORKtimeout for adnl query query" # node is not synced yet - or e.result["message"] + or e.result.message == "LITE_SERVER_NETWORK" # node is not listening the socket ) ): @@ -281,7 +280,6 @@ async def wait_mc_block(self, seqno: int): pass raise - mc_info = tonlib_api.Blocks_masterchainInfo.from_dict(raw_result) assert mc_info.last is not None if mc_info.last.seqno >= seqno: @@ -452,14 +450,10 @@ async def tonlib_client(self) -> TonlibClient: validator=self._get_or_generate_zerostate().as_validator_config(), ) - keystore_dir = self._directory / "lc-keystore" - keystore_dir.mkdir() - self._client = TonlibClient( ls_index=0, - config=config.to_dict(), - keystore=str(keystore_dir), - cdll_path=str(self._install.tonlibjson), + config=config, + cdll_path=self._install.tonlibjson, verbosity_level=3, ) await self._client.init() @@ -469,5 +463,5 @@ async def tonlib_client(self) -> TonlibClient: @override async def stop(self): if self._client: - await self._client.close() + await self._client.aclose() await super().stop() diff --git a/test/tontester/tests/tonlib/test_tonlib.py b/test/tontester/tests/tonlib/test_tonlib.py new file mode 100644 index 0000000000..b2ee6643e8 --- /dev/null +++ b/test/tontester/tests/tonlib/test_tonlib.py @@ -0,0 +1,248 @@ +# pyright: reportPrivateUsage=false + +import asyncio +import base64 +import json +import threading +from pathlib import Path +from typing import cast + +import pytest +from tontester.install import Install +from tontester.tl import ton_api + +from tl import JSONSerializable +from tonlib import TonlibClient, TonlibError + +config: dict[str, JSONSerializable] = { + "@type": "liteclient.config.global", + "dht": { + "@type": "dht.config.global", + "k": 6, + "a": 3, + "static_nodes": {"@type": "dht.nodes", "nodes": []}, + }, + "liteservers": [ + { + "ip": 123, + "port": 123, + "id": {"@type": "pub.ed25519", "key": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="}, + } + ], + "validator": { + "@type": "validator.config.global", + "zero_state": { + "workchain": -1, + "shard": -9223372036854775808, + "seqno": 0, + "root_hash": "F6OpKZKqvqeFp6CQmFomXNMfMj2EnaUSOXN+Mh+wVWk=", + "file_hash": "XplPz01CXAps5qeSWUtxcyBfdAo5zVb1N979KLSKD24=", + }, + "init_block": { + "workchain": -1, + "shard": -9223372036854775808, + "seqno": 53986204, + "root_hash": "uBltuZIIhio2c9EiCTI9cExzf2UhALUDXvUora+eIPg=", + "file_hash": "JDx4DGuIs+BFo6qhLQnKZmBaYBlKTCeJrO0CgMzjqjQ=", + }, + "hardforks": [ + { + "workchain": -1, + "shard": -9223372036854775808, + "seqno": 8536841, + "root_hash": "08Kpc9XxrMKC6BF/FeNHPS3MEL1/Vi/fQU/C9ELUrkc=", + "file_hash": "t/9VBPODF7Zdh4nsnA49dprO69nQNMqYL+zk5bCjV/8=", + } + ], + }, +} + + +@pytest.fixture() +def tonlib_client() -> TonlibClient: + repo_root = Path(__file__).resolve().parents[4] + install = Install(repo_root / "build", repo_root) + + return TonlibClient( + verbosity_level=5, + config=ton_api.Liteclient_config_global.from_dict(config), + ls_index=0, + cdll_path=install.tonlibjson, + ) + + +@pytest.mark.asyncio +async def test_client_init(tonlib_client: TonlibClient): + await tonlib_client.init() + assert tonlib_client._tonlib_wrapper is not None + await tonlib_client.aclose() + + +@pytest.mark.asyncio +async def test_request(tonlib_client: TonlibClient, monkeypatch: pytest.MonkeyPatch): + h = "FsRgTb2HymFDIkV82C0aA0CPbdQKAVEzZBgVq5rjPNI=" + + def mock_send(_, request_json: str) -> None: + q: dict[str, JSONSerializable] = cast(dict[str, JSONSerializable], json.loads(request_json)) + assert q["@type"] == "blocks.getMasterchainInfo" + res = { + "@type": "blocks.masterchainInfo", + "@extra": q["@extra"], + "last": { + "@type": "ton.blockIdExt", + "workchain": -1, + "shard": -9223372036854775808, + "seqno": 1, + "root_hash": "FsRgTb2HymFDIkV82C0aA0CPbdQKAVEzZBgVq5rjPNI=", + "file_hash": "JKzTCHqi/c0or9o78mCwo1kigYXdagBQqQ4B2RLXFXY=", + }, + "state_root_hash": "9DcJlDUeelZCBniBzWseg6KyjbjtkB8r6rX4x6BDT9o=", + "init": { + "@type": "ton.blockIdExt", + "workchain": -1, + "shard": 0, + "seqno": 0, + "root_hash": "4DWqijlo0wfJCuJUZeKkWOvENlS3DdL9z2DXkFQ1UtE=", + "file_hash": "nv5sO4rQHhrT5PGLTs1f01AtWYuuI5tH41c87vU1zac=", + }, + } + monkeypatch.setattr( + tonlib_client._tonlib_wrapper, + "_tonlib_json_client_receive", + lambda *_: json.dumps(res).encode("utf-8"), # pyright: ignore[reportUnknownLambdaType, reportUnknownArgumentType] + ) + + async with tonlib_client: + monkeypatch.setattr( + tonlib_client._tonlib_wrapper, + "_tonlib_json_client_send", + mock_send, # pyright: ignore[reportUnknownArgumentType] + ) + blk = await tonlib_client.get_masterchain_info() + assert blk.last is not None + assert blk.last.workchain == -1 + assert blk.last.seqno == 1 + assert blk.last.root_hash == base64.b64decode(h) + + +@pytest.mark.asyncio +async def test_timeout(tonlib_client: TonlibClient, monkeypatch: pytest.MonkeyPatch): + async with tonlib_client: + monkeypatch.setattr( + tonlib_client._tonlib_wrapper, + "_tonlib_json_client_send", + lambda *_: None, # pyright: ignore[reportUnknownLambdaType, reportUnknownArgumentType] + ) + with pytest.raises(TimeoutError): + _ = await asyncio.wait_for(tonlib_client.get_masterchain_info(), timeout=2) + assert tonlib_client._tonlib_wrapper is not None + assert not tonlib_client._tonlib_wrapper._futures + + +@pytest.mark.asyncio +async def test_error_response(tonlib_client: TonlibClient, monkeypatch: pytest.MonkeyPatch): + def mock_send(_, request_json: str) -> None: + q: dict[str, JSONSerializable] = cast(dict[str, JSONSerializable], json.loads(request_json)) + assert q["@type"] == "blocks.getMasterchainInfo" + res = {"@type": "error", "@extra": q["@extra"], "code": 504, "message": "Timeout"} + monkeypatch.setattr( + tonlib_client._tonlib_wrapper, + "_tonlib_json_client_receive", + lambda *_: json.dumps(res).encode("utf-8"), # pyright: ignore[reportUnknownLambdaType, reportUnknownArgumentType] + ) + + async with tonlib_client: + monkeypatch.setattr( + tonlib_client._tonlib_wrapper, + "_tonlib_json_client_send", + mock_send, # pyright: ignore[reportUnknownArgumentType] + ) + with pytest.raises(TonlibError) as e: + _ = await tonlib_client.get_masterchain_info() + assert e.value.code == 504 + assert str(e.value) == "Timeout" + + +@pytest.mark.asyncio +async def test_read_results_error_cancels_inflight_requests( + tonlib_client: TonlibClient, monkeypatch: pytest.MonkeyPatch +): + send_event = threading.Event() + call_count = 0 + + def mock_receive(*_) -> bytes: + nonlocal call_count + call_count += 1 + + _ = send_event.wait() + + if call_count == 1: + raise RuntimeError("Simulated error in _read_results") + + assert False + + def mock_send(*_) -> None: + send_event.set() + + async with tonlib_client: + monkeypatch.setattr( + tonlib_client._tonlib_wrapper, + "_tonlib_json_client_send", + mock_send, # pyright: ignore[reportUnknownArgumentType] + ) + monkeypatch.setattr( + tonlib_client._tonlib_wrapper, + "_tonlib_json_client_receive", + mock_receive, # pyright: ignore[reportUnknownArgumentType] + ) + + task1 = asyncio.create_task(tonlib_client.get_masterchain_info()) + task2 = asyncio.create_task(tonlib_client.get_masterchain_info()) + task3 = asyncio.create_task(tonlib_client.get_masterchain_info()) + + send_event.set() + + with pytest.raises(RuntimeError, match="Simulated error in _read_results"): + await task1 + with pytest.raises(RuntimeError, match="Simulated error in _read_results"): + await task2 + with pytest.raises(RuntimeError, match="Simulated error in _read_results"): + await task3 + + with pytest.raises(AssertionError, match="TonLib failed with state"): + _ = await tonlib_client.get_masterchain_info() + + +@pytest.mark.asyncio +async def test_aclose_cancels_requests( + tonlib_client: TonlibClient, monkeypatch: pytest.MonkeyPatch +): + call_count = 0 + + send_event = asyncio.Event() + + def mock_send(*_): + nonlocal call_count + call_count += 1 + if call_count == 3: + send_event.set() + + async with tonlib_client: + monkeypatch.setattr( + tonlib_client._tonlib_wrapper, + "_tonlib_json_client_send", + mock_send, # pyright: ignore[reportUnknownArgumentType] + ) + + task1 = asyncio.create_task(tonlib_client.get_masterchain_info()) + task2 = asyncio.create_task(tonlib_client.get_masterchain_info()) + task3 = asyncio.create_task(tonlib_client.get_masterchain_info()) + + _ = await send_event.wait() + + with pytest.raises(asyncio.CancelledError): + await task1 + with pytest.raises(asyncio.CancelledError): + await task2 + with pytest.raises(asyncio.CancelledError): + await task3 diff --git a/uv.lock b/uv.lock index 4a3ecc729e..d1db128997 100644 --- a/uv.lock +++ b/uv.lock @@ -20,103 +20,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/38/87/d95da7ef033c26d71a7153623e7b14fd9cc2c8676e03cf4e9199320cbaf5/basedpyright-1.33.0-py3-none-any.whl", hash = "sha256:1ab813683e28184aa60fd4000731f6fe9dbd083c9bd489420bb20f3e5dbb6ab8", size = 11859149, upload-time = "2025-11-07T11:49:28.694Z" }, ] -[[package]] -name = "bitarray" -version = "3.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/95/06/92fdc84448d324ab8434b78e65caf4fb4c6c90b4f8ad9bdd4c8021bfaf1e/bitarray-3.8.0.tar.gz", hash = "sha256:3eae38daffd77c9621ae80c16932eea3fb3a4af141fb7cc724d4ad93eff9210d", size = 151991, upload-time = "2025-11-02T21:41:15.117Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3e/35/480364d4baf1e34c79076750914664373f561c58abb5c31c35b3fae613ff/bitarray-3.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:18214bac86341f1cc413772e66447d6cca10981e2880b70ecaf4e826c04f95e9", size = 148582, upload-time = "2025-11-02T21:39:42.268Z" }, - { url = "https://files.pythonhosted.org/packages/5e/a8/718b95524c803937f4edbaaf6480f39c80f6ed189d61357b345e8361ffb6/bitarray-3.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:01c5f0dc080b0ebb432f7a68ee1e88a76bd34f6d89c9568fcec65fb16ed71f0e", size = 145433, upload-time = "2025-11-02T21:39:43.552Z" }, - { url = "https://files.pythonhosted.org/packages/03/66/4a10f30dc9e2e01e3b4ecd44a511219f98e63c86b0e0f704c90fac24059b/bitarray-3.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:86685fa04067f7175f9718489ae755f6acde03593a1a9ca89305554af40e14fd", size = 332986, upload-time = "2025-11-02T21:39:44.656Z" }, - { url = "https://files.pythonhosted.org/packages/53/25/4c08774d847f80a1166e4c704b4e0f1c417c0afe6306eae0bc5e70d35faa/bitarray-3.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56896ceeffe25946c4010320629e2d858ca763cd8ded273c81672a5edbcb1e0a", size = 360634, upload-time = "2025-11-02T21:39:45.798Z" }, - { url = "https://files.pythonhosted.org/packages/a5/8f/bf8ad26169ebd0b2746d5c7564db734453ca467f8aab87e9d43b0a794383/bitarray-3.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9858dcbc23ba7eaadcd319786b982278a1a2b2020720b19db43e309579ff76fb", size = 371992, upload-time = "2025-11-02T21:39:46.968Z" }, - { url = "https://files.pythonhosted.org/packages/a9/16/ce166754e7c9d10650e02914552fa637cf3b2591f7ed16632bbf6b783312/bitarray-3.8.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aa7dec53c25f1949513457ef8b0ea1fb40e76c672cc4d2daa8ad3c8d6b73491a", size = 340315, upload-time = "2025-11-02T21:39:48.182Z" }, - { url = "https://files.pythonhosted.org/packages/de/2a/fbba3a106ddd260e84b9a624f730257c32ba51a8a029565248dfedfdf6f2/bitarray-3.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:15a2eff91f54d2b1f573cca8ca6fb58763ce8fea80e7899ab028f3987ef71cd5", size = 330473, upload-time = "2025-11-02T21:39:49.705Z" }, - { url = "https://files.pythonhosted.org/packages/68/97/56cf3c70196e7307ad32318a9d6ed969dbdc6a4534bbe429112fa7dfe42e/bitarray-3.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b1572ee0eb1967e71787af636bb7d1eb9c6735d5337762c450650e7f51844594", size = 358129, upload-time = "2025-11-02T21:39:51.189Z" }, - { url = "https://files.pythonhosted.org/packages/fd/be/afd391a5c0896d3339613321b2f94af853f29afc8bd3fbc327431244c642/bitarray-3.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5bfac7f236ba1a4d402644bdce47fb9db02a7cf3214a1f637d3a88390f9e5428", size = 356005, upload-time = "2025-11-02T21:39:52.355Z" }, - { url = "https://files.pythonhosted.org/packages/ae/08/a8e1a371babba29bad3378bb3a2cdca2b012170711e7fe1f22031a6b7b95/bitarray-3.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f0a55cf02d2cdd739b40ce10c09bbdd520e141217696add7a48b56e67bdfdfe6", size = 336862, upload-time = "2025-11-02T21:39:54.345Z" }, - { url = "https://files.pythonhosted.org/packages/ee/8a/6dc1d0fdc06991c8dc3b1fcfe1ae49fbaced42064cd1b5f24278e73fe05f/bitarray-3.8.0-cp313-cp313-win32.whl", hash = "sha256:a2ba92f59e30ce915e9e79af37649432e3a212ddddf416d4d686b1b4825bcdb2", size = 143018, upload-time = "2025-11-02T21:39:56.361Z" }, - { url = "https://files.pythonhosted.org/packages/2e/72/76e13f5cd23b8b9071747909663ce3b02da24a5e7e22c35146338625db35/bitarray-3.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:1c8f2a5d8006db5a555e06f9437e76bf52537d3dfd130cb8ae2b30866aca32c9", size = 149977, upload-time = "2025-11-02T21:39:57.718Z" }, - { url = "https://files.pythonhosted.org/packages/01/37/60f336c32336cc3ec03b0c61076f16ea2f05d5371c8a56e802161d218b77/bitarray-3.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:50ddbe3a7b4b6ab96812f5a4d570f401a2cdb95642fd04c062f98939610bbeee", size = 146930, upload-time = "2025-11-02T21:39:59.308Z" }, - { url = "https://files.pythonhosted.org/packages/1b/b0/411327a6c7f6b2bead64bb06fe60b92e0344957ec1ab0645d5ccc25fdafe/bitarray-3.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8cbd4bfc933b33b85c43ef4c1f4d5e3e9d91975ea6368acf5fbac02bac06ea89", size = 148563, upload-time = "2025-11-02T21:40:01.006Z" }, - { url = "https://files.pythonhosted.org/packages/2a/bc/ff80d97c627d774f879da0ea93223adb1267feab7e07d5c17580ffe6d632/bitarray-3.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9d35d8f8a1c9ed4e2b08187b513f8a3c71958600129db3aa26d85ea3abfd1310", size = 145422, upload-time = "2025-11-02T21:40:02.535Z" }, - { url = "https://files.pythonhosted.org/packages/66/e7/b4cb6c5689aacd0a32f3aa8a507155eaa33528c63de2f182b60843fbf700/bitarray-3.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:99f55e14e7c56f4fafe1343480c32b110ef03836c21ff7c48bae7add6818f77c", size = 332852, upload-time = "2025-11-02T21:40:03.645Z" }, - { url = "https://files.pythonhosted.org/packages/e7/91/fbd1b047e3e2f4b65590f289c8151df1d203d75b005f5aae4e072fe77d76/bitarray-3.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:dfbe2aa45b273f49e715c5345d94874cb65a28482bf231af408891c260601b8d", size = 360801, upload-time = "2025-11-02T21:40:04.827Z" }, - { url = "https://files.pythonhosted.org/packages/ef/4a/63064c593627bac8754fdafcb5343999c93ab2aeb27bcd9d270a010abea5/bitarray-3.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:64af877116edf051375b45f0bda648143176a017b13803ec7b3a3111dc05f4c5", size = 371408, upload-time = "2025-11-02T21:40:05.985Z" }, - { url = "https://files.pythonhosted.org/packages/46/97/ddc07723767bdafd170f2ff6e173c940fa874192783ee464aa3c1dedf07d/bitarray-3.8.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cdfbb27f2c46bb5bbdcee147530cbc5ca8ab858d7693924e88e30ada21b2c5e2", size = 340033, upload-time = "2025-11-02T21:40:07.189Z" }, - { url = "https://files.pythonhosted.org/packages/ad/1e/e1ea9f1146fd4af032817069ff118918d73e5de519854ce3860e2ed560ff/bitarray-3.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4d73d4948dcc5591d880db8933004e01f1dd2296df9de815354d53469beb26fe", size = 330774, upload-time = "2025-11-02T21:40:08.496Z" }, - { url = "https://files.pythonhosted.org/packages/cf/9f/8242296c124a48d1eab471fd0838aeb7ea9c6fd720302d99ab7855d3e6d3/bitarray-3.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:28a85b056c0eb7f5d864c0ceef07034117e8ebfca756f50648c71950a568ba11", size = 358337, upload-time = "2025-11-02T21:40:10.035Z" }, - { url = "https://files.pythonhosted.org/packages/b5/6b/9095d75264c67d479f298c80802422464ce18c3cdd893252eeccf4997611/bitarray-3.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:79ec4498a545733ecace48d780d22407411b07403a2e08b9a4d7596c0b97ebd7", size = 355639, upload-time = "2025-11-02T21:40:11.485Z" }, - { url = "https://files.pythonhosted.org/packages/a0/af/c93c0ae5ef824136e90ac7ddf6cceccb1232f34240b2f55a922f874da9b4/bitarray-3.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:33af25c4ff7723363cb8404dfc2eefeab4110b654f6c98d26aba8a08c745d860", size = 336999, upload-time = "2025-11-02T21:40:12.709Z" }, - { url = "https://files.pythonhosted.org/packages/81/0f/72c951f5997b2876355d5e671f78dd2362493254876675cf22dbd24389ae/bitarray-3.8.0-cp314-cp314-win32.whl", hash = "sha256:2c3bb96b6026643ce24677650889b09073f60b9860a71765f843c99f9ab38b25", size = 142169, upload-time = "2025-11-02T21:40:14.031Z" }, - { url = "https://files.pythonhosted.org/packages/8a/55/ef1b4de8107bf13823da8756c20e1fbc9452228b4e837f46f6d9ddba3eb3/bitarray-3.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:847c7f61964225fc489fe1d49eda7e0e0d253e98862c012cecf845f9ad45cdf4", size = 148737, upload-time = "2025-11-02T21:40:15.436Z" }, - { url = "https://files.pythonhosted.org/packages/5f/26/bc0784136775024ac56cc67c0d6f9aa77a7770de7f82c3a7c9be11c217cd/bitarray-3.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:a2cb35a6efaa0e3623d8272471371a12c7e07b51a33e5efce9b58f655d864b4e", size = 146083, upload-time = "2025-11-02T21:40:17.135Z" }, - { url = "https://files.pythonhosted.org/packages/6e/64/57984e64264bf43d93a1809e645972771566a2d0345f4896b041ce20b000/bitarray-3.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:15e8d0597cc6e8496de6f4dea2a6880c57e1251502a7072f5631108a1aa28521", size = 149455, upload-time = "2025-11-02T21:40:18.558Z" }, - { url = "https://files.pythonhosted.org/packages/81/c0/0d5f2eaef1867f462f764bdb07d1e116c33a1bf052ea21889aefe4282f5b/bitarray-3.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8ffe660e963ae711cb9e2b8d8461c9b1ad6167823837fc17d59d5e539fb898fa", size = 146491, upload-time = "2025-11-02T21:40:19.665Z" }, - { url = "https://files.pythonhosted.org/packages/65/c6/bc1261f7a8862c0c59220a484464739e52235fd1e2afcb24d7f7d3fb5702/bitarray-3.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4779f356083c62e29b4198d290b7b17a39a69702d150678b7efff0fdddf494a8", size = 339721, upload-time = "2025-11-02T21:40:21.277Z" }, - { url = "https://files.pythonhosted.org/packages/81/d8/289ca55dd2939ea17b1108dc53bffc0fdc5160ba44f77502dfaae35d08c6/bitarray-3.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:025d133bf4ca8cf75f904eeb8ea946228d7c043231866143f31946a6f4dd0bf3", size = 367823, upload-time = "2025-11-02T21:40:22.463Z" }, - { url = "https://files.pythonhosted.org/packages/91/a2/61e7461ca9ac0fcb70f327a2e84b006996d2a840898e69037a39c87c6d06/bitarray-3.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:451f9958850ea98440d542278368c8d1e1ea821e2494b204570ba34a340759df", size = 377341, upload-time = "2025-11-02T21:40:23.789Z" }, - { url = "https://files.pythonhosted.org/packages/6c/87/4a0c9c8bdb13916d443e04d8f8542eef9190f31425da3c17c3478c40173f/bitarray-3.8.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6d79f659965290af60d6acc8e2716341865fe74609a7ede2a33c2f86ad893b8f", size = 344985, upload-time = "2025-11-02T21:40:25.261Z" }, - { url = "https://files.pythonhosted.org/packages/17/4c/ff9259b916efe53695b631772e5213699c738efc2471b5ffe273f4000994/bitarray-3.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fbf05678c2ae0064fb1b8de7e9e8f0fc30621b73c8477786dd0fb3868044a8c8", size = 336796, upload-time = "2025-11-02T21:40:26.942Z" }, - { url = "https://files.pythonhosted.org/packages/0f/4b/51b2468bbddbade5e2f3b8d5db08282c5b309e8687b0f02f75a8b5ff559c/bitarray-3.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:c396358023b876cff547ce87f4e8ff8a2280598873a137e8cc69e115262260b8", size = 365085, upload-time = "2025-11-02T21:40:28.224Z" }, - { url = "https://files.pythonhosted.org/packages/bf/79/53473bfc2e052c6dbb628cdc1b156be621c77aaeb715918358b01574be55/bitarray-3.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed3493a369fe849cce98542d7405c88030b355e4d2e113887cb7ecc86c205773", size = 361012, upload-time = "2025-11-02T21:40:29.635Z" }, - { url = "https://files.pythonhosted.org/packages/c4/b1/242bf2e44bfc69e73fa2b954b425d761a8e632f78ea31008f1c3cfad0854/bitarray-3.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c764fb167411d5afaef88138542a4bfa28bd5e5ded5e8e42df87cef965efd6e9", size = 340644, upload-time = "2025-11-02T21:40:31.089Z" }, - { url = "https://files.pythonhosted.org/packages/cf/01/12e5ecf30a5de28a32485f226cad4b8a546845f65f755ce0365057ab1e92/bitarray-3.8.0-cp314-cp314t-win32.whl", hash = "sha256:e12769d3adcc419e65860de946df8d2ed274932177ac1cdb05186e498aaa9149", size = 143630, upload-time = "2025-11-02T21:40:32.351Z" }, - { url = "https://files.pythonhosted.org/packages/b6/92/6b6ade587b08024a8a890b07724775d29da9cf7497be5c3cbe226185e463/bitarray-3.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0ca70ccf789446a6dfde40b482ec21d28067172cd1f8efd50d5548159fccad9e", size = 150250, upload-time = "2025-11-02T21:40:33.596Z" }, - { url = "https://files.pythonhosted.org/packages/ed/40/be3858ffed004e47e48a2cefecdbf9b950d41098b780f9dc3aa609a88351/bitarray-3.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2a3d1b05ffdd3e95687942ae7b13c63689f85d3f15c39b33329e3cb9ce6c015f", size = 147015, upload-time = "2025-11-02T21:40:35.064Z" }, -] - -[[package]] -name = "certifi" -version = "2025.10.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4c/5b/b6ce21586237c77ce67d01dc5507039d444b630dd76611bbca2d8e5dcd91/certifi-2025.10.5.tar.gz", hash = "sha256:47c09d31ccf2acf0be3f701ea53595ee7e0b8fa08801c6624be771df09ae7b43", size = 164519, upload-time = "2025-10-05T04:12:15.808Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e4/37/af0d2ef3967ac0d6113837b44a4f0bfe1328c2b9763bd5b1744520e5cfed/certifi-2025.10.5-py3-none-any.whl", hash = "sha256:0f212c2744a9bb6de0c56639a6f68afe01ecd92d91f14ae897c4fe7bbeeef0de", size = 163286, upload-time = "2025-10-05T04:12:14.03Z" }, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/13/69/33ddede1939fdd074bce5434295f38fae7136463422fe4fd3e0e89b98062/charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a", size = 129418, upload-time = "2025-10-14T04:42:32.879Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/97/45/4b3a1239bbacd321068ea6e7ac28875b03ab8bc0aa0966452db17cd36714/charset_normalizer-3.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794", size = 208091, upload-time = "2025-10-14T04:41:13.346Z" }, - { url = "https://files.pythonhosted.org/packages/7d/62/73a6d7450829655a35bb88a88fca7d736f9882a27eacdca2c6d505b57e2e/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed", size = 147936, upload-time = "2025-10-14T04:41:14.461Z" }, - { url = "https://files.pythonhosted.org/packages/89/c5/adb8c8b3d6625bef6d88b251bbb0d95f8205831b987631ab0c8bb5d937c2/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72", size = 144180, upload-time = "2025-10-14T04:41:15.588Z" }, - { url = "https://files.pythonhosted.org/packages/91/ed/9706e4070682d1cc219050b6048bfd293ccf67b3d4f5a4f39207453d4b99/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328", size = 161346, upload-time = "2025-10-14T04:41:16.738Z" }, - { url = "https://files.pythonhosted.org/packages/d5/0d/031f0d95e4972901a2f6f09ef055751805ff541511dc1252ba3ca1f80cf5/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede", size = 158874, upload-time = "2025-10-14T04:41:17.923Z" }, - { url = "https://files.pythonhosted.org/packages/f5/83/6ab5883f57c9c801ce5e5677242328aa45592be8a00644310a008d04f922/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894", size = 153076, upload-time = "2025-10-14T04:41:19.106Z" }, - { url = "https://files.pythonhosted.org/packages/75/1e/5ff781ddf5260e387d6419959ee89ef13878229732732ee73cdae01800f2/charset_normalizer-3.4.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1", size = 150601, upload-time = "2025-10-14T04:41:20.245Z" }, - { url = "https://files.pythonhosted.org/packages/d7/57/71be810965493d3510a6ca79b90c19e48696fb1ff964da319334b12677f0/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490", size = 150376, upload-time = "2025-10-14T04:41:21.398Z" }, - { url = "https://files.pythonhosted.org/packages/e5/d5/c3d057a78c181d007014feb7e9f2e65905a6c4ef182c0ddf0de2924edd65/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44", size = 144825, upload-time = "2025-10-14T04:41:22.583Z" }, - { url = "https://files.pythonhosted.org/packages/e6/8c/d0406294828d4976f275ffbe66f00266c4b3136b7506941d87c00cab5272/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133", size = 162583, upload-time = "2025-10-14T04:41:23.754Z" }, - { url = "https://files.pythonhosted.org/packages/d7/24/e2aa1f18c8f15c4c0e932d9287b8609dd30ad56dbe41d926bd846e22fb8d/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3", size = 150366, upload-time = "2025-10-14T04:41:25.27Z" }, - { url = "https://files.pythonhosted.org/packages/e4/5b/1e6160c7739aad1e2df054300cc618b06bf784a7a164b0f238360721ab86/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e", size = 160300, upload-time = "2025-10-14T04:41:26.725Z" }, - { url = "https://files.pythonhosted.org/packages/7a/10/f882167cd207fbdd743e55534d5d9620e095089d176d55cb22d5322f2afd/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc", size = 154465, upload-time = "2025-10-14T04:41:28.322Z" }, - { url = "https://files.pythonhosted.org/packages/89/66/c7a9e1b7429be72123441bfdbaf2bc13faab3f90b933f664db506dea5915/charset_normalizer-3.4.4-cp313-cp313-win32.whl", hash = "sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac", size = 99404, upload-time = "2025-10-14T04:41:29.95Z" }, - { url = "https://files.pythonhosted.org/packages/c4/26/b9924fa27db384bdcd97ab83b4f0a8058d96ad9626ead570674d5e737d90/charset_normalizer-3.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14", size = 107092, upload-time = "2025-10-14T04:41:31.188Z" }, - { url = "https://files.pythonhosted.org/packages/af/8f/3ed4bfa0c0c72a7ca17f0380cd9e4dd842b09f664e780c13cff1dcf2ef1b/charset_normalizer-3.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2", size = 100408, upload-time = "2025-10-14T04:41:32.624Z" }, - { url = "https://files.pythonhosted.org/packages/2a/35/7051599bd493e62411d6ede36fd5af83a38f37c4767b92884df7301db25d/charset_normalizer-3.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd", size = 207746, upload-time = "2025-10-14T04:41:33.773Z" }, - { url = "https://files.pythonhosted.org/packages/10/9a/97c8d48ef10d6cd4fcead2415523221624bf58bcf68a802721a6bc807c8f/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb", size = 147889, upload-time = "2025-10-14T04:41:34.897Z" }, - { url = "https://files.pythonhosted.org/packages/10/bf/979224a919a1b606c82bd2c5fa49b5c6d5727aa47b4312bb27b1734f53cd/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e", size = 143641, upload-time = "2025-10-14T04:41:36.116Z" }, - { url = "https://files.pythonhosted.org/packages/ba/33/0ad65587441fc730dc7bd90e9716b30b4702dc7b617e6ba4997dc8651495/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14", size = 160779, upload-time = "2025-10-14T04:41:37.229Z" }, - { url = "https://files.pythonhosted.org/packages/67/ed/331d6b249259ee71ddea93f6f2f0a56cfebd46938bde6fcc6f7b9a3d0e09/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191", size = 159035, upload-time = "2025-10-14T04:41:38.368Z" }, - { url = "https://files.pythonhosted.org/packages/67/ff/f6b948ca32e4f2a4576aa129d8bed61f2e0543bf9f5f2b7fc3758ed005c9/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838", size = 152542, upload-time = "2025-10-14T04:41:39.862Z" }, - { url = "https://files.pythonhosted.org/packages/16/85/276033dcbcc369eb176594de22728541a925b2632f9716428c851b149e83/charset_normalizer-3.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6", size = 149524, upload-time = "2025-10-14T04:41:41.319Z" }, - { url = "https://files.pythonhosted.org/packages/9e/f2/6a2a1f722b6aba37050e626530a46a68f74e63683947a8acff92569f979a/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e", size = 150395, upload-time = "2025-10-14T04:41:42.539Z" }, - { url = "https://files.pythonhosted.org/packages/60/bb/2186cb2f2bbaea6338cad15ce23a67f9b0672929744381e28b0592676824/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c", size = 143680, upload-time = "2025-10-14T04:41:43.661Z" }, - { url = "https://files.pythonhosted.org/packages/7d/a5/bf6f13b772fbb2a90360eb620d52ed8f796f3c5caee8398c3b2eb7b1c60d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090", size = 162045, upload-time = "2025-10-14T04:41:44.821Z" }, - { url = "https://files.pythonhosted.org/packages/df/c5/d1be898bf0dc3ef9030c3825e5d3b83f2c528d207d246cbabe245966808d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152", size = 149687, upload-time = "2025-10-14T04:41:46.442Z" }, - { url = "https://files.pythonhosted.org/packages/a5/42/90c1f7b9341eef50c8a1cb3f098ac43b0508413f33affd762855f67a410e/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828", size = 160014, upload-time = "2025-10-14T04:41:47.631Z" }, - { url = "https://files.pythonhosted.org/packages/76/be/4d3ee471e8145d12795ab655ece37baed0929462a86e72372fd25859047c/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec", size = 154044, upload-time = "2025-10-14T04:41:48.81Z" }, - { url = "https://files.pythonhosted.org/packages/b0/6f/8f7af07237c34a1defe7defc565a9bc1807762f672c0fde711a4b22bf9c0/charset_normalizer-3.4.4-cp314-cp314-win32.whl", hash = "sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9", size = 99940, upload-time = "2025-10-14T04:41:49.946Z" }, - { url = "https://files.pythonhosted.org/packages/4b/51/8ade005e5ca5b0d80fb4aff72a3775b325bdc3d27408c8113811a7cbe640/charset_normalizer-3.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c", size = 107104, upload-time = "2025-10-14T04:41:51.051Z" }, - { url = "https://files.pythonhosted.org/packages/da/5f/6b8f83a55bb8278772c5ae54a577f3099025f9ade59d0136ac24a0df4bde/charset_normalizer-3.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2", size = 100743, upload-time = "2025-10-14T04:41:52.122Z" }, - { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" }, -] - [[package]] name = "colorama" version = "0.4.6" @@ -126,65 +29,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] -[[package]] -name = "crc16" -version = "0.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a6/e0/70a44c4385f2b33df82e518005aae16b5c1feaf082c73c0acebe3426fc0a/crc16-0.1.1.tar.gz", hash = "sha256:c1f86aa0390f4baf07d2631b16b979580eae1d9a973a826ce45353a22ee8d396", size = 9631, upload-time = "2011-06-04T23:20:40.62Z" } - -[[package]] -name = "crc32c" -version = "2.8" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e3/66/7e97aa77af7cf6afbff26e3651b564fe41932599bc2d3dce0b2f73d4829a/crc32c-2.8.tar.gz", hash = "sha256:578728964e59c47c356aeeedee6220e021e124b9d3e8631d95d9a5e5f06e261c", size = 48179, upload-time = "2025-10-17T06:20:13.61Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/d8/3ae227890b3be40955a7144106ef4dd97d6123a82c2a5310cdab58ca49d8/crc32c-2.8-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:36f1e03ee9e9c6938e67d3bcb60e36f260170aa5f37da1185e04ef37b56af395", size = 66380, upload-time = "2025-10-17T06:19:18.009Z" }, - { url = "https://files.pythonhosted.org/packages/bd/8b/178d3f987cd0e049b484615512d3f91f3d2caeeb8ff336bb5896ae317438/crc32c-2.8-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b2f3226b94b85a8dd9b3533601d7a63e9e3e8edf03a8a169830ee8303a199aeb", size = 63048, upload-time = "2025-10-17T06:19:18.853Z" }, - { url = "https://files.pythonhosted.org/packages/f2/a1/48145ae2545ebc0169d3283ebe882da580ea4606bfb67cf4ca922ac3cfc3/crc32c-2.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6e08628bc72d5b6bc8e0730e8f142194b610e780a98c58cb6698e665cb885a5b", size = 61530, upload-time = "2025-10-17T06:19:19.974Z" }, - { url = "https://files.pythonhosted.org/packages/06/4b/cf05ed9d934cc30e5ae22f97c8272face420a476090e736615d9a6b53de0/crc32c-2.8-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:086f64793c5ec856d1ab31a026d52ad2b895ac83d7a38fce557d74eb857f0a82", size = 80001, upload-time = "2025-10-17T06:19:20.784Z" }, - { url = "https://files.pythonhosted.org/packages/15/ab/4b04801739faf36345f6ba1920be5b1c70282fec52f8280afd3613fb13e2/crc32c-2.8-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcf72ee7e0135b3d941c34bb2c26c3fc6bc207106b49fd89aaafaeae223ae209", size = 81543, upload-time = "2025-10-17T06:19:21.557Z" }, - { url = "https://files.pythonhosted.org/packages/a9/1b/6e38dde5bfd2ea69b7f2ab6ec229fcd972a53d39e2db4efe75c0ac0382ce/crc32c-2.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8a717dd9c3fd777d9bc6603717eae172887d402c4ab589d124ebd0184a83f89e", size = 80644, upload-time = "2025-10-17T06:19:22.325Z" }, - { url = "https://files.pythonhosted.org/packages/ce/45/012176ffee90059ae8ec7131019c71724ea472aa63e72c0c8edbd1fad1d7/crc32c-2.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0450bb845b3c3c7b9bdc0b4e95620ec9a40824abdc8c86d6285c919a90743c1a", size = 79919, upload-time = "2025-10-17T06:19:23.101Z" }, - { url = "https://files.pythonhosted.org/packages/f0/2b/f557629842f9dec2b3461cb3a0d854bb586ec45b814cea58b082c32f0dde/crc32c-2.8-cp313-cp313-win32.whl", hash = "sha256:765d220bfcbcffa6598ac11eb1e10af0ee4802b49fe126aa6bf79f8ddb9931d1", size = 64896, upload-time = "2025-10-17T06:19:23.88Z" }, - { url = "https://files.pythonhosted.org/packages/d0/db/fd0f698c15d1e21d47c64181a98290665a08fcbb3940cd559e9c15bda57e/crc32c-2.8-cp313-cp313-win_amd64.whl", hash = "sha256:171ff0260d112c62abcce29332986950a57bddee514e0a2418bfde493ea06bb3", size = 66646, upload-time = "2025-10-17T06:19:24.702Z" }, - { url = "https://files.pythonhosted.org/packages/db/b9/8e5d7054fe8e7eecab10fd0c8e7ffb01439417bdb6de1d66a81c38fc4a20/crc32c-2.8-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b977a32a3708d6f51703c8557008f190aaa434d7347431efb0e86fcbe78c2a50", size = 66203, upload-time = "2025-10-17T06:19:25.872Z" }, - { url = "https://files.pythonhosted.org/packages/55/5f/cc926c70057a63cc0c98a3c8a896eb15fc7e74d3034eadd53c94917c6cc3/crc32c-2.8-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7399b01db4adaf41da2fb36fe2408e75a8d82a179a9564ed7619412e427b26d6", size = 62956, upload-time = "2025-10-17T06:19:26.652Z" }, - { url = "https://files.pythonhosted.org/packages/a1/8a/0660c44a2dd2cb6ccbb529eb363b9280f5c766f1017bc8355ed8d695bd94/crc32c-2.8-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4379f73f9cdad31958a673d11a332ec725ca71572401ca865867229f5f15e853", size = 61442, upload-time = "2025-10-17T06:19:27.74Z" }, - { url = "https://files.pythonhosted.org/packages/f5/5a/6108d2dfc0fe33522ce83ba07aed4b22014911b387afa228808a278e27cd/crc32c-2.8-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e68264555fab19bab08331550dab58573e351a63ed79c869d455edd3b0aa417", size = 79109, upload-time = "2025-10-17T06:19:28.535Z" }, - { url = "https://files.pythonhosted.org/packages/84/1e/c054f9e390090c197abf3d2936f4f9effaf0c6ee14569ae03d6ddf86958a/crc32c-2.8-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b48f2486727b8d0e7ccbae4a34cb0300498433d2a9d6b49cb13cb57c2e3f19cb", size = 80987, upload-time = "2025-10-17T06:19:29.305Z" }, - { url = "https://files.pythonhosted.org/packages/c8/ad/1650e5c3341e4a485f800ea83116d72965030c5d48ccc168fcc685756e4d/crc32c-2.8-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ecf123348934a086df8c8fde7f9f2d716d523ca0707c5a1367b8bb00d8134823", size = 79994, upload-time = "2025-10-17T06:19:30.109Z" }, - { url = "https://files.pythonhosted.org/packages/d7/3b/f2ed924b177729cbb2ab30ca2902abff653c31d48c95e7b66717a9ca9fcc/crc32c-2.8-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e636ac60f76de538f7a2c0d0f3abf43104ee83a8f5e516f6345dc283ed1a4df7", size = 79046, upload-time = "2025-10-17T06:19:30.894Z" }, - { url = "https://files.pythonhosted.org/packages/4b/80/413b05ee6ace613208b31b3670c3135ee1cf451f0e72a9c839b4946acc04/crc32c-2.8-cp313-cp313t-win32.whl", hash = "sha256:8dd4a19505e0253892e1b2f1425cc3bd47f79ae5a04cb8800315d00aad7197f2", size = 64837, upload-time = "2025-10-17T06:19:32.03Z" }, - { url = "https://files.pythonhosted.org/packages/3b/1b/85eddb6ac5b38496c4e35c20298aae627970c88c3c624a22ab33e84f16c7/crc32c-2.8-cp313-cp313t-win_amd64.whl", hash = "sha256:4bb18e4bd98fb266596523ffc6be9c5b2387b2fa4e505ec56ca36336f49cb639", size = 66574, upload-time = "2025-10-17T06:19:33.143Z" }, - { url = "https://files.pythonhosted.org/packages/aa/df/50e9079b532ff53dbfc0e66eed781374bd455af02ed5df8b56ad538de4ff/crc32c-2.8-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3a3b2e4bcf7b3ee333050e7d3ff38e2ba46ea205f1d73d8949b248aaffe937ac", size = 66399, upload-time = "2025-10-17T06:19:34.279Z" }, - { url = "https://files.pythonhosted.org/packages/5a/2e/67e3b0bc3d30e46ea5d16365cc81203286387671e22f2307eb41f19abb9c/crc32c-2.8-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:445e559e66dff16be54f8a4ef95aa6b01db799a639956d995c5498ba513fccc2", size = 63044, upload-time = "2025-10-17T06:19:35.062Z" }, - { url = "https://files.pythonhosted.org/packages/36/ea/1723b17437e4344ed8d067456382ecb1f5b535d83fdc5aaebab676c6d273/crc32c-2.8-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bf3040919e17afa5782e01b1875d6a05f44b8f19c05f211d8b9f8a1deb8bbd9c", size = 61541, upload-time = "2025-10-17T06:19:36.204Z" }, - { url = "https://files.pythonhosted.org/packages/4c/6a/cbec8a235c5b46a01f319939b538958662159aec0ed3a74944e3a6de21f1/crc32c-2.8-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5607ab8221e1ffd411f64aa40dbb6850cf06dd2908c9debd05d371e1acf62ff3", size = 80139, upload-time = "2025-10-17T06:19:37.351Z" }, - { url = "https://files.pythonhosted.org/packages/21/31/d096722fe74b692d6e8206c27da1ea5f6b2a12ff92c54a62a6ba2f376254/crc32c-2.8-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f5db4f16816926986d3c94253314920689706ae13a9bf4888b47336c6735ce", size = 81736, upload-time = "2025-10-17T06:19:38.16Z" }, - { url = "https://files.pythonhosted.org/packages/f6/a2/f75ef716ff7e3c22f385ba6ef30c5de80c19a21ebe699dc90824a1903275/crc32c-2.8-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:70b0153c4d418b673309d3529334d117e1074c4a3b2d7f676e430d72c14de67b", size = 80795, upload-time = "2025-10-17T06:19:38.948Z" }, - { url = "https://files.pythonhosted.org/packages/d8/94/6d647a12d96ab087d9b8eacee3da073f981987827d57c7072f89ffc7b6cd/crc32c-2.8-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c8933531442042438753755a5c8a9034e4d88b01da9eb796f7e151b31a7256c", size = 80042, upload-time = "2025-10-17T06:19:39.725Z" }, - { url = "https://files.pythonhosted.org/packages/cd/dc/32b8896b40a0afee7a3c040536d0da5a73e68df2be9fadd21770fd158e16/crc32c-2.8-cp314-cp314-win32.whl", hash = "sha256:cdc83a3fe6c4e5df9457294cfd643de7d95bd4e9382c1dd6ed1e0f0f9169172c", size = 64914, upload-time = "2025-10-17T06:19:40.527Z" }, - { url = "https://files.pythonhosted.org/packages/f2/b4/4308b27d307e8ecaf8dd1dcc63bbb0e47ae1826d93faa3e62d1ee00ee2d5/crc32c-2.8-cp314-cp314-win_amd64.whl", hash = "sha256:509e10035106df66770fe24b9eb8d9e32b6fb967df17744402fb67772d8b2bc7", size = 66723, upload-time = "2025-10-17T06:19:42.449Z" }, - { url = "https://files.pythonhosted.org/packages/90/d5/a19d2489fa997a143bfbbf971a5c9a43f8b1ba9e775b1fb362d8fb15260c/crc32c-2.8-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:864359a39777a07b09b28eb31337c0cc603d5c1bf0fc328c3af736a8da624ec0", size = 66201, upload-time = "2025-10-17T06:19:43.273Z" }, - { url = "https://files.pythonhosted.org/packages/98/c2/5f82f22d2c1242cb6f6fe92aa9a42991ebea86de994b8f9974d9c1d128e2/crc32c-2.8-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:14511d7cfc5d9f5e1a6c6b64caa6225c2bdc1ed00d725e9a374a3e84073ce180", size = 62956, upload-time = "2025-10-17T06:19:44.099Z" }, - { url = "https://files.pythonhosted.org/packages/9b/61/3d43d33489cf974fb78bfb3500845770e139ae6d1d83473b660bd8f79a6c/crc32c-2.8-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:918b7999b52b5dcbcea34081e9a02d46917d571921a3f209956a9a429b2e06e5", size = 61443, upload-time = "2025-10-17T06:19:44.89Z" }, - { url = "https://files.pythonhosted.org/packages/52/6d/f306ce64a352a3002f76b0fc88a1373f4541f9d34fad3668688610bab14b/crc32c-2.8-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cc445da03fc012a5a03b71da1df1b40139729e6a5571fd4215ab40bfb39689c7", size = 79106, upload-time = "2025-10-17T06:19:45.688Z" }, - { url = "https://files.pythonhosted.org/packages/a5/b7/1f74965dd7ea762954a69d172dfb3a706049c84ffa45d31401d010a4a126/crc32c-2.8-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e3dde2ec59a8a830511d72a086ead95c0b0b7f0d418f93ea106244c5e77e350", size = 80983, upload-time = "2025-10-17T06:19:46.792Z" }, - { url = "https://files.pythonhosted.org/packages/1b/50/af93f0d91ccd61833ce77374ebfbd16f5805f5c17d18c6470976d9866d76/crc32c-2.8-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:61d51681a08b6a2a2e771b7f0cd1947fb87cb28f38ed55a01cb7c40b2ac4cdd8", size = 80009, upload-time = "2025-10-17T06:19:47.619Z" }, - { url = "https://files.pythonhosted.org/packages/ee/fa/94f394beb68a88258af694dab2f1284f55a406b615d7900bdd6235283bc4/crc32c-2.8-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:67c0716c3b1a02d5235be649487b637eed21f2d070f2b3f63f709dcd2fefb4c7", size = 79066, upload-time = "2025-10-17T06:19:48.409Z" }, - { url = "https://files.pythonhosted.org/packages/91/c6/a6050e0c64fd73c67a97da96cb59f08b05111e00b958fb87ecdce99f17ac/crc32c-2.8-cp314-cp314t-win32.whl", hash = "sha256:2e8fe863fbbd8bdb6b414a2090f1b0f52106e76e9a9c96a413495dbe5ebe492a", size = 64869, upload-time = "2025-10-17T06:19:49.197Z" }, - { url = "https://files.pythonhosted.org/packages/08/1f/c7735034e401cb1ea14f996a224518e3a3fa9987cb13680e707328a7d779/crc32c-2.8-cp314-cp314t-win_amd64.whl", hash = "sha256:20a9cfb897693eb6da19e52e2a7be2026fd4d9fc8ae318f086c0d71d5dd2d8e0", size = 66633, upload-time = "2025-10-17T06:19:50.003Z" }, -] - -[[package]] -name = "idna" -version = "3.11" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, -] - [[package]] name = "iniconfig" version = "2.3.0" @@ -254,32 +98,15 @@ wheels = [ ] [[package]] -name = "pytonlib" -version = "0.0.68" +name = "pytest-asyncio" +version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "crc16" }, - { name = "requests" }, - { name = "tvm-valuetypes" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/17/9b/9a3d40841dc3a460e308b7a284152b00f4378c57fcfd57ef5b09ca21e8d8/pytonlib-0.0.68.tar.gz", hash = "sha256:a2414cd2692388edf9114a00dc81cbae38c7197a67042844f886275297b82279", size = 27425030, upload-time = "2025-11-03T08:05:30.977Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/c1/dc76ccabe5818dccbb131ef6e069400b3fea39f14f5a5f3ff07761f6731d/pytonlib-0.0.68-py3-none-any.whl", hash = "sha256:53ced393bbac68c6a294c73e74b8f060c0a4a2349d8e74622ea521655645d010", size = 27531020, upload-time = "2025-11-03T08:05:28.243Z" }, -] - -[[package]] -name = "requests" -version = "2.32.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "charset-normalizer" }, - { name = "idna" }, - { name = "urllib3" }, + { name = "pytest" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } +sdist = { url = "https://files.pythonhosted.org/packages/90/2c/8af215c0f776415f3590cac4f9086ccefd6fd463befeae41cd4d3f193e5a/pytest_asyncio-1.3.0.tar.gz", hash = "sha256:d7f52f36d231b80ee124cd216ffb19369aa168fc10095013c6b014a34d3ee9e5", size = 50087, upload-time = "2025-11-10T16:07:47.256Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, + { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, ] [[package]] @@ -308,21 +135,13 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/30/bd/4168a751ddbbf43e86544b4de8b5c3b7be8d7167a2a5cb977d274e04f0a1/ruff-0.14.4-py3-none-win_arm64.whl", hash = "sha256:dd09c292479596b0e6fec8cd95c65c3a6dc68e9ad17b8f2382130f87ff6a75bb", size = 12663065, upload-time = "2025-11-06T22:07:42.603Z" }, ] -[[package]] -name = "setuptools" -version = "80.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" }, -] - [[package]] name = "ton" version = "0.1.0" source = { virtual = "." } dependencies = [ { name = "pytest" }, + { name = "pytest-asyncio" }, { name = "tontester" }, ] @@ -335,6 +154,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "pytest", specifier = ">=9.0.1" }, + { name = "pytest-asyncio" }, { name = "tontester", editable = "test/tontester" }, ] @@ -348,35 +168,3 @@ dev = [ name = "tontester" version = "0.1.0" source = { editable = "test/tontester" } -dependencies = [ - { name = "pytonlib" }, - { name = "setuptools" }, -] - -[package.metadata] -requires-dist = [ - { name = "pytonlib", specifier = ">=0.0.68" }, - { name = "setuptools", specifier = "<81" }, -] - -[[package]] -name = "tvm-valuetypes" -version = "0.0.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "bitarray" }, - { name = "crc32c" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/96/8c/6e856cf86ab94363333d3e3e1a3754f0c56694fab7a7c8b328493eaef00d/tvm_valuetypes-0.0.12.tar.gz", hash = "sha256:1f9304126be99e30ecb827e68e6e36e1620b8dc6a781dca1b0fdcc1e31d70983", size = 9044, upload-time = "2023-07-18T19:28:37.89Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/58/d98de1ffd588dcf5cda8c9d105b5949aa66c69cb3af7d0a25bae679536c1/tvm_valuetypes-0.0.12-py3-none-any.whl", hash = "sha256:e4af62baebdcc8864a1f4aac40082f3221f1e4232f79b8dfb7411d7c78856880", size = 9471, upload-time = "2023-07-18T19:28:36.304Z" }, -] - -[[package]] -name = "urllib3" -version = "2.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, -] From 1c95ae0e6c1a44de3d6e0e80e512df100a5ce499 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Tue, 25 Nov 2025 23:08:12 +0400 Subject: [PATCH 490/524] codex via openapi key (#1920) --- .github/workflows/codex-review.yml | 74 ++++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 10 deletions(-) diff --git a/.github/workflows/codex-review.yml b/.github/workflows/codex-review.yml index 1806566336..151d22acb4 100644 --- a/.github/workflows/codex-review.yml +++ b/.github/workflows/codex-review.yml @@ -4,13 +4,9 @@ on: pull_request_target: types: [opened, reopened, synchronize] -permissions: - contents: read - issues: write - pull-requests: write - jobs: - request-codex-review: + codex: + # Only run Codex for PRs authored by allowed users if: | github.event.pull_request.user.login == 'EmelyanenkoK' || github.event.pull_request.user.login == 'tolya-yanot' || @@ -31,15 +27,73 @@ jobs: github.event.pull_request.user.login == 'tvorogme' || github.event.pull_request.user.login == 'krigga' runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + outputs: + final_message: ${{ steps.run_codex.outputs['final-message'] }} + + steps: + # 1. Checkout the trusted base commit + - name: Checkout base branch (trusted) + uses: actions/checkout@v5 + with: + ref: ${{ github.event.pull_request.base.sha }} + fetch-depth: 0 + persist-credentials: false + + # 2. Fetch PR head as a separate local branch, without checking it out + - name: Fetch PR head + run: | + git fetch origin \ + pull/${{ github.event.pull_request.number }}/head:pr-${{ github.event.pull_request.number }} + + # Optional diagnostics + - name: Show diff summary + run: | + git diff --stat \ + ${{ github.event.pull_request.base.sha }} pr-${{ github.event.pull_request.number }} + + # 3. Run Codex safely + - name: Run Codex + id: run_codex + uses: openai/codex-action@02e7b2943818fbac9f077c3d1249a198ab358352 + with: + # IMPORTANT: this is safe only because: + # - workflow file lives in base repo (attacker cannot change it) + # - we do not run arbitrary PR code, only git commands + openai-api-key: ${{ secrets.OPENAI_API_KEY }} + prompt: | + This is PR #${{ github.event.pull_request.number }} for ${{ github.repository }}. + + Review ONLY the changes introduced by this PR. + + Diff range: + ${{ github.event.pull_request.base.sha }}...pr-${{ github.event.pull_request.number }} + + Pull request title and body: + ---- + ${{ github.event.pull_request.title }} + ${{ github.event.pull_request.body }} + + post_feedback: + runs-on: ubuntu-latest + needs: codex + if: needs.codex.outputs.final_message != '' + permissions: + issues: write + pull-requests: write steps: - - name: Comment to trigger Codex + - name: Report Codex feedback uses: actions/github-script@v7 + env: + CODEX_FINAL_MESSAGE: ${{ needs.codex.outputs.final_message }} with: github-token: ${{ github.token }} script: | await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, - issue_number: context.issue.number, - body: "@codex review" - }) + issue_number: context.payload.pull_request.number, + body: process.env.CODEX_FINAL_MESSAGE, + }); From e7db1ce5e9c888fe7f3242dc736aeba83ed6981d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 26 Nov 2025 12:29:25 +0300 Subject: [PATCH 491/524] Various bugfixes in node (#1922) * Fix constructing LoadedCell * Remove unused VirtualizationParameters * Fix cleanup of fast-sync member certificates in node config * Fix checking cancellation in boc.h * Add mc_block_id to collation stats --- crypto/vm/boc.h | 3 + crypto/vm/cells/CellSlice.cpp | 4 +- crypto/vm/cells/VirtualizationParameters.h | 72 ---------------------- tl/generate/scheme/ton_api.tl | 3 +- validator-engine/validator-engine.cpp | 2 +- validator/impl/collator.cpp | 1 + validator/interfaces/validator-manager.h | 3 +- 7 files changed, 11 insertions(+), 77 deletions(-) delete mode 100644 crypto/vm/cells/VirtualizationParameters.h diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 28f908b3df..c71da63b58 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -219,6 +219,7 @@ class BagOfCellsLogger { log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD); last_speed_log_ = td::Timestamp::now(); processed_cells_ = 0; + last_token_check_ = 0; timer_ = {}; stage_ = std::move(stage); } @@ -235,7 +236,9 @@ class BagOfCellsLogger { double period = td::Timestamp::now().at() - last_speed_log_.at(); LOG(WARNING) << "serializer: " << stage_ << " " << (double)processed_cells_ / period << " cells/s"; + TRY_STATUS(cancellation_token_.check()); processed_cells_ = 0; + last_token_check_ = 0; last_speed_log_ = td::Timestamp::now(); log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD); } diff --git a/crypto/vm/cells/CellSlice.cpp b/crypto/vm/cells/CellSlice.cpp index e8d9d58a2c..8a0a248b71 100644 --- a/crypto/vm/cells/CellSlice.cpp +++ b/crypto/vm/cells/CellSlice.cpp @@ -74,7 +74,7 @@ CellSlice::CellSlice(NoVmOrd, Ref ref) : CellSlice(load_cell_nothrow(std:: } CellSlice::CellSlice(NoVmSpec, Ref ref) : CellSlice(load_cell_nothrow(std::move(ref), 2)) { } -CellSlice::CellSlice(Ref ref) : CellSlice(VirtualCell::LoadedCell{std::move(ref), {}, {}}) { +CellSlice::CellSlice(Ref ref) : CellSlice(VirtualCell::LoadedCell{std::move(ref), Cell::max_level, {}}) { } CellSlice::CellSlice(const CellSlice& cs) = default; @@ -100,7 +100,7 @@ bool CellSlice::load(NoVmSpec, Ref cell_ref) { return load(load_cell_nothrow(std::move(cell_ref), 2)); } bool CellSlice::load(Ref dc_ref) { - return load(VirtualCell::LoadedCell{std::move(dc_ref), {}, {}}); + return load(VirtualCell::LoadedCell{std::move(dc_ref), Cell::max_level, {}}); } /* diff --git a/crypto/vm/cells/VirtualizationParameters.h b/crypto/vm/cells/VirtualizationParameters.h deleted file mode 100644 index fb25f8fa88..0000000000 --- a/crypto/vm/cells/VirtualizationParameters.h +++ /dev/null @@ -1,72 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -#include - -#include "td/utils/int_types.h" -#include "td/utils/logging.h" - -namespace vm { -namespace detail { -class VirtualizationParameters { - public: - static constexpr td::uint8 max_level() { - return std::numeric_limits::max(); - } - - VirtualizationParameters() = default; - - VirtualizationParameters(td::uint8 level, td::uint8 virtualization) : level_(level), virtualization_(virtualization) { - CHECK(virtualization_ != 0 || empty()); - } - - bool empty() const { - return level_ == max_level() && virtualization_ == 0; - } - - VirtualizationParameters apply(VirtualizationParameters outer) const { - if (outer.level_ >= level_) { - return *this; - } - CHECK(virtualization_ <= outer.virtualization_); - return {outer.level_, outer.virtualization_}; - } - - td::uint8 get_level() const { - return level_; - } - - td::uint8 get_virtualization() const { - return virtualization_; - } - - bool operator==(const VirtualizationParameters &other) const { - return level_ == other.level_ && virtualization_ == other.virtualization_; - } - - private: - td::uint8 level_ = max_level(); - td::uint8 virtualization_ = 0; -}; -inline td::StringBuilder &operator<<(td::StringBuilder &sb, const VirtualizationParameters &virt) { - return sb << "{level: " << virt.get_level() << ", virtualization: " << virt.get_virtualization() << "}"; -} -} // namespace detail -} // namespace vm diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index c0e57cb813..09ae4fa04f 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -960,7 +960,8 @@ validatorStats.blockStats.neighborStats shard:tonNode.shardId is_trivial:Bool is validatorStats.blockStats ext_msgs:validatorStats.blockStats.extMsgsStats transactions:int shard_configuration:(vector tonNode.blockIdExt) old_out_msg_queue_size:long new_out_msg_queue_size:long msg_queue_cleaned:int - neighbors:(vector validatorStats.blockStats.neighborStats) = validatorStats.BlockStats; + neighbors:(vector validatorStats.blockStats.neighborStats) + mc_block_id:tonNode.blockIdExt = validatorStats.BlockStats; validatorStats.storageStatCacheStats small_cnt:long small_cells:long hit_cnt:long hit_cells:long miss_cnt:long miss_cells:long = validatorStats.StorageStatCacheStats; diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 2e5532dec1..8f0636ace3 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1458,7 +1458,7 @@ void ValidatorEngine::alarm() { need_write = true; std::erase_if(config_.fast_sync_member_certificates, [&](const std::pair &e) { - return !fs_to_del.contains(e.first); + return fs_to_del.contains(e.first); }); } } diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 00f9cb5d10..2d2954747d 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -6704,6 +6704,7 @@ void Collator::finalize_stats() { stats_.neighbors.push_back(std::move(neighbors_stats[i])); } } + stats_.mc_block_id = mc_block_id_; } /** diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index d6257f7c80..34e1210e7b 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -121,6 +121,7 @@ struct CollationStats { } }; std::vector neighbors; + BlockIdExt mc_block_id; double load_fraction_queue_cleanup = -1.0; double load_fraction_dispatch = -1.0; @@ -169,7 +170,7 @@ struct CollationStats { create_tl_object(ext_msgs_total, ext_msgs_filtered, ext_msgs_accepted, ext_msgs_rejected), transactions, std::move(shards_obj), old_out_msg_queue_size, new_out_msg_queue_size, msg_queue_cleaned, - std::move(neighbors_obj)); + std::move(neighbors_obj), create_tl_block_id(mc_block_id)); return create_tl_object( create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes, actual_collated_data_bytes, attempt, self.bits256_value(), is_validator, total_time, work_time.total.real, From d840e36857ac0e60aefd8438631808c252ae7872 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Wed, 26 Nov 2025 21:56:52 +0400 Subject: [PATCH 492/524] make debug output only to vm_log (#1921) * make debug output part of vm_log too * make debug output only to vm_log * remove comments --- crypto/vm/debugops.cpp | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/crypto/vm/debugops.cpp b/crypto/vm/debugops.cpp index 29a8345a61..e87d545c84 100644 --- a/crypto/vm/debugops.cpp +++ b/crypto/vm/debugops.cpp @@ -76,16 +76,20 @@ int exec_dump_stack(VmState* st) { } Stack& stack = st->get_stack(); int d = stack.depth(); - std::cerr << "#DEBUG#: stack(" << d << " values) : "; + + std::ostringstream os; + os << "#DEBUG#: stack(" << d << " values) : "; if (d > 255) { - std::cerr << "... "; + os << "... "; d = 255; } for (int i = d; i > 0; i--) { - stack[i - 1].print_list(std::cerr); - std::cerr << ' '; + stack[i - 1].print_list(os); + os << ' '; } - std::cerr << std::endl; + + VM_LOG(st) << os.str(); + return 0; } @@ -97,11 +101,13 @@ int exec_dump_value(VmState* st, unsigned arg) { } Stack& stack = st->get_stack(); if ((int)arg < stack.depth()) { - std::cerr << "#DEBUG#: s" << arg << " = "; - stack[arg].print_list(std::cerr); - std::cerr << std::endl; + std::ostringstream os; + os << "#DEBUG#: s" << arg << " = "; + stack[arg].print_list(os); + + VM_LOG(st) << os.str(); } else { - std::cerr << "#DEBUG#: s" << arg << " is absent" << std::endl; + VM_LOG(st) << "#DEBUG#: s" << arg << " is absent"; } return 0; } @@ -127,16 +133,16 @@ int exec_dump_string(VmState* st) { cs.write().fetch_bytes(tmp, cnt); std::string s{tmp, tmp + cnt}; - std::cerr << "#DEBUG#: " << s << std::endl; + VM_LOG(st) << "#DEBUG#: " << s; } else { - std::cerr << "#DEBUG#: slice contains not valid bits count" << std::endl; + VM_LOG(st) << "#DEBUG#: slice contains not valid bits count"; } } else { - std::cerr << "#DEBUG#: is not a slice" << std::endl; + VM_LOG(st) << "#DEBUG#: is not a slice"; } } else { - std::cerr << "#DEBUG#: s0 is absent" << std::endl; + VM_LOG(st) << "#DEBUG#: s0 is absent"; } return 0; From ddb173b16f4ff8fb314175b9751720dbfc79e77e Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Fri, 28 Nov 2025 12:32:23 -0500 Subject: [PATCH 493/524] Refactor CMake tonlibjson declaration (#1927) * Refactor CMake tonlibjson declaration This will allow us to do white-box testing of the library in the future. * Add test that ensures installed tonlib can be imported and used ... at least on Linux and MacOS. --- .../build-ton-linux-arm64-shared.yml | 6 +- .../build-ton-linux-x86-64-shared.yml | 4 ++ .../build-ton-macos-15-arm64-shared.yml | 4 ++ .../build-ton-macos-15-x86-64-shared.yml | 4 ++ .../build-ton-macos-arm64-shared.yml | 4 ++ assembly/native/build-macos-shared.sh | 5 +- assembly/native/build-ubuntu-shared.sh | 5 +- test/tonlib/integration/CMakeLists.txt | 10 +++ test/tonlib/integration/test_integration.sh | 31 +++++++++ .../integration/tonlibjson_consumer.cpp | 8 +++ tonlib/CMakeLists.txt | 69 +++++++++++-------- 11 files changed, 118 insertions(+), 32 deletions(-) create mode 100644 test/tonlib/integration/CMakeLists.txt create mode 100755 test/tonlib/integration/test_integration.sh create mode 100644 test/tonlib/integration/tonlibjson_consumer.cpp diff --git a/.github/workflows/build-ton-linux-arm64-shared.yml b/.github/workflows/build-ton-linux-arm64-shared.yml index ec3f10e2ae..1c5ffae8f0 100644 --- a/.github/workflows/build-ton-linux-arm64-shared.yml +++ b/.github/workflows/build-ton-linux-arm64-shared.yml @@ -63,5 +63,9 @@ jobs: - name: Run Tests run: | - cd build + cd build ctest --output-on-failure --timeout 1800 + + - name: Run tonlib integration test + run: | + ./test/tonlib/integration/test_integration.sh diff --git a/.github/workflows/build-ton-linux-x86-64-shared.yml b/.github/workflows/build-ton-linux-x86-64-shared.yml index fffaae1560..87b9dc15a7 100644 --- a/.github/workflows/build-ton-linux-x86-64-shared.yml +++ b/.github/workflows/build-ton-linux-x86-64-shared.yml @@ -78,3 +78,7 @@ jobs: echo $script uv run $script done + + - name: Run tonlib integration test + run: | + ./test/tonlib/integration/test_integration.sh diff --git a/.github/workflows/build-ton-macos-15-arm64-shared.yml b/.github/workflows/build-ton-macos-15-arm64-shared.yml index 77953bc4fe..0105db8980 100644 --- a/.github/workflows/build-ton-macos-15-arm64-shared.yml +++ b/.github/workflows/build-ton-macos-15-arm64-shared.yml @@ -43,3 +43,7 @@ jobs: run: | cd build ctest --output-on-failure --timeout 1800 + + - name: Run tonlib integration test + run: | + ./test/tonlib/integration/test_integration.sh diff --git a/.github/workflows/build-ton-macos-15-x86-64-shared.yml b/.github/workflows/build-ton-macos-15-x86-64-shared.yml index 506b356e85..993dccdcac 100644 --- a/.github/workflows/build-ton-macos-15-x86-64-shared.yml +++ b/.github/workflows/build-ton-macos-15-x86-64-shared.yml @@ -43,3 +43,7 @@ jobs: run: | cd build ctest --output-on-failure --timeout 1800 + + - name: Run tonlib integration test + run: | + ./test/tonlib/integration/test_integration.sh diff --git a/.github/workflows/build-ton-macos-arm64-shared.yml b/.github/workflows/build-ton-macos-arm64-shared.yml index e9c70f15d4..4a8fa89ebb 100644 --- a/.github/workflows/build-ton-macos-arm64-shared.yml +++ b/.github/workflows/build-ton-macos-arm64-shared.yml @@ -43,3 +43,7 @@ jobs: run: | cd build ctest --output-on-failure --timeout 1800 + + - name: Run tonlib integration test + run: | + ./test/tonlib/integration/test_integration.sh diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index 13e6208345..2d0638c1cc 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -83,7 +83,8 @@ cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \ -DCMAKE_SYSROOT=$(xcrun --show-sdk-path) \ -DLZ4_FOUND=1 \ -DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a \ --DLZ4_INCLUDE_DIRS=$lz4Path/lib +-DLZ4_INCLUDE_DIRS=$lz4Path/lib \ +-DCMAKE_INSTALL_PREFIX="$(pwd)/install" test $? -eq 0 || { echo "Can't configure ton"; exit 1; } @@ -91,7 +92,7 @@ if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \ - http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver all-tests + http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver all-tests install test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli blockchain-explorer \ diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index 22756b68d7..4db66a11e7 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -56,7 +56,8 @@ cmake -GNinja -DTON_USE_JEMALLOC=ON .. \ -DCMAKE_BUILD_TYPE=Release \ -DOPENSSL_ROOT_DIR=$opensslPath \ -DOPENSSL_INCLUDE_DIR=$opensslPath/include \ --DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.so +-DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.so \ +-DCMAKE_INSTALL_PREFIX="$(pwd)/install" test $? -eq 0 || { echo "Can't configure ton"; exit 1; } @@ -65,7 +66,7 @@ if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy dht-ping-servers dht-resolve \ - adnl-proxy create-state emulator proxy-liteserver all-tests + adnl-proxy create-state emulator proxy-liteserver all-tests install test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ diff --git a/test/tonlib/integration/CMakeLists.txt b/test/tonlib/integration/CMakeLists.txt new file mode 100644 index 0000000000..cb98e749a9 --- /dev/null +++ b/test/tonlib/integration/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.16) +project(TonlibConsumerTest) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +find_package(Tonlib REQUIRED) + +add_executable(tonlibjson_consumer tonlibjson_consumer.cpp) +target_link_libraries(tonlibjson_consumer PRIVATE Tonlib::TonlibJson) diff --git a/test/tonlib/integration/test_integration.sh b/test/tonlib/integration/test_integration.sh new file mode 100755 index 0000000000..84d429796b --- /dev/null +++ b/test/tonlib/integration/test_integration.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +SOURCE_DIR="$(cd "$SCRIPT_DIR/../../.." >/dev/null && pwd)" +BUILD_DIR="$SOURCE_DIR/build" +INSTALL_PREFIX="$BUILD_DIR/install" +CONSUMER_BUILD_DIR="$BUILD_DIR/test/tonlib/integration" + +if [ -d "$CONSUMER_BUILD_DIR" ]; then + rm -rf "$CONSUMER_BUILD_DIR" +fi +mkdir -p "$CONSUMER_BUILD_DIR" + +cleanup() { + rm -rf "$CONSUMER_BUILD_DIR" +} +trap cleanup EXIT + +echo "=== Building consumer project ===" +cmake -S "$SCRIPT_DIR" -B "$CONSUMER_BUILD_DIR" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="$INSTALL_PREFIX" + +cmake --build "$CONSUMER_BUILD_DIR" -j + +echo "=== Running consumer tests ===" +"$CONSUMER_BUILD_DIR/tonlibjson_consumer" +echo "tonlibjson_consumer: OK" + +echo "=== Integration test passed ===" diff --git a/test/tonlib/integration/tonlibjson_consumer.cpp b/test/tonlib/integration/tonlibjson_consumer.cpp new file mode 100644 index 0000000000..ee8528e924 --- /dev/null +++ b/test/tonlib/integration/tonlibjson_consumer.cpp @@ -0,0 +1,8 @@ +#include "tonlib/tonlib_client_json.h" + +int main() { + void* client = tonlib_client_json_create(); + tonlib_client_json_destroy(client); + + return 0; +} diff --git a/tonlib/CMakeLists.txt b/tonlib/CMakeLists.txt index a59bd9459b..62aa76f7c1 100644 --- a/tonlib/CMakeLists.txt +++ b/tonlib/CMakeLists.txt @@ -70,32 +70,50 @@ endif() add_executable(tonlib-cli tonlib/tonlib-cli.cpp) target_link_libraries(tonlib-cli tonlib tdactor tdutils terminal ton_crypto git) -if (NOT CMAKE_CROSSCOMPILING) - if (TONLIB_ENABLE_JNI) - #FIXME - #add_dependencies(tonlib tonlib_generate_java_api) - endif() +# ===== tonlibjson ===== +if (USE_EMSCRIPTEN) + set(TONLIBJSON_STATIC ON) endif() -add_library(tonlibjson_private STATIC tonlib/ClientJson.cpp tonlib/ClientJson.h) -target_include_directories(tonlibjson_private PUBLIC + +add_library(tonlibjson_public_interface INTERFACE) +target_include_directories(tonlibjson_public_interface INTERFACE + $ $ - $) -target_link_libraries(tonlibjson_private PUBLIC tonlib PRIVATE tl_tonlib_api_json) +) -set(TONLIB_JSON_HEADERS tonlib/tonlib_client_json.h) -set(TONLIB_JSON_SOURCE tonlib/tonlib_client_json.cpp) +add_library(tonlibjson_private_interface INTERFACE) +target_link_libraries(tonlibjson_private_interface INTERFACE + tonlib tl_tonlib_api_json tonlibjson_public_interface +) -include(GenerateExportHeader) -if (TONLIBJSON_STATIC OR USE_EMSCRIPTEN) - add_library(tonlibjson STATIC ${TONLIB_JSON_SOURCE}) +add_library(tonlibjson_objects OBJECT + tonlib/ClientJson.cpp + tonlib/tonlib_client_json.cpp +) +target_link_libraries(tonlibjson_objects + PRIVATE tonlibjson_private_interface + PUBLIC tonlibjson_public_interface +) + +if (TONLIBJSON_STATIC) + add_library(tonlibjson STATIC $) else() - add_library(tonlibjson SHARED ${TONLIB_JSON_SOURCE}) + add_library(tonlibjson SHARED $) endif() -if (PORTABLE AND NOT APPLE) - target_link_libraries(tonlibjson PRIVATE tonlibjson_private -static-libgcc -static-libstdc++) -else() - target_link_libraries(tonlibjson PRIVATE tonlibjson_private) +target_link_libraries(tonlibjson + PRIVATE tonlibjson_private_interface + PUBLIC tonlibjson_public_interface +) + +include(GenerateExportHeader) +generate_export_header(tonlibjson EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/tonlib/tonlibjson_export.h) +if (TONLIBJSON_STATIC) + target_compile_definitions(tonlibjson_public_interface INTERFACE TONLIBJSON_STATIC_DEFINE) +endif() + +if (NOT TONLIBJSON_STATIC AND PORTABLE AND NOT APPLE) + target_link_options(tonlibjson PRIVATE -static-libgcc -static-libstdc++) endif() if (APPLE) @@ -109,13 +127,6 @@ if (APPLE AND PORTABLE) set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-libstdc++") endif() -generate_export_header(tonlibjson EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/tonlib/tonlibjson_export.h) -if (TONLIBJSON_STATIC OR USE_EMSCRIPTEN) - target_compile_definitions(tonlibjson PUBLIC TONLIBJSON_STATIC_DEFINE) -endif() -target_include_directories(tonlibjson PUBLIC - $ - $) if (APPLE) set_target_properties(tonlibjson PROPERTIES LINK_FLAGS "-Wl,-exported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/tonlibclientjson_export_list") endif() @@ -130,7 +141,7 @@ target_link_libraries(Tonlib INTERFACE tonlib) add_library(Tonlib::Tonlib ALIAS Tonlib) -install(TARGETS tonlibjson TonlibJson EXPORT Tonlib +install(TARGETS tonlibjson_public_interface tonlibjson TonlibJson EXPORT Tonlib LIBRARY DESTINATION lib ARCHIVE DESTINATION lib RUNTIME DESTINATION bin @@ -162,6 +173,10 @@ install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/../tdutils/td/utils/port/platform.h DE install(FILES ${CMAKE_CURRENT_BINARY_DIR}/../tdutils/td/utils/config.h DESTINATION include/td/utils/) endif() +set(TONLIB_JSON_HEADERS + tonlib/tonlib_client_json.h +) + install(FILES ${TONLIB_JSON_HEADERS} ${CMAKE_CURRENT_BINARY_DIR}/tonlib/tonlibjson_export.h DESTINATION include/tonlib/) if (NOT USE_EMSCRIPTEN AND NOT TONLIBJSON_STATIC) From 1d5c0d5f2f1007c54277889dbc1fdc91a00f2ae8 Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Fri, 28 Nov 2025 12:33:25 -0500 Subject: [PATCH 494/524] [tontester] Fix type annotation in tonlibjson.py (#1926) No behavior change. --- test/tontester/src/tonlib/tonlibjson.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tontester/src/tonlib/tonlibjson.py b/test/tontester/src/tonlib/tonlibjson.py index 914e03fd9e..a73276c529 100644 --- a/test/tontester/src/tonlib/tonlibjson.py +++ b/test/tontester/src/tonlib/tonlibjson.py @@ -72,7 +72,7 @@ def __init__( tonlib_json_client_receive = tonlib.tonlib_client_json_receive tonlib_json_client_receive.restype = ctypes.c_char_p tonlib_json_client_receive.argtypes = [ctypes.c_void_p, ctypes.c_double] - self._tonlib_json_client_receive: typing.Callable[[int, int], bytes | None] = ( + self._tonlib_json_client_receive: typing.Callable[[int, float], bytes | None] = ( tonlib_json_client_receive ) From 75cbcd7e09937b09fbe2d583ed4e268b64b0aedc Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Fri, 28 Nov 2025 12:34:12 -0500 Subject: [PATCH 495/524] Make td/actor/core/ActorTypeStat.h usable as a standalone header (#1923) We will need ActorTypeStatManager::get_class_name demangler from it in the future. --- tdactor/td/actor/core/ActorTypeStat.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tdactor/td/actor/core/ActorTypeStat.h b/tdactor/td/actor/core/ActorTypeStat.h index 2b048b0d22..71d86efff6 100644 --- a/tdactor/td/actor/core/ActorTypeStat.h +++ b/tdactor/td/actor/core/ActorTypeStat.h @@ -1,8 +1,11 @@ #pragma once + #include +#include #include #include +#include "td/utils/common.h" #include "td/utils/int_types.h" #include "td/utils/port/Clocks.h" @@ -393,4 +396,4 @@ class ActorTypeStatManager { } // namespace core } // namespace actor -} // namespace td \ No newline at end of file +} // namespace td From 0926873a02e85a9b93574ed60dd762e8c2a7a771 Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Mon, 1 Dec 2025 10:36:43 -0500 Subject: [PATCH 496/524] Add `reverse-continue-to-actor-caller` GDB command (#1924) This obviously only works under rr. --- .gdbinit | 1 + CMakeLists.txt | 5 ++ assembly/gdb/rc_to_actor_caller.py | 84 +++++++++++++++++++++++++ pyproject.toml | 1 + tdactor/td/actor/core/ActorExecutor.cpp | 34 +++++++++- tdactor/td/actor/core/ActorMailbox.h | 24 ++++++- uv.lock | 11 ++++ 7 files changed, 155 insertions(+), 5 deletions(-) create mode 100644 .gdbinit create mode 100644 assembly/gdb/rc_to_actor_caller.py diff --git a/.gdbinit b/.gdbinit new file mode 100644 index 0000000000..4713275fbc --- /dev/null +++ b/.gdbinit @@ -0,0 +1 @@ +source assembly/gdb/rc_to_actor_caller.py diff --git a/CMakeLists.txt b/CMakeLists.txt index fb4b193d89..2cf917f5c1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,6 +69,7 @@ option(TON_USE_COVERAGE "Use \"ON\" to enable code coverage with gcov." OFF) set(TON_ARCH "native" CACHE STRING "Architecture, will be passed to -march=") option(TON_PRINT_BACKTRACE_ON_CRASH "Attempt to print a backtrace when a fatal signal is caught" ON) +option(TON_INSERT_GDB_HOOKS "Insert GDB hooks" OFF) option(TON_USE_LLD "Use LLD for linking" OFF) @@ -300,6 +301,10 @@ if (NOT ANDROID) # _FILE_OFFSET_BITS is broken in ndk r15 and r15b and doesn't w add_definitions(-D_FILE_OFFSET_BITS=64) endif() +if (TON_INSERT_GDB_HOOKS) + add_compile_definitions(-DTON_INSERT_GDB_HOOKS=1) +endif() + set(INTERNAL_COMPILE "0") #BEGIN internal add_definitions(-D_INTERNAL_COMPILE=1) diff --git a/assembly/gdb/rc_to_actor_caller.py b/assembly/gdb/rc_to_actor_caller.py new file mode 100644 index 0000000000..79af3c3fbb --- /dev/null +++ b/assembly/gdb/rc_to_actor_caller.py @@ -0,0 +1,84 @@ +import contextlib +from typing import final, override + +import gdb # pyright: ignore[reportMissingModuleSource] + +MESSAGE_RUN_HOOK = "td::actor::core::gdb::hook_message_run" +MESSAGE_PUSHED_HOOK = "td::actor::core::gdb::hook_message_pushed_to_mailbox" +MESSAGE_DELAYED_HOOK = "td::actor::core::gdb::hook_message_delayed" + +FLUSH_FN = "td::actor::core::ActorExecutor::flush_one_message" + + +def has_flush_frame(): + f = gdb.newest_frame() + while f: + if f.name() == FLUSH_FN: + return True + f = f.older() + return False + + +@final +class MallocReturnBP(gdb.Breakpoint): + def __init__(self, addr: int): + super().__init__("malloc", internal=True) + self.condition = f"$_retval == (void*){addr}" + self.silent = True + + +def reverse_continue_until_any(breakpoints: list[gdb.Breakpoint]): + while True: + _ = gdb.execute("reverse-continue", to_string=True) + if any(bp.hit_count > 0 for bp in breakpoints): + return + + +@contextlib.contextmanager +def with_breakpoint(bp: gdb.Breakpoint): + try: + yield bp + finally: + bp.delete() + + +def run(): + if not has_flush_frame(): + print("flush_one_message not present in the backtrace") + return + + thread_name = gdb.selected_thread().num + + with with_breakpoint(gdb.Breakpoint(MESSAGE_RUN_HOOK, internal=True)) as bp: + bp.silent = True + bp.thread = thread_name + reverse_continue_until_any([bp]) + + mailbox = gdb.parse_and_eval("&mailbox") + message = gdb.parse_and_eval("message.impl_.get()") + + with ( + with_breakpoint(gdb.Breakpoint(MESSAGE_PUSHED_HOOK, internal=True)) as bp_pushed, + with_breakpoint(gdb.Breakpoint(MESSAGE_DELAYED_HOOK, internal=True)) as bp_delayed, + ): + bp_pushed.silent = True + bp_pushed.condition = f"&mailbox == {mailbox} && message.impl_.get() == {message}" + + bp_delayed.silent = True + bp_delayed.condition = f"&mailbox == {mailbox} && message.impl_.get() == {message}" + + reverse_continue_until_any([bp_pushed, bp_delayed]) + + +class ActorTraceCmd(gdb.Command): + def __init__(self): + super().__init__("reverse-continue-to-actor-caller", gdb.COMMAND_USER) + + @override + def invoke(self, argument: str, from_tty: bool): + run() + + +_ = ActorTraceCmd() +_ = gdb.execute("alias rc-actor = reverse-continue-to-actor-caller", to_string=True) +_ = gdb.execute("alias rca = reverse-continue-to-actor-caller", to_string=True) diff --git a/pyproject.toml b/pyproject.toml index 700bb8565c..4ec6f68d58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ dependencies = [ "pytest>=9.0.1", "pytest-asyncio", "tontester", + "types-gdb>=16.3.0.20250920", ] [dependency-groups] diff --git a/tdactor/td/actor/core/ActorExecutor.cpp b/tdactor/td/actor/core/ActorExecutor.cpp index 474b5d2da1..cab346cd81 100644 --- a/tdactor/td/actor/core/ActorExecutor.cpp +++ b/tdactor/td/actor/core/ActorExecutor.cpp @@ -22,13 +22,41 @@ namespace td { namespace actor { namespace core { + +namespace gdb { + +#ifdef TON_INSERT_GDB_HOOKS +[[gnu::noinline]] void hook_message_delayed(ActorMailbox& mailbox, ActorMessage& message, auto&& continuation) { + asm volatile("" : : "r"(&message) : "memory"); + asm volatile("" : : "r"(&mailbox) : "memory"); + continuation(); +} + +[[gnu::noinline]] void hook_message_run(ActorMailbox& mailbox, ActorMessage& message, auto&& continuation) { + asm volatile("" : : "r"(&message) : "memory"); + asm volatile("" : : "r"(&mailbox) : "memory"); + continuation(); +} +#else +void hook_message_delayed(ActorMailbox&, ActorMessage&, auto&& continuation) { + continuation(); +} + +void hook_message_run(ActorMailbox&, ActorMessage&, auto&& continuation) { + continuation(); +} +#endif + +} // namespace gdb + void ActorExecutor::send_immediate(ActorMessage message) { CHECK(can_send_immediate()); if (is_closed()) { return; } if (message.is_big()) { - actor_info_.mailbox().reader().delay(std::move(message)); + gdb::hook_message_delayed(actor_info_.mailbox(), message, + [&]() { actor_info_.mailbox().reader().delay(std::move(message)); }); pending_signals_.add_signal(ActorSignals::Message); actor_execute_context_.set_pause(); return; @@ -192,7 +220,7 @@ void ActorExecutor::finish() noexcept { //LOG(ERROR) << "DO FINISH " << actor_info_.get_name() << " " << flags(); } -bool ActorExecutor::flush_one_signal(ActorSignals &signals) { +bool ActorExecutor::flush_one_signal(ActorSignals& signals) { auto signal = signals.first_signal(); if (!signal) { return false; @@ -260,7 +288,7 @@ bool ActorExecutor::flush_one_message() { actor_execute_context_.set_link_token(message.get_link_token()); auto message_timer = actor_stats_.create_message_timer(); - message.run(); + gdb::hook_message_run(actor_info_.mailbox(), message, [&] { message.run(); }); return true; } diff --git a/tdactor/td/actor/core/ActorMailbox.h b/tdactor/td/actor/core/ActorMailbox.h index 9e8b03091d..cf8e5ba3d0 100644 --- a/tdactor/td/actor/core/ActorMailbox.h +++ b/tdactor/td/actor/core/ActorMailbox.h @@ -24,6 +24,26 @@ namespace td { namespace actor { namespace core { + +class ActorMailbox; + +namespace gdb { + +#ifdef TON_INSERT_GDB_HOOKS +[[gnu::noinline]] inline auto hook_message_pushed_to_mailbox(ActorMailbox &mailbox, ActorMessage &message, + auto &&continuation) { + asm volatile("" : : "r"(&message) : "memory"); + asm volatile("" : : "r"(&mailbox) : "memory"); + return continuation(); +} +#else +inline auto hook_message_pushed_to_mailbox(ActorMailbox &, ActorMessage &, auto &&continuation) { + return continuation(); +} +#endif + +} // namespace gdb + class ActorMailbox { public: ActorMailbox() = default; @@ -35,10 +55,10 @@ class ActorMailbox { clear(); } void push(ActorMessage message) { - queue_.push(std::move(message)); + gdb::hook_message_pushed_to_mailbox(*this, message, [&] { queue_.push(std::move(message)); }); } void push_unsafe(ActorMessage message) { - queue_.push_unsafe(std::move(message)); + gdb::hook_message_pushed_to_mailbox(*this, message, [&] { queue_.push_unsafe(std::move(message)); }); } td::MpscLinkQueue::Reader &reader() { diff --git a/uv.lock b/uv.lock index d1db128997..524cb4597b 100644 --- a/uv.lock +++ b/uv.lock @@ -143,6 +143,7 @@ dependencies = [ { name = "pytest" }, { name = "pytest-asyncio" }, { name = "tontester" }, + { name = "types-gdb" }, ] [package.dev-dependencies] @@ -156,6 +157,7 @@ requires-dist = [ { name = "pytest", specifier = ">=9.0.1" }, { name = "pytest-asyncio" }, { name = "tontester", editable = "test/tontester" }, + { name = "types-gdb", specifier = ">=16.3.0.20250920" }, ] [package.metadata.requires-dev] @@ -168,3 +170,12 @@ dev = [ name = "tontester" version = "0.1.0" source = { editable = "test/tontester" } + +[[package]] +name = "types-gdb" +version = "16.3.0.20250920" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/9c/f1fbeedd534f7228c04cfb11281a47a1f4338238d27635199958c247b0e5/types_gdb-16.3.0.20250920.tar.gz", hash = "sha256:4c16cd289428ff6c0cb2fac26e1913d5eab2d890da9951091a11133d5d0a589c", size = 25335, upload-time = "2025-09-20T02:45:11.072Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/a5/e0246aece6b052e4b8251fbb0c80f490b6b7bae9a9c3d92978d04dfc930d/types_gdb-16.3.0.20250920-py3-none-any.whl", hash = "sha256:10873fd469cdccb0b7e5ad2321eeb751cb6676cb05a67f3589e753d5f4482ae7", size = 31308, upload-time = "2025-09-20T02:45:10.034Z" }, +] From af6e91eaca64e307b029db0f99ec5bb45600ee85 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Mon, 1 Dec 2025 19:42:09 +0400 Subject: [PATCH 497/524] shows a real diff that will be applied on merge (only PR changes) (#1928) --- .github/workflows/codex-review.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codex-review.yml b/.github/workflows/codex-review.yml index 151d22acb4..57dc39a280 100644 --- a/.github/workflows/codex-review.yml +++ b/.github/workflows/codex-review.yml @@ -52,7 +52,7 @@ jobs: - name: Show diff summary run: | git diff --stat \ - ${{ github.event.pull_request.base.sha }} pr-${{ github.event.pull_request.number }} + ${{ github.event.pull_request.base.sha }}...pr-${{ github.event.pull_request.number }} # 3. Run Codex safely - name: Run Codex From c704891f7127ba8eef474de2eca43ebca94dcbaf Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 1 Dec 2025 18:45:29 +0300 Subject: [PATCH 498/524] fix(emulator): returned support for `web` launching `wasm` in the browser (#1933) Co-authored-by: Danil Ovchinnikov --- emulator/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emulator/CMakeLists.txt b/emulator/CMakeLists.txt index a802d8df30..ccbeb0f64d 100644 --- a/emulator/CMakeLists.txt +++ b/emulator/CMakeLists.txt @@ -63,7 +63,7 @@ if (USE_EMSCRIPTEN) target_link_options(emulator-emscripten PRIVATE -sIGNORE_MISSING_MAIN=1) target_link_options(emulator-emscripten PRIVATE -sAUTO_NATIVE_LIBRARIES=0) target_link_options(emulator-emscripten PRIVATE -sMODULARIZE=1) - target_link_options(emulator-emscripten PRIVATE -sENVIRONMENT=node) + target_link_options(emulator-emscripten PRIVATE -sENVIRONMENT=web,node) target_link_options(emulator-emscripten PRIVATE -sFORCE_FILESYSTEM=1) target_link_options(emulator-emscripten PRIVATE -sALLOW_MEMORY_GROWTH=1) target_link_options(emulator-emscripten PRIVATE -fexceptions) From 07dc2f298c15f75a0f35d0c763a7ea9a2f6376ce Mon Sep 17 00:00:00 2001 From: neodix42 Date: Mon, 1 Dec 2025 20:07:57 +0400 Subject: [PATCH 499/524] update blst to v0.3.15 (#1932) * update blst to v0.3.15 * remove redundant submodule entry --- third-party/blst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third-party/blst b/third-party/blst index 3dd0f804b1..6d960cd05d 160000 --- a/third-party/blst +++ b/third-party/blst @@ -1 +1 @@ -Subproject commit 3dd0f804b1819e5d03fb22ca2e6fac105932043a +Subproject commit 6d960cd05d6fe2b5bc9ba161edf0c1a131b87c4c From 9c2b5cd232730bd13d3fc2afc0afb68ee5f683e1 Mon Sep 17 00:00:00 2001 From: "Dr. Awesome Doge" Date: Tue, 2 Dec 2025 00:08:42 +0800 Subject: [PATCH 500/524] Fix promise_type ambiguity in coroutine declarations for GCC 11+ compatibility (#1929) This commit resolves a compilation error that occurs when building TON with GCC 11 or newer. The error was caused by ambiguous promise_type declarations in Task and StartedTask structs. Changes: - Replace 'using promise_type = promise_type' with fully-qualified 'using promise_type = td::actor::promise_type' at lines 295 and 367 in tdactor/td/actor/coro_task.h This fix enables successful compilation with modern GCC versions while maintaining backward compatibility with Clang. Tested with: - GCC 11.4.0 on Ubuntu 22.04 - Clang 14.0.0 Fixes build errors on systems using default GCC 11+ compilers. --- tdactor/td/actor/coro_task.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdactor/td/actor/coro_task.h b/tdactor/td/actor/coro_task.h index 420f2e4e35..4e5c904f6c 100644 --- a/tdactor/td/actor/coro_task.h +++ b/tdactor/td/actor/coro_task.h @@ -292,7 +292,7 @@ template struct [[nodiscard]] Task { using value_type = T; - using promise_type = promise_type; + using promise_type = td::actor::promise_type; using Handle = std::coroutine_handle; Handle h{}; @@ -364,7 +364,7 @@ template struct [[nodiscard]] StartedTask { using value_type = T; - using promise_type = promise_type; + using promise_type = td::actor::promise_type; using Handle = std::coroutine_handle; Handle h{}; From aaa46759285c8c875b6d5a2e6e0bf1574bf324bc Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Tue, 2 Dec 2025 15:25:15 +0300 Subject: [PATCH 501/524] [Fift] Fix PREPAREDICT in Asm.fif (#1934) A Fift compilation error could occur in some cases when one get method (id>65536) was called from another --- crypto/fift/lib/Asm.fif | 2 +- tolk-tester/tests/try-catch-tests.tolk | 21 ++++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index b69e582ea0..fddb0c88ea 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -1069,7 +1069,7 @@ x{EDFB} @Defop SAMEALTSAVE } dup : JMP : JMPDICT { dup 14 ufits { Date: Wed, 3 Dec 2025 19:09:41 +0400 Subject: [PATCH 502/524] fetch libmicrohttpd from github instead of ftpmirror.gnu.org (#1935) * fetch libmicrohttpd from github instead of ftpmirror.gnu.org * remove redundant snippet * remove redundant snippet * minor * use libmicrohttpd repo out of ton-blockchain org --- assembly/native/build-macos-portable.sh | 11 ++++------- assembly/native/build-ubuntu-portable-libs.sh | 13 +++++-------- assembly/native/build-ubuntu-portable.sh | 13 +++++-------- 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index 870ae73849..a86435a4fa 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -109,18 +109,15 @@ else fi if [ ! -d "../3pp/libmicrohttpd" ]; then - mkdir -p ../3pp/libmicrohttpd - wget -O ../3pp/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz - cd ../3pp/libmicrohttpd/ - tar xf libmicrohttpd-1.0.1.tar.gz - cd libmicrohttpd-1.0.1 + git clone https://github.com/ton-blockchain/libmicrohttpd.git ../3pp/libmicrohttpd + cd ../3pp/libmicrohttpd libmicrohttpdPath=`pwd` ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic make -j4 test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } - cd ../../../build + cd ../../build else - libmicrohttpdPath=$(pwd)/../3pp/libmicrohttpd/libmicrohttpd-1.0.1 + libmicrohttpdPath=$(pwd)/../3pp/libmicrohttpd echo "Using compiled libmicrohttpd" fi diff --git a/assembly/native/build-ubuntu-portable-libs.sh b/assembly/native/build-ubuntu-portable-libs.sh index c7bffd40ee..cf797cd233 100644 --- a/assembly/native/build-ubuntu-portable-libs.sh +++ b/assembly/native/build-ubuntu-portable-libs.sh @@ -94,18 +94,15 @@ else fi if [ ! -d "../3pp/libmicrohttpd" ]; then - mkdir -p ../3pp/libmicrohttpd - wget -O ../3pp/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz - cd ../3pp/libmicrohttpd/ - tar xf libmicrohttpd-1.0.1.tar.gz - cd libmicrohttpd-1.0.1 + git clone https://github.com/ton-blockchain/libmicrohttpd.git ../3pp/libmicrohttpd + cd ../3pp/libmicrohttpd libmicrohttpdPath=`pwd` ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - make -j$(nproc) + make -j4 test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } - cd ../../../build + cd ../../build else - libmicrohttpdPath=$(pwd)/../3pp/libmicrohttpd/libmicrohttpd-1.0.1 + libmicrohttpdPath=$(pwd)/../3pp/libmicrohttpd echo "Using compiled libmicrohttpd" fi diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh index 49b2863c30..49a0205f68 100644 --- a/assembly/native/build-ubuntu-portable.sh +++ b/assembly/native/build-ubuntu-portable.sh @@ -95,18 +95,15 @@ else fi if [ ! -d "../3pp/libmicrohttpd" ]; then - mkdir -p ../3pp/libmicrohttpd - wget -O ../3pp/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz - cd ../3pp/libmicrohttpd/ - tar xf libmicrohttpd-1.0.1.tar.gz - cd libmicrohttpd-1.0.1 + git clone https://github.com/ton-blockchain/libmicrohttpd.git ../3pp/libmicrohttpd + cd ../3pp/libmicrohttpd libmicrohttpdPath=`pwd` ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - make -j$(nproc) + make -j4 test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } - cd ../../../build + cd ../../build else - libmicrohttpdPath=$(pwd)/../3pp/libmicrohttpd/libmicrohttpd-1.0.1 + libmicrohttpdPath=$(pwd)/../3pp/libmicrohttpd echo "Using compiled libmicrohttpd" fi From ca818c201a18671ddc14ef032027830465b6cbd5 Mon Sep 17 00:00:00 2001 From: Vadim Averin Date: Wed, 3 Dec 2025 18:10:18 +0300 Subject: [PATCH 503/524] Add ratelimiter for full node shard requests (#1918) * Add ratelimiter for full node shard requests * Fill in limits for some functions * Fix linter errors * Share limiter between shards * Fix includes * Add synchronization * Some fixes * Move configuration * Fixes + formatting * Fix * FIx request groups --- validator-engine/validator-engine.cpp | 38 +++++++-- validator-engine/validator-engine.hpp | 29 +++++-- validator/full-node-shard.cpp | 32 ++++---- validator/full-node-shard.h | 5 +- validator/full-node-shard.hpp | 10 ++- validator/full-node.cpp | 36 ++++++-- validator/full-node.h | 5 +- validator/full-node.hpp | 5 ++ validator/rate-limiter.h | 114 ++++++++++++++++++++++++++ 9 files changed, 233 insertions(+), 41 deletions(-) create mode 100644 validator/rate-limiter.h diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 8f0636ace3..838bd5fbab 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -2231,11 +2231,8 @@ void ValidatorEngine::start_full_node() { R.ensure(); td::actor::send_closure(SelfId, &ValidatorEngine::started_full_node); }); - ton::validator::fullnode::FullNodeOptions full_node_options{ - .config_ = config_.full_node_config, - .public_broadcast_speed_multiplier_ = broadcast_speed_multiplier_public_, - .private_broadcast_speed_multiplier_ = broadcast_speed_multiplier_private_, - .initial_sync_delay_ = initial_sync_delay_}; + ton::validator::fullnode::FullNodeOptions full_node_options = full_node_options_; + full_node_options.config_ = config_.full_node_config; full_node_ = ton::validator::fullnode::FullNode::create( short_id, full_node_id_, validator_options_->zero_block_id().file_hash, full_node_options, keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(), @@ -5552,6 +5549,37 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_initial_sync_delay, v); }); return td::Status::OK(); }); + p.add_checked_option( + 0, "fullnode-ratelimit-window-size", "ratelimit tracking window size (in seconds)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v < 0) { + return td::Status::Error("ratelimit-window-size should be non-negative"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_ratelimit_window_size, v); }); + return td::Status::OK(); + }); + p.add_checked_option( + 0, "fullnode-ratelimit-global", "ratelimit for all kind of requests (in counts per window)", + [&](td::Slice s) -> td::Status { + TRY_RESULT(v, td::to_integer_safe(s)); + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_ratelimit_global, v); }); + return td::Status::OK(); + }); + p.add_checked_option( + 0, "fullnode-ratelimit-heavy", "ratelimit for heavy requests (in counts per window)", + [&](td::Slice s) -> td::Status { + TRY_RESULT(v, td::to_integer_safe(s)); + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_ratelimit_heavy, v); }); + return td::Status::OK(); + }); + p.add_checked_option( + 0, "fullnode-ratelimit-medium", "ratelimit for medium requests (in counts per window)", + [&](td::Slice s) -> td::Status { + TRY_RESULT(v, td::to_integer_safe(s)); + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_ratelimit_medium, v); }); + return td::Status::OK(); + }); p.add_checked_option( '\0', "sync-shards-upto", "stop syncing shards on this masterchain seqno", [&](td::Slice s) -> td::Status { TRY_RESULT(v, td::to_integer_safe(s)); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 1a72869631..19991a065b 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -250,15 +250,20 @@ class ValidatorEngine : public td::actor::Actor { std::vector add_shard_cmds_; bool state_serializer_disabled_flag_ = false; double broadcast_speed_multiplier_catchain_ = 3.33; - double broadcast_speed_multiplier_public_ = 3.33; - double broadcast_speed_multiplier_private_ = 3.33; bool permanent_celldb_ = false; bool skip_key_sync_ = false; td::optional sync_shards_upto_; ton::adnl::AdnlNodeIdShort shard_block_retainer_adnl_id_ = ton::adnl::AdnlNodeIdShort::zero(); bool shard_block_retainer_adnl_id_fullnode_ = false; bool parallel_validation_ = false; - double initial_sync_delay_ = 60.0; + ton::validator::fullnode::FullNodeOptions full_node_options_ = {.config_ = {}, + .public_broadcast_speed_multiplier_ = 3.33, + .private_broadcast_speed_multiplier_ = 3.33, + .initial_sync_delay_ = 60.0, + .ratelimit_window_size_ = 0, + .ratelimit_global_ = 0, + .ratelimit_heavy_ = 0, + .ratelimit_medium_ = 0}; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -363,10 +368,10 @@ class ValidatorEngine : public td::actor::Actor { broadcast_speed_multiplier_catchain_ = value; } void set_broadcast_speed_multiplier_public(double value) { - broadcast_speed_multiplier_public_ = value; + full_node_options_.public_broadcast_speed_multiplier_ = value; } void set_broadcast_speed_multiplier_private(double value) { - broadcast_speed_multiplier_private_ = value; + full_node_options_.private_broadcast_speed_multiplier_ = value; } void set_permanent_celldb(bool value) { permanent_celldb_ = value; @@ -387,7 +392,19 @@ class ValidatorEngine : public td::actor::Actor { parallel_validation_ = value; } void set_initial_sync_delay(double value) { - initial_sync_delay_ = value; + full_node_options_.initial_sync_delay_ = value; + } + void set_ratelimit_window_size(double seconds) { + full_node_options_.ratelimit_window_size_ = seconds; + } + void set_ratelimit_global(size_t count) { + full_node_options_.ratelimit_global_ = count; + } + void set_ratelimit_heavy(size_t count) { + full_node_options_.ratelimit_heavy_ = count; + } + void set_ratelimit_medium(size_t count) { + full_node_options_.ratelimit_medium_ = count; } void start_up() override; diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 813f795938..d0f75f82c0 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -736,7 +736,12 @@ void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice promise.set_error(td::Status::Error(ErrorCode::protoviolation, "cannot parse tonnode query")); return; } - ton_api::downcast_call(*B.move_as_ok().get(), [&](auto &obj) { this->process_query(src, obj, std::move(promise)); }); + auto fun_ptr = B.move_as_ok(); + if (!limiter_->check_in(fun_ptr->get_id())) { + promise.set_error(td::Status::Error(ErrorCode::failure, "too many requests")); + return; + } + ton_api::downcast_call(*fun_ptr.get(), [&](auto &obj) { this->process_query(src, obj, std::move(promise)); }); } void FullNodeShardImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) { @@ -1391,14 +1396,12 @@ void FullNodeShardImpl::get_stats_extra(td::Promise promise) { promise.set_result(td::json_encode(td::ToJson(*res), true)); } -FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, FullNodeOptions opts, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId rldp2, - td::actor::ActorId overlays, - td::actor::ActorId validator_manager, - td::actor::ActorId client, - td::actor::ActorId full_node, bool active) +FullNodeShardImpl::FullNodeShardImpl( + ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, + FullNodeOptions opts, std::shared_ptr> limiter, td::actor::ActorId keyring, + td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, + td::actor::ActorId overlays, td::actor::ActorId validator_manager, + td::actor::ActorId client, td::actor::ActorId full_node, bool active) : shard_(shard) , local_id_(local_id) , adnl_id_(adnl_id) @@ -1412,18 +1415,19 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, , client_(client) , full_node_(full_node) , active_(active) - , opts_(opts) { + , opts_(opts) + , limiter_(std::move(limiter)) { } td::actor::ActorOwn FullNodeShard::create( ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId rldp2, + FullNodeOptions opts, std::shared_ptr> limiter, td::actor::ActorId keyring, + td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, td::actor::ActorId full_node, bool active) { return td::actor::create_actor(PSTRING() << "tonnode" << shard.to_str(), shard, local_id, adnl_id, - zero_state_file_hash, opts, keyring, adnl, rldp, rldp2, overlays, - validator_manager, client, full_node, active); + zero_state_file_hash, opts, std::move(limiter), keyring, adnl, rldp, + rldp2, overlays, validator_manager, client, full_node, active); } } // namespace fullnode diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 59df18c24b..6bc36bb279 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -22,6 +22,7 @@ #include "validator/interfaces/block-handle.h" #include "full-node.h" +#include "rate-limiter.h" namespace ton { @@ -78,8 +79,8 @@ class FullNodeShard : public td::actor::Actor { static td::actor::ActorOwn create( ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId rldp2, + FullNodeOptions opts, std::shared_ptr> limiter, td::actor::ActorId keyring, + td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, td::actor::ActorId full_node, bool active); }; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index fbc13a4685..3fdbb761b1 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -25,6 +25,7 @@ #include "td/utils/port/Poll.h" #include "full-node-shard.h" +#include "rate-limiter.h" namespace ton { @@ -236,9 +237,10 @@ class FullNodeShardImpl : public FullNodeShard { } FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, FullNodeOptions opts, td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId rldp2, td::actor::ActorId overlays, + FileHash zero_state_file_hash, FullNodeOptions opts, std::shared_ptr> limiter, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId rldp2, + td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, td::actor::ActorId full_node, bool active); @@ -288,6 +290,8 @@ class FullNodeShardImpl : public FullNodeShard { std::set my_ext_msg_broadcasts_; std::set processed_ext_msg_broadcasts_; td::Timestamp cleanup_processed_ext_msg_at_; + + std::shared_ptr> limiter_; }; } // namespace fullnode diff --git a/validator/full-node.cpp b/validator/full-node.cpp index a27555f9ad..f3f6e2a00e 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -282,8 +282,9 @@ void FullNodeImpl::on_new_masterchain_block(td::Ref state, std void FullNodeImpl::update_shard_actor(ShardIdFull shard, bool active) { ShardInfo &info = shards_[shard]; if (info.actor.empty()) { - info.actor = FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, opts_, keyring_, adnl_, rldp_, - rldp2_, overlays_, validator_manager_, client_, actor_id(this), active); + info.actor = + FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, opts_, limiter_, keyring_, adnl_, + rldp_, rldp2_, overlays_, validator_manager_, client_, actor_id(this), active); if (!all_validators_.empty()) { td::actor::send_closure(info.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } @@ -849,7 +850,8 @@ FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id , client_(client) , db_root_(db_root) , started_promise_(std::move(started_promise)) - , opts_(opts) { + , opts_(opts) + , limiter_(make_limiter(opts)) { } td::actor::ActorOwn FullNode::create( @@ -870,12 +872,6 @@ FullNodeConfig::FullNodeConfig(const tl_object_ptr FullNodeConfig::tl() const { return create_tl_object(ext_messages_broadcast_disabled_); } -bool FullNodeConfig::operator==(const FullNodeConfig &rhs) const { - return ext_messages_broadcast_disabled_ == rhs.ext_messages_broadcast_disabled_; -} -bool FullNodeConfig::operator!=(const FullNodeConfig &rhs) const { - return !(*this == rhs); -} bool CustomOverlayParams::send_shard(const ShardIdFull &shard) const { return sender_shards_.empty() || @@ -902,6 +898,28 @@ CustomOverlayParams CustomOverlayParams::fetch(const ton_api::engine_validator_c return c; } +decltype(FullNodeImpl::limiter_) FullNodeImpl::make_limiter(const FullNodeOptions &opts) { + double w_size = opts.ratelimit_window_size_; + size_t h_limit = opts.ratelimit_heavy_; + size_t m_limit = opts.ratelimit_medium_; + size_t g_limit = opts.ratelimit_global_; + return std::make_shared>( + RateLimit{w_size, g_limit}, + std::map{{ton_api::tonNode_getArchiveSlice::ID, {w_size, h_limit}}, + {ton_api::tonNode_downloadPersistentStateSliceV2::ID, {w_size, h_limit}}, + {ton_api::tonNode_downloadPersistentStateSlice::ID, {w_size, h_limit}}, + {ton_api::tonNode_downloadZeroState::ID, {w_size, h_limit}}, + + {ton_api::tonNode_downloadBlock::ID, {w_size, m_limit}}, + {ton_api::tonNode_downloadBlockFull::ID, {w_size, m_limit}}, + {ton_api::tonNode_downloadNextBlockFull::ID, {w_size, m_limit}}, + {ton_api::tonNode_downloadBlockProof::ID, {w_size, m_limit}}, + {ton_api::tonNode_downloadBlockProofLink::ID, {w_size, m_limit}}, + {ton_api::tonNode_downloadKeyBlockProof::ID, {w_size, m_limit}}, + {ton_api::tonNode_downloadKeyBlockProofLink::ID, {w_size, m_limit}}, + {ton_api::tonNode_getOutMsgQueueProof::ID, {w_size, m_limit}}}); +} + } // namespace fullnode } // namespace validator diff --git a/validator/full-node.h b/validator/full-node.h index a69a75ba04..600209d98d 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -47,8 +47,7 @@ struct FullNodeConfig { FullNodeConfig() = default; FullNodeConfig(const tl_object_ptr& obj); tl_object_ptr tl() const; - bool operator==(const FullNodeConfig& rhs) const; - bool operator!=(const FullNodeConfig& rhs) const; + bool operator==(const FullNodeConfig& rhs) const = default; bool ext_messages_broadcast_disabled_ = false; }; @@ -58,6 +57,8 @@ struct FullNodeOptions { double public_broadcast_speed_multiplier_ = 1.0; double private_broadcast_speed_multiplier_ = 1.0; double initial_sync_delay_ = 60.0; + double ratelimit_window_size_ = 0; + size_t ratelimit_global_ = 0, ratelimit_heavy_ = 0, ratelimit_medium_ = 0; }; struct CustomOverlayParams { diff --git a/validator/full-node.hpp b/validator/full-node.hpp index ecc446da37..3efb3d131b 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -31,6 +31,7 @@ #include "full-node-custom-overlays.hpp" #include "full-node-fast-sync-overlays.hpp" +#include "rate-limiter.h" namespace ton { @@ -179,6 +180,10 @@ class FullNodeImpl : public FullNode { td::actor::ActorOwn out_msg_queue_query_token_manager_ = td::actor::create_actor("tokens", /* max_tokens = */ 1); + + std::shared_ptr> limiter_; + + decltype(limiter_) make_limiter(const FullNodeOptions& opts); }; } // namespace fullnode diff --git a/validator/rate-limiter.h b/validator/rate-limiter.h new file mode 100644 index 0000000000..a0355fa554 --- /dev/null +++ b/validator/rate-limiter.h @@ -0,0 +1,114 @@ +#pragma once +#include +#include +#include + +#include "td/utils/Time.h" + +namespace ton::validator::fullnode { +struct LimiterWindow { + double size; + size_t limit; + std::deque timestamps = {}; + + bool check(td::Timestamp time); + void insert(td::Timestamp time); +}; + +inline bool LimiterWindow::check(td::Timestamp time) { + if (size == 0) { + return true; + } + if (limit == 0) { + return false; + } + return timestamps.size() < limit || time - timestamps.back() > size; +} + +inline void LimiterWindow::insert(td::Timestamp time) { + if (size == 0) { + return; + } + if (timestamps.size() == limit) { + timestamps.pop_back(); + } + timestamps.push_front(time); +} + +struct RateLimit { + double window_size; + size_t window_limit; +}; + +template +class RateLimiter { + public: + RateLimiter(RateLimit global_limit, std::map request_limits) + : global_limit_(global_limit), request_limits_(std::move(request_limits)) { + global_window_ = {.size = global_limit.window_size, .limit = global_limit.window_limit}; + } + + bool check_in(RequestID request, td::Timestamp time = td::Timestamp::now()); + + private: + bool check(td::Timestamp time); + bool check(RequestID request, td::Timestamp time); + void insert(td::Timestamp time); + void insert(RequestID request, td::Timestamp time); + + const RateLimit global_limit_; + const std::map request_limits_; + + LimiterWindow global_window_; + std::map request_windows_; + + std::mutex mutex_; +}; + +template +bool RateLimiter::check_in(RequestID request, td::Timestamp time) { + if (!request_limits_.contains(request)) { + return true; + } + std::unique_lock lock(mutex_); + if (check(time) && check(request, time)) { + insert(time); + insert(request, time); + return true; + } + return false; +} + +template +bool RateLimiter::check(td::Timestamp time) { + if (global_limit_.window_size == 0) { + return true; + } + return global_window_.check(time); +} + +template +bool RateLimiter::check(RequestID request, td::Timestamp time) { + if (!request_limits_.contains(request)) { + return true; + } + if (!request_windows_.contains(request)) { + RateLimit limit = request_limits_.at(request); + request_windows_.emplace(std::pair{request, LimiterWindow{limit.window_size, limit.window_limit}}); + } + return request_windows_.at(request).check(time); +} + +template +void RateLimiter::insert(td::Timestamp time) { + global_window_.insert(time); +} + +template +void RateLimiter::insert(RequestID request, td::Timestamp time) { + if (auto it = request_windows_.find(request); it != request_windows_.end()) { + it->second.insert(time); + } +} + +} // namespace ton::validator::fullnode \ No newline at end of file From 1f8f62e909be3066ddaae313a5b824f5050a8022 Mon Sep 17 00:00:00 2001 From: Oleg Vallas Date: Wed, 3 Dec 2025 16:11:10 +0100 Subject: [PATCH 504/524] Compression depth balance (#1892) * Fix depth balance decompression * [compression-depth-balance] Bug fix and refactor depth balance decompression * [compression-depth-balance] Refactor balance parsing, change CMakeLists to avoid cyclic dependencies with block-auto.h * [compression-depth-balance] Optimize decompression * [compression-depth-balance] Add benchmark logs for testing on devnet * Add benchmark logs to lefotver methods * Improve benchmark logs * [broadcast-benchmark-logs] Create comprehensive logs for broadcast benchmark. Other refactoring. * [compression-depth-balance] Fix formatting --- crypto/CMakeLists.txt | 2 +- crypto/vm/boc-compression.cpp | 233 ++++++++++++++++-- test/test-validator-session-state.cpp | 3 + validator-session/candidate-serializer.cpp | 47 +++- validator-session/candidate-serializer.h | 12 +- .../validator-session-description.h | 1 + .../validator-session-description.hpp | 7 + validator-session/validator-session.cpp | 5 +- validator/collator-node/utils.cpp | 100 ++++---- validator/full-node-custom-overlays.cpp | 16 +- validator/full-node-fast-sync-overlays.cpp | 16 +- validator/full-node-serializer.cpp | 207 ++++++++++++---- validator/full-node-serializer.hpp | 11 +- validator/full-node-shard.cpp | 16 +- 14 files changed, 516 insertions(+), 160 deletions(-) diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 1051f0d18e..fb42810f40 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -10,7 +10,6 @@ set(TON_CRYPTO_CORE_SOURCE openssl/residue.cpp openssl/rand.cpp vm/boc.cpp - vm/boc-compression.cpp vm/large-boc-serializer.cpp tl/tlblib.cpp @@ -82,6 +81,7 @@ set(TON_CRYPTO_CORE_SOURCE ellcurve/p256.cpp) set(TON_CRYPTO_SOURCE + vm/boc-compression.cpp vm/stack.cpp vm/atom.cpp vm/continuation.cpp diff --git a/crypto/vm/boc-compression.cpp b/crypto/vm/boc-compression.cpp index 41095efded..9b9858c135 100644 --- a/crypto/vm/boc-compression.cpp +++ b/crypto/vm/boc-compression.cpp @@ -18,7 +18,11 @@ */ #include #include +#include +#include "common/refint.h" +#include "crypto/block/block-auto.h" +#include "crypto/block/block-parse.h" #include "td/utils/Slice-decl.h" #include "td/utils/lz4.h" #include "vm/boc-writers.h" @@ -77,7 +81,46 @@ inline td::Result read_uint(td::BitSlice& bs, int bits) { return result; } +// Decode DepthBalanceInfo and extract grams using TLB methods +td::RefInt256 extract_balance_from_depth_balance_info(vm::CellSlice& cs) { + // Check hashmap label is empty ('00') + if (cs.size() < 2 || cs.fetch_ulong(2) != 0) { + return td::RefInt256{}; + } + + int split_depth; + Ref balance_cs_ref; + if (!block::gen::t_DepthBalanceInfo.unpack_depth_balance(cs, split_depth, balance_cs_ref)) { + return td::RefInt256{}; + } + if (split_depth != 0) { + return td::RefInt256{}; + } + if (!cs.empty()) { + return td::RefInt256{}; + } + auto balance_cs = balance_cs_ref.write(); + auto res = block::tlb::t_Grams.as_integer_skip(balance_cs); + if (balance_cs.size() != 1 || balance_cs.fetch_ulong(1) != 0) { + return td::RefInt256{}; + } + return res; +} + +// Process ShardAccounts vertex and compute balance difference (right - left) +td::RefInt256 process_shard_accounts_vertex(vm::CellSlice& cs_left, vm::CellSlice& cs_right) { + auto balance_left = extract_balance_from_depth_balance_info(cs_left); + auto balance_right = extract_balance_from_depth_balance_info(cs_right); + if (balance_left.not_null() && balance_right.not_null()) { + td::RefInt256 diff = balance_right; + diff -= balance_left; + return diff; + } + return td::RefInt256{}; +} + td::Result boc_compress_improved_structure_lz4(const std::vector>& boc_roots) { + const bool kMURemoveSubtreeSums = true; // Input validation if (boc_roots.empty()) { return td::Status::Error("No root cells were provided for serialization"); @@ -99,7 +142,9 @@ td::Result boc_compress_improved_structure_lz4(const std::vecto size_t total_size_estimate = 0; // Build graph representation using recursive lambda - const auto build_graph = [&](auto&& self, td::Ref cell) -> td::Result { + const auto build_graph = [&](auto&& self, td::Ref cell, td::Ref left_cell = td::Ref(), + bool under_mu_right = false, + td::RefInt256* sum_diff_out = nullptr) -> td::Result { if (cell.is_null()) { return td::Status::Error("Error while importing a cell during serialization: cell is null"); } @@ -139,9 +184,37 @@ td::Result boc_compress_improved_structure_lz4(const std::vecto total_size_estimate += cell_bitslice.size(); // Process cell references - for (int i = 0; i < cell_slice.size_refs(); ++i) { - TRY_RESULT(child_id, self(self, cell_slice.prefetch_ref(i))); - boc_graph[current_cell_id][i] = child_id; + if (kMURemoveSubtreeSums && cell_slice.special_type() == vm::CellTraits::SpecialType::MerkleUpdate) { + // Left branch: traverse normally + TRY_RESULT(child_left_id, self(self, cell_slice.prefetch_ref(0))); + boc_graph[current_cell_id][0] = child_left_id; + // Right branch: traverse paired with left and compute diffs inline + TRY_RESULT(child_right_id, self(self, cell_slice.prefetch_ref(1), cell_slice.prefetch_ref(0), true)); + boc_graph[current_cell_id][1] = child_right_id; + } else if (under_mu_right && left_cell.not_null()) { + // Inline computation for RIGHT subtree nodes under MerkleUpdate + vm::CellSlice cs_left(NoVm(), left_cell); + td::RefInt256 sum_child_diff = td::make_refint(0); + // Recurse children first + for (int i = 0; i < cell_slice.size_refs(); ++i) { + TRY_RESULT(child_id, self(self, cell_slice.prefetch_ref(i), cs_left.prefetch_ref(i), true, &sum_child_diff)); + boc_graph[current_cell_id][i] = child_id; + } + + // Compute this vertex diff and check skippable condition + td::RefInt256 vertex_diff = process_shard_accounts_vertex(cs_left, cell_slice); + if (!is_special && vertex_diff.not_null() && sum_child_diff.not_null() && cmp(sum_child_diff, vertex_diff) == 0) { + cell_data[current_cell_id] = td::BitSlice(); + prunned_branch_level[current_cell_id] = 9; + } + if (sum_diff_out && vertex_diff.not_null()) { + *sum_diff_out += vertex_diff; + } + } else { + for (int i = 0; i < cell_slice.size_refs(); ++i) { + TRY_RESULT(child_id, self(self, cell_slice.prefetch_ref(i))); + boc_graph[current_cell_id][i] = child_id; + } } return current_cell_id; @@ -241,11 +314,11 @@ td::Result boc_compress_improved_structure_lz4(const std::vecto // Store cell types and sizes for (int i = 0; i < node_count; ++i) { size_t node = topo_order[i]; - size_t currrent_cell_type = bool(cell_type[node]) + prunned_branch_level[node]; - append_uint(result, currrent_cell_type, 4); + size_t current_cell_type = bool(cell_type[node]) + prunned_branch_level[node]; + append_uint(result, current_cell_type, 4); append_uint(result, refs_cnt[node], 4); - if (cell_type[node] != 1) { + if (cell_type[node] != 1 && current_cell_type != 9) { if (is_data_small[node]) { append_uint(result, 1, 1); append_uint(result, cell_data[node].size(), 7); @@ -269,6 +342,9 @@ td::Result boc_compress_improved_structure_lz4(const std::vecto // Store cell data for (size_t node : topo_order) { + if (prunned_branch_level[node] == 9) { + continue; + } if (cell_type[node] != 1 && !is_data_small[node]) { continue; } @@ -308,6 +384,9 @@ td::Result boc_compress_improved_structure_lz4(const std::vecto // Store remaining cell data for (size_t node : topo_order) { + if (prunned_branch_level[node] == 9) { + continue; + } if (cell_type[node] == 1 || is_data_small[node]) { size_t prefix_size = cell_data[node].size() % 8; result.append(cell_data[node].subslice(prefix_size, cell_data[node].size() - prefix_size)); @@ -342,6 +421,30 @@ td::Result boc_compress_improved_structure_lz4(const std::vecto return compressed_with_size; } +// Helper: write ShardAccounts augmentation (DepthBalanceInfo with grams) into builder +bool write_depth_balance_grams(vm::CellBuilder& cb, const td::RefInt256& grams) { + if (!cb.store_zeroes_bool(7)) { // empty HmLabel and split_depth + return false; + } + if (!block::tlb::t_CurrencyCollection.pack_special(cb, grams, td::Ref())) { + return false; + } + return true; +} + +// Helper: detect MerkleUpdate (is_special AND first byte == 0x04) without finalizing +bool is_merkle_update_node(bool is_special, const vm::CellBuilder& cb) { + if (!is_special) { + return false; + } + // Need at least one full byte in data to read the tag + if (cb.get_bits() < 8) { + return false; + } + unsigned first_byte = cb.get_data()[0]; + return first_byte == 0x04; +} + td::Result>> boc_decompress_improved_structure_lz4(td::Slice compressed, int max_decompressed_size) { constexpr size_t kMaxCellDataLengthBits = 1024; @@ -404,7 +507,7 @@ td::Result>> boc_decompress_improved_structure_lz4 // Initialize data structures std::vector cell_data_length(node_count), is_data_small(node_count); - std::vector is_special(node_count), cell_refs_cnt(node_count); + std::vector is_special(node_count), cell_refs_cnt(node_count), is_depth_balance(node_count); std::vector prunned_branch_level(node_count, 0); std::vector cell_builders(node_count); @@ -418,7 +521,8 @@ td::Result>> boc_decompress_improved_structure_lz4 } size_t cell_type = bit_reader.bits().get_uint(4); - is_special[i] = bool(cell_type); + is_special[i] = (cell_type == 9 ? false : bool(cell_type)); + is_depth_balance[i] = cell_type == 9; if (is_special[i]) { prunned_branch_level[i] = cell_type - 1; } @@ -429,8 +533,9 @@ td::Result>> boc_decompress_improved_structure_lz4 if (cell_refs_cnt[i] > 4) { return td::Status::Error("BOC decompression failed: invalid cell refs count"); } - - if (prunned_branch_level[i]) { + if (is_depth_balance[i]) { + cell_data_length[i] = 0; + } else if (prunned_branch_level[i]) { size_t coef = std::bitset<4>(prunned_branch_level[i]).count(); cell_data_length[i] = (256 + 16) * coef; } else { @@ -470,6 +575,9 @@ td::Result>> boc_decompress_improved_structure_lz4 // Read initial cell data for (int i = 0; i < node_count; ++i) { + if (is_depth_balance[i]) { + continue; + } if (prunned_branch_level[i]) { cell_builders[i].store_long((1 << 8) + prunned_branch_level[i], 16); } @@ -538,6 +646,9 @@ td::Result>> boc_decompress_improved_structure_lz4 // Read remaining cell data for (int i = 0; i < node_count; ++i) { + if (is_depth_balance[i]) { + continue; + } size_t padding_bits = 0; if (!prunned_branch_level[i] && !is_data_small[i]) { while (bit_reader.size() > 0 && bit_reader.bits()[0] == 0) { @@ -561,20 +672,106 @@ td::Result>> boc_decompress_improved_structure_lz4 // Build cell tree std::vector> nodes(node_count); - for (int i = node_count - 1; i >= 0; --i) { + + // Helper: finalize a node by storing refs and finalizing the builder + auto finalize_node = [&](size_t idx) -> td::Status { try { - for (int child_index = 0; child_index < cell_refs_cnt[i]; ++child_index) { - size_t child = boc_graph[i][child_index]; - cell_builders[i].store_ref(nodes[child]); + for (int j = 0; j < cell_refs_cnt[idx]; ++j) { + cell_builders[idx].store_ref(nodes[boc_graph[idx][j]]); } try { - nodes[i] = cell_builders[i].finalize(is_special[i]); + nodes[idx] = cell_builders[idx].finalize(is_special[idx]); } catch (vm::CellBuilder::CellWriteError& e) { - return td::Status::Error("BOC decompression failed: write error while finalizing cell."); + return td::Status::Error(PSTRING() << "BOC decompression failed: failed to finalize node (CellWriteError)"); } } catch (vm::VmError& e) { - return td::Status::Error("BOC decompression failed: VM error during cell construction"); + return td::Status::Error(PSTRING() << "BOC decompression failed: failed to finalize node (VmError)"); } + return td::Status::OK(); + }; + + // Recursively build right subtree under MerkleUpdate, pairing with left subtree, computing sum diffs. + // Sum is accumulated into sum_diff_out (if non-null), similar to compression flow. + std::function build_right_under_mu = + [&](size_t right_idx, size_t left_idx, td::RefInt256* sum_diff_out) -> td::Status { + if (nodes[right_idx].not_null()) { + if (left_idx != std::numeric_limits::max() && sum_diff_out) { + vm::CellSlice cs_left(NoVm(), nodes[left_idx]); + vm::CellSlice cs_right(NoVm(), nodes[right_idx]); + td::RefInt256 vertex_diff = process_shard_accounts_vertex(cs_left, cs_right); + if (vertex_diff.not_null()) { + *sum_diff_out += vertex_diff; + } + } + return td::Status::OK(); + } + td::RefInt256 cur_right_left_diff; + // Build children first + td::RefInt256 sum_child_diff = td::make_refint(0); + for (int j = 0; j < cell_refs_cnt[right_idx]; ++j) { + size_t right_child = boc_graph[right_idx][j]; + size_t left_child = (left_idx != std::numeric_limits::max() && j < cell_refs_cnt[left_idx]) + ? boc_graph[left_idx][j] + : std::numeric_limits::max(); + TRY_STATUS(build_right_under_mu(right_child, left_child, &sum_child_diff)); + } + // If this vertex was depth-balance-compressed, reconstruct its data from left + children sum + if (is_depth_balance[right_idx]) { + vm::CellSlice cs_left(NoVm(), nodes[left_idx]); + td::RefInt256 left_grams = extract_balance_from_depth_balance_info(cs_left); + if (left_grams.is_null()) { + return td::Status::Error("BOC decompression failed: depth-balance left vertex has no grams"); + } + td::RefInt256 expected_right_grams = left_grams; + expected_right_grams += sum_child_diff; + if (!write_depth_balance_grams(cell_builders[right_idx], expected_right_grams)) { + return td::Status::Error("BOC decompression failed: failed to write depth-balance grams"); + } + cur_right_left_diff = sum_child_diff; + } + + // Store children refs and finalize this right node + TRY_STATUS(finalize_node(right_idx)); + + // Compute this vertex diff (right - left) to propagate upward + if (cur_right_left_diff.is_null() && left_idx != std::numeric_limits::max()) { + vm::CellSlice cs_left(NoVm(), nodes[left_idx]); + vm::CellSlice cs_right(NoVm(), nodes[right_idx]); + cur_right_left_diff = process_shard_accounts_vertex(cs_left, cs_right); + } + if (sum_diff_out && cur_right_left_diff.not_null()) { + *sum_diff_out += cur_right_left_diff; + } + return td::Status::OK(); + }; + + // General recursive build that handles MerkleUpdate by pairing left/right subtrees + std::function build_node = [&](size_t idx) -> td::Status { + if (nodes[idx].not_null()) { + return td::Status::OK(); + } + // If this node is a MerkleUpdate, build left subtree normally first, then right subtree paired with left + if (is_merkle_update_node(is_special[idx], cell_builders[idx])) { + size_t left_idx = boc_graph[idx][0]; + size_t right_idx = boc_graph[idx][1]; + TRY_STATUS(build_node(left_idx)); + TRY_STATUS(build_right_under_mu(right_idx, left_idx, nullptr)); + TRY_STATUS(finalize_node(idx)); + return td::Status::OK(); + } else { + // Default: build children normally then finalize + for (int j = 0; j < cell_refs_cnt[idx]; ++j) { + TRY_STATUS(build_node(boc_graph[idx][j])); + } + } + + TRY_STATUS(finalize_node(idx)); + return td::Status::OK(); + }; + + // Build from roots using DFS + for (size_t index : root_indexes) { + TRY_STATUS(build_node(index)); } std::vector> root_nodes; diff --git a/test/test-validator-session-state.cpp b/test/test-validator-session-state.cpp index c854e4bf75..226e20ec15 100644 --- a/test/test-validator-session-state.cpp +++ b/test/test-validator-session-state.cpp @@ -74,6 +74,9 @@ class Description : public ton::validatorsession::ValidatorSessionDescription { auto y = x.as_array(); return y[0]; } + td::Result get_source_idx_safe(ton::PublicKeyHash id) const override { + return get_source_idx(id); + } ton::ValidatorWeight get_node_weight(td::uint32 idx) const override { return 1; } diff --git a/validator-session/candidate-serializer.cpp b/validator-session/candidate-serializer.cpp index 94dfe20480..8a4cfb6a28 100644 --- a/validator-session/candidate-serializer.cpp +++ b/validator-session/candidate-serializer.cpp @@ -14,6 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ +#include "td/utils/Time.h" #include "td/utils/lz4.h" #include "tl-utils/tl-utils.hpp" #include "vm/boc-compression.h" @@ -27,10 +28,17 @@ namespace ton::validatorsession { td::Result serialize_candidate(const tl_object_ptr& block, bool compression_enabled) { if (!compression_enabled) { - return serialize_tl_object(block, true); + auto t_compression_start = td::Time::now(); + auto res = serialize_tl_object(block, true); + LOG(DEBUG) << "Broadcast_benchmark serialize_candidate block_id=" << block->root_hash_.to_hex() + << " time_sec=" << (td::Time::now() - t_compression_start) << " compression=" << "none" + << " original_size=" << block->data_.size() + block->collated_data_.size() + << " compressed_size=" << block->data_.size() + block->collated_data_.size(); + return res; } size_t decompressed_size; - TRY_RESULT(compressed, compress_candidate_data(block->data_, block->collated_data_, decompressed_size)) + TRY_RESULT(compressed, + compress_candidate_data(block->data_, block->collated_data_, decompressed_size, block->root_hash_)) return create_serialize_tl_object( 0, block->src_, block->round_, block->root_hash_, (int)decompressed_size, std::move(compressed)); } @@ -40,7 +48,12 @@ td::Result> deserialize_candi int max_decompressed_data_size, int proto_version) { if (!compression_enabled) { - return fetch_tl_object(data, true); + auto t_decompression_start = td::Time::now(); + TRY_RESULT(res, fetch_tl_object(data, true)); + LOG(DEBUG) << "Broadcast_benchmark deserialize_candidate block_id=" << res->root_hash_.to_hex() + << " time_sec=" << (td::Time::now() - t_decompression_start) << " compression=" << "none" + << " compressed_size=" << res->data_.size() + res->collated_data_.size(); + return std::move(res); } TRY_RESULT(f, fetch_tl_object(data, true)); td::Result> res; @@ -55,7 +68,7 @@ td::Result> deserialize_candi return td::Status::Error("decompressed size is too big"); } TRY_RESULT(p, decompress_candidate_data(c.data_, false, c.decompressed_size_, - max_decompressed_data_size, proto_version)); + max_decompressed_data_size, proto_version, c.root_hash_)); return create_tl_object(c.src_, c.round_, c.root_hash_, std::move(p.first), std::move(p.second)); }(); @@ -65,7 +78,8 @@ td::Result> deserialize_candi if (c.data_.size() > max_decompressed_data_size) { return td::Status::Error("Compressed data is too big"); } - TRY_RESULT(p, decompress_candidate_data(c.data_, true, 0, max_decompressed_data_size, proto_version)); + TRY_RESULT(p, decompress_candidate_data(c.data_, true, 0, max_decompressed_data_size, proto_version, + c.root_hash_)); return create_tl_object(c.src_, c.round_, c.root_hash_, std::move(p.first), std::move(p.second)); }(); @@ -73,8 +87,8 @@ td::Result> deserialize_candi return res; } -td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, - size_t& decompressed_size) { +td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, size_t& decompressed_size, + td::Bits256 root_hash) { vm::BagOfCells boc1, boc2; TRY_STATUS(boc1.deserialize(block)); if (boc1.get_root_count() != 1) { @@ -85,27 +99,36 @@ td::Result compress_candidate_data(td::Slice block, td::Slice c for (int i = 0; i < boc2.get_root_count(); ++i) { roots.push_back(boc2.get_root_cell(i)); } + auto t_compression_start = td::Time::now(); TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(roots), 2)); decompressed_size = data.size(); td::BufferSlice compressed = td::lz4_compress(data); LOG(DEBUG) << "Compressing block candidate: " << block.size() + collated_data.size() << " -> " << compressed.size(); + LOG(DEBUG) << "Broadcast_benchmark compress_candidate_data block_id=" << root_hash.to_hex() + << " time_sec=" << (td::Time::now() - t_compression_start) << " compression=" << "compressed" + << " original_size=" << block.size() + collated_data.size() << " compressed_size=" << compressed.size(); return compressed; } -td::Result> decompress_candidate_data(td::Slice compressed, - bool improved_compression, - int decompressed_size, - int max_decompressed_size, - int proto_version) { +td::Result> decompress_candidate_data( + td::Slice compressed, bool improved_compression, int decompressed_size, int max_decompressed_size, + int proto_version, td::Bits256 root_hash) { std::vector> roots; + auto t_decompression_start = td::Time::now(); if (!improved_compression) { TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size)); if (decompressed.size() != (size_t)decompressed_size) { return td::Status::Error("decompressed size mismatch"); } TRY_RESULT_ASSIGN(roots, vm::std_boc_deserialize_multi(decompressed)); + LOG(DEBUG) << "Broadcast_benchmark decompress_candidate_data block_id=" << root_hash.to_hex() + << " time_sec=" << (td::Time::now() - t_decompression_start) << " compression=" << "compressed" + << " compressed_size=" << compressed.size(); } else { TRY_RESULT_ASSIGN(roots, vm::boc_decompress(compressed, max_decompressed_size)); + LOG(DEBUG) << "Broadcast_benchmark decompress_candidate_data block_id=" << root_hash.to_hex() + << " time_sec=" << (td::Time::now() - t_decompression_start) << " compression=" << "compressedV2" + << " compressed_size=" << compressed.size(); } if (roots.empty()) { return td::Status::Error("boc is empty"); diff --git a/validator-session/candidate-serializer.h b/validator-session/candidate-serializer.h index cf688a6921..75dc1c6cf8 100644 --- a/validator-session/candidate-serializer.h +++ b/validator-session/candidate-serializer.h @@ -27,12 +27,10 @@ td::Result> deserialize_candi int max_decompressed_data_size, int proto_version); -td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, - size_t& decompressed_size); -td::Result> decompress_candidate_data(td::Slice compressed, - bool improved_compression, - int decompressed_size, - int max_decompressed_size, - int proto_version); +td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, size_t& decompressed_size, + td::Bits256 root_hash); +td::Result> decompress_candidate_data( + td::Slice compressed, bool improved_compression, int decompressed_size, int max_decompressed_size, + int proto_version, td::Bits256 root_hash); } // namespace ton::validatorsession diff --git a/validator-session/validator-session-description.h b/validator-session/validator-session-description.h index 03d9d77f6c..197ef3a23f 100644 --- a/validator-session/validator-session-description.h +++ b/validator-session/validator-session-description.h @@ -78,6 +78,7 @@ class ValidatorSessionDescription { virtual PublicKey get_source_public_key(td::uint32 idx) const = 0; virtual adnl::AdnlNodeIdShort get_source_adnl_id(td::uint32 idx) const = 0; virtual td::uint32 get_source_idx(PublicKeyHash id) const = 0; + virtual td::Result get_source_idx_safe(PublicKeyHash id) const = 0; virtual ValidatorWeight get_node_weight(td::uint32 idx) const = 0; virtual td::uint32 get_total_nodes() const = 0; virtual ValidatorWeight get_cutoff_weight() const = 0; diff --git a/validator-session/validator-session-description.hpp b/validator-session/validator-session-description.hpp index af379d0098..7487899af4 100644 --- a/validator-session/validator-session-description.hpp +++ b/validator-session/validator-session-description.hpp @@ -99,6 +99,13 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription { CHECK(it != rev_sources_.end()); return it->second; } + td::Result get_source_idx_safe(PublicKeyHash id) const override { + auto it = rev_sources_.find(id); + if (it == rev_sources_.end()) { + return td::Status::Error("unknown source id"); + } + return it->second; + } ValidatorWeight get_node_weight(td::uint32 idx) const override { CHECK(idx < sources_.size()); return sources_[idx].weight; diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 6f2499adeb..87ae1e0825 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -452,8 +452,9 @@ void ValidatorSessionImpl::process_query(PublicKeyHash src, td::BufferSlice data return; } const SentBlock *block = nullptr; - auto id = description().candidate_id(description().get_source_idx(PublicKeyHash{f->id_->src_}), f->id_->root_hash_, - f->id_->file_hash_, f->id_->collated_data_file_hash_); + TRY_RESULT_PROMISE(promise, source_idx, description().get_source_idx_safe(PublicKeyHash{f->id_->src_})); + auto id = + description().candidate_id(source_idx, f->id_->root_hash_, f->id_->file_hash_, f->id_->collated_data_file_hash_); if (round_id < real_state_->cur_round_seqno()) { block = real_state_->get_committed_block(description(), round_id); if (!block || SentBlock::get_block_id(block) != id) { diff --git a/validator/collator-node/utils.cpp b/validator/collator-node/utils.cpp index 41b6dc4329..26b1095d40 100644 --- a/validator/collator-node/utils.cpp +++ b/validator/collator-node/utils.cpp @@ -31,7 +31,8 @@ tl_object_ptr serialize_candidate(const BlockCa } size_t decompressed_size; td::BufferSlice compressed = - validatorsession::compress_candidate_data(block.data, block.collated_data, decompressed_size).move_as_ok(); + validatorsession::compress_candidate_data(block.data, block.collated_data, decompressed_size, block.id.root_hash) + .move_as_ok(); return create_tl_object( 0, PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), (int)decompressed_size, std::move(compressed)); @@ -40,54 +41,55 @@ tl_object_ptr serialize_candidate(const BlockCa td::Result deserialize_candidate(tl_object_ptr f, int max_decompressed_data_size, int proto_version) { td::Result res; - ton_api::downcast_call( - *f, td::overloaded( - [&](ton_api::collatorNode_candidate& c) { - res = [&]() -> td::Result { - auto hash = td::sha256_bits256(c.collated_data_); - auto key = PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), hash, std::move(c.data_), - std::move(c.collated_data_)}; - }(); - }, - [&](ton_api::collatorNode_compressedCandidate& c) { - res = [&]() -> td::Result { - if (c.decompressed_size_ <= 0) { - return td::Status::Error("invalid decompressed size"); - } - if (c.decompressed_size_ > max_decompressed_data_size) { - return td::Status::Error("decompressed size is too big"); - } - TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, false, c.decompressed_size_, - max_decompressed_data_size, proto_version)); - auto collated_data_hash = td::sha256_bits256(p.second); - auto key = PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), - std::move(p.second)}; - }(); - }, - [&](ton_api::collatorNode_compressedCandidateV2& c) { - res = [&]() -> td::Result { - TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, true, 0, - max_decompressed_data_size, proto_version)); - auto collated_data_hash = td::sha256_bits256(p.second); - auto key = PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), - std::move(p.second)}; - }(); - })); + ton_api::downcast_call(*f, td::overloaded( + [&](ton_api::collatorNode_candidate& c) { + res = [&]() -> td::Result { + auto hash = td::sha256_bits256(c.collated_data_); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), hash, std::move(c.data_), + std::move(c.collated_data_)}; + }(); + }, + [&](ton_api::collatorNode_compressedCandidate& c) { + res = [&]() -> td::Result { + if (c.decompressed_size_ <= 0) { + return td::Status::Error("invalid decompressed size"); + } + if (c.decompressed_size_ > max_decompressed_data_size) { + return td::Status::Error("decompressed size is too big"); + } + TRY_RESULT(p, validatorsession::decompress_candidate_data( + c.data_, false, c.decompressed_size_, max_decompressed_data_size, + proto_version, create_block_id(c.id_).root_hash)); + auto collated_data_hash = td::sha256_bits256(p.second); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, + std::move(p.first), std::move(p.second)}; + }(); + }, + [&](ton_api::collatorNode_compressedCandidateV2& c) { + res = [&]() -> td::Result { + TRY_RESULT(p, validatorsession::decompress_candidate_data( + c.data_, true, 0, max_decompressed_data_size, proto_version, + create_block_id(c.id_).root_hash)); + auto collated_data_hash = td::sha256_bits256(p.second); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, + std::move(p.first), std::move(p.second)}; + }(); + })); return res; } diff --git a/validator/full-node-custom-overlays.cpp b/validator/full-node-custom-overlays.cpp index 7cd4182eb7..e571e1e4b3 100644 --- a/validator/full-node-custom-overlays.cpp +++ b/validator/full-node-custom-overlays.cpp @@ -26,6 +26,12 @@ namespace ton::validator::fullnode { +namespace { + +constexpr const char *k_called_from_custom = "custom"; + +} // namespace + void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { process_block_broadcast(src, query); } @@ -44,7 +50,7 @@ void FullNodeCustomOverlay::process_block_broadcast(PublicKeyHash src, ton_api:: << src; return; } - auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); + auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size(), k_called_from_custom); if (B.is_error()) { LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); return; @@ -92,7 +98,7 @@ void FullNodeCustomOverlay::process_block_candidate_broadcast(PublicKeyHash src, td::uint32 validator_set_hash; td::BufferSlice data; auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, - overlay::Overlays::max_fec_broadcast_size()); + overlay::Overlays::max_fec_broadcast_size(), k_called_from_custom); if (S.is_error()) { LOG(DEBUG) << "dropped broadcast: " << S; return; @@ -144,7 +150,7 @@ void FullNodeCustomOverlay::send_broadcast(BlockBroadcast broadcast) { } VLOG(FULL_NODE_DEBUG) << "Sending block broadcast to custom overlay \"" << name_ << "\": " << broadcast.block_id.to_str(); - auto B = serialize_block_broadcast(broadcast, true); // compression_enabled = true + auto B = serialize_block_broadcast(broadcast, true, k_called_from_custom); // compression_enabled = true if (B.is_error()) { VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); return; @@ -158,8 +164,8 @@ void FullNodeCustomOverlay::send_block_candidate(BlockIdExt block_id, CatchainSe if (!inited_) { return; } - auto B = - serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled + auto B = serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true, + k_called_from_custom); // compression enabled if (B.is_error()) { VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); return; diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index e25f636233..d480ec0454 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -27,6 +27,12 @@ namespace ton::validator::fullnode { +namespace { + +constexpr const char *k_called_from_fast_sync = "fast-sync"; + +} // namespace + void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { process_block_broadcast(src, query); } @@ -40,7 +46,7 @@ void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonN } void FullNodeFastSyncOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { - auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); + auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size(), k_called_from_fast_sync); if (B.is_error()) { LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); return; @@ -109,7 +115,7 @@ void FullNodeFastSyncOverlay::process_block_candidate_broadcast(PublicKeyHash sr td::uint32 validator_set_hash; td::BufferSlice data; auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, - overlay::Overlays::max_fec_broadcast_size()); + overlay::Overlays::max_fec_broadcast_size(), k_called_from_fast_sync); if (S.is_error()) { LOG(DEBUG) << "dropped broadcast: " << S; return; @@ -191,7 +197,7 @@ void FullNodeFastSyncOverlay::send_broadcast(BlockBroadcast broadcast) { } VLOG(FULL_NODE_DEBUG) << "Sending block broadcast in fast sync overlay (with compression): " << broadcast.block_id.to_str(); - auto B = serialize_block_broadcast(broadcast, true); // compression_enabled = true + auto B = serialize_block_broadcast(broadcast, true, k_called_from_fast_sync); // compression_enabled = true if (B.is_error()) { VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); return; @@ -205,8 +211,8 @@ void FullNodeFastSyncOverlay::send_block_candidate(BlockIdExt block_id, Catchain if (!inited_) { return; } - auto B = - serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled + auto B = serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true, + k_called_from_fast_sync); // compression enabled if (B.is_error()) { VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); return; diff --git a/validator/full-node-serializer.cpp b/validator/full-node-serializer.cpp index 4181d69b64..db7390476e 100644 --- a/validator/full-node-serializer.cpp +++ b/validator/full-node-serializer.cpp @@ -15,6 +15,7 @@ along with TON Blockchain Library. If not, see . */ #include "auto/tl/ton_api.hpp" +#include "td/utils/Time.h" #include "td/utils/lz4.h" #include "td/utils/overloaded.h" #include "tl-utils/common-utils.hpp" @@ -28,46 +29,79 @@ namespace ton::validator::fullnode { -td::Result serialize_block_broadcast(const BlockBroadcast& broadcast, bool compression_enabled) { +td::Result serialize_block_broadcast(const BlockBroadcast& broadcast, bool compression_enabled, + std::string called_from) { std::vector> sigs; + size_t total_signatures_size = 0; for (auto& sig : broadcast.signatures) { + total_signatures_size += sig.signature.size(); sigs.emplace_back(create_tl_object(sig.node, sig.signature.clone())); } if (!compression_enabled) { - return create_serialize_tl_object( + auto t_compression_start = td::Time::now(); + auto res = create_serialize_tl_object( create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs), broadcast.proof.clone(), broadcast.data.clone()); + LOG(DEBUG) << "Broadcast_benchmark serialize_block_broadcast block_id=" << broadcast.block_id.to_str() + << " called_from=" << called_from << " time_sec=" << (td::Time::now() - t_compression_start) + << " compression=" << "none" + << " original_size=" << broadcast.data.size() + broadcast.proof.size() + total_signatures_size + << " compressed_size=" << broadcast.data.size() + broadcast.proof.size() + total_signatures_size; + return res; } TRY_RESULT(proof_root, vm::std_boc_deserialize(broadcast.proof)); TRY_RESULT(data_root, vm::std_boc_deserialize(broadcast.data)); + + auto t_compression_start = td::Time::now(); TRY_RESULT(boc, vm::std_boc_serialize_multi({proof_root, data_root}, 2)); td::BufferSlice data = create_serialize_tl_object(std::move(sigs), std::move(boc)); td::BufferSlice compressed = td::lz4_compress(data); + size_t compressed_size = compressed.size(); VLOG(FULL_NODE_DEBUG) << "Compressing block broadcast: " << broadcast.data.size() + broadcast.proof.size() + broadcast.signatures.size() * 96 << " -> " << compressed.size(); - return create_serialize_tl_object( + auto res = create_serialize_tl_object( create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, 0, std::move(compressed)); + LOG(DEBUG) << "Broadcast_benchmark serialize_block_broadcast block_id=" << broadcast.block_id.to_str() + << " called_from=" << called_from << " time_sec=" << (td::Time::now() - t_compression_start) + << " compression=" << "compressed" + << " original_size=" << broadcast.data.size() + broadcast.proof.size() + total_signatures_size + << " compressed_size=" << compressed_size; + return res; } -static td::Result deserialize_block_broadcast(ton_api::tonNode_blockBroadcast& f) { +static td::Result deserialize_block_broadcast(ton_api::tonNode_blockBroadcast& f, + std::string called_from) { + auto block_id = create_block_id(f.id_); + auto t_decompression_start = td::Time::now(); + std::vector signatures; + size_t total_signatures_size = 0; for (auto& sig : f.signatures_) { + total_signatures_size += sig->signature_.size(); signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); } - return BlockBroadcast{create_block_id(f.id_), - std::move(signatures), - static_cast(f.catchain_seqno_), - static_cast(f.validator_set_hash_), - std::move(f.data_), - std::move(f.proof_)}; + auto result = BlockBroadcast{block_id, + std::move(signatures), + static_cast(f.catchain_seqno_), + static_cast(f.validator_set_hash_), + std::move(f.data_), + std::move(f.proof_)}; + LOG(DEBUG) << "Broadcast_benchmark deserialize_block_broadcast block_id=" << block_id.to_str() + << " called_from=" << called_from << " time_sec=" << (td::Time::now() - t_decompression_start) + << " compression=" << "none" + << " compressed_size=" << result.data.size() + result.proof.size() + total_signatures_size; + return result; } static td::Result deserialize_block_broadcast(ton_api::tonNode_blockBroadcastCompressed& f, - int max_decompressed_size) { + int max_decompressed_size, std::string called_from) { + auto block_id = create_block_id(f.id_); + auto t_decompression_start = td::Time::now(); + TRY_RESULT(decompressed, td::lz4_decompress(f.compressed_, max_decompressed_size)); TRY_RESULT(f2, fetch_tl_object(decompressed, true)); std::vector signatures; @@ -78,11 +112,15 @@ static td::Result deserialize_block_broadcast(ton_api::tonNode_b if (roots.size() != 2) { return td::Status::Error("expected 2 roots in boc"); } + LOG(DEBUG) << "Broadcast_benchmark deserialize_block_broadcast block_id=" << block_id.to_str() + << " called_from=" << called_from << " time_sec=" << (td::Time::now() - t_decompression_start) + << " compression=" << "compressed" + << " compressed_size=" << f.compressed_.size(); TRY_RESULT(proof, vm::std_boc_serialize(roots[0], 0)); TRY_RESULT(data, vm::std_boc_serialize(roots[1], 31)); VLOG(FULL_NODE_DEBUG) << "Decompressing block broadcast: " << f.compressed_.size() << " -> " << data.size() + proof.size() + signatures.size() * 96; - return BlockBroadcast{create_block_id(f.id_), + return BlockBroadcast{block_id, std::move(signatures), static_cast(f.catchain_seqno_), static_cast(f.validator_set_hash_), @@ -91,15 +129,24 @@ static td::Result deserialize_block_broadcast(ton_api::tonNode_b } static td::Result deserialize_block_broadcast(ton_api::tonNode_blockBroadcastCompressedV2& f, - int max_decompressed_size) { + int max_decompressed_size, std::string called_from) { + auto block_id = create_block_id(f.id_); + auto t_decompression_start = td::Time::now(); + std::vector signatures; + size_t total_signatures_size = 0; for (auto& sig : f.signatures_) { + total_signatures_size += sig->signature_.size(); signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); } TRY_RESULT(roots, vm::boc_decompress(f.compressed_, max_decompressed_size)); if (roots.size() != 2) { return td::Status::Error("expected 2 roots in boc"); } + LOG(DEBUG) << "Broadcast_benchmark deserialize_block_broadcast block_id=" << block_id.to_str() + << " called_from=" << called_from << " time_sec=" << (td::Time::now() - t_decompression_start) + << " compression=" << "compressedV2" + << " compressed_size=" << f.compressed_.size() + total_signatures_size; TRY_RESULT(proof, vm::std_boc_serialize(roots[0], 0)); TRY_RESULT(data, vm::std_boc_serialize(roots[1], 31)); VLOG(FULL_NODE_DEBUG) << "Decompressing block broadcast: " << f.compressed_.size() << " -> " @@ -112,56 +159,76 @@ static td::Result deserialize_block_broadcast(ton_api::tonNode_b std::move(proof)}; } -td::Result deserialize_block_broadcast(ton_api::tonNode_Broadcast& obj, - int max_decompressed_data_size) { +td::Result deserialize_block_broadcast(ton_api::tonNode_Broadcast& obj, int max_decompressed_data_size, + std::string called_from) { td::Result B; - ton_api::downcast_call(obj, - td::overloaded([&](ton_api::tonNode_blockBroadcast& f) { B = deserialize_block_broadcast(f); }, - [&](ton_api::tonNode_blockBroadcastCompressed& f) { - B = deserialize_block_broadcast(f, max_decompressed_data_size); - }, - [&](ton_api::tonNode_blockBroadcastCompressedV2& f) { - B = deserialize_block_broadcast(f, max_decompressed_data_size); - }, - [&](auto&) { B = td::Status::Error("unknown broadcast type"); })); + ton_api::downcast_call( + obj, td::overloaded([&](ton_api::tonNode_blockBroadcast& f) { B = deserialize_block_broadcast(f, called_from); }, + [&](ton_api::tonNode_blockBroadcastCompressed& f) { + B = deserialize_block_broadcast(f, max_decompressed_data_size, called_from); + }, + [&](ton_api::tonNode_blockBroadcastCompressedV2& f) { + B = deserialize_block_broadcast(f, max_decompressed_data_size, called_from); + }, + [&](auto&) { B = td::Status::Error("unknown broadcast type"); })); return B; } td::Result serialize_block_full(const BlockIdExt& id, td::Slice proof, td::Slice data, bool is_proof_link, bool compression_enabled) { if (!compression_enabled) { - return create_serialize_tl_object(create_tl_block_id(id), td::BufferSlice(proof), - td::BufferSlice(data), is_proof_link); + auto t_compression_start = td::Time::now(); + auto res = create_serialize_tl_object(create_tl_block_id(id), td::BufferSlice(proof), + td::BufferSlice(data), is_proof_link); + LOG(DEBUG) << "Broadcast_benchmark serialize_block_full block_id=" << id.to_str() + << " time_sec=" << (td::Time::now() - t_compression_start) << " compression=" << "none" + << " original_size=" << data.size() + proof.size() << " compressed_size=" << data.size() + proof.size(); + return res; } TRY_RESULT(proof_root, vm::std_boc_deserialize(proof)); TRY_RESULT(data_root, vm::std_boc_deserialize(data)); + auto t_compression_start = td::Time::now(); TRY_RESULT(boc, vm::std_boc_serialize_multi({proof_root, data_root}, 2)); td::BufferSlice compressed = td::lz4_compress(boc); + size_t compressed_size = compressed.size(); VLOG(FULL_NODE_DEBUG) << "Compressing block full: " << data.size() + proof.size() << " -> " << compressed.size(); - return create_serialize_tl_object(create_tl_block_id(id), 0, - std::move(compressed), is_proof_link); + auto res = create_serialize_tl_object(create_tl_block_id(id), 0, + std::move(compressed), is_proof_link); + LOG(DEBUG) << "Broadcast_benchmark serialize_block_full block_id=" << id.to_str() + << " time_sec=" << (td::Time::now() - t_compression_start) << " compression=" << "compressed" + << " original_size=" << data.size() + proof.size() << " compressed_size=" << compressed_size; + return res; } static td::Status deserialize_block_full(ton_api::tonNode_dataFull& f, BlockIdExt& id, td::BufferSlice& proof, td::BufferSlice& data, bool& is_proof_link) { id = create_block_id(f.id_); + auto t_decompression_start = td::Time::now(); proof = std::move(f.proof_); data = std::move(f.block_); is_proof_link = f.is_link_; + LOG(DEBUG) << "Broadcast_benchmark deserialize_block_full block_id=" << id.to_str() + << " time_sec=" << (td::Time::now() - t_decompression_start) << " compression=" << "none" + << " compressed_size=" << proof.size() + data.size(); return td::Status::OK(); } static td::Status deserialize_block_full(ton_api::tonNode_dataFullCompressed& f, BlockIdExt& id, td::BufferSlice& proof, td::BufferSlice& data, bool& is_proof_link, int max_decompressed_size) { + id = create_block_id(f.id_); + auto t_decompression_start = td::Time::now(); + TRY_RESULT(decompressed, td::lz4_decompress(f.compressed_, max_decompressed_size)); TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed, 2)); if (roots.size() != 2) { return td::Status::Error("expected 2 roots in boc"); } + LOG(DEBUG) << "Broadcast_benchmark deserialize_block_full block_id=" << id.to_str() + << " time_sec=" << (td::Time::now() - t_decompression_start) << " compression=" << "compressed" + << " compressed_size=" << f.compressed_.size(); TRY_RESULT_ASSIGN(proof, vm::std_boc_serialize(roots[0], 0)); TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(roots[1], 31)); VLOG(FULL_NODE_DEBUG) << "Decompressing block full: " << f.compressed_.size() << " -> " << data.size() + proof.size(); - id = create_block_id(f.id_); is_proof_link = f.is_link_; return td::Status::OK(); } @@ -169,15 +236,20 @@ static td::Status deserialize_block_full(ton_api::tonNode_dataFullCompressed& f, static td::Status deserialize_block_full(ton_api::tonNode_dataFullCompressedV2& f, BlockIdExt& id, td::BufferSlice& proof, td::BufferSlice& data, bool& is_proof_link, int max_decompressed_size) { + id = create_block_id(f.id_); + auto t_decompression_start = td::Time::now(); + TRY_RESULT(roots, vm::boc_decompress(f.compressed_, max_decompressed_size)); if (roots.size() != 2) { return td::Status::Error("expected 2 roots in boc"); } + LOG(DEBUG) << "Broadcast_benchmark deserialize_block_full block_id=" << id.to_str() + << " time_sec=" << (td::Time::now() - t_decompression_start) << " compression=" << "compressedV2" + << " compressed_size=" << f.compressed_.size(); TRY_RESULT_ASSIGN(proof, vm::std_boc_serialize(roots[0], 0)); TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(roots[1], 31)); VLOG(FULL_NODE_DEBUG) << "Decompressing block full V2: " << f.compressed_.size() << " -> " << data.size() + proof.size(); - id = create_block_id(f.id_); is_proof_link = f.is_link_; return td::Status::OK(); } @@ -200,40 +272,64 @@ td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id td::Result serialize_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::Slice data, - bool compression_enabled) { + bool compression_enabled, std::string called_from) { if (!compression_enabled) { - return create_serialize_tl_object( + auto t_compression_start = td::Time::now(); + auto res = create_serialize_tl_object( create_tl_block_id(block_id), cc_seqno, validator_set_hash, create_tl_object(Bits256::zero(), td::BufferSlice()), td::BufferSlice(data)); + LOG(DEBUG) << "Broadcast_benchmark serialize_block_candidate_broadcast block_id=" << block_id.to_str() + << " called_from=" << called_from << " time_sec=" << (td::Time::now() - t_compression_start) + << " compression=" << "none" + << " original_size=" << data.size() << " compressed_size=" << data.size(); + return res; } TRY_RESULT(root, vm::std_boc_deserialize(data)); + auto t_compression_start = td::Time::now(); TRY_RESULT(data_new, vm::std_boc_serialize(root, 2)); td::BufferSlice compressed = td::lz4_compress(data_new); - VLOG(FULL_NODE_DEBUG) << "Compressing block candidate broadcast: " << data.size() << " -> " << compressed.size(); - return create_serialize_tl_object( + auto compressed_size = compressed.size(); + VLOG(FULL_NODE_DEBUG) << "Compressing block candidate broadcast: " << data.size() << " -> " << compressed_size; + auto res = create_serialize_tl_object( create_tl_block_id(block_id), cc_seqno, validator_set_hash, create_tl_object(Bits256::zero(), td::BufferSlice()), 0, std::move(compressed)); + LOG(DEBUG) << "Broadcast_benchmark serialize_block_candidate_broadcast block_id=" << block_id.to_str() + << " called_from=" << called_from << " time_sec=" << (td::Time::now() - t_compression_start) + << " compression=" << "compressed" + << " original_size=" << data.size() << " compressed_size=" << compressed_size; + return res; } static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBlockCandidateBroadcast& obj, BlockIdExt& block_id, CatchainSeqno& cc_seqno, - td::uint32& validator_set_hash, td::BufferSlice& data) { + td::uint32& validator_set_hash, td::BufferSlice& data, + std::string called_from) { + auto t_decompression_start = td::Time::now(); block_id = create_block_id(obj.id_); cc_seqno = obj.catchain_seqno_; validator_set_hash = obj.validator_set_hash_; data = std::move(obj.data_); + LOG(DEBUG) << "Broadcast_benchmark deserialize_block_candidate_broadcast block_id=" << block_id.to_str() + << " called_from=" << called_from << " time_sec=" << (td::Time::now() - t_decompression_start) + << " compression=" << "none" + << " compressed_size=" << data.size(); return td::Status::OK(); } static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBlockCandidateBroadcastCompressed& obj, BlockIdExt& block_id, CatchainSeqno& cc_seqno, td::uint32& validator_set_hash, td::BufferSlice& data, - int max_decompressed_data_size) { + int max_decompressed_data_size, std::string called_from) { block_id = create_block_id(obj.id_); + auto t_decompression_start = td::Time::now(); cc_seqno = obj.catchain_seqno_; validator_set_hash = obj.validator_set_hash_; TRY_RESULT(decompressed, td::lz4_decompress(obj.compressed_, max_decompressed_data_size)); TRY_RESULT(root, vm::std_boc_deserialize(decompressed)); + LOG(DEBUG) << "Broadcast_benchmark deserialize_block_candidate_broadcast block_id=" << block_id.to_str() + << " called_from=" << called_from << " time_sec=" << (td::Time::now() - t_decompression_start) + << " compression=" << "compressed" + << " compressed_size=" << obj.compressed_.size(); TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(root, 31)); VLOG(FULL_NODE_DEBUG) << "Decompressing block candidate broadcast: " << obj.compressed_.size() << " -> " << data.size(); @@ -243,14 +339,19 @@ static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBloc static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBlockCandidateBroadcastCompressedV2& obj, BlockIdExt& block_id, CatchainSeqno& cc_seqno, td::uint32& validator_set_hash, td::BufferSlice& data, - int max_decompressed_data_size) { + int max_decompressed_data_size, std::string called_from) { block_id = create_block_id(obj.id_); + auto t_decompression_start = td::Time::now(); cc_seqno = obj.catchain_seqno_; validator_set_hash = obj.validator_set_hash_; TRY_RESULT(roots, vm::boc_decompress(obj.compressed_, max_decompressed_data_size)); if (roots.size() != 1) { return td::Status::Error("expected 1 root in boc"); } + LOG(DEBUG) << "Broadcast_benchmark deserialize_block_candidate_broadcast block_id=" << block_id.to_str() + << " called_from=" << called_from << " time_sec=" << (td::Time::now() - t_decompression_start) + << " compression=" << "compressedV2" + << " compressed_size=" << obj.compressed_.size(); auto root = std::move(roots[0]); TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(root, 31)); VLOG(FULL_NODE_DEBUG) << "Decompressing block candidate broadcast V2: " << obj.compressed_.size() << " -> " @@ -260,22 +361,24 @@ static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBloc td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_Broadcast& obj, BlockIdExt& block_id, CatchainSeqno& cc_seqno, td::uint32& validator_set_hash, - td::BufferSlice& data, int max_decompressed_data_size) { + td::BufferSlice& data, int max_decompressed_data_size, + std::string called_from) { td::Status S; - ton_api::downcast_call(obj, td::overloaded( - [&](ton_api::tonNode_newBlockCandidateBroadcast& f) { - S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, - data); - }, - [&](ton_api::tonNode_newBlockCandidateBroadcastCompressed& f) { - S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, - data, max_decompressed_data_size); - }, - [&](ton_api::tonNode_newBlockCandidateBroadcastCompressedV2& f) { - S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, - data, max_decompressed_data_size); - }, - [&](auto&) { S = td::Status::Error("unknown data type"); })); + ton_api::downcast_call(obj, + td::overloaded( + [&](ton_api::tonNode_newBlockCandidateBroadcast& f) { + S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, + data, called_from); + }, + [&](ton_api::tonNode_newBlockCandidateBroadcastCompressed& f) { + S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, + data, max_decompressed_data_size, called_from); + }, + [&](ton_api::tonNode_newBlockCandidateBroadcastCompressedV2& f) { + S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, + data, max_decompressed_data_size, called_from); + }, + [&](auto&) { S = td::Status::Error("unknown data type"); })); return S; } diff --git a/validator/full-node-serializer.hpp b/validator/full-node-serializer.hpp index d28495f691..48871f982f 100644 --- a/validator/full-node-serializer.hpp +++ b/validator/full-node-serializer.hpp @@ -20,8 +20,10 @@ namespace ton::validator::fullnode { -td::Result serialize_block_broadcast(const BlockBroadcast& broadcast, bool compression_enabled); -td::Result deserialize_block_broadcast(ton_api::tonNode_Broadcast& obj, int max_decompressed_data_size); +td::Result serialize_block_broadcast(const BlockBroadcast& broadcast, bool compression_enabled, + std::string called_from); +td::Result deserialize_block_broadcast(ton_api::tonNode_Broadcast& obj, int max_decompressed_data_size, + std::string called_from); td::Result serialize_block_full(const BlockIdExt& id, td::Slice proof, td::Slice data, bool is_proof_link, bool compression_enabled); @@ -30,9 +32,10 @@ td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id td::Result serialize_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::Slice data, - bool compression_enabled); + bool compression_enabled, std::string called_from); td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_Broadcast& obj, BlockIdExt& block_id, CatchainSeqno& cc_seqno, td::uint32& validator_set_hash, - td::BufferSlice& data, int max_decompressed_data_size); + td::BufferSlice& data, int max_decompressed_data_size, + std::string called_from); } // namespace ton::validator::fullnode diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index d0f75f82c0..60a69ce18c 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -49,6 +49,12 @@ namespace validator { namespace fullnode { +namespace { + +constexpr const char *k_called_from_public = "public"; + +} // namespace + Neighbour Neighbour::zero = Neighbour{adnl::AdnlNodeIdShort::zero()}; void Neighbour::update_proto_version(ton_api::tonNode_capabilities &q) { @@ -791,7 +797,7 @@ void FullNodeShardImpl::process_block_candidate_broadcast(PublicKeyHash src, ton td::uint32 validator_set_hash; td::BufferSlice data; auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, - overlay::Overlays::max_fec_broadcast_size()); + overlay::Overlays::max_fec_broadcast_size(), k_called_from_public); if (data.size() > FullNode::max_block_size()) { VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; return; @@ -818,7 +824,7 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_bl } void FullNodeShardImpl::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { - auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); + auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size(), k_called_from_public); if (B.is_error()) { LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); return; @@ -915,8 +921,8 @@ void FullNodeShardImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno UNREACHABLE(); return; } - auto B = - serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled + auto B = serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true, + k_called_from_public); // compression enabled if (B.is_error()) { VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); return; @@ -932,7 +938,7 @@ void FullNodeShardImpl::send_broadcast(BlockBroadcast broadcast) { return; } VLOG(FULL_NODE_DEBUG) << "Sending block broadcast in private overlay: " << broadcast.block_id.to_str(); - auto B = serialize_block_broadcast(broadcast, false); // compression_enabled = false + auto B = serialize_block_broadcast(broadcast, false, k_called_from_public); // compression_enabled = false if (B.is_error()) { VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); return; From 85ac200bb92b9d4cfca6d4a93e8f3706022046ab Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Thu, 4 Dec 2025 23:51:03 -0500 Subject: [PATCH 505/524] Do not require approve signatures in run_accept_block_query (#1937) They are unused in AcceptBlockQuery. Co-authored-by: SpyCheese --- validator/fabric.h | 4 ++-- validator/impl/accept-block.cpp | 5 ++--- validator/impl/accept-block.hpp | 6 ++---- validator/impl/fabric.cpp | 10 +++++----- validator/validator-group.cpp | 2 +- 5 files changed, 12 insertions(+), 15 deletions(-) diff --git a/validator/fabric.h b/validator/fabric.h index a2e7c6c040..94b5cc54b5 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -85,8 +85,8 @@ void run_check_external_message(td::Ref message, td::actor::ActorId< void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, int send_broadcast_mode, bool apply, - td::actor::ActorId manager, td::Promise promise); + int send_broadcast_mode, bool apply, td::actor::ActorId manager, + td::Promise promise); void run_fake_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::actor::ActorId manager, td::Promise promise); diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index 310f687123..822699d94c 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -39,14 +39,13 @@ using namespace std::literals::string_literals; AcceptBlockQuery::AcceptBlockQuery(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, int send_broadcast_mode, bool apply, - td::actor::ActorId manager, td::Promise promise) + int send_broadcast_mode, bool apply, td::actor::ActorId manager, + td::Promise promise) : id_(id) , data_(std::move(data)) , prev_(std::move(prev)) , validator_set_(std::move(validator_set)) , signatures_(std::move(signatures)) - , approve_signatures_(std::move(approve_signatures)) , is_fake_(false) , is_fork_(false) , send_broadcast_mode_(send_broadcast_mode) diff --git a/validator/impl/accept-block.hpp b/validator/impl/accept-block.hpp index 61aa323989..7267cbcb08 100644 --- a/validator/impl/accept-block.hpp +++ b/validator/impl/accept-block.hpp @@ -50,9 +50,8 @@ class AcceptBlockQuery : public td::actor::Actor { struct IsFake {}; struct ForceFork {}; AcceptBlockQuery(BlockIdExt id, td::Ref data, std::vector prev, - td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, int send_broadcast_mode, bool apply, - td::actor::ActorId manager, td::Promise promise); + td::Ref validator_set, td::Ref signatures, int send_broadcast_mode, + bool apply, td::actor::ActorId manager, td::Promise promise); AcceptBlockQuery(IsFake fake, BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::actor::ActorId manager, td::Promise promise); @@ -98,7 +97,6 @@ class AcceptBlockQuery : public td::actor::Actor { std::vector prev_; Ref validator_set_; Ref signatures_; - Ref approve_signatures_; bool is_fake_; bool is_fork_; int send_broadcast_mode_{0}; diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 7b5b439f4b..3f98cce258 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -131,11 +131,11 @@ td::Result> create_ihr_message(td::BufferSlice data) { void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, int send_broadcast_mode, bool apply, - td::actor::ActorId manager, td::Promise promise) { - td::actor::create_actor( - PSTRING() << "accept" << id.id.to_str(), id, std::move(data), prev, std::move(validator_set), - std::move(signatures), std::move(approve_signatures), send_broadcast_mode, apply, manager, std::move(promise)) + int send_broadcast_mode, bool apply, td::actor::ActorId manager, + td::Promise promise) { + td::actor::create_actor(PSTRING() << "accept" << id.id.to_str(), id, std::move(data), prev, + std::move(validator_set), std::move(signatures), send_broadcast_mode, apply, + manager, std::move(promise)) .release(); } diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index f603414bdc..6e46211074 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -345,7 +345,7 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref }); run_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, std::move(sig_set), - std::move(approve_sig_set), send_broadcast_mode, monitoring_shard_, manager_, std::move(P)); + send_broadcast_mode, monitoring_shard_, manager_, std::move(P)); } void ValidatorGroup::skip_round(td::uint32 round_id) { From f801e1c1a4ddb5abf41033f51a7c74ab72dfdc07 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Fri, 5 Dec 2025 10:44:12 +0300 Subject: [PATCH 506/524] Update changelog --- Changelog.md | 20 ++++++++++++++++++++ recent_changelog.md | 24 +++++++++++++++++------- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/Changelog.md b/Changelog.md index b5ae89f86a..3580073ddc 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,23 @@ +## 2025.12 Update + +1. `celldb-v2` enabled by default. +2. Fast state serializer (sharded serialization) enabled by default. +3. Using `_` instead of `:` in package filenames for better compatibility with Windows and network protocols. +4. Introduced parallelism in the validator engine. +5. Added Python-based testing framework. +6. Various fixes in emulator, node, TVM, `Asm.fif`. +7. Introduced improved network traffic compression. +8. BLST updated. + +Internal code changes: + +1. Introduced support for coroutines in actors. +2. Enabled `clang-format` 21. +3. Removed the virtualization level concept. +4. Minor code style and cleanliness improvements. + +Besides the work of the core team, this update also includes contributions from Tonstudio team members: @pyAndr3w (`Asm.fif`), @Gusarich (TVM fixes in `arithops`), @Kaladin13 (node fixes). + ## 2025.11 Update 1. [TVM version v12](./doc/GlobalVersions.md) update: [forbid unused high bits in extra_flags](https://github.com/ton-blockchain/TEPs/pull/503/commits/d949d70d5a69026d273cbbc07653d12c4373117a), [bounce extra_flags equal to initial message extra_flags](https://github.com/ton-blockchain/TEPs/pull/503/commits/d33ff342d69de04f1c33d11360dcf06b63a6c21e), [new TVM opcodes](https://github.com/ton-blockchain/ton/commit/ecd8fbb833c408eb34ec1aa4516e9e4344b54a22). diff --git a/recent_changelog.md b/recent_changelog.md index 422d4d8a1f..5ac2d295d9 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,9 +1,19 @@ -## 2025.11 Update +## 2025.12 Update -1. [TVM version v12](./doc/GlobalVersions.md) update: [forbid unused high bits in extra_flags](https://github.com/ton-blockchain/TEPs/pull/503/commits/d949d70d5a69026d273cbbc07653d12c4373117a), [bounce extra_flags equal to initial message extra_flags](https://github.com/ton-blockchain/TEPs/pull/503/commits/d33ff342d69de04f1c33d11360dcf06b63a6c21e), [new TVM opcodes](https://github.com/ton-blockchain/ton/commit/ecd8fbb833c408eb34ec1aa4516e9e4344b54a22). -2. Abseil upgrade -3. Improvements in node synchronisation -4. Fixing rare ArchiveManager issues -5. Various improvements in logging, builds, DHT node behavior, private net launching, failure handlers. +1. `celldb-v2` enabled by default. +2. Fast state serializer (sharded serialization) enabled by default. +3. Using `_` instead of `:` in package filenames for better compatibility with Windows and network protocols. +4. Introduced parallelism in the validator engine. +5. Added Python-based testing framework. +6. Various fixes in emulator, node, TVM, `Asm.fif`. +7. Introduced improved network traffic compression. +8. BLST updated. -Besides the work of the core team, this update is based on the efforts of the @Lapo4kaKek and [Vahagn x.com/vah_13](https://x.com/vah_13). +Internal code changes: + +1. Introduced support for coroutines in actors. +2. Enabled `clang-format` 21. +3. Removed the virtualization level concept. +4. Minor code style and cleanliness improvements. + +Besides the work of the core team, this update also includes contributions from: @pyAndr3w (`Asm.fif`), @Gusarich (TVM fixes in `arithops`), @Kaladin13 (node fixes). From 55a0b52e111947ebb7ce41e9368dc4233684b8ac Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 15 Dec 2025 18:40:32 +0300 Subject: [PATCH 507/524] Merge pull request #1947 from ton-blockchain/ext-messages Broadcast multiple messages to one wallet --- create-hardfork/create-hardfork.cpp | 5 +- crypto/block/block.tlb | 2 + tdactor/td/actor/coro_task.h | 4 +- test/test-ton-collator.cpp | 5 +- validator-engine/validator-engine.cpp | 7 +- validator-engine/validator-engine.hpp | 4 +- validator/fabric.h | 6 +- validator/full-node-custom-overlays.cpp | 5 +- validator/full-node-shard.cpp | 40 ++- validator/full-node-shard.hpp | 2 + validator/impl/CMakeLists.txt | 2 + validator/impl/ext-message-pool.cpp | 330 +++++++++++++++++++++++ validator/impl/ext-message-pool.hpp | 165 ++++++++++++ validator/impl/external-message.cpp | 256 +++++++++++++++--- validator/impl/external-message.hpp | 13 +- validator/impl/fabric.cpp | 13 +- validator/impl/liteserver.cpp | 31 +-- validator/interfaces/validator-manager.h | 1 - validator/manager-disk.cpp | 11 +- validator/manager-disk.hpp | 8 +- validator/manager-hardfork.cpp | 9 +- validator/manager-hardfork.hpp | 8 +- validator/manager.cpp | 293 ++++---------------- validator/manager.hpp | 116 +------- validator/validator-options.cpp | 6 +- validator/validator-options.hpp | 8 +- validator/validator.h | 13 +- 27 files changed, 865 insertions(+), 498 deletions(-) create mode 100644 validator/impl/ext-message-pool.cpp create mode 100644 validator/impl/ext-message-pool.hpp diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 5cabac038a..db9335c61b 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -209,8 +209,9 @@ class HardforkCreator : public td::actor::Actor { validator_manager_ = ton::validator::ValidatorManagerHardforkFactory::create(opts, shard_, shard_top_block_id_, db_root_); for (auto &msg : ext_msgs_) { - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManager::new_external_message, - std::move(msg), 0); + td::actor::ask(validator_manager_, &ton::validator::ValidatorManager::new_external_message_broadcast, + std::move(msg), 0) + .detach(); } for (auto &topmsg : top_shard_descrs_) { td::actor::send_closure(validator_manager_, diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 03b92cd2b4..c269440a06 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -159,6 +159,8 @@ message$_ {X:Type} info:CommonMsgInfo message$_ {X:Type} info:CommonMsgInfoRelaxed init:(Maybe (Either StateInit ^StateInit)) body:(Either X ^X) = MessageRelaxed X; + +_ init:(Maybe (Either StateInit ^StateInit)) = EitherStateInit; _ (Message Any) = MessageAny; diff --git a/tdactor/td/actor/coro_task.h b/tdactor/td/actor/coro_task.h index 4e5c904f6c..d624bbe5f1 100644 --- a/tdactor/td/actor/coro_task.h +++ b/tdactor/td/actor/coro_task.h @@ -288,7 +288,7 @@ struct promise_type : promise_value> { } }; -template +template struct [[nodiscard]] Task { using value_type = T; @@ -360,7 +360,7 @@ struct [[nodiscard]] Task { } }; -template +template struct [[nodiscard]] StartedTask { using value_type = T; diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 7900bb474b..d43213f759 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -292,8 +292,9 @@ class TestNode : public td::actor::Actor { validator_manager_ = ton::validator::ValidatorManagerDiskFactory::create(ton::PublicKeyHash::zero(), opts, shard_, shard_top_block_id_, db_root_); for (auto &msg : ext_msgs_) { - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManager::new_external_message, - std::move(msg), 0); + td::actor::ask(validator_manager_, &ton::validator::ValidatorManager::new_external_message_broadcast, + std::move(msg), 0) + .detach(); } for (auto &topmsg : top_shard_descrs_) { td::actor::send_closure(validator_manager_, diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 838bd5fbab..cfe51f052e 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -5278,11 +5278,8 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_state_ttl, v); }); return td::Status::OK(); }); - p.add_checked_option('m', "mempool-num", "Maximal number of mempool external message", [&](td::Slice fname) { - auto v = td::to_double(fname); - if (v < 0) { - return td::Status::Error("mempool-num should be non-negative"); - } + p.add_checked_option('m', "mempool-num", "Maximal number of mempool external message", [&](td::Slice s) { + TRY_RESULT(v, td::to_integer_safe(s)); acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); }); return td::Status::OK(); }); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 19991a065b..074dfcc973 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -224,7 +224,7 @@ class ValidatorEngine : public td::actor::Actor { std::map control_permissions_; double state_ttl_ = 0; - double max_mempool_num_ = 0; + size_t max_mempool_num_ = 0; double block_ttl_ = 0; double sync_ttl_ = 0; double archive_ttl_ = 0; @@ -288,7 +288,7 @@ class ValidatorEngine : public td::actor::Actor { void set_state_ttl(double t) { state_ttl_ = t; } - void set_max_mempool_num(double t) { + void set_max_mempool_num(size_t t) { max_mempool_num_ = t; } void set_block_ttl(double t) { diff --git a/validator/fabric.h b/validator/fabric.h index 94b5cc54b5..fdef9aad90 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -20,6 +20,7 @@ #include "interfaces/db.h" #include "interfaces/validator-manager.h" +#include "td/actor/coro_utils.h" #include "validator.h" @@ -80,9 +81,6 @@ td::Result>> create_new_shard_bloc td::Ref create_signature_set(std::vector sig_set); -void run_check_external_message(td::Ref message, td::actor::ActorId manager, - td::Promise> promise); - void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, int send_broadcast_mode, bool apply, td::actor::ActorId manager, @@ -118,6 +116,8 @@ void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, td::Promise, UnixTime, LogicalTime, std::unique_ptr>> promise); +td::actor::Task, UnixTime, LogicalTime, std::unique_ptr>> +run_fetch_account_state(WorkchainId wc, StdSmcAddress addr, td::actor::ActorId manager); void run_validate_shard_block_description(td::BufferSlice data, BlockHandle masterchain_block, td::Ref masterchain_state, td::actor::ActorId manager, td::Timestamp timeout, diff --git a/validator/full-node-custom-overlays.cpp b/validator/full-node-custom-overlays.cpp index e571e1e4b3..8ab441e92c 100644 --- a/validator/full-node-custom-overlays.cpp +++ b/validator/full-node-custom-overlays.cpp @@ -69,8 +69,9 @@ void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNod } VLOG(FULL_NODE_DEBUG) << "Got external message in custom overlay \"" << name_ << "\" from " << src << " (priority=" << it->second << ")"; - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_external_message, - std::move(query.message_->data_), it->second); + td::actor::ask(validator_manager_, &ValidatorManagerInterface::new_external_message_broadcast, + std::move(query.message_->data_), it->second) + .detach(); } void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 60a69ce18c..a4094a6d86 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -122,34 +122,31 @@ void FullNodeShardImpl::create_overlay() { } void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broadcast, td::Promise promise) { - if (!active_) { - return promise.set_error(td::Status::Error("cannot check broadcast: shard is not active")); - } - auto B = fetch_tl_object(std::move(broadcast), true); - if (B.is_error()) { - return promise.set_error(B.move_as_error_prefix("failed to parse external message broadcast: ")); + TRY_RESULT_PROMISE(promise, message, + fetch_tl_object(std::move(broadcast), true)); + if (opts_.config_.ext_messages_broadcast_disabled_) { + promise.set_error(td::Status::Error("rebroadcasting external messages is disabled")); + promise = [](td::Result) {}; } + process_external_message_broadcast(*message, std::move(promise)); +} - auto q = B.move_as_ok(); - auto hash = td::sha256_bits256(q->message_->data_); +void FullNodeShardImpl::process_external_message_broadcast(ton_api::tonNode_externalMessageBroadcast &message, + td::Promise promise) { + if (!active_) { + return promise.set_error(td::Status::Error("cannot process broadcast: shard is not active")); + } + auto hash = td::sha256_bits256(message.message_->data_); if (!processed_ext_msg_broadcasts_.insert(hash).second) { return promise.set_error(td::Status::Error("duplicate external message broadcast")); } - if (opts_.config_.ext_messages_broadcast_disabled_) { - promise.set_error(td::Status::Error("rebroadcasting external messages is disabled")); - promise = [manager = validator_manager_, message = q->message_->data_.clone()](td::Result R) mutable { - if (R.is_ok()) { - td::actor::send_closure(manager, &ValidatorManagerInterface::new_external_message, std::move(message), 0); - } - }; - } - if (my_ext_msg_broadcasts_.count(hash)) { - // Don't re-check messages that were sent by us + if (my_ext_msg_broadcasts_.contains(hash)) { + // Don't process messages that were sent by us promise.set_result(td::Unit()); return; } - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_external_message, - std::move(q->message_->data_), promise.wrap([](td::Ref) { return td::Unit(); })); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_external_message_broadcast, + std::move(message.message_->data_), 0, std::move(promise)); } void FullNodeShardImpl::remove_neighbour(adnl::AdnlNodeIdShort id) { @@ -766,8 +763,7 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ih } void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query) { - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_external_message, - std::move(query.message_->data_), 0); + process_external_message_broadcast(query, [](td::Result) {}); } void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 3fdbb761b1..83e2962d7f 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -170,6 +170,8 @@ class FullNodeShardImpl : public FullNodeShard { void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void check_broadcast(PublicKeyHash src, td::BufferSlice query, td::Promise promise); + void process_external_message_broadcast(ton_api::tonNode_externalMessageBroadcast &message, + td::Promise promise); void get_stats_extra(td::Promise promise); void remove_neighbour(adnl::AdnlNodeIdShort id); diff --git a/validator/impl/CMakeLists.txt b/validator/impl/CMakeLists.txt index 37ec9d4254..22680fe2f7 100644 --- a/validator/impl/CMakeLists.txt +++ b/validator/impl/CMakeLists.txt @@ -6,6 +6,7 @@ set(TON_VALIDATOR_SOURCE collator.cpp config.cpp external-message.cpp + ext-message-pool.cpp fabric.cpp ihr-message.cpp liteserver.cpp @@ -26,6 +27,7 @@ set(TON_VALIDATOR_SOURCE collator.h config.hpp external-message.hpp + ext-message-pool.hpp ihr-message.hpp liteserver.hpp liteserver-cache.hpp diff --git a/validator/impl/ext-message-pool.cpp b/validator/impl/ext-message-pool.cpp new file mode 100644 index 0000000000..72569e801b --- /dev/null +++ b/validator/impl/ext-message-pool.cpp @@ -0,0 +1,330 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "td/utils/Random.h" + +#include "ext-message-pool.hpp" +#include "external-message.hpp" +#include "fabric.h" +#include "transaction.h" + +namespace ton::validator { +td::actor::Task ExtMessagePool::check_add_external_message(td::BufferSlice data, + int priority, + bool add_to_mempool) { + if (last_masterchain_state_.is_null()) { + co_return td::Status::Error(ErrorCode::notready, "not ready"); + } + auto message = co_await create_ext_message(std::move(data), last_masterchain_state_->get_ext_msg_limits()); + WorkchainId wc = message->wc(); + StdSmcAddress addr = message->addr(); + if (checked_ext_msg_counter_.get_msg_count(wc, addr) >= MAX_EXT_MSG_PER_ADDR) { + co_return td::Status::Error(PSTRING() << "too many external messages to address " << wc << ":" << addr.to_hex()); + } + td::optional msg_seqno; + auto result = co_await check_message(message, msg_seqno).wrap(); + ++(result.is_ok() ? total_check_ext_messages_ok_ : total_check_ext_messages_error_); + if (result.is_error()) { + co_return result.move_as_error(); + } + if (checked_ext_msg_counter_.inc_msg_count(wc, addr) > MAX_EXT_MSG_PER_ADDR) { + co_return td::Status::Error(PSTRING() << "too many external messages to address " << wc << ":" << addr.to_hex()); + } + if (add_to_mempool) { + add_message_to_mempool(message, priority, msg_seqno); + } + co_return result.move_as_ok(); +} + +std::vector, int>> ExtMessagePool::get_external_messages_for_collator(ShardIdFull shard) { + td::Timer t; + size_t processed = 0, deleted = 0; + std::vector, int>> res; + MessageId left{AccountIdPrefixFull{shard.workchain, shard.shard & (shard.shard - 1)}, Bits256::zero()}; + size_t total_msgs = 0; + td::Random::Fast rnd; + for (auto iter = ext_msgs_.rbegin(); iter != ext_msgs_.rend(); ++iter) { + std::vector, int>> cur_res; + int priority = iter->first; + auto &msgs = iter->second; + auto it = msgs.ext_messages_.lower_bound(left); + while (it != msgs.ext_messages_.end()) { + auto s = it->first; + if (!shard_contains(shard, s.dst)) { + break; + } + ++processed; + if (it->second->expired()) { + msgs.ext_addr_messages_[it->second->address()].erase(it->first.hash); + ext_messages_hashes_.erase(it->first.hash); + it = msgs.ext_messages_.erase(it); + ++deleted; + continue; + } + if (it->second->is_active()) { + cur_res.emplace_back(it->second->message, priority); + } + ++it; + } + td::random_shuffle(td::as_mutable_span(cur_res), rnd); + res.insert(res.end(), cur_res.begin(), cur_res.end()); + total_msgs += msgs.ext_messages_.size(); + } + + // Sort messages to each account by msg_seqno, if present + std::map, std::vector>> wallet_msg_idxs_; + for (size_t i = 0; i < res.size(); ++i) { + auto &[message, priority] = res[i]; + MessageId id{res[i].first->shard(), res[i].first->hash()}; + auto msg_seqno = ext_msgs_[priority].ext_messages_[id]->msg_seqno; + if (msg_seqno) { + wallet_msg_idxs_[{message->wc(), message->addr()}].emplace_back(msg_seqno.value(), i); + } + } + for (auto &[_, idxs] : wallet_msg_idxs_) { + auto sorted_idxs = idxs; + std::sort(sorted_idxs.begin(), sorted_idxs.end()); + std::vector, int>> new_res; + new_res.reserve(idxs.size()); + for (auto [_, i] : sorted_idxs) { + new_res.push_back(res[i]); + } + for (size_t j = 0; j < new_res.size(); ++j) { + res[idxs[j].second] = new_res[j]; + } + } + + if (!res.empty() || deleted > 0) { + LOG(WARNING) << "get_external_messages to shard " << shard.to_str() << " : time=" << t.elapsed() + << " result_size=" << res.size() << " processed=" << processed << " expired=" << deleted + << " total_size=" << total_msgs; + } + return res; +} + +void ExtMessagePool::cleanup_external_messages(ShardIdFull shard) { + get_external_messages_for_collator(shard); +} + +void ExtMessagePool::complete_external_messages(std::vector to_delay, + std::vector to_delete) { + for (auto &hash : to_delete) { + auto it = ext_messages_hashes_.find(hash); + if (it != ext_messages_hashes_.end()) { + int priority = it->second.first; + auto msg_id = it->second.second; + ext_msgs_[priority].erase(msg_id); + ext_messages_hashes_.erase(it); + } + } + for (auto &hash : to_delay) { + auto it = ext_messages_hashes_.find(hash); + if (it != ext_messages_hashes_.end()) { + int priority = it->second.first; + auto msg_id = it->second.second; + auto &msgs = ext_msgs_[priority]; + auto it2 = msgs.ext_messages_.find(msg_id); + if (msgs.ext_messages_.size() < SOFT_MEMPOOL_LIMIT && it2->second->can_postpone()) { + it2->second->postpone(); + } else { + msgs.erase(msg_id); + ext_messages_hashes_.erase(it); + } + } + } +} + +std::vector> ExtMessagePool::prepare_stats() { + std::vector> vec; + vec.emplace_back("total.ext_msg_check", + PSTRING() << "ok:" << total_check_ext_messages_ok_ << " error:" << total_check_ext_messages_error_); + return vec; +} + +void ExtMessagePool::alarm() { + if (cleanup_mempool_at_.is_in_past()) { + cleanup_external_messages(ShardIdFull{masterchainId, shardIdAll}); + cleanup_external_messages(ShardIdFull{basechainId, shardIdAll}); + cleanup_mempool_at_ = td::Timestamp::in(250.0); + } + alarm_timestamp().relax(cleanup_mempool_at_); +} + +void ExtMessagePool::add_message_to_mempool(td::Ref message, int priority, + td::optional msg_seqno) { + WorkchainId wc = message->wc(); + StdSmcAddress addr = message->addr(); + auto &msgs = ext_msgs_[priority]; + if (msgs.ext_messages_.size() > opts_->max_mempool_num()) { + LOG(INFO) << "cannot add message addr=" << wc << ":" << addr.to_hex() << " prio=" << priority + << " to mempool: mempool is full (limit=" << opts_->max_mempool_num() << ")"; + return; + } + auto msg = std::make_unique(message); + msg->msg_seqno = msg_seqno; + MessageId id{message->shard(), message->hash()}; + auto address = msg->address(); + auto it = msgs.ext_addr_messages_.find(address); + if (it != msgs.ext_addr_messages_.end() && it->second.size() >= PER_ADDRESS_LIMIT) { + LOG(INFO) << "cannot add message addr=" << wc << ":" << addr.to_hex() << " prio=" << priority + << " to mempool: per address limit reached (limit=" << PER_ADDRESS_LIMIT << ")"; + return; + } + auto it2 = ext_messages_hashes_.find(id.hash); + if (it2 != ext_messages_hashes_.end()) { + int old_priority = it2->second.first; + if (old_priority >= priority) { + LOG(INFO) << "cannot add message addr=" << wc << ":" << addr.to_hex() << " prio=" << priority + << " to mempool: already exists"; + return; + } + ext_msgs_[old_priority].erase(id); + } + msgs.ext_messages_.emplace(id, std::move(msg)); + msgs.ext_addr_messages_[address].emplace(id.hash, id); + ext_messages_hashes_[id.hash] = {priority, id}; + LOG(INFO) << "adding message addr=" << wc << ":" << addr.to_hex() << " prio=" << priority << " to mempool"; +} + +td::actor::Task ExtMessagePool::check_message(td::Ref message, + td::optional &msg_seqno) { + WorkchainId wc = message->wc(); + StdSmcAddress addr = message->addr(); + auto [shard_acc, utime, lt, config] = co_await run_fetch_account_state(wc, addr, manager_); + bool special = wc == masterchainId && config->is_special_smartcontract(addr); + block::Account acc; + if (!acc.unpack(shard_acc, utime, special)) { + co_return td::Status::Error(PSLICE() << "Failed to unpack account state"); + } + acc.block_lt = lt; + + auto [wait_allow_broadcast, allow_broadcast_promise] = td::actor::StartedTask<>::make_bridge(); + CheckResult check_result{.message = message, .wait_allow_broadcast = std::move(wait_allow_broadcast)}; + + const WalletMessageProcessor *wallet = + acc.code.not_null() ? WalletMessageProcessor::get(acc.code->get_hash().bits()) : nullptr; + if (wallet != nullptr) { + msg_seqno = co_await check_message_to_wallet(message, wallet, std::move(acc), utime, lt, std::move(config), + std::move(allow_broadcast_promise)); + co_return check_result; + } + wallets_.erase({wc, addr}); + co_await ExtMessageQ::run_message_on_account(wc, &acc, utime, lt + 1, message->root_cell(), std::move(config)); + allow_broadcast_promise.set_value(td::Unit{}); + co_return check_result; +} + +td::Result ExtMessagePool::check_message_to_wallet(td::Ref message, + const WalletMessageProcessor *wallet, block::Account acc, + UnixTime utime, LogicalTime lt, + std::unique_ptr config, + td::Promise allow_broadcast_promise) { + WorkchainId wc = message->wc(); + StdSmcAddress addr = message->addr(); + LOG(DEBUG) << "Checking external message to " << wc << ":" << addr.to_hex() << ", " << wallet->name(); + TRY_RESULT(wallet_seqno, wallet->get_wallet_seqno(acc.data)); + auto &wallet_info = wallets_[{wc, addr}]; + SCOPE_EXIT { + if (wallet_info.messages.empty()) { + wallets_.erase({wc, addr}); + } + }; + wallet_info.process_messages(wallet_seqno, utime); + TRY_RESULT(parsed_message, wallet->parse_message(message->root_cell())); + auto [msg_seqno, msg_valid_until] = parsed_message; + LOG(DEBUG) << "External message to " << wallet->name() << ": msg_seqno=" << msg_seqno + << ", msg_ttl=" << msg_valid_until << ", wallet_seqno=" << wallet_seqno; + if (msg_valid_until <= (UnixTime)td::Clocks::system()) { + return td::Status::Error("valid_until is in the past"); + } + if (msg_seqno < wallet_seqno) { + return td::Status::Error(PSTRING() << "Too old seqno: msg_seqno=" << msg_seqno + << ", wallet_seqno=" << wallet_seqno); + } + if (msg_seqno - wallet_seqno > MAX_WALLET_SEQNO_DIFF) { + return td::Status::Error(PSTRING() << "Too new seqno: msg_seqno=" << msg_seqno + << ", wallet_seqno=" << wallet_seqno); + } + if (wallet_info.messages.contains(msg_seqno)) { + return td::Status::Error(PSTRING() << "Duplicate msg_seqno " << msg_seqno); + } + TRY_RESULT_ASSIGN(acc.data, wallet->set_wallet_seqno(acc.data, msg_seqno)); + acc.storage_dict_hash = acc.orig_storage_dict_hash = {}; + TRY_STATUS(ExtMessageQ::run_message_on_account(wc, &acc, utime, lt + 1, message->root_cell(), std::move(config))); + wallet_info.messages[msg_seqno] = + WalletMessageInfo{.valid_until = msg_valid_until, .allow_broadcast_promise = std::move(allow_broadcast_promise)}; + wallet_info.process_messages(wallet_seqno, utime); + LOG(DEBUG) << "Checked external message to " << wc << ":" << addr.to_hex() << ", " << wallet->name(); + return msg_seqno; +} + +void ExtMessagePool::WalletInfo::process_messages(td::uint32 wallet_seqno, UnixTime utime) { + for (auto it = messages.begin(); it != messages.end();) { + auto &[seqno, message] = *it; + if (seqno < wallet_seqno) { + if (message.allow_broadcast_promise) { + message.allow_broadcast_promise.set_error( + td::Status::Error(PSTRING() << "Too old seqno: msg_seqno=" << seqno << ", wallet_seqno=" << wallet_seqno)); + } + it = messages.erase(it); + continue; + } + if (message.valid_until <= utime) { + if (message.allow_broadcast_promise) { + message.allow_broadcast_promise.set_error(td::Status::Error("valid_until is in the past")); + } + it = messages.erase(it); + continue; + } + ++it; + } + for (td::uint32 seqno = wallet_seqno;; ++seqno) { + auto it = messages.find(seqno); + if (it == messages.end()) { + break; + } + if (it->second.allow_broadcast_promise) { + it->second.allow_broadcast_promise.set_value(td::Unit{}); + } + } +} + +size_t ExtMessagePool::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) { + before_query(); + auto it1 = counter_cur_.find({wc, addr}); + auto it2 = counter_prev_.find({wc, addr}); + return (it1 == counter_cur_.end() ? 0 : it1->second) + (it2 == counter_prev_.end() ? 0 : it2->second); +} + +size_t ExtMessagePool::CheckedExtMsgCounter::inc_msg_count(WorkchainId wc, StdSmcAddress addr) { + before_query(); + auto it2 = counter_prev_.find({wc, addr}); + return (it2 == counter_prev_.end() ? 0 : it2->second) + ++counter_cur_[{wc, addr}]; +} + +void ExtMessagePool::CheckedExtMsgCounter::before_query() { + while (cleanup_at_.is_in_past()) { + counter_prev_ = std::move(counter_cur_); + counter_cur_.clear(); + if (counter_prev_.empty()) { + cleanup_at_ = td::Timestamp::in(MAX_EXT_MSG_PER_ADDR_TIME_WINDOW / 2.0); + break; + } + cleanup_at_ += MAX_EXT_MSG_PER_ADDR_TIME_WINDOW / 2.0; + } +} + +} // namespace ton::validator diff --git a/validator/impl/ext-message-pool.hpp b/validator/impl/ext-message-pool.hpp new file mode 100644 index 0000000000..cbef720712 --- /dev/null +++ b/validator/impl/ext-message-pool.hpp @@ -0,0 +1,165 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "interfaces/validator-manager.h" +#include "td/actor/coro_utils.h" + +#include "external-message.hpp" + +namespace ton::validator { + +class ExtMessagePool : public td::actor::Actor { + public: + ExtMessagePool(td::Ref opts, td::actor::ActorId manager) + : opts_(opts), manager_(manager) { + } + + struct CheckResult { + td::Ref message; + td::actor::StartedTask<> wait_allow_broadcast; + }; + td::actor::Task check_add_external_message(td::BufferSlice data, int priority, bool add_to_mempool); + std::vector, int>> get_external_messages_for_collator(ShardIdFull shard); + void cleanup_external_messages(ShardIdFull shard); + void complete_external_messages(std::vector to_delay, std::vector to_delete); + + void update_last_masterchain_state(td::Ref state) { + last_masterchain_state_ = std::move(state); + } + void update_options(td::Ref opts) { + opts_ = std::move(opts); + } + std::vector> prepare_stats(); + + void alarm() override; + + private: + struct MessageId { + AccountIdPrefixFull dst; + ExtMessage::Hash hash; + + bool operator<(const MessageId &msg) const { + if (dst < msg.dst) { + return true; + } + if (msg.dst < dst) { + return false; + } + return hash < msg.hash; + } + }; + struct MempoolMsg { + td::Ref message; + td::uint32 generation = 0; + bool active = true; + td::Timestamp reactivate_at; + td::Timestamp delete_at; + td::optional msg_seqno; + + auto address() const { + return std::make_pair(message->wc(), message->addr()); + } + bool is_active() { + if (!active) { + if (reactivate_at.is_in_past()) { + active = true; + generation++; + } + } + return active; + } + bool can_postpone() const { + return generation <= 2; + } + void postpone() { + if (!active) { + return; + } + active = false; + reactivate_at = td::Timestamp::in(generation * 5.0); + } + bool expired() const { + return delete_at.is_in_past(); + } + explicit MempoolMsg(td::Ref msg) : message(std::move(msg)) { + delete_at = td::Timestamp::in(600); + } + }; + + td::Ref opts_; + td::actor::ActorId manager_; + td::Ref last_masterchain_state_; + + struct ExtMessages { + std::map> ext_messages_; + std::map, std::map> ext_addr_messages_; + void erase(const MessageId &id) { + auto it = ext_messages_.find(id); + CHECK(it != ext_messages_.end()); + ext_addr_messages_[it->second->address()].erase(id.hash); + ext_messages_.erase(it); + } + }; + std::map ext_msgs_; // priority -> messages + std::map> ext_messages_hashes_; // hash -> priority + + struct CheckedExtMsgCounter { + std::map, size_t> counter_cur_, counter_prev_; + td::Timestamp cleanup_at_ = td::Timestamp::now(); + + size_t get_msg_count(WorkchainId wc, StdSmcAddress addr); + size_t inc_msg_count(WorkchainId wc, StdSmcAddress addr); + void before_query(); + } checked_ext_msg_counter_; + td::uint64 total_check_ext_messages_ok_{0}, total_check_ext_messages_error_{0}; + + td::Timestamp cleanup_mempool_at_ = td::Timestamp::now(); + + void add_message_to_mempool(td::Ref message, int priority, td::optional msg_seqno); + + struct WalletMessageInfo { + td::uint32 valid_until; + td::Promise allow_broadcast_promise; + }; + struct WalletInfo { + std::map messages; + ~WalletInfo() { + for (auto &[_, message] : messages) { + if (message.allow_broadcast_promise) { + message.allow_broadcast_promise.set_error(td::Status::Error("wallet is no longer valid")); + } + } + } + void process_messages(td::uint32 wallet_seqno, UnixTime utime); + }; + std::map, WalletInfo> wallets_; + + td::actor::Task check_message(td::Ref message, td::optional &msg_seqno); + td::Result check_message_to_wallet(td::Ref message, const WalletMessageProcessor *wallet, + block::Account acc, UnixTime utime, LogicalTime lt, + std::unique_ptr config, + td::Promise allow_broadcast_promise); + + static constexpr double MAX_EXT_MSG_PER_ADDR_TIME_WINDOW = 10.0; + static constexpr size_t MAX_EXT_MSG_PER_ADDR = 3 * 10; + static constexpr size_t PER_ADDRESS_LIMIT = 256; + static constexpr size_t SOFT_MEMPOOL_LIMIT = 1024; + static constexpr td::uint32 MAX_WALLET_SEQNO_DIFF = 16; +}; + +} // namespace ton::validator diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index 0ab839b443..c5678cc7f0 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -27,7 +27,6 @@ #include "collator-impl.h" #include "external-message.hpp" -#include "fabric.h" namespace ton { @@ -88,45 +87,6 @@ td::Result> ExtMessageQ::create_ext_message(td::BufferSlice dat return Ref{true, std::move(data), std::move(ext_msg), dest_prefix, wc, addr}; } -void ExtMessageQ::run_message(td::Ref message, td::actor::ActorId manager, - td::Promise> promise) { - auto root = message->root_cell(); - block::gen::CommonMsgInfo::Record_ext_in_msg_info info; - tlb::unpack_cell_inexact(root, info); // checked in create message - ton::StdSmcAddress addr = message->addr(); - ton::WorkchainId wc = message->wc(); - - run_fetch_account_state( - wc, addr, manager, - [promise = std::move(promise), msg_root = root, wc, addr, message]( - td::Result, UnixTime, LogicalTime, std::unique_ptr>> - res) mutable { - if (res.is_error()) { - promise.set_error(td::Status::Error(PSLICE() << "Failed to get account state")); - } else { - auto tuple = res.move_as_ok(); - block::Account acc; - auto shard_acc = std::move(std::get<0>(tuple)); - auto utime = std::get<1>(tuple); - auto lt = std::get<2>(tuple); - auto config = std::move(std::get<3>(tuple)); - bool special = wc == masterchainId && config->is_special_smartcontract(addr); - if (!acc.unpack(shard_acc, utime, special)) { - promise.set_error(td::Status::Error(PSLICE() << "Failed to unpack account state")); - } else { - acc.block_lt = lt; - auto status = run_message_on_account(wc, &acc, utime, lt + 1, msg_root, std::move(config)); - if (status.is_ok()) { - promise.set_value(std::move(message)); - } else { - promise.set_error(td::Status::Error(PSLICE() << "External message was not accepted\n" - << status.message())); - } - } - } - }); -} - td::Status ExtMessageQ::run_message_on_account(ton::WorkchainId wc, block::Account* acc, UnixTime utime, LogicalTime lt, td::Ref msg_root, std::unique_ptr config) { Ref old_mparams; @@ -144,7 +104,7 @@ td::Status ExtMessageQ::run_message_on_account(ton::WorkchainId wc, block::Accou if (fetch_res.is_error()) { auto error = fetch_res.move_as_error(); LOG(DEBUG) << "Cannot fetch config params: " << error.message(); - return error.move_as_error_prefix("Cannot fetch config params: "); + return error.move_as_error_prefix("External message was not accepted: cannot fetch config params: "); } compute_phase_cfg_.libraries = std::make_unique(config->get_libraries_root(), 256); compute_phase_cfg_.with_vm_log = true; @@ -156,17 +116,227 @@ td::Status ExtMessageQ::run_message_on_account(ton::WorkchainId wc, block::Accou if (res.is_error()) { auto error = res.move_as_error(); LOG(DEBUG) << "Cannot run message on account: " << error.message(); - return error.move_as_error_prefix("Cannot run message on account: "); + return error.move_as_error_prefix("External message was not accepted: cannot run message on account: "); } std::unique_ptr trans = res.move_as_ok(); auto trans_root = trans->commit(*acc); if (trans_root.is_null()) { LOG(DEBUG) << "Cannot commit new transaction for smart contract"; - return td::Status::Error("Cannot commit new transaction for smart contract"); + return td::Status::Error("External message was not accepted: cannot commit new transaction for smart contract"); } return td::Status::OK(); } +namespace { +class WalletMessageProcessorImpl : public WalletMessageProcessor { + td::Result> parse_message(td::Ref msg_root) const override { + if (msg_root.is_null()) { + return td::Status::Error("msg is null"); + } + vm::CellSlice cs{vm::NoVmOrd{}, msg_root}; + block::gen::CommonMsgInfo::Record_ext_in_msg_info info; + block::gen::EitherStateInit::Record init; + if (!tlb::unpack(cs, info) || !tlb::unpack(cs, init) || cs.size() < 1) { + return td::Status::Error("cannot unpack external message"); + } + vm::CellSlice body; + if (cs.fetch_ulong(1) == 0) { + body = cs; + } else { + td::Ref ref = cs.prefetch_ref(); + if (ref.is_null()) { + return td::Status::Error("cannot unpack external message"); + } + body = vm::CellSlice{vm::NoVmOrd{}, std::move(ref)}; + } + return parse_message_body(std::move(body)); + } + + virtual td::Result> parse_message_body(vm::CellSlice body) const = 0; + + td::Result get_wallet_seqno(td::Ref data_root) const override { + if (data_root.is_null()) { + return td::Status::Error("data is null"); + } + vm::CellSlice cs{vm::NoVmOrd{}, data_root}; + if (cs.size() < 32) { + return td::Status::Error("invalid data"); + } + return (td::uint32)cs.prefetch_ulong(32); + } + + td::Result> set_wallet_seqno(td::Ref data_root, td::uint32 new_seqno) const override { + if (data_root.is_null()) { + return td::Status::Error("data is null"); + } + vm::CellSlice cs{vm::NoVmOrd{}, data_root}; + if (cs.size() < 32) { + return td::Status::Error("invalid data"); + } + cs.skip_first(32); + vm::CellBuilder cb; + cb.store_long(new_seqno, 32); + cb.append_cellslice(cs); + return cb.finalize_novm(); + } +}; + +class WalletV1 : public WalletMessageProcessorImpl { + public: + std::string name() const override { + return "wallet-v1"; + } + + td::Result> parse_message_body(vm::CellSlice body) const override { + // signature, msg_seqno + if (body.size() < 512 + 32) { + return td::Status::Error("invalid message body"); + } + body.skip_first(512); + auto msg_seqno = (td::uint32)body.fetch_ulong(32); + return std::make_pair(msg_seqno, std::numeric_limits::max()); + } +}; + +class WalletV2 : public WalletMessageProcessorImpl { + public: + std::string name() const override { + return "wallet-v2"; + } + + td::Result> parse_message_body(vm::CellSlice body) const override { + // signature, msg_seqno, valid_until + if (body.size() < 512 + 32 + 32) { + return td::Status::Error("invalid message body"); + } + body.skip_first(512); + auto msg_seqno = (td::uint32)body.fetch_ulong(32); + auto valid_until = (UnixTime)body.fetch_ulong(32); + return std::make_pair(msg_seqno, valid_until); + } +}; + +class WalletV3 : public WalletMessageProcessorImpl { + public: + std::string name() const override { + return "wallet-v3"; + } + + td::Result> parse_message_body(vm::CellSlice body) const override { + // signature, subwallet_id, valid_until, msg_seqno + if (body.size() < 512 + 32 + 32 + 32) { + return td::Status::Error("invalid message body"); + } + body.skip_first(512 + 32); + auto valid_until = (UnixTime)body.fetch_ulong(32); + auto msg_seqno = (td::uint32)body.fetch_ulong(32); + return std::make_pair(msg_seqno, valid_until); + } +}; + +class WalletV4 : public WalletMessageProcessorImpl { + public: + std::string name() const override { + return "wallet-v4"; + } + + td::Result> parse_message_body(vm::CellSlice body) const override { + // signature, subwallet_id, valid_until, msg_seqno + if (body.size() < 512 + 32 + 32 + 32) { + return td::Status::Error("invalid message body"); + } + body.skip_first(512 + 32); + auto valid_until = (UnixTime)body.fetch_ulong(32); + auto msg_seqno = (td::uint32)body.fetch_ulong(32); + return std::make_pair(msg_seqno, valid_until); + } +}; + +class WalletV5 : public WalletMessageProcessorImpl { + public: + std::string name() const override { + return "wallet-v5"; + } + + td::Result> parse_message_body(vm::CellSlice body) const override { + // tag, subwallet_id, valid_until, msg_seqno + if (body.size() < 32 + 32 + 32 + 32) { + return td::Status::Error("invalid message body"); + } + body.skip_first(32 + 32); + auto valid_until = (UnixTime)body.fetch_ulong(32); + auto msg_seqno = (td::uint32)body.fetch_ulong(32); + return std::make_pair(msg_seqno, valid_until); + } + + td::Result get_wallet_seqno(td::Ref data_root) const override { + if (data_root.is_null()) { + return td::Status::Error("data is null"); + } + vm::CellSlice cs{vm::NoVmOrd{}, data_root}; + if (cs.size() < 33) { + return td::Status::Error("invalid data"); + } + cs.skip_first(1); + return (td::uint32)cs.prefetch_ulong(32); + } + + td::Result> set_wallet_seqno(td::Ref data_root, td::uint32 new_seqno) const override { + if (data_root.is_null()) { + return td::Status::Error("data is null"); + } + vm::CellSlice cs{vm::NoVmOrd{}, data_root}; + if (cs.size() < 33) { + return td::Status::Error("invalid data"); + } + bool flag = cs.fetch_long(1); + cs.skip_first(32); + vm::CellBuilder cb; + cb.store_long(flag, 1); + cb.store_long(new_seqno, 32); + cb.append_cellslice(cs); + return cb.finalize_novm(); + } +}; + +} // namespace + +const WalletMessageProcessor* WalletMessageProcessor::get(td::Bits256 code_hash) { + static auto wallets = []() -> std::map> { + std::map> wallets; + auto add_wallet = [&](td::Slice s, std::shared_ptr wallet) { + td::Bits256 code_hash; + CHECK(code_hash.from_hex(s) == 256); + wallets[code_hash] = wallet; + // Make library cell + vm::CellBuilder cb; + cb.store_long((int)vm::Cell::SpecialType::Library, 8); + cb.store_bytes(code_hash.as_slice()); + td::Bits256 library_code_hash = cb.finalize_novm(true)->get_hash().bits(); + wallets[library_code_hash] = wallet; + }; + + add_wallet("A0CFC2C48AEE16A271F2CFC0B7382D81756CECB1017D077FAAAB3BB602F6868C", std::make_shared()); + add_wallet("D4902FCC9FAD74698FA8E353220A68DA0DCF72E32BCB2EB9EE04217C17D3062C", std::make_shared()); + add_wallet("587CC789EFF1C84F46EC3797E45FC809A14FF5AE24F1E0C7A6A99CC9DC9061FF", std::make_shared()); + + add_wallet("5C9A5E68C108E18721A07C42F9956BFB39AD77EC6D624B60C576EC88EEE65329", std::make_shared()); + add_wallet("FE9530D3243853083EF2EF0B4C2908C0ABF6FA1C31EA243AACAA5BF8C7D753F1", std::make_shared()); + + add_wallet("B61041A58A7980B946E8FB9E198E3C904D24799FFA36574EA4251C41A566F581", std::make_shared()); + add_wallet("84DAFA449F98A6987789BA232358072BC0F76DC4524002A5D0918B9A75D2D599", std::make_shared()); + + add_wallet("64DD54805522C5BE8A9DB59CEA0105CCF0D08786CA79BEB8CB79E880A8D7322D", std::make_shared()); + add_wallet("FEB5FF6820E2FF0D9483E7E0D62C817D846789FB4AE580C878866D959DABD5C0", std::make_shared()); + + add_wallet("20834B7B72B112147E1B2FB457B84E74D1A30F04F737D4F62A668E9552D2B72F", std::make_shared()); + + return wallets; + }(); + auto it = wallets.find(code_hash); + return it == wallets.end() ? nullptr : it->second.get(); +} + } // namespace validator } // namespace ton diff --git a/validator/impl/external-message.hpp b/validator/impl/external-message.hpp index fee2cef977..15e77585b3 100644 --- a/validator/impl/external-message.hpp +++ b/validator/impl/external-message.hpp @@ -61,12 +61,21 @@ class ExtMessageQ : public ExtMessage { ton::StdSmcAddress addr); static td::Result> create_ext_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits); - static void run_message(td::Ref message, td::actor::ActorId manager, - td::Promise> promise); static td::Status run_message_on_account(ton::WorkchainId wc, block::Account* acc, UnixTime utime, LogicalTime lt, td::Ref msg_root, std::unique_ptr config); }; +class WalletMessageProcessor { + public: + virtual ~WalletMessageProcessor() = default; + virtual std::string name() const = 0; + virtual td::Result> parse_message(td::Ref msg_root) const = 0; + virtual td::Result get_wallet_seqno(td::Ref data_root) const = 0; + virtual td::Result> set_wallet_seqno(td::Ref data_root, td::uint32 new_seqno) const = 0; + + static const WalletMessageProcessor* get(td::Bits256 code_hash); +}; + } // namespace validator } // namespace ton diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 3f98cce258..c717be69f8 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -119,11 +119,6 @@ td::Result> create_ext_message(td::BufferSlice data, block:: return std::move(res); } -void run_check_external_message(Ref message, td::actor::ActorId manager, - td::Promise> promise) { - ExtMessageQ::run_message(std::move(message), std::move(manager), std::move(promise)); -} - td::Result> create_ihr_message(td::BufferSlice data) { TRY_RESULT(res, IhrMessageQ::create_ihr_message(std::move(data))); return std::move(res); @@ -237,6 +232,14 @@ void run_fetch_account_state( LiteQuery::fetch_account_state(wc, addr, std::move(manager), std::move(promise)); } +td::actor::Task, UnixTime, LogicalTime, std::unique_ptr>> +run_fetch_account_state(WorkchainId wc, StdSmcAddress addr, td::actor::ActorId manager) { + auto [task, promise] = td::actor::StartedTask< + std::tuple, UnixTime, LogicalTime, std::unique_ptr>>::make_bridge(); + run_fetch_account_state(wc, addr, std::move(manager), std::move(promise)); + co_return co_await std::move(task); +} + void run_validate_shard_block_description(td::BufferSlice data, BlockHandle masterchain_block, td::Ref masterchain_state, td::actor::ActorId manager, td::Timestamp timeout, diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 8850f2a3b5..edf94f6c51 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -547,23 +547,20 @@ void LiteQuery::continue_getZeroState(BlockIdExt blkid, td::BufferSlice state) { void LiteQuery::perform_sendMessage(td::BufferSlice data) { LOG(INFO) << "started a sendMessage(<" << data.size() << " bytes>) liteserver query"; - auto copy = data.clone(); - td::actor::send_closure_later( - manager_, &ValidatorManager::check_external_message, std::move(copy), - [Self = actor_id(this), data = std::move(data), manager = manager_, cache = cache_, - cache_key = cache_key_](td::Result> res) mutable { - if (res.is_error()) { - // Don't cache errors - td::actor::send_closure(cache, &LiteServerCache::drop_send_message_from_cache, cache_key); - td::actor::send_closure(Self, &LiteQuery::abort_query, - res.move_as_error_prefix("cannot apply external message to current state : "s)); - } else { - LOG(INFO) << "sending an external message to validator manager"; - td::actor::send_closure_later(manager, &ValidatorManager::send_external_message, res.move_as_ok()); - auto b = ton::create_serialize_tl_object(1); - td::actor::send_closure(Self, &LiteQuery::finish_query, std::move(b), false); - } - }); + td::actor::send_closure( + manager_, &ValidatorManager::new_external_message_query, std::move(data), + td::PromiseCreator::lambda( + [Self = actor_id(this), cache = cache_, cache_key = cache_key_](td::Result res) mutable { + if (res.is_error()) { + // Don't cache errors + td::actor::send_closure(cache, &LiteServerCache::drop_send_message_from_cache, cache_key); + td::actor::send_closure(Self, &LiteQuery::abort_query, + res.move_as_error_prefix("cannot apply external message to current state : "s)); + } else { + auto b = ton::create_serialize_tl_object(1); + td::actor::send_closure(Self, &LiteQuery::finish_query, std::move(b), false); + } + })); } void LiteQuery::get_block_handle_checked(BlockIdExt blkid, td::Promise promise) { diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 34e1210e7b..2c270457ff 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -324,7 +324,6 @@ class ValidatorManager : public ValidatorManagerInterface { td::Promise promise) = 0; virtual void send_get_next_key_blocks_request(BlockIdExt block_id, td::uint32 priority, td::Promise> promise) = 0; - virtual void send_external_message(td::Ref message) = 0; virtual void send_ihr_message(td::Ref message) = 0; virtual void send_top_shard_block_description(td::Ref desc) = 0; virtual void send_block_broadcast(BlockBroadcast broadcast, int mode) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 0bba54df6e..dd8a8dcdd6 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -269,14 +269,13 @@ void ValidatorManagerImpl::get_key_block_proof_link(BlockIdExt block_id, td::Pro td::actor::send_closure(db_, &Db::get_key_block_proof, block_id, std::move(P)); } -void ValidatorManagerImpl::new_external_message(td::BufferSlice data, int priority) { +td::actor::Task<> ValidatorManagerImpl::new_external_message_broadcast(td::BufferSlice data, int priority) { if (last_masterchain_state_.is_null()) { - return; - } - auto R = create_ext_message(std::move(data), last_masterchain_state_->get_ext_msg_limits()); - if (R.is_ok()) { - ext_messages_.emplace_back(R.move_as_ok()); + co_return td::Status::Error(ErrorCode::notready, "not ready"); } + auto msg = co_await create_ext_message(std::move(data), last_masterchain_state_->get_ext_msg_limits()); + ext_messages_.emplace_back(std::move(msg)); + co_return td::Unit{}; } void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 0bef96ee9f..43524d62d8 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -130,10 +130,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_key_block_proof_link(BlockIdExt block_id, td::Promise promise) override; //void get_block_description(BlockIdExt block_id, td::Promise promise) override; - void new_external_message(td::BufferSlice data, int priority) override; - void check_external_message(td::BufferSlice data, td::Promise> promise) override { - UNREACHABLE(); - } + td::actor::Task<> new_external_message_broadcast(td::BufferSlice data, int priority) override; void new_ihr_message(td::BufferSlice data) override; void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; @@ -267,9 +264,6 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { UNREACHABLE(); } - void send_external_message(td::Ref message) override { - new_external_message(message->serialize(), 0); - } void send_ihr_message(td::Ref message) override { new_ihr_message(message->serialize()); } diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index 950e595238..68657379e8 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -152,11 +152,10 @@ void ValidatorManagerImpl::get_key_block_proof_link(BlockIdExt block_id, td::Pro td::actor::send_closure(db_, &Db::get_key_block_proof_link, block_id, std::move(P)); } -void ValidatorManagerImpl::new_external_message(td::BufferSlice data, int priority) { - auto R = create_ext_message(std::move(data), block::SizeLimitsConfig::ExtMsgLimits()); - if (R.is_ok()) { - ext_messages_.emplace_back(R.move_as_ok()); - } +td::actor::Task<> ValidatorManagerImpl::new_external_message_broadcast(td::BufferSlice data, int priority) { + auto msg = co_await create_ext_message(std::move(data), block::SizeLimitsConfig::ExtMsgLimits()); + ext_messages_.emplace_back(std::move(msg)); + co_return td::Unit{}; } void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 77cef2a1df..5a5a4a8b1b 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -151,10 +151,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_key_block_proof(BlockIdExt block_id, td::Promise promise) override; void get_key_block_proof_link(BlockIdExt block_id, td::Promise promise) override; - void new_external_message(td::BufferSlice data, int priority) override; - void check_external_message(td::BufferSlice data, td::Promise> promise) override { - UNREACHABLE(); - } + td::actor::Task<> new_external_message_broadcast(td::BufferSlice data, int priority) override; void new_ihr_message(td::BufferSlice data) override; void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { @@ -342,9 +339,6 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { UNREACHABLE(); } - void send_external_message(td::Ref message) override { - new_external_message(message->serialize(), 0); - } void send_ihr_message(td::Ref message) override { new_ihr_message(message->serialize()); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 101c871c7b..0cd65a05e0 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -396,104 +396,50 @@ void ValidatorManagerImpl::get_key_block_proof_link(BlockIdExt block_id, td::Pro td::actor::send_closure(db_, &Db::get_key_block_proof, block_id, std::move(P)); } -void ValidatorManagerImpl::new_external_message(td::BufferSlice data, int priority) { - if (!validating_masterchain() && collator_nodes_.empty() && - (!is_validator() || !opts_->get_collators_list()->self_collate)) { - return; - } - if (last_masterchain_state_.is_null()) { - VLOG(VALIDATOR_NOTICE) << "dropping ext message: validator is not ready"; - return; - } - if (ext_msgs_[priority].ext_messages_.size() > (size_t)max_mempool_num()) { - return; - } - auto R = create_ext_message(std::move(data), last_masterchain_state_->get_ext_msg_limits()); - if (R.is_error()) { - VLOG(VALIDATOR_NOTICE) << "dropping bad ext message: " << R.move_as_error(); - return; - } - add_external_message(R.move_as_ok(), priority); -} - -void ValidatorManagerImpl::add_external_message(td::Ref msg, int priority) { - auto &msgs = ext_msgs_[priority]; - auto message = std::make_unique>(msg); - auto id = message->ext_id(); - auto address = message->address(); - unsigned long per_address_limit = 256; - auto it = msgs.ext_addr_messages_.find(address); - if (it != msgs.ext_addr_messages_.end() && it->second.size() >= per_address_limit) { - return; - } - auto it2 = ext_messages_hashes_.find(id.hash); - if (it2 != ext_messages_hashes_.end()) { - int old_priority = it2->second.first; - if (old_priority >= priority) { - return; - } - ext_msgs_[old_priority].erase(id); - } - msgs.ext_messages_.emplace(id, std::move(message)); - msgs.ext_addr_messages_[address].emplace(id.hash, id); - ext_messages_hashes_[id.hash] = {priority, id}; -} -void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Promise> promise) { - if (!started_) { - promise.set_error(td::Status::Error(ErrorCode::notready, "node not synced")); - return; - } - auto state = do_get_last_liteserver_state(); - if (state.is_null()) { - promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); - return; - } - auto R = create_ext_message(std::move(data), state->get_ext_msg_limits()); - if (R.is_error()) { - promise.set_error(R.move_as_error_prefix("failed to parse external message: ")); - return; - } - auto message = R.move_as_ok(); - WorkchainId wc = message->wc(); - StdSmcAddress addr = message->addr(); - if (checked_ext_msg_counter_.get_msg_count(wc, addr) >= max_ext_msg_per_addr()) { - promise.set_error( - td::Status::Error(PSTRING() << "too many external messages to address " << wc << ":" << addr.to_hex())); - return; +td::actor::Task<> ValidatorManagerImpl::new_external_message_broadcast(td::BufferSlice data, int priority) { + auto r_check_result = + co_await td::actor::ask(ext_message_pool_, &ExtMessagePool::check_add_external_message, std::move(data), priority, + /* add_to_mempool = */ is_validator() || !collator_nodes_.empty()) + .wrap(); + if (r_check_result.is_error()) { + VLOG(VALIDATOR_DEBUG) << "Dropping external message broadcast (prio=" << priority + << ") : " << r_check_result.error(); + co_return r_check_result.move_as_error(); + } + auto check_result = r_check_result.move_as_ok(); + auto allow_broadcast = co_await std::move(check_result.wait_allow_broadcast).wrap(); + if (allow_broadcast.is_error()) { + VLOG(VALIDATOR_DEBUG) << "Dropping external message broadcast (prio=" << priority + << ") : " << allow_broadcast.error(); + co_return allow_broadcast.move_as_error(); + } + VLOG(VALIDATOR_DEBUG) << "Checked external message broadcast to " << check_result.message->wc() << ":" + << check_result.message->addr().to_hex() << " (prio=" << priority << ")"; + co_return td::Unit{}; +} + +td::actor::Task<> ValidatorManagerImpl::new_external_message_query(td::BufferSlice data) { + auto [message, wait_allow_broadcast] = + co_await td::actor::ask(ext_message_pool_, &ExtMessagePool::check_add_external_message, std::move(data), 0, + /* add_to_mempool = */ is_validator() || !collator_nodes_.empty()); + new_external_message_query_cont(std::move(message), std::move(wait_allow_broadcast)).start().detach(); + co_return td::Unit{}; +} + +td::actor::Task<> ValidatorManagerImpl::new_external_message_query_cont(td::Ref message, + td::actor::StartedTask<> wait_allow_broadcast) { + auto result = co_await std::move(wait_allow_broadcast).wrap(); + if (result.is_error()) { + LOG(INFO) << "Cannot send external message to " << message->wc() << ":" << message->addr().to_hex() << " : " + << result.error(); + } else { + LOG(INFO) << "Sending external message to " << message->wc() << ":" << message->addr().to_hex(); + callback_->send_ext_message(message->shard(), message->serialize()); } - - promise = [self = this, wc, addr, promise = std::move(promise), - SelfId = actor_id(this)](td::Result> R) mutable { - td::actor::send_lambda(SelfId, [=, promise = std::move(promise), R = std::move(R)]() mutable { - ++(R.is_ok() ? self->total_check_ext_messages_ok_ : self->total_check_ext_messages_error_); - TRY_RESULT_PROMISE(promise, message, std::move(R)); - if (self->checked_ext_msg_counter_.inc_msg_count(wc, addr) > max_ext_msg_per_addr()) { - promise.set_error( - td::Status::Error(PSTRING() << "too many external messages to address " << wc << ":" << addr.to_hex())); - return; - } - promise.set_result(std::move(message)); - }); - }; - ++ls_stats_check_ext_messages_; - run_check_external_message(std::move(message), actor_id(this), std::move(promise)); + co_return td::Unit{}; } void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { - if (collator_nodes_.empty() && (!is_validator() || !opts_->get_collators_list()->self_collate)) { - return; - } - auto R = create_ihr_message(std::move(data)); - if (R.is_error()) { - VLOG(VALIDATOR_NOTICE) << "dropping bad ihr message: " << R.move_as_error(); - return; - } - auto M = std::make_unique>(R.move_as_ok()); - auto id = M->ext_id(); - if (ihr_messages_hashes_.count(id.hash) == 0) { - ihr_messages_.emplace(id, std::move(M)); - ihr_messages_hashes_.emplace(id.hash, id); - } } void ValidatorManagerImpl::new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, @@ -1076,65 +1022,11 @@ void ValidatorManagerImpl::wait_block_message_queue_short(BlockIdExt block_id, t void ValidatorManagerImpl::get_external_messages( ShardIdFull shard, td::Promise, int>>> promise) { - td::Timer t; - size_t processed = 0, deleted = 0; - std::vector, int>> res; - MessageId left{AccountIdPrefixFull{shard.workchain, shard.shard & (shard.shard - 1)}, Bits256::zero()}; - size_t total_msgs = 0; - td::Random::Fast rnd; - for (auto iter = ext_msgs_.rbegin(); iter != ext_msgs_.rend(); ++iter) { - std::vector, int>> cur_res; - int priority = iter->first; - auto &msgs = iter->second; - auto it = msgs.ext_messages_.lower_bound(left); - while (it != msgs.ext_messages_.end()) { - auto s = it->first; - if (!shard_contains(shard, s.dst)) { - break; - } - ++processed; - if (it->second->expired()) { - msgs.ext_addr_messages_[it->second->address()].erase(it->first.hash); - ext_messages_hashes_.erase(it->first.hash); - it = msgs.ext_messages_.erase(it); - ++deleted; - continue; - } - if (it->second->is_active()) { - cur_res.emplace_back(it->second->message(), priority); - } - it++; - } - td::random_shuffle(td::as_mutable_span(cur_res), rnd); - res.insert(res.end(), cur_res.begin(), cur_res.end()); - total_msgs += msgs.ext_messages_.size(); - } - LOG(WARNING) << "get_external_messages to shard " << shard.to_str() << " : time=" << t.elapsed() - << " result_size=" << res.size() << " processed=" << processed << " expired=" << deleted - << " total_size=" << total_msgs; - promise.set_value(std::move(res)); + td::actor::send_closure(ext_message_pool_, &ExtMessagePool::get_external_messages_for_collator, shard, + std::move(promise)); } void ValidatorManagerImpl::get_ihr_messages(ShardIdFull shard, td::Promise>> promise) { - std::vector> res; - MessageId left{AccountIdPrefixFull{shard.workchain, shard.shard & (shard.shard - 1)}, Bits256::zero()}; - auto it = ihr_messages_.lower_bound(left); - while (it != ihr_messages_.end()) { - auto s = it->first; - if (!shard_contains(shard, s.dst)) { - break; - } - if (it->second->expired()) { - ihr_messages_hashes_.erase(it->first.hash); - it = ihr_messages_.erase(it); - continue; - } - if (it->second->is_active()) { - res.push_back(it->second->message()); - } - it++; - } - promise.set_value(std::move(res)); } void ValidatorManagerImpl::get_shard_blocks_for_collator( @@ -1151,55 +1043,12 @@ void ValidatorManagerImpl::get_shard_blocks_for_collator( void ValidatorManagerImpl::complete_external_messages(std::vector to_delay, std::vector to_delete) { - for (auto &hash : to_delete) { - auto it = ext_messages_hashes_.find(hash); - if (it != ext_messages_hashes_.end()) { - int priority = it->second.first; - auto msg_id = it->second.second; - ext_msgs_[priority].erase(msg_id); - ext_messages_hashes_.erase(it); - } - } - unsigned long soft_mempool_limit = 1024; - for (auto &hash : to_delay) { - auto it = ext_messages_hashes_.find(hash); - if (it != ext_messages_hashes_.end()) { - int priority = it->second.first; - auto msg_id = it->second.second; - auto &msgs = ext_msgs_[priority]; - auto it2 = msgs.ext_messages_.find(msg_id); - if ((msgs.ext_messages_.size() < soft_mempool_limit) && it2->second->can_postpone()) { - it2->second->postpone(); - } else { - msgs.erase(msg_id); - ext_messages_hashes_.erase(it); - } - } - } + td::actor::send_closure(ext_message_pool_, &ExtMessagePool::complete_external_messages, std::move(to_delay), + std::move(to_delete)); } void ValidatorManagerImpl::complete_ihr_messages(std::vector to_delay, std::vector to_delete) { - for (auto &hash : to_delete) { - auto it = ihr_messages_hashes_.find(hash); - if (it != ihr_messages_hashes_.end()) { - ihr_messages_.erase(it->second); - ihr_messages_hashes_.erase(it); - } - } - for (auto &hash : to_delay) { - auto it = ihr_messages_hashes_.find(hash); - if (it != ihr_messages_hashes_.end()) { - auto it2 = ihr_messages_.find(it->second); - CHECK(it2 != ihr_messages_.end()); - if (it2->second->can_postpone()) { - it2->second->postpone(); - } else { - ihr_messages_.erase(it2); - ihr_messages_hashes_.erase(it); - } - } - } } void ValidatorManagerImpl::get_block_data_from_db(ConstBlockHandle handle, td::Promise> promise) { @@ -1843,13 +1692,7 @@ void ValidatorManagerImpl::send_get_next_key_blocks_request(BlockIdExt block_id, callback_->get_next_key_blocks(block_id, td::Timestamp::in(10.0), std::move(promise)); } -void ValidatorManagerImpl::send_external_message(td::Ref message) { - callback_->send_ext_message(message->shard(), message->serialize()); - add_external_message(std::move(message), 0); -} - void ValidatorManagerImpl::send_ihr_message(td::Ref message) { - callback_->send_ihr_message(message->shard(), message->serialize()); } void ValidatorManagerImpl::send_top_shard_block_description(td::Ref desc) { @@ -1951,6 +1794,7 @@ void ValidatorManagerImpl::start_up() { lite_server_cache_ = create_liteserver_cache_actor(actor_id(this), db_root_); token_manager_ = td::actor::create_actor("tokenmanager"); storage_stat_cache_ = td::actor::create_actor("storagestatcache"); + ext_message_pool_ = td::actor::create_actor("extmessages", opts_, actor_id(this)); td::mkdir(db_root_ + "/tmp/").ensure(); td::mkdir(db_root_ + "/catchains/").ensure(); @@ -2298,6 +2142,7 @@ void ValidatorManagerImpl::new_masterchain_block() { for (auto &[_, actor] : shard_block_retainers_) { td::actor::send_closure(actor, &ShardBlockRetainer::update_masterchain_state, last_masterchain_state_); } + td::actor::send_closure(ext_message_pool_, &ExtMessagePool::update_last_masterchain_state, last_masterchain_state_); if (last_masterchain_seqno_ % 1024 == 0) { LOG(WARNING) << "applied masterchain block " << last_masterchain_block_id_.to_str(); } else { @@ -2653,8 +2498,7 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group if (check_gc_list_.count(session_id) == 1) { return td::actor::ActorOwn{}; } else { - // Call get_external_messages to cleanup mempool for the shard - get_external_messages(shard, [](td::Result, int>>>) {}); + td::actor::send_closure(ext_message_pool_, &ExtMessagePool::cleanup_external_messages, shard); auto validator_id = get_validator(shard, validator_set); CHECK(!validator_id.is_zero()); @@ -2929,7 +2773,7 @@ void ValidatorManagerImpl::alarm() { alarm_timestamp().relax(check_shard_clients_); if (log_ls_stats_at_.is_in_past()) { - if (!ls_stats_.empty() || ls_stats_check_ext_messages_ != 0) { + if (!ls_stats_.empty()) { td::StringBuilder sb; sb << "Liteserver stats (1 minute):"; td::uint32 total = 0; @@ -2940,26 +2784,12 @@ void ValidatorManagerImpl::alarm() { if (total > 0) { sb << " TOTAL:" << total; } - if (ls_stats_check_ext_messages_ > 0) { - sb << " checkExtMessage:" << ls_stats_check_ext_messages_; - } LOG(WARNING) << sb.as_cslice(); } ls_stats_.clear(); - ls_stats_check_ext_messages_ = 0; log_ls_stats_at_ = td::Timestamp::in(60.0); } alarm_timestamp().relax(log_ls_stats_at_); - if (cleanup_mempool_at_.is_in_past()) { - if (is_validator() || !collator_nodes_.empty()) { - get_external_messages(ShardIdFull{masterchainId, shardIdAll}, - [](td::Result, int>>>) {}); - get_external_messages(ShardIdFull{basechainId, shardIdAll}, - [](td::Result, int>>>) {}); - } - cleanup_mempool_at_ = td::Timestamp::in(250.0); - } - alarm_timestamp().relax(cleanup_mempool_at_); } void ValidatorManagerImpl::update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) { @@ -3128,8 +2958,6 @@ void ValidatorManagerImpl::prepare_stats(td::Promise opts) if (!shard_block_verifier_.empty()) { td::actor::send_closure(shard_block_verifier_, &ShardBlockVerifier::update_options, opts); } + td::actor::send_closure(ext_message_pool_, &ExtMessagePool::update_options, opts); opts_ = std::move(opts); } @@ -3701,29 +3531,6 @@ void ValidatorManagerImpl::unregister_stats_provider(td::uint64 idx) { stats_providers_.erase(idx); } -size_t ValidatorManagerImpl::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) { - before_query(); - auto it1 = counter_cur_.find({wc, addr}); - auto it2 = counter_prev_.find({wc, addr}); - return (it1 == counter_cur_.end() ? 0 : it1->second) + (it2 == counter_prev_.end() ? 0 : it2->second); -} -size_t ValidatorManagerImpl::CheckedExtMsgCounter::inc_msg_count(WorkchainId wc, StdSmcAddress addr) { - before_query(); - auto it2 = counter_prev_.find({wc, addr}); - return (it2 == counter_prev_.end() ? 0 : it2->second) + ++counter_cur_[{wc, addr}]; -} -void ValidatorManagerImpl::CheckedExtMsgCounter::before_query() { - while (cleanup_at_.is_in_past()) { - counter_prev_ = std::move(counter_cur_); - counter_cur_.clear(); - if (counter_prev_.empty()) { - cleanup_at_ = td::Timestamp::in(max_ext_msg_per_addr_time_window() / 2.0); - break; - } - cleanup_at_ += max_ext_msg_per_addr_time_window() / 2.0; - } -} - template void ValidatorManagerImpl::write_session_stats(const T &obj) { std::string fname = opts_->get_session_logs_file(); diff --git a/validator/manager.hpp b/validator/manager.hpp index f5a1668711..3625033451 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -26,6 +26,7 @@ #include "collator-node/collator-node.hpp" #include "common/refcnt.hpp" #include "impl/candidates-buffer.hpp" +#include "impl/ext-message-pool.hpp" #include "interfaces/db.h" #include "interfaces/validator-manager.h" #include "rldp/rldp.h" @@ -57,75 +58,6 @@ class WaitZeroState; class WaitShardState; class WaitBlockData; -template -struct MessageId { - AccountIdPrefixFull dst; - typename MType::Hash hash; - - bool operator<(const MessageId &msg) const { - if (dst < msg.dst) { - return true; - } - if (msg.dst < dst) { - return false; - } - return hash < msg.hash; - } -}; - -template -class MessageExt { - public: - auto shard() const { - return message_->shard(); - } - auto ext_id() const { - auto shard = message_->shard(); - return MessageId{shard, message_->hash()}; - } - auto message() const { - return message_; - } - auto hash() const { - return message_->hash(); - } - auto address() const { - return std::make_pair(message_->wc(), message_->addr()); - } - bool is_active() { - if (!active_) { - if (reactivate_at_.is_in_past()) { - active_ = true; - generation_++; - } - } - return active_; - } - bool can_postpone() const { - return generation_ <= 2; - } - void postpone() { - if (!active_) { - return; - } - active_ = false; - reactivate_at_ = td::Timestamp::in(generation_ * 5.0); - } - bool expired() const { - return delete_at_.is_in_past(); - } - MessageExt(td::Ref msg) : message_(std::move(msg)) { - delete_at_ = td::Timestamp::in(600); - } - - private: - td::Ref message_; - td::uint32 generation_ = 0; - bool active_ = true; - td::Timestamp reactivate_at_; - td::Timestamp delete_at_; -}; - class BlockHandleLru : public td::ListNode { public: BlockHandle handle() const { @@ -272,32 +204,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::LRUCache cached_block_data_{/* max_size = */ 128}; td::LRUCache cached_checked_shard_block_descriptions_{/* max_size = */ 1024}; - struct ExtMessages { - std::map, std::unique_ptr>> ext_messages_; - std::map, std::map>> - ext_addr_messages_; - void erase(const MessageId &id) { - auto it = ext_messages_.find(id); - CHECK(it != ext_messages_.end()); - ext_addr_messages_[it->second->address()].erase(id.hash); - ext_messages_.erase(it); - } - }; - std::map ext_msgs_; // priority -> messages - std::map>> ext_messages_hashes_; // hash -> priority - td::Timestamp cleanup_mempool_at_; - // IHR ? - std::map, std::unique_ptr>> ihr_messages_; - std::map> ihr_messages_hashes_; - - struct CheckedExtMsgCounter { - std::map, size_t> counter_cur_, counter_prev_; - td::Timestamp cleanup_at_ = td::Timestamp::now(); - - size_t get_msg_count(WorkchainId wc, StdSmcAddress addr); - size_t inc_msg_count(WorkchainId wc, StdSmcAddress addr); - void before_query(); - } checked_ext_msg_counter_; + td::actor::ActorOwn ext_message_pool_; private: // VALIDATOR GROUPS @@ -428,9 +335,10 @@ class ValidatorManagerImpl : public ValidatorManager { void get_key_block_proof_link(BlockIdExt block_id, td::Promise promise) override; //void get_block_description(BlockIdExt block_id, td::Promise promise) override; - void new_external_message(td::BufferSlice data, int priority) override; - void add_external_message(td::Ref message, int priority); - void check_external_message(td::BufferSlice data, td::Promise> promise) override; + td::actor::Task<> new_external_message_broadcast(td::BufferSlice data, int priority) override; + td::actor::Task<> new_external_message_query(td::BufferSlice data) override; + td::actor::Task<> new_external_message_query_cont(td::Ref message, + td::actor::StartedTask<> wait_allow_broadcast); void new_ihr_message(td::BufferSlice data) override; void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, @@ -553,7 +461,6 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise promise) override; void send_get_next_key_blocks_request(BlockIdExt block_id, td::uint32 priority, td::Promise> promise) override; - void send_external_message(td::Ref message) override; void send_ihr_message(td::Ref message) override; void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast, int mode) override; @@ -769,15 +676,6 @@ class ValidatorManagerImpl : public ValidatorManager { double state_ttl() const { return opts_->state_ttl(); } - double max_mempool_num() const { - return opts_->max_mempool_num(); - } - static double max_ext_msg_per_addr_time_window() { - return 10.0; - } - static size_t max_ext_msg_per_addr() { - return 3 * 10; - } void got_persistent_state_descriptions(std::vector> descs); void add_persistent_state_description_impl(td::Ref desc); @@ -793,11 +691,9 @@ class ValidatorManagerImpl : public ValidatorManager { td::Timestamp log_ls_stats_at_; std::map ls_stats_; // lite_api ID -> count, 0 for unknown - td::uint32 ls_stats_check_ext_messages_{0}; UnixTime started_at_ = (UnixTime)td::Clocks::system(); std::map total_ls_queries_ok_, total_ls_queries_error_; // lite_api ID -> count, 0 for unknown - td::uint64 total_check_ext_messages_ok_{0}, total_check_ext_messages_error_{0}; td::uint64 total_collated_blocks_master_ok_{0}, total_collated_blocks_master_error_{0}; td::uint64 total_validated_blocks_master_ok_{0}, total_validated_blocks_master_error_{0}; td::uint64 total_collated_blocks_shard_ok_{0}, total_collated_blocks_shard_error_{0}; diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index 2508b6d2c5..504c9e0771 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -98,9 +98,9 @@ td::Status ShardBlockVerifierConfig::unpack(const ton_api::engine_validator_shar td::Ref ValidatorManagerOptions::create(BlockIdExt zero_block_id, BlockIdExt init_block_id, bool allow_blockchain_init, double sync_blocks_before, - double block_ttl, double state_ttl, - double max_mempool_num, double archive_ttl, - double key_proof_ttl, bool initial_sync_disabled) { + double block_ttl, double state_ttl, double archive_ttl, + double key_proof_ttl, size_t max_mempool_num, + bool initial_sync_disabled) { return td::make_ref(zero_block_id, init_block_id, allow_blockchain_init, sync_blocks_before, block_ttl, state_ttl, max_mempool_num, archive_ttl, key_proof_ttl, initial_sync_disabled); diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index fd536b1dff..1a46a72f45 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -49,7 +49,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { double state_ttl() const override { return state_ttl_; } - double max_mempool_num() const override { + size_t max_mempool_num() const override { return max_mempool_num_; } double archive_ttl() const override { @@ -198,7 +198,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_state_ttl(double value) override { state_ttl_ = value; } - void set_max_mempool_num(double value) override { + void set_max_mempool_num(size_t value) override { max_mempool_num_ = value; } void set_archive_ttl(double value) override { @@ -307,7 +307,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { } ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, bool allow_blockchain_init, - double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, + double sync_blocks_before, double block_ttl, double state_ttl, size_t max_mempool_num, double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) : zero_block_id_(zero_block_id) , init_block_id_(init_block_id) @@ -329,7 +329,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { double sync_blocks_before_; double block_ttl_; double state_ttl_; - double max_mempool_num_; + size_t max_mempool_num_; double archive_ttl_; double key_proof_ttl_; bool initial_sync_disabled_; diff --git a/validator/validator.h b/validator/validator.h index 3e64522fbf..a2f15f5b58 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -35,6 +35,7 @@ #include "interfaces/validator-set.h" #include "overlay/overlays.h" #include "td/actor/actor.h" +#include "td/actor/coro_utils.h" #include "ton/ton-types.h" namespace ton { @@ -111,7 +112,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual double sync_blocks_before() const = 0; virtual double block_ttl() const = 0; virtual double state_ttl() const = 0; - virtual double max_mempool_num() const = 0; + virtual size_t max_mempool_num() const = 0; virtual double archive_ttl() const = 0; virtual double key_proof_ttl() const = 0; virtual bool initial_sync_disabled() const = 0; @@ -158,7 +159,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_sync_blocks_before(double value) = 0; virtual void set_block_ttl(double value) = 0; virtual void set_state_ttl(double value) = 0; - virtual void set_max_mempool_num(double value) = 0; + virtual void set_max_mempool_num(size_t value) = 0; virtual void set_archive_ttl(double value) = 0; virtual void set_key_proof_ttl(double value) = 0; virtual void set_initial_sync_disabled(bool value) = 0; @@ -195,7 +196,7 @@ struct ValidatorManagerOptions : public td::CntObject { bool allow_blockchain_init = false, double sync_blocks_before = 3600, double block_ttl = 86400, double state_ttl = 86400, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650, - double max_mempool_num = 999999, bool initial_sync_disabled = false); + size_t max_mempool_num = 999999, bool initial_sync_disabled = false); }; class ValidatorManagerInterface : public td::actor::Actor { @@ -301,8 +302,10 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_next_block(BlockIdExt block_id, td::Promise promise) = 0; virtual void write_handle(BlockHandle handle, td::Promise promise) = 0; - virtual void new_external_message(td::BufferSlice data, int priority) = 0; - virtual void check_external_message(td::BufferSlice data, td::Promise> promise) = 0; + virtual td::actor::Task<> new_external_message_broadcast(td::BufferSlice data, int priority) = 0; + virtual td::actor::Task<> new_external_message_query(td::BufferSlice data) { + co_return td::Status::Error("not implemented"); + } virtual void new_ihr_message(td::BufferSlice data) = 0; virtual void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; From bb43a89d5dcfb54a8774dc97d46329e3f5ba9af6 Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Mon, 15 Dec 2025 10:41:27 -0500 Subject: [PATCH 508/524] Remove downloadPersistentStateSlice and getPersistentStateSize requests (#1954) Everyone should be using v2 versions of these requests. --- tl/generate/scheme/ton_api.tl | 3 --- validator/full-node-master.cpp | 13 ------------- validator/full-node-master.hpp | 4 ---- validator/full-node-shard.cpp | 13 ------------- validator/full-node-shard.hpp | 4 ---- validator/full-node.cpp | 1 - 6 files changed, 38 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 09ae4fa04f..345623130c 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -507,10 +507,7 @@ tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data; tonNode.getOutMsgQueueProof dst_shard:tonNode.shardId blocks:(vector tonNode.blockIdExt) limits:tonNode.importedMsgQueueLimits = tonNode.OutMsgQueueProof; -tonNode.downloadPersistentStateSlice block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt offset:long max_size:long = tonNode.Data; -tonNode.getPersistentStateSize block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PersistentStateSize; tonNode.preparePersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PreparedState; - tonNode.downloadPersistentStateSliceV2 state:tonNode.persistentStateIdV2 offset:long max_size:long = tonNode.Data; tonNode.getPersistentStateSizeV2 state:tonNode.persistentStateIdV2 = tonNode.PersistentStateSize; diff --git a/validator/full-node-master.cpp b/validator/full-node-master.cpp index 931c45f23d..22567f9136 100644 --- a/validator/full-node-master.cpp +++ b/validator/full-node-master.cpp @@ -288,12 +288,6 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo masterchain_block_id, UnsplitStateType{}, std::move(P)); } -void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, - td::Promise promise) { - auto query_v2 = create_tl_object(persistent_state_id_from_v1_query(query)); - return process_query(src, *query_v2, std::move(promise)); -} - void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise) { auto cnt = static_cast(query.max_size_); @@ -337,13 +331,6 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_zero_state, block_id, std::move(P)); } -void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, - td::Promise promise) { - auto query_v2 = create_tl_object( - persistent_state_id_from_v1_query(query), query.offset_, query.max_size_); - return process_query(src, *query_v2, std::move(promise)); -} - void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise) { promise.set_value( diff --git a/validator/full-node-master.hpp b/validator/full-node-master.hpp index b3ada4b101..d227209587 100644 --- a/validator/full-node-master.hpp +++ b/validator/full-node-master.hpp @@ -66,14 +66,10 @@ class FullNodeMasterImpl : public FullNodeMaster { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise); - void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, - td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadZeroState &query, td::Promise promise); - void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, - td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_slave_sendExtMessage &query, diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index a4094a6d86..cf4be9aec7 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -524,12 +524,6 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod masterchain_block_id, UnsplitStateType{}, std::move(P)); } -void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, - td::Promise promise) { - auto query_v2 = create_tl_object(persistent_state_id_from_v1_query(query)); - return process_query(src, *query_v2, std::move(promise)); -} - void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise) { auto cnt = static_cast(query.max_size_); @@ -579,13 +573,6 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_zero_state, block_id, std::move(P)); } -void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, - td::Promise promise) { - auto query_v2 = create_tl_object( - persistent_state_id_from_v1_query(query), query.offset_, query.max_size_); - return process_query(src, *query_v2, std::move(promise)); -} - void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise) { VLOG(FULL_NODE_DEBUG) << "Got query getCapabilities from " << src; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 83e2962d7f..56fa3b2ca1 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -126,14 +126,10 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise); - void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, - td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadZeroState &query, td::Promise promise); - void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, - td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, diff --git a/validator/full-node.cpp b/validator/full-node.cpp index f3f6e2a00e..2e242343b2 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -907,7 +907,6 @@ decltype(FullNodeImpl::limiter_) FullNodeImpl::make_limiter(const FullNodeOption RateLimit{w_size, g_limit}, std::map{{ton_api::tonNode_getArchiveSlice::ID, {w_size, h_limit}}, {ton_api::tonNode_downloadPersistentStateSliceV2::ID, {w_size, h_limit}}, - {ton_api::tonNode_downloadPersistentStateSlice::ID, {w_size, h_limit}}, {ton_api::tonNode_downloadZeroState::ID, {w_size, h_limit}}, {ton_api::tonNode_downloadBlock::ID, {w_size, m_limit}}, From 381b4b32305816f0600ce841599b2dfed3bf4d7c Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Mon, 15 Dec 2025 10:42:21 -0500 Subject: [PATCH 509/524] Ask clang-format to insert new line at EOF and use west const (#1955) * Simplify clang-format by actually basing it on Google style There is no benefit in not using `BasedOnStyle` rule. The new config differs from the old one in a few minor rules that do not introduce any formatting changes. * Ask clang-format to insert new line at EOF * Use west const everywhere --- .clang-format | 96 ++----------------- crypto/block/account-storage-stat.cpp | 2 +- crypto/block/mc-config.h | 2 +- .../PrecompiledSmartContract.h | 2 +- crypto/block/precompiled-smc/common.h | 2 +- crypto/func/func.cpp | 2 +- crypto/func/func.h | 4 +- crypto/funcfiftlib/funcfiftlib.cpp | 4 +- crypto/smc-envelope/WalletV4.h | 2 +- crypto/test/Ed25519.cpp | 2 +- crypto/vm/Hasher.h | 2 +- crypto/vm/boc-writers.h | 10 +- crypto/vm/cells/CellHash.h | 2 +- crypto/vm/cells/DataCell.cpp | 4 +- crypto/vm/cells/DataCell.h | 10 +- crypto/vm/tonops.cpp | 2 +- emulator/StringLog.h | 2 +- emulator/emulator-emscripten.cpp | 2 +- emulator/tvm-emulator.hpp | 2 +- lite-client/query-utils.cpp | 2 +- rldp-http-proxy/DNSResolver.h | 2 +- storage/db.h | 2 +- storage/storage-daemon/StorageManager.cpp | 2 +- storage/storage-daemon/StorageProvider.h | 2 +- storage/storage-daemon/smc-util.h | 2 +- tdactor/benchmark/gbench-coro.cpp | 2 +- .../benchmark/third_party/MoodyCamelQueue.h | 54 +++++------ tdactor/example/actor-example-coroutines.cpp | 2 +- tdactor/td/actor/coro.h | 2 +- tdutils/td/utils/date.h | 82 ++++++++-------- tdutils/td/utils/tests.h | 6 +- tonlib/tonlib/TonlibClientWrapper.cpp | 2 +- validator/db/archive-manager.cpp | 8 +- validator/db/archive-manager.hpp | 2 +- validator/db/db-utils.cpp | 2 +- validator/db/db-utils.h | 2 +- validator/db/statedb.cpp | 14 +-- validator/downloaders/download-state.cpp | 6 +- validator/downloaders/wait-block-state.cpp | 2 +- validator/impl/collator.cpp | 2 +- validator/interfaces/liteserver.h | 2 +- validator/interfaces/persistent-state.h | 8 +- validator/manager.cpp | 4 +- validator/queue-size-counter.cpp | 2 +- validator/rate-limiter.h | 2 +- validator/state-serializer.cpp | 2 +- validator/state-serializer.hpp | 2 +- 47 files changed, 150 insertions(+), 226 deletions(-) diff --git a/.clang-format b/.clang-format index 886aa8df99..f4f7d099f8 100644 --- a/.clang-format +++ b/.clang-format @@ -1,55 +1,12 @@ --- -Language: Cpp -# BasedOnStyle: Google -AccessModifierOffset: -1 -AlignAfterOpenBracket: Align -AlignConsecutiveAssignments: false -AlignConsecutiveDeclarations: false -AlignEscapedNewlinesLeft: true -AlignOperands: true -AlignTrailingComments: true -AllowAllParametersOfDeclarationOnNextLine: true -AllowShortBlocksOnASingleLine: false -AllowShortCaseLabelsOnASingleLine: false -AllowShortFunctionsOnASingleLine: None # All -AllowShortIfStatementsOnASingleLine: false # true -AllowShortLoopsOnASingleLine: false # true -AlwaysBreakAfterDefinitionReturnType: None -AlwaysBreakAfterReturnType: None -AlwaysBreakBeforeMultilineStrings: true -AlwaysBreakTemplateDeclarations: true -BinPackArguments: true -BinPackParameters: true -BraceWrapping: - AfterClass: false - AfterControlStatement: false - AfterEnum: false - AfterFunction: false - AfterNamespace: false - AfterObjCDeclaration: false - AfterStruct: false - AfterUnion: false - BeforeCatch: false - BeforeElse: false - IndentBraces: false -BreakBeforeBinaryOperators: None -BreakBeforeBraces: Attach -BreakBeforeTernaryOperators: true -BreakConstructorInitializersBeforeComma: true # false -# BreakAfterJavaFieldAnnotations: false -BreakStringLiterals: true -ColumnLimit: 120 # 80 -CommentPragmas: '^ IWYU pragma:' -ConstructorInitializerAllOnOneLineOrOnePerLine: true -ConstructorInitializerIndentWidth: 4 -ContinuationIndentWidth: 4 -Cpp11BracedListStyle: true +Language: Cpp +AllowShortFunctionsOnASingleLine: None +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +BasedOnStyle: Google +BreakConstructorInitializers: BeforeComma +ColumnLimit: 120 DerivePointerAlignment: true -DisableFormat: false -ExperimentalAutoDetectBinPacking: false -FixNamespaceComments: true -# ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] -IncludeBlocks: Regroup IncludeCategories: - Regex: '>$' Priority: -3 @@ -59,40 +16,7 @@ IncludeCategories: Priority: -1 - Regex: '^\"[^/]*\"$' Priority: 0 -IndentCaseLabels: true -IndentWidth: 2 -IndentWrappedFunctionNames: false -# JavaScriptQuotes: Leave -# JavaScriptWrapImports: true -KeepEmptyLinesAtTheStartOfBlocks: false -MacroBlockBegin: '' -MacroBlockEnd: '' -MaxEmptyLinesToKeep: 1 -NamespaceIndentation: None -# ObjCBlockIndentWidth: 2 -# ObjCSpaceAfterProperty: false -# ObjCSpaceBeforeProtocolList: false -PenaltyBreakBeforeFirstCallParameter: 1 -PenaltyBreakComment: 300 -PenaltyBreakFirstLessLess: 120 -PenaltyBreakString: 1000 -PenaltyExcessCharacter: 1000000 -PenaltyReturnTypeOnItsOwnLine: 200 -PointerAlignment: Left -ReflowComments: false # true -SortIncludes: true -SpaceAfterCStyleCast: false -SpaceAfterTemplateKeyword: true -SpaceBeforeAssignmentOperators: true -SpaceBeforeParens: ControlStatements -SpaceInEmptyParentheses: false -SpacesBeforeTrailingComments: 2 -SpacesInAngles: false -SpacesInContainerLiterals: true -SpacesInCStyleCastParentheses: false -SpacesInParentheses: false -SpacesInSquareBrackets: false -Standard: Auto -TabWidth: 100 # 8 -UseTab: Never +InsertNewlineAtEOF: true +ReflowComments: Never +QualifierAlignment: Left ... diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index a6159c2245..b7f6c592d5 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -279,4 +279,4 @@ td::Status AccountStorageStat::finalize_entry(Entry& e) { return td::Status::OK(); } -} // namespace block \ No newline at end of file +} // namespace block diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 378439ae8f..03ff2dc475 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -658,7 +658,7 @@ class Config { const WorkchainSet& get_workchain_list() const { return workchains_; } - std::shared_ptr const& get_cur_validator_set() const& { + const std::shared_ptr& get_cur_validator_set() const& { return cur_validators_; } std::pair get_validator_set_start_stop(int next = 0) const; diff --git a/crypto/block/precompiled-smc/PrecompiledSmartContract.h b/crypto/block/precompiled-smc/PrecompiledSmartContract.h index e20cf90ece..7e3c8b8f41 100644 --- a/crypto/block/precompiled-smc/PrecompiledSmartContract.h +++ b/crypto/block/precompiled-smc/PrecompiledSmartContract.h @@ -121,4 +121,4 @@ class PrecompiledSmartContract { std::unique_ptr get_implementation(td::Bits256 code_hash); void set_precompiled_execution_enabled(bool value); // disabled by default -} // namespace block::precompiled \ No newline at end of file +} // namespace block::precompiled diff --git a/crypto/block/precompiled-smc/common.h b/crypto/block/precompiled-smc/common.h index 174db76785..1d8b0db53f 100644 --- a/crypto/block/precompiled-smc/common.h +++ b/crypto/block/precompiled-smc/common.h @@ -19,4 +19,4 @@ #include "vm/cellops.h" #include "vm/tonops.h" -#include "PrecompiledSmartContract.h" \ No newline at end of file +#include "PrecompiledSmartContract.h" diff --git a/crypto/func/func.cpp b/crypto/func/func.cpp index 39fc76cf7a..7610630c52 100644 --- a/crypto/func/func.cpp +++ b/crypto/func/func.cpp @@ -260,4 +260,4 @@ int func_proceed(const std::vector& sources, std::ostream& outs, st return 0; } -} // namespace funC \ No newline at end of file +} // namespace funC diff --git a/crypto/func/func.h b/crypto/func/func.h index e02729de9d..7b1e469a53 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -857,8 +857,8 @@ extern std::set prohibited_var_names; class ReadCallback { public: /// Noncopyable. - ReadCallback(ReadCallback const&) = delete; - ReadCallback& operator=(ReadCallback const&) = delete; + ReadCallback(const ReadCallback&) = delete; + ReadCallback& operator=(const ReadCallback&) = delete; enum class Kind { ReadFile, Realpath }; diff --git a/crypto/funcfiftlib/funcfiftlib.cpp b/crypto/funcfiftlib/funcfiftlib.cpp index aa9af93670..69264a22ce 100644 --- a/crypto/funcfiftlib/funcfiftlib.cpp +++ b/crypto/funcfiftlib/funcfiftlib.cpp @@ -89,12 +89,12 @@ td::Result compile_internal(char* config_json) { /// said storage after func_compile returns. /// /// If the callback is not supported, *o_contents and *o_error must be set to NULL. -typedef void (*CStyleReadFileCallback)(char const* _kind, char const* _data, char** o_contents, char** o_error); +typedef void (*CStyleReadFileCallback)(const char* _kind, const char* _data, char** o_contents, char** o_error); funC::ReadCallback::Callback wrapReadCallback(CStyleReadFileCallback _readCallback) { funC::ReadCallback::Callback readCallback; if (_readCallback) { - readCallback = [=](funC::ReadCallback::Kind _kind, char const* _data) -> td::Result { + readCallback = [=](funC::ReadCallback::Kind _kind, const char* _data) -> td::Result { char* contents_c = nullptr; char* error_c = nullptr; _readCallback(funC::ReadCallback::kindString(_kind).data(), _data, &contents_c, &error_c); diff --git a/crypto/smc-envelope/WalletV4.h b/crypto/smc-envelope/WalletV4.h index 313647ac29..812f2685f0 100644 --- a/crypto/smc-envelope/WalletV4.h +++ b/crypto/smc-envelope/WalletV4.h @@ -44,4 +44,4 @@ class WalletV4 : public WalletBase { td::Result get_wallet_id() const override; }; -} // namespace ton \ No newline at end of file +} // namespace ton diff --git a/crypto/test/Ed25519.cpp b/crypto/test/Ed25519.cpp index 5109aa243b..f8a2d720d1 100644 --- a/crypto/test/Ed25519.cpp +++ b/crypto/test/Ed25519.cpp @@ -246,4 +246,4 @@ TEST(Crypto, ed25519_benchmark) { bench(ed25519_signBench()); bench(ed25519_shared_secretBench()); bench(ed25519_verifyBench()); -} \ No newline at end of file +} diff --git a/crypto/vm/Hasher.h b/crypto/vm/Hasher.h index 9d1f4fdb77..e2cd859685 100644 --- a/crypto/vm/Hasher.h +++ b/crypto/vm/Hasher.h @@ -56,4 +56,4 @@ class Hasher { std::unique_ptr impl_; }; -} // namespace vm \ No newline at end of file +} // namespace vm diff --git a/crypto/vm/boc-writers.h b/crypto/vm/boc-writers.h index ffd09ebcab..451360fe57 100644 --- a/crypto/vm/boc-writers.h +++ b/crypto/vm/boc-writers.h @@ -49,7 +49,7 @@ struct BufferWriter { } DCHECK(!bytes); } - void store_bytes(unsigned char const* data, size_t s) { + void store_bytes(const unsigned char* data, size_t s) { store_ptr += s; chk(); memcpy(store_ptr - s, data, s); @@ -88,13 +88,13 @@ struct FileWriter { flush_if_needed(bytes); writer.store_uint(value, bytes); } - void store_bytes(unsigned char const* data, size_t s) { + void store_bytes(const unsigned char* data, size_t s) { flush_if_needed(s); writer.store_bytes(data, s); } unsigned get_crc32() const { - unsigned char const* start = buf.data(); - unsigned char const* end = start + writer.position(); + const unsigned char* start = buf.data(); + const unsigned char* end = start + writer.position(); return td::crc32c_extend(current_crc32, td::Slice(start, end)); } @@ -145,4 +145,4 @@ struct FileWriter { td::Status res = td::Status::OK(); }; } // namespace boc_writers -} // namespace vm \ No newline at end of file +} // namespace vm diff --git a/crypto/vm/cells/CellHash.h b/crypto/vm/cells/CellHash.h index 9bbfd4cc69..24b8a19e92 100644 --- a/crypto/vm/cells/CellHash.h +++ b/crypto/vm/cells/CellHash.h @@ -82,7 +82,7 @@ template <> struct hash { typedef vm::CellHash argument_type; typedef std::size_t result_type; - result_type operator()(argument_type const& s) const noexcept { + result_type operator()(const argument_type& s) const noexcept { return cell_hash_slice_hash(s.as_slice()); } }; diff --git a/crypto/vm/cells/DataCell.cpp b/crypto/vm/cells/DataCell.cpp index e342b732c5..505a7d7f42 100644 --- a/crypto/vm/cells/DataCell.cpp +++ b/crypto/vm/cells/DataCell.cpp @@ -114,11 +114,11 @@ class CellChecker { return virtualized_; } - std::array const& depths() const { + const std::array& depths() const { return depth_; } - std::array const& hashes() const { + const std::array& hashes() const { return hash_; } diff --git a/crypto/vm/cells/DataCell.h b/crypto/vm/cells/DataCell.h index 9e2e52b0e3..daf43d7814 100644 --- a/crypto/vm/cells/DataCell.h +++ b/crypto/vm/cells/DataCell.h @@ -56,7 +56,7 @@ class DataCell final : public Cell { } } - DataCell(DataCell const&) = delete; + DataCell(const DataCell&) = delete; DataCell(DataCell&&) = delete; ~DataCell(); @@ -106,8 +106,8 @@ class DataCell final : public Cell { return bit_length_; } - unsigned char const* get_data() const { - return reinterpret_cast(trailer_ + sizeof(detail::LevelInfo) * (level_ + 1)); + const unsigned char* get_data() const { + return reinterpret_cast(trailer_ + sizeof(detail::LevelInfo) * (level_ + 1)); } Ref get_ref(unsigned idx) const { @@ -171,8 +171,8 @@ class DataCell final : public Cell { DataCell(int bit_length, size_t refs_cnt, Cell::SpecialType type, LevelMask level_mask, bool allocated_in_arena, bool virtualized); - detail::LevelInfo const* level_info() const { - return reinterpret_cast(trailer_); + const detail::LevelInfo* level_info() const { + return reinterpret_cast(trailer_); } virtual td::uint16 do_get_depth(td::uint32 level) const override { diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index a98fd9f703..1f414a39c8 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -2203,7 +2203,7 @@ int exec_send_message(VmState* st) { }; compute_fees(); - auto stored_grams_len = [](td::RefInt256 const& x) -> unsigned { + auto stored_grams_len = [](const td::RefInt256& x) -> unsigned { unsigned bits = x->bit_size(false); return 4 + ((bits + 7) & ~7); }; diff --git a/emulator/StringLog.h b/emulator/StringLog.h index 46c456a81c..74a4275d92 100644 --- a/emulator/StringLog.h +++ b/emulator/StringLog.h @@ -25,4 +25,4 @@ class StringLog : public td::LogInterface { std::string str; }; -#endif //TON_STRINGLOG_H \ No newline at end of file +#endif //TON_STRINGLOG_H diff --git a/emulator/emulator-emscripten.cpp b/emulator/emulator-emscripten.cpp index 545b7fed99..bb9ec51975 100644 --- a/emulator/emulator-emscripten.cpp +++ b/emulator/emulator-emscripten.cpp @@ -299,4 +299,4 @@ const char* run_get_method(const char* params, const char* stack, const char* co const char* version() { return emulator_version(); } -} \ No newline at end of file +} diff --git a/emulator/tvm-emulator.hpp b/emulator/tvm-emulator.hpp index 55c0ca6aa4..4bdf3e5001 100644 --- a/emulator/tvm-emulator.hpp +++ b/emulator/tvm-emulator.hpp @@ -69,4 +69,4 @@ class TvmEmulator { return smc_.send_internal_message(message_body, args.set_amount(amount)); } }; -} // namespace emulator \ No newline at end of file +} // namespace emulator diff --git a/lite-client/query-utils.cpp b/lite-client/query-utils.cpp index 2c20da6a0b..53e6d3732d 100644 --- a/lite-client/query-utils.cpp +++ b/lite-client/query-utils.cpp @@ -405,4 +405,4 @@ td::Result> LiteServerConfig::parse_global_config( return servers; } -} // namespace liteclient \ No newline at end of file +} // namespace liteclient diff --git a/rldp-http-proxy/DNSResolver.h b/rldp-http-proxy/DNSResolver.h index 4afcc86453..5ac6a62149 100644 --- a/rldp-http-proxy/DNSResolver.h +++ b/rldp-http-proxy/DNSResolver.h @@ -47,4 +47,4 @@ class DNSResolver : public td::actor::Actor { double created_at_; }; std::map cache_; -}; \ No newline at end of file +}; diff --git a/storage/db.h b/storage/db.h index d3dedbe332..4843c172ab 100644 --- a/storage/db.h +++ b/storage/db.h @@ -53,4 +53,4 @@ inline td::Result> db_get(td::KeyValue& db, td::Bits256 ke return ton::fetch_tl_object(td::Slice(value), true); } -} // namespace db \ No newline at end of file +} // namespace db diff --git a/storage/storage-daemon/StorageManager.cpp b/storage/storage-daemon/StorageManager.cpp index 68b2165642..4c0d154aa9 100644 --- a/storage/storage-daemon/StorageManager.cpp +++ b/storage/storage-daemon/StorageManager.cpp @@ -389,4 +389,4 @@ void StorageManager::set_upload_speed_limit(double max_speed) { db_store_config(); } -} // namespace ton \ No newline at end of file +} // namespace ton diff --git a/storage/storage-daemon/StorageProvider.h b/storage/storage-daemon/StorageProvider.h index 7690d00d28..6f5a53f0c1 100644 --- a/storage/storage-daemon/StorageProvider.h +++ b/storage/storage-daemon/StorageProvider.h @@ -127,4 +127,4 @@ class StorageProvider : public td::actor::Actor { void sent_next_proof(ContractAddress address); }; -} // namespace ton \ No newline at end of file +} // namespace ton diff --git a/storage/storage-daemon/smc-util.h b/storage/storage-daemon/smc-util.h index 63c3a11a03..7714917167 100644 --- a/storage/storage-daemon/smc-util.h +++ b/storage/storage-daemon/smc-util.h @@ -184,4 +184,4 @@ struct StorageContractData { void get_storage_contract_data(ContractAddress address, td::actor::ActorId client, td::Promise promise); -} // namespace ton \ No newline at end of file +} // namespace ton diff --git a/tdactor/benchmark/gbench-coro.cpp b/tdactor/benchmark/gbench-coro.cpp index 259727c535..4e73ca36b4 100644 --- a/tdactor/benchmark/gbench-coro.cpp +++ b/tdactor/benchmark/gbench-coro.cpp @@ -997,4 +997,4 @@ int main(int argc, char** argv) { //benchmark::SetBenchmarkFilter("BM_Ask"); benchmark::RunSpecifiedBenchmarks(); return 0; -} \ No newline at end of file +} diff --git a/tdactor/benchmark/third_party/MoodyCamelQueue.h b/tdactor/benchmark/third_party/MoodyCamelQueue.h index cd03e3e97b..21a5fb0491 100644 --- a/tdactor/benchmark/third_party/MoodyCamelQueue.h +++ b/tdactor/benchmark/third_party/MoodyCamelQueue.h @@ -77,7 +77,7 @@ template struct thread_id_converter { typedef thread_id_t thread_id_numeric_size_t; typedef thread_id_t thread_id_hash_t; - static thread_id_hash_t prehash(thread_id_t const& x) { + static thread_id_hash_t prehash(const thread_id_t& x) { return x; } }; @@ -519,14 +519,14 @@ static inline void swap_relaxed(std::atomic& left, std::atomic& right) { } template -static inline T const& nomove(T const& x) { +static inline const T& nomove(const T& x) { return x; } template struct nomove_if { template - static inline T const& eval(T const& x) { + static inline const T& eval(const T& x) { return x; } }; @@ -588,8 +588,8 @@ class ThreadExitNotifier { private: ThreadExitNotifier() : tail(nullptr) { } - ThreadExitNotifier(ThreadExitNotifier const&) MOODYCAMEL_DELETE_FUNCTION; - ThreadExitNotifier& operator=(ThreadExitNotifier const&) MOODYCAMEL_DELETE_FUNCTION; + ThreadExitNotifier(const ThreadExitNotifier&) MOODYCAMEL_DELETE_FUNCTION; + ThreadExitNotifier& operator=(const ThreadExitNotifier&) MOODYCAMEL_DELETE_FUNCTION; ~ThreadExitNotifier() { // This thread is about to exit, let everyone know! @@ -698,8 +698,8 @@ struct ProducerToken { } // Disable copying and assignment - ProducerToken(ProducerToken const&) MOODYCAMEL_DELETE_FUNCTION; - ProducerToken& operator=(ProducerToken const&) MOODYCAMEL_DELETE_FUNCTION; + ProducerToken(const ProducerToken&) MOODYCAMEL_DELETE_FUNCTION; + ProducerToken& operator=(const ProducerToken&) MOODYCAMEL_DELETE_FUNCTION; private: template @@ -738,8 +738,8 @@ struct ConsumerToken { } // Disable copying and assignment - ConsumerToken(ConsumerToken const&) MOODYCAMEL_DELETE_FUNCTION; - ConsumerToken& operator=(ConsumerToken const&) MOODYCAMEL_DELETE_FUNCTION; + ConsumerToken(const ConsumerToken&) MOODYCAMEL_DELETE_FUNCTION; + ConsumerToken& operator=(const ConsumerToken&) MOODYCAMEL_DELETE_FUNCTION; private: template @@ -909,8 +909,8 @@ class ConcurrentQueue { } // Disable copying and copy assignment - ConcurrentQueue(ConcurrentQueue const&) MOODYCAMEL_DELETE_FUNCTION; - ConcurrentQueue& operator=(ConcurrentQueue const&) MOODYCAMEL_DELETE_FUNCTION; + ConcurrentQueue(const ConcurrentQueue&) MOODYCAMEL_DELETE_FUNCTION; + ConcurrentQueue& operator=(const ConcurrentQueue&) MOODYCAMEL_DELETE_FUNCTION; // Moving is supported, but note that it is *not* a thread-safe operation. // Nobody can use the queue while it's being moved, and the memory effects @@ -998,7 +998,7 @@ class ConcurrentQueue { // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0, // or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). // Thread-safe. - inline bool enqueue(T const& item) { + inline bool enqueue(const T& item) { if (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; return inner_enqueue(item); @@ -1019,7 +1019,7 @@ class ConcurrentQueue { // Allocates memory if required. Only fails if memory allocation fails (or // Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). // Thread-safe. - inline bool enqueue(producer_token_t const& token, T const& item) { + inline bool enqueue(const producer_token_t& token, const T& item) { return inner_enqueue(token, item); } @@ -1027,7 +1027,7 @@ class ConcurrentQueue { // Allocates memory if required. Only fails if memory allocation fails (or // Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). // Thread-safe. - inline bool enqueue(producer_token_t const& token, T&& item) { + inline bool enqueue(const producer_token_t& token, T&& item) { return inner_enqueue(token, std::move(item)); } @@ -1051,7 +1051,7 @@ class ConcurrentQueue { // instead of copied. // Thread-safe. template - bool enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) { + bool enqueue_bulk(const producer_token_t& token, It itemFirst, size_t count) { return inner_enqueue_bulk(token, itemFirst, count); } @@ -1060,7 +1060,7 @@ class ConcurrentQueue { // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE // is 0). // Thread-safe. - inline bool try_enqueue(T const& item) { + inline bool try_enqueue(const T& item) { if (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; return inner_enqueue(item); @@ -1080,14 +1080,14 @@ class ConcurrentQueue { // Enqueues a single item (by copying it) using an explicit producer token. // Does not allocate memory. Fails if not enough room to enqueue. // Thread-safe. - inline bool try_enqueue(producer_token_t const& token, T const& item) { + inline bool try_enqueue(const producer_token_t& token, const T& item) { return inner_enqueue(token, item); } // Enqueues a single item (by moving it, if possible) using an explicit producer token. // Does not allocate memory. Fails if not enough room to enqueue. // Thread-safe. - inline bool try_enqueue(producer_token_t const& token, T&& item) { + inline bool try_enqueue(const producer_token_t& token, T&& item) { return inner_enqueue(token, std::move(item)); } @@ -1111,7 +1111,7 @@ class ConcurrentQueue { // instead of copied. // Thread-safe. template - bool try_enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) { + bool try_enqueue_bulk(const producer_token_t& token, It itemFirst, size_t count) { return inner_enqueue_bulk(token, itemFirst, count); } @@ -1292,7 +1292,7 @@ class ConcurrentQueue { // was checked (so, the queue is likely but not guaranteed to be empty). // Never allocates. Thread-safe. template - inline bool try_dequeue_from_producer(producer_token_t const& producer, U& item) { + inline bool try_dequeue_from_producer(const producer_token_t& producer, U& item) { return static_cast(producer.producer)->dequeue(item); } @@ -1304,7 +1304,7 @@ class ConcurrentQueue { // was checked (so, the queue is likely but not guaranteed to be empty). // Never allocates. Thread-safe. template - inline size_t try_dequeue_bulk_from_producer(producer_token_t const& producer, It itemFirst, size_t max) { + inline size_t try_dequeue_bulk_from_producer(const producer_token_t& producer, It itemFirst, size_t max) { return static_cast(producer.producer)->dequeue_bulk(itemFirst, max); } @@ -1349,7 +1349,7 @@ class ConcurrentQueue { /////////////////////////////// template - inline bool inner_enqueue(producer_token_t const& token, U&& element) { + inline bool inner_enqueue(const producer_token_t& token, U&& element) { return static_cast(token.producer) ->ConcurrentQueue::ExplicitProducer::template enqueue(std::forward(element)); } @@ -1363,7 +1363,7 @@ class ConcurrentQueue { } template - inline bool inner_enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) { + inline bool inner_enqueue_bulk(const producer_token_t& token, It itemFirst, size_t count) { return static_cast(token.producer) ->ConcurrentQueue::ExplicitProducer::template enqueue_bulk(itemFirst, count); } @@ -1442,8 +1442,8 @@ class ConcurrentQueue { details::swap_relaxed(freeListHead, other.freeListHead); } - FreeList(FreeList const&) MOODYCAMEL_DELETE_FUNCTION; - FreeList& operator=(FreeList const&) MOODYCAMEL_DELETE_FUNCTION; + FreeList(const FreeList&) MOODYCAMEL_DELETE_FUNCTION; + FreeList& operator=(const FreeList&) MOODYCAMEL_DELETE_FUNCTION; inline void add(N* node) { #ifdef MCDBGQ_NOLOCKFREE_FREELIST @@ -1650,8 +1650,8 @@ class ConcurrentQueue { return static_cast(static_cast(elements)) + static_cast(idx & static_cast(BLOCK_SIZE - 1)); } - inline T const* operator[](index_t idx) const MOODYCAMEL_NOEXCEPT { - return static_cast(static_cast(elements)) + + inline const T* operator[](index_t idx) const MOODYCAMEL_NOEXCEPT { + return static_cast(static_cast(elements)) + static_cast(idx & static_cast(BLOCK_SIZE - 1)); } diff --git a/tdactor/example/actor-example-coroutines.cpp b/tdactor/example/actor-example-coroutines.cpp index a2d6beab79..d40d446404 100644 --- a/tdactor/example/actor-example-coroutines.cpp +++ b/tdactor/example/actor-example-coroutines.cpp @@ -351,4 +351,4 @@ int main() { scheduler.run(); LOG(INFO) << "DONE"; return 0; -} \ No newline at end of file +} diff --git a/tdactor/td/actor/coro.h b/tdactor/td/actor/coro.h index 1dbeef3fde..03cadf39a7 100644 --- a/tdactor/td/actor/coro.h +++ b/tdactor/td/actor/coro.h @@ -6,4 +6,4 @@ #include "td/actor/coro_types.h" #include "td/actor/coro_utils.h" -namespace td::actor {} // namespace td::actor \ No newline at end of file +namespace td::actor {} // namespace td::actor diff --git a/tdutils/td/utils/date.h b/tdutils/td/utils/date.h index 08f53edc72..a2d7ee8d8b 100644 --- a/tdutils/td/utils/date.h +++ b/tdutils/td/utils/date.h @@ -1376,14 +1376,14 @@ inline bool operator>=(const month& x, const month& y) NOEXCEPT { CONSTCD14 inline months operator-(const month& x, const month& y) NOEXCEPT { - auto const d = static_cast(x) - static_cast(y); + const auto d = static_cast(x) - static_cast(y); return months(d <= 11 ? d : d + 12); } CONSTCD14 inline month operator+(const month& x, const months& y) NOEXCEPT { - auto const mu = static_cast(static_cast(x)) + y.count() - 1; - auto const yr = (mu >= 0 ? mu : mu - 11) / 12; + const auto mu = static_cast(static_cast(x)) + y.count() - 1; + const auto yr = (mu >= 0 ? mu : mu - 11) / 12; return month{static_cast(mu - yr * 12 + 1)}; } @@ -1598,15 +1598,15 @@ inline bool operator!=(const weekday& x, const weekday& y) NOEXCEPT { CONSTCD14 inline days operator-(const weekday& x, const weekday& y) NOEXCEPT { - auto const wdu = x.wd_ - y.wd_; - auto const wk = (wdu >= 0 ? wdu : wdu - 6) / 7; + const auto wdu = x.wd_ - y.wd_; + const auto wk = (wdu >= 0 ? wdu : wdu - 6) / 7; return days{wdu - wk * 7}; } CONSTCD14 inline weekday operator+(const weekday& x, const days& y) NOEXCEPT { - auto const wdu = static_cast(static_cast(x.wd_)) + y.count(); - auto const wk = (wdu >= 0 ? wdu : wdu - 6) / 7; + const auto wdu = static_cast(static_cast(x.wd_)) + y.count(); + const auto wk = (wdu >= 0 ? wdu : wdu - 6) / 7; return weekday{static_cast(wdu - wk * 7)}; } @@ -2261,13 +2261,13 @@ inline days year_month_day::to_days() const NOEXCEPT { "This algorithm has not been ported to a 16 bit unsigned integer"); static_assert(std::numeric_limits::digits >= 20, "This algorithm has not been ported to a 16 bit signed integer"); - auto const y = static_cast(y_) - (m_ <= February); - auto const m = static_cast(m_); - auto const d = static_cast(d_); - auto const era = (y >= 0 ? y : y - 399) / 400; - auto const yoe = static_cast(y - era * 400); // [0, 399] - auto const doy = (153 * (m > 2 ? m - 3 : m + 9) + 2) / 5 + d - 1; // [0, 365] - auto const doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; // [0, 146096] + const auto y = static_cast(y_) - (m_ <= February); + const auto m = static_cast(m_); + const auto d = static_cast(d_); + const auto era = (y >= 0 ? y : y - 399) / 400; + const auto yoe = static_cast(y - era * 400); // [0, 399] + const auto doy = (153 * (m > 2 ? m - 3 : m + 9) + 2) / 5 + d - 1; // [0, 365] + const auto doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; // [0, 146096] return days{era * 146097 + static_cast(doe) - 719468}; } @@ -2343,15 +2343,15 @@ inline year_month_day year_month_day::from_days(days dp) NOEXCEPT { "This algorithm has not been ported to a 16 bit unsigned integer"); static_assert(std::numeric_limits::digits >= 20, "This algorithm has not been ported to a 16 bit signed integer"); - auto const z = dp.count() + 719468; - auto const era = (z >= 0 ? z : z - 146096) / 146097; - auto const doe = static_cast(z - era * 146097); // [0, 146096] - auto const yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; // [0, 399] - auto const y = static_cast(yoe) + era * 400; - auto const doy = doe - (365 * yoe + yoe / 4 - yoe / 100); // [0, 365] - auto const mp = (5 * doy + 2) / 153; // [0, 11] - auto const d = doy - (153 * mp + 2) / 5 + 1; // [1, 31] - auto const m = mp < 10 ? mp + 3 : mp - 9; // [1, 12] + const auto z = dp.count() + 719468; + const auto era = (z >= 0 ? z : z - 146096) / 146097; + const auto doe = static_cast(z - era * 146097); // [0, 146096] + const auto yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; // [0, 399] + const auto y = static_cast(yoe) + era * 400; + const auto doy = doe - (365 * yoe + yoe / 4 - yoe / 100); // [0, 365] + const auto mp = (5 * doy + 2) / 153; // [0, 11] + const auto d = doy - (153 * mp + 2) / 5 + 1; // [1, 31] + const auto m = mp < 10 ? mp + 3 : mp - 9; // [1, 12] return year_month_day{date::year{y + (m <= 2)}, date::month(m), date::day(d)}; } @@ -2473,8 +2473,8 @@ inline bool year_month_weekday::ok() const NOEXCEPT { CONSTCD14 inline year_month_weekday year_month_weekday::from_days(days d) NOEXCEPT { sys_days dp{d}; - auto const wd = date::weekday(dp); - auto const ymd = year_month_day(dp); + const auto wd = date::weekday(dp); + const auto ymd = year_month_day(dp); return {ymd.year(), ymd.month(), wd[(static_cast(ymd.day()) - 1) / 7 + 1]}; } @@ -2597,7 +2597,7 @@ inline bool year_month_weekday_last::ok() const NOEXCEPT { CONSTCD14 inline days year_month_weekday_last::to_days() const NOEXCEPT { - auto const d = sys_days(y_ / m_ / last); + const auto d = sys_days(y_ / m_ / last); return (d - (date::weekday{d} - wdl_.weekday())).time_since_epoch(); } @@ -3068,7 +3068,7 @@ class hh_mm_ss { private: template - friend std::basic_ostream& operator<<(std::basic_ostream& os, hh_mm_ss const& tod) { + friend std::basic_ostream& operator<<(std::basic_ostream& os, const hh_mm_ss& tod) { if (tod.is_negative()) os << '-'; if (tod.h_ < std::chrono::hours{10}) @@ -3092,12 +3092,12 @@ class hh_mm_ss { std::chrono::minutes* offset); }; -inline CONSTCD14 bool is_am(std::chrono::hours const& h) NOEXCEPT { +inline CONSTCD14 bool is_am(const std::chrono::hours& h) NOEXCEPT { using std::chrono::hours; return hours{0} <= h && h < hours{12}; } -inline CONSTCD14 bool is_pm(std::chrono::hours const& h) NOEXCEPT { +inline CONSTCD14 bool is_pm(const std::chrono::hours& h) NOEXCEPT { using std::chrono::hours; return hours{12} <= h && h < hours{24}; } @@ -3138,7 +3138,7 @@ inline typename std::enable_if::value, std::basic_ostream&>::type operator<<(std::basic_ostream& os, const sys_time& tp) { - auto const dp = date::floor(tp); + const auto dp = date::floor(tp); return os << year_month_day(dp) << ' ' << make_time(tp - dp); } @@ -3173,8 +3173,8 @@ class string_literal { public: using const_iterator = const CharT*; - string_literal(string_literal const&) = default; - string_literal& operator=(string_literal const&) = delete; + string_literal(const string_literal&) = default; + string_literal& operator=(const string_literal&) = delete; template ::type> CONSTCD11 string_literal(CharT c) NOEXCEPT : p_{c} { @@ -3200,7 +3200,7 @@ class string_literal { } template ::value>::type> - CONSTCD14 string_literal(string_literal const& a) NOEXCEPT : p_{} { + CONSTCD14 string_literal(const string_literal& a) NOEXCEPT : p_{} { for (std::size_t i = 0; i < N; ++i) p_[i] = a[i]; } @@ -3219,7 +3219,7 @@ class string_literal { return p_ + N - 1; } - CONSTCD11 CharT const& operator[](std::size_t n) const NOEXCEPT { + CONSTCD11 const CharT& operator[](std::size_t n) const NOEXCEPT { return p_[n]; } @@ -3757,7 +3757,7 @@ std::basic_ostream& to_stream(std::basic_ostream& os.setstate(std::ios::failbit); #if !ONLY_C_LOCALE tm = std::tm{}; - auto const& ymd = fds.ymd; + const auto& ymd = fds.ymd; auto ld = local_days(ymd); if (*fmt == 'c') { tm.tm_sec = static_cast(fds.tod.seconds().count()); @@ -3789,7 +3789,7 @@ std::basic_ostream& to_stream(std::basic_ostream& } else // *fmt == 'x' { - auto const& ymd = fds.ymd; + const auto& ymd = fds.ymd; save_ostream _(os); os.fill('0'); os.flags(std::ios::dec | std::ios::right); @@ -3884,7 +3884,7 @@ std::basic_ostream& to_stream(std::basic_ostream& if (modified == CharT{}) { if (!fds.ymd.ok()) os.setstate(std::ios::failbit); - auto const& ymd = fds.ymd; + const auto& ymd = fds.ymd; save_ostream _(os); os.fill('0'); os.flags(std::ios::dec | std::ios::right); @@ -3907,7 +3907,7 @@ std::basic_ostream& to_stream(std::basic_ostream& if (modified == CharT{}) { if (!fds.ymd.ok()) os.setstate(std::ios::failbit); - auto const& ymd = fds.ymd; + const auto& ymd = fds.ymd; save_ostream _(os); os.imbue(std::locale::classic()); os.fill('0'); @@ -4271,7 +4271,7 @@ std::basic_ostream& to_stream(std::basic_ostream& if (modified == CharT{'E'}) os << CharT{'%'} << modified << *fmt; else { - auto const& ymd = fds.ymd; + const auto& ymd = fds.ymd; if (!ymd.ok()) os.setstate(std::ios::failbit); auto ld = local_days(ymd); @@ -4332,7 +4332,7 @@ std::basic_ostream& to_stream(std::basic_ostream& #if !ONLY_C_LOCALE else if (modified == CharT{'O'}) { const CharT f[] = {'%', modified, *fmt}; - auto const& ymd = fds.ymd; + const auto& ymd = fds.ymd; tm.tm_year = static_cast(ymd.year()) - 1900; tm.tm_wday = static_cast(extract_weekday(os, fds)); if (os.fail()) @@ -4380,7 +4380,7 @@ std::basic_ostream& to_stream(std::basic_ostream& if (modified == CharT{'E'}) os << CharT{'%'} << modified << *fmt; else { - auto const& ymd = fds.ymd; + const auto& ymd = fds.ymd; if (!ymd.ok()) os.setstate(std::ios::failbit); auto ld = local_days(ymd); diff --git a/tdutils/td/utils/tests.h b/tdutils/td/utils/tests.h index de93f370cf..b8d9a15183 100644 --- a/tdutils/td/utils/tests.h +++ b/tdutils/td/utils/tests.h @@ -191,7 +191,7 @@ inline vector rand_split(Slice str) { namespace detail { -std::optional stringify(auto const &value) { +std::optional stringify(const auto &value) { if constexpr (requires(std::ostringstream builder) { builder << value; }) { std::ostringstream builder; builder << value; @@ -202,7 +202,7 @@ std::optional stringify(auto const &value) { return std::nullopt; } -inline std::optional check(bool condition, char const *msg) { +inline std::optional check(bool condition, const char *msg) { if (condition) { return std::nullopt; } @@ -210,7 +210,7 @@ inline std::optional check(bool condition, char const *msg) { return PSTRING() << "Expectation failed: " << msg << "!"; } -std::optional check_eq(auto const &a_value, auto const &b_value, char const *a_expr, char const *b_expr) { +std::optional check_eq(const auto &a_value, const auto &b_value, const char *a_expr, const char *b_expr) { if (a_value == b_value) { return std::nullopt; } diff --git a/tonlib/tonlib/TonlibClientWrapper.cpp b/tonlib/tonlib/TonlibClientWrapper.cpp index 8dff18749c..fb43fda2b0 100644 --- a/tonlib/tonlib/TonlibClientWrapper.cpp +++ b/tonlib/tonlib/TonlibClientWrapper.cpp @@ -67,4 +67,4 @@ void TonlibClientWrapper::receive_request_result(td::uint64 id, promise.set_result(std::move(R)); } -} // namespace tonlib \ No newline at end of file +} // namespace tonlib diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index d234397faf..6e21455069 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -362,11 +362,11 @@ namespace { FileReferenceShort create_persistent_state_id(BlockIdExt block_id, BlockIdExt mc_block_id, PersistentStateType type) { FileReferenceShort result; type.visit(td::overloaded( - [&](UnsplitStateType const &) { result = fileref::PersistentStateShort::create(block_id, mc_block_id); }, - [&](SplitAccountStateType const &account_state) { + [&](const UnsplitStateType &) { result = fileref::PersistentStateShort::create(block_id, mc_block_id); }, + [&](const SplitAccountStateType &account_state) { result = fileref::SplitAccountState::create(block_id, mc_block_id, account_state.effective_shard_id); }, - [&](SplitPersistentStateType const &persistent_state) { + [&](const SplitPersistentStateType &persistent_state) { result = fileref::SplitPersistentState::create(block_id, mc_block_id); })); return result; @@ -400,7 +400,7 @@ void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt ma } void ArchiveManager::add_persistent_state_impl( - FileReferenceShort const &id, td::Promise promise, + const FileReferenceShort &id, td::Promise promise, std::function)> create_writer) { if (perm_states_.find({id.seqno_of_persistent_state(), id.hash()}) != perm_states_.end()) { promise.set_value(td::Unit()); diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index 247e0a99b3..f651661a24 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -223,7 +223,7 @@ class ArchiveManager : public td::actor::Actor { PackageId get_max_temp_file_desc_idx(); PackageId get_prev_temp_file_desc_idx(PackageId id); - void add_persistent_state_impl(FileReferenceShort const &id, td::Promise promise, + void add_persistent_state_impl(const FileReferenceShort &id, td::Promise promise, std::function)> create_writer); void register_perm_state(FileReferenceShort id); diff --git a/validator/db/db-utils.cpp b/validator/db/db-utils.cpp index b7f2f849e5..6ea99d6e60 100644 --- a/validator/db/db-utils.cpp +++ b/validator/db/db-utils.cpp @@ -51,4 +51,4 @@ void PercentileStats::clear() { values_.clear(); } -} // namespace ton::validator \ No newline at end of file +} // namespace ton::validator diff --git a/validator/db/db-utils.h b/validator/db/db-utils.h index 8ffbf5c97c..d6e1e8fcdd 100644 --- a/validator/db/db-utils.h +++ b/validator/db/db-utils.h @@ -30,4 +30,4 @@ class PercentileStats { std::multiset values_; }; -} // namespace ton::validator \ No newline at end of file +} // namespace ton::validator diff --git a/validator/db/statedb.cpp b/validator/db/statedb.cpp index 6c5d87ce42..c5bef78a81 100644 --- a/validator/db/statedb.cpp +++ b/validator/db/statedb.cpp @@ -279,7 +279,7 @@ void StateDb::add_persistent_state_description(td::Reflist_.resize(new_size); bool can_be_stored_as_v1 = true; - for (auto const& [block_id, split_depth] : desc->shard_blocks) { + for (const auto& [block_id, split_depth] : desc->shard_blocks) { if (split_depth != 0) { can_be_stored_as_v1 = false; break; @@ -290,14 +290,14 @@ void StateDb::add_persistent_state_description(td::Ref> shard_blocks; - for (auto const& [block_id, _] : desc->shard_blocks) { + for (const auto& [block_id, _] : desc->shard_blocks) { shard_blocks.push_back(create_tl_block_id(block_id)); } serialized_shards = create_serialize_tl_object(std::move(shard_blocks)); } else { std::vector> shard_blocks; - for (auto const& [block_id, split_depth] : desc->shard_blocks) { + for (const auto& [block_id, split_depth] : desc->shard_blocks) { shard_blocks.push_back(create_tl_object( create_tl_block_id(block_id), static_cast(split_depth))); } @@ -350,16 +350,16 @@ void StateDb::get_persistent_state_descriptions(td::Promiseblock_), .split_depth = static_cast(shard_description->split_depth_), diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index 866899c37f..74f4dba9b2 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -86,14 +86,14 @@ class SplitStateDeserializer { } return parts; - } catch (vm::VmVirtError const&) { + } catch (const vm::VmVirtError &) { return td::Status::Error("Insufficient number of cells in split state header"); } } - td::Ref merge(std::vector> const& parts) { + td::Ref merge(const std::vector> &parts) { vm::AugmentedDictionary accounts{256, block::tlb::aug_ShardAccounts}; - for (auto const& part_root : parts) { + for (const auto &part_root : parts) { vm::AugmentedDictionary part{ vm::load_cell_slice_ref(part_root), 256, diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index 2631f00730..0769c086b5 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -129,7 +129,7 @@ void WaitBlockState::start() { BlockIdExt masterchain_id = persistent_state_desc_->masterchain_id; td::uint32 split_depth = 0; bool block_found = false; - for (auto const& [block, block_split_depth] : persistent_state_desc_->shard_blocks) { + for (const auto& [block, block_split_depth] : persistent_state_desc_->shard_blocks) { if (block == handle_->id()) { split_depth = block_split_depth; block_found = true; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 2d2954747d..19b3a127ac 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -3528,7 +3528,7 @@ td::Result> Collator::impl_crea if (!trans->compute_phase->accepted) { if (external) { // inbound external message was not accepted - auto const& cp = *trans->compute_phase; + const auto& cp = *trans->compute_phase; return td::Status::Error( -701, PSLICE() << "inbound external message rejected by transaction " << acc->addr.to_hex() << ":\n" << "exitcode=" << cp.exit_code << ", steps=" << cp.vm_steps << ", gas_used=" << cp.gas_used diff --git a/validator/interfaces/liteserver.h b/validator/interfaces/liteserver.h index cbf4cf85ff..74049d1d2a 100644 --- a/validator/interfaces/liteserver.h +++ b/validator/interfaces/liteserver.h @@ -35,4 +35,4 @@ class LiteServerCache : public td::actor::Actor { virtual void drop_send_message_from_cache(td::Bits256 key) = 0; }; -} // namespace ton::validator \ No newline at end of file +} // namespace ton::validator diff --git a/validator/interfaces/persistent-state.h b/validator/interfaces/persistent-state.h index 74b5100965..62785dd572 100644 --- a/validator/interfaces/persistent-state.h +++ b/validator/interfaces/persistent-state.h @@ -36,13 +36,13 @@ struct SplitPersistentStateType {}; using PersistentStateType = td::Variant; -auto persistent_state_id_from_v1_query(auto const &query) { +auto persistent_state_id_from_v1_query(const auto &query) { auto block = create_tl_block_id(create_block_id(query.block_)); auto mc_block = create_tl_block_id(create_block_id(query.masterchain_block_)); return create_tl_object(std::move(block), std::move(mc_block), 0); } -auto persistent_state_from_v2_query(auto const &query) { +auto persistent_state_from_v2_query(const auto &query) { auto block = create_block_id(query.state_->block_); auto mc_block = create_block_id(query.state_->masterchain_block_); ShardId effective_shard = static_cast(query.state_->effective_shard_); @@ -61,7 +61,7 @@ auto persistent_state_from_v2_query(auto const &query) { return std::tuple{block, mc_block, PersistentStateType{SplitAccountStateType{effective_shard}}}; } -inline ShardId persistent_state_to_effective_shard(ShardIdFull const &shard, PersistentStateType const &type) { +inline ShardId persistent_state_to_effective_shard(const ShardIdFull &shard, const PersistentStateType &type) { ShardId result = 0; type.visit(td::overloaded([](UnsplitStateType) {}, [&](SplitAccountStateType type) { result = type.effective_shard_id; }, @@ -69,7 +69,7 @@ inline ShardId persistent_state_to_effective_shard(ShardIdFull const &shard, Per return result; } -inline std::string persistent_state_type_to_string(ShardIdFull const &shard, PersistentStateType const &state) { +inline std::string persistent_state_type_to_string(const ShardIdFull &shard, const PersistentStateType &state) { std::string result; state.visit(td::overloaded([&](UnsplitStateType) { result = "unsplit"; }, [&](SplitAccountStateType type) { diff --git a/validator/manager.cpp b/validator/manager.cpp index 0cd65a05e0..69d5bee524 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3447,7 +3447,7 @@ void ValidatorManagerImpl::add_persistent_state_description(td::Refsecond; if (prev_desc->end_time <= now) { - for (auto const &[block_id, _] : prev_desc->shard_blocks) { + for (const auto &[block_id, _] : prev_desc->shard_blocks) { persistent_state_blocks_.erase(block_id); } it = persistent_state_descriptions_.erase(it); @@ -3464,7 +3464,7 @@ void ValidatorManagerImpl::add_persistent_state_description_impl(td::Refmasterchain_id.to_str() << " start_time=" << desc->start_time << " end_time=" << desc->end_time; - for (auto const &[block_id, _] : desc->shard_blocks) { + for (const auto &[block_id, _] : desc->shard_blocks) { persistent_state_blocks_[block_id] = desc; LOG(DEBUG) << "Persistent state description: shard block " << block_id.to_str(); } diff --git a/validator/queue-size-counter.cpp b/validator/queue-size-counter.cpp index 59d3a13039..4cf4f05051 100644 --- a/validator/queue-size-counter.cpp +++ b/validator/queue-size-counter.cpp @@ -301,4 +301,4 @@ void QueueSizeCounter::alarm() { alarm_timestamp() = td::Timestamp::in(td::Random::fast(20.0, 40.0)); } -} // namespace ton::validator \ No newline at end of file +} // namespace ton::validator diff --git a/validator/rate-limiter.h b/validator/rate-limiter.h index a0355fa554..345d4a2046 100644 --- a/validator/rate-limiter.h +++ b/validator/rate-limiter.h @@ -111,4 +111,4 @@ void RateLimiter::insert(RequestID request, td::Timestamp time) { } } -} // namespace ton::validator::fullnode \ No newline at end of file +} // namespace ton::validator::fullnode diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index 1c577b663f..a522144215 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -391,7 +391,7 @@ void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard, cache = std::make_shared(std::move(cells)); } -void AsyncStateSerializer::PreviousStateCache::add_new_cells(vm::CellDbReader& reader, Ref const& cell) { +void AsyncStateSerializer::PreviousStateCache::add_new_cells(vm::CellDbReader& reader, const Ref& cell) { if (!cell->is_loaded()) { return; } diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index e02a2e66c7..6fd7c43a0d 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -68,7 +68,7 @@ class AsyncStateSerializer : public td::actor::Actor { std::vector cur_shards; void prepare_cache(ShardIdFull shard, PersistentStateType type); - void add_new_cells(vm::CellDbReader& reader, Ref const& cell); + void add_new_cells(vm::CellDbReader& reader, const Ref& cell); void cleanup_big_state_files(Ref mc_state); }; std::shared_ptr previous_state_cache_; From 95747d3026e04663cb62af7f7e6e18d4e4f2ab32 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 15 Dec 2025 18:48:43 +0300 Subject: [PATCH 510/524] Remove "proto_version" from deserialize_candidate (#1956) --- validator-session/candidate-serializer.cpp | 19 ++-- validator-session/candidate-serializer.h | 11 +- validator-session/validator-session.cpp | 3 +- validator/collation-manager.cpp | 16 ++- validator/collation-manager.hpp | 7 +- .../collator-node/collator-node-session.cpp | 3 +- validator/collator-node/utils.cpp | 100 +++++++++--------- validator/collator-node/utils.hpp | 2 +- validator/impl/collator.cpp | 3 +- validator/validator-group.cpp | 4 +- 10 files changed, 81 insertions(+), 87 deletions(-) diff --git a/validator-session/candidate-serializer.cpp b/validator-session/candidate-serializer.cpp index 8a4cfb6a28..fa02e9c66f 100644 --- a/validator-session/candidate-serializer.cpp +++ b/validator-session/candidate-serializer.cpp @@ -45,8 +45,7 @@ td::Result serialize_candidate(const tl_object_ptr> deserialize_candidate(td::Slice data, bool compression_enabled, - int max_decompressed_data_size, - int proto_version) { + int max_decompressed_data_size) { if (!compression_enabled) { auto t_decompression_start = td::Time::now(); TRY_RESULT(res, fetch_tl_object(data, true)); @@ -68,7 +67,7 @@ td::Result> deserialize_candi return td::Status::Error("decompressed size is too big"); } TRY_RESULT(p, decompress_candidate_data(c.data_, false, c.decompressed_size_, - max_decompressed_data_size, proto_version, c.root_hash_)); + max_decompressed_data_size, c.root_hash_)); return create_tl_object(c.src_, c.round_, c.root_hash_, std::move(p.first), std::move(p.second)); }(); @@ -78,8 +77,7 @@ td::Result> deserialize_candi if (c.data_.size() > max_decompressed_data_size) { return td::Status::Error("Compressed data is too big"); } - TRY_RESULT(p, decompress_candidate_data(c.data_, true, 0, max_decompressed_data_size, proto_version, - c.root_hash_)); + TRY_RESULT(p, decompress_candidate_data(c.data_, true, 0, max_decompressed_data_size, c.root_hash_)); return create_tl_object(c.src_, c.round_, c.root_hash_, std::move(p.first), std::move(p.second)); }(); @@ -110,9 +108,11 @@ td::Result compress_candidate_data(td::Slice block, td::Slice c return compressed; } -td::Result> decompress_candidate_data( - td::Slice compressed, bool improved_compression, int decompressed_size, int max_decompressed_size, - int proto_version, td::Bits256 root_hash) { +td::Result> decompress_candidate_data(td::Slice compressed, + bool improved_compression, + int decompressed_size, + int max_decompressed_size, + td::Bits256 root_hash) { std::vector> roots; auto t_decompression_start = td::Time::now(); if (!improved_compression) { @@ -135,8 +135,7 @@ td::Result> decompress_candidate_dat } TRY_RESULT(block_data, vm::std_boc_serialize(roots[0], 31)); roots.erase(roots.begin()); - int collated_data_mode = proto_version >= 5 ? 2 : 31; - TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), collated_data_mode)); + TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), 2)); LOG(DEBUG) << "Decompressing block candidate " << (improved_compression ? "V2:" : ":") << compressed.size() << " -> " << block_data.size() + collated_data.size(); return std::make_pair(std::move(block_data), std::move(collated_data)); diff --git a/validator-session/candidate-serializer.h b/validator-session/candidate-serializer.h index 75dc1c6cf8..60a680f48b 100644 --- a/validator-session/candidate-serializer.h +++ b/validator-session/candidate-serializer.h @@ -24,13 +24,14 @@ td::Result serialize_candidate(const tl_object_ptr> deserialize_candidate(td::Slice data, bool compression_enabled, - int max_decompressed_data_size, - int proto_version); + int max_decompressed_data_size); td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, size_t& decompressed_size, td::Bits256 root_hash); -td::Result> decompress_candidate_data( - td::Slice compressed, bool improved_compression, int decompressed_size, int max_decompressed_size, - int proto_version, td::Bits256 root_hash); +td::Result> decompress_candidate_data(td::Slice compressed, + bool improved_compression, + int decompressed_size, + int max_decompressed_size, + td::Bits256 root_hash); } // namespace ton::validatorsession diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 87ae1e0825..78532bfb5c 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -242,8 +242,7 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice td::Timer deserialize_timer; auto R = deserialize_candidate(data, compress_block_candidates_, - description().opts().max_block_size + description().opts().max_collated_data_size + 1024, - description().opts().proto_version); + description().opts().max_block_size + description().opts().max_collated_data_size + 1024); double deserialize_time = deserialize_timer.elapsed(); if (R.is_error()) { VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index a2127ab6a1..6f5e80288c 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -59,7 +59,7 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha std::vector prev, Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, - td::Promise promise, int proto_version) { + td::Promise promise) { if (shard.is_masterchain()) { run_collate_query( CollateParams{.shard = shard, @@ -74,8 +74,7 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha return; } collate_shard_block(shard, min_masterchain_block_id, std::move(prev), creator, priority, std::move(validator_set), - max_answer_size, std::move(cancellation_token), std::move(promise), td::Timestamp::in(10.0), - proto_version); + max_answer_size, std::move(cancellation_token), std::move(promise), td::Timestamp::in(10.0)); } void CollationManager::collate_block_optimistic(ShardIdFull shard, BlockIdExt min_masterchain_block_id, @@ -83,7 +82,7 @@ void CollationManager::collate_block_optimistic(ShardIdFull shard, BlockIdExt mi Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, - td::Promise promise, int proto_version) { + td::Promise promise) { if (shard.is_masterchain()) { TRY_RESULT_PROMISE(promise, prev_block_data, create_block(prev_block_id, std::move(prev_block))); run_collate_query( @@ -118,7 +117,7 @@ void CollationManager::collate_block_optimistic(ShardIdFull shard, BlockIdExt mi collate_shard_block(shard, min_masterchain_block_id, {prev_block_id}, creator, priority, std::move(validator_set), max_answer_size, std::move(cancellation_token), std::move(promise), td::Timestamp::in(10.0), - proto_version, true); + true); } void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, @@ -126,7 +125,7 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise, td::Timestamp timeout, - int proto_version, bool is_optimistic) { + bool is_optimistic) { TRY_STATUS_PROMISE(promise, cancellation_token.check()); ShardInfo* s = select_shard_info(shard); if (s == nullptr) { @@ -246,7 +245,7 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas [=, promise = std::move(promise)]() mutable { td::actor::send_closure(SelfId, &CollationManager::collate_shard_block, shard, min_masterchain_block_id, prev, creator, priority, validator_set, max_answer_size, cancellation_token, - std::move(promise), timeout, proto_version, is_optimistic); + std::move(promise), timeout, is_optimistic); }, retry_at); }; @@ -279,8 +278,7 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas return; } TRY_RESULT_PROMISE(P, f, fetch_tl_object(data, true)); - TRY_RESULT_PROMISE(P, candidate, - deserialize_candidate(std::move(f), td::narrow_cast(max_answer_size), proto_version)); + TRY_RESULT_PROMISE(P, candidate, deserialize_candidate(std::move(f), td::narrow_cast(max_answer_size))); if (candidate.pubkey.as_bits256() != creator.as_bits256()) { P.set_error(td::Status::Error("collate query: block candidate source mismatch")); return; diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp index 387df2d206..184432dbc1 100644 --- a/validator/collation-manager.hpp +++ b/validator/collation-manager.hpp @@ -40,13 +40,12 @@ class CollationManager : public td::actor::Actor { void collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, - td::Promise promise, int proto_version); + td::Promise promise); void collate_block_optimistic(ShardIdFull shard, BlockIdExt min_masterchain_block_id, BlockIdExt prev_block_id, td::BufferSlice prev_block, Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, - td::CancellationToken cancellation_token, td::Promise promise, - int proto_version); + td::CancellationToken cancellation_token, td::Promise promise); void update_options(td::Ref opts); @@ -68,7 +67,7 @@ class CollationManager : public td::actor::Actor { Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise, - td::Timestamp timeout, int proto_version, bool is_optimistic = false); + td::Timestamp timeout, bool is_optimistic = false); void update_collators_list(const CollatorsList& collators_list); diff --git a/validator/collator-node/collator-node-session.cpp b/validator/collator-node/collator-node-session.cpp index 11838de9df..d884d9b7db 100644 --- a/validator/collator-node/collator-node-session.cpp +++ b/validator/collator-node/collator-node-session.cpp @@ -293,8 +293,7 @@ void CollatorNodeSession::process_request_optimistic_cont2(BlockIdExt prev_block "failed to download prev block data for optimistic collation: "); TRY_RESULT_PROMISE_PREFIX(promise, f, fetch_tl_object(response, true), "failed to download prev block data for optimistic collation: "); - TRY_RESULT_PROMISE_PREFIX(promise, candidate, - deserialize_candidate(std::move(f), max_candidate_size_, proto_version_), + TRY_RESULT_PROMISE_PREFIX(promise, candidate, deserialize_candidate(std::move(f), max_candidate_size_), "failed to download prev block data for optimistic collation: "); TRY_RESULT_PROMISE_PREFIX(promise, prev_block, create_block(prev_block_id, std::move(candidate.data)), "invalid prev block data from validator: "); diff --git a/validator/collator-node/utils.cpp b/validator/collator-node/utils.cpp index 26b1095d40..bbec460865 100644 --- a/validator/collator-node/utils.cpp +++ b/validator/collator-node/utils.cpp @@ -39,57 +39,57 @@ tl_object_ptr serialize_candidate(const BlockCa } td::Result deserialize_candidate(tl_object_ptr f, - int max_decompressed_data_size, int proto_version) { + int max_decompressed_data_size) { td::Result res; - ton_api::downcast_call(*f, td::overloaded( - [&](ton_api::collatorNode_candidate& c) { - res = [&]() -> td::Result { - auto hash = td::sha256_bits256(c.collated_data_); - auto key = PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), hash, std::move(c.data_), - std::move(c.collated_data_)}; - }(); - }, - [&](ton_api::collatorNode_compressedCandidate& c) { - res = [&]() -> td::Result { - if (c.decompressed_size_ <= 0) { - return td::Status::Error("invalid decompressed size"); - } - if (c.decompressed_size_ > max_decompressed_data_size) { - return td::Status::Error("decompressed size is too big"); - } - TRY_RESULT(p, validatorsession::decompress_candidate_data( - c.data_, false, c.decompressed_size_, max_decompressed_data_size, - proto_version, create_block_id(c.id_).root_hash)); - auto collated_data_hash = td::sha256_bits256(p.second); - auto key = PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, - std::move(p.first), std::move(p.second)}; - }(); - }, - [&](ton_api::collatorNode_compressedCandidateV2& c) { - res = [&]() -> td::Result { - TRY_RESULT(p, validatorsession::decompress_candidate_data( - c.data_, true, 0, max_decompressed_data_size, proto_version, - create_block_id(c.id_).root_hash)); - auto collated_data_hash = td::sha256_bits256(p.second); - auto key = PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, - std::move(p.first), std::move(p.second)}; - }(); - })); + ton_api::downcast_call( + *f, td::overloaded( + [&](ton_api::collatorNode_candidate& c) { + res = [&]() -> td::Result { + auto hash = td::sha256_bits256(c.collated_data_); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), hash, std::move(c.data_), + std::move(c.collated_data_)}; + }(); + }, + [&](ton_api::collatorNode_compressedCandidate& c) { + res = [&]() -> td::Result { + if (c.decompressed_size_ <= 0) { + return td::Status::Error("invalid decompressed size"); + } + if (c.decompressed_size_ > max_decompressed_data_size) { + return td::Status::Error("decompressed size is too big"); + } + TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, false, c.decompressed_size_, + max_decompressed_data_size, + create_block_id(c.id_).root_hash)); + auto collated_data_hash = td::sha256_bits256(p.second); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), + std::move(p.second)}; + }(); + }, + [&](ton_api::collatorNode_compressedCandidateV2& c) { + res = [&]() -> td::Result { + TRY_RESULT(p, validatorsession::decompress_candidate_data( + c.data_, true, 0, max_decompressed_data_size, create_block_id(c.id_).root_hash)); + auto collated_data_hash = td::sha256_bits256(p.second); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), + std::move(p.second)}; + }(); + })); return res; } diff --git a/validator/collator-node/utils.hpp b/validator/collator-node/utils.hpp index 8ceba01f22..fb26981c8e 100644 --- a/validator/collator-node/utils.hpp +++ b/validator/collator-node/utils.hpp @@ -23,5 +23,5 @@ namespace ton::validator { tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress); td::Result deserialize_candidate(tl_object_ptr f, - int max_decompressed_data_size, int proto_version); + int max_decompressed_data_size); } // namespace ton::validator diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 19b3a127ac..1be9ec0593 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -6411,8 +6411,7 @@ bool Collator::create_block_candidate() { if (res.is_error()) { return fatal_error(res.move_as_error()); } - int cdata_serialize_mode = consensus_config.proto_version >= 5 ? 2 : 31; - auto cdata_res = boc_collated.serialize_to_slice(cdata_serialize_mode); + auto cdata_res = boc_collated.serialize_to_slice(2); if (cdata_res.is_error()) { LOG(ERROR) << "cannot serialize collated data"; return fatal_error(cdata_res.move_as_error()); diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 6e46211074..2e796f31b5 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -99,7 +99,7 @@ void ValidatorGroup::generate_block_candidate_cont(validatorsession::BlockSource td::actor::send_closure(collation_manager_, &CollationManager::collate_block, shard_, min_masterchain_block_id_, prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, source_info.priority, validator_set_, max_answer_size, std::move(cancellation_token), - std::move(promise), config_.proto_version); + std::move(promise)); } void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo source_info, @@ -394,7 +394,7 @@ void ValidatorGroup::generate_block_optimistic(validatorsession::BlockSourceInfo min_masterchain_block_id_, block_id, std::move(prev_block), Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, source_info.priority, validator_set_, max_answer_size, optimistic_generation_->cancellation_token_source.get_cancellation_token(), - std::move(P), config_.proto_version); + std::move(P)); } void ValidatorGroup::generated_block_optimistic(validatorsession::BlockSourceInfo source_info, From 9a3a3108670b0f3e5377a4f10ab666cf91497006 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 16 Dec 2025 07:41:00 +0300 Subject: [PATCH 511/524] Check that node is synced in new_external_message_broadcast (#1957) --- validator/manager.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/validator/manager.cpp b/validator/manager.cpp index 69d5bee524..296a7ba451 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -397,6 +397,9 @@ void ValidatorManagerImpl::get_key_block_proof_link(BlockIdExt block_id, td::Pro } td::actor::Task<> ValidatorManagerImpl::new_external_message_broadcast(td::BufferSlice data, int priority) { + if (!started_) { + co_return td::Status::Error(ErrorCode::notready, "node not synced"); + } auto r_check_result = co_await td::actor::ask(ext_message_pool_, &ExtMessagePool::check_add_external_message, std::move(data), priority, /* add_to_mempool = */ is_validator() || !collator_nodes_.empty()) From 5a0ccfea1329693e5c6bd8ae38d338bd17d652fb Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Thu, 18 Dec 2025 01:26:28 -0500 Subject: [PATCH 512/524] Make clangd highlight invalid td::actor::ask calls with a wiggly line (#1969) Co-authored-by: birydrad <> --- tdactor/td/actor/coro_utils.h | 50 ++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/tdactor/td/actor/coro_utils.h b/tdactor/td/actor/coro_utils.h index b0da730168..2faa1b54ab 100644 --- a/tdactor/td/actor/coro_utils.h +++ b/tdactor/td/actor/coro_utils.h @@ -165,6 +165,21 @@ struct unified_result { static constexpr UnifiedKind kind = is_promise ? UnifiedKind::PromiseArgument : UnifiedKind::Void; }; +// Concept: arguments must match the member function signature +template +concept AskArgsValid = [] { + using Meta = unified_result; + using C = detail::memfn_meta>::cls; + + if constexpr (Meta::kind == UnifiedKind::PromiseArgument) { + using T = Meta::type; + using P = StartedTask, td::Unit, T>>::ExternalPromise; + return std::is_invocable_v; + } else { + return std::is_invocable_v; + } +}(); + template auto ask_impl(TargetId&& to, MemFn mf, Args&&... args) { using Meta = unified_result; @@ -218,29 +233,34 @@ auto ask_new_impl(TargetId&& to, MemFn mf, Args&&... args) { } } -template -auto ask_new(Args&&... args) { - return ask_new_impl(std::forward(args)...); +template + requires AskArgsValid +auto ask_new(TargetId&& to, MemFn mf, Args&&... args) { + return ask_new_impl(std::forward(to), mf, std::forward(args)...); } -template -auto ask_new_immediate(Args&&... args) { - return ask_new_impl(std::forward(args)...); +template + requires AskArgsValid +auto ask_new_immediate(TargetId&& to, MemFn mf, Args&&... args) { + return ask_new_impl(std::forward(to), mf, std::forward(args)...); } -template -auto ask(Args&&... args) { - return ask_impl(std::forward(args)...); +template + requires AskArgsValid +auto ask(TargetId&& to, MemFn mf, Args&&... args) { + return ask_impl(std::forward(to), mf, std::forward(args)...); } -template -auto ask_immediate(Args&&... args) { - return ask_impl(std::forward(args)...); +template + requires AskArgsValid +auto ask_immediate(TargetId&& to, MemFn mf, Args&&... args) { + return ask_impl(std::forward(to), mf, std::forward(args)...); } -template -auto ask_promise(Args&&... args) { - return ask(std::forward(args)...); +template + requires AskArgsValid +auto ask_promise(TargetId&& to, MemFn mf, Args&&... args) { + return ask(std::forward(to), mf, std::forward(args)...); } template From 64128aca06e3f3d9f53166981fd6770bb1de74ce Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Thu, 18 Dec 2025 17:17:02 +0300 Subject: [PATCH 513/524] Add version check for source address validation (#1958) * Add version check for source address validation * fix formatting --- crypto/block/transaction.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index cb789ea427..577435f102 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -2710,6 +2710,9 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, // it must be either our source address, or empty if (!check_replace_src_addr(info.src)) { LOG(DEBUG) << "invalid source address in a proposed outbound message"; + if (cfg.global_version >= 13) { + return check_skip_invalid(35); + } return 35; // invalid source address } bool to_mc = false; From ff037b54a03b5d440eb8584412d2b0dd38af1fa6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 19 Dec 2025 15:09:48 +0300 Subject: [PATCH 514/524] Improve accessing nonfinal blocks through LS (#1941) * Improve accessing nonfinal blocks through LS * Fix formatting * Send shard block descriptions to custom overlays * Publish db.event.blockApplied for masterchain when all shards are ready * Fix incorrect merge * Add newline at end of db-event-publisher.hpp Added a newline at the end of the file for consistency. --------- Co-authored-by: EmelyanenkoK --- lite-client/query-utils.cpp | 1 + tl/generate/scheme/lite_api.tl | 2 + tl/generate/scheme/ton_api.tl | 6 +- validator-engine/validator-engine.cpp | 4 + validator-engine/validator-engine.hpp | 4 + validator/CMakeLists.txt | 2 + validator/db/db-event-publisher.cpp | 143 +++++++++++++ validator/db/db-event-publisher.hpp | 51 +++++ validator/full-node-custom-overlays.cpp | 22 ++ validator/full-node-custom-overlays.hpp | 2 + validator/full-node-fast-sync-overlays.cpp | 4 +- validator/full-node-shard.cpp | 4 +- validator/full-node.cpp | 49 +++-- validator/full-node.h | 2 + validator/full-node.hpp | 10 +- validator/impl/CMakeLists.txt | 2 - validator/impl/candidates-buffer.cpp | 213 -------------------- validator/impl/candidates-buffer.hpp | 64 ------ validator/impl/liteserver.cpp | 23 +++ validator/impl/liteserver.hpp | 1 + validator/impl/shard.cpp | 7 + validator/impl/shard.hpp | 1 + validator/interfaces/shard.h | 1 + validator/interfaces/validator-manager.h | 5 + validator/manager-disk.hpp | 2 - validator/manager-hardfork.hpp | 3 - validator/manager.cpp | 224 ++++++++++++++------- validator/manager.hpp | 25 ++- validator/validator-options.hpp | 7 + validator/validator.h | 8 +- 30 files changed, 511 insertions(+), 381 deletions(-) create mode 100644 validator/db/db-event-publisher.cpp create mode 100644 validator/db/db-event-publisher.hpp delete mode 100644 validator/impl/candidates-buffer.cpp delete mode 100644 validator/impl/candidates-buffer.hpp diff --git a/lite-client/query-utils.cpp b/lite-client/query-utils.cpp index 53e6d3732d..5b53a8166c 100644 --- a/lite-client/query-utils.cpp +++ b/lite-client/query-utils.cpp @@ -210,6 +210,7 @@ QueryInfo get_query_info(const lite_api::Function& f) { [&](const lite_api::liteServer_getShardBlockProof& q) { from_block_id(q.id_); }, [&](const lite_api::liteServer_nonfinal_getCandidate& q) { /* t_simple */ }, [&](const lite_api::liteServer_nonfinal_getValidatorGroups& q) { /* t_simple */ }, + [&](const lite_api::liteServer_nonfinal_getPendingShardBlocks& q) { /* t_simple */ }, [&](const lite_api::liteServer_getOutMsgQueueSizes& q) { // This query is expected to be removed, as it is not fully compatible with separated liteservers /* t_simple */ diff --git a/tl/generate/scheme/lite_api.tl b/tl/generate/scheme/lite_api.tl index b00951da40..a9adfa8453 100644 --- a/tl/generate/scheme/lite_api.tl +++ b/tl/generate/scheme/lite_api.tl @@ -74,6 +74,7 @@ liteServer.nonfinal.candidate id:liteServer.nonfinal.candidateId data:bytes coll liteServer.nonfinal.candidateInfo id:liteServer.nonfinal.candidateId available:Bool approved_weight:long signed_weight:long total_weight:long = liteServer.nonfinal.CandidateInfo; liteServer.nonfinal.validatorGroupInfo next_block_id:tonNode.blockId cc_seqno:int prev:(vector tonNode.blockIdExt) candidates:(vector liteServer.nonfinal.candidateInfo) = liteServer.nonfinal.ValidatorGroupInfo; liteServer.nonfinal.validatorGroups groups:(vector liteServer.nonfinal.validatorGroupInfo) = liteServer.nonfinal.ValidatorGroups; +liteServer.nonfinal.pendingShardBlocks signed_blocks:(vector tonNode.blockIdExt) candidates:(vector tonNode.blockIdExt) = liteServer.nonfinal.PendingShardBlocks; ---functions--- @@ -111,6 +112,7 @@ liteServer.getDispatchQueueMessages mode:# id:tonNode.blockIdExt addr:int256 aft liteServer.nonfinal.getValidatorGroups mode:# wc:mode.0?int shard:mode.0?long = liteServer.nonfinal.ValidatorGroups; liteServer.nonfinal.getCandidate id:liteServer.nonfinal.candidateId = liteServer.nonfinal.Candidate; +liteServer.nonfinal.getPendingShardBlocks mode:# wc:mode.0?int shard:mode.0?long = liteServer.nonfinal.PendingShardBlocks; liteServer.queryPrefix = Object; liteServer.query data:bytes = Object; diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 345623130c..1cb7ac6ef0 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -610,7 +610,11 @@ db.files.package.key package_id:int key:Bool temp:Bool = db.files.Key; db.files.index.value packages:(vector int) key_packages:(vector int) temp_packages:(vector int) = db.files.index.Value; db.files.package.firstBlock workchain:int shard:long seqno:int unixtime:int lt:long = db.files.package.FirstBlock; db.files.package.value package_id:int key:Bool temp:Bool firstblocks:(vector db.files.package.firstBlock) deleted:Bool - = db.files.package.Value; + = db.files.package.Value; + +db.event.blockApplied block_id:tonNode.blockIdExt = db.Event; +db.event.blockSigned block_id:tonNode.blockIdExt = db.Event; +db.event.blockCandidateReceived block_id:tonNode.blockIdExt = db.Event; ---functions--- diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index cfe51f052e..0d00a07d57 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1639,6 +1639,7 @@ td::Status ValidatorEngine::load_global_config() { validator_options_.write().set_hardforks(std::move(h)); validator_options_.write().set_catchain_broadcast_speed_multiplier(broadcast_speed_multiplier_catchain_); validator_options_.write().set_parallel_validation(parallel_validation_); + validator_options_.write().set_db_event_fifo_path(db_event_fifo_path_); for (auto &id : config_.collator_node_whitelist) { validator_options_.write().set_collator_node_whitelisted_validator(id, true); @@ -5605,6 +5606,9 @@ int main(int argc, char *argv[]) { p.add_option('\0', "parallel-validation", "parallel validation over different accounts", [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_parallel_validation, true); }); }); + p.add_option('\0', "db-event-fifo", "path to FIFO pipe for publishing DB events", [&](td::Slice s) { + acts.push_back([&x, s = s.str()]() { td::actor::send_closure(x, &ValidatorEngine::set_db_event_fifo_path, s); }); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 074dfcc973..e16e6f315b 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -256,6 +256,7 @@ class ValidatorEngine : public td::actor::Actor { ton::adnl::AdnlNodeIdShort shard_block_retainer_adnl_id_ = ton::adnl::AdnlNodeIdShort::zero(); bool shard_block_retainer_adnl_id_fullnode_ = false; bool parallel_validation_ = false; + std::string db_event_fifo_path_; ton::validator::fullnode::FullNodeOptions full_node_options_ = {.config_ = {}, .public_broadcast_speed_multiplier_ = 3.33, .private_broadcast_speed_multiplier_ = 3.33, @@ -391,6 +392,9 @@ class ValidatorEngine : public td::actor::Actor { void set_parallel_validation(bool value) { parallel_validation_ = value; } + void set_db_event_fifo_path(std::string value) { + db_event_fifo_path_ = std::move(value); + } void set_initial_sync_delay(double value) { full_node_options_.initial_sync_delay_ = value; } diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index dd4b84d1c6..ea6d1032f7 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -23,6 +23,8 @@ set(VALIDATOR_DB_SOURCE db/staticfilesdb.hpp db/db-utils.cpp db/db-utils.h + db/db-event-publisher.cpp + db/db-event-publisher.hpp db/package.hpp db/package.cpp diff --git a/validator/db/db-event-publisher.cpp b/validator/db/db-event-publisher.cpp new file mode 100644 index 0000000000..fed3ceba6a --- /dev/null +++ b/validator/db/db-event-publisher.cpp @@ -0,0 +1,143 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#include "tl-utils/common-utils.hpp" +#include "tl-utils/tl-utils.hpp" + +#include "db-event-publisher.hpp" + +#if TD_PORT_POSIX +#include +#include +#include +#include +#include +#endif + +namespace ton::validator { + +void DbEventPublisher::publish(tl_object_ptr event) { +#if !TD_PORT_POSIX + if (!unsupported_logged_) { + LOG(WARNING) << "DB events FIFO is not supported on this platform"; + unsupported_logged_ = true; + } + return; +#else + if (disabled_) { + return; + } + auto status = ensure_ready(); + if (status.is_error()) { + if (!ready_error_logged_) { + LOG(ERROR) << "Failed to prepare DB events FIFO '" << fifo_path_ << "': " << status; + ready_error_logged_ = true; + } + disabled_ = true; + return; + } + td::BufferSlice data = serialize_tl_object(event, true); + switch (write_once(data)) { + case WriteStatus::Ok: + no_reader_logged_ = false; + temp_error_logged_ = false; + break; + case WriteStatus::NoReader: + if (!no_reader_logged_) { + LOG(INFO) << "DB events FIFO '" << fifo_path_ << "' has no reader. Dropping event"; + no_reader_logged_ = true; + } + break; + case WriteStatus::TemporaryError: + if (!temp_error_logged_) { + LOG(WARNING) << "DB events FIFO '" << fifo_path_ << "' is temporarily unavailable. Dropping event"; + temp_error_logged_ = true; + } + break; + case WriteStatus::FatalError: + if (!write_error_logged_) { + LOG(ERROR) << "Failed to publish DB event to '" << fifo_path_ << "', disabling events"; + write_error_logged_ = true; + } + disabled_ = true; + break; + } +#endif +} + +#if TD_PORT_POSIX +td::Status DbEventPublisher::ensure_ready() { + if (fifo_ready_) { + return td::Status::OK(); + } + struct stat st; + if (lstat(fifo_path_.c_str(), &st) == 0) { + if (!S_ISFIFO(st.st_mode)) { + return td::Status::Error(PSLICE() << "path '" << fifo_path_ << "' exists and is not a FIFO"); + } + } else { + if (errno != ENOENT) { + return td::Status::PosixError(errno, PSLICE() << "stat failed for '" << fifo_path_ << "'"); + } + if (mkfifo(fifo_path_.c_str(), 0660) != 0 && errno != EEXIST) { + return td::Status::PosixError(errno, PSLICE() << "mkfifo failed for '" << fifo_path_ << "'"); + } + } + fifo_ready_ = true; + return td::Status::OK(); +} + +DbEventPublisher::WriteStatus DbEventPublisher::write_once(td::Slice data) { + td::int32 flags = O_WRONLY | O_NONBLOCK | O_CLOEXEC; + int fd = open(fifo_path_.c_str(), flags); + if (fd < 0) { + if (errno == ENXIO || errno == EPIPE) { + return WriteStatus::NoReader; + } + if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { + return WriteStatus::TemporaryError; + } + return WriteStatus::FatalError; + } + auto guard = td::ScopeExit() + [&] { + while (close(fd) != 0) { + if (errno != EINTR) { + break; + } + } + }; + while (!data.empty()) { + auto written = write(fd, data.data(), data.size()); + if (written < 0) { + if (errno == EINTR) { + continue; + } + if (errno == EAGAIN || errno == EWOULDBLOCK) { + return WriteStatus::TemporaryError; + } + if (errno == EPIPE) { + return WriteStatus::NoReader; + } + return WriteStatus::FatalError; + } + data.remove_prefix(static_cast(written)); + } + return WriteStatus::Ok; +} +#endif + +} // namespace ton::validator diff --git a/validator/db/db-event-publisher.hpp b/validator/db/db-event-publisher.hpp new file mode 100644 index 0000000000..dfc3b91d08 --- /dev/null +++ b/validator/db/db-event-publisher.hpp @@ -0,0 +1,51 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#pragma once +#include + +#include "auto/tl/ton_api.h" +#include "td/actor/actor.h" +#include "td/actor/coro_utils.h" +#include "tl/TlObject.h" + +namespace ton::validator { + +class DbEventPublisher : public td::actor::Actor { + public: + explicit DbEventPublisher(std::string fifo_path) : fifo_path_(std::move(fifo_path)) { + } + void publish(tl_object_ptr event); + + private: + std::string fifo_path_; + bool fifo_ready_ = false; + bool disabled_ = false; + bool ready_error_logged_ = false; + bool write_error_logged_ = false; + bool no_reader_logged_ = false; + bool temp_error_logged_ = false; + bool unsupported_logged_ = false; + +#if TD_PORT_POSIX + enum class WriteStatus { Ok, NoReader, TemporaryError, FatalError }; + td::Status ensure_ready(); + WriteStatus write_once(td::Slice data); +#endif +}; + +} // namespace ton::validator diff --git a/validator/full-node-custom-overlays.cpp b/validator/full-node-custom-overlays.cpp index 8ab441e92c..44bede1d02 100644 --- a/validator/full-node-custom-overlays.cpp +++ b/validator/full-node-custom-overlays.cpp @@ -118,6 +118,14 @@ void FullNodeCustomOverlay::process_block_candidate_broadcast(PublicKeyHash src, validator_set_hash, std::move(data)); } +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { + BlockIdExt block_id = create_block_id(query.block_->block_); + VLOG(FULL_NODE_DEBUG) << "Received newShardBlockBroadcast in custom overlay \"" << name_ << "\" from " << src << ": " + << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_shard_block_info_broadcast, block_id, query.block_->cc_seqno_, + std::move(query.block_->data_)); +} + void FullNodeCustomOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { if (adnl::AdnlNodeIdShort{src} == local_id_) { return; @@ -176,6 +184,20 @@ void FullNodeCustomOverlay::send_block_candidate(BlockIdExt block_id, CatchainSe local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); } +void FullNodeCustomOverlay::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { + VLOG(FULL_NODE_DEBUG) << "Sending newShardBlockBroadcast in custom overlay \"" << name_ + << "\": " << block_id.to_str(); + auto B = create_serialize_tl_object( + create_tl_object(create_tl_block_id(block_id), cc_seqno, std::move(data))); + if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(B)); + } else { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); + } +} + void FullNodeCustomOverlay::start_up() { std::sort(nodes_.begin(), nodes_.end()); nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end()); diff --git a/validator/full-node-custom-overlays.hpp b/validator/full-node-custom-overlays.hpp index 90c8d403b7..6ce723a5c6 100644 --- a/validator/full-node-custom-overlays.hpp +++ b/validator/full-node-custom-overlays.hpp @@ -36,6 +36,7 @@ class FullNodeCustomOverlay : public td::actor::Actor { void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); template void process_broadcast(PublicKeyHash, T &) { @@ -47,6 +48,7 @@ class FullNodeCustomOverlay : public td::actor::Actor { void send_broadcast(BlockBroadcast broadcast); void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data); + void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data); void set_config(FullNodeConfig config) { opts_.config_ = std::move(config); diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index d480ec0454..bb6a4621d6 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -91,8 +91,8 @@ void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonN BlockIdExt block_id = create_block_id(query.block_->block_); VLOG(FULL_NODE_DEBUG) << "Received newShardBlockBroadcast in fast sync overlay from " << src << ": " << block_id.to_str(); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block_description_broadcast, - block_id, query.block_->cc_seqno_, std::move(query.block_->data_)); + td::actor::send_closure(full_node_, &FullNode::process_shard_block_info_broadcast, block_id, query.block_->cc_seqno_, + std::move(query.block_->data_)); } void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index cf4be9aec7..7c5f64f002 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -756,8 +756,8 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ex void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { BlockIdExt block_id = create_block_id(query.block_->block_); VLOG(FULL_NODE_DEBUG) << "Received newShardBlockBroadcast from " << src << ": " << block_id.to_str(); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block_description_broadcast, - block_id, query.block_->cc_seqno_, std::move(query.block_->data_)); + td::actor::send_closure(full_node_, &FullNode::process_shard_block_info_broadcast, block_id, query.block_->cc_seqno_, + std::move(query.block_->data_)); } void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 2e242343b2..6a0ff9d89c 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -19,6 +19,7 @@ #include "common/delay.h" #include "impl/out-msg-queue-proof.hpp" #include "td/actor/MultiPromise.h" +#include "td/actor/coro_utils.h" #include "td/utils/Random.h" #include "ton/ton-io.hpp" #include "ton/ton-tl.hpp" @@ -334,6 +335,7 @@ void FullNodeImpl::send_ext_message(AccountIdPrefixFull dst, td::BufferSlice dat } void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { + send_shard_block_info_to_custom_overlays(block_id, cc_seqno, data); auto shard = get_shard(ShardIdFull{masterchainId}); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; @@ -621,9 +623,16 @@ void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast) { void FullNodeImpl::process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data) { send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); - // ignore cc_seqno and validator_hash for now - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_block_candidate_broadcast, block_id, - std::move(data)); + td::actor::ask(validator_manager_, &ValidatorManagerInterface::new_block_candidate_broadcast, block_id, cc_seqno, + std::move(data)) + .detach(); +} + +void FullNodeImpl::process_shard_block_info_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) { + send_shard_block_info_to_custom_overlays(block_id, cc_seqno, data); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block_description_broadcast, + block_id, cc_seqno, std::move(data)); } void FullNodeImpl::get_out_msg_queue_query_token(td::Promise> promise) { @@ -786,14 +795,10 @@ void FullNodeImpl::update_custom_overlay(CustomOverlayInfo &overlay) { } void FullNodeImpl::send_block_broadcast_to_custom_overlays(const BlockBroadcast &broadcast) { - if (!custom_overlays_sent_broadcasts_.insert(broadcast.block_id).second) { + if (custom_overlays_sent_broadcasts_.contains(broadcast.block_id)) { return; } - custom_overlays_sent_broadcasts_lru_.push(broadcast.block_id); - if (custom_overlays_sent_broadcasts_lru_.size() > 256) { - custom_overlays_sent_broadcasts_.erase(custom_overlays_sent_broadcasts_lru_.front()); - custom_overlays_sent_broadcasts_lru_.pop(); - } + custom_overlays_sent_broadcasts_.put(broadcast.block_id, {}); for (auto &[_, private_overlay] : custom_overlays_) { if (private_overlay.params_.send_shard(broadcast.block_id.shard_full())) { for (auto &[local_id, actor] : private_overlay.actors_) { @@ -809,14 +814,10 @@ void FullNodeImpl::send_block_candidate_broadcast_to_custom_overlays(const Block td::uint32 validator_set_hash, const td::BufferSlice &data) { // Same cache of sent broadcasts as in send_block_broadcast_to_custom_overlays - if (!custom_overlays_sent_broadcasts_.insert(block_id).second) { + if (custom_overlays_sent_broadcasts_.contains(block_id)) { return; } - custom_overlays_sent_broadcasts_lru_.push(block_id); - if (custom_overlays_sent_broadcasts_lru_.size() > 256) { - custom_overlays_sent_broadcasts_.erase(custom_overlays_sent_broadcasts_lru_.front()); - custom_overlays_sent_broadcasts_lru_.pop(); - } + custom_overlays_sent_broadcasts_.put(block_id, {}); for (auto &[_, private_overlay] : custom_overlays_) { if (private_overlay.params_.send_shard(block_id.shard_full())) { for (auto &[local_id, actor] : private_overlay.actors_) { @@ -829,6 +830,24 @@ void FullNodeImpl::send_block_candidate_broadcast_to_custom_overlays(const Block } } +void FullNodeImpl::send_shard_block_info_to_custom_overlays(BlockIdExt block_id, CatchainSeqno cc_seqno, + const td::BufferSlice &data) { + if (custom_overlays_sent_shard_block_desc_.contains(block_id)) { + return; + } + custom_overlays_sent_shard_block_desc_.put(block_id, {}); + for (auto &[_, private_overlay] : custom_overlays_) { + if (private_overlay.params_.send_shard(block_id.shard_full())) { + for (auto &[local_id, actor] : private_overlay.actors_) { + if (private_overlay.params_.block_senders_.contains(local_id)) { + td::actor::send_closure(actor, &FullNodeCustomOverlay::send_shard_block_info, block_id, cc_seqno, + data.clone()); + } + } + } + } +} + FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, diff --git a/validator/full-node.h b/validator/full-node.h index 600209d98d..bdea158dd5 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -99,6 +99,8 @@ class FullNode : public td::actor::Actor { virtual void process_block_broadcast(BlockBroadcast broadcast) = 0; virtual void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data) = 0; + virtual void process_shard_block_info_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) = 0; virtual void get_out_msg_queue_query_token(td::Promise> promise) = 0; virtual void set_validator_telemetry_filename(std::string value) = 0; diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 3efb3d131b..5389d99558 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -28,6 +28,7 @@ #include "interfaces/proof.h" #include "interfaces/shard.h" +#include "td/utils/LRUCache.h" #include "full-node-custom-overlays.hpp" #include "full-node-fast-sync-overlays.hpp" @@ -69,7 +70,7 @@ class FullNodeImpl : public FullNode { void initial_read_complete(BlockHandle top_block); void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data); void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data); - void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqnp, td::BufferSlice data); + void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data); void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data, int mode); void send_broadcast(BlockBroadcast broadcast, int mode); @@ -96,6 +97,7 @@ class FullNodeImpl : public FullNode { void process_block_broadcast(BlockBroadcast broadcast) override; void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data) override; + void process_shard_block_info_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; void get_out_msg_queue_query_token(td::Promise> promise) override; void set_validator_telemetry_filename(std::string value) override; @@ -164,14 +166,16 @@ class FullNodeImpl : public FullNode { std::map> actors_; // our local id -> actor }; std::map custom_overlays_; - std::set custom_overlays_sent_broadcasts_; - std::queue custom_overlays_sent_broadcasts_lru_; + td::LRUCache custom_overlays_sent_broadcasts_{256}; + td::LRUCache custom_overlays_sent_shard_block_desc_{256}; void update_private_overlays(); void update_custom_overlay(CustomOverlayInfo& overlay); void send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast); void send_block_candidate_broadcast_to_custom_overlays(const BlockIdExt& block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, const td::BufferSlice& data); + void send_shard_block_info_to_custom_overlays(BlockIdExt block_id, CatchainSeqno cc_seqno, + const td::BufferSlice& data); std::string validator_telemetry_filename_; PublicKeyHash validator_telemetry_collector_key_ = PublicKeyHash::zero(); diff --git a/validator/impl/CMakeLists.txt b/validator/impl/CMakeLists.txt index 22680fe2f7..dad23baa75 100644 --- a/validator/impl/CMakeLists.txt +++ b/validator/impl/CMakeLists.txt @@ -1,7 +1,6 @@ set(TON_VALIDATOR_SOURCE accept-block.cpp block.cpp - candidates-buffer.cpp check-proof.cpp collator.cpp config.cpp @@ -21,7 +20,6 @@ set(TON_VALIDATOR_SOURCE accept-block.hpp block.hpp - candidates-buffer.hpp check-proof.hpp collator-impl.h collator.h diff --git a/validator/impl/candidates-buffer.cpp b/validator/impl/candidates-buffer.cpp deleted file mode 100644 index e24913b676..0000000000 --- a/validator/impl/candidates-buffer.cpp +++ /dev/null @@ -1,213 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . -*/ -#include "candidates-buffer.hpp" -#include "fabric.h" - -namespace ton::validator { - -void CandidatesBuffer::start_up() { - alarm_timestamp() = td::Timestamp::in(60.0); -} - -void CandidatesBuffer::alarm() { - alarm_timestamp() = td::Timestamp::in(60.0); - for (auto it = candidates_.begin(); it != candidates_.end();) { - Candidate &entry = it->second; - if (entry.ttl_.is_in_past()) { - for (auto &p : entry.data_waiters_) { - p.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); - } - for (auto &p : entry.state_waiters_) { - p.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); - } - it = candidates_.erase(it); - } else { - ++it; - } - } -} - -void CandidatesBuffer::add_new_candidate(BlockIdExt id, PublicKey source, FileHash collated_data_file_hash) { - auto it = candidates_.emplace(id, Candidate{}); - Candidate &entry = it.first->second; - entry.ttl_ = td::Timestamp::in(120.0); - if (!it.second) { // not inserted - return; - } - LOG(DEBUG) << "New block candidate " << id.to_str(); - entry.source_ = source; - entry.collated_data_file_hash_ = collated_data_file_hash; -} - -void CandidatesBuffer::get_block_data(BlockIdExt id, td::Promise> promise) { - auto it = candidates_.find(id); - if (it == candidates_.end()) { - promise.set_error(td::Status::Error(ErrorCode::notready, "unknown block candidate")); - return; - } - Candidate &entry = it->second; - if (entry.data_.not_null()) { - promise.set_result(entry.data_); - return; - } - entry.data_waiters_.push_back(std::move(promise)); - if (entry.data_requested_) { - return; - } - entry.data_requested_ = true; - td::actor::send_closure(manager_, &ValidatorManager::get_block_candidate_from_db, entry.source_, id, - entry.collated_data_file_hash_, [SelfId = actor_id(this), id](td::Result R) { - td::actor::send_closure(SelfId, &CandidatesBuffer::got_block_candidate, id, std::move(R)); - }); -} - -void CandidatesBuffer::got_block_candidate(BlockIdExt id, td::Result R) { - if (R.is_error()) { - finish_get_block_data(id, R.move_as_error()); - return; - } - BlockCandidate cand = R.move_as_ok(); - CHECK(cand.id == id); - finish_get_block_data(id, create_block(id, std::move(cand.data))); -} - -void CandidatesBuffer::get_block_state(BlockIdExt id, td::Promise> promise) { - auto it = candidates_.find(id); - if (it == candidates_.end()) { - promise.set_error(td::Status::Error(ErrorCode::notready, "unknown block candidate")); - return; - } - Candidate &entry = it->second; - if (entry.state_.not_null()) { - promise.set_result(entry.state_); - return; - } - entry.state_waiters_.push_back(std::move(promise)); - if (entry.state_requested_) { - return; - } - entry.state_requested_ = true; - get_block_data(id, [SelfId = actor_id(this), id](td::Result> R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &CandidatesBuffer::finish_get_block_state, id, R.move_as_error()); - return; - } - td::actor::send_closure(SelfId, &CandidatesBuffer::get_block_state_cont, id, R.move_as_ok()); - }); -} - -void CandidatesBuffer::get_block_state_cont(BlockIdExt id, td::Ref data) { - CHECK(id == data->block_id()); - std::vector prev; - BlockIdExt mc_blkid; - bool after_split; - auto S = block::unpack_block_prev_blk_ext(data->root_cell(), id, prev, mc_blkid, after_split); - if (S.is_error()) { - finish_get_block_state(id, std::move(S)); - return; - } - get_block_state_cont2(std::move(data), std::move(prev), {}); -} - -void CandidatesBuffer::get_block_state_cont2(td::Ref block, std::vector prev, - std::vector> prev_states) { - if (prev_states.size() < prev.size()) { - BlockIdExt prev_id = prev[prev_states.size()]; - td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db_short, prev_id, - [SelfId = actor_id(this), block = std::move(block), prev = std::move(prev), - prev_states = std::move(prev_states)](td::Result> R) mutable { - if (R.is_error()) { - td::actor::send_closure(SelfId, &CandidatesBuffer::finish_get_block_state, - block->block_id(), R.move_as_error()); - return; - } - prev_states.push_back(R.move_as_ok()); - td::actor::send_closure(SelfId, &CandidatesBuffer::get_block_state_cont2, - std::move(block), std::move(prev), std::move(prev_states)); - }); - return; - } - - BlockIdExt id = block->block_id(); - td::Ref state; - CHECK(prev_states.size() == 1 || prev_states.size() == 2); - if (prev_states.size() == 2) { // after merge - auto R = prev_states[0]->merge_with(*prev_states[1]); - if (R.is_error()) { - finish_get_block_state(id, R.move_as_error()); - return; - } - state = R.move_as_ok(); - } else if (id.shard_full() != prev[0].shard_full()) { // after split - auto R = prev_states[0]->split(); - if (R.is_error()) { - finish_get_block_state(id, R.move_as_error()); - return; - } - auto s = R.move_as_ok(); - state = is_left_child(id.shard_full()) ? std::move(s.first) : std::move(s.second); - } else { // no split/merge - state = std::move(prev_states[0]); - } - - auto S = state.write().apply_block(id, std::move(block)); - if (S.is_error()) { - finish_get_block_state(id, std::move(S)); - return; - } - finish_get_block_state(id, std::move(state)); -} - -void CandidatesBuffer::finish_get_block_data(BlockIdExt id, td::Result> res) { - auto it = candidates_.find(id); - if (it == candidates_.end()) { - return; - } - Candidate &entry = it->second; - for (auto &p : entry.data_waiters_) { - p.set_result(res.clone()); - } - entry.data_waiters_.clear(); - entry.data_requested_ = false; - if (res.is_ok()) { - entry.data_ = res.move_as_ok(); - LOG(DEBUG) << "Loaded block data for " << id.to_str(); - } else { - LOG(DEBUG) << "Failed to load block data for " << id.to_str() << ": " << res.move_as_error(); - } -} - -void CandidatesBuffer::finish_get_block_state(BlockIdExt id, td::Result> res) { - auto it = candidates_.find(id); - if (it == candidates_.end()) { - return; - } - Candidate &entry = it->second; - for (auto &p : entry.state_waiters_) { - p.set_result(res.clone()); - } - entry.state_waiters_.clear(); - entry.state_requested_ = false; - if (res.is_ok()) { - entry.state_ = res.move_as_ok(); - LOG(DEBUG) << "Loaded block state for " << id.to_str(); - } else { - LOG(DEBUG) << "Failed to load block state for " << id.to_str() << ": " << res.move_as_error(); - } -} - -} // namespace ton::validator diff --git a/validator/impl/candidates-buffer.hpp b/validator/impl/candidates-buffer.hpp deleted file mode 100644 index 90914cff33..0000000000 --- a/validator/impl/candidates-buffer.hpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . -*/ -#pragma once -#include "interfaces/validator-manager.h" -#include "td/actor/actor.h" -#include "ton/ton-types.h" - -namespace ton::validator { - -class CandidatesBuffer : public td::actor::Actor { - public: - explicit CandidatesBuffer(td::actor::ActorId manager) : manager_(std::move(manager)) { - } - - void start_up() override; - void alarm() override; - - void add_new_candidate(BlockIdExt id, PublicKey source, FileHash collated_data_file_hash); - void get_block_data(BlockIdExt id, td::Promise> promise); - void get_block_state(BlockIdExt id, td::Promise> promise); - - private: - td::actor::ActorId manager_; - - struct Candidate { - PublicKey source_; - FileHash collated_data_file_hash_; - td::Timestamp ttl_; - - td::Ref data_; - std::vector>> data_waiters_; - bool data_requested_{false}; - - td::Ref state_; - std::vector>> state_waiters_; - bool state_requested_{false}; - }; - std::map candidates_; - - void got_block_candidate(BlockIdExt id, td::Result R); - - void get_block_state_cont(BlockIdExt id, td::Ref data); - void get_block_state_cont2(td::Ref block, std::vector prev, - std::vector> prev_states); - - void finish_get_block_data(BlockIdExt id, td::Result> res); - void finish_get_block_state(BlockIdExt id, td::Result> res); -}; - -} // namespace ton::validator diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index edf94f6c51..44c4e23c7f 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -277,6 +277,9 @@ void LiteQuery::perform() { [&](lite_api::liteServer_nonfinal_getValidatorGroups& q) { this->perform_nonfinal_getValidatorGroups(q.mode_, ShardIdFull{q.wc_, (ShardId)q.shard_}); }, + [&](lite_api::liteServer_nonfinal_getPendingShardBlocks& q) { + this->perform_nonfinal_getPendingShardBlocks(q.mode_, ShardIdFull{q.wc_, (ShardId)q.shard_}); + }, [&](lite_api::liteServer_getOutMsgQueueSizes& q) { this->perform_getOutMsgQueueSizes(q.mode_ & 1 ? ShardIdFull(q.wc_, q.shard_) : td::optional()); }, @@ -3742,5 +3745,25 @@ void LiteQuery::perform_nonfinal_getValidatorGroups(int mode, ShardIdFull shard) }); } +void LiteQuery::perform_nonfinal_getPendingShardBlocks(int mode, ShardIdFull shard) { + bool with_shard = mode & 1; + LOG(INFO) << "started a nonfinal.getPendingShardBlocks" << (with_shard ? shard.to_str() : "(all)") + << " liteserver query"; + td::optional maybe_shard; + if (with_shard) { + maybe_shard = shard; + } + td::actor::send_closure( + manager_, &ValidatorManager::get_pending_shard_blocks_for_litequery, maybe_shard, + [Self = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::finish_query, serialize_tl_object(R.move_as_ok(), true), + false); + } + }); +} + } // namespace validator } // namespace ton diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index 05b7900b3a..a3bdba9029 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -188,6 +188,7 @@ class LiteQuery : public td::actor::Actor { void perform_nonfinal_getCandidate(td::Bits256 source, BlockIdExt blkid, td::Bits256 collated_data_hash); void perform_nonfinal_getValidatorGroups(int mode, ShardIdFull shard); + void perform_nonfinal_getPendingShardBlocks(int mode, ShardIdFull shard); void load_prevKeyBlock(ton::BlockIdExt blkid, td::Promise>>); void continue_loadPrevKeyBlock(ton::BlockIdExt blkid, td::Result, BlockIdExt>> res, diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index d0d858a6b7..a76a385ed1 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -505,6 +505,13 @@ td::Ref MasterchainStateQ::get_shard_from_config(ShardIdFull shard, return config_->get_shard_hash(shard, exact); } +CatchainSeqno MasterchainStateQ::get_shard_cc_seqno(ShardIdFull shard) const { + if (!config_) { + return std::numeric_limits::max(); + } + return config_->get_shard_cc_seqno(shard); +} + bool MasterchainStateQ::rotated_all_shards() const { if (!config_) { return false; diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index ac6ade5d26..a0c30d9ac5 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -116,6 +116,7 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { bool rotated_all_shards() const override; std::vector> get_shards() const override; td::Ref get_shard_from_config(ShardIdFull shard, bool exact) const override; + CatchainSeqno get_shard_cc_seqno(ShardIdFull shard) const override; bool ancestor_is_valid(BlockIdExt id) const override { return check_old_mc_block_id(id); } diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 07524f66f0..e0fee64a42 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -71,6 +71,7 @@ class MasterchainState : virtual public ShardState { virtual bool rotated_all_shards() const = 0; virtual std::vector> get_shards() const = 0; virtual td::Ref get_shard_from_config(ShardIdFull shard, bool exact = true) const = 0; + virtual CatchainSeqno get_shard_cc_seqno(ShardIdFull shard) const = 0; virtual bool workchain_is_active(WorkchainId workchain_id) const = 0; virtual td::uint32 persistent_state_split_depth(WorkchainId workchain_id) const = 0; virtual td::uint32 monitor_min_split_depth(WorkchainId workchain_id) const = 0; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 2c270457ff..497c5db6ca 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -384,6 +384,11 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void get_validator_groups_info_for_litequery( td::optional shard, td::Promise> promise) = 0; + virtual void get_pending_shard_blocks_for_litequery( + td::optional shard, + td::Promise> promise) { + promise.set_error(td::Status::Error("not implemented")); + } virtual void add_lite_query_stats(int lite_query_id, bool success) { } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 43524d62d8..367dca97e1 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -134,8 +134,6 @@ class ValidatorManagerImpl : public ValidatorManager { void new_ihr_message(td::BufferSlice data) override; void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; - void new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) override { - } void add_ext_server_id(adnl::AdnlNodeIdShort id) override { UNREACHABLE(); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 5a5a4a8b1b..42b4a3d453 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -157,9 +157,6 @@ class ValidatorManagerImpl : public ValidatorManager { td::BufferSlice data) override { UNREACHABLE(); } - void new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) override { - UNREACHABLE(); - } void add_ext_server_id(adnl::AdnlNodeIdShort id) override { UNREACHABLE(); diff --git a/validator/manager.cpp b/validator/manager.cpp index 296a7ba451..f50859b571 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -26,6 +26,7 @@ #include "downloaders/wait-block-state-merge.hpp" #include "downloaders/wait-block-state.hpp" #include "td/actor/MultiPromise.h" +#include "td/actor/coro_utils.h" #include "td/utils/JsonBuilder.h" #include "td/utils/Random.h" #include "td/utils/buffer.h" @@ -33,6 +34,7 @@ #include "td/utils/port/path.h" #include "tl-utils/lite-utils.hpp" #include "tl/tl_json.h" +#include "ton/lite-tl.hpp" #include "ton/ton-io.hpp" #include "ton/ton-tl.hpp" #include "validator/stats-merger.h" @@ -222,7 +224,7 @@ void ValidatorManagerImpl::new_block_broadcast(BlockBroadcast broadcast, td::Pro promise = [SelfId = actor_id(this), promise = std::move(promise), block_id = broadcast.block_id, cc_seqno = broadcast.catchain_seqno](td::Result R) mutable { if (R.is_ok()) { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::validated_block_broadcast, block_id, cc_seqno); + td::actor::ask(SelfId, &ValidatorManagerImpl::validated_block_broadcast, block_id, cc_seqno).detach(); } promise.set_result(std::move(R)); }; @@ -234,12 +236,18 @@ void ValidatorManagerImpl::new_block_broadcast(BlockBroadcast broadcast, td::Pro .release(); } -void ValidatorManagerImpl::validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno) { +td::actor::Task<> ValidatorManagerImpl::validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno) { for (auto &[_, collator_node] : collator_nodes_) { if (collator_node.can_collate_shard(block_id.shard_full())) { td::actor::send_closure(collator_node.actor, &CollatorNode::new_shard_block_accepted, block_id, cc_seqno); } } + if (opts_->nonfinal_ls_queries_enabled()) { + co_await td::actor::ask(actor_id(this), &ValidatorManagerImpl::wait_block_state_short, block_id, 0, + td::Timestamp::in(60.0), true); + } + process_accepted_nonfinal_block(block_id, cc_seqno); + co_return td::Unit{}; } void ValidatorManagerImpl::sync_complete(td::Promise promise) { @@ -474,16 +482,41 @@ void ValidatorManagerImpl::new_shard_block_description_broadcast(BlockIdExt bloc actor_id(this), td::Timestamp::in(2.0), std::move(P)); } -void ValidatorManagerImpl::new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) { +td::actor::Task<> ValidatorManagerImpl::new_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) { if (!last_masterchain_block_handle_ || !started_) { VLOG(VALIDATOR_DEBUG) << "dropping top shard block broadcast: not inited"; - return; + co_return td::Unit{}; } if (!need_monitor(block_id.shard_full())) { VLOG(VALIDATOR_DEBUG) << "dropping block candidate broadcast: not monitoring shard"; - return; + co_return td::Unit{}; + } + add_cached_block_data(block_id, data.clone()); + + if (opts_->nonfinal_ls_queries_enabled()) { + auto handle = co_await td::actor::ask(actor_id(this), &ValidatorManagerImpl::get_block_handle, block_id, true); + Ref block = co_await create_block(block_id, std::move(data)); + co_await td::actor::ask(actor_id(this), &ValidatorManagerImpl::set_block_data, handle, std::move(block)); + co_await td::actor::ask(actor_id(this), &ValidatorManagerImpl::wait_block_state, handle, 0, td::Timestamp::in(60.0), + true); + VLOG(VALIDATOR_DEBUG) << "Processed candidate broadcast " << block_id.to_str(); + if (is_valid_nonfinal_group(block_id.shard_full(), cc_seqno)) { + NonfinalGroupInfo &info = nonfinal_info_[{block_id.shard_full(), cc_seqno}]; + if (!info.last_accepted.is_valid() || info.last_accepted.seqno() < block_id.seqno()) { + if (!info.last_candidate.is_valid() || info.last_candidate.seqno() < block_id.seqno()) { + info.last_candidate = block_id; + } + } + } } - add_cached_block_data(block_id, std::move(data)); + if (!db_event_publisher_.empty() && is_valid_nonfinal_group(block_id.shard_full(), cc_seqno)) { + VLOG(VALIDATOR_DEBUG) << "DB Event: blockCandidateReceived " << block_id.to_str(); + td::actor::ask(db_event_publisher_, &DbEventPublisher::publish, + create_tl_object(create_tl_block_id(block_id))) + .detach(); + } + co_return td::Unit{}; } void ValidatorManagerImpl::add_shard_block_description(td::Ref desc) { @@ -505,7 +538,13 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refshard())) { + td::actor::send_closure(collator_node.actor, &CollatorNode::new_shard_block_accepted, desc->block_id(), + desc->catchain_seqno()); + } + } + if (!is_validator() && !opts_->nonfinal_ls_queries_enabled()) { return; } auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); @@ -516,7 +555,8 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id().shard_full(), desc->catchain_seqno()}].latest_desc = desc; VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); if (need_monitor(desc->block_id().shard_full())) { - auto P = td::PromiseCreator::lambda([](td::Result> R) { + auto P = td::PromiseCreator::lambda([block_id = desc->block_id(), cc_seqno = desc->catchain_seqno(), + SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { auto S = R.move_as_error(); if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { @@ -524,7 +564,9 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id(), 0, td::Timestamp::in(60.0), true, std::move(P)); } @@ -544,12 +586,6 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refshard())) { - td::actor::send_closure(collator_node.actor, &CollatorNode::new_shard_block_accepted, desc->block_id(), - desc->catchain_seqno()); - } - } } void ValidatorManagerImpl::preload_msg_queue_to_masterchain(td::Ref desc, @@ -1370,10 +1406,6 @@ void ValidatorManagerImpl::set_next_block(BlockIdExt block_id, BlockIdExt next, void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::Promise promise) { - if (!candidates_buffer_.empty()) { - td::actor::send_closure(candidates_buffer_, &CandidatesBuffer::add_new_candidate, id, - PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}, candidate.collated_file_hash); - } if (!id.is_masterchain()) { add_cached_block_data(id, candidate.data.clone()); } @@ -1500,6 +1532,12 @@ void ValidatorManagerImpl::new_block_cont(BlockHandle handle, td::Refset_processed(); promise.set_value(td::Unit()); } + if (!db_event_publisher_.empty() && !handle->id().is_masterchain()) { + VLOG(VALIDATOR_DEBUG) << "DB Event: blockApplied " << handle->id().to_str(); + td::actor::ask(db_event_publisher_, &DbEventPublisher::publish, + create_tl_object(create_tl_block_id(handle->id()))) + .detach(); + } } void ValidatorManagerImpl::new_block(BlockHandle handle, td::Ref state, td::Promise promise) { @@ -1862,6 +1900,10 @@ void ValidatorManagerImpl::start_up() { validator_manager_init(opts_, actor_id(this), db_.get(), std::move(P)); + if (!opts_->get_db_event_fifo_path().empty()) { + db_event_publisher_ = td::actor::create_actor("db-events", opts_->get_db_event_fifo_path()); + } + check_waiters_at_ = td::Timestamp::in(1.0); alarm_timestamp().relax(check_waiters_at_); } @@ -1908,10 +1950,6 @@ void ValidatorManagerImpl::started(ValidatorManagerInitResult R) { }); td::actor::send_closure(db_, &Db::get_destroyed_validator_sessions, std::move(P)); - if (opts_->nonfinal_ls_queries_enabled()) { - candidates_buffer_ = td::actor::create_actor("candidates-buffer", actor_id(this)); - } - auto Q = td::PromiseCreator::lambda( [SelfId = actor_id(this)](td::Result>> R) { if (R.is_error()) { @@ -2653,14 +2691,22 @@ void ValidatorManagerImpl::update_shard_client_block_handle(BlockHandle handle, shard_client_handle_ = std::move(handle); auto seqno = shard_client_handle_->id().seqno(); if (state.not_null()) { + shard_client_state_ = state; shard_client_shards_ = state->get_shards(); if (last_liteserver_state_.is_null() || last_liteserver_state_->get_block_id().seqno() < seqno) { last_liteserver_state_ = std::move(state); } } + if (!db_event_publisher_.empty()) { + VLOG(VALIDATOR_DEBUG) << "DB Event: blockApplied " << shard_client_handle_->id().to_str(); + td::actor::ask(db_event_publisher_, &DbEventPublisher::publish, + create_tl_object(create_tl_block_id(shard_client_handle_->id()))) + .detach(); + } for (auto &c : collator_nodes_) { td::actor::send_closure(c.second.actor, &CollatorNode::update_shard_client_handle, shard_client_handle_); } + cleanup_nonfinal_groups(); shard_client_update(seqno); promise.set_value(td::Unit()); } @@ -3060,56 +3106,22 @@ void ValidatorManagerImpl::get_block_handle_for_litequery(BlockIdExt block_id, t } void ValidatorManagerImpl::get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) { - if (candidates_buffer_.empty()) { - get_block_handle_for_litequery( - block_id, [manager = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - TRY_RESULT_PROMISE(promise, handle, std::move(R)); - td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, std::move(handle), - std::move(promise)); - }); - } else { - td::actor::send_closure( - candidates_buffer_, &CandidatesBuffer::get_block_data, block_id, - [manager = actor_id(this), promise = std::move(promise), block_id](td::Result> R) mutable { - if (R.is_ok()) { - promise.set_result(R.move_as_ok()); - return; - } - td::actor::send_closure(manager, &ValidatorManagerImpl::get_block_handle_for_litequery, block_id, - [manager, promise = std::move(promise)](td::Result R) mutable { - TRY_RESULT_PROMISE(promise, handle, std::move(R)); - td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, - std::move(handle), std::move(promise)); - }); - }); - } + get_block_handle_for_litequery( + block_id, [manager = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, std::move(handle), + std::move(promise)); + }); } void ValidatorManagerImpl::get_block_state_for_litequery(BlockIdExt block_id, td::Promise> promise) { - if (candidates_buffer_.empty()) { - get_block_handle_for_litequery( - block_id, [manager = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - TRY_RESULT_PROMISE(promise, handle, std::move(R)); - td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, std::move(handle), - std::move(promise)); - }); - } else { - td::actor::send_closure( - candidates_buffer_, &CandidatesBuffer::get_block_state, block_id, - [manager = actor_id(this), promise = std::move(promise), block_id](td::Result> R) mutable { - if (R.is_ok()) { - promise.set_result(R.move_as_ok()); - return; - } - td::actor::send_closure(manager, &ValidatorManagerImpl::get_block_handle_for_litequery, block_id, - [manager, promise = std::move(promise)](td::Result R) mutable { - TRY_RESULT_PROMISE(promise, handle, std::move(R)); - td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, - std::move(handle), std::move(promise)); - }); - }); - } + get_block_handle_for_litequery( + block_id, [manager = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, std::move(handle), + std::move(promise)); + }); } void ValidatorManagerImpl::get_block_by_lt_for_litequery(AccountIdPrefixFull account, LogicalTime lt, @@ -3307,6 +3319,28 @@ void ValidatorManagerImpl::get_validator_groups_info_for_litequery( td::actor::create_actor("get-validator-groups-info", std::move(groups), std::move(promise)).release(); } +void ValidatorManagerImpl::get_pending_shard_blocks_for_litequery( + td::optional shard, + td::Promise> promise) { + if (!opts_->nonfinal_ls_queries_enabled()) { + promise.set_error(td::Status::Error("query is not allowed")); + return; + } + auto response = create_tl_object(); + for (auto &[p, info] : nonfinal_info_) { + if (shard && !shard_intersects(shard.value(), p.first)) { + continue; + } + if (info.last_accepted.is_valid()) { + response->signed_blocks_.push_back(create_tl_lite_block_id(info.last_accepted)); + } + if (info.last_candidate.is_valid()) { + response->candidates_.push_back(create_tl_lite_block_id(info.last_candidate)); + } + } + promise.set_value(std::move(response)); +} + void ValidatorManagerImpl::update_options(td::Ref opts) { if (!shard_client_.empty()) { td::actor::send_closure(shard_client_, &ShardClient::update_options, opts); @@ -3579,6 +3613,60 @@ void ValidatorManagerImpl::iterate_temp_block_handles(std::functionget_shard_cc_seqno(shard) <= cc_seqno; +} + +void ValidatorManagerImpl::process_accepted_nonfinal_block(BlockIdExt block_id, CatchainSeqno cc_seqno) { + if (!is_valid_nonfinal_group(block_id.shard_full(), cc_seqno)) { + return; + } + if (opts_->nonfinal_ls_queries_enabled()) { + NonfinalGroupInfo &info = nonfinal_info_[{block_id.shard_full(), cc_seqno}]; + if (!info.last_accepted.is_valid() || info.last_accepted.seqno() < block_id.seqno()) { + info.last_accepted = block_id; + if (info.last_candidate.is_valid() && info.last_candidate.seqno() <= block_id.seqno()) { + info.last_candidate = {}; + } + } + } + if (!db_event_publisher_.empty()) { + VLOG(VALIDATOR_DEBUG) << "DB Event: blockSigned " << block_id.to_str(); + td::actor::ask(db_event_publisher_, &DbEventPublisher::publish, + create_tl_object(create_tl_block_id(block_id))) + .detach(); + } +} + +void ValidatorManagerImpl::cleanup_nonfinal_groups() { + for (auto it = nonfinal_info_.begin(); it != nonfinal_info_.end();) { + const auto &[shard, cc_seqno] = it->first; + if (!is_valid_nonfinal_group(shard, cc_seqno)) { + it = nonfinal_info_.erase(it); + continue; + } + NonfinalGroupInfo &info = it->second; + auto desc = shard_client_state_->get_shard_from_config(ShardIdFull(shard.workchain, shard.shard | 1), false); + if (desc.not_null()) { + BlockSeqno top_seqno = desc->top_block_id().seqno(); + if (info.last_accepted.is_valid() && info.last_accepted.seqno() <= top_seqno) { + info.last_accepted = {}; + } + if (info.last_candidate.is_valid() && info.last_candidate.seqno() <= top_seqno) { + info.last_candidate = {}; + } + } + if (info.last_accepted.is_valid() || info.last_candidate.is_valid()) { + ++it; + } else { + it = nonfinal_info_.erase(it); + } + } +} + } // namespace validator } // namespace ton diff --git a/validator/manager.hpp b/validator/manager.hpp index 3625033451..039f053380 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -25,7 +25,7 @@ #include "collator-node/collator-node.hpp" #include "common/refcnt.hpp" -#include "impl/candidates-buffer.hpp" +#include "db/db-event-publisher.hpp" #include "impl/ext-message-pool.hpp" #include "interfaces/db.h" #include "interfaces/validator-manager.h" @@ -33,6 +33,7 @@ #include "rldp2/rldp.h" #include "td/actor/ActorStats.h" #include "td/actor/PromiseFuture.h" +#include "td/actor/coro_task.h" #include "td/utils/LRUCache.h" #include "td/utils/SharedSlice.h" #include "td/utils/buffer.h" @@ -240,6 +241,7 @@ class ValidatorManagerImpl : public ValidatorManager { BlockHandle last_known_key_block_handle_; BlockHandle shard_client_handle_; std::vector> shard_client_shards_; + td::Ref shard_client_state_; td::Ref last_liteserver_state_; td::Ref do_get_last_liteserver_state(); @@ -310,7 +312,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise promise) override; void validate_block(ReceivedBlock block, td::Promise promise) override; void new_block_broadcast(BlockBroadcast broadcast, td::Promise promise) override; - void validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno); + td::actor::Task<> validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno); //void create_validate_block(BlockId block, td::BufferSlice data, td::Promise promise) = 0; void sync_complete(td::Promise promise) override; @@ -343,7 +345,8 @@ class ValidatorManagerImpl : public ValidatorManager { void new_ihr_message(td::BufferSlice data) override; void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; - void new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) override; + td::actor::Task<> new_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) override; void add_ext_server_id(adnl::AdnlNodeIdShort id) override; void add_ext_server_port(td::uint16 port) override; @@ -605,6 +608,9 @@ class ValidatorManagerImpl : public ValidatorManager { void get_validator_groups_info_for_litequery( td::optional shard, td::Promise> promise) override; + void get_pending_shard_blocks_for_litequery( + td::optional shard, + td::Promise> promise) override; void add_lite_query_stats(int lite_query_id, bool success) override { ++ls_stats_[lite_query_id]; @@ -701,8 +707,6 @@ class ValidatorManagerImpl : public ValidatorManager { size_t active_validator_groups_master_{0}, active_validator_groups_shard_{0}; - td::actor::ActorOwn candidates_buffer_; - void log_collate_query_stats(CollationStats stats) override; void log_validate_query_stats(ValidationStats stats) override; void log_collator_node_response_stats(CollatorNodeResponseStats stats) override; @@ -743,6 +747,17 @@ class ValidatorManagerImpl : public ValidatorManager { std::map> shard_block_retainers_; void init_shard_block_verifier(adnl::AdnlNodeIdShort local_id); + + td::actor::ActorOwn db_event_publisher_; + + struct NonfinalGroupInfo { + BlockIdExt last_accepted, last_candidate; + }; + std::map, NonfinalGroupInfo> nonfinal_info_; + + bool is_valid_nonfinal_group(ShardIdFull shard, CatchainSeqno cc_seqno); + void process_accepted_nonfinal_block(BlockIdExt block_id, CatchainSeqno cc_seqno); + void cleanup_nonfinal_groups(); }; } // namespace validator diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 1a46a72f45..5d79e36952 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -176,6 +176,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool get_parallel_validation() const override { return parallel_validation; } + std::string get_db_event_fifo_path() const override { + return db_event_fifo_path_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -301,6 +304,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_parallel_validation(bool value) override { parallel_validation = value; } + void set_db_event_fifo_path(std::string value) override { + db_event_fifo_path_ = std::move(value); + } ValidatorManagerOptionsImpl* make_copy() const override { return new ValidatorManagerOptionsImpl(*this); @@ -360,6 +366,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool collator_node_whitelist_enabled_ = false; td::Ref shard_block_verifier_config_{true}; bool parallel_validation = false; + std::string db_event_fifo_path_; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index a2f15f5b58..e5697e7610 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -35,6 +35,7 @@ #include "interfaces/validator-set.h" #include "overlay/overlays.h" #include "td/actor/actor.h" +#include "td/actor/coro_task.h" #include "td/actor/coro_utils.h" #include "ton/ton-types.h" @@ -151,6 +152,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual td::Ref get_collators_list() const = 0; virtual bool check_collator_node_whitelist(adnl::AdnlNodeIdShort id) const = 0; virtual td::Ref get_shard_block_verifier_config() const = 0; + virtual std::string get_db_event_fifo_path() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; @@ -191,6 +193,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_collator_node_whitelist_enabled(bool enabled) = 0; virtual void set_shard_block_verifier_config(td::Ref config) = 0; virtual void set_parallel_validation(bool value) = 0; + virtual void set_db_event_fifo_path(std::string value) = 0; static td::Ref create(BlockIdExt zero_block_id, BlockIdExt init_block_id, bool allow_blockchain_init = false, double sync_blocks_before = 3600, @@ -309,7 +312,10 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void new_ihr_message(td::BufferSlice data) = 0; virtual void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; - virtual void new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) = 0; + virtual td::actor::Task<> new_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) { + co_return td::Unit{}; + } virtual void add_ext_server_id(adnl::AdnlNodeIdShort id) = 0; virtual void add_ext_server_port(td::uint16 port) = 0; From da79f303061ec399075c0856df7599056d1cfabc Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Sat, 20 Dec 2025 06:12:49 -0500 Subject: [PATCH 515/524] Introduce td::Formattable and use it to gate some operator<< overloads (#1976) * Use C++20 in example/android * Introduce td::Formattable and use it to gate some operator<< overloads This helps clangd to issue error at call site when you try to format a type without a suitable formatter. --- crypto/common/refcnt.hpp | 2 +- example/android/CMakeLists.txt | 4 ++-- tdutils/td/utils/Status.h | 7 ++++--- tdutils/td/utils/StringBuilder.h | 5 +++++ tdutils/td/utils/format.h | 6 +++--- tdutils/td/utils/logging.h | 2 +- 6 files changed, 16 insertions(+), 10 deletions(-) diff --git a/crypto/common/refcnt.hpp b/crypto/common/refcnt.hpp index f31a0faa87..4a27d269e0 100644 --- a/crypto/common/refcnt.hpp +++ b/crypto/common/refcnt.hpp @@ -363,7 +363,7 @@ Ref> make_cnt_ref(Args&&... args) { return Ref>{true, std::forward(args)...}; } -template +template td::StringBuilder& operator<<(td::StringBuilder& sb, const Ref& ref) { if (ref.is_null()) { return sb << "nullptr"; diff --git a/example/android/CMakeLists.txt b/example/android/CMakeLists.txt index b0f998a8b3..3f7818e6b4 100644 --- a/example/android/CMakeLists.txt +++ b/example/android/CMakeLists.txt @@ -10,9 +10,9 @@ project(TON_ANDROID VERSION 0.5 LANGUAGES C CXX) option(TONLIB_ENABLE_JNI "Enable JNI-compatible TonLib API" ON) if (${CMAKE_SYSTEM_NAME} MATCHES "Windows") - string(APPEND CMAKE_CXX_FLAGS " -std=c++14 -fno-omit-frame-pointer -ffunction-sections -fdata-sections") + string(APPEND CMAKE_CXX_FLAGS " -std=c++20 -fno-omit-frame-pointer -ffunction-sections -fdata-sections") else() - string(APPEND CMAKE_CXX_FLAGS " -std=c++14 -Wall -Wextra -Wno-unused-parameter -Wno-deprecated-declarations -Wconversion -Wno-sign-conversion -fno-omit-frame-pointer -ffunction-sections -fdata-sections") + string(APPEND CMAKE_CXX_FLAGS " -std=c++20 -Wall -Wextra -Wno-unused-parameter -Wno-deprecated-declarations -Wconversion -Wno-sign-conversion -fno-omit-frame-pointer -ffunction-sections -fdata-sections") endif() if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin") string(APPEND CMAKE_SHARED_LINKER_FLAGS " -Wl,--gc-sections -Wl,--exclude-libs,ALL") diff --git a/tdutils/td/utils/Status.h b/tdutils/td/utils/Status.h index d43e4fd97d..b154d9074f 100644 --- a/tdutils/td/utils/Status.h +++ b/tdutils/td/utils/Status.h @@ -443,6 +443,10 @@ class Status { } }; +inline StringBuilder &operator<<(StringBuilder &string_builder, const Status &status) { + return status.print(string_builder); +} + // Forward declarations for Result wrappers template struct ResultUnwrap; @@ -643,9 +647,6 @@ inline Result::Result(Status &&status) : status_(std::move(status)) { // no assert } -inline StringBuilder &operator<<(StringBuilder &string_builder, const Status &status) { - return status.print(string_builder); -} template StringBuilder &operator<<(StringBuilder &sb, const Result &result) { if (result.is_ok()) { diff --git a/tdutils/td/utils/StringBuilder.h b/tdutils/td/utils/StringBuilder.h index 8f48f864fd..769a3f1cc0 100644 --- a/tdutils/td/utils/StringBuilder.h +++ b/tdutils/td/utils/StringBuilder.h @@ -140,6 +140,11 @@ class StringBuilder { bool reserve_inner(size_t size); }; +template +concept Formattable = requires(td::StringBuilder &builder, const T &other) { + { builder << other } -> std::same_as; +}; + template std::enable_if_t::value, string> to_string(const T &x) { const size_t buf_size = 1000; diff --git a/tdutils/td/utils/format.h b/tdutils/td/utils/format.h index 4960a2da80..c79bb37883 100644 --- a/tdutils/td/utils/format.h +++ b/tdutils/td/utils/format.h @@ -339,16 +339,16 @@ Lambda lambda(const LambdaT &lambda) { using format::tag; -template +template StringBuilder &operator<<(StringBuilder &sb, const std::pair &p) { return sb << "[" << p.first << ";" << p.second << "]"; } -template +template StringBuilder &operator<<(StringBuilder &stream, const vector &vec) { return stream << format::as_array(vec); } -template +template StringBuilder &operator<<(StringBuilder &stream, const std::set &vec) { return stream << format::as_array(vec); } diff --git a/tdutils/td/utils/logging.h b/tdutils/td/utils/logging.h index f8580d44f6..9993cc95e8 100644 --- a/tdutils/td/utils/logging.h +++ b/tdutils/td/utils/logging.h @@ -259,7 +259,7 @@ class Logger { Logger(LogInterface &log, const LogOptions &options, int log_level, Slice file_name, int line_num, Slice comment); - template + template Logger &operator<<(const T &other) { sb_ << other; return *this; From 2384d662faf76d59a8e123c7110cbcdbaedaf231 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Sun, 21 Dec 2025 13:47:27 +0300 Subject: [PATCH 516/524] Coroutines fixes and improvements; add libbacktrace (#1977) * framed_{write,read}: dynamic max_message_size * experimental libbacktrace support * do not build git target if it is already exists * coroutines: detach now logs failed coroutines by default * coroutines use-after-free bugfix * Context.h: get returns reference and CHECKS that current context is not null * fix CE * fix co_return {} * clang format * add libbacktrace as submodule --------- Co-authored-by: birydrad <> --- .gitmodules | 3 + CMake/BuildLibbacktrace.cmake | 110 ++++++++ CMakeLists.txt | 24 +- adnl/adnl-ext-connection.hpp | 6 +- crypto/test/test-db.cpp | 6 +- crypto/util/mintless-proof-generator.cpp | 2 +- http/http-connection.h | 6 +- lite-client/lite-client.h | 2 +- rldp-http-proxy/rldp-http-proxy.cpp | 6 +- storage/storage-cli.cpp | 2 +- storage/test/storage.cpp | 4 +- tdactor/benchmark/benchmark-coro.cpp | 4 +- tdactor/benchmark/benchmark.cpp | 22 +- tdactor/example/actor-example-coroutines.cpp | 6 +- tdactor/td/actor/ActorId.h | 6 +- tdactor/td/actor/ActorShared.h | 2 +- tdactor/td/actor/ActorStats.cpp | 2 +- tdactor/td/actor/common.h | 16 +- tdactor/td/actor/core/Actor.h | 10 +- tdactor/td/actor/core/ActorExecutor.cpp | 3 + tdactor/td/actor/core/Context.h | 6 +- tdactor/td/actor/core/CpuWorker.cpp | 2 +- tdactor/td/actor/core/IoWorker.cpp | 12 +- tdactor/td/actor/core/Scheduler.cpp | 2 +- tdactor/td/actor/core/Scheduler.h | 6 +- tdactor/td/actor/coro_executor.h | 6 +- tdactor/td/actor/coro_task.h | 30 ++- tdactor/test/actors_core.cpp | 46 ++-- tdactor/test/actors_promise.cpp | 4 +- tdactor/test/test-coro.cpp | 112 +++++++- tddb/test/binlog.cpp | 4 +- tddb/test/io-bench.cpp | 6 +- tddb/test/key_value.cpp | 4 +- tdnet/example/tcp_ping_pong.cpp | 6 +- tdnet/example/udp_ping_pong.cpp | 2 +- tdnet/td/net/FdListener.cpp | 4 +- tdnet/td/net/FramedPipe.cpp | 13 +- tdnet/td/net/FramedPipe.h | 8 +- tdnet/td/net/Pipe.cpp | 6 +- tdnet/td/net/TcpListener.cpp | 6 +- tdnet/td/net/UdpServer.cpp | 10 +- tdnet/test/net-test.cpp | 2 +- tdutils/CMakeLists.txt | 16 ++ tdutils/td/utils/optional.h | 8 + tdutils/td/utils/port/signals.cpp | 1 + tdutils/td/utils/port/stacktrace.cpp | 247 ++++++++++++++++++ tdutils/td/utils/port/stacktrace.h | 1 + terminal/terminal.cpp | 6 +- third-party/libbacktrace | 1 + tonlib/tonlib/Client.cpp | 2 +- tonlib/tonlib/tonlib-cli.cpp | 2 +- .../validator-engine-console.h | 2 +- validator/validator-telemetry.cpp | 4 +- 53 files changed, 682 insertions(+), 147 deletions(-) create mode 100644 CMake/BuildLibbacktrace.cmake create mode 160000 third-party/libbacktrace diff --git a/.gitmodules b/.gitmodules index 4b1d0203da..241489812c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -21,3 +21,6 @@ [submodule "third-party/tl-parser"] path = third-party/tl-parser url = https://github.com/ton-blockchain/tl-parser.git +[submodule "third-party/libbacktrace"] + path = third-party/libbacktrace + url = https://github.com/ianlancetaylor/libbacktrace.git diff --git a/CMake/BuildLibbacktrace.cmake b/CMake/BuildLibbacktrace.cmake new file mode 100644 index 0000000000..d8e569db7b --- /dev/null +++ b/CMake/BuildLibbacktrace.cmake @@ -0,0 +1,110 @@ +# Build libbacktrace from source +# libbacktrace is a library from GCC for getting backtraces with file and line information +# +# Usage options: +# 1. Use pre-built library: +# set(LIBBACKTRACE_LIBRARY "/path/to/libbacktrace.a") +# set(LIBBACKTRACE_INCLUDE_DIR "/path/to/include") +# 2. Build from submodule (default): uses third-party/libbacktrace + +include(ExternalProject) + +# Skip libbacktrace on Android - it doesn't work well with Android's unwinder +if(ANDROID) + message(STATUS "Skipping libbacktrace on Android (not supported)") + set(LIBBACKTRACE_FOUND FALSE CACHE BOOL "libbacktrace found" FORCE) + return() +endif() + +# Check if user provided pre-built library +if(LIBBACKTRACE_LIBRARY AND LIBBACKTRACE_INCLUDE_DIR) + if(EXISTS "${LIBBACKTRACE_LIBRARY}" AND EXISTS "${LIBBACKTRACE_INCLUDE_DIR}/backtrace.h") + message(STATUS "Using pre-built libbacktrace: ${LIBBACKTRACE_LIBRARY}") + add_library(libbacktrace STATIC IMPORTED GLOBAL) + set_target_properties(libbacktrace PROPERTIES + IMPORTED_LOCATION "${LIBBACKTRACE_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${LIBBACKTRACE_INCLUDE_DIR}" + ) + set(LIBBACKTRACE_FOUND TRUE CACHE BOOL "libbacktrace found" FORCE) + return() + endif() +endif() + +# Set default paths for building +if(NOT LIBBACKTRACE_ROOT_DIR) + set(LIBBACKTRACE_ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/libbacktrace") +endif() + +if(NOT LIBBACKTRACE_INCLUDE_DIR) + set(LIBBACKTRACE_INCLUDE_DIR "${LIBBACKTRACE_ROOT_DIR}/include") +endif() + +if(NOT LIBBACKTRACE_LIBRARY) + set(LIBBACKTRACE_LIBRARY "${LIBBACKTRACE_ROOT_DIR}/lib/libbacktrace.a") +endif() + +# Create directories early to avoid CMake errors +file(MAKE_DIRECTORY "${LIBBACKTRACE_INCLUDE_DIR}") +file(MAKE_DIRECTORY "${LIBBACKTRACE_ROOT_DIR}/lib") + +# Use submodule path if not overridden +if(NOT LIBBACKTRACE_SOURCE_DIR) + set(LIBBACKTRACE_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third-party/libbacktrace") +endif() + +message(STATUS "Building libbacktrace from: ${LIBBACKTRACE_SOURCE_DIR}") + +# Build configure arguments - handle cross-compilation +set(LIBBACKTRACE_CONFIGURE_ARGS + --prefix= + --enable-static + --disable-shared + --with-pic +) + +# For cross-compilation, pass the host triplet +if(CMAKE_CROSSCOMPILING AND CMAKE_C_COMPILER_TARGET) + list(APPEND LIBBACKTRACE_CONFIGURE_ARGS --host=${CMAKE_C_COMPILER_TARGET}) +endif() + +# Set up environment for configure +set(LIBBACKTRACE_ENV + CC=${CMAKE_C_COMPILER} +) + +# For cross-compilation, also set AR and RANLIB if available +if(CMAKE_AR) + list(APPEND LIBBACKTRACE_ENV AR=${CMAKE_AR}) +endif() +if(CMAKE_RANLIB) + list(APPEND LIBBACKTRACE_ENV RANLIB=${CMAKE_RANLIB}) +endif() + +# Pass C flags (important for cross-compilation sysroot, etc.) +if(CMAKE_C_FLAGS) + list(APPEND LIBBACKTRACE_ENV "CFLAGS=${CMAKE_C_FLAGS}") +endif() + +ExternalProject_Add( + libbacktrace_external + PREFIX "${LIBBACKTRACE_ROOT_DIR}" + SOURCE_DIR "${LIBBACKTRACE_SOURCE_DIR}" + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${LIBBACKTRACE_ENV} + /configure ${LIBBACKTRACE_CONFIGURE_ARGS} + BUILD_COMMAND make + INSTALL_COMMAND make install + BUILD_BYPRODUCTS "${LIBBACKTRACE_LIBRARY}" +) + +# Create imported target +add_library(libbacktrace STATIC IMPORTED GLOBAL) +set_target_properties(libbacktrace PROPERTIES + IMPORTED_LOCATION "${LIBBACKTRACE_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${LIBBACKTRACE_INCLUDE_DIR}" +) + +# Make libbacktrace depend on the external project +add_dependencies(libbacktrace libbacktrace_external) + +set(LIBBACKTRACE_FOUND TRUE CACHE BOOL "libbacktrace found" FORCE) + diff --git a/CMakeLists.txt b/CMakeLists.txt index 2cf917f5c1..afbcbe6762 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,14 +4,17 @@ project(TON VERSION 0.5 LANGUAGES C CXX) set(CMAKE_POSITION_INDEPENDENT_CODE ON) # Define the two required variables before including the source code for watching a git repository. -set(PRE_CONFIGURE_FILE "git.cc.in") -set(POST_CONFIGURE_FILE "${CMAKE_CURRENT_BINARY_DIR}/git.cc") -include(git_watcher.cmake) +# Only set up git tracking if not already done at a higher level +if(NOT TARGET git) + set(PRE_CONFIGURE_FILE "git.cc.in") + set(POST_CONFIGURE_FILE "${CMAKE_CURRENT_BINARY_DIR}/git.cc") + include(git_watcher.cmake) -# Create a library out of the compiled post-configure file. -add_library(git STATIC ${POST_CONFIGURE_FILE}) -target_include_directories(git PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -add_dependencies(git check_git) + # Create a library out of the compiled post-configure file. + add_library(git STATIC ${POST_CONFIGURE_FILE}) + target_include_directories(git PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) + add_dependencies(git check_git) +endif() # Prevent in-source build get_filename_component(TON_REAL_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" REALPATH) @@ -194,6 +197,7 @@ set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH}) include(BuildBLST) include(BuildSECP256K1) +include(BuildLibbacktrace) # Configure CCache if available find_program(CCACHE_FOUND ccache) @@ -276,6 +280,12 @@ elseif (CLANG OR GCC) #use "-Wl,-exported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/export_list" for exported symbols set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fvisibility=hidden -Wl,-dead_strip,-x,-S") set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} -fvisibility=hidden -Wl,-dead_strip,-x,-S") + # Generate dSYM bundles for proper stack traces with libbacktrace (for Ninja/Make generators) + if(NOT DSYMUTIL_LINK_CONFIGURED AND NOT CMAKE_GENERATOR MATCHES "Xcode" AND CMAKE_BUILD_TYPE MATCHES "Debug|RelWithDebInfo") + set(DSYMUTIL_LINK_CONFIGURED TRUE CACHE INTERNAL "dSYM generation configured in linker") + set(CMAKE_C_LINK_EXECUTABLE "${CMAKE_C_LINK_EXECUTABLE} && dsymutil ") + set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} && dsymutil ") + endif() else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffunction-sections -fdata-sections") if (NOT USE_EMSCRIPTEN) diff --git a/adnl/adnl-ext-connection.hpp b/adnl/adnl-ext-connection.hpp index 484f2611c0..a801f62399 100644 --- a/adnl/adnl-ext-connection.hpp +++ b/adnl/adnl-ext-connection.hpp @@ -98,8 +98,8 @@ class AdnlExtConnection : public td::actor::Actor, public td::ObserverBase { self_ = actor_id(this); // Subscribe for socket updates // NB: Interface will be changed - td::actor::SchedulerContext::get()->get_poll().subscribe(buffered_fd_.get_poll_info().extract_pollable_fd(this), - td::PollFlags::ReadWrite()); + td::actor::SchedulerContext::get().get_poll().subscribe(buffered_fd_.get_poll_info().extract_pollable_fd(this), + td::PollFlags::ReadWrite()); update_timer(); notify(); } @@ -127,7 +127,7 @@ class AdnlExtConnection : public td::actor::Actor, public td::ObserverBase { } // unsubscribe from socket updates // nb: interface will be changed - td::actor::SchedulerContext::get()->get_poll().unsubscribe(buffered_fd_.get_poll_info().get_pollable_fd_ref()); + td::actor::SchedulerContext::get().get_poll().unsubscribe(buffered_fd_.get_poll_info().get_pollable_fd_ref()); } void update_timer() { diff --git a/crypto/test/test-db.cpp b/crypto/test/test-db.cpp index 443287a9a6..724238390f 100644 --- a/crypto/test/test-db.cpp +++ b/crypto/test/test-db.cpp @@ -80,7 +80,7 @@ class ActorExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { class Worker : public td::actor::Actor { public: void close() { - td::actor::core::SchedulerContext::get()->stop(); + td::actor::core::SchedulerContext::get().stop(); stop(); } void execute_sync(std::function f) { @@ -101,7 +101,7 @@ class ActorExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { private: std::function f_; }; - auto context = td::actor::SchedulerContext::get(); + auto context = td::actor::SchedulerContext::get_ptr(); if (context) { td::actor::create_actor("executeasync", std::move(f)).release(); } else { @@ -111,7 +111,7 @@ class ActorExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { } void execute_sync(std::function f) override { - auto context = td::actor::SchedulerContext::get(); + auto context = td::actor::SchedulerContext::get_ptr(); if (context) { td::actor::send_closure(worker_, &Worker::execute_sync, std::move(f)); } else { diff --git a/crypto/util/mintless-proof-generator.cpp b/crypto/util/mintless-proof-generator.cpp index e4dbe037df..8f2b243a96 100644 --- a/crypto/util/mintless-proof-generator.cpp +++ b/crypto/util/mintless-proof-generator.cpp @@ -283,7 +283,7 @@ class MakeAllProofsActor : public td::actor::core::Actor { LOG_CHECK(!out_file_.fail()) << "Failed to write to " << out_filename_; LOG(INFO) << "Written " << written_count_ << " entries to " << out_filename_; stop(); - td::actor::SchedulerContext::get()->stop(); + td::actor::SchedulerContext::get().stop(); } } diff --git a/http/http-connection.h b/http/http-connection.h index 8db50884c4..08e1371bc9 100644 --- a/http/http-connection.h +++ b/http/http-connection.h @@ -101,8 +101,8 @@ class HttpConnection : public td::actor::Actor, public td::ObserverBase { self_ = actor_id(this); // Subscribe for socket updates // NB: Interface will be changed - td::actor::SchedulerContext::get()->get_poll().subscribe(buffered_fd_.get_poll_info().extract_pollable_fd(this), - td::PollFlags::ReadWrite() | td::PollFlags::Close()); + td::actor::SchedulerContext::get().get_poll().subscribe(buffered_fd_.get_poll_info().extract_pollable_fd(this), + td::PollFlags::ReadWrite() | td::PollFlags::Close()); notify(); } @@ -130,7 +130,7 @@ class HttpConnection : public td::actor::Actor, public td::ObserverBase { } // unsubscribe from socket updates // nb: interface will be changed - td::actor::SchedulerContext::get()->get_poll().unsubscribe(buffered_fd_.get_poll_info().get_pollable_fd_ref()); + td::actor::SchedulerContext::get().get_poll().unsubscribe(buffered_fd_.get_poll_info().get_pollable_fd_ref()); buffered_fd_.close(); } }; diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 7353edf453..945eed931e 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -454,7 +454,7 @@ class TestNode : public td::actor::Actor { } void tear_down() override { // FIXME: do not work in windows - //td::actor::SchedulerContext::get()->stop(); + //td::actor::SchedulerContext::get().stop(); } void got_result(td::Result R, td::Promise promise); diff --git a/rldp-http-proxy/rldp-http-proxy.cpp b/rldp-http-proxy/rldp-http-proxy.cpp index 792da10ec1..f7e4fbdddb 100644 --- a/rldp-http-proxy/rldp-http-proxy.cpp +++ b/rldp-http-proxy/rldp-http-proxy.cpp @@ -602,7 +602,7 @@ class RldpTcpTunnel : public td::actor::Actor, private td::ObserverBase { void tear_down() override { LOG(INFO) << "RldpTcpTunnel: tear_down"; - td::actor::SchedulerContext::get()->get_poll().unsubscribe(fd_.get_poll_info().get_pollable_fd_ref()); + td::actor::SchedulerContext::get().get_poll().unsubscribe(fd_.get_poll_info().get_pollable_fd_ref()); } void registered_sender(RegisteredPayloadSenderGuard guard) { @@ -1577,8 +1577,8 @@ void HttpRldpPayloadSender::start_up() { void RldpTcpTunnel::start_up() { self_ = actor_id(this); - td::actor::SchedulerContext::get()->get_poll().subscribe(fd_.get_poll_info().extract_pollable_fd(this), - td::PollFlags::ReadWrite()); + td::actor::SchedulerContext::get().get_poll().subscribe(fd_.get_poll_info().extract_pollable_fd(this), + td::PollFlags::ReadWrite()); td::actor::send_closure( proxy_, &RldpHttpProxy::register_payload_sender, id_, [SelfId = actor_id(this)](ton::tl_object_ptr f, diff --git a/storage/storage-cli.cpp b/storage/storage-cli.cpp index de0be0eeff..a80e32833b 100644 --- a/storage/storage-cli.cpp +++ b/storage/storage-cli.cpp @@ -604,7 +604,7 @@ class StorageCli : public td::actor::Actor { } void tear_down() override { - td::actor::SchedulerContext::get()->stop(); + td::actor::SchedulerContext::get().stop(); } }; diff --git a/storage/test/storage.cpp b/storage/test/storage.cpp index 08846d7f5e..5c4d9f3b3c 100644 --- a/storage/test/storage.cpp +++ b/storage/test/storage.cpp @@ -655,7 +655,7 @@ struct RldpBasicTest { void on_closed() { cnt_--; if (cnt_ == 0) { - td::actor::SchedulerContext::get()->stop(); + td::actor::SchedulerContext::get().stop(); //LOG(ERROR) << "STOP"; stop(); } @@ -1318,7 +1318,7 @@ TEST(Torrent, Peer) { auto info = torrent.get_info(); - auto stop_watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get()->stop(); }); + auto stop_watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get().stop(); }); auto guard = std::make_shared>>(); auto complete_watcher = td::create_shared_destructor([guard] {}); diff --git a/tdactor/benchmark/benchmark-coro.cpp b/tdactor/benchmark/benchmark-coro.cpp index 98af490fe0..f3f31c9d9d 100644 --- a/tdactor/benchmark/benchmark-coro.cpp +++ b/tdactor/benchmark/benchmark-coro.cpp @@ -17,7 +17,7 @@ class CoroBenchmark final : public td::actor::Actor { public: void start_up() override { db_ = td::actor::create_actor("BenchmarkDatabase").release(); - flow().start_immediate().detach(); + flow().start_immediate().detach("BenchmakrDatabase"); } Task run_benchmarks() { @@ -107,7 +107,7 @@ class CoroBenchmark final : public td::actor::Actor { LOG(INFO) << "Starting benchmarks"; (void)co_await run_benchmarks(); LOG(INFO) << "Benchmarks completed"; - td::actor::SchedulerContext::get()->stop(); + td::actor::SchedulerContext::get().stop(); stop(); co_return td::Unit(); } diff --git a/tdactor/benchmark/benchmark.cpp b/tdactor/benchmark/benchmark.cpp index a65cf34bb9..9420e707c6 100644 --- a/tdactor/benchmark/benchmark.cpp +++ b/tdactor/benchmark/benchmark.cpp @@ -378,7 +378,7 @@ class ActorExecutorBenchmark : public td::Benchmark { //{ //ActorExecutor executor(*actor, dispatcher, ActorExecutor::Options()); //executor.send( - //ActorMessageCreator::lambda([&] { static_cast(ActorExecuteContext::get()->actor()).close(); })); + //ActorMessageCreator::lambda([&] { static_cast(ActorExecuteContext::get().actor()).close(); })); //} dispatcher.queue.clear(); } @@ -435,7 +435,7 @@ class ActorSignalQuery : public td::Benchmark { Scheduler scheduler{{threads_count}}; scheduler.run_in_context([&] { - auto watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get()->stop(); }); + auto watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get().stop(); }); create_actor(ActorOptions().with_name(PSLICE() << "Master"), watcher, n).release(); }); @@ -492,7 +492,7 @@ class ActorQuery : public td::Benchmark { Scheduler scheduler({threads_count}); scheduler.run_in_context([&] { - auto watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get()->stop(); }); + auto watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get().stop(); }); create_actor(ActorOptions().with_name(PSLICE() << "Master"), watcher, n).release(); }); @@ -548,7 +548,7 @@ class ActorDummyQuery : public td::Benchmark { Scheduler scheduler({threads_count}); scheduler.run_in_context([&] { - auto watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get()->stop(); }); + auto watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get().stop(); }); create_actor(ActorOptions().with_name(PSLICE() << "Master"), watcher, n).release(); }); @@ -607,7 +607,7 @@ class ActorTaskQuery : public td::Benchmark { Scheduler scheduler({threads_count}); scheduler.run_in_context([&] { - auto watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get()->stop(); }); + auto watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get().stop(); }); create_actor(ActorOptions().with_name(PSLICE() << "Master"), watcher, n).release(); }); @@ -626,7 +626,7 @@ class BlockSha256Actors { Scheduler scheduler({threads_count}); scheduler.run_in_context([&] { - auto watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get()->stop(); }); + auto watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get().stop(); }); class Worker : public td::actor::Actor { public: Worker(std::shared_ptr watcher, td::Promise<> promise) @@ -1434,7 +1434,7 @@ class ChainedSpawn : public td::Benchmark { .release(); sem.wait(); } - td::actor::SchedulerContext::get()->stop(); + td::actor::SchedulerContext::get().stop(); }); sch.join(); @@ -1481,7 +1481,7 @@ class ChainedSpawnInplace : public td::Benchmark { .release(); sem.wait(); } - td::actor::SchedulerContext::get()->stop(); + td::actor::SchedulerContext::get().stop(); }); sch.join(); @@ -1537,7 +1537,7 @@ class PingPong : public td::Benchmark { send_closure(a, &Task::ping, 1000); sem.wait(2); } - td::actor::SchedulerContext::get()->stop(); + td::actor::SchedulerContext::get().stop(); }); sch.join(); @@ -1579,7 +1579,7 @@ class SpawnMany : public td::Benchmark { } sem.wait(spawn_cnt); } - td::actor::SchedulerContext::get()->stop(); + td::actor::SchedulerContext::get().stop(); }); sch.join(); } @@ -1632,7 +1632,7 @@ class YieldMany : public td::Benchmark { } }); - scheduler.run_in_context_external([&] { td::actor::SchedulerContext::get()->stop(); }); + scheduler.run_in_context_external([&] { td::actor::SchedulerContext::get().stop(); }); sch.join(); } diff --git a/tdactor/example/actor-example-coroutines.cpp b/tdactor/example/actor-example-coroutines.cpp index d40d446404..5b58bebbea 100644 --- a/tdactor/example/actor-example-coroutines.cpp +++ b/tdactor/example/actor-example-coroutines.cpp @@ -115,7 +115,7 @@ Task example_actor() { } void start_up() override { // it is usual actor all coroutines create FROM actor, will be executed ON actor - run().start().detach(); + run().start().detach("example_actor"); } Task run() { state_ = 19; @@ -233,7 +233,7 @@ Task example_echo_server() { void accept(td::SocketFd fd) override { LOG(INFO) << "Accepting new connection"; auto pipe = td::make_socket_pipe(std::move(fd)); - spawn_task_actor("echo_conn", std::move(pipe)).detach(); + spawn_task_actor("echo_conn", std::move(pipe)).detach("EchoConnection"); } }; @@ -338,7 +338,7 @@ Task example() { (co_await run_all_examples().wrap()).ensure(); LOG(INFO) << "Finish example coroutine and stop scheduler"; - td::actor::SchedulerContext::get()->stop(); + td::actor::SchedulerContext::get().stop(); co_return td::Unit(); } diff --git a/tdactor/td/actor/ActorId.h b/tdactor/td/actor/ActorId.h index 003fc70488..a9489eaa1d 100644 --- a/tdactor/td/actor/ActorId.h +++ b/tdactor/td/actor/ActorId.h @@ -116,12 +116,12 @@ namespace core { // for ADL template ActorId actor_id(SelfT *self) { CHECK(self); - CHECK(static_cast(self) == &core::ActorExecuteContext::get()->actor()); - return ActorId(core::ActorExecuteContext::get()->actor().get_actor_info_ptr()); + CHECK(static_cast(self) == &core::ActorExecuteContext::get().actor()); + return ActorId(core::ActorExecuteContext::get().actor().get_actor_info_ptr()); } inline ActorId<> actor_id() { - return actor_id(&core::ActorExecuteContext::get()->actor()); + return actor_id(&core::ActorExecuteContext::get().actor()); } } // namespace core using core::actor_id; diff --git a/tdactor/td/actor/ActorShared.h b/tdactor/td/actor/ActorShared.h index 48f793a2f0..654e161df7 100644 --- a/tdactor/td/actor/ActorShared.h +++ b/tdactor/td/actor/ActorShared.h @@ -112,7 +112,7 @@ ActorShared actor_shared(SelfT *self, uint64 id = static_cast(-1) } inline ActorShared<> actor_shared() { - return actor_shared(&core::ActorExecuteContext::get()->actor()); + return actor_shared(&core::ActorExecuteContext::get().actor()); } } // namespace core using core::actor_shared; diff --git a/tdactor/td/actor/ActorStats.cpp b/tdactor/td/actor/ActorStats.cpp index 79f7a6016c..b4ab1e5f8b 100644 --- a/tdactor/td/actor/ActorStats.cpp +++ b/tdactor/td/actor/ActorStats.cpp @@ -209,7 +209,7 @@ std::string ActorStats::prepare_stats() { }; std::sort(stats.begin(), stats.end(), [&](auto &left, auto &right) { return main_key(left.first) > main_key(right.first); }); - auto debug = Debug(SchedulerContext::get()->scheduler_group()); + auto debug = Debug(SchedulerContext::get().scheduler_group()); debug.dump(sb); sb << "All actors:\n"; for (auto &it : stats) { diff --git a/tdactor/td/actor/common.h b/tdactor/td/actor/common.h index aa0072e1a1..80d6af923a 100644 --- a/tdactor/td/actor/common.h +++ b/tdactor/td/actor/common.h @@ -261,11 +261,11 @@ struct ActorRef { template T ¤t_actor() { - return static_cast(core::ActorExecuteContext::get()->actor()); + return static_cast(core::ActorExecuteContext::get().actor()); } inline void send_message(core::ActorInfo &actor_info, core::ActorMessage message) { - auto scheduler_context_ptr = core::SchedulerContext::get(); + auto scheduler_context_ptr = core::SchedulerContext::get_ptr(); if (scheduler_context_ptr == nullptr) { //LOG(ERROR) << "send to actor is silently ignored"; return; @@ -282,7 +282,7 @@ inline void send_message(ActorRef actor_ref, core::ActorMessage message) { } inline void send_message_later(core::ActorInfo &actor_info, core::ActorMessage message) { - auto scheduler_context_ptr = core::SchedulerContext::get(); + auto scheduler_context_ptr = core::SchedulerContext::get_ptr(); if (scheduler_context_ptr == nullptr) { //LOG(ERROR) << "send to actor is silently ignored"; return; @@ -307,7 +307,7 @@ auto run_on_current_actor(ClosureT &&closure) { template void send_immediate(ActorRef actor_ref, ExecuteF &&execute, ToMessageF &&to_message) { - auto scheduler_context_ptr = core::SchedulerContext::get(); + auto scheduler_context_ptr = core::SchedulerContext::get_ptr(); if (scheduler_context_ptr == nullptr) { //LOG(ERROR) << "send to actor is silently ignored"; return; @@ -433,7 +433,7 @@ void send_closure_later(ActorRef actor_ref, ArgsT &&...args) { } inline void send_signals(ActorRef actor_ref, ActorSignals signals) { - auto scheduler_context_ptr = core::SchedulerContext::get(); + auto scheduler_context_ptr = core::SchedulerContext::get_ptr(); if (scheduler_context_ptr == nullptr) { //LOG(ERROR) << "send to actor is silently ignored"; return; @@ -445,7 +445,7 @@ inline void send_signals(ActorRef actor_ref, ActorSignals signals) { } inline void send_signals_later(ActorRef actor_ref, ActorSignals signals) { - auto scheduler_context_ptr = core::SchedulerContext::get(); + auto scheduler_context_ptr = core::SchedulerContext::get_ptr(); if (scheduler_context_ptr == nullptr) { //LOG(ERROR) << "send to actor is silently ignored"; return; @@ -460,12 +460,12 @@ inline void send_signals_later(ActorRef actor_ref, ActorSignals signals) { inline void register_actor_info_ptr(core::ActorInfoPtr actor_info_ptr) { auto state = actor_info_ptr->state().get_flags_unsafe(); actor_info_ptr->on_add_to_queue(); - core::SchedulerContext::get()->add_to_queue(std::move(actor_info_ptr), state.get_scheduler_id(), !state.is_shared()); + core::SchedulerContext::get().add_to_queue(std::move(actor_info_ptr), state.get_scheduler_id(), !state.is_shared()); } template core::ActorInfoPtr create_actor(core::ActorOptions &options, ArgsT &&...args) noexcept { - auto *scheduler_context = core::SchedulerContext::get(); + auto *scheduler_context = core::SchedulerContext::get_ptr(); if (!options.has_scheduler()) { options.on_scheduler(scheduler_context->get_scheduler_id()); } diff --git a/tdactor/td/actor/core/Actor.h b/tdactor/td/actor/core/Actor.h index 0c9a00c2c9..eb6c0250c7 100644 --- a/tdactor/td/actor/core/Actor.h +++ b/tdactor/td/actor/core/Actor.h @@ -70,23 +70,23 @@ class Actor { // Useful functions void yield() { // send wakeup signal to itself - ActorExecuteContext::get()->set_yield(); + ActorExecuteContext::get().set_yield(); } void stop() { // send Kill signal to itself - ActorExecuteContext::get()->set_stop(); + ActorExecuteContext::get().set_stop(); } Timestamp &alarm_timestamp() { - return ActorExecuteContext::get()->alarm_timestamp(); + return ActorExecuteContext::get().alarm_timestamp(); } Timestamp get_alarm_timestamp() { - return ActorExecuteContext::get()->get_alarm_timestamp(); + return ActorExecuteContext::get().get_alarm_timestamp(); } CSlice get_name() { return actor_info_ptr_->get_name(); } uint64 get_link_token() { - return ActorExecuteContext::get()->get_link_token(); + return ActorExecuteContext::get().get_link_token(); } //set context that will be inherited by all childrens diff --git a/tdactor/td/actor/core/ActorExecutor.cpp b/tdactor/td/actor/core/ActorExecutor.cpp index cab346cd81..e154a6eebd 100644 --- a/tdactor/td/actor/core/ActorExecutor.cpp +++ b/tdactor/td/actor/core/ActorExecutor.cpp @@ -304,6 +304,9 @@ void ActorExecutor::flush_context_flags() { actor_info_.actor().tear_down(); actor_stats_.destroyed(); } + // Clear actor pointer BEFORE destroy_actor() so any code during + // destruction (actor members, mailbox clear) sees no current actor + actor_execute_context_.clear_actor(); actor_info_.destroy_actor(); } else { if (actor_execute_context_.get_pause()) { diff --git a/tdactor/td/actor/core/Context.h b/tdactor/td/actor/core/Context.h index bb32f66896..71780f5101 100644 --- a/tdactor/td/actor/core/Context.h +++ b/tdactor/td/actor/core/Context.h @@ -26,7 +26,11 @@ namespace core { template class Context { public: - static Impl *get() { + static Impl &get() { + CHECK(context_); + return *context_; + } + static Impl *get_ptr() { return context_; } class Guard { diff --git a/tdactor/td/actor/core/CpuWorker.cpp b/tdactor/td/actor/core/CpuWorker.cpp index 7083b6fef4..4179fe97d1 100644 --- a/tdactor/td/actor/core/CpuWorker.cpp +++ b/tdactor/td/actor/core/CpuWorker.cpp @@ -28,7 +28,7 @@ namespace actor { namespace core { void CpuWorker::run() { auto thread_id = get_thread_id(); - auto &dispatcher = *SchedulerContext::get(); + auto &dispatcher = SchedulerContext::get(); MpmcWaiter::Slot slot; waiter_.init_slot(slot, thread_id); diff --git a/tdactor/td/actor/core/IoWorker.cpp b/tdactor/td/actor/core/IoWorker.cpp index 8238a5cc4c..b40cd3c272 100644 --- a/tdactor/td/actor/core/IoWorker.cpp +++ b/tdactor/td/actor/core/IoWorker.cpp @@ -25,24 +25,24 @@ namespace actor { namespace core { void IoWorker::start_up() { #if TD_PORT_POSIX - auto &poll = SchedulerContext::get()->get_poll(); + auto &poll = SchedulerContext::get().get_poll(); poll.subscribe(queue_.reader_get_event_fd().get_poll_info().extract_pollable_fd(nullptr), PollFlags::Read()); #endif } void IoWorker::tear_down() { #if TD_PORT_POSIX - auto &poll = SchedulerContext::get()->get_poll(); + auto &poll = SchedulerContext::get().get_poll(); poll.unsubscribe(queue_.reader_get_event_fd().get_poll_info().get_pollable_fd_ref()); #endif } bool IoWorker::run_once(double timeout, bool skip_timeouts) { - auto &dispatcher = *SchedulerContext::get(); + auto &dispatcher = SchedulerContext::get(); #if TD_PORT_POSIX - auto &poll = SchedulerContext::get()->get_poll(); + auto &poll = SchedulerContext::get().get_poll(); #endif - auto &heap = SchedulerContext::get()->get_heap(); - auto &debug = SchedulerContext::get()->get_debug(); + auto &heap = SchedulerContext::get().get_heap(); + auto &debug = SchedulerContext::get().get_debug(); auto now = Time::now(); // update Time::now_cached() while (!heap.empty() && heap.top_key() <= now) { diff --git a/tdactor/td/actor/core/Scheduler.cpp b/tdactor/td/actor/core/Scheduler.cpp index 9412cb9232..9221e17542 100644 --- a/tdactor/td/actor/core/Scheduler.cpp +++ b/tdactor/td/actor/core/Scheduler.cpp @@ -101,7 +101,7 @@ void Scheduler::start() { bool Scheduler::run(double timeout) { bool res; run_in_context_impl(*info_->io_worker, [this, timeout, &res] { - if (SchedulerContext::get()->is_stop_requested()) { + if (SchedulerContext::get().is_stop_requested()) { res = false; } else { res = io_worker_->run_once(timeout, skip_timeouts_); diff --git a/tdactor/td/actor/core/Scheduler.h b/tdactor/td/actor/core/Scheduler.h index 461b43dab8..bec6274204 100644 --- a/tdactor/td/actor/core/Scheduler.h +++ b/tdactor/td/actor/core/Scheduler.h @@ -217,7 +217,7 @@ class Scheduler { // Just syntactic sugar void stop() { - run_in_context([] { SchedulerContext::get()->stop(); }); + run_in_context([] { SchedulerContext::get().stop(); }); } SchedulerId get_scheduler_id() const { @@ -297,13 +297,13 @@ class Scheduler { class ActorMessageHangup : public core::ActorMessageImpl { public: void run() override { - ActorExecuteContext::get()->actor().hangup(); + ActorExecuteContext::get().actor().hangup(); } }; class ActorMessageHangupShared : public core::ActorMessageImpl { public: void run() override { - ActorExecuteContext::get()->actor().hangup_shared(); + ActorExecuteContext::get().actor().hangup_shared(); } }; } // namespace core diff --git a/tdactor/td/actor/coro_executor.h b/tdactor/td/actor/coro_executor.h index 765b9ad14f..4bccc18982 100644 --- a/tdactor/td/actor/coro_executor.h +++ b/tdactor/td/actor/coro_executor.h @@ -14,7 +14,7 @@ namespace td::actor { namespace detail { inline ActorId<> get_current_actor_id() noexcept { - auto context = core::ActorExecuteContext::get(); + auto context = core::ActorExecuteContext::get_ptr(); if (context && context->actor_ptr()) { return actor_id(context->actor_ptr()); } @@ -98,9 +98,7 @@ struct SchedulerExecutor { } void schedule(std::coroutine_handle<> cont) noexcept { auto token = reinterpret_cast(encode_continuation(cont)); - auto ctx = td::actor::core::SchedulerContext::get(); - CHECK(ctx); - ctx->add_token_to_cpu_queue(token, td::actor::core::SchedulerId{}); + td::actor::core::SchedulerContext::get().add_token_to_cpu_queue(token, td::actor::core::SchedulerId{}); } }; diff --git a/tdactor/td/actor/coro_task.h b/tdactor/td/actor/coro_task.h index d624bbe5f1..f04cd2e8ea 100644 --- a/tdactor/td/actor/coro_task.h +++ b/tdactor/td/actor/coro_task.h @@ -139,7 +139,9 @@ struct promise_value : promise_common { result = std::forward(v); } - struct ExternalResult {}; + struct ExternalResult { + explicit ExternalResult() = default; + }; void return_value(ExternalResult&&) noexcept { } @@ -163,6 +165,15 @@ struct promise_type : promise_value> { static_assert(!std::is_void_v, "Task is not supported; use Task instead"); using Handle = std::coroutine_handle; + // Bring base class return_value overloads into scope + using promise_value>::return_value; + + // Allow co_return {}; to work by constructing T{} (e.g., Unit{}) + // This fixes a bug where co_return {}; was equivalent to co_return td::Status::Error(-1); + void return_value(T v) noexcept { + this->result = std::move(v); + } + auto self() noexcept { return Handle::from_promise(*this); } @@ -383,9 +394,22 @@ struct [[nodiscard]] StartedTask { StartedTask& operator=(const StartedTask&) = delete; ~StartedTask() noexcept { - detach(); + detach_silent(); } - void detach() { + void detach(std::string description = "UnknownTask") && { + if (!h) { + return; + } + [](auto self, std::string description) -> Task { + co_await become_lightweight(); + auto r = co_await std::move(self).wrap(); + LOG_IF(ERROR, r.is_error()) << "Detached task <" << description << "> failed: " << r.error(); + co_return td::Unit{}; + }(std::move(*this), std::move(description)) + .start_immediate() + .detach_silent(); + } + void detach_silent() { if (!h) { return; } diff --git a/tdactor/test/actors_core.cpp b/tdactor/test/actors_core.cpp index 3eba4d08c7..a9780f86aa 100644 --- a/tdactor/test/actors_core.cpp +++ b/tdactor/test/actors_core.cpp @@ -337,7 +337,7 @@ TEST(Actor2, executor_simple) { { ActorExecutor executor(*actor, dispatcher, ActorExecutor::Options()); executor.send( - ActorMessageCreator::lambda([&] { static_cast(ActorExecuteContext::get()->actor()).close(); })); + ActorMessageCreator::lambda([&] { static_cast(ActorExecuteContext::get().actor()).close(); })); } LOG_CHECK(sb.as_cslice() == "TearDown") << sb.as_cslice(); sb.clear(); @@ -345,7 +345,7 @@ TEST(Actor2, executor_simple) { { ActorExecutor executor(*actor, dispatcher, ActorExecutor::Options()); executor.send( - ActorMessageCreator::lambda([&] { static_cast(ActorExecuteContext::get()->actor()).close(); })); + ActorMessageCreator::lambda([&] { static_cast(ActorExecuteContext::get().actor()).close(); })); } CHECK(dispatcher.queue.empty()); CHECK(sb.as_cslice() == ""); @@ -364,7 +364,7 @@ TEST(Actor2, executor_simple) { sb.clear(); auto a_msg = ActorMessageCreator::lambda([&] { sb << "big pause"; - ActorExecuteContext::get()->set_pause(); + ActorExecuteContext::get().set_pause(); }); a_msg.set_big(); executor.send(std::move(a_msg)); @@ -392,7 +392,7 @@ TEST(Actor2, executor_simple) { { ActorExecutor executor(*actor, dispatcher, ActorExecutor::Options()); executor.send( - ActorMessageCreator::lambda([&] { static_cast(ActorExecuteContext::get()->actor()).close(); })); + ActorMessageCreator::lambda([&] { static_cast(ActorExecuteContext::get().actor()).close(); })); } LOG_CHECK(sb.as_cslice() == "TearDown") << sb.as_cslice(); sb.clear(); @@ -428,7 +428,7 @@ class Master : public Actor { l++; if (l == r) { if (!--global_cnt) { - SchedulerContext::get()->stop(); + SchedulerContext::get().stop(); } detail::send_closure(*worker, &Worker::close); stop(); @@ -480,7 +480,7 @@ TEST(Actor2, actor_id_simple) { ~A() { sb << "~A"; if (--global_cnt <= 0) { - SchedulerContext::get()->stop(); + SchedulerContext::get().stop(); } } @@ -521,14 +521,14 @@ TEST(Actor2, actor_creation) { } void check() { - auto &context = *SchedulerContext::get(); + auto &context = SchedulerContext::get(); CHECK(context.has_poll()); context.get_poll(); } void tear_down() override { if (--global_cnt <= 0) { - SchedulerContext::get()->stop(); + SchedulerContext::get().stop(); } } }; @@ -540,14 +540,14 @@ TEST(Actor2, actor_creation) { private: void start_up() override { - auto &context = *SchedulerContext::get(); + auto &context = SchedulerContext::get(); CHECK(!context.has_poll()); send_closure(a_, &A::f); stop(); } void tear_down() override { if (--global_cnt <= 0) { - SchedulerContext::get()->stop(); + SchedulerContext::get().stop(); } } ActorId a_; @@ -567,7 +567,7 @@ TEST(Actor2, actor_timeout_simple) { sb.clear(); scheduler.start(); - auto watcher = td::create_shared_destructor([] { SchedulerContext::get()->stop(); }); + auto watcher = td::create_shared_destructor([] { SchedulerContext::get().stop(); }); scheduler.run_in_context([watcher = std::move(watcher)] { class A : public Actor { public: @@ -612,7 +612,7 @@ TEST(Actor2, actor_timeout_simple2) { sb.clear(); scheduler.start(); - auto watcher = td::create_shared_destructor([] { SchedulerContext::get()->stop(); }); + auto watcher = td::create_shared_destructor([] { SchedulerContext::get().stop(); }); scheduler.run_in_context([watcher = std::move(watcher)] { class A : public Actor { public: @@ -668,7 +668,7 @@ TEST(Actor2, actor_function_result) { sb.clear(); scheduler.start(); - auto watcher = td::create_shared_destructor([] { SchedulerContext::get()->stop(); }); + auto watcher = td::create_shared_destructor([] { SchedulerContext::get().stop(); }); scheduler.run_in_context([watcher = std::move(watcher)] { class B : public Actor { public: @@ -732,7 +732,7 @@ TEST(Actor2, actor_ping_pong) { sb.clear(); scheduler.start(); - auto watcher = td::create_shared_destructor([] { SchedulerContext::get()->stop(); }); + auto watcher = td::create_shared_destructor([] { SchedulerContext::get().stop(); }); td::actor::set_debug(true); for (int i = 0; i < 2000; i++) { scheduler.run_in_context([watcher] { @@ -828,7 +828,7 @@ TEST(Actor2, SchedulerZeroCpuThreads) { scheduler.run_in_context([] { class A : public Actor { void start_up() override { - SchedulerContext::get()->stop(); + SchedulerContext::get().stop(); } }; create_actor(ActorOptions().with_name("A").with_poll(false)).release(); @@ -841,16 +841,16 @@ TEST(Actor2, SchedulerTwo) { class B : public Actor { public: void start_up() override { - CHECK(SchedulerContext::get()->get_scheduler_id() == SchedulerId{1}); + CHECK(SchedulerContext::get().get_scheduler_id() == SchedulerId{1}); } void close() { - CHECK(SchedulerContext::get()->get_scheduler_id() == SchedulerId{1}); - SchedulerContext::get()->stop(); + CHECK(SchedulerContext::get().get_scheduler_id() == SchedulerId{1}); + SchedulerContext::get().stop(); } }; class A : public Actor { void start_up() override { - CHECK(SchedulerContext::get()->get_scheduler_id() == SchedulerId{0}); + CHECK(SchedulerContext::get().get_scheduler_id() == SchedulerId{0}); auto id = create_actor(ActorOptions().with_name("B").with_poll(false).on_scheduler(SchedulerId{1})).release(); send_closure(id, &B::close); @@ -867,7 +867,7 @@ TEST(Actor2, ActorIdDynamicCast) { public: void close() { CHECK(actor_id().actor_info_ptr() == get_actor_info_ptr()); - SchedulerContext::get()->stop(); + SchedulerContext::get().stop(); } }; auto actor_own_a = create_actor(ActorOptions().with_name("A").with_poll(false)); @@ -901,7 +901,7 @@ TEST(Actor2, send_vs_close) { for (int it = 0; it < 100; it++) { Scheduler scheduler({8}); - auto watcher = td::create_shared_destructor([] { SchedulerContext::get()->stop(); }); + auto watcher = td::create_shared_destructor([] { SchedulerContext::get().stop(); }); scheduler.run_in_context([watcher = std::move(watcher)] { class To : public Actor { public: @@ -1035,7 +1035,7 @@ TEST(Actor2, send_vs_close2) { for (int it = 0; it < 100; it++) { Scheduler scheduler({8}); - auto watcher = td::create_shared_destructor([] { SchedulerContext::get()->stop(); }); + auto watcher = td::create_shared_destructor([] { SchedulerContext::get().stop(); }); //std::shared_ptr watcher; scheduler.run_in_context([watcher = std::move(watcher)] { class To : public Actor { @@ -1115,7 +1115,7 @@ TEST(Actor2, test_stats) { Scheduler scheduler({8}); td::actor::set_debug(true); - auto watcher = td::create_shared_destructor([] { SchedulerContext::get()->stop(); }); + auto watcher = td::create_shared_destructor([] { SchedulerContext::get().stop(); }); scheduler.run_in_context([watcher = std::move(watcher)] { class SleepWorker : public Actor { void loop() override { diff --git a/tdactor/test/actors_promise.cpp b/tdactor/test/actors_promise.cpp index b6ee511764..01a48c469d 100644 --- a/tdactor/test/actors_promise.cpp +++ b/tdactor/test/actors_promise.cpp @@ -215,7 +215,7 @@ TEST(Actor2, actor_lost_promise) { auto watcher = td::create_shared_destructor([] { LOG(ERROR) << "STOP"; - SchedulerContext::get()->stop(); + SchedulerContext::get().stop(); }); scheduler.run_in_context([watcher = std::move(watcher)] { class B : public Actor { @@ -559,7 +559,7 @@ TEST(ActorCoro, Simple) { auto watcher = td::create_shared_destructor([] { LOG(ERROR) << "STOP"; - SchedulerContext::get()->stop(); + SchedulerContext::get().stop(); }); scheduler.run_in_context([watcher = std::move(watcher)] { create_actor(ActorOptions().with_name("SampleActor").with_poll(), watcher).release(); diff --git a/tdactor/test/test-coro.cpp b/tdactor/test/test-coro.cpp index 8bb909069b..71935d49fd 100644 --- a/tdactor/test/test-coro.cpp +++ b/tdactor/test/test-coro.cpp @@ -160,11 +160,11 @@ class CoroSpec final : public td::actor::Actor { [](Task test) -> Task { (co_await std::move(test).wrap()).ensure(); co_await yield_on_current(); - td::actor::SchedulerContext::get()->stop(); + td::actor::SchedulerContext::get().stop(); co_return td::Unit{}; }(run_all()) .start_immediate() - .detach(); + .detach("CoroSpec"); } Task unified_queries() { @@ -473,7 +473,7 @@ class CoroSpec final : public td::actor::Actor { }(round) .start(); td::usleep_for(td::Random::fast(0, 1000)); - task.detach(); + task.detach_silent(); td::usleep_for(100); } @@ -847,6 +847,109 @@ class CoroSpec final : public td::actor::Actor { co_return td::Unit{}; } + // Test 1: Promise stored in MAILBOX when actor stops + // BUG: mailbox.clear() runs AFTER destroy_actor(), context still points to freed actor + // FIX: Call actor_execute_context_.clear_actor() after destroy_actor() + Task promise_destroy_in_mailbox() { + LOG(INFO) << "=== promise_destroy_in_mailbox ==="; + + class Target : public td::actor::Actor { + public: + explicit Target(StartedTask::ExternalPromise p) : promise_(std::move(p)) { + } + + void start_up() override { + // Queue the promise to ourselves - it goes to the mailbox + send_closure_later(actor_id(this), &Target::receive_promise, std::move(promise_)); + stop(); // Promise in mailbox destroyed during mailbox.clear() AFTER destroy_actor() + } + + void receive_promise(StartedTask::ExternalPromise) { + LOG(FATAL) << "Should not reach"; + } + + private: + StartedTask::ExternalPromise promise_; + }; + + auto [task, promise] = StartedTask::make_bridge(); + create_actor("Target", std::move(promise)).release(); + + auto result = co_await std::move(task).wrap(); + expect_true(result.is_error(), "Task should fail"); + LOG(INFO) << "Got expected error: " << result.error(); + co_return td::Unit{}; + } + + // Test 2: Promise stored in ACTOR MEMBER when actor stops + // Promise is destroyed during Actor destructor, context might still point to actor + Task promise_destroy_in_actor_member() { + LOG(INFO) << "=== promise_destroy_in_actor_member ==="; + + class Target : public td::actor::Actor { + public: + explicit Target(StartedTask::ExternalPromise p) : promise_(std::move(p)) { + } + + void start_up() override { + // Keep the promise in the actor member, don't move it anywhere + stop(); // Promise destroyed during Actor destructor + } + + private: + StartedTask::ExternalPromise promise_; // Destroyed BEFORE actor_info_ptr_ (base class) + }; + + auto [task, promise] = StartedTask::make_bridge(); + create_actor("Target", std::move(promise)).release(); + + auto result = co_await std::move(task).wrap(); + expect_true(result.is_error(), "Task should fail"); + LOG(INFO) << "Got expected error: " << result.error(); + co_return td::Unit{}; + } + + // Test that co_return {}; works correctly for Task + // Bug: co_return {}; was equivalent to co_return td::Status::Error(-1); + // because {} matched ExternalResult via aggregate initialization + Task co_return_empty_braces() { + LOG(INFO) << "=== co_return_empty_braces ==="; + + // Test co_return {}; in Task - should succeed, not return error + auto test_task = []() -> Task { + co_return {}; // This was buggy - was equivalent to co_return td::Status::Error(-1); + }; + + auto result = co_await test_task().wrap(); + expect_true(result.is_ok(), "co_return {}; should succeed for Task"); + + // Also verify co_return td::Unit{}; still works + auto test_task2 = []() -> Task { co_return td::Unit{}; }; + auto result2 = co_await test_task2().wrap(); + expect_true(result2.is_ok(), "co_return td::Unit{}; should succeed"); + + // Test designated initializers with simple structs + struct SimpleStruct { + int a; + int b; + }; + auto test_designated = []() -> Task { co_return {.a = 1, .b = 2}; }; + auto result3 = co_await test_designated().wrap(); + expect_true(result3.is_ok(), "co_return {.a=1, .b=2}; should succeed"); + expect_eq(result3.ok().a, 1, "designated init .a"); + expect_eq(result3.ok().b, 2, "designated init .b"); + + // Test simple brace init for structs + auto test_brace = []() -> Task { co_return {10, 20}; }; + auto result4 = co_await test_brace().wrap(); + expect_true(result4.is_ok(), "co_return {10, 20}; should succeed"); + expect_eq(result4.ok().a, 10, "brace init .a"); + expect_eq(result4.ok().b, 20, "brace init .b"); + + LOG(INFO) << "co_return_empty_braces test passed"; + co_return td::Unit{}; + } + // Master runner Task run_all() { LOG(ERROR) << "Run tests"; @@ -866,6 +969,9 @@ class CoroSpec final : public td::actor::Actor { co_await combinators(); co_await try_awaitable(); co_await stop_actor(); + co_await promise_destroy_in_mailbox(); + co_await promise_destroy_in_actor_member(); + co_await co_return_empty_braces(); (void)co_await ask(logger_, &TestLogger::log, std::string("All tests passed")); co_return td::Unit(); diff --git a/tddb/test/binlog.cpp b/tddb/test/binlog.cpp index aaba8ffbf8..f8b549cc42 100644 --- a/tddb/test/binlog.cpp +++ b/tddb/test/binlog.cpp @@ -562,7 +562,7 @@ void test_binlog(td::Slice data, td::optional> events = {}) { res.ensure(); binlog_writer.close([&](Result res) { res.ensure(); - td::actor::SchedulerContext::get()->stop(); + td::actor::SchedulerContext::get().stop(); }); }); }); @@ -613,7 +613,7 @@ void test_binlog(td::Slice data, td::optional> events = {}) { scheduler.run_in_context([&]() mutable { binlog.replay_async(binlog_reader, [](Result res) { res.ensure(); - td::actor::SchedulerContext::get()->stop(); + td::actor::SchedulerContext::get().stop(); }); }); diff --git a/tddb/test/io-bench.cpp b/tddb/test/io-bench.cpp index 365de9f5ad..87367b7dcb 100644 --- a/tddb/test/io-bench.cpp +++ b/tddb/test/io-bench.cpp @@ -152,7 +152,7 @@ void read_async(td::CSlice path, size_t buffer_size) { } void on_closed(td::Status status) override { LOG(ERROR) << processor.result(); - td::actor::SchedulerContext::get()->stop(); + td::actor::SchedulerContext::get().stop(); } private: @@ -520,7 +520,7 @@ void write_async(td::CSlice path, size_t buffer_size) { } } void hangup_shared() override { - td::actor::SchedulerContext::get()->stop(); + td::actor::SchedulerContext::get().stop(); stop(); } }; @@ -580,7 +580,7 @@ void write_async2(td::CSlice path, size_t buffer_size) { worker_.reset(); } void hangup_shared() override { - td::actor::SchedulerContext::get()->stop(); + td::actor::SchedulerContext::get().stop(); stop(); } }; diff --git a/tddb/test/key_value.cpp b/tddb/test/key_value.cpp index 76d89dd50f..b53c3c4a3d 100644 --- a/tddb/test/key_value.cpp +++ b/tddb/test/key_value.cpp @@ -79,7 +79,7 @@ TEST(KeyValue, async_simple) { td::RocksDb::destroy(db_name).ignore(); td::actor::Scheduler scheduler({6}); - auto watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get()->stop(); }); + auto watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get().stop(); }); class Worker : public td::actor::Actor { public: @@ -190,7 +190,7 @@ TEST(KeyValue, Stress) { } td::actor::Scheduler scheduler({6}); - auto watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get()->stop(); }); + auto watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get().stop(); }); class Worker : public td::actor::Actor { public: diff --git a/tdnet/example/tcp_ping_pong.cpp b/tdnet/example/tcp_ping_pong.cpp index bece8f2655..40b2c339e4 100644 --- a/tdnet/example/tcp_ping_pong.cpp +++ b/tdnet/example/tcp_ping_pong.cpp @@ -54,8 +54,8 @@ class PingClient : public td::actor::Actor, td::ObserverBase { LOG(INFO) << "Start"; // Subscribe for socket updates // NB: Interface will be changed - td::actor::SchedulerContext::get()->get_poll().subscribe(buffered_fd_.get_poll_info().extract_pollable_fd(this), - td::PollFlags::ReadWrite()); + td::actor::SchedulerContext::get().get_poll().subscribe(buffered_fd_.get_poll_info().extract_pollable_fd(this), + td::PollFlags::ReadWrite()); alarm_timestamp() = td::Timestamp::now(); } @@ -64,7 +64,7 @@ class PingClient : public td::actor::Actor, td::ObserverBase { LOG(INFO) << "Close"; // unsubscribe from socket updates // nb: interface will be changed - td::actor::SchedulerContext::get()->get_poll().unsubscribe(buffered_fd_.get_poll_info().get_pollable_fd_ref()); + td::actor::SchedulerContext::get().get_poll().unsubscribe(buffered_fd_.get_poll_info().get_pollable_fd_ref()); } void loop() override { diff --git a/tdnet/example/udp_ping_pong.cpp b/tdnet/example/udp_ping_pong.cpp index 1c1472aa3f..4f75a831cd 100644 --- a/tdnet/example/udp_ping_pong.cpp +++ b/tdnet/example/udp_ping_pong.cpp @@ -106,7 +106,7 @@ class PingPong : public td::actor::Actor { stop(); } void tear_down() override { - td::actor::SchedulerContext::get()->stop(); + td::actor::SchedulerContext::get().stop(); } }; diff --git a/tdnet/td/net/FdListener.cpp b/tdnet/td/net/FdListener.cpp index a807408cb8..bf67fb7b3b 100644 --- a/tdnet/td/net/FdListener.cpp +++ b/tdnet/td/net/FdListener.cpp @@ -21,10 +21,10 @@ namespace td { void FdListener::start_up() { fd_.add_flags(PollFlags::ReadWrite()); - td::actor::SchedulerContext::get()->get_poll().subscribe(std::move(fd_), PollFlags::ReadWrite()); + td::actor::SchedulerContext::get().get_poll().subscribe(std::move(fd_), PollFlags::ReadWrite()); } void FdListener::tear_down() { - td::actor::SchedulerContext::get()->get_poll().unsubscribe(fd_ref_); + td::actor::SchedulerContext::get().get_poll().unsubscribe(fd_ref_); } } // namespace td diff --git a/tdnet/td/net/FramedPipe.cpp b/tdnet/td/net/FramedPipe.cpp index d6c074404a..3b4bcb2a48 100644 --- a/tdnet/td/net/FramedPipe.cpp +++ b/tdnet/td/net/FramedPipe.cpp @@ -3,9 +3,10 @@ namespace td { -Status framed_write(ChainBufferWriter &writer, Slice message) { - if (message.size() > MAX_FRAMED_MESSAGE_SIZE) { - return Status::Error("Too large message"); +Status framed_write(ChainBufferWriter &writer, Slice message, size_t max_message_size) { + if (message.size() > max_message_size) { + return Status::Error(PSLICE() << "Invalid message size: " << message.size() << " (max: " << max_message_size + << ")"); } char head[4]; as(head) = static_cast(message.size()); @@ -14,7 +15,7 @@ Status framed_write(ChainBufferWriter &writer, Slice message) { return Status::OK(); } -Result framed_read(ChainBufferReader &reader, BufferSlice &dst) { +Result framed_read(ChainBufferReader &reader, BufferSlice &dst, size_t max_message_size) { auto input = reader.clone(); if (input.size() < 4) { return 4; @@ -22,8 +23,8 @@ Result framed_read(ChainBufferReader &reader, BufferSlice &dst) { uint32 size = as(input.cut_head(4).move_as_buffer_slice().as_slice().data()); // Validate size to prevent integer overflow and excessive memory allocation - if (size > MAX_FRAMED_MESSAGE_SIZE) { - return Status::Error(PSLICE() << "Invalid message size: " << size << " (max: " << MAX_FRAMED_MESSAGE_SIZE << ")"); + if (size > max_message_size) { + return Status::Error(PSLICE() << "Invalid message size: " << size << " (max: " << max_message_size << ")"); } // Check if we have enough data diff --git a/tdnet/td/net/FramedPipe.h b/tdnet/td/net/FramedPipe.h index 021ebeea24..cacd3ba775 100644 --- a/tdnet/td/net/FramedPipe.h +++ b/tdnet/td/net/FramedPipe.h @@ -8,17 +8,19 @@ namespace td { // Simple framing protocol: 4-byte length prefix + message // Maximum message size to prevent excessive memory allocation -constexpr size_t MAX_FRAMED_MESSAGE_SIZE = 16536; +constexpr size_t DEFAULT_MAX_FRAMED_MESSAGE_SIZE = 16536; // Write a framed message to a ChainBufferWriter // Format: [4-byte length][message data] -Status framed_write(ChainBufferWriter &writer, Slice message); +Status framed_write(ChainBufferWriter &writer, Slice message, + size_t max_message_size = DEFAULT_MAX_FRAMED_MESSAGE_SIZE); // Read a framed message from a ChainBufferReader // Returns: // - 0 if a complete message was read (stored in dst) // - >0 indicating how many more bytes are needed // - Error if message size is invalid -Result framed_read(ChainBufferReader &reader, BufferSlice &dst); +Result framed_read(ChainBufferReader &reader, BufferSlice &dst, + size_t max_message_size = DEFAULT_MAX_FRAMED_MESSAGE_SIZE); } // namespace td diff --git a/tdnet/td/net/Pipe.cpp b/tdnet/td/net/Pipe.cpp index 1de405c087..151fe21ffa 100644 --- a/tdnet/td/net/Pipe.cpp +++ b/tdnet/td/net/Pipe.cpp @@ -19,14 +19,14 @@ struct PollFdActor : public td::actor::Actor { void unsubscribe() { CHECK(!closed_); - td::actor::SchedulerContext::get()->get_poll().unsubscribe(fd_ref_); + td::actor::SchedulerContext::get().get_poll().unsubscribe(fd_ref_); closed_ = true; stop(); } void destroy() { CHECK(!closed_); - td::actor::SchedulerContext::get()->get_poll().unsubscribe_before_close(fd_ref_); + td::actor::SchedulerContext::get().get_poll().unsubscribe_before_close(fd_ref_); closed_ = true; stop(); } @@ -38,7 +38,7 @@ struct PollFdActor : public td::actor::Actor { bool closed_{false}; void start_up() override { - td::actor::SchedulerContext::get()->get_poll().subscribe(std::move(fd_), td::PollFlags::ReadWrite()); + td::actor::SchedulerContext::get().get_poll().subscribe(std::move(fd_), td::PollFlags::ReadWrite()); } void tear_down() override { diff --git a/tdnet/td/net/TcpListener.cpp b/tdnet/td/net/TcpListener.cpp index 019212dafb..34c9fd0da5 100644 --- a/tdnet/td/net/TcpListener.cpp +++ b/tdnet/td/net/TcpListener.cpp @@ -49,15 +49,15 @@ void TcpListener::start_up() { // Subscribe for socket updates // NB: Interface will be changed - td::actor::SchedulerContext::get()->get_poll().subscribe(server_socket_fd_.get_poll_info().extract_pollable_fd(this), - PollFlags::Read()); + td::actor::SchedulerContext::get().get_poll().subscribe(server_socket_fd_.get_poll_info().extract_pollable_fd(this), + PollFlags::Read()); } void TcpListener::tear_down() { if (!server_socket_fd_.empty()) { // unsubscribe from socket updates // nb: interface will be changed - td::actor::SchedulerContext::get()->get_poll().unsubscribe(server_socket_fd_.get_poll_info().get_pollable_fd_ref()); + td::actor::SchedulerContext::get().get_poll().unsubscribe(server_socket_fd_.get_poll_info().get_pollable_fd_ref()); } } diff --git a/tdnet/td/net/UdpServer.cpp b/tdnet/td/net/UdpServer.cpp index 6ce0f5c91b..37b6137103 100644 --- a/tdnet/td/net/UdpServer.cpp +++ b/tdnet/td/net/UdpServer.cpp @@ -69,7 +69,7 @@ UdpServerImpl::UdpServerImpl(td::UdpSocketFd fd, std::unique_ptr callb } void UdpServerImpl::start_up() { - //CHECK(td::actor::SchedulerContext::get()->has_poll() == false); + //CHECK(td::actor::SchedulerContext::get().has_poll() == false); class Observer : public td::ObserverBase, public Destructor { public: Observer(td::actor::ActorShared udp_server) : udp_server_(std::move(udp_server)) { @@ -97,7 +97,7 @@ void UdpServerImpl::loop() { if (is_closing_) { return; } - //CHECK(td::actor::SchedulerContext::get()->has_poll() == false); + //CHECK(td::actor::SchedulerContext::get().has_poll() == false); sync_with_poll(fd_); VLOG(udp_server) << "loop " << td::tag("can read", can_read(fd_)) << " " << td::tag("can write", can_write(fd_)); Status status; @@ -169,8 +169,8 @@ class TcpClient : public td::actor::Actor, td::ObserverBase { LOG(INFO) << "Start"; // Subscribe for socket updates // NB: Interface will be changed - td::actor::SchedulerContext::get()->get_poll().subscribe(buffered_fd_.get_poll_info().extract_pollable_fd(this), - PollFlags::ReadWrite()); + td::actor::SchedulerContext::get().get_poll().subscribe(buffered_fd_.get_poll_info().extract_pollable_fd(this), + PollFlags::ReadWrite()); alarm_timestamp() = Timestamp::in(10); notify(); } @@ -179,7 +179,7 @@ class TcpClient : public td::actor::Actor, td::ObserverBase { LOG(INFO) << "Close"; // unsubscribe from socket updates // nb: interface will be changed - td::actor::SchedulerContext::get()->get_poll().unsubscribe(buffered_fd_.get_poll_info().get_pollable_fd_ref()); + td::actor::SchedulerContext::get().get_poll().unsubscribe(buffered_fd_.get_poll_info().get_pollable_fd_ref()); callback_->on_closed(actor_id(this)); } diff --git a/tdnet/test/net-test.cpp b/tdnet/test/net-test.cpp index d20be50413..c5814c6673 100644 --- a/tdnet/test/net-test.cpp +++ b/tdnet/test/net-test.cpp @@ -139,7 +139,7 @@ class PingPong : public td::actor::Actor { stop(); } void tear_down() override { - td::actor::SchedulerContext::get()->stop(); + td::actor::SchedulerContext::get().stop(); } }; diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index c0fa067d28..e9d053c652 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -1,4 +1,5 @@ option(TDUTILS_MIME_TYPE "Generate mime types conversion (gperf is required)" ON) +option(TDUTILS_USE_LIBBACKTRACE "Use libbacktrace for better stack traces" ON) set(TD_HAVE_OPENSSL 1) @@ -31,6 +32,16 @@ if (ABSL_FOUND) set(TD_HAVE_ABSL 1) endif() +# libbacktrace support - check if target exists (built at higher level) +if (TDUTILS_USE_LIBBACKTRACE) + if (TARGET libbacktrace AND NOT WIN32) + set(TD_HAVE_LIBBACKTRACE 1) + message(STATUS "Using libbacktrace for enhanced stack traces") + else() + message(STATUS "libbacktrace target not found, using basic backtrace") + endif() +endif() + configure_file(td/utils/config.h.in td/utils/config.h @ONLY) add_subdirectory(generate) @@ -361,6 +372,11 @@ if (ABSL_FOUND) target_link_libraries_system(tdutils absl::flat_hash_map absl::flat_hash_set absl::hash) endif() +if (TD_HAVE_LIBBACKTRACE) + target_link_libraries(tdutils PRIVATE libbacktrace) + target_compile_definitions(tdutils PRIVATE TD_HAVE_LIBBACKTRACE=1) +endif() + if (WIN32 AND WINGETOPT_FOUND) target_link_libraries(tdutils PRIVATE wingetopt) endif() diff --git a/tdutils/td/utils/optional.h b/tdutils/td/utils/optional.h index 2c96dc3ff7..ee057ce599 100644 --- a/tdutils/td/utils/optional.h +++ b/tdutils/td/utils/optional.h @@ -119,4 +119,12 @@ struct optional : optional { ~optional() = default; }; +template +StringBuilder &operator<<(StringBuilder &sb, const optional &v) { + if (v) { + return sb << "Some{" << v.value() << "}"; + } + return sb << "None"; +} + } // namespace td diff --git a/tdutils/td/utils/port/signals.cpp b/tdutils/td/utils/port/signals.cpp index fd14a8bfa2..bd2dfde369 100644 --- a/tdutils/td/utils/port/signals.cpp +++ b/tdutils/td/utils/port/signals.cpp @@ -340,6 +340,7 @@ static void block_stdin() { Stacktrace::PrintOptions options; options.use_gdb = true; + options.use_libbacktrace = true; Stacktrace::print_to_stderr(options); #if TD_PORT_POSIX diff --git a/tdutils/td/utils/port/stacktrace.cpp b/tdutils/td/utils/port/stacktrace.cpp index 2c993536ae..b47bf848e2 100644 --- a/tdutils/td/utils/port/stacktrace.cpp +++ b/tdutils/td/utils/port/stacktrace.cpp @@ -16,6 +16,11 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include +#include +#include +#include + #include "td/utils/port/signals.h" #include "td/utils/port/stacktrace.h" @@ -23,10 +28,17 @@ #include #else #if TD_DARWIN || __GLIBC__ +#include #include #endif #endif +#ifdef TD_HAVE_LIBBACKTRACE +#include +#include +#include +#endif + #if TD_LINUX || TD_FREEBSD #include #include @@ -37,10 +49,216 @@ #endif #endif +#if TD_DARWIN +#include +#endif + namespace td { namespace { +#ifdef TD_HAVE_LIBBACKTRACE +struct BacktraceState { + int skip; + int index; +}; + +// Hex number formatter (builds string backwards in embedded buffer) +class SafeHex { + public: + SafeHex(uintptr_t value) { + char *ptr = buf_ + sizeof(buf_) - 1; + *ptr = '\0'; + if (value == 0) { + *--ptr = '0'; + } else { + while (value != 0) { + int digit = static_cast(value & 0xF); + *--ptr = static_cast(digit < 10 ? '0' + digit : 'a' + digit - 10); + value >>= 4; + } + } + slice_ = td::Slice(ptr, buf_ + sizeof(buf_) - 1); + } + + operator td::Slice() const { + return slice_; + } + + private: + char buf_[20]; + td::Slice slice_; +}; + +// Decimal number formatter (builds string backwards in embedded buffer) +class SafeDec { + public: + SafeDec(int value) { + char *ptr = buf_ + sizeof(buf_) - 1; + *ptr = '\0'; + if (value == 0) { + *--ptr = '0'; + } else { + while (value > 0) { + *--ptr = '0' + (value % 10); + value /= 10; + } + } + slice_ = td::Slice(ptr, buf_ + sizeof(buf_) - 1); + } + + operator td::Slice() const { + return slice_; + } + + private: + char buf_[20]; + td::Slice slice_; +}; + +// Simple writer for building strings safely in signal handlers +class SafeWriter { + public: + SafeWriter(char *buf, size_t size) : begin_(buf), remaining_(buf, size) { + } + + SafeWriter &operator<<(td::Slice str) { + size_t to_copy = std::min(str.size(), remaining_.size()); + if (to_copy > 0) { + std::memcpy(remaining_.data(), str.data(), to_copy); + remaining_ = remaining_.substr(to_copy); + } + return *this; + } + + td::Slice as_slice() const { + size_t written = remaining_.begin() - begin_; + return td::Slice(begin_, written); + } + + private: + char *begin_; + td::MutableSlice remaining_; +}; + +void backtrace_error_callback(void *data, const char *msg, int errnum) { + signal_safe_write("libbacktrace error: ", false); + signal_safe_write(td::Slice(msg ? msg : "unknown error"), false); + signal_safe_write("\n", false); +} + +int backtrace_full_callback(void *data, uintptr_t pc, const char *filename, int lineno, const char *function) { + BacktraceState *state = static_cast(data); + + if (state->skip > 0) { + state->skip--; + return 0; + } + + // Get module info for this PC + Dl_info dlinfo; + Slice module_cstr = "unknown"; + uintptr_t module_base = 0; + if (dladdr(reinterpret_cast(pc), &dlinfo) && dlinfo.dli_fname) { + module_cstr = td::CSlice(dlinfo.dli_fname); + auto pos = module_cstr.rfind('/'); + if (pos != CSlice::npos) { + module_cstr.remove_prefix(pos + 1); + } + module_base = reinterpret_cast(dlinfo.dli_fbase); + } + + // Choose best function name: libbacktrace > dladdr > unknown + const char *func_name = function ? function : (dlinfo.dli_sname ? dlinfo.dli_sname : nullptr); + + // Demangle C++ function names + char *demangled = nullptr; + if (func_name) { + int status; + demangled = abi::__cxa_demangle(func_name, nullptr, nullptr, &status); + if (status == 0 && demangled) { + func_name = demangled; + } + } + + // Extract file basename + const char *file_cstr = nullptr; + if (filename && filename[0]) { + const char *slash = strrchr(filename, '/'); + file_cstr = slash ? slash + 1 : filename; + } + + // Build output: " #N function at file:line (module+0xoffset) [0xaddress]\n" + char buf[2048]; + SafeWriter w(buf, sizeof(buf)); + + w << " #" << SafeDec(state->index) << " " << td::Slice(func_name ? func_name : "??").truncate(400); + + if (file_cstr) { + w << " at " << td::Slice(file_cstr).truncate(80); + if (lineno > 0) { + w << ":" << SafeDec(lineno); + } + } + + uintptr_t offset = module_base ? (pc - module_base) : pc; + w << " (" << module_cstr.truncate(100) << "+0x" << SafeHex(offset) << ") [0x" << SafeHex(pc) << "]\n"; + + signal_safe_write(w.as_slice(), false); + + free(demangled); + state->index++; + return 0; +} + +// Single atomic lock for backtrace operations +static std::atomic backtrace_lock{false}; + +// Deleter that releases the lock +struct BacktraceUnlock { + void operator()(backtrace_state *) const { + backtrace_lock.store(false, std::memory_order_release); + } +}; + +using LockedBacktraceState = std::unique_ptr; + +// Try to get backtrace state with lock. Returns nullptr if lock is taken. +static LockedBacktraceState get_locked_backtrace_state() { + // Try to acquire lock + bool expected = false; + if (!backtrace_lock.compare_exchange_strong(expected, true, std::memory_order_acquire)) { + return nullptr; // Lock taken by another thread + } + + // Get or initialize state (protected by lock, no atomics needed) + static backtrace_state *state = nullptr; + + if (state == nullptr) { + // Initialize on first use + const char *exe_path = nullptr; +#if TD_LINUX || TD_FREEBSD + static char path_buf[512]; + ssize_t res = readlink("/proc/self/exe", path_buf, sizeof(path_buf) - 1); + if (res >= 0) { + path_buf[res] = '\0'; + exe_path = path_buf; + } +#elif TD_DARWIN + static char path_buf[512]; + uint32_t size = sizeof(path_buf); + if (_NSGetExecutablePath(path_buf, &size) == 0) { + exe_path = path_buf; + } +#endif + + state = backtrace_create_state(exe_path, 1, backtrace_error_callback, nullptr); + } + + return LockedBacktraceState(state, BacktraceUnlock{}); +} +#endif + void print_backtrace(void) { #if TD_WINDOWS void *stack[100]; @@ -86,6 +304,28 @@ void print_backtrace(void) { #endif } +void print_backtrace_libbacktrace(void) { +#ifdef TD_HAVE_LIBBACKTRACE + signal_safe_write("--- Enhanced Backtrace (libbacktrace) ---\n", false); + + auto locked_state = get_locked_backtrace_state(); + if (!locked_state) { + signal_safe_write("(another thread is printing backtrace)\n", false); + signal_safe_write("------------------------------------------\n", false); + return; + } + + BacktraceState state; + state.skip = 0; + state.index = 0; + + // Skip 1 frame: this function itself + backtrace_full(locked_state.get(), 1, backtrace_full_callback, backtrace_error_callback, &state); + + signal_safe_write("------------------------------------------\n", false); +#endif +} + void print_backtrace_gdb(void) { #if TD_LINUX || TD_FREEBSD char pid_buf[30]; @@ -161,12 +401,19 @@ void print_backtrace_gdb(void) { void Stacktrace::print_to_stderr(const PrintOptions &options) { print_backtrace(); + if (options.use_libbacktrace) { + print_backtrace_libbacktrace(); + } if (options.use_gdb) { print_backtrace_gdb(); } } void Stacktrace::init() { +#ifdef TD_HAVE_LIBBACKTRACE + // Initialize libbacktrace state early (no lock needed during init) + get_locked_backtrace_state(); +#endif #if TD_DARWIN || __GLIBC__ // backtrace needs to be called once to ensure that next calls are async-signal-safe void *buffer[1]; diff --git a/tdutils/td/utils/port/stacktrace.h b/tdutils/td/utils/port/stacktrace.h index 40bb9c9a8d..9cb32322ec 100644 --- a/tdutils/td/utils/port/stacktrace.h +++ b/tdutils/td/utils/port/stacktrace.h @@ -24,6 +24,7 @@ class Stacktrace { public: struct PrintOptions { bool use_gdb = false; + bool use_libbacktrace = true; PrintOptions() { } }; diff --git a/terminal/terminal.cpp b/terminal/terminal.cpp index c4a383dbe5..1013ed20fd 100644 --- a/terminal/terminal.cpp +++ b/terminal/terminal.cpp @@ -115,15 +115,15 @@ void TerminalIOImpl::start_up() { #endif if (!no_input_) { - td::actor::SchedulerContext::get()->get_poll().subscribe(stdin_.get_poll_info().extract_pollable_fd(this), - td::PollFlags::Read()); + td::actor::SchedulerContext::get().get_poll().subscribe(stdin_.get_poll_info().extract_pollable_fd(this), + td::PollFlags::Read()); loop(); } } void TerminalIOImpl::tear_down() { log_interface = default_log_interface; - td::actor::SchedulerContext::get()->get_poll().unsubscribe(stdin_.get_poll_info().get_pollable_fd_ref()); + td::actor::SchedulerContext::get().get_poll().unsubscribe(stdin_.get_poll_info().get_pollable_fd_ref()); out_mutex_.lock(); #ifdef USE_READLINE if (use_readline_) { diff --git a/third-party/libbacktrace b/third-party/libbacktrace new file mode 160000 index 0000000000..b9e40069c0 --- /dev/null +++ b/third-party/libbacktrace @@ -0,0 +1 @@ +Subproject commit b9e40069c0b47a722286b94eb5231f7f05c08713 diff --git a/tonlib/tonlib/Client.cpp b/tonlib/tonlib/Client.cpp index 58aea5829a..756ffcd368 100644 --- a/tonlib/tonlib/Client.cpp +++ b/tonlib/tonlib/Client.cpp @@ -97,7 +97,7 @@ class Client::Impl final { receive(10); } LOG(ERROR) << "Stop"; - scheduler_.run_in_context_external([] { td::actor::SchedulerContext::get()->stop(); }); + scheduler_.run_in_context_external([] { td::actor::SchedulerContext::get().stop(); }); LOG(ERROR) << "join"; scheduler_thread_.join(); LOG(ERROR) << "join - done"; diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 6c9961f0e9..2f20cdcef6 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -279,7 +279,7 @@ class TonlibCli : public td::actor::Actor { } } void tear_down() override { - td::actor::SchedulerContext::get()->stop(); + td::actor::SchedulerContext::get().stop(); } void on_wait() { diff --git a/validator-engine-console/validator-engine-console.h b/validator-engine-console/validator-engine-console.h index 3cf3587345..2dcca796c7 100644 --- a/validator-engine-console/validator-engine-console.h +++ b/validator-engine-console/validator-engine-console.h @@ -121,7 +121,7 @@ class ValidatorEngineConsole : public td::actor::Actor { } void tear_down() override { // FIXME: do not work in windows - //td::actor::SchedulerContext::get()->stop(); + //td::actor::SchedulerContext::get().stop(); io_.reset(); std::_Exit(0); } diff --git a/validator/validator-telemetry.cpp b/validator/validator-telemetry.cpp index 38c7c18d2c..15a64518ba 100644 --- a/validator/validator-telemetry.cpp +++ b/validator/validator-telemetry.cpp @@ -77,8 +77,8 @@ void ValidatorTelemetry::send_telemetry() { telemetry->ram_size_ = ram_size_; telemetry->cpu_cores_ = cpu_cores_; telemetry->node_threads_ = (td::int32)td::actor::SchedulerContext::get() - ->scheduler_group() - ->schedulers.at(td::actor::SchedulerContext::get()->get_scheduler_id().value()) + .scheduler_group() + ->schedulers.at(td::actor::SchedulerContext::get().get_scheduler_id().value()) .cpu_threads_count; LOG(DEBUG) << "Sending validator telemetry for adnl id " << local_id_; From 54af04fd793da72ff5f09683a7de32fb9d6a6ba2 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Mon, 22 Dec 2025 14:34:59 +0800 Subject: [PATCH 517/524] adjust assembly scripts for clang 21 (#1952) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * clang-21 wip * adjust build scripts * use bundled libc++ on macos * force new llvm usage on macos * revert force new llvm usage on macos, clear gh cache * stop using homebrew libc++ headers and force it to use Apple’s * fetch libmicrohttpd from github instead of ftpmirror.gnu.org * for macos >=15 use AppleClang, otherwise homebrew's * compile 3pp with sdk, macos-14 still fails (ok) * try clang-20 for macos * Revert "try clang-20 for macos" This reverts commit b120dda52bbc0be17db0e6e71c314145da2881ef. * stop using homebrew libc++ headers and force it to use Apple’s * update blst to v0.3.15 * update blst to v0.3.15 * remove redundant submodule entry * remove redundant snippet * remove redundant snippet * remove redundant snippet * minor * minor * use libmicrohttpd repo out of ton-blockchain org * minor * clear gh cache and rerun * review fixes * review fixes --------- Co-authored-by: EmelyanenkoK --- .../build-ton-linux-arm64-appimage.yml | 6 +- .../build-ton-linux-arm64-shared.yml | 15 +++- .../build-ton-linux-x86-64-appimage.yml | 4 +- .../build-ton-linux-x86-64-shared.yml | 11 ++- .../workflows/build-ton-wasm-emscripten.yml | 4 +- Dockerfile | 6 +- README.md | 4 +- assembly/android/build-android-tonlib.sh | 4 +- assembly/native/build-macos-portable.sh | 68 +++++++++++-------- assembly/native/build-macos-shared.sh | 52 +++++++------- assembly/native/build-ubuntu-appimages.sh | 12 ++-- assembly/native/build-ubuntu-portable-libs.sh | 30 ++++---- assembly/native/build-ubuntu-portable.sh | 30 ++++---- assembly/native/build-ubuntu-shared.sh | 10 +-- assembly/wasm/fift-func-wasm-build-ubuntu.sh | 20 +++--- 15 files changed, 151 insertions(+), 125 deletions(-) diff --git a/.github/workflows/build-ton-linux-arm64-appimage.yml b/.github/workflows/build-ton-linux-arm64-appimage.yml index cb8a49844c..cccebce358 100644 --- a/.github/workflows/build-ton-linux-arm64-appimage.yml +++ b/.github/workflows/build-ton-linux-arm64-appimage.yml @@ -24,13 +24,13 @@ jobs: sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache libgsl-dev libblas-dev libgslcblas0 mkdir ~/.ccache 3pp - - name: Install clang-16 + - name: Install clang-21 run: | lsb_release -a wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-16 main" | sudo tee /etc/apt/sources.list.d/llvm.list + echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list sudo apt-get -y update - sudo apt-get install -y clang-16 + sudo apt-get install -y clang-21 - name: Cache 3pp id: cache-3pp diff --git a/.github/workflows/build-ton-linux-arm64-shared.yml b/.github/workflows/build-ton-linux-arm64-shared.yml index 1c5ffae8f0..5c5e86d17a 100644 --- a/.github/workflows/build-ton-linux-arm64-shared.yml +++ b/.github/workflows/build-ton-linux-arm64-shared.yml @@ -29,13 +29,22 @@ jobs: mkdir ~/.ccache - if: matrix.os != 'ubuntu-24.04-arm' - name: Install llvm-16 + name: Install llvm-21 run: | lsb_release -a wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-16 main" | sudo tee /etc/apt/sources.list.d/llvm.list + echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list sudo apt-get -y update - sudo apt-get install -y clang-16 + sudo apt-get install -y clang-21 + + - if: matrix.os == 'ubuntu-24.04-arm' + name: Install llvm-21 + run: | + lsb_release -a + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list + sudo apt-get -y update + sudo apt-get install -y clang-21 - name: Cache OpenSSL id: cache-openssl diff --git a/.github/workflows/build-ton-linux-x86-64-appimage.yml b/.github/workflows/build-ton-linux-x86-64-appimage.yml index 0bebcbadc1..aa9787cd69 100644 --- a/.github/workflows/build-ton-linux-x86-64-appimage.yml +++ b/.github/workflows/build-ton-linux-x86-64-appimage.yml @@ -24,11 +24,11 @@ jobs: sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache libgsl-dev libblas-dev libgslcblas0 mkdir ~/.ccache 3pp - - name: Install clang-16 + - name: Install clang-21 run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 16 clang + sudo ./llvm.sh 21 clang - name: Cache 3pp id: cache-3pp diff --git a/.github/workflows/build-ton-linux-x86-64-shared.yml b/.github/workflows/build-ton-linux-x86-64-shared.yml index 87b9dc15a7..bccfab3af6 100644 --- a/.github/workflows/build-ton-linux-x86-64-shared.yml +++ b/.github/workflows/build-ton-linux-x86-64-shared.yml @@ -32,7 +32,16 @@ jobs: run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 16 clang + sudo ./llvm.sh 21 clang + + - if: matrix.os == 'ubuntu-24.04' + name: Install llvm-21 + run: | + lsb_release -a + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list + sudo apt -y update + sudo apt install -y clang-21 - name: Cache OpenSSL id: cache-openssl diff --git a/.github/workflows/build-ton-wasm-emscripten.yml b/.github/workflows/build-ton-wasm-emscripten.yml index ce09f530e1..6e575b3999 100644 --- a/.github/workflows/build-ton-wasm-emscripten.yml +++ b/.github/workflows/build-ton-wasm-emscripten.yml @@ -18,11 +18,11 @@ jobs: sudo apt-get install -y build-essential git openssl cmake ninja-build zlib1g-dev libssl-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev mkdir -p 3pp_emscripten - - name: Install clang-16 + - name: Install clang-21 run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 16 clang + sudo ./llvm.sh 21 clang - name: Cache OpenSSL id: cache-openssl diff --git a/Dockerfile b/Dockerfile index 7a53500db6..6e7668b00d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,11 +11,11 @@ RUN apt-get update && \ RUN wget https://apt.llvm.org/llvm.sh && \ chmod +x llvm.sh && \ - ./llvm.sh 16 all && \ + ./llvm.sh 21 all && \ rm -rf /var/lib/apt/lists/* -ENV CC=/usr/bin/clang-16 -ENV CXX=/usr/bin/clang++-16 +ENV CC=/usr/bin/clang-21 +ENV CXX=/usr/bin/clang++-21 ENV CCACHE_DISABLE=1 WORKDIR / diff --git a/README.md b/README.md index 897ba809a4..9833cecd92 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ Install additional system libraries wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 16 all + sudo ./llvm.sh 21 clang ``` Compile TON binaries ```bash @@ -117,7 +117,7 @@ Install additional system libraries on Ubuntu wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 16 all + sudo ./llvm.sh 21 clang ``` Compile TON binaries with emscripten ```bash diff --git a/assembly/android/build-android-tonlib.sh b/assembly/android/build-android-tonlib.sh index dda74c1664..b1bff6c4ab 100644 --- a/assembly/android/build-android-tonlib.sh +++ b/assembly/android/build-android-tonlib.sh @@ -8,8 +8,8 @@ while getopts 'a' flag; do esac done -export CC=$(which clang-16) -export CXX=$(which clang++-16) +export CC=$(which clang-21) +export CXX=$(which clang++-21) export CCACHE_DISABLE=1 if [ ! -d android-ndk-r27d ]; then diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index a86435a4fa..29bb51409f 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -6,6 +6,13 @@ with_ccache=false OSX_TARGET=11.0 +MACOS_MAJOR=0 +if [ "$(uname)" = "Darwin" ]; then + MACOS_MAJOR=$(sw_vers -productVersion | cut -d. -f1) + export SDKROOT="$(xcrun --sdk macosx --show-sdk-path)" + echo "Using SDKROOT=$SDKROOT" +fi + while getopts 'taco:' flag; do case "${flag}" in t) with_tests=true ;; @@ -21,14 +28,26 @@ if [ ! -d "build" ]; then mkdir build cd build else - cd build + cd build || exit rm -rf .ninja* CMakeCache.txt fi export NONINTERACTIVE=1 brew install ninja pkg-config automake libtool autoconf texinfo export PATH=/usr/local/opt/ccache/libexec:$PATH -brew install llvm@16 + +if [ "$(uname)" = "Darwin" ]; then + if [ "$MACOS_MAJOR" -ge 15 ]; then + echo "macOS $MACOS_MAJOR detected -> using AppleClang (Xcode toolchain), NOT llvm@21" + export CC="$(xcrun --find clang)" + export CXX="$(xcrun --find clang++)" + else + echo "macOS $MACOS_MAJOR detected -> using Homebrew llvm@21" + brew install llvm@21 + export CC="$(brew --prefix llvm@21)"/bin/clang + export CXX="$(brew --prefix llvm@21)"/bin/clang++ + fi +fi if [ "$with_ccache" = true ]; then brew install ccache @@ -40,24 +59,15 @@ else export CCACHE_DISABLE=1 fi - -if [ -f /opt/homebrew/opt/llvm@16/bin/clang ]; then - export CC=/opt/homebrew/opt/llvm@16/bin/clang - export CXX=/opt/homebrew/opt/llvm@16/bin/clang++ -else - export CC=/usr/local/opt/llvm@16/bin/clang - export CXX=/usr/local/opt/llvm@16/bin/clang++ -fi - if [ ! -d "../3pp/lz4" ]; then mkdir -p ../3pp git clone https://github.com/lz4/lz4.git ../3pp/lz4 -cd ../3pp/lz4 +cd ../3pp/lz4 || exit lz4Path=`pwd` git checkout v1.9.4 -make -j4 +make -j4 CC="$CC" CFLAGS="--sysroot=$SDKROOT" test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } -cd ../../build +cd ../../build || exit # ./lib/liblz4.a # ./lib else @@ -67,15 +77,15 @@ fi if [ ! -d "../3pp/libsodium-1.0.18" ]; then export LIBSODIUM_FULL_BUILD=1 - cd ../3pp + cd ../3pp || exit curl -LO https://download.libsodium.org/libsodium/releases/libsodium-1.0.18.tar.gz tar -xzf libsodium-1.0.18.tar.gz - cd libsodium-1.0.18 + cd libsodium-1.0.18 || exit sodiumPath=`pwd` ./configure --with-pic --enable-static - make -j4 + make -j4 CC="$CC" CFLAGS="--sysroot=$SDKROOT" test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } - cd ../../build + cd ../../build || exit else sodiumPath=$(pwd)/../3pp/libsodium-1.0.18 echo "Using compiled libsodium" @@ -83,13 +93,13 @@ fi if [ ! -d "../3pp/openssl_3" ]; then git clone https://github.com/openssl/openssl ../3pp/openssl_3 - cd ../3pp/openssl_3 + cd ../3pp/openssl_3 || exit opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make build_libs -j4 + make build_libs -j4 CC="$CC" CFLAGS="--sysroot=$SDKROOT" test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd ../../build + cd ../../build || exit else opensslPath=$(pwd)/../3pp/openssl_3 echo "Using compiled openssl_3" @@ -97,12 +107,12 @@ fi if [ ! -d "../3pp/zlib" ]; then git clone https://github.com/madler/zlib.git ../3pp/zlib - cd ../3pp/zlib + cd ../3pp/zlib || exit zlibPath=`pwd` ./configure --static - make -j4 + make -j4 CC="$CC" CFLAGS="--sysroot=$SDKROOT" test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } - cd ../../build + cd ../../build || exit else zlibPath=$(pwd)/../3pp/zlib echo "Using compiled zlib" @@ -110,12 +120,12 @@ fi if [ ! -d "../3pp/libmicrohttpd" ]; then git clone https://github.com/ton-blockchain/libmicrohttpd.git ../3pp/libmicrohttpd - cd ../3pp/libmicrohttpd + cd ../3pp/libmicrohttpd || exit libmicrohttpdPath=`pwd` ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - make -j4 + make -j4 CC="$CC" CFLAGS="--sysroot=$SDKROOT" test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } - cd ../../build + cd ../../build || exit else libmicrohttpdPath=$(pwd)/../3pp/libmicrohttpd echo "Using compiled libmicrohttpd" @@ -124,8 +134,8 @@ fi cmake -GNinja .. \ -DPORTABLE=1 \ -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=$OSX_TARGET \ --DCMAKE_CXX_FLAGS="-stdlib=libc++" \ --DCMAKE_SYSROOT=$(xcrun --show-sdk-path) \ +-DCMAKE_CXX_FLAGS="-nostdinc++ -isystem ${SDKROOT}/usr/include/c++/v1 -isystem ${SDKROOT}/usr/include" \ +-DCMAKE_SYSROOT="$(xcrun --show-sdk-path)" \ -DCMAKE_BUILD_TYPE=Release \ -DOPENSSL_FOUND=1 \ -DOPENSSL_INCLUDE_DIR=$opensslPath/include \ diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index 2d0638c1cc..698343d3d3 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -6,6 +6,13 @@ with_ccache=false OSX_TARGET=11.0 +MACOS_MAJOR=0 +if [ "$(uname)" = "Darwin" ]; then + MACOS_MAJOR=$(sw_vers -productVersion | cut -d. -f1) + export SDKROOT="$(xcrun --sdk macosx --show-sdk-path)" + echo "Using SDKROOT=$SDKROOT" +fi + while getopts 'taco:' flag; do case "${flag}" in t) with_tests=true ;; @@ -21,14 +28,26 @@ if [ ! -d "build" ]; then mkdir build cd build else - cd build + cd build || exit rm -rf .ninja* CMakeCache.txt fi export NONINTERACTIVE=1 brew install ninja libsodium libmicrohttpd pkg-config automake libtool autoconf gnutls export PATH=/usr/local/opt/ccache/libexec:$PATH -brew install llvm@16 + +if [ "$(uname)" = "Darwin" ]; then + if [ "$MACOS_MAJOR" -ge 15 ]; then + echo "macOS $MACOS_MAJOR detected -> using AppleClang (Xcode toolchain), NOT llvm@21" + export CC="$(xcrun --find clang)" + export CXX="$(xcrun --find clang++)" + else + echo "macOS $MACOS_MAJOR detected -> using Homebrew llvm@21" + brew install llvm@21 + export CC="$(brew --prefix llvm@21)"/bin/clang + export CXX="$(brew --prefix llvm@21)"/bin/clang++ + fi +fi if [ "$with_ccache" = true ]; then brew install ccache @@ -40,20 +59,12 @@ else export CCACHE_DISABLE=1 fi -if [ -f /opt/homebrew/opt/llvm@16/bin/clang ]; then - export CC=/opt/homebrew/opt/llvm@16/bin/clang - export CXX=/opt/homebrew/opt/llvm@16/bin/clang++ -else - export CC=/usr/local/opt/llvm@16/bin/clang - export CXX=/usr/local/opt/llvm@16/bin/clang++ -fi - if [ ! -d "lz4" ]; then git clone https://github.com/lz4/lz4 - cd lz4 + cd lz4 || exit lz4Path=`pwd` git checkout v1.9.4 - make -j4 + make -j4 CC="$CC" CFLAGS="--sysroot=$SDKROOT" test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } cd .. else @@ -61,26 +72,13 @@ else echo "Using compiled lz4" fi -if [ ! -d "zlib" ]; then - git clone https://github.com/madler/zlib.git - cd zlib - zlibPath=`pwd` - ./configure --static - make -j4 - test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } - cd .. -else - zlibPath=$(pwd)/zlib - echo "Using compiled zlib" -fi - brew unlink openssl@1.1 brew install openssl@3 brew unlink openssl@3 && brew link --overwrite openssl@3 cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \ --DCMAKE_CXX_FLAGS="-stdlib=libc++" \ --DCMAKE_SYSROOT=$(xcrun --show-sdk-path) \ +-DCMAKE_CXX_FLAGS="-nostdinc++ -isystem ${SDKROOT}/usr/include/c++/v1 -isystem ${SDKROOT}/usr/include" \ +-DCMAKE_SYSROOT="$(xcrun --show-sdk-path)" \ -DLZ4_FOUND=1 \ -DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a \ -DLZ4_INCLUDE_DIRS=$lz4Path/lib \ diff --git a/assembly/native/build-ubuntu-appimages.sh b/assembly/native/build-ubuntu-appimages.sh index ef54f06321..d6ca99f2ff 100644 --- a/assembly/native/build-ubuntu-appimages.sh +++ b/assembly/native/build-ubuntu-appimages.sh @@ -27,22 +27,22 @@ if [ ! -d "build" ]; then mkdir build cd build else - cd build + cd build || exit rm -rf .ninja* CMakeCache.txt fi -export CC=$(which clang-16) -export CXX=$(which clang++-16) +export CC=$(which clang-21) +export CXX=$(which clang++-21) if [ ! -d "../openssl_3" ]; then git clone https://github.com/openssl/openssl ../openssl_3 - cd ../openssl_3 + cd ../openssl_3 || exit opensslPath=`pwd` git checkout openssl-3.1.4 ./config make build_libs -j$(nproc) test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd ../build + cd ../build || exit else opensslPath=$(pwd)/../openssl_3 echo "Using compiled openssl_3" @@ -110,6 +110,6 @@ if [ "$with_artifacts" = true ]; then fi if [ "$with_tests" = true ]; then - cd build + cd build || exit ctest --output-on-failure --timeout 1800 fi diff --git a/assembly/native/build-ubuntu-portable-libs.sh b/assembly/native/build-ubuntu-portable-libs.sh index cf797cd233..f4dccdea99 100644 --- a/assembly/native/build-ubuntu-portable-libs.sh +++ b/assembly/native/build-ubuntu-portable-libs.sh @@ -28,22 +28,22 @@ if [ ! -d "build" ]; then mkdir build cd build else - cd build + cd build || exit rm -rf .ninja* CMakeCache.txt fi -export CC=$(which clang-16) -export CXX=$(which clang++-16) +export CC=$(which clang-21) +export CXX=$(which clang++-21) if [ ! -d "../3pp/lz4" ]; then mkdir -p ../3pp git clone https://github.com/lz4/lz4.git ../3pp/lz4 -cd ../3pp/lz4 +cd ../3pp/lz4 || exit lz4Path=`pwd` git checkout v1.9.4 CFLAGS="-fPIC" make -j$(nproc) test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } -cd ../../build +cd ../../build || exit else lz4Path=$(pwd)/../3pp/lz4 echo "Using compiled lz4" @@ -53,14 +53,14 @@ if [ ! -d "../3pp/libsodium" ]; then export LIBSODIUM_FULL_BUILD=1 mkdir -p ../3pp/libsodium wget -O ../3pp/libsodium/libsodium-1.0.18.tar.gz https://github.com/jedisct1/libsodium/releases/download/1.0.18-RELEASE/libsodium-1.0.18.tar.gz - cd ../3pp/libsodium + cd ../3pp/libsodium || exit tar xf libsodium-1.0.18.tar.gz - cd libsodium-1.0.18 + cd libsodium-1.0.18 || exit sodiumPath=`pwd` ./configure --with-pic --enable-static make -j$(nproc) test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } - cd ../../../build + cd ../../../build || exit else sodiumPath=$(pwd)/../3pp/libsodium/libsodium-1.0.18 echo "Using compiled libsodium" @@ -68,13 +68,13 @@ fi if [ ! -d "../3pp/openssl_3" ]; then git clone https://github.com/openssl/openssl ../3pp/openssl_3 - cd ../3pp/openssl_3 + cd ../3pp/openssl_3 || exit opensslPath=`pwd` git checkout openssl-3.1.4 ./config make build_libs -j$(nproc) test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd ../../build + cd ../../build || exit else opensslPath=$(pwd)/../3pp/openssl_3 echo "Using compiled openssl_3" @@ -82,12 +82,12 @@ fi if [ ! -d "../3pp/zlib" ]; then git clone https://github.com/madler/zlib.git ../3pp/zlib - cd ../3pp/zlib + cd ../3pp/zlib || exit zlibPath=`pwd` ./configure --static make -j$(nproc) test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } - cd ../../build + cd ../../build || exit else zlibPath=$(pwd)/../3pp/zlib echo "Using compiled zlib" @@ -95,12 +95,12 @@ fi if [ ! -d "../3pp/libmicrohttpd" ]; then git clone https://github.com/ton-blockchain/libmicrohttpd.git ../3pp/libmicrohttpd - cd ../3pp/libmicrohttpd + cd ../3pp/libmicrohttpd || exit libmicrohttpdPath=`pwd` ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - make -j4 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } - cd ../../build + cd ../../build || exit else libmicrohttpdPath=$(pwd)/../3pp/libmicrohttpd echo "Using compiled libmicrohttpd" diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh index 49a0205f68..38b0ddd2ba 100644 --- a/assembly/native/build-ubuntu-portable.sh +++ b/assembly/native/build-ubuntu-portable.sh @@ -30,22 +30,22 @@ if [ ! -d "build" ]; then mkdir build cd build else - cd build + cd build || exit rm -rf .ninja* CMakeCache.txt fi -export CC=$(which clang-16) -export CXX=$(which clang++-16) +export CC=$(which clang-21) +export CXX=$(which clang++-21) if [ ! -d "../3pp/lz4" ]; then mkdir -p ../3pp git clone https://github.com/lz4/lz4.git ../3pp/lz4 -cd ../3pp/lz4 +cd ../3pp/lz4 || exit lz4Path=`pwd` git checkout v1.9.4 CFLAGS="-fPIC" make -j$(nproc) test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } -cd ../../build +cd ../../build || exit else lz4Path=$(pwd)/../3pp/lz4 echo "Using compiled lz4" @@ -54,14 +54,14 @@ fi if [ ! -d "../3pp/libsodium" ]; then export LIBSODIUM_FULL_BUILD=1 git clone https://github.com/jedisct1/libsodium.git ../3pp/libsodium - cd ../3pp/libsodium + cd ../3pp/libsodium || exit sodiumPath=`pwd` git checkout 1.0.18 ./autogen.sh ./configure --with-pic --enable-static make -j$(nproc) test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } - cd ../../build + cd ../../build || exit else sodiumPath=$(pwd)/../3pp/libsodium echo "Using compiled libsodium" @@ -69,13 +69,13 @@ fi if [ ! -d "../3pp/openssl_3" ]; then git clone https://github.com/openssl/openssl ../3pp/openssl_3 - cd ../3pp/openssl_3 + cd ../3pp/openssl_3 || exit opensslPath=`pwd` git checkout openssl-3.1.4 ./config make build_libs -j$(nproc) test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd ../../build + cd ../../build || exit else opensslPath=$(pwd)/../3pp/openssl_3 echo "Using compiled openssl_3" @@ -83,12 +83,12 @@ fi if [ ! -d "../3pp/zlib" ]; then git clone https://github.com/madler/zlib.git ../3pp/zlib - cd ../3pp/zlib + cd ../3pp/zlib || exit zlibPath=`pwd` ./configure --static make -j$(nproc) test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } - cd ../../build + cd ../../build || exit else zlibPath=$(pwd)/../3pp/zlib echo "Using compiled zlib" @@ -96,12 +96,12 @@ fi if [ ! -d "../3pp/libmicrohttpd" ]; then git clone https://github.com/ton-blockchain/libmicrohttpd.git ../3pp/libmicrohttpd - cd ../3pp/libmicrohttpd + cd ../3pp/libmicrohttpd || exit libmicrohttpdPath=`pwd` ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - make -j4 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } - cd ../../build + cd ../../build || exit else libmicrohttpdPath=$(pwd)/../3pp/libmicrohttpd echo "Using compiled libmicrohttpd" @@ -170,7 +170,7 @@ if [ "$with_artifacts" = true ]; then fi if [ "$with_tests" = true ]; then - cd build + cd build || exit # ctest --output-on-failure -E "test-catchain|test-actors|test-smartcont|test-adnl|test-validator-session-state|test-dht|test-rldp" ctest --output-on-failure -E "test-adnl" fi diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index 4db66a11e7..5020a56582 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -30,23 +30,23 @@ if [ ! -d "build" ]; then mkdir build cd build else - cd build + cd build || exit rm -rf .ninja* CMakeCache.txt fi -export CC=$(which clang-16) -export CXX=$(which clang++-16) +export CC=$(which clang-21) +export CXX=$(which clang++-21) if [ ! -d "../openssl_3" ]; then git clone https://github.com/openssl/openssl ../openssl_3 - cd ../openssl_3 + cd ../openssl_3 || exit opensslPath=`pwd` git checkout openssl-3.1.4 ./config make build_libs -j$(nproc) test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd ../build + cd ../build || exit else opensslPath=$(pwd)/../openssl_3 echo "Using compiled openssl_3" diff --git a/assembly/wasm/fift-func-wasm-build-ubuntu.sh b/assembly/wasm/fift-func-wasm-build-ubuntu.sh index a99f27392b..0c08c8c47c 100644 --- a/assembly/wasm/fift-func-wasm-build-ubuntu.sh +++ b/assembly/wasm/fift-func-wasm-build-ubuntu.sh @@ -6,7 +6,7 @@ # wget https://apt.llvm.org/llvm.sh # chmod +x llvm.sh -# sudo ./llvm.sh 16 all +# sudo ./llvm.sh 21 clang with_artifacts=false scratch_new=false @@ -20,8 +20,8 @@ while getopts 'af' flag; do esac done -export CC=$(which clang-16) -export CXX=$(which clang++-16) +export CC=$(which clang-21) +export CXX=$(which clang++-21) export CCACHE_DISABLE=1 echo `pwd` @@ -32,7 +32,7 @@ fi if [ ! -d "openssl_3" ]; then git clone https://github.com/openssl/openssl openssl_3 - cd openssl_3 + cd openssl_3 || exit opensslPath=`pwd` git checkout openssl-3.1.4 ./config @@ -69,7 +69,7 @@ echo echo Using cloned emsdk fi -cd emsdk +cd emsdk || exit ./emsdk install 4.0.17 ./emsdk activate 4.0.17 EMSDK_DIR=`pwd` @@ -84,7 +84,7 @@ cd .. if [ ! -f "3pp_emscripten/openssl_em/openssl_em" ]; then mkdir -p 3pp_emscripten git clone https://github.com/openssl/openssl 3pp_emscripten/openssl_em - cd 3pp_emscripten/openssl_em + cd 3pp_emscripten/openssl_em || exit emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test no-tests no-fuzz-afl no-fuzz-libfuzzer sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile sed -i 's/-ldl//g' Makefile @@ -102,7 +102,7 @@ fi if [ ! -d "3pp_emscripten/zlib" ]; then git clone https://github.com/madler/zlib.git 3pp_emscripten/zlib - cd 3pp_emscripten/zlib + cd 3pp_emscripten/zlib || exit git checkout v1.3.1 ZLIB_DIR=`pwd` emconfigure ./configure --static @@ -116,7 +116,7 @@ fi if [ ! -d "3pp_emscripten/lz4" ]; then git clone https://github.com/lz4/lz4.git 3pp_emscripten/lz4 - cd 3pp_emscripten/lz4 + cd 3pp_emscripten/lz4 || exit git checkout v1.9.4 LZ4_DIR=`pwd` emmake make -j$(nproc) @@ -129,7 +129,7 @@ fi if [ ! -d "3pp_emscripten/libsodium" ]; then git clone https://github.com/jedisct1/libsodium 3pp_emscripten/libsodium - cd 3pp_emscripten/libsodium + cd 3pp_emscripten/libsodium || exit git checkout 1.0.18-RELEASE SODIUM_DIR=`pwd` emconfigure ./configure --disable-ssp @@ -141,7 +141,7 @@ else echo Using compiled libsodium with emscripten at $SODIUM_DIR fi -cd build +cd build || exit emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ -DZLIB_FOUND=1 \ From 154ed68a69410986b46dc13fe84126ad344618ae Mon Sep 17 00:00:00 2001 From: Danil Ovchinnikov Date: Mon, 22 Dec 2025 09:35:47 +0300 Subject: [PATCH 518/524] fix(build): bug in scripts for `Windows` (#1975) Co-authored-by: EmelyanenkoK --- assembly/native/build-windows-2019.bat | 2 +- assembly/native/build-windows-2022.bat | 2 +- assembly/native/build-windows.bat | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/assembly/native/build-windows-2019.bat b/assembly/native/build-windows-2019.bat index c8dedbf200..befda03107 100644 --- a/assembly/native/build-windows-2019.bat +++ b/assembly/native/build-windows-2019.bat @@ -35,8 +35,8 @@ IF %errorlevel% NEQ 0 ( echo Installing nasm... choco install -y nasm -where nasm SET PATH=%PATH%;C:\Program Files\NASM +where nasm IF %errorlevel% NEQ 0 ( echo Can't install nasm exit /b %errorlevel% diff --git a/assembly/native/build-windows-2022.bat b/assembly/native/build-windows-2022.bat index bfcf3089e5..3b5c1e3d56 100644 --- a/assembly/native/build-windows-2022.bat +++ b/assembly/native/build-windows-2022.bat @@ -35,8 +35,8 @@ IF %errorlevel% NEQ 0 ( echo Installing nasm... choco install -y nasm -where nasm SET PATH=%PATH%;C:\Program Files\NASM +where nasm IF %errorlevel% NEQ 0 ( echo Can't install nasm exit /b %errorlevel% diff --git a/assembly/native/build-windows.bat b/assembly/native/build-windows.bat index 4612affe32..8356ad00a9 100644 --- a/assembly/native/build-windows.bat +++ b/assembly/native/build-windows.bat @@ -35,8 +35,8 @@ IF %errorlevel% NEQ 0 ( echo Installing nasm... choco install -y nasm -where nasm SET PATH=%PATH%;C:\Program Files\NASM +where nasm IF %errorlevel% NEQ 0 ( echo Can't install nasm exit /b %errorlevel% From 70d73c87ad489f422a09f64a78f4fdc74edcb596 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Mon, 22 Dec 2025 04:36:55 -0500 Subject: [PATCH 519/524] Zero-initialize BlockIdExt::{root,file}_hash in default constructor (#1982) Let's not create unnecessary footguns. --- ton/ton-types.h | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ton/ton-types.h b/ton/ton-types.h index 010106a5a4..cc218ae6cc 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -245,8 +245,8 @@ inline bool operator<(const ShardIdFull& x, const BlockId& y) { struct BlockIdExt { BlockId id; - RootHash root_hash; - FileHash file_hash; + RootHash root_hash{}; + FileHash file_hash{}; BlockIdExt(WorkchainId workchain, ShardId shard, BlockSeqno seqno, const RootHash& root_hash, const FileHash& file_hash) : id{workchain, shard, seqno}, root_hash(root_hash), file_hash(file_hash) { @@ -255,11 +255,8 @@ struct BlockIdExt { : id(id), root_hash(root_hash), file_hash(file_hash) { } BlockIdExt(BlockId id, const FileHash& file_hash) : id(id), file_hash(file_hash) { - root_hash.set_zero(); } explicit BlockIdExt(BlockId id) : id(id) { - root_hash.set_zero(); - file_hash.set_zero(); } BlockIdExt() : id(workchainIdNotYet, 0, 0) { } From 3b32ebde0cfa5ee6fbfd21ee628290dd636923bf Mon Sep 17 00:00:00 2001 From: Evgeny Kapun Date: Wed, 31 Dec 2025 10:04:36 +0100 Subject: [PATCH 520/524] Remove spaces at EOL (#1989) * Remove spaces at EOL * Add a lint check to detect spaces at EOL --- .../build-ton-linux-arm64-shared.yml | 2 +- .../build-ton-linux-x86-64-appimage.yml | 2 +- .../build-ton-linux-x86-64-werror.yml | 4 + .github/workflows/create-release.yml | 2 +- CMake/UseLATEX.cmake | 2 +- Changelog.md | 6 +- LICENSE.LGPL | 8 +- README.md | 14 +- adnl/adnl-packet.h | 14 +- adnl/test/adnl-test-ping.cpp | 16 +- .../blockchain-explorer-http.hpp | 16 +- .../blockchain-explorer-query.cpp | 14 +- .../blockchain-explorer-query.hpp | 16 +- blockchain-explorer/blockchain-explorer.cpp | 16 +- blockchain-explorer/blockchain-explorer.hpp | 16 +- catchain/CMakeLists.txt | 6 +- common/CMakeLists.txt | 8 +- crypto/block/Binlog.cpp | 4 +- crypto/block/Binlog.h | 4 +- crypto/block/adjust-block.cpp | 16 +- crypto/block/block-db-impl.h | 4 +- crypto/block/block-db.cpp | 10 +- crypto/block/block.cpp | 4 +- crypto/block/block.tlb | 196 +++++++++--------- crypto/block/mc-config.cpp | 14 +- crypto/block/mc-config.h | 2 +- crypto/block/output-queue-merger.cpp | 6 +- crypto/block/test-block.cpp | 18 +- crypto/block/test-weight-distr.cpp | 16 +- crypto/block/transaction.cpp | 22 +- crypto/fift/Continuation.h | 4 +- crypto/fift/lib/Asm.fif | 52 ++--- crypto/fift/lib/Disasm.fif | 4 +- crypto/fift/lib/Fift.fif | 2 +- crypto/fift/lib/FiftExt.fif | 20 +- crypto/fift/lib/GetOpt.fif | 4 +- crypto/fift/lib/Lisp.fif | 18 +- crypto/fift/lib/Lists.fif | 2 +- crypto/fift/lib/Stack.fif | 14 +- crypto/fift/lib/TonUtil.fif | 12 +- crypto/func/abscode.cpp | 4 +- crypto/func/analyzer.cpp | 4 +- crypto/func/asmops.cpp | 4 +- .../bsc-bridge-collector/votes-collector.fc | 4 +- .../legacy_tests/config/config-code.fc | 4 +- .../legacy_tests/elector/elector-code.fc | 14 +- .../jetton-minter/imports/utils.fc | 2 +- .../jetton-minter/jetton-minter.fc | 2 +- .../jetton-wallet/imports/utils.fc | 2 +- .../jetton-wallet/jetton-wallet.fc | 6 +- .../jetton_JettonDefaultWallet.code.fc | 16 +- .../tact-examples/jetton_SampleJetton.code.fc | 20 +- .../maps_MapTestContract.code.fc | 26 +-- .../tact-examples/treasure_Treasure.code.fc | 14 +- .../whales-nominators/modules/get.fc | 2 +- .../whales-nominators/modules/model.fc | 14 +- .../whales-nominators/modules/op-common.fc | 12 +- .../modules/op-controller.fc | 40 ++-- .../modules/op-nominators.fc | 4 +- .../whales-nominators/modules/op-owner.fc | 2 +- .../whales-nominators/modules/utils-config.fc | 2 +- .../whales-nominators/modules/utils.fc | 10 +- .../whales-nominators/nominators.fc | 2 +- .../legacy_tests/whales-nominators/proxy.fc | 2 +- crypto/func/auto-tests/tests/w2.fc | 2 +- crypto/func/builtins.cpp | 6 +- crypto/func/codegen.cpp | 4 +- crypto/func/func.h | 32 +-- crypto/func/gen-abscode.cpp | 4 +- crypto/func/keywords.cpp | 4 +- crypto/func/optimize.cpp | 4 +- crypto/func/parse-func.cpp | 4 +- crypto/func/stack-transform.cpp | 4 +- crypto/func/test/w2.fc | 2 +- crypto/func/test/w4.fc | 2 +- crypto/func/test/w8.fc | 6 +- crypto/func/unify-types.cpp | 4 +- crypto/smartcont/CreateState.fif | 14 +- crypto/smartcont/LICENSE.LGPL | 8 +- crypto/smartcont/asm-to-cpp.fif | 4 +- crypto/smartcont/auto-dns.fif | 6 +- crypto/smartcont/complaint-vote-req.fif | 2 +- crypto/smartcont/config-code.fc | 4 +- crypto/smartcont/config-proposal-vote-req.fif | 4 +- .../smartcont/config-proposal-vote-signed.fif | 4 +- crypto/smartcont/create-config-proposal.fif | 10 +- .../create-config-upgrade-proposal.fif | 8 +- .../create-elector-upgrade-proposal.fif | 8 +- crypto/smartcont/dns-auto-code.fc | 58 +++--- crypto/smartcont/dns-manual-code.fc | 14 +- crypto/smartcont/elector-code.fc | 14 +- crypto/smartcont/envelope-complaint.fif | 2 +- crypto/smartcont/gen-zerostate-test.fif | 22 +- crypto/smartcont/gen-zerostate.fif | 18 +- crypto/smartcont/highload-wallet-v2-code.fc | 2 +- crypto/smartcont/highload-wallet-v2-one.fif | 6 +- crypto/smartcont/highload-wallet-v2.fif | 6 +- crypto/smartcont/highload-wallet.fif | 6 +- crypto/smartcont/manual-dns-manage.fif | 8 +- crypto/smartcont/mathlib.fc | 24 +-- crypto/smartcont/multisig-code.fc | 8 +- crypto/smartcont/new-highload-wallet-v2.fif | 4 +- crypto/smartcont/new-highload-wallet.fif | 6 +- crypto/smartcont/new-restricted-wallet.fif | 2 +- crypto/smartcont/new-restricted-wallet2.fif | 2 +- crypto/smartcont/new-restricted-wallet3.fif | 12 +- crypto/smartcont/new-wallet-v2.fif | 4 +- crypto/smartcont/new-wallet-v3.fif | 2 +- crypto/smartcont/new-wallet.fif | 6 +- crypto/smartcont/payment-channel-code.fc | 66 +++--- crypto/smartcont/show-addr.fif | 4 +- crypto/smartcont/testgiver.fif | 4 +- crypto/smartcont/tolk-stdlib/common.tolk | 52 ++--- .../smartcont/tolk-stdlib/exotic-cells.tolk | 4 +- crypto/smartcont/validator-elect-signed.fif | 2 +- crypto/smartcont/wallet-v2.fif | 4 +- crypto/smartcont/wallet-v3.fif | 4 +- crypto/smartcont/wallet.fif | 4 +- crypto/test/fift/cmdline.fif | 2 +- crypto/test/fift/fixed.fif | 2 +- crypto/test/fift/sort.fif | 2 +- crypto/test/fift/testdict.fif | 4 +- crypto/test/fift/testdict2.fif | 2 +- crypto/test/fift/testvm3.fif | 2 +- crypto/test/fift/testvm4.fif | 4 +- crypto/test/fift/testvm4a.fif | 4 +- crypto/test/fift/testvm4b.fif | 4 +- crypto/test/fift/testvm4c.fif | 4 +- crypto/test/fift/testvm4e.fif | 2 +- crypto/test/fift/testvm5.fif | 4 +- crypto/test/fift/testvm7.fif | 4 +- crypto/tl/boc.tlb | 16 +- crypto/tl/hashmap.tlb | 24 +-- crypto/tl/tlbc-gen-cpp.cpp | 4 +- crypto/tl/tlbc.cpp | 56 ++--- crypto/util/mintless-proof-generator.cpp | 2 +- crypto/vm/boc.cpp | 8 +- crypto/vm/cells/CellBuilder.cpp | 4 +- crypto/vm/dict.cpp | 18 +- crypto/vm/fmt.hpp | 4 +- dht-server/dht-server.hpp | 16 +- dht/CMakeLists.txt | 6 +- dht/utils/dht-ping-servers.cpp | 16 +- dht/utils/dht-resolve.cpp | 16 +- doc/ConfigParam-HOWTO | 24 +-- doc/DNS-HOWTO | 26 +-- doc/GlobalVersions.md | 2 +- doc/LiteClient-HOWTO | 42 ++-- doc/TonSites-HOWTO | 2 +- doc/Validator-HOWTO | 22 +- doc/catchain.tex | 6 +- doc/fiftbase.tex | 64 +++--- doc/smc-guidelines.txt | 2 +- doc/tblkch.tex | 134 ++++++------ doc/ton.tex | 10 +- doc/tvm.tex | 160 +++++++------- docker/README.md | 42 ++-- emulator/README.md | 2 +- emulator/emulator-extern.h | 80 +++---- example/android/README.md | 4 +- example/cpp/tonjson_example.cpp | 16 +- fec/CMakeLists.txt | 6 +- lite-client/lite-client.h | 16 +- overlay/CMakeLists.txt | 6 +- rldp/CMakeLists.txt | 6 +- rldp2/CMakeLists.txt | 6 +- .../smartcont/storage-contract.fif | 24 +-- .../smartcont/storage-provider.fif | 22 +- tdactor/benchmark/CMakeLists.txt | 4 +- tdactor/benchmark/benchmark.cpp | 16 +- tddb/test/io-bench.cpp | 16 +- tdfec/benchmark/benchmark.cpp | 16 +- tdnet/example/tcp_ping_pong.cpp | 16 +- tdnet/example/udp_ping_pong.cpp | 16 +- .../generate/generate_mime_types_gperf.cpp | 16 +- terminal/CMakeLists.txt | 8 +- test/generate-test-node-config.py | 24 +-- test/test-adnl.cpp | 16 +- test/test-catchain.cpp | 16 +- test/test-dht.cpp | 16 +- test/test-hello-world.cpp | 16 +- test/test-http.cpp | 16 +- test/test-overlay.cpp | 16 +- test/test-rldp.cpp | 16 +- test/test-rldp2.cpp | 16 +- test/test-rocksdb.cpp | 16 +- test/test-td-main.cpp | 16 +- test/test-ton-collator.cpp | 16 +- test/test-validator-session-state.cpp | 16 +- third-party/wingetopt/LICENSE | 12 +- third-party/wingetopt/README.md | 12 +- third-party/wingetopt/src/getopt.h | 6 +- tl/CMakeLists.txt | 16 +- tl/generate/generate_common.cpp | 16 +- tl/generate/generate_java.cpp | 16 +- tl/generate/scheme/lite_api.tl | 4 +- tl/generate/scheme/ton_api.tl | 96 ++++----- tolk-tester/tests/cells-slices.tolk | 18 +- tolk-tester/tests/comments-tests.tolk | 2 +- tolk-tester/tests/constants-tests.tolk | 2 +- tolk-tester/tests/dicts-demo.tolk | 2 +- tolk-tester/tests/enums-tests.tolk | 38 ++-- tolk-tester/tests/generics-4.tolk | 10 +- .../tests/invalid-declaration/err-1505.tolk | 2 +- .../tests/invalid-semantics/err-4090.tolk | 2 +- .../tests/invalid-semantics/err-4130.tolk | 2 +- .../tests/invalid-semantics/err-4180.tolk | 2 +- .../tests/invalid-semantics/err-4370.tolk | 2 +- .../tests/invalid-semantics/err-4372.tolk | 2 +- .../tests/invalid-semantics/err-4919.tolk | 2 +- .../tests/invalid-serialization/err-7883.tolk | 2 +- .../tests/invalid-symbol/err-2190.tolk | 2 +- .../tests/invalid-syntax/err-3710.tolk | 8 +- .../tests/invalid-syntax/err-3853.tolk | 2 +- .../tests/invalid-syntax/err-3967.tolk | 2 +- .../tests/invalid-syntax/err-3981.tolk | 2 +- .../tests/invalid-typing/err-6210.tolk | 2 +- .../tests/invalid-typing/err-6483.tolk | 2 +- .../tests/invalid-typing/err-6610.tolk | 2 +- .../tests/invalid-typing/err-6728.tolk | 2 +- tolk-tester/tests/lambdas-tests.tolk | 16 +- tolk-tester/tests/lazy-load-tests.tolk | 2 +- tolk-tester/tests/maps-tests.tolk | 28 +-- tolk-tester/tests/match-by-expr-tests.tolk | 8 +- tolk-tester/tests/meaningful-1.tolk | 2 +- tolk-tester/tests/methods-tests.tolk | 2 +- tolk-tester/tests/mutate-methods.tolk | 2 +- tolk-tester/tests/never-type-tests.tolk | 2 +- tolk-tester/tests/overloads-tests.tolk | 2 +- tolk-tester/tests/pack-unpack-2.tolk | 4 +- tolk-tester/tests/pack-unpack-5.tolk | 8 +- tolk-tester/tests/pack-unpack-8.tolk | 12 +- tolk-tester/tests/send-msg-3.tolk | 4 +- tolk-tester/tests/smart-cast-tests.tolk | 8 +- tolk-tester/tests/some-tests-4.tolk | 4 +- tolk-tester/tests/struct-tests.tolk | 10 +- tolk-tester/tests/type-aliases-tests.tolk | 2 +- tolk-tester/tests/union-types-tests.tolk | 4 +- .../warnings-not-errors/unreachable-6.tolk | 2 +- .../tests/warnings-not-errors/warnings-5.tolk | 2 +- tolk/abscode.cpp | 4 +- tolk/analyzer.cpp | 4 +- tolk/asmops.cpp | 4 +- tolk/ast-from-tokens.cpp | 10 +- tolk/ast-replicator.h | 2 +- tolk/ast.h | 4 +- tolk/builtins.cpp | 8 +- tolk/codegen.cpp | 4 +- tolk/compilation-errors.cpp | 6 +- tolk/constant-evaluator.cpp | 14 +- tolk/generics-helpers.cpp | 4 +- tolk/lexer.h | 2 +- tolk/maps-kv-api.cpp | 28 +-- tolk/optimize.cpp | 20 +- tolk/overload-resolution.cpp | 6 +- tolk/pack-unpack-api.cpp | 2 +- tolk/pack-unpack-serializers.cpp | 14 +- tolk/pipe-ast-to-legacy.cpp | 4 +- tolk/pipe-calc-rvalue-lvalue.cpp | 4 +- tolk/pipe-check-const-expr.cpp | 2 +- tolk/pipe-check-inferred-types.cpp | 6 +- tolk/pipe-check-private-fields.cpp | 6 +- tolk/pipe-check-serialized-fields.cpp | 6 +- tolk/pipe-infer-types-and-calls.cpp | 10 +- tolk/pipe-lazy-load-insertions.cpp | 4 +- tolk/pipe-mini-borrow-checker.cpp | 6 +- tolk/pipe-resolve-identifiers.cpp | 2 +- tolk/pipe-resolve-types.cpp | 6 +- tolk/pretty-print.cpp | 4 +- tolk/send-message-api.cpp | 4 +- tolk/smart-casts-cfg.cpp | 4 +- tolk/src-file.cpp | 2 +- tolk/src-file.h | 4 +- tolk/stack-transform.cpp | 4 +- tolk/tolk.h | 22 +- tolk/type-system.cpp | 2 +- tolk/type-system.h | 4 +- tonlib/test/online.cpp | 16 +- utils/pack-viewer.cpp | 16 +- .../validator-engine-console-query.cpp | 14 +- .../validator-engine-console-query.h | 16 +- .../validator-engine-console.h | 16 +- validator-engine/validator-engine.hpp | 14 +- validator-session/CMakeLists.txt | 6 +- validator/CMakeLists.txt | 34 +-- validator/impl/CMakeLists.txt | 8 +- 286 files changed, 1710 insertions(+), 1706 deletions(-) diff --git a/.github/workflows/build-ton-linux-arm64-shared.yml b/.github/workflows/build-ton-linux-arm64-shared.yml index 5c5e86d17a..cebfb39f6c 100644 --- a/.github/workflows/build-ton-linux-arm64-shared.yml +++ b/.github/workflows/build-ton-linux-arm64-shared.yml @@ -13,7 +13,7 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v3 - with: + with: submodules: 'recursive' - name: Date Stamp diff --git a/.github/workflows/build-ton-linux-x86-64-appimage.yml b/.github/workflows/build-ton-linux-x86-64-appimage.yml index aa9787cd69..8f93ce90c5 100644 --- a/.github/workflows/build-ton-linux-x86-64-appimage.yml +++ b/.github/workflows/build-ton-linux-x86-64-appimage.yml @@ -21,7 +21,7 @@ jobs: - name: Install system libraries run: | sudo apt update - sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache libgsl-dev libblas-dev libgslcblas0 + sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache libgsl-dev libblas-dev libgslcblas0 mkdir ~/.ccache 3pp - name: Install clang-21 diff --git a/.github/workflows/build-ton-linux-x86-64-werror.yml b/.github/workflows/build-ton-linux-x86-64-werror.yml index 1835030224..d1ac635289 100644 --- a/.github/workflows/build-ton-linux-x86-64-werror.yml +++ b/.github/workflows/build-ton-linux-x86-64-werror.yml @@ -42,6 +42,10 @@ jobs: -DTON_WERROR_BUILD=On ninja -C build + - name: Check spaces at EOL + run: | + ! git grep --cached -Il '\s$' :!example/android/third_party + - name: Check C++ formatting run: | # TODO: Run clang-format only on changed files. diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index a09f0fc9bf..b44f35f653 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -504,7 +504,7 @@ jobs: run: | mkdir smartcont_lib cd smartcont_lib - cp -r ../artifacts/ton-x86_64-linux/{smartcont,lib} . + cp -r ../artifacts/ton-x86_64-linux/{smartcont,lib} . zip -r smartcont_lib.zip . gh release upload ${{ steps.tag.outputs.TAG }} smartcont_lib.zip diff --git a/CMake/UseLATEX.cmake b/CMake/UseLATEX.cmake index 5380522525..a3acd9d07c 100644 --- a/CMake/UseLATEX.cmake +++ b/CMake/UseLATEX.cmake @@ -1589,7 +1589,7 @@ function(add_latex_targets_internal) else() set(bib_compiler ${BIBTEX_COMPILER}) set(bib_compiler_flags ${BIBTEX_COMPILER_ARGS}) - endif() + endif() if(LATEX_MULTIBIB_NEWCITES) foreach (multibib_auxfile ${LATEX_MULTIBIB_NEWCITES}) latex_get_filename_component(multibib_target ${multibib_auxfile} NAME_WE) diff --git a/Changelog.md b/Changelog.md index 3580073ddc..2b549d2a51 100644 --- a/Changelog.md +++ b/Changelog.md @@ -127,7 +127,7 @@ Besides the work of the core team, this update is based on the efforts of @krigg ## 2024.08 Update -1. Introduction of dispatch queues, message envelopes with transaction chain metadata, and explicitly stored msg_queue size, which will be activated by `Config8.version >= 8` and new `Config8.capabilities` bits: `capStoreOutMsgQueueSize`, `capMsgMetadata`, `capDeferMessages`. +1. Introduction of dispatch queues, message envelopes with transaction chain metadata, and explicitly stored msg_queue size, which will be activated by `Config8.version >= 8` and new `Config8.capabilities` bits: `capStoreOutMsgQueueSize`, `capMsgMetadata`, `capDeferMessages`. 2. A number of changes to transaction executor which will activated for `Config8.version >= 8`: - Check mode on invalid `action_send_msg`. Ignore action if `IGNORE_ERROR` (+2) bit is set, bounce if `BOUNCE_ON_FAIL` (+16) bit is set. - Slightly change random seed generation to fix mix of `addr_rewrite` and `addr`. @@ -146,7 +146,7 @@ Besides the work of the core team, this update is based on the efforts of @krigg 1. Make Jemalloc default allocator 2. Add candidate broadcasting and caching -3. Limit per address speed for external messages broadcast by reasonably large number +3. Limit per address speed for external messages broadcast by reasonably large number 4. Overlay improvements: fix dropping peers in small custom overlays, fix wrong certificate on missed keyblocks 5. Extended statistics and logs for celldb usage, session stats, persistent state serialization 6. Tonlib and explorer fixes @@ -253,7 +253,7 @@ Besides the work of the core team, this update is based on the efforts of @aleks ## 2023.04 Update 1. CPU load optimization: previous DHT reconnect policy was too aggressive -2. Network throughput improvements: granular control on external message broadcast, optimize celldb GC, adjust state serialization and block downloading timings, rldp2 for states and archives +2. Network throughput improvements: granular control on external message broadcast, optimize celldb GC, adjust state serialization and block downloading timings, rldp2 for states and archives 3. Update for Fift (namespaces) and Fift libraries (list of improvements: https://github.com/ton-blockchain/ton/issues/631) 4. Better handling of incorrect inputs in funC: fix UB and prevent crashes on some inputs, improve optimizing int consts and unused variables in FunC, fix analyzing repeat loop. FunC version is increase to 0.4.3. 5. `listBlockTransactionsExt` in liteserver added diff --git a/LICENSE.LGPL b/LICENSE.LGPL index b482fc4ed4..464e46b9f7 100644 --- a/LICENSE.LGPL +++ b/LICENSE.LGPL @@ -133,7 +133,7 @@ such a program is covered only if its contents constitute a work based on the Library (independent of the use of the Library in a tool for writing it). Whether that is true depends on what the Library does and what the program that uses the Library does. - + 1. You may copy and distribute verbatim copies of the Library's complete source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an @@ -203,7 +203,7 @@ instead of to this License. (If a newer version than version 2 of the ordinary GNU General Public License has appeared, then you can specify that version instead if you wish.) Do not make any other change in these notices. - + Once this change is made in a given copy, it is irreversible for that copy, so the ordinary GNU General Public License applies to all subsequent copies and derivative works made from that copy. @@ -308,7 +308,7 @@ restrictions of other proprietary libraries that do not normally accompany the operating system. Such a contradiction means you cannot use both them and the Library together in an executable that you distribute. - + 7. You may place library facilities that are a work based on the Library side-by-side in a single library together with other library facilities not covered by this License, and distribute such a combined @@ -401,7 +401,7 @@ conditions either of that version or of any later version published by the Free Software Foundation. If the Library does not specify a license version number, you may choose any version ever published by the Free Software Foundation. - + 14. If you wish to incorporate parts of the Library into other free programs whose distribution conditions are incompatible with these, write to the author to ask for permission. For software which is diff --git a/README.md b/README.md index 9833cecd92..254a41e7a3 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@
-## +##

@@ -76,7 +76,7 @@ Install additional system libraries ```bash sudo apt-get update sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev - + wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh sudo ./llvm.sh 21 clang @@ -85,7 +85,7 @@ Compile TON binaries ```bash cp assembly/native/build-ubuntu-shared.sh . chmod +x build-ubuntu-shared.sh - ./build-ubuntu-shared.sh + ./build-ubuntu-shared.sh ``` ### MacOS 11, 12 (x86-64, aarch64) @@ -99,11 +99,11 @@ Compile TON binaries You need to install `MS Visual Studio 2022` first. Go to https://www.visualstudio.com/downloads/ and download `MS Visual Studio 2022 Community`. -Launch installer and select `Desktop development with C++`. +Launch installer and select `Desktop development with C++`. After installation, also make sure that `cmake` is globally available by adding `C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin` to the system `PATH` (adjust the path per your needs). -Open an elevated (Run as Administrator) `x86-64 Native Tools Command Prompt for VS 2022`, go to the root folder and execute: +Open an elevated (Run as Administrator) `x86-64 Native Tools Command Prompt for VS 2022`, go to the root folder and execute: ```bash copy assembly\native\build-windows.bat . build-windows.bat @@ -114,7 +114,7 @@ Install additional system libraries on Ubuntu ```bash sudo apt-get update sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev - + wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh sudo ./llvm.sh 21 clang @@ -144,7 +144,7 @@ Compile TON tonlib library ### TON portable binaries Linux portable binaries are wrapped into AppImages, at the same time MacOS portable binaries are statically linked executables. -Linux and MacOS binaries are available for both x86-64 and arm64 architectures. +Linux and MacOS binaries are available for both x86-64 and arm64 architectures. ## Running tests diff --git a/adnl/adnl-packet.h b/adnl/adnl-packet.h index 6e12a7df88..7ffa969524 100644 --- a/adnl/adnl-packet.h +++ b/adnl/adnl-packet.h @@ -26,19 +26,19 @@ namespace ton { namespace adnl { /* - from:flags.0?PublicKey + from:flags.0?PublicKey from_short:flags.1?adnl.id.short - message:flags.2?adnl.Message + message:flags.2?adnl.Message messages:flags.3?(vector adnl.Message) - address:flags.4?adnl.addressList + address:flags.4?adnl.addressList priority_address:flags.5?adnl.addressList - seqno:flags.6?long - confirm_seqno:flags.7?long + seqno:flags.6?long + confirm_seqno:flags.7?long recv_addr_list_version:flags.8?int recv_priority_addr_list_version:flags.9?int - reinit_date:flags.10?int + reinit_date:flags.10?int dst_reinit_date:flags.10?int - signature:flags.11?bytes + signature:flags.11?bytes */ // total packet length: diff --git a/adnl/test/adnl-test-ping.cpp b/adnl/test/adnl-test-ping.cpp index 818487abb5..c348938191 100644 --- a/adnl/test/adnl-test-ping.cpp +++ b/adnl/test/adnl-test-ping.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/blockchain-explorer/blockchain-explorer-http.hpp b/blockchain-explorer/blockchain-explorer-http.hpp index c0f4f15874..35030f16e7 100644 --- a/blockchain-explorer/blockchain-explorer-http.hpp +++ b/blockchain-explorer/blockchain-explorer-http.hpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/blockchain-explorer/blockchain-explorer-query.cpp b/blockchain-explorer/blockchain-explorer-query.cpp index c07f3e216b..5aa001559c 100644 --- a/blockchain-explorer/blockchain-explorer-query.cpp +++ b/blockchain-explorer/blockchain-explorer-query.cpp @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. along with TON Blockchain. If not, see . diff --git a/blockchain-explorer/blockchain-explorer-query.hpp b/blockchain-explorer/blockchain-explorer-query.hpp index 911d8ec298..d9c6d2990a 100644 --- a/blockchain-explorer/blockchain-explorer-query.hpp +++ b/blockchain-explorer/blockchain-explorer-query.hpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/blockchain-explorer/blockchain-explorer.cpp b/blockchain-explorer/blockchain-explorer.cpp index 1a0d909a38..7d377aee1f 100644 --- a/blockchain-explorer/blockchain-explorer.cpp +++ b/blockchain-explorer/blockchain-explorer.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -13,13 +13,13 @@ You should have received a copy of the GNU General Public License - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. along with TON Blockchain. If not, see . diff --git a/blockchain-explorer/blockchain-explorer.hpp b/blockchain-explorer/blockchain-explorer.hpp index 432552ed52..6c41b473af 100644 --- a/blockchain-explorer/blockchain-explorer.hpp +++ b/blockchain-explorer/blockchain-explorer.hpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/catchain/CMakeLists.txt b/catchain/CMakeLists.txt index 46b7c13420..62bacbbc2d 100644 --- a/catchain/CMakeLists.txt +++ b/catchain/CMakeLists.txt @@ -23,9 +23,9 @@ set(CATCHAIN_SOURCE add_library(catchain STATIC ${CATCHAIN_SOURCE}) -target_include_directories(overlay PUBLIC - $ - $/.. +target_include_directories(overlay PUBLIC + $ + $/.. ${OPENSSL_INCLUDE_DIR} ) target_link_libraries(catchain PRIVATE tdutils tdactor adnl tl_api dht tdfec overlay) diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 9252178b50..61c2b71b79 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -1,6 +1,6 @@ set(COMMON_SOURCE checksum.h - errorcode.h + errorcode.h status.h io.hpp @@ -12,9 +12,9 @@ set(COMMON_SOURCE add_library(common STATIC ${COMMON_SOURCE}) -target_include_directories(common PUBLIC - $ - $/.. +target_include_directories(common PUBLIC + $ + $/.. ${OPENSSL_INCLUDE_DIR} ) target_link_libraries(common PRIVATE tdutils ton_crypto ) diff --git a/crypto/block/Binlog.cpp b/crypto/block/Binlog.cpp index fc5b055c2c..5e29d616c5 100644 --- a/crypto/block/Binlog.cpp +++ b/crypto/block/Binlog.cpp @@ -25,9 +25,9 @@ namespace block { /* - * + * * GENERIC BINLOG (move to separate file) - * + * */ BinlogBuffer::BinlogBuffer(std::unique_ptr cb, std::size_t _max_size, td::FileFd fd) diff --git a/crypto/block/Binlog.h b/crypto/block/Binlog.h index 64142fe802..566aa44764 100644 --- a/crypto/block/Binlog.h +++ b/crypto/block/Binlog.h @@ -23,9 +23,9 @@ namespace block { /* - * + * * BINLOG (BUFFERS) -- move to separate file - * + * */ class BinlogBuffer; diff --git a/crypto/block/adjust-block.cpp b/crypto/block/adjust-block.cpp index 6bc270fd9f..475f7e8159 100644 --- a/crypto/block/adjust-block.cpp +++ b/crypto/block/adjust-block.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/crypto/block/block-db-impl.h b/crypto/block/block-db-impl.h index d7e99bdd49..bf38e8c432 100644 --- a/crypto/block/block-db-impl.h +++ b/crypto/block/block-db-impl.h @@ -29,9 +29,9 @@ namespace block { using td::Ref; /* - * + * * BLOCK DATABASE - * + * */ class BlockDbImpl; diff --git a/crypto/block/block-db.cpp b/crypto/block/block-db.cpp index d9eef85d40..233a2e568b 100644 --- a/crypto/block/block-db.cpp +++ b/crypto/block/block-db.cpp @@ -124,9 +124,9 @@ FileHash compute_file_hash(td::Slice data) { } /* - * + * * ZEROSTATE CONFIGURATION - * + * */ td::Status ZerostateInfo::base_check() { @@ -154,9 +154,9 @@ td::Status ZerostateInfo::base_check() { } /* - * - * BLOCK DATABASE - * + * + * BLOCK DATABASE + * */ std::string compute_db_filename(std::string base_dir, const FileHash& file_hash, int depth) { diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index d7b3cc1b83..8426e4aad2 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -1787,9 +1787,9 @@ void MtCarloComputeShare::gen_vset() { } /* - * + * * Other block-related functions - * + * */ bool store_UInt7(vm::CellBuilder& cb, unsigned long long value) { diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index c269440a06..8a7e0ee31b 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -19,11 +19,11 @@ bit$_ (## 1) = Bit; */ // ordinary Hashmap / HashmapE, with fixed length keys // -hm_edge#_ {n:#} {X:Type} {l:#} {m:#} label:(HmLabel ~l n) +hm_edge#_ {n:#} {X:Type} {l:#} {m:#} label:(HmLabel ~l n) {n = (~m) + l} node:(HashmapNode m X) = Hashmap n X; hmn_leaf#_ {X:Type} value:X = HashmapNode 0 X; -hmn_fork#_ {n:#} {X:Type} left:^(Hashmap n X) +hmn_fork#_ {n:#} {X:Type} left:^(Hashmap n X) right:^(Hashmap n X) = HashmapNode (n + 1) X; hml_short$0 {m:#} {n:#} len:(Unary ~n) {n <= m} s:(n * Bit) = HmLabel ~n m; @@ -39,38 +39,38 @@ hme_root$1 {n:#} {X:Type} root:^(Hashmap n X) = HashmapE n X; // true#_ = True; _ {n:#} _:(Hashmap n True) = BitstringSet n; -// HashmapAug, hashmap with an extra value +// HashmapAug, hashmap with an extra value // (augmentation) of type Y at every node // -ahm_edge#_ {n:#} {X:Type} {Y:Type} {l:#} {m:#} - label:(HmLabel ~l n) {n = (~m) + l} +ahm_edge#_ {n:#} {X:Type} {Y:Type} {l:#} {m:#} + label:(HmLabel ~l n) {n = (~m) + l} node:(HashmapAugNode m X Y) = HashmapAug n X Y; ahmn_leaf#_ {X:Type} {Y:Type} extra:Y value:X = HashmapAugNode 0 X Y; ahmn_fork#_ {n:#} {X:Type} {Y:Type} left:^(HashmapAug n X Y) right:^(HashmapAug n X Y) extra:Y = HashmapAugNode (n + 1) X Y; -ahme_empty$0 {n:#} {X:Type} {Y:Type} extra:Y +ahme_empty$0 {n:#} {X:Type} {Y:Type} extra:Y = HashmapAugE n X Y; -ahme_root$1 {n:#} {X:Type} {Y:Type} root:^(HashmapAug n X Y) +ahme_root$1 {n:#} {X:Type} {Y:Type} root:^(HashmapAug n X Y) extra:Y = HashmapAugE n X Y; // VarHashmap / VarHashmapE, with variable-length keys // -vhm_edge#_ {n:#} {X:Type} {l:#} {m:#} label:(HmLabel ~l n) - {n = (~m) + l} node:(VarHashmapNode m X) +vhm_edge#_ {n:#} {X:Type} {l:#} {m:#} label:(HmLabel ~l n) + {n = (~m) + l} node:(VarHashmapNode m X) = VarHashmap n X; vhmn_leaf$00 {n:#} {X:Type} value:X = VarHashmapNode n X; -vhmn_fork$01 {n:#} {X:Type} left:^(VarHashmap n X) - right:^(VarHashmap n X) value:(Maybe X) +vhmn_fork$01 {n:#} {X:Type} left:^(VarHashmap n X) + right:^(VarHashmap n X) value:(Maybe X) = VarHashmapNode (n + 1) X; -vhmn_cont$1 {n:#} {X:Type} branch:Bit child:^(VarHashmap n X) +vhmn_cont$1 {n:#} {X:Type} branch:Bit child:^(VarHashmap n X) value:X = VarHashmapNode (n + 1) X; // nothing$0 {X:Type} = Maybe X; // just$1 {X:Type} value:X = Maybe X; vhme_empty$0 {n:#} {X:Type} = VarHashmapE n X; -vhme_root$1 {n:#} {X:Type} root:^(VarHashmap n X) +vhme_root$1 {n:#} {X:Type} root:^(VarHashmap n X) = VarHashmapE n X; // @@ -78,16 +78,16 @@ vhme_root$1 {n:#} {X:Type} root:^(VarHashmap n X) // constituting a prefix code // -phm_edge#_ {n:#} {X:Type} {l:#} {m:#} label:(HmLabel ~l n) - {n = (~m) + l} node:(PfxHashmapNode m X) +phm_edge#_ {n:#} {X:Type} {l:#} {m:#} label:(HmLabel ~l n) + {n = (~m) + l} node:(PfxHashmapNode m X) = PfxHashmap n X; phmn_leaf$0 {n:#} {X:Type} value:X = PfxHashmapNode n X; -phmn_fork$1 {n:#} {X:Type} left:^(PfxHashmap n X) +phmn_fork$1 {n:#} {X:Type} left:^(PfxHashmap n X) right:^(PfxHashmap n X) = PfxHashmapNode (n + 1) X; phme_empty$0 {n:#} {X:Type} = PfxHashmapE n X; -phme_root$1 {n:#} {X:Type} root:^(PfxHashmap n X) +phme_root$1 {n:#} {X:Type} root:^(PfxHashmap n X) = PfxHashmapE n X; /* * @@ -98,42 +98,42 @@ phme_root$1 {n:#} {X:Type} root:^(PfxHashmap n X) // TON BLOCK LAYOUT // addr_none$00 = MsgAddressExt; -addr_extern$01 len:(## 9) external_address:(bits len) +addr_extern$01 len:(## 9) external_address:(bits len) = MsgAddressExt; anycast_info$_ depth:(#<= 30) { depth >= 1 } rewrite_pfx:(bits depth) = Anycast; -addr_std$10 anycast:(Maybe Anycast) +addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; -addr_var$11 anycast:(Maybe Anycast) addr_len:(## 9) +addr_var$11 anycast:(Maybe Anycast) addr_len:(## 9) workchain_id:int32 address:(bits addr_len) = MsgAddressInt; _ _:MsgAddressInt = MsgAddress; _ _:MsgAddressExt = MsgAddress; // var_uint$_ {n:#} len:(#< n) value:(uint (len * 8)) = VarUInteger n; -var_int$_ {n:#} len:(#< n) value:(int (len * 8)) +var_int$_ {n:#} len:(#< n) value:(int (len * 8)) = VarInteger n; nanograms$_ amount:(VarUInteger 16) = Grams; _ grams:Grams = Coins; // -extra_currencies$_ dict:(HashmapE 32 (VarUInteger 32)) +extra_currencies$_ dict:(HashmapE 32 (VarUInteger 32)) = ExtraCurrencyCollection; -currencies$_ grams:Grams other:ExtraCurrencyCollection +currencies$_ grams:Grams other:ExtraCurrencyCollection = CurrencyCollection; // int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool - src:MsgAddressInt dest:MsgAddressInt + src:MsgAddressInt dest:MsgAddressInt value:CurrencyCollection extra_flags:(VarUInteger 16) fwd_fee:Grams created_lt:uint64 created_at:uint32 = CommonMsgInfo; -ext_in_msg_info$10 src:MsgAddressExt dest:MsgAddressInt +ext_in_msg_info$10 src:MsgAddressExt dest:MsgAddressInt import_fee:Grams = CommonMsgInfo; ext_out_msg_info$11 src:MsgAddressInt dest:MsgAddressExt created_lt:uint64 created_at:uint32 = CommonMsgInfo; int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool - src:MsgAddress dest:MsgAddressInt + src:MsgAddress dest:MsgAddressInt value:CurrencyCollection extra_flags:(VarUInteger 16) fwd_fee:Grams created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; ext_out_msg_info$11 src:MsgAddress dest:MsgAddressExt @@ -161,7 +161,7 @@ message$_ {X:Type} info:CommonMsgInfoRelaxed body:(Either X ^X) = MessageRelaxed X; _ init:(Maybe (Either StateInit ^StateInit)) = EitherStateInit; - + _ (Message Any) = MessageAny; @@ -175,14 +175,14 @@ new_bounce_body#fffffffe = NewBounceBody; // -interm_addr_regular$0 use_dest_bits:(#<= 96) +interm_addr_regular$0 use_dest_bits:(#<= 96) = IntermediateAddress; -interm_addr_simple$10 workchain_id:int8 addr_pfx:uint64 +interm_addr_simple$10 workchain_id:int8 addr_pfx:uint64 = IntermediateAddress; interm_addr_ext$11 workchain_id:int32 addr_pfx:uint64 = IntermediateAddress; -msg_envelope#4 cur_addr:IntermediateAddress - next_addr:IntermediateAddress fwd_fee_remaining:Grams +msg_envelope#4 cur_addr:IntermediateAddress + next_addr:IntermediateAddress fwd_fee_remaining:Grams msg:^(Message Any) = MsgEnvelope; msg_metadata#0 depth:uint32 initiator_addr:MsgAddressInt initiator_lt:uint64 = MsgMetadata; msg_envelope_v2#5 cur_addr:IntermediateAddress @@ -191,45 +191,45 @@ msg_envelope_v2#5 cur_addr:IntermediateAddress emitted_lt:(Maybe uint64) metadata:(Maybe MsgMetadata) = MsgEnvelope; // -msg_import_ext$000 msg:^(Message Any) transaction:^Transaction +msg_import_ext$000 msg:^(Message Any) transaction:^Transaction = InMsg; -msg_import_ihr$010 msg:^(Message Any) transaction:^Transaction +msg_import_ihr$010 msg:^(Message Any) transaction:^Transaction ihr_fee:Grams proof_created:^Cell = InMsg; msg_import_imm$011 in_msg:^MsgEnvelope transaction:^Transaction fwd_fee:Grams = InMsg; -msg_import_fin$100 in_msg:^MsgEnvelope +msg_import_fin$100 in_msg:^MsgEnvelope transaction:^Transaction fwd_fee:Grams = InMsg; -msg_import_tr$101 in_msg:^MsgEnvelope out_msg:^MsgEnvelope +msg_import_tr$101 in_msg:^MsgEnvelope out_msg:^MsgEnvelope transit_fee:Grams = InMsg; -msg_discard_fin$110 in_msg:^MsgEnvelope transaction_id:uint64 +msg_discard_fin$110 in_msg:^MsgEnvelope transaction_id:uint64 fwd_fee:Grams = InMsg; -msg_discard_tr$111 in_msg:^MsgEnvelope transaction_id:uint64 +msg_discard_tr$111 in_msg:^MsgEnvelope transaction_id:uint64 fwd_fee:Grams proof_delivered:^Cell = InMsg; msg_import_deferred_fin$00100 in_msg:^MsgEnvelope transaction:^Transaction fwd_fee:Grams = InMsg; msg_import_deferred_tr$00101 in_msg:^MsgEnvelope out_msg:^MsgEnvelope = InMsg; // -import_fees$_ fees_collected:Grams +import_fees$_ fees_collected:Grams value_imported:CurrencyCollection = ImportFees; _ (HashmapAugE 256 InMsg ImportFees) = InMsgDescr; msg_export_ext$000 msg:^(Message Any) transaction:^Transaction = OutMsg; -msg_export_imm$010 out_msg:^MsgEnvelope +msg_export_imm$010 out_msg:^MsgEnvelope transaction:^Transaction reimport:^InMsg = OutMsg; -msg_export_new$001 out_msg:^MsgEnvelope +msg_export_new$001 out_msg:^MsgEnvelope transaction:^Transaction = OutMsg; -msg_export_tr$011 out_msg:^MsgEnvelope +msg_export_tr$011 out_msg:^MsgEnvelope imported:^InMsg = OutMsg; msg_export_deq$1100 out_msg:^MsgEnvelope import_block_lt:uint63 = OutMsg; msg_export_deq_short$1101 msg_env_hash:bits256 next_workchain:int32 next_addr_pfx:uint64 import_block_lt:uint64 = OutMsg; -msg_export_tr_req$111 out_msg:^MsgEnvelope +msg_export_tr_req$111 out_msg:^MsgEnvelope imported:^InMsg = OutMsg; -msg_export_deq_imm$100 out_msg:^MsgEnvelope +msg_export_deq_imm$100 out_msg:^MsgEnvelope reimport:^InMsg = OutMsg; msg_export_new_defer$10100 out_msg:^MsgEnvelope transaction:^Transaction = OutMsg; @@ -243,7 +243,7 @@ _ (HashmapAugE 256 OutMsg CurrencyCollection) = OutMsgDescr; _ (HashmapAugE 352 EnqueuedMsg uint64) = OutMsgQueue; processed_upto$_ last_msg_lt:uint64 last_msg_hash:bits256 = ProcessedUpto; -// key is [ shard:uint64 mc_seqno:uint32 ] +// key is [ shard:uint64 mc_seqno:uint32 ] _ (HashmapE 96 ProcessedUpto) = ProcessedInfo; ihr_pending$_ import_lt:uint64 = IhrPendingSince; @@ -272,7 +272,7 @@ account$1 addr:MsgAddressInt storage_stat:StorageInfo storage:AccountStorage = Account; account_storage$_ last_trans_lt:uint64 - balance:CurrencyCollection state:AccountState + balance:CurrencyCollection state:AccountState = AccountStorage; account_uninit$00 = AccountState; @@ -284,14 +284,14 @@ acc_state_frozen$01 = AccountStatus; acc_state_active$10 = AccountStatus; acc_state_nonexist$11 = AccountStatus; -account_descr$_ account:^Account last_trans_hash:bits256 +account_descr$_ account:^Account last_trans_hash:bits256 last_trans_lt:uint64 = ShardAccount; depth_balance$_ split_depth:(#<= 30) balance:CurrencyCollection = DepthBalanceInfo; _ (HashmapAugE 256 ShardAccount DepthBalanceInfo) = ShardAccounts; -transaction$0111 account_addr:bits256 lt:uint64 +transaction$0111 account_addr:bits256 lt:uint64 prev_trans_hash:bits256 prev_trans_lt:uint64 now:uint32 outmsg_cnt:uint15 orig_status:AccountStatus end_status:AccountStatus @@ -312,7 +312,7 @@ acc_trans#5 account_addr:bits256 _ (HashmapAugE 256 AccountBlock CurrencyCollection) = ShardAccountBlocks; // -tr_phase_storage$_ storage_fees_collected:Grams +tr_phase_storage$_ storage_fees_collected:Grams storage_fees_due:(Maybe Grams) status_change:AccStatusChange = TrStoragePhase; @@ -326,7 +326,7 @@ tr_phase_credit$_ due_fees_collected:(Maybe Grams) tr_phase_compute_skipped$0 reason:ComputeSkipReason = TrComputePhase; -tr_phase_compute_vm$1 success:Bool msg_state_used:Bool +tr_phase_compute_vm$1 success:Bool msg_state_used:Bool account_activated:Bool gas_fees:Grams ^[ gas_used:(VarUInteger 7) gas_limit:(VarUInteger 7) gas_credit:(Maybe (VarUInteger 3)) @@ -343,7 +343,7 @@ tr_phase_action$_ success:Bool valid:Bool no_funds:Bool status_change:AccStatusChange total_fwd_fees:(Maybe Grams) total_action_fees:(Maybe Grams) result_code:int32 result_arg:(Maybe int32) tot_actions:uint16 - spec_actions:uint16 skipped_actions:uint16 msgs_created:uint16 + spec_actions:uint16 skipped_actions:uint16 msgs_created:uint16 action_list_hash:bits256 tot_msg_size:StorageUsed = TrActionPhase; @@ -392,7 +392,7 @@ trans_merge_install$0111 split_info:SplitMergeInfo = TransactionDescr; smc_info#076ef1ea actions:uint16 msgs_sent:uint16 - unixtime:uint32 block_lt:uint64 trans_lt:uint64 + unixtime:uint32 block_lt:uint64 trans_lt:uint64 rand_seed:bits256 balance_remaining:CurrencyCollection myself:MsgAddressInt global_config:(Maybe Cell) = SmartContractInfo; // @@ -400,7 +400,7 @@ smc_info#076ef1ea actions:uint16 msgs_sent:uint16 out_list_empty$_ = OutList 0; out_list$_ {n:#} prev:^(OutList n) action:OutAction = OutList (n + 1); -action_send_msg#0ec3c86d mode:(## 8) +action_send_msg#0ec3c86d mode:(## 8) out_msg:^(MessageRelaxed Any) = OutAction; action_set_code#ad4de08e new_code:^Cell = OutAction; action_reserve_currency#36e6b809 mode:(## 8) @@ -413,11 +413,11 @@ action_change_library#26fa1dd4 mode:(## 7) out_list_node$_ prev:^Cell action:OutAction = OutListNode; // // -shard_ident$00 shard_pfx_bits:(#<= 60) +shard_ident$00 shard_pfx_bits:(#<= 60) workchain_id:int32 shard_prefix:uint64 = ShardIdent; ext_blk_ref$_ end_lt:uint64 - seq_no:uint32 root_hash:bits256 file_hash:bits256 + seq_no:uint32 root_hash:bits256 file_hash:bits256 = ExtBlkRef; block_id_ext$_ shard_id:ShardIdent seq_no:uint32 @@ -426,7 +426,7 @@ block_id_ext$_ shard_id:ShardIdent seq_no:uint32 master_info$_ master:ExtBlkRef = BlkMasterInfo; shard_state#9023afe2 global_id:int32 - shard_id:ShardIdent + shard_id:ShardIdent seq_no:uint32 vert_seq_no:# gen_utime:uint32 gen_lt:uint64 min_ref_mc_seqno:uint32 @@ -440,22 +440,22 @@ shard_state#9023afe2 global_id:int32 master_ref:(Maybe BlkMasterInfo) ] custom:(Maybe ^McStateExtra) = ShardStateUnsplit; - + _ ShardStateUnsplit = ShardState; split_state#5f327da5 left:^ShardStateUnsplit right:^ShardStateUnsplit = ShardState; shared_lib_descr$00 lib:^Cell publishers:(Hashmap 256 True) = LibDescr; -block_info#9bc7a987 version:uint32 - not_master:(## 1) - after_merge:(## 1) before_split:(## 1) - after_split:(## 1) +block_info#9bc7a987 version:uint32 + not_master:(## 1) + after_merge:(## 1) before_split:(## 1) + after_split:(## 1) want_split:Bool want_merge:Bool key_block:Bool vert_seqno_incr:(## 1) flags:(## 8) { flags <= 1 } - seq_no:# vert_seq_no:# { vert_seq_no >= vert_seqno_incr } - { prev_seq_no:# } { ~prev_seq_no + 1 = seq_no } + seq_no:# vert_seq_no:# { vert_seq_no >= vert_seqno_incr } + { prev_seq_no:# } { ~prev_seq_no + 1 = seq_no } shard:ShardIdent gen_utime:uint32 start_lt:uint64 end_lt:uint64 gen_validator_list_hash_short:uint32 @@ -463,7 +463,7 @@ block_info#9bc7a987 version:uint32 min_ref_mc_seqno:uint32 prev_key_block_seqno:uint32 gen_software:flags . 0?GlobalVersion - master_ref:not_master?^BlkMasterInfo + master_ref:not_master?^BlkMasterInfo prev_ref:^(BlkPrevInfo after_merge) prev_vert_ref:vert_seqno_incr?^(BlkPrevInfo 0) = BlockInfo; @@ -473,7 +473,7 @@ prev_blks_info$_ prev1:^ExtBlkRef prev2:^ExtBlkRef = BlkPrevInfo 1; block#11ef55aa global_id:int32 info:^BlockInfo value_flow:^ValueFlow - state_update:^(MERKLE_UPDATE ShardState) + state_update:^(MERKLE_UPDATE ShardState) extra:^BlockExtra = Block; block_extra in_msg_descr:^InMsgDescr @@ -483,7 +483,7 @@ block_extra in_msg_descr:^InMsgDescr created_by:bits256 custom:(Maybe ^McBlockExtra) = BlockExtra; // -value_flow#b8e48dfb ^[ from_prev_blk:CurrencyCollection +value_flow#b8e48dfb ^[ from_prev_blk:CurrencyCollection to_next_blk:CurrencyCollection imported:CurrencyCollection exported:CurrencyCollection ] @@ -511,7 +511,7 @@ value_flow_v2#3ebf98b7 ^[ from_prev_blk:CurrencyCollection // // bt_leaf$0 {X:Type} leaf:X = BinTree X; -bt_fork$1 {X:Type} left:^(BinTree X) right:^(BinTree X) +bt_fork$1 {X:Type} left:^(BinTree X) right:^(BinTree X) = BinTree X; fsm_none$0 = FutureSplitMerge; @@ -520,7 +520,7 @@ fsm_merge$11 merge_utime:uint32 interval:uint32 = FutureSplitMerge; shard_descr#b seq_no:uint32 reg_mc_seqno:uint32 start_lt:uint64 end_lt:uint64 - root_hash:bits256 file_hash:bits256 + root_hash:bits256 file_hash:bits256 before_split:Bool before_merge:Bool want_split:Bool want_merge:Bool nx_cc_updated:Bool flags:(## 3) { flags = 0 } @@ -532,7 +532,7 @@ shard_descr#b seq_no:uint32 reg_mc_seqno:uint32 shard_descr_new#a seq_no:uint32 reg_mc_seqno:uint32 start_lt:uint64 end_lt:uint64 - root_hash:bits256 file_hash:bits256 + root_hash:bits256 file_hash:bits256 before_split:Bool before_merge:Bool want_split:Bool want_merge:Bool nx_cc_updated:Bool flags:(## 3) { flags = 0 } @@ -545,23 +545,23 @@ shard_descr_new#a seq_no:uint32 reg_mc_seqno:uint32 _ (HashmapE 32 ^(BinTree ShardDescr)) = ShardHashes; bta_leaf$0 {X:Type} {Y:Type} extra:Y leaf:X = BinTreeAug X Y; -bta_fork$1 {X:Type} {Y:Type} left:^(BinTreeAug X Y) +bta_fork$1 {X:Type} {Y:Type} left:^(BinTreeAug X Y) right:^(BinTreeAug X Y) extra:Y = BinTreeAug X Y; _ fees:CurrencyCollection create:CurrencyCollection = ShardFeeCreated; _ (HashmapAugE 96 ShardFeeCreated ShardFeeCreated) = ShardFees; -_ config_addr:bits256 config:^(Hashmap 32 ^Cell) +_ config_addr:bits256 config:^(Hashmap 32 ^Cell) = ConfigParams; validator_info$_ - validator_list_hash_short:uint32 + validator_list_hash_short:uint32 catchain_seqno:uint32 nx_cc_updated:Bool = ValidatorInfo; validator_base_info$_ - validator_list_hash_short:uint32 + validator_list_hash_short:uint32 catchain_seqno:uint32 = ValidatorBaseInfo; @@ -571,8 +571,8 @@ _ key:Bool blk_ref:ExtBlkRef = KeyExtBlkRef; _ (HashmapAugE 32 KeyExtBlkRef KeyMaxLt) = OldMcBlocksInfo; -counters#_ last_updated:uint32 total:uint64 cnt2048:uint64 cnt65536:uint64 = Counters; -creator_info#4 mc_blocks:Counters shard_blocks:Counters = CreatorStats; +counters#_ last_updated:uint32 total:uint64 cnt2048:uint64 cnt65536:uint64 = Counters; +creator_info#4 mc_blocks:Counters shard_blocks:Counters = CreatorStats; block_create_stats#17 counters:(HashmapE 256 CreatorStats) = BlockCreateStats; block_create_stats_ext#34 counters:(HashmapAugE 256 CreatorStats uint32) = BlockCreateStats; @@ -618,11 +618,11 @@ masterchain_block_extra#cca5 validator#53 public_key:SigPubKey weight:uint64 = ValidatorDescr; validator_addr#73 public_key:SigPubKey weight:uint64 adnl_addr:bits256 = ValidatorDescr; -validators#11 utime_since:uint32 utime_until:uint32 - total:(## 16) main:(## 16) { main <= total } { main >= 1 } +validators#11 utime_since:uint32 utime_until:uint32 + total:(## 16) main:(## 16) { main <= total } { main >= 1 } list:(Hashmap 16 ValidatorDescr) = ValidatorSet; -validators_ext#12 utime_since:uint32 utime_until:uint32 - total:(## 16) main:(## 16) { main <= total } { main >= 1 } +validators_ext#12 utime_since:uint32 utime_until:uint32 + total:(## 16) main:(## 16) { main <= total } { main >= 1 } total_weight:uint64 list:(HashmapE 16 ValidatorDescr) = ValidatorSet; _ config_addr:bits256 = ConfigParam 0; @@ -648,15 +648,15 @@ cfg_vote_cfg#36 min_tot_rounds:uint8 max_tot_rounds:uint8 min_wins:uint8 max_los cfg_vote_setup#91 normal_params:^ConfigProposalSetup critical_params:^ConfigProposalSetup = ConfigVotingSetup; _ ConfigVotingSetup = ConfigParam 11; -cfg_proposal#f3 param_id:int32 param_value:(Maybe ^Cell) if_hash_equal:(Maybe uint256) +cfg_proposal#f3 param_id:int32 param_value:(Maybe ^Cell) if_hash_equal:(Maybe uint256) = ConfigProposal; cfg_proposal_status#ce expires:uint32 proposal:^ConfigProposal is_critical:Bool - voters:(HashmapE 16 True) remaining_weight:int64 validator_set_id:uint256 + voters:(HashmapE 16 True) remaining_weight:int64 validator_set_id:uint256 rounds_remaining:uint8 wins:uint8 losses:uint8 = ConfigProposalStatus; wfmt_basic#1 vm_version:int32 vm_mode:uint64 = WorkchainFormat 1; wfmt_ext#0 min_addr_len:(## 12) max_addr_len:(## 12) addr_len_step:(## 12) - { min_addr_len >= 64 } { min_addr_len <= max_addr_len } + { min_addr_len >= 64 } { min_addr_len <= max_addr_len } { max_addr_len <= 1023 } { addr_len_step <= 1023 } workchain_type_id:(## 32) { workchain_type_id >= 1 } = WorkchainFormat 0; @@ -687,37 +687,37 @@ workchain_v2#a7 enabled_since:uint32 monitor_min_split:(## 8) _ workchains:(HashmapE 32 WorkchainDescr) = ConfigParam 12; -complaint_prices#1a deposit:Grams bit_price:Grams cell_price:Grams = ComplaintPricing; +complaint_prices#1a deposit:Grams bit_price:Grams cell_price:Grams = ComplaintPricing; _ ComplaintPricing = ConfigParam 13; block_grams_created#6b masterchain_block_fee:Grams basechain_block_fee:Grams = BlockCreateFees; _ BlockCreateFees = ConfigParam 14; -_ validators_elected_for:uint32 elections_start_before:uint32 +_ validators_elected_for:uint32 elections_start_before:uint32 elections_end_before:uint32 stake_held_for:uint32 = ConfigParam 15; - -_ max_validators:(## 16) max_main_validators:(## 16) min_validators:(## 16) - { max_validators >= max_main_validators } - { max_main_validators >= min_validators } + +_ max_validators:(## 16) max_main_validators:(## 16) min_validators:(## 16) + { max_validators >= max_main_validators } + { max_main_validators >= min_validators } { min_validators >= 1 } = ConfigParam 16; _ min_stake:Grams max_stake:Grams min_total_stake:Grams max_stake_factor:uint32 = ConfigParam 17; -_#cc utime_since:uint32 bit_price_ps:uint64 cell_price_ps:uint64 +_#cc utime_since:uint32 bit_price_ps:uint64 cell_price_ps:uint64 mc_bit_price_ps:uint64 mc_cell_price_ps:uint64 = StoragePrices; _ (Hashmap 32 StoragePrices) = ConfigParam 18; _ global_id:int32 = ConfigParam 19; -gas_prices#dd gas_price:uint64 gas_limit:uint64 gas_credit:uint64 - block_gas_limit:uint64 freeze_due_limit:uint64 delete_due_limit:uint64 +gas_prices#dd gas_price:uint64 gas_limit:uint64 gas_credit:uint64 + block_gas_limit:uint64 freeze_due_limit:uint64 delete_due_limit:uint64 = GasLimitsPrices; -gas_prices_ext#de gas_price:uint64 gas_limit:uint64 special_gas_limit:uint64 gas_credit:uint64 - block_gas_limit:uint64 freeze_due_limit:uint64 delete_due_limit:uint64 +gas_prices_ext#de gas_price:uint64 gas_limit:uint64 special_gas_limit:uint64 gas_credit:uint64 + block_gas_limit:uint64 freeze_due_limit:uint64 delete_due_limit:uint64 = GasLimitsPrices; gas_flat_pfx#d1 flat_gas_limit:uint64 flat_gas_price:uint64 other:GasLimitsPrices @@ -749,7 +749,7 @@ config_mc_fwd_prices#_ MsgForwardPrices = ConfigParam 24; // used for all other messages config_fwd_prices#_ MsgForwardPrices = ConfigParam 25; -catchain_config#c1 mc_catchain_lifetime:uint32 shard_catchain_lifetime:uint32 +catchain_config#c1 mc_catchain_lifetime:uint32 shard_catchain_lifetime:uint32 shard_validators_lifetime:uint32 shard_validators_num:uint32 = CatchainConfig; catchain_config_new#c2 flags:(## 7) { flags = 0 } shuffle_mc_validators:Bool @@ -771,7 +771,7 @@ consensus_config_v3#d8 flags:(## 7) { flags = 0 } new_catchain_ids:Bool round_candidates:(## 8) { round_candidates >= 1 } next_candidate_delay_ms:uint32 consensus_timeout_ms:uint32 fast_attempts:uint32 attempt_duration:uint32 catchain_max_deps:uint32 - max_block_bytes:uint32 max_collated_bytes:uint32 + max_block_bytes:uint32 max_collated_bytes:uint32 proto_version:uint16 = ConsensusConfig; consensus_config_v4#d9 flags:(## 7) { flags = 0 } new_catchain_ids:Bool @@ -796,7 +796,7 @@ validator_temp_key#3 adnl_addr:bits256 temp_public_key:SigPubKey seqno:# valid_u signed_temp_key#4 key:^ValidatorTempKey signature:CryptoSignature = ValidatorSignedTempKey; _ (HashmapE 256 ValidatorSignedTempKey) = ConfigParam 39; -misbehaviour_punishment_config_v1#01 +misbehaviour_punishment_config_v1#01 default_flat_fine:Grams default_proportional_fine:uint32 severity_flat_mult:uint16 severity_proportional_mult:uint16 unpunishable_interval:uint16 @@ -851,7 +851,7 @@ block_proof#c3 proof_for:BlockIdExt root:^Cell signatures:(Maybe ^BlockSignature chain_empty$_ = ProofChain 0; chain_link$_ {n:#} root:^Cell prev:n?^(ProofChain n) = ProofChain (n + 1); -top_block_descr#d5 proof_for:BlockIdExt signatures:(Maybe ^BlockSignatures) +top_block_descr#d5 proof_for:BlockIdExt signatures:(Maybe ^BlockSignatures) len:(## 8) { len >= 1 } { len <= 8 } chain:(ProofChain len) = TopBlockDescr; // @@ -905,7 +905,7 @@ vmc_std$00 cdata:VmControlData code:VmCellSlice = VmCont; vmc_envelope$01 cdata:VmControlData next:^VmCont = VmCont; vmc_quit$1000 exit_code:int32 = VmCont; vmc_quit_exc$1001 = VmCont; -vmc_repeat$10100 count:uint63 body:^VmCont after:^VmCont = VmCont; +vmc_repeat$10100 count:uint63 body:^VmCont after:^VmCont = VmCont; vmc_until$110000 body:^VmCont after:^VmCont = VmCont; vmc_again$110001 body:^VmCont = VmCont; vmc_while_cond$110010 cond:^VmCont body:^VmCont @@ -949,7 +949,7 @@ dns_storage_address#7473 bag_id:bits256 = DNSRecord; // PAYMENT CHANNELS // -chan_config$_ init_timeout:uint32 close_timeout:uint32 a_key:bits256 b_key:bits256 +chan_config$_ init_timeout:uint32 close_timeout:uint32 a_key:bits256 b_key:bits256 a_addr:^MsgAddressInt b_addr:^MsgAddressInt channel_id:uint64 min_A_extra:Grams = ChanConfig; chan_state_init$000 signed_A:Bool signed_B:Bool min_A:Grams min_B:Grams expire_at:uint32 A:Grams B:Grams = ChanState; diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 8af5247a87..bd9a8f55a2 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -14,13 +14,13 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 03ff2dc475..6d33802971 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or diff --git a/crypto/block/output-queue-merger.cpp b/crypto/block/output-queue-merger.cpp index 43c59276fe..4864929905 100644 --- a/crypto/block/output-queue-merger.cpp +++ b/crypto/block/output-queue-merger.cpp @@ -21,9 +21,9 @@ namespace block { /* - * - * OUTPUT QUEUE MERGER - * + * + * OUTPUT QUEUE MERGER + * */ bool OutputQueueMerger::MsgKeyValue::operator<(const MsgKeyValue& other) const { diff --git a/crypto/block/test-block.cpp b/crypto/block/test-block.cpp index 70433ecec7..8a0c1ef8ed 100644 --- a/crypto/block/test-block.cpp +++ b/crypto/block/test-block.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2019 Telegram Systems LLP @@ -151,7 +151,7 @@ void test1() { std::cout << " cb = " << cb.finalize() << std::endl; } } - /* + /* { vm::CellBuilder cb; td::BitArray<256> hash; diff --git a/crypto/block/test-weight-distr.cpp b/crypto/block/test-weight-distr.cpp index 9c364e8451..6935747976 100644 --- a/crypto/block/test-weight-distr.cpp +++ b/crypto/block/test-weight-distr.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2020 Telegram Systems LLP diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 577435f102..d372b527f6 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -96,9 +96,9 @@ Ref ComputePhaseConfig::lookup_library(td::ConstBitPtr key) const { } /* - * + * * ACCOUNTS - * + * */ /** @@ -148,7 +148,7 @@ bool Account::check_addr_rewrite_length(int length) const { /** * Parses anycast data of the account address. - * + * * Initializes addr_rewrite. * * @param cs The cell slice containing partially-parsed account address. @@ -244,7 +244,7 @@ bool Account::unpack_address(vm::CellSlice& addr_cs) { /** * Unpacks storage information from a CellSlice. - * + * * Storage information is serialized using StorageInfo TLB-scheme. * * @param cs The CellSlice containing the storage information. @@ -434,7 +434,7 @@ bool Account::init_rewrite_addr(int rewrite_length, td::ConstBitPtr orig_addr_re /** * Unpacks the account information from the provided CellSlice. - * + * * Used to unpack previously existing accounts. * * @param shard_account The ShardAccount to unpack. @@ -2345,7 +2345,7 @@ int Transaction::try_action_change_library(vm::CellSlice& cs, ActionPhase& ap, c /** * Computes the forward fees for a message based on the number of cells and bits. - * + * * msg_fwd_fees = (lump_price + ceil((bit_price * msg.bits + cell_price * msg.cells)/2^16)) nanograms * ihr_fwd_fees = ceil((msg_fwd_fees * ihr_price_factor)/2^16) nanograms * bits in the root cell of a message are not included in msg.bits (lump_price pays for them) @@ -3403,9 +3403,9 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { } // namespace transaction /* - * + * * SERIALIZE PREPARED TRANSACTION - * + * */ /** @@ -3634,7 +3634,7 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { /** * Serializes the transaction object using Transaction TLB-scheme. - * + * * Updates root. * * @param cfg The configuration for the serialization. @@ -3951,9 +3951,9 @@ bool Transaction::update_limits(block::BlockLimitStatus& blimst, bool with_gas, } /* - * + * * COMMIT TRANSACTION - * + * */ /** diff --git a/crypto/fift/Continuation.h b/crypto/fift/Continuation.h index 3b2e1b6b2f..511bce68fb 100644 --- a/crypto/fift/Continuation.h +++ b/crypto/fift/Continuation.h @@ -28,9 +28,9 @@ using td::Ref; struct IntCtx; /* - * + * * FIFT CONTINUATIONS - * + * */ class FiftCont : public td::CntObject { diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index fddb0c88ea..0053ff8873 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -184,7 +184,7 @@ x{6E} @Defop ISNULL x{6F0} @Defop(4u) TUPLE x{6F00} @Defop NIL x{6F01} @Defop SINGLE -x{6F02} dup @Defop PAIR @Defop CONS +x{6F02} dup @Defop PAIR @Defop CONS x{6F03} @Defop TRIPLE x{6F1} @Defop(4u) INDEX x{6F10} dup @Defop FIRST @Defop CAR @@ -251,7 +251,7 @@ x{7F} @Defop TRUE { dup 16 fits { abort"integer too large" 8 + 2dup fits } until - > 2- 5 u, -rot i, + > 2- 5 u, -rot i, } cond } cond } cond @@ -918,24 +918,24 @@ recursive IFELSE-cont2 { { 1 { swap @normal? swap IFELSE-cont2 } does @doafter<{ } : @doifelse { 1 { swap @normal? IFELSE-cont2 } does @doafter<{ } : @doifnotelse { - { dup `else eq? + { dup `else eq? { drop @doifelse } { dup `else: eq? { drop IFJMP-cont } { @normal? IF-cont - } cond - } cond - } @doafter<{ + } cond + } cond + } @doafter<{ } : IF:<{ { - { dup `else eq? + { dup `else eq? { drop @doifnotelse } { dup `else: eq? { drop IFNOTJMP-cont } { @normal? IFNOT-cont - } cond - } cond - } @doafter<{ + } cond + } cond + } @doafter<{ } : IFNOT:<{ x{E304} @Defop CONDSEL @@ -963,12 +963,12 @@ x{EB} dup @Defop AGAINEND @Defop AGAIN: { }> PUSHCONT UNTIL } : }>UNTIL { { @normal? PUSHCONT UNTIL } @doafter<{ } : UNTIL:<{ { PUSHCONT { @normal? PUSHCONT WHILE } @doafter<{ } : @dowhile -{ - { dup `do eq? - { drop @dowhile } - { `do: eq? not abort"`}>DO<{` expected" PUSHCONT WHILEEND - } cond - } @doafter<{ +{ + { dup `do eq? + { drop @dowhile } + { `do: eq? not abort"`}>DO<{` expected" PUSHCONT WHILEEND + } cond + } @doafter<{ } : WHILE:<{ { }> PUSHCONT AGAIN } : }>AGAIN { { @normal? PUSHCONT AGAIN } @doafter<{ } : AGAIN:<{ @@ -988,11 +988,11 @@ x{E31B} dup @Defop AGAINENDBRK @Defop AGAINBRK: { { @normal? PUSHCONT UNTILBRK } @doafter<{ } : UNTILBRK:<{ { PUSHCONT { @normal? PUSHCONT WHILEBRK } @doafter<{ } : @dowhile { - { dup `do eq? - { drop @dowhile } + { dup `do eq? + { drop @dowhile } { `do: eq? not abort"`}>DO<{` expected" PUSHCONT WHILEENDBRK - } cond - } @doafter<{ + } cond + } @doafter<{ } : WHILEBRK:<{ { }> PUSHCONT AGAINBRK } : }>AGAINBRK { { @normal? PUSHCONT AGAINBRK } @doafter<{ } : AGAINBRK:<{ @@ -1018,8 +1018,8 @@ x{ED1F} @Defop BLESSVARARGS { c4 PUSHCTR } : PUSHROOT { c4 POPCTR } : POPROOT x{ED6} dup @Defop(c) SETCONTCTR @Defop(c) SETCONT -x{ED7} @Defop(c) SETRETCTR -x{ED8} @Defop(c) SETALTCTR +x{ED7} @Defop(c) SETRETCTR +x{ED8} @Defop(c) SETALTCTR x{ED9} dup @Defop(c) POPSAVE @Defop(c) POPCTRSAVE x{EDA} dup @Defop(c) SAVE @Defop(c) SAVECTR x{EDB} dup @Defop(c) SAVEALT @Defop(c) SAVEALTCTR @@ -1076,7 +1076,7 @@ x{EDFB} @Defop SAMEALTSAVE { dup sbits { @addop } { - dup srefs // + dup srefs // { ref@ CALLREF } { drop } cond @@ -1113,9 +1113,9 @@ x{F2FF} @Defop TRY x{F3} @Defop(4u,4u) TRYARGS { `catch @endblk } : }>CATCH<{ { PUSHCONT { @normal? PUSHCONT TRY } @doafter<{ } : @trycatch -{ +{ { `catch eq? not abort"`}>CATCH<{` expected" @trycatch - } @doafter<{ + } @doafter<{ } : TRY:<{ // // dictionary manipulation @@ -1265,7 +1265,7 @@ x{F4A1} @Defop DICTUGETJMP x{F4A2} @Defop DICTIGETEXEC x{F4A3} @Defop DICTUGETEXEC { dup sbitrefs tuck 1 > swap 1 <> or abort"not a dictionary" swap 1 u@ over <> abort"not a dictionary" } : @chkdicts -{ dup null? tuck { <{" cr +indent disasm -indent .indent ."}>" cr true - } swap ' idictforeach ' dictforeach cond drop + } swap ' idictforeach ' dictforeach cond drop -indent .indent ."}" cr } : show-const-dict-op diff --git a/crypto/fift/lib/Fift.fif b/crypto/fift/lib/Fift.fif index 0fde5cb945..a252a6d1f6 100644 --- a/crypto/fift/lib/Fift.fif +++ b/crypto/fift/lib/Fift.fif @@ -17,7 +17,7 @@ 10 constant ten { bl word 1 { find 0= abort"word not found" } } :: (') { bl word find not abort"-?" 0 swap } :: [compile] -{ bl word 1 { +{ bl word 1 { dup find { " -?" $+ abort } ifnot nip execute } } :: @' { bl word 1 { swap 1 'nop does swap 0 (create) } diff --git a/crypto/fift/lib/FiftExt.fif b/crypto/fift/lib/FiftExt.fif index 6ed677d7f2..b1d2dd67d6 100644 --- a/crypto/fift/lib/FiftExt.fif +++ b/crypto/fift/lib/FiftExt.fif @@ -1,7 +1,7 @@ -{ ?dup { 1+ { execute } { 0 swap } cond } +{ ?dup { 1+ { execute } { 0 swap } cond } { (number) ?dup 0= abort"-?" 'nop } cond } : (interpret-prepare) -{ { include-depth 0= (seekeof?) not } { +{ { include-depth 0= (seekeof?) not } { (word-prefix-find) (interpret-prepare) (execute) } while } : interpret @@ -10,7 +10,7 @@ (}) swap execute } : begin-block { swap 0 'nop } : end-block -{ { 1 'nop } `{ begin-block } +{ { 1 'nop } `{ begin-block } { { swap `{ eq? not abort"} without {" swap execute } end-block } :: } :: { @@ -18,7 +18,7 @@ { eq? not abort"unexpected" } : ?pairs { dup `if eq? swap `ifnot eq? over or not abort"without if{" } : if-ifnot? // cond then ? -- exec -{ { ' if } { ' ifnot } cond rot ({) 0 rot (compile) -rot 1 swap (compile) (}) +{ { ' if } { ' ifnot } cond rot ({) 0 rot (compile) -rot 1 swap (compile) (}) } : (make-if) // cond then else -- exec { rot ({) 0 rot (compile) -rot 2 ' cond (compile) (}) @@ -29,7 +29,7 @@ { { over `else eq? } { nip rot if-ifnot? ' swap ifnot (make-cond) } while - swap `noelse ?pairs 0 swap + swap `noelse ?pairs 0 swap } : finish-else-chain { swap dup if-ifnot? drop `then { swap `then ?pairs @@ -52,7 +52,7 @@ } end-block-does :: }do{ // repeat{ ... }until{ ... } -{ swap ({) 0 rot (compile) 0 rot (compile) (}) 1 ' until does } : (make-until) +{ swap ({) 0 rot (compile) 0 rot (compile) (}) 1 ' until does } : (make-until) { `repeat begin-block } :: repeat{ { swap `repeat eq? not abort"without repeat{" `until { swap `until ?pairs (make-until) 0 swap @@ -61,7 +61,7 @@ // def { ... } instead of { ... } : { bl word swap bl word "{" $cmp abort"{ expected" `def { - swap `def ?pairs -rot 3 ' (create) + swap `def ?pairs -rot 3 ' (create) } `{ begin-block } : (def) { 0 (def) } :: def @@ -77,10 +77,10 @@ def .sgn { if{ ?dup 0= }then{ ."zero" }elseif{ 0> }then{ - ."positive" + ."positive" }else{ - ."negative" - } + ."negative" + } cr } // equivalent to: { ?dup 0= { ."zero" } { 0> { ."positive" } { ."negative" } cond } cond cr } : .sgn diff --git a/crypto/fift/lib/GetOpt.fif b/crypto/fift/lib/GetOpt.fif index 442552b636..a683727eaa 100644 --- a/crypto/fift/lib/GetOpt.fif +++ b/crypto/fift/lib/GetOpt.fif @@ -20,7 +20,7 @@ recursive list-delete-range { } cond } cond } swap ! // ( n c -- ) deletes $n .. $(n+c-1) from the argument list $* -{ swap 1- $* @ swap rot list-delete-range $* ! } : $*del.. +{ swap 1- $* @ swap rot list-delete-range $* ! } : $*del.. // ( s s' -- ? ) checks whether s' is a prefix of s { tuck $len over $len over >= { $| drop $= } { 2drop drop false } cond } : $pfx? @@ -60,7 +60,7 @@ recursive list-delete-range { { dup null? { drop 0 } { car get-opt-flags -256 and } cond } : get-global-option-flags variable options-list -// ( l -- i or 0 ) +// ( l -- i or 0 ) // parses command line arguments according to option description list l // and returns index i of first incorrect option { dup options-list ! get-global-option-flags diff --git a/crypto/fift/lib/Lisp.fif b/crypto/fift/lib/Lisp.fif index fb91408d33..b044a97e03 100644 --- a/crypto/fift/lib/Lisp.fif +++ b/crypto/fift/lib/Lisp.fif @@ -38,7 +38,7 @@ recursive lisp-ctx-eval { recursive lisp-ctx-eval-list { over null? { drop } { swap uncons -rot over lisp-ctx-eval -rot lisp-ctx-eval-list cons - } cond + } cond } swap ! // (exprs) ctx -- val { null rot { @@ -205,7 +205,7 @@ forget 0L: forget 1L: forget 2L: { 3 exch 3 pick lisp-ctx-eval lisp-true? ' swap if nip swap lisp-ctx-eval } `if 3 lisp-fixed-lazy-primitive // (e) ctx -{ #t -rot +{ #t -rot { over null? { 2drop true } { swap uncons swap 2 pick lisp-ctx-eval dup lisp-true? // v' c t v ? { swap 2swap nip false } { -rot 2drop nip true } cond @@ -228,14 +228,14 @@ forget 0L: forget 1L: forget 2L: } : eval-cond-clause // (clauses) ctx -- v { { over null? { no-answer true } { - swap uncons -rot over eval-cond-clause } cond + swap uncons -rot over eval-cond-clause } cond } until -rot 2drop } `cond lisp-lazy-primitive { lisp-dict @ lookup-in { hole tuck lisp-dict-int-define } ifnot } : lisp-create-global-var // a e ctx -- old (simple) define { drop over atom? not abort"only a variable can be define'd" - over lisp-create-global-var swap lisp-eval swap ! + over lisp-create-global-var swap lisp-eval swap ! } drop // `define 2 lisp-fixed-lazy-primitive { tuck lisp-ctx-eval rot dup atom? not abort"only a variable can be set" rot lookup-or-fail dup @ -rot ! @@ -255,7 +255,7 @@ forget 0L: forget 1L: forget 2L: } : is-quote? recursive match-arg-list-acc // l (vars) (args) -- ((var . arg) ...)+l -1 or ? 0 -{ over atom? { over `_ eq? { 2drop } { pair swap cons } cond true } { +{ over atom? { over `_ eq? { 2drop } { pair swap cons } cond true } { over null? { nip null? } { // (vars) (args) over tuple? not { 2drop false } { over is-quote? { eq? nip } { // (v) (a) @@ -279,7 +279,7 @@ recursive match-arg-list-acc nip } : extend-ctx-by-list // ((vars) body) ctx -{ swap uncons -rot +{ swap uncons -rot dup lambda-var-list? not abort"invalid formal parameter list" { // l-expr ctx' [_ body ctx (vars)] -rot 2 pick 3 [] swap rot // [_ body ...] (vars) ctx' l-expr @@ -291,7 +291,7 @@ recursive match-arg-list-acc } : make-lambda { make-lambda } `lambda lisp-lazy-primitive // (a e) ctx -- more sophisticated (define a e) -{ drop uncons swap dup atom? { // (e) a +{ drop uncons swap dup atom? { // (e) a tuck lisp-create-global-var swap lisp-dict @ lisp-ctx-eval-list-last swap ! } { // (e) (a v..) @@ -356,8 +356,8 @@ lisp-dict @ constant original-lisp-dict { original-lisp-dict lisp-dict ! } : reset-lisp { ' drop { lisp-eval .l cr } List-generic( } :_ LISP-EVAL-PRINT( // LISP-EVAL-PRINT((+ 3 4) (* 5 6)) computes and prints 12 and 30 -{ hole dup 1 { @ nip } does swap - 1 { swap lisp-eval swap ! } does +{ hole dup 1 { @ nip } does swap + 1 { swap lisp-eval swap ! } does List-generic( } :_ LISP-EVAL( // LISP-EVAL((+ 3 4) (* 5 6)) computes 12 and 30, returns only 30 diff --git a/crypto/fift/lib/Lists.fif b/crypto/fift/lib/Lists.fif index b59e40a0d9..0737269d8c 100644 --- a/crypto/fift/lib/Lists.fif +++ b/crypto/fift/lib/Lists.fif @@ -152,7 +152,7 @@ forget curloop forget loopdump // variable ') 'nop box constant ', -{ ") without (" abort } ') ! +{ ") without (" abort } ') ! { ') @ execute } : ) anon constant dot-marker // m x1 ... xn t m -- (x1 ... xn . t) diff --git a/crypto/fift/lib/Stack.fif b/crypto/fift/lib/Stack.fif index 6efc0a060f..dae2c61be1 100644 --- a/crypto/fift/lib/Stack.fif +++ b/crypto/fift/lib/Stack.fif @@ -114,7 +114,7 @@ variable emul-stk // new s -- ops registered { { over null? not } { - // .sl .e get-op-list .l cr + // .sl .e get-op-list .l cr // get-op-list list-length 100 > abort"too long" emul-depth over > { over emul-tos swap list-member-eqv? not } { false } cond { @@ -139,14 +139,14 @@ variable emul-stk } : generate-reorder-ops // old new -- op-list -{ emul-stk @ op-rlist @ 2swap +{ emul-stk @ op-rlist @ 2swap swap list>tuple emul-stk ! clear-op-list 0 generate-reorder-ops get-op-list -rot op-rlist ! emul-stk ! } : generate-reorder { parse-stk-list( generate-reorder } :_ SG( -// op-list rewriting according to a ruleset +// op-list rewriting according to a ruleset // l f l1 l2 -- l' -1 or l f with l' = l2 + (l - l1) { push(3) rot list- { list+ nip nip true } { drop } cond } : try-rule @@ -186,7 +186,7 @@ LIST( // helpers for creating Fift source strings for one fift-op // i j -- s { minmax over { "xchg(" rot (.) $+ +"," swap (.) $+ +")" } - { nip dup 1 = { drop "swap" } { + { nip dup 1 = { drop "swap" } { ?dup { "xchg0(" swap (.) $+ +")" } { "" } cond } cond } cond } : source- @@ -202,7 +202,7 @@ LIST( } : source- // lit -- s { dup string? { char " chr swap $+ char " hold } { (.) } cond -} : source- +} : source- // dictionary with all fift op compilation/source creation { 0 swap (compile) } : fop-compile @@ -228,7 +228,7 @@ variable 'fop-entry-exec { dup first dup fift-op-dict @ assq { report-unknown-op } ifnot dup second 1+ push(3) count <> abort"incorrect param count" nip swap explode dup roll drop 1- roll // o2 .. on entry - 'fop-entry-exec @ execute + 'fop-entry-exec @ execute } : process-fift-op // compile op-list into Fift wordlist @@ -244,7 +244,7 @@ variable 'fop-entry-exec // transform op-list into Fift source // ls op -- ls' -{ fift-ops-rewrite +{ fift-ops-rewrite { 3 [] execute } 'fop-entry-exec ! null ' process-fift-op foldl dup null? { drop "" } { { +" " swap $+ } foldr-ne } cond diff --git a/crypto/fift/lib/TonUtil.fif b/crypto/fift/lib/TonUtil.fif index f31b591eb3..721b06147e 100644 --- a/crypto/fift/lib/TonUtil.fif +++ b/crypto/fift/lib/TonUtil.fif @@ -16,22 +16,22 @@ library TonUtil // TON Blockchain Fift Library // Private key load/generate // ( fname -- pubkey privkey ) -{ dup ."Loading private key from file " type cr +{ dup ."Loading private key from file " type cr file>B dup Blen 32 <> abort"Private key must be exactly 32 bytes long" - dup priv>pub swap + dup priv>pub swap } : load-keypair // ( fname -- pubkey privkey ) { dup file-exists? { load-keypair } { dup newkeypair swap rot over swap B>file - rot ."Saved new private key to file " type cr + rot ."Saved new private key to file " type cr } cond } : load-generate-keypair // Parse smart-contract address // ( S -- workchain addr bounce? ) { $>smca not abort"invalid smart-contract address" - 1 and 0= + 1 and 0= } : parse-smc-addr // ( x -- ) Displays a 64-digit hex number @@ -48,7 +48,7 @@ library TonUtil // TON Blockchain Fift Library } : save-address-verbose // ( fname -- wc addr ) Load address from file -{ file>B 32 B| +{ file>B 32 B| dup Blen { 32 B>i@ } { drop Basechain } cond swap 256 B>u@ } : load-address @@ -69,7 +69,7 @@ library TonUtil // TON Blockchain Fift Library } : parse-adnl-address // ( b wc addr -- b' ) Serializes address into Builder b -{ -rot 8 i, swap 256 u, } : addr, +{ -rot 8 i, swap 256 u, } : addr, { over 8 fits { rot b{100} s, -rot addr, } { rot b{110} s, 256 9 u, rot 32 i, swap 256 u, } cond } : Addr, diff --git a/crypto/func/abscode.cpp b/crypto/func/abscode.cpp index f1ffcfa493..3ee3903e64 100644 --- a/crypto/func/abscode.cpp +++ b/crypto/func/abscode.cpp @@ -21,9 +21,9 @@ namespace funC { /* - * + * * ABSTRACT CODE - * + * */ TmpVar::TmpVar(var_idx_t _idx, int _cls, TypeExpr* _type, SymDef* sym, const SrcLocation* loc) diff --git a/crypto/func/analyzer.cpp b/crypto/func/analyzer.cpp index 8a30458185..2e3bf5f8ac 100644 --- a/crypto/func/analyzer.cpp +++ b/crypto/func/analyzer.cpp @@ -23,9 +23,9 @@ namespace funC { /* - * + * * ANALYZE AND PREPROCESS ABSTRACT CODE - * + * */ void CodeBlob::simplify_var_types() { diff --git a/crypto/func/asmops.cpp b/crypto/func/asmops.cpp index 93fd58f8c0..b794c4b0ac 100644 --- a/crypto/func/asmops.cpp +++ b/crypto/func/asmops.cpp @@ -25,9 +25,9 @@ namespace funC { /* - * + * * ASM-OP LIST FUNCTIONS - * + * */ int is_pos_pow2(td::RefInt256 x) { diff --git a/crypto/func/auto-tests/legacy_tests/bsc-bridge-collector/votes-collector.fc b/crypto/func/auto-tests/legacy_tests/bsc-bridge-collector/votes-collector.fc index cec4ed211b..49377eecbc 100644 --- a/crypto/func/auto-tests/legacy_tests/bsc-bridge-collector/votes-collector.fc +++ b/crypto/func/auto-tests/legacy_tests/bsc-bridge-collector/votes-collector.fc @@ -63,7 +63,7 @@ cell load_data() inline_ref { } save_data(external_votings); - return send_receipt_message(s_addr, 0x10000 + 6, query_id, 0, 0, 64); ;; thanks + return send_receipt_message(s_addr, 0x10000 + 6, query_id, 0, 0, 64); ;; thanks } () recv_internal(int msg_value, cell in_msg_cell, slice in_msg) impure { @@ -83,7 +83,7 @@ cell load_data() inline_ref { return (); } int query_id = in_msg~load_uint(64); - + if (op == 5) { ;; submit signatures int voting_id = in_msg~load_uint(256); slice signature = in_msg~load_bits(520); diff --git a/crypto/func/auto-tests/legacy_tests/config/config-code.fc b/crypto/func/auto-tests/legacy_tests/config/config-code.fc index 85a4ee00cc..71841a5c90 100644 --- a/crypto/func/auto-tests/legacy_tests/config/config-code.fc +++ b/crypto/func/auto-tests/legacy_tests/config/config-code.fc @@ -132,8 +132,8 @@ _ perform_action(cfg_dict, public_key, action, cs) inline_ref { (cell, int, cell) get_current_vset() inline_ref { var vset = config_param(34); var cs = begin_parse(vset); - ;; validators_ext#12 utime_since:uint32 utime_until:uint32 - ;; total:(## 16) main:(## 16) { main <= total } { main >= 1 } + ;; validators_ext#12 utime_since:uint32 utime_until:uint32 + ;; total:(## 16) main:(## 16) { main <= total } { main >= 1 } ;; total_weight:uint64 throw_unless(40, cs~load_uint(8) == 0x12); cs~skip_bits(32 + 32 + 16 + 16); diff --git a/crypto/func/auto-tests/legacy_tests/elector/elector-code.fc b/crypto/func/auto-tests/legacy_tests/elector/elector-code.fc index a065244235..a1b12cc760 100644 --- a/crypto/func/auto-tests/legacy_tests/elector/elector-code.fc +++ b/crypto/func/auto-tests/legacy_tests/elector/elector-code.fc @@ -101,7 +101,7 @@ builder pack_complaint(int validator_pubkey, cell description, int created_at, i .store_uint(suggested_fine_part, 32); } -;; complaint_prices#1a deposit:Grams bit_price:Grams cell_price:Grams = ComplaintPricing; +;; complaint_prices#1a deposit:Grams bit_price:Grams cell_price:Grams = ComplaintPricing; (int, int, int) parse_complaint_prices(cell info) inline { var cs = info.begin_parse(); throw_unless(9, cs~load_uint(8) == 0x1a); @@ -127,8 +127,8 @@ builder pack_complaint(int validator_pubkey, cell description, int created_at, i (cell, int, cell) get_current_vset() inline_ref { var vset = config_param(34); var cs = begin_parse(vset); - ;; validators_ext#12 utime_since:uint32 utime_until:uint32 - ;; total:(## 16) main:(## 16) { main <= total } { main >= 1 } + ;; validators_ext#12 utime_since:uint32 utime_until:uint32 + ;; total:(## 16) main:(## 16) { main <= total } { main >= 1 } ;; total_weight:uint64 throw_unless(40, cs~load_uint(8) == 0x12); cs~skip_bits(32 + 32 + 16 + 16); @@ -177,7 +177,7 @@ builder pack_complaint(int validator_pubkey, cell description, int created_at, i () send_validator_set_to_config(config_addr, vset, query_id) impure inline_ref { var msg = begin_cell() - .store_uint(0xc4ff, 17) ;; 0 11000100 0xff + .store_uint(0xc4ff, 17) ;; 0 11000100 0xff .store_uint(config_addr, 256) .store_grams(1 << 30) ;; ~1 gram of value to process and obtain answer .store_uint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) @@ -639,7 +639,7 @@ int proceed_register_vote(election_id, chash, idx, weight) impure inline_ref { return config_set_confirmed(s_addr, in_msg, query_id, cfg_ok); } if (op == 0x52674370) { - ;; new complaint + ;; new complaint var price = register_complaint(s_addr, in_msg, msg_value); int mode = 64; int ans_tag = - price; @@ -798,7 +798,7 @@ _ compute_total_stake(l, n, m_stake) inline_ref { i += 1; } until (l.null?()); throw_unless(49, tot_stake == best_stake); - return (credits, vset, tot_weight, frozen, tot_stake, m); + return (credits, vset, tot_weight, frozen, tot_stake, m); } int conduct_elections(ds, elect, credits) impure { @@ -848,7 +848,7 @@ int conduct_elections(ds, elect, credits) impure { .store_uint(0x12, 8) ;; validators_ext#12 .store_uint(start, 32) ;; utime_since:uint32 .store_uint(start + elect_for, 32) ;; utime_until:uint32 - .store_uint(cnt, 16) ;; total:(## 16) + .store_uint(cnt, 16) ;; total:(## 16) .store_uint(min(cnt, main_validators), 16) ;; main:(## 16) .store_uint(total_weight, 64) ;; total_weight:uint64 .store_dict(vdict) ;; list:(HashmapE 16 ValidatorDescr) diff --git a/crypto/func/auto-tests/legacy_tests/jetton-minter/imports/utils.fc b/crypto/func/auto-tests/legacy_tests/jetton-minter/imports/utils.fc index 230a8f7988..c066b9b6bd 100644 --- a/crypto/func/auto-tests/legacy_tests/jetton-minter/imports/utils.fc +++ b/crypto/func/auto-tests/legacy_tests/jetton-minter/imports/utils.fc @@ -4,6 +4,6 @@ .store_slice(address) ;; 267 bit address .store_grams(amount) .store_uint(0, 107) ;; 106 zeroes + 0 as an indicator that there is no cell with the data - .end_cell(); + .end_cell(); send_raw_message(msg, 3); ;; mode, 2 for ignoring errors, 1 for sender pays fees, 64 for returning inbound message value } diff --git a/crypto/func/auto-tests/legacy_tests/jetton-minter/jetton-minter.fc b/crypto/func/auto-tests/legacy_tests/jetton-minter/jetton-minter.fc index 73776e6469..2eb16857ee 100644 --- a/crypto/func/auto-tests/legacy_tests/jetton-minter/jetton-minter.fc +++ b/crypto/func/auto-tests/legacy_tests/jetton-minter/jetton-minter.fc @@ -55,7 +55,7 @@ return (); } slice sender_address = cs~load_msg_addr(); - + int op = in_msg_body~load_uint(32); int query_id = in_msg_body~load_uint(64); diff --git a/crypto/func/auto-tests/legacy_tests/jetton-wallet/imports/utils.fc b/crypto/func/auto-tests/legacy_tests/jetton-wallet/imports/utils.fc index 230a8f7988..c066b9b6bd 100644 --- a/crypto/func/auto-tests/legacy_tests/jetton-wallet/imports/utils.fc +++ b/crypto/func/auto-tests/legacy_tests/jetton-wallet/imports/utils.fc @@ -4,6 +4,6 @@ .store_slice(address) ;; 267 bit address .store_grams(amount) .store_uint(0, 107) ;; 106 zeroes + 0 as an indicator that there is no cell with the data - .end_cell(); + .end_cell(); send_raw_message(msg, 3); ;; mode, 2 for ignoring errors, 1 for sender pays fees, 64 for returning inbound message value } diff --git a/crypto/func/auto-tests/legacy_tests/jetton-wallet/jetton-wallet.fc b/crypto/func/auto-tests/legacy_tests/jetton-wallet/jetton-wallet.fc index 01cc91bce0..7576e8bcd5 100644 --- a/crypto/func/auto-tests/legacy_tests/jetton-wallet/jetton-wallet.fc +++ b/crypto/func/auto-tests/legacy_tests/jetton-wallet/jetton-wallet.fc @@ -45,7 +45,7 @@ const gas_consumption = 10000000; ;; 0.01 TON internal_transfer query_id:uint64 amount:(VarUInteger 16) from:MsgAddress response_address:MsgAddress forward_ton_amount:(VarUInteger 16) - forward_payload:(Either Cell ^Cell) + forward_payload:(Either Cell ^Cell) = InternalMsgBody; -} @@ -103,7 +103,7 @@ const gas_consumption = 10000000; ;; 0.01 TON internal_transfer query_id:uint64 amount:(VarUInteger 16) from:MsgAddress response_address:MsgAddress forward_ton_amount:(VarUInteger 16) - forward_payload:(Either Cell ^Cell) + forward_payload:(Either Cell ^Cell) = InternalMsgBody; -} @@ -223,7 +223,7 @@ const gas_consumption = 10000000; ;; 0.01 TON cs~load_coins(); ;; skip value cs~skip_bits(1); ;; skip extracurrency collection cs~load_coins(); ;; skip ihr_fee - int fwd_fee = muldiv(cs~load_coins(), 3, 2); ;; we use message fwd_fee for estimation of forward_payload costs + int fwd_fee = muldiv(cs~load_coins(), 3, 2); ;; we use message fwd_fee for estimation of forward_payload costs int op = in_msg_body~load_uint(32); diff --git a/crypto/func/auto-tests/legacy_tests/tact-examples/jetton_JettonDefaultWallet.code.fc b/crypto/func/auto-tests/legacy_tests/tact-examples/jetton_JettonDefaultWallet.code.fc index 522fa9e11c..9f25e22577 100644 --- a/crypto/func/auto-tests/legacy_tests/tact-examples/jetton_JettonDefaultWallet.code.fc +++ b/crypto/func/auto-tests/legacy_tests/tact-examples/jetton_JettonDefaultWallet.code.fc @@ -197,7 +197,7 @@ _ __gen_StateInit_get_code((cell, cell) v) inline { } (int, slice, slice, cell) __gen_JettonWalletData_to_external(((int, slice, slice, cell)) v) { - var (v'balance, v'owner, v'master, v'walletCode) = v; + var (v'balance, v'owner, v'master, v'walletCode) = v; return (v'balance, v'owner, v'master, v'walletCode); } @@ -287,7 +287,7 @@ cell $__gen_JettonDefaultWallet_init(cell sys', slice $master, slice $owner) { slice sc' = sys'.begin_parse(); cell source = sc'~load_dict(); cell contracts = new_dict(); - + ;; Contract Code: JettonDefaultWallet cell mine = __tact_dict_get_code(source, 55471); contracts = __tact_dict_set_code(contracts, 55471, mine); @@ -376,7 +376,7 @@ _ $__gen_get_get_wallet_data() method_id(97026) { () recv_internal(int msg_value, cell in_msg_cell, slice in_msg) impure { - + ;; Parse incoming message int op = 0; if (slice_bits(in_msg) >= 32) { @@ -387,7 +387,7 @@ _ $__gen_get_get_wallet_data() method_id(97026) { var msg_bounced = ((msg_flags & 1) == 1 ? true : false); slice msg_sender_addr = cs~load_msg_addr(); __tact_context = (msg_bounced, msg_sender_addr, msg_value, cs); - + ;; Handle bounced messages if (msg_bounced) { var self = __gen_load_JettonDefaultWallet(); @@ -395,7 +395,7 @@ _ $__gen_get_get_wallet_data() method_id(97026) { __gen_store_JettonDefaultWallet(self); return (); } - + ;; Receive TokenTransfer message if (op == 260734629) { var self = __gen_load_JettonDefaultWallet(); @@ -404,7 +404,7 @@ _ $__gen_get_get_wallet_data() method_id(97026) { __gen_store_JettonDefaultWallet(self); return (); } - + ;; Receive TokenTransferInternal message if (op == 395134233) { var self = __gen_load_JettonDefaultWallet(); @@ -413,7 +413,7 @@ _ $__gen_get_get_wallet_data() method_id(97026) { __gen_store_JettonDefaultWallet(self); return (); } - + ;; Receive TokenBurn message if (op == 1499400124) { var self = __gen_load_JettonDefaultWallet(); @@ -422,7 +422,7 @@ _ $__gen_get_get_wallet_data() method_id(97026) { __gen_store_JettonDefaultWallet(self); return (); } - + throw(130); } diff --git a/crypto/func/auto-tests/legacy_tests/tact-examples/jetton_SampleJetton.code.fc b/crypto/func/auto-tests/legacy_tests/tact-examples/jetton_SampleJetton.code.fc index 30bff4e1fa..31d4732e97 100644 --- a/crypto/func/auto-tests/legacy_tests/tact-examples/jetton_SampleJetton.code.fc +++ b/crypto/func/auto-tests/legacy_tests/tact-examples/jetton_SampleJetton.code.fc @@ -165,7 +165,7 @@ _ __gen_Context_get_sender((int, slice, int, slice) v) inline { } (int, int, slice, cell, cell) __gen_JettonData_to_external(((int, int, slice, cell, cell)) v) { - var (v'totalSupply, v'mintable, v'owner, v'content, v'walletCode) = v; + var (v'totalSupply, v'mintable, v'owner, v'content, v'walletCode) = v; return (v'totalSupply, v'mintable, v'owner, v'content, v'walletCode); } @@ -258,7 +258,7 @@ cell $__gen_JettonDefaultWallet_init(cell sys', slice $master, slice $owner) { slice sc' = sys'.begin_parse(); cell source = sc'~load_dict(); cell contracts = new_dict(); - + ;; Contract Code: JettonDefaultWallet cell mine = __tact_dict_get_code(source, 55471); contracts = __tact_dict_set_code(contracts, 55471, mine); @@ -366,7 +366,7 @@ _ $__gen_get_owner() method_id(83229) { () recv_internal(int msg_value, cell in_msg_cell, slice in_msg) impure { - + ;; Parse incoming message int op = 0; if (slice_bits(in_msg) >= 32) { @@ -377,12 +377,12 @@ _ $__gen_get_owner() method_id(83229) { var msg_bounced = ((msg_flags & 1) == 1 ? true : false); slice msg_sender_addr = cs~load_msg_addr(); __tact_context = (msg_bounced, msg_sender_addr, msg_value, cs); - + ;; Handle bounced messages if (msg_bounced) { return (); } - + ;; Receive Mint message if (op == 33240155) { var self = __gen_load_SampleJetton(); @@ -391,7 +391,7 @@ _ $__gen_get_owner() method_id(83229) { __gen_store_SampleJetton(self); return (); } - + ;; Receive TokenUpdateContent message if (op == 201882270) { var self = __gen_load_SampleJetton(); @@ -400,7 +400,7 @@ _ $__gen_get_owner() method_id(83229) { __gen_store_SampleJetton(self); return (); } - + ;; Receive TokenBurnNotification message if (op == 2078119902) { var self = __gen_load_SampleJetton(); @@ -409,11 +409,11 @@ _ $__gen_get_owner() method_id(83229) { __gen_store_SampleJetton(self); return (); } - + ;; Text Receivers if (op == 0) { var text_op = slice_hash(in_msg); - + ;; Receive "Mint!" message if (text_op == 0xcd0d986cb1a2f468ae7089f4fc3162c116e5f53fbd11a6839f52dbf5040830b2) { var self = __gen_load_SampleJetton(); @@ -422,7 +422,7 @@ _ $__gen_get_owner() method_id(83229) { return (); } } - + throw(130); } diff --git a/crypto/func/auto-tests/legacy_tests/tact-examples/maps_MapTestContract.code.fc b/crypto/func/auto-tests/legacy_tests/tact-examples/maps_MapTestContract.code.fc index 833e9d1aa0..c9c9836283 100644 --- a/crypto/func/auto-tests/legacy_tests/tact-examples/maps_MapTestContract.code.fc +++ b/crypto/func/auto-tests/legacy_tests/tact-examples/maps_MapTestContract.code.fc @@ -231,8 +231,8 @@ tuple __gen_SomeStruct_to_tuple(((int)) v) { } tuple __gen_SomeStruct_opt_to_tuple(tuple v) inline { - if (null?(v)) { return null(); } - return __gen_SomeStruct_to_tuple(__gen_SomeStruct_not_null(v)); + if (null?(v)) { return null(); } + return __gen_SomeStruct_to_tuple(__gen_SomeStruct_not_null(v)); } tuple __gen_SomeStruct_opt_to_external(tuple v) inline { @@ -499,7 +499,7 @@ _ $__gen_get_addrMap4Value(slice $$key) method_id(100021) { () recv_internal(int msg_value, cell in_msg_cell, slice in_msg) impure { - + ;; Parse incoming message int op = 0; if (slice_bits(in_msg) >= 32) { @@ -510,12 +510,12 @@ _ $__gen_get_addrMap4Value(slice $$key) method_id(100021) { var msg_bounced = ((msg_flags & 1) == 1 ? true : false); slice msg_sender_addr = cs~load_msg_addr(); __tact_context = (msg_bounced, msg_sender_addr, msg_value, cs); - + ;; Handle bounced messages if (msg_bounced) { return (); } - + ;; Receive SetIntMap1 message if (op == 1510253336) { var self = __gen_load_MapTestContract(); @@ -524,7 +524,7 @@ _ $__gen_get_addrMap4Value(slice $$key) method_id(100021) { __gen_store_MapTestContract(self); return (); } - + ;; Receive SetIntMap2 message if (op == 1629867766) { var self = __gen_load_MapTestContract(); @@ -533,7 +533,7 @@ _ $__gen_get_addrMap4Value(slice $$key) method_id(100021) { __gen_store_MapTestContract(self); return (); } - + ;; Receive SetIntMap3 message if (op == 3613954633) { var self = __gen_load_MapTestContract(); @@ -542,7 +542,7 @@ _ $__gen_get_addrMap4Value(slice $$key) method_id(100021) { __gen_store_MapTestContract(self); return (); } - + ;; Receive SetIntMap4 message if (op == 383013829) { var self = __gen_load_MapTestContract(); @@ -551,7 +551,7 @@ _ $__gen_get_addrMap4Value(slice $$key) method_id(100021) { __gen_store_MapTestContract(self); return (); } - + ;; Receive SetAddrMap1 message if (op == 1749966413) { var self = __gen_load_MapTestContract(); @@ -560,7 +560,7 @@ _ $__gen_get_addrMap4Value(slice $$key) method_id(100021) { __gen_store_MapTestContract(self); return (); } - + ;; Receive SetAddrMap2 message if (op == 624157584) { var self = __gen_load_MapTestContract(); @@ -569,7 +569,7 @@ _ $__gen_get_addrMap4Value(slice $$key) method_id(100021) { __gen_store_MapTestContract(self); return (); } - + ;; Receive SetAddrMap3 message if (op == 4276365062) { var self = __gen_load_MapTestContract(); @@ -578,7 +578,7 @@ _ $__gen_get_addrMap4Value(slice $$key) method_id(100021) { __gen_store_MapTestContract(self); return (); } - + ;; Receive SetAddrMap4 message if (op == 1683777913) { var self = __gen_load_MapTestContract(); @@ -587,7 +587,7 @@ _ $__gen_get_addrMap4Value(slice $$key) method_id(100021) { __gen_store_MapTestContract(self); return (); } - + throw(130); } diff --git a/crypto/func/auto-tests/legacy_tests/tact-examples/treasure_Treasure.code.fc b/crypto/func/auto-tests/legacy_tests/tact-examples/treasure_Treasure.code.fc index e4fc974fb1..eab9de18a9 100644 --- a/crypto/func/auto-tests/legacy_tests/tact-examples/treasure_Treasure.code.fc +++ b/crypto/func/auto-tests/legacy_tests/tact-examples/treasure_Treasure.code.fc @@ -164,7 +164,7 @@ _ $__gen_get_owner() method_id(83229) { () recv_internal(int msg_value, cell in_msg_cell, slice in_msg) impure { - + ;; Parse incoming message int op = 0; if (slice_bits(in_msg) >= 32) { @@ -175,12 +175,12 @@ _ $__gen_get_owner() method_id(83229) { var msg_bounced = ((msg_flags & 1) == 1 ? true : false); slice msg_sender_addr = cs~load_msg_addr(); __tact_context = (msg_bounced, msg_sender_addr, msg_value, cs); - + ;; Handle bounced messages if (msg_bounced) { return (); } - + ;; Receive Withdraw message if (op == 1672521544) { var self = __gen_load_Treasure(); @@ -189,7 +189,7 @@ _ $__gen_get_owner() method_id(83229) { __gen_store_Treasure(self); return (); } - + ;; Receive ChangeOwner message if (op == 256331011) { var self = __gen_load_Treasure(); @@ -198,11 +198,11 @@ _ $__gen_get_owner() method_id(83229) { __gen_store_Treasure(self); return (); } - + ;; Text Receivers if (op == 0) { var text_op = slice_hash(in_msg); - + ;; Receive "Destroy" message if (text_op == 0x986c2ba124bb9287eb4a0bd8d3104e1c0067a3c93952d889c74d08185bd30d4d) { var self = __gen_load_Treasure(); @@ -211,7 +211,7 @@ _ $__gen_get_owner() method_id(83229) { return (); } } - + throw(130); } diff --git a/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/get.fc b/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/get.fc index c9152e1cc9..db39a0cc2f 100644 --- a/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/get.fc +++ b/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/get.fc @@ -72,7 +72,7 @@ _ get_params() method_id { _ get_member_balance(slice address) method_id { load_base_data(); load_member(parse_work_addr(address)); - + member_update_balance(); return (ctx_member_balance, ctx_member_pending_deposit, ctx_member_pending_withdraw, ctx_member_withdraw); } diff --git a/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/model.fc b/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/model.fc index 9c28b42a1a..3e176e3ea7 100644 --- a/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/model.fc +++ b/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/model.fc @@ -5,7 +5,7 @@ () add_member_pending_withdraw(int delta) impure inline { ctx_balance_pending_withdraw = ctx_balance_pending_withdraw + delta; ctx_member_pending_withdraw = ctx_member_pending_withdraw + delta; -} +} () set_member_pending_withdraw(int value) impure inline { add_member_pending_withdraw(value - ctx_member_pending_withdraw); } @@ -32,7 +32,7 @@ int apply_profit(int value, int value_profit, int profit) { ;; () member_update_balance() impure { - + ;; Update profit (for non-owner) if (ctx_member != owner_id()) { if (ctx_profit_per_coin != ctx_member_profit_per_coin) { @@ -127,7 +127,7 @@ int apply_profit(int value, int value_profit, int profit) { ;; Trying to withdraw immediatelly var remaining = value; var withdrawed = 0; - + ;; Try to withdraw from pending deposit if ((remaining > 0) & (ctx_member_pending_deposit >= 0)) { int delta = min(ctx_member_pending_deposit, remaining); @@ -165,7 +165,7 @@ int apply_profit(int value, int value_profit, int profit) { } () member_accept_withdraw() impure { - + ;; Checks if there are pending withdrawals throw_unless(error::invalid_message(), ctx_member_pending_withdraw > 0); @@ -248,10 +248,10 @@ int apply_profit(int value, int value_profit, int profit) { () on_locked() impure { if (~ ctx_locked) { - + ;; Allow locking only on no pending withdrawals throw_unless(error::invalid_message(), ctx_balance_pending_withdraw == 0); - + ;; Update state ctx_locked = true; } @@ -284,7 +284,7 @@ int owned_balance() { () on_stake_recovered(int stake) impure { ;; Calculate profit - ;; NOTE: ctx_locked is true meaning that ctx_balance + ;; NOTE: ctx_locked is true meaning that ctx_balance ;; have the same value as was when stake was sent ;; balances are going to be unlocked after profit distribution var profit = stake - ctx_balance_sent; diff --git a/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/op-common.fc b/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/op-common.fc index b83afdbdf0..21067639f6 100644 --- a/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/op-common.fc +++ b/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/op-common.fc @@ -3,7 +3,7 @@ ;; Read extras var (enabled, udpates_enabled, min_stake, deposit_fee, withdraw_fee, pool_fee, receipt_price) = ctx_extras; throw_unless(error::invalid_message(), enabled); - + ;; Read stake value int fee = receipt_price + deposit_fee; int stake = value - fee; @@ -67,7 +67,7 @@ if (member != owner_id()) { address = serialize_work_addr(member); } - + ;; Send receipt if (ctx_query_id == 0) { send_text_message( @@ -76,7 +76,7 @@ send_mode::default(), all ? begin_cell().store_withdraw_completed() : begin_cell().store_withdraw_delayed() ); - } else { + } else { send_empty_std_message( address, withdrawed + receipt_price, @@ -92,10 +92,10 @@ } () op_donate(int value) impure { - + ;; Check value throw_unless(error::invalid_message(), value >= 2 * coins::1()); - + ;; Distribute profit to everyone distribute_profit(value - coins::1()); @@ -108,7 +108,7 @@ ;; Read extras var (enabled, udpates_enabled, min_stake, deposit_fee, withdraw_fee, pool_fee, receipt_price) = ctx_extras; throw_unless(error::invalid_message(), udpates_enabled); - + ;; Check value throw_unless(error::too_low_value(), value >= fees::deploy()); diff --git a/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/op-controller.fc b/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/op-controller.fc index 98a8fe59f1..6e47a34ca2 100644 --- a/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/op-controller.fc +++ b/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/op-controller.fc @@ -36,7 +36,7 @@ ;; Lock stakes on_locked(); - + ;; Update operation state proxy_stake_at = stake_at; proxy_stake_until = stake_at + electedFor + stakeHeldFor; @@ -50,11 +50,11 @@ ;; Send message to elector send_std_message( - ctx_proxy, - stake + coins::1(), - send_mode::separate_gas(), - elector::stake::request(), - proxy_stored_query_id, + ctx_proxy, + stake + coins::1(), + send_mode::separate_gas(), + elector::stake::request(), + proxy_stored_query_id, begin_cell() .store_uint(validator_pubkey, 256) .store_uint(stake_at, 32) @@ -136,7 +136,7 @@ () op_stake_recover(int value) impure { ;; NOTE: We never block stake recover operation - ;; in case of misbehaviour of something anyone always can get + ;; in case of misbehaviour of something anyone always can get ;; coins from elector after lockup period is up ;; Allow request only if stake is exited lockup period @@ -154,10 +154,10 @@ ;; Send message to elector send_empty_std_message( - ctx_proxy, - 0, + ctx_proxy, + 0, send_mode::carry_remaining_value(), - elector::refund::request(), + elector::refund::request(), proxy_stored_query_id ); @@ -169,7 +169,7 @@ () op_elector_recover_response(int value, slice in_msg) impure { if ((proxy_stake_until != 0) & (now() > proxy_stake_until)) { - + ;; Reset state: all stake is returned proxy_stake_sent = 0; proxy_stake_at = 0; @@ -181,12 +181,12 @@ proxy_stored_query_stake = 0; ;; Handle stake recovered event - ;; NOTE: Any stake recovery outside this condition might be just a noise and + ;; NOTE: Any stake recovery outside this condition might be just a noise and ;; effect of various race condirtions that doesn't carry any substantianal vakue on_stake_recovered(value - fees::stake_fees()); ;; Reset lock state - ;; NOTE: MUST be after on_stake_recovered since it adjusts withdrawals and + ;; NOTE: MUST be after on_stake_recovered since it adjusts withdrawals and ;; modifies global balance on_unlocked(); } @@ -207,10 +207,10 @@ ;; Send message to controller send_empty_std_message( - ctx_controller, - 0, - send_mode::carry_remaining_balance(), - op::withdraw_unowned::response(), + ctx_controller, + 0, + send_mode::carry_remaining_balance(), + op::withdraw_unowned::response(), ctx_query_id ); } @@ -293,7 +293,7 @@ (member, var cs, var f) = members.udict_get_next?(256, member); if (f) { - ;; Reject owner kicking + ;; Reject owner kicking throw_if(error::invalid_message(), member == owner_id()); ;; Kick member from a pool @@ -312,7 +312,7 @@ op::force_kick::notification(), ctx_query_id ); - + ;; Persist membership store_member(); } @@ -388,7 +388,7 @@ ctx_query_id = query_id; ;; Bounced - ;; It seems that handling doesn't make sence sicne there are no throws (?) + ;; It seems that handling doesn't make sence sicne there are no throws (?) ;; in elector contract if (flags & 1) { return (); diff --git a/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/op-nominators.fc b/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/op-nominators.fc index 233f13972d..f6e673c40a 100644 --- a/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/op-nominators.fc +++ b/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/op-nominators.fc @@ -25,7 +25,7 @@ } () op_nominators(int member, int flags, int value, slice in_msg) impure { - + ;; Ignore bounced if (flags & 1) { return (); @@ -60,7 +60,7 @@ op_stake_recover(value); return (); } - + ;; Unknown message throw(error::invalid_message()); return (); diff --git a/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/op-owner.fc b/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/op-owner.fc index 231e68898d..bf81727d27 100644 --- a/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/op-owner.fc +++ b/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/op-owner.fc @@ -1,5 +1,5 @@ () op_owner(int flags, int value, slice in_msg) impure { - + ;; Ignore bounced if (flags & 1) { return (); diff --git a/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/utils-config.fc b/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/utils-config.fc index 84e07e01a6..00c7a0be88 100644 --- a/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/utils-config.fc +++ b/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/utils-config.fc @@ -23,7 +23,7 @@ } int lockup_lift_time(int stake_at, int stake_untill) { - + ;; Resolve previous cycle parameters var (timeSince, timeUntil) = get_previous_cycle(); diff --git a/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/utils.fc b/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/utils.fc index 50e4e57c1d..b28c22f9c3 100644 --- a/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/utils.fc +++ b/crypto/func/auto-tests/legacy_tests/whales-nominators/modules/utils.fc @@ -37,8 +37,8 @@ builder store_coins(builder b, int x) asm "STGRAMS"; () send_std_message( slice to_addr, int value, - int mode, - int op, + int mode, + int op, int query_id, builder content ) impure { @@ -63,8 +63,8 @@ builder store_coins(builder b, int x) asm "STGRAMS"; () send_empty_std_message( slice to_addr, int value, - int mode, - int op, + int mode, + int op, int query_id ) impure { @@ -87,7 +87,7 @@ builder store_coins(builder b, int x) asm "STGRAMS"; () send_text_message( slice to_addr, int value, - int mode, + int mode, builder content ) impure { diff --git a/crypto/func/auto-tests/legacy_tests/whales-nominators/nominators.fc b/crypto/func/auto-tests/legacy_tests/whales-nominators/nominators.fc index 8dee4e5ecb..daa9758872 100644 --- a/crypto/func/auto-tests/legacy_tests/whales-nominators/nominators.fc +++ b/crypto/func/auto-tests/legacy_tests/whales-nominators/nominators.fc @@ -14,7 +14,7 @@ #include "modules/get.fc"; () recv_internal(int msg_value, cell in_msg_cell, slice in_msg) impure { - + ;; Prepare message context var cs = in_msg_cell.begin_parse(); var flags = cs~load_uint(4); ;; int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool diff --git a/crypto/func/auto-tests/legacy_tests/whales-nominators/proxy.fc b/crypto/func/auto-tests/legacy_tests/whales-nominators/proxy.fc index 805b60989e..5ba1914826 100644 --- a/crypto/func/auto-tests/legacy_tests/whales-nominators/proxy.fc +++ b/crypto/func/auto-tests/legacy_tests/whales-nominators/proxy.fc @@ -1,7 +1,7 @@ (int) equal_slices (slice s1, slice s2) asm "SDEQ"; () recv_internal(cell in_msg_cell, slice in_msg) { - + ;; Parse message var cs = in_msg_cell.begin_parse(); var flags = cs~load_uint(4); ;; int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool diff --git a/crypto/func/auto-tests/tests/w2.fc b/crypto/func/auto-tests/tests/w2.fc index b9e5fb0e50..a0572bc260 100644 --- a/crypto/func/auto-tests/tests/w2.fc +++ b/crypto/func/auto-tests/tests/w2.fc @@ -1,5 +1,5 @@ _ f(cs) { - return (cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), + return (cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), diff --git a/crypto/func/builtins.cpp b/crypto/func/builtins.cpp index db0b9e7302..52f23e134c 100644 --- a/crypto/func/builtins.cpp +++ b/crypto/func/builtins.cpp @@ -24,7 +24,7 @@ using namespace std::literals::string_literals; /* * * SYMBOL VALUES - * + * */ int glob_func_cnt, undef_func_cnt, glob_var_cnt, const_cnt; @@ -98,9 +98,9 @@ bool SymValAsmFunc::compile(AsmOpList& dest, std::vector& out, std::ve } /* - * + * * DEFINE BUILT-IN FUNCTIONS - * + * */ int emulate_negate(int a) { diff --git a/crypto/func/codegen.cpp b/crypto/func/codegen.cpp index aa2972c242..8667685ee6 100644 --- a/crypto/func/codegen.cpp +++ b/crypto/func/codegen.cpp @@ -21,9 +21,9 @@ namespace funC { /* - * + * * GENERATE TVM STACK CODE - * + * */ StackLayout Stack::vars() const { diff --git a/crypto/func/func.h b/crypto/func/func.h index 7b1e469a53..8a216e430a 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -144,9 +144,9 @@ class IdSc { // 0 = else /* - * + * * TYPE EXPRESSIONS - * + * */ struct TypeExpr { @@ -294,9 +294,9 @@ void unify(TypeExpr*& te1, TypeExpr*& te2); // extern int TypeExpr::holes; /* - * + * * ABSTRACT CODE - * + * */ using src::Lexem; @@ -751,7 +751,7 @@ struct CodeBlob { /* * * SYMBOL VALUES - * + * */ struct SymVal : sym::SymValBase { @@ -849,9 +849,9 @@ extern std::vector glob_func, glob_vars; extern std::set prohibited_var_names; /* - * + * * PARSE SOURCE - * + * */ class ReadCallback { @@ -885,9 +885,9 @@ bool parse_source_stdin(); extern std::stack inclusion_locations; /* - * + * * EXPRESSIONS - * + * */ struct Expr { @@ -981,9 +981,9 @@ struct Expr { }; /* - * + * * GENERATE CODE - * + * */ typedef std::vector StackLayout; @@ -1268,9 +1268,9 @@ int is_pos_pow2(td::RefInt256 x); int is_neg_pow2(td::RefInt256 x); /* - * + * * STACK TRANSFORMS - * + * */ /* @@ -1458,9 +1458,9 @@ inline std::ostream& operator<<(std::ostream& os, const StackTransform& trans) { bool apply_op(StackTransform& trans, const AsmOp& op); /* - * + * * STACK OPERATION OPTIMIZER - * + * */ struct Optimizer { @@ -1678,7 +1678,7 @@ struct Stack { * * SPECIFIC SYMBOL VALUES, * BUILT-IN FUNCTIONS AND OPERATIONS - * + * */ typedef std::function&, std::vector&, const SrcLocation)> simple_compile_func_t; diff --git a/crypto/func/gen-abscode.cpp b/crypto/func/gen-abscode.cpp index 0a6998f640..840baed234 100644 --- a/crypto/func/gen-abscode.cpp +++ b/crypto/func/gen-abscode.cpp @@ -25,9 +25,9 @@ using namespace std::literals::string_literals; namespace funC { /* - * + * * EXPRESSIONS - * + * */ Expr* Expr::copy() const { diff --git a/crypto/func/keywords.cpp b/crypto/func/keywords.cpp index cd73c9a8fc..189e11daa5 100644 --- a/crypto/func/keywords.cpp +++ b/crypto/func/keywords.cpp @@ -29,9 +29,9 @@ int lexem_is_special(std::string str) { namespace funC { /* - * + * * KEYWORD DEFINITION - * + * */ void define_keywords() { diff --git a/crypto/func/optimize.cpp b/crypto/func/optimize.cpp index 258663901a..ee45143c94 100644 --- a/crypto/func/optimize.cpp +++ b/crypto/func/optimize.cpp @@ -21,9 +21,9 @@ namespace funC { /* - * + * * PEEPHOLE OPTIMIZER - * + * */ void Optimizer::set_code(AsmOpConsList code) { diff --git a/crypto/func/parse-func.cpp b/crypto/func/parse-func.cpp index d2f2219bf1..779ad56c39 100644 --- a/crypto/func/parse-func.cpp +++ b/crypto/func/parse-func.cpp @@ -60,9 +60,9 @@ inline bool is_special_ident(sym_idx_t idx) { } /* - * + * * PARSE SOURCE - * + * */ // TE ::= TA | TA -> TE diff --git a/crypto/func/stack-transform.cpp b/crypto/func/stack-transform.cpp index b529060864..6b19a7ae00 100644 --- a/crypto/func/stack-transform.cpp +++ b/crypto/func/stack-transform.cpp @@ -21,9 +21,9 @@ namespace funC { /* - * + * * GENERIC STACK TRANSFORMATIONS - * + * */ StackTransform::StackTransform(std::initializer_list list) { diff --git a/crypto/func/test/w2.fc b/crypto/func/test/w2.fc index 3bbcd415ab..41d60895ce 100644 --- a/crypto/func/test/w2.fc +++ b/crypto/func/test/w2.fc @@ -1,5 +1,5 @@ _ f(cs) { - return (cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), + return (cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), diff --git a/crypto/func/test/w4.fc b/crypto/func/test/w4.fc index 1c0e2cefe1..6c5c90fad5 100644 --- a/crypto/func/test/w4.fc +++ b/crypto/func/test/w4.fc @@ -2,7 +2,7 @@ _ main(cell dict, int t3) { int index = -1; do { (index, slice value, int found) = dict.udict_get_next?(32, index); - if (found) { + if (found) { (int temp1, int temp2) = (value~load_uint(16), value~load_uint(32)); if (t3 > temp2) { dict~udict_delete_get?(32, index); diff --git a/crypto/func/test/w8.fc b/crypto/func/test/w8.fc index bd0696e736..ee995a2383 100644 --- a/crypto/func/test/w8.fc +++ b/crypto/func/test/w8.fc @@ -11,11 +11,11 @@ int check_signatures(msg_hash, signatures, signers, bitmask_size) impure { if (_found){ if (check_signature(msg_hash, sig, public_key)){ var signer_index = cs~load_uint(bitmask_size); - bitmask = bitmask | (1 << (signer_index - 1)); - } + bitmask = bitmask | (1 << (signer_index - 1)); + } } } until (~ _found); - ;; signature~touch(); + ;; signature~touch(); } } until (~ f); return bitmask; diff --git a/crypto/func/unify-types.cpp b/crypto/func/unify-types.cpp index f9b639c0e5..8f860c3bc2 100644 --- a/crypto/func/unify-types.cpp +++ b/crypto/func/unify-types.cpp @@ -21,9 +21,9 @@ namespace funC { /* - * + * * TYPE EXPRESSIONS - * + * */ int TypeExpr::holes = 0, TypeExpr::type_vars = 0; // not thread safe, but it is ok for now diff --git a/crypto/smartcont/CreateState.fif b/crypto/smartcont/CreateState.fif index 25873eefe6..b504677366 100644 --- a/crypto/smartcont/CreateState.fif +++ b/crypto/smartcont/CreateState.fif @@ -86,7 +86,7 @@ validators-weight 0! } : serialize-adnl-validator // ( weight val-cell -- ) { swap validators-weight +! - @@ -145,8 +145,8 @@ variable special-dict } : config.storage_prices! { 16 << } : sg* { 16 <$ type ." -> " RWCode2 // code - // data empty_cell // libs 3 roll // balance diff --git a/crypto/smartcont/LICENSE.LGPL b/crypto/smartcont/LICENSE.LGPL index b482fc4ed4..464e46b9f7 100644 --- a/crypto/smartcont/LICENSE.LGPL +++ b/crypto/smartcont/LICENSE.LGPL @@ -133,7 +133,7 @@ such a program is covered only if its contents constitute a work based on the Library (independent of the use of the Library in a tool for writing it). Whether that is true depends on what the Library does and what the program that uses the Library does. - + 1. You may copy and distribute verbatim copies of the Library's complete source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an @@ -203,7 +203,7 @@ instead of to this License. (If a newer version than version 2 of the ordinary GNU General Public License has appeared, then you can specify that version instead if you wish.) Do not make any other change in these notices. - + Once this change is made in a given copy, it is irreversible for that copy, so the ordinary GNU General Public License applies to all subsequent copies and derivative works made from that copy. @@ -308,7 +308,7 @@ restrictions of other proprietary libraries that do not normally accompany the operating system. Such a contradiction means you cannot use both them and the Library together in an executable that you distribute. - + 7. You may place library facilities that are a work based on the Library side-by-side in a single library together with other library facilities not covered by this License, and distribute such a combined @@ -401,7 +401,7 @@ conditions either of that version or of any later version published by the Free Software Foundation. If the Library does not specify a license version number, you may choose any version ever published by the Free Software Foundation. - + 14. If you wish to incorporate parts of the Library into other free programs whose distribution conditions are incompatible with these, write to the author to ask for permission. For software which is diff --git a/crypto/smartcont/asm-to-cpp.fif b/crypto/smartcont/asm-to-cpp.fif index 21ff34bb2d..4633c13e2d 100755 --- a/crypto/smartcont/asm-to-cpp.fif +++ b/crypto/smartcont/asm-to-cpp.fif @@ -18,8 +18,8 @@ $3 =: name source include boc>B -."Convert boc to base64" cr -B>base64 +."Convert boc to base64" cr +B>base64 { char " chr tuck $+ $+ } : wrp wrp // "" diff --git a/crypto/smartcont/auto-dns.fif b/crypto/smartcont/auto-dns.fif index 21420d1ee0..dd59f0fad0 100644 --- a/crypto/smartcont/auto-dns.fif +++ b/crypto/smartcont/auto-dns.fif @@ -45,13 +45,13 @@ variable Values dictnew Values ! { over 0= abort"category cannot be zero" udict!+ not abort"duplicate category id" Values ! -} : register-value +} : register-value { @end? abort"category number expected" @next (number) 1 <> abort"category must be integer" dup 256 fits not abort"category does not fit into 256 bit integer" dup 0= abort"category must be non-zero" } : parse-cat-num -{ @end? abort"smart contract address expected" +{ @end? abort"smart contract address expected" @next false parse-load-address drop } : cl-parse-smc-addr { @end? abort"adnl address expected" @@ -91,7 +91,7 @@ parse-actions main-op ( _( `add 0x72656764 ) _( `update 0x75706464 ) _( `prolong 0x70726f6c ) ) assq-val not abort"unknown main operation" -=: op-id +=: op-id ."Automatic DNS smart contract address = " dest-addr 2dup .addr cr 6 .Addr cr diff --git a/crypto/smartcont/complaint-vote-req.fif b/crypto/smartcont/complaint-vote-req.fif index f771f6fda9..1d2e0d94f2 100644 --- a/crypto/smartcont/complaint-vote-req.fif +++ b/crypto/smartcont/complaint-vote-req.fif @@ -23,7 +23,7 @@ $4 savefile replace-if-null =: savefile ."Creating a request to vote for complaint 0x" compl-hash 64x. ."of past validator set " elect-id . ."on behalf of current validator with index " val-idx . cr -B{56744350} val-idx 16 u>B B+ elect-id 32 u>B B+ compl-hash 256 u>B B+ +B{56744350} val-idx 16 u>B B+ elect-id 32 u>B B+ compl-hash 256 u>B B+ dup Bx. cr dup B>base64url type cr savefile tuck B>file ."Saved to file " type cr diff --git a/crypto/smartcont/config-code.fc b/crypto/smartcont/config-code.fc index b3aa04c46e..4c22159e3d 100644 --- a/crypto/smartcont/config-code.fc +++ b/crypto/smartcont/config-code.fc @@ -132,8 +132,8 @@ _ perform_action(cfg_dict, public_key, action, cs) inline_ref { (cell, int, cell) get_current_vset() inline_ref { var vset = config_param(34); var cs = begin_parse(vset); - ;; validators_ext#12 utime_since:uint32 utime_until:uint32 - ;; total:(## 16) main:(## 16) { main <= total } { main >= 1 } + ;; validators_ext#12 utime_since:uint32 utime_until:uint32 + ;; total:(## 16) main:(## 16) { main <= total } { main >= 1 } ;; total_weight:uint64 throw_unless(40, cs~load_uint(8) == 0x12); cs~skip_bits(32 + 32 + 16 + 16); diff --git a/crypto/smartcont/config-proposal-vote-req.fif b/crypto/smartcont/config-proposal-vote-req.fif index d1eaff9d68..77772ceeb5 100644 --- a/crypto/smartcont/config-proposal-vote-req.fif +++ b/crypto/smartcont/config-proposal-vote-req.fif @@ -33,14 +33,14 @@ $2 parse-int dup =: prop-hash 256 ufits not abort"invalid proposal hash" $3 savefile replace-if-null =: savefile -."Creating a request " internal? { ."with seqno " seqno . ."expiring at " expire-at . } ifnot +."Creating a request " internal? { ."with seqno " seqno . ."expiring at " expire-at . } ifnot ."to vote for configuration proposal 0x" prop-hash 64x. ."on behalf of validator with index " val-idx . cr internal? { B{566f7445} } { B{566f7465} seqno 32 u>B B+ expire-at 32 u>B B+ } cond -val-idx 16 u>B B+ prop-hash 256 u>B B+ +val-idx 16 u>B B+ prop-hash 256 u>B B+ dup Bx. cr dup B>base64url type cr savefile tuck B>file ."Saved to file " type cr diff --git a/crypto/smartcont/config-proposal-vote-signed.fif b/crypto/smartcont/config-proposal-vote-signed.fif index 5aaf127063..83e103c908 100644 --- a/crypto/smartcont/config-proposal-vote-signed.fif +++ b/crypto/smartcont/config-proposal-vote-signed.fif @@ -47,7 +47,7 @@ internal? { { ."Creating an external message to configuration smart contract " config-addr 2dup 6 .Addr ." = " .addr cr } cond -."containing a signed request " internal? { ."expiring at " expire-at . } ifnot +."containing a signed request " internal? { ."expiring at " expire-at . } ifnot ."to vote for configuration proposal 0x" prop-hash 64x. ."on behalf of validator with index " val-idx . "and public key" pubkey Bx. cr @@ -65,7 +65,7 @@ now 32 << prop-hash 32 1<< mod + =: query-id -cr +cr internal? { ."Internal message body is " } { ."External message is " } cond dup = tuck 1 i, -rot { 256 u, } { drop } cond b> ref, += tuck 1 i, -rot { 256 u, } { drop } cond b> ref, critical 1 i, b> dup ."resulting internal message body: " B dup Bx. cr param-value dup null? { drop } { - totalcsize swap ."(a total of " . ."data bits, " . ."cell references -> " + totalcsize swap ."(a total of " . ."data bits, " . ."cell references -> " drop dup Blen . ."BoC data bytes)" cr } cond diff --git a/crypto/smartcont/create-config-upgrade-proposal.fif b/crypto/smartcont/create-config-upgrade-proposal.fif index 35caa64b18..874cbf2df1 100644 --- a/crypto/smartcont/create-config-upgrade-proposal.fif +++ b/crypto/smartcont/create-config-upgrade-proposal.fif @@ -27,7 +27,7 @@ begin-options src-file $+ +"' by default)" option-help "H" "--old-hash" { (hex-number) not abort"256-bit hex number expected as hash" =: old-hash } short-long-option-arg - "Sets the required cell hash of existing parameter value (0 means no value)" option-help + "Sets the required cell hash of existing parameter value (0 means no value)" option-help "h" "--help" { usage } short-long-option "Shows a help message" option-help parse-options @@ -51,9 +51,9 @@ now 32 << param-idx + =: query-id ."Query id is " query-id . cr // create message body -= tuck 1 i, -rot { 256 u, } { drop } cond b> ref, += tuck 1 i, -rot { 256 u, } { drop } cond b> ref, critical 1 i, b> dup ."resulting internal message body: " = tuck 1 i, -rot { 256 u, } { drop } cond b> ref, += tuck 1 i, -rot { 256 u, } { drop } cond b> ref, critical 1 i, b> dup ."resulting internal message body: " :HashmapUInt<32b>,CatTable>:domains] [OptRef<1b+1r?>:Hashmap(Time|Hash128)->Slice(DomName)>:gc] [UInt<32b>:stdperiod] [Gram:PPReg] [Gram:PPCell] [Gram:PPBit] - [UInt<32b>:lasthousekeeping] + [UInt<32b>:lasthousekeeping] := HashmapE 256 (~~16~~) ^DNSRecord - + STORED DOMAIN NAME SLICE FORMAT: (#ZeroChars<7b>) (Domain name value) #Zeros allows to simultaneously store, for example, com\0 and com\0google\0 That will be stored as \1com\0 and \2com\0google\0 (pfx tree has restricitons) @@ -75,9 +75,9 @@ cell udict_get_ref_(cell dict, int key_len, int index) asm(index dict key_len) " global var query_info; -() send_message(slice addr, int tag, int query_id, +() send_message(slice addr, int tag, int query_id, int body, int grams, int mode) impure { - ;; int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool + ;; int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool ;; src:MsgAddress -> 011000 0x18 var msg = begin_cell() .store_uint (0x18, 6) @@ -175,11 +175,11 @@ int check_owner(cell cat_table, cell owner_info, int src_wc, int src_addr, int s int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool src:MsgAddressInt dest:MsgAddressInt value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams - created_lt:uint64 created_at:uint32 + created_lt:uint64 created_at:uint32 Internal message data structure: - [UInt<32b>:op] [UInt<64b>:query_id] [Ref<1r>:domain] + [UInt<32b>:op] [UInt<64b>:query_id] [Ref<1r>:domain] (if not prolong: [Ref<1r>:value->CatTable]) - + -} ;; Control operations: permitted only to the owner of this smartcontract @@ -219,27 +219,27 @@ int check_owner(cell cat_table, cell owner_info, int src_wc, int src_addr, int s ;; Must send at least GR$1 more for possible gas fees! () recv_internal(int msg_value, cell in_msg_cell, slice in_msg) impure { ;; this time very interested in internal messages - if (in_msg.slice_bits() < 32) { + if (in_msg.slice_bits() < 32) { return (); ;; simple transfer or short } slice cs = in_msg_cell.begin_parse(); int flags = cs~load_uint(4); - if (flags & 1) { + if (flags & 1) { return (); ;; bounced messages } slice s_addr = cs~load_msg_addr(); (int src_wc, int src_addr) = s_addr.parse_std_addr(); int op = in_msg~load_uint(32); - ifnot (op) { + ifnot (op) { return (); ;; simple transfer with comment } int query_id = 0; - if (in_msg.slice_bits() >= 64) { + if (in_msg.slice_bits() >= 64) { query_id = in_msg~load_uint(64); } - + query_info = (s_addr, query_id, op); - + if (op & (1 << 31)) { return (); ;; an answer to our query } @@ -247,15 +247,15 @@ int check_owner(cell cat_table, cell owner_info, int src_wc, int src_addr, int s ;; Control operations return perform_ctl_op(op, src_wc, src_addr, in_msg); } - + int qt = (op == 0x72656764) * 1 + (op == 0x70726f6c) * 2 + (op == 0x75706464) * 4 + (op == 0x676f6763) * 8; ifnot (qt) { ;; unknown query, return error return send_error(0xffffffff); } qt = - qt; - + (cell ctl, cell domdata, cell gc, [int, int, int, int] prices, int nhk, int lhk) = load_data(); - + if (qt == 8) { ;; 0x676f6763 -> GO, GC! go!!! ;; Manual garbage collection iteration int max_steps = in_msg~load_int(32); ;; -1 = infty @@ -283,7 +283,7 @@ int check_owner(cell cat_table, cell owner_info, int src_wc, int src_addr, int s if (fail) { return send_error(0xee6f5c30); } - + int n = now(); cell cat_table = cell owner_info = null(); int key = int exp = int zeros = 0; @@ -309,23 +309,23 @@ int check_owner(cell cat_table, cell owner_info, int src_wc, int src_addr, int s } } } - + if (zeros > 4) { ;; too much zero chars (overflow): ov\0 return send_error(0xef765c30); } ;; ########################################################################## - + int err = check_owner(cat_table, owner_info, src_wc, src_addr, qt != 1); if (err) { return send_error(err); } ;; ########################################################################## - + ;; load desired data (reuse old for a "prolong" operation) cell data = null(); - + if (qt != 2) { ;; not a "prolong", load data dictionary data = in_msg~load_ref(); ;; basic integrity check of (client-provided) dictionary @@ -356,7 +356,7 @@ int check_owner(cell cat_table, cell owner_info, int src_wc, int src_addr, int s if (msg_value - (1 << 30) < price) { ;; gr regd | register domain ifnot (cat_table.null?()) { ;; domain already exists: return alre | 2^31 @@ -398,7 +398,7 @@ int check_owner(cell cat_table, cell owner_info, int src_wc, int src_addr, int s return send_ok(price); } ;; ########################################################################## - + return (); ;; should NEVER reach this part of code! } @@ -445,7 +445,7 @@ int check_owner(cell cat_table, cell owner_info, int src_wc, int src_addr, int s } var ds = get_data().begin_parse(); (_, cell root) = (ds~load_ref(), ds~load_dict()); - + slice val = null(); int tail_bits = -1; slice tail = domain; @@ -480,10 +480,10 @@ int check_owner(cell cat_table, cell owner_info, int src_wc, int src_addr, int s category = "dns_next_resolver"H; ;; 0x19f02441ee588fdb26ee24b2568dd035c3c9206e11ab979be62e55558a1d17ff ;; update: next resolver is now sha256("dns_next_resolver") instead of -1 } - + int pfx_bits = pfx.slice_bits(); - - ;; pfx.slice_bits() will contain 8m, where m is number of bytes in subdomain + + ;; pfx.slice_bits() will contain 8m, where m is number of bytes in subdomain ;; COUNTING the zero byte (if structurally correct: no multiple-ZB keys) ;; which corresponds to 8m, m=one plus the number of bytes in the subdomain found) ifnot (category) { diff --git a/crypto/smartcont/dns-manual-code.fc b/crypto/smartcont/dns-manual-code.fc index 555e5952d8..78fb5f748c 100644 --- a/crypto/smartcont/dns-manual-code.fc +++ b/crypto/smartcont/dns-manual-code.fc @@ -150,11 +150,11 @@ cell udict_get_ref_(cell dict, int key_len, int index) asm(index dict key_len) " ;; operations with codes 20..29 replace / delete dict, no need if (op < 20) { ;; lets resolve the name here so as not to duplicate the code - (slice pfx, cell val, slice tail, int succ) = + (slice pfx, cell val, slice tail, int succ) = root.pfxdict_get_ref(1023, name); if (succ) { ;; must match EXACTLY to prevent accident changes - throw_unless(35, tail.slice_empty?()); + throw_unless(35, tail.slice_empty?()); cat_table = val; } ;; otherwise cat_table is null which is reasonable for actions @@ -169,7 +169,7 @@ cell udict_get_ref_(cell dict, int key_len, int index) asm(index dict key_len) " ;; 12 VDel: delete specified subdomain->category value if (op == 12) { if (cat_table~udict_delete?(256, cat)) { ;; update: category length now u256 instead of i16 - root~pfxdict_set_ref(1023, name, cat_table); + root~pfxdict_set_ref(1023, name, cat_table); } return (root, ops); } @@ -262,10 +262,10 @@ cell process_ops(cell root, slice ops) inline_ref { {- Data structure: - Root cell: [UInt<32b>:seqno] [UInt<256b>:owner_public_key] + Root cell: [UInt<32b>:seqno] [UInt<256b>:owner_public_key] [OptRef<1b+1r?>:HashmapCatTable>:domains] := HashmapE 256 (~~16~~) ^DNSRecord - + STORED DOMAIN NAME SLICE FORMAT: (#ZeroChars<7b>) (Domain name value) #Zeros allows to simultaneously store, for example, com\0 and com\0google\0 That will be stored as \1com\0 and \2com\0google\0 (pfx tree has restricitons) @@ -316,7 +316,7 @@ int get_public_key() method_id { bits -= 8; } (_, _, _, cell root, _) = load_data(); - + slice cname = subdomain; int zeros = 0; repeat (bits >> 3) { @@ -349,7 +349,7 @@ int get_public_key() method_id { } int pfx_bits = pfx.slice_bits() - 7; cell cat_table = val; - ;; pfx.slice_bits() will contain 8m, where m is number of bytes in subdomain + ;; pfx.slice_bits() will contain 8m, where m is number of bytes in subdomain ;; COUNTING the zero byte (if structurally correct: no multiple-ZB keys) ;; which corresponds to 8m, m=one plus the number of bytes in the subdomain found) if (category == 0) { diff --git a/crypto/smartcont/elector-code.fc b/crypto/smartcont/elector-code.fc index 97185fc216..9b9addcf1d 100644 --- a/crypto/smartcont/elector-code.fc +++ b/crypto/smartcont/elector-code.fc @@ -99,7 +99,7 @@ builder pack_complaint(int validator_pubkey, cell description, int created_at, i .store_uint(suggested_fine_part, 32); } -;; complaint_prices#1a deposit:Grams bit_price:Grams cell_price:Grams = ComplaintPricing; +;; complaint_prices#1a deposit:Grams bit_price:Grams cell_price:Grams = ComplaintPricing; (int, int, int) parse_complaint_prices(cell info) inline { var cs = info.begin_parse(); throw_unless(9, cs~load_uint(8) == 0x1a); @@ -125,8 +125,8 @@ builder pack_complaint(int validator_pubkey, cell description, int created_at, i (cell, int, cell) get_current_vset() inline_ref { var vset = config_param(34); var cs = begin_parse(vset); - ;; validators_ext#12 utime_since:uint32 utime_until:uint32 - ;; total:(## 16) main:(## 16) { main <= total } { main >= 1 } + ;; validators_ext#12 utime_since:uint32 utime_until:uint32 + ;; total:(## 16) main:(## 16) { main <= total } { main >= 1 } ;; total_weight:uint64 throw_unless(40, cs~load_uint(8) == 0x12); cs~skip_bits(32 + 32 + 16 + 16); @@ -175,7 +175,7 @@ builder pack_complaint(int validator_pubkey, cell description, int created_at, i () send_validator_set_to_config(config_addr, vset, query_id) impure inline_ref { var msg = begin_cell() - .store_uint(0xc4ff, 17) ;; 0 11000100 0xff + .store_uint(0xc4ff, 17) ;; 0 11000100 0xff .store_uint(config_addr, 256) .store_grams(1 << 30) ;; ~1 gram of value to process and obtain answer .store_uint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) @@ -637,7 +637,7 @@ int proceed_register_vote(election_id, chash, idx, weight) impure inline_ref { return config_set_confirmed(s_addr, in_msg, query_id, cfg_ok); } if (op == 0x52674370) { - ;; new complaint + ;; new complaint var price = register_complaint(s_addr, in_msg, msg_value); int mode = 64; int ans_tag = - price; @@ -796,7 +796,7 @@ _ compute_total_stake(l, n, m_stake) inline_ref { i += 1; } until (l.null?()); throw_unless(49, tot_stake == best_stake); - return (credits, vset, tot_weight, frozen, tot_stake, m); + return (credits, vset, tot_weight, frozen, tot_stake, m); } int conduct_elections(ds, elect, credits) impure { @@ -846,7 +846,7 @@ int conduct_elections(ds, elect, credits) impure { .store_uint(0x12, 8) ;; validators_ext#12 .store_uint(start, 32) ;; utime_since:uint32 .store_uint(start + elect_for, 32) ;; utime_until:uint32 - .store_uint(cnt, 16) ;; total:(## 16) + .store_uint(cnt, 16) ;; total:(## 16) .store_uint(min(cnt, main_validators), 16) ;; main:(## 16) .store_uint(total_weight, 64) ;; total_weight:uint64 .store_dict(vdict) ;; list:(HashmapE 16 ValidatorDescr) diff --git a/crypto/smartcont/envelope-complaint.fif b/crypto/smartcont/envelope-complaint.fif index e9a65c7265..c15fbe5a57 100644 --- a/crypto/smartcont/envelope-complaint.fif +++ b/crypto/smartcont/envelope-complaint.fif @@ -43,7 +43,7 @@ dup ."resulting internal message body: " B dup Bx. cr complaint -totalcsize swap ."(a total of " . ."data bits, " . ."cell references -> " +totalcsize swap ."(a total of " . ."data bits, " . ."cell references -> " drop dup Blen . ."BoC data bytes)" cr savefile tuck B>file diff --git a/crypto/smartcont/gen-zerostate-test.fif b/crypto/smartcont/gen-zerostate-test.fif index d98c7a2630..b6d9f46c5d 100644 --- a/crypto/smartcont/gen-zerostate-test.fif +++ b/crypto/smartcont/gen-zerostate-test.fif @@ -3,7 +3,7 @@ "Asm.fif" include "Lists.fif" include -def? $1 { @' $1 } { "" } cond constant suffix +def? $1 { @' $1 } { "" } cond constant suffix { suffix $+ } : +suffix 256 1<<1- 15 / constant AllOnes @@ -12,7 +12,7 @@ wc_master setworkchain // Initial state of Workchain 0 (Basic workchain) -0 mkemptyShardState +0 mkemptyShardState cr ."initial basechain state is:" cr dup B dup Bx. cr @@ -50,9 +50,9 @@ config.workchains! INC NEWC 32 STU 256 STU ENDC c4 POPCTR }>c // code - // data Libs{ x{ABACABADABACABA} s>c public_lib @@ -64,8 +64,8 @@ GR$4999990000 // balance AllOnes 0 * // address 6 // mode: create+setaddr register_smc -dup make_special dup constant smc1_addr -Masterchain over +dup make_special dup constant smc1_addr +Masterchain over 2dup ."wallet address = " .addr cr 2dup 6 .Addr cr "main-wallet" +suffix +".addr" save-address-verbose @@ -90,7 +90,7 @@ empty_cell // libraries GR$1000 // initial balance (1k test Grams) 0 0 AllOnes 6 * 6 register_smc dup make_special dup constant smc2_addr -Masterchain over +Masterchain over 2dup ."free test gram giver address = " .addr cr 2dup 6 .Addr cr "testgiver" +suffix +".addr" save-address-verbose @@ -110,7 +110,7 @@ PROGRAM{ // send raw message from Cell ZERO SENDRAWMSG -17 INT 256 STIR 130000000 INT STGRAMS - 107 INT STZEROES ENDC + 107 INT STZEROES ENDC ZERO // another message with 0.13 Grams to account -17 NEWC b{11000100100000} "test" $>s |+ STSLICECONST 123456789 INT STGRAMS @@ -178,7 +178,7 @@ elector_addr config.elector_smc! 1 500 1000 500000 config.storage_prices! config.special! -// gas_price gas_limit special_gas_limit gas_credit block_gas_limit freeze_due_limit delete_due_limit flat_gas_limit flat_gas_price -- +// gas_price gas_limit special_gas_limit gas_credit block_gas_limit freeze_due_limit delete_due_limit flat_gas_limit flat_gas_price -- 1000 sg* 1 *M dup 10000 10 *M GR$0.1 GR$1.0 1000 1000000 config.gas_prices! 10000 sg* 1 *M 10 *M 10000 10 *M GR$0.1 GR$1.0 1000 10000000 config.mc_gas_prices! // lump_price bit_price cell_price ihr_factor first_frac next_frac @@ -217,7 +217,7 @@ GR$100 1 500 config.complaint_prices! "validator-keys" +suffix +".pub" file>B { dup Blen } { 32 B| swap dup ."Validator public key = " Bx. cr 17 add-validator } while drop -// newkeypair nip dup ."Validator #1 public key = " Bx. cr +// newkeypair nip dup ."Validator #1 public key = " Bx. cr // 17 add-validator // newkeypair nip dup ."Validator #2 public key = " Bx. cr // 239 add-validator @@ -250,7 +250,7 @@ Masterchain swap * Initial wallets (test) * */ - + // pubkey amount `create-wallet1` or pubkey amount `create-wallet2` PK'PuZPPXK5Rff9SvtoS7Y9lUuEixvy-J6aishYFj3Qn6P0pJMb GR$1000000000 create-wallet1 PK'PuYiB1zAWzr4p8j6I681+sGUrRGcn6Ylf7vXl0xaUl/w6Xfg GR$1700000000 create-wallet0 diff --git a/crypto/smartcont/gen-zerostate.fif b/crypto/smartcont/gen-zerostate.fif index d41abaee90..c9e09df01a 100644 --- a/crypto/smartcont/gen-zerostate.fif +++ b/crypto/smartcont/gen-zerostate.fif @@ -3,7 +3,7 @@ "Asm.fif" include "Lists.fif" include -def? $1 { @' $1 } { "" } cond constant suffix +def? $1 { @' $1 } { "" } cond constant suffix { suffix $+ } : +suffix 256 1<<1- 15 / constant AllOnes @@ -13,7 +13,7 @@ wc_master setworkchain // Initial state of Workchain 0 (Basic workchain) -0 mkemptyShardState +0 mkemptyShardState cr ."initial basechain state is:" cr dup B dup Bx. cr @@ -67,7 +67,7 @@ StB 999. PubMMGvqM08jx_6BibYldMclwjl-D88r7-u0_IEcDXHA30-G WCode3 // code // data empty_cell // libs GR$4999999000 allocated-balance - // balance @@ -76,8 +76,8 @@ GR$4999999000 allocated-balance - // balance AllOnes 1 * // address 6 // mode: create+setaddr register_smc -dup make_special dup constant smc1_addr -Masterchain over +dup make_special dup constant smc1_addr +Masterchain over 2dup ."main wallet address = " .addr cr 2dup 6 .Addr cr "main-wallet" +suffix +".addr" save-address-verbose @@ -126,7 +126,7 @@ elector_addr config.elector_smc! 1 500 1000 500000 config.storage_prices! config.special! -// gas_price gas_limit special_gas_limit gas_credit block_gas_limit freeze_due_limit delete_due_limit flat_gas_limit flat_gas_price -- +// gas_price gas_limit special_gas_limit gas_credit block_gas_limit freeze_due_limit delete_due_limit flat_gas_limit flat_gas_price -- 1000 sg* 1 *M dup 10000 10 *M GR$0.1 GR$1.0 1000 1000000 config.gas_prices! 10000 sg* 1 *M 10 *M 10000 10 *M GR$0.1 GR$1.0 1000 10000000 config.mc_gas_prices! // lump_price bit_price cell_price ihr_factor first_frac next_frac @@ -162,7 +162,7 @@ config.param_proposals_setup! // deposit bit_pps cell_pps GR$100 1 500 config.complaint_prices! -{ file>B { dup Blen } { +{ file>B { dup Blen } { 32 B| swap dup ."Validator public key = " Bx. cr 17 add-validator } while drop @@ -176,7 +176,7 @@ keys-from-file B64{8E9AbL8blhRiBUoZt/b/vIo3+iZzglLKiL8MTf0g6Sg=} 256 B>u@ 1 add-adnl-validator } cond -// newkeypair nip dup ."Validator #1 public key = " Bx. cr +// newkeypair nip dup ."Validator #1 public key = " Bx. cr // 17 add-validator // newkeypair nip dup ."Validator #2 public key = " Bx. cr // 239 add-validator @@ -204,7 +204,7 @@ Masterchain swap ."config smart contract address = " 2dup .addr cr 2dup 7 .Addr cr "config-master" +suffix +".addr" save-address-verbose // Other data - + /* * * Create state diff --git a/crypto/smartcont/highload-wallet-v2-code.fc b/crypto/smartcont/highload-wallet-v2-code.fc index b7626bbe59..73b867f73c 100644 --- a/crypto/smartcont/highload-wallet-v2-code.fc +++ b/crypto/smartcont/highload-wallet-v2-code.fc @@ -5,7 +5,7 @@ ;; Note, when dealing with highload-wallet the following limits need to be checked and taken into account: -;; 1) Storage size limit. Currently, size of contract storage should be less than 65535 cells. If size of +;; 1) Storage size limit. Currently, size of contract storage should be less than 65535 cells. If size of ;; old_queries will grow above this limit, exception in ActionPhase will be thrown and transaction will fail. ;; Failed transaction may be replayed. ;; 2) Gas limit. Currently, gas limit is 1'000'000 gas units, that means that there is a limit of how much diff --git a/crypto/smartcont/highload-wallet-v2-one.fif b/crypto/smartcont/highload-wallet-v2-one.fif index bd7fa5aa0a..735d9644f9 100644 --- a/crypto/smartcont/highload-wallet-v2-one.fif +++ b/crypto/smartcont/highload-wallet-v2-one.fif @@ -56,7 +56,7 @@ def? body-boc-file { @' body-boc-file file>B B>boc } { comment simple-transfer-b constant body-cell ."Transferring " amount .GR+cc ."to account " -dest_addr 2dup bounce 7 + .Addr ." = " .addr +dest_addr 2dup bounce 7 + .Addr ." = " .addr ."subwallet-id=0x" subwallet-id x. ."timeout=" timeout . ."bounce=" bounce . cr ."Body of transfer message is " body-cell } : create-int-msg // ng wc addr bnc -- -{ ."Transferring " 3 roll .GR ."to account " +{ ."Transferring " 3 roll .GR ."to account " -rot 2dup 4 pick 7 + .Addr ." = " .addr ." bounce=" . cr } : .transfer // addr$ ng -- c -{ swap parse-smc-addr force-bounce or allow-bounce and // ng wc addr bnc +{ swap parse-smc-addr force-bounce or allow-bounce and // ng wc addr bnc 2over 2over .transfer create-int-msg } : create-simple-transfer diff --git a/crypto/smartcont/highload-wallet-v2.fif b/crypto/smartcont/highload-wallet-v2.fif index fd71f52aae..e70a5b7184 100755 --- a/crypto/smartcont/highload-wallet-v2.fif +++ b/crypto/smartcont/highload-wallet-v2.fif @@ -12,7 +12,7 @@ false =: force-bounce begin-options " [-n|-b] [-t] []" +cr +tab +"Creates a request with up to 254 orders loaded from to high-load (sub)wallet created by new-highload-v2-wallet.fif, with private key loaded from file .pk " - +"and address from .addr, and saves it into .boc ('wallet-query.boc' by default)" +cr + +"and address from .addr, and saves it into .boc ('wallet-query.boc' by default)" +cr +" is a text file with lines `SEND `" disable-digit-options generic-help-setopt "n" "--no-bounce" { false =: allow-bounce } short-long-option @@ -58,11 +58,11 @@ variable order# order# 0! 0 9 64 32 + + 1+ u, swap append-msg-body b> } : create-int-msg // ng wc addr bnc -- -{ ."Transferring " 3 roll .GR ."to account " +{ ."Transferring " 3 roll .GR ."to account " -rot 2dup 4 pick 7 + .Addr ." = " .addr ." bounce=" . cr } : .transfer // addr$ ng -- c -{ swap parse-smc-addr force-bounce or allow-bounce and // ng wc addr bnc +{ swap parse-smc-addr force-bounce or allow-bounce and // ng wc addr bnc 2over 2over .transfer create-int-msg } : create-simple-transfer diff --git a/crypto/smartcont/highload-wallet.fif b/crypto/smartcont/highload-wallet.fif index 36e321eff5..2e3f4d3f34 100755 --- a/crypto/smartcont/highload-wallet.fif +++ b/crypto/smartcont/highload-wallet.fif @@ -12,7 +12,7 @@ false =: force-bounce begin-options " [-n|-b] [-t] []" +cr +tab +"Creates a request with up to 254 orders loaded from to high-load (sub)wallet created by new-highload-wallet.fif, with private key loaded from file .pk " - +"and address from .addr, and saves it into .boc ('wallet-query.boc' by default)" +cr + +"and address from .addr, and saves it into .boc ('wallet-query.boc' by default)" +cr +" is a text file with lines `SEND `" disable-digit-options generic-help-setopt "n" "--no-bounce" { false =: allow-bounce } short-long-option @@ -59,11 +59,11 @@ variable order# order# 0! 0 9 64 32 + + 1+ u, swap append-msg-body b> } : create-int-msg // ng wc addr bnc -- -{ ."Transferring " 3 roll .GR ."to account " +{ ."Transferring " 3 roll .GR ."to account " -rot 2dup 4 pick 7 + .Addr ." = " .addr ." bounce=" . cr } : .transfer // addr$ ng -- c -{ swap parse-smc-addr force-bounce or allow-bounce and // ng wc addr bnc +{ swap parse-smc-addr force-bounce or allow-bounce and // ng wc addr bnc 2over 2over .transfer create-int-msg } : create-simple-transfer diff --git a/crypto/smartcont/manual-dns-manage.fif b/crypto/smartcont/manual-dns-manage.fif index 98559da948..ce3a6acc35 100644 --- a/crypto/smartcont/manual-dns-manage.fif +++ b/crypto/smartcont/manual-dns-manage.fif @@ -14,7 +14,7 @@ begin-options +cr +" is an operation description, one of" +cr +tab +"add cat (smc | next | adnl | text )" +cr +tab +"delete cat " +cr +tab - +"drop " + +"drop " disable-digit-options generic-help-setopt "t" "--timeout" { parse-int =: timeout } short-long-option-arg "Sets expiration timeout in seconds (" timeout (.) $+ +" by default)" option-help @@ -47,7 +47,7 @@ variable Actions } : parse-cat-num { @end? abort"`cat` expected" @next "cat" $= not abort"`cat` expected" parse-cat-num } : parse-cat -{ @end? abort"smart contract address expected" +{ @end? abort"smart contract address expected" @next false parse-load-address drop triple } : cl-parse-smc-addr { @end? abort"adnl address expected" @@ -96,11 +96,11 @@ file-base +"-dns" +contractid +".addr" load-address dup `smc eq? { drop untriple 2swap drop x{9fd3} s, -rot Addr, 0 8 u, } { dup `next eq? { drop untriple 2swap drop x{ba93} s, -rot Addr, } { dup `adnl eq? { drop second swap x{ad01} s, swap 256 u, 0 8 u, } { - dup `text eq? { drop second swap x{1eda01} s, over $len 8 u, swap $, } { + dup `text eq? { drop second swap x{1eda01} s, over $len 8 u, swap $, } { abort"unknown value type" } cond } cond } cond } cond } : value, -{ subdomain>s dup sbits 3 >> +{ subdomain>s dup sbits 3 >> dup 63 > { drop s>c dict, } { rot swap 7 u, swap s, } cond } : subdomain, // ( A -- b ) diff --git a/crypto/smartcont/mathlib.fc b/crypto/smartcont/mathlib.fc index f2dfd73f57..a38a3701a2 100644 --- a/crypto/smartcont/mathlib.fc +++ b/crypto/smartcont/mathlib.fc @@ -111,7 +111,7 @@ int tanh_f258(int x, int n); ;; fixed257 expm1(fixed257 x); int expm1_f257(int x); -;; computes (sin(x+xe),-cos(x+xe)) for |x| <= Pi/4, xe very small +;; computes (sin(x+xe),-cos(x+xe)) for |x| <= Pi/4, xe very small ;; this function is very accurate, error less than 0.7 ulp (consumes ~ 5500 gas) ;; (fixed256, fixed256) sincosn(fixed256 x, fixed259 xe) (int, int) sincosn_f256(int x, int xe); @@ -194,7 +194,7 @@ int lshift256divr(int x, int y) asm "256 LSHIFT#DIVR"; (int, int) rshiftr4mod(int x) asm "4 RSHIFTR#MOD"; (int, int) rshift3mod(int x) asm "3 RSHIFT#MOD"; -;; computes y - x (FunC compiler does not try to use this by itself) +;; computes y - x (FunC compiler does not try to use this by itself) int sub_rev(int x, int y) asm "SUBR"; int nan() asm "PUSHNAN"; @@ -314,7 +314,7 @@ int tanh_f258(int x, int n) inline_ref { } a = (touch(3) << 254) + muldivr(x2, 1 << 243, a); ;; a := 3+x^2/a as fixed254 ;; y = x/(1+a') = x - x*a'/(1+a') = x - x*x^2/(a+x^2) where a' = x^2/a - return x - (muldivr(x, x2, a + (x2 ~>> 7)) ~>> 7); + return x - (muldivr(x, x2, a + (x2 ~>> 7)) ~>> 7); } ;; fixed257 expm1_f257(fixed257 x) @@ -372,7 +372,7 @@ int tan_f260_inlined(int x) inline { } a = (touch(3) << 254) - muldivr(x2, 1 << 240, a); ;; a := 3-x^2/a as fixed254 ;; y = x/(1-a') = x + x*a'/(1-a') = x + x*x^2/(a-x^2) where a' = x^2/a - return x + (muldivr(x / 2, x2, a - (x2 ~>> 10)) ~>> 9); + return x + (muldivr(x / 2, x2, a - (x2 ~>> 10)) ~>> 9); } ;; fixed260 tan(fixed260 x); @@ -391,7 +391,7 @@ int tan_f258_inlined(int x) inline { } a = (touch(3) << 254) - muldivr(x2, 1 << 244, a); ;; a := 3-x^2/a as fixed254 ;; y = x/(1-a') = x + x*a'/(1-a') = x + x*x^2/(a-x^2) where a' = x^2/a - return x + (muldivr(x / 2, x2, a - (x2 ~>> 6)) ~>> 5); + return x + (muldivr(x / 2, x2, a - (x2 ~>> 6)) ~>> 5); } ;; fixed258 tan(fixed258 x); @@ -414,7 +414,7 @@ int tan_f258(int x) inline_ref { return sincosm1_f259_inlined(x); } -;; computes (sin(x+xe),-cos(x+xe)) for |x| <= Pi/4, xe very small +;; computes (sin(x+xe),-cos(x+xe)) for |x| <= Pi/4, xe very small ;; this function is very accurate, error less than 0.7 ulp (consumes ~ 5500 gas) ;; (fixed256, fixed256) sincosn(fixed256 x, fixed259 xe) (int, int) sincosn_f256(int x, int xe) inline_ref { @@ -430,7 +430,7 @@ int tan_f258(int x) inline_ref { ;; (b, a) = (lshift256divr(b, c), lshift256divr(a, c)); (b, int br) = lshift256divmodr(b, c); br = muldivr(br, 128, c); (a, int ar) = lshift256divmodr(a, c); ar = muldivr(ar, 128, c); - return (sgn(x) * (((mulrshiftr256(b, co) - br) ~/ 16 - mulrshiftr256(a, si)) ~/ 8 - b), + return (sgn(x) * (((mulrshiftr256(b, co) - br) ~/ 16 - mulrshiftr256(a, si)) ~/ 8 - b), a - ((mulrshiftr256(a, co) - ar) ~/ 16 + mulrshiftr256(b, si)) ~/ 8); } @@ -624,7 +624,7 @@ int pow33b(int m) inline { ;; int Log33_32_l = -3769; ;; log(33/32) = Log33_32 / 2^256 + Log33_32_l / 2^269 yh += (yl * 512 + q * -3769) ~>> 13; ;; compensation, may be removed if slightly worse accuracy is acceptable int Log33_32 = 3563114646320977386603103333812068872452913448227778071188132859183498739150; ;; log(33/32) as fixed256 - return (yh + q * Log33_32, s); + return (yh + q * Log33_32, s); } ;; returns (y, s) such that log2(x) = y/2^256 + s for positive integer x @@ -635,7 +635,7 @@ int pow33b(int m) inline { y = lshift256divr(y, log2_const_f256()) ~>> 4; ;; y/log(2) as fixed256 int Log33_32 = 5140487830366106860412008603913034462883915832139695448455767612111363481357; ;; log_2(33/32) as fixed256 ;; Log33_32/2^256 happens to be a very precise approximation to log_2(33/32), no compensation required - return (y + q * Log33_32, s); + return (y + q * Log33_32, s); } ;; functions log_aux_f256() and log2_aux_f256() may be used to implement specific fixed-point instances of log() and log2() @@ -731,7 +731,7 @@ int atan_f261(int x, int n) inline_ref { int t1 = muldivr(xu - tc, 1 << 88, xu * tc + (1 << 48)); ;; tan(x') as fixed64 where x'=atan(x)-13*theta ;; t1/(3+t1^2) * 3073/32 = x'/3 * 3072/32 = x' / (96/3072) = x' / theta int q = muldivr(t1 * 3073, 1 << 59, t1 * t1 + (touch(3) << 128)) + 13; ;; approximately round(atan(x)/theta), 0<=q<=25 - var (pa, pb) = (33226912, 5232641); ;; (32+I)^5 + var (pa, pb) = (33226912, 5232641); ;; (32+I)^5 var (qh, ql) = q /% 5; var (a, b) = (1 << (5 * (51 - q)), 0); ;; (1/32^q, 0) as fixed255 repeat (ql) { ;; a+b*I *= 32+I @@ -857,7 +857,7 @@ int fixed248::atan(int x) inline_ref { } var (q, z) = atan_aux_f256(x); ;; now atan(x) = z + q*atan(1/32) + s*(Pi/2), z is fixed261 - return (z ~/ 64 + s * Pi_const_f254() + muldivr(q, Atan1_32_f261(), 64)) ~/ 128; ;; compute in fixed255, then convert + return (z ~/ 64 + s * Pi_const_f254() + muldivr(q, Atan1_32_f261(), 64)) ~/ 128; ;; compute in fixed255, then convert } ;; fixed248 acot(fixed248 x); @@ -873,7 +873,7 @@ int fixed248::acot(int x) inline_ref { } var (q, z) = atan_aux_f256(x); ;; now acot(x) = - z - q*atan(1/32) + s*(Pi/2), z is fixed261 - return (s * Pi_const_f254() - z ~/ 64 - muldivr(q, Atan1_32_f261(), 64)) ~/ 128; ;; compute in fixed255, then convert + return (s * Pi_const_f254() - z ~/ 64 - muldivr(q, Atan1_32_f261(), 64)) ~/ 128; ;; compute in fixed255, then convert } {--------------------- PSEUDO-RANDOM NUMBERS -------------------} diff --git a/crypto/smartcont/multisig-code.fc b/crypto/smartcont/multisig-code.fc index 243262b67e..8503e4c4e8 100644 --- a/crypto/smartcont/multisig-code.fc +++ b/crypto/smartcont/multisig-code.fc @@ -30,7 +30,7 @@ _ unpack_owner_info(slice cs) inline_ref { (int, int) check_signatures(cell public_keys, cell signatures, int hash, int cnt_bits) inline_ref { int cnt = 0; - + do { slice cs = signatures.begin_parse(); slice signature = cs~load_bits(512); @@ -170,7 +170,7 @@ _ unpack_owner_info(slice cs) inline_ref { set_data(pack_state(pending_queries, owner_infos, last_cleaned, k, n, wallet_id)); commit(); - + int need_save = 0; ifnot (cell_null?(signatures) | (cnt >= k)) { (int new_cnt, cnt_bits) = check_signatures(owner_infos, signatures, hash, cnt_bits); @@ -241,11 +241,11 @@ cell merge_list(cell a, cell b) { cell tail = merge_list(as~load_ref(), b); return begin_cell().store_slice(as).store_ref(tail).end_cell(); } - + as~skip_last_bits(1); ;; as~skip_bits(1); return begin_cell().store_slice(as).store_dict(b).end_cell(); - + } cell get_public_keys() method_id { diff --git a/crypto/smartcont/new-highload-wallet-v2.fif b/crypto/smartcont/new-highload-wallet-v2.fif index d0183b2250..af275ac890 100755 --- a/crypto/smartcont/new-highload-wallet-v2.fif +++ b/crypto/smartcont/new-highload-wallet-v2.fif @@ -4,7 +4,7 @@ { ."usage: " @' $0 type ." []" cr ."Creates a new v2 high-load wallet in the specified workchain, with the controlling private key saved to or loaded from .pk " - ."('new-wallet.pk' by default)" cr + ."('new-wallet.pk' by default)" cr ." is the 32-bit identifier of this subwallet among all controlled by the same private key" cr 1 halt } : usage $# 2- -2 and ' usage if @@ -16,7 +16,7 @@ $2 parse-int dup =: subwallet-id // parse subwallet-id def? $3 { @' $3 } { "new-wallet" } cond constant file-base 65536 constant timeout // init query times out in 65536 seconds -."Creating new v2 high-load wallet in workchain " wc . +."Creating new v2 high-load wallet in workchain " wc . ."with subwallet id " subwallet-id . cr // Create new high-load wallet; source code included from `auto/highload-wallet-v2-code.fif` diff --git a/crypto/smartcont/new-highload-wallet.fif b/crypto/smartcont/new-highload-wallet.fif index 9ead3ff103..525da76a6e 100755 --- a/crypto/smartcont/new-highload-wallet.fif +++ b/crypto/smartcont/new-highload-wallet.fif @@ -4,7 +4,7 @@ { ."usage: " @' $0 type ." []" cr ."Creates a new high-load wallet in the specified workchain, with the controlling private key saved to or loaded from .pk " - ."('new-wallet.pk' by default)" cr + ."('new-wallet.pk' by default)" cr ." is the 32-bit identifier of this subwallet among all controlled by the same private key" cr 1 halt } : usage $# 2- -2 and ' usage if @@ -15,7 +15,7 @@ $2 parse-int dup =: subwallet-id // parse subwallet-id { subwallet-id (.) $+ } : +subwallet def? $3 { @' $3 } { "new-wallet" } cond constant file-base -."Creating new high-load wallet in workchain " wc . +."Creating new high-load wallet in workchain " wc . ."with subwallet id " subwallet-id . cr // Create new high-load wallet; source code included from `auto/highload-wallet-code.fif` @@ -24,7 +24,7 @@ def? $3 { @' $3 } { "new-wallet" } cond constant file-base // data null // no libraries // create StateInit diff --git a/crypto/smartcont/new-restricted-wallet.fif b/crypto/smartcont/new-restricted-wallet.fif index 881f8ad24b..f2db317a3b 100644 --- a/crypto/smartcont/new-restricted-wallet.fif +++ b/crypto/smartcont/new-restricted-wallet.fif @@ -25,7 +25,7 @@ dup hashu wc swap 2dup 2constant wallet_addr 2dup file-base +".addr" save-address-verbose ."Non-bounceable address (for init): " 2dup 7 .Addr cr ."Bounceable address (for later access): " 6 .Addr cr - dup ."External message for initialization is " B dup Bx. cr diff --git a/crypto/smartcont/new-restricted-wallet2.fif b/crypto/smartcont/new-restricted-wallet2.fif index 90f544953d..034cbb8d2e 100644 --- a/crypto/smartcont/new-restricted-wallet2.fif +++ b/crypto/smartcont/new-restricted-wallet2.fif @@ -39,7 +39,7 @@ dup hashu wc swap 2dup 2constant wallet_addr 2dup file-base +".addr" save-address-verbose ."Non-bounceable address (for init): " 2dup 7 .Addr cr ."Bounceable address (for later access): " 6 .Addr cr - dup ."External message for initialization is " B dup Bx. cr diff --git a/crypto/smartcont/new-restricted-wallet3.fif b/crypto/smartcont/new-restricted-wallet3.fif index a53f35dffe..513ef4317a 100644 --- a/crypto/smartcont/new-restricted-wallet3.fif +++ b/crypto/smartcont/new-restricted-wallet3.fif @@ -27,11 +27,11 @@ begin-options "Expiration time of the initialization message (" expires-in (.) $+ +" seconds by default)" option-help "t" "--start-at" { dup "now" $= { drop now } { parse-int } cond =: start-at } short-long-option-arg - "Restriction start Unixtime (`now` by default)" option-help + "Restriction start Unixtime (`now` by default)" option-help "h" "--help" { usage } short-long-option "Shows a help message" option-help parse-options - + $# dup 3 < swap 4 > or ' usage if 4 :$1..n @@ -52,7 +52,7 @@ start-at 32 ufits not abort"invalid restriction start time" restrict-mode dup 0 < swap 2 > or abort"unknown restriction mode" filename-base +".pk" load-keypair =: init_pk =: init_pubkey -."Creating new restricted lockup wallet v3 in workchain " wc . +."Creating new restricted lockup wallet v3 in workchain " wc . ."with restriction mode " restrict-mode . ."and nominal amount " amount .GR cr ."controlled by public key " PubKey .pubkey ." and initialized by public key " init_pubkey 256 B>u@ .pubkey cr ."(subwallet id is " subwallet-id ._ .")" cr @@ -85,8 +85,8 @@ filename-base +".pk" load-keypair =: init_pk =: init_pubkey amount restrict-mode ?dup { - 1 = { make-rdict1 } { make-rdict2 } cond -} { make-rdict0 } cond + 1 = { make-rdict1 } { make-rdict2 } cond +} { make-rdict0 } cond =: rdict ."Restrictions start at " start-at ._ .": " cr @@ -108,7 +108,7 @@ dup =: StateInit hashu wc swap 2dup 2constant wallet_addr dup ."signing message: " dup ."External message for initialization is " B dup Bx. cr diff --git a/crypto/smartcont/new-wallet-v2.fif b/crypto/smartcont/new-wallet-v2.fif index 0bb2bcd19c..c7d9c86205 100755 --- a/crypto/smartcont/new-wallet-v2.fif +++ b/crypto/smartcont/new-wallet-v2.fif @@ -39,10 +39,10 @@ def? $2 { @' $2 } { "new-wallet" } cond constant file-base NEWC 32 STU 256 STU ENDC c4 POP }>c // >libref // code - // data null // no libraries // create StateInit diff --git a/crypto/smartcont/new-wallet-v3.fif b/crypto/smartcont/new-wallet-v3.fif index 3b31b35c6b..4821d27038 100644 --- a/crypto/smartcont/new-wallet-v3.fif +++ b/crypto/smartcont/new-wallet-v3.fif @@ -23,7 +23,7 @@ $3 "new-wallet" replace-if-null =: file-base // data null // no libraries // create StateInit diff --git a/crypto/smartcont/new-wallet.fif b/crypto/smartcont/new-wallet.fif index b1aead9000..194fdc58d7 100755 --- a/crypto/smartcont/new-wallet.fif +++ b/crypto/smartcont/new-wallet.fif @@ -28,7 +28,7 @@ def? $2 { @' $2 } { "new-wallet" } cond constant file-base CHKSIGNU // pubk cs cnt ? 34 THROWIFNOT // signature mismatch ACCEPT - SWAP 32 LDU NIP + SWAP 32 LDU NIP DUP SREFS IF:<{ // 3 INT 35 LSHIFT# 3 INT RAWRESERVE // reserve all but 103 Grams from the balance 8 LDU LDREF // pubk cnt mode msg cs @@ -38,10 +38,10 @@ def? $2 { @' $2 } { "new-wallet" } cond constant file-base INC NEWC 32 STU 256 STU ENDC c4 POPCTR }>c // >libref // code - // data null // no libraries // Libs{ x{ABACABADABACABA} drop x{AAAA} s>c public_lib x{1234} x{5678} |_ s>c public_lib }Libs diff --git a/crypto/smartcont/payment-channel-code.fc b/crypto/smartcont/payment-channel-code.fc index b4b14d5cd5..727d4ab05e 100644 --- a/crypto/smartcont/payment-channel-code.fc +++ b/crypto/smartcont/payment-channel-code.fc @@ -27,7 +27,7 @@ int state:payout() asm "2 PUSHINT"; int min_fee() asm "1000000000 PUSHINT"; -;; A - initial balance of Alice, +;; A - initial balance of Alice, ;; B - initial balance of B ;; ;; To determine balance we track nondecreasing list of promises @@ -37,7 +37,7 @@ int min_fee() asm "1000000000 PUSHINT"; ;; diff - balance between Alice and Bob. 0 in the beginning ;; diff = promise_B - promise_A; ;; diff = clamp(diff, -A, +B); -;; +;; ;; final_A = A + diff; ;; final_B = B + diff; @@ -61,11 +61,11 @@ _ pack_data(cell config, cell state) impure inline_ref { _ unpack_config(cell config) { var cs = config.begin_parse(); var res = ( - cs~load_uint(32), - cs~load_uint(32), - cs~load_uint(256), - cs~load_uint(256), - cs~load_ref().begin_parse(), + cs~load_uint(32), + cs~load_uint(32), + cs~load_uint(256), + cs~load_uint(256), + cs~load_ref().begin_parse(), cs~load_ref().begin_parse(), cs~load_uint(64), cs~load_grams()); @@ -102,13 +102,13 @@ _ unpack_config(cell config) { ;; stateInit signed_A?:Bool signed_B?:Bool min_A:Grams min_B:Grams expire_at:uint32 A:Grams B:Grams = State; _ unpack_state_init(slice state) { return ( - state~load_int(1), - state~load_int(1), - state~load_grams(), - state~load_grams(), - state~load_uint(32), - state~load_grams(), - state~load_grams()); + state~load_int(1), + state~load_int(1), + state~load_grams(), + state~load_grams(), + state~load_uint(32), + state~load_grams(), + state~load_grams()); } _ pack_state_init(int signed_A?, int signed_B?, int min_A, int min_B, int expire_at, int A, int B) { @@ -126,13 +126,13 @@ _ pack_state_init(int signed_A?, int signed_B?, int min_A, int min_B, int expire ;; stateClosing$10 signed_A?:bool signed_B?:Bool promise_A:Grams promise_B:Grams exipire_at:uint32 A:Grams B:Grams = State; _ unpack_state_close(slice state) { return ( - state~load_int(1), - state~load_int(1), - state~load_grams(), - state~load_grams(), - state~load_uint(32), - state~load_grams(), - state~load_grams()); + state~load_int(1), + state~load_int(1), + state~load_grams(), + state~load_grams(), + state~load_uint(32), + state~load_grams(), + state~load_grams()); } _ pack_state_close(int signed_A?, int signed_B?, int promise_A, int promise_B, int expire_at, int A, int B) { @@ -186,7 +186,7 @@ cell do_payout(int promise_A, int promise_B, int A, int B, slice a_addr, slice b ;; ;; init$000 inc_A:Grams inc_B:Grams min_A:Grams min_B:Grams = Message; ;; -cell with_init(slice state, int msg_value, slice msg, int msg_signed_A?, int msg_signed_B?, +cell with_init(slice state, int msg_value, slice msg, int msg_signed_A?, int msg_signed_B?, slice a_addr, slice b_addr, int init_timeout, int channel_id, int min_A_extra) { ;; parse state (int signed_A?, int signed_B?, int min_A, int min_B, int expire_at, int A, int B) = unpack_state_init(state); @@ -203,14 +203,14 @@ cell with_init(slice state, int msg_value, slice msg, int msg_signed_A?, int msg throw_unless(err:expected_init(), op == msg:init()); ;; unpack init message - (int inc_A, int inc_B, int upd_min_A, int upd_min_B, int got_channel_id) = + (int inc_A, int inc_B, int upd_min_A, int upd_min_B, int got_channel_id) = (msg~load_grams(), msg~load_grams(), msg~load_grams(), msg~load_grams(), msg~load_uint(64)); throw_unless(err:wrong_channel_id(), got_channel_id == channel_id); ;; TODO: we should reserve some part of the value for comission throw_if(err:msg_value_too_small(), msg_value < inc_A + inc_B); throw_unless(err:replay_protection(), (msg_signed_A? < signed_A?) | (msg_signed_B? < signed_B?)); - + A += inc_A; B += inc_B; @@ -223,14 +223,14 @@ cell with_init(slice state, int msg_value, slice msg, int msg_signed_A?, int msg if (min_B < upd_min_B) { min_B = upd_min_B; } - + if (signed_A? & signed_B?) { A -= min_A_extra; if ((min_A > A) | (min_B > B)) { return do_payout(0, 0, A, B, a_addr, b_addr, channel_id); } - return pack_state_close(0, 0, 0, 0, 0, A, B); + return pack_state_close(0, 0, 0, 0, 0, A, B); } return pack_state_init(signed_A?, signed_B?, min_A, min_B, expire_at, A, B); @@ -238,7 +238,7 @@ cell with_init(slice state, int msg_value, slice msg, int msg_signed_A?, int msg ;; close$001 extra_A:Grams extra_B:Grams sig:Maybe promise_A:Grams promise_B:Grams = Message; -cell with_close(slice cs, slice msg, int msg_signed_A?, int msg_signed_B?, int a_key, int b_key, +cell with_close(slice cs, slice msg, int msg_signed_A?, int msg_signed_B?, int a_key, int b_key, slice a_addr, slice b_addr, int expire_timeout, int channel_id) { ;; parse state (int signed_A?, int signed_B?, int promise_A, int promise_B, int expire_at, int A, int B) = unpack_state_close(cs); @@ -276,19 +276,19 @@ cell with_close(slice cs, slice msg, int msg_signed_A?, int msg_signed_B?, int a } else { throw_unless(err:no_promise_signature(), msg_signed_A? & msg_signed_B?); extra_A = 0; - extra_B = 0; + extra_B = 0; } (int got_channel_id, int update_promise_A, int update_promise_B) = (msg~load_uint(64), msg~load_grams(), msg~load_grams()); throw_unless(err:wrong_channel_id(), got_channel_id == channel_id); - + accept_message(); update_promise_A += extra_A; - if (promise_A < update_promise_A) { + if (promise_A < update_promise_A) { promise_A = update_promise_A; } update_promise_B += extra_B; - if (promise_B < update_promise_B) { + if (promise_B < update_promise_B) { promise_B = update_promise_B; } @@ -323,7 +323,7 @@ cell with_close(slice cs, slice msg, int msg_signed_A?, int msg_signed_B?, int a throw_unless(err:unknown_op(), op == op:pchan_cmd()); (cell config, cell state) = unpack_data(); - (int init_timeout, int close_timeout, int a_key, int b_key, + (int init_timeout, int close_timeout, int a_key, int b_key, slice a_addr, slice b_addr, int channel_id, int min_A_extra) = config.unpack_config(); (int msg_signed_A?, int msg_signed_B?) = msg~unwrap_signatures(a_key, b_key); @@ -331,7 +331,7 @@ cell with_close(slice cs, slice msg, int msg_signed_A?, int msg_signed_B?, int a int state_type = cs~load_uint(3); if (state_type == state:init()) { ;; init - state = with_init(cs, msg_value, msg, msg_signed_A?, msg_signed_B?, a_addr, b_addr, init_timeout, channel_id, min_A_extra); + state = with_init(cs, msg_value, msg, msg_signed_A?, msg_signed_B?, a_addr, b_addr, init_timeout, channel_id, min_A_extra); } if (state_type == state:close()) { state = with_close(cs, msg, msg_signed_A?, msg_signed_B?, a_key, b_key, a_addr, b_addr, close_timeout, channel_id); } if (state_type == state:payout()) { diff --git a/crypto/smartcont/show-addr.fif b/crypto/smartcont/show-addr.fif index c421a8790d..daf4865193 100755 --- a/crypto/smartcont/show-addr.fif +++ b/crypto/smartcont/show-addr.fif @@ -9,7 +9,7 @@ $# 1 > ' usage if 1 :$1..n $1 "new-wallet" replace-if-null =: file-base -file-base +".addr" dup ."Loading wallet address from " type cr file>B 32 B| +file-base +".addr" dup ."Loading wallet address from " type cr file>B 32 B| dup Blen { 32 B>i@ } { drop Basechain } cond constant wallet_wc 256 B>u@ dup constant wallet_addr ."Source wallet address = " wallet_wc swap 2dup .addr cr @@ -19,6 +19,6 @@ dup Blen { 32 B>i@ } { drop Basechain } cond constant wallet_wc file-base +".pk" dup file-exists? { dup file>B dup Blen 32 <> abort"Private key must be exactly 32 bytes long" tuck =: wallet_pk ."Private key available in file " type cr - priv>pub 256 B>u@ + priv>pub 256 B>u@ dup ."Corresponding public key is " .pubkey ." = " 64X. cr } { ."Private key file " type ." not found" cr } cond diff --git a/crypto/smartcont/testgiver.fif b/crypto/smartcont/testgiver.fif index c48735aca1..7e08f9884b 100755 --- a/crypto/smartcont/testgiver.fif +++ b/crypto/smartcont/testgiver.fif @@ -8,7 +8,7 @@ $# 3 - -2 and ' usage if -// "testgiver.addr" load-address +// "testgiver.addr" load-address Masterchain 0xfcb91a3a3816d0f7b8c2c76108b8a9bc5a6b7a55bd79f8ab101c52db29232260 2constant giver_addr ."Test giver address = " giver_addr 2dup .addr cr 6 .Addr cr @@ -23,7 +23,7 @@ dest_addr 2dup bounce 7 + .Addr ." = " .addr ."seqno=0x" seqno x. ."bounce=" bounce . cr // create a message (NB: 01b00.., b = bounce) - dup ."enveloping message: " (self): T fun tuple.pop(mutate self): T asm "TPOP" -/// Packs any object from a stack to a tuple. +/// Packs any object from a stack to a tuple. /// An object occupies N slots on a stack — the tuple will be of size N. /// Example: /// ``` /// struct Point { x: int, y: int } -/// +/// /// var p: Point = { x: 1, y: 2 }; /// var t = p.toTuple(); // [ 1 2 ] /// p = Point.fromTuple(t); // back @@ -328,7 +328,7 @@ fun contract.getOriginalBalanceWithExtraCurrencies(): [coins, ExtraCurrenciesMap fun contract.getData(): cell asm "c4 PUSH" -/// Sets the persistent contract storage. +/// Sets the persistent contract storage. fun contract.setData(c: cell): void asm "c4 POP" @@ -620,7 +620,7 @@ fun slice.hash(self): uint256 fun builder.hash(self): uint256 asm "HASHBU" -/// Computes sha256 of the data bits of a slice (without refs). +/// Computes sha256 of the data bits of a slice (without refs). /// If the bit length is not divisible by eight, throws a cell underflow exception. @pure fun slice.bitsHash(self): uint256 @@ -731,7 +731,7 @@ fun builder.depth(self): int /// Returns the number of stack slots `anyVariable` occupies (works at compile-time). /// - sizeof(nullableInt) = 1, because `int?` is 1 TVM slot holding either NULL or a value. /// - sizeof(somePoint) = 2 for `struct Point { x:int, y:int }`: two fields one slot per each. -/// +/// /// Useful for debugging or when preparing stack contents for RUNVM. @pure fun sizeof(anyVariable: T): int @@ -763,7 +763,7 @@ fun debug.dumpStack(): void /* Implementation of `map` and methods over it. - Note: maps in TVM are called "dictionaries". They are stored as nested cells + Note: maps in TVM are called "dictionaries". They are stored as nested cells (each value in a separate cell, probably with intermediate cells-nodes). Constructing even small dicts is gas-expensive, because cell creation costs a lot. @@ -794,7 +794,7 @@ fun map.toLowLevelDict(self): dict asm "NOP" /// Checks whether a map is empty (whether a cell is null). -/// +/// /// Note: a check `m == null` will not work, use `m.isEmpty()`. @pure fun map.isEmpty(self): bool @@ -861,7 +861,7 @@ fun map.addIfNotExists(mutate self, key: K, value: V): bool fun map.addOrGetExisting(mutate self, key: K, value: V): MapLookupResult builtin -/// Delete an element at the key. Returns whether an element was deleted. +/// Delete an element at the key. Returns whether an element was deleted. @pure fun map.delete(mutate self, key: K): bool builtin @@ -1322,7 +1322,7 @@ fun address.getWorkchain(self): int8 /// Creates a slice representing "none external address" (TL addr_none$00 — two zero bits). /// Note! Use this in combination with EXTERNAL addresses (e.g. for [createExternalLogMessage]). -/// To represent "internal or none" address (which you most likely need to), +/// To represent "internal or none" address (which you most likely need to), /// use `address?` (nullable) and `null` accordingly. @pure fun createAddressNone(): any_address @@ -1335,7 +1335,7 @@ fun any_address.isNone(self): bool asm "b{00} SDBEGINSQ" "NIP" /// Returns if "any address" is a standard (internal) address: workchain + hash. -/// Then it can be safely cast to `address`. +/// Then it can be safely cast to `address`. /// In TL/B, it's addr_std$10. @pure fun any_address.isInternal(self): bool @@ -1453,7 +1453,7 @@ const SEND_MODE_ESTIMATE_FEE_ONLY = 1024 /// `ExtraCurrenciesMap` represents a dictionary of "extra currencies" other than TON coin. /// They can be attached to a message (incoming and outgoing) and stored on contract's balance. -/// It's a dictionary `[currencyID => amount]` (amount is like `coins`, but encoded with a high precision). +/// It's a dictionary `[currencyID => amount]` (amount is like `coins`, but encoded with a high precision). type ExtraCurrenciesMap = map /// `ContractState` is "code + data" of a contract. @@ -1485,7 +1485,7 @@ struct AddressShardingOptions { /// ``` /// You just provide code+data, and the compiler automatically calculates the destination address, /// because in TON, the address of a contract, by definition, is a hash of its initial state. -/// You can also use it without sending a message. +/// You can also use it without sending a message. /// See [AutoDeployAddress.calculateAddress]. struct AutoDeployAddress { workchain: int8 = BASECHAIN @@ -1533,10 +1533,10 @@ fun AutoDeployAddress.addressMatches(self, addr: address): bool /// bounce: BounceMode.Only256BitsOfBody, /// ... /// }); -/// +/// /// fun onBouncedMessage(in: InMessageBounced) { /// // in.bouncedBody is populated depending on BounceMode -/// // for instance, "Only256BitsOfBody" means +/// // for instance, "Only256BitsOfBody" means /// // "bounced prefix 0xFFFFFFFF" and 256 bits of data /// in.bouncedBody.skipBouncedPrefix(); /// val originalOpcode = in.bouncedBody.loadUint(32); @@ -1596,7 +1596,7 @@ fun createMessage(options: CreateMessageOptions): OutMessage /// It includes some "topic" (arbitrary number), that determines the format of the message body. /// For example, you emit "deposit event" (reserving topic "deposit" = 123): /// ``` -/// dest: ExtOutLogBucket { topic: 123 }, +/// dest: ExtOutLogBucket { topic: 123 }, /// body: DepositData { ... } /// ``` /// and external indexers can index your emitted logs by destination address without parsing body. @@ -1752,7 +1752,7 @@ fun address.fromValidBuilder(b: builder): address /// dest: (workchain, hash), // a pair can be passed instead of `address` /// ... // so, don't call this function in such a case /// ``` -fun address.fromWorkchainAndHash(workchain: int8, hash: uint256): address +fun address.fromWorkchainAndHash(workchain: int8, hash: uint256): address asm (hash workchain) "NEWC" "b{100} STSLICECONST" // addr_std$10 + 0 (no anycast) @@ -1796,7 +1796,7 @@ struct InMessage { /// ``` /// fun onBouncedMessage(in: InMessageBounced) { /// // in.bouncedBody is populated depending on BounceMode -/// // for instance, "Only256BitsOfBody" means +/// // for instance, "Only256BitsOfBody" means /// // "bounced prefix 0xFFFFFFFF" and 256 bits of data /// in.bouncedBody.skipBouncedPrefix(); /// val originalOpcode = in.bouncedBody.loadUint(32); @@ -1829,7 +1829,7 @@ struct (0xfffffffe) RichBounceBody { computePhase: RichBounceComputePhaseInfo? // filled if compute phase executed } -/// Describes compute phase details of a bounced message. +/// Describes compute phase details of a bounced message. struct RichBounceComputePhaseInfo { gasUsed: uint32 vmSteps: uint32 diff --git a/crypto/smartcont/tolk-stdlib/exotic-cells.tolk b/crypto/smartcont/tolk-stdlib/exotic-cells.tolk index ef7b73c7fa..e277bd7023 100644 --- a/crypto/smartcont/tolk-stdlib/exotic-cells.tolk +++ b/crypto/smartcont/tolk-stdlib/exotic-cells.tolk @@ -9,8 +9,8 @@ enum ExoticCellType: int8 { MerkleUpdate = 4 } -/// Transforms an ordinary or exotic cell into a slice, as if it were an ordinary cell. -/// A flag is returned indicating whether c is exotic. +/// Transforms an ordinary or exotic cell into a slice, as if it were an ordinary cell. +/// A flag is returned indicating whether c is exotic. /// If so, ExoticCellType can later be deserialized from the first eight bits of s. /// ``` /// var (s, isExotic) = c.beginParseSpecial(); diff --git a/crypto/smartcont/validator-elect-signed.fif b/crypto/smartcont/validator-elect-signed.fif index 4e681f07dc..d911674b1e 100755 --- a/crypto/smartcont/validator-elect-signed.fif +++ b/crypto/smartcont/validator-elect-signed.fif @@ -36,7 +36,7 @@ to_sign signature pubkey ed25519_chksign not abort"Ed25519 signature is invalid" ."Provided a valid Ed25519 signature " signature Bx. ." with validator public key " pubkey Bx. cr now dup constant query_id ."query_id set to " . cr - ref, b> cr ."Message body is " dup B B>boc } { comment simple-transfer-b constant body-cell ."Transferring " amount .GR+cc ."to account " -dest_addr 2dup bounce 7 + .Addr ." = " .addr +dest_addr 2dup bounce 7 + .Addr ." = " .addr ."seqno=0x" seqno x. ."bounce=" bounce . cr ."Body of transfer message is " body-cell B B>boc B B>boc " cr +{ ."usage: " $0 type ." " cr ."Computes the product of two integers." cr bye } : usage { ' usage if } : ?usage $# 2 <> ?usage diff --git a/crypto/test/fift/fixed.fif b/crypto/test/fift/fixed.fif index 779920cc54..a27a7f593c 100644 --- a/crypto/test/fift/fixed.fif +++ b/crypto/test/fift/fixed.fif @@ -1,5 +1,5 @@ 1 { 10 * } 70 times constant One -{ bl word (number) dup 0= abort"invalid fixed-point constant" +{ bl word (number) dup 0= abort"invalid fixed-point constant" 1- { One swap */r } { One * } cond 1 'nop } ::_ F$ ' + : F+ ' - : F- diff --git a/crypto/test/fift/sort.fif b/crypto/test/fift/sort.fif index ac66cd60b8..5d88f3a776 100644 --- a/crypto/test/fift/sort.fif +++ b/crypto/test/fift/sort.fif @@ -2,7 +2,7 @@ { dup null? not } { uncons swap rot cons -rot } while drop } : split -variable 'sort variable 'merge variable 'compare +variable 'sort variable 'merge variable 'compare { 'sort @ execute } : sort { 'merge @ execute } : merge { 'compare @ execute } : compare diff --git a/crypto/test/fift/testdict.fif b/crypto/test/fift/testdict.fif index 28d711ade7..d8e5ce9d19 100644 --- a/crypto/test/fift/testdict.fif +++ b/crypto/test/fift/testdict.fif @@ -20,10 +20,10 @@ ( _( 13 169 ) _( 17 289 ) _( 10 100 ) ) mkdict =: Dict _( 4 16 ) _( 9 81 ) Dict +dictpair +dictpair =: Dict1 _( 4 20 ) _( 101 10201 ) Dict +dictpair +dictpair =: Dict2 -."Dict1 = " Dict1 show-dict +."Dict1 = " Dict1 show-dict Dict1 dict>list .l cr Dict1 c 13 x{218010F40C} runvmcode .s drop swap csr. diff --git a/crypto/test/fift/testvm4.fif b/crypto/test/fift/testvm4.fif index f6cd5a2ef4..976d08923e 100644 --- a/crypto/test/fift/testvm4.fif +++ b/crypto/test/fift/testvm4.fif @@ -1,6 +1,6 @@ -169 13 +169 13 239 dup dup * swap -289 17 +289 17 3 x{6D01E502C8CB0F028010F442} runvmcode .s drop dup 16 dictionary" } : xdict(16,16) { xdict(16,16) ."{ " { swap . ."-> " . ."; " } swap times ."}" cr } : .dict(16,16) { xdict(16,16) { dup 0< not { ."+" } if ._ 64 + emit } swap times space } : .v -169 13 +169 13 239 dup dup * swap -289 17 +289 17 3 mkdict(16,16) constant dict diff --git a/crypto/test/fift/testvm4e.fif b/crypto/test/fift/testvm4e.fif index 323c15d431..1e6c267edd 100644 --- a/crypto/test/fift/testvm4e.fif +++ b/crypto/test/fift/testvm4e.fif @@ -31,7 +31,7 @@ D <{ 0 INT 0 INT ROT 16 INT SUBDICTUGET }>s dup csr. runvmcode drop dup dict>s csr. .dict(16,16) ." <-- SubDict(D,_)" cr D <{ 0x1 INT 4 INT ROT 16 INT SUBDICTUGET }>s runvmcode drop -dup dict>s csr. .dict(16,16) ." <-- SubDict(D,0001)" cr +dup dict>s csr. .dict(16,16) ." <-- SubDict(D,0001)" cr D <{ 0x10 INT 8 INT ROT 16 INT SUBDICTUGET }>s runvmcode drop dup dict>s csr. .dict(16,16) ." <-- SubDict(D,00010000)" cr diff --git a/crypto/test/fift/testvm5.fif b/crypto/test/fift/testvm5.fif index 9578fc55f8..496f7d757b 100644 --- a/crypto/test/fift/testvm5.fif +++ b/crypto/test/fift/testvm5.fif @@ -3,9 +3,9 @@ <{ 2SWAP MUL s2 POP FALSE }>s b{1} <{ 2SWAP SUB s2 POP FALSE }>s b{01} 3 -<{ NEWDICT SWAP REPEAT: 8 INT PFXDICTADD 11 THROWIFNOT }>s runvmcode +<{ NEWDICT SWAP REPEAT: 8 INT PFXDICTADD 11 THROWIFNOT }>s runvmcode abort"cannot create prefix code dictionary" -dup dict>s csr. +dup dict>s csr. <{ <{ rot 8 PFXDICTSWITCH TRUE }>UNTIL SDEMPTY 11 THROWIFNOT }>s dup csr. 1 2 3 4 5 b{001011} 6 roll runvmcode .s diff --git a/crypto/test/fift/testvm7.fif b/crypto/test/fift/testvm7.fif index 796c771ca6..c5c32754dd 100644 --- a/crypto/test/fift/testvm7.fif +++ b/crypto/test/fift/testvm7.fif @@ -5,7 +5,7 @@ 3 3 BLKPUSH // -ROT OVER ADD ROT OVER ADD -ROT OVER ADD ROT s2 s1 PUXC ADD TUCK ADD s2 s1 PUXC ADD SWAP - DUP ISPOS + DUP ISPOS }>DO<{ s4 POP s4 POP s4 POP INC }> @@ -33,7 +33,7 @@ drop }> DUP 250 QFITS ISNAN }>UNTIL - 2DROP s3 POP s3 POP DROP + 2DROP s3 POP s3 POP DROP }>s dup csr. 1 { 10 * } 75 times constant One { One swap */r } : frac*One diff --git a/crypto/tl/boc.tlb b/crypto/tl/boc.tlb index d283c78c10..88810390cb 100644 --- a/crypto/tl/boc.tlb +++ b/crypto/tl/boc.tlb @@ -3,8 +3,8 @@ // serialized_boc_idx#68ff65f3 size:(## 8) { size <= 4 } - off_bytes:(## 8) { off_bytes <= 8 } - cells:(##(size * 8)) + off_bytes:(## 8) { off_bytes <= 8 } + cells:(##(size * 8)) roots:(##(size * 8)) { roots = 1 } absent:(##(size * 8)) { roots + absent <= cells } tot_cells_size:(##(off_bytes * 8)) @@ -13,8 +13,8 @@ serialized_boc_idx#68ff65f3 size:(## 8) { size <= 4 } = BagOfCells; serialized_boc_idx_crc32c#acc3a728 size:(## 8) { size <= 4 } - off_bytes:(## 8) { off_bytes <= 8 } - cells:(##(size * 8)) + off_bytes:(## 8) { off_bytes <= 8 } + cells:(##(size * 8)) roots:(##(size * 8)) { roots = 1 } absent:(##(size * 8)) { roots + absent <= cells } tot_cells_size:(##(off_bytes * 8)) @@ -22,11 +22,11 @@ serialized_boc_idx_crc32c#acc3a728 size:(## 8) { size <= 4 } cell_data:(tot_cells_size * [ uint8 ]) crc32c:uint32 = BagOfCells; -serialized_boc#b5ee9c72 has_idx:(## 1) has_crc32c:(## 1) +serialized_boc#b5ee9c72 has_idx:(## 1) has_crc32c:(## 1) has_cache_bits:(## 1) flags:(## 2) { flags = 0 } size:(## 3) { size <= 4 } - off_bytes:(## 8) { off_bytes <= 8 } - cells:(##(size * 8)) + off_bytes:(## 8) { off_bytes <= 8 } + cells:(##(size * 8)) roots:(##(size * 8)) { roots >= 1 } absent:(##(size * 8)) { roots + absent <= cells } tot_cells_size:(##(off_bytes * 8)) @@ -43,4 +43,4 @@ compiled_smart_contract compiler_version:(Maybe ^TinyString) ] = CompiledSmartContract; -tiny_string#_ len:(#<= 126) str:(len * [ uint8 ]) = TinyString; +tiny_string#_ len:(#<= 126) str:(len * [ uint8 ]) = TinyString; diff --git a/crypto/tl/hashmap.tlb b/crypto/tl/hashmap.tlb index 0ec55a6596..58f64644db 100644 --- a/crypto/tl/hashmap.tlb +++ b/crypto/tl/hashmap.tlb @@ -2,11 +2,11 @@ bit#_ _:(## 1) = Bit; // ordinary Hashmap / HashmapE, with fixed length keys // -hm_edge#_ {n:#} {X:Type} {l:#} {m:#} label:(HmLabel ~l n) +hm_edge#_ {n:#} {X:Type} {l:#} {m:#} label:(HmLabel ~l n) {n = (~m) + l} node:(HashmapNode m X) = Hashmap n X; hmn_leaf#_ {X:Type} value:X = HashmapNode 0 X; -hmn_fork#_ {n:#} {X:Type} left:^(Hashmap n X) +hmn_fork#_ {n:#} {X:Type} left:^(Hashmap n X) right:^(Hashmap n X) = HashmapNode (n + 1) X; hml_short$0 {m:#} {n:#} len:(Unary ~n) s:(n * Bit) = HmLabel ~n m; @@ -25,21 +25,21 @@ _ {n:#} _:(Hashmap n True) = BitstringSet n; // VarHashmap / VarHashmapE, with variable-length keys // -vhm_edge#_ {n:#} {X:Type} {l:#} {m:#} label:(HmLabel ~l n) - {n = (~m) + l} node:(VarHashmapNode m X) +vhm_edge#_ {n:#} {X:Type} {l:#} {m:#} label:(HmLabel ~l n) + {n = (~m) + l} node:(VarHashmapNode m X) = VarHashmap n X; vhmn_leaf$00 {n:#} {X:Type} value:X = VarHashmapNode n X; -vhmn_fork$01 {n:#} {X:Type} left:^(VarHashmap n X) - right:^(VarHashmap n X) value:(Maybe X) +vhmn_fork$01 {n:#} {X:Type} left:^(VarHashmap n X) + right:^(VarHashmap n X) value:(Maybe X) = VarHashmapNode (n + 1) X; -vhmn_cont$1 {n:#} {X:Type} branch:Bit child:^(VarHashmap n X) +vhmn_cont$1 {n:#} {X:Type} branch:Bit child:^(VarHashmap n X) value:X = VarHashmapNode (n + 1) X; nothing$0 {X:Type} = Maybe X; just$1 {X:Type} value:X = Maybe X; vhme_empty$0 {n:#} {X:Type} = VarHashmapE n X; -vhme_root$1 {n:#} {X:Type} root:^(VarHashmap n X) +vhme_root$1 {n:#} {X:Type} root:^(VarHashmap n X) = VarHashmapE n X; // @@ -47,14 +47,14 @@ vhme_root$1 {n:#} {X:Type} root:^(VarHashmap n X) // constituting a prefix code // -phm_edge#_ {n:#} {X:Type} {l:#} {m:#} label:(HmLabel ~l n) - {n = (~m) + l} node:(PfxHashmapNode m X) +phm_edge#_ {n:#} {X:Type} {l:#} {m:#} label:(HmLabel ~l n) + {n = (~m) + l} node:(PfxHashmapNode m X) = PfxHashmap n X; phmn_leaf$0 {n:#} {X:Type} value:X = PfxHashmapNode n X; -phmn_fork$1 {n:#} {X:Type} left:^(PfxHashmap n X) +phmn_fork$1 {n:#} {X:Type} left:^(PfxHashmap n X) right:^(PfxHashmap n X) = PfxHashmapNode (n + 1) X; phme_empty$0 {n:#} {X:Type} = PfxHashmapE n X; -phme_root$1 {n:#} {X:Type} root:^(PfxHashmap n X) +phme_root$1 {n:#} {X:Type} root:^(PfxHashmap n X) = PfxHashmapE n X; diff --git a/crypto/tl/tlbc-gen-cpp.cpp b/crypto/tl/tlbc-gen-cpp.cpp index f7b5067365..75cd79e3ea 100644 --- a/crypto/tl/tlbc-gen-cpp.cpp +++ b/crypto/tl/tlbc-gen-cpp.cpp @@ -24,9 +24,9 @@ namespace tlbc { /* - * + * * C++ CODE GENERATION - * + * */ CppIdentSet global_cpp_ids; diff --git a/crypto/tl/tlbc.cpp b/crypto/tl/tlbc.cpp index 294f7deaf5..d979ea7fa4 100644 --- a/crypto/tl/tlbc.cpp +++ b/crypto/tl/tlbc.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP @@ -60,9 +60,9 @@ int verbosity; namespace src { /* - * + * * KEYWORD DEFINITION - * + * */ enum { _Eof = -1, _Ident = 0, _Number, _Special, _Eq = 0x80, _Leq, _Geq, _Neq, _Type, _EMPTY }; @@ -223,9 +223,9 @@ namespace tlbc { td::LinearAllocator AR(1 << 22); /* - * + * * AUXILIARY DATA TYPES - * + * */ // headers are in tlbc-aux.h @@ -847,9 +847,9 @@ using src::Lexer; using sym::sym_idx_t; /* - * + * * DATA TYPES: Type Expressions, Types, Constructors - * + * */ // headers in tlbc-data.h @@ -1568,8 +1568,8 @@ bool TypeExpr::bind_value(bool value_negated, Constructor& cs, bool checking_typ // if checking_type == true: // value_negated must be false /* (debug output) - std::cerr << "binding " << (value_negated ? "negative" : "positive") << " value to expression " << (checking_type ? "of type " : ""); - show(std::cerr, &cs); + std::cerr << "binding " << (value_negated ? "negative" : "positive") << " value to expression " << (checking_type ? "of type " : ""); + show(std::cerr, &cs); std::cerr << std::endl; */ if (!checking_type) { @@ -1975,9 +1975,9 @@ bool Field::isomorphic_to(const Field& f, bool allow_other_names) const { } /* - * + * * TL-B SOURCE PARSER - * + * */ void parse_field_list(Lexer& lex, Constructor& cs); @@ -2410,9 +2410,9 @@ void parse_constructor_def(Lexer& lex) { } /* - * + * * SOURCE PARSER (TOP LEVEL) - * + * */ std::vector source_fdescr; @@ -2447,9 +2447,9 @@ bool parse_source_stdin() { } /* - * + * * BUILT-IN TYPE DEFINITIONS - * + * */ Type* define_builtin_type(std::string name_str, std::string args, bool produces_nat, int size = -1, int min_size = -1, @@ -2531,9 +2531,9 @@ void define_builtins() { } /* - * + * * SCHEME PROCESSING AND CHECKING - * + * */ bool Type::cons_all_exact() const { @@ -3041,9 +3041,9 @@ void dump_all_constexpr() { } /* - * + * * CODE GENERATION - * + * */ std::vector source_list; @@ -3057,9 +3057,9 @@ void register_source(std::string source) { #include "tlbc-gen-cpp.cpp" /* - * + * * TLBC MAIN - * + * */ void usage(const char* progname) { diff --git a/crypto/util/mintless-proof-generator.cpp b/crypto/util/mintless-proof-generator.cpp index 8f2b243a96..74d721a085 100644 --- a/crypto/util/mintless-proof-generator.cpp +++ b/crypto/util/mintless-proof-generator.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain Library. TON Blockchain Library is free software: you can redistribute it and/or modify diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index cc59b4864c..13db319494 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -947,9 +947,9 @@ unsigned long long BagOfCells::get_idx_entry_raw(int index) { } /* - * + * * Simple BoC serialization/deserialization functions - * + * */ td::Result> std_boc_deserialize(td::Slice data, bool can_be_empty, bool allow_nonzero_level) { @@ -1040,9 +1040,9 @@ td::Status std_boc_serialize_to_file(Ref root, td::FileFd& fd, int mode, } /* - * + * * Cell storage statistics - * + * */ td::Result CellStorageStat::compute_used_storage(Ref cs_ref, bool kill_dup, diff --git a/crypto/vm/cells/CellBuilder.cpp b/crypto/vm/cells/CellBuilder.cpp index 21d0b1dcbb..e782b04d07 100644 --- a/crypto/vm/cells/CellBuilder.cpp +++ b/crypto/vm/cells/CellBuilder.cpp @@ -29,9 +29,9 @@ using td::Ref; using td::RefAny; /* - * + * * CELL BUILDERS - * + * */ CellBuilder::~CellBuilder() { diff --git a/crypto/vm/dict.cpp b/crypto/vm/dict.cpp index e8c0e03a51..67fbb4d7f6 100644 --- a/crypto/vm/dict.cpp +++ b/crypto/vm/dict.cpp @@ -27,9 +27,9 @@ namespace vm { /* - * + * * DictionaryBase : basic (common) dictionary manipulation - * + * */ DictionaryBase::DictionaryBase(Ref _root, int _n, bool validate) @@ -236,7 +236,7 @@ bool DictionaryFixed::check_fork_raw(Ref cs_ref, int n) const { } /* - * + * * Label parser (HmLabel n ~l) for all dictionary types * */ @@ -408,9 +408,9 @@ int LabelParser::copy_label_prefix_to(td::BitPtr to, int max_len) const { } // namespace dict /* - * + * * Usual Dictionary - * + * */ using dict::LabelParser; @@ -2575,9 +2575,9 @@ bool DictionaryFixed::validate_all() { } /* - * + * * PREFIX DICTIONARIES - * + * */ std::pair, int> PrefixDictionary::lookup_prefix(td::ConstBitPtr key, int key_len) { @@ -2668,9 +2668,9 @@ Ref PrefixDictionary::lookup_delete(td::ConstBitPtr key, int key_len) } /* - * + * * AUGMENTED DICTIONARIES - * + * */ namespace dict { diff --git a/crypto/vm/fmt.hpp b/crypto/vm/fmt.hpp index 37b2cee97d..f103e7b4e0 100644 --- a/crypto/vm/fmt.hpp +++ b/crypto/vm/fmt.hpp @@ -371,9 +371,9 @@ Int u64(T& val) { } /* - * + * * non-integer types - * + * */ // cr(Ref& cell_ref) for (de)serializing cell references diff --git a/dht-server/dht-server.hpp b/dht-server/dht-server.hpp index 4853ce9ae9..fde7c40904 100644 --- a/dht-server/dht-server.hpp +++ b/dht-server/dht-server.hpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/dht/CMakeLists.txt b/dht/CMakeLists.txt index a2f63af58c..030f7c610b 100644 --- a/dht/CMakeLists.txt +++ b/dht/CMakeLists.txt @@ -17,9 +17,9 @@ set(DHT_SOURCE add_library(dht STATIC ${DHT_SOURCE}) -target_include_directories(dht PUBLIC - $ - $/.. +target_include_directories(dht PUBLIC + $ + $/.. ${OPENSSL_INCLUDE_DIR} ) target_link_libraries(dht PRIVATE tdutils tdactor adnl tl_api) diff --git a/dht/utils/dht-ping-servers.cpp b/dht/utils/dht-ping-servers.cpp index f01008a5a3..59b6086d6d 100644 --- a/dht/utils/dht-ping-servers.cpp +++ b/dht/utils/dht-ping-servers.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/dht/utils/dht-resolve.cpp b/dht/utils/dht-resolve.cpp index 615e1d4999..0e4697467d 100644 --- a/dht/utils/dht-resolve.cpp +++ b/dht/utils/dht-resolve.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/doc/ConfigParam-HOWTO b/doc/ConfigParam-HOWTO index 639014bb33..89c51ce7b4 100644 --- a/doc/ConfigParam-HOWTO +++ b/doc/ConfigParam-HOWTO @@ -142,7 +142,7 @@ Loading new value of configuration parameter 8 from file config-param8.boc x{C400000001000000000000000E} Non-critical configuration proposal will expire at 1586779536 (in 1100000 seconds) -Query id is 6810441749056454664 +Query id is 6810441749056454664 resulting internal message body: x{6E5650525E838CB0000000085E9455904_} x{F300000008A_} x{C400000001000000000000000E} @@ -163,9 +163,9 @@ We see that the address of the configuration smart contract is -1:5555...5555. B ==================================================== > runmethod -1:5555555555555555555555555555555555555555555555555555555555555555 proposal_storage_price 0 1100000 104 0 ... -arguments: [ 0 1100000 104 0 75077 ] -result: [ 2340800000 ] -remote result (not to be trusted): [ 2340800000 ] +arguments: [ 0 1100000 104 0 75077 ] +result: [ 2340800000 ] +remote result (not to be trusted): [ 2340800000 ] ==================================================== The parameters to get-method `proposal_storage_price` are the critical flag (0 in this case), the time interval during which this proposal will be active (1.1 Megaseconds), the total amount of bits (104) and cell references (0) in the data. The latter two quantities can be seen in the output of `create-config-proposal.fif`. @@ -175,7 +175,7 @@ We see that one has to pay 2.3408 test Grams to create this proposal. It is bett ==================================================== $ fift -s wallet.fif my-wallet -1:5555555555555555555555555555555555555555555555555555555555555555 31 4. -B config-msg-body.boc ... -Transferring GR$4. to account kf9VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVQft = -1:5555555555555555555555555555555555555555555555555555555555555555 seqno=0x1c bounce=-1 +Transferring GR$4. to account kf9VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVQft = -1:5555555555555555555555555555555555555555555555555555555555555555 seqno=0x1c bounce=-1 Body of transfer message is x{6E5650525E835154000000085E9293944_} x{F300000008A_} x{C400000001000000000000000E} @@ -205,9 +205,9 @@ After waiting for some time, we can inspect the incoming messages of our wallet ==================================================== > runmethod -1:5555555555555555555555555555555555555555555555555555555555555555 list_proposals ... -arguments: [ 107394 ] -result: [ ([64654898543692093106630260209820256598623953458404398631153796624848083036321 [1586779536 0 [8 C{FDCD887EAF7ACB51DA592348E322BBC0BD3F40F9A801CB6792EFF655A7F43BBC} -1] 112474791597373109254579258586921297140142226044620228506108869216416853782998 () 864691128455135209 3 0 0]]) ] -remote result (not to be trusted): [ ([64654898543692093106630260209820256598623953458404398631153796624848083036321 [1586779536 0 [8 C{FDCD887EAF7ACB51DA592348E322BBC0BD3F40F9A801CB6792EFF655A7F43BBC} -1] 112474791597373109254579258586921297140142226044620228506108869216416853782998 () 864691128455135209 3 0 0]]) ] +arguments: [ 107394 ] +result: [ ([64654898543692093106630260209820256598623953458404398631153796624848083036321 [1586779536 0 [8 C{FDCD887EAF7ACB51DA592348E322BBC0BD3F40F9A801CB6792EFF655A7F43BBC} -1] 112474791597373109254579258586921297140142226044620228506108869216416853782998 () 864691128455135209 3 0 0]]) ] +remote result (not to be trusted): [ ([64654898543692093106630260209820256598623953458404398631153796624848083036321 [1586779536 0 [8 C{FDCD887EAF7ACB51DA592348E322BBC0BD3F40F9A801CB6792EFF655A7F43BBC} -1] 112474791597373109254579258586921297140142226044620228506108869216416853782998 () 864691128455135209 3 0 0]]) ] ... caching cell FDCD887EAF7ACB51DA592348E322BBC0BD3F40F9A801CB6792EFF655A7F43BBC ==================================================== @@ -238,8 +238,8 @@ Note that the configuration proposal is henceforth identified by its 256-bit has > runmethod -1:5555555555555555555555555555555555555555555555555555555555555555 get_proposal 64654898543692093106630260209820256598623953458404398631153796624848083036321 ... -arguments: [ 64654898543692093106630260209820256598623953458404398631153796624848083036321 94347 ] -result: [ [1586779536 0 [8 C{FDCD887EAF7ACB51DA592348E322BBC0BD3F40F9A801CB6792EFF655A7F43BBC} -1] 112474791597373109254579258586921297140142226044620228506108869216416853782998 () 864691128455135209 3 0 0] ] +arguments: [ 64654898543692093106630260209820256598623953458404398631153796624848083036321 94347 ] +result: [ [1586779536 0 [8 C{FDCD887EAF7ACB51DA592348E322BBC0BD3F40F9A801CB6792EFF655A7F43BBC} -1] 112474791597373109254579258586921297140142226044620228506108869216416853782998 () 864691128455135209 3 0 0] ] We obtain essentially the same result as before, but for only one configuration proposal, and without the identifier of the configuration proposal at the beginning. @@ -254,7 +254,7 @@ Voting for a configuration proposal is possible only for current validators, lis - The operator invokes special Fift script `config-proposal-vote-req.fif`, found in directory `crypto/smartcont' of the source tree, indicating *val-idx* and *config-proposal-id* as its arguments: $ fift -s config-proposal-vote-req.fif -i 0 64654898543692093106630260209820256598623953458404398631153796624848083036321 - Creating a request to vote for configuration proposal 0x8ef1603180dad5b599fa854806991a7aa9f280dbdb81d67ce1bedff9d66128a1 on behalf of validator with index 0 + Creating a request to vote for configuration proposal 0x8ef1603180dad5b599fa854806991a7aa9f280dbdb81d67ce1bedff9d66128a1 on behalf of validator with index 0 566F744500008EF1603180DAD5B599FA854806991A7AA9F280DBDB81D67CE1BEDFF9D66128A1 Vm90RQAAjvFgMYDa1bWZ-oVIBpkaeqnygNvbgdZ84b7f-dZhKKE= Saved to file validator-to-sign.req @@ -274,7 +274,7 @@ You can monitor answer messages from the configuration smart contract to the con > runmethod -1:5555555555555555555555555555555555555555555555555555555555555555 get_proposal 64654898543692093106630260209820256598623953458404398631153796624848083036321 ... -arguments: [ 64654898543692093106630260209820256598623953458404398631153796624848083036321 94347 ] +arguments: [ 64654898543692093106630260209820256598623953458404398631153796624848083036321 94347 ] result: [ [1586779536 0 [8 C{FDCD887EAF7ACB51DA592348E322BBC0BD3F40F9A801CB6792EFF655A7F43BBC} -1] 112474791597373109254579258586921297140142226044620228506108869216416853782998 (0) 864691128455135209 3 0 0] ] This time the list of indices of validators that voted for this configuration proposal should be non-empty, and it should contain the index of your validator. In this example, this list is (0), meaning that only the validator with index 0 in configuration parameter #34 has voted. If the list becomes large enough, the last-but-one integer (the first zero in "3 0 0") in the proposal status will increase by one, indicating a new win by this proposal. If the number of wins becomes greater than or equal to the value indicated in configuration parameter #11, then the configuration proposal is automatically accepted and the proposed changes become effective immediately. On the other hand, when the validator set changes, then the list of validators that have already voted becomes empty, the value of *rounds_remaining* (three in "3 0 0") is decreased by one, and if it becomes negative, the configuration proposal is destroyed. If it is not destroyed, and if it did not win in this round, then the number of losses (the second zero in "3 0 0") is increased. If it becomes larger than a value specified in configuration parameter #11, then the configuration proposal is discarded. In this way all validators that have abstained from voting in a round have implicitly voted against. diff --git a/doc/DNS-HOWTO b/doc/DNS-HOWTO index 3f4fb4b3f6..69d404a9ed 100644 --- a/doc/DNS-HOWTO +++ b/doc/DNS-HOWTO @@ -73,14 +73,14 @@ We see that the address of this automatic DNS smart contract is EQAZC9dW9sDnlI3C ================================================ > runmethod EQAZC9dW9sDnlI3CbLR5aDIxd_sgNE-PmlCRjK-H7LNLeUXN getstdperiod ... -arguments: [ 67418 ] -result: [ 700000 ] -remote result (not to be trusted): [ 700000 ] +arguments: [ 67418 ] +result: [ 700000 ] +remote result (not to be trusted): [ 700000 ] > runmethod EQAZC9dW9sDnlI3CbLR5aDIxd_sgNE-PmlCRjK-H7LNLeUXN getppr ... -arguments: [ 109522 ] -result: [ 100000000 ] -remote result (not to be trusted): [ 100000000 ] +arguments: [ 109522 ] +result: [ 100000000 ] +remote result (not to be trusted): [ 100000000 ] ================================================ We see that subdomains are registered for 700000 seconds (about eight days), and that the registration price is 100000000ng = 0.1 test Grams per domain, plus a price for each bit and cell of stored data, which can be learned by running get-methods `getppb` and `getppc`. @@ -93,10 +93,10 @@ For example: =============================================== $ fift -s auto-dns.fif EQAZC9dW9sDnlI3CbLR5aDIxd_sgNE-PmlCRjK-H7LNLeUXN add 'mysite' 700000 owner kQABzslAMKOVwkSkkWfelS1pYSDOSyTcgn0yY_loQvyo_ZgI cat 1 adnl vcqmha5j3ceve35ammfrhqty46rkhi455otydstv66pk2tmf7rl25f3 -Automatic DNS smart contract address = 0:190bd756f6c0e7948dc26cb47968323177fb20344f8f9a50918caf87ecb34b79 +Automatic DNS smart contract address = 0:190bd756f6c0e7948dc26cb47968323177fb20344f8f9a50918caf87ecb34b79 kQAZC9dW9sDnlI3CbLR5aDIxd_sgNE-PmlCRjK-H7LNLef5H -Action: add mysite 1583865040 -Operation code: 0x72656764 +Action: add mysite 1583865040 +Operation code: 0x72656764 Value: x{2_} x{BC000C_} x{AD0145061C1D4EC44A937D0318589E13C73D151D1CEF5D3C0E53AFBCF56A6C2FE2BD00} @@ -111,7 +111,7 @@ Internal message body is: x{726567645E5D2E700481CE3F0EDAF2E6D2E8CA00BCCFB9A1_} x{9FD3800039D928061472B84894922CFBD2A5AD2C2419C9649B904FA64C7F2D085F951FA01_} B5EE9C7241010601007800012F726567645E5D2E700481CE3F0EDAF2E6D2E8CA00BCCFB9A10102012002030105BC000C040105BFFFF4050046AD0145061C1D4EC44A937D0318589E13C73D151D1CEF5D3C0E53AFBCF56A6C2FE2BD0000499FD3800039D928061472B84894922CFBD2A5AD2C2419C9649B904FA64C7F2D085F951FA01070E6337D -Query_id is 6799642071046147647 = 0x5E5D2E700481CE3F +Query_id is 6799642071046147647 = 0x5E5D2E700481CE3F (Saved to file dns-msg-body.boc) ================================================ @@ -119,10 +119,10 @@ We see that the internal message body for this query has been created and saved ================================================ $ fift -s wallet.fif my_new_wallet EQAZC9dW9sDnlI3CbLR5aDIxd_sgNE-PmlCRjK-H7LNLeUXN 1 1.7 -B dns-msg-body.boc -Source wallet address = 0:01cec94030a395c244a49167de952d696120ce4b24dc827d3263f96842fca8fd +Source wallet address = 0:01cec94030a395c244a49167de952d696120ce4b24dc827d3263f96842fca8fd kQABzslAMKOVwkSkkWfelS1pYSDOSyTcgn0yY_loQvyo_ZgI Loading private key from file my_new_wallet.pk -Transferring GR$1.7 to account kQAZC9dW9sDnlI3CbLR5aDIxd_sgNE-PmlCRjK-H7LNLef5H = 0:190bd756f6c0e7948dc26cb47968323177fb20344f8f9a50918caf87ecb34b79 seqno=0x1 bounce=-1 +Transferring GR$1.7 to account kQAZC9dW9sDnlI3CbLR5aDIxd_sgNE-PmlCRjK-H7LNLef5H = 0:190bd756f6c0e7948dc26cb47968323177fb20344f8f9a50918caf87ecb34b79 seqno=0x1 bounce=-1 Body of transfer message is x{726567645E5D2E700481CE3F0EDAF2E6D2E8CA00BCCFB9A1_} x{2_} x{BC000C_} @@ -153,7 +153,7 @@ B5EE9C72410207010001170001CF8800039D928061472B84894922CFBD2A5AD2C2419C9649B904FA (You have to replace 1 with the correct sequence number for your wallet.) Once you obtain a signed external message in `wallet-query.boc`, addressed to your wallet and instructing it to transfer 1.7 test Grams to the automatic DNS smart contract along with the description of your new domain to be registered, you can upload this message using the LiteClient by typing ===================================================== -> sendfile wallet-query.boc +> sendfile wallet-query.boc [ 1][t 1][!testnode] sending query from file wallet-query.boc [ 3][t 1][!query] external message status is 1 ===================================================== diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index ac60331a11..9503cb3961 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -47,7 +47,7 @@ __Enabled in mainnet on 2024-02-03__ ### Gas limits Version 5 enables higher gas limits for special contracts. -* Gas limit for all transactions on special contracts is set to `special_gas_limit` from `ConfigParam 20` (which is 35M at the moment of writing). +* Gas limit for all transactions on special contracts is set to `special_gas_limit` from `ConfigParam 20` (which is 35M at the moment of writing). Previously only ticktock transactions had this limit, while ordinary transactions had a default limit of `gas_limit` gas (1M). * Gas usage of special contracts is not taken into account when checking block limits. This allows keeping masterchain block limits low while having high gas limits for elector. diff --git a/doc/LiteClient-HOWTO b/doc/LiteClient-HOWTO index 272c4472dc..0be1a7ff5c 100644 --- a/doc/LiteClient-HOWTO +++ b/doc/LiteClient-HOWTO @@ -28,7 +28,7 @@ in the "raw" form (notice that uppercase Latin letters 'A'..'F' may be used inst and -kf/8uRo6OBbQ97jCx2EIuKm8Wmt6Vb15-KsQHFLbKSMiYIny (base64) +kf/8uRo6OBbQ97jCx2EIuKm8Wmt6Vb15-KsQHFLbKSMiYIny (base64) kf_8uRo6OBbQ97jCx2EIuKm8Wmt6Vb15-KsQHFLbKSMiYIny (base64url) in the "user-friendly" form (to be displayed by user-friendly clients). Notice that both forms (base64 and base64url) are valid and must be accepted. @@ -78,12 +78,12 @@ account state is (account split_depth:nothing special:nothing code:(just - value:(raw@^Cell + value:(raw@^Cell x{} x{FF0020DD2082014C97BA9730ED44D0D70B1FE0A4F260D31F01ED44D0D31FD166BAF2A1F8000120D74A8E11D307D459821804A817C80073FB0201FB00DED1A4C8CB1FC9ED54} )) data:(just - value:(raw@^Cell + value:(raw@^Cell x{} x{00009A15} )) @@ -148,7 +148,7 @@ def? $2 { @' $2 } { "new-wallet" } cond constant file-base CHKSIGNU // pubk cs cnt ? 34 THROWIFNOT // signature mismatch ACCEPT - SWAP 32 LDU NIP + SWAP 32 LDU NIP DUP SREFS IF:<{ // 3 INT 35 LSHIFT# 3 INT RAWRESERVE // reserve all but 103 Grams from the balance 8 LDU LDREF // pubk cnt mode msg cs @@ -158,10 +158,10 @@ def? $2 { @' $2 } { "new-wallet" } cond constant file-base INC NEWC 32 STU 256 STU ENDC c4 POPCTR }>c // >libref // code - // data null // no libraries // Libs{ x{ABACABADABACABA} drop x{AAAA} s>c public_lib x{1234} x{5678} |_ s>c public_lib }Libs @@ -199,13 +199,13 @@ instead of indicating the complete search paths in the command line. If everything worked, you'll see something like the following -------------------------------------------- -Creating new wallet in workchain 0 +Creating new wallet in workchain 0 Saved new private key to file my_wallet_name.pk StateInit: x{34_} x{FF0020DD2082014C97BA9730ED44D0D70B1FE0A4F260810200D71820D70B1FED44D0D31FD3FFD15112BAF2A122F901541044F910F2A2F80001D31F3120D74A96D307D402FB00DED1A4C8CB1FCBFFC9ED54} x{00000000C59DC52962CC568AC5E72735EABB025C5BDF457D029AEEA6C2FFA5EB2A945446} -new wallet address = 0:2ee9b4fd4f077c9b223280c35763df9edab0b41ac20d36f4009677df95c3afe2 +new wallet address = 0:2ee9b4fd4f077c9b223280c35763df9edab0b41ac20d36f4009677df95c3afe2 (Saving address to file my_wallet_name.addr) Non-bounceable address (for init): 0QAu6bT9Twd8myIygMNXY9-e2rC0GsINNvQAlnfflcOv4uVb Bounceable address (for later access): kQAu6bT9Twd8myIygMNXY9-e2rC0GsINNvQAlnfflcOv4rie @@ -264,8 +264,8 @@ got account state for -1 : FCB91A3A3816D0F7B8C2C76108B8A9BC5A6B7A55BD79F8AB101C5 creating VM starting VM to run method `seqno` (85143) of smart contract -1:FCB91A3A3816D0F7B8C2C76108B8A9BC5A6B7A55BD79F8AB101C52DB29232260 ... -arguments: [ 85143 ] -result: [ 39445 ] +arguments: [ 85143 ] +result: [ 39445 ] -------------------------------------------- Next, you create an external message to the test giver asking it to send another message to your (uninitialized) smart contract carrying a specified amount of test Grams. There is a special Fift script for generating this external message located at crypto/smartcont/testgiver.fif: @@ -281,7 +281,7 @@ Next, you create an external message to the test giver asking it to send another $# 3 - -2 and ' usage if -// "testgiver.addr" load-address +// "testgiver.addr" load-address Masterchain 0xfcb91a3a3816d0f7b8c2c76108b8a9bc5a6b7a55bd79f8ab101c52db29232260 2constant giver_addr ."Test giver address = " giver_addr 2dup .addr cr 6 .Addr cr @@ -296,7 +296,7 @@ dest_addr 2dup bounce 7 + .Addr ." = " .addr ."seqno=0x" seqno x. ."bounce=" bounce . cr // create a message (NB: 01b00.., b = bounce) - dup ."enveloping message: " B B>boc } { } cond constant body-cell ."Transferring " amount .GR ."to account " -dest_addr 2dup bounce 7 + .Addr ." = " .addr +dest_addr 2dup bounce 7 + .Addr ." = " .addr ."seqno=0x" seqno x. ."bounce=" bounce . cr ."Body of transfer message is " body-cell diff --git a/doc/TonSites-HOWTO b/doc/TonSites-HOWTO index 8975a39ffb..493110f878 100644 --- a/doc/TonSites-HOWTO +++ b/doc/TonSites-HOWTO @@ -1,4 +1,4 @@ -The aim of this document is to provide a gentle introduction into TON Sites, which are (TON) Web sites accessed through the TON Network. TON Sites may be used as a convenient entry point for other TON Services. In particular, HTML pages downloaded from TON Sites may contain links to ton://... URIs representing payments that can be performed by the user by clicking to the link, provided a TON Wallet is installed on the user's device. +The aim of this document is to provide a gentle introduction into TON Sites, which are (TON) Web sites accessed through the TON Network. TON Sites may be used as a convenient entry point for other TON Services. In particular, HTML pages downloaded from TON Sites may contain links to ton://... URIs representing payments that can be performed by the user by clicking to the link, provided a TON Wallet is installed on the user's device. From the technical perspective, TON Sites are very much like the usual Web sites, but they are accessed through the TON Network (which is an overlay network inside the Internet) instead of the Internet. More specifically, they have an ADNL address (instead of a more customary IPv4 or IPv6 address), and they accept HTTP queries via RLDP protocol (which is a higher-level RPC protocol built upon ADNL, the main protocol of TON Network) instead of the usual TCP/IP. All encryption is handled by ADNL, so there is no need to use HTTPS (i.e., TLS). diff --git a/doc/Validator-HOWTO b/doc/Validator-HOWTO index b538823132..77a08231a2 100644 --- a/doc/Validator-HOWTO +++ b/doc/Validator-HOWTO @@ -29,7 +29,7 @@ $ crypto/fift -I /crypto/fift/lib:/crypto/smartcont -s n Once you run this script, the address of the new smart contract is displayed: ... -new wallet address = -1:af17db43f40b6aa24e7203a9f8c8652310c88c125062d1129fe883eaa1bd6763 +new wallet address = -1:af17db43f40b6aa24e7203a9f8c8652310c88c125062d1129fe883eaa1bd6763 (Saving address to file my_wallet_id.addr) Non-bounceable address (for init): 0f-vF9tD9Atqok5yA6n4yGUjEMiMElBi0RKf6IPqob1nYzqK Bounceable address (for later access): kf-vF9tD9Atqok5yA6n4yGUjEMiMElBi0RKf6IPqob1nY2dP @@ -66,8 +66,8 @@ Namely, running > runmethod -1:A4C2C7C05B093D470DE2316DBA089FA0DD775FD9B1EBFC9DC9D04B498D3A2DDA active_election_id ... -arguments: [ 86535 ] -result: [ 1567633899 ] +arguments: [ 86535 ] +result: [ 1567633899 ] (or lite-client -C -c "runmethod -1: active_election_id" in batch mode) will return the identifier of the currently active elections (a non-zero integer, typically the Unix time of the start of the service term of the validator group being elected), or 0, if no elections are currently active. In this example, the identifier of the active elections is 1567633899. @@ -122,7 +122,7 @@ or, if you have created the controlling wallet by means of new-wallet.fif, you c --------------------------------------- $ fift -s validator-elect-req.fif @my_wallet_id.addr 1567633899 2.7 C5C2B94529405FB07D1DDFB4C42BFB07727E7BA07006B2DB569FBF23060B9E5C -Creating a request to participate in validator elections at time 1567633899 from smart contract Uf+vF9tD9Atqok5yA6n4yGUjEMiMElBi0RKf6IPqob1nY4EA = -1:af17db43f40b6aa24e7203a9f8c8652310c88c125062d1129fe883eaa1bd6763 with maximal stake factor with respect to the minimal stake 176947/65536 and validator ADNL address c5c2b94529405fb07d1ddfb4c42bfb07727e7ba07006b2db569fbf23060b9e5c +Creating a request to participate in validator elections at time 1567633899 from smart contract Uf+vF9tD9Atqok5yA6n4yGUjEMiMElBi0RKf6IPqob1nY4EA = -1:af17db43f40b6aa24e7203a9f8c8652310c88c125062d1129fe883eaa1bd6763 with maximal stake factor with respect to the minimal stake 176947/65536 and validator ADNL address c5c2b94529405fb07d1ddfb4c42bfb07727e7ba07006b2db569fbf23060b9e5c 654C50745D7031EB0002B333AF17DB43F40B6AA24E7203A9F8C8652310C88C125062D1129FE883EAA1BD6763C5C2B94529405FB07D1DDFB4C42BFB07727E7BA07006B2DB569FBF23060B9E5C ZUxQdF1wMesAArMzrxfbQ_QLaqJOcgOp-MhlIxDIjBJQYtESn-iD6qG9Z2PFwrlFKUBfsH0d37TEK_sHcn57oHAGsttWn78jBgueXA== --------------------------------------- @@ -140,10 +140,10 @@ Now you have to run another script validator-elect-signed.fif, which also requir ------------------------------------ $ fift -s validator-elect-signed.fif @my_wallet_id.addr 1567633899 2.7 C5C2B94529405FB07D1DDFB4C42BFB07727E7BA07006B2DB569FBF23060B9E5C xrQTSIQEsqZkWnoADMiBnyBFRUUweXTvzRQFqb5nHd5xmeE6 ovf9cmr2J/speJEtMU+tZm6zH/GBEyZCPpaukqL3mmNH9Wipyoys63VFh0yR386bARHKMPpfKAYBYslOjdSjCQ== -Creating a request to participate in validator elections at time 1567633899 from smart contract Uf+vF9tD9Atqok5yA6n4yGUjEMiMElBi0RKf6IPqob1nY4EA = -1:af17db43f40b6aa24e7203a9f8c8652310c88c125062d1129fe883eaa1bd6763 with maximal stake factor with respect to the minimal stake 176947/65536 and validator ADNL address c5c2b94529405fb07d1ddfb4c42bfb07727e7ba07006b2db569fbf23060b9e5c +Creating a request to participate in validator elections at time 1567633899 from smart contract Uf+vF9tD9Atqok5yA6n4yGUjEMiMElBi0RKf6IPqob1nY4EA = -1:af17db43f40b6aa24e7203a9f8c8652310c88c125062d1129fe883eaa1bd6763 with maximal stake factor with respect to the minimal stake 176947/65536 and validator ADNL address c5c2b94529405fb07d1ddfb4c42bfb07727e7ba07006b2db569fbf23060b9e5c String to sign is: 654C50745D7031EB0002B333AF17DB43F40B6AA24E7203A9F8C8652310C88C125062D1129FE883EAA1BD6763C5C2B94529405FB07D1DDFB4C42BFB07727E7BA07006B2DB569FBF23060B9E5C Provided a valid Ed25519 signature A2F7FD726AF627FB2978912D314FAD666EB31FF1811326423E96AE92A2F79A6347F568A9CA8CACEB7545874C91DFCE9B0111CA30FA5F28060162C94E8DD4A309 with validator public key 8404B2A6645A7A000CC8819F20454545307974EFCD1405A9BE671DDE7199E13A -query_id set to 1567632790 +query_id set to 1567632790 Message body is x{4E73744B000000005D702D968404B2A6645A7A000CC8819F20454545307974EFCD1405A9BE671DDE7199E13A5D7031EB0002B333C5C2B94529405FB07D1DDFB4C42BFB07727E7BA07006B2DB569FBF23060B9E5C} x{A2F7FD726AF627FB2978912D314FAD666EB31FF1811326423E96AE92A2F79A6347F568A9CA8CACEB7545874C91DFCE9B0111CA30FA5F28060162C94E8DD4A309} @@ -156,11 +156,11 @@ Alternatively, if you are running validator-engine-console on the same machine a Now you have a message body containing your elections participation request. You must send it from the controlling smart contract, carrying the stake as its value (plus one extra Gram for sending confirmation). If you use the simple wallet smart contract, this can be done by using the `-B` command-line argument to wallet.fif: -------------------------------------------- -$ fift -s wallet.fif my_wallet_id -1:A4C2C7C05B093D470DE2316DBA089FA0DD775FD9B1EBFC9DC9D04B498D3A2DDA 1 100001. -B validator-query.boc -Source wallet address = -1:af17db43f40b6aa24e7203a9f8c8652310c88c125062d1129fe883eaa1bd6763 +$ fift -s wallet.fif my_wallet_id -1:A4C2C7C05B093D470DE2316DBA089FA0DD775FD9B1EBFC9DC9D04B498D3A2DDA 1 100001. -B validator-query.boc +Source wallet address = -1:af17db43f40b6aa24e7203a9f8c8652310c88c125062d1129fe883eaa1bd6763 kf-vF9tD9Atqok5yA6n4yGUjEMiMElBi0RKf6IPqob1nY2dP Loading private key from file my_wallet_id.pk -Transferring GR$100001. to account kf-kwsfAWwk9Rw3iMW26CJ-g3Xdf2bHr_J3J0EtJjTot2lHQ = -1:a4c2c7c05b093d470de2316dba089fa0dd775fd9b1ebfc9dc9d04b498d3a2dda seqno=0x1 bounce=-1 +Transferring GR$100001. to account kf-kwsfAWwk9Rw3iMW26CJ-g3Xdf2bHr_J3J0EtJjTot2lHQ = -1:a4c2c7c05b093d470de2316dba089fa0dd775fd9b1ebfc9dc9d04b498d3a2dda seqno=0x1 bounce=-1 Body of transfer message is x{4E73744B000000005D702D968404B2A6645A7A000CC8819F20454545307974EFCD1405A9BE671DDE7199E13A5D7031EB0002B333C5C2B94529405FB07D1DDFB4C42BFB07727E7BA07006B2DB569FBF23060B9E5C} x{A2F7FD726AF627FB2978912D314FAD666EB31FF1811326423E96AE92A2F79A6347F568A9CA8CACEB7545874C91DFCE9B0111CA30FA5F28060162C94E8DD4A309} @@ -203,7 +203,7 @@ If the result is zero, nothing is due to you. Otherwise, you'll see part or all ----------------------------- $ fift -s recover-stake.fif -query_id for stake recovery message is set to 1567634299 +query_id for stake recovery message is set to 1567634299 Message body is x{47657424000000005D70337B} @@ -216,7 +216,7 @@ $ fift -s wallet.fif my_wallet_id -B For example, -$ fift -s wallet.fif my_wallet_id -1:A4C2C7C05B093D470DE2316DBA089FA0DD775FD9B1EBFC9DC9D04B498D3A2DDA 2 1. -B recover-query.boc +$ fift -s wallet.fif my_wallet_id -1:A4C2C7C05B093D470DE2316DBA089FA0DD775FD9B1EBFC9DC9D04B498D3A2DDA 2 1. -B recover-query.boc ... (Saved to file wallet-query.boc) diff --git a/doc/catchain.tex b/doc/catchain.tex index b1a27164d4..771973b0c7 100644 --- a/doc/catchain.tex +++ b/doc/catchain.tex @@ -300,7 +300,7 @@ \item The algorithm keeps track of $\sigma_{C_i}$, where $C_i$ is the current dependence cone, i.e., the set of all messages $m$ delivered (to the current process $i$). The initial value of $\sigma_{C_i}$ is $\sigma_\emptyset$. \item When a new message $m$ is delivered, the value of $\sigma_{D^m}$ is computed by a repeated application of $g$ since $D_m=\bigcup_{m'\in D_m}D^+_{m'}=\bigcup_{m'\in\Max(D_m)}D^+_{m'}$; therefore, if $\Max(D_m)=\{m'_1,\ldots,m'_k\}$, then \begin{equation}\label{eq:merge.many} - \sigma_{D_m}=g\Bigl(\ldots g\bigl(g(\sigma_{D^+_{m'_1}},\sigma_{D^+_{m'_2}}),\sigma_{D^+_{m'_3}}\bigr),\ldots \sigma_{D^+_{m'_k}}\Bigr)\quad. + \sigma_{D_m}=g\Bigl(\ldots g\bigl(g(\sigma_{D^+_{m'_1}},\sigma_{D^+_{m'_2}}),\sigma_{D^+_{m'_3}}\bigr),\ldots \sigma_{D^+_{m'_k}}\Bigr)\quad. \end{equation} The set $\Max(D_m)$ is explicitly listed in the header of message $m$ in some fixed order $m'_1$, \dots, $m'_k$; the above formula is applied with respect to this order (so the computation of $D_m$ is deterministic). The first element in this list always is the previous message of the sender of $m$, i.e., if $m=m_{i,s+1}$, then $m'_1=m_{i,s}$. \item After that, the value of $\sigma_{D^+_m}$ is computed by an application of $f$: $\sigma_{D^+_m}=f(\sigma_{D_m},m)$. This value is memorized for future use. @@ -312,7 +312,7 @@ \item Once a new message $m$ is generated and locally delivered, so that $C_i$ becomes equal to $D^+_m$, the previously computed value of $\sigma_{C_i}$ is discarded and replaced with $\sigma_{D^+_m}$ computed according to the general algorithm described above. If $g$ is not commutative or not associative (for example, it may happen that $g(x,y)$ and $g(y,x)$ are different but equivalent representations of the same state), then this might lead to a slight change of the current ``virtual'' state of process $i$. \item If the lower-level (catchain) protocol reports to the higher-level protocol that a certain process $j\not\in i$ is ``bad'' (i.e., $j$ is found out to have created a fork, cf.~\ptref{sp:bad.proc.set}, or to have knowingly endorsed a fork by another process, cf.~\ptref{sp:no.bad.accept}), then the current (virtual) state $\sigma_{C_i}$ is recomputed from scratch using the new set $C'_i=\bigcup_{\text{$m\in C_i$, $m$ was created by ``good'' process $k$}}D^+_m$ and the ``merge'' function $g$ applied to the set of $\sigma_{D^+_m}$ where $m$ runs through the set of last messages of the processes known to be good (or through the set of maximal elements of this set). The next created outbound message will depend only on the messages from $C'_i$. \end{itemize} - + \nxsubpoint\emb{Necessity to know the internal state of the other processes} Formula \eqref{eq:merge.many} implies that process~$i$ must also keep track of $\sigma_{D^+_m}$ for all messages $m$, created by this process or not. However, this is possible since these internal states are also computed by appropriate applications of the update algorithm. Therefore, BCP computes and remembers all $\sigma_{D^+_m}$ as well. @@ -537,7 +537,7 @@ \begin{thebibliography}{2} \bibitem{Birman} {\sc K.~Birman}, {\sl Reliable Distributed Systems: Technologies, Web Services and Applications}, Springer, 2005. - + \bibitem{PBFT} {\sc M.~Castro, B.~Liskov, et al.}, {\sl Practical byzantine fault tolerance}, {\it Proceedings of the Third Symposium on Operating Systems Design and Implementation\/} (1999), p.~173--186, available at \url{http://pmg.csail.mit.edu/papers/osdi99.pdf}. diff --git a/doc/fiftbase.tex b/doc/fiftbase.tex index c6131b2e2e..ea45fe7c1c 100644 --- a/doc/fiftbase.tex +++ b/doc/fiftbase.tex @@ -350,8 +350,8 @@ \section*{Introduction} If one needs to use such Gram literals often, one can introduce a new active prefix word {\tt GR\$} as follows: \begin{verbatim} -{ bl word (number) ?dup 0= abort"not a valid Gram amount" - 1- { Gram swap */r } { Gram * } cond +{ bl word (number) ?dup 0= abort"not a valid Gram amount" + 1- { Gram swap */r } { Gram * } cond 1 'nop } ::_ GR$ \end{verbatim} @@ -533,11 +533,11 @@ \section*{Introduction} \end{verbatim} produces \begin{verbatim} -x=2 y=5 x*y=10 -x=2 y=6 x*y=12 -x=3 y=6 x*y=18 -x=3 y=7 x*y=21 -x=7 y=3 x*y=21 +x=2 y=5 x*y=10 +x=2 y=6 x*y=12 +x=3 y=6 x*y=18 +x=3 y=7 x*y=21 +x=7 y=3 x*y=21 \end{verbatim} \mysubsection{Tuples and arrays}\label{p:tuples} @@ -596,9 +596,9 @@ \section*{Introduction} \end{verbatim} produces \begin{verbatim} -[ [ 1 2 3 ] [ 4 5 6 ] [ 7 8 9 ] ] -6 -[ 7 8 9 ] +[ [ 1 2 3 ] [ 4 5 6 ] [ 7 8 9 ] ] +6 +[ 7 8 9 ] \end{verbatim} Notice that {\em Tuple\/}s are somewhat similar to arrays of other programming languages, but are immutable: we cannot change one individual component of a {\em Tuple}. If we still want to create something like an array, we need a {\em Tuple\/} of {\em Box\/}es (cf.~\ptref{p:variables}): @@ -612,7 +612,7 @@ \section*{Introduction} { over @ over @ swap rot ! swap ! } : swap-values-of { B swap [] } : B[] { B[] swap B[] swap-values-of } : swap-B -{ B[] @ . } : .B[] +{ B[] @ . } : .B[] 0 1 swap-B 1 3 swap-B 0 2 swap-B 0 .B[] 1 .B[] 2 .B[] 3 .B[] \end{verbatim} @@ -640,14 +640,14 @@ \section*{Introduction} For instance, \begin{verbatim} 2 3 9 3 tuple .dump cr -2 3 9 3 list dup .dump space dup .l cr +2 3 9 3 list dup .dump space dup .l cr "test" swap cons .l cr \end{verbatim} produces \begin{verbatim} -[ 2 3 9 ] -[ 2 [ 3 [ 9 (null) ] ] ] (2 3 9) -("test" 2 3 9) +[ 2 3 9 ] +[ 2 [ 3 [ 9 (null) ] ] ] (2 3 9) +("test" 2 3 9) \end{verbatim} Notice that the three-element list $(2\ 3\ 9)$ is distinct from the triple $(2,3,9)$. @@ -656,7 +656,7 @@ \section*{Introduction} \begin{itemize} \item {\tt (atom)} ($S$ $x$ -- $a$ $-1$ or $0$), returns the only {\em Atom\/} $a$ with the name given by {\em String\/}~$S$. If there is no such {\em Atom\/} yet, either creates it (if {\em Integer\/}~$x$ is non-zero) or returns a single zero to indicate failure (if $x$ is zero). \item {\tt atom} ($S$ -- $a$), returns the only {\em Atom\/}~$a$ with the name~$S$, creating such an atom if necessary. Equivalent to {\tt true (atom) drop}. -\item {\tt `$\langle\textit{word\/}\rangle$} ( -- $a$), introduces an {\em Atom\/} literal, equal to the only {\em Atom\/} with the name equal to $\langle\textit{word\/}\rangle$. Equivalent to {\tt "$\langle\textit{word\/}\rangle$" atom}. +\item {\tt `$\langle\textit{word\/}\rangle$} ( -- $a$), introduces an {\em Atom\/} literal, equal to the only {\em Atom\/} with the name equal to $\langle\textit{word\/}\rangle$. Equivalent to {\tt "$\langle\textit{word\/}\rangle$" atom}. \item {\tt anon} ( -- $a$), creates a new unique anonymous {\em Atom}. \item {\tt atom?} ($u$ -- $?$), checks whether $u$ is an {\em Atom}. \item {\tt eq?} ($u$ $v$ -- $?$), checks whether $u$ and $v$ are equal {\em Integer\/}s, {\em Atom\/}s, or {\em Null\/}s. If they are not equal, or if they are of different types, or not of one of the types listed, returns zero. @@ -676,21 +676,21 @@ \section*{Introduction} { 'eval @ execute } : eval { dup tuple? { uncons uncons uncons - null? not abort"three-element list expected" + null? not abort"three-element list expected" swap eval swap eval rot dup `+ eq? { drop + } { dup `- eq? { drop - } { - `* eq? not abort"unknown operation" * + `* eq? not abort"unknown operation" * } cond } cond - } if + } if } 'eval ! `+ 2 `* 3 4 3 list 3 list dup .l cr eval . cr \end{verbatim} prints \begin{verbatim} -(+ 2 (* 3 4)) -14 +(+ 2 (* 3 4)) +14 \end{verbatim} If we load {\tt Lisp.fif} to enable Lisp-style list syntax, we can enter \begin{verbatim} @@ -700,7 +700,7 @@ \section*{Introduction} with the same result as before. The word {\tt (}, defined in {\tt Lisp.fif}, uses an anonymous {\em Atom\/} created by {\tt anon} to mark the current stack position, and then {\tt )} builds a list from several top stack entries, scanning the stack until the anonymous {\em Atom\/} marker is found: \begin{verbatim} variable ') -{ ") without (" abort } ') ! +{ ") without (" abort } ') ! { ') @ execute } : ) { null { -rot 2dup eq? not } { swap rot cons } while 2drop } : list-until-marker @@ -948,10 +948,10 @@ \section*{Introduction} \end{verbatim} producing the following output: \begin{verbatim} -(3 1 4 1 5 9 2 6 5) -(1 1 2 3 4 5 5 6 9) -("once" "upon" "a" "time" "there" "lived" "a" "kitten") -("a" "a" "kitten" "lived" "once" "there" "time" "upon") +(3 1 4 1 5 9 2 6 5) +(1 1 2 3 4 5 5 6 9) +("once" "upon" "a" "time" "there" "lived" "a" "kitten") +("a" "a" "kitten" "lived" "once" "there" "time" "upon") \end{verbatim} \mysubsection{Throwing exceptions}\label{p:exception.ops} @@ -1618,7 +1618,7 @@ \section*{Introduction} If we choose to keep a conditional branch in a separate cell, we can use the {\tt <\{ \dots\ \}>c} construct along with instructions such as {\tt IFJMPREF}: \begin{verbatim} <{ DUP 1 INT AND - <{ 3 MULCONST INC }>c IFJMPREF + <{ 3 MULCONST INC }>c IFJMPREF 1 RSHIFT# }>s dup csr. 3 swap runvmcode .s @@ -1634,7 +1634,7 @@ \section*{Introduction} execute MULINT 3 execute INC execute implicit RET -10 0 +10 0 \end{verbatim} \mysubsection{Macro definitions} @@ -1770,7 +1770,7 @@ \section*{Introduction} dup csr. runvmdict .s \end{verbatim} -which produces +which produces \begin{verbatim} x{F24B9D5331A85331A8A15044A859A8A075715C24D85C24D8588100EF7F58D9} implicit PUSH 0 at start @@ -1792,7 +1792,7 @@ \section*{Introduction} execute implicit RET 114244 114244 0 \end{verbatim} - + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % % @@ -1992,7 +1992,7 @@ \section*{Introduction} \item {\tt []} ($t$ $i$ -- $x$), returns the $(i+1)$-st component $t_{i+1}$ of {\em Tuple\/}~$t$, where $0\leq i<|t|$, cf.~\ptref{p:tuples}. \item {\tt \underline{[compile]} $\langle\textit{word-name}\rangle$} ( -- ), compiles $\langle\textit{word-name}\rangle$ as if it were an ordinary word, even if it is active, cf.~\ptref{p:dict.lookup}. Essentially equivalent to {\tt ' $\langle\textit{word-name}\rangle$ execute}. \item {\tt \underline{]}} ($x_1$ \dots $x_n$ $n$ -- ), closes an internal interpreter session opened by {\tt [} and invokes {\tt (compile)} or {\tt (execute)} afterwards depending on whether {\tt state} is greater than zero. For instance, {\tt \{ [ 2 3 + 1 ] * \}} is equivalent to {\tt \{ 5 * \}}. -\item {\tt \underline{`}$\langle\textit{word\/}\rangle$} ( -- $a$), introduces an {\em Atom\/} literal, equal to the only {\em Atom\/} with the name equal to $\langle\textit{word\/}\rangle$, cf.~\ptref{p:atoms}. Equivalent to {\tt "$\langle\textit{word\/}\rangle$" atom}. +\item {\tt \underline{`}$\langle\textit{word\/}\rangle$} ( -- $a$), introduces an {\em Atom\/} literal, equal to the only {\em Atom\/} with the name equal to $\langle\textit{word\/}\rangle$, cf.~\ptref{p:atoms}. Equivalent to {\tt "$\langle\textit{word\/}\rangle$" atom}. \item {\tt abort} ($S$ -- ), throws an exception with an error message taken from {\em String\/}~$S$, cf.~\ptref{p:exception.ops}. \item {\tt \underline{abort"}$\langle\textit{message}\rangle$"} ($x$ -- ), throws an exception with the error message $\langle\textit{message}\rangle$ if the {\em Integer\/}~$x$ is non-zero, cf.~\ptref{p:exception.ops}. \item {\tt abs} ($x$ -- $|x|$), computes the absolute value $|x|=\max(x,-x)$ of {\em Integer\/}~$x$. Equivalent to {\tt dup negate max}. @@ -2076,7 +2076,7 @@ \section*{Introduction} \item {\tt gasrunvmctx} (\dots $s$ $c$ $t$ $z$ -- \dots $x$ $c'$ $z'$), a gas-aware version of {\tt runvmctx}, cf.~\ptref{p:tvm.ops}. Differs from {\tt gasrunmv} in that it initializes {\tt c7} with {\em Tuple\/}~$t$. \item {\tt gasrunvmdict} (\dots $s$ $z$ -- \dots $x$ $z'$), a gas-aware version of {\tt runvmdict}, cf.~\ptref{p:tvm.ops}: invokes a new instance of TVM with the current continuation {\tt cc} initialized from {\em Slice\/} $s$ and sets the gas limit to $z$ similarly to {\tt gasrunvmcode}, but also initializes the special register {\tt c3} with the same value, and pushes a zero into the initial TVM stack before the TVM execution begins. The actually consumed gas is returned as an {\em Integer\/} $z'$. In a typical application {\em Slice\/}~$s$ consists of a subroutine selection code that uses the top-of-stack {\em Integer\/} to select the subroutine to be executed, thus enabling the definition and execution of several mutually-recursive subroutines (cf.~\cite[4.6]{TVM} and~\ptref{p:asm.prog}). The selector equal to zero corresponds to the {\tt main()} subroutine in a large TVM program. \item {\tt halt} ($x$ -- ), quits to the operating system similarly to {\tt bye}, but uses {\em Integer\/} $x$ as the exit code, cf.~\ptref{p:exit.fift}. -\item {\tt hash} ($c$ -- $x$), a deprecated version of {\tt hashu}. Use {\tt hashu} or {\tt hashB} instead. +\item {\tt hash} ($c$ -- $x$), a deprecated version of {\tt hashu}. Use {\tt hashu} or {\tt hashB} instead. \item {\tt hashB} ($c$ -- $B$), computes the $\Sha$-based representation hash of {\em Cell\/}~$c$ (cf.~\cite[3.1]{TVM}), which unambiguously defines $c$ and all its descendants (provided there are no collisions for $\Sha$), cf.~\ptref{p:hash.ops}. The result is returned as a {\em Bytes\/} value consisting of exactly 32 bytes. \item {\tt hashu} ($c$ -- $x$), computes the $\Sha$-based representation hash of {\em Cell\/}~$c$ similarly to {\tt hashB}, but returns the result as a big-endian unsigned 256-bit {\em Integer}. \item {\tt hold} ($S$ $x$ -- $S'$), appends to {\em String\/}~$S$ one UTF-8 encoded character with Unicode codepoint~$x$. Equivalent to {\tt chr \$+}. diff --git a/doc/smc-guidelines.txt b/doc/smc-guidelines.txt index 2a4bc1146c..bfac450d49 100644 --- a/doc/smc-guidelines.txt +++ b/doc/smc-guidelines.txt @@ -34,7 +34,7 @@ Most smart contracts should not perform non-trivial actions or reject the inboun * op = 0xffffffff means "operation not supported". It is followed by the 64-bit *query_id* extracted from the original query, and the 32-bit *op* of the original query. All but the simplest smart contracts should return this error when they receive a query with an unknown *op* in the range 1 .. 2^31-1. * op = 0xfffffffe means "operation not allowed". It is followed by the 64-bit *query_id* of the original query, followed by the 32-bit *op* extracted from the original query. -Notice that unknown "responses" (with an *op* in the range 2^31 .. 2^32-1) should be ignored (in particular, no response with an *op* equal to 0xffffffff should be generated in response to them), just as unexpected bounced messages (with "bounced" flag set). +Notice that unknown "responses" (with an *op* in the range 2^31 .. 2^32-1) should be ignored (in particular, no response with an *op* equal to 0xffffffff should be generated in response to them), just as unexpected bounced messages (with "bounced" flag set). 2. Paying for processing queries and sending responses ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/tblkch.tex b/doc/tblkch.tex index e7b3b1957b..e209919035 100644 --- a/doc/tblkch.tex +++ b/doc/tblkch.tex @@ -180,7 +180,7 @@ \section*{Introduction} Most cell operations do not observe any reference cells or other ``exotic'' kinds of cells; they see only data cells, with any reference cell transparently replaced by the cell referred to. For example, when the {\em transparent\/} cell hash $\Hash^\flat(c)$ is recursively computed, the hash of a reference cell is set to be equal to the hash of the cell referred to, not the hash of the standard representation of the reference cell. \nxsubpoint\emb{Transparent hash and representation hash of a cell} -In this way, $\Sha^\flat(c)=\Hash^\flat(c)$ is the {\em transparent hash} of a cell $c$ (or the tree of cells rooted in $c$). +In this way, $\Sha^\flat(c)=\Hash^\flat(c)$ is the {\em transparent hash} of a cell $c$ (or the tree of cells rooted in $c$). However, sometimes we need to reason about the exact representation of a tree of cells present in a block. To this end, a {\em representation hash\/} $\Hash^\sharp(c)$ is defined, which is not transparent with respect to reference cells and other exotic types of cells. We often say that the representation hash of~$c$ is ``the'' hash of~$c$, because it is the most frequently used hash of a cell. @@ -865,12 +865,12 @@ \section*{Introduction} The serialization of source and destination addresses is defined by the following TL-B scheme: \begin{verbatim} addr_none$00 = MsgAddressExt; -addr_extern$01 len:(## 9) external_address:(len * Bit) +addr_extern$01 len:(## 9) external_address:(len * Bit) = MsgAddressExt; anycast_info$_ depth:(## 5) rewrite_pfx:(depth * Bit) = Anycast; -addr_std$10 anycast:(Maybe Anycast) +addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:uint256 = MsgAddressInt; -addr_var$11 anycast:(Maybe Anycast) addr_len:(## 9) +addr_var$11 anycast:(Maybe Anycast) addr_len:(## 9) workchain_id:int32 address:(addr_len * Bit) = MsgAddressInt; _ MsgAddressInt = MsgAddress; _ MsgAddressExt = MsgAddress; @@ -892,9 +892,9 @@ \section*{Introduction} \begin{verbatim} var_uint$_ {n:#} len:(#< n) value:(uint (len * 8)) = VarUInteger n; -var_int$_ {n:#} len:(#< n) value:(int (len * 8)) +var_int$_ {n:#} len:(#< n) value:(int (len * 8)) = VarInteger n; -nanograms$_ amount:(VarUInteger 16) = Grams; +nanograms$_ amount:(VarUInteger 16) = Grams; \end{verbatim} If one wants to represent $x$ nanograms, one selects an integer $\ell<16$ such that $x<2^{8\ell}$, and serializes first $\ell$ as an unsigned 4-bit integer, then $x$ itself as an unsigned $8\ell$-bit integer. Notice that four zero bits represent a zero amount of Grams. @@ -905,9 +905,9 @@ \section*{Introduction} When some amounts of one or several such cryptocurrencies need to be represented, a dictionary (cf.~\cite[3.3]{TVM}) with 32-bit $\currencyid$s as keys and \texttt{VarUInteger 32} values is used: \begin{verbatim} -extra_currencies$_ dict:(HashmapE 32 (VarUInteger 32)) +extra_currencies$_ dict:(HashmapE 32 (VarUInteger 32)) = ExtraCurrencyCollection; -currencies$_ grams:Grams other:ExtraCurrencyCollection +currencies$_ grams:Grams other:ExtraCurrencyCollection = CurrencyCollection; \end{verbatim} The value attached to an internal message is represented by a value of the \texttt{CurrencyCollection} type, which may describe a certain (non-negative) amount of (nano)grams as well as some additional currencies, if needed. Notice that if no additional currencies are required, \texttt{other} reduces to just one zero bit. @@ -916,10 +916,10 @@ \section*{Introduction} A message consists of its {\em header\/} followed by its {\em body}, or {\em payload}. The body is essentially arbitrary, to be interpreted by the destination smart contract. The message header is standard and is organized as follows: \begin{verbatim} int_msg_info$0 ihr_disabled:Bool bounce:Bool - src:MsgAddressInt dest:MsgAddressInt + src:MsgAddressInt dest:MsgAddressInt value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams created_lt:uint64 created_at:uint32 = CommonMsgInfo; -ext_in_msg_info$10 src:MsgAddressExt dest:MsgAddressInt +ext_in_msg_info$10 src:MsgAddressExt dest:MsgAddressInt import_fee:Grams = CommonMsgInfo; ext_out_msg_info$11 src:MsgAddressInt dest:MsgAddressExt created_lt:uint64 created_at:uint32 = CommonMsgInfo; @@ -970,14 +970,14 @@ \section*{Introduction} \nxsubpoint\label{sp:tl.msg.env}\emb{Enveloped messages} {\em Message envelopes\/} are used for attaching routing information, such as the current (transit) address and the next-hop address, to inbound, transit, and outbound messages (cf.~\ptref{sp:msg.env}). The message itself is kept in a separate cell and referred to from the message envelope by a cell reference. \begin{verbatim} -interm_addr_regular$0 use_src_bits:(#<= 96) +interm_addr_regular$0 use_src_bits:(#<= 96) = IntermediateAddress; -interm_addr_simple$10 workchain_id:int8 addr_pfx:(64 * Bit) +interm_addr_simple$10 workchain_id:int8 addr_pfx:(64 * Bit) = IntermediateAddress; interm_addr_ext$11 workchain_id:int32 addr_pfx:(64 * Bit) = IntermediateAddress; -msg_envelope cur_addr:IntermediateAddress - next_addr:IntermediateAddress fwd_fee_remaining:Grams +msg_envelope cur_addr:IntermediateAddress + next_addr:IntermediateAddress fwd_fee_remaining:Grams msg:^(Message Any) = MsgEnvelope; \end{verbatim} The \texttt{IntermediateAddress} type is used to describe the intermediate addresses of a message---that is, its current (or transit) address \texttt{cur\_addr}, and its next-hop address~\texttt{next\_addr}. The first constructor \texttt{interm\_addr\_regular} represents the intermediate address using the optimization described in~\ptref{sp:repr.interm.addr}, by storing the number of the first bits of the intermediate address that are the same as in the source address; the two other explicitly store the workchain identifier and the first 64 bits of the address inside that workchain (the remaining bits can be taken from the source address). The \texttt{fwd\_fee\_remaining} field is used to explicitly represent the maximum amount of message forwarding fees that can be deducted from the message value during the remaining HR steps; it cannot exceed the value of \texttt{fwd\_fee} indicated in the message itself. @@ -1002,19 +1002,19 @@ \section*{Introduction} \nxsubpoint\label{sp:in.msg.d}\emb{Descriptor of an inbound message} Each inbound message is described by an instance of the \texttt{InMsg} type, which has six constructors corresponding to the cases listed above in~\ptref{sp:inb.msg.classes}: \begin{verbatim} -msg_import_ext$000 msg:^(Message Any) transaction:^Transaction +msg_import_ext$000 msg:^(Message Any) transaction:^Transaction = InMsg; -msg_import_ihr$010 msg:^(Message Any) transaction:^Transaction +msg_import_ihr$010 msg:^(Message Any) transaction:^Transaction ihr_fee:Grams proof_created:^Cell = InMsg; msg_import_imm$011 in_msg:^MsgEnvelope transaction:^Transaction fwd_fee:Grams = InMsg; -msg_import_fin$100 in_msg:^MsgEnvelope +msg_import_fin$100 in_msg:^MsgEnvelope transaction:^Transaction fwd_fee:Grams = InMsg; -msg_import_tr$101 in_msg:^MsgEnvelope out_msg:^MsgEnvelope +msg_import_tr$101 in_msg:^MsgEnvelope out_msg:^MsgEnvelope transit_fee:Grams = InMsg; -msg_discard_fin$110 in_msg:^MsgEnvelope transaction_id:uint64 +msg_discard_fin$110 in_msg:^MsgEnvelope transaction_id:uint64 fwd_fee:Grams = InMsg; -msg_discard_tr$111 in_msg:^MsgEnvelope transaction_id:uint64 +msg_discard_tr$111 in_msg:^MsgEnvelope transaction_id:uint64 fwd_fee:Grams proof_delivered:^Cell = InMsg; \end{verbatim} Notice that the processing transaction is referred to in the first four constructors directly by a cell reference to \texttt{Transaction}, even though the logical time of the transaction \texttt{transaction\_lt:uint64} would suffice for this purpose. Internal consistency conditions ensure that the transaction referred to does belong to the destination smart contract indicated in the message, and that the inbound message processed by that transaction is indeed the one being described in this {\em InMsg\/} instance. @@ -1051,16 +1051,16 @@ \section*{Introduction} \nxsubpoint\label{sp:aug.hm.ser}\emb{Serialization of augmented hashmaps} The serialization of augmented hashmaps with $n$-bit keys, values of type $X$, and augmentation values of type $Y$ is given by the following TL-B scheme, which is an extension of the one provided in~\cite[3.3.3]{TVM}: \begin{verbatim} -ahm_edge#_ {n:#} {X:Type} {Y:Type} {l:#} {m:#} - label:(HmLabel ~l n) {n = (~m) + l} +ahm_edge#_ {n:#} {X:Type} {Y:Type} {l:#} {m:#} + label:(HmLabel ~l n) {n = (~m) + l} node:(HashmapAugNode m X Y) = HashmapAug n X Y; -ahmn_leaf#_ {X:Type} {Y:Type} extra:Y value:X +ahmn_leaf#_ {X:Type} {Y:Type} extra:Y value:X = HashmapAugNode 0 X Y; ahmn_fork#_ {n:#} {X:Type} {Y:Type} - left:^(HashmapAug n X Y) right:^(HashmapAug n X Y) extra:Y + left:^(HashmapAug n X Y) right:^(HashmapAug n X Y) extra:Y = HashmapAugNode (n + 1) X Y; -ahme_empty$0 {n:#} {X:Type} {Y:Type} extra:Y +ahme_empty$0 {n:#} {X:Type} {Y:Type} extra:Y = HashmapAugE n X Y; ahme_root$1 {n:#} {X:Type} {Y:Type} root:^(HashmapAug n X Y) extra:Y = HashmapAugE n X Y; @@ -1069,7 +1069,7 @@ \section*{Introduction} \nxsubpoint\label{sp:in.msg.augm}\emb{Augmentation of {\em InMsgDescr}} The collection of inbound message descriptors is augmented by a vector of two currency values, representing the imported value and the forwarding and transit fees collected from a message or a collection of messages: \begin{verbatim} -import_fees$_ fees_collected:Grams +import_fees$_ fees_collected:Grams value_imported:CurrencyCollection = ImportFees; \end{verbatim} @@ -1105,15 +1105,15 @@ \section*{Introduction} \begin{verbatim} msg_export_ext$000 msg:^(Message Any) transaction:^Transaction = OutMsg; -msg_export_imm$010 out_msg:^MsgEnvelope +msg_export_imm$010 out_msg:^MsgEnvelope transaction:^Transaction reimport:^InMsg = OutMsg; -msg_export_new$001 out_msg:^MsgEnvelope +msg_export_new$001 out_msg:^MsgEnvelope transaction:^Transaction = OutMsg; -msg_export_tr$011 out_msg:^MsgEnvelope +msg_export_tr$011 out_msg:^MsgEnvelope imported:^InMsg = OutMsg; -msg_export_deq$110 out_msg:^MsgEnvelope +msg_export_deq$110 out_msg:^MsgEnvelope import_block_lt:uint64 = OutMsg; -msg_export_tr_req$111 out_msg:^MsgEnvelope +msg_export_tr_req$111 out_msg:^MsgEnvelope imported:^InMsg = OutMsg; \end{verbatim} The last two descriptions have the effect of removing (dequeueing) the message from {\em OutMsgQueue\/} instead of inserting it. The last one re-inserts the message into {\em OutMsgQueue\/} with a new envelope after performing the internal routing (cf.~\ptref{sp:hr.int.route}). @@ -1189,8 +1189,8 @@ \section*{Introduction} \nxsubpoint\emb{Storage profile of an account} The {\em storage profile\/} of an account is a data structure describing the amount of persistent blockchain state storage used by that account. It describes the total amount of cells, data bits, and internal and external cell references used. \begin{verbatim} -storage_used$_ cells:(VarUInteger 7) bits:(VarUInteger 7) - ext_refs:(VarUInteger 7) int_refs:(VarUInteger 7) +storage_used$_ cells:(VarUInteger 7) bits:(VarUInteger 7) + ext_refs:(VarUInteger 7) int_refs:(VarUInteger 7) public_cells:(VarUInteger 7) = StorageUsed; \end{verbatim} The same type {\tt StorageUsed} may represent the storage profile of a message, as required, for instance, to compute {\tt fwd\_fee}, the total forwarding fee for Hypercube Routing. The storage profile of an account has some additional fields indicating the last time when the storage fees were exacted: @@ -1208,7 +1208,7 @@ \section*{Introduction} storage:AccountStorage = Account; account_storage$_ last_trans_lt:uint64 - balance:CurrencyCollection state:AccountState + balance:CurrencyCollection state:AccountState = AccountStorage; account_uninit$00 = AccountState; @@ -1372,7 +1372,7 @@ \section*{Introduction} \begin{verbatim} transaction$_ account_addr:uint256 lt:uint64 outmsg_cnt:uint15 orig_status:AccountStatus end_status:AccountStatus - in_msg:(Maybe ^(Message Any)) + in_msg:(Maybe ^(Message Any)) out_msgs:(HashmapE 15 ^(Message Any)) total_fees:Grams state_update:^(MERKLE_UPDATE Account) description:^TransactionDescr = Transaction; @@ -1450,7 +1450,7 @@ \section*{Introduction} \nxsubpoint\emb{Description of a storage phase} The storage phase is present in several kinds of transactions, so a common representation for this phase is used: \begin{verbatim} -tr_phase_storage$_ storage_fees_collected:Grams +tr_phase_storage$_ storage_fees_collected:Grams storage_fees_due:(Maybe Grams) status_change:AccStatusChange = TrStoragePhase; @@ -1473,7 +1473,7 @@ \section*{Introduction} \begin{verbatim} tr_phase_compute_skipped$0 reason:ComputeSkipReason = TrComputePhase; -tr_phase_compute_vm$1 success:Bool msg_state_used:Bool +tr_phase_compute_vm$1 success:Bool msg_state_used:Bool account_activated:Bool gas_fees:Grams _:^[ gas_used:(VarUInteger 7) gas_limit:(VarUInteger 7) gas_credit:(Maybe (VarUInteger 3)) @@ -1503,7 +1503,7 @@ \section*{Introduction} \item The {\tt gas\_fees} parameter reflects the total gas fees collected by the validators for executing this transaction. It must be equal to the product of {\tt gas\_used} and {\tt gas\_price} from the current block header. \item The {\tt gas\_limit} parameter reflects the gas limit for this instance of TVM. It equals the lesser of either the Grams credited in the credit phase from the value of the inbound message divided by the current gas price, or the global per-transaction gas limit. \item The {\tt gas\_credit} parameter may be non-zero only for external inbound messages. It is the lesser of either the amount of gas that can be paid from the account balance or the maximum gas credit. -\item The {\tt exit\_code} and {\tt exit\_args} parameters represent the status values returned by TVM. +\item The {\tt exit\_code} and {\tt exit\_args} parameters represent the status values returned by TVM. \item The {\tt vm\_init\_state\_hash} and {\tt vm\_final\_state\_hash} parameters are the representation hashes of the original and resulting states of TVM, and {\tt vm\_steps} is the total number of steps performed by TVM (usually equal to two plus the number of instructions executed, including implicit {\tt RET}s).\footnote{Notice that this record does not represent a change in the state of the account, because the transaction may still be aborted during the action phase. In that case, the new persistent data indirectly referenced by {\tt vm\_final\_state\_hash} will be discarded.} \end{itemize} @@ -1514,8 +1514,8 @@ \section*{Introduction} status_change:AccStatusChange total_fwd_fees:(Maybe Grams) total_action_fees:(Maybe Grams) result_code:int32 result_arg:(Maybe int32) tot_actions:int16 - spec_actions:int16 msgs_created:int16 - action_list_hash:uint256 tot_msg_size:StorageUsed + spec_actions:int16 msgs_created:int16 + action_list_hash:uint256 tot_msg_size:StorageUsed = TrActionPhase; \end{verbatim} @@ -1524,7 +1524,7 @@ \section*{Introduction} tr_phase_bounce_negfunds$00 = TrBouncePhase; tr_phase_bounce_nofunds$01 msg_size:StorageUsed req_fwd_fees:Grams = TrBouncePhase; -tr_phase_bounce_ok$1 msg_size:StorageUsed +tr_phase_bounce_ok$1 msg_size:StorageUsed msg_fees:Grams fwd_fees:Grams = TrBouncePhase; \end{verbatim} @@ -1687,9 +1687,9 @@ \section*{Introduction} The smart-contract information structure {\em SmartContractInfo}, passed in the first component of the {\em Tuple\/} contained in control register {\tt c7}, is also a {\em Tuple} containing the following data: \begin{verbatim} [ magic:0x076ef1ea actions:Integer msgs_sent:Integer - unixtime:Integer block_lt:Integer trans_lt:Integer + unixtime:Integer block_lt:Integer trans_lt:Integer rand_seed:Integer balance_remaining:[Integer (Maybe Cell)] - myself:MsgAddressInt global_config:(Maybe Cell) + myself:MsgAddressInt global_config:(Maybe Cell) ] = SmartContractInfo; \end{verbatim} In other words, the first component of this tuple is an {\em Integer\/} {\tt magic\/} always equal to {\tt 0x076ef1ea}, the second component is an {\em Integer\/} {\tt actions}, originally initialized by zero, but incremented by one whenever an output action is installed by a non-{\tt RAW} output action primitive of the TVM, and so on. The remaining balance is represented by a pair, i.e., a two-component {\em Tuple}: the first component is the nanogram balance, and the second component is a dictionary with 32-bit keys representing all other currencies, if any (cf.~\ptref{sp:extra.curr}). @@ -1755,10 +1755,10 @@ \section*{Introduction} master_info$_ master:ExtBlkRef = BlkMasterInfo; -shard_ident$00 shard_pfx_bits:(## 6) +shard_ident$00 shard_pfx_bits:(## 6) workchain_id:int32 shard_prefix:uint64 = ShardIdent; -shard_state shard_id:ShardIdent +shard_state shard_id:ShardIdent out_msg_queue:OutMsgQueue total_balance:CurrencyCollection total_validator_fees:CurrencyCollection @@ -1781,13 +1781,13 @@ \section*{Introduction} \nxsubpoint\emb{TL-B scheme for an unsigned shardchain block} The precise format of an {\em unsigned\/} (cf.~\ptref{sp:val.sign}) shardchain block is given by the following TL-B scheme: \begin{verbatim} -block_info version:uint32 - not_master:(## 1) +block_info version:uint32 + not_master:(## 1) after_merge:(## 1) before_split:(## 1) flags:(## 13) - seq_no:# vert_seq_no:# + seq_no:# vert_seq_no:# shard:ShardIdent gen_utime:uint32 start_lt:uint64 end_lt:uint64 - master_ref:not_master?^BlkMasterInfo + master_ref:not_master?^BlkMasterInfo prev_ref:seq_no?^(BlkPrevInfo after_merge) prev_vert_ref:vert_seq_no?^(BlkPrevInfo 0) = BlockInfo; @@ -1796,7 +1796,7 @@ \section*{Introduction} prev_alt:merged?ExtBlkRef = BlkPrevInfo merged; unsigned_block info:^BlockInfo value_flow:^ValueFlow - state_update:^(MERKLE_UPDATE ShardState) + state_update:^(MERKLE_UPDATE ShardState) extra:^BlockExtra = Block; block_extra in_msg_descr:^InMsgDescr @@ -1810,7 +1810,7 @@ \section*{Introduction} \nxsubpoint\emb{Description of total value flow through a block} The total value flow through a block is serialized according to the following TL-B scheme: \begin{verbatim} -value_flow _:^[ from_prev_blk:CurrencyCollection +value_flow _:^[ from_prev_blk:CurrencyCollection to_next_blk:CurrencyCollection imported:CurrencyCollection exported:CurrencyCollection ] @@ -1824,9 +1824,9 @@ \section*{Introduction} Recall that \texttt{\_:\caret[}\dots\texttt{]} is a TL-B construction indicating that a group of fields has been moved into a separate cell. The last three fields may be non-zero only in masterchain blocks. \nxsubpoint\label{sp:signed.blk}\emb{Signed shardchain block} -A signed shardchain block is just an unsigned block augmented by a collection of validator signatures: +A signed shardchain block is just an unsigned block augmented by a collection of validator signatures: \begin{verbatim} -ed25519_signature#5 R:uint256 s:uint256 = CryptoSignature; +ed25519_signature#5 R:uint256 s:uint256 = CryptoSignature; signed_block block:^Block blk_serialize_hash:uint256 signatures:(HashmapE 64 CryptoSignature) @@ -1871,14 +1871,14 @@ \section*{Introduction} {\em ShardHashes\/} is represented by a dictionary with 32-bit $\workchainid$s as keys, and ``shard binary trees'', represented by TL-B type {\em BinTree ShardDescr}, as values. Each leaf of this shard binary tree contains a value of type {\em ShardDescr}, which describes a single shard by indicating the sequence number {\tt seq\_no}, the logical time {\tt lt}, and the hash {\tt hash} of the latest (signed) block of the corresponding shardchain. \begin{verbatim} bt_leaf$0 {X:Type} leaf:X = BinTree X; -bt_fork$1 {X:Type} left:^(BinTree X) right:^(BinTree X) +bt_fork$1 {X:Type} left:^(BinTree X) right:^(BinTree X) = BinTree X; fsm_none$0 = FutureSplitMerge; fsm_split$10 mc_seqno:uint32 = FutureSplitMerge; fsm_merge$11 mc_seqno:uint32 = FutureSplitMerge; -shard_descr$_ seq_no:uint32 lt:uint64 hash:uint256 +shard_descr$_ seq_no:uint32 lt:uint64 hash:uint256 split_merge_at:FutureSplitMerge = ShardDescr; _ (HashmapE 32 ^(BinTree ShardDescr)) = ShardHashes; @@ -1891,7 +1891,7 @@ \section*{Introduction} {\em ShardFees\/} is a masterchain structure used to reflect the total fees collected so far by the validators of a shardchain. The total fees reflected in this structure are accumulated in the masterchain by crediting them to a special account, whose address is a configurable parameter. Typically this account is the smart contract that computes and distributes the rewards to all validators. \begin{verbatim} bta_leaf$0 {X:Type} {Y:Type} leaf:X extra:Y = BinTreeAug X Y; -bta_fork$1 {X:Type} {Y:Type} left:^(BinTreeAug X Y) +bta_fork$1 {X:Type} {Y:Type} left:^(BinTreeAug X Y) right:^(BinTreeAug X Y) extra:Y = BinTreeAug X Y; _ (HashmapAugE 32 ^(BinTreeAug True CurrencyCollection) @@ -1902,7 +1902,7 @@ \section*{Introduction} \nxsubpoint\emb{Description of {\em ConfigParams}} Recall that the {\em configurable parameters\/} or the {\em configuration dictionary\/} is a dictionary {\tt config} with 32-bit keys kept inside the first cell reference of the persistent data of the configuration smart contract $\gamma$ (cf.~\ptref{p:conf.params}). The address $\gamma$ of the configuration smart contract and a copy of the configuration dictionary are duplicated in fields {\tt config\_addr} and {\tt config} of a {\em ConfigParams\/} structure, explicitly included into masterchain state to facilitate access to the current values of the configurable parameters (cf.~\ptref{sp:conf.par.qa}): \begin{verbatim} -_ config_addr:uint256 config:^(Hashmap 32 ^Cell) +_ config_addr:uint256 config:^(Hashmap 32 ^Cell) = ConfigParams; \end{verbatim} @@ -2002,11 +2002,11 @@ \section*{Introduction} \nxsubpoint\label{sp:boc.ser.sch}\emb{TL-B scheme for serializing bags of cells} Several TL-B constructors can be used to serialize bags of cells into octet (i.e., 8-bit byte) sequences. The only one that is currently used to serialize new bags of cell is \begin{verbatim} -serialized_boc#b5ee9c72 has_idx:(## 1) has_crc32c:(## 1) +serialized_boc#b5ee9c72 has_idx:(## 1) has_crc32c:(## 1) has_cache_bits:(## 1) flags:(## 2) { flags = 0 } size:(## 3) { size <= 4 } - off_bytes:(## 8) { off_bytes <= 8 } - cells:(##(size * 8)) + off_bytes:(## 8) { off_bytes <= 8 } + cells:(##(size * 8)) roots:(##(size * 8)) { roots >= 1 } absent:(##(size * 8)) { roots + absent <= cells } tot_cells_size:(##(off_bytes * 8)) @@ -2021,8 +2021,8 @@ \section*{Introduction} Two older constructors are still supported in the bag-of-cells deserialization functions: \begin{verbatim} serialized_boc_idx#68ff65f3 size:(## 8) { size <= 4 } - off_bytes:(## 8) { off_bytes <= 8 } - cells:(##(size * 8)) + off_bytes:(## 8) { off_bytes <= 8 } + cells:(##(size * 8)) roots:(##(size * 8)) { roots = 1 } absent:(##(size * 8)) { roots + absent <= cells } tot_cells_size:(##(off_bytes * 8)) @@ -2031,8 +2031,8 @@ \section*{Introduction} = BagOfCells; serialized_boc_idx_crc32c#acc3a728 size:(## 8) { size <= 4 } - off_bytes:(## 8) { off_bytes <= 8 } - cells:(##(size * 8)) + off_bytes:(## 8) { off_bytes <= 8 } + cells:(##(size * 8)) roots:(##(size * 8)) { roots = 1 } absent:(##(size * 8)) { roots + absent <= cells } tot_cells_size:(##(off_bytes * 8)) @@ -2051,7 +2051,7 @@ \section*{Introduction} compiler_version:(Maybe ^TinyString) ] = CompiledSmartContract; -tiny_string#_ len:(#<= 126) str:(len * [ uint8 ]) = TinyString; +tiny_string#_ len:(#<= 126) str:(len * [ uint8 ]) = TinyString; \end{verbatim} Then a compiled smart contract may be represented by a value of type {\em CompiledSmartContract}, transformed into a tree of cells and then into a bag of cells, and then serialized using one of the constructors listed in~\ptref{sp:boc.ser.sch}. The resulting octet string may be then written into a file with suffix {\tt .tvc} (``TVM smart contract''), and this file may be used to distribute the compiled smart contract, download it into a wallet application for deploying into the TON Blockchain, and so on. @@ -2172,10 +2172,10 @@ \section*{Introduction} a finite base field $\bbF_q$ (if $q=p$ is a prime, it is sufficient to fix prime~$p$), an elliptic curve $E/\bbF_q$ (usually represented by the coefficients of its Weierstrass form \eqref{eq:weier.form} or~\eqref{eq:gen.weier.form}), the base point $O$ (which usually is the point at infinity of an elliptic curve written in Weierstrass form), and the generator $G\in E(\bbF_q)$ (usually determined by its coordinates $(x,y)$ with respect to the equation of the elliptic curve) of a cyclic subgroup of large prime order~$\ell$. Prime number $\ell$ and the cofactor $c$ are usually also part of the elliptic cryptography data. \nxsubpoint\emb{Main operations of elliptic curve cryptography} -Elliptic curve cryptography usually deals with a fixed cyclic subgroup $C$ of a large prime order $\ell$ inside the group of points of an elliptic curve $E$ over a finite field~$\bbF_q$. A generator $G$ of $C$ is usually fixed. It is usually assumed that, given a point $X$ of $C$, one cannot find its ``discrete logarithm base $G$'' (i.e., a residue $n$ modulo $\ell$ such that $X=[n]G$) faster than in $O(\sqrt\ell)$ operations. The most important operations used in elliptic curve cryptography are the addition of points from $C\subset E(\bbF_q)$ and the computation of their powers, or multiples. +Elliptic curve cryptography usually deals with a fixed cyclic subgroup $C$ of a large prime order $\ell$ inside the group of points of an elliptic curve $E$ over a finite field~$\bbF_q$. A generator $G$ of $C$ is usually fixed. It is usually assumed that, given a point $X$ of $C$, one cannot find its ``discrete logarithm base $G$'' (i.e., a residue $n$ modulo $\ell$ such that $X=[n]G$) faster than in $O(\sqrt\ell)$ operations. The most important operations used in elliptic curve cryptography are the addition of points from $C\subset E(\bbF_q)$ and the computation of their powers, or multiples. \nxsubpoint\emb{Private and public keys for elliptic curve cryptography} -Usually a private key for elliptic curve cryptography described by the data listed in~\ptref{sp:ecc.data} is a ``random'' integer $0= 1 } rewrite_pfx:(bits depth) = Anycast; -addr_std$10 anycast:(Maybe Anycast) +addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; -addr_var$11 anycast:(Maybe Anycast) addr_len:(## 9) +addr_var$11 anycast:(Maybe Anycast) addr_len:(## 9) workchain_id:int32 address:(bits addr_len) = MsgAddressInt; _ _:MsgAddressInt = MsgAddress; _ _:MsgAddressExt = MsgAddress; int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool - src:MsgAddress dest:MsgAddressInt + src:MsgAddress dest:MsgAddressInt value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; ext_out_msg_info$11 src:MsgAddress dest:MsgAddressExt @@ -2382,9 +2382,9 @@ \section*{Introduction} vm_stk_int#0201_ value:int257 = VmStackValue; vm_stk_nan#02FF = VmStackValue; vm_stk_cell#03 cell:^Cell = VmStackValue; -_ cell:^Cell st_bits:(## 10) end_bits:(## 10) - { st_bits <= end_bits } - st_ref:(#<= 4) end_ref:(#<= 4) +_ cell:^Cell st_bits:(## 10) end_bits:(## 10) + { st_bits <= end_bits } + st_ref:(#<= 4) end_ref:(#<= 4) { st_ref <= end_ref } = VmCellSlice; vm_stk_slice#04 _:VmCellSlice = VmStackValue; vm_stk_builder#05 cell:^Cell = VmStackValue; @@ -2396,7 +2396,7 @@ \section*{Introduction} The TVM stack can be serialized as follows: \begin{verbatim} vm_stack#_ depth:(## 24) stack:(VmStackList depth) = VmStack; -vm_stk_cons#_ {n:#} head:VmStackValue tail:^(VmStackList n) +vm_stk_cons#_ {n:#} head:VmStackValue tail:^(VmStackList n) = VmStackList (n + 1); vm_stk_nil#_ = VmStackList 0; \end{verbatim} @@ -2410,7 +2410,7 @@ \section*{Introduction} \nxsubpoint\emb{TVM gas limits} Gas limits in TVM can be serialized as follows: \begin{verbatim} -gas_limits#_ remaining:int64 _:^[ +gas_limits#_ remaining:int64 _:^[ max_limit:int64 cur_limit:int64 credit:int64 ] = VmGasLimits; \end{verbatim} @@ -2426,13 +2426,13 @@ \section*{Introduction} \begin{verbatim} vmc_std$00 nargs:(## 22) stack:(Maybe VmStack) save:VmSaveList cp:int16 code:VmCellSlice = VmCont; -vmc_envelope$01 nargs:(## 22) stack:(Maybe VmStack) +vmc_envelope$01 nargs:(## 22) stack:(Maybe VmStack) save:VmSaveList next:^VmCont = VmCont; vmc_quit$1000 exit_code:int32 = VmCont; vmc_quit_exc$1001 = VmCont; vmc_until$1010 body:^VmCont after:^VmCont = VmCont; vmc_again$1011 body:^VmCont = VmCont; -vmc_while_cond$1100 cond:^VmCont body:^VmCont +vmc_while_cond$1100 cond:^VmCont body:^VmCont after:^VmCont = VmCont; vmc_while_body$1101 cond:^VmCont body:^VmCont after:^VmCont = VmCont; @@ -2442,17 +2442,17 @@ \section*{Introduction} \nxsubpoint\emb{TVM state} The total state of TVM can be serialized as follows: \begin{verbatim} -vms_init$00 cp:int16 step:int32 gas:GasLimits +vms_init$00 cp:int16 step:int32 gas:GasLimits stack:(Maybe VmStack) save:VmSaveList code:VmCellSlice lib:VmLibraries = VmState; -vms_exception$01 cp:int16 step:int32 gas:GasLimits - exc_no:int32 exc_arg:VmStackValue +vms_exception$01 cp:int16 step:int32 gas:GasLimits + exc_no:int32 exc_arg:VmStackValue save:VmSaveList lib:VmLibraries = VmState; vms_running$10 cp:int16 step:int32 gas:GasLimits stack:VmStack save:VmSaveList code:VmCellSlice lib:VmLibraries = VmState; -vms_finished$11 cp:int16 step:int32 gas:GasLimits - exit_code:int32 no_gas:Boolean stack:VmStack +vms_finished$11 cp:int16 step:int32 gas:GasLimits + exit_code:int32 no_gas:Boolean stack:VmStack save:VmSaveList lib:VmLibraries = VmState; \end{verbatim} When TVM is initialized, its state is described by a {\tt vms\_init}, usually with {\tt step} set to zero. The step function of TVM does nothing to a {\tt vms\_finished} state, and transforms all other states into {\tt vms\_running}, {\tt vms\_exception}, or {\tt vms\_finished}, with {\tt step} increased by one. @@ -2529,7 +2529,7 @@ \section*{Introduction} IDIV r1,r1,r6 // y := Dy/D RET \end{verbatim} -We have used 12 operations and at least 23 bytes (each operation uses $3\times 4=12$ bits to indicate the three registers involved, and at least 4 bits to indicate the operation performed; thus we need two or three bytes to encode each operation). A more realistic estimate would be 34 (three bytes for each arithmetic operation) or 31 bytes (two bytes for addition and subtraction, three bytes for multiplication and division). +We have used 12 operations and at least 23 bytes (each operation uses $3\times 4=12$ bits to indicate the three registers involved, and at least 4 bits to indicate the operation performed; thus we need two or three bytes to encode each operation). A more realistic estimate would be 34 (three bytes for each arithmetic operation) or 31 bytes (two bytes for addition and subtraction, three bytes for multiplication and division). \nxsubpoint\label{sp:cmp1.2addr}\emb{Two-address register machine} The machine code for a two-address register machine might look as follows: @@ -2707,7 +2707,7 @@ \section*{Introduction} \hline &\multicolumn{3}{|c|}{Operations}&\multicolumn{3}{|c|}{Code bytes}&\multicolumn{3}{|c|}{Opcode space}\\ Machine&data&arith&total&data&arith&total&data&arith&total\\ -\hline +\hline 3-addr. (opt.)& 0 & 11 & 12 & 0 & 22 & 23 & 0/256 & 64/256 & 65/256 \\ 3-addr. (real.)& 0 & 11 & 12 & 0 & 30 & 31 & 0/256 & 34/256 & 35/256 \\ \hline @@ -2722,7 +2722,7 @@ \section*{Introduction} } \caption{A summary of machine code properties for hypothetical 3-address, 2-address, 1-address, and stack machines, generated for a sample leaf function (cf.~\ptref{sp:cmp1.source}). The two most important columns, reflecting {\bf code density} and {\bf extendability} to other operations, are marked by bold font. Smaller values are better in both of these columns.}\label{tab:cmp1.code} \end{table} - + The ``code bytes'' column reflects the density of the code for the specific sample source. However, ``opcode space'' is also important, because it reflects the extendability of the achieved density to other classes of operations (e.g., if one were to complement arithmetic operations with string manipulation operations and so on). Here the ``arithmetic'' subcolumn is more important than the ``data'' subcolumn, because no further data manipulation operations would be required for such extensions. We see that the three-address register machine with the ``optimistic'' encoding, assuming two-byte encodings for all three-register arithmetic operations, achieves the best code density, requiring only 23 bytes. However, this comes at a price: each arithmetic operation consumes 1/16 of the opcode space, so the four operations already use a quarter of the opcode space. At most 11 other operations, arithmetic or not, might be added to this architecture while preserving such high code density. On the other hand, when we consider the ``realistic'' encoding for the three-address machine, using two-byte encodings only for the most frequently used addition/subtraction operations (and longer encodings for less frequently used multiplication/division operations, reflecting the fact that the possible extension operations would likely fall in this class), then the three-address machine ceases to offer such attractive code density. @@ -2762,7 +2762,7 @@ \section*{Introduction} \hline &&\multicolumn{3}{|c|}{Operations}&\multicolumn{3}{|c|}{Code bytes}&\multicolumn{3}{|c|}{Opcode space}\\ Machine&$r$&data&arith&total&data&arith&total&data&arith&total\\ -\hline +\hline 3-addr. (opt.)& 4 & 8 & 11 & 20 & 8 & 22 & 31 & 32/256 & 64/256 & 97/256 \\ 3-addr. (real.)& 4 & 8 & 11 & 20 & 8 & 30 & 39 & 32/256 & 34/256 & 67/256 \\ \hline @@ -2788,7 +2788,7 @@ \section*{Introduction} \hline &&\multicolumn{3}{|c|}{Operations}&\multicolumn{3}{|c|}{Code bytes}&\multicolumn{3}{|c|}{Opcode space}\\ Machine&$r$&data&arith&total&data&arith&total&data&arith&total\\ -\hline +\hline 3-addr. (opt.)& 0 & 0 & 11 & 12 & 0 & 22 & 23 & 32/256 & 64/256 & 97/256 \\ 3-addr. (real.)& 0 & 0 & 11 & 12 & 0 & 30 & 31 & 32/256 & 34/256 & 67/256 \\ \hline @@ -2821,7 +2821,7 @@ \section*{Introduction} \hline &&\multicolumn{3}{|c|}{Operations}&\multicolumn{3}{|c|}{Code bytes}&\multicolumn{3}{|c|}{Opcode space}\\ Machine&$r$&data&arith&total&data&arith&total&data&arith&total\\ -\hline +\hline 3-addr.& 0 & 0 & 11 & 12 & 0 & 27.5 & 28.5 & 64/256 & 4/256 & 69/256 \\ \hline 2-addr. & 0 & 4 & 11 & 16 & 6 & 22 & 29 & 64/256 & 4/256 & 69/256 \\ @@ -3136,12 +3136,12 @@ \section*{Introduction} \hline &&\multicolumn{3}{|c|}{Operations}&\multicolumn{3}{|c|}{Code bytes}&\multicolumn{3}{|c|}{Opcode space}\\ Machine&$m$&data&cont.&total&data&cont.&total&data&arith&total\\ -\hline -\tworow{3-addr.}& 0,8 & 29 & 12 & 41 & 42 & 34 & 76 & \tworow{35/256} & \tworow{34/256} & \tworow{72/256} \\ +\hline +\tworow{3-addr.}& 0,8 & 29 & 12 & 41 & 42 & 34 & 76 & \tworow{35/256} & \tworow{34/256} & \tworow{72/256} \\ & 16 & 27 & 12 & 39 & 44 & 34 & 78 &&& \\ \hline -\tworow{2-addr.}& 0,8 & 29 & 12 & 41 & 42 & 34 & 76 & \tworow{37/256} & \tworow{4/256} & \tworow{44/256} \\ - & 16 & 27 & 12 & 39 & 44 & 34 & 78 &&&\\ +\tworow{2-addr.}& 0,8 & 29 & 12 & 41 & 42 & 34 & 76 & \tworow{37/256} & \tworow{4/256} & \tworow{44/256} \\ + & 16 & 27 & 12 & 39 & 44 & 34 & 78 &&&\\ \hline \tworow{1-addr.}& 0,8 & 33 & 12 & 45 & 33 & 34 & 67 & \tworow{97/256} & \tworow{64/256} & \tworow{164/256} \\ & 16 & 28 & 12 & 40 & 39 & 34 & 73 &&& \\ @@ -3167,12 +3167,12 @@ \section*{Introduction} \hline &&\multicolumn{3}{|c|}{Operations}&\multicolumn{3}{|c|}{Code bytes}&\multicolumn{3}{|c|}{Opcode space}\\ Machine&$m$&data&cont.&total&data&cont.&total&data&arith&total\\ -\hline -\tworow{3-addr.}& 0,8 & 29 & 12 & 41 & 35.5 & 34 & 69.5 & \tworow{110/256} & \tworow{4/256} & \tworow{117/256} \\ +\hline +\tworow{3-addr.}& 0,8 & 29 & 12 & 41 & 35.5 & 34 & 69.5 & \tworow{110/256} & \tworow{4/256} & \tworow{117/256} \\ & 16 & 27 & 12 & 39 & 35.5 & 34 & 69.5 &&& \\ \hline -\tworow{2-addr.}& 0,8 & 29 & 12 & 41 & 35.5 & 34 & 69.5 & \tworow{110/256} & \tworow{4/256} & \tworow{117/256} \\ - & 16 & 27 & 12 & 39 & 35.5 & 34 & 69.5 &&&\\ +\tworow{2-addr.}& 0,8 & 29 & 12 & 41 & 35.5 & 34 & 69.5 & \tworow{110/256} & \tworow{4/256} & \tworow{117/256} \\ + & 16 & 27 & 12 & 39 & 35.5 & 34 & 69.5 &&&\\ \hline \tworow{1-addr.}& 0,8 & 33 & 12 & 45 & 33 & 34 & 67 & \tworow{112/256} & \tworow{4/256} & \tworow{119/256} \\ & 16 & 28 & 12 & 40 & 33.5 & 34 & 67.5 &&& \\ diff --git a/docker/README.md b/docker/README.md index 47e109dbfa..a65666fd03 100644 --- a/docker/README.md +++ b/docker/README.md @@ -8,12 +8,12 @@ 6. [Troubleshooting](#troubleshooting) ## Prerequisites -The TON node, whether it is validator or fullnode, requires a public IP address. +The TON node, whether it is validator or fullnode, requires a public IP address. If your server is within an internal network or kubernetes you have to make sure that the required ports are available from the outside. -Also pay attention at [hardware requirements](https://docs.ton.org/participate/run-nodes/full-node) for TON fullnodes and validators. Pods and StatefulSets in this guide imply these requirements. +Also pay attention at [hardware requirements](https://docs.ton.org/participate/run-nodes/full-node) for TON fullnodes and validators. Pods and StatefulSets in this guide imply these requirements. -It is recommended to everyone to read Docker chapter first in order to get a better understanding about TON Docker image and its parameters. +It is recommended to everyone to read Docker chapter first in order to get a better understanding about TON Docker image and its parameters. ## Docker @@ -21,8 +21,8 @@ It is recommended to everyone to read Docker chapter first in order to get a bet ```docker pull ghcr.io/ton-blockchain/ton:latest``` ### Configuration -TON validator-engine supports number of command line parameters, -these parameters can be handed over to the container via environment variables. +TON validator-engine supports number of command line parameters, +these parameters can be handed over to the container via environment variables. Below is the list of supported arguments and their default values: | Argument | Description | Mandatory? | Default value | @@ -66,7 +66,7 @@ docker run -d --name ton-node -v /data/db:/var/ton-work/db \ If you don't need Lite-server, then remove -e "LITESERVER=true". ### Run the node - isolated way -In production environments it is recommended to use **Port mapping** feature of Docker's default bridge network. +In production environments it is recommended to use **Port mapping** feature of Docker's default bridge network. When you use port mapping, Docker allocates a specific port on the host to forward traffic to a port inside the container. This is ideal for running multiple containers with isolated networks on the same host. ``` @@ -82,7 +82,7 @@ docker run -d --name ton-node -v /data/db:/var/ton-work/db \ -p 443:443/tcp \ -it ghcr.io/ton-blockchain/ton ``` -Adjust ports per your need. +Adjust ports per your need. Check your firewall configuration and make sure that customized ports (443/udp, 88/tcp and 443/tcp in this example) are publicly available. ### Verify if TON node is operating correctly @@ -106,7 +106,7 @@ finished downloading state (-1,8000000000000000,38585739):9E86E166AE7E24BAA22762 getnextkey: [Error : 651 : not inited] getnextkey: [Error : 651 : not inited] ``` -As you noticed we have mounted docker volume to a local folder **/data/db**. +As you noticed we have mounted docker volume to a local folder **/data/db**. Go inside this folder on your server and check if its size is growing (```sudo du -h .*```) Now connect to the running container: @@ -164,12 +164,12 @@ docker stop ton-node ## Kubernetes ### Deploy in a quick way (without load balancer) -If the nodes within your kubernetes cluster have external IPs, +If the nodes within your kubernetes cluster have external IPs, make sure that the PUBLIC_IP used for validator-engine matches the node's external IP. If all Kubernetes nodes are inside DMZ - skip this section. #### Prepare -If you are using **flannel** network driver you can find node's IP this way: +If you are using **flannel** network driver you can find node's IP this way: ```yaml kubectl get nodes kubectl describe node | grep public-ip @@ -187,15 +187,15 @@ kubectl delete pod validator-engine-pod If IPs do not match, refer to the sections where load balancers are used. Now do the following: -* Add a label to this particular node. -* By this label our pod will know where to be deployed and what storage to use: +* Add a label to this particular node. +* By this label our pod will know where to be deployed and what storage to use: ``` kubectl label nodes node_type=ton-validator ``` * Replace **** (and ports if needed) in file [ton-node-port.yaml](ton-node-port.yaml). * Replace **** with a real path on host for Persistent Volume. * If you change the ports, make sure you specify appropriate env vars in Pod section. -* If you want to use dynamic storage provisioning via volumeClaimTemplates, feel free to create own StorageClass. +* If you want to use dynamic storage provisioning via volumeClaimTemplates, feel free to create own StorageClass. #### Install ```yaml @@ -224,7 +224,7 @@ or go inside the pod and check if blockchain size is growing: kubectl exec --stdin --tty validator-engine-pod -- /bin/bash du -h . ``` -### Deploy on-premises with metalLB load balancer +### Deploy on-premises with metalLB load balancer Often Kubernetes cluster is located in DMZ, is behind corporate firewall and access is controlled via proxy configuration. In this case we can't use host's network stack (**hostNetwork: true**) within a Pod and must manually proxy the access to the pod. @@ -323,13 +323,13 @@ Use the commands from the previous chapter to see if node operates properly. ```kubectl apply -f ton-aws.yaml``` #### Verify installation -Use instructions from the previous sections. +Use instructions from the previous sections. ### Deploy on GCP (Google Cloud Platform) #### Prepare * Kubernetes cluster of type Standard (not Autopilot). -* Premium static IP address. +* Premium static IP address. * Adjust firewall rules and security groups to allow ports 30001/udp, 30002/tcp and 30003/tcp (default ones). * Replace **** (and ports if needed) in file [ton-gcp.yaml](ton-gcp.yaml). * Adjust StorageClass name. Make sure you are providing fast storage. @@ -360,7 +360,7 @@ As a result CLB (classic internal Load Balancer) will be created automatically w Use instructions from the previous sections. ## Troubleshooting -## Docker +## Docker ### TON node cannot synchronize, constantly see messages [Error : 651 : no nodes] in the log Start the new container without starting validator-engine: @@ -380,7 +380,7 @@ identify your PUBLIC_IP: ``` curl -4 ifconfig.me ``` -compare if resulted IP coincides with your . +compare if resulted IP coincides with your . If it doesn't, exit container and launch it with the correct public IP. Then open UDP port (inside the container) you plan to allocate for TON node using netcat utility: ``` @@ -393,7 +393,7 @@ echo "test" | nc -u 30001 as a result inside the container you have to receive the "test" message. If you don't get the message inside the docker container, that means that either your firewall, LoadBalancer, NAT or proxy is blocking it. -Ask your system administrator for assistance. +Ask your system administrator for assistance. In the same way you can check if TCP port is available: @@ -455,7 +455,7 @@ kubernetes.io/cluster/: owner ``` --- #### AWS Load Balancer works, but I still see ```[no nodes]``` in validator's log -It is required to add the security group for the EC2 instances to the load balancer along with the default security group. +It is required to add the security group for the EC2 instances to the load balancer along with the default security group. It's a misleading that the default security group has "everything open." Add security group (default name is usually something like 'launch-wizard-1'). @@ -467,7 +467,7 @@ You can also set inbound and outbound rules of new security group to allow ALL p #### Pending PersistentVolumeClaim ```Waiting for a volume to be created either by the external provisioner 'ebs.csi.aws.com' or manually by the system administrator.``` -Solution: +Solution: Configure Amazon EBS CSI driver for working PersistentVolumes in EKS. diff --git a/emulator/README.md b/emulator/README.md index d4e2244e62..b60a55305d 100644 --- a/emulator/README.md +++ b/emulator/README.md @@ -27,6 +27,6 @@ Emulator output contains: ## TVM Emulator -TVM emulator is intended to run get methods or emulate sending message on TVM level. It is initialized with smart contract code and data cells. +TVM emulator is intended to run get methods or emulate sending message on TVM level. It is initialized with smart contract code and data cells. - To run get method you pass *initial stack* and *method id* (as integer). - To emulate sending message you pass *message body* and in case of internal message *amount* in nanograms. diff --git a/emulator/emulator-extern.h b/emulator/emulator-extern.h index 8410cb1bc5..a7e468b02e 100644 --- a/emulator/emulator-extern.h +++ b/emulator/emulator-extern.h @@ -59,7 +59,7 @@ EMULATOR_EXPORT bool transaction_emulator_set_ignore_chksig(void *transaction_em /** * @brief Set config for emulation * @param transaction_emulator Pointer to TransactionEmulator object - * @param config_boc Base64 encoded BoC serialized Config dictionary (Hashmap 32 ^Cell) + * @param config_boc Base64 encoded BoC serialized Config dictionary (Hashmap 32 ^Cell) * @return true in case of success, false in case of error */ EMULATOR_EXPORT bool transaction_emulator_set_config(void *transaction_emulator, const char *config_boc); @@ -102,18 +102,18 @@ EMULATOR_EXPORT bool transaction_emulator_set_prev_blocks_info(void *transaction * @param shard_account_boc Base64 encoded BoC serialized ShardAccount * @param message_boc Base64 encoded BoC serialized inbound Message (internal or external) * @return Json object with error: - * { - * "success": false, + * { + * "success": false, * "error": "Error description", * "external_not_accepted": false, * // and optional fields "vm_exit_code", "vm_log", "elapsed_time" in case external message was not accepted. - * } + * } * Or success: - * { - * "success": true, - * "transaction": "Base64 encoded Transaction boc", - * "shard_account": "Base64 encoded new ShardAccount boc", - * "vm_log": "execute DUP...", + * { + * "success": true, + * "transaction": "Base64 encoded Transaction boc", + * "shard_account": "Base64 encoded new ShardAccount boc", + * "vm_log": "execute DUP...", * "actions": "Base64 encoded compute phase actions boc (OutList n)", * "elapsed_time": 0.02 * } @@ -128,17 +128,17 @@ EMULATOR_EXPORT const char *transaction_emulator_emulate_transaction(void *trans * @param shard_account_boc Base64 encoded BoC serialized ShardAccount of special account * @param is_tock True for tock transactions, false for tick * @return Json object with error: - * { - * "success": false, + * { + * "success": false, * "error": "Error description", * "external_not_accepted": false - * } + * } * Or success: - * { - * "success": true, - * "transaction": "Base64 encoded Transaction boc", - * "shard_account": "Base64 encoded new ShardAccount boc", - * "vm_log": "execute DUP...", + * { + * "success": true, + * "transaction": "Base64 encoded Transaction boc", + * "shard_account": "Base64 encoded new ShardAccount boc", + * "vm_log": "execute DUP...", * "actions": "Base64 encoded compute phase actions boc (OutList n)", * "elapsed_time": 0.02 * } @@ -183,7 +183,7 @@ EMULATOR_EXPORT bool tvm_emulator_set_libraries(void *tvm_emulator, const char * * @param balance Smart contract balance * @param rand_seed_hex Random seed as hex string of length 64 * @param config Base64 encoded BoC serialized Config dictionary (Hashmap 32 ^Cell). Optional. - * @return true in case of success, false in case of error + * @return true in case of success, false in case of error */ EMULATOR_EXPORT bool tvm_emulator_set_c7(void *tvm_emulator, const char *address, uint32_t unixtime, uint64_t balance, const char *rand_seed_hex, const char *config); @@ -192,7 +192,7 @@ EMULATOR_EXPORT bool tvm_emulator_set_c7(void *tvm_emulator, const char *address * @brief Set extra currencies balance * @param tvm_emulator Pointer to TVM emulator * @param extra_currencies String with extra currencies balance in format "currency_id1=balance1 currency_id2=balance2 ..." - * @return true in case of success, false in case of error + * @return true in case of success, false in case of error */ EMULATOR_EXPORT bool tvm_emulator_set_extra_currencies(void *tvm_emulator, const char *extra_currencies); @@ -234,17 +234,17 @@ EMULATOR_EXPORT bool tvm_emulator_set_debug_enabled(void *tvm_emulator, bool deb * @param method_id Integer method id * @param stack_boc Base64 encoded BoC serialized stack (VmStack) * @return Json object with error: - * { - * "success": false, + * { + * "success": false, * "error": "Error description" - * } + * } * Or success: * { * "success": true - * "vm_log": "...", - * "vm_exit_code": 0, - * "stack": "Base64 encoded BoC serialized stack (VmStack)", - * "missing_library": null, + * "vm_log": "...", + * "vm_exit_code": 0, + * "stack": "Base64 encoded BoC serialized stack (VmStack)", + * "missing_library": null, * "gas_used": 1212 * } */ @@ -275,7 +275,7 @@ EMULATOR_EXPORT void *tvm_emulator_emulate_run_method_detailed(uint32_t len, con /** * @brief Destroy detailed result of "tvm_emulator_emulate_run_method_detailed" * @param detailed_result Pointer to detailed result struct returned by "tvm_emulator_emulate_run_method_detailed" - * + * * Caller should not use string_destroy() for fields of this struct, * as they are already freed in this function. */ @@ -286,19 +286,19 @@ EMULATOR_EXPORT void run_method_detailed_result_destroy(void *detailed_result); * @param tvm_emulator Pointer to TVM emulator * @param message_body_boc Base64 encoded BoC serialized message body cell. * @return Json object with error: - * { - * "success": false, + * { + * "success": false, * "error": "Error description" - * } + * } * Or success: * { * "success": true, * "new_code": "Base64 boc decoded new code cell", * "new_data": "Base64 boc decoded new data cell", * "accepted": true, - * "vm_exit_code": 0, - * "vm_log": "...", - * "missing_library": null, + * "vm_exit_code": 0, + * "vm_log": "...", + * "missing_library": null, * "gas_used": 1212, * "actions": "Base64 boc decoded actions cell of type (OutList n)" * } @@ -311,19 +311,19 @@ EMULATOR_EXPORT const char *tvm_emulator_send_external_message(void *tvm_emulato * @param message_body_boc Base64 encoded BoC serialized message body cell. * @param amount Amount of nanograms attached with internal message. * @return Json object with error: - * { - * "success": false, + * { + * "success": false, * "error": "Error description" - * } + * } * Or success: * { * "success": true, * "new_code": "Base64 boc decoded new code cell", * "new_data": "Base64 boc decoded new data cell", * "accepted": true, - * "vm_exit_code": 0, - * "vm_log": "...", - * "missing_library": null, + * "vm_exit_code": 0, + * "vm_log": "...", + * "missing_library": null, * "gas_used": 1212, * "actions": "Base64 boc decoded actions cell of type (OutList n)" * } @@ -346,7 +346,7 @@ EMULATOR_EXPORT void emulator_config_destroy(void *config); /** * @brief Destroy string created by emulator library * @param string Pointer to string to destroy - * + * * This function should be used to free strings returned by emulator library functions. * It is not safe to use caller's free() on them, as they may have been allocated using a different allocator. */ diff --git a/example/android/README.md b/example/android/README.md index cf12ba3047..b40cc16a59 100644 --- a/example/android/README.md +++ b/example/android/README.md @@ -1,8 +1,8 @@ # Generation of Tonlib libraries for Android OS **Tl;dr** Download the latest version of Tonlib libraries for Android from TON release page or check the artifacts from [Android JNI GitHub action](https://github.com/ton-blockchain/ton/actions/workflows/tonlib-android-jni.yml). -## Compile Tonlib for Android manually -Prerequisite: installed Java and set environment variable JAVA_HOME. +## Compile Tonlib for Android manually +Prerequisite: installed Java and set environment variable JAVA_HOME. ```bash git clone --recursive https://github.com/ton-blockchain/ton.git cd ton diff --git a/example/cpp/tonjson_example.cpp b/example/cpp/tonjson_example.cpp index a2ca01e80c..de4c8403d8 100644 --- a/example/cpp/tonjson_example.cpp +++ b/example/cpp/tonjson_example.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/fec/CMakeLists.txt b/fec/CMakeLists.txt index 721f988de7..648993eaae 100644 --- a/fec/CMakeLists.txt +++ b/fec/CMakeLists.txt @@ -5,9 +5,9 @@ set(FEC_SOURCE add_library(fec STATIC ${FEC_SOURCE}) -target_include_directories(fec PUBLIC - $ - $/.. +target_include_directories(fec PUBLIC + $ + $/.. ${OPENSSL_INCLUDE_DIR} ) target_link_libraries(fec PRIVATE tl_api) diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 945eed931e..8015d2f419 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/overlay/CMakeLists.txt b/overlay/CMakeLists.txt index e7b91d6e24..11ba257b75 100644 --- a/overlay/CMakeLists.txt +++ b/overlay/CMakeLists.txt @@ -17,9 +17,9 @@ set(OVERLAY_SOURCE add_library(overlay STATIC ${OVERLAY_SOURCE}) -target_include_directories(overlay PUBLIC - $ - $/.. +target_include_directories(overlay PUBLIC + $ + $/.. ${OPENSSL_INCLUDE_DIR} ) target_link_libraries(overlay PRIVATE tdutils tdactor adnl tl_api dht fec) diff --git a/rldp/CMakeLists.txt b/rldp/CMakeLists.txt index 82e238d3ca..361b350948 100644 --- a/rldp/CMakeLists.txt +++ b/rldp/CMakeLists.txt @@ -10,9 +10,9 @@ set(RLDP_SOURCE add_library(rldp STATIC ${RLDP_SOURCE}) -target_include_directories(rldp PUBLIC - $ - $/.. +target_include_directories(rldp PUBLIC + $ + $/.. ${OPENSSL_INCLUDE_DIR} ) target_link_libraries(rldp PRIVATE tdutils tdactor fec adnl tl_api) diff --git a/rldp2/CMakeLists.txt b/rldp2/CMakeLists.txt index bf0c212c23..47860b7f01 100644 --- a/rldp2/CMakeLists.txt +++ b/rldp2/CMakeLists.txt @@ -42,9 +42,9 @@ set(RLDP_SOURCE add_library(rldp2 STATIC ${RLDP_SOURCE}) -target_include_directories(rldp PUBLIC - $ - $/.. +target_include_directories(rldp PUBLIC + $ + $/.. ${OPENSSL_INCLUDE_DIR} ) if (GSL_FOUND AND BLAS_FOUND) diff --git a/storage/storage-daemon/smartcont/storage-contract.fif b/storage/storage-daemon/smartcont/storage-contract.fif index 05641521f1..8606e2d102 100644 --- a/storage/storage-daemon/smartcont/storage-contract.fif +++ b/storage/storage-daemon/smartcont/storage-contract.fif @@ -1,5 +1,5 @@ "Asm.fif" include -// automatically generated from `../../../crypto/smartcont/stdlib.fc` `storage-contract.fc` incl:`constants.fc` +// automatically generated from `../../../crypto/smartcont/stdlib.fc` `storage-contract.fc` incl:`constants.fc` PROGRAM{ DECLPROC check_proof DECLPROC add_to_balance @@ -14,21 +14,21 @@ PROGRAM{ x{D739} s, // merkle_hash byte_to_proof file_size cs special NOT // merkle_hash byte_to_proof file_size cs _7 IFJMP:<{ // merkle_hash byte_to_proof file_size cs - 4 BLKDROP // + 4 BLKDROP // FALSE // _8 }> // merkle_hash byte_to_proof file_size cs 8 LDU // merkle_hash byte_to_proof file_size _9 cs SWAP // merkle_hash byte_to_proof file_size cs _9 3 NEQINT // merkle_hash byte_to_proof file_size cs _13 IFJMP:<{ // merkle_hash byte_to_proof file_size cs - 4 BLKDROP // + 4 BLKDROP // FALSE // _14 }> // merkle_hash byte_to_proof file_size cs 256 LDU // merkle_hash byte_to_proof file_size _15 cs s0 s4 XCHG // cs byte_to_proof file_size _15 merkle_hash NEQ // cs byte_to_proof file_size _18 IFJMP:<{ // cs byte_to_proof file_size - 3 BLKDROP // + 3 BLKDROP // FALSE // _19 }> // cs byte_to_proof file_size s0 s2 XCHG // file_size byte_to_proof cs @@ -51,9 +51,9 @@ PROGRAM{ DICTUGET NULLSWAPIFNOT // _45 _46 NIP // found? - IFJMP:<{ // + IFJMP:<{ // TRUE // _35 - }> // + }> // FALSE // _36 }> add_to_balance PROCREF:<{ @@ -91,7 +91,7 @@ PROGRAM{ 1 PUSHINT // msg_value in_msg_body cs flags _9=1 AND // msg_value in_msg_body cs _10 IFJMP:<{ // msg_value in_msg_body cs - 3 BLKDROP // + 3 BLKDROP // }> // msg_value in_msg_body cs LDMSGADDR // msg_value in_msg_body _421 _420 DROP // msg_value in_msg_body sender_address @@ -382,11 +382,11 @@ PROGRAM{ 66 PUSHINT // msg _416 SENDRAWMSG }>ELSE<{ - 2DROP // + 2DROP // }> }> get_storage_contract_data PROC:<{ - // + // c4 PUSH // _1 CTOS // ds 1 LDI // _13 ds @@ -401,18 +401,18 @@ PROGRAM{ get_client_data CALLDICT // active balance provider merkle_hash file_size next_proof rate_per_mb_day max_span last_proof_time client torrent_hash }> get_torrent_hash PROC:<{ - // + // get_storage_contract_data CALLDICT // _12 _13 _14 _15 _16 _17 _18 _19 _20 _21 _22 10 1 BLKDROP2 // torrent_hash }> is_active PROC:<{ - // + // c4 PUSH // _0 CTOS // _1 1 PLDI // _3 }> get_next_proof_info PROC:<{ - // + // get_storage_contract_data CALLDICT // _12 _13 _14 _15 _16 _17 _18 _19 _20 _21 _22 2DROP s2 POP diff --git a/storage/storage-daemon/smartcont/storage-provider.fif b/storage/storage-daemon/smartcont/storage-provider.fif index d926c1c6c9..c0198ab402 100644 --- a/storage/storage-daemon/smartcont/storage-provider.fif +++ b/storage/storage-daemon/smartcont/storage-provider.fif @@ -1,5 +1,5 @@ "Asm.fif" include -// automatically generated from `../../../crypto/smartcont/stdlib.fc` `storage-provider.fc` incl:`constants.fc` +// automatically generated from `../../../crypto/smartcont/stdlib.fc` `storage-provider.fc` incl:`constants.fc` PROGRAM{ DECLPROC calculate_address_by_stateinit DECLPROC build_storage_contract_stateinit @@ -125,7 +125,7 @@ PROGRAM{ SEMPTY // msg_value in_msg_body cs _10 _11 OR // msg_value in_msg_body cs _12 IFJMP:<{ // msg_value in_msg_body cs - 3 BLKDROP // + 3 BLKDROP // }> // msg_value in_msg_body cs LDMSGADDR // msg_value in_msg_body _141 _140 DROP // msg_value in_msg_body sender_address @@ -134,7 +134,7 @@ PROGRAM{ OVER // msg_value sender_address op in_msg_body op 0 EQINT // msg_value sender_address op in_msg_body _21 IFJMP:<{ // msg_value sender_address op in_msg_body - 4 BLKDROP // + 4 BLKDROP // }> // msg_value sender_address op in_msg_body 64 LDU // msg_value sender_address op query_id in_msg_body s2 PUSH @@ -171,7 +171,7 @@ PROGRAM{ 3055775075 PUSHINT // in_msg_body sender_address op op _58=3055775075 EQUAL // in_msg_body sender_address op _59 IFJMP:<{ // in_msg_body sender_address op - 3 BLKDROP // + 3 BLKDROP // }> // in_msg_body sender_address op DUP 1408453846 PUSHINT // in_msg_body sender_address op op _60=1408453846 @@ -182,7 +182,7 @@ PROGRAM{ SDEQ // in_msg_body sender_address op _63 NOT // in_msg_body sender_address op _64 IFJMP:<{ // in_msg_body sender_address op - 3 BLKDROP // + 3 BLKDROP // RETALT }> // in_msg_body sender_address op c4 PUSH // in_msg_body sender_address op _66 @@ -209,7 +209,7 @@ PROGRAM{ SDEQ // in_msg_body _93 NOT // in_msg_body _94 IFJMP:<{ // in_msg_body - DROP // + DROP // RETALT }> // in_msg_body c4 PUSH // in_msg_body _96 @@ -237,7 +237,7 @@ PROGRAM{ ENDC // _136 c4 POP }>ELSE<{ - 2DROP // + 2DROP // }> }> recv_external PROC:<{ @@ -293,13 +293,13 @@ PROGRAM{ c4 POP }> seqno PROC:<{ - // + // c4 PUSH // _0 CTOS // _1 32 PLDU // _3 }> get_public_key PROC:<{ - // + // c4 PUSH // _1 CTOS // cs 64 LDU // _9 _8 @@ -307,7 +307,7 @@ PROGRAM{ 256 PLDU // _7 }> get_wallet_params PROC:<{ - // + // c4 PUSH // _1 CTOS // ds 32 LDU // _6 ds @@ -316,7 +316,7 @@ PROGRAM{ DROP // stored_seqno stored_subwallet public_key }> get_storage_params PROC:<{ - // + // c4 PUSH // _1 CTOS // ds 320 PUSHINT // ds _14 diff --git a/tdactor/benchmark/CMakeLists.txt b/tdactor/benchmark/CMakeLists.txt index d22a8f7405..6e7b9c1842 100644 --- a/tdactor/benchmark/CMakeLists.txt +++ b/tdactor/benchmark/CMakeLists.txt @@ -30,9 +30,9 @@ if(PkgConfig_FOUND) target_link_libraries(gbench-coro -lbenchmark) target_include_directories(gbench-coro PRIVATE ${GBENCH_INCLUDE_DIRS}) target_compile_options(gbench-coro PRIVATE ${GBENCH_CFLAGS_OTHER}) - + message(STATUS "Google Benchmark found via pkg-config - gbench-coro target enabled") - + # Check for Folly find_package(folly CONFIG QUIET) if(folly_FOUND) diff --git a/tdactor/benchmark/benchmark.cpp b/tdactor/benchmark/benchmark.cpp index 9420e707c6..9402cc8ba1 100644 --- a/tdactor/benchmark/benchmark.cpp +++ b/tdactor/benchmark/benchmark.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/tddb/test/io-bench.cpp b/tddb/test/io-bench.cpp index 87367b7dcb..2dbe75f2c9 100644 --- a/tddb/test/io-bench.cpp +++ b/tddb/test/io-bench.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/tdfec/benchmark/benchmark.cpp b/tdfec/benchmark/benchmark.cpp index 7179102181..a8611a9ff0 100644 --- a/tdfec/benchmark/benchmark.cpp +++ b/tdfec/benchmark/benchmark.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/tdnet/example/tcp_ping_pong.cpp b/tdnet/example/tcp_ping_pong.cpp index 40b2c339e4..872beca84a 100644 --- a/tdnet/example/tcp_ping_pong.cpp +++ b/tdnet/example/tcp_ping_pong.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/tdnet/example/udp_ping_pong.cpp b/tdnet/example/udp_ping_pong.cpp index 4f75a831cd..47c9745290 100644 --- a/tdnet/example/udp_ping_pong.cpp +++ b/tdnet/example/udp_ping_pong.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/tdutils/generate/generate_mime_types_gperf.cpp b/tdutils/generate/generate_mime_types_gperf.cpp index 03edb535cf..573b5dd294 100644 --- a/tdutils/generate/generate_mime_types_gperf.cpp +++ b/tdutils/generate/generate_mime_types_gperf.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/terminal/CMakeLists.txt b/terminal/CMakeLists.txt index d3b06abb7a..b5c3ef85f0 100644 --- a/terminal/CMakeLists.txt +++ b/terminal/CMakeLists.txt @@ -6,13 +6,13 @@ set(TERMINAL_SOURCE ) add_library(terminal STATIC ${TERMINAL_SOURCE}) -target_include_directories(terminal PUBLIC - $ - $/.. +target_include_directories(terminal PUBLIC + $ + $/.. ${OPENSSL_INCLUDE_DIR} ) target_link_libraries(terminal PRIVATE tdutils tdactor ) - + if (NOT READLINE_FOUND) find_package(Readline) endif() diff --git a/test/generate-test-node-config.py b/test/generate-test-node-config.py index 0f22abdb15..49a9734249 100644 --- a/test/generate-test-node-config.py +++ b/test/generate-test-node-config.py @@ -29,7 +29,7 @@ def get_addr_list(node): def generate_dht_node(node): global binary addr_list = get_addr_list(node) - r = subprocess.run([binary, '-k', node['spk'], '-a', addr_list, '-m', 'dht'], capture_output=True) + r = subprocess.run([binary, '-k', node['spk'], '-a', addr_list, '-m', 'dht'], capture_output=True) s = r.stdout.decode("utf-8").split("\n") return json.loads(s[0]) @@ -47,11 +47,11 @@ def add_id(s): options = t[3:] pub = '-' short = '-' - rand = None + rand = None spk = pk if (pk[0] != '-'): - r = subprocess.run([binary, '-k', pk, '-m', 'id'], capture_output=True) + r = subprocess.run([binary, '-k', pk, '-m', 'id'], capture_output=True) s = r.stdout.decode("utf-8").split("\n") pk = json.loads(pk) pub = json.loads(s[1]) @@ -73,7 +73,7 @@ def add_id(s): ids.append({'ip' : ip, 'port' : port, 'options' : options, 'pk' : pk, 'spk' : spk, 'pub' : pub, 'short' : short, 'rand' : rand}) else: for x in range(0, rand): - r = subprocess.run([binary, '-m', 'id'], capture_output=True) + r = subprocess.run([binary, '-m', 'id'], capture_output=True) s = r.stdout.decode("utf-8").split("\n") pk = json.loads(s[0]) pub = json.loads(s[1]) @@ -81,13 +81,13 @@ def add_id(s): ids.append({'ip' : ip, 'port' : port, 'options' : options, 'pk' : pk, 'spk' : spk, 'pub' : pub, 'short' : short, 'rand' : None}) def readconfig(name): - with open(name) as f: + with open(name) as f: for s in f.readlines(): add_id(s.strip()) def generate_global_config(): global ids - + config={'@type':'config.global'} dht={'@type':'dht.config.global','k':6,'a':3} @@ -97,7 +97,7 @@ def generate_global_config(): if '+dhtstatic' in node['options']: dht_nodes.append(generate_dht_node(node)) - dht['static_nodes'] = {'@type':'dht.nodes','nodes':dht_nodes} + dht['static_nodes'] = {'@type':'dht.nodes','nodes':dht_nodes} config['dht'] = dht @@ -111,16 +111,16 @@ def generate_global_config(): catchains.append(name) cc=[] - for name in catchains: + for name in catchains: catchain={'@type':'catchain.config.global','tag':base64.b64encode(hashlib.sha256(name.encode("utf-8")).digest()).decode("utf-8")} catchain_nodes=[] for node in ids: if ('+catchain'+name) in node['options']: catchain_nodes.append(node['pub']) - catchain['nodes'] = catchain_nodes + catchain['nodes'] = catchain_nodes cc.append(catchain) config['catchains'] = cc - + liteservers=[] for node in ids: for opt in node['options']: @@ -134,7 +134,7 @@ def generate_global_config(): "workchain" : -1, "shard" : -9223372036854775808, "seqno" : 0, - "root_hash": "DXduYJkakj2d+rB7Qpj2SCkjTCG7AGXAA9G7EHQEyG0=", + "root_hash": "DXduYJkakj2d+rB7Qpj2SCkjTCG7AGXAA9G7EHQEyG0=", "file_hash": "4xASfDALy6Hk5Tg01IyBoFUepur9YJg2pg3cm0zocJk=" }} @@ -170,7 +170,7 @@ def generate_local_config(ip,port): cnt=node['rand'] dht_config.append({'@type':'dht.config.random.local', 'cnt':cnt}) config['dht'] = dht_config - + liteservers=[] for node in ids: for opt in node['options']: diff --git a/test/test-adnl.cpp b/test/test-adnl.cpp index 772d1ac62e..b6a0e740e9 100644 --- a/test/test-adnl.cpp +++ b/test/test-adnl.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/test/test-catchain.cpp b/test/test-catchain.cpp index 74616e0683..b9975fa883 100644 --- a/test/test-catchain.cpp +++ b/test/test-catchain.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/test/test-dht.cpp b/test/test-dht.cpp index ff3b613348..c24b235489 100644 --- a/test/test-dht.cpp +++ b/test/test-dht.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/test/test-hello-world.cpp b/test/test-hello-world.cpp index f2faaab537..69aac509cb 100644 --- a/test/test-hello-world.cpp +++ b/test/test-hello-world.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -13,13 +13,13 @@ You should have received a copy of the GNU General Public License - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. along with TON Blockchain. If not, see . diff --git a/test/test-http.cpp b/test/test-http.cpp index 8d07b79eed..54eaad0792 100644 --- a/test/test-http.cpp +++ b/test/test-http.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/test/test-overlay.cpp b/test/test-overlay.cpp index 39659c7af3..214a6a1d32 100644 --- a/test/test-overlay.cpp +++ b/test/test-overlay.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/test/test-rldp.cpp b/test/test-rldp.cpp index ce3bbf2c1a..d20d9d3b49 100644 --- a/test/test-rldp.cpp +++ b/test/test-rldp.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/test/test-rldp2.cpp b/test/test-rldp2.cpp index f7acb01d4e..af3ff5ab7c 100644 --- a/test/test-rldp2.cpp +++ b/test/test-rldp2.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/test/test-rocksdb.cpp b/test/test-rocksdb.cpp index 6f8dbb3aa1..68ab67c05e 100644 --- a/test/test-rocksdb.cpp +++ b/test/test-rocksdb.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/test/test-td-main.cpp b/test/test-td-main.cpp index 2c7f81ad6a..37a475aef5 100644 --- a/test/test-td-main.cpp +++ b/test/test-td-main.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index d43213f759..0e59826bdf 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/test/test-validator-session-state.cpp b/test/test-validator-session-state.cpp index 226e20ec15..08d86317c0 100644 --- a/test/test-validator-session-state.cpp +++ b/test/test-validator-session-state.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/third-party/wingetopt/LICENSE b/third-party/wingetopt/LICENSE index b1ded6d834..4cf4cf7dad 100644 --- a/third-party/wingetopt/LICENSE +++ b/third-party/wingetopt/LICENSE @@ -7,11 +7,11 @@ #### LICENSE Copyright (c) 2002 Todd C. Miller - + Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. - + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR @@ -19,7 +19,7 @@ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - + Sponsored in part by the Defense Advanced Research Projects Agency (DARPA) and Air Force Research Laboratory, Air Force Materiel Command, USAF, under agreement number F39502-99-1-0512. @@ -28,10 +28,10 @@ Copyright (c) 2000 The NetBSD Foundation, Inc. All rights reserved. - + This code is derived from software contributed to The NetBSD Foundation by Dieter Baron and Thomas Klausner. - + Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -40,7 +40,7 @@ 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR diff --git a/third-party/wingetopt/README.md b/third-party/wingetopt/README.md index 16b855a614..492870a042 100644 --- a/third-party/wingetopt/README.md +++ b/third-party/wingetopt/README.md @@ -16,11 +16,11 @@ The sources were taken from MinGW-runtime project. #### LICENSE Copyright (c) 2002 Todd C. Miller - + Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. - + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR @@ -28,7 +28,7 @@ The sources were taken from MinGW-runtime project. WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - + Sponsored in part by the Defense Advanced Research Projects Agency (DARPA) and Air Force Research Laboratory, Air Force Materiel Command, USAF, under agreement number F39502-99-1-0512. @@ -37,10 +37,10 @@ The sources were taken from MinGW-runtime project. Copyright (c) 2000 The NetBSD Foundation, Inc. All rights reserved. - + This code is derived from software contributed to The NetBSD Foundation by Dieter Baron and Thomas Klausner. - + Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -49,7 +49,7 @@ The sources were taken from MinGW-runtime project. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR diff --git a/third-party/wingetopt/src/getopt.h b/third-party/wingetopt/src/getopt.h index 1426b69ce3..2ebc68e1e7 100644 --- a/third-party/wingetopt/src/getopt.h +++ b/third-party/wingetopt/src/getopt.h @@ -4,9 +4,9 @@ * This file has no copyright assigned and is placed in the Public Domain. * This file is a part of the w64 mingw-runtime package. * - * The w64 mingw-runtime package and its code is distributed in the hope that it - * will be useful but WITHOUT ANY WARRANTY. ALL WARRANTIES, EXPRESSED OR - * IMPLIED ARE HEREBY DISCLAIMED. This includes but is not limited to + * The w64 mingw-runtime package and its code is distributed in the hope that it + * will be useful but WITHOUT ANY WARRANTY. ALL WARRANTIES, EXPRESSED OR + * IMPLIED ARE HEREBY DISCLAIMED. This includes but is not limited to * warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ diff --git a/tl/CMakeLists.txt b/tl/CMakeLists.txt index 168a5d73ef..3eebd585df 100644 --- a/tl/CMakeLists.txt +++ b/tl/CMakeLists.txt @@ -1,9 +1,9 @@ add_subdirectory(generate) set_source_files_properties(${TL_TON_API} PROPERTIES GENERATED TRUE) -add_library(tl_api STATIC - ${TL_TON_API} - tl/tl_object_parse.h - tl/tl_object_store.h +add_library(tl_api STATIC + ${TL_TON_API} + tl/tl_object_parse.h + tl/tl_object_store.h tl/TlObject.h) add_dependencies(tl_api tl_generate_common) target_link_libraries(tl_api tdutils) @@ -13,10 +13,10 @@ target_include_directories(tl_api PUBLIC $ $) set_source_files_properties(${TL_LITE_API} PROPERTIES GENERATED TRUE) -add_library(tl_lite_api STATIC - ${TL_LITE_API} - tl/tl_object_parse.h - tl/tl_object_store.h +add_library(tl_lite_api STATIC + ${TL_LITE_API} + tl/tl_object_parse.h + tl/tl_object_store.h tl/TlObject.h) add_dependencies(tl_lite_api tl_generate_common) target_link_libraries(tl_lite_api tdutils) diff --git a/tl/generate/generate_common.cpp b/tl/generate/generate_common.cpp index 28f0f5eee2..0b9b3d1d4b 100644 --- a/tl/generate/generate_common.cpp +++ b/tl/generate/generate_common.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/tl/generate/generate_java.cpp b/tl/generate/generate_java.cpp index 7e837ca34f..a3bd461088 100644 --- a/tl/generate/generate_java.cpp +++ b/tl/generate/generate_java.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/tl/generate/scheme/lite_api.tl b/tl/generate/scheme/lite_api.tl index a9adfa8453..8abfcd1896 100644 --- a/tl/generate/scheme/lite_api.tl +++ b/tl/generate/scheme/lite_api.tl @@ -22,7 +22,7 @@ tonNode.zeroStateIdExt workchain:int root_hash:int256 file_hash:int256 = tonNode adnl.message.query query_id:int256 query:bytes = adnl.Message; adnl.message.answer query_id:int256 answer:bytes = adnl.Message; -liteServer.error code:int message:string = liteServer.Error; +liteServer.error code:int message:string = liteServer.Error; liteServer.accountId workchain:int id:int256 = liteServer.AccountId; liteServer.libraryEntry hash:int256 data:bytes = liteServer.LibraryEntry; @@ -114,6 +114,6 @@ liteServer.nonfinal.getValidatorGroups mode:# wc:mode.0?int shard:mode.0?long = liteServer.nonfinal.getCandidate id:liteServer.nonfinal.candidateId = liteServer.nonfinal.Candidate; liteServer.nonfinal.getPendingShardBlocks mode:# wc:mode.0?int shard:mode.0?long = liteServer.nonfinal.PendingShardBlocks; -liteServer.queryPrefix = Object; +liteServer.queryPrefix = Object; liteServer.query data:bytes = Object; liteServer.waitMasterchainSeqno seqno:int timeout_ms:int = Object; // query prefix diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 1cb7ac6ef0..454fe91e6d 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -67,7 +67,7 @@ adnl.address.udp6 ip:int128 port:int = adnl.Address; //adnl.address.tcp ip:int port:int = adnl.Address; //adnl.address.tcp6 ip:int128 port:int = adnl.Address; -adnl.address.tunnel to:int256 pubkey:PublicKey = adnl.Address; +adnl.address.tunnel to:int256 pubkey:PublicKey = adnl.Address; adnl.address.reverse = adnl.Address; adnl.addressList addrs:(vector adnl.Address) version:int reinit_date:int priority:int expire_at:int = adnl.AddressList; @@ -79,34 +79,34 @@ adnl.nodes nodes:(vector adnl.node) = adnl.Nodes; ---types--- -adnl.packetContents - rand1:bytes - flags:# - from:flags.0?PublicKey +adnl.packetContents + rand1:bytes + flags:# + from:flags.0?PublicKey from_short:flags.1?adnl.id.short - message:flags.2?adnl.Message + message:flags.2?adnl.Message messages:flags.3?(vector adnl.Message) - address:flags.4?adnl.addressList + address:flags.4?adnl.addressList priority_address:flags.5?adnl.addressList - seqno:flags.6?long - confirm_seqno:flags.7?long + seqno:flags.6?long + confirm_seqno:flags.7?long recv_addr_list_version:flags.8?int recv_priority_addr_list_version:flags.9?int - reinit_date:flags.10?int + reinit_date:flags.10?int dst_reinit_date:flags.10?int - signature:flags.11?bytes - rand2:bytes + signature:flags.11?bytes + rand2:bytes = adnl.PacketContents; -adnl.tunnelPacketContents - rand1:bytes - flags:# +adnl.tunnelPacketContents + rand1:bytes + flags:# from_ip:flags.0?int from_port:flags.0?int - message:flags.1?bytes + message:flags.1?bytes statistics:flags.2?bytes payment:flags.3?bytes - rand2:bytes + rand2:bytes = adnl.TunnelPacketContents; @@ -114,7 +114,7 @@ adnl.tunnelPacketContents // flag 17 - control packet adnl.proxyPacketHeader proxy_id:int256 - flags:# + flags:# ip:flags.0?int port:flags.0?int adnl_start_time:flags.1?int @@ -196,7 +196,7 @@ dht.updateRule.overlayNodes = dht.UpdateRule; dht.keyDescription key:dht.key id:PublicKey update_rule:dht.UpdateRule signature:bytes = dht.KeyDescription; -dht.value key:dht.keyDescription value:bytes ttl:int signature:bytes = dht.Value; +dht.value key:dht.keyDescription value:bytes ttl:int signature:bytes = dht.Value; dht.pong random_id:long = dht.Pong; @@ -217,8 +217,8 @@ dht.db.key.bucket id:int = dht.db.Key; dht.ping random_id:long = dht.Pong; dht.store value:dht.value = dht.Stored; -dht.findNode key:int256 k:int = dht.Nodes; -dht.findValue key:int256 k:int = dht.ValueResult; +dht.findNode key:int256 k:int = dht.Nodes; +dht.findValue key:int256 k:int = dht.ValueResult; dht.getSignedAddressList = dht.Node; dht.registerReverseConnection node:PublicKey ttl:int signature:bytes = dht.Stored; dht.requestReversePing target:adnl.Node signature:bytes client:int256 k:int = dht.ReversePingResult; @@ -245,7 +245,7 @@ overlay.broadcastList hashes:(vector int256) = overlay.BroadcastList; overlay.fec.received hash:int256 = overlay.Broadcast; overlay.fec.completed hash:int256 = overlay.Broadcast; -overlay.broadcast.id src:int256 data_hash:int256 flags:int = overlay.broadcast.Id; +overlay.broadcast.id src:int256 data_hash:int256 flags:int = overlay.broadcast.Id; overlay.broadcastFec.id src:int256 type:int256 data_hash:int256 size:int flags:int = overlay.broadcastFec.Id; overlay.broadcastFec.partId broadcast_hash:int256 data_hash:int256 seqno:int = overlay.broadcastFec.PartId; @@ -265,7 +265,7 @@ overlay.unicast data:bytes = overlay.Broadcast; overlay.broadcast src:PublicKey certificate:overlay.Certificate flags:int data:bytes date:int signature:bytes = overlay.Broadcast; overlay.broadcastFec src:PublicKey certificate:overlay.Certificate data_hash:int256 data_size:int flags:int data:bytes seqno:int fec:fec.Type date:int signature:bytes = overlay.Broadcast; -overlay.broadcastFecShort src:PublicKey certificate:overlay.Certificate broadcast_hash:int256 part_data_hash:int256 seqno:int signature:bytes = overlay.Broadcast; +overlay.broadcastFecShort src:PublicKey certificate:overlay.Certificate broadcast_hash:int256 part_data_hash:int256 seqno:int signature:bytes = overlay.Broadcast; overlay.broadcastNotFound = overlay.Broadcast; ---functions--- @@ -325,7 +325,7 @@ validatorSession.candidate.id round:int256 block_hash:int256 = validatorSession. validatorSession.message.startSession = validatorSession.Message; validatorSession.message.finishSession = validatorSession.Message; -validatorSession.message.submittedBlock round:int root_hash:int256 file_hash:int256 +validatorSession.message.submittedBlock round:int root_hash:int256 file_hash:int256 collated_data_file_hash:int256 = validatorSession.round.Message; validatorSession.message.approvedBlock round:int candidate:int256 signature:bytes = validatorSession.round.Message; validatorSession.message.rejectedBlock round:int candidate:int256 reason:bytes = validatorSession.round.Message; @@ -361,7 +361,7 @@ validatorSession.configVersionedV2 catchain_opts:validatorSession.CatChainOption ---functions--- validatorSession.ping hash:long = validatorSession.Pong; -validatorSession.downloadCandidate round:int id:validatorSession.candidateId = validatorSession.Candidate; +validatorSession.downloadCandidate round:int id:validatorSession.candidateId = validatorSession.Candidate; ---types--- @@ -377,7 +377,7 @@ hashable.validatorSessionRoundAttempt seqno:int votes:int precommitted:int vote_ hashable.validatorSessionRound locked_round:int locked_block:int seqno:int precommitted:Bool first_attempt:int approved_blocks:int signatures:int attempts:int = Hashable; hashable.blockSignature signature:int = Hashable; -hashable.sentBlock src:int root_hash:int file_hash:int collated_data_file_hash:int = Hashable; +hashable.sentBlock src:int root_hash:int file_hash:int collated_data_file_hash:int = Hashable; hashable.sentBlockEmpty = Hashable; hashable.vote block:int node:int = Hashable; hashable.blockCandidate block:int approved:int = Hashable; @@ -393,7 +393,7 @@ hashable.validatorSession ts:int old_rounds:int cur_round:int = Hashable; ---types--- -tonNode.sessionId workchain:int shard:long cc_seqno:int opts_hash:int256 = tonNode.SessionId; +tonNode.sessionId workchain:int shard:long cc_seqno:int opts_hash:int256 = tonNode.SessionId; tonNode.blockSignature who:int256 signature:bytes = tonNode.BlockSignature; @@ -529,8 +529,8 @@ tonNode.requestFastSyncOverlayMemberCertificate sign_by:int256 adnl_id:int256 sl db.root.dbDescription version:int first_masterchain_block_id:tonNode.blockIdExt flags:int = db.root.DbDescription; -db.root.key.cellDb version:int = db.root.Key; -db.root.key.blockDb version:int = db.root.Key; +db.root.key.cellDb version:int = db.root.Key; +db.root.key.blockDb version:int = db.root.Key; db.root.config celldb_version:int blockdb_version:int = db.root.Config; db.root.key.config = db.root.Key; @@ -542,9 +542,9 @@ db.block.info#4ac6e727 id:tonNode.blockIdExt flags:# prev_left:flags.1?tonNode.b prev_right:flags.2?tonNode.blockIdExt next_left:flags.3?tonNode.blockIdExt next_right:flags.4?tonNode.blockIdExt - lt:flags.13?long + lt:flags.13?long ts:flags.14?int - state:flags.17?int256 + state:flags.17?int256 masterchain_ref_seqno:flags.23?int = db.block.Info; db.block.packedInfo id:tonNode.blockIdExt unixtime:int offset:long = db.block.Info; db.block.archivedInfo id:tonNode.blockIdExt flags:# next:flags.0?tonNode.blockIdExt = db.block.Info; @@ -555,11 +555,11 @@ db.blockdb.key.lru id:tonNode.blockIdExt = db.blockdb.Key; db.blockdb.key.value id:tonNode.blockIdExt = db.blockdb.Key; db.candidate source:PublicKey id:tonNode.blockIdExt data:bytes collated_data:bytes = db.Candidate; -db.candidate.id source:PublicKey id:tonNode.blockIdExt collated_data_file_hash:int256 = db.candidate.Id; +db.candidate.id source:PublicKey id:tonNode.blockIdExt collated_data_file_hash:int256 = db.candidate.Id; db.filedb.key.empty = db.filedb.Key; -db.filedb.key.blockFile block_id:tonNode.blockIdExt = db.filedb.Key; -db.filedb.key.zeroStateFile block_id:tonNode.blockIdExt = db.filedb.Key; +db.filedb.key.blockFile block_id:tonNode.blockIdExt = db.filedb.Key; +db.filedb.key.zeroStateFile block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.persistentStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.splitAccountStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt effective_shard:long = db.filedb.Key; db.filedb.key.splitPersistentStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt = db.filedb.Key; @@ -600,8 +600,8 @@ db.lt.desc.key workchain:int shard:long = db.lt.Key; db.lt.shard.key idx:int = db.lt.Key; db.lt.status.key = db.lt.Key; db.lt.el.value id:tonNode.blockIdExt lt:long ts:int = db.lt.el.Value; -db.lt.desc.value first_idx:int last_idx:int last_seqno:int last_lt:long last_ts:int = db.lt.desc.Value; -db.lt.shard.value workchain:int shard:long = db.lt.shard.Value; +db.lt.desc.value first_idx:int last_idx:int last_seqno:int last_lt:long last_ts:int = db.lt.desc.Value; +db.lt.shard.value workchain:int shard:long = db.lt.shard.Value; db.lt.status.value total_shards:int = db.lt.status.Value; db.files.index.key = db.files.Key; @@ -609,7 +609,7 @@ db.files.package.key package_id:int key:Bool temp:Bool = db.files.Key; db.files.index.value packages:(vector int) key_packages:(vector int) temp_packages:(vector int) = db.files.index.Value; db.files.package.firstBlock workchain:int shard:long seqno:int unixtime:int lt:long = db.files.package.FirstBlock; -db.files.package.value package_id:int key:Bool temp:Bool firstblocks:(vector db.files.package.firstBlock) deleted:Bool +db.files.package.value package_id:int key:Bool temp:Bool firstblocks:(vector db.files.package.firstBlock) deleted:Bool = db.files.package.Value; db.event.blockApplied block_id:tonNode.blockIdExt = db.Event; @@ -620,10 +620,10 @@ db.event.blockCandidateReceived block_id:tonNode.blockIdExt = db.Event; ---types--- -validator.groupMember public_key_hash:int256 adnl:int256 weight:long = engine.validator.GroupMember; -validator.group workchain:int shard:long catchain_seqno:int config_hash:int256 members:(vector validator.groupMember) = validator.Group; -validator.groupEx workchain:int shard:long vertical_seqno:int catchain_seqno:int config_hash:int256 members:(vector validator.groupMember) = validator.Group; -validator.groupNew workchain:int shard:long vertical_seqno:int last_key_block_seqno:int catchain_seqno:int config_hash:int256 members:(vector validator.groupMember) = validator.Group; +validator.groupMember public_key_hash:int256 adnl:int256 weight:long = engine.validator.GroupMember; +validator.group workchain:int shard:long catchain_seqno:int config_hash:int256 members:(vector validator.groupMember) = validator.Group; +validator.groupEx workchain:int shard:long vertical_seqno:int catchain_seqno:int config_hash:int256 members:(vector validator.groupMember) = validator.Group; +validator.groupNew workchain:int shard:long vertical_seqno:int last_key_block_seqno:int catchain_seqno:int config_hash:int256 members:(vector validator.groupMember) = validator.Group; validator.telemetry flags:# timestamp:double adnl_id:int256 node_version:string os_version:string node_started_at:int @@ -663,7 +663,7 @@ liteclient.config.global liteservers:(vector liteserver.desc) liteservers_v2:(ve engine.adnl id:int256 category:int = engine.Adnl; engine.addr ip:int port:int categories:(vector int) priority_categories:(vector int) = engine.Addr; -engine.addrProxy in_ip:int in_port:int out_ip:int out_port:int +engine.addrProxy in_ip:int in_port:int out_ip:int out_port:int proxy_type:adnl.Proxy categories:(vector int) priority_categories:(vector int) = engine.Addr; engine.dht id:int256 = engine.Dht; engine.validatorTempKey key:int256 expire_at:int = engine.ValidatorTempKey; @@ -672,7 +672,7 @@ engine.validator id:int256 temp_keys:(vector engine.validatorTempKey) adnl_addrs engine.collator adnl_id:int256 shard:tonNode.shardId = engine.Collator; engine.liteServer id:int256 port:int = engine.LiteServer; engine.controlProcess id:int256 permissions:int = engine.ControlProcess; -engine.controlInterface id:int256 port:int allowed:(vector engine.controlProcess) = engine.ControlInterface; +engine.controlInterface id:int256 port:int allowed:(vector engine.controlProcess) = engine.ControlInterface; engine.gc ids:(vector int256) = engine.Gc; engine.dht.config dht:(vector engine.dht) gc:engine.gc = engine.dht.Config; @@ -751,7 +751,7 @@ engine.validator.success = engine.validator.Success; engine.validator.jsonConfig data:string = engine.validator.JsonConfig; -engine.validator.electionBid election_date:int perm_key:int256 adnl_addr:int256 to_send_payload:bytes = engine.validator.ElectionBid; +engine.validator.electionBid election_date:int perm_key:int256 adnl_addr:int256 to_send_payload:bytes = engine.validator.ElectionBid; engine.validator.proposalVote perm_key:int256 to_send:bytes = engine.validator.ProposalVote; engine.validator.dhtServerStatus id:int256 status:int = engine.validator.DhtServerStatus; @@ -803,7 +803,7 @@ engine.validator.addDhtId key_hash:int256 = engine.validator.Success; engine.validator.addValidatorPermanentKey key_hash:int256 election_date:int ttl:int = engine.validator.Success; engine.validator.addValidatorTempKey permanent_key_hash:int256 key_hash:int256 ttl:int = engine.validator.Success; engine.validator.addValidatorAdnlAddress permanent_key_hash:int256 key_hash:int256 ttl:int = engine.validator.Success; -engine.validator.changeFullNodeAdnlAddress adnl_id:int256 = engine.validator.Success; +engine.validator.changeFullNodeAdnlAddress adnl_id:int256 = engine.validator.Success; engine.validator.addLiteserver key_hash:int256 port:int = engine.validator.Success; engine.validator.addControlInterface key_hash:int256 port:int = engine.validator.Success; engine.validator.addControlProcess key_hash:int256 port:int peer_key:int256 permissions:int = engine.validator.Success; @@ -825,11 +825,11 @@ engine.validator.exportAllPrivateKeys encryption_key:PublicKey = engine.validato engine.validator.getStats = engine.validator.Stats; engine.validator.getConfig = engine.validator.JsonConfig; -engine.validator.setVerbosity verbosity:int = engine.validator.Success; +engine.validator.setVerbosity verbosity:int = engine.validator.Success; engine.validator.createElectionBid election_date:int election_addr:string wallet:string = engine.validator.ElectionBid; -engine.validator.createProposalVote vote:bytes = engine.validator.ProposalVote; -engine.validator.createComplaintVote election_id:int vote:bytes = engine.validator.ProposalVote; +engine.validator.createProposalVote vote:bytes = engine.validator.ProposalVote; +engine.validator.createComplaintVote election_id:int vote:bytes = engine.validator.ProposalVote; engine.validator.checkDhtServers id:int256 = engine.validator.DhtServersStatus; diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index fb9899c050..3aad9e052a 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -451,7 +451,7 @@ fun test34(p: int, n: int) { fun test35(overflowMode: int) { try { val b: builder = match (overflowMode) { - 1 => triggerOverflowIntConst(), + 1 => triggerOverflowIntConst(), 2 => triggerOverflowUintConst(), 3 => beginCell().storeUint(10, -4), 4 => beginCell().storeInt(100, 1), @@ -541,7 +541,7 @@ fun test42() { var c1 = beginCell().endCell(); var c2 = beginCell().endCellSpecial(false); return ( - c1.hash() == c1.hashOfLevel(3), + c1.hash() == c1.hashOfLevel(3), c1.depth() == c1.depthOfLevel(3), c1.hash() == c2.hash(), c2.depth() == 0, @@ -552,7 +552,7 @@ fun test42() { fun test43() { var c3 = beginCell().storeUint(123, 32).endCell(); var c3lib = c3.toLibraryReference(); - + var (s, isExotic) = c3lib.beginParseSpecial(); assert (isExotic) throw 123; val cellType = s.loadInt(8) as ExoticCellType; @@ -768,7 +768,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - triggerOverflowIntConst() PROC:<{ + triggerOverflowIntConst() PROC:<{ 10 PUSHINT NEWC 4 STI @@ -777,7 +777,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - triggerOverflowUintConst() PROC:<{ + triggerOverflowUintConst() PROC:<{ 123 PUSHINT NEWC 6 STU @@ -786,7 +786,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - test36() PROC:<{ + test36() PROC:<{ 0 PUSHINT NEWC OVER @@ -804,7 +804,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - test38() PROC:<{ + test38() PROC:<{ x{} PUSHSLICE SBITS }> @@ -812,14 +812,14 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - test39() PROC:<{ + test39() PROC:<{ PUSHREF }> """ @fif_codegen """ - test40() PROC:<{ + test40() PROC:<{ x{} PUSHSLICE SBITREFS }> diff --git a/tolk-tester/tests/comments-tests.tolk b/tolk-tester/tests/comments-tests.tolk index 0c577674fd..54431a3b64 100644 --- a/tolk-tester/tests/comments-tests.tolk +++ b/tolk-tester/tests/comments-tests.tolk @@ -89,7 +89,7 @@ fun main() { @fif_codegen """ - call_some_calc() PROC:<{ // + call_some_calc() PROC:<{ // // 48: return some_calc(-10) -10 PUSHINT // '0=-10 // 31: calcAbs: bool = true diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index f28000d525..be565240aa 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -195,7 +195,7 @@ fun test14() { @method_id(115) fun test15() { return ( - opts1.equals(), opts1.equals(opts1), opts1.equals(opts2), opts1.equals(opts3), opts1.equals(opts4), + opts1.equals(), opts1.equals(opts1), opts1.equals(opts2), opts1.equals(opts3), opts1.equals(opts4), opts4.equals(), opts4.equals(opts1), opts4.equals(opts2), opts4.equals(opts3), opts4.equals(opts4), 777, opts1.equals({ closeTo: null, allowPenalty: true }), diff --git a/tolk-tester/tests/dicts-demo.tolk b/tolk-tester/tests/dicts-demo.tolk index 990d57a86b..708f8ef341 100644 --- a/tolk-tester/tests/dicts-demo.tolk +++ b/tolk-tester/tests/dicts-demo.tolk @@ -130,7 +130,7 @@ fun main() {} @testcase | 101 | 1 1 1 | 1 2 3 @testcase | 101 | 1 2 1 | 1 -1 2 @testcase | 101 | 1 2 3 | 1 -1 -1 -@testcase | 102 | | [ [ 4 104 ] [ 3 103 ] [ 2 102 ] [ 1 101 ] ] +@testcase | 102 | | [ [ 4 104 ] [ 3 103 ] [ 2 102 ] [ 1 101 ] ] @testcase | 103 | | 1 1 2 1 3 (null) @testcase | 104 | | 12 34 56 78 0 @testcase | 105 | -1 | 1 diff --git a/tolk-tester/tests/enums-tests.tolk b/tolk-tester/tests/enums-tests.tolk index c88a724c56..caeacd7f8b 100644 --- a/tolk-tester/tests/enums-tests.tolk +++ b/tolk-tester/tests/enums-tests.tolk @@ -40,8 +40,8 @@ enum Err { DisabledId, } -struct Box { - value: T +struct Box { + value: T } fun checkIsBlue(c: Color) { @@ -83,7 +83,7 @@ fun Color.mapToNumber(self) { @method_id(101) fun test1() { var c = Color.Red; - + return ( checkIsBlue(Color.Blue), checkIsBlue(c), checkIsBlue(Color.createBlue()), Color.Blue.isBlue(), Color.Red.isBlue(), Color.createBlue()!!.isBlue(), C_BLUE.isBlue(), @@ -119,7 +119,7 @@ fun test3() { fun test4(chooseGreen: bool) { var c: ColorAlias = chooseGreen ? Color.Green : ColorAlias.Blue; return (c is Color, (c!) is ColorAlias, ColorAlias.Blue, ColorAlias.createBlue().isBlue(), c); -} +} @method_id(105) fun test5() { @@ -141,16 +141,16 @@ fun test7() { WithColor{c1: -100 as Color, c2: Color.createBlue()} ) } - + @method_id(108) fun test8() { var (c0: Color | int, c1: Color | int) = (Color.Red, 123); var (u1, u2) = (Color.Red as Color | int, 0 as Color | int); var (b1, b2, b3) = (Color.Red as Color | EPartialAssigned | builder, EPartialAssigned.Max as ColorAlias | builder | EPartialAssigned, beginCell() as builder | Color | EPartialAssigned); return ( - checkIsColor(u1), checkIsColor(u2), + checkIsColor(u1), checkIsColor(u2), checkIsColor(b1), checkIsColor(b2), checkIsColor(b3), - b2 is EPartialAssigned, + b2 is EPartialAssigned, u1, match (u1) { int => -100, Color => -200 } ) @@ -161,7 +161,7 @@ fun test9() { var invalidEx = 0; try { invalidEx = (99 as Color).mapToNumber() } catch (ex) { invalidEx = ex } - + return ( Color.Red.mapToNumber(), Color.Green.mapToNumber(), Color.Blue.mapToNumber(), invalidEx @@ -283,21 +283,21 @@ fun main(c: Color) { @testcase | 107 | | 0 1 -100 2 @testcase | 108 | | -1 0 -1 0 0 -1 0 typeid-1 -200 @testcase | 109 | | 100 200 300 5 -@testcase | 110 | 0 | [ 1 1 -100 88 -100 1 ] -@testcase | 110 | 1 | [ -100 2 -100 88 2 2 ] -@testcase | 110 | 2 | [ -100 3 -100 88 3 3 ] -@testcase | 111 | 2 | 0 -1 -1 0 +@testcase | 110 | 0 | [ 1 1 -100 88 -100 1 ] +@testcase | 110 | 1 | [ -100 2 -100 88 2 2 ] +@testcase | 110 | 2 | [ -100 3 -100 88 3 3 ] +@testcase | 111 | 2 | 0 -1 -1 0 @testcase | 112 | 0 1 | [ 210 120 ] @testcase | 112 | 2 2 | [ 330 310 ] -@testcase | 113 | -8 | 256 -1 -@testcase | 113 | 111 | 257 0 -@testcase | 113 | 10 | 258 0 -@testcase | 113 | 50 | 0 0 -@testcase | 114 | 2 | 2 1 0 1000000000 -1 +@testcase | 113 | -8 | 256 -1 +@testcase | 113 | 111 | 257 0 +@testcase | 113 | 10 | 258 0 +@testcase | 113 | 50 | 0 0 +@testcase | 114 | 2 | 2 1 0 1000000000 -1 @fif_codegen """ - test1() PROC:<{ + test1() PROC:<{ TRUE FALSE TRUE @@ -312,7 +312,7 @@ fun main(c: Color) { @fif_codegen """ test2() PROC:<{ // c - DROP // + DROP // -1 PUSHINT // '1=-1 0 PUSHINT // '1=-1 '2 0 PUSHINT // '1=-1 '2 '3=0 diff --git a/tolk-tester/tests/generics-4.tolk b/tolk-tester/tests/generics-4.tolk index 5b05064d2b..f86887abcd 100644 --- a/tolk-tester/tests/generics-4.tolk +++ b/tolk-tester/tests/generics-4.tolk @@ -217,16 +217,16 @@ fun main() { __expect_type(cb2, "() -> [int, cell, int, builder]?"); __expect_type(cb3, "() -> [int, cell, slice, builder]?"); __expect_type(cb4, "() -> [int, cell, slice, tuple]?"); - - var o2 = MultipleWithDefaults{}; - var o3 = MultipleWithDefaults{}; + + var o2 = MultipleWithDefaults{}; + var o3 = MultipleWithDefaults{}; var o4 = MultipleWithDefaults{}; __expect_type(o2.t, "[int, cell, int?, builder]?"); __expect_type(o3.t, "[int, cell, slice, builder]?"); __expect_type(o4.t, "[int, cell, slice?, tuple]?"); - + __expect_type(o2.getT(), "[int, cell, int?, builder]?"); - + __expect_type(o4.method1(), "[slice?, int?]"); __expect_type(o4.method1>(), "[slice?, Container1?]"); __expect_type(o4.method2(), "[tuple, cell?, int?]"); diff --git a/tolk-tester/tests/invalid-declaration/err-1505.tolk b/tolk-tester/tests/invalid-declaration/err-1505.tolk index ad861fd888..1e566e5bcc 100644 --- a/tolk-tester/tests/invalid-declaration/err-1505.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1505.tolk @@ -1,5 +1,5 @@ fun onInternalMessage(mutate in_msg_body: slice) { - + } /** diff --git a/tolk-tester/tests/invalid-semantics/err-4090.tolk b/tolk-tester/tests/invalid-semantics/err-4090.tolk index 7dfa456cdc..b445e40782 100644 --- a/tolk-tester/tests/invalid-semantics/err-4090.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4090.tolk @@ -16,5 +16,5 @@ fun main() { /** @compilation_should_fail -@stderr field `value` missed in initialization of struct `TryHackCompilerByNever` +@stderr field `value` missed in initialization of struct `TryHackCompilerByNever` */ diff --git a/tolk-tester/tests/invalid-semantics/err-4130.tolk b/tolk-tester/tests/invalid-semantics/err-4130.tolk index 907727f98e..19c50506f9 100644 --- a/tolk-tester/tests/invalid-semantics/err-4130.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4130.tolk @@ -18,5 +18,5 @@ fun f(l: ROLine) { /** @compilation_should_fail -@stderr modifying readonly field `ROLine.p2` +@stderr modifying readonly field `ROLine.p2` */ diff --git a/tolk-tester/tests/invalid-semantics/err-4180.tolk b/tolk-tester/tests/invalid-semantics/err-4180.tolk index 8935f3f62c..5652062cc3 100644 --- a/tolk-tester/tests/invalid-semantics/err-4180.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4180.tolk @@ -5,5 +5,5 @@ fun main() { /** @compilation_should_fail @stderr `mutate` is not available in lambdas -@stderr ^^^^^^ +@stderr ^^^^^^ */ diff --git a/tolk-tester/tests/invalid-semantics/err-4370.tolk b/tolk-tester/tests/invalid-semantics/err-4370.tolk index 82f867f66d..f7f9836b9c 100644 --- a/tolk-tester/tests/invalid-semantics/err-4370.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4370.tolk @@ -17,5 +17,5 @@ fun test(x: int, y: int) { /** @compilation_should_fail @stderr can not borrow `x` for mutation once again, it is already being mutated by `int.incrementBy` -@stderr ([y, x] = [0, 0]).0 +@stderr ([y, x] = [0, 0]).0 */ diff --git a/tolk-tester/tests/invalid-semantics/err-4372.tolk b/tolk-tester/tests/invalid-semantics/err-4372.tolk index 6012771adc..f0c0861992 100644 --- a/tolk-tester/tests/invalid-semantics/err-4372.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4372.tolk @@ -17,5 +17,5 @@ fun test(p: Point) { /** @compilation_should_fail -@stderr can not borrow `p` for mutation, because `p.x` is already being mutated by `assignment operator` +@stderr can not borrow `p` for mutation, because `p.x` is already being mutated by `assignment operator` */ diff --git a/tolk-tester/tests/invalid-semantics/err-4919.tolk b/tolk-tester/tests/invalid-semantics/err-4919.tolk index 1a26b4fa07..6fcdfdaecf 100644 --- a/tolk-tester/tests/invalid-semantics/err-4919.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4919.tolk @@ -21,5 +21,5 @@ fun main() { /** @compilation_should_fail @stderr method `method1` not found for type `WithDefaults` -@stderr (but it exists for type `WithDefaults` +@stderr (but it exists for type `WithDefaults` */ diff --git a/tolk-tester/tests/invalid-serialization/err-7883.tolk b/tolk-tester/tests/invalid-serialization/err-7883.tolk index fd824f2002..1f04dc4cf4 100644 --- a/tolk-tester/tests/invalid-serialization/err-7883.tolk +++ b/tolk-tester/tests/invalid-serialization/err-7883.tolk @@ -17,5 +17,5 @@ fun main() { /** @compilation_should_fail @stderr because field `TryHackCompilerByNever.value` of type `never` can't be serialized -@stderr because type `never` is not serializable +@stderr because type `never` is not serializable */ diff --git a/tolk-tester/tests/invalid-symbol/err-2190.tolk b/tolk-tester/tests/invalid-symbol/err-2190.tolk index b15475538e..eba27c17fb 100644 --- a/tolk-tester/tests/invalid-symbol/err-2190.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2190.tolk @@ -10,5 +10,5 @@ fun main(c: Color) { /** @compilation_should_fail -@stderr field `Red` doesn't exist in type `Color` +@stderr field `Red` doesn't exist in type `Color` */ diff --git a/tolk-tester/tests/invalid-syntax/err-3710.tolk b/tolk-tester/tests/invalid-syntax/err-3710.tolk index 00ff2e865a..aed3d5b899 100644 --- a/tolk-tester/tests/invalid-syntax/err-3710.tolk +++ b/tolk-tester/tests/invalid-syntax/err-3710.tolk @@ -1,7 +1,7 @@ -struct A { - private readonly a: () - readonly b: () - private private e: () +struct A { + private readonly a: () + readonly b: () + private private e: () } /** diff --git a/tolk-tester/tests/invalid-syntax/err-3853.tolk b/tolk-tester/tests/invalid-syntax/err-3853.tolk index 947c666906..be81833edb 100644 --- a/tolk-tester/tests/invalid-syntax/err-3853.tolk +++ b/tolk-tester/tests/invalid-syntax/err-3853.tolk @@ -1,5 +1,5 @@ fun main(int): int { - + } /** diff --git a/tolk-tester/tests/invalid-syntax/err-3967.tolk b/tolk-tester/tests/invalid-syntax/err-3967.tolk index 9a158a390f..1eca43d6db 100644 --- a/tolk-tester/tests/invalid-syntax/err-3967.tolk +++ b/tolk-tester/tests/invalid-syntax/err-3967.tolk @@ -1,5 +1,5 @@ int main() { - + } /** diff --git a/tolk-tester/tests/invalid-syntax/err-3981.tolk b/tolk-tester/tests/invalid-syntax/err-3981.tolk index 8158063ff7..1ac65d7147 100644 --- a/tolk-tester/tests/invalid-syntax/err-3981.tolk +++ b/tolk-tester/tests/invalid-syntax/err-3981.tolk @@ -3,5 +3,5 @@ struct(0x23456666666666666666666666666666666666666666666666666666666666666666666 /** @compilation_should_fail @stderr invalid integer constant -@stderr ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +@stderr ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ */ diff --git a/tolk-tester/tests/invalid-typing/err-6210.tolk b/tolk-tester/tests/invalid-typing/err-6210.tolk index 76a3b7c99b..d8e12eccbb 100644 --- a/tolk-tester/tests/invalid-typing/err-6210.tolk +++ b/tolk-tester/tests/invalid-typing/err-6210.tolk @@ -4,5 +4,5 @@ fun main() { /** @compilation_should_fail -@stderr can not pass `(slice) -> int` to `slice` +@stderr can not pass `(slice) -> int` to `slice` */ diff --git a/tolk-tester/tests/invalid-typing/err-6483.tolk b/tolk-tester/tests/invalid-typing/err-6483.tolk index 89ac9c5f1c..cdf9e015d2 100644 --- a/tolk-tester/tests/invalid-typing/err-6483.tolk +++ b/tolk-tester/tests/invalid-typing/err-6483.tolk @@ -7,5 +7,5 @@ fun recursive() { /** @compilation_should_fail @stderr could not infer return type of `recursive`, because it appears in a recursive call chain -@stderr declare `fun (...): ` manually +@stderr declare `fun (...): ` manually */ diff --git a/tolk-tester/tests/invalid-typing/err-6610.tolk b/tolk-tester/tests/invalid-typing/err-6610.tolk index 4e9de9056a..c513fdc0b4 100644 --- a/tolk-tester/tests/invalid-typing/err-6610.tolk +++ b/tolk-tester/tests/invalid-typing/err-6610.tolk @@ -10,7 +10,7 @@ fun main() { /** @compilation_should_fail -@stderr can not convert type `int` to return type `slice` +@stderr can not convert type `int` to return type `slice` @stderr in lambda in function `some` @stderr return c */ diff --git a/tolk-tester/tests/invalid-typing/err-6728.tolk b/tolk-tester/tests/invalid-typing/err-6728.tolk index 6da940f0af..7479cbad4c 100644 --- a/tolk-tester/tests/invalid-typing/err-6728.tolk +++ b/tolk-tester/tests/invalid-typing/err-6728.tolk @@ -3,7 +3,7 @@ struct Point { y: int8 } -fun Cell.myMethod(self, v: U): int +fun Cell.myMethod(self, v: U): int asm "DROP" "HASHCU" fun main() { diff --git a/tolk-tester/tests/lambdas-tests.tolk b/tolk-tester/tests/lambdas-tests.tolk index 6448e7d56f..8565923a6e 100644 --- a/tolk-tester/tests/lambdas-tests.tolk +++ b/tolk-tester/tests/lambdas-tests.tolk @@ -179,7 +179,7 @@ fun test11(v: bool) { var nullable_cb = v ? null : fun(x: int) { return x * 2 }; var union_int: int | (int -> int) = v ? v as int : fun(x: int) { return x * 2 }; var union_cn: (()->int) | (int->int) = v ? fun() { return 500 } : fun(x : int) { return x * 2 }; - + return ( nullable_cb != null ? nullable_cb(10) : null, (union_int is int) ? union_int : union_int(10), @@ -199,13 +199,13 @@ fun main() { @testcase | 101 | | 10 30 0 30 @testcase | 102 | 10 -20 | -20 -10 0 30 @testcase | 103 | | 16 2550 8 9 1 -@testcase | 104 | | 30 -@testcase | 105 | | 30 30 -@testcase | 106 | | 5 10 123 500 -@testcase | 107 | | -1 -@testcase | 108 | | -1 -1 123 typeid-1 -@testcase | 109 | | -1000 8 -@testcase | 110 | | 20 400 60 +@testcase | 104 | | 30 +@testcase | 105 | | 30 30 +@testcase | 106 | | 5 10 123 500 +@testcase | 107 | | -1 +@testcase | 108 | | -1 -1 123 typeid-1 +@testcase | 109 | | -1000 8 +@testcase | 110 | | 20 400 60 @testcase | 111 | -1 | (null) -1 -1 500 @testcase | 111 | 0 | 20 20 20 20 */ \ No newline at end of file diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index 122df21666..faf0fdffd2 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -1386,7 +1386,7 @@ fun demo312() { var st = lazy WithInternalAddress.fromCell(correct); __expect_lazy("[st] skip (bits8) (address), load f3"); val f3 = st.f3; - + val wrong = beginCell().storeInt(0, 8).storeAddressNone().storeAddress(addr).storeInt(-1, 8).endCell(); var st2 = lazy WithInternalAddress.fromCell(wrong); // check that `address` (st2.f2) is validated even though unused diff --git a/tolk-tester/tests/maps-tests.tolk b/tolk-tester/tests/maps-tests.tolk index a446f0325e..2255f99c09 100644 --- a/tolk-tester/tests/maps-tests.tolk +++ b/tolk-tester/tests/maps-tests.tolk @@ -117,7 +117,7 @@ fun test6() { m.set(2, 20); m.set(3, 30); return ( - m.findKeyGreater(1).getKey(), m.findKeyGreaterOrEqual(1).getKey(), + m.findKeyGreater(1).getKey(), m.findKeyGreaterOrEqual(1).getKey(), m.findKeyLess(3).getKey(), m.findKeyLessOrEqual(3).getKey(), m.findKeyLess(3).loadValue(), m.findKeyLessOrEqual(3).loadValue(), m.findKeyLessOrEqual(3).isFound, m.findKeyGreater(4).isFound, @@ -324,7 +324,7 @@ fun test26() { val f2 = m.deleteAndGetDeleted(1); val f3 = m.setAndGetPrevious(1, stringHexToSlice("01010101")); return ( - f1.isFound, f2.isFound, f3.isFound, + f1.isFound, f2.isFound, f3.isFound, f1.loadValue().remainingBitsCount(), f2.loadValue().remainingBitsCount(), m.mustGet(1).remainingBitsCount(), ); @@ -360,8 +360,8 @@ fun test29() { val a3 = m.addIfNotExists(3, DataWithCell{a: null, v: 0, p: Point{x:30,y:30}.toCell(), payload: createEmptyCell()}.toCell().beginParse()); val r4 = m.replaceAndGetPrevious(4, ""); return ( - a1, a2, a3, r4, - DataWithCell.fromSlice(m.mustGet(2)).payload, + a1, a2, a3, r4, + DataWithCell.fromSlice(m.mustGet(2)).payload, m.get(3).loadValue().loadAny().p!.load(), m.get(3).loadValue().remainingRefsCount(), m.findLast().getKey(), @@ -436,14 +436,14 @@ fun test35() { assert (!f1.isFound) throw 123; m.get(1).loadValue().set(10, 100); assert (m.mustGet(1).isEmpty()) throw 123; // the previous set() changed a copy - + var nested = createEmptyMap(); nested.set(1,10).set(2, 20); m.replaceIfExists(1, nested); nested.set(3, 30); assert (nested.calculateSize() == 3) throw 123; assert (m.mustGet(1).calculateSize() == 2) throw 123; - + val (k1, v2) = (m.mustGet(1).findFirst().getKey(), m.get(1).loadValue().findLast().loadValue()); val s1 = m.findFirst().loadValue().calculateSize(); m.mustGet(1).delete(10); @@ -520,7 +520,7 @@ fun test41() { val k7 = m.get(-7); return ( - m.exists(8), m.mustGet(111) == HashMode.TotalHash, + m.exists(8), m.mustGet(111) == HashMode.TotalHash, k7.isFound, k7.loadValue() == HashMode.NoHash, k7.loadValue() == HashMode.PartialHash ); } @@ -594,7 +594,7 @@ fun main() { @fif_codegen """ - test1() PROC:<{ // + test1() PROC:<{ // NEWDICT // m 1 PUSHINT // m '2=1 x{0000000a} PUSHSLICE @@ -638,7 +638,7 @@ fun main() { SWAP 267 PUSHINT DICTGET - 9 THROWIFNOT + 9 THROWIFNOT 8 LDI ENDS }> @@ -646,7 +646,7 @@ fun main() { @fif_codegen """ - test16() PROC:<{ + test16() PROC:<{ NEWDICT 100 PUSHINT x{0a14} PUSHSLICE @@ -657,7 +657,7 @@ fun main() { @fif_codegen """ - test25() PROC:<{ // + test25() PROC:<{ // NEWDICT // m 1 PUSHINT // m '2=1 x{0101} PUSHSLICE // m '2=1 '3 @@ -669,7 +669,7 @@ fun main() { @fif_codegen """ - test30() PROC:<{ + test30() PROC:<{ NEWDICT 1 PUSHINT PUSHREF @@ -680,7 +680,7 @@ fun main() { @fif_codegen """ - test33() PROC:<{ + test33() PROC:<{ NEWDICT 123 PUSHINT x{0a0a} PUSHSLICE @@ -714,7 +714,7 @@ fun main() { @fif_codegen """ - test36() PROC:<{ // + test36() PROC:<{ // x{9FFDE075922FCD6EC7F0A0F3E32D6828C8AE03395763E4C5ECF19503052F1829F45_} PUSHSLICE // k NEWDICT // k m x{0000000a} PUSHSLICE diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk index df09d51bc2..8414034902 100644 --- a/tolk-tester/tests/match-by-expr-tests.tolk +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -171,13 +171,13 @@ fun test13() { var ex2 = 0; // will throw (the compiler generates "else" implicitly for enums) - try { ex1 = mapRole(invalidAsmRole()); } + try { ex1 = mapRole(invalidAsmRole()); } catch (ex) { ex1 = ex } // will not throw (the compiler compiles "match" for bool as true => else =>) try { ex2 = mapBool(invalidAsmBool()); } catch (ex) { ex2 = ex } - + return (ex1, ex2); } @@ -344,9 +344,9 @@ fun main() { @fif_codegen """ 1 EQINT // '6 - IFNOTJMP:<{ // + IFNOTJMP:<{ // 5 THROW - }> // + }> // 200 PUSHINT // '1=200 """ diff --git a/tolk-tester/tests/meaningful-1.tolk b/tolk-tester/tests/meaningful-1.tolk index 45278ad647..616f6e1fd8 100644 --- a/tolk-tester/tests/meaningful-1.tolk +++ b/tolk-tester/tests/meaningful-1.tolk @@ -27,7 +27,7 @@ fun fakeParseMessage(mode: int): MyMessage { fun onInternalMessage(curCounter: int, mode: int) { var m = fakeParseMessage(mode); __expect_type(m, "MyMessage"); - + val nextCounter: int = match (m) { CounterReset => m.initialValue, CounterIncrement => curCounter + m.byValue, diff --git a/tolk-tester/tests/methods-tests.tolk b/tolk-tester/tests/methods-tests.tolk index c4d5948bcc..abb42a9b1a 100644 --- a/tolk-tester/tests/methods-tests.tolk +++ b/tolk-tester/tests/methods-tests.tolk @@ -297,7 +297,7 @@ fun AssetList.validate(self): int2 { return 0 } fun test19(b: BalanceList, a: AssetList) { __expect_type(b.validate(), "int1"); __expect_type(a.validate(), "int2"); - + if (b == null) {} match (a) { null => {} diff --git a/tolk-tester/tests/mutate-methods.tolk b/tolk-tester/tests/mutate-methods.tolk index cff11ebd53..e00d3e9afd 100644 --- a/tolk-tester/tests/mutate-methods.tolk +++ b/tolk-tester/tests/mutate-methods.tolk @@ -415,7 +415,7 @@ fun main(){ @fif_codegen """ - testReceiverAndNot() PROC:<{ // + testReceiverAndNot() PROC:<{ // 30 PUSHINT // '6=30 40 PUSHINT // obj.f1=30 obj.f2=40 OVER diff --git a/tolk-tester/tests/never-type-tests.tolk b/tolk-tester/tests/never-type-tests.tolk index 45043ff0e6..8a61d6423c 100644 --- a/tolk-tester/tests/never-type-tests.tolk +++ b/tolk-tester/tests/never-type-tests.tolk @@ -42,7 +42,7 @@ fun main() { @fif_codegen """ - bigThrow() PROC:<{ + bigThrow() PROC:<{ 1222222222222222222222222222222223 PUSHINT THROWANY }> diff --git a/tolk-tester/tests/overloads-tests.tolk b/tolk-tester/tests/overloads-tests.tolk index 70baa48b58..49ce79418a 100644 --- a/tolk-tester/tests/overloads-tests.tolk +++ b/tolk-tester/tests/overloads-tests.tolk @@ -263,7 +263,7 @@ type MMap8Of32 = map fun test14( m1: map>, m2: map>, m3: map>, m4: map, m5: MMap8Of32, - m6: map?>, + m6: map?>, m7: map, m8: map, m9: map, ) { diff --git a/tolk-tester/tests/pack-unpack-2.tolk b/tolk-tester/tests/pack-unpack-2.tolk index 8bdf1a6286..a79c7a7f65 100644 --- a/tolk-tester/tests/pack-unpack-2.tolk +++ b/tolk-tester/tests/pack-unpack-2.tolk @@ -792,7 +792,7 @@ fun test_serializeMaps() { } return (w.m1.isEmpty(), w.m2.isEmpty(), d3.isFound, d3.loadValue(), kl); } - + @method_id(231) fun test_nullableMaps() { var o: WithNullableMaps = { @@ -908,7 +908,7 @@ fun test_WithAnyAddress() { a6: createAddressNone().toCell(), a7: null, }, stringHexToSlice("2084_").appendRef(stringHexToSlice("2_"))); - + return true; } diff --git a/tolk-tester/tests/pack-unpack-5.tolk b/tolk-tester/tests/pack-unpack-5.tolk index 3b37aeec15..0b358d6a20 100644 --- a/tolk-tester/tests/pack-unpack-5.tolk +++ b/tolk-tester/tests/pack-unpack-5.tolk @@ -216,8 +216,8 @@ type EUint123Alias = EUint123 @method_id(121) fun test21() { return ( - Color.estimate(), EInt9.estimate(), EUint6.estimate(), - EUint1.estimate(), EInt1.estimate(), EInt1.estimate(), + Color.estimate(), EInt9.estimate(), EUint6.estimate(), + EUint1.estimate(), EInt1.estimate(), EInt1.estimate(), EUint123.estimate(), EUint123Alias.estimate(), ) } @@ -255,6 +255,6 @@ fun main() { @testcase | 110 | | [ 32 9999 0 4 ] [ 33 9999 0 8 ] @testcase | 111 | | [ 1023 1023 0 1 ] [ 0 0 2 2 ] @testcase | 120 | | 16 4128 1 1 -@testcase | 121 | | [ 2 2 0 0 ] [ 9 9 0 0 ] [ 6 6 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 123 123 0 0 ] [ 123 123 0 0 ] -@testcase | 122 | | [ 267 267 0 0 ] [ 2 267 0 0 ] [ 267 267 0 0 ] [ 1 268 0 0 ] [ 267 267 0 0 ] [ 2 267 0 0 ] [ 2 523 0 0 ] [ 1 524 0 0 ] +@testcase | 121 | | [ 2 2 0 0 ] [ 9 9 0 0 ] [ 6 6 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 123 123 0 0 ] [ 123 123 0 0 ] +@testcase | 122 | | [ 267 267 0 0 ] [ 2 267 0 0 ] [ 267 267 0 0 ] [ 1 268 0 0 ] [ 267 267 0 0 ] [ 2 267 0 0 ] [ 2 523 0 0 ] [ 1 524 0 0 ] */ diff --git a/tolk-tester/tests/pack-unpack-8.tolk b/tolk-tester/tests/pack-unpack-8.tolk index 16b24318c0..b62ede4e77 100644 --- a/tolk-tester/tests/pack-unpack-8.tolk +++ b/tolk-tester/tests/pack-unpack-8.tolk @@ -25,7 +25,7 @@ enum EFits8Bits { } const MAX_UINT = 115792089237316195423570985008687907853269984665640564039457584007913129639935 - + enum EMinMax { MIN_INT = -115792089237316195423570985008687907853269984665640564039457584007913129639935 - 1, MAX_INT = +115792089237316195423570985008687907853269984665640564039457584007913129639935, @@ -39,7 +39,7 @@ enum E0Max { struct WithEnumsUnion { u: EFits8Bits | EStartFromM2 } - + enum Role: int8 { Admin, User, } @@ -93,7 +93,7 @@ fun test3() { fun test4(v: EFillsAllUint3) { return EFillsAllUint3.fromCell(v.toCell()) } - + @method_id(105) fun test5(s: slice) { return EStartFromM2.fromSlice(s, {assertEndAfterReading: false}) @@ -145,7 +145,7 @@ fun test10() { w3.u is EStartFromM2 && w3.u == EStartFromM2.ZERO, w4.u is EStartFromM2 && w4.u == EStartFromM2.P3, w4.u is EFits8Bits && w4.u == EFits8Bits.E220, - ) + ) } @method_id(111) @@ -190,7 +190,7 @@ fun test14() { } fun main() { - + } /** @@ -219,7 +219,7 @@ fun main() { @fif_codegen """ - test1() PROC:<{ + test1() PROC:<{ b{00} PUSHSLICE 2 LDU OVER diff --git a/tolk-tester/tests/send-msg-3.tolk b/tolk-tester/tests/send-msg-3.tolk index c2965e5007..21a0e98535 100644 --- a/tolk-tester/tests/send-msg-3.tolk +++ b/tolk-tester/tests/send-msg-3.tolk @@ -77,8 +77,8 @@ fun test3(eventId: int) { val bodyAddress = address("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf") as any_address; val someRef = MyBody { queryId: 800 }.toCell(); var b = createExternalLogMessage({ - body: UnsafeBodyNoRef { - forceInline: Body3 { bodyAddress, someRef, amount: ton("0.05") } + body: UnsafeBodyNoRef { + forceInline: Body3 { bodyAddress, someRef, amount: ton("0.05") } }, dest: { topic: eventId }, }); diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 589bf65231..a8d37f0802 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -729,20 +729,20 @@ fun test64(d: dict, b: BalanceDict) { } __expect_type(d, "dict"); __expect_type(d.myDepth(), "int3"); - + match (b) { null => { __expect_type(b.myDepth(), "int4") } cell => { __expect_type(b.myDepth(), "int1") } } __expect_type(b, "BalanceDict"); __expect_type(b.myDepth(), "int3"); - + if (d == null) { return; } if (b != null) { return; } __expect_type(d.myDepth(), "int1"); __expect_type(b.myDepth(), "int4"); - + if ([0].0) { (b, d) = (createEmptyCell(), null) } @@ -779,7 +779,7 @@ fun test65(w: WithDict, t: (dict, BalanceDict)) { fun main(x: int?): int { // mark used to codegen them test55; - + return x == null ? -1 : x; } diff --git a/tolk-tester/tests/some-tests-4.tolk b/tolk-tester/tests/some-tests-4.tolk index e6f6a070c9..7956a8fb62 100644 --- a/tolk-tester/tests/some-tests-4.tolk +++ b/tolk-tester/tests/some-tests-4.tolk @@ -101,9 +101,9 @@ fun main() { @fif_codegen """ test2() PROC:<{ - IFJMP:<{ + IFJMP:<{ 305482242253527345283975278648594775678 PUSHINT // '1=305482242253527345283975278648594775678 - }> + }> 305482242253527345283975278648594775678 PUSHINT // '2=305482242253527345283975278648594775678 }> """ diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 262c968ffb..3bd84a5626 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -627,10 +627,10 @@ fun test45() { otherTup: [] as tuple, }; return WithTensorInside{ - coords: (1, 2), - tup: [3, 4], - otherCoords: (5, 6), - otherTup: WithTensorInside.fromTuple(inner.toTuple()).toTuple() + coords: (1, 2), + tup: [3, 4], + otherCoords: (5, 6), + otherTup: WithTensorInside.fromTuple(inner.toTuple()).toTuple() }.toTuple(); } @@ -711,7 +711,7 @@ type PointAlias = Point; @testcase | 143 | | 0 0 20 0 @testcase | 144 | | [ 10 20 0 ] [ 10 ] [ 0 0 ] 1 -1 @testcase | 145 | | [ 1 2 [ 3 4 ] 5 6 typeid-8 [ 10 20 [ -1 -2 ] (null) (null) 0 [] ] ] -@testcase | 146 | | 20 [ 11 12 13 14 15 ] +@testcase | 146 | | 20 [ 11 12 13 14 15 ] @fif_codegen """ diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk index 587e16433e..b32b3e8f92 100644 --- a/tolk-tester/tests/type-aliases-tests.tolk +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -326,7 +326,7 @@ fun test15() { m = i; m = t; t = i; t = m; } - + return ( i.plus0(), i.plus1(), i.plus2(), m.plus0(), m.plus1(), m.plus2(), diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 1427f598d6..bcd5d33b17 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -619,7 +619,7 @@ fun test51(init: int) { slice => { if (a !is int) { a = a.loadInt(32) as int8; - } + } } } __expect_type(a, "int"); @@ -787,7 +787,7 @@ fun test62() { ((5 as int64) as int32 | int64).toTuple(), myFromTuple([10, 48] as tuple) is int64, myFromTuple([10, 48] as tuple).typeNameOfObject().bitsEqual("int32 | int64"), - ) + ) } diff --git a/tolk-tester/tests/warnings-not-errors/unreachable-6.tolk b/tolk-tester/tests/warnings-not-errors/unreachable-6.tolk index 9471a88161..90574d50f1 100644 --- a/tolk-tester/tests/warnings-not-errors/unreachable-6.tolk +++ b/tolk-tester/tests/warnings-not-errors/unreachable-6.tolk @@ -22,7 +22,7 @@ fun checkBool(b: bool) { match (b) { true => return 1, false => return 100, - } + } throw 456; } diff --git a/tolk-tester/tests/warnings-not-errors/warnings-5.tolk b/tolk-tester/tests/warnings-not-errors/warnings-5.tolk index 009464c4da..bacc1831bd 100644 --- a/tolk-tester/tests/warnings-not-errors/warnings-5.tolk +++ b/tolk-tester/tests/warnings-not-errors/warnings-5.tolk @@ -10,5 +10,5 @@ fun main(x: int, y: int) { @testcase | 0 | 0 0 | 21 @stderr assignment inside condition, probably it's a misprint @stderr if (x = 1) -@stderr assert ((y = 20)) +@stderr assert ((y = 20)) */ diff --git a/tolk/abscode.cpp b/tolk/abscode.cpp index 6fd0565196..eeb4351d4e 100644 --- a/tolk/abscode.cpp +++ b/tolk/abscode.cpp @@ -21,9 +21,9 @@ namespace tolk { /* - * + * * ABSTRACT CODE - * + * */ void TmpVar::show_as_stack_comment(std::ostream& os) const { diff --git a/tolk/analyzer.cpp b/tolk/analyzer.cpp index 356d5b017a..85f42562ff 100644 --- a/tolk/analyzer.cpp +++ b/tolk/analyzer.cpp @@ -30,9 +30,9 @@ static bool does_function_always_throw(FunctionPtr fun_ref) { } /* - * + * * ANALYZE AND PREPROCESS ABSTRACT CODE - * + * */ bool CodeBlob::compute_used_code_vars() { diff --git a/tolk/asmops.cpp b/tolk/asmops.cpp index df0c5ef778..7e558a9c7e 100644 --- a/tolk/asmops.cpp +++ b/tolk/asmops.cpp @@ -21,9 +21,9 @@ namespace tolk { /* - * + * * ASM-OP LIST FUNCTIONS - * + * */ int is_pos_pow2(td::RefInt256 x) { diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index f90c6ee44f..25e1819acc 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -846,7 +846,7 @@ static V parse_match_expression(Lexer& lex) { static V parse_lambda_fun_expression(Lexer& lex) { SrcRange range = lex.range_start(); lex.expect(tok_fun, "`fun`"); - + V v_param_list = parse_parameter_list(lex, nullptr, true); AnyTypeV ret_type = nullptr; @@ -857,7 +857,7 @@ static V parse_lambda_fun_expression(Lexer& lex) { auto v_body = parse_block_statement(lex); range.end(v_body->range); - return createV(range, v_param_list, v_body, ret_type); + return createV(range, v_param_list, v_body, ret_type); } static V parse_lazy_operator(Lexer& lex) { @@ -1084,7 +1084,7 @@ static AnyExprV parse_expr75(Lexer& lex) { AnyExprV rhs = parse_expr75(lex); range.end(rhs->range); - // convert `-1` to `int(-1)`, not to a tree `unary(-) > int(1)` right here + // convert `-1` to `int(-1)`, not to a tree `unary(-) > int(1)` right here if (auto rhs_int = rhs->try_as(); rhs_int && (t == tok_minus || t == tok_plus)) { td::RefInt256 intval = rhs_int->intval; tolk_assert(!intval.is_null()); @@ -1710,7 +1710,7 @@ static AnyV parse_struct_field(Lexer& lex) { lex.next(); is_readonly = true; } - + auto v_ident = parse_identifier(lex, "field name"); lex.expect(tok_colon, "`: `"); AnyTypeV declared_type = parse_type_from_tokens(lex); @@ -1813,7 +1813,7 @@ static AnyV parse_enum_member(Lexer& lex) { range.end(init_value->range); } - return createV(range, v_ident, init_value); + return createV(range, v_ident, init_value); } static V parse_enum_body(Lexer& lex) { diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index c07207591a..5a06eec3b4 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -375,7 +375,7 @@ class ASTReplicator final { // convert a lambda expression `fun(params) { ... }` into a full function declaration // (the instantiated function will be added to G.all_functions and exist as a standalone function) - static V clone_lambda_as_standalone(V v_lambda) { + static V clone_lambda_as_standalone(V v_lambda) { return createV( v_lambda->range, createV(v_lambda->keyword_range(), "lambda"), // it's not a real name, for AST only diff --git a/tolk/ast.h b/tolk/ast.h index 4d07244e95..ef97d72a7e 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -183,7 +183,7 @@ struct ASTNodeBase { if (range.is_defined() && !range.is_valid()) { throw Fatal("forgot to end() range in ast"); } -#endif +#endif } ASTNodeBase(const ASTNodeBase&) = delete; @@ -1059,7 +1059,7 @@ struct Vertex final : ASTExprLeaf { auto get_param(int i) const { return parameters->get_param(i); } auto get_body() const { return body; } SrcRange keyword_range() const { return SrcRange::span(range, 3); } - + Vertex* mutate() const { return const_cast(this); } void assign_lambda_ref(FunctionPtr lambda_ref); diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 9f324aa46a..a99b80d0da 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -87,9 +87,9 @@ void FunctionBodyAsm::compile(AsmOpList& dest, AnyV origin) const { /* - * + * * DEFINE BUILT-IN FUNCTIONS - * + * */ int emulate_negate(int a) { @@ -1267,7 +1267,7 @@ static AsmOp compile_debug_print_to_string(std::vector&, std::vector& res, std::vector& args, AnyV origin) { tolk_assert(res.size() == 1); int n_slots = static_cast(args.size()); - std::string op_make_tuple = std::to_string(n_slots) + (n_slots > 15 ? " PUSHINT TUPLEVAR" : " TUPLE"); + std::string op_make_tuple = std::to_string(n_slots) + (n_slots > 15 ? " PUSHINT TUPLEVAR" : " TUPLE"); return exec_op(origin, op_make_tuple, n_slots, 1); } @@ -1275,7 +1275,7 @@ static AsmOp compile_T_to_tuple(std::vector& res, std::vector& res, std::vector& args, AnyV origin) { tolk_assert(args.size() == 1); int n_slots = static_cast(res.size()); - std::string op_un_tuple = std::to_string(n_slots) + (n_slots > 15 ? " PUSHINT UNTUPLEVAR" : " UNTUPLE"); + std::string op_un_tuple = std::to_string(n_slots) + (n_slots > 15 ? " PUSHINT UNTUPLEVAR" : " UNTUPLE"); return exec_op(origin, op_un_tuple, 1, n_slots); } diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp index 9aa0e8b7e8..2581d32991 100644 --- a/tolk/codegen.cpp +++ b/tolk/codegen.cpp @@ -28,9 +28,9 @@ static void sanitize_fift_name(std::string &name) { } /* - * + * * GENERATE TVM STACK CODE - * + * */ StackLayout Stack::vars() const { diff --git a/tolk/compilation-errors.cpp b/tolk/compilation-errors.cpp index 8d952eacca..c25e601427 100644 --- a/tolk/compilation-errors.cpp +++ b/tolk/compilation-errors.cpp @@ -27,7 +27,7 @@ static std::string str_in_function(FunctionPtr f) { if (f->is_lambda()) { return "in lambda " + str_in_function(f->base_fun_ref); } - return "in function `" + f->as_human_readable() + "`"; + return "in function `" + f->as_human_readable() + "`"; } void on_assertion_failed(const char *description, const char *file_name, int line_number) { @@ -158,11 +158,11 @@ Error ErrorBuilder::build() const { } void Error::fire(AnyV at, FunctionPtr in_function) const { - throw ThrownParseError(str_in_function(in_function), at->range, message); + throw ThrownParseError(str_in_function(in_function), at->range, message); } void Error::fire(SrcRange range, FunctionPtr in_function) const { - throw ThrownParseError(str_in_function(in_function), range, message); + throw ThrownParseError(str_in_function(in_function), range, message); } void Error::warning(AnyV at, FunctionPtr in_function) const { diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index ed64e33dca..45924e7dde 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -260,7 +260,7 @@ static ConstValExpression parse_vertex_call_to_compile_time_function(V int(-5), `!true` => false + // `-5` => int(-5), `!true` => false static ConstValExpression handle_unary_operator(V v) { ConstValExpression expr = eval_any_v_or_fire(v->get_rhs()); @@ -296,7 +296,7 @@ class ConstExpressionEvaluator { // `2 + 3` => int(5), `10 > 3` => true, `true & false` => 0 static ConstValExpression handle_binary_operator(V v) { - ConstValExpression expr_lhs = eval_any_v_or_fire(v->get_lhs()); + ConstValExpression expr_lhs = eval_any_v_or_fire(v->get_lhs()); ConstValExpression expr_rhs = eval_any_v_or_fire(v->get_rhs()); td::RefInt256 lhs; @@ -306,7 +306,7 @@ class ConstExpressionEvaluator { lhs = i_lhs->int_val; } else if (const ConstValBool* b_lhs = std::get_if(&expr_lhs)) { lhs = td::make_refint(b_lhs->bool_val ? -1 : 0); - } + } if (const ConstValInt* i_rhs = std::get_if(&expr_rhs)) { rhs = i_rhs->int_val; @@ -385,10 +385,10 @@ class ConstExpressionEvaluator { if (!fun_ref || !fun_ref->is_compile_time_const_val()) { err_not_a_constant_expression().fire(v); } - + return parse_vertex_call_to_compile_time_function(v, fun_ref->name); } - + // `const a = ANOTHER`, or in field default, enum member, etc. static ConstValExpression handle_reference(V v) { GlobalConstPtr const_ref = v->sym->try_as(); @@ -467,7 +467,7 @@ class ConstExpressionEvaluator { std::vector> fields; fields.reserve(v_body->size()); for (int i = 0; i < v_body->size(); ++i) { - AnyExprV field_init_val = v_body->get_field(i)->get_init_val(); + AnyExprV field_init_val = v_body->get_field(i)->get_init_val(); check_expression_is_constant_or_fire(field_init_val); fields.emplace_back(v_body->get_field(i)->field_ref, field_init_val); } @@ -493,7 +493,7 @@ ConstValExpression eval_constant_expression_or_fire(AnyExprV v_expr) { if (auto v_int = v_expr->try_as()) { return ConstValInt{v_int->intval}; } - return ConstExpressionEvaluator::eval_any_v_or_fire(v_expr); + return ConstExpressionEvaluator::eval_any_v_or_fire(v_expr); } ConstValExpression eval_and_cache_const_init_val(GlobalConstPtr const_ref) { diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index a675dcdf10..2ec0b6b80c 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -554,7 +554,7 @@ FunctionPtr instantiate_lambda_function(AnyV v_lambda, FunctionPtr parent_fun_re lambda_ref->mutate()->assign_resolved_type(return_type); run_pipeline_for_cloned_function(lambda_ref); - return lambda_ref; + return lambda_ref; } // a function `tuple.push(self, v: T) asm "TPUSH"` can't be called with T=Point (2 stack slots); @@ -567,7 +567,7 @@ bool is_allowed_asm_generic_function_with_non1_width_T(FunctionPtr fun_ref, int // allow "Cell.hash", "map.isEmpty" and other methods that don't depend on internal structure if (fun_ref->is_method() && idxT < fun_ref->genericTs->n_from_receiver) { - TypePtr receiver = fun_ref->receiver_type->unwrap_alias(); + TypePtr receiver = fun_ref->receiver_type->unwrap_alias(); if (const auto* r_withTs = receiver->try_as()) { return r_withTs->struct_ref && r_withTs->struct_ref->name == "Cell"; } diff --git a/tolk/lexer.h b/tolk/lexer.h index f99134a4c8..5fc0456b3d 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -159,7 +159,7 @@ class Lexer { int cur_token_offset = 0; void update_location() { - cur_token_offset = static_cast(p_next - p_start); + cur_token_offset = static_cast(p_next - p_start); } public: diff --git a/tolk/maps-kv-api.cpp b/tolk/maps-kv-api.cpp index 0c7e674aac..9d92af5b3e 100644 --- a/tolk/maps-kv-api.cpp +++ b/tolk/maps-kv-api.cpp @@ -107,7 +107,7 @@ static bool is_TValue_raw_slice(TypePtr TValue) { // `map>` can emit SETREF instructions static bool is_TValue_cell_or_CellT(TypePtr TValue) { - return TValue->unwrap_alias() == TypeDataCell::create() || is_type_cellT(TValue->unwrap_alias()); + return TValue->unwrap_alias() == TypeDataCell::create() || is_type_cellT(TValue->unwrap_alias()); } bool check_mapKV_TKey_is_valid(TypePtr TKey, std::string& because_msg) { @@ -115,7 +115,7 @@ bool check_mapKV_TKey_is_valid(TypePtr TKey, std::string& because_msg) { return true; } - // okay, not a trivial key — it must be a serializable struct of a constant size + // okay, not a trivial key — it must be a serializable struct of a constant size if (TKey->unwrap_alias() == TypeDataSlice::create()) { // a dedicated error message for `map` because_msg = "because it does not specify keyLen for a dictionary\n""hint: use `address` if a key is an internal address\n""hint: use `bits128` and similar if a key represents fixed-width data"; return false; @@ -153,7 +153,7 @@ bool check_mapKV_TValue_is_valid(TypePtr TValue, std::string& because_msg) { } // note that `struct A { s: slice }` can not be used as a value (not serializable), // although `slice` can, because in stdlib behavior for TValue=slice is overloaded (no deserialization) - + return true; } @@ -301,7 +301,7 @@ static std::vector create_ir_MapEntry(CodeBlob& code, AnyV origin) { return code.create_tmp_var(TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create(), TypeDataInt::create()}), origin, "(entry)"); } -// see a comment above construct_MapEntry_with_non_trivial_key() +// see a comment above construct_MapEntry_with_non_trivial_key() static std::vector finalize_ir_MapEntry(CodeBlob& code, AnyV origin, std::vector&& ir_entry, TypePtr TKey) { if (!is_TKey_TVM_int(TKey) && !is_TKey_TVM_uint(TKey) && !is_TKey_TVM_slice(TKey)) { ir_entry = construct_MapEntry_with_non_trivial_key(code, origin, std::move(ir_entry), TKey); @@ -321,7 +321,7 @@ static std::string choose_dict_op(std::string_view op_slice, VarDescr& var_with_ DictKeyKind key_kind = static_cast(var_with_key_kind.int_const->to_long()); var_with_key_kind.unused(); - + if (key_kind == DictKeyKind::UIntKey) op = "DICTU" + op.substr(4); if (key_kind == DictKeyKind::IntKey) op = "DICTI" + op.substr(4); @@ -533,7 +533,7 @@ std::vector generate_mapKV_setGet(FunctionPtr called_f, CodeBlob& cod std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_value_val(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; code.emplace_back(origin, Op::_Call, ir_map_and_lookup, std::move(dict_args), lookup_function("__dict.setGet")); - return ir_map_and_lookup; + return ir_map_and_lookup; } // fun map.replaceIfExists(mutate self, key: K, value: V): bool @@ -616,7 +616,7 @@ std::vector generate_mapKV_delGet(FunctionPtr called_f, CodeBlob& cod std::vector generate_mapKV_findFirst(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); DictKeyValue kv(code, origin, TKey, nullptr, TypeDataSlice::create(), nullptr); - + // on a stack (ir_entry) we will have: either (x k −1) or (null null 0) std::vector ir_entry = create_ir_MapEntry(code, origin); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), args[0][0], kv.ir_key_len() }; @@ -629,7 +629,7 @@ std::vector generate_mapKV_findFirst(FunctionPtr called_f, CodeBlob& std::vector generate_mapKV_findLast(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); DictKeyValue kv(code, origin, TKey, nullptr, TypeDataSlice::create(), nullptr); - + std::vector ir_entry = create_ir_MapEntry(code, origin); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), args[0][0], kv.ir_key_len() }; code.emplace_back(origin, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getMax")); @@ -641,7 +641,7 @@ std::vector generate_mapKV_findLast(FunctionPtr called_f, CodeBlob& c std::vector generate_mapKV_findKeyGreater(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); DictKeyValue kv(code, origin, TKey, &args[1], TypeDataSlice::create(), nullptr); - + std::vector ir_entry = create_ir_MapEntry(code, origin); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; code.emplace_back(origin, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getNext")); @@ -653,7 +653,7 @@ std::vector generate_mapKV_findKeyGreater(FunctionPtr called_f, CodeB std::vector generate_mapKV_findKeyGreaterOrEqual(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); DictKeyValue kv(code, origin, TKey, &args[1], TypeDataSlice::create(), nullptr); - + std::vector ir_entry = create_ir_MapEntry(code, origin); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; code.emplace_back(origin, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getNextEq")); @@ -665,7 +665,7 @@ std::vector generate_mapKV_findKeyGreaterOrEqual(FunctionPtr called_f std::vector generate_mapKV_findKeyLess(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); DictKeyValue kv(code, origin, TKey, &args[1], TypeDataSlice::create(), nullptr); - + std::vector ir_entry = create_ir_MapEntry(code, origin); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; code.emplace_back(origin, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getPrev")); @@ -677,7 +677,7 @@ std::vector generate_mapKV_findKeyLess(FunctionPtr called_f, CodeBlob std::vector generate_mapKV_findKeyLessOrEqual(FunctionPtr called_f, CodeBlob& code, AnyV origin, const std::vector>& args) { TypePtr TKey = called_f->substitutedTs->typeT_at(0); DictKeyValue kv(code, origin, TKey, &args[1], TypeDataSlice::create(), nullptr); - + std::vector ir_entry = create_ir_MapEntry(code, origin); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; code.emplace_back(origin, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getPrevEq")); @@ -690,7 +690,7 @@ std::vector generate_mapKV_iterateNext(FunctionPtr called_f, CodeBlob std::vector ir_pivot_key(args[1].begin() + 1, args[1].end() - 1); TypePtr TKey = called_f->substitutedTs->typeT_at(0); DictKeyValue kv(code, origin, TKey, &ir_pivot_key, TypeDataSlice::create(), nullptr); - + std::vector ir_entry = create_ir_MapEntry(code, origin); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; code.emplace_back(origin, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getNext")); @@ -703,7 +703,7 @@ std::vector generate_mapKV_iteratePrev(FunctionPtr called_f, CodeBlob std::vector ir_pivot_key(args[1].begin() + 1, args[1].end() - 1); TypePtr TKey = called_f->substitutedTs->typeT_at(0); DictKeyValue kv(code, origin, TKey, &ir_pivot_key, TypeDataSlice::create(), nullptr); - + std::vector ir_entry = create_ir_MapEntry(code, origin); std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; code.emplace_back(origin, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getPrev")); diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index 371cb8620b..67a973d8aa 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -19,9 +19,9 @@ namespace tolk { /* - * + * * PEEPHOLE OPTIMIZER - * + * */ void Optimizer::set_code(AsmOpConsList code) { @@ -454,7 +454,7 @@ bool Optimizer::detect_rewrite_0EQINT_THROWIF() { return true; } } - + return false; } @@ -480,7 +480,7 @@ bool Optimizer::detect_rewrite_DICTSETB_DICTSET() { std::string new_op = op_[4]->op; // "DICTSET" / "DICTSETGET NULLSWAPIFNOT" for (size_t i = 0; i < std::size(contains_b); ++i) { if (size_t pos = new_op.find(contains_b[i]); pos == 4 || pos == 5) { - new_op.replace(pos, std::strlen(contains_b[i]), repl_with[i]); + new_op.replace(pos, std::strlen(contains_b[i]), repl_with[i]); p_ = 5; q_ = 4; oq_[0] = std::make_unique(AsmOp::Custom(op_[1]->origin, op_[1]->op.substr(0, op_[1]->op.rfind(' ')) + " PUSHSLICE", 0, 1)); @@ -488,7 +488,7 @@ bool Optimizer::detect_rewrite_DICTSETB_DICTSET() { oq_[2] = std::move(op_[3]); oq_[3] = std::make_unique(AsmOp::Custom(op_[4]->origin, new_op)); return true; - } + } } return false; @@ -555,7 +555,7 @@ bool Optimizer::detect_rewrite_ENDC_HASHCU() { // pattern `NEWC` + `BTOS` -> `x{} PUSHSLICE` bool Optimizer::detect_rewrite_NEWC_BTOS() { bool first_newc = op_[0]->is_custom() && op_[0]->op == "NEWC"; - if (!first_newc || pb_ < 2) { + if (!first_newc || pb_ < 2) { return false; } @@ -573,7 +573,7 @@ bool Optimizer::detect_rewrite_NEWC_BTOS() { // pattern `NEWC` + `x{...} STSLICECONST` + `BTOS` -> `x{...} PUSHSLICE` bool Optimizer::detect_rewrite_NEWC_STSLICECONST_BTOS() { bool first_newc = op_[0]->is_custom() && op_[0]->op == "NEWC"; - if (!first_newc || pb_ < 3) { + if (!first_newc || pb_ < 3) { return false; } @@ -594,13 +594,13 @@ bool Optimizer::detect_rewrite_NEWC_STSLICECONST_BTOS() { // pattern `NEWC` + `ENDC` + `CTOS` -> `x{} PUSHSLICE` bool Optimizer::detect_rewrite_NEWC_ENDC_CTOS() { bool first_newc = op_[0]->is_custom() && op_[0]->op == "NEWC"; - if (!first_newc || pb_ < 3) { + if (!first_newc || pb_ < 3) { return false; } bool next_endc = op_[1]->is_custom() && op_[1]->op == "ENDC"; bool next_ctos = op_[2]->is_custom() && op_[2]->op == "CTOS"; - if (!next_endc || !next_ctos) { + if (!next_endc || !next_ctos) { return false; } @@ -613,7 +613,7 @@ bool Optimizer::detect_rewrite_NEWC_ENDC_CTOS() { // pattern `NEWC` + `ENDC` -> ` PUSHREF` bool Optimizer::detect_rewrite_NEWC_ENDC() { bool first_newc = op_[0]->is_custom() && op_[0]->op == "NEWC"; - if (!first_newc || pb_ < 2) { + if (!first_newc || pb_ < 2) { return false; } diff --git a/tolk/overload-resolution.cpp b/tolk/overload-resolution.cpp index 9bb05c50e9..3f620545a7 100644 --- a/tolk/overload-resolution.cpp +++ b/tolk/overload-resolution.cpp @@ -111,7 +111,7 @@ static ShapeScore calculate_shape_score(TypePtr t) { // tries to find Ts in `pattern` to reach `actual`; // example: pattern=`map`, actual=`map` => T=int -// example: pattern=`Container`, actual=`Container>` => T=Container +// example: pattern=`Container`, actual=`Container>` => T=Container static bool can_substitute_Ts_to_reach_actual(TypePtr pattern, TypePtr actual, const GenericsDeclaration* genericTs) { GenericSubstitutionsDeducing deducingTs(genericTs); TypePtr replaced = deducingTs.auto_deduce_from_argument(pattern, actual); @@ -120,7 +120,7 @@ static bool can_substitute_Ts_to_reach_actual(TypePtr pattern, TypePtr actual, c // checks whether a generic typeA is more specific than typeB; // example: `map` dominates `map`; -// example: `map>` dominates `map>` dominates `map>`; +// example: `map>` dominates `map>` dominates `map>`; // example: `map` and `map` are not comparable; static bool is_more_specific_generic(TypePtr typeA, TypePtr typeB, const GenericsDeclaration* genericTsA, const GenericsDeclaration* genericTsB) { // exists θ: θ(B)=A && not exists φ: φ(A)=B @@ -197,7 +197,7 @@ std::vector resolve_methods_for_call(TypePtr provided_recei best_shape = s; } } - + std::vector best_by_shape; for (const MethodCallCandidate& candidate : viable) { if (calculate_shape_score(candidate.original_receiver) == best_shape) { diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index 29be8e738f..4b3ead3b4c 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -127,7 +127,7 @@ class PackUnpackAvailabilityChecker { } return {}; } - + if (const auto* t_union = any_type->try_as()) { // a union can almost always be serialized if every of its variants can: // - `T?` is TL/B `(Maybe T)` diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index beb5d979b7..e2d8b8d302 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -68,7 +68,7 @@ void get_custom_pack_unpack_function(TypePtr receiver_type, FunctionPtr& f_pack, f_pack = sym->try_as(); } if (const Symbol* sym = lookup_global_symbol(receiver_name + ".unpackFromSlice")) { - f_unpack = sym->try_as(); + f_unpack = sym->try_as(); } } } @@ -1130,7 +1130,7 @@ struct S_IntegerEnum final : ISerializer { // then validation is: "throw if vB", but "LESSINT + THROWIF" 2 times is more generalized td::RefInt256 min_value = enum_ref->members.front()->computed_value; bool dont_check_min = intN != nullptr && intN->is_unsigned && min_value == 0; - if (!dont_check_min) { // LDU can't load < 0 + if (!dont_check_min) { // LDU can't load < 0 std::vector ir_min_value = code.create_tmp_var(TypeDataInt::create(), origin, "(enum-min)"); code.emplace_back(origin, Op::_IntConst, ir_min_value, min_value); std::vector ir_lt_min = code.create_tmp_var(TypeDataInt::create(), origin, "(enum-lt-min)"); @@ -1273,7 +1273,7 @@ std::vector auto_generate_opcodes_for_union(TypePtr union_type, std: // examples: `int32 | int64 | int128` / `int32 | A | null` / `A | B` / `A | B | C`; // if `null` exists, it's 0, all others are 1+tree: A|B|C|D|null => 0 | 100+A | 101+B | 110+C | 111+D; // if no `null`, just distribute sequentially: A|B|C => 00+A | 01+B | 10+C - int n_without_null = t_union->size() - has_null; + int n_without_null = t_union->size() - has_null; int prefix_len = static_cast(std::ceil(std::log2(n_without_null))); int cur_prefix = 0; for (TypePtr variant : t_union->variants) { @@ -1295,7 +1295,7 @@ TypePtr calculate_intN_to_serialize_enum(EnumDefPtr enum_ref) { if (enum_ref->colon_type) { // intN / coins return enum_ref->colon_type; } - + bool is_unsigned = false; int n_bits = 1; for (; n_bits <= 256; ++n_bits) { @@ -1326,7 +1326,7 @@ std::vector create_default_PackOptions(CodeBlob& code, AnyV origin) { std::vector ir_defaults = { code.create_int(origin, 0, "(zero)"), // skipBitsNFieldsValidation }; - code.emplace_back(origin, Op::_Let, ir_options, std::move(ir_defaults)); + code.emplace_back(origin, Op::_Let, ir_options, std::move(ir_defaults)); return ir_options; } @@ -1340,7 +1340,7 @@ std::vector create_default_UnpackOptions(CodeBlob& code, AnyV origin) code.create_int(origin, -1, "(true)"), // assertEndAfterReading code.create_int(origin, 63, "(excno)"), // throwIfOpcodeDoesNotMatch }; - code.emplace_back(origin, Op::_Let, ir_options, std::move(ir_defaults)); + code.emplace_back(origin, Op::_Let, ir_options, std::move(ir_defaults)); return ir_options; } @@ -1486,7 +1486,7 @@ std::vector UnpackContext::generate_lazy_match_any(TypePtr any_type, if (auto* s = dynamic_cast(serializer.get())) { return s->lazy_match(this, code, origin, options); } - tolk_assert(false); + tolk_assert(false); } PackSize EstimateContext::estimate_any(TypePtr any_type, PrefixEstimateMode prefix_mode) const { diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index b5bd3dda69..a270abedbf 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1140,7 +1140,7 @@ std::vector transition_to_target_type(std::vector&& rvect, } // convert a constant value (calculated by a "constant-evaluator") to IR vars; -// every init_val of `const XXX = ...` is calculated once (into ConstValExpression) and cached +// every init_val of `const XXX = ...` is calculated once (into ConstValExpression) and cached static std::vector pre_compile_constant_expression(const ConstValExpression& value, CodeBlob& code, AnyV origin) { if (const ConstValInt* val = std::get_if(&value)) { std::vector rvect = code.create_tmp_var(TypeDataInt::create(), origin, "(int-const)"); @@ -1489,7 +1489,7 @@ static std::vector process_match_expression(V v } // how to compare subject and branches for `match` expression, similar to operator `==` which can handle non-integers - FunctionPtr eq_fn = lookup_function("_==_"); // for int/bool/enum + FunctionPtr eq_fn = lookup_function("_==_"); // for int/bool/enum if (subject_type->unwrap_alias()->try_as()) { eq_fn = lookup_function("slice.bitsEqual"); } diff --git a/tolk/pipe-calc-rvalue-lvalue.cpp b/tolk/pipe-calc-rvalue-lvalue.cpp index ea7fd8b433..ff97045825 100644 --- a/tolk/pipe-calc-rvalue-lvalue.cpp +++ b/tolk/pipe-calc-rvalue-lvalue.cpp @@ -73,7 +73,7 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { mark_vertex(v); } - + void visit(V v) override { mark_vertex(v); MarkingState saved = enter_rvalue_if_none(); @@ -101,7 +101,7 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { parent::visit(v); restore_state(saved); } - + void visit(V v) override { mark_vertex(v); MarkingState saved = enter_rvalue_if_none(); diff --git a/tolk/pipe-check-const-expr.cpp b/tolk/pipe-check-const-expr.cpp index e1028dea4f..cb8eb889ea 100644 --- a/tolk/pipe-check-const-expr.cpp +++ b/tolk/pipe-check-const-expr.cpp @@ -82,7 +82,7 @@ void pipeline_check_constant_expressions() { } } } - + // assign `enum` members values (either auto-compute sequentially or use manual initializers) for (EnumDefPtr enum_ref : get_all_declared_enums()) { std::vector values = calculate_enum_members_with_values(enum_ref); diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 1d64a2b18c..bcff182540 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -211,7 +211,7 @@ static bool check_eq_neq_operator(TypePtr lhs_type, TypePtr rhs_type, bool& not_ if (lhs_enum && rhs_enum) { // allow `someColor == anotherColor`, don't allow `someColor == 123` return lhs_enum->enum_ref == rhs_enum->enum_ref; } - + return false; } @@ -273,7 +273,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } else { err_cannot_apply_operator(v->operator_name, lhs, rhs).fire(v->operator_range, cur_f); } - } + } if (not_integer_comparison) { // special handling at IR generation like for `address` v->mutate()->assign_fun_ref(nullptr); } @@ -628,7 +628,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } if (subject_enum) { auto l_dot = v_arm->get_pattern_expr()->try_as(); - if (!l_dot || !l_dot->is_target_enum_member()) { // match (someColor) { anotherColor => } + if (!l_dot || !l_dot->is_target_enum_member()) { // match (someColor) { anotherColor => } err("wrong pattern matching: `match` should contain members of a enum").fire(v_arm->get_pattern_expr(), cur_f); } EnumMemberPtr member_ref = std::get(l_dot->target); diff --git a/tolk/pipe-check-private-fields.cpp b/tolk/pipe-check-private-fields.cpp index e9468500bb..78ce775ac5 100644 --- a/tolk/pipe-check-private-fields.cpp +++ b/tolk/pipe-check-private-fields.cpp @@ -30,7 +30,7 @@ static bool is_private_field_usage_allowed(FunctionPtr cur_f, StructPtr struct_r if (!cur_f->is_method()) { return false; } - const TypeDataStruct* receiver_struct = cur_f->receiver_type->unwrap_alias()->try_as(); + const TypeDataStruct* receiver_struct = cur_f->receiver_type->unwrap_alias()->try_as(); if (receiver_struct && receiver_struct->struct_ref == struct_ref) { return true; } @@ -40,7 +40,7 @@ static bool is_private_field_usage_allowed(FunctionPtr cur_f, StructPtr struct_r const auto* receiver_Ts = cur_f->base_fun_ref->receiver_type->try_as(); return receiver_Ts && receiver_Ts->struct_ref == struct_ref->base_struct_ref; } - + return false; } @@ -71,7 +71,7 @@ class CheckPrivateFieldsUsageVisitor final : public ASTVisitorFunctionBody { } } } - + public: bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); diff --git a/tolk/pipe-check-serialized-fields.cpp b/tolk/pipe-check-serialized-fields.cpp index d49cc91350..92614a7394 100644 --- a/tolk/pipe-check-serialized-fields.cpp +++ b/tolk/pipe-check-serialized-fields.cpp @@ -46,10 +46,10 @@ static void check_map_TKey_TValue(SrcRange range, TypePtr TKey, TypePtr TValue) std::string because_msg; if (!check_mapKV_TKey_is_valid(TKey, because_msg)) { err("invalid `map`: type `{}` can not be used as a key\n{}", TKey, because_msg).fire(range); - } + } if (!check_mapKV_TValue_is_valid(TValue, because_msg)) { err("invalid `map`: type `{}` can not be used as a value\n{}", TValue, because_msg).fire(range); - } + } } GNU_ATTRIBUTE_NOINLINE @@ -120,7 +120,7 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody check_map_TKey_TValue(v->range, fun_ref->substitutedTs->typeT_at(0), fun_ref->substitutedTs->typeT_at(1)); return; } - + TypePtr serialized_type = nullptr; bool is_pack = false; if (f_name == "Cell.load" || f_name == "T.fromSlice" || f_name == "T.fromCell" || f_name == "T.toCell" || diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 8d1c8c11d6..f0f6ba8b3b 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -133,7 +133,7 @@ static Error err_method_or_field_not_found(TypePtr receiver_type, std::string_vi } if (!called_as_method && !is_static_dot) { return err("field `{}` doesn't exist in type `{}`", field_name, receiver_type); - } + } if (std::vector other = lookup_methods_with_name(field_name); !other.empty()) { return err("method `{}` not found for type `{}`\n(but it exists for type `{}`)", field_name, receiver_type, other.front()->receiver_type); } @@ -847,7 +847,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // for `genericF` user should provide two Ts // for `genericF` where T2 has a default — one T1 // for `Container.wrap` — one U (and one T is implicitly from receiver) - int n_provided = instantiationT_list->size() + genericTs->n_from_receiver; + int n_provided = instantiationT_list->size() + genericTs->n_from_receiver; if (n_provided < genericTs->size_no_defaults() || n_provided > genericTs->size()) { err("expected {} type arguments, got {}", genericTs->size() - genericTs->n_from_receiver, instantiationT_list->size()).fire(instantiationT_list, cur_f); } @@ -1806,11 +1806,11 @@ void pipeline_infer_types_and_calls_and_fields() { // loop over user-defined functions visit_ast_of_all_functions(); - // assign inferred_type to built-in functions like __throw() + // assign inferred_type to built-in functions like __throw() for (FunctionPtr fun_ref : get_all_builtin_functions()) { if (LaunchInferTypesAndMethodsOnce::should_visit_function(fun_ref)) { - infer_and_save_return_type_of_function(fun_ref); - } + infer_and_save_return_type_of_function(fun_ref); + } } // analyze constants that weren't referenced by any function diff --git a/tolk/pipe-lazy-load-insertions.cpp b/tolk/pipe-lazy-load-insertions.cpp index fafc060a49..b324be3ed2 100644 --- a/tolk/pipe-lazy-load-insertions.cpp +++ b/tolk/pipe-lazy-load-insertions.cpp @@ -297,7 +297,7 @@ struct ExprUsagesWhileCollecting { } void on_used_reassigned_type() { - used_reassigned_type++; + used_reassigned_type++; } bool is_self_or_field_used_for_reading() const { @@ -660,7 +660,7 @@ class CollectUsagesInStatementVisitor final : public ASTVisitorFunctionBody { lazy_expr->on_used_reassigned_type(); } // for `obj.f.method()`, mark lazy_expr=obj.f "used" anyway - if (s_expr.index_path) { + if (s_expr.index_path) { lazy_expr->on_used_rw(false); } // if we have `st.save()` / `p.getX()` / `obj.f.method()`, which will be inlined when transforming to IR, diff --git a/tolk/pipe-mini-borrow-checker.cpp b/tolk/pipe-mini-borrow-checker.cpp index 195b4bd882..e9882869a3 100644 --- a/tolk/pipe-mini-borrow-checker.cpp +++ b/tolk/pipe-mini-borrow-checker.cpp @@ -43,7 +43,7 @@ * * In order to prevent `x += ([x] = rhs).0`, we need to carefully dig into lhs of assignment. Traversing top-down, * we can't just mark "we are inside lhs of assignment" and treat all references as mutated there, because - * `getObj(x).field` is a valid lhs, where `x` is not mutated. + * `getObj(x).field` is a valid lhs, where `x` is not mutated. */ namespace tolk { @@ -88,10 +88,10 @@ class BorrowedForWriteCtx { void borrow_or_fire_if_twice(FunctionPtr cur_f, SinkExpression s_expr, AnyExprV where, FunctionPtr by_function) { for (const BorrowedVarOrField& existing : expressions) { if (existing.s_expr == s_expr) { - err("can not borrow `{}` for mutation once again, it is already being mutated by `{}`\n""hint: split a complex expression into several simple ones", s_expr.to_string(), existing.stringify_by_function()).fire(where, cur_f); + err("can not borrow `{}` for mutation once again, it is already being mutated by `{}`\n""hint: split a complex expression into several simple ones", s_expr.to_string(), existing.stringify_by_function()).fire(where, cur_f); } if (existing.s_expr.is_child_of(s_expr) || s_expr.is_child_of(existing.s_expr)) { - err("can not borrow `{}` for mutation, because `{}` is already being mutated by `{}`\n""hint: split a complex expression into several simple ones", s_expr.to_string(), existing.s_expr.to_string(), existing.stringify_by_function()).fire(where, cur_f); + err("can not borrow `{}` for mutation, because `{}` is already being mutated by `{}`\n""hint: split a complex expression into several simple ones", s_expr.to_string(), existing.s_expr.to_string(), existing.stringify_by_function()).fire(where, cur_f); } } expressions.emplace_front(BorrowedVarOrField{s_expr, by_function}); diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index ca2a93822e..c5177d5ec3 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -289,7 +289,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { } void start_visiting_enum_members(EnumDefPtr enum_ref) { - // member `Red = Another.Blue`, resolve `Another` + // member `Red = Another.Blue`, resolve `Another` for (EnumMemberPtr member_ref : enum_ref->members) { if (member_ref->has_init_value()) { parent::visit(member_ref->init_value); diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 99b55a9bd9..2e9ede3b9d 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -290,7 +290,7 @@ class TypeNodesVisitorResolver { if (const TypeDataStruct* t_struct = type_to_instantiate->try_as(); t_struct && t_struct->struct_ref->is_generic_struct()) { StructPtr struct_ref = t_struct->struct_ref; - int n_provided = static_cast(type_arguments.size()); + int n_provided = static_cast(type_arguments.size()); if (n_provided < struct_ref->genericTs->size_no_defaults() || n_provided > struct_ref->genericTs->size()) { err("struct `{}` expects {} type arguments, but {} provided", struct_ref, struct_ref->genericTs->size(), type_arguments.size()).fire(range, cur_f); } @@ -302,7 +302,7 @@ class TypeNodesVisitorResolver { } if (const TypeDataAlias* t_alias = type_to_instantiate->try_as(); t_alias && t_alias->alias_ref->is_generic_alias()) { AliasDefPtr alias_ref = t_alias->alias_ref; - int n_provided = static_cast(type_arguments.size()); + int n_provided = static_cast(type_arguments.size()); if (n_provided < alias_ref->genericTs->size_no_defaults() || n_provided > alias_ref->genericTs->size()) { err("type `{}` expects {} type arguments, but {} provided", alias_ref, alias_ref->genericTs->size(), type_arguments.size()).fire(range, cur_f); } @@ -576,7 +576,7 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { for (int i = 0; i < cur_f->get_num_params(); ++i) { LocalVarPtr param_ref = &cur_f->parameters[i]; // types for parameters in regular functions are mandatory: `fun f(a: int)`, so type_node always exists; - // but types for lambdas may be missed out; they are inferred at usage, and declared_type filled before instantiation + // but types for lambdas may be missed out; they are inferred at usage, and declared_type filled before instantiation if (param_ref->type_node) { TypePtr declared_type = finalize_type_node(param_ref->type_node); param_ref->mutate()->assign_resolved_type(declared_type); diff --git a/tolk/pretty-print.cpp b/tolk/pretty-print.cpp index 47eb04211c..c5b386d3fc 100644 --- a/tolk/pretty-print.cpp +++ b/tolk/pretty-print.cpp @@ -28,7 +28,7 @@ using namespace tolk; * Every function should be named `debug_print` and accept a single const-ref argument. * The rest will work automatically. * No changes for "prettified" classes are required: these functions are fully standalone. - * + * * NOTE! When adding a new function, its argument type should be listed in lldb_addons.py. * * See .lldbinit and lldb_addons.py. @@ -250,7 +250,7 @@ std::string debug_print(const AsmOpList* op_list) { std::string debug_print(const Stack* stack) { std::ostringstream os; - + if (stack->s.empty()) { os << "(empty) "; } else { diff --git a/tolk/send-message-api.cpp b/tolk/send-message-api.cpp index be98399e72..20901d625a 100644 --- a/tolk/send-message-api.cpp +++ b/tolk/send-message-api.cpp @@ -114,7 +114,7 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co // since TVM 12, field `bounce` is a union: `bounce: BounceMode | bool` std::vector ir_bounce_is_bool = pre_compile_is_type(code, t_bounce, TypeDataBool::create(), ir_bounce, origin, "(bounce-is-bool)"); - + // field `dest` is `dest: address | AutoDeployAddress | (int8, uint256) | builder`; // struct AutoDeployAddress { workchain: int8; stateInit: ContractState | cell; toShard: AddressShardingOptions?; } // struct ContractState { code: cell; data: cell; } @@ -177,7 +177,7 @@ std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& co code.emplace_back(origin, Op::_Call, ir_not_NoBounce, std::vector{ir_bounce[0], ir_zero}, lookup_function("_!=_")); ctx.storeBool(ir_not_NoBounce[0]); code.close_pop_cur(origin); - } + } // fill `bounced:Bool` + `src:MsgAddress` 00 ctx.storeUint(ir_zero, 1 + 2); diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index 34b577c457..56aebb18ce 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -120,8 +120,8 @@ bool SinkExpression::is_child_of(SinkExpression rhs) const { while (rhs_path != 0) { mask = (mask << 8) + 0xFF; rhs_path >>= 8; - } - return var_ref == rhs.var_ref && index_path != rhs.index_path && (index_path & mask) == rhs.index_path; + } + return var_ref == rhs.var_ref && index_path != rhs.index_path && (index_path & mask) == rhs.index_path; } SinkExpression SinkExpression::get_child_s_expr(int field_idx) const { diff --git a/tolk/src-file.cpp b/tolk/src-file.cpp index 5ebb8d5994..dd2eb4ada8 100644 --- a/tolk/src-file.cpp +++ b/tolk/src-file.cpp @@ -163,7 +163,7 @@ static void output_line_to_fif(std::ostream& os, int indent, std::string_view li os << ' '; } os << "// " << (dots ? "..." : "") << line_no << ": " << s.substr(b, e - b + 1) << std::endl; - } + } } // when generating Fift output, every block of asm instructions originated from the same Tolk line, diff --git a/tolk/src-file.h b/tolk/src-file.h index e119b21130..1f0ef7430a 100644 --- a/tolk/src-file.h +++ b/tolk/src-file.h @@ -95,11 +95,11 @@ class SrcRange { static SrcRange overlap(SrcRange start, SrcRange end) { return SrcRange(start.file_id, start.start_offset, end.end_offset); } - + static SrcRange empty_at_start(SrcRange range) { return SrcRange(range.file_id, range.start_offset, range.start_offset); } - + static SrcRange empty_at_end(SrcRange range) { return SrcRange(range.file_id, range.end_offset, range.end_offset); } diff --git a/tolk/stack-transform.cpp b/tolk/stack-transform.cpp index d86afaec68..f855405915 100644 --- a/tolk/stack-transform.cpp +++ b/tolk/stack-transform.cpp @@ -19,9 +19,9 @@ namespace tolk { /* - * + * * GENERIC STACK TRANSFORMATIONS - * + * */ StackTransform::StackTransform(std::initializer_list list) { diff --git a/tolk/tolk.h b/tolk/tolk.h index 5925fdbb13..d7979f37d4 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -33,9 +33,9 @@ namespace tolk { /* - * + * * ABSTRACT CODE - * + * */ typedef int const_idx_t; @@ -390,9 +390,9 @@ struct FunctionBodyCode { }; /* - * + * * GENERATE CODE - * + * */ typedef std::vector StackLayout; @@ -629,9 +629,9 @@ int is_pos_pow2(td::RefInt256 x); int is_neg_pow2(td::RefInt256 x); /* - * + * * STACK TRANSFORMS - * + * */ /* @@ -790,9 +790,9 @@ inline std::ostream& operator<<(std::ostream& os, const StackTransform& trans) { bool apply_op(StackTransform& trans, const AsmOp& op); /* - * + * * STACK OPERATION OPTIMIZER - * + * */ struct Optimizer { @@ -1023,12 +1023,12 @@ struct Stack { * * SPECIFIC SYMBOL VALUES, * BUILT-IN FUNCTIONS AND OPERATIONS - * + * */ struct FunctionBodyBuiltinAsmOp { using CompileToAsmOpImpl = AsmOp(std::vector&, std::vector&, AnyV origin); - + std::function simple_compile; explicit FunctionBodyBuiltinAsmOp(std::function compile) @@ -1041,7 +1041,7 @@ struct FunctionBodyBuiltinGenerateOps { using GenerateOpsImpl = std::vector(FunctionPtr, CodeBlob&, AnyV origin, const std::vector>&); std::function generate_ops; - + explicit FunctionBodyBuiltinGenerateOps(std::function generate_ops) : generate_ops(std::move(generate_ops)) {} }; diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index c3d24878ba..c11bf89df3 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -578,7 +578,7 @@ TypePtr TypeDataMapKV::replace_children_custom(const ReplacerCallbackT& callback bool TypeDataAlias::can_rhs_be_assigned(TypePtr rhs) const { if (const TypeDataAlias* rhs_alias = rhs->try_as()) { // having `type UserId = int` and `type OwnerId = int`, make them NOT assignable without `as` - // (although they both have the same type_id) + // (although they both have the same type_id) if (underlying_type->equal_to(rhs_alias->underlying_type)) { return !are_two_equal_type_aliases_different(this, rhs_alias); } diff --git a/tolk/type-system.h b/tolk/type-system.h index ea8be25e81..2dd5a27c36 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -65,7 +65,7 @@ class TypeData { type_id_int128 = 50, type_id_int256 = 52, }; - + enum flag_mask { flag_contains_genericT_inside = 1 << 2, flag_contains_type_alias_inside = 1 << 3, @@ -650,7 +650,7 @@ class TypeDataMapKV final : public TypeData { public: TypePtr TKey; TypePtr TValue; - + static TypePtr create(TypePtr TKey, TypePtr TValue); int get_type_id() const override; diff --git a/tonlib/test/online.cpp b/tonlib/test/online.cpp index 96a0e0c021..1ecbaa1a20 100644 --- a/tonlib/test/online.cpp +++ b/tonlib/test/online.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/utils/pack-viewer.cpp b/utils/pack-viewer.cpp index 1280c4303b..320d6528ce 100644 --- a/utils/pack-viewer.cpp +++ b/utils/pack-viewer.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 59b4efbd8f..ab6a2aed55 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 13aeba5d2a..a389ed2986 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/validator-engine-console/validator-engine-console.h b/validator-engine-console/validator-engine-console.h index 2dcca796c7..28dfbbbd3d 100644 --- a/validator-engine-console/validator-engine-console.h +++ b/validator-engine-console/validator-engine-console.h @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index e16e6f315b..58c3ed525a 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -14,13 +14,13 @@ You should have received a copy of the GNU General Public License along with TON Blockchain. If not, see . - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. Copyright 2017-2020 Telegram Systems LLP diff --git a/validator-session/CMakeLists.txt b/validator-session/CMakeLists.txt index c57fad54ea..f800e3ad06 100644 --- a/validator-session/CMakeLists.txt +++ b/validator-session/CMakeLists.txt @@ -17,9 +17,9 @@ set(VALIDATOR_SESSION_SOURCE add_library(validatorsession STATIC ${VALIDATOR_SESSION_SOURCE}) -target_include_directories(validatorsession PUBLIC - $ - $/.. +target_include_directories(validatorsession PUBLIC + $ + $/.. ${OPENSSL_INCLUDE_DIR} ) target_link_libraries(validatorsession PRIVATE tdutils tdactor adnl rldp2 tl_api dht tdfec overlay catchain) diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index ea6d1032f7..13030f47e4 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -52,7 +52,7 @@ set(VALIDATOR_HEADERS interfaces/validator-manager.h interfaces/validator-set.h invariants.hpp - + import-db-slice.hpp import-db-slice-local.hpp queue-size-counter.hpp @@ -189,31 +189,31 @@ add_library(validator-disk STATIC ${DISK_VALIDATOR_SOURCE}) add_library(validator-hardfork STATIC ${HARDFORK_VALIDATOR_SOURCE}) add_library(full-node STATIC ${FULL_NODE_SOURCE}) -target_include_directories(validator PUBLIC - $ - $/.. - $/../crypto +target_include_directories(validator PUBLIC + $ + $/.. + $/../crypto ${OPENSSL_INCLUDE_DIR} ) -target_include_directories(validator-disk PUBLIC - $ - $/.. - $/../crypto +target_include_directories(validator-disk PUBLIC + $ + $/.. + $/../crypto ${OPENSSL_INCLUDE_DIR} ) -target_include_directories(validator-hardfork PUBLIC - $ - $/.. - $/../crypto +target_include_directories(validator-hardfork PUBLIC + $ + $/.. + $/../crypto ${OPENSSL_INCLUDE_DIR} ) -target_include_directories(full-node PUBLIC - $ - $/.. - $/../crypto +target_include_directories(full-node PUBLIC + $ + $/.. + $/../crypto ${OPENSSL_INCLUDE_DIR} ) diff --git a/validator/impl/CMakeLists.txt b/validator/impl/CMakeLists.txt index dad23baa75..9f61e207d0 100644 --- a/validator/impl/CMakeLists.txt +++ b/validator/impl/CMakeLists.txt @@ -40,10 +40,10 @@ set(TON_VALIDATOR_SOURCE add_library(ton_validator STATIC ${TON_VALIDATOR_SOURCE}) -target_include_directories(ton_validator PUBLIC - $ - $/.. - $/../crypto +target_include_directories(ton_validator PUBLIC + $ + $/.. + $/../crypto ${OPENSSL_INCLUDE_DIR} ) From 971069d3a8b72fa21a8d6f73eda0de915ff37b38 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Tue, 6 Jan 2026 09:38:32 +0300 Subject: [PATCH 521/524] Twostep broadcast (#1990) * Rename broadcast files * Refactor broadcast-simple * Refactor broadcast-fec * Add broadcast-twostep * Configure mtu for peers in rldp2 * Improve twostep broadcasts 1) Fix use after move in process_broadcast 2) Sign src adnl id to verify original sender 3) Don't send to self and src in rebroadcast() 4) deliver_broadcast on send * Enable twostep broadcast in overlay options * Pass rldp to CatChainReceiverImpl * Use new broadcasts in custom overlays * Init peer mtu in custom overlays * Register broadcast in BroadcastsTwostep::send * Run broadcasts_twostep_.gc * Enable processing twostep broadcasts in catchain overlays * Improve setting rldp2 peer mtu * Reformat code --------- Co-authored-by: Evgeny Kapun Co-authored-by: SpyCheese --- catchain/catchain-receiver-interface.h | 7 +- catchain/catchain-receiver.cpp | 17 +- catchain/catchain-receiver.hpp | 2 + catchain/catchain.cpp | 15 +- catchain/catchain.h | 1 + catchain/catchain.hpp | 4 + lite-client/query-utils.cpp | 1 - overlay/CMakeLists.txt | 12 +- overlay/broadcast-fec.cpp | 563 ++++++++++++++++++++++++ overlay/broadcast-fec.hpp | 64 +++ overlay/broadcast-simple.cpp | 241 ++++++++++ overlay/broadcast-simple.hpp | 65 +++ overlay/broadcast-twostep.cpp | 321 ++++++++++++++ overlay/broadcast-twostep.hpp | 74 ++++ overlay/overlay-broadcast.cpp | 191 -------- overlay/overlay-broadcast.hpp | 121 ----- overlay/overlay-fec-broadcast.cpp | 365 --------------- overlay/overlay-fec-broadcast.hpp | 345 --------------- overlay/overlay-fec.cpp | 77 ---- overlay/overlay-fec.hpp | 60 --- overlay/overlay-manager.cpp | 4 +- overlay/overlay.cpp | 221 ++++------ overlay/overlay.hpp | 57 +-- overlay/overlays.h | 5 +- rldp2/CMakeLists.txt | 1 + rldp2/RldpConnection.h | 4 +- rldp2/rldp-in.hpp | 4 + rldp2/rldp-utils.h | 73 +++ rldp2/rldp.cpp | 41 +- rldp2/rldp.h | 2 + test/test-catchain.cpp | 3 +- test/test-overlay.cpp | 1 + tl/generate/scheme/ton_api.tl | 6 + validator-session/validator-session.cpp | 4 +- validator/full-node-custom-overlays.cpp | 8 +- validator/full-node-custom-overlays.hpp | 3 + validator/validator-group.cpp | 6 + validator/validator-group.hpp | 2 + 38 files changed, 1625 insertions(+), 1366 deletions(-) create mode 100644 overlay/broadcast-fec.cpp create mode 100644 overlay/broadcast-fec.hpp create mode 100644 overlay/broadcast-simple.cpp create mode 100644 overlay/broadcast-simple.hpp create mode 100644 overlay/broadcast-twostep.cpp create mode 100644 overlay/broadcast-twostep.hpp delete mode 100644 overlay/overlay-broadcast.cpp delete mode 100644 overlay/overlay-broadcast.hpp delete mode 100644 overlay/overlay-fec-broadcast.cpp delete mode 100644 overlay/overlay-fec-broadcast.hpp delete mode 100644 overlay/overlay-fec.cpp delete mode 100644 overlay/overlay-fec.hpp create mode 100644 rldp2/rldp-utils.h diff --git a/catchain/catchain-receiver-interface.h b/catchain/catchain-receiver-interface.h index c05fb77da4..2a70294bc3 100644 --- a/catchain/catchain-receiver-interface.h +++ b/catchain/catchain-receiver-interface.h @@ -59,9 +59,10 @@ class CatChainReceiverInterface : public td::actor::Actor { static td::actor::ActorOwn create( std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId overlay_manager, - const std::vector &ids, const PublicKeyHash &local_id, const CatChainSessionId &unique_hash, - std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync); + td::actor::ActorId adnl, td::actor::ActorId adnl_sender, + td::actor::ActorId overlay_manager, const std::vector &ids, + const PublicKeyHash &local_id, const CatChainSessionId &unique_hash, std::string db_root, std::string db_suffix, + bool allow_unsafe_self_blocks_resync); ~CatChainReceiverInterface() override = default; }; diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp index 29111688ee..1df9c01ffd 100644 --- a/catchain/catchain-receiver.cpp +++ b/catchain/catchain-receiver.cpp @@ -462,6 +462,7 @@ void CatChainReceiverImpl::debug_add_fork(td::BufferSlice payload, CatChainBlock CatChainReceiverImpl::CatChainReceiverImpl(std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId adnl_sender, td::actor::ActorId overlay_manager, const std::vector &ids, const PublicKeyHash &local_id, const CatChainSessionId &unique_hash, std::string db_root, @@ -470,6 +471,7 @@ CatChainReceiverImpl::CatChainReceiverImpl(std::unique_ptr callback, c , opts_(opts) , keyring_(std::move(keyring)) , adnl_(std::move(adnl)) + , adnl_sender_(std::move(adnl_sender)) , overlay_manager_(std::move(overlay_manager)) , local_id_(local_id) , db_root_(std::move(db_root)) @@ -523,6 +525,9 @@ void CatChainReceiverImpl::start_up() { overlay::OverlayOptions overlay_options; overlay_options.broadcast_speed_multiplier_ = opts_.broadcast_speed_multiplier; overlay_options.private_ping_peers_ = true; + overlay_options.twostep_broadcast_sender_ = adnl_sender_; + // Uncomment this to enable sending twostep broadcasts in catchain overlays: + // overlay_options.send_twostep_broadcast_ = true; td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay_ex, get_source(local_idx_)->get_adnl_id(), overlay_full_id_.clone(), std::move(ids), make_callback(), overlay::OverlayPrivacyRules{0, 0, std::move(root_keys)}, @@ -653,12 +658,14 @@ void CatChainReceiverImpl::read_db() { td::actor::ActorOwn CatChainReceiverInterface::create( std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId overlay_manager, - const std::vector &ids, const PublicKeyHash &local_id, const CatChainSessionId &unique_hash, - std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync) { + td::actor::ActorId adnl, td::actor::ActorId adnl_sender, + td::actor::ActorId overlay_manager, const std::vector &ids, + const PublicKeyHash &local_id, const CatChainSessionId &unique_hash, std::string db_root, std::string db_suffix, + bool allow_unsafe_self_blocks_resync) { auto A = td::actor::create_actor( - "catchainreceiver", std::move(callback), opts, std::move(keyring), std::move(adnl), std::move(overlay_manager), - ids, local_id, unique_hash, std::move(db_root), std::move(db_suffix), allow_unsafe_self_blocks_resync); + "catchainreceiver", std::move(callback), opts, std::move(keyring), std::move(adnl), std::move(adnl_sender), + std::move(overlay_manager), ids, local_id, unique_hash, std::move(db_root), std::move(db_suffix), + allow_unsafe_self_blocks_resync); return std::move(A); } diff --git a/catchain/catchain-receiver.hpp b/catchain/catchain-receiver.hpp index e25329413d..1d05f2d4f1 100644 --- a/catchain/catchain-receiver.hpp +++ b/catchain/catchain-receiver.hpp @@ -134,6 +134,7 @@ class CatChainReceiverImpl final : public CatChainReceiver { CatChainReceiverImpl(std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId adnl_sender, td::actor::ActorId overlays, const std::vector &ids, const PublicKeyHash &local_id, const CatChainBlockHash &unique_hash, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync); @@ -199,6 +200,7 @@ class CatChainReceiverImpl final : public CatChainReceiver { td::actor::ActorId keyring_; td::actor::ActorId adnl_; + td::actor::ActorId adnl_sender_; td::actor::ActorId overlay_manager_; overlay::OverlayIdShort overlay_id_; overlay::OverlayIdFull overlay_full_id_; diff --git a/catchain/catchain.cpp b/catchain/catchain.cpp index 6b205d98c4..b6cc7dba30 100644 --- a/catchain/catchain.cpp +++ b/catchain/catchain.cpp @@ -229,6 +229,7 @@ void CatChainImpl::on_receiver_started() { CatChainImpl::CatChainImpl(std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId adnl_sender, td::actor::ActorId overlay_manager, std::vector ids, const PublicKeyHash &local_id, const CatChainSessionId &unique_hash, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync) @@ -248,8 +249,8 @@ CatChainImpl::CatChainImpl(std::unique_ptr callback, const CatChainOpt blamed_sources_.resize(ids.size(), false); top_source_blocks_.resize(ids.size(), nullptr); - args_ = std::make_unique(std::move(keyring), std::move(adnl), std::move(overlay_manager), std::move(ids), - local_id, unique_hash); + args_ = std::make_unique(std::move(keyring), std::move(adnl), std::move(adnl_sender), + std::move(overlay_manager), std::move(ids), local_id, unique_hash); } void CatChainImpl::alarm() { @@ -291,7 +292,7 @@ void CatChainImpl::start_up() { auto cb = std::make_unique(actor_id(this)); - receiver_ = CatChainReceiverInterface::create(std::move(cb), opts_, args_->keyring, args_->adnl, + receiver_ = CatChainReceiverInterface::create(std::move(cb), opts_, args_->keyring, args_->adnl, args_->adnl_sender, args_->overlay_manager, args_->ids, args_->local_id, args_->unique_hash, db_root_, db_suffix_, allow_unsafe_self_blocks_resync_); args_ = nullptr; @@ -301,13 +302,15 @@ void CatChainImpl::start_up() { td::actor::ActorOwn CatChain::create(std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId adnl_sender, td::actor::ActorId overlay_manager, std::vector ids, const PublicKeyHash &local_id, const CatChainSessionId &unique_hash, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync) { - return td::actor::create_actor( - "catchain", std::move(callback), opts, std::move(keyring), std::move(adnl), std::move(overlay_manager), - std::move(ids), local_id, unique_hash, std::move(db_root), std::move(db_suffix), allow_unsafe_self_blocks_resync); + return td::actor::create_actor("catchain", std::move(callback), opts, std::move(keyring), + std::move(adnl), std::move(adnl_sender), std::move(overlay_manager), + std::move(ids), local_id, unique_hash, std::move(db_root), + std::move(db_suffix), allow_unsafe_self_blocks_resync); } CatChainBlock *CatChainImpl::get_block(CatChainBlockHash hash) const { diff --git a/catchain/catchain.h b/catchain/catchain.h index 110939b3f0..bee20cad7e 100644 --- a/catchain/catchain.h +++ b/catchain/catchain.h @@ -103,6 +103,7 @@ class CatChain : public td::actor::Actor { static td::actor::ActorOwn create(std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId adnl_sender, td::actor::ActorId overlay_manager, std::vector ids, const PublicKeyHash &local_id, const CatChainSessionId &unique_hash, std::string db_root, diff --git a/catchain/catchain.hpp b/catchain/catchain.hpp index 73f4f29ef0..416fc1634f 100644 --- a/catchain/catchain.hpp +++ b/catchain/catchain.hpp @@ -62,16 +62,19 @@ class CatChainImpl : public CatChain { struct Args { td::actor::ActorId keyring; td::actor::ActorId adnl; + td::actor::ActorId adnl_sender; td::actor::ActorId overlay_manager; std::vector ids; PublicKeyHash local_id; CatChainSessionId unique_hash; Args(td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId adnl_sender, td::actor::ActorId overlay_manager, std::vector ids, const PublicKeyHash &local_id, const CatChainSessionId &unique_hash) : keyring(std::move(keyring)) , adnl(std::move(adnl)) + , adnl_sender(std::move(adnl_sender)) , overlay_manager(std::move(overlay_manager)) , ids(std::move(ids)) , local_id(local_id) @@ -128,6 +131,7 @@ class CatChainImpl : public CatChain { void destroy() override; CatChainImpl(std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId adnl_sender, td::actor::ActorId overlay_manager, std::vector ids, const PublicKeyHash &local_id, const CatChainSessionId &unique_hash, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync); diff --git a/lite-client/query-utils.cpp b/lite-client/query-utils.cpp index 5b53a8166c..cc8db76f05 100644 --- a/lite-client/query-utils.cpp +++ b/lite-client/query-utils.cpp @@ -18,7 +18,6 @@ #include "auto/tl/lite_api.hpp" #include "block/block-auto.h" -#include "overlay/overlay-broadcast.hpp" #include "td/utils/overloaded.h" #include "tl-utils/common-utils.hpp" #include "tl-utils/lite-utils.hpp" diff --git a/overlay/CMakeLists.txt b/overlay/CMakeLists.txt index 11ba257b75..58dab3c1a0 100644 --- a/overlay/CMakeLists.txt +++ b/overlay/CMakeLists.txt @@ -1,14 +1,14 @@ set(OVERLAY_SOURCE + broadcast-fec.cpp + broadcast-simple.cpp + broadcast-twostep.cpp overlay-manager.cpp overlay.cpp - overlay-fec.cpp - overlay-fec-broadcast.cpp - overlay-broadcast.cpp overlay-peers.cpp - overlay-fec.hpp - overlay-broadcast.hpp - overlay-fec-broadcast.hpp + broadcast-fec.hpp + broadcast-simple.hpp + broadcast-twostep.hpp overlay-manager.h overlay.h overlay.hpp diff --git a/overlay/broadcast-fec.cpp b/overlay/broadcast-fec.cpp new file mode 100644 index 0000000000..23d2fd161c --- /dev/null +++ b/overlay/broadcast-fec.cpp @@ -0,0 +1,563 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ + +#include "keys/encryptor.h" + +#include "broadcast-fec.hpp" +#include "overlay.hpp" + +namespace ton { + +namespace overlay { + +static Overlay::BroadcastHash compute_broadcast_id(PublicKeyHash source, const fec::FecType &fec_type, + Overlay::BroadcastDataHash data_hash, td::uint32 size, + td::uint32 flags) { + return get_tl_object_sha_bits256(create_tl_object( + (flags & Overlays::BroadcastFlagAnySender()) ? PublicKeyHash::zero().tl() : source.tl(), + get_tl_object_sha_bits256(fec_type.tl()), data_hash, size, flags)); +} + +static Overlay::BroadcastPartHash compute_broadcast_part_id(Overlay::BroadcastHash broadcast_hash, + Overlay::BroadcastDataHash data_hash, td::uint32 seqno) { + return get_tl_object_sha_bits256( + create_tl_object(broadcast_hash, data_hash, seqno)); +} + +class BroadcastFec : public td::ListNode { + friend class BroadcastFecPart; + friend class BroadcastsFec; + + public: + BroadcastFec(Overlay::BroadcastHash hash, Overlay::BroadcastDataHash data_hash, td::uint32 flags, td::uint32 date, + PublicKey src, fec::FecType fec_type) + : hash_(hash) + , data_hash_(data_hash) + , flags_(flags) + , date_(date) + , src_(std::move(src)) + , fec_type_(std::move(fec_type)) { + } + + td::Status is_eligible_sender(PublicKey src) { + if (flags_ & Overlays::BroadcastFlagAnySender()) { + return td::Status::OK(); + } else { + if (src == src_) { + return td::Status::OK(); + } else { + return td::Status::Error(ErrorCode::protoviolation, "bad source"); + } + } + } + + td::Status add_part(td::uint32 seqno, td::BufferSlice data, td::BufferSlice serialized_fec_part_short, + td::BufferSlice serialized_fec_part) { + if (decoder_) { + td::fec::Symbol s; + s.id = seqno; + s.data = std::move(data); + + decoder_->add_symbol(std::move(s)); + } + parts_[seqno] = std::pair(std::move(serialized_fec_part_short), + std::move(serialized_fec_part)); + + return td::Status::OK(); + } + + td::Result finish() { + CHECK(decoder_); + if (!decoder_->may_try_decode()) { + return td::Status::Error(ErrorCode::notready, "need more parts"); + } + TRY_RESULT(D, decoder_->try_decode(true)); + if (sha256_bits256(D.data.as_slice()) != data_hash_) { + return td::Status::Error(ErrorCode::protoviolation, "bad hash"); + } + encoder_ = std::move(D.encoder); + CHECK(encoder_ != nullptr); + ready_ = true; + decoder_ = nullptr; + data_ = D.data.clone(); + return std::move(D.data); + } + + td::BufferSlice get_part(td::uint32 seqno) { + CHECK(ready_); + CHECK(encoder_ != nullptr); + auto R = encoder_->gen_symbol(seqno); + CHECK(R.id == seqno); + return std::move(R.data); + } + + td::Status init_fec_type() { + TRY_RESULT(D, fec_type_.create_decoder()); + decoder_ = std::move(D); + return td::Status::OK(); + } + + td::Status run_checks(); + + bool neighbour_received(adnl::AdnlNodeIdShort id) const { + return received_neighbours_.find(id) != received_neighbours_.end(); + } + + void add_received(adnl::AdnlNodeIdShort id) { + received_neighbours_.insert(id); + } + + bool neighbour_completed(adnl::AdnlNodeIdShort id) const { + return completed_neighbours_.find(id) != completed_neighbours_.end(); + } + + void add_completed(adnl::AdnlNodeIdShort id) { + completed_neighbours_.insert(id); + } + + static BroadcastFec *from_list_node(ListNode *node) { + return static_cast(node); + } + + bool received_part(td::uint32 seqno) const { + if (seqno + 64 < next_seqno_) { + return true; + } + if (seqno >= next_seqno_) { + return false; + } + return received_parts_ & (1ull << (next_seqno_ - seqno - 1)); + } + + void add_received_part(td::uint32 seqno) { + CHECK(!received_part(seqno)); + if (seqno < next_seqno_) { + received_parts_ |= (1ull << (next_seqno_ - seqno - 1)); + } else { + auto old = next_seqno_; + next_seqno_ = seqno + 1; + if (next_seqno_ - old >= 64) { + received_parts_ = 1; + } else { + received_parts_ = received_parts_ << (next_seqno_ - old); + received_parts_ |= 1; + } + } + } + + void broadcast_checked(OverlayImpl *overlay, td::Result R); + + void set_src_peer_id(adnl::AdnlNodeIdShort src_peer_id) { + src_peer_id_ = src_peer_id; + } + + td::Status distribute_part(OverlayImpl *overlay, td::uint32 seqno); + + private: + Overlay::BroadcastHash hash_; + Overlay::BroadcastDataHash data_hash_; + td::uint32 flags_; + td::uint32 date_; + PublicKey src_; + fec::FecType fec_type_; + + bool ready_ = false; + bool is_checked_ = false; + std::unique_ptr decoder_; + std::unique_ptr encoder_; + std::set received_neighbours_; + std::set completed_neighbours_; + td::uint32 next_seqno_ = 0; + td::uint64 received_parts_ = 0; + std::map> parts_; + adnl::AdnlNodeIdShort src_peer_id_ = adnl::AdnlNodeIdShort::zero(); + td::BufferSlice data_; +}; + +td::Status BroadcastFec::run_checks() { + if (fec_type_.size() > Overlays::max_fec_broadcast_size()) { + return td::Status::Error(ErrorCode::protoviolation, "too big fec broadcast"); + } + return td::Status::OK(); +} + +void BroadcastFec::broadcast_checked(OverlayImpl *overlay, td::Result R) { + if (R.is_error()) { + td::actor::send_closure(actor_id(overlay), &OverlayImpl::update_peer_err_ctr, src_peer_id_, true); + return; + } + overlay->deliver_broadcast(src_.compute_short_id(), data_.clone()); + while (!parts_.empty()) { + distribute_part(overlay, parts_.begin()->first); + } + + is_checked_ = true; +} + +// Do we need status here?? +td::Status BroadcastFec::distribute_part(OverlayImpl *overlay, td::uint32 seqno) { + auto i = parts_.find(seqno); + if (i == parts_.end()) { + VLOG(OVERLAY_WARNING) << "not distibuting empty part " << seqno; + // should not get here + return td::Status::OK(); + } + auto tls = std::move(i->second); + parts_.erase(i); + td::BufferSlice data_short = std::move(tls.first); + td::BufferSlice data = std::move(tls.second); + + auto nodes = overlay->get_neighbours(overlay->propagate_broadcast_to()); + auto manager = overlay->overlay_manager(); + + for (auto &n : nodes) { + if (neighbour_completed(n)) { + continue; + } + if (neighbour_received(n)) { + td::actor::send_closure(manager, &OverlayManager::send_message, n, overlay->local_id(), overlay->overlay_id(), + data_short.clone()); + } else { + if (hash_.count_leading_zeroes() >= 12) { + VLOG(OVERLAY_INFO) << "broadcast " << hash_ << ": sending part " << seqno << " to " << n; + } + td::actor::send_closure(manager, &OverlayManager::send_message, n, overlay->local_id(), overlay->overlay_id(), + data.clone()); + } + } + return td::Status::OK(); +} + +class BroadcastFecPart { + friend class BroadcastsFec; + + public: + BroadcastFecPart(Overlay::BroadcastHash broadcast_hash, Overlay::BroadcastPartHash part_hash, PublicKey source, + std::shared_ptr cert, Overlay::BroadcastDataHash data_hash, td::uint32 data_size, + td::uint32 flags, Overlay::BroadcastDataHash part_data_hash, td::BufferSlice data, td::uint32 seqno, + fec::FecType fec_type, td::uint32 date, td::BufferSlice signature, bool is_short, + adnl::AdnlNodeIdShort src_peer_id) + : broadcast_hash_(broadcast_hash) + , part_hash_(part_hash) + , source_(std::move(source)) + , cert_(std::move(cert)) + , broadcast_data_hash_(data_hash) + , broadcast_size_(data_size) + , flags_(flags) + , part_data_hash_(part_data_hash) + , data_(std::move(data)) + , seqno_(seqno) + , fec_type_(std::move(fec_type)) + , date_(date) + , signature_(std::move(signature)) + , is_short_(is_short) + , src_peer_id_(src_peer_id) { + } + + td::BufferSlice to_sign(); + + td::Status run_checks(OverlayImpl *overlay, BroadcastFec *bcast); + td::Status run(OverlayImpl *overlay, BroadcastFec &bcast); + + private: + Overlay::BroadcastHash broadcast_hash_; + Overlay::BroadcastPartHash part_hash_; + + PublicKey source_; + std::shared_ptr cert_; + Overlay::BroadcastDataHash broadcast_data_hash_; + td::uint32 broadcast_size_; + td::uint32 flags_; + Overlay::BroadcastDataHash part_data_hash_; + td::BufferSlice data_; + td::uint32 seqno_; + fec::FecType fec_type_; + td::uint32 date_; + td::BufferSlice signature_; + + bool is_short_; + bool untrusted_{false}; + + adnl::AdnlNodeIdShort src_peer_id_ = adnl::AdnlNodeIdShort::zero(); +}; + +td::Status BroadcastFecPart::run_checks(OverlayImpl *overlay, BroadcastFec *bcast) { + if (bcast && bcast->received_part(seqno_)) { + return td::Status::Error(ErrorCode::notready, "duplicate part"); + } + auto r = overlay->check_source_eligible(source_, cert_.get(), broadcast_size_, true); + if (r == BroadcastCheckResult::Forbidden) { + return td::Status::Error(ErrorCode::error, "broadcast is forbidden"); + } + + if (r == BroadcastCheckResult::NeedCheck) { + untrusted_ = true; + } else if (bcast) { + TRY_STATUS(bcast->is_eligible_sender(source_)); + } + TRY_RESULT(encryptor, overlay->get_encryptor(source_)); + TRY_STATUS(encryptor->check_signature(to_sign().as_slice(), signature_.as_slice())); + return td::Status::OK(); +} + +td::Status BroadcastFecPart::run(OverlayImpl *overlay, BroadcastFec &bcast) { + if (bcast.received_part(seqno_)) { + return td::Status::Error(ErrorCode::notready, "duplicate part"); + } + bcast.add_received_part(seqno_); + bcast.set_src_peer_id(src_peer_id_); + + TRY_STATUS(bcast.add_part( + seqno_, data_.clone(), + create_serialize_tl_object( + source_.tl(), cert_ ? cert_->tl() : Certificate::empty_tl(), broadcast_hash_, part_data_hash_, seqno_, + signature_.clone()), + create_serialize_tl_object( + source_.tl(), cert_ ? cert_->tl() : Certificate::empty_tl(), bcast.data_hash_, bcast.fec_type_.size(), + bcast.flags_, data_.clone(), seqno_, bcast.fec_type_.tl(), bcast.date_, signature_.clone()))); + if (!bcast.ready_) { + auto R = bcast.finish(); + if (R.is_error()) { + auto S = R.move_as_error(); + if (S.code() != ErrorCode::notready) { + return S; + } + } else { + if (untrusted_) { + auto P = td::PromiseCreator::lambda( + [overlay_id = actor_id(overlay), broadcast_hash = broadcast_hash_](td::Result R) mutable { + td::actor::send_closure(overlay_id, &OverlayImpl::broadcast_fec_checked, std::move(broadcast_hash), + std::move(R)); + }); + overlay->check_broadcast(bcast.src_.compute_short_id(), R.move_as_ok(), std::move(P)); + } else { + overlay->deliver_broadcast(bcast.src_.compute_short_id(), R.move_as_ok()); + } + } + } + if (!untrusted_ || bcast.is_checked_) { + TRY_STATUS(bcast.distribute_part(overlay, seqno_)); + } + return td::Status::OK(); +} + +td::BufferSlice BroadcastFecPart::to_sign() { + auto obj = create_tl_object(part_hash_, date_); + return serialize_tl_object(obj, true); +} + +class BroadcastFecActor : public td::actor::Actor { + public: + BroadcastFecActor(td::BufferSlice data, td::uint32 flags, td::actor::ActorId overlay, + PublicKeyHash local_id, double speed_multiplier = 1.0); + + void start_up() override; + void alarm() override; + + private: + const td::uint32 symbol_size_ = 768; + td::uint32 to_send_; + td::uint32 seqno_ = 0; + PublicKeyHash local_id_; + Overlay::BroadcastDataHash data_hash_; + td::uint32 flags_ = 0; + double delay_ = 0.010; + td::int32 date_; + std::unique_ptr encoder_; + td::actor::ActorId overlay_; + fec::FecType fec_type_; +}; + +BroadcastFecActor::BroadcastFecActor(td::BufferSlice data, td::uint32 flags, td::actor::ActorId overlay, + PublicKeyHash local_id, double speed_multiplier) + : flags_(flags) { + delay_ /= speed_multiplier; + CHECK(data.size() <= (1 << 27)); + local_id_ = local_id; + overlay_ = std::move(overlay); + date_ = static_cast(td::Clocks::system()); + to_send_ = (static_cast(data.size()) / symbol_size_ + 1) * 2; + + data_hash_ = td::sha256_bits256(data); + + fec_type_ = td::fec::RaptorQEncoder::Parameters{data.size(), symbol_size_, 0}; + auto E = fec_type_.create_encoder(std::move(data)); + E.ensure(); + encoder_ = E.move_as_ok(); +} + +void BroadcastFecActor::start_up() { + encoder_->prepare_more_symbols(); + alarm(); +} + +void BroadcastFecActor::alarm() { + for (td::uint32 i = 0; i < 4; i++) { + auto X = encoder_->gen_symbol(seqno_++); + CHECK(X.data.size() <= 1000); + td::actor::send_closure(overlay_, &OverlayImpl::send_new_fec_broadcast_part, local_id_, data_hash_, + fec_type_.size(), flags_, std::move(X.data), X.id, fec_type_, date_); + } + + alarm_timestamp() = td::Timestamp::in(delay_); + + if (seqno_ >= to_send_) { + stop(); + } +} + +BroadcastsFec::BroadcastsFec() = default; + +BroadcastsFec::~BroadcastsFec() = default; + +void BroadcastsFec::send(OverlayImpl *overlay, PublicKeyHash send_as, td::BufferSlice data, td::uint32 flags, + double speed_multiplier) { + td::actor::create_actor(td::actor::ActorOptions().with_name("bcast"), std::move(data), flags, + actor_id(overlay), send_as, speed_multiplier) + .release(); +} + +void BroadcastsFec::send_part(OverlayImpl *overlay, PublicKeyHash send_as, Overlay::BroadcastDataHash data_hash, + td::uint32 size, td::uint32 flags, td::BufferSlice part, td::uint32 seqno, + fec::FecType fec_type, td::uint32 date) { + auto broadcast_hash = compute_broadcast_id(send_as, fec_type, data_hash, size, flags); + auto part_data_hash = sha256_bits256(part.as_slice()); + auto part_hash = compute_broadcast_part_id(broadcast_hash, part_data_hash, seqno); + auto part_obj = std::make_unique( + broadcast_hash, part_hash, PublicKey{}, overlay->get_certificate(send_as), data_hash, size, flags, part_data_hash, + std::move(part), seqno, std::move(fec_type), date, td::BufferSlice{}, false, adnl::AdnlNodeIdShort::zero()); + auto to_sign = part_obj->to_sign(); + auto P = td::PromiseCreator::lambda([overlay = actor_id(overlay), part = std::move(part_obj)]( + td::Result> R) mutable { + td::actor::send_closure(overlay, &OverlayImpl::broadcast_fec_signed, std::move(part), std::move(R)); + }); + td::actor::send_closure(overlay->keyring(), &keyring::Keyring::sign_add_get_public_key, send_as, std::move(to_sign), + std::move(P)); +} + +void BroadcastsFec::signed_(OverlayImpl *overlay, std::unique_ptr &&part, + td::Result> &&R) { + if (R.is_error()) { + td::Status reason = R.move_as_error(); + if (reason.code() == ErrorCode::notready) { + LOG(DEBUG) << "failed to send fec broadcast: " << reason; + } else { + LOG(WARNING) << "failed to send fec broadcast: " << reason; + } + return; + } + auto V = R.move_as_ok(); + part->source_ = std::move(V.second); + part->signature_ = std::move(V.first); + part->cert_ = overlay->get_certificate(part->source_.compute_short_id()); + td::Status S = process(overlay, *part); + if (S.is_error() && S.code() != ErrorCode::notready) { + LOG(WARNING) << "failed to process fec broadcast: " << S; + } +} + +td::Status BroadcastsFec::process_broadcast(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + tl_object_ptr broadcast) { + TRY_STATUS(overlay->check_date(broadcast->date_)); + PublicKey source(broadcast->src_); + auto part_data_hash = sha256_bits256(broadcast->data_.as_slice()); + TRY_RESULT(fec_type, fec::FecType::create(std::move(broadcast->fec_))); + auto broadcast_hash = compute_broadcast_id(source.compute_short_id(), fec_type, broadcast->data_hash_, + broadcast->data_size_, broadcast->flags_); + auto part_hash = compute_broadcast_part_id(broadcast_hash, part_data_hash, broadcast->seqno_); + TRY_RESULT(cert, Certificate::create(std::move(broadcast->certificate_))); + BroadcastFecPart part(broadcast_hash, part_hash, source, std::move(cert), broadcast->data_hash_, + static_cast(broadcast->data_size_), static_cast(broadcast->flags_), + part_data_hash, std::move(broadcast->data_), static_cast(broadcast->seqno_), + std::move(fec_type), static_cast(broadcast->date_), + std::move(broadcast->signature_), false, src_peer_id); + TRY_STATUS(process(overlay, part)); + return td::Status::OK(); +} + +td::Status BroadcastsFec::process_broadcast(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + tl_object_ptr broadcast) { + auto it = broadcasts_.find(broadcast->broadcast_hash_); + if (it == broadcasts_.end()) { + return td::Status::Error(ErrorCode::notready, "short part of unknown broadcast"); + } + auto &bcast = *it->second; + if (!bcast.ready_) { + return td::Status::Error(ErrorCode::protoviolation, "short part of not finished broadcast"); + } + TRY_STATUS(overlay->check_date(bcast.date_)); + + auto source = PublicKey{broadcast->src_}; + auto part_data_hash = broadcast->part_data_hash_; + auto broadcast_hash = bcast.hash_; + auto part_hash = compute_broadcast_part_id(broadcast_hash, part_data_hash, broadcast->seqno_); + TRY_RESULT(cert, Certificate::create(std::move(broadcast->certificate_))); + td::uint32 seqno = static_cast(broadcast->seqno_); + BroadcastFecPart part(broadcast_hash, part_hash, source, std::move(cert), bcast.data_hash_, bcast.fec_type_.size(), + bcast.flags_, part_data_hash, bcast.get_part(seqno), seqno, bcast.fec_type_, bcast.date_, + std::move(broadcast->signature_), true, src_peer_id); + TRY_STATUS(part.run_checks(overlay, &bcast)); + TRY_STATUS(part.run(overlay, bcast)); + return td::Status::OK(); +} + +void BroadcastsFec::checked(OverlayImpl *overlay, Overlay::BroadcastHash &&hash, td::Result &&R) { + auto it = broadcasts_.find(hash); + if (it != broadcasts_.end()) { + it->second->broadcast_checked(overlay, std::move(R)); + } +} + +void BroadcastsFec::gc(OverlayImpl *overlay) { + while (!broadcasts_.empty()) { + auto bcast = BroadcastFec::from_list_node(lru_.prev); + CHECK(bcast); + if (bcast->date_ > td::Clocks::system() - 60) { + break; + } + auto hash = bcast->hash_; + CHECK(broadcasts_.count(hash) == 1); + broadcasts_.erase(hash); + overlay->register_delivered_broadcast(hash); + } +} + +td::Status BroadcastsFec::process(OverlayImpl *overlay, BroadcastFecPart &part) { + auto it = broadcasts_.find(part.broadcast_hash_); + if (it == broadcasts_.end()) { + if (overlay->is_delivered(part.broadcast_hash_)) { + return td::Status::Error(ErrorCode::notready, "duplicate broadcast"); + } + TRY_STATUS(part.run_checks(overlay, nullptr)); + auto bcast = std::make_unique(part.broadcast_hash_, part.broadcast_data_hash_, part.flags_, + part.date_, part.source_, part.fec_type_); + TRY_STATUS(bcast->run_checks()); + TRY_STATUS(bcast->init_fec_type()); + lru_.put(bcast.get()); + it = broadcasts_.emplace(part.broadcast_hash_, std::move(bcast)).first; + } else { + TRY_STATUS(part.run_checks(overlay, it->second.get())); + } + TRY_STATUS(part.run(overlay, *it->second)); + return td::Status::OK(); +} + +} // namespace overlay + +} // namespace ton diff --git a/overlay/broadcast-fec.hpp b/overlay/broadcast-fec.hpp new file mode 100644 index 0000000000..0bc41c7c6a --- /dev/null +++ b/overlay/broadcast-fec.hpp @@ -0,0 +1,64 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ +#pragma once + +#include + +#include "adnl/adnl-node-id.hpp" +#include "auto/tl/ton_api.h" +#include "fec/fec.h" +#include "keys/keys.hpp" +#include "overlay/overlay.h" +#include "td/utils/List.h" + +namespace ton { + +namespace overlay { + +class OverlayImpl; +class BroadcastFec; +class BroadcastFecPart; + +class BroadcastsFec { + public: + BroadcastsFec(); + ~BroadcastsFec(); + void send(OverlayImpl *overlay, PublicKeyHash send_as, td::BufferSlice data, td::uint32 flags, + double speed_multiplier); + void send_part(OverlayImpl *overlay, PublicKeyHash send_as, Overlay::BroadcastDataHash data_hash, td::uint32 size, + td::uint32 flags, td::BufferSlice part, td::uint32 seqno, fec::FecType fec_type, td::uint32 date); + void signed_(OverlayImpl *overlay, std::unique_ptr &&part, + td::Result> &&R); + td::Status process_broadcast(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + tl_object_ptr broadcast); + td::Status process_broadcast(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + tl_object_ptr broadcast); + void checked(OverlayImpl *overlay, Overlay::BroadcastHash &&hash, td::Result &&R); + void gc(OverlayImpl *overlay); + + private: + td::Status process(OverlayImpl *overlay, BroadcastFecPart &part); + + std::map> broadcasts_; + td::ListNode lru_; +}; + +} // namespace overlay + +} // namespace ton diff --git a/overlay/broadcast-simple.cpp b/overlay/broadcast-simple.cpp new file mode 100644 index 0000000000..920077d4c5 --- /dev/null +++ b/overlay/broadcast-simple.cpp @@ -0,0 +1,241 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ + +#include "adnl/adnl-node-id.hpp" +#include "common/util.h" +#include "keys/encryptor.h" +#include "td/actor/PromiseFuture.h" +#include "td/actor/actor.h" +#include "td/utils/Status.h" +#include "td/utils/port/Stat.h" + +#include "broadcast-simple.hpp" +#include "overlay.hpp" + +namespace ton { + +namespace overlay { + +static constexpr std::size_t MAX_BCASTS = 100; + +static Overlay::BroadcastHash compute_broadcast_id(PublicKeyHash source, Overlay::BroadcastDataHash data_hash, + td::uint32 flags) { + auto obj = create_tl_object( + (flags & Overlays::BroadcastFlagAnySender()) ? PublicKeyHash::zero().tl() : source.tl(), data_hash, flags); + return get_tl_object_sha_bits256(obj); +} + +class BroadcastSimple : public td::ListNode { + friend class BroadcastsSimple; + + public: + BroadcastSimple(Overlay::BroadcastHash broadcast_hash, PublicKey source, std::shared_ptr cert, + td::uint32 flags, td::BufferSlice data, td::uint32 date, td::BufferSlice signature, bool is_valid, + adnl::AdnlNodeIdShort src_peer_id) + : broadcast_hash_(broadcast_hash) + , source_(std::move(source)) + , cert_(std::move(cert)) + , flags_(flags) + , data_(std::move(data)) + , date_(date) + , signature_(std::move(signature)) + , is_valid_(is_valid) + , src_peer_id_(src_peer_id) { + } + + td::Status run(OverlayImpl *overlay); + void checked(OverlayImpl *overlay, td::Result R); + void run_continue(OverlayImpl *overlay); + + td::BufferSlice serialize(); + td::BufferSlice to_sign(); + + static BroadcastSimple *from_list_node(ListNode *node) { + return static_cast(node); + } + + private: + Overlay::BroadcastHash broadcast_hash_; + PublicKey source_; + std::shared_ptr cert_; + td::uint32 flags_; + td::BufferSlice data_; + td::uint32 date_; + td::BufferSlice signature_; + bool is_valid_; + adnl::AdnlNodeIdShort src_peer_id_; +}; + +td::Status BroadcastSimple::run(OverlayImpl *overlay) { + auto r = overlay->check_source_eligible(source_, cert_.get(), static_cast(data_.size()), false); + if (r == BroadcastCheckResult::Forbidden) { + return td::Status::Error(ErrorCode::error, "broadcast is forbidden"); + } + is_valid_ = r == BroadcastCheckResult::Allowed; + TRY_RESULT(encryptor, overlay->get_encryptor(source_)); + TRY_STATUS(encryptor->check_signature(to_sign().as_slice(), signature_.as_slice())); + if (!is_valid_) { + auto P = td::PromiseCreator::lambda( + [overlay = actor_id(overlay), hash = broadcast_hash_](td::Result R) mutable { + td::actor::send_closure(overlay, &OverlayImpl::broadcast_simple_checked, std::move(hash), std::move(R)); + }); + overlay->check_broadcast(source_.compute_short_id(), data_.clone(), std::move(P)); + } else { + run_continue(overlay); + } + return td::Status::OK(); +} + +void BroadcastSimple::checked(OverlayImpl *overlay, td::Result R) { + if (R.is_error()) { + overlay->update_peer_err_ctr(src_peer_id_, false); + return; + } + is_valid_ = true; + run_continue(overlay); +} + +void BroadcastSimple::run_continue(OverlayImpl *overlay) { + auto B = serialize(); + auto nodes = overlay->get_neighbours(overlay->propagate_broadcast_to()); + auto manager = overlay->overlay_manager(); + for (auto &n : nodes) { + td::actor::send_closure(manager, &OverlayManager::send_message, n, overlay->local_id(), overlay->overlay_id(), + B.clone()); + } + overlay->deliver_broadcast(source_.compute_short_id(), data_.clone()); +} + +td::BufferSlice BroadcastSimple::serialize() { + return create_serialize_tl_object( + source_.tl(), cert_ ? cert_->tl() : Certificate::empty_tl(), flags_, data_.clone(), date_, signature_.clone()); +} + +td::BufferSlice BroadcastSimple::to_sign() { + return create_serialize_tl_object(broadcast_hash_, date_); +} + +BroadcastsSimple::BroadcastsSimple() = default; + +BroadcastsSimple::~BroadcastsSimple() = default; + +void BroadcastsSimple::send(OverlayImpl *overlay, PublicKeyHash send_as, td::BufferSlice data, td::uint32 flags) { + auto data_hash = sha256_bits256(data.as_slice()); + auto broadcast_hash = compute_broadcast_id(send_as, data_hash, flags); + if (has(broadcast_hash) || overlay->is_delivered(broadcast_hash)) { + LOG(DEBUG) << "failed to send simple broadcast: duplicate broadcast"; + return; + } + auto date = static_cast(td::Clocks::system()); + auto bcast = std::make_unique(broadcast_hash, PublicKey{}, nullptr, flags, std::move(data), date, + td::BufferSlice{}, false, adnl::AdnlNodeIdShort::zero()); + auto to_sign = bcast->to_sign(); + auto P = td::PromiseCreator::lambda([overlay = actor_id(overlay), bcast = std::move(bcast)]( + td::Result> R) mutable { + td::actor::send_closure(overlay, &OverlayImpl::broadcast_simple_signed, std::move(bcast), std::move(R)); + }); + td::actor::send_closure(overlay->keyring(), &keyring::Keyring::sign_add_get_public_key, send_as, std::move(to_sign), + std::move(P)); +} + +void BroadcastsSimple::signed_(OverlayImpl *overlay, std::unique_ptr &&bcast, + td::Result> &&R) { + if (R.is_error()) { + td::Status reason = R.move_as_error(); + if (reason.code() == ErrorCode::notready) { + LOG(DEBUG) << "failed to send simple broadcast: " << reason; + } else { + LOG(WARNING) << "failed to send simple broadcast: " << reason; + } + return; + } + auto V = R.move_as_ok(); + bcast->source_ = std::move(V.second); + bcast->signature_ = std::move(V.first); + bcast->cert_ = overlay->get_certificate(bcast->source_.compute_short_id()); + auto S = bcast->run(overlay); + if (S.is_error() && S.code() != ErrorCode::notready) { + LOG(WARNING) << "failed to send simple broadcast: " << S; + } + register_(overlay, std::move(bcast)); +} + +td::Status BroadcastsSimple::process_broadcast(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + tl_object_ptr broadcast) { + TRY_STATUS(overlay->check_date(broadcast->date_)); + auto src = PublicKey{broadcast->src_}; + auto data_hash = sha256_bits256(broadcast->data_.as_slice()); + auto broadcast_hash = compute_broadcast_id(src.compute_short_id(), data_hash, broadcast->flags_); + if (has(broadcast_hash) || overlay->is_delivered(broadcast_hash)) { + return td::Status::Error(ErrorCode::notready, "duplicate broadcast"); + } + TRY_RESULT(cert, Certificate::create(std::move(broadcast->certificate_))); + auto B = std::make_unique(broadcast_hash, src, std::move(cert), broadcast->flags_, + std::move(broadcast->data_), broadcast->date_, + std::move(broadcast->signature_), false, src_peer_id); + TRY_STATUS(B->run(overlay)); + register_(overlay, std::move(B)); + return td::Status::OK(); +} + +void BroadcastsSimple::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, + td::Promise promise) { + auto it = broadcasts_.find(query.hash_); + if (it == broadcasts_.end()) { + VLOG(OVERLAY_NOTICE) << this << ": received getBroadcastQuery(" << query.hash_ << ") from " << src + << " but broadcast is unknown"; + promise.set_value(create_serialize_tl_object()); + return; + } + VLOG(OVERLAY_DEBUG) << this << ": received getBroadcastQuery(" << query.hash_ << ") from " << src + << " sending broadcast"; + promise.set_value(it->second->serialize()); +} + +void BroadcastsSimple::checked(OverlayImpl *overlay, Overlay::BroadcastHash &&hash, td::Result &&R) { + auto it = broadcasts_.find(hash); + if (it != broadcasts_.end()) { + it->second->checked(overlay, std::move(R)); + } +} + +void BroadcastsSimple::gc(OverlayImpl *overlay) { + while (broadcasts_.size() > MAX_BCASTS) { + auto bcast = BroadcastSimple::from_list_node(lru_.get()); + CHECK(bcast); + auto hash = bcast->broadcast_hash_; + broadcasts_.erase(hash); + overlay->register_delivered_broadcast(hash); + } +} + +bool BroadcastsSimple::has(const Overlay::BroadcastHash &hash) { + return broadcasts_.find(hash) != broadcasts_.end(); +} + +void BroadcastsSimple::register_(OverlayImpl *overlay, std::unique_ptr bcast) { + auto hash = bcast->broadcast_hash_; + lru_.put(bcast.get()); + broadcasts_.emplace(hash, std::move(bcast)); + gc(overlay); +} + +} // namespace overlay + +} // namespace ton diff --git a/overlay/broadcast-simple.hpp b/overlay/broadcast-simple.hpp new file mode 100644 index 0000000000..7dc53422cb --- /dev/null +++ b/overlay/broadcast-simple.hpp @@ -0,0 +1,65 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ +#pragma once + +#include +#include + +#include "adnl/adnl-node-id.hpp" +#include "auto/tl/ton_api.h" +#include "keys/keys.hpp" +#include "overlay/overlay.h" +#include "td/actor/PromiseFuture.h" +#include "td/utils/List.h" +#include "td/utils/Status.h" +#include "td/utils/buffer.h" +#include "td/utils/common.h" + +namespace ton { + +namespace overlay { + +class OverlayImpl; +class BroadcastSimple; + +class BroadcastsSimple { + public: + BroadcastsSimple(); + ~BroadcastsSimple(); + void send(OverlayImpl *overlay, PublicKeyHash send_as, td::BufferSlice data, td::uint32 flags); + void signed_(OverlayImpl *overlay, std::unique_ptr &&bcast, + td::Result> &&R); + td::Status process_broadcast(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + tl_object_ptr broadcast); + void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, + td::Promise promise); + void checked(OverlayImpl *overlay, Overlay::BroadcastHash &&hash, td::Result &&R); + void gc(OverlayImpl *overlay); + + private: + bool has(const Overlay::BroadcastHash &hash); + void register_(OverlayImpl *overlay, std::unique_ptr bcast); + + std::map> broadcasts_; + td::ListNode lru_; +}; + +} // namespace overlay + +} // namespace ton diff --git a/overlay/broadcast-twostep.cpp b/overlay/broadcast-twostep.cpp new file mode 100644 index 0000000000..e11baaf9d0 --- /dev/null +++ b/overlay/broadcast-twostep.cpp @@ -0,0 +1,321 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ + +#include +#include +#include +#include +#include + +#include "adnl/adnl-node-id.hpp" +#include "auto/tl/ton_api.h" +#include "crypto/common/bitstring.h" +#include "keys/keys.hpp" +#include "td/actor/actor.h" +#include "td/fec/raptorq/Decoder.h" +#include "td/fec/raptorq/Encoder.h" +#include "td/utils/List.h" +#include "td/utils/Status.h" +#include "td/utils/buffer.h" +#include "td/utils/common.h" +#include "td/utils/int_types.h" +#include "td/utils/port/Clocks.h" + +#include "broadcast-twostep.hpp" +#include "overlay.hpp" + +namespace ton { + +namespace overlay { + +static constexpr std::size_t FEC_MIN_BYTES = 513; +static constexpr std::size_t FEC_MIN_OTHER_NODES = 4; + +static constexpr std::size_t fec_k(std::size_t other_nodes) { + return (other_nodes * 2 - 2) / 3; +} + +struct BroadcastTwostep : td::ListNode { + Overlay::BroadcastHash broadcast_id; + td::uint32 date; + std::unique_ptr decoder; +}; + +struct BroadcastTwostepDataSimple { + td::uint32 flags; + td::uint32 date; + adnl::AdnlNodeIdShort src; + std::vector dsts; + td::BufferSlice data; +}; + +struct BroadcastTwostepDataFec { + td::uint32 flags; + td::uint32 date; + adnl::AdnlNodeIdShort src; + adnl::AdnlNodeIdShort dst; + td::Bits256 data_hash; + td::uint32 data_size; + td::uint32 seqno; + td::BufferSlice part; +}; + +BroadcastsTwostep::BroadcastsTwostep() = default; + +BroadcastsTwostep::~BroadcastsTwostep() = default; + +void BroadcastsTwostep::send(OverlayImpl *overlay, PublicKeyHash send_as, td::BufferSlice data, td::uint32 flags) { + std::size_t data_size = data.size(); + td::Bits256 data_hash = sha256_bits256(data.as_slice()); + td::uint32 date = static_cast(td::Clocks::system()); + std::vector other_nodes; + overlay->iterate_all_peers([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) { + if (overlay->is_persistent_node(key) && key.pubkey_hash() != send_as) { + other_nodes.push_back(key); + } + }); + td::Bits256 broadcast_id; + if (data_size >= FEC_MIN_BYTES && other_nodes.size() >= FEC_MIN_OTHER_NODES) { + std::size_t k = fec_k(other_nodes.size()); + std::size_t part_size = (data_size + k - 1) / k; + CHECK(part_size < data_size); + broadcast_id = get_tl_object_sha_bits256(create_tl_object( + flags, date, send_as.bits256_value(), overlay->local_id().bits256_value(), data_hash, + static_cast(part_size))); + auto R = td::raptorq::Encoder::create(part_size, data.clone()); + if (R.is_error()) { + VLOG(OVERLAY_WARNING) << overlay << ": cannot create FEC encoder: " << R.move_as_error(); + return; + } + auto encoder = R.move_as_ok(); + encoder->precalc(); + for (std::size_t i = 0; i < other_nodes.size(); i++) { + td::uint32 seqno = static_cast(i); + td::BufferSlice part(part_size); + td::Status S = encoder->gen_symbol(seqno, part.as_slice()); + if (S.is_error()) { + VLOG(OVERLAY_WARNING) << overlay << ": cannot generate symbol: " << S; + continue; + } + td::BufferSlice to_sign = create_serialize_tl_object( + broadcast_id, static_cast(data_size), static_cast(seqno), part.clone()); + BroadcastTwostepDataFec passdata{flags, date, adnl::AdnlNodeIdShort(send_as), + other_nodes[i], data_hash, static_cast(data_size), + seqno, std::move(part)}; + auto P = td::PromiseCreator::lambda([overlay = actor_id(overlay), data = std::move(passdata)]( + td::Result> R) mutable { + td::actor::send_closure(overlay, &OverlayImpl::broadcast_twostep_signed_fec, std::move(data), std::move(R)); + }); + td::actor::send_closure(overlay->keyring(), &keyring::Keyring::sign_add_get_public_key, send_as, + std::move(to_sign), std::move(P)); + } + } else { + broadcast_id = get_tl_object_sha_bits256(create_tl_object( + flags, date, send_as.bits256_value(), overlay->local_id().bits256_value(), data_hash, + static_cast(data_size))); + td::BufferSlice to_sign = + create_serialize_tl_object(broadcast_id, data.clone()); + BroadcastTwostepDataSimple passdata{flags, date, adnl::AdnlNodeIdShort(send_as), std::move(other_nodes), + data.clone()}; + auto P = td::PromiseCreator::lambda([overlay = actor_id(overlay), data = std::move(passdata)]( + td::Result> R) mutable { + td::actor::send_closure(overlay, &OverlayImpl::broadcast_twostep_signed_simple, std::move(data), std::move(R)); + }); + td::actor::send_closure(overlay->keyring(), &keyring::Keyring::sign_add_get_public_key, send_as, std::move(to_sign), + std::move(P)); + } + if (!overlay->is_delivered(broadcast_id)) { + overlay->register_delivered_broadcast(broadcast_id); + overlay->deliver_broadcast(send_as, std::move(data)); + } +} + +static bool handle_error(const td::Result> &R) { + if (R.is_error()) { + if (R.error().code() == ErrorCode::notready) { + LOG(DEBUG) << "failed to send twostep broadcast: " << R.error(); + } else { + LOG(WARNING) << "failed to send twostep broadcast: " << R.error(); + } + return true; + } + return false; +} + +void BroadcastsTwostep::signed_simple(OverlayImpl *overlay, BroadcastTwostepDataSimple &&data, + td::Result> &&R) { + if (handle_error(R)) { + return; + } + auto V = R.move_as_ok(); + auto cert = overlay->get_certificate(data.src.pubkey_hash()); + td::BufferSlice broadcast = create_serialize_tl_object( + data.flags, data.date, V.second.tl(), overlay->local_id().bits256_value(), + cert ? cert->tl() : Certificate::empty_tl(), std::move(data.data), std::move(V.first)); + for (auto &dst : data.dsts) { + td::actor::send_closure(overlay->overlay_manager(), &Overlays::send_message_via, dst, overlay->local_id(), + overlay->overlay_id(), broadcast.clone(), sender_); + } +} + +void BroadcastsTwostep::signed_fec(OverlayImpl *overlay, BroadcastTwostepDataFec &&data, + td::Result> &&R) { + if (handle_error(R)) { + return; + } + auto V = R.move_as_ok(); + auto cert = overlay->get_certificate(data.src.pubkey_hash()); + td::BufferSlice broadcast = create_serialize_tl_object( + data.flags, data.date, V.second.tl(), overlay->local_id().bits256_value(), + cert ? cert->tl() : Certificate::empty_tl(), data.data_hash, data.data_size, data.seqno, std::move(data.part), + std::move(V.first)); + td::actor::send_closure(overlay->overlay_manager(), &Overlays::send_message_via, data.dst, overlay->local_id(), + overlay->overlay_id(), std::move(broadcast), sender_); +} + +static td::Result check_signature_and_certificate( + OverlayImpl *overlay, const PublicKey &src_key, const PublicKeyHash &src_keyhash, const td::BufferSlice &to_sign, + const td::BufferSlice &signature, const tl_object_ptr &certificate, + td::uint32 data_size) { + TRY_RESULT(encryptor, overlay->get_encryptor(src_key)); + TRY_STATUS(encryptor->check_signature(to_sign.as_slice(), signature.as_slice())); + TRY_RESULT(cert, Certificate::create(certificate)); + auto r = overlay->check_source_eligible(src_keyhash, cert.get(), data_size, true); + if (r == BroadcastCheckResult::Forbidden) { + return td::Status::Error(ErrorCode::error, "broadcast is forbidden"); + } + return r; +} + +void BroadcastsTwostep::rebroadcast(OverlayImpl *overlay, const adnl::AdnlNodeIdShort &bcast_src_adnl_id, + const td::BufferSlice &data) { + overlay->iterate_all_peers([&](const adnl::AdnlNodeIdShort &peer_id, OverlayPeer &) { + if (peer_id != bcast_src_adnl_id && peer_id != overlay->local_id()) { + td::actor::send_closure(overlay->overlay_manager(), &Overlays::send_message_via, peer_id, overlay->local_id(), + overlay->overlay_id(), data.clone(), sender_); + } + }); +} + +static void check_and_deliver(OverlayImpl *overlay, PublicKeyHash src, BroadcastCheckResult check_result, + td::BufferSlice data) { + if (check_result == BroadcastCheckResult::Allowed) { + overlay->deliver_broadcast(src, std::move(data)); + } else { + auto P = td::PromiseCreator::lambda( + [overlay = actor_id(overlay), src, data = data.clone()](td::Result R) mutable { + td::actor::send_closure(overlay, &OverlayImpl::broadcast_twostep_checked, std::move(src), std::move(data), + std::move(R)); + }); + overlay->check_broadcast(src, std::move(data), std::move(P)); + } +} + +td::Status BroadcastsTwostep::process_broadcast(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + tl_object_ptr broadcast) { + TRY_STATUS(overlay->check_date(broadcast->date_)); + PublicKey src_key(broadcast->src_); + PublicKeyHash src_keyhash(src_key.compute_short_id()); + adnl::AdnlNodeIdShort bcast_src_adnl_id{broadcast->src_adnl_id_}; + td::Bits256 broadcast_id = get_tl_object_sha_bits256(create_tl_object( + broadcast->flags_, broadcast->date_, src_keyhash.bits256_value(), bcast_src_adnl_id.bits256_value(), + sha256_bits256(broadcast->data_.as_slice()), static_cast(broadcast->data_.size()))); + td::BufferSlice to_sign = create_serialize_tl_object( + broadcast_id, broadcast->data_.clone()); + TRY_RESULT(check_result, check_signature_and_certificate(overlay, src_key, src_keyhash, to_sign, + broadcast->signature_, broadcast->certificate_, + static_cast(broadcast->data_.size()))); + if (src_peer_id == bcast_src_adnl_id) { + rebroadcast(overlay, bcast_src_adnl_id, serialize_tl_object(broadcast, true)); + } + if (overlay->is_delivered(broadcast_id)) { + return td::Status::Error(ErrorCode::notready, "duplicate broadcast"); + } + check_and_deliver(overlay, src_keyhash, check_result, std::move(broadcast->data_)); + return td::Status::OK(); +} + +td::Status BroadcastsTwostep::process_broadcast(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + tl_object_ptr broadcast) { + td::uint32 date = static_cast(broadcast->date_); + TRY_STATUS(overlay->check_date(date)); + PublicKey src_key(broadcast->src_); + PublicKeyHash src_keyhash(src_key.compute_short_id()); + adnl::AdnlNodeIdShort bcast_src_adnl_id{broadcast->src_adnl_id_}; + std::size_t data_size = static_cast(static_cast(broadcast->data_size_)); + std::size_t part_size = broadcast->part_.size(); + td::Bits256 broadcast_id = get_tl_object_sha_bits256(create_tl_object( + broadcast->flags_, broadcast->date_, src_keyhash.bits256_value(), bcast_src_adnl_id.bits256_value(), + broadcast->data_hash_, static_cast(part_size))); + td::BufferSlice to_sign = create_serialize_tl_object( + broadcast_id, broadcast->data_size_, broadcast->seqno_, broadcast->part_.clone()); + TRY_RESULT(check_result, + check_signature_and_certificate(overlay, src_key, src_keyhash, to_sign, broadcast->signature_, + broadcast->certificate_, static_cast(data_size))); + if (src_peer_id == bcast_src_adnl_id) { + rebroadcast(overlay, bcast_src_adnl_id, serialize_tl_object(broadcast, true)); + } + auto it = broadcasts_.find(broadcast_id); + if (it == broadcasts_.end()) { + if (overlay->is_delivered(broadcast_id)) { + return td::Status::Error(ErrorCode::notready, "duplicate broadcast"); + } + td::Result> R; + if (part_size == 0 || + (R = td::raptorq::Decoder::create({(data_size + part_size - 1) / part_size, part_size, data_size})) + .is_error()) { + return td::Status::Error(ErrorCode::protoviolation, "invalid FEC parameters"); + } + std::unique_ptr bcast(new BroadcastTwostep{{}, broadcast_id, date, R.move_as_ok()}); + lru_.put(bcast.get()); + it = broadcasts_.emplace(broadcast_id, std::move(bcast)).first; + } + auto bcast = it->second.get(); + TRY_STATUS(bcast->decoder->add_symbol({static_cast(broadcast->seqno_), broadcast->part_.as_slice()})); + if (bcast->decoder->may_try_decode()) { + TRY_RESULT(R, bcast->decoder->try_decode(false)); + broadcasts_.erase(it); + overlay->register_delivered_broadcast(broadcast_id); + check_and_deliver(overlay, src_keyhash, check_result, std::move(R.data)); + } + return td::Status::OK(); +} + +void BroadcastsTwostep::checked(OverlayImpl *overlay, PublicKeyHash &&src, td::BufferSlice &&data, + td::Result &&R) { + if (R.is_ok()) { + overlay->deliver_broadcast(src, std::move(data)); + } +} + +void BroadcastsTwostep::gc(OverlayImpl *overlay) { + while (!broadcasts_.empty()) { + auto bcast = static_cast(lru_.get()); + if (bcast->date > td::Clocks::system() - 60) { + break; + } + auto broadcast_id = bcast->broadcast_id; + broadcasts_.erase(broadcast_id); + overlay->register_delivered_broadcast(broadcast_id); + } +} + +} // namespace overlay + +} // namespace ton diff --git a/overlay/broadcast-twostep.hpp b/overlay/broadcast-twostep.hpp new file mode 100644 index 0000000000..914fee9e89 --- /dev/null +++ b/overlay/broadcast-twostep.hpp @@ -0,0 +1,74 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ +#pragma once + +#include +#include +#include + +#include "adnl/adnl-node-id.hpp" +#include "auto/tl/ton_api.h" +#include "keys/keys.hpp" +#include "td/utils/List.h" +#include "td/utils/Status.h" +#include "td/utils/buffer.h" +#include "td/utils/common.h" +#include "td/utils/int_types.h" + +#include "overlay.h" + +namespace ton { + +namespace overlay { + +class OverlayImpl; +struct BroadcastTwostep; +struct BroadcastTwostepDataSimple; +struct BroadcastTwostepDataFec; + +class BroadcastsTwostep { + public: + BroadcastsTwostep(); + ~BroadcastsTwostep(); + void send(OverlayImpl *overlay, PublicKeyHash send_as, td::BufferSlice data, td::uint32 flags); + void signed_simple(OverlayImpl *overlay, BroadcastTwostepDataSimple &&data, + td::Result> &&R); + void signed_fec(OverlayImpl *overlay, BroadcastTwostepDataFec &&data, + td::Result> &&R); + td::Status process_broadcast(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + tl_object_ptr broadcast); + td::Status process_broadcast(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + tl_object_ptr broadcast); + void checked(OverlayImpl *overlay, PublicKeyHash &&src, td::BufferSlice &&data, td::Result &&R); + void gc(OverlayImpl *overlay); + + void init_sender(td::actor::ActorId sender) { + sender_ = std::move(sender); + } + + private: + td::actor::ActorId sender_; + std::map> broadcasts_; + td::ListNode lru_; + + void rebroadcast(OverlayImpl *overlay, const adnl::AdnlNodeIdShort &bcast_src_adnl_id, const td::BufferSlice &data); +}; +} // namespace overlay + +} // namespace ton diff --git a/overlay/overlay-broadcast.cpp b/overlay/overlay-broadcast.cpp deleted file mode 100644 index 308257b517..0000000000 --- a/overlay/overlay-broadcast.cpp +++ /dev/null @@ -1,191 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "adnl/adnl-node-id.hpp" -#include "common/util.h" -#include "keys/encryptor.h" -#include "td/actor/PromiseFuture.h" -#include "td/actor/actor.h" -#include "td/utils/Status.h" -#include "td/utils/port/Stat.h" - -#include "overlay-broadcast.hpp" -#include "overlay.hpp" - -namespace ton { - -namespace overlay { - -td::Status BroadcastSimple::check_time() { - return overlay_->check_date(date_); -} - -td::Status BroadcastSimple::check_duplicate() { - return overlay_->check_delivered(broadcast_hash_); -} - -td::Status BroadcastSimple::check_source() { - auto r = overlay_->check_source_eligible(source_, cert_.get(), data_size(), false); - if (r == BroadcastCheckResult::Forbidden) { - return td::Status::Error(ErrorCode::error, "broadcast is forbidden"); - } - - is_valid_ = r == BroadcastCheckResult::Allowed; - return td::Status::OK(); -} - -td::BufferSlice BroadcastSimple::to_sign() { - return create_serialize_tl_object(broadcast_hash_, date_); -} - -td::Status BroadcastSimple::check_signature() { - TRY_RESULT(encryptor, overlay_->get_encryptor(source_)); - - return encryptor->check_signature(to_sign().as_slice(), signature_.as_slice()); -} - -td::Status BroadcastSimple::run_checks() { - TRY_STATUS(check_time()); - TRY_STATUS(check_duplicate()); - TRY_STATUS(check_source()); - TRY_STATUS(check_signature()); - return td::Status::OK(); -} - -td::Status BroadcastSimple::distribute() { - auto B = serialize(); - auto nodes = overlay_->get_neighbours(overlay_->propagate_broadcast_to()); - - auto manager = overlay_->overlay_manager(); - for (auto &n : nodes) { - td::actor::send_closure(manager, &OverlayManager::send_message, n, overlay_->local_id(), overlay_->overlay_id(), - B.clone()); - } - return td::Status::OK(); -} - -void BroadcastSimple::broadcast_checked(td::Result R) { - if (R.is_error()) { - td::actor::send_closure(actor_id(overlay_), &OverlayImpl::update_peer_err_ctr, src_peer_id_, false); - return; - } - is_valid_ = true; - run_continue().ignore(); -} - -tl_object_ptr BroadcastSimple::tl() const { - return create_tl_object(source_.tl(), cert_ ? cert_->tl() : Certificate::empty_tl(), - flags_, data_.clone(), date_, signature_.clone()); -} - -td::BufferSlice BroadcastSimple::serialize() { - return serialize_tl_object(tl(), true); -} - -td::Status BroadcastSimple::run_continue() { - TRY_STATUS(distribute()); - deliver(); - return td::Status::OK(); -} - -td::Status BroadcastSimple::run() { - TRY_STATUS(run_checks()); - if (!is_valid_) { - auto P = td::PromiseCreator::lambda( - [id = broadcast_hash_, overlay_id = actor_id(overlay_)](td::Result R) mutable { - td::actor::send_closure(std::move(overlay_id), &OverlayImpl::broadcast_checked, id, std::move(R)); - }); - overlay_->check_broadcast(source_.compute_short_id(), data_.clone(), std::move(P)); - return td::Status::OK(); - } - return run_continue(); -} - -td::Status BroadcastSimple::create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, - tl_object_ptr broadcast) { - auto src = PublicKey{broadcast->src_}; - auto data_hash = sha256_bits256(broadcast->data_.as_slice()); - auto broadcast_hash = compute_broadcast_id(src, data_hash, broadcast->flags_); - - TRY_STATUS(overlay->check_date(broadcast->date_)); - TRY_STATUS(overlay->check_delivered(broadcast_hash)); - TRY_RESULT(cert, Certificate::create(std::move(broadcast->certificate_))); - - auto B = std::make_unique(broadcast_hash, src, std::move(cert), broadcast->flags_, - std::move(broadcast->data_), broadcast->date_, - std::move(broadcast->signature_), false, overlay, src_peer_id); - TRY_STATUS(B->run()); - overlay->register_simple_broadcast(std::move(B)); - return td::Status::OK(); -} - -td::Status BroadcastSimple::create_new(td::actor::ActorId overlay, - td::actor::ActorId keyring, PublicKeyHash local_id, - td::BufferSlice data, td::uint32 flags) { - auto data_hash = sha256_bits256(data.as_slice()); - auto broadcast_hash = compute_broadcast_id(local_id, data_hash, flags); - auto date = static_cast(td::Clocks::system()); - - auto B = std::make_unique(broadcast_hash, PublicKey{}, nullptr, flags, std::move(data), date, - td::BufferSlice{}, false, nullptr, adnl::AdnlNodeIdShort::zero()); - - auto to_sign = B->to_sign(); - auto P = td::PromiseCreator::lambda( - [id = overlay, B = std::move(B)](td::Result> R) mutable { - if (R.is_error()) { - td::actor::send_closure(id, &OverlayImpl::failed_to_create_simple_broadcast, R.move_as_error()); - return; - } - auto V = R.move_as_ok(); - auto pub_id = V.second; - B->update_source(pub_id); - B->update_signature(std::move(V.first)); - td::actor::send_closure(id, &OverlayImpl::created_simple_broadcast, std::move(B)); - }); - td::actor::send_closure(keyring, &keyring::Keyring::sign_add_get_public_key, local_id, std::move(to_sign), - std::move(P)); - return td::Status::OK(); -} - -Overlay::BroadcastHash BroadcastSimple::compute_broadcast_id(PublicKeyHash source, Overlay::BroadcastDataHash data_hash, - td::uint32 flags) { - auto obj = create_tl_object( - (flags & Overlays::BroadcastFlagAnySender()) ? PublicKeyHash::zero().tl() : source.tl(), data_hash, flags); - return get_tl_object_sha_bits256(obj); -} - -Overlay::BroadcastHash BroadcastSimple::compute_broadcast_id(PublicKey source, Overlay::BroadcastDataHash data_hash, - td::uint32 flags) { - return compute_broadcast_id(source.compute_short_id(), data_hash, flags); -} - -void BroadcastSimple::update_overlay(OverlayImpl *overlay) { - if (overlay_) { - return; - } - overlay_ = overlay; - cert_ = overlay->get_certificate(source_.compute_short_id()); -} - -void BroadcastSimple::deliver() { - overlay_->deliver_broadcast(source_.compute_short_id(), data_.clone()); -} - -} // namespace overlay - -} // namespace ton diff --git a/overlay/overlay-broadcast.hpp b/overlay/overlay-broadcast.hpp deleted file mode 100644 index 4a77e082e0..0000000000 --- a/overlay/overlay-broadcast.hpp +++ /dev/null @@ -1,121 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -#include "adnl/adnl-local-id.h" -#include "adnl/adnl-node-id.hpp" -#include "auto/tl/ton_api.h" -#include "common/refcnt.hpp" -#include "overlay/overlay.h" -#include "td/actor/PromiseFuture.h" -#include "td/utils/List.h" -#include "td/utils/Status.h" -#include "td/utils/buffer.h" -#include "td/utils/common.h" - -namespace ton { - -namespace overlay { - -class OverlayImpl; - -class BroadcastSimple : public td::ListNode { - private: - Overlay::BroadcastHash broadcast_hash_; - - PublicKey source_; - std::shared_ptr cert_; - td::uint32 flags_; - td::BufferSlice data_; - td::uint32 date_; - td::BufferSlice signature_; - bool is_valid_{false}; - - OverlayImpl *overlay_; - - adnl::AdnlNodeIdShort src_peer_id_ = adnl::AdnlNodeIdShort::zero(); - - td::Status check_time(); - td::Status check_duplicate(); - td::Status check_source(); - td::Status check_signature(); - - td::Status run_checks(); - td::Status distribute(); - td::BufferSlice to_sign(); - - public: - BroadcastSimple(Overlay::BroadcastHash broadcast_hash, PublicKey source, std::shared_ptr cert, - td::uint32 flags, td::BufferSlice data, td::uint32 date, td::BufferSlice signature, bool is_valid, - OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id) - : broadcast_hash_(broadcast_hash) - , source_(std::move(source)) - , cert_(std::move(cert)) - , flags_(flags) - , data_(std::move(data)) - , date_(date) - , signature_(std::move(signature)) - , is_valid_(is_valid) - , overlay_(overlay) - , src_peer_id_(src_peer_id) { - } - - Overlay::BroadcastHash get_hash() const { - return broadcast_hash_; - } - - td::uint32 data_size() const { - return static_cast(data_.size()); - } - - void update_source(PublicKey source) { - source_ = source; - } - void update_signature(td::BufferSlice signature) { - signature_ = std::move(signature); - } - void deliver(); - - td::Status run(); - td::Status run_continue(); - - tl_object_ptr tl() const; - td::BufferSlice serialize(); - - void update_overlay(OverlayImpl *overlay); - void broadcast_checked(td::Result R); - - static td::Status create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, - tl_object_ptr broadcast); - static td::Status create_new(td::actor::ActorId overlay, td::actor::ActorId keyring, - PublicKeyHash local_id, td::BufferSlice data, td::uint32 flags); - - static Overlay::BroadcastHash compute_broadcast_id(PublicKey source, Overlay::BroadcastDataHash data_hash, - td::uint32 flags); - static Overlay::BroadcastHash compute_broadcast_id(PublicKeyHash source, Overlay::BroadcastDataHash data_hash, - td::uint32 flags); - - static BroadcastSimple *from_list_node(ListNode *node) { - return static_cast(node); - } -}; - -} // namespace overlay - -} // namespace ton diff --git a/overlay/overlay-fec-broadcast.cpp b/overlay/overlay-fec-broadcast.cpp deleted file mode 100644 index c67f613bb7..0000000000 --- a/overlay/overlay-fec-broadcast.cpp +++ /dev/null @@ -1,365 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "keys/encryptor.h" - -#include "overlay-fec-broadcast.hpp" -#include "overlay.hpp" - -namespace ton { - -namespace overlay { - -td::Result> BroadcastFec::create(Overlay::BroadcastHash hash, PublicKey src, - Overlay::BroadcastDataHash data_hash, td::uint32 flags, - td::uint32 date, fec::FecType fec_type) { - auto F = std::make_unique(hash, std::move(src), data_hash, flags, date, std::move(fec_type)); - TRY_STATUS(F->run_checks()); - TRY_STATUS(F->init_fec_type()); - return std::move(F); -} - -td::Status BroadcastFec::run_checks() { - if (fec_type_.size() > Overlays::max_fec_broadcast_size()) { - return td::Status::Error(ErrorCode::protoviolation, "too big fec broadcast"); - } - return td::Status::OK(); -} - -td::Status OverlayFecBroadcastPart::check_time() { - return overlay_->check_date(date_); -} - -td::Status OverlayFecBroadcastPart::check_duplicate() { - TRY_STATUS(overlay_->check_delivered(broadcast_hash_)); - - if (bcast_ && bcast_->received_part(seqno_)) { - return td::Status::Error(ErrorCode::notready, "duplicate part"); - } - return td::Status::OK(); -} - -td::Status OverlayFecBroadcastPart::check_source() { - auto r = overlay_->check_source_eligible(source_, cert_.get(), broadcast_size_, true); - if (r == BroadcastCheckResult::Forbidden) { - return td::Status::Error(ErrorCode::error, "broadcast is forbidden"); - } - - if (r == BroadcastCheckResult::NeedCheck) { - untrusted_ = true; - return td::Status::OK(); - } - - if (bcast_) { - TRY_STATUS(bcast_->is_eligible_sender(source_)); - } - - return td::Status::OK(); -} - -td::Status OverlayFecBroadcastPart::check_signature() { - TRY_RESULT(encryptor, overlay_->get_encryptor(source_)); - - return encryptor->check_signature(to_sign().as_slice(), signature_.as_slice()); -} - -td::Status OverlayFecBroadcastPart::run_checks() { - TRY_STATUS(check_time()); - TRY_STATUS(check_duplicate()); - TRY_STATUS(check_source()); - TRY_STATUS(check_signature()); - return td::Status::OK(); -} - -void BroadcastFec::broadcast_checked(td::Result R) { - if (R.is_error()) { - td::actor::send_closure(actor_id(overlay_), &OverlayImpl::update_peer_err_ctr, src_peer_id_, true); - return; - } - overlay_->deliver_broadcast(get_source().compute_short_id(), data_.clone()); - auto manager = overlay_->overlay_manager(); - while (!parts_.empty()) { - distribute_part(parts_.begin()->first); - } - - is_checked_ = true; -} - -// Do we need status here?? -td::Status BroadcastFec::distribute_part(td::uint32 seqno) { - auto i = parts_.find(seqno); - if (i == parts_.end()) { - VLOG(OVERLAY_WARNING) << "not distibuting empty part " << seqno; - // should not get here - return td::Status::OK(); - } - auto tls = std::move(i->second); - parts_.erase(i); - td::BufferSlice data_short = std::move(tls.first); - td::BufferSlice data = std::move(tls.second); - - auto nodes = overlay_->get_neighbours(overlay_->propagate_broadcast_to()); - auto manager = overlay_->overlay_manager(); - - for (auto &n : nodes) { - if (neighbour_completed(n)) { - continue; - } - if (neighbour_received(n)) { - td::actor::send_closure(manager, &OverlayManager::send_message, n, overlay_->local_id(), overlay_->overlay_id(), - data_short.clone()); - } else { - if (hash_.count_leading_zeroes() >= 12) { - VLOG(OVERLAY_INFO) << "broadcast " << hash_ << ": sending part " << seqno << " to " << n; - } - td::actor::send_closure(manager, &OverlayManager::send_message, n, overlay_->local_id(), overlay_->overlay_id(), - data.clone()); - } - } - return td::Status::OK(); -} - -td::Status OverlayFecBroadcastPart::apply() { - if (!bcast_) { - bcast_ = overlay_->get_fec_broadcast(broadcast_hash_); - } - if (!bcast_) { - if (is_short_) { - return td::Status::Error(ErrorCode::protoviolation, "short broadcast part for incomplete broadcast"); - } - TRY_RESULT(B, BroadcastFec::create(broadcast_hash_, source_, broadcast_data_hash_, flags_, date_, fec_type_)); - bcast_ = B.get(); - overlay_->register_fec_broadcast(std::move(B)); - } - if (bcast_->received_part(seqno_)) { - return td::Status::Error(ErrorCode::notready, "duplicate part"); - } else { - bcast_->add_received_part(seqno_); - } - - if (!bcast_->finalized() && is_short_) { - return td::Status::Error(ErrorCode::protoviolation, "short broadcast part for incomplete broadcast"); - } - - if (!bcast_->finalized()) { - bcast_->set_overlay(overlay_); - bcast_->set_src_peer_id(src_peer_id_); - TRY_STATUS(bcast_->add_part(seqno_, data_.clone(), export_serialized_short(), export_serialized())); - auto R = bcast_->finish(); - if (R.is_error()) { - auto S = R.move_as_error(); - if (S.code() != ErrorCode::notready) { - return S; - } - } else { - if (untrusted_) { - auto P = td::PromiseCreator::lambda( - [id = broadcast_hash_, overlay_id = actor_id(overlay_)](td::Result RR) mutable { - td::actor::send_closure(std::move(overlay_id), &OverlayImpl::broadcast_checked, id, std::move(RR)); - }); - overlay_->check_broadcast(bcast_->get_source().compute_short_id(), R.move_as_ok(), std::move(P)); - } else { - overlay_->deliver_broadcast(bcast_->get_source().compute_short_id(), R.move_as_ok()); - } - } - } else { - bcast_->set_overlay(overlay_); - bcast_->set_src_peer_id(src_peer_id_); - TRY_STATUS(bcast_->add_part(seqno_, data_.clone(), export_serialized_short(), export_serialized())); - } - return td::Status::OK(); -} - -td::Status OverlayFecBroadcastPart::distribute() { - TRY_STATUS(bcast_->distribute_part(seqno_)); - return td::Status::OK(); -} - -tl_object_ptr OverlayFecBroadcastPart::export_tl() { - if (data_.size() == 0) { - data_ = bcast_->get_part(seqno_); - } - - return create_tl_object( - source_.tl(), cert_ ? cert_->tl() : Certificate::empty_tl(), bcast_->get_data_hash(), bcast_->get_size(), - bcast_->get_flags(), data_.clone(), seqno_, bcast_->get_fec_type().tl(), bcast_->get_date(), signature_.clone()); -} - -tl_object_ptr OverlayFecBroadcastPart::export_tl_short() { - return create_tl_object( - source_.tl(), cert_ ? cert_->tl() : Certificate::empty_tl(), broadcast_hash_, part_data_hash_, seqno_, - signature_.clone()); -} - -td::BufferSlice OverlayFecBroadcastPart::export_serialized() { - return serialize_tl_object(export_tl(), true); -} - -td::BufferSlice OverlayFecBroadcastPart::export_serialized_short() { - return serialize_tl_object(export_tl_short(), true); -} - -td::BufferSlice OverlayFecBroadcastPart::to_sign() { - auto obj = create_tl_object(part_hash_, date_); - return serialize_tl_object(obj, true); -} - -td::Status OverlayFecBroadcastPart::create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, - tl_object_ptr broadcast) { - TRY_STATUS(overlay->check_date(broadcast->date_)); - auto source = PublicKey{broadcast->src_}; - auto part_data_hash = sha256_bits256(broadcast->data_.as_slice()); - - TRY_RESULT(fec_type, fec::FecType::create(std::move(broadcast->fec_))); - auto broadcast_hash = - compute_broadcast_id(source, fec_type, broadcast->data_hash_, broadcast->data_size_, broadcast->flags_); - auto part_hash = compute_broadcast_part_id(broadcast_hash, part_data_hash, broadcast->seqno_); - - if (broadcast_hash.count_leading_zeroes() >= 12) { - VLOG(OVERLAY_INFO) << "broadcast " << broadcast_hash << ": received part " << part_hash; - } - - TRY_STATUS(overlay->check_delivered(broadcast_hash)); - TRY_RESULT(cert, Certificate::create(std::move(broadcast->certificate_))); - - OverlayFecBroadcastPart B{broadcast_hash, - part_hash, - source, - std::move(cert), - broadcast->data_hash_, - static_cast(broadcast->data_size_), - static_cast(broadcast->flags_), - part_data_hash, - std::move(broadcast->data_), - static_cast(broadcast->seqno_), - std::move(fec_type), - static_cast(broadcast->date_), - std::move(broadcast->signature_), - false, - overlay->get_fec_broadcast(broadcast_hash), - overlay, - src_peer_id}; - TRY_STATUS(B.run()); - return td::Status::OK(); -} - -td::Status OverlayFecBroadcastPart::create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, - tl_object_ptr broadcast) { - auto bcast = overlay->get_fec_broadcast(broadcast->broadcast_hash_); - if (!bcast) { - return td::Status::Error(ErrorCode::notready, "short part of unknown broadcast"); - } - if (!bcast->finalized()) { - return td::Status::Error(ErrorCode::protoviolation, "short part of not finished broadcast"); - } - - TRY_STATUS(overlay->check_date(bcast->get_date())); - - auto source = PublicKey{broadcast->src_}; - auto part_data_hash = broadcast->part_data_hash_; - - auto broadcast_hash = bcast->get_hash(); - auto part_hash = compute_broadcast_part_id(broadcast_hash, part_data_hash, broadcast->seqno_); - - TRY_STATUS(overlay->check_delivered(broadcast_hash)); - TRY_RESULT(cert, Certificate::create(std::move(broadcast->certificate_))); - - OverlayFecBroadcastPart B{broadcast_hash, - part_hash, - source, - std::move(cert), - bcast->get_data_hash(), - bcast->get_size(), - bcast->get_flags(), - part_data_hash, - td::BufferSlice{}, - static_cast(broadcast->seqno_), - bcast->get_fec_type(), - bcast->get_date(), - std::move(broadcast->signature_), - true, - bcast, - overlay, - src_peer_id}; - TRY_STATUS(B.run()); - return td::Status::OK(); -} - -td::Status OverlayFecBroadcastPart::create_new(OverlayImpl *overlay, td::actor::ActorId overlay_actor_id, - PublicKeyHash local_id, Overlay::BroadcastDataHash data_hash, - td::uint32 size, td::uint32 flags, td::BufferSlice part, - td::uint32 seqno, fec::FecType fec_type, td::uint32 date) { - auto broadcast_hash = compute_broadcast_id(local_id, fec_type, data_hash, size, flags); - auto part_data_hash = sha256_bits256(part.as_slice()); - auto part_hash = compute_broadcast_part_id(broadcast_hash, part_data_hash, seqno); - - auto B = std::make_unique( - broadcast_hash, part_hash, PublicKey{}, overlay->get_certificate(local_id), data_hash, size, flags, - part_data_hash, std::move(part), seqno, std::move(fec_type), date, td::BufferSlice{}, false, nullptr, overlay, - adnl::AdnlNodeIdShort::zero()); - auto to_sign = B->to_sign(); - - auto P = td::PromiseCreator::lambda( - [id = overlay_actor_id, local_id, B = std::move(B)](td::Result> R) mutable { - if (R.is_error()) { - td::actor::send_closure(id, &OverlayImpl::failed_to_create_fec_broadcast, R.move_as_error()); - return; - } - auto V = R.move_as_ok(); - auto pub_id = V.second; - B->update_source(pub_id); - B->update_signature(std::move(V.first)); - td::actor::send_closure(id, &OverlayImpl::created_fec_broadcast, local_id, std::move(B)); - }); - td::actor::send_closure(overlay->keyring(), &keyring::Keyring::sign_add_get_public_key, local_id, std::move(to_sign), - std::move(P)); - return td::Status::OK(); -} - -Overlay::BroadcastHash OverlayFecBroadcastPart::compute_broadcast_id(PublicKey source, const fec::FecType &fec_type, - Overlay::BroadcastDataHash data_hash, - td::uint32 size, td::uint32 flags) { - return compute_broadcast_id(source.compute_short_id(), fec_type, data_hash, size, flags); -} - -Overlay::BroadcastHash OverlayFecBroadcastPart::compute_broadcast_id(PublicKeyHash source, const fec::FecType &fec_type, - Overlay::BroadcastDataHash data_hash, - td::uint32 size, td::uint32 flags) { - return get_tl_object_sha_bits256(create_tl_object( - (flags & Overlays::BroadcastFlagAnySender()) ? PublicKeyHash::zero().tl() : source.tl(), - get_tl_object_sha_bits256(fec_type.tl()), data_hash, size, flags)); -} - -Overlay::BroadcastPartHash OverlayFecBroadcastPart::compute_broadcast_part_id(Overlay::BroadcastHash broadcast_hash, - Overlay::BroadcastDataHash data_hash, - td::uint32 seqno) { - return get_tl_object_sha_bits256( - create_tl_object(broadcast_hash, data_hash, seqno)); -} - -void OverlayFecBroadcastPart::update_overlay(OverlayImpl *overlay) { - if (overlay_) { - return; - } - overlay_ = overlay; - cert_ = overlay_->get_certificate(source_.compute_short_id()); -} - -} // namespace overlay - -} // namespace ton diff --git a/overlay/overlay-fec-broadcast.hpp b/overlay/overlay-fec-broadcast.hpp deleted file mode 100644 index e56a0aa676..0000000000 --- a/overlay/overlay-fec-broadcast.hpp +++ /dev/null @@ -1,345 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -#include - -#include "auto/tl/ton_api.h" -#include "common/checksum.h" -#include "fec/fec.h" -#include "overlay/overlay.h" -#include "td/utils/List.h" - -namespace ton { - -namespace overlay { - -class OverlayImpl; - -class BroadcastFec : public td::ListNode { - public: - bool finalized() const { - return ready_; - } - - bool has_encoder() const { - return encoder_ != nullptr; - } - - auto get_hash() const { - return hash_; - } - - auto get_data_hash() const { - return data_hash_; - } - - td::uint32 get_date() const { - return date_; - } - - auto get_flags() const { - return flags_; - } - - const auto &get_fec_type() const { - return fec_type_; - } - - auto get_source() const { - return src_; - } - - auto get_size() const { - return fec_type_.size(); - } - - td::Status is_eligible_sender(PublicKey src) { - if (flags_ & Overlays::BroadcastFlagAnySender()) { - return td::Status::OK(); - } else { - if (src == src_) { - return td::Status::OK(); - } else { - return td::Status::Error(ErrorCode::protoviolation, "bad source"); - } - } - } - - td::Status add_part(td::uint32 seqno, td::BufferSlice data, td::BufferSlice serialized_fec_part_short, - td::BufferSlice serialized_fec_part) { - if (decoder_) { - td::fec::Symbol s; - s.id = seqno; - s.data = std::move(data); - - decoder_->add_symbol(std::move(s)); - } - parts_[seqno] = std::pair(std::move(serialized_fec_part_short), - std::move(serialized_fec_part)); - - return td::Status::OK(); - } - - td::Result finish() { - CHECK(decoder_); - if (!decoder_->may_try_decode()) { - return td::Status::Error(ErrorCode::notready, "need more parts"); - } - TRY_RESULT(D, decoder_->try_decode(true)); - if (sha256_bits256(D.data.as_slice()) != data_hash_) { - return td::Status::Error(ErrorCode::protoviolation, "bad hash"); - } - encoder_ = std::move(D.encoder); - CHECK(encoder_ != nullptr); - ready_ = true; - decoder_ = nullptr; - data_ = D.data.clone(); - return std::move(D.data); - } - - td::BufferSlice get_part(td::uint32 seqno) { - CHECK(ready_); - CHECK(encoder_ != nullptr); - auto R = encoder_->gen_symbol(seqno); - CHECK(R.id == seqno); - return std::move(R.data); - } - - td::Status init_fec_type() { - TRY_RESULT(D, fec_type_.create_decoder()); - decoder_ = std::move(D); - return td::Status::OK(); - } - - td::Status run_checks(); - - BroadcastFec(Overlay::BroadcastHash hash, PublicKey src, Overlay::BroadcastDataHash data_hash, td::uint32 flags, - td::uint32 date, fec::FecType fec_type) - : hash_(hash) - , data_hash_(data_hash) - , flags_(flags) - , date_(date) - , src_(std::move(src)) - , fec_type_(std::move(fec_type)) { - } - - static td::Result> create(Overlay::BroadcastHash hash, PublicKey src, - Overlay::BroadcastDataHash data_hash, td::uint32 flags, - td::uint32 date, fec::FecType fec_type); - - bool neighbour_received(adnl::AdnlNodeIdShort id) const { - return received_neighbours_.find(id) != received_neighbours_.end(); - } - - void add_received(adnl::AdnlNodeIdShort id) { - received_neighbours_.insert(id); - } - - bool neighbour_completed(adnl::AdnlNodeIdShort id) const { - return completed_neighbours_.find(id) != completed_neighbours_.end(); - } - - void add_completed(adnl::AdnlNodeIdShort id) { - completed_neighbours_.insert(id); - } - - static BroadcastFec *from_list_node(ListNode *node) { - return static_cast(node); - } - - bool received_part(td::uint32 seqno) const { - if (seqno + 64 < next_seqno_) { - return true; - } - if (seqno >= next_seqno_) { - return false; - } - return received_parts_ & (1ull << (next_seqno_ - seqno - 1)); - } - - void add_received_part(td::uint32 seqno) { - CHECK(!received_part(seqno)); - if (seqno < next_seqno_) { - received_parts_ |= (1ull << (next_seqno_ - seqno - 1)); - } else { - auto old = next_seqno_; - next_seqno_ = seqno + 1; - if (next_seqno_ - old >= 64) { - received_parts_ = 1; - } else { - received_parts_ = received_parts_ << (next_seqno_ - old); - received_parts_ |= 1; - } - } - } - - void broadcast_checked(td::Result R); - void set_overlay(OverlayImpl *overlay) { - overlay_ = overlay; - } - void set_src_peer_id(adnl::AdnlNodeIdShort src_peer_id) { - src_peer_id_ = src_peer_id; - } - - td::Status distribute_part(td::uint32 seqno); - - bool is_checked() const { - return is_checked_; - } - - private: - bool ready_ = false; - bool is_checked_ = false; - - Overlay::BroadcastHash hash_; - Overlay::BroadcastDataHash data_hash_; - - td::uint32 flags_; - td::uint32 date_; - - PublicKey src_; - fec::FecType fec_type_; - - std::unique_ptr decoder_; - std::unique_ptr encoder_; - - std::set received_neighbours_; - std::set completed_neighbours_; - - td::uint32 next_seqno_ = 0; - td::uint64 received_parts_ = 0; - - std::map> parts_; - OverlayImpl *overlay_; - adnl::AdnlNodeIdShort src_peer_id_ = adnl::AdnlNodeIdShort::zero(); - td::BufferSlice data_; -}; - -class OverlayFecBroadcastPart : public td::ListNode { - private: - Overlay::BroadcastHash broadcast_hash_; - Overlay::BroadcastPartHash part_hash_; - - PublicKey source_; - std::shared_ptr cert_; - Overlay::BroadcastDataHash broadcast_data_hash_; - td::uint32 broadcast_size_; - td::uint32 flags_; - Overlay::BroadcastDataHash part_data_hash_; - td::BufferSlice data_; - td::uint32 seqno_; - fec::FecType fec_type_; - td::uint32 date_; - td::BufferSlice signature_; - - bool is_short_; - bool untrusted_{false}; - - BroadcastFec *bcast_; - OverlayImpl *overlay_; - adnl::AdnlNodeIdShort src_peer_id_ = adnl::AdnlNodeIdShort::zero(); - - td::Status check_time(); - td::Status check_duplicate(); - td::Status check_source(); - td::Status check_signature(); - - td::Status run_checks(); - td::Status apply(); - td::Status distribute(); - - public: - OverlayFecBroadcastPart(Overlay::BroadcastHash broadcast_hash, Overlay::BroadcastPartHash part_hash, PublicKey source, - std::shared_ptr cert, Overlay::BroadcastDataHash data_hash, td::uint32 data_size, - td::uint32 flags, Overlay::BroadcastDataHash part_data_hash, td::BufferSlice data, - td::uint32 seqno, fec::FecType fec_type, td::uint32 date, td::BufferSlice signature, - bool is_short, BroadcastFec *bcast, OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id) - : broadcast_hash_(broadcast_hash) - , part_hash_(part_hash) - , source_(std::move(source)) - , cert_(std::move(cert)) - , broadcast_data_hash_(data_hash) - , broadcast_size_(data_size) - , flags_(flags) - , part_data_hash_(part_data_hash) - , data_(std::move(data)) - , seqno_(seqno) - , fec_type_(std::move(fec_type)) - , date_(date) - , signature_(std::move(signature)) - , is_short_(is_short) - , bcast_(bcast) - , overlay_(overlay) - , src_peer_id_(src_peer_id) { - } - - td::uint32 data_size() const { - return static_cast(data_.size()); - } - - Overlay::BroadcastPartHash get_hash() const { - return part_hash_; - } - - void update_source(PublicKey source) { - source_ = source; - } - void update_signature(td::BufferSlice signature) { - signature_ = std::move(signature); - } - void update_overlay(OverlayImpl *overlay); - - tl_object_ptr export_tl(); - tl_object_ptr export_tl_short(); - td::BufferSlice export_serialized(); - td::BufferSlice export_serialized_short(); - td::BufferSlice to_sign(); - - td::Status run() { - TRY_STATUS(run_checks()); - TRY_STATUS(apply()); - if (!untrusted_ || bcast_->is_checked()) { - TRY_STATUS(distribute()); - } - return td::Status::OK(); - } - - static td::Status create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, - tl_object_ptr broadcast); - static td::Status create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, - tl_object_ptr broadcast); - static td::Status create_new(OverlayImpl *overlay, td::actor::ActorId overlay_actor_id, - PublicKeyHash local_id, Overlay::BroadcastDataHash data_hash, td::uint32 size, - td::uint32 flags, td::BufferSlice part, td::uint32 seqno, fec::FecType fec_type, - td::uint32 date); - - static Overlay::BroadcastHash compute_broadcast_id(PublicKey source, const fec::FecType &fec_type, - Overlay::BroadcastDataHash data_hash, td::uint32 size, - td::uint32 flags); - static Overlay::BroadcastHash compute_broadcast_id(PublicKeyHash source, const fec::FecType &fec_type, - Overlay::BroadcastDataHash data_hash, td::uint32 size, - td::uint32 flags); - static Overlay::BroadcastPartHash compute_broadcast_part_id(Overlay::BroadcastHash broadcast_hash, - Overlay::BroadcastDataHash data_hash, td::uint32 seqno); -}; - -} // namespace overlay - -} // namespace ton diff --git a/overlay/overlay-fec.cpp b/overlay/overlay-fec.cpp deleted file mode 100644 index 42e7cc6a85..0000000000 --- a/overlay/overlay-fec.cpp +++ /dev/null @@ -1,77 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "adnl/utils.hpp" - -#include "overlay-fec.hpp" -#include "overlay.hpp" - -namespace ton { - -namespace overlay { - -void OverlayOutboundFecBroadcast::alarm() { - for (td::uint32 i = 0; i < 4; i++) { - auto X = encoder_->gen_symbol(seqno_++); - CHECK(X.data.size() <= 1000); - td::actor::send_closure(overlay_, &OverlayImpl::send_new_fec_broadcast_part, local_id_, data_hash_, - fec_type_.size(), flags_, std::move(X.data), X.id, fec_type_, date_); - } - - alarm_timestamp() = td::Timestamp::in(delay_); - - if (seqno_ >= to_send_) { - stop(); - } -} - -void OverlayOutboundFecBroadcast::start_up() { - encoder_->prepare_more_symbols(); - alarm(); -} - -OverlayOutboundFecBroadcast::OverlayOutboundFecBroadcast(td::BufferSlice data, td::uint32 flags, - td::actor::ActorId overlay, - PublicKeyHash local_id, double speed_multiplier) - : flags_(flags) { - delay_ /= speed_multiplier; - CHECK(data.size() <= (1 << 27)); - local_id_ = local_id; - overlay_ = std::move(overlay); - date_ = static_cast(td::Clocks::system()); - to_send_ = (static_cast(data.size()) / symbol_size_ + 1) * 2; - - data_hash_ = td::sha256_bits256(data); - - fec_type_ = td::fec::RaptorQEncoder::Parameters{data.size(), symbol_size_, 0}; - auto E = fec_type_.create_encoder(std::move(data)); - E.ensure(); - encoder_ = E.move_as_ok(); -} - -td::actor::ActorId OverlayOutboundFecBroadcast::create( - td::BufferSlice data, td::uint32 flags, td::actor::ActorId overlay, PublicKeyHash local_id, - double speed_multiplier) { - return td::actor::create_actor( - td::actor::ActorOptions().with_name("bcast"), std::move(data), flags, overlay, local_id, speed_multiplier) - .release(); -} - -} // namespace overlay - -} // namespace ton diff --git a/overlay/overlay-fec.hpp b/overlay/overlay-fec.hpp deleted file mode 100644 index 65169bfd11..0000000000 --- a/overlay/overlay-fec.hpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -#include "fec/fec.h" - -#include "overlay-manager.h" -#include "overlay.h" - -namespace ton { - -namespace overlay { - -class OverlayImpl; - -class OverlayOutboundFecBroadcast : public td::actor::Actor { - private: - const td::uint32 symbol_size_ = 768; - td::uint32 to_send_; - - td::uint32 seqno_ = 0; - PublicKeyHash local_id_; - Overlay::BroadcastDataHash data_hash_; - td::uint32 flags_ = 0; - double delay_ = 0.010; - td::int32 date_; - std::unique_ptr encoder_; - td::actor::ActorId overlay_; - fec::FecType fec_type_; - - public: - static td::actor::ActorId create(td::BufferSlice data, td::uint32 flags, - td::actor::ActorId overlay, - PublicKeyHash local_id, double speed_multiplier = 1.0); - OverlayOutboundFecBroadcast(td::BufferSlice data, td::uint32 flags, td::actor::ActorId overlay, - PublicKeyHash local_id, double speed_multiplier = 1.0); - - void alarm() override; - void start_up() override; -}; - -} // namespace overlay - -} // namespace ton diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index 35b59f107f..63bedd2538 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -278,8 +278,6 @@ void OverlayManager::send_query_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdS void OverlayManager::send_message_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, td::BufferSlice object, td::actor::ActorId via) { - CHECK(object.size() <= adnl::Adnl::huge_packet_max_size()); - auto extra = create_tl_object(); extra->flags_ = 0; @@ -551,7 +549,7 @@ const PublicKey &Certificate::issuer() const { return issued_by_.get(); } -td::Result> Certificate::create(tl_object_ptr cert) { +td::Result> Certificate::create(const tl_object_ptr &cert) { std::shared_ptr res; ton_api::downcast_call(*cert.get(), td::overloaded([&](ton_api::overlay_emptyCertificate &obj) { res = nullptr; }, diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index d653f4a4d1..2db266825b 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -24,6 +24,7 @@ #include "common/delay.h" #include "dht/dht.h" #include "keys/encryptor.h" +#include "rldp2/rldp.h" #include "td/utils/Random.h" #include "td/utils/Status.h" #include "td/utils/StringBuilder.h" @@ -105,6 +106,20 @@ OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor auto nodes_size = static_cast(nodes.size()); OverlayImpl::update_root_member_list(std::move(nodes), std::move(root_public_keys), std::move(cert)); update_neighbours(nodes_size); + + if (overlay_type_ == OverlayType::Public && + (!opts_.twostep_broadcast_sender_.empty() || opts_.send_twostep_broadcast_)) { + VLOG(OVERLAY_WARNING) << "Cannot enable twostep broadcasts in public overlay"; + opts_.twostep_broadcast_sender_ = {}; + opts_.send_twostep_broadcast_ = false; + } + if (opts_.send_twostep_broadcast_ && opts_.twostep_broadcast_sender_.empty()) { + VLOG(OVERLAY_WARNING) << "Twostep broadcast sender is not set"; + opts_.send_twostep_broadcast_ = false; + } + if (!opts_.twostep_broadcast_sender_.empty()) { + broadcasts_twostep_.init_sender(opts_.twostep_broadcast_sender_); + } } void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeers &query, @@ -140,23 +155,7 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_ping void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, td::Promise promise) { - auto it = broadcasts_.find(query.hash_); - if (it == broadcasts_.end()) { - VLOG(OVERLAY_NOTICE) << this << ": received getBroadcastQuery(" << query.hash_ << ") from " << src - << " but broadcast is unknown"; - promise.set_value(create_serialize_tl_object()); - return; - } - if (delivered_broadcasts_.find(query.hash_) != delivered_broadcasts_.end()) { - VLOG(OVERLAY_DEBUG) << this << ": received getBroadcastQuery(" << query.hash_ << ") from " << src - << " but broadcast already deleted"; - promise.set_value(create_serialize_tl_object()); - return; - } - - VLOG(OVERLAY_DEBUG) << this << ": received getBroadcastQuery(" << query.hash_ << ") from " << src - << " sending broadcast"; - promise.set_value(it->second->serialize()); + broadcasts_simple_.process_query(src, query, std::move(promise)); } void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcastList &query, @@ -198,7 +197,7 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, if (peer_list_.local_member_flags_ & OverlayMemberFlags::DoNotReceiveBroadcasts) { return td::Status::OK(); } - return BroadcastSimple::create(this, message_from, std::move(bcast)); + return broadcasts_simple_.process_broadcast(this, message_from, std::move(bcast)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, @@ -206,7 +205,7 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, if (peer_list_.local_member_flags_ & OverlayMemberFlags::DoNotReceiveBroadcasts) { return td::Status::OK(); } - return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); + return broadcasts_fec_.process_broadcast(this, message_from, std::move(b)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, @@ -214,7 +213,7 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, if (peer_list_.local_member_flags_ & OverlayMemberFlags::DoNotReceiveBroadcasts) { return td::Status::OK(); } - return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); + return broadcasts_fec_.process_broadcast(this, message_from, std::move(b)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, @@ -225,31 +224,11 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { - return td::Status::OK(); // disable this logic for now - auto it = fec_broadcasts_.find(msg->hash_); - if (it != fec_broadcasts_.end()) { - VLOG(OVERLAY_DEBUG) << this << ": received fec opt-out message from " << message_from << " for broadcast " - << msg->hash_; - it->second->add_received(message_from); - } else { - VLOG(OVERLAY_DEBUG) << this << ": received fec opt-out message from " << message_from << " for unknown broadcast " - << msg->hash_; - } return td::Status::OK(); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { - return td::Status::OK(); // disable this logic for now - auto it = fec_broadcasts_.find(msg->hash_); - if (it != fec_broadcasts_.end()) { - VLOG(OVERLAY_DEBUG) << this << ": received fec completed message from " << message_from << " for broadcast " - << msg->hash_; - it->second->add_completed(message_from); - } else { - VLOG(OVERLAY_DEBUG) << this << ": received fec completed message from " << message_from << " for unknown broadcast " - << msg->hash_; - } return td::Status::OK(); } @@ -260,6 +239,22 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, return td::Status::OK(); } +td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, + tl_object_ptr bcast) { + if (opts_.twostep_broadcast_sender_.empty()) { + return td::Status::Error("twostep broadcasts are not enabled"); + } + return broadcasts_twostep_.process_broadcast(this, message_from, std::move(bcast)); +} + +td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, + tl_object_ptr bcast) { + if (opts_.twostep_broadcast_sender_.empty()) { + return td::Status::Error("twostep broadcasts are not enabled"); + } + return broadcasts_twostep_.process_broadcast(this, message_from, std::move(bcast)); +} + void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, tl_object_ptr extra, td::BufferSlice data) { if (!is_valid_peer(src, extra ? extra->certificate_.get() : nullptr)) { @@ -450,28 +445,9 @@ void OverlayImpl::update_dht_nodes(OverlayNode node) { } void OverlayImpl::bcast_gc() { - while (broadcasts_.size() > max_data_bcasts()) { - auto bcast = BroadcastSimple::from_list_node(bcast_data_lru_.get()); - CHECK(bcast); - auto hash = bcast->get_hash(); - broadcasts_.erase(hash); - if (delivered_broadcasts_.insert(hash).second) { - bcast_lru_.push(hash); - } - } - while (fec_broadcasts_.size() > 0) { - auto bcast = BroadcastFec::from_list_node(bcast_fec_lru_.prev); - CHECK(bcast); - if (bcast->get_date() > td::Clocks::system() - 60) { - break; - } - auto hash = bcast->get_hash(); - CHECK(fec_broadcasts_.count(hash) == 1); - fec_broadcasts_.erase(hash); - if (delivered_broadcasts_.insert(hash).second) { - bcast_lru_.push(hash); - } - } + broadcasts_simple_.gc(this); + broadcasts_fec_.gc(this); + broadcasts_twostep_.gc(this); while (bcast_lru_.size() > max_bcasts()) { auto Id = bcast_lru_.front(); bcast_lru_.pop(); @@ -490,10 +466,7 @@ void OverlayImpl::send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::Bu VLOG(OVERLAY_WARNING) << "broadcast source certificate is invalid"; return; } - auto S = BroadcastSimple::create_new(actor_id(this), keyring_, send_as, std::move(data), flags); - if (S.is_error()) { - LOG(WARNING) << "failed to send broadcast: " << S; - } + broadcasts_simple_.send(this, send_as, std::move(data), flags); } void OverlayImpl::send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { @@ -506,8 +479,11 @@ void OverlayImpl::send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td VLOG(OVERLAY_WARNING) << "broadcast source certificate is invalid"; return; } - OverlayOutboundFecBroadcast::create(std::move(data), flags, actor_id(this), send_as, - opts_.broadcast_speed_multiplier_); + if (opts_.send_twostep_broadcast_) { + broadcasts_twostep_.send(this, send_as, std::move(data), flags); + } else { + broadcasts_fec_.send(this, send_as, std::move(data), flags, opts_.broadcast_speed_multiplier_); + } } void OverlayImpl::print(td::StringBuilder &sb) { @@ -565,30 +541,6 @@ BroadcastCheckResult OverlayImpl::check_source_eligible(PublicKey source, const return check_source_eligible(source.compute_short_id(), cert, size, is_fec); } -td::Status OverlayImpl::check_delivered(BroadcastHash hash) { - if (delivered_broadcasts_.count(hash) == 1 || broadcasts_.count(hash) == 1) { - return td::Status::Error(ErrorCode::notready, "duplicate broadcast"); - } else { - return td::Status::OK(); - } -} - -BroadcastFec *OverlayImpl::get_fec_broadcast(BroadcastHash hash) { - auto it = fec_broadcasts_.find(hash); - if (it == fec_broadcasts_.end()) { - return nullptr; - } else { - return it->second.get(); - } -} - -void OverlayImpl::register_fec_broadcast(std::unique_ptr bcast) { - auto hash = bcast->get_hash(); - bcast_fec_lru_.put(bcast.get()); - fec_broadcasts_.emplace(hash, std::move(bcast)); - bcast_gc(); -} - void OverlayImpl::get_self_node(td::Promise promise) { OverlayNode s{local_id_, overlay_id_, peer_list_.local_member_flags_}; auto to_sign = s.to_sign(); @@ -615,57 +567,33 @@ void OverlayImpl::get_self_node(td::Promise promise) { void OverlayImpl::send_new_fec_broadcast_part(PublicKeyHash local_id, Overlay::BroadcastDataHash data_hash, td::uint32 size, td::uint32 flags, td::BufferSlice part, td::uint32 seqno, fec::FecType fec_type, td::uint32 date) { - auto S = OverlayFecBroadcastPart::create_new(this, actor_id(this), local_id, data_hash, size, flags, std::move(part), - seqno, std::move(fec_type), date); - if (S.is_error() && S.code() != ErrorCode::notready) { - LOG(WARNING) << "failed to send broadcast part: " << S; - } + broadcasts_fec_.send_part(this, local_id, data_hash, size, flags, std::move(part), seqno, std::move(fec_type), date); } -void OverlayImpl::deliver_broadcast(PublicKeyHash source, td::BufferSlice data) { - callback_->receive_broadcast(source, overlay_id_, std::move(data)); +void OverlayImpl::broadcast_twostep_signed_simple(BroadcastTwostepDataSimple &&data, + td::Result> &&R) { + broadcasts_twostep_.signed_simple(this, std::move(data), std::move(R)); } -void OverlayImpl::failed_to_create_fec_broadcast(td::Status reason) { - if (reason.code() == ErrorCode::notready) { - LOG(DEBUG) << "failed to receive fec broadcast: " << reason; - } else { - LOG(WARNING) << "failed to receive fec broadcast: " << reason; - } +void OverlayImpl::broadcast_twostep_signed_fec(BroadcastTwostepDataFec &&data, + td::Result> &&R) { + broadcasts_twostep_.signed_fec(this, std::move(data), std::move(R)); } -void OverlayImpl::created_fec_broadcast(PublicKeyHash local_id, std::unique_ptr bcast) { - bcast->update_overlay(this); - auto S = bcast->run(); - if (S.is_error() && S.code() != ErrorCode::notready) { - LOG(WARNING) << "failed to send fec broadcast: " << S; - } +void OverlayImpl::broadcast_twostep_checked(PublicKeyHash &&src, td::BufferSlice &&data, td::Result &&R) { + broadcasts_twostep_.checked(this, std::move(src), std::move(data), std::move(R)); } -void OverlayImpl::failed_to_create_simple_broadcast(td::Status reason) { - if (reason.code() == ErrorCode::notready) { - LOG(DEBUG) << "failed to send simple broadcast: " << reason; - } else { - LOG(WARNING) << "failed to send simple broadcast: " << reason; - } +void OverlayImpl::deliver_broadcast(PublicKeyHash source, td::BufferSlice data) { + callback_->receive_broadcast(source, overlay_id_, std::move(data)); } -void OverlayImpl::created_simple_broadcast(std::unique_ptr bcast) { - bcast->update_overlay(this); - auto S = bcast->run(); - register_simple_broadcast(std::move(bcast)); - if (S.is_error() && S.code() != ErrorCode::notready) { - LOG(WARNING) << "failed to receive fec broadcast: " << S; +void OverlayImpl::register_delivered_broadcast(const BroadcastHash &hash) { + if (delivered_broadcasts_.insert(hash).second) { + bcast_lru_.push(hash); } } -void OverlayImpl::register_simple_broadcast(std::unique_ptr bcast) { - auto hash = bcast->get_hash(); - bcast_data_lru_.put(bcast.get()); - broadcasts_.emplace(hash, std::move(bcast)); - bcast_gc(); -} - td::Result OverlayImpl::get_encryptor(PublicKey source) { auto short_id = source.compute_short_id(); auto it = encryptor_map_.find(short_id); @@ -694,23 +622,30 @@ void OverlayImpl::set_privacy_rules(OverlayPrivacyRules rules) { rules_ = std::move(rules); } +bool OverlayImpl::is_delivered(const BroadcastHash &hash) { + return delivered_broadcasts_.find(hash) != delivered_broadcasts_.end(); +} + void OverlayImpl::check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise promise) { callback_->check_broadcast(src, overlay_id_, std::move(data), std::move(promise)); } -void OverlayImpl::broadcast_checked(Overlay::BroadcastHash hash, td::Result R) { - { - auto it = broadcasts_.find(hash); - if (it != broadcasts_.end()) { - it->second->broadcast_checked(std::move(R)); - } - } - { - auto it = fec_broadcasts_.find(hash); - if (it != fec_broadcasts_.end()) { - it->second->broadcast_checked(std::move(R)); - } - } +void OverlayImpl::broadcast_simple_signed(std::unique_ptr &&bcast, + td::Result> &&R) { + broadcasts_simple_.signed_(this, std::move(bcast), std::move(R)); +} + +void OverlayImpl::broadcast_simple_checked(Overlay::BroadcastHash &&hash, td::Result &&R) { + broadcasts_simple_.checked(this, std::move(hash), std::move(R)); +} + +void OverlayImpl::broadcast_fec_signed(std::unique_ptr &&part, + td::Result> &&R) { + broadcasts_fec_.signed_(this, std::move(part), std::move(R)); +} + +void OverlayImpl::broadcast_fec_checked(Overlay::BroadcastHash &&hash, td::Result &&R) { + broadcasts_fec_.checked(this, std::move(hash), std::move(R)); } void OverlayImpl::get_stats(td::Promise> promise) { diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 64c2116001..3fa9fee2bb 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -32,6 +32,7 @@ #include "auto/tl/ton_api.hpp" #include "fec/fec.h" #include "keys/encryptor.h" +#include "rldp2/rldp.h" #include "td/utils/DecTree.h" #include "td/utils/List.h" #include "td/utils/Status.h" @@ -41,9 +42,9 @@ #include "td/utils/port/signals.h" #include "tl-utils/common-utils.hpp" -#include "overlay-broadcast.hpp" -#include "overlay-fec-broadcast.hpp" -#include "overlay-fec.hpp" +#include "broadcast-fec.hpp" +#include "broadcast-simple.hpp" +#include "broadcast-twostep.hpp" #include "overlay-id.hpp" #include "overlay-manager.h" #include "overlay.h" @@ -218,9 +219,6 @@ class OverlayImpl : public Overlay { void update_neighbours(td::uint32 nodes_to_change); - void finish_fec_bcast(BroadcastHash id) { - out_fec_bcasts_.erase(id); - } struct PrintId { OverlayIdShort overlay_id; adnl::AdnlNodeIdShort local_id; @@ -236,24 +234,29 @@ class OverlayImpl : public Overlay { BroadcastCheckResult check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, bool is_fec); BroadcastCheckResult check_source_eligible(const PublicKeyHash &source, const Certificate *cert, td::uint32 size, bool is_fec); - td::Status check_delivered(BroadcastHash hash); - void broadcast_checked(Overlay::BroadcastHash hash, td::Result R); + void deliver_broadcast(PublicKeyHash source, td::BufferSlice data); + void register_delivered_broadcast(const BroadcastHash &hash); + bool is_delivered(const BroadcastHash &hash); void check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise promise); - void update_peer_err_ctr(adnl::AdnlNodeIdShort peer_id, bool is_fec); - - BroadcastFec *get_fec_broadcast(BroadcastHash hash); - void register_fec_broadcast(std::unique_ptr bcast); - void register_simple_broadcast(std::unique_ptr bcast); - void created_simple_broadcast(std::unique_ptr bcast); - void failed_to_create_simple_broadcast(td::Status reason); - void created_fec_broadcast(PublicKeyHash local_id, std::unique_ptr bcast); - void failed_to_create_fec_broadcast(td::Status reason); - void deliver_broadcast(PublicKeyHash source, td::BufferSlice data); + void broadcast_simple_signed(std::unique_ptr &&bcast, + td::Result> &&R); + void broadcast_simple_checked(Overlay::BroadcastHash &&hash, td::Result &&R); + void broadcast_fec_signed(std::unique_ptr &&part, + td::Result> &&R); + void broadcast_fec_checked(Overlay::BroadcastHash &&hash, td::Result &&R); void send_new_fec_broadcast_part(PublicKeyHash local_id, Overlay::BroadcastDataHash data_hash, td::uint32 size, td::uint32 flags, td::BufferSlice part, td::uint32 seqno, fec::FecType fec_type, td::uint32 date); + + void broadcast_twostep_signed_simple(BroadcastTwostepDataSimple &&data, + td::Result> &&R); + void broadcast_twostep_signed_fec(BroadcastTwostepDataFec &&data, + td::Result> &&R); + void broadcast_twostep_checked(PublicKeyHash &&src, td::BufferSlice &&data, td::Result &&R); + + void update_peer_err_ctr(adnl::AdnlNodeIdShort peer_id, bool is_fec); std::vector get_neighbours(td::uint32 max_size = 0) const; td::actor::ActorId overlay_manager() const { return manager_; @@ -288,10 +291,8 @@ class OverlayImpl : public Overlay { bool is_valid_peer(const adnl::AdnlNodeIdShort &id, const ton_api::overlay_MemberCertificate *certificate); bool is_persistent_node(const adnl::AdnlNodeIdShort &id); + void iterate_all_peers(std::function cb); - td::uint32 max_data_bcasts() const { - return 100; - } td::uint32 max_bcasts() const { return 1000; } @@ -354,6 +355,10 @@ class OverlayImpl : public Overlay { td::Status process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg); td::Status process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg); td::Status process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg); + td::Status process_broadcast(adnl::AdnlNodeIdShort message_from, + tl_object_ptr bcast); + td::Status process_broadcast(adnl::AdnlNodeIdShort message_from, + tl_object_ptr bcast); td::Status validate_peer_certificate(const adnl::AdnlNodeIdShort &node, const OverlayMemberCertificate &cert); td::Status validate_peer_certificate(const adnl::AdnlNodeIdShort &node, const OverlayMemberCertificate *cert); @@ -366,7 +371,6 @@ class OverlayImpl : public Overlay { void del_peer(const adnl::AdnlNodeIdShort &id); void del_from_neighbour_list(OverlayPeer *P); void del_from_neighbour_list(const adnl::AdnlNodeIdShort &id); - void iterate_all_peers(std::function cb); OverlayPeer *get_random_peer(bool only_alive = false); bool is_root_public_key(const PublicKeyHash &key) const; bool has_good_peers() const; @@ -401,16 +405,13 @@ class OverlayImpl : public Overlay { std::unique_ptr callback_; - std::map> broadcasts_; - std::map> fec_broadcasts_; + BroadcastsSimple broadcasts_simple_; + BroadcastsFec broadcasts_fec_; + BroadcastsTwostep broadcasts_twostep_; std::set delivered_broadcasts_; - td::ListNode bcast_data_lru_; - td::ListNode bcast_fec_lru_; std::queue bcast_lru_; - std::map> out_fec_bcasts_; - void bcast_gc(); static BroadcastHash get_broadcast_hash(adnl::AdnlNodeIdShort &src, td::Bits256 &data_hash) { diff --git a/overlay/overlays.h b/overlay/overlays.h index b5533151ec..69995b49d2 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -153,7 +153,7 @@ class Certificate { const PublicKey &issuer() const; const PublicKeyHash issuer_hash() const; - static td::Result> create(tl_object_ptr cert); + static td::Result> create(const tl_object_ptr &cert); static tl_object_ptr empty_tl(); private: @@ -269,6 +269,9 @@ struct OverlayOptions { td::uint32 default_permanent_members_flags_ = 0; double broadcast_speed_multiplier_ = 1.0; bool private_ping_peers_ = false; + + td::actor::ActorId twostep_broadcast_sender_ = {}; + bool send_twostep_broadcast_ = false; }; class Overlays : public td::actor::Actor { diff --git a/rldp2/CMakeLists.txt b/rldp2/CMakeLists.txt index 47860b7f01..217429b19b 100644 --- a/rldp2/CMakeLists.txt +++ b/rldp2/CMakeLists.txt @@ -33,6 +33,7 @@ set(RLDP_SOURCE Pacer.h rldp.h rldp.hpp + rldp-utils.h RldpReceiver.h RldpSender.h RldpConnection.h diff --git a/rldp2/RldpConnection.h b/rldp2/RldpConnection.h index fc53cd3cb9..a71fe31c00 100644 --- a/rldp2/RldpConnection.h +++ b/rldp2/RldpConnection.h @@ -64,8 +64,10 @@ class RldpConnection { return default_mtu_; } + static constexpr td::uint64 DEFAULT_MTU = 7680; + private: - td::uint64 default_mtu_ = 7680; + td::uint64 default_mtu_ = DEFAULT_MTU; std::map outbound_transfers_; td::uint32 in_flight_count_{0}; diff --git a/rldp2/rldp-in.hpp b/rldp2/rldp-in.hpp index be1df7171c..791096f2ab 100644 --- a/rldp2/rldp-in.hpp +++ b/rldp2/rldp-in.hpp @@ -93,6 +93,8 @@ class RldpIn : public RldpImpl { td::Promise promise) override; void set_default_mtu(td::uint64 mtu) override; + void add_peer_mtu_limit(adnl::AdnlNodeIdShort local_id, adnl::AdnlNodeIdShort peer_id, td::uint64 mtu) override; + void remove_peer_mtu_limit(adnl::AdnlNodeIdShort local_id, adnl::AdnlNodeIdShort peer_id, td::uint64 mtu) override; RldpIn(td::actor::ActorId adnl) : adnl_(adnl) { } @@ -110,8 +112,10 @@ class RldpIn : public RldpImpl { std::set local_ids_; td::optional custom_default_mtu_; + std::map, std::multiset> custom_peer_mtu_; td::actor::ActorId create_connection(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst); + td::uint64 get_peer_mtu(adnl::AdnlNodeIdShort local_id, adnl::AdnlNodeIdShort peer_id); }; } // namespace rldp2 diff --git a/rldp2/rldp-utils.h b/rldp2/rldp-utils.h new file mode 100644 index 0000000000..2a8b0adb0b --- /dev/null +++ b/rldp2/rldp-utils.h @@ -0,0 +1,73 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "rldp.h" + +namespace ton::rldp2 { + +class PeersMtuLimitGuard { + public: + PeersMtuLimitGuard() = default; + PeersMtuLimitGuard(td::actor::ActorId rldp, adnl::AdnlNodeIdShort local_id, + std::vector peer_ids, td::uint64 mtu) + : rldp_(std::move(rldp)), local_id_(local_id), peer_ids_(std::move(peer_ids)), mtu_(mtu) { + for (const adnl::AdnlNodeIdShort peer_id : peer_ids_) { + td::actor::send_closure(rldp_, &Rldp::add_peer_mtu_limit, local_id_, peer_id, mtu_); + } + } + PeersMtuLimitGuard(const PeersMtuLimitGuard&) = delete; + PeersMtuLimitGuard(PeersMtuLimitGuard&& other) noexcept + : rldp_(std::move(other.rldp_)) + , local_id_(other.local_id_) + , peer_ids_(std::move(other.peer_ids_)) + , mtu_(other.mtu_) { + other.rldp_ = {}; + } + ~PeersMtuLimitGuard() { + reset(); + } + PeersMtuLimitGuard& operator=(PeersMtuLimitGuard&& other) noexcept { + if (this == &other) { + return *this; + } + reset(); + rldp_ = std::move(other.rldp_); + local_id_ = other.local_id_; + peer_ids_ = std::move(other.peer_ids_); + mtu_ = other.mtu_; + other.rldp_ = {}; + return *this; + } + + private: + td::actor::ActorId rldp_; + adnl::AdnlNodeIdShort local_id_; + std::vector peer_ids_; + td::uint64 mtu_ = 0; + + void reset() { + if (rldp_.empty()) { + return; + } + for (const adnl::AdnlNodeIdShort peer_id : peer_ids_) { + td::actor::send_closure(rldp_, &Rldp::remove_peer_mtu_limit, local_id_, peer_id, mtu_); + } + } +}; + +} // namespace ton::rldp2 diff --git a/rldp2/rldp.cpp b/rldp2/rldp.cpp index 86660a5faa..e78bf9799a 100644 --- a/rldp2/rldp.cpp +++ b/rldp2/rldp.cpp @@ -133,14 +133,21 @@ td::actor::ActorId RldpIn::create_connection(adnl::AdnlNode return it->second.get(); } auto connection = td::actor::create_actor("RldpConnection", actor_id(this), src, dst, adnl_); - if (custom_default_mtu_) { - td::actor::send_closure(connection, &RldpConnectionActor::set_default_mtu, custom_default_mtu_.value()); - } + td::actor::send_closure(connection, &RldpConnectionActor::set_default_mtu, get_peer_mtu(src, dst)); auto res = connection.get(); connections_[std::make_pair(src, dst)] = std::move(connection); return res; } +td::uint64 RldpIn::get_peer_mtu(adnl::AdnlNodeIdShort local_id, adnl::AdnlNodeIdShort peer_id) { + td::uint64 mtu = custom_default_mtu_ ? custom_default_mtu_.value() : RldpConnection::DEFAULT_MTU; + auto it = custom_peer_mtu_.find({local_id, peer_id}); + if (it != custom_peer_mtu_.end()) { + mtu = std::max(mtu, *it->second.rbegin()); + } + return mtu; +} + void RldpIn::receive_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, td::Result r_data) { if (r_data.is_error()) { @@ -230,8 +237,32 @@ void RldpIn::get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p void RldpIn::set_default_mtu(td::uint64 mtu) { custom_default_mtu_ = mtu; - for (auto &connection : connections_) { - td::actor::send_closure(connection.second, &RldpConnectionActor::set_default_mtu, mtu); + for (auto &[p, connection] : connections_) { + td::actor::send_closure(connection, &RldpConnectionActor::set_default_mtu, get_peer_mtu(p.first, p.second)); + } +} + +void RldpIn::add_peer_mtu_limit(adnl::AdnlNodeIdShort local_id, adnl::AdnlNodeIdShort peer_id, td::uint64 mtu) { + custom_peer_mtu_[{local_id, peer_id}].insert(mtu); + auto it = connections_.find({local_id, peer_id}); + if (it != connections_.end()) { + auto &[p, connection] = *it; + td::actor::send_closure(connection, &RldpConnectionActor::set_default_mtu, get_peer_mtu(p.first, p.second)); + } +} + +void RldpIn::remove_peer_mtu_limit(adnl::AdnlNodeIdShort local_id, adnl::AdnlNodeIdShort peer_id, td::uint64 mtu) { + auto &map = custom_peer_mtu_[{local_id, peer_id}]; + auto it = map.find(mtu); + CHECK(it != map.end()); + map.erase(it); + if (map.empty()) { + custom_peer_mtu_.erase({local_id, peer_id}); + } + auto it2 = connections_.find({local_id, peer_id}); + if (it2 != connections_.end()) { + auto &[p, connection] = *it2; + td::actor::send_closure(connection, &RldpConnectionActor::set_default_mtu, get_peer_mtu(p.first, p.second)); } } diff --git a/rldp2/rldp.h b/rldp2/rldp.h index 97fc126a13..72d02868e7 100644 --- a/rldp2/rldp.h +++ b/rldp2/rldp.h @@ -38,6 +38,8 @@ class Rldp : public adnl::AdnlSenderInterface { td::BufferSlice data) = 0; virtual void set_default_mtu(td::uint64 mtu) = 0; + virtual void add_peer_mtu_limit(adnl::AdnlNodeIdShort local_id, adnl::AdnlNodeIdShort peer_id, td::uint64 mtu) = 0; + virtual void remove_peer_mtu_limit(adnl::AdnlNodeIdShort local_id, adnl::AdnlNodeIdShort peer_id, td::uint64 mtu) = 0; static td::actor::ActorOwn create(td::actor::ActorId adnl); }; diff --git a/test/test-catchain.cpp b/test/test-catchain.cpp index b9975fa883..6a5a9c7bfa 100644 --- a/test/test-catchain.cpp +++ b/test/test-catchain.cpp @@ -25,6 +25,7 @@ Copyright 2017-2020 Telegram Systems LLP */ + #include "adnl/adnl-test-loopback-implementation.h" #include "adnl/adnl.h" #include "adnl/utils.hpp" @@ -130,7 +131,7 @@ class CatChainInst : public td::actor::Actor { nodes.push_back(ton::catchain::CatChainNode{n.adnl_id, n.id_full}); } catchain_ = - ton::catchain::CatChain::create(make_callback(), opts, keyring_, adnl_, overlay_manager_, std::move(nodes), + ton::catchain::CatChain::create(make_callback(), opts, keyring_, adnl_, {}, overlay_manager_, std::move(nodes), nodes_[idx_].id, unique_hash_, std::string(""), "", false); } diff --git a/test/test-overlay.cpp b/test/test-overlay.cpp index 214a6a1d32..d7a3acaffd 100644 --- a/test/test-overlay.cpp +++ b/test/test-overlay.cpp @@ -25,6 +25,7 @@ Copyright 2017-2020 Telegram Systems LLP */ + #include #include diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 454fe91e6d..47a20b6d01 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -268,6 +268,12 @@ overlay.broadcastFec src:PublicKey certificate:overlay.Certificate data_hash:int overlay.broadcastFecShort src:PublicKey certificate:overlay.Certificate broadcast_hash:int256 part_data_hash:int256 seqno:int signature:bytes = overlay.Broadcast; overlay.broadcastNotFound = overlay.Broadcast; +overlay.broadcastTwostepSimple flags:int date:int src:PublicKey src_adnl_id:int256 certificate:overlay.Certificate data:bytes signature:bytes = overlay.Broadcast; +overlay.broadcastTwostepFec flags:int date:int src:PublicKey src_adnl_id:int256 certificate:overlay.Certificate data_hash:int256 data_size:int seqno:int part:bytes signature:bytes = overlay.Broadcast; +overlay.broadcastTwostep.id flags:int date:int src:int256 src_adnl_id:int256 data_hash:int256 part_size:int = overlay.broadcastTwostep.Id; +overlay.broadcastTwostepSimple.toSign id:int256 data:bytes = overlay.broadcastTwostepSimple.ToSign; +overlay.broadcastTwostepFec.toSign id:int256 data_size:int seqno:int part:bytes = overlay.broadcastTwostepFec.ToSign; + ---functions--- overlay.getRandomPeers peers:overlay.nodes = overlay.Nodes; diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 78532bfb5c..6f6a48578b 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -1168,8 +1168,8 @@ void ValidatorSessionImpl::start() { auto w = description().export_catchain_nodes(); catchain_ = catchain::CatChain::create(make_catchain_callback(), description().opts().catchain_opts, keyring_, adnl_, - overlay_manager_, std::move(w), local_id(), unique_hash_, db_root_, db_suffix_, - allow_unsafe_self_blocks_resync_); + rldp_, overlay_manager_, std::move(w), local_id(), unique_hash_, db_root_, + db_suffix_, allow_unsafe_self_blocks_resync_); check_all(); } diff --git a/validator/full-node-custom-overlays.cpp b/validator/full-node-custom-overlays.cpp index 44bede1d02..7b3a3f166e 100644 --- a/validator/full-node-custom-overlays.cpp +++ b/validator/full-node-custom-overlays.cpp @@ -227,6 +227,9 @@ void FullNodeCustomOverlay::try_init() { } void FullNodeCustomOverlay::init() { + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); + td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); + LOG(FULL_NODE_WARNING) << "Creating custom overlay \"" << name_ << "\" for adnl id " << local_id_ << " : " << nodes_.size() << " nodes, " << msg_senders_.size() << " msg senders, " << block_senders_.size() << " block senders, overlay_id=" << overlay_id_; @@ -260,14 +263,15 @@ void FullNodeCustomOverlay::init() { overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), 0, std::move(authorized_keys)}; overlay::OverlayOptions overlay_options; overlay_options.broadcast_speed_multiplier_ = opts_.private_broadcast_speed_multiplier_; + overlay_options.send_twostep_broadcast_ = true; + overlay_options.twostep_broadcast_sender_ = rldp2_; td::actor::send_closure( overlays_, &overlay::Overlays::create_private_overlay_ex, local_id_, overlay_id_full_.clone(), nodes_, std::make_unique(actor_id(this)), rules, PSTRING() << R"({ "type": "custom-overlay", "name": ")" << td::format::Escaped{name_} << R"(" })", overlay_options); + rldp_limit_guard_ = rldp2::PeersMtuLimitGuard(rldp2_, local_id_, nodes_, FullNode::max_block_size() + 1024); - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); - td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); inited_ = true; } diff --git a/validator/full-node-custom-overlays.hpp b/validator/full-node-custom-overlays.hpp index 6ce723a5c6..8340d29e83 100644 --- a/validator/full-node-custom-overlays.hpp +++ b/validator/full-node-custom-overlays.hpp @@ -18,6 +18,8 @@ #include +#include "rldp2/rldp-utils.h" + #include "full-node.h" #include "validator-telemetry.hpp" @@ -99,6 +101,7 @@ class FullNodeCustomOverlay : public td::actor::Actor { bool inited_ = false; overlay::OverlayIdFull overlay_id_full_; overlay::OverlayIdShort overlay_id_; + rldp2::PeersMtuLimitGuard rldp_limit_guard_; void try_init(); void init(); diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 2e796f31b5..1c84d8150d 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -16,6 +16,8 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include + #include "collator-node/collator-node.hpp" #include "common/delay.h" #include "td/utils/Random.h" @@ -543,6 +545,7 @@ void ValidatorGroup::create_session() { CHECK(!init_); init_ = true; std::vector vec; + std::vector adnl_ids; auto v = validator_set_->export_vector(); bool found = false; for (auto &el : v) { @@ -560,12 +563,15 @@ void ValidatorGroup::create_session() { local_id_full_ = n.pub_key; local_adnl_id_ = n.adnl_id; } + adnl_ids.push_back(n.adnl_id); vec.emplace_back(std::move(n)); } CHECK(found); td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_adnl_id_); td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_adnl_id_); + rldp_limit_guard_ = rldp2::PeersMtuLimitGuard(rldp2_, local_adnl_id_, adnl_ids, + config_.max_block_size + config_.max_collated_data_size + 1024); config_.catchain_opts.broadcast_speed_multiplier = opts_->get_catchain_broadcast_speed_multiplier(); if (!config_.new_catchain_ids) { diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 33200471db..37ded553da 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -22,6 +22,7 @@ #include "interfaces/validator-manager.h" #include "rldp/rldp.h" +#include "rldp2/rldp-utils.h" #include "rldp2/rldp.h" #include "validator-session/validator-session.h" @@ -147,6 +148,7 @@ class ValidatorGroup : public td::actor::Actor { td::actor::ActorId collation_manager_; td::actor::ActorOwn session_; adnl::AdnlNodeIdShort local_adnl_id_; + rldp2::PeersMtuLimitGuard rldp_limit_guard_; bool init_ = false; bool started_ = false; From ee1ffb0692fde6889fe3eb384fc20588ff04d480 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Wed, 7 Jan 2026 09:20:30 -0500 Subject: [PATCH 522/524] Include RocksDB before setting sanitizer compile flags (#2061) For some reason, RocksDB conditionally enables sanitizer suppressions only when it has its own WITH_{A,T,UB}SAN options set to true. --- CMakeLists.txt | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index afbcbe6762..db88f5b8e4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -103,6 +103,28 @@ if (TON_WERROR_BUILD) add_compile_options(-Werror) endif() +if (TON_USE_ROCKSDB) + message("Add rocksdb") + function(rocksdb_scope) + if (ANDROID) + set(PORTABLE ON CACHE BOOL "") + endif() + set(WITH_GFLAGS OFF CACHE BOOL "") + set(WITH_TESTS OFF CACHE BOOL "") + set(WITH_TOOLS OFF CACHE BOOL "") + set(USE_RTTI ON CACHE BOOL "") + set(FAIL_ON_WARNINGS OFF CACHE BOOL "") + set(WITH_ASAN ${TON_USE_ASAN} CACHE BOOL "") + set(WITH_TSAN ${TON_USE_TSAN} CACHE BOOL "") + set(WITH_UBSAN ${TON_USE_UBSAN} CACHE BOOL "") + add_subdirectory(third-party/rocksdb EXCLUDE_FROM_ALL) + # Broken CMake in rocksdb alters properties it has no business changing. + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE) + set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK) + endfunction() + rocksdb_scope() +endif() + if (TON_USE_ASAN) add_compile_options(-fsanitize=address) add_link_options(-fsanitize=address) @@ -154,22 +176,6 @@ endfunction() crc32_scope() set(CRC32C_FOUND 1) -if (TON_USE_ROCKSDB) - if (ANDROID) - set(PORTABLE ON CACHE BOOL "portable") - endif() - set(WITH_GFLAGS OFF CACHE BOOL "build with GFlags") - set(WITH_TESTS OFF CACHE BOOL "build with tests") - set(WITH_TOOLS OFF CACHE BOOL "build with tools") - set(USE_RTTI ON CACHE BOOL "use rtti") - set(FAIL_ON_WARNINGS OFF CACHE BOOL "fail on warnings") - message("Add rocksdb") - add_subdirectory(third-party/rocksdb EXCLUDE_FROM_ALL) - # Broken CMake in rocksdb alters properties it has no business changing. - set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE) - set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK) -endif() - option(USE_COROUTINES "experimental support of coroutines" OFF) if (USE_COROUTINES) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") From 00c6a6b6c56113db9be68e4787f9e8b1955ac470 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Wed, 7 Jan 2026 09:20:51 -0500 Subject: [PATCH 523/524] [tontester] Save session stats in work_dir/session-logs (#2062) --- test/tontester/src/tontester/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tontester/src/tontester/network.py b/test/tontester/src/tontester/network.py index 8a4c049de5..7ef949cefd 100644 --- a/test/tontester/src/tontester/network.py +++ b/test/tontester/src/tontester/network.py @@ -431,7 +431,7 @@ async def run(self, *, debug: DebugType = None): self._install.validator_engine_exe, self._local_config, zerostate.as_validator_config(), - ["--initial-sync-delay", "5"], + ["--initial-sync-delay", "5", "--session-logs", str(self._directory / "session-logs")], debug=debug, ) From 48439e0d43bd9048f96b39e80777b450103a7cda Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 8 Jan 2026 11:28:09 +0300 Subject: [PATCH 524/524] Fix BroadcastTwostep::gc (#2063) --- overlay/broadcast-twostep.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/overlay/broadcast-twostep.cpp b/overlay/broadcast-twostep.cpp index e11baaf9d0..f3d88dd89f 100644 --- a/overlay/broadcast-twostep.cpp +++ b/overlay/broadcast-twostep.cpp @@ -306,12 +306,13 @@ void BroadcastsTwostep::checked(OverlayImpl *overlay, PublicKeyHash &&src, td::B void BroadcastsTwostep::gc(OverlayImpl *overlay) { while (!broadcasts_.empty()) { - auto bcast = static_cast(lru_.get()); + auto bcast = static_cast(lru_.prev); + CHECK(bcast); if (bcast->date > td::Clocks::system() - 60) { break; } auto broadcast_id = bcast->broadcast_id; - broadcasts_.erase(broadcast_id); + CHECK(broadcasts_.erase(broadcast_id)); overlay->register_delivered_broadcast(broadcast_id); } }

(point: P) { return point.x + point.y; } + +@method_id(118) +fun test18() { + var p = `Point` { x: 8, `y`: 9 } as `PointAlias`?; + if (p != null) { + return sumXY(p); + } + return sumXY(p = getStorage1(80).lastPoint); +} + +global gPoint: Point; +global gPointN: Point?; + +@method_id(119) +fun test19() { + gPoint = { x: 1, y: 2 }; + assignCoords(mutate gPoint); + gPointN = getStorage1(9).lastPoint; + return (gPoint, gPointN, gPointN!.x); +} + +@method_id(120) +fun test20(setNull: bool) { + gPoint = generatePoint(8, 9); + gPointN = setNull ? null : gPoint; + if (!setNull) { + gPoint.x = gPointN!.y = 80; + } else { + (gPoint as Point?)!.y += 5; + } + return (gPoint, gPointN); +} + +global gJustInt: JustInt; +global gJustIntN: JustInt?; + +@method_id(121) +fun test21(setNull: bool) { + var value = 8; + gJustInt = { value }; + gJustIntN = setNull ? null : { value: value += 9 }; + (gJustInt.value, value) = (value, gJustInt.value); + return (gJustInt, gJustIntN, value, value * (gJustIntN == null ? 1 : gJustIntN!.value)); +} + +fun asm_func(x1: JustInt, x2: int, x3: JustIntAlias, x4: JustIntAlias, x5: int, x6: JustInt, x7: int): + (int, JustInt, int, JustInt, int, int, JustIntAlias2) + asm (x4 x5 x6 x7 x1 x2 x3->0 1 2 3 4 5 6) "NOP"; + +fun someOp(mutate self: JustInt, y: int): int { + val (newX, newY) = (self.value + y, y * 10); + self = { value: newX }; + return newY; +} + +@method_id(122) +fun test22(x: JustInt) { + return asm_func(x, x.value += 1, x, x, x.someOp(x.value / 20), x, x.value = x.value * 2); +} + +@method_id(123) +fun test23(p: Point) { + var complex = p as Point | (JustInt | Storage); + return match (complex) { + JustInt => 10 * complex.value, + PointAlias => 20, + Storage => 30 + complex.lastPoint.x, + }; +} + +@method_id(124) +fun test24(x: JustInt | JustIntAlias2 | JustIntAlias): JustInt { + return match (x) { + JustIntAlias => x + }; +} + +fun getPointOrStorage(getPoint: bool): Point | Storage { + return getPoint + ? Point { x: 10, y: 20 } + : Storage { owner: { id: 10, name: "" }, lastPoint: { x: 30, y: 40 } }; +} + +@method_id(125) +fun test25() { + var r1 = getPointOrStorage(true); + if (r1 !is Storage && r1 !is builder && r1 != null) { + match (val d = getPointOrStorage(false)) { + Point => { + r1.x += d.x; + r1.y += d.y; + } + Storage => { + r1.x += d.lastPoint.x; + r1.y += d.lastPoint.y; + } + } + } + return r1; +} + +fun acceptSomeUnion(r: Point | int | builder) { + return (r is Point) ? r.x + r.y : (r is int) ? r : null; +} + +@method_id(126) +fun test26() { + __expect_type(acceptSomeUnion, "(Point | int | builder) -> int?"); + return acceptSomeUnion({ x: 8, y: 10 })! + acceptSomeUnion(100)!; +} + +struct Has2EmptyNullable { + f1: Empty?; + f2: Empty? +} + +fun get2EmptyNullable(null1: bool, null2: bool): Has2EmptyNullable { + return { f1: null1 ? null : {}, f2: null2 ? null : {} }; +} + +@method_id(127) +fun test27() { + var (t1, t2) = (get2EmptyNullable(true, true), get2EmptyNullable(false, false)); + var (t3, t4) = (t1 as Has2EmptyNullable?, null as Has2EmptyNullable?); + return (t1, t2, 777, t1.f1 == null, t1.f2 == null, t2.f1 == null, t2.f2 == null, 777, t3, 777, t3!, 777, t4); +} + +fun getEmptyOrIntInt(getEmpty: bool): Empty | (int, int) { + return getEmpty ? {} : (1, 2); +} + +@method_id(128) +fun test28(getEmpty: bool) { + var t1: Empty | (int, int) = getEmptyOrIntInt(getEmpty); + var t2 = (getEmpty ? Empty{} : Has2EmptyNullable{ f1: null, f2: {} }) as EmptyAlias | Has2EmptyNullable | null; + return (t1, t1 is Empty, t1 is (int, int), t1 is builder, 777, t2, t2 is Empty, t2 is Has2EmptyNullable, t2 == null); +} + +struct Empty1 {} +struct Empty2 {} + +@method_id(129) +fun test29() { + var c = (null as Empty1 | Empty2?); + if (c == null) { + return c; + } + return null; +} + +@method_id(130) +fun test30(is1: bool) { + var e: Empty1 | Empty2 = is1 ? Empty1{} : Empty2 {}; + var n: Empty1 | int | Empty2 = 4; + var c: Empty1 | Empty2? = null; + if (is1) { + n = e; + } else { + c = e; + } + __expect_type(test29(), "null"); + return (e, 777, n, 777, c, 777, (test29() as Empty1? | Empty2), (test29() as Empty1 | Empty2 | null)!); +} + +struct IntOrNull { + e: Empty; + x: int?; +} + +@method_id(131) +fun test31() { + var s1 = { x: 5, e: {} } as IntOrNull?; + var s2 = { e: {}, x: null } as IntOrNull?; + var s3 = null as IntOrNull?; + return (s1, s2, s3, 777, s1 == null, s2 == null, s3 == null, 777, s1!.x == null, s2!.x == null); +} + + + + + +fun main(x: int8, y: MInt) { + __expect_type(PointAlias{x,y}, "Point"); + __expect_type(Point{x,y} as Point, "Point"); + __expect_type(test3(), "Point"); + __expect_type(maxCoord, "(Point) -> int8"); +} + +type PointAlias = Point; + +/** +@testcase | 101 | | 1 2 3 [ 1 5 ] +@testcase | 102 | | 1 2 3 [ 1 5 ] +@testcase | 103 | | 5 6 +@testcase | 104 | | 0 10 20 30 40 -1 +@testcase | 105 | | 35 30 45 +@testcase | 106 | | 10 20 +@testcase | 107 | | 5 5 10 20 15 +@testcase | 108 | | 777 131 777 0 777 777 777 131 777 132 777 777 +@testcase | 109 | | 70 30 20 20 -80 +@testcase | 110 | 0 | (null) (null) 0 +@testcase | 110 | -1 | 10 20 133 +@testcase | 111 | 0 | 0 2 10 20 30 +@testcase | 111 | -1 | 0 0 3 4 7 +@testcase | 112 | 0 | (null) (null) 0 +@testcase | 112 | -1 | 1 2 133 +@testcase | 113 | | (null) (null) 0 +@testcase | 114 | | 1 2 +@testcase | 115 | | (null) (null) (null) (null) 0 +@testcase | 116 | | -1 -4 [ 8 4 ] 7 (null) 0 [ 10 11 ] 135 +@testcase | 117 | 5 | 5 5 5 5 5 +@testcase | 117 | null | (null) (null) (null) -1 (null) +@testcase | 118 | | 17 +@testcase | 119 | | 10 20 9 10 133 9 +@testcase | 120 | 0 | 80 9 8 80 133 +@testcase | 120 | -1 | 8 14 (null) (null) 0 +@testcase | 121 | 0 | 17 17 8 136 +@testcase | 121 | -1 | 8 (null) 8 8 +@testcase | 122 | 100 | 101 50 106 212 100 101 101 +@testcase | 124 | 66 | 66 +@testcase | 125 | | (null) (null) 40 60 133 +@testcase | 126 | | 118 +@testcase | 127 | | 0 0 131 131 777 -1 -1 0 0 777 0 0 138 777 0 0 777 (null) (null) 0 +@testcase | 128 | -1 | (null) (null) 131 -1 0 0 777 (null) (null) 131 -1 0 0 +@testcase | 128 | 0 | 1 2 129 0 -1 0 777 0 131 138 0 -1 0 +@testcase | 129 | | (null) +@testcase | 130 | -1 | 140 777 (null) 140 777 0 777 0 0 +@testcase | 130 | 0 | 139 777 4 1 777 139 777 0 0 +@testcase | 131 | | 5 141 (null) 141 (null) 0 777 0 0 -1 777 0 -1 + +@fif_codegen +""" + test6 PROC:<{ + // + 10 PUSHINT // p.x + 20 PUSHINT // p.x p.y + }> +""" + +@fif_codegen +""" + sumXY PROC:<{ + // point.x point.y + ADD // '2 + }> +""" +@fif_codegen_avoid sumXY + +@fif_codegen +""" + test23 PROC:<{ + // p.x p.y + 2DROP + 20 PUSHINT // '10=20 + }> +""" + +@fif_codegen +""" + test29 PROC:<{ + // + PUSHNULL // '5 + }> +""" + + */ diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk index aa67bcad73..68935733cd 100644 --- a/tolk-tester/tests/type-aliases-tests.tolk +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -7,6 +7,9 @@ type Pair2_v2 = (MInt, MInt); type MBool = bool; type Tuple2Int = [int, int]; +struct Point { x: int; y: int } +type PointAlias = Point; +type PointAlias2 = PointAlias; fun test1(x: MInt): MVoid { var y = x; @@ -36,6 +39,10 @@ fun test1(x: MInt): MVoid { __expect_type(x as MInt?, "MInt?"); __expect_type(x as MIntN, "MIntN"); + __expect_type(PointAlias{x:0,y:0}, "Point"); + __expect_type(PointAlias2{x:0,y:0}, "Point"); + __expect_type(Point{x:0,y:0} as PointAlias, "PointAlias"); + if (x) { return; } } diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index 3411f4a066..e4318dc15e 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -222,6 +222,26 @@ fun testCallbackUnion(call1st: bool): (int, int) | int { return cb(10) as int } +struct WithCallbacks { + f1: (int -> int)?, + beginCell: builder -> cell, +} + +fun doEndCellActually(w: WithCallbacks, b: builder) { return w.beginCell(b); } + +@method_id(115) +fun testCallableFields() { + var w: WithCallbacks = { f1: null, beginCell: endCell }; + if (10 > 3) { + w.f1 = justAdd2; + } + return ( + w.f1!(5), + w.doEndCellActually(beginCell().storeInt(6, 32)).beginParse().loadInt(32), + beginParse(w.beginCell(beginCell().storeInt(7, 32))).loadInt(32) + ); +} + fun main() {} /** @@ -246,4 +266,5 @@ fun main() {} @testcase | 113 | -1 | 12 -1 @testcase | 114 | -1 | 2 3 130 @testcase | 114 | 0 | (null) 12 1 +@testcase | 115 | | 7 6 7 */ diff --git a/tolk/abscode.cpp b/tolk/abscode.cpp index 90c7fbcf17..984d6ffc88 100644 --- a/tolk/abscode.cpp +++ b/tolk/abscode.cpp @@ -401,7 +401,14 @@ std::vector CodeBlob::create_var(TypePtr var_type, SrcLocation loc, s std::vector ir_idx; int stack_w = var_type->get_width_on_stack(); ir_idx.reserve(stack_w); - if (const TypeDataTensor* t_tensor = var_type->try_as()) { + if (const TypeDataStruct* t_struct = var_type->try_as()) { + for (int i = 0; i < t_struct->struct_ref->get_num_fields(); ++i) { + StructFieldPtr field_ref = t_struct->struct_ref->get_field(i); + std::string sub_name = name.empty() || t_struct->struct_ref->get_num_fields() == 1 ? name : name + "." + field_ref->name; + std::vector nested = create_var(field_ref->declared_type, loc, std::move(sub_name)); + ir_idx.insert(ir_idx.end(), nested.begin(), nested.end()); + } + } else if (const TypeDataTensor* t_tensor = var_type->try_as()) { for (int i = 0; i < t_tensor->size(); ++i) { std::string sub_name = name.empty() ? name : name + "." + std::to_string(i); std::vector nested = create_var(t_tensor->items[i], loc, std::move(sub_name)); diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index ec9d689d47..6fc23457b9 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -385,6 +385,40 @@ static V parse_maybe_instantiationTs_after_identifier(L } } +static V parse_object_field(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.check(tok_identifier, "field name"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + + if (lex.tok() == tok_comma || lex.tok() == tok_clbrace) { + auto v_same_ident = createV(v_ident->loc, v_ident->name); + auto v_same_expr = createV(v_ident->loc, v_same_ident, nullptr); + return createV(loc, v_ident, v_same_expr); + } + + lex.expect(tok_colon, "`:`"); + return createV(loc, v_ident, parse_expr(lex)); +} + +static V parse_object_body(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_opbrace, "`{`"); + + std::vector fields; + while (lex.tok() != tok_clbrace) { + fields.push_back(parse_object_field(lex)); + if (lex.tok() == tok_comma) { + lex.next(); + } else if (lex.tok() != tok_clbrace) { + lex.unexpected("`,`"); + } + } + lex.expect(tok_clbrace, "`}`"); + + return createV(loc, std::move(fields)); +} + // `throw code` / `throw (code)` / `throw (code, arg)` // it's technically a statement (can't occur "in any place of expression"), // but inside `match` arm it can appear without semicolon: `pattern => throw 123` @@ -587,8 +621,14 @@ static AnyExprV parse_expr100(Lexer& lex) { if (lex.tok() == tok_lt) { v_instantiationTs = parse_maybe_instantiationTs_after_identifier(lex); } + if (lex.tok() == tok_opbrace) { + auto explicit_ref = createV(loc, v_ident, v_instantiationTs); + return createV(loc, explicit_ref, parse_object_body(lex)); + } return createV(loc, v_ident, v_instantiationTs); } + case tok_opbrace: + return createV(loc, createV(loc), parse_object_body(lex)); case tok_match: return parse_match_expression(lex); default: @@ -1227,6 +1267,52 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(loc, v_ident, v_param_list, v_body, ret_type, genericsT_list, std::move(method_id), flags); } +static AnyV parse_struct_field(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.check(tok_identifier, "field name"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + lex.expect(tok_colon, "`: `"); + TypePtr declared_type = parse_type_from_tokens(lex); + + if (lex.tok() == tok_assign) { // `id: int = 3` + lex.error("default values for fields not supported yet"); + } + + return createV(loc, v_ident, declared_type); +} + +static V parse_struct_body(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_opbrace, "`{`"); + + std::vector fields; + while (lex.tok() != tok_clbrace) { + fields.push_back(parse_struct_field(lex)); + if (lex.tok() == tok_comma || lex.tok() == tok_semicolon) { + lex.next(); + } else if (lex.tok() != tok_clbrace) { + lex.unexpected("`;` or `,`"); + } + } + lex.expect(tok_clbrace, "`}`"); + + return createV(loc, std::move(fields)); +} + +static AnyV parse_struct_declaration(Lexer& lex, const std::vector>& annotations) { + SrcLocation loc = lex.cur_location(); + if (!annotations.empty()) { + lex.error("@annotations are not applicable to type alias declaration"); + } + lex.expect(tok_struct, "`struct`"); + lex.check(tok_identifier, "identifier"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + + return createV(loc, v_ident, parse_struct_body(lex)); +} + static AnyV parse_tolk_required_version(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.next_special(tok_semver, "semver"); // syntax: "tolk 0.6" @@ -1301,9 +1387,12 @@ AnyV parse_src_file_to_ast(const SrcFile* file) { toplevel_declarations.push_back(parse_function_declaration(lex, annotations)); annotations.clear(); break; + case tok_struct: + toplevel_declarations.push_back(parse_struct_declaration(lex, annotations)); + annotations.clear(); + break; case tok_export: - case tok_struct: case tok_enum: case tok_operator: case tok_infix: diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 417ff411c2..7108b95b08 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -119,6 +119,9 @@ class ASTReplacerInFunctionBody : public ASTReplacer { virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } // statements virtual AnyV replace(V v) { return replace_children(v); } virtual AnyV replace(V v) { return replace_children(v); } @@ -160,6 +163,9 @@ class ASTReplacerInFunctionBody : public ASTReplacer { case ast_not_null_operator: return replace(v->as()); case ast_match_expression: return replace(v->as()); case ast_match_arm: return replace(v->as()); + case ast_object_field: return replace(v->as()); + case ast_object_body: return replace(v->as()); + case ast_object_literal: return replace(v->as()); default: throw UnexpectedASTNodeType(v, "ASTReplacerInFunctionBody::replace"); } diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 66781f1f0d..1217fa45b2 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -136,6 +136,15 @@ class ASTReplicatorFunction : public ASTReplicator { virtual V clone(V v) { return createV(v->loc, v->pattern_kind, clone(v->exact_type), clone(v->get_pattern_expr()), clone(v->get_body())); } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_field_identifier()), clone(v->get_init_val())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_all_fields())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_maybe_reference()), clone(v->get_body())); + } // statements @@ -220,6 +229,9 @@ class ASTReplicatorFunction : public ASTReplicator { case ast_not_null_operator: return clone(v->as()); case ast_match_expression: return clone(v->as()); case ast_match_arm: return clone(v->as()); + case ast_object_field: return clone(v->as()); + case ast_object_body: return clone(v->as()); + case ast_object_literal: return clone(v->as()); default: throw UnexpectedASTNodeType(v, "ASTReplicatorFunction::clone"); } diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index a232888f28..7782c506d4 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -61,6 +61,9 @@ class ASTStringifier final : public ASTVisitor { {ast_not_null_operator, "ast_not_null_operator"}, {ast_match_expression, "ast_match_expression"}, {ast_match_arm, "ast_match_arm"}, + {ast_object_field, "ast_object_field"}, + {ast_object_body, "ast_object_body"}, + {ast_object_literal, "ast_object_literal"}, // statements {ast_empty_statement, "ast_empty_statement"}, {ast_block_statement, "ast_block_statement"}, @@ -85,6 +88,9 @@ class ASTStringifier final : public ASTVisitor { {ast_global_var_declaration, "ast_global_var_declaration"}, {ast_constant_declaration, "ast_constant_declaration"}, {ast_type_alias_declaration, "ast_type_alias_declaration"}, + {ast_struct_field, "ast_struct_field"}, + {ast_struct_body, "ast_struct_body"}, + {ast_struct_declaration, "ast_struct_declaration"}, {ast_tolk_required_version, "ast_tolk_required_version"}, {ast_import_directive, "ast_import_directive"}, {ast_tolk_file, "ast_tolk_file"}, @@ -163,6 +169,10 @@ class ASTStringifier final : public ASTVisitor { return static_cast(v->as()->get_identifier()->name); case ast_type_alias_declaration: return "type " + static_cast(v->as()->get_identifier()->name); + case ast_struct_field: + return static_cast(v->as()->get_identifier()->name) + ": " + v->as()->declared_type->as_human_readable(); + case ast_struct_declaration: + return "struct " + static_cast(v->as()->get_identifier()->name); case ast_assign: return "="; case ast_set_assign: @@ -224,6 +234,10 @@ class ASTStringifier final : public ASTVisitor { return "(expression)"; } return "(else)"; + case ast_object_field: + return static_cast(v->as()->get_field_name()); + case ast_object_literal: + return "↓" + std::to_string(v->as()->get_body()->get_num_fields()); case ast_tolk_required_version: return static_cast(v->as()->semver); case ast_import_directive: @@ -289,6 +303,9 @@ class ASTStringifier final : public ASTVisitor { case ast_not_null_operator: return handle_vertex(v->as()); case ast_match_expression: return handle_vertex(v->as()); case ast_match_arm: return handle_vertex(v->as()); + case ast_object_field: return handle_vertex(v->as()); + case ast_object_body: return handle_vertex(v->as()); + case ast_object_literal: return handle_vertex(v->as()); // statements case ast_empty_statement: return handle_vertex(v->as()); case ast_block_statement: return handle_vertex(v->as()); @@ -313,6 +330,9 @@ class ASTStringifier final : public ASTVisitor { case ast_global_var_declaration: return handle_vertex(v->as()); case ast_constant_declaration: return handle_vertex(v->as()); case ast_type_alias_declaration: return handle_vertex(v->as()); + case ast_struct_field: return handle_vertex(v->as()); + case ast_struct_body: return handle_vertex(v->as()); + case ast_struct_declaration: return handle_vertex(v->as()); case ast_tolk_required_version: return handle_vertex(v->as()); case ast_import_directive: return handle_vertex(v->as()); case ast_tolk_file: return handle_vertex(v->as()); diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index eac7e98ef7..256f8d8869 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -118,6 +118,9 @@ class ASTVisitorFunctionBody : public ASTVisitor { virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } // statements virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } @@ -160,6 +163,9 @@ class ASTVisitorFunctionBody : public ASTVisitor { case ast_not_null_operator: return visit(v->as()); case ast_match_expression: return visit(v->as()); case ast_match_arm: return visit(v->as()); + case ast_object_field: return visit(v->as()); + case ast_object_body: return visit(v->as()); + case ast_object_literal: return visit(v->as()); // statements case ast_empty_statement: return visit(v->as()); case ast_block_statement: return visit(v->as()); diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 9fadd5f6c7..083abca0a6 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -176,6 +176,18 @@ void Vertex::assign_resolved_type(TypePtr underlying this->underlying_type = underlying_type; } +void Vertex::assign_field_ref(StructFieldPtr field_ref) { + this->field_ref = field_ref; +} + +void Vertex::assign_resolved_type(TypePtr declared_type) { + this->declared_type = declared_type; +} + +void Vertex::assign_struct_ref(StructPtr struct_ref) { + this->struct_ref = struct_ref; +} + void Vertex::assign_resolved_type(TypePtr substituted_type) { this->substituted_type = substituted_type; } @@ -208,6 +220,14 @@ void Vertex::assign_target(const DotTarget& target) { this->target = target; } +void Vertex::assign_field_ref(StructFieldPtr field_ref) { + this->field_ref = field_ref; +} + +void Vertex::assign_struct_ref(StructPtr struct_ref) { + this->struct_ref = struct_ref; +} + void Vertex::assign_fun_ref(FunctionPtr fun_ref) { this->fun_ref = fun_ref; } diff --git a/tolk/ast.h b/tolk/ast.h index 9dce458ecd..6531d50a87 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -93,6 +93,9 @@ enum ASTNodeType { ast_not_null_operator, ast_match_expression, ast_match_arm, + ast_object_field, + ast_object_body, + ast_object_literal, // statements ast_empty_statement, ast_block_statement, @@ -117,6 +120,9 @@ enum ASTNodeType { ast_global_var_declaration, ast_constant_declaration, ast_type_alias_declaration, + ast_struct_field, + ast_struct_body, + ast_struct_declaration, ast_tolk_required_version, ast_import_directive, ast_tolk_file, @@ -562,11 +568,13 @@ struct Vertex final : ASTExprUnary { typedef std::variant< FunctionPtr, // for `t.tupleAt` target is `tupleAt` global function + StructFieldPtr, // for `user.id` target is field `id` of struct `User` int // for `t.0` target is "indexed access" 0 > DotTarget; - DotTarget target = static_cast(nullptr); // filled at type inferring + DotTarget target = static_cast(nullptr); // filled at type inferring bool is_target_fun_ref() const { return std::holds_alternative(target); } + bool is_target_struct_field() const { return std::holds_alternative(target); } bool is_target_indexed_access() const { return std::holds_alternative(target); } AnyExprV get_obj() const { return child; } @@ -780,6 +788,59 @@ struct Vertex final : ASTExprBinary { , pattern_kind(pattern_kind), exact_type(exact_type) {} }; +template<> +// ast_object_field is one field at object creation +// example: `Point { x: 2, y: 3 }` is object creation, its body contains 2 fields +struct Vertex final : ASTExprUnary { +private: + V identifier; + +public: + StructFieldPtr field_ref = nullptr; // assigned at type inferring + + AnyExprV get_init_val() const { return child; } + std::string_view get_field_name() const { return identifier->name; } + V get_field_identifier() const { return identifier; } + + Vertex* mutate() const { return const_cast(this); } + void assign_field_ref(StructFieldPtr field_ref); + + Vertex(SrcLocation loc, V name_identifier, AnyExprV init_val) + : ASTExprUnary(ast_object_field, loc, init_val) + , identifier(name_identifier) {} +}; + +template<> +// ast_object_body is `{ ... }` inside object initialization, it contains fields +// examples: see below +struct Vertex final : ASTExprVararg { + int get_num_fields() const { return size(); } + auto get_field(int i) const { return child(i)->as(); } + std::vector get_all_fields() const { return children; } + + Vertex(SrcLocation loc, std::vector&& fields) + : ASTExprVararg(ast_object_body, loc, std::move(fields)) {} +}; + +template<> +// ast_object_literal is creating an instance of a struct with initial values of fields, like objects in TypeScript +// example: `Point { ... }` (object creation has explicit ref and body) +// example: `var v: Point = { ... }` (object creation has only body, struct_ref is determined from the left) +struct Vertex final : ASTExprBinary { + StructPtr struct_ref = nullptr; // assigned at type inferring + + bool has_explicit_ref() const { return lhs->type != ast_empty_expression; } + auto get_explicit_ref() const { return lhs->as(); } + AnyExprV get_maybe_reference() const { return lhs; } // ast_empty_expression or ast_reference + auto get_body() const { return rhs->as(); } + + Vertex* mutate() const { return const_cast(this); } + void assign_struct_ref(StructPtr struct_ref); + + Vertex(SrcLocation loc, AnyExprV maybe_reference, V body) + : ASTExprBinary(ast_object_literal, loc, maybe_reference, body) {} +}; + // // --------------------------------------------------------- @@ -1127,6 +1188,52 @@ struct Vertex final : ASTOtherVararg { , underlying_type(underlying_type) {} }; +template<> +// ast_struct_field is one field at struct declaration +// example: `struct Point { x: int, y: int }` is struct declaration, its body contains 2 fields +struct Vertex final : ASTOtherVararg { + StructFieldPtr field_ref = nullptr; // filled after register + TypePtr declared_type; + + auto get_identifier() const { return children.at(0)->as(); } + + Vertex* mutate() const { return const_cast(this); } + void assign_field_ref(StructFieldPtr field_ref); + void assign_resolved_type(TypePtr declared_type); + + Vertex(SrcLocation loc, V name_identifier, TypePtr declared_type) + : ASTOtherVararg(ast_struct_field, loc, {name_identifier}) + , declared_type(declared_type) {} +}; + +template<> +// ast_struct_body is `{ ... }` inside struct declaration, it contains fields +// example: `struct Storage { owner: User; validUntil: int }` its body contains 2 fields +struct Vertex final : ASTOtherVararg { + int get_num_fields() const { return size(); } + auto get_field(int i) const { return children.at(i)->as(); } + + Vertex(SrcLocation loc, std::vector&& fields) + : ASTOtherVararg(ast_struct_body, loc, std::move(fields)) {} +}; + +template<> +// ast_struct_declaration is declaring a struct with fields (each having declared_type), like interfaces in TypeScript +// example: `struct Storage { owner: User; validUntil: int }` +// currently, Tolk doesn't have "implements" or whatever, so struct declaration contains only body +struct Vertex final : ASTOtherVararg { + StructPtr struct_ref = nullptr; // filled after register + + auto get_identifier() const { return children.at(0)->as(); } + auto get_struct_body() const { return children.at(1)->as(); } + + Vertex* mutate() const { return const_cast(this); } + void assign_struct_ref(StructPtr struct_ref); + + Vertex(SrcLocation loc, V name_identifier, V struct_body) + : ASTOtherVararg(ast_struct_declaration, loc, {name_identifier, struct_body}) {} +}; + template<> // ast_tolk_required_version is a preamble fixating compiler's version at the top of the file // example: `tolk 0.6` diff --git a/tolk/fwd-declarations.h b/tolk/fwd-declarations.h index 7255a85884..ff2e32a367 100644 --- a/tolk/fwd-declarations.h +++ b/tolk/fwd-declarations.h @@ -32,12 +32,16 @@ struct FunctionData; struct GlobalVarData; struct GlobalConstData; struct AliasDefData; +struct StructFieldData; +struct StructData; using LocalVarPtr = const LocalVarData*; using FunctionPtr = const FunctionData*; using GlobalVarPtr = const GlobalVarData*; using GlobalConstPtr = const GlobalConstData*; using AliasDefPtr = const AliasDefData*; +using StructFieldPtr = const StructFieldData*; +using StructPtr = const StructData*; class TypeData; using TypePtr = const TypeData*; diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 3778b8cab6..8f4a8ddab1 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -100,6 +100,23 @@ class VarsModificationWatcher { static VarsModificationWatcher vars_modification_watcher; +static int calc_offset_on_stack(const TypeDataTensor* t_tensor, int index_at) { + int stack_offset = 0; + for (int i = 0; i < index_at; ++i) { + stack_offset += t_tensor->items[i]->get_width_on_stack(); + } + return stack_offset; +} + +static int calc_offset_on_stack(const TypeDataStruct* t_struct, int field_idx) { + int stack_offset = 0; + for (int i = 0; i < field_idx; ++i) { + stack_offset += t_struct->struct_ref->fields[i]->declared_type->get_width_on_stack(); + } + return stack_offset; +} + + // Main goal of LValContext is to handle non-primitive lvalues. At IR level, a usual local variable // exists, but on its change, something non-trivial should happen. // Example: `globalVar = 9` actually does `Const $5 = 9` + `Let $6 = $5` + `SetGlob "globVar" = $6` @@ -143,20 +160,25 @@ class LValContext { // example: `global v: (int, int); v.1 = 5`, implicit var is created `$tmp = 5`, and when it's modified, // we need to partially update w; essentially, apply_partially_rewrite() above will be called struct ModifiedFieldOfGlobal { - AnyExprV tensor_obj; - int index_at; + AnyExprV tensor_obj; // it's a tensor or struct + int index_at; // for tensors, it's index_at; for structs, it's field_idx std::vector lval_ir_idx; void apply(CodeBlob& code, SrcLocation loc) const { LValContext local_lval; local_lval.enter_rval_inside_lval(); std::vector obj_ir_idx = pre_compile_expr(tensor_obj, code, nullptr, &local_lval); - const TypeDataTensor* t_tensor = tensor_obj->inferred_type->unwrap_alias()->try_as(); - tolk_assert(t_tensor); - int stack_width = t_tensor->items[index_at]->get_width_on_stack(); - int stack_offset = 0; - for (int i = 0; i < index_at; ++i) { - stack_offset += t_tensor->items[i]->get_width_on_stack(); + + int stack_width, stack_offset; + TypePtr obj_type = tensor_obj->inferred_type->unwrap_alias(); + if (const TypeDataTensor* t_tensor = obj_type->try_as()) { + stack_width = t_tensor->items[index_at]->get_width_on_stack(); + stack_offset = calc_offset_on_stack(t_tensor, index_at); + } else if (const TypeDataStruct* t_struct = obj_type->try_as()) { + stack_width = t_struct->struct_ref->get_field(index_at)->declared_type->get_width_on_stack(); + stack_offset = calc_offset_on_stack(t_struct, index_at); + } else { + tolk_assert(false); } std::vector field_ir_idx = {obj_ir_idx.begin() + stack_offset, obj_ir_idx.begin() + stack_offset + stack_width}; tolk_assert(field_ir_idx.size() == lval_ir_idx.size()); @@ -270,7 +292,7 @@ static AnyExprV unwrap_not_null_operator(AnyExprV v) { static V calc_sink_leftmost_obj(V v) { AnyExprV leftmost_obj = unwrap_not_null_operator(v->get_obj()); while (auto v_dot = leftmost_obj->try_as()) { - if (!v_dot->is_target_indexed_access()) { + if (!v_dot->is_target_indexed_access() && !v_dot->is_target_struct_field()) { break; } leftmost_obj = unwrap_not_null_operator(v_dot->get_obj()); @@ -350,13 +372,15 @@ static std::vector> pre_compile_tensor_inner(CodeBlob& co } static std::vector pre_compile_tensor(CodeBlob& code, const std::vector& args, - LValContext* lval_ctx = nullptr) { - std::vector types_list; - types_list.reserve(args.size()); - for (AnyExprV item : args) { - types_list.push_back(item->inferred_type); + LValContext* lval_ctx = nullptr, const TypeDataTensor* tensor_target_type = nullptr) { + if (tensor_target_type == nullptr) { + std::vector types_list; + types_list.reserve(args.size()); + for (AnyExprV item : args) { + types_list.push_back(item->inferred_type); + } + tensor_target_type = TypeDataTensor::create(std::move(types_list))->try_as(); } - const TypeDataTensor* tensor_target_type = TypeDataTensor::create(std::move(types_list))->try_as(); std::vector> res_lists = pre_compile_tensor_inner(code, args, tensor_target_type, lval_ctx); std::vector res; for (const std::vector& list : res_lists) { @@ -489,10 +513,8 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectoror_null->get_width_on_stack() == 1) { - // a generalized union type "T | ..." takes at least 2 slots, but "T | null" can be 1 - tolk_assert(t_union->or_null); + // in general, pass `T1` to `T2?` when `T2?` still occupies 1 stack slot (value or TVM NULL) + if (t_union && t_union->is_primitive_nullable() && orig_w == 1) { // rvect has 1 slot, either value or TVM NULL return rvect; } @@ -502,8 +524,8 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectoror_null->get_width_on_stack() == 1) { - tolk_assert(o_union->or_null); + if (o_union && o_union->is_primitive_nullable() && target_w == 1) { + // rvect has 1 slot, but its contents is compile-time guaranteed to match target_type return rvect; } @@ -518,7 +540,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector 0 && t_union->or_null->get_width_on_stack() == 1) { + if (t_union && t_union->is_primitive_nullable() && orig_w > 0) { // nothing except "T1 | T2 | ... null" can be cast to 1-slot nullable `T1?` tolk_assert(o_union && o_union->has_null() && o_union->has_variant_with_type_id(t_union->or_null)); // here we exploit rvect shape, how union types and multi-slot nullables are stored on a stack @@ -527,7 +549,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector transition_expr_to_runtime_type_impl(std::vectoror_null && t_union->or_null->get_width_on_stack() == 0); + tolk_assert(t_union->has_null() && target_w == 1); std::vector new_rvect = code.create_tmp_var(TypeDataInt::create(), loc, "(UTag)"); code.emplace_back(loc, Op::_IntConst, new_rvect, td::make_refint(0)); return new_rvect; @@ -569,10 +592,11 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectoror_null); + tolk_assert(orig_w == 1 && o_union->has_null()); FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); std::vector new_rvect = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); code.emplace_back(loc, Op::_Call, new_rvect, std::vector{}, null_sym); @@ -583,7 +607,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectoror_null->get_width_on_stack() == 1) { + if (o_union && o_union->is_primitive_nullable() && t_union) { tolk_assert(t_union->has_null() && t_union->has_variant_with_type_id(o_union->or_null) && target_w > 1); // the transformation is tricky: // when value is null, we need to achieve "... (null) 0" (value is already null, so "... value 0") @@ -1089,10 +1113,31 @@ static std::vector process_dot_access(V v, CodeBlob& // it's `t.0`, `getUser().id`, and `t.tupleSize` (as a reference, not as a call) if (!v->is_target_fun_ref()) { TypePtr obj_type = v->get_obj()->inferred_type->unwrap_alias(); - int index_at = std::get(v->target); + // `user.id`; internally, a struct (an object) is a tensor + if (const auto* t_struct = obj_type->try_as()) { + StructFieldPtr field_ref = std::get(v->target); + // handle `globalObj.field = rhs`, special case, then the global will be read on demand + if (lval_ctx && !lval_ctx->is_rval_inside_lval()) { + if (auto sink = calc_sink_leftmost_obj(v); sink && sink->sym->try_as()) { + std::vector lval_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(lval-global-field)"); + lval_ctx->capture_field_of_global_modification(v->get_obj(), field_ref->field_idx, lval_ir_idx); + return lval_ir_idx; + } + } + std::vector lhs_vars = pre_compile_expr(v->get_obj(), code, nullptr, lval_ctx); + int stack_width = field_ref->declared_type->get_width_on_stack(); + int stack_offset = calc_offset_on_stack(t_struct, field_ref->field_idx); + std::vector rvect{lhs_vars.begin() + stack_offset, lhs_vars.begin() + stack_offset + stack_width}; + // an object field might be smart cast at this point, for example we're in `if (user.t != null)` + // it means that we must drop the null flag (if `user.t` is a tensor), or maybe perform other stack transformations + // (from original rvect = (vars of user.t) to fit smart cast) + rvect = transition_to_target_type(std::move(rvect), code, field_ref->declared_type, v->inferred_type, v->loc); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } // `tensorVar.0` if (const auto* t_tensor = obj_type->try_as()) { - // handle `tensorVar.0 = rhs` if tensors is a global, special case, then the global will be read on demand + int index_at = std::get(v->target); + // handle `globalTensorVar.0 = rhs`, special case, then the global will be read on demand if (lval_ctx && !lval_ctx->is_rval_inside_lval()) { if (auto sink = calc_sink_leftmost_obj(v); sink && sink->sym->try_as()) { std::vector lval_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(lval-global-tensor)"); @@ -1103,10 +1148,7 @@ static std::vector process_dot_access(V v, CodeBlob& // since a tensor of N elems are N vars on a stack actually, calculate offset std::vector lhs_vars = pre_compile_expr(v->get_obj(), code, nullptr, lval_ctx); int stack_width = t_tensor->items[index_at]->get_width_on_stack(); - int stack_offset = 0; - for (int i = 0; i < index_at; ++i) { - stack_offset += t_tensor->items[i]->get_width_on_stack(); - } + int stack_offset = calc_offset_on_stack(t_tensor, index_at); std::vector rvect{lhs_vars.begin() + stack_offset, lhs_vars.begin() + stack_offset + stack_width}; // a tensor index might be smart cast at this point, for example we're in `if (t.1 != null)` // it means that we must drop the null flag (if `t.1` is a tensor), or maybe perform other stack transformations @@ -1116,6 +1158,7 @@ static std::vector process_dot_access(V v, CodeBlob& } // `tupleVar.0` if (obj_type->try_as() || obj_type->try_as()) { + int index_at = std::get(v->target); // handle `tupleVar.0 = rhs`, "0 SETINDEX" will be called when this was is modified if (lval_ctx && !lval_ctx->is_rval_inside_lval() && calc_sink_leftmost_obj(v)) { std::vector lval_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(lval-tuple-field)"); @@ -1281,6 +1324,32 @@ static std::vector process_typed_tuple(V v, CodeBlob return transition_to_target_type(std::move(left), code, target_type, v); } +static std::vector process_object_literal(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + // an object (an instance of a struct) is actually a tensor at low-level + // for example, `struct User { id: int; name: slice; }` occupies 2 slots + // fields of a tensor are placed in order of declaration (in a literal they might be shuffled) + std::vector tensor_items; + std::vector target_types; + tensor_items.reserve(v->struct_ref->get_num_fields()); + target_types.reserve(v->struct_ref->get_num_fields()); + for (StructFieldPtr field_ref : v->struct_ref->fields) { + AnyExprV v_init_val = nullptr; + for (int i = 0; i < v->get_body()->get_num_fields(); ++i) { + auto v_field = v->get_body()->get_field(i); + if (v_field->get_field_name() == field_ref->name) { + v_init_val = v_field->get_init_val(); + break; + } + } + tolk_assert(v_init_val); + tensor_items.push_back(v_init_val); + target_types.push_back(field_ref->declared_type); + } + const auto* tensor_target_type = TypeDataTensor::create(std::move(target_types))->try_as(); + std::vector rvect = pre_compile_tensor(code, tensor_items, lval_ctx, tensor_target_type); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + static std::vector process_int_const(V v, CodeBlob& code, TypePtr target_type) { std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(int-const)"); code.emplace_back(v->loc, Op::_IntConst, rvect, v->intval); @@ -1370,6 +1439,8 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ return process_tensor(v->as(), code, target_type, lval_ctx); case ast_typed_tuple: return process_typed_tuple(v->as(), code, target_type, lval_ctx); + case ast_object_literal: + return process_object_literal(v->as(), code, target_type, lval_ctx); case ast_int_const: return process_int_const(v->as(), code, target_type); case ast_string_const: diff --git a/tolk/pipe-calc-rvalue-lvalue.cpp b/tolk/pipe-calc-rvalue-lvalue.cpp index b910eface6..b00a766965 100644 --- a/tolk/pipe-calc-rvalue-lvalue.cpp +++ b/tolk/pipe-calc-rvalue-lvalue.cpp @@ -226,6 +226,25 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { parent::visit(v); } + void visit(V v) override { + tolk_assert(cur_state == MarkingState::RValue); + mark_vertex(v); + parent::visit(v); + } + + void visit(V v) override { + tolk_assert(cur_state == MarkingState::RValue); + mark_vertex(v); + parent::visit(v); + } + + void visit(V v) override { + mark_vertex(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + void visit(V v) override { tolk_assert(cur_state == MarkingState::LValue); mark_vertex(v); diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 4b0f8d53f5..46de607c83 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -128,6 +128,9 @@ static void handle_possible_compiler_internal_call(FunctionPtr cur_f, Vtry_as()) { return TypeDataAlias::create(alias_ref); } + if (StructPtr struct_ref = sym->try_as()) { + return struct_ref->struct_type; + } tolk_assert(false); } return child; @@ -468,6 +471,8 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (!lhs->inferred_type->can_rhs_be_assigned(rhs_type)) { if (lhs->try_as()) { fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to variable of type " + to_string(lhs)); + } else if (lhs->try_as()) { + fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to field of type " + to_string(lhs)); } else { fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to " + to_string(lhs)); } @@ -616,6 +621,14 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } + void visit(V v) override { + parent::visit(v->get_init_val()); + + if (!v->field_ref->declared_type->can_rhs_be_assigned(v->get_init_val()->inferred_type)) { + fire(cur_f, v->get_init_val()->loc, "can not assign " + to_string(v->get_init_val()) + " to field of type " + to_string(v->field_ref->declared_type)); + } + } + void visit(V v) override { parent::visit(v); diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp index 8860911276..fed6de13d4 100644 --- a/tolk/pipe-check-rvalue-lvalue.cpp +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -149,6 +149,31 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { } void visit(V v) override { + // check for `immutableVal.field = rhs` or any other mutation of an immutable tensor/tuple/object + // don't allow cheating like `((immutableVal!)).field = rhs` + if (v->is_lvalue) { + AnyExprV leftmost_obj = v->get_obj(); + while (true) { + if (auto as_dot = leftmost_obj->try_as()) { + leftmost_obj = as_dot->get_obj(); + } else if (auto as_par = leftmost_obj->try_as()) { + leftmost_obj = as_par->get_expr(); + } else if (auto as_cast = leftmost_obj->try_as()) { + leftmost_obj = as_cast->get_expr(); + } else if (auto as_nn = leftmost_obj->try_as()) { + leftmost_obj = as_nn->get_expr(); + } else { + break; + } + } + + if (auto as_ref = leftmost_obj->try_as()) { + if (LocalVarPtr var_ref = as_ref->sym->try_as(); var_ref && var_ref->is_immutable()) { + fire_error_modifying_immutable_variable(cur_f, leftmost_obj, var_ref); + } + } + } + // a reference to a method used as rvalue, like `var v = t.tupleAt` if (v->is_rvalue && v->is_target_fun_ref()) { validate_function_used_as_noncall(cur_f, v, std::get(v->target)); diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 0e1c5af983..9cca8a2921 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -120,6 +120,11 @@ static std::string to_string(FunctionPtr fun_ref) { return "`" + fun_ref->as_human_readable() + "`"; } +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(std::string_view string_view) { + return static_cast(string_view); +} + // fire a general error, just a wrapper over `throw` GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { @@ -247,7 +252,7 @@ class InferTypesAndCallsAndFieldsVisitor final { case ast_braced_expression: return infer_braced_expression(v->as(), std::move(flow), used_as_condition); case ast_reference: - return infer_reference(v->as(), std::move(flow), used_as_condition); + return infer_reference(v->as(), std::move(flow), used_as_condition, hint); case ast_dot_access: return infer_dot_access(v->as(), std::move(flow), used_as_condition, hint); case ast_function_call: @@ -260,6 +265,8 @@ class InferTypesAndCallsAndFieldsVisitor final { return infer_null_keyword(v->as(), std::move(flow), used_as_condition); case ast_match_expression: return infer_match_expression(v->as(), std::move(flow), used_as_condition, hint); + case ast_object_literal: + return infer_object_literal(v->as(), std::move(flow), used_as_condition, hint); case ast_underscore: return infer_underscore(v->as(), std::move(flow), used_as_condition, hint); case ast_empty_expression: @@ -685,7 +692,7 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } - ExprFlow infer_reference(V v, FlowContext&& flow, bool used_as_condition) { + ExprFlow infer_reference(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) const { if (LocalVarPtr var_ref = v->sym->try_as()) { TypePtr declared_or_smart_casted = flow.smart_cast_if_exists(SinkExpression(var_ref)); tolk_assert(declared_or_smart_casted != nullptr); // all local vars are presented in flow @@ -728,6 +735,9 @@ class InferTypesAndCallsAndFieldsVisitor final { } else if (AliasDefPtr alias_ref = v->sym->try_as()) { fire(cur_f, v->loc, "type `" + alias_ref->name + "` can not be used as a value"); + } else if (StructPtr struct_ref = v->sym->try_as()) { + fire(cur_f, v->loc, "struct `" + struct_ref->name + "` can not be used as a value"); + } else { tolk_assert(false); } @@ -790,8 +800,23 @@ class InferTypesAndCallsAndFieldsVisitor final { V v_instantiationTs = v->get_instantiationTs(); std::string_view field_name = v_ident->name; - // it can be indexed access (`tensorVar.0`, `tupleVar.1`) or a method (`t.tupleSize`) - // at first, check for indexed access + // check for field access (`user.id`), when obj is a struct + if (const TypeDataStruct* obj_struct = unwrapped_obj_type->try_as()) { + if (StructFieldPtr field_ref = obj_struct->struct_ref->find_field(field_name)) { + v->mutate()->assign_target(field_ref); + TypePtr inferred_type = field_ref->declared_type; + if (SinkExpression s_expr = extract_sink_expression_from_vertex(v)) { + if (TypePtr smart_casted = flow.smart_cast_if_exists(s_expr)) { + inferred_type = smart_casted; + } + } + assign_inferred_type(v, inferred_type); + return ExprFlow(std::move(flow), used_as_condition); + } + // if field_name doesn't exist, don't fire an error now — maybe, it's `user.globalFunction()` + } + + // check for indexed access (`tensorVar.0` / `tupleVar.1`) if (field_name[0] >= '0' && field_name[0] <= '9') { int index_at = std::stoi(std::string(field_name)); if (const auto* t_tensor = unwrapped_obj_type->try_as()) { @@ -836,14 +861,28 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, item_type); return ExprFlow(std::move(flow), used_as_condition); } - fire(cur_f, v_ident->loc, "type " + to_string(obj_type) + " is not indexable"); + // numeric field not found, fire an error + if (unwrapped_obj_type->try_as()) { // `user.100500` + fire(cur_f, v_ident->loc, "field `" + static_cast(field_name) + "` doesn't exist in type " + to_string(obj_type)); + } else { + fire(cur_f, v_ident->loc, "type " + to_string(obj_type) + " is not indexable"); + } } - // for now, Tolk doesn't have fields and object-scoped methods; `t.tupleSize` is a global function `tupleSize` + // check for method (`t.tupleSize` / `user.globalFunction`) + // for now, Tolk doesn't have object-scoped methods; `t.tupleSize` is a global function `tupleSize` const Symbol* sym = lookup_global_symbol(field_name); FunctionPtr fun_ref = sym ? sym->try_as() : nullptr; + + // not a field, not a method — fire an error if (!fun_ref) { - fire(cur_f, v_ident->loc, "non-existing field `" + static_cast(field_name) + "` of type " + to_string(obj_type)); + // as a special case, handle accessing fields of nullable objects, to show a more precise error + if (const TypeDataUnion* as_union = unwrapped_obj_type->try_as(); as_union && as_union->or_null) { + if (const TypeDataStruct* n_struct = as_union->or_null->try_as(); n_struct && n_struct->struct_ref->find_field(field_name)) { + fire(cur_f, v_ident->loc, "can not access field `" + static_cast(field_name) + "` of a possibly nullable object " + to_string(obj_type) + "; check it via `obj != null` or use non-null assertion `obj!` operator"); + } + } + fire(cur_f, v_ident->loc, "field `" + static_cast(field_name) + "` doesn't exist in type " + to_string(obj_type)); } // `t.tupleSize` is ok, `cs.tupleSize` not @@ -896,7 +935,9 @@ class InferTypesAndCallsAndFieldsVisitor final { // it can be indexed access (`tensorVar.0()`, `tupleVar.1()`) or a method (`t.tupleSize()`) std::string_view field_name = v_dot->get_field_name(); - if (field_name[0] >= '0' && field_name[0] <= '9') { + if (const TypeDataStruct* t_struct = dot_obj->inferred_type->unwrap_alias()->try_as(); t_struct && t_struct->struct_ref->find_field(field_name)) { + // `t.someField`, it's a field, probably a callable, fun_ref remains nullptr + } else if (field_name[0] >= '0' && field_name[0] <= '9') { // indexed access `ab.2()`, then treat `ab.2` just like an expression, fun_ref remains nullptr // infer_dot_access() will be called for a callee, it will check index correctness } else { @@ -904,7 +945,7 @@ class InferTypesAndCallsAndFieldsVisitor final { const Symbol* sym = lookup_global_symbol(field_name); fun_ref = sym ? sym->try_as() : nullptr; if (!fun_ref) { - fire(cur_f, v_dot->get_identifier()->loc, "non-existing method `" + static_cast(field_name) + "` of type " + to_string(dot_obj)); + fire(cur_f, v_dot->get_identifier()->loc, "non-existing method `" + to_string(field_name) + "` of type " + to_string(dot_obj)); } } @@ -1112,6 +1153,76 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(match_out_flow), used_as_condition); } + ExprFlow infer_object_literal(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + StructPtr struct_ref = nullptr; + if (v->has_explicit_ref()) { + V v_ref = v->get_explicit_ref(); + struct_ref = v_ref->sym->try_as(); + if (!struct_ref) { + if (AliasDefPtr as_alias = v_ref->sym->try_as()) { + if (const TypeDataStruct* as_struct = as_alias->underlying_type->unwrap_alias()->try_as()) { + struct_ref = as_struct->struct_ref; + } + } + } + if (!struct_ref) { + fire(cur_f, v_ref->loc, "`" + to_string(v_ref->get_name()) + "` does not name a struct"); + } + if (UNLIKELY(v_ref->has_instantiationTs())) { + fire(cur_f, v_ref->get_instantiationTs()->loc, "generic T not expected here"); + } + } + if (!struct_ref && hint) { + if (const TypeDataStruct* hint_struct = hint->unwrap_alias()->try_as()) { + struct_ref = hint_struct->struct_ref; + } + if (const TypeDataUnion* hint_union = hint->unwrap_alias()->try_as()) { + StructPtr last_struct = nullptr; + int n_structs = 0; + for (TypePtr variant : hint_union->variants) { + if (const TypeDataStruct* variant_struct = variant->unwrap_alias()->try_as()) { + last_struct = variant_struct->struct_ref; + n_structs++; + } + } + if (n_structs == 1) { + struct_ref = last_struct; + } + } + } + if (!struct_ref) { + fire(cur_f, v->loc, "can not detect struct name; use either `var v: StructName = { ... }` or `var v = StructName { ... }`"); + } + + uint64_t occurred_mask = 0; + for (int i = 0; i < v->get_body()->get_num_fields(); ++i) { + auto v_field = v->get_body()->get_field(i); + std::string_view field_name = v_field->get_field_name(); + StructFieldPtr field_ref = struct_ref->find_field(field_name); + if (!field_ref) { + fire(cur_f, v_field->loc, "field `" + to_string(field_name) + "` not found in struct `" + struct_ref->name + "`"); + } + v_field->mutate()->assign_field_ref(field_ref); + + if (occurred_mask & (1ULL << field_ref->field_idx)) { + fire(cur_f, v_field->loc, "duplicate field initialization"); + } + occurred_mask |= 1ULL << field_ref->field_idx; + flow = infer_any_expr(v_field->get_init_val(), std::move(flow), false, field_ref->declared_type).out_flow; + assign_inferred_type(v_field, v_field->get_init_val()); + } + for (StructFieldPtr field_ref : struct_ref->fields) { + if (!(occurred_mask & (1ULL << field_ref->field_idx))) { + fire(cur_f, v->get_body()->loc, "field `" + field_ref->name + "` missed in initialization of struct `" + struct_ref->name + "`"); + } + } + + v->mutate()->assign_struct_ref(struct_ref); + assign_inferred_type(v, struct_ref->struct_type); + assign_inferred_type(v->get_body(), v); + return ExprFlow(std::move(flow), used_as_condition); + } + static ExprFlow infer_underscore(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { // if execution is here, underscore is either used as lhs of assignment, or incorrectly, like `f(_)` // more precise is to always set unknown here, but for incorrect usages, instead of an error diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 363952cbdc..7185471987 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -140,6 +140,24 @@ static void register_type_alias(V v) { v->mutate()->assign_alias_ref(a_sym); } +static void register_struct(V v) { + auto v_body = v->get_struct_body(); + + std::vector fields; + fields.reserve(v_body->get_num_fields()); + for (int i = 0; i < v_body->get_num_fields(); ++i) { + auto v_field = v_body->get_field(i); + StructFieldData* field_ref = new StructFieldData(static_cast(v_field->get_identifier()->name), v_field->loc, i, v_field->declared_type); + fields.emplace_back(field_ref); + v_field->mutate()->assign_field_ref(field_ref); + } + + StructData* s_sym = new StructData(static_cast(v->get_identifier()->name), v->loc, std::move(fields)); + + G.symtable.add_struct(s_sym); + v->mutate()->assign_struct_ref(s_sym); +} + static LocalVarData register_parameter(V v, int idx) { if (v->is_underscore()) { return {"", v->loc, v->declared_type, 0, idx}; @@ -248,6 +266,9 @@ static void iterate_through_file_symbols(const SrcFile* file) { case ast_type_alias_declaration: register_type_alias(v->as()); break; + case ast_struct_declaration: + register_struct(v->as()); + break; case ast_function_declaration: register_function(v->as()); break; diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 797795afd9..304a3497d1 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -160,6 +160,12 @@ struct TypeDataResolver { } return TypeDataAlias::create(alias_ref); } + if (StructPtr struct_ref = sym->try_as()) { + if (!struct_ref->struct_type) { + resolve_and_mutate_struct_and_fields(struct_ref); + } + return struct_ref->struct_type; + } } if (un->text == "auto") { throw ParseError(cur_f, un->loc, "`auto` type does not exist; just omit a type for local variable (will be inferred from assignment); parameters should always be typed"); @@ -187,6 +193,27 @@ struct TypeDataResolver { alias_ref->mutate()->assign_resolved_type(underlying_type); called_stack.pop_back(); } + + static void resolve_and_mutate_struct_and_fields(StructPtr struct_ref) { + static std::vector called_stack; + + // prevent recursion like `struct A { field: A }` + // currently, a struct is a tensor, and recursion always leads to infinite size (`A?` also, it's also on a stack) + // if there would be an annotation to store a struct in a tuple, then it has to be reconsidered + bool contains = std::find(called_stack.begin(), called_stack.end(), struct_ref) != called_stack.end(); + if (contains) { + throw ParseError(struct_ref->loc, "struct `" + struct_ref->name + "` size is infinity due to recursive fields"); + } + + called_stack.push_back(struct_ref); + for (int i = 0; i < struct_ref->get_num_fields(); ++i) { + StructFieldPtr field_ref = struct_ref->get_field(i); + TypePtr declared_type = finalize_type_data(nullptr, field_ref->declared_type, nullptr); + field_ref->mutate()->assign_resolved_type(declared_type); + } + struct_ref->mutate()->assign_resolved_type(TypeDataStruct::create(struct_ref)); + called_stack.pop_back(); + } }; static TypePtr finalize_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { @@ -280,7 +307,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { switch (v->pattern_kind) { case MatchArmKind::exact_type: { if (const TypeDataUnresolved* maybe_ident = v->exact_type->try_as()) { - if (const Symbol* sym = current_scope.lookup_symbol(maybe_ident->text); sym && !sym->try_as()) { + if (const Symbol* sym = current_scope.lookup_symbol(maybe_ident->text); sym && !sym->try_as() && !sym->try_as()) { auto v_ident = createV(v->loc, sym->name); AnyExprV pattern_expr = createV(v->loc, v_ident, nullptr); parent::visit(pattern_expr); @@ -425,6 +452,14 @@ void pipeline_resolve_identifiers_and_assign_symbols() { } else if (auto v_alias = v->try_as()) { TypeDataResolver::resolve_and_mutate_type_alias(v_alias->alias_ref); v_alias->mutate()->assign_resolved_type(v_alias->alias_ref->underlying_type); + + } else if (auto v_struct = v->try_as()) { + TypeDataResolver::resolve_and_mutate_struct_and_fields(v_struct->struct_ref); + auto v_body = v_struct->get_struct_body(); + for (int i = 0; i < v_body->get_num_fields(); ++i) { + auto v_field = v_body->get_field(i); + v_field->mutate()->assign_resolved_type(v_struct->struct_ref->get_field(i)->declared_type); + } } } } diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index 65ab7fb800..b4974ef489 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -98,9 +98,16 @@ namespace tolk { std::string SinkExpression::to_string() const { std::string result = var_ref->name; uint64_t cur_path = index_path; + TypePtr cur_type = var_ref->declared_type; while (cur_path != 0) { result += "."; - result += std::to_string((cur_path & 0xFF) - 1); + if (const TypeDataStruct* t_struct = cur_type->try_as()) { + StructFieldPtr field_ref = t_struct->struct_ref->get_field((cur_path & 0xFF) - 1); + result += field_ref->name; + cur_type = field_ref->declared_type; + } else { + result += std::to_string((cur_path & 0xFF) - 1); + } cur_path >>= 8; } return result; @@ -400,11 +407,13 @@ SinkExpression extract_sink_expression_from_vertex(AnyExprV v) { } } - if (auto as_dot = v->try_as(); as_dot && as_dot->is_target_indexed_access()) { + if (auto as_dot = v->try_as()) { V cur_dot = as_dot; uint64_t index_path = 0; - while (cur_dot->is_target_indexed_access()) { - int index_at = std::get(cur_dot->target); + while (cur_dot->is_target_indexed_access() || cur_dot->is_target_struct_field()) { + int index_at = cur_dot->is_target_indexed_access() + ? std::get(cur_dot->target) + : std::get(cur_dot->target)->field_idx; index_path = (index_path << 8) + index_at + 1; if (auto parent_dot = unwrap_not_null_operator(cur_dot->get_obj())->try_as()) { cur_dot = parent_dot; @@ -413,7 +422,7 @@ SinkExpression extract_sink_expression_from_vertex(AnyExprV v) { } } if (auto as_ref = unwrap_not_null_operator(cur_dot->get_obj())->try_as()) { - if (LocalVarPtr var_ref = as_ref->sym->try_as()) { + if (LocalVarPtr var_ref = as_ref->sym->try_as(); var_ref && index_path) { return SinkExpression(var_ref, index_path); } } @@ -448,14 +457,20 @@ TypePtr calc_declared_type_before_smart_cast(AnyExprV v) { } } - if (auto as_dot = v->try_as(); as_dot && as_dot->is_target_indexed_access()) { + if (auto as_dot = v->try_as()) { TypePtr obj_type = as_dot->get_obj()->inferred_type->unwrap_alias(); // v already inferred; hence, index_at is correct - int index_at = std::get(as_dot->target); - if (const auto* t_tensor = obj_type->try_as()) { - return t_tensor->items[index_at]; + if (as_dot->is_target_struct_field()) { + StructFieldPtr field_ref = std::get(as_dot->target); + return field_ref->declared_type; } - if (const auto* t_tuple = obj_type->try_as()) { - return t_tuple->items[index_at]; + if (as_dot->is_target_indexed_access()) { + int index_at = std::get(as_dot->target); + if (const auto* t_tensor = obj_type->try_as()) { + return t_tensor->items[index_at]; + } + if (const auto* t_tuple = obj_type->try_as()) { + return t_tuple->items[index_at]; + } } } diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 4942fe8701..ccd56a011e 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -117,6 +117,23 @@ void AliasDefData::assign_resolved_type(TypePtr underlying_type) { this->underlying_type = underlying_type; } +void StructFieldData::assign_resolved_type(TypePtr declared_type) { + this->declared_type = declared_type; +} + +void StructData::assign_resolved_type(TypePtr struct_type) { + this->struct_type = struct_type; +} + +StructFieldPtr StructData::find_field(std::string_view field_name) const { + for (StructFieldPtr field : fields) { + if (field->name == field_name) { + return field; + } + } + return nullptr; +} + GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_redefinition_of_symbol(SrcLocation loc, const Symbol* previous) { SrcLocation prev_loc = previous->loc; @@ -161,6 +178,14 @@ void GlobalSymbolTable::add_type_alias(AliasDefPtr a_sym) { } } +void GlobalSymbolTable::add_struct(StructPtr s_sym) { + auto key = key_hash(s_sym->name); + auto [it, inserted] = entries.emplace(key, s_sym); + if (!inserted) { + fire_error_redefinition_of_symbol(s_sym->loc, it->second); + } +} + const Symbol* lookup_global_symbol(std::string_view name) { return G.symtable.lookup(name); } diff --git a/tolk/symtable.h b/tolk/symtable.h index a27aecd44d..269aee90fb 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -228,6 +228,37 @@ struct AliasDefData final : Symbol { void assign_resolved_type(TypePtr underlying_type); }; +struct StructFieldData final : Symbol { + int field_idx; + TypePtr declared_type; + + StructFieldData* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr declared_type); + + StructFieldData(std::string name, SrcLocation loc, int field_idx, TypePtr declared_type) + : Symbol(std::move(name), loc) + , field_idx(field_idx) + , declared_type(declared_type) { + } +}; + +struct StructData final : Symbol { + std::vector fields; + const TypeData* struct_type = nullptr; // it's TypeDataStruct, assigned at resolve identifiers + + int get_num_fields() const { return static_cast(fields.size()); } + StructFieldPtr get_field(int i) const { return fields.at(i); } + StructFieldPtr find_field(std::string_view field_name) const; + + StructData* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr struct_type); + + StructData(std::string name, SrcLocation loc, std::vector&& fields) + : Symbol(std::move(name), loc) + , fields(std::move(fields)) { + } +}; + class GlobalSymbolTable { std::unordered_map entries; @@ -240,6 +271,7 @@ class GlobalSymbolTable { void add_global_var(GlobalVarPtr g_sym); void add_global_const(GlobalConstPtr c_sym); void add_type_alias(AliasDefPtr a_sym); + void add_struct(StructPtr s_sym); const Symbol* lookup(std::string_view name) const { const auto it = entries.find(key_hash(name)); diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index a63d555c1d..402b79b387 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -218,6 +218,20 @@ TypePtr TypeDataGenericT::create(std::string&& nameT) { return hash.register_unique(new TypeDataGenericT(std::move(nameT))); } +TypePtr TypeDataStruct::create(StructPtr struct_ref) { + TypeDataHasherForUnique hash(8315986401043319583ULL); + hash.feed_string(struct_ref->name); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + int width_on_stack = 0; + for (StructFieldPtr field_ref : struct_ref->fields) { + width_on_stack += field_ref->declared_type->get_width_on_stack(); + } + return hash.register_unique(new TypeDataStruct(width_on_stack, struct_ref)); +} + TypePtr TypeDataTensor::create(std::vector&& items) { TypeDataHasherForUnique hash(3159238551239480381ULL); for (TypePtr item : items) { @@ -385,6 +399,10 @@ int TypeDataGenericT::get_type_id() const { return TypeIdCalculation::assign_type_id(this); } +int TypeDataStruct::get_type_id() const { + return TypeIdCalculation::assign_type_id(this); +} + int TypeDataTensor::get_type_id() const { assert(!has_genericT_inside()); return TypeIdCalculation::assign_type_id(this); @@ -451,6 +469,10 @@ std::string TypeDataFunCallable::as_human_readable() const { return result; } +std::string TypeDataStruct::as_human_readable() const { + return struct_ref->name; +} + std::string TypeDataTensor::as_human_readable() const { std::string result = "("; for (TypePtr item : items) { @@ -731,6 +753,16 @@ bool TypeDataGenericT::can_rhs_be_assigned(TypePtr rhs) const { return false; } +bool TypeDataStruct::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } + return rhs == TypeDataNever::create(); +} + bool TypeDataTensor::can_rhs_be_assigned(TypePtr rhs) const { if (const auto* as_tensor = rhs->try_as(); as_tensor && as_tensor->size() == size()) { for (int i = 0; i < size(); ++i) { @@ -986,6 +1018,16 @@ bool TypeDataGenericT::can_be_casted_with_as_operator(TypePtr cast_to) const { return true; } +bool TypeDataStruct::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + return cast_to == this; +} + bool TypeDataTensor::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_tensor = cast_to->try_as(); to_tensor && to_tensor->size() == size()) { for (int i = 0; i < size(); ++i) { @@ -1108,6 +1150,18 @@ bool TypeDataAlias::can_hold_tvm_null_instead() const { return underlying_type->can_hold_tvm_null_instead(); } +bool TypeDataStruct::can_hold_tvm_null_instead() const { + if (get_width_on_stack() != 1) { // example that can hold null: `{ field: int }` + return false; // another example: `{ e: Empty, field: ((), int) }` + } // examples can NOT: `{ field1: int, field2: int }`, `{ field1: int? }` + for (StructFieldPtr field : struct_ref->fields) { + if (field->declared_type->get_width_on_stack() == 1 && !field->declared_type->can_hold_tvm_null_instead()) { + return false; + } + } + return true; +} + bool TypeDataTensor::can_hold_tvm_null_instead() const { if (get_width_on_stack() != 1) { // `(int, int)` / `()` can not hold null instead, since null is 1 slot return false; // only `((), int)` and similar can: diff --git a/tolk/type-system.h b/tolk/type-system.h index 56fd02e5ce..6a6b1625aa 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -17,6 +17,7 @@ #pragma once #include "src-file.h" +#include "fwd-declarations.h" #include #include #include @@ -337,6 +338,28 @@ class TypeDataGenericT final : public TypeData { bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; +/* + * `A`, `User`, `SomeStruct` is TypeDataStruct. At TVM level, structs are tensors. + * In the code, creating a struct is either `var v: A = { ... }` (by hint) or `var v = A { ... }`. + * Fields of a struct have their own types (accessed by struct_ref). + */ +class TypeDataStruct final : public TypeData { + TypeDataStruct(int width_on_stack, StructPtr struct_ref) + : TypeData(0, width_on_stack) + , struct_ref(struct_ref) {} + +public: + StructPtr struct_ref; + + static TypePtr create(StructPtr struct_ref); + + int get_type_id() const override; + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + bool can_hold_tvm_null_instead() const override; +}; + /* * `(int, slice)` is TypeDataTensor of 2 elements. Tensor of N elements occupies N stack slots. * Of course, there may be nested tensors, like `(int, (int, slice), cell)`. From a01053789ae92d990892b20a057523eea556b1bb Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 17 Apr 2025 08:22:45 +0300 Subject: [PATCH 229/524] [Tolk] Support default values for fields, simplify constants Now fields can have default values, even complex ones: > tensor: (int, coins) = (2, ton("0.05")) Initial value must be a constant expression. Global constants can also now be of any type, including tensors and nullables. Constant evaluator was almost dropped off, evaluation is now done at IR (Ops) level. --- tolk-tester/tests/constants-tests.tolk | 21 +- .../tests/invalid-semantics/err-4190.tolk | 10 + .../tests/invalid-semantics/err-4270.tolk | 8 + .../tests/invalid-symbol/err-2098.tolk | 9 + .../tests/invalid-syntax/err-3150.tolk | 9 - .../tests/invalid-syntax/err-3650.tolk | 8 + .../tests/invalid-typing/err-6740.tolk | 9 + tolk-tester/tests/match-by-expr-tests.tolk | 4 +- tolk-tester/tests/struct-tests.tolk | 33 +++ tolk-tester/tests/use-before-declare.tolk | 2 +- tolk/ast-from-tokens.cpp | 8 +- tolk/ast-replacer.h | 1 + tolk/ast-visitor.h | 1 + tolk/ast.cpp | 2 +- tolk/ast.h | 10 +- tolk/compiler-state.cpp | 4 + tolk/compiler-state.h | 1 + tolk/constant-evaluator.cpp | 197 +++++------------- tolk/constant-evaluator.h | 34 +-- tolk/pipe-ast-to-legacy.cpp | 20 +- tolk/pipe-check-inferred-types.cpp | 17 ++ tolk/pipe-constant-folding.cpp | 49 +++-- tolk/pipe-infer-types-and-calls.cpp | 18 +- tolk/pipe-register-symbols.cpp | 8 +- tolk/pipe-resolve-identifiers.cpp | 6 + tolk/symtable.cpp | 8 +- tolk/symtable.h | 13 +- 27 files changed, 266 insertions(+), 244 deletions(-) create mode 100644 tolk-tester/tests/invalid-semantics/err-4190.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4270.tolk create mode 100644 tolk-tester/tests/invalid-symbol/err-2098.tolk delete mode 100644 tolk-tester/tests/invalid-syntax/err-3150.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3650.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6740.tolk diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index d5519b0ef9..128a5f7ea3 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -29,6 +29,12 @@ const true3 = true1 && true2; const false1 = !true; const false2 = false1 || false; +const tens1 = (1, 2); +const tens2: (int, int, (int8, int16)) = (tens1.1, tens1.0 << 2, tens1); + +const intOrN: int? = null; +const int32Or64: int32 | int64 = 7 as int64; + fun iget1(): int { return int1; } fun iget2(): int { return int2; } fun iget3(): int { return int1+int2; } @@ -71,6 +77,17 @@ fun test3() { return (false1, false2); } +@method_id(104) +fun test4() { + __expect_type(tens1, "(int, int)"); + return (tens1.0, tens2.2); +} + +@method_id(105) +fun test5() { + return (intOrN == null, int32Or64 is int32, int32Or64); +} + fun main() { var i1: int = iget1(); var i2: int = iget2(); @@ -98,6 +115,8 @@ fun main() { @testcase | 101 | | 0 -1 @testcase | 102 | | -1 -1 -1 @testcase | 103 | | 0 0 +@testcase | 104 | | 1 1 2 +@testcase | 105 | | -1 0 7 48 -@code_hash 28102194299745406750019953961984060488024870092664444642078578246708959881688 +@code_hash 80040709432962217077682091261201772251141677197885524779745956896218368868623 */ diff --git a/tolk-tester/tests/invalid-semantics/err-4190.tolk b/tolk-tester/tests/invalid-semantics/err-4190.tolk new file mode 100644 index 0000000000..70355bf983 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4190.tolk @@ -0,0 +1,10 @@ +struct A { + ok: coins = ton("0.05"), + err: int = now(), +} + +/** +@compilation_should_fail +@stderr not a constant expression +@stderr err: int + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4270.tolk b/tolk-tester/tests/invalid-semantics/err-4270.tolk new file mode 100644 index 0000000000..14aef95ca3 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4270.tolk @@ -0,0 +1,8 @@ +struct A { + tens: (int, int) = (1, 2 + now()), +} + +/** +@compilation_should_fail +@stderr not a constant expression + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2098.tolk b/tolk-tester/tests/invalid-symbol/err-2098.tolk new file mode 100644 index 0000000000..5c2b952bb0 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2098.tolk @@ -0,0 +1,9 @@ +struct A { + a: int = 1 << CCC; +} + +/** +@compilation_should_fail +@stderr undefined symbol `CCC` +@stderr 1 << CCC + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3150.tolk b/tolk-tester/tests/invalid-syntax/err-3150.tolk deleted file mode 100644 index 8828fcadff..0000000000 --- a/tolk-tester/tests/invalid-syntax/err-3150.tolk +++ /dev/null @@ -1,9 +0,0 @@ -struct A { - id: int = 3; -} - -/** -@compilation_should_fail -@stderr default values for fields not supported yet -@stderr id: int = 3 -*/ diff --git a/tolk-tester/tests/invalid-syntax/err-3650.tolk b/tolk-tester/tests/invalid-syntax/err-3650.tolk new file mode 100644 index 0000000000..5482e7b9c2 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3650.tolk @@ -0,0 +1,8 @@ +struct A { + v1 = 0, +} + +/** +@compilation_should_fail +@stderr expected `: `, got `=` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6740.tolk b/tolk-tester/tests/invalid-typing/err-6740.tolk new file mode 100644 index 0000000000..e3dd3a4706 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6740.tolk @@ -0,0 +1,9 @@ +struct A { + v1: int = "asdf"; +} + +/** +@compilation_should_fail +@stderr can not assign `slice` to `int` +@stderr v1: int = "asdf" + */ diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk index d9ecb3cfec..80287c9bbd 100644 --- a/tolk-tester/tests/match-by-expr-tests.tolk +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -155,12 +155,12 @@ fun main() {} 100 PUSHINT // '1=100 }>ELSE<{ // x DUP // x x - 2 EQINT // x '6 + 2 EQINT // x '8 IF:<{ // x DROP // 200 PUSHINT // '1=200 }>ELSE<{ // x - 3 EQINT // '9 + 3 EQINT // '13 IF:<{ // 300 PUSHINT // '1=300 }>ELSE<{ // diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 91dc005435..4a676a3db1 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -428,7 +428,38 @@ fun test31() { return (s1, s2, s3, 777, s1 == null, s2 == null, s3 == null, 777, s1!.x == null, s2!.x == null); } +struct PointDef0 { + x: int = 0, + y: int = 0, +} + +@method_id(132) +fun test32() { + var p1: PointDef0 = { x: 10, y: 20 }; + var p2: PointDef0 = { x: 10 }; + var p3: PointDef0 = { y: 20 }; + var p4: PointDef0 = { }; + return (p1, p2, p3, p4, PointDef0{}); +} +struct WithDefaults { + f1: (bool, int) = (true, 0), + f2: int, + f3: slice? = stringHexToSlice("010203"), + f4: PointDef0? = null, + f5: int32 | int64 = 0 as int32, +} + +@method_id(133) +fun test33(): WithDefaults { + var w1: WithDefaults = { f2: 0 }; + assert(w1.f1.0 && w1.f1.1 == 0 && w1.f3!.getRemainingBitsCount() == 24 && w1.f4 == null && w1.f5 is int32) throw 100; + var w2: WithDefaults? = { f1: (false, 55), f2: 10, f5: 8 as int64 }; + assert(w2.f1.0 != true && w2.f3!.getRemainingBitsCount() == 24 && w2.f4 == null && w2.f5 is int64 && w2.f5 == 8) throw 100; + var w3: (int, WithDefaults) = (0, { f2: 7, f4: {y: 20} }); + assert(w3.1.f4 != null && w3.1.f4.x == 0 && w3.1.f4.y == 20) throw 100; + return { f2: 5, f3: null }; +} @@ -480,6 +511,8 @@ type PointAlias = Point; @testcase | 130 | -1 | 140 777 (null) 140 777 0 777 0 0 @testcase | 130 | 0 | 139 777 4 1 777 139 777 0 0 @testcase | 131 | | 5 141 (null) 141 (null) 0 777 0 0 -1 777 0 -1 +@testcase | 132 | | 10 20 10 0 0 20 0 0 0 0 +@testcase | 133 | | -1 0 5 (null) (null) (null) 0 0 46 @fif_codegen """ diff --git a/tolk-tester/tests/use-before-declare.tolk b/tolk-tester/tests/use-before-declare.tolk index 9c5f063496..f444c48ef4 100644 --- a/tolk-tester/tests/use-before-declare.tolk +++ b/tolk-tester/tests/use-before-declare.tolk @@ -58,7 +58,7 @@ const first = 1; """ test2 PROC:<{ // - 2 PUSHINT // '0=2 + 2 PUSHINT // '2 }> """ */ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 6fc23457b9..939dd799c0 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -1275,11 +1275,15 @@ static AnyV parse_struct_field(Lexer& lex) { lex.expect(tok_colon, "`: `"); TypePtr declared_type = parse_type_from_tokens(lex); + AnyExprV default_value = nullptr; if (lex.tok() == tok_assign) { // `id: int = 3` - lex.error("default values for fields not supported yet"); + lex.next(); + default_value = parse_expr(lex); + } else { + default_value = createV(lex.cur_location()); } - return createV(loc, v_ident, declared_type); + return createV(loc, v_ident, default_value, declared_type); } static V parse_struct_body(Lexer& lex) { diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 7108b95b08..40cd8e4896 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -206,6 +206,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { const std::vector& get_all_not_builtin_functions(); const std::vector& get_all_declared_constants(); +const std::vector& get_all_declared_structs(); template void replace_ast_of_all_functions() { diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index 256f8d8869..12e65e2707 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -197,6 +197,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { const std::vector& get_all_not_builtin_functions(); const std::vector& get_all_declared_constants(); +const std::vector& get_all_declared_structs(); template void visit_ast_of_all_functions() { diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 083abca0a6..399947446e 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -126,7 +126,7 @@ void Vertex::assign_sym(const Symbol* sym) { this->sym = sym; } -void Vertex::assign_literal_value(ConstantValue&& literal_value) { +void Vertex::assign_literal_value(std::string&& literal_value) { this->literal_value = std::move(literal_value); } diff --git a/tolk/ast.h b/tolk/ast.h index 6531d50a87..71bc9c40b9 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -501,10 +501,10 @@ template<> // note, that TVM doesn't have strings, it has only slices, so "hello" has type slice struct Vertex final : ASTExprLeaf { std::string_view str_val; - ConstantValue literal_value; // value of type `slice`, calculated after type inferring, at constants evaluation + std::string literal_value; // when "some_str" is a standalone string, value of type `slice`, for x{...} Fift output Vertex* mutate() const { return const_cast(this); } - void assign_literal_value(ConstantValue&& literal_value); + void assign_literal_value(std::string&& literal_value); Vertex(SrcLocation loc, std::string_view str_val) : ASTExprLeaf(ast_string_const, loc) @@ -1196,13 +1196,15 @@ struct Vertex final : ASTOtherVararg { TypePtr declared_type; auto get_identifier() const { return children.at(0)->as(); } + bool has_default_value() const { return children.at(1)->type != ast_empty_expression; } + auto get_default_value() const { return child_as_expr(1); } Vertex* mutate() const { return const_cast(this); } void assign_field_ref(StructFieldPtr field_ref); void assign_resolved_type(TypePtr declared_type); - Vertex(SrcLocation loc, V name_identifier, TypePtr declared_type) - : ASTOtherVararg(ast_struct_field, loc, {name_identifier}) + Vertex(SrcLocation loc, V name_identifier, AnyExprV default_value, TypePtr declared_type) + : ASTOtherVararg(ast_struct_field, loc, {name_identifier, default_value}) , declared_type(declared_type) {} }; diff --git a/tolk/compiler-state.cpp b/tolk/compiler-state.cpp index e02d0220a5..f5bda29e6f 100644 --- a/tolk/compiler-state.cpp +++ b/tolk/compiler-state.cpp @@ -74,4 +74,8 @@ const std::vector& get_all_declared_constants() { return G.all_constants; } +const std::vector& get_all_declared_structs() { + return G.all_structs; +} + } // namespace tolk diff --git a/tolk/compiler-state.h b/tolk/compiler-state.h index 1d166a3ab3..b39e62fb3f 100644 --- a/tolk/compiler-state.h +++ b/tolk/compiler-state.h @@ -99,6 +99,7 @@ struct CompilerState { std::vector all_get_methods; std::vector all_global_vars; std::vector all_constants; + std::vector all_structs; AllRegisteredSrcFiles all_src_files; bool is_verbosity(int gt_eq) const { return settings.verbosity >= gt_eq; } diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index ff8dbbce18..a281cfbe31 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -137,16 +137,11 @@ static td::RefInt256 parse_nanotons_as_floating_string(SrcLocation loc, std::str return td::make_refint(is_negative ? -result : result); } -static ConstantValue parse_vertex_string_const(V v) { - td::Slice str_slice = td::Slice(v->str_val.data(), v->str_val.size()); - return ConstantValue(td::hex_encode(str_slice)); -} - // given `ton("0.05")` evaluate it to 50000000 // given `stringCrc32("some_str")` evaluate it // etc. // currently, all compile-time functions accept 1 argument, a literal string -static ConstantValue parse_vertex_call_to_compile_time_function(V v, std::string_view f_name) { +static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(V v, std::string_view f_name) { tolk_assert(v->get_num_args() == 1); // checked by type inferring AnyExprV v_arg = v->get_arg(0)->get_expr(); @@ -163,27 +158,27 @@ static ConstantValue parse_vertex_call_to_compile_time_function(Vloc, str)); + return parse_nanotons_as_floating_string(v_arg->loc, str); } if (f_name == "stringCrc32") { // previously, postfix "..."c - return ConstantValue(td::make_refint(td::crc32(td::Slice{str.data(), str.size()}))); + return td::make_refint(td::crc32(td::Slice{str.data(), str.size()})); } if (f_name == "stringCrc16") { // previously, there was no postfix in FunC, no way to calc at compile-time - return ConstantValue(td::make_refint(td::crc16(td::Slice{str.data(), str.size()}))); + return td::make_refint(td::crc16(td::Slice{str.data(), str.size()})); } if (f_name == "stringSha256") { // previously, postfix "..."H unsigned char hash[32]; digest::hash_str(hash, str.data(), str.size()); - return ConstantValue(td::bits_to_refint(hash, 256, false)); + return td::bits_to_refint(hash, 256, false); } if (f_name == "stringSha256_32") { // previously, postfix "..."h unsigned char hash[32]; digest::hash_str(hash, str.data(), str.size()); - return ConstantValue(td::bits_to_refint(hash, 32, false)); + return td::bits_to_refint(hash, 32, false); } if (f_name == "stringAddressToSlice") { // previously, postfix "..."a @@ -202,7 +197,7 @@ static ConstantValue parse_vertex_call_to_compile_time_function(V(4) << (64 - 3), 3); td::bitstring::bits_store_long_top(data, 3, static_cast(workchain) << (64 - 8), 8); td::bitstring::bits_memcpy(data, 3 + 8, addr.bits().ptr, 0, ton::StdSmcAddress::size()); - return ConstantValue(td::BitSlice{data, sizeof(data)}.to_hex()); + return td::BitSlice{data, sizeof(data)}.to_hex(); } if (f_name == "stringHexToSlice") { // previously, postfix "..."s @@ -211,7 +206,7 @@ static ConstantValue parse_vertex_call_to_compile_time_function(Verror("invalid hex bitstring constant"); } - return ConstantValue(static_cast(str)); + return static_cast(str); } if (f_name == "stringToBase256") { // previously, postfix "..."u @@ -222,149 +217,56 @@ static ConstantValue parse_vertex_call_to_compile_time_function(Verror("too long integer ascii-constant"); } - return ConstantValue(std::move(intval)); + return std::move(intval); } tolk_assert(false); - return ConstantValue(); } -struct ConstantEvaluator { - static bool is_overflow(const td::RefInt256& intval) { - return intval.is_null() || !intval->signed_fits_bits(257); +struct ConstantExpressionChecker { + static void handle_unary_operator(V v) { + visit(v->get_rhs()); } - static ConstantValue handle_unary_operator(V v, const ConstantValue& rhs) { - tolk_assert(rhs.is_int()); // type inferring has passed before, so it's int/bool - td::RefInt256 intval = rhs.as_int(); - - switch (v->tok) { - case tok_minus: - intval = -intval; - break; - case tok_plus: - break; - case tok_bitwise_not: - intval = ~intval; - break; - case tok_logical_not: - intval = td::make_refint(intval == 0 ? -1 : 0); - break; - default: - v->error("not a constant expression"); - } - - if (is_overflow(intval)) { - v->error("integer overflow"); - } - return ConstantValue(std::move(intval)); - } - - static ConstantValue handle_binary_operator(V v, const ConstantValue& lhs, const ConstantValue& rhs) { - tolk_assert(lhs.is_int() && rhs.is_int()); // type inferring has passed before, so they are int/bool - td::RefInt256 lhs_intval = lhs.as_int(); - td::RefInt256 rhs_intval = rhs.as_int(); - td::RefInt256 intval; - - switch (v->tok) { - case tok_minus: - intval = lhs_intval - rhs_intval; - break; - case tok_plus: - intval = lhs_intval + rhs_intval; - break; - case tok_mul: - intval = lhs_intval * rhs_intval; - break; - case tok_div: - intval = lhs_intval / rhs_intval; - break; - case tok_mod: - intval = lhs_intval % rhs_intval; - break; - case tok_lshift: - intval = lhs_intval << static_cast(rhs_intval->to_long()); - break; - case tok_rshift: - intval = lhs_intval >> static_cast(rhs_intval->to_long()); - break; - case tok_bitwise_and: - intval = lhs_intval & rhs_intval; - break; - case tok_bitwise_or: - intval = lhs_intval | rhs_intval; - break; - case tok_bitwise_xor: - intval = lhs_intval ^ rhs_intval; - break; - case tok_eq: - intval = td::make_refint(lhs_intval == rhs_intval ? -1 : 0); - break; - case tok_lt: - intval = td::make_refint(lhs_intval < rhs_intval ? -1 : 0); - break; - case tok_gt: - intval = td::make_refint(lhs_intval > rhs_intval ? -1 : 0); - break; - case tok_leq: - intval = td::make_refint(lhs_intval <= rhs_intval ? -1 : 0); - break; - case tok_geq: - intval = td::make_refint(lhs_intval >= rhs_intval ? -1 : 0); - break; - case tok_neq: - intval = td::make_refint(lhs_intval != rhs_intval ? -1 : 0); - break; - case tok_logical_and: - intval = td::make_refint(lhs_intval != 0 && rhs_intval != 0 ? -1 : 0); - break; - case tok_logical_or: - intval = td::make_refint(lhs_intval != 0 || rhs_intval != 0 ? -1 : 0); - break; - default: - v->error("unsupported binary operator in constant expression"); - } - - if (is_overflow(intval)) { - v->error("integer overflow"); - } - return ConstantValue(std::move(intval)); + static void handle_binary_operator(V v) { + visit(v->get_lhs()); + visit(v->get_rhs()); } // `const a = 1 + b`, we met `b` - static ConstantValue handle_reference(V v) { + static void handle_reference(V v) { GlobalConstPtr const_ref = v->sym->try_as(); if (!const_ref) { v->error("symbol `" + static_cast(v->get_name()) + "` is not a constant"); } - - if (!const_ref->value.initialized()) { // maybe, `b` was already calculated - eval_and_assign_const_init_value(const_ref); // if not, dig recursively into `b` - } - return const_ref->value; } // `const a = ton("0.05")`, we met `ton("0.05")` - static ConstantValue handle_function_call(V v) { + static void handle_function_call(V v) { if (v->fun_maybe && v->fun_maybe->is_compile_time_only()) { - return parse_vertex_call_to_compile_time_function(v, v->fun_maybe->name); + // `ton(local_var)` is denied; it's validated not here, but when replacing its value with a calculated one + return; } v->error("not a constant expression"); } - static ConstantValue visit(AnyExprV v) { - if (auto v_int = v->try_as()) { - return ConstantValue(v_int->intval); + // `const a = (1, 2)`, why not; or it's a default value of a field + static void handle_tensor(V v) { + for (int i = 0; i < v->size(); ++i) { + visit(v->get_item(i)); } - if (auto v_bool = v->try_as()) { - return ConstantValue(v_bool->bool_val ? -1 : 0); + } + + static void visit(AnyExprV v) { + if (v->try_as() || v->try_as() || v->try_as() || v->try_as()) { + return; } if (auto v_unop = v->try_as()) { - return handle_unary_operator(v_unop, visit(v_unop->get_rhs())); + return handle_unary_operator(v_unop); } if (auto v_binop = v->try_as()) { - return handle_binary_operator(v_binop, visit(v_binop->get_lhs()), visit(v_binop->get_rhs())); + return handle_binary_operator(v_binop); } if (auto v_ref = v->try_as()) { return handle_reference(v_ref); @@ -372,50 +274,45 @@ struct ConstantEvaluator { if (auto v_call = v->try_as()) { return handle_function_call(v_call); } + if (auto v_tensor = v->try_as()) { + return handle_tensor(v_tensor); + } if (auto v_par = v->try_as()) { return visit(v_par->get_expr()); } if (auto v_cast_to = v->try_as()) { return visit(v_cast_to->get_expr()); } - if (auto v_string = v->try_as()) { - return parse_vertex_string_const(v_string); + if (auto v_dot = v->try_as(); v_dot && (v_dot->is_target_indexed_access() || v_dot->is_target_fun_ref())) { + return visit(v_dot->get_obj()); } v->error("not a constant expression"); } - // evaluate `2 + 3` into 5 - // type inferring has already passed, to types are correct, `1 + ""` can not occur + // check that `2 + 3` is constant + // type inferring has already passed, so types are correct, `1 + ""` can not occur // if v is not a constant expression like `foo()`, an exception is thrown - static ConstantValue eval_expression_expected_to_be_constant(AnyExprV v) { - return visit(v); + static void check_expression_expected_to_be_constant(AnyExprV v) { + visit(v); } }; -ConstantValue eval_string_const_standalone(AnyExprV v_string) { +void check_expression_is_constant(AnyExprV v_expr) { + ConstantExpressionChecker::check_expression_expected_to_be_constant(v_expr); +} + +std::string eval_string_const_standalone(AnyExprV v_string) { auto v = v_string->try_as(); tolk_assert(v); - return parse_vertex_string_const(v); + td::Slice str_slice = td::Slice(v->str_val.data(), v->str_val.size()); + return td::hex_encode(str_slice); } -ConstantValue eval_call_to_compile_time_function(AnyExprV v_call) { +CompileTimeFunctionResult eval_call_to_compile_time_function(AnyExprV v_call) { auto v = v_call->try_as(); tolk_assert(v && v->fun_maybe->is_compile_time_only()); return parse_vertex_call_to_compile_time_function(v, v->fun_maybe->name); } -// evaluate `2 + 3` into `5` -// if v is not a constant expression like `gVar`, an exception is thrown -ConstantValue eval_expression_expected_to_be_constant(AnyExprV v) { - return ConstantEvaluator::eval_expression_expected_to_be_constant(v); -} - -// evaluate `const a = 2 + 3` into `const a = 5` -// recursive initializers `const a = b; const b = a` don't exist, checked on type inferring -// if init_value is not a constant expression like `const a = foo()`, an exception is thrown -void eval_and_assign_const_init_value(GlobalConstPtr const_ref) { - ConstantValue init_value = ConstantEvaluator::eval_expression_expected_to_be_constant(const_ref->init_value); - const_ref->mutate()->assign_const_value(std::move(init_value)); -} } // namespace tolk diff --git a/tolk/constant-evaluator.h b/tolk/constant-evaluator.h index 1a7f364fbe..b7a9fc7fe1 100644 --- a/tolk/constant-evaluator.h +++ b/tolk/constant-evaluator.h @@ -22,34 +22,10 @@ namespace tolk { -class ConstantValue { - std::variant< - td::RefInt256, // is set for int, intN, coins, bool - std::string // is set for slice, bytesN - > value; - -public: - ConstantValue() = default; // by default, not initialized - - explicit ConstantValue(int value) - : value(td::make_refint(value)) {} - explicit ConstantValue(td::RefInt256 value) - : value(std::move(value)) {} - explicit ConstantValue(std::string value) - : value(std::move(value)) {} - - bool initialized() const { return is_slice() || std::get(value).not_null(); } - - bool is_int() const { return std::holds_alternative(value); } - bool is_slice() const { return std::holds_alternative(value); } - - td::RefInt256 as_int() const { return std::get(value); } - const std::string& as_slice() const { return std::get(value); } -}; - -ConstantValue eval_string_const_standalone(AnyExprV v_string); -ConstantValue eval_call_to_compile_time_function(AnyExprV v_call); -ConstantValue eval_expression_expected_to_be_constant(AnyExprV v); -void eval_and_assign_const_init_value(GlobalConstPtr const_ref); +typedef std::variant CompileTimeFunctionResult; + +void check_expression_is_constant(AnyExprV v_expr); +std::string eval_string_const_standalone(AnyExprV v_string); +CompileTimeFunctionResult eval_call_to_compile_time_function(AnyExprV v_call); } // namespace tolk diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 8f4a8ddab1..cd6e38f0ca 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -20,7 +20,6 @@ #include "ast-visitor.h" #include "type-system.h" #include "common/refint.h" -#include "constant-evaluator.h" #include "smart-casts-cfg.h" #include @@ -876,15 +875,8 @@ std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, Co return local_ir_idx; } if (GlobalConstPtr const_ref = sym->try_as()) { - if (const_ref->value.is_int()) { - std::vector rvect = code.create_tmp_var(TypeDataInt::create(), loc, "(glob-const)"); - code.emplace_back(loc, Op::_IntConst, rvect, const_ref->value.as_int()); - return rvect; - } else { - std::vector rvect = code.create_tmp_var(TypeDataSlice::create(), loc, "(glob-const)"); - code.emplace_back(loc, Op::_SliceConst, rvect, const_ref->value.as_slice()); - return rvect; - } + tolk_assert(lval_ctx == nullptr); + return pre_compile_expr(const_ref->init_value, code, const_ref->declared_type); } if (FunctionPtr fun_ref = sym->try_as()) { std::vector rvect = code.create_tmp_var(fun_ref->inferred_full_type, loc, "(glob-var-fun)"); @@ -1341,7 +1333,10 @@ static std::vector process_object_literal(V v, Co break; } } - tolk_assert(v_init_val); + if (!v_init_val) { + tolk_assert(field_ref->has_default_value()); + v_init_val = field_ref->default_value; // it may be a complex expression, but it's constant, checked earlier + } tensor_items.push_back(v_init_val); target_types.push_back(field_ref->declared_type); } @@ -1359,9 +1354,8 @@ static std::vector process_int_const(V v, CodeBlob& co } static std::vector process_string_const(V v, CodeBlob& code, TypePtr target_type) { - tolk_assert(v->literal_value.is_slice()); std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(str-const)"); - code.emplace_back(v->loc, Op::_SliceConst, rvect, v->literal_value.as_slice()); + code.emplace_back(v->loc, Op::_SliceConst, rvect, v->literal_value); return transition_to_target_type(std::move(rvect), code, target_type, v); } diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 46de607c83..2b87d77cbe 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -747,6 +747,16 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } } + + // given struct field `a: int = 2 + 3` check types within its default_value + void start_visiting_field_default(StructFieldPtr field_ref) { + parent::visit(field_ref->default_value); + + TypePtr inferred_type = field_ref->default_value->inferred_type; + if (!field_ref->declared_type->can_rhs_be_assigned(inferred_type)) { + throw ParseError(field_ref->loc, "can not assign " + to_string(inferred_type) + " to " + to_string(field_ref->declared_type)); + } + } }; void pipeline_check_inferred_types() { @@ -756,6 +766,13 @@ void pipeline_check_inferred_types() { for (GlobalConstPtr const_ref : get_all_declared_constants()) { visitor.start_visiting_constant(const_ref); } + for (StructPtr struct_ref : get_all_declared_structs()) { + for (StructFieldPtr field_ref : struct_ref->fields) { + if (field_ref->has_default_value()) { + visitor.start_visiting_field_default(field_ref); + } + } + } } } // namespace tolk diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index c64f7bff50..a5e68c6bf3 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -18,6 +18,7 @@ #include "ast.h" #include "ast-replacer.h" #include "type-system.h" +#include "constant-evaluator.h" /* * This pipe is supposed to do constant folding, like replacing `2 + 3` with `5`. @@ -57,8 +58,8 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { return inner; } - static V create_string_const(SrcLocation loc, ConstantValue&& literal_value) { - auto v_string = createV(loc, literal_value.as_slice()); + static V create_string_const(SrcLocation loc, std::string&& literal_value) { + auto v_string = createV(loc, literal_value); v_string->assign_inferred_type(TypeDataSlice::create()); v_string->assign_literal_value(std::move(literal_value)); v_string->assign_rvalue_true(); @@ -112,14 +113,12 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { // replace `ton("0.05")` with 50000000 / `stringCrc32("some_str")` with calculated value / etc. if (v->fun_maybe && v->fun_maybe->is_compile_time_only()) { - ConstantValue value = eval_call_to_compile_time_function(v); - if (value.is_int()) { - return create_int_const(v->loc, value.as_int()); + CompileTimeFunctionResult value = eval_call_to_compile_time_function(v); + if (std::holds_alternative(value)) { + return create_int_const(v->loc, std::move(std::get(value))); + } else { + return create_string_const(v->loc, std::move(std::get(value))); } - if (value.is_slice()) { - return create_string_const(v->loc, std::move(value)); - } - tolk_assert(false); } return v; @@ -128,7 +127,7 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { AnyExprV replace(V v) override { // when "some_str" occurs as a standalone constant (not inside `stringCrc32("some_str")`, // it's actually a slice - ConstantValue literal_value = eval_string_const_standalone(v); + std::string literal_value = eval_string_const_standalone(v); v->mutate()->assign_literal_value(std::move(literal_value)); return v; @@ -140,10 +139,7 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { // replace `2 + 3 => ...` with `5 => ...` // non-constant expressions like `foo() => ...` fire an error here if (v->pattern_kind == MatchArmKind::const_expression && v->get_pattern_expr()->type != ast_int_const) { - ConstantValue value = eval_expression_expected_to_be_constant(v->get_pattern_expr()); - tolk_assert(value.is_int()); - AnyExprV v_new_pattern = create_int_const(v->get_pattern_expr()->loc, value.as_int()); - v->mutate()->assign_resolved_pattern(MatchArmKind::const_expression, nullptr, v_new_pattern); + check_expression_is_constant(v->get_pattern_expr()); } return v; @@ -153,15 +149,32 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } + + // used to replace `ton("0.05")` and other compile-time functions inside fields defaults, etc. + AnyExprV replace_in_expression(AnyExprV init_value) { + return parent::replace(init_value); + } }; void pipeline_constant_folding() { - // here (after type inferring) evaluate `const a = 2 + 3` into `5` + ConstantFoldingReplacer replacer; + + // here (after type inferring) check that `const a = 2 + 3` is a valid constant expression // non-constant expressions like `const a = foo()` fire an error here + // also, replace `const a = ton("0.05")` with `const a = 50000000` for (GlobalConstPtr const_ref : get_all_declared_constants()) { - // for `const a = b`, `b` could be already calculated while calculating `a` - if (!const_ref->value.initialized()) { - eval_and_assign_const_init_value(const_ref); + check_expression_is_constant(const_ref->init_value); + AnyExprV replaced = replacer.replace_in_expression(const_ref->init_value); + const_ref->mutate()->assign_init_value(replaced); + } + // do the same for default values of struct fields, they must be constant expressions + for (StructPtr struct_ref : get_all_declared_structs()) { + for (StructFieldPtr field_ref : struct_ref->fields) { + if (field_ref->has_default_value()) { + check_expression_is_constant(field_ref->default_value); + AnyExprV replaced = replacer.replace_in_expression(field_ref->default_value); + field_ref->mutate()->assign_default_value(replaced); + } } } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 9cca8a2921..ee2e0f27b5 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -1212,7 +1212,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v_field, v_field->get_init_val()); } for (StructFieldPtr field_ref : struct_ref->fields) { - if (!(occurred_mask & (1ULL << field_ref->field_idx))) { + if (!(occurred_mask & (1ULL << field_ref->field_idx)) && !field_ref->has_default_value()) { fire(cur_f, v->get_body()->loc, "field `" + field_ref->name + "` missed in initialization of struct `" + struct_ref->name + "`"); } } @@ -1439,6 +1439,12 @@ class InferTypesAndCallsAndFieldsVisitor final { infer_any_expr(const_ref->init_value, std::move(const_flow), false, const_ref->declared_type); const_ref->mutate()->assign_inferred_type(const_ref->declared_type == nullptr ? const_ref->init_value->inferred_type : const_ref->declared_type); } + + // given struct field `a: int = 2 + 3` infer that default value is int, assign inferred_type to all nodes + void start_visiting_field_default(StructFieldPtr field_ref) { + FlowContext field_flow; + infer_any_expr(field_ref->default_value, std::move(field_flow), false, field_ref->declared_type); + } }; class LaunchInferTypesAndMethodsOnce final { @@ -1510,7 +1516,15 @@ void pipeline_infer_types_and_calls_and_fields() { visitor.start_visiting_constant(const_ref); } } - // (later, at constant folding, `const a = 2 + 3` will be evaluated to 5) + + // infer types for default values in structs + for (StructPtr struct_ref : get_all_declared_structs()) { + for (StructFieldPtr field_ref : struct_ref->fields) { + if (field_ref->has_default_value()) { + visitor.start_visiting_field_default(field_ref); + } + } + } } void pipeline_infer_types_and_calls_and_fields(FunctionPtr fun_ref) { diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 7185471987..6bb93d55ae 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -19,7 +19,6 @@ #include "src-file.h" #include "ast.h" #include "compiler-state.h" -#include "constant-evaluator.h" #include "generics-helpers.h" #include "td/utils/crypto.h" #include "type-system.h" @@ -116,9 +115,6 @@ static const GenericsDeclaration* construct_genericTs(V v_li static void register_constant(V v) { GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->declared_type, v->get_init_value()); - // init value of constant is not evaluated here - // at first, it will be type checked (in type inference pipe) - // then, at constant folding pipe, `const a = 2 + 3` will be evaluated to 5 G.symtable.add_global_const(c_sym); G.all_constants.push_back(c_sym); @@ -147,7 +143,8 @@ static void register_struct(V v) { fields.reserve(v_body->get_num_fields()); for (int i = 0; i < v_body->get_num_fields(); ++i) { auto v_field = v_body->get_field(i); - StructFieldData* field_ref = new StructFieldData(static_cast(v_field->get_identifier()->name), v_field->loc, i, v_field->declared_type); + AnyExprV default_value = v_field->has_default_value() ? v_field->get_default_value() : nullptr; + StructFieldData* field_ref = new StructFieldData(static_cast(v_field->get_identifier()->name), v_field->loc, i, v_field->declared_type, default_value); fields.emplace_back(field_ref); v_field->mutate()->assign_field_ref(field_ref); } @@ -155,6 +152,7 @@ static void register_struct(V v) { StructData* s_sym = new StructData(static_cast(v->get_identifier()->name), v->loc, std::move(fields)); G.symtable.add_struct(s_sym); + G.all_structs.push_back(s_sym); v->mutate()->assign_struct_ref(s_sym); } diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 304a3497d1..545cf50fc8 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -422,6 +422,11 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { // `const a = b`, resolve `b` parent::visit(v->get_init_value()); } + + void start_visiting_struct_field(V v) { + // field `a: int = C`, resolve `C` + parent::visit(v->get_default_value()); + } }; NameAndScopeResolver AssignSymInsideFunctionVisitor::current_scope; @@ -458,6 +463,7 @@ void pipeline_resolve_identifiers_and_assign_symbols() { auto v_body = v_struct->get_struct_body(); for (int i = 0; i < v_body->get_num_fields(); ++i) { auto v_field = v_body->get_field(i); + visitor.start_visiting_struct_field(v_field); v_field->mutate()->assign_resolved_type(v_struct->struct_ref->get_field(i)->declared_type); } } diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index ccd56a011e..97e48f7fce 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -97,8 +97,8 @@ void GlobalConstData::assign_inferred_type(TypePtr inferred_type) { this->inferred_type = inferred_type; } -void GlobalConstData::assign_const_value(ConstantValue&& value) { - this->value = std::move(value); +void GlobalConstData::assign_init_value(AnyExprV init_value) { + this->init_value = init_value; } void LocalVarData::assign_ir_idx(std::vector&& ir_idx) { @@ -121,6 +121,10 @@ void StructFieldData::assign_resolved_type(TypePtr declared_type) { this->declared_type = declared_type; } +void StructFieldData::assign_default_value(AnyExprV default_value) { + this->default_value = default_value; +} + void StructData::assign_resolved_type(TypePtr struct_type) { this->struct_type = struct_type; } diff --git a/tolk/symtable.h b/tolk/symtable.h index 269aee90fb..9ab7f7b965 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -18,7 +18,6 @@ #include "src-file.h" #include "fwd-declarations.h" -#include "constant-evaluator.h" #include "crypto/common/refint.h" #include #include @@ -200,7 +199,6 @@ struct GlobalVarData final : Symbol { struct GlobalConstData final : Symbol { AnyExprV init_value; - ConstantValue value; TypePtr declared_type; // `const a: int = ...`; nullptr for `const a = ...` TypePtr inferred_type = nullptr; // filled at type inferring pass @@ -213,7 +211,7 @@ struct GlobalConstData final : Symbol { GlobalConstData* mutate() const { return const_cast(this); } void assign_resolved_type(TypePtr declared_type); void assign_inferred_type(TypePtr inferred_type); - void assign_const_value(ConstantValue&& value); + void assign_init_value(AnyExprV init_value); }; struct AliasDefData final : Symbol { @@ -231,14 +229,19 @@ struct AliasDefData final : Symbol { struct StructFieldData final : Symbol { int field_idx; TypePtr declared_type; + AnyExprV default_value; // nullptr if no default + + bool has_default_value() const { return default_value != nullptr; } StructFieldData* mutate() const { return const_cast(this); } void assign_resolved_type(TypePtr declared_type); + void assign_default_value(AnyExprV default_value); - StructFieldData(std::string name, SrcLocation loc, int field_idx, TypePtr declared_type) + StructFieldData(std::string name, SrcLocation loc, int field_idx, TypePtr declared_type, AnyExprV default_value) : Symbol(std::move(name), loc) , field_idx(field_idx) - , declared_type(declared_type) { + , declared_type(declared_type) + , default_value(default_value) { } }; From 09019fea5d924d8dbb424867d3f453864ca4bd29 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 20 Feb 2025 19:10:08 +0400 Subject: [PATCH 230/524] [Tolk] Refactor parsing types, introduce AST type nodes Until this MR, parsing any type from tokens immediately resulted in TypePtr. This MR introduces an intermediate AST representation for types, which are resolved at a later step. It's a necessary refactoring towards generics. --- .../tests/invalid-semantics/err-4517.tolk | 2 +- .../tests/invalid-syntax/err-3457.tolk | 16 +- tolk-tester/tests/match-by-expr-tests.tolk | 13 +- tolk-tester/tests/nullable-tensors.tolk | 6 +- tolk-tester/tests/smart-cast-tests.tolk | 8 +- tolk-tester/tests/struct-tests.tolk | 2 +- tolk-tester/tests/union-types-tests.tolk | 38 +- tolk-tester/tests/var-apply-tests.tolk | 2 +- tolk/CMakeLists.txt | 1 + tolk/ast-from-tokens.cpp | 183 +++++-- tolk/ast-replacer.h | 12 +- tolk/ast-replicator.h | 105 ++-- tolk/ast-stringifier.h | 85 ++-- tolk/ast-visitor.h | 22 +- tolk/ast.cpp | 66 +-- tolk/ast.h | 322 ++++++++----- tolk/fwd-declarations.h | 2 + tolk/generics-helpers.cpp | 135 ++---- tolk/generics-helpers.h | 4 +- tolk/lexer.cpp | 9 - tolk/lexer.h | 1 - tolk/pipe-ast-to-legacy.cpp | 42 +- tolk/pipe-calc-rvalue-lvalue.cpp | 2 +- tolk/pipe-check-inferred-types.cpp | 66 +-- tolk/pipe-check-rvalue-lvalue.cpp | 6 - tolk/pipe-constant-folding.cpp | 12 +- tolk/pipe-infer-types-and-calls.cpp | 83 ++-- tolk/pipe-optimize-boolean-expr.cpp | 2 +- tolk/pipe-refine-lvalue-for-mutate.cpp | 4 +- tolk/pipe-register-symbols.cpp | 61 +-- tolk/pipe-resolve-identifiers.cpp | 231 ++------- tolk/pipe-resolve-types.cpp | 455 ++++++++++++++++++ tolk/pipeline.h | 3 + tolk/smart-casts-cfg.cpp | 8 +- tolk/symtable.cpp | 10 +- tolk/symtable.h | 80 ++- tolk/tolk.cpp | 3 +- tolk/tolk.h | 6 + tolk/type-system.cpp | 278 +---------- tolk/type-system.h | 68 +-- 40 files changed, 1328 insertions(+), 1126 deletions(-) create mode 100644 tolk/pipe-resolve-types.cpp diff --git a/tolk-tester/tests/invalid-semantics/err-4517.tolk b/tolk-tester/tests/invalid-semantics/err-4517.tolk index f84e3ac1b9..5497ebb41b 100644 --- a/tolk-tester/tests/invalid-semantics/err-4517.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4517.tolk @@ -10,5 +10,5 @@ fun f(x: int): int { /** @compilation_should_fail -@stderr symbol `g` is not a constant +@stderr unknown type name `g` */ diff --git a/tolk-tester/tests/invalid-syntax/err-3457.tolk b/tolk-tester/tests/invalid-syntax/err-3457.tolk index 8cfcf8e6ca..44c1b89544 100644 --- a/tolk-tester/tests/invalid-syntax/err-3457.tolk +++ b/tolk-tester/tests/invalid-syntax/err-3457.tolk @@ -1,4 +1,5 @@ type asdf = int; +const gcon = 10; fun main() { match (10) { @@ -7,12 +8,21 @@ fun main() { var asdf = 5; match (10) { - asdf => 2, // it's match by expression now + asdf => 2, // also match by type }; + + match (10) { + gcon => 2, // match by expression (it's constant) + }; + + var ten = 10; + match (10) { + ten => 10, // not a constant, tries to lookup a type, fails + } } /** @compilation_should_fail -@stderr symbol `asdf` is not a constant -@stderr asdf => 2 +@stderr unknown type name `ten` +@stderr ten => 10 */ diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk index 80287c9bbd..7941b46001 100644 --- a/tolk-tester/tests/match-by-expr-tests.tolk +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -117,7 +117,18 @@ fun test11(x: bool) { } -fun main() {} +type asdf = int; + +fun main() { + match (10) { + asdf => 1, // it's match by type + }; + + var asdf = 5; + return match (10) { + asdf => 2, // also match by type, regardless of local var + }; +} /** @testcase | 101 | 1 | 100 diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index c1de8c0b7b..41b3d62b86 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -519,14 +519,14 @@ fun main(){} @testcase | 132 | | -1 0 -1 0 777 (null) (null) -1 0 0 @testcase | 133 | | 60 @testcase | 134 | | 11 21 129 -@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 146 (null) 146 (null) 0 777 10 143 (null) 143 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 +@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 143 (null) 143 (null) 0 777 10 144 (null) 144 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 @testcase | 136 | 9 | 9 0 @testcase | 136 | null | (null) -1 -@testcase | 140 | 8 9 | 8 9 144 (null) (null) 0 +@testcase | 140 | 8 9 | 8 9 145 (null) (null) 0 @testcase | 141 | | (null) 10 @testcase | 142 | | 3 3 1 2 @testcase | 143 | -1 0 | 1 2 1 3 777 (null) (null) (null) (null) 0 -@testcase | 143 | 0 -1 | 1 (null) 0 3 777 1 2 1 3 145 +@testcase | 143 | 0 -1 | 1 (null) 0 3 777 1 2 1 3 146 @fif_codegen """ diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 77374bdcef..aac1028f3d 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -727,16 +727,16 @@ fun main(x: int?): int { @testcase | 139 | | 16 @testcase | 140 | 5 | 25 @testcase | 141 | | 1 2 -@testcase | 142 | | 5 3 (null) (null) 0 132 3 (null) (null) 0 +@testcase | 142 | | 5 3 (null) (null) 0 133 3 (null) (null) 0 @testcase | 143 | | 10 11 (null) 10 11 (null) (null) 0 @testcase | 144 | | 10 11 (null) 10 11 (null) (null) 0 -@testcase | 145 | | 5 3 (null) (null) 0 132 3 (null) (null) (null) (null) 0 3 (null) (null) 0 -@testcase | 146 | | 3 4 5 3 4 5 131 +@testcase | 145 | | 5 3 (null) (null) 0 133 3 (null) (null) (null) (null) 0 3 (null) (null) 0 +@testcase | 146 | | 3 4 5 3 4 5 132 @testcase | 147 | | (null) (null) 100 (null) 100 (null) (null) 0 @testcase | 158 | | 123 10 123 5 @testcase | 160 | | 101 109 @testcase | 161 | 9 9 | (null) (null) 0 (null) (null) -@testcase | 161 | 19 0 | 19 0 133 19 0 +@testcase | 161 | 19 0 | 19 0 129 19 0 @stderr warning: expression of type `int` can never be `null`, this condition is always true @stderr warning: unreachable code diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 4a676a3db1..8dd2f3455a 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -506,7 +506,7 @@ type PointAlias = Point; @testcase | 126 | | 118 @testcase | 127 | | 0 0 131 131 777 -1 -1 0 0 777 0 0 138 777 0 0 777 (null) (null) 0 @testcase | 128 | -1 | (null) (null) 131 -1 0 0 777 (null) (null) 131 -1 0 0 -@testcase | 128 | 0 | 1 2 129 0 -1 0 777 0 131 138 0 -1 0 +@testcase | 128 | 0 | 1 2 130 0 -1 0 777 0 131 138 0 -1 0 @testcase | 129 | | (null) @testcase | 130 | -1 | 140 777 (null) 140 777 0 777 0 0 @testcase | 130 | 0 | 139 777 4 1 777 139 777 0 0 diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 8cd97f9419..0bc88c3660 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -791,11 +791,11 @@ fun main() { @testcase | 105 | | 5 5 1 5 1 5 @testcase | 106 | 1 | (null) 0 @testcase | 106 | 2 | 2 1 -@testcase | 107 | 1 | (null) 2 3 131 (null) 2 3 131 -@testcase | 107 | 3 | 6 7 8 132 6 7 8 132 -@testcase | 108 | | (null) 2 3 131 (null) (null) (null) 0 (null) 2 3 131 (null) 2 3 131 (null) (null) (null) 0 -@testcase | 109 | | 6 7 8 132 0 (null) -1 (null) (null) (null) 0 -1 (null) (null) (null) 0 -@testcase | 110 | | 1 (null) (null) 0 (null) (null) 0 134 +@testcase | 107 | 1 | (null) 2 3 129 (null) 2 3 129 +@testcase | 107 | 3 | 6 7 8 130 6 7 8 130 +@testcase | 108 | | (null) 2 3 129 (null) (null) (null) 0 (null) 2 3 129 (null) 2 3 129 (null) (null) (null) 0 +@testcase | 109 | | 6 7 8 130 0 (null) -1 (null) (null) (null) 0 -1 (null) (null) (null) 0 +@testcase | 110 | | 1 (null) (null) 0 (null) (null) 0 135 @testcase | 120 | | 5 @testcase | 121 | | -1 0 0 -1 -1 0 @testcase | 122 | 0 0 1 | 36 @@ -807,21 +807,21 @@ fun main() { @testcase | 125 | 1 1 | 42 @testcase | 126 | | 42 @testcase | 127 | | 1 2 5 1 -@testcase | 128 | | (null) (null) 0 777 (null) 2 1 777 3 4 131 +@testcase | 128 | | (null) (null) 0 777 (null) 2 1 777 3 4 129 @testcase | 129 | | (null) (null) 0 (null) 5 1 -@testcase | 130 | 0 | (null) 5 1 777 (null) (null) 0 777 1 2 131 777 (null) [ 6 ] 135 777 (null) (null) 0 -@testcase | 130 | -1 | (null) (null) 0 777 (null) (null) 0 777 1 2 131 777 (null) [ 6 ] 135 777 (null) (null) 0 -@testcase | 131 | | (null) 777 5 777 1 2 777 (null) 777 5 777 (null) (null) 0 777 1 2 131 -@testcase | 132 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 131 777 (null) (null) (null) 0 -@testcase | 133 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 131 777 (null) (null) (null) 0 -@testcase | 134 | | (null) 5 1 777 (null) 5 1 777 1 2 131 777 1 2 131 777 (null) 5 42 777 5 42 777 5 42 -@testcase | 135 | | (null) 5 1 777 (null) (null) 1 2 131 777 (null) (null) 1 2 131 777 (null) (null) 0 -@testcase | 136 | | 1 2 131 777 1 2 140 777 1 2 141 777 (null) 1 2 138 +@testcase | 130 | 0 | (null) 5 1 777 (null) (null) 0 777 1 2 129 777 (null) [ 6 ] 136 777 (null) (null) 0 +@testcase | 130 | -1 | (null) (null) 0 777 (null) (null) 0 777 1 2 129 777 (null) [ 6 ] 136 777 (null) (null) 0 +@testcase | 131 | | (null) 777 5 777 1 2 777 (null) 777 5 777 (null) (null) 0 777 1 2 129 +@testcase | 132 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 129 777 (null) (null) (null) 0 +@testcase | 133 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 129 777 (null) (null) (null) 0 +@testcase | 134 | | (null) 5 1 777 (null) 5 1 777 1 2 129 777 1 2 129 777 (null) 5 42 777 5 42 777 5 42 +@testcase | 135 | | (null) 5 1 777 (null) (null) 1 2 129 777 (null) (null) 1 2 129 777 (null) (null) 0 +@testcase | 136 | | 1 2 129 777 1 2 140 777 1 2 141 777 (null) 1 2 142 @testcase | 137 | 1 2 | 1 2 777 1 2 777 1 2 777 [ 1 2 ] -@testcase | 138 | | 1 (null) (null) (null) (null) 0 2 149 777 1 (null) 0 2 143 777 1 5 1 2 143 777 1 (null) 5 1 2 144 +@testcase | 138 | | 1 (null) (null) (null) (null) 0 2 146 777 1 (null) 0 2 147 777 1 5 1 2 147 777 1 (null) 5 1 2 148 @testcase | 139 | | 1 (null) (null) (null) (null) 0 2 777 1 (null) 0 2 777 1 5 1 2 777 1 (null) 5 1 2 @testcase | 140 | 5 | 5 42 -@testcase | 140 | 15 | 15 129 +@testcase | 140 | 15 | 15 131 @testcase | 140 | 90 | 90 49 @testcase | 141 | 7 8 | 7 7 1 8 8 1 @testcase | 141 | null null | (null) (null) 0 (null) (null) 0 @@ -842,9 +842,9 @@ fun main() { @testcase | 154 | | 100 1 @testcase | 155 | | 5 1 5 1 @testcase | 156 | 1 2 -1 | 2 44 2 2 44 2 2 44 2 44 2 2 2 -@testcase | 157 | 1 | (null) 0 (null) 145 777 1 2 131 777 145 145 777 0 0 -1 0 -1 0 0 777 -1 0 -@testcase | 157 | 0 | (null) 0 (null) 145 777 (null) (null) 145 777 0 0 777 0 0 -1 0 0 -1 0 777 0 -1 -@testcase | 158 | | (null) (null) 145 (null) (null) 145 (null) 0 (null) +@testcase | 157 | 1 | (null) 0 (null) 149 777 1 2 129 777 149 149 777 0 0 -1 0 -1 0 0 777 -1 0 +@testcase | 157 | 0 | (null) 0 (null) 149 777 (null) (null) 149 777 0 0 777 0 0 -1 0 0 -1 0 777 0 -1 +@testcase | 158 | | (null) (null) 149 (null) (null) 149 (null) 0 (null) @testcase | 159 | 0 4 | 456 @testcase | 159 | null 0 | 123 @testcase | 160 | | 10 0 diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index e4318dc15e..aec77391f0 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -264,7 +264,7 @@ fun main() {} @testcase | 112 | -1 -10 -20 | -1 @testcase | 113 | 0 | 12 12 @testcase | 113 | -1 | 12 -1 -@testcase | 114 | -1 | 2 3 130 +@testcase | 114 | -1 | 2 3 131 @testcase | 114 | 0 | (null) 12 1 @testcase | 115 | | 7 6 7 */ diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index c4abfec04a..c6e992c270 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -9,6 +9,7 @@ set(TOLK_SOURCE pipe-discover-parse-sources.cpp pipe-register-symbols.cpp pipe-resolve-identifiers.cpp + pipe-resolve-types.cpp pipe-calc-rvalue-lvalue.cpp pipe-infer-types-and-calls.cpp pipe-check-inferred-types.cpp diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 939dd799c0..36c657c261 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -76,12 +76,12 @@ static void fire_error_mix_and_or_no_parenthesis(SrcLocation loc, std::string_vi // that's why if rhs->type == ast_binary_operator, it's not surrounded by parenthesis) static void diagnose_bitwise_precedence(SrcLocation loc, std::string_view operator_name, AnyExprV lhs, AnyExprV rhs) { // handle "flags & 0xFF != 0" (rhs = "0xFF != 0") - if (rhs->type == ast_binary_operator && is_comparison_binary_op(rhs->as()->tok)) { + if (rhs->kind == ast_binary_operator && is_comparison_binary_op(rhs->as()->tok)) { fire_error_lower_precedence(loc, operator_name, rhs->as()->operator_name); } // handle "0 != flags & 0xFF" (lhs = "0 != flags") - if (lhs->type == ast_binary_operator && is_comparison_binary_op(lhs->as()->tok)) { + if (lhs->kind == ast_binary_operator && is_comparison_binary_op(lhs->as()->tok)) { fire_error_lower_precedence(loc, operator_name, lhs->as()->operator_name); } } @@ -105,21 +105,22 @@ static void diagnose_and_or_precedence(SrcLocation loc, AnyExprV lhs, TokenType // diagnose "a << 8 + 1" (equivalent to "a << 9", probably unexpected) static void diagnose_addition_in_bitshift(SrcLocation loc, std::string_view bitshift_operator_name, AnyExprV rhs) { - if (rhs->type == ast_binary_operator && is_add_or_sub_binary_op(rhs->as()->tok)) { + if (rhs->kind == ast_binary_operator && is_add_or_sub_binary_op(rhs->as()->tok)) { fire_error_lower_precedence(loc, bitshift_operator_name, rhs->as()->operator_name); } } // replace (a == null) and similar to ast_is_type_operator(a, null) (as if `a is null` was written) static AnyExprV maybe_replace_eq_null_with_isNull_check(V v) { - bool has_null = v->get_lhs()->type == ast_null_keyword || v->get_rhs()->type == ast_null_keyword; + bool has_null = v->get_lhs()->kind == ast_null_keyword || v->get_rhs()->kind == ast_null_keyword; bool replace = has_null && (v->tok == tok_eq || v->tok == tok_neq); if (!replace) { return v; } - AnyExprV v_nullable = v->get_lhs()->type == ast_null_keyword ? v->get_rhs() : v->get_lhs(); - return createV(v->loc, v_nullable, TypeDataNullLiteral::create(), v->tok == tok_neq); + AnyExprV v_nullable = v->get_lhs()->kind == ast_null_keyword ? v->get_rhs() : v->get_lhs(); + AnyTypeV rhs_null_type = createV(v->loc, "null"); + return createV(v->loc, v_nullable, rhs_null_type, v->tok == tok_neq); } // parse `123` / `0xFF` / `0b10001` to td::RefInt256 @@ -142,11 +143,116 @@ static td::RefInt256 parse_tok_int_const(std::string_view text) { -/* - * - * PARSE SOURCE - * - */ +// -------------------------------------------- +// parsing type from tokens +// +// here we implement parsing types (mostly after colon) to AnyTypeV +// example: `var v: int` is leaf "int" +// example: `var v: (User?, [cell])` is tensor(nullable(leaf "User"), brackets(leaf "cell")) +// +// later, after all symbols are registered, types are resolved to TypePtr, see pipe-resolve-types.cpp +// + +static AnyTypeV parse_type_expression(Lexer& lex); + +static std::vector parse_nested_type_list(Lexer& lex, TokenType tok_op, const char* s_op, TokenType tok_cl, const char* s_cl) { + lex.expect(tok_op, s_op); + std::vector sub_types; + while (true) { + if (lex.tok() == tok_cl) { // empty lists allowed + lex.next(); + break; + } + + sub_types.emplace_back(parse_type_expression(lex)); + if (lex.tok() == tok_comma) { + lex.next(); + } else if (lex.tok() != tok_cl) { + lex.unexpected(s_cl); + } + } + return sub_types; +} + +static std::vector parse_nested_type_list_in_parenthesis(Lexer& lex) { + return parse_nested_type_list(lex, tok_oppar, "`(`", tok_clpar, "`)` or `,`"); +} + +static AnyTypeV parse_simple_type(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + switch (lex.tok()) { + case tok_self: + case tok_identifier: + case tok_null: { + std::string_view text = lex.cur_str(); + lex.next(); + return createV(loc, text); + } + case tok_oppar: { + return createV(loc, parse_nested_type_list_in_parenthesis(lex)); + } + case tok_opbracket: { + return createV(loc, parse_nested_type_list(lex, tok_opbracket, "`[`", tok_clbracket, "`]` or `,`")); + } + default: + lex.unexpected(""); + } +} + +static AnyTypeV parse_type_nullable(Lexer& lex) { + AnyTypeV result = parse_simple_type(lex); + + if (lex.tok() == tok_question) { + lex.next(); + result = createV(result->loc, result); + } + + return result; +} + +static AnyTypeV parse_type_expression(Lexer& lex) { + if (lex.tok() == tok_bitwise_or) { // `type T = | T1 | T2 | ...` (each per line) (like in TypeScript) + lex.next(); + } + + AnyTypeV result = parse_type_nullable(lex); + + if (lex.tok() == tok_bitwise_or) { // `int | slice`, `Pair2 | (Pair3 | null)` + std::vector items; + items.emplace_back(result); + while (lex.tok() == tok_bitwise_or) { + lex.next(); + items.emplace_back(parse_type_nullable(lex)); + } + result = createV(result->loc, std::move(items)); + } + + if (lex.tok() == tok_arrow) { // `int -> int`, `(cell, slice) -> void`, `int -> int -> int`, `int | cell -> void` + lex.next(); + std::vector params_and_return; + if (auto p_tensor = result->try_as()) { + params_and_return.reserve(p_tensor->get_items().size()); + params_and_return.insert(params_and_return.begin(), p_tensor->get_items().begin(), p_tensor->get_items().end()); + } else { + params_and_return.reserve(2); + params_and_return.push_back(result); + } + params_and_return.push_back(parse_type_expression(lex)); + result = createV(result->loc, std::move(params_and_return)); + } + + return result; +} + +static AnyTypeV parse_type_from_tokens(Lexer& lex) { + return parse_type_expression(lex); +} + + + +// -------------------------------------------- +// parsing expressions and statements +// AnyExprV parse_expr(Lexer& lex); @@ -178,7 +284,7 @@ static AnyV parse_parameter(Lexer& lex, bool is_first) { // parameter type after colon are mandatory lex.expect(tok_colon, "`: `"); - TypePtr param_type = parse_type_from_tokens(lex); + AnyTypeV param_type = parse_type_from_tokens(lex); return createV(loc, param_name, param_type, declared_as_mutate); } @@ -193,7 +299,7 @@ static AnyV parse_global_var_declaration(Lexer& lex, const std::vector(lex.cur_location(), lex.cur_str()); lex.next(); lex.expect(tok_colon, "`:`"); - TypePtr declared_type = parse_type_from_tokens(lex); + AnyTypeV declared_type = parse_type_from_tokens(lex); if (lex.tok() == tok_comma) { lex.error("multiple declarations are not allowed, split globals on separate lines"); } @@ -213,7 +319,7 @@ static AnyV parse_constant_declaration(Lexer& lex, const std::vector(lex.cur_location(), lex.cur_str()); lex.next(); - TypePtr declared_type = nullptr; + AnyTypeV declared_type = nullptr; if (lex.tok() == tok_colon) { lex.next(); declared_type = parse_type_from_tokens(lex); @@ -237,7 +343,7 @@ static AnyV parse_type_alias_declaration(Lexer& lex, const std::vector(lex.cur_location(), lex.cur_str()); lex.next(); lex.expect(tok_assign, "`=`"); - TypePtr underlying_type = parse_type_from_tokens(lex); + AnyTypeV underlying_type = parse_type_from_tokens(lex); lex.expect(tok_semicolon, "`;`"); return createV(loc, v_ident, underlying_type); } @@ -267,11 +373,11 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { args.push_back(parse_var_declaration_lhs(lex, is_immutable)); } lex.expect(tok_clbracket, "`]`"); - return createV(loc, std::move(args)); + return createV(loc, std::move(args)); } if (lex.tok() == tok_identifier) { auto v_ident = createV(loc, lex.cur_str()); - TypePtr declared_type = nullptr; + AnyTypeV declared_type = nullptr; bool marked_as_redef = false; lex.next(); if (lex.tok() == tok_colon) { @@ -284,7 +390,7 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { return createV(loc, v_ident, declared_type, is_immutable, marked_as_redef); } if (lex.tok() == tok_underscore) { - TypePtr declared_type = nullptr; + AnyTypeV declared_type = nullptr; lex.next(); if (lex.tok() == tok_colon) { lex.next(); @@ -449,7 +555,7 @@ static AnyV parse_throw_expression(Lexer& lex) { static V parse_match_arm(Lexer& lex) { SrcLocation loc = lex.cur_location(); MatchArmKind pattern_kind = static_cast(-1); - TypePtr exact_type = nullptr; + AnyTypeV exact_type = nullptr; AnyExprV pattern_expr = nullptr; Lexer::SavedPositionForLookahead backup = lex.save_parsing_position(); @@ -525,7 +631,7 @@ static V parse_match_expression(Lexer& lex) { if (lex.tok() == tok_clbrace) { break; } - if (!was_comma && v_arm->get_body()->type != ast_braced_expression) { + if (!was_comma && v_arm->get_body()->kind != ast_braced_expression) { lex.unexpected("`,`"); } } @@ -566,7 +672,7 @@ static AnyExprV parse_expr100(Lexer& lex) { lex.next(); if (lex.tok() == tok_clbracket) { lex.next(); - return createV(loc, {}); + return createV(loc, {}); } std::vector items(1, parse_expr(lex)); while (lex.tok() == tok_comma) { @@ -577,7 +683,7 @@ static AnyExprV parse_expr100(Lexer& lex) { items.emplace_back(parse_expr(lex)); } lex.expect(tok_clbracket, "`]`"); - return createV(loc, std::move(items)); + return createV(loc, std::move(items)); } case tok_int_const: { std::string_view orig_str = lex.cur_str(); @@ -707,13 +813,13 @@ static AnyExprV parse_expr40(Lexer& lex) { if (lex.tok() == tok_as) { SrcLocation loc = lex.cur_location(); lex.next(); - TypePtr cast_to_type = parse_type_from_tokens(lex); + AnyTypeV cast_to_type = parse_type_from_tokens(lex); lhs = createV(loc, lhs, cast_to_type); } else if (lex.tok() == tok_is) { SrcLocation loc = lex.cur_location(); lex.next(); - TypePtr rhs_type = parse_type_from_tokens(lex); - bool is_negated = lhs->type == ast_not_null_operator; // `a !is ...`, now lhs = `a!` + AnyTypeV rhs_type = parse_type_from_tokens(lex); + bool is_negated = lhs->kind == ast_not_null_operator; // `a !is ...`, now lhs = `a!` if (is_negated) { lhs = lhs->as()->get_expr(); } @@ -862,7 +968,9 @@ static V parse_block_statement(Lexer& lex) { if (lex.tok() == tok_clbrace) { break; } - bool does_end_with_brace = v->type == ast_if_statement || v->type == ast_while_statement || v->type == ast_match_expression || v->type == ast_try_catch_statement || v->type == ast_repeat_statement || v->type == ast_block_statement; + bool does_end_with_brace = v->kind == ast_if_statement || v->kind == ast_while_statement + || v->kind == ast_match_expression + || v->kind == ast_try_catch_statement || v->kind == ast_repeat_statement || v->kind == ast_block_statement; if (!does_end_with_brace) { lex.expect(tok_semicolon, "`;`"); } @@ -1038,6 +1146,12 @@ AnyV parse_statement(Lexer& lex) { } } + +// -------------------------------------------- +// parsing top-level declarations +// + + static AnyV parse_func_body(Lexer& lex) { return parse_block_statement(lex); } @@ -1136,7 +1250,7 @@ static V parse_annotation(Lexer& lex) { v_arg = createV(loc, {}); break; case AnnotationKind::method_id: - if (!v_arg || v_arg->size() != 1 || v_arg->get_item(0)->type != ast_int_const) { + if (!v_arg || v_arg->size() != 1 || v_arg->get_item(0)->kind != ast_int_const) { throw ParseError(loc, "expecting `(number)` after " + static_cast(name)); } break; @@ -1178,7 +1292,7 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vectorempty() && v_param_list->get_param(0)->param_name == "self"; int n_mutate_params = v_param_list->get_mutate_params_count(); - TypePtr ret_type = nullptr; + AnyTypeV ret_type = nullptr; bool returns_self = false; if (lex.tok() == tok_colon) { // : (if absent, it means "auto infer", not void) lex.next(); @@ -1186,9 +1300,9 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(lex.cur_location(), "void"); + lex.next(); } else { ret_type = parse_type_from_tokens(lex); } @@ -1273,7 +1387,7 @@ static AnyV parse_struct_field(Lexer& lex) { auto v_ident = createV(lex.cur_location(), lex.cur_str()); lex.next(); lex.expect(tok_colon, "`: `"); - TypePtr declared_type = parse_type_from_tokens(lex); + AnyTypeV declared_type = parse_type_from_tokens(lex); AnyExprV default_value = nullptr; if (lex.tok() == tok_assign) { // `id: int = 3` @@ -1344,7 +1458,12 @@ static AnyV parse_import_directive(Lexer& lex) { return createV(loc, v_str); // semicolon is not necessary } -// the main (exported) function + +// -------------------------------------------- +// parse .tolk source file to AST +// (the main, exported, function) +// + AnyV parse_src_file_to_ast(const SrcFile* file) { std::vector toplevel_declarations; std::vector> annotations; diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 40cd8e4896..54d1f8e26e 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -96,7 +96,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } - virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } @@ -135,12 +135,12 @@ class ASTReplacerInFunctionBody : public ASTReplacer { virtual AnyV replace(V v) { return replace_children(v); } AnyExprV replace(AnyExprV v) final { - switch (v->type) { + switch (v->kind) { case ast_empty_expression: return replace(v->as()); case ast_parenthesized_expression: return replace(v->as()); case ast_braced_expression: return replace(v->as()); case ast_tensor: return replace(v->as()); - case ast_typed_tuple: return replace(v->as()); + case ast_bracket_tuple: return replace(v->as()); case ast_reference: return replace(v->as()); case ast_local_var_lhs: return replace(v->as()); case ast_local_vars_declaration: return replace(v->as()); @@ -167,12 +167,12 @@ class ASTReplacerInFunctionBody : public ASTReplacer { case ast_object_body: return replace(v->as()); case ast_object_literal: return replace(v->as()); default: - throw UnexpectedASTNodeType(v, "ASTReplacerInFunctionBody::replace"); + throw UnexpectedASTNodeKind(v, "ASTReplacerInFunctionBody::replace"); } } AnyV replace(AnyV v) final { - switch (v->type) { + switch (v->kind) { case ast_empty_statement: return replace(v->as()); case ast_block_statement: return replace(v->as()); case ast_return_statement: return replace(v->as()); @@ -185,7 +185,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { case ast_try_catch_statement: return replace(v->as()); #ifdef TOLK_DEBUG case ast_asm_body: - throw UnexpectedASTNodeType(v, "ASTReplacer::replace"); + throw UnexpectedASTNodeKind(v, "ASTReplacer::replace"); #endif default: { // be very careful, don't forget to handle all statements (not expressions) above! diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 1217fa45b2..3b0b458e4b 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -25,7 +25,7 @@ class ASTReplicator { protected: virtual AnyV clone(AnyV v) = 0; virtual AnyExprV clone(AnyExprV v) = 0; - virtual TypePtr clone(TypePtr) = 0; + virtual AnyTypeV clone(AnyTypeV v) = 0; public: virtual ~ASTReplicator() = default; @@ -53,6 +53,36 @@ class ASTReplicatorFunction : public ASTReplicator { return result; } + std::vector clone(const std::vector& items) { + std::vector result; + result.reserve(items.size()); + for (AnyTypeV item : items) { + result.push_back(clone(item)); + } + return result; + } + + // types + + virtual V clone(V v) { + return createV(v->loc, v->text); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_inner())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_items())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_items())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_params_and_return())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_variants())); + } + // expressions virtual V clone(V v) { @@ -67,14 +97,14 @@ class ASTReplicatorFunction : public ASTReplicator { virtual V clone(V v) { return createV(v->loc, clone(v->get_items())); } - virtual V clone(V v) { - return createV(v->loc, clone(v->get_items())); + virtual V clone(V v) { + return createV(v->loc, clone(v->get_items())); } virtual V clone(V v) { return createV(v->loc, clone(v->get_identifier()), v->has_instantiationTs() ? clone(v->get_instantiationTs()) : nullptr); } virtual V clone(V v) { - return createV(v->loc, clone(v->get_identifier()), clone(v->declared_type), v->is_immutable, v->marked_as_redef); + return createV(v->loc, clone(v->get_identifier()), clone(v->type_node), v->is_immutable, v->marked_as_redef); } virtual V clone(V v) { return createV(v->loc, clone(v->get_expr())); @@ -122,10 +152,10 @@ class ASTReplicatorFunction : public ASTReplicator { return createV(v->loc, clone(v->get_cond()), clone(v->get_when_true()), clone(v->get_when_false())); } virtual V clone(V v) { - return createV(v->loc, clone(v->get_expr()), clone(v->cast_to_type)); + return createV(v->loc, clone(v->get_expr()), clone(v->type_node)); } virtual V clone(V v) { - return createV(v->loc, clone(v->get_expr()), clone(v->rhs_type), v->is_negated); + return createV(v->loc, clone(v->get_expr()), clone(v->type_node), v->is_negated); } virtual V clone(V v) { return createV(v->loc, clone(v->get_expr())); @@ -134,7 +164,7 @@ class ASTReplicatorFunction : public ASTReplicator { return createV(v->loc, clone(v->get_all_children())); } virtual V clone(V v) { - return createV(v->loc, v->pattern_kind, clone(v->exact_type), clone(v->get_pattern_expr()), clone(v->get_body())); + return createV(v->loc, v->pattern_kind, clone(v->pattern_type_node), clone(v->get_pattern_expr()), clone(v->get_body())); } virtual V clone(V v) { return createV(v->loc, clone(v->get_field_identifier()), clone(v->get_init_val())); @@ -187,26 +217,48 @@ class ASTReplicatorFunction : public ASTReplicator { virtual V clone(V v) { return createV(v->loc, v->name); } + virtual V clone(V v) { + return createV(v->loc, v->nameT); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_items())); + } virtual V clone(V v) { - return createV(v->loc, clone(v->substituted_type)); + return createV(v->loc, clone(v->type_node)); } virtual V clone(V v) { return createV(v->loc, clone(v->get_items())); } virtual V clone(V v) { - return createV(v->loc, v->param_name, clone(v->declared_type), v->declared_as_mutate); + return createV(v->loc, v->param_name, clone(v->type_node), v->declared_as_mutate); } virtual V clone(V v) { return createV(v->loc, clone(v->get_params())); } + AnyTypeV clone(AnyTypeV v) final { + if (v == nullptr) { + return nullptr; + } + switch (v->kind) { + case ast_type_leaf_text: return clone(v->as()); + case ast_type_question_nullable: return clone(v->as()); + case ast_type_parenthesis_tensor: return clone(v->as()); + case ast_type_bracket_tuple: return clone(v->as()); + case ast_type_arrow_callable: return clone(v->as()); + case ast_type_vertical_bar_union: return clone(v->as()); + default: + throw UnexpectedASTNodeKind(v, "ASTReplicatorFunction::clone"); + } + } + AnyExprV clone(AnyExprV v) final { - switch (v->type) { + switch (v->kind) { case ast_empty_expression: return clone(v->as()); case ast_parenthesized_expression: return clone(v->as()); case ast_braced_expression: return clone(v->as()); case ast_tensor: return clone(v->as()); - case ast_typed_tuple: return clone(v->as()); + case ast_bracket_tuple: return clone(v->as()); case ast_reference: return clone(v->as()); case ast_local_var_lhs: return clone(v->as()); case ast_local_vars_declaration: return clone(v->as()); @@ -233,12 +285,12 @@ class ASTReplicatorFunction : public ASTReplicator { case ast_object_body: return clone(v->as()); case ast_object_literal: return clone(v->as()); default: - throw UnexpectedASTNodeType(v, "ASTReplicatorFunction::clone"); + throw UnexpectedASTNodeKind(v, "ASTReplicatorFunction::clone"); } } AnyV clone(AnyV v) final { - switch (v->type) { + switch (v->kind) { case ast_empty_statement: return clone(v->as()); case ast_block_statement: return clone(v->as()); case ast_return_statement: return clone(v->as()); @@ -252,6 +304,8 @@ class ASTReplicatorFunction : public ASTReplicator { case ast_asm_body: return clone(v->as()); // other AST nodes that can be children of ast nodes of function body case ast_identifier: return clone(v->as()); + case ast_genericsT_item: return clone(v->as()); + case ast_genericsT_list: return clone(v->as()); case ast_instantiationT_item: return clone(v->as()); case ast_instantiationT_list: return clone(v->as()); case ast_parameter: return clone(v->as()); @@ -265,21 +319,18 @@ class ASTReplicatorFunction : public ASTReplicator { } } - TypePtr clone(TypePtr t) override { - return t; - } - - public: - virtual V clone_function_body(V v_function) { +public: + // the cloned function becomes a deep copy, all AST nodes are copied, no previous pointers left + V clone_function_ast(V v_orig, V new_name_ident) { return createV( - v_function->loc, - clone(v_function->get_identifier()), - clone(v_function->get_param_list()), - clone(v_function->get_body()->as()), - clone(v_function->declared_return_type), - v_function->genericsT_list, - v_function->method_id, - v_function->flags + v_orig->loc, + new_name_ident, + clone(v_orig->get_param_list()), + clone(v_orig->get_body()), + clone(v_orig->return_type_node), + clone(v_orig->genericsT_list), + v_orig->method_id, + v_orig->flags ); } }; diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index 7782c506d4..ea991da20f 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -31,14 +31,21 @@ namespace tolk { class ASTStringifier final : public ASTVisitor { - constexpr static std::pair name_pairs[] = { + constexpr static std::pair name_pairs[] = { {ast_identifier, "ast_identifier"}, + // types + {ast_type_leaf_text, "ast_type_leaf_text"}, + {ast_type_question_nullable, "ast_type_question_nullable"}, + {ast_type_parenthesis_tensor, "ast_type_parenthesis_tensor"}, + {ast_type_bracket_tuple, "ast_type_bracket_tuple"}, + {ast_type_arrow_callable, "ast_type_arrow_callable"}, + {ast_type_vertical_bar_union, "ast_type_vertical_bar_union"}, // expressions {ast_empty_expression, "ast_empty_expression"}, {ast_parenthesized_expression, "ast_parenthesized_expression"}, {ast_braced_expression, "ast_braced_expression"}, {ast_tensor, "ast_tensor"}, - {ast_typed_tuple, "ast_typed_tuple"}, + {ast_bracket_tuple, "ast_bracket_tuple"}, {ast_reference, "ast_reference"}, {ast_local_var_lhs, "ast_local_var_lhs"}, {ast_local_vars_declaration, "ast_local_vars_declaration"}, @@ -108,19 +115,19 @@ class ASTStringifier final : public ASTVisitor { static_assert(std::size(annotation_kinds) == static_cast(AnnotationKind::unknown), "annotation_kinds needs to be updated"); - template - constexpr static const char* ast_node_type_to_string() { - return name_pairs[node_type].second; + template + constexpr static const char* ast_node_kind_to_string() { + return name_pairs[node_kind].second; } int depth = 0; std::string out; bool colored = false; - template - void handle_vertex(V v) { + template + void handle_vertex(V v) { out += std::string(depth * 2, ' '); - out += ast_node_type_to_string(); + out += ast_node_kind_to_string(); if (std::string postfix = specific_str(v); !postfix.empty()) { out += colored ? " \x1b[34m" : " // "; out += postfix; @@ -133,7 +140,9 @@ class ASTStringifier final : public ASTVisitor { } static std::string specific_str(AnyV v) { - switch (v->type) { + switch (v->kind) { + case ast_type_leaf_text: + return static_cast(v->as()->text); case ast_identifier: return static_cast(v->as()->name); case ast_reference: { @@ -170,7 +179,7 @@ class ASTStringifier final : public ASTVisitor { case ast_type_alias_declaration: return "type " + static_cast(v->as()->get_identifier()->name); case ast_struct_field: - return static_cast(v->as()->get_identifier()->name) + ": " + v->as()->declared_type->as_human_readable(); + return static_cast(v->as()->get_identifier()->name) + ": " + ast_type_node_to_string(v->as()->type_node); case ast_struct_declaration: return "struct " + static_cast(v->as()->get_identifier()->name); case ast_assign: @@ -182,24 +191,21 @@ class ASTStringifier final : public ASTVisitor { case ast_binary_operator: return static_cast(v->as()->operator_name); case ast_cast_as_operator: - return v->as()->cast_to_type->as_human_readable(); + return ast_type_node_to_string(v->as()->type_node); case ast_is_type_operator: { std::string prefix = v->as()->is_negated ? "!is " : "is "; - return prefix + v->as()->rhs_type->as_human_readable(); + return prefix + ast_type_node_to_string(v->as()->type_node); } case ast_block_statement: return "↓" + std::to_string(v->as()->get_items().size()); case ast_instantiationT_item: - return v->as()->substituted_type->as_human_readable(); + return ast_type_node_to_string(v->as()->type_node); case ast_if_statement: return v->as()->is_ifnot ? "ifnot" : ""; case ast_annotation: return annotation_kinds[static_cast(v->as()->kind)].second; - case ast_parameter: { - std::ostringstream os; - os << v->as()->declared_type; - return static_cast(v->as()->param_name) + ": " + os.str(); - } + case ast_parameter: + return static_cast(v->as()->param_name) + ": " + ast_type_node_to_string(v->as()->type_node); case ast_function_declaration: { std::string param_names; for (int i = 0; i < v->as()->get_num_params(); i++) { @@ -210,25 +216,24 @@ class ASTStringifier final : public ASTVisitor { return "fun " + static_cast(v->as()->get_identifier()->name) + "(" + param_names + ")"; } case ast_local_var_lhs: { - std::ostringstream os; - os << (v->as()->inferred_type ? v->as()->inferred_type->as_human_readable() : v->as()->declared_type->as_human_readable()); + std::string str_type = v->as()->inferred_type ? v->as()->inferred_type->as_human_readable() : ast_type_node_to_string(v->as()->type_node); if (v->as()->get_name().empty()) { - return "_: " + os.str(); + return "_: " + str_type; } - return static_cast(v->as()->get_name()) + ":" + os.str(); + return static_cast(v->as()->get_name()) + ": " + str_type; } case ast_instantiationT_list: { std::string result = "<"; for (AnyV item : v->as()->get_items()) { if (result.size() > 1) result += ","; - result += item->as()->substituted_type->as_human_readable(); + result += ast_type_node_to_string(item->as()->type_node); } return result + ">"; } case ast_match_arm: if (v->as()->pattern_kind == MatchArmKind::exact_type) { - return v->as()->exact_type->as_human_readable(); + return ast_type_node_to_string(v->as()->pattern_type_node); } if (v->as()->pattern_kind == MatchArmKind::const_expression) { return "(expression)"; @@ -260,7 +265,7 @@ class ASTStringifier final : public ASTVisitor { } static std::string to_string_without_children(AnyV v) { - std::string result = ast_node_type_to_string(v->type); + std::string result = ast_node_kind_to_string(v->kind); if (std::string postfix = specific_str(v); !postfix.empty()) { result += ' '; result += specific_str(v); @@ -268,19 +273,39 @@ class ASTStringifier final : public ASTVisitor { return result; } - static const char* ast_node_type_to_string(ASTNodeType node_type) { - return name_pairs[node_type].second; + static std::string ast_type_node_to_string(AnyTypeV type_node) { + if (type_node == nullptr) { + return ""; + } + if (auto v_leaf = type_node->try_as()) { + return static_cast(v_leaf->text); + } + if (auto v_nullable = type_node->try_as()) { + return ast_type_node_to_string(v_nullable->get_inner()) + "?"; + } + return ast_node_kind_to_string(type_node->kind); + } + + static const char* ast_node_kind_to_string(ASTNodeKind node_kind) { + return name_pairs[node_kind].second; } void visit(AnyV v) override { - switch (v->type) { + switch (v->kind) { case ast_identifier: return handle_vertex(v->as()); + // types + case ast_type_leaf_text: return handle_vertex(v->as()); + case ast_type_question_nullable: return handle_vertex(v->as()); + case ast_type_parenthesis_tensor: return handle_vertex(v->as()); + case ast_type_bracket_tuple: return handle_vertex(v->as()); + case ast_type_arrow_callable: return handle_vertex(v->as()); + case ast_type_vertical_bar_union: return handle_vertex(v->as()); // expressions case ast_empty_expression: return handle_vertex(v->as()); case ast_parenthesized_expression: return handle_vertex(v->as()); case ast_braced_expression: return handle_vertex(v->as()); case ast_tensor: return handle_vertex(v->as()); - case ast_typed_tuple: return handle_vertex(v->as()); + case ast_bracket_tuple: return handle_vertex(v->as()); case ast_reference: return handle_vertex(v->as()); case ast_local_var_lhs: return handle_vertex(v->as()); case ast_local_vars_declaration: return handle_vertex(v->as()); @@ -337,7 +362,7 @@ class ASTStringifier final : public ASTVisitor { case ast_import_directive: return handle_vertex(v->as()); case ast_tolk_file: return handle_vertex(v->as()); default: - throw UnexpectedASTNodeType(v, "ASTStringifier::visit"); + throw UnexpectedASTNodeKind(v, "ASTStringifier::visit"); } } }; diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index 12e65e2707..11ecbe80f7 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -21,7 +21,7 @@ /* * A module implementing base functionality of read-only traversing a vertex tree. - * Since a vertex in general doesn't store a vector of children, iterating is possible only for concrete node_type. + * Since a vertex in general doesn't store a vector of children, iterating is possible only for concrete node_kind. * E.g., for ast_if_statement, visit nodes cond, if-body and else-body. For ast_string_const, nothing. And so on. * Visitors below are helpers to inherit from and handle specific vertex types. * @@ -37,6 +37,16 @@ namespace tolk { class ASTVisitor { protected: + GNU_ATTRIBUTE_ALWAYS_INLINE static void visit_children(const ASTTypeLeaf* v) { + static_cast(v); + } + + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTTypeVararg* v) { + for (AnyTypeV child : v->children) { + visit(child); + } + } + GNU_ATTRIBUTE_ALWAYS_INLINE static void visit_children(const ASTExprLeaf* v) { static_cast(v); } @@ -95,7 +105,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } - virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } @@ -134,13 +144,13 @@ class ASTVisitorFunctionBody : public ASTVisitor { virtual void visit(V v) { return visit_children(v); } void visit(AnyV v) final { - switch (v->type) { + switch (v->kind) { // expressions case ast_empty_expression: return visit(v->as()); case ast_parenthesized_expression: return visit(v->as()); case ast_braced_expression: return visit(v->as()); case ast_tensor: return visit(v->as()); - case ast_typed_tuple: return visit(v->as()); + case ast_bracket_tuple: return visit(v->as()); case ast_reference: return visit(v->as()); case ast_local_var_lhs: return visit(v->as()); case ast_local_vars_declaration: return visit(v->as()); @@ -179,10 +189,10 @@ class ASTVisitorFunctionBody : public ASTVisitor { case ast_try_catch_statement: return visit(v->as()); #ifdef TOLK_DEBUG case ast_asm_body: - throw UnexpectedASTNodeType(v, "ASTVisitor; forgot to filter out asm functions in should_visit_function()?"); + throw UnexpectedASTNodeKind(v, "ASTVisitor; forgot to filter out asm functions in should_visit_function()?"); #endif default: - throw UnexpectedASTNodeType(v, "ASTVisitorFunctionBody::visit"); + throw UnexpectedASTNodeKind(v, "ASTVisitorFunctionBody::visit"); } } diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 399947446e..24c797bd2e 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -36,10 +36,10 @@ void ASTNodeBase::debug_print() const { #endif // TOLK_DEBUG -UnexpectedASTNodeType::UnexpectedASTNodeType(AnyV v_unexpected, const char* place_where): v_unexpected(v_unexpected) { - message = "Unexpected ASTNodeType "; +UnexpectedASTNodeKind::UnexpectedASTNodeKind(AnyV v_unexpected, const char* place_where): v_unexpected(v_unexpected) { + message = "Unexpected ASTNodeKind "; #ifdef TOLK_DEBUG - message += ASTStringifier::ast_node_type_to_string(v_unexpected->type); + message += ASTStringifier::ast_node_kind_to_string(v_unexpected->kind); message += " "; #endif message += "in "; @@ -105,6 +105,10 @@ int Vertex::get_mutate_params_count() const { // Therefore, there is a guarantee, that all AST mutations are done via these methods, // easily searched by usages, and there is no another way to modify any other field. +void ASTNodeDeclaredTypeBase::assign_resolved_type(TypePtr resolved_type) { + this->resolved_type = resolved_type; +} + void ASTNodeExpressionBase::assign_inferred_type(TypePtr type) { this->inferred_type = type; } @@ -134,72 +138,32 @@ void Vertex::assign_fun_ref(FunctionPtr fun_ref) { this->fun_maybe = fun_ref; } -void Vertex::assign_resolved_type(TypePtr cast_to_type) { - this->cast_to_type = cast_to_type; -} - -void Vertex::assign_resolved_type(TypePtr rhs_type) { - this->rhs_type = rhs_type; -} - void Vertex::assign_is_negated(bool is_negated) { this->is_negated = is_negated; } -void Vertex::assign_resolved_pattern(MatchArmKind pattern_kind, TypePtr exact_type, AnyExprV pattern_expr) { +void Vertex::assign_resolved_pattern(MatchArmKind pattern_kind, AnyExprV pattern_expr) { + this->pattern_type_node = nullptr; this->pattern_kind = pattern_kind; - this->exact_type = exact_type; this->lhs = pattern_expr; } -void Vertex::assign_var_ref(GlobalVarPtr var_ref) { - this->var_ref = var_ref; -} - -void Vertex::assign_resolved_type(TypePtr declared_type) { - this->declared_type = declared_type; +void Vertex::assign_glob_ref(GlobalVarPtr glob_ref) { + this->glob_ref = glob_ref; } void Vertex::assign_const_ref(GlobalConstPtr const_ref) { this->const_ref = const_ref; } -void Vertex::assign_resolved_type(TypePtr declared_type) { - this->declared_type = declared_type; -} - void Vertex::assign_alias_ref(AliasDefPtr alias_ref) { this->alias_ref = alias_ref; } -void Vertex::assign_resolved_type(TypePtr underlying_type) { - this->underlying_type = underlying_type; -} - -void Vertex::assign_field_ref(StructFieldPtr field_ref) { - this->field_ref = field_ref; -} - -void Vertex::assign_resolved_type(TypePtr declared_type) { - this->declared_type = declared_type; -} - void Vertex::assign_struct_ref(StructPtr struct_ref) { this->struct_ref = struct_ref; } -void Vertex::assign_resolved_type(TypePtr substituted_type) { - this->substituted_type = substituted_type; -} - -void Vertex::assign_param_ref(LocalVarPtr param_ref) { - this->param_ref = param_ref; -} - -void Vertex::assign_resolved_type(TypePtr declared_type) { - this->declared_type = declared_type; -} - void Vertex::assign_fun_ref(FunctionPtr fun_ref) { this->fun_ref = fun_ref; } @@ -232,18 +196,10 @@ void Vertex::assign_fun_ref(FunctionPtr fun_ref) { this->fun_ref = fun_ref; } -void Vertex::assign_resolved_type(TypePtr declared_return_type) { - this->declared_return_type = declared_return_type; -} - void Vertex::assign_var_ref(LocalVarPtr var_ref) { this->var_ref = var_ref; } -void Vertex::assign_resolved_type(TypePtr declared_type) { - this->declared_type = declared_type; -} - void Vertex::assign_src_file(const SrcFile* file) { this->file = file; } diff --git a/tolk/ast.h b/tolk/ast.h index 71bc9c40b9..dbe1e02b02 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -27,35 +27,33 @@ * Here we introduce AST representation of Tolk source code. * Historically, in FunC, there was no AST: while lexing, symbols were registered, types were inferred, and so on. * There was no way to perform any more or less semantic analysis. - * In Tolk, I've implemented parsing .tolk files into AST at first, and then converting this AST - * into legacy representation (see pipe-ast-to-legacy.cpp). - * In the future, more and more code analysis will be moved out of legacy to AST-level. + * In Tolk, all files are parsed into AST, and all semantic analysis is done at the AST level. * * From the user's point of view, all AST vertices are constant. All API is based on constancy. * Even though fields of vertex structs are public, they can't be modified, since vertices are accepted by const ref. * Generally, there are three ways of accepting a vertex: * * AnyV (= const ASTNodeBase*) - * the only you can do with this vertex is to see v->type (ASTNodeType) and to cast via v->as() + * the only you can do with this vertex is to see v->kind (ASTNodeKind) and to cast via v->as() * * AnyExprV (= const ASTNodeExpressionBase*) * in contains expression-specific properties (lvalue/rvalue, inferred type) - * * V (= const Vertex*) + * * V (= const Vertex*) * a specific type of vertex, you can use its fields and methods * There is one way of creating a vertex: - * * createV(...constructor_args) (= new Vertex(...)) + * * createV(...constructor_args) (= new Vertex(...)) * vertices are currently created on a heap, without any custom memory arena, just allocated and never deleted * The only way to modify a field is to use "mutate()" method (drops constancy, the only point of mutation) - * and then to call "assign_*" method, like "assign_sym", "assign_src_file", etc. + * and then to call "assign_*" method, like "assign_sym", "assign_src_file", etc. * - * Having AnyV and knowing its node_type, a call - * v->as() + * Having AnyV and knowing its node_kind, a call + * v->as() * will return a typed vertex. - * There is also a shorthand v->try_as() which returns V or nullptr if types don't match: + * There is also a shorthand v->try_as() which returns V or nullptr if types don't match: * if (auto v_int = v->try_as()) * Note, that there casts are NOT DYNAMIC. ASTNode is not a virtual base, it has no vtable. * So, as<...>() is just a compile-time casting, without any runtime overhead. * * Note, that ASTNodeBase doesn't store any vector of children. That's why there is no way to loop over - * a random (unknown) vertex. Only a concrete Vertex stores its children (if any). + * a random (unknown) vertex. Only a concrete Vertex stores its children (if any). * Hence, to iterate over a custom vertex (e.g., a function body), one should inherit some kind of ASTVisitor. * Besides read-only visiting, there is a "visit and replace" pattern. * See ast-visitor.h and ast-replacer.h. @@ -63,14 +61,21 @@ namespace tolk { -enum ASTNodeType { +enum ASTNodeKind { ast_identifier, + // types + ast_type_leaf_text, + ast_type_question_nullable, + ast_type_parenthesis_tensor, + ast_type_bracket_tuple, + ast_type_arrow_callable, + ast_type_vertical_bar_union, // expressions ast_empty_expression, ast_parenthesized_expression, ast_braced_expression, ast_tensor, - ast_typed_tuple, + ast_bracket_tuple, ast_reference, ast_local_var_lhs, ast_local_vars_declaration, @@ -143,19 +148,19 @@ enum class MatchArmKind { // for `match` expression, each of arms `pattern => else_branch, // `else => body` }; -template +template struct Vertex; -template -using V = const Vertex*; +template +using V = const Vertex*; #define createV new Vertex -struct UnexpectedASTNodeType final : std::exception { +struct UnexpectedASTNodeKind final : std::exception { AnyV v_unexpected; std::string message; - explicit UnexpectedASTNodeType(AnyV v_unexpected, const char* place_where); + explicit UnexpectedASTNodeKind(AnyV v_unexpected, const char* place_where); const char* what() const noexcept override { return message.c_str(); @@ -165,25 +170,25 @@ struct UnexpectedASTNodeType final : std::exception { // --------------------------------------------------------- struct ASTNodeBase { - const ASTNodeType type; + const ASTNodeKind kind; const SrcLocation loc; - ASTNodeBase(ASTNodeType type, SrcLocation loc) : type(type), loc(loc) {} + ASTNodeBase(ASTNodeKind kind, SrcLocation loc) : kind(kind), loc(loc) {} ASTNodeBase(const ASTNodeBase&) = delete; - template - V as() const { + template + V as() const { #ifdef TOLK_DEBUG - if (type != node_type) { - throw Fatal("v->as<...> to wrong node_type"); + if (kind != node_kind) { + throw Fatal("v->as<...> to wrong node_kind"); } #endif - return static_cast>(this); + return static_cast>(this); } - template - V try_as() const { - return type == node_type ? static_cast>(this) : nullptr; + template + V try_as() const { + return kind == node_kind ? static_cast>(this) : nullptr; } #ifdef TOLK_DEBUG @@ -196,9 +201,16 @@ struct ASTNodeBase { void error(const std::string& err_msg) const; }; -struct ASTNodeExpressionBase : ASTNodeBase { - friend class ASTDuplicatorFunction; +struct ASTNodeDeclaredTypeBase : ASTNodeBase { + TypePtr resolved_type = nullptr; + + ASTNodeDeclaredTypeBase* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr resolved_type); + + ASTNodeDeclaredTypeBase(ASTNodeKind kind, SrcLocation loc) : ASTNodeBase(kind, loc) {} +}; +struct ASTNodeExpressionBase : ASTNodeBase { TypePtr inferred_type = nullptr; bool is_rvalue: 1 = false; bool is_lvalue: 1 = false; @@ -211,11 +223,27 @@ struct ASTNodeExpressionBase : ASTNodeBase { void assign_lvalue_true(); void assign_always_true_or_false(int flow_true_false_state); - ASTNodeExpressionBase(ASTNodeType type, SrcLocation loc) : ASTNodeBase(type, loc) {} + ASTNodeExpressionBase(ASTNodeKind kind, SrcLocation loc) : ASTNodeBase(kind, loc) {} }; struct ASTNodeStatementBase : ASTNodeBase { - ASTNodeStatementBase(ASTNodeType type, SrcLocation loc) : ASTNodeBase(type, loc) {} + ASTNodeStatementBase(ASTNodeKind kind, SrcLocation loc) : ASTNodeBase(kind, loc) {} +}; + +struct ASTTypeLeaf : ASTNodeDeclaredTypeBase { +protected: + ASTTypeLeaf(ASTNodeKind kind, SrcLocation loc) + : ASTNodeDeclaredTypeBase(kind, loc) {} +}; + +struct ASTTypeVararg : ASTNodeDeclaredTypeBase { + friend class ASTVisitor; + +protected: + std::vector children; + + ASTTypeVararg(ASTNodeKind kind, SrcLocation loc, std::vector&& children) + : ASTNodeDeclaredTypeBase(kind, loc), children(std::move(children)) {} }; struct ASTExprLeaf : ASTNodeExpressionBase { @@ -223,8 +251,8 @@ struct ASTExprLeaf : ASTNodeExpressionBase { friend class ASTReplacer; protected: - ASTExprLeaf(ASTNodeType type, SrcLocation loc) - : ASTNodeExpressionBase(type, loc) {} + ASTExprLeaf(ASTNodeKind kind, SrcLocation loc) + : ASTNodeExpressionBase(kind, loc) {} }; struct ASTExprUnary : ASTNodeExpressionBase { @@ -234,8 +262,8 @@ struct ASTExprUnary : ASTNodeExpressionBase { protected: AnyExprV child; - ASTExprUnary(ASTNodeType type, SrcLocation loc, AnyExprV child) - : ASTNodeExpressionBase(type, loc), child(child) {} + ASTExprUnary(ASTNodeKind kind, SrcLocation loc, AnyExprV child) + : ASTNodeExpressionBase(kind, loc), child(child) {} }; struct ASTExprBinary : ASTNodeExpressionBase { @@ -246,8 +274,8 @@ struct ASTExprBinary : ASTNodeExpressionBase { AnyExprV lhs; AnyExprV rhs; - ASTExprBinary(ASTNodeType type, SrcLocation loc, AnyExprV lhs, AnyExprV rhs) - : ASTNodeExpressionBase(type, loc), lhs(lhs), rhs(rhs) {} + ASTExprBinary(ASTNodeKind kind, SrcLocation loc, AnyExprV lhs, AnyExprV rhs) + : ASTNodeExpressionBase(kind, loc), lhs(lhs), rhs(rhs) {} }; struct ASTExprVararg : ASTNodeExpressionBase { @@ -259,8 +287,8 @@ struct ASTExprVararg : ASTNodeExpressionBase { AnyExprV child(int i) const { return children.at(i); } - ASTExprVararg(ASTNodeType type, SrcLocation loc, std::vector&& children) - : ASTNodeExpressionBase(type, loc), children(std::move(children)) {} + ASTExprVararg(ASTNodeKind kind, SrcLocation loc, std::vector&& children) + : ASTNodeExpressionBase(kind, loc), children(std::move(children)) {} public: int size() const { return static_cast(children.size()); } @@ -274,8 +302,8 @@ struct ASTExprBlockOfStatements : ASTNodeExpressionBase { protected: AnyV child_block_statement; - ASTExprBlockOfStatements(ASTNodeType type, SrcLocation loc, AnyV child_block_statement) - : ASTNodeExpressionBase(type, loc), child_block_statement(child_block_statement) {} + ASTExprBlockOfStatements(ASTNodeKind kind, SrcLocation loc, AnyV child_block_statement) + : ASTNodeExpressionBase(kind, loc), child_block_statement(child_block_statement) {} }; struct ASTStatementUnary : ASTNodeStatementBase { @@ -287,8 +315,8 @@ struct ASTStatementUnary : ASTNodeStatementBase { AnyExprV child_as_expr() const { return reinterpret_cast(child); } - ASTStatementUnary(ASTNodeType type, SrcLocation loc, AnyV child) - : ASTNodeStatementBase(type, loc), child(child) {} + ASTStatementUnary(ASTNodeKind kind, SrcLocation loc, AnyV child) + : ASTNodeStatementBase(kind, loc), child(child) {} }; struct ASTStatementVararg : ASTNodeStatementBase { @@ -300,8 +328,8 @@ struct ASTStatementVararg : ASTNodeStatementBase { AnyExprV child_as_expr(int i) const { return reinterpret_cast(children.at(i)); } - ASTStatementVararg(ASTNodeType type, SrcLocation loc, std::vector children) - : ASTNodeStatementBase(type, loc), children(std::move(children)) {} + ASTStatementVararg(ASTNodeKind kind, SrcLocation loc, std::vector&& children) + : ASTNodeStatementBase(kind, loc), children(std::move(children)) {} public: int size() const { return static_cast(children.size()); } @@ -313,8 +341,8 @@ struct ASTOtherLeaf : ASTNodeBase { friend class ASTReplacer; protected: - ASTOtherLeaf(ASTNodeType type, SrcLocation loc) - : ASTNodeBase(type, loc) {} + ASTOtherLeaf(ASTNodeKind kind, SrcLocation loc) + : ASTNodeBase(kind, loc) {} }; struct ASTOtherVararg : ASTNodeBase { @@ -326,8 +354,8 @@ struct ASTOtherVararg : ASTNodeBase { AnyExprV child_as_expr(int i) const { return reinterpret_cast(children.at(i)); } - ASTOtherVararg(ASTNodeType type, SrcLocation loc, std::vector children) - : ASTNodeBase(type, loc), children(std::move(children)) {} + ASTOtherVararg(ASTNodeKind kind, SrcLocation loc, std::vector&& children) + : ASTNodeBase(kind, loc), children(std::move(children)) {} public: int size() const { return static_cast(children.size()); } @@ -352,6 +380,74 @@ struct Vertex final : ASTOtherLeaf { }; +// +// --------------------------------------------------------- +// types +// + + +template<> +// ast_type_leaf_text is a type node without children: "int", "User", "T", etc. +// after resolving, it will become TypeDataInt / TypeDataStruct / etc. +struct Vertex final : ASTTypeLeaf { + std::string_view text; + + Vertex(SrcLocation loc, std::string_view text) + : ASTTypeLeaf(ast_type_leaf_text, loc) + , text(text) {} +}; + +template<> +// ast_type_question_nullable is "T?" +// after resolving, it will become a union "T | null", but at AST level, it's a separate node +struct Vertex final : ASTTypeVararg { + AnyTypeV get_inner() const { return children.at(0); } + + Vertex(SrcLocation loc, AnyTypeV inner) + : ASTTypeVararg(ast_type_question_nullable, loc, {inner}) {} +}; + +template<> +// ast_type_parenthesis_tensor is "(T1, T2, ...)" +// after resolving, it will become TypeDataTensor +struct Vertex final : ASTTypeVararg { + const std::vector& get_items() const { return children; } + + Vertex(SrcLocation loc, std::vector&& items) + : ASTTypeVararg(ast_type_parenthesis_tensor, loc, std::move(items)) {} +}; + +template<> +// ast_type_bracket_tuple is "[T1, T2, ...]" +// after resolving, it will become TypeDataBrackets +struct Vertex final : ASTTypeVararg { + const std::vector& get_items() const { return children; } + + Vertex(SrcLocation loc, std::vector&& items) + : ASTTypeVararg(ast_type_bracket_tuple, loc, std::move(items)) {} +}; + +template<> +// ast_type_arrow_callable is "(T1, T2, ...) -> TResult" +// after resolving, it will become TypeDataFunCallable +struct Vertex final : ASTTypeVararg { + const std::vector& get_params_and_return() const { return children; } + + Vertex(SrcLocation loc, std::vector&& params_and_return) + : ASTTypeVararg(ast_type_arrow_callable, loc, std::move(params_and_return)) {} +}; + +template<> +// ast_type_vertical_bar_union is "T1 | T2 | ..." +// after resolving, it will become TypeDataUnion +struct Vertex final : ASTTypeVararg { + const std::vector& get_variants() const { return children; } + + Vertex(SrcLocation loc, std::vector&& variants) + : ASTTypeVararg(ast_type_vertical_bar_union, loc, std::move(variants)) {} +}; + + // // --------------------------------------------------------- // expressions @@ -404,16 +500,16 @@ struct Vertex final : ASTExprVararg { }; template<> -// ast_typed_tuple is a set of expressions in [square brackets] +// ast_bracket_tuple is a set of expressions in [square brackets] // in TVM, it's a TVM tuple, that occupies 1 slot, but the compiler knows its "typed structure" // example: `[1, x]`, `[[0]]` (nested) // typed tuples can be assigned to N variables, like `[one, _, three] = [1,2,3]` -struct Vertex final : ASTExprVararg { +struct Vertex final : ASTExprVararg { const std::vector& get_items() const { return children; } AnyExprV get_item(int i) const { return child(i); } Vertex(SrcLocation loc, std::vector items) - : ASTExprVararg(ast_typed_tuple, loc, std::move(items)) {} + : ASTExprVararg(ast_bracket_tuple, loc, std::move(items)) {} }; template<> @@ -453,7 +549,7 @@ struct Vertex final : ASTExprLeaf { public: LocalVarPtr var_ref = nullptr; // filled on resolve identifiers; for `redef` points to declared above; for underscore, name is empty - TypePtr declared_type; // not null for `var x: int = rhs`, otherwise nullptr + AnyTypeV type_node; // exists for `var x: int = rhs`, otherwise nullptr bool is_immutable; // declared via 'val', not 'var' bool marked_as_redef; // var (existing_var redef, new_var: int) = ... @@ -462,11 +558,10 @@ struct Vertex final : ASTExprLeaf { Vertex* mutate() const { return const_cast(this); } void assign_var_ref(LocalVarPtr var_ref); - void assign_resolved_type(TypePtr declared_type); - Vertex(SrcLocation loc, V identifier, TypePtr declared_type, bool is_immutable, bool marked_as_redef) + Vertex(SrcLocation loc, V identifier, AnyTypeV type_node, bool is_immutable, bool marked_as_redef) : ASTExprLeaf(ast_local_var_lhs, loc) - , identifier(identifier), declared_type(declared_type), is_immutable(is_immutable), marked_as_redef(marked_as_redef) {} + , identifier(identifier), type_node(type_node), is_immutable(is_immutable), marked_as_redef(marked_as_redef) {} }; template<> @@ -475,7 +570,7 @@ template<> // for `var (x, [y])` its expr is "tensor (local var, typed tuple (local var))" // for assignment `var x = 5`, this node is `var x`, lhs of assignment struct Vertex final : ASTExprUnary { - AnyExprV get_expr() const { return child; } // ast_local_var_lhs / ast_tensor / ast_typed_tuple + AnyExprV get_expr() const { return child; } // ast_local_var_lhs / ast_tensor / ast_bracket_tuple Vertex(SrcLocation loc, AnyExprV expr) : ASTExprUnary(ast_local_vars_declaration, loc, expr) {} @@ -602,7 +697,7 @@ struct Vertex final : ASTExprBinary { FunctionPtr fun_maybe = nullptr; // filled while type inferring for `globalF()` / `obj.f()`; remains nullptr for `local_var()` / `getF()()` AnyExprV get_callee() const { return lhs; } - bool is_dot_call() const { return lhs->type == ast_dot_access; } + bool is_dot_call() const { return lhs->kind == ast_dot_access; } AnyExprV get_dot_obj() const { return lhs->as()->get_obj(); } auto get_arg_list() const { return rhs->as(); } int get_num_args() const { return rhs->as()->size(); } @@ -711,16 +806,13 @@ template<> // ast_cast_as_operator is explicit casting with "as" keyword // examples: `arg as int` / `null as cell` / `t.tupleAt(2) as slice` struct Vertex final : ASTExprUnary { - AnyExprV get_expr() const { return child; } - - TypePtr cast_to_type; + AnyTypeV type_node; - Vertex* mutate() const { return const_cast(this); } - void assign_resolved_type(TypePtr cast_to_type); + AnyExprV get_expr() const { return child; } - Vertex(SrcLocation loc, AnyExprV expr, TypePtr cast_to_type) + Vertex(SrcLocation loc, AnyExprV expr, AnyTypeV type_node) : ASTExprUnary(ast_cast_as_operator, loc, expr) - , cast_to_type(cast_to_type) {} + , type_node(type_node) {} }; template<> @@ -729,16 +821,15 @@ template<> struct Vertex final : ASTExprUnary { AnyExprV get_expr() const { return child; } - TypePtr rhs_type; - bool is_negated; // `!is type`, `!= null` + AnyTypeV type_node; + bool is_negated; // `!is type`, `!= null` Vertex* mutate() const { return const_cast(this); } - void assign_resolved_type(TypePtr rhs_type); void assign_is_negated(bool is_negated); - Vertex(SrcLocation loc, AnyExprV expr, TypePtr rhs_type, bool is_negated) + Vertex(SrcLocation loc, AnyExprV expr, AnyTypeV type_node, bool is_negated) : ASTExprUnary(ast_is_type_operator, loc, expr) - , rhs_type(rhs_type), is_negated(is_negated) {} + , type_node(type_node), is_negated(is_negated) {} }; template<> @@ -775,17 +866,17 @@ template<> // example: `a+b => { ...; return 0; }` (match by expression, inferred_type of body is "never" (unreachable end)) struct Vertex final : ASTExprBinary { MatchArmKind pattern_kind; - TypePtr exact_type; // for MatchArmKind::exact_type; otherwise, nullptr + AnyTypeV pattern_type_node; // for MatchArmKind::exact_type; otherwise nullptr AnyExprV get_pattern_expr() const { return lhs; } AnyExprV get_body() const { return rhs; } // remember, it may be V Vertex* mutate() const { return const_cast(this); } - void assign_resolved_pattern(MatchArmKind pattern_kind, TypePtr exact_type, AnyExprV pattern_expr); + void assign_resolved_pattern(MatchArmKind pattern_kind, AnyExprV pattern_expr); - Vertex(SrcLocation loc, MatchArmKind pattern_kind, TypePtr exact_type, AnyExprV pattern_expr, AnyExprV body) + Vertex(SrcLocation loc, MatchArmKind pattern_kind, AnyTypeV pattern_type_node, AnyExprV pattern_expr, AnyExprV body) : ASTExprBinary(ast_match_arm, loc, pattern_expr, body) - , pattern_kind(pattern_kind), exact_type(exact_type) {} + , pattern_kind(pattern_kind), pattern_type_node(pattern_type_node) {} }; template<> @@ -829,7 +920,7 @@ template<> struct Vertex final : ASTExprBinary { StructPtr struct_ref = nullptr; // assigned at type inferring - bool has_explicit_ref() const { return lhs->type != ast_empty_expression; } + bool has_explicit_ref() const { return lhs->kind != ast_empty_expression; } auto get_explicit_ref() const { return lhs->as(); } AnyExprV get_maybe_reference() const { return lhs; } // ast_empty_expression or ast_reference auto get_body() const { return rhs->as(); } @@ -882,7 +973,7 @@ template<> // note, that for `return;` (without a value, meaning "void"), in AST, it's stored as empty expression struct Vertex : ASTStatementUnary { AnyExprV get_return_value() const { return child_as_expr(); } - bool has_return_value() const { return child->type != ast_empty_expression; } + bool has_return_value() const { return child->kind != ast_empty_expression; } Vertex(SrcLocation loc, AnyExprV child) : ASTStatementUnary(ast_return_statement, loc, child) {} @@ -944,7 +1035,7 @@ template<> // when thrown arg is missing, it's stored as empty expression struct Vertex final : ASTStatementVararg { AnyExprV get_thrown_code() const { return child_as_expr(0); } - bool has_thrown_arg() const { return child_as_expr(1)->type != ast_empty_expression; } + bool has_thrown_arg() const { return child_as_expr(1)->kind != ast_empty_expression; } AnyExprV get_thrown_arg() const { return child_as_expr(1); } Vertex(SrcLocation loc, AnyExprV thrown_code, AnyExprV thrown_arg) @@ -1028,14 +1119,11 @@ template<> // ast_instantiationT_item is manual substitution of generic T used in code, mostly for func calls // examples: `g()` / `t.tupleFirst()` / `f<(int, slice), builder>()` struct Vertex final : ASTOtherLeaf { - TypePtr substituted_type; + AnyTypeV type_node; - Vertex* mutate() const { return const_cast(this); } - void assign_resolved_type(TypePtr substituted_type); - - Vertex(SrcLocation loc, TypePtr substituted_type) + Vertex(SrcLocation loc, AnyTypeV type_node) : ASTOtherLeaf(ast_instantiationT_item, loc) - , substituted_type(substituted_type) {} + , type_node(type_node) {} }; template<> @@ -1053,20 +1141,15 @@ template<> // ast_parameter is a parameter of a function in its declaration // example: `fun f(a: int, mutate b: slice)` has 2 parameters struct Vertex final : ASTOtherLeaf { - LocalVarPtr param_ref = nullptr; // filled on resolve identifiers std::string_view param_name; - TypePtr declared_type; + AnyTypeV type_node; // always exists, typing parameters is mandatory bool declared_as_mutate; // declared as `mutate param_name` bool is_underscore() const { return param_name.empty(); } - Vertex* mutate() const { return const_cast(this); } - void assign_param_ref(LocalVarPtr param_ref); - void assign_resolved_type(TypePtr declared_type); - - Vertex(SrcLocation loc, std::string_view param_name, TypePtr declared_type, bool declared_as_mutate) + Vertex(SrcLocation loc, std::string_view param_name, AnyTypeV type_node, bool declared_as_mutate) : ASTOtherLeaf(ast_parameter, loc) - , param_name(param_name), declared_type(declared_type), declared_as_mutate(declared_as_mutate) {} + , param_name(param_name), type_node(type_node), declared_as_mutate(declared_as_mutate) {} }; template<> @@ -1107,28 +1190,27 @@ template<> // their body is either sequence (regular code function), or `asm`, or `builtin` struct Vertex final : ASTOtherVararg { auto get_identifier() const { return children.at(0)->as(); } - int get_num_params() const { return children.at(1)->as()->size(); } + int get_num_params() const { return children.at(1)->as()->size(); } auto get_param_list() const { return children.at(1)->as(); } auto get_param(int i) const { return children.at(1)->as()->get_param(i); } AnyV get_body() const { return children.at(2); } // ast_block_statement / ast_asm_body FunctionPtr fun_ref = nullptr; // filled after register - TypePtr declared_return_type; // filled at ast parsing; if unspecified (nullptr), means "auto infer" + AnyTypeV return_type_node; // if unspecified (nullptr), means "auto infer" V genericsT_list; // for non-generics it's nullptr td::RefInt256 method_id; // specified via @method_id annotation int flags; // from enum in FunctionData - bool is_asm_function() const { return children.at(2)->type == ast_asm_body; } - bool is_code_function() const { return children.at(2)->type == ast_block_statement; } - bool is_builtin_function() const { return children.at(2)->type == ast_empty_statement; } + bool is_asm_function() const { return children.at(2)->kind == ast_asm_body; } + bool is_code_function() const { return children.at(2)->kind == ast_block_statement; } + bool is_builtin_function() const { return children.at(2)->kind == ast_empty_statement; } Vertex* mutate() const { return const_cast(this); } void assign_fun_ref(FunctionPtr fun_ref); - void assign_resolved_type(TypePtr declared_return_type); - Vertex(SrcLocation loc, V name_identifier, V parameters, AnyV body, TypePtr declared_return_type, V genericsT_list, td::RefInt256 method_id, int flags) + Vertex(SrcLocation loc, V name_identifier, V parameters, AnyV body, AnyTypeV return_type_node, V genericsT_list, td::RefInt256 method_id, int flags) : ASTOtherVararg(ast_function_declaration, loc, {name_identifier, parameters, body}) - , declared_return_type(declared_return_type), genericsT_list(genericsT_list), method_id(std::move(method_id)), flags(flags) {} + , return_type_node(return_type_node), genericsT_list(genericsT_list), method_id(std::move(method_id)), flags(flags) {} }; template<> @@ -1136,18 +1218,17 @@ template<> // example: `global g: int;` // note, that globals don't have default values, since there is no single "entrypoint" for a contract struct Vertex final : ASTOtherVararg { - GlobalVarPtr var_ref = nullptr; // filled after register - TypePtr declared_type; // filled always, typing globals is mandatory + GlobalVarPtr glob_ref = nullptr; // filled after register + AnyTypeV type_node; // always exists, typing globals is mandatory auto get_identifier() const { return children.at(0)->as(); } Vertex* mutate() const { return const_cast(this); } - void assign_var_ref(GlobalVarPtr var_ref); - void assign_resolved_type(TypePtr declared_type); + void assign_glob_ref(GlobalVarPtr glob_ref); - Vertex(SrcLocation loc, V name_identifier, TypePtr declared_type) + Vertex(SrcLocation loc, V name_identifier, AnyTypeV type_node) : ASTOtherVararg(ast_global_var_declaration, loc, {name_identifier}) - , declared_type(declared_type) {} + , type_node(type_node) {} }; template<> @@ -1155,18 +1236,17 @@ template<> // example: `const op = 0x123;` struct Vertex final : ASTOtherVararg { GlobalConstPtr const_ref = nullptr; // filled after register - TypePtr declared_type; // not null for `const op: int = ...` + AnyTypeV type_node; // exists for `const op: int = rhs`, otherwise nullptr auto get_identifier() const { return children.at(0)->as(); } AnyExprV get_init_value() const { return child_as_expr(1); } Vertex* mutate() const { return const_cast(this); } void assign_const_ref(GlobalConstPtr const_ref); - void assign_resolved_type(TypePtr declared_type); - Vertex(SrcLocation loc, V name_identifier, TypePtr declared_type, AnyExprV init_value) + Vertex(SrcLocation loc, V name_identifier, AnyTypeV type_node, AnyExprV init_value) : ASTOtherVararg(ast_constant_declaration, loc, {name_identifier, init_value}) - , declared_type(declared_type) {} + , type_node(type_node) {} }; template<> @@ -1174,38 +1254,32 @@ template<> // example: `type UserId = int;` // see TypeDataAlias in type-system.h struct Vertex final : ASTOtherVararg { - AliasDefPtr alias_ref = nullptr; // filled after register, contains TypeDataAlias - TypePtr underlying_type; // at the right of `=` + AliasDefPtr alias_ref = nullptr; // filled after register + AnyTypeV underlying_type_node; // at the right of `=` auto get_identifier() const { return children.at(0)->as(); } Vertex* mutate() const { return const_cast(this); } void assign_alias_ref(AliasDefPtr alias_ref); - void assign_resolved_type(TypePtr underlying_type); - Vertex(SrcLocation loc, V name_identifier, TypePtr underlying_type) + Vertex(SrcLocation loc, V name_identifier, AnyTypeV underlying_type_node) : ASTOtherVararg(ast_type_alias_declaration, loc, {name_identifier}) - , underlying_type(underlying_type) {} + , underlying_type_node(underlying_type_node) {} }; template<> // ast_struct_field is one field at struct declaration // example: `struct Point { x: int, y: int }` is struct declaration, its body contains 2 fields struct Vertex final : ASTOtherVararg { - StructFieldPtr field_ref = nullptr; // filled after register - TypePtr declared_type; + AnyTypeV type_node; // always exists, typing struct fields is mandatory auto get_identifier() const { return children.at(0)->as(); } - bool has_default_value() const { return children.at(1)->type != ast_empty_expression; } + bool has_default_value() const { return children.at(1)->kind != ast_empty_expression; } auto get_default_value() const { return child_as_expr(1); } - Vertex* mutate() const { return const_cast(this); } - void assign_field_ref(StructFieldPtr field_ref); - void assign_resolved_type(TypePtr declared_type); - - Vertex(SrcLocation loc, V name_identifier, AnyExprV default_value, TypePtr declared_type) + Vertex(SrcLocation loc, V name_identifier, AnyExprV default_value, AnyTypeV type_node) : ASTOtherVararg(ast_struct_field, loc, {name_identifier, default_value}) - , declared_type(declared_type) {} + , type_node(type_node) {} }; template<> diff --git a/tolk/fwd-declarations.h b/tolk/fwd-declarations.h index ff2e32a367..cda17c4247 100644 --- a/tolk/fwd-declarations.h +++ b/tolk/fwd-declarations.h @@ -19,10 +19,12 @@ namespace tolk { struct ASTNodeBase; +struct ASTNodeDeclaredTypeBase; struct ASTNodeExpressionBase; struct ASTNodeStatementBase; using AnyV = const ASTNodeBase*; +using AnyTypeV = const ASTNodeDeclaredTypeBase*; using AnyExprV = const ASTNodeExpressionBase*; using AnyStatementV = const ASTNodeStatementBase*; diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index 94a43e9fc6..e73639b866 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -97,9 +97,9 @@ void GenericSubstitutionsDeduceForCall::consider_next_condition(TypePtr param_ty consider_next_condition(p_tensor->items[i], a_tensor->items[i]); } } - } else if (const auto* p_tuple = param_type->try_as()) { + } else if (const auto* p_tuple = param_type->try_as()) { // `arg: [int, T]` called as `f([5, cs])` => T is slice - if (const auto* a_tuple = arg_type->unwrap_alias()->try_as(); a_tuple && a_tuple->size() == p_tuple->size()) { + if (const auto* a_tuple = arg_type->unwrap_alias()->try_as(); a_tuple && a_tuple->size() == p_tuple->size()) { for (int i = 0; i < a_tuple->size(); ++i) { consider_next_condition(p_tuple->items[i], a_tuple->items[i]); } @@ -146,41 +146,6 @@ int GenericSubstitutionsDeduceForCall::get_first_not_deduced_idx() const { return -1; } -// clone the body of `f` replacing T everywhere with a substitution -// before: `fun f(v: T) { var cp: [T] = [v]; }` -// after: `fun f(v: int) { var cp: [int] = [v]; }` -// an instantiated function becomes a deep copy, all AST nodes are copied, no previous pointers left -class GenericFunctionReplicator final : public ASTReplicatorFunction { - const GenericsDeclaration* genericTs; - const std::vector& substitutionTs; - -protected: - using ASTReplicatorFunction::clone; - - TypePtr clone(TypePtr t) override { - return replace_genericT_with_deduced(t, genericTs, substitutionTs); - } - -public: - GenericFunctionReplicator(const GenericsDeclaration* genericTs, const std::vector& substitutionTs) - : genericTs(genericTs) - , substitutionTs(substitutionTs) { - } - - V clone_function_body(V v_function) override { - return createV( - v_function->loc, - clone(v_function->get_identifier()), - clone(v_function->get_param_list()), - clone(v_function->get_body()), - clone(v_function->declared_return_type), - nullptr, // a newly-created function is not generic - v_function->method_id, - v_function->flags - ); - } -}; - std::string GenericsDeclaration::as_human_readable() const { std::string result = "<"; for (const GenericsItem& item : itemsT) { @@ -202,16 +167,9 @@ int GenericsDeclaration::find_nameT(std::string_view nameT) const { return -1; } -// after creating a deep copy of `f` like `f`, its new and fresh body needs the previous pipeline to run -// for example, all local vars need to be registered as symbols, etc. -static void run_pipeline_for_instantiated_function(FunctionPtr inst_fun_ref) { - // these pipes are exactly the same as in tolk.cpp — all preceding (and including) type inferring - pipeline_resolve_identifiers_and_assign_symbols(inst_fun_ref); - pipeline_calculate_rvalue_lvalue(inst_fun_ref); - pipeline_infer_types_and_calls_and_fields(inst_fun_ref); -} - -std::string generate_instantiated_name(const std::string& orig_name, const std::vector& substitutions) { +// when cloning `f`, original name is "f", we need a new name for symtable and output +// name of an instantiated function will be "f" and similar (yes, with "<" symbol, it's okay to Fift) +static std::string generate_instantiated_name(const std::string& orig_name, const std::vector& substitutions) { // an instantiated function name will be "{orig_name}<{T1,T2,...}>" std::string name = orig_name; name += "<"; @@ -226,53 +184,56 @@ std::string generate_instantiated_name(const std::string& orig_name, const std:: return name; } -FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, const std::string& inst_name, std::vector&& substitutionTs) { - tolk_assert(fun_ref->genericTs); +FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, std::vector&& substitutionTs) { + tolk_assert(fun_ref->is_generic_function() && !fun_ref->is_method_id_not_empty()); - // if `f` was earlier instantiated, return it - if (const auto* existing = lookup_global_symbol(inst_name)) { - FunctionPtr inst_ref = existing->try_as(); - tolk_assert(inst_ref); - return inst_ref; + // fun_ref->name = "f", inst_name will be "f" and similar + std::string new_name = generate_instantiated_name(fun_ref->name, substitutionTs); + if (const Symbol* existing_sym = lookup_global_symbol(new_name)) { + FunctionPtr existing_ref = existing_sym->try_as(); + tolk_assert(existing_ref); + return existing_ref; } - std::vector parameters; - parameters.reserve(fun_ref->get_num_params()); - for (const LocalVarData& orig_p : fun_ref->parameters) { - parameters.emplace_back(orig_p.name, orig_p.loc, replace_genericT_with_deduced(orig_p.declared_type, fun_ref->genericTs, substitutionTs), orig_p.flags, orig_p.param_idx); - } - TypePtr declared_return_type = replace_genericT_with_deduced(fun_ref->declared_return_type, fun_ref->genericTs, substitutionTs); const GenericsInstantiation* instantiationTs = new GenericsInstantiation(loc, std::move(substitutionTs)); + ASTReplicatorFunction replicator; - if (fun_ref->is_asm_function()) { - FunctionData* inst_ref = new FunctionData(inst_name, fun_ref->loc, declared_return_type, std::move(parameters), fun_ref->flags, nullptr, instantiationTs, new FunctionBodyAsm, fun_ref->ast_root); - inst_ref->arg_order = fun_ref->arg_order; - inst_ref->ret_order = fun_ref->ret_order; - G.symtable.add_function(inst_ref); - G.all_functions.push_back(inst_ref); - run_pipeline_for_instantiated_function(inst_ref); - return inst_ref; - } - + // built-in functions don't have AST to clone, types of parameters don't exist in AST, etc. + // nevertheless, for outer code to follow a single algorithm, + // when calling `debugPrint(x)`, we clone it as "debugPrint", replace types, and insert into symtable if (fun_ref->is_builtin_function()) { - FunctionData* inst_ref = new FunctionData(inst_name, fun_ref->loc, declared_return_type, std::move(parameters), fun_ref->flags, nullptr, instantiationTs, fun_ref->body, fun_ref->ast_root); - inst_ref->arg_order = fun_ref->arg_order; - inst_ref->ret_order = fun_ref->ret_order; - G.symtable.add_function(inst_ref); - return inst_ref; - } - - GenericFunctionReplicator replicator(fun_ref->genericTs, instantiationTs->substitutions); - V inst_root = replicator.clone_function_body(fun_ref->ast_root->as()); + std::vector new_parameters; + new_parameters.reserve(fun_ref->get_num_params()); + for (const LocalVarData& orig_p : fun_ref->parameters) { + TypePtr new_param_type = replace_genericT_with_deduced(orig_p.declared_type, fun_ref->genericTs, instantiationTs->substitutions); + new_parameters.emplace_back(orig_p.name, orig_p.loc, new_param_type, orig_p.flags, orig_p.param_idx); + } + TypePtr new_return_type = replace_genericT_with_deduced(fun_ref->declared_return_type, fun_ref->genericTs, instantiationTs->substitutions); + FunctionData* new_fun_ref = new FunctionData(new_name, fun_ref->loc, new_return_type, std::move(new_parameters), fun_ref->flags, nullptr, instantiationTs, fun_ref->body, fun_ref->ast_root); + new_fun_ref->arg_order = fun_ref->arg_order; + new_fun_ref->ret_order = fun_ref->ret_order; + G.symtable.add_function(new_fun_ref); + return new_fun_ref; + } + + // for `f` (both asm and regular), create "f" with AST fully cloned + // it means, that types still contain T: `f(v: T): T`, but since type resolving knows + // it's instantiation, when resolving types, it substitutes T=int + V orig_root = fun_ref->ast_root->as(); + V new_name_ident = createV(orig_root->get_identifier()->loc, new_name); + V new_root = replicator.clone_function_ast(orig_root, new_name_ident); + + FunctionPtr new_fun_ref = pipeline_register_instantiated_generic_function(new_root, instantiationTs); + tolk_assert(new_fun_ref); + // body of a cloned function (it's cloned at type inferring step) needs the previous pipeline to run + // for example, all local vars need to be registered as symbols, etc. + // these pipes are exactly the same as in tolk.cpp — all preceding (and including) type inferring + pipeline_resolve_identifiers_and_assign_symbols(new_fun_ref); + pipeline_resolve_types_and_aliases(new_fun_ref); + pipeline_calculate_rvalue_lvalue(new_fun_ref); + pipeline_infer_types_and_calls_and_fields(new_fun_ref); - FunctionData* inst_ref = new FunctionData(inst_name, fun_ref->loc, declared_return_type, std::move(parameters), fun_ref->flags, nullptr, instantiationTs, new FunctionBodyCode, inst_root); - inst_ref->arg_order = fun_ref->arg_order; - inst_ref->ret_order = fun_ref->ret_order; - inst_root->mutate()->assign_fun_ref(inst_ref); - G.symtable.add_function(inst_ref); - G.all_functions.push_back(inst_ref); - run_pipeline_for_instantiated_function(inst_ref); - return inst_ref; + return new_fun_ref; } } // namespace tolk diff --git a/tolk/generics-helpers.h b/tolk/generics-helpers.h index 5ed245affc..140d278159 100644 --- a/tolk/generics-helpers.h +++ b/tolk/generics-helpers.h @@ -41,7 +41,6 @@ struct GenericsDeclaration { std::string as_human_readable() const; size_t size() const { return itemsT.size(); } - bool has_nameT(std::string_view nameT) const { return find_nameT(nameT) != -1; } int find_nameT(std::string_view nameT) const; std::string get_nameT(int idx) const { return static_cast(itemsT[idx].nameT); } }; @@ -96,7 +95,6 @@ struct GenericDeduceError final : std::exception { } }; -std::string generate_instantiated_name(const std::string& orig_name, const std::vector& substitutions); -FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, const std::string& inst_name, std::vector&& substitutionTs); +FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, std::vector&& substitutionTs); } // namespace tolk diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index 822abe931f..32989b3fa2 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -557,15 +557,6 @@ Lexer::Lexer(const SrcFile* file) next(); } -Lexer::Lexer(std::string_view text) - : file(nullptr) - , p_start(text.data()) - , p_end(p_start + text.size()) - , p_next(p_start) - , location() { - next(); -} - void Lexer::next() { while (cur_token_idx == last_token_idx && !is_eof()) { update_location(); diff --git a/tolk/lexer.h b/tolk/lexer.h index 6e9ab6a41b..9c8545bf39 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -168,7 +168,6 @@ class Lexer { }; explicit Lexer(const SrcFile* file); - explicit Lexer(std::string_view text); Lexer(const Lexer&) = delete; Lexer &operator=(const Lexer&) = delete; diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index cd6e38f0ca..2139ae714d 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -296,7 +296,7 @@ static V calc_sink_leftmost_obj(V v) { } leftmost_obj = unwrap_not_null_operator(v_dot->get_obj()); } - return leftmost_obj->type == ast_reference ? leftmost_obj->as() : nullptr; + return leftmost_obj->kind == ast_reference ? leftmost_obj->as() : nullptr; } @@ -390,12 +390,12 @@ static std::vector pre_compile_tensor(CodeBlob& code, const std::vect static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyExprV rhs, SrcLocation loc) { // [lhs] = [rhs]; since type checking is ok, it's the same as "lhs = rhs" - if (lhs->type == ast_typed_tuple && rhs->type == ast_typed_tuple) { + if (lhs->kind == ast_bracket_tuple && rhs->kind == ast_bracket_tuple) { // note: there are no type transitions (adding nullability flag, etc.), since only 1-slot elements allowed in tuples LValContext local_lval; - std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); + std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); vars_modification_watcher.trigger_callbacks(left, loc); - std::vector rvect = pre_compile_tensor(code, rhs->as()->get_items()); + std::vector rvect = pre_compile_tensor(code, rhs->as()->get_items()); code.emplace_back(loc, Op::_Let, left, rvect); local_lval.after_let(std::move(left), code, loc); std::vector right = code.create_tmp_var(TypeDataTuple::create(), loc, "(tuple)"); @@ -403,12 +403,12 @@ static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyE return right; } // [lhs] = rhs; it's un-tuple to N left vars - if (lhs->type == ast_typed_tuple) { + if (lhs->kind == ast_bracket_tuple) { LValContext local_lval; - std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); + std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); vars_modification_watcher.trigger_callbacks(left, loc); std::vector right = pre_compile_expr(rhs, code, nullptr); - const TypeDataTypedTuple* inferred_tuple = rhs->inferred_type->unwrap_alias()->try_as(); + const TypeDataBrackets* inferred_tuple = rhs->inferred_type->unwrap_alias()->try_as(); std::vector types_list = inferred_tuple->items; std::vector rvect = code.create_tmp_var(TypeDataTensor::create(std::move(types_list)), rhs->loc, "(unpack-tuple)"); code.emplace_back(lhs->loc, Op::_UnTuple, rvect, std::move(right)); @@ -417,7 +417,7 @@ static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyE return right; } // small optimization: `var x = rhs` or `local_var = rhs` (90% cases), LValContext not needed actually - if (lhs->type == ast_local_var_lhs || (lhs->type == ast_reference && lhs->as()->sym->try_as())) { + if (lhs->kind == ast_local_var_lhs || (lhs->kind == ast_reference && lhs->as()->sym->try_as())) { std::vector left = pre_compile_expr(lhs, code, nullptr); // effectively, local_var->ir_idx vars_modification_watcher.trigger_callbacks(left, loc); std::vector right = pre_compile_expr(rhs, code, lhs->inferred_type); @@ -818,7 +818,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as() && original_type->try_as()) { + if (target_type->try_as() && original_type->try_as()) { tolk_assert(target_w == 1 && orig_w == 1); return rvect; } @@ -969,7 +969,7 @@ static std::vector process_binary_operator(V v, return transition_to_target_type(std::move(rvect), code, target_type, v); } - throw UnexpectedASTNodeType(v, "process_binary_operator"); + throw UnexpectedASTNodeKind(v, "process_binary_operator"); } static std::vector process_unary_operator(V v, CodeBlob& code, TypePtr target_type) { @@ -1001,14 +1001,14 @@ static std::vector process_ternary_operator(V v } static std::vector process_cast_as_operator(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { - TypePtr child_target_type = v->cast_to_type; + TypePtr child_target_type = v->type_node->resolved_type; std::vector rvect = pre_compile_expr(v->get_expr(), code, child_target_type, lval_ctx); return transition_to_target_type(std::move(rvect), code, target_type, v); } static std::vector process_is_type_operator(V v, CodeBlob& code, TypePtr target_type) { TypePtr lhs_type = v->get_expr()->inferred_type->unwrap_alias(); - TypePtr cmp_type = v->rhs_type->unwrap_alias(); + TypePtr cmp_type = v->type_node->resolved_type->unwrap_alias(); bool is_null_check = cmp_type == TypeDataNullLiteral::create(); // `v == null`, not `v is T` tolk_assert(!cmp_type->try_as()); // `v is int|slice` is a type checker error @@ -1061,7 +1061,7 @@ static std::vector process_match_expression(V v auto v_ith_arm = v->get_arm(i); std::vector eq_ith_ir_idx; if (is_match_by_type) { - TypePtr cmp_type = v_ith_arm->exact_type->unwrap_alias(); + TypePtr cmp_type = v_ith_arm->pattern_type_node->resolved_type->unwrap_alias(); tolk_assert(!cmp_type->try_as()); // `match` over `int|slice` is a type checker error eq_ith_ir_idx = pre_compile_is_type(code, lhs_type, cmp_type, subj_ir_idx, v_ith_arm->loc, "(arm-cond-eq)"); } else { @@ -1149,7 +1149,7 @@ static std::vector process_dot_access(V v, CodeBlob& return transition_to_target_type(std::move(rvect), code, target_type, v); } // `tupleVar.0` - if (obj_type->try_as() || obj_type->try_as()) { + if (obj_type->try_as() || obj_type->try_as()) { int index_at = std::get(v->target); // handle `tupleVar.0 = rhs`, "0 SETINDEX" will be called when this was is modified if (lval_ctx && !lval_ctx->is_rval_inside_lval() && calc_sink_leftmost_obj(v)) { @@ -1218,7 +1218,7 @@ static std::vector process_function_call(V v, Code if (delta_self) { args.push_back(v->get_dot_obj()); obj_leftmost = v->get_dot_obj(); - while (obj_leftmost->type == ast_function_call && obj_leftmost->as()->is_dot_call() && obj_leftmost->as()->fun_maybe && obj_leftmost->as()->fun_maybe->does_return_self()) { + while (obj_leftmost->kind == ast_function_call && obj_leftmost->as()->is_dot_call() && obj_leftmost->as()->fun_maybe && obj_leftmost->as()->fun_maybe->does_return_self()) { obj_leftmost = obj_leftmost->as()->get_dot_obj(); } } @@ -1306,7 +1306,7 @@ static std::vector process_tensor(V v, CodeBlob& code, Ty return transition_to_target_type(std::move(rvect), code, target_type, v); } -static std::vector process_typed_tuple(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { +static std::vector process_typed_tuple(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { if (lval_ctx) { // todo some time, make "var (a, [b,c]) = (1, [2,3])" work v->error("[...] can not be used as lvalue here"); } @@ -1400,7 +1400,7 @@ static std::vector process_empty_expression(V v } std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { - switch (v->type) { + switch (v->kind) { case ast_reference: return process_reference(v->as(), code, target_type, lval_ctx); case ast_assign: @@ -1431,8 +1431,8 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ return process_braced_expression(v->as(), code, target_type); case ast_tensor: return process_tensor(v->as(), code, target_type, lval_ctx); - case ast_typed_tuple: - return process_typed_tuple(v->as(), code, target_type, lval_ctx); + case ast_bracket_tuple: + return process_typed_tuple(v->as(), code, target_type, lval_ctx); case ast_object_literal: return process_object_literal(v->as(), code, target_type, lval_ctx); case ast_int_const: @@ -1452,7 +1452,7 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ case ast_empty_expression: return process_empty_expression(v->as(), code, target_type); default: - throw UnexpectedASTNodeType(v, "pre_compile_expr"); + throw UnexpectedASTNodeKind(v, "pre_compile_expr"); } } @@ -1646,7 +1646,7 @@ static void append_implicit_return_statement(SrcLocation loc_end, CodeBlob& code void process_any_statement(AnyV v, CodeBlob& code) { - switch (v->type) { + switch (v->kind) { case ast_block_statement: return process_block_statement(v->as(), code); case ast_return_statement: diff --git a/tolk/pipe-calc-rvalue-lvalue.cpp b/tolk/pipe-calc-rvalue-lvalue.cpp index b00a766965..b6a239116c 100644 --- a/tolk/pipe-calc-rvalue-lvalue.cpp +++ b/tolk/pipe-calc-rvalue-lvalue.cpp @@ -95,7 +95,7 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { restore_state(saved); } - void visit(V v) override { + void visit(V v) override { mark_vertex(v); MarkingState saved = enter_rvalue_if_none(); parent::visit(v); diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 2b87d77cbe..8c42b3f54b 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -31,6 +31,11 @@ static std::string to_string(AnyExprV v_with_type) { return "`" + v_with_type->inferred_type->as_human_readable() + "`"; } +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(std::string_view string_view) { + return static_cast(string_view); +} + GNU_ATTRIBUTE_NOINLINE static std::string expression_as_string(AnyExprV v) { if (auto v_ref = v->try_as()) { @@ -44,12 +49,6 @@ static std::string expression_as_string(AnyExprV v) { return "expression"; } -// fire a general "type mismatch" error, just a wrapper over `throw` -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { - throw ParseError(cur_f, loc, message); -} - // fire an error on `!cell` / `+slice` GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_cannot_apply_operator(FunctionPtr cur_f, SrcLocation loc, std::string_view operator_name, AnyExprV unary_expr) { @@ -117,28 +116,10 @@ static void handle_possible_compiler_internal_call(FunctionPtr cur_f, Vname == "__expect_type") { tolk_assert(v->get_num_args() == 2); - TypePtr expected_type = parse_type_from_string(v->get_arg(1)->get_expr()->as()->str_val); - if (expected_type->has_unresolved_inside()) { // only aliases can be inside - expected_type = expected_type->replace_children_custom([cur_f, v](TypePtr child) -> TypePtr { - if (const auto* as_unresolved = child->try_as()) { - const Symbol* sym = lookup_global_symbol(as_unresolved->text); - if (!sym) { - throw ParseError(cur_f, v->loc, "invalid __expect_type"); - } - if (AliasDefPtr alias_ref = sym->try_as()) { - return TypeDataAlias::create(alias_ref); - } - if (StructPtr struct_ref = sym->try_as()) { - return struct_ref->struct_type; - } - tolk_assert(false); - } - return child; - }); - } + std::string_view expected_type_str = v->get_arg(1)->get_expr()->as()->str_val; TypePtr expr_type = v->get_arg(0)->inferred_type; - if (expected_type->as_human_readable() != expr_type->as_human_readable()) { - fire(cur_f, v->loc, "__expect_type failed: expected " + to_string(expected_type) + ", got " + to_string(expr_type)); + if (expected_type_str != expr_type->as_human_readable()) { + fire(cur_f, v->loc, "__expect_type failed: expected `" + to_string(expected_type_str) + "`, got " + to_string(expr_type)); } } } @@ -282,23 +263,24 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { parent::visit(v->get_expr()); - if (!v->get_expr()->inferred_type->can_be_casted_with_as_operator(v->cast_to_type)) { - fire(cur_f, v->loc, "type " + to_string(v->get_expr()) + " can not be cast to " + to_string(v->cast_to_type)); + if (!v->get_expr()->inferred_type->can_be_casted_with_as_operator(v->type_node->resolved_type)) { + fire(cur_f, v->loc, "type " + to_string(v->get_expr()) + " can not be cast to " + to_string(v->type_node->resolved_type)); } } void visit(V v) override { parent::visit(v->get_expr()); + TypePtr rhs_type = v->type_node->resolved_type; - if (v->rhs_type->unwrap_alias()->try_as()) { // `v is T1 | T2` / `v is T?` is disallowed + if (rhs_type->unwrap_alias()->try_as()) { // `v is T1 | T2` / `v is T?` is disallowed fire(cur_f, v->loc, "union types are not allowed, use concrete types in `is`"); } if ((v->is_always_true && !v->is_negated) || (v->is_always_false && v->is_negated)) { - v->loc.show_warning(expression_as_string(v->get_expr()) + " is always " + to_string(v->rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); + v->loc.show_warning(expression_as_string(v->get_expr()) + " is always " + to_string(rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); } if ((v->is_always_false && !v->is_negated) || (v->is_always_true && v->is_negated)) { - v->loc.show_warning(expression_as_string(v->get_expr()) + " of type " + to_string(v->get_expr()) + " can never be " + to_string(v->rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); + v->loc.show_warning(expression_as_string(v->get_expr()) + " of type " + to_string(v->get_expr()) + " can never be " + to_string(rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); } } @@ -312,7 +294,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // if operator `!` used for non-nullable, probably a warning should be printed } - void visit(V v) override { + void visit(V v) override { parent::visit(v); for (int i = 0; i < v->size(); ++i) { @@ -328,7 +310,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (v->is_target_indexed_access()) { TypePtr obj_type = v->get_obj()->inferred_type->unwrap_alias(); - if (v->inferred_type->get_width_on_stack() != 1 && (obj_type->try_as() || obj_type->try_as())) { + if (v->inferred_type->get_width_on_stack() != 1 && (obj_type->try_as() || obj_type->try_as())) { fire_error_cannot_put_non1_stack_width_arg_to_tuple(cur_f, v->loc, v->inferred_type); } } @@ -405,7 +387,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // inside `var v: int = rhs` / `var _ = rhs` / `var v redef = rhs` (lhs is "v" / "_" / "v") if (auto lhs_var = lhs->try_as()) { - TypePtr declared_type = lhs_var->declared_type; // `var v: int = rhs` (otherwise, nullptr) + TypePtr declared_type = lhs_var->type_node ? lhs_var->type_node->resolved_type : nullptr; if (lhs_var->marked_as_redef) { tolk_assert(lhs_var->var_ref && lhs_var->var_ref->declared_type); declared_type = lhs_var->var_ref->declared_type; @@ -416,7 +398,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } else { if (rhs_type == TypeDataNullLiteral::create()) { - fire_error_assign_always_null_to_variable(cur_f, err_loc->loc, lhs_var->var_ref->try_as(), corresponding_maybe_rhs && corresponding_maybe_rhs->type == ast_null_keyword); + fire_error_assign_always_null_to_variable(cur_f, err_loc->loc, lhs_var->var_ref->try_as(), corresponding_maybe_rhs && corresponding_maybe_rhs->kind == ast_null_keyword); } } return; @@ -441,15 +423,15 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // `[v1, v2] = rhs` / `var [v1, v2] = rhs` (rhs may be `[1,2]` or `tupleVar` or `someF()`, doesn't matter) // dig recursively into v1 and v2 with corresponding rhs i-th item of a tuple - if (auto lhs_tuple = lhs->try_as()) { - const TypeDataTypedTuple* rhs_type_tuple = rhs_type->unwrap_alias()->try_as(); + if (auto lhs_tuple = lhs->try_as()) { + const TypeDataBrackets* rhs_type_tuple = rhs_type->unwrap_alias()->try_as(); if (!rhs_type_tuple) { fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to a tuple"); } if (lhs_tuple->size() != rhs_type_tuple->size()) { fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + ", sizes mismatch"); } - V rhs_tuple_maybe = corresponding_maybe_rhs ? corresponding_maybe_rhs->try_as() : nullptr; + V rhs_tuple_maybe = corresponding_maybe_rhs ? corresponding_maybe_rhs->try_as() : nullptr; for (int i = 0; i < lhs_tuple->size(); ++i) { process_assignment_lhs(lhs_tuple->get_item(i), rhs_type_tuple->items[i], rhs_tuple_maybe ? rhs_tuple_maybe->get_item(i) : nullptr); } @@ -497,7 +479,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { static bool is_expr_valid_as_return_self(AnyExprV return_expr) { // `return self` - if (return_expr->type == ast_reference && return_expr->as()->get_name() == "self") { + if (return_expr->kind == ast_reference && return_expr->as()->get_name() == "self") { return true; } // `return self.someMethod()` @@ -548,7 +530,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } has_type_arm = true; - TypePtr lhs_type = v_arm->exact_type->unwrap_alias(); // `lhs_type => ...` + TypePtr lhs_type = v_arm->pattern_type_node->resolved_type->unwrap_alias(); // `lhs_type => ...` if (lhs_type->try_as()) { fire(cur_f, v_arm->loc, "wrong pattern matching: union types are not allowed, use concrete types in `match`"); } @@ -605,7 +587,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { missing += to_string(variant); } } - throw ParseError(cur_f, v->loc, "`match` does not cover all possible types; missing types are: " + missing); + fire(cur_f, v->loc, "`match` does not cover all possible types; missing types are: " + missing); } // `match` by expression, if it's not statement, should have `else` (unless it's match over bool with const true/false) if (has_expr_arm && !has_else_arm && !v->is_statement()) { diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp index fed6de13d4..aa274a436c 100644 --- a/tolk/pipe-check-rvalue-lvalue.cpp +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -29,12 +29,6 @@ namespace tolk { -// fire a general error, just a wrapper over `throw` -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { - throw ParseError(cur_f, loc, message); -} - GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_cannot_be_used_as_lvalue(FunctionPtr cur_f, AnyV v, const std::string& details) { // example: `f() = 32` diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index a5e68c6bf3..30d0ff562c 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -71,7 +71,7 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { TokenType t = v->tok; // convert "-1" (tok_minus tok_int_const) to a const -1 - if (t == tok_minus && v->get_rhs()->type == ast_int_const) { + if (t == tok_minus && v->get_rhs()->kind == ast_int_const) { td::RefInt256 intval = v->get_rhs()->as()->intval; tolk_assert(!intval.is_null()); intval = -intval; @@ -81,16 +81,16 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { return create_int_const(v->loc, std::move(intval)); } // same for "+1" - if (t == tok_plus && v->get_rhs()->type == ast_int_const) { + if (t == tok_plus && v->get_rhs()->kind == ast_int_const) { return v->get_rhs(); } // `!true` / `!false` - if (t == tok_logical_not && v->get_rhs()->type == ast_bool_const) { + if (t == tok_logical_not && v->get_rhs()->kind == ast_bool_const) { return create_bool_const(v->loc, !v->get_rhs()->as()->bool_val); } // `!0` - if (t == tok_logical_not && v->get_rhs()->type == ast_int_const) { + if (t == tok_logical_not && v->get_rhs()->kind == ast_int_const) { return create_bool_const(v->loc, v->get_rhs()->as()->intval == 0); } @@ -101,7 +101,7 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { parent::replace(v); // `null == null` / `null != null` - if (v->get_expr()->type == ast_null_keyword && v->rhs_type == TypeDataNullLiteral::create()) { + if (v->get_expr()->kind == ast_null_keyword && v->type_node->resolved_type == TypeDataNullLiteral::create()) { return create_bool_const(v->loc, !v->is_negated); } @@ -138,7 +138,7 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { // replace `2 + 3 => ...` with `5 => ...` // non-constant expressions like `foo() => ...` fire an error here - if (v->pattern_kind == MatchArmKind::const_expression && v->get_pattern_expr()->type != ast_int_const) { + if (v->pattern_kind == MatchArmKind::const_expression && v->get_pattern_expr()->kind != ast_int_const) { check_expression_is_constant(v->get_pattern_expr()); } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index ee2e0f27b5..f1e5b4c25c 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -125,12 +125,6 @@ static std::string to_string(std::string_view string_view) { return static_cast(string_view); } -// fire a general error, just a wrapper over `throw` -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { - throw ParseError(cur_f, loc, message); -} - // fire an error when `fun f(...) asm ...` is called with T=(int,int) or other non-1 width on stack // asm functions generally can't handle it, they expect T to be a TVM primitive // (in FunC, `forall` type just couldn't be unified with non-primitives; in Tolk, generic T is expectedly inferred) @@ -161,7 +155,7 @@ class InferTypesAndCallsAndFieldsVisitor final { GNU_ATTRIBUTE_ALWAYS_INLINE static void assign_inferred_type(AnyExprV dst, AnyExprV src) { #ifdef TOLK_DEBUG - tolk_assert(src->inferred_type != nullptr && !src->inferred_type->has_unresolved_inside() && !src->inferred_type->has_genericT_inside()); + tolk_assert(src->inferred_type != nullptr && !src->inferred_type->has_genericT_inside()); #endif dst->mutate()->assign_inferred_type(src->inferred_type); } @@ -169,28 +163,28 @@ class InferTypesAndCallsAndFieldsVisitor final { GNU_ATTRIBUTE_ALWAYS_INLINE static void assign_inferred_type(AnyExprV dst, TypePtr inferred_type) { #ifdef TOLK_DEBUG - tolk_assert(inferred_type != nullptr && !inferred_type->has_unresolved_inside() && !inferred_type->has_genericT_inside()); + tolk_assert(inferred_type != nullptr && !inferred_type->has_genericT_inside()); #endif dst->mutate()->assign_inferred_type(inferred_type); } static void assign_inferred_type(LocalVarPtr local_var_or_param, TypePtr inferred_type) { #ifdef TOLK_DEBUG - tolk_assert(inferred_type != nullptr && !inferred_type->has_unresolved_inside() && !inferred_type->has_genericT_inside()); + tolk_assert(inferred_type != nullptr && !inferred_type->has_genericT_inside()); #endif local_var_or_param->mutate()->assign_inferred_type(inferred_type); } static void assign_inferred_type(FunctionPtr fun_ref, TypePtr inferred_return_type, TypePtr inferred_full_type) { #ifdef TOLK_DEBUG - tolk_assert(inferred_return_type != nullptr && !inferred_return_type->has_unresolved_inside() && !inferred_return_type->has_genericT_inside()); + tolk_assert(inferred_return_type != nullptr && !inferred_return_type->has_genericT_inside()); #endif fun_ref->mutate()->assign_inferred_type(inferred_return_type, inferred_full_type); } // traverse children in any statement FlowContext process_any_statement(AnyV v, FlowContext&& flow) { - switch (v->type) { + switch (v->kind) { case ast_block_statement: return process_block_statement(v->as(), std::move(flow)); case ast_return_statement: @@ -220,7 +214,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // returns ExprFlow: out_facts that are "definitely known" after evaluating the whole expression // if used_as_condition, true_facts/false_facts are also calculated (don't calculate them always for optimization) ExprFlow infer_any_expr(AnyExprV v, FlowContext&& flow, bool used_as_condition, TypePtr hint = nullptr) { - switch (v->type) { + switch (v->kind) { case ast_int_const: return infer_int_const(v->as(), std::move(flow), used_as_condition); case ast_string_const: @@ -259,8 +253,8 @@ class InferTypesAndCallsAndFieldsVisitor final { return infer_function_call(v->as(), std::move(flow), used_as_condition, hint); case ast_tensor: return infer_tensor(v->as(), std::move(flow), used_as_condition, hint); - case ast_typed_tuple: - return infer_typed_tuple(v->as(), std::move(flow), used_as_condition, hint); + case ast_bracket_tuple: + return infer_typed_tuple(v->as(), std::move(flow), used_as_condition, hint); case ast_null_keyword: return infer_null_keyword(v->as(), std::move(flow), used_as_condition); case ast_match_expression: @@ -272,7 +266,7 @@ class InferTypesAndCallsAndFieldsVisitor final { case ast_empty_expression: return infer_empty_expression(v->as(), std::move(flow), used_as_condition); default: - throw UnexpectedASTNodeType(v, "infer_any_expr"); + throw UnexpectedASTNodeKind(v, "infer_any_expr"); } } @@ -323,7 +317,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (v->marked_as_redef) { assign_inferred_type(v, v->var_ref->declared_type); } else { - assign_inferred_type(v, v->declared_type ? v->declared_type : TypeDataUnknown::create()); + assign_inferred_type(v, v->type_node ? v->type_node->resolved_type : TypeDataUnknown::create()); } return ExprFlow(std::move(flow), used_as_condition); } @@ -357,14 +351,14 @@ class InferTypesAndCallsAndFieldsVisitor final { } assign_inferred_type(lhs, TypeDataTensor::create(std::move(types_list))); - } else if (auto lhs_tuple = lhs->try_as()) { + } else if (auto lhs_tuple = lhs->try_as()) { std::vector types_list; types_list.reserve(lhs_tuple->size()); for (int i = 0; i < lhs_tuple->size(); ++i) { flow = infer_left_side_of_assignment(lhs_tuple->get_item(i), std::move(flow)); types_list.push_back(lhs_tuple->get_item(i)->inferred_type); } - assign_inferred_type(lhs, TypeDataTypedTuple::create(std::move(types_list))); + assign_inferred_type(lhs, TypeDataBrackets::create(std::move(types_list))); } else if (auto lhs_par = lhs->try_as()) { flow = infer_left_side_of_assignment(lhs_par->get_expr(), std::move(flow)); @@ -399,7 +393,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // inside `var v: int = rhs` / `var _ = rhs` / `var v redef = rhs` (lhs is "v" / "_" / "v") if (auto lhs_var = lhs->try_as()) { - TypePtr declared_type = lhs_var->marked_as_redef ? lhs_var->var_ref->declared_type : lhs_var->declared_type; + TypePtr declared_type = lhs_var->marked_as_redef ? lhs_var->var_ref->declared_type : lhs_var->type_node ? lhs_var->type_node->resolved_type : nullptr; if (lhs_var->inferred_type == TypeDataUnknown::create()) { assign_inferred_type(lhs_var, rhs_type); assign_inferred_type(lhs_var->var_ref, rhs_type); @@ -426,8 +420,8 @@ class InferTypesAndCallsAndFieldsVisitor final { // `[v1, v2] = rhs` / `var [v1, v2] = rhs` (rhs may be `[1,2]` or `tupleVar` or `someF()`, doesn't matter) // dig recursively into v1 and v2 with corresponding rhs i-th item of a tuple - if (auto lhs_tuple = lhs->try_as()) { - const TypeDataTypedTuple* rhs_type_tuple = rhs_type->unwrap_alias()->try_as(); + if (auto lhs_tuple = lhs->try_as()) { + const TypeDataBrackets* rhs_type_tuple = rhs_type->unwrap_alias()->try_as(); std::vector types_list; types_list.reserve(lhs_tuple->size()); for (int i = 0; i < lhs_tuple->size(); ++i) { @@ -435,7 +429,7 @@ class InferTypesAndCallsAndFieldsVisitor final { process_assignment_lhs_after_infer_rhs(lhs_tuple->get_item(i), ith_rhs_type, out_flow); types_list.push_back(lhs_tuple->get_item(i)->inferred_type); } - assign_inferred_type(lhs, TypeDataTypedTuple::create(std::move(types_list))); + assign_inferred_type(lhs, TypeDataBrackets::create(std::move(types_list))); return; } @@ -619,8 +613,8 @@ class InferTypesAndCallsAndFieldsVisitor final { ExprFlow infer_cast_as_operator(V v, FlowContext&& flow, bool used_as_condition) { // for `expr as `, use this type for hint, so that `t.tupleAt(0) as int` is ok - ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false, v->cast_to_type); - assign_inferred_type(v, v->cast_to_type); + ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false, v->type_node->resolved_type); + assign_inferred_type(v, v->type_node->resolved_type); if (!used_as_condition) { return after_expr; @@ -632,10 +626,10 @@ class InferTypesAndCallsAndFieldsVisitor final { ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); assign_inferred_type(v, TypeDataBool::create()); - TypePtr rhs_type = v->rhs_type->unwrap_alias(); + TypePtr rhs_type = v->type_node->resolved_type->unwrap_alias(); TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); TypePtr non_rhs_type = calculate_type_subtract_rhs_type(expr_type, rhs_type); - if (expr_type->equal_to(v->rhs_type)) { // `expr is ` is always true + if (expr_type->equal_to(rhs_type)) { // `expr is ` is always true v->mutate()->assign_always_true_or_false(v->is_negated ? 2 : 1); } else if (non_rhs_type == TypeDataNever::create()) { // `expr is ` is always false v->mutate()->assign_always_true_or_false(v->is_negated ? 1 : 2); @@ -760,7 +754,7 @@ class InferTypesAndCallsAndFieldsVisitor final { std::vector substitutions; substitutions.reserve(instantiationT_list->size()); for (int i = 0; i < instantiationT_list->size(); ++i) { - substitutions.push_back(instantiationT_list->get_item(i)->substituted_type); + substitutions.push_back(instantiationT_list->get_item(i)->type_node->resolved_type); } return substitutions; @@ -782,10 +776,9 @@ class InferTypesAndCallsAndFieldsVisitor final { } } - std::string inst_name = generate_instantiated_name(fun_ref->name, substitutionTs); // make deep clone of `f` with substitutionTs // (if `f` was already instantiated, it will be immediately returned from a symbol table) - return instantiate_generic_function(loc, fun_ref, inst_name, std::move(substitutionTs)); + return instantiate_generic_function(loc, fun_ref, std::move(substitutionTs)); } ExprFlow infer_dot_access(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { @@ -833,7 +826,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); } - if (const auto* t_tuple = unwrapped_obj_type->try_as()) { + if (const auto* t_tuple = unwrapped_obj_type->try_as()) { if (index_at >= t_tuple->size()) { fire(cur_f, v_ident->loc, "invalid tuple index, expected 0.." + std::to_string(t_tuple->items.size() - 1)); } @@ -1095,8 +1088,8 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } - ExprFlow infer_typed_tuple(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { - const TypeDataTypedTuple* tuple_hint = hint ? hint->unwrap_alias()->try_as() : nullptr; + ExprFlow infer_typed_tuple(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + const TypeDataBrackets* tuple_hint = hint ? hint->unwrap_alias()->try_as() : nullptr; std::vector types_list; types_list.reserve(v->get_items().size()); for (int i = 0; i < v->size(); ++i) { @@ -1104,7 +1097,7 @@ class InferTypesAndCallsAndFieldsVisitor final { flow = infer_any_expr(item, std::move(flow), false, tuple_hint && i < tuple_hint->size() ? tuple_hint->items[i] : nullptr).out_flow; types_list.emplace_back(item->inferred_type); } - assign_inferred_type(v, TypeDataTypedTuple::create(std::move(types_list))); + assign_inferred_type(v, TypeDataBrackets::create(std::move(types_list))); return ExprFlow(std::move(flow), used_as_condition); } @@ -1125,7 +1118,7 @@ class InferTypesAndCallsAndFieldsVisitor final { auto v_arm = v->get_arm(i); FlowContext arm_flow = infer_any_expr(v_arm->get_pattern_expr(), arms_entry_facts.clone(), false, nullptr).out_flow; if (s_expr && v_arm->pattern_kind == MatchArmKind::exact_type) { - arm_flow.register_known_type(s_expr, v_arm->exact_type); + arm_flow.register_known_type(s_expr, v_arm->pattern_type_node->resolved_type); } arm_flow = infer_any_expr(v_arm->get_body(), std::move(arm_flow), false, hint).out_flow; match_out_flow = i == 0 ? std::move(arm_flow) : FlowContext::merge_flow(std::move(match_out_flow), std::move(arm_flow)); @@ -1218,7 +1211,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } v->mutate()->assign_struct_ref(struct_ref); - assign_inferred_type(v, struct_ref->struct_type); + assign_inferred_type(v, TypeDataStruct::create(struct_ref)); assign_inferred_type(v->get_body(), v); return ExprFlow(std::move(flow), used_as_condition); } @@ -1241,7 +1234,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // (but don't print a warning if it's already unreachable, for example we're inside always-false if) bool initially_unreachable = flow.is_unreachable(); for (AnyV item : v->get_items()) { - if (flow.is_unreachable() && !initially_unreachable && !v->first_unreachable && item->type != ast_empty_statement) { + if (flow.is_unreachable() && !initially_unreachable && !v->first_unreachable && item->kind != ast_empty_statement) { v->mutate()->assign_first_unreachable(item); // a warning will be printed later, after type checking } flow = process_any_statement(item, std::move(flow)); @@ -1435,15 +1428,17 @@ class InferTypesAndCallsAndFieldsVisitor final { // given `const a = 2 + 3` infer that it's int // for `const a: int = ...` still infer all sub expressions (to be checked in a later pipe) void start_visiting_constant(GlobalConstPtr const_ref) { - FlowContext const_flow; - infer_any_expr(const_ref->init_value, std::move(const_flow), false, const_ref->declared_type); + infer_any_expr(const_ref->init_value, FlowContext(), false, const_ref->declared_type); const_ref->mutate()->assign_inferred_type(const_ref->declared_type == nullptr ? const_ref->init_value->inferred_type : const_ref->declared_type); } // given struct field `a: int = 2 + 3` infer that default value is int, assign inferred_type to all nodes - void start_visiting_field_default(StructFieldPtr field_ref) { - FlowContext field_flow; - infer_any_expr(field_ref->default_value, std::move(field_flow), false, field_ref->declared_type); + void start_visiting_struct_fields(StructPtr struct_ref) { + for (StructFieldPtr field_ref : struct_ref->fields) { + if (field_ref->has_default_value()) { + infer_any_expr(field_ref->default_value, FlowContext(), false, field_ref->declared_type); + } + } } }; @@ -1519,11 +1514,7 @@ void pipeline_infer_types_and_calls_and_fields() { // infer types for default values in structs for (StructPtr struct_ref : get_all_declared_structs()) { - for (StructFieldPtr field_ref : struct_ref->fields) { - if (field_ref->has_default_value()) { - visitor.start_visiting_field_default(field_ref); - } - } + visitor.start_visiting_struct_fields(struct_ref); } } diff --git a/tolk/pipe-optimize-boolean-expr.cpp b/tolk/pipe-optimize-boolean-expr.cpp index 09282cfd99..4b05923c2a 100644 --- a/tolk/pipe-optimize-boolean-expr.cpp +++ b/tolk/pipe-optimize-boolean-expr.cpp @@ -117,7 +117,7 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { if (v->tok == tok_eq || v->tok == tok_neq) { AnyExprV lhs = v->get_lhs(); AnyExprV rhs = v->get_rhs(); - if (expect_boolean(lhs->inferred_type) && rhs->type == ast_bool_const) { + if (expect_boolean(lhs->inferred_type) && rhs->kind == ast_bool_const) { // `boolVar == true` / `boolVar != false` if (rhs->as()->bool_val ^ (v->tok == tok_neq)) { return lhs; diff --git a/tolk/pipe-refine-lvalue-for-mutate.cpp b/tolk/pipe-refine-lvalue-for-mutate.cpp index a8b4f1ae52..5187457b2a 100644 --- a/tolk/pipe-refine-lvalue-for-mutate.cpp +++ b/tolk/pipe-refine-lvalue-for-mutate.cpp @@ -35,7 +35,7 @@ namespace tolk { GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_invalid_mutate_arg_passed(AnyExprV v, FunctionPtr fun_ref, const LocalVarData& p_sym, bool called_as_method, bool arg_passed_as_mutate, AnyV arg_expr) { - std::string arg_str(arg_expr->type == ast_reference ? arg_expr->as()->get_name() : "obj"); + std::string arg_str(arg_expr->kind == ast_reference ? arg_expr->as()->get_name() : "obj"); // case: `loadInt(cs, 32)`; suggest: `cs.loadInt(32)` if (p_sym.is_mutate_parameter() && !arg_passed_as_mutate && !called_as_method && p_sym.param_idx == 0 && fun_ref->does_accept_self()) { @@ -92,7 +92,7 @@ class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBod break; } } - bool will_be_extracted_as_tmp_var = leftmost_obj->type == ast_function_call; + bool will_be_extracted_as_tmp_var = leftmost_obj->kind == ast_function_call; if (!will_be_extracted_as_tmp_var) { leftmost_obj->mutate()->assign_lvalue_true(); v->get_dot_obj()->mutate()->assign_lvalue_true(); diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 6bb93d55ae..7a09defae4 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -21,7 +21,6 @@ #include "compiler-state.h" #include "generics-helpers.h" #include "td/utils/crypto.h" -#include "type-system.h" #include /* @@ -60,10 +59,7 @@ static int calculate_method_id_by_func_name(std::string_view func_name) { return static_cast(crc & 0xffff) | 0x10000; } -static void validate_arg_ret_order_of_asm_function(V v_body, int n_params, TypePtr ret_type) { - if (!ret_type) { - v_body->error("asm function must declare return type (before asm instructions)"); - } +static void validate_arg_ret_order_of_asm_function(V v_body, int n_params) { if (n_params > 16) { v_body->error("asm function can have at most 16 parameters"); } @@ -114,7 +110,7 @@ static const GenericsDeclaration* construct_genericTs(V v_li } static void register_constant(V v) { - GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->declared_type, v->get_init_value()); + GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->type_node, v->get_init_value()); G.symtable.add_global_const(c_sym); G.all_constants.push_back(c_sym); @@ -122,15 +118,15 @@ static void register_constant(V v) { } static void register_global_var(V v) { - GlobalVarData* g_sym = new GlobalVarData(static_cast(v->get_identifier()->name), v->loc, v->declared_type); + GlobalVarData* g_sym = new GlobalVarData(static_cast(v->get_identifier()->name), v->loc, v->type_node); G.symtable.add_global_var(g_sym); G.all_global_vars.push_back(g_sym); - v->mutate()->assign_var_ref(g_sym); + v->mutate()->assign_glob_ref(g_sym); } static void register_type_alias(V v) { - AliasDefData* a_sym = new AliasDefData(static_cast(v->get_identifier()->name), v->loc, v->underlying_type); + AliasDefData* a_sym = new AliasDefData(static_cast(v->get_identifier()->name), v->loc, v->underlying_type_node); G.symtable.add_type_alias(a_sym); v->mutate()->assign_alias_ref(a_sym); @@ -144,9 +140,7 @@ static void register_struct(V v) { for (int i = 0; i < v_body->get_num_fields(); ++i) { auto v_field = v_body->get_field(i); AnyExprV default_value = v_field->has_default_value() ? v_field->get_default_value() : nullptr; - StructFieldData* field_ref = new StructFieldData(static_cast(v_field->get_identifier()->name), v_field->loc, i, v_field->declared_type, default_value); - fields.emplace_back(field_ref); - v_field->mutate()->assign_field_ref(field_ref); + fields.emplace_back(new StructFieldData(static_cast(v_field->get_identifier()->name), v_field->loc, i, v_field->type_node, default_value)); } StructData* s_sym = new StructData(static_cast(v->get_identifier()->name), v->loc, std::move(fields)); @@ -158,7 +152,7 @@ static void register_struct(V v) { static LocalVarData register_parameter(V v, int idx) { if (v->is_underscore()) { - return {"", v->loc, v->declared_type, 0, idx}; + return LocalVarData{"", v->loc, v->type_node, 0, idx}; } int flags = 0; @@ -168,30 +162,21 @@ static LocalVarData register_parameter(V v, int idx) { if (!v->declared_as_mutate && idx == 0 && v->param_name == "self") { flags |= LocalVarData::flagImmutable; } - return LocalVarData(static_cast(v->param_name), v->loc, v->declared_type, flags, idx); + return LocalVarData(static_cast(v->param_name), v->loc, v->type_node, flags, idx); } -static void register_function(V v) { +static void register_function(V v, const GenericsInstantiation* instantiationTs = nullptr) { std::string_view func_name = v->get_identifier()->name; - // calculate TypeData of a function - std::vector arg_types; std::vector parameters; - int n_params = v->get_num_params(); int n_mutate_params = 0; - arg_types.reserve(n_params); - parameters.reserve(n_params); - for (int i = 0; i < n_params; ++i) { + parameters.reserve(v->get_num_params()); + for (int i = 0; i < v->get_num_params(); ++i) { auto v_param = v->get_param(i); - arg_types.emplace_back(v_param->declared_type); parameters.emplace_back(register_parameter(v_param, i)); n_mutate_params += static_cast(v_param->declared_as_mutate); } - const GenericsDeclaration* genericTs = nullptr; - if (v->genericsT_list) { - genericTs = construct_genericTs(v->genericsT_list); - } if (v->is_builtin_function()) { const Symbol* sym = lookup_global_symbol(func_name); FunctionPtr fun_ref = sym ? sym->try_as() : nullptr; @@ -202,15 +187,15 @@ static void register_function(V v) { return; } - if (G.is_verbosity(1) && v->is_code_function()) { - std::cerr << "fun " << func_name << " : " << v->declared_return_type << std::endl; - } - - FunctionBody f_body = v->get_body()->type == ast_block_statement ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); - FunctionData* f_sym = new FunctionData(static_cast(func_name), v->loc, v->declared_return_type, std::move(parameters), 0, genericTs, nullptr, f_body, v); + const GenericsDeclaration* genericTs = v->genericsT_list ? construct_genericTs(v->genericsT_list) : nullptr; + FunctionBody f_body = v->get_body()->kind == ast_block_statement ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); + FunctionData* f_sym = new FunctionData(static_cast(func_name), v->loc, v->return_type_node, std::move(parameters), 0, genericTs, instantiationTs, f_body, v); - if (const auto* v_asm = v->get_body()->try_as()) { - validate_arg_ret_order_of_asm_function(v_asm, v->get_num_params(), v->declared_return_type); + if (auto v_asm = v->get_body()->try_as()) { + if (!v->return_type_node) { + v_asm->error("asm function must declare return type (before asm instructions)"); + } + validate_arg_ret_order_of_asm_function(v_asm, v->get_num_params()); f_sym->arg_order = v_asm->arg_order; f_sym->ret_order = v_asm->ret_order; } @@ -248,7 +233,7 @@ static void iterate_through_file_symbols(const SrcFile* file) { tolk_assert(file && file->ast); for (AnyV v : file->ast->as()->get_toplevel_declarations()) { - switch (v->type) { + switch (v->kind) { case ast_import_directive: // on `import "another-file.tolk"`, register symbols from that file at first // (for instance, it can calculate constants, which are used in init_val of constants in current file below import) @@ -282,4 +267,10 @@ void pipeline_register_global_symbols() { } } +FunctionPtr pipeline_register_instantiated_generic_function(AnyV cloned_v, const GenericsInstantiation* instantiationTs) { + auto v = cloned_v->as(); + register_function(v, instantiationTs); + return lookup_global_symbol(v->get_identifier()->name)->try_as(); +} + } // namespace tolk diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 545cf50fc8..9f089b0644 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -18,32 +18,21 @@ #include "platform-utils.h" #include "compiler-state.h" #include "src-file.h" -#include "generics-helpers.h" #include "ast.h" #include "ast-visitor.h" -#include "type-system.h" #include /* - * This pipe resolves identifiers (local variables and types) in all functions bodies. + * This pipe resolves identifiers (local variables, globals, constants, etc.) in all functions bodies. * It happens before type inferring, but after all global symbols are registered. * It means, that for any symbol `x` we can look up whether it's a global name or not. * - * About resolving variables. * Example: `var x = 10; x = 20;` both `x` point to one LocalVarData. * Example: `x = 20` undefined symbol `x` is also here (unless it's a global) * Variables scoping and redeclaration are also here. * Note, that `x` is stored as `ast_reference (ast_identifier "x")`. More formally, "references" are resolved. * "Reference" in AST, besides the identifier, stores optional generics instantiation. `x` is grammar-valid. * - * About resolving types. At the moment of parsing, `int`, `cell` and other predefined are parsed as TypeDataInt, etc. - * All the others are stored as TypeDataUnresolved, to be resolved here, after global symtable is filled. - * Example: `var x: T = 0` unresolved "T" is replaced by TypeDataGenericT inside `f`. - * Example: `f()` unresolved "MyAlias" is replaced by TypeDataAlias inside the reference. - * Example: `fun f(): KKK` unresolved "KKK" fires an error "unknown type name". - * When structures and type aliases are implemented, their resolving will also be done here. - * See finalize_type_data(). - * * Note, that functions/methods binding is NOT here. * In other words, for ast_function_call `beginCell()` and `t.tupleAt(0)`, their fun_ref is NOT filled here. * Functions/methods binding is done later, simultaneously with type inferring and generics instantiation. @@ -53,7 +42,6 @@ * As a result of this step, * * every V::sym is filled, pointing either to a local var/parameter, or to a global symbol * (exceptional for function calls and methods, their references are bound later) - * * all TypeData in all symbols is ready for analyzing, TypeDataUnresolved won't occur later in pipeline */ namespace tolk { @@ -67,11 +55,6 @@ static void fire_error_undefined_symbol(FunctionPtr cur_f, V v) } } -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_unknown_type_name(FunctionPtr cur_f, SrcLocation loc, const std::string &text) { - throw ParseError(cur_f, loc, "unknown type name `" + text + "`"); -} - static void check_import_exists_when_using_sym(FunctionPtr cur_f, AnyV v_usage, const Symbol* used_sym) { SrcLocation sym_loc = used_sym->loc; if (!v_usage->loc.is_symbol_from_same_or_builtin_file(sym_loc)) { @@ -135,100 +118,13 @@ struct NameAndScopeResolver { } }; -struct TypeDataResolver { - static TypePtr finalize_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { - if (type_data) { - if (type_data->has_unresolved_inside()) { - type_data = resolve_identifiers_in_type_data(cur_f, type_data, genericTs); - } - } - return type_data; - } - - GNU_ATTRIBUTE_NOINLINE - static TypePtr resolve_identifiers_in_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { - return type_data->replace_children_custom([cur_f, genericTs](TypePtr child) { - if (const TypeDataUnresolved* un = child->try_as()) { - if (genericTs && genericTs->has_nameT(un->text)) { - std::string nameT = un->text; - return TypeDataGenericT::create(std::move(nameT)); - } - if (const Symbol* sym = lookup_global_symbol(un->text)) { - if (AliasDefPtr alias_ref = sym->try_as()) { - if (alias_ref->underlying_type->has_unresolved_inside()) { - resolve_and_mutate_type_alias(alias_ref); - } - return TypeDataAlias::create(alias_ref); - } - if (StructPtr struct_ref = sym->try_as()) { - if (!struct_ref->struct_type) { - resolve_and_mutate_struct_and_fields(struct_ref); - } - return struct_ref->struct_type; - } - } - if (un->text == "auto") { - throw ParseError(cur_f, un->loc, "`auto` type does not exist; just omit a type for local variable (will be inferred from assignment); parameters should always be typed"); - } - if (un->text == "self") { - throw ParseError(cur_f, un->loc, "`self` type can be used only as a return type of a function (enforcing it to be chainable)"); - } - fire_error_unknown_type_name(cur_f, un->loc, un->text); - } - return child; - }); - } - - static void resolve_and_mutate_type_alias(AliasDefPtr alias_ref) { - static std::vector called_stack; - - // prevent recursion like `type A = B; type B = A` - bool contains = std::find(called_stack.begin(), called_stack.end(), alias_ref) != called_stack.end(); - if (contains) { - throw ParseError(alias_ref->loc, "type `" + alias_ref->name + "` circularly references itself"); - } - - called_stack.push_back(alias_ref); - TypePtr underlying_type = finalize_type_data(nullptr, alias_ref->underlying_type, nullptr); - alias_ref->mutate()->assign_resolved_type(underlying_type); - called_stack.pop_back(); - } - - static void resolve_and_mutate_struct_and_fields(StructPtr struct_ref) { - static std::vector called_stack; - - // prevent recursion like `struct A { field: A }` - // currently, a struct is a tensor, and recursion always leads to infinite size (`A?` also, it's also on a stack) - // if there would be an annotation to store a struct in a tuple, then it has to be reconsidered - bool contains = std::find(called_stack.begin(), called_stack.end(), struct_ref) != called_stack.end(); - if (contains) { - throw ParseError(struct_ref->loc, "struct `" + struct_ref->name + "` size is infinity due to recursive fields"); - } - - called_stack.push_back(struct_ref); - for (int i = 0; i < struct_ref->get_num_fields(); ++i) { - StructFieldPtr field_ref = struct_ref->get_field(i); - TypePtr declared_type = finalize_type_data(nullptr, field_ref->declared_type, nullptr); - field_ref->mutate()->assign_resolved_type(declared_type); - } - struct_ref->mutate()->assign_resolved_type(TypeDataStruct::create(struct_ref)); - called_stack.pop_back(); - } -}; - -static TypePtr finalize_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { - return TypeDataResolver::finalize_type_data(cur_f, type_data, genericTs); -} - - class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { // more correctly this field shouldn't be static, but currently there is no need to make it a part of state static NameAndScopeResolver current_scope; static FunctionPtr cur_f; - static const GenericsDeclaration* current_genericTs; - static LocalVarPtr create_local_var_sym(std::string_view name, SrcLocation loc, TypePtr declared_type, bool immutable) { - LocalVarData* v_sym = new LocalVarData(static_cast(name), loc, declared_type, immutable * LocalVarData::flagImmutable, -1); + static LocalVarPtr create_local_var_sym(std::string_view name, SrcLocation loc, AnyTypeV declared_type_node, bool immutable) { + LocalVarData* v_sym = new LocalVarData(static_cast(name), loc, declared_type_node, immutable * LocalVarData::flagImmutable, -1); current_scope.add_local_var(v_sym); return v_sym; } @@ -253,9 +149,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { } v->mutate()->assign_var_ref(var_ref); } else { - TypePtr declared_type = finalize_type_data(cur_f, v->declared_type, current_genericTs); - LocalVarPtr var_ref = create_local_var_sym(v->get_name(), v->loc, declared_type, v->is_immutable); - v->mutate()->assign_resolved_type(declared_type); + LocalVarPtr var_ref = create_local_var_sym(v->get_name(), v->loc, v->type_node, v->is_immutable); v->mutate()->assign_var_ref(var_ref); } } @@ -276,15 +170,6 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { if (!sym->try_as()) { check_import_exists_when_using_sym(cur_f, v, sym); } - - // for `f` / `f`, resolve "MyAlias" and "T" - // (for function call `f()`, this v (ast_reference `f`) is callee) - if (auto v_instantiationTs = v->get_instantiationTs()) { - for (int i = 0; i < v_instantiationTs->size(); ++i) { - TypePtr substituted_type = finalize_type_data(cur_f, v_instantiationTs->get_item(i)->substituted_type, current_genericTs); - v_instantiationTs->get_item(i)->mutate()->assign_resolved_type(substituted_type); - } - } } void visit(V v) override { @@ -306,17 +191,14 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { switch (v->pattern_kind) { case MatchArmKind::exact_type: { - if (const TypeDataUnresolved* maybe_ident = v->exact_type->try_as()) { - if (const Symbol* sym = current_scope.lookup_symbol(maybe_ident->text); sym && !sym->try_as() && !sym->try_as()) { + if (auto maybe_ident = v->pattern_type_node->try_as()) { + if (const Symbol* sym = current_scope.lookup_symbol(maybe_ident->text); sym && sym->try_as()) { auto v_ident = createV(v->loc, sym->name); AnyExprV pattern_expr = createV(v->loc, v_ident, nullptr); parent::visit(pattern_expr); - v->mutate()->assign_resolved_pattern(MatchArmKind::const_expression, nullptr, pattern_expr); - return; + v->mutate()->assign_resolved_pattern(MatchArmKind::const_expression, pattern_expr); } } - TypePtr resolved_exact_type = finalize_type_data(cur_f, v->exact_type, current_genericTs); - v->mutate()->assign_resolved_pattern(MatchArmKind::exact_type, resolved_exact_type, v->get_pattern_expr()); break; } case MatchArmKind::const_expression: { @@ -329,30 +211,6 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { } } - void visit(V v) override { - // for `t.tupleAt` / `obj.method`, resolve "MyAlias" and "T" - // (for function call `t.tupleAt()`, this v (ast_dot_access `t.tupleAt`) is callee) - if (auto v_instantiationTs = v->get_instantiationTs()) { - for (int i = 0; i < v_instantiationTs->size(); ++i) { - TypePtr substituted_type = finalize_type_data(cur_f, v_instantiationTs->get_item(i)->substituted_type, current_genericTs); - v_instantiationTs->get_item(i)->mutate()->assign_resolved_type(substituted_type); - } - } - parent::visit(v->get_obj()); - } - - void visit(V v) override { - TypePtr cast_to_type = finalize_type_data(cur_f, v->cast_to_type, current_genericTs); - v->mutate()->assign_resolved_type(cast_to_type); - parent::visit(v->get_expr()); - } - - void visit(V v) override { - TypePtr rhs_type = finalize_type_data(cur_f, v->rhs_type, current_genericTs); - v->mutate()->assign_resolved_type(rhs_type); - parent::visit(v->get_expr()); - } - void visit(V v) override { if (v->empty()) { return; @@ -382,56 +240,41 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { public: bool should_visit_function(FunctionPtr fun_ref) override { - // this pipe is done just after parsing - // visit both asm and code functions, resolve identifiers in parameter/return types everywhere - // for generic functions, unresolved "T" will be replaced by TypeDataGenericT - return true; + return fun_ref->is_code_function(); } void start_visiting_function(FunctionPtr fun_ref, V v) override { cur_f = fun_ref; - current_genericTs = fun_ref->genericTs; + auto v_block = v->get_body()->as(); + current_scope.open_scope(v->loc); for (int i = 0; i < v->get_num_params(); ++i) { - const LocalVarData& param_var = fun_ref->parameters[i]; - TypePtr declared_type = finalize_type_data(cur_f, param_var.declared_type, fun_ref->genericTs); - v->get_param(i)->mutate()->assign_param_ref(¶m_var); - v->get_param(i)->mutate()->assign_resolved_type(declared_type); - param_var.mutate()->assign_resolved_type(declared_type); - } - TypePtr return_type = finalize_type_data(cur_f, fun_ref->declared_return_type, fun_ref->genericTs); - v->mutate()->assign_resolved_type(return_type); - fun_ref->mutate()->assign_resolved_type(return_type); - - if (fun_ref->is_code_function()) { - auto v_block = v->get_body()->as(); - current_scope.open_scope(v->loc); - for (int i = 0; i < v->get_num_params(); ++i) { - current_scope.add_local_var(&fun_ref->parameters[i]); - } - parent::visit(v_block); - current_scope.close_scope(v_block->loc_end); - tolk_assert(current_scope.scopes.empty()); + current_scope.add_local_var(&fun_ref->parameters[i]); } + parent::visit(v_block); + current_scope.close_scope(v_block->loc_end); + tolk_assert(current_scope.scopes.empty()); - current_genericTs = nullptr; cur_f = nullptr; } - void start_visiting_constant(V v) { + void start_visiting_constant(GlobalConstPtr const_ref) { // `const a = b`, resolve `b` - parent::visit(v->get_init_value()); + parent::visit(const_ref->init_value); } - void start_visiting_struct_field(V v) { + void start_visiting_struct_fields(StructPtr struct_ref) { // field `a: int = C`, resolve `C` - parent::visit(v->get_default_value()); + for (StructFieldPtr field_ref : struct_ref->fields) { + if (field_ref->has_default_value()) { + parent::visit(field_ref->default_value); + } + } } }; NameAndScopeResolver AssignSymInsideFunctionVisitor::current_scope; FunctionPtr AssignSymInsideFunctionVisitor::cur_f = nullptr; -const GenericsDeclaration* AssignSymInsideFunctionVisitor::current_genericTs = nullptr; void pipeline_resolve_identifiers_and_assign_symbols() { AssignSymInsideFunctionVisitor visitor; @@ -439,33 +282,17 @@ void pipeline_resolve_identifiers_and_assign_symbols() { for (AnyV v : file->ast->as()->get_toplevel_declarations()) { if (auto v_func = v->try_as()) { tolk_assert(v_func->fun_ref); - visitor.start_visiting_function(v_func->fun_ref, v_func); - - } else if (auto v_global = v->try_as()) { - TypePtr declared_type = finalize_type_data(nullptr, v_global->var_ref->declared_type, nullptr); - v_global->mutate()->assign_resolved_type(declared_type); - v_global->var_ref->mutate()->assign_resolved_type(declared_type); - - } else if (auto v_const = v->try_as()) { - visitor.start_visiting_constant(v_const); - if (v_const->declared_type) { - TypePtr declared_type = finalize_type_data(nullptr, v_const->const_ref->declared_type, nullptr); - v_const->mutate()->assign_resolved_type(declared_type); - v_const->const_ref->mutate()->assign_resolved_type(declared_type); + if (visitor.should_visit_function(v_func->fun_ref)) { + visitor.start_visiting_function(v_func->fun_ref, v_func); } - } else if (auto v_alias = v->try_as()) { - TypeDataResolver::resolve_and_mutate_type_alias(v_alias->alias_ref); - v_alias->mutate()->assign_resolved_type(v_alias->alias_ref->underlying_type); + } else if (auto v_const = v->try_as()) { + tolk_assert(v_const->const_ref); + visitor.start_visiting_constant(v_const->const_ref); } else if (auto v_struct = v->try_as()) { - TypeDataResolver::resolve_and_mutate_struct_and_fields(v_struct->struct_ref); - auto v_body = v_struct->get_struct_body(); - for (int i = 0; i < v_body->get_num_fields(); ++i) { - auto v_field = v_body->get_field(i); - visitor.start_visiting_struct_field(v_field); - v_field->mutate()->assign_resolved_type(v_struct->struct_ref->get_field(i)->declared_type); - } + tolk_assert(v_struct->struct_ref); + visitor.start_visiting_struct_fields(v_struct->struct_ref); } } } diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp new file mode 100644 index 0000000000..3af32fa96c --- /dev/null +++ b/tolk/pipe-resolve-types.cpp @@ -0,0 +1,455 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include "platform-utils.h" +#include "compiler-state.h" +#include "ast.h" +#include "ast-visitor.h" +#include "generics-helpers.h" +#include "type-system.h" +#include + +namespace tolk { + +/* + * This pipe transforms AST of types into TypePtr. + * It happens after all global symbols were registered, and all local references were bound. + * + * At the moment of parsing, `int`, `cell` and other were parsed as AnyTypeV (ast_type_leaf_text and others). + * Example: `var x: int = ...` to TypeDataInt + * Example: `fun f(a: cell): (int, User)` param to TypeDataCell, return type to TypeDataTensor(TypeDataInt, TypeDataStruct) + * Example: `var x: T = 0` to TypeDataGenericT inside `f` + * Example: `f()` to TypeDataAlias inside instantiation list + * Example: `fun f(): KKK` fires an error "unknown type name" + * + * Types resolving is done everywhere: inside functions bodies, in struct fields, inside globals declaration, etc. + * See finalize_type_node(). + * + * Note, that resolving T to TypeDataGenericT (and replacing T with substitution when instantiating a generic type) + * is also done here, see genericTs and instantiationTs. + */ + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_unknown_type_name(FunctionPtr cur_f, SrcLocation loc, std::string_view text) { + if (text == "auto") { + fire(cur_f, loc, "`auto` type does not exist; just omit a type for local variable (will be inferred from assignment); parameters should always be typed"); + } + if (text == "self") { + fire(cur_f, loc, "`self` type can be used only as a return type of a function (enforcing it to be chainable)"); + } + fire(cur_f, loc, "unknown type name `" + static_cast(text) + "`"); +} + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_void_type_not_allowed_inside_union(FunctionPtr cur_f, SrcLocation loc, TypePtr disallowed_variant) { + fire(cur_f, loc, "type `" + disallowed_variant->as_human_readable() + "` is not allowed inside a union"); +} + +static TypePtr parse_intN(std::string_view strN, bool is_unsigned) { + int n; + auto result = std::from_chars(strN.data() + 3 + static_cast(is_unsigned), strN.data() + strN.size(), n); + bool parsed = result.ec == std::errc() && result.ptr == strN.data() + strN.size(); + if (!parsed || n <= 0 || n > 256 + static_cast(is_unsigned)) { + return nullptr; // `int1000`, maybe it's user-defined alias, let it be unresolved + } + return TypeDataIntN::create(is_unsigned, false, n); +} + +static TypePtr parse_bytesN(std::string_view strN, bool is_bits) { + int n; + auto result = std::from_chars(strN.data() + 5 - static_cast(is_bits), strN.data() + strN.size(), n); + bool parsed = result.ec == std::errc() && result.ptr == strN.data() + strN.size(); + if (!parsed || n <= 0 || n > 1024) { + return nullptr; // `bytes9999`, maybe it's user-defined alias, let it be unresolved + } + return TypeDataBytesN::create(is_bits, n); +} + +static TypePtr try_parse_predefined_type(std::string_view str) { + switch (str.size()) { + case 3: + if (str == "int") return TypeDataInt::create(); + break; + case 4: + if (str == "cell") return TypeDataCell::create(); + if (str == "void") return TypeDataVoid::create(); + if (str == "bool") return TypeDataBool::create(); + if (str == "null") return TypeDataNullLiteral::create(); + break; + case 5: + if (str == "slice") return TypeDataSlice::create(); + if (str == "tuple") return TypeDataTuple::create(); + if (str == "coins") return TypeDataCoins::create(); + if (str == "never") return TypeDataNever::create(); + break; + case 7: + if (str == "builder") return TypeDataBuilder::create(); + break; + case 8: + if (str == "varint16") return TypeDataIntN::create(false, true, 16); + if (str == "varint32") return TypeDataIntN::create(false, true, 32); + break; + case 12: + if (str == "continuation") return TypeDataContinuation::create(); + break; + default: + break; + } + + if (str.starts_with("int")) { + if (TypePtr intN = parse_intN(str, false)) { + return intN; + } + } + if (str.size() > 4 && str.starts_with("uint")) { + if (TypePtr uintN = parse_intN(str, true)) { + return uintN; + } + } + if (str.size() > 4 && str.starts_with("bits")) { + if (TypePtr bitsN = parse_bytesN(str, true)) { + return bitsN; + } + } + if (str.size() > 5 && str.starts_with("bytes")) { + if (TypePtr bytesN = parse_bytesN(str, false)) { + return bytesN; + } + } + + return nullptr; +} + +class TypeNodesVisitorResolver { + FunctionPtr cur_f; // exists if we're inside its body + const GenericsDeclaration* genericTs; // `` if we're inside `f` or `f` + const GenericsInstantiation* instantiationTs; // `` if we're inside `f` + + TypePtr parse_ast_type_node(AnyTypeV v) { + switch (v->kind) { + case ast_type_leaf_text: { + std::string_view text = v->as()->text; + if (TypePtr predefined_type = try_parse_predefined_type(text)) { + return predefined_type; + } + if (genericTs) { + // if we're inside `f`, replace "T" with TypeDataGenericT + // if we're inside `f`, replace "T" with TypeDataInt (substitution) + if (int idx = genericTs->find_nameT(text); idx != -1) { + if (instantiationTs) { + return instantiationTs->substitutions[idx]; + } + return TypeDataGenericT::create(static_cast(text)); + } + } + if (const Symbol* sym = lookup_global_symbol(text)) { + if (TypePtr struct_or_other = try_resolve_user_defined_type(sym)) { + return struct_or_other; + } + } + fire_error_unknown_type_name(cur_f, v->as()->loc, text); + } + + case ast_type_question_nullable: { + TypePtr inner = finalize_type_node(v->as()->get_inner()); + TypePtr result = TypeDataUnion::create_nullable(inner); + if (const TypeDataUnion* t_union = result->try_as()) { + validate_resulting_union_type(t_union, cur_f, v->loc); + } + return result; + } + + case ast_type_parenthesis_tensor: { + std::vector items = finalize_type_node(v->as()->get_items()); + if (items.size() == 1) { + return items.front(); + } + return TypeDataTensor::create(std::move(items)); + } + + case ast_type_bracket_tuple: { + std::vector items = finalize_type_node(v->as()->get_items()); + return TypeDataBrackets::create(std::move(items)); + } + + case ast_type_arrow_callable: { + std::vector params_and_return = finalize_type_node(v->as()->get_params_and_return()); + TypePtr return_type = params_and_return.back(); + params_and_return.pop_back(); + return TypeDataFunCallable::create(std::move(params_and_return), return_type); + } + + case ast_type_vertical_bar_union: { + std::vector variants = finalize_type_node(v->as()->get_variants()); + TypePtr result = TypeDataUnion::create(std::move(variants)); + if (const TypeDataUnion* t_union = result->try_as()) { + validate_resulting_union_type(t_union, cur_f, v->loc); + } + return result; + } + + default: + throw UnexpectedASTNodeKind(v, "resolve_ast_type_node"); + } + } + + static TypePtr try_resolve_user_defined_type(const Symbol* sym) { + if (AliasDefPtr alias_ref = sym->try_as()) { + if (!alias_ref->was_visited_by_resolver()) { + visit_symbol(alias_ref); + } + return TypeDataAlias::create(alias_ref); + } + if (StructPtr struct_ref = sym->try_as()) { + if (!struct_ref->was_visited_by_resolver()) { + visit_symbol(struct_ref); + } + return TypeDataStruct::create(struct_ref); + } + return nullptr; + } + + static void validate_resulting_union_type(const TypeDataUnion* t_union, FunctionPtr cur_f, SrcLocation loc) { + for (TypePtr variant : t_union->variants) { + if (variant == TypeDataVoid::create() || variant == TypeDataNever::create()) { + fire_void_type_not_allowed_inside_union(cur_f, loc, variant); + } + } + } + +public: + + TypeNodesVisitorResolver(FunctionPtr cur_f, const GenericsDeclaration* genericTs, const GenericsInstantiation* instantiationTs) + : cur_f(cur_f) + , genericTs(genericTs) + , instantiationTs(instantiationTs) {} + + TypePtr finalize_type_node(AnyTypeV type_node) { +#ifdef TOLK_DEBUG + tolk_assert(type_node != nullptr); +#endif + TypePtr resolved_type = parse_ast_type_node(type_node); + type_node->mutate()->assign_resolved_type(resolved_type); + return resolved_type; + } + + std::vector finalize_type_node(const std::vector& type_node_array) { + std::vector result; + result.reserve(type_node_array.size()); + for (AnyTypeV v : type_node_array) { + result.push_back(finalize_type_node(v)); + } + return result; + } + + static void visit_symbol(GlobalVarPtr glob_ref) { + TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr); + TypePtr declared_type = visitor.finalize_type_node(glob_ref->type_node); + glob_ref->mutate()->assign_resolved_type(declared_type); + } + + static void visit_symbol(GlobalConstPtr const_ref) { + if (!const_ref->type_node) { + return; + } + + TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr); + TypePtr declared_type = visitor.finalize_type_node(const_ref->type_node); + const_ref->mutate()->assign_resolved_type(declared_type); + } + + static void visit_symbol(AliasDefPtr alias_ref) { + static std::vector called_stack; + + // prevent recursion like `type A = B; type B = A` + bool contains = std::find(called_stack.begin(), called_stack.end(), alias_ref) != called_stack.end(); + if (contains) { + throw ParseError(alias_ref->loc, "type `" + alias_ref->name + "` circularly references itself"); + } + + called_stack.push_back(alias_ref); + TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr); + TypePtr underlying_type = visitor.finalize_type_node(alias_ref->underlying_type_node); + alias_ref->mutate()->assign_resolved_type(underlying_type); + alias_ref->mutate()->assign_visited_by_resolver(); + called_stack.pop_back(); + } + + static void visit_symbol(StructPtr struct_ref) { + static std::vector called_stack; + + // prevent recursion like `struct A { field: A }` + // currently, a struct is a tensor, and recursion always leads to infinite size (`A?` also, it's also on a stack) + // if there would be an annotation to store a struct in a tuple, then it has to be reconsidered + bool contains = std::find(called_stack.begin(), called_stack.end(), struct_ref) != called_stack.end(); + if (contains) { + throw ParseError(struct_ref->loc, "struct `" + struct_ref->name + "` size is infinity due to recursive fields"); + } + + called_stack.push_back(struct_ref); + TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr); + for (int i = 0; i < struct_ref->get_num_fields(); ++i) { + StructFieldPtr field_ref = struct_ref->get_field(i); + TypePtr declared_type = visitor.finalize_type_node(field_ref->type_node); + field_ref->mutate()->assign_resolved_type(declared_type); + } + struct_ref->mutate()->assign_visited_by_resolver(); + called_stack.pop_back(); + } +}; + +class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { + static TypeNodesVisitorResolver type_nodes_visitor; + + static TypePtr finalize_type_node(AnyTypeV type_node) { + return type_nodes_visitor.finalize_type_node(type_node); + } + +protected: + + void visit(V v) override { + if (v->type_node) { + TypePtr declared_type = finalize_type_node(v->type_node); + v->var_ref->mutate()->assign_resolved_type(declared_type); + } + } + + void visit(V v) override { + // for `f` / `f`, resolve "MyAlias" and "T" + // (for function call `f()`, this v (ast_reference `f`) is callee) + if (auto v_instantiationTs = v->get_instantiationTs()) { + for (int i = 0; i < v_instantiationTs->size(); ++i) { + finalize_type_node(v_instantiationTs->get_item(i)->type_node); + } + } + } + + void visit(V v) override { + if (v->pattern_type_node) { + finalize_type_node(v->pattern_type_node); + } + parent::visit(v->get_pattern_expr()); + parent::visit(v->get_body()); + } + + void visit(V v) override { + // for `t.tupleAt` / `obj.method`, resolve "MyAlias" and "T" + // (for function call `t.tupleAt()`, this v (ast_dot_access `t.tupleAt`) is callee) + if (auto v_instantiationTs = v->get_instantiationTs()) { + for (int i = 0; i < v_instantiationTs->size(); ++i) { + finalize_type_node(v_instantiationTs->get_item(i)->type_node); + } + } + parent::visit(v->get_obj()); + } + + void visit(V v) override { + finalize_type_node(v->type_node); + parent::visit(v->get_expr()); + } + + void visit(V v) override { + finalize_type_node(v->type_node); + parent::visit(v->get_expr()); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return !fun_ref->is_builtin_function(); + } + + void start_visiting_function(FunctionPtr fun_ref, V v) override { + type_nodes_visitor = TypeNodesVisitorResolver(fun_ref, fun_ref->genericTs, fun_ref->instantiationTs); + + for (int i = 0; i < v->get_num_params(); ++i) { + const LocalVarData& param_var = fun_ref->parameters[i]; + TypePtr declared_type = finalize_type_node(param_var.type_node); + param_var.mutate()->assign_resolved_type(declared_type); + } + if (fun_ref->return_type_node) { + TypePtr declared_return_type = finalize_type_node(fun_ref->return_type_node); + fun_ref->mutate()->assign_resolved_type(declared_return_type); + } + + if (fun_ref->is_code_function()) { + parent::visit(v->get_body()->as()); + } + + type_nodes_visitor = TypeNodesVisitorResolver(nullptr, nullptr, nullptr); + } + + void start_visiting_constant(GlobalConstPtr const_ref) { + // `const a = 0 as int8`, resolve types there + // same for struct field `v: int8 = 0 as int8` + parent::visit(const_ref->init_value); + } + + void start_visiting_struct_fields(StructPtr struct_ref) { + // same for struct field `v: int8 = 0 as int8` + for (StructFieldPtr field_ref : struct_ref->fields) { + if (field_ref->has_default_value()) { + parent::visit(field_ref->default_value); + } + } + } +}; + +TypeNodesVisitorResolver ResolveTypesInsideFunctionVisitor::type_nodes_visitor(nullptr, nullptr, nullptr); + +void pipeline_resolve_types_and_aliases() { + ResolveTypesInsideFunctionVisitor visitor; + + for (const SrcFile* file : G.all_src_files) { + for (AnyV v : file->ast->as()->get_toplevel_declarations()) { + if (auto v_func = v->try_as()) { + tolk_assert(v_func->fun_ref); + if (visitor.should_visit_function(v_func->fun_ref)) { + visitor.start_visiting_function(v_func->fun_ref, v_func); + } + + } else if (auto v_global = v->try_as()) { + TypeNodesVisitorResolver::visit_symbol(v_global->glob_ref); + + } else if (auto v_const = v->try_as()) { + if (v_const->type_node) { + TypeNodesVisitorResolver::visit_symbol(v_const->const_ref); + } + visitor.start_visiting_constant(v_const->const_ref); + + } else if (auto v_alias = v->try_as()) { + if (!v_alias->alias_ref->was_visited_by_resolver()) { + TypeNodesVisitorResolver::visit_symbol(v_alias->alias_ref); + } + + } else if (auto v_struct = v->try_as()) { + if (!v_struct->struct_ref->was_visited_by_resolver()) { + TypeNodesVisitorResolver::visit_symbol(v_struct->struct_ref); + } + visitor.start_visiting_struct_fields(v_struct->struct_ref); + } + } + } +} + +void pipeline_resolve_types_and_aliases(FunctionPtr fun_ref) { + ResolveTypesInsideFunctionVisitor visitor; + if (visitor.should_visit_function(fun_ref)) { + visitor.start_visiting_function(fun_ref, fun_ref->ast_root->as()); + } +} + +} // namespace tolk diff --git a/tolk/pipeline.h b/tolk/pipeline.h index 0a71d751de..acd0fd87a4 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -34,6 +34,7 @@ void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, con void pipeline_register_global_symbols(); void pipeline_resolve_identifiers_and_assign_symbols(); +void pipeline_resolve_types_and_aliases(); void pipeline_calculate_rvalue_lvalue(); void pipeline_infer_types_and_calls_and_fields(); void pipeline_check_inferred_types(); @@ -49,7 +50,9 @@ void pipeline_generate_fif_output_to_std_cout(); // these pipes also can be called per-function individually // they are called for instantiated generics functions, when `f` is deeply cloned as `f` +FunctionPtr pipeline_register_instantiated_generic_function(AnyV cloned_v, const GenericsInstantiation* instantiationTs); void pipeline_resolve_identifiers_and_assign_symbols(FunctionPtr); +void pipeline_resolve_types_and_aliases(FunctionPtr); void pipeline_calculate_rvalue_lvalue(FunctionPtr); void pipeline_detect_unreachable_statements(FunctionPtr); void pipeline_infer_types_and_calls_and_fields(FunctionPtr); diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index b4974ef489..dd6a5abbfb 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -178,8 +178,8 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b, bool* became_union = nul return TypeDataTensor::create(std::move(types_lca)); } - const auto* tuple1 = a->try_as(); - const auto* tuple2 = b->try_as(); + const auto* tuple1 = a->try_as(); + const auto* tuple2 = b->try_as(); if (tuple1 && tuple2 && tuple1->size() == tuple2->size()) { std::vector types_lca; types_lca.reserve(tuple1->size()); @@ -190,7 +190,7 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b, bool* became_union = nul } types_lca.push_back(next); } - return TypeDataTypedTuple::create(std::move(types_lca)); + return TypeDataBrackets::create(std::move(types_lca)); } if (const auto* a_alias = a->try_as()) { @@ -468,7 +468,7 @@ TypePtr calc_declared_type_before_smart_cast(AnyExprV v) { if (const auto* t_tensor = obj_type->try_as()) { return t_tensor->items[index_at]; } - if (const auto* t_tuple = obj_type->try_as()) { + if (const auto* t_tuple = obj_type->try_as()) { return t_tuple->items[index_at]; } } diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 97e48f7fce..e2a0a1836a 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -22,7 +22,7 @@ namespace tolk { std::string FunctionData::as_human_readable() const { - if (!genericTs) { + if (!is_generic_function()) { return name; // if it's generic instantiation like `f`, its name is "f", not "f" } return name + genericTs->as_human_readable(); @@ -113,6 +113,10 @@ void LocalVarData::assign_inferred_type(TypePtr inferred_type) { this->declared_type = inferred_type; } +void AliasDefData::assign_visited_by_resolver() { + this->flags |= flagVisitedByResolver; +} + void AliasDefData::assign_resolved_type(TypePtr underlying_type) { this->underlying_type = underlying_type; } @@ -125,8 +129,8 @@ void StructFieldData::assign_default_value(AnyExprV default_value) { this->default_value = default_value; } -void StructData::assign_resolved_type(TypePtr struct_type) { - this->struct_type = struct_type; +void StructData::assign_visited_by_resolver() { + this->flags |= flagVisitedByResolver; } StructFieldPtr StructData::find_field(std::string_view field_name) const { diff --git a/tolk/symtable.h b/tolk/symtable.h index 9ab7f7b965..4252386c0d 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -51,13 +51,21 @@ struct LocalVarData final : Symbol { flagImmutable = 2, // variable was declared via `val` (not `var`) }; - TypePtr declared_type; // either at declaration `var x:int`, or if omitted, from assigned value `var x=2` + AnyTypeV type_node; // either at declaration `var x:int`, or if omitted, from assigned value `var x=2` + TypePtr declared_type = nullptr; // = resolved type_node int flags; int param_idx; // 0...N for function parameters, -1 for local vars std::vector ir_idx; + LocalVarData(std::string name, SrcLocation loc, AnyTypeV type_node, int flags, int param_idx) + : Symbol(std::move(name), loc) + , type_node(type_node) + , flags(flags) + , param_idx(param_idx) { + } LocalVarData(std::string name, SrcLocation loc, TypePtr declared_type, int flags, int param_idx) : Symbol(std::move(name), loc) + , type_node(nullptr) // for built-in functions (their parameters) , declared_type(declared_type) , flags(flags) , param_idx(param_idx) { @@ -110,19 +118,31 @@ struct FunctionData final : Symbol { std::vector parameters; std::vector arg_order, ret_order; - TypePtr declared_return_type; // may be nullptr, meaning "auto infer" + AnyTypeV return_type_node; // may be nullptr, meaning "auto infer" + TypePtr declared_return_type = nullptr; // = resolved return_type_node TypePtr inferred_return_type = nullptr; // assigned on type inferring TypePtr inferred_full_type = nullptr; // assigned on type inferring, it's TypeDataFunCallable(params -> return) const GenericsDeclaration* genericTs; const GenericsInstantiation* instantiationTs; FunctionBody body; - AnyV ast_root; // V for user-defined (not builtin) + AnyV ast_root; // V for user-defined (not builtin) + FunctionData(std::string name, SrcLocation loc, AnyTypeV return_type_node, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsInstantiation* instantiationTs, FunctionBody body, AnyV ast_root) + : Symbol(std::move(name), loc) + , flags(initial_flags) + , parameters(std::move(parameters)) + , return_type_node(return_type_node) + , genericTs(genericTs) + , instantiationTs(instantiationTs) + , body(body) + , ast_root(ast_root) { + } FunctionData(std::string name, SrcLocation loc, TypePtr declared_return_type, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsInstantiation* instantiationTs, FunctionBody body, AnyV ast_root) : Symbol(std::move(name), loc) , flags(initial_flags) , parameters(std::move(parameters)) + , return_type_node(nullptr) // for built-in functions, defined in sources , declared_return_type(declared_return_type) , genericTs(genericTs) , instantiationTs(instantiationTs) @@ -146,7 +166,7 @@ struct FunctionData final : Symbol { bool is_asm_function() const { return std::holds_alternative(body); } bool is_builtin_function() const { return ast_root == nullptr; } - bool is_generic_function() const { return genericTs != nullptr; } + bool is_generic_function() const { return genericTs != nullptr && instantiationTs == nullptr; } bool is_instantiation_of_generic_function() const { return instantiationTs != nullptr; } bool is_inline() const { return flags & flagInline; } @@ -182,12 +202,13 @@ struct GlobalVarData final : Symbol { flagReallyUsed = 1, // calculated via dfs from used functions; unused globals are not codegenerated }; - TypePtr declared_type; // always exists, declaring globals without type is prohibited + AnyTypeV type_node; // `global a: int;` always exists, declaring globals without type is prohibited + TypePtr declared_type = nullptr; // = resolved type_node int flags = 0; - GlobalVarData(std::string name, SrcLocation loc, TypePtr declared_type) + GlobalVarData(std::string name, SrcLocation loc, AnyTypeV type_node) : Symbol(std::move(name), loc) - , declared_type(declared_type) { + , type_node(type_node) { } bool is_really_used() const { return flags & flagReallyUsed; } @@ -198,14 +219,15 @@ struct GlobalVarData final : Symbol { }; struct GlobalConstData final : Symbol { + AnyTypeV type_node; // exists for `const op: int = rhs`, otherwise nullptr + TypePtr declared_type = nullptr; // = resolved type_node + TypePtr inferred_type = nullptr; AnyExprV init_value; - TypePtr declared_type; // `const a: int = ...`; nullptr for `const a = ...` - TypePtr inferred_type = nullptr; // filled at type inferring pass - GlobalConstData(std::string name, SrcLocation loc, TypePtr declared_type, AnyExprV init_value) + GlobalConstData(std::string name, SrcLocation loc, AnyTypeV type_node, AnyExprV init_value) : Symbol(std::move(name), loc) - , init_value(init_value) - , declared_type(declared_type) { + , type_node(type_node) + , init_value(init_value) { } GlobalConstData* mutate() const { return const_cast(this); } @@ -215,21 +237,31 @@ struct GlobalConstData final : Symbol { }; struct AliasDefData final : Symbol { - TypePtr underlying_type; + enum { + flagVisitedByResolver = 1, + }; + + AnyTypeV underlying_type_node; + TypePtr underlying_type = nullptr; // = resolved underlying_type_node + int flags = 0; - AliasDefData(std::string name, SrcLocation loc, TypePtr underlying_type) + AliasDefData(std::string name, SrcLocation loc, AnyTypeV underlying_type_node) : Symbol(std::move(name), loc) - , underlying_type(underlying_type) { + , underlying_type_node(underlying_type_node) { } + bool was_visited_by_resolver() const { return flags & flagVisitedByResolver; } + AliasDefData* mutate() const { return const_cast(this); } + void assign_visited_by_resolver(); void assign_resolved_type(TypePtr underlying_type); }; struct StructFieldData final : Symbol { int field_idx; - TypePtr declared_type; - AnyExprV default_value; // nullptr if no default + AnyTypeV type_node; + TypePtr declared_type = nullptr; // = resolved type_node + AnyExprV default_value; // nullptr if no default bool has_default_value() const { return default_value != nullptr; } @@ -237,24 +269,30 @@ struct StructFieldData final : Symbol { void assign_resolved_type(TypePtr declared_type); void assign_default_value(AnyExprV default_value); - StructFieldData(std::string name, SrcLocation loc, int field_idx, TypePtr declared_type, AnyExprV default_value) + StructFieldData(std::string name, SrcLocation loc, int field_idx, AnyTypeV type_node, AnyExprV default_value) : Symbol(std::move(name), loc) , field_idx(field_idx) - , declared_type(declared_type) + , type_node(type_node) , default_value(default_value) { } }; struct StructData final : Symbol { + enum { + flagVisitedByResolver = 1, + }; + std::vector fields; - const TypeData* struct_type = nullptr; // it's TypeDataStruct, assigned at resolve identifiers + int flags = 0; int get_num_fields() const { return static_cast(fields.size()); } StructFieldPtr get_field(int i) const { return fields.at(i); } StructFieldPtr find_field(std::string_view field_name) const; + bool was_visited_by_resolver() const { return flags & flagVisitedByResolver; } + StructData* mutate() const { return const_cast(this); } - void assign_resolved_type(TypePtr struct_type); + void assign_visited_by_resolver(); StructData(std::string name, SrcLocation loc, std::vector&& fields) : Symbol(std::move(name), loc) diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp index 71d1969de8..fc38571bc4 100644 --- a/tolk/tolk.cpp +++ b/tolk/tolk.cpp @@ -57,6 +57,7 @@ int tolk_proceed(const std::string &entrypoint_filename) { pipeline_register_global_symbols(); pipeline_resolve_identifiers_and_assign_symbols(); + pipeline_resolve_types_and_aliases(); pipeline_calculate_rvalue_lvalue(); pipeline_infer_types_and_calls_and_fields(); pipeline_check_inferred_types(); @@ -77,7 +78,7 @@ int tolk_proceed(const std::string &entrypoint_filename) { } catch (ParseError& error) { std::cerr << error << std::endl; return 2; - } catch (UnexpectedASTNodeType& error) { + } catch (UnexpectedASTNodeKind& error) { std::cerr << "fatal: " << error.what() << std::endl; std::cerr << "It's a compiler bug, please report to developers" << std::endl; return 2; diff --git a/tolk/tolk.h b/tolk/tolk.h index 74d8fdd9fc..12bc6b6941 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -34,6 +34,12 @@ namespace tolk { GNU_ATTRIBUTE_COLD GNU_ATTRIBUTE_NORETURN void on_assertion_failed(const char *description, const char *file_name, int line_number); +// fire a general error, just a wrapper over `throw` +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +inline void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { + throw ParseError(cur_f, loc, message); +} + /* * * ABSTRACT CODE diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index 402b79b387..7f2101ca4d 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -249,7 +249,7 @@ TypePtr TypeDataTensor::create(std::vector&& items) { return hash.register_unique(new TypeDataTensor(hash.children_flags(), width_on_stack, std::move(items))); } -TypePtr TypeDataTypedTuple::create(std::vector&& items) { +TypePtr TypeDataBrackets::create(std::vector&& items) { TypeDataHasherForUnique hash(9189266157349499320ULL); for (TypePtr item : items) { hash.feed_child(item); @@ -259,7 +259,7 @@ TypePtr TypeDataTypedTuple::create(std::vector&& items) { if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataTypedTuple(hash.children_flags(), std::move(items))); + return hash.register_unique(new TypeDataBrackets(hash.children_flags(), std::move(items))); } TypePtr TypeDataIntN::create(bool is_unsigned, bool is_variadic, int n_bits) { @@ -296,12 +296,12 @@ TypePtr TypeDataUnion::create(std::vector&& variants) { return existing; } - // at the moment of parsing, union type can contain unresolved symbols + // for generics, like `f(v: T?)`, no type_id exists // in this case, don't try to flatten: we have no info - // after symbols resolving, a new union type (with resolved variants) will be created + // after instantiation, a new union type (with resolved variants) will be created bool not_ready_yet = false; for (TypePtr variant : variants) { - not_ready_yet |= variant->has_unresolved_inside() || variant->has_genericT_inside(); + not_ready_yet |= variant->has_genericT_inside(); } if (not_ready_yet) { TypePtr or_null = nullptr; @@ -368,17 +368,6 @@ TypePtr TypeDataUnion::create_nullable(TypePtr nullable) { return create({nullable, TypeDataNullLiteral::create()}); } -TypePtr TypeDataUnresolved::create(std::string&& text, SrcLocation loc) { - TypeDataHasherForUnique hash(3680147223540048162ULL); - hash.feed_string(text); - // hash.feed_hash(*reinterpret_cast(&loc)); - - if (TypePtr existing = hash.get_existing()) { - return existing; - } - return hash.register_unique(new TypeDataUnresolved(std::move(text), loc)); -} - // -------------------------------------------- // get_type_id() @@ -396,7 +385,8 @@ int TypeDataFunCallable::get_type_id() const { } int TypeDataGenericT::get_type_id() const { - return TypeIdCalculation::assign_type_id(this); + assert(false); // generics must have been instantiated in advance + throw Fatal("unexpected get_type_id() call"); } int TypeDataStruct::get_type_id() const { @@ -408,7 +398,7 @@ int TypeDataTensor::get_type_id() const { return TypeIdCalculation::assign_type_id(this); } -int TypeDataTypedTuple::get_type_id() const { +int TypeDataBrackets::get_type_id() const { assert(!has_genericT_inside()); return TypeIdCalculation::assign_type_id(this); } @@ -439,11 +429,6 @@ int TypeDataUnknown::get_type_id() const { throw Fatal("unexpected get_type_id() call"); } -int TypeDataUnresolved::get_type_id() const { - assert(false); // unresolved can be inside a union at parsing, but is resolved is advance - throw Fatal("unexpected get_type_id() call"); -} - // -------------------------------------------- // as_human_readable() @@ -485,7 +470,7 @@ std::string TypeDataTensor::as_human_readable() const { return result; } -std::string TypeDataTypedTuple::as_human_readable() const { +std::string TypeDataBrackets::as_human_readable() const { std::string result = "["; for (TypePtr item : items) { if (result.size() > 1) { @@ -534,48 +519,6 @@ std::string TypeDataUnion::as_human_readable() const { } -// -------------------------------------------- -// traverse() -// -// invokes a callback for TypeData itself and all its children -// only non-trivial implementations are here; by default (no children), `callback(this)` is executed -// - -void TypeDataAlias::traverse(const TraverserCallbackT& callback) const { - callback(this); - underlying_type->traverse(callback); -} - -void TypeDataFunCallable::traverse(const TraverserCallbackT& callback) const { - callback(this); - for (TypePtr param : params_types) { - param->traverse(callback); - } - return_type->traverse(callback); -} - -void TypeDataTensor::traverse(const TraverserCallbackT& callback) const { - callback(this); - for (TypePtr item : items) { - item->traverse(callback); - } -} - -void TypeDataTypedTuple::traverse(const TraverserCallbackT& callback) const { - callback(this); - for (TypePtr item : items) { - item->traverse(callback); - } -} - -void TypeDataUnion::traverse(const TraverserCallbackT& callback) const { - callback(this); - for (TypePtr variant : variants) { - variant->traverse(callback); - } -} - - // -------------------------------------------- // replace_children_custom() // @@ -602,7 +545,7 @@ TypePtr TypeDataTensor::replace_children_custom(const ReplacerCallbackT& callbac return callback(create(std::move(mapped))); } -TypePtr TypeDataTypedTuple::replace_children_custom(const ReplacerCallbackT& callback) const { +TypePtr TypeDataBrackets::replace_children_custom(const ReplacerCallbackT& callback) const { std::vector mapped; mapped.reserve(items.size()); for (TypePtr item : items) { @@ -778,8 +721,8 @@ bool TypeDataTensor::can_rhs_be_assigned(TypePtr rhs) const { return rhs == TypeDataNever::create(); } -bool TypeDataTypedTuple::can_rhs_be_assigned(TypePtr rhs) const { - if (const auto* as_tuple = rhs->try_as(); as_tuple && as_tuple->size() == size()) { +bool TypeDataBrackets::can_rhs_be_assigned(TypePtr rhs) const { + if (const auto* as_tuple = rhs->try_as(); as_tuple && as_tuple->size() == size()) { for (int i = 0; i < size(); ++i) { if (!items[i]->can_rhs_be_assigned(as_tuple->items[i])) { return false; @@ -852,11 +795,6 @@ bool TypeDataUnknown::can_rhs_be_assigned(TypePtr rhs) const { return true; } -bool TypeDataUnresolved::can_rhs_be_assigned(TypePtr rhs) const { - assert(false); - return false; -} - bool TypeDataNever::can_rhs_be_assigned(TypePtr rhs) const { return true; } @@ -1046,8 +984,8 @@ bool TypeDataTensor::can_be_casted_with_as_operator(TypePtr cast_to) const { return false; } -bool TypeDataTypedTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_tuple = cast_to->try_as(); to_tuple && to_tuple->size() == size()) { +bool TypeDataBrackets::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_tuple = cast_to->try_as(); to_tuple && to_tuple->size() == size()) { for (int i = 0; i < size(); ++i) { if (!items[i]->can_be_casted_with_as_operator(to_tuple->items[i])) { return false; @@ -1124,10 +1062,6 @@ bool TypeDataUnknown::can_be_casted_with_as_operator(TypePtr cast_to) const { return cast_to->get_width_on_stack() == 1; } -bool TypeDataUnresolved::can_be_casted_with_as_operator(TypePtr cast_to) const { - return false; -} - bool TypeDataNever::can_be_casted_with_as_operator(TypePtr cast_to) const { return true; } @@ -1251,190 +1185,6 @@ TypePtr TypeDataUnion::calculate_exact_variant_to_fit_rhs(TypePtr rhs_type) cons } -// -------------------------------------------- -// parsing type from tokens -// -// here we implement parsing types (mostly after colon) to TypeData -// example: `var v: int` is TypeDataInt -// example: `var v: (builder?, [cell])` is TypeDataTensor(TypeDataUnion(TypeDataBuilder,TypeDataNullLiteral), TypeDataTypedTuple(TypeDataCell)) -// example: `fun f(): ()` is TypeDataTensor() (an empty one) -// -// note, that unrecognized type names (MyEnum, MyStruct, T) are parsed as TypeDataUnresolved, -// and later, when all files are parsed and all symbols registered, such identifiers are resolved -// example: `fun f(v: T)` at first v is TypeDataUnresolved("T"), later becomes TypeDataGenericT -// see finalize_type_data() -// -// note, that `self` does not name a type, it can appear only as a return value of a function (parsed specially) -// when `self` appears as a type, it's parsed as TypeDataUnresolved, and later an error is emitted -// - -static TypePtr parse_type_expression(Lexer& lex); - -std::vector parse_nested_type_list(Lexer& lex, TokenType tok_op, const char* s_op, TokenType tok_cl, const char* s_cl) { - lex.expect(tok_op, s_op); - std::vector sub_types; - while (true) { - if (lex.tok() == tok_cl) { // empty lists allowed - lex.next(); - break; - } - - sub_types.emplace_back(parse_type_expression(lex)); - if (lex.tok() == tok_comma) { - lex.next(); - } else if (lex.tok() != tok_cl) { - lex.unexpected(s_cl); - } - } - return sub_types; -} - -std::vector parse_nested_type_list_in_parenthesis(Lexer& lex) { - return parse_nested_type_list(lex, tok_oppar, "`(`", tok_clpar, "`)` or `,`"); -} - -static TypePtr parse_intN(std::string_view strN, bool is_unsigned) { - int n; - auto result = std::from_chars(strN.data() + 3 + static_cast(is_unsigned), strN.data() + strN.size(), n); - bool parsed = result.ec == std::errc() && result.ptr == strN.data() + strN.size(); - if (!parsed || n <= 0 || n > 256 + static_cast(is_unsigned)) { - return nullptr; // `int1000`, maybe it's user-defined alias, let it be unresolved - } - return TypeDataIntN::create(is_unsigned, false, n); -} - -static TypePtr parse_bytesN(std::string_view strN, bool is_bits) { - int n; - auto result = std::from_chars(strN.data() + 5 - static_cast(is_bits), strN.data() + strN.size(), n); - bool parsed = result.ec == std::errc() && result.ptr == strN.data() + strN.size(); - if (!parsed || n <= 0 || n > 1024) { - return nullptr; // `bytes9999`, maybe it's user-defined alias, let it be unresolved - } - return TypeDataBytesN::create(is_bits, n); -} - -static TypePtr parse_simple_type(Lexer& lex) { - switch (lex.tok()) { - case tok_self: - case tok_identifier: { - SrcLocation loc = lex.cur_location(); - std::string_view str = lex.cur_str(); - lex.next(); - switch (str.size()) { - case 3: - if (str == "int") return TypeDataInt::create(); - break; - case 4: - if (str == "cell") return TypeDataCell::create(); - if (str == "void") return TypeDataVoid::create(); - if (str == "bool") return TypeDataBool::create(); - break; - case 5: - if (str == "slice") return TypeDataSlice::create(); - if (str == "tuple") return TypeDataTuple::create(); - if (str == "coins") return TypeDataCoins::create(); - if (str == "never") return TypeDataNever::create(); - break; - case 7: - if (str == "builder") return TypeDataBuilder::create(); - break; - case 8: - if (str == "varint16") return TypeDataIntN::create(false, true, 16); - if (str == "varint32") return TypeDataIntN::create(false, true, 32); - break; - case 12: - if (str == "continuation") return TypeDataContinuation::create(); - break; - default: - break; - } - if (str.starts_with("int")) { - if (TypePtr intN = parse_intN(str, false)) { - return intN; - } - } - if (str.size() > 4 && str.starts_with("uint")) { - if (TypePtr uintN = parse_intN(str, true)) { - return uintN; - } - } - if (str.size() > 4 && str.starts_with("bits")) { - if (TypePtr bitsN = parse_bytesN(str, true)) { - return bitsN; - } - } - if (str.size() > 5 && str.starts_with("bytes")) { - if (TypePtr bytesN = parse_bytesN(str, false)) { - return bytesN; - } - } - return TypeDataUnresolved::create(std::string(str), loc); - } - case tok_null: - lex.next(); - return TypeDataNullLiteral::create(); - case tok_oppar: { - std::vector items = parse_nested_type_list_in_parenthesis(lex); - if (items.size() == 1) { - return items.front(); - } - return TypeDataTensor::create(std::move(items)); - } - case tok_opbracket: { - std::vector items = parse_nested_type_list(lex, tok_opbracket, "`[`", tok_clbracket, "`]` or `,`"); - return TypeDataTypedTuple::create(std::move(items)); - } - default: - lex.unexpected(""); - } -} - -static TypePtr parse_type_nullable(Lexer& lex) { - TypePtr result = parse_simple_type(lex); - - if (lex.tok() == tok_question) { - lex.next(); - result = TypeDataUnion::create_nullable(result); - } - - return result; -} - -static TypePtr parse_type_expression(Lexer& lex) { - TypePtr result = parse_type_nullable(lex); - - if (lex.tok() == tok_bitwise_or) { // `int | slice`, `Pair2 | (Pair3 | null)` - std::vector items; - items.emplace_back(result); - while (lex.tok() == tok_bitwise_or) { - lex.next(); - items.emplace_back(parse_type_nullable(lex)); - } - result = TypeDataUnion::create(std::move(items)); - } - - if (lex.tok() == tok_arrow) { // `int -> int`, `(cell, slice) -> void`, `int -> int -> int`, `int | cell -> void` - lex.next(); - TypePtr return_type = parse_type_expression(lex); - std::vector params_types = {result}; - if (const auto* as_tensor = result->try_as()) { - params_types = as_tensor->items; - } - result = TypeDataFunCallable::create(std::move(params_types), return_type); - } - - return result; -} - -TypePtr parse_type_from_tokens(Lexer& lex) { - return parse_type_expression(lex); -} - -// for internal usage only -TypePtr parse_type_from_string(std::string_view text) { - Lexer lex(text); - return parse_type_expression(lex); -} std::ostream& operator<<(std::ostream& os, TypePtr type_data) { return os << (type_data ? type_data->as_human_readable() : "(nullptr-type)"); diff --git a/tolk/type-system.h b/tolk/type-system.h index 6a6b1625aa..cd0c34581b 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -16,11 +16,10 @@ */ #pragma once -#include "src-file.h" #include "fwd-declarations.h" -#include -#include #include +#include +#include namespace tolk { @@ -30,21 +29,14 @@ namespace tolk { * Every unique TypeData is created only once, so for example TypeDataTensor::create(int, int) * returns one and the same pointer always. * - * In Tolk code, types after colon `var v: (int, T)` are parsed to TypeData. - * See parse_type_from_tokens(). - * So, AST nodes which can have declared types (local/global variables and others) store a pointer to TypeData. - * - * Type inferring also creates TypeData for inferred expressions. All AST expression nodes have inferred_type. + * Type inferring creates TypeData for inferred expressions. All AST expression nodes have inferred_type. * For example, `1 + 2`, both operands are TypeDataInt, its result is also TypeDataInt. * Type checking also uses TypeData. For example, `var i: slice = 1 + 2`, at first rhs (TypeDataInt) is inferred, * then lhs (TypeDataSlice from declaration) is checked whether rhs can be assigned. * See can_rhs_be_assigned(). * - * Note, that while initial parsing Tolk files to AST, known types (`int`, `cell`, etc.) are created as-is, - * but user-defined types (`T`, `MyStruct`, `MyAlias`) are saved as TypeDataUnresolved. - * After all symbols have been registered, resolving identifiers step is executed, where particularly - * all TypeDataUnresolved instances are converted to a resolved type. At inferring, no unresolved remain. - * For instance, `fun f(v: T)`, at first "T" of `v` is unresolved, and then converted to TypeDataGenericT. + * At the moment of parsing, types after colon `var v: (int, T)` are parsed to AST (AnyTypeV), + * and all symbols have been registered, AST representation resolved to TypeData, see pipe-resolve-types.cpp. */ class TypeData { // bits of flag_mask, to store often-used properties and return them without tree traversing @@ -58,8 +50,7 @@ class TypeData { enum flag_mask { flag_contains_unknown_inside = 1 << 1, flag_contains_genericT_inside = 1 << 2, - flag_contains_unresolved_inside = 1 << 3, - flag_contains_type_alias_inside = 1 << 4, + flag_contains_type_alias_inside = 1 << 3, }; explicit TypeData(int flags_with_children, int width_on_stack) @@ -89,10 +80,8 @@ class TypeData { bool has_unknown_inside() const { return flags & flag_contains_unknown_inside; } bool has_genericT_inside() const { return flags & flag_contains_genericT_inside; } - bool has_unresolved_inside() const { return flags & flag_contains_unresolved_inside; } bool has_type_alias_inside() const { return flags & flag_contains_type_alias_inside; } - using TraverserCallbackT = std::function; using ReplacerCallbackT = std::function; virtual int get_type_id() const = 0; @@ -104,10 +93,6 @@ class TypeData { return true; } - virtual void traverse(const TraverserCallbackT& callback) const { - callback(this); - } - virtual TypePtr replace_children_custom(const ReplacerCallbackT& callback) const { return callback(this); } @@ -135,7 +120,6 @@ class TypeDataAlias final : public TypeData { std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; - void traverse(const TraverserCallbackT& callback) const override; bool can_hold_tvm_null_instead() const override; }; @@ -313,7 +297,6 @@ class TypeDataFunCallable final : public TypeData { std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; - void traverse(const TraverserCallbackT& callback) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; }; @@ -382,18 +365,17 @@ class TypeDataTensor final : public TypeData { std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; - void traverse(const TraverserCallbackT& callback) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; bool can_hold_tvm_null_instead() const override; }; /* - * `[int, slice]` is TypeDataTypedTuple, a TVM 'tuple' under the hood, contained in 1 stack slot. + * `[int, slice]` is TypeDataBrackets, a TVM 'tuple' under the hood, contained in 1 stack slot. * Unlike TypeDataTuple (untyped tuples), it has a predefined inner structure and can be assigned as - * `var [i, cs] = [0, ""]` (where a and b become two separate variables on a stack, int and slice). + * `var [i, cs] = [0, ""]` (where i and cs become two separate variables on a stack, int and slice). */ -class TypeDataTypedTuple final : public TypeData { - TypeDataTypedTuple(int children_flags, std::vector&& items) +class TypeDataBrackets final : public TypeData { + TypeDataBrackets(int children_flags, std::vector&& items) : TypeData(children_flags, 1) , items(std::move(items)) {} @@ -408,7 +390,6 @@ class TypeDataTypedTuple final : public TypeData { std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; - void traverse(const TraverserCallbackT& callback) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; }; @@ -533,7 +514,6 @@ class TypeDataUnion final : public TypeData { std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; - void traverse(const TraverserCallbackT& callback) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; bool can_hold_tvm_null_instead() const override; }; @@ -559,30 +539,6 @@ class TypeDataUnknown final : public TypeData { bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; -/* - * "Unresolved" is not actually a type — it's an intermediate state between parsing and resolving. - * At parsing to AST, unrecognized type names (MyEnum, MyStruct, T) are parsed as TypeDataUnresolved, - * and after all source files parsed and global symbols registered, they are replaced by actual ones. - * Example: `fun f(v: T)` at first v is TypeDataUnresolved("T"), later becomes TypeDataGenericT. - */ -class TypeDataUnresolved final : public TypeData { - TypeDataUnresolved(std::string&& text, SrcLocation loc) - : TypeData(flag_contains_unresolved_inside, -999999) - , text(std::move(text)) - , loc(loc) {} - -public: - const std::string text; - const SrcLocation loc; - - static TypePtr create(std::string&& text, SrcLocation loc); - - int get_type_id() const override; - std::string as_human_readable() const override { return text + "*"; } - bool can_rhs_be_assigned(TypePtr rhs) const override; - bool can_be_casted_with_as_operator(TypePtr cast_to) const override; -}; - /* * `never` is a special type meaning "no value can be hold". * Is may appear due to smart casts, for example `if (x == null && x != null)` makes x "never". @@ -629,10 +585,6 @@ class TypeDataVoid final : public TypeData { // -------------------------------------------- -class Lexer; -TypePtr parse_type_from_tokens(Lexer& lex); -TypePtr parse_type_from_string(std::string_view text); - void type_system_init(); } // namespace tolk From ff5e49403a7cc7df6004c7088d27526731b58c27 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 29 Apr 2025 14:23:19 +0300 Subject: [PATCH 231/524] Fix level calculation in ProofStorageStat --- crypto/vm/boc.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index d3ef57b742..837624563a 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -1255,6 +1255,7 @@ bool VmStorageStat::add_storage(const CellSlice& cs) { } void ProofStorageStat::add_loaded_cell(const Ref& cell, td::uint8 max_level) { + max_level = std::min(max_level, Cell::max_level); auto& [status, size] = cells_[cell->get_hash(max_level)]; if (status == c_loaded) { return; From 0f9dd15b9a5ab1474470b8e8ee04983ee4efd606 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 27 Feb 2025 18:48:20 +0400 Subject: [PATCH 232/524] [Tolk] Implement generic structs and aliases Now structures can be generic: Container and similar. The same goes for type aliases: type A = Container. Like generic functions, every struct is instantiated, "Container" and "Container" become different symbols, and on instantiation they walk though the pipeline. Type inferring and checking are done only upon instantiation. --- tolk-tester/tests/generics-1.tolk | 17 +- tolk-tester/tests/generics-2.tolk | 464 ++++++++++++++++++ tolk-tester/tests/generics-3.tolk | 146 ++++++ .../tests/invalid-declaration/err-1285.tolk | 14 + .../tests/invalid-declaration/err-1378.tolk | 11 + .../tests/invalid-declaration/err-1876.tolk | 11 + .../tests/invalid-semantics/err-4081.tolk | 20 + .../tests/invalid-semantics/err-4119.tolk | 16 + .../tests/invalid-semantics/err-4193.tolk | 2 +- .../tests/invalid-semantics/err-4218.tolk | 14 + .../tests/invalid-semantics/err-4260.tolk | 2 +- .../tests/invalid-semantics/err-4291.tolk | 7 + .../tests/invalid-semantics/err-4378.tolk | 11 + .../tests/invalid-semantics/err-4387.tolk | 17 + .../tests/invalid-semantics/err-4429.tolk | 20 + .../tests/invalid-semantics/err-4493.tolk | 13 + .../tests/invalid-semantics/err-4512.tolk | 8 + .../tests/invalid-semantics/err-4519.tolk | 8 +- .../tests/invalid-semantics/err-4530.tolk | 17 + .../tests/invalid-semantics/err-4580.tolk | 12 + .../tests/invalid-semantics/err-4628.tolk | 16 + .../tests/invalid-semantics/err-4630.tolk | 13 + .../tests/invalid-semantics/err-4711.tolk | 22 + .../tests/invalid-semantics/err-4862.tolk | 18 + .../tests/invalid-semantics/err-4880.tolk | 2 +- .../tests/invalid-semantics/err-4925.tolk | 20 + .../tests/invalid-syntax/err-3173.tolk | 8 + .../tests/invalid-syntax/err-3450.tolk | 12 + .../tests/invalid-typing/err-6070.tolk | 26 + .../tests/invalid-typing/err-6110.tolk | 2 +- .../tests/invalid-typing/err-6145.tolk | 20 + .../tests/invalid-typing/err-6369.tolk | 2 +- .../tests/invalid-typing/err-6409.tolk | 10 + .../tests/invalid-typing/err-6473.tolk | 18 + .../tests/invalid-typing/err-6480.tolk | 27 + .../tests/invalid-typing/err-6618.tolk | 17 + .../tests/invalid-typing/err-6620.tolk | 16 + .../tests/invalid-typing/err-6642.tolk | 21 + .../tests/invalid-typing/err-6919.tolk | 13 + .../tests/invalid-typing/err-6948.tolk | 16 + tolk-tester/tests/var-apply-tests.tolk | 18 + .../tests/warnings-not-errors/warnings-4.tolk | 15 + tolk/ast-from-tokens.cpp | 89 +++- tolk/ast-replicator.h | 248 +++++----- tolk/ast-stringifier.h | 2 + tolk/ast-visitor.h | 4 +- tolk/ast.h | 41 +- tolk/fwd-declarations.h | 2 + tolk/generics-helpers.cpp | 277 ++++++++--- tolk/generics-helpers.h | 70 +-- tolk/lexer.cpp | 6 + tolk/lexer.h | 1 + tolk/pipe-ast-to-legacy.cpp | 9 +- tolk/pipe-check-inferred-types.cpp | 2 +- tolk/pipe-constant-folding.cpp | 2 +- tolk/pipe-infer-types-and-calls.cpp | 284 ++++++++--- tolk/pipe-register-symbols.cpp | 64 ++- tolk/pipe-resolve-identifiers.cpp | 4 + tolk/pipe-resolve-types.cpp | 154 ++++-- tolk/pipeline.h | 9 +- tolk/smart-casts-cfg.cpp | 6 +- tolk/smart-casts-cfg.h | 1 + tolk/symtable.cpp | 14 + tolk/symtable.h | 50 +- tolk/type-system.cpp | 87 +++- tolk/type-system.h | 40 +- 66 files changed, 2227 insertions(+), 401 deletions(-) create mode 100644 tolk-tester/tests/generics-2.tolk create mode 100644 tolk-tester/tests/generics-3.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1285.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1378.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1876.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4081.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4119.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4218.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4291.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4378.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4387.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4429.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4493.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4512.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4530.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4580.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4628.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4630.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4711.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4862.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4925.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3173.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3450.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6070.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6145.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6409.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6473.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6480.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6618.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6620.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6642.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6919.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6948.tolk create mode 100644 tolk-tester/tests/warnings-not-errors/warnings-4.tolk diff --git a/tolk-tester/tests/generics-1.tolk b/tolk-tester/tests/generics-1.tolk index feb7a8393e..ba46f3440e 100644 --- a/tolk-tester/tests/generics-1.tolk +++ b/tolk-tester/tests/generics-1.tolk @@ -17,8 +17,15 @@ fun getTwo(): X { return 2 as X; } fun takeInt(a: int) { return a; } +struct Wrapper { + value: T; +} +struct Err { + errPayload: T +} + @method_id(102) -fun test102(): (int, int, int, [int, int]) { +fun test102(): (int, int, int, [int, int], Wrapper< Wrapper >) { var a: int = getTwo(); var _: int = getTwo(); var b = getTwo() as int; @@ -32,7 +39,10 @@ fun test102(): (int, int, int, [int, int]) { ab_tup.0 = getTwo(); ab_tup.1.1 = getTwo(); var ab_tens_al: Tensor2Int = (getTwo(), getTwo()); - return (eq1(a), eq2(b), takeInt(getTwo()), [getTwo(), ab_tens.1.1]); + var cint: Wrapper = { value: getTwo() }; + var cuni: Wrapper | Err = Wrapper { value: getTwo() }; + cuni = Err { errPayload: getTwo() }; + return (eq1(a), eq2(b), takeInt(getTwo()), [getTwo(), ab_tens.1.1], eq1({ value: { value: getTwo() } } as Wrapper< Wrapper >)); } @method_id(103) @@ -204,7 +214,7 @@ fun main(x: int): (int, [Tup2Int]) { /** @testcase | 0 | 1 | 1 [ [ 1 2 ] ] @testcase | 101 | 0 | 0 0 0 [ 0 0 ] 0 0 0 [ 0 0 ] 0 0 0 [] -@testcase | 102 | | 2 2 2 [ 2 2 ] +@testcase | 102 | | 2 2 2 [ 2 2 ] 2 @testcase | 103 | 0 | 0 100 100 @testcase | 104 | 0 | [ 1 (null) 2 ] [ 2 -1 0 ] @testcase | 105 | | 3 @@ -224,6 +234,7 @@ fun main(x: int): (int, [Tup2Int]) { @fif_codegen DECLPROC eq1<[int,int]> @fif_codegen DECLPROC eq1 @fif_codegen DECLPROC eq1 +@fif_codegen DECLPROC eq1>> @fif_codegen DECLPROC getTwo @fif_codegen_avoid DECLPROC eq1 diff --git a/tolk-tester/tests/generics-2.tolk b/tolk-tester/tests/generics-2.tolk new file mode 100644 index 0000000000..cde9d99297 --- /dev/null +++ b/tolk-tester/tests/generics-2.tolk @@ -0,0 +1,464 @@ +struct Empty {} + +struct Wrapper { + value: T; +} + +type PairAlias = Pair; +struct Pair { + first: T1, + second: T2 +} + +type MyInt = int; +type WrappedInt = Wrapper; +type WrapperAlias = Wrapper; + +struct DeepNested { + wrapper: WrapperAlias>; + pair: PairAlias; +} + + +fun eq(v: T): T { return v; } + +fun swap(p: Pair): Pair { + return { first: p.second, second: p.first }; +} + +fun wrap(value: T): Wrapper { + return { value }; +} + +fun wrap2(value: T) { + return WrapperAlias { value }; +} + +fun makePair(a: A, b: B) { + var c: Pair = { first: a, second: b }; + return c; +} + + +@method_id(101) +fun test1() { + var c1: Wrapper = { value: 23 }; + __expect_type(c1, "Wrapper"); + var c2: WrapperAlias = { value: 23 }; + __expect_type(c2, "WrapperAlias"); + __expect_type(c2 as Wrapper, "Wrapper"); + __expect_type(c2 as WrappedInt, "WrappedInt"); + return (c1, c2); +} + +@method_id(102) +fun test2(c: Wrapper) { + var c2 = Wrapper { value: c.value + 1 }; + var c3 = Wrapper { value: c.value + c2.value }; + __expect_type(c3, "Wrapper"); + var c4: Wrapper? = c.value > 10 ? null : { value: c.value }; + return (c, c2, c3, c4); +} + +@pure +fun getWrappervalue1(c: Wrapper) { + return c.value; +} + +@pure +fun getWrappervalue2(c: T) { + return c.value; +} + +@method_id(103) +fun test3() { + var c1 = wrap(10); + var c2 = wrap(10 as int?); + var c3 = wrap2<(int, int)?>(null); + __expect_type(c1, "Wrapper"); + __expect_type(c2, "Wrapper"); + __expect_type(c3, "Wrapper<(int, int)?>"); + return (getWrappervalue1(c1), getWrappervalue1(c2), getWrappervalue1(c3), 777, getWrappervalue2(c1), getWrappervalue2(c2), getWrappervalue2(c3)); +} + +@method_id(104) +fun test4() { + var c1 = wrap(10); + var c2 = wrap(c1); + var c3 = wrap(Wrapper { value: beginCell() }); + __expect_type(c2, "Wrapper>"); + __expect_type(c3, "Wrapper>"); + __expect_type(c3.value, "Wrapper"); + __expect_type(c3.value.value, "builder"); + c3.value.value.storeInt(1, 32).storeInt(2, 32).storeInt(3, 32); + var c4 = wrap2(Wrapper { value: getWrappervalue1(c3.value).endCell().beginParse() }); + __expect_type(c4, "Wrapper>"); + var sumOf1 = 0; + sumOf1 += getWrappervalue1(getWrappervalue1(c4)).loadInt(32); // a function returns a copy + sumOf1 += getWrappervalue2(c4.value).loadInt(32); // so original slice isn't mutated + return (c1.value += c4.value.value.loadInt(32), c1.getWrappervalue1(), c1.value += c4.value.value.loadInt(32), c1.getWrappervalue2(), c1, sumOf1); +} + +@method_id(105) +fun test5(setNull: bool) { + var c1: Wrapper < WrappedInt >? = { value: { value: 80 } }; + __expect_type(c1, "Wrapper"); + if (setNull) { + c1 = null; + } + __expect_type(c1, "Wrapper?"); + return c1; +} + +fun createNestedWrapperOrNull(setNull: bool, initVal: T) { + if (setNull) { + return null; + } + var c1 = Wrapper> {value:{value:initVal}}; + var c2: Wrapper< Wrapper<(T)> > = {value:{value:initVal}}; + var c3: Wrapper< Wrapper<((T | T))> > = WrapperAlias {value:{value:initVal}}; + var c4: Wrapper< Wrapper<((T | T))> > = Wrapper> {value:c3.value}; + var c5: Wrapper< WrapperAlias<((T | T))> > = WrapperAlias> {value:c3.value}; + var c6: WrapperAlias< WrapperAlias<((T | T))> > = Wrapper> {value:c3.value}; + return c1; +} + +@method_id(106) +fun test6() { + var c1 = createNestedWrapperOrNull(true, (1, 2)); + var c2 = createNestedWrapperOrNull(false, (1, 2)); + __expect_type(c1, "Wrapper>?"); + getWrappervalue1(c2!); + getWrappervalue2(c2!); + return (c1, 777, c2); +} + +@method_id(107) +fun test7(i8: int8) { + var c1 = Pair { first: 10, second: i8 }; + var c2 = eq(c1); + __expect_type(c2, "Pair"); + var c3: Pair = { first: null, second: null }; + return (c2.first += 10, c2, swap(c2), 777, c3); +} + +@method_id(108) +fun test8(lastNull: bool) { + var c1 = Wrapper { value: () }; + var c2 = Pair> { first: {}, second: {first:{},second:{}} }; + var c3 = Pair { first: c1, second: Wrapper { value: Empty{} } }; + var c4 = Pair { first: c1, second: Wrapper { value: lastNull ? null : Empty{} } }; + __expect_type(c4, "Pair, Wrapper>"); + return (c1, c2, c3, 777, c4, 777, c4.first, c4.second.value); +} + +@method_id(109) +fun test9() { + var c1: WrappedInt = { value: 10 }; + var c2: WrappedInt = WrappedInt { value: 20 }; + var c3: WrappedInt = Wrapper { value: 30 }; + var c4: Wrapper = { value: 40 }; + c1 = c2 = c3 = c4; + __expect_type(c1, "WrappedInt"); + __expect_type(c1 as Wrapper, "Wrapper"); + + var c5: Pair, WrappedInt> = { first: {value:50}, second: {value:50} }; + c5.first = c1; c5.first = c4; + c5.second = c1; c5.second = c4; + + var c6 = Wrapper> { value: { value: 60 } }; + var c7 = Wrapper { value: { value: 70 } }; + c6 = c7; c7 = c6; + c6 as Wrapper; + c7 as Wrapper>; + + var p1: PairAlias = { first: 42, second: WrappedInt { value: 200 } }; + __expect_type(p1, "PairAlias"); + __expect_type(p1 as Pair>, "Pair>"); + var p2: Pair> = p1; + + __expect_type(DeepNested { + wrapper: wrap(wrap(999)), + pair: makePair(10, 20), + }, "DeepNested"); + + return (c5, c7); +} + +type Int16Or8 = int8 | int16; + +@method_id(110) +fun test10() { + var p1 = Pair { first: 5, second: 10 as int8 }; + var p2: Pair = { first: 10, second: 5 as int16 }; + var p3 = Pair { first: 20 as int|builder, second: (20 as int8) as Int16Or8 }; + __expect_type(p1, "Pair"); + __expect_type(p2, "Pair"); + __expect_type(p3, "Pair"); + p1 = p2; p1 = p3; p2 = p1; p2 = p3; p3 = p1; p3 = p2; + p2 as Pair; p3 as Pair; + p1 as Pair; p3 as Pair; + p1 as Pair; p3 as Pair; + return p3; +} + +@method_id(111) +fun test11() { + // type_id of these instantiations are the same + var c1 = { value: 5 } as int | Wrapper; + var c2 = { value: 5 } as int | Wrapper; + var c3 = { value: 5 } as int | Wrapper; + var c4 = { value: 5 } as int | Wrapper; + c1 = c1; c1 = c2; c1 = c3; c1 = c4; + c2 = c1; c2 = c2; c2 = c3; c2 = c4; + c3 = c1; c3 = c2; c3 = c3; c3 = c4; + c4 = c1; c4 = c2; c4 = c3; c4 = c4; + return (c1, c2, c3, c4); +} + +@method_id(112) +fun test12(is1: bool) { + var c1: Wrapper = { value: 10 }; + var c2: Wrapper = { value: 20 }; + var c3: Wrapper | WrapperAlias = is1 ? c1 : c2; + __expect_type(c3, "Wrapper"); + var m3: Wrapper | Wrapper = is1 ? c1 : Wrapper { value: null }; + __expect_type(m3, "Wrapper | Wrapper"); + return (c3, 777, m3); +} + +fun makeIdentity(): (T) -> T { + return eq; +} + +@method_id(113) +fun test13() { + return makeIdentity>()({ first: 30, second: true }); +} + +struct Storage { + Wrapper: Wrapper; + backup: T?; +} + +@method_id(114) +fun test14() { + var s1 = Storage { Wrapper: wrap(999), backup: null }; + var s2 = Storage { Wrapper: wrap(makePair(1, "")), backup: makePair(2, "") }; + __expect_type(s1, "Storage"); + __expect_type(s2, "Storage>"); + return (s1, s2.backup!.first); +} + +struct DualStorage { + primary: Wrapper; + secondary: Wrapper; +} + +type IntOrBuilder = int | builder; + +@method_id(115) +fun test15() { + var s1 = DualStorage { primary: wrap(10), secondary: wrap(false), }; + __expect_type(s1, "DualStorage"); + var s2: DualStorage = { primary: wrap(100 as IntOrBuilder), secondary: wrap(true), }; + var s3 = DualStorage { primary: wrap(200 as IntOrBuilder), secondary: wrap(true), }; + return (s1, s2 = s3, s3 = s2); +} + +type ComplexWrapPair = Wrapper>; + +@method_id(116) +fun test16() { + var c1: ComplexWrapPair = wrap(makePair(1, 2)); + __expect_type(c1, "ComplexWrapPair"); + var c2 = wrap(makePair(3, 4)); + __expect_type(c2, "Wrapper>"); + if (0) { c1 = c2; c2 = c1; } + c1 as Wrapper>; c2 as ComplexWrapPair; + return (c1, c2); +} + +fun transform(x: T, f: (T) -> U): U { + return f(x); +} + +@method_id(117) +fun test17() { + var id = makeIdentity(); + var result = transform(100 as MyInt, id); + __expect_type(result, "MyInt"); + var r2 = transform(wrap(123), wrap>); + __expect_type(r2, "Wrapper>"); + return (result, r2); +} + +struct UnionHolder { + item: Wrapper; + extra: Wrapper; +} + +@method_id(118) +fun test18() { + var uh: UnionHolder = { item: wrap(123 as int|slice), extra: wrap("" as int|slice|bool) }; + __expect_type(uh, "UnionHolder"); + __expect_type(uh.item, "Wrapper"); + __expect_type(uh.extra, "Wrapper"); + + var ub = UnionHolder { item: wrap(123 as int|bool), extra: wrap(true as int|bool) }; + __expect_type(ub.extra, "Wrapper"); + + return (uh.item, 777, ub); +} + +type StrangeInt = int; +type StrangeAlsoInt = StrangeInt<()>; + +@method_id(119) +fun test19() { + var i1: StrangeInt = 20; + var i2: StrangeInt = 30; + var i3: StrangeInt> = 40 as StrangeAlsoInt; + i1 = i2; i2 = i1; i1 = i3; i3 = i1; + return (i1 as StrangeInt<()>, i3 as StrangeAlsoInt, i1 is StrangeInt, i1 is StrangeInt, i3 is StrangeInt, i3 is StrangeAlsoInt); +} + +@method_id(120) +fun test20() { + var w1: Wrapper | int = { value: 10 }; + var w2: int | Wrapper = { value: 30 }; + var w3: slice | Wrapper<(int8, int16)> = { value: (50, 30) }; + __expect_type(w3.value, "(int8, int16)"); + var w4 = { value: null } as int | Wrapper; + var w5 = { value: null } as int | Wrapper?>; + var w6: int | Wrapper | null | slice | builder | (int, int) = { value: 10 }; + if (10 > 3) { + w6 = beginCell(); + } + __expect_type(w6, "builder | Wrapper"); + return (w4, 777, w5, w5 is Wrapper, w6 is builder); +} + +struct AnotherWrapper { + value: int; +} + +fun constructAnotherWrapper(value: slice): AnotherWrapper { + return AnotherWrapper { value }; +} + +fun test21(w: AnotherWrapper) { + var w2: AnotherWrapper = { value: beginCell() }; + var w3 = constructAnotherWrapper(beginCell()); + __expect_type(w2, "AnotherWrapper"); + __expect_type(w3, "AnotherWrapper"); + __expect_type(constructAnotherWrapper>, "(Wrapper) -> AnotherWrapper>"); + w.value = w2.value; + w3.value = beginCell(); + w = w2; +} + +fun test22(w: int | WrapperAlias | slice) { + match (w) { + int => {} + WrapperAlias => {} + slice => {} + } + __expect_type(w, "int | Wrapper | slice"); + match (w) { + MyInt => {} + slice => {} + WrapperAlias => {} + } +}; + +@method_id(123) +fun test23() { + var w1: Wrapper = { value: 10 }; + var w2: Wrapper> = { value: w1 }; + var w3: Wrapper>> = { value: w2 }; + var w4: Wrapper>>> = { value: w3 }; + var w5: Wrapper>>>> = { value: w4 }; + eq>(w1); + eq>>(w2); + eq>>>(w3); + return (w1, w2, w3, w4, w5); +} + +fun takeSomethingAndWrapper(something: T, w: Wrapper) { + w.value = something; +} + +struct HasSomethingAndWrapper { + something: T; + w: Wrapper; +} + +fun test24() { + // check that parameter w can be understood from context (after deducing parameter something) + takeSomethingAndWrapper(0, {value: 0}); + takeSomethingAndWrapper(null as slice?, {value: null}); + takeSomethingAndWrapper(Wrapper { value: 10 }, {value: { value: 20 }}); + // same for creating an object and deducing Ts one by one + HasSomethingAndWrapper { something: 0, w: { value: 0 } }; + val o = HasSomethingAndWrapper { something: 5 as uint8, w: { value: 5 } }; + __expect_type(o, "HasSomethingAndWrapper"); +} + + +fun main(c: Wrapper, d: WrappedInt) { + __expect_type(c, "Wrapper"); + __expect_type(c!, "Wrapper"); + __expect_type(d, "WrappedInt"); + __expect_type(d.value, "int"); + __expect_type(WrappedInt { value: 200 }, "Wrapper"); + __expect_type(Wrapper{ value: c.value }, "Wrapper"); + __expect_type(Wrapper{ value: c.value as bytes32 }, "Wrapper"); + __expect_type(Wrapper{ value: Wrapper { value: (c.value as bytes32?)! }.value }, "Wrapper"); + __expect_type(Wrapper{ value: c.value }, "Wrapper"); + __expect_type(Wrapper{ value: c.value.loadInt(32) }, "Wrapper"); + __expect_type(WrapperAlias{ value: 10 }, "Wrapper"); + __expect_type(WrapperAlias{ value: 10 } as WrapperAlias, "WrapperAlias"); + __expect_type(swap(Pair { first: 3, second: beginCell() }), "Pair"); +} + +/** +@testcase | 101 | | 23 23 +@testcase | 102 | 9 | 9 10 19 9 +@testcase | 102 | 19 | 19 20 39 (null) +@testcase | 103 | | 10 10 (null) (null) 0 777 10 10 (null) (null) 0 +@testcase | 104 | | 11 11 13 13 13 2 +@testcase | 105 | 0 | 80 +@testcase | 105 | -1 | (null) +@testcase | 106 | | (null) (null) 0 777 1 2 142 +@testcase | 107 | 5 | 20 20 5 5 20 777 (null) (null) (null) 0 +@testcase | 108 | 0 | 777 143 777 143 +@testcase | 108 | -1 | 777 0 777 0 +@testcase | 109 | | 40 40 70 +@testcase | 110 | | 20 1 20 42 +@testcase | 111 | | 5 1 132 5 1 132 5 1 132 5 1 132 +@testcase | 112 | -1 | 10 1 777 10 1 133 +@testcase | 112 | 0 | 20 1 777 (null) 0 134 +@testcase | 113 | | 30 -1 +@testcase | 114 | | 999 (null) 2 +@testcase | 115 | | 10 0 200 1 -1 200 1 -1 +@testcase | 116 | | 1 2 3 4 +@testcase | 117 | | 100 123 +@testcase | 118 | | 123 1 777 123 1 -1 2 +@testcase | 119 | | 40 40 -1 -1 -1 -1 +@testcase | 120 | | (null) 137 777 (null) (null) 0 139 -1 -1 +@testcase | 123 | | 10 10 10 10 10 + + +@fif_codegen +""" + test1 PROC:<{ + // + 23 PUSHINT // c1=23 + DUP // c1=23 c2=23 + }> +""" + +@fif_codegen DECLPROC getWrappervalue1> +@fif_codegen DECLPROC getWrappervalue2>> + */ diff --git a/tolk-tester/tests/generics-3.tolk b/tolk-tester/tests/generics-3.tolk new file mode 100644 index 0000000000..4ec7b29b40 --- /dev/null +++ b/tolk-tester/tests/generics-3.tolk @@ -0,0 +1,146 @@ +struct Ok { + result: T; +} + +struct Err { + errPayload: T; +} + +type OkAlias = Ok; + +type Response = Ok | Err; + +@pure +fun getResponse(success: bool): Response { + return success ? Ok { result: 10 } : Err { errPayload: beginCell().storeInt(-1,32).endCell().beginParse() }; +} + +@method_id(101) +fun test1(getOk: bool) { + return match (var r = getResponse(getOk)) { + Ok => r.result, + Err => r.errPayload.loadInt(32) + }; +} + +fun test2() { + var r = getResponse(true); + match (r) { + Ok => r as OkAlias, + Err => r as Err, + } + match (r) { + OkAlias => r as Ok, + Err => {} + } + match (r) { + Ok => { __expect_type(r, "Ok"); } + Err => { __expect_type(r, "Err"); } + } + + if (r is Ok) { __expect_type(r, "Ok"); } + if (r is Ok) { __expect_type(r, "Ok"); } + if (r is OkAlias) { __expect_type(r, "Ok"); } + if (r !is OkAlias) { __expect_type(r, "Err"); } + if (r is Ok && r !is Err && r is Ok) { __expect_type(r, "Ok"); } +} + +@method_id(103) +fun test3() { + var r: Response = getResponse(true); + r = Err { errPayload: "" }; + __expect_type(r, "Err"); + r = OkAlias { result: 10 }; + __expect_type(r, "Ok"); + return (r, r is Ok, (r as Response) is Err); +} + +@method_id(104) +fun test4() { + var r1 = OkAlias { result: OkAlias { result: 10 } } as Response< Ok, slice >; + match (r1) { + Ok => { __expect_type(r1, "Ok>"); r1.result.result; } + Err => {} + } + var r2: Response< Response, slice > = Err { errPayload: "" }; + __expect_type(r2, "Err"); + r2 = 10>3 ? Ok { result: Ok { result: 12 } } : r2; + var mm1 = match (r2) { + OkAlias> => true, + Err => false + }; + var mm2 = match (r2) { + Ok => true, + Err => false + }; + return (r2, mm1, mm2, r2 is Ok>, r2 is OkAlias>, r2 is Err, r2 is Ok, r2 is Err, r2 !is Ok); +} + +@method_id(105) +fun test5() { + var o1: Ok = { result: 1 }; + var o2: OkAlias = { result: 2 }; + return (o1 is Ok, o1 is OkAlias, o2 is Ok, o2 is OkAlias, o1 is Ok, o1 is OkAlias); +} + +type OkInt = Ok; + +fun test6(w: Response | int) { + match (w) { + int => {} + Ok => {} + Err => {} + } + match (w) { + OkInt => {} + int => {} + Err => {} + } +} + +fun test7(w: OkInt | Err | int) { + match (w) { + Ok => {} + Err => {} + int => {} + } +} + +struct WithDef { + f1: T? = null; + f2: int? = null as int?; + f3: T? = null as T?; + price: coins = ton("0.05"), + slice: slice = stringHexToSlice("010203"), +} + +@method_id(108) +fun test8() { + var w1: WithDef = {}; + var w2: WithDef> = { price: ton("0.1"), f3: { result: 12 } }; + return (w1.price, w1.f1, w1.f3, w2.price, w2.f3!.result, w2.f3, w2.slice.getRemainingBitsCount()); +} + + +fun main() { + __expect_type(getResponse(true), "Response"); +} + +/** +@testcase | 101 | -1 | 10 +@testcase | 101 | 0 | -1 +@testcase | 103 | | 10 -1 0 +@testcase | 104 | | 12 129 132 -1 -1 -1 0 0 -1 0 0 +@testcase | 105 | | -1 -1 -1 -1 0 0 +@testcase | 108 | | 50000000 (null) (null) 100000000 12 12 24 + +@fif_codegen +""" + test3 PROC:<{ + // + 10 PUSHINT // r.USlot1=10 + -1 PUSHINT // r.USlot1=10 '11=-1 + FALSE // r.USlot1=10 '11=-1 '13 + }> +""" + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1285.tolk b/tolk-tester/tests/invalid-declaration/err-1285.tolk new file mode 100644 index 0000000000..c647a15359 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1285.tolk @@ -0,0 +1,14 @@ +struct Pair { + first: T1; + second: T2; +} + +struct DoesntWork { + p: Pair; +} + +/** +@compilation_should_fail +@stderr struct `Pair` expects 2 type arguments, but 1 provided +@stderr p: Pair + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1378.tolk b/tolk-tester/tests/invalid-declaration/err-1378.tolk new file mode 100644 index 0000000000..bd2750fcb8 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1378.tolk @@ -0,0 +1,11 @@ +struct Wrapper { + value: T; +} + +type ErrWrapper = int | Wrapper; + +/** +@compilation_should_fail +@stderr type `Wrapper` is generic, you should provide type arguments +@stderr int | Wrapper; + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1876.tolk b/tolk-tester/tests/invalid-declaration/err-1876.tolk new file mode 100644 index 0000000000..281a4af7d8 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1876.tolk @@ -0,0 +1,11 @@ +struct Pair { + first: T1; + second: T2; + first: int; +} + +/** +@compilation_should_fail +@stderr redeclaration of field `first` +@stderr first: int + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4081.tolk b/tolk-tester/tests/invalid-semantics/err-4081.tolk new file mode 100644 index 0000000000..4442cf2d50 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4081.tolk @@ -0,0 +1,20 @@ +struct Wrapper { + field: T; +} + +type MyInt = int; +type WrapperAlias = Wrapper; + +fun main(w: int | WrapperAlias | WrapperAlias) { + match (w) { + MyInt => {} + Wrapper => {} + WrapperAlias => {} + } +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: duplicated `Wrapper` +@stderr WrapperAlias + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4119.tolk b/tolk-tester/tests/invalid-semantics/err-4119.tolk new file mode 100644 index 0000000000..bb68393151 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4119.tolk @@ -0,0 +1,16 @@ +struct Wrapper { + value: T; +} + +type WrapperAlias = Wrapper; + +fun main() { + var w1 = WrapperAlias { value: 10 } as WrapperAlias | int | Wrapper; + w1 is Wrapper; +} + +/** +@compilation_should_fail +@stderr can not deduce type arguments for `Wrapper`, provide them manually +@stderr w1 is Wrapper + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4193.tolk b/tolk-tester/tests/invalid-semantics/err-4193.tolk index b0b31fa7da..6e7198c61b 100644 --- a/tolk-tester/tests/invalid-semantics/err-4193.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4193.tolk @@ -8,6 +8,6 @@ fun cantInstantiateNonGenericStruct() { /** @compilation_should_fail -@stderr generic T not expected here +@stderr type `User` is not generic @stderr User */ diff --git a/tolk-tester/tests/invalid-semantics/err-4218.tolk b/tolk-tester/tests/invalid-semantics/err-4218.tolk new file mode 100644 index 0000000000..baab5a2a3d --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4218.tolk @@ -0,0 +1,14 @@ +struct Wrapper { + value: T; +} + +type nested = Wrapper>>; + +fun main() { + var w3: nested = {}; +} + +/** +@compilation_should_fail +@stderr field `value` missed in initialization of struct `Wrapper>>` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4260.tolk b/tolk-tester/tests/invalid-semantics/err-4260.tolk index 6f174a595c..9c9a73b31e 100644 --- a/tolk-tester/tests/invalid-semantics/err-4260.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4260.tolk @@ -10,6 +10,6 @@ fun cantInstantiateNonGenericStruct() { /** @compilation_should_fail -@stderr generic T not expected here +@stderr type `UserAlias` is not generic @stderr UserAlias */ diff --git a/tolk-tester/tests/invalid-semantics/err-4291.tolk b/tolk-tester/tests/invalid-semantics/err-4291.tolk new file mode 100644 index 0000000000..907cc76c85 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4291.tolk @@ -0,0 +1,7 @@ +struct T1 { field: T1; } +struct CollisionName { item: T1; } + +/** +@compilation_should_fail +@stderr type `T1` is not generic + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4378.tolk b/tolk-tester/tests/invalid-semantics/err-4378.tolk new file mode 100644 index 0000000000..7f1225670f --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4378.tolk @@ -0,0 +1,11 @@ +struct Wrapper { + field: T; +} + +fun f(w: Wrapper) { +} + +/** +@compilation_should_fail +@type `Wrapper` is generic, you should provide type arguments + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4387.tolk b/tolk-tester/tests/invalid-semantics/err-4387.tolk new file mode 100644 index 0000000000..50e02ed7bc --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4387.tolk @@ -0,0 +1,17 @@ +struct Wrapper { + value: T; +} + +type nested = Wrapper>>; + +fun main(d: nested) { + var c1 = d; + + var c2 = Wrapper>>; +} + +/** +@compilation_should_fail +@stderr struct `Wrapper` can not be used as a value +@stderr c2 = Wrapper + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4429.tolk b/tolk-tester/tests/invalid-semantics/err-4429.tolk new file mode 100644 index 0000000000..028ae10788 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4429.tolk @@ -0,0 +1,20 @@ +struct Wrapper { + field: T; +} + +type MyInt = int; + +fun main(w: int | Wrapper | Wrapper) { + w is Wrapper; + w is Wrapper; + w is Wrapper; + w is Wrapper; + + w is Wrapper; +} + +/** +@compilation_should_fail +@stderr can not deduce type arguments for `Wrapper`, provide them manually +@stderr Wrapper + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4493.tolk b/tolk-tester/tests/invalid-semantics/err-4493.tolk new file mode 100644 index 0000000000..8a386f8c36 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4493.tolk @@ -0,0 +1,13 @@ +struct Wrapper { + value: T; +} + +fun my() { + var w: Wrapper = Wrapper { value: 30 }; +} + +/** +@compilation_should_fail +@stderr type `Wrapper` is generic, you should provide type arguments +@stderr var w: Wrapper = + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4512.tolk b/tolk-tester/tests/invalid-semantics/err-4512.tolk new file mode 100644 index 0000000000..77c18cb899 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4512.tolk @@ -0,0 +1,8 @@ +fun f() { + var n: int = 10; +} + +/** +@compilation_should_fail +@stderr type `int` is not generic + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4519.tolk b/tolk-tester/tests/invalid-semantics/err-4519.tolk index b0377b7199..a76d20a967 100644 --- a/tolk-tester/tests/invalid-semantics/err-4519.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4519.tolk @@ -2,11 +2,13 @@ struct value { id: int; } fun main() { var value = 100; - var obj = value { id: 100 }; + var value2 = 200; + var obj1 = value { id: 100 }; // "value" refs to a struct, it's parsed as a type + var obj2 = value2 { id: 200 }; // no struct "value2" } /** @compilation_should_fail -@stderr `value` does not name a struct -@stderr obj = value +@stderr unknown type name `value2` +@stderr obj2 = value2 */ diff --git a/tolk-tester/tests/invalid-semantics/err-4530.tolk b/tolk-tester/tests/invalid-semantics/err-4530.tolk new file mode 100644 index 0000000000..c7adb10d72 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4530.tolk @@ -0,0 +1,17 @@ +struct Wrapper { + field: T; +} + +fun main() { + var w1: int | Wrapper | null | Wrapper | (int, int) = Wrapper { field: 10 }; + var w2: int | Wrapper | null | Wrapper | (int, int) = Wrapper { field: 10 }; + var w3: int | Wrapper | null | Wrapper | (int, int) = Wrapper { field: 10 }; + + var w4: int | Wrapper | null | Wrapper | (int, int) = { field: 20 }; +} + +/** +@compilation_should_fail +@stderr can not detect struct name +@stderr { field: 20 } + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4580.tolk b/tolk-tester/tests/invalid-semantics/err-4580.tolk new file mode 100644 index 0000000000..bd414eeb66 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4580.tolk @@ -0,0 +1,12 @@ +struct Wrapper { + item: T; +} + +fun main() { + var c = Wrapper { }; +} + +/** +@compilation_should_fail +@stderr field `item` missed in initialization of struct `Wrapper` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4628.tolk b/tolk-tester/tests/invalid-semantics/err-4628.tolk new file mode 100644 index 0000000000..3efe9069da --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4628.tolk @@ -0,0 +1,16 @@ +struct Wrapper { + item: T; + value: int; +} + +fun main() { + var c = Wrapper { + value: null + }; +} + +/** +@compilation_should_fail +@stderr field `item` missed in initialization of struct `Wrapper` +@stderr = Wrapper { + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4630.tolk b/tolk-tester/tests/invalid-semantics/err-4630.tolk new file mode 100644 index 0000000000..d404adbde9 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4630.tolk @@ -0,0 +1,13 @@ +struct Ok { result: T; } + +type OkInt = Ok; +type OkSlice = Ok; + +fun f(w: OkInt | int | OkSlice) { + if (w !is Ok) {} +} + +/** +@compilation_should_fail +@stderr can not deduce type arguments for `Ok`, provide them manually + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4711.tolk b/tolk-tester/tests/invalid-semantics/err-4711.tolk new file mode 100644 index 0000000000..01f4884878 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4711.tolk @@ -0,0 +1,22 @@ +struct Pair { + first: T1; + second: T2; +} + +fun main() { + var ok1: Pair = { first: null, second: null }; + var ok2: Pair = Pair { first: null, second: null }; + var ok3 = Pair { first: null, second: null }; + var ok4 = Pair { first: null as int?, second: null as (int, int)? }; + var ok5 = Pair { first: null as int?, second: null as slice? }; // ok since doesn't reach type checking + var ok6 = Pair { first: null, second: null }; + + var err = Pair { first: null, second: null }; +} + +/** +@compilation_should_fail +@stderr can not deduce T1 for generic struct `Pair` +@stderr instantiate it manually with `Pair<...>` +@stderr var err = + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4862.tolk b/tolk-tester/tests/invalid-semantics/err-4862.tolk new file mode 100644 index 0000000000..490c9d2f36 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4862.tolk @@ -0,0 +1,18 @@ +struct Pair { + first: T1; + second: T2; +} + +fun cantProvideDuplicateField() { + var p: Pair = { + first: null, + second: null, + first: 3, + }; +} + +/** +@compilation_should_fail +@stderr duplicate field initialization +@stderr first: 3 + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4880.tolk b/tolk-tester/tests/invalid-semantics/err-4880.tolk index 0bbdeee67d..4564a4a814 100644 --- a/tolk-tester/tests/invalid-semantics/err-4880.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4880.tolk @@ -6,5 +6,5 @@ fun failCantDeduceWithoutArgument() { /** @compilation_should_fail -@stderr too few arguments in call to `f`, expected 2, have 1 +@stderr too few arguments in call to `f`, expected 2, have 1 */ diff --git a/tolk-tester/tests/invalid-semantics/err-4925.tolk b/tolk-tester/tests/invalid-semantics/err-4925.tolk new file mode 100644 index 0000000000..858ff6f779 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4925.tolk @@ -0,0 +1,20 @@ +struct Wrapper { + field: T; +} + +type MyInt = int; +type WrapperAlias = Wrapper; + +fun main(w: int | WrapperAlias | WrapperAlias) { + match (w) { + MyInt => {} + WrapperAlias => {} + WrapperAlias => {} + } +} + +/** +@compilation_should_fail +@stderr can not deduce type arguments for `WrapperAlias`, provide them manually +@stderr WrapperAlias => {} + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3173.tolk b/tolk-tester/tests/invalid-syntax/err-3173.tolk new file mode 100644 index 0000000000..2c3e23b7d8 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3173.tolk @@ -0,0 +1,8 @@ +struct Pair`, got `{` + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3450.tolk b/tolk-tester/tests/invalid-syntax/err-3450.tolk new file mode 100644 index 0000000000..5075284ec2 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3450.tolk @@ -0,0 +1,12 @@ +struct Wrapper { + value: T; +} + +fun main() { + var w3: Wrapper> = {}; +} + +/** +@compilation_should_fail +@stderr expected `>` or `,`, got `=` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6070.tolk b/tolk-tester/tests/invalid-typing/err-6070.tolk new file mode 100644 index 0000000000..e10560210b --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6070.tolk @@ -0,0 +1,26 @@ +struct Wrapper { + value: T; +} + +struct JustInt { + value: int; +} + +fun takeSmth(obj: T) { return obj.value; } +fun takeW(obj: Wrapper) { return obj.value; } + +fun cantPassJustIntAsWrapper() { + var w: Wrapper = { value: 10 }; + var j: JustInt = { value: 10 }; + + takeSmth(w); + takeSmth(j); + takeW(w); + takeW(j); +} + +/** +@compilation_should_fail +@stderr can not pass `JustInt` to `Wrapper` +@stderr takeW(j) + */ diff --git a/tolk-tester/tests/invalid-typing/err-6110.tolk b/tolk-tester/tests/invalid-typing/err-6110.tolk index fbef5abe67..49e9f30c74 100644 --- a/tolk-tester/tests/invalid-typing/err-6110.tolk +++ b/tolk-tester/tests/invalid-typing/err-6110.tolk @@ -10,6 +10,6 @@ fun cantDeduceDefiniteUnion(a: int | slice | builder) { /** @compilation_should_fail -@stderr can not deduce T1 for generic function `makeNullable` +@stderr can not deduce T1 for generic function `makeNullable` @stderr makeNullable(a); */ diff --git a/tolk-tester/tests/invalid-typing/err-6145.tolk b/tolk-tester/tests/invalid-typing/err-6145.tolk new file mode 100644 index 0000000000..54893cb07d --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6145.tolk @@ -0,0 +1,20 @@ +struct Wrapper { + value: T; +} + +fun analyze(w: Wrapper) {} + +type MInt = int; + +fun main() { + var w1: Wrapper = { value: 10 }; + var w2 = Wrapper { value: 10 as int7 }; + + analyze(w1); + analyze(w2); +} + +/** +@compilation_should_fail +@stderr can not pass `Wrapper` to `Wrapper` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6369.tolk b/tolk-tester/tests/invalid-typing/err-6369.tolk index 72b7df0ec6..3d9391b11e 100644 --- a/tolk-tester/tests/invalid-typing/err-6369.tolk +++ b/tolk-tester/tests/invalid-typing/err-6369.tolk @@ -6,6 +6,6 @@ fun failIncompatibleTypesForT() { /** @compilation_should_fail -@stderr T is both int and slice for generic function `f` +@stderr can not pass `slice` to `int` @stderr f(32 */ diff --git a/tolk-tester/tests/invalid-typing/err-6409.tolk b/tolk-tester/tests/invalid-typing/err-6409.tolk new file mode 100644 index 0000000000..93c347edf7 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6409.tolk @@ -0,0 +1,10 @@ +struct Wrapper { value: T } + +fun f(w: Wrapper | Wrapper) { + w = Wrapper { value: beginCell() }; +} + +/** +@compilation_should_fail +@stderr can not assign `Wrapper` to variable of type `Wrapper | Wrapper` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6473.tolk b/tolk-tester/tests/invalid-typing/err-6473.tolk new file mode 100644 index 0000000000..a6859e03b0 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6473.tolk @@ -0,0 +1,18 @@ +struct Wrapper { + value: T; +} + +type IntOrWrapped8 = Wrapper | int; + +fun main() { + var w1: IntOrWrapped8 | Wrapper = Wrapper { value: 10 as int8 }; + var w2: IntOrWrapped8 | Wrapper = Wrapper { value: 10 as int16 }; + + var w3: IntOrWrapped8 | Wrapper = Wrapper { value: 10 as int32 }; +} + +/** +@compilation_should_fail +@stderr can not assign `Wrapper` to variable of type `Wrapper | int | Wrapper` +@stderr var w3 + */ diff --git a/tolk-tester/tests/invalid-typing/err-6480.tolk b/tolk-tester/tests/invalid-typing/err-6480.tolk new file mode 100644 index 0000000000..25a85e0bc5 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6480.tolk @@ -0,0 +1,27 @@ +struct All { + f1: A; + f2: A; + f3: A; +} + +fun testInferGenericOneByOne(x: int?) { + // ok + __expect_type(All { + f1: x, + f2: 5, + f3: null + }, "All"); + // err + All { + f1: x, + f2: null, + f3: beginCell() // type checker error + }; +} + +/** +@compilation_should_fail +@stderr can not assign `builder` to field of type `int?` +@stderr f3: beginCell() + */ + diff --git a/tolk-tester/tests/invalid-typing/err-6618.tolk b/tolk-tester/tests/invalid-typing/err-6618.tolk new file mode 100644 index 0000000000..ab505c1249 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6618.tolk @@ -0,0 +1,17 @@ +struct Ok { result: T } +struct Err { errPayload: T } + +type Response = Ok | Err; + +fun f(w: Response | Response) { + match (w) { + Ok => {} + Err => {} + } +} + +/** +@compilation_should_fail +@stderr `match` does not cover all possible types +@stderr missing types are: `Ok`, `Err` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6620.tolk b/tolk-tester/tests/invalid-typing/err-6620.tolk new file mode 100644 index 0000000000..47e30e99c6 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6620.tolk @@ -0,0 +1,16 @@ +struct Wrapper { + value: T; +} + +fun main() { + var c1: Wrapper< Wrapper<(int,int)> > = {value:{value:(1,2)}}; // ok + var c2: Wrapper< Wrapper<(int?,int?)> > = {value:{value:(null,null)}}; // ok + + var c3: Wrapper< Wrapper<(int,int)> > = {value:{value:0}}; +} + +/** +@compilation_should_fail +@stderr can not assign `int` to field of type `(int, int)` +@stderr {value:0} + */ diff --git a/tolk-tester/tests/invalid-typing/err-6642.tolk b/tolk-tester/tests/invalid-typing/err-6642.tolk new file mode 100644 index 0000000000..82563e3221 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6642.tolk @@ -0,0 +1,21 @@ +struct Wrapper { + value: T; +} + +type MyInt = int; + +fun cantAssignFieldsOfDifferentTypes( + w1: Wrapper, w2: Wrapper, w3: Wrapper, w4: Wrapper<()> +) { + w1.value = w2.value; // ok + w1.value = w3.value; // ok + w3.value = w2.value; // ok + + w1.value = w4.value; +} + +/** +@compilation_should_fail +@stderr can not assign `()` to field of type `int` +@stderr w1.value = w4.value + */ diff --git a/tolk-tester/tests/invalid-typing/err-6919.tolk b/tolk-tester/tests/invalid-typing/err-6919.tolk new file mode 100644 index 0000000000..f7a9885679 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6919.tolk @@ -0,0 +1,13 @@ +struct Gen { + price: coins = (1 as T) + "", +} + +fun main() { + var c: Gen = {}; + return c; +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int8` and `slice` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6948.tolk b/tolk-tester/tests/invalid-typing/err-6948.tolk new file mode 100644 index 0000000000..448d07a469 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6948.tolk @@ -0,0 +1,16 @@ +struct Wrapper { + field: T; +} + +struct AnotherWrapper { + field: T; +} + +fun main() { + var w: Wrapper = AnotherWrapper { field: 10 }; +} + +/** +@compilation_should_fail +@stderr can not assign `AnotherWrapper` to variable of type `Wrapper` + */ diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index aec77391f0..9d9a0a0b87 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -242,6 +242,23 @@ fun testCallableFields() { ); } +struct Wrapper { + value: T; +} + +fun wrap(value: T): Wrapper { + return{value}; +} + +@method_id(116) +fun testCallableGenericField() { + var w1: Wrapper<(int) -> int> = { value: justAdd2 }; + var w2: Wrapper = { value: justAdd2 }; + var w_f = wrapMInt>; + return (w1.value(5), w2.value!(9), wrap(w_f(justAdd2)).value.value(78)); +} + + fun main() {} /** @@ -267,4 +284,5 @@ fun main() {} @testcase | 114 | -1 | 2 3 131 @testcase | 114 | 0 | (null) 12 1 @testcase | 115 | | 7 6 7 +@testcase | 116 | | 7 11 80 */ diff --git a/tolk-tester/tests/warnings-not-errors/warnings-4.tolk b/tolk-tester/tests/warnings-not-errors/warnings-4.tolk new file mode 100644 index 0000000000..bc843ecc7a --- /dev/null +++ b/tolk-tester/tests/warnings-not-errors/warnings-4.tolk @@ -0,0 +1,15 @@ +struct Wrapper { + value: T; +} + +fun main() { + var w = Wrapper { value: 16 as int16 }; + return (w is Wrapper, w is Wrapper, w is Wrapper, w !is Wrapper< Wrapper >); +} + +/** +@testcase | 0 | | -1 -1 0 -1 + +@stderr warning: variable `w` of type `Wrapper` can never be `Wrapper`, this condition is always false +@stderr warning: variable `w` of type `Wrapper` can never be `Wrapper>`, this condition is always true + */ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 36c657c261..2999d1a10e 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -168,14 +168,20 @@ static std::vector parse_nested_type_list(Lexer& lex, TokenType tok_op if (lex.tok() == tok_comma) { lex.next(); } else if (lex.tok() != tok_cl) { + // overcome the `>>` problem, like `Wrapper>`: + // treat token `>>` like two `>`; consume one here doing nothing (break) and leave the second `>` in a lexer + if (tok_cl == tok_gt && lex.tok() == tok_rshift) { + lex.hack_replace_rshift_with_one_triangle(); + break; + } lex.unexpected(s_cl); } } return sub_types; } -static std::vector parse_nested_type_list_in_parenthesis(Lexer& lex) { - return parse_nested_type_list(lex, tok_oppar, "`(`", tok_clpar, "`)` or `,`"); +static std::vector parse_nested_type_list_in_triangles(Lexer& lex) { + return parse_nested_type_list(lex, tok_lt, "`<`", tok_gt, "`>` or `,`"); } static AnyTypeV parse_simple_type(Lexer& lex) { @@ -189,7 +195,7 @@ static AnyTypeV parse_simple_type(Lexer& lex) { return createV(loc, text); } case tok_oppar: { - return createV(loc, parse_nested_type_list_in_parenthesis(lex)); + return createV(loc, parse_nested_type_list(lex, tok_oppar, "`(`", tok_clpar, "`)` or `,`")); } case tok_opbracket: { return createV(loc, parse_nested_type_list(lex, tok_opbracket, "`[`", tok_clbracket, "`]` or `,`")); @@ -202,6 +208,15 @@ static AnyTypeV parse_simple_type(Lexer& lex) { static AnyTypeV parse_type_nullable(Lexer& lex) { AnyTypeV result = parse_simple_type(lex); + if (lex.tok() == tok_lt) { + std::vector args = parse_nested_type_list_in_triangles(lex); + std::vector outer_and_args; + outer_and_args.reserve(1 + args.size()); + outer_and_args.push_back(result); + outer_and_args.insert(outer_and_args.end(), args.begin(), args.end()); + result = createV(result->loc, std::move(outer_and_args)); + } + if (lex.tok() == tok_question) { lex.next(); result = createV(result->loc, result); @@ -258,6 +273,24 @@ static AnyTypeV parse_type_from_tokens(Lexer& lex) { AnyExprV parse_expr(Lexer& lex); AnyV parse_statement(Lexer& lex); +static V parse_genericsT_list(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + std::vector genericsT_items; + lex.expect(tok_lt, "`<`"); + while (true) { + lex.check(tok_identifier, "T"); + std::string_view nameT = lex.cur_str(); + genericsT_items.emplace_back(createV(lex.cur_location(), nameT)); + lex.next(); + if (lex.tok() != tok_comma) { + break; + } + lex.next(); + } + lex.expect(tok_gt, "`>`"); + return createV{loc, std::move(genericsT_items)}; +} + static AnyV parse_parameter(Lexer& lex, bool is_first) { SrcLocation loc = lex.cur_location(); @@ -342,10 +375,16 @@ static AnyV parse_type_alias_declaration(Lexer& lex, const std::vector(lex.cur_location(), lex.cur_str()); lex.next(); + + V genericsT_list = nullptr; + if (lex.tok() == tok_lt) { // 'type Response' + genericsT_list = parse_genericsT_list(lex); + } + lex.expect(tok_assign, "`=`"); AnyTypeV underlying_type = parse_type_from_tokens(lex); lex.expect(tok_semicolon, "`;`"); - return createV(loc, v_ident, underlying_type); + return createV(loc, v_ident, genericsT_list, underlying_type); } static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { @@ -728,13 +767,22 @@ static AnyExprV parse_expr100(Lexer& lex) { v_instantiationTs = parse_maybe_instantiationTs_after_identifier(lex); } if (lex.tok() == tok_opbrace) { - auto explicit_ref = createV(loc, v_ident, v_instantiationTs); - return createV(loc, explicit_ref, parse_object_body(lex)); + AnyTypeV type_node = createV(v_ident->loc, v_ident->name); // `Pair { ... }` + if (v_instantiationTs) { // `Pair { ... }` + std::vector ident_and_args; + ident_and_args.reserve(1 + v_instantiationTs->size()); + ident_and_args.push_back(type_node); + for (int i = 0; i < v_instantiationTs->size(); ++i) { + ident_and_args.push_back(v_instantiationTs->get_item(i)->type_node); + } + type_node = createV(v_ident->loc, std::move(ident_and_args)); + } + return createV(loc, type_node, parse_object_body(lex)); } return createV(loc, v_ident, v_instantiationTs); } case tok_opbrace: - return createV(loc, createV(loc), parse_object_body(lex)); + return createV(loc, nullptr, parse_object_body(lex)); case tok_match: return parse_match_expression(lex); default: @@ -1195,24 +1243,6 @@ static AnyV parse_asm_func_body(Lexer& lex, V param_list) { return createV(loc, std::move(arg_order), std::move(ret_order), std::move(asm_commands)); } -static AnyV parse_genericsT_list(Lexer& lex) { - SrcLocation loc = lex.cur_location(); - std::vector genericsT_items; - lex.expect(tok_lt, "`<`"); - while (true) { - lex.check(tok_identifier, "T"); - std::string_view nameT = lex.cur_str(); - genericsT_items.emplace_back(createV(lex.cur_location(), nameT)); - lex.next(); - if (lex.tok() != tok_comma) { - break; - } - lex.next(); - } - lex.expect(tok_gt, "`>`"); - return createV{loc, std::move(genericsT_items)}; -} - static V parse_annotation(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.check(tok_annotation_at, "`@`"); @@ -1285,7 +1315,7 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector genericsT_list = nullptr; if (lex.tok() == tok_lt) { // 'fun f' - genericsT_list = parse_genericsT_list(lex)->as(); + genericsT_list = parse_genericsT_list(lex); } V v_param_list = parse_parameter_list(lex)->as(); @@ -1428,7 +1458,12 @@ static AnyV parse_struct_declaration(Lexer& lex, const std::vector(lex.cur_location(), lex.cur_str()); lex.next(); - return createV(loc, v_ident, parse_struct_body(lex)); + V genericsT_list = nullptr; + if (lex.tok() == tok_lt) { // 'struct Wrapper' + genericsT_list = parse_genericsT_list(lex); + } + + return createV(loc, v_ident, genericsT_list, parse_struct_body(lex)); } static AnyV parse_tolk_required_version(Lexer& lex) { diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 3b0b458e4b..42e6915252 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -21,21 +21,8 @@ namespace tolk { -class ASTReplicator { -protected: - virtual AnyV clone(AnyV v) = 0; - virtual AnyExprV clone(AnyExprV v) = 0; - virtual AnyTypeV clone(AnyTypeV v) = 0; - -public: - virtual ~ASTReplicator() = default; -}; - -class ASTReplicatorFunction : public ASTReplicator { -protected: - using parent = ASTReplicatorFunction; - - std::vector clone(const std::vector& items) { +class ASTReplicator final { + static std::vector clone(const std::vector& items) { std::vector result; result.reserve(items.size()); for (AnyV item : items) { @@ -44,7 +31,7 @@ class ASTReplicatorFunction : public ASTReplicator { return result; } - std::vector clone(const std::vector& items) { + static std::vector clone(const std::vector& items) { std::vector result; result.reserve(items.size()); for (AnyExprV item : items) { @@ -53,7 +40,7 @@ class ASTReplicatorFunction : public ASTReplicator { return result; } - std::vector clone(const std::vector& items) { + static std::vector clone(const std::vector& items) { std::vector result; result.reserve(items.size()); for (AnyTypeV item : items) { @@ -64,195 +51,221 @@ class ASTReplicatorFunction : public ASTReplicator { // types - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->text); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_inner())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_items())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_items())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_params_and_return())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_variants())); } + static V clone(V v) { + return createV(v->loc, clone(v->get_inner_and_args())); + } // expressions - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_expr())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_block_statement())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_items())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_items())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_identifier()), v->has_instantiationTs() ? clone(v->get_instantiationTs()) : nullptr); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_identifier()), clone(v->type_node), v->is_immutable, v->marked_as_redef); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_expr())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->intval, v->orig_str); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->str_val); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->bool_val); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_expr()), v->passed_as_mutate); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_arguments())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_obj()), clone(v->get_identifier()), v->has_instantiationTs() ? clone(v->get_instantiationTs()) : nullptr); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_callee()), clone(v->get_arg_list())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_lhs()), clone(v->get_rhs())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->operator_name, v->tok, clone(v->get_lhs()), clone(v->get_rhs())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->operator_name, v->tok, clone(v->get_rhs())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->operator_name, v->tok, clone(v->get_lhs()), clone(v->get_rhs())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_cond()), clone(v->get_when_true()), clone(v->get_when_false())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_expr()), clone(v->type_node)); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_expr()), clone(v->type_node), v->is_negated); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_expr())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_all_children())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->pattern_kind, clone(v->pattern_type_node), clone(v->get_pattern_expr()), clone(v->get_body())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_field_identifier()), clone(v->get_init_val())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_all_fields())); } - virtual V clone(V v) { - return createV(v->loc, clone(v->get_maybe_reference()), clone(v->get_body())); + static V clone(V v) { + return createV(v->loc, clone(v->type_node), clone(v->get_body())); } // statements - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->loc_end, clone(v->get_items())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_return_value())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->is_ifnot, clone(v->get_cond()), clone(v->get_if_body()), clone(v->get_else_body())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_cond()), clone(v->get_body())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_cond()), clone(v->get_body())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_body()), clone(v->get_cond())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_thrown_code()), clone(v->get_thrown_arg())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_cond()), clone(v->get_thrown_code())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_try_body()), clone(v->get_catch_expr()), clone(v->get_catch_body())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->arg_order, v->ret_order, clone(v->get_asm_commands())); } - // other + // other (common) - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->name); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->nameT); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_items())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->type_node)); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_items())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->param_name, clone(v->type_node), v->declared_as_mutate); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_params())); } + static V clone(V v) { + return createV(v->loc, clone(v->get_identifier()), clone(v->get_default_value()), clone(v->type_node)); + } + static V clone(V v) { + return createV(v->loc, clone(v->get_all_fields())); + } - AnyTypeV clone(AnyTypeV v) final { - if (v == nullptr) { - return nullptr; - } + + static AnyV clone(AnyV v) { switch (v->kind) { - case ast_type_leaf_text: return clone(v->as()); - case ast_type_question_nullable: return clone(v->as()); - case ast_type_parenthesis_tensor: return clone(v->as()); - case ast_type_bracket_tuple: return clone(v->as()); - case ast_type_arrow_callable: return clone(v->as()); - case ast_type_vertical_bar_union: return clone(v->as()); - default: - throw UnexpectedASTNodeKind(v, "ASTReplicatorFunction::clone"); + case ast_empty_statement: return clone(v->as()); + case ast_block_statement: return clone(v->as()); + case ast_return_statement: return clone(v->as()); + case ast_if_statement: return clone(v->as()); + case ast_repeat_statement: return clone(v->as()); + case ast_while_statement: return clone(v->as()); + case ast_do_while_statement: return clone(v->as()); + case ast_throw_statement: return clone(v->as()); + case ast_assert_statement: return clone(v->as()); + case ast_try_catch_statement: return clone(v->as()); + case ast_asm_body: return clone(v->as()); + // other AST nodes that can be children of ast nodes of function/struct body + case ast_identifier: return clone(v->as()); + case ast_genericsT_item: return clone(v->as()); + case ast_genericsT_list: return clone(v->as()); + case ast_instantiationT_item: return clone(v->as()); + case ast_instantiationT_list: return clone(v->as()); + case ast_parameter: return clone(v->as()); + case ast_parameter_list: return clone(v->as()); + case ast_struct_field: return clone(v->as()); + case ast_struct_body: return clone(v->as()); + + default: { + // be very careful, don't forget to handle all statements/other (not expressions) above! + AnyExprV as_expr = reinterpret_cast(v); + return clone(as_expr); + } } } - AnyExprV clone(AnyExprV v) final { + static AnyExprV clone(AnyExprV v) { switch (v->kind) { case ast_empty_expression: return clone(v->as()); case ast_parenthesized_expression: return clone(v->as()); @@ -285,43 +298,30 @@ class ASTReplicatorFunction : public ASTReplicator { case ast_object_body: return clone(v->as()); case ast_object_literal: return clone(v->as()); default: - throw UnexpectedASTNodeKind(v, "ASTReplicatorFunction::clone"); + throw UnexpectedASTNodeKind(v, "ASTReplicatorFunction::clone(AnyExprV)"); } } - AnyV clone(AnyV v) final { + static AnyTypeV clone(AnyTypeV v) { + if (v == nullptr) { + return nullptr; + } switch (v->kind) { - case ast_empty_statement: return clone(v->as()); - case ast_block_statement: return clone(v->as()); - case ast_return_statement: return clone(v->as()); - case ast_if_statement: return clone(v->as()); - case ast_repeat_statement: return clone(v->as()); - case ast_while_statement: return clone(v->as()); - case ast_do_while_statement: return clone(v->as()); - case ast_throw_statement: return clone(v->as()); - case ast_assert_statement: return clone(v->as()); - case ast_try_catch_statement: return clone(v->as()); - case ast_asm_body: return clone(v->as()); - // other AST nodes that can be children of ast nodes of function body - case ast_identifier: return clone(v->as()); - case ast_genericsT_item: return clone(v->as()); - case ast_genericsT_list: return clone(v->as()); - case ast_instantiationT_item: return clone(v->as()); - case ast_instantiationT_list: return clone(v->as()); - case ast_parameter: return clone(v->as()); - case ast_parameter_list: return clone(v->as()); - - default: { - // be very careful, don't forget to handle all statements/other (not expressions) above! - AnyExprV as_expr = reinterpret_cast(v); - return clone(as_expr); - } + case ast_type_leaf_text: return clone(v->as()); + case ast_type_question_nullable: return clone(v->as()); + case ast_type_parenthesis_tensor: return clone(v->as()); + case ast_type_bracket_tuple: return clone(v->as()); + case ast_type_arrow_callable: return clone(v->as()); + case ast_type_vertical_bar_union: return clone(v->as()); + case ast_type_triangle_args: return clone(v->as()); + default: + throw UnexpectedASTNodeKind(v, "ASTReplicator::clone(AnyTypeV)"); } } public: // the cloned function becomes a deep copy, all AST nodes are copied, no previous pointers left - V clone_function_ast(V v_orig, V new_name_ident) { + static V clone_function_ast(V v_orig, V new_name_ident) { return createV( v_orig->loc, new_name_ident, @@ -333,6 +333,26 @@ class ASTReplicatorFunction : public ASTReplicator { v_orig->flags ); } + + // the cloned struct becomes a deep copy, all AST nodes are copied, no previous pointers left + static V clone_struct_ast(V v_orig, V new_name_ident) { + return createV( + v_orig->loc, + new_name_ident, + clone(v_orig->genericsT_list), + clone(v_orig->get_struct_body()) + ); + } + + // the cloned type alias becomes a deep copy, all AST nodes are copied, no previous pointers left + static V clone_type_alias_ast(V v_orig, V new_name_ident) { + return createV( + v_orig->loc, + new_name_ident, + clone(v_orig->genericsT_list), + clone(v_orig->underlying_type_node) + ); + } }; } // namespace tolk diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index ea991da20f..c4af598d44 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -40,6 +40,7 @@ class ASTStringifier final : public ASTVisitor { {ast_type_bracket_tuple, "ast_type_bracket_tuple"}, {ast_type_arrow_callable, "ast_type_arrow_callable"}, {ast_type_vertical_bar_union, "ast_type_vertical_bar_union"}, + {ast_type_triangle_args, "ast_type_triangle_args"}, // expressions {ast_empty_expression, "ast_empty_expression"}, {ast_parenthesized_expression, "ast_parenthesized_expression"}, @@ -300,6 +301,7 @@ class ASTStringifier final : public ASTVisitor { case ast_type_bracket_tuple: return handle_vertex(v->as()); case ast_type_arrow_callable: return handle_vertex(v->as()); case ast_type_vertical_bar_union: return handle_vertex(v->as()); + case ast_type_triangle_args: return handle_vertex(v->as()); // expressions case ast_empty_expression: return handle_vertex(v->as()); case ast_parenthesized_expression: return handle_vertex(v->as()); diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index 11ecbe80f7..1bb72f44c4 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -212,7 +212,9 @@ const std::vector& get_all_declared_structs(); template void visit_ast_of_all_functions() { BodyVisitorT visitor; - for (FunctionPtr fun_ref : get_all_not_builtin_functions()) { + const std::vector& all = get_all_not_builtin_functions(); + for (size_t i = 0; i < all.size(); ++i) { // NOLINT(*-loop-convert) + FunctionPtr fun_ref = all[i]; // not range-base loop to prevent iterator invalidation (push_back at generics) if (visitor.should_visit_function(fun_ref)) { visitor.start_visiting_function(fun_ref, fun_ref->ast_root->as()); } diff --git a/tolk/ast.h b/tolk/ast.h index dbe1e02b02..95677fcda2 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -70,6 +70,7 @@ enum ASTNodeKind { ast_type_bracket_tuple, ast_type_arrow_callable, ast_type_vertical_bar_union, + ast_type_triangle_args, // expressions ast_empty_expression, ast_parenthesized_expression, @@ -448,6 +449,18 @@ struct Vertex final : ASTTypeVararg { }; +template<> +// ast_type_triangle_args is "T1" +// example: `type OkInt = Ok`, then "Ok" is triangle args, and at resolving, it's instantiated into TypeDataStruct +// example: `type A = Ok`, then "Ok" is triangle args, but at resolving, kept as TypeDataGenericTypeWithTs +struct Vertex final : ASTTypeVararg { + const std::vector& get_inner_and_args() const { return children; } + + Vertex(SrcLocation loc, std::vector&& inner_and_args) + : ASTTypeVararg(ast_type_triangle_args, loc, std::move(inner_and_args)) {} +}; + + // // --------------------------------------------------------- // expressions @@ -915,21 +928,21 @@ struct Vertex final : ASTExprVararg { template<> // ast_object_literal is creating an instance of a struct with initial values of fields, like objects in TypeScript -// example: `Point { ... }` (object creation has explicit ref and body) +// example: `Point { ... }` (object creation has type_node and body) // example: `var v: Point = { ... }` (object creation has only body, struct_ref is determined from the left) -struct Vertex final : ASTExprBinary { +// example: `Wrapper { ... }` (also type_node and body, this type_node is resolved as instantiated generic struct) +struct Vertex final : ASTExprUnary { StructPtr struct_ref = nullptr; // assigned at type inferring + AnyTypeV type_node; // not null for `T { ... }`, nullptr for plain `{ ... }` - bool has_explicit_ref() const { return lhs->kind != ast_empty_expression; } - auto get_explicit_ref() const { return lhs->as(); } - AnyExprV get_maybe_reference() const { return lhs; } // ast_empty_expression or ast_reference - auto get_body() const { return rhs->as(); } + auto get_body() const { return child->as(); } Vertex* mutate() const { return const_cast(this); } void assign_struct_ref(StructPtr struct_ref); - Vertex(SrcLocation loc, AnyExprV maybe_reference, V body) - : ASTExprBinary(ast_object_literal, loc, maybe_reference, body) {} + Vertex(SrcLocation loc, AnyTypeV type_node, V body) + : ASTExprUnary(ast_object_literal, loc, body) + , type_node(type_node) {} }; @@ -1255,6 +1268,7 @@ template<> // see TypeDataAlias in type-system.h struct Vertex final : ASTOtherVararg { AliasDefPtr alias_ref = nullptr; // filled after register + V genericsT_list; // exists for `type Response`; otherwise, nullptr AnyTypeV underlying_type_node; // at the right of `=` auto get_identifier() const { return children.at(0)->as(); } @@ -1262,9 +1276,9 @@ struct Vertex final : ASTOtherVararg { Vertex* mutate() const { return const_cast(this); } void assign_alias_ref(AliasDefPtr alias_ref); - Vertex(SrcLocation loc, V name_identifier, AnyTypeV underlying_type_node) + Vertex(SrcLocation loc, V name_identifier, V genericsT_list, AnyTypeV underlying_type_node) : ASTOtherVararg(ast_type_alias_declaration, loc, {name_identifier}) - , underlying_type_node(underlying_type_node) {} + , genericsT_list(genericsT_list), underlying_type_node(underlying_type_node) {} }; template<> @@ -1288,6 +1302,7 @@ template<> struct Vertex final : ASTOtherVararg { int get_num_fields() const { return size(); } auto get_field(int i) const { return children.at(i)->as(); } + const std::vector& get_all_fields() const { return children; } Vertex(SrcLocation loc, std::vector&& fields) : ASTOtherVararg(ast_struct_body, loc, std::move(fields)) {} @@ -1299,6 +1314,7 @@ template<> // currently, Tolk doesn't have "implements" or whatever, so struct declaration contains only body struct Vertex final : ASTOtherVararg { StructPtr struct_ref = nullptr; // filled after register + V genericsT_list; // exists for `Wrapper`; otherwise, nullptr auto get_identifier() const { return children.at(0)->as(); } auto get_struct_body() const { return children.at(1)->as(); } @@ -1306,8 +1322,9 @@ struct Vertex final : ASTOtherVararg { Vertex* mutate() const { return const_cast(this); } void assign_struct_ref(StructPtr struct_ref); - Vertex(SrcLocation loc, V name_identifier, V struct_body) - : ASTOtherVararg(ast_struct_declaration, loc, {name_identifier, struct_body}) {} + Vertex(SrcLocation loc, V name_identifier, V genericsT_list, V struct_body) + : ASTOtherVararg(ast_struct_declaration, loc, {name_identifier, struct_body}) + , genericsT_list(genericsT_list) {} }; template<> diff --git a/tolk/fwd-declarations.h b/tolk/fwd-declarations.h index cda17c4247..97cba7200b 100644 --- a/tolk/fwd-declarations.h +++ b/tolk/fwd-declarations.h @@ -48,6 +48,8 @@ using StructPtr = const StructData*; class TypeData; using TypePtr = const TypeData*; +struct GenericsSubstitutions; + struct SrcFile; } // namespace tolk diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index e73639b866..9983ca49dd 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -24,49 +24,86 @@ namespace tolk { -// given orig = "(int, T)" and substitutions = [slice], return "(int, slice)" -static TypePtr replace_genericT_with_deduced(TypePtr orig, const GenericsDeclaration* genericTs, const std::vector& substitutionTs) { +// given orig `(int, T)` and substitutions [T=slice], return `(int, slice)` +static TypePtr replace_genericT_with_deduced(TypePtr orig, const GenericsSubstitutions* substitutedTs) { if (!orig || !orig->has_genericT_inside()) { return orig; } - tolk_assert(genericTs->size() == substitutionTs.size()); - return orig->replace_children_custom([genericTs, substitutionTs](TypePtr child) { + return orig->replace_children_custom([substitutedTs](TypePtr child) { if (const TypeDataGenericT* asT = child->try_as()) { - int idx = genericTs->find_nameT(asT->nameT); - if (idx == -1) { - throw Fatal("can not replace generic " + asT->nameT); + TypePtr typeT = substitutedTs->get_substitution_for_nameT(asT->nameT); + if (typeT == nullptr) { // T was not deduced yet, leave T as generic + typeT = child; } - if (substitutionTs[idx] == nullptr) { - throw GenericDeduceError("can not deduce " + asT->nameT); + return typeT; + } + if (const TypeDataGenericTypeWithTs* as_instTs = child->try_as(); as_instTs && !as_instTs->has_genericT_inside()) { + if (StructPtr struct_ref = as_instTs->struct_ref) { + struct_ref = instantiate_generic_struct(struct_ref, GenericsSubstitutions(struct_ref->genericTs, as_instTs->type_arguments)); + return TypeDataStruct::create(struct_ref); + } + if (AliasDefPtr alias_ref = as_instTs->alias_ref) { + alias_ref = instantiate_generic_alias(as_instTs->alias_ref, GenericsSubstitutions(alias_ref->genericTs, as_instTs->type_arguments)); + return TypeDataAlias::create(alias_ref); } - return substitutionTs[idx]; } return child; }); } -GenericSubstitutionsDeduceForCall::GenericSubstitutionsDeduceForCall(FunctionPtr fun_ref) - : fun_ref(fun_ref) { - substitutionTs.resize(fun_ref->genericTs->size()); // filled with nullptr (nothing deduced) +std::string GenericsSubstitutions::as_human_readable() const { + std::string result; + for (int i = 0; i < size(); ++i) { + if (!result.empty()) { + result += ", "; + } + result += genericTs->get_nameT(i); + if (valuesTs[i] == nullptr) { + result += "=nullptr"; + } else { + result += "=`"; + result += valuesTs[i]->as_human_readable(); + result += "`"; + } + } + return result; } -void GenericSubstitutionsDeduceForCall::provide_deducedT(const std::string& nameT, TypePtr deduced) { - if (deduced == TypeDataNullLiteral::create() || deduced->has_unknown_inside()) { - return; // just 'null' doesn't give sensible info +void GenericsSubstitutions::set_typeT(std::string_view nameT, TypePtr typeT) { + for (int i = 0; i < size(); ++i) { + if (genericTs->get_nameT(i) == nameT) { + if (valuesTs[i] == nullptr) { + if (typeT == TypeDataNullLiteral::create() || typeT->has_unknown_inside()) { + throw GenericDeduceError(nameT); + } + tolk_assert(!typeT->has_genericT_inside()); + valuesTs[i] = typeT; + } + break; + } } +} - int idx = fun_ref->genericTs->find_nameT(nameT); - if (substitutionTs[idx] == nullptr) { - substitutionTs[idx] = deduced; - } else if (substitutionTs[idx] != deduced) { - throw GenericDeduceError(nameT + " is both " + substitutionTs[idx]->as_human_readable() + " and " + deduced->as_human_readable()); +GenericsSubstitutions::GenericsSubstitutions(const GenericsDeclaration* genericTs, const std::vector& type_arguments) + : genericTs(genericTs) { + assert(genericTs != nullptr && genericTs->size() == static_cast(type_arguments.size())); + valuesTs.resize(genericTs->size()); + for (int i = 0; i < genericTs->size(); ++i) { + valuesTs[i] = type_arguments[i]; } } -void GenericSubstitutionsDeduceForCall::provide_manually_specified(std::vector&& substitutionTs) { - this->substitutionTs = std::move(substitutionTs); - this->manually_specified = true; +GenericSubstitutionsDeducing::GenericSubstitutionsDeducing(FunctionPtr fun_ref) + : fun_ref(fun_ref) + , struct_ref(nullptr) + , deducedTs(fun_ref->genericTs) { +} + +GenericSubstitutionsDeducing::GenericSubstitutionsDeducing(StructPtr struct_ref) + : fun_ref(nullptr) + , struct_ref(struct_ref) + , deducedTs(struct_ref->genericTs) { } // purpose: having `f(value: T)` and call `f(5)`, deduce T = int @@ -75,12 +112,18 @@ void GenericSubstitutionsDeduceForCall::provide_manually_specified(std::vector(a: int, b: T1, c: (T1, T2))` and call `f(6, 7, (8, cs))` // - `a` does not affect, it doesn't depend on generic Ts // - next condition: param_type = `T1`, arg_type = `int`, deduce T1 = int -// - next condition: param_type = `(T1, T2)`, arg_type = `(int, slice)`, deduce T1 = int, T2 = slice -// for call `f(6, cs, (8, cs))` T1 will be both `slice` and `int`, fired an error -void GenericSubstitutionsDeduceForCall::consider_next_condition(TypePtr param_type, TypePtr arg_type) { +// - next condition: param_type = `(T1, T2)` = `(int, T2)`, arg_type = `(int, slice)`, deduce T2 = slice +// for call `f(6, cs, (8, cs))` both T1 and T2 will become `slice`, firing a type mismatch error later +void GenericSubstitutionsDeducing::consider_next_condition(TypePtr param_type, TypePtr arg_type) { + // all Ts deduced up to this point are apriori + param_type = replace_genericT_with_deduced(param_type, &deducedTs); + if (!param_type->has_genericT_inside()) { + return; + } + if (const auto* asT = param_type->try_as()) { // `(arg: T)` called as `f([1, 2])` => T is [int, int] - provide_deducedT(asT->nameT, arg_type); + deducedTs.set_typeT(asT->nameT, arg_type); } else if (const auto* p_nullable = param_type->try_as(); p_nullable && p_nullable->or_null) { // `arg: T?` called as `f(nullableInt)` => T is int if (const auto* a_nullable = arg_type->unwrap_alias()->try_as(); a_nullable && a_nullable->or_null) { @@ -114,43 +157,90 @@ void GenericSubstitutionsDeduceForCall::consider_next_condition(TypePtr param_ty } } else if (const auto* p_union = param_type->try_as()) { // `arg: T1 | T2` called as `f(intOrBuilder)` => T1 is int, T2 is builder - if (const auto* a_union = arg_type->unwrap_alias()->try_as(); a_union && a_union->variants.size() == p_union->variants.size()) { - for (int i = 0; i < static_cast(p_union->variants.size()); ++i) { - consider_next_condition(p_union->variants[i], a_union->variants[i]); + // `arg: int | T1` called as `f(builderOrIntOrSlice)` => T1 is builder|slice + if (const auto* a_union = arg_type->unwrap_alias()->try_as()) { + std::vector p_generic; + std::vector a_sub_p = a_union->variants; + bool is_sub_correct = true; + for (TypePtr p_variant : p_union->variants) { + if (!p_variant->has_genericT_inside()) { + auto it = std::find(a_sub_p.begin(), a_sub_p.end(), p_variant); + if (it != a_sub_p.end()) { + a_sub_p.erase(it); + } else { + is_sub_correct = false; + } + } else { + p_generic.push_back(p_variant); + } + } + if (is_sub_correct && p_generic.size() == 1 && a_sub_p.size() > 1) { + consider_next_condition(p_generic[0], TypeDataUnion::create(std::move(a_sub_p))); + } else if (is_sub_correct && p_generic.size() == a_sub_p.size()) { + for (int i = 0; i < static_cast(p_generic.size()); ++i) { + consider_next_condition(p_generic[i], a_sub_p[i]); + } + } + } + } else if (const auto* p_instSt = param_type->try_as(); p_instSt && p_instSt->struct_ref) { + // `arg: Wrapper` called as `f(wrappedInt)` => T is int + if (const auto* a_struct = arg_type->try_as(); a_struct && a_struct->struct_ref->is_instantiation_of_generic_struct() && a_struct->struct_ref->base_struct_ref == p_instSt->struct_ref) { + tolk_assert(p_instSt->size() == a_struct->struct_ref->substitutedTs->size()); + for (int i = 0; i < p_instSt->size(); ++i) { + consider_next_condition(p_instSt->type_arguments[i], a_struct->struct_ref->substitutedTs->typeT_at(i)); + } + } + } else if (const auto* p_instAl = param_type->try_as(); p_instAl && p_instAl->alias_ref) { + // `arg: WrapperAlias` called as `f(wrappedInt)` => T is int + if (const auto* a_alias = arg_type->try_as(); a_alias && a_alias->alias_ref->is_instantiation_of_generic_alias() && a_alias->alias_ref->base_alias_ref == p_instAl->alias_ref) { + tolk_assert(p_instAl->size() == a_alias->alias_ref->substitutedTs->size()); + for (int i = 0; i < p_instAl->size(); ++i) { + consider_next_condition(p_instAl->type_arguments[i], a_alias->alias_ref->substitutedTs->typeT_at(i)); } } } } -TypePtr GenericSubstitutionsDeduceForCall::replace_by_manually_specified(TypePtr param_type) const { - return replace_genericT_with_deduced(param_type, fun_ref->genericTs, substitutionTs); +TypePtr GenericSubstitutionsDeducing::replace_Ts_with_currently_deduced(TypePtr orig) const { + return replace_genericT_with_deduced(orig, &deducedTs); } -TypePtr GenericSubstitutionsDeduceForCall::auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type) { +TypePtr GenericSubstitutionsDeducing::auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type) { try { - if (!manually_specified) { - consider_next_condition(param_type, arg_type); + consider_next_condition(param_type, arg_type); + param_type = replace_genericT_with_deduced(param_type, &deducedTs); + if (param_type->has_genericT_inside()) { + fire_error_can_not_deduce(cur_f, loc, get_first_not_deduced_nameT()); } - return replace_genericT_with_deduced(param_type, fun_ref->genericTs, substitutionTs); + return param_type; } catch (const GenericDeduceError& ex) { - throw ParseError(cur_f, loc, ex.message + " for generic function `" + fun_ref->as_human_readable() + "`; instantiate it manually with `" + fun_ref->name + "<...>()`"); + fire_error_can_not_deduce(cur_f, loc, ex.nameT); + return nullptr; } } -int GenericSubstitutionsDeduceForCall::get_first_not_deduced_idx() const { - for (int i = 0; i < static_cast(substitutionTs.size()); ++i) { - if (substitutionTs[i] == nullptr) { - return i; +std::string_view GenericSubstitutionsDeducing::get_first_not_deduced_nameT() const { + for (int i = 0; i < deducedTs.size(); ++i) { + if (deducedTs.typeT_at(i) == nullptr) { + return deducedTs.nameT_at(i); } } - return -1; + return ""; +} + +void GenericSubstitutionsDeducing::fire_error_can_not_deduce(FunctionPtr cur_f, SrcLocation loc, std::string_view nameT) const { + if (fun_ref) { + fire(cur_f, loc, "can not deduce " + static_cast(nameT) + " for generic function `" + fun_ref->as_human_readable() + "`; instantiate it manually with `" + fun_ref->name + "<...>()`"); + } else { + fire(cur_f, loc, "can not deduce " + static_cast(nameT) + " for generic struct `" + struct_ref->as_human_readable() + "`; instantiate it manually with `" + struct_ref->name + "<...>`"); + } } std::string GenericsDeclaration::as_human_readable() const { std::string result = "<"; for (const GenericsItem& item : itemsT) { if (result.size() > 1) { - result += ","; + result += ", "; } result += item.nameT; } @@ -167,36 +257,61 @@ int GenericsDeclaration::find_nameT(std::string_view nameT) const { return -1; } +bool GenericsSubstitutions::has_nameT(std::string_view nameT) const { + return genericTs->find_nameT(nameT) != -1; +} + +TypePtr GenericsSubstitutions::get_substitution_for_nameT(std::string_view nameT) const { + int idx = genericTs->find_nameT(nameT); + return idx == -1 ? nullptr : valuesTs[idx]; +} + +// given this= and rhs=, check that T1 is equal to T2 in terms of "equal_to" of TypePtr +// for example, `Wrapper>` / `Wrapper>` / `Wrapper` are equal +bool GenericsSubstitutions::equal_to(const GenericsSubstitutions* rhs) const { + if (size() != rhs->size()) { + return false; + } + for (int i = 0; i < size(); ++i) { + if (!valuesTs[i]->equal_to(rhs->valuesTs[i])) { + return false; + } + } + return true; +} + // when cloning `f`, original name is "f", we need a new name for symtable and output // name of an instantiated function will be "f" and similar (yes, with "<" symbol, it's okay to Fift) -static std::string generate_instantiated_name(const std::string& orig_name, const std::vector& substitutions) { +static std::string generate_instantiated_name(const std::string& orig_name, const GenericsSubstitutions& substitutedTs, bool allow_spaces) { // an instantiated function name will be "{orig_name}<{T1,T2,...}>" std::string name = orig_name; name += "<"; - for (TypePtr subs : substitutions) { + for (int i = 0; i < substitutedTs.size(); ++i) { if (name.size() > orig_name.size() + 1) { - name += ","; + name += ", "; } - name += subs->as_human_readable(); + name += substitutedTs.typeT_at(i)->as_human_readable(); + } + if (!allow_spaces) { + name.erase(std::remove(name.begin(), name.end(), ' '), name.end()); } - name.erase(std::remove(name.begin(), name.end(), ' '), name.end()); name += ">"; return name; } -FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, std::vector&& substitutionTs) { +FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs) { tolk_assert(fun_ref->is_generic_function() && !fun_ref->is_method_id_not_empty()); // fun_ref->name = "f", inst_name will be "f" and similar - std::string new_name = generate_instantiated_name(fun_ref->name, substitutionTs); + std::string new_name = generate_instantiated_name(fun_ref->name, substitutedTs, false); if (const Symbol* existing_sym = lookup_global_symbol(new_name)) { FunctionPtr existing_ref = existing_sym->try_as(); tolk_assert(existing_ref); return existing_ref; } - const GenericsInstantiation* instantiationTs = new GenericsInstantiation(loc, std::move(substitutionTs)); - ASTReplicatorFunction replicator; + // to store permanently, allocate an object in heap + const GenericsSubstitutions* allocatedTs = new GenericsSubstitutions(std::move(substitutedTs)); // built-in functions don't have AST to clone, types of parameters don't exist in AST, etc. // nevertheless, for outer code to follow a single algorithm, @@ -205,13 +320,14 @@ FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, s std::vector new_parameters; new_parameters.reserve(fun_ref->get_num_params()); for (const LocalVarData& orig_p : fun_ref->parameters) { - TypePtr new_param_type = replace_genericT_with_deduced(orig_p.declared_type, fun_ref->genericTs, instantiationTs->substitutions); + TypePtr new_param_type = replace_genericT_with_deduced(orig_p.declared_type, allocatedTs); new_parameters.emplace_back(orig_p.name, orig_p.loc, new_param_type, orig_p.flags, orig_p.param_idx); } - TypePtr new_return_type = replace_genericT_with_deduced(fun_ref->declared_return_type, fun_ref->genericTs, instantiationTs->substitutions); - FunctionData* new_fun_ref = new FunctionData(new_name, fun_ref->loc, new_return_type, std::move(new_parameters), fun_ref->flags, nullptr, instantiationTs, fun_ref->body, fun_ref->ast_root); + TypePtr new_return_type = replace_genericT_with_deduced(fun_ref->declared_return_type, allocatedTs); + FunctionData* new_fun_ref = new FunctionData(new_name, fun_ref->loc, new_return_type, std::move(new_parameters), fun_ref->flags, nullptr, allocatedTs, fun_ref->body, fun_ref->ast_root); new_fun_ref->arg_order = fun_ref->arg_order; new_fun_ref->ret_order = fun_ref->ret_order; + new_fun_ref->base_fun_ref = fun_ref; G.symtable.add_function(new_fun_ref); return new_fun_ref; } @@ -221,9 +337,9 @@ FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, s // it's instantiation, when resolving types, it substitutes T=int V orig_root = fun_ref->ast_root->as(); V new_name_ident = createV(orig_root->get_identifier()->loc, new_name); - V new_root = replicator.clone_function_ast(orig_root, new_name_ident); + V new_root = ASTReplicator::clone_function_ast(orig_root, new_name_ident); - FunctionPtr new_fun_ref = pipeline_register_instantiated_generic_function(new_root, instantiationTs); + FunctionPtr new_fun_ref = pipeline_register_instantiated_generic_function(fun_ref, new_root, allocatedTs); tolk_assert(new_fun_ref); // body of a cloned function (it's cloned at type inferring step) needs the previous pipeline to run // for example, all local vars need to be registered as symbols, etc. @@ -236,4 +352,49 @@ FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, s return new_fun_ref; } +StructPtr instantiate_generic_struct(StructPtr struct_ref, GenericsSubstitutions&& substitutedTs) { + tolk_assert(struct_ref->is_generic_struct()); + + // if `Wrapper` was earlier instantiated, return it + std::string new_name = generate_instantiated_name(struct_ref->name, substitutedTs, true); + if (const Symbol* existing_sym = lookup_global_symbol(new_name)) { + StructPtr existing_ref = existing_sym->try_as(); + tolk_assert(existing_ref); + return existing_ref; + } + + const GenericsSubstitutions* allocatedTs = new GenericsSubstitutions(std::move(substitutedTs)); + V orig_root = struct_ref->ast_root->as(); + V new_name_ident = createV(orig_root->get_identifier()->loc, new_name); + V new_root = ASTReplicator::clone_struct_ast(orig_root, new_name_ident); + + StructPtr new_struct_ref = pipeline_register_instantiated_generic_struct(struct_ref, new_root, allocatedTs); + tolk_assert(new_struct_ref); + pipeline_resolve_identifiers_and_assign_symbols(new_struct_ref); + pipeline_resolve_types_and_aliases(new_struct_ref); + return new_struct_ref; +} + +AliasDefPtr instantiate_generic_alias(AliasDefPtr alias_ref, GenericsSubstitutions&& substitutedTs) { + tolk_assert(alias_ref->is_generic_alias()); + + // if `Response` was earlier instantiated, return it + std::string new_name = generate_instantiated_name(alias_ref->name, substitutedTs, true); + if (const Symbol* existing_sym = lookup_global_symbol(new_name)) { + AliasDefPtr existing_ref = existing_sym->try_as(); + tolk_assert(existing_ref); + return existing_ref; + } + + const GenericsSubstitutions* allocatedTs = new GenericsSubstitutions(std::move(substitutedTs)); + V orig_root = alias_ref->ast_root->as(); + V new_name_ident = createV(orig_root->get_identifier()->loc, new_name); + V new_root = ASTReplicator::clone_type_alias_ast(orig_root, new_name_ident); + + AliasDefPtr new_alias_ref = pipeline_register_instantiated_generic_alias(alias_ref, new_root, allocatedTs); + tolk_assert(new_alias_ref); + pipeline_resolve_types_and_aliases(new_alias_ref); + return new_alias_ref; +} + } // namespace tolk diff --git a/tolk/generics-helpers.h b/tolk/generics-helpers.h index 140d278159..7d324e625e 100644 --- a/tolk/generics-helpers.h +++ b/tolk/generics-helpers.h @@ -40,61 +40,75 @@ struct GenericsDeclaration { std::string as_human_readable() const; - size_t size() const { return itemsT.size(); } + int size() const { return static_cast(itemsT.size()); } int find_nameT(std::string_view nameT) const; - std::string get_nameT(int idx) const { return static_cast(itemsT[idx].nameT); } + std::string_view get_nameT(int idx) const { return itemsT[idx].nameT; } }; // when a function call is `f()`, this "" is represented as this class -struct GenericsInstantiation { - const std::vector substitutions; // for genericTs - const SrcLocation loc; // first instantiation location +// same for `Wrapper`, "" is substitution +struct GenericsSubstitutions { +private: + const GenericsDeclaration* genericTs; // [T1, T2] + std::vector valuesTs; // [SomeStruct, int] - explicit GenericsInstantiation(SrcLocation loc, std::vector&& substitutions) - : substitutions(std::move(substitutions)) - , loc(loc) { +public: + explicit GenericsSubstitutions(const GenericsDeclaration* genericTs) + : genericTs(genericTs) + , valuesTs(genericTs == nullptr ? 0 : genericTs->size()) { } + explicit GenericsSubstitutions(const GenericsDeclaration* genericTs, const std::vector& type_arguments); + + std::string as_human_readable() const; + + int size() const { return static_cast(valuesTs.size()); } + bool has_nameT(std::string_view nameT) const; + TypePtr get_substitution_for_nameT(std::string_view nameT) const; + std::string_view nameT_at(int idx) const { return genericTs->get_nameT(idx); } + TypePtr typeT_at(int idx) const { return valuesTs.at(idx); } + bool equal_to(const GenericsSubstitutions* rhs) const; + + void set_typeT(std::string_view nameT, TypePtr typeT); }; // this class helps to deduce Ts on the fly // purpose: having `f(value: T)` and call `f(5)`, deduce T = int // while analyzing a call, arguments are handled one by one, by `auto_deduce_from_argument()` -// this class also handles manually specified substitutions like `f(5)` -class GenericSubstitutionsDeduceForCall { +// note, that manually specified substitutions like `f(5)` are NOT handled by this class, it's not deducing +class GenericSubstitutionsDeducing { FunctionPtr fun_ref; - std::vector substitutionTs; - bool manually_specified = false; + StructPtr struct_ref; + GenericsSubstitutions deducedTs; - void provide_deducedT(const std::string& nameT, TypePtr deduced); void consider_next_condition(TypePtr param_type, TypePtr arg_type); public: - explicit GenericSubstitutionsDeduceForCall(FunctionPtr fun_ref); - - bool is_manually_specified() const { - return manually_specified; - } + explicit GenericSubstitutionsDeducing(FunctionPtr fun_ref); + explicit GenericSubstitutionsDeducing(StructPtr struct_ref); - void provide_manually_specified(std::vector&& substitutionTs); - TypePtr replace_by_manually_specified(TypePtr param_type) const; + TypePtr replace_Ts_with_currently_deduced(TypePtr orig) const; TypePtr auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type); - int get_first_not_deduced_idx() const; + std::string_view get_first_not_deduced_nameT() const; + void fire_error_can_not_deduce(FunctionPtr cur_f, SrcLocation loc, std::string_view nameT) const; - std::vector&& flush() { - return std::move(substitutionTs); + GenericsSubstitutions&& flush() { + return std::move(deducedTs); } }; struct GenericDeduceError final : std::exception { - std::string message; - explicit GenericDeduceError(std::string message) - : message(std::move(message)) { } + std::string nameT; + + explicit GenericDeduceError(std::string_view nameT) + : nameT(nameT) { } const char* what() const noexcept override { - return message.c_str(); + return nameT.c_str(); } }; -FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, std::vector&& substitutionTs); +FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs); +StructPtr instantiate_generic_struct(StructPtr struct_ref, GenericsSubstitutions&& substitutedTs); +AliasDefPtr instantiate_generic_alias(AliasDefPtr alias_ref, GenericsSubstitutions&& substitutedTs); } // namespace tolk diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index 32989b3fa2..d34a863995 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -591,6 +591,12 @@ void Lexer::restore_position(SavedPositionForLookahead saved) { location = saved.loc; } +void Lexer::hack_replace_rshift_with_one_triangle() { + // overcome the `>>` problem when parsing generics, leave only `>` here, see comments at usage + assert(cur_token.type == tok_rshift); + cur_token = Token(tok_gt, ">"); +} + void Lexer::error(const std::string& err_msg) const { throw ParseError(cur_location(), err_msg); } diff --git a/tolk/lexer.h b/tolk/lexer.h index 9c8545bf39..55bdf4edbb 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -212,6 +212,7 @@ class Lexer { SavedPositionForLookahead save_parsing_position() const; void restore_position(SavedPositionForLookahead saved); + void hack_replace_rshift_with_one_triangle(); void check(TokenType next_tok, const char* str_expected) const { if (cur_token.type != next_tok) { diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 2139ae714d..cefdb5b8d6 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -681,7 +681,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectorvariants.size() >= o_union->variants.size()) { + if (t_union && o_union && t_union->size() >= o_union->size()) { tolk_assert(target_w >= orig_w && t_union->has_all_variants_of(o_union)); std::vector prepend_nulls; prepend_nulls.reserve(target_w - orig_w); @@ -830,6 +830,13 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector> is ok to Wrapper> + if (original_type->try_as() && target_type->try_as()) { + tolk_assert(target_type->can_rhs_be_assigned(original_type) && orig_w == target_w); + return rvect; + } + throw Fatal("unhandled transition_expr_to_runtime_type_impl() combination"); } diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 8c42b3f54b..1df6821404 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -750,7 +750,7 @@ void pipeline_check_inferred_types() { } for (StructPtr struct_ref : get_all_declared_structs()) { for (StructFieldPtr field_ref : struct_ref->fields) { - if (field_ref->has_default_value()) { + if (field_ref->has_default_value() && !struct_ref->is_generic_struct()) { visitor.start_visiting_field_default(field_ref); } } diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index 30d0ff562c..32b7c28c2a 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -170,7 +170,7 @@ void pipeline_constant_folding() { // do the same for default values of struct fields, they must be constant expressions for (StructPtr struct_ref : get_all_declared_structs()) { for (StructFieldPtr field_ref : struct_ref->fields) { - if (field_ref->has_default_value()) { + if (field_ref->has_default_value() && !struct_ref->is_generic_struct()) { check_expression_is_constant(field_ref->default_value); AnyExprV replaced = replacer.replace_in_expression(field_ref->default_value); field_ref->mutate()->assign_default_value(replaced); diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index f1e5b4c25c..06e658ac91 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -120,6 +120,16 @@ static std::string to_string(FunctionPtr fun_ref) { return "`" + fun_ref->as_human_readable() + "`"; } +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(StructPtr struct_ref) { + return "`" + struct_ref->as_human_readable() + "`"; +} + +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(AliasDefPtr alias_ref) { + return "`" + alias_ref->as_human_readable() + "`"; +} + GNU_ATTRIBUTE_NOINLINE static std::string to_string(std::string_view string_view) { return static_cast(string_view); @@ -129,8 +139,8 @@ static std::string to_string(std::string_view string_view) { // asm functions generally can't handle it, they expect T to be a TVM primitive // (in FunC, `forall` type just couldn't be unified with non-primitives; in Tolk, generic T is expectedly inferred) GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_calling_asm_function_with_non1_stack_width_arg(FunctionPtr cur_f, SrcLocation loc, FunctionPtr fun_ref, const std::vector& substitutions, int arg_idx) { - fire(cur_f, loc, "can not call `" + fun_ref->as_human_readable() + "` with " + fun_ref->genericTs->get_nameT(arg_idx) + "=" + substitutions[arg_idx]->as_human_readable() + ", because it occupies " + std::to_string(substitutions[arg_idx]->get_width_on_stack()) + " stack slots in TVM, not 1"); +static void fire_error_calling_asm_function_with_non1_stack_width_arg(FunctionPtr cur_f, SrcLocation loc, FunctionPtr fun_ref, const GenericsSubstitutions& substitutions, int arg_idx) { + fire(cur_f, loc, "can not call `" + fun_ref->as_human_readable() + "` with " + to_string(substitutions.nameT_at(arg_idx)) + "=" + substitutions.typeT_at(arg_idx)->as_human_readable() + ", because it occupies " + std::to_string(substitutions.typeT_at(arg_idx)->get_width_on_stack()) + " stack slots in TVM, not 1"); } // fire an error on `untypedTupleVar.0` when used without a hint @@ -140,6 +150,51 @@ static void fire_error_cannot_deduce_untyped_tuple_access(FunctionPtr cur_f, Src fire(cur_f, loc, "can not deduce type of `" + idx_access + "`; either assign it to variable like `var c: int = " + idx_access + "` or cast the result like `" + idx_access + " as int`"); } +// helper function: given hint = `Ok | Err` and struct `Ok`, return `Ok` +// example: `match (...) { Ok => ... }` we need to deduce `Ok` based on subject +static TypePtr try_pick_instantiated_generic_from_hint(TypePtr hint, StructPtr lookup_ref) { + // example: `var w: Ok = Ok { ... }`, hint is `Ok`, lookup is `Ok` + if (const TypeDataStruct* h_struct = hint->unwrap_alias()->try_as()) { + if (lookup_ref == h_struct->struct_ref->base_struct_ref) { + return h_struct; + } + } + // example: `fun f(): Response { return Err { ... } }`, hint is `Ok | Err`, lookup is `Err` + if (const TypeDataUnion* h_union = hint->unwrap_alias()->try_as()) { + TypePtr only_variant = nullptr; // hint `Ok | Ok` is ambiguous + for (TypePtr h_variant : h_union->variants) { + if (const TypeDataStruct* variant_struct = h_variant->unwrap_alias()->try_as()) { + if (lookup_ref == variant_struct->struct_ref->base_struct_ref) { + if (only_variant) { + return nullptr; + } + only_variant = variant_struct; + } + } + } + return only_variant; + } + return nullptr; +} + +// helper function, similar to the above, but for generic type aliases +// example: `v is OkAlias`, need to deduce `OkAlias` based on type of v +static TypePtr try_pick_instantiated_generic_from_hint(TypePtr hint, AliasDefPtr lookup_ref) { + // when a generic type alias points to a generic struct actually: `type WrapperAlias = Wrapper` + if (const TypeDataGenericTypeWithTs* as_instT = lookup_ref->underlying_type->try_as()) { + return as_instT->struct_ref + ? try_pick_instantiated_generic_from_hint(hint, as_instT->struct_ref) + : try_pick_instantiated_generic_from_hint(hint, as_instT->alias_ref); + } + // it's something weird, when a generic alias refs non-generic type + // example: `type StrangeInt = int`, hint is `StrangeInt`, lookup `StrangeInt` + if (const TypeDataAlias* h_alias = hint->try_as()) { + if (lookup_ref == h_alias->alias_ref->base_alias_ref) { + return h_alias; + } + } + return nullptr; +} /* * This class handles all types of AST vertices and traverses them, filling all AnyExprV::inferred_type. @@ -626,7 +681,28 @@ class InferTypesAndCallsAndFieldsVisitor final { ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); assign_inferred_type(v, TypeDataBool::create()); - TypePtr rhs_type = v->type_node->resolved_type->unwrap_alias(); + TypePtr rhs_type = v->type_node->resolved_type; + + if (const auto* t_struct = rhs_type->try_as(); t_struct && t_struct->struct_ref->is_generic_struct()) { + // `v is Wrapper`, detect T based on type of v (`Wrapper | int` => `Wrapper`) + if (TypePtr inst_rhs_type = try_pick_instantiated_generic_from_hint(v->get_expr()->inferred_type, t_struct->struct_ref)) { + rhs_type = inst_rhs_type; + v->type_node->mutate()->assign_resolved_type(rhs_type); + } else { + fire(cur_f, v->type_node->loc, "can not deduce type arguments for " + to_string(t_struct->struct_ref) + ", provide them manually"); + } + } + if (const auto* t_alias = rhs_type->try_as(); t_alias && t_alias->alias_ref->is_generic_alias()) { + // `v is WrapperAlias`, detect T similar to structures + if (TypePtr inst_rhs_type = try_pick_instantiated_generic_from_hint(v->get_expr()->inferred_type, t_alias->alias_ref)) { + rhs_type = inst_rhs_type; + v->type_node->mutate()->assign_resolved_type(rhs_type); + } else { + fire(cur_f, v->type_node->loc, "can not deduce type arguments for " + to_string(t_alias->alias_ref) + ", provide them manually"); + } + } + + rhs_type = rhs_type->unwrap_alias(); TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); TypePtr non_rhs_type = calculate_type_subtract_rhs_type(expr_type, rhs_type); if (expr_type->equal_to(rhs_type)) { // `expr is ` is always true @@ -712,8 +788,8 @@ class InferTypesAndCallsAndFieldsVisitor final { } else if (fun_ref->is_generic_function()) { // `genericFn` is valid, it's a reference to instantiation - std::vector substitutions = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref, v_instantiationTs); - fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutions)); + GenericsSubstitutions substitutedTs = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref->genericTs, v_instantiationTs); + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); v->mutate()->assign_sym(fun_ref); } else if (v_instantiationTs != nullptr && !fun_ref->is_instantiation_of_generic_function()) { @@ -727,10 +803,10 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } else if (AliasDefPtr alias_ref = v->sym->try_as()) { - fire(cur_f, v->loc, "type `" + alias_ref->name + "` can not be used as a value"); + fire(cur_f, v->loc, "type " + to_string(alias_ref) + " can not be used as a value"); } else if (StructPtr struct_ref = v->sym->try_as()) { - fire(cur_f, v->loc, "struct `" + struct_ref->name + "` can not be used as a value"); + fire(cur_f, v->loc, "struct " + to_string(struct_ref) + " can not be used as a value"); } else { tolk_assert(false); @@ -745,19 +821,19 @@ class InferTypesAndCallsAndFieldsVisitor final { // given `genericF` / `t.tupleFirst` (the user manually specified instantiation Ts), // validate and collect them - // returns: [int, slice] / [cell] - std::vector collect_fun_generic_substitutions_from_manually_specified(SrcLocation loc, FunctionPtr fun_ref, V instantiationT_list) const { - if (fun_ref->genericTs->size() != instantiationT_list->get_items().size()) { - fire(cur_f, loc, "wrong count of generic T: expected " + std::to_string(fun_ref->genericTs->size()) + ", got " + std::to_string(instantiationT_list->size())); + // returns: [T1=int, T2=slice] / [T=cell] (with flag auto_deduced = false, meaning they are manually specified) + GenericsSubstitutions collect_fun_generic_substitutions_from_manually_specified(SrcLocation loc, const GenericsDeclaration* genericTs, V instantiationT_list) const { + if (instantiationT_list->size() != genericTs->size()) { + fire(cur_f, loc, "wrong count of generic T: expected " + std::to_string(genericTs->size()) + ", got " + std::to_string(instantiationT_list->size())); } - std::vector substitutions; - substitutions.reserve(instantiationT_list->size()); + std::vector type_arguments; + type_arguments.reserve(instantiationT_list->size()); for (int i = 0; i < instantiationT_list->size(); ++i) { - substitutions.push_back(instantiationT_list->get_item(i)->type_node->resolved_type); + type_arguments.push_back(instantiationT_list->get_item(i)->type_node->resolved_type); } - return substitutions; + return GenericsSubstitutions(genericTs, type_arguments); } // when generic Ts have been collected from user-specified or deduced from arguments, @@ -766,19 +842,19 @@ class InferTypesAndCallsAndFieldsVisitor final { // example: was `t.tuplePush(2)`, read , instantiate `tuplePush` (will later fail type check) // example: was `var cb = t.tupleFirst;` (used as reference, as non-call), instantiate `tupleFirst` // returns fun_ref to instantiated function - FunctionPtr check_and_instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, std::vector&& substitutionTs) const { + FunctionPtr check_and_instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs) const { // T for asm function must be a TVM primitive (width 1), otherwise, asm would act incorrectly if (fun_ref->is_asm_function() || fun_ref->is_builtin_function()) { - for (int i = 0; i < static_cast(substitutionTs.size()); ++i) { - if (substitutionTs[i]->get_width_on_stack() != 1) { - fire_error_calling_asm_function_with_non1_stack_width_arg(cur_f, loc, fun_ref, substitutionTs, i); + for (int i = 0; i < substitutedTs.size(); ++i) { + if (substitutedTs.typeT_at(i)->get_width_on_stack() != 1) { + fire_error_calling_asm_function_with_non1_stack_width_arg(cur_f, loc, fun_ref, substitutedTs, i); } } } - // make deep clone of `f` with substitutionTs + // make deep clone of `f` with substitutedTs // (if `f` was already instantiated, it will be immediately returned from a symbol table) - return instantiate_generic_function(loc, fun_ref, std::move(substitutionTs)); + return instantiate_generic_function(fun_ref, std::move(substitutedTs)); } ExprFlow infer_dot_access(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { @@ -889,8 +965,8 @@ class InferTypesAndCallsAndFieldsVisitor final { } else if (fun_ref->is_generic_function()) { // `t.tupleAt` is valid, it's a reference to instantiation - std::vector substitutions = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref, v_instantiationTs); - fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutions)); + GenericsSubstitutions substitutedTs = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref->genericTs, v_instantiationTs); + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); } else if (UNLIKELY(v_instantiationTs != nullptr)) { // non-generic method referenced like `var cb = c.cellHash;` @@ -915,7 +991,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (auto v_ref = callee->try_as()) { // `globalF()` / `globalF()` / `local_var()` / `SOME_CONST()` - fun_ref = v_ref->sym->try_as(); // not null for `globalF` + fun_ref = v_ref->sym->try_as(); // not null for `globalF` v_instantiationTs = v_ref->get_instantiationTs(); // present for `globalF()` } else if (auto v_dot = callee->try_as()) { @@ -970,29 +1046,36 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } + // for a call `f(...)`, create and instantiate function "f" right now + // so that further, fun_ref will be not a generic, but an instantiated function + if (fun_ref->is_generic_function() && v_instantiationTs) { + GenericsSubstitutions substitutedTs = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref->genericTs, v_instantiationTs); + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); + } else if (UNLIKELY(v_instantiationTs != nullptr)) { + // `c.cellHash()` / `beginCell()` + fire(cur_f, v_instantiationTs->loc, "calling a not generic function with generic T"); + } + // so, we have a call `f(args)` or `obj.f(args)`, f is a global function (fun_ref) (code / asm / builtin) - // we're going to iterate over passed arguments, and (if generic) infer substitutionTs + // we're going to iterate over passed arguments, and (if generic) infer substitutedTs // at first, check arguments count (Tolk doesn't have optional parameters, so just compare counts) int n_arguments = v->get_num_args() + delta_self; int n_parameters = fun_ref->get_num_params(); if (!n_parameters && dot_obj) { - fire(cur_f, v->loc, "`" + fun_ref->name + "` has no parameters and can not be called as method"); + fire(cur_f, v->loc, to_string(fun_ref) + " has no parameters and can not be called as method"); } if (n_parameters < n_arguments) { - fire(cur_f, v->loc, "too many arguments in call to `" + fun_ref->name + "`, expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); + fire(cur_f, v->loc, "too many arguments in call to " + to_string(fun_ref) + ", expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); } if (n_arguments < n_parameters) { - fire(cur_f, v->loc, "too few arguments in call to `" + fun_ref->name + "`, expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); + fire(cur_f, v->loc, "too few arguments in call to " + to_string(fun_ref) + ", expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); } // now, for every passed argument, we need to infer its type // for regular functions, it's obvious - // but for generic functions, we need to infer type arguments (substitutionTs) on the fly + // but for generic functions, we need to infer type arguments (substitutedTs) on the fly // (unless Ts are specified by a user like `f(args)` / `t.tupleAt()`, take them) - GenericSubstitutionsDeduceForCall* deducingTs = fun_ref->is_generic_function() ? new GenericSubstitutionsDeduceForCall(fun_ref) : nullptr; - if (deducingTs && v_instantiationTs) { - deducingTs->provide_manually_specified(collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref, v_instantiationTs)); - } + GenericSubstitutionsDeducing deducingTs(fun_ref); // loop over every argument, for `obj.method()` obj is the first one // if genericT deducing has a conflict, ParseError is thrown @@ -1002,7 +1085,7 @@ class InferTypesAndCallsAndFieldsVisitor final { const LocalVarData& param_0 = fun_ref->parameters[0]; TypePtr param_type = param_0.declared_type; if (param_type->has_genericT_inside()) { - param_type = deducingTs->auto_deduce_from_argument(cur_f, dot_obj->loc, param_type, dot_obj->inferred_type); + param_type = deducingTs.auto_deduce_from_argument(cur_f, dot_obj->loc, param_type, dot_obj->inferred_type); } if (param_0.is_mutate_parameter() && dot_obj->inferred_type->unwrap_alias() != param_type->unwrap_alias()) { if (SinkExpression s_expr = extract_sink_expression_from_vertex(dot_obj)) { @@ -1015,16 +1098,12 @@ class InferTypesAndCallsAndFieldsVisitor final { const LocalVarData& param_i = fun_ref->parameters[delta_self + i]; AnyExprV arg_i = v->get_arg(i)->get_expr(); TypePtr param_type = param_i.declared_type; - if (param_type->has_genericT_inside() && deducingTs->is_manually_specified()) { // `f(a)` - param_type = deducingTs->replace_by_manually_specified(param_type); + if (param_type->has_genericT_inside()) { + param_type = deducingTs.replace_Ts_with_currently_deduced(param_type); } + flow = infer_any_expr(arg_i, std::move(flow), false, param_type).out_flow; if (param_type->has_genericT_inside()) { // `f(a)` where f is generic: use `a` to infer param type - // then arg_i is inferred without any hint - flow = infer_any_expr(arg_i, std::move(flow), false).out_flow; - param_type = deducingTs->auto_deduce_from_argument(cur_f, arg_i->loc, param_type, arg_i->inferred_type); - } else { - // param_type is hint, helps infer arg_i - flow = infer_any_expr(arg_i, std::move(flow), false, param_type).out_flow; + param_type = deducingTs.auto_deduce_from_argument(cur_f, arg_i->loc, param_type, arg_i->inferred_type); } assign_inferred_type(v->get_arg(i), arg_i); // arg itself is an expression if (param_i.is_mutate_parameter() && arg_i->inferred_type->unwrap_alias() != param_type->unwrap_alias()) { @@ -1040,22 +1119,17 @@ class InferTypesAndCallsAndFieldsVisitor final { if (fun_ref->is_generic_function()) { // if `f(args)` was called, Ts were inferred; check that all of them are known - int idx = deducingTs->get_first_not_deduced_idx(); - if (idx != -1 && hint && fun_ref->declared_return_type->has_genericT_inside()) { + std::string_view nameT_unknown = deducingTs.get_first_not_deduced_nameT(); + if (!nameT_unknown.empty() && hint && fun_ref->declared_return_type && fun_ref->declared_return_type->has_genericT_inside()) { // example: `t.tupleFirst()`, T doesn't depend on arguments, but is determined by return type // if used like `var x: int = t.tupleFirst()` / `t.tupleFirst() as int` / etc., use hint - deducingTs->auto_deduce_from_argument(cur_f, v->loc, fun_ref->declared_return_type, hint); - idx = deducingTs->get_first_not_deduced_idx(); + deducingTs.auto_deduce_from_argument(cur_f, v->loc, fun_ref->declared_return_type, hint); + nameT_unknown = deducingTs.get_first_not_deduced_nameT(); } - if (idx != -1) { - fire(cur_f, v->loc, "can not deduce " + fun_ref->genericTs->get_nameT(idx)); + if (!nameT_unknown.empty()) { + deducingTs.fire_error_can_not_deduce(cur_f, v->get_arg_list()->loc, nameT_unknown); } - fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, deducingTs->flush()); - delete deducingTs; - - } else if (UNLIKELY(v_instantiationTs != nullptr)) { - // non-generic function/method called with type arguments, like `c.cellHash()` / `beginCell()` - fire(cur_f, v_instantiationTs->loc, "calling a not generic function with generic T"); + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, deducingTs.flush()); } v->mutate()->assign_fun_ref(fun_ref); @@ -1118,7 +1192,26 @@ class InferTypesAndCallsAndFieldsVisitor final { auto v_arm = v->get_arm(i); FlowContext arm_flow = infer_any_expr(v_arm->get_pattern_expr(), arms_entry_facts.clone(), false, nullptr).out_flow; if (s_expr && v_arm->pattern_kind == MatchArmKind::exact_type) { - arm_flow.register_known_type(s_expr, v_arm->pattern_type_node->resolved_type); + TypePtr exact_type = v_arm->pattern_type_node->resolved_type; + if (const auto* t_struct = exact_type->try_as(); t_struct && t_struct->struct_ref->is_generic_struct()) { + // `Wrapper => ...`, detect T based on type of subject (`Wrapper | int` => `Wrapper`) + if (TypePtr inst_exact_type = try_pick_instantiated_generic_from_hint(v->get_subject()->inferred_type, t_struct->struct_ref)) { + exact_type = inst_exact_type; + v_arm->pattern_type_node->mutate()->assign_resolved_type(exact_type); + } else { + fire(cur_f, v_arm->loc, "can not deduce type arguments for " + to_string(t_struct->struct_ref) + ", provide them manually"); + } + } + if (const auto* t_alias = exact_type->try_as(); t_alias && t_alias->alias_ref->is_generic_alias()) { + // `WrapperAlias => ...`, detect T similar to structures + if (TypePtr inst_exact_type = try_pick_instantiated_generic_from_hint(v->get_subject()->inferred_type, t_alias->alias_ref)) { + exact_type = inst_exact_type; + v_arm->pattern_type_node->mutate()->assign_resolved_type(exact_type); + } else { + fire(cur_f, v_arm->loc, "can not deduce type arguments for " + to_string(t_alias->alias_ref) + ", provide them manually"); + } + } + arm_flow.register_known_type(s_expr, exact_type); } arm_flow = infer_any_expr(v_arm->get_body(), std::move(arm_flow), false, hint).out_flow; match_out_flow = i == 0 ? std::move(arm_flow) : FlowContext::merge_flow(std::move(match_out_flow), std::move(arm_flow)); @@ -1147,22 +1240,26 @@ class InferTypesAndCallsAndFieldsVisitor final { } ExprFlow infer_object_literal(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + // the goal is to detect struct_ref + // either by lhs hint `var u: User = { ... }, or by explicitly provided ref `User { ... }` StructPtr struct_ref = nullptr; - if (v->has_explicit_ref()) { - V v_ref = v->get_explicit_ref(); - struct_ref = v_ref->sym->try_as(); - if (!struct_ref) { - if (AliasDefPtr as_alias = v_ref->sym->try_as()) { - if (const TypeDataStruct* as_struct = as_alias->underlying_type->unwrap_alias()->try_as()) { - struct_ref = as_struct->struct_ref; - } - } + + // `User { ... }` / `UserAlias { ... }` / `Wrapper { ... }` / `Wrapper { ... }` + if (v->type_node) { + TypePtr provided_type = v->type_node->resolved_type->unwrap_alias(); + if (const TypeDataStruct* hint_struct = provided_type->try_as()) { + struct_ref = hint_struct->struct_ref; // `Wrapper` / `Wrapper` + } else if (const TypeDataGenericTypeWithTs* hint_instTs = provided_type->try_as()) { + struct_ref = hint_instTs->struct_ref; // if `type WAlias = Wrapper`, here `Wrapper` (generic struct) } if (!struct_ref) { - fire(cur_f, v_ref->loc, "`" + to_string(v_ref->get_name()) + "` does not name a struct"); + fire(cur_f, v->type_node->loc, to_string(v->type_node->resolved_type) + " does not name a struct"); } - if (UNLIKELY(v_ref->has_instantiationTs())) { - fire(cur_f, v_ref->get_instantiationTs()->loc, "generic T not expected here"); + // example: `var v: Ok = Ok { ... }`, now struct_ref is "Ok", take "Ok" from hint + if (struct_ref->is_generic_struct() && hint) { + if (TypePtr inst_explicit_ref = try_pick_instantiated_generic_from_hint(hint, struct_ref)) { + struct_ref = inst_explicit_ref->try_as()->struct_ref; + } } } if (!struct_ref && hint) { @@ -1187,26 +1284,55 @@ class InferTypesAndCallsAndFieldsVisitor final { fire(cur_f, v->loc, "can not detect struct name; use either `var v: StructName = { ... }` or `var v = StructName { ... }`"); } + // so, we have struct_ref, so we can check field names and infer values + // if it's a generic struct, we need to deduce Ts by field values, like for a function call + GenericSubstitutionsDeducing deducingTs(struct_ref); + uint64_t occurred_mask = 0; for (int i = 0; i < v->get_body()->get_num_fields(); ++i) { - auto v_field = v->get_body()->get_field(i); - std::string_view field_name = v_field->get_field_name(); + auto field_i = v->get_body()->get_field(i); + std::string_view field_name = field_i->get_field_name(); StructFieldPtr field_ref = struct_ref->find_field(field_name); if (!field_ref) { - fire(cur_f, v_field->loc, "field `" + to_string(field_name) + "` not found in struct `" + struct_ref->name + "`"); + fire(cur_f, field_i->loc, "field `" + to_string(field_name) + "` not found in struct " + to_string(struct_ref)); } - v_field->mutate()->assign_field_ref(field_ref); + field_i->mutate()->assign_field_ref(field_ref); if (occurred_mask & (1ULL << field_ref->field_idx)) { - fire(cur_f, v_field->loc, "duplicate field initialization"); + fire(cur_f, field_i->loc, "duplicate field initialization"); } occurred_mask |= 1ULL << field_ref->field_idx; - flow = infer_any_expr(v_field->get_init_val(), std::move(flow), false, field_ref->declared_type).out_flow; - assign_inferred_type(v_field, v_field->get_init_val()); + + AnyExprV val_i = field_i->get_init_val(); + TypePtr field_type = field_ref->declared_type; + if (field_type->has_genericT_inside()) { + field_type = deducingTs.replace_Ts_with_currently_deduced(field_type); + } + if (field_type->has_genericT_inside()) { // `item: v` where field `item` is generic: use `v` to infer T + flow = infer_any_expr(val_i, std::move(flow), false).out_flow; + deducingTs.auto_deduce_from_argument(cur_f, field_i->loc, field_type, val_i->inferred_type); + } else { + // field_type is hint, helps infer val_i + flow = infer_any_expr(val_i, std::move(flow), false, field_type).out_flow; + } + assign_inferred_type(field_i, val_i); } for (StructFieldPtr field_ref : struct_ref->fields) { if (!(occurred_mask & (1ULL << field_ref->field_idx)) && !field_ref->has_default_value()) { - fire(cur_f, v->get_body()->loc, "field `" + field_ref->name + "` missed in initialization of struct `" + struct_ref->name + "`"); + fire(cur_f, v->get_body()->loc, "field `" + field_ref->name + "` missed in initialization of struct " + to_string(struct_ref)); + } + } + + // if it's a generic struct `Wrapper`, we need to instantiate it, like `Wrapper` + if (struct_ref->is_generic_struct()) { + if (std::string_view nameT = deducingTs.get_first_not_deduced_nameT(); !nameT.empty()) { + deducingTs.fire_error_can_not_deduce(cur_f, v->loc, nameT); + } + struct_ref = instantiate_generic_struct(struct_ref, deducingTs.flush()); + // re-assign field_ref (it was earlier assigned into a field of a generic struct) + for (int i = 0; i < v->get_body()->get_num_fields(); ++i) { + auto field_i = v->get_body()->get_field(i); + field_i->mutate()->assign_field_ref(struct_ref->find_field(field_i->get_field_name())); } } @@ -1406,13 +1532,13 @@ class InferTypesAndCallsAndFieldsVisitor final { if (has_void_returns && has_non_void_returns) { for (AnyExprV return_expr : return_statements) { if (return_expr->inferred_type == TypeDataVoid::create()) { - fire(fun_ref, return_expr->loc, "mixing void and non-void returns in function `" + fun_ref->as_human_readable() + "`"); + fire(fun_ref, return_expr->loc, "mixing void and non-void returns in function " + to_string(fun_ref)); } } } if (return_unifier.is_union_of_different_types()) { // `return intVar` + `return sliceVar` results in `int | slice`, probably unexpected - fire(fun_ref, v_function->get_body()->loc, "function `" + fun_ref->as_human_readable() + "` calculated return type is " + to_string(inferred_return_type) + "; probably, it's not what you expected; declare `fun (...): ` manually"); + fire(fun_ref, v_function->get_body()->loc, "function " + to_string(fun_ref) + " calculated return type is " + to_string(inferred_return_type) + "; probably, it's not what you expected; declare `fun (...): ` manually"); } } @@ -1514,7 +1640,9 @@ void pipeline_infer_types_and_calls_and_fields() { // infer types for default values in structs for (StructPtr struct_ref : get_all_declared_structs()) { - visitor.start_visiting_struct_fields(struct_ref); + if (!struct_ref->is_generic_struct()) { + visitor.start_visiting_struct_fields(struct_ref); + } } } diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 7a09defae4..29754580bb 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -109,45 +109,68 @@ static const GenericsDeclaration* construct_genericTs(V v_li return new GenericsDeclaration(std::move(itemsT)); } -static void register_constant(V v) { +static GlobalConstPtr register_constant(V v) { GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->type_node, v->get_init_value()); G.symtable.add_global_const(c_sym); G.all_constants.push_back(c_sym); v->mutate()->assign_const_ref(c_sym); + return c_sym; } -static void register_global_var(V v) { +static GlobalVarPtr register_global_var(V v) { GlobalVarData* g_sym = new GlobalVarData(static_cast(v->get_identifier()->name), v->loc, v->type_node); G.symtable.add_global_var(g_sym); G.all_global_vars.push_back(g_sym); v->mutate()->assign_glob_ref(g_sym); + return g_sym; } -static void register_type_alias(V v) { - AliasDefData* a_sym = new AliasDefData(static_cast(v->get_identifier()->name), v->loc, v->underlying_type_node); +static AliasDefPtr register_type_alias(V v, AliasDefPtr base_alias_ref = nullptr, const GenericsSubstitutions* substitutedTs = nullptr) { + const GenericsDeclaration* genericTs = nullptr; + if (v->genericsT_list && !substitutedTs) { + genericTs = construct_genericTs(v->genericsT_list); + } + + AliasDefData* a_sym = new AliasDefData(static_cast(v->get_identifier()->name), v->loc, v->underlying_type_node, genericTs, substitutedTs, v); + a_sym->base_alias_ref = base_alias_ref; // for `Response`, here is `Response` G.symtable.add_type_alias(a_sym); v->mutate()->assign_alias_ref(a_sym); + return a_sym; } -static void register_struct(V v) { +static StructPtr register_struct(V v, StructPtr base_struct_ref = nullptr, const GenericsSubstitutions* substitutedTs = nullptr) { auto v_body = v->get_struct_body(); std::vector fields; fields.reserve(v_body->get_num_fields()); for (int i = 0; i < v_body->get_num_fields(); ++i) { auto v_field = v_body->get_field(i); + std::string field_name = static_cast(v_field->get_identifier()->name); AnyExprV default_value = v_field->has_default_value() ? v_field->get_default_value() : nullptr; + + for (StructFieldPtr prev : fields) { + if (UNLIKELY(prev->name == field_name)) { + v_field->error("redeclaration of field `" + field_name + "`"); + } + } fields.emplace_back(new StructFieldData(static_cast(v_field->get_identifier()->name), v_field->loc, i, v_field->type_node, default_value)); } - StructData* s_sym = new StructData(static_cast(v->get_identifier()->name), v->loc, std::move(fields)); + const GenericsDeclaration* genericTs = nullptr; + if (v->genericsT_list && !substitutedTs) { + genericTs = construct_genericTs(v->genericsT_list); + } + + StructData* s_sym = new StructData(static_cast(v->get_identifier()->name), v->loc, std::move(fields), genericTs, substitutedTs, v); + s_sym->base_struct_ref = base_struct_ref; // for `Container`, here is `Container` G.symtable.add_struct(s_sym); G.all_structs.push_back(s_sym); v->mutate()->assign_struct_ref(s_sym); + return s_sym; } static LocalVarData register_parameter(V v, int idx) { @@ -165,7 +188,7 @@ static LocalVarData register_parameter(V v, int idx) { return LocalVarData(static_cast(v->param_name), v->loc, v->type_node, flags, idx); } -static void register_function(V v, const GenericsInstantiation* instantiationTs = nullptr) { +static FunctionPtr register_function(V v, FunctionPtr base_fun_ref = nullptr, const GenericsSubstitutions* substitutedTs = nullptr) { std::string_view func_name = v->get_identifier()->name; std::vector parameters; @@ -184,12 +207,17 @@ static void register_function(V v, const GenericsInsta v->error("`builtin` used for non-builtin function"); } v->mutate()->assign_fun_ref(fun_ref); - return; + return fun_ref; + } + + const GenericsDeclaration* genericTs = nullptr; + if (v->genericsT_list && !substitutedTs) { + genericTs = construct_genericTs(v->genericsT_list); } - const GenericsDeclaration* genericTs = v->genericsT_list ? construct_genericTs(v->genericsT_list) : nullptr; FunctionBody f_body = v->get_body()->kind == ast_block_statement ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); - FunctionData* f_sym = new FunctionData(static_cast(func_name), v->loc, v->return_type_node, std::move(parameters), 0, genericTs, instantiationTs, f_body, v); + FunctionData* f_sym = new FunctionData(static_cast(func_name), v->loc, v->return_type_node, std::move(parameters), 0, genericTs, substitutedTs, f_body, v); + f_sym->base_fun_ref = base_fun_ref; // for `f`, here is `f` if (auto v_asm = v->get_body()->try_as()) { if (!v->return_type_node) { @@ -223,6 +251,7 @@ static void register_function(V v, const GenericsInsta G.all_get_methods.push_back(f_sym); } v->mutate()->assign_fun_ref(f_sym); + return f_sym; } static void iterate_through_file_symbols(const SrcFile* file) { @@ -267,10 +296,19 @@ void pipeline_register_global_symbols() { } } -FunctionPtr pipeline_register_instantiated_generic_function(AnyV cloned_v, const GenericsInstantiation* instantiationTs) { +FunctionPtr pipeline_register_instantiated_generic_function(FunctionPtr base_fun_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs) { auto v = cloned_v->as(); - register_function(v, instantiationTs); - return lookup_global_symbol(v->get_identifier()->name)->try_as(); + return register_function(v, base_fun_ref, substitutedTs); +} + +StructPtr pipeline_register_instantiated_generic_struct(StructPtr base_struct_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs) { + auto v = cloned_v->as(); + return register_struct(v, base_struct_ref, substitutedTs); +} + +AliasDefPtr pipeline_register_instantiated_generic_alias(AliasDefPtr base_alias_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs) { + auto v = cloned_v->as(); + return register_type_alias(v, base_alias_ref, substitutedTs); } } // namespace tolk diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 9f089b0644..667172034b 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -305,4 +305,8 @@ void pipeline_resolve_identifiers_and_assign_symbols(FunctionPtr fun_ref) { } } +void pipeline_resolve_identifiers_and_assign_symbols(StructPtr struct_ref) { + AssignSymInsideFunctionVisitor().start_visiting_struct_fields(struct_ref); +} + } // namespace tolk diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 3af32fa96c..8f805f741e 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -34,13 +34,17 @@ namespace tolk { * Example: `fun f(a: cell): (int, User)` param to TypeDataCell, return type to TypeDataTensor(TypeDataInt, TypeDataStruct) * Example: `var x: T = 0` to TypeDataGenericT inside `f` * Example: `f()` to TypeDataAlias inside instantiation list + * Example: `arg: Wrapper` instantiates "Wrapper" right here and returns TypeDataStruct to it * Example: `fun f(): KKK` fires an error "unknown type name" * * Types resolving is done everywhere: inside functions bodies, in struct fields, inside globals declaration, etc. * See finalize_type_node(). * * Note, that resolving T to TypeDataGenericT (and replacing T with substitution when instantiating a generic type) - * is also done here, see genericTs and instantiationTs. + * is also done here, see genericTs and substitutedTs. + * Note, that instantiating generic structs and aliases is also done here (if they don't have generic Ts inside). + * Example: `type OkInt = Ok`, struct "Ok" is instantiated (as a clone of `Ok` substituting T=int) + * Example: `type A = Ok`, then `Ok` is not ready yet, it's left as TypeDataGenericTypeWithTs. */ GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD @@ -59,6 +63,11 @@ static void fire_void_type_not_allowed_inside_union(FunctionPtr cur_f, SrcLocati fire(cur_f, loc, "type `" + disallowed_variant->as_human_readable() + "` is not allowed inside a union"); } +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_generic_type_used_without_T(FunctionPtr cur_f, SrcLocation loc, const std::string& type_name_with_Ts) { + fire(cur_f, loc, "type `" + type_name_with_Ts + "` is generic, you should provide type arguments"); +} + static TypePtr parse_intN(std::string_view strN, bool is_unsigned) { int n; auto result = std::from_chars(strN.data() + 3 + static_cast(is_unsigned), strN.data() + strN.size(), n); @@ -137,31 +146,30 @@ static TypePtr try_parse_predefined_type(std::string_view str) { class TypeNodesVisitorResolver { FunctionPtr cur_f; // exists if we're inside its body const GenericsDeclaration* genericTs; // `` if we're inside `f` or `f` - const GenericsInstantiation* instantiationTs; // `` if we're inside `f` + const GenericsSubstitutions* substitutedTs; // `T=int` if we're inside `f` - TypePtr parse_ast_type_node(AnyTypeV v) { + TypePtr parse_ast_type_node(AnyTypeV v, bool allow_without_type_arguments) { switch (v->kind) { case ast_type_leaf_text: { std::string_view text = v->as()->text; - if (TypePtr predefined_type = try_parse_predefined_type(text)) { - return predefined_type; - } - if (genericTs) { + SrcLocation loc = v->as()->loc; + if (genericTs && genericTs->find_nameT(text) != -1) { // if we're inside `f`, replace "T" with TypeDataGenericT - // if we're inside `f`, replace "T" with TypeDataInt (substitution) - if (int idx = genericTs->find_nameT(text); idx != -1) { - if (instantiationTs) { - return instantiationTs->substitutions[idx]; - } - return TypeDataGenericT::create(static_cast(text)); - } + return TypeDataGenericT::create(static_cast(text)); + } + if (substitutedTs && substitutedTs->has_nameT(text)) { + // if we're inside `f`, replace "T" with TypeDataInt + return substitutedTs->get_substitution_for_nameT(text); } if (const Symbol* sym = lookup_global_symbol(text)) { - if (TypePtr struct_or_other = try_resolve_user_defined_type(sym)) { - return struct_or_other; + if (TypePtr custom_type = try_resolve_user_defined_type(cur_f, loc, sym, allow_without_type_arguments)) { + return custom_type; } } - fire_error_unknown_type_name(cur_f, v->as()->loc, text); + if (TypePtr predefined_type = try_parse_predefined_type(text)) { + return predefined_type; + } + fire_error_unknown_type_name(cur_f, loc, text); } case ast_type_question_nullable: { @@ -202,19 +210,41 @@ class TypeNodesVisitorResolver { return result; } + case ast_type_triangle_args: { + const std::vector& inner_and_args = v->as()->get_inner_and_args(); + TypePtr inner = finalize_type_node(inner_and_args.front(), true); + std::vector type_arguments; + type_arguments.reserve(inner_and_args.size() - 1); + for (size_t i = 1; i < inner_and_args.size(); ++i) { + type_arguments.push_back(finalize_type_node(inner_and_args[i])); + } + return instantiate_generic_type_or_fire(cur_f, v->loc, inner, std::move(type_arguments)); + } + default: - throw UnexpectedASTNodeKind(v, "resolve_ast_type_node"); + throw UnexpectedASTNodeKind(v, "parse_ast_type_node"); } } - static TypePtr try_resolve_user_defined_type(const Symbol* sym) { + // given `dict` / `User` / `Wrapper` / `WrapperAlias`, find it in a symtable + // for generic types, like `Wrapper`, fire that it's used without type arguments (unless allowed) + // example: `var w: Wrapper = ...`, here will be an error of generic usage without T + // example: `w is Wrapper`, here not, it's allowed (instantiated at type inferring later) + // example: `var w: KKK`, nullptr will be returned + static TypePtr try_resolve_user_defined_type(FunctionPtr cur_f, SrcLocation loc, const Symbol* sym, bool allow_without_type_arguments) { if (AliasDefPtr alias_ref = sym->try_as()) { + if (alias_ref->is_generic_alias() && !allow_without_type_arguments) { + fire_error_generic_type_used_without_T(cur_f, loc, alias_ref->as_human_readable()); + } if (!alias_ref->was_visited_by_resolver()) { visit_symbol(alias_ref); } return TypeDataAlias::create(alias_ref); } if (StructPtr struct_ref = sym->try_as()) { + if (struct_ref->is_generic_struct() && !allow_without_type_arguments) { + fire_error_generic_type_used_without_T(cur_f, loc, struct_ref->as_human_readable()); + } if (!struct_ref->was_visited_by_resolver()) { visit_symbol(struct_ref); } @@ -223,6 +253,40 @@ class TypeNodesVisitorResolver { return nullptr; } + // given `Wrapper` / `Pair` / `Response`, instantiate a generic struct/alias + // an error for invalid usage `Pair` / `cell` is also here + static TypePtr instantiate_generic_type_or_fire(FunctionPtr cur_f, SrcLocation loc, TypePtr type_to_instantiate, std::vector&& type_arguments) { + // example: `type WrapperAlias = Wrapper`, we are at `Wrapper`, type_arguments = `` + // they contain generics, so the struct is not ready to be instantiated yet + bool is_still_generic = false; + for (TypePtr argT : type_arguments) { + is_still_generic |= argT->has_genericT_inside(); + } + + if (const TypeDataStruct* t_struct = type_to_instantiate->try_as(); t_struct && t_struct->struct_ref->is_generic_struct()) { + StructPtr struct_ref = t_struct->struct_ref; + if (struct_ref->genericTs->size() != static_cast(type_arguments.size())) { + fire(cur_f, loc, "struct `" + struct_ref->as_human_readable() + "` expects " + std::to_string(struct_ref->genericTs->size()) + " type arguments, but " + std::to_string(type_arguments.size()) + " provided"); + } + if (is_still_generic) { + return TypeDataGenericTypeWithTs::create(struct_ref, nullptr, std::move(type_arguments)); + } + return TypeDataStruct::create(instantiate_generic_struct(struct_ref, GenericsSubstitutions(struct_ref->genericTs, type_arguments))); + } + if (const TypeDataAlias* t_alias = type_to_instantiate->try_as(); t_alias && t_alias->alias_ref->is_generic_alias()) { + AliasDefPtr alias_ref = t_alias->alias_ref; + if (alias_ref->genericTs->size() != static_cast(type_arguments.size())) { + fire(cur_f, loc, "type `" + alias_ref->as_human_readable() + "` expects " + std::to_string(alias_ref->genericTs->size()) + " type arguments, but " + std::to_string(type_arguments.size()) + " provided"); + } + if (is_still_generic) { + return TypeDataGenericTypeWithTs::create(nullptr, alias_ref, std::move(type_arguments)); + } + return TypeDataAlias::create(instantiate_generic_alias(alias_ref, GenericsSubstitutions(alias_ref->genericTs, type_arguments))); + } + // `User` / `cell` + fire(cur_f, loc, "type `" + type_to_instantiate->as_human_readable() + "` is not generic"); + } + static void validate_resulting_union_type(const TypeDataUnion* t_union, FunctionPtr cur_f, SrcLocation loc) { for (TypePtr variant : t_union->variants) { if (variant == TypeDataVoid::create() || variant == TypeDataNever::create()) { @@ -233,16 +297,16 @@ class TypeNodesVisitorResolver { public: - TypeNodesVisitorResolver(FunctionPtr cur_f, const GenericsDeclaration* genericTs, const GenericsInstantiation* instantiationTs) + TypeNodesVisitorResolver(FunctionPtr cur_f, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs) : cur_f(cur_f) , genericTs(genericTs) - , instantiationTs(instantiationTs) {} + , substitutedTs(substitutedTs) {} - TypePtr finalize_type_node(AnyTypeV type_node) { + TypePtr finalize_type_node(AnyTypeV type_node, bool allow_without_type_arguments = false) { #ifdef TOLK_DEBUG tolk_assert(type_node != nullptr); #endif - TypePtr resolved_type = parse_ast_type_node(type_node); + TypePtr resolved_type = parse_ast_type_node(type_node, allow_without_type_arguments); type_node->mutate()->assign_resolved_type(resolved_type); return resolved_type; } @@ -282,7 +346,7 @@ class TypeNodesVisitorResolver { } called_stack.push_back(alias_ref); - TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr); + TypeNodesVisitorResolver visitor(nullptr, alias_ref->genericTs, alias_ref->substitutedTs); TypePtr underlying_type = visitor.finalize_type_node(alias_ref->underlying_type_node); alias_ref->mutate()->assign_resolved_type(underlying_type); alias_ref->mutate()->assign_visited_by_resolver(); @@ -301,7 +365,7 @@ class TypeNodesVisitorResolver { } called_stack.push_back(struct_ref); - TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr); + TypeNodesVisitorResolver visitor(nullptr, struct_ref->genericTs, struct_ref->substitutedTs); for (int i = 0; i < struct_ref->get_num_fields(); ++i) { StructFieldPtr field_ref = struct_ref->get_field(i); TypePtr declared_type = visitor.finalize_type_node(field_ref->type_node); @@ -313,10 +377,10 @@ class TypeNodesVisitorResolver { }; class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { - static TypeNodesVisitorResolver type_nodes_visitor; + TypeNodesVisitorResolver type_nodes_visitor{nullptr, nullptr, nullptr}; - static TypePtr finalize_type_node(AnyTypeV type_node) { - return type_nodes_visitor.finalize_type_node(type_node); + TypePtr finalize_type_node(AnyTypeV type_node, bool allow_without_type_arguments = false) { + return type_nodes_visitor.finalize_type_node(type_node, allow_without_type_arguments); } protected: @@ -329,6 +393,8 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { } void visit(V v) override { + tolk_assert(v->sym != nullptr); + // for `f` / `f`, resolve "MyAlias" and "T" // (for function call `f()`, this v (ast_reference `f`) is callee) if (auto v_instantiationTs = v->get_instantiationTs()) { @@ -340,7 +406,9 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { if (v->pattern_type_node) { - finalize_type_node(v->pattern_type_node); + // before `=>` we allow referencing generic types, type inferring will guess + // example: `struct Ok` + `type Response = Ok | ErrCode` + `match (resp) { Ok => ... }` + finalize_type_node(v->pattern_type_node, true); } parent::visit(v->get_pattern_expr()); parent::visit(v->get_body()); @@ -363,17 +431,24 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { } void visit(V v) override { - finalize_type_node(v->type_node); + finalize_type_node(v->type_node, true); parent::visit(v->get_expr()); } + void visit(V v) override { + if (v->type_node) { + finalize_type_node(v->type_node, true); + } + parent::visit(v->get_body()); + } + public: bool should_visit_function(FunctionPtr fun_ref) override { return !fun_ref->is_builtin_function(); } void start_visiting_function(FunctionPtr fun_ref, V v) override { - type_nodes_visitor = TypeNodesVisitorResolver(fun_ref, fun_ref->genericTs, fun_ref->instantiationTs); + type_nodes_visitor = TypeNodesVisitorResolver(fun_ref, fun_ref->genericTs, fun_ref->substitutedTs); for (int i = 0; i < v->get_num_params(); ++i) { const LocalVarData& param_var = fun_ref->parameters[i]; @@ -388,17 +463,19 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { if (fun_ref->is_code_function()) { parent::visit(v->get_body()->as()); } - - type_nodes_visitor = TypeNodesVisitorResolver(nullptr, nullptr, nullptr); } void start_visiting_constant(GlobalConstPtr const_ref) { + type_nodes_visitor = TypeNodesVisitorResolver(nullptr, nullptr, nullptr); + // `const a = 0 as int8`, resolve types there // same for struct field `v: int8 = 0 as int8` parent::visit(const_ref->init_value); } void start_visiting_struct_fields(StructPtr struct_ref) { + type_nodes_visitor = TypeNodesVisitorResolver(nullptr, struct_ref->genericTs, struct_ref->substitutedTs); + // same for struct field `v: int8 = 0 as int8` for (StructFieldPtr field_ref : struct_ref->fields) { if (field_ref->has_default_value()) { @@ -408,8 +485,6 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { } }; -TypeNodesVisitorResolver ResolveTypesInsideFunctionVisitor::type_nodes_visitor(nullptr, nullptr, nullptr); - void pipeline_resolve_types_and_aliases() { ResolveTypesInsideFunctionVisitor visitor; @@ -452,4 +527,13 @@ void pipeline_resolve_types_and_aliases(FunctionPtr fun_ref) { } } +void pipeline_resolve_types_and_aliases(StructPtr struct_ref) { + ResolveTypesInsideFunctionVisitor().start_visiting_struct_fields(struct_ref); + TypeNodesVisitorResolver::visit_symbol(struct_ref); +} + +void pipeline_resolve_types_and_aliases(AliasDefPtr alias_ref) { + TypeNodesVisitorResolver::visit_symbol(alias_ref); +} + } // namespace tolk diff --git a/tolk/pipeline.h b/tolk/pipeline.h index acd0fd87a4..c0a51b096e 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -50,12 +50,19 @@ void pipeline_generate_fif_output_to_std_cout(); // these pipes also can be called per-function individually // they are called for instantiated generics functions, when `f` is deeply cloned as `f` -FunctionPtr pipeline_register_instantiated_generic_function(AnyV cloned_v, const GenericsInstantiation* instantiationTs); +FunctionPtr pipeline_register_instantiated_generic_function(FunctionPtr base_fun_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs); + void pipeline_resolve_identifiers_and_assign_symbols(FunctionPtr); void pipeline_resolve_types_and_aliases(FunctionPtr); void pipeline_calculate_rvalue_lvalue(FunctionPtr); void pipeline_detect_unreachable_statements(FunctionPtr); void pipeline_infer_types_and_calls_and_fields(FunctionPtr); +StructPtr pipeline_register_instantiated_generic_struct(StructPtr base_struct_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs); +void pipeline_resolve_identifiers_and_assign_symbols(StructPtr); +void pipeline_resolve_types_and_aliases(StructPtr); + +AliasDefPtr pipeline_register_instantiated_generic_alias(AliasDefPtr base_alias_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs); +void pipeline_resolve_types_and_aliases(AliasDefPtr); } // namespace tolk diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index dd6a5abbfb..31b19558fc 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -368,7 +368,7 @@ TypePtr calculate_type_subtract_rhs_type(TypePtr type, TypePtr subtract_type) { if (const TypeDataUnion* sub_union = subtract_type->try_as()) { if (lhs_union->has_all_variants_of(sub_union)) { - rest_variants.reserve(lhs_union->variants.size() - sub_union->variants.size()); + rest_variants.reserve(lhs_union->size() - sub_union->size()); for (TypePtr lhs_variant : lhs_union->variants) { if (!sub_union->has_variant_with_type_id(lhs_variant)) { rest_variants.push_back(lhs_variant); @@ -376,7 +376,7 @@ TypePtr calculate_type_subtract_rhs_type(TypePtr type, TypePtr subtract_type) { } } } else if (lhs_union->has_variant_with_type_id(subtract_type)) { - rest_variants.reserve(lhs_union->variants.size() - 1); + rest_variants.reserve(lhs_union->size() - 1); for (TypePtr lhs_variant : lhs_union->variants) { if (lhs_variant->get_type_id() != subtract_type->get_type_id()) { rest_variants.push_back(lhs_variant); @@ -496,7 +496,7 @@ TypePtr calc_smart_cast_type_on_assignment(TypePtr lhs_declared_type, TypePtr rh for (TypePtr rhs_variant : rhs_union->variants) { lhs_has_all_variants_of_rhs &= lhs_union->has_variant_with_type_id(rhs_variant); } - if (lhs_has_all_variants_of_rhs && rhs_union->variants.size() < lhs_union->variants.size()) { + if (lhs_has_all_variants_of_rhs && rhs_union->size() < lhs_union->size()) { std::vector subtypes_of_lhs; for (TypePtr lhs_variant : lhs_union->variants) { if (rhs_union->has_variant_with_type_id(lhs_variant)) { diff --git a/tolk/smart-casts-cfg.h b/tolk/smart-casts-cfg.h index 31d4ebe0d4..1e24a7b696 100644 --- a/tolk/smart-casts-cfg.h +++ b/tolk/smart-casts-cfg.h @@ -18,6 +18,7 @@ #include "fwd-declarations.h" #include "type-system.h" +#include #include #include diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index e2a0a1836a..2449de59df 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -28,6 +28,20 @@ std::string FunctionData::as_human_readable() const { return name + genericTs->as_human_readable(); } +std::string AliasDefData::as_human_readable() const { + if (!is_generic_alias()) { + return name; + } + return name + genericTs->as_human_readable(); +} + +std::string StructData::as_human_readable() const { + if (!is_generic_struct()) { + return name; + } + return name + genericTs->as_human_readable(); +} + bool FunctionData::does_need_codegen() const { // when a function is declared, but not referenced from code in any way, don't generate its body if (!is_really_used() && G.settings.remove_unused_functions) { diff --git a/tolk/symtable.h b/tolk/symtable.h index 4252386c0d..cde3a83974 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -86,7 +86,6 @@ struct FunctionBodyCode; struct FunctionBodyAsm; struct FunctionBodyBuiltin; struct GenericsDeclaration; -struct GenericsInstantiation; typedef std::variant< FunctionBodyCode*, @@ -124,28 +123,29 @@ struct FunctionData final : Symbol { TypePtr inferred_full_type = nullptr; // assigned on type inferring, it's TypeDataFunCallable(params -> return) const GenericsDeclaration* genericTs; - const GenericsInstantiation* instantiationTs; + const GenericsSubstitutions* substitutedTs; + FunctionPtr base_fun_ref = nullptr; // for `f`, here is `f` FunctionBody body; AnyV ast_root; // V for user-defined (not builtin) - FunctionData(std::string name, SrcLocation loc, AnyTypeV return_type_node, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsInstantiation* instantiationTs, FunctionBody body, AnyV ast_root) + FunctionData(std::string name, SrcLocation loc, AnyTypeV return_type_node, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) : Symbol(std::move(name), loc) , flags(initial_flags) , parameters(std::move(parameters)) , return_type_node(return_type_node) , genericTs(genericTs) - , instantiationTs(instantiationTs) + , substitutedTs(substitutedTs) , body(body) , ast_root(ast_root) { } - FunctionData(std::string name, SrcLocation loc, TypePtr declared_return_type, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsInstantiation* instantiationTs, FunctionBody body, AnyV ast_root) + FunctionData(std::string name, SrcLocation loc, TypePtr declared_return_type, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) : Symbol(std::move(name), loc) , flags(initial_flags) , parameters(std::move(parameters)) , return_type_node(nullptr) // for built-in functions, defined in sources , declared_return_type(declared_return_type) , genericTs(genericTs) - , instantiationTs(instantiationTs) + , substitutedTs(substitutedTs) , body(body) , ast_root(ast_root) { } @@ -166,8 +166,8 @@ struct FunctionData final : Symbol { bool is_asm_function() const { return std::holds_alternative(body); } bool is_builtin_function() const { return ast_root == nullptr; } - bool is_generic_function() const { return genericTs != nullptr && instantiationTs == nullptr; } - bool is_instantiation_of_generic_function() const { return instantiationTs != nullptr; } + bool is_generic_function() const { return genericTs != nullptr; } + bool is_instantiation_of_generic_function() const { return substitutedTs != nullptr; } bool is_inline() const { return flags & flagInline; } bool is_inline_ref() const { return flags & flagInlineRef; } @@ -245,11 +245,24 @@ struct AliasDefData final : Symbol { TypePtr underlying_type = nullptr; // = resolved underlying_type_node int flags = 0; - AliasDefData(std::string name, SrcLocation loc, AnyTypeV underlying_type_node) + const GenericsDeclaration* genericTs; + const GenericsSubstitutions* substitutedTs; + AliasDefPtr base_alias_ref = nullptr; // for `Response`, here is `Response` + AnyV ast_root; // V + + AliasDefData(std::string name, SrcLocation loc, AnyTypeV underlying_type_node, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, AnyV ast_root) : Symbol(std::move(name), loc) - , underlying_type_node(underlying_type_node) { + , underlying_type_node(underlying_type_node) + , genericTs(genericTs) + , substitutedTs(substitutedTs) + , ast_root(ast_root) { } + std::string as_human_readable() const; + + bool is_generic_alias() const { return genericTs != nullptr; } + bool is_instantiation_of_generic_alias() const { return substitutedTs != nullptr; } + bool was_visited_by_resolver() const { return flags & flagVisitedByResolver; } AliasDefData* mutate() const { return const_cast(this); } @@ -285,19 +298,32 @@ struct StructData final : Symbol { std::vector fields; int flags = 0; + const GenericsDeclaration* genericTs; + const GenericsSubstitutions* substitutedTs; + StructPtr base_struct_ref = nullptr; // for `Container`, here is `Container` + AnyV ast_root; // V + int get_num_fields() const { return static_cast(fields.size()); } StructFieldPtr get_field(int i) const { return fields.at(i); } StructFieldPtr find_field(std::string_view field_name) const; + bool is_generic_struct() const { return genericTs != nullptr; } + bool is_instantiation_of_generic_struct() const { return substitutedTs != nullptr; } + bool was_visited_by_resolver() const { return flags & flagVisitedByResolver; } StructData* mutate() const { return const_cast(this); } void assign_visited_by_resolver(); - StructData(std::string name, SrcLocation loc, std::vector&& fields) + StructData(std::string name, SrcLocation loc, std::vector&& fields, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, AnyV ast_root) : Symbol(std::move(name), loc) - , fields(std::move(fields)) { + , fields(std::move(fields)) + , genericTs(genericTs) + , substitutedTs(substitutedTs) + , ast_root(ast_root) { } + + std::string as_human_readable() const; }; class GlobalSymbolTable { diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index 7f2101ca4d..39431b4a3e 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -15,8 +15,8 @@ along with TON Blockchain Library. If not, see . */ #include "type-system.h" -#include "lexer.h" #include "platform-utils.h" +#include "generics-helpers.h" #include "compiler-state.h" #include #include @@ -81,6 +81,7 @@ class TypeDataHasherForUnique { class TypeIdCalculation { static int last_type_id; static std::unordered_map map_ptr_to_type_id; + static std::vector instantiated_structs; public: static int assign_type_id(TypePtr self) { @@ -91,6 +92,21 @@ class TypeIdCalculation { return it->second; } + // make `Wrapper>` and `Wrapper>` and `Wrapper` have equal type_id + if (const TypeDataStruct* t_struct = self->try_as(); t_struct && t_struct->struct_ref->is_instantiation_of_generic_struct()) { + StructPtr struct_ref = t_struct->struct_ref; + instantiated_structs.push_back(struct_ref); + for (StructPtr another_ref : instantiated_structs) { + if (struct_ref != another_ref && self->equal_to(TypeDataStruct::create(another_ref))) { + auto it = map_ptr_to_type_id.find(TypeDataStruct::create(another_ref)); + assert(it != map_ptr_to_type_id.end()); + int type_id = it->second; + map_ptr_to_type_id[self] = type_id; + return type_id; + } + } + } + int type_id = ++last_type_id; map_ptr_to_type_id[self] = type_id; return type_id; @@ -110,6 +126,7 @@ class TypeIdCalculation { int TypeIdCalculation::last_type_id = 128; // below 128 reserved for built-in types std::unordered_map TypeDataHasherForUnique::all_unique_occurred_types; std::unordered_map TypeIdCalculation::map_ptr_to_type_id; +std::vector TypeIdCalculation::instantiated_structs; TypePtr TypeDataInt::singleton; TypePtr TypeDataBool::singleton; @@ -156,6 +173,12 @@ bool TypeData::equal_to_slow_path(TypePtr lhs, TypePtr rhs) { return lhs_union->variants.size() == rhs_union->variants.size() && lhs_union->has_all_variants_of(rhs_union); } } + if (const TypeDataStruct* lhs_struct = lhs->try_as(); lhs_struct && lhs_struct->struct_ref->is_instantiation_of_generic_struct()) { + if (const TypeDataStruct* rhs_struct = rhs->try_as(); rhs_struct && rhs_struct->struct_ref->is_instantiation_of_generic_struct()) { + return lhs_struct->struct_ref->base_struct_ref == rhs_struct->struct_ref->base_struct_ref + && lhs_struct->struct_ref->substitutedTs->equal_to(rhs_struct->struct_ref->substitutedTs); + } + } return false; } @@ -218,6 +241,22 @@ TypePtr TypeDataGenericT::create(std::string&& nameT) { return hash.register_unique(new TypeDataGenericT(std::move(nameT))); } +TypePtr TypeDataGenericTypeWithTs::create(StructPtr struct_ref, AliasDefPtr alias_ref, std::vector&& type_arguments) { + TypeDataHasherForUnique hash(7451094818554079348ULL); // use it only to calculate children_flags + if (struct_ref) { + assert(alias_ref == nullptr && struct_ref->is_generic_struct()); + hash.feed_string(struct_ref->name); + } else { + assert(struct_ref == nullptr && alias_ref->is_generic_alias()); + hash.feed_string(alias_ref->name); + } + for (TypePtr argT : type_arguments) { + hash.feed_child(argT); + } + + return new TypeDataGenericTypeWithTs(hash.children_flags(), struct_ref, alias_ref, std::move(type_arguments)); +} + TypePtr TypeDataStruct::create(StructPtr struct_ref) { TypeDataHasherForUnique hash(8315986401043319583ULL); hash.feed_string(struct_ref->name); @@ -225,7 +264,7 @@ TypePtr TypeDataStruct::create(StructPtr struct_ref) { if (TypePtr existing = hash.get_existing()) { return existing; } - int width_on_stack = 0; + int width_on_stack = 0; // if a struct is generic, it will be incorrect, it's okay for (StructFieldPtr field_ref : struct_ref->fields) { width_on_stack += field_ref->declared_type->get_width_on_stack(); } @@ -377,6 +416,7 @@ TypePtr TypeDataUnion::create_nullable(TypePtr nullable) { // int TypeDataAlias::get_type_id() const { + assert(!alias_ref->is_generic_alias()); return underlying_type->get_type_id(); } @@ -389,7 +429,13 @@ int TypeDataGenericT::get_type_id() const { throw Fatal("unexpected get_type_id() call"); } +int TypeDataGenericTypeWithTs::get_type_id() const { + assert(false); // `Wrapper` has to be resolved in advance + throw Fatal("unexpected get_type_id() call"); +} + int TypeDataStruct::get_type_id() const { + assert(!struct_ref->is_generic_struct()); return TypeIdCalculation::assign_type_id(this); } @@ -454,6 +500,19 @@ std::string TypeDataFunCallable::as_human_readable() const { return result; } +std::string TypeDataGenericTypeWithTs::as_human_readable() const { + std::string result = struct_ref ? struct_ref->name : alias_ref->name; + result += '<'; + for (TypePtr argT : type_arguments) { + if (result[result.size() - 1] != '<') { + result += ", "; + } + result += argT->as_human_readable(); + } + result += '>'; + return result; +} + std::string TypeDataStruct::as_human_readable() const { return struct_ref->name; } @@ -536,6 +595,15 @@ TypePtr TypeDataFunCallable::replace_children_custom(const ReplacerCallbackT& ca return callback(create(std::move(mapped), return_type->replace_children_custom(callback))); } +TypePtr TypeDataGenericTypeWithTs::replace_children_custom(const ReplacerCallbackT& callback) const { + std::vector mapped; + mapped.reserve(type_arguments.size()); + for (TypePtr argT : type_arguments) { + mapped.push_back(argT->replace_children_custom(callback)); + } + return callback(create(struct_ref, alias_ref, std::move(mapped))); +} + TypePtr TypeDataTensor::replace_children_custom(const ReplacerCallbackT& callback) const { std::vector mapped; mapped.reserve(items.size()); @@ -692,7 +760,10 @@ bool TypeDataFunCallable::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataGenericT::can_rhs_be_assigned(TypePtr rhs) const { - assert(false); + return false; +} + +bool TypeDataGenericTypeWithTs::can_rhs_be_assigned(TypePtr rhs) const { return false; } @@ -703,6 +774,9 @@ bool TypeDataStruct::can_rhs_be_assigned(TypePtr rhs) const { if (const TypeDataAlias* rhs_alias = rhs->try_as()) { return can_rhs_be_assigned(rhs_alias->underlying_type); } + if (const TypeDataStruct* rhs_struct = rhs->try_as()) { // C> = C + return equal_to(rhs_struct); + } return rhs == TypeDataNever::create(); } @@ -956,6 +1030,10 @@ bool TypeDataGenericT::can_be_casted_with_as_operator(TypePtr cast_to) const { return true; } +bool TypeDataGenericTypeWithTs::can_be_casted_with_as_operator(TypePtr cast_to) const { + return true; +} + bool TypeDataStruct::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataUnion* to_union = cast_to->try_as()) { return can_be_casted_to_union(this, to_union); @@ -963,6 +1041,9 @@ bool TypeDataStruct::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } + if (const TypeDataStruct* to_struct = cast_to->try_as()) { // C> as C + return equal_to(to_struct); + } return cast_to == this; } diff --git a/tolk/type-system.h b/tolk/type-system.h index cd0c34581b..40b2910154 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -103,6 +103,8 @@ class TypeData { * It never occurs at runtime: at IR generation it's erased, replaced by an underlying type. * But until IR generation, aliases exists, and `var t: MyTensor2 = (1,2)` is alias "MyTensor", not tensor (int,int). * That's why lots of code comparing types use `type->unwrap_alias()` or `try_as`. + * Note, that generic aliases, when instantiated, are inserted into in symtable (like structs and functions), + * so for `WrapperAlias` alias_ref points to a generic alias, and for `WrapperAlias` to an instantiated one. */ class TypeDataAlias final : public TypeData { explicit TypeDataAlias(int children_flags, AliasDefPtr alias_ref, TypePtr underlying_type) @@ -301,8 +303,9 @@ class TypeDataFunCallable final : public TypeData { }; /* - * `T` inside generic functions is TypeDataGenericT. + * `T` inside generic functions and structs is TypeDataGenericT. * Example: `fun f(a: X, b: Y): [X, Y]` (here X and Y are). + * Example: `struct Wrapper { value: T }` (type of field is generic T). * On instantiation like `f(1,"")`, a new function `f` is created with type `fun(int,slice)->[int,slice]`. */ class TypeDataGenericT final : public TypeData { @@ -322,9 +325,40 @@ class TypeDataGenericT final : public TypeData { }; /* - * `A`, `User`, `SomeStruct` is TypeDataStruct. At TVM level, structs are tensors. + * `Wrapper` when T is a generic (a struct is not ready to instantiate) is TypeDataGenericTypeWithTs. + * `Wrapper` is NOT here, it's an instantiated struct. Here is only when type arguments contain generics. + * Example: `type WrapperAlias = Wrapper`, then `Wrapper` (underlying type of alias) is here. + * Since structs and type aliases both can be generic, either struct_ref of alias_ref is filled. + */ +class TypeDataGenericTypeWithTs final : public TypeData { + TypeDataGenericTypeWithTs(int children_flags, StructPtr struct_ref, AliasDefPtr alias_ref, std::vector&& type_arguments) + : TypeData(children_flags, -999999) + , struct_ref(struct_ref) + , alias_ref(alias_ref) + , type_arguments(std::move(type_arguments)) {} + +public: + const StructPtr struct_ref; // for `Wrapper`, then alias_ref = nullptr + const AliasDefPtr alias_ref; // for `PairAlias`, then struct_ref = nullptr + const std::vector type_arguments; // ``, ``, at least one of them contains generic T + + static TypePtr create(StructPtr struct_ref, AliasDefPtr alias_ref, std::vector&& type_arguments); + + int size() const { return static_cast(type_arguments.size()); } + + int get_type_id() const override; + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; +}; + +/* + * `A`, `User`, `SomeStruct`, `Wrapper` is TypeDataStruct. At TVM level, structs are tensors. * In the code, creating a struct is either `var v: A = { ... }` (by hint) or `var v = A { ... }`. * Fields of a struct have their own types (accessed by struct_ref). + * Note, that instantiated structs like "Wrapper" exist in symtable (like aliases and functions), + * so for `Wrapper` struct_ref points to a generic struct, and for `Wrapper` to an instantiated one. */ class TypeDataStruct final : public TypeData { TypeDataStruct(int width_on_stack, StructPtr struct_ref) @@ -487,6 +521,8 @@ class TypeDataUnion final : public TypeData { static TypePtr create(std::vector&& variants); static TypePtr create_nullable(TypePtr nullable); + int size() const { return static_cast(variants.size()); } + // "primitive nullable" is `T?` which holds TVM NULL in the same slot (it other words, has no UTag slot) // true : `int?`, `slice?`, `StructWith1IntField?` // false: `(int, int)?`, `ComplexStruct?`, `()?` From 720a173d0e8afdd8f919de265998d7394dad87b0 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 28 Apr 2025 20:35:22 +0400 Subject: [PATCH 233/524] [Tolk] Output original .tolk lines into Fift as comments This update greatly enhances reading Fift output: 1) stack comments are aligned 2) original lines from .tolk files are output as comments, grouping asm instructions Internally, every AsmOp now has SrcLocation. When outputting them one by one, an original .tolk line is inserted if locations differ. This can be optimized later by storing an index to fast mapping of location (offset) into a line in a file. --- .../tests/allow-post-modification.tolk | 8 +- tolk-tester/tests/assignment-tests.tolk | 19 +- tolk-tester/tests/bit-operators.tolk | 35 +- tolk-tester/tests/cells-slices.tolk | 1 - tolk-tester/tests/codegen-check-demo.tolk | 4 +- tolk-tester/tests/comments-tests.tolk | 42 ++ tolk-tester/tests/generics-2.tolk | 1 - tolk-tester/tests/generics-3.tolk | 1 - tolk-tester/tests/if-else-tests.tolk | 14 +- tolk-tester/tests/indexed-access.tolk | 8 +- tolk-tester/tests/intN-tests.tolk | 1 - .../tests/invalid-declaration/err-1918.tolk | 2 +- tolk-tester/tests/logical-operators.tolk | 19 +- tolk-tester/tests/match-by-expr-tests.tolk | 10 +- tolk-tester/tests/mutate-methods.tolk | 8 +- tolk-tester/tests/no-spaces.tolk | 3 - tolk-tester/tests/null-keyword.tolk | 3 +- tolk-tester/tests/nullable-tensors.tolk | 6 +- tolk-tester/tests/op-priority.tolk | 21 +- tolk-tester/tests/self-keyword.tolk | 19 +- tolk-tester/tests/smart-cast-tests.tolk | 1 - tolk-tester/tests/some-tests-2.tolk | 3 - tolk-tester/tests/struct-tests.tolk | 8 +- tolk-tester/tests/try-catch-tests.tolk | 9 +- tolk-tester/tests/type-aliases-tests.tolk | 7 +- tolk-tester/tests/union-types-tests.tolk | 9 +- tolk-tester/tests/use-before-declare.tolk | 2 - tolk-tester/tolk-tester.js | 11 +- tolk-tester/tolk-tester.py | 5 + tolk/abscode.cpp | 3 - tolk/analyzer.cpp | 20 +- tolk/asmops.cpp | 209 +++++---- tolk/ast-aux-data.h | 38 ++ tolk/ast-from-tokens.cpp | 2 +- tolk/ast-replacer.h | 2 + tolk/ast-replicator.h | 4 + tolk/ast-stringifier.h | 2 + tolk/ast-visitor.h | 2 + tolk/ast.h | 20 + tolk/builtins.cpp | 416 +++++++++--------- tolk/codegen.cpp | 277 ++++++------ tolk/compiler-state.h | 1 + tolk/optimize.cpp | 72 ++- tolk/pipe-ast-to-legacy.cpp | 31 +- tolk/pipe-generate-fif-output.cpp | 24 +- tolk/pipe-resolve-identifiers.cpp | 2 +- tolk/src-file.cpp | 55 ++- tolk/src-file.h | 4 +- tolk/tolk-main.cpp | 6 +- tolk/tolk-wasm.cpp | 2 + tolk/tolk.h | 305 ++++++------- 51 files changed, 944 insertions(+), 833 deletions(-) create mode 100644 tolk-tester/tests/comments-tests.tolk create mode 100644 tolk/ast-aux-data.h diff --git a/tolk-tester/tests/allow-post-modification.tolk b/tolk-tester/tests/allow-post-modification.tolk index 10c2437407..49a2eaec58 100644 --- a/tolk-tester/tests/allow-post-modification.tolk +++ b/tolk-tester/tests/allow-post-modification.tolk @@ -146,16 +146,14 @@ fun main() { @fif_codegen """ - ~inc PROC:<{ - // self y - inc CALLDICT // self newY + ~inc PROC:<{ // self y + inc CALLDICT // self newY }> """ @fif_codegen """ - test_assign_tensor_global PROC:<{ - // x.0 x.1 + test_assign_tensor_global PROC:<{ // x.0 x.1 """ @code_hash 6737917279814799680932710799951154408447028229503449454536845752635763933556 diff --git a/tolk-tester/tests/assignment-tests.tolk b/tolk-tester/tests/assignment-tests.tolk index f4d2c0d78e..2301c9b70c 100644 --- a/tolk-tester/tests/assignment-tests.tolk +++ b/tolk-tester/tests/assignment-tests.tolk @@ -245,16 +245,15 @@ fun main(value: int, ) { @fif_codegen """ - test116 PROC:<{ - // - 1 PUSHINT // '10=1 - 2 PUSHINT // '10=1 '11=2 - 3 PUSHINT // '10=1 '11=2 '12=3 - 4 PUSHINT // '10=1 '11=2 '12=3 '13=4 - 4 TUPLE // rhs - DUP // rhs rhs - 4 UNTUPLE // rhs2 a b c d - 4 ROLL // a b c d rhs2 + test116 PROC:<{ // + 1 PUSHINT // '10=1 + 2 PUSHINT // '10=1 '11=2 + 3 PUSHINT // '10=1 '11=2 '12=3 + 4 PUSHINT // '10=1 '11=2 '12=3 '13=4 + 4 TUPLE // rhs + DUP // rhs rhs + 4 UNTUPLE // rhs2 a b c d + 4 ROLL // a b c d rhs2 }> """ */ diff --git a/tolk-tester/tests/bit-operators.tolk b/tolk-tester/tests/bit-operators.tolk index b010688390..9ae04090c2 100644 --- a/tolk-tester/tests/bit-operators.tolk +++ b/tolk-tester/tests/bit-operators.tolk @@ -126,18 +126,16 @@ fun testBoolCompareOptimized(x: bool) { @fif_codegen """ boolWithBitwiseConst PROC:<{ - // - 0 PUSHINT // '3 - -1 PUSHINT // '3 '5 - 0 PUSHINT // '3 '5 '7 - -1 PUSHINT // '3 '5 '7 '8 + 0 PUSHINT // '3 + -1 PUSHINT // '3 '5 + 0 PUSHINT // '3 '5 '7 + -1 PUSHINT // '3 '5 '7 '8 }> """ @fif_codegen """ - testDoUntilCodegen PROC:<{ - // i n + testDoUntilCodegen PROC:<{ // i n 0 PUSHINT // i n cnt=0 UNTIL:<{ INC // i n cnt @@ -164,7 +162,6 @@ fun testBoolCompareOptimized(x: bool) { @fif_codegen """ testConstNegateCodegen PROC:<{ - // TRUE // '0 FALSE // '0 '1 FALSE // '0 '1 '2 @@ -176,26 +173,24 @@ fun testBoolCompareOptimized(x: bool) { @fif_codegen """ - testBoolNegateOptimized PROC:<{ - // x - DUP // x x - NOT // x '1 - OVER // x '1 x - NOT // x '1 '2 + testBoolNegateOptimized PROC:<{ // x + DUP // x x + NOT // x '1 + OVER // x '1 x + NOT // x '1 '2 s2 s(-1) PUXC - TRUE // x '1 x '2 '3 + TRUE // x '1 x '2 '3 }> """ @fif_codegen """ - testBoolCompareOptimized PROC:<{ - // x - DUP // x x - NOT // x '1 + testBoolCompareOptimized PROC:<{ // x + DUP // x x + NOT // x '1 OVER // x '1 x eqX CALLDICT // x '1 '2 - NOT // x '1 '3 + NOT // x '1 '3 s2 PUSH // x '1 '3 x eqX CALLDICT // x '1 '3 '4 s3 PUSH // x '1 '3 '4 x diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 772812eb10..e900268bba 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -216,7 +216,6 @@ Note, that since 'compute-asm-ltr' became on be default, chaining methods codege @fif_codegen """ test6 PROC:<{ - // NEWC // '0 1 PUSHINT // '0 '1=1 SWAP // '1=1 '0 diff --git a/tolk-tester/tests/codegen-check-demo.tolk b/tolk-tester/tests/codegen-check-demo.tolk index 5b46c09357..ff2ca3c2cc 100644 --- a/tolk-tester/tests/codegen-check-demo.tolk +++ b/tolk-tester/tests/codegen-check-demo.tolk @@ -33,8 +33,7 @@ Below, I just give examples of @fif_codegen tag: @fif_codegen """ -main PROC:<{ - // s +main PROC:<{ // s 17 PUSHINT // s '3=17 OVER // s z=17 t WHILE:<{ @@ -78,7 +77,6 @@ main PROC:<{ @fif_codegen """ test1 PROC:<{ -// FALSE }> """ diff --git a/tolk-tester/tests/comments-tests.tolk b/tolk-tester/tests/comments-tests.tolk new file mode 100644 index 0000000000..3d00a0b1a8 --- /dev/null +++ b/tolk-tester/tests/comments-tests.tolk @@ -0,0 +1,42 @@ + +struct WithDef { + f1: int, + f2: int = 10, + f3: int, + f4: int, +} + +const C_20 = 20; + +fun demo_fields_def(x: int) { + var w: WithDef = { + f1: x + C_20, + f3: 100, + f4: C_20 + }; + return (w.f4 + 5, w.f3, w.f2, w.f1); +} + +fun main() { + return 1; +} + +/** +@testcase | 0 | | 1 + +@fif_codegen_enable_comments +@fif_codegen +""" + demo_fields_def PROC:<{ // x + // 13: f1: x + C_20 + 20 ADDCONST // '6 + 10 PUSHINT // '6 '7=10 + // 14: f3: 100 + 100 PUSHINT // w.f1 w.f2=10 w.f3=100 + // 17: return (w.f4 + 5, w.f3, w.f2, w.f1) + 25 PUSHINT // w.f1 w.f2=10 w.f3=100 '11 + s2 s3 XCHG2 // '11 w.f3=100 w.f2=10 w.f1 + }> +""" + +*/ \ No newline at end of file diff --git a/tolk-tester/tests/generics-2.tolk b/tolk-tester/tests/generics-2.tolk index cde9d99297..b3e0ea5b9d 100644 --- a/tolk-tester/tests/generics-2.tolk +++ b/tolk-tester/tests/generics-2.tolk @@ -453,7 +453,6 @@ fun main(c: Wrapper, d: WrappedInt) { @fif_codegen """ test1 PROC:<{ - // 23 PUSHINT // c1=23 DUP // c1=23 c2=23 }> diff --git a/tolk-tester/tests/generics-3.tolk b/tolk-tester/tests/generics-3.tolk index 4ec7b29b40..77d0461134 100644 --- a/tolk-tester/tests/generics-3.tolk +++ b/tolk-tester/tests/generics-3.tolk @@ -137,7 +137,6 @@ fun main() { @fif_codegen """ test3 PROC:<{ - // 10 PUSHINT // r.USlot1=10 -1 PUSHINT // r.USlot1=10 '11=-1 FALSE // r.USlot1=10 '11=-1 '13 diff --git a/tolk-tester/tests/if-else-tests.tolk b/tolk-tester/tests/if-else-tests.tolk index f40336a9ec..8c314b8edf 100644 --- a/tolk-tester/tests/if-else-tests.tolk +++ b/tolk-tester/tests/if-else-tests.tolk @@ -128,15 +128,14 @@ fun main() { @fif_codegen """ - test3 PROC:<{ - // x - DUP // x x + test3 PROC:<{ // x + DUP // x x 20 NEQINT // x '2 IFNOTJMP:<{ // x - DROP // + DROP // 20 PUSHINT // '3=20 - }> // x - DUP // x x + }> // x + DUP // x x 50 EQINT // x '5 IFNOTJMP:<{ // x """ @@ -144,7 +143,6 @@ fun main() { @fif_codegen """ withNoElse PROC:<{ - // op DUP 123 EQINT IFJMP:<{ @@ -170,7 +168,6 @@ fun main() { @fif_codegen """ withElse PROC:<{ - // op DUP 123 EQINT IFJMP:<{ @@ -196,7 +193,6 @@ fun main() { @fif_codegen """ withMatch PROC:<{ - // op DUP 123 EQINT IFJMP:<{ diff --git a/tolk-tester/tests/indexed-access.tolk b/tolk-tester/tests/indexed-access.tolk index 4b3a0a4b26..f6a376ad3d 100644 --- a/tolk-tester/tests/indexed-access.tolk +++ b/tolk-tester/tests/indexed-access.tolk @@ -286,7 +286,6 @@ fun main(){} @fif_codegen """ testCodegenSimple PROC:<{ - // 1 PUSHINT // '2=1 SINGLE // t1 2 PUSHINT // t1 '3=2 @@ -336,15 +335,13 @@ fun main(){} @fif_codegen """ testCodegenNoPureIndexedAccess PROC:<{ - // 0 PUSHINT // '8=0 }> """ @fif_codegen """ - testCodegenIndexPostfix1 PROC:<{ - // x.0 x.1 + testCodegenIndexPostfix1 PROC:<{ // x.0 x.1 // ab.1 ab.0 SWAP // ab.0 ab.1 }> @@ -352,8 +349,7 @@ fun main(){} @fif_codegen """ - testCodegenIndexPostfix2 PROC:<{ - // x.0 x.1.0 x.1.1 x.2 + testCodegenIndexPostfix2 PROC:<{ // x.0 x.1.0 x.1.1 x.2 s2 POP // y.0 y.2 y.1.1 s1 s2 XCHG // y.2 y.0 y.1.1 }> diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index d12436f5e0..4b2fccb25c 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -254,7 +254,6 @@ fun main() { @fif_codegen """ test12 PROC:<{ - // 100000000 PUSHINT // '4=100000000 }> """ diff --git a/tolk-tester/tests/invalid-declaration/err-1918.tolk b/tolk-tester/tests/invalid-declaration/err-1918.tolk index ae801de265..8fec98925e 100644 --- a/tolk-tester/tests/invalid-declaration/err-1918.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1918.tolk @@ -8,5 +8,5 @@ fun hello(): int { @compilation_should_fail @stderr fun hello() @stderr redefinition of symbol, previous was at -@stderr err-1918.tolk:1:1 +@stderr err-1918.tolk:1 */ diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk index 7e3168c7c9..2fd62db5b1 100644 --- a/tolk-tester/tests/logical-operators.tolk +++ b/tolk-tester/tests/logical-operators.tolk @@ -218,7 +218,6 @@ fun main() { @fif_codegen """ simpleAllConst PROC:<{ - // TRUE 0 PUSHINT TRUE @@ -233,8 +232,7 @@ fun main() { @fif_codegen """ - compileTimeEval1 PROC:<{ - // x + compileTimeEval1 PROC:<{ // x DUP // x x 0 EQINT // x '1 FALSE // x '1 '4 @@ -250,9 +248,9 @@ fun main() { @fif_codegen """ - withIfNot PROC:<{ + withIfNot PROC:<{ // x y c2 SAVE - SAMEALTSAVE // x y + SAMEALTSAVE OVER // x y x IFNOTJMP:<{ // x y 2DROP // @@ -271,7 +269,6 @@ fun main() { @fif_codegen """ testAndConstCodegen PROC:<{ - // FALSE 0 PUSHINT DUP @@ -294,7 +291,6 @@ fun main() { @fif_codegen """ testOrConstCodegen PROC:<{ - // -1 PUSHINT TRUE FALSE @@ -319,8 +315,7 @@ For example, `a && b` can be expressed without IFs. These are moments of future optimizations. For now, it's more than enough. @fif_codegen """ - testAndSimpleCodegen PROC:<{ - // a b + testAndSimpleCodegen PROC:<{ // a b SWAP // b a IF:<{ // b 0 NEQINT // '2 @@ -333,8 +328,7 @@ These are moments of future optimizations. For now, it's more than enough. @fif_codegen """ - testOrSimpleCodegen PROC:<{ - // a b + testOrSimpleCodegen PROC:<{ // a b SWAP // b a 0 GTINT // b '3 IF:<{ // b @@ -349,8 +343,7 @@ These are moments of future optimizations. For now, it's more than enough. @fif_codegen """ - testConvertIfToIfnot PROC:<{ - // x + testConvertIfToIfnot PROC:<{ // x DUP // x x 100 THROWIF DUP // x x diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk index 7941b46001..5599193532 100644 --- a/tolk-tester/tests/match-by-expr-tests.tolk +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -157,8 +157,7 @@ fun main() { @fif_codegen """ - test1 PROC:<{ - // x + test1 PROC:<{ // x DUP // x x 1 EQINT // x '3 IF:<{ // x @@ -185,15 +184,13 @@ fun main() { @fif_codegen """ test2 PROC:<{ - // 300 PUSHINT }> """ @fif_codegen """ - test3 PROC:<{ - // init + test3 PROC:<{ // init DUP // init init isGt10 CALLDICT // init '2 -1 EQINT // init '5 @@ -206,8 +203,7 @@ fun main() { @fif_codegen """ - test8 PROC:<{ - // x + test8 PROC:<{ // x DROP // 30 PUSHINT // r1 }> diff --git a/tolk-tester/tests/mutate-methods.tolk b/tolk-tester/tests/mutate-methods.tolk index 9ebf8b1d46..59da684843 100644 --- a/tolk-tester/tests/mutate-methods.tolk +++ b/tolk-tester/tests/mutate-methods.tolk @@ -295,8 +295,7 @@ fun main(){} @fif_codegen """ - incrementInPlace PROC:<{ - // self byValue + incrementInPlace PROC:<{ // self byValue ADD // self }> """ @@ -315,8 +314,7 @@ fun main(){} @fif_codegen """ - load_next PROC:<{ - // cs + load_next PROC:<{ // cs 32 LDI // '4 cs SWAP // cs '4 }> @@ -325,7 +323,6 @@ fun main(){} @fif_codegen """ testStoreUintPureUnusedResult PROC:<{ - // 0 PUSHINT // '11=0 }> """ @@ -333,7 +330,6 @@ fun main(){} @fif_codegen """ testStoreUintImpureUnusedResult PROC:<{ - // NEWC // b STIX // '2 DROP // diff --git a/tolk-tester/tests/no-spaces.tolk b/tolk-tester/tests/no-spaces.tolk index 3fe2573a94..33959917e4 100644 --- a/tolk-tester/tests/no-spaces.tolk +++ b/tolk-tester/tests/no-spaces.tolk @@ -80,7 +80,6 @@ fun`main`(){} @fif_codegen """ get_+-+1 PROC:<{ - // -1 PUSHINT }> """ @@ -88,7 +87,6 @@ fun`main`(){} @fif_codegen """ unary+bitwise-constant PROC:<{ - // -4 PUSHINT 6 PUSHINT -4 PUSHINT @@ -99,7 +97,6 @@ fun`main`(){} @fif_codegen """ unary_const_check PROC:<{ - // -1 PUSHINT // fst1=-1 DUP // fst1=-1 snd1=-1 2 PUSHINT // fst1=-1 snd1=-1 trd1=2 diff --git a/tolk-tester/tests/null-keyword.tolk b/tolk-tester/tests/null-keyword.tolk index 65890a92c2..db54973e92 100644 --- a/tolk-tester/tests/null-keyword.tolk +++ b/tolk-tester/tests/null-keyword.tolk @@ -89,10 +89,10 @@ fun main() { @testcase | 103 | 15 | -1 @testcase | 104 | | (null) @testcase | 107 | | -11 + @fif_codegen """ test1 PROC:<{ - // PUSHNULL // numbers 1 PUSHINT // numbers '2=1 SWAP // '2=1 numbers @@ -114,7 +114,6 @@ fun main() { @fif_codegen """ main PROC:<{ - // 1 PUSHINT // '3=1 }> """ diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index 41b3d62b86..af051ce939 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -530,9 +530,8 @@ fun main(){} @fif_codegen """ - isTensorNull PROC:<{ - // t.0 t.1 t.UTag - 2 1 BLKDROP2 // t.UTag + isTensorNull PROC:<{ // t.0 t.1 t.UTag + 2 1 BLKDROP2 // t.UTag 0 EQINT // '3 }> """ @@ -540,7 +539,6 @@ fun main(){} @fif_codegen """ test113 PROC:<{ - // 1 PUSHINT // '2=1 PUSHNULL // '2=1 '3 PAIR // t diff --git a/tolk-tester/tests/op-priority.tolk b/tolk-tester/tests/op-priority.tolk index e3c6bb6e87..3a1dee21e0 100644 --- a/tolk-tester/tests/op-priority.tolk +++ b/tolk-tester/tests/op-priority.tolk @@ -92,28 +92,25 @@ fun main() { @fif_codegen """ - unary_minus_1 PROC:<{ - // a b c + unary_minus_1 PROC:<{ // a b c -ROT // c a b - ADD // c '3 + ADD // c '3 NEGATE // c '4 SWAP // '4 c - MUL // '5 + MUL // '5 }> - unary_minus_2 PROC:<{ - // a b c + unary_minus_2 PROC:<{ // a b c -ROT // c a b - ADD // c '3 + ADD // c '3 NEGATE // c '4 SWAP // '4 c - MUL // '5 + MUL // '5 }> - unary_minus_3 PROC:<{ - // a b c + unary_minus_3 PROC:<{ // a b c -ROT // c a b - ADD // c '3 + ADD // c '3 SWAP // '3 c - MUL // '4 + MUL // '4 NEGATE // '5 }> """ diff --git a/tolk-tester/tests/self-keyword.tolk b/tolk-tester/tests/self-keyword.tolk index b0567696de..37f3cb1784 100644 --- a/tolk-tester/tests/self-keyword.tolk +++ b/tolk-tester/tests/self-keyword.tolk @@ -222,29 +222,24 @@ fun main() { } @fif_codegen """ - incChained PROC:<{ - // self - INC // self + incChained PROC:<{ // self + INC // self }> - incChained2 PROC:<{ - // self + incChained2 PROC:<{ // self incChained CALLDICT // self }> - incChained3 PROC:<{ - // self + incChained3 PROC:<{ // self incChained CALLDICT // self }> - incChained4 PROC:<{ - // self + incChained4 PROC:<{ // self incChained CALLDICT // self }> """ @fif_codegen """ - testIncChainedCodegen PROC:<{ - // x - incChained CALLDICT // x + testIncChainedCodegen PROC:<{ // x + incChained CALLDICT // x incChained2 CALLDICT // x incChained3 CALLDICT // x incChained4 CALLDICT // x diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index aac1028f3d..24baa34f1a 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -748,7 +748,6 @@ fun main(x: int?): int { @fif_codegen """ test60 PROC:<{ - // 101 PUSHINT // cc1 109 PUSHINT // cc1 cc2 }> diff --git a/tolk-tester/tests/some-tests-2.tolk b/tolk-tester/tests/some-tests-2.tolk index 9d24f38da2..4e46f26335 100644 --- a/tolk-tester/tests/some-tests-2.tolk +++ b/tolk-tester/tests/some-tests-2.tolk @@ -106,7 +106,6 @@ fun test95() { @fif_codegen """ touchCodegen2 PROC:<{ - // get10 CALLDICT // f }> """ @@ -125,7 +124,6 @@ fun test95() { @fif_codegen """ testStartBalanceCodegen1 PROC:<{ - // BALANCE // t FIRST // first }> @@ -134,7 +132,6 @@ fun test95() { @fif_codegen """ testStartBalanceCodegen2 PROC:<{ - // BALANCE FIRST // first }> diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 8dd2f3455a..81f3b9f298 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -517,7 +517,6 @@ type PointAlias = Point; @fif_codegen """ test6 PROC:<{ - // 10 PUSHINT // p.x 20 PUSHINT // p.x p.y }> @@ -525,8 +524,7 @@ type PointAlias = Point; @fif_codegen """ - sumXY PROC:<{ - // point.x point.y + sumXY PROC:<{ // point.x point.y ADD // '2 }> """ @@ -534,8 +532,7 @@ type PointAlias = Point; @fif_codegen """ - test23 PROC:<{ - // p.x p.y + test23 PROC:<{ // p.x p.y 2DROP 20 PUSHINT // '10=20 }> @@ -544,7 +541,6 @@ type PointAlias = Point; @fif_codegen """ test29 PROC:<{ - // PUSHNULL // '5 }> """ diff --git a/tolk-tester/tests/try-catch-tests.tolk b/tolk-tester/tests/try-catch-tests.tolk index 4ac86d9661..b0a50aacb3 100644 --- a/tolk-tester/tests/try-catch-tests.tolk +++ b/tolk-tester/tests/try-catch-tests.tolk @@ -269,8 +269,7 @@ fun main() { @fif_codegen """ - testCodegen1 PROC:<{ - // x + testCodegen1 PROC:<{ // x DUP // x x 10 GTINT // x '2 IFJMP:<{ // x @@ -288,8 +287,7 @@ fun main() { @fif_codegen """ - testCodegen2 PROC:<{ - // x + testCodegen2 PROC:<{ // x DUP // x x 10 GTINT // x '2 IFJMP:<{ // x @@ -307,8 +305,7 @@ fun main() { @fif_codegen """ - testCodegen3 PROC:<{ - // numberId paramVal + testCodegen3 PROC:<{ // numberId paramVal SWAP -1000 PUSHINT // paramVal numberId '2=-1000 EQUAL // paramVal '3 diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk index 68935733cd..d8ab0782cb 100644 --- a/tolk-tester/tests/type-aliases-tests.tolk +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -184,12 +184,11 @@ fun main(x: MInt, y: MInt?) { @fif_codegen """ - test9 PROC:<{ - // b - DUP // b b + test9 PROC:<{ // b + DUP // b b IFNOTJMP:<{ // b NOT // '2 - }> // b + }> // b DROP // TRUE // '5 }> diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 0bc88c3660..11dae3f8cd 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -854,17 +854,15 @@ fun main() { @fif_codegen """ test26 PROC:<{ - // 42 PUSHINT // result }> """ @fif_codegen """ - checkSimple PROC:<{ - // a.USlot1 a.UTag - NIP // a.UTag - DUP // a.UTag a.UTag + checkSimple PROC:<{ // a.USlot1 a.UTag + NIP // a.UTag + DUP // a.UTag a.UTag 1 EQINT // a.UTag '3 IF:<{ // a.UTag DROP // @@ -883,7 +881,6 @@ fun main() { @fif_codegen """ test61 PROC:<{ - // i.USlot1 i.UTag NIP DUP 46 EQINT diff --git a/tolk-tester/tests/use-before-declare.tolk b/tolk-tester/tests/use-before-declare.tolk index f444c48ef4..baae0b400d 100644 --- a/tolk-tester/tests/use-before-declare.tolk +++ b/tolk-tester/tests/use-before-declare.tolk @@ -49,7 +49,6 @@ const first = 1; @fif_codegen """ test1 PROC:<{ - // 30 PUSHINT // '10 }> """ @@ -57,7 +56,6 @@ const first = 1; @fif_codegen """ test2 PROC:<{ - // 2 PUSHINT // '2 }> """ diff --git a/tolk-tester/tolk-tester.js b/tolk-tester/tolk-tester.js index 0aaf0738f8..8b70007246 100644 --- a/tolk-tester/tolk-tester.js +++ b/tolk-tester/tolk-tester.js @@ -273,6 +273,8 @@ class TolkTestFile { this.expected_hash = null /** @type {string | null} */ this.experimental_options = null + /** @type {boolean} */ + this.enable_tolk_lines_comments = false } parse_input_from_tolk_file() { @@ -292,6 +294,8 @@ class TolkTestFile { this.stderr_includes.push(new TolkTestCaseStderr(this.parse_string_value(lines), false)) } else if (line.startsWith("@fif_codegen_avoid")) { this.fif_codegen.push(new TolkTestCaseFifCodegen(this.parse_string_value(lines), true)) + } else if (line.startsWith("@fif_codegen_enable_comments")) { + this.enable_tolk_lines_comments = true } else if (line.startsWith("@fif_codegen")) { this.fif_codegen.push(new TolkTestCaseFifCodegen(this.parse_string_value(lines), false)) } else if (line.startsWith("@code_hash")) { @@ -345,9 +349,9 @@ class TolkTestFile { async run_and_check() { const wasmModule = await compileWasm(TOLKFIFTLIB_MODULE, TOLKFIFTLIB_WASM) - let res = compileFile(wasmModule, this.tolk_filename, this.experimental_options) + let res = compileFile(wasmModule, this.tolk_filename, this.experimental_options, this.enable_tolk_lines_comments) let exit_code = res.status === 'ok' ? 0 : 1 - let stderr = res.message + let stderr = res.message || res.stderr let stdout = '' if (exit_code === 0 && this.compilation_should_fail) @@ -488,7 +492,7 @@ function copyFromCString(mod, ptr) { } /** @return {{status: string, message: string, fiftCode: string, codeBoc: string, codeHashHex: string}} */ -function compileFile(mod, filename, experimentalOptions) { +function compileFile(mod, filename, experimentalOptions, withSrcLineComments) { // see tolk-wasm.cpp: typedef void (*WasmFsReadCallback)(int, char const*, char**, char**) const callbackPtr = mod.addFunction((kind, dataPtr, destContents, destError) => { if (kind === 0) { // realpath @@ -520,6 +524,7 @@ function compileFile(mod, filename, experimentalOptions) { const config = { optimizationLevel: 2, withStackComments: true, + withSrcLineComments: withSrcLineComments, experimentalOptions: experimentalOptions || undefined, entrypointFileName: filename }; diff --git a/tolk-tester/tolk-tester.py b/tolk-tester/tolk-tester.py index f8a7a5512d..126b079308 100644 --- a/tolk-tester/tolk-tester.py +++ b/tolk-tester/tolk-tester.py @@ -260,6 +260,7 @@ def __init__(self, tolk_filename: str, artifacts_folder: str): self.fif_codegen: List[TolkTestCaseFifCodegen] = [] self.expected_hash: TolkTestCaseExpectedHash | None = None self.experimental_options: str | None = None + self.enable_tolk_lines_comments = False def parse_input_from_tolk_file(self): with open(self.tolk_filename, "r") as fd: @@ -279,6 +280,8 @@ def parse_input_from_tolk_file(self): self.stderr_includes.append(TolkTestCaseStderr(self.parse_string_value(lines), False)) elif line.startswith("@fif_codegen_avoid"): self.fif_codegen.append(TolkTestCaseFifCodegen(self.parse_string_value(lines), True)) + elif line.startswith("@fif_codegen_enable_comments"): + self.enable_tolk_lines_comments = True elif line.startswith("@fif_codegen"): self.fif_codegen.append(TolkTestCaseFifCodegen(self.parse_string_value(lines), False)) elif line.startswith("@code_hash"): @@ -326,6 +329,8 @@ def run_and_check(self): cmd_args = [TOLK_EXECUTABLE, "-o", self.get_compiled_fif_filename()] if self.experimental_options: cmd_args = cmd_args + ["-x", self.experimental_options] + if not self.enable_tolk_lines_comments: + cmd_args = cmd_args + ["-L"] res = subprocess.run(cmd_args + [self.tolk_filename], capture_output=True, timeout=10) exit_code = res.returncode stderr = str(res.stderr, "utf-8") diff --git a/tolk/abscode.cpp b/tolk/abscode.cpp index 984d6ffc88..02215d8847 100644 --- a/tolk/abscode.cpp +++ b/tolk/abscode.cpp @@ -202,9 +202,6 @@ void Op::show(std::ostream& os, const std::vector& vars, std::string pfx dis += " "; } switch (cl) { - case _Undef: - os << pfx << dis << "???\n"; - break; case _Nop: os << pfx << dis << "NOP\n"; break; diff --git a/tolk/analyzer.cpp b/tolk/analyzer.cpp index 935c8b02c7..480165a169 100644 --- a/tolk/analyzer.cpp +++ b/tolk/analyzer.cpp @@ -501,7 +501,7 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) { } default: std::cerr << "fatal: unknown operation in compute_used_vars()\n"; - throw ParseError{where, "unknown operation"}; + throw ParseError(loc, "unknown operation"); } } @@ -583,7 +583,7 @@ bool prune_unreachable(std::unique_ptr& ops) { op.cl = Op::_If; std::unique_ptr new_op = std::move(op.block0); op.block0 = std::move(op.block1); - op.block1 = std::make_unique(op.next->where, Op::_Nop); + op.block1 = std::make_unique(op.next->loc, Op::_Nop); new_op->last().next = std::move(ops); ops = std::move(new_op); } @@ -629,7 +629,7 @@ bool prune_unreachable(std::unique_ptr& ops) { } default: std::cerr << "fatal: unknown operation \n"; - throw ParseError{op.where, "unknown operation in prune_unreachable()"}; + throw ParseError(op.loc, "unknown operation in prune_unreachable()"); } if (reach) { return prune_unreachable(op.next); @@ -643,7 +643,7 @@ bool prune_unreachable(std::unique_ptr& ops) { void CodeBlob::prune_unreachable_code() { if (prune_unreachable(ops)) { - throw ParseError{loc, "control reaches end of function"}; + throw ParseError(fun_ref->loc, "control reaches end of function"); } } @@ -704,10 +704,8 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { res.emplace_back(i); } AsmOpList tmp; - if (f_sym->is_asm_function()) { - std::get(f_sym->body)->compile(tmp); // abstract interpretation of res := f (args) - } else { - std::get(f_sym->body)->compile(tmp, res, args, where); + if (!f_sym->is_asm_function()) { + std::get(f_sym->body)->compile(tmp, res, args, loc); } int j = 0; for (var_idx_t i : left) { @@ -822,7 +820,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { } default: std::cerr << "fatal: unknown operation \n"; - throw ParseError{where, "unknown operation in fwd_analyze()"}; + throw ParseError(loc, "unknown operation in fwd_analyze()"); } if (next) { return next->fwd_analyze(std::move(values)); @@ -892,7 +890,7 @@ bool Op::mark_noreturn() { } last_in_block1->next = std::move(next); next = std::move(block1); - block1 = std::make_unique(where, Op::_Nop); + block1 = std::make_unique(loc, Op::_Nop); block1->var_info = std::move(block1_var_info); } else { block1->mark_noreturn(); @@ -916,7 +914,7 @@ bool Op::mark_noreturn() { return set_noreturn(next->mark_noreturn()); default: std::cerr << "fatal: unknown operation \n"; - throw ParseError{where, "unknown operation in mark_noreturn()"}; + throw ParseError(loc, "unknown operation in mark_noreturn()"); } } diff --git a/tolk/asmops.cpp b/tolk/asmops.cpp index 2618ed26eb..44887633bb 100644 --- a/tolk/asmops.cpp +++ b/tolk/asmops.cpp @@ -52,30 +52,50 @@ std::ostream& operator<<(std::ostream& os, AsmOp::SReg stack_reg) { } } -AsmOp AsmOp::Const(int arg, const std::string& push_op) { +// mirror the above operator<< formatting, but calculate resulting strlen +// used to align comments in Fift output +int AsmOp::SReg::calc_out_strlen() const { + int i = idx; + if (i >= 0) { + if (i < 10) { + return 2; + } else if (i < 16) { + return 3; + } else { + return 6; + } + } else if (i >= -2) { + return 5; + } else { + return 6; + } +} + + +AsmOp AsmOp::Const(SrcLocation loc, int arg, const std::string& push_op) { std::ostringstream os; os << arg << ' ' << push_op; - return AsmOp::Const(os.str()); + return AsmOp::Const(loc, os.str()); } -AsmOp AsmOp::make_stk2(int a, int b, const char* str, int delta) { +AsmOp AsmOp::make_stk2(SrcLocation loc, int a, int b, const char* str, int delta) { std::ostringstream os; os << SReg(a) << ' ' << SReg(b) << ' ' << str; int c = std::max(a, b) + 1; - return AsmOp::Custom(os.str(), c, c + delta); + return AsmOp::Custom(loc, os.str(), c, c + delta); } -AsmOp AsmOp::make_stk3(int a, int b, int c, const char* str, int delta) { +AsmOp AsmOp::make_stk3(SrcLocation loc, int a, int b, int c, const char* str, int delta) { std::ostringstream os; os << SReg(a) << ' ' << SReg(b) << ' ' << SReg(c) << ' ' << str; int m = std::max(a, std::max(b, c)) + 1; - return AsmOp::Custom(os.str(), m, m + delta); + return AsmOp::Custom(loc, os.str(), m, m + delta); } -AsmOp AsmOp::BlkSwap(int a, int b) { +AsmOp AsmOp::BlkSwap(SrcLocation loc, int a, int b) { std::ostringstream os; if (a == 1 && b == 1) { - return AsmOp::Xchg(0, 1); + return AsmOp::Xchg(loc, 0, 1); } else if (a == 1) { if (b == 2) { os << "ROT"; @@ -91,125 +111,125 @@ AsmOp AsmOp::BlkSwap(int a, int b) { } else { os << a << " " << b << " BLKSWAP"; } - return AsmOp::Custom(os.str(), a + b, a + b); + return AsmOp::Custom(loc, os.str(), a + b, a + b); } -AsmOp AsmOp::BlkPush(int a, int b) { +AsmOp AsmOp::BlkPush(SrcLocation loc, int a, int b) { std::ostringstream os; if (a == 1) { - return AsmOp::Push(b); + return AsmOp::Push(loc, b); } else if (a == 2 && b == 1) { os << "2DUP"; } else { os << a << " " << b << " BLKPUSH"; } - return AsmOp::Custom(os.str(), b + 1, a + b + 1); + return AsmOp::Custom(loc, os.str(), b + 1, a + b + 1); } -AsmOp AsmOp::BlkDrop(int a) { +AsmOp AsmOp::BlkDrop(SrcLocation loc, int a) { std::ostringstream os; if (a == 1) { - return AsmOp::Pop(); + return AsmOp::Pop(loc, 0); } else if (a == 2) { os << "2DROP"; } else { os << a << " BLKDROP"; } - return AsmOp::Custom(os.str(), a, 0); + return AsmOp::Custom(loc, os.str(), a, 0); } -AsmOp AsmOp::BlkDrop2(int a, int b) { +AsmOp AsmOp::BlkDrop2(SrcLocation loc, int a, int b) { if (!b) { - return BlkDrop(a); + return BlkDrop(loc, a); } std::ostringstream os; os << a << " " << b << " BLKDROP2"; - return AsmOp::Custom(os.str(), a + b, b); + return AsmOp::Custom(loc, os.str(), a + b, b); } -AsmOp AsmOp::BlkReverse(int a, int b) { +AsmOp AsmOp::BlkReverse(SrcLocation loc, int a, int b) { std::ostringstream os; os << a << " " << b << " REVERSE"; - return AsmOp::Custom(os.str(), a + b, a + b); + return AsmOp::Custom(loc, os.str(), a + b, a + b); } -AsmOp AsmOp::Tuple(int a) { +AsmOp AsmOp::Tuple(SrcLocation loc, int a) { switch (a) { case 1: - return AsmOp::Custom("SINGLE", 1, 1); + return AsmOp::Custom(loc, "SINGLE", 1, 1); case 2: - return AsmOp::Custom("PAIR", 2, 1); + return AsmOp::Custom(loc, "PAIR", 2, 1); case 3: - return AsmOp::Custom("TRIPLE", 3, 1); + return AsmOp::Custom(loc, "TRIPLE", 3, 1); } std::ostringstream os; os << a << " TUPLE"; - return AsmOp::Custom(os.str(), a, 1); + return AsmOp::Custom(loc, os.str(), a, 1); } -AsmOp AsmOp::UnTuple(int a) { +AsmOp AsmOp::UnTuple(SrcLocation loc, int a) { switch (a) { case 1: - return AsmOp::Custom("UNSINGLE", 1, 1); + return AsmOp::Custom(loc, "UNSINGLE", 1, 1); case 2: - return AsmOp::Custom("UNPAIR", 1, 2); + return AsmOp::Custom(loc, "UNPAIR", 1, 2); case 3: - return AsmOp::Custom("UNTRIPLE", 1, 3); + return AsmOp::Custom(loc, "UNTRIPLE", 1, 3); } std::ostringstream os; os << a << " UNTUPLE"; - return AsmOp::Custom(os.str(), 1, a); + return AsmOp::Custom(loc, os.str(), 1, a); } -AsmOp AsmOp::IntConst(const td::RefInt256& x) { +AsmOp AsmOp::IntConst(SrcLocation loc, const td::RefInt256& x) { if (x->signed_fits_bits(8)) { - return AsmOp::Const(dec_string(x) + " PUSHINT"); + return AsmOp::Const(loc, dec_string(x) + " PUSHINT"); } if (!x->is_valid()) { - return AsmOp::Const("PUSHNAN"); + return AsmOp::Const(loc, "PUSHNAN"); } int k = is_pos_pow2(x); if (k >= 0) { - return AsmOp::Const(k, "PUSHPOW2"); + return AsmOp::Const(loc, k, "PUSHPOW2"); } k = is_pos_pow2(x + 1); if (k >= 0) { - return AsmOp::Const(k, "PUSHPOW2DEC"); + return AsmOp::Const(loc, k, "PUSHPOW2DEC"); } k = is_pos_pow2(-x); if (k >= 0) { - return AsmOp::Const(k, "PUSHNEGPOW2"); + return AsmOp::Const(loc, k, "PUSHNEGPOW2"); } if (!x->mod_pow2_short(23)) { - return AsmOp::Const(dec_string(x) + " PUSHINTX"); + return AsmOp::Const(loc, dec_string(x) + " PUSHINTX"); } - return AsmOp::Const(dec_string(x) + " PUSHINT"); + return AsmOp::Const(loc, dec_string(x) + " PUSHINT"); } -AsmOp AsmOp::BoolConst(bool f) { - return AsmOp::Const(f ? "TRUE" : "FALSE"); +AsmOp AsmOp::BoolConst(SrcLocation loc, bool f) { + return AsmOp::Const(loc, f ? "TRUE" : "FALSE"); } -AsmOp AsmOp::Parse(const std::string& custom_op) { +AsmOp AsmOp::Parse(SrcLocation loc, const std::string& custom_op) { if (custom_op == "NOP") { - return AsmOp::Nop(); + return AsmOp::Nop(loc); } else if (custom_op == "SWAP") { - return AsmOp::Xchg(1); + return AsmOp::Xchg(loc, 1); } else if (custom_op == "DROP") { - return AsmOp::Pop(0); + return AsmOp::Pop(loc, 0); } else if (custom_op == "NIP") { - return AsmOp::Pop(1); + return AsmOp::Pop(loc, 1); } else if (custom_op == "DUP") { - return AsmOp::Push(0); + return AsmOp::Push(loc, 0); } else if (custom_op == "OVER") { - return AsmOp::Push(1); + return AsmOp::Push(loc, 1); } else { - return AsmOp::Custom(custom_op); + return AsmOp::Custom(loc, custom_op); } } -AsmOp AsmOp::Parse(std::string custom_op, int args, int retv) { - auto res = Parse(custom_op); +AsmOp AsmOp::Parse(SrcLocation loc, std::string custom_op, int args, int retv) { + auto res = Parse(loc, custom_op); if (res.is_custom()) { res.a = args; res.b = retv; @@ -217,48 +237,50 @@ AsmOp AsmOp::Parse(std::string custom_op, int args, int retv) { return res; } -void AsmOp::out(std::ostream& os) const { +int AsmOp::out(std::ostream& os) const { if (!op.empty()) { os << op; - return; + return static_cast(op.size()); // return strlen to align a comment at the right } switch (t) { - case a_none: - break; + case a_nop: + case a_comment: + return 0; case a_xchg: if (!a && !(b & -2)) { os << (b ? "SWAP" : "NOP"); - break; + return b ? 4 : 3; } os << SReg(a) << ' ' << SReg(b) << " XCHG"; - break; + return SReg(a).calc_out_strlen() + 1 + SReg(b).calc_out_strlen() + 5; case a_push: if (!(a & -2)) { os << (a ? "OVER" : "DUP"); - break; + return a ? 4 : 3; } os << SReg(a) << " PUSH"; - break; + return SReg(a).calc_out_strlen() + 5; case a_pop: if (!(a & -2)) { os << (a ? "NIP" : "DROP"); - break; + return a ? 3 : 4; } os << SReg(a) << " POP"; - break; + return SReg(a).calc_out_strlen() + 4; default: - throw Fatal{"unknown assembler operation"}; + throw Fatal("unknown assembler operation"); } } -void AsmOp::out_indent_nl(std::ostream& os, bool no_eol) const { - for (int i = 0; i < indent; i++) { - os << " "; +int AsmOp::out_indented(std::ostream& os, bool print_src_line_above) const { + static int last_line_no = -1; + if (loc.is_defined() && print_src_line_above) { + loc.show_line_to_fif_output(os, indent, &last_line_no); } - out(os); - if (!no_eol) { - os << std::endl; + for (int i = 0; i < indent * 2; i++) { + os << ' '; } + return out(os) + indent * 2; } std::string AsmOp::to_string() const { @@ -271,16 +293,7 @@ std::string AsmOp::to_string() const { } } -bool AsmOpList::append(const std::vector& ops) { - for (const auto& op : ops) { - if (!append(op)) { - return false; - } - } - return true; -} - -const_idx_t AsmOpList::register_const(Const new_const) { +const_idx_t AsmOpList::register_const(td::RefInt256 new_const) { if (new_const.is_null()) { return not_const; } @@ -294,7 +307,7 @@ const_idx_t AsmOpList::register_const(Const new_const) { return (const_idx_t)idx; } -Const AsmOpList::get_const(const_idx_t idx) { +td::RefInt256 AsmOpList::get_const(const_idx_t idx) { if ((unsigned)idx < constants_.size()) { return constants_[idx]; } else { @@ -316,25 +329,27 @@ void AsmOpList::show_var_ext(std::ostream& os, std::pair } void AsmOpList::out(std::ostream& os, int mode) const { - if (!(mode & 2)) { - for (const auto& op : list_) { - op.out_indent_nl(os); - } - } else { - std::size_t n = list_.size(); - for (std::size_t i = 0; i < n; i++) { - const auto& op = list_[i]; - if (!op.is_comment() && i + 1 < n && list_[i + 1].is_comment()) { - op.out_indent_nl(os, true); - os << '\t'; - do { - i++; - } while (i + 1 < n && list_[i + 1].is_comment()); - list_[i].out(os); - os << std::endl; - } else { - op.out_indent_nl(os, false); + std::size_t n = list_.size(); + for (std::size_t i = 0; i < n; i++) { + const AsmOp& op = list_[i]; + if (!op.is_comment() && i + 1 < n && list_[i + 1].is_comment()) { + int len = op.out_indented(os, mode & Stack::_LineComments); + while (len < 28) { // align stack comments at the right + os << ' '; + len++; } + os << '\t'; + do { + i++; + } while (i + 1 < n && list_[i + 1].is_comment()); + list_[i].out(os); + os << std::endl; + } else if (op.is_comment()) { + op.out(os); + os << std::endl; + } else { + op.out_indented(os, mode & Stack::_LineComments); + os << std::endl; } } } @@ -344,7 +359,7 @@ bool apply_op(StackTransform& trans, const AsmOp& op) { return false; } switch (op.t) { - case AsmOp::a_none: + case AsmOp::a_nop: return true; case AsmOp::a_xchg: return trans.apply_xchg(op.a, op.b, true); diff --git a/tolk/ast-aux-data.h b/tolk/ast-aux-data.h new file mode 100644 index 0000000000..59006b1fcc --- /dev/null +++ b/tolk/ast-aux-data.h @@ -0,0 +1,38 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "ast.h" + +/* + * This file contains a schema of aux_data inside ast_artificial_aux_vertex + * (it's a compiler-inserted vertex that can't occur in source code). + */ + +namespace tolk { + +// AuxData_ForceFiftLocation is created when transforming AST to IR; +// it wraps constants to force codegen location point to usage, not to init_val AST nodes +struct AuxData_ForceFiftLocation final : ASTAuxData { + SrcLocation forced_loc; + + explicit AuxData_ForceFiftLocation(SrcLocation forced_loc) + : forced_loc(forced_loc) { + } +}; + +} // namespace tolk diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 2999d1a10e..74ecbd2827 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -1209,7 +1209,7 @@ static AnyV parse_asm_func_body(Lexer& lex, V param_list) { lex.expect(tok_asm, "`asm`"); size_t n_params = param_list->size(); if (n_params > 16) { - throw ParseError{loc, "assembler built-in function can have at most 16 arguments"}; + throw ParseError(loc, "assembler built-in function can have at most 16 arguments"); } std::vector arg_order, ret_order; if (lex.tok() == tok_oppar) { diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 54d1f8e26e..382f2bd715 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -95,6 +95,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } @@ -139,6 +140,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { case ast_empty_expression: return replace(v->as()); case ast_parenthesized_expression: return replace(v->as()); case ast_braced_expression: return replace(v->as()); + case ast_artificial_aux_vertex: return replace(v->as()); case ast_tensor: return replace(v->as()); case ast_bracket_tuple: return replace(v->as()); case ast_reference: return replace(v->as()); diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 42e6915252..b056db314c 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -84,6 +84,9 @@ class ASTReplicator final { static V clone(V v) { return createV(v->loc, clone(v->get_block_statement())); } + static V clone(V v) { + return createV(v->loc, clone(v->get_wrapped_expr()), v->aux_data, v->inferred_type); + } static V clone(V v) { return createV(v->loc, clone(v->get_items())); } @@ -270,6 +273,7 @@ class ASTReplicator final { case ast_empty_expression: return clone(v->as()); case ast_parenthesized_expression: return clone(v->as()); case ast_braced_expression: return clone(v->as()); + case ast_artificial_aux_vertex: return clone(v->as()); case ast_tensor: return clone(v->as()); case ast_bracket_tuple: return clone(v->as()); case ast_reference: return clone(v->as()); diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index c4af598d44..4e6f795e93 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -45,6 +45,7 @@ class ASTStringifier final : public ASTVisitor { {ast_empty_expression, "ast_empty_expression"}, {ast_parenthesized_expression, "ast_parenthesized_expression"}, {ast_braced_expression, "ast_braced_expression"}, + {ast_artificial_aux_vertex, "ast_artificial_aux_vertex"}, {ast_tensor, "ast_tensor"}, {ast_bracket_tuple, "ast_bracket_tuple"}, {ast_reference, "ast_reference"}, @@ -306,6 +307,7 @@ class ASTStringifier final : public ASTVisitor { case ast_empty_expression: return handle_vertex(v->as()); case ast_parenthesized_expression: return handle_vertex(v->as()); case ast_braced_expression: return handle_vertex(v->as()); + case ast_artificial_aux_vertex: return handle_vertex(v->as()); case ast_tensor: return handle_vertex(v->as()); case ast_bracket_tuple: return handle_vertex(v->as()); case ast_reference: return handle_vertex(v->as()); diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index 1bb72f44c4..926ec5808a 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -104,6 +104,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } @@ -149,6 +150,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { case ast_empty_expression: return visit(v->as()); case ast_parenthesized_expression: return visit(v->as()); case ast_braced_expression: return visit(v->as()); + case ast_artificial_aux_vertex: return visit(v->as()); case ast_tensor: return visit(v->as()); case ast_bracket_tuple: return visit(v->as()); case ast_reference: return visit(v->as()); diff --git a/tolk/ast.h b/tolk/ast.h index 95677fcda2..927be5b71f 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -75,6 +75,7 @@ enum ASTNodeKind { ast_empty_expression, ast_parenthesized_expression, ast_braced_expression, + ast_artificial_aux_vertex, ast_tensor, ast_bracket_tuple, ast_reference, @@ -149,6 +150,10 @@ enum class MatchArmKind { // for `match` expression, each of arms `pattern => else_branch, // `else => body` }; +struct ASTAuxData { // base class for data in ast_artificial_aux_vertex, see ast-aux-data.h + virtual ~ASTAuxData() = default; +}; + template struct Vertex; @@ -498,6 +503,21 @@ struct Vertex final : ASTExprBlockOfStatements { : ASTExprBlockOfStatements(ast_braced_expression, loc, child_block_statement) {} }; +template<> +// ast_artificial_aux_vertex is a compiler-inserted vertex that can't occur in source code +// example: special vertex to force location in Fift output at constant usage +struct Vertex final : ASTExprUnary { + const ASTAuxData* aux_data; // custom payload, see ast-aux-data.h + + AnyExprV get_wrapped_expr() const { return child; } + + Vertex(SrcLocation loc, AnyExprV wrapped_expr, const ASTAuxData* aux_data, TypePtr inferred_type) + : ASTExprUnary(ast_artificial_aux_vertex, loc, wrapped_expr) + , aux_data(aux_data) { + assign_inferred_type(inferred_type); + } +}; + template<> // ast_tensor is a set of expressions embraced by (parenthesis) // in most languages, it's called "tuple", but in TVM, "tuple" is a TVM primitive, that's why "tensor" diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 75ead3458f..7de0b3d843 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -44,11 +44,6 @@ static void define_builtin_func(const std::string& name, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const AsmOp& macro, int flags) { - auto* f_sym = new FunctionData(name, {}, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(make_simple_compile(macro)), nullptr); - G.symtable.add_function(f_sym); -} - static void define_builtin_func(const std::string& name, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags, std::initializer_list arg_order, std::initializer_list ret_order) { auto* f_sym = new FunctionData(name, {}, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); @@ -58,12 +53,16 @@ static void define_builtin_func(const std::string& name, const std::vector& out, std::vector& in, - SrcLocation where) const { - dest.append(simple_compile(out, in, where)); + SrcLocation loc) const { + dest << simple_compile(out, in, loc); } -void FunctionBodyAsm::compile(AsmOpList& dest) const { - dest.append(ops); +void FunctionBodyAsm::compile(AsmOpList& dest, SrcLocation loc) const { + for (const AsmOp& op : ops) { + AsmOp copy = op; + copy.loc = loc; + dest << std::move(copy); + } } @@ -331,101 +330,89 @@ bool VarDescr::always_neq(const VarDescr& other) const { (always_odd() && other.always_even()); } -AsmOp exec_op(std::string op) { - return AsmOp::Custom(op); -} - -AsmOp exec_op(std::string op, int args, int retv = 1) { - return AsmOp::Custom(op, args, retv); +AsmOp exec_op(SrcLocation loc, std::string op) { + return AsmOp::Custom(loc, op); } -AsmOp exec_arg_op(std::string op, long long arg) { - std::ostringstream os; - os << arg << ' ' << op; - return AsmOp::Custom(os.str()); -} - -AsmOp exec_arg_op(std::string op, long long arg, int args, int retv) { - std::ostringstream os; - os << arg << ' ' << op; - return AsmOp::Custom(os.str(), args, retv); +AsmOp exec_op(SrcLocation loc, std::string op, int args, int retv = 1) { + return AsmOp::Custom(loc, op, args, retv); } -AsmOp exec_arg_op(std::string op, td::RefInt256 arg) { +AsmOp exec_arg_op(SrcLocation loc, std::string op, long long arg, int args, int retv) { std::ostringstream os; os << arg << ' ' << op; - return AsmOp::Custom(os.str()); + return AsmOp::Custom(loc, os.str(), args, retv); } -AsmOp exec_arg_op(std::string op, td::RefInt256 arg, int args, int retv) { +AsmOp exec_arg_op(SrcLocation loc, std::string op, td::RefInt256 arg, int args, int retv) { std::ostringstream os; os << arg << ' ' << op; - return AsmOp::Custom(os.str(), args, retv); + return AsmOp::Custom(loc, os.str(), args, retv); } -AsmOp exec_arg2_op(std::string op, long long imm1, long long imm2, int args, int retv) { +AsmOp exec_arg2_op(SrcLocation loc, std::string op, long long imm1, long long imm2, int args, int retv) { std::ostringstream os; os << imm1 << ' ' << imm2 << ' ' << op; - return AsmOp::Custom(os.str(), args, retv); + return AsmOp::Custom(loc, os.str(), args, retv); } -AsmOp push_const(td::RefInt256 x) { - return AsmOp::IntConst(std::move(x)); +AsmOp push_const(SrcLocation loc, td::RefInt256 x) { + return AsmOp::IntConst(loc, std::move(x)); } -AsmOp compile_add(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_add(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const + y.int_const); if (!r.int_const->is_valid()) { - throw ParseError(where, "integer overflow"); + throw ParseError(loc, "integer overflow"); } x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_add(x.val, y.val); if (y.is_int_const() && y.int_const->signed_fits_bits(8)) { y.unused(); if (y.always_zero()) { - return AsmOp::Nop(); + return AsmOp::Nop(loc); } if (*y.int_const == 1) { - return exec_op("INC", 1); + return exec_op(loc, "INC", 1); } if (*y.int_const == -1) { - return exec_op("DEC", 1); + return exec_op(loc, "DEC", 1); } - return exec_arg_op("ADDCONST", y.int_const, 1); + return exec_arg_op(loc, "ADDCONST", y.int_const, 1); } if (x.is_int_const() && x.int_const->signed_fits_bits(8)) { x.unused(); if (x.always_zero()) { - return AsmOp::Nop(); + return AsmOp::Nop(loc); } if (*x.int_const == 1) { - return exec_op("INC", 1); + return exec_op(loc, "INC", 1); } if (*x.int_const == -1) { - return exec_op("DEC", 1); + return exec_op(loc, "DEC", 1); } - return exec_arg_op("ADDCONST", x.int_const, 1); + return exec_arg_op(loc, "ADDCONST", x.int_const, 1); } - return exec_op("ADD", 2); + return exec_op(loc, "ADD", 2); } -AsmOp compile_sub(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_sub(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const - y.int_const); if (!r.int_const->is_valid()) { - throw ParseError(where, "integer overflow"); + throw ParseError(loc, "integer overflow"); } x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_sub(x.val, y.val); if (y.is_int_const() && (-y.int_const)->signed_fits_bits(8)) { @@ -434,121 +421,121 @@ AsmOp compile_sub(std::vector& res, std::vector& args, SrcLo return {}; } if (*y.int_const == 1) { - return exec_op("DEC", 1); + return exec_op(loc, "DEC", 1); } if (*y.int_const == -1) { - return exec_op("INC", 1); + return exec_op(loc, "INC", 1); } - return exec_arg_op("ADDCONST", -y.int_const, 1); + return exec_arg_op(loc, "ADDCONST", -y.int_const, 1); } if (x.always_zero()) { x.unused(); - return exec_op("NEGATE", 1); + return exec_op(loc, "NEGATE", 1); } - return exec_op("SUB", 2); + return exec_op(loc, "SUB", 2); } -AsmOp compile_unary_minus(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_unary_minus(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(-x.int_const); if (!r.int_const->is_valid()) { - throw ParseError(where, "integer overflow"); + throw ParseError(loc, "integer overflow"); } x.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_negate(x.val); - return exec_op("NEGATE", 1); + return exec_op(loc, "NEGATE", 1); } -AsmOp compile_unary_plus(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_unary_plus(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(x.int_const); x.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = x.val; - return AsmOp::Nop(); + return AsmOp::Nop(loc); } -AsmOp compile_logical_not(std::vector& res, std::vector& args, SrcLocation where, bool for_int_arg) { +static AsmOp compile_logical_not(std::vector& res, std::vector& args, SrcLocation loc, bool for_int_arg) { tolk_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(x.int_const == 0 ? -1 : 0); x.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = VarDescr::ValBool; // for integers, `!var` is `var != 0` // for booleans, `!var` can be shortened to `~var` (works the same for 0/-1 but consumes less) - return for_int_arg ? exec_op("0 EQINT", 1) : exec_op("NOT", 1); + return for_int_arg ? exec_op(loc, "0 EQINT", 1) : exec_op(loc, "NOT", 1); } -AsmOp compile_bitwise_and(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_bitwise_and(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const & y.int_const); x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_bitwise_and(x.val, y.val); - return exec_op("AND", 2); + return exec_op(loc, "AND", 2); } -AsmOp compile_bitwise_or(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_bitwise_or(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const | y.int_const); x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_bitwise_or(x.val, y.val); - return exec_op("OR", 2); + return exec_op(loc, "OR", 2); } -AsmOp compile_bitwise_xor(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_bitwise_xor(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const ^ y.int_const); x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_bitwise_xor(x.val, y.val); - return exec_op("XOR", 2); + return exec_op(loc, "XOR", 2); } -AsmOp compile_bitwise_not(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_bitwise_not(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(~x.int_const); x.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_bitwise_not(x.val); - return exec_op("NOT", 1); + return exec_op(loc, "NOT", 1); } -AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation where) { +static AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation loc) { if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const * y.int_const); if (!r.int_const->is_valid()) { - throw ParseError(where, "integer overflow"); + throw ParseError(loc, "integer overflow"); } x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_mul(x.val, y.val); if (y.is_int_const()) { @@ -559,23 +546,23 @@ AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation wh // dubious optimization: NaN * 0 = ? r.set_const(y.int_const); x.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } if (*y.int_const == 1 && x.always_finite()) { - return AsmOp::Nop(); + return AsmOp::Nop(loc); } if (*y.int_const == -1) { - return exec_op("NEGATE", 1); + return exec_op(loc, "NEGATE", 1); } - return exec_arg_op("MULCONST", y.int_const, 1); + return exec_arg_op(loc, "MULCONST", y.int_const, 1); } if (k > 0) { y.unused(); - return exec_arg_op("LSHIFT#", k, 1); + return exec_arg_op(loc, "LSHIFT#", k, 1); } if (k == 0) { y.unused(); - return AsmOp::Nop(); + return AsmOp::Nop(loc); } } if (x.is_int_const()) { @@ -586,48 +573,48 @@ AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation wh // dubious optimization: NaN * 0 = ? r.set_const(x.int_const); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } if (*x.int_const == 1 && y.always_finite()) { - return AsmOp::Nop(); + return AsmOp::Nop(loc); } if (*x.int_const == -1) { - return exec_op("NEGATE", 1); + return exec_op(loc, "NEGATE", 1); } - return exec_arg_op("MULCONST", x.int_const, 1); + return exec_arg_op(loc, "MULCONST", x.int_const, 1); } if (k > 0) { x.unused(); - return exec_arg_op("LSHIFT#", k, 1); + return exec_arg_op(loc, "LSHIFT#", k, 1); } if (k == 0) { x.unused(); - return AsmOp::Nop(); + return AsmOp::Nop(loc); } } - return exec_op("MUL", 2); + return exec_op(loc, "MUL", 2); } -AsmOp compile_mul(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_mul(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); - return compile_mul_internal(res[0], args[0], args[1], where); + return compile_mul_internal(res[0], args[0], args[1], loc); } -AsmOp compile_lshift(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_lshift(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (y.is_int_const()) { auto yv = y.int_const->to_long(); if (yv < 0 || yv > 256) { - throw ParseError(where, "lshift argument is out of range"); + throw ParseError(loc, "lshift argument is out of range"); } else if (x.is_int_const()) { r.set_const(x.int_const << (int)yv); if (!r.int_const->is_valid()) { - throw ParseError(where, "integer overflow"); + throw ParseError(loc, "integer overflow"); } x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } } r.val = emulate_lshift(x.val, y.val); @@ -636,38 +623,38 @@ AsmOp compile_lshift(std::vector& res, std::vector& args, Sr if (!k /* && x.always_finite() */) { // dubious optimization: what if x=NaN ? y.unused(); - return AsmOp::Nop(); + return AsmOp::Nop(loc); } y.unused(); - return exec_arg_op("LSHIFT#", k, 1); + return exec_arg_op(loc, "LSHIFT#", k, 1); } if (x.is_int_const()) { auto xv = x.int_const->to_long(); if (xv == 1) { x.unused(); - return exec_op("POW2", 1); + return exec_op(loc, "POW2", 1); } if (xv == -1) { x.unused(); - return exec_op("-1 PUSHINT SWAP LSHIFT", 1); + return exec_op(loc, "-1 PUSHINT SWAP LSHIFT", 1); } } - return exec_op("LSHIFT", 2); + return exec_op(loc, "LSHIFT", 2); } -AsmOp compile_rshift(std::vector& res, std::vector& args, SrcLocation where, +static AsmOp compile_rshift(std::vector& res, std::vector& args, SrcLocation loc, int round_mode) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (y.is_int_const()) { auto yv = y.int_const->to_long(); if (yv < 0 || yv > 256) { - throw ParseError(where, "rshift argument is out of range"); + throw ParseError(loc, "rshift argument is out of range"); } else if (x.is_int_const()) { r.set_const(td::rshift(x.int_const, (int)yv, round_mode)); x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } } r.val = emulate_rshift(x.val, y.val); @@ -677,36 +664,36 @@ AsmOp compile_rshift(std::vector& res, std::vector& args, Sr if (!k /* && x.always_finite() */) { // dubious optimization: what if x=NaN ? y.unused(); - return AsmOp::Nop(); + return AsmOp::Nop(loc); } y.unused(); - return exec_arg_op(rshift + "#", k, 1); + return exec_arg_op(loc, rshift + "#", k, 1); } - return exec_op(rshift, 2); + return exec_op(loc, rshift, 2); } -AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation where, int round_mode) { +static AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation loc, int round_mode) { if (x.is_int_const() && y.is_int_const()) { r.set_const(div(x.int_const, y.int_const, round_mode)); if (!r.int_const->is_valid()) { - throw ParseError(where, *y.int_const == 0 ? "division by zero" : "integer overflow"); + throw ParseError(loc, *y.int_const == 0 ? "division by zero" : "integer overflow"); } x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_div(x.val, y.val); if (y.is_int_const()) { if (*y.int_const == 0) { - throw ParseError(where, "division by zero"); + throw ParseError(loc, "division by zero"); } if (*y.int_const == 1 && x.always_finite()) { y.unused(); - return AsmOp::Nop(); + return AsmOp::Nop(loc); } if (*y.int_const == -1) { y.unused(); - return exec_op("NEGATE", 1); + return exec_op(loc, "NEGATE", 1); } int k = is_pos_pow2(y.int_const); if (k > 0) { @@ -715,44 +702,44 @@ AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation wh if (round_mode >= 0) { op += (round_mode > 0 ? 'C' : 'R'); } - return exec_arg_op(op + '#', k, 1); + return exec_arg_op(loc, op + '#', k, 1); } } std::string op = "DIV"; if (round_mode >= 0) { op += (round_mode > 0 ? 'C' : 'R'); } - return exec_op(op, 2); + return exec_op(loc, op, 2); } -AsmOp compile_div(std::vector& res, std::vector& args, SrcLocation where, int round_mode) { +static AsmOp compile_div(std::vector& res, std::vector& args, SrcLocation loc, int round_mode) { tolk_assert(res.size() == 1 && args.size() == 2); - return compile_div_internal(res[0], args[0], args[1], where, round_mode); + return compile_div_internal(res[0], args[0], args[1], loc, round_mode); } -AsmOp compile_mod(std::vector& res, std::vector& args, SrcLocation where, +static AsmOp compile_mod(std::vector& res, std::vector& args, SrcLocation loc, int round_mode) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(mod(x.int_const, y.int_const, round_mode)); if (!r.int_const->is_valid()) { - throw ParseError(where, *y.int_const == 0 ? "division by zero" : "integer overflow"); + throw ParseError(loc, *y.int_const == 0 ? "division by zero" : "integer overflow"); } x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_mod(x.val, y.val); if (y.is_int_const()) { if (*y.int_const == 0) { - throw ParseError(where, "division by zero"); + throw ParseError(loc, "division by zero"); } if ((*y.int_const == 1 || *y.int_const == -1) && x.always_finite()) { x.unused(); y.unused(); r.set_const(td::zero_refint()); - return push_const(r.int_const); + return push_const(loc, r.int_const); } int k = is_pos_pow2(y.int_const); if (k > 0) { @@ -761,29 +748,29 @@ AsmOp compile_mod(std::vector& res, std::vector& args, SrcLo if (round_mode >= 0) { op += (round_mode > 0 ? 'C' : 'R'); } - return exec_arg_op(op + '#', k, 1); + return exec_arg_op(loc, op + '#', k, 1); } } std::string op = "MOD"; if (round_mode >= 0) { op += (round_mode > 0 ? 'C' : 'R'); } - return exec_op(op, 2); + return exec_op(loc, op, 2); } -AsmOp compile_muldiv(std::vector& res, std::vector& args, SrcLocation where, +static AsmOp compile_muldiv(std::vector& res, std::vector& args, SrcLocation loc, int round_mode) { tolk_assert(res.size() == 1 && args.size() == 3); VarDescr &r = res[0], &x = args[0], &y = args[1], &z = args[2]; if (x.is_int_const() && y.is_int_const() && z.is_int_const()) { r.set_const(muldiv(x.int_const, y.int_const, z.int_const, round_mode)); if (!r.int_const->is_valid()) { - throw ParseError(where, *z.int_const == 0 ? "division by zero" : "integer overflow"); + throw ParseError(loc, *z.int_const == 0 ? "division by zero" : "integer overflow"); } x.unused(); y.unused(); z.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } if (x.always_zero() || y.always_zero()) { // dubious optimization for z=0... @@ -791,26 +778,26 @@ AsmOp compile_muldiv(std::vector& res, std::vector& args, Sr y.unused(); z.unused(); r.set_const(td::make_refint(0)); - return push_const(r.int_const); + return push_const(loc, r.int_const); } char c = (round_mode < 0) ? 0 : (round_mode > 0 ? 'C' : 'R'); r.val = emulate_div(emulate_mul(x.val, y.val), z.val); if (z.is_int_const()) { if (*z.int_const == 0) { - throw ParseError(where, "division by zero"); + throw ParseError(loc, "division by zero"); } if (*z.int_const == 1) { z.unused(); - return compile_mul_internal(r, x, y, where); + return compile_mul_internal(r, x, y, loc); } } if (y.is_int_const() && *y.int_const == 1) { y.unused(); - return compile_div_internal(r, x, z, where, round_mode); + return compile_div_internal(r, x, z, loc, round_mode); } if (x.is_int_const() && *x.int_const == 1) { x.unused(); - return compile_div_internal(r, y, z, where, round_mode); + return compile_div_internal(r, y, z, loc, round_mode); } if (z.is_int_const()) { int k = is_pos_pow2(z.int_const); @@ -820,7 +807,7 @@ AsmOp compile_muldiv(std::vector& res, std::vector& args, Sr if (c) { op += c; } - return exec_arg_op(op + '#', k, 2); + return exec_arg_op(loc, op + '#', k, 2); } } if (y.is_int_const()) { @@ -831,7 +818,7 @@ AsmOp compile_muldiv(std::vector& res, std::vector& args, Sr if (c) { op += c; } - return exec_arg_op(op, k, 2); + return exec_arg_op(loc, op, k, 2); } } if (x.is_int_const()) { @@ -842,17 +829,22 @@ AsmOp compile_muldiv(std::vector& res, std::vector& args, Sr if (c) { op += c; } - return exec_arg_op(op, k, 2); + return exec_arg_op(loc, op, k, 2); } } std::string op = "MULDIV"; if (c) { op += c; } - return exec_op(op, 3); + return exec_op(loc, op, 3); } -int compute_compare(td::RefInt256 x, td::RefInt256 y, int mode) { +// fun mulDivMod(x: int, y: int, z: int): (int, int) asm "MULDIVMOD"; +static AsmOp compile_muldivmod(std::vector&, std::vector&, SrcLocation loc) { + return AsmOp::Custom(loc, "MULDIVMOD", 3, 2); +} + +static int compute_compare(td::RefInt256 x, td::RefInt256 y, int mode) { int s = td::cmp(x, y); if (mode == 7) { return s; @@ -866,7 +858,7 @@ int compute_compare(td::RefInt256 x, td::RefInt256 y, int mode) { // 2 -> constant 0 // 1 -> constant -1 // 3 -> 0 or -1 -int compute_compare(const VarDescr& x, const VarDescr& y, int mode) { +static int compute_compare(const VarDescr& x, const VarDescr& y, int mode) { switch (mode) { case 1: // > return x.always_greater(y) ? 1 : (x.always_leq(y) ? 2 : 3); @@ -893,7 +885,7 @@ int compute_compare(const VarDescr& x, const VarDescr& y, int mode) { } } -AsmOp compile_cmp_int(std::vector& res, std::vector& args, int mode) { +static AsmOp compile_cmp_int(std::vector& res, std::vector& args, SrcLocation loc, int mode) { tolk_assert(mode >= 1 && mode <= 7); tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; @@ -902,7 +894,7 @@ AsmOp compile_cmp_int(std::vector& res, std::vector& args, i r.set_const(v); x.unused(); y.unused(); - return mode == 7 ? push_const(r.int_const) : AsmOp::BoolConst(v != 0); + return mode == 7 ? push_const(loc, r.int_const) : AsmOp::BoolConst(loc, v != 0); } int v = compute_compare(x, y, mode); // std::cerr << "compute_compare(" << x << ", " << y << ", " << mode << ") = " << v << std::endl; @@ -911,7 +903,7 @@ AsmOp compile_cmp_int(std::vector& res, std::vector& args, i r.set_const(v - (v >> 2) - 2); x.unused(); y.unused(); - return mode == 7 ? push_const(r.int_const) : AsmOp::BoolConst(v & 1); + return mode == 7 ? push_const(loc, r.int_const) : AsmOp::BoolConst(loc, v & 1); } r.val = ~0; if (v & 1) { @@ -930,29 +922,29 @@ AsmOp compile_cmp_int(std::vector& res, std::vector& args, i if (mode != 7) { if (y.is_int_const() && y.int_const >= -128 && y.int_const <= 127) { y.unused(); - return exec_arg_op(cmp_int_names[mode], y.int_const + cmp_int_delta[mode], 1); + return exec_arg_op(loc, cmp_int_names[mode], y.int_const + cmp_int_delta[mode], 1); } if (x.is_int_const() && x.int_const >= -128 && x.int_const <= 127) { x.unused(); mode = ((mode & 4) >> 2) | (mode & 2) | ((mode & 1) << 2); - return exec_arg_op(cmp_int_names[mode], x.int_const + cmp_int_delta[mode], 1); + return exec_arg_op(loc, cmp_int_names[mode], x.int_const + cmp_int_delta[mode], 1); } } - return exec_op(cmp_names[mode], 2); + return exec_op(loc, cmp_names[mode], 2); } -AsmOp compile_throw(std::vector& res, std::vector& args, SrcLocation) { +static AsmOp compile_throw(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.empty() && args.size() == 1); VarDescr& x = args[0]; if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { x.unused(); - return exec_arg_op("THROW", x.int_const, 0, 0); + return exec_arg_op(loc, "THROW", x.int_const, 0, 0); } else { - return exec_op("THROWANY", 1, 0); + return exec_op(loc, "THROWANY", 1, 0); } } -AsmOp compile_throw_if_unless(std::vector& res, std::vector& args, SrcLocation) { +static AsmOp compile_throw_if_unless(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.empty() && args.size() == 3); VarDescr &x = args[0], &y = args[1], &z = args[2]; if (!z.always_true() && !z.always_false()) { @@ -967,40 +959,40 @@ AsmOp compile_throw_if_unless(std::vector& res, std::vector& skip_cond = true; if (y.always_true() != mode) { x.unused(); - return AsmOp::Nop(); + return AsmOp::Nop(loc); } } if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { x.unused(); - return skip_cond ? exec_arg_op("THROW", x.int_const, 0, 0) : exec_arg_op("THROW"s + suff, x.int_const, 1, 0); + return skip_cond ? exec_arg_op(loc, "THROW", x.int_const, 0, 0) : exec_arg_op(loc, "THROW"s + suff, x.int_const, 1, 0); } else { - return skip_cond ? exec_op("THROWANY", 1, 0) : exec_op("THROWANY"s + suff, 2, 0); + return skip_cond ? exec_op(loc, "THROWANY", 1, 0) : exec_op(loc, "THROWANY"s + suff, 2, 0); } } -AsmOp compile_throw_arg(std::vector& res, std::vector& args, SrcLocation) { +static AsmOp compile_throw_arg(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.empty() && args.size() == 2); VarDescr &x = args[1]; if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { x.unused(); - return exec_arg_op("THROWARG", x.int_const, 1, 0); + return exec_arg_op(loc, "THROWARG", x.int_const, 1, 0); } else { - return exec_op("THROWARGANY", 2, 0); + return exec_op(loc, "THROWARGANY", 2, 0); } } -AsmOp compile_bool_const(std::vector& res, std::vector& args, bool val) { +static AsmOp compile_bool_const(std::vector& res, std::vector& args, SrcLocation loc, bool val) { tolk_assert(res.size() == 1 && args.empty()); VarDescr& r = res[0]; r.set_const(val ? -1 : 0); - return AsmOp::Const(val ? "TRUE" : "FALSE"); + return AsmOp::Const(loc, val ? "TRUE" : "FALSE"); } // fun loadInt (mutate s: slice, len: int): int asm(s len -> 1 0) "LDIX"; // fun loadUint (mutate s: slice, len: int): int asm( -> 1 0) "LDUX"; // fun preloadInt (s: slice, len: int): int asm "PLDIX"; // fun preloadUint(s: slice, len: int): int asm "PLDUX"; -AsmOp compile_fetch_int(std::vector& res, std::vector& args, bool fetch, bool sgnd) { +static AsmOp compile_fetch_int(std::vector& res, std::vector& args, SrcLocation loc, bool fetch, bool sgnd) { tolk_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); auto &y = args[1], &r = res.back(); r.val = (sgnd ? VarDescr::FiniteInt : VarDescr::FiniteUInt); @@ -1015,27 +1007,27 @@ AsmOp compile_fetch_int(std::vector& res, std::vector& args, } if (v > 0) { y.unused(); - return exec_arg_op((fetch ? "LD"s : "PLD"s) + (sgnd ? 'I' : 'U'), v, 1, 1 + (unsigned)fetch); + return exec_arg_op(loc, (fetch ? "LD"s : "PLD"s) + (sgnd ? 'I' : 'U'), v, 1, 1 + (unsigned)fetch); } } - return exec_op((fetch ? "LD"s : "PLD"s) + (sgnd ? "IX" : "UX"), 2, 1 + (unsigned)fetch); + return exec_op(loc, (fetch ? "LD"s : "PLD"s) + (sgnd ? "IX" : "UX"), 2, 1 + (unsigned)fetch); } // fun storeInt (mutate self: builder, x: int, len: int): self asm(x b len) "STIX"; // fun storeUint (mutate self: builder, x: int, len: int): self asm(x b len) "STUX"; -AsmOp compile_store_int(std::vector& res, std::vector& args, bool sgnd) { +static AsmOp compile_store_int(std::vector& res, std::vector& args, SrcLocation loc, bool sgnd) { tolk_assert(args.size() == 3 && res.size() == 1); auto& z = args[2]; if (z.is_int_const() && z.int_const > 0 && z.int_const <= 256) { z.unused(); - return exec_arg_op("ST"s + (sgnd ? 'I' : 'U'), z.int_const, 2, 1); + return exec_arg_op(loc, "ST"s + (sgnd ? 'I' : 'U'), z.int_const, 2, 1); } - return exec_op("ST"s + (sgnd ? "IX" : "UX"), 3, 1); + return exec_op(loc, "ST"s + (sgnd ? "IX" : "UX"), 3, 1); } // fun loadBits (mutate self: slice, len: int): self asm(s len -> 1 0) "LDSLICEX" // fun preloadBits(self: slice, len: int): slice asm(s len -> 1 0) "PLDSLICEX" -AsmOp compile_fetch_slice(std::vector& res, std::vector& args, bool fetch) { +static AsmOp compile_fetch_slice(std::vector& res, std::vector& args, SrcLocation loc, bool fetch) { tolk_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); auto& y = args[1]; int v = -1; @@ -1043,47 +1035,73 @@ AsmOp compile_fetch_slice(std::vector& res, std::vector& arg v = (int)y.int_const->to_long(); if (v > 0) { y.unused(); - return exec_arg_op(fetch ? "LDSLICE" : "PLDSLICE", v, 1, 1 + (unsigned)fetch); + return exec_arg_op(loc, fetch ? "LDSLICE" : "PLDSLICE", v, 1, 1 + (unsigned)fetch); } } - return exec_op(fetch ? "LDSLICEX" : "PLDSLICEX", 2, 1 + (unsigned)fetch); + return exec_op(loc, fetch ? "LDSLICEX" : "PLDSLICEX", 2, 1 + (unsigned)fetch); } // fun tupleAt(t: tuple, index: int): X asm "INDEXVAR"; -AsmOp compile_tuple_at(std::vector& res, std::vector& args, SrcLocation) { +static AsmOp compile_tuple_at(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 2 && res.size() == 1); auto& y = args[1]; if (y.is_int_const() && y.int_const >= 0 && y.int_const < 16) { y.unused(); - return exec_arg_op("INDEX", y.int_const, 1, 1); + return exec_arg_op(loc, "INDEX", y.int_const, 1, 1); } - return exec_op("INDEXVAR", 2, 1); + return exec_op(loc, "INDEXVAR", 2, 1); } // fun tupleSetAt(mutate self: tuple, value: X, index: int): void asm "SETINDEXVAR"; -AsmOp compile_tuple_set_at(std::vector& res, std::vector& args, SrcLocation) { +static AsmOp compile_tuple_set_at(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 3 && res.size() == 1); auto& y = args[2]; if (y.is_int_const() && y.int_const >= 0 && y.int_const < 16) { y.unused(); - return exec_arg_op("SETINDEX", y.int_const, 1, 1); + return exec_arg_op(loc, "SETINDEX", y.int_const, 1, 1); } - return exec_op("SETINDEXVAR", 2, 1); + return exec_op(loc, "SETINDEXVAR", 2, 1); +} + +// fun debugDumpStack(): void asm "DUMPSTK"; +static AsmOp compile_dumpstk(std::vector&, std::vector&, SrcLocation loc) { + return AsmOp::Custom(loc, "DUMPSTK", 0, 0); +} + +// fun debugPrintString(x: T): void asm "STRDUMP"; +static AsmOp compile_strdump(std::vector&, std::vector&, SrcLocation loc) { + return AsmOp::Custom(loc, "STRDUMP DROP", 1, 1); +} + +// fun debugPrint(x: T): void; +static AsmOp compile_debug_print_to_string(std::vector&, std::vector&, SrcLocation loc) { + return AsmOp::Custom(loc, "s0 DUMP DROP", 1, 1); } // fun ton(amount: slice): coins; ton("0.05") replaced by 50000000 at compile-time // same for stringCrc32(constString: slice) and others -AsmOp compile_time_only_function(std::vector&, std::vector&, SrcLocation) { +AsmOp compile_time_only_function(std::vector&, std::vector&, SrcLocation loc) { // all ton() invocations are constants, replaced by integers; no dynamic values allowed, no work at runtime tolk_assert(false); - return AsmOp::Nop(); + return AsmOp::Nop(loc); +} + +// `null` literal is under the hood transformed to PUSHNULL +static AsmOp compile_push_null(std::vector&, std::vector&, SrcLocation loc) { + return AsmOp::Const(loc, "PUSHNULL"); } // fun __isNull(X arg): bool -AsmOp compile_is_null(std::vector& res, std::vector& args, SrcLocation) { +static AsmOp compile_is_null(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 1 && res.size() == 1); res[0].val = VarDescr::ValBool; - return exec_op("ISNULL", 1, 1); + return exec_op(loc, "ISNULL", 1, 1); +} + +// fun __expect_type(, ""): void; +static AsmOp compile_expect_type(std::vector&, std::vector&, SrcLocation loc) { + // handled by type checker, does nothing at runtime + return AsmOp::Nop(loc); } @@ -1175,37 +1193,37 @@ void define_builtins() { compile_bitwise_xor, FunctionData::flagMarkedAsPure); define_builtin_func("_==_", ParamsInt2, Int, nullptr, // also works for bool - std::bind(compile_cmp_int, _1, _2, 2), + std::bind(compile_cmp_int, _1, _2, _3, 2), FunctionData::flagMarkedAsPure); define_builtin_func("_!=_", ParamsInt2, Int, nullptr, // also works for bool - std::bind(compile_cmp_int, _1, _2, 5), + std::bind(compile_cmp_int, _1, _2, _3, 5), FunctionData::flagMarkedAsPure); define_builtin_func("_<_", ParamsInt2, Int, nullptr, - std::bind(compile_cmp_int, _1, _2, 4), + std::bind(compile_cmp_int, _1, _2, _3, 4), FunctionData::flagMarkedAsPure); define_builtin_func("_>_", ParamsInt2, Int, nullptr, - std::bind(compile_cmp_int, _1, _2, 1), + std::bind(compile_cmp_int, _1, _2, _3, 1), FunctionData::flagMarkedAsPure); define_builtin_func("_<=_", ParamsInt2, Int, nullptr, - std::bind(compile_cmp_int, _1, _2, 6), + std::bind(compile_cmp_int, _1, _2, _3, 6), FunctionData::flagMarkedAsPure); define_builtin_func("_>=_", ParamsInt2, Int, nullptr, - std::bind(compile_cmp_int, _1, _2, 3), + std::bind(compile_cmp_int, _1, _2, _3, 3), FunctionData::flagMarkedAsPure); define_builtin_func("_<=>_", ParamsInt2, Int, nullptr, - std::bind(compile_cmp_int, _1, _2, 7), + std::bind(compile_cmp_int, _1, _2, _3, 7), FunctionData::flagMarkedAsPure); // special function used for internal compilation of some lexical constructs // for example, `throw 123;` is actually calling `__throw(123)` define_builtin_func("__true", {}, Bool, nullptr, /* AsmOp::Const("TRUE") */ - std::bind(compile_bool_const, _1, _2, true), + std::bind(compile_bool_const, _1, _2, _3, true), FunctionData::flagMarkedAsPure); define_builtin_func("__false", {}, Bool, nullptr, /* AsmOp::Const("FALSE") */ - std::bind(compile_bool_const, _1, _2, false), + std::bind(compile_bool_const, _1, _2, _3, false), FunctionData::flagMarkedAsPure); define_builtin_func("__null", {}, typeT, declGenericT, - AsmOp::Const("PUSHNULL"), + compile_push_null, FunctionData::flagMarkedAsPure); define_builtin_func("__isNull", {typeT}, Bool, declGenericT, compile_is_null, @@ -1260,35 +1278,35 @@ void define_builtins() { std::bind(compile_muldiv, _1, _2, _3, 1), FunctionData::flagMarkedAsPure); define_builtin_func("mulDivMod", ParamsInt3, TypeDataTensor::create({Int, Int}), nullptr, - AsmOp::Custom("MULDIVMOD", 3, 2), + compile_muldivmod, FunctionData::flagMarkedAsPure); define_builtin_func("loadInt", ParamsSliceInt, Int, nullptr, - std::bind(compile_fetch_int, _1, _2, true, true), + std::bind(compile_fetch_int, _1, _2, _3, true, true), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, {}, {1, 0}); define_builtin_func("loadUint", ParamsSliceInt, Int, nullptr, - std::bind(compile_fetch_int, _1, _2, true, false), + std::bind(compile_fetch_int, _1, _2, _3, true, false), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, {}, {1, 0}); define_builtin_func("loadBits", ParamsSliceInt, Slice, nullptr, - std::bind(compile_fetch_slice, _1, _2, true), + std::bind(compile_fetch_slice, _1, _2, _3, true), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, {}, {1, 0}); define_builtin_func("preloadInt", ParamsSliceInt, Int, nullptr, - std::bind(compile_fetch_int, _1, _2, false, true), + std::bind(compile_fetch_int, _1, _2, _3, false, true), FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); define_builtin_func("preloadUint", ParamsSliceInt, Int, nullptr, - std::bind(compile_fetch_int, _1, _2, false, false), + std::bind(compile_fetch_int, _1, _2, _3, false, false), FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); define_builtin_func("preloadBits", ParamsSliceInt, Slice, nullptr, - std::bind(compile_fetch_slice, _1, _2, false), + std::bind(compile_fetch_slice, _1, _2, _3, false), FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); define_builtin_func("storeInt", {Builder, Int, Int}, Unit, nullptr, - std::bind(compile_store_int, _1, _2, true), + std::bind(compile_store_int, _1, _2, _3, true), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, {1, 0, 2}, {}); define_builtin_func("storeUint", {Builder, Int, Int}, Unit, nullptr, - std::bind(compile_store_int, _1, _2, false), + std::bind(compile_store_int, _1, _2, _3, false), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, {1, 0, 2}, {}); define_builtin_func("tupleAt", {Tuple, Int}, typeT, declGenericT, @@ -1298,20 +1316,20 @@ void define_builtins() { compile_tuple_set_at, FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf); define_builtin_func("debugPrint", {typeT}, Unit, declGenericT, - AsmOp::Custom("s0 DUMP DROP", 1, 1), + compile_debug_print_to_string, 0); define_builtin_func("debugPrintString", {typeT}, Unit, declGenericT, - AsmOp::Custom("STRDUMP DROP", 1, 1), + compile_strdump, 0); define_builtin_func("debugDumpStack", {}, Unit, nullptr, - AsmOp::Custom("DUMPSTK", 0, 0), + compile_dumpstk, 0); // functions not presented in stdlib at all // used in tolk-tester to check/expose internal compiler state // each of them is handled in a special way, search by its name define_builtin_func("__expect_type", {TypeDataUnknown::create(), Slice}, Unit, nullptr, - AsmOp::Nop(), + compile_expect_type, FunctionData::flagMarkedAsPure); } diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp index ac1cf63916..6c970269e5 100644 --- a/tolk/codegen.cpp +++ b/tolk/codegen.cpp @@ -82,38 +82,41 @@ void Stack::forget_const() { } } -void Stack::issue_pop(int i) { +void Stack::issue_pop(SrcLocation loc, int i) { validate(i); if (output_enabled()) { - o << AsmOp::Pop(i); + o << AsmOp::Pop(loc, i); } at(i) = get(0); s.pop_back(); modified(); + opt_show(); } -void Stack::issue_push(int i) { +void Stack::issue_push(SrcLocation loc, int i) { validate(i); if (output_enabled()) { - o << AsmOp::Push(i); + o << AsmOp::Push(loc, i); } s.push_back(get(i)); modified(); + opt_show(); } -void Stack::issue_xchg(int i, int j) { +void Stack::issue_xchg(SrcLocation loc, int i, int j) { validate(i); validate(j); if (i != j && get(i) != get(j)) { if (output_enabled()) { - o << AsmOp::Xchg(i, j); + o << AsmOp::Xchg(loc, i, j); } std::swap(at(i), at(j)); modified(); + opt_show(); } } -int Stack::drop_vars_except(const VarDescrList& var_info, int excl_var) { +int Stack::drop_vars_except(SrcLocation loc, const VarDescrList& var_info, int excl_var) { int dropped = 0, changes; do { changes = 0; @@ -122,7 +125,7 @@ int Stack::drop_vars_except(const VarDescrList& var_info, int excl_var) { var_idx_t idx = at(i).first; if (((!var_info[idx] || var_info[idx]->is_unused()) && idx != excl_var) || find(idx, 0, i - 1) >= 0) { // unneeded - issue_pop(i); + issue_pop(loc, i); changes = 1; break; } @@ -138,7 +141,7 @@ void Stack::show() { os << ' '; o.show_var_ext(os, i); } - o << AsmOp::Comment(os.str()); + o << AsmOp::Comment({}, os.str()); mode |= _Shown; } @@ -172,17 +175,17 @@ void Stack::assign_var(var_idx_t new_idx, var_idx_t old_idx) { } } -void Stack::do_copy_var(var_idx_t new_idx, var_idx_t old_idx) { +void Stack::do_copy_var(SrcLocation loc, var_idx_t new_idx, var_idx_t old_idx) { int i = find(old_idx); tolk_assert(i >= 0 && "variable not found in stack"); if (find(old_idx, i + 1) < 0) { - issue_push(i); + issue_push(loc, i); tolk_assert(at(0).first == old_idx); } assign_var(new_idx, old_idx); } -void Stack::enforce_state(const StackLayout& req_stack) { +void Stack::enforce_state(SrcLocation loc, const StackLayout& req_stack) { int k = (int)req_stack.size(); for (int i = 0; i < k; i++) { var_idx_t x = req_stack[i]; @@ -191,18 +194,18 @@ void Stack::enforce_state(const StackLayout& req_stack) { } while (depth() > 0 && std::find(req_stack.cbegin(), req_stack.cend(), get(0).first) == req_stack.cend()) { // current TOS entry is unused in req_stack, drop it - issue_pop(0); + issue_pop(loc, 0); } int j = find(x); if (j >= depth() - i) { - issue_push(j); + issue_push(loc, j); j = 0; } - issue_xchg(j, depth() - i - 1); + issue_xchg(loc, j, depth() - i - 1); tolk_assert(s[i].first == x); } while (depth() > k) { - issue_pop(0); + issue_pop(loc, 0); } tolk_assert(depth() == k); for (int i = 0; i < k; i++) { @@ -220,12 +223,12 @@ void Stack::merge_const(const Stack& req_stack) { } } -void Stack::merge_state(const Stack& req_stack) { - enforce_state(req_stack.vars()); +void Stack::merge_state(SrcLocation loc, const Stack& req_stack) { + enforce_state(loc, req_stack.vars()); merge_const(req_stack); } -void Stack::rearrange_top(const StackLayout& top, std::vector last) { +void Stack::rearrange_top(SrcLocation loc, const StackLayout& top, std::vector last) { while (last.size() < top.size()) { last.push_back(false); } @@ -250,24 +253,24 @@ void Stack::rearrange_top(const StackLayout& top, std::vector last) { int j = find_outside(x, ss, ss + i); if (last[i]) { // rearrange x to be at s(ss-1) - issue_xchg(--ss, j); + issue_xchg(loc, --ss, j); tolk_assert(get(ss).first == x); } else { // create a new copy of x - issue_push(j); - issue_xchg(0, ss); + issue_push(loc, j); + issue_xchg(loc, 0, ss); tolk_assert(get(ss).first == x); } } tolk_assert(!ss); } -void Stack::rearrange_top(var_idx_t top, bool last) { +void Stack::rearrange_top(SrcLocation loc, var_idx_t top, bool last) { int i = find(top); if (last) { - issue_xchg(0, i); + issue_xchg(loc, 0, i); } else { - issue_push(i); + issue_push(loc, i); } tolk_assert(get(0).first == top); } @@ -280,7 +283,7 @@ bool Op::generate_code_step(Stack& stack) { bool will_now_immediate_throw = (cl == _Call && f_sym->is_builtin_function() && f_sym->name == "__throw") || (cl == _IntConst && next->cl == _Call && next->f_sym->is_builtin_function() && next->f_sym->name == "__throw"); if (!will_now_immediate_throw) { - stack.drop_vars_except(var_info); + stack.drop_vars_except(loc, var_info); stack.opt_show(); } @@ -290,9 +293,9 @@ bool Op::generate_code_step(Stack& stack) { case _Import: return true; case _Return: { - stack.enforce_state(left); + stack.enforce_state(loc, left); if (stack.o.retalt_ && (stack.mode & Stack::_NeedRetAlt)) { - stack.o << "RETALT"; + stack.o << AsmOp::Custom(loc, "RETALT"); stack.o.retalt_inserted_ = true; } stack.opt_show(); @@ -306,11 +309,11 @@ bool Op::generate_code_step(Stack& stack) { auto cidx = stack.o.register_const(int_const); int i = stack.find_const(cidx); if (i < 0) { - stack.o << push_const(int_const); + stack.o << push_const(loc, int_const); stack.push_new_const(left[0], cidx); } else { tolk_assert(stack.at(i).second == cidx); - stack.do_copy_var(left[0], stack[i]); + stack.do_copy_var(loc, left[0], stack[i]); } return true; } @@ -319,7 +322,7 @@ bool Op::generate_code_step(Stack& stack) { if (!p || p->is_unused()) { return true; } - stack.o << AsmOp::Const("x{" + str_const + "} PUSHSLICE"); + stack.o << AsmOp::Const(loc, "x{" + str_const + "} PUSHSLICE"); stack.push_new_var(left[0]); return true; } @@ -335,10 +338,10 @@ bool Op::generate_code_step(Stack& stack) { if (!used || disabled()) { return true; } - stack.o << AsmOp::Custom(g_sym->name + " GETGLOB", 0, 1); + stack.o << AsmOp::Custom(loc, g_sym->name + " GETGLOB", 0, 1); if (left.size() != 1) { tolk_assert(left.size() <= 15); - stack.o << AsmOp::UnTuple((int)left.size()); + stack.o << AsmOp::UnTuple(loc, (int)left.size()); } for (auto i : left) { stack.push_new_var(i); @@ -350,7 +353,7 @@ bool Op::generate_code_step(Stack& stack) { if (!p || p->is_unused() || disabled()) { return true; } - stack.o << "CONT:<{"; + stack.o << AsmOp::Custom(loc, "CONT:<{"); stack.o.indent(); if (f_sym->is_asm_function() || f_sym->is_builtin_function()) { // TODO: create and compile a true lambda instead of this (so that arg_order and ret_order would work correctly) @@ -368,15 +371,15 @@ bool Op::generate_code_step(Stack& stack) { args0.emplace_back(0); } if (f_sym->is_asm_function()) { - std::get(f_sym->body)->compile(stack.o); // compile res := f (args0) + std::get(f_sym->body)->compile(stack.o, loc); // compile res := f (args0) } else { - std::get(f_sym->body)->compile(stack.o, res, args0, where); // compile res := f (args0) + std::get(f_sym->body)->compile(stack.o, res, args0, loc); // compile res := f (args0) } } else { - stack.o << AsmOp::Custom(f_sym->name + " CALLDICT", (int)right.size(), (int)left.size()); + stack.o << AsmOp::Custom(loc, f_sym->name + " CALLDICT", (int)right.size(), (int)left.size()); } stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); stack.push_new_var(left.at(0)); return true; } @@ -408,7 +411,7 @@ bool Op::generate_code_step(Stack& stack) { if (is_last) { stack.assign_var(--i, x); } else { - stack.do_copy_var(--i, x); + stack.do_copy_var(loc, --i, x); } } i = 0; @@ -428,15 +431,15 @@ bool Op::generate_code_step(Stack& stack) { for (var_idx_t x : right) { last.push_back(var_info[x] && var_info[x]->is_last()); } - stack.rearrange_top(right, std::move(last)); + stack.rearrange_top(loc, right, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right.size(); tolk_assert(k >= 0); if (cl == _Tuple) { - stack.o << AsmOp::Tuple((int)right.size()); + stack.o << AsmOp::Tuple(loc, (int)right.size()); tolk_assert(left.size() == 1); } else { - stack.o << AsmOp::UnTuple((int)left.size()); + stack.o << AsmOp::UnTuple(loc, (int)left.size()); tolk_assert(right.size() == 1); } stack.s.resize(k); @@ -475,24 +478,21 @@ bool Op::generate_code_step(Stack& stack) { for (var_idx_t x : right1) { last.push_back(var_info[x] && var_info[x]->is_last()); } - stack.rearrange_top(right1, std::move(last)); + stack.rearrange_top(loc, right1, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right1.size(); tolk_assert(k >= 0); for (int i = 0; i < (int)right1.size(); i++) { - if (stack.s[k + i].first != right1[i]) { - std::cerr << stack.o; - } tolk_assert(stack.s[k + i].first == right1[i]); } auto exec_callxargs = [&](int args, int ret) { if (args <= 15 && ret <= 15) { - stack.o << exec_arg2_op("CALLXARGS", args, ret, args + 1, ret); + stack.o << exec_arg2_op(loc, "CALLXARGS", args, ret, args + 1, ret); } else { tolk_assert(args <= 254 && ret <= 254); - stack.o << AsmOp::Const(PSTRING() << args << " PUSHINT"); - stack.o << AsmOp::Const(PSTRING() << ret << " PUSHINT"); - stack.o << AsmOp::Custom("CALLXVARARGS", args + 3, ret); + stack.o << AsmOp::Const(loc, PSTRING() << args << " PUSHINT"); + stack.o << AsmOp::Const(loc, PSTRING() << ret << " PUSHINT"); + stack.o << AsmOp::Custom(loc, "CALLXVARARGS", args + 3, ret); } }; if (cl == _CallInd) { @@ -504,20 +504,21 @@ bool Op::generate_code_step(Stack& stack) { res.emplace_back(i); } if (f_sym->is_asm_function()) { - std::get(f_sym->body)->compile(stack.o); // compile res := f (args) + std::get(f_sym->body)->compile(stack.o, loc); // compile res := f (args) } else { - std::get(f_sym->body)->compile(stack.o, res, args, where); // compile res := f (args) + std::get(f_sym->body)->compile(stack.o, res, args, loc); // compile res := f (args) } } else { if (f_sym->is_inline() || f_sym->is_inline_ref()) { - stack.o << AsmOp::Custom(f_sym->name + " INLINECALLDICT", (int)right.size(), (int)left.size()); + stack.o << AsmOp::Custom(loc, f_sym->name + " INLINECALLDICT", (int)right.size(), (int)left.size()); } else if (f_sym->is_code_function() && std::get(f_sym->body)->code->require_callxargs) { - stack.o << AsmOp::Custom(f_sym->name + (" PREPAREDICT"), 0, 2); + stack.o << AsmOp::Custom(loc, f_sym->name + (" PREPAREDICT"), 0, 2); exec_callxargs((int)right.size() + 1, (int)left.size()); } else { - stack.o << AsmOp::Custom(f_sym->name + " CALLDICT", (int)right.size(), (int)left.size()); + stack.o << AsmOp::Custom(loc, f_sym->name + " CALLDICT", (int)right.size(), (int)left.size()); } } + stack.modified(); stack.s.resize(k); for (int i = 0; i < (int)left.size(); i++) { int j = ret_order ? ret_order->at(i) : i; @@ -531,21 +532,19 @@ bool Op::generate_code_step(Stack& stack) { for (var_idx_t x : right) { last.push_back(var_info[x] && var_info[x]->is_last()); } - stack.rearrange_top(right, std::move(last)); + stack.rearrange_top(loc, right, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right.size(); tolk_assert(k >= 0); for (int i = 0; i < (int)right.size(); i++) { - if (stack.s[k + i].first != right[i]) { - std::cerr << stack.o; - } tolk_assert(stack.s[k + i].first == right[i]); } if (right.size() > 1) { - stack.o << AsmOp::Tuple((int)right.size()); + stack.o << AsmOp::Tuple(loc, (int)right.size()); } if (!right.empty()) { - stack.o << AsmOp::Custom(g_sym->name + " SETGLOB", 1, 0); + stack.o << AsmOp::Custom(loc, g_sym->name + " SETGLOB", 1, 0); + stack.modified(); } stack.s.resize(k); return true; @@ -558,7 +557,7 @@ bool Op::generate_code_step(Stack& stack) { stack.o.retalt_ = true; } var_idx_t x = left[0]; - stack.rearrange_top(x, var_info[x] && var_info[x]->is_last()); + stack.rearrange_top(loc, x, var_info[x] && var_info[x]->is_last()); tolk_assert(stack[0] == x); stack.opt_show(); stack.s.pop_back(); @@ -568,19 +567,19 @@ bool Op::generate_code_step(Stack& stack) { Op* block_noreturn = is0 ? block0.get() : block1.get(); Op* block_other = is0 ? block1.get() : block0.get(); stack.mode &= ~Stack::_InlineFunc; - stack.o << (is0 ? "IF:<{" : "IFNOT:<{"); + stack.o << AsmOp::Custom(loc, is0 ? "IF:<{" : "IFNOT:<{"); stack.o.indent(); Stack stack_copy{stack}; block_noreturn->generate_code_all(stack_copy); stack.o.undent(); - stack.o << "}>ELSE<{"; + stack.o << AsmOp::Custom({}, "}>ELSE<{"); stack.o.indent(); block_other->generate_code_all(stack); if (!block_other->noreturn()) { next->generate_code_all(stack); } stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return false; } if (block1->is_empty() || block0->is_empty()) { @@ -589,88 +588,88 @@ bool Op::generate_code_step(Stack& stack) { // if (left) block0; ... // if (!left) block1; ... if (block->noreturn()) { - stack.o << (is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); + stack.o << AsmOp::Custom(loc, is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); stack.o.indent(); Stack stack_copy{stack}; stack_copy.mode &= ~Stack::_InlineFunc; stack_copy.mode |= next->noreturn() ? 0 : Stack::_NeedRetAlt; block->generate_code_all(stack_copy); stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return true; } - stack.o << (is0 ? "IF:<{" : "IFNOT:<{"); + stack.o << AsmOp::Custom(loc, is0 ? "IF:<{" : "IFNOT:<{"); stack.o.indent(); Stack stack_copy{stack}, stack_target{stack}; stack_target.disable_output(); - stack_target.drop_vars_except(next->var_info); + stack_target.drop_vars_except(loc, next->var_info); stack_copy.mode &= ~Stack::_InlineFunc; block->generate_code_all(stack_copy); - stack_copy.drop_vars_except(var_info); + stack_copy.drop_vars_except(loc, var_info); stack_copy.opt_show(); if ((is0 && stack_copy == stack) || (!is0 && stack_copy.vars() == stack.vars())) { stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); if (!is0) { stack.merge_const(stack_copy); } return true; } // stack_copy.drop_vars_except(next->var_info); - stack_copy.enforce_state(stack_target.vars()); + stack_copy.enforce_state(loc, stack_target.vars()); stack_copy.opt_show(); if (stack_copy.vars() == stack.vars()) { stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); stack.merge_const(stack_copy); return true; } stack.o.undent(); - stack.o << "}>ELSE<{"; + stack.o << AsmOp::Custom({}, "}>ELSE<{"); stack.o.indent(); - stack.merge_state(stack_copy); + stack.merge_state(loc, stack_copy); stack.opt_show(); stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return true; } if (block0->noreturn() || block1->noreturn()) { bool is0 = block0->noreturn(); Op* block_noreturn = is0 ? block0.get() : block1.get(); Op* block_other = is0 ? block1.get() : block0.get(); - stack.o << (is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); + stack.o << AsmOp::Custom(loc, is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); stack.o.indent(); Stack stack_copy{stack}; stack_copy.mode &= ~Stack::_InlineFunc; stack_copy.mode |= (block_other->noreturn() || next->noreturn()) ? 0 : Stack::_NeedRetAlt; block_noreturn->generate_code_all(stack_copy); stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); block_other->generate_code_all(stack); return !block_other->noreturn(); } - stack.o << "IF:<{"; + stack.o << AsmOp::Custom(loc, "IF:<{"); stack.o.indent(); Stack stack_copy{stack}; stack_copy.mode &= ~Stack::_InlineFunc; block0->generate_code_all(stack_copy); - stack_copy.drop_vars_except(next->var_info); + stack_copy.drop_vars_except(loc, next->var_info); stack_copy.opt_show(); stack.o.undent(); - stack.o << "}>ELSE<{"; + stack.o << AsmOp::Custom({}, "}>ELSE<{"); stack.o.indent(); stack.mode &= ~Stack::_InlineFunc; block1->generate_code_all(stack); - stack.merge_state(stack_copy); + stack.merge_state(loc, stack_copy); stack.opt_show(); stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return true; } case _Repeat: { var_idx_t x = left[0]; //stack.drop_vars_except(block0->var_info, x); - stack.rearrange_top(x, var_info[x] && var_info[x]->is_last()); + stack.rearrange_top(loc, x, var_info[x] && var_info[x]->is_last()); tolk_assert(stack[0] == x); stack.opt_show(); stack.s.pop_back(); @@ -679,7 +678,7 @@ bool Op::generate_code_step(Stack& stack) { stack.o.retalt_ = true; } if (true || !next->is_empty()) { - stack.o << "REPEAT:<{"; + stack.o << AsmOp::Custom(loc, "REPEAT:<{"); stack.o.indent(); stack.forget_const(); if (block0->noreturn()) { @@ -693,47 +692,47 @@ bool Op::generate_code_step(Stack& stack) { stack.mode &= ~Stack::_InlineFunc; stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); - stack.enforce_state(std::move(layout1)); + stack.enforce_state(loc, std::move(layout1)); stack.opt_show(); } stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return true; } else { - stack.o << "REPEATEND"; + stack.o << AsmOp::Custom(loc, "REPEATEND"); stack.forget_const(); StackLayout layout1 = stack.vars(); block0->generate_code_all(stack); - stack.enforce_state(std::move(layout1)); + stack.enforce_state(loc, std::move(layout1)); stack.opt_show(); return false; } } case _Again: { - stack.drop_vars_except(block0->var_info); + stack.drop_vars_except(loc, block0->var_info); stack.opt_show(); if (block0->noreturn()) { stack.o.retalt_ = true; } if (!next->is_empty() || inline_func) { - stack.o << "AGAIN:<{"; + stack.o << AsmOp::Custom(loc, "AGAIN:<{"); stack.o.indent(); stack.forget_const(); StackLayout layout1 = stack.vars(); stack.mode &= ~Stack::_InlineFunc; stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); - stack.enforce_state(std::move(layout1)); + stack.enforce_state(loc, std::move(layout1)); stack.opt_show(); stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return true; } else { - stack.o << "AGAINEND"; + stack.o << AsmOp::Custom(loc, "AGAINEND"); stack.forget_const(); StackLayout layout1 = stack.vars(); block0->generate_code_all(stack); - stack.enforce_state(std::move(layout1)); + stack.enforce_state(loc, std::move(layout1)); stack.opt_show(); return false; } @@ -745,7 +744,7 @@ bool Op::generate_code_step(Stack& stack) { stack.o.retalt_ = true; } if (true || !next->is_empty()) { - stack.o << "UNTIL:<{"; + stack.o << AsmOp::Custom(loc, "UNTIL:<{"); stack.o.indent(); stack.forget_const(); auto layout1 = stack.vars(); @@ -753,20 +752,20 @@ bool Op::generate_code_step(Stack& stack) { stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); layout1.push_back(left[0]); - stack.enforce_state(std::move(layout1)); + stack.enforce_state(loc, std::move(layout1)); stack.opt_show(); stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); stack.s.pop_back(); stack.modified(); return true; } else { - stack.o << "UNTILEND"; + stack.o << AsmOp::Custom(loc, "UNTILEND"); stack.forget_const(); StackLayout layout1 = stack.vars(); block0->generate_code_all(stack); layout1.push_back(left[0]); - stack.enforce_state(std::move(layout1)); + stack.enforce_state(loc, std::move(layout1)); stack.opt_show(); return false; } @@ -774,36 +773,36 @@ bool Op::generate_code_step(Stack& stack) { case _While: { // while (block0 | left) block1; ...next var_idx_t x = left[0]; - stack.drop_vars_except(block0->var_info); + stack.drop_vars_except(loc, block0->var_info); stack.opt_show(); StackLayout layout1 = stack.vars(); bool next_empty = false && next->is_empty(); if (block0->noreturn()) { stack.o.retalt_ = true; } - stack.o << "WHILE:<{"; + stack.o << AsmOp::Custom(loc, "WHILE:<{"); stack.o.indent(); stack.forget_const(); stack.mode &= ~Stack::_InlineFunc; stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); - stack.rearrange_top(x, !next->var_info[x] && !block1->var_info[x]); + stack.rearrange_top(loc, x, !next->var_info[x] && !block1->var_info[x]); stack.opt_show(); stack.s.pop_back(); stack.modified(); stack.o.undent(); Stack stack_copy{stack}; - stack.o << (next_empty ? "}>DO:" : "}>DO<{"); + stack.o << AsmOp::Custom(loc, next_empty ? "}>DO:" : "}>DO<{"); if (!next_empty) { stack.o.indent(); } stack_copy.opt_show(); block1->generate_code_all(stack_copy); - stack_copy.enforce_state(std::move(layout1)); + stack_copy.enforce_state(loc, std::move(layout1)); stack_copy.opt_show(); if (!next_empty) { stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return true; } else { return false; @@ -816,7 +815,7 @@ bool Op::generate_code_step(Stack& stack) { if (block0->noreturn() || block1->noreturn()) { stack.o.retalt_ = true; } - Stack catch_stack{stack.o}; + Stack catch_stack{stack.o, 0}; std::vector catch_vars; std::vector catch_last; for (const VarDescr& var : block1->var_info.list) { @@ -834,37 +833,37 @@ bool Op::generate_code_step(Stack& stack) { } catch_stack.push_new_var(left[0]); catch_stack.push_new_var(left[1]); - stack.rearrange_top(catch_vars, catch_last); + stack.rearrange_top(loc, catch_vars, catch_last); stack.opt_show(); - stack.o << "c1 PUSH"; - stack.o << "c3 PUSH"; - stack.o << "c4 PUSH"; - stack.o << "c5 PUSH"; - stack.o << "c7 PUSH"; - stack.o << "<{"; + stack.o << AsmOp::Custom(loc, "c1 PUSH"); + stack.o << AsmOp::Custom(loc, "c3 PUSH"); + stack.o << AsmOp::Custom(loc, "c4 PUSH"); + stack.o << AsmOp::Custom(loc, "c5 PUSH"); + stack.o << AsmOp::Custom(loc, "c7 PUSH"); + stack.o << AsmOp::Custom(loc, "<{"); stack.o.indent(); if (block1->noreturn()) { catch_stack.mode |= Stack::_NeedRetAlt; } block1->generate_code_all(catch_stack); - catch_stack.drop_vars_except(next->var_info); + catch_stack.drop_vars_except(loc, next->var_info); catch_stack.opt_show(); stack.o.undent(); - stack.o << "}>CONT"; - stack.o << "c7 SETCONT"; - stack.o << "c5 SETCONT"; - stack.o << "c4 SETCONT"; - stack.o << "c3 SETCONT"; - stack.o << "c1 SETCONT"; + stack.o << AsmOp::Custom({}, "}>CONT"); + stack.o << AsmOp::Custom(loc, "c7 SETCONT"); + stack.o << AsmOp::Custom(loc, "c5 SETCONT"); + stack.o << AsmOp::Custom(loc, "c4 SETCONT"); + stack.o << AsmOp::Custom(loc, "c3 SETCONT"); + stack.o << AsmOp::Custom(loc, "c1 SETCONT"); for (size_t begin = catch_vars.size(), end = begin; end > 0; end = begin) { begin = end >= block_size ? end - block_size : 0; - stack.o << std::to_string(end - begin) + " PUSHINT"; - stack.o << "-1 PUSHINT"; - stack.o << "SETCONTVARARGS"; + stack.o << AsmOp::Custom(loc, std::to_string(end - begin) + " PUSHINT"); + stack.o << AsmOp::Custom(loc, "-1 PUSHINT"); + stack.o << AsmOp::Custom(loc, "SETCONTVARARGS"); } stack.s.erase(stack.s.end() - catch_vars.size(), stack.s.end()); stack.modified(); - stack.o << "<{"; + stack.o << AsmOp::Custom(loc, "<{"); stack.o.indent(); if (block0->noreturn()) { stack.mode |= Stack::_NeedRetAlt; @@ -873,20 +872,20 @@ bool Op::generate_code_step(Stack& stack) { if (block0->noreturn()) { stack.s = std::move(catch_stack.s); } else if (!block1->noreturn()) { - stack.merge_state(catch_stack); + stack.merge_state(loc, catch_stack); } stack.opt_show(); stack.o.undent(); - stack.o << "}>CONT"; - stack.o << "c1 PUSH"; - stack.o << "COMPOSALT"; - stack.o << "SWAP"; - stack.o << "TRY"; + stack.o << AsmOp::Custom({}, "}>CONT"); + stack.o << AsmOp::Custom(loc, "c1 PUSH"); + stack.o << AsmOp::Custom(loc, "COMPOSALT"); + stack.o << AsmOp::Custom(loc, "SWAP"); + stack.o << AsmOp::Custom(loc, "TRY"); return true; } default: std::cerr << "fatal: unknown operation \n"; - throw ParseError{where, "unknown operation in generate_code()"}; + throw ParseError(loc, "unknown operation in generate_code()"); } } @@ -899,20 +898,16 @@ void Op::generate_code_all(Stack& stack) { } } -void CodeBlob::generate_code(AsmOpList& out, int mode) { - Stack stack{out, mode}; +void CodeBlob::generate_code(std::ostream& os, int mode, int indent) const { + AsmOpList out_list(indent, &vars); + Stack stack{out_list, mode}; tolk_assert(ops && ops->cl == Op::_Import); - auto args = (int)ops->left.size(); + int n_import_width = static_cast(ops->left.size()); for (var_idx_t x : ops->left) { stack.push_new_var(x); } ops->generate_code_all(stack); - stack.apply_wrappers(require_callxargs && (mode & Stack::_InlineAny) ? args : -1); -} - -void CodeBlob::generate_code(std::ostream& os, int mode, int indent) { - AsmOpList out_list(indent, &vars); - generate_code(out_list, mode); + stack.apply_wrappers(fun_ref->loc, require_callxargs && (mode & Stack::_InlineAny) ? n_import_width : -1); if (G.settings.optimization_level >= 2) { optimize_code(out_list); } diff --git a/tolk/compiler-state.h b/tolk/compiler-state.h index b39e62fb3f..9b67293b57 100644 --- a/tolk/compiler-state.h +++ b/tolk/compiler-state.h @@ -52,6 +52,7 @@ struct CompilerSettings { int verbosity = 0; int optimization_level = 2; bool stack_layout_comments = true; + bool tolk_src_as_line_comments = true; std::string output_filename; std::string boc_output_filename; diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index 76d7563868..332069c03f 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -125,13 +125,6 @@ void Optimizer::show_right() const { std::cerr << std::endl; } -bool Optimizer::say(std::string str) const { - if (debug_) { - std::cerr << str << std::endl; - } - return true; -} - bool Optimizer::find_const_op(int* op_idx, int cst) { for (int i = 0; i < l2_; i++) { if (op_[i]->is_gconst() && tr_[i].get(0) == cst) { @@ -157,7 +150,7 @@ bool Optimizer::rewrite_push_const(int i, int c) { show_left(); oq_[1] = std::move(op_[idx]); oq_[0] = std::move(op_[!idx]); - *oq_[0] = AsmOp::Push(i); + *oq_[0] = AsmOp::Push(oq_[0]->loc, i); show_right(); return true; } @@ -176,7 +169,7 @@ bool Optimizer::rewrite_const_rot(int c) { show_left(); oq_[0] = std::move(op_[idx]); oq_[1] = std::move(op_[!idx]); - *oq_[1] = AsmOp::Custom("ROT", 3, 3); + *oq_[1] = AsmOp::Custom(oq_[0]->loc, "ROT", 3, 3); show_right(); return true; } @@ -195,7 +188,7 @@ bool Optimizer::rewrite_const_pop(int c, int i) { show_left(); oq_[0] = std::move(op_[idx]); oq_[1] = std::move(op_[!idx]); - *oq_[1] = AsmOp::Pop(i); + *oq_[1] = AsmOp::Pop(oq_[0]->loc, i); show_right(); return true; } @@ -553,40 +546,41 @@ bool Optimizer::find_at_least(int pb) { pb_ = pb; // show_stack_transforms(); int i, j, k, l, c; + SrcLocation loc; // for asm ops inserted by optimizer, leave location empty (in fift output, it'll be attached to above) return (is_push_const(&i, &c) && rewrite_push_const(i, c)) || (is_nop() && rewrite_nop()) || (!(mode_ & 1) && is_const_rot(&c) && rewrite_const_rot(c)) || (is_const_push_xchgs() && rewrite_const_push_xchgs()) || (is_const_pop(&c, &i) && rewrite_const_pop(c, i)) || - (is_xchg(&i, &j) && rewrite(AsmOp::Xchg(i, j))) || (is_push(&i) && rewrite(AsmOp::Push(i))) || - (is_pop(&i) && rewrite(AsmOp::Pop(i))) || (is_pop_pop(&i, &j) && rewrite(AsmOp::Pop(i), AsmOp::Pop(j))) || - (is_xchg_xchg(&i, &j, &k, &l) && rewrite(AsmOp::Xchg(i, j), AsmOp::Xchg(k, l))) || + (is_xchg(&i, &j) && rewrite(AsmOp::Xchg(loc, i, j))) || (is_push(&i) && rewrite(AsmOp::Push(loc, i))) || + (is_pop(&i) && rewrite(AsmOp::Pop(loc, i))) || (is_pop_pop(&i, &j) && rewrite(AsmOp::Pop(loc, i), AsmOp::Pop(loc, j))) || + (is_xchg_xchg(&i, &j, &k, &l) && rewrite(AsmOp::Xchg(loc, i, j), AsmOp::Xchg(loc, k, l))) || (!(mode_ & 1) && - ((is_rot() && rewrite(AsmOp::Custom("ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom("-ROT", 3, 3))) || - (is_2dup() && rewrite(AsmOp::Custom("2DUP", 2, 4))) || - (is_2swap() && rewrite(AsmOp::Custom("2SWAP", 2, 4))) || - (is_2over() && rewrite(AsmOp::Custom("2OVER", 2, 4))) || - (is_tuck() && rewrite(AsmOp::Custom("TUCK", 2, 3))) || - (is_2drop() && rewrite(AsmOp::Custom("2DROP", 2, 0))) || (is_xchg2(&i, &j) && rewrite(AsmOp::Xchg2(i, j))) || - (is_xcpu(&i, &j) && rewrite(AsmOp::XcPu(i, j))) || (is_puxc(&i, &j) && rewrite(AsmOp::PuXc(i, j))) || - (is_push2(&i, &j) && rewrite(AsmOp::Push2(i, j))) || (is_blkswap(&i, &j) && rewrite(AsmOp::BlkSwap(i, j))) || - (is_blkpush(&i, &j) && rewrite(AsmOp::BlkPush(i, j))) || (is_blkdrop(&i) && rewrite(AsmOp::BlkDrop(i))) || - (is_push_rot(&i) && rewrite(AsmOp::Push(i), AsmOp::Custom("ROT"))) || - (is_push_rotrev(&i) && rewrite(AsmOp::Push(i), AsmOp::Custom("-ROT"))) || - (is_push_xchg(&i, &j, &k) && rewrite(AsmOp::Push(i), AsmOp::Xchg(j, k))) || - (is_reverse(&i, &j) && rewrite(AsmOp::BlkReverse(i, j))) || - (is_blkdrop2(&i, &j) && rewrite(AsmOp::BlkDrop2(i, j))) || - (is_nip_seq(&i, &j) && rewrite(AsmOp::Xchg(i, j), AsmOp::BlkDrop(i))) || - (is_pop_blkdrop(&i, &k) && rewrite(AsmOp::Pop(i), AsmOp::BlkDrop(k))) || + ((is_rot() && rewrite(AsmOp::Custom(loc, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(loc, "-ROT", 3, 3))) || + (is_2dup() && rewrite(AsmOp::Custom(loc, "2DUP", 2, 4))) || + (is_2swap() && rewrite(AsmOp::Custom(loc, "2SWAP", 2, 4))) || + (is_2over() && rewrite(AsmOp::Custom(loc, "2OVER", 2, 4))) || + (is_tuck() && rewrite(AsmOp::Custom(loc, "TUCK", 2, 3))) || + (is_2drop() && rewrite(AsmOp::Custom(loc, "2DROP", 2, 0))) || (is_xchg2(&i, &j) && rewrite(AsmOp::Xchg2(loc, i, j))) || + (is_xcpu(&i, &j) && rewrite(AsmOp::XcPu(loc, i, j))) || (is_puxc(&i, &j) && rewrite(AsmOp::PuXc(loc, i, j))) || + (is_push2(&i, &j) && rewrite(AsmOp::Push2(loc, i, j))) || (is_blkswap(&i, &j) && rewrite(AsmOp::BlkSwap(loc, i, j))) || + (is_blkpush(&i, &j) && rewrite(AsmOp::BlkPush(loc, i, j))) || (is_blkdrop(&i) && rewrite(AsmOp::BlkDrop(loc, i))) || + (is_push_rot(&i) && rewrite(AsmOp::Push(loc, i), AsmOp::Custom(loc, "ROT"))) || + (is_push_rotrev(&i) && rewrite(AsmOp::Push(loc, i), AsmOp::Custom(loc, "-ROT"))) || + (is_push_xchg(&i, &j, &k) && rewrite(AsmOp::Push(loc, i), AsmOp::Xchg(loc, j, k))) || + (is_reverse(&i, &j) && rewrite(AsmOp::BlkReverse(loc, i, j))) || + (is_blkdrop2(&i, &j) && rewrite(AsmOp::BlkDrop2(loc, i, j))) || + (is_nip_seq(&i, &j) && rewrite(AsmOp::Xchg(loc, i, j), AsmOp::BlkDrop(loc, i))) || + (is_pop_blkdrop(&i, &k) && rewrite(AsmOp::Pop(loc, i), AsmOp::BlkDrop(loc, k))) || (is_2pop_blkdrop(&i, &j, &k) && (k >= 3 && k <= 13 && i != j + 1 && i <= 15 && j <= 14 - ? rewrite(AsmOp::Xchg2(j + 1, i), AsmOp::BlkDrop(k + 2)) - : rewrite(AsmOp::Pop(i), AsmOp::Pop(j), AsmOp::BlkDrop(k)))) || - (is_xchg3(&i, &j, &k) && rewrite(AsmOp::Xchg3(i, j, k))) || - (is_xc2pu(&i, &j, &k) && rewrite(AsmOp::Xc2Pu(i, j, k))) || - (is_xcpuxc(&i, &j, &k) && rewrite(AsmOp::XcPuXc(i, j, k))) || - (is_xcpu2(&i, &j, &k) && rewrite(AsmOp::XcPu2(i, j, k))) || - (is_puxc2(&i, &j, &k) && rewrite(AsmOp::PuXc2(i, j, k))) || - (is_puxcpu(&i, &j, &k) && rewrite(AsmOp::PuXcPu(i, j, k))) || - (is_pu2xc(&i, &j, &k) && rewrite(AsmOp::Pu2Xc(i, j, k))) || - (is_push3(&i, &j, &k) && rewrite(AsmOp::Push3(i, j, k))))); + ? rewrite(AsmOp::Xchg2(loc, j + 1, i), AsmOp::BlkDrop(loc, k + 2)) + : rewrite(AsmOp::Pop(loc, i), AsmOp::Pop(loc, j), AsmOp::BlkDrop(loc, k)))) || + (is_xchg3(&i, &j, &k) && rewrite(AsmOp::Xchg3(loc, i, j, k))) || + (is_xc2pu(&i, &j, &k) && rewrite(AsmOp::Xc2Pu(loc, i, j, k))) || + (is_xcpuxc(&i, &j, &k) && rewrite(AsmOp::XcPuXc(loc, i, j, k))) || + (is_xcpu2(&i, &j, &k) && rewrite(AsmOp::XcPu2(loc, i, j, k))) || + (is_puxc2(&i, &j, &k) && rewrite(AsmOp::PuXc2(loc, i, j, k))) || + (is_puxcpu(&i, &j, &k) && rewrite(AsmOp::PuXcPu(loc, i, j, k))) || + (is_pu2xc(&i, &j, &k) && rewrite(AsmOp::Pu2Xc(loc, i, j, k))) || + (is_push3(&i, &j, &k) && rewrite(AsmOp::Push3(loc, i, j, k))))); } bool Optimizer::find() { diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index cefdb5b8d6..53447277b9 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -17,6 +17,7 @@ #include "tolk.h" #include "src-file.h" #include "ast.h" +#include "ast-aux-data.h" #include "ast-visitor.h" #include "type-system.h" #include "common/refint.h" @@ -883,7 +884,9 @@ std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, Co } if (GlobalConstPtr const_ref = sym->try_as()) { tolk_assert(lval_ctx == nullptr); - return pre_compile_expr(const_ref->init_value, code, const_ref->declared_type); + ASTAuxData* aux_data = new AuxData_ForceFiftLocation(loc); + auto v_force_loc = createV(loc, const_ref->init_value, aux_data, const_ref->inferred_type); + return pre_compile_expr(v_force_loc, code, const_ref->declared_type); } if (FunctionPtr fun_ref = sym->try_as()) { std::vector rvect = code.create_tmp_var(fun_ref->inferred_full_type, loc, "(glob-var-fun)"); @@ -1327,6 +1330,7 @@ static std::vector process_object_literal(V v, Co // an object (an instance of a struct) is actually a tensor at low-level // for example, `struct User { id: int; name: slice; }` occupies 2 slots // fields of a tensor are placed in order of declaration (in a literal they might be shuffled) + SrcLocation prev_loc = v->loc; std::vector tensor_items; std::vector target_types; tensor_items.reserve(v->struct_ref->get_num_fields()); @@ -1342,10 +1346,13 @@ static std::vector process_object_literal(V v, Co } if (!v_init_val) { tolk_assert(field_ref->has_default_value()); - v_init_val = field_ref->default_value; // it may be a complex expression, but it's constant, checked earlier + ASTAuxData *aux_data = new AuxData_ForceFiftLocation(prev_loc); + auto v_force_loc = createV(v->loc, field_ref->default_value, aux_data, field_ref->declared_type); + v_init_val = v_force_loc; // it may be a complex expression, but it's a constant, checked earlier } tensor_items.push_back(v_init_val); target_types.push_back(field_ref->declared_type); + prev_loc = v_init_val->loc; } const auto* tensor_target_type = TypeDataTensor::create(std::move(target_types))->try_as(); std::vector rvect = pre_compile_tensor(code, tensor_items, lval_ctx, tensor_target_type); @@ -1406,6 +1413,18 @@ static std::vector process_empty_expression(V v return transition_to_target_type(std::move(empty_rvect), code, target_type, v); } +static std::vector process_artificial_aux_vertex(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + if (const auto* data = dynamic_cast(v->aux_data)) { + SrcLocation backup = code.forced_loc; + code.forced_loc = data->forced_loc; + std::vector rvect = pre_compile_expr(v->get_wrapped_expr(), code, target_type, lval_ctx); + code.forced_loc = backup; + return rvect; + } + + tolk_assert(false); +} + std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { switch (v->kind) { case ast_reference: @@ -1458,6 +1477,8 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ return process_underscore(v->as(), code); case ast_empty_expression: return process_empty_expression(v->as(), code, target_type); + case ast_artificial_aux_vertex: + return process_artificial_aux_vertex(v->as(), code, target_type, lval_ctx); default: throw UnexpectedASTNodeKind(v, "pre_compile_expr"); } @@ -1681,7 +1702,7 @@ void process_any_statement(AnyV v, CodeBlob& code) { static void convert_function_body_to_CodeBlob(FunctionPtr fun_ref, FunctionBodyCode* code_body) { auto v_body = fun_ref->ast_root->as()->get_body()->as(); - CodeBlob* blob = new CodeBlob{fun_ref->name, fun_ref->loc, fun_ref}; + CodeBlob* blob = new CodeBlob(fun_ref); std::vector rvect_import; int total_arg_width = 0; @@ -1720,7 +1741,7 @@ static void convert_asm_body_to_AsmOp(FunctionPtr fun_ref, FunctionBodyAsm* asm_ for (char c : ops) { if (c == '\n' || c == '\r') { if (!op.empty()) { - asm_ops.push_back(AsmOp::Parse(op, cnt, width)); + asm_ops.push_back(AsmOp::Parse({}, op, cnt, width)); if (asm_ops.back().is_custom()) { cnt = width; } @@ -1731,7 +1752,7 @@ static void convert_asm_body_to_AsmOp(FunctionPtr fun_ref, FunctionBodyAsm* asm_ } } if (!op.empty()) { - asm_ops.push_back(AsmOp::Parse(op, cnt, width)); + asm_ops.push_back(AsmOp::Parse({}, op, cnt, width)); if (asm_ops.back().is_custom()) { cnt = width; } diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp index 57f481f07f..b40ce6860d 100644 --- a/tolk/pipe-generate-fif-output.cpp +++ b/tolk/pipe-generate-fif-output.cpp @@ -84,10 +84,24 @@ static void generate_output_func(FunctionPtr fun_ref) { } else if (fun_ref->is_inline_ref()) { modifier = "REF"; } - std::cout << std::string(2, ' ') << fun_ref->name << " PROC" << modifier << ":<{\n"; + if (G.settings.tolk_src_as_line_comments) { + std::cout << " // " << fun_ref->loc << std::endl; + } + std::cout << " " << fun_ref->name << " PROC" << modifier << ":<{"; int mode = 0; if (G.settings.stack_layout_comments) { - mode |= Stack::_StkCmt | Stack::_CptStkCmt; + mode |= Stack::_StackComments; + size_t len = 2 + fun_ref->name.size() + 5 + std::strlen(modifier) + 3; + while (len < 28) { // a bit weird, but okay for now: + std::cout << ' '; // insert space after "xxx PROC" before `// stack state` + len++; // (the first AsmOp-comment that will be code generated) + } // space is the same as used to align comments in asmops.cpp + std::cout << '\t'; + } else { + std::cout << std::endl; + } + if (G.settings.tolk_src_as_line_comments) { + mode |= Stack::_LineComments; } if (fun_ref->is_inline() && code->ops->noreturn()) { mode |= Stack::_InlineFunc; @@ -96,7 +110,7 @@ static void generate_output_func(FunctionPtr fun_ref) { mode |= Stack::_InlineAny; } code->generate_code(std::cout, mode, 2); - std::cout << std::string(2, ' ') << "}>\n"; + std::cout << " " << "}>\n"; if (G.is_verbosity(2)) { std::cerr << "--------------\n"; } @@ -131,7 +145,7 @@ void pipeline_generate_fif_output_to_std_cout() { has_main_procedure = true; } - std::cout << std::string(2, ' '); + std::cout << " "; if (fun_ref->is_method_id_not_empty()) { std::cout << fun_ref->method_id << " DECLMETHOD " << fun_ref->name << "\n"; } else { @@ -151,7 +165,7 @@ void pipeline_generate_fif_output_to_std_cout() { continue; } - std::cout << std::string(2, ' ') << "DECLGLOBVAR " << var_ref->name << "\n"; + std::cout << " " << "DECLGLOBVAR " << var_ref->name << "\n"; } for (FunctionPtr fun_ref : G.all_functions) { diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 667172034b..d4cda33fb9 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -86,7 +86,7 @@ struct NameAndScopeResolver { void close_scope([[maybe_unused]] SrcLocation loc) { // std::cerr << "close_scope " << scopes.size() << " at " << loc << std::endl; if (UNLIKELY(scopes.empty())) { - throw Fatal{"cannot close the outer scope"}; + throw Fatal("cannot close the outer scope"); } scopes.pop_back(); } diff --git a/tolk/src-file.cpp b/tolk/src-file.cpp index 1286c1f9c7..d976085609 100644 --- a/tolk/src-file.cpp +++ b/tolk/src-file.cpp @@ -24,15 +24,6 @@ namespace tolk { static_assert(sizeof(SrcLocation) == 8); -const SrcFile* AllRegisteredSrcFiles::find_file(int file_id) const { - for (const SrcFile* file : all_src_files) { - if (file->file_id == file_id) { - return file; - } - } - return nullptr; -} - const SrcFile* AllRegisteredSrcFiles::find_file(const std::string& abs_filename) const { for (const SrcFile* file : all_src_files) { if (file->abs_filename == abs_filename) { @@ -64,7 +55,8 @@ const SrcFile* AllRegisteredSrcFiles::locate_and_register_source_file(const std: throw Fatal("Failed to read " + rel_filename + ": " + text.move_as_error().message().str()); } - SrcFile* created = new SrcFile(++last_registered_file_id, rel_filename, std::move(abs_filename), text.move_as_ok()); + int file_id = static_cast(all_src_files.size()); // SrcFile::file_id is the index in all files + SrcFile* created = new SrcFile(file_id, rel_filename, std::move(abs_filename), text.move_as_ok()); if (G.is_verbosity(1)) { std::cerr << "register file_id " << created->file_id << " " << created->abs_filename << std::endl; } @@ -73,6 +65,7 @@ const SrcFile* AllRegisteredSrcFiles::locate_and_register_source_file(const std: } SrcFile* AllRegisteredSrcFiles::get_next_unparsed_file() { + int last_registered_file_id = static_cast(all_src_files.size() - 1); if (last_parsed_file_id >= last_registered_file_id) { return nullptr; } @@ -93,6 +86,10 @@ SrcFile::SrcPosition SrcFile::convert_offset(int offset) const { return SrcPosition{offset, -1, -1, "invalid offset"}; } + // currently, converting offset to line number is O(N): just read file contents char by char and detect lines + // since original Tolk src lines are now printed into Fift output, this is invoked for every asm instruction + // but anyway, it consumes a small amount of time relative to other work of the compiler + // in the future, it can be optimized by making lines index aside just std::string_view text int line_idx = 0; int char_idx = 0; int line_offset = 0; @@ -129,7 +126,7 @@ std::ostream& operator<<(std::ostream& os, const Fatal& fatal) { } const SrcFile* SrcLocation::get_src_file() const { - return G.all_src_files.find_file(file_id); + return G.all_src_files.get_file(file_id); } void SrcLocation::show(std::ostream& os) const { @@ -137,7 +134,10 @@ void SrcLocation::show(std::ostream& os) const { os << src_file; if (src_file && src_file->is_offset_valid(char_offset)) { SrcFile::SrcPosition pos = src_file->convert_offset(char_offset); - os << ':' << pos.line_no << ':' << pos.char_no; + os << ':' << pos.line_no; + if (pos.char_no != 1) { + os << ':' << pos.char_no; + } } } @@ -157,6 +157,37 @@ void SrcLocation::show_context(std::ostream& os) const { os << '^' << "\n"; } +// when generating Fift output, every block of asm instructions originated from the same Tolk line, +// is preceded by an original line as a comment +void SrcLocation::show_line_to_fif_output(std::ostream& os, int indent, int* last_line_no) const { + SrcFile::SrcPosition pos = G.all_src_files.get_file(file_id)->convert_offset(char_offset); + + // avoid duplicating one line multiple times in fift output + if (pos.line_no == *last_line_no) { + return; + } + *last_line_no = pos.line_no; + + // trim some characters from start and end to see `else if (x)` not `} else if (x) {` + std::string_view s = pos.line_str; + int b = 0, e = static_cast(s.size() - 1); + while (std::isspace(s[b]) || s[b] == '}') { + if (b < e) b++; + else break; + } + while (std::isspace(s[e]) || s[e] == '{' || s[e] == ';' || s[e] == ',') { + if (e > b) e--; + else break; + } + + if (b < e) { + for (int i = 0; i < indent * 2; ++i) { + os << ' '; + } + os << "// " << pos.line_no << ": " << s.substr(b, e - b + 1) << std::endl; + } +} + std::string SrcLocation::to_string() const { std::ostringstream os; show(os); diff --git a/tolk/src-file.h b/tolk/src-file.h index b0f9cba371..1f3eb030e8 100644 --- a/tolk/src-file.h +++ b/tolk/src-file.h @@ -85,6 +85,7 @@ class SrcLocation { void show(std::ostream& os) const; void show_context(std::ostream& os) const; + void show_line_to_fif_output(std::ostream& os, int indent, int* last_line_no) const; std::string to_string() const; void show_general_error(std::ostream& os, const std::string& message, const std::string& err_type) const; @@ -97,11 +98,10 @@ std::ostream& operator<<(std::ostream& os, SrcLocation loc); class AllRegisteredSrcFiles { std::vector all_src_files; - int last_registered_file_id = -1; int last_parsed_file_id = -1; public: - const SrcFile* find_file(int file_id) const; + const SrcFile* get_file(int file_id) const { return all_src_files.at(file_id); } const SrcFile* find_file(const std::string& abs_filename) const; const SrcFile* locate_and_register_source_file(const std::string& rel_filename, SrcLocation included_from); diff --git a/tolk/tolk-main.cpp b/tolk/tolk-main.cpp index 7f939670b7..4e9c1f4d86 100644 --- a/tolk/tolk-main.cpp +++ b/tolk/tolk-main.cpp @@ -50,6 +50,7 @@ void usage(const char* progname) { "-O\tSets optimization level (2 by default)\n" "-x\tEnables experimental options, comma-separated\n" "-S\tDon't include stack layout comments into Fift output\n" + "-L\tDon't include original lines from Tolk src into Fift output\n" "-e\tIncreases verbosity level (extra output into stderr)\n" "-v\tOutput version of Tolk and exit\n"; std::exit(2); @@ -214,7 +215,7 @@ class StdCoutRedirectToFile { int main(int argc, char* const argv[]) { int i; - while ((i = getopt(argc, argv, "o:b:O:x:Sevh")) != -1) { + while ((i = getopt(argc, argv, "o:b:O:x:SLevh")) != -1) { switch (i) { case 'o': G.settings.output_filename = optarg; @@ -231,6 +232,9 @@ int main(int argc, char* const argv[]) { case 'S': G.settings.stack_layout_comments = false; break; + case 'L': + G.settings.tolk_src_as_line_comments = false; + break; case 'e': G.settings.verbosity++; break; diff --git a/tolk/tolk-wasm.cpp b/tolk/tolk-wasm.cpp index e74589ce88..3c8cb87dd7 100644 --- a/tolk/tolk-wasm.cpp +++ b/tolk/tolk-wasm.cpp @@ -40,12 +40,14 @@ static td::Result compile_internal(char *config_json) { TRY_RESULT(opt_level, td::get_json_object_int_field(config, "optimizationLevel", true, 2)); TRY_RESULT(stack_comments, td::get_json_object_bool_field(config, "withStackComments", true, false)); + TRY_RESULT(src_line_comments, td::get_json_object_bool_field(config, "withSrcLineComments", true, false)); TRY_RESULT(entrypoint_filename, td::get_json_object_string_field(config, "entrypointFileName", false)); TRY_RESULT(experimental_options, td::get_json_object_string_field(config, "experimentalOptions", true)); G.settings.verbosity = 0; G.settings.optimization_level = std::max(0, opt_level); G.settings.stack_layout_comments = stack_comments; + G.settings.tolk_src_as_line_comments = src_line_comments; if (!experimental_options.empty()) { G.settings.parse_experimental_options_cmd_arg(experimental_options.c_str()); } diff --git a/tolk/tolk.h b/tolk/tolk.h index 12bc6b6941..9fb39dae27 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -267,7 +267,6 @@ struct Stack; struct Op { enum OpKind { - _Undef, _Nop, _Call, _CallInd, @@ -293,48 +292,48 @@ struct Op { std::unique_ptr next; FunctionPtr f_sym = nullptr; GlobalVarPtr g_sym = nullptr; - SrcLocation where; + SrcLocation loc; VarDescrList var_info; std::vector args; std::vector left, right; std::unique_ptr block0, block1; td::RefInt256 int_const; std::string str_const; - Op(SrcLocation _where = {}, OpKind _cl = _Undef) : cl(_cl), flags(0), f_sym(nullptr), where(_where) { + Op(SrcLocation loc, OpKind cl) : cl(cl), flags(0), loc(loc) { } - Op(SrcLocation _where, OpKind _cl, const std::vector& _left) - : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left) { + Op(SrcLocation loc, OpKind cl, const std::vector& left) + : cl(cl), flags(0), loc(loc), left(left) { } - Op(SrcLocation _where, OpKind _cl, std::vector&& _left) - : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(std::move(_left)) { + Op(SrcLocation loc, OpKind cl, std::vector&& left) + : cl(cl), flags(0), loc(loc), left(std::move(left)) { } - Op(SrcLocation _where, OpKind _cl, const std::vector& _left, td::RefInt256 _const) - : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left), int_const(_const) { + Op(SrcLocation loc, OpKind cl, const std::vector& left, td::RefInt256 int_const) + : cl(cl), flags(0), loc(loc), left(left), int_const(std::move(int_const)) { } - Op(SrcLocation _where, OpKind _cl, const std::vector& _left, std::string _const) - : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left), str_const(_const) { + Op(SrcLocation loc, OpKind cl, const std::vector& left, std::string str_const) + : cl(cl), flags(0), loc(loc), left(left), str_const(std::move(str_const)) { } - Op(SrcLocation _where, OpKind _cl, const std::vector& _left, const std::vector& _right) - : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left), right(_right) { + Op(SrcLocation loc, OpKind cl, const std::vector& left, const std::vector& right) + : cl(cl), flags(0), loc(loc), left(left), right(right) { } - Op(SrcLocation _where, OpKind _cl, std::vector&& _left, std::vector&& _right) - : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(std::move(_left)), right(std::move(_right)) { + Op(SrcLocation loc, OpKind cl, std::vector&& left, std::vector&& right) + : cl(cl), flags(0), loc(loc), left(std::move(left)), right(std::move(right)) { } - Op(SrcLocation _where, OpKind _cl, const std::vector& _left, const std::vector& _right, + Op(SrcLocation loc, OpKind cl, const std::vector& left, const std::vector& right, FunctionPtr _fun) - : cl(_cl), flags(0), f_sym(_fun), where(_where), left(_left), right(_right) { + : cl(cl), flags(0), f_sym(_fun), loc(loc), left(left), right(right) { } - Op(SrcLocation _where, OpKind _cl, std::vector&& _left, std::vector&& _right, - FunctionPtr _fun) - : cl(_cl), flags(0), f_sym(_fun), where(_where), left(std::move(_left)), right(std::move(_right)) { + Op(SrcLocation loc, OpKind cl, std::vector&& left, std::vector&& right, + FunctionPtr fun_ref) + : cl(cl), flags(0), f_sym(fun_ref), loc(loc), left(std::move(left)), right(std::move(right)) { } - Op(SrcLocation _where, OpKind _cl, const std::vector& _left, const std::vector& _right, - GlobalVarPtr _gvar) - : cl(_cl), flags(0), g_sym(_gvar), where(_where), left(_left), right(_right) { + Op(SrcLocation loc, OpKind cl, const std::vector& left, const std::vector& right, + GlobalVarPtr glob_ref) + : cl(cl), flags(0), g_sym(glob_ref), loc(loc), left(left), right(right) { } - Op(SrcLocation _where, OpKind _cl, std::vector&& _left, std::vector&& _right, + Op(SrcLocation loc, OpKind cl, std::vector&& left, std::vector&& right, GlobalVarPtr _gvar) - : cl(_cl), flags(0), g_sym(_gvar), where(_where), left(std::move(_left)), right(std::move(_right)) { + : cl(cl), flags(0), g_sym(_gvar), loc(loc), left(std::move(left)), right(std::move(right)) { } bool disabled() const { return flags & _Disabled; } @@ -408,45 +407,46 @@ typedef std::vector StackLayout; typedef std::pair var_const_idx_t; typedef std::vector StackLayoutExt; constexpr const_idx_t not_const = -1; -using Const = td::RefInt256; struct AsmOp { - enum Type { a_none, a_xchg, a_push, a_pop, a_const, a_custom, a_magic }; - Type t{a_none}; + enum Type { a_nop, a_comment, a_xchg, a_push, a_pop, a_const, a_custom }; + Type t; + SrcLocation loc; int indent{0}; int a, b; bool gconst{false}; std::string op; struct SReg { int idx; - SReg(int _idx) : idx(_idx) { + explicit SReg(int _idx) : idx(_idx) { } + int calc_out_strlen() const; }; AsmOp() = default; - AsmOp(Type _t) : t(_t) { + AsmOp(Type t, SrcLocation loc) : t(t), loc(loc) { } - AsmOp(Type _t, std::string _op) : t(_t), op(std::move(_op)) { + AsmOp(Type t, SrcLocation loc, std::string _op) : t(t), loc(loc), op(std::move(_op)) { } - AsmOp(Type _t, int _a) : t(_t), a(_a) { + AsmOp(Type t, SrcLocation loc, int a) : t(t), loc(loc), a(a) { } - AsmOp(Type _t, int _a, std::string _op) : t(_t), a(_a), op(std::move(_op)) { + AsmOp(Type t, SrcLocation loc, int a, std::string _op) : t(t), loc(loc), a(a), op(std::move(_op)) { } - AsmOp(Type _t, int _a, int _b) : t(_t), a(_a), b(_b) { + AsmOp(Type t, SrcLocation loc, int a, int b) : t(t), loc(loc), a(a), b(b) { } - AsmOp(Type _t, int _a, int _b, std::string _op) : t(_t), a(_a), b(_b), op(std::move(_op)) { + AsmOp(Type t, SrcLocation loc, int a, int b, std::string op) : t(t), loc(loc), a(a), b(b), op(std::move(op)) { compute_gconst(); } - void out(std::ostream& os) const; - void out_indent_nl(std::ostream& os, bool no_nl = false) const; + int out(std::ostream& os) const; + int out_indented(std::ostream& os, bool print_src_line_above) const; std::string to_string() const; void compute_gconst() { gconst = (is_custom() && (op == "PUSHNULL" || op == "NEWC" || op == "NEWB" || op == "TRUE" || op == "FALSE" || op == "NOW")); } bool is_nop() const { - return t == a_none && op.empty(); + return t == a_nop; } bool is_comment() const { - return t == a_none && !op.empty(); + return t == a_comment; } bool is_custom() const { return t == a_custom; @@ -493,80 +493,80 @@ struct AsmOp { bool is_gconst() const { return !a && b == 1 && (t == a_const || gconst); } - static AsmOp Nop() { - return AsmOp(a_none); + static AsmOp Nop(SrcLocation loc) { + return AsmOp(a_nop, loc); } - static AsmOp Xchg(int a, int b = 0) { - return a == b ? AsmOp(a_none) : (a < b ? AsmOp(a_xchg, a, b) : AsmOp(a_xchg, b, a)); + static AsmOp Xchg(SrcLocation loc, int a, int b = 0) { + return a == b ? AsmOp(a_nop, loc) : (a < b ? AsmOp(a_xchg, loc, a, b) : AsmOp(a_xchg, loc, b, a)); } - static AsmOp Push(int a) { - return AsmOp(a_push, a); + static AsmOp Push(SrcLocation loc, int a) { + return AsmOp(a_push, loc, a); } - static AsmOp Pop(int a = 0) { - return AsmOp(a_pop, a); + static AsmOp Pop(SrcLocation loc, int a) { + return AsmOp(a_pop, loc, a); } - static AsmOp Xchg2(int a, int b) { - return make_stk2(a, b, "XCHG2", 0); + static AsmOp Xchg2(SrcLocation loc, int a, int b) { + return make_stk2(loc, a, b, "XCHG2", 0); } - static AsmOp XcPu(int a, int b) { - return make_stk2(a, b, "XCPU", 1); + static AsmOp XcPu(SrcLocation loc, int a, int b) { + return make_stk2(loc, a, b, "XCPU", 1); } - static AsmOp PuXc(int a, int b) { - return make_stk2(a, b, "PUXC", 1); + static AsmOp PuXc(SrcLocation loc, int a, int b) { + return make_stk2(loc, a, b, "PUXC", 1); } - static AsmOp Push2(int a, int b) { - return make_stk2(a, b, "PUSH2", 2); + static AsmOp Push2(SrcLocation loc, int a, int b) { + return make_stk2(loc, a, b, "PUSH2", 2); } - static AsmOp Xchg3(int a, int b, int c) { - return make_stk3(a, b, c, "XCHG3", 0); + static AsmOp Xchg3(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "XCHG3", 0); } - static AsmOp Xc2Pu(int a, int b, int c) { - return make_stk3(a, b, c, "XC2PU", 1); + static AsmOp Xc2Pu(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "XC2PU", 1); } - static AsmOp XcPuXc(int a, int b, int c) { - return make_stk3(a, b, c, "XCPUXC", 1); + static AsmOp XcPuXc(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "XCPUXC", 1); } - static AsmOp XcPu2(int a, int b, int c) { - return make_stk3(a, b, c, "XCPU2", 3); + static AsmOp XcPu2(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "XCPU2", 3); } - static AsmOp PuXc2(int a, int b, int c) { - return make_stk3(a, b, c, "PUXC2", 3); + static AsmOp PuXc2(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "PUXC2", 3); } - static AsmOp PuXcPu(int a, int b, int c) { - return make_stk3(a, b, c, "PUXCPU", 3); + static AsmOp PuXcPu(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "PUXCPU", 3); } - static AsmOp Pu2Xc(int a, int b, int c) { - return make_stk3(a, b, c, "PU2XC", 3); + static AsmOp Pu2Xc(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "PU2XC", 3); } - static AsmOp Push3(int a, int b, int c) { - return make_stk3(a, b, c, "PUSH3", 3); + static AsmOp Push3(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "PUSH3", 3); } - static AsmOp BlkSwap(int a, int b); - static AsmOp BlkPush(int a, int b); - static AsmOp BlkDrop(int a); - static AsmOp BlkDrop2(int a, int b); - static AsmOp BlkReverse(int a, int b); - static AsmOp make_stk2(int a, int b, const char* str, int delta); - static AsmOp make_stk3(int a, int b, int c, const char* str, int delta); - static AsmOp IntConst(const td::RefInt256& x); - static AsmOp BoolConst(bool f); - static AsmOp Const(std::string push_op) { - return AsmOp(a_const, 0, 1, std::move(push_op)); + static AsmOp BlkSwap(SrcLocation loc, int a, int b); + static AsmOp BlkPush(SrcLocation loc, int a, int b); + static AsmOp BlkDrop(SrcLocation loc, int a); + static AsmOp BlkDrop2(SrcLocation loc, int a, int b); + static AsmOp BlkReverse(SrcLocation loc, int a, int b); + static AsmOp make_stk2(SrcLocation loc, int a, int b, const char* str, int delta); + static AsmOp make_stk3(SrcLocation loc, int a, int b, int c, const char* str, int delta); + static AsmOp IntConst(SrcLocation loc, const td::RefInt256& x); + static AsmOp BoolConst(SrcLocation loc, bool f); + static AsmOp Const(SrcLocation loc, std::string push_op) { + return AsmOp(a_const, loc, 0, 1, std::move(push_op)); } - static AsmOp Const(int arg, const std::string& push_op); - static AsmOp Comment(const std::string& comment) { - return AsmOp(a_none, std::string{"// "} + comment); + static AsmOp Const(SrcLocation loc, int arg, const std::string& push_op); + static AsmOp Comment(SrcLocation loc, const std::string& comment) { + return AsmOp(a_comment, loc, std::string{"// "} + comment); } - static AsmOp Custom(const std::string& custom_op) { - return AsmOp(a_custom, 255, 255, custom_op); + static AsmOp Custom(SrcLocation loc, const std::string& custom_op) { + return AsmOp(a_custom, loc, 255, 255, custom_op); } - static AsmOp Parse(const std::string& custom_op); - static AsmOp Custom(const std::string& custom_op, int args, int retv = 1) { - return AsmOp(a_custom, args, retv, custom_op); + static AsmOp Parse(SrcLocation loc, const std::string& custom_op); + static AsmOp Custom(SrcLocation loc, const std::string& custom_op, int args, int retv = 1) { + return AsmOp(a_custom, loc, args, retv, custom_op); } - static AsmOp Parse(std::string custom_op, int args, int retv = 1); - static AsmOp Tuple(int a); - static AsmOp UnTuple(int a); + static AsmOp Parse(SrcLocation loc, std::string custom_op, int args, int retv = 1); + static AsmOp Tuple(SrcLocation loc, int a); + static AsmOp UnTuple(SrcLocation loc, int a); }; inline std::ostream& operator<<(std::ostream& os, const AsmOp& op) { @@ -581,38 +581,19 @@ struct AsmOpList { std::vector list_; int indent_{0}; const std::vector* var_names_{nullptr}; - std::vector constants_; + std::vector constants_; bool retalt_{false}; bool retalt_inserted_{false}; void out(std::ostream& os, int mode = 0) const; AsmOpList(int indent = 0, const std::vector* var_names = nullptr) : indent_(indent), var_names_(var_names) { } - template - AsmOpList& add(Args&&... args) { - append(AsmOp(std::forward(args)...)); + AsmOpList& operator<<(AsmOp&& op) { + list_.emplace_back(op); adjust_last(); return *this; } - bool append(const AsmOp& op) { - list_.push_back(op); - adjust_last(); - return true; - } - bool append(const std::vector& ops); - bool append(std::initializer_list ops) { - return append(std::vector(std::move(ops))); - } - AsmOpList& operator<<(const AsmOp& op) { - return add(op); - } - AsmOpList& operator<<(AsmOp&& op) { - return add(std::move(op)); - } - AsmOpList& operator<<(std::string str) { - return add(AsmOp::Type::a_custom, 255, 255, str); - } - const_idx_t register_const(Const new_const); - Const get_const(const_idx_t idx); + const_idx_t register_const(td::RefInt256 new_const); + td::RefInt256 get_const(const_idx_t idx); void show_var_ext(std::ostream& os, std::pair idx_pair) const; void adjust_last() { if (list_.back().is_nop()) { @@ -627,11 +608,8 @@ struct AsmOpList { void undent() { --indent_; } - void set_indent(int new_indent) { - indent_ = new_indent; - } - void insert(size_t pos, std::string str) { - insert(pos, AsmOp(AsmOp::a_custom, 255, 255, str)); + void insert(size_t pos, SrcLocation loc, std::string str) { + insert(pos, AsmOp(AsmOp::a_custom, loc, 255, 255, std::move(str))); } void insert(size_t pos, const AsmOp& op) { auto ip = list_.begin() + pos; @@ -645,11 +623,6 @@ struct AsmOpList { } }; -inline std::ostream& operator<<(std::ostream& os, const AsmOpList& op_list) { - op_list.out(os); - return os; -} - struct AsmOpCons { std::unique_ptr car; std::unique_ptr cdr; @@ -884,7 +857,6 @@ struct Optimizer { bool find(); bool optimize(); bool compute_stack_transforms(); - bool say(std::string str) const; bool show_stack_transforms() const; void show_head() const; void show_left() const; @@ -960,17 +932,17 @@ struct Stack { StackLayoutExt s; AsmOpList& o; enum { - _StkCmt = 1, _CptStkCmt = 2, _DisableOut = 128, _Shown = 256, + _StackComments = 1, _LineComments = 2, _DisableOut = 128, _Shown = 256, _InlineFunc = 512, _NeedRetAlt = 1024, _InlineAny = 2048, _ModeSave = _InlineFunc | _NeedRetAlt | _InlineAny, _Garbage = -0x10000 }; int mode; - Stack(AsmOpList& _o, int _mode = 0) : o(_o), mode(_mode) { + Stack(AsmOpList& _o, int _mode) : o(_o), mode(_mode) { } - Stack(AsmOpList& _o, const StackLayoutExt& _s, int _mode = 0) : s(_s), o(_o), mode(_mode) { + Stack(AsmOpList& _o, const StackLayoutExt& _s, int _mode) : s(_s), o(_o), mode(_mode) { } - Stack(AsmOpList& _o, StackLayoutExt&& _s, int _mode = 0) : s(std::move(_s)), o(_o), mode(_mode) { + Stack(AsmOpList& _o, StackLayoutExt&& _s, int _mode) : s(std::move(_s)), o(_o), mode(_mode) { } int depth() const { return (int)s.size(); @@ -1007,55 +979,56 @@ struct Stack { void forget_const(); void validate(int i) const { if (i > 255) { - throw Fatal{"Too deep stack"}; + throw Fatal("Too deep stack"); } tolk_assert(i >= 0 && i < depth() && "invalid stack reference"); } void modified() { mode &= ~_Shown; } - void issue_pop(int i); - void issue_push(int i); - void issue_xchg(int i, int j); - int drop_vars_except(const VarDescrList& var_info, int excl_var = 0x80000000); + void issue_pop(SrcLocation loc, int i); + void issue_push(SrcLocation loc, int i); + void issue_xchg(SrcLocation loc, int i, int j); + int drop_vars_except(SrcLocation loc, const VarDescrList& var_info, int excl_var = 0x80000000); void forget_var(var_idx_t idx); void push_new_var(var_idx_t idx); void push_new_const(var_idx_t idx, const_idx_t cidx); void assign_var(var_idx_t new_idx, var_idx_t old_idx); - void do_copy_var(var_idx_t new_idx, var_idx_t old_idx); - void enforce_state(const StackLayout& req_stack); - void rearrange_top(const StackLayout& top, std::vector last); - void rearrange_top(var_idx_t top, bool last); + void do_copy_var(SrcLocation loc, var_idx_t new_idx, var_idx_t old_idx); + void enforce_state(SrcLocation loc, const StackLayout& req_stack); + void rearrange_top(SrcLocation loc, const StackLayout& top, std::vector last); + void rearrange_top(SrcLocation loc, var_idx_t top, bool last); void merge_const(const Stack& req_stack); - void merge_state(const Stack& req_stack); + void merge_state(SrcLocation loc, const Stack& req_stack); void show(); void opt_show() { - if ((mode & (_StkCmt | _Shown)) == _StkCmt) { + if ((mode & (_StackComments | _Shown)) == _StackComments) { show(); } } bool operator==(const Stack& y) const & { return s == y.s; } - void apply_wrappers(int callxargs_count) { + void apply_wrappers(SrcLocation loc, int callxargs_count) { + int pos0 = (mode & _StackComments && !o.list_.empty() && o.list_[0].is_comment()) ? 1 : 0; bool is_inline = mode & _InlineFunc; if (o.retalt_inserted_) { - o.insert(0, "SAMEALTSAVE"); - o.insert(0, "c2 SAVE"); + o.insert(pos0, loc, "SAMEALTSAVE"); + o.insert(pos0, loc, "c2 SAVE"); } if (callxargs_count != -1 || (is_inline && o.retalt_)) { o.indent_all(); - o.insert(0, "CONT:<{"); - o << "}>"; + o.insert(pos0, loc, "CONT:<{"); + o << AsmOp::Custom(loc, "}>"); if (callxargs_count != -1) { if (callxargs_count <= 15) { - o << AsmOp::Custom(PSTRING() << callxargs_count << " -1 CALLXARGS"); + o << AsmOp::Custom(loc, PSTRING() << callxargs_count << " -1 CALLXARGS"); } else { tolk_assert(callxargs_count <= 254); - o << AsmOp::Custom(PSTRING() << callxargs_count << " PUSHINT -1 PUSHINT CALLXVARARGS"); + o << AsmOp::Custom(loc, PSTRING() << callxargs_count << " PUSHINT -1 PUSHINT CALLXVARARGS"); } } else { - o << "EXECUTE"; + o << AsmOp::Custom(loc, "EXECUTE"); } } } @@ -1070,31 +1043,27 @@ struct Stack { typedef std::function&, std::vector&, SrcLocation)> simple_compile_func_t; -inline simple_compile_func_t make_simple_compile(AsmOp op) { - return [op](std::vector& out, std::vector& in, SrcLocation) -> AsmOp { return op; }; -} - struct FunctionBodyBuiltin { simple_compile_func_t simple_compile; explicit FunctionBodyBuiltin(simple_compile_func_t compile) : simple_compile(std::move(compile)) {} - void compile(AsmOpList& dest, std::vector& out, std::vector& in, SrcLocation where) const; + void compile(AsmOpList& dest, std::vector& out, std::vector& in, SrcLocation loc) const; }; struct FunctionBodyAsm { std::vector ops; void set_code(std::vector&& code); - void compile(AsmOpList& dest) const; + void compile(AsmOpList& dest, SrcLocation loc) const; }; struct CodeBlob { int var_cnt, in_var_cnt; FunctionPtr fun_ref; std::string name; - SrcLocation loc; + SrcLocation forced_loc; std::vector vars; std::unique_ptr ops; std::unique_ptr* cur_ops; @@ -1103,12 +1072,15 @@ struct CodeBlob { #endif std::stack*> cur_ops_stack; bool require_callxargs = false; - CodeBlob(std::string name, SrcLocation loc, FunctionPtr fun_ref) - : var_cnt(0), in_var_cnt(0), fun_ref(fun_ref), name(std::move(name)), loc(loc), cur_ops(&ops) { + explicit CodeBlob(FunctionPtr fun_ref) + : var_cnt(0), in_var_cnt(0), fun_ref(fun_ref), name(fun_ref->name), cur_ops(&ops) { } template Op& emplace_back(Args&&... args) { Op& res = *(*cur_ops = std::make_unique(args...)); + if (forced_loc.is_defined()) { + res.loc = forced_loc; + } cur_ops = &(res.next); #ifdef TOLK_DEBUG _vector_of_ops.push_back(&res); @@ -1146,17 +1118,14 @@ struct CodeBlob { void prune_unreachable_code(); void fwd_analyze(); void mark_noreturn(); - void generate_code(AsmOpList& out_list, int mode = 0); - void generate_code(std::ostream& os, int mode = 0, int indent = 0); + void generate_code(std::ostream& os, int mode = 0, int indent = 0) const; }; // defined in builtins.cpp -AsmOp exec_arg_op(std::string op, long long arg); -AsmOp exec_arg_op(std::string op, long long arg, int args, int retv = 1); -AsmOp exec_arg_op(std::string op, td::RefInt256 arg); -AsmOp exec_arg_op(std::string op, td::RefInt256 arg, int args, int retv = 1); -AsmOp exec_arg2_op(std::string op, long long imm1, long long imm2, int args, int retv = 1); -AsmOp push_const(td::RefInt256 x); +AsmOp exec_arg_op(SrcLocation loc, std::string op, long long arg, int args, int retv = 1); +AsmOp exec_arg_op(SrcLocation loc, std::string op, td::RefInt256 arg, int args, int retv = 1); +AsmOp exec_arg2_op(SrcLocation loc, std::string op, long long imm1, long long imm2, int args, int retv = 1); +AsmOp push_const(SrcLocation loc, td::RefInt256 x); void define_builtins(); From 578df21a28720cf26b2f8811c42479f1de3a5a96 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Wed, 19 Mar 2025 21:50:30 +0400 Subject: [PATCH 234/524] [Tolk] Implement methods for any types as extension functions In FunC, any function could be called `f(arg)` or `arg.f()`. You had to call `cell.cell_hash()` / `slice.slice_hash()`: these were two different global-scope functions. Now, Tolk, as other languages, separates functions from methods. It drops the ability to call a function via dot; you can only call a method (for an object type). > fun cell.hash(self): int ... > fun slice.hash(self): int ... With methods, stdlib functions became short: `t.size()` instead of `t.tupleSize()`, and so on. Methods can be declared for any type, including generics. Calling a method, the compiler does type matching to detect the only method from many equally named. This could be generalized to functions overloading some day. --- crypto/smartcont/tolk-stdlib/common.tolk | 325 ++++++++------- .../smartcont/tolk-stdlib/gas-payments.tolk | 18 +- crypto/smartcont/tolk-stdlib/lisp-lists.tolk | 8 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 134 +++--- .../smartcont/tolk-stdlib/tvm-lowlevel.tolk | 2 +- .../tests/allow-post-modification.tolk | 10 +- tolk-tester/tests/asm-arg-order.tolk | 7 +- tolk-tester/tests/assignment-tests.tolk | 22 +- tolk-tester/tests/bytesN-tests.tolk | 8 +- tolk-tester/tests/cells-slices.tolk | 50 ++- tolk-tester/tests/constants-tests.tolk | 4 +- tolk-tester/tests/dicts-demo.tolk | 10 +- tolk-tester/tests/generics-1.tolk | 27 +- tolk-tester/tests/generics-2.tolk | 21 + tolk-tester/tests/generics-3.tolk | 2 +- tolk-tester/tests/if-else-tests.tolk | 2 +- tolk-tester/tests/indexed-access.tolk | 55 ++- tolk-tester/tests/inference-tests.tolk | 10 +- tolk-tester/tests/intN-tests.tolk | 23 +- .../tests/invalid-declaration/err-1074.tolk | 6 + .../tests/invalid-declaration/err-1140.tolk | 3 +- .../tests/invalid-declaration/err-1193.tolk | 7 + .../tests/invalid-declaration/err-1226.tolk | 14 + .../tests/invalid-declaration/err-1458.tolk | 13 + .../tests/invalid-declaration/err-1556.tolk | 7 + .../tests/invalid-declaration/err-1601.tolk | 8 - .../tests/invalid-declaration/err-1663.tolk | 11 + .../tests/invalid-declaration/err-1701.tolk | 12 +- .../tests/invalid-declaration/err-1790.tolk | 2 +- .../tests/invalid-declaration/err-1794.tolk | 4 +- .../tests/invalid-declaration/err-1912.tolk | 9 + .../tests/invalid-declaration/err-1973.tolk | 7 + .../tests/invalid-semantics/err-4057.tolk | 17 + .../tests/invalid-semantics/err-4064.tolk | 7 +- .../tests/invalid-semantics/err-4080.tolk | 4 +- .../tests/invalid-semantics/err-4083.tolk | 2 +- .../tests/invalid-semantics/err-4104.tolk | 18 +- .../tests/invalid-semantics/err-4115.tolk | 14 + .../tests/invalid-semantics/err-4127.tolk | 2 +- .../tests/invalid-semantics/err-4190.tolk | 2 +- .../tests/invalid-semantics/err-4195.tolk | 4 + .../tests/invalid-semantics/err-4256.tolk | 18 + .../tests/invalid-semantics/err-4266.tolk | 4 +- .../tests/invalid-semantics/err-4270.tolk | 2 +- .../tests/invalid-semantics/err-4291.tolk | 3 +- .../tests/invalid-semantics/err-4304.tolk | 2 +- .../tests/invalid-semantics/err-4319.tolk | 17 + .../tests/invalid-semantics/err-4385.tolk | 10 +- .../tests/invalid-semantics/err-4387.tolk | 2 +- .../tests/invalid-semantics/err-4413.tolk | 10 +- .../tests/invalid-semantics/err-4473.tolk | 2 +- .../tests/invalid-semantics/err-4480.tolk | 10 + .../tests/invalid-semantics/err-4506.tolk | 16 - .../tests/invalid-semantics/err-4510.tolk | 2 +- .../tests/invalid-semantics/err-4520.tolk | 2 +- .../tests/invalid-semantics/err-4542.tolk | 4 +- .../tests/invalid-semantics/err-4588.tolk | 17 + .../tests/invalid-semantics/err-4598.tolk | 16 + .../tests/invalid-semantics/err-4604.tolk | 2 +- .../tests/invalid-semantics/err-4610.tolk | 13 + .../tests/invalid-semantics/err-4682.tolk | 2 +- .../tests/invalid-semantics/err-4711.tolk | 22 - .../tests/invalid-semantics/err-4739.tolk | 12 + .../tests/invalid-semantics/err-4828.tolk | 12 - .../tests/invalid-semantics/err-4841.tolk | 12 - .../tests/invalid-semantics/err-4847.tolk | 16 + .../tests/invalid-semantics/err-4851.tolk | 2 +- .../tests/invalid-semantics/err-4884.tolk | 13 + .../tests/invalid-semantics/err-4899.tolk | 8 - .../tests/invalid-semantics/err-4941.tolk | 10 - .../tests/invalid-semantics/err-4951.tolk | 2 +- .../tests/invalid-semantics/err-4956.tolk | 2 +- .../tests/invalid-symbol/err-2188.tolk | 2 +- .../tests/invalid-symbol/err-2374.tolk | 4 +- .../tests/invalid-symbol/err-2619.tolk | 8 + .../tests/invalid-symbol/err-2788.tolk | 9 + .../tests/invalid-symbol/err-2914.tolk | 10 - .../tests/invalid-syntax/err-3407.tolk | 2 +- .../tests/invalid-syntax/err-3967.tolk | 2 +- .../tests/invalid-typing/err-6013.tolk | 18 + .../tests/invalid-typing/err-6080.tolk | 2 +- .../tests/invalid-typing/err-6085.tolk | 17 - .../tests/invalid-typing/err-6087.tolk | 2 +- .../tests/invalid-typing/err-6134.tolk | 4 +- .../tests/invalid-typing/err-6148.tolk | 2 +- .../tests/invalid-typing/err-6149.tolk | 18 + .../tests/invalid-typing/err-6204.tolk | 8 +- .../tests/invalid-typing/err-6249.tolk | 6 +- .../tests/invalid-typing/err-6257.tolk | 4 +- .../tests/invalid-typing/err-6281.tolk | 12 + .../tests/invalid-typing/err-6288.tolk | 2 +- .../tests/invalid-typing/err-6337.tolk | 13 - .../tests/invalid-typing/err-6386.tolk | 10 - .../tests/invalid-typing/err-6391.tolk | 2 +- .../tests/invalid-typing/err-6393.tolk | 3 + .../tests/invalid-typing/err-6407.tolk | 2 +- .../tests/invalid-typing/err-6519.tolk | 13 - .../tests/invalid-typing/err-6629.tolk | 2 +- .../tests/invalid-typing/err-6637.tolk | 2 +- .../tests/invalid-typing/err-6659.tolk | 18 +- .../tests/invalid-typing/err-6731.tolk | 2 +- .../tests/invalid-typing/err-6820.tolk | 4 +- .../tests/invalid-typing/err-6839.tolk | 6 +- .../tests/invalid-typing/err-6844.tolk | 2 +- .../tests/invalid-typing/err-6942.tolk | 4 +- .../tests/invalid-typing/err-6981.tolk | 10 +- tolk-tester/tests/logical-operators.tolk | 2 +- tolk-tester/tests/methods-tests.tolk | 311 ++++++++++++++ tolk-tester/tests/mutate-methods.tolk | 66 ++- tolk-tester/tests/no-spaces.tolk | 2 +- tolk-tester/tests/null-keyword.tolk | 4 +- tolk-tester/tests/nullable-tensors.tolk | 26 +- tolk-tester/tests/nullable-types.tolk | 13 +- tolk-tester/tests/parse-address.tolk | 4 +- tolk-tester/tests/pure-functions.tolk | 4 +- tolk-tester/tests/self-keyword.tolk | 66 +-- tolk-tester/tests/smart-cast-tests.tolk | 74 ++-- tolk-tester/tests/some-tests-2.tolk | 28 +- tolk-tester/tests/strings-tests.tolk | 6 +- tolk-tester/tests/struct-tests.tolk | 25 +- tolk-tester/tests/test-math.tolk | 12 +- tolk-tester/tests/type-aliases-tests.tolk | 23 +- tolk-tester/tests/union-types-tests.tolk | 4 +- tolk-tester/tests/use-before-declare.tolk | 2 +- tolk-tester/tests/var-apply-tests.tolk | 90 ++++- .../tests/warnings-not-errors/warnings-1.tolk | 2 +- tolk/ast-from-tokens.cpp | 82 ++-- tolk/ast-replicator.h | 9 +- tolk/ast-stringifier.h | 7 +- tolk/ast.cpp | 3 +- tolk/ast.h | 20 +- tolk/builtins.cpp | 87 ++-- tolk/compiler-state.h | 5 +- tolk/generics-helpers.cpp | 195 ++++++--- tolk/generics-helpers.h | 47 +-- tolk/lexer.cpp | 1 - tolk/lexer.h | 1 - tolk/pipe-ast-to-legacy.cpp | 19 +- tolk/pipe-check-inferred-types.cpp | 22 +- tolk/pipe-check-pure-impure.cpp | 23 +- tolk/pipe-check-rvalue-lvalue.cpp | 4 +- tolk/pipe-find-unused-symbols.cpp | 2 +- tolk/pipe-generate-fif-output.cpp | 4 +- tolk/pipe-infer-types-and-calls.cpp | 382 ++++++++++-------- tolk/pipe-refine-lvalue-for-mutate.cpp | 77 ++-- tolk/pipe-register-symbols.cpp | 104 ++--- tolk/pipe-resolve-identifiers.cpp | 33 +- tolk/pipe-resolve-types.cpp | 93 ++++- tolk/pipeline.h | 6 +- tolk/src-file.cpp | 22 +- tolk/symtable.cpp | 13 + tolk/symtable.h | 36 +- tolk/tolk.h | 1 + 153 files changed, 2236 insertions(+), 1227 deletions(-) create mode 100644 tolk-tester/tests/invalid-declaration/err-1074.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1193.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1226.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1458.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1556.tolk delete mode 100644 tolk-tester/tests/invalid-declaration/err-1601.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1663.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1912.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1973.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4057.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4115.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4256.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4319.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4480.tolk delete mode 100644 tolk-tester/tests/invalid-semantics/err-4506.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4588.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4598.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4610.tolk delete mode 100644 tolk-tester/tests/invalid-semantics/err-4711.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4739.tolk delete mode 100644 tolk-tester/tests/invalid-semantics/err-4828.tolk delete mode 100644 tolk-tester/tests/invalid-semantics/err-4841.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4847.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4884.tolk delete mode 100644 tolk-tester/tests/invalid-semantics/err-4899.tolk delete mode 100644 tolk-tester/tests/invalid-semantics/err-4941.tolk create mode 100644 tolk-tester/tests/invalid-symbol/err-2619.tolk create mode 100644 tolk-tester/tests/invalid-symbol/err-2788.tolk delete mode 100644 tolk-tester/tests/invalid-symbol/err-2914.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6013.tolk delete mode 100644 tolk-tester/tests/invalid-typing/err-6085.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6149.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6281.tolk delete mode 100644 tolk-tester/tests/invalid-typing/err-6337.tolk delete mode 100644 tolk-tester/tests/invalid-typing/err-6386.tolk delete mode 100644 tolk-tester/tests/invalid-typing/err-6519.tolk create mode 100644 tolk-tester/tests/methods-tests.tolk diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 6b737ef3ee..f22f4695f3 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -10,9 +10,9 @@ tolk 0.11 type dict = cell?; /** - Tuple manipulation primitives. - Elements of a tuple can be of arbitrary type. - Note that atomic type `tuple` can't be cast to composite tuple type (e.g. `[int, cell]`) and vise versa. + Tuple manipulation primitives. + Elements of a tuple can be of arbitrary type. + Note that atomic type `tuple` can't be cast to composite tuple type (e.g. `[int, cell]`) and vise versa. */ /// Creates a tuple with zero elements. @@ -23,36 +23,36 @@ fun createEmptyTuple(): tuple /// Appends a value to tuple, resulting in `Tuple t' = (x1, ..., xn, value)`. /// If its size exceeds 255, throws a type check exception. @pure -fun tuplePush(mutate self: tuple, value: T): void +fun tuple.push(mutate self, value: T): void asm "TPUSH"; /// Returns the first element of a non-empty tuple. -/// `t.0` is actually the same as `t.tupleFirst()` +/// `t.0` is actually the same as `t.first()` @pure -fun tupleFirst(self: tuple): T +fun tuple.first(self): T asm "FIRST"; /// Returns the [`index`]-th element of a tuple. -/// `t.i` is actually the same as `t.tupleAt(i)` +/// `t.i` is actually the same as `t.get(i)` @pure -fun tupleAt(self: tuple, index: int): T +fun tuple.get(self, index: int): T builtin; /// Sets the [`index`]-th element of a tuple to a specified value /// (element with this index must already exist, a new element isn't created). -/// `t.i = value` is actually the same as `t.tupleSetAt(value, i)` +/// `t.i = value` is actually the same as `t.set(value, i)` @pure -fun tupleSetAt(mutate self: tuple, value: T, index: int): void +fun tuple.set(mutate self, value: T, index: int): void builtin; /// Returns the size of a tuple (elements count in it). @pure -fun tupleSize(self: tuple): int +fun tuple.size(self): int asm "TLEN"; /// Returns the last element of a non-empty tuple. @pure -fun tupleLast(self: tuple): T +fun tuple.last(self): T asm "LAST"; @@ -78,6 +78,7 @@ fun max(x: int, y: int): int asm "MAX"; /// Sorts two integers. +/// Example: `minMax(x, y)` with (x=20, y=10) and with (x=10, y=20) returns (10, 20) @pure fun minMax(x: int, y: int): (int, int) asm "MINMAX"; @@ -125,70 +126,83 @@ fun mulDivMod(x: int, y: int, z: int): (int, int) /** - Global getters of environment and contract state. + Global getters and setters of current contract state. */ -const MASTERCHAIN = -1; -const BASECHAIN = 0; - -/// Returns current Unix timestamp (in seconds). -@pure -fun now(): int - asm "NOW"; +/// `contract` is a built-in struct, it has only static methods. +/// Example: `contract.getCode()` and other methods. +struct contract {} /// Returns the internal address of the current smart contract as a Slice with a `MsgAddressInt`. /// If necessary, it can be parsed further using primitives such as [parseStandardAddress]. @pure -fun getMyAddress(): slice +fun contract.getAddress(): slice asm "MYADDR"; /// Returns the balance (in nanotoncoins) of the smart contract at the start of Computation Phase. /// Note that RAW primitives such as [sendMessage] do not update this field. @pure -fun getMyOriginalBalance(): int +fun contract.getOriginalBalance(): coins asm "BALANCE" "FIRST"; -/// Same as [getMyOriginalBalance], but returns a tuple: +/// Same as [contract.getOriginalBalance], but returns a tuple: /// `int` — balance in nanotoncoins; /// `dict` — a dictionary with 32-bit keys representing the balance of "extra currencies". @pure -fun getMyOriginalBalanceWithExtraCurrencies(): [int, dict] +fun contract.getOriginalBalanceWithExtraCurrencies(): [coins, dict] asm "BALANCE"; -/// Returns the logical time of the current transaction. -@pure -fun getLogicalTime(): int - asm "LTIME"; - -/// Returns the starting logical time of the current block. -@pure -fun getCurrentBlockLogicalTime(): int - asm "BLOCKLT"; - -/// Returns the value of the global configuration parameter with integer index `i` as a `cell` or `null` value. -@pure -fun getBlockchainConfigParam(x: int): cell? - asm "CONFIGOPTPARAM"; - /// Returns the persistent contract storage cell. It can be parsed or modified with slice and builder primitives later. @pure -fun getContractData(): cell +fun contract.getData(): cell asm "c4 PUSH"; /// Sets `cell` [c] as persistent contract data. You can update persistent contract storage with this primitive. -fun setContractData(c: cell): void +fun contract.setData(c: cell): void asm "c4 POP"; /// Retrieves code of smart-contract from c7 @pure -fun getContractCode(): cell +fun contract.getCode(): cell asm "MYCODE"; /// Creates an output action that would change this smart contract code to that given by cell [newCode]. /// Notice that this change will take effect only after the successful termination of the current run of the smart contract. -fun setContractCodePostponed(newCode: cell): void +fun contract.setCodePostponed(newCode: cell): void asm "SETCODE"; + +/** + Global getters of current blockchain (environment) state. +*/ + +const MASTERCHAIN = -1; +const BASECHAIN = 0; + +/// `blockchain` is a built-in struct, it has only static methods. +/// Example: `blockchain.configParam(16)` and other methods. +struct blockchain {} + +/// Returns current Unix timestamp (in seconds). +@pure +fun blockchain.now(): int + asm "NOW"; + +/// Returns the logical time of the current transaction. +@pure +fun blockchain.logicalTime(): int + asm "LTIME"; + +/// Returns the starting logical time of the current block. +@pure +fun blockchain.currentBlockLogicalTime(): int + asm "BLOCKLT"; + +/// Returns the value of the global configuration parameter with integer index `i` as a `cell` or `null` value. +@pure +fun blockchain.configParam(x: int): cell? + asm "CONFIGOPTPARAM"; + /// Commits the current state of registers `c4` (“persistent data”) and `c5` (“actions”) /// so that the current execution is considered “successful” with the saved values even if an exception /// in Computation Phase is thrown later. @@ -197,7 +211,7 @@ fun commitContractDataAndActions(): void /** - Signature checks, hashing, cryptography. + Signature checks, hashing, cryptography. */ /// Compile-time function that calculates crc32 of a constant string. @@ -236,23 +250,23 @@ fun stringSha256_32(constString: slice): int fun stringToBase256(constString: slice): int builtin; -/// Computes the representation hash of a `cell` [c] and returns it as a 256-bit unsigned integer `x`. +/// Computes the representation hash of a `cell` and returns it as a 256-bit unsigned integer `x`. /// Useful for signing and checking signatures of arbitrary entities represented by a tree of cells. @pure -fun cellHash(c: cell): int +fun cell.hash(self): int asm "HASHCU"; -/// Computes the hash of a `slice s` and returns it as a 256-bit unsigned integer `x`. +/// Computes the hash of a `slice` and returns it as a 256-bit unsigned integer `x`. /// The result is the same as if an ordinary cell containing only data and references from `s` had been created -/// and its hash computed by [cellHash]. +/// and its hash computed by [cell.hash]. @pure -fun sliceHash(s: slice): int +fun slice.hash(self): int asm "HASHSU"; -/// Computes sha256 of the data bits of `slice` [s]. If the bit length of `s` is not divisible by eight, +/// Computes sha256 of the data bits of a `slice`. If the bit length of `s` is not divisible by eight, /// throws a cell underflow exception. The hash value is returned as a 256-bit unsigned integer `x`. @pure -fun sliceBitsHash(s: slice): int +fun slice.bitsHash(self): int asm "SHA256U"; /// Checks the Ed25519-`signature` of a `hash` (a 256-bit unsigned integer, usually computed as the hash of some data) @@ -275,36 +289,43 @@ fun isSignatureValid(hash: int, signature: slice, publicKey: int): bool fun isSliceSignatureValid(data: slice, signature: slice, publicKey: int): bool asm "CHKSIGNS"; +/// `random` is a built-in struct, it has only static methods. +/// Example: `random.uint256()` and other methods. +struct random {} + /// Generates a new pseudo-random unsigned 256-bit integer x. -fun random(): int +/// Ensure you've called [random.initialize] in advance to make it unpredictable! +fun random.uint256(): uint256 asm "RANDU256"; -/// Generates a new pseudo-random integer z in the range 0..range−1 (or range..−1, if range < 0). -/// More precisely, an unsigned random value x is generated as in random; then z := x * range / 2^256 is computed. -fun randomRange(range: int): int +/// Generates a new pseudo-random integer z in the range 0..limit−1 (or limit..−1, if upto < 0). +/// More precisely, an unsigned random value x is generated as in random; then z := x * limit / 2^256 is computed. +/// Ensure you've called [random.initialize] in advance to make it unpredictable! +fun random.range(limit: int): int asm "RAND"; -/// Returns the current random seed as an unsigned 256-bit integer. +/// Returns the current random seed used to generate pseudo-random numbers. @pure -fun randomGetSeed(): int +fun random.getSeed(): uint256 asm "RANDSEED"; -/// Sets the random seed to unsigned 256-bit seed. -fun randomSetSeed(seed: int): void +/// Sets the random seed to the provided value. +fun random.setSeed(seed: uint256): void asm "SETRAND"; -/// Initializes (mixes) random seed with unsigned 256-bit integer x. -fun randomizeBy(x: int): void +/// Initializes (mixes) random seed with the provided value. +fun random.initializeBy(mixSeedWith: uint256): void asm "ADDRAND"; -/// Initializes random seed using current time. Don't forget to call this before calling `random`! -fun randomizeByLogicalTime(): void +/// Initializes random seed with current time to make random generation unpredictable. +/// Typically, you call this function once before calling [random.uint256] / [random.range]. +fun random.initialize(): void asm "LTIME" "ADDRAND"; /** - Size computation primitives. - They may be useful for computing storage fees of user-provided data. + Size computation primitives. + They may be useful for computing storage fees of user-provided data. */ /// Returns `(x, y, z, -1)` or `(null, null, null, 0)`. @@ -317,69 +338,73 @@ fun randomizeByLogicalTime(): void /// otherwise the computation is aborted before visiting the `(maxCells + 1)`-st cell and /// a zero flag is returned to indicate failure. If [c] is `null`, returns `x = y = z = 0`. @pure -fun calculateCellSize(c: cell, maxCells: int): (int, int, int, bool) +fun cell.calculateSize(self, maxCells: int): (int, int, int, bool) asm "CDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT"; -/// Similar to [calculateCellSize], but accepting a `slice` [s] instead of a `cell`. +/// Similar to [cell.calculateSize], but accepting a `slice` [s] instead of a `cell`. /// The returned value of `x` does not take into account the cell that contains the `slice` [s] itself; /// however, the data bits and the cell references of [s] are accounted for in `y` and `z`. @pure -fun calculateSliceSize(s: slice, maxCells: int): (int, int, int, bool) +fun slice.calculateSize(self, maxCells: int): (int, int, int, bool) asm "SDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT"; -/// A non-quiet version of [calculateCellSize] that throws a cell overflow exception (`8`) on failure. -fun calculateCellSizeStrict(c: cell, maxCells: int): (int, int, int) +/// A non-quiet version of [cell.calculateSize] that throws a cell overflow exception (`8`) on failure. +fun cell.calculateSizeStrict(self, maxCells: int): (int, int, int) asm "CDATASIZE"; -/// A non-quiet version of [calculateSliceSize] that throws a cell overflow exception (`8`) on failure. -fun calculateSliceSizeStrict(s: slice, maxCells: int): (int, int, int) +/// A non-quiet version of [cell.calculateSize] that throws a cell overflow exception (`8`) on failure. +fun slice.calculateSizeStrict(self, maxCells: int): (int, int, int) asm "SDATASIZE"; -/// Returns the depth of `cell` [c]. +/// Returns the depth of a `cell`. /// If [c] has no references, then return `0`; /// otherwise the returned value is one plus the maximum of depths of cells referred to from [c]. /// If [c] is a `null` instead of a cell, returns zero. @pure -fun getCellDepth(c: cell?): int +fun cell?.depth(self): int asm "CDEPTH"; -/// Returns the depth of `slice` [s]. +/// Returns the depth of a `slice`. /// If [s] has no references, then returns `0`; /// otherwise the returned value is one plus the maximum of depths of cells referred to from [s]. @pure -fun getSliceDepth(s: slice): int +fun slice.depth(self): int asm "SDEPTH"; -/// Returns the depth of `builder` [b]. +/// Returns the depth of a `builder`. /// If no cell references are stored in [b], then returns 0; /// otherwise the returned value is one plus the maximum of depths of cells referred to from [b]. @pure -fun getBuilderDepth(b: builder): int +fun builder.depth(self): int asm "BDEPTH"; /** - Debug primitives. - Only works for local TVM execution with debug level verbosity. + Debug primitives. + Only works for local TVM execution with debug level verbosity. */ +/// `debug` is a built-in struct, it has only static methods. +/// Example: `debug.print(v)` and other methods. +struct debug {} + /// Dump a variable [x] to the debug log. -fun debugPrint(x: T): void +fun debug.print(x: T): void builtin; /// Dump a string [x] to the debug log. -fun debugPrintString(x: T): void +fun debug.printString(x: T): void builtin; /// Dumps the stack (at most the top 255 values) and shows the total stack depth. -fun debugDumpStack(): void +fun debug.dumpStack(): void builtin; /** - Slice primitives: parsing cells. - When you _load_ some data, you mutate the slice (shifting an internal pointer on the stack). - When you _preload_ some data, you just get the result without mutating the slice. + Slice primitives: parsing cells. + When you _load_ some data, you mutate the slice (shifting an internal pointer on the stack). + When you _preload_ some data, you just get the result without mutating the slice. */ /// Compile-time function that converts a constant string to TL-encoded MsgAddressInt (TVM slice). @@ -398,124 +423,124 @@ fun stringAddressToSlice(constStringAddress: slice): slice fun stringHexToSlice(constStringBytesHex: slice): slice builtin; -/// Converts a `cell` [c] into a `slice`. Notice that [c] must be either an ordinary cell, +/// Converts a `cell` into a `slice`. Notice that [c] must be either an ordinary cell, /// or an exotic cell (see [TVM.pdf](https://ton-blockchain.github.io/docs/tvm.pdf), 3.1.2) /// which is automatically loaded to yield an ordinary cell `c'`, converted into a `slice` afterwards. @pure -fun beginParse(c: cell): slice +fun cell.beginParse(self): slice asm "CTOS"; /// Checks if slice is empty. If not, throws an exception. -fun assertEndOfSlice(self: slice): void +fun slice.assertEnd(self): void asm "ENDS"; /// Loads the next reference from the slice. @pure -fun loadRef(mutate self: slice): cell +fun slice.loadRef(mutate self): cell asm( -> 1 0) "LDREF"; /// Preloads the next reference from the slice. @pure -fun preloadRef(self: slice): cell +fun slice.preloadRef(self): cell asm "PLDREF"; /// Loads a signed [len]-bit integer from a slice. @pure -fun loadInt(mutate self: slice, len: int): int +fun slice.loadInt(mutate self, len: int): int builtin; /// Loads an unsigned [len]-bit integer from a slice. @pure -fun loadUint(mutate self: slice, len: int): int +fun slice.loadUint(mutate self, len: int): int builtin; /// Loads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate slice `s''`. @pure -fun loadBits(mutate self: slice, len: int): slice +fun slice.loadBits(mutate self, len: int): slice builtin; /// Preloads a signed [len]-bit integer from a slice. @pure -fun preloadInt(self: slice, len: int): int +fun slice.preloadInt(self, len: int): int builtin; /// Preloads an unsigned [len]-bit integer from a slice. @pure -fun preloadUint(self: slice, len: int): int +fun slice.preloadUint(self, len: int): int builtin; /// Preloads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate slice. @pure -fun preloadBits(self: slice, len: int): slice +fun slice.preloadBits(self, len: int): slice builtin; /// Loads serialized amount of Toncoins (any unsigned integer up to `2^120 - 1`). @pure -fun loadCoins(mutate self: slice): coins +fun slice.loadCoins(mutate self): coins asm( -> 1 0) "LDGRAMS"; /// Loads bool (-1 or 0) from a slice @pure -fun loadBool(mutate self: slice): bool +fun slice.loadBool(mutate self): bool asm( -> 1 0) "1 LDI"; /// Shifts a slice pointer to [len] bits forward, mutating the slice. @pure -fun skipBits(mutate self: slice, len: int): self +fun slice.skipBits(mutate self, len: int): self asm "SDSKIPFIRST"; /// Returns the first `0 ≤ len ≤ 1023` bits of a slice. @pure -fun getFirstBits(self: slice, len: int): slice +fun slice.getFirstBits(self, len: int): slice asm "SDCUTFIRST"; /// Returns all but the last `0 ≤ len ≤ 1023` bits of a slice. @pure -fun removeLastBits(mutate self: slice, len: int): self +fun slice.removeLastBits(mutate self, len: int): self asm "SDSKIPLAST"; /// Returns the last `0 ≤ len ≤ 1023` bits of a slice. @pure -fun getLastBits(self: slice, len: int): slice +fun slice.getLastBits(self, len: int): slice asm "SDCUTLAST"; /// Loads a dictionary (TL HashMapE structure, represented as TVM cell) from a slice. /// Returns `null` if `nothing` constructor is used. @pure -fun loadDict(mutate self: slice): dict +fun slice.loadDict(mutate self): dict asm( -> 1 0) "LDDICT"; /// Preloads a dictionary (cell) from a slice. @pure -fun preloadDict(self: slice): dict +fun slice.preloadDict(self): dict asm "PLDDICT"; -/// Loads a dictionary as [loadDict], but returns only the remainder of the slice. +/// Loads a dictionary as [slice.loadDict], but returns only the remainder of the slice. @pure -fun skipDict(mutate self: slice): self +fun slice.skipDict(mutate self): self asm "SKIPDICT"; /// Loads (Maybe ^Cell) from a slice. /// In other words, loads 1 bit: if it's true, loads the first ref, otherwise returns `null`. @pure -fun loadMaybeRef(mutate self: slice): cell? +fun slice.loadMaybeRef(mutate self): cell? asm( -> 1 0) "LDOPTREF"; /// Preloads (Maybe ^Cell) from a slice. @pure -fun preloadMaybeRef(self: slice): cell? +fun slice.preloadMaybeRef(self): cell? asm "PLDOPTREF"; /// Loads (Maybe ^Cell), but returns only the remainder of the slice. @pure -fun skipMaybeRef(mutate self: slice): self +fun slice.skipMaybeRef(mutate self): self asm "SKIPOPTREF"; /** - Builder primitives: constructing cells. - When you _store_ some data, you mutate the builder (shifting an internal pointer on the stack). - All the primitives below first check whether there is enough space in the `builder`, - and only then check the range of the value being serialized. + Builder primitives: constructing cells. + When you _store_ some data, you mutate the builder (shifting an internal pointer on the stack). + All the primitives below first check whether there is enough space in the `builder`, + and only then check the range of the value being serialized. */ /// Creates a new empty builder. @@ -525,116 +550,116 @@ fun beginCell(): builder /// Converts a builder into an ordinary `cell`. @pure -fun endCell(self: builder): cell +fun builder.endCell(self): cell asm "ENDC"; /// Stores a reference to a cell into a builder. @pure -fun storeRef(mutate self: builder, c: cell): self +fun builder.storeRef(mutate self, c: cell): self asm(c self) "STREF"; /// Stores a signed [len]-bit integer into a builder (`0 ≤ len ≤ 257`). @pure -fun storeInt(mutate self: builder, x: int, len: int): self +fun builder.storeInt(mutate self, x: int, len: int): self builtin; /// Stores an unsigned [len]-bit integer into a builder (`0 ≤ len ≤ 256`). @pure -fun storeUint(mutate self: builder, x: int, len: int): self +fun builder.storeUint(mutate self, x: int, len: int): self builtin; /// Stores a slice into a builder. @pure -fun storeSlice(mutate self: builder, s: slice): self +fun builder.storeSlice(mutate self, s: slice): self asm "STSLICER"; /// Stores amount of Toncoins into a builder. @pure -fun storeCoins(mutate self: builder, x: coins): self +fun builder.storeCoins(mutate self, x: coins): self asm "STGRAMS"; /// Stores bool (-1 or 0) into a builder. /// Attention: true value is `-1`, not 1! If you pass `1` here, TVM will throw an exception. @pure -fun storeBool(mutate self: builder, x: bool): self +fun builder.storeBool(mutate self, x: bool): self asm(x self) "1 STI"; /// Stores dictionary (represented by TVM `cell` or `null`) into a builder. /// In other words, stores a `1`-bit and a reference to [c] if [c] is not `null` and `0`-bit otherwise. @pure -fun storeDict(mutate self: builder, c: dict): self +fun builder.storeDict(mutate self, c: dict): self asm(c self) "STDICT"; /// Stores (Maybe ^Cell) into a builder. /// In other words, if cell is `null`, store '0' bit; otherwise, store '1' and a ref to [c]. @pure -fun storeMaybeRef(mutate self: builder, c: cell?): self +fun builder.storeMaybeRef(mutate self, c: cell?): self asm(c self) "STOPTREF"; /// Concatenates two builders. @pure -fun storeBuilder(mutate self: builder, from: builder): self +fun builder.storeBuilder(mutate self, from: builder): self asm "STBR"; /// Stores a slice representing TL addr_none$00 (two `0` bits). @pure -fun storeAddressNone(mutate self: builder): self +fun builder.storeAddressNone(mutate self): self asm "b{00} STSLICECONST"; /** - Slice size primitives. + Slice size primitives. */ /// Returns the number of references in a slice. @pure -fun getRemainingRefsCount(self: slice): int +fun slice.remainingRefsCount(self): int asm "SREFS"; /// Returns the number of data bits in a slice. @pure -fun getRemainingBitsCount(self: slice): int +fun slice.remainingBitsCount(self): int asm "SBITS"; /// Returns both the number of data bits and the number of references in a slice. @pure -fun getRemainingBitsAndRefsCount(self: slice): (int, int) +fun slice.remainingBitsAndRefsCount(self): (int, int) asm "SBITREFS"; /// Checks whether a slice is empty (i.e., contains no bits of data and no cell references). @pure -fun isEndOfSlice(self: slice): bool +fun slice.isEnd(self): bool asm "SEMPTY"; /// Checks whether a slice has no bits of data. @pure -fun isEndOfSliceBits(self: slice): bool +fun slice.isEndOfBits(self): bool asm "SDEMPTY"; /// Checks whether a slice has no references. @pure -fun isEndOfSliceRefs(self: slice): bool +fun slice.isEndOfRefs(self): bool asm "SREMPTY"; /// Checks whether data parts of two slices coinside. @pure -fun isSliceBitsEqual(self: slice, b: slice): bool +fun slice.bitsEqual(self, b: slice): bool asm "SDEQ"; /// Returns the number of cell references already stored in a builder. @pure -fun getBuilderRefsCount(self: builder): int +fun builder.refsCount(self): int asm "BREFS"; /// Returns the number of data bits already stored in a builder. @pure -fun getBuilderBitsCount(self: builder): int +fun builder.bitsCount(self): int asm "BBITS"; /** - Address manipulation primitives. - The address manipulation primitives listed below serialize and deserialize values according to the following TL-B scheme: + Address manipulation primitives. + The address manipulation primitives listed below serialize and deserialize values according to the following TL-B scheme: ```TL-B addr_none$00 = MsgAddressExt; addr_extern$01 len:(## 8) external_address:(bits len) @@ -672,7 +697,7 @@ fun getBuilderBitsCount(self: builder): int /// Loads from slice [s] the only prefix that is a valid `MsgAddress`, /// and returns both this prefix `s'` and the remainder `s''` of [s] as slices. @pure -fun loadAddress(mutate self: slice): slice +fun slice.loadAddress(mutate self): slice asm( -> 1 0) "LDMSGADDR"; /// Decomposes slice [s] containing a valid `MsgAddress` into a `tuple t` with separate fields of this `MsgAddress`. @@ -703,7 +728,7 @@ fun addressIsNone(s: slice): bool /** - Reserving Toncoins on balance and its flags. + Reserving Toncoins on balance and its flags. */ /// mode = 0: Reserve exact amount of nanotoncoins @@ -721,12 +746,12 @@ const RESERVE_MODE_BOUNCE_ON_ACTION_FAIL = 16; /// Creates an output action which would reserve Toncoins on balance. /// For [reserveMode] consider constants above. -fun reserveToncoinsOnBalance(nanoTonCoins: int, reserveMode: int): void +fun reserveToncoinsOnBalance(nanoTonCoins: coins, reserveMode: int): void asm "RAWRESERVE"; /// Similar to [reserveToncoinsOnBalance], but also accepts a dictionary extraAmount (represented by a cell or null) /// with extra currencies. In this way currencies other than Toncoin can be reserved. -fun reserveExtraCurrenciesOnBalance(nanoTonCoins: int, extraAmount: dict, reserveMode: int): void +fun reserveExtraCurrenciesOnBalance(nanoTonCoins: coins, extraAmount: dict, reserveMode: int): void asm "RAWRESERVEX"; @@ -745,7 +770,7 @@ const NON_BOUNCEABLE = 0x10; /// Load msgFlags from incoming message body (4 bits). @pure -fun loadMessageFlags(mutate self: slice): int +fun slice.loadMessageFlags(mutate self): int asm( -> 1 0) "4 LDU"; /// Having msgFlags (4 bits), check that a message is bounced. @@ -756,33 +781,33 @@ fun isMessageBounced(msgFlags: int): bool /// Skip 0xFFFFFFFF prefix (when a message is bounced). @pure -fun skipBouncedPrefix(mutate self: slice): self +fun slice.skipBouncedPrefix(mutate self): self asm "32 PUSHINT" "SDSKIPFIRST"; /// The guideline recommends to start the body of an internal message with uint32 `op` and uint64 `queryId`. @pure -fun loadMessageOp(mutate self: slice): int +fun slice.loadMessageOp(mutate self): int asm( -> 1 0) "32 LDU"; @pure -fun skipMessageOp(mutate self: slice): self +fun slice.skipMessageOp(mutate self): self asm "32 PUSHINT" "SDSKIPFIRST"; @pure -fun storeMessageOp(mutate self: builder, op: int): self +fun builder.storeMessageOp(mutate self, op: int): self asm(op self) "32 STU"; /// The guideline recommends that uint64 `queryId` should follow uint32 `op`. @pure -fun loadMessageQueryId(mutate self: slice): int +fun slice.loadMessageQueryId(mutate self): int asm( -> 1 0) "64 LDU"; @pure -fun skipMessageQueryId(mutate self: slice): self +fun slice.skipMessageQueryId(mutate self): self asm "64 PUSHINT" "SDSKIPFIRST"; @pure -fun storeMessageQueryId(mutate self: builder, queryId: int): self +fun builder.storeMessageQueryId(mutate self, queryId: int): self asm(queryId self) "64 STU"; /// SEND MODES - https://docs.ton.org/tvm.pdf page 137, SENDRAWMSG @@ -809,8 +834,6 @@ const SEND_MODE_ESTIMATE_FEE_ONLY = 1024; /// Sends a raw message — a correctly serialized TL object `Message X`. /// For `mode`, see constants above (except SEND_MODE_ESTIMATE_FEE_ONLY). -/// This function is still available, but deprecated: consider using [sendMessage]. -@deprecated fun sendRawMessage(msg: cell, mode: int): void asm "SENDRAWMSG"; diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 0e8305e1fd..b0cab4de8d 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -6,7 +6,7 @@ tolk 0.11 */ /// Returns amount of gas (in gas units) consumed in current Computation Phase. -fun getGasConsumedAtTheMoment(): int +fun getGasConsumedAtTheMoment(): coins asm "GASCONSUMED"; /// This function is required to be called when you process an external message (from an outer world) @@ -33,37 +33,37 @@ fun setGasLimit(limit: int): void asm "SETGASLIMIT"; /// Calculates fee (amount in nanotoncoins to be paid) for a transaction which consumed [gasUsed] gas units. -fun calculateGasFee(workchain: int, gasUsed: int): int +fun calculateGasFee(workchain: int8, gasUsed: int): coins asm(gasUsed workchain) "GETGASFEE"; /// Same as [calculateGasFee], but without flat price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) -fun calculateGasFeeWithoutFlatPrice(workchain: int, gasUsed: int): int +fun calculateGasFeeWithoutFlatPrice(workchain: int8, gasUsed: coins): coins asm(gasUsed workchain) "GETGASFEESIMPLE"; /// Calculates amount of nanotoncoins you should pay for storing a contract of provided size for [seconds]. /// [bits] and [cells] represent contract state (code + data). -fun calculateStorageFee(workchain: int, seconds: int, bits: int, cells: int): int +fun calculateStorageFee(workchain: int8, seconds: int, bits: int, cells: int): coins asm(cells bits seconds workchain) "GETSTORAGEFEE"; /// Calculates amount of nanotoncoins you should pay to send a message of specified size. -fun calculateMessageFee(workchain: int, bits: int, cells: int): int +fun calculateMessageFee(workchain: int8, bits: int, cells: int): coins asm(cells bits workchain) "GETFORWARDFEE"; /// Same as [calculateMessageFee], but without lump price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) -fun calculateMessageFeeWithoutLumpPrice(workchain: int, bits: int, cells: int): int +fun calculateMessageFeeWithoutLumpPrice(workchain: int8, bits: int, cells: int): coins asm(cells bits workchain) "GETFORWARDFEESIMPLE"; /// Calculates fee that was paid by the sender of an incoming internal message. -fun calculateOriginalMessageFee(workchain: int, incomingFwdFee: int): int +fun calculateOriginalMessageFee(workchain: int8, incomingFwdFee: coins): coins asm(incomingFwdFee workchain) "GETORIGINALFWDFEE"; /// Returns the amount of nanotoncoins current contract debts for storage. ("due" and "debt" are synonyms) /// If it has no debt, `0` is returned. -fun getMyStorageDuePayment(): int +fun contract.getStorageDuePayment(): coins asm "DUEPAYMENT"; /// Returns the amount of nanotoncoins charged for storage. /// (during storage phase preceeding to current computation phase) @pure -fun getMyStoragePaidPayment(): int +fun contract.getStoragePaidPayment(): coins asm "STORAGEFEES"; diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk index a08b2c11c8..9dd6c1da9a 100644 --- a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -2,7 +2,7 @@ tolk 0.11 /** - Lisp-style lists are nested 2-elements tuples: `(1, (2, (3, null)))` represents list `[1, 2, 3]`. + Lisp-style lists are nested 2-elements tuples: `[1, [2, [3, null]]]` represents list `[1, 2, 3]`. Elements of a list can be of different types. Empty list is conventionally represented as TVM `null` value. */ @@ -22,12 +22,6 @@ fun listPrepend(head: X, tail: tuple?): tuple fun listSplit(list: tuple): (X, tuple?) asm "UNCONS"; -/// Extracts the tail and the head of lisp-style list. -/// After extracting the last element, tuple is assigned to null. -@pure -fun listNext(mutate self: tuple?): X - asm( -> 1 0) "UNCONS"; - /// Returns the head of lisp-style list. @pure fun listGetHead(list: tuple): X diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index 00ae594c6d..5de989aed9 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -24,278 +24,278 @@ fun createEmptyDict(): dict /// Checks whether a dictionary is empty. @pure -fun dictIsEmpty(self: dict): bool +fun dict.dictIsEmpty(self): bool asm "DICTEMPTY"; @pure -fun iDictGet(self: dict, keyLen: int, key: int): (slice?, bool) +fun dict.iDictGet(self, keyLen: int, key: int): (slice?, bool) asm(key self keyLen) "DICTIGET" "NULLSWAPIFNOT"; @pure -fun uDictGet(self: dict, keyLen: int, key: int): (slice?, bool) +fun dict.uDictGet(self, keyLen: int, key: int): (slice?, bool) asm(key self keyLen) "DICTUGET" "NULLSWAPIFNOT"; @pure -fun sDictGet(self: dict, keyLen: int, key: slice): (slice?, bool) +fun dict.sDictGet(self, keyLen: int, key: slice): (slice?, bool) asm(key self keyLen) "DICTGET" "NULLSWAPIFNOT"; @pure -fun iDictSet(mutate self: dict, keyLen: int, key: int, value: slice): void +fun dict.iDictSet(mutate self, keyLen: int, key: int, value: slice): void asm(value key self keyLen) "DICTISET"; @pure -fun uDictSet(mutate self: dict, keyLen: int, key: int, value: slice): void +fun dict.uDictSet(mutate self, keyLen: int, key: int, value: slice): void asm(value key self keyLen) "DICTUSET"; @pure -fun sDictSet(mutate self: dict, keyLen: int, key: slice, value: slice): void +fun dict.sDictSet(mutate self, keyLen: int, key: slice, value: slice): void asm(value key self keyLen) "DICTSET"; @pure -fun iDictSetRef(mutate self: dict, keyLen: int, key: int, value: cell): void +fun dict.iDictSetRef(mutate self, keyLen: int, key: int, value: cell): void asm(value key self keyLen) "DICTISETREF"; @pure -fun uDictSetRef(mutate self: dict, keyLen: int, key: int, value: cell): void +fun dict.uDictSetRef(mutate self, keyLen: int, key: int, value: cell): void asm(value key self keyLen) "DICTUSETREF"; @pure -fun sDictSetRef(mutate self: dict, keyLen: int, key: slice, value: cell): void +fun dict.sDictSetRef(mutate self, keyLen: int, key: slice, value: cell): void asm(value key self keyLen) "DICTSETREF"; @pure -fun iDictSetIfNotExists(mutate self: dict, keyLen: int, key: int, value: slice): bool +fun dict.iDictSetIfNotExists(mutate self, keyLen: int, key: int, value: slice): bool asm(value key self keyLen) "DICTIADD"; @pure -fun uDictSetIfNotExists(mutate self: dict, keyLen: int, key: int, value: slice): bool +fun dict.uDictSetIfNotExists(mutate self, keyLen: int, key: int, value: slice): bool asm(value key self keyLen) "DICTUADD"; @pure -fun iDictSetIfExists(mutate self: dict, keyLen: int, key: int, value: slice): bool +fun dict.iDictSetIfExists(mutate self, keyLen: int, key: int, value: slice): bool asm(value key self keyLen) "DICTIREPLACE"; @pure -fun uDictSetIfExists(mutate self: dict, keyLen: int, key: int, value: slice): bool +fun dict.uDictSetIfExists(mutate self, keyLen: int, key: int, value: slice): bool asm(value key self keyLen) "DICTUREPLACE"; @pure -fun iDictGetRef(self: dict, keyLen: int, key: int): (dict, bool) +fun dict.iDictGetRef(self, keyLen: int, key: int): (dict, bool) asm(key self keyLen) "DICTIGETREF" "NULLSWAPIFNOT"; @pure -fun uDictGetRef(self: dict, keyLen: int, key: int): (dict, bool) +fun dict.uDictGetRef(self, keyLen: int, key: int): (dict, bool) asm(key self keyLen) "DICTUGETREF" "NULLSWAPIFNOT"; @pure -fun sDictGetRef(self: dict, keyLen: int, key: slice): (dict, bool) +fun dict.sDictGetRef(self, keyLen: int, key: slice): (dict, bool) asm(key self keyLen) "DICTGETREF" "NULLSWAPIFNOT"; @pure -fun iDictGetRefOrNull(self: dict, keyLen: int, key: int): dict +fun dict.iDictGetRefOrNull(self, keyLen: int, key: int): dict asm(key self keyLen) "DICTIGETOPTREF"; @pure -fun uDictGetRefOrNull(self: dict, keyLen: int, key: int): dict +fun dict.uDictGetRefOrNull(self, keyLen: int, key: int): dict asm(key self keyLen) "DICTUGETOPTREF"; @pure -fun sDictGetRefOrNull(self: dict, keyLen: int, key: slice): dict +fun dict.sDictGetRefOrNull(self, keyLen: int, key: slice): dict asm(key self keyLen) "DICTGETOPTREF"; @pure -fun iDictDelete(mutate self: dict, keyLen: int, key: int): bool +fun dict.iDictDelete(mutate self, keyLen: int, key: int): bool asm(key self keyLen) "DICTIDEL"; @pure -fun uDictDelete(mutate self: dict, keyLen: int, key: int): bool +fun dict.uDictDelete(mutate self, keyLen: int, key: int): bool asm(key self keyLen) "DICTUDEL"; @pure -fun sDictDelete(mutate self: dict, keyLen: int, key: slice): bool +fun dict.sDictDelete(mutate self, keyLen: int, key: slice): bool asm(key self keyLen) "DICTDEL"; @pure -fun iDictSetAndGet(mutate self: dict, keyLen: int, key: int, value: slice): (slice?, bool) +fun dict.iDictSetAndGet(mutate self, keyLen: int, key: int, value: slice): (slice?, bool) asm(value key self keyLen) "DICTISETGET" "NULLSWAPIFNOT"; @pure -fun uDictSetAndGet(mutate self: dict, keyLen: int, key: int, value: slice): (slice?, bool) +fun dict.uDictSetAndGet(mutate self, keyLen: int, key: int, value: slice): (slice?, bool) asm(value key self keyLen) "DICTUSETGET" "NULLSWAPIFNOT"; @pure -fun sDictSetAndGet(mutate self: dict, keyLen: int, key: slice, value: slice): (slice?, bool) +fun dict.sDictSetAndGet(mutate self, keyLen: int, key: slice, value: slice): (slice?, bool) asm(value key self keyLen) "DICTSETGET" "NULLSWAPIFNOT"; @pure -fun iDictSetAndGetRefOrNull(mutate self: dict, keyLen: int, key: int, value: cell): dict +fun dict.iDictSetAndGetRefOrNull(mutate self, keyLen: int, key: int, value: cell): dict asm(value key self keyLen) "DICTISETGETOPTREF"; @pure -fun uDictSetAndGetRefOrNull(mutate self: dict, keyLen: int, key: int, value: cell): dict +fun dict.uDictSetAndGetRefOrNull(mutate self, keyLen: int, key: int, value: cell): dict asm(value key self keyLen) "DICTUSETGETOPTREF"; @pure -fun iDictDeleteAndGet(mutate self: dict, keyLen: int, key: int): (slice?, bool) +fun dict.iDictDeleteAndGet(mutate self, keyLen: int, key: int): (slice?, bool) asm(key self keyLen) "DICTIDELGET" "NULLSWAPIFNOT"; @pure -fun uDictDeleteAndGet(mutate self: dict, keyLen: int, key: int): (slice?, bool) +fun dict.uDictDeleteAndGet(mutate self, keyLen: int, key: int): (slice?, bool) asm(key self keyLen) "DICTUDELGET" "NULLSWAPIFNOT"; @pure -fun sDictDeleteAndGet(mutate self: dict, keyLen: int, key: slice): (slice?, bool) +fun dict.sDictDeleteAndGet(mutate self, keyLen: int, key: slice): (slice?, bool) asm(key self keyLen) "DICTDELGET" "NULLSWAPIFNOT"; @pure -fun iDictSetBuilder(mutate self: dict, keyLen: int, key: int, value: builder): void +fun dict.iDictSetBuilder(mutate self, keyLen: int, key: int, value: builder): void asm(value key self keyLen) "DICTISETB"; @pure -fun uDictSetBuilder(mutate self: dict, keyLen: int, key: int, value: builder): void +fun dict.uDictSetBuilder(mutate self, keyLen: int, key: int, value: builder): void asm(value key self keyLen) "DICTUSETB"; @pure -fun sDictSetBuilder(mutate self: dict, keyLen: int, key: slice, value: builder): void +fun dict.sDictSetBuilder(mutate self, keyLen: int, key: slice, value: builder): void asm(value key self keyLen) "DICTSETB"; @pure -fun iDictSetBuilderIfNotExists(mutate self: dict, keyLen: int, key: int, value: builder): bool +fun dict.iDictSetBuilderIfNotExists(mutate self, keyLen: int, key: int, value: builder): bool asm(value key self keyLen) "DICTIADDB"; @pure -fun uDictSetBuilderIfNotExists(mutate self: dict, keyLen: int, key: int, value: builder): bool +fun dict.uDictSetBuilderIfNotExists(mutate self, keyLen: int, key: int, value: builder): bool asm(value key self keyLen) "DICTUADDB"; @pure -fun iDictSetBuilderIfExists(mutate self: dict, keyLen: int, key: int, value: builder): bool +fun dict.iDictSetBuilderIfExists(mutate self, keyLen: int, key: int, value: builder): bool asm(value key self keyLen) "DICTIREPLACEB"; @pure -fun uDictSetBuilderIfExists(mutate self: dict, keyLen: int, key: int, value: builder): bool +fun dict.uDictSetBuilderIfExists(mutate self, keyLen: int, key: int, value: builder): bool asm(value key self keyLen) "DICTUREPLACEB"; @pure -fun iDictDeleteFirstAndGet(mutate self: dict, keyLen: int): (int?, slice?, bool) +fun dict.iDictDeleteFirstAndGet(mutate self, keyLen: int): (int?, slice?, bool) asm(-> 0 2 1 3) "DICTIREMMIN" "NULLSWAPIFNOT2"; @pure -fun uDictDeleteFirstAndGet(mutate self: dict, keyLen: int): (int?, slice?, bool) +fun dict.uDictDeleteFirstAndGet(mutate self, keyLen: int): (int?, slice?, bool) asm(-> 0 2 1 3) "DICTUREMMIN" "NULLSWAPIFNOT2"; @pure -fun sDictDeleteFirstAndGet(mutate self: dict, keyLen: int): (slice?, slice?, bool) +fun dict.sDictDeleteFirstAndGet(mutate self, keyLen: int): (slice?, slice?, bool) asm(-> 0 2 1 3) "DICTREMMIN" "NULLSWAPIFNOT2"; @pure -fun iDictDeleteLastAndGet(mutate self: dict, keyLen: int): (int?, slice?, bool) +fun dict.iDictDeleteLastAndGet(mutate self, keyLen: int): (int?, slice?, bool) asm(-> 0 2 1 3) "DICTIREMMAX" "NULLSWAPIFNOT2"; @pure -fun uDictDeleteLastAndGet(mutate self: dict, keyLen: int): (int?, slice?, bool) +fun dict.uDictDeleteLastAndGet(mutate self, keyLen: int): (int?, slice?, bool) asm(-> 0 2 1 3) "DICTUREMMAX" "NULLSWAPIFNOT2"; @pure -fun sDictDeleteLastAndGet(mutate self: dict, keyLen: int): (slice?, slice?, bool) +fun dict.sDictDeleteLastAndGet(mutate self, keyLen: int): (slice?, slice?, bool) asm(-> 0 2 1 3) "DICTREMMAX" "NULLSWAPIFNOT2"; @pure -fun iDictGetFirst(self: dict, keyLen: int): (int?, slice?, bool) +fun dict.iDictGetFirst(self, keyLen: int): (int?, slice?, bool) asm (-> 1 0 2) "DICTIMIN" "NULLSWAPIFNOT2"; @pure -fun uDictGetFirst(self: dict, keyLen: int): (int?, slice?, bool) +fun dict.uDictGetFirst(self, keyLen: int): (int?, slice?, bool) asm (-> 1 0 2) "DICTUMIN" "NULLSWAPIFNOT2"; @pure -fun sDictGetFirst(self: dict, keyLen: int): (slice?, slice?, bool) +fun dict.sDictGetFirst(self, keyLen: int): (slice?, slice?, bool) asm (-> 1 0 2) "DICTMIN" "NULLSWAPIFNOT2"; @pure -fun iDictGetFirstAsRef(self: dict, keyLen: int): (int?, dict, bool) +fun dict.iDictGetFirstAsRef(self, keyLen: int): (int?, dict, bool) asm (-> 1 0 2) "DICTIMINREF" "NULLSWAPIFNOT2"; @pure -fun uDictGetFirstAsRef(self: dict, keyLen: int): (int?, dict, bool) +fun dict.uDictGetFirstAsRef(self, keyLen: int): (int?, dict, bool) asm (-> 1 0 2) "DICTUMINREF" "NULLSWAPIFNOT2"; @pure -fun sDictGetFirstAsRef(self: dict, keyLen: int): (slice?, dict, bool) +fun dict.sDictGetFirstAsRef(self, keyLen: int): (slice?, dict, bool) asm (-> 1 0 2) "DICTMINREF" "NULLSWAPIFNOT2"; @pure -fun iDictGetLast(self: dict, keyLen: int): (int?, slice?, bool) +fun dict.iDictGetLast(self, keyLen: int): (int?, slice?, bool) asm (-> 1 0 2) "DICTIMAX" "NULLSWAPIFNOT2"; @pure -fun uDictGetLast(self: dict, keyLen: int): (int?, slice?, bool) +fun dict.uDictGetLast(self, keyLen: int): (int?, slice?, bool) asm (-> 1 0 2) "DICTUMAX" "NULLSWAPIFNOT2"; @pure -fun sDictGetLast(self: dict, keyLen: int): (slice?, slice?, bool) +fun dict.sDictGetLast(self, keyLen: int): (slice?, slice?, bool) asm (-> 1 0 2) "DICTMAX" "NULLSWAPIFNOT2"; @pure -fun iDictGetLastAsRef(self: dict, keyLen: int): (int?, dict, bool) +fun dict.iDictGetLastAsRef(self, keyLen: int): (int?, dict, bool) asm (-> 1 0 2) "DICTIMAXREF" "NULLSWAPIFNOT2"; @pure -fun uDictGetLastAsRef(self: dict, keyLen: int): (int?, dict, bool) +fun dict.uDictGetLastAsRef(self, keyLen: int): (int?, dict, bool) asm (-> 1 0 2) "DICTUMAXREF" "NULLSWAPIFNOT2"; @pure -fun sDictGetLastAsRef(self: dict, keyLen: int): (slice?, dict, bool) +fun dict.sDictGetLastAsRef(self, keyLen: int): (slice?, dict, bool) asm (-> 1 0 2) "DICTMAXREF" "NULLSWAPIFNOT2"; @pure -fun iDictGetNext(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) +fun dict.iDictGetNext(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXT" "NULLSWAPIFNOT2"; @pure -fun uDictGetNext(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) +fun dict.uDictGetNext(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXT" "NULLSWAPIFNOT2"; @pure -fun iDictGetNextOrEqual(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) +fun dict.iDictGetNextOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXTEQ" "NULLSWAPIFNOT2"; @pure -fun uDictGetNextOrEqual(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) +fun dict.uDictGetNextOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXTEQ" "NULLSWAPIFNOT2"; @pure -fun iDictGetPrev(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) +fun dict.iDictGetPrev(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETPREV" "NULLSWAPIFNOT2"; @pure -fun uDictGetPrev(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) +fun dict.uDictGetPrev(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETPREV" "NULLSWAPIFNOT2"; @pure -fun iDictGetPrevOrEqual(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) +fun dict.iDictGetPrevOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETPREVEQ" "NULLSWAPIFNOT2"; @pure -fun uDictGetPrevOrEqual(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) +fun dict.uDictGetPrevOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETPREVEQ" "NULLSWAPIFNOT2"; @@ -304,13 +304,13 @@ fun uDictGetPrevOrEqual(self: dict, keyLen: int, pivot: int): (int?, slice?, boo */ @pure -fun prefixDictGet(self: dict, keyLen: int, key: slice): (slice, slice?, slice?, bool) +fun dict.prefixDictGet(self, keyLen: int, key: slice): (slice, slice?, slice?, bool) asm(key self keyLen) "PFXDICTGETQ" "NULLSWAPIFNOT2"; @pure -fun prefixDictSet(mutate self: dict, keyLen: int, key: slice, value: slice): bool +fun dict.prefixDictSet(mutate self, keyLen: int, key: slice, value: slice): bool asm(value key self keyLen) "PFXDICTSET"; @pure -fun prefixDictDelete(mutate self: dict, keyLen: int, key: slice): bool +fun dict.prefixDictDelete(mutate self, keyLen: int, key: slice): bool asm(key self keyLen) "PFXDICTDEL"; diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index a7f1582485..0755145397 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -21,5 +21,5 @@ fun transformSliceToContinuation(s: slice): continuation /// Moves a variable or a value [x] to the top of the stack. @pure -fun stackMoveToTop(mutate self: X): void +fun T.stackMoveToTop(mutate self): void asm "NOP"; diff --git a/tolk-tester/tests/allow-post-modification.tolk b/tolk-tester/tests/allow-post-modification.tolk index 49a2eaec58..6d3abf0573 100644 --- a/tolk-tester/tests/allow-post-modification.tolk +++ b/tolk-tester/tests/allow-post-modification.tolk @@ -4,7 +4,7 @@ fun unsafe_tuple(x: X): tuple fun inc(x: int, y: int): (int, int) { return (x + y, y * 10); } -fun `~inc`(mutate self: int, y: int): int { +fun int.`~inc`(mutate self, y: int): int { val (newX, newY) = inc(self, y); self = newX; return newY; @@ -13,7 +13,7 @@ fun `~inc`(mutate self: int, y: int): int { fun eq(v: X): X { return v; } fun eq2(v: (int, int)) { return v; } fun mul2(mutate dest: int, v: int): int { dest = v*2; return dest; } -fun multens(mutate self: (int, int), v: (int, int)): (int, int) { var (f, s) = self; var (m1, m2) = v; self = (f*m1, s*m2); return self; } +fun (int, int).multens(mutate self, v: (int, int)): (int, int) { var (f, s) = self; var (m1, m2) = v; self = (f*m1, s*m2); return self; } @method_id(11) fun test_return(x: int): (int, int, int, int, int, int, int) { @@ -102,14 +102,14 @@ fun test_assign_with_mutate(x: int) { @method_id(23) fun test_assign_tensor(x: (int, int)) { var fs = (0, 0); - return (x, x = (20, 30), fs = x.multens((1, 2)), fs.multens(multens(mutate x, (-1, -1))), x, fs); + return (x, x = (20, 30), fs = x.multens((1, 2)), fs.multens(x.multens((-1, -1))), x, fs); } global fs: (int, int); @method_id(24) fun test_assign_tensor_global(x: (int, int)) { fs = (0, 0); - return (x, x = (20, 30), fs = x.multens((1, 2)), fs.multens(multens(mutate x, (-1, -1))), x, fs); + return (x, x = (20, 30), fs = x.multens((1, 2)), fs.multens(x.multens((-1, -1))), x, fs); } struct Point1D { @@ -146,7 +146,7 @@ fun main() { @fif_codegen """ - ~inc PROC:<{ // self y + int.~inc PROC:<{ // self y inc CALLDICT // self newY }> """ diff --git a/tolk-tester/tests/asm-arg-order.tolk b/tolk-tester/tests/asm-arg-order.tolk index b96e09ecb1..841dc95645 100644 --- a/tolk-tester/tests/asm-arg-order.tolk +++ b/tolk-tester/tests/asm-arg-order.tolk @@ -2,7 +2,7 @@ fun empty_tuple2(): tuple asm "NIL"; @pure -fun tpush2(mutate self: tuple, x: X): void +fun tuple.tpush2(mutate self, x: X): void asm "TPUSH"; @pure @@ -15,11 +15,14 @@ asm (z y x -> 0) "3 TUPLE"; fun asm_func_3(x: int, y: int, z: int): tuple asm (y z x -> 0) "3 TUPLE"; @pure +fun int.asm_func_3(self, y: int, z: int): tuple +asm (y z self -> 0) "3 TUPLE"; +@pure fun asm_func_4(a: int, b: (int, (int, int)), c: int): tuple asm (b a c -> 0) "5 TUPLE"; @pure -fun asm_func_modify(mutate self: tuple, b: int, c: int): void +fun tuple.asm_func_modify(mutate self, b: int, c: int): void asm (c b self) "SWAP TPUSH SWAP TPUSH"; global t: tuple; diff --git a/tolk-tester/tests/assignment-tests.tolk b/tolk-tester/tests/assignment-tests.tolk index 2301c9b70c..a52b3ca50e 100644 --- a/tolk-tester/tests/assignment-tests.tolk +++ b/tolk-tester/tests/assignment-tests.tolk @@ -21,7 +21,7 @@ fun typesAsIdentifiers(builder: builder) { { var cell: cell = cell; var tuple: tuple = createEmptyTuple(); - var bool: bool = tuple.tupleAt(0) > 0; + var bool: bool = tuple.get(0) > 0; } return int; } @@ -29,37 +29,37 @@ fun typesAsIdentifiers(builder: builder) { global callOrder: tuple; fun getTensor_12() { - callOrder.tuplePush(100); + callOrder.push(100); return (1, 2); } fun getTensor_1X(x: int) { - callOrder.tuplePush(101); + callOrder.push(101); return (1, x); } fun getTuple_12() { - callOrder.tuplePush(110); + callOrder.push(110); return [1, 2, ]; } fun getTuple_1X(x: int) { - callOrder.tuplePush(111); + callOrder.push(111); return [1, x]; } fun getUntypedTuple_12() { - callOrder.tuplePush(120); - var t = createEmptyTuple(); t.tuplePush(1); t.tuplePush(2); + callOrder.push(120); + var t = createEmptyTuple(); t.push(1); t.push(2); return t; } fun getUntypedTuple_1X(x: int) { - callOrder.tuplePush(121); - var t = createEmptyTuple(); t.tuplePush(1); t.tuplePush(x); + callOrder.push(121); + var t = createEmptyTuple(); t.push(1); t.push(x); return t; } fun getIntValue5() { - callOrder.tuplePush(10); + callOrder.push(10); return 5; } fun getIntValueX(x: int) { - callOrder.tuplePush(11); + callOrder.push(11); return x; } diff --git a/tolk-tester/tests/bytesN-tests.tolk b/tolk-tester/tests/bytesN-tests.tolk index 32b975bf78..7f1a76e2f3 100644 --- a/tolk-tester/tests/bytesN-tests.tolk +++ b/tolk-tester/tests/bytesN-tests.tolk @@ -6,8 +6,8 @@ const someBytes = "asdf" as bytes16; const anotherBytes: bytes16 = "asdf" as bytes16; fun autoInferBytes16() { - if (random()) { return someBytes; } - else if (random()) { return true ? "" as bytes16 : anotherBytes; } + if (random.uint256()) { return someBytes; } + else if (random.uint256()) { return true ? "" as bytes16 : anotherBytes; } else { return someBytes!; } } @@ -20,7 +20,7 @@ fun test1() { __expect_type(ss as bytes128, "bytes128"); __expect_type(someBytes, "bytes16"); __expect_type(10>3 ? (null, ss as bits1) : (ss as bytes1, null), "(bytes1?, bits1?)"); - return (getRemainingBitsCount(ss as slice), loadInt(mutate ss as slice, 32), (ss as slice).loadInt(32)); + return ((ss as slice).remainingBitsCount(), (ss as slice).loadInt(32), (ss as slice).loadInt(32)); } fun test2(a: bytes8, b: bits64) { @@ -35,7 +35,7 @@ fun test3() { var slice = beginCell().storeInt(1, 32).storeInt(2, 32).endCell().beginParse(); var bq = slice as bits16?; if (bq != null) { - return (bq as slice).loadInt(32 as uint8) + (slice = bq as slice).getRemainingBitsCount(); + return (bq as slice).loadInt(32 as uint8) + (slice = bq as slice).remainingBitsCount(); } return -1; } diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index e900268bba..63a823ed9f 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -1,15 +1,21 @@ -fun store_u32(mutate self: builder, value: int): self { +fun builder.store_u32(mutate self, value: int): self { return self.storeUint(value, 32); } -fun load_u32(mutate self: slice): int { +fun slice.load_u32(mutate self): int { return self.loadUint(32); } -fun myLoadInt(mutate self: slice, len: int): int +fun slice.myLoadInt(mutate self, len: int): int asm(-> 1 0) "LDIX"; -fun myStoreInt(mutate self: builder, x: int, len: int): self +fun builder.myStoreInt(mutate self, x: int, len: int): self asm(x self len) "STIX"; +@pure +fun endCell(b: builder): cell + asm "ENDC"; +@pure +fun beginParse(c: cell): slice + asm "CTOS"; @method_id(101) fun test1(): [int,int,int,int,int] { @@ -80,7 +86,7 @@ fun test7() { var uri_builder = beginCell(); var uri_slice = uri_builder.storeSlice(".json").endCell().beginParse(); var image_slice = uri_builder.storeSlice(".png").endCell().beginParse(); - return (uri_builder.getBuilderBitsCount(), uri_slice.getRemainingBitsCount(), image_slice.getRemainingBitsCount()); + return (uri_builder.bitsCount(), uri_slice.remainingBitsCount(), image_slice.remainingBitsCount()); } @method_id(108) @@ -90,13 +96,13 @@ fun test8() { var uri_slice = fresh.storeSlice(".json").endCell().beginParse(); var fresh redef = uri_builder; var image_slice = fresh.storeSlice(".png").endCell().beginParse(); - return (uri_builder.getBuilderBitsCount(), uri_slice.getRemainingBitsCount(), image_slice.getRemainingBitsCount()); + return (uri_builder.bitsCount(), uri_slice.remainingBitsCount(), image_slice.remainingBitsCount()); } -fun sumNumbersInSlice(mutate self: slice): int { +fun slice.sumNumbersInSlice(mutate self): int { var result = 0; - while (!self.isEndOfSliceBits()) { + while (!self.isEndOfBits()) { result += self.loadUint(32); } return result; @@ -106,27 +112,27 @@ fun sumNumbersInSlice(mutate self: slice): int { fun test10() { var ref = beginCell().storeInt(100, 32).endCell(); var s: slice = beginCell().storeInt(1, 32).storeInt(2, 32).storeRef(ref).endCell().beginParse(); - var result = (getRemainingBitsCount(s), s.sumNumbersInSlice(), getRemainingBitsCount(s), isEndOfSlice(s), isEndOfSliceBits(s), isEndOfSliceRefs(s)); + var result = (s.remainingBitsCount(), s.sumNumbersInSlice(), s.remainingBitsCount(), s.isEnd(), s.isEndOfBits(), s.isEndOfRefs()); var ref2: cell = s.loadRef(); var s2: slice = ref2.beginParse(); - s.assertEndOfSlice(); - return (result, s2.loadInt(32), s2.isEndOfSlice()); + s.assertEnd(); + return (result, s2.loadInt(32), s2.isEnd()); } @method_id(111) fun test11() { var s: slice = beginCell().storeInt(1, 32).storeInt(2, 32).storeInt(3, 32).storeInt(4, 32).storeInt(5, 32).storeInt(6, 32).storeInt(7, 32).endCell().beginParse(); - var size1 = getRemainingBitsCount(s); + var size1 = s.remainingBitsCount(); s!.skipBits(32); var s1: slice = s.getFirstBits(64); var n1 = s1.loadInt(32); - var size2 = getRemainingBitsCount(s); + var size2 = s.remainingBitsCount(); s.loadInt(32); - var size3 = getRemainingBitsCount(s); + var size3 = s.remainingBitsCount(); s.removeLastBits(32); - var size4 = getRemainingBitsCount(s); + var size4 = s.remainingBitsCount(); var n2 = s.loadInt(32); - var size5 = getRemainingBitsCount(s); + var size5 = s.remainingBitsCount(); return (n1, n2, size1, size2, size3, size4, size5); } @@ -134,13 +140,13 @@ fun test11() { fun test12() { var (result1, result2) = (0, 0); try { - beginCell().storeRef(beginCell().endCell()).endCell().beginParse().assertEndOfSlice(); + beginCell().storeRef(beginCell().endCell()).endCell().beginParse().assertEnd(); result1 = 100; } catch (code) { result1 = code; } try { - beginCell().endCell().beginParse().assertEndOfSlice(); + beginCell().endCell().beginParse().assertEnd(); result2 = 100; } catch (code) { result2 = code; @@ -153,12 +159,12 @@ fun test13() { var ref2 = beginCell().storeInt(1, 32).endCell(); var ref1 = beginCell().storeInt(1, 32).storeRef(ref2).endCell(); var c = beginCell().storeInt(444, 32).storeRef(ref1).storeRef(ref1).storeRef(ref1).storeRef(ref2).storeInt(4, 32).endCell(); - var (n_cells1, n_bits1, n_refs1) = c.calculateCellSizeStrict(10); + var (n_cells1, n_bits1, n_refs1) = c.calculateSizeStrict(10); var s = c.beginParse(); s.loadRef(); s.loadRef(); var n = s.loadInt(32); - var (n_cells2, n_bits2, n_refs2) = s.calculateSliceSizeStrict(10); + var (n_cells2, n_bits2, n_refs2) = s.calculateSizeStrict(10); return ([n_cells1, n_bits1, n_refs1], [n_cells2, n_bits2, n_refs2], n); } @@ -177,7 +183,7 @@ fun test111() { .endCell().beginParse(); var op1 = s.loadUint(32); var q1 = s.loadUint(64); - if (s.addressIsNone()) { + if (addressIsNone(s)) { s.skipBits(2); } if (s.loadBool() == false) { @@ -187,7 +193,7 @@ fun test111() { var op2 = s.loadMessageOp(); var q2 = s.loadMessageQueryId(); s.skipBits(64); - s.assertEndOfSlice(); + s.assertEnd(); assert(isMessageBounced(0x001) && !isMessageBounced(0x002)) throw 444; return (op1, q1, op2, q2); } diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index 128a5f7ea3..ba213093bb 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -53,13 +53,13 @@ fun iget240(): MInt { return int240; } fun newc(): builder asm "NEWC"; @pure -fun endcs(b: builder): slice +fun builder.endcs(self): slice asm "ENDC" "CTOS"; @pure fun sdeq(s1: slice, s2: slice): MInt asm "SDEQ"; @pure -fun stslicer(b: builder, s: slice): builder +fun builder.stslicer(self, s: slice): builder asm "STSLICER"; @method_id(101) diff --git a/tolk-tester/tests/dicts-demo.tolk b/tolk-tester/tests/dicts-demo.tolk index 606318cb36..1ff4cd8c3e 100644 --- a/tolk-tester/tests/dicts-demo.tolk +++ b/tolk-tester/tests/dicts-demo.tolk @@ -1,6 +1,6 @@ import "@stdlib/tvm-dicts" -fun addIntToIDict(mutate self: cell?, key: int, number: int): void { +fun cell?.addIntToIDict(mutate self, key: int, number: int): void { return self.iDictSetBuilder(32, key, beginCell().storeInt(number, 32)); } @@ -14,7 +14,7 @@ fun calculateDictLen(d: cell?) { return len; } -fun loadTwoDigitNumberFromSlice(mutate self: slice): int { +fun slice.loadTwoDigitNumberFromSlice(mutate self): int { var n1 = self.loadInt(8); var n2 = self.loadInt(8); return (n1 - 48) * 10 + (n2 - 48); @@ -47,7 +47,7 @@ fun test102() { while (!shouldBreak) { var (kDel, kVal, wasDel) = dict.iDictDeleteLastAndGet(32); if (wasDel) { - deleted.tuplePush([kDel, kVal!.loadInt(32)]); + deleted.push([kDel, kVal!.loadInt(32)]); } else { shouldBreak = true; } @@ -82,8 +82,8 @@ fun test104() { var (old2, _) = dict.sDictDeleteAndGet(32, "key1"); var (restK, restV, _) = dict.sDictGetFirst(32); var (restK1, restV1, _) = dict.sDictDeleteLastAndGet(32); - assert (restK!.isSliceBitsEqual(restK1!)) throw 123; - assert (restV!.isSliceBitsEqual(restV1!)) throw 123; + assert (restK!.bitsEqual(restK1!)) throw 123; + assert (restV!.bitsEqual(restV1!)) throw 123; return ( old1!.loadTwoDigitNumberFromSlice(), old2!.loadTwoDigitNumberFromSlice(), diff --git a/tolk-tester/tests/generics-1.tolk b/tolk-tester/tests/generics-1.tolk index ba46f3440e..0984dcd515 100644 --- a/tolk-tester/tests/generics-1.tolk +++ b/tolk-tester/tests/generics-1.tolk @@ -7,6 +7,10 @@ fun eq2(value: X) { return value; } fun eq3(value: X): X { var cp: [X] = [eq1(value)]; var ((([v: X]))) = cp; return v; } fun eq4(value: X) { return eq1(value); } +@pure +fun tuplePush(mutate t: tuple, value: T): void + asm "TPUSH"; + @method_id(101) fun test101(x: int) { var (a, b, c) = (x, (x,x), [x,x]); @@ -49,12 +53,12 @@ fun test102(): (int, int, int, [int, int], Wrapper< Wrapper >) { fun test103(first: int): (int, int, int) { var t = createEmptyTuple(); var cs = beginCell().storeInt(100, 32).endCell().beginParse(); - t.tuplePush(first); - t.tuplePush(2); - t.tuplePush(cs); - cs = t.tupleAt(2); - cs = t.tupleAt(2) as slice; - return (t.tupleAt(0), cs.loadInt(32), t.tupleAt(2).loadInt(32)); + t.push(first); + t.push(2); + t.push(cs); + cs = t.get(2); + cs = t.get(2) as slice; + return (t.get(0), cs.loadInt(32), t.get(2).loadInt(32)); } fun manyEq(a: T1, b: T2, c: T3): [T1, T2, T3] { @@ -101,10 +105,11 @@ fun test106() { ]; } -fun callTupleFirst(t: X): Y { return t.tupleFirst(); } -fun callTuplePush(mutate self: T, v1: V, v2: V): self { self.tuplePush(v1); tuplePush(mutate self, v2); return self; } -fun getTupleLastInt(t: tuple) { return t.tupleLast(); } -fun getTupleSize(t: MTuple) { return t.tupleSize(); } +fun X.callTupleFirst(self): Y { return self.first(); } +fun callTupleFirst(t: X): Y { return t.first(); } +fun T.callTuplePush(mutate self, v1: V, v2: V): self { self.push(v1); tuplePush(mutate self, v2); return self; } +fun getTupleLastInt(t: tuple) { return t.last(); } +fun getTupleSize(t: MTuple) { return t.size(); } fun callAnyFn(f: (TObj) -> TResult, arg: TObj) { return f(arg); } fun callAnyFn2(f: TCallback, arg: tuple) { return f(arg); } @@ -114,7 +119,7 @@ type MTuple = tuple; @method_id(107) fun test107() { t107 = createEmptyTuple(); - callTuplePush(mutate t107, 1, 2); + t107.callTuplePush(1, 2); t107.callTuplePush(3, 4).callTuplePush(5, 6); var first: int = t107.callTupleFirst(); return ( diff --git a/tolk-tester/tests/generics-2.tolk b/tolk-tester/tests/generics-2.tolk index b3e0ea5b9d..8e95d884b0 100644 --- a/tolk-tester/tests/generics-2.tolk +++ b/tolk-tester/tests/generics-2.tolk @@ -65,11 +65,21 @@ fun getWrappervalue1(c: Wrapper) { return c.value; } +@pure +fun Wrapper.getWrappervalue1(self) { + return self.value; +} + @pure fun getWrappervalue2(c: T) { return c.value; } +@pure +fun T.getWrappervalue2(self) { + return self.value; +} + @method_id(103) fun test3() { var c1 = wrap(10); @@ -405,6 +415,17 @@ fun test24() { __expect_type(o, "HasSomethingAndWrapper"); } +fun test25() { + var p1 = Pair { first: null, second: null }; + __expect_type(p1, "Pair"); + + try {} + catch(e, second) { + var p2 = Pair { first: null, second }; + __expect_type(p2, "Pair"); + } +} + fun main(c: Wrapper, d: WrappedInt) { __expect_type(c, "Wrapper"); diff --git a/tolk-tester/tests/generics-3.tolk b/tolk-tester/tests/generics-3.tolk index 77d0461134..4539d61446 100644 --- a/tolk-tester/tests/generics-3.tolk +++ b/tolk-tester/tests/generics-3.tolk @@ -118,7 +118,7 @@ struct WithDef { fun test8() { var w1: WithDef = {}; var w2: WithDef> = { price: ton("0.1"), f3: { result: 12 } }; - return (w1.price, w1.f1, w1.f3, w2.price, w2.f3!.result, w2.f3, w2.slice.getRemainingBitsCount()); + return (w1.price, w1.f1, w1.f3, w2.price, w2.f3!.result, w2.f3, w2.slice.remainingBitsCount()); } diff --git a/tolk-tester/tests/if-else-tests.tolk b/tolk-tester/tests/if-else-tests.tolk index 8c314b8edf..159a86bd57 100644 --- a/tolk-tester/tests/if-else-tests.tolk +++ b/tolk-tester/tests/if-else-tests.tolk @@ -68,7 +68,7 @@ fun doNothing(): void {} @method_id(107) fun test7() { - if (random()) { return doNothing() } + if (random.uint256()) { return doNothing() } // here we test that no "missing return" error } diff --git a/tolk-tester/tests/indexed-access.tolk b/tolk-tester/tests/indexed-access.tolk index f6a376ad3d..f84d2d93ed 100644 --- a/tolk-tester/tests/indexed-access.tolk +++ b/tolk-tester/tests/indexed-access.tolk @@ -2,10 +2,21 @@ type Tup2Int = [int, MInt]; type Tensor2Int = (int, MInt); type MInt = int; -fun increment(mutate self: MInt) { +@pure +fun tuplePush(mutate t: tuple, value: T): void + asm "TPUSH"; +@pure +fun getBuilderBitsCount(b: builder): int + asm "BBITS"; + +fun MInt.increment(mutate self) { self += 1; } +fun increment(mutate v: MInt) { + v += 1; +} + fun increment2(mutate a: int, mutate b: int) { a += 1; b += 1; @@ -16,12 +27,18 @@ fun assign1020(mutate a: int, mutate b: MInt) { b = 20; } -fun plus(mutate self: int, y: int): int { +fun int.plus(mutate self, y: int): int { val newVals = (self + y, y * 10); self = newVals.0; return newVals.1; } +fun plus(mutate x: int, y: int): int { + val newVals = (x + y, y * 10); + x = newVals.0; + return newVals.1; +} + fun eq(v: X): X { return v; } global gTup: [int]; @@ -31,16 +48,16 @@ global gTens: (int, int); fun testCodegenSimple() { var t1 = [1]; t1.0 = 2; - debugPrintString(""); + debug.printString(""); var t2 = [[1]]; t2.0.0 = 2; - debugPrintString(""); + debug.printString(""); gTup = [1]; gTup.0 = 2; - debugPrintString(""); + debug.printString(""); gTens = (1,2); gTens.1 = 4; - debugPrintString(""); + debug.printString(""); return (t1, t2, gTup, gTens); } @@ -90,8 +107,8 @@ fun test104() { @method_id(105) fun test105(x: int, y: int): (tuple, int, (int?, int), int, int) { - var ab = (createEmptyTuple(), (x as int?, y), tupleSize); - ab.0.tuplePush(1); + var ab = (createEmptyTuple(), (x as int?, y), tuple.size); + ab.0.push(1); tuplePush(mutate ab.0, 2); ab.1.0 = null; ab.1.1 += 10; @@ -101,8 +118,8 @@ fun test105(x: int, y: int): (tuple, int, (int?, int), int, int) { @method_id(106) fun test106(x: int, y: int) { - var ab = [createEmptyTuple(), [x as int?, y], tupleSize]; - ab.0.tuplePush(1); + var ab = [createEmptyTuple(), [x as int?, y], tuple.size]; + ab.0.push(1); tuplePush(mutate ab.0, 2); ab.1.0 = null; ab.1.1 += 10; @@ -113,8 +130,8 @@ fun test106(x: int, y: int) { @method_id(107) fun test107() { var ab = createEmptyTuple(); - ab.tuplePush(1); - ab.tuplePush(beginCell().storeInt(1, 32)); + ab.push(1); + ab.push(beginCell().storeInt(1, 32)); return (ab.0 as int, getBuilderBitsCount(ab.1)); } @@ -187,7 +204,7 @@ fun test115() { @method_id(116) fun test116() { var t = createEmptyTuple(); - t.tuplePush(1); + t.push(1); try { return t.100500 as int; } catch(excNo) { @@ -198,7 +215,7 @@ fun test116() { @method_id(117) fun test117() { var t = createEmptyTuple(); - t.tuplePush(1); + t.push(1); try { return (t.0 as tuple).0 as MInt; } catch(excNo) { @@ -228,14 +245,18 @@ fun test120() { @method_id(121) fun test121(zero: int) { var t = createEmptyTuple(); - t.tuplePush(-100); - t.tupleSetAt(0, zero); + t.push(-100); + t.set(0, zero); (t.0 as int).increment(); (((t.0) as int) as int).increment(); increment(mutate t.0 as int); return t; } +fun (T1, T2).isFirstComponentGt0(self): bool { + return self.0 > 0; +} + fun isFirstComponentGt0(t: (T1, T2)): bool { return t.0 > 0; } @@ -244,7 +265,7 @@ fun isFirstComponentGt0(t: (T1, T2)): bool { fun test122(x: (int, int)) { return ( isFirstComponentGt0(x), isFirstComponentGt0((2, beginCell())), isFirstComponentGt0((0, null)), - x.isFirstComponentGt0(), (2, beginCell()).isFirstComponentGt0(), (0, null).isFirstComponentGt0() + x.isFirstComponentGt0(), (2, beginCell()).isFirstComponentGt0(), (0, null as slice?).isFirstComponentGt0() ); } diff --git a/tolk-tester/tests/inference-tests.tolk b/tolk-tester/tests/inference-tests.tolk index affe42ee1e..9c836baf4f 100644 --- a/tolk-tester/tests/inference-tests.tolk +++ b/tolk-tester/tests/inference-tests.tolk @@ -15,7 +15,7 @@ fun test1(x: int, y: int) { __expect_type(x = x, "int"); __expect_type(x += x, "int"); __expect_type(x &= x, "int"); - __expect_type(random() ? x : y, "int"); + __expect_type(random.uint256() ? x : y, "int"); __expect_type(eq(x), "int"); __expect_type(eq(x), "int"); __expect_type(eq(null), "int?"); @@ -47,7 +47,7 @@ fun test2(x: int, y: bool) { __expect_type(x <= x, "bool"); __expect_type(x <=> x, "bool"); __expect_type(x <=> x, "bool"); - __expect_type(!random(), "bool"); + __expect_type(!random.uint256(), "bool"); __expect_type(!!(x != null), "bool"); __expect_type(x ? x != null : null == x, "bool"); __expect_type(y & true, "bool"); @@ -60,7 +60,7 @@ fun test2(x: int, y: bool) { fun test3() { __expect_type(true as int, "int"); - __expect_type(!random() as int, "int"); + __expect_type(!random.uint256() as int, "int"); } fun test4(x: int) { @@ -75,13 +75,13 @@ fun test5(x: int) { __expect_type([x, x >= 1], "[int, bool]"); __expect_type([x, x >= 1, null as slice?], "[int, bool, slice?]"); __expect_type((x, [x], [[x], x]), "(int, [int], [[int], int])"); - __expect_type(getMyOriginalBalanceWithExtraCurrencies(), "[int, dict]"); + __expect_type(contract.getOriginalBalanceWithExtraCurrencies(), "[coins, dict]"); } fun test6() { var t = createEmptyTuple(); __expect_type(t, "tuple"); - t.tuplePush(1); + t.push(1); __expect_type(t, "tuple"); } diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index 4b2fccb25c..6f44ed0305 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -10,28 +10,29 @@ const someN_coins = 10 as coins; const cost1 = ton("1.234"); const cost2 = ton("0.05") + ton("0.001"); +fun rand(): uint256 { return random.uint256(); } fun autoInferInt8(x: int8) { - if (random()) { return x; } - else if (random()) { return 0 as int8; } - else if (random()) { return x!; } + if (rand()) { return x; } + else if (rand()) { return 0 as int8; } + else if (rand()) { return x!; } else { return x += x; } } fun autoInferUInt16Opt(x: uint16) { - if (random()) { return autoInferInt8(x as int8) as uint16; } + if (rand()) { return autoInferInt8(x as int8) as uint16; } return null; } fun inferInt_v1(): int { - if (random()) { return 0; } - else if (random()) { return 0 as uint16; } + if (rand()) { return 0; } + else if (rand()) { return 0 as uint16; } else { return 0 as int8; } } fun inferInt_v2(): int { - if (random()) { return 0 as uint1; } - else if (random()) { return 0 as int123; } + if (rand()) { return 0 as uint1; } + else if (rand()) { return 0 as int123; } else { return 0 as int8; } } @@ -218,9 +219,9 @@ fun assign0(mutate v: T) { v = 0; } fun main() { var t = createEmptyTuple(); - t.tuplePush(1); - t.tuplePush(2); - t.tuplePush(3); + t.push(1); + t.push(2); + t.push(3); assign0(mutate t.0 as int8); assign0(mutate t.1 as uint16); assign0(mutate t.2 as int); diff --git a/tolk-tester/tests/invalid-declaration/err-1074.tolk b/tolk-tester/tests/invalid-declaration/err-1074.tolk new file mode 100644 index 0000000000..b3298436fc --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1074.tolk @@ -0,0 +1,6 @@ +fun Container.create() {} + +/** +@compilation_should_fail +@stderr unknown type name `Container` + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1140.tolk b/tolk-tester/tests/invalid-declaration/err-1140.tolk index eb1ca04e57..ab902fc265 100644 --- a/tolk-tester/tests/invalid-declaration/err-1140.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1140.tolk @@ -21,6 +21,7 @@ fun autoInferUnionType(a: int8, b: int16) { /** @compilation_should_fail -@stderr function `autoInferUnionType` calculated return type is `int8 | int16`; probably, it's not what you expected; declare `fun (...): ` manually +@stderr function `autoInferUnionType` calculated return type is `int8 | int16`; probably, it's not what you expected +@stderr declare `fun (...): ` manually @stderr fun autoInferUnionType */ diff --git a/tolk-tester/tests/invalid-declaration/err-1193.tolk b/tolk-tester/tests/invalid-declaration/err-1193.tolk new file mode 100644 index 0000000000..45124998e2 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1193.tolk @@ -0,0 +1,7 @@ +fun int.smth(self: int) { +} + +/** +@compilation_should_fail +@stderr `self` parameter should not have a type + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1226.tolk b/tolk-tester/tests/invalid-declaration/err-1226.tolk new file mode 100644 index 0000000000..12620289c3 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1226.tolk @@ -0,0 +1,14 @@ +type MInt = int; +type MInt2 = MInt; + +fun int.check(self) {} +fun MInt.check(self) {} +fun MInt2.check(self) {} +fun `MInt`.`check`(self) {} + +/** +@compilation_should_fail +@stderr redefinition of symbol +@stderr err-1226.tolk:7 +@stderr err-1226.tolk:5 + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1458.tolk b/tolk-tester/tests/invalid-declaration/err-1458.tolk new file mode 100644 index 0000000000..c1d24dfa2d --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1458.tolk @@ -0,0 +1,13 @@ +struct Pair { + first: A; + second: B; +} + +type PairAlias = Pair; + +fun PairAlias.create() {} + +/** +@compilation_should_fail +@stderr type `PairAlias` expects 2 type arguments, but 1 provided + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1556.tolk b/tolk-tester/tests/invalid-declaration/err-1556.tolk new file mode 100644 index 0000000000..831ab2d496 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1556.tolk @@ -0,0 +1,7 @@ +fun nothing(self) { +} + +/** +@compilation_should_fail +@stderr `self` can only be the first parameter of a method + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1601.tolk b/tolk-tester/tests/invalid-declaration/err-1601.tolk deleted file mode 100644 index 588c70ab27..0000000000 --- a/tolk-tester/tests/invalid-declaration/err-1601.tolk +++ /dev/null @@ -1,8 +0,0 @@ -fun increment(x: int, self: int): int { - return x + self; -} - -/** -@compilation_should_fail -@stderr `self` can only be the first parameter - */ diff --git a/tolk-tester/tests/invalid-declaration/err-1663.tolk b/tolk-tester/tests/invalid-declaration/err-1663.tolk new file mode 100644 index 0000000000..0c41e4c2c6 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1663.tolk @@ -0,0 +1,11 @@ +struct Container { + item: T; +} + +fun Container.wrap(item: T) {} + +/** +@compilation_should_fail +@stderr duplicate generic parameter `T` +@stderr fun Container.wrap + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1701.tolk b/tolk-tester/tests/invalid-declaration/err-1701.tolk index 6a7f1ca7f2..e942ca470b 100644 --- a/tolk-tester/tests/invalid-declaration/err-1701.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1701.tolk @@ -1,10 +1,12 @@ +// try to cheat fun moddiv2(x: int, y: int): (int, int) builtin; +// but actually, it remains unknown symbol + +fun main() { + moddiv2(1, 2); +} /** @compilation_should_fail -@stderr -""" -`builtin` used for non-builtin function -fun moddiv2 -""" +@stderr undefined symbol `moddiv2` */ diff --git a/tolk-tester/tests/invalid-declaration/err-1790.tolk b/tolk-tester/tests/invalid-declaration/err-1790.tolk index 75ebb450bc..4a845db8d7 100644 --- a/tolk-tester/tests/invalid-declaration/err-1790.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1790.tolk @@ -1,7 +1,7 @@ // this function is declared incorrectly, // since it should return 2 values onto a stack (1 for returned slice, 1 for mutated int) // but contains not 2 numbers in asm ret_order -fun loadAddress2(mutate self: int): slice +fun int.loadAddress2(mutate self): slice asm( -> 1 0 2) "LDMSGADDR"; fun main(){} diff --git a/tolk-tester/tests/invalid-declaration/err-1794.tolk b/tolk-tester/tests/invalid-declaration/err-1794.tolk index 42cb7b9533..a064a633c2 100644 --- a/tolk-tester/tests/invalid-declaration/err-1794.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1794.tolk @@ -1,8 +1,8 @@ -get seqno(self: int) { +get int.seqno(self) { return 0; } /** @compilation_should_fail -@stderr get methods can't have `mutate` and `self` params +@stderr invalid declaration of a get method */ diff --git a/tolk-tester/tests/invalid-declaration/err-1912.tolk b/tolk-tester/tests/invalid-declaration/err-1912.tolk new file mode 100644 index 0000000000..2674580cad --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1912.tolk @@ -0,0 +1,9 @@ +struct Point { x: int, y: int } + +fun Point.create() {} + +/** +@compilation_should_fail +@stderr type `Point` is not generic +@stderr fun Point + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1973.tolk b/tolk-tester/tests/invalid-declaration/err-1973.tolk new file mode 100644 index 0000000000..d00819e4b0 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1973.tolk @@ -0,0 +1,7 @@ +fun int.nothing(zero: int, self: slice) { +} + +/** +@compilation_should_fail +@stderr `self` can only be the first parameter of a method + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4057.tolk b/tolk-tester/tests/invalid-semantics/err-4057.tolk new file mode 100644 index 0000000000..a5de199dc6 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4057.tolk @@ -0,0 +1,17 @@ +fun int16.plus1(self) { return self + 100; } +fun int8.plus1(self) { return self + 1; } + +fun main() { + (10 as int16).plus1(); // ok + (10 as int8).plus1(); // ok + + 10.plus1(); +} + +/** +@compilation_should_fail +@stderr call to method `plus1` for type `int` is ambiguous +@stderr candidate function: `int16.plus1` +@stderr candidate function: `int8.plus1` +@stderr 10.plus1 + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4064.tolk b/tolk-tester/tests/invalid-semantics/err-4064.tolk index 4f0e9142af..e2336ab3be 100644 --- a/tolk-tester/tests/invalid-semantics/err-4064.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4064.tolk @@ -12,9 +12,6 @@ fun main(): int { /** @compilation_should_fail -@stderr -""" -an impure operation in a pure function -return f_impure(); -""" +@stderr an impure operation in a pure function +@stderr in function `f_pure` */ diff --git a/tolk-tester/tests/invalid-semantics/err-4080.tolk b/tolk-tester/tests/invalid-semantics/err-4080.tolk index c8f7dcebfe..df569995fc 100644 --- a/tolk-tester/tests/invalid-semantics/err-4080.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4080.tolk @@ -1,4 +1,4 @@ -fun methodWith1Param(self: int, param: int) { +fun int.methodWith1Param(self, param: int) { } @@ -9,5 +9,5 @@ fun main() { /** @compilation_should_fail -@stderr too many arguments in call to `methodWith1Param`, expected 1, have 2 +@stderr too many arguments in call to `int.methodWith1Param`, expected 1, have 2 */ diff --git a/tolk-tester/tests/invalid-semantics/err-4083.tolk b/tolk-tester/tests/invalid-semantics/err-4083.tolk index 9da6e25340..aac7417bd3 100644 --- a/tolk-tester/tests/invalid-semantics/err-4083.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4083.tolk @@ -1,6 +1,6 @@ fun cantCallMutatingFunctionWithAssignmentLValue() { var t: tuple = createEmptyTuple(); - (t = createEmptyTuple()).tuplePush(1); + (t = createEmptyTuple()).push(1); } /** diff --git a/tolk-tester/tests/invalid-semantics/err-4104.tolk b/tolk-tester/tests/invalid-semantics/err-4104.tolk index 73e6403fd5..3ca7ef1bc0 100644 --- a/tolk-tester/tests/invalid-semantics/err-4104.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4104.tolk @@ -1,10 +1,20 @@ -fun eq(t: X) { return t; } +struct Container { + item: T; +} + +fun Container.createFrom(item: U): Container { + return { item }; +} + +fun cantUseAsNonCall() { + Container.createFrom; // ok + Container.createFrom>; // ok -fun failUsingGenericFunctionPartially() { - var cb = createEmptyTuple().eq().eq().tuplePush; + Container.createFrom; } /** @compilation_should_fail -@stderr can not use a generic function `tuplePush` as non-call +@stderr can not use a generic function `Container.createFrom` as non-call +@stderr Container.createFrom; */ diff --git a/tolk-tester/tests/invalid-semantics/err-4115.tolk b/tolk-tester/tests/invalid-semantics/err-4115.tolk new file mode 100644 index 0000000000..26bcf0c1f5 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4115.tolk @@ -0,0 +1,14 @@ +fun (T|int).equals(self, v: U) { return self == v; } +fun T?.equals(self, v: U) { return self == v; } + +fun main() { + (10 as int?).equals(20); +} + +/** +@compilation_should_fail +@stderr error: call to method `equals` for type `int?` is ambiguous +@stderr candidate function: `(T|int).equals` with T=`null` +@stderr candidate function: `T?.equals` with T=`int` +@stderr (10 as int?).equals(20); + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4127.tolk b/tolk-tester/tests/invalid-semantics/err-4127.tolk index 339addfb57..cae958ab60 100644 --- a/tolk-tester/tests/invalid-semantics/err-4127.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4127.tolk @@ -3,5 +3,5 @@ const asdf = ttt; /** @compilation_should_fail -@stderr generic T not expected here +@stderr type arguments not expected here */ diff --git a/tolk-tester/tests/invalid-semantics/err-4190.tolk b/tolk-tester/tests/invalid-semantics/err-4190.tolk index 70355bf983..97497bd629 100644 --- a/tolk-tester/tests/invalid-semantics/err-4190.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4190.tolk @@ -1,6 +1,6 @@ struct A { ok: coins = ton("0.05"), - err: int = now(), + err: int = blockchain.now(), } /** diff --git a/tolk-tester/tests/invalid-semantics/err-4195.tolk b/tolk-tester/tests/invalid-semantics/err-4195.tolk index bb8cde0587..a4170ffdd6 100644 --- a/tolk-tester/tests/invalid-semantics/err-4195.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4195.tolk @@ -1,5 +1,9 @@ fun getNullableTuple(): tuple? { return createEmptyTuple(); } +@pure +fun tuplePush(mutate t: tuple, value: T): void + asm "TPUSH"; + fun cantUseLValueUnwrappedNotNull() { tuplePush(mutate getNullableTuple()!, 1); } diff --git a/tolk-tester/tests/invalid-semantics/err-4256.tolk b/tolk-tester/tests/invalid-semantics/err-4256.tolk new file mode 100644 index 0000000000..6b29b8aaba --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4256.tolk @@ -0,0 +1,18 @@ +type MInt = int; +type IntOrSlice = MInt|slice; + +fun (int | slice).method(self) {} +fun (slice | MInt).method(self) {} +fun IntOrSlice.method(self) {} + +fun main(p: int|slice) { + p.method(); +} + +/** +@compilation_should_fail +@stderr call to method `method` for type `int | slice` is ambiguous +@stderr candidate function: `(int|slice).method` +@stderr candidate function: `(slice|MInt).method` +@stderr candidate function: `IntOrSlice.method` +*/ diff --git a/tolk-tester/tests/invalid-semantics/err-4266.tolk b/tolk-tester/tests/invalid-semantics/err-4266.tolk index c7f72bf4d8..6d6ec061bb 100644 --- a/tolk-tester/tests/invalid-semantics/err-4266.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4266.tolk @@ -1,9 +1,9 @@ fun invalidReferencingGenericMethodWithoutGeneric() { var t = createEmptyTuple(); - var cb = t.tupleLast; + var cb = t.last; } /** @compilation_should_fail -@stderr can not use a generic function `tupleLast` as non-call +@stderr can not use a generic function `tuple.last` as non-call */ diff --git a/tolk-tester/tests/invalid-semantics/err-4270.tolk b/tolk-tester/tests/invalid-semantics/err-4270.tolk index 14aef95ca3..455c163451 100644 --- a/tolk-tester/tests/invalid-semantics/err-4270.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4270.tolk @@ -1,5 +1,5 @@ struct A { - tens: (int, int) = (1, 2 + now()), + tens: (int, int) = (1, 2 + blockchain.now()), } /** diff --git a/tolk-tester/tests/invalid-semantics/err-4291.tolk b/tolk-tester/tests/invalid-semantics/err-4291.tolk index 907cc76c85..a7fadaa2d1 100644 --- a/tolk-tester/tests/invalid-semantics/err-4291.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4291.tolk @@ -3,5 +3,6 @@ struct CollisionName { item: T1; } /** @compilation_should_fail -@stderr type `T1` is not generic +@stderr unknown type name `T1` +@stderr item: T1 */ diff --git a/tolk-tester/tests/invalid-semantics/err-4304.tolk b/tolk-tester/tests/invalid-semantics/err-4304.tolk index 9636934140..032586aaa1 100644 --- a/tolk-tester/tests/invalid-semantics/err-4304.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4304.tolk @@ -6,5 +6,5 @@ fun main() { /** @compilation_should_fail -@stderr type `MInt` can not be used as a value +@stderr `MInt` only refers to a type, but is being used as a value here */ diff --git a/tolk-tester/tests/invalid-semantics/err-4319.tolk b/tolk-tester/tests/invalid-semantics/err-4319.tolk new file mode 100644 index 0000000000..f1438ea330 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4319.tolk @@ -0,0 +1,17 @@ +type MInt = int; + +fun (int | slice).method(self) {} +fun (MInt|null).method(self) {} +fun int8?.method(self) {} + +fun main(p: int8) { + p.method(); +} + +/** +@compilation_should_fail +@stderr call to method `method` for type `int8` is ambiguous +@stderr candidate function: `(int|slice).method` +@stderr candidate function: `MInt?.method` +@stderr candidate function: `int8?.method` +*/ diff --git a/tolk-tester/tests/invalid-semantics/err-4385.tolk b/tolk-tester/tests/invalid-semantics/err-4385.tolk index 31d4f02132..767f524f6a 100644 --- a/tolk-tester/tests/invalid-semantics/err-4385.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4385.tolk @@ -1,6 +1,6 @@ @pure fun validate_input(input: cell): (int, int) { - var (x, y, z, correct) = calculateCellSize(input, 10); + var (x, y, z, correct) = input.calculateSize(10); assert(correct) throw 102; return (x, y); } @@ -16,9 +16,7 @@ fun main() {} /** @compilation_should_fail -@stderr -""" -an impure operation in a pure function -assert(correct) -""" +@stderr an impure operation in a pure function +@stderr in function `validate_input` +@stderr assert(correct) */ diff --git a/tolk-tester/tests/invalid-semantics/err-4387.tolk b/tolk-tester/tests/invalid-semantics/err-4387.tolk index 50e02ed7bc..0de2871acb 100644 --- a/tolk-tester/tests/invalid-semantics/err-4387.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4387.tolk @@ -12,6 +12,6 @@ fun main(d: nested) { /** @compilation_should_fail -@stderr struct `Wrapper` can not be used as a value +@stderr `Wrapper` only refers to a type, but is being used as a value here @stderr c2 = Wrapper */ diff --git a/tolk-tester/tests/invalid-semantics/err-4413.tolk b/tolk-tester/tests/invalid-semantics/err-4413.tolk index 2132068347..6ee576dcc4 100644 --- a/tolk-tester/tests/invalid-semantics/err-4413.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4413.tolk @@ -15,9 +15,7 @@ fun main(): int { /** @compilation_should_fail -@stderr -""" -an impure operation in a pure function -g = g + 1; -""" -*/ +@stderr an impure operation in a pure function +@stderr in function `f_pure` +@stderr g = g + 1; + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4473.tolk b/tolk-tester/tests/invalid-semantics/err-4473.tolk index dfc69851cf..9d4ad1cc04 100644 --- a/tolk-tester/tests/invalid-semantics/err-4473.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4473.tolk @@ -1,4 +1,4 @@ -fun load32(self: slice): int { +fun slice.load32(self): int { return self.loadUint(32); } diff --git a/tolk-tester/tests/invalid-semantics/err-4480.tolk b/tolk-tester/tests/invalid-semantics/err-4480.tolk new file mode 100644 index 0000000000..a0f0da32dc --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4480.tolk @@ -0,0 +1,10 @@ +fun main() { + Point2.create(); +} + +/** +@compilation_should_fail +@stderr unknown type name `Point2` +@stderr in function `main` +@stderr Point2.create() + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4506.tolk b/tolk-tester/tests/invalid-semantics/err-4506.tolk deleted file mode 100644 index 8cd37c5171..0000000000 --- a/tolk-tester/tests/invalid-semantics/err-4506.tolk +++ /dev/null @@ -1,16 +0,0 @@ -fun increment(mutate x: int) { - x = x + 1; -} - -fun cantCallMutatingAsAMember() { - var x = 0; - x.increment(); - return x; -} - -/** -@compilation_should_fail -@stderr function `increment` mutates parameter `x` -@stderr consider calling `increment(mutate x)`, not `x.increment`() -@stderr alternatively, rename parameter to `self` to make it a method - */ diff --git a/tolk-tester/tests/invalid-semantics/err-4510.tolk b/tolk-tester/tests/invalid-semantics/err-4510.tolk index a399bc9179..82aaaaccfa 100644 --- a/tolk-tester/tests/invalid-semantics/err-4510.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4510.tolk @@ -7,5 +7,5 @@ fun main() { /** @compilation_should_fail -@stderr generic T not expected here +@stderr type arguments not expected here */ diff --git a/tolk-tester/tests/invalid-semantics/err-4520.tolk b/tolk-tester/tests/invalid-semantics/err-4520.tolk index 199aa681a4..994440163b 100644 --- a/tolk-tester/tests/invalid-semantics/err-4520.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4520.tolk @@ -6,5 +6,5 @@ fun main() { /** @compilation_should_fail -@stderr type `int` is not indexable +@stderr method `3` not found for type `int` */ diff --git a/tolk-tester/tests/invalid-semantics/err-4542.tolk b/tolk-tester/tests/invalid-semantics/err-4542.tolk index 9327f07d81..e1ef833e58 100644 --- a/tolk-tester/tests/invalid-semantics/err-4542.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4542.tolk @@ -1,5 +1,5 @@ @pure -fun tupleMut(mutate self: tuple): int +fun tuple.tupleMut(mutate self): int asm "TLEN"; fun main() { @@ -9,5 +9,5 @@ fun main() { /** @compilation_should_fail -@stderr saving `tupleMut` into a variable is impossible, since it has `mutate` parameters +@stderr saving `tuple.tupleMut` into a variable is impossible, since it has `mutate` parameters */ diff --git a/tolk-tester/tests/invalid-semantics/err-4588.tolk b/tolk-tester/tests/invalid-semantics/err-4588.tolk new file mode 100644 index 0000000000..9c4ded675b --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4588.tolk @@ -0,0 +1,17 @@ +struct Point { + x: int; + y: int; +} + +fun getX(i: int) { return i; } +fun Point.getX(self) { return self.x; } + +fun main() { + var Point = 10; + Point.getX(); +} + +/** +@compilation_should_fail +@stderr method `getX` not found for type `int` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4598.tolk b/tolk-tester/tests/invalid-semantics/err-4598.tolk new file mode 100644 index 0000000000..3ef699380a --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4598.tolk @@ -0,0 +1,16 @@ +struct Container { + item: T; +} + +fun Container.wrap(item: T) {} + +fun main(c: Container) { + c.wrap(); +} + +/** +@compilation_should_fail +@stderr method `Container.wrap` can not be called via dot +@stderr it's a static method, it does not accept `self` +@stderr c.wrap(); + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4604.tolk b/tolk-tester/tests/invalid-semantics/err-4604.tolk index 73fd6f87c0..07637d9dab 100644 --- a/tolk-tester/tests/invalid-semantics/err-4604.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4604.tolk @@ -4,5 +4,5 @@ fun invalidProvidingGenericTsToNotGeneric() { /** @compilation_should_fail -@stderr calling a not generic function with generic T +@stderr type arguments not expected here */ diff --git a/tolk-tester/tests/invalid-semantics/err-4610.tolk b/tolk-tester/tests/invalid-semantics/err-4610.tolk new file mode 100644 index 0000000000..2c495602d5 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4610.tolk @@ -0,0 +1,13 @@ +fun int.create(value: int) { + return value; +} + +fun main() { + 10.create(); +} + +/** +@compilation_should_fail +@stderr method `int.create` can not be called via dot +@stderr it's a static method, it does not accept `self` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4682.tolk b/tolk-tester/tests/invalid-semantics/err-4682.tolk index 054a6a4032..0a042411b4 100644 --- a/tolk-tester/tests/invalid-semantics/err-4682.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4682.tolk @@ -8,6 +8,6 @@ fun main() { /** @compilation_should_fail -@stderr struct `User` can not be used as a value +@stderr `User` only refers to a type, but is being used as a value here @stderr absk(User) */ diff --git a/tolk-tester/tests/invalid-semantics/err-4711.tolk b/tolk-tester/tests/invalid-semantics/err-4711.tolk deleted file mode 100644 index 01f4884878..0000000000 --- a/tolk-tester/tests/invalid-semantics/err-4711.tolk +++ /dev/null @@ -1,22 +0,0 @@ -struct Pair { - first: T1; - second: T2; -} - -fun main() { - var ok1: Pair = { first: null, second: null }; - var ok2: Pair = Pair { first: null, second: null }; - var ok3 = Pair { first: null, second: null }; - var ok4 = Pair { first: null as int?, second: null as (int, int)? }; - var ok5 = Pair { first: null as int?, second: null as slice? }; // ok since doesn't reach type checking - var ok6 = Pair { first: null, second: null }; - - var err = Pair { first: null, second: null }; -} - -/** -@compilation_should_fail -@stderr can not deduce T1 for generic struct `Pair` -@stderr instantiate it manually with `Pair<...>` -@stderr var err = - */ diff --git a/tolk-tester/tests/invalid-semantics/err-4739.tolk b/tolk-tester/tests/invalid-semantics/err-4739.tolk new file mode 100644 index 0000000000..58fa5efe53 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4739.tolk @@ -0,0 +1,12 @@ +fun T.pairWith(self, value: U) { + return [self, value]; +} + +fun main() { + 10.pairWith(); +} + +/** +@compilation_should_fail +@stderr too few arguments in call to `T.pairWith`, expected 1, have 0 + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4828.tolk b/tolk-tester/tests/invalid-semantics/err-4828.tolk deleted file mode 100644 index f6874fb8c5..0000000000 --- a/tolk-tester/tests/invalid-semantics/err-4828.tolk +++ /dev/null @@ -1,12 +0,0 @@ -fun asdf(mutate cs: slice) {} - -fun main(cs: slice) { - cs.asdf(); -} - -/** -@compilation_should_fail -@stderr function `asdf` mutates parameter `cs` -@stderr consider calling `asdf(mutate cs)`, not `cs.asdf`() -@stderr alternatively, rename parameter to `self` to make it a method - */ diff --git a/tolk-tester/tests/invalid-semantics/err-4841.tolk b/tolk-tester/tests/invalid-semantics/err-4841.tolk deleted file mode 100644 index cbf598066f..0000000000 --- a/tolk-tester/tests/invalid-semantics/err-4841.tolk +++ /dev/null @@ -1,12 +0,0 @@ -fun nothing() { -} - -fun main() { - val x = 0; - return x.nothing(); -} - -/** -@compilation_should_fail -@stderr `nothing` has no parameters and can not be called as method - */ diff --git a/tolk-tester/tests/invalid-semantics/err-4847.tolk b/tolk-tester/tests/invalid-semantics/err-4847.tolk new file mode 100644 index 0000000000..215c245c35 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4847.tolk @@ -0,0 +1,16 @@ +fun increment(mutate x: int) { + x += 1; +} + +fun testCantCallWithoutMutate() { + var x = 0; + increment(x); +} + +/** +@compilation_should_fail +@stderr function `increment` mutates parameter `x` +@stderr you need to specify `mutate` when passing an argument, like `mutate x` +@stderr in function `testCantCallWithoutMutate` +@stderr increment(x) + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4851.tolk b/tolk-tester/tests/invalid-semantics/err-4851.tolk index 3489a2889a..5195da0700 100644 --- a/tolk-tester/tests/invalid-semantics/err-4851.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4851.tolk @@ -1,4 +1,4 @@ -fun increment(self: int) { +fun int.increment(self) { self = self + 1; } diff --git a/tolk-tester/tests/invalid-semantics/err-4884.tolk b/tolk-tester/tests/invalid-semantics/err-4884.tolk new file mode 100644 index 0000000000..0515a4b2e8 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4884.tolk @@ -0,0 +1,13 @@ +fun T.copy(self) { + return self; +} + +fun main() { + int.copy(); +} + +/** +@compilation_should_fail +@stderr too few arguments in call to `int.copy`, expected 1, have 0 +@stderr int.copy(); + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4899.tolk b/tolk-tester/tests/invalid-semantics/err-4899.tolk deleted file mode 100644 index 2ba645d133..0000000000 --- a/tolk-tester/tests/invalid-semantics/err-4899.tolk +++ /dev/null @@ -1,8 +0,0 @@ -fun main(cs: slice) { - return loadInt(cs, 32); -} - -/** -@compilation_should_fail -@stderr `loadInt` is a mutating method; consider calling `cs.loadInt()`, not `loadInt(cs)` - */ diff --git a/tolk-tester/tests/invalid-semantics/err-4941.tolk b/tolk-tester/tests/invalid-semantics/err-4941.tolk deleted file mode 100644 index bb8cde0587..0000000000 --- a/tolk-tester/tests/invalid-semantics/err-4941.tolk +++ /dev/null @@ -1,10 +0,0 @@ -fun getNullableTuple(): tuple? { return createEmptyTuple(); } - -fun cantUseLValueUnwrappedNotNull() { - tuplePush(mutate getNullableTuple()!, 1); -} - -/** -@compilation_should_fail -@stderr function call can not be used as lvalue - */ diff --git a/tolk-tester/tests/invalid-semantics/err-4951.tolk b/tolk-tester/tests/invalid-semantics/err-4951.tolk index 87eb61e84c..6766b2490e 100644 --- a/tolk-tester/tests/invalid-semantics/err-4951.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4951.tolk @@ -6,5 +6,5 @@ fun main() { /** @compilation_should_fail -@stderr calling a not generic function with generic T +@stderr type arguments not expected here */ diff --git a/tolk-tester/tests/invalid-semantics/err-4956.tolk b/tolk-tester/tests/invalid-semantics/err-4956.tolk index 5a8c9fa5d7..de5309dc92 100644 --- a/tolk-tester/tests/invalid-semantics/err-4956.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4956.tolk @@ -10,5 +10,5 @@ fun main() { /** @compilation_should_fail -@stderr `mutate` used for non-mutate argument +@stderr `mutate` used for non-mutate parameter */ diff --git a/tolk-tester/tests/invalid-symbol/err-2188.tolk b/tolk-tester/tests/invalid-symbol/err-2188.tolk index cf8c788c14..6511fd40d2 100644 --- a/tolk-tester/tests/invalid-symbol/err-2188.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2188.tolk @@ -9,6 +9,6 @@ fun main() { /** @compilation_should_fail -@stderr non-existing method `storeUnexisting` of type `builder` +@stderr method `storeUnexisting` not found for type `builder` @stderr .storeUnexisting() */ diff --git a/tolk-tester/tests/invalid-symbol/err-2374.tolk b/tolk-tester/tests/invalid-symbol/err-2374.tolk index 08a86f1764..3a3ba567cd 100644 --- a/tolk-tester/tests/invalid-symbol/err-2374.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2374.tolk @@ -4,11 +4,11 @@ fun main(x: int): int { } else { var y: slice = "20"; } - debugPrint(y); + debug.print(y); } /** @compilation_should_fail -@stderr debugPrint(y); +@stderr debug.print(y); @stderr undefined symbol `y` */ diff --git a/tolk-tester/tests/invalid-symbol/err-2619.tolk b/tolk-tester/tests/invalid-symbol/err-2619.tolk new file mode 100644 index 0000000000..8eef3bc07a --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2619.tolk @@ -0,0 +1,8 @@ +fun main() { + return random(); +} + +/** +@compilation_should_fail +@stderr `random` is not a function, you probably want `random.uint256()` + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2788.tolk b/tolk-tester/tests/invalid-symbol/err-2788.tolk new file mode 100644 index 0000000000..64135e7017 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2788.tolk @@ -0,0 +1,9 @@ +fun main() { + var f = stringHexToSlice; + f("asdf"); +} + +/** +@compilation_should_fail +@stderr can not get reference to this function, it's compile-time only + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2914.tolk b/tolk-tester/tests/invalid-symbol/err-2914.tolk deleted file mode 100644 index 7435bb3cf0..0000000000 --- a/tolk-tester/tests/invalid-symbol/err-2914.tolk +++ /dev/null @@ -1,10 +0,0 @@ -const asdf = 1; - -fun main(x: int) { - return x.asdf(); -} - -/** -@compilation_should_fail -@stderr non-existing method `asdf` of type `int` - */ diff --git a/tolk-tester/tests/invalid-syntax/err-3407.tolk b/tolk-tester/tests/invalid-syntax/err-3407.tolk index 807e7be888..e37402deb9 100644 --- a/tolk-tester/tests/invalid-syntax/err-3407.tolk +++ b/tolk-tester/tests/invalid-syntax/err-3407.tolk @@ -7,5 +7,5 @@ not nested /** @compilation_should_fail -@stderr error: expected fun or get, got `*` +@stderr error: expected top-level declaration, got `*` */ diff --git a/tolk-tester/tests/invalid-syntax/err-3967.tolk b/tolk-tester/tests/invalid-syntax/err-3967.tolk index 3edc09fda9..9a158a390f 100644 --- a/tolk-tester/tests/invalid-syntax/err-3967.tolk +++ b/tolk-tester/tests/invalid-syntax/err-3967.tolk @@ -4,5 +4,5 @@ int main() { /** @compilation_should_fail -@stderr expected fun or get, got `int` +@stderr expected top-level declaration, got `int` */ diff --git a/tolk-tester/tests/invalid-typing/err-6013.tolk b/tolk-tester/tests/invalid-typing/err-6013.tolk new file mode 100644 index 0000000000..c5ec6783d7 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6013.tolk @@ -0,0 +1,18 @@ +struct Pair { + first: A; + second: B; +} + +fun Pair.demoInvalid(self) { + return self.first == (null as int?); +} + +fun main(p: Pair) { + p.demoInvalid(); +} + +/** +@compilation_should_fail +@stderr can not apply operator `==` to `int` and `int?` +@stderr in function `Pair.demoInvalid` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6080.tolk b/tolk-tester/tests/invalid-typing/err-6080.tolk index cf985addf7..d026750976 100644 --- a/tolk-tester/tests/invalid-typing/err-6080.tolk +++ b/tolk-tester/tests/invalid-typing/err-6080.tolk @@ -1,5 +1,5 @@ fun incrementOrSetNull(mutate x: int?) { - if (random()) { x! += 1; } + if (random.uint256()) { x! += 1; } else { x = null; } } diff --git a/tolk-tester/tests/invalid-typing/err-6085.tolk b/tolk-tester/tests/invalid-typing/err-6085.tolk deleted file mode 100644 index eb3adc9212..0000000000 --- a/tolk-tester/tests/invalid-typing/err-6085.tolk +++ /dev/null @@ -1,17 +0,0 @@ -fun eq(v: X) {} - -fun cantDeduceWhenNotInferred() { - // at type inferring (before type checking) they are unknown - var (x, y) = 2; - - eq(x as int); // ok (since execution doesn't reach type checking) - eq(x); // ok (since execution doesn't reach type checking) - eq(x); -} - -/** -@compilation_should_fail -@stderr in function `cantDeduceWhenNotInferred` -@stderr can not deduce X for generic function `eq` -@stderr eq(x); - */ diff --git a/tolk-tester/tests/invalid-typing/err-6087.tolk b/tolk-tester/tests/invalid-typing/err-6087.tolk index 463ba4871e..aabd956eac 100644 --- a/tolk-tester/tests/invalid-typing/err-6087.tolk +++ b/tolk-tester/tests/invalid-typing/err-6087.tolk @@ -5,5 +5,5 @@ fun cantAutoCastBytesNToSlice() { /** @compilation_should_fail -@stderr can not call method for `slice` with object of type `bits32` +@stderr method `loadInt` not found for type `bits32` */ diff --git a/tolk-tester/tests/invalid-typing/err-6134.tolk b/tolk-tester/tests/invalid-typing/err-6134.tolk index 1ee71290c1..b3972a7af6 100644 --- a/tolk-tester/tests/invalid-typing/err-6134.tolk +++ b/tolk-tester/tests/invalid-typing/err-6134.tolk @@ -1,4 +1,4 @@ -fun incNotChained(mutate self: int) { +fun int.incNotChained(mutate self) { self = self + 1; } @@ -8,5 +8,5 @@ fun cantCallNotChainedMethodsInAChain(x: int) { /** @compilation_should_fail -@stderr can not call method for `int` with object of type `void` +@stderr method `incNotChained` not found for type `void` */ diff --git a/tolk-tester/tests/invalid-typing/err-6148.tolk b/tolk-tester/tests/invalid-typing/err-6148.tolk index 1621bab19f..cbd305f216 100644 --- a/tolk-tester/tests/invalid-typing/err-6148.tolk +++ b/tolk-tester/tests/invalid-typing/err-6148.tolk @@ -9,6 +9,6 @@ fun testSmartCastsDropAfterMutate() { /** @compilation_should_fail -@stderr type `(int, int)?` is not indexable +@stderr field `1` doesn't exist in type `(int, int)?` @stderr return x.1 */ diff --git a/tolk-tester/tests/invalid-typing/err-6149.tolk b/tolk-tester/tests/invalid-typing/err-6149.tolk new file mode 100644 index 0000000000..7fd34b6462 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6149.tolk @@ -0,0 +1,18 @@ +struct Pair { + first: A; + second: B; +} + +fun Pair.compareWith(self, f: U, s: V) { + return self.first == f && self.second == s; +} + +fun main(p: Pair) { + p.compareWith(null, null); +} + +/** +@compilation_should_fail +@stderr can not apply operator `==` to `int` and `int?` +@stderr in function `Pair.compareWith` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6204.tolk b/tolk-tester/tests/invalid-typing/err-6204.tolk index 8f2b540da0..50c191f1f5 100644 --- a/tolk-tester/tests/invalid-typing/err-6204.tolk +++ b/tolk-tester/tests/invalid-typing/err-6204.tolk @@ -1,9 +1,9 @@ fun cantUnifyCoinsAndUInt8(n: int8, c: coins) { - __expect_type(random() ? n : c as int8, "int8"); - __expect_type(random() ? n as int16 : c as int16, "int16"); + __expect_type(random.uint256() ? n : c as int8, "int8"); + __expect_type(random.uint256() ? n as int16 : c as int16, "int16"); - var withHint: int = random() ? n : c; // ok - var withoutHint = random() ? n : c; // error + var withHint: int = random.uint256() ? n : c; // ok + var withoutHint = random.uint256() ? n : c; // error } /** diff --git a/tolk-tester/tests/invalid-typing/err-6249.tolk b/tolk-tester/tests/invalid-typing/err-6249.tolk index 274f51b4c2..1553bf75ee 100644 --- a/tolk-tester/tests/invalid-typing/err-6249.tolk +++ b/tolk-tester/tests/invalid-typing/err-6249.tolk @@ -1,7 +1,7 @@ fun cantUnifyBytesNAndSlice(n: slice, c: bytes16) { - __expect_type(random() ? n : c as slice, "slice"); - __expect_type(random() ? n : c as slice?, "slice?"); - random() ? n : c; + __expect_type(random.uint256() ? n : c as slice, "slice"); + __expect_type(random.uint256() ? n : c as slice?, "slice?"); + random.uint256() ? n : c; } /** diff --git a/tolk-tester/tests/invalid-typing/err-6257.tolk b/tolk-tester/tests/invalid-typing/err-6257.tolk index a007a93c27..5e18b376ca 100644 --- a/tolk-tester/tests/invalid-typing/err-6257.tolk +++ b/tolk-tester/tests/invalid-typing/err-6257.tolk @@ -1,9 +1,9 @@ -fun increment(mutate self: int): self { +fun int.increment(mutate self): self { self = self + 1; return self; } -fun cantReturnAnotherSelf(mutate self: int): self { +fun int.cantReturnAnotherSelf(mutate self): self { self = self + 1; var x = 0; return x.increment(); diff --git a/tolk-tester/tests/invalid-typing/err-6281.tolk b/tolk-tester/tests/invalid-typing/err-6281.tolk new file mode 100644 index 0000000000..7d77f411fa --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6281.tolk @@ -0,0 +1,12 @@ +struct Wrapper { item: T; } + +fun Wrapper.create(initial: T): Wrapper { return { item: initial }; } + +fun main() { + Wrapper>.create(123); +} + +/** +@compilation_should_fail +@stderr can not pass `int` to `Wrapper` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6288.tolk b/tolk-tester/tests/invalid-typing/err-6288.tolk index 40b54f1635..3ea5c02a1a 100644 --- a/tolk-tester/tests/invalid-typing/err-6288.tolk +++ b/tolk-tester/tests/invalid-typing/err-6288.tolk @@ -1,4 +1,4 @@ -fun cantReturnFromSelf(mutate self: int): self { +fun int.cantReturnFromSelf(mutate self): self { return 2; } diff --git a/tolk-tester/tests/invalid-typing/err-6337.tolk b/tolk-tester/tests/invalid-typing/err-6337.tolk deleted file mode 100644 index 5c9cbdc4c8..0000000000 --- a/tolk-tester/tests/invalid-typing/err-6337.tolk +++ /dev/null @@ -1,13 +0,0 @@ -@pure -fun myDictDeleteStrict(mutate self: cell, keyLen: int, key: int): bool - asm(key self keyLen) "DICTIDEL"; - - -fun testCantCallDictMethodsOnNullable(c: cell) { - c.beginParse().loadDict().myDictDeleteStrict(16, 1); -} - -/** -@compilation_should_fail -@stderr can not call method for `cell` with object of type `dict` - */ diff --git a/tolk-tester/tests/invalid-typing/err-6386.tolk b/tolk-tester/tests/invalid-typing/err-6386.tolk deleted file mode 100644 index 1559443383..0000000000 --- a/tolk-tester/tests/invalid-typing/err-6386.tolk +++ /dev/null @@ -1,10 +0,0 @@ -fun f(v: int, x: T) {} - -fun failCantDeduceWithPlainNull() { - return f(0, null); -} - -/** -@compilation_should_fail -@stderr can not deduce T for generic function `f` - */ diff --git a/tolk-tester/tests/invalid-typing/err-6391.tolk b/tolk-tester/tests/invalid-typing/err-6391.tolk index f6eb2f9fd1..1da57d8834 100644 --- a/tolk-tester/tests/invalid-typing/err-6391.tolk +++ b/tolk-tester/tests/invalid-typing/err-6391.tolk @@ -1,4 +1,4 @@ -fun acceptMutateNullableTensor(mutate self: (int, int)?) { +fun (int, int)?.acceptMutateNullableTensor(mutate self) { } fun cantModifyTupleIndexWithTypeTransition() { diff --git a/tolk-tester/tests/invalid-typing/err-6393.tolk b/tolk-tester/tests/invalid-typing/err-6393.tolk index a0d5ee04e5..28a9f76ca8 100644 --- a/tolk-tester/tests/invalid-typing/err-6393.tolk +++ b/tolk-tester/tests/invalid-typing/err-6393.tolk @@ -1,3 +1,6 @@ +fun tupleLast(t: tuple): T + asm "LAST"; + fun getTupleLastGetter(): tuple -> X { return tupleLast; } diff --git a/tolk-tester/tests/invalid-typing/err-6407.tolk b/tolk-tester/tests/invalid-typing/err-6407.tolk index 68c6c80432..8f0265de20 100644 --- a/tolk-tester/tests/invalid-typing/err-6407.tolk +++ b/tolk-tester/tests/invalid-typing/err-6407.tolk @@ -1,5 +1,5 @@ fun invalidNever(): never { - if (random()) { throw 123; } + if (random.uint256()) { throw 123; } } /** diff --git a/tolk-tester/tests/invalid-typing/err-6519.tolk b/tolk-tester/tests/invalid-typing/err-6519.tolk deleted file mode 100644 index 9d8cd480dc..0000000000 --- a/tolk-tester/tests/invalid-typing/err-6519.tolk +++ /dev/null @@ -1,13 +0,0 @@ -fun incNotChained(mutate self: int) { - self = self + 1; -} - -fun failWhenReturnANotChainedValue(x: int): int { - return x.incNotChained(); -} - -/** -@compilation_should_fail -@stderr x.incNotChained() -@stderr can not convert type `void` to return type `int` - */ diff --git a/tolk-tester/tests/invalid-typing/err-6629.tolk b/tolk-tester/tests/invalid-typing/err-6629.tolk index d2c24e5320..c68082fa6f 100644 --- a/tolk-tester/tests/invalid-typing/err-6629.tolk +++ b/tolk-tester/tests/invalid-typing/err-6629.tolk @@ -6,6 +6,6 @@ fun wrongTCountPassed() { /** @compilation_should_fail -@stderr wrong count of generic T: expected 2, got 1 +@stderr expected 2 type arguments, got 1 @stderr */ diff --git a/tolk-tester/tests/invalid-typing/err-6637.tolk b/tolk-tester/tests/invalid-typing/err-6637.tolk index 657ab5f4a2..26c7ce108c 100644 --- a/tolk-tester/tests/invalid-typing/err-6637.tolk +++ b/tolk-tester/tests/invalid-typing/err-6637.tolk @@ -1,6 +1,6 @@ fun autoGetIntOrNull() { - if (random()) { return 1; } + if (random.uint256()) { return 1; } return null; } diff --git a/tolk-tester/tests/invalid-typing/err-6659.tolk b/tolk-tester/tests/invalid-typing/err-6659.tolk index 82fa462b6a..326e239d81 100644 --- a/tolk-tester/tests/invalid-typing/err-6659.tolk +++ b/tolk-tester/tests/invalid-typing/err-6659.tolk @@ -1,15 +1,17 @@ +fun rand() { return random.uint256() } + fun cantAutoInferUnionInTernary() { - var dd: int | slice = random() ? 1 : ""; // ok - var cc = (random() ? 1 : "") as int|slice; // ok - random() ? 1 as int|slice : "" as int|slice; // ok - random() ? 1 as int|slice : ""; // ok - random() ? 1 : "" as int|slice|builder; // ok - random() ? 1 as int|slice|null : "" as slice?; // ok + var dd: int | slice = rand() ? 1 : ""; // ok + var cc = (rand() ? 1 : "") as int|slice; // ok + rand() ? 1 as int|slice : "" as int|slice; // ok + rand() ? 1 as int|slice : ""; // ok + rand() ? 1 : "" as int|slice|builder; // ok + rand() ? 1 as int|slice|null : "" as slice?; // ok - var sub = random() ? (1, 2 as int|slice) : (1, ""); + var sub = rand() ? (1, 2 as int|slice) : (1, ""); __expect_type(sub, "(int, int | slice)"); - random() ? 1 : ""; + rand() ? 1 : ""; } /** diff --git a/tolk-tester/tests/invalid-typing/err-6731.tolk b/tolk-tester/tests/invalid-typing/err-6731.tolk index 6a33e6968f..461e48d559 100644 --- a/tolk-tester/tests/invalid-typing/err-6731.tolk +++ b/tolk-tester/tests/invalid-typing/err-6731.tolk @@ -4,5 +4,5 @@ fun main(cs: slice) { /** @compilation_should_fail -@stderr referencing a method for `tuple` with object of type `slice` +@stderr field `tupleSize` doesn't exist in type `slice` */ diff --git a/tolk-tester/tests/invalid-typing/err-6820.tolk b/tolk-tester/tests/invalid-typing/err-6820.tolk index 47ebb1a2fb..0928582902 100644 --- a/tolk-tester/tests/invalid-typing/err-6820.tolk +++ b/tolk-tester/tests/invalid-typing/err-6820.tolk @@ -1,6 +1,6 @@ fun cantUnifyBytesNAndBytesM(n: bytes8, c: bytes16) { - __expect_type(random() ? n : c as bytes8, "bytes8"); - random() ? n : c; + __expect_type(random.uint256() ? n : c as bytes8, "bytes8"); + random.uint256() ? n : c; } /** diff --git a/tolk-tester/tests/invalid-typing/err-6839.tolk b/tolk-tester/tests/invalid-typing/err-6839.tolk index ac019a4216..91cca9c4fd 100644 --- a/tolk-tester/tests/invalid-typing/err-6839.tolk +++ b/tolk-tester/tests/invalid-typing/err-6839.tolk @@ -1,9 +1,9 @@ -fun incInt(mutate self: int): self { +fun int.incInt(mutate self): self { self += 1; return self; } -fun appendBuilder(mutate self: builder): self { +fun builder.appendBuilder(mutate self): self { self.storeUint(1, 32); return self; } @@ -15,5 +15,5 @@ fun cantMixDifferentThis() { /** @compilation_should_fail -@stderr can not call method for `builder` with object of type `int` +@stderr method `appendBuilder` not found for type `int` */ diff --git a/tolk-tester/tests/invalid-typing/err-6844.tolk b/tolk-tester/tests/invalid-typing/err-6844.tolk index 0be6b9e4df..3f719f2048 100644 --- a/tolk-tester/tests/invalid-typing/err-6844.tolk +++ b/tolk-tester/tests/invalid-typing/err-6844.tolk @@ -1,4 +1,4 @@ -fun cantReturnNothingFromSelf(mutate self: int): self { +fun int.cantReturnNothingFromSelf(mutate self): self { self = self + 1; } diff --git a/tolk-tester/tests/invalid-typing/err-6942.tolk b/tolk-tester/tests/invalid-typing/err-6942.tolk index 0ffe5c218b..95d0b9ad38 100644 --- a/tolk-tester/tests/invalid-typing/err-6942.tolk +++ b/tolk-tester/tests/invalid-typing/err-6942.tolk @@ -1,14 +1,12 @@ struct Point { x: int; y: int; } -fun methodOverNullable(p: Point?) {} -fun method(p: Point) {} +fun Point?.methodOverNullable(self) {} fun cantAssignNullableObjectField(a: Point?) { if (a != null) { a.x = 10; // ok } a.methodOverNullable(); // ok - a.method(); // also ok, would have failed on type checking, but doesn't reach type checking a.y = 20; } diff --git a/tolk-tester/tests/invalid-typing/err-6981.tolk b/tolk-tester/tests/invalid-typing/err-6981.tolk index 62a6f5da49..af3bebb8cb 100644 --- a/tolk-tester/tests/invalid-typing/err-6981.tolk +++ b/tolk-tester/tests/invalid-typing/err-6981.tolk @@ -1,15 +1,15 @@ -fun getTwo(): X { return 2; } +fun getTwo(): X { return 2 as X; } fun cantDeduceNonArgumentGeneric() { var t1: [int] = [0]; t1.0 = getTwo(); // ok var t2 = createEmptyTuple(); - t2.tuplePush(0); - t2.0 = getTwo(); // error, can't decude X + t2.push(0); + t2.0 = getTwo(); // deduced X = unknown } /** @compilation_should_fail -@stderr can not deduce X for generic function `getTwo` -@stderr t2.0 = getTwo(); +@stderr in function `getTwo` +@stderr type `int` can not be cast to `unknown` */ diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk index 2fd62db5b1..c925b8c1a8 100644 --- a/tolk-tester/tests/logical-operators.tolk +++ b/tolk-tester/tests/logical-operators.tolk @@ -120,7 +120,7 @@ fun testLogicalOps2(first: int) { (10 == s.loadUint(32)) || (20 == s.loadUint(32)) || (3 == s.loadUint(32)) || (4 == s.loadUint(32)); sum += s.loadUint(32); } - return (s.getRemainingBitsCount(), sum); + return (s.remainingBitsCount(), sum); } @method_id(112) diff --git a/tolk-tester/tests/methods-tests.tolk b/tolk-tester/tests/methods-tests.tolk new file mode 100644 index 0000000000..a54fe94157 --- /dev/null +++ b/tolk-tester/tests/methods-tests.tolk @@ -0,0 +1,311 @@ +import "@stdlib/common.tolk" +type MInt = int; + +global callOrder: tuple; + +fun int.zero() { return 0; } +fun int.plus1(self) { return self + 1; } +fun MInt.plusN(self, n: int) { return self + n; } +fun int.main(self) { return self; } + +struct Point { + x: int; + y: int; +} + +fun Point.create0() { return Point.create(0, 0); } +fun Point.create(x: int, y: int): Point { return {x,y}; } + +fun Point.getMaxCoord(self) { return self.x > self.y ? self.x : self.y; } +fun Point.incX(mutate self) { self.x += 1; } +fun Point.incXMutateY(mutate self, mutate curY: int) { self.x += 1; curY = self.y; } + +fun Point?.isNull(self) { return self == null; } + +struct Pair { + first: A; + second: B; +} + +fun Pair.create0(): Pair { + return { first: 0, second: 0 }; +} +fun Pair.create0(): Pair { + return { first: 0, second: 0 }; +} +fun Pair.createFrom(first: U, second: V): Pair { + return { first: first as A, second: second as B }; +} +fun Pair.is0(self) { + return self.first == 0 && self.second == 0; +} +fun Pair.compareWith(self, f: U, s: V) { + return self.first == f && self.second == s; +} + +type Tuple2Int = [int, int]; +fun [int, int].getLast(self) { return self.1; } +fun [int, MInt].assignLast(mutate self, v: int) { self.1 = v; } +fun Tuple2Int.getSum(self) { return self.0 + self.1; } + +fun int.calcExactF(self) { return self; } +fun int8.calcExactF(self) { return self * 2; } +fun int8?.calcExactF(self) { return self == null ? 0 : self * 3; } + +fun T.copy(self) { return self; } +fun T.assign(mutate self, value: T2) { self = value; } + +struct Wrapper { item: T; } + +fun Wrapper.create(initial: T): Wrapper { return { item: initial }; } +fun Wrapper.getItem(self) { return self.item; } +fun Wrapper.getItem(self) { return 100500; } +fun createWrapper(item: T) { return Wrapper.create(item); } + +@method_id(101) +fun test1() { + var i = int.zero() + MInt.zero(); + i = i.plus1(); + return (i.plus1(), i, i.plusN(5), i.plusN(i.plus1())); +} + +@method_id(102) +fun test2(a: int8, b: int16) { + var c = b as coins?; + return (a.plus1(), b.plusN(a), c!.plus1()); +} + +@method_id(103) +fun test3() { + var p = Point.create0(); + var yy = 10; + p.incX(); + Point.incX(mutate p); + p.incXMutateY(mutate yy); + p.y = p.x; + Point.incXMutateY(mutate p, mutate yy); + return (p, p.getMaxCoord(), Point.getMaxCoord(p), yy); +} + +@method_id(104) +fun test4() { + var p0 = Point.create0(); + var p5 = Point.create(5,5) as Point?; + var pN = null as Point?; + return (p0.isNull(), p5.isNull(), pN.isNull(), Point.isNull(null), Point.isNull({x:3,y:3})); +} + +@method_id(105) +fun test5(a: MInt, b: int) { + var t = [a,b]; + return (t.getSum(), t.assignLast(a+b), t.getSum(), Tuple2Int.getLast(t)); +} + +@method_id(106) +fun test6() { + var x = 5; + var x8 = x as int8; + return ( + x.calcExactF(), x8.calcExactF(), (8 as int8?).calcExactF(), (null as int8?).calcExactF(), + (5 as int16).calcExactF(), (x8 as coins).calcExactF(), (x8 as MInt).calcExactF(), + int8.calcExactF(x) + ); +} + +@method_id(107) +fun test7() { + __expect_type(5.copy(), "int"); + __expect_type((5 as int99).copy(), "int99"); + __expect_type(beginCell().storeInt(1,32).copy(), "builder"); + __expect_type(((null, null) as (MInt?,bool?)).copy(), "(MInt?, bool?)"); + return ( + beginCell().storeInt(1,32).endCell().beginParse().loadInt(32).copy() + 1, + stringHexToSlice("01").copy().loadInt(8), + stringHexToSlice("FF").loadInt(8), + ); +} + +@method_id(108) +fun test8(x: int32) { + var cp = x.copy(); + x.assign(10); + int32.assign(mutate x, x+cp); + (x as int18).assign(x+2); + __expect_type(x, "int32"); + return (x, cp); +} + +@method_id(109) +fun test9() { + var w1 = Wrapper.create(5); + var w2 = Wrapper.create(10); + var w3 = createWrapper(20 as int16); + var w4 = Wrapper>.create({item:beginCell().storeInt(30,32).storeInt(31,32).endCell().beginParse()}); + __expect_type(w1, "Wrapper"); + __expect_type(w2, "Wrapper"); + __expect_type(w3, "Wrapper"); + __expect_type(w4, "Wrapper>"); + __expect_type(w4.getItem(), "Wrapper"); + __expect_type(w4.getItem().getItem(), "slice"); + return ( + w1.getItem(), w2.getItem(), w3.getItem(), w4.item.item.loadInt(32), + Wrapper.getItem(w1), w1.getItem(), Wrapper>.getItem(w4).getItem().loadInt(32) + ); +} + + +fun T.isAnotherNull(self) { callOrder.push(1); return self == null; } +fun int.isAnotherNull(self) { callOrder.push(2); return false; } +fun int?.isAnotherNull(self) { callOrder.push(3); return false; } + +@method_id(110) +fun test10() { + callOrder = createEmptyTuple(); + ( + 5.isAnotherNull(), (5 as int?).isAnotherNull(), beginCell().isAnotherNull(), (beginCell() as builder?).isAnotherNull(), + null.isAnotherNull(), true.isAnotherNull(), ((1,null) as (int,slice?)).isAnotherNull() + ); + return (callOrder, null.isNull()); +} + +fun int.anyMethod(self) { callOrder.push(1); } +fun slice?.anyMethod(self) { callOrder.push(2); } +fun T.anyMethod(self) { callOrder.push(3); } +fun callAnyMethod(obj: T) { return obj.anyMethod(); } + +@method_id(111) +fun test11() { + callOrder = createEmptyTuple(); + (callAnyMethod(10), callAnyMethod(null as slice?), callAnyMethod(null), + callAnyMethod(null), callAnyMethod(null)); + return callOrder; +} + +fun int?.arbitraryMethod(self) { return self; } +fun Wrapper.arbitraryMethod(self) { return self; } +fun Wrapper.arbitraryMethod(self) { return beginCell(); } +fun Wrapper>.arbitraryMethod(self) { return beginCell(); } +fun T?.arbitraryMethod(self): never { throw 123; } + +fun test12() { + __expect_type(10.arbitraryMethod(), "int?"); + __expect_type((10 as int8?).arbitraryMethod(), "int?"); + __expect_type((10 as coins?)!.arbitraryMethod(), "int?"); + + __expect_type(null.arbitraryMethod(), "int?"); + __expect_type((10 as coins?).arbitraryMethod(), "int?"); + + __expect_type(Wrapper { item: 10 }.arbitraryMethod(), "Wrapper"); + __expect_type(Wrapper { item: 10 as int8 }.arbitraryMethod(), "Wrapper"); + + __expect_type(Wrapper { item: 10 as int8 }.arbitraryMethod(), "builder"); + __expect_type(Wrapper { item: Wrapper { item: 10 } }.arbitraryMethod(), "builder"); + + __expect_type((Wrapper { item: 10 } as Wrapper?).arbitraryMethod(), "never"); + __expect_type((null as slice?).arbitraryMethod(), "never"); +} + +type BuilderOrInt = builder|int; +fun (int|builder).isInt(self) { return self is int; } + +@method_id(113) +fun test13() { + var u1 = beginCell() as builder|int; + var u2 = 10 as int|builder; + var u3 = 20 as BuilderOrInt; + return (u1.isInt(), u2.isInt(), u3.isInt(), 10.isInt(), beginCell().isInt(), (8 as int8).isInt()); +} + +fun (int|builder|T).isBuilder(self) { return self is builder; } + +@method_id(114) +fun test14() { + var b1 = beginCell() as BuilderOrInt | slice; + var b2 = beginCell() as builder|slice|int|Point; + var i1 = 5 as int|builder|continuation|(int, int); + var c = beginCell().endCell() as cell?|builder|int; + var cn = null as cell?|int|int8|builder; + return (b1.isBuilder(), b2.isBuilder(), i1.isBuilder(), c.isBuilder(), cn.isBuilder()); +} + +@method_id(115) +fun test15() { + var p1 = Pair.create0(); + __expect_type(p1, "Pair"); + var p2 = Pair.create0(); + __expect_type(p2, "Pair"); + var p3 = Pair.createFrom(10, (20 as coins?)!); + __expect_type(p3, "Pair"); + return (p2.is0(), p2.compareWith(p3.first, p3.second)); +} + +struct CounterIncrement { inc_by: int; } +struct CounterReset { initial_value: int; } +type CounterMsg = CounterIncrement | CounterReset; + +fun CounterReset.onInternalMessage(self) { + throw 123; +} +fun CounterMsg.onInternalMessage(self) { + return match (self) { + CounterIncrement => self.inc_by, + CounterReset => self.initial_value, + }; +} + +@method_id(116) +fun test16() { + var inc: CounterIncrement = { inc_by: 100 }; + var msg = inc as CounterMsg; + return (inc.onInternalMessage(), msg.onInternalMessage(), CounterMsg.onInternalMessage(CounterReset{initial_value:5})); +} + +fun int.`0`() { return 0; } +fun int.`1`(self) { return self + 1; } + +@method_id(117) +fun test17() { + var zero = int.0(); + return zero.1() + int.0().1() + int.1(int.0()); +} + +fun Wrapper.createNull(): Wrapper { return { item: null }; } +fun Wrapper.createFrom(item: U): Wrapper { return {item}; } + +@method_id(118) +fun test18() { + __expect_type(10.copy, "(int) -> int"); + __expect_type(Wrapper{item:null as int8?}.copy, "(Wrapper) -> Wrapper"); + __expect_type(Wrapper.createNull, "() -> Wrapper"); + __expect_type(Wrapper?>.createNull, "() -> Wrapper?>"); + __expect_type(Wrapper.createFrom, "(int8) -> Wrapper"); + + var cb = Wrapper.createFrom; + return cb(10); +} + +fun main() {} + +/** +@testcase | 101 | | 2 1 6 3 +@testcase | 102 | 5 6 | 6 11 7 +@testcase | 103 | | 4 3 4 4 3 +@testcase | 104 | | 0 0 -1 -1 0 +@testcase | 105 | 5 6 | 11 16 11 +@testcase | 106 | | 5 10 24 0 5 5 5 10 +@testcase | 107 | | 2 1 -1 +@testcase | 108 | 5 | 17 5 +@testcase | 109 | | 5 100500 20 30 5 5 31 +@testcase | 110 | | [ 2 3 1 1 3 1 1 ] -1 +@testcase | 111 | | [ 1 2 3 3 2 ] +@testcase | 113 | | 0 -1 -1 -1 0 -1 +@testcase | 114 | | -1 -1 0 0 0 +@testcase | 115 | | -1 0 +@testcase | 116 | | 100 100 5 +@testcase | 117 | | 3 +@testcase | 118 | | 10 + +@fif_codegen DECLPROC Pair.createFrom +@fif_codegen DECLPROC Pair.compareWith +@fif_codegen DECLPROC CounterMsg.onInternalMessage + */ diff --git a/tolk-tester/tests/mutate-methods.tolk b/tolk-tester/tests/mutate-methods.tolk index 59da684843..18cb644045 100644 --- a/tolk-tester/tests/mutate-methods.tolk +++ b/tolk-tester/tests/mutate-methods.tolk @@ -1,8 +1,18 @@ -fun incrementInPlace(mutate self: int, byValue: int): void { +fun incrementInPlace(mutate x: int, byValue: int): void { + x = x + byValue; +} + +fun int.incrementInPlace(mutate self, byValue: int): void { self = self + byValue; } -fun incrementTwoInPlace(mutate self: int, mutate y: int, byValue: int): int { +fun incrementTwoInPlace(mutate x: int, mutate y: int, byValue: int): int { + x.incrementInPlace(byValue); + y += byValue; + return x + y; +} + +fun int.incrementTwoInPlace(mutate self, mutate y: int, byValue: int): int { self.incrementInPlace(byValue); y += byValue; return self + y; @@ -31,12 +41,12 @@ fun testIncrement2() { fun load_next(mutate cs: slice): int { - return loadInt(mutate cs, 32); + return cs.loadInt(32); } -fun myLoadInt(mutate self: slice, len: int): int +fun slice.myLoadInt(mutate self, len: int): int asm(-> 1 0) "LDIX"; -fun myStoreInt(mutate self: builder, x: int, len: int): self +fun builder.myStoreInt(mutate self, x: int, len: int): self asm(x self len) "STIX"; @inline_ref @@ -58,7 +68,7 @@ fun testSlices1() { return (first, cs.myLoadInt(32), cs.loadInt(32)); } -fun load_decimal_symbol(mutate self: slice): int { +fun slice.load_decimal_symbol(mutate self): int { // load decimal from bits using utf-8 table var n: int = self.loadUint(8); n = n - 48; @@ -113,10 +123,15 @@ fun testNameShadowing() { return (x, sum); } -fun updateTwoItems(mutate self: (int, int), byValue: int) { +type Pair2 = (int, int); +fun Pair2.updateTwoItems(mutate self, byValue: int) { val (first, second) = self; self = (first + byValue, second + byValue); } +fun updateTwoItems(mutate pair: Pair2, byValue: int) { + val (first, second) = pair; + pair = (first + byValue, second + byValue); +} global t107_1: int; global t107_2: int; @@ -134,18 +149,29 @@ fun testMutableTensor() { } @pure -fun myStoreUint(mutate self: builder, x: int, len: int): self +fun builder.myStoreUint(mutate self, x: int, len: int): self asm(x self len) "STIX"; @pure -fun myStoreU32(mutate self: builder, x: int): self { +fun myStoreUint(mutate b: builder, x: int, len: int): void + asm(x b len) "STUX"; +@pure +fun storeUint(mutate b: builder, x: int, len: int): void + asm(x b len) "STUX"; + +@pure +fun builder.myStoreU32(mutate self, x: int): self { return self.storeUint(x, 32); } +@pure +fun myStoreU32(mutate b: builder, x: int) { + b.storeUint(x, 32); +} fun getSumOfNumbersInCell(c: cell): int { var sum = 0; var s = c.beginParse(); - var n_numbers = s.getRemainingBitsCount() / 32; + var n_numbers = s.remainingBitsCount() / 32; repeat (n_numbers) { sum += s.loadUint(32); } @@ -161,7 +187,7 @@ fun testStoreChaining() { b = b.storeUint(8, 32); b = b.storeUint(9, 32).storeUint(10, 32); - return getBuilderBitsCount(b); + return b.bitsCount(); } @method_id(111) @@ -185,7 +211,7 @@ fun testStoreChainingCustom() { return (sum1, sum2); } -fun myStoreU32_and_mutate_x(mutate self: builder, mutate x: int): void { +fun builder.myStoreU32_and_mutate_x(mutate self, mutate x: int): void { return myStoreUint(mutate self, x += 10, 32); } @@ -215,7 +241,7 @@ fun testStoreChainingForGlobal() { ccc = ccc.myStoreU32(8); ccc = ccc.storeUint(9, 32).myStoreUint(10, 32); - return getBuilderBitsCount(ccc); + return ccc.bitsCount(); } fun alwaysThrows(): int { throw 123; return 123; } @@ -233,10 +259,10 @@ fun testLoadIntForTemporaryObject() { } @pure -fun myStoreUint_pure(mutate self: builder): void +fun builder.myStoreUint_pure(mutate self): void asm "STIX"; -fun myStoreUint_impure(mutate self: builder): void +fun builder.myStoreUint_impure(mutate self): void asm "STIX"; fun testStoreUintPureUnusedResult() { @@ -257,11 +283,11 @@ fun testStoreUintImpureUnusedResult() { global counter: int; -fun writeNext2(mutate self: builder): self { +fun builder.writeNext2(mutate self): self { return self.storeUint(counter += 1, 32).storeUint(counter += 1, 32); } -fun resetCounter(mutate self: builder): self { +fun builder.resetCounter(mutate self): self { counter = 0; return self; } @@ -270,7 +296,7 @@ fun resetCounter(mutate self: builder): self { fun testExplicitReturn() { counter = 0; return ( - beginCell().writeNext2().writeNext2().resetCounter().writeNext2().endCell().getSumOfNumbersInCell(), + getSumOfNumbersInCell(beginCell().writeNext2().writeNext2().resetCounter().writeNext2().endCell()), counter ); } @@ -295,7 +321,7 @@ fun main(){} @fif_codegen """ - incrementInPlace PROC:<{ // self byValue + int.incrementInPlace PROC:<{ // self byValue ADD // self }> """ @@ -307,7 +333,7 @@ fun main(){} incrementTwoInPlace CALLDICT // x y sum1 -ROT 10 PUSHINT // sum1 x y '11=10 - incrementTwoInPlace CALLDICT // sum1 x y sum2 + int.incrementTwoInPlace CALLDICT // sum1 x y sum2 s1 s3 s0 XCHG3 // x y sum1 sum2 }> """ diff --git a/tolk-tester/tests/no-spaces.tolk b/tolk-tester/tests/no-spaces.tolk index 33959917e4..50b2761d7f 100644 --- a/tolk-tester/tests/no-spaces.tolk +++ b/tolk-tester/tests/no-spaces.tolk @@ -47,7 +47,7 @@ fun add3(a: int, b: int, c: int) { return a+b+c; } return [add3(fst2,snd2,trd2),add3(fst1,snd1,trd1)]; } -fun `load:u32`(mutate self: slice): int { +fun slice.`load:u32`(mutate self): int { return self.loadUint(32) } diff --git a/tolk-tester/tests/null-keyword.tolk b/tolk-tester/tests/null-keyword.tolk index db54973e92..98b90811f2 100644 --- a/tolk-tester/tests/null-keyword.tolk +++ b/tolk-tester/tests/null-keyword.tolk @@ -14,8 +14,8 @@ fun test1() { (_, _) = (null, null); var t = createEmptyTuple(); do { - var num: int = numbers.listNext(); - t.tuplePush(num); + var (num: int, numbers redef) = listSplit(numbers!); + t.push(num); } while (numbers != null); return (h, numbers == null, t); diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index af051ce939..4811c41447 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -13,7 +13,18 @@ fun isTensorNull(t: (int, int)?) { return t == null; } -fun incrementNullableTensorComponents(mutate self: (int, int)?): self { +fun (int, int)?.isTensorNull(self) { + return self == null; +} + +fun incrementNullableTensorComponents(mutate t: (int, int)?) { + if (t != null) { + t!.0 += 1; + t!.1 += 1; + } +} + +fun (int, int)?.incrementNullableTensorComponents(mutate self): self { if (self != null) { self!.0 += 1; self!.1 += 1; @@ -21,7 +32,12 @@ fun incrementNullableTensorComponents(mutate self: (int, int)?): self { return self; } -fun incrementTensorComponents(mutate self: (int, int)): self { +fun incrementTensorComponents(mutate t: (int, int)) { + t.0 += 1; + t.1 += 1; +} + +fun (int, int).incrementTensorComponents(mutate self): self { self.0 += 1; self.1 += 1; return self; @@ -120,6 +136,10 @@ fun isTensorNullGen(t: (T1, T2)?) { return t == null; } +fun (T1, T2)?.isTensorNullGen(self) { + return self == null; +} + @method_id(109) fun test109() { var x1 = (1, 2); @@ -128,7 +148,7 @@ fun test109() { return ( isTensorNullGen(x1), isTensorNullGen(x2), isTensorNullGen(null), isTensorNullGen(x1), isTensorNullGen(x3), - x1.isTensorNullGen(), x2.isTensorNullGen(), x3.isTensorNullGen(), null.isTensorNullGen() + x1.isTensorNullGen(), x2.isTensorNullGen(), x3.isTensorNullGen(), (null as (int,int)?).isTensorNullGen() ); } diff --git a/tolk-tester/tests/nullable-types.tolk b/tolk-tester/tests/nullable-types.tolk index 74e8f1b1be..39906eae44 100644 --- a/tolk-tester/tests/nullable-types.tolk +++ b/tolk-tester/tests/nullable-types.tolk @@ -9,6 +9,13 @@ fun eq(x: T) { return x } fun unwrap(x: T?): T { return x!; } fun intOr0(x: int?): int { return null == x ? 0 : x!; } +@pure +fun storeInt(mutate b: builder, x: int, len: int): void + asm(x b len) "STIX"; +@pure +fun tuplePush(mutate t: tuple, value: T): void + asm "TPUSH"; + @method_id(101) fun test101(x: int) { var re = x == 0 ? null : 100; @@ -61,8 +68,8 @@ fun test106() { gInt106! += 5; var int106: int? = 0; var gTup106 = createEmptyTuple(); - gTup106!.tuplePush(createEmptyTuple()); - (gTup106!.0 as tuple?)!.tuplePush(0 as int?); + gTup106!.push(createEmptyTuple()); + (gTup106!.0 as tuple?)!.push(0 as int?); tuplePush(mutate gTup106!, gInt106); tuplePush(mutate gTup106!.0, int106! += 1); return (gTup106 == null, null != gTup106, gTup106, gTup106!.0 as tuple?); @@ -75,7 +82,7 @@ fun test107() { b = b!.storeInt(3, 32); storeInt(mutate b!, 4, 32); (b! as builder).storeInt(5, 32); - return b!.getBuilderBitsCount(); + return b!.bitsCount(); } @method_id(108) diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index 64ca5d67e5..912fb84c28 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -2,7 +2,7 @@ const cc1 = stringAddressToSlice("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce const cc2 = stringAddressToSlice("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); fun verifyAddr(addr: slice, workchain: int, number: int) { - assert (addr.getRemainingBitsCount() == 3 + 8 + 256) throw 112; + assert (addr.remainingBitsCount() == 3 + 8 + 256) throw 112; addr.skipBits(3); assert (addr.loadUint(8) == workchain) throw 111; assert (addr.loadUint(256) == number) throw 111; @@ -105,7 +105,7 @@ fun main() { verifyAddr(stringAddressToSlice("99:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 99, 37304138005561100291416421295333982606153966175434134130332440738068913455320); verifyAddr(stringAddressToSlice("-1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 255, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - return cc1.isSliceBitsEqual(cc2); + return cc1.bitsEqual(cc2); } /** diff --git a/tolk-tester/tests/pure-functions.tolk b/tolk-tester/tests/pure-functions.tolk index bb1a7d5935..285320dec3 100644 --- a/tolk-tester/tests/pure-functions.tolk +++ b/tolk-tester/tests/pure-functions.tolk @@ -11,7 +11,7 @@ fun f_pure2(): int { @pure fun get_contract_data(): (int, int) { - var c: cell = getContractData(); + var c: cell = contract.getData(); var cs: slice = c.beginParse(); cs.loadBits(32); var value: int = cs.loadUint(16); @@ -20,7 +20,7 @@ fun get_contract_data(): (int, int) { fun save_contract_data(value: int) { var b: builder = beginCell().storeInt(1, 32).storeUint(value, 16); - setContractData(b.endCell()); + contract.setData(b.endCell()); } @pure diff --git a/tolk-tester/tests/self-keyword.tolk b/tolk-tester/tests/self-keyword.tolk index 37f3cb1784..e504b5c484 100644 --- a/tolk-tester/tests/self-keyword.tolk +++ b/tolk-tester/tests/self-keyword.tolk @@ -1,18 +1,18 @@ -fun incChained(mutate self: int): self { +fun int.incChained(mutate self): self { self = self + 1; return self; } -fun incChained2(mutate self: int): self { +fun int.incChained2(mutate self): self { return self.incChained(); } -fun incChained3(mutate self: int): self { - incChained(mutate self); +fun int.incChained3(mutate self): self { + self.incChained(); return self; } -fun incChained4(mutate self: int): self { +fun int.incChained4(mutate self): self { self.incChained(); return self; } @@ -25,7 +25,7 @@ fun testIncChainedCodegen(x: int) { @method_id(102) fun testIncChained() { var x: int = 10; - incChained(mutate x); + x.incChained(); x.incChained(); x.incChained2(); x.incChained2().incChained(); @@ -34,7 +34,7 @@ fun testIncChained() { return x.incChained(); } -fun incChainedWithMiddleReturn(mutate self: int, maxValue: int): self { +fun int.incChainedWithMiddleReturn(mutate self, maxValue: int): self { if (self >= maxValue) { return self; } @@ -49,7 +49,7 @@ fun testIncChainedWithMiddleReturn(x: int) { return x.incChainedWithMiddleReturn(10).incChainedWithMiddleReturn(999); } -fun incChainedMutatingBoth(mutate self: int, mutate y: int): self { +fun int.incChainedMutatingBoth(mutate self, mutate y: int): self { self += 1; y += 1; return self; @@ -62,12 +62,12 @@ fun testIncChainedMutatingBoth() { var (x, y) = (0, 0); c104 = 0; x.incChainedMutatingBoth(mutate y).incChainedMutatingBoth(mutate y); - incChainedMutatingBoth(mutate x, mutate y); + x.incChainedMutatingBoth(mutate y); x = x.incChainedMutatingBoth(mutate c104).incChainedMutatingBoth(mutate c104).incChainedMutatingBoth(mutate y); return (x, y, c104); } -fun incTensorChained(mutate self: (int, int)): self { +fun (int, int).incTensorChained(mutate self): self { val (f, s) = self; self = (f + 1, s + 1); return self; @@ -80,7 +80,7 @@ fun testIncTensorChained(f: int, s: int) { return tens.incTensorChained().incTensorChained(); } -fun incConditionalChainable(mutate self: int, mutate another: int, ifLessThan: int): self { +fun int.incConditionalChainable(mutate self, mutate another: int, ifLessThan: int): self { another += 1; return self.incChained() < ifLessThan ? self.incChained().incChained() : self; } @@ -93,7 +93,7 @@ fun testIncConditionalChainable(x: int) { return (x.incConditionalChainable(mutate y, 5), y); } -fun checkNotEq(self: int, throwIfEq: int): void { +fun int.checkNotEq(self, throwIfEq: int): void { if (self == throwIfEq) { throw 100 + throwIfEq; } @@ -113,7 +113,7 @@ fun testNotMutatingSelf(arg: int) { global c108: int; -fun checkNotEqChainable(self: int, throwIfEq: int): self { +fun int.checkNotEqChainable(self, throwIfEq: int): self { c108 += 1; if (self != throwIfEq) { return self; @@ -136,7 +136,7 @@ fun testNotMutatingChainableSelf(arg: int) { global onceFailed109: int; -fun checkNotEqChainableMutateAnother(self: int, throwIfEq: int, mutate toInc: int): self { +fun int.checkNotEqChainableMutateAnother(self, throwIfEq: int, mutate toInc: int): self { if (onceFailed109) { return self; } toInc += 1; try { return self.checkNotEqChainable(throwIfEq); } @@ -158,9 +158,9 @@ fun testNotMutatingChainableSelfMutateAnother(initial: int) { return (arg, c108, c109, x); } -fun pickG110(mutate self: int, mutate pushTo: tuple): self { +fun int.pickG110(mutate self, mutate pushTo: tuple): self { self += 10; - pushTo.tuplePush(c110); + pushTo.push(c110); return self; } @@ -175,13 +175,13 @@ fun testMutateGlobalsLValue(init: int) { return (c110, tup110); } -fun myTuplePush(mutate self: tuple, value: T): self { - self.tuplePush(value); +fun tuple.myTuplePush(mutate self, value: T): self { + self.push(value); return self; } -fun myTupleAt(self: tuple, idx: int): T { - return self.tupleAt(idx); +fun tuple.myTupleAt(self, idx: int): T { + return self.get(idx); } global tup111: tuple; @@ -192,7 +192,7 @@ fun testForallFunctionsWithSelf(): (int, int, tuple) { tup111 = createEmptyTuple(); t.myTuplePush(10); tup111.myTuplePush(1).myTuplePush(2).myTuplePush(3); - return (t.myTupleAt(0), tup111.myTupleAt(tup111.tupleSize() - 1), tup111); + return (t.myTupleAt(0), tup111.myTupleAt(tup111.size() - 1), tup111); } @@ -222,27 +222,27 @@ fun main() { } @fif_codegen """ - incChained PROC:<{ // self - INC // self + int.incChained PROC:<{ // self + INC // self }> - incChained2 PROC:<{ // self - incChained CALLDICT // self + int.incChained2 PROC:<{ // self + int.incChained CALLDICT // self }> - incChained3 PROC:<{ // self - incChained CALLDICT // self + int.incChained3 PROC:<{ // self + int.incChained CALLDICT // self }> - incChained4 PROC:<{ // self - incChained CALLDICT // self + int.incChained4 PROC:<{ // self + int.incChained CALLDICT // self }> """ @fif_codegen """ testIncChainedCodegen PROC:<{ // x - incChained CALLDICT // x - incChained2 CALLDICT // x - incChained3 CALLDICT // x - incChained4 CALLDICT // x + int.incChained CALLDICT // x + int.incChained2 CALLDICT // x + int.incChained3 CALLDICT // x + int.incChained4 CALLDICT // x }> """ */ diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 24baa34f1a..106d93840b 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -12,13 +12,19 @@ struct PointOfNulls { sub: Point?; } +fun rand(): int + asm "RANDU256"; + fun getNullableInt(): int? { return 5; } fun getNullableSlice(): slice? { return null; } fun takeNullableInt(a: int?) {} fun takeNullableSlice(a: slice?) {} -fun increment(mutate self: int) { self += 1; } -fun assignToInt(mutate self: int, value: int) { self = value; } -fun assignToNullableInt(mutate self: int?, value: int) { self = value; } +fun increment(mutate x: int) { x += 1; } +fun int.increment(mutate self) { self += 1; } +fun assignToInt(mutate x: int, value: int) { x = value; } +fun int.assignToInt(mutate self, value: int) { self = value; } +fun assignToNullableInt(mutate x: int?, value: int) { x = value; } +fun int?.assignToNullableInt(mutate self, value: int) { self = value; } fun sameTensor(t: (int, int)) { return t; } fun sameTensor2(t: (int?, Pair4N)) { return t; } fun eq(v: T) { return v; } @@ -58,7 +64,7 @@ fun test3(): int { } return x; } - if (random() > -1) { + if (rand() > -1) { if (y == null) { return -1 } else { return y } } @@ -88,7 +94,7 @@ fun test5() { var (a, (b, c)) = (getNullableInt(), (getNullableInt(), getNullableInt())); if (a == null) { return -1; } if (!(b != null)) { return -2; } - if (random() ? c == null && c == null : c == null) { return -3; } + if (rand() ? c == null && c == null : c == null) { return -3; } return a + b + c; } @@ -99,10 +105,10 @@ fun test6() { __expect_type(a == null ? "" : a, "int"); takeNullableInt(a); __expect_type(a, "int"); - if (random()) { + if (rand()) { a = null; } else { - if (random()) { a = null; } + if (rand()) { a = null; } else { a = null; } } __expect_type(a, "null"); @@ -121,7 +127,7 @@ fun test7() { if (a == null && true) { return -1; } if (true && true && 1 && !0 && b == null) { return -2; } if (true ? c == null && (((c))) == null && true : false) { return -3; } - if (!true ? random() > 0 : a != null && (d == null && b != null)) { return -4; } + if (!true ? rand() > 0 : a != null && (d == null && b != null)) { return -4; } return a + b + c + d; } @@ -129,7 +135,7 @@ fun test8(x: int?, y: int?) { var allGt1 = x != null && x > 1 && y != null && y > 1; var xGtY = x != null && y != null && x > y; var xLtEq0 = x == null || x < 0; - (x = 0) < random() || x > 10; + (x = 0) < rand() || x > 10; return x + 0; } @@ -170,7 +176,7 @@ fun test11() { fun test12() { var (x, y) = (getNullableInt(), getNullableInt()); - if (random() ? x == null || y == null : x == null || y == null) { return -1; } + if (rand() ? x == null || y == null : x == null || y == null) { return -1; } __expect_type(x, "int"); __expect_type(y, "int"); return x + y; @@ -192,7 +198,7 @@ fun test14() { x = 0; } if (y == null) { - if (random()) { return 0; } + if (rand()) { return 0; } else { y = 0; } } return x + y; @@ -262,45 +268,45 @@ fun test24(x: int?) { fun test25() { var x = (getNullableInt(), getNullableInt(), getNullableInt()); - x.0 = x.2 = random(); + x.0 = x.2 = rand(); return (x.0) + ((x.2)); } fun test26() { var x = [getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()]; - if (~(x.0 = random())) { return; } - if ((x.1 = random()) < (x.2 = random())) { return; } - else if (!(x.2 <=> (x.3 = random()))) { return; } - x.5 = (x.4 = random()) ? (x.6 = random()) : (x.6 = random()); - if ((x.7 = random()) as int) { return; } - if (((((x.8 = random()) != null)))) { return; } - if ([x.1, (x.9 = random())!].1) { return; } + if (~(x.0 = rand())) { return; } + if ((x.1 = rand()) < (x.2 = rand())) { return; } + else if (!(x.2 <=> (x.3 = rand()))) { return; } + x.5 = (x.4 = rand()) ? (x.6 = rand()) : (x.6 = rand()); + if ((x.7 = rand()) as int) { return; } + if (((((x.8 = rand()) != null)))) { return; } + if ([x.1, (x.9 = rand())!].1) { return; } val result = x.0+x.1+x.2+x.3+x.4+x.5+x.6+x.7+x.8+x.9; } fun test27() { var (x, _) = ([getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()], []); - +(x.0 = random()); - x.0 += [((x.1 = random()) < (x.2 = random() + x.1)) as int].0; - !(x.2 <=> (x.3 = random() + x.2)); - x.5 = (x.4 = random()) ? (x.6 = random()) : (x.6 = random()); - (x.7 = random()) as int; - (((((x.8 = random()) != null)))); - [x.1, (x.9 = random())!].1; + +(x.0 = rand()); + x.0 += [((x.1 = rand()) < (x.2 = rand() + x.1)) as int].0; + !(x.2 <=> (x.3 = rand() + x.2)); + x.5 = (x.4 = rand()) ? (x.6 = rand()) : (x.6 = rand()); + (x.7 = rand()) as int; + (((((x.8 = rand()) != null)))); + [x.1, (x.9 = rand())!].1; return x.0+x.1+x.2+x.3+x.4+x.5+x.6+x.7+x.8+x.9; } fun test28() { var x = (getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()); - __expect_type((x.0 = random(), x.0 += (x.1 = random()) as int, !(x.1 <=> (x.2 = random() + x.0)) == null, (x.3 = random()) ? x.3 : (!x.3) as int), + __expect_type((x.0 = rand(), x.0 += (x.1 = rand()) as int, !(x.1 <=> (x.2 = rand() + x.0)) == null, (x.3 = rand()) ? x.3 : (!x.3) as int), "(int, int, bool, int)"); } fun test29() { var x = (getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()); - __expect_type([x.0 = random(), ((x.0 += (x.1 = random()) as int)), !(x.1 <=> (x.2 = random() + x.0)) == null, (x.3 = random()) ? x.3 : (!x.3) as int], + __expect_type([x.0 = rand(), ((x.0 += (x.1 = rand()) as int)), !(x.1 <=> (x.2 = rand() + x.0)) == null, (x.3 = rand()) ? x.3 : (!x.3) as int], "[int, int, bool, int]"); } @@ -357,7 +363,7 @@ fun test33(): int { fun test34() { var (x, y) = (getNullableInt(), getNullableInt()); - if (random()) { throw (x = 1, y = 2) } + if (rand()) { throw (x = 1, y = 2) } else { throw (x = 3, y = (1, getNullableInt()!).1) } return x + y; } @@ -411,8 +417,8 @@ fun test38() { assignNull2(mutate t.0, mutate t.1); __expect_type(t.0, "int?"); __expect_type(t.1, "slice?"); - t.0 != null && t.1 != null ? t.0 + loadInt(mutate t.1, 32) : -1; - return t.0 != null && t.1 != null ? t.0 + loadInt(mutate t.1, 32) : -1; + t.0 != null && t.1 != null ? t.0 + t.1.loadInt(32) : -1; + return t.0 != null && t.1 != null ? t.0 + t.1.loadInt(32) : -1; } @method_id(139) @@ -585,7 +591,7 @@ fun test54() { x1 = getNullableInt(); __expect_type(x1, "int?"); assignToNullableInt(mutate x2, 5); - if (random()) { x3.assignToNullableInt(5); } + if (rand()) { x3.assignToNullableInt(5); } x11 = 10; assignToInt(mutate x12, 5); } while (x1 != null); @@ -606,7 +612,7 @@ fun test55() { // (checked via codegen) eq55(x); __expect_type(x, "int?"); // types are checked (unlike generics instantiated) after inferring - x = random() ? 1 : null; + x = rand() ? 1 : null; } __expect_type(x, "int?"); } @@ -694,7 +700,7 @@ fun test62() { __expect_type(p.sub, "Point"); __expect_type(p!!.sub!!, "Point"); p.sub.x + p.x! + p.y!; - return random() ? null : p; + return rand() ? null : p; } fun test63() { diff --git a/tolk-tester/tests/some-tests-2.tolk b/tolk-tester/tests/some-tests-2.tolk index 4e46f26335..b3b756b199 100644 --- a/tolk-tester/tests/some-tests-2.tolk +++ b/tolk-tester/tests/some-tests-2.tolk @@ -1,6 +1,6 @@ import "@stdlib/tvm-lowlevel" -fun pair_first(p: [X, Y]): X asm "FIRST"; +fun [X, Y].pair_first(self): X asm "FIRST"; fun one(dummy: tuple?) { return 1; @@ -37,11 +37,11 @@ fun test88(x: int) { @method_id(89) fun test89(last: int): (int, int, int, int) { var t: tuple = createEmptyTuple(); - t.tuplePush(1); - t.tuplePush(2); - t.tuplePush(3); - t.tuplePush(last); - return (t.tupleAt(0), t.tupleAt(t.tupleSize() - 1), t.tupleFirst(), t.tupleLast()); + t.push(1); + t.push(2); + t.push(3); + t.push(last); + return (t.get(0), t.get(t.size() - 1), t.first(), t.last()); } @pure fun get10() { return 10; } @@ -56,25 +56,25 @@ fun touchCodegen2() { @method_id(92) fun testDumpDontPolluteStack() { var f = get10(); - f.debugPrint(); - debugPrint(10); + debug.print(f); + debug.print(10); var s = "asdf"; - s.debugPrintString(); - debugDumpStack(); - debugPrintString("my"); - return (f, getRemainingBitsCount(s)); + debug.printString(s); + debug.dumpStack(); + debug.printString("my"); + return (f, s.remainingBitsCount()); } @method_id(93) fun testStartBalanceCodegen1() { - var t = getMyOriginalBalanceWithExtraCurrencies(); + var t = contract.getOriginalBalanceWithExtraCurrencies(); var first = t.pair_first(); return first; } @method_id(94) fun testStartBalanceCodegen2() { - var first = getMyOriginalBalance(); + var first = contract.getOriginalBalance(); return first; } diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index af8c6edeb0..1316595714 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -33,7 +33,7 @@ get fun string_crc16(): int { @pure fun newc(): builder asm "NEWC"; -fun endcs(b: builder): slice +fun builder.endcs(self): slice asm "ENDC" "CTOS"; @pure fun sdeq(s1: slice, s2: slice): int @@ -42,8 +42,8 @@ asm "SDEQ"; fun calcSliceBytes(slice: slice): tuple { var t = createEmptyTuple(); var n = 0; - while (n = slice.getRemainingBitsCount()) { - t.tuplePush(slice.loadUint(min(8, n))); + while (n = slice.remainingBitsCount()) { + t.push(slice.loadUint(min(8, n))); } return t; } diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 81f3b9f298..c7c8205e41 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -41,6 +41,10 @@ fun sumCoords(p: Point) { return p.x + p.y; } +fun Point.sumCoords(self) { + return self.x + self.y; +} + fun getStorage1(x: int): Storage { val owner = User { id: 1, name: "u" }; return { @@ -53,7 +57,12 @@ fun generatePoint(x: int, y: int): Point { return { x, y }; } -fun assignCoords(mutate self: Point) { +fun assignCoords(mutate p: Point) { + p.x = 10; + p.y = 20; +} + +fun Point.assignCoords(mutate self) { self.x = 10; self.y = 20; } @@ -149,6 +158,10 @@ fun maxCoord(p: Point): int8 { return p.x > p.y ? p.x : p.y; } +fun Point.maxCoord(self): int8 { + return self.x > self.y ? self.x : self.y; +} + @method_id(109) fun test9() { var p = Point { x: 80, y: 100 }; @@ -222,7 +235,7 @@ fun test15() { } fun pushZero(mutate t: tuple) { - t.tuplePush(0); + t.push(0); } @method_id(116) @@ -237,7 +250,7 @@ fun test16(): WithTensorInside? { wt.coords.1 -= 2; wt.tup.0 += 5; wt.otherCoords!.0 = 7; - wt.otherTup.tuplePush(0); + wt.otherTup.push(0); pushZero(mutate wt.otherTup); (wt.otherTup.1, wt.otherTup.0) = (11, 10); return wt; @@ -300,7 +313,7 @@ fun asm_func(x1: JustInt, x2: int, x3: JustIntAlias, x4: JustIntAlias, x5: int, (int, JustInt, int, JustInt, int, int, JustIntAlias2) asm (x4 x5 x6 x7 x1 x2 x3->0 1 2 3 4 5 6) "NOP"; -fun someOp(mutate self: JustInt, y: int): int { +fun JustInt.someOp(mutate self, y: int): int { val (newX, newY) = (self.value + y, y * 10); self = { value: newX }; return newY; @@ -453,9 +466,9 @@ struct WithDefaults { @method_id(133) fun test33(): WithDefaults { var w1: WithDefaults = { f2: 0 }; - assert(w1.f1.0 && w1.f1.1 == 0 && w1.f3!.getRemainingBitsCount() == 24 && w1.f4 == null && w1.f5 is int32) throw 100; + assert(w1.f1.0 && w1.f1.1 == 0 && w1.f3!.remainingBitsCount() == 24 && w1.f4 == null && w1.f5 is int32) throw 100; var w2: WithDefaults? = { f1: (false, 55), f2: 10, f5: 8 as int64 }; - assert(w2.f1.0 != true && w2.f3!.getRemainingBitsCount() == 24 && w2.f4 == null && w2.f5 is int64 && w2.f5 == 8) throw 100; + assert(w2.f1.0 != true && w2.f3!.remainingBitsCount() == 24 && w2.f4 == null && w2.f5 is int64 && w2.f5 == 8) throw 100; var w3: (int, WithDefaults) = (0, { f2: 7, f4: {y: 20} }); assert(w3.1.f4 != null && w3.1.f4.x == 0 && w3.1.f4.y == 20) throw 100; return { f2: 5, f3: null }; diff --git a/tolk-tester/tests/test-math.tolk b/tolk-tester/tests/test-math.tolk index 6b147c776c..55c518800b 100644 --- a/tolk-tester/tests/test-math.tolk +++ b/tolk-tester/tests/test-math.tolk @@ -921,7 +921,7 @@ fun nrand_f252(): int { var (x, s, t, A, B, r0) = (nan(), 29483 << 236, -3167 << 239, 12845, 16693, 9043); // 4/sqrt(e*Pi) = 1.369 loop iterations on average do { - var (u, v) = (random() / 16 + 1, mulDivRound(random() - (1 << 255), 7027, 1 << 16)); // fixed252; 7027=ceil(sqrt(8/e)*2^12) + var (u, v) = (random.uint256() / 16 + 1, mulDivRound(random.uint256() - (1 << 255), 7027, 1 << 16)); // fixed252; 7027=ceil(sqrt(8/e)*2^12) var va: int = abs(v); var (u1, v1) = (u - s, va - t); // (u - 29483/2^16, abs(v) + 3167/2^13) as fixed252 // Q := u1^2 + v1 * (A*v1 - B*u1) as fixed252 where A=12845/2^16, B=16693/2^16 @@ -952,7 +952,7 @@ fun nrand_f252(): int { fun nrand_fast_f252(): int { var t: int = -3 << 253; // -6. as fixed252 repeat (12) { - t += random() / 16; // add together 12 uniformly random numbers + t += random.uint256() / 16; // add together 12 uniformly random numbers } return t; } @@ -961,7 +961,7 @@ fun nrand_fast_f252(): int { /// fixed248 random(); @inline fun fixed248_random(): int { - return random() >> 8; + return random.uint256() >> 8; } /// random number with standard normal distribution @@ -979,7 +979,7 @@ fun fixed248_nrand_fast(): int { } @pure -fun tset(mutate self: tuple, idx: int, value: X): void +fun tuple.tset(mutate self, idx: int, value: X): void asm(self value idx) "SETINDEXVAR"; // computes 1-acos(x)/Pi by a very simple, extremely slow (~70k gas) and imprecise method @@ -1014,12 +1014,12 @@ fun asin_slow_f255(x: int): int { fun test_nrand(n: int): tuple { var t: tuple = createEmptyTuple(); repeat (255) { - t.tuplePush(0); + t.push(0); } repeat (n) { var x: int = fixed248_nrand(); var bucket: int = (abs(x) >> 243); // 255 buckets starting from x=0, each 1/32 wide - var at_bucket: int = t.tupleAt(bucket); + var at_bucket: int = t.get(bucket); t.tset(bucket, at_bucket + 1); } return t; diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk index d8ab0782cb..ed0cca11d2 100644 --- a/tolk-tester/tests/type-aliases-tests.tolk +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -11,6 +11,9 @@ struct Point { x: int; y: int } type PointAlias = Point; type PointAlias2 = PointAlias; +fun rand(): uint256 + asm "RANDU256"; + fun test1(x: MInt): MVoid { var y = x; var z: MInt = 2; @@ -24,14 +27,14 @@ fun test1(x: MInt): MVoid { __expect_type(~x, "int"); __expect_type(x as int, "int"); __expect_type(x as int8, "int8"); - __expect_type(random() ? x : y, "MInt"); + __expect_type(rand() ? x : y, "MInt"); __expect_type((x, 1, y), "(MInt, int, MInt)"); - __expect_type(random() ? (1, 2) : (1, 2) as Pair2_v1, "(int, int)"); - __expect_type(random() ? (1, 2) : (1, 2) as Pair2_v2, "(int, int)"); - __expect_type(random() ? (1, 2) as Pair2_v1 : (1, 2), "Pair2_v1"); - __expect_type(random() ? (1, 2) as Pair2_v1 : (1, 2) as Pair2_v2, "(int, int)"); - __expect_type(random() ? (1, 2) as Pair2_v2 : (1, 2) as Pair2_v2, "Pair2_v2"); + __expect_type(rand() ? (1, 2) : (1, 2) as Pair2_v1, "(int, int)"); + __expect_type(rand() ? (1, 2) : (1, 2) as Pair2_v2, "(int, int)"); + __expect_type(rand() ? (1, 2) as Pair2_v1 : (1, 2), "Pair2_v1"); + __expect_type(rand() ? (1, 2) as Pair2_v1 : (1, 2) as Pair2_v2, "(int, int)"); + __expect_type(rand() ? (1, 2) as Pair2_v2 : (1, 2) as Pair2_v2, "Pair2_v2"); __expect_type(!x, "bool"); @@ -73,10 +76,10 @@ fun test4(x: MIntN, y: MIntN) { } __expect_type(x, "MInt?"); __expect_type(x!, "MInt"); - __expect_type(random() ? x : y, "MInt?"); - __expect_type(random() ? x : 0, "MInt?"); - __expect_type(random() ? 0 : x, "int?"); - __expect_type(random() ? 0 : x!, "int"); + __expect_type(rand() ? x : y, "MInt?"); + __expect_type(rand() ? x : 0, "MInt?"); + __expect_type(rand() ? 0 : x, "int?"); + __expect_type(rand() ? 0 : x!, "int"); return y!; } diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 11dae3f8cd..692c06e1cd 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -596,7 +596,7 @@ fun test50() { if (match(b) { int => 1, null => null } !is int) { return 100 } - return (b! + (match (a) { int => a, slice => a, null => a } as int|slice?).test49()); + return (b! + test49(match (a) { int => a, slice => a, null => a } as int|slice?)); } @method_id(151) @@ -671,7 +671,7 @@ fun test54(): int|slice { __expect_type(a, "int"); a = beginCell().storeInt(100, 32).endCell().beginParse(); __expect_type(a, "slice"); - if (a.getRemainingBitsCount() < 100) { a = a.loadInt(32); } + if (a.remainingBitsCount() < 100) { a = a.loadInt(32); } else { a = 0; } __expect_type(a, "int"); return a; diff --git a/tolk-tester/tests/use-before-declare.tolk b/tolk-tester/tests/use-before-declare.tolk index baae0b400d..42afe8b00a 100644 --- a/tolk-tester/tests/use-before-declare.tolk +++ b/tolk-tester/tests/use-before-declare.tolk @@ -9,7 +9,7 @@ fun main(): int { fun my_begin_cell(): builder asm "NEWC"; @pure -fun my_end_cell(b: builder): cell +fun builder.my_end_cell(self): cell asm "ENDC"; @pure fun my_begin_parse(c: cell): slice diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index 9d9a0a0b87..1c0f164cba 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -3,6 +3,14 @@ type MIntN = int?; type MBuilder = builder; type CallbackIntToInt = int -> int; +@pure +fun beginParse(c: cell): slice + asm "CTOS"; + +@pure +fun endCell(b: builder): cell + asm "ENDC"; + fun getBeginCell() { return beginCell } @@ -71,24 +79,24 @@ fun demo_handler(op: int, query_id: int, a: int, b: int): int { fun testVarApplyInTernary() { var t: tuple = createEmptyTuple(); try { - t.tuplePush(demo_handler(0xF2, 122, 100, 200)); + t.push(demo_handler(0xF2, 122, 100, 200)); } catch(code) { - t.tuplePush(code); + t.push(code); } try { - t.tuplePush(demo_handler(0xF4, 122, 100, 200)) + t.push(demo_handler(0xF4, 122, 100, 200)) } catch(code) { - t.tuplePush(code) + t.push(code) } try { - t.tuplePush(demo_handler(0xF2, 122, 10, 10)) + t.push(demo_handler(0xF2, 122, 10, 10)) } catch(code) { - t.tuplePush(code) + t.push(code) } try { - t.tuplePush(demo_handler(0xF2, 123, 10, 10)) + t.push(demo_handler(0xF2, 123, 10, 10)) } catch(code) { - t.tuplePush(code) + t.push(code) } return t; } @@ -114,22 +122,22 @@ fun testGlobalVarApply() { fun testVarApply2() { var creator = createEmptyTuple; var t = creator(); - t.tuplePush(1); - var sizer = t.tupleSize; + t.push(1); + var sizer = t.size; return sizer(t); } fun getTupleLastGetter(): (tuple) -> X { - return tupleLast; + return X.last; } @method_id(106) fun testVarApply3() { var t = createEmptyTuple(); - t.tuplePush(1); - t.tuplePush([2]); - var getIntAt = t.tupleAt; - var getTupleFirstInt = createEmptyTuple().tupleFirst; + t.push(1); + t.push([2]); + var getIntAt = t.get; + var getTupleFirstInt = createEmptyTuple().first; var getTupleLastTuple = getTupleLastGetter(); return (getIntAt(t, 0), getTupleFirstInt(t), getTupleLastTuple(t), getTupleLastGetter()(t)); } @@ -227,7 +235,7 @@ struct WithCallbacks { beginCell: builder -> cell, } -fun doEndCellActually(w: WithCallbacks, b: builder) { return w.beginCell(b); } +fun WithCallbacks.doEndCellActually(self, b: builder) { return self.beginCell(b); } @method_id(115) fun testCallableFields() { @@ -258,6 +266,54 @@ fun testCallableGenericField() { return (w1.value(5), w2.value!(9), wrap(w_f(justAdd2)).value.value(78)); } +struct Point { + x: int; + y: int; +} + +fun Point.createXY(x: int, y: int): Point { return {x,y}; } +fun Point.createFrom(x: U, y: V): Point { return {x,y}; } + +fun Point.maxCoord(self) { return self.x > self.y ? self.x : self.y; } + +@method_id(117) +fun testStaticAndInstanceMethods() { + var fPointCreateXY = Point.createXY; + var p1 = fPointCreateXY(5,6); + var fPointCreateFrom = Point.createFrom; + var p2 = fPointCreateFrom(7,8); + __expect_type(p1, "Point"); + __expect_type(p2, "Point"); + var getM1 = Point.maxCoord; + var getM2 = p1.maxCoord; + return (getM1(p1), (getM2 = p2.maxCoord)(p2)); +} + +fun Wrapper.createFromNull(): Wrapper { + return {value: null}; +} +fun Wrapper.createFrom(value: U): Wrapper { + return {value: value as T}; +} +fun Wrapper.equalsNotNull(self, anotherW: Wrapper) { + return self.value != null && anotherW.value != null && self.value == anotherW.value; +} + +@method_id(118) +fun testMethodsOfGenericStruct() { + var creator1 = Wrapper.createFrom; + var w1 = creator1(10); + __expect_type(w1, "Wrapper"); + var creator2 = Wrapper.createFrom; + var w2 = creator2(null as int16?); + __expect_type(w2, "Wrapper"); + var eq = Wrapper.equalsNotNull; + __expect_type(eq, "(Wrapper, Wrapper) -> bool"); + var creator3 = Wrapper.createFromNull; + __expect_type(creator3, "() -> Wrapper"); + return (eq(w1, w2), w2.value = w1.value as int, eq(w1, w2), creator3()); +} + fun main() {} @@ -285,4 +341,6 @@ fun main() {} @testcase | 114 | 0 | (null) 12 1 @testcase | 115 | | 7 6 7 @testcase | 116 | | 7 11 80 +@testcase | 117 | | 6 8 +@testcase | 118 | | 0 10 -1 (null) */ diff --git a/tolk-tester/tests/warnings-not-errors/warnings-1.tolk b/tolk-tester/tests/warnings-not-errors/warnings-1.tolk index 5dc3acdc3e..ec77a2a66e 100644 --- a/tolk-tester/tests/warnings-not-errors/warnings-1.tolk +++ b/tolk-tester/tests/warnings-not-errors/warnings-1.tolk @@ -6,7 +6,7 @@ fun main() { if (c == null) {} var d: int? = c; - if (((d)) != null && tupleSize(createEmptyTuple())) {} + if (((d)) != null && createEmptyTuple().size()) {} var e: int? = getNullableInt(); if (e != null) { diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 74ecbd2827..73791e3dd5 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -291,7 +291,7 @@ static V parse_genericsT_list(Lexer& lex) { return createV{loc, std::move(genericsT_items)}; } -static AnyV parse_parameter(Lexer& lex, bool is_first) { +static AnyV parse_parameter(Lexer& lex, AnyTypeV self_type) { SrcLocation loc = lex.cur_location(); // optional keyword `mutate` meaning that a function will mutate a passed argument (like passed by reference) @@ -303,21 +303,28 @@ static AnyV parse_parameter(Lexer& lex, bool is_first) { // parameter name (or underscore for an unnamed parameter) std::string_view param_name; + bool is_self = false; if (lex.tok() == tok_identifier) { param_name = lex.cur_str(); } else if (lex.tok() == tok_self) { - if (!is_first) { - lex.error("`self` can only be the first parameter"); + if (!self_type) { + lex.error("`self` can only be the first parameter of a method"); } + is_self = true; param_name = "self"; } else if (lex.tok() != tok_underscore) { lex.unexpected("parameter name"); } lex.next(); - // parameter type after colon are mandatory - lex.expect(tok_colon, "`: `"); - AnyTypeV param_type = parse_type_from_tokens(lex); + // parameter type after colon is mandatory + AnyTypeV param_type = self_type; + if (!is_self) { + lex.expect(tok_colon, "`: `"); + param_type = parse_type_from_tokens(lex); + } else if (lex.tok() == tok_colon) { + lex.error("`self` parameter should not have a type"); + } return createV(loc, param_name, param_type, declared_as_mutate); } @@ -459,18 +466,19 @@ static AnyExprV parse_local_vars_declaration_assignment(Lexer& lex) { } // "parameters" are at function declaration: `fun f(param1: int, mutate param2: slice)` -static V parse_parameter_list(Lexer& lex) { +// for methods like `fun builder.storeUint(self, i: int)`, receiver_type = builder (type of self) +static V parse_parameter_list(Lexer& lex, AnyTypeV receiver_type) { SrcLocation loc = lex.cur_location(); std::vector params; lex.expect(tok_oppar, "parameter list"); if (lex.tok() != tok_clpar) { - params.push_back(parse_parameter(lex, true)); + params.push_back(parse_parameter(lex, receiver_type)); while (lex.tok() == tok_comma) { lex.next(); if (lex.tok() == tok_clpar) { // trailing comma break; } - params.push_back(parse_parameter(lex, false)); + params.push_back(parse_parameter(lex, nullptr)); } } lex.expect(tok_clpar, "`)`"); @@ -1273,12 +1281,14 @@ static V parse_annotation(Lexer& lex) { case AnnotationKind::inline_simple: case AnnotationKind::inline_ref: case AnnotationKind::pure: - case AnnotationKind::deprecated: if (v_arg) { throw ParseError(v_arg->loc, "arguments aren't allowed for " + static_cast(name)); } v_arg = createV(loc, {}); break; + case AnnotationKind::deprecated: + // allowed with and without arguments; it's IDE-only, the compiler doesn't analyze @deprecated + break; case AnnotationKind::method_id: if (!v_arg || v_arg->size() != 1 || v_arg->get_item(0)->kind != ast_int_const) { throw ParseError(loc, "expecting `(number)` after " + static_cast(name)); @@ -1291,21 +1301,31 @@ static V parse_annotation(Lexer& lex) { static AnyV parse_function_declaration(Lexer& lex, const std::vector>& annotations) { SrcLocation loc = lex.cur_location(); - bool is_get_method = lex.tok() == tok_get; + bool is_contract_getter = lex.cur_str() == "get"; lex.next(); - if (is_get_method && lex.tok() == tok_fun) { + if (is_contract_getter && lex.tok() == tok_fun) { lex.next(); // 'get f()' and 'get fun f()' both correct } + AnyTypeV receiver_type = nullptr; + auto backup = lex.save_parsing_position(); + try { + receiver_type = parse_type_expression(lex); + lex.expect(tok_dot, ""); + } catch (const ParseError&) { + receiver_type = nullptr; + lex.restore_position(backup); + } + lex.check(tok_identifier, "function name identifier"); std::string_view f_name = lex.cur_str(); - bool is_entrypoint = + bool is_entrypoint = !receiver_type && ( f_name == "main" || f_name == "onInternalMessage" || f_name == "onExternalMessage" || - f_name == "onRunTickTock" || f_name == "onSplitPrepare" || f_name == "onSplitInstall"; - bool is_FunC_entrypoint = + f_name == "onRunTickTock" || f_name == "onSplitPrepare" || f_name == "onSplitInstall"); + bool is_FunC_entrypoint = !receiver_type && ( f_name == "recv_internal" || f_name == "recv_external" || - f_name == "run_ticktock" || f_name == "split_prepare" || f_name == "split_install"; + f_name == "run_ticktock" || f_name == "split_prepare" || f_name == "split_install"); if (is_FunC_entrypoint) { lex.error("this is a reserved FunC/Fift identifier; you need `onInternalMessage`"); } @@ -1318,7 +1338,7 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector v_param_list = parse_parameter_list(lex)->as(); + V v_param_list = parse_parameter_list(lex, receiver_type)->as(); bool accepts_self = !v_param_list->empty() && v_param_list->get_param(0)->param_name == "self"; int n_mutate_params = v_param_list->get_mutate_params_count(); @@ -1338,11 +1358,11 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vectorkind) { case AnnotationKind::inline_simple: @@ -1389,14 +1409,14 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vectorerror("@method_id can be specified only for regular functions"); } auto v_int = v_annotation->get_arg()->get_item(0)->as(); if (v_int->intval.is_null() || !v_int->intval->signed_fits_bits(32)) { v_int->error("invalid integer constant"); } - method_id = v_int->intval; + tvm_method_id = v_int->intval; break; } case AnnotationKind::deprecated: @@ -1408,7 +1428,7 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(loc, v_ident, v_param_list, v_body, ret_type, genericsT_list, std::move(method_id), flags); + return createV(loc, v_ident, v_param_list, v_body, receiver_type, ret_type, genericsT_list, std::move(tvm_method_id), flags); } static AnyV parse_struct_field(Lexer& lex) { @@ -1541,7 +1561,6 @@ AnyV parse_src_file_to_ast(const SrcFile* file) { annotations.clear(); break; case tok_fun: - case tok_get: toplevel_declarations.push_back(parse_function_declaration(lex, annotations)); annotations.clear(); break; @@ -1556,8 +1575,15 @@ AnyV parse_src_file_to_ast(const SrcFile* file) { case tok_infix: lex.error("`" + static_cast(lex.cur_str()) +"` is not supported yet"); + case tok_identifier: + if (lex.cur_str() == "get") { // tok-level "get", contract getter + toplevel_declarations.push_back(parse_function_declaration(lex, annotations)); + annotations.clear(); + break; + } + // fallthrough default: - lex.unexpected("fun or get"); + lex.unexpected("top-level declaration"); } } diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index b056db314c..b3e1702eb6 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -325,15 +325,16 @@ class ASTReplicator final { public: // the cloned function becomes a deep copy, all AST nodes are copied, no previous pointers left - static V clone_function_ast(V v_orig, V new_name_ident) { + static V clone_function_ast(V v_orig) { return createV( v_orig->loc, - new_name_ident, + clone(v_orig->get_identifier()), clone(v_orig->get_param_list()), clone(v_orig->get_body()), + clone(v_orig->receiver_type_node), clone(v_orig->return_type_node), - clone(v_orig->genericsT_list), - v_orig->method_id, + v_orig->genericsT_list ? clone(v_orig->genericsT_list) : nullptr, + v_orig->tvm_method_id, v_orig->flags ); } diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index 4e6f795e93..8ced76a0ba 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -215,7 +215,12 @@ class ASTStringifier final : public ASTVisitor { param_names += ","; param_names += v->as()->get_param(i)->param_name; } - return "fun " + static_cast(v->as()->get_identifier()->name) + "(" + param_names + ")"; + std::string decl = "fun "; + if (auto receiver_node = v->as()->receiver_type_node) { + decl += specific_str(receiver_node); + decl += "."; + } + return decl + static_cast(v->as()->get_identifier()->name) + "(" + param_names + ")"; } case ast_local_var_lhs: { std::string str_type = v->as()->inferred_type ? v->as()->inferred_type->as_human_readable() : ast_type_node_to_string(v->as()->type_node); diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 24c797bd2e..6e725b7816 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -134,8 +134,9 @@ void Vertex::assign_literal_value(std::string&& literal_value) this->literal_value = std::move(literal_value); } -void Vertex::assign_fun_ref(FunctionPtr fun_ref) { +void Vertex::assign_fun_ref(FunctionPtr fun_ref, bool dot_obj_is_self) { this->fun_maybe = fun_ref; + this->dot_obj_is_self = dot_obj_is_self; } void Vertex::assign_is_negated(bool is_negated) { diff --git a/tolk/ast.h b/tolk/ast.h index 927be5b71f..013ac2a788 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -684,7 +684,7 @@ struct Vertex final : ASTExprVararg { template<> // ast_dot_access is "object before dot, identifier + optional after dot" -// examples: `tensorVar.0` / `obj.field` / `getObj().method` / `t.tupleFirst` +// examples: `tensorVar.0` / `obj.field` / `getObj().method` / `t.tupleFirst` / `Point.create` // from traversing point of view, it's an unary expression: only obj is expression, field name is not // note, that `obj.method()` is a function call with "dot access `obj.method`" callee struct Vertex final : ASTExprUnary { @@ -725,19 +725,20 @@ template<> // example: `globalF()` then callee is reference (with instantiation Ts filled) // example: `local_var()` then callee is reference (points to local var, filled at resolve identifiers) // example: `getF()()` then callee is another func call (which type is TypeDataFunCallable) -// example: `obj.method()` then callee is dot access (resolved while type inferring) +// example: `obj.method()` then callee is dot access, self_obj = obj +// example: `Point.create()` then callee is dot access, self_obj = nullptr struct Vertex final : ASTExprBinary { FunctionPtr fun_maybe = nullptr; // filled while type inferring for `globalF()` / `obj.f()`; remains nullptr for `local_var()` / `getF()()` + bool dot_obj_is_self = false; // true for `obj.method()` (obj will be `self` in method); false for `globalF()` / `Point.create()` AnyExprV get_callee() const { return lhs; } - bool is_dot_call() const { return lhs->kind == ast_dot_access; } - AnyExprV get_dot_obj() const { return lhs->as()->get_obj(); } + AnyExprV get_self_obj() const { return dot_obj_is_self ? lhs->as()->get_obj() : nullptr; } auto get_arg_list() const { return rhs->as(); } int get_num_args() const { return rhs->as()->size(); } auto get_arg(int i) const { return rhs->as()->get_arg(i); } Vertex* mutate() const { return const_cast(this); } - void assign_fun_ref(FunctionPtr fun_ref); + void assign_fun_ref(FunctionPtr fun_ref, bool dot_obj_is_self); Vertex(SrcLocation loc, AnyExprV lhs_f, V arguments) : ASTExprBinary(ast_function_call, loc, lhs_f, arguments) {} @@ -1166,7 +1167,7 @@ struct Vertex final : ASTOtherVararg { std::vector get_items() const { return children; } auto get_item(int i) const { return children.at(i)->as(); } - Vertex(SrcLocation loc, std::vector instantiationTs) + Vertex(SrcLocation loc, std::vector&& instantiationTs) : ASTOtherVararg(ast_instantiationT_list, loc, std::move(instantiationTs)) {} }; @@ -1229,9 +1230,10 @@ struct Vertex final : ASTOtherVararg { AnyV get_body() const { return children.at(2); } // ast_block_statement / ast_asm_body FunctionPtr fun_ref = nullptr; // filled after register + AnyTypeV receiver_type_node; // for `fun builder.storeInt`, here is `builder` AnyTypeV return_type_node; // if unspecified (nullptr), means "auto infer" V genericsT_list; // for non-generics it's nullptr - td::RefInt256 method_id; // specified via @method_id annotation + td::RefInt256 tvm_method_id; // specified via @method_id annotation int flags; // from enum in FunctionData bool is_asm_function() const { return children.at(2)->kind == ast_asm_body; } @@ -1241,9 +1243,9 @@ struct Vertex final : ASTOtherVararg { Vertex* mutate() const { return const_cast(this); } void assign_fun_ref(FunctionPtr fun_ref); - Vertex(SrcLocation loc, V name_identifier, V parameters, AnyV body, AnyTypeV return_type_node, V genericsT_list, td::RefInt256 method_id, int flags) + Vertex(SrcLocation loc, V name_identifier, V parameters, AnyV body, AnyTypeV receiver_type_node, AnyTypeV return_type_node, V genericsT_list, td::RefInt256 tvm_method_id, int flags) : ASTOtherVararg(ast_function_declaration, loc, {name_identifier, parameters, body}) - , return_type_node(return_type_node), genericsT_list(genericsT_list), method_id(std::move(method_id)), flags(flags) {} + , receiver_type_node(receiver_type_node), return_type_node(return_type_node), genericsT_list(genericsT_list), tvm_method_id(std::move(tvm_method_id)), flags(flags) {} }; template<> diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 7de0b3d843..5b962f604f 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -40,16 +40,18 @@ static std::vector define_builtin_parameters(const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags) { - auto* f_sym = new FunctionData(name, {}, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); + auto* f_sym = new FunctionData(name, {}, "", nullptr, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); G.symtable.add_function(f_sym); } -static void define_builtin_func(const std::string& name, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags, - std::initializer_list arg_order, std::initializer_list ret_order) { - auto* f_sym = new FunctionData(name, {}, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); +static void define_builtin_method(const std::string& name, TypePtr receiver_type, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags, + std::initializer_list arg_order = {}, std::initializer_list ret_order = {}) { + std::string method_name = name.substr(name.find('.') + 1); + auto* f_sym = new FunctionData(name, {}, std::move(method_name), receiver_type, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); f_sym->arg_order = arg_order; f_sym->ret_order = ret_order; G.symtable.add_function(f_sym); + G.all_methods.push_back(f_sym); } void FunctionBodyBuiltin::compile(AsmOpList& dest, std::vector& out, std::vector& in, @@ -988,10 +990,10 @@ static AsmOp compile_bool_const(std::vector& res, std::vector 1 0) "LDIX"; -// fun loadUint (mutate s: slice, len: int): int asm( -> 1 0) "LDUX"; -// fun preloadInt (s: slice, len: int): int asm "PLDIX"; -// fun preloadUint(s: slice, len: int): int asm "PLDUX"; +// fun slice.loadInt (mutate self, len: int): int asm(s len -> 1 0) "LDIX"; +// fun slice.loadUint (mutate self, len: int): int asm( -> 1 0) "LDUX"; +// fun slice.preloadInt (self, len: int): int asm "PLDIX"; +// fun slice.preloadUint(self, len: int): int asm "PLDUX"; static AsmOp compile_fetch_int(std::vector& res, std::vector& args, SrcLocation loc, bool fetch, bool sgnd) { tolk_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); auto &y = args[1], &r = res.back(); @@ -1013,8 +1015,8 @@ static AsmOp compile_fetch_int(std::vector& res, std::vector return exec_op(loc, (fetch ? "LD"s : "PLD"s) + (sgnd ? "IX" : "UX"), 2, 1 + (unsigned)fetch); } -// fun storeInt (mutate self: builder, x: int, len: int): self asm(x b len) "STIX"; -// fun storeUint (mutate self: builder, x: int, len: int): self asm(x b len) "STUX"; +// fun builder.storeInt (mutate self, x: int, len: int): self asm(x b len) "STIX"; +// fun builder.storeUint (mutate self, x: int, len: int): self asm(x b len) "STUX"; static AsmOp compile_store_int(std::vector& res, std::vector& args, SrcLocation loc, bool sgnd) { tolk_assert(args.size() == 3 && res.size() == 1); auto& z = args[2]; @@ -1025,8 +1027,8 @@ static AsmOp compile_store_int(std::vector& res, std::vector return exec_op(loc, "ST"s + (sgnd ? "IX" : "UX"), 3, 1); } -// fun loadBits (mutate self: slice, len: int): self asm(s len -> 1 0) "LDSLICEX" -// fun preloadBits(self: slice, len: int): slice asm(s len -> 1 0) "PLDSLICEX" +// fun slice.loadBits (mutate self, len: int): self asm(s len -> 1 0) "LDSLICEX" +// fun slice.preloadBits(self, len: int): slice asm(s len -> 1 0) "PLDSLICEX" static AsmOp compile_fetch_slice(std::vector& res, std::vector& args, SrcLocation loc, bool fetch) { tolk_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); auto& y = args[1]; @@ -1041,8 +1043,8 @@ static AsmOp compile_fetch_slice(std::vector& res, std::vector(t: tuple, index: int): X asm "INDEXVAR"; -static AsmOp compile_tuple_at(std::vector& res, std::vector& args, SrcLocation loc) { +// fun tuple.get(t: tuple, index: int): X asm "INDEXVAR"; +static AsmOp compile_tuple_get(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 2 && res.size() == 1); auto& y = args[1]; if (y.is_int_const() && y.int_const >= 0 && y.int_const < 16) { @@ -1052,7 +1054,7 @@ static AsmOp compile_tuple_at(std::vector& res, std::vector& return exec_op(loc, "INDEXVAR", 2, 1); } -// fun tupleSetAt(mutate self: tuple, value: X, index: int): void asm "SETINDEXVAR"; +// fun tuple.set(mutate self: tuple, value: X, index: int): void asm "SETINDEXVAR"; static AsmOp compile_tuple_set_at(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 3 && res.size() == 1); auto& y = args[2]; @@ -1063,17 +1065,17 @@ static AsmOp compile_tuple_set_at(std::vector& res, std::vector&, std::vector&, SrcLocation loc) { return AsmOp::Custom(loc, "DUMPSTK", 0, 0); } -// fun debugPrintString(x: T): void asm "STRDUMP"; +// fun debug.printString(x: T): void asm "STRDUMP"; static AsmOp compile_strdump(std::vector&, std::vector&, SrcLocation loc) { return AsmOp::Custom(loc, "STRDUMP DROP", 1, 1); } -// fun debugPrint(x: T): void; +// fun debug.print(x: T): void; static AsmOp compile_debug_print_to_string(std::vector&, std::vector&, SrcLocation loc) { return AsmOp::Custom(loc, "s0 DUMP DROP", 1, 1); } @@ -1116,16 +1118,18 @@ void define_builtins() { TypePtr Tuple = TypeDataTuple::create(); TypePtr Never = TypeDataNever::create(); - std::vector itemsT; - itemsT.emplace_back("T"); TypePtr typeT = TypeDataGenericT::create("T"); - const GenericsDeclaration* declGenericT = new GenericsDeclaration(std::move(itemsT)); + const GenericsDeclaration* declGenericT = new GenericsDeclaration(std::vector{"T"}, 0); std::vector ParamsInt1 = {Int}; std::vector ParamsInt2 = {Int, Int}; std::vector ParamsInt3 = {Int, Int, Int}; std::vector ParamsSliceInt = {Slice, Int}; + // these types are defined in stdlib, currently unknown + // see patch_builtins_after_stdlib_loaded() below + TypePtr debug = TypeDataUnknown::create(); + // builtin operators // they are internally stored as functions, because at IR level, there is no difference // between calling `userAdd(a,b)` and `_+_(a,b)` @@ -1280,48 +1284,48 @@ void define_builtins() { define_builtin_func("mulDivMod", ParamsInt3, TypeDataTensor::create({Int, Int}), nullptr, compile_muldivmod, FunctionData::flagMarkedAsPure); - define_builtin_func("loadInt", ParamsSliceInt, Int, nullptr, + define_builtin_method("slice.loadInt", Slice, ParamsSliceInt, Int, nullptr, std::bind(compile_fetch_int, _1, _2, _3, true, true), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, {}, {1, 0}); - define_builtin_func("loadUint", ParamsSliceInt, Int, nullptr, + define_builtin_method("slice.loadUint", Slice, ParamsSliceInt, Int, nullptr, std::bind(compile_fetch_int, _1, _2, _3, true, false), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, {}, {1, 0}); - define_builtin_func("loadBits", ParamsSliceInt, Slice, nullptr, + define_builtin_method("slice.loadBits", Slice, ParamsSliceInt, Slice, nullptr, std::bind(compile_fetch_slice, _1, _2, _3, true), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, {}, {1, 0}); - define_builtin_func("preloadInt", ParamsSliceInt, Int, nullptr, + define_builtin_method("slice.preloadInt", Slice, ParamsSliceInt, Int, nullptr, std::bind(compile_fetch_int, _1, _2, _3, false, true), FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); - define_builtin_func("preloadUint", ParamsSliceInt, Int, nullptr, + define_builtin_method("slice.preloadUint", Slice, ParamsSliceInt, Int, nullptr, std::bind(compile_fetch_int, _1, _2, _3, false, false), FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); - define_builtin_func("preloadBits", ParamsSliceInt, Slice, nullptr, + define_builtin_method("slice.preloadBits", Slice, ParamsSliceInt, Slice, nullptr, std::bind(compile_fetch_slice, _1, _2, _3, false), FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); - define_builtin_func("storeInt", {Builder, Int, Int}, Unit, nullptr, + define_builtin_method("builder.storeInt", Builder, {Builder, Int, Int}, Unit, nullptr, std::bind(compile_store_int, _1, _2, _3, true), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, {1, 0, 2}, {}); - define_builtin_func("storeUint", {Builder, Int, Int}, Unit, nullptr, + define_builtin_method("builder.storeUint", Builder, {Builder, Int, Int}, Unit, nullptr, std::bind(compile_store_int, _1, _2, _3, false), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, {1, 0, 2}, {}); - define_builtin_func("tupleAt", {Tuple, Int}, typeT, declGenericT, - compile_tuple_at, + define_builtin_method("tuple.get", Tuple, {Tuple, Int}, typeT, declGenericT, + compile_tuple_get, FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); - define_builtin_func("tupleSetAt", {Tuple, typeT, Int}, Unit, declGenericT, + define_builtin_method("tuple.set", Tuple, {Tuple, typeT, Int}, Unit, declGenericT, compile_tuple_set_at, FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf); - define_builtin_func("debugPrint", {typeT}, Unit, declGenericT, + define_builtin_method("debug.print", debug, {typeT}, Unit, declGenericT, compile_debug_print_to_string, 0); - define_builtin_func("debugPrintString", {typeT}, Unit, declGenericT, + define_builtin_method("debug.printString", debug, {typeT}, Unit, declGenericT, compile_strdump, 0); - define_builtin_func("debugDumpStack", {}, Unit, nullptr, + define_builtin_method("debug.dumpStack", debug, {}, Unit, nullptr, compile_dumpstk, 0); @@ -1333,4 +1337,17 @@ void define_builtins() { FunctionData::flagMarkedAsPure); } +// there are some built-in functions that operate on types declared in stdlib (like Cell) +// that's why these symbols were undefined, and when builtins were registered, they were set to unknown +// after all files have been loaded, symbols have been registered, and aliases exist, +// we patch that earlier registered built-in functions providing types that now exist +void patch_builtins_after_stdlib_loaded() { + StructPtr struct_debug = lookup_global_symbol("debug")->try_as(); + TypePtr debug = TypeDataStruct::create(struct_debug); + + lookup_function("debug.print")->mutate()->receiver_type = debug; + lookup_function("debug.printString")->mutate()->receiver_type = debug; + lookup_function("debug.dumpStack")->mutate()->receiver_type = debug; +} + } // namespace tolk diff --git a/tolk/compiler-state.h b/tolk/compiler-state.h index 9b67293b57..2193503a22 100644 --- a/tolk/compiler-state.h +++ b/tolk/compiler-state.h @@ -96,8 +96,9 @@ struct CompilerState { GlobalSymbolTable symtable; PersistentHeapAllocator persistent_mem; - std::vector all_functions; // all user-defined (not built-in) functions, with generic instantiations - std::vector all_get_methods; + std::vector all_functions; // all user-defined (not built-in) global-scope functions, with generic instantiations + std::vector all_methods; // all user-defined and built-in extension methods for arbitrary types (receivers) + std::vector all_contract_getters; std::vector all_global_vars; std::vector all_constants; std::vector all_structs; diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index 9983ca49dd..0f8e6356ce 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -24,7 +24,7 @@ namespace tolk { -// given orig `(int, T)` and substitutions [T=slice], return `(int, slice)` +// given orig `(int, T)` and substitutions [slice], return `(int, slice)` static TypePtr replace_genericT_with_deduced(TypePtr orig, const GenericsSubstitutions* substitutedTs) { if (!orig || !orig->has_genericT_inside()) { return orig; @@ -52,9 +52,18 @@ static TypePtr replace_genericT_with_deduced(TypePtr orig, const GenericsSubstit }); } -std::string GenericsSubstitutions::as_human_readable() const { +GenericsSubstitutions::GenericsSubstitutions(const GenericsDeclaration* genericTs, const std::vector& type_arguments) + : genericTs(genericTs) + , valuesTs(genericTs->size()) { + provide_type_arguments(type_arguments); +} + +std::string GenericsSubstitutions::as_human_readable(bool show_nullptr) const { std::string result; for (int i = 0; i < size(); ++i) { + if (valuesTs[i] == nullptr && !show_nullptr) { + continue;; + } if (!result.empty()) { result += ", "; } @@ -74,9 +83,6 @@ void GenericsSubstitutions::set_typeT(std::string_view nameT, TypePtr typeT) { for (int i = 0; i < size(); ++i) { if (genericTs->get_nameT(i) == nameT) { if (valuesTs[i] == nullptr) { - if (typeT == TypeDataNullLiteral::create() || typeT->has_unknown_inside()) { - throw GenericDeduceError(nameT); - } tolk_assert(!typeT->has_genericT_inside()); valuesTs[i] = typeT; } @@ -85,12 +91,12 @@ void GenericsSubstitutions::set_typeT(std::string_view nameT, TypePtr typeT) { } } -GenericsSubstitutions::GenericsSubstitutions(const GenericsDeclaration* genericTs, const std::vector& type_arguments) - : genericTs(genericTs) { - assert(genericTs != nullptr && genericTs->size() == static_cast(type_arguments.size())); - valuesTs.resize(genericTs->size()); - for (int i = 0; i < genericTs->size(); ++i) { - valuesTs[i] = type_arguments[i]; +void GenericsSubstitutions::provide_type_arguments(const std::vector& type_arguments) { + tolk_assert(genericTs != nullptr); + int start_from = genericTs->n_from_receiver; // for `Container.wrap` user should specify only U + tolk_assert(static_cast(type_arguments.size()) + start_from == genericTs->size()); + for (int i = start_from; i < genericTs->size(); ++i) { + valuesTs[i] = type_arguments[i - start_from]; } } @@ -205,18 +211,17 @@ TypePtr GenericSubstitutionsDeducing::replace_Ts_with_currently_deduced(TypePtr return replace_genericT_with_deduced(orig, &deducedTs); } +TypePtr GenericSubstitutionsDeducing::auto_deduce_from_argument(TypePtr param_type, TypePtr arg_type) { + consider_next_condition(param_type, arg_type); + return replace_genericT_with_deduced(param_type, &deducedTs); +} + TypePtr GenericSubstitutionsDeducing::auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type) { - try { - consider_next_condition(param_type, arg_type); - param_type = replace_genericT_with_deduced(param_type, &deducedTs); - if (param_type->has_genericT_inside()) { - fire_error_can_not_deduce(cur_f, loc, get_first_not_deduced_nameT()); - } - return param_type; - } catch (const GenericDeduceError& ex) { - fire_error_can_not_deduce(cur_f, loc, ex.nameT); - return nullptr; + param_type = auto_deduce_from_argument(param_type, arg_type); + if (param_type->has_genericT_inside()) { + fire_error_can_not_deduce(cur_f, loc, get_first_not_deduced_nameT()); } + return param_type; } std::string_view GenericSubstitutionsDeducing::get_first_not_deduced_nameT() const { @@ -236,27 +241,43 @@ void GenericSubstitutionsDeducing::fire_error_can_not_deduce(FunctionPtr cur_f, } } -std::string GenericsDeclaration::as_human_readable() const { - std::string result = "<"; - for (const GenericsItem& item : itemsT) { - if (result.size() > 1) { - result += ", "; +std::string GenericsDeclaration::as_human_readable(bool include_from_receiver) const { + std::string result; + if (int start_from = include_from_receiver ? 0 : n_from_receiver; start_from < size()) { + result += '<'; + for (int i = start_from; i < size(); ++i) { + if (result.size() > 1) { + result += ", "; + } + result += namesT[i]; } - result += item.nameT; + result += '>'; } - result += ">"; return result; } int GenericsDeclaration::find_nameT(std::string_view nameT) const { - for (int i = 0; i < static_cast(itemsT.size()); ++i) { - if (itemsT[i].nameT == nameT) { + for (int i = 0; i < static_cast(namesT.size()); ++i) { + if (namesT[i] == nameT) { return i; } } return -1; } +void GenericsDeclaration::append_ast_list_checking_duplicates(AnyV v_genericsT_list, std::vector& out) { + for (int i = 0; i < v_genericsT_list->as()->size(); ++i) { + auto v_item = v_genericsT_list->as()->get_item(i); + auto it_existing = std::find_if(out.begin(), out.end(), [v_item](const std::string_view& prevT) { + return prevT == v_item->nameT; + }); + if (it_existing != out.end()) { + v_item->error("duplicate generic parameter `" + static_cast(v_item->nameT) + "`"); + } + out.emplace_back(v_item->nameT); + } +} + bool GenericsSubstitutions::has_nameT(std::string_view nameT) const { return genericTs->find_nameT(nameT) != -1; } @@ -282,28 +303,34 @@ bool GenericsSubstitutions::equal_to(const GenericsSubstitutions* rhs) const { // when cloning `f`, original name is "f", we need a new name for symtable and output // name of an instantiated function will be "f" and similar (yes, with "<" symbol, it's okay to Fift) -static std::string generate_instantiated_name(const std::string& orig_name, const GenericsSubstitutions& substitutedTs, bool allow_spaces) { +static std::string generate_instantiated_name(const std::string& orig_name, const GenericsSubstitutions& substitutedTs, bool allow_spaces, int size_from_receiver = 0) { // an instantiated function name will be "{orig_name}<{T1,T2,...}>" std::string name = orig_name; - name += "<"; - for (int i = 0; i < substitutedTs.size(); ++i) { - if (name.size() > orig_name.size() + 1) { - name += ", "; + if (size_from_receiver < substitutedTs.size()) { + name += '<'; + for (int i = size_from_receiver; i < substitutedTs.size(); ++i) { + if (name.size() > orig_name.size() + 1) { + name += ", "; + } + name += substitutedTs.typeT_at(i)->as_human_readable(); } - name += substitutedTs.typeT_at(i)->as_human_readable(); + name += '>'; } if (!allow_spaces) { name.erase(std::remove(name.begin(), name.end(), ' '), name.end()); } - name += ">"; return name; } FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs) { - tolk_assert(fun_ref->is_generic_function() && !fun_ref->is_method_id_not_empty()); + tolk_assert(fun_ref->is_generic_function() && !fun_ref->has_tvm_method_id()); // fun_ref->name = "f", inst_name will be "f" and similar - std::string new_name = generate_instantiated_name(fun_ref->name, substitutedTs, false); + std::string fun_name = fun_ref->name; + if (fun_ref->is_method() && fun_ref->receiver_type->has_genericT_inside()) { + fun_name = replace_genericT_with_deduced(fun_ref->receiver_type, &substitutedTs)->as_human_readable() + "." + fun_ref->method_name; + } + std::string new_name = generate_instantiated_name(fun_name, substitutedTs, false, fun_ref->genericTs->n_from_receiver); if (const Symbol* existing_sym = lookup_global_symbol(new_name)) { FunctionPtr existing_ref = existing_sym->try_as(); tolk_assert(existing_ref); @@ -324,7 +351,8 @@ FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstituti new_parameters.emplace_back(orig_p.name, orig_p.loc, new_param_type, orig_p.flags, orig_p.param_idx); } TypePtr new_return_type = replace_genericT_with_deduced(fun_ref->declared_return_type, allocatedTs); - FunctionData* new_fun_ref = new FunctionData(new_name, fun_ref->loc, new_return_type, std::move(new_parameters), fun_ref->flags, nullptr, allocatedTs, fun_ref->body, fun_ref->ast_root); + TypePtr new_receiver_type = replace_genericT_with_deduced(fun_ref->receiver_type, allocatedTs); + FunctionData* new_fun_ref = new FunctionData(new_name, fun_ref->loc, fun_ref->method_name, new_receiver_type, new_return_type, std::move(new_parameters), fun_ref->flags, nullptr, allocatedTs, fun_ref->body, fun_ref->ast_root); new_fun_ref->arg_order = fun_ref->arg_order; new_fun_ref->ret_order = fun_ref->ret_order; new_fun_ref->base_fun_ref = fun_ref; @@ -336,10 +364,9 @@ FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstituti // it means, that types still contain T: `f(v: T): T`, but since type resolving knows // it's instantiation, when resolving types, it substitutes T=int V orig_root = fun_ref->ast_root->as(); - V new_name_ident = createV(orig_root->get_identifier()->loc, new_name); - V new_root = ASTReplicator::clone_function_ast(orig_root, new_name_ident); + V new_root = ASTReplicator::clone_function_ast(orig_root); - FunctionPtr new_fun_ref = pipeline_register_instantiated_generic_function(fun_ref, new_root, allocatedTs); + FunctionPtr new_fun_ref = pipeline_register_instantiated_generic_function(fun_ref, new_root, std::move(new_name), allocatedTs); tolk_assert(new_fun_ref); // body of a cloned function (it's cloned at type inferring step) needs the previous pipeline to run // for example, all local vars need to be registered as symbols, etc. @@ -368,7 +395,7 @@ StructPtr instantiate_generic_struct(StructPtr struct_ref, GenericsSubstitutions V new_name_ident = createV(orig_root->get_identifier()->loc, new_name); V new_root = ASTReplicator::clone_struct_ast(orig_root, new_name_ident); - StructPtr new_struct_ref = pipeline_register_instantiated_generic_struct(struct_ref, new_root, allocatedTs); + StructPtr new_struct_ref = pipeline_register_instantiated_generic_struct(struct_ref, new_root, std::move(new_name), allocatedTs); tolk_assert(new_struct_ref); pipeline_resolve_identifiers_and_assign_symbols(new_struct_ref); pipeline_resolve_types_and_aliases(new_struct_ref); @@ -391,10 +418,88 @@ AliasDefPtr instantiate_generic_alias(AliasDefPtr alias_ref, GenericsSubstitutio V new_name_ident = createV(orig_root->get_identifier()->loc, new_name); V new_root = ASTReplicator::clone_type_alias_ast(orig_root, new_name_ident); - AliasDefPtr new_alias_ref = pipeline_register_instantiated_generic_alias(alias_ref, new_root, allocatedTs); + AliasDefPtr new_alias_ref = pipeline_register_instantiated_generic_alias(alias_ref, new_root, std::move(new_name), allocatedTs); tolk_assert(new_alias_ref); pipeline_resolve_types_and_aliases(new_alias_ref); return new_alias_ref; } +// find `builder.storeInt` for called_receiver = "builder" and called_name = "storeInt" +// most practical case, when a direct method for receiver exists +FunctionPtr match_exact_method_for_call_not_generic(TypePtr called_receiver, std::string_view called_name) { + FunctionPtr exact_found = nullptr; + + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == called_name && !method_ref->receiver_type->has_genericT_inside()) { + if (method_ref->receiver_type->equal_to(called_receiver)) { + if (exact_found) { + return nullptr; + } + exact_found = method_ref; + } + } + } + + return exact_found; +} + +// find `int?.copy` / `T.copy` for called_receiver = "int" and called_name = "copy" +std::vector match_methods_for_call_including_generic(TypePtr called_receiver, std::string_view called_name) { + std::vector candidates; + + // step1: find all methods where a receiver equals to provided, e.g. `MInt.copy` + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == called_name && !method_ref->receiver_type->has_genericT_inside()) { + if (method_ref->receiver_type->equal_to(called_receiver)) { + candidates.emplace_back(method_ref, GenericsSubstitutions(method_ref->genericTs)); + } + } + } + if (!candidates.empty()) { + return candidates; + } + + // step2: find all methods where a receiver can accept provided, e.g. `int8.copy` / `int?.copy` / `(int|slice).copy` + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == called_name && !method_ref->receiver_type->has_genericT_inside()) { + if (method_ref->receiver_type->can_rhs_be_assigned(called_receiver)) { + candidates.emplace_back(method_ref, GenericsSubstitutions(method_ref->genericTs)); + } + } + } + if (!candidates.empty()) { + return candidates; + } + + // step 3: try to match generic receivers, e.g. `Container.copy` / `(T?|slice).copy` but NOT `T.copy` + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == called_name && method_ref->receiver_type->has_genericT_inside() && !method_ref->receiver_type->try_as()) { + try { + GenericSubstitutionsDeducing deducingTs(method_ref); + TypePtr replaced = deducingTs.auto_deduce_from_argument(method_ref->receiver_type, called_receiver); + if (!replaced->has_genericT_inside()) { + candidates.emplace_back(method_ref, deducingTs.flush()); + } + } catch (...) {} + } + } + if (!candidates.empty()) { + return candidates; + } + + // step 4: try to match `T.copy` + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == called_name && method_ref->receiver_type->try_as()) { + try { + GenericSubstitutionsDeducing deducingTs(method_ref); + TypePtr replaced = deducingTs.auto_deduce_from_argument(method_ref->receiver_type, called_receiver); + if (!replaced->has_genericT_inside()) { + candidates.emplace_back(method_ref, deducingTs.flush()); + } + } catch (...) {} + } + } + return candidates; +} + } // namespace tolk diff --git a/tolk/generics-helpers.h b/tolk/generics-helpers.h index 7d324e625e..08dfcc9812 100644 --- a/tolk/generics-helpers.h +++ b/tolk/generics-helpers.h @@ -24,25 +24,26 @@ namespace tolk { // when a function is declared `f`, this "" is represented as this class -// (not at AST, but at symbol storage level) +// - `fun tuple.push(self, value:T)` namesT = [T] +// - `struct Pair` namesT = [A,B] +// - `fun Container.compareWith` namesT = [T,U], n_from_receiver = 1 +// - `fun Pair.createFrom` namesT = [U,V] +// - `fun Pair.create` namesT = [A,B], n_from_receiver = 2 struct GenericsDeclaration { - struct GenericsItem { - std::string_view nameT; + explicit GenericsDeclaration(std::vector&& namesT, int n_from_receiver) + : namesT(std::move(namesT)) + , n_from_receiver(n_from_receiver) {} - explicit GenericsItem(std::string_view nameT) - : nameT(nameT) {} - }; + const std::vector namesT; + const int n_from_receiver; - explicit GenericsDeclaration(std::vector&& itemsT) - : itemsT(std::move(itemsT)) {} + std::string as_human_readable(bool include_from_receiver = false) const; - const std::vector itemsT; - - std::string as_human_readable() const; - - int size() const { return static_cast(itemsT.size()); } + int size() const { return static_cast(namesT.size()); } int find_nameT(std::string_view nameT) const; - std::string_view get_nameT(int idx) const { return itemsT[idx].nameT; } + std::string_view get_nameT(int idx) const { return namesT[idx]; } + + static void append_ast_list_checking_duplicates(AnyV v_genericsT_list, std::vector& out); }; // when a function call is `f()`, this "" is represented as this class @@ -59,7 +60,7 @@ struct GenericsSubstitutions { } explicit GenericsSubstitutions(const GenericsDeclaration* genericTs, const std::vector& type_arguments); - std::string as_human_readable() const; + std::string as_human_readable(bool show_nullptr) const; int size() const { return static_cast(valuesTs.size()); } bool has_nameT(std::string_view nameT) const; @@ -69,6 +70,7 @@ struct GenericsSubstitutions { bool equal_to(const GenericsSubstitutions* rhs) const; void set_typeT(std::string_view nameT, TypePtr typeT); + void provide_type_arguments(const std::vector& type_arguments); }; // this class helps to deduce Ts on the fly @@ -87,6 +89,7 @@ class GenericSubstitutionsDeducing { explicit GenericSubstitutionsDeducing(StructPtr struct_ref); TypePtr replace_Ts_with_currently_deduced(TypePtr orig) const; + TypePtr auto_deduce_from_argument(TypePtr param_type, TypePtr arg_type); TypePtr auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type); std::string_view get_first_not_deduced_nameT() const; void fire_error_can_not_deduce(FunctionPtr cur_f, SrcLocation loc, std::string_view nameT) const; @@ -96,19 +99,13 @@ class GenericSubstitutionsDeducing { } }; -struct GenericDeduceError final : std::exception { - std::string nameT; - - explicit GenericDeduceError(std::string_view nameT) - : nameT(nameT) { } - - const char* what() const noexcept override { - return nameT.c_str(); - } -}; +typedef std::pair MethodCallCandidate; FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs); StructPtr instantiate_generic_struct(StructPtr struct_ref, GenericsSubstitutions&& substitutedTs); AliasDefPtr instantiate_generic_alias(AliasDefPtr alias_ref, GenericsSubstitutions&& substitutedTs); +FunctionPtr match_exact_method_for_call_not_generic(TypePtr called_receiver, std::string_view called_name); +std::vector match_methods_for_call_including_generic(TypePtr called_receiver, std::string_view called_name); + } // namespace tolk diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index d34a863995..c31f48762c 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -344,7 +344,6 @@ struct ChunkIdentifierOrKeyword final : ChunkLexerBase { if (str == "var") return tok_var; if (str == "fun") return tok_fun; if (str == "asm") return tok_asm; - if (str == "get") return tok_get; if (str == "try") return tok_try; if (str == "val") return tok_val; break; diff --git a/tolk/lexer.h b/tolk/lexer.h index 55bdf4edbb..19ccb5c105 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -26,7 +26,6 @@ enum TokenType { tok_empty, tok_fun, - tok_get, tok_type, tok_enum, tok_struct, diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 53447277b9..0f86b2ad1f 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -205,7 +205,7 @@ class LValContext { code.emplace_back(loc, Op::_IntConst, index_ir_idx, td::make_refint(index_at)); vars_modification_watcher.trigger_callbacks(tuple_ir_idx, loc); - FunctionPtr builtin_sym = lookup_global_symbol("tupleSetAt")->try_as(); + FunctionPtr builtin_sym = lookup_global_symbol("tuple.set")->try_as(); code.emplace_back(loc, Op::_Call, std::vector{tuple_ir_idx}, std::vector{tuple_ir_idx[0], lval_ir_idx[0], index_ir_idx[0]}, builtin_sym); local_lval.after_let(std::move(tuple_ir_idx), code, loc); } @@ -1167,13 +1167,13 @@ static std::vector process_dot_access(V v, CodeBlob& lval_ctx->capture_tuple_index_modification(v->get_obj(), index_at, lval_ir_idx); return lval_ir_idx; } - // `tupleVar.0` as rvalue: the same as "tupleAt(tupleVar, 0)" written in terms of IR vars + // `tupleVar.0` as rvalue: the same as "tuple.get(tupleVar, 0)" written in terms of IR vars std::vector tuple_ir_idx = pre_compile_expr(v->get_obj(), code); std::vector index_ir_idx = code.create_tmp_var(TypeDataInt::create(), v->get_identifier()->loc, "(tuple-idx)"); code.emplace_back(v->loc, Op::_IntConst, index_ir_idx, td::make_refint(index_at)); std::vector field_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(tuple-field)"); tolk_assert(tuple_ir_idx.size() == 1 && field_ir_idx.size() == 1); // tuples contain only 1-slot values - FunctionPtr builtin_sym = lookup_global_symbol("tupleAt")->try_as(); + FunctionPtr builtin_sym = lookup_global_symbol("tuple.get")->try_as(); code.emplace_back(v->loc, Op::_Call, field_ir_idx, std::vector{tuple_ir_idx[0], index_ir_idx[0]}, builtin_sym); if (lval_ctx && calc_sink_leftmost_obj(v)) { // `tupleVar.0.1 = rhs`, then `tupleVar.0` is rval inside lval lval_ctx->capture_tuple_index_modification(v->get_obj(), index_at, field_ir_idx); @@ -1221,15 +1221,14 @@ static std::vector process_function_call(V v, Code return transition_to_target_type(std::move(rvect), code, target_type, v); } - int delta_self = v->is_dot_call(); - AnyExprV obj_leftmost = nullptr; + AnyExprV obj_leftmost = v->get_self_obj(); + int delta_self = obj_leftmost != nullptr; std::vector args; args.reserve(delta_self + v->get_num_args()); if (delta_self) { - args.push_back(v->get_dot_obj()); - obj_leftmost = v->get_dot_obj(); - while (obj_leftmost->kind == ast_function_call && obj_leftmost->as()->is_dot_call() && obj_leftmost->as()->fun_maybe && obj_leftmost->as()->fun_maybe->does_return_self()) { - obj_leftmost = obj_leftmost->as()->get_dot_obj(); + args.push_back(obj_leftmost); + while (obj_leftmost->kind == ast_function_call && obj_leftmost->as()->get_self_obj() && obj_leftmost->as()->fun_maybe && obj_leftmost->as()->fun_maybe->does_return_self()) { + obj_leftmost = obj_leftmost->as()->get_self_obj(); } } for (int i = 0; i < v->get_num_args(); ++i) { @@ -1243,7 +1242,7 @@ static std::vector process_function_call(V v, Code TypePtr op_call_type = v->inferred_type; TypePtr real_ret_type = v->inferred_type; - if (delta_self && fun_ref->does_return_self()) { + if (obj_leftmost && fun_ref->does_return_self()) { real_ret_type = TypeDataVoid::create(); if (!fun_ref->parameters[0].is_mutate_parameter()) { op_call_type = TypeDataVoid::create(); diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 1df6821404..a863fc5929 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -98,7 +98,7 @@ static void check_function_argument_mutate_back(FunctionPtr cur_f, TypePtr param GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_assign_always_null_to_variable(FunctionPtr cur_f, SrcLocation loc, LocalVarPtr assigned_var, bool is_assigned_null_literal) { std::string var_name = assigned_var->name; - fire(cur_f, loc, "can not infer type of `" + var_name + "`, it's always null; specify its type with `" + var_name + ": `" + (is_assigned_null_literal ? " or use `null as `" : "")); + fire(cur_f, loc, "can not infer type of `" + var_name + "`, it's always null\nspecify its type with `" + var_name + ": `" + (is_assigned_null_literal ? " or use `null as `" : "")); } // fire an error on `untypedTupleVar.0` when inferred as (int,int), or `[int, (int,int)]`, or other non-1 width in a tuple @@ -334,20 +334,16 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { return; } - // so, we have a call `f(args)` or `obj.f(args)`, f is a global function (fun_ref) (code / asm / builtin) - int delta_self = 0; - AnyExprV dot_obj = nullptr; - if (auto v_dot = v->get_callee()->try_as()) { - delta_self = 1; - dot_obj = v_dot->get_obj(); - } + // so, we have a call `f(args)` or `obj.f(args)`, fun_ref is a function/method (code / asm / builtin) + AnyExprV self_obj = v->get_self_obj(); + int delta_self = self_obj != nullptr; - if (dot_obj) { + if (self_obj) { const LocalVarData& param_0 = fun_ref->parameters[0]; TypePtr param_type = param_0.declared_type; - check_function_argument_passed(cur_f, param_type, dot_obj, true); + check_function_argument_passed(cur_f, param_type, self_obj, true); if (param_0.is_mutate_parameter()) { - check_function_argument_mutate_back(cur_f, param_type, dot_obj, true); + check_function_argument_mutate_back(cur_f, param_type, self_obj, true); } } for (int i = 0; i < v->get_num_args(); ++i) { @@ -483,8 +479,8 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { return true; } // `return self.someMethod()` - if (auto v_call = return_expr->try_as(); v_call && v_call->is_dot_call()) { - return v_call->fun_maybe && v_call->fun_maybe->does_return_self() && is_expr_valid_as_return_self(v_call->get_dot_obj()); + if (auto v_call = return_expr->try_as(); v_call && v_call->get_self_obj()) { + return v_call->fun_maybe && v_call->fun_maybe->does_return_self() && is_expr_valid_as_return_self(v_call->get_self_obj()); } // `return cond ? ... : ...` if (auto v_ternary = return_expr->try_as()) { diff --git a/tolk/pipe-check-pure-impure.cpp b/tolk/pipe-check-pure-impure.cpp index 366ff1607a..1a91438750 100644 --- a/tolk/pipe-check-pure-impure.cpp +++ b/tolk/pipe-check-pure-impure.cpp @@ -27,15 +27,17 @@ namespace tolk { GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_impure_operation_inside_pure_function(AnyV v) { - v->error("an impure operation in a pure function"); +static void fire_error_impure_operation_inside_pure_function(FunctionPtr cur_f, SrcLocation loc) { + fire(cur_f, loc, "an impure operation in a pure function"); } class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFunctionBody { - static void fire_if_global_var(AnyExprV v) { + FunctionPtr cur_f = nullptr; + + void fire_if_global_var(AnyExprV v) const { if (auto v_ident = v->try_as()) { if (v_ident->sym->try_as()) { - fire_error_impure_operation_inside_pure_function(v); + fire_error_impure_operation_inside_pure_function(cur_f, v->loc); } } } @@ -54,11 +56,11 @@ class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFuncti // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` if (!v->fun_maybe) { // `local_var(args)` is always impure, no considerations about what's there at runtime - fire_error_impure_operation_inside_pure_function(v); + fire_error_impure_operation_inside_pure_function(cur_f, v->loc); } if (!v->fun_maybe->is_marked_as_pure()) { - fire_error_impure_operation_inside_pure_function(v); + fire_error_impure_operation_inside_pure_function(cur_f, v->loc); } parent::visit(v); @@ -73,17 +75,22 @@ class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFuncti } void visit(V v) override { - fire_error_impure_operation_inside_pure_function(v); + fire_error_impure_operation_inside_pure_function(cur_f, v->loc); } void visit(V v) override { - fire_error_impure_operation_inside_pure_function(v); + fire_error_impure_operation_inside_pure_function(cur_f, v->loc); } public: bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function() && fun_ref->is_marked_as_pure(); } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + } }; void pipeline_check_pure_impure_operations() { diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp index aa274a436c..be4db27499 100644 --- a/tolk/pipe-check-rvalue-lvalue.cpp +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -183,8 +183,8 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { } // for `f()` don't visit ast_reference `f`, to detect `f` usage as non-call, like `var cb = f` // same for `obj.method()`, don't visit ast_reference method, visit only obj - if (v->is_dot_call()) { - parent::visit(v->get_dot_obj()); + if (AnyExprV self_obj = v->get_self_obj()) { + parent::visit(self_obj); } for (int i = 0; i < v->get_num_args(); ++i) { diff --git a/tolk/pipe-find-unused-symbols.cpp b/tolk/pipe-find-unused-symbols.cpp index 2b7e555786..cce03cc901 100644 --- a/tolk/pipe-find-unused-symbols.cpp +++ b/tolk/pipe-find-unused-symbols.cpp @@ -67,7 +67,7 @@ static void mark_function_used_dfs(const std::unique_ptr& op) { void pipeline_find_unused_symbols() { for (FunctionPtr fun_ref : G.all_functions) { - if (fun_ref->is_method_id_not_empty()) { // get methods, main and other entrypoints, regular functions with @method_id + if (fun_ref->has_tvm_method_id()) { // get methods, main and other entrypoints, regular functions with @method_id mark_function_used(fun_ref); } } diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp index b40ce6860d..6cfd506831 100644 --- a/tolk/pipe-generate-fif-output.cpp +++ b/tolk/pipe-generate-fif-output.cpp @@ -146,8 +146,8 @@ void pipeline_generate_fif_output_to_std_cout() { } std::cout << " "; - if (fun_ref->is_method_id_not_empty()) { - std::cout << fun_ref->method_id << " DECLMETHOD " << fun_ref->name << "\n"; + if (fun_ref->has_tvm_method_id()) { + std::cout << fun_ref->tvm_method_id << " DECLMETHOD " << fun_ref->name << "\n"; } else { std::cout << "DECLPROC " << fun_ref->name << "\n"; } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 06e658ac91..2f324f5610 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -147,7 +147,7 @@ static void fire_error_calling_asm_function_with_non1_stack_width_arg(FunctionPt GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_cannot_deduce_untyped_tuple_access(FunctionPtr cur_f, SrcLocation loc, int index) { std::string idx_access = "." + std::to_string(index); - fire(cur_f, loc, "can not deduce type of `" + idx_access + "`; either assign it to variable like `var c: int = " + idx_access + "` or cast the result like `" + idx_access + " as int`"); + fire(cur_f, loc, "can not deduce type of `" + idx_access + "`\neither assign it to variable like `var c: int = " + idx_access + "` or cast the result like `" + idx_access + " as int`"); } // helper function: given hint = `Ok | Err` and struct `Ok`, return `Ok` @@ -196,6 +196,39 @@ static TypePtr try_pick_instantiated_generic_from_hint(TypePtr hint, AliasDefPtr return nullptr; } +// given `p.create` (called_receiver = Point, called_name = "create") +// look up a corresponding method (it may be `Point.create` / `Point?.create` / `T.create`) +static MethodCallCandidate choose_only_method_to_call(FunctionPtr cur_f, SrcLocation loc, TypePtr called_receiver, std::string_view called_name) { + // most practical cases: `builder.storeInt` etc., when a direct method for receiver exists + if (FunctionPtr exact_method = match_exact_method_for_call_not_generic(called_receiver, called_name)) { + return {exact_method, GenericsSubstitutions(exact_method->genericTs)}; + } + + // else, try to match, for example `10.copy` with `int?.copy`, with `T.copy`, etc. + std::vector candidates = match_methods_for_call_including_generic(called_receiver, called_name); + if (candidates.size() == 1) { + return candidates[0]; + } + if (candidates.empty()) { // return nullptr, the caller side decides how to react on this + return {nullptr, GenericsSubstitutions(nullptr)}; + } + + std::ostringstream msg; + msg << "call to method `" << called_name << "` for type `" << called_receiver << "` is ambiguous\n"; + for (const auto& [method_ref, substitutedTs] : candidates) { + msg << "candidate function: `" << method_ref->as_human_readable() << "`"; + if (method_ref->is_generic_function()) { + msg << " with " << substitutedTs.as_human_readable(false); + } + if (method_ref->loc.is_defined()) { + msg << " (declared at " << method_ref->loc << ")\n"; + } else if (method_ref->is_builtin_function()) { + msg << " (builtin)\n"; + } + } + fire(cur_f, loc, msg.str()); +} + /* * This class handles all types of AST vertices and traverses them, filling all AnyExprV::inferred_type. * Note, that it isn't derived from ASTVisitor, it has manual `switch` over all existing vertex types. @@ -656,7 +689,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // `... ? intVar : sliceVar` results in `int | slice`, probably it's not what the user expected // example: `var v = ternary`, show an inference error // do NOT show an error for `var v: T = ternary` (T is hint); it will be checked by type checker later - if (hint == nullptr || hint == TypeDataUnknown::create()) { + if (hint == nullptr || hint == TypeDataUnknown::create() || hint->has_genericT_inside()) { fire(cur_f, v->loc, "types of ternary branches are incompatible: " + to_string(v->get_when_true()) + " and " + to_string(v->get_when_false())); } } @@ -762,11 +795,54 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } - ExprFlow infer_reference(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) const { + // given `genericF` / `t.tupleFirst` (the user manually specified instantiation Ts), + // validate and collect them + // returns: [int, slice] / [cell] + std::vector collect_type_arguments_for_fun(SrcLocation loc, const GenericsDeclaration* genericTs, V instantiationT_list) const { + // for `genericF` user should provide two Ts + // for `Container.wrap` — one U (and one T is implicitly from receiver) + if (instantiationT_list->size() != genericTs->size() - genericTs->n_from_receiver) { + fire(cur_f, loc, "expected " + std::to_string(genericTs->size() - genericTs->n_from_receiver) + " type arguments, got " + std::to_string(instantiationT_list->size())); + } + + std::vector type_arguments; + type_arguments.reserve(instantiationT_list->size()); + for (int i = 0; i < instantiationT_list->size(); ++i) { + type_arguments.push_back(instantiationT_list->get_item(i)->type_node->resolved_type); + } + + return type_arguments; + } + + // when substitutedTs have been collected from `f` or deduced from arguments, instantiate a generic function `f` + // example: was `t.push(2)`, deduced , instantiate `tuple.push` + // example: was `t.push(2)`, collected , instantiate `tuple.push` (will later fail type check) + // example: was `var cb = t.first;` (used as reference, as non-call), instantiate `tuple.first` + // returns fun_ref to instantiated function + FunctionPtr check_and_instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs) const { + // T for asm function must be a TVM primitive (width 1), otherwise, asm would act incorrectly + if (fun_ref->is_asm_function() || fun_ref->is_builtin_function()) { + for (int i = 0; i < substitutedTs.size(); ++i) { + if (substitutedTs.typeT_at(i)->get_width_on_stack() != 1) { + fire_error_calling_asm_function_with_non1_stack_width_arg(cur_f, loc, fun_ref, substitutedTs, i); + } + } + } + + // make deep clone of `f` with substitutedTs (or immediately return from symtable if already instantiated) + return instantiate_generic_function(fun_ref, std::move(substitutedTs)); + } + + ExprFlow infer_reference(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint, FunctionPtr* out_f_called = nullptr) const { + // at current point, v is a reference: + // - either a standalone: `local_var` / `SOME_CONST` / `globalF` / `genericFn` + // - or inside a call: `globalF()` / `genericFn()` / `genericFn()` / `local_var()` + if (LocalVarPtr var_ref = v->sym->try_as()) { TypePtr declared_or_smart_casted = flow.smart_cast_if_exists(SinkExpression(var_ref)); tolk_assert(declared_or_smart_casted != nullptr); // all local vars are presented in flow assign_inferred_type(v, declared_or_smart_casted); + // it might be `local_var()` also, don't fill out_f_called, we have no fun_ref, it's a call of arbitrary expression } else if (GlobalConstPtr const_ref = v->sym->try_as()) { if (!const_ref->inferred_type) { @@ -779,98 +855,83 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, glob_ref->declared_type); } else if (FunctionPtr fun_ref = v->sym->try_as()) { - // it's `globalF` / `globalF` - references to functions used as non-call + // it's `globalF` / `globalF` / `globalF()` / `globalF()` + // if it's a call, then out_f_called is present, we should fill it V v_instantiationTs = v->get_instantiationTs(); - if (fun_ref->is_generic_function() && !v_instantiationTs) { + if (fun_ref->is_generic_function() && !v_instantiationTs && !out_f_called) { // `genericFn` is invalid as non-call, can't be used without fire(cur_f, v->loc, "can not use a generic function " + to_string(fun_ref) + " as non-call"); - } else if (fun_ref->is_generic_function()) { - // `genericFn` is valid, it's a reference to instantiation - GenericsSubstitutions substitutedTs = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref->genericTs, v_instantiationTs); + } else if (fun_ref->is_generic_function() && v_instantiationTs) { + // `genericFn` is valid as non-call, it's a reference to instantiation + // `genericFn()` is also ok, we'll assign an instantiated fun_ref to out_f_called + GenericsSubstitutions substitutedTs(fun_ref->genericTs); + substitutedTs.provide_type_arguments(collect_type_arguments_for_fun(v->loc, fun_ref->genericTs, v_instantiationTs)); fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); v->mutate()->assign_sym(fun_ref); - } else if (v_instantiationTs != nullptr && !fun_ref->is_instantiation_of_generic_function()) { + } else if (UNLIKELY(v_instantiationTs != nullptr)) { // non-generic function referenced like `return beginCell;` - fire(cur_f, v_instantiationTs->loc, "not generic function used with generic T"); + fire(cur_f, v_instantiationTs->loc, "type arguments not expected here"); } - fun_ref->mutate()->assign_is_used_as_noncall(); - get_or_infer_return_type(fun_ref); - assign_inferred_type(v, fun_ref->inferred_full_type); + if (out_f_called) { // so, it's `globalF()` / `genericFn()` / `genericFn()` + *out_f_called = fun_ref; // (it's still may be a generic one, then Ts will be deduced from arguments) + } else { // so, it's `globalF` / `genericFn` as a reference + if (fun_ref->is_compile_time_only()) { + fire(cur_f, v->loc, "can not get reference to this function, it's compile-time only"); + } + fun_ref->mutate()->assign_is_used_as_noncall(); + get_or_infer_return_type(fun_ref); + assign_inferred_type(v, fun_ref->inferred_full_type); + } return ExprFlow(std::move(flow), used_as_condition); - } else if (AliasDefPtr alias_ref = v->sym->try_as()) { - fire(cur_f, v->loc, "type " + to_string(alias_ref) + " can not be used as a value"); - - } else if (StructPtr struct_ref = v->sym->try_as()) { - fire(cur_f, v->loc, "struct " + to_string(struct_ref) + " can not be used as a value"); - } else { tolk_assert(false); } // for non-functions: `local_var` and similar not allowed if (UNLIKELY(v->has_instantiationTs())) { - fire(cur_f, v->get_instantiationTs()->loc, "generic T not expected here"); + fire(cur_f, v->get_instantiationTs()->loc, "type arguments not expected here"); } return ExprFlow(std::move(flow), used_as_condition); } - // given `genericF` / `t.tupleFirst` (the user manually specified instantiation Ts), - // validate and collect them - // returns: [T1=int, T2=slice] / [T=cell] (with flag auto_deduced = false, meaning they are manually specified) - GenericsSubstitutions collect_fun_generic_substitutions_from_manually_specified(SrcLocation loc, const GenericsDeclaration* genericTs, V instantiationT_list) const { - if (instantiationT_list->size() != genericTs->size()) { - fire(cur_f, loc, "wrong count of generic T: expected " + std::to_string(genericTs->size()) + ", got " + std::to_string(instantiationT_list->size())); - } - - std::vector type_arguments; - type_arguments.reserve(instantiationT_list->size()); - for (int i = 0; i < instantiationT_list->size(); ++i) { - type_arguments.push_back(instantiationT_list->get_item(i)->type_node->resolved_type); - } + ExprFlow infer_dot_access(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint, FunctionPtr* out_f_called = nullptr, AnyExprV* out_dot_obj = nullptr) { + // at current point, v is a dot access to a field / index / method: + // - either a standalone: `user.id` / `getUser().id` / `var.0` / `t.size` / `Point.create` / `t.tupleAt` + // - or inside a call: `user.getId()` / `.method()` / `Point.create()` / `t.tupleAt(1)` - return GenericsSubstitutions(genericTs, type_arguments); - } + AnyExprV dot_obj = nullptr; // to be filled for `.(field/index/method)`, nullptr for `Point.create` + FunctionPtr fun_ref = nullptr; // to be filled for `.method` / `Point.create` (both standalone or in a call) + GenericsSubstitutions substitutedTs(nullptr); + V v_ident = v->get_identifier(); // field/method name vertex - // when generic Ts have been collected from user-specified or deduced from arguments, - // instantiate a generic function - // example: was `t.tuplePush(2)`, deduced , instantiate `tuplePush` - // example: was `t.tuplePush(2)`, read , instantiate `tuplePush` (will later fail type check) - // example: was `var cb = t.tupleFirst;` (used as reference, as non-call), instantiate `tupleFirst` - // returns fun_ref to instantiated function - FunctionPtr check_and_instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs) const { - // T for asm function must be a TVM primitive (width 1), otherwise, asm would act incorrectly - if (fun_ref->is_asm_function() || fun_ref->is_builtin_function()) { - for (int i = 0; i < substitutedTs.size(); ++i) { - if (substitutedTs.typeT_at(i)->get_width_on_stack() != 1) { - fire_error_calling_asm_function_with_non1_stack_width_arg(cur_f, loc, fun_ref, substitutedTs, i); + // handle `Point.create` / `Container.wrap`: no dot_obj expression actually, lhs is a type, looking up a method + if (auto obj_ref = v->get_obj()->try_as()) { + if (const auto* obj_as_type = obj_ref->sym->try_as()) { + TypePtr receiver_type = obj_as_type->resolved_type; + std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, v_ident->loc, receiver_type, v->get_field_name()); + if (!fun_ref) { + fire(cur_f, v_ident->loc, "method `" + to_string(v->get_field_name()) + "` not found for type " + to_string(receiver_type)); } } } + // handle other (most, actually) cases: `.field` / `.method` + if (!fun_ref) { + dot_obj = v->get_obj(); + flow = infer_any_expr(dot_obj, std::move(flow), false).out_flow; + } - // make deep clone of `f` with substitutedTs - // (if `f` was already instantiated, it will be immediately returned from a symbol table) - return instantiate_generic_function(fun_ref, std::move(substitutedTs)); - } - - ExprFlow infer_dot_access(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { - // it's NOT a method call `t.tupleSize()` (since such cases are handled by infer_function_call) - // it's `t.0`, `getUser().id`, and `t.tupleSize` (as a reference, not as a call) - flow = infer_any_expr(v->get_obj(), std::move(flow), false).out_flow; - - TypePtr obj_type = v->get_obj()->inferred_type; - TypePtr unwrapped_obj_type = obj_type->unwrap_alias(); - // our goal is to fill v->target knowing type of obj - V v_ident = v->get_identifier(); // field/method name vertex + // our goal is to fill v->target (field/index/method) knowing type of obj + TypePtr obj_type = dot_obj ? dot_obj->inferred_type->unwrap_alias() : TypeDataUnknown::create(); V v_instantiationTs = v->get_instantiationTs(); std::string_view field_name = v_ident->name; // check for field access (`user.id`), when obj is a struct - if (const TypeDataStruct* obj_struct = unwrapped_obj_type->try_as()) { + if (const TypeDataStruct* obj_struct = obj_type->try_as()) { if (StructFieldPtr field_ref = obj_struct->struct_ref->find_field(field_name)) { v->mutate()->assign_target(field_ref); TypePtr inferred_type = field_ref->declared_type; @@ -882,13 +943,13 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); } - // if field_name doesn't exist, don't fire an error now — maybe, it's `user.globalFunction()` + // if field_name doesn't exist, don't fire an error now — maybe, it's `user.method()` } // check for indexed access (`tensorVar.0` / `tupleVar.1`) - if (field_name[0] >= '0' && field_name[0] <= '9') { + if (!fun_ref && field_name[0] >= '0' && field_name[0] <= '9') { int index_at = std::stoi(std::string(field_name)); - if (const auto* t_tensor = unwrapped_obj_type->try_as()) { + if (const auto* t_tensor = obj_type->try_as()) { if (index_at >= t_tensor->size()) { fire(cur_f, v_ident->loc, "invalid tensor index, expected 0.." + std::to_string(t_tensor->items.size() - 1)); } @@ -902,7 +963,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); } - if (const auto* t_tuple = unwrapped_obj_type->try_as()) { + if (const auto* t_tuple = obj_type->try_as()) { if (index_at >= t_tuple->size()) { fire(cur_f, v_ident->loc, "invalid tuple index, expected 0.." + std::to_string(t_tuple->items.size() - 1)); } @@ -916,12 +977,12 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); } - if (unwrapped_obj_type->try_as()) { + if (obj_type->try_as()) { TypePtr item_type = nullptr; if (v->is_lvalue && !hint) { // left side of assignment item_type = TypeDataUnknown::create(); } else { - if (hint == nullptr) { + if (hint == nullptr || hint == TypeDataUnknown::create() || hint->has_genericT_inside()) { fire_error_cannot_deduce_untyped_tuple_access(cur_f, v->loc, index_at); } item_type = hint; @@ -930,53 +991,62 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, item_type); return ExprFlow(std::move(flow), used_as_condition); } - // numeric field not found, fire an error - if (unwrapped_obj_type->try_as()) { // `user.100500` - fire(cur_f, v_ident->loc, "field `" + static_cast(field_name) + "` doesn't exist in type " + to_string(obj_type)); - } else { - fire(cur_f, v_ident->loc, "type " + to_string(obj_type) + " is not indexable"); - } } - // check for method (`t.tupleSize` / `user.globalFunction`) - // for now, Tolk doesn't have object-scoped methods; `t.tupleSize` is a global function `tupleSize` - const Symbol* sym = lookup_global_symbol(field_name); - FunctionPtr fun_ref = sym ? sym->try_as() : nullptr; + // check for method (`t.size` / `user.getId`); even `i.0()` can be here if `fun int.0(self)` exists + // for `T.copy` / `Container.create`, substitution for T is also returned + if (!fun_ref) { + std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, dot_obj->loc, obj_type, field_name); + } // not a field, not a method — fire an error if (!fun_ref) { // as a special case, handle accessing fields of nullable objects, to show a more precise error - if (const TypeDataUnion* as_union = unwrapped_obj_type->try_as(); as_union && as_union->or_null) { + if (const TypeDataUnion* as_union = obj_type->try_as(); as_union && as_union->or_null) { if (const TypeDataStruct* n_struct = as_union->or_null->try_as(); n_struct && n_struct->struct_ref->find_field(field_name)) { - fire(cur_f, v_ident->loc, "can not access field `" + static_cast(field_name) + "` of a possibly nullable object " + to_string(obj_type) + "; check it via `obj != null` or use non-null assertion `obj!` operator"); + fire(cur_f, v_ident->loc, "can not access field `" + static_cast(field_name) + "` of a possibly nullable object " + to_string(dot_obj) + "\ncheck it via `obj != null` or use non-null assertion `obj!` operator"); } } - fire(cur_f, v_ident->loc, "field `" + static_cast(field_name) + "` doesn't exist in type " + to_string(obj_type)); + if (out_f_called) { + fire(cur_f, v_ident->loc, "method `" + to_string(v->get_field_name()) + "` not found for type " + to_string(obj_type)); + } else { + fire(cur_f, v_ident->loc, "field `" + static_cast(field_name) + "` doesn't exist in type " + to_string(dot_obj)); + } } - // `t.tupleSize` is ok, `cs.tupleSize` not - if (!fun_ref->parameters[0].declared_type->can_rhs_be_assigned(obj_type)) { - fire(cur_f, v_ident->loc, "referencing a method for " + to_string(fun_ref->parameters[0].declared_type) + " with object of type " + to_string(obj_type)); + // if `fun T.copy(self)` and reference `int.copy` — all generic parameters are determined by the receiver, we know it + if (fun_ref->is_generic_function() && fun_ref->genericTs->size() == fun_ref->genericTs->n_from_receiver) { + tolk_assert(substitutedTs.typeT_at(0) != nullptr); + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); } - if (fun_ref->is_generic_function() && !v_instantiationTs) { - // `genericFn` and `t.tupleAt` are invalid as non-call, they can't be used without + if (fun_ref->is_generic_function() && !v_instantiationTs && !out_f_called) { + // `t.tupleAt` is invalid as non-call, can't be used without fire(cur_f, v->loc, "can not use a generic function " + to_string(fun_ref) + " as non-call"); - } else if (fun_ref->is_generic_function()) { - // `t.tupleAt` is valid, it's a reference to instantiation - GenericsSubstitutions substitutedTs = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref->genericTs, v_instantiationTs); + } else if (fun_ref->is_generic_function() && v_instantiationTs) { + // `t.tupleAt` is valid as non-call, it's a reference to instantiation + // `t.tupleAt()` is also ok, we'll assign an instantiated fun_ref to out_f_called + substitutedTs.provide_type_arguments(collect_type_arguments_for_fun(v->loc, fun_ref->genericTs, v_instantiationTs)); fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); } else if (UNLIKELY(v_instantiationTs != nullptr)) { - // non-generic method referenced like `var cb = c.cellHash;` - fire(cur_f, v_instantiationTs->loc, "not generic function used with generic T"); + // non-generic method referenced like `var cb = c.hash;` + fire(cur_f, v_instantiationTs->loc, "type arguments not expected here"); } - fun_ref->mutate()->assign_is_used_as_noncall(); - v->mutate()->assign_target(fun_ref); - get_or_infer_return_type(fun_ref); - assign_inferred_type(v, fun_ref->inferred_full_type); // type of `t.tupleSize` is TypeDataFunCallable + if (out_f_called) { // so, it's `user.method()` / `t.tupleAt()` / `t.tupleAt()` / `Point.create` + *out_f_called = fun_ref; // (it's still may be a generic one, then Ts will be deduced from arguments) + *out_dot_obj = dot_obj; + } else { // so, it's `user.method` / `t.tupleAt` as a reference + if (fun_ref->is_compile_time_only()) { + fire(cur_f, v->get_identifier()->loc, "can not get reference to this method, it's compile-time only"); + } + fun_ref->mutate()->assign_is_used_as_noncall(); + v->mutate()->assign_target(fun_ref); + get_or_infer_return_type(fun_ref); + assign_inferred_type(v, fun_ref->inferred_full_type); + } return ExprFlow(std::move(flow), used_as_condition); } @@ -984,50 +1054,20 @@ class InferTypesAndCallsAndFieldsVisitor final { AnyExprV callee = v->get_callee(); // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` - int delta_self = 0; - AnyExprV dot_obj = nullptr; + AnyExprV self_obj = nullptr; // for `obj.method()`, obj will be here (but for `Point.create()`, no obj exists) FunctionPtr fun_ref = nullptr; - V v_instantiationTs = nullptr; - - if (auto v_ref = callee->try_as()) { - // `globalF()` / `globalF()` / `local_var()` / `SOME_CONST()` - fun_ref = v_ref->sym->try_as(); // not null for `globalF` - v_instantiationTs = v_ref->get_instantiationTs(); // present for `globalF()` - - } else if (auto v_dot = callee->try_as()) { - // `obj.someMethod()` / `obj.someMethod()` / `getF().someMethod()` / `obj.SOME_CONST()` - // note, that dot_obj->target is not filled yet, since callee was not inferred yet - delta_self = 1; - dot_obj = v_dot->get_obj(); - v_instantiationTs = v_dot->get_instantiationTs(); // present for `obj.someMethod()` - flow = infer_any_expr(dot_obj, std::move(flow), false).out_flow; - - // it can be indexed access (`tensorVar.0()`, `tupleVar.1()`) or a method (`t.tupleSize()`) - std::string_view field_name = v_dot->get_field_name(); - if (const TypeDataStruct* t_struct = dot_obj->inferred_type->unwrap_alias()->try_as(); t_struct && t_struct->struct_ref->find_field(field_name)) { - // `t.someField`, it's a field, probably a callable, fun_ref remains nullptr - } else if (field_name[0] >= '0' && field_name[0] <= '9') { - // indexed access `ab.2()`, then treat `ab.2` just like an expression, fun_ref remains nullptr - // infer_dot_access() will be called for a callee, it will check index correctness - } else { - // for now, Tolk doesn't have fields and object-scoped methods; `t.tupleSize` is a global function `tupleSize` - const Symbol* sym = lookup_global_symbol(field_name); - fun_ref = sym ? sym->try_as() : nullptr; - if (!fun_ref) { - fire(cur_f, v_dot->get_identifier()->loc, "non-existing method `" + to_string(field_name) + "` of type " + to_string(dot_obj)); - } - } + if (callee->kind == ast_reference) { + flow = infer_reference(callee->as(), std::move(flow), false, nullptr, &fun_ref).out_flow; + } else if (callee->kind == ast_dot_access) { + flow = infer_dot_access(callee->as(), std::move(flow), false, nullptr, &fun_ref, &self_obj).out_flow; } else { - // `getF()()` / `5()` - // fun_ref remains nullptr + flow = infer_any_expr(callee, std::move(flow), false).out_flow; } // handle `local_var()` / `getF()()` / `5()` / `SOME_CONST()` / `obj.method()()()` / `tensorVar.0()` if (!fun_ref) { - // treat callee like a usual expression - flow = infer_any_expr(callee, std::move(flow), false).out_flow; - // it must have "callable" inferred type + // callee must have "callable" inferred type const TypeDataFunCallable* f_callable = callee->inferred_type->unwrap_alias()->try_as(); if (!f_callable) { // `5()` / `SOME_CONST()` / `null()` fire(cur_f, v->loc, "calling a non-function " + to_string(callee->inferred_type)); @@ -1041,28 +1081,18 @@ class InferTypesAndCallsAndFieldsVisitor final { flow = infer_any_expr(arg_i, std::move(flow), false, f_callable->params_types[i]).out_flow; assign_inferred_type(v->get_arg(i), arg_i); } - v->mutate()->assign_fun_ref(nullptr); // no fun_ref to a global function assign_inferred_type(v, f_callable->return_type); return ExprFlow(std::move(flow), used_as_condition); } - // for a call `f(...)`, create and instantiate function "f" right now - // so that further, fun_ref will be not a generic, but an instantiated function - if (fun_ref->is_generic_function() && v_instantiationTs) { - GenericsSubstitutions substitutedTs = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref->genericTs, v_instantiationTs); - fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); - } else if (UNLIKELY(v_instantiationTs != nullptr)) { - // `c.cellHash()` / `beginCell()` - fire(cur_f, v_instantiationTs->loc, "calling a not generic function with generic T"); - } - - // so, we have a call `f(args)` or `obj.f(args)`, f is a global function (fun_ref) (code / asm / builtin) + // so, we have a call `f(args)` or `obj.f(args)`, f is fun_ref (function / method) (code / asm / builtin) // we're going to iterate over passed arguments, and (if generic) infer substitutedTs // at first, check arguments count (Tolk doesn't have optional parameters, so just compare counts) + int delta_self = self_obj != nullptr; int n_arguments = v->get_num_args() + delta_self; int n_parameters = fun_ref->get_num_params(); - if (!n_parameters && dot_obj) { - fire(cur_f, v->loc, to_string(fun_ref) + " has no parameters and can not be called as method"); + if (!fun_ref->does_accept_self() && self_obj) { // static method `Point.create(...)` called as `p.create()` + fire(cur_f, v->loc, "method " + to_string(fun_ref) + " can not be called via dot\n(it's a static method, it does not accept `self`)"); } if (n_parameters < n_arguments) { fire(cur_f, v->loc, "too many arguments in call to " + to_string(fun_ref) + ", expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); @@ -1071,34 +1101,38 @@ class InferTypesAndCallsAndFieldsVisitor final { fire(cur_f, v->loc, "too few arguments in call to " + to_string(fun_ref) + ", expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); } - // now, for every passed argument, we need to infer its type - // for regular functions, it's obvious - // but for generic functions, we need to infer type arguments (substitutedTs) on the fly - // (unless Ts are specified by a user like `f(args)` / `t.tupleAt()`, take them) + // for every passed argument, we need to infer its type + // for generic functions, we need to infer type arguments (substitutedTs) on the fly + // (if they are specified by a user like `f(args)` / `t.tupleAt()`, fun_ref is already instantiated) GenericSubstitutionsDeducing deducingTs(fun_ref); - // loop over every argument, for `obj.method()` obj is the first one - // if genericT deducing has a conflict, ParseError is thrown - // note, that deducing Ts one by one is important to manage control flow (mutate params work like assignments) - // a corner case, e.g. `f(v1:T?, v2:T?)` and `f(null,2)` will fail on first argument, won't try the second one - if (dot_obj) { + // for `obj.method()` obj is the first argument (passed to `self` parameter) + if (self_obj) { const LocalVarData& param_0 = fun_ref->parameters[0]; TypePtr param_type = param_0.declared_type; if (param_type->has_genericT_inside()) { - param_type = deducingTs.auto_deduce_from_argument(cur_f, dot_obj->loc, param_type, dot_obj->inferred_type); + param_type = deducingTs.auto_deduce_from_argument(cur_f, self_obj->loc, param_type, self_obj->inferred_type); } - if (param_0.is_mutate_parameter() && dot_obj->inferred_type->unwrap_alias() != param_type->unwrap_alias()) { - if (SinkExpression s_expr = extract_sink_expression_from_vertex(dot_obj)) { - assign_inferred_type(dot_obj, calc_declared_type_before_smart_cast(dot_obj)); + if (param_0.is_mutate_parameter() && self_obj->inferred_type->unwrap_alias() != param_type->unwrap_alias()) { + if (SinkExpression s_expr = extract_sink_expression_from_vertex(self_obj)) { + assign_inferred_type(self_obj, calc_declared_type_before_smart_cast(self_obj)); flow.register_known_type(s_expr, param_type); } } } + // for static generic method call `Container.create` of fun_ref = `Container.create`, gather T=int + if (!self_obj && fun_ref->receiver_type && fun_ref->receiver_type->has_genericT_inside() && callee->kind == ast_dot_access && callee->as()->get_obj()->kind == ast_reference) { + const auto* t_static = callee->as()->get_obj()->as()->sym->try_as(); + tolk_assert(t_static); + deducingTs.auto_deduce_from_argument(cur_f, v->loc, fun_ref->receiver_type, t_static->resolved_type); + } + + // loop over every argument, one by one, like control flow goes for (int i = 0; i < v->get_num_args(); ++i) { const LocalVarData& param_i = fun_ref->parameters[delta_self + i]; AnyExprV arg_i = v->get_arg(i)->get_expr(); TypePtr param_type = param_i.declared_type; - if (param_type->has_genericT_inside()) { + if (param_type->has_genericT_inside()) { // `fun f(a:T, b:T)` T was fixated on `a`, use it as hint for `b` param_type = deducingTs.replace_Ts_with_currently_deduced(param_type); } flow = infer_any_expr(arg_i, std::move(flow), false, param_type).out_flow; @@ -1120,7 +1154,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (fun_ref->is_generic_function()) { // if `f(args)` was called, Ts were inferred; check that all of them are known std::string_view nameT_unknown = deducingTs.get_first_not_deduced_nameT(); - if (!nameT_unknown.empty() && hint && fun_ref->declared_return_type && fun_ref->declared_return_type->has_genericT_inside()) { + if (!nameT_unknown.empty() && hint && fun_ref->declared_return_type) { // example: `t.tupleFirst()`, T doesn't depend on arguments, but is determined by return type // if used like `var x: int = t.tupleFirst()` / `t.tupleFirst() as int` / etc., use hint deducingTs.auto_deduce_from_argument(cur_f, v->loc, fun_ref->declared_return_type, hint); @@ -1132,20 +1166,15 @@ class InferTypesAndCallsAndFieldsVisitor final { fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, deducingTs.flush()); } - v->mutate()->assign_fun_ref(fun_ref); - // since for `t.tupleAt()`, infer_dot_access() not called for callee = "t.tupleAt", assign its target here - if (v->is_dot_call()) { - v->get_callee()->as()->mutate()->assign_target(fun_ref); - } // get return type either from user-specified declaration or infer here on demand traversing its body + v->mutate()->assign_fun_ref(fun_ref, self_obj != nullptr); get_or_infer_return_type(fun_ref); - TypePtr inferred_type = dot_obj && fun_ref->does_return_self() ? dot_obj->inferred_type : fun_ref->inferred_return_type; + TypePtr inferred_type = fun_ref->does_return_self() ? self_obj->inferred_type : fun_ref->inferred_return_type; assign_inferred_type(v, inferred_type); assign_inferred_type(callee, fun_ref->inferred_full_type); if (inferred_type == TypeDataNever::create()) { flow.mark_unreachable(UnreachableKind::CallNeverReturnFunction); } - // note, that mutate params don't affect typing, they are handled when converting to IR return ExprFlow(std::move(flow), used_as_condition); } @@ -1229,8 +1258,8 @@ class InferTypesAndCallsAndFieldsVisitor final { } if (branches_unifier.is_union_of_different_types()) { // same as in ternary: `match (...) { t1 => someSlice, t2 => someInt }` is `int|slice`, probably unexpected - if (hint == nullptr || hint == TypeDataUnknown::create()) { - fire(cur_f, v->loc, "type of `match` was inferred as " + to_string(branches_unifier.get_result()) + "; probably, it's not what you expected; assign it to a variable `var v: = match (...) { ... }` manually"); + if (hint == nullptr || hint == TypeDataUnknown::create() || hint->has_genericT_inside()) { + fire(cur_f, v->loc, "type of `match` was inferred as " + to_string(branches_unifier.get_result()) + "; probably, it's not what you expected\nassign it to a variable `var v: = match (...) { ... }` manually"); } } assign_inferred_type(v, branches_unifier.get_result()); @@ -1281,7 +1310,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } } if (!struct_ref) { - fire(cur_f, v->loc, "can not detect struct name; use either `var v: StructName = { ... }` or `var v = StructName { ... }`"); + fire(cur_f, v->loc, "can not detect struct name\nuse either `var v: StructName = { ... }` or `var v = StructName { ... }`"); } // so, we have struct_ref, so we can check field names and infer values @@ -1308,12 +1337,9 @@ class InferTypesAndCallsAndFieldsVisitor final { if (field_type->has_genericT_inside()) { field_type = deducingTs.replace_Ts_with_currently_deduced(field_type); } - if (field_type->has_genericT_inside()) { // `item: v` where field `item` is generic: use `v` to infer T - flow = infer_any_expr(val_i, std::move(flow), false).out_flow; + flow = infer_any_expr(val_i, std::move(flow), false, field_type).out_flow; + if (field_type->has_genericT_inside()) { deducingTs.auto_deduce_from_argument(cur_f, field_i->loc, field_type, val_i->inferred_type); - } else { - // field_type is hint, helps infer val_i - flow = infer_any_expr(val_i, std::move(flow), false, field_type).out_flow; } assign_inferred_type(field_i, val_i); } @@ -1538,7 +1564,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } if (return_unifier.is_union_of_different_types()) { // `return intVar` + `return sliceVar` results in `int | slice`, probably unexpected - fire(fun_ref, v_function->get_body()->loc, "function " + to_string(fun_ref) + " calculated return type is " + to_string(inferred_return_type) + "; probably, it's not what you expected; declare `fun (...): ` manually"); + fire(fun_ref, v_function->get_body()->loc, "function " + to_string(fun_ref) + " calculated return type is " + to_string(inferred_return_type) + "; probably, it's not what you expected\ndeclare `fun (...): ` manually"); } } @@ -1598,7 +1624,7 @@ static void infer_and_save_return_type_of_function(FunctionPtr fun_ref) { // prevent recursion of untyped functions, like `fun f() { return g(); } fun g() { return f(); }` bool contains = std::find(called_stack.begin(), called_stack.end(), fun_ref) != called_stack.end(); if (contains) { - fire(fun_ref, fun_ref->loc, "could not infer return type of " + to_string(fun_ref) + ", because it appears in a recursive call chain; specify `: ` manually"); + fire(fun_ref, fun_ref->loc, "could not infer return type of " + to_string(fun_ref) + ", because it appears in a recursive call chain\ndeclare `fun (...): ` manually"); } // dig into g's body; it's safe, since the compiler is single-threaded diff --git a/tolk/pipe-refine-lvalue-for-mutate.cpp b/tolk/pipe-refine-lvalue-for-mutate.cpp index 5187457b2a..7692b0a2b2 100644 --- a/tolk/pipe-refine-lvalue-for-mutate.cpp +++ b/tolk/pipe-refine-lvalue-for-mutate.cpp @@ -34,30 +34,22 @@ namespace tolk { GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_invalid_mutate_arg_passed(AnyExprV v, FunctionPtr fun_ref, const LocalVarData& p_sym, bool called_as_method, bool arg_passed_as_mutate, AnyV arg_expr) { +static void fire_error_invalid_mutate_arg_passed(FunctionPtr cur_f, SrcLocation loc, FunctionPtr fun_ref, const LocalVarData& p_sym, bool arg_passed_as_mutate, AnyV arg_expr) { std::string arg_str(arg_expr->kind == ast_reference ? arg_expr->as()->get_name() : "obj"); - // case: `loadInt(cs, 32)`; suggest: `cs.loadInt(32)` - if (p_sym.is_mutate_parameter() && !arg_passed_as_mutate && !called_as_method && p_sym.param_idx == 0 && fun_ref->does_accept_self()) { - v->error("`" + fun_ref->name + "` is a mutating method; consider calling `" + arg_str + "." + fun_ref->name + "()`, not `" + fun_ref->name + "(" + arg_str + ")`"); - } - // case: `cs.mutating_function()`; suggest: `mutating_function(mutate cs)` or make it a method - if (p_sym.is_mutate_parameter() && called_as_method && p_sym.param_idx == 0 && !fun_ref->does_accept_self()) { - v->error("function `" + fun_ref->name + "` mutates parameter `" + p_sym.name + "`; consider calling `" + fun_ref->name + "(mutate " + arg_str + ")`, not `" + arg_str + "." + fun_ref->name + "`(); alternatively, rename parameter to `self` to make it a method"); - } - // case: `mutating_function(arg)`; suggest: `mutate arg` if (p_sym.is_mutate_parameter() && !arg_passed_as_mutate) { - v->error("function `" + fun_ref->name + "` mutates parameter `" + p_sym.name + "`; you need to specify `mutate` when passing an argument, like `mutate " + arg_str + "`"); + // called `mutating_function(arg)`; suggest: `mutate arg` + fire(cur_f, loc, "function `" + fun_ref->as_human_readable() + "` mutates parameter `" + p_sym.name + "`\nyou need to specify `mutate` when passing an argument, like `mutate " + arg_str + "`"); + } else { + // called `usual_function(mutate arg)` + fire(cur_f, loc, "incorrect `mutate`, since `" + fun_ref->as_human_readable() + "` does not mutate this parameter"); } - // case: `usual_function(mutate arg)` - if (!p_sym.is_mutate_parameter() && arg_passed_as_mutate) { - v->error("incorrect `mutate`, since `" + fun_ref->name + "` does not mutate this parameter"); - } - throw Fatal("unreachable"); } class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; + void visit(V v) override { // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` FunctionPtr fun_ref = v->fun_maybe; @@ -66,41 +58,35 @@ class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBod for (int i = 0; i < v->get_num_args(); ++i) { auto v_arg = v->get_arg(i); if (v_arg->passed_as_mutate) { - v_arg->error("`mutate` used for non-mutate argument"); + v_arg->error("`mutate` used for non-mutate parameter"); } } return; } - int delta_self = v->is_dot_call(); + int delta_self = v->get_self_obj() != nullptr; tolk_assert(fun_ref->get_num_params() == delta_self + v->get_num_args()); - if (v->is_dot_call()) { - if (fun_ref->does_mutate_self()) { - // for `b.storeInt()`, `b` should become lvalue, since `storeInt` is a method mutating self - // but: `beginCell().storeInt()`, then `beginCell()` is not lvalue - // (it will be extracted as tmp var when transforming AST to IR) - AnyExprV leftmost_obj = v->get_dot_obj(); - while (true) { - if (auto as_par = leftmost_obj->try_as()) { - leftmost_obj = as_par->get_expr(); - } else if (auto as_cast = leftmost_obj->try_as()) { - leftmost_obj = as_cast->get_expr(); - } else if (auto as_nn = leftmost_obj->try_as()) { - leftmost_obj = as_nn->get_expr(); - } else { - break; - } - } - bool will_be_extracted_as_tmp_var = leftmost_obj->kind == ast_function_call; - if (!will_be_extracted_as_tmp_var) { - leftmost_obj->mutate()->assign_lvalue_true(); - v->get_dot_obj()->mutate()->assign_lvalue_true(); + if (delta_self && fun_ref->does_mutate_self()) { + // for `b.storeInt()`, `b` should become lvalue, since `storeInt` is a method mutating self + // but: `beginCell().storeInt()`, then `beginCell()` is not lvalue + // (it will be extracted as tmp var when transforming AST to IR) + AnyExprV leftmost_obj = v->get_self_obj(); + while (true) { + if (auto as_par = leftmost_obj->try_as()) { + leftmost_obj = as_par->get_expr(); + } else if (auto as_cast = leftmost_obj->try_as()) { + leftmost_obj = as_cast->get_expr(); + } else if (auto as_nn = leftmost_obj->try_as()) { + leftmost_obj = as_nn->get_expr(); + } else { + break; } } - - if (!fun_ref->does_accept_self() && fun_ref->parameters[0].is_mutate_parameter()) { - fire_error_invalid_mutate_arg_passed(v, fun_ref, fun_ref->parameters[0], true, false, v->get_dot_obj()); + bool will_be_extracted_as_tmp_var = leftmost_obj->kind == ast_function_call; + if (!will_be_extracted_as_tmp_var) { + leftmost_obj->mutate()->assign_lvalue_true(); + v->get_self_obj()->mutate()->assign_lvalue_true(); } } @@ -108,7 +94,7 @@ class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBod const LocalVarData& p_sym = fun_ref->parameters[delta_self + i]; auto arg_i = v->get_arg(i); if (p_sym.is_mutate_parameter() != arg_i->passed_as_mutate) { - fire_error_invalid_mutate_arg_passed(arg_i, fun_ref, p_sym, false, arg_i->passed_as_mutate, arg_i->get_expr()); + fire_error_invalid_mutate_arg_passed(cur_f, arg_i->loc, fun_ref, p_sym, arg_i->passed_as_mutate, arg_i->get_expr()); } parent::visit(arg_i); } @@ -119,6 +105,11 @@ class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBod bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + } }; void pipeline_refine_lvalue_for_mutate_arguments() { diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 29754580bb..69e24c1a8a 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -35,7 +35,7 @@ namespace tolk { -static int calculate_method_id_for_entrypoint(std::string_view func_name) { +static int calculate_tvm_method_id_for_entrypoint(std::string_view func_name) { if (func_name == "main" || func_name == "onInternalMessage") { return 0; } @@ -54,7 +54,7 @@ static int calculate_method_id_for_entrypoint(std::string_view func_name) { tolk_assert(false); } -static int calculate_method_id_by_func_name(std::string_view func_name) { +static int calculate_tvm_method_id_by_func_name(std::string_view func_name) { unsigned int crc = td::crc16(static_cast(func_name)); return static_cast(crc & 0xffff) | 0x10000; } @@ -92,21 +92,10 @@ static void validate_arg_ret_order_of_asm_function(V v_body, int n } static const GenericsDeclaration* construct_genericTs(V v_list) { - std::vector itemsT; - itemsT.reserve(v_list->size()); - - for (int i = 0; i < v_list->size(); ++i) { - auto v_item = v_list->get_item(i); - auto it_existing = std::find_if(itemsT.begin(), itemsT.end(), [v_item](const GenericsDeclaration::GenericsItem& prev) { - return prev.nameT == v_item->nameT; - }); - if (it_existing != itemsT.end()) { - v_item->error("duplicate generic parameter `" + static_cast(v_item->nameT) + "`"); - } - itemsT.emplace_back(v_item->nameT); - } - - return new GenericsDeclaration(std::move(itemsT)); + std::vector namesT; + namesT.reserve(v_list->size()); + GenericsDeclaration::append_ast_list_checking_duplicates(v_list, namesT); + return new GenericsDeclaration(std::move(namesT), 0); } static GlobalConstPtr register_constant(V v) { @@ -127,13 +116,17 @@ static GlobalVarPtr register_global_var(V v) { return g_sym; } -static AliasDefPtr register_type_alias(V v, AliasDefPtr base_alias_ref = nullptr, const GenericsSubstitutions* substitutedTs = nullptr) { +static AliasDefPtr register_type_alias(V v, AliasDefPtr base_alias_ref = nullptr, std::string override_name = {}, const GenericsSubstitutions* substitutedTs = nullptr) { const GenericsDeclaration* genericTs = nullptr; if (v->genericsT_list && !substitutedTs) { genericTs = construct_genericTs(v->genericsT_list); } - AliasDefData* a_sym = new AliasDefData(static_cast(v->get_identifier()->name), v->loc, v->underlying_type_node, genericTs, substitutedTs, v); + std::string name = std::move(override_name); + if (name.empty()) { + name = v->get_identifier()->name; + } + AliasDefData* a_sym = new AliasDefData(std::move(name), v->loc, v->underlying_type_node, genericTs, substitutedTs, v); a_sym->base_alias_ref = base_alias_ref; // for `Response`, here is `Response` G.symtable.add_type_alias(a_sym); @@ -141,7 +134,7 @@ static AliasDefPtr register_type_alias(V v, AliasDef return a_sym; } -static StructPtr register_struct(V v, StructPtr base_struct_ref = nullptr, const GenericsSubstitutions* substitutedTs = nullptr) { +static StructPtr register_struct(V v, StructPtr base_struct_ref = nullptr, std::string override_name = {}, const GenericsSubstitutions* substitutedTs = nullptr) { auto v_body = v->get_struct_body(); std::vector fields; @@ -164,7 +157,11 @@ static StructPtr register_struct(V v, StructPtr base_str genericTs = construct_genericTs(v->genericsT_list); } - StructData* s_sym = new StructData(static_cast(v->get_identifier()->name), v->loc, std::move(fields), genericTs, substitutedTs, v); + std::string name = std::move(override_name); + if (name.empty()) { + name = v->get_identifier()->name; + } + StructData* s_sym = new StructData(std::move(name), v->loc, std::move(fields), genericTs, substitutedTs, v); s_sym->base_struct_ref = base_struct_ref; // for `Container`, here is `Container` G.symtable.add_struct(s_sym); @@ -188,8 +185,12 @@ static LocalVarData register_parameter(V v, int idx) { return LocalVarData(static_cast(v->param_name), v->loc, v->type_node, flags, idx); } -static FunctionPtr register_function(V v, FunctionPtr base_fun_ref = nullptr, const GenericsSubstitutions* substitutedTs = nullptr) { - std::string_view func_name = v->get_identifier()->name; +static FunctionPtr register_function(V v, FunctionPtr base_fun_ref = nullptr, std::string override_name = {}, const GenericsSubstitutions* substitutedTs = nullptr) { + if (v->is_builtin_function()) { + return nullptr; + } + + std::string_view f_identifier = v->get_identifier()->name; // function or method name std::vector parameters; int n_mutate_params = 0; @@ -200,23 +201,22 @@ static FunctionPtr register_function(V v, FunctionPtr n_mutate_params += static_cast(v_param->declared_as_mutate); } - if (v->is_builtin_function()) { - const Symbol* sym = lookup_global_symbol(func_name); - FunctionPtr fun_ref = sym ? sym->try_as() : nullptr; - if (!fun_ref || !fun_ref->is_builtin_function()) { - v->error("`builtin` used for non-builtin function"); - } - v->mutate()->assign_fun_ref(fun_ref); - return fun_ref; - } - const GenericsDeclaration* genericTs = nullptr; if (v->genericsT_list && !substitutedTs) { genericTs = construct_genericTs(v->genericsT_list); } + std::string method_name; + if (v->receiver_type_node) { + method_name = f_identifier; + } + std::string name = std::move(override_name); + if (name.empty()) { + name = f_identifier; + } + FunctionBody f_body = v->get_body()->kind == ast_block_statement ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); - FunctionData* f_sym = new FunctionData(static_cast(func_name), v->loc, v->return_type_node, std::move(parameters), 0, genericTs, substitutedTs, f_body, v); + FunctionData* f_sym = new FunctionData(std::move(name), v->loc, std::move(method_name), v->receiver_type_node, v->return_type_node, std::move(parameters), 0, genericTs, substitutedTs, f_body, v); f_sym->base_fun_ref = base_fun_ref; // for `f`, here is `f` if (auto v_asm = v->get_body()->try_as()) { @@ -228,27 +228,31 @@ static FunctionPtr register_function(V v, FunctionPtr f_sym->ret_order = v_asm->ret_order; } - if (v->method_id.not_null()) { - f_sym->method_id = static_cast(v->method_id->to_long()); - } else if (v->flags & FunctionData::flagGetMethod) { - f_sym->method_id = calculate_method_id_by_func_name(func_name); - for (FunctionPtr other : G.all_get_methods) { - if (other->method_id == f_sym->method_id) { + if (v->tvm_method_id.not_null()) { + f_sym->tvm_method_id = static_cast(v->tvm_method_id->to_long()); + } else if (v->flags & FunctionData::flagContractGetter) { + f_sym->tvm_method_id = calculate_tvm_method_id_by_func_name(f_identifier); + for (FunctionPtr other : G.all_contract_getters) { + if (other->tvm_method_id == f_sym->tvm_method_id) { v->error(PSTRING() << "GET methods hash collision: `" << other->name << "` and `" << f_sym->name << "` produce the same hash. Consider renaming one of these functions."); } } } else if (v->flags & FunctionData::flagIsEntrypoint) { - f_sym->method_id = calculate_method_id_for_entrypoint(func_name); + f_sym->tvm_method_id = calculate_tvm_method_id_for_entrypoint(f_identifier); } f_sym->flags |= v->flags; if (n_mutate_params) { f_sym->flags |= FunctionData::flagHasMutateParams; } - G.symtable.add_function(f_sym); + if (!f_sym->receiver_type_node) { + G.symtable.add_function(f_sym); + } else if (!substitutedTs) { + G.all_methods.push_back(f_sym); + } G.all_functions.push_back(f_sym); - if (f_sym->is_get_method()) { - G.all_get_methods.push_back(f_sym); + if (f_sym->is_contract_getter()) { + G.all_contract_getters.push_back(f_sym); } v->mutate()->assign_fun_ref(f_sym); return f_sym; @@ -296,19 +300,19 @@ void pipeline_register_global_symbols() { } } -FunctionPtr pipeline_register_instantiated_generic_function(FunctionPtr base_fun_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs) { +FunctionPtr pipeline_register_instantiated_generic_function(FunctionPtr base_fun_ref, AnyV cloned_v, std::string&& name, const GenericsSubstitutions* substitutedTs) { auto v = cloned_v->as(); - return register_function(v, base_fun_ref, substitutedTs); + return register_function(v, base_fun_ref, std::move(name), substitutedTs); } -StructPtr pipeline_register_instantiated_generic_struct(StructPtr base_struct_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs) { +StructPtr pipeline_register_instantiated_generic_struct(StructPtr base_struct_ref, AnyV cloned_v, std::string&& name, const GenericsSubstitutions* substitutedTs) { auto v = cloned_v->as(); - return register_struct(v, base_struct_ref, substitutedTs); + return register_struct(v, base_struct_ref, std::move(name), substitutedTs); } -AliasDefPtr pipeline_register_instantiated_generic_alias(AliasDefPtr base_alias_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs) { +AliasDefPtr pipeline_register_instantiated_generic_alias(AliasDefPtr base_alias_ref, AnyV cloned_v, std::string&& name, const GenericsSubstitutions* substitutedTs) { auto v = cloned_v->as(); - return register_type_alias(v, base_alias_ref, substitutedTs); + return register_type_alias(v, base_alias_ref, std::move(name), substitutedTs); } } // namespace tolk diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index d4cda33fb9..4452ea5738 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -49,9 +49,18 @@ namespace tolk { GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_undefined_symbol(FunctionPtr cur_f, V v) { if (v->name == "self") { - throw ParseError(cur_f, v->loc, "using `self` in a non-member function (it does not accept the first `self` parameter)"); + fire(cur_f, v->loc, "using `self` in a non-member function (it does not accept the first `self` parameter)"); } else { - throw ParseError(cur_f, v->loc, "undefined symbol `" + static_cast(v->name) + "`"); + fire(cur_f, v->loc, "undefined symbol `" + static_cast(v->name) + "`"); + } +} + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_type_used_as_symbol(FunctionPtr cur_f, V v) { + if (v->name == "random") { // calling `random()`, but it's a struct, correct is `random.uint256()` + fire(cur_f, v->loc, "`random` is not a function, you probably want `random.uint256()`"); + } else { + fire(cur_f, v->loc, "`" + static_cast(v->name) + "` only refers to a type, but is being used as a value here"); } } @@ -164,6 +173,9 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { if (!sym) { fire_error_undefined_symbol(cur_f, v->get_identifier()); } + if (sym->try_as() || sym->try_as()) { + fire_error_type_used_as_symbol(cur_f, v->get_identifier()); + } v->mutate()->assign_sym(sym); // for global functions, global vars and constants, `import` must exist @@ -172,6 +184,19 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { } } + void visit(V v) override { + try { + parent::visit(v->get_obj()); + } catch (const ParseError&) { + if (v->get_obj()->kind == ast_reference) { + // for `Point.create` / `int.zero`, "undefined symbol" is fired for Point/int + // suppress this exception till a later pipe, it will be tried to be resolved as a type + return; + } + throw; + } + } + void visit(V v) override { current_scope.open_scope(v->loc); parent::visit(v->get_block_statement()); @@ -248,7 +273,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { auto v_block = v->get_body()->as(); current_scope.open_scope(v->loc); - for (int i = 0; i < v->get_num_params(); ++i) { + for (int i = 0; i < fun_ref->get_num_params(); ++i) { current_scope.add_local_var(&fun_ref->parameters[i]); } parent::visit(v_block); @@ -280,7 +305,7 @@ void pipeline_resolve_identifiers_and_assign_symbols() { AssignSymInsideFunctionVisitor visitor; for (const SrcFile* file : G.all_src_files) { for (AnyV v : file->ast->as()->get_toplevel_declarations()) { - if (auto v_func = v->try_as()) { + if (auto v_func = v->try_as(); v_func && !v_func->is_builtin_function()) { tolk_assert(v_func->fun_ref); if (visitor.should_visit_function(v_func->fun_ref)) { visitor.start_visiting_function(v_func->fun_ref, v_func); diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 8f805f741e..084a48c91b 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -143,10 +143,33 @@ static TypePtr try_parse_predefined_type(std::string_view str) { return nullptr; } +static const GenericsDeclaration* combine_genericTs_from_receiver_and_declaration(TypePtr receiver_type, V v_list) { + std::vector namesT; + receiver_type->replace_children_custom([&namesT](TypePtr child) { + if (const TypeDataGenericT* asT = child->try_as()) { + auto it_existing = std::find_if(namesT.begin(), namesT.end(), [asT](const std::string_view& prevT) { + return prevT == asT->nameT; + }); + if (it_existing == namesT.end()) { + namesT.emplace_back(asT->nameT); + } + } + return child; + }); + int n_from_receiver = static_cast(namesT.size()); + + if (v_list) { + GenericsDeclaration::append_ast_list_checking_duplicates(v_list, namesT); + } + + return new GenericsDeclaration(std::move(namesT), n_from_receiver); +} + class TypeNodesVisitorResolver { FunctionPtr cur_f; // exists if we're inside its body const GenericsDeclaration* genericTs; // `` if we're inside `f` or `f` const GenericsSubstitutions* substitutedTs; // `T=int` if we're inside `f` + bool treat_unresolved_as_genericT; // used for receivers `fun Container.create()`, T becomes generic TypePtr parse_ast_type_node(AnyTypeV v, bool allow_without_type_arguments) { switch (v->kind) { @@ -169,6 +192,9 @@ class TypeNodesVisitorResolver { if (TypePtr predefined_type = try_parse_predefined_type(text)) { return predefined_type; } + if (treat_unresolved_as_genericT) { + return TypeDataGenericT::create(static_cast(text)); + } fire_error_unknown_type_name(cur_f, loc, text); } @@ -283,6 +309,9 @@ class TypeNodesVisitorResolver { } return TypeDataAlias::create(instantiate_generic_alias(alias_ref, GenericsSubstitutions(alias_ref->genericTs, type_arguments))); } + if (const TypeDataGenericT* asT = type_to_instantiate->try_as()) { + fire_error_unknown_type_name(cur_f, loc, asT->nameT); + } // `User` / `cell` fire(cur_f, loc, "type `" + type_to_instantiate->as_human_readable() + "` is not generic"); } @@ -297,10 +326,11 @@ class TypeNodesVisitorResolver { public: - TypeNodesVisitorResolver(FunctionPtr cur_f, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs) + TypeNodesVisitorResolver(FunctionPtr cur_f, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, bool treat_unresolved_as_genericT) : cur_f(cur_f) , genericTs(genericTs) - , substitutedTs(substitutedTs) {} + , substitutedTs(substitutedTs) + , treat_unresolved_as_genericT(treat_unresolved_as_genericT) {} TypePtr finalize_type_node(AnyTypeV type_node, bool allow_without_type_arguments = false) { #ifdef TOLK_DEBUG @@ -321,7 +351,7 @@ class TypeNodesVisitorResolver { } static void visit_symbol(GlobalVarPtr glob_ref) { - TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr); + TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr, false); TypePtr declared_type = visitor.finalize_type_node(glob_ref->type_node); glob_ref->mutate()->assign_resolved_type(declared_type); } @@ -331,7 +361,7 @@ class TypeNodesVisitorResolver { return; } - TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr); + TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr, false); TypePtr declared_type = visitor.finalize_type_node(const_ref->type_node); const_ref->mutate()->assign_resolved_type(declared_type); } @@ -346,7 +376,7 @@ class TypeNodesVisitorResolver { } called_stack.push_back(alias_ref); - TypeNodesVisitorResolver visitor(nullptr, alias_ref->genericTs, alias_ref->substitutedTs); + TypeNodesVisitorResolver visitor(nullptr, alias_ref->genericTs, alias_ref->substitutedTs, false); TypePtr underlying_type = visitor.finalize_type_node(alias_ref->underlying_type_node); alias_ref->mutate()->assign_resolved_type(underlying_type); alias_ref->mutate()->assign_visited_by_resolver(); @@ -365,7 +395,7 @@ class TypeNodesVisitorResolver { } called_stack.push_back(struct_ref); - TypeNodesVisitorResolver visitor(nullptr, struct_ref->genericTs, struct_ref->substitutedTs); + TypeNodesVisitorResolver visitor(nullptr, struct_ref->genericTs, struct_ref->substitutedTs, false); for (int i = 0; i < struct_ref->get_num_fields(); ++i) { StructFieldPtr field_ref = struct_ref->get_field(i); TypePtr declared_type = visitor.finalize_type_node(field_ref->type_node); @@ -377,7 +407,7 @@ class TypeNodesVisitorResolver { }; class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { - TypeNodesVisitorResolver type_nodes_visitor{nullptr, nullptr, nullptr}; + TypeNodesVisitorResolver type_nodes_visitor{nullptr, nullptr, nullptr, false}; TypePtr finalize_type_node(AnyTypeV type_node, bool allow_without_type_arguments = false) { return type_nodes_visitor.finalize_type_node(type_node, allow_without_type_arguments); @@ -415,6 +445,27 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { } void visit(V v) override { + // for static method calls, like "int.zero()" or "Point.create()", dot obj symbol is unresolved for now + // so, resolve it as a type and store as a "type reference symbol" + if (auto obj_ref = v->get_obj()->try_as()) { + if (obj_ref->sym == nullptr) { + std::string_view obj_type_name = obj_ref->get_identifier()->name; + AnyTypeV obj_type_node = createV(obj_ref->loc, obj_type_name); + if (obj_ref->has_instantiationTs()) { // Container.create + std::vector inner_and_args; + inner_and_args.reserve(1 + obj_ref->get_instantiationTs()->size()); + inner_and_args.push_back(obj_type_node); + for (int i = 0; i < obj_ref->get_instantiationTs()->size(); ++i) { + inner_and_args.push_back(obj_ref->get_instantiationTs()->get_item(i)->type_node); + } + obj_type_node = createV(obj_ref->loc, std::move(inner_and_args)); + } + TypePtr type_as_reference = finalize_type_node(obj_type_node); + const Symbol* type_as_symbol = new TypeReferenceUsedAsSymbol(static_cast(obj_type_name), obj_ref->loc, type_as_reference); + obj_ref->mutate()->assign_sym(type_as_symbol); + } + } + // for `t.tupleAt` / `obj.method`, resolve "MyAlias" and "T" // (for function call `t.tupleAt()`, this v (ast_dot_access `t.tupleAt`) is callee) if (auto v_instantiationTs = v->get_instantiationTs()) { @@ -448,9 +499,25 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { } void start_visiting_function(FunctionPtr fun_ref, V v) override { - type_nodes_visitor = TypeNodesVisitorResolver(fun_ref, fun_ref->genericTs, fun_ref->substitutedTs); + if (fun_ref->receiver_type_node) { + TypeNodesVisitorResolver receiver_visitor(fun_ref, fun_ref->genericTs, fun_ref->substitutedTs, true); + TypePtr receiver_type = receiver_visitor.finalize_type_node(fun_ref->receiver_type_node); + const GenericsDeclaration* genericTs = fun_ref->genericTs; + if (receiver_type->has_genericT_inside()) { + genericTs = combine_genericTs_from_receiver_and_declaration(receiver_type, v->genericsT_list); + } + std::string name_prefix = receiver_type->as_human_readable(); + bool embrace = receiver_type->try_as() && !receiver_type->try_as()->or_null; + if (embrace) { + name_prefix = "(" + name_prefix + ")"; + } + fun_ref->mutate()->assign_resolved_receiver_type(receiver_type, genericTs, std::move(name_prefix)); + G.symtable.add_function(fun_ref); + } - for (int i = 0; i < v->get_num_params(); ++i) { + type_nodes_visitor = TypeNodesVisitorResolver(fun_ref, fun_ref->genericTs, fun_ref->substitutedTs, false); + + for (int i = 0; i < fun_ref->get_num_params(); ++i) { const LocalVarData& param_var = fun_ref->parameters[i]; TypePtr declared_type = finalize_type_node(param_var.type_node); param_var.mutate()->assign_resolved_type(declared_type); @@ -466,7 +533,7 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { } void start_visiting_constant(GlobalConstPtr const_ref) { - type_nodes_visitor = TypeNodesVisitorResolver(nullptr, nullptr, nullptr); + type_nodes_visitor = TypeNodesVisitorResolver(nullptr, nullptr, nullptr, false); // `const a = 0 as int8`, resolve types there // same for struct field `v: int8 = 0 as int8` @@ -474,7 +541,7 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { } void start_visiting_struct_fields(StructPtr struct_ref) { - type_nodes_visitor = TypeNodesVisitorResolver(nullptr, struct_ref->genericTs, struct_ref->substitutedTs); + type_nodes_visitor = TypeNodesVisitorResolver(nullptr, struct_ref->genericTs, struct_ref->substitutedTs, false); // same for struct field `v: int8 = 0 as int8` for (StructFieldPtr field_ref : struct_ref->fields) { @@ -490,7 +557,7 @@ void pipeline_resolve_types_and_aliases() { for (const SrcFile* file : G.all_src_files) { for (AnyV v : file->ast->as()->get_toplevel_declarations()) { - if (auto v_func = v->try_as()) { + if (auto v_func = v->try_as(); v_func && !v_func->is_builtin_function()) { tolk_assert(v_func->fun_ref); if (visitor.should_visit_function(v_func->fun_ref)) { visitor.start_visiting_function(v_func->fun_ref, v_func); @@ -518,6 +585,8 @@ void pipeline_resolve_types_and_aliases() { } } } + + patch_builtins_after_stdlib_loaded(); } void pipeline_resolve_types_and_aliases(FunctionPtr fun_ref) { diff --git a/tolk/pipeline.h b/tolk/pipeline.h index c0a51b096e..a46e3e0375 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -50,7 +50,7 @@ void pipeline_generate_fif_output_to_std_cout(); // these pipes also can be called per-function individually // they are called for instantiated generics functions, when `f` is deeply cloned as `f` -FunctionPtr pipeline_register_instantiated_generic_function(FunctionPtr base_fun_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs); +FunctionPtr pipeline_register_instantiated_generic_function(FunctionPtr base_fun_ref, AnyV cloned_v, std::string&& name, const GenericsSubstitutions* substitutedTs); void pipeline_resolve_identifiers_and_assign_symbols(FunctionPtr); void pipeline_resolve_types_and_aliases(FunctionPtr); @@ -58,11 +58,11 @@ void pipeline_calculate_rvalue_lvalue(FunctionPtr); void pipeline_detect_unreachable_statements(FunctionPtr); void pipeline_infer_types_and_calls_and_fields(FunctionPtr); -StructPtr pipeline_register_instantiated_generic_struct(StructPtr base_struct_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs); +StructPtr pipeline_register_instantiated_generic_struct(StructPtr base_struct_ref, AnyV cloned_v, std::string&& name, const GenericsSubstitutions* substitutedTs); void pipeline_resolve_identifiers_and_assign_symbols(StructPtr); void pipeline_resolve_types_and_aliases(StructPtr); -AliasDefPtr pipeline_register_instantiated_generic_alias(AliasDefPtr base_alias_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs); +AliasDefPtr pipeline_register_instantiated_generic_alias(AliasDefPtr base_alias_ref, AnyV cloned_v, std::string&& name, const GenericsSubstitutions* substitutedTs); void pipeline_resolve_types_and_aliases(AliasDefPtr); } // namespace tolk diff --git a/tolk/src-file.cpp b/tolk/src-file.cpp index d976085609..ab54ce7b0c 100644 --- a/tolk/src-file.cpp +++ b/tolk/src-file.cpp @@ -226,7 +226,27 @@ std::ostream& operator<<(std::ostream& os, const ParseError& error) { } void ParseError::show(std::ostream& os) const { - os << loc << ": error: " << message << std::endl; + if (message.find('\n') == std::string::npos) { + // just print a single-line message + os << loc << ": error: " << message << std::endl; + } else { + // print "location: line1 \n (spaces) line2 \n ..." + std::string_view message = this->message; + std::string loc_text = loc.to_string(); + std::string loc_spaces(std::min(static_cast(loc_text.size()), 30), ' '); + size_t start = 0, end; + os << loc_text << ": error: "; + while ((end = message.find('\n', start)) != std::string::npos) { + if (start > 0) { + os << loc_spaces << " "; + } + os << message.substr(start, end - start) << std::endl; + start = end + 1; + } + if (start < message.size()) { + os << loc_spaces << " " << message.substr(start) << std::endl; + } + } if (current_function) { os << " // in function `" << current_function->as_human_readable() << "`" << std::endl; } diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 2449de59df..2e9fbabe5a 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -66,6 +66,15 @@ bool FunctionData::does_need_codegen() const { return true; } +void FunctionData::assign_resolved_receiver_type(TypePtr receiver_type, const GenericsDeclaration* genericTs, std::string&& name_prefix) { + this->receiver_type = receiver_type; + if (!this->substitutedTs) { // after receiver has been resolve, update name to "receiver.method" + name_prefix.erase(std::remove(name_prefix.begin(), name_prefix.end(), ' '), name_prefix.end()); + this->name = name_prefix + "." + this->method_name; + this->genericTs = genericTs; + } +} + void FunctionData::assign_resolved_type(TypePtr declared_return_type) { this->declared_return_type = declared_return_type; } @@ -212,4 +221,8 @@ const Symbol* lookup_global_symbol(std::string_view name) { return G.symtable.lookup(name); } +FunctionPtr lookup_function(std::string_view name) { + return G.symtable.lookup(name)->try_as(); +} + } // namespace tolk diff --git a/tolk/symtable.h b/tolk/symtable.h index cde3a83974..9c80d19924 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -94,7 +94,7 @@ typedef std::variant< > FunctionBody; struct FunctionData final : Symbol { - static constexpr int EMPTY_METHOD_ID = -10; + static constexpr int EMPTY_TVM_METHOD_ID = -10; enum { flagInline = 1, // marked `@inline` @@ -103,7 +103,7 @@ struct FunctionData final : Symbol { flagUsedAsNonCall = 8, // used not only as `f()`, but as a 1-st class function (assigned to var, pushed to tuple, etc.) flagMarkedAsPure = 16, // declared as `pure`, can't call impure and access globals, unused invocations are optimized out flagImplicitReturn = 32, // control flow reaches end of function, so it needs implicit return at the end - flagGetMethod = 64, // was declared via `get func(): T`, method_id is auto-assigned + flagContractGetter = 64, // was declared via `get func(): T`, tvm_method_id is auto-assigned flagIsEntrypoint = 128, // it's `main` / `onExternalMessage` / etc. flagHasMutateParams = 256, // has parameters declared as `mutate` flagAcceptsSelf = 512, // is a member function (has `self` first parameter) @@ -112,9 +112,13 @@ struct FunctionData final : Symbol { flagCompileTimeOnly = 4096, // calculated only at compile-time for constant arguments: `ton("0.05")`, `stringCrc32`, and others }; - int method_id = EMPTY_METHOD_ID; + int tvm_method_id = EMPTY_TVM_METHOD_ID; int flags; + std::string method_name; // for `fun Container.store` here is "store" + AnyTypeV receiver_type_node; // for `fun Container.store` here is `Container` + TypePtr receiver_type = nullptr; // = resolved receiver_type_node + std::vector parameters; std::vector arg_order, ret_order; AnyTypeV return_type_node; // may be nullptr, meaning "auto infer" @@ -128,9 +132,11 @@ struct FunctionData final : Symbol { FunctionBody body; AnyV ast_root; // V for user-defined (not builtin) - FunctionData(std::string name, SrcLocation loc, AnyTypeV return_type_node, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) + FunctionData(std::string name, SrcLocation loc, std::string method_name, AnyTypeV receiver_type_node, AnyTypeV return_type_node, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) : Symbol(std::move(name), loc) , flags(initial_flags) + , method_name(std::move(method_name)) + , receiver_type_node(receiver_type_node) , parameters(std::move(parameters)) , return_type_node(return_type_node) , genericTs(genericTs) @@ -138,9 +144,12 @@ struct FunctionData final : Symbol { , body(body) , ast_root(ast_root) { } - FunctionData(std::string name, SrcLocation loc, TypePtr declared_return_type, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) + FunctionData(std::string name, SrcLocation loc, std::string method_name, TypePtr receiver_type, TypePtr declared_return_type, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) : Symbol(std::move(name), loc) , flags(initial_flags) + , method_name(std::move(method_name)) + , receiver_type_node(nullptr) + , receiver_type(receiver_type) , parameters(std::move(parameters)) , return_type_node(nullptr) // for built-in functions, defined in sources , declared_return_type(declared_return_type) @@ -165,6 +174,8 @@ struct FunctionData final : Symbol { bool is_code_function() const { return std::holds_alternative(body); } bool is_asm_function() const { return std::holds_alternative(body); } bool is_builtin_function() const { return ast_root == nullptr; } + bool is_method() const { return !method_name.empty(); } + bool is_static_method() const { return is_method() && !does_accept_self(); } bool is_generic_function() const { return genericTs != nullptr; } bool is_instantiation_of_generic_function() const { return substitutedTs != nullptr; } @@ -175,8 +186,8 @@ struct FunctionData final : Symbol { bool is_used_as_noncall() const { return flags & flagUsedAsNonCall; } bool is_marked_as_pure() const { return flags & flagMarkedAsPure; } bool is_implicit_return() const { return flags & flagImplicitReturn; } - bool is_get_method() const { return flags & flagGetMethod; } - bool is_method_id_not_empty() const { return method_id != EMPTY_METHOD_ID; } + bool is_contract_getter() const { return flags & flagContractGetter; } + bool has_tvm_method_id() const { return tvm_method_id != EMPTY_TVM_METHOD_ID; } bool is_entrypoint() const { return flags & flagIsEntrypoint; } bool has_mutate_params() const { return flags & flagHasMutateParams; } bool does_accept_self() const { return flags & flagAcceptsSelf; } @@ -188,6 +199,7 @@ struct FunctionData final : Symbol { bool does_need_codegen() const; FunctionData* mutate() const { return const_cast(this); } + void assign_resolved_receiver_type(TypePtr receiver_type, const GenericsDeclaration* genericTs, std::string&& name_prefix); void assign_resolved_type(TypePtr declared_return_type); void assign_inferred_type(TypePtr inferred_return_type, TypePtr inferred_full_type); void assign_is_used_as_noncall(); @@ -326,6 +338,15 @@ struct StructData final : Symbol { std::string as_human_readable() const; }; +struct TypeReferenceUsedAsSymbol final : Symbol { + TypePtr resolved_type; + + TypeReferenceUsedAsSymbol(std::string name, SrcLocation loc, TypePtr resolved_type) + : Symbol(std::move(name), loc) + , resolved_type(resolved_type) { + } +}; + class GlobalSymbolTable { std::unordered_map entries; @@ -347,5 +368,6 @@ class GlobalSymbolTable { }; const Symbol* lookup_global_symbol(std::string_view name); +FunctionPtr lookup_function(std::string_view name); } // namespace tolk diff --git a/tolk/tolk.h b/tolk/tolk.h index 9fb39dae27..55d9d7677c 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -1128,6 +1128,7 @@ AsmOp exec_arg2_op(SrcLocation loc, std::string op, long long imm1, long long im AsmOp push_const(SrcLocation loc, td::RefInt256 x); void define_builtins(); +void patch_builtins_after_stdlib_loaded(); From 76c9f9eaaf0679e2299167ab7744320c5a9ee5a3 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 28 Apr 2025 20:40:22 +0400 Subject: [PATCH 235/524] [Tolk] Support defaults for generic type arguments This is now valid: > struct Container { item: T = null } > Container{} // does Container Defaults for Ts work both for functions and structs. Their main purpose is to have the `never` default: > struct WithOptField { f: T; } I've added a rule that if a struct has the `never` field, it can be missed out from a literal while creation (like it doesn't exist at all). This will be used in stdlib later. --- tolk-tester/tests/generics-4.tolk | 181 ++++++++++++++++++ .../tests/invalid-semantics/err-4628.tolk | 2 +- .../tests/invalid-semantics/err-4629.tolk | 16 ++ .../tests/invalid-semantics/err-4907.tolk | 25 +++ .../tests/invalid-typing/err-6380.tolk | 14 ++ tolk-tester/tests/struct-tests.tolk | 13 ++ tolk/ast-from-tokens.cpp | 8 +- tolk/ast-replicator.h | 2 +- tolk/ast.h | 6 +- tolk/builtins.cpp | 2 +- tolk/generics-helpers.cpp | 58 ++++-- tolk/generics-helpers.h | 35 ++-- tolk/pipe-ast-to-legacy.cpp | 3 + tolk/pipe-infer-types-and-calls.cpp | 35 +++- tolk/pipe-register-symbols.cpp | 25 +-- tolk/pipe-resolve-types.cpp | 80 +++++--- tolk/symtable.cpp | 19 +- tolk/symtable.h | 5 +- tolk/type-system.cpp | 2 +- 19 files changed, 434 insertions(+), 97 deletions(-) create mode 100644 tolk-tester/tests/generics-4.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4629.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4907.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6380.tolk diff --git a/tolk-tester/tests/generics-4.tolk b/tolk-tester/tests/generics-4.tolk new file mode 100644 index 0000000000..30ce19e352 --- /dev/null +++ b/tolk-tester/tests/generics-4.tolk @@ -0,0 +1,181 @@ +fun eqUnusedT(v: int) { return v; } +fun eqUnusedU(v: T) { if (v is int123) { __expect_type(v as U, "never"); } return v; } + +fun dup(x: T1, y: T2): (T1, T2) { return (x, y) } + +struct Container1 { + item: T; +} + +fun getItemOf(v: Container1) { + return v.item; +} + +struct WithDef1 { + body: T? = null; +} + +fun getBodyOf(o: WithDef1) { + return o.body; +} + +struct WithNever { + f1: int; + f2: T; +} + +fun eqNever(o: WithNever) { return o; } + +struct MyInit { + value: coins; + data: TInit; +} + +fun MyInit.getValue(self) { + return self.value; +} + +struct Parameters { + bounce: bool; + body: TBody; + init: builder | MyInit | null = null; +} + +fun createParameters(bounce: bool, body: TBody, data: TInit): Parameters { + return { bounce, body, init: { value: ton("0"), data } }; +} + +fun mySend(p: Parameters): int { + var total = 0; + if (p.bounce) { + total += 1; + } + if (p.body !is never) { + assert(p.body is TBody, 101); + total += 10; + } + if (p.init is MyInit) { + total += 100 + p.init.getValue(); + } + return total; +} + +fun test1() { + eqUnusedT(100); + eqUnusedU(100); + eqUnusedU(beginCell()); + + __expect_type(dup(null, null), "(null, null)"); + __expect_type(dup(createEmptyTuple(), 6), "(tuple, int)"); +} + +fun test2() { + var w1 = Container1 { item: 123 }; + var w2 = Container1 { item: null }; + __expect_type(w1, "Container1"); + __expect_type(w2, "Container1"); + __expect_type(getItemOf(w1), "int"); + __expect_type(getItemOf(w2), "null"); + + __expect_type(getItemOf({item: null as slice?}), "slice?"); + __expect_type(getItemOf({item: null}), "null"); +} + +@method_id(103) +fun test3() { + __expect_type(WithNever{f1:10}, "WithNever"); + __expect_type(WithNever{f1:10,f2:20}, "WithNever"); + + __expect_type(eqNever({f1:10}), "WithNever"); + __expect_type(eqNever({f1:10,f2:20}), "WithNever"); + __expect_type(eqNever({f1:10,f2:null}), "WithNever"); + + var a: WithNever = {f1:10}; + return (a, WithNever{f1:20}, eqNever({f1:30}), 777, eqNever({f1:40,f2:40})); +} + +@method_id(104) +fun test4() { + __expect_type(getBodyOf({body: 123}), "int?"); + __expect_type(getBodyOf({body: null}), "null"); + __expect_type(getBodyOf({}), "null"); + + __expect_type(WithDef1{}, "WithDef1"); + __expect_type(WithDef1{}.body, "null"); + __expect_type(WithDef1{body: null}.body, "null"); + __expect_type(WithDef1{body: 123}.body, "int?"); + + return (getBodyOf({body: null}), getBodyOf({}), WithDef1{}); +} + +@method_id(105) +fun test5() { + __expect_type(Parameters { bounce: true }, "Parameters"); + __expect_type(Parameters { bounce: false, body: 179 }, "Parameters"); + __expect_type(Parameters { bounce: true, init: beginCell() }, "Parameters"); + __expect_type(Parameters { bounce: false, body: beginCell(), init: { value: 123, data: 123 } }, "Parameters"); + + __expect_type(createParameters(true, null, null), "Parameters"); + __expect_type(createParameters(true, beginCell(), "123"), "Parameters"); + + __expect_type(createParameters(true, null, null).body, "null"); + __expect_type(createParameters(true, 123, null).body, "int"); + __expect_type(createParameters(true, null, null).init, "builder | MyInit | null"); + __expect_type(createParameters(true, 123, 456).init, "builder | MyInit | null"); + + return (createParameters(true, null, null), 777, createParameters(false, 123, 456)); +} + +@method_id(106) +fun test6() { + var p: Parameters = { + bounce: true, + body: 123, + init: { value: ton("0"), data: beginCell().endCell() } + }; + return (p.body is int, p.init is cell, p.init is MyInit, p.init is MyInit && p.init.data.depth() == 0); +} + +@method_id(107) +fun test7() { + __expect_type(Parameters{ bounce: false, body: 123, init: { value: ton("0"), data: beginCell() }}, "Parameters"); + var v1 = mySend({ bounce: true }); + var v2 = mySend({ bounce: false, body: 123, init: beginCell() }); + var v3 = mySend({ bounce: false, body: 123, init: { value: ton("0"), data: beginCell() }}); + var v4 = mySend({ bounce: true, init: { value: 16, data: null as [int]? }}); + return (v1, v2, v3, v4); +} + +fun main() { +} + +/** +@testcase | 103 | | 10 20 30 777 40 40 +@testcase | 104 | | (null) (null) (null) +@testcase | 105 | | -1 (null) 0 (null) 133 777 0 123 0 456 132 +@testcase | 106 | | -1 0 -1 -1 +@testcase | 107 | | 1 10 110 117 + +@fif_codegen DECLPROC eqUnusedT +@fif_codegen DECLPROC eqUnusedU +@fif_codegen DECLPROC mySend +@fif_codegen DECLPROC mySend +@fif_codegen DECLPROC mySend +@fif_codegen DECLPROC mySend + +@fif_codegen +""" + test6 PROC:<{ // + NEWC // '8 + ENDC // p.init.USlot2 + -1 PUSHINT // p.init.USlot2 '11=-1 + FALSE // p.init.USlot2 '11=-1 '12 + TRUE // p.init.USlot2 '11=-1 '12 '14 + s0 s3 XCHG // '14 '11=-1 '12 p.init.USlot2 + CDEPTH // '14 '11=-1 '12 '19 + 0 EQINT // '14 '11=-1 '12 '21 + 0 NEQINT // '14 '11=-1 '12 '18 + s1 s3 s0 XCHG3 // '11=-1 '12 '14 '18 + }> +""" + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4628.tolk b/tolk-tester/tests/invalid-semantics/err-4628.tolk index 3efe9069da..1f7debb5e7 100644 --- a/tolk-tester/tests/invalid-semantics/err-4628.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4628.tolk @@ -11,6 +11,6 @@ fun main() { /** @compilation_should_fail -@stderr field `item` missed in initialization of struct `Wrapper` +@stderr can not deduce T for generic struct `Wrapper` @stderr = Wrapper { */ diff --git a/tolk-tester/tests/invalid-semantics/err-4629.tolk b/tolk-tester/tests/invalid-semantics/err-4629.tolk new file mode 100644 index 0000000000..dc69a83be5 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4629.tolk @@ -0,0 +1,16 @@ +struct Wrapper { + item: T; + value: int; +} + +fun main() { + var c = Wrapper { + value: null + }; +} + +/** +@compilation_should_fail +@stderr field `item` missed in initialization of struct `Wrapper` +@stderr = Wrapper { + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4907.tolk b/tolk-tester/tests/invalid-semantics/err-4907.tolk new file mode 100644 index 0000000000..0674c933b8 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4907.tolk @@ -0,0 +1,25 @@ +struct Data { + data: T; +} + +struct Parameters { + bounce: bool, + dest: slice | Data, + body: TBody, +} + +fun f(params: Parameters) { +} + +fun main() { + f({ + dest: "sss", + body: 123, + }) +} + +/** +@compilation_should_fail +@stderr can not deduce TData for generic struct `Parameters` +@stderr dest: "sss" + */ diff --git a/tolk-tester/tests/invalid-typing/err-6380.tolk b/tolk-tester/tests/invalid-typing/err-6380.tolk new file mode 100644 index 0000000000..b7ab81a33c --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6380.tolk @@ -0,0 +1,14 @@ +struct A { + f1: never; +} + +fun main() { + var a: A = {}; // f1 is never, can be omitted + a.f1 = 100; +} + +/** +@compilation_should_fail +@stderr can not assign `int` to field of type `never` +@stderr a.f1 = 100 + */ diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index c7c8205e41..a484475a21 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -474,6 +474,18 @@ fun test33(): WithDefaults { return { f2: 5, f3: null }; } +struct WithNever { + f1: int; + f2: never; + f3: int; +} + +@method_id(134) +fun test34() { + var o1: WithNever = { f1: 10, f3: 20 }; // f2 is `never`, it can be omitted + __expect_type(o1.f2, "never"); + return o1; +} fun main(x: int8, y: MInt) { @@ -526,6 +538,7 @@ type PointAlias = Point; @testcase | 131 | | 5 141 (null) 141 (null) 0 777 0 0 -1 777 0 -1 @testcase | 132 | | 10 20 10 0 0 20 0 0 0 0 @testcase | 133 | | -1 0 5 (null) (null) (null) 0 0 46 +@testcase | 134 | | 10 20 @fif_codegen """ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 73791e3dd5..ff13ed386f 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -279,9 +279,15 @@ static V parse_genericsT_list(Lexer& lex) { lex.expect(tok_lt, "`<`"); while (true) { lex.check(tok_identifier, "T"); + SrcLocation locT = lex.cur_location(); std::string_view nameT = lex.cur_str(); - genericsT_items.emplace_back(createV(lex.cur_location(), nameT)); lex.next(); + AnyTypeV default_type = nullptr; + if (lex.tok() == tok_assign) { // + lex.next(); + default_type = parse_type_expression(lex); + } + genericsT_items.emplace_back(createV(locT, nameT, default_type)); if (lex.tok() != tok_comma) { break; } diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index b3e1702eb6..fc1dafa22b 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -211,7 +211,7 @@ class ASTReplicator final { return createV(v->loc, v->name); } static V clone(V v) { - return createV(v->loc, v->nameT); + return createV(v->loc, v->nameT, clone(v->default_type_node)); } static V clone(V v) { return createV(v->loc, clone(v->get_items())); diff --git a/tolk/ast.h b/tolk/ast.h index 013ac2a788..ee550064c4 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -1127,12 +1127,14 @@ struct Vertex final : ASTStatementVararg { template<> // ast_genericsT_item is generics T at declaration // example: `fun f` has a list of 2 generic Ts +// example: `struct Params` has 1 generic T with default struct Vertex final : ASTOtherLeaf { std::string_view nameT; + AnyTypeV default_type_node; // exists for ``, nullptr otherwise - Vertex(SrcLocation loc, std::string_view nameT) + Vertex(SrcLocation loc, std::string_view nameT, AnyTypeV default_type_node) : ASTOtherLeaf(ast_genericsT_item, loc) - , nameT(nameT) {} + , nameT(nameT), default_type_node(default_type_node) {} }; template<> diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 5b962f604f..70944cd4c2 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1119,7 +1119,7 @@ void define_builtins() { TypePtr Never = TypeDataNever::create(); TypePtr typeT = TypeDataGenericT::create("T"); - const GenericsDeclaration* declGenericT = new GenericsDeclaration(std::vector{"T"}, 0); + const GenericsDeclaration* declGenericT = new GenericsDeclaration(std::vector{{"T", nullptr}}, 0); std::vector ParamsInt1 = {Int}; std::vector ParamsInt2 = {Int, Int}; diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index 0f8e6356ce..5c0cc88673 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -25,16 +25,22 @@ namespace tolk { // given orig `(int, T)` and substitutions [slice], return `(int, slice)` -static TypePtr replace_genericT_with_deduced(TypePtr orig, const GenericsSubstitutions* substitutedTs) { +static TypePtr replace_genericT_with_deduced(TypePtr orig, const GenericsSubstitutions* substitutedTs, bool apply_defaultTs = false, std::string_view* out_unknownT = nullptr) { if (!orig || !orig->has_genericT_inside()) { return orig; } - return orig->replace_children_custom([substitutedTs](TypePtr child) { + return orig->replace_children_custom([substitutedTs, apply_defaultTs, &out_unknownT](TypePtr child) { if (const TypeDataGenericT* asT = child->try_as()) { TypePtr typeT = substitutedTs->get_substitution_for_nameT(asT->nameT); + if (typeT == nullptr && apply_defaultTs) { + typeT = substitutedTs->get_default_for_nameT(asT->nameT); + } if (typeT == nullptr) { // T was not deduced yet, leave T as generic typeT = child; + if (out_unknownT && out_unknownT->empty()) { + *out_unknownT = asT->nameT; + } } return typeT; } @@ -100,6 +106,14 @@ void GenericsSubstitutions::provide_type_arguments(const std::vector& t } } +void GenericsSubstitutions::rewrite_missing_with_defaults() { + for (int i = 0; i < size(); ++i) { + if (valuesTs[i] == nullptr) { + valuesTs[i] = genericTs->get_defaultT(i); // if no default, left nullptr + } + } +} + GenericSubstitutionsDeducing::GenericSubstitutionsDeducing(FunctionPtr fun_ref) : fun_ref(fun_ref) , struct_ref(nullptr) @@ -188,6 +202,12 @@ void GenericSubstitutionsDeducing::consider_next_condition(TypePtr param_type, T } } } + // `arg: int | MyData` called as `f(MyData)` => T is int + else { + for (TypePtr p_variant : p_union->variants) { + consider_next_condition(p_variant, arg_type); + } + } } else if (const auto* p_instSt = param_type->try_as(); p_instSt && p_instSt->struct_ref) { // `arg: Wrapper` called as `f(wrappedInt)` => T is int if (const auto* a_struct = arg_type->try_as(); a_struct && a_struct->struct_ref->is_instantiation_of_generic_struct() && a_struct->struct_ref->base_struct_ref == p_instSt->struct_ref) { @@ -217,9 +237,11 @@ TypePtr GenericSubstitutionsDeducing::auto_deduce_from_argument(TypePtr param_ty } TypePtr GenericSubstitutionsDeducing::auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type) { - param_type = auto_deduce_from_argument(param_type, arg_type); + std::string_view unknown_nameT; + consider_next_condition(param_type, arg_type); + param_type = replace_genericT_with_deduced(param_type, &deducedTs, true, &unknown_nameT); if (param_type->has_genericT_inside()) { - fire_error_can_not_deduce(cur_f, loc, get_first_not_deduced_nameT()); + fire_error_can_not_deduce(cur_f, loc, unknown_nameT); } return param_type; } @@ -233,6 +255,10 @@ std::string_view GenericSubstitutionsDeducing::get_first_not_deduced_nameT() con return ""; } +void GenericSubstitutionsDeducing::apply_defaults_from_declaration() { + deducedTs.rewrite_missing_with_defaults(); +} + void GenericSubstitutionsDeducing::fire_error_can_not_deduce(FunctionPtr cur_f, SrcLocation loc, std::string_view nameT) const { if (fun_ref) { fire(cur_f, loc, "can not deduce " + static_cast(nameT) + " for generic function `" + fun_ref->as_human_readable() + "`; instantiate it manually with `" + fun_ref->name + "<...>()`"); @@ -249,7 +275,7 @@ std::string GenericsDeclaration::as_human_readable(bool include_from_receiver) c if (result.size() > 1) { result += ", "; } - result += namesT[i]; + result += itemsT[i].nameT; } result += '>'; } @@ -257,27 +283,14 @@ std::string GenericsDeclaration::as_human_readable(bool include_from_receiver) c } int GenericsDeclaration::find_nameT(std::string_view nameT) const { - for (int i = 0; i < static_cast(namesT.size()); ++i) { - if (namesT[i] == nameT) { + for (int i = 0; i < static_cast(itemsT.size()); ++i) { + if (itemsT[i].nameT == nameT) { return i; } } return -1; } -void GenericsDeclaration::append_ast_list_checking_duplicates(AnyV v_genericsT_list, std::vector& out) { - for (int i = 0; i < v_genericsT_list->as()->size(); ++i) { - auto v_item = v_genericsT_list->as()->get_item(i); - auto it_existing = std::find_if(out.begin(), out.end(), [v_item](const std::string_view& prevT) { - return prevT == v_item->nameT; - }); - if (it_existing != out.end()) { - v_item->error("duplicate generic parameter `" + static_cast(v_item->nameT) + "`"); - } - out.emplace_back(v_item->nameT); - } -} - bool GenericsSubstitutions::has_nameT(std::string_view nameT) const { return genericTs->find_nameT(nameT) != -1; } @@ -287,6 +300,11 @@ TypePtr GenericsSubstitutions::get_substitution_for_nameT(std::string_view nameT return idx == -1 ? nullptr : valuesTs[idx]; } +TypePtr GenericsSubstitutions::get_default_for_nameT(std::string_view nameT) const { + int idx = genericTs->find_nameT(nameT); + return idx == -1 ? nullptr : genericTs->get_defaultT(idx); +} + // given this= and rhs=, check that T1 is equal to T2 in terms of "equal_to" of TypePtr // for example, `Wrapper>` / `Wrapper>` / `Wrapper` are equal bool GenericsSubstitutions::equal_to(const GenericsSubstitutions* rhs) const { diff --git a/tolk/generics-helpers.h b/tolk/generics-helpers.h index 08dfcc9812..c8f46ff27d 100644 --- a/tolk/generics-helpers.h +++ b/tolk/generics-helpers.h @@ -24,26 +24,34 @@ namespace tolk { // when a function is declared `f`, this "" is represented as this class -// - `fun tuple.push(self, value:T)` namesT = [T] -// - `struct Pair` namesT = [A,B] -// - `fun Container.compareWith` namesT = [T,U], n_from_receiver = 1 -// - `fun Pair.createFrom` namesT = [U,V] -// - `fun Pair.create` namesT = [A,B], n_from_receiver = 2 +// - `fun tuple.push(self, value:T)` itemsT = [T] +// - `struct Pair` itemsT = [A,B] +// - `fun Container.compareWith` itemsT = [T,U], n_from_receiver = 1 +// - `fun Pair.createFrom` itemsT = [U,V] +// - `fun Pair.create` itemsT = [A,B], n_from_receiver = 2 +// - `struct Opts` itemsT = [T default_type=never] struct GenericsDeclaration { - explicit GenericsDeclaration(std::vector&& namesT, int n_from_receiver) - : namesT(std::move(namesT)) + struct ItemT { + std::string_view nameT; + TypePtr default_type; // exists for ``, nullptr otherwise + + ItemT(std::string_view nameT, TypePtr default_type) + : nameT(nameT), default_type(default_type) {} + }; + + explicit GenericsDeclaration(std::vector&& itemsT, int n_from_receiver) + : itemsT(std::move(itemsT)) , n_from_receiver(n_from_receiver) {} - const std::vector namesT; + const std::vector itemsT; const int n_from_receiver; std::string as_human_readable(bool include_from_receiver = false) const; - int size() const { return static_cast(namesT.size()); } + int size() const { return static_cast(itemsT.size()); } int find_nameT(std::string_view nameT) const; - std::string_view get_nameT(int idx) const { return namesT[idx]; } - - static void append_ast_list_checking_duplicates(AnyV v_genericsT_list, std::vector& out); + std::string_view get_nameT(int idx) const { return itemsT[idx].nameT; } + TypePtr get_defaultT(int idx) const { return itemsT[idx].default_type; } }; // when a function call is `f()`, this "" is represented as this class @@ -65,12 +73,14 @@ struct GenericsSubstitutions { int size() const { return static_cast(valuesTs.size()); } bool has_nameT(std::string_view nameT) const; TypePtr get_substitution_for_nameT(std::string_view nameT) const; + TypePtr get_default_for_nameT(std::string_view nameT) const; std::string_view nameT_at(int idx) const { return genericTs->get_nameT(idx); } TypePtr typeT_at(int idx) const { return valuesTs.at(idx); } bool equal_to(const GenericsSubstitutions* rhs) const; void set_typeT(std::string_view nameT, TypePtr typeT); void provide_type_arguments(const std::vector& type_arguments); + void rewrite_missing_with_defaults(); }; // this class helps to deduce Ts on the fly @@ -92,6 +102,7 @@ class GenericSubstitutionsDeducing { TypePtr auto_deduce_from_argument(TypePtr param_type, TypePtr arg_type); TypePtr auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type); std::string_view get_first_not_deduced_nameT() const; + void apply_defaults_from_declaration(); void fire_error_can_not_deduce(FunctionPtr cur_f, SrcLocation loc, std::string_view nameT) const; GenericsSubstitutions&& flush() { diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 0f86b2ad1f..ecabb5c713 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1344,6 +1344,9 @@ static std::vector process_object_literal(V v, Co } } if (!v_init_val) { + if (field_ref->declared_type == TypeDataNever::create()) { + continue; // field of `never` type can be missed out of object literal (useful in generics defaults) + } // (it occupies 0 slots, nothing is assignable to it — like this field is missing from a struct) tolk_assert(field_ref->has_default_value()); ASTAuxData *aux_data = new AuxData_ForceFiftLocation(prev_loc); auto v_force_loc = createV(v->loc, field_ref->default_value, aux_data, field_ref->declared_type); diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 2f324f5610..abeaffa1f6 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -1160,6 +1160,10 @@ class InferTypesAndCallsAndFieldsVisitor final { deducingTs.auto_deduce_from_argument(cur_f, v->loc, fun_ref->declared_return_type, hint); nameT_unknown = deducingTs.get_first_not_deduced_nameT(); } + if (!nameT_unknown.empty()) { + deducingTs.apply_defaults_from_declaration(); + nameT_unknown = deducingTs.get_first_not_deduced_nameT(); + } if (!nameT_unknown.empty()) { deducingTs.fire_error_can_not_deduce(cur_f, v->get_arg_list()->loc, nameT_unknown); } @@ -1303,11 +1307,18 @@ class InferTypesAndCallsAndFieldsVisitor final { last_struct = variant_struct->struct_ref; n_structs++; } + if (const TypeDataGenericTypeWithTs* hint_withTs = variant->unwrap_alias()->try_as()) { + last_struct = hint_withTs->struct_ref; + n_structs++; + } } if (n_structs == 1) { struct_ref = last_struct; } } + if (const TypeDataGenericTypeWithTs* hint_withTs = hint->unwrap_alias()->try_as()) { + struct_ref = hint_withTs->struct_ref; + } } if (!struct_ref) { fire(cur_f, v->loc, "can not detect struct name\nuse either `var v: StructName = { ... }` or `var v = StructName { ... }`"); @@ -1343,16 +1354,16 @@ class InferTypesAndCallsAndFieldsVisitor final { } assign_inferred_type(field_i, val_i); } - for (StructFieldPtr field_ref : struct_ref->fields) { - if (!(occurred_mask & (1ULL << field_ref->field_idx)) && !field_ref->has_default_value()) { - fire(cur_f, v->get_body()->loc, "field `" + field_ref->name + "` missed in initialization of struct " + to_string(struct_ref)); - } - } // if it's a generic struct `Wrapper`, we need to instantiate it, like `Wrapper` if (struct_ref->is_generic_struct()) { - if (std::string_view nameT = deducingTs.get_first_not_deduced_nameT(); !nameT.empty()) { - deducingTs.fire_error_can_not_deduce(cur_f, v->loc, nameT); + std::string_view nameT_unknown = deducingTs.get_first_not_deduced_nameT(); + if (!nameT_unknown.empty()) { + deducingTs.apply_defaults_from_declaration(); + nameT_unknown = deducingTs.get_first_not_deduced_nameT(); + } + if (!nameT_unknown.empty()) { + deducingTs.fire_error_can_not_deduce(cur_f, v->loc, nameT_unknown); } struct_ref = instantiate_generic_struct(struct_ref, deducingTs.flush()); // re-assign field_ref (it was earlier assigned into a field of a generic struct) @@ -1362,6 +1373,16 @@ class InferTypesAndCallsAndFieldsVisitor final { } } + // check that all fields are present (do it after potential generic instantiation, when types are known) + for (StructFieldPtr field_ref : struct_ref->fields) { + if (!(occurred_mask & (1ULL << field_ref->field_idx))) { + bool allow_missing = field_ref->has_default_value() || field_ref->declared_type == TypeDataNever::create(); + if (!allow_missing) { + fire(cur_f, v->get_body()->loc, "field `" + field_ref->name + "` missed in initialization of struct " + to_string(struct_ref)); + } + } + } + v->mutate()->assign_struct_ref(struct_ref); assign_inferred_type(v, TypeDataStruct::create(struct_ref)); assign_inferred_type(v->get_body(), v); diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 69e24c1a8a..bbb105e428 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -91,13 +91,6 @@ static void validate_arg_ret_order_of_asm_function(V v_body, int n } } -static const GenericsDeclaration* construct_genericTs(V v_list) { - std::vector namesT; - namesT.reserve(v_list->size()); - GenericsDeclaration::append_ast_list_checking_duplicates(v_list, namesT); - return new GenericsDeclaration(std::move(namesT), 0); -} - static GlobalConstPtr register_constant(V v) { GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->type_node, v->get_init_value()); @@ -117,15 +110,11 @@ static GlobalVarPtr register_global_var(V v) { } static AliasDefPtr register_type_alias(V v, AliasDefPtr base_alias_ref = nullptr, std::string override_name = {}, const GenericsSubstitutions* substitutedTs = nullptr) { - const GenericsDeclaration* genericTs = nullptr; - if (v->genericsT_list && !substitutedTs) { - genericTs = construct_genericTs(v->genericsT_list); - } - std::string name = std::move(override_name); if (name.empty()) { name = v->get_identifier()->name; } + const GenericsDeclaration* genericTs = nullptr; // at registering it's null; will be assigned after types resolving AliasDefData* a_sym = new AliasDefData(std::move(name), v->loc, v->underlying_type_node, genericTs, substitutedTs, v); a_sym->base_alias_ref = base_alias_ref; // for `Response`, here is `Response` @@ -152,15 +141,11 @@ static StructPtr register_struct(V v, StructPtr base_str fields.emplace_back(new StructFieldData(static_cast(v_field->get_identifier()->name), v_field->loc, i, v_field->type_node, default_value)); } - const GenericsDeclaration* genericTs = nullptr; - if (v->genericsT_list && !substitutedTs) { - genericTs = construct_genericTs(v->genericsT_list); - } - std::string name = std::move(override_name); if (name.empty()) { name = v->get_identifier()->name; } + const GenericsDeclaration* genericTs = nullptr; // at registering it's null; will be assigned after types resolving StructData* s_sym = new StructData(std::move(name), v->loc, std::move(fields), genericTs, substitutedTs, v); s_sym->base_struct_ref = base_struct_ref; // for `Container`, here is `Container` @@ -201,11 +186,6 @@ static FunctionPtr register_function(V v, FunctionPtr n_mutate_params += static_cast(v_param->declared_as_mutate); } - const GenericsDeclaration* genericTs = nullptr; - if (v->genericsT_list && !substitutedTs) { - genericTs = construct_genericTs(v->genericsT_list); - } - std::string method_name; if (v->receiver_type_node) { method_name = f_identifier; @@ -215,6 +195,7 @@ static FunctionPtr register_function(V v, FunctionPtr name = f_identifier; } + const GenericsDeclaration* genericTs = nullptr; // at registering it's null; will be assigned after types resolving FunctionBody f_body = v->get_body()->kind == ast_block_statement ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); FunctionData* f_sym = new FunctionData(std::move(name), v->loc, std::move(method_name), v->receiver_type_node, v->return_type_node, std::move(parameters), 0, genericTs, substitutedTs, f_body, v); f_sym->base_fun_ref = base_fun_ref; // for `f`, here is `f` diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 084a48c91b..915c01f72b 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -143,28 +143,6 @@ static TypePtr try_parse_predefined_type(std::string_view str) { return nullptr; } -static const GenericsDeclaration* combine_genericTs_from_receiver_and_declaration(TypePtr receiver_type, V v_list) { - std::vector namesT; - receiver_type->replace_children_custom([&namesT](TypePtr child) { - if (const TypeDataGenericT* asT = child->try_as()) { - auto it_existing = std::find_if(namesT.begin(), namesT.end(), [asT](const std::string_view& prevT) { - return prevT == asT->nameT; - }); - if (it_existing == namesT.end()) { - namesT.emplace_back(asT->nameT); - } - } - return child; - }); - int n_from_receiver = static_cast(namesT.size()); - - if (v_list) { - GenericsDeclaration::append_ast_list_checking_duplicates(v_list, namesT); - } - - return new GenericsDeclaration(std::move(namesT), n_from_receiver); -} - class TypeNodesVisitorResolver { FunctionPtr cur_f; // exists if we're inside its body const GenericsDeclaration* genericTs; // `` if we're inside `f` or `f` @@ -375,6 +353,11 @@ class TypeNodesVisitorResolver { throw ParseError(alias_ref->loc, "type `" + alias_ref->name + "` circularly references itself"); } + if (auto v_genericsT_list = alias_ref->ast_root->as()->genericsT_list) { + const GenericsDeclaration* genericTs = construct_genericTs(nullptr, v_genericsT_list); + alias_ref->mutate()->assign_resolved_genericTs(genericTs); + } + called_stack.push_back(alias_ref); TypeNodesVisitorResolver visitor(nullptr, alias_ref->genericTs, alias_ref->substitutedTs, false); TypePtr underlying_type = visitor.finalize_type_node(alias_ref->underlying_type_node); @@ -394,6 +377,11 @@ class TypeNodesVisitorResolver { throw ParseError(struct_ref->loc, "struct `" + struct_ref->name + "` size is infinity due to recursive fields"); } + if (auto v_genericsT_list = struct_ref->ast_root->as()->genericsT_list) { + const GenericsDeclaration* genericTs = construct_genericTs(nullptr, v_genericsT_list); + struct_ref->mutate()->assign_resolved_genericTs(genericTs); + } + called_stack.push_back(struct_ref); TypeNodesVisitorResolver visitor(nullptr, struct_ref->genericTs, struct_ref->substitutedTs, false); for (int i = 0; i < struct_ref->get_num_fields(); ++i) { @@ -404,6 +392,44 @@ class TypeNodesVisitorResolver { struct_ref->mutate()->assign_visited_by_resolver(); called_stack.pop_back(); } + + static const GenericsDeclaration* construct_genericTs(TypePtr receiver_type, V v_list) { + std::vector itemsT; + if (receiver_type && receiver_type->has_genericT_inside()) { + receiver_type->replace_children_custom([&itemsT](TypePtr child) { + if (const TypeDataGenericT* asT = child->try_as()) { + auto it_existing = std::find_if(itemsT.begin(), itemsT.end(), [asT](const GenericsDeclaration::ItemT& prevT) { + return prevT.nameT == asT->nameT; + }); + if (it_existing == itemsT.end()) { + itemsT.emplace_back(asT->nameT, nullptr); + } + } + return child; + }); + } + int n_from_receiver = static_cast(itemsT.size()); + + if (v_list) { + TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr, false); + for (int i = 0; i < v_list->size(); ++i) { + auto v_item = v_list->get_item(i); + auto it_existing = std::find_if(itemsT.begin(), itemsT.end(), [v_item](const GenericsDeclaration::ItemT& prevT) { + return prevT.nameT == v_item->nameT; + }); + if (it_existing != itemsT.end()) { + v_item->error("duplicate generic parameter `" + static_cast(v_item->nameT) + "`"); + } + TypePtr default_type = nullptr; + if (v_list->get_item(i)->default_type_node) { + default_type = visitor.finalize_type_node(v_list->get_item(i)->default_type_node); + } + itemsT.emplace_back(v_item->nameT, default_type); + } + } + + return new GenericsDeclaration(std::move(itemsT), n_from_receiver); + } }; class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { @@ -502,18 +528,18 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { if (fun_ref->receiver_type_node) { TypeNodesVisitorResolver receiver_visitor(fun_ref, fun_ref->genericTs, fun_ref->substitutedTs, true); TypePtr receiver_type = receiver_visitor.finalize_type_node(fun_ref->receiver_type_node); - const GenericsDeclaration* genericTs = fun_ref->genericTs; - if (receiver_type->has_genericT_inside()) { - genericTs = combine_genericTs_from_receiver_and_declaration(receiver_type, v->genericsT_list); - } std::string name_prefix = receiver_type->as_human_readable(); bool embrace = receiver_type->try_as() && !receiver_type->try_as()->or_null; if (embrace) { name_prefix = "(" + name_prefix + ")"; } - fun_ref->mutate()->assign_resolved_receiver_type(receiver_type, genericTs, std::move(name_prefix)); + fun_ref->mutate()->assign_resolved_receiver_type(receiver_type, std::move(name_prefix)); G.symtable.add_function(fun_ref); } + if (v->genericsT_list || (fun_ref->receiver_type && fun_ref->receiver_type->has_genericT_inside())) { + const GenericsDeclaration* genericTs = TypeNodesVisitorResolver::construct_genericTs(fun_ref->receiver_type, v->genericsT_list); + fun_ref->mutate()->assign_resolved_genericTs(genericTs); + } type_nodes_visitor = TypeNodesVisitorResolver(fun_ref, fun_ref->genericTs, fun_ref->substitutedTs, false); diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 2e9fbabe5a..1485eae75a 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -66,11 +66,16 @@ bool FunctionData::does_need_codegen() const { return true; } -void FunctionData::assign_resolved_receiver_type(TypePtr receiver_type, const GenericsDeclaration* genericTs, std::string&& name_prefix) { +void FunctionData::assign_resolved_receiver_type(TypePtr receiver_type, std::string&& name_prefix) { this->receiver_type = receiver_type; if (!this->substitutedTs) { // after receiver has been resolve, update name to "receiver.method" name_prefix.erase(std::remove(name_prefix.begin(), name_prefix.end(), ' '), name_prefix.end()); this->name = name_prefix + "." + this->method_name; + } +} + +void FunctionData::assign_resolved_genericTs(const GenericsDeclaration* genericTs) { + if (this->substitutedTs == nullptr) { this->genericTs = genericTs; } } @@ -140,6 +145,12 @@ void AliasDefData::assign_visited_by_resolver() { this->flags |= flagVisitedByResolver; } +void AliasDefData::assign_resolved_genericTs(const GenericsDeclaration* genericTs) { + if (this->substitutedTs == nullptr) { + this->genericTs = genericTs; + } +} + void AliasDefData::assign_resolved_type(TypePtr underlying_type) { this->underlying_type = underlying_type; } @@ -156,6 +167,12 @@ void StructData::assign_visited_by_resolver() { this->flags |= flagVisitedByResolver; } +void StructData::assign_resolved_genericTs(const GenericsDeclaration* genericTs) { + if (this->substitutedTs == nullptr) { + this->genericTs = genericTs; + } +} + StructFieldPtr StructData::find_field(std::string_view field_name) const { for (StructFieldPtr field : fields) { if (field->name == field_name) { diff --git a/tolk/symtable.h b/tolk/symtable.h index 9c80d19924..38f090e7b4 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -199,7 +199,8 @@ struct FunctionData final : Symbol { bool does_need_codegen() const; FunctionData* mutate() const { return const_cast(this); } - void assign_resolved_receiver_type(TypePtr receiver_type, const GenericsDeclaration* genericTs, std::string&& name_prefix); + void assign_resolved_receiver_type(TypePtr receiver_type, std::string&& name_prefix); + void assign_resolved_genericTs(const GenericsDeclaration* genericTs); void assign_resolved_type(TypePtr declared_return_type); void assign_inferred_type(TypePtr inferred_return_type, TypePtr inferred_full_type); void assign_is_used_as_noncall(); @@ -279,6 +280,7 @@ struct AliasDefData final : Symbol { AliasDefData* mutate() const { return const_cast(this); } void assign_visited_by_resolver(); + void assign_resolved_genericTs(const GenericsDeclaration* genericTs); void assign_resolved_type(TypePtr underlying_type); }; @@ -326,6 +328,7 @@ struct StructData final : Symbol { StructData* mutate() const { return const_cast(this); } void assign_visited_by_resolver(); + void assign_resolved_genericTs(const GenericsDeclaration* genericTs); StructData(std::string name, SrcLocation loc, std::vector&& fields, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, AnyV ast_root) : Symbol(std::move(name), loc) diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index 39431b4a3e..b419a2e3f5 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -870,7 +870,7 @@ bool TypeDataUnknown::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataNever::can_rhs_be_assigned(TypePtr rhs) const { - return true; + return rhs == TypeDataNever::create(); } bool TypeDataVoid::can_rhs_be_assigned(TypePtr rhs) const { From 44705a1bcbb0b1e98ccf662e9ec6d6c778ebec3d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 29 Apr 2025 14:24:09 +0300 Subject: [PATCH 236/524] Load blocks from DB in ShardBlockRetainer, fix is_block_outdated --- tdutils/td/utils/tl_parsers.cpp | 3 ++- validator/db/archive-manager.cpp | 9 +++++++++ validator/db/archive-manager.hpp | 2 ++ validator/db/archive-slice.cpp | 18 ++++++++++++++++++ validator/db/archive-slice.hpp | 2 ++ validator/db/rootdb.cpp | 4 ++++ validator/db/rootdb.hpp | 1 + validator/interfaces/db.h | 2 ++ validator/interfaces/validator-manager.h | 3 +++ validator/manager.cpp | 4 ++++ validator/manager.hpp | 2 ++ validator/shard-block-retainer.cpp | 24 +++++++++++++++++++++++- validator/shard-block-retainer.hpp | 2 ++ validator/shard-block-verifier.cpp | 4 +++- 14 files changed, 77 insertions(+), 3 deletions(-) diff --git a/tdutils/td/utils/tl_parsers.cpp b/tdutils/td/utils/tl_parsers.cpp index eb78d3e1c1..c41cfeb95d 100644 --- a/tdutils/td/utils/tl_parsers.cpp +++ b/tdutils/td/utils/tl_parsers.cpp @@ -33,7 +33,8 @@ TlParser::TlParser(Slice slice) { if (data_len <= small_data_array.size() * sizeof(int32)) { buf = &small_data_array[0]; } else { - LOG(ERROR) << "Unexpected big unaligned data pointer of length " << slice.size() << " at " << slice.begin(); + LOG(DEBUG) << "Unexpected big unaligned data pointer of length " << slice.size() << " at " + << (const void *)slice.begin(); data_buf = std::make_unique(1 + data_len / sizeof(int32)); buf = data_buf.get(); } diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 9e6ea797e3..b39f4374f4 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -1224,6 +1224,15 @@ void ArchiveManager::prepare_stats(td::Promise f) { + for (auto &[_, file] : temp_files_) { + if (file.deleted) { + continue; + } + td::actor::send_closure(file.file_actor_id(), &ArchiveSlice::iterate_block_handles, f); + } +} + void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise) { index_->begin_transaction().ensure(); td::MultiPromise mp; diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index 33dd1f3d6f..9245aa61d0 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -84,6 +84,8 @@ class ArchiveManager : public td::actor::Actor { void prepare_stats(td::Promise>> promise); + void iterate_temp_block_handles(std::function f); + static constexpr td::uint32 archive_size() { return 20000; } diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index 41ed509978..635df6dbb3 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -686,6 +686,24 @@ void ArchiveSlice::close_files() { } } +void ArchiveSlice::iterate_block_handles(std::function f) { + before_query(); + td::uint32 range_start = ton_api::db_blockdb_key_value::ID; + td::uint32 range_end = ton_api::db_blockdb_key_value::ID + 1; + kv_->for_each_in_range(td::Slice{(char *)&range_start, 4}, td::Slice{(char *)&range_end, 4}, + [&](td::Slice key, td::Slice value) -> td::Status { + auto r_key = fetch_tl_object(key, true); + if (r_key.is_error()) { + return td::Status::OK(); + } + auto r_handle = create_block_handle(value); + if (r_handle.is_ok()) { + f(*r_handle.ok()); + } + return td::Status::OK(); + }); +} + void ArchiveSlice::do_close() { if (destroyed_) { return; diff --git a/validator/db/archive-slice.hpp b/validator/db/archive-slice.hpp index a027ec0ff5..b955c270aa 100644 --- a/validator/db/archive-slice.hpp +++ b/validator/db/archive-slice.hpp @@ -127,6 +127,8 @@ class ArchiveSlice : public td::actor::Actor { void open_files(); void close_files(); + void iterate_block_handles(std::function f); + private: void before_query(); void do_close(); diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 90546fbef9..0b3730edea 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -527,6 +527,10 @@ void RootDb::get_persistent_state_descriptions(td::Promise f) { + td::actor::send_closure(archive_db_, &ArchiveManager::iterate_temp_block_handles, std::move(f)); +} + } // namespace validator } // namespace ton diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 2ccd7f942d..05187fde78 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -139,6 +139,7 @@ class RootDb : public Db { void add_persistent_state_description(td::Ref desc, td::Promise promise) override; void get_persistent_state_descriptions(td::Promise>> promise) override; + void iterate_temp_block_handles(std::function f) override; private: td::actor::ActorId validator_manager_; diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index f21c6f9028..c714e1e34c 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -129,6 +129,8 @@ class Db : public td::actor::Actor { td::Promise promise) = 0; virtual void get_persistent_state_descriptions( td::Promise>> promise) = 0; + + virtual void iterate_temp_block_handles(std::function f) = 0; }; } // namespace validator diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 140f01ff20..c6151f1b7f 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -274,6 +274,9 @@ class ValidatorManager : public ValidatorManagerInterface { promise.set_result(td::Unit()); } + virtual void iterate_temp_block_handles(std::function f) { + } + static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/manager.cpp b/validator/manager.cpp index f68cdf941b..fa4c7bef98 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3710,6 +3710,10 @@ void ValidatorManagerImpl::add_shard_block_retainer(adnl::AdnlNodeIdShort id) { "shardblockretainer", id, last_masterchain_state_, opts_, actor_id(this), adnl_, rldp_); } +void ValidatorManagerImpl::iterate_temp_block_handles(std::function f) { + td::actor::send_closure(db_, &Db::iterate_temp_block_handles, std::move(f)); +} + } // namespace validator } // namespace ton diff --git a/validator/manager.hpp b/validator/manager.hpp index fce2ceee4c..31cd6feab0 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -778,6 +778,8 @@ class ValidatorManagerImpl : public ValidatorManager { void add_shard_block_retainer(adnl::AdnlNodeIdShort id) override; + void iterate_temp_block_handles(std::function f) override; + std::map> validator_telemetry_; void init_validator_telemetry(); diff --git a/validator/shard-block-retainer.cpp b/validator/shard-block-retainer.cpp index 4fb96502e7..c36a9bac6a 100644 --- a/validator/shard-block-retainer.cpp +++ b/validator/shard-block-retainer.cpp @@ -15,6 +15,7 @@ along with TON Blockchain Library. If not, see . */ #include "shard-block-retainer.hpp" +#include namespace ton::validator { @@ -87,6 +88,18 @@ void ShardBlockRetainer::update_masterchain_state(td::Ref stat ++it; } } + if (!inited_) { + delay_action( + [SelfId = actor_id(this), manager = manager_]() { + td::actor::send_closure( + manager, &ValidatorManager::iterate_temp_block_handles, [SelfId](const BlockHandleInterface& handle) { + if (!handle.id().is_masterchain() && handle.received_state()) { + td::actor::send_closure(SelfId, &ShardBlockRetainer::got_block_from_db, handle.id()); + } + }); + }, + td::Timestamp::in(1.0)); + } inited_ = true; } @@ -171,8 +184,17 @@ void ShardBlockRetainer::confirm_block(BlockIdExt block_id) { LOG(INFO) << "Confirmed block " << block_id.to_str() << ", sending " << sent << " confirmations"; } +void ShardBlockRetainer::got_block_from_db(BlockIdExt block_id) { + if (!is_block_outdated(block_id)) { + LOG(INFO) << "Loaded confirmed block from DB: " << block_id.to_str(); + confirm_block(block_id); + } +} + bool ShardBlockRetainer::is_block_outdated(const BlockIdExt& block_id) const { - auto shard_desc = last_masterchain_state_->get_shard_from_config(block_id.shard_full(), false); + ShardIdFull shard = block_id.shard_full(); + shard.shard |= 1; + auto shard_desc = last_masterchain_state_->get_shard_from_config(shard, false); return shard_desc.not_null() && shard_desc->top_block_id().seqno() >= block_id.seqno(); } diff --git a/validator/shard-block-retainer.hpp b/validator/shard-block-retainer.hpp index 14a6cfa8ae..ca8d4c8776 100644 --- a/validator/shard-block-retainer.hpp +++ b/validator/shard-block-retainer.hpp @@ -62,6 +62,8 @@ class ShardBlockRetainer : public td::actor::Actor { void confirm_shard_block_description(td::Ref desc); void confirm_block(BlockIdExt block_id); + void got_block_from_db(BlockIdExt block_id); + bool is_block_outdated(const BlockIdExt& block_id) const; static constexpr double SUBSCRIPTION_TTL = 60.0; diff --git a/validator/shard-block-verifier.cpp b/validator/shard-block-verifier.cpp index 5beb1e9a3d..000260a362 100644 --- a/validator/shard-block-verifier.cpp +++ b/validator/shard-block-verifier.cpp @@ -138,7 +138,9 @@ int ShardBlockVerifier::get_config_shard_idx(const ShardIdFull& shard_id) const } bool ShardBlockVerifier::is_block_outdated(const BlockIdExt& block_id) const { - auto shard_desc = last_masterchain_state_->get_shard_from_config(block_id.shard_full(), false); + ShardIdFull shard = block_id.shard_full(); + shard.shard |= 1; + auto shard_desc = last_masterchain_state_->get_shard_from_config(shard, false); return shard_desc.not_null() && shard_desc->top_block_id().seqno() >= block_id.seqno(); } From f361e78b5a7e8dcda0945b599f9beae16587f93c Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 1 May 2025 15:54:21 +0400 Subject: [PATCH 237/524] [Tolk] Optimize arg_order in advance when it's safe This partially "reverts" the behavior of pragma 'compute-asm-ltr' from FunC, which was always on in Tolk. Now, if it's safe, for asm functions with arg_order, arguments are evaluated and placed onto the stack in a desired order. When it's unsafe (the purpose of this pragma, actually), arguments are evaluated left-to-right. --- tolk-tester/tests/asm-arg-order.tolk | 117 ++++++++++++++++++++++++- tolk-tester/tests/cells-slices.tolk | 49 ++++++++--- tolk-tester/tests/constants-tests.tolk | 2 +- tolk-tester/tests/strings-tests.tolk | 2 +- tolk/analyzer.cpp | 4 + tolk/codegen.cpp | 6 +- tolk/pipe-ast-to-legacy.cpp | 84 ++++++++++++++++-- tolk/tolk.h | 5 +- 8 files changed, 248 insertions(+), 21 deletions(-) diff --git a/tolk-tester/tests/asm-arg-order.tolk b/tolk-tester/tests/asm-arg-order.tolk index 841dc95645..6d55b2da3f 100644 --- a/tolk-tester/tests/asm-arg-order.tolk +++ b/tolk-tester/tests/asm-arg-order.tolk @@ -120,6 +120,67 @@ fun test_new_dot(): (tuple, tuple) { return (t, t2); } +@pure +fun asmAPlus1TimesB(a: int, b: int): int + asm(b a) "1 ADDCONST MUL"; + +@pure +fun int.plus1TimesB(self, b: int): int + asm(b self) "1 ADDCONST MUL"; + +@pure +fun get2Pure() { return 2; } +@pure +fun get10Pure() { return 10; } + +fun get2Impure() { return 2; } +fun get10Impure() { return 10; } + +global g2: int; +global g10: int; + +fun setG2(v: int) { g2 = v; return v; } + +@method_id(27) +fun test27() { + return asmAPlus1TimesB(2, 10); +} + +@method_id(28) +fun test28() { + return asmAPlus1TimesB(get2Pure(), get10Pure()); +} + +@method_id(29) +fun test29() { + return asmAPlus1TimesB(get2Impure(), get10Impure()); +} + +@method_id(30) +fun test30() { + g2 = 2; + g10 = 10; + return asmAPlus1TimesB(g2, g10); +} + +@method_id(31) +fun test31() { + g2 = 2; + g10 = 10; + return asmAPlus1TimesB(g2 += 2, g10 += g2); +} + +@method_id(32) +fun test32() { + return 2.plus1TimesB(10); +} + +@method_id(33) +fun test33(x: int) { + return ((x += 10).plus1TimesB(2), (x += 20).plus1TimesB(x), ((x /= (g2=2)).plus1TimesB(x*g2)), setG2(7).plus1TimesB(g2)); +} + + fun main() { } @@ -137,6 +198,60 @@ fun main() { @testcase | 24 | | [ 11 22 33 44 55 ] [ 220 330 440 110 550 ] @testcase | 25 | | [ 22 33 ] [ 220 330 ] @testcase | 26 | | [ 11 22 33 ] [ 220 330 110 ] +@testcase | 27 | | 30 +@testcase | 28 | | 30 +@testcase | 29 | | 30 +@testcase | 30 | | 30 +@testcase | 31 | | 70 +@testcase | 32 | | 30 +@testcase | 33 | 0 | 22 930 480 56 + +@fif_codegen +""" + test27 PROC:<{ + 10 PUSHINT + 2 PUSHINT + 1 ADDCONST MUL + }> +""" + +@fif_codegen +""" + test28 PROC:<{ + get10Pure CALLDICT + get2Pure CALLDICT + 1 ADDCONST MUL + }> +""" + +@fif_codegen +""" + test29 PROC:<{ + get2Impure CALLDICT + get10Impure CALLDICT + SWAP + 1 ADDCONST MUL + }> +""" + +@fif_codegen +""" + test30 PROC:<{ + ... + g10 GETGLOB + g2 GETGLOB + 1 ADDCONST MUL + }> +""" + +@fif_codegen +""" + test32 PROC:<{ + 10 PUSHINT + 2 PUSHINT + 1 ADDCONST MUL + }> +""" -@code_hash 93068291567112337250118419287631047120002003622184251973082208096953112184588 +@code_hash 78671986831403867804966279036762472603849672357801214378328975900111280733054 */ diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 63a823ed9f..70c40b531e 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -198,6 +198,37 @@ fun test111() { return (op1, q1, op2, q2); } +global g8: int; +global g10: int; +global g16: int; +global g32: int; + +@inline_ref +fun initGlobals() { + g8 = 8; + g10 = 10; + g16 = 16; + g32 = 32; +} + +@method_id(116) +fun test16() { + initGlobals(); + var b1 = beginCell().storeUint(g8, g16).storeUint(0xFF, g32).storeUint(g8, g16 * 2); + var b2 = beginCell().storeUint(8, 16).storeUint(0xFF, 32).storeUint(8, 16 * 2); + assert(b1.bitsCount() == b2.bitsCount(), 400); + var c1 = b1.endCell().beginParse(); + var c2 = b2.endCell().beginParse(); + assert(c1.bitsEqual(c2), 400); + assert(c1.loadUint(g16) == g8, 400); + assert(c1.loadUint(g32) == 0xFF, 400); + assert(c1.loadUint(2 * g16) == 8, 400); + return b1; + // 00140008000000ff00000008 + // 00140008000000ff00000008 +} + + fun main(): int { return 0; } @@ -217,21 +248,19 @@ fun main(): int { @testcase | 114 | -1 | -1 0 -1 @testcase | 114 | 0 | 0 0 0 @testcase | 115 | | 123 456 123 456 +@testcase | 116 | | BC{00140008000000ff00000008} Note, that since 'compute-asm-ltr' became on be default, chaining methods codegen is not quite optimal. @fif_codegen """ test6 PROC:<{ - NEWC // '0 - 1 PUSHINT // '0 '1=1 - SWAP // '1=1 '0 - 32 STU // '0 - 2 PUSHINT // '0 '4=2 - SWAP // '4=2 '0 - 32 STU // '0 - 3 PUSHINT // '0 '7=3 - SWAP // '7=3 '0 - 32 STU // '0 + 3 PUSHINT // '0=3 + 2 PUSHINT // '0=3 '1=2 + 1 PUSHINT // '0=3 '1=2 '2=1 + NEWC // '0=3 '1=2 '2=1 '3 + 32 STU // '0=3 '1=2 '3 + 32 STU // '0=3 '3 + 32 STU // '3 }> """ */ diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index ba213093bb..ed510c13c6 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -118,5 +118,5 @@ fun main() { @testcase | 104 | | 1 1 2 @testcase | 105 | | -1 0 7 48 -@code_hash 80040709432962217077682091261201772251141677197885524779745956896218368868623 +@code_hash 49556957179018386976033482229516007597784982050169632168468608374010225644988 */ diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index 1316595714..9feed3a1c2 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -78,5 +78,5 @@ fun test1() { @testcase | 0 | | 0 @testcase | 101 | | [ 65 66 67 68 ] -@code_hash 38184847030631877916087987911699475358017315230885358090110033079289166112584 +@code_hash 55974318379341089957961227475446008591490555692181953973486962465702042912657 */ diff --git a/tolk/analyzer.cpp b/tolk/analyzer.cpp index 480165a169..f9721f5f1f 100644 --- a/tolk/analyzer.cpp +++ b/tolk/analyzer.cpp @@ -851,6 +851,10 @@ void Op::set_impure_flag() { flags |= _Impure; } +void Op::set_arg_order_already_equals_asm_flag() { + flags |= _ArgOrderAlreadyEqualsAsm; +} + bool Op::mark_noreturn() { switch (cl) { case _Nop: diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp index 6c970269e5..ff6c39676b 100644 --- a/tolk/codegen.cpp +++ b/tolk/codegen.cpp @@ -453,8 +453,10 @@ bool Op::generate_code_step(Stack& stack) { if (disabled()) { return true; } - // f_sym can be nullptr for Op::_CallInd (invoke a variable, not a function) - const std::vector* arg_order = f_sym ? f_sym->get_arg_order() : nullptr; + // f_sym can be nullptr for Op::_CallInd (invoke a variable, not a function); + // if f has arg_order, when it's safe, the compiler evaluates arguments in that order in advance (for fewer stack manipulations); + // when it's unsafe, arguments are evaluated left-to-right, and we need to match asm arg_order here + const std::vector* arg_order = f_sym && !arg_order_already_equals_asm() ? f_sym->get_arg_order() : nullptr; const std::vector* ret_order = f_sym ? f_sym->get_ret_order() : nullptr; tolk_assert(!arg_order || arg_order->size() == right.size()); tolk_assert(!ret_order || ret_order->size() == left.size()); diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index ecabb5c713..a45a4ccb6f 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -278,6 +278,53 @@ class LValContext { } }; +// the purpose of this class is having a call `f(a1,a2,...)` when f has asm arg_order, to check +// whether it's safe to rearrange arguments (to evaluate them in arg_order right here for fewer stack manipulations) +// or it's unsafe, and we should evaluate them left-to-right; +// example: `f(1,2,3)` / `b.storeUint(2,32)` is safe +// example: `f(x,x+=5,x)` / `f(impureF1(), global_var)` is unsafe +class CheckReorderingForAsmArgOrderIsSafeVisitor final : public ASTVisitorFunctionBody { + bool has_side_effects = false; + +protected: + void visit(V v) override { + has_side_effects |= v->fun_maybe == nullptr || !v->fun_maybe->is_marked_as_pure(); + parent::visit(v); + } + + void visit(V v) override { + has_side_effects = true; + parent::visit(v); + } + + void visit(V v) override { + has_side_effects = true; + parent::visit(v); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + tolk_assert(false); + } + + static bool is_safe_to_reorder(V v) { + for (const LocalVarData& param : v->fun_maybe->parameters) { + if (param.declared_type->get_width_on_stack() != 1) { + return false; + } + } + + CheckReorderingForAsmArgOrderIsSafeVisitor visitor; + for (int i = 0; i < v->get_num_args(); ++i) { + visitor.ASTVisitorFunctionBody::visit(v->get_arg(i)->get_expr()); + } + if (v->dot_obj_is_self) { + visitor.ASTVisitorFunctionBody::visit(v->get_self_obj()); + } + return !visitor.has_side_effects; + } +}; + // given `{some_expr}!`, return some_expr static AnyExprV unwrap_not_null_operator(AnyExprV v) { while (auto v_notnull = v->try_as()) { @@ -469,12 +516,16 @@ static std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_t } static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcLocation loc, - std::vector&& args_vars, FunctionPtr fun_ref, const char* debug_desc) { + std::vector&& args_vars, FunctionPtr fun_ref, const char* debug_desc, + bool arg_order_already_equals_asm = false) { std::vector rvect = code.create_tmp_var(ret_type, loc, debug_desc); Op& op = code.emplace_back(loc, Op::_Call, rvect, std::move(args_vars), fun_ref); if (!fun_ref->is_marked_as_pure()) { op.set_impure_flag(); } + if (arg_order_already_equals_asm) { + op.set_arg_order_already_equals_asm_flag(); + } return rvect; } @@ -1234,9 +1285,31 @@ static std::vector process_function_call(V v, Code for (int i = 0; i < v->get_num_args(); ++i) { args.push_back(v->get_arg(i)->get_expr()); } + // the purpose of tensor_tt ("tensor target type") is to transition `null` to `(int, int)?` and so on // the purpose of calling `pre_compile_tensor_inner` is to have 0-th IR vars to handle return self std::vector params_types = fun_ref->inferred_full_type->try_as()->params_types; + + // if fun_ref has asm arg_order, maybe it's safe to swap arguments here (to put them onto a stack in the right way); + // (if it's not safe, arguments are evaluated left-to-right, involving stack transformations later) + bool arg_order_already_equals_asm = false; + int asm_self_idx = 0; + if (!fun_ref->arg_order.empty() && CheckReorderingForAsmArgOrderIsSafeVisitor::is_safe_to_reorder(v)) { + std::vector new_args(args.size()); + std::vector new_params_types(params_types.size()); + for (int i = 0; i < static_cast(fun_ref->arg_order.size()); ++i) { + int real_i = fun_ref->arg_order[i]; + new_args[i] = args[real_i]; + new_params_types[i] = params_types[real_i]; + if (real_i == 0) { + asm_self_idx = i; + } + } + args = std::move(new_args); + params_types = std::move(new_params_types); + arg_order_already_equals_asm = true; + } + const TypeDataTensor* tensor_tt = TypeDataTensor::create(std::move(params_types))->try_as(); std::vector> vars_per_arg = pre_compile_tensor_inner(code, args, tensor_tt, nullptr); @@ -1263,20 +1336,21 @@ static std::vector process_function_call(V v, Code for (const std::vector& list : vars_per_arg) { args_vars.insert(args_vars.end(), list.cbegin(), list.cend()); } - std::vector rvect_apply = gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)"); + std::vector rvect_apply = gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)", arg_order_already_equals_asm); if (fun_ref->has_mutate_params()) { LValContext local_lval; std::vector left; for (int i = 0; i < delta_self + v->get_num_args(); ++i) { + int real_i = arg_order_already_equals_asm ? i == 0 && delta_self ? asm_self_idx : fun_ref->arg_order[i - delta_self] : i; if (fun_ref->parameters[i].is_mutate_parameter()) { - AnyExprV arg_i = obj_leftmost && i == 0 ? obj_leftmost : args[i]; + AnyExprV arg_i = obj_leftmost && i == 0 ? obj_leftmost : args[real_i]; tolk_assert(arg_i->is_lvalue || i == 0); if (arg_i->is_lvalue) { std::vector ith_var_idx = pre_compile_expr(arg_i, code, nullptr, &local_lval); left.insert(left.end(), ith_var_idx.begin(), ith_var_idx.end()); } else { - left.insert(left.end(), vars_per_arg[0].begin(), vars_per_arg[0].end()); + left.insert(left.end(), vars_per_arg[asm_self_idx].begin(), vars_per_arg[asm_self_idx].end()); } } } @@ -1292,7 +1366,7 @@ static std::vector process_function_call(V v, Code if (obj_leftmost->is_lvalue) { // to handle if obj is global var, potentially re-assigned inside a chain rvect_apply = pre_compile_expr(obj_leftmost, code, nullptr); } else { // temporary object, not lvalue, pre_compile_expr - rvect_apply = vars_per_arg[0]; + rvect_apply = vars_per_arg[asm_self_idx]; } } diff --git a/tolk/tolk.h b/tolk/tolk.h index 55d9d7677c..ad2a5898d3 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -287,7 +287,7 @@ struct Op { _SliceConst, }; OpKind cl; - enum { _Disabled = 1, _NoReturn = 4, _Impure = 24 }; + enum { _Disabled = 1, _NoReturn = 2, _Impure = 4, _ArgOrderAlreadyEqualsAsm = 8 }; int flags; std::unique_ptr next; FunctionPtr f_sym = nullptr; @@ -347,6 +347,9 @@ struct Op { bool impure() const { return flags & _Impure; } void set_impure_flag(); + bool arg_order_already_equals_asm() const { return flags & _ArgOrderAlreadyEqualsAsm; } + void set_arg_order_already_equals_asm_flag(); + void show(std::ostream& os, const std::vector& vars, std::string pfx = "", int mode = 0) const; void show_var_list(std::ostream& os, const std::vector& idx_list, const std::vector& vars) const; void show_var_list(std::ostream& os, const std::vector& list, const std::vector& vars) const; From 50b8873d47045466b7dd36683e1dba87529d32f8 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 10 Apr 2025 22:25:58 +0400 Subject: [PATCH 238/524] [Tolk] Bump version to v0.12 --- crypto/smartcont/tolk-stdlib/common.tolk | 2 +- crypto/smartcont/tolk-stdlib/gas-payments.tolk | 2 +- crypto/smartcont/tolk-stdlib/lisp-lists.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk | 2 +- tolk/tolk-version.h | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index f22f4695f3..8ff2010bcb 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -1,7 +1,7 @@ // Standard library for Tolk (LGPL licence). // It contains common functions that are available out of the box, the user doesn't have to import anything. // More specific functions are required to be imported explicitly, like "@stdlib/tvm-dicts". -tolk 0.11 +tolk 0.12 /// In Tolk v1.x there would be a type `map`. /// Currently, working with dictionaries is still low-level, with raw cells. diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index b0cab4de8d..2c35faf930 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.11 +tolk 0.12 /** Gas and payment related primitives. diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk index 9dd6c1da9a..ada7a72bc7 100644 --- a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.11 +tolk 0.12 /** Lisp-style lists are nested 2-elements tuples: `[1, [2, [3, null]]]` represents list `[1, 2, 3]`. diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index 5de989aed9..90b392f615 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.11 +tolk 0.12 /** Dictionaries are represented as `cell` data type (cells can store anything, dicts in particular). diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index 0755145397..47b4a300ee 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.11 +tolk 0.12 /// Usually `c3` has a continuation initialized by the whole code of the contract. It is used for function calls. /// The primitive returns the current value of `c3`. diff --git a/tolk/tolk-version.h b/tolk/tolk-version.h index b8f2893eb2..dd9134f453 100644 --- a/tolk/tolk-version.h +++ b/tolk/tolk-version.h @@ -18,6 +18,6 @@ namespace tolk { -constexpr const char* TOLK_VERSION = "0.11.0"; +constexpr const char* TOLK_VERSION = "0.12.0"; } // namespace tolk From 0b024a648d8fdcae43875a82f2b918bc183a70c5 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 9 May 2025 19:02:08 +0300 Subject: [PATCH 239/524] Improve processing block candidate broadcasts, prepare to broadcast shard blocks without signatures (#1642) --- create-hardfork/create-hardfork.cpp | 7 +- tdutils/CMakeLists.txt | 1 + tdutils/td/utils/LRUCache.h | 123 ++++++++++++++++++++++ test/test-ton-collator.cpp | 7 +- validator/full-node-private-overlay.cpp | 4 +- validator/full-node-shard.cpp | 4 +- validator/full-node.cpp | 36 +++---- validator/full-node.hpp | 3 +- validator/impl/accept-block.cpp | 5 + validator/impl/top-shard-descr.hpp | 3 + validator/interfaces/shard-block.h | 1 + validator/interfaces/validator-manager.h | 2 +- validator/manager-disk.cpp | 14 +-- validator/manager-disk.hpp | 9 +- validator/manager-hardfork.hpp | 11 +- validator/manager.cpp | 124 +++++++++++++---------- validator/manager.hpp | 19 ++-- validator/validator-group.cpp | 27 +++++ validator/validator-group.hpp | 8 +- validator/validator.h | 9 +- 20 files changed, 299 insertions(+), 118 deletions(-) create mode 100644 tdutils/td/utils/LRUCache.h diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 72bffae56e..20e58f006b 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -216,8 +216,9 @@ class HardforkCreator : public td::actor::Actor { std::move(msg), 0); } for (auto &topmsg : top_shard_descrs_) { - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManager::new_shard_block, ton::BlockIdExt{}, - 0, std::move(topmsg)); + td::actor::send_closure(validator_manager_, + &ton::validator::ValidatorManager::new_shard_block_description_broadcast, + ton::BlockIdExt{}, 0, std::move(topmsg)); } class Callback : public ton::validator::ValidatorManagerInterface::Callback { private: @@ -246,7 +247,7 @@ class HardforkCreator : public td::actor::Actor { void send_shard_block_info(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::BufferSlice data) override { } void send_block_candidate(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) override { + td::BufferSlice data, int mode) override { } void send_broadcast(ton::BlockBroadcast broadcast, int mode) override { } diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index 716ec322a2..b4a7fec211 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -214,6 +214,7 @@ set(TDUTILS_SOURCE td/utils/invoke.h td/utils/JsonBuilder.h td/utils/List.h + td/utils/LRUCache.h td/utils/logging.h td/utils/MemoryLog.h td/utils/misc.h diff --git a/tdutils/td/utils/LRUCache.h b/tdutils/td/utils/LRUCache.h new file mode 100644 index 0000000000..5c30f66938 --- /dev/null +++ b/tdutils/td/utils/LRUCache.h @@ -0,0 +1,123 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include +#include +#include "List.h" +#include "check.h" + +namespace td { + +template +class LRUCache { + public: + explicit LRUCache(size_t max_size) : max_size_(max_size) { + CHECK(max_size_ > 0); + } + LRUCache(const LRUCache&) = delete; + LRUCache& operator=(const LRUCache&) = delete; + + V* get_if_exists(const K& key, bool update = true) { + auto it = cache_.find(key); + if (it == cache_.end()) { + return nullptr; + } + Entry* entry = it->get(); + if (update) { + entry->remove(); + lru_.put(entry); + } + return &entry->value; + } + + bool contains(const K& key) const { + return cache_.contains(key); + } + + bool put(const K& key, V value, bool update = true) { + bool added = false; + auto it = cache_.find(key); + if (it == cache_.end()) { + update = true; + it = cache_.insert(std::make_unique(key, std::move(value))).first; + added = true; + } else { + (*it)->value = std::move(value); + if (update) { + (*it)->remove(); + } + } + if (update) { + lru_.put(it->get()); + cleanup(); + } + return added; + } + + V& get(const K& key, bool update = true) { + auto it = cache_.find(key); + if (it == cache_.end()) { + update = true; + it = cache_.insert(std::make_unique(key)).first; + } else if (update) { + (*it)->remove(); + } + V& result = (*it)->value; + if (update) { + lru_.put(it->get()); + cleanup(); + } + return result; + } + + private: + struct Entry : ListNode { + explicit Entry(K key) : key(std::move(key)) { + } + Entry(K key, V value) : key(std::move(key)), value(std::move(value)) { + } + K key; + V value; + }; + struct Cmp { + using is_transparent = void; + bool operator()(const std::unique_ptr& a, const std::unique_ptr& b) const { + return a->key < b->key; + } + bool operator()(const std::unique_ptr& a, const K& b) const { + return a->key < b; + } + bool operator()(const K& a, const std::unique_ptr& b) const { + return a < b->key; + } + }; + std::set, Cmp> cache_; + ListNode lru_; + size_t max_size_; + + void cleanup() { + while (cache_.size() > max_size_) { + auto to_remove = (Entry*)lru_.get(); + CHECK(to_remove); + to_remove->remove(); + cache_.erase(cache_.find(to_remove->key)); + } + } +}; + +} // namespace td diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 0fde1e68db..a062ddbe5b 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -303,8 +303,9 @@ class TestNode : public td::actor::Actor { std::move(msg), 0); } for (auto &topmsg : top_shard_descrs_) { - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManager::new_shard_block, ton::BlockIdExt{}, - 0, std::move(topmsg)); + td::actor::send_closure(validator_manager_, + &ton::validator::ValidatorManager::new_shard_block_description_broadcast, + ton::BlockIdExt{}, 0, std::move(topmsg)); } class Callback : public ton::validator::ValidatorManagerInterface::Callback { private: @@ -347,7 +348,7 @@ class TestNode : public td::actor::Actor { } } void send_block_candidate(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) override { + td::BufferSlice data, int mode) override { } void send_broadcast(ton::BlockBroadcast broadcast, int mode) override { } diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index 1fb9c15079..386eba94ab 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -49,8 +49,8 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, ton_api:: BlockIdExt block_id = create_block_id(query.block_->block_); VLOG(FULL_NODE_DEBUG) << "Received newShardBlockBroadcast in private overlay from " << src << ": " << block_id.to_str(); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, block_id, - query.block_->cc_seqno_, std::move(query.block_->data_)); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block_description_broadcast, + block_id, query.block_->cc_seqno_, std::move(query.block_->data_)); } void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index a8ddb338ad..2543431a81 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -780,8 +780,8 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ex void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { BlockIdExt block_id = create_block_id(query.block_->block_); VLOG(FULL_NODE_DEBUG) << "Received newShardBlockBroadcast from " << src << ": " << block_id.to_str(); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, block_id, - query.block_->cc_seqno_, std::move(query.block_->data_)); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block_description_broadcast, + block_id, query.block_->cc_seqno_, std::move(query.block_->data_)); } void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { diff --git a/validator/full-node.cpp b/validator/full-node.cpp index e1951c365f..953e0c599c 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -313,18 +313,20 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s } void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) { - send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); - auto shard = get_shard(ShardIdFull{masterchainId, shardIdAll}); - if (shard.empty()) { - VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; - return; + td::BufferSlice data, int mode) { + if (mode & broadcast_mode_custom) { + send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); } - if (!private_block_overlays_.empty()) { + if ((mode & broadcast_mode_private_block) && !private_block_overlays_.empty()) { td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_block_candidate, block_id, cc_seqno, validator_set_hash, data.clone()); } - if (broadcast_block_candidates_in_public_overlay_) { + if (mode & broadcast_mode_public) { + auto shard = get_shard(ShardIdFull{masterchainId, shardIdAll}); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; + return; + } td::actor::send_closure(shard, &FullNodeShard::send_block_candidate, block_id, cc_seqno, validator_set_hash, std::move(data)); } @@ -334,11 +336,6 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, int mode) { if (mode & broadcast_mode_custom) { send_block_broadcast_to_custom_overlays(broadcast); } - auto shard = get_shard(broadcast.block_id.shard_full()); - if (shard.empty()) { - VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; - return; - } if (mode & broadcast_mode_private_block) { if (!private_block_overlays_.empty()) { td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, @@ -346,6 +343,11 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, int mode) { } } if (mode & broadcast_mode_public) { + auto shard = get_shard(broadcast.block_id.shard_full()); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; + return; + } td::actor::send_closure(shard, &FullNodeShard::send_broadcast, std::move(broadcast)); } } @@ -556,7 +558,7 @@ void FullNodeImpl::send_validator_telemetry(PublicKeyHash key, tl_object_ptr R) { if (R.is_error()) { if (R.error().code() == ErrorCode::notready) { @@ -572,7 +574,7 @@ void FullNodeImpl::process_block_candidate_broadcast(BlockIdExt block_id, Catcha td::uint32 validator_set_hash, td::BufferSlice data) { send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); // ignore cc_seqno and validator_hash for now - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_block_candidate, block_id, + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_block_candidate_broadcast, block_id, std::move(data)); } @@ -630,9 +632,9 @@ void FullNodeImpl::start_up() { td::actor::send_closure(id_, &FullNodeImpl::send_shard_block_info, block_id, cc_seqno, std::move(data)); } void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) override { + td::BufferSlice data, int mode) override { td::actor::send_closure(id_, &FullNodeImpl::send_block_candidate, block_id, cc_seqno, validator_set_hash, - std::move(data)); + std::move(data), mode); } void send_broadcast(BlockBroadcast broadcast, int mode) override { td::actor::send_closure(id_, &FullNodeImpl::send_broadcast, std::move(broadcast), mode); diff --git a/validator/full-node.hpp b/validator/full-node.hpp index b4c79363e5..e2d5c72b08 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -66,7 +66,7 @@ class FullNodeImpl : public FullNode { void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data); void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqnp, td::BufferSlice data); void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data); + td::BufferSlice data, int mode); void send_broadcast(BlockBroadcast broadcast, int mode); void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, @@ -144,7 +144,6 @@ class FullNodeImpl : public FullNode { FullNodeOptions opts_; std::map> private_block_overlays_; - bool broadcast_block_candidates_in_public_overlay_ = false; struct CustomOverlayInfo { CustomOverlayParams params_; diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index de48626d75..fa68580d45 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -951,6 +951,11 @@ void AcceptBlockQuery::applied() { // do not wait for answer td::actor::send_closure_later(manager_, &ValidatorManager::send_block_broadcast, std::move(b), send_broadcast_mode_); + // Do this for shard blocks later: + // td::actor::send_closure(manager_, &ValidatorManager::send_block_candidate_broadcast, id_, + // validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), + // std::move(b.data), send_broadcast_mode_); + finish_query(); } diff --git a/validator/impl/top-shard-descr.hpp b/validator/impl/top-shard-descr.hpp index 5c866a6e33..e1fd5778f0 100644 --- a/validator/impl/top-shard-descr.hpp +++ b/validator/impl/top-shard-descr.hpp @@ -74,6 +74,9 @@ class ShardTopBlockDescrQ final : public ShardTopBlockDescrQBase { std::size_t size() const { return chain_blk_ids_.size(); } + const std::vector& get_chain_blocks() const override { + return chain_blk_ids_; + } UnixTime generated_at() const override { return gen_utime_; } diff --git a/validator/interfaces/shard-block.h b/validator/interfaces/shard-block.h index e88a970bba..6e31cc136c 100644 --- a/validator/interfaces/shard-block.h +++ b/validator/interfaces/shard-block.h @@ -35,6 +35,7 @@ class ShardTopBlockDescription : public td::CntObject { virtual bool after_split() const = 0; virtual bool after_merge() const = 0; virtual CatchainSeqno catchain_seqno() const = 0; + virtual const std::vector& get_chain_blocks() const = 0; virtual UnixTime generated_at() const = 0; // if method returns false this shard block description is discarded diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 1f08c4ef5c..077ce7849a 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -104,7 +104,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::Promise promise) = 0; virtual void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) = 0; + td::BufferSlice data, int mode) = 0; virtual void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 8e4a4d0868..82081cd11a 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -64,7 +64,7 @@ void ValidatorManagerImpl::validate_block(ReceivedBlock block, td::Promise promise) { +void ValidatorManagerImpl::new_block_broadcast(BlockBroadcast broadcast, td::Promise promise) { UNREACHABLE(); } @@ -276,7 +276,8 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } } -void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { +void ValidatorManagerImpl::new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) { if (!last_masterchain_block_handle_) { shard_blocks_raw_.push_back(std::move(data)); return; @@ -528,7 +529,7 @@ void ValidatorManagerImpl::get_shard_blocks(BlockIdExt masterchain_block_id, } if (!shard_blocks_raw_.empty()) { for (auto &raw : shard_blocks_raw_) { - new_shard_block(BlockIdExt{}, 0, std::move(raw)); + new_shard_block_description_broadcast(BlockIdExt{}, 0, std::move(raw)); } shard_blocks_raw_.clear(); } @@ -786,8 +787,9 @@ void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate can } void ValidatorManagerImpl::send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, - td::uint32 validator_set_hash, td::BufferSlice data) { - callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data)); + td::uint32 validator_set_hash, td::BufferSlice data, + int mode) { + callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data), mode); } void ValidatorManagerImpl::write_handle(BlockHandle handle, td::Promise promise) { @@ -947,7 +949,7 @@ void ValidatorManagerImpl::update_shard_blocks() { } if (!shard_blocks_raw_.empty()) { for (auto &raw : shard_blocks_raw_) { - new_shard_block(BlockIdExt{}, 0, std::move(raw)); + new_shard_block_description_broadcast(BlockIdExt{}, 0, std::move(raw)); } shard_blocks_raw_.clear(); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 8526ffa4d6..cdf6c1caef 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -96,7 +96,7 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } void validate_block(ReceivedBlock block, td::Promise promise) override; - void prevalidate_block(BlockBroadcast broadcast, td::Promise promise) override; + void new_block_broadcast(BlockBroadcast broadcast, td::Promise promise) override; //void create_validate_block(BlockId block, td::BufferSlice data, td::Promise promise) = 0; void sync_complete(td::Promise promise) override; @@ -133,8 +133,9 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } void new_ihr_message(td::BufferSlice data) override; - void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; - void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override { + void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) override; + void new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) override { } void add_ext_server_id(adnl::AdnlNodeIdShort id) override { @@ -186,7 +187,7 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::Promise promise) override; void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) override; + td::BufferSlice data, int mode) override; void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index b2bd9250b8..fbb40901f5 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -107,7 +107,7 @@ class ValidatorManagerImpl : public ValidatorManager { void validate_block(ReceivedBlock block, td::Promise promise) override { UNREACHABLE(); } - void prevalidate_block(BlockBroadcast broadcast, td::Promise promise) override { + void new_block_broadcast(BlockBroadcast broadcast, td::Promise promise) override { UNREACHABLE(); } @@ -154,10 +154,11 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } void new_ihr_message(td::BufferSlice data) override; - void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { + void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) override { UNREACHABLE(); } - void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override { + void new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) override { UNREACHABLE(); } @@ -228,8 +229,8 @@ class ValidatorManagerImpl : public ValidatorManager { promise.set_value(td::Unit()); } void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) { - callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data)); + td::BufferSlice data, int mode) { + callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data), mode); } void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, diff --git a/validator/manager.cpp b/validator/manager.cpp index 85f3a30abc..a94f9c07ca 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -210,7 +210,7 @@ void ValidatorManagerImpl::validate_block(ReceivedBlock block, td::Promise promise) { +void ValidatorManagerImpl::new_block_broadcast(BlockBroadcast broadcast, td::Promise promise) { if (!started_) { promise.set_error(td::Status::Error(ErrorCode::notready, "node not started")); return; @@ -487,20 +487,22 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } } -void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - if (!is_validator() && !cached_block_candidates_.count(block_id)) { +void ValidatorManagerImpl::new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) { + if (!last_masterchain_block_handle_ || !started_) { + VLOG(VALIDATOR_DEBUG) << "dropping shard block description broadcast: not inited"; return; } - if (!last_masterchain_block_handle_) { - VLOG(VALIDATOR_DEBUG) << "dropping top shard block broadcast: not inited"; + if (!is_validator() && !opts_->need_monitor(block_id.shard_full(), last_masterchain_state_)) { return; } - if (!started_) { + if (cached_checked_shard_block_descriptions_.contains(block_id)) { + VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block description broadcast"; return; } auto it = shard_blocks_.find(ShardTopBlockDescriptionId{block_id.shard_full(), cc_seqno}); if (it != shard_blocks_.end() && block_id.id.seqno <= it->second->block_id().id.seqno) { - VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; + VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block description broadcast"; return; } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { @@ -514,25 +516,37 @@ void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc actor_id(this), td::Timestamp::in(2.0), std::move(P)); } -void ValidatorManagerImpl::new_block_candidate(BlockIdExt block_id, td::BufferSlice data) { - if (!last_masterchain_block_handle_) { +void ValidatorManagerImpl::new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) { + if (!last_masterchain_block_handle_ || !started_) { VLOG(VALIDATOR_DEBUG) << "dropping top shard block broadcast: not inited"; return; } - if (!started_) { - return; - } if (!need_monitor(block_id.shard_full())) { VLOG(VALIDATOR_DEBUG) << "dropping block candidate broadcast: not monitoring shard"; return; } - add_cached_block_candidate(ReceivedBlock{block_id, std::move(data)}); + add_cached_block_data(block_id, std::move(data)); } void ValidatorManagerImpl::add_shard_block_description(td::Ref desc) { + for (const BlockIdExt &block_id : desc->get_chain_blocks()) { + if (cached_checked_shard_block_descriptions_.put(block_id, td::Unit())) { + if (cached_block_data_.contains(block_id)) { + wait_block_data_short(block_id, 0, td::Timestamp::in(60.0), [id = block_id](td::Result> R) { + if (R.is_error()) { + LOG(WARNING) << "Failed to store block data from cache for new shard block description " << id.to_str() + << " : " << R.move_as_error(); + } + }); + } + } + } if (!desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { return; } + if (!is_validator()) { + return; + } auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second->block_id().id.seqno) { VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; @@ -555,33 +569,41 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refsecond.actor_, &WaitBlockData::loaded_block_data, r_block.move_as_ok()); - } + td::BufferSlice& block_data = cached_block_data_.get(block_id); + if (!block_data.empty()) { + return; + } + block_data = std::move(data); + { + auto it = wait_block_data_.find(block_id); + if (it != wait_block_data_.end()) { + auto r_block = create_block(ReceivedBlock{block_id, block_data.clone()}); + if (r_block.is_ok()) { + td::actor::send_closure(it->second.actor_, &WaitBlockData::loaded_block_data, r_block.move_as_ok()); + } else { + LOG(WARNING) << "Failed to parse cached block " << block_id.to_str() << " : " << r_block.move_as_error(); } } - { - auto it = wait_state_.find(id); - if (it != wait_state_.end()) { - // Proof link is not ready at this point, but this will force WaitBlockState to redo send_get_proof_link_request - td::actor::send_closure(it->second.actor_, &WaitBlockState::after_get_proof_link); - } + } + { + auto it = wait_state_.find(block_id); + if (it != wait_state_.end()) { + // Proof link is not ready at this point, but this will force WaitBlockState to redo send_get_proof_link_request + td::actor::send_closure(it->second.actor_, &WaitBlockState::after_get_proof_link); } } - if (cached_block_candidates_lru_.size() > max_cached_candidates()) { - CHECK(cached_block_candidates_.erase(cached_block_candidates_lru_.front())); - cached_block_candidates_lru_.pop_front(); + + if (cached_checked_shard_block_descriptions_.contains(block_id)) { + wait_block_data_short(block_id, 0, td::Timestamp::in(60.0), [id = block_id](td::Result> R) { + if (R.is_error()) { + LOG(WARNING) << "Failed to store block data from cache for checked shard block description " << id.to_str() + << " : " << R.move_as_error(); + } + }); } } @@ -1058,9 +1080,8 @@ void ValidatorManagerImpl::get_block_candidate_from_db(PublicKey source, BlockId } void ValidatorManagerImpl::get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) { - auto it = cached_block_candidates_.find(id); - if (it != cached_block_candidates_.end()) { - promise.set_result(it->second.data.clone()); + if (auto cached = cached_block_data_.get_if_exists(id, false)) { + promise.set_result(cached->clone()); return; } td::actor::send_closure(db_, &Db::get_block_candidate_by_block_id, id, @@ -1307,14 +1328,15 @@ void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate can PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}, candidate.collated_file_hash); } if (!id.is_masterchain()) { - add_cached_block_candidate(ReceivedBlock{id, candidate.data.clone()}); + add_cached_block_data(id, candidate.data.clone()); } td::actor::send_closure(db_, &Db::store_block_candidate, std::move(candidate), std::move(promise)); } void ValidatorManagerImpl::send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, - td::uint32 validator_set_hash, td::BufferSlice data) { - callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data)); + td::uint32 validator_set_hash, td::BufferSlice data, + int mode) { + callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data), mode); } void ValidatorManagerImpl::write_handle(BlockHandle handle, td::Promise promise) { @@ -1570,13 +1592,11 @@ void ValidatorManagerImpl::get_last_liteserver_state_block( void ValidatorManagerImpl::send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) { - { - auto it = cached_block_candidates_.find(id); - if (it != cached_block_candidates_.end()) { - LOG(DEBUG) << "send_get_block_request: got result from candidates cache for " << id.to_str(); - return promise.set_value(it->second.clone()); - } + if (auto cached = cached_block_data_.get_if_exists(id, false)) { + LOG(DEBUG) << "send_get_block_request: got result from block data cache for " << id.to_str(); + return promise.set_value(ReceivedBlock{id, cached->clone()}); } + LOG(ERROR) << "XXXX download " << id.id.to_str(); callback_->download_block(id, priority, td::Timestamp::in(10.0), std::move(promise)); } @@ -1600,19 +1620,19 @@ void ValidatorManagerImpl::send_get_block_proof_request(BlockIdExt block_id, td: void ValidatorManagerImpl::send_get_block_proof_link_request(BlockIdExt block_id, td::uint32 priority, td::Promise promise) { if (!block_id.is_masterchain()) { - auto it = cached_block_candidates_.find(block_id); - if (it != cached_block_candidates_.end()) { - // Proof link can be created from the cached block candidate - LOG(DEBUG) << "send_get_block_proof_link_request: creating proof link from cached caniddate for " + if (auto cached = cached_block_data_.get_if_exists(block_id, false)) { + // Proof link can be created from the cached block data + LOG(DEBUG) << "send_get_block_proof_link_request: creating proof link from cached block data for " << block_id.to_str(); - TRY_RESULT_PROMISE_PREFIX(promise, block_root, vm::std_boc_deserialize(it->second.data), + TRY_RESULT_PROMISE_PREFIX(promise, block_root, vm::std_boc_deserialize(*cached), "failed to create proof link: "); - TRY_RESULT_PROMISE_PREFIX(promise, proof_link, WaitBlockData::generate_proof_link(it->second.id, block_root), + TRY_RESULT_PROMISE_PREFIX(promise, proof_link, WaitBlockData::generate_proof_link(block_id, block_root), "failed to create proof link: "); promise.set_result(std::move(proof_link)); return; } } + LOG(ERROR) << "XXXX download link " << block_id.id.to_str(); callback_->download_block_proof_link(block_id, priority, td::Timestamp::in(10.0), std::move(promise)); } diff --git a/validator/manager.hpp b/validator/manager.hpp index c406d36b04..a321684b77 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -36,6 +36,7 @@ #include "queue-size-counter.hpp" #include "validator-telemetry.hpp" #include "impl/candidates-buffer.hpp" +#include "td/utils/LRUCache.h" #include #include @@ -234,8 +235,8 @@ class ValidatorManagerImpl : public ValidatorManager { // DATA FOR COLLATOR std::map> shard_blocks_; - std::map cached_block_candidates_; - std::list cached_block_candidates_lru_; + td::LRUCache cached_block_data_{/* max_size = */ 128}; + td::LRUCache cached_checked_shard_block_descriptions_{/* max_size = */ 1024}; struct ExtMessages { std::map, std::unique_ptr>> ext_messages_; @@ -366,7 +367,7 @@ class ValidatorManagerImpl : public ValidatorManager { void validate_block_proof_rel(BlockIdExt block_id, BlockIdExt rel_block_id, td::BufferSlice proof, td::Promise promise) override; void validate_block(ReceivedBlock block, td::Promise promise) override; - void prevalidate_block(BlockBroadcast broadcast, td::Promise promise) override; + void new_block_broadcast(BlockBroadcast broadcast, td::Promise promise) override; void validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno); //void create_validate_block(BlockId block, td::BufferSlice data, td::Promise promise) = 0; @@ -396,8 +397,9 @@ class ValidatorManagerImpl : public ValidatorManager { void check_external_message(td::BufferSlice data, td::Promise> promise) override; void new_ihr_message(td::BufferSlice data) override; - void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; - void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override; + void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) override; + void new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) override; void add_ext_server_id(adnl::AdnlNodeIdShort id) override; void add_ext_server_port(td::uint16 port) override; @@ -445,7 +447,7 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::Promise promise) override; void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) override; + td::BufferSlice data, int mode) override; void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; @@ -545,7 +547,7 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_shard_block_description(td::Ref desc); - void add_cached_block_candidate(ReceivedBlock block); + void add_cached_block_data(BlockIdExt block_id, td::BufferSlice data); void register_block_handle(BlockHandle handle); @@ -696,9 +698,6 @@ class ValidatorManagerImpl : public ValidatorManager { double max_mempool_num() const { return opts_->max_mempool_num(); } - size_t max_cached_candidates() const { - return 128; - } static double max_ext_msg_per_addr_time_window() { return 10.0; } diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 110ccd8138..498e5ce20e 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -185,6 +185,7 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so auto block = block_data.size() > 0 ? create_block(next_block_id, std::move(block_data)).move_as_ok() : td::Ref{}; + // OLD BROADCAST BEHAVIOR: // Creator of the block sends broadcast to public overlays // Creator of the block sends broadcast to private block overlay unless candidate broadcast was sent // Any node sends broadcast to custom overlays unless candidate broadcast was sent @@ -200,6 +201,23 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so send_broadcast_mode |= fullnode::FullNode::broadcast_mode_custom; } + // NEW BROADCAST BEHAVIOR (activate later): + // Masterchain block are broadcasted as Block Broadcast (with signatures). Shard blocks are broadcasted as Block Candidate Broadcast (only block data). + // Public and private overlays: creator sends masterchain blocks, all validators send shard blocks. + // Custom overlays: all nodes send all blocks. + // If the block was broadcasted earlier as a candidate (to private and custom overlays), the broadcast is not repeated. + /*int send_broadcast_mode = 0; + bool sent_candidate = sent_candidate_broadcasts_.contains(next_block_id); + if (!shard_.is_masterchain() || source_info.source.compute_short_id() == local_id_) { + send_broadcast_mode |= fullnode::FullNode::broadcast_mode_public; + if (!sent_candidate) { + send_broadcast_mode |= fullnode::FullNode::broadcast_mode_private_block; + } + } + if (!sent_candidate) { + send_broadcast_mode |= fullnode::FullNode::broadcast_mode_custom; + }*/ + auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), block_id = next_block_id, prev = prev_block_ids_, promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -516,6 +534,15 @@ void ValidatorGroup::get_validator_group_info_for_litequery_cont( promise.set_result(std::move(result)); } +void ValidatorGroup::send_block_candidate_broadcast(BlockIdExt id, td::BufferSlice data) { + if (sent_candidate_broadcasts_.insert(id).second) { + td::actor::send_closure( + manager_, &ValidatorManager::send_block_candidate_broadcast, id, validator_set_->get_catchain_seqno(), + validator_set_->get_validator_set_hash(), std::move(data), + fullnode::FullNode::broadcast_mode_private_block | fullnode::FullNode::broadcast_mode_custom); + } +} + } // namespace validator } // namespace ton diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index db55614f4b..fee7ee5800 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -165,13 +165,7 @@ class ValidatorGroup : public td::actor::Actor { std::set sent_candidate_broadcasts_; - void send_block_candidate_broadcast(BlockIdExt id, td::BufferSlice data) { - if (sent_candidate_broadcasts_.insert(id).second) { - td::actor::send_closure(manager_, &ValidatorManager::send_block_candidate_broadcast, id, - validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), - std::move(data)); - } - } + void send_block_candidate_broadcast(BlockIdExt id, td::BufferSlice data); }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 7868bc16f8..dc0f5761ad 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -178,7 +178,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; virtual void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) = 0; + td::BufferSlice data, int mode) = 0; virtual void send_broadcast(BlockBroadcast broadcast, int mode) = 0; virtual void download_block(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; @@ -216,7 +216,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void validate_block_proof_rel(BlockIdExt block_id, BlockIdExt rel_block_id, td::BufferSlice proof, td::Promise promise) = 0; virtual void validate_block(ReceivedBlock block, td::Promise promise) = 0; - virtual void prevalidate_block(BlockBroadcast broadcast, td::Promise promise) = 0; + virtual void new_block_broadcast(BlockBroadcast broadcast, td::Promise promise) = 0; //virtual void create_validate_block(BlockId block, td::BufferSlice data, td::Promise promise) = 0; virtual void sync_complete(td::Promise promise) = 0; @@ -252,8 +252,9 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void new_external_message(td::BufferSlice data, int priority) = 0; virtual void check_external_message(td::BufferSlice data, td::Promise> promise) = 0; virtual void new_ihr_message(td::BufferSlice data) = 0; - virtual void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; - virtual void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) = 0; + virtual void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) = 0; + virtual void new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) = 0; virtual void add_ext_server_id(adnl::AdnlNodeIdShort id) = 0; virtual void add_ext_server_port(td::uint16 port) = 0; From 5a86f0ac2d5216a64c33e14f16cb2f3a360124ff Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 9 May 2025 19:12:21 +0300 Subject: [PATCH 240/524] Fix calculating storage dict, simplify storage stat code (#1643) --- crypto/block/account-storage-stat.cpp | 47 +++++++++++++-------------- crypto/block/account-storage-stat.h | 3 +- crypto/block/transaction.cpp | 32 +++++++++++++----- 3 files changed, 47 insertions(+), 35 deletions(-) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index 56cc8f1e1b..82359f827f 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -63,7 +63,7 @@ td::Status AccountStorageStat::replace_roots(std::vector> new_root for (const Ref& root : to_add) { TRY_RESULT(info, add_cell(root)); if (check_merkle_depth && info.max_merkle_depth > MAX_MERKLE_DEPTH) { - return td::Status::Error("too big Merkle depth"); + return td::Status::Error(errorcode_limits_exceeded, "too big Merkle depth"); } } for (const Ref& root : to_del) { @@ -92,10 +92,10 @@ void AccountStorageStat::add_hint(const td::HashSet& hint) { return; } } + e.max_merkle_depth = 0; if (hint.contains(cell->get_hash())) { bool spec; vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); - e.size_bits = cs.size(); for (unsigned i = 0; i < cs.size_refs(); ++i) { dfs(cs.prefetch_ref(i), false); } @@ -108,11 +108,11 @@ void AccountStorageStat::add_hint(const td::HashSet& hint) { td::Result AccountStorageStat::add_cell(const Ref& cell) { Entry& e = get_entry(cell); - if (!e.exists_known || e.refcnt_diff < 0) { + if (!e.exists_known) { TRY_STATUS(fetch_from_dict(e)); } ++e.refcnt_diff; - if (e.exists || e.refcnt_diff > 1 || (e.refcnt && e.refcnt.value() + e.refcnt_diff != 1)) { + if (e.exists || e.refcnt_diff > 1) { if (!e.max_merkle_depth) { TRY_STATUS(fetch_from_dict(e)); if (!e.max_merkle_depth) { @@ -125,7 +125,6 @@ td::Result AccountStorageStat::add_cell(const Ref< td::uint32 max_merkle_depth = 0; bool spec; vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); - e.size_bits = cs.size(); for (unsigned i = 0; i < cs.size_refs(); ++i) { TRY_RESULT(info, add_cell(cs.prefetch_ref(i))); max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); @@ -137,6 +136,8 @@ td::Result AccountStorageStat::add_cell(const Ref< max_merkle_depth = std::min(max_merkle_depth, MERKLE_DEPTH_LIMIT); Entry& e2 = get_entry(cell); e2.max_merkle_depth = max_merkle_depth; + ++total_cells_; + total_bits_ += cs.size(); return CellInfo{max_merkle_depth}; } @@ -158,16 +159,28 @@ td::Status AccountStorageStat::remove_cell(const Ref& cell) { } bool spec; vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); - e.size_bits = cs.size(); for (unsigned i = 0; i < cs.size_refs(); ++i) { TRY_STATUS(remove_cell(cs.prefetch_ref(i))); } + --total_cells_; + total_bits_ -= cs.size(); return td::Status::OK(); } td::Result> AccountStorageStat::get_dict_root() { if (!dict_up_to_date_) { std::vector>> values; + if (parent_ && !parent_->dict_up_to_date_) { + for (auto& [_, ep] : parent_->cache_) { + if (ep.dict_refcnt_diff == 0) { + continue; + } + Entry& e = cache_[ep.hash]; + if (!e.inited) { + e = ep; + } + } + } for (auto& [_, e] : cache_) { if (e.dict_refcnt_diff == 0) { continue; @@ -259,25 +272,9 @@ td::Status AccountStorageStat::finalize_entry(Entry& e) { e.refcnt.value() += e.refcnt_diff; e.dict_refcnt_diff += e.refcnt_diff; e.refcnt_diff = 0; - if (e.refcnt.value() == 0) { - if (!e.size_bits) { - return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown cell bits"); - } - --total_cells_; - total_bits_ -= e.size_bits.value(); - e.exists = false; - } else { - if (!e.exists) { - if (!e.size_bits) { - return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown cell bits"); - } - ++total_cells_; - total_bits_ += e.size_bits.value(); - } - e.exists = true; - if (!e.max_merkle_depth) { - return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown merkle depth"); - } + e.exists = (e.refcnt.value() != 0); + if (e.exists && !e.max_merkle_depth) { + return td::Status::Error(PSTRING() << "Error on entry " << e.hash.to_hex() << " : unknown merkle depth"); } return td::Status::OK(); } diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h index 645f0744d8..e939a0ff51 100644 --- a/crypto/block/account-storage-stat.h +++ b/crypto/block/account-storage-stat.h @@ -59,6 +59,8 @@ class AccountStorageStat { void apply_child_stat(AccountStorageStat &&child); + static constexpr int errorcode_limits_exceeded = 999; + private: vm::Dictionary dict_; bool dict_up_to_date_ = true; @@ -76,7 +78,6 @@ class AccountStorageStat { struct Entry { bool inited = false; vm::CellHash hash; - td::optional size_bits; bool exists_known = false; bool exists = false; td::optional refcnt, max_merkle_depth; diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index b892b5a216..1b6c3582fc 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -1930,12 +1930,17 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { ap.action_fine = td::zero_refint(); td::Ref old_code = new_code, old_data = new_data, old_library = new_library; - auto enforce_state_limits = [&]() { + // 1 - ok, 0 - limits exceeded, -1 - fatal error + auto enforce_state_limits = [&]() -> int { if (account.is_special) { - return true; + return 1; } auto S = check_state_limits(cfg.size_limits); if (S.is_error()) { + if (S.code() != AccountStorageStat::errorcode_limits_exceeded) { + LOG(ERROR) << "Account storage stat error: " << S.move_as_error(); + return -1; + } // Rollback changes to state, fail action phase LOG(INFO) << "Account state size exceeded limits: " << S.move_as_error(); new_account_storage_stat = {}; @@ -1944,9 +1949,9 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { new_library = old_library; ap.result_code = 50; ap.state_exceeds_limits = true; - return false; + return 0; } - return true; + return 1; }; int n = 0; @@ -2057,7 +2062,9 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { } LOG(DEBUG) << "invalid action " << ap.result_arg << " in action list: error code " << ap.result_code; // This is required here because changes to libraries are applied even if action phase fails - enforce_state_limits(); + if (enforce_state_limits() == -1) { + return false; + } if (cfg.action_fine_enabled) { ap.action_fine = std::min(ap.action_fine, balance.grams); ap.total_action_fees = ap.action_fine; @@ -2079,7 +2086,11 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { new_code = ap.new_code; } new_data = compute_phase->new_data; // tentative persistent data update applied - if (!enforce_state_limits()) { + int res = enforce_state_limits(); + if (res == -1) { + return false; + } + if (res == 0) { if (cfg.extra_currency_v2) { end_lt = ap.end_lt = start_lt + 1; if (cfg.action_fine_enabled) { @@ -3054,7 +3065,8 @@ static td::uint32 get_public_libraries_diff_count(const td::Ref& old_l * * @returns A `td::Status` indicating the result of the check. * - If the state limits are within the allowed range, returns OK. - * - If the state limits exceed the maximum allowed range, returns an error. + * - If the state limits exceed the maximum allowed range, returns an error with AccountStorageStat::errorcode_limits_exceeded code. + * - If an error occurred during storage stat calculation, returns other error. */ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, bool update_storage_stat) { auto cell_equal = [](const td::Ref& a, const td::Ref& b) -> bool { @@ -3079,7 +3091,8 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, if (storage_stat.get_total_cells() > size_limits.max_acc_state_cells || storage_stat.get_total_bits() > size_limits.max_acc_state_bits) { - return td::Status::Error(PSTRING() << "account state is too big: cells=" << storage_stat.get_total_cells() + return td::Status::Error(AccountStorageStat::errorcode_limits_exceeded, + PSTRING() << "account state is too big: cells=" << storage_stat.get_total_cells() << ", bits=" << storage_stat.get_total_bits() << " (max cells=" << size_limits.max_acc_state_cells << ", max bits=" << size_limits.max_acc_state_bits << ")"); @@ -3087,7 +3100,8 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, if (account.is_masterchain() && !cell_equal(account.library, new_library)) { auto libraries_count = get_public_libraries_count(new_library); if (libraries_count > size_limits.max_acc_public_libraries) { - return td::Status::Error(PSTRING() << "too many public libraries: " << libraries_count << " (max " + return td::Status::Error(AccountStorageStat::errorcode_limits_exceeded, + PSTRING() << "too many public libraries: " << libraries_count << " (max " << size_limits.max_acc_public_libraries << ")"); } } From 9e5109d56bc4f2345a00b2271c3711103841b799 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 9 May 2025 19:13:21 +0300 Subject: [PATCH 241/524] Fix type check in INMSGPARAM (#1652) --- crypto/vm/tonops.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index b62e5fb991..135ed70268 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -166,6 +166,9 @@ int exec_get_in_msg_param(VmState* st, unsigned idx, const char* name) { VM_LOG(st) << "execute " << name; } Ref t = get_param(st, inmsgparams_idx).as_tuple(); + if (t.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; + } st->get_stack().push(tuple_index(t, idx)); return 0; } From a6449abf21a88c39000d4e3d9183ee7906920b61 Mon Sep 17 00:00:00 2001 From: Evgeny Kapun Date: Tue, 13 May 2025 09:18:59 +0300 Subject: [PATCH 242/524] Adnl optimizations (#1620) * Remove AdnlStaticNodesManager * Remove AdnlPeer --------- Co-authored-by: EmelyanenkoK --- adnl/CMakeLists.txt | 3 - adnl/adnl-peer-table.cpp | 155 ++++++++++++++++++++++++-------- adnl/adnl-peer-table.h | 4 +- adnl/adnl-peer-table.hpp | 31 ++++--- adnl/adnl-peer.cpp | 178 +------------------------------------ adnl/adnl-peer.h | 36 -------- adnl/adnl-peer.hpp | 59 +----------- adnl/adnl-static-nodes.cpp | 55 ------------ adnl/adnl-static-nodes.h | 43 --------- adnl/adnl-static-nodes.hpp | 42 --------- 10 files changed, 140 insertions(+), 466 deletions(-) delete mode 100644 adnl/adnl-static-nodes.cpp delete mode 100644 adnl/adnl-static-nodes.h delete mode 100644 adnl/adnl-static-nodes.hpp diff --git a/adnl/CMakeLists.txt b/adnl/CMakeLists.txt index f1d1bfbfb4..3604dfb3a8 100644 --- a/adnl/CMakeLists.txt +++ b/adnl/CMakeLists.txt @@ -23,8 +23,6 @@ set(ADNL_HEADERS adnl-peer.h adnl-peer.hpp adnl-query.h - adnl-static-nodes.h - adnl-static-nodes.hpp adnl-proxy-types.h adnl-proxy-types.hpp adnl.h @@ -46,7 +44,6 @@ set(ADNL_SOURCE adnl-peer.cpp adnl-query.cpp adnl-channel.cpp - adnl-static-nodes.cpp adnl-proxy-types.cpp utils.cpp ${ADNL_HEADERS} diff --git a/adnl/adnl-peer-table.cpp b/adnl/adnl-peer-table.cpp index b8aab5671e..5a92ca1c11 100644 --- a/adnl/adnl-peer-table.cpp +++ b/adnl/adnl-peer-table.cpp @@ -84,6 +84,32 @@ void AdnlPeerTableImpl::receive_packet(td::IPAddress addr, AdnlCategoryMask cat_ << " (len=" << (data.size() + 32) << ")"; } +void AdnlPeerTableImpl::update_id(AdnlPeerTableImpl::PeerInfo &peer_info, AdnlNodeIdFull &&peer_id) { + if (peer_info.peer_id.empty()) { + peer_info.peer_id = std::move(peer_id); + for (auto &e: peer_info.peers) { + td::actor::send_closure(e.second, &AdnlPeerPair::update_peer_id, peer_info.peer_id); + } + } +} + +td::actor::ActorOwn &AdnlPeerTableImpl::get_peer_pair( + AdnlNodeIdShort peer_id, AdnlPeerTableImpl::PeerInfo &peer_info, + AdnlNodeIdShort local_id, AdnlPeerTableImpl::LocalIdInfo &local_id_info) { + auto it = peer_info.peers.find(local_id); + if (it == peer_info.peers.end()) { + it = peer_info.peers.emplace(local_id, + AdnlPeerPair::create(network_manager_, actor_id(this), + local_id_info.mode, local_id_info.local_id.get(), + dht_node_, local_id, peer_id)) + .first; + if (!peer_info.peer_id.empty()) { + td::actor::send_closure(it->second, &AdnlPeerPair::update_peer_id, peer_info.peer_id); + } + } + return it->second; +} + void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket packet, td::uint64 serialized_size) { packet.run_basic_checks().ensure(); @@ -91,8 +117,9 @@ void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket VLOG(ADNL_INFO) << this << ": dropping IN message [?->" << dst << "]: destination not set"; return; } + AdnlNodeIdShort src = packet.from_short(); - auto it = peers_.find(packet.from_short()); + auto it = peers_.find(src); if (it == peers_.end()) { if (!packet.inited_from()) { VLOG(ADNL_NOTICE) << this << ": dropping IN message [" << packet.from_short() << "->" << dst @@ -105,11 +132,7 @@ void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket return; } - it = peers_ - .emplace(packet.from_short(), - AdnlPeer::create(network_manager_, actor_id(this), dht_node_, packet.from_short())) - .first; - CHECK(it != peers_.end()); + it = peers_.try_emplace(src).first; } auto it2 = local_ids_.find(dst); @@ -118,8 +141,13 @@ void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket << "]: unknown dst (but how did we decrypt message?)"; return; } - td::actor::send_closure(it->second, &AdnlPeer::receive_packet, dst, it2->second.mode, it2->second.local_id.get(), - std::move(packet), serialized_size); + + if (packet.inited_from()) { + update_id(it->second, packet.from()); + } + + td::actor::send_closure(get_peer_pair(src, it->second, dst, it2->second), + &AdnlPeerPair::receive_packet, std::move(packet), serialized_size); } void AdnlPeerTableImpl::add_peer(AdnlNodeIdShort local_id, AdnlNodeIdFull id, AdnlAddressList addr_list) { @@ -129,31 +157,25 @@ void AdnlPeerTableImpl::add_peer(AdnlNodeIdShort local_id, AdnlNodeIdFull id, Ad auto it2 = local_ids_.find(local_id); CHECK(it2 != local_ids_.end()); - auto it = peers_.find(id_short); - if (it == peers_.end()) { - it = peers_.emplace(id_short, AdnlPeer::create(network_manager_, actor_id(this), dht_node_, id_short)).first; - CHECK(it != peers_.end()); - } - td::actor::send_closure(it->second, &AdnlPeer::update_id, std::move(id)); + auto &peer_info = peers_[id_short]; + update_id(peer_info, std::move(id)); if (!addr_list.empty()) { - td::actor::send_closure(it->second, &AdnlPeer::update_addr_list, local_id, it2->second.mode, - it2->second.local_id.get(), std::move(addr_list)); + td::actor::send_closure(get_peer_pair(id_short, peer_info, local_id, it2->second), + &AdnlPeerPair::update_addr_list, std::move(addr_list)); } } void AdnlPeerTableImpl::add_static_nodes_from_config(AdnlNodesList nodes) { - for (auto &it : nodes.nodes()) { - add_static_node(it); + for (auto &node : nodes.nodes()) { + auto id_short = node.compute_short_id(); + VLOG(ADNL_INFO) << "[staticnodes] adding static node " << id_short; + static_nodes_.emplace(id_short, std::move(node)); } } void AdnlPeerTableImpl::send_message_in(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlMessage message, td::uint32 flags) { - auto it = peers_.find(dst); - - if (it == peers_.end()) { - it = peers_.emplace(dst, AdnlPeer::create(network_manager_, actor_id(this), dht_node_, dst)).first; - } + auto &peer_info = peers_[dst]; auto it2 = local_ids_.find(src); if (it2 == local_ids_.end()) { @@ -161,8 +183,10 @@ void AdnlPeerTableImpl::send_message_in(AdnlNodeIdShort src, AdnlNodeIdShort dst return; } - td::actor::send_closure(it->second, &AdnlPeer::send_one_message, src, it2->second.mode, it2->second.local_id.get(), - OutboundAdnlMessage{std::move(message), flags}); + std::vector messages; + messages.push_back(OutboundAdnlMessage{std::move(message), flags}); + td::actor::send_closure(get_peer_pair(dst, peer_info, src, it2->second), + &AdnlPeerPair::send_messages, std::move(messages)); } void AdnlPeerTableImpl::answer_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlQueryId query_id, @@ -182,11 +206,7 @@ void AdnlPeerTableImpl::send_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, std VLOG(ADNL_WARNING) << "DUMP: " << td::buffer_to_hex(data.as_slice().truncate(128)); return; } - auto it = peers_.find(dst); - - if (it == peers_.end()) { - it = peers_.emplace(dst, AdnlPeer::create(network_manager_, actor_id(this), dht_node_, dst)).first; - } + auto &peer_info = peers_[dst]; auto it2 = local_ids_.find(src); if (it2 == local_ids_.end()) { @@ -194,8 +214,8 @@ void AdnlPeerTableImpl::send_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, std return; } - td::actor::send_closure(it->second, &AdnlPeer::send_query, src, it2->second.mode, it2->second.local_id.get(), name, - std::move(promise), timeout, std::move(data), 0); + td::actor::send_closure(get_peer_pair(dst, peer_info, src, it2->second), + &AdnlPeerPair::send_query, name, std::move(promise), timeout, std::move(data), 0); } void AdnlPeerTableImpl::add_id_ex(AdnlNodeIdFull id, AdnlAddressList addr_list, td::uint8 cat, td::uint32 mode) { @@ -247,8 +267,10 @@ void AdnlPeerTableImpl::unsubscribe(AdnlNodeIdShort dst, std::string prefix) { void AdnlPeerTableImpl::register_dht_node(td::actor::ActorId dht_node) { dht_node_ = dht_node; - for (auto &peer : peers_) { - td::actor::send_closure(peer.second, &AdnlPeer::update_dht_node, dht_node_); + for (auto &e : peers_) { + for (auto &e2 : e.second.peers) { + td::actor::send_closure(e2.second, &AdnlPeerPair::update_dht_node, dht_node_); + } } for (auto &local_id : local_ids_) { td::actor::send_closure(local_id.second.local_id, &AdnlLocalId::update_dht_node, dht_node_); @@ -332,8 +354,6 @@ void AdnlPeerTableImpl::get_addr_list_from_db(AdnlNodeIdShort local_id, AdnlNode AdnlPeerTableImpl::AdnlPeerTableImpl(std::string db_root, td::actor::ActorId keyring) { keyring_ = keyring; - static_nodes_manager_ = AdnlStaticNodesManager::create(); - if (!db_root.empty()) { db_ = AdnlDb::create(db_root + "/adnl"); } @@ -382,7 +402,64 @@ void AdnlPeerTableImpl::get_conn_ip_str(AdnlNodeIdShort l_id, AdnlNodeIdShort p_ promise.set_value("undefined"); return; } - td::actor::send_closure(it->second, &AdnlPeer::get_conn_ip_str, l_id, std::move(promise)); + auto it2 = it->second.peers.find(l_id); + if (it2 == it->second.peers.end()) { + promise.set_value("undefined"); + return; + } + td::actor::send_closure(it2->second, &AdnlPeerPair::get_conn_ip_str, std::move(promise)); +} + +void AdnlPeerTableImpl::get_stats_peer(AdnlNodeIdShort peer_id, AdnlPeerTableImpl::PeerInfo &peer_info, bool all, + td::Promise>> promise) { + class Cb : public td::actor::Actor { + public: + explicit Cb(td::Promise>> promise) + : promise_(std::move(promise)) { + } + + void got_peer_pair_stats(tl_object_ptr peer_pair) { + if (peer_pair) { + result_.push_back(std::move(peer_pair)); + } + dec_pending(); + } + + void inc_pending() { + ++pending_; + } + + void dec_pending() { + CHECK(pending_ > 0); + --pending_; + if (pending_ == 0) { + promise_.set_result(std::move(result_)); + stop(); + } + } + + private: + td::Promise>> promise_; + size_t pending_ = 1; + std::vector> result_; + }; + auto callback = td::actor::create_actor("adnlpeerstats", std::move(promise)).release(); + + for (auto &[local_id, peer_pair] : peer_info.peers) { + td::actor::send_closure(callback, &Cb::inc_pending); + td::actor::send_closure(peer_pair, &AdnlPeerPair::get_stats, all, + [local_id = local_id, peer_id = peer_id, + callback](td::Result> R) { + if (R.is_error()) { + VLOG(ADNL_NOTICE) << "failed to get stats for peer pair " << peer_id << "->" << local_id + << " : " << R.move_as_error(); + td::actor::send_closure(callback, &Cb::dec_pending); + } else { + td::actor::send_closure(callback, &Cb::got_peer_pair_stats, R.move_as_ok()); + } + }); + } + td::actor::send_closure(callback, &Cb::dec_pending); } void AdnlPeerTableImpl::get_stats(bool all, td::Promise> promise) { @@ -453,8 +530,8 @@ void AdnlPeerTableImpl::get_stats(bool all, td::Promise>> R) { if (R.is_error()) { VLOG(ADNL_NOTICE) << "failed to get stats for peer " << id << " : " << R.move_as_error(); diff --git a/adnl/adnl-peer-table.h b/adnl/adnl-peer-table.h index 055f32ac1f..13b4adb537 100644 --- a/adnl/adnl-peer-table.h +++ b/adnl/adnl-peer-table.h @@ -97,9 +97,7 @@ class AdnlPeerTable : public Adnl { td::actor::ActorId channel) = 0; virtual void unregister_channel(AdnlChannelIdShort id) = 0; - virtual void add_static_node(AdnlNode node) = 0; - virtual void del_static_node(AdnlNodeIdShort id) = 0; - virtual void get_static_node(AdnlNodeIdShort id, td::Promise promise) = 0; + virtual td::Result get_static_node(AdnlNodeIdShort id) = 0; virtual void write_new_addr_list_to_db(AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id, AdnlDbItem node, td::Promise promise) = 0; diff --git a/adnl/adnl-peer-table.hpp b/adnl/adnl-peer-table.hpp index 9ad61b6536..408f846d0d 100644 --- a/adnl/adnl-peer-table.hpp +++ b/adnl/adnl-peer-table.hpp @@ -28,7 +28,6 @@ #include "adnl-local-id.h" #include "adnl-query.h" #include "utils.hpp" -#include "adnl-static-nodes.h" #include "adnl-ext-server.h" #include "adnl-address-list.h" @@ -86,15 +85,12 @@ class AdnlPeerTableImpl : public AdnlPeerTable { void get_addr_list_from_db(AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id, td::Promise promise) override; - void add_static_node(AdnlNode node) override { - CHECK(!static_nodes_manager_.empty()); - td::actor::send_closure(static_nodes_manager_, &AdnlStaticNodesManager::add_node, std::move(node)); - } - void del_static_node(AdnlNodeIdShort id) override { - td::actor::send_closure(static_nodes_manager_, &AdnlStaticNodesManager::del_node, id); - } - void get_static_node(AdnlNodeIdShort id, td::Promise promise) override { - td::actor::send_closure(static_nodes_manager_, &AdnlStaticNodesManager::get_node, id, std::move(promise)); + td::Result get_static_node(AdnlNodeIdShort id) override { + auto it = static_nodes_.find(id); + if (it == static_nodes_.end()) { + return td::Status::Error(ErrorCode::notready, "static node not found"); + } + return it->second; } void deliver(AdnlNodeIdShort src, AdnlNodeIdShort dst, td::BufferSlice data) override; void deliver_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, td::BufferSlice data, @@ -116,20 +112,26 @@ class AdnlPeerTableImpl : public AdnlPeerTable { } private: + struct PeerInfo { + AdnlNodeIdFull peer_id; + std::map> peers; + }; + struct LocalIdInfo { td::actor::ActorOwn local_id; td::uint8 cat; td::uint32 mode; }; + td::actor::ActorId keyring_; td::actor::ActorId network_manager_; td::actor::ActorId dht_node_; - td::actor::ActorOwn static_nodes_manager_; + std::map static_nodes_; void deliver_one_message(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlMessage message); - std::map> peers_; + std::map peers_; std::map local_ids_; std::map, td::uint8>> channels_; @@ -140,6 +142,11 @@ class AdnlPeerTableImpl : public AdnlPeerTable { AdnlNodeIdShort proxy_addr_; //std::map> out_queries_; //td::uint64 last_query_id_ = 1; + + static void update_id(PeerInfo &peer_info, AdnlNodeIdFull &&peer_id); + td::actor::ActorOwn &get_peer_pair(AdnlNodeIdShort peer_id, PeerInfo &peer_info, AdnlNodeIdShort local_id, LocalIdInfo &local_id_info); + static void get_stats_peer(AdnlNodeIdShort peer_id, PeerInfo &peer_info, bool all, + td::Promise>> promise); }; inline td::StringBuilder &operator<<(td::StringBuilder &sb, const AdnlPeerTableImpl::PrintId &id) { diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index 88a38f7b04..4913216ee4 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -475,13 +475,12 @@ void AdnlPeerPairImpl::alarm_query(AdnlQueryId id) { AdnlPeerPairImpl::AdnlPeerPairImpl(td::actor::ActorId network_manager, td::actor::ActorId peer_table, td::uint32 local_mode, - td::actor::ActorId local_actor, td::actor::ActorId peer, + td::actor::ActorId local_actor, td::actor::ActorId dht_node, AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id) { network_manager_ = network_manager; peer_table_ = peer_table; local_actor_ = local_actor; - peer_ = peer; dht_node_ = dht_node; mode_ = local_mode; @@ -871,19 +870,6 @@ void AdnlPeerPairImpl::get_stats(bool all, td::Promise peer, td::actor::ActorId network_manager, td::actor::ActorId adnl) { @@ -902,171 +888,13 @@ void AdnlPeerPairImpl::conn_change_state(AdnlConnectionIdShort id, bool ready) { td::actor::ActorOwn AdnlPeerPair::create( td::actor::ActorId network_manager, td::actor::ActorId peer_table, - td::uint32 local_mode, td::actor::ActorId local_actor, td::actor::ActorId peer_actor, + td::uint32 local_mode, td::actor::ActorId local_actor, td::actor::ActorId dht_node, AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id) { auto X = td::actor::create_actor("peerpair", network_manager, peer_table, local_mode, local_actor, - peer_actor, dht_node, local_id, peer_id); + dht_node, local_id, peer_id); return td::actor::ActorOwn(std::move(X)); } -td::actor::ActorOwn AdnlPeer::create(td::actor::ActorId network_manager, - td::actor::ActorId peer_table, - td::actor::ActorId dht_node, AdnlNodeIdShort peer_id) { - auto X = td::actor::create_actor("peer", network_manager, peer_table, dht_node, peer_id); - return td::actor::ActorOwn(std::move(X)); -} - -void AdnlPeerImpl::receive_packet(AdnlNodeIdShort dst, td::uint32 dst_mode, td::actor::ActorId dst_actor, - AdnlPacket packet, td::uint64 serialized_size) { - if (packet.inited_from()) { - update_id(packet.from()); - } - - auto it = peer_pairs_.find(dst); - if (it == peer_pairs_.end()) { - auto X = AdnlPeerPair::create(network_manager_, peer_table_, dst_mode, dst_actor, actor_id(this), dht_node_, dst, - peer_id_short_); - peer_pairs_.emplace(dst, std::move(X)); - it = peer_pairs_.find(dst); - CHECK(it != peer_pairs_.end()); - - if (!peer_id_.empty()) { - td::actor::send_closure(it->second.get(), &AdnlPeerPair::update_peer_id, peer_id_); - } - } - - td::actor::send_closure(it->second.get(), &AdnlPeerPair::receive_packet, std::move(packet), serialized_size); -} - -void AdnlPeerImpl::send_messages(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - std::vector messages) { - auto it = peer_pairs_.find(src); - if (it == peer_pairs_.end()) { - auto X = AdnlPeerPair::create(network_manager_, peer_table_, src_mode, src_actor, actor_id(this), dht_node_, src, - peer_id_short_); - peer_pairs_.emplace(src, std::move(X)); - it = peer_pairs_.find(src); - CHECK(it != peer_pairs_.end()); - - if (!peer_id_.empty()) { - td::actor::send_closure(it->second.get(), &AdnlPeerPair::update_peer_id, peer_id_); - } - } - - td::actor::send_closure(it->second, &AdnlPeerPair::send_messages, std::move(messages)); -} - -void AdnlPeerImpl::send_query(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - std::string name, td::Promise promise, td::Timestamp timeout, - td::BufferSlice data, td::uint32 flags) { - auto it = peer_pairs_.find(src); - if (it == peer_pairs_.end()) { - auto X = AdnlPeerPair::create(network_manager_, peer_table_, src_mode, src_actor, actor_id(this), dht_node_, src, - peer_id_short_); - peer_pairs_.emplace(src, std::move(X)); - it = peer_pairs_.find(src); - CHECK(it != peer_pairs_.end()); - - if (!peer_id_.empty()) { - td::actor::send_closure(it->second.get(), &AdnlPeerPair::update_peer_id, peer_id_); - } - } - - td::actor::send_closure(it->second, &AdnlPeerPair::send_query, name, std::move(promise), timeout, std::move(data), - flags); -} - -void AdnlPeerImpl::del_local_id(AdnlNodeIdShort local_id) { - peer_pairs_.erase(local_id); -} - -void AdnlPeerImpl::update_dht_node(td::actor::ActorId dht_node) { - dht_node_ = dht_node; - for (auto it = peer_pairs_.begin(); it != peer_pairs_.end(); it++) { - td::actor::send_closure(it->second, &AdnlPeerPair::update_dht_node, dht_node_); - } -} - -void AdnlPeerImpl::get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise promise) { - auto it = peer_pairs_.find(l_id); - if (it == peer_pairs_.end()) { - promise.set_value("undefined"); - return; - } - - td::actor::send_closure(it->second, &AdnlPeerPair::get_conn_ip_str, std::move(promise)); -} - -void AdnlPeerImpl::update_addr_list(AdnlNodeIdShort local_id, td::uint32 local_mode, - td::actor::ActorId local_actor, AdnlAddressList addr_list) { - auto it = peer_pairs_.find(local_id); - if (it == peer_pairs_.end()) { - auto X = AdnlPeerPair::create(network_manager_, peer_table_, local_mode, local_actor, actor_id(this), dht_node_, - local_id, peer_id_short_); - peer_pairs_.emplace(local_id, std::move(X)); - it = peer_pairs_.find(local_id); - CHECK(it != peer_pairs_.end()); - - if (!peer_id_.empty()) { - td::actor::send_closure(it->second.get(), &AdnlPeerPair::update_peer_id, peer_id_); - } - } - - td::actor::send_closure(it->second, &AdnlPeerPair::update_addr_list, std::move(addr_list)); -} - -void AdnlPeerImpl::get_stats(bool all, td::Promise>> promise) { - class Cb : public td::actor::Actor { - public: - explicit Cb(td::Promise>> promise) - : promise_(std::move(promise)) { - } - - void got_peer_pair_stats(tl_object_ptr peer_pair) { - if (peer_pair) { - result_.push_back(std::move(peer_pair)); - } - dec_pending(); - } - - void inc_pending() { - ++pending_; - } - - void dec_pending() { - CHECK(pending_ > 0); - --pending_; - if (pending_ == 0) { - promise_.set_result(std::move(result_)); - stop(); - } - } - - private: - td::Promise>> promise_; - size_t pending_ = 1; - std::vector> result_; - }; - auto callback = td::actor::create_actor("adnlpeerstats", std::move(promise)).release(); - - for (auto &[local_id, peer_pair] : peer_pairs_) { - td::actor::send_closure(callback, &Cb::inc_pending); - td::actor::send_closure(peer_pair, &AdnlPeerPair::get_stats, all, - [local_id = local_id, peer_id = peer_id_short_, - callback](td::Result> R) { - if (R.is_error()) { - VLOG(ADNL_NOTICE) << "failed to get stats for peer pair " << peer_id << "->" << local_id - << " : " << R.move_as_error(); - td::actor::send_closure(callback, &Cb::dec_pending); - } else { - td::actor::send_closure(callback, &Cb::got_peer_pair_stats, R.move_as_ok()); - } - }); - } - td::actor::send_closure(callback, &Cb::dec_pending); -} - - void AdnlPeerPairImpl::got_data_from_db(td::Result R) { received_from_db_ = false; if (R.is_error()) { diff --git a/adnl/adnl-peer.h b/adnl/adnl-peer.h index 1215f71dab..4d6812a891 100644 --- a/adnl/adnl-peer.h +++ b/adnl/adnl-peer.h @@ -64,46 +64,10 @@ class AdnlPeerPair : public td::actor::Actor { static td::actor::ActorOwn create(td::actor::ActorId network_manager, td::actor::ActorId peer_table, td::uint32 local_mode, td::actor::ActorId local_actor, - td::actor::ActorId peer_actor, td::actor::ActorId dht_node, AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id); }; -class AdnlPeer : public td::actor::Actor { - public: - virtual void receive_packet(AdnlNodeIdShort dst, td::uint32 dst_mode, td::actor::ActorId dst_actor, - AdnlPacket message, td::uint64 serialized_size) = 0; - virtual void send_messages(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - std::vector messages) = 0; - virtual void send_query(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - std::string name, td::Promise promise, td::Timestamp timeout, - td::BufferSlice data, td::uint32 flags) = 0; - void send_one_message(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - OutboundAdnlMessage message) { - std::vector vec; - vec.push_back(std::move(message)); - send_messages(src, src_mode, src_actor, std::move(vec)); - } - - void send_message(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - td::BufferSlice data, td::uint32 flags) { - auto M = OutboundAdnlMessage{adnlmessage::AdnlMessageCustom{std::move(data)}, flags}; - send_one_message(src, src_mode, src_actor, std::move(M)); - } - - static td::actor::ActorOwn create(td::actor::ActorId network_manager, - td::actor::ActorId peer_table, - td::actor::ActorId dht_node, AdnlNodeIdShort peer_id); - - virtual void del_local_id(AdnlNodeIdShort local_id) = 0; - virtual void update_id(AdnlNodeIdFull id) = 0; - virtual void update_addr_list(AdnlNodeIdShort local_id, td::uint32 local_mode, - td::actor::ActorId local_actor, AdnlAddressList addr_list) = 0; - virtual void update_dht_node(td::actor::ActorId dht_node) = 0; - virtual void get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise promise) = 0; - virtual void get_stats(bool all, td::Promise>> promise) = 0; -}; - } // namespace adnl } // namespace ton diff --git a/adnl/adnl-peer.hpp b/adnl/adnl-peer.hpp index 243974ba12..b1d9835789 100644 --- a/adnl/adnl-peer.hpp +++ b/adnl/adnl-peer.hpp @@ -60,7 +60,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair { AdnlPeerPairImpl(td::actor::ActorId network_manager, td::actor::ActorId peer_table, td::uint32 local_mode, td::actor::ActorId local_actor, - td::actor::ActorId peer, td::actor::ActorId dht_node, AdnlNodeIdShort local_id, + td::actor::ActorId dht_node, AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id); void start_up() override; void alarm() override; @@ -209,7 +209,6 @@ class AdnlPeerPairImpl : public AdnlPeerPair { td::actor::ActorId network_manager_; td::actor::ActorId peer_table_; td::actor::ActorId local_actor_; - td::actor::ActorId peer_; td::actor::ActorId dht_node_; td::uint32 priority_ = 0; @@ -287,68 +286,12 @@ class AdnlPeerPairImpl : public AdnlPeerPair { void prepare_packet_stats(); }; -class AdnlPeerImpl : public AdnlPeer { - public: - void receive_packet(AdnlNodeIdShort dst, td::uint32 dst_mode, td::actor::ActorId dst_actor, - AdnlPacket packet, td::uint64 serialized_size) override; - void send_messages(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - std::vector messages) override; - void send_query(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, std::string name, - td::Promise promise, td::Timestamp timeout, td::BufferSlice data, - td::uint32 flags) override; - - void del_local_id(AdnlNodeIdShort local_id) override; - void update_id(AdnlNodeIdFull id) override; - void update_addr_list(AdnlNodeIdShort local_id, td::uint32 local_mode, td::actor::ActorId local_actor, - AdnlAddressList addr_list) override; - void update_dht_node(td::actor::ActorId dht_node) override; - void get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise promise) override; - void get_stats(bool all, td::Promise>> promise) override; - //void check_signature(td::BufferSlice data, td::BufferSlice signature, td::Promise promise) override; - - AdnlPeerImpl(td::actor::ActorId network_manager, td::actor::ActorId peer_table, - td::actor::ActorId dht_node, AdnlNodeIdShort peer_id) - : peer_id_short_(peer_id), dht_node_(dht_node), peer_table_(peer_table), network_manager_(network_manager) { - } - - struct PrintId { - AdnlNodeIdShort peer_id; - }; - - PrintId print_id() const { - return PrintId{peer_id_short_}; - } - - private: - AdnlNodeIdShort peer_id_short_; - AdnlNodeIdFull peer_id_; - std::map> peer_pairs_; - td::actor::ActorId dht_node_; - td::actor::ActorId peer_table_; - td::actor::ActorId network_manager_; -}; - } // namespace adnl } // namespace ton namespace td { -inline td::StringBuilder &operator<<(td::StringBuilder &sb, const ton::adnl::AdnlPeerImpl::PrintId &id) { - sb << "[peer " << id.peer_id << "]"; - return sb; -} - -inline td::StringBuilder &operator<<(td::StringBuilder &sb, const ton::adnl::AdnlPeerImpl &peer) { - sb << peer.print_id(); - return sb; -} - -inline td::StringBuilder &operator<<(td::StringBuilder &sb, const ton::adnl::AdnlPeerImpl *peer) { - sb << peer->print_id(); - return sb; -} - inline td::StringBuilder &operator<<(td::StringBuilder &sb, const ton::adnl::AdnlPeerPairImpl::PrintId &id) { sb << "[peerpair " << id.peer_id << "-" << id.local_id << "]"; return sb; diff --git a/adnl/adnl-static-nodes.cpp b/adnl/adnl-static-nodes.cpp deleted file mode 100644 index 251715be89..0000000000 --- a/adnl/adnl-static-nodes.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "adnl-static-nodes.h" -#include "adnl-static-nodes.hpp" - -#include "utils.hpp" - -namespace ton { - -namespace adnl { - -void AdnlStaticNodesManagerImpl::add_node(AdnlNode node) { - auto id_short = node.compute_short_id(); - VLOG(ADNL_INFO) << "[staticnodes] adding static node " << id_short; - - nodes_.emplace(id_short, std::move(node)); -} - -void AdnlStaticNodesManagerImpl::del_node(AdnlNodeIdShort id) { - nodes_.erase(id); -} - -td::Result AdnlStaticNodesManagerImpl::get_node(AdnlNodeIdShort id) { - auto it = nodes_.find(id); - if (it == nodes_.end()) { - return td::Status::Error(ErrorCode::notready, "static node not found"); - } - - return it->second; -} - -td::actor::ActorOwn AdnlStaticNodesManager::create() { - auto X = td::actor::create_actor("staticnodesmanager"); - return td::actor::ActorOwn(std::move(X)); -} - -} // namespace adnl - -} // namespace ton diff --git a/adnl/adnl-static-nodes.h b/adnl/adnl-static-nodes.h deleted file mode 100644 index 91e3bed0c1..0000000000 --- a/adnl/adnl-static-nodes.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -#include "td/actor/actor.h" -#include "td/utils/Status.h" -#include "td/actor/PromiseFuture.h" -#include "auto/tl/ton_api.h" - -#include "adnl-peer-table.h" - -namespace ton { - -namespace adnl { - -class AdnlStaticNodesManager : public td::actor::Actor { - public: - virtual void add_node(AdnlNode node) = 0; - virtual void del_node(AdnlNodeIdShort id) = 0; - virtual td::Result get_node(AdnlNodeIdShort id) = 0; - - static td::actor::ActorOwn create(); -}; - -} // namespace adnl - -} // namespace ton diff --git a/adnl/adnl-static-nodes.hpp b/adnl/adnl-static-nodes.hpp deleted file mode 100644 index 166be5aa16..0000000000 --- a/adnl/adnl-static-nodes.hpp +++ /dev/null @@ -1,42 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -#include -#include "adnl-static-nodes.h" - -namespace ton { - -namespace adnl { - -class AdnlStaticNodesManagerImpl : public AdnlStaticNodesManager { - public: - void add_node(AdnlNode node) override; - void del_node(AdnlNodeIdShort id) override; - td::Result get_node(AdnlNodeIdShort id) override; - AdnlStaticNodesManagerImpl() { - } - - private: - std::map nodes_; -}; - -} // namespace adnl - -} // namespace ton From b8251cb4500cb625790aaadd2460bca791daa737 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 13 May 2025 21:19:58 +0300 Subject: [PATCH 243/524] Remove debug logs --- validator/manager.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index 2118ffb379..266079a5a9 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1601,7 +1601,6 @@ void ValidatorManagerImpl::send_get_block_request(BlockIdExt id, td::uint32 prio LOG(DEBUG) << "send_get_block_request: got result from block data cache for " << id.to_str(); return promise.set_value(ReceivedBlock{id, cached->clone()}); } - LOG(ERROR) << "XXXX download " << id.id.to_str(); callback_->download_block(id, priority, td::Timestamp::in(10.0), std::move(promise)); } @@ -1637,7 +1636,6 @@ void ValidatorManagerImpl::send_get_block_proof_link_request(BlockIdExt block_id return; } } - LOG(ERROR) << "XXXX download link " << block_id.id.to_str(); callback_->download_block_proof_link(block_id, priority, td::Timestamp::in(10.0), std::move(promise)); } From 215b157177351d3c7e28e0f8644d54bc4463eb5b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 13 May 2025 21:36:52 +0300 Subject: [PATCH 244/524] Fix downloading initial proof --- validator/manager-init.cpp | 25 ++++++++++++++++++------- validator/manager-init.hpp | 2 +- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index 3b57eb9b8e..3552d9164a 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -74,7 +74,7 @@ void ValidatorManagerMasterchainReiniter::got_masterchain_handle(BlockHandle han download_proof_link(); } -void ValidatorManagerMasterchainReiniter::download_proof_link() { +void ValidatorManagerMasterchainReiniter::download_proof_link(bool try_local) { if (handle_->id().id.seqno == 0) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); @@ -84,18 +84,29 @@ void ValidatorManagerMasterchainReiniter::download_proof_link() { td::Timestamp::in(3600), std::move(P)) .release(); } else { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { - LOG(WARNING) << "failed to download proof link: " << R.move_as_error(); + if (try_local) { + LOG(DEBUG) << "failed to get proof link from local import: " << R.move_as_error(); + } else { + LOG(WARNING) << "failed to download proof link: " << R.move_as_error(); + } delay_action( - [SelfId]() { td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link); }, + [SelfId]() { + td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link, false); + }, td::Timestamp::in(1.0)); } else { td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::downloaded_proof_link, R.move_as_ok()); } }); - td::actor::send_closure(manager_, &ValidatorManager::get_block_proof_link_from_import, handle_->id(), handle_->id(), - std::move(P)); + if (try_local) { + td::actor::send_closure(manager_, &ValidatorManager::get_block_proof_link_from_import, handle_->id(), + handle_->id(), std::move(P)); + } else { + td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, handle_->id(), 2, + std::move(P)); + } } } @@ -111,7 +122,7 @@ void ValidatorManagerMasterchainReiniter::downloaded_proof_link(td::BufferSlice auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), db = db_, proof](td::Result R) { if (R.is_error()) { LOG(WARNING) << "downloaded proof link failed: " << R.move_as_error(); - td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link); + td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link, false); } else { auto P = td::PromiseCreator::lambda([SelfId, handle = R.move_as_ok()](td::Result R) { R.ensure(); diff --git a/validator/manager-init.hpp b/validator/manager-init.hpp index 901b826bdb..90dc5ccc97 100644 --- a/validator/manager-init.hpp +++ b/validator/manager-init.hpp @@ -44,7 +44,7 @@ class ValidatorManagerMasterchainReiniter : public td::actor::Actor { void start_up() override; void written_hardforks(); void got_masterchain_handle(BlockHandle handle); - void download_proof_link(); + void download_proof_link(bool try_local = true); void downloaded_proof_link(td::BufferSlice data); void downloaded_zero_state(); From 6d04184337bc3f578f26cff9c08e9b72d444a2c6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 16 May 2025 16:17:48 +0300 Subject: [PATCH 245/524] Fix check in overlay-peers.cpp --- overlay/overlay-peers.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index 923f0fb127..52cc40cf61 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -196,7 +196,6 @@ void OverlayImpl::add_peer(OverlayNode node) { if (R.is_error()) { VLOG(OVERLAY_WARNING) << this << ": bad peer certificate node=" << node.adnl_id_short() << ": " << R.move_as_error(); - UNREACHABLE(); return; } } From d65e054734b216829ab5a4d189c22bfe988793b2 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 19 May 2025 17:56:59 +0300 Subject: [PATCH 246/524] Fix tracking storage stat updates --- crypto/block/transaction.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 1c4cbe35b1..0dded1ff74 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -3199,6 +3199,11 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, } StorageStatCalculationContext context{is_account_stat}; StorageStatCalculationContext::Guard guard{&context}; + if (is_account_stat) { + storage_stats_updates.push_back(new_code); + storage_stats_updates.push_back(new_data); + storage_stats_updates.push_back(new_library); + } TRY_STATUS(storage_stat.replace_roots({new_code, new_data, new_library}, /* check_merkle_depth = */ true)); if (timer.elapsed() > 0.1) { LOG(INFO) << "Compute used storage (1) took " << timer.elapsed() << "s"; @@ -3224,9 +3229,6 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, if (is_account_stat) { // storage_stat will be reused in compute_state() new_account_storage_stat.value_force() = std::move(storage_stat); - storage_stats_updates.push_back(new_code); - storage_stats_updates.push_back(new_data); - storage_stats_updates.push_back(new_library); } return td::Status::OK(); } From 18a8ed8a5ab01107b98ebeedc12b9b5ad815756a Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 20 May 2025 15:11:21 +0300 Subject: [PATCH 247/524] Bugfix in AccountStorageStat::remove_cell --- crypto/block/account-storage-stat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index 82359f827f..a6159c2245 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -154,7 +154,7 @@ td::Status AccountStorageStat::remove_cell(const Ref& cell) { if (e.refcnt_diff < 0 && !e.refcnt) { TRY_STATUS(fetch_from_dict(e)); } - if (e.refcnt.value() + e.refcnt_diff != 0) { + if (e.refcnt_diff >= 0 || e.refcnt.value() + e.refcnt_diff != 0) { return td::Status::OK(); } bool spec; From ed3860f838900bb4c2780c7f03ca624ca12ee90d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 20 May 2025 15:11:54 +0300 Subject: [PATCH 248/524] Fix returning null as c4/c5 in RUNVM --- crypto/vm/vm.cpp | 19 ++++++++++++++----- doc/GlobalVersions.md | 5 ++++- test/regression-tests.ans | 2 +- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/crypto/vm/vm.cpp b/crypto/vm/vm.cpp index 272e3930e6..356f0df223 100644 --- a/crypto/vm/vm.cpp +++ b/crypto/vm/vm.cpp @@ -785,11 +785,20 @@ void VmState::restore_parent_vm(int res) { cur_stack.push(std::move(child_state.stack->at(i))); } cur_stack.push_smallint(res); - if (parent->return_data) { - cur_stack.push_cell(child_state.get_committed_state().c4); - } - if (parent->return_actions) { - cur_stack.push_cell(child_state.get_committed_state().c5); + if (global_version >= 11 && !child_state.get_committed_state().committed) { + if (parent->return_data) { + cur_stack.push_null(); + } + if (parent->return_actions) { + cur_stack.push_null(); + } + } else { + if (parent->return_data) { + cur_stack.push_cell(child_state.get_committed_state().c4); + } + if (parent->return_actions) { + cur_stack.push_cell(child_state.get_committed_state().c5); + } } if (parent->return_gas) { cur_stack.push_smallint(child_state.gas.gas_consumed()); diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 7bd9b0cc98..d4113b3f1b 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -221,4 +221,7 @@ Reserve modes `+1`, `+4` and `+8` ("reserve all except", "add original balance" Along with the storage stat (cells and bits count), each account now stores the hash of the **storage dict**. **Storage dict** is the dictionary that stores refcnt for each cell in the account state. -This is required to help computing storage stats in the future, after collator-validator separation. \ No newline at end of file +This is required to help computing storage stats in the future, after collator-validator separation. + +### Other TVM changes +- Fix returning `null` as `c4` and `c5` (when VM state is not committed) in `RUNVM`. \ No newline at end of file diff --git a/test/regression-tests.ans b/test/regression-tests.ans index 22e7826eac..5831ebe740 100644 --- a/test/regression-tests.ans +++ b/test/regression-tests.ans @@ -25,7 +25,7 @@ Test_Fift_test_rist255_default f4d7558f200a656934f986145c19b1dedbe2ad029292a5a97 Test_Fift_test_secp256k1_default 3118450dace6af05fcdbd54a87d9446162ce11ac6ef6dfc57998cf113587d602 Test_Fift_test_sort2_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a Test_Fift_test_sort_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a -Test_Fift_test_tvm_runvm_default ff3d2a4031b543c18d6b555f0a1f1a891c7825e6d1e2e9beb4bf13b37441450b +Test_Fift_test_tvm_runvm_default b3e0d70c00f0e8bdf6d45c56c7c0817fa5be7a3fc540190786233a394de72e42 Test_Fift_testvm2_default 8a6e35fc0224398be9d2de39d31c86ea96965ef1eca2aa9e0af2303150ed4a7b Test_Fift_testvm3_default 3c1b77471c5fd914ed8b5f528b9faed618e278693f5030b953ff150e543864ae Test_Fift_testvm4_default 8a6e35fc0224398be9d2de39d31c86ea96965ef1eca2aa9e0af2303150ed4a7b From d8b5fad74e799c15388f297523a88312dafeb7d4 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 20 May 2025 20:10:23 +0300 Subject: [PATCH 249/524] Fix collated data calculation in Collator::process_account_storage_dict --- crypto/block/account-storage-stat.cpp | 6 +++- crypto/block/account-storage-stat.h | 3 +- crypto/block/transaction.cpp | 14 ++++++---- crypto/block/transaction.h | 4 +-- validator/impl/collator-impl.h | 1 + validator/impl/collator.cpp | 40 +++++++++++++-------------- 6 files changed, 37 insertions(+), 31 deletions(-) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index 82359f827f..f25caadef2 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -36,7 +36,8 @@ AccountStorageStat::AccountStorageStat(const AccountStorageStat* parent) CHECK(parent_->parent_ == nullptr); } -td::Status AccountStorageStat::replace_roots(std::vector> new_roots, bool check_merkle_depth) { +td::Status AccountStorageStat::replace_roots(std::vector> new_roots, bool check_merkle_depth, + td::HashSet* store_added) { std::erase_if(new_roots, [](const Ref& c) { return c.is_null(); }); if (new_roots.empty()) { roots_.clear(); @@ -73,6 +74,9 @@ td::Status AccountStorageStat::replace_roots(std::vector> new_root roots_ = std::move(new_roots); dict_up_to_date_ = false; for (auto& [_, e] : cache_) { + if (store_added && !e.exists && e.refcnt_diff > 0) { + store_added->insert(e.hash); + } TRY_STATUS(finalize_entry(e)); } return td::Status::OK(); diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h index e939a0ff51..7322c991d3 100644 --- a/crypto/block/account-storage-stat.h +++ b/crypto/block/account-storage-stat.h @@ -39,7 +39,8 @@ class AccountStorageStat { AccountStorageStat &operator=(const AccountStorageStat &other) = delete; AccountStorageStat &operator=(AccountStorageStat &&other) = default; - td::Status replace_roots(std::vector> new_roots, bool check_merkle_depth = false); + td::Status replace_roots(std::vector> new_roots, bool check_merkle_depth = false, + td::HashSet *store_added = nullptr); void add_hint(const td::HashSet &visited); td::uint64 get_total_cells() const { diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 0dded1ff74..b3d5c8c692 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -602,9 +602,11 @@ static td::Ref storage_without_extra_currencies(td::Ref> Account::compute_account_storage_dict() const { +td::Result> Account::compute_account_storage_dict(td::HashSet* storage_cells) const { if (storage.is_null()) { return td::Status::Error("cannot compute storage dict: empty storage"); } @@ -616,7 +618,7 @@ td::Result> Account::compute_account_storage_dict() const { if (storage_for_stat.is_null()) { return td::Status::Error("cannot compute storage dict: invalid storage"); } - TRY_STATUS(stat.replace_roots(storage_for_stat->prefetch_all_refs())); + TRY_STATUS(stat.replace_roots(storage_for_stat->prefetch_all_refs(), false, storage_cells)); // Root of AccountStorage is not counted in AccountStorageStat td::uint64 expected_cells = stat.get_total_cells() + 1; td::uint64 expected_bits = stat.get_total_bits() + storage->size(); @@ -3200,9 +3202,9 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, StorageStatCalculationContext context{is_account_stat}; StorageStatCalculationContext::Guard guard{&context}; if (is_account_stat) { - storage_stats_updates.push_back(new_code); - storage_stats_updates.push_back(new_data); - storage_stats_updates.push_back(new_library); + storage_stat_updates.push_back(new_code); + storage_stat_updates.push_back(new_data); + storage_stat_updates.push_back(new_library); } TRY_STATUS(storage_stat.replace_roots({new_code, new_data, new_library}, /* check_merkle_depth = */ true)); if (timer.elapsed() > 0.1) { @@ -3506,7 +3508,7 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { AccountStorageStat& stats = new_account_storage_stat.value_force(); // Don't check Merkle depth and size here - they were checked in check_state_limits auto roots = new_storage_for_stat->prefetch_all_refs(); - storage_stats_updates.insert(storage_stats_updates.end(), roots.begin(), roots.end()); + storage_stat_updates.insert(storage_stat_updates.end(), roots.begin(), roots.end()); { StorageStatCalculationContext context{true}; StorageStatCalculationContext::Guard guard{&context}; diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index b7050afa07..4e9440e12e 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -300,7 +300,7 @@ struct Account { bool set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr); bool unpack(Ref account, ton::UnixTime now, bool special); bool init_new(ton::UnixTime now); - td::Result> compute_account_storage_dict() const; + td::Result> compute_account_storage_dict(td::HashSet* storage_cells = nullptr) const; td::Status init_account_storage_stat(Ref dict_root); bool deactivate(); bool recompute_tmp_addr(Ref& tmp_addr, int fixed_prefix_length, td::ConstBitPtr orig_addr_rewrite) const; @@ -397,7 +397,7 @@ struct Transaction { td::optional new_account_storage_stat; td::optional new_storage_dict_hash; bool gas_limit_overridden{false}; - std::vector> storage_stats_updates; + std::vector> storage_stat_updates; Transaction(const Account& _account, int ttype, ton::LogicalTime req_start_lt, ton::UnixTime _now, Ref _inmsg = {}); bool unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* cfg); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 405fad3346..303672f5d5 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -227,6 +227,7 @@ class Collator final : public td::actor::Actor { vm::ProofStorageStat proof_stat; bool add_to_collated_data = false; std::vector> storage_stat_updates; + td::HashSet original_storage_cells; }; std::map account_storage_dicts_; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 6e7a5b87aa..069abc0f98 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -2625,9 +2625,6 @@ bool Collator::init_account_storage_dict(block::Account& account) { if (!full_collated_data_ || is_masterchain() || !account.storage_dict_hash || account.storage.is_null()) { return true; } - if (account.storage_used.cells < 10) { // TODO: some other threshold? - return true; - } td::Bits256 storage_dict_hash = account.storage_dict_hash.value(); if (storage_dict_hash.is_zero()) { return true; @@ -2637,7 +2634,7 @@ bool Collator::init_account_storage_dict(block::Account& account) { dict.inited = true; // don't mark cells in account state as loaded during compute_account_storage_dict state_usage_tree_->set_ignore_loads(true); - auto res = account.compute_account_storage_dict(); + auto res = account.compute_account_storage_dict(&dict.original_storage_cells); state_usage_tree_->set_ignore_loads(false); if (res.is_error()) { return fatal_error(res.move_as_error_prefix(PSTRING() << "Failed to init account storage dict for " @@ -2740,37 +2737,38 @@ bool Collator::process_account_storage_dict(const block::Account& account) { td::HashSet visited; bool calculate_proof_size_diff = true; td::int64 proof_size_diff = 0; - std::function&, bool)> dfs = [&](const Ref& cell, bool from_old_state) { + std::function&)> dfs = [&](const Ref& cell) { if (cell.is_null() || !visited.emplace(cell->get_hash()).second) { return; } auto loaded_cell = cell->load_cell().move_as_ok(); - if (!loaded_cell.tree_node.empty()) { - from_old_state = true; - } - if (calculate_proof_size_diff && from_old_state) { - switch (collated_data_stat.get_cell_status(cell->get_hash())) { - case vm::ProofStorageStat::c_none: - proof_size_diff += vm::ProofStorageStat::estimate_serialized_size(loaded_cell.data_cell); + if (dict.original_storage_cells.contains(cell->get_hash())) { + if (calculate_proof_size_diff) { + switch (collated_data_stat.get_cell_status(cell->get_hash())) { + case vm::ProofStorageStat::c_none: + proof_size_diff += vm::ProofStorageStat::estimate_serialized_size(loaded_cell.data_cell); break; - case vm::ProofStorageStat::c_prunned: - proof_size_diff -= vm::ProofStorageStat::estimate_prunned_size(); + case vm::ProofStorageStat::c_prunned: + proof_size_diff -= vm::ProofStorageStat::estimate_prunned_size(); proof_size_diff += vm::ProofStorageStat::estimate_serialized_size(loaded_cell.data_cell); break; - case vm::ProofStorageStat::c_loaded: - break; + case vm::ProofStorageStat::c_loaded: + break; + } + } else { + collated_data_stat.add_loaded_cell(loaded_cell.data_cell, loaded_cell.virt.get_level()); } } - vm::CellSlice cs{std::move(loaded_cell)}; + vm::CellSlice cs{std::move(loaded_cell.data_cell)}; for (unsigned i = 0; i < cs.size_refs(); ++i) { - dfs(cs.prefetch_ref(i), from_old_state); + dfs(cs.prefetch_ref(i)); } }; // Visit cells that were used in storage stat computation to calculate collated data increase state_usage_tree_->set_ignore_loads(true); for (const auto& cell : dict.storage_stat_updates) { - dfs(cell, false); + dfs(cell); } state_usage_tree_->set_ignore_loads(false); @@ -2789,7 +2787,7 @@ bool Collator::process_account_storage_dict(const block::Account& account) { calculate_proof_size_diff = false; visited.clear(); for (const auto& cell : dict.storage_stat_updates) { - dfs(cell, false); + dfs(cell); } } @@ -5465,7 +5463,7 @@ void Collator::update_account_storage_dict_info(const block::transaction::Transa if (it == account_storage_dicts_.end()) { return; } - for (const Ref& cell : trans.storage_stats_updates) { + for (const Ref& cell : trans.storage_stat_updates) { if (cell.not_null()) { it->second.storage_stat_updates.push_back(cell); } From ffa872c2fb8677e7851dcca4a8dcc4d59013a7c7 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Wed, 21 May 2025 14:20:48 +0300 Subject: [PATCH 250/524] Update note on +2 RUNVM flag --- crypto/vm/contops.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/vm/contops.cpp b/crypto/vm/contops.cpp index 1ccf53dafe..e4499aa027 100644 --- a/crypto/vm/contops.cpp +++ b/crypto/vm/contops.cpp @@ -214,7 +214,7 @@ int exec_ret_data(VmState* st) { // Mode: // +1 = same_c3 (set c3 to code) -// +2 = push_0 (push an implicit 0 before running the code) +// +2 = push_0 (push an implicit 0 before running the code); only works with +1 enabled // +4 = load c4 (persistent data) from stack and return its final value // +8 = load gas limit from stack and return consumed gas // +16 = load c7 (smart-contract context) From 7d9f081e38cf639604397e1007a3b0336afc7cb6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 21 May 2025 16:15:43 +0300 Subject: [PATCH 251/524] Improve downloading shard archives --- validator/import-db-slice.cpp | 31 +++++++++++++++++++++++++------ validator/import-db-slice.hpp | 2 +- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index 96dba7ea86..55c2cd7946 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -54,7 +54,7 @@ ArchiveImporter::ArchiveImporter(std::string db_root, td::Ref void ArchiveImporter::start_up() { if (use_imported_files_) { LOG(INFO) << "Importing archive for masterchain seqno #" << start_import_seqno_ << " from disk"; - for (const std::string& path : to_import_files_) { + for (const std::string &path : to_import_files_) { LOG(INFO) << "Importing file from disk " << path; td::Status S = process_package(path, true); if (S.is_error()) { @@ -271,13 +271,13 @@ void ArchiveImporter::applied_masterchain_block(BlockHandle handle) { LOG(DEBUG) << "Applied masterchain block #" << handle->id().seqno(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); - td::actor::send_closure(SelfId, &ArchiveImporter::got_new_materchain_state, + td::actor::send_closure(SelfId, &ArchiveImporter::got_new_masterchain_state, td::Ref(R.move_as_ok())); }); td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle, std::move(P)); } -void ArchiveImporter::got_new_materchain_state(td::Ref state) { +void ArchiveImporter::got_new_masterchain_state(td::Ref state) { last_masterchain_state_ = std::move(state); imported_any_ = true; check_masterchain_block(last_masterchain_state_->get_block_id().seqno() + 1); @@ -300,6 +300,20 @@ void ArchiveImporter::checked_all_masterchain_blocks() { }); } +static bool have_new_shards(td::Ref new_state, td::Ref old_state, + ShardIdFull shard_prefix) { + for (auto &shard : new_state->get_shards()) { + if (!shard_intersects(shard_prefix, shard->shard())) { + continue; + } + auto old_shard = old_state->get_shard_from_config(shard->shard()); + if (old_shard.is_null() || old_shard->top_block_id() != shard->top_block_id()) { + return true; + } + } + return false; +} + void ArchiveImporter::download_shard_archives(td::Ref start_state) { start_state_ = start_state; td::uint32 monitor_min_split = start_state->monitor_min_split_depth(basechainId); @@ -310,9 +324,14 @@ void ArchiveImporter::download_shard_archives(td::Ref start_st for (td::uint64 i = 0; i < (1ULL << monitor_min_split); ++i) { ShardIdFull shard_prefix{basechainId, (i * 2 + 1) << (64 - monitor_min_split - 1)}; if (opts_->need_monitor(shard_prefix, start_state)) { - ++pending_shard_archives_; - LOG(DEBUG) << "Downloading shard archive #" << start_import_seqno_ << " " << shard_prefix.to_str(); - download_shard_archive(shard_prefix); + if (have_new_shards(last_masterchain_state_, start_state_, shard_prefix)) { + ++pending_shard_archives_; + LOG(INFO) << "Downloading shard archive #" << start_import_seqno_ << " " << shard_prefix.to_str(); + download_shard_archive(shard_prefix); + } else { + LOG(INFO) << "Not downloading shard archive #" << start_import_seqno_ << " " << shard_prefix.to_str() + << " : no new shard blocks"; + } } } } else { diff --git a/validator/import-db-slice.hpp b/validator/import-db-slice.hpp index 4cf6ce40ac..fc295216ae 100644 --- a/validator/import-db-slice.hpp +++ b/validator/import-db-slice.hpp @@ -44,7 +44,7 @@ class ArchiveImporter : public td::actor::Actor { void check_masterchain_block(BlockSeqno seqno); void checked_masterchain_proof(BlockHandle handle, td::Ref data); void applied_masterchain_block(BlockHandle handle); - void got_new_materchain_state(td::Ref state); + void got_new_masterchain_state(td::Ref state); void checked_all_masterchain_blocks(); void download_shard_archives(td::Ref start_state); From 80a968ee201300c78c2164e727159c313b6e2e67 Mon Sep 17 00:00:00 2001 From: Evgeny Kapun Date: Thu, 22 May 2025 21:23:12 +0300 Subject: [PATCH 252/524] Remove pow-miner (#1683) --- assembly/native/build-macos-portable.sh | 4 +- assembly/native/build-macos-shared.sh | 4 +- assembly/native/build-ubuntu-appimages.sh | 4 +- assembly/native/build-ubuntu-portable.sh | 4 +- assembly/native/build-ubuntu-shared.sh | 4 +- assembly/native/build-windows-2019.bat | 4 +- assembly/native/build-windows.bat | 4 +- crypto/CMakeLists.txt | 8 - crypto/util/Miner.cpp | 129 ------------ crypto/util/Miner.h | 43 ---- crypto/util/pow-miner.cpp | 245 ---------------------- doc/TestGrams-HOWTO | 163 -------------- tonlib/CMakeLists.txt | 4 +- tonlib/tonlib/tonlib-cli.cpp | 219 ------------------- 14 files changed, 16 insertions(+), 823 deletions(-) delete mode 100644 crypto/util/Miner.cpp delete mode 100644 crypto/util/Miner.h delete mode 100644 crypto/util/pow-miner.cpp delete mode 100644 doc/TestGrams-HOWTO diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index c46ea416e4..c3dec11d20 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -150,7 +150,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \ test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ @@ -159,7 +159,7 @@ if [ "$with_tests" = true ]; then else ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index a326694907..c769e95797 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -75,7 +75,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \ test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ @@ -84,7 +84,7 @@ if [ "$with_tests" = true ]; then else ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi diff --git a/assembly/native/build-ubuntu-appimages.sh b/assembly/native/build-ubuntu-appimages.sh index 458ca9d0bf..b21d7df8c9 100644 --- a/assembly/native/build-ubuntu-appimages.sh +++ b/assembly/native/build-ubuntu-appimages.sh @@ -60,7 +60,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ - validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ @@ -69,7 +69,7 @@ ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib- test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ - validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ adnl-proxy create-state emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh index f208148c9a..d2e733954f 100644 --- a/assembly/native/build-ubuntu-portable.sh +++ b/assembly/native/build-ubuntu-portable.sh @@ -134,7 +134,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ - validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ @@ -143,7 +143,7 @@ ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib- test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ - validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ adnl-proxy create-state emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index 68ab6aba62..3fc6db3156 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -63,7 +63,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ - validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ @@ -72,7 +72,7 @@ ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib- test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ - validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ adnl-proxy create-state emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } diff --git a/assembly/native/build-windows-2019.bat b/assembly/native/build-windows-2019.bat index 0b96978ca7..f5fd0be367 100644 --- a/assembly/native/build-windows-2019.bat +++ b/assembly/native/build-windows-2019.bat @@ -145,7 +145,7 @@ IF %errorlevel% NEQ 0 ( IF "%1"=="-t" ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ -tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net ^ test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ @@ -156,7 +156,7 @@ IF %errorlevel% NEQ 0 ( ) ) else ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ -tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver IF %errorlevel% NEQ 0 ( echo Can't compile TON diff --git a/assembly/native/build-windows.bat b/assembly/native/build-windows.bat index c513140782..41697eb0fd 100644 --- a/assembly/native/build-windows.bat +++ b/assembly/native/build-windows.bat @@ -145,7 +145,7 @@ IF %errorlevel% NEQ 0 ( IF "%1"=="-t" ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ -tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net ^ test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ @@ -156,7 +156,7 @@ IF %errorlevel% NEQ 0 ( ) ) else ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ -tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver IF %errorlevel% NEQ 0 ( echo Can't compile TON diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 79d1117eaa..cd2c7e47eb 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -421,16 +421,8 @@ if (WINGETOPT_FOUND) target_link_libraries_system(tlbc wingetopt) endif() -add_library(pow-miner-lib util/Miner.cpp util/Miner.h) -target_include_directories(pow-miner-lib PUBLIC $) -target_link_libraries(pow-miner-lib PUBLIC ton_crypto) - -add_executable(pow-miner util/pow-miner.cpp) -target_link_libraries(pow-miner PRIVATE ton_crypto pow-miner-lib git) - if (WINGETOPT_FOUND) target_link_libraries_system(fift wingetopt) - target_link_libraries_system(pow-miner wingetopt) endif() add_executable(mintless-proof-generator util/mintless-proof-generator.cpp) diff --git a/crypto/util/Miner.cpp b/crypto/util/Miner.cpp deleted file mode 100644 index 3e26fac6b1..0000000000 --- a/crypto/util/Miner.cpp +++ /dev/null @@ -1,129 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "Miner.h" - -#include "td/utils/Random.h" -#include "td/utils/misc.h" -#include "td/utils/crypto.h" -#include "td/utils/port/Clocks.h" -#include - -namespace ton { -#pragma pack(push, 1) -struct HData { - unsigned char op[4]; - signed char flags = -4; - unsigned char expire[4] = {}, myaddr[32] = {}, rdata1[32] = {}, pseed[16] = {}, rdata2[32] = {}; - void inc() { - for (long i = 31; !(rdata1[i] = ++(rdata2[i])); --i) { - } - } - void set_expire(unsigned x) { - for (int i = 3; i >= 0; --i) { - expire[i] = (x & 0xff); - x >>= 8; - } - } - - td::Slice as_slice() const { - return td::Slice(reinterpret_cast(this), sizeof(*this)); - } -}; - -struct HDataEnv { - unsigned char d1 = 0, d2 = sizeof(HData) * 2; - HData body; - - td::Slice as_slice() const { - return td::Slice(reinterpret_cast(this), sizeof(*this)); - } - - void init(const block::StdAddress& my_address, td::Slice seed) { - std::memcpy(body.myaddr, my_address.addr.data(), sizeof(body.myaddr)); - body.flags = static_cast(my_address.workchain * 4 + my_address.bounceable); - CHECK(seed.size() == 16); - std::memcpy(body.pseed, seed.data(), 16); - std::memcpy(body.op, "Mine", 4); - - td::Random::secure_bytes(body.rdata1, 32); - std::memcpy(body.rdata2, body.rdata1, 32); - } -}; - -static_assert(std::is_trivially_copyable::value, "HDataEnv must be a trivial type"); -#pragma pack(pop) - -td::optional Miner::run(const Options& options) { - HDataEnv H; - H.init(options.my_address, td::Slice(options.seed.data(), options.seed.size())); - - td::Slice data = H.as_slice(); - CHECK(data.size() == 123); - - constexpr size_t prefix_size = 72; - constexpr size_t guard_pos = prefix_size - (72 - 28); - CHECK(0 <= guard_pos && guard_pos < 32); - size_t got_prefix_size = (const unsigned char*)H.body.rdata1 + guard_pos + 1 - (const unsigned char*)&H; - CHECK(prefix_size == got_prefix_size); - - auto head = data.substr(0, prefix_size); - auto tail = data.substr(prefix_size); - - SHA256_CTX shactx1, shactx2; - std::array hash; - SHA256_Init(&shactx1); - auto guard = head.back(); - - td::int64 i = 0, i0 = 0; - for (; i < options.max_iterations; i++) { - if (!(i & 0xfffff) || head.back() != guard) { - if (options.token_) { - break; - } - if (options.hashes_computed) { - *options.hashes_computed += i - i0; - } - i0 = i; - if (options.expire_at && options.expire_at.value().is_in_past(td::Timestamp::now())) { - break; - } - H.body.set_expire((unsigned)td::Clocks::system() + 900); - guard = head.back(); - SHA256_Init(&shactx1); - SHA256_Update(&shactx1, head.ubegin(), head.size()); - } - shactx2 = shactx1; - SHA256_Update(&shactx2, tail.ubegin(), tail.size()); - SHA256_Final(hash.data(), &shactx2); - - if (memcmp(hash.data(), options.complexity.data(), 32) < 0) { - // FOUND - if (options.hashes_computed) { - *options.hashes_computed += i - i0; - } - return H.body.as_slice().str(); - } - H.body.inc(); - } - if (options.hashes_computed) { - *options.hashes_computed += i - i0; - } - return {}; -} -} // namespace ton diff --git a/crypto/util/Miner.h b/crypto/util/Miner.h deleted file mode 100644 index f91a668662..0000000000 --- a/crypto/util/Miner.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -#include "block/block.h" -#include "td/utils/CancellationToken.h" -#include "td/utils/optional.h" -#include "td/utils/Time.h" -#include -#include - -namespace ton { -class Miner { - public: - struct Options { - block::StdAddress my_address; - std::array seed; - std::array complexity; - td::optional expire_at; - td::int64 max_iterations = std::numeric_limits::max(); - std::atomic* hashes_computed{nullptr}; - td::CancellationToken token_; - }; - - static td::optional run(const Options& options); -}; -} // namespace ton diff --git a/crypto/util/pow-miner.cpp b/crypto/util/pow-miner.cpp deleted file mode 100644 index c065fdc701..0000000000 --- a/crypto/util/pow-miner.cpp +++ /dev/null @@ -1,245 +0,0 @@ -/* - This file is part of TON Blockchain source code. - - TON Blockchain is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - TON Blockchain is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with TON Blockchain. If not, see . - - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement - from all source files in the program, then also delete it here. - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "common/bigint.hpp" -#include "common/refint.h" -#include "block/block.h" -#include "td/utils/benchmark.h" -#include "td/utils/filesystem.h" -#include "vm/boc.h" -#include "openssl/digest.hpp" -#include -#include -#include -#include -#include -#include "git.h" -#include "Miner.h" - -const char* progname; - -int usage() { - std::cerr - << "usage: " << progname - << " [-v][-B][-w] [-t] [ " - "] [-V]\n" - "Outputs a valid value for proof-of-work testgiver after computing at most hashes " - "or terminates with non-zero exit code\n"; - std::exit(2); -} - -td::RefInt256 parse_bigint(std::string str, int bits) { - int len = (int)str.size(); - auto num = td::make_refint(); - auto& x = num.write(); - if (len >= 3 && str[0] == '0' && str[1] == 'x') { - if (x.parse_hex(str.data() + 2, len - 2) != len - 2) { - return {}; - } - } else if (!len || x.parse_dec(str.data(), len) != len) { - return {}; - } - return x.unsigned_fits_bits(bits) ? std::move(num) : td::RefInt256{}; -} - -td::RefInt256 parse_bigint_chk(std::string str, int bits) { - auto x = parse_bigint(std::move(str), bits); - if (x.is_null()) { - std::cerr << "fatal: `" << str << "` is not an integer" << std::endl; - usage(); - } - return x; -} - -void parse_addr(std::string str, block::StdAddress& addr) { - if (!addr.parse_addr(str) || (addr.workchain != -1 && addr.workchain != 0)) { - std::cerr << "fatal: `" << str.c_str() << "` is not a valid blockchain address" << std::endl; - usage(); - } -} - -bool make_boc = false; -std::string boc_filename; -block::StdAddress miner_address; - -int verbosity = 0; -std::atomic hashes_computed{0}; -td::Timestamp start_at; - -void print_stats() { - auto passed = td::Timestamp::now().at() - start_at.at(); - if (passed < 1e-9) { - passed = 1; - } - std::cerr << "[ hashes computed: " << hashes_computed << " ]" << std::endl; - std::cerr << "[ speed: " << static_cast(hashes_computed) / passed << " hps ]" << std::endl; -} - -int found(td::Slice data) { - for (unsigned i = 0; i < data.size(); i++) { - printf("%02X", data.ubegin()[i]); - } - printf("\n"); - if (make_boc) { - vm::CellBuilder cb; - td::Ref ext_msg, body; - CHECK(cb.store_bytes_bool(data) // body - && cb.finalize_to(body) // -> body - && cb.store_long_bool(0x44, 7) // ext_message_in$10 ... - && cb.store_long_bool(miner_address.workchain, 8) // workchain - && cb.store_bytes_bool(miner_address.addr.as_slice()) // miner addr - && cb.store_long_bool(1, 6) // amount:Grams ... - && cb.store_ref_bool(std::move(body)) // body:^Cell - && cb.finalize_to(ext_msg)); - auto boc = vm::std_boc_serialize(std::move(ext_msg), 2).move_as_ok(); - std::cerr << "Saving " << boc.size() << " bytes of serialized external message into file `" << boc_filename << "`" - << std::endl; - td::write_file(boc_filename, boc).ensure(); - } - if (verbosity > 0) { - print_stats(); - } - std::exit(0); - return 0; -} - -void miner(const ton::Miner::Options& options) { - auto res = ton::Miner::run(options); - if (res) { - found(res.value()); - } -} - -class MinerBench : public td::Benchmark { - public: - std::string get_description() const override { - return "Miner"; - } - - void run(int n) override { - ton::Miner::Options options; - options.my_address.parse_addr("EQDU86V5wyPrLd4nQ0RHPcCLPZq_y1O5wFWyTsMw63vjXTOv"); - std::fill(options.seed.begin(), options.seed.end(), 0xa7); - std::fill(options.complexity.begin(), options.complexity.end(), 0); - options.max_iterations = n; - CHECK(!ton::Miner::run(options)); - } -}; - -int main(int argc, char* const argv[]) { - ton::Miner::Options options; - - progname = argv[0]; - int i, threads = 0; - bool bounce = false, benchmark = false; - while ((i = getopt(argc, argv, "bnvw:t:Bh:V")) != -1) { - switch (i) { - case 'v': - ++verbosity; - break; - case 'w': - threads = atoi(optarg); - CHECK(threads > 0 && threads <= 256); - break; - case 't': { - int timeout = atoi(optarg); - CHECK(timeout > 0); - options.expire_at = td::Timestamp::in(timeout); - break; - } - case 'B': - benchmark = true; - break; - case 'b': - bounce = true; - break; - case 'n': - bounce = false; - break; - case 'V': - std::cout << "pow-miner build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; - std::exit(0); - break; - case 'h': - return usage(); - default: - std::cerr << "unknown option" << std::endl; - return usage(); - } - } - if (benchmark && argc == optind) { - td::bench(MinerBench()); - return 0; - } - - if (argc != optind + 4 && argc != optind + 6) { - return usage(); - } - - parse_addr(argv[optind], options.my_address); - options.my_address.bounceable = bounce; - CHECK(parse_bigint_chk(argv[optind + 1], 128)->export_bytes(options.seed.data(), 16, false)); - - auto cmplx = parse_bigint_chk(argv[optind + 2], 256); - CHECK(cmplx->export_bytes(options.complexity.data(), 32, false)); - CHECK(!cmplx->unsigned_fits_bits(256 - 62)); - td::BigInt256 bigpower, hrate; - bigpower.set_pow2(256).mod_div(*cmplx, hrate); - long long hash_rate = hrate.to_long(); - options.max_iterations = parse_bigint_chk(argv[optind + 3], 50)->to_long(); - if (argc == optind + 6) { - make_boc = true; - parse_addr(argv[optind + 4], miner_address); - boc_filename = argv[optind + 5]; - } - - if (verbosity >= 2) { - std::cerr << "[ expected required hashes for success: " << hash_rate << " ]" << std::endl; - } - if (benchmark) { - td::bench(MinerBench()); - } - - start_at = td::Timestamp::now(); - - options.hashes_computed = &hashes_computed; - // may invoke several miner threads - if (threads <= 0) { - miner(options); - } else { - std::vector T; - for (int i = 0; i < threads; i++) { - T.emplace_back(miner, options); - } - for (auto& thr : T) { - thr.join(); - } - } - if (verbosity > 0) { - print_stats(); - } -} diff --git a/doc/TestGrams-HOWTO b/doc/TestGrams-HOWTO deleted file mode 100644 index 8156e97571..0000000000 --- a/doc/TestGrams-HOWTO +++ /dev/null @@ -1,163 +0,0 @@ -The aim of this text is to describe how to quickly obtain a small amount of test Grams for test purposes, or a larger amount of test Grams for running a validator in the test network. We assume familiarity with the TON Blockchain LiteClient as explained in the LiteClient-HOWTO, and with the procedure required to compile the LiteClient and other software. For obtaining larger amount of test Grams required for running a validator, we also assume acquaintance with the FullNode-HOWTO and Validator-HOWTO. You will also need a dedicated server powerful enough for running a Full Node in order to obtain the larger amount of test Grams. Obtaining small amounts of test Grams does not require a dedicated server and may be done in several minutes on a home computer. - -1. Proof-of-Work TestGiver smart contracts -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In order to prevent a small number of malicious parties to collect all test Grams reserved for test purposes, a special kind of "Proof-of-Work TestGiver" smart contracts have been deployed in the masterchain of the test network. The addresses of these smart contacts are: - -Small testgivers (deliver from 10 to 100 test Grams every several minutes): - -kf-kkdY_B7p-77TLn2hUhM6QidWrrsl8FYWCIvBMpZKprBtN -kf8SYc83pm5JkGt0p3TQRkuiM58O9Cr3waUtR9OoFq716lN- -kf-FV4QTxLl-7Ct3E6MqOtMt-RGXMxi27g4I645lw6MTWraV -kf_NSzfDJI1A3rOM0GQm7xsoUXHTgmdhN5-OrGD8uwL2JMvQ -kf8gf1PQy4u2kURl-Gz4LbS29eaN4sVdrVQkPO-JL80VhOe6 -kf8kO6K6Qh6YM4ddjRYYlvVAK7IgyW8Zet-4ZvNrVsmQ4EOF -kf-P_TOdwcCh0AXHhBpICDMxStxHenWdLCDLNH5QcNpwMHJ8 -kf91o4NNTryJ-Cw3sDGt9OTiafmETdVFUMvylQdFPoOxIsLm -kf9iWhwk9GwAXjtwKG-vN7rmXT3hLIT23RBY6KhVaynRrIK7 -kf8JfFUEJhhpRW80_jqD7zzQteH6EBHOzxiOhygRhBdt4z2N - -Large testgivers (deliver 10000 test Grams at least once a day): - -kf8guqdIbY6kpMykR8WFeVGbZcP2iuBagXfnQuq0rGrxgE04 -kf9CxReRyaGj0vpSH0gRZkOAitm_yDHvgiMGtmvG-ZTirrMC -kf-WXA4CX4lqyVlN4qItlQSWPFIy00NvO2BAydgC4CTeIUme -kf8yF4oXfIj7BZgkqXM6VsmDEgCqWVSKECO1pC0LXWl399Vx -kf9nNY69S3_heBBSUtpHRhIzjjqY0ChugeqbWcQGtGj-gQxO -kf_wUXx-l1Ehw0kfQRgFtWKO07B6WhSqcUQZNyh4Jmj8R4zL -kf_6keW5RniwNQYeq3DNWGcohKOwI85p-V2MsPk4v23tyO3I -kf_NSPpF4ZQ7mrPylwk-8XQQ1qFD5evLnx5_oZVNywzOjSfh -kf-uNWj4JmTJefr7IfjBSYQhFbd3JqtQ6cxuNIsJqDQ8SiEA -kf8mO4l6ZB_eaMn1OqjLRrrkiBcSt7kYTvJC_dzJLdpEDKxn - -The first ten smart contracts enable a tester willing to obtain a small amount of test Grams to obtain some without spending too much computing power (typically, several minutes of work of a home computer should suffice). The remaining smart contracts are for obtaining larger amounts of test Grams required for running a validator in the test network; typically, a day of work of a dedicated server powerful enough to run a validator should suffice to obtain the necessary amount. - -You should randomly choose one of these "proof-of-work testgiver" smart contracts (from one of these two lists depending on your purpose) and obtain test Grams from this smart contract by a procedure similar to "mining". Essentially, you have to present an external message containing the proof of work and the address of your wallet to the chosen "proof-of-work testgiver" smart contract, and then the necessary amount will be sent to you. - -2. The "mining" process -~~~~~~~~~~~~~~~~~~~~~~~ - -In order to create an external message containing the "proof of work", you should run a special "mining" utility, compiled from the TON sources located in the GitHub repository. The utility is located in file './crypto/pow-miner' with respect to the build directory, and can be compiled by typing 'make pow-miner' in the build directory. - -However, before running "pow-miner", you need to know the actual values of "seed" and "complexity" parameters of the chosen "proof-of-work testgiver" smart contract. This can be done by invoking get-method "get_pow_params" of this smart contract. For instance, if you use testgiver smart contract kf-kkdY_B7p-77TLn2hUhM6QidWrrsl8FYWCIvBMpZKprBtN, you can simply type - - > runmethod kf-kkdY_B7p-77TLn2hUhM6QidWrrsl8FYWCIvBMpZKprBtN get_pow_params - -in the LiteClient console and obtain output like - - ... - arguments: [ 101616 ] - result: [ 229760179690128740373110445116482216837 53919893334301279589334030174039261347274288845081144962207220498432 100000000000 256 ] - remote result (not to be trusted): [ 229760179690128740373110445116482216837 53919893334301279589334030174039261347274288845081144962207220498432 100000000000 256 ] - -The two first large numbers in the "result:" line are the "seed" and the "complexity" of this smart contract. In this example, the seed is 229760179690128740373110445116482216837 and the complexity is 53919893334301279589334030174039261347274288845081144962207220498432. - -Next, you invoke the pow-miner utility as follows: - - $ crypto/pow-miner -vv -w -t - -Here is the number of CPU cores that you want to use for mining, is the maximal amount of seconds that the miner would run before admitting failure, is the address of your wallet (possibly not initialized yet), either in the masterchain or in the workchain (note that you need a masterchain wallet to control a validator), and are the most recent values obtained by running get-method 'get-pow-params', is the address of the chosen proof-of-work testgiver smartcontract, and is the filename of the output file where the external message with the proof of work will be saved in the case of success. - -For example, if your wallet address is kQBWkNKqzCAwA9vjMwRmg7aY75Rf8lByPA9zKXoqGkHi8SM7, you might run - - $ crypto/pow-miner -vv -w7 -t100 kQBWkNKqzCAwA9vjMwRmg7aY75Rf8lByPA9zKXoqGkHi8SM7 229760179690128740373110445116482216837 53919893334301279589334030174039261347274288845081144962207220498432 100000000000 kf-kkdY_B7p-77TLn2hUhM6QidWrrsl8FYWCIvBMpZKprBtN mined.boc - -The program will run for some time (at most 100 seconds in this case) and either terminate successfully (with zero exit code) and save the required proof of work into file "mined.boc", or terminate with a non-zero exit code if no proof of work was found. - -In the case of failure, you will see something like - - [ expected required hashes for success: 2147483648 ] - [ hashes computed: 1192230912 ] - -and the program will terminate with a non-zero exit code. Then you have to obtain the "seed" and "complexity" again (because they may have changed in the meantime as a result of processing requests from more successful miners) and re-run the "pow-miner" with the new parameters, repeating the process again and again until success. - -In the case of success, you will see something like - - [ expected required hashes for success: 2147483648 ] - 4D696E65005EFE49705690D2AACC203003DBE333046683B698EF945FF250723C0F73297A2A1A41E2F1A1F533B3BC4F5664D6C743C1C5C74BB3342F3A7314364B3D0DA698E6C80C1EA4ACDA33755876665780BAE9BE8A4D6385A1F533B3BC4F5664D6C743C1C5C74BB3342F3A7314364B3D0DA698E6C80C1EA4 - Saving 176 bytes of serialized external message into file `mined.boc` - [ hashes computed: 1122036095 ] - -Then you can use the LiteClient to send external message from file "mined.boc" to the proof-of-work testgiver smart contract (and you must do this as soon as possible): - -> sendfile mined.boc -... external message status is 1 - -You can wait for several seconds and check the state of your wallet: - -> last -> getaccount kQBWkNKqzCAwA9vjMwRmg7aY75Rf8lByPA9zKXoqGkHi8SM7 -... -account state is (account - addr:(addr_std - anycast:nothing workchain_id:0 address:x5690D2AACC203003DBE333046683B698EF945FF250723C0F73297A2A1A41E2F1) - storage_stat:(storage_info - used:(storage_used - cells:(var_uint len:1 value:1) - bits:(var_uint len:1 value:111) - public_cells:(var_uint len:0 value:0)) last_paid:1593722498 - due_payment:nothing) - storage:(account_storage last_trans_lt:7720869000002 - balance:(currencies - grams:(nanograms - amount:(var_uint len:5 value:100000000000)) - other:(extra_currencies - dict:hme_empty)) - state:account_uninit)) -x{C005690D2AACC203003DBE333046683B698EF945FF250723C0F73297A2A1A41E2F12025BC2F7F2341000001C169E9DCD0945D21DBA0004_} -last transaction lt = 7720869000001 hash = 83C15CDED025970FEF7521206E82D2396B462AADB962C7E1F4283D88A0FAB7D4 -account balance is 100000000000ng - -If nobody has sent a valid proof of work with this *seed* and *complexity* before you, the proof-of-work testgiver will accept your proof of work and this will be reflected in the balance of your wallet (10 or 20 seconds may elapse after sending the external message before this happens; be sure to make several attempts and type "last" each time before checking the balance of your wallet to refresh the LiteClient state). In the case of success, you will see that the balance has been increased (and even that your wallet has been created in uninitialized state if it did not exist before). In the case of failure, you will have to obtain the new "seed" and "complexity" and repeat the mining process from the very beginning. - -If you have been lucky and the balance of your wallet has been increased, you may want to initialize the wallet if it wasn't initialized before (more information on wallet creation can be found in LiteClient-HOWTO): - -> sendfile new-wallet-query.boc -... external message status is 1 -> last -> getaccount kQBWkNKqzCAwA9vjMwRmg7aY75Rf8lByPA9zKXoqGkHi8SM7 -... -account state is (account - addr:(addr_std - anycast:nothing workchain_id:0 address:x5690D2AACC203003DBE333046683B698EF945FF250723C0F73297A2A1A41E2F1) - storage_stat:(storage_info - used:(storage_used - cells:(var_uint len:1 value:3) - bits:(var_uint len:2 value:1147) - public_cells:(var_uint len:0 value:0)) last_paid:1593722691 - due_payment:nothing) - storage:(account_storage last_trans_lt:7720945000002 - balance:(currencies - grams:(nanograms - amount:(var_uint len:5 value:99995640998)) - other:(extra_currencies - dict:hme_empty)) - state:(account_active - ( - split_depth:nothing - special:nothing - code:(just - value:(raw@^Cell - x{} - x{FF0020DD2082014C97BA218201339CBAB19C71B0ED44D0D31FD70BFFE304E0A4F260810200D71820D70B1FED44D0D31FD3FFD15112BAF2A122F901541044F910F2A2F80001D31F3120D74A96D307D402FB00DED1A4C8CB1FCBFFC9ED54} - )) - data:(just - value:(raw@^Cell - x{} - x{00000001CE6A50A6E9467C32671667F8C00C5086FC8D62E5645652BED7A80DF634487715} - )) - library:hme_empty)))) -x{C005690D2AACC203003DBE333046683B698EF945FF250723C0F73297A2A1A41E2F1206811EC2F7F23A1800001C16B0BC790945D20D1929934_} - x{FF0020DD2082014C97BA218201339CBAB19C71B0ED44D0D31FD70BFFE304E0A4F260810200D71820D70B1FED44D0D31FD3FFD15112BAF2A122F901541044F910F2A2F80001D31F3120D74A96D307D402FB00DED1A4C8CB1FCBFFC9ED54} - x{00000001CE6A50A6E9467C32671667F8C00C5086FC8D62E5645652BED7A80DF634487715} -last transaction lt = 7720945000001 hash = 73353151859661AB0202EA5D92FF409747F201D10F1E52BD0CBB93E1201676BF -account balance is 99995640998ng - -Now you are a happy owner of 100 test Grams that can be used for whatever testing purposes you had in mind. Congratulations! - -3. Automating the mining process in the case of failure -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you fail to obtain your test Grams for a long time, this may happen because too many other testers are simultaneously "mining" from the same proof-of-work testgiver smart contract. Maybe you should choose another proof-of-work testgiver smart contract from one of the lists given above. Alternatively, you can write a simple script to automatically run `pow-miner` with the correct parameters again and again until success (detected by checking the exit code of `pow-miner`) and invoke the lite-client with parameter -c 'sendfile mined.boc' to send the external message immediately after it is found. - diff --git a/tonlib/CMakeLists.txt b/tonlib/CMakeLists.txt index f86364411e..f281762232 100644 --- a/tonlib/CMakeLists.txt +++ b/tonlib/CMakeLists.txt @@ -72,14 +72,14 @@ if (TONLIB_ENABLE_JNI AND NOT ANDROID) # jni is available by default on Android endif() add_executable(tonlib-cli tonlib/tonlib-cli.cpp) -target_link_libraries(tonlib-cli tonlib tdactor tdutils terminal pow-miner-lib git) +target_link_libraries(tonlib-cli tonlib tdactor tdutils terminal ton_crypto git) if (NOT CMAKE_CROSSCOMPILING) if (TONLIB_ENABLE_JNI) #FIXME #add_dependencies(tonlib tonlib_generate_java_api) endif() -endif() +endif() add_library(tonlibjson_private STATIC tonlib/ClientJson.cpp tonlib/ClientJson.h) target_include_directories(tonlibjson_private PUBLIC $ diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 2c7100f24b..45eec07de9 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -52,7 +52,6 @@ #include "auto/tl/tonlib_api.hpp" -#include "crypto/util/Miner.h" #include "vm/boc.h" #include "vm/cells/CellBuilder.h" #include "lite-client/ext-client.h" @@ -329,11 +328,6 @@ class TonlibCli : public td::actor::Actor { td::TerminalIO::out() << "rwallet address \n"; td::TerminalIO::out() << "rwallet init [: ...]\n"; } - void pminer_help() { - td::TerminalIO::out() << "pminer help\n"; - td::TerminalIO::out() << "pminer start \n"; - td::TerminalIO::out() << "pminer stop\n"; - } void parse_line(td::BufferSlice line) { if (is_closing_) { @@ -393,7 +387,6 @@ class TonlibCli : public td::actor::Actor { dns_help(); pchan_help(); rwallet_help(); - pminer_help(); td::TerminalIO::out() << "blockmode auto|manual\tWith auto mode, all queries will be executed with respect to the latest block. " @@ -504,8 +497,6 @@ class TonlibCli : public td::actor::Actor { run_pchan_cmd(parser, std::move(cmd_promise)); } else if (cmd == "rwallet") { run_rwallet_cmd(parser, std::move(cmd_promise)); - } else if (cmd == "pminer") { - run_pminer_cmd(parser, std::move(cmd_promise)); } else if (cmd == "gethistory") { get_history(parser.read_word(), std::move(cmd_promise)); } else if (cmd == "guessrevision") { @@ -590,216 +581,6 @@ class TonlibCli : public td::actor::Actor { promise.set_error(td::Status::Error("Unknown command")); } - class PowMiner : public td::actor::Actor { - public: - class Callback { - public: - }; - struct Options { - Address giver_address; - Address my_address; - }; - - PowMiner(Options options, td::actor::ActorId client) - : options_(std::move(options)), client_(std::move(client)) { - } - - private: - Options options_; - td::actor::ActorId client_; - - td::optional miner_options_; - static constexpr double QUERY_EACH = 5.0; - td::Timestamp next_options_query_at_; - - bool need_run_miners_{false}; - td::CancellationTokenSource source_; - ton::Miner::Options miner_options_copy_; - std::size_t threads_alive_{0}; - std::vector threads_; - - bool close_flag_{false}; - - template - void send_query(QueryT query, td::Promise promise) { - td::actor::send_lambda(client_, - [self = client_, query = std::move(query), promise = std::move(promise)]() mutable { - self.get_actor_unsafe().make_request(std::move(query), std::move(promise)); - }); - } - - void start_up() override { - next_options_query_at_ = td::Timestamp::now(); - loop(); - } - void hangup() override { - close_flag_ = true; - source_.cancel(); - try_stop(); - } - void try_stop() { - if (threads_alive_ == 0) { - td::TerminalIO::out() << "pminer: stopped\n"; - stop(); - } - } - - void loop() override { - if (close_flag_) { - try_stop(); - return; - } - if (next_options_query_at_ && next_options_query_at_.is_in_past()) { - send_query(tonlib_api::smc_load(options_.giver_address.tonlib_api()), - promise_send_closure(td::actor::actor_id(this), &PowMiner::with_giver_state)); - next_options_query_at_ = {}; - } - - if (miner_options_ && threads_.empty() && need_run_miners_) { - td::TerminalIO::out() << "pminer: start workers\n"; - need_run_miners_ = false; - miner_options_copy_ = miner_options_.value(); - miner_options_copy_.token_ = source_.get_cancellation_token(); - auto n = td::thread::hardware_concurrency(); - threads_alive_ = n; - for (td::uint32 i = 0; i < n; i++) { - threads_.emplace_back([this, actor_id = actor_id(this)] { - auto res = ton::Miner::run(miner_options_copy_); - global_scheduler_->run_in_context_external( - [&] { send_closure(actor_id, &PowMiner::got_answer, std::move(res)); }); - }); - } - } - - alarm_timestamp().relax(next_options_query_at_); - } - - void got_answer(td::optional answer) { - source_.cancel(); - if (--threads_alive_ == 0) { - threads_.clear(); - } - if (answer) { - td::TerminalIO::out() << "pminer: got some result - sending query to the giver\n"; - vm::CellBuilder cb; - cb.store_bytes(answer.unwrap()); - send_query(tonlib_api::raw_createAndSendMessage( - options_.giver_address.tonlib_api(), "", - vm::std_boc_serialize(cb.finalize_novm()).move_as_ok().as_slice().str()), - promise_send_closure(td::actor::actor_id(this), &PowMiner::on_query_sent)); - } - loop(); - } - - void on_query_sent(td::Result> r_ok) { - LOG_IF(ERROR, r_ok.is_error()) << "pminer: " << r_ok.error(); - } - - void with_giver_state(td::Result> r_info) { - if (r_info.is_error()) { - return with_giver_info(r_info.move_as_error()); - } - send_query(tonlib_api::smc_runGetMethod(r_info.ok()->id_, - make_object("get_pow_params"), {}), - promise_send_closure(td::actor::actor_id(this), &PowMiner::with_giver_info)); - } - - void with_giver_info(td::Result> r_info) { - auto status = do_with_giver_info(std::move(r_info)); - LOG_IF(ERROR, status.is_error()) << "pminer: " << status; - next_options_query_at_ = td::Timestamp::in(QUERY_EACH); - return loop(); - } - - td::Result to_number(const tonlib_api::object_ptr& entry, - td::int32 bits) { - if (entry->get_id() != tonlib_api::tvm_stackEntryNumber::ID) { - return td::Status::Error("Expected stackEntryNumber"); - } - auto& number_str = static_cast(*entry.get()).number_->number_; - auto num = td::make_refint(); - if (num.write().parse_dec(number_str.data(), (int)number_str.size()) < (int)number_str.size()) { - return td::Status::Error("Failed to parse a number"); - } - if (!num->unsigned_fits_bits(bits)) { - return td::Status::Error(PSLICE() << "Number is too big " << num->to_dec_string() << " " << bits); - } - return num; - } - - td::Status do_with_giver_info(td::Result> r_info) { - TRY_RESULT(info, std::move(r_info)); - if (info->stack_.size() < 2) { - return td::Status::Error("Unexpected `get_pow_params` result format"); - } - TRY_RESULT(seed, to_number(info->stack_[0], 128)); - TRY_RESULT(complexity, to_number(info->stack_[1], 256)); - ton::Miner::Options options; - seed->export_bytes(options.seed.data(), 16, false); - complexity->export_bytes(options.complexity.data(), 32, false); - - TRY_RESULT(address, block::StdAddress::parse(options_.my_address.address->account_address_)); - options.my_address = std::move(address); - options.token_ = source_.get_cancellation_token(); - - if (miner_options_ && miner_options_.value().seed == options.seed) { - return td::Status::OK(); - } - - td::TerminalIO::out() << "pminer: got new options\n"; - td::BigInt256 bigpower, hrate; - bigpower.set_pow2(256).mod_div(*complexity, hrate); - long long hash_rate = hrate.to_long(); - td::TerminalIO::out() << "[ expected required hashes for success: " << hash_rate << " ]\n"; - miner_options_ = std::move(options); - need_run_miners_ = true; - source_.cancel(); - - return td::Status::OK(); - } - }; - - td::uint64 pow_miner_id_{0}; - std::map> pow_miners_; - - void pminer_start(td::ConstParser& parser, td::Promise promise) { - if (!pow_miners_.empty()) { - promise.set_error(td::Status::Error("One pminer is already running")); - } - TRY_RESULT_PROMISE_PREFIX(promise, giver_address, to_account_address(parser.read_word(), false), "giver address"); - TRY_RESULT_PROMISE_PREFIX(promise, my_address, to_account_address(parser.read_word(), false), "my address"); - - auto id = ++pow_miner_id_; - - PowMiner::Options options; - options.giver_address = std::move(giver_address); - options.my_address = std::move(my_address); - pow_miners_.emplace(id, td::actor::create_actor("PowMiner", std::move(options), client_.get())); - td::TerminalIO::out() << "Miner #" << id << " created"; - promise.set_value({}); - } - - void pminer_stop(td::ConstParser& parser, td::Promise promise) { - pow_miners_.clear(); - promise.set_value({}); - } - - void run_pminer_cmd(td::ConstParser& parser, td::Promise promise) { - auto cmd = parser.read_word(); - if (cmd == "help") { - pminer_help(); - return promise.set_value(td::Unit()); - } - - if (cmd == "start") { - return pminer_start(parser, std::move(promise)); - } - if (cmd == "stop") { - return pminer_stop(parser, std::move(promise)); - } - promise.set_error(td::Status::Error("Unknown command")); - } - void run_pchan_cmd(td::ConstParser& parser, td::Promise promise) { auto cmd = parser.read_word(); if (cmd == "help") { From d1e00f5e80e6759f295474ea63e75b83bdf0a3d2 Mon Sep 17 00:00:00 2001 From: Evgeny Kapun Date: Thu, 22 May 2025 21:24:03 +0300 Subject: [PATCH 253/524] Remove legacy Ed25519 code (#1684) --- CMakeLists.txt | 3 +- assembly/native/build-macos-portable.sh | 2 +- assembly/native/build-macos-shared.sh | 2 +- assembly/native/build-ubuntu-appimages.sh | 2 +- assembly/native/build-ubuntu-portable.sh | 2 +- assembly/native/build-ubuntu-shared.sh | 2 +- assembly/native/build-windows-2019.bat | 2 +- assembly/native/build-windows.bat | 2 +- crypto/CMakeLists.txt | 13 - crypto/Ed25519.cpp | 97 ------- crypto/ellcurve/Ed25519.cpp | 280 ------------------- crypto/ellcurve/Ed25519.h | 188 ------------- crypto/ellcurve/Fp25519.cpp | 33 --- crypto/ellcurve/Fp25519.h | 32 --- crypto/ellcurve/Montgomery.cpp | 138 ---------- crypto/ellcurve/Montgomery.h | 123 --------- crypto/ellcurve/TwEdwards.cpp | 255 ------------------ crypto/ellcurve/TwEdwards.h | 145 ---------- crypto/test/Ed25519.cpp | 1 - crypto/test/test-ed25519-crypto.cpp | 314 ---------------------- doc/Tests.md | 4 +- 21 files changed, 10 insertions(+), 1630 deletions(-) delete mode 100644 crypto/ellcurve/Ed25519.cpp delete mode 100644 crypto/ellcurve/Ed25519.h delete mode 100644 crypto/ellcurve/Fp25519.cpp delete mode 100644 crypto/ellcurve/Fp25519.h delete mode 100644 crypto/ellcurve/Montgomery.cpp delete mode 100644 crypto/ellcurve/Montgomery.h delete mode 100644 crypto/ellcurve/TwEdwards.cpp delete mode 100644 crypto/ellcurve/TwEdwards.h delete mode 100644 crypto/test/test-ed25519-crypto.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b002b2844c..5cc0a32da9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -181,7 +181,7 @@ endif() if (TON_ARCH AND NOT MSVC) CHECK_CXX_COMPILER_FLAG( "-march=${TON_ARCH}" COMPILER_OPT_ARCH_SUPPORTED ) if (TON_ARCH STREQUAL "apple-m1") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=${TON_ARCH}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=${TON_ARCH}") elseif(COMPILER_OPT_ARCH_SUPPORTED) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=${TON_ARCH}") elseif(NOT TON_ARCH STREQUAL "native") @@ -534,7 +534,6 @@ endif() enable_testing() set(TEST_OPTIONS "--regression ${CMAKE_CURRENT_SOURCE_DIR}/test/regression-tests.ans --filter -Bench") separate_arguments(TEST_OPTIONS) -add_test(test-ed25519-crypto crypto/test-ed25519-crypto) add_test(test-ed25519 test-ed25519) add_test(test-bigint test-bigint) add_test(test-vm test-vm ${TEST_OPTIONS}) diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index c3dec11d20..e112ab499e 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -152,7 +152,7 @@ if [ "$with_tests" = true ]; then tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ lite-client validator-engine-console generate-random-id json2tlo dht-server \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ - test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \ + test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont \ test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index c769e95797..b21f21f893 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -77,7 +77,7 @@ if [ "$with_tests" = true ]; then tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ lite-client validator-engine-console generate-random-id json2tlo dht-server \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ - test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \ + test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont \ test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } diff --git a/assembly/native/build-ubuntu-appimages.sh b/assembly/native/build-ubuntu-appimages.sh index b21d7df8c9..015b33cb78 100644 --- a/assembly/native/build-ubuntu-appimages.sh +++ b/assembly/native/build-ubuntu-appimages.sh @@ -62,7 +62,7 @@ if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ + adnl-proxy create-state emulator test-ed25519 test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh index d2e733954f..3a65ddd2bb 100644 --- a/assembly/native/build-ubuntu-portable.sh +++ b/assembly/native/build-ubuntu-portable.sh @@ -136,7 +136,7 @@ if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ + adnl-proxy create-state emulator test-ed25519 test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index 3fc6db3156..0bdf4072ba 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -65,7 +65,7 @@ if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ + adnl-proxy create-state emulator test-ed25519 test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver diff --git a/assembly/native/build-windows-2019.bat b/assembly/native/build-windows-2019.bat index f5fd0be367..2c635d3a63 100644 --- a/assembly/native/build-windows-2019.bat +++ b/assembly/native/build-windows-2019.bat @@ -147,7 +147,7 @@ IF "%1"=="-t" ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ -test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net ^ +test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^ test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver IF %errorlevel% NEQ 0 ( diff --git a/assembly/native/build-windows.bat b/assembly/native/build-windows.bat index 41697eb0fd..6ff7166af3 100644 --- a/assembly/native/build-windows.bat +++ b/assembly/native/build-windows.bat @@ -147,7 +147,7 @@ IF "%1"=="-t" ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ -test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net ^ +test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^ test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver IF %errorlevel% NEQ 0 ( diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index cd2c7e47eb..fd3c41c09d 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -10,10 +10,6 @@ set(TON_CRYPTO_CORE_SOURCE common/bigexp.cpp common/bitstring.cpp common/util.cpp - ellcurve/Ed25519.cpp - ellcurve/Fp25519.cpp - ellcurve/Montgomery.cpp - ellcurve/TwEdwards.cpp openssl/bignum.cpp openssl/residue.cpp openssl/rand.cpp @@ -32,11 +28,6 @@ set(TON_CRYPTO_CORE_SOURCE common/linalloc.hpp common/promiseop.hpp - ellcurve/Ed25519.h - ellcurve/Fp25519.h - ellcurve/Montgomery.h - ellcurve/TwEdwards.h - openssl/bignum.h openssl/digest.hpp openssl/rand.hpp @@ -352,10 +343,6 @@ target_include_directories(ton_db PUBLIC $) target_link_libraries(ton_db PUBLIC tdutils tddb ton_crypto) -add_executable(test-ed25519-crypto test/test-ed25519-crypto.cpp) -target_include_directories(test-ed25519-crypto PUBLIC $) -target_link_libraries(test-ed25519-crypto PUBLIC ton_crypto) - add_library(fift-lib STATIC ${FIFT_SOURCE}) target_include_directories(fift-lib PUBLIC $) target_link_libraries(fift-lib PUBLIC ton_crypto) diff --git a/crypto/Ed25519.cpp b/crypto/Ed25519.cpp index c263a0cf1a..fa5b4f2f6d 100644 --- a/crypto/Ed25519.cpp +++ b/crypto/Ed25519.cpp @@ -18,12 +18,8 @@ */ #include "crypto/Ed25519.h" -#if TD_HAVE_OPENSSL - #include -#if OPENSSL_VERSION_NUMBER >= 0x10101000L && OPENSSL_VERSION_NUMBER != 0x20000000L || defined(OPENSSL_IS_BORINGSSL) - #include "td/utils/base64.h" #include "td/utils/BigNum.h" #include "td/utils/format.h" @@ -35,12 +31,6 @@ #include #include -#else - -#include "crypto/ellcurve/Ed25519.h" - -#endif - namespace td { Ed25519::PublicKey::PublicKey(SecureString octet_string) : octet_string_(std::move(octet_string)) { @@ -57,8 +47,6 @@ SecureString Ed25519::PrivateKey::as_octet_string() const { return octet_string_.copy(); } -#if OPENSSL_VERSION_NUMBER >= 0x10101000L && OPENSSL_VERSION_NUMBER != 0x20000000L || defined(OPENSSL_IS_BORINGSSL) - namespace detail { static Result X25519_key_from_PKEY(EVP_PKEY *pkey, bool is_private) { @@ -314,89 +302,4 @@ int Ed25519::version() { return OPENSSL_VERSION_NUMBER; } -#else - -Result Ed25519::generate_private_key() { - crypto::Ed25519::PrivateKey private_key; - if (!private_key.random_private_key(true)) { - return Status::Error("Can't generate random private key"); - } - SecureString private_key_buf(32); - if (!private_key.export_private_key(private_key_buf.as_mutable_slice())) { - return Status::Error("Failed to export private key"); - } - return PrivateKey(std::move(private_key_buf)); -} - -Result Ed25519::PrivateKey::get_public_key() const { - crypto::Ed25519::PrivateKey private_key; - if (!private_key.import_private_key(Slice(octet_string_).ubegin())) { - return Status::Error("Bad private key"); - } - SecureString public_key(32); - if (!private_key.get_public_key().export_public_key(public_key.as_mutable_slice())) { - return Status::Error("Failed to export public key"); - } - return PublicKey(std::move(public_key)); -} - -Result Ed25519::PrivateKey::as_pem(Slice password) const { - return Status::Error("Not supported"); -} - -Result Ed25519::PrivateKey::from_pem(Slice pem, Slice password) { - return Status::Error("Not supported"); -} - -Result Ed25519::PrivateKey::sign(Slice data) const { - crypto::Ed25519::PrivateKey private_key; - if (!private_key.import_private_key(Slice(octet_string_).ubegin())) { - return Status::Error("Bad private key"); - } - SecureString signature(crypto::Ed25519::sign_bytes, '\0'); - if (!private_key.sign_message(signature.as_mutable_slice(), data)) { - return Status::Error("Failed to sign message"); - } - return std::move(signature); -} - -Status Ed25519::PublicKey::verify_signature(Slice data, Slice signature) const { - if (signature.size() != crypto::Ed25519::sign_bytes) { - return Status::Error("Signature has invalid length"); - } - - crypto::Ed25519::PublicKey public_key; - if (!public_key.import_public_key(Slice(octet_string_).ubegin())) { - return Status::Error("Bad public key"); - } - if (public_key.check_message_signature(signature, data)) { - return Status::OK(); - } - return Status::Error("Wrong signature"); -} - -Result Ed25519::compute_shared_secret(const PublicKey &public_key, const PrivateKey &private_key) { - crypto::Ed25519::PrivateKey tmp_private_key; - if (!tmp_private_key.import_private_key(Slice(private_key.as_octet_string()).ubegin())) { - return Status::Error("Bad private key"); - } - crypto::Ed25519::PublicKey tmp_public_key; - if (!tmp_public_key.import_public_key(Slice(public_key.as_octet_string()).ubegin())) { - return Status::Error("Bad public key"); - } - SecureString shared_secret(32, '\0'); - if (!tmp_private_key.compute_shared_secret(shared_secret.as_mutable_slice(), tmp_public_key)) { - return Status::Error("Failed to compute shared secret"); - } - return std::move(shared_secret); -} - -int Ed25519::version() { - return 0; -} - -#endif - } // namespace td - -#endif diff --git a/crypto/ellcurve/Ed25519.cpp b/crypto/ellcurve/Ed25519.cpp deleted file mode 100644 index a38eb079e0..0000000000 --- a/crypto/ellcurve/Ed25519.cpp +++ /dev/null @@ -1,280 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "Ed25519.h" - -#include "td/utils/Random.h" - -namespace crypto { -namespace Ed25519 { - -bool all_bytes_same(const unsigned char *str, std::size_t size) { - unsigned char c = str[0]; - for (std::size_t i = 0; i < size; i++) { - if (str[i] != c) { - return false; - } - } - return true; -} - -void PublicKey::clear(void) { - if (inited != pk_empty) { - std::memset(pubkey, 0, pubkey_bytes); - PubKey.zeroize(); - PubKey_xz.zeroize(); - } - inited = pk_empty; -} - -PublicKey::PublicKey(const unsigned char pub_key[pubkey_bytes]) - : inited(pk_empty), PubKey(ellcurve::Fp25519()), PubKey_xz(ellcurve::Fp25519()) { - import_public_key(pub_key); -} - -PublicKey::PublicKey(const ellcurve::TwEdwardsCurve::SegrePoint &Pub_Key) - : inited(pk_empty), PubKey(ellcurve::Fp25519()), PubKey_xz(ellcurve::Fp25519()) { - import_public_key(Pub_Key); -} - -bool PublicKey::import_public_key(const unsigned char pub_key[pubkey_bytes]) { - clear(); - if (all_bytes_same(pub_key, pubkey_bytes)) { - return false; - } - bool ok = false; - PubKey = ellcurve::Ed25519().import_point(pub_key, ok); - if (!ok) { - clear(); - return false; - } - std::memcpy(pubkey, pub_key, pubkey_bytes); - PubKey_xz.X = PubKey.Z + PubKey.Y; - PubKey_xz.Z = PubKey.Z - PubKey.Y; - inited = pk_init; - return true; -} - -bool PublicKey::import_public_key(const ellcurve::TwEdwardsCurve::SegrePoint &Pub_Key) { - clear(); - if (!Pub_Key.is_valid()) { - return false; - } - PubKey = Pub_Key; - PubKey_xz.X = PubKey.Z + PubKey.Y; - PubKey_xz.Z = PubKey.Z - PubKey.Y; - inited = pk_init; - - if (!PubKey.export_point(pubkey)) { - clear(); - return false; - } - return true; -} - -bool PublicKey::export_public_key(unsigned char pubkey_buffer[pubkey_bytes]) const { - if (inited != pk_init) { - std::memset(pubkey_buffer, 0, pubkey_bytes); - return false; - } else { - std::memcpy(pubkey_buffer, pubkey, pubkey_bytes); - return true; - } -} - -bool PublicKey::check_message_signature(const unsigned char signature[sign_bytes], const unsigned char *message, - std::size_t msg_size) { - if (inited != pk_init) { - return false; - } - unsigned char hash[64]; - { - digest::SHA512 hasher(signature, 32); - hasher.feed(pubkey, 32); - hasher.feed(message, msg_size); - hasher.extract(hash); - } - auto &E = ellcurve::Ed25519(); - const arith::Bignum &L = E.get_ell(); - arith::Bignum H, S; - S.import_lsb(signature + 32, 32); - H.import_lsb(hash, 64); - H %= L; - H = L - H; - auto sG = E.power_gen(S); - auto hA = E.power_point(PubKey, H); - auto pR1 = E.add_points(sG, hA); - unsigned char pR1_bytes[32]; - if (!pR1.export_point(pR1_bytes)) { - return false; - } - return !std::memcmp(pR1_bytes, signature, 32); -} - -// --------------------- -class PrivateKey; - -bool PrivateKey::random_private_key(bool strong) { - inited = false; - if (!prng::rand_gen().rand_bytes(privkey, privkey_bytes, strong)) { - clear(); - return false; - } - return process_private_key(); -} - -void PrivateKey::clear(void) { - std::memset(privkey, 0, privkey_bytes); - std::memset(priv_salt, 0, sizeof(priv_salt)); - priv_exp.clear(); - PubKey.clear(); - inited = false; -} - -bool PrivateKey::import_private_key(const unsigned char pk[privkey_bytes]) { - clear(); - if (all_bytes_same(pk, privkey_bytes)) { - return false; - } - std::memcpy(privkey, pk, privkey_bytes); - return process_private_key(); -} - -bool PrivateKey::export_private_key(unsigned char pk[privkey_bytes]) const { // careful! - if (!inited) { - std::memset(pk, 0, privkey_bytes); - return false; - } else { - std::memcpy(pk, privkey, privkey_bytes); - return true; - } -} - -bool PrivateKey::process_private_key() { - unsigned char buff[64]; - digest::hash_str(buff, privkey, privkey_bytes); - std::memcpy(priv_salt, buff + 32, 32); - buff[0] = (unsigned char)(buff[0] & -8); - buff[31] = (unsigned char)((buff[31] | 0x40) & ~0x80); - priv_exp.import_lsb(buff, 32); - PubKey = ellcurve::Ed25519().power_gen(priv_exp, true); // uniform - inited = PubKey.ok(); - if (!inited) { - clear(); - } - return inited; -} - -bool PrivateKey::compute_shared_secret(unsigned char secret[shared_secret_bytes], const PublicKey &Pub) { - if (!inited || !Pub.ok()) { - std::memset(secret, 0, shared_secret_bytes); - *(long *)secret = static_cast(td::Random::fast_uint64()); - return false; - } - // uniform power! - auto P = ellcurve::Curve25519().power_xz(Pub.get_point_xz(), priv_exp); - if (P.is_infty()) { - std::memset(secret, 0, shared_secret_bytes); - *(long *)secret = static_cast(td::Random::fast_uint64()); - return false; - } - P.export_point_u(secret); - return true; -} - -bool PrivateKey::compute_temp_shared_secret(unsigned char secret[shared_secret_bytes], - const unsigned char temp_pub_key[pubkey_bytes]) { - PublicKey tempPubkey(temp_pub_key); - if (!tempPubkey.ok()) { - return false; - } - return compute_shared_secret(secret, tempPubkey); -} - -bool PrivateKey::sign_message(unsigned char signature[sign_bytes], const unsigned char *message, std::size_t msg_size) { - if (!inited) { - std::memset(signature, 0, sign_bytes); - return false; - } - unsigned char r_bytes[64]; - digest::hash_two_str(r_bytes, priv_salt, 32, message, msg_size); - const arith::Bignum &L = ellcurve::Ed25519().get_ell(); - arith::Bignum eR; - eR.import_lsb(r_bytes, 64); - eR %= L; - std::memset(r_bytes, 0, sizeof(r_bytes)); - - // uniform power - auto pR = ellcurve::Ed25519().power_gen(eR, true); - - auto ok = pR.export_point(signature, true); - (void)ok; - assert(ok); - { - digest::SHA512 hasher(signature, 32); - hasher.feed(PubKey.get_pubkey_ptr(), 32); - hasher.feed(message, msg_size); - hasher.extract(r_bytes); - } - arith::Bignum S; - S.import_lsb(r_bytes, 64); - S %= L; - S *= priv_exp; - S += eR; - S %= L; - eR.clear(); - S.export_lsb(signature + 32, 32); - return true; -} - -// --------------------------------- -class TempKeyGenerator; - -unsigned char *TempKeyGenerator::get_temp_private_key(unsigned char *to, const unsigned char *message, std::size_t size, - const unsigned char *rand, - std::size_t rand_size) { // rand may be 0 - digest::SHA256 hasher(message, size); - hasher.feed(random_salt, salt_size); - if (rand && rand_size) { - hasher.feed(rand, rand_size); - } - if (!to) { - to = buffer; - } - hasher.extract(to); - //++ *((long *)random_salt); - return to; -} - -void TempKeyGenerator::create_temp_private_key(PrivateKey &pk, const unsigned char *message, std::size_t size, - const unsigned char *rand, std::size_t rand_size) { - pk.import_private_key(get_temp_private_key(buffer, message, size, rand, rand_size)); - std::memset(buffer, 0, privkey_bytes); -} - -bool TempKeyGenerator::create_temp_shared_secret(unsigned char temp_pub_key[pubkey_bytes], - unsigned char shared_secret[shared_secret_bytes], - const PublicKey &recipientPubKey, const unsigned char *message, - std::size_t size, const unsigned char *rand, std::size_t rand_size) { - PrivateKey tmpPk; - create_temp_private_key(tmpPk, message, size, rand, rand_size); - return tmpPk.export_public_key(temp_pub_key) && tmpPk.compute_shared_secret(shared_secret, recipientPubKey); -} - -} // namespace Ed25519 -} // namespace crypto diff --git a/crypto/ellcurve/Ed25519.h b/crypto/ellcurve/Ed25519.h deleted file mode 100644 index 4b9e6348f3..0000000000 --- a/crypto/ellcurve/Ed25519.h +++ /dev/null @@ -1,188 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once -#include "ellcurve/Montgomery.h" -#include "ellcurve/TwEdwards.h" -#include "openssl/digest.hpp" -#include "openssl/rand.hpp" -#include -#include - -#include "td/utils/buffer.h" - -namespace crypto { -namespace Ed25519 { - -const int privkey_bytes = 32; -const int pubkey_bytes = 32; -const int sign_bytes = 64; -const int shared_secret_bytes = 32; - -class PublicKey { - enum { pk_empty, pk_xz, pk_init } inited; - unsigned char pubkey[pubkey_bytes]; - ellcurve::TwEdwardsCurve::SegrePoint PubKey; - ellcurve::MontgomeryCurve::PointXZ PubKey_xz; - - public: - PublicKey() : inited(pk_empty), PubKey(ellcurve::Fp25519()), PubKey_xz(ellcurve::Fp25519()) { - } - PublicKey(const unsigned char pub_key[pubkey_bytes]); - PublicKey(td::Slice pub_key) : PublicKey(pub_key.ubegin()) { - CHECK(pub_key.size() == pubkey_bytes); - } - PublicKey(const ellcurve::TwEdwardsCurve::SegrePoint &Pub_Key); - - bool import_public_key(const unsigned char pub_key[pubkey_bytes]); - bool import_public_key(td::Slice pub_key) { - CHECK(pub_key.size() == pubkey_bytes); - return import_public_key(pub_key.ubegin()); - } - bool import_public_key(const ellcurve::TwEdwardsCurve::SegrePoint &Pub_Key); - bool export_public_key(unsigned char pubkey_buffer[pubkey_bytes]) const; - bool export_public_key(td::MutableSlice pubk) const { - CHECK(pubk.size() == pubkey_bytes); - return export_public_key(pubk.ubegin()); - } - bool check_message_signature(const unsigned char signature[sign_bytes], const unsigned char *message, - std::size_t msg_size); - bool check_message_signature(td::Slice signature, td::Slice message) { - CHECK(signature.size() == sign_bytes); - return check_message_signature(signature.ubegin(), message.ubegin(), message.size()); - } - - void clear(); - bool ok() const { - return inited == pk_init; - } - - const unsigned char *get_pubkey_ptr() const { - return inited == pk_init ? pubkey : 0; - } - const ellcurve::TwEdwardsCurve::SegrePoint &get_point() const { - return PubKey; - } - const ellcurve::MontgomeryCurve::PointXZ &get_point_xz() const { - return PubKey_xz; - } -}; - -class PrivateKey { - public: - struct priv_key_no_copy {}; - PrivateKey() : inited(false) { - memset(privkey, 0, privkey_bytes); - } - PrivateKey(const unsigned char pk[privkey_bytes]) : inited(false) { - memset(privkey, 0, privkey_bytes); - import_private_key(pk); - } - PrivateKey(td::Slice pk) : inited(false) { - CHECK(pk.size() == privkey_bytes); - memset(privkey, 0, privkey_bytes); - import_private_key(pk.ubegin()); - } - ~PrivateKey() { - clear(); - } - bool random_private_key(bool strong = false); - bool import_private_key(const unsigned char pk[privkey_bytes]); - bool import_private_key(td::Slice pk) { - CHECK(pk.size() == privkey_bytes); - return import_private_key(pk.ubegin()); - } - bool export_private_key(unsigned char pk[privkey_bytes]) const; // careful! - bool export_private_key(td::MutableSlice pk) const { // careful! - return export_private_key(pk.ubegin()); - } - bool export_public_key(unsigned char pubk[pubkey_bytes]) const { - return PubKey.export_public_key(pubk); - } - bool export_public_key(td::MutableSlice pubk) const { - return PubKey.export_public_key(pubk); - } - void clear(); - bool ok() const { - return inited; - } - - // used for EdDSA (sign) - bool sign_message(unsigned char signature[sign_bytes], const unsigned char *message, std::size_t msg_size); - bool sign_message(td::MutableSlice signature, td::Slice message) { - CHECK(signature.size() == sign_bytes); - return sign_message(signature.ubegin(), message.ubegin(), message.size()); - } - // used for ECDH (encrypt / decrypt) - bool compute_shared_secret(unsigned char secret[shared_secret_bytes], const PublicKey &Pub); - bool compute_shared_secret(td::MutableSlice secret, const PublicKey &Pub) { - CHECK(secret.size() == shared_secret_bytes); - return compute_shared_secret(secret.ubegin(), Pub); - } - // used for EC asymmetric decryption - bool compute_temp_shared_secret(unsigned char secret[shared_secret_bytes], - const unsigned char temp_pub_key[pubkey_bytes]); - - const PublicKey &get_public_key() const { - return PubKey; - } - - private: - bool inited; - unsigned char privkey[privkey_bytes]; - unsigned char priv_salt[32]; - arith::Bignum priv_exp; - PublicKey PubKey; - - bool process_private_key(); - PrivateKey(const PrivateKey &) { - throw priv_key_no_copy(); - } - PrivateKey &operator=(const PrivateKey &) { - throw priv_key_no_copy(); - } -}; - -// use one TempKeyGenerator object a lot of times -class TempKeyGenerator { - enum { salt_size = 64 }; - unsigned char random_salt[salt_size]; - unsigned char buffer[privkey_bytes]; - - public: - TempKeyGenerator() { - prng::rand_gen().strong_rand_bytes(random_salt, salt_size); - } - ~TempKeyGenerator() { - memset(random_salt, 0, salt_size); - memset(buffer, 0, privkey_bytes); - } - - unsigned char *get_temp_private_key(unsigned char *to, const unsigned char *message, std::size_t size, - const unsigned char *rand = 0, std::size_t rand_size = 0); // rand may be 0 - void create_temp_private_key(PrivateKey &pk, const unsigned char *message, std::size_t size, - const unsigned char *rand = 0, std::size_t rand_size = 0); - - // sets temp_pub_key and shared_secret for one-time asymmetric encryption of message - bool create_temp_shared_secret(unsigned char temp_pub_key[pubkey_bytes], unsigned char secret[shared_secret_bytes], - const PublicKey &recipientPubKey, const unsigned char *message, std::size_t size, - const unsigned char *rand = 0, std::size_t rand_size = 0); -}; - -} // namespace Ed25519 -} // namespace crypto diff --git a/crypto/ellcurve/Fp25519.cpp b/crypto/ellcurve/Fp25519.cpp deleted file mode 100644 index 7b248f64ad..0000000000 --- a/crypto/ellcurve/Fp25519.cpp +++ /dev/null @@ -1,33 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "ellcurve/Fp25519.h" - -namespace ellcurve { -using namespace arith; - -const Bignum& P25519() { - static const Bignum P25519 = (Bignum(1) << 255) - 19; - return P25519; -} - -td::Ref Fp25519() { - static const td::Ref Fp25519(true, P25519()); - return Fp25519; -} -} // namespace ellcurve diff --git a/crypto/ellcurve/Fp25519.h b/crypto/ellcurve/Fp25519.h deleted file mode 100644 index b6b63be5e4..0000000000 --- a/crypto/ellcurve/Fp25519.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once -#include "common/refcnt.hpp" -#include "openssl/residue.h" - -namespace ellcurve { -using namespace arith; - -// returns 2^255-19 -const Bignum& P25519(); - -// residue ring modulo P25519 -td::Ref Fp25519(); - -} // namespace ellcurve diff --git a/crypto/ellcurve/Montgomery.cpp b/crypto/ellcurve/Montgomery.cpp deleted file mode 100644 index ed71910d57..0000000000 --- a/crypto/ellcurve/Montgomery.cpp +++ /dev/null @@ -1,138 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "ellcurve/Montgomery.h" - -#include -#include - -namespace ellcurve { -using namespace arith; - -class MontgomeryCurve; - -void MontgomeryCurve::init() { - assert(!((a_short + 2) & 3) && a_short >= 0); -} - -void MontgomeryCurve::set_order_cofactor(const Bignum& order, int cof) { - assert(order > 0); - assert(cof >= 0); - assert(cof == 0 || (order % cof) == 0); - Order = order; - cofactor = cofactor_short = cof; - if (cof > 0) { - L = order / cof; - assert(is_prime(L)); - } - assert(!power_gen_xz(1).is_infty()); - assert(power_gen_xz(Order).is_infty()); -} - -// computes u(P+Q)*u(P-Q) as X/Z -MontgomeryCurve::PointXZ MontgomeryCurve::add_xz(const MontgomeryCurve::PointXZ& P, - const MontgomeryCurve::PointXZ& Q) const { - Residue u = (P.X + P.Z) * (Q.X - Q.Z); - Residue v = (P.X - P.Z) * (Q.X + Q.Z); - return MontgomeryCurve::PointXZ(sqr(u + v), sqr(u - v)); -} - -// computes u(2P) as X/Z -MontgomeryCurve::PointXZ MontgomeryCurve::double_xz(const MontgomeryCurve::PointXZ& P) const { - Residue u = sqr(P.X + P.Z); - Residue v = sqr(P.X - P.Z); - Residue w = u - v; - return PointXZ(u * v, w * (v + Residue(a_short, ring) * w)); -} - -MontgomeryCurve::PointXZ MontgomeryCurve::power_gen_xz(const Bignum& n) const { - return power_xz(Gu, n); -} - -MontgomeryCurve::PointXZ MontgomeryCurve::power_xz(const Residue& u, const Bignum& n) const { - return power_xz(PointXZ(u), n); -} - -// computes u([n]P) in form X/Z -MontgomeryCurve::PointXZ MontgomeryCurve::power_xz(const PointXZ& A, const Bignum& n) const { - assert(n >= 0); - if (n == 0) { - return PointXZ(ring); - } - - int k = n.num_bits(); - PointXZ P(A); - PointXZ Q(double_xz(P)); - for (int i = k - 2; i >= 0; --i) { - PointXZ PQ(add_xz(P, Q)); - PQ.X *= A.Z; - PQ.Z *= A.X; - if (n[i]) { - P = PQ; - Q = double_xz(Q); - } else { - Q = PQ; - P = double_xz(P); - } - } - return P; -} - -bool MontgomeryCurve::PointXZ::export_point_y(unsigned char buffer[32]) const { - if ((X + Z).is_zero()) { - std::memset(buffer, 0xff, 32); - return false; - } else { - get_y().extract().export_lsb(buffer, 32); - return true; - } -} - -bool MontgomeryCurve::PointXZ::export_point_u(unsigned char buffer[32]) const { - if (Z.is_zero()) { - std::memset(buffer, 0xff, 32); - return false; - } else { - get_u().extract().export_lsb(buffer, 32); - return true; - } -} - -MontgomeryCurve::PointXZ MontgomeryCurve::import_point_u(const unsigned char point[32]) const { - Bignum u; - u.import_lsb(point, 32); - u[255] = 0; - return PointXZ(Residue(u, ring)); -} - -MontgomeryCurve::PointXZ MontgomeryCurve::import_point_y(const unsigned char point[32]) const { - Bignum y; - y.import_lsb(point, 32); - y[255] = 0; - return PointXZ(Residue(y, ring), true); -} - -const MontgomeryCurve& Curve25519() { - static const MontgomeryCurve Curve25519 = [] { - MontgomeryCurve res(486662, 9, Fp25519()); - res.set_order_cofactor(hex_string{"80000000000000000000000000000000a6f7cef517bce6b2c09318d2e7ae9f68"}, 8); - return res; - }(); - return Curve25519; -} -} // namespace ellcurve diff --git a/crypto/ellcurve/Montgomery.h b/crypto/ellcurve/Montgomery.h deleted file mode 100644 index 94d852c06c..0000000000 --- a/crypto/ellcurve/Montgomery.h +++ /dev/null @@ -1,123 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once -#include -#include - -#include "openssl/bignum.h" -#include "openssl/residue.h" -#include "ellcurve/Fp25519.h" - -namespace ellcurve { -using namespace arith; - -class MontgomeryCurve { - td::Ref ring; - int A_short; // v^2 = u^2 + Au + 1 - int Gu_short; // u(G) - int a_short; // (A+2)/4 - Residue A_; - Residue Gu; - Bignum P_; - Bignum L; - Bignum Order; - Bignum cofactor; - int cofactor_short; - - void init(); - - public: - MontgomeryCurve(int _A, int _Gu, td::Ref _R) - : ring(_R) - , A_short(_A) - , Gu_short(_Gu) - , a_short((_A + 2) / 4) - , A_(_A, _R) - , Gu(_Gu, _R) - , P_(_R->get_modulus()) - , cofactor_short(0) { - init(); - } - - const Residue& get_gen_u() const { - return Gu; - } - const Bignum& get_ell() const { - return L; - } - const Bignum& get_order() const { - return Order; - } - td::Ref get_base_ring() const { - return ring; - } - const Bignum& get_p() const { - return P_; - } - - void set_order_cofactor(const Bignum& order, int cof); - - struct PointXZ { - Residue X, Z; - PointXZ(Residue x, Residue z) : X(x), Z(z) { - x.same_ring(z); - } - PointXZ(td::Ref r) : X(r->one()), Z(r->zero()) { - } - explicit PointXZ(Residue u) : X(u), Z(u.ring_of().one()) { - } - explicit PointXZ(Residue y, bool) : X(y.ring_of().one() + y), Z(y.ring_of().one() - y) { - } - PointXZ(const PointXZ& P) : X(P.X), Z(P.Z) { - } - PointXZ& operator=(const PointXZ& P) { - X = P.X; - Z = P.Z; - return *this; - } - Residue get_u() const { - return X * inverse(Z); - } - Residue get_v(bool sign_v = false) const; - bool is_infty() const { - return Z.is_zero(); - } - Residue get_y() const { - return (X - Z) * inverse(X + Z); - } - bool export_point_y(unsigned char buffer[32]) const; - bool export_point_u(unsigned char buffer[32]) const; - void zeroize() { - X = Z = Z.ring_of().zero(); - } - }; - - PointXZ power_gen_xz(const Bignum& n) const; - PointXZ power_xz(const Residue& u, const Bignum& n) const; - PointXZ power_xz(const PointXZ& P, const Bignum& n) const; - PointXZ add_xz(const PointXZ& P, const PointXZ& Q) const; - PointXZ double_xz(const PointXZ& P) const; - - PointXZ import_point_u(const unsigned char point[32]) const; - PointXZ import_point_y(const unsigned char point[32]) const; -}; - -const MontgomeryCurve& Curve25519(); - -} // namespace ellcurve diff --git a/crypto/ellcurve/TwEdwards.cpp b/crypto/ellcurve/TwEdwards.cpp deleted file mode 100644 index eb131f7025..0000000000 --- a/crypto/ellcurve/TwEdwards.cpp +++ /dev/null @@ -1,255 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "ellcurve/TwEdwards.h" -#include -#include - -namespace ellcurve { -using namespace arith; - -class TwEdwardsCurve; - -TwEdwardsCurve::TwEdwardsCurve(const Residue& _D, const Residue& _Gy, td::Ref _R) - : ring(_R) - , D(_D) - , D2(_D + _D) - , Gy(_Gy) - , P_(_R->get_modulus()) - , cofactor_short(0) - , G(_R) - , O(_R) - , table_lines(0) - , table() { - init(); -} - -TwEdwardsCurve::~TwEdwardsCurve() { -} - -void TwEdwardsCurve::init() { - assert(D != ring->zero() && D != ring->convert(-1)); - O.X = O.Z = ring->one(); - G = SegrePoint(*this, Gy, 0); - assert(!G.XY.is_zero()); -} - -void TwEdwardsCurve::set_order_cofactor(const Bignum& order, int cof) { - assert(order > 0); - assert(cof >= 0); - assert(cof == 0 || (order % cof) == 0); - Order = order; - cofactor = cofactor_short = cof; - if (cof > 0) { - L = order / cof; - assert(is_prime(L)); - assert(!power_gen(1).is_zero()); - assert(power_gen(L).is_zero()); - } -} - -TwEdwardsCurve::SegrePoint::SegrePoint(const TwEdwardsCurve& E, const Residue& y, bool x_sign) - : XY(y), X(E.get_base_ring()), Y(y), Z(E.get_base_ring()->one()) { - Residue x(y.ring_ref()); - if (E.recover_x(x, y, x_sign)) { - XY *= x; - X = x; - } else { - XY = Y = Z = E.get_base_ring()->zero(); - } -} - -bool TwEdwardsCurve::recover_x(Residue& x, const Residue& y, bool x_sign) const { - // recovers x from equation -x^2+y^2 = 1+d*x^2*y^2 - Residue z = inverse(ring->one() + D * sqr(y)); - if (z.is_zero()) { - return false; - } - z *= sqr(y) - ring->one(); - Residue t = sqrt(z); - if (sqr(t) == z) { - x = (t.extract().odd() == x_sign) ? t : -t; - //std::cout << "x=" << x << ", y=" << y << std::endl; - return true; - } else { - return false; - } -} - -void TwEdwardsCurve::add_points(SegrePoint& Res, const SegrePoint& P, const SegrePoint& Q) const { - Residue a((P.X + P.Y) * (Q.X + Q.Y)); - Residue b((P.X - P.Y) * (Q.X - Q.Y)); - Residue c(P.Z * Q.Z * ring->convert(2)); - Residue d(P.XY * Q.XY * D2); - Residue x_num(a - b); // 2(x1y2+x2y1) - Residue y_num(a + b); // 2(x1x2+y1y2) - Residue x_den(c + d); // 2(1+dx1x2y1y2) - Residue y_den(c - d); // 2(1-dx1x2y1y2) - Res.X = x_num * y_den; // x = x_num/x_den, y = y_num/y_den - Res.Y = y_num * x_den; - Res.XY = x_num * y_num; - Res.Z = x_den * y_den; -} - -TwEdwardsCurve::SegrePoint TwEdwardsCurve::add_points(const SegrePoint& P, const SegrePoint& Q) const { - SegrePoint Res(ring); - add_points(Res, P, Q); - return Res; -} - -void TwEdwardsCurve::double_point(SegrePoint& Res, const SegrePoint& P) const { - add_points(Res, P, P); -} - -TwEdwardsCurve::SegrePoint TwEdwardsCurve::double_point(const SegrePoint& P) const { - SegrePoint Res(ring); - double_point(Res, P); - return Res; -} - -// computes u([n]P) in form (xy,x,y,1)*Z -TwEdwardsCurve::SegrePoint TwEdwardsCurve::power_point(const SegrePoint& A, const Bignum& n, bool uniform) const { - assert(n >= 0); - if (n == 0) { - return O; - } - - int k = n.num_bits(); - SegrePoint P(A); - - if (uniform) { - SegrePoint Q(double_point(A)); - - for (int i = k - 2; i >= 0; --i) { - if (n[i]) { - add_points(P, P, Q); - double_point(Q, Q); - } else { - // we do more operations than necessary for uniformicity - add_points(Q, P, Q); - double_point(P, P); - } - } - } else { - for (int i = k - 2; i >= 0; --i) { - double_point(P, P); - if (n[i]) { - add_points(P, P, A); // may optimize further if A.z = 1 - } - } - } - return P; -} - -int TwEdwardsCurve::build_table() { - if (table.size()) { - return -1; - } - table_lines = (P_.num_bits() >> 2) + 2; - table.reserve(table_lines * 15 + 1); - table.emplace_back(get_base_point()); - for (int i = 0; i < table_lines; i++) { - for (int j = 0; j < 15; j++) { - table.emplace_back(add_points(table[15 * i + j], table[15 * i])); - } - } - return 1; -} - -int get_nibble(const Bignum& n, int idx) { - return n[idx * 4 + 3] * 8 + n[idx * 4 + 2] * 4 + n[idx * 4 + 1] * 2 + n[idx * 4]; -} - -TwEdwardsCurve::SegrePoint TwEdwardsCurve::power_gen(const Bignum& n, bool uniform) const { - if (uniform || n.num_bits() > table_lines * 4) { - return power_point(G, n, uniform); - } else if (n.is_zero()) { - return O; - } else { - int k = (n.num_bits() + 3) >> 2; - assert(k > 0 && k <= table_lines); - int x = get_nibble(n, k - 1); - assert(x > 0 && x < 16); - SegrePoint P(table[15 * (k - 1) + x - 1]); - for (int i = k - 2; i >= 0; i--) { - x = get_nibble(n, i); - assert(x >= 0 && x < 16); - if (x > 0) { - add_points(P, P, table[15 * i + x - 1]); - } - } - return P; - } -} - -bool TwEdwardsCurve::SegrePoint::export_point(unsigned char buffer[32], bool need_x) const { - if (!is_normalized()) { - if (Z.is_zero()) { - std::memset(buffer, 0xff, 32); - return false; - } - Residue f(inverse(Z)); - Bignum y((Y * f).extract()); - assert(!y[255]); - if (need_x) { - y[255] = (X * f).extract().odd(); - } - y.export_lsb(buffer, 32); - } else { - Bignum y(Y.extract()); - assert(!y[255]); - if (need_x) { - y[255] = X.extract().odd(); - } - y.export_lsb(buffer, 32); - } - return true; -} - -bool TwEdwardsCurve::SegrePoint::export_point_u(unsigned char buffer[32]) const { - if (Z == Y) { - std::memset(buffer, 0xff, 32); - return false; - } - Residue f(inverse(Z - Y)); - ((Z + Y) * f).extract().export_lsb(buffer, 32); - assert(!(buffer[31] & 0x80)); - return true; -} - -TwEdwardsCurve::SegrePoint TwEdwardsCurve::import_point(const unsigned char point[32], bool& ok) const { - Bignum y; - y.import_lsb(point, 32); - bool x_sign = y[255]; - y[255] = 0; - Residue yr(y, ring); - Residue xr(ring); - ok = recover_x(xr, yr, x_sign); - return ok ? SegrePoint(xr, yr) : SegrePoint(ring); -} - -const TwEdwardsCurve& Ed25519() { - static const TwEdwardsCurve Ed25519 = [] { - TwEdwardsCurve res(Fp25519()->frac(-121665, 121666), Fp25519()->frac(4, 5), Fp25519()); - res.set_order_cofactor(hex_string{"80000000000000000000000000000000a6f7cef517bce6b2c09318d2e7ae9f68"}, 8); - res.build_table(); - return res; - }(); - return Ed25519; -} -} // namespace ellcurve diff --git a/crypto/ellcurve/TwEdwards.h b/crypto/ellcurve/TwEdwards.h deleted file mode 100644 index c720ecca9e..0000000000 --- a/crypto/ellcurve/TwEdwards.h +++ /dev/null @@ -1,145 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once -#include -#include "common/refcnt.hpp" -#include "openssl/residue.h" -#include "ellcurve/Fp25519.h" - -namespace ellcurve { -using namespace arith; - -class TwEdwardsCurve { - public: - struct SegrePoint { - Residue XY, X, Y, Z; // if x=X/Z and y=Y/T, stores (xy,x,y,1)*Z*T - SegrePoint(td::Ref R) : XY(R), X(R), Y(R), Z(R) { - } - SegrePoint(const Residue& x, const Residue& y) : XY(x * y), X(x), Y(y), Z(y.ring_of().one()) { - } - SegrePoint(const TwEdwardsCurve& E, const Residue& y, bool x_sign); - SegrePoint(const SegrePoint& P) : XY(P.XY), X(P.X), Y(P.Y), Z(P.Z) { - } - SegrePoint& operator=(const SegrePoint& P) { - XY = P.XY; - X = P.X; - Y = P.Y; - Z = P.Z; - return *this; - } - bool is_zero() const { - return X.is_zero() && (Y == Z); - } - bool is_valid() const { - return (XY * Z == X * Y) && !(XY.is_zero() && X.is_zero() && Y.is_zero() && Z.is_zero()); - } - bool is_finite() const { - return !Z.is_zero(); - } - bool is_normalized() const { - return Z == Z.ring_of().one(); - } - SegrePoint& normalize() { - auto f = inverse(Z); - XY *= f; - X *= f; - Y *= f; - Z = Z.ring_of().one(); - return *this; - } - SegrePoint& zeroize() { - XY = X = Y = Z = Z.ring_of().zero(); - return *this; - } - bool export_point(unsigned char buffer[32], bool need_x = true) const; - bool export_point_y(unsigned char buffer[32]) const { - return export_point(buffer, false); - } - bool export_point_u(unsigned char buffer[32]) const; - Residue get_y() const { - return Y * inverse(Z); - } - Residue get_x() const { - return X * inverse(Z); - } - Residue get_u() const { - return (Z + Y) * inverse(Z - Y); - } - void negate() { - XY.negate(); - X.negate(); - } - }; - - private: - td::Ref ring; - Residue D; - Residue D2; - Residue Gy; - Bignum P_; - Bignum L; - Bignum Order; - Bignum cofactor; - int cofactor_short; - SegrePoint G; - SegrePoint O; - int table_lines; - std::vector table; - - void init(); - - public: - TwEdwardsCurve(const Residue& _D, const Residue& _Gy, td::Ref _R); - ~TwEdwardsCurve(); - const Residue& get_gen_y() const { - return Gy; - } - const Bignum& get_ell() const { - return L; - } - const Bignum& get_order() const { - return Order; - } - td::Ref get_base_ring() const { - return ring; - } - const Bignum& get_p() const { - return P_; - } - const SegrePoint& get_base_point() const { - return G; - } - - void set_order_cofactor(const Bignum& order, int cof); - bool recover_x(Residue& x, const Residue& y, bool x_sign) const; - - void add_points(SegrePoint& R, const SegrePoint& P, const SegrePoint& Q) const; - SegrePoint add_points(const SegrePoint& P, const SegrePoint& Q) const; - void double_point(SegrePoint& R, const SegrePoint& P) const; - SegrePoint double_point(const SegrePoint& P) const; - SegrePoint power_point(const SegrePoint& A, const Bignum& n, bool uniform = false) const; - SegrePoint power_gen(const Bignum& n, bool uniform = false) const; - int build_table(); - - SegrePoint import_point(const unsigned char point[32], bool& ok) const; -}; - -std::ostream& operator<<(std::ostream& os, const TwEdwardsCurve::SegrePoint& P); -const TwEdwardsCurve& Ed25519(); -} // namespace ellcurve diff --git a/crypto/test/Ed25519.cpp b/crypto/test/Ed25519.cpp index 131bfe92ee..940b50796b 100644 --- a/crypto/test/Ed25519.cpp +++ b/crypto/test/Ed25519.cpp @@ -17,7 +17,6 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "crypto/Ed25519.h" -#include "ellcurve/Ed25519.h" #include "td/utils/logging.h" #include "td/utils/misc.h" diff --git a/crypto/test/test-ed25519-crypto.cpp b/crypto/test/test-ed25519-crypto.cpp deleted file mode 100644 index 3e3dab896e..0000000000 --- a/crypto/test/test-ed25519-crypto.cpp +++ /dev/null @@ -1,314 +0,0 @@ -/* - This file is part of TON Blockchain source code. - - TON Blockchain is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - TON Blockchain is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with TON Blockchain. If not, see . - - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement - from all source files in the program, then also delete it here. - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include -#include -#include -#include - -#include "crypto/ellcurve/Ed25519.h" - -static void my_assert_impl(bool cond, const char* str, const char* file, int line) { - if (!cond) { - std::cerr << "Failed " << str << " in " << file << " at " << line << ".\n"; - } -} -#define my_assert(x) my_assert_impl(x, #x, __FILE__, __LINE__) - -void print_buffer(const unsigned char buffer[32]) { - for (int i = 0; i < 32; i++) { - char buff[4]; - sprintf(buff, "%02x", buffer[i]); - std::cout << buff; - } -} - -std::string buffer_to_hex(const unsigned char* buffer, std::size_t size = 32) { - const char* hex = "0123456789ABCDEF"; - std::string res(2 * size, '\0'); - for (std::size_t i = 0; i < size; i++) { - auto c = buffer[i]; - res[2 * i] = hex[c & 15]; - res[2 * i + 1] = hex[c >> 4]; - } - return res; -} - -// export of (17/12)G on twisted Edwards curve -unsigned char test_vector1[32] = {0xfc, 0xb7, 0x42, 0x1e, 0x26, 0xad, 0x1b, 0x17, 0xf6, 0xb1, 0x52, - 0x0c, 0xdb, 0x8a, 0x64, 0x7d, 0x28, 0xa7, 0x56, 0x69, 0xd4, 0xb6, - 0x0c, 0xec, 0x63, 0x72, 0x5e, 0xe6, 0x32, 0x4d, 0xf7, 0xe6}; - -unsigned char rfc7748_output[32] = { - 0x95, 0xcb, 0xde, 0x94, 0x76, 0xe8, 0x90, 0x7d, 0x7a, 0xad, 0xe4, 0x5c, 0xb4, 0xb8, 0x73, 0xf8, - 0x8b, 0x59, 0x5a, 0x68, 0x79, 0x9f, 0xa1, 0x52, 0xe6, 0xf8, 0xf7, 0x64, 0x7a, 0xac, 0x79, 0x57, -}; - -bool test_ed25519_impl(void) { - std::cout << "************** Testing Curve25519 / Ed25519 operations ************\n"; - auto& E = ellcurve::Curve25519(); - auto& Edw = ellcurve::Ed25519(); - arith::Bignum L = E.get_ell(); - my_assert(arith::is_prime(L)); - my_assert(L == Edw.get_ell()); - arith::ResidueRing Fl(L); - arith::Bignum s = Fl.frac(17, 12).extract(); - arith::Bignum t = Fl.frac(12, 17).extract(); - std::cout << "l = " << L << std::endl; - std::cout << "s = 17/12 mod l = " << s << std::endl; - std::cout << "t = 12/17 mod l = " << t << std::endl; - auto sG = E.power_gen_xz(s); - auto u_sG = sG.get_u(); - std::cout << "Curve25519 u(sG) = " << sG.get_u().extract() << std::endl; - std::cout << "Curve25519 y(sG) = " << sG.get_y().extract() << std::endl; - auto sG1 = Edw.power_gen(s); - std::cout << "Ed25519 u(sG) = " << sG1.get_u().extract() << std::endl; - std::cout << "Ed25519 y(sG) = " << sG1.get_y().extract() << std::endl; - std::cout << "Ed25519 x(sG) = " << sG1.get_x().extract() << std::endl; - my_assert(sG1.get_x().extract() != sG1.get_y().extract()); - my_assert(sG.get_u() == sG1.get_u()); - my_assert(sG.get_y() == sG1.get_y()); - - my_assert( - sG1.get_x().extract() == - arith::Bignum(arith::dec_string{"9227429025021714590777223519505276506601225973596506606120015751301699519597"})); - my_assert(sG1.get_y().extract() == - arith::Bignum( - arith::dec_string{"46572854587220149033453000581008590225032365765275643343836649812808016508924"})); - - auto sG2 = Edw.power_gen(s, true); - my_assert(sG1.get_u() == sG2.get_u()); - my_assert(sG1.get_y() == sG2.get_y()); - unsigned char buff[32]; - std::memset(buff, 0, 32); - my_assert(sG1.export_point(buff)); - std::cout << "sG export = " << buffer_to_hex(buff) << std::endl; - bool ok; - auto sG3 = Edw.import_point(buff, ok); - my_assert(ok); - my_assert(!std::memcmp(buff, test_vector1, 32)); - my_assert(sG3.get_u() == sG1.get_u()); - my_assert(sG2.get_x() == sG2.get_x()); - my_assert(sG2.get_y() == sG2.get_y()); - - auto stG = E.power_xz(u_sG, t); - std::cout << "Curve25519 u(stG) = " << stG.get_u().extract() << std::endl; - my_assert(stG.get_u().extract() == 9); - auto stG1 = Edw.power_point(sG1, t); - std::cout << "Ed25519 u(stG) = " << stG1.get_u().extract() << std::endl; - my_assert(stG1.get_u().extract() == 9); - stG1.normalize(); - my_assert(stG1.XY == Edw.get_base_point().XY); - my_assert(stG1.X == Edw.get_base_point().X); - my_assert(stG1.Y == Edw.get_base_point().Y); - my_assert(stG1.Z == Edw.get_base_point().Z); - auto stG2 = Edw.power_point(sG2, t, true); - my_assert(stG2.get_u().extract() == 9); - stG2.normalize(); - my_assert(stG2.XY == stG1.XY && stG2.X == stG1.X && stG2.Y == stG1.Y); - auto stG3 = Edw.power_point(sG3, t).normalize(); - auto stG4 = Edw.power_point(sG3, t, true).normalize(); - my_assert(stG3.XY == stG1.XY && stG3.X == stG1.X && stG3.Y == stG1.Y); - my_assert(stG4.XY == stG1.XY && stG4.X == stG1.X && stG4.Y == stG1.Y); - - // RFC7748 test vector - auto u = - arith::Bignum(arith::dec_string{"8883857351183929894090759386610649319417338800022198945255395922347792736741"}); - //u[255] = 0; - auto n = - arith::Bignum(arith::dec_string{"35156891815674817266734212754503633747128614016119564763269015315466259359304"}); - //n[255] = 0; n[254] = 1; - //n[0] = n[1] = n[2] = 0; - auto umodp = arith::Residue(u, E.get_base_ring()); - auto nP = E.power_xz(umodp, n); - std::cout << "u(P) = " << u.to_hex() << std::endl; - std::cout << "n = " << n.to_hex() << std::endl; - std::cout << "u(nP) = " << nP.get_u().extract().to_hex() << std::endl; - unsigned char buffer[32]; - std::memset(buffer, 0, 32); - nP.export_point_u(buffer); - std::cout << "u(nP) export = " << buffer_to_hex(buffer) << std::endl; - my_assert(!std::memcmp(buffer, rfc7748_output, 32)); - - std::cout << "********* ok\n\n"; - return true; -} - -unsigned char fixed_privkey[32] = "abacabadabacabaeabacabadabacaba"; -unsigned char fixed_pubkey[32] = {0x6f, 0x9e, 0x5b, 0xde, 0xce, 0x87, 0x21, 0xeb, 0x57, 0x37, 0xfb, - 0xb5, 0x92, 0x28, 0xba, 0x07, 0xf7, 0x88, 0x0f, 0x73, 0xce, 0x5b, - 0xfa, 0xa1, 0xb7, 0x15, 0x73, 0x03, 0xd4, 0x20, 0x1e, 0x74}; - -unsigned char rfc8032_secret_key1[32] = {0x9d, 0x61, 0xb1, 0x9d, 0xef, 0xfd, 0x5a, 0x60, 0xba, 0x84, 0x4a, - 0xf4, 0x92, 0xec, 0x2c, 0xc4, 0x44, 0x49, 0xc5, 0x69, 0x7b, 0x32, - 0x69, 0x19, 0x70, 0x3b, 0xac, 0x03, 0x1c, 0xae, 0x7f, 0x60}; - -unsigned char rfc8032_public_key1[32] = {0xd7, 0x5a, 0x98, 0x01, 0x82, 0xb1, 0x0a, 0xb7, 0xd5, 0x4b, 0xfe, - 0xd3, 0xc9, 0x64, 0x07, 0x3a, 0x0e, 0xe1, 0x72, 0xf3, 0xda, 0xa6, - 0x23, 0x25, 0xaf, 0x02, 0x1a, 0x68, 0xf7, 0x07, 0x51, 0x1a}; - -unsigned char rfc8032_signature1[64] = { - 0xe5, 0x56, 0x43, 0x00, 0xc3, 0x60, 0xac, 0x72, 0x90, 0x86, 0xe2, 0xcc, 0x80, 0x6e, 0x82, 0x8a, - 0x84, 0x87, 0x7f, 0x1e, 0xb8, 0xe5, 0xd9, 0x74, 0xd8, 0x73, 0xe0, 0x65, 0x22, 0x49, 0x01, 0x55, - 0x5f, 0xb8, 0x82, 0x15, 0x90, 0xa3, 0x3b, 0xac, 0xc6, 0x1e, 0x39, 0x70, 0x1c, 0xf9, 0xb4, 0x6b, - 0xd2, 0x5b, 0xf5, 0xf0, 0x59, 0x5b, 0xbe, 0x24, 0x65, 0x51, 0x41, 0x43, 0x8e, 0x7a, 0x10, 0x0b, -}; - -unsigned char rfc8032_secret_key2[32] = { - 0xc5, 0xaa, 0x8d, 0xf4, 0x3f, 0x9f, 0x83, 0x7b, 0xed, 0xb7, 0x44, 0x2f, 0x31, 0xdc, 0xb7, 0xb1, - 0x66, 0xd3, 0x85, 0x35, 0x07, 0x6f, 0x09, 0x4b, 0x85, 0xce, 0x3a, 0x2e, 0x0b, 0x44, 0x58, 0xf7, -}; - -unsigned char rfc8032_public_key2[32] = { - 0xfc, 0x51, 0xcd, 0x8e, 0x62, 0x18, 0xa1, 0xa3, 0x8d, 0xa4, 0x7e, 0xd0, 0x02, 0x30, 0xf0, 0x58, - 0x08, 0x16, 0xed, 0x13, 0xba, 0x33, 0x03, 0xac, 0x5d, 0xeb, 0x91, 0x15, 0x48, 0x90, 0x80, 0x25, -}; - -unsigned char rfc8032_message2[2] = {0xaf, 0x82}; - -unsigned char rfc8032_signature2[64] = { - 0x62, 0x91, 0xd6, 0x57, 0xde, 0xec, 0x24, 0x02, 0x48, 0x27, 0xe6, 0x9c, 0x3a, 0xbe, 0x01, 0xa3, - 0x0c, 0xe5, 0x48, 0xa2, 0x84, 0x74, 0x3a, 0x44, 0x5e, 0x36, 0x80, 0xd7, 0xdb, 0x5a, 0xc3, 0xac, - 0x18, 0xff, 0x9b, 0x53, 0x8d, 0x16, 0xf2, 0x90, 0xae, 0x67, 0xf7, 0x60, 0x98, 0x4d, 0xc6, 0x59, - 0x4a, 0x7c, 0x15, 0xe9, 0x71, 0x6e, 0xd2, 0x8d, 0xc0, 0x27, 0xbe, 0xce, 0xea, 0x1e, 0xc4, 0x0a, -}; - -bool test_ed25519_crypto() { - std::cout << "************** Testing Curve25519 / Ed25519 cryptographic primitives ************\n"; - crypto::Ed25519::PrivateKey PK1, PK2, PK3, PK4, PK5; - PK1.random_private_key(); - PK2.import_private_key(fixed_privkey); - unsigned char priv2_export[32]; - bool ok = PK1.export_private_key(priv2_export); - std::cout << "PK1 = " << ok << " " << buffer_to_hex(priv2_export) << std::endl; - my_assert(ok); - ok = PK2.export_private_key(priv2_export); - std::cout << "PK2 = " << ok << " " << buffer_to_hex(priv2_export) << std::endl; - my_assert(ok); - PK3.import_private_key(priv2_export); - std::cout << "PK3 = " << PK3.ok() << std::endl; - my_assert(PK3.ok()); - - unsigned char pub_export[32]; - ok = PK1.export_public_key(pub_export); - std::cout << "PubK1 = " << ok << " " << buffer_to_hex(pub_export) << std::endl; - my_assert(ok); - crypto::Ed25519::PublicKey PubK1(pub_export); - ok = PK2.export_public_key(pub_export); - std::cout << "PubK2 = " << ok << " " << buffer_to_hex(pub_export) << std::endl; - my_assert(ok); - my_assert(!std::memcmp(pub_export, fixed_pubkey, 32)); - crypto::Ed25519::PublicKey PubK2(pub_export); - ok = PK3.export_public_key(pub_export); - std::cout << "PubK3 = " << ok << " " << buffer_to_hex(pub_export) << std::endl; - my_assert(ok); - my_assert(!std::memcmp(pub_export, fixed_pubkey, 32)); - crypto::Ed25519::PublicKey PubK3(pub_export); - ok = PubK1.export_public_key(pub_export); - std::cout << "PubK1 = " << ok << " " << buffer_to_hex(pub_export) << std::endl; - my_assert(ok); - - unsigned char secret22[32]; - ok = PK2.compute_shared_secret(secret22, PubK3); - std::cout << "secret(PK2,PubK2)=" << ok << " " << buffer_to_hex(secret22) << std::endl; - my_assert(ok); - - unsigned char secret12[32], secret21[32]; - ok = PK1.compute_shared_secret(secret12, PubK3); - std::cout << "secret(PK1,PubK2)=" << ok << " " << buffer_to_hex(secret12) << std::endl; - my_assert(ok); - ok = PK2.compute_shared_secret(secret21, PubK1); - std::cout << "secret(PK2,PubK1)=" << ok << " " << buffer_to_hex(secret21) << std::endl; - my_assert(ok); - my_assert(!std::memcmp(secret12, secret21, 32)); - - // for (int i = 0; i < 1000; i++) { - // ok = PK1.compute_shared_secret(secret12, PubK3); - // my_assert(ok); - // ok = PK2.compute_shared_secret(secret21, PubK1); - // my_assert(ok); - // } - - unsigned char signature[64]; - ok = PK1.sign_message(signature, (const unsigned char*)"abc", 3); - std::cout << "PK1.signature=" << ok << " " << buffer_to_hex(signature, 64) << std::endl; - my_assert(ok); - - // signature[63] ^= 1; - ok = PubK1.check_message_signature(signature, (const unsigned char*)"abc", 3); - std::cout << "PubK1.check_signature=" << ok << std::endl; - my_assert(ok); - - PK4.import_private_key(rfc8032_secret_key1); - PK4.export_public_key(pub_export); - std::cout << "PK4.private_key = " << buffer_to_hex(rfc8032_secret_key1) << std::endl; - std::cout << "PK4.public_key = " << buffer_to_hex(pub_export) << std::endl; - my_assert(!std::memcmp(pub_export, rfc8032_public_key1, 32)); - ok = PK4.sign_message(signature, (const unsigned char*)"", 0); - std::cout << "PK4.signature('') = " << buffer_to_hex(signature, 64) << std::endl; - my_assert(ok); - my_assert(!std::memcmp(signature, rfc8032_signature1, 32)); - - PK5.import_private_key(rfc8032_secret_key2); - PK5.export_public_key(pub_export); - std::cout << "PK5.private_key = " << buffer_to_hex(rfc8032_secret_key2) << std::endl; - std::cout << "PK5.public_key = " << buffer_to_hex(pub_export) << std::endl; - my_assert(!std::memcmp(pub_export, rfc8032_public_key2, 32)); - ok = PK5.sign_message(signature, rfc8032_message2, 2); - std::cout << "PK5.signature('') = " << buffer_to_hex(signature, 64) << std::endl; - my_assert(ok); - my_assert(!std::memcmp(signature, rfc8032_signature2, 32)); - crypto::Ed25519::PublicKey PubK5(pub_export); - - // for (int i = 0; i < 10000; i++) { - // ok = PK5.sign_message (signature, rfc8032_message2, 2); - // my_assert (ok); - // } - // for (int i = 0; i < 10000; i++) { - // ok = PubK5.check_message_signature (signature, rfc8032_message2, 2); - // my_assert (ok); - // } - - unsigned char temp_pubkey[32]; - crypto::Ed25519::TempKeyGenerator TKG; // use one generator a lot of times - - TKG.create_temp_shared_secret(temp_pubkey, secret12, PubK1, (const unsigned char*)"abc", 3); - std::cout << "secret12=" << buffer_to_hex(secret12) << "; temp_pubkey=" << buffer_to_hex(temp_pubkey) << std::endl; - - PK1.compute_temp_shared_secret(secret21, temp_pubkey); - std::cout << "secret21=" << buffer_to_hex(secret21) << std::endl; - my_assert(!std::memcmp(secret12, secret21, 32)); - - std::cout << "********* ok\n\n"; - return true; -} - -int main(void) { - test_ed25519_impl(); - test_ed25519_crypto(); - return 0; -} diff --git a/doc/Tests.md b/doc/Tests.md index c883731a92..4ae21a75ae 100644 --- a/doc/Tests.md +++ b/doc/Tests.md @@ -3,13 +3,13 @@ TON contains multiple unit-tests, that facilitate detection of erroneous blockch ## Build tests Go inside the build directory and, if you use ninja, build the tests using the following command: -```ninja test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state``` +```ninja test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state``` For more details on how to build TON artifacts, please refer to any of Github actions. For cmake use: -```cmake --build . --target test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state``` +```cmake --build . --target test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state``` ## Run tests Go inside the build directory and with ninja execute: From 611cab74ada795ad81d55a76c8d7f7e537f92a95 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 23 May 2025 17:22:54 +0300 Subject: [PATCH 254/524] More session stats --- tl/generate/scheme/ton_api.tl | 6 ++++-- tl/generate/scheme/ton_api.tlo | Bin 113468 -> 113716 bytes validator-session/validator-session-types.h | 11 ++++++++--- validator/impl/collator.cpp | 2 ++ validator/interfaces/validator-manager.h | 8 ++++++-- validator/validator-group.cpp | 1 + 6 files changed, 21 insertions(+), 7 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index c5bc4afe3d..5d42796ec0 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -936,6 +936,8 @@ validatorStats.blockLimitsStatus limits_log:string = validatorStats.BlockLimitsStatus; validatorStats.extMsgsStats ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validatorStats.ExtMsgsStats; +validatorStats.blockStats + ext_msgs:validatorStats.extMsgsStats transactions:int = validatorStats.BlockStats; validatorStats.collatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 cc_seqno:int collated_at:double @@ -943,7 +945,7 @@ validatorStats.collatedBlock self:int256 is_validator:Bool total_time:double work_time:double cpu_work_time:double time_stats:string block_limits:validatorStats.blockLimitsStatus - ext_msgs_stats:validatorStats.extMsgsStats = validatorSession.stats.CollatedBlock; + block_stats:validatorStats.blockStats = validatorSession.stats.CollatedBlock; validatorStats.validatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 validated_at:double @@ -954,7 +956,7 @@ validatorStats.validatedBlock validatorStats.newValidatorGroup.node id:int256 pubkey:PublicKey adnl_id:int256 weight:long = validatorStats.newValidatorGroup.Node; validatorStats.newValidatorGroup session_id:int256 shard:tonNode.shardId cc_seqno:int - last_key_block_seqno:int started_at:double + last_key_block_seqno:int started_at:double prev:(vector tonNode.blockIdExt) self_idx:int self:int256 nodes:(vector validatorStats.newValidatorGroup.node) = validatorStats.NewValidatorGroup; validatorStats.endValidatorGroup.node id:int256 catchain_blocks:int = validatorStats.endValidatorGroup.Node; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 744323b6de215969871ddd09ac1cfb41e3619ed6..7b0b3750ac301801cd4b6efe14317c597d86146d 100644 GIT binary patch delta 362 zcmdn5sSn#Ho7nANYKayE!LF;RN*2fH0C_HGx?ZC+!)gpu(=bD-n)Ku5;)f~+7TWTr0+XB3?-70k%R z3F6;8Cgj6EIk8E5`h*BZjqMG=jC&Y$xRXHU#TSFj204Pmw(TIuT1Kdgik#VWKw=;R zw&#R08Zaeba~0SWumc$wK=L5Fn1Dt|KfABVky=p_pIe*`HXmw*|D1|!m=z#3JS9bm zdBuszC7JnolMOr+SU`qspAgO{!ORNspu*&Z4hlR#%^(iQaL(zBk&N2YH`Fr9Ojn3z hteM=?BrrKfCWcWV9st^9f%pIb delta 181 zcmdn;gKf_@Hr_|G^{p77z;Pq5vK`9mTWWbkNQ4*h9oL(GXT#{H)3^tO(w(X$*oQiBls0BsN zY&sw@kQLh(gfbd1ZBK|`B$iPj F9snI!K6wBD diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index e613b08a5c..18f6b702b0 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -223,19 +223,24 @@ struct NewValidatorGroupStats { CatchainSeqno cc_seqno = 0; BlockSeqno last_key_block_seqno = 0; double started_at = -1.0; + std::vector prev; td::uint32 self_idx = 0; PublicKeyHash self = PublicKeyHash::zero(); std::vector nodes; tl_object_ptr tl() const { + std::vector> prev_arr; + for (const auto &p : prev) { + prev_arr.push_back(create_tl_block_id(p)); + } std::vector> nodes_arr; for (const auto &node : nodes) { nodes_arr.push_back(create_tl_object( node.id.bits256_value(), node.pubkey.tl(), node.adnl_id.bits256_value(), node.weight)); } - return create_tl_object(session_id, create_tl_shard_id(shard), cc_seqno, - last_key_block_seqno, started_at, self_idx, - self.bits256_value(), std::move(nodes_arr)); + return create_tl_object( + session_id, create_tl_shard_id(shard), cc_seqno, last_key_block_seqno, started_at, std::move(prev_arr), + self_idx, self.bits256_value(), std::move(nodes_arr)); } }; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 069abc0f98..93fbaa1984 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -3063,6 +3063,7 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t update_max_lt(acc->last_trans_end_lt_); block::MsgMetadata new_msg_metadata{0, acc->workchain, acc->addr, trans->start_lt}; register_new_msgs(*trans, std::move(new_msg_metadata)); + ++stats_.transactions; return true; } @@ -3170,6 +3171,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, register_new_msgs(*trans, std::move(new_msg_metadata)); update_max_lt(acc->last_trans_end_lt_); value_flow_.burned += trans->blackhole_burned; + ++stats_.transactions; return trans_root; } diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index b09a8f2448..d1394c942c 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -69,6 +69,7 @@ struct CollationStats { td::uint32 estimated_bytes = 0, gas = 0, lt_delta = 0, estimated_collated_data_bytes = 0; int cat_bytes = 0, cat_gas = 0, cat_lt_delta = 0, cat_collated_data_bytes = 0; std::string limits_log; + td::uint32 transactions = 0; td::uint32 ext_msgs_total = 0; td::uint32 ext_msgs_filtered = 0; td::uint32 ext_msgs_accepted = 0; @@ -77,6 +78,10 @@ struct CollationStats { std::string time_stats; tl_object_ptr tl() const { + auto block_stats = create_tl_object( + create_tl_object(ext_msgs_total, ext_msgs_filtered, ext_msgs_accepted, + ext_msgs_rejected), + transactions); return create_tl_object( create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes, actual_collated_data_bytes, attempt, self.bits256_value(), is_validator, total_time, work_time, cpu_work_time, @@ -84,8 +89,7 @@ struct CollationStats { create_tl_object(estimated_bytes, gas, lt_delta, estimated_collated_data_bytes, cat_bytes, cat_gas, cat_lt_delta, cat_collated_data_bytes, limits_log), - create_tl_object(ext_msgs_total, ext_msgs_filtered, ext_msgs_accepted, - ext_msgs_rejected)); + std::move(block_stats)); } }; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 8ba4febcf4..7a2fa187a5 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -432,6 +432,7 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch .cc_seqno = validator_set_->get_catchain_seqno(), .last_key_block_seqno = last_key_block_seqno_, .started_at = td::Clocks::system(), + .prev = prev, .self = local_id_}; td::uint32 idx = 0; for (const auto &node : validator_set_->export_vector()) { From d4fcef4374b3e2c85751096ba1d9ac6a6a97abf9 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 26 May 2025 15:20:50 +0300 Subject: [PATCH 255/524] Version 11: fix account storage stat and null in RUNVM (#1690) * Bugfix in AccountStorageStat::remove_cell * Fix returning null as c4/c5 in RUNVM --- crypto/block/account-storage-stat.cpp | 2 +- crypto/vm/vm.cpp | 19 ++++++++++++++----- doc/GlobalVersions.md | 5 ++++- test/regression-tests.ans | 2 +- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index 82359f827f..a6159c2245 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -154,7 +154,7 @@ td::Status AccountStorageStat::remove_cell(const Ref& cell) { if (e.refcnt_diff < 0 && !e.refcnt) { TRY_STATUS(fetch_from_dict(e)); } - if (e.refcnt.value() + e.refcnt_diff != 0) { + if (e.refcnt_diff >= 0 || e.refcnt.value() + e.refcnt_diff != 0) { return td::Status::OK(); } bool spec; diff --git a/crypto/vm/vm.cpp b/crypto/vm/vm.cpp index 272e3930e6..356f0df223 100644 --- a/crypto/vm/vm.cpp +++ b/crypto/vm/vm.cpp @@ -785,11 +785,20 @@ void VmState::restore_parent_vm(int res) { cur_stack.push(std::move(child_state.stack->at(i))); } cur_stack.push_smallint(res); - if (parent->return_data) { - cur_stack.push_cell(child_state.get_committed_state().c4); - } - if (parent->return_actions) { - cur_stack.push_cell(child_state.get_committed_state().c5); + if (global_version >= 11 && !child_state.get_committed_state().committed) { + if (parent->return_data) { + cur_stack.push_null(); + } + if (parent->return_actions) { + cur_stack.push_null(); + } + } else { + if (parent->return_data) { + cur_stack.push_cell(child_state.get_committed_state().c4); + } + if (parent->return_actions) { + cur_stack.push_cell(child_state.get_committed_state().c5); + } } if (parent->return_gas) { cur_stack.push_smallint(child_state.gas.gas_consumed()); diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 7bd9b0cc98..d4113b3f1b 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -221,4 +221,7 @@ Reserve modes `+1`, `+4` and `+8` ("reserve all except", "add original balance" Along with the storage stat (cells and bits count), each account now stores the hash of the **storage dict**. **Storage dict** is the dictionary that stores refcnt for each cell in the account state. -This is required to help computing storage stats in the future, after collator-validator separation. \ No newline at end of file +This is required to help computing storage stats in the future, after collator-validator separation. + +### Other TVM changes +- Fix returning `null` as `c4` and `c5` (when VM state is not committed) in `RUNVM`. \ No newline at end of file diff --git a/test/regression-tests.ans b/test/regression-tests.ans index 22e7826eac..5831ebe740 100644 --- a/test/regression-tests.ans +++ b/test/regression-tests.ans @@ -25,7 +25,7 @@ Test_Fift_test_rist255_default f4d7558f200a656934f986145c19b1dedbe2ad029292a5a97 Test_Fift_test_secp256k1_default 3118450dace6af05fcdbd54a87d9446162ce11ac6ef6dfc57998cf113587d602 Test_Fift_test_sort2_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a Test_Fift_test_sort_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a -Test_Fift_test_tvm_runvm_default ff3d2a4031b543c18d6b555f0a1f1a891c7825e6d1e2e9beb4bf13b37441450b +Test_Fift_test_tvm_runvm_default b3e0d70c00f0e8bdf6d45c56c7c0817fa5be7a3fc540190786233a394de72e42 Test_Fift_testvm2_default 8a6e35fc0224398be9d2de39d31c86ea96965ef1eca2aa9e0af2303150ed4a7b Test_Fift_testvm3_default 3c1b77471c5fd914ed8b5f528b9faed618e278693f5030b953ff150e543864ae Test_Fift_testvm4_default 8a6e35fc0224398be9d2de39d31c86ea96965ef1eca2aa9e0af2303150ed4a7b From 9b2933cb8480c99f72da7d47a24fb329706d4229 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 26 May 2025 15:22:29 +0300 Subject: [PATCH 256/524] Store shard archives separately even if monitor_min_split = 0 (#1689) --- validator/db/archive-slice.cpp | 35 +++++++++++++++++----------------- validator/db/archive-slice.hpp | 1 + validator/import-db-slice.cpp | 8 ++++---- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index 41ed509978..0691f67406 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -554,14 +554,14 @@ void ArchiveSlice::get_slice(td::uint64 archive_id, td::uint64 offset, td::uint3 before_query(); auto value = static_cast(archive_id >> 32); PackageInfo *p; - if (shard_split_depth_ == 0) { - TRY_RESULT_PROMISE_ASSIGN(promise, p, choose_package(value, ShardIdFull{masterchainId}, false)); - } else { + if (shard_separated_) { if (value >= packages_.size()) { promise.set_error(td::Status::Error(ErrorCode::notready, "no such package")); return; } p = &packages_[value]; + } else { + TRY_RESULT_PROMISE_ASSIGN(promise, p, choose_package(value, ShardIdFull{masterchainId}, false)); } promise = begin_async_query(std::move(promise)); td::actor::create_actor("readfile", p->path, offset, limit, 0, std::move(promise)).release(); @@ -574,10 +574,10 @@ void ArchiveSlice::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shar promise.set_result(archive_id_); } else { TRY_RESULT_PROMISE(promise, p, choose_package(masterchain_seqno, shard_prefix, false)); - if (shard_split_depth_ == 0) { - promise.set_result(p->seqno * (1ull << 32) + archive_id_); - } else { + if (shard_separated_) { promise.set_result(p->idx * (1ull << 32) + archive_id_); + } else { + promise.set_result(p->seqno * (1ull << 32) + archive_id_); } } } @@ -609,8 +609,10 @@ void ArchiveSlice::before_query() { if (R2.move_as_ok() == td::KeyValue::GetStatus::Ok) { shard_split_depth_ = td::to_integer(value); CHECK(shard_split_depth_ <= 60); + shard_separated_ = true; } else { shard_split_depth_ = 0; + shard_separated_ = false; } for (td::uint32 i = 0; i < tot; i++) { R2 = kv_->get(PSTRING() << "status." << i, value); @@ -625,16 +627,16 @@ void ArchiveSlice::before_query() { } td::uint32 seqno; ShardIdFull shard_prefix; - if (shard_split_depth_ == 0) { - seqno = archive_id_ + slice_size_ * i; - shard_prefix = ShardIdFull{masterchainId}; - } else { + if (shard_separated_) { R2 = kv_->get(PSTRING() << "info." << i, value); R2.ensure(); CHECK(R2.move_as_ok() == td::KeyValue::GetStatus::Ok); unsigned long long shard; CHECK(sscanf(value.c_str(), "%u.%d:%016llx", &seqno, &shard_prefix.workchain, &shard) == 3); shard_prefix.shard = shard; + } else { + seqno = archive_id_ + slice_size_ * i; + shard_prefix = ShardIdFull{masterchainId}; } add_package(seqno, shard_prefix, len, ver); } @@ -651,10 +653,9 @@ void ArchiveSlice::before_query() { kv_->set("slice_size", td::to_string(slice_size_)).ensure(); kv_->set("status.0", "0").ensure(); kv_->set("version.0", td::to_string(default_package_version())).ensure(); - if (shard_split_depth_ > 0) { - kv_->set("info.0", package_info_to_str(archive_id_, ShardIdFull{masterchainId})).ensure(); - kv_->set("shard_split_depth", td::to_string(shard_split_depth_)).ensure(); - } + shard_separated_ = true; + kv_->set("info.0", package_info_to_str(archive_id_, ShardIdFull{masterchainId})).ensure(); + kv_->set("shard_split_depth", td::to_string(shard_split_depth_)).ensure(); kv_->commit_transaction().ensure(); add_package(archive_id_, ShardIdFull{masterchainId}, 0, default_package_version()); } else { @@ -779,7 +780,7 @@ td::Result ArchiveSlice::choose_package(BlockSeqno } masterchain_seqno -= (masterchain_seqno - archive_id_) % slice_size_; CHECK((masterchain_seqno - archive_id_) % slice_size_ == 0); - if (shard_split_depth_ == 0) { + if (!shard_separated_) { shard_prefix = ShardIdFull{masterchainId}; } else if (!shard_prefix.is_masterchain()) { shard_prefix.shard |= 1; // In case length is < split depth @@ -795,7 +796,7 @@ td::Result ArchiveSlice::choose_package(BlockSeqno kv_->set("slices", td::to_string(v + 1)).ensure(); kv_->set(PSTRING() << "status." << v, "0").ensure(); kv_->set(PSTRING() << "version." << v, td::to_string(default_package_version())).ensure(); - if (shard_split_depth_ > 0) { + if (shard_separated_) { kv_->set(PSTRING() << "info." << v, package_info_to_str(masterchain_seqno, shard_prefix)).ensure(); } commit_transaction(); @@ -1079,7 +1080,7 @@ void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle, td:: package.idx = i; kv_->set(PSTRING() << "status." << i, td::to_string(package.package->size())).ensure(); kv_->set(PSTRING() << "version." << i, td::to_string(package.version)).ensure(); - if (shard_split_depth_ > 0) { + if (shard_separated_) { kv_->set(PSTRING() << "info." << i, package_info_to_str(package.seqno, package.shard_prefix)).ensure(); } id_to_package_[{package.seqno, package.shard_prefix}] = i; diff --git a/validator/db/archive-slice.hpp b/validator/db/archive-slice.hpp index a027ec0ff5..0cb2c9fbfa 100644 --- a/validator/db/archive-slice.hpp +++ b/validator/db/archive-slice.hpp @@ -159,6 +159,7 @@ class ArchiveSlice : public td::actor::Actor { bool sliced_mode_{false}; td::uint32 huge_transaction_size_ = 0; td::uint32 slice_size_{100}; + bool shard_separated_{false}; td::uint32 shard_split_depth_ = 0; enum Status { diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index 3e1392bef9..c0d502cb2a 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -300,10 +300,10 @@ void ArchiveImporter::checked_all_masterchain_blocks() { void ArchiveImporter::download_shard_archives(td::Ref start_state) { start_state_ = start_state; td::uint32 monitor_min_split = start_state->monitor_min_split_depth(basechainId); - LOG(DEBUG) << "Monitor min split = " << monitor_min_split; - // If monitor_min_split == 0, we use the old archive format (packages are not separated by shard) + LOG(DEBUG) << "Monitor min split = " << monitor_min_split + << (have_shard_blocks_ ? ", shard blocks in the main package" : ", no shard blocks in the main package"); // If masterchain package has shard blocks then it's old archive format, don't need to download shards - if (monitor_min_split > 0 && !have_shard_blocks_ && !use_imported_files_) { + if (!have_shard_blocks_ && !use_imported_files_) { for (td::uint64 i = 0; i < (1ULL << monitor_min_split); ++i) { ShardIdFull shard_prefix{basechainId, (i * 2 + 1) << (64 - monitor_min_split - 1)}; if (opts_->need_monitor(shard_prefix, start_state)) { @@ -313,7 +313,7 @@ void ArchiveImporter::download_shard_archives(td::Ref start_st } } } else { - LOG(DEBUG) << "Skip downloading shard archives"; + LOG(INFO) << "Skip downloading shard archives"; } if (pending_shard_archives_ == 0) { check_next_shard_client_seqno(shard_client_seqno_ + 1); From 6a1dbc72a3c0205c413b08f9c4394f38d32c37e3 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 26 May 2025 15:46:57 +0300 Subject: [PATCH 257/524] Don't destroy ValidatorGroup immediately, fix flags in session stats --- validator-session/validator-session-types.h | 4 +- validator/validator-group.cpp | 44 +++++++++++++++------ validator/validator-group.hpp | 2 + 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index 18f6b702b0..5b1f5c426c 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -143,7 +143,9 @@ struct ValidatorSessionStats { signers_str[i] = '0' + signers[i]; } int flags = - (block_status != status_none ? ton_api::validatorStats_stats_producer::Flags::BLOCK_ID_MASK : 0) | + (block_status != status_none || !candidate_id.is_zero() + ? ton_api::validatorStats_stats_producer::Flags::BLOCK_ID_MASK + : 0) | (collated_at >= 0.0 ? ton_api::validatorStats_stats_producer::Flags::COLLATED_AT_MASK : 0) | (!collator_node_id.is_zero() ? ton_api::validatorStats_stats_producer::Flags::COLLATOR_NODE_ID_MASK : 0) | (validated_at >= 0.0 ? ton_api::validatorStats_stats_producer::Flags::VALIDATED_AT_MASK : 0) | diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 7a2fa187a5..f9a8f27325 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -38,6 +38,10 @@ static bool need_send_candidate_broadcast(const validatorsession::BlockSourceInf void ValidatorGroup::generate_block_candidate(validatorsession::BlockSourceInfo source_info, td::Promise promise) { + if (destroying_) { + promise.set_error(td::Status::Error("validator session finished")); + return; + } td::uint32 round_id = source_info.priority.round; if (round_id > last_known_round_id_) { last_known_round_id_ = round_id; @@ -99,6 +103,10 @@ void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block, td::Promise> promise) { + if (destroying_) { + promise.set_error(td::Status::Error("validator session finished")); + return; + } td::uint32 round_id = source_info.priority.round; if (round_id > last_known_round_id_) { last_known_round_id_ = round_id; @@ -451,6 +459,28 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch } void ValidatorGroup::destroy() { + if (destroying_) { + return; + } + destroying_ = true; + if (!session_.empty()) { + td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_end_stats, + [manager = manager_](td::Result R) { + if (R.is_error()) { + LOG(DEBUG) << "Failed to get validator session end stats: " << R.move_as_error(); + return; + } + auto stats = R.move_as_ok(); + td::actor::send_closure(manager, &ValidatorManager::log_end_validator_group_stats, + std::move(stats)); + }); + } + cancellation_token_source_.cancel(); + delay_action([SelfId = actor_id(this)]() { td::actor::send_closure(SelfId, &ValidatorGroup::destroy_cont); }, + td::Timestamp::in(10.0)); +} + +void ValidatorGroup::destroy_cont() { if (!session_.empty()) { td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_current_stats, [manager = manager_, cc_seqno = validator_set_->get_catchain_seqno(), @@ -469,21 +499,9 @@ void ValidatorGroup::destroy() { td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats, std::move(stats)); }); - td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_end_stats, - [manager = manager_](td::Result R) { - if (R.is_error()) { - LOG(DEBUG) << "Failed to get validator session end stats: " << R.move_as_error(); - return; - } - auto stats = R.move_as_ok(); - td::actor::send_closure(manager, &ValidatorManager::log_end_validator_group_stats, - std::move(stats)); - }); auto ses = session_.release(); - delay_action([ses]() mutable { td::actor::send_closure(ses, &validatorsession::ValidatorSession::destroy); }, - td::Timestamp::in(10.0)); + td::actor::send_closure(ses, &validatorsession::ValidatorSession::destroy); } - cancellation_token_source_.cancel(); stop(); } diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 971fcc17b4..c7f9404128 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -102,6 +102,7 @@ class ValidatorGroup : public td::actor::Actor { private: std::unique_ptr make_validator_session_callback(); + void destroy_cont(); struct PostponedAccept { RootHash root_hash; @@ -143,6 +144,7 @@ class ValidatorGroup : public td::actor::Actor { td::Ref opts_; td::uint32 last_known_round_id_ = 0; bool monitoring_shard_ = true; + bool destroying_ = false; struct CachedCollatedBlock { td::optional result; From 3fbab2c601380eba5ba68048f45d24a359bd2936 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 26 May 2025 17:27:06 +0300 Subject: [PATCH 258/524] Always set ihr_disabled in internal messages (#1691) --- crypto/block/transaction.cpp | 4 ++++ crypto/block/transaction.h | 1 + crypto/vm/tonops.cpp | 2 +- doc/GlobalVersions.md | 5 +++-- validator/impl/validate-query.cpp | 1 + 5 files changed, 10 insertions(+), 3 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 1b6c3582fc..8a72f8e48e 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -2569,6 +2569,9 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, fwd_fee = block::tlb::t_Grams.as_integer(info.fwd_fee); ihr_fee = block::tlb::t_Grams.as_integer(info.ihr_fee); } + if (cfg.disable_ihr_flag) { + info.ihr_disabled = true; + } } // set created_at and created_lt to correct values info.created_at = now; @@ -4080,6 +4083,7 @@ td::Status FetchConfigParams::fetch_config_params( action_phase_cfg->mc_blackhole_addr = config.get_burning_config().blackhole_addr; action_phase_cfg->extra_currency_v2 = config.get_global_version() >= 10; action_phase_cfg->disable_anycast = config.get_global_version() >= 10; + action_phase_cfg->disable_ihr_flag = config.get_global_version() >= 11; } { serialize_cfg->extra_currency_v2 = config.get_global_version() >= 10; diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index fe70060067..0b3a4358d7 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -174,6 +174,7 @@ struct ActionPhaseConfig { bool disable_custom_fess{false}; bool reserve_extra_enabled{false}; bool extra_currency_v2{false}; + bool disable_ihr_flag{false}; td::optional mc_blackhole_addr; bool disable_anycast{false}; const MsgPrices& fetch_msg_prices(bool is_masterchain) const { diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index 135ed70268..b06fb1b77b 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -1834,7 +1834,7 @@ int exec_send_message(VmState* st) { if (!tlb::csr_unpack(msg.info, info)) { throw VmError{Excno::unknown, "invalid message"}; } - ihr_disabled = info.ihr_disabled; + ihr_disabled = info.ihr_disabled || st->get_global_version() >= 11; dest = std::move(info.dest); Ref extra; if (!block::tlb::t_CurrencyCollection.unpack_special(info.value.write(), value, extra)) { diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index d4113b3f1b..ad5eba948f 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -223,5 +223,6 @@ Along with the storage stat (cells and bits count), each account now stores the **Storage dict** is the dictionary that stores refcnt for each cell in the account state. This is required to help computing storage stats in the future, after collator-validator separation. -### Other TVM changes -- Fix returning `null` as `c4` and `c5` (when VM state is not committed) in `RUNVM`. \ No newline at end of file +### Other changes +- Fix returning `null` as `c4` and `c5` (when VM state is not committed) in `RUNVM`. +- In new internal messages `ihr_disabled` is automatically set to `1`, `ihr_fee` is always zero. \ No newline at end of file diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 88ae031e2c..8c974b431b 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1007,6 +1007,7 @@ bool ValidateQuery::fetch_config_params() { action_phase_cfg_.mc_blackhole_addr = config_->get_burning_config().blackhole_addr; action_phase_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; action_phase_cfg_.disable_anycast = config_->get_global_version() >= 10; + action_phase_cfg_.disable_ihr_flag = config_->get_global_version() >= 11; } { serialize_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; From a213e463ee4690a6d629562b6392b1ea806502b4 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 27 May 2025 14:23:47 +0300 Subject: [PATCH 259/524] "--shard-block-retainer fullnode" option --- validator-engine/validator-engine.cpp | 33 ++++++++++++++++++--------- validator-engine/validator-engine.hpp | 4 ++++ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 8c15b66c7c..a7d28ea7ff 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -2194,6 +2194,9 @@ void ValidatorEngine::start_validator() { } } + if (shard_block_retainer_adnl_id_fullnode_) { + shard_block_retainer_adnl_id_ = ton::adnl::AdnlNodeIdShort{config_.full_node}; + } if (!shard_block_retainer_adnl_id_.is_zero()) { if (config_.adnl_ids.contains(shard_block_retainer_adnl_id_.pubkey_hash())) { td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_shard_block_retainer, @@ -5575,17 +5578,25 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_sync_shards_upto, v); }); return td::Status::OK(); }); - p.add_checked_option( - '\0', "shard-block-retainer", "adnl id for shard block retainer (hex)", [&](td::Slice arg) -> td::Status { - td::Bits256 v; - if (v.from_hex(arg) != 256) { - return td::Status::Error("invalid adnl-id"); - } - acts.push_back([&x, v]() { - td::actor::send_closure(x, &ValidatorEngine::set_shard_block_retainer_adnl_id, ton::adnl::AdnlNodeIdShort{v}); - }); - return td::Status::OK(); - }); + p.add_checked_option('\0', "shard-block-retainer", + "adnl id for shard block retainer (hex), or \"fullnode\" for full node id", + [&](td::Slice arg) -> td::Status { + if (arg == "fullnode") { + acts.push_back([&x]() { + td::actor::send_closure(x, &ValidatorEngine::set_shard_block_retainer_adnl_id_fullnode); + }); + } else { + td::Bits256 v; + if (v.from_hex(arg) != 256) { + return td::Status::Error("invalid adnl-id"); + } + acts.push_back([&x, v]() { + td::actor::send_closure(x, &ValidatorEngine::set_shard_block_retainer_adnl_id, + ton::adnl::AdnlNodeIdShort{v}); + }); + } + return td::Status::OK(); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index b8ec238e98..ea6ef0da47 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -259,6 +259,7 @@ class ValidatorEngine : public td::actor::Actor { bool skip_key_sync_ = false; td::optional sync_shards_upto_; ton::adnl::AdnlNodeIdShort shard_block_retainer_adnl_id_ = ton::adnl::AdnlNodeIdShort::zero(); + bool shard_block_retainer_adnl_id_fullnode_ = false; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -386,6 +387,9 @@ class ValidatorEngine : public td::actor::Actor { void set_shard_block_retainer_adnl_id(ton::adnl::AdnlNodeIdShort id) { shard_block_retainer_adnl_id_ = id; } + void set_shard_block_retainer_adnl_id_fullnode() { + shard_block_retainer_adnl_id_fullnode_ = true; + } void start_up() override; ValidatorEngine() { From b2028ae139cbdd426a47219aea004d62dd37b123 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 31 Mar 2025 20:55:01 +0300 Subject: [PATCH 260/524] [Tolk] Lateinit variables Allow a variable be unassigned at declaration if it is definitely assigned before the first usage --- tolk-tester/tests/assignment-tests.tolk | 38 +++++++++++++++++++ .../tests/invalid-declaration/err-1190.tolk | 9 +++++ .../tests/invalid-declaration/err-1617.tolk | 8 ++++ .../tests/invalid-semantics/err-4301.tolk | 15 ++++++++ .../tests/invalid-semantics/err-4309.tolk | 32 ++++++++++++++++ tolk/ast-from-tokens.cpp | 34 +++++++++++------ tolk/ast-replicator.h | 2 +- tolk/ast.h | 5 ++- tolk/pipe-ast-to-legacy.cpp | 1 + tolk/pipe-calc-rvalue-lvalue.cpp | 2 +- tolk/pipe-infer-types-and-calls.cpp | 22 +++++++++++ tolk/pipe-resolve-identifiers.cpp | 8 ++-- tolk/symtable.h | 2 + 13 files changed, 158 insertions(+), 20 deletions(-) create mode 100644 tolk-tester/tests/invalid-declaration/err-1190.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1617.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4301.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4309.tolk diff --git a/tolk-tester/tests/assignment-tests.tolk b/tolk-tester/tests/assignment-tests.tolk index a52b3ca50e..e53020d050 100644 --- a/tolk-tester/tests/assignment-tests.tolk +++ b/tolk-tester/tests/assignment-tests.tolk @@ -212,6 +212,43 @@ fun test117() { return c; } +struct Storage { + owner: slice; +} + +@method_id(118) +fun test118(x: int) { + var i1: int; + if (10 > 3) { + i1 = 10; + } else { { (_, i1) = (5, 20); } } + + var i2: int10; + match (x) { + 1 => i2 = 1, + 2 => i2 = 2, + 3 => i2 = 3, + else => i2 = 4, + } + + var st: Storage?; + if (x > 100) { + throw 123; + } else { + if (x < -100) { throw 456; } + else { st = { owner: "" }} + } + + var i3: (int, int) | builder; + match (i1) { + int => i3 = (1, 2), + } + + var unused: int8; + + return (i1, i2, st.owner.remainingBitsCount(), i3.0); +} + fun main(value: int, ) { @@ -241,6 +278,7 @@ fun main(value: int, ) { @testcase | 115 | | [ 101 111 ] 9 9 @testcase | 116 | | 1 2 3 4 [ 1 2 3 4 ] @testcase | 117 | | [ 20 ] +@testcase | 118 | 3 | 10 3 0 1 @fif_codegen diff --git a/tolk-tester/tests/invalid-declaration/err-1190.tolk b/tolk-tester/tests/invalid-declaration/err-1190.tolk new file mode 100644 index 0000000000..510e1d1cc0 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1190.tolk @@ -0,0 +1,9 @@ +fun main() { + var i; +} + +/** +@compilation_should_fail +@stderr provide a type for a variable, because its default value is omitted: +@stderr var i: ; + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1617.tolk b/tolk-tester/tests/invalid-declaration/err-1617.tolk new file mode 100644 index 0000000000..dbfe897194 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1617.tolk @@ -0,0 +1,8 @@ +fun main() { + val (s, i); +} + +/** +@compilation_should_fail +@stderr variables declaration must be followed by assignment + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4301.tolk b/tolk-tester/tests/invalid-semantics/err-4301.tolk new file mode 100644 index 0000000000..800495da3b --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4301.tolk @@ -0,0 +1,15 @@ +fun main() { + var i: int; + match (random.uint256()) { + 0 => { i = 0; debug.print(i) }, + 1 => { if (true) { i = 1 } }, + 2 => throw i = 2, + } + if (i == 0) {} +} + +/** +@compilation_should_fail +@stderr using variable `i` before it's definitely assigned +@stderr if (i == 0) + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4309.tolk b/tolk-tester/tests/invalid-semantics/err-4309.tolk new file mode 100644 index 0000000000..3fa24df913 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4309.tolk @@ -0,0 +1,32 @@ +fun increment(mutate x: int) { + x += 1; +} + +fun main() { + var i: int; + if (true) { + i = 10; + } + increment(mutate i); // ok + + var j: int8 | int16; + if (random.uint256()) { + (j, _) = (10 as int8, i); + } else { + return; + } + match (j) { // ok + int8 => increment(mutate i), + int16 => increment(mutate j), + } + + var k: int; + { + increment(mutate k); + } +} + +/** +@compilation_should_fail +@stderr using variable `k` before it's definitely assigned + */ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index ff13ed386f..28f0ba182e 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -400,11 +400,11 @@ static AnyV parse_type_alias_declaration(Lexer& lex, const std::vector(loc, v_ident, genericsT_list, underlying_type); } -static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { +static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable, bool allow_lateinit) { SrcLocation loc = lex.cur_location(); if (lex.tok() == tok_oppar) { lex.next(); - AnyExprV first = parse_var_declaration_lhs(lex, is_immutable); + AnyExprV first = parse_var_declaration_lhs(lex, is_immutable, false); if (lex.tok() == tok_clpar) { lex.next(); return first; @@ -412,17 +412,17 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { std::vector args(1, first); while (lex.tok() == tok_comma) { lex.next(); - args.push_back(parse_var_declaration_lhs(lex, is_immutable)); + args.push_back(parse_var_declaration_lhs(lex, is_immutable, false)); } lex.expect(tok_clpar, "`)`"); return createV(loc, std::move(args)); } if (lex.tok() == tok_opbracket) { lex.next(); - std::vector args(1, parse_var_declaration_lhs(lex, is_immutable)); + std::vector args(1, parse_var_declaration_lhs(lex, is_immutable, false)); while (lex.tok() == tok_comma) { lex.next(); - args.push_back(parse_var_declaration_lhs(lex, is_immutable)); + args.push_back(parse_var_declaration_lhs(lex, is_immutable, false)); } lex.expect(tok_clbracket, "`]`"); return createV(loc, std::move(args)); @@ -431,6 +431,7 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { auto v_ident = createV(loc, lex.cur_str()); AnyTypeV declared_type = nullptr; bool marked_as_redef = false; + bool is_lateinit = false; lex.next(); if (lex.tok() == tok_colon) { lex.next(); @@ -439,7 +440,13 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { lex.next(); marked_as_redef = true; } - return createV(loc, v_ident, declared_type, is_immutable, marked_as_redef); + if (lex.tok() == tok_semicolon && allow_lateinit) { + if (declared_type == nullptr) { + lex.error("provide a type for a variable, because its default value is omitted:\n> var " + static_cast(v_ident->name) + ": ;"); + } + is_lateinit = true; + } + return createV(loc, v_ident, declared_type, is_immutable, is_lateinit, marked_as_redef); } if (lex.tok() == tok_underscore) { AnyTypeV declared_type = nullptr; @@ -448,18 +455,21 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { lex.next(); declared_type = parse_type_from_tokens(lex); } - return createV(loc, createV(loc, ""), declared_type, true, false); + return createV(loc, createV(loc, ""), declared_type, true, false, false); } lex.unexpected("variable name"); } -static AnyExprV parse_local_vars_declaration_assignment(Lexer& lex) { +static AnyExprV parse_local_vars_declaration(Lexer& lex, bool allow_lateinit) { SrcLocation loc = lex.cur_location(); bool is_immutable = lex.tok() == tok_val; lex.next(); - AnyExprV lhs = createV(loc, parse_var_declaration_lhs(lex, is_immutable)); + AnyExprV lhs = parse_var_declaration_lhs(lex, is_immutable, allow_lateinit); if (lex.tok() != tok_assign) { + if (auto lhs_var = lhs->try_as(); lhs_var && lhs_var->is_lateinit) { + return lhs; // just ast_local_var_lhs inside AST tree + } lex.error("variables declaration must be followed by assignment: `var xxx = ...`"); } lex.next(); @@ -468,7 +478,7 @@ static AnyExprV parse_local_vars_declaration_assignment(Lexer& lex) { if (lex.tok() == tok_comma) { lex.error("multiple declarations are not allowed, split variables on separate lines"); } - return createV(loc, lhs, rhs); + return createV(loc, createV(loc, lhs), rhs); } // "parameters" are at function declaration: `fun f(param1: int, mutate param2: slice)` @@ -666,7 +676,7 @@ static V parse_match_expression(Lexer& lex) { lex.expect(tok_oppar, "`(`"); AnyExprV subject = lex.tok() == tok_var || lex.tok() == tok_val // `match (var x = rhs)` - ? parse_local_vars_declaration_assignment(lex) + ? parse_local_vars_declaration(lex, false) : parse_expr(lex); lex.expect(tok_clpar, "`)`"); @@ -1179,7 +1189,7 @@ AnyV parse_statement(Lexer& lex) { switch (lex.tok()) { case tok_var: // `var x = 0` is technically an expression, but can not appear in "any place", case tok_val: // only as a separate declaration - return parse_local_vars_declaration_assignment(lex); + return parse_local_vars_declaration(lex, true); case tok_opbrace: return parse_block_statement(lex); case tok_return: diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index fc1dafa22b..d72513906c 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -97,7 +97,7 @@ class ASTReplicator final { return createV(v->loc, clone(v->get_identifier()), v->has_instantiationTs() ? clone(v->get_instantiationTs()) : nullptr); } static V clone(V v) { - return createV(v->loc, clone(v->get_identifier()), clone(v->type_node), v->is_immutable, v->marked_as_redef); + return createV(v->loc, clone(v->get_identifier()), clone(v->type_node), v->is_immutable, v->is_lateinit, v->marked_as_redef); } static V clone(V v) { return createV(v->loc, clone(v->get_expr())); diff --git a/tolk/ast.h b/tolk/ast.h index ee550064c4..0770d9b844 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -584,6 +584,7 @@ struct Vertex final : ASTExprLeaf { LocalVarPtr var_ref = nullptr; // filled on resolve identifiers; for `redef` points to declared above; for underscore, name is empty AnyTypeV type_node; // exists for `var x: int = rhs`, otherwise nullptr bool is_immutable; // declared via 'val', not 'var' + bool is_lateinit; // var st: Storage lateinit (no assignment) bool marked_as_redef; // var (existing_var redef, new_var: int) = ... V get_identifier() const { return identifier; } @@ -592,9 +593,9 @@ struct Vertex final : ASTExprLeaf { Vertex* mutate() const { return const_cast(this); } void assign_var_ref(LocalVarPtr var_ref); - Vertex(SrcLocation loc, V identifier, AnyTypeV type_node, bool is_immutable, bool marked_as_redef) + Vertex(SrcLocation loc, V identifier, AnyTypeV type_node, bool is_immutable, bool is_lateinit, bool marked_as_redef) : ASTExprLeaf(ast_local_var_lhs, loc) - , identifier(identifier), type_node(type_node), is_immutable(is_immutable), marked_as_redef(marked_as_redef) {} + , identifier(identifier), type_node(type_node), is_immutable(is_immutable), is_lateinit(is_lateinit), marked_as_redef(marked_as_redef) {} }; template<> diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index a45a4ccb6f..370460d3cf 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1476,6 +1476,7 @@ static std::vector process_local_var(V v, CodeBlob static std::vector process_local_vars_declaration(V, CodeBlob&) { // it can not appear as a standalone expression // `var ... = rhs` is handled by ast_assign + // `var rhs: int lateinit` is ast_local_var_lhs tolk_assert(false); } diff --git a/tolk/pipe-calc-rvalue-lvalue.cpp b/tolk/pipe-calc-rvalue-lvalue.cpp index b6a239116c..d451a7c601 100644 --- a/tolk/pipe-calc-rvalue-lvalue.cpp +++ b/tolk/pipe-calc-rvalue-lvalue.cpp @@ -246,7 +246,7 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { } void visit(V v) override { - tolk_assert(cur_state == MarkingState::LValue); + tolk_assert(cur_state == MarkingState::LValue || v->is_lateinit); mark_vertex(v); parent::visit(v); } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index abeaffa1f6..71149cd4b1 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -150,6 +150,12 @@ static void fire_error_cannot_deduce_untyped_tuple_access(FunctionPtr cur_f, Src fire(cur_f, loc, "can not deduce type of `" + idx_access + "`\neither assign it to variable like `var c: int = " + idx_access + "` or cast the result like `" + idx_access + " as int`"); } +// fire an error on using lateinit variable before definite assignment +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_using_lateinit_variable_uninitialized(FunctionPtr cur_f, SrcLocation loc, std::string_view name) { + fire(cur_f, loc, "using variable `" + static_cast(name) + "` before it's definitely assigned"); +} + // helper function: given hint = `Ok | Err` and struct `Ok`, return `Ok` // example: `match (...) { Ok => ... }` we need to deduce `Ok` based on subject static TypePtr try_pick_instantiated_generic_from_hint(TypePtr hint, StructPtr lookup_ref) { @@ -406,6 +412,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, v->var_ref->declared_type); } else { assign_inferred_type(v, v->type_node ? v->type_node->resolved_type : TypeDataUnknown::create()); + flow.register_known_type(SinkExpression(v->var_ref), TypeDataUnknown::create()); // it's unknown before assigned } return ExprFlow(std::move(flow), used_as_condition); } @@ -842,6 +849,9 @@ class InferTypesAndCallsAndFieldsVisitor final { TypePtr declared_or_smart_casted = flow.smart_cast_if_exists(SinkExpression(var_ref)); tolk_assert(declared_or_smart_casted != nullptr); // all local vars are presented in flow assign_inferred_type(v, declared_or_smart_casted); + if (var_ref->is_lateinit() && declared_or_smart_casted == TypeDataUnknown::create() && v->is_rvalue) { + fire_error_using_lateinit_variable_uninitialized(cur_f, v->loc, v->get_name()); + } // it might be `local_var()` also, don't fill out_f_called, we have no fun_ref, it's a call of arbitrary expression } else if (GlobalConstPtr const_ref = v->sym->try_as()) { @@ -1220,6 +1230,8 @@ class InferTypesAndCallsAndFieldsVisitor final { TypeInferringUnifyStrategy branches_unifier; FlowContext arms_entry_facts = flow.clone(); FlowContext match_out_flow; + bool has_expr_arm = false; + bool has_else_arm = false; for (int i = 0; i < v->get_arms_count(); ++i) { auto v_arm = v->get_arm(i); @@ -1245,7 +1257,13 @@ class InferTypesAndCallsAndFieldsVisitor final { } } arm_flow.register_known_type(s_expr, exact_type); + + } else if (v_arm->pattern_kind == MatchArmKind::const_expression) { + has_expr_arm = true; + } else if (v_arm->pattern_kind == MatchArmKind::else_branch) { + has_else_arm = true; } + arm_flow = infer_any_expr(v_arm->get_body(), std::move(arm_flow), false, hint).out_flow; match_out_flow = i == 0 ? std::move(arm_flow) : FlowContext::merge_flow(std::move(match_out_flow), std::move(arm_flow)); branches_unifier.unify_with(v_arm->get_body()->inferred_type, hint); @@ -1253,6 +1271,10 @@ class InferTypesAndCallsAndFieldsVisitor final { if (v->get_arms_count() == 0) { match_out_flow = std::move(arms_entry_facts); } + if (has_expr_arm && !has_else_arm) { + FlowContext else_flow = process_any_statement(createV(v->loc), std::move(arms_entry_facts)); + match_out_flow = FlowContext::merge_flow(std::move(match_out_flow), std::move(else_flow)); + } if (v->is_statement()) { assign_inferred_type(v, TypeDataVoid::create()); diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 4452ea5738..ab17c5b42c 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -132,15 +132,15 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { static NameAndScopeResolver current_scope; static FunctionPtr cur_f; - static LocalVarPtr create_local_var_sym(std::string_view name, SrcLocation loc, AnyTypeV declared_type_node, bool immutable) { - LocalVarData* v_sym = new LocalVarData(static_cast(name), loc, declared_type_node, immutable * LocalVarData::flagImmutable, -1); + static LocalVarPtr create_local_var_sym(std::string_view name, SrcLocation loc, AnyTypeV declared_type_node, bool immutable, bool lateinit) { + LocalVarData* v_sym = new LocalVarData(static_cast(name), loc, declared_type_node, immutable * LocalVarData::flagImmutable + lateinit * LocalVarData::flagLateInit, -1); current_scope.add_local_var(v_sym); return v_sym; } static void process_catch_variable(AnyExprV catch_var) { if (auto v_ref = catch_var->try_as()) { - LocalVarPtr var_ref = create_local_var_sym(v_ref->get_name(), catch_var->loc, nullptr, true); + LocalVarPtr var_ref = create_local_var_sym(v_ref->get_name(), catch_var->loc, nullptr, true, false); v_ref->mutate()->assign_sym(var_ref); } } @@ -158,7 +158,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { } v->mutate()->assign_var_ref(var_ref); } else { - LocalVarPtr var_ref = create_local_var_sym(v->get_name(), v->loc, v->type_node, v->is_immutable); + LocalVarPtr var_ref = create_local_var_sym(v->get_name(), v->loc, v->type_node, v->is_immutable, v->is_lateinit); v->mutate()->assign_var_ref(var_ref); } } diff --git a/tolk/symtable.h b/tolk/symtable.h index 38f090e7b4..e6b2f1ce64 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -49,6 +49,7 @@ struct LocalVarData final : Symbol { enum { flagMutateParameter = 1, // parameter was declared with `mutate` keyword flagImmutable = 2, // variable was declared via `val` (not `var`) + flagLateInit = 4, // variable was declared via `lateinit` (not assigned at declaration) }; AnyTypeV type_node; // either at declaration `var x:int`, or if omitted, from assigned value `var x=2` @@ -74,6 +75,7 @@ struct LocalVarData final : Symbol { bool is_parameter() const { return param_idx >= 0; } bool is_immutable() const { return flags & flagImmutable; } + bool is_lateinit() const { return flags & flagLateInit; } bool is_mutate_parameter() const { return flags & flagMutateParameter; } LocalVarData* mutate() const { return const_cast(this); } From 68b67d4d7a3baebbc1f3c4fdc0e087116486cb91 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 28 May 2025 17:08:28 +0300 Subject: [PATCH 261/524] Fix loop in collation manager --- validator/collation-manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index 27af220df2..e987de917c 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -94,7 +94,7 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas } case CollatorsList::mode_round_robin: { size_t iters = 0; - for (size_t i = s->cur_idx; iters < s->collators.size(); (++i) %= s->collators.size()) { + for (size_t i = s->cur_idx; iters < s->collators.size(); (++i) %= s->collators.size(), ++iters) { adnl::AdnlNodeIdShort& collator = s->collators[i]; if (collators_[collator].alive) { selected_collator = collator; From 7e01156c4eafc5c72e8406e1026eca9a4ba366ac Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 29 May 2025 11:21:04 +0300 Subject: [PATCH 262/524] Use rldp2 in collator node and shard block retainer --- validator-engine/validator-engine.cpp | 2 +- validator/collation-manager.cpp | 6 +++--- validator/collation-manager.hpp | 6 +++--- validator/collator-node.cpp | 4 ++-- validator/collator-node.hpp | 5 +++-- validator/manager.cpp | 12 ++++++------ validator/manager.h | 2 ++ validator/manager.hpp | 13 +++++++++++-- validator/shard-block-retainer.cpp | 4 ++-- validator/shard-block-retainer.hpp | 6 +++--- validator/shard-block-verifier.cpp | 4 ++-- validator/shard-block-verifier.hpp | 6 +++--- 12 files changed, 41 insertions(+), 29 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index a7d28ea7ff..c78e0acc36 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -2182,7 +2182,7 @@ void ValidatorEngine::start_validator() { load_collator_options(); validator_manager_ = ton::validator::ValidatorManagerFactory::create( - validator_options_, db_root_, keyring_.get(), adnl_.get(), rldp_.get(), overlay_manager_.get()); + validator_options_, db_root_, keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(), overlay_manager_.get()); for (auto &v : config_.validators) { td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_permanent_key, v.first, diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index e987de917c..853ac37aea 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -27,7 +27,7 @@ namespace ton::validator { void CollationManager::start_up() { - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); + td::actor::send_closure(rldp_, &rldp2::Rldp::add_id, local_id_); update_collators_list(*opts_->get_collators_list()); } @@ -187,7 +187,7 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas << selected_collator << ") in " << timer.elapsed() << "s"; P.set_result(std::move(candidate)); }; - td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_id_, selected_collator, "collatequery", + td::actor::send_closure(rldp_, &rldp2::Rldp::send_query_ex, local_id_, selected_collator, "collatequery", std::move(P2), timeout, std::move(query), max_answer_size); } @@ -339,7 +339,7 @@ void CollationManager::alarm() { td::actor::send_closure(SelfId, &CollationManager::got_pong, id, std::move(R)); }; LOG(DEBUG) << "sending ping to " << id; - td::actor::send_closure(rldp_, &rldp::Rldp::send_query, local_id_, id, "ping", std::move(P), + td::actor::send_closure(rldp_, &rldp2::Rldp::send_query, local_id_, id, "ping", std::move(P), td::Timestamp::in(2.0), std::move(query)); } else { alarm_timestamp().relax(collator.ping_at); diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp index 048d30812b..5b8d2b7a07 100644 --- a/validator/collation-manager.hpp +++ b/validator/collation-manager.hpp @@ -17,7 +17,7 @@ #pragma once #include "interfaces/validator-manager.h" -#include "rldp/rldp.h" +#include "rldp2/rldp.h" #include namespace ton::validator { @@ -27,7 +27,7 @@ class ValidatorManager; class CollationManager : public td::actor::Actor { public: CollationManager(adnl::AdnlNodeIdShort local_id, td::Ref opts, - td::actor::ActorId manager, td::actor::ActorId rldp) + td::actor::ActorId manager, td::actor::ActorId rldp) : local_id_(local_id), opts_(opts), manager_(manager), rldp_(rldp) { } @@ -50,7 +50,7 @@ class CollationManager : public td::actor::Actor { adnl::AdnlNodeIdShort local_id_; td::Ref opts_; td::actor::ActorId manager_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; void collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, BlockCandidatePriority priority, diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 238eb89218..4b0d7e140b 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -29,7 +29,7 @@ namespace ton::validator { CollatorNode::CollatorNode(adnl::AdnlNodeIdShort local_id, td::Ref opts, td::actor::ActorId manager, td::actor::ActorId adnl, - td::actor::ActorId rldp) + td::actor::ActorId rldp) : local_id_(local_id) , opts_(std::move(opts)) , manager_(std::move(manager)) @@ -58,7 +58,7 @@ void CollatorNode::start_up() { td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, adnl::Adnl::int_to_bytestring(ton_api::collatorNode_ping::ID), std::make_unique(actor_id(this))); - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, adnl::AdnlNodeIdShort(local_id_)); + td::actor::send_closure(rldp_, &rldp2::Rldp::add_id, adnl::AdnlNodeIdShort(local_id_)); } void CollatorNode::tear_down() { diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index 604338f0b6..a4df036816 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -18,6 +18,7 @@ #include "interfaces/validator-manager.h" #include "rldp/rldp.h" +#include "rldp2/rldp.h" #include #include @@ -29,7 +30,7 @@ class CollatorNode : public td::actor::Actor { public: CollatorNode(adnl::AdnlNodeIdShort local_id, td::Ref opts, td::actor::ActorId manager, td::actor::ActorId adnl, - td::actor::ActorId rldp); + td::actor::ActorId rldp); void start_up() override; void tear_down() override; void add_shard(ShardIdFull shard); @@ -53,7 +54,7 @@ class CollatorNode : public td::actor::Actor { td::Ref opts_; td::actor::ActorId manager_; td::actor::ActorId adnl_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; std::vector collating_shards_; std::set validator_adnl_ids_; diff --git a/validator/manager.cpp b/validator/manager.cpp index 47c243eb45..8cf85a9dc0 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2643,7 +2643,7 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group td::actor::ActorId ValidatorManagerImpl::get_collation_manager(adnl::AdnlNodeIdShort adnl_id) { auto &actor = collation_managers_[adnl_id]; if (actor.empty()) { - actor = td::actor::create_actor("collation", adnl_id, opts_, actor_id(this), rldp_); + actor = td::actor::create_actor("collation", adnl_id, opts_, actor_id(this), rldp2_); } return actor.get(); } @@ -3486,7 +3486,7 @@ void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh auto it = collator_nodes_.find(id); if (it == collator_nodes_.end()) { it = collator_nodes_.emplace(id, Collator()).first; - it->second.actor = td::actor::create_actor("collatornode", id, opts_, actor_id(this), adnl_, rldp_); + it->second.actor = td::actor::create_actor("collatornode", id, opts_, actor_id(this), adnl_, rldp2_); if (last_masterchain_state_.not_null()) { td::actor::send_closure(it->second.actor, &CollatorNode::new_masterchain_block_notification, last_masterchain_state_); @@ -3633,10 +3633,10 @@ td::Ref ValidatorManagerImpl::get_block_persistent_s td::actor::ActorOwn ValidatorManagerFactory::create( td::Ref opts, std::string db_root, td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays) { return td::actor::create_actor("manager", std::move(opts), db_root, keyring, adnl, - rldp, overlays); + rldp, rldp2, overlays); } void ValidatorManagerImpl::log_collate_query_stats(CollationStats stats) { @@ -3783,7 +3783,7 @@ void ValidatorManagerImpl::init_shard_block_verifier(adnl::AdnlNodeIdShort local shard_block_verifier_ = {}; } else { shard_block_verifier_ = td::actor::create_actor( - "shardblockverifier", local_id, last_masterchain_state_, opts_, actor_id(this), adnl_, rldp_); + "shardblockverifier", local_id, last_masterchain_state_, opts_, actor_id(this), adnl_, rldp2_); } } } @@ -3799,7 +3799,7 @@ void ValidatorManagerImpl::wait_verify_shard_blocks(std::vector bloc void ValidatorManagerImpl::add_shard_block_retainer(adnl::AdnlNodeIdShort id) { shard_block_retainers_[id] = td::actor::create_actor( - "shardblockretainer", id, last_masterchain_state_, opts_, actor_id(this), adnl_, rldp_); + "shardblockretainer", id, last_masterchain_state_, opts_, actor_id(this), adnl_, rldp2_); } void ValidatorManagerImpl::iterate_temp_block_handles(std::function f) { diff --git a/validator/manager.h b/validator/manager.h index 4fe2037fce..652b775885 100644 --- a/validator/manager.h +++ b/validator/manager.h @@ -20,6 +20,7 @@ #include "validator/validator.h" #include "adnl/adnl.h" #include "rldp/rldp.h" +#include "rldp2/rldp.h" namespace ton { @@ -32,6 +33,7 @@ class ValidatorManagerFactory { td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId rldp2, td::actor::ActorId overlays); }; diff --git a/validator/manager.hpp b/validator/manager.hpp index 82986a0602..ae90f651f3 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -32,6 +32,7 @@ #include "manager-init.h" #include "state-serializer.hpp" #include "rldp/rldp.h" +#include "rldp2/rldp.h" #include "token-manager.h" #include "queue-size-counter.hpp" #include "validator-telemetry.hpp" @@ -595,8 +596,15 @@ class ValidatorManagerImpl : public ValidatorManager { ValidatorManagerImpl(td::Ref opts, std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays) - : opts_(std::move(opts)), db_root_(db_root), keyring_(keyring), adnl_(adnl), rldp_(rldp), overlays_(overlays) { + td::actor::ActorId rldp, td::actor::ActorId rldp2, + td::actor::ActorId overlays) + : opts_(std::move(opts)) + , db_root_(db_root) + , keyring_(keyring) + , adnl_(adnl) + , rldp_(rldp) + , rldp2_(rldp2) + , overlays_(overlays) { } public: @@ -714,6 +722,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::actor::ActorId keyring_; td::actor::ActorId adnl_; td::actor::ActorId rldp_; + td::actor::ActorId rldp2_; td::actor::ActorId overlays_; td::actor::ActorOwn serializer_; diff --git a/validator/shard-block-retainer.cpp b/validator/shard-block-retainer.cpp index c36a9bac6a..528f6bcf78 100644 --- a/validator/shard-block-retainer.cpp +++ b/validator/shard-block-retainer.cpp @@ -41,7 +41,7 @@ void ShardBlockRetainer::start_up() { td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, adnl::Adnl::int_to_bytestring(ton_api::shardBlockVerifier_subscribe::ID), std::make_unique(actor_id(this))); - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); + td::actor::send_closure(rldp_, &rldp2::Rldp::add_id, local_id_); } void ShardBlockRetainer::tear_down() { @@ -154,7 +154,7 @@ void ShardBlockRetainer::send_confirmations(adnl::AdnlNodeIdShort dst, std::vect for (size_t i = l; i < r; ++i) { query->blocks_.push_back(create_tl_block_id(blocks[i])); } - td::actor::send_closure(rldp_, &rldp::Rldp::send_message, local_id_, dst, serialize_tl_object(query, true)); + td::actor::send_closure(rldp_, &rldp2::Rldp::send_message, local_id_, dst, serialize_tl_object(query, true)); } } diff --git a/validator/shard-block-retainer.hpp b/validator/shard-block-retainer.hpp index ca8d4c8776..366a6d506e 100644 --- a/validator/shard-block-retainer.hpp +++ b/validator/shard-block-retainer.hpp @@ -16,7 +16,7 @@ */ #pragma once #include "interfaces/validator-manager.h" -#include "rldp.h" +#include "rldp2/rldp.h" #include namespace ton::validator { @@ -25,7 +25,7 @@ class ShardBlockRetainer : public td::actor::Actor { public: ShardBlockRetainer(adnl::AdnlNodeIdShort local_id, td::Ref last_masterchain_state, td::Ref opts, td::actor::ActorId manager, - td::actor::ActorId adnl, td::actor::ActorId rldp) + td::actor::ActorId adnl, td::actor::ActorId rldp) : local_id_(local_id) , last_masterchain_state_(last_masterchain_state) , opts_(std::move(opts)) @@ -49,7 +49,7 @@ class ShardBlockRetainer : public td::actor::Actor { td::Ref opts_; td::actor::ActorId manager_; td::actor::ActorId adnl_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; bool inited_ = false; std::set validator_adnl_ids_; diff --git a/validator/shard-block-verifier.cpp b/validator/shard-block-verifier.cpp index 000260a362..ebd6111440 100644 --- a/validator/shard-block-verifier.cpp +++ b/validator/shard-block-verifier.cpp @@ -41,7 +41,7 @@ void ShardBlockVerifier::start_up() { td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, adnl::Adnl::int_to_bytestring(ton_api::shardBlockVerifier_confirmBlocks::ID), std::make_unique(actor_id(this))); - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); + td::actor::send_closure(rldp_, &rldp2::Rldp::add_id, local_id_); } void ShardBlockVerifier::tear_down() { @@ -107,7 +107,7 @@ void ShardBlockVerifier::alarm() { LOG(WARNING) << "Subscribe to " << node_id << " for " << shard.to_str() << " : " << R.move_as_error(); } }; - td::actor::send_closure(rldp_, &rldp::Rldp::send_query, local_id_, node_id, "subscribe", std::move(P), + td::actor::send_closure(rldp_, &rldp2::Rldp::send_query, local_id_, node_id, "subscribe", std::move(P), td::Timestamp::in(3.0), create_serialize_tl_object( create_tl_shard_id(shard_config.shard_id), 0)); diff --git a/validator/shard-block-verifier.hpp b/validator/shard-block-verifier.hpp index 82c1e3738c..025996681e 100644 --- a/validator/shard-block-verifier.hpp +++ b/validator/shard-block-verifier.hpp @@ -16,7 +16,7 @@ */ #pragma once #include "interfaces/validator-manager.h" -#include "rldp.h" +#include "rldp2/rldp.h" #include namespace ton::validator { @@ -25,7 +25,7 @@ class ShardBlockVerifier : public td::actor::Actor { public: ShardBlockVerifier(adnl::AdnlNodeIdShort local_id, td::Ref last_masterchain_state, td::Ref opts, td::actor::ActorId manager, - td::actor::ActorId adnl, td::actor::ActorId rldp) + td::actor::ActorId adnl, td::actor::ActorId rldp) : local_id_(local_id) , last_masterchain_state_(last_masterchain_state) , opts_(std::move(opts)) @@ -54,7 +54,7 @@ class ShardBlockVerifier : public td::actor::Actor { td::Ref opts_; td::actor::ActorId manager_; td::actor::ActorId adnl_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; td::Ref config_; From d6db65bc630c937b513f8c900f70c68dda2bfa34 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 14 Apr 2025 16:30:19 +0300 Subject: [PATCH 263/524] [Tolk] Built-in type `address` It holds internal/external/none addresses. It's a slice at runtime, but a dedicated type from the type system point of view. Auto-serialization will easily pack/unpack it. Operators == and != work on addresses. --- crypto/smartcont/tolk-stdlib/common.tolk | 89 +++--- tolk-tester/tests/cells-slices.tolk | 2 +- tolk-tester/tests/constants-tests.tolk | 11 +- .../tests/invalid-syntax/err-3208.tolk | 8 + .../tests/invalid-syntax/err-3357.tolk | 8 + .../tests/invalid-typing/err-6710.tolk | 15 + tolk-tester/tests/parse-address.tolk | 266 +++++++++++------- tolk-tester/tests/strings-tests.tolk | 8 +- tolk/builtins.cpp | 4 +- tolk/constant-evaluator.cpp | 50 ++-- tolk/pipe-ast-to-legacy.cpp | 26 ++ tolk/pipe-check-inferred-types.cpp | 19 +- tolk/pipe-optimize-boolean-expr.cpp | 9 + tolk/pipe-resolve-types.cpp | 1 + tolk/type-system.cpp | 32 ++- tolk/type-system.h | 19 ++ 16 files changed, 398 insertions(+), 169 deletions(-) create mode 100644 tolk-tester/tests/invalid-syntax/err-3208.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3357.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6710.tolk diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 8ff2010bcb..642337df5e 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -133,10 +133,10 @@ fun mulDivMod(x: int, y: int, z: int): (int, int) /// Example: `contract.getCode()` and other methods. struct contract {} -/// Returns the internal address of the current smart contract as a Slice with a `MsgAddressInt`. -/// If necessary, it can be parsed further using primitives such as [parseStandardAddress]. +/// Returns the internal address of the current smart contract. +/// If necessary, it can be parsed further using [address.getWorkchain] and others. @pure -fun contract.getAddress(): slice +fun contract.getAddress(): address asm "MYADDR"; /// Returns the balance (in nanotoncoins) of the smart contract at the start of Computation Phase. @@ -407,15 +407,6 @@ fun debug.dumpStack(): void When you _preload_ some data, you just get the result without mutating the slice. */ -/// Compile-time function that converts a constant string to TL-encoded MsgAddressInt (TVM slice). -/// Example: stringAddressToSlice("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5") -/// Example: stringAddressToSlice("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8") -/// Note: it's a compile-time function, `stringAddressToSlice(slice_var)` does not work. -/// Use `parseStandardAddress` to decode a slice at runtime into workchain and hash. -@pure -fun stringAddressToSlice(constStringAddress: slice): slice - builtin; - /// Compile-time function that converts a constant hex-encoded string to N/2 bytes. /// Example: `const v = stringHexToSlice("abcdef")` = slice with 3 bytes `[ 0xAB, 0xCD, 0xEF ]` /// Note: stringHexToSlice(slice_var) does not work! It accepts a constant string and works at compile-time. @@ -573,6 +564,11 @@ fun builder.storeUint(mutate self, x: int, len: int): self fun builder.storeSlice(mutate self, s: slice): self asm "STSLICER"; +/// Stores an address into a builder. +@pure +fun builder.storeAddress(mutate self, addr: address): self + asm "STSLICER"; + /// Stores amount of Toncoins into a builder. @pure fun builder.storeCoins(mutate self, x: coins): self @@ -694,37 +690,64 @@ fun builder.bitsCount(self): int where `u`, `x`, and `s` have the same meaning as for `addr_std`. */ -/// Loads from slice [s] the only prefix that is a valid `MsgAddress`, -/// and returns both this prefix `s'` and the remainder `s''` of [s] as slices. +/// Compile-time function that parses a valid contract address. +/// Example: address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5") +/// Example: address("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8") +/// Returns `address`, which can be stored in a builder, compared with `==`, etc. @pure -fun slice.loadAddress(mutate self): slice - asm( -> 1 0) "LDMSGADDR"; +fun address(stdAddress: slice): address + builtin; + +/// Creates a slice representing TL addr_none$00 (two `0` bits). +@pure +fun createAddressNone(): address + asm "b{00} PUSHSLICE"; + +/// Returns if it's an empty address. +/// Don't confuse it with null! Empty address is a slice with two `0` bits. +/// In TL/B, it's addr_none$00. +@pure +fun address.isNone(self): bool + asm "b{00} SDBEGINSQ" "NIP"; -/// Decomposes slice [s] containing a valid `MsgAddress` into a `tuple t` with separate fields of this `MsgAddress`. -/// If [s] is not a valid `MsgAddress`, a cell deserialization exception is thrown. +/// Returns if it's a standard (internal) address. Such addresses contain workchain (8 bits) and hash (256 bits). +/// All contract addresses are internal, so it's the most practical use case. +/// In TL/B it's addr_std$10. +/// For internal addresses, you can call [address.getWorkchain] and [address.getWorkchainAndHash]. @pure -fun parseAddress(s: slice): tuple - asm "PARSEMSGADDR"; +fun address.isInternal(self): bool + asm "b{10} SDBEGINSQ" "NIP"; -/// Parses slice [s] containing a valid `MsgAddressInt` (usually a `msg_addr_std`), -/// applies rewriting from the anycast (if present) to the same-length prefix of the address, -/// and returns both the workchain and the 256-bit address as integers. -/// If the address is not 256-bit, or if [s] is not a valid serialization of `MsgAddressInt`, -/// throws a cell deserialization exception. +/// Returns if it's an external address, used to communication with the outside world. +/// In TL/B it's addr_extern$01. @pure -fun parseStandardAddress(s: slice): (int, int) +fun address.isExternal(self): bool + asm "b{01} SDBEGINSQ" "NIP"; + +/// Extracts workchain and hash from a standard (internal) address. +/// If the address is not internal, throws a cell deserialization exception. +@pure +fun address.getWorkchainAndHash(self): (int8, uint256) asm "REWRITESTDADDR"; -/// Creates a slice representing TL addr_none$00 (two `0` bits). +/// Extracts workchain from a standard (internal) address. +/// If the address is not internal, throws a cell deserialization exception. @pure -fun createAddressNone(): slice - asm "b{00} PUSHSLICE"; +fun address.getWorkchain(self): int8 + asm "REWRITESTDADDR" "DROP"; -/// Returns if a slice pointer contains an empty address. -/// In other words, a slice starts with two `0` bits (TL addr_none$00). +/// Checks whether two addresses are equal. Equivalent to `a == b`. +/// Deprecated! Left for smoother transition from FunC, where you used `slice` everywhere. +/// Use just `a == b` and `a != b` to compare addresses, don't use bitsEqual. @pure -fun addressIsNone(s: slice): bool - asm "2 PLDU" "0 EQINT"; +@deprecated("use `senderAddress == ownerAddress`, not `senderAddress.bitsEqual(ownerAddress)`") +fun address.bitsEqual(self, b: address): bool + asm "SDEQ"; + +/// Loads from slice [s] a valid `MsgAddress` (none/internal/external). +@pure +fun slice.loadAddress(mutate self): address + asm( -> 1 0) "LDMSGADDR"; /** diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 70c40b531e..e5b89424a6 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -183,7 +183,7 @@ fun test111() { .endCell().beginParse(); var op1 = s.loadUint(32); var q1 = s.loadUint(64); - if (addressIsNone(s)) { + if ((s as address).isNone()) { s.skipBits(2); } if (s.loadBool() == false) { diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index ed510c13c6..838a7dae33 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -22,6 +22,8 @@ const nibbles: int = 4; const strange_zero = (!10 as int); const strange_minus_1: MInt = (!0 as int); +const addr1 = address("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"); + const true1 = true; const true2 = !!true; const true3 = true1 && true2; @@ -88,6 +90,12 @@ fun test5() { return (intOrN == null, int32Or64 is int32, int32Or64); } +@method_id(106) +fun test6() { + __expect_type(addr1, "address"); + return (addr1 == addr1, addr1 != addr1, addr1 == createAddressNone(), addr1.getWorkchain()); +} + fun main() { var i1: int = iget1(); var i2: int = iget2(); @@ -117,6 +125,7 @@ fun main() { @testcase | 103 | | 0 0 @testcase | 104 | | 1 1 2 @testcase | 105 | | -1 0 7 48 +@testcase | 106 | | -1 0 0 -1 -@code_hash 49556957179018386976033482229516007597784982050169632168468608374010225644988 +@code_hash 85012002134196298607946339234948178079079623823648671175958399073436861460061 */ diff --git a/tolk-tester/tests/invalid-syntax/err-3208.tolk b/tolk-tester/tests/invalid-syntax/err-3208.tolk new file mode 100644 index 0000000000..75292b2529 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3208.tolk @@ -0,0 +1,8 @@ +fun main() { + return address("0:gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg"); +} + +/** +@compilation_should_fail +@stderr invalid standard address + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3357.tolk b/tolk-tester/tests/invalid-syntax/err-3357.tolk new file mode 100644 index 0000000000..de5569da32 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3357.tolk @@ -0,0 +1,8 @@ +fun main() { + return ton("1000000000"); +} + +/** +@compilation_should_fail +@stderr argument is too big and leads to overflow + */ diff --git a/tolk-tester/tests/invalid-typing/err-6710.tolk b/tolk-tester/tests/invalid-typing/err-6710.tolk new file mode 100644 index 0000000000..d0fae1d56d --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6710.tolk @@ -0,0 +1,15 @@ +fun f(s: slice) { + +} + +fun testCantPassAddressToSlice() { + var a = createAddressNone(); + f(a as slice); // ok + f(a); +} + +/** +@compilation_should_fail +@stderr can not pass `address` to `slice` +@stderr f(a); + */ diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index 912fb84c28..cf74af3f10 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -1,113 +1,171 @@ -const cc1 = stringAddressToSlice("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e"); -const cc2 = stringAddressToSlice("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); +const cc1 = address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e"); +const cc2 = address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); -fun verifyAddr(addr: slice, workchain: int, number: int) { - assert (addr.remainingBitsCount() == 3 + 8 + 256) throw 112; - addr.skipBits(3); - assert (addr.loadUint(8) == workchain) throw 111; - assert (addr.loadUint(256) == number) throw 111; +fun verifyAddr(addr: address, workchain: int, number: int) { + assert (addr == addr) throw 111; + assert (!(addr != addr)) throw 111; + assert (addr.isInternal(), 111); + assert (!addr.isExternal(), 111); + assert (!addr.isNone(), 111); + assert (addr.bitsEqual(addr), 111); + + var (wc, h) = addr.getWorkchainAndHash(); + assert (wc == workchain) throw 111; + assert (h == number) throw 111; + assert (addr.getWorkchain() == workchain) throw 111; + + var s = addr as slice; + assert (s.remainingBitsCount() == 3 + 8 + 256) throw 112; + s.skipBits(3); + assert (s.loadInt(8) == workchain) throw 111; + assert (s.loadUint(256) == number) throw 111; +} + +fun check0(addr: address) { + assert (addr.getWorkchain() == 0) throw 111; +} + +fun codegenAddrEq(a: address, b: address) { + if (a == b) { return 1; } + if (a != b) { return 2; } + return 3; } fun main() { - verifyAddr(stringAddressToSlice("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"), 255, 23158417847463239084714197001737581570653996933128112807891516801582625927987); - verifyAddr(stringAddressToSlice("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), 0, 0); - verifyAddr(stringAddressToSlice("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"), 0, 65607996509792174074532427555986248720836864382484024657400295821210434460432); - verifyAddr(stringAddressToSlice("UQCOgxbCOjOLH_cEuQdGgS23zBM5SrQQepMFedjK-oixYbis"), 0, 64460038539088394980732229180523693489682583665805557562964506609821558550881); - verifyAddr(stringAddressToSlice("EQDa4VOnTYlLvDJ0gZjNYm5PXfSmmtL6Vs6A_CZEtXCNICq_"), 0, 99002318936150612861744867526221033858534811876886359650897405270877291973920); - verifyAddr(stringAddressToSlice("Ef8BtXO9bcTMXjg9bgivKh4lhJmZWQPP6_rb9vfjlTP5FJtM"), 255, 772910975127952880303441415761050161913031788763061162001556772893733681428); - verifyAddr(stringAddressToSlice("Ef89xh-uy860-mCcvS8zcAUs8bApmxLGygDLEKjUk5RL-311"), 255, 27941138149036269893630478666581900122707382189183906805784676408403709676539); - verifyAddr(stringAddressToSlice("Ef_vA6yRfmt2P4UHnxlrQUZFcBnKux8mL2eMqBgpeMFPorr4"), 255, 108109262375472472702582493362335418330829651067377177643099076957184687427490); - verifyAddr(stringAddressToSlice("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), 255, 18502444830824300068094395885436326119386947594392869497312068745716154912158); - verifyAddr(stringAddressToSlice("Ef_fvrd0hBoVJUxoi7wH173Zk8NPiyVvxh5IoYSjEYZbOhsu"), 255, 101202732337223525952216789200341489000836292542250083765062769181728788863802); - verifyAddr(stringAddressToSlice("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"), 255, 46952625717497919357580310066854892621799390294920450816077086267929711460872); - verifyAddr(stringAddressToSlice("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"), 255, 48545777798729612074233611768739897492467685225150339217043102685589809464695); - verifyAddr(stringAddressToSlice("Ef9LynHHKgBxY6-l-W_dWN-CtGT2_ji5rN3EzOI-p9zWEfq6"), 255, 34281152017620085319078796986198022632548048219136747083019177301186013091345); - verifyAddr(stringAddressToSlice("Ef9hMd78gzSiVsK0zz0AHtEja8x1UoB_NDZMjn-l86NQK_2Y"), 255, 43962460814164090767878334494257755557842170134382045184921495822637115592747); - verifyAddr(stringAddressToSlice("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"), 255, 23557057702048801338698514499604413540742716310574705490458593067566768087319); - verifyAddr(stringAddressToSlice("Ef_fdIbThooPs4_r2DE_Z6ZsWycJdHLnsuKAJHTcbaZaipez"), 255, 101071650030310556115830521522496708686577365303530257137459798093298869361290); - verifyAddr(stringAddressToSlice("Ef_lva0qEiZhWrrZJl-IJxyCcTQmmTo71fIWyQ31HxJ8NurV"), 255, 103914771557158282349484109182290824591675204108148026180964788916630125182006); - verifyAddr(stringAddressToSlice("Ef8sMGKypw006AeRYqimLjmY2Ufp-SHk8C0ZJBNgVBlzw_Nr"), 255, 19987255184378161380023126214650814972824352533523055905552702178965809886147); - verifyAddr(stringAddressToSlice("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff+W72r5gqPrHF"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr(stringAddressToSlice("EQCaSCHVak-jIc9ANutTAfHpZNM3YdGky7yaDzsTrg0WhFlm"), 0, 69783625181781015447914682554083924798054947959007050695795761257887453484676); - verifyAddr(stringAddressToSlice("EQBS9U3AfD15fGmOtRMXQAxcPVBwNuItfLcDni9fkbTyyNX0"), 0, 37523067738561024305547433298623118197038688994386001017161816416175242146504); - verifyAddr(stringAddressToSlice("EQBiMNL9qNWMAkJHuM0BFneYcuHL17kzS4pswpaEO-NGWrFG"), 0, 44412924025649114419413541526870954696667907029239618728289150652715284776538); - verifyAddr(stringAddressToSlice("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi"), 0, 9407242825041766837311851458322335726136775042891143504070507665010681354284); - verifyAddr(stringAddressToSlice("EQD-nhrinjv0B4LTgr0dRHTHwH1MOsgGhKBXJZd7vESMZUf1"), 0, 115166810931401616117484448645661180241548402534908005320733783571353775148133); - verifyAddr(stringAddressToSlice("EQAVD3Fni9I6j8XeSIl-wAGBEhqhame6OtAY0GScKT0D9X6f"), 0, 9525855215156855607080079714361451576383963668563135377495902959388099150837); - verifyAddr(stringAddressToSlice("EQC6ACq3VANZjqfRBy7JMHkpLwqQ9qyYJsCIGx1mYbQgxaKw"), 0, 84130484652351964071210477536969520113177637645401392541565606610268614566085); - verifyAddr(stringAddressToSlice("EQCIJLNFIko5CvpKn9oAkrDgLocDOoD4vwmHxNx_fsG_LkwW"), 0, 61579391178099797614367237687950512448308156724136883899001108680249616482094); - verifyAddr(stringAddressToSlice("EQCe4AYIBce1pAk2qJJPSs1OzyZRlKjkfq8zuC8D7erv6DUP"), 0, 71861245445432818728925844931259040612664802586395398157190478191760507596776); - verifyAddr(stringAddressToSlice("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"), 0, 78559023162479717496981724991265882229440558807791659796411897368395464230649); - verifyAddr(stringAddressToSlice("EQBBlraAps0OZaB9Q8ePQn2wVAaL1G411A-dNppyWe3X3GIT"), 0, 29666621803903557832193058147214384979915773445007872807927344851911086823388); - verifyAddr(stringAddressToSlice("EQBiASqUqaVizrozLRbszkWC2kETbkhpO2qniDVDPPg2_0W8"), 0, 44328719889509369519441680467651025944540360433148852643949783408843779749631); - verifyAddr(stringAddressToSlice("EQBu2Q1EO8gIoNA1qoGWnHUudKfmqlKEDTQE-DxN-_4sdg14"), 0, 50137910719490808065414827264266674858051167131188257457782826342827836714102); - verifyAddr(stringAddressToSlice("EQA5bvxWd5-q2vJUVqR9AlbEIfdFysLR0PXGgVlBf8x5hWuF"), 0, 25977927117604457079092522008276392864656238504700352770597256138254994667909); - verifyAddr(stringAddressToSlice("EQBguMSHjFv5bfoOdshr3ruS9ymSZzhRKMovoNrxGxZXvmee"), 0, 43748489720571123896506696370504498290006245978262404519821633796370658121662); - verifyAddr(stringAddressToSlice("EQAxL0oF1-zNgimPKthbDnYS4xj94rHtfNRN7_Pd1r2LNNv3"), 0, 22246882279393590648219842750911786376759362211171398419754426796438233910068); - verifyAddr(stringAddressToSlice("EQANX1uRKGZfyPIwEaIXrR0ZOqadct5q10dvKxWIxx7SQqzW"), 0, 6048549475100840191738010856156544571222758030966479209409932714701987172930); - verifyAddr(stringAddressToSlice("EQBitdFDoU5DWSjfKq7AsO29RIwAnBzzvcVVSn5ekQoB9Liv"), 0, 44647902768175374073183447303109856895983123510911038181495138821771906122228); - verifyAddr(stringAddressToSlice("EQBgbux7VSjqJHP7ByRK1q4QuVZbpSCesNgvz5qad3lfXX_B"), 0, 43618018778298854282398238948198420936771670943015013768514626213000552996701); - verifyAddr(stringAddressToSlice("EQDisBd8U7M3CEOZ8gcWCdetdmJi3AI31zIT5qBwOdmUbsxY"), 0, 102533830955233207294921564956803510155400341370448300698800842506363763004526); - verifyAddr(stringAddressToSlice("EQAZpn_eynVlf7Ii2d6jP_p1URPrdF9F3S7DiudQyelkjzwE"), 0, 11602000355550451044739442929923326898313570892134000961608306166632391730319); - verifyAddr(stringAddressToSlice("EQDE0HBgfkOiqHezLtExBGTvOs8eitthHQosBjW3BmDy1y2K"), 0, 89021598108837008984355105304701054698583123510131754065320641619941010764503); - verifyAddr(stringAddressToSlice("EQDyT36zktBN9PVWvZ1joRxhIfEUgCPt4F2isa-enUA_d6CP"), 0, 109600164736599393471831241268953938618560132398692391517933933264745646800759); - verifyAddr(stringAddressToSlice("EQDSMUGwt25IQd3_yHjI03F71G8Kp2GMaMEv2TiWoTKbsyRH"), 0, 95072727086440754059372943502908629555499501854161516009430039520728770059187); - verifyAddr(stringAddressToSlice("EQAgK1EcrvEuL9sCtoj3cNhVNOuf3lo5GIPE2gn1fwZZYB3j"), 0, 14550545393206146289454646242321274637527057595221202748348667645886114191712); - verifyAddr(stringAddressToSlice("EQCDKqL5w_6MD-Z7AOButu-uR-ZJTsgNU1fu464hn9grY81U"), 0, 59328315557704100696483472039557119625141880163887490602190749720459366378339); - verifyAddr(stringAddressToSlice("EQB1aVMyFBhnlYXmQjsma0S63kvxKU7ccZKFNCFTwX7ASPv4"), 0, 53106696421104300082516512931084483581353095629408473618166869610568148238408); - verifyAddr(stringAddressToSlice("EQBbjrXHoxDyh1ZYGBdBoQgLaScxW6pZR1hEhJC8BqF-5Kgq"), 0, 41412616102566803060532874463898939692666425753852274254609049615175463829220); - verifyAddr(stringAddressToSlice("EQC-QeZ13QP0lszxNKt380fCWuaV94vwC/bfuqmrlg1/fJPA"), 0, 86055876869280374285292827775555707420719385459150221433115419095878595346300); - verifyAddr(stringAddressToSlice("EQAiUwpF27vXCngqNhf_TQ5E_06ah0G4zuSrnfU7CLLaht5H"), 0, 15525356059048115813946213102829493539706126913595626308144289257869196581510); - verifyAddr(stringAddressToSlice("EQBqiVjmhe2iVGmgOSDO1FGjSiz_AMtb1w7lLEiP4XIF_SFy"), 0, 48187833566271418625754761625661652107159264793429628379411792200127405491709); - verifyAddr(stringAddressToSlice("EQDmwvaK2d_SbaPdpOM60effPWeKsksgDVwFPEyxuftM396K"), 0, 104376425077737068747642645125299653296942252727305637929378053253273342397663); - verifyAddr(stringAddressToSlice("EQDWtPZZgF7wvIMUHZQojuD3utiuivsW7WslRJ33dgv-5yc8"), 0, 97114682311034709685427168495629428400170984047839002197324103884924936519399); - verifyAddr(stringAddressToSlice("EQAA7z0JI0JKqbN-1uENKz9JrxIO5ZRY-ehMeg9fPncx50Ck"), 0, 422697701361909095759185681783393186844038628935759044330165207027374567911); - verifyAddr(stringAddressToSlice("EQBVUHRoCq6coQYUwOAhGSoAmQ6Mpm7dFlDYon6HMgWV8Ftr"), 0, 38588743302295548905191533977469452945717219128199196974980570837505276220912); - verifyAddr(stringAddressToSlice("EQCTdvDCf0bA5dOPI1-44tB2ZfNcMGiklzvg27TovgDEqM6E"), 0, 66700138358140658950710678965721715920748906761125730971082529064117803730088); - verifyAddr(stringAddressToSlice("EQBDBKE5WGKIlnoi3OOzw7vkKKIX55eWjPvgxJWwek8AyL2J"), 0, 30313140970524770883308749215942283658935592719811899513010665548955593408712); - verifyAddr(stringAddressToSlice("EQAvCSyLCo21GrqLAifdov4WkOxuGQCjMRxgF1cXSaNzLHZe"), 0, 21274912932379789207153885262858116665851037273450532982121514600400844714796); - verifyAddr(stringAddressToSlice("EQCsLpDeHB2qpRbmsCb_0xmsYVNx1NgeYrvHGT1TDrHkDgL4"), 0, 77880084760844670718511036557364450189323549135231036747480176919181282894862); - verifyAddr(stringAddressToSlice("EQCTQ8kPwyX92r48gCIL_pLN_RcQT9ghZygnmDTYkOkuW_j5"), 0, 66609755171046741472463433430016629628609840960137226492652665879987546041947); - verifyAddr(stringAddressToSlice("EQCTrFRSHt-tfk7WxK9ZHQmqLcgxXxTK7wGfCEbqgY2W9Mcx"), 0, 66794468397542182731534559853537484892417154018190804733043974345563210356468); - verifyAddr(stringAddressToSlice("EQCv28y49GdaLncoclv0ISdDlMUY_cxDPGNWFCPT8t4GuqUJ"), 0, 79543100951881731989812212377176715376834409392116644269458867858071577560762); - verifyAddr(stringAddressToSlice("EQCVL-k6deDR56Z8pcb0Btg0lGfaivOGfdDCD1vvyRsyL9vS"), 0, 67479265933941008511790471646564661743401752930295407567346938670637286896175); - verifyAddr(stringAddressToSlice("EQD6t2dXDjZxF1DqunKF-8dEWivJdliY_0FYiCXnthuqnDCa"), 0, 113402258385556889021060606279033166272577193563727959698596277924908309916316); - verifyAddr(stringAddressToSlice("EQDE98XNzXiPq7VnbJJ2M4-Ht3tX_OWR0xUTTnDC8NObLmyU"), 0, 89091094739778473356272490822716056624384395256388481953562551087642791090990); - verifyAddr(stringAddressToSlice("EQDfeRDE1TDhwt478CDR0Q7MDwqcTUhfjqyTT59mgoAaF6f7"), 0, 101079669463449311486034260688909914923832300293253430457119371423825321269783); - verifyAddr(stringAddressToSlice("EQDijcEyUKa-QgCbeGlggQk1uBtt2ZRHyW4Y4gB4R6MN6RLW"), 0, 102473162609487797404330889623966425536887610061087715571345738626121871855081); - verifyAddr(stringAddressToSlice("EQDOtFOt41skbjBkZF89oYXpoDECjlxIzD-ShWAOYyzuxqLA"), 0, 93495056812773926196963707371665512785268729004579280701087533371037976424134); - verifyAddr(stringAddressToSlice("EQDuJKSFWU7AYqH6KLFfAbYvMuz346eWmJvG6_2NYE42_B4T"), 0, 107715199938628393100813870735031557263256555616273999363057194279168054802172); - verifyAddr(stringAddressToSlice("EQDwGu4vFv1e3wn8min_iy7OPJXegOYTFQ5bZFZ5a5ZPiBpX"), 0, 108602665568837301744601989570019709742180613578164394799026726718721456754568); - verifyAddr(stringAddressToSlice("EQC4G2ph6AS_mD_-cIv4aIYm1z5jAgCW_TTDEr72ygXOP2X-"), 0, 83274003234732023403481554420859495155084746906198543572711543697320249249343); - verifyAddr(stringAddressToSlice("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"), 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); - verifyAddr(stringAddressToSlice("EQDoIA20MF1qEcSPtROdCu5ukGx9dVjgMeJh1oQ4A4cf_Jif"), 0, 104993214557977037193613824776415934089204193426692473563548548423424814817276); - verifyAddr(stringAddressToSlice("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"), 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); - verifyAddr(stringAddressToSlice("EQClLO4EnZ_rTyV1GVpWy53pLgWJRki5c4ZzuM_1O_ClBkO9"), 0, 74711004027159342540251007601464500186374346239921204216319145006974068892934); - verifyAddr(stringAddressToSlice("EQDmkj65Ab_m0aZaW8IpKw4kYqIgITw_HRstYEkVQ6NIYCyW"), 0, 104290347741656803921830951060768893809692975574470790497562993373950614128736); - verifyAddr(stringAddressToSlice("EQCqNTwAYUNhPFS0RgqZoTLGJcQQxbAJ7csUo4YO3_TONLab"), 0, 76987241268612358571638783428744566580605181728938801022059780105627411729972); - verifyAddr(stringAddressToSlice("EQCL3DmCynaRK7-vsfeNmd4Jj-UxAIHPvA4qS2xwaL6UpLbF"), 0, 63260589232981964910240894899061676480139492286430589202252472895352724165796); - verifyAddr(stringAddressToSlice("EQDbU1SVEjBE73oUqgAoM9gDcShUkM5EC2PgoCjuwVUKo-Ee"), 0, 99203745911752606845646497420891218522647962685916739950275357890977532807843); - verifyAddr(stringAddressToSlice("EQD02VdcF4TDbCKLLhZJ39NQTu6aWq2LnLjp0oXqbNu_BANK"), 0, 110748343802097970709980079967961144373090790244250392237586606542170934198020); - verifyAddr(stringAddressToSlice("EQBynBO23ywHy_CgarY9NK9FTz0yDsG82PtcbSTQgGoXwiuA"), 0, 51839428943991432793039248316067731096592274748149794482308513726460953499586); - verifyAddr(stringAddressToSlice("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr(stringAddressToSlice("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"), 0, 9183547432069678364603018431103042146626948674383548774683927217595824907333); - verifyAddr(stringAddressToSlice("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"), 0, 45985353862647206060987594732861817093328871106941773337270673759241903247880); - verifyAddr(stringAddressToSlice("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr(stringAddressToSlice("kQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPgpP"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr(stringAddressToSlice("kf-Dfdg-YQXaR2Q97gZJ4fGBtmV1DHOU1y1RPyyZZtRy_Ikh"), 255, 59475331506450494976393625198911249698879029820580340449086829444312920781564); - verifyAddr(stringAddressToSlice("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 0, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr(stringAddressToSlice("0:0000000000000000000000000000000000000000000000000000000000000000"), 0, 0); - verifyAddr(stringAddressToSlice("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"), 0, 115792089237316195423570985008687907853269984665640564039457584007913129639935); - verifyAddr(stringAddressToSlice("0:zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"), 0, 23158417847463239084714197001737581570653996933128112807891516801582625927987); - verifyAddr(stringAddressToSlice("0:0000000000000000000000000000000000000000000000000000000000000000"), 0, 0); - verifyAddr(stringAddressToSlice("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 1, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr(stringAddressToSlice("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 9, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr(stringAddressToSlice("99:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 99, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr(stringAddressToSlice("-1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 255, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + __expect_type(cc1, "address"); + + verifyAddr(address("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"), -1, 23158417847463239084714197001737581570653996933128112807891516801582625927987); + verifyAddr(address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), 0, 0); + verifyAddr(address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"), 0, 65607996509792174074532427555986248720836864382484024657400295821210434460432); + verifyAddr(address("UQCOgxbCOjOLH_cEuQdGgS23zBM5SrQQepMFedjK-oixYbis"), 0, 64460038539088394980732229180523693489682583665805557562964506609821558550881); + verifyAddr(address("EQDa4VOnTYlLvDJ0gZjNYm5PXfSmmtL6Vs6A_CZEtXCNICq_"), 0, 99002318936150612861744867526221033858534811876886359650897405270877291973920); + verifyAddr(address("Ef8BtXO9bcTMXjg9bgivKh4lhJmZWQPP6_rb9vfjlTP5FJtM"), -1, 772910975127952880303441415761050161913031788763061162001556772893733681428); + verifyAddr(address("Ef89xh-uy860-mCcvS8zcAUs8bApmxLGygDLEKjUk5RL-311"), -1, 27941138149036269893630478666581900122707382189183906805784676408403709676539); + verifyAddr(address("Ef_vA6yRfmt2P4UHnxlrQUZFcBnKux8mL2eMqBgpeMFPorr4"), -1, 108109262375472472702582493362335418330829651067377177643099076957184687427490); + verifyAddr(address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), -1, 18502444830824300068094395885436326119386947594392869497312068745716154912158); + verifyAddr(address("Ef_fvrd0hBoVJUxoi7wH173Zk8NPiyVvxh5IoYSjEYZbOhsu"), -1, 101202732337223525952216789200341489000836292542250083765062769181728788863802); + verifyAddr(address("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"), -1, 46952625717497919357580310066854892621799390294920450816077086267929711460872); + verifyAddr(address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"), -1, 48545777798729612074233611768739897492467685225150339217043102685589809464695); + verifyAddr(address("Ef9LynHHKgBxY6-l-W_dWN-CtGT2_ji5rN3EzOI-p9zWEfq6"), -1, 34281152017620085319078796986198022632548048219136747083019177301186013091345); + verifyAddr(address("Ef9hMd78gzSiVsK0zz0AHtEja8x1UoB_NDZMjn-l86NQK_2Y"), -1, 43962460814164090767878334494257755557842170134382045184921495822637115592747); + verifyAddr(address("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"), -1, 23557057702048801338698514499604413540742716310574705490458593067566768087319); + verifyAddr(address("Ef_fdIbThooPs4_r2DE_Z6ZsWycJdHLnsuKAJHTcbaZaipez"), -1, 101071650030310556115830521522496708686577365303530257137459798093298869361290); + verifyAddr(address("Ef_lva0qEiZhWrrZJl-IJxyCcTQmmTo71fIWyQ31HxJ8NurV"), -1, 103914771557158282349484109182290824591675204108148026180964788916630125182006); + verifyAddr(address("Ef8sMGKypw006AeRYqimLjmY2Ufp-SHk8C0ZJBNgVBlzw_Nr"), -1, 19987255184378161380023126214650814972824352533523055905552702178965809886147); + verifyAddr(address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff+W72r5gqPrHF"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(address("EQCaSCHVak-jIc9ANutTAfHpZNM3YdGky7yaDzsTrg0WhFlm"), 0, 69783625181781015447914682554083924798054947959007050695795761257887453484676); + verifyAddr(address("EQBS9U3AfD15fGmOtRMXQAxcPVBwNuItfLcDni9fkbTyyNX0"), 0, 37523067738561024305547433298623118197038688994386001017161816416175242146504); + verifyAddr(address("EQBiMNL9qNWMAkJHuM0BFneYcuHL17kzS4pswpaEO-NGWrFG"), 0, 44412924025649114419413541526870954696667907029239618728289150652715284776538); + verifyAddr(address("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi"), 0, 9407242825041766837311851458322335726136775042891143504070507665010681354284); + verifyAddr(address("EQD-nhrinjv0B4LTgr0dRHTHwH1MOsgGhKBXJZd7vESMZUf1"), 0, 115166810931401616117484448645661180241548402534908005320733783571353775148133); + verifyAddr(address("EQAVD3Fni9I6j8XeSIl-wAGBEhqhame6OtAY0GScKT0D9X6f"), 0, 9525855215156855607080079714361451576383963668563135377495902959388099150837); + verifyAddr(address("EQC6ACq3VANZjqfRBy7JMHkpLwqQ9qyYJsCIGx1mYbQgxaKw"), 0, 84130484652351964071210477536969520113177637645401392541565606610268614566085); + verifyAddr(address("EQCIJLNFIko5CvpKn9oAkrDgLocDOoD4vwmHxNx_fsG_LkwW"), 0, 61579391178099797614367237687950512448308156724136883899001108680249616482094); + verifyAddr(address("EQCe4AYIBce1pAk2qJJPSs1OzyZRlKjkfq8zuC8D7erv6DUP"), 0, 71861245445432818728925844931259040612664802586395398157190478191760507596776); + verifyAddr(address("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"), 0, 78559023162479717496981724991265882229440558807791659796411897368395464230649); + verifyAddr(address("EQBBlraAps0OZaB9Q8ePQn2wVAaL1G411A-dNppyWe3X3GIT"), 0, 29666621803903557832193058147214384979915773445007872807927344851911086823388); + verifyAddr(address("EQBiASqUqaVizrozLRbszkWC2kETbkhpO2qniDVDPPg2_0W8"), 0, 44328719889509369519441680467651025944540360433148852643949783408843779749631); + verifyAddr(address("EQBu2Q1EO8gIoNA1qoGWnHUudKfmqlKEDTQE-DxN-_4sdg14"), 0, 50137910719490808065414827264266674858051167131188257457782826342827836714102); + verifyAddr(address("EQA5bvxWd5-q2vJUVqR9AlbEIfdFysLR0PXGgVlBf8x5hWuF"), 0, 25977927117604457079092522008276392864656238504700352770597256138254994667909); + verifyAddr(address("EQBguMSHjFv5bfoOdshr3ruS9ymSZzhRKMovoNrxGxZXvmee"), 0, 43748489720571123896506696370504498290006245978262404519821633796370658121662); + verifyAddr(address("EQAxL0oF1-zNgimPKthbDnYS4xj94rHtfNRN7_Pd1r2LNNv3"), 0, 22246882279393590648219842750911786376759362211171398419754426796438233910068); + verifyAddr(address("EQANX1uRKGZfyPIwEaIXrR0ZOqadct5q10dvKxWIxx7SQqzW"), 0, 6048549475100840191738010856156544571222758030966479209409932714701987172930); + verifyAddr(address("EQBitdFDoU5DWSjfKq7AsO29RIwAnBzzvcVVSn5ekQoB9Liv"), 0, 44647902768175374073183447303109856895983123510911038181495138821771906122228); + verifyAddr(address("EQBgbux7VSjqJHP7ByRK1q4QuVZbpSCesNgvz5qad3lfXX_B"), 0, 43618018778298854282398238948198420936771670943015013768514626213000552996701); + verifyAddr(address("EQDisBd8U7M3CEOZ8gcWCdetdmJi3AI31zIT5qBwOdmUbsxY"), 0, 102533830955233207294921564956803510155400341370448300698800842506363763004526); + verifyAddr(address("EQAZpn_eynVlf7Ii2d6jP_p1URPrdF9F3S7DiudQyelkjzwE"), 0, 11602000355550451044739442929923326898313570892134000961608306166632391730319); + verifyAddr(address("EQDE0HBgfkOiqHezLtExBGTvOs8eitthHQosBjW3BmDy1y2K"), 0, 89021598108837008984355105304701054698583123510131754065320641619941010764503); + verifyAddr(address("EQDyT36zktBN9PVWvZ1joRxhIfEUgCPt4F2isa-enUA_d6CP"), 0, 109600164736599393471831241268953938618560132398692391517933933264745646800759); + verifyAddr(address("EQDSMUGwt25IQd3_yHjI03F71G8Kp2GMaMEv2TiWoTKbsyRH"), 0, 95072727086440754059372943502908629555499501854161516009430039520728770059187); + verifyAddr(address("EQAgK1EcrvEuL9sCtoj3cNhVNOuf3lo5GIPE2gn1fwZZYB3j"), 0, 14550545393206146289454646242321274637527057595221202748348667645886114191712); + verifyAddr(address("EQCDKqL5w_6MD-Z7AOButu-uR-ZJTsgNU1fu464hn9grY81U"), 0, 59328315557704100696483472039557119625141880163887490602190749720459366378339); + verifyAddr(address("EQB1aVMyFBhnlYXmQjsma0S63kvxKU7ccZKFNCFTwX7ASPv4"), 0, 53106696421104300082516512931084483581353095629408473618166869610568148238408); + verifyAddr(address("EQBbjrXHoxDyh1ZYGBdBoQgLaScxW6pZR1hEhJC8BqF-5Kgq"), 0, 41412616102566803060532874463898939692666425753852274254609049615175463829220); + verifyAddr(address("EQC-QeZ13QP0lszxNKt380fCWuaV94vwC/bfuqmrlg1/fJPA"), 0, 86055876869280374285292827775555707420719385459150221433115419095878595346300); + verifyAddr(address("EQAiUwpF27vXCngqNhf_TQ5E_06ah0G4zuSrnfU7CLLaht5H"), 0, 15525356059048115813946213102829493539706126913595626308144289257869196581510); + verifyAddr(address("EQBqiVjmhe2iVGmgOSDO1FGjSiz_AMtb1w7lLEiP4XIF_SFy"), 0, 48187833566271418625754761625661652107159264793429628379411792200127405491709); + verifyAddr(address("EQDmwvaK2d_SbaPdpOM60effPWeKsksgDVwFPEyxuftM396K"), 0, 104376425077737068747642645125299653296942252727305637929378053253273342397663); + verifyAddr(address("EQDWtPZZgF7wvIMUHZQojuD3utiuivsW7WslRJ33dgv-5yc8"), 0, 97114682311034709685427168495629428400170984047839002197324103884924936519399); + verifyAddr(address("EQAA7z0JI0JKqbN-1uENKz9JrxIO5ZRY-ehMeg9fPncx50Ck"), 0, 422697701361909095759185681783393186844038628935759044330165207027374567911); + verifyAddr(address("EQBVUHRoCq6coQYUwOAhGSoAmQ6Mpm7dFlDYon6HMgWV8Ftr"), 0, 38588743302295548905191533977469452945717219128199196974980570837505276220912); + verifyAddr(address("EQCTdvDCf0bA5dOPI1-44tB2ZfNcMGiklzvg27TovgDEqM6E"), 0, 66700138358140658950710678965721715920748906761125730971082529064117803730088); + verifyAddr(address("EQBDBKE5WGKIlnoi3OOzw7vkKKIX55eWjPvgxJWwek8AyL2J"), 0, 30313140970524770883308749215942283658935592719811899513010665548955593408712); + verifyAddr(address("EQAvCSyLCo21GrqLAifdov4WkOxuGQCjMRxgF1cXSaNzLHZe"), 0, 21274912932379789207153885262858116665851037273450532982121514600400844714796); + verifyAddr(address("EQCsLpDeHB2qpRbmsCb_0xmsYVNx1NgeYrvHGT1TDrHkDgL4"), 0, 77880084760844670718511036557364450189323549135231036747480176919181282894862); + verifyAddr(address("EQCTQ8kPwyX92r48gCIL_pLN_RcQT9ghZygnmDTYkOkuW_j5"), 0, 66609755171046741472463433430016629628609840960137226492652665879987546041947); + verifyAddr(address("EQCTrFRSHt-tfk7WxK9ZHQmqLcgxXxTK7wGfCEbqgY2W9Mcx"), 0, 66794468397542182731534559853537484892417154018190804733043974345563210356468); + verifyAddr(address("EQCv28y49GdaLncoclv0ISdDlMUY_cxDPGNWFCPT8t4GuqUJ"), 0, 79543100951881731989812212377176715376834409392116644269458867858071577560762); + verifyAddr(address("EQCVL-k6deDR56Z8pcb0Btg0lGfaivOGfdDCD1vvyRsyL9vS"), 0, 67479265933941008511790471646564661743401752930295407567346938670637286896175); + verifyAddr(address("EQD6t2dXDjZxF1DqunKF-8dEWivJdliY_0FYiCXnthuqnDCa"), 0, 113402258385556889021060606279033166272577193563727959698596277924908309916316); + verifyAddr(address("EQDE98XNzXiPq7VnbJJ2M4-Ht3tX_OWR0xUTTnDC8NObLmyU"), 0, 89091094739778473356272490822716056624384395256388481953562551087642791090990); + verifyAddr(address("EQDfeRDE1TDhwt478CDR0Q7MDwqcTUhfjqyTT59mgoAaF6f7"), 0, 101079669463449311486034260688909914923832300293253430457119371423825321269783); + verifyAddr(address("EQDijcEyUKa-QgCbeGlggQk1uBtt2ZRHyW4Y4gB4R6MN6RLW"), 0, 102473162609487797404330889623966425536887610061087715571345738626121871855081); + verifyAddr(address("EQDOtFOt41skbjBkZF89oYXpoDECjlxIzD-ShWAOYyzuxqLA"), 0, 93495056812773926196963707371665512785268729004579280701087533371037976424134); + verifyAddr(address("EQDuJKSFWU7AYqH6KLFfAbYvMuz346eWmJvG6_2NYE42_B4T"), 0, 107715199938628393100813870735031557263256555616273999363057194279168054802172); + verifyAddr(address("EQDwGu4vFv1e3wn8min_iy7OPJXegOYTFQ5bZFZ5a5ZPiBpX"), 0, 108602665568837301744601989570019709742180613578164394799026726718721456754568); + verifyAddr(address("EQC4G2ph6AS_mD_-cIv4aIYm1z5jAgCW_TTDEr72ygXOP2X-"), 0, 83274003234732023403481554420859495155084746906198543572711543697320249249343); + verifyAddr(address("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"), 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); + verifyAddr(address("EQDoIA20MF1qEcSPtROdCu5ukGx9dVjgMeJh1oQ4A4cf_Jif"), 0, 104993214557977037193613824776415934089204193426692473563548548423424814817276); + verifyAddr(address("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"), 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); + verifyAddr(address("EQClLO4EnZ_rTyV1GVpWy53pLgWJRki5c4ZzuM_1O_ClBkO9"), 0, 74711004027159342540251007601464500186374346239921204216319145006974068892934); + verifyAddr(address("EQDmkj65Ab_m0aZaW8IpKw4kYqIgITw_HRstYEkVQ6NIYCyW"), 0, 104290347741656803921830951060768893809692975574470790497562993373950614128736); + verifyAddr(address("EQCqNTwAYUNhPFS0RgqZoTLGJcQQxbAJ7csUo4YO3_TONLab"), 0, 76987241268612358571638783428744566580605181728938801022059780105627411729972); + verifyAddr(address("EQCL3DmCynaRK7-vsfeNmd4Jj-UxAIHPvA4qS2xwaL6UpLbF"), 0, 63260589232981964910240894899061676480139492286430589202252472895352724165796); + verifyAddr(address("EQDbU1SVEjBE73oUqgAoM9gDcShUkM5EC2PgoCjuwVUKo-Ee"), 0, 99203745911752606845646497420891218522647962685916739950275357890977532807843); + verifyAddr(address("EQD02VdcF4TDbCKLLhZJ39NQTu6aWq2LnLjp0oXqbNu_BANK"), 0, 110748343802097970709980079967961144373090790244250392237586606542170934198020); + verifyAddr(address("EQBynBO23ywHy_CgarY9NK9FTz0yDsG82PtcbSTQgGoXwiuA"), 0, 51839428943991432793039248316067731096592274748149794482308513726460953499586); + verifyAddr(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(address("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"), 0, 9183547432069678364603018431103042146626948674383548774683927217595824907333); + verifyAddr(address("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"), 0, 45985353862647206060987594732861817093328871106941773337270673759241903247880); + verifyAddr(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(address("kQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPgpP"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(address("kf-Dfdg-YQXaR2Q97gZJ4fGBtmV1DHOU1y1RPyyZZtRy_Ikh"), -1, 59475331506450494976393625198911249698879029820580340449086829444312920781564); + verifyAddr(address("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 0, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(address("0:0000000000000000000000000000000000000000000000000000000000000000"), 0, 0); + verifyAddr(address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"), 0, 115792089237316195423570985008687907853269984665640564039457584007913129639935); + verifyAddr(address("0:0000000000000000000000000000000000000000000000000000000000000000"), 0, 0); + verifyAddr(address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 1, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 9, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(address("99:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 99, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(address("-1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), -1, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - return cc1.bitsEqual(cc2); + return ( + cc1 == cc2, + cc2 != cc1, + (cc1 as slice).bitsEqual(((cc2 as slice) as address) as slice), + createAddressNone() == cc1, + createAddressNone() == createAddressNone(), + check0(address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff")) + ); } /** -@testcase | 0 | | -1 +@testcase | 0 | | -1 0 -1 0 -1 + +@fif_codegen +""" + check0 PROC:<{ // addr + REWRITESTDADDR + DROP // '2 + 0 EQINT // '4 + 111 THROWIFNOT // + }> +""" + +@fif_codegen +""" + codegenAddrEq PROC:<{ // a b + 2DUP // a b a b + SDEQ // a b '2 + IFJMP:<{ // a b + 2DROP // + 1 PUSHINT // '3=1 + }> // a b + SDEQ // '4 + IFNOTJMP:<{ // + 2 PUSHINT // '5=2 + }> // + 3 PUSHINT // '6=3 + }> +""" */ diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index 9feed3a1c2..9fb5e36185 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -6,8 +6,8 @@ get raw_slice(): slice { return stringHexToSlice("abcdef"); } -get addr_slice(): slice { - return stringAddressToSlice("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"); +get addr_slice(): address { + return address("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"); } get string_hex(): int { @@ -51,7 +51,7 @@ fun calcSliceBytes(slice: slice): tuple { fun main() { var s_ascii: slice = ascii_slice(); var s_raw: slice = raw_slice(); - var s_addr: slice = addr_slice(); + var s_addr: address = addr_slice(); var i_hex: int = string_hex(); var i_mini: int = string_minihash(); var i_maxi: int = string_maxihash(); @@ -59,7 +59,7 @@ fun main() { var i_crc16: int = string_crc16(); assert(sdeq(s_ascii, newc().storeUint(0x737472696E67, 12 * 4).endcs())) throw 101; assert(sdeq(s_raw, newc().storeUint(0xABCDEF, 6 * 4).endcs())) throw 102; - assert(sdeq(s_addr, newc().storeUint(4, 3).storeInt(-1, 8) + assert(sdeq(s_addr as slice, newc().storeUint(4, 3).storeInt(-1, 8) .storeUint(0x3333333333333333333333333333333333333333333333333333333333333333, 256).endcs()), 103); assert(i_hex == 0x4142434445464748494A4B4C4D4E4F505152535455565758595A303132333435) throw 104; assert(i_mini == 0x7a62e8a8) throw 105; diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 70944cd4c2..7bed41a56c 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1263,10 +1263,10 @@ void define_builtins() { define_builtin_func("stringToBase256", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); - define_builtin_func("stringAddressToSlice", {TypeDataUnknown::create()}, TypeDataSlice::create(), nullptr, + define_builtin_func("stringHexToSlice", {TypeDataUnknown::create()}, TypeDataSlice::create(), nullptr, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); - define_builtin_func("stringHexToSlice", {TypeDataUnknown::create()}, TypeDataSlice::create(), nullptr, + define_builtin_func("address", {TypeDataUnknown::create()}, TypeDataAddress::create(), nullptr, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index a281cfbe31..a68de6c7c8 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -73,9 +73,9 @@ static bool parse_raw_address(std::string_view acc_string, int& workchain, ton:: int x; if (c >= '0' && c <= '9') { x = c - '0'; - } else if (c >= 'a' && c <= 'z') { + } else if (c >= 'a' && c <= 'f') { x = c - 'a' + 10; - } else if (c >= 'A' && c <= 'Z') { + } else if (c >= 'A' && c <= 'F') { x = c - 'A' + 10; } else { return false; @@ -90,6 +90,23 @@ static bool parse_raw_address(std::string_view acc_string, int& workchain, ton:: return true; } +static void parse_any_std_address(std::string_view str, SrcLocation loc, unsigned char (*data)[3 + 8 + 256]) { + ton::WorkchainId workchain; + ton::StdSmcAddress addr; + bool correct = (str.size() == 48 && parse_friendly_address(str.data(), workchain, addr)) || + (str.size() != 48 && parse_raw_address(str, workchain, addr)); + if (!correct) { + throw ParseError(loc, "invalid standard address"); + } + if (workchain < -128 || workchain >= 128) { + throw ParseError(loc, "anycast addresses not supported"); + } + + td::bitstring::bits_store_long_top(*data, 0, static_cast(4) << (64 - 3), 3); + td::bitstring::bits_store_long_top(*data, 3, static_cast(workchain) << (64 - 8), 8); + td::bitstring::bits_memcpy(*data, 3 + 8, addr.bits().ptr, 0, ton::StdSmcAddress::size()); +} + // internal helper: for `ton("0.05")`, parse string literal "0.05" to 50000000 static td::RefInt256 parse_nanotons_as_floating_string(SrcLocation loc, std::string_view str) { bool is_negative = false; @@ -106,6 +123,7 @@ static td::RefInt256 parse_nanotons_as_floating_string(SrcLocation loc, std::str // parse "0.05" into integer part (before dot) and fractional (after) int64_t integer_part = 0; int64_t fractional_part = 0; + int integer_digits = 0; int fractional_digits = 0; bool seen_dot = false; @@ -119,6 +137,9 @@ static td::RefInt256 parse_nanotons_as_floating_string(SrcLocation loc, std::str } else if (c >= '0' && c <= '9') { if (!seen_dot) { integer_part = integer_part * 10 + (c - '0'); + if (++integer_digits > 9) { + throw ParseError(loc, "argument is too big and leads to overflow"); + } } else if (fractional_digits < 9) { fractional_part = fractional_part * 10 + (c - '0'); fractional_digits++; @@ -161,6 +182,12 @@ static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(Vloc, str); } + if (f_name == "address") { // previously, postfix "..."a, but it returned `slice` (now returns `address`) + unsigned char data[3 + 8 + 256]; // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; + parse_any_std_address(str, v_arg->loc, &data); + return td::BitSlice{data, sizeof(data)}.to_hex(); + } + if (f_name == "stringCrc32") { // previously, postfix "..."c return td::make_refint(td::crc32(td::Slice{str.data(), str.size()})); } @@ -181,25 +208,6 @@ static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(Verror("invalid standard address"); - } - if (workchain < -128 || workchain >= 128) { - v_arg->error("anycast addresses not supported"); - } - - unsigned char data[3 + 8 + 256]; // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; - td::bitstring::bits_store_long_top(data, 0, static_cast(4) << (64 - 3), 3); - td::bitstring::bits_store_long_top(data, 3, static_cast(workchain) << (64 - 8), 8); - td::bitstring::bits_memcpy(data, 3 + 8, addr.bits().ptr, 0, ton::StdSmcAddress::size()); - return td::BitSlice{data, sizeof(data)}.to_hex(); - } - if (f_name == "stringHexToSlice") { // previously, postfix "..."s unsigned char buff[128]; long bits = td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.data(), str.data() + str.size()); diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 370460d3cf..6c376491e5 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -889,6 +889,18 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector process_binary_operator(V v, code.close_pop_cur(v->loc); return transition_to_target_type(std::move(rvect), code, target_type, v); } + if (t == tok_eq || t == tok_neq) { + if (v->get_lhs()->inferred_type->unwrap_alias() == TypeDataAddress::create() && v->get_rhs()->inferred_type->unwrap_alias() == TypeDataAddress::create()) { + FunctionPtr f_sliceEq = lookup_global_symbol("slice.bitsEqual")->try_as(); + std::vector ir_lhs_slice = pre_compile_expr(v->get_lhs(), code); + std::vector ir_rhs_slice = pre_compile_expr(v->get_rhs(), code); + std::vector rvect = code.create_tmp_var(TypeDataBool::create(), v->loc, "(addr-eq)"); + code.emplace_back(v->loc, Op::_Call, rvect, std::vector{ir_lhs_slice[0], ir_rhs_slice[0]}, f_sliceEq); + if (t == tok_neq) { + FunctionPtr not_sym = lookup_global_symbol("!b_")->try_as(); + code.emplace_back(v->loc, Op::_Call, rvect, rvect, not_sym); + } + return transition_to_target_type(std::move(rvect), code, target_type, v); + } + } throw UnexpectedASTNodeKind(v, "process_binary_operator"); } diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index a863fc5929..dc17fa6523 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -155,6 +155,20 @@ static bool expect_boolean(AnyExprV v_inferred) { return expect_boolean(v_inferred->inferred_type); } +static bool expect_address(TypePtr inferred_type) { + if (inferred_type == TypeDataAddress::create()) { + return true; + } + if (const TypeDataAlias* as_alias = inferred_type->try_as()) { + return expect_address(as_alias->underlying_type); + } + return false; +} + +static bool expect_address(AnyExprV v_inferred) { + return expect_address(v_inferred->inferred_type); +} + class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { FunctionPtr cur_f = nullptr; // may be nullptr if checking `const a = ...` init_value @@ -211,7 +225,10 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { bool both_int = expect_integer(lhs) && expect_integer(rhs); bool both_bool = expect_boolean(lhs) && expect_boolean(rhs); if (!both_int && !both_bool) { - if (lhs->inferred_type->equal_to(rhs->inferred_type)) { // compare slice with slice, int? with int? + bool both_address = expect_address(lhs) && expect_address(rhs); + if (both_address) { // address can be compared with ==, but it's not integer comparison, it's handled specially + v->mutate()->assign_fun_ref(nullptr); + } else if (lhs->inferred_type->equal_to(rhs->inferred_type)) { // compare slice with slice, int? with int? fire(cur_f, v->loc, "type " + to_string(lhs) + " can not be compared with `== !=`"); } else { fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); diff --git a/tolk/pipe-optimize-boolean-expr.cpp b/tolk/pipe-optimize-boolean-expr.cpp index 4b05923c2a..ef6c8208cb 100644 --- a/tolk/pipe-optimize-boolean-expr.cpp +++ b/tolk/pipe-optimize-boolean-expr.cpp @@ -145,6 +145,15 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { v_cond_istype->mutate()->assign_is_negated(!v_cond_istype->is_negated); v = createV(v->loc, !v->is_ifnot, v_cond_istype, v->get_if_body(), v->get_else_body()); } + // `if (addr1 != addr2)` -> ifnot(addr1 == addr2) + if (auto v_cond_neq = v->get_cond()->try_as()) { + if (v_cond_neq->tok == tok_neq && v_cond_neq->get_lhs()->inferred_type->unwrap_alias() == TypeDataAddress::create() && v_cond_neq->get_rhs()->inferred_type->unwrap_alias() == TypeDataAddress::create()) { + auto v_cond_eq = createV(v_cond_neq->loc, "==", tok_eq, v_cond_neq->get_lhs(), v_cond_neq->get_rhs()); + v_cond_eq->mutate()->assign_inferred_type(v_cond_neq->inferred_type); + v_cond_eq->mutate()->assign_rvalue_true(); + v = createV(v->loc, !v->is_ifnot, v_cond_eq, v->get_if_body(), v->get_else_body()); + } + } return v; } diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 915c01f72b..78c61d8965 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -107,6 +107,7 @@ static TypePtr try_parse_predefined_type(std::string_view str) { break; case 7: if (str == "builder") return TypeDataBuilder::create(); + if (str == "address") return TypeDataAddress::create(); break; case 8: if (str == "varint16") return TypeDataIntN::create(false, true, 16); diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index b419a2e3f5..58376d63bc 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -135,6 +135,7 @@ TypePtr TypeDataSlice::singleton; TypePtr TypeDataBuilder::singleton; TypePtr TypeDataTuple::singleton; TypePtr TypeDataContinuation::singleton; +TypePtr TypeDataAddress::singleton; TypePtr TypeDataNullLiteral::singleton; TypePtr TypeDataCoins::singleton; TypePtr TypeDataUnknown::singleton; @@ -149,6 +150,7 @@ void type_system_init() { TypeDataBuilder::singleton = new TypeDataBuilder; TypeDataTuple::singleton = new TypeDataTuple; TypeDataContinuation::singleton = new TypeDataContinuation; + TypeDataAddress::singleton = new TypeDataAddress; TypeDataNullLiteral::singleton = new TypeDataNullLiteral; TypeDataCoins::singleton = new TypeDataCoins; TypeDataUnknown::singleton = new TypeDataUnknown; @@ -691,7 +693,7 @@ bool TypeDataSlice::can_rhs_be_assigned(TypePtr rhs) const { if (const TypeDataAlias* rhs_alias = rhs->try_as()) { return can_rhs_be_assigned(rhs_alias->underlying_type); } - return rhs == TypeDataNever::create(); // note, that bytesN is NOT automatically cast to slice without `as` operator + return rhs == TypeDataNever::create(); // note, that bytesN/address is NOT automatically cast to slice without `as` operator } bool TypeDataBuilder::can_rhs_be_assigned(TypePtr rhs) const { @@ -724,6 +726,16 @@ bool TypeDataContinuation::can_rhs_be_assigned(TypePtr rhs) const { return rhs == TypeDataNever::create(); } +bool TypeDataAddress::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } + return rhs == TypeDataNever::create(); // note, that slice is NOT automatically cast to address without `as` operator +} + bool TypeDataNullLiteral::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; @@ -952,6 +964,9 @@ bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to->try_as()) { // `slice` to `bytes32` / `slice` to `bits8` return true; } + if (cast_to == TypeDataAddress::create()) { + return true; + } if (const TypeDataUnion* to_union = cast_to->try_as()) { return can_be_casted_to_union(this, to_union); } @@ -991,6 +1006,19 @@ bool TypeDataContinuation::can_be_casted_with_as_operator(TypePtr cast_to) const return cast_to == this; } +bool TypeDataAddress::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to == TypeDataSlice::create()) { + return true; + } + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + return cast_to == this; +} + bool TypeDataNullLiteral::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataUnion* to_union = cast_to->try_as()) { // `null` to `T?` / `null` to `... | null` return can_be_casted_to_union(this, to_union); @@ -1106,7 +1134,7 @@ bool TypeDataBytesN::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == TypeDataSlice::create(); + return cast_to == TypeDataSlice::create() || cast_to == TypeDataAddress::create(); } bool TypeDataCoins::can_be_casted_with_as_operator(TypePtr cast_to) const { diff --git a/tolk/type-system.h b/tolk/type-system.h index 40b2910154..862a00667f 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -255,6 +255,25 @@ class TypeDataContinuation final : public TypeData { bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; +/* + * `address` is TypeDataAddress — TVM slice under the hood, but since it's a very common use case, + * it's extracted as a separate type (not as a struct with slice field, but just a dedicated type). + */ +class TypeDataAddress final : public TypeData { + TypeDataAddress() : TypeData(0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + int get_type_id() const override { return 8; } + std::string as_human_readable() const override { return "address"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + /* * `null` has TypeDataNullLiteral type. * It can be assigned only to nullable types (`int?`, etc.), to ensure null safety. From 768bad66daf87a33d0f0a65720e981b70b1879c0 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 22 May 2025 22:26:01 +0300 Subject: [PATCH 264/524] [Tolk] Better handle recursive structs and infinity size This will help to create snake structs like > struct Snake { > data: ...; > next: Cell; > } --- tolk-tester/tests/generics-4.tolk | 23 ++++- .../tests/invalid-declaration/err-1585.tolk | 9 +- tolk/pipe-resolve-types.cpp | 72 ++++++++++---- tolk/symtable.cpp | 8 -- tolk/symtable.h | 16 ---- tolk/type-system.cpp | 93 +++++++++++++------ tolk/type-system.h | 72 +++++++------- 7 files changed, 190 insertions(+), 103 deletions(-) diff --git a/tolk-tester/tests/generics-4.tolk b/tolk-tester/tests/generics-4.tolk index 30ce19e352..cfe5714ff1 100644 --- a/tolk-tester/tests/generics-4.tolk +++ b/tolk-tester/tests/generics-4.tolk @@ -146,15 +146,36 @@ fun test7() { return (v1, v2, v3, v4); } +struct FakeGeneric8 { + alwaysInt: int; +} + +struct Snake8 { + next: FakeGeneric8; // it's not a recursive struct, it's okay + next2: FakeGeneric8?; + next3: FakeGeneric8>; +} + +@method_id(108) +fun test8() { + var sn: Snake8 = { + next: { alwaysInt: 10 }, + next2: null, + next3: { alwaysInt: 20 } + }; + return sn; +} + fun main() { } /** @testcase | 103 | | 10 20 30 777 40 40 @testcase | 104 | | (null) (null) (null) -@testcase | 105 | | -1 (null) 0 (null) 133 777 0 123 0 456 132 +@testcase | 105 | | -1 (null) 0 (null) 134 777 0 123 0 456 133 @testcase | 106 | | -1 0 -1 -1 @testcase | 107 | | 1 10 110 117 +@testcase | 108 | | 10 (null) 20 @fif_codegen DECLPROC eqUnusedT @fif_codegen DECLPROC eqUnusedU diff --git a/tolk-tester/tests/invalid-declaration/err-1585.tolk b/tolk-tester/tests/invalid-declaration/err-1585.tolk index 0c1fef3328..5abf000178 100644 --- a/tolk-tester/tests/invalid-declaration/err-1585.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1585.tolk @@ -3,8 +3,13 @@ struct A { b: BAlias } type BAlias = B; +fun test(a: A) { + a.b; + a.b.a; +} + /** @compilation_should_fail -@stderr struct `B` size is infinity due to recursive fields -@stderr struct B { +(error message not stable, sometimes about A, sometimes B, it's okay, they are in a hashmap) +@stderr size is infinity due to recursive fields */ diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 78c61d8965..5a2aa0a4af 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -47,6 +47,9 @@ namespace tolk { * Example: `type A = Ok`, then `Ok` is not ready yet, it's left as TypeDataGenericTypeWithTs. */ +static std::unordered_map visited_structs; +static std::unordered_map visited_aliases; + GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_unknown_type_name(FunctionPtr cur_f, SrcLocation loc, std::string_view text) { if (text == "auto") { @@ -241,7 +244,7 @@ class TypeNodesVisitorResolver { if (alias_ref->is_generic_alias() && !allow_without_type_arguments) { fire_error_generic_type_used_without_T(cur_f, loc, alias_ref->as_human_readable()); } - if (!alias_ref->was_visited_by_resolver()) { + if (!visited_aliases.contains(alias_ref)) { visit_symbol(alias_ref); } return TypeDataAlias::create(alias_ref); @@ -250,7 +253,7 @@ class TypeNodesVisitorResolver { if (struct_ref->is_generic_struct() && !allow_without_type_arguments) { fire_error_generic_type_used_without_T(cur_f, loc, struct_ref->as_human_readable()); } - if (!struct_ref->was_visited_by_resolver()) { + if (!visited_structs.contains(struct_ref)) { visit_symbol(struct_ref); } return TypeDataStruct::create(struct_ref); @@ -348,7 +351,7 @@ class TypeNodesVisitorResolver { static void visit_symbol(AliasDefPtr alias_ref) { static std::vector called_stack; - // prevent recursion like `type A = B; type B = A` + // prevent recursion like `type A = B; type B = A` (we can't create TypeDataAlias without a resolved underlying type) bool contains = std::find(called_stack.begin(), called_stack.end(), alias_ref) != called_stack.end(); if (contains) { throw ParseError(alias_ref->loc, "type `" + alias_ref->name + "` circularly references itself"); @@ -363,35 +366,24 @@ class TypeNodesVisitorResolver { TypeNodesVisitorResolver visitor(nullptr, alias_ref->genericTs, alias_ref->substitutedTs, false); TypePtr underlying_type = visitor.finalize_type_node(alias_ref->underlying_type_node); alias_ref->mutate()->assign_resolved_type(underlying_type); - alias_ref->mutate()->assign_visited_by_resolver(); + visited_aliases.insert({alias_ref, 1}); called_stack.pop_back(); } static void visit_symbol(StructPtr struct_ref) { - static std::vector called_stack; - - // prevent recursion like `struct A { field: A }` - // currently, a struct is a tensor, and recursion always leads to infinite size (`A?` also, it's also on a stack) - // if there would be an annotation to store a struct in a tuple, then it has to be reconsidered - bool contains = std::find(called_stack.begin(), called_stack.end(), struct_ref) != called_stack.end(); - if (contains) { - throw ParseError(struct_ref->loc, "struct `" + struct_ref->name + "` size is infinity due to recursive fields"); - } + visited_structs.insert({struct_ref, 1}); if (auto v_genericsT_list = struct_ref->ast_root->as()->genericsT_list) { const GenericsDeclaration* genericTs = construct_genericTs(nullptr, v_genericsT_list); struct_ref->mutate()->assign_resolved_genericTs(genericTs); } - called_stack.push_back(struct_ref); TypeNodesVisitorResolver visitor(nullptr, struct_ref->genericTs, struct_ref->substitutedTs, false); for (int i = 0; i < struct_ref->get_num_fields(); ++i) { StructFieldPtr field_ref = struct_ref->get_field(i); TypePtr declared_type = visitor.finalize_type_node(field_ref->type_node); field_ref->mutate()->assign_resolved_type(declared_type); } - struct_ref->mutate()->assign_visited_by_resolver(); - called_stack.pop_back(); } static const GenericsDeclaration* construct_genericTs(TypePtr receiver_type, V v_list) { @@ -579,6 +571,46 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { } }; +// prevent recursion like `struct A { field: A }`; +// currently, a struct is a tensor, and recursion always leads to infinite size (`A?` also, it's also on a stack); +// if there is an annotation to store a struct in a tuple, then it has to be reconsidered; +// it's crucial to detect it here; otherwise, get_width_on_stack() will silently face stack overflow +class InfiniteStructSizeDetector { + static TypePtr visit_type_deeply(TypePtr type) { + return type->replace_children_custom([](TypePtr child) { + if (const TypeDataStruct* child_struct = child->try_as()) { + check_struct_for_infinite_size(child_struct->struct_ref); + } + if (const TypeDataAlias* child_alias = child->try_as()) { + return visit_type_deeply(child_alias->underlying_type); + } + return child; + }); + }; + + static void check_struct_for_infinite_size(StructPtr struct_ref) { + static std::vector called_stack; + + bool contains = std::find(called_stack.begin(), called_stack.end(), struct_ref) != called_stack.end(); + if (contains) { + throw ParseError(struct_ref->loc, "struct `" + struct_ref->name + "` size is infinity due to recursive fields"); + } + + called_stack.push_back(struct_ref); + for (StructFieldPtr field_ref : struct_ref->fields) { + visit_type_deeply(field_ref->declared_type); + } + called_stack.pop_back(); + } + +public: + static void detect_and_fire_if_any_struct_is_infinite() { + for (auto [struct_ref, _] : visited_structs) { + check_struct_for_infinite_size(struct_ref); + } + } +}; + void pipeline_resolve_types_and_aliases() { ResolveTypesInsideFunctionVisitor visitor; @@ -600,12 +632,12 @@ void pipeline_resolve_types_and_aliases() { visitor.start_visiting_constant(v_const->const_ref); } else if (auto v_alias = v->try_as()) { - if (!v_alias->alias_ref->was_visited_by_resolver()) { + if (!visited_aliases.contains(v_alias->alias_ref)) { TypeNodesVisitorResolver::visit_symbol(v_alias->alias_ref); } } else if (auto v_struct = v->try_as()) { - if (!v_struct->struct_ref->was_visited_by_resolver()) { + if (!visited_structs.contains(v_struct->struct_ref)) { TypeNodesVisitorResolver::visit_symbol(v_struct->struct_ref); } visitor.start_visiting_struct_fields(v_struct->struct_ref); @@ -613,6 +645,10 @@ void pipeline_resolve_types_and_aliases() { } } + InfiniteStructSizeDetector::detect_and_fire_if_any_struct_is_infinite(); + visited_structs.clear(); + visited_aliases.clear(); + patch_builtins_after_stdlib_loaded(); } diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 1485eae75a..9e5d4ad69a 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -141,10 +141,6 @@ void LocalVarData::assign_inferred_type(TypePtr inferred_type) { this->declared_type = inferred_type; } -void AliasDefData::assign_visited_by_resolver() { - this->flags |= flagVisitedByResolver; -} - void AliasDefData::assign_resolved_genericTs(const GenericsDeclaration* genericTs) { if (this->substitutedTs == nullptr) { this->genericTs = genericTs; @@ -163,10 +159,6 @@ void StructFieldData::assign_default_value(AnyExprV default_value) { this->default_value = default_value; } -void StructData::assign_visited_by_resolver() { - this->flags |= flagVisitedByResolver; -} - void StructData::assign_resolved_genericTs(const GenericsDeclaration* genericTs) { if (this->substitutedTs == nullptr) { this->genericTs = genericTs; diff --git a/tolk/symtable.h b/tolk/symtable.h index e6b2f1ce64..3b8075b64e 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -252,13 +252,8 @@ struct GlobalConstData final : Symbol { }; struct AliasDefData final : Symbol { - enum { - flagVisitedByResolver = 1, - }; - AnyTypeV underlying_type_node; TypePtr underlying_type = nullptr; // = resolved underlying_type_node - int flags = 0; const GenericsDeclaration* genericTs; const GenericsSubstitutions* substitutedTs; @@ -278,10 +273,7 @@ struct AliasDefData final : Symbol { bool is_generic_alias() const { return genericTs != nullptr; } bool is_instantiation_of_generic_alias() const { return substitutedTs != nullptr; } - bool was_visited_by_resolver() const { return flags & flagVisitedByResolver; } - AliasDefData* mutate() const { return const_cast(this); } - void assign_visited_by_resolver(); void assign_resolved_genericTs(const GenericsDeclaration* genericTs); void assign_resolved_type(TypePtr underlying_type); }; @@ -307,12 +299,7 @@ struct StructFieldData final : Symbol { }; struct StructData final : Symbol { - enum { - flagVisitedByResolver = 1, - }; - std::vector fields; - int flags = 0; const GenericsDeclaration* genericTs; const GenericsSubstitutions* substitutedTs; @@ -326,10 +313,7 @@ struct StructData final : Symbol { bool is_generic_struct() const { return genericTs != nullptr; } bool is_instantiation_of_generic_struct() const { return substitutedTs != nullptr; } - bool was_visited_by_resolver() const { return flags & flagVisitedByResolver; } - StructData* mutate() const { return const_cast(this); } - void assign_visited_by_resolver(); void assign_resolved_genericTs(const GenericsDeclaration* genericTs); StructData(std::string name, SrcLocation loc, std::vector&& fields, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, AnyV ast_root) diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index 58376d63bc..cd249a5c37 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -266,11 +266,7 @@ TypePtr TypeDataStruct::create(StructPtr struct_ref) { if (TypePtr existing = hash.get_existing()) { return existing; } - int width_on_stack = 0; // if a struct is generic, it will be incorrect, it's okay - for (StructFieldPtr field_ref : struct_ref->fields) { - width_on_stack += field_ref->declared_type->get_width_on_stack(); - } - return hash.register_unique(new TypeDataStruct(width_on_stack, struct_ref)); + return hash.register_unique(new TypeDataStruct(struct_ref)); } TypePtr TypeDataTensor::create(std::vector&& items) { @@ -283,11 +279,7 @@ TypePtr TypeDataTensor::create(std::vector&& items) { if (TypePtr existing = hash.get_existing()) { return existing; } - int width_on_stack = 0; - for (TypePtr item : items) { - width_on_stack += item->get_width_on_stack(); - } - return hash.register_unique(new TypeDataTensor(hash.children_flags(), width_on_stack, std::move(items))); + return hash.register_unique(new TypeDataTensor(hash.children_flags(), std::move(items))); } TypePtr TypeDataBrackets::create(std::vector&& items) { @@ -351,7 +343,7 @@ TypePtr TypeDataUnion::create(std::vector&& variants) { or_null = variants[variants[0] == TypeDataNullLiteral::create()]; } } - return hash.register_unique(new TypeDataUnion(hash.children_flags(), -999999, or_null, std::move(variants))); + return hash.register_unique(new TypeDataUnion(hash.children_flags(), or_null, std::move(variants))); } // flatten variants and remove duplicates @@ -375,24 +367,10 @@ TypePtr TypeDataUnion::create(std::vector&& variants) { } } - int width_on_stack; - if (or_null && or_null->can_hold_tvm_null_instead()) { - width_on_stack = 1; - } else { - // `T1 | T2 | ...` occupy max(W[i]) + 1 slot for UTag (stores type_id or 0 for null) - int max_child_width = 0; - for (TypePtr i : flat_variants) { - if (i != TypeDataNullLiteral::create()) { // `Empty | () | null` totally should be 1 (0 + 1 for UTag) - max_child_width = std::max(max_child_width, i->get_width_on_stack()); - } - } - width_on_stack = max_child_width + 1; - } - if (flat_variants.size() == 1) { // `int | int` return flat_variants[0]; } - return hash.register_unique(new TypeDataUnion(hash.children_flags(), width_on_stack, or_null, std::move(flat_variants))); + return hash.register_unique(new TypeDataUnion(hash.children_flags(), or_null, std::move(flat_variants))); } TypePtr TypeDataUnion::create_nullable(TypePtr nullable) { @@ -410,6 +388,69 @@ TypePtr TypeDataUnion::create_nullable(TypePtr nullable) { } +// -------------------------------------------- +// get_width_on_stack() +// +// calculate, how many stack slots the type occupies, e.g. `int`=1, `(int,int)`=2, `(int,int)?`=3 +// it's calculated dynamically (not saved at TypeData*::create) to overcome problems with +// - recursive struct mentions (to create TypeDataStruct without knowing width of children) +// - uninitialized generics (that don't make any sense upon being instantiated) +// + +int TypeDataAlias::get_width_on_stack() const { + return underlying_type->get_width_on_stack(); +} + +int TypeDataGenericT::get_width_on_stack() const { + assert(false); + return -99999; +} + +int TypeDataGenericTypeWithTs::get_width_on_stack() const { + assert(false); + return -99999; +} + +int TypeDataStruct::get_width_on_stack() const { + int width_on_stack = 0; + for (StructFieldPtr field_ref : struct_ref->fields) { + width_on_stack += field_ref->declared_type->get_width_on_stack(); + } + return width_on_stack; +} + +int TypeDataTensor::get_width_on_stack() const { + int width_on_stack = 0; + for (TypePtr item : items) { + width_on_stack += item->get_width_on_stack(); + } + return width_on_stack; +} + +int TypeDataUnion::get_width_on_stack() const { + if (or_null && or_null->can_hold_tvm_null_instead()) { + return 1; + } + + // `T1 | T2 | ...` occupy max(W[i]) + 1 slot for UTag (stores type_id or 0 for null) + int max_child_width = 0; + for (TypePtr i : variants) { + if (i != TypeDataNullLiteral::create()) { // `Empty | () | null` totally should be 1 (0 + 1 for UTag) + max_child_width = std::max(max_child_width, i->get_width_on_stack()); + } + } + return max_child_width + 1; +} + +int TypeDataNever::get_width_on_stack() const { + return 0; +} + +int TypeDataVoid::get_width_on_stack() const { + return 0; +} + + // -------------------------------------------- // get_type_id() // diff --git a/tolk/type-system.h b/tolk/type-system.h index 862a00667f..c1f9cbb6db 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -41,8 +41,6 @@ namespace tolk { class TypeData { // bits of flag_mask, to store often-used properties and return them without tree traversing const int flags; - // how many slots on a stack this type occupies (calculated on creation), e.g. `int`=1, `(int,int)`=2, `(int,int)?`=3 - const int width_on_stack; friend class TypeDataHasherForUnique; @@ -53,9 +51,8 @@ class TypeData { flag_contains_type_alias_inside = 1 << 3, }; - explicit TypeData(int flags_with_children, int width_on_stack) - : flags(flags_with_children) - , width_on_stack(width_on_stack) { + explicit TypeData(int flags_with_children) + : flags(flags_with_children) { } static bool equal_to_slow_path(TypePtr lhs, TypePtr rhs); @@ -69,7 +66,10 @@ class TypeData { return dynamic_cast(this); } - int get_width_on_stack() const { return width_on_stack; } + // how many slots on a stack this type occupies, e.g. `int`=1, `(int,int)`=2, `(int,int)?`=3 + virtual int get_width_on_stack() const { + return 1; // most types occupy 1 stack slot (int, cell, slice, etc.) + } bool equal_to(TypePtr rhs) const { return this == rhs || equal_to_slow_path(this, rhs); @@ -108,7 +108,7 @@ class TypeData { */ class TypeDataAlias final : public TypeData { explicit TypeDataAlias(int children_flags, AliasDefPtr alias_ref, TypePtr underlying_type) - : TypeData(children_flags | flag_contains_type_alias_inside, underlying_type->get_width_on_stack()) + : TypeData(children_flags | flag_contains_type_alias_inside) , alias_ref(alias_ref) , underlying_type(underlying_type) {} @@ -118,6 +118,7 @@ class TypeDataAlias final : public TypeData { static TypePtr create(AliasDefPtr alias_ref); + int get_width_on_stack() const override; int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; @@ -129,7 +130,7 @@ class TypeDataAlias final : public TypeData { * `int` is TypeDataInt, representation of TVM int. */ class TypeDataInt final : public TypeData { - TypeDataInt() : TypeData(0, 1) {} + TypeDataInt() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -148,7 +149,7 @@ class TypeDataInt final : public TypeData { * From the type system point of view, int and bool are different, not-autocastable types. */ class TypeDataBool final : public TypeData { - TypeDataBool() : TypeData(0, 1) {} + TypeDataBool() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -166,7 +167,7 @@ class TypeDataBool final : public TypeData { * `cell` is TypeDataCell, representation of TVM cell. */ class TypeDataCell final : public TypeData { - TypeDataCell() : TypeData(0, 1) {} + TypeDataCell() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -184,7 +185,7 @@ class TypeDataCell final : public TypeData { * `slice` is TypeDataSlice, representation of TVM slice. */ class TypeDataSlice final : public TypeData { - TypeDataSlice() : TypeData(0, 1) {} + TypeDataSlice() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -202,7 +203,7 @@ class TypeDataSlice final : public TypeData { * `builder` is TypeDataBuilder, representation of TVM builder. */ class TypeDataBuilder final : public TypeData { - TypeDataBuilder() : TypeData(0, 1) {} + TypeDataBuilder() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -222,7 +223,7 @@ class TypeDataBuilder final : public TypeData { * so getting its element results in TypeDataUnknown (which must be assigned/cast explicitly). */ class TypeDataTuple final : public TypeData { - TypeDataTuple() : TypeData(0, 1) {} + TypeDataTuple() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -241,7 +242,7 @@ class TypeDataTuple final : public TypeData { * It's like "untyped callable", not compatible with other types. */ class TypeDataContinuation final : public TypeData { - TypeDataContinuation() : TypeData(0, 1) {} + TypeDataContinuation() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -260,7 +261,7 @@ class TypeDataContinuation final : public TypeData { * it's extracted as a separate type (not as a struct with slice field, but just a dedicated type). */ class TypeDataAddress final : public TypeData { - TypeDataAddress() : TypeData(0, 1) {} + TypeDataAddress() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -281,7 +282,7 @@ class TypeDataAddress final : public TypeData { * (it's much better for user to see an error here than when he passes this variable somewhere). */ class TypeDataNullLiteral final : public TypeData { - TypeDataNullLiteral() : TypeData(0, 1) {} + TypeDataNullLiteral() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -302,7 +303,7 @@ class TypeDataNullLiteral final : public TypeData { */ class TypeDataFunCallable final : public TypeData { TypeDataFunCallable(int children_flags, std::vector&& params_types, TypePtr return_type) - : TypeData(children_flags, 1) + : TypeData(children_flags) , params_types(std::move(params_types)) , return_type(return_type) {} @@ -329,7 +330,7 @@ class TypeDataFunCallable final : public TypeData { */ class TypeDataGenericT final : public TypeData { explicit TypeDataGenericT(std::string&& nameT) - : TypeData(flag_contains_genericT_inside, -999999) // width undefined until instantiated + : TypeData(flag_contains_genericT_inside) , nameT(std::move(nameT)) {} public: @@ -337,6 +338,7 @@ class TypeDataGenericT final : public TypeData { static TypePtr create(std::string&& nameT); + int get_width_on_stack() const override; int get_type_id() const override; std::string as_human_readable() const override { return nameT; } bool can_rhs_be_assigned(TypePtr rhs) const override; @@ -351,7 +353,7 @@ class TypeDataGenericT final : public TypeData { */ class TypeDataGenericTypeWithTs final : public TypeData { TypeDataGenericTypeWithTs(int children_flags, StructPtr struct_ref, AliasDefPtr alias_ref, std::vector&& type_arguments) - : TypeData(children_flags, -999999) + : TypeData(children_flags) , struct_ref(struct_ref) , alias_ref(alias_ref) , type_arguments(std::move(type_arguments)) {} @@ -365,6 +367,7 @@ class TypeDataGenericTypeWithTs final : public TypeData { int size() const { return static_cast(type_arguments.size()); } + int get_width_on_stack() const override; int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; @@ -380,8 +383,8 @@ class TypeDataGenericTypeWithTs final : public TypeData { * so for `Wrapper` struct_ref points to a generic struct, and for `Wrapper` to an instantiated one. */ class TypeDataStruct final : public TypeData { - TypeDataStruct(int width_on_stack, StructPtr struct_ref) - : TypeData(0, width_on_stack) + explicit TypeDataStruct(StructPtr struct_ref) + : TypeData(0) , struct_ref(struct_ref) {} public: @@ -389,6 +392,7 @@ class TypeDataStruct final : public TypeData { static TypePtr create(StructPtr struct_ref); + int get_width_on_stack() const override; int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; @@ -403,8 +407,8 @@ class TypeDataStruct final : public TypeData { * A tensor can be empty. */ class TypeDataTensor final : public TypeData { - TypeDataTensor(int children_flags, int width_on_stack, std::vector&& items) - : TypeData(children_flags, width_on_stack) + TypeDataTensor(int children_flags, std::vector&& items) + : TypeData(children_flags) , items(std::move(items)) {} public: @@ -414,6 +418,7 @@ class TypeDataTensor final : public TypeData { int size() const { return static_cast(items.size()); } + int get_width_on_stack() const override; int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; @@ -429,7 +434,7 @@ class TypeDataTensor final : public TypeData { */ class TypeDataBrackets final : public TypeData { TypeDataBrackets(int children_flags, std::vector&& items) - : TypeData(children_flags, 1) + : TypeData(children_flags) , items(std::move(items)) {} public: @@ -454,7 +459,7 @@ class TypeDataBrackets final : public TypeData { */ class TypeDataIntN final : public TypeData { TypeDataIntN(bool is_unsigned, bool is_variadic, int n_bits) - : TypeData(0, 1) + : TypeData(0) , is_unsigned(is_unsigned) , is_variadic(is_variadic) , n_bits(n_bits) {} @@ -477,7 +482,7 @@ class TypeDataIntN final : public TypeData { * Example: `var cost = ton("0.05")` has type `coins`. */ class TypeDataCoins final : public TypeData { - TypeDataCoins() : TypeData(0, 1) {} + TypeDataCoins() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -499,7 +504,7 @@ class TypeDataCoins final : public TypeData { */ class TypeDataBytesN final : public TypeData { TypeDataBytesN(bool is_bits, int n_width) - : TypeData(0, 1) + : TypeData(0) , is_bits(is_bits) , n_width(n_width) {} @@ -525,8 +530,8 @@ class TypeDataBytesN final : public TypeData { * - `T1 | T2 | ...` is a tagged union: occupy max(T_i)+1 slots (1 for type_id) */ class TypeDataUnion final : public TypeData { - TypeDataUnion(int children_flags, int width_on_stack, TypePtr or_null, std::vector&& variants) - : TypeData(children_flags, width_on_stack) + TypeDataUnion(int children_flags, TypePtr or_null, std::vector&& variants) + : TypeData(children_flags) , or_null(or_null) , variants(std::move(variants)) {} @@ -565,6 +570,7 @@ class TypeDataUnion final : public TypeData { TypePtr calculate_exact_variant_to_fit_rhs(TypePtr rhs_type) const; bool has_all_variants_of(const TypeDataUnion* rhs_type) const; + int get_width_on_stack() const override; int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; @@ -580,7 +586,7 @@ class TypeDataUnion final : public TypeData { * The only thing available to do with unknown is to cast it: `catch (excNo, arg) { var i = arg as int; }` */ class TypeDataUnknown final : public TypeData { - TypeDataUnknown() : TypeData(flag_contains_unknown_inside, 1) {} + TypeDataUnknown() : TypeData(flag_contains_unknown_inside) {} static TypePtr singleton; friend void type_system_init(); @@ -601,7 +607,7 @@ class TypeDataUnknown final : public TypeData { * Such variables can not be cast to any other types, all their usage will trigger type mismatch errors. */ class TypeDataNever final : public TypeData { - TypeDataNever() : TypeData(0, 0) {} + TypeDataNever() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -609,6 +615,7 @@ class TypeDataNever final : public TypeData { public: static TypePtr create() { return singleton; } + int get_width_on_stack() const override; int get_type_id() const override { return 19; } std::string as_human_readable() const override { return "never"; } bool can_rhs_be_assigned(TypePtr rhs) const override; @@ -622,7 +629,7 @@ class TypeDataNever final : public TypeData { * Empty tensor is not compatible with void, although at IR level they are similar, 0 stack slots. */ class TypeDataVoid final : public TypeData { - TypeDataVoid() : TypeData(0, 0) {} + TypeDataVoid() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -630,6 +637,7 @@ class TypeDataVoid final : public TypeData { public: static TypePtr create() { return singleton; } + int get_width_on_stack() const override; int get_type_id() const override { return 10; } std::string as_human_readable() const override { return "void"; } bool can_rhs_be_assigned(TypePtr rhs) const override; From e4bc23501162e077d11361db318e6ae111e6ed48 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 22 May 2025 23:06:14 +0300 Subject: [PATCH 265/524] [Tolk] Better error messages for unknown method When `a.method()` is unknown, show a better error: - maybe, it's a global function - maybe, it exists for another type --- .../tests/invalid-semantics/err-4520.tolk | 2 +- .../tests/invalid-symbol/err-2188.tolk | 2 +- .../tests/invalid-symbol/err-2218.tolk | 13 ++++++++++ .../tests/invalid-symbol/err-2770.tolk | 14 +++++++++++ tolk-tester/tests/type-aliases-tests.tolk | 25 +++++++++++++++++++ tolk/generics-helpers.cpp | 6 ++++- tolk/pipe-infer-types-and-calls.cpp | 14 ++++++++++- tolk/symtable.cpp | 10 ++++++++ tolk/symtable.h | 1 + 9 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 tolk-tester/tests/invalid-symbol/err-2218.tolk create mode 100644 tolk-tester/tests/invalid-symbol/err-2770.tolk diff --git a/tolk-tester/tests/invalid-semantics/err-4520.tolk b/tolk-tester/tests/invalid-semantics/err-4520.tolk index 994440163b..47b1d2d558 100644 --- a/tolk-tester/tests/invalid-semantics/err-4520.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4520.tolk @@ -6,5 +6,5 @@ fun main() { /** @compilation_should_fail -@stderr method `3` not found for type `int` +@stderr method `3` not found */ diff --git a/tolk-tester/tests/invalid-symbol/err-2188.tolk b/tolk-tester/tests/invalid-symbol/err-2188.tolk index 6511fd40d2..4d0c66d3db 100644 --- a/tolk-tester/tests/invalid-symbol/err-2188.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2188.tolk @@ -9,6 +9,6 @@ fun main() { /** @compilation_should_fail -@stderr method `storeUnexisting` not found for type `builder` +@stderr method `storeUnexisting` not found @stderr .storeUnexisting() */ diff --git a/tolk-tester/tests/invalid-symbol/err-2218.tolk b/tolk-tester/tests/invalid-symbol/err-2218.tolk new file mode 100644 index 0000000000..a9a1e78874 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2218.tolk @@ -0,0 +1,13 @@ +fun demo(x: int) { + +} + +fun main() { + 10.demo(); +} + +/** +@compilation_should_fail +@stderr method `demo` not found, but there is a global function named `demo` +@stderr (a function should be called `foo(arg)`, not `arg.foo()`) + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2770.tolk b/tolk-tester/tests/invalid-symbol/err-2770.tolk new file mode 100644 index 0000000000..06935c93d8 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2770.tolk @@ -0,0 +1,14 @@ + +fun int.demo(self) { + +} + +fun main() { + (10 as int?).demo(); +} + +/** +@compilation_should_fail +@stderr method `demo` not found for type `int?` +@stderr (but it exists for type `int`) + */ diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk index ed0cca11d2..5d725ecf5a 100644 --- a/tolk-tester/tests/type-aliases-tests.tolk +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -1,3 +1,5 @@ +import "@stdlib/tvm-dicts.tolk" + type MIntN = MInt?; type MInt = int; type MInt_v2 = int; @@ -175,6 +177,27 @@ fun test11() { __expect_type(analyzeTensor2(x3), "(MInt, MInt)?"); } +fun dict.getFakeDepth(self) { return 1; } +fun cell.getFakeDepth(self) { return 2; } + +@method_id(112) +fun test12(makeNotNull: bool) { + var d = createEmptyDict(); + if (makeNotNull) { + d.iDictSet(32, 123, ""); + } + var t = createEmptyTuple(); + if (d != null) { + __expect_type(d.getFakeDepth, "(cell) -> int"); + t.push(d.getFakeDepth()); + } else { + __expect_type(d.getFakeDepth, "(dict) -> int"); + t.push(d.getFakeDepth()); + } + t.push(d.getFakeDepth()); + return t; +} + fun main(x: MInt, y: MInt?) { return y == null ? x : x + y; @@ -184,6 +207,8 @@ fun main(x: MInt, y: MInt?) { @testcase | 0 | 3 4 | 7 @testcase | 104 | 1 2 | 3 @testcase | 110 | | 41 +@testcase | 112 | 0 | [ 1 1 ] +@testcase | 112 | -1 | [ 2 1 ] @fif_codegen """ diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index 5c0cc88673..34d2f8f333 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -443,7 +443,11 @@ AliasDefPtr instantiate_generic_alias(AliasDefPtr alias_ref, GenericsSubstitutio } // find `builder.storeInt` for called_receiver = "builder" and called_name = "storeInt" -// most practical case, when a direct method for receiver exists +// most practical case, when a direct method for receiver exists; +// note, that having an alias `type WorkchainNum = int` and methods `WorkchainNum.isMasterchain()`, +// it's okay to call `-1.isMasterchain()`, because int equals to any alias; +// currently there is no chance to change this logic, say, `type AssetList = dict` to have separate methods, +// due to smart casts, types merge of control flow rejoin, etc., which immediately become `cell?` FunctionPtr match_exact_method_for_call_not_generic(TypePtr called_receiver, std::string_view called_name) { FunctionPtr exact_found = nullptr; diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 71149cd4b1..de929d70f8 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -156,6 +156,18 @@ static void fire_error_using_lateinit_variable_uninitialized(FunctionPtr cur_f, fire(cur_f, loc, "using variable `" + static_cast(name) + "` before it's definitely assigned"); } +// fire an error when `obj.f()`, method `f` not found, try to locate a method for another type +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_method_not_found(FunctionPtr cur_f, SrcLocation loc, TypePtr receiver_type, std::string_view method_name) { + if (std::vector other = lookup_methods_with_name(method_name); !other.empty()) { + fire(cur_f, loc, "method `" + to_string(method_name) + "` not found for type " + to_string(receiver_type) + "\n(but it exists for type " + to_string(other.front()->receiver_type) + ")"); + } + if (const Symbol* sym = lookup_global_symbol(method_name); sym && sym->try_as()) { + fire(cur_f, loc, "method `" + to_string(method_name) + "` not found, but there is a global function named `" + to_string(method_name) + "`\n(a function should be called `foo(arg)`, not `arg.foo()`)"); + } + fire(cur_f, loc, "method `" + to_string(method_name) + "` not found"); +} + // helper function: given hint = `Ok | Err` and struct `Ok`, return `Ok` // example: `match (...) { Ok => ... }` we need to deduce `Ok` based on subject static TypePtr try_pick_instantiated_generic_from_hint(TypePtr hint, StructPtr lookup_ref) { @@ -1018,7 +1030,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } } if (out_f_called) { - fire(cur_f, v_ident->loc, "method `" + to_string(v->get_field_name()) + "` not found for type " + to_string(obj_type)); + fire_error_method_not_found(cur_f, v_ident->loc, dot_obj->inferred_type, v->get_field_name()); } else { fire(cur_f, v_ident->loc, "field `" + static_cast(field_name) + "` doesn't exist in type " + to_string(dot_obj)); } diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 9e5d4ad69a..7efccc75ee 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -234,4 +234,14 @@ FunctionPtr lookup_function(std::string_view name) { return G.symtable.lookup(name)->try_as(); } +std::vector lookup_methods_with_name(std::string_view name) { + std::vector result; + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == name) { + result.push_back(method_ref); + } + } + return result; +} + } // namespace tolk diff --git a/tolk/symtable.h b/tolk/symtable.h index 3b8075b64e..ccc8bc5279 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -358,5 +358,6 @@ class GlobalSymbolTable { const Symbol* lookup_global_symbol(std::string_view name); FunctionPtr lookup_function(std::string_view name); +std::vector lookup_methods_with_name(std::string_view name); } // namespace tolk From 49729a1de9089549c30aa21c4cfee8602ef514d7 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 29 May 2025 18:34:46 +0300 Subject: [PATCH 266/524] [Tolk] Fix potential overflow on parsing `obj.index_at` --- tolk-tester/tests/invalid-semantics/err-4208.tolk | 6 ++++++ tolk/pipe-infer-types-and-calls.cpp | 12 +++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 tolk-tester/tests/invalid-semantics/err-4208.tolk diff --git a/tolk-tester/tests/invalid-semantics/err-4208.tolk b/tolk-tester/tests/invalid-semantics/err-4208.tolk new file mode 100644 index 0000000000..c40dda98cb --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4208.tolk @@ -0,0 +1,6 @@ +const flyp = 9999999.77777777777777; + +/** +@compilation_should_fail +@stderr invalid numeric index + */ diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index de929d70f8..8e56e62b6a 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -21,6 +21,7 @@ #include "generics-helpers.h" #include "type-system.h" #include "smart-casts-cfg.h" +#include /* * This is a complicated and crucial part of the pipeline. It simultaneously does the following: @@ -168,6 +169,12 @@ static void fire_error_method_not_found(FunctionPtr cur_f, SrcLocation loc, Type fire(cur_f, loc, "method `" + to_string(method_name) + "` not found"); } +// safe version of std::stoi that does not crash on long numbers +static bool try_parse_string_to_int(std::string_view str, int& out) { + auto result = std::from_chars(str.data(), str.data() + str.size(), out); + return result.ec == std::errc() && result.ptr == str.data() + str.size(); +} + // helper function: given hint = `Ok | Err` and struct `Ok`, return `Ok` // example: `match (...) { Ok => ... }` we need to deduce `Ok` based on subject static TypePtr try_pick_instantiated_generic_from_hint(TypePtr hint, StructPtr lookup_ref) { @@ -970,7 +977,10 @@ class InferTypesAndCallsAndFieldsVisitor final { // check for indexed access (`tensorVar.0` / `tupleVar.1`) if (!fun_ref && field_name[0] >= '0' && field_name[0] <= '9') { - int index_at = std::stoi(std::string(field_name)); + int index_at; + if (!try_parse_string_to_int(field_name, index_at)) { + fire(cur_f, v_ident->loc, "invalid numeric index"); + } if (const auto* t_tensor = obj_type->try_as()) { if (index_at >= t_tensor->size()) { fire(cur_f, v_ident->loc, "invalid tensor index, expected 0.." + std::to_string(t_tensor->items.size() - 1)); From 711ca2f1a47728a5b089f6b5d479fd6e39eb3f74 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sat, 24 May 2025 15:59:53 +0300 Subject: [PATCH 267/524] [Tolk] Correctly handle object literals with shuffled fields > var p: Point = { > y: getY(), > x: getX() > } Now, `getY` will be evaluated first, even though `x` is declared first in a struct. --- tolk-tester/tests/asm-arg-order.tolk | 21 +++++- tolk-tester/tests/cells-slices.tolk | 16 +++-- tolk-tester/tests/strings-tests.tolk | 2 +- tolk-tester/tests/struct-tests.tolk | 104 +++++++++++++++++++++++++++ tolk/pipe-ast-to-legacy.cpp | 83 ++++++++++++++++++++- 5 files changed, 214 insertions(+), 12 deletions(-) diff --git a/tolk-tester/tests/asm-arg-order.tolk b/tolk-tester/tests/asm-arg-order.tolk index 6d55b2da3f..cb7b6f1e30 100644 --- a/tolk-tester/tests/asm-arg-order.tolk +++ b/tolk-tester/tests/asm-arg-order.tolk @@ -180,6 +180,12 @@ fun test33(x: int) { return ((x += 10).plus1TimesB(2), (x += 20).plus1TimesB(x), ((x /= (g2=2)).plus1TimesB(x*g2)), setG2(7).plus1TimesB(g2)); } +@method_id(34) +fun test34() { + var cs = stringHexToSlice("020a"); + return asmAPlus1TimesB(cs.loadUint(8), cs.loadUint(8)); +} + fun main() { } @@ -205,6 +211,7 @@ fun main() { @testcase | 31 | | 70 @testcase | 32 | | 30 @testcase | 33 | 0 | 22 930 480 56 +@testcase | 34 | | 30 @fif_codegen """ @@ -253,5 +260,17 @@ fun main() { }> """ -@code_hash 78671986831403867804966279036762472603849672357801214378328975900111280733054 +@fif_codegen +""" + test34 PROC:<{ + x{020a} PUSHSLICE + 8 LDU + 8 LDU + DROP + SWAP + 1 ADDCONST MUL + }> +""" + +@code_hash 93297578247504549901423325446957614083213743835412767087757830691482809332788 */ diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index e5b89424a6..253ebfff67 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -254,13 +254,15 @@ Note, that since 'compute-asm-ltr' became on be default, chaining methods codege @fif_codegen """ test6 PROC:<{ - 3 PUSHINT // '0=3 - 2 PUSHINT // '0=3 '1=2 - 1 PUSHINT // '0=3 '1=2 '2=1 - NEWC // '0=3 '1=2 '2=1 '3 - 32 STU // '0=3 '1=2 '3 - 32 STU // '0=3 '3 - 32 STU // '3 + 1 PUSHINT // '0=1 + NEWC // '0=1 '1 + 32 STU // '1 + 2 PUSHINT // '1 '4=2 + SWAP // '4=2 '1 + 32 STU // '1 + 3 PUSHINT // '1 '7=3 + SWAP // '7=3 '1 + 32 STU // '1 }> """ */ diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index 9fb5e36185..287dc0df3c 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -78,5 +78,5 @@ fun test1() { @testcase | 0 | | 0 @testcase | 101 | | [ 65 66 67 68 ] -@code_hash 55974318379341089957961227475446008591490555692181953973486962465702042912657 +@code_hash 58447050269190289721084012139099481162782788646785441106022886746601529758643 */ diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index a484475a21..4bd407f59e 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -487,6 +487,81 @@ fun test34() { return o1; } +@pure fun getXPure() { return 1; } +@pure fun getYPure() { return 2; } +global t_impure: tuple; +fun getXImpure() { t_impure.push(1); return 1; } +fun getYImpure() { t_impure.push(2); return 2; } + +@method_id(135) +fun test35() { + var p: Point = { + y: getYPure(), + x: getXPure(), + }; + return p; +} + +@method_id(136) +fun test36() { + t_impure = createEmptyTuple(); + var p: Point = { + y: getYImpure(), + x: getXImpure(), + }; + return (p, t_impure); +} + +@method_id(137) +fun test37() { + var num = 0; + var p: Point = { + y: num += 5, + x: (num *= 10) - 2, + }; + return (p, num); +} + +@method_id(138) +fun test38(num: int) { + var p: Point = { + y: num += 5, + x: (num *= 10) - 2, + }; + return (p, num); +} + +struct TwoPoints { + p1: Point; + p2: Point; +} + +@method_id(139) +fun test39(): (TwoPoints, int) { + var cs = stringHexToSlice("0102030405"); + return ({ + p2: { y: cs.loadUint(8), x: cs.loadUint(8) }, + p1: { x: cs.loadUint(8), y: cs.loadUint(8) }, + }, cs.remainingBitsCount()); +} + +@method_id(140) +fun test40() { + var cs = stringHexToSlice("0102030405"); + return TwoPoints { + p1: { y: cs.loadUint(8), x: cs.loadUint(8) }, + p2: { y: cs.loadUint(8), x: cs.loadUint(8) }, + } +} + +@method_id(141) +fun test41(rev: bool) { + var cs = stringHexToSlice("0102030405"); + return TwoPoints { + p2: rev ? { y: cs.loadUint(8), x: cs.loadUint(8) } : { x: cs.loadUint(8), y: cs.loadUint(8) }, + p1: rev ? { y: cs.loadUint(8), x: cs.loadUint(8) } : { x: cs.loadUint(8), y: cs.loadUint(8) }, + } +} fun main(x: int8, y: MInt) { __expect_type(PointAlias{x,y}, "Point"); @@ -539,6 +614,14 @@ type PointAlias = Point; @testcase | 132 | | 10 20 10 0 0 20 0 0 0 0 @testcase | 133 | | -1 0 5 (null) (null) (null) 0 0 46 @testcase | 134 | | 10 20 +@testcase | 135 | | 1 2 +@testcase | 136 | | 1 2 [ 2 1 ] +@testcase | 137 | | 48 5 50 +@testcase | 138 | 3 | 78 8 80 +@testcase | 139 | | 3 4 2 1 8 +@testcase | 140 | | 2 1 4 3 +@testcase | 141 | -1 | 4 3 2 1 +@testcase | 141 | 0 | 3 4 1 2 @fif_codegen """ @@ -571,4 +654,25 @@ type PointAlias = Point; }> """ +@fif_codegen +""" + test35 PROC:<{ // + getXPure CALLDICT // '2 + getYPure CALLDICT // p.x p.y + }> +""" + +@fif_codegen +""" + test36 PROC:<{ // + NIL // '0 + t_impure SETGLOB // + getYImpure CALLDICT // '4 + getXImpure CALLDICT // p.y p.x + t_impure GETGLOB // p.y p.x g_t_impure + s1 s2 XCHG // p.x p.y g_t_impure + }> +""" + + */ diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 6c376491e5..31bccca150 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -281,14 +281,15 @@ class LValContext { // the purpose of this class is having a call `f(a1,a2,...)` when f has asm arg_order, to check // whether it's safe to rearrange arguments (to evaluate them in arg_order right here for fewer stack manipulations) // or it's unsafe, and we should evaluate them left-to-right; -// example: `f(1,2,3)` / `b.storeUint(2,32)` is safe -// example: `f(x,x+=5,x)` / `f(impureF1(), global_var)` is unsafe +// example: `f(1,2,3)` / `b.storeUint(2,32)` is safe; +// example: `f(x,x+=5,x)` / `f(impureF1(), global_var)` / `f(s.loadInt(), s.loadInt())` is unsafe; +// the same rules are used to check an object literal: is it safe to convert `{y:expr, x:expr}` to declaration order {x,y} class CheckReorderingForAsmArgOrderIsSafeVisitor final : public ASTVisitorFunctionBody { bool has_side_effects = false; protected: void visit(V v) override { - has_side_effects |= v->fun_maybe == nullptr || !v->fun_maybe->is_marked_as_pure(); + has_side_effects |= v->fun_maybe == nullptr || !v->fun_maybe->is_marked_as_pure() || v->fun_maybe->has_mutate_params(); parent::visit(v); } @@ -323,6 +324,14 @@ class CheckReorderingForAsmArgOrderIsSafeVisitor final : public ASTVisitorFuncti } return !visitor.has_side_effects; } + + static bool is_safe_to_reorder(V v) { + CheckReorderingForAsmArgOrderIsSafeVisitor visitor; + for (int i = 0; i < v->get_num_fields(); ++i) { + visitor.ASTVisitorFunctionBody::visit(v->get_field(i)->get_init_val()); + } + return !visitor.has_side_effects; + } }; // given `{some_expr}!`, return some_expr @@ -1425,10 +1434,78 @@ static std::vector process_typed_tuple(V v, CodeBl return transition_to_target_type(std::move(left), code, target_type, v); } +static std::vector process_object_literal_shuffled(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + // creating an object like `Point { y: getY(), x: getX() }`, where fields order doesn't match declaration; + // as opposed to a non-shuffled version `{x:..., y:...}`, we should at first evaluate fields as they created, + // and then to place them in a correct order + std::vector tensor_items; // create a tensor of literal fields values + std::vector target_types; + tensor_items.reserve(v->get_body()->get_num_fields()); + target_types.reserve(v->get_body()->get_num_fields()); + for (int i = 0; i < v->get_body()->get_num_fields(); ++i) { + auto v_field = v->get_body()->get_field(i); + StructFieldPtr field_ref = v->struct_ref->find_field(v_field->get_field_name()); + tensor_items.push_back(v_field->get_init_val()); + target_types.push_back(field_ref->declared_type); + } + const auto* tensor_target_type = TypeDataTensor::create(std::move(target_types))->try_as(); + std::vector literal_rvect = pre_compile_tensor(code, tensor_items, lval_ctx, tensor_target_type); + + std::vector rvect = code.create_tmp_var(TypeDataStruct::create(v->struct_ref), v->loc, "(object)"); + int stack_offset = 0; + for (StructFieldPtr field_ref : v->struct_ref->fields) { + int stack_width = field_ref->declared_type->get_width_on_stack(); + std::vector field_rvect(rvect.begin() + stack_offset, rvect.begin() + stack_offset + stack_width); + stack_offset += stack_width; + + int tensor_offset = 0; + bool exists_in_literal = false; + for (int i = 0; i < v->get_body()->get_num_fields(); ++i) { + auto v_field = v->get_body()->get_field(i); + int tensor_item_width = v_field->field_ref->declared_type->get_width_on_stack(); + if (v_field->get_field_name() == field_ref->name) { + exists_in_literal = true; + std::vector literal_field_rvect(literal_rvect.begin() + tensor_offset, literal_rvect.begin() + tensor_offset + tensor_item_width); + code.emplace_back(v->loc, Op::_Let, std::move(field_rvect), std::move(literal_field_rvect)); + break; + } + tensor_offset += tensor_item_width; + } + if (exists_in_literal || field_ref->declared_type == TypeDataNever::create()) { + continue; + } + + tolk_assert(field_ref->has_default_value()); + SrcLocation last_loc = v->get_body()->empty() ? v->loc : v->get_body()->get_all_fields().back()->loc; + ASTAuxData *aux_data = new AuxData_ForceFiftLocation(last_loc); + auto v_force_loc = createV(v->loc, field_ref->default_value, aux_data, field_ref->declared_type); + std::vector def_rvect = pre_compile_expr(v_force_loc, code, field_ref->declared_type); + code.emplace_back(v->loc, Op::_Let, std::move(field_rvect), std::move(def_rvect)); + } + + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + static std::vector process_object_literal(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { // an object (an instance of a struct) is actually a tensor at low-level // for example, `struct User { id: int; name: slice; }` occupies 2 slots // fields of a tensor are placed in order of declaration (in a literal they might be shuffled) + bool are_fields_shuffled = false; + for (int i = 1; i < v->get_body()->get_num_fields(); ++i) { + StructFieldPtr field_ref = v->struct_ref->find_field(v->get_body()->get_field(i)->get_field_name()); + StructFieldPtr prev_field_ref = v->struct_ref->find_field(v->get_body()->get_field(i - 1)->get_field_name()); + are_fields_shuffled |= prev_field_ref->field_idx > field_ref->field_idx; + } + + // if fields are created {y,x} (not {x,y}), maybe, it's nevertheless safe to evaluate them as {x,y}; + // for example, if they are just constants, calls to pure non-mutating functions, etc.; + // generally, rules of "can we evaluate {x,y} instead of {y,x}" follows the same logic + // as passing of calling `f(x,y)` with asm arg_order, is it safe to avoid SWAP + if (are_fields_shuffled && !CheckReorderingForAsmArgOrderIsSafeVisitor::is_safe_to_reorder(v->get_body())) { + // okay, we have `{y: getY(), x: getX()}` / `{y: v += 1, x: v}`, evaluate them in created order + return process_object_literal_shuffled(v, code, target_type, lval_ctx); + } + SrcLocation prev_loc = v->loc; std::vector tensor_items; std::vector target_types; From 865ee8b31fed3977d1c1fd403bec287e158f8e33 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Wed, 28 May 2025 14:46:54 +0300 Subject: [PATCH 268/524] [Tolk] Auto-packing to/from cells/builders/slices A fully working auto-serialization is made within a big single commit. All types are supported. fromCell, toCell, loadAny, storeAny, etc. Typed cells, overflow detection, error messages, postfixes and opcodes, and more. --- crypto/smartcont/tolk-stdlib/common.tolk | 140 ++- .../smartcont/tolk-stdlib/gas-payments.tolk | 2 +- tolk-tester/tests/annotations-tests.tolk | 31 + .../tests/invalid-declaration/err-1795.tolk | 13 + .../tests/invalid-serialization/err-7008.tolk | 16 + .../tests/invalid-serialization/err-7114.tolk | 22 + .../tests/invalid-serialization/err-7215.tolk | 23 + .../tests/invalid-serialization/err-7419.tolk | 24 + .../tests/invalid-serialization/err-7514.tolk | 47 + .../tests/invalid-serialization/err-7563.tolk | 20 + .../tests/invalid-serialization/err-7564.tolk | 16 + .../tests/invalid-serialization/err-7810.tolk | 20 + .../tests/invalid-serialization/err-7811.tolk | 19 + .../tests/invalid-serialization/err-7812.tolk | 20 + .../tests/invalid-serialization/err-7813.tolk | 13 + .../tests/invalid-serialization/err-7911.tolk | 22 + tolk-tester/tests/pack-unpack-1.tolk | 130 +++ tolk-tester/tests/pack-unpack-2.tolk | 682 +++++++++++ tolk-tester/tests/pack-unpack-3.tolk | 284 +++++ tolk-tester/tests/pack-unpack-4.tolk | 162 +++ tolk-tester/tests/pack-unpack-5.tolk | 183 +++ tolk-tester/tests/some-tests-2.tolk | 16 + tolk/CMakeLists.txt | 3 + tolk/abscode.cpp | 11 + tolk/ast-from-tokens.cpp | 101 +- tolk/ast-replicator.h | 2 + tolk/ast-stringifier.h | 12 +- tolk/ast.cpp | 6 + tolk/ast.h | 21 +- tolk/builtins.cpp | 108 +- tolk/constant-evaluator.cpp | 4 +- tolk/pack-unpack-api.cpp | 257 ++++ tolk/pack-unpack-api.h | 35 + tolk/pack-unpack-serializers.cpp | 1031 +++++++++++++++++ tolk/pack-unpack-serializers.h | 137 +++ tolk/pipe-ast-to-legacy.cpp | 127 +- tolk/pipe-check-inferred-types.cpp | 4 + tolk/pipe-check-serialized-fields.cpp | 115 ++ tolk/pipe-constant-folding.cpp | 8 +- tolk/pipe-infer-types-and-calls.cpp | 14 +- tolk/pipe-optimize-boolean-expr.cpp | 4 +- tolk/pipe-register-symbols.cpp | 21 +- tolk/pipeline.h | 1 + tolk/symtable.cpp | 24 + tolk/symtable.h | 33 +- tolk/tolk.cpp | 1 + tolk/tolk.h | 1 + tolk/type-system.cpp | 11 + 48 files changed, 3900 insertions(+), 97 deletions(-) create mode 100644 tolk-tester/tests/annotations-tests.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1795.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7008.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7114.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7215.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7419.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7514.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7563.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7564.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7810.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7811.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7812.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7813.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7911.tolk create mode 100644 tolk-tester/tests/pack-unpack-1.tolk create mode 100644 tolk-tester/tests/pack-unpack-2.tolk create mode 100644 tolk-tester/tests/pack-unpack-3.tolk create mode 100644 tolk-tester/tests/pack-unpack-4.tolk create mode 100644 tolk-tester/tests/pack-unpack-5.tolk create mode 100644 tolk/pack-unpack-api.cpp create mode 100644 tolk/pack-unpack-api.h create mode 100644 tolk/pack-unpack-serializers.cpp create mode 100644 tolk/pack-unpack-serializers.h create mode 100644 tolk/pipe-check-serialized-fields.cpp diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 642337df5e..db3dfc0ff2 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -55,6 +55,11 @@ fun tuple.size(self): int fun tuple.last(self): T asm "LAST"; +/// Pops and returns the last element of a non-empty tuple. +@pure +fun tuple.pop(mutate self): T + asm "TPOP"; + /** Mathematical primitives. @@ -210,6 +215,139 @@ fun commitContractDataAndActions(): void asm "COMMIT"; +/** + Auto packing structures to/from cells. +*/ + +/// Convert anything to a cell (most likely, you'll call it for structures). +/// Example: +/// ``` +/// var st: MyStorage = { ... }; +/// contract.setData(st.toCell()); +/// ``` +/// Internally, a builder is created, all fields are serialized one by one, and a builder is flushed +/// (beginCell() + serialize fields + endCell()). +@pure +fun T.toCell(self): Cell + builtin; + +/// Parse anything from a cell (most likely, you'll call it for structures). +/// Example: +/// ``` +/// var st = MyStorage.fromCell(contract.getData()); +/// ``` +/// Internally, a cell is unpacked to a slice, and that slice is parsed +/// (packedCell.beginParse() + read from slice). +@pure +fun T.fromCell(packedCell: cell): T + builtin; + +/// Parse anything from a slice (most likely, you'll call it for structures). +/// Example: +/// ``` +/// var msg = CounterIncrement.fromSlice(cs); +/// ``` +/// All fields are read from a slice immediately. +/// If a slice is corrupted, an exception is thrown (most likely, excode 9 "cell underflow"). +/// Note, that a passed slice is NOT mutated, its internal pointer is NOT shifted. +/// If you need to mutate it, like `cs.loadInt()`, consider calling `cs.loadAny()`. +@pure +fun T.fromSlice(rawSlice: slice): T + builtin; + +/// Parse anything from a slice, shifting its internal pointer. +/// Similar to `slice.loadUint()` and others, but allows loading structures. +/// Example: +/// ``` +/// var st: MyStorage = cs.loadAny(); // or cs.loadAny() +/// ``` +/// Similar to `MyStorage.fromSlice(cs)`, but called as a slice method and mutates the slice. +@pure +fun slice.loadAny(mutate self): T + builtin; + +/// Skip anything in a slice, shifting its internal pointer. +/// Similar to `slice.skipBits()` and others, but allows skipping structures. +/// Example: +/// ``` +/// struct TwoInts { a: int32; b: int32; } +/// cs.skipAny(); // skips 64 bits +/// ``` +@pure +fun slice.skipAny(mutate self): self + builtin; + +/// Store anything to a builder. +/// Similar to `builder.storeUint()` and others, but allows storing structures. +/// Example: +/// ``` +/// var b = beginCell().storeUint(32).storeAny(msgBody).endCell(); +/// ``` +@pure +fun builder.storeAny(mutate self, v: T): self + builtin; + +/// Cell represents a typed cell reference (as opposed to untyped `cell`). +/// Example: +/// ``` +/// struct ExtraData { ... } +/// +/// struct MyStorage { +/// ... +/// extra: Cell; // TL-B `^ExtraData` +/// optional: Cell?; // TL-B `(Maybe ^ExtraData)` +/// code: cell; // TL-B `^Cell` +/// data: cell?; // TL-B `(Maybe ^Cell)` +/// } +/// ``` +/// Note, that `st = MyStorage.fromSlice(s)` does NOT deep-load any refs; `st.extra` is `Cell`, not `T`; +/// you should manually call `st.extra.load()` to get T (ExtraData in this example). +struct Cell { + tvmCell: cell; +} + +/// Parse data from already loaded cell reference. +/// Example: +/// ``` +/// struct MyStorage { ... extra: Cell; } +/// +/// var st = MyStorage.fromCell(contract.getData()); +/// // st.extra is cell; if we need to unpack it, we do +/// var extra = st.extra.load(); // it's ExtraData, unpacked from loaded ref +/// ``` +@pure +fun Cell.load(self): T + builtin; + +/// Converts a typed cell into a slice. +@pure +fun Cell.beginParse(self): slice + asm "CTOS"; + +/// RemainingBitsAndRefs is a special built-in type to get "all the rest" slice tail on reading. +/// Example: +/// ``` +/// struct JettonMessage { +/// ... some fields +/// forwardPayload: RemainingBitsAndRefs; +/// } +/// ``` +/// When you deserialize JettonMessage, forwardPayload contains "everything left after reading fields above". +type RemainingBitsAndRefs = slice; + +/// Creates a cell with zero bits and references. +/// Equivalent to `beginCell().endCell()` but cheaper. +@pure +fun createEmptyCell(): cell + asm " PUSHREF"; + +/// Creates a slice with zero remaining bits and references. +/// Equivalent to `beginCell().endCell().beginParse()` but cheaper. +@pure +fun createEmptySlice(): slice + asm "x{} PUSHSLICE"; + + /** Signature checks, hashing, cryptography. */ @@ -421,7 +559,7 @@ fun stringHexToSlice(constStringBytesHex: slice): slice fun cell.beginParse(self): slice asm "CTOS"; -/// Checks if slice is empty. If not, throws an exception. +/// Checks if slice is empty. If not, throws an exception with code 9. fun slice.assertEnd(self): void asm "ENDS"; diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 2c35faf930..89366cf4ff 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -6,7 +6,7 @@ tolk 0.12 */ /// Returns amount of gas (in gas units) consumed in current Computation Phase. -fun getGasConsumedAtTheMoment(): coins +fun getGasConsumedAtTheMoment(): int asm "GASCONSUMED"; /// This function is required to be called when you process an external message (from an outer world) diff --git a/tolk-tester/tests/annotations-tests.tolk b/tolk-tester/tests/annotations-tests.tolk new file mode 100644 index 0000000000..730b0ee0bd --- /dev/null +++ b/tolk-tester/tests/annotations-tests.tolk @@ -0,0 +1,31 @@ +@deprecated("anything") +@custom("anything") +global a: int; + +@deprecated +@custom +const ASDF = 1; + +@custom("props", {allowed: false, ids: [1,2,3]}) +@deprecated +fun f() {} + +@custom({ + `type`: 123, + value: 19 +}) +@custom("another", 12, "annotation") +struct TTT {} + +@deprecated +type MyMsg = int; + +@custom(1,2,3,4) +@custom({function: "main", contract: self}) +fun main() { + return 0; +} + +/** +@testcase | 0 | | 0 + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1795.tolk b/tolk-tester/tests/invalid-declaration/err-1795.tolk new file mode 100644 index 0000000000..67b062cbe7 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1795.tolk @@ -0,0 +1,13 @@ +@overflow1023_policy("ignorje") +struct A { + d1: bits256; + d2: bits256; + d3: bits256; + d4: bits256; + d5: bits256; +} + +/** +@compilation_should_fail +@stderr incorrect value for @overflow1023_policy + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7008.tolk b/tolk-tester/tests/invalid-serialization/err-7008.tolk new file mode 100644 index 0000000000..9203ea228e --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7008.tolk @@ -0,0 +1,16 @@ +struct Point { + x: int; + y: int; +} + +fun main(p: Point) { + p.toCell(); +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `Point` +@stderr because field `Point.x` of type `int` can't be serialized +@stderr because type `int` is not serializable, it doesn't define binary width +@stderr hint: replace `int` with `int32` / `uint64` / `coins` / etc. + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7114.tolk b/tolk-tester/tests/invalid-serialization/err-7114.tolk new file mode 100644 index 0000000000..fe228d6019 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7114.tolk @@ -0,0 +1,22 @@ +type NotSerializableTensor = (int8, slice); + +struct Demo { + a: int8; + b: int8 | int16; + c: NotSerializableTensor; +} + +fun main(p: Demo?) { + p.toCell(); +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `Demo?` +@stderr because field `Demo.c` of type `NotSerializableTensor` can't be serialized +@stderr because alias `NotSerializableTensor` expands to `(int8, slice)` +@stderr because element `tensor.1` of type `slice` can't be serialized +@stderr because type `slice` is not serializable, it doesn't define binary width +@stderr hint: replace `slice` with `address` if it's an address, actually +@stderr hint: replace `slice` with `bits128` and similar if it represents fixed-width data without refs + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7215.tolk b/tolk-tester/tests/invalid-serialization/err-7215.tolk new file mode 100644 index 0000000000..63b46affc6 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7215.tolk @@ -0,0 +1,23 @@ +type MInt = int; + +struct CantBe { + a: int8; + b: MInt?; +} + +struct Container { + item: T; +} + +fun main(s: slice) { + Container.fromSlice(s); +} + +/** +@compilation_should_fail +@stderr auto-serialization via fromSlice() is not available for type `Container` +@stderr because field `Container.item` of type `CantBe` can't be serialized +@stderr because field `CantBe.b` of type `MInt?` can't be serialized +@stderr because alias `MInt` expands to `int` +@stderr because type `int` is not serializable + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7419.tolk b/tolk-tester/tests/invalid-serialization/err-7419.tolk new file mode 100644 index 0000000000..4133c80a05 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7419.tolk @@ -0,0 +1,24 @@ +struct(0x1) Ok1 {} +struct(0x2) Ok2 {} +struct(0x3) Ok3 {} +struct NoPrefix {} + +type MsgOk1 = Ok1 | Ok2 | Ok3; + +type MsgCantBe = Ok1 | Ok2 | NoPrefix | Ok3; + +fun main() { + MsgOk1.fromCell(beginCell().endCell()); + + MsgCantBe.fromCell(beginCell().endCell()); +} + +/** +@compilation_should_fail +@stderr auto-serialization via fromCell() is not available for type `MsgCantBe` +@stderr because alias `MsgCantBe` expands to `Ok1 | Ok2 | NoPrefix | Ok3` +@stderr because could not automatically generate serialization prefixes for a union +@stderr because struct `Ok3` has opcode, but `NoPrefix` does not +@stderr hint: manually specify opcodes to all structures +@stderr err-7419.tolk:13 + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7514.tolk b/tolk-tester/tests/invalid-serialization/err-7514.tolk new file mode 100644 index 0000000000..11728a6c7b --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7514.tolk @@ -0,0 +1,47 @@ +struct In4 { + b: builder; +} + +struct In3 { + i: In4; +} + +struct In2 { + i: In3?; +} + +struct In1 { + i: (In2, slice); +} + +struct MaybeNothing {} +struct MaybeJust { value: T } +type Maybe = MaybeNothing | MaybeJust; + +struct CantBe { + a: address; + b: address?; + i: Maybe; +} + +fun main(c: cell) { + var d: CantBe = CantBe.fromCell(c); +} + +/** +@compilation_should_fail +@stderr auto-serialization via fromCell() is not available for type `CantBe` +@stderr because field `CantBe.i` of type `Maybe` can't be serialized +@stderr because alias `Maybe` expands to `MaybeNothing | MaybeJust` +@stderr because variant #2 of type `MaybeJust` can't be serialized +@stderr because field `MaybeJust.value` of type `In1` can't be serialized +@stderr because field `In1.i` of type `(In2, slice)` can't be serialized +@stderr because element `tensor.0` of type `In2` can't be serialized +@stderr because field `In2.i` of type `In3?` can't be serialized +@stderr because field `In3.i` of type `In4` can't be serialized +@stderr because field `In4.b` of type `builder` can't be serialized +@stderr because type `builder` can not be used for reading, only for writing +@stderr hint: use `bitsN` or `RemainingBitsAndRefs` for reading +@stderr hint: using generics, you can substitute `builder` for writing and something other for reading +@stderr CantBe.fromCell(c) + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7563.tolk b/tolk-tester/tests/invalid-serialization/err-7563.tolk new file mode 100644 index 0000000000..cc83a0cd59 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7563.tolk @@ -0,0 +1,20 @@ +struct A { + d1: bits256; + d2: bits256; + d3: bits256; + d4: bits256; + d5: bits256; +} + +struct B { + a: Cell; +} + +fun main(b: B) { + b.toCell(); +} + +/** +@compilation_should_fail +@stderr struct `A` can exceed 1023 bits in serialization (estimated size: 1280..1280 bits) + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7564.tolk b/tolk-tester/tests/invalid-serialization/err-7564.tolk new file mode 100644 index 0000000000..36eb63d4fa --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7564.tolk @@ -0,0 +1,16 @@ +struct A { + d1: bits256; + d2: bits256; + d3: bits256; + d4: bits256; + d5: bits256; +} + +fun main(o: A | int32) { + o.toCell(); +} + +/** +@compilation_should_fail +@stderr struct `A` can exceed 1023 bits in serialization (estimated size: 1280..1280 bits) + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7810.tolk b/tolk-tester/tests/invalid-serialization/err-7810.tolk new file mode 100644 index 0000000000..2cf0d32e54 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7810.tolk @@ -0,0 +1,20 @@ +struct(0b01) B {} +struct(0b00) C {} + +struct A { + multiple: B | C | int32; +} + +fun main() { + var a: A = { multiple: B{} }; + return a.toCell(); +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `A` +@stderr because field `A.multiple` of type `B | C | int32` can't be serialized +@stderr because could not automatically generate serialization prefixes for a union +@stderr because of mixing primitives and struct `C` with serialization prefix +@stderr hint: extract primitives to single-field structs and provide prefixes + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7811.tolk b/tolk-tester/tests/invalid-serialization/err-7811.tolk new file mode 100644 index 0000000000..5dbcab7e6b --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7811.tolk @@ -0,0 +1,19 @@ +struct(0b01) B {} +struct(0b00) C {} + +struct A { + multiple: B | C | null; +} + +fun main() { + var a: A = { multiple: B{} }; + return a.toCell(); +} + +/** +@compilation_should_fail +@stderr error: auto-serialization via toCell() is not available for type `A` +@stderr because field `A.multiple` of type `B | C | null` can't be serialized +@stderr because could not automatically generate serialization prefixes for a union +@stderr because of mixing primitives and struct `C` with serialization prefix + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7812.tolk b/tolk-tester/tests/invalid-serialization/err-7812.tolk new file mode 100644 index 0000000000..0b69956330 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7812.tolk @@ -0,0 +1,20 @@ +struct(0b000) B {} +struct C {} + +struct A { + multiple: (int32, B | C); +} + +fun main() { + var a: A = { multiple: (5, B{}) }; + return a.toCell(); +} + +/** +@compilation_should_fail +@stderr error: auto-serialization via toCell() is not available for type `A` +@stderr because field `A.multiple` of type `(int32, B | C)` can't be serialized +@stderr because element `tensor.1` of type `B | C` can't be serialized +@stderr because could not automatically generate serialization prefixes for a union +@stderr because struct `B` has opcode, but `C` does not + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7813.tolk b/tolk-tester/tests/invalid-serialization/err-7813.tolk new file mode 100644 index 0000000000..720678cffb --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7813.tolk @@ -0,0 +1,13 @@ +struct (0x0F) A {} + +fun f(x: int32 | A) { + x.toCell(); +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `int32 | A` +@stderr because could not automatically generate serialization prefixes for a union +@stderr because of mixing primitives and struct `A` with serialization prefix +@stderr hint: extract primitives to single-field structs and provide prefixes + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7911.tolk b/tolk-tester/tests/invalid-serialization/err-7911.tolk new file mode 100644 index 0000000000..ab045dd400 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7911.tolk @@ -0,0 +1,22 @@ +struct ExtraData { + owner: address; + lastTime: int; +} + +struct Storage { + more: Cell; +} + +fun main() { + Storage.fromSlice(""); +} + +/** +@compilation_should_fail +@stderr error: auto-serialization via fromSlice() is not available for type `Storage` +@stderr because field `Storage.more` of type `Cell` can't be serialized +@stderr because type `ExtraData` can't be serialized +@stderr because field `ExtraData.lastTime` of type `int` can't be serialized +@stderr because type `int` is not serializable, it doesn't define binary width +@stderr hint: replace `int` with `int32` / `uint64` / `coins` / etc. + */ diff --git a/tolk-tester/tests/pack-unpack-1.tolk b/tolk-tester/tests/pack-unpack-1.tolk new file mode 100644 index 0000000000..00767042c6 --- /dev/null +++ b/tolk-tester/tests/pack-unpack-1.tolk @@ -0,0 +1,130 @@ +struct JustInt32 { + value: int32; +} + +struct Point { + x: int32; + y: int32; +} + +@method_id(101) +fun test1(value: int) { + var t: JustInt32 = { value }; + var c = t.toCell(); + var t2 = c.load(); + __expect_type(c, "Cell"); + __expect_type(t2, "JustInt32"); + if (0) { + var rawCell: cell = c; // Cell is assignable to raw cell + contract.setData(t.toCell()); // (for this purpose, since we don't have overloads) + } + + var t3 = Cell.load(c); + var t4 = JustInt32.fromCell(c); + var t5 = JustInt32.fromSlice(c.beginParse()); + return (t2.value, t3.value, t4.value, t5.value); +} + +@method_id(102) +fun test2() { + var t: JustInt32 = { value: 10 }; + var c = t.toCell(); + var s = c.tvmCell.beginParse(); + __expect_type(s.skipAny(), "slice"); + s.assertEnd(); + return true; +} + +@method_id(103) +fun test3() { + var yy = 20; + var p: Point = { x: 10, y: yy }; + var c = p.toCell(); // p is constant, storing its fields are joined into a single STI + var s = c.beginParse(); + s.skipAny(); // skipping is also merged to skip 64 bits + return s.remainingBitsCount(); +} + +@method_id(104) +fun test4() { + var s = stringHexToSlice("000000010000000200000003"); + var p = Point.fromSlice(s); // does not mutate s + return (p, s.remainingBitsCount()); +} + +@method_id(105) +fun test5() { + var s = stringHexToSlice("000000010000000200000003"); + var p: Point = s.loadAny(); // mutates s + return (p, s.remainingBitsCount()); +} + +@method_id(106) +fun test6() { + var b = beginCell().storeInt(1, 32); + var p: Point = { x: 10, y: 20 }; + b.storeAny(p); + var s = b.endCell().beginParse(); + return (s.remainingBitsCount(), b.bitsCount(), Point.fromSlice(s.getLastBits(64)), s.remainingBitsCount()); +} + +// this function is used in serialization codegen, it's not exposed to stdlib; +// it constructs "x{...}" slices for SDBEGINSQ, used for opcode matching; +// here we write some unit tests and fif codegen tests for it +fun slice.tryStripPrefix(mutate self, prefix: int, prefixLen: int): bool + builtin; + +@method_id(107) +fun test7(s: slice) { + if (s.tryStripPrefix(0x10, 32)) { return (1, s.remainingBitsCount()); } + if (s.tryStripPrefix(0x40, 31)) { return (2, s.remainingBitsCount()); } + if (s.tryStripPrefix(0xFF18, 16)) { return (3, s.remainingBitsCount()); } + if (s.tryStripPrefix(0b1001, 8)) { return (4, s.remainingBitsCount()); } + if (s.tryStripPrefix(0b01, 3)) { return (5, s.remainingBitsCount()); } + return (6, s.remainingBitsCount()); +} + +fun main(c: cell) { + c as Cell; + (c as Cell) as cell; + (c as Cell) as cell?; + __expect_type((c as Cell?)!, "Cell"); + c = c as Cell; // Cell implicitly converts to cell +} + +/** +@testcase | 101 | 10 | 10 10 10 10 +@testcase | 102 | | -1 +@testcase | 103 | | 0 +@testcase | 104 | | 1 2 96 +@testcase | 105 | | 1 2 32 +@testcase | 106 | | 96 96 10 20 96 +@testcase | 107 | x{00000080} | 2 1 +@testcase | 107 | x{09332} | 4 12 +@testcase | 107 | x{2} | 5 1 +@testcase | 107 | x{0234} | 6 16 + +@fif_codegen +""" + test3 PROC:<{ + 20 PUSHINT // yy=20 + 10 PUSHINT // p.y=20 p.x=10 + NEWC // p.y=20 p.x=10 b + 32 STI // p.y=20 b + 32 STI // b + ENDC // c + CTOS // s + 32 PUSHINT // s '12=32 + SDSKIPFIRST // s + 32 PUSHINT // s '13=32 + SDSKIPFIRST // s + SBITS // '14 + }> +""" + +@fif_codegen x{00000010} SDBEGINSQ +@fif_codegen b{0000000000000000000000001000000} SDBEGINSQ +@fif_codegen x{ff18} SDBEGINSQ +@fif_codegen x{09} SDBEGINSQ +@fif_codegen b{001} SDBEGINSQ + */ diff --git a/tolk-tester/tests/pack-unpack-2.tolk b/tolk-tester/tests/pack-unpack-2.tolk new file mode 100644 index 0000000000..75f20a5fdd --- /dev/null +++ b/tolk-tester/tests/pack-unpack-2.tolk @@ -0,0 +1,682 @@ + +struct MaybeNothing {} +struct MaybeJust { value: T; } +type Maybe = MaybeNothing | MaybeJust; + +struct EitherLeft { value: T } +struct EitherRight { value: T } +type Either = EitherLeft | EitherRight; + +@inline +fun makeExternalAddress(hash: int, len: int): address { + return beginCell().storeUint(0b01, 2).storeUint(len, 9).storeUint(hash, len).endCell().beginParse() as address; +} + + +fun slice.assertEqDeeply(self, rhs: slice): slice { + var lhs = self; + assert(lhs.bitsEqual(rhs), 400); + assert(lhs.remainingRefsCount() == rhs.remainingRefsCount(), 400); + while (lhs.remainingRefsCount()) { + lhs.loadRef().beginParse().assertEqDeeply(rhs.loadRef().beginParse()); + } + return self; +} + +@inline +fun generateSlice_44_with_ref45(): slice { + return generateCell_44_with_ref45().beginParse(); +} + +@inline +fun generateCell_44_with_ref45(): cell { + return beginCell().storeInt(44, 32).storeRef(beginCell().storeInt(45, 32).endCell()).endCell(); +} + +fun assert_slice_is_44_and_ref45(s: slice) { + assert(s.loadInt(32) == 44, 400); + var ref = s.loadRef().beginParse(); + assert(ref.loadInt(32) == 45, 400); + ref.assertEnd(); + s.assertEnd(); +} + +@inline +fun slice.appendRef(self, refSlice: slice): slice { + return beginCell().storeSlice(self).storeRef(beginCell().storeSlice(refSlice).endCell()).endCell().beginParse(); +} + +fun run(input: TInputStruct, ans: slice) { + repeat (2) { + var s = input.toCell().beginParse(); + input = TInputStruct.fromSlice(s.assertEqDeeply(ans)); + } + input.toCell().beginParse().skipAny().assertEnd(); +} + + +/* + value:int32 + = JustInt32; +*/ + +struct JustInt32 { + value: int32; +} + +/* + value:(Maybe int32) + = JustMaybeInt32; + */ + +struct JustMaybeInt32 { + value: int32?; +} + +/* + op:int32 + amount:Grams + = TwoInts32AndCoins; +*/ + +struct TwoInts32AndCoins { + op: int32; + amount: coins; +} + +/* + op:int32 + query_id:uint64 + = TwoInts32And64; +*/ + +struct TwoInts32And64 { + op: int32; + query_id: uint64; +} + +/* + op:int32 + query_id_ref: ^[uint64] + = TwoInts32AndRef64; +*/ + +struct TwoInts32AndRef64 { + op: int32; + query_id_ref: Cell; +} + +/* + op:int32 + query_id:(Maybe uint64) + demo_bool_field:Bool + = TwoInts32AndMaybe64; +*/ + +struct TwoInts32AndMaybe64 { + op: int32; + query_id: uint64?; + demo_bool_field: bool; +} + +/* + addr:MsgAddressInt + = JustAddress; +*/ + +struct JustAddress { + addr: address; +} + +/* + op:int32 + addr:MsgAddressExt + query_id:uint64 + = TwoInts32And64SepByAddress; +*/ + +struct TwoInts32And64SepByAddress { + op: int32; + addr_e: address; + query_id: uint64; +} + +/* + op:int32 + i8or256:(Either int8 int256) + = IntAndEitherInt8Or256; +*/ + +struct IntAndEitherInt8Or256 { + op: int32; + i8or256: int8 | int256; +} + +/* + query_id_ref:uint64 = Inner1; + i64_in_ref:int64 = Inner2; + + op:int32 + i32orRef:(Either int32 ^Inner2) + query_id_maybe_ref: (Maybe ^Inner1) + = IntAndEither32OrRef64; +*/ + +struct Inner1 { + query_id_ref: uint64; +} +struct Inner2 { + i64_in_ref: int64; +} + +struct IntAndEither32OrRef64 { + op: int32; + i32orRef: int32 | Cell; + query_id_maybe_ref: Cell?; +} + +/* + value:(Either int8 (Maybe int256)) + op:int32 + = IntAndEither8OrMaybe256; +*/ + +struct IntAndEither8OrMaybe256 { + value: Either; + op: int32; +} + +/* + value:(Either (Maybe int8) int256) + op:int32 + = IntAndEitherMaybe8Or256; +*/ + +struct IntAndEitherMaybe8Or256 { + value: Either; + op: int32; +} + +/* + value:(Maybe (Maybe int8)) + op:int32 + = IntAndMaybeMaybe8; +*/ + +struct IntAndMaybeMaybe8 { + value: Maybe>; + op: int32; +} + +/* + f1:bits8 + f2:bits3 + f3:(Maybe bits20) + f4:(Either bits100 bits200) + = SomeBytesFields; +*/ + +struct SomeBytesFields { + f1: bytes1; + f2: bits3; + f3: bits20?; + f4: bits100 | bits200; +} + +/* + op:int32 + rest:Cell + = IntAndRestInlineCell; +*/ + +struct IntAndRestInlineCell { + op: int32; + rest: RemainingBitsAndRefs; +} + +/* + op:int32 + rest:^Cell + = IntAndRestRefCell; +*/ + +struct IntAndRestRefCell { + op: int32; + rest: cell; +} + +/* + op:int32 + rest:(Either Cell ^Cell) + = IntAndRestEitherCellOrRefCell; +*/ + +struct IntAndRestEitherCellOrRefCell { + op: int32; + rest: RemainingBitsAndRefs | cell; +} + +/* + op:int32 + ref1m:(Maybe ^Cell) + ref2m:(Maybe ^Cell) + ref3:^Cell + ref4m32:(Maybe ^JustInt32) + query_id:int64 + = DifferentMaybeRefs; + */ + +struct DifferentMaybeRefs { + op: int32; + ref1m: cell?; + ref2m: dict; + ref3: cell; + ref4m32: Cell?; + query_id: int64; +} + +/* + ji:JustInt32 + jmi:JustMaybeInt32 + jiMaybe:(Maybe JustInt32) + jmiMaybe:(Maybe JustMaybeInt32) + = DifferentIntsWithMaybe; +*/ + +struct DifferentIntsWithMaybe { + ji: JustInt32; + jmi: JustMaybeInt32; + jiMaybe: JustInt32?; + jmiMaybe: JustMaybeInt32?; +} + +/* + ja1:JustAddress + ja2m:(Maybe JustAddress) + imm:IntAndMaybeMaybe8 + tis:TwoInts32And64SepByAddress + = DifferentMix1; +*/ + +@overflow1023_policy("suppress") +struct DifferentMix1 { + ja1: JustAddress; + ja2m: JustAddress?; + ext_nn: address; + imm: IntAndMaybeMaybe8; + tis: TwoInts32And64SepByAddress; +} + +/* + iae:^IntAndEither32OrRef64 + tic:TwoInts32AndCoins + rest:Cell + = DifferentMix2; +*/ + +struct DifferentMix2 { + iae: Cell; + tic: TwoInts32AndCoins; + rest: RemainingBitsAndRefs; +} + +/* + bod:(Either ^TwoInts32AndCoins ^JustInt32) + tim:(Maybe TwoInts32AndCoins) + pairm:(Maybe (Both int32 int64)) + = DifferentMix3; + */ + +struct DifferentMix3 { + bod: Cell | Cell; + tim: TwoInts32AndCoins?; + pairm: (int32, int64)?; +} + +/* + Test: write with builder, read with other struct + (type `builder` is available for writing, but not for reading) + */ + +struct WriteWithBuilder { + f1: int32; + rest: builder; +} + +struct ReadWrittenWithBuilder { + f1: int32; + someInt: uint32; + someCell: cell?; +} + +struct ReadWriteRest { + f1: int32; + f2: coins; + rest: T; +} + +struct Tail224 { + ji: JustInt32, + addr: address, + ref1: Cell?, + ref2: Cell, +} + +type ReadRest_Remaining = ReadWriteRest; + +struct ReadWriteMid { + f1: int32; + mid: T; + f3: coins; +} + + + +// --------------------------------------------- + + +@method_id(200) +fun test_JustInt32_1() { + var b = beginCell(); + b.storeAny(JustInt32{ value: 123 }); + b.storeAny(JustInt32{ value: 456 }); + return b; +} + +@method_id(201) +fun test_JustInt32() { + run({ value: 255}, stringHexToSlice("000000FF")); + var s = stringHexToSlice("0000007b000001c8"); + return (s.loadAny(), s.loadAny()); +} + +@method_id(202) +fun test_JustMaybeInt32() { + run({ value: 255 }, stringHexToSlice("8000007FC_")); + run({ value: null }, stringHexToSlice("4_")); + return true; +} + +@method_id(203) +fun test_TwoInts32AndCoins() { + run({ op: 123, amount: 0 }, stringHexToSlice("0000007B0")); + run({ op: 123, amount: 1000000000 }, stringHexToSlice("0000007B43B9ACA00")); + return true; +} + +@method_id(204) +fun test_TwoInts32And64() { + run({ op: 123, query_id: 255 }, stringHexToSlice("0000007B00000000000000FF")); + return true; +} + +@method_id(205) +fun test_TwoInts32AndRef64() { + run({ op: 123, query_id_ref: { tvmCell: beginCell().storeUint(255,64).endCell() } }, stringHexToSlice("0000007B").appendRef(stringHexToSlice("00000000000000FF"))); + return true; +} + +@method_id(206) +fun test_TwoInts32AndMaybe64() { + run({ op: 123, query_id: 255, demo_bool_field: true }, stringHexToSlice("0000007B800000000000007FE_")); + run({ op: 123, query_id: null, demo_bool_field: true }, stringHexToSlice("0000007B6_")); + run({ op: 123, query_id: null, demo_bool_field: false }, stringHexToSlice("0000007B2_")); + return true; +} + +@method_id(207) +fun test_JustAddress() { + run({ addr: address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e") }, stringHexToSlice("80194DC6438F99D3D9DBE151944925D90B2492954BF6B9C070FBFF2DDED5F30547D_")); + return true; +} + +@method_id(208) +fun test_TwoInts32And64SepByAddress() { + run({ op: 123, addr_e: makeExternalAddress(70, 10), query_id: 255 } , stringHexToSlice("0000007B41423000000000000007FC_")); + run({ op: 123, addr_e: makeExternalAddress(666, 20), query_id: 254 } , stringHexToSlice("0000007B4280053400000000000001FD_")); + run({ op: 123, addr_e: createAddressNone(), query_id: 253 } , stringHexToSlice("0000007B000000000000003F6_")); + return true; +} + +@method_id(209) +fun test_IntAndEitherInt8Or256() { + run({ op: 123, i8or256: 80 as int8 }, stringHexToSlice("0000007B284_")); + run({ op: 123, i8or256: 65535 as int256 }, stringHexToSlice("0000007B8000000000000000000000000000000000000000000000000000000000007FFFC_")); + return true; +} + +@method_id(210) +fun test_IntAndEither32OrRef64() { + run({ op: 123, i32orRef: Inner2{ i64_in_ref: 555 }.toCell(), query_id_maybe_ref: Inner1{ query_id_ref: 888 }.toCell() }, stringHexToSlice("0000007BE_").appendRef(stringHexToSlice("000000000000022B")).appendRef(stringHexToSlice("0000000000000378"))); + run({ op: 123, i32orRef: Inner2{ i64_in_ref: 555 }.toCell(), query_id_maybe_ref: null }, stringHexToSlice("0000007BA_").appendRef(stringHexToSlice("000000000000022B"))); + run({ op: 123, i32orRef: 555, query_id_maybe_ref: Inner1{ query_id_ref: 888 }.toCell() }, stringHexToSlice("0000007B00000115E_").appendRef(stringHexToSlice("0000000000000378"))); + run({ op: 123, i32orRef: 555, query_id_maybe_ref: null }, stringHexToSlice("0000007B00000115A_")); + return IntAndEither32OrRef64.fromSlice(stringHexToSlice("0000007B00000115A_")); +} + +@method_id(211) +fun test_IntAndEither8OrMaybe256() { + run({ value: EitherLeft { value: 100 }, op: 123 }, stringHexToSlice("320000003DC_")); + run({ value: EitherRight { value: 10000 }, op: 123 }, stringHexToSlice("C0000000000000000000000000000000000000000000000000000000000009C40000001EE_")); + run({ value: EitherRight { value: null }, op: 123 }, stringHexToSlice("8000001EE_")); + return IntAndEither8OrMaybe256.fromSlice(stringHexToSlice("8000001EE_")); +} + +@method_id(212) +fun test_IntAndEitherMaybe8Or256() { + run({ value: EitherLeft { value: 100 }, op: 123 }, stringHexToSlice("590000001EE_")); + run({ value: EitherRight { value: 10000 }, op: 123 }, stringHexToSlice("80000000000000000000000000000000000000000000000000000000000013880000003DC_")); + run({ value: EitherLeft { value: null }, op: 123 }, stringHexToSlice("0000001EE_")); + return IntAndEitherMaybe8Or256.fromSlice(stringHexToSlice("0000001EE_")); +} + +@method_id(213) +fun test_SomeBytesFields() { + run({ f1: stringHexToSlice("A4") as bytes1, f2: stringHexToSlice("7_") as bits3, f3: null, f4: stringHexToSlice("BBA87684B3DAA58C0FCC75230C4302C9D156102139D631FF56") as bits200 }, stringHexToSlice("A46DDD43B4259ED52C607E63A9186218164E8AB08109CEB18FFAB4_")); + run({ f1: stringHexToSlice("E6") as bytes1, f2: stringHexToSlice("D_") as bits3, f3: stringHexToSlice("2531C") as bits20, f4: stringHexToSlice("927E88FAB2D327D9468547217") as bits100 }, stringHexToSlice("E6D2531C493F447D596993ECA342A390BC_")); + return SomeBytesFields.fromSlice(stringHexToSlice("E6D2531C493F447D596993ECA342A390BC_")).f4 is bits100; +} + +@method_id(214) +fun test_IntAndMaybeMaybe8() { + run({ value: MaybeJust { value: MaybeJust { value: 88 } }, op: 123 }, stringHexToSlice("D60000001EE_")); + run({ value: MaybeJust { value: MaybeNothing{} }, op: 123 }, stringHexToSlice("8000001EE_")); + + val t1 = IntAndMaybeMaybe8.fromSlice(stringHexToSlice("D60000001EE_")); // (88) 123 + val v1 = t1.value; + __expect_type(v1, "Maybe>"); + assert(v1 is MaybeJust && v1.value is MaybeJust && v1.value.value == 88 && t1.op == 123, 400); + __expect_type(v1, "MaybeJust>"); + __expect_type(v1.value, "MaybeJust"); + __expect_type(v1.value.value, "int8"); + val t2 = IntAndMaybeMaybe8.fromSlice(stringHexToSlice("8000001EE_")); // (()) 123 + val v2 = t2.value; + assert(v2 is MaybeJust && v2.value is MaybeNothing && t2.op == 123, 400); + __expect_type(v2, "MaybeJust>"); + __expect_type(v2.value, "MaybeNothing"); + + return true; +} + +@method_id(215) +fun test_IntAndRestInlineCell() { + run({ op: 123, rest: generateSlice_44_with_ref45() }, stringHexToSlice("0000007B0000002C").appendRef(stringHexToSlice("0000002D"))); + return true; +} + +@method_id(216) +fun test_IntAndRestRefCell() { + run({ op: 123, rest: generateCell_44_with_ref45() }, stringHexToSlice("0000007B").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D")))); + return true; +} + +@method_id(217) +fun test_IntAndRestEitherCellOrRefCell() { + val input1: IntAndRestEitherCellOrRefCell = { op: 123, rest: generateCell_44_with_ref45() }; + var s1 = input1.toCell().beginParse(); + s1.assertEqDeeply(stringHexToSlice("0000007BC_").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D")))); + match (val rest = IntAndRestEitherCellOrRefCell.fromSlice(s1).rest) { + cell => assert_slice_is_44_and_ref45(rest.beginParse()), + RemainingBitsAndRefs => assert_slice_is_44_and_ref45(rest) + } + + val input2: IntAndRestEitherCellOrRefCell = { op: 123, rest: generateSlice_44_with_ref45() }; + var s2 = input2.toCell().beginParse(); + s2.assertEqDeeply(stringHexToSlice("0000007B000000164_").appendRef(stringHexToSlice("0000002D"))); + match (val rest = IntAndRestEitherCellOrRefCell.fromSlice(s2).rest) { + cell => assert_slice_is_44_and_ref45(rest.beginParse()), + RemainingBitsAndRefs => assert_slice_is_44_and_ref45(rest) + } + + return true; +} + +@method_id(218) +fun test_DifferentMaybeRefs() { + run({ op: 123, ref1m: null, ref2m: null, ref3: generateCell_44_with_ref45(), ref4m32: null, query_id: 456}, stringHexToSlice("0000007B00000000000000391_").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D")))); + run({ op: 123, ref1m: generateCell_44_with_ref45(), ref2m: null, ref3: generateCell_44_with_ref45(), ref4m32: JustInt32{value: 999}.toCell(), query_id: 456}, stringHexToSlice("0000007BA0000000000000391_").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef(stringHexToSlice("000003E7"))); + run({ op: 123, ref1m: null, ref2m: beginCell().endCell(), ref3: generateCell_44_with_ref45(), ref4m32: JustInt32{value: 998}.toCell(), query_id: 456}, stringHexToSlice("0000007B60000000000000391_").appendRef("").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef(stringHexToSlice("000003E6"))); + run({ op: 123, ref1m: generateCell_44_with_ref45(), ref2m: generateCell_44_with_ref45(), ref3: beginCell().endCell(), ref4m32: null, query_id: 456}, stringHexToSlice("0000007BC0000000000000391_").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef("")); + run({ op: 123, ref1m: generateCell_44_with_ref45(), ref2m: beginCell().endCell(), ref3: generateCell_44_with_ref45(), ref4m32: JustInt32{value: 997}.toCell(), query_id: 456}, stringHexToSlice("0000007BE0000000000000391_").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef("").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef(stringHexToSlice("000003E5"))); + return true; +} + +@method_id(219) +fun test_DifferentIntsWithMaybe() { + run({ ji: { value: 44 }, jmi: { value: 45 }, jiMaybe: null, jmiMaybe: null }, stringHexToSlice("0000002C800000169_")); + run({ ji: { value: 44 }, jmi: { value: null }, jiMaybe: { value: 45 }, jmiMaybe: { value: null } }, stringHexToSlice("0000002C4000000B6")); + run({ ji: { value: 44 }, jmi: { value: null }, jiMaybe: null, jmiMaybe: { value: 46 } }, stringHexToSlice("0000002C30000002E")); + return DifferentIntsWithMaybe.fromSlice(stringHexToSlice("0000002C30000002E")); +} + +@method_id(220) +fun test_DifferentMix1() { + run({ ja1: { addr: address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5") }, ja2m: { addr: address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5") }, ext_nn: makeExternalAddress(1234,30), imm: { op: 78, value: MaybeJust { value: MaybeJust { value: 78 } } }, tis: { op: 123, query_id: 889128, addr_e: makeExternalAddress(1234, 80) } }, + stringHexToSlice("80122199EC3C49BA84BA73C79F764BF1A4C1400717E303DC86E737C60A3E3B1B62180122199EC3C49BA84BA73C79F764BF1A4C1400717E303DC86E737C60A3E3B1B62087800004D2D3800000138000001ED2800000000000000000269000000000006C8944_")); + run({ ja1: { addr: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c") }, ja2m: null, ext_nn: makeExternalAddress(0,3), imm: { op: 99, value: MaybeJust { value: MaybeJust { value: 99 } } }, tis: { op: 1234, query_id: 889129, addr_e: createAddressNone() } }, + stringHexToSlice("800000000000000000000000000000000000000000000000000000000000000000040636300000063000004D2000000000003644A6_")); + + val o = DifferentMix1.fromSlice(stringHexToSlice("800000000000000000000000000000000000000000000000000000000000000000040636300000063000004D2000000000003644A6_")); + return (o.ja1.addr.getWorkchainAndHash(), o.ja2m, o.imm, o.tis.op, o.tis.addr_e.isNone() ? null : 0, o.tis.query_id, (o.ext_nn as slice).remainingBitsCount()); +} + +@method_id(221) +fun test_DifferentMix2() { + run({ iae: IntAndEither32OrRef64{ op: 777, i32orRef: 2983, query_id_maybe_ref: null }.toCell(), tic: { op: 123, amount: 829290000 }, rest: generateSlice_44_with_ref45() }, + stringHexToSlice("0000007B4316DF6100000002C").appendRef(stringHexToSlice("00000309000005D3A_")).appendRef(stringHexToSlice("0000002D"))); + val r1 = DifferentMix2.fromSlice(stringHexToSlice("0000007B4316DF6100000002C").appendRef(stringHexToSlice("00000309000005D3A_")).appendRef(stringHexToSlice("0000002D"))); + val iae1 = r1.iae.load(); + assert(iae1.i32orRef is int32, 400); + assert(iae1.i32orRef == 2983, 400); + assert(iae1.query_id_maybe_ref == null, 400); + assert_slice_is_44_and_ref45(r1.rest); + + run({ iae: { tvmCell: IntAndEither32OrRef64{ op: 778, i32orRef: { tvmCell: Inner2{ i64_in_ref: 9919992 }.toCell() }, query_id_maybe_ref: Inner1{ query_id_ref: 889477 }.toCell() }.toCell() }, tic: { op: 123, amount: 500000 }, rest: beginCell().endCell().beginParse() }, + stringHexToSlice("0000007B307A120").appendRef(stringHexToSlice("0000030AE_").appendRef(stringHexToSlice("0000000000975DF8")).appendRef(stringHexToSlice("00000000000D9285")))); + var r2 = DifferentMix2.fromSlice(stringHexToSlice("0000007B307A120").appendRef(stringHexToSlice("0000030AE_").appendRef(stringHexToSlice("0000000000975DF8")).appendRef(stringHexToSlice("00000000000D9285")))); + val iae2 = r2.iae.load(); + assert(iae2.i32orRef is Cell, 400); + __expect_type(iae2.i32orRef.load(), "Inner2"); + assert(iae2.i32orRef.load().i64_in_ref == 9919992, 400); + assert(iae2.query_id_maybe_ref != null, 400); + __expect_type(iae2.query_id_maybe_ref.load(), "Inner1"); + assert(iae2.query_id_maybe_ref.load().query_id_ref == 889477, 400); + assert(r2.tic.amount == 500000, 400); + r2.rest.assertEnd(); + + return true; +} + +@method_id(222) +fun test_DifferentMix3() { + run({ bod: TwoInts32AndCoins{op:123, amount:80000}.toCell(), tim: {op: 456, amount:0}, pairm: null }, stringHexToSlice("4000007201_").appendRef(stringHexToSlice("0000007B3013880"))); + run({ bod: TwoInts32AndCoins{op:124, amount:10}.toCell(), tim: null, pairm: (100000,100000) }, stringHexToSlice("200030D400000000000030D41_").appendRef(stringHexToSlice("0000007C10A"))); + run({ bod: JustInt32{value:255}.toCell(), tim: null, pairm: (90,90) }, stringHexToSlice("A000000B400000000000000B5_").appendRef(stringHexToSlice("000000FF"))); + run({ bod: JustInt32{value:510}.toCell(), tim: {op: 567, amount:9392843922}, pairm: (81923,81923) }, stringHexToSlice("C000008DD408BF6DB24A000280060000000000028007_").appendRef(stringHexToSlice("000001FE"))); + + val o4 = DifferentMix3.fromSlice(stringHexToSlice("C000008DD408BF6DB24A000280060000000000028007_").appendRef(stringHexToSlice("000001FE"))); + val o2 = DifferentMix3.fromSlice(stringHexToSlice("200030D400000000000030D41_").appendRef(stringHexToSlice("0000007C10A"))); + return ( + (o4.bod is Cell) ? o4.bod.load().value as int : -1, o4.tim, o4.pairm, + 777, + o2.bod is Cell, o2.bod is Cell, o2.tim, o2.pairm, + ); +} + +@method_id(223) +fun test_WriteWithBuilderReadWithOther() { + var b = beginCell().storeUint(55, 32).storeMaybeRef(null); + var w: WriteWithBuilder = { f1: 10, rest: b }; + return ReadWrittenWithBuilder.fromCell(w.toCell()); +} + +@method_id(224) +fun test_RestIsBuilderOrRemaining() { + var remainingB = beginCell() + .storeUint(5, 32) + .storeAddress(address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8")) + .storeMaybeRef(JustInt32 { value: 123 }.toCell()) + .storeRef(JustAddress { addr: createAddressNone() }.toCell()); + var w: ReadWriteRest = { + f1: 60, + f2: ton("0.05"), + rest: remainingB + }; + var r: ReadRest_Remaining = w.toCell().beginParse().loadAny(); + __expect_type(r.rest, "RemainingBitsAndRefs"); + var rest = Tail224.fromSlice(r.rest); + return (r.f1, r.f2, rest.ji.value, rest.addr.getWorkchain(), rest.ref1!.load(), rest.ref2.load().addr.isNone()); +} + +@method_id(225) +fun test_MidIsBuilderOrBitsN() { + var bits40_b = beginCell().storeSlice(stringHexToSlice("0000FFFF00")); + var typedCell = ReadWriteMid { + f1: 5, + mid: bits40_b, + f3: ton("0.05") + }.toCell(); + __expect_type(typedCell, "Cell>"); + var r = ReadWriteMid.fromCell(typedCell); + var mid = r.mid as slice; + return (r.f1, mid.remainingBitsCount(), mid.loadAny(), mid.remainingBitsCount(), r.f3); +} + + +fun main() { + var t: JustInt32 = { value: 10 }; + var c = t.toCell(); + var t2 = c.load(); + var t3 = JustInt32.fromSlice(c.beginParse()); + return (t2.value, t3.value); +} + +/** +@testcase | 0 | | 10 10 +@testcase | 200 | | BC{00100000007b000001c8} +@testcase | 201 | | 123 456 +@testcase | 202 | | -1 +@testcase | 203 | | -1 +@testcase | 204 | | -1 +@testcase | 205 | | -1 +@testcase | 206 | | -1 +@testcase | 207 | | -1 +@testcase | 208 | | -1 +@testcase | 209 | | -1 +@testcase | 210 | | 123 555 46 (null) +@testcase | 211 | | (null) 132 123 +@testcase | 212 | | (null) 133 123 +@testcase | 213 | | -1 +@testcase | 214 | | -1 +@testcase | 215 | | -1 +@testcase | 216 | | -1 +@testcase | 217 | | -1 +@testcase | 218 | | -1 +@testcase | 219 | | 44 (null) (null) 46 143 +@testcase | 220 | | 0 0 (null) 99 136 137 99 1234 (null) 889129 14 +@testcase | 221 | | -1 +@testcase | 222 | | 510 567 9392843922 146 81923 81923 147 777 0 -1 (null) (null) 0 100000 100000 147 +@testcase | 223 | | 10 55 (null) +@testcase | 224 | | 60 50000000 5 9 123 -1 +@testcase | 225 | | 5 40 65535 8 50000000 + */ diff --git a/tolk-tester/tests/pack-unpack-3.tolk b/tolk-tester/tests/pack-unpack-3.tolk new file mode 100644 index 0000000000..74fa1b8e92 --- /dev/null +++ b/tolk-tester/tests/pack-unpack-3.tolk @@ -0,0 +1,284 @@ + +struct EitherLeft { value: T } +struct EitherRight { value: T } +type Either = EitherLeft | EitherRight; + +@inline +fun makeExternalAddress(hash: int, len: int): address { + return beginCell().storeUint(0b01, 2).storeUint(len, 9).storeUint(hash, len).endCell().beginParse() as address; +} + + +fun slice.assertEqDeeply(self, rhs: slice): slice { + var lhs = self; + assert(lhs.bitsEqual(rhs), 400); + assert(lhs.remainingRefsCount() == rhs.remainingRefsCount(), 400); + while (lhs.remainingRefsCount()) { + lhs.loadRef().beginParse().assertEqDeeply(rhs.loadRef().beginParse()); + } + return self; +} + +@inline +fun slice.appendRef(self, refSlice: slice): slice { + return beginCell().storeSlice(self).storeRef(beginCell().storeSlice(refSlice).endCell()).endCell().beginParse(); +} + +@inline +fun generateSlice_44_with_ref45(): slice { + return generateCell_44_with_ref45().beginParse(); +} + +@inline +fun generateCell_44_with_ref45(): cell { + return beginCell().storeInt(44, 32).storeRef(beginCell().storeInt(45, 32).endCell()).endCell(); +} + + +fun run(input: TInputStruct, ans: slice) { + repeat (2) { + var s = input.toCell().beginParse(); + input = TInputStruct.fromSlice(s.assertEqDeeply(ans)); + } + input.toCell().beginParse().skipAny().assertEnd(); +} + + +/* +single_prefix32#87654321 + amount1:Grams + amount2:Grams + = MsgSinglePrefix32; +*/ + +struct(0x87654321) MsgSinglePrefix32 { + amount1: coins; + amount2: coins; +} + +/* +single_prefix48#876543211234 + amount:(Either Grams uint64) + = MsgSinglePrefix48; +*/ + +struct(0x876543211234) MsgSinglePrefix48 { + amount: coins | uint64; +} + +/* +counterIncrement#12345678 + counter_id:int8 + inc_by:int32 + = MsgCounter1; +*/ + +struct(0x12345678) CounterIncrement { + counter_id: int8; + inc_by: int32; +} + +/* +counterDecrement#23456789 + counter_id:int8 + dec_by:int32 + = MsgCounter1; +*/ + +struct(0x23456789) CounterDecrement { + counter_id: int8; + dec_by: int32; +} + +/* +counterReset0#34567890 + counter_id:int8 + = MsgCounter1; +*/ + +struct(0x34567890) CounterReset0 { + counter_id: int8; +} + +/* +counterResetTo#00184300 + counter_id:int8 + initial_value:int64 + = MsgCounter1; +*/ + +struct(0x00184300) CounterResetTo { + counter_id: int8; + initial_value: int64; +} + +type MsgCounter1 = + CounterIncrement | + CounterDecrement | + CounterReset0 | + CounterResetTo | +; + +/* +bodyPayload1$001 + should_forward:Bool + n_times:int32 + content:Cell + = BodyPayload; +*/ + +struct(0b001) BodyPayload1 { + should_forward: bool; + n_times: int32; + content: RemainingBitsAndRefs; +} + +/* +bodyPayload2$01 + master_id:int8 + owner_address:MsgAddressInt + = BodyPayload; +*/ + +struct(0b01) BodyPayload2 { + master_id: int8; + owner_address: address; +} + +type BodyPayload = BodyPayload1 | BodyPayload2; + +/* +sayHiAndGoodbye#89 + dest_addr:(Maybe MsgAddressInt) + body:BodyPayload + = MsgExternal1; +*/ + +struct(0x89) SayHiAndGoodbye { + dest_addr: address?; + body: BodyPayload; +} + +/* +sayStoreInChain#0013 + in_masterchain:Bool + contents:^BodyPayload + = MsgExternal1; + */ + +struct(0x0013) SayStoreInChain { + in_masterchain: bool; + contents: Cell; +} + +type MsgExternal1 = SayHiAndGoodbye | SayStoreInChain; + +/* +transferParams1#794 + dest_int:MsgAddressInt + amount:Grams + dest_ext:MsgAddressExt + = TransferParams; +*/ + +struct(0x794) TransferParams1 { + dest_int: address; + amount: coins; + dest_ext: address; +} + +/* +transferParams2#9 + intVector:(Both int32 (Both (Maybe Grams) uint64)) + needs_more:^Bit + = TransferParams; + */ + +struct(0x9) TransferParams2 { + intVector: (int32, coins?, uint64); + needs_more: Cell; +} + +type TransferParams = TransferParams1 | TransferParams2; + +/* +_#FB3701FF + params:(Either TransferParams ^TransferParams) + = MsgTransfer; + */ + +struct(0xFB3701FF) MsgTransfer { + params: Either>; +} + + + +// --------------------------------------------- + + +@method_id(201) +fun test_MsgSinglePrefix32() { + run({ amount1: 80, amount2: 800000000 }, stringHexToSlice("8765432115042FAF0800")); + + return MsgSinglePrefix32.fromSlice(stringHexToSlice("8765432115042FAF0800")); +} + +@method_id(202) +fun test_MsgSinglePrefix48() { + run({ amount: 80 as uint64 }, stringHexToSlice("87654321123480000000000000284_")); + run({ amount: 800000000 as coins }, stringHexToSlice("876543211234217D784004_")); + + var o = MsgSinglePrefix48.fromSlice(stringHexToSlice("876543211234217D784004_")); + return (o, 777, o.amount is coins, o.amount is uint64); +} + +@method_id(203) +fun test_MsgCounter1() { + run(CounterIncrement{ counter_id: 123, inc_by: 78 }, stringHexToSlice("123456787B0000004E")); + run(CounterDecrement{ counter_id: 0, dec_by: -38 }, stringHexToSlice("2345678900FFFFFFDA")); + run(CounterReset0{ counter_id: 0 }, stringHexToSlice("3456789000")); + run(CounterResetTo{ counter_id: 0, initial_value: 29874329774732 }, stringHexToSlice("001843000000001B2BA8D06A8C")); + + val o = MsgCounter1.fromSlice(stringHexToSlice("3456789000")); + return (o, 777, o is CounterReset0, o is CounterIncrement); +} + +@method_id(204) +fun test_MsgExternal1() { + run(SayHiAndGoodbye{ dest_addr: null, body: BodyPayload2 { master_id: 10, owner_address: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c") } }, stringHexToSlice("892150000000000000000000000000000000000000000000000000000000000000000002_")); + run(SayHiAndGoodbye{ dest_addr: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), body: BodyPayload1 { should_forward: false, n_times: 85, content: generateSlice_44_with_ref45() } }, stringHexToSlice("89C0000000000000000000000000000000000000000000000000000000000000000002000000550000002C").appendRef(stringHexToSlice("0000002D"))); + run(SayHiAndGoodbye{ dest_addr: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), body: BodyPayload2 { master_id: -5, owner_address: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi") } }, stringHexToSlice("89CFF28E8033DB1467CACEA8B158F0F61E682DE06E8A5947504C904F1F703D2BE4D9E7EE7F9474019ED8A33E5675458AC787B0F3416F037452CA3A82648278FB81E95F26CF4_")); + run(SayStoreInChain{ in_masterchain: true, contents: { tvmCell: BodyPayload1{ should_forward: true, n_times: 20, content: generateSlice_44_with_ref45() }.toCell()} }, stringHexToSlice("0013C_").appendRef(stringHexToSlice("3000000140000002C").appendRef(stringHexToSlice("0000002D")))); + run(SayStoreInChain{ in_masterchain: false, contents: { tvmCell: BodyPayload2{ master_id: 37, owner_address: address(("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c")) }.toCell()} }, stringHexToSlice("00134_").appendRef(stringHexToSlice("4960000000000000000000000000000000000000000000000000000000000000000004_"))); + + val o = MsgExternal1.fromSlice(stringHexToSlice("00134_").appendRef(stringHexToSlice("4960000000000000000000000000000000000000000000000000000000000000000004_"))); + assert(o is SayStoreInChain, 400); + val contents = o.contents.load(); + return (contents is BodyPayload2 && contents.owner_address == address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), contents is BodyPayload1, contents is BodyPayload2); +} + +@method_id(205) +fun test_MsgTransfer() { + run({ params: EitherLeft { value: TransferParams1 { dest_int: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), amount: 80000000, dest_ext: makeExternalAddress(1234,80) } } }, stringHexToSlice("FB3701FF3CA4FF28E8033DB1467CACEA8B158F0F61E682DE06E8A5947504C904F1F703D2BE4D9E404C4B4004A0000000000000000009A5_")); + run({ params: EitherLeft { value: TransferParams2 { intVector: (123, 1234567890123456, 1234567890123456), needs_more: {tvmCell: beginCell().storeBool(true).endCell()} } } }, stringHexToSlice("FB3701FF48000003DDC118B54F22AEB0000118B54F22AEB02_").appendRef(stringHexToSlice("C_"))); + run({ params: EitherRight { value: { tvmCell: TransferParams1{ dest_int: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), amount: 80000000, dest_ext: makeExternalAddress(1234,70) }.toCell()} } }, stringHexToSlice("FB3701FFC_").appendRef(stringHexToSlice("7948000000000000000000000000000000000000000000000000000000000000000000809896800918000000000000004D2"))); + run({ params: EitherRight { value: { tvmCell: TransferParams2{ intVector: (123, null, 0), needs_more: {tvmCell: beginCell().storeBool(false).endCell()} }.toCell()} } }, stringHexToSlice("FB3701FFC_").appendRef(stringHexToSlice("90000007B00000000000000004_").appendRef(stringHexToSlice("4_")))); + + val o = MsgTransfer.fromSlice(stringHexToSlice("FB3701FFC_").appendRef(stringHexToSlice("90000007B00000000000000004_").appendRef(stringHexToSlice("4_")))); + return ( + o.params is EitherLeft, o.params is EitherRight, + o.params is EitherRight && TransferParams.fromCell(o.params.value.tvmCell) is TransferParams1, + o.params is EitherRight && TransferParams.fromCell(o.params.value.tvmCell) is TransferParams2, + ); +} + + +fun main() {} + +/** +@testcase | 201 | | 80 800000000 +@testcase | 202 | | 800000000 17 777 -1 0 +@testcase | 203 | | (null) 0 131 777 -1 0 +@testcase | 204 | | -1 0 -1 +@testcase | 205 | | 0 -1 0 -1 + */ diff --git a/tolk-tester/tests/pack-unpack-4.tolk b/tolk-tester/tests/pack-unpack-4.tolk new file mode 100644 index 0000000000..4839032dce --- /dev/null +++ b/tolk-tester/tests/pack-unpack-4.tolk @@ -0,0 +1,162 @@ + +fun slice.assertEqDeeply(self, rhs: slice): slice { + var lhs = self; + assert(lhs.bitsEqual(rhs), 400); + assert(lhs.remainingRefsCount() == rhs.remainingRefsCount(), 400); + while (lhs.remainingRefsCount()) { + lhs.loadRef().beginParse().assertEqDeeply(rhs.loadRef().beginParse()); + } + return self; +} + + +fun run(input: TInputStruct, ans: slice) { + repeat (2) { + var s = input.toCell().beginParse(); + input = TInputStruct.fromSlice(s.assertEqDeeply(ans)); + } + input.toCell().beginParse().skipAny().assertEnd(); +} + + +type Union_8_16_32 = int8 | int16 | int32; + +fun ans101_8(): slice asm "b{0000001111} PUSHSLICE"; +fun ans101_16(): slice asm "b{010000000000001111} PUSHSLICE"; +fun ans101_32(): slice asm "b{1000000000000000000000000000001111} PUSHSLICE"; + +@method_id(101) +fun test1() { + run(15 as int8, ans101_8()); + run(15 as int16, ans101_16()); + run(15 as int32, ans101_32()); + + return 0; +} + + +type Union_8_16_32_n = int8 | null | int16 | int32; + +fun ans102n_8(): slice asm "b{0100001111} PUSHSLICE"; +fun ans102n_16(): slice asm "b{100000000000001111} PUSHSLICE"; +fun ans102n_32(): slice asm "b{1100000000000000000000000000001111} PUSHSLICE"; +fun ans102n_null(): slice asm "b{00} PUSHSLICE"; + +@method_id(102) +fun test2() { + run(15 as int8, ans102n_8()); + run(15 as int16, ans102n_16()); + run(15 as int32, ans102n_32()); + run(null, ans102n_null()); + + return 0; +} + + +struct Test4_8 { a: int8 } +struct Test4_16 { a: int16 } +struct Test4_32 { a: int32 } + +type UnionStructs_8_16_32 = int8 | Test4_16 | Test4_32; +type UnionStructs_8_16_32_n = Test4_8 | null | int16 | int32; + +fun ans104_8(): slice asm "b{0000001111} PUSHSLICE"; +fun ans104_16(): slice asm "b{010000000000001111} PUSHSLICE"; +fun ans104_32(): slice asm "b{1000000000000000000000000000001111} PUSHSLICE"; + +fun ans104n_8(): slice asm "b{0100001111} PUSHSLICE"; +fun ans104n_16(): slice asm "b{100000000000001111} PUSHSLICE"; +fun ans104n_32(): slice asm "b{1100000000000000000000000000001111} PUSHSLICE"; +fun ans104n_null(): slice asm "b{00} PUSHSLICE"; + +@method_id(104) +fun test4() { + // when mixing primitives and structs with no opcode, it's like mixing primitives + run(15, ans104_8()); + run(Test4_16{a:15}, ans104_16()); + run(Test4_32{a:15}, ans104_32()); + // with null — the same behavior + run({a:15}, ans104n_8()); + run(15 as int16, ans104n_16()); + run(15 as int32, ans104n_32()); + run(null, ans104n_null()); + + return 0; +} + + +type U105 = int8 | int32 | int64; // auto-prefixes 0b00 0b01 0b10 + +fun invalid105_slice(): slice asm "b{1100} PUSHSLICE"; // prefix 0b11 doesn't exist + +@method_id(105) +fun test5() { + try { + var u = U105.fromSlice(invalid105_slice()); + return (u is int8) ? 8 : -8; + } catch (excode) { + return excode; + } +} + + +type U106 = int8 | int16 | int32 | int64; // exhaustive prefixes, checked via codegen + +fun s106_int16(): slice asm "b{010000000000001111} PUSHSLICE"; + +@method_id(106) +fun test6(): int16 { + var u = U106.fromSlice(s106_int16()); + if (u is int16) { return u; } + else { return -1; } +} + + +fun main() {} + +/** +@testcase | 101 | | 0 +@testcase | 102 | | 0 +@testcase | 104 | | 0 +@testcase | 105 | | 9 +@testcase | 106 | | 15 + +@fif_codegen +""" + test6 PROC:<{ // + b{010000000000001111} PUSHSLICE // s + b{00} SDBEGINSQ // s '6 + IF:<{ // s + 8 LDI // '10 s + DROP // '10 + 42 PUSHINT // 'USlot1 'UTag=42 + }>ELSE<{ // s + b{01} SDBEGINSQ // s '6 + IF:<{ // s + 16 LDI // '15 s + DROP // '15 + 44 PUSHINT // 'USlot1 'UTag=44 + }>ELSE<{ // s + b{10} SDBEGINSQ // s '6 + IF:<{ // s + 32 LDI // '20 s + DROP // '20 + 46 PUSHINT // 'USlot1 'UTag=46 + }>ELSE<{ // s + b{11} SDBEGINSQ // s '6 + DROP // s + 64 LDI // '25 s + DROP // '25 + 48 PUSHINT // 'USlot1 'UTag=48 + }> + }> + }> // u.USlot1 u.UTag + 44 EQINT // u.USlot1 '27 + IFJMP:<{ // u.USlot1 + }> // u.USlot1 + DROP // + -1 PUSHINT // '29=-1 + }> +""" + */ + diff --git a/tolk-tester/tests/pack-unpack-5.tolk b/tolk-tester/tests/pack-unpack-5.tolk new file mode 100644 index 0000000000..068d031f23 --- /dev/null +++ b/tolk-tester/tests/pack-unpack-5.tolk @@ -0,0 +1,183 @@ +struct JustInt32 { + value: int32; +} + +type MaybeInt32 = int32?; +type MaybeCell = cell?; +type Int16Or32 = int16 | int32; + +struct JustMaybeInt32 { + value: MaybeInt32; +} + + +@method_id(101) +fun test1() { + return ( + int32.estimatePackSize(), + uint64.estimatePackSize(), + int1.estimatePackSize(), + bool.estimatePackSize(), + RemainingBitsAndRefs.estimatePackSize(), + coins.estimatePackSize(), + bits6.estimatePackSize(), + bytes8.estimatePackSize(), + ); +} + +@method_id(102) +fun test2() { + return ( + JustInt32.estimatePackSize(), + JustMaybeInt32.estimatePackSize(), + MaybeInt32.estimatePackSize(), + Int16Or32.estimatePackSize(), + ); +} + +struct Test3_1 { + f1: JustInt32, + f2: JustMaybeInt32, + f3: JustMaybeInt32, + f4: JustInt32 | uint100, +} + +struct Test3_2 { + f1: Test3_1; + f2: Test3_1?; + f3: null | Test3_1 | null; +} + +@method_id(103) +fun test3() { + return (Test3_1.estimatePackSize(), Test3_2.estimatePackSize()); +} + +struct Test4_1 { + f1: address; + f2: address; +} + +struct Test4_2 { + f1: address?; + f2: address?; +} + +@overflow1023_policy("suppress") +struct Test4_3 { + f: (Test4_1, Test4_2); +} + +@method_id(104) +fun test4() { + return (Test4_1.estimatePackSize(), Test4_2.estimatePackSize(), Test4_3.estimatePackSize()); +} + +struct Test5_1 { + f1: cell; + f2: cell?; + f3: Cell; + f4: Cell?; +} + +struct Test5_2 { + f1: Cell?; + f2: Cell?; + f3: Cell>?; + f4: Cell?; + f5: Cell?; + rest: RemainingBitsAndRefs; +} + +struct Test5_3 { + f1: int8 | Cell; + f2: bytes2 | Cell; + f3: Cell | cell; + f4: Cell | coins; +} + +@method_id(105) +fun test5() { + return (Test5_1.estimatePackSize(), Test5_2.estimatePackSize(), Test5_3.estimatePackSize()); +} + +struct(0x00112233) Test6_1 { + f1: int32; +} + +struct(0b0010) Test6_2 { + f1: int32?; + f2: Cell; +} + +type Test6_or = Test6_1 | Test6_2; + +@method_id(106) +fun test6() { + return (Test6_1.estimatePackSize(), Test6_2.estimatePackSize(), Test6_or.estimatePackSize()); +} + +struct(0x1020) Test7_1 { } +struct(0x1030) Test7_2 { } +struct(0x1040) Test7_3 { } + +type Test7_or = Test7_1 | Test7_2 | Test7_3 |; + +@method_id(107) +fun test7() { + assert((Test7_1{} as Test7_or).toCell().beginParse().remainingBitsCount() == Test7_or.estimatePackSize().0, 400); + return (Test7_1.estimatePackSize(), Test7_or.estimatePackSize()); +} + +struct(0x10) Inner8_1 { + ea: address; +} +struct(0b1) CellInner8_1 { + ref: Cell; +} +struct Inner8_2 { + t: (bits32, int1, coins?); +} + +@overflow1023_policy("suppress") +struct Test8 { + f1: Inner8_1; + f2: Inner8_2; + f3: Inner8_1?; + f4: Inner8_2?; + f5: Inner8_1 | CellInner8_1; +} + +@method_id(108) +fun test8() { + return (Inner8_1.estimatePackSize(), Inner8_2.estimatePackSize(), Test8.estimatePackSize()); +} + +struct Test9_bits2 { f: bits2; } +struct Test9_bits4 { f: bits4; } + +type Test9_f1 = int32 | int64 | int128; // auto-generated 2-bit prefix +type Test9_f2 = int32 | Inner8_2; // auto-generated 1-bit prefix (Either) +type Test9_f3 = bits1 | Test9_bits2 | bits3 | bits4 | bits5; // auto-generated 3-bit prefix +type Test9_f4 = bits1 | Test9_bits2 | bits3 | Test9_bits4?; // auto-generated 3-bit prefix + +@method_id(109) +fun test9() { + return (Test9_f1.estimatePackSize(), Test9_f2.estimatePackSize(), Test9_f3.estimatePackSize(), Test9_f4.estimatePackSize()); +} + +fun main() { + __expect_type(int8.estimatePackSize(), "[int, int, int, int]"); +} + +/** +@testcase | 101 | | [ 32 32 0 0 ] [ 64 64 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 0 0 0 0 ] [ 4 124 0 0 ] [ 6 6 0 0 ] [ 64 64 0 0 ] +@testcase | 102 | | [ 32 32 0 0 ] [ 1 33 0 0 ] [ 1 33 0 0 ] [ 17 33 0 0 ] +@testcase | 103 | | [ 67 199 0 0 ] [ 69 599 0 0 ] +@testcase | 104 | | [ 4 534 0 0 ] [ 2 536 0 0 ] [ 6 1070 0 0 ] +@testcase | 105 | | [ 2 2 2 4 ] [ 5 5 0 5 ] [ 4 152 1 4 ] +@testcase | 106 | | [ 64 64 0 0 ] [ 5 37 1 1 ] [ 5 65 0 1 ] +@testcase | 107 | | [ 16 16 0 0 ] [ 16 16 0 0 ] +@testcase | 108 | | [ 10 275 0 0 ] [ 34 158 0 0 ] [ 47 1143 0 1 ] +@testcase | 109 | | [ 34 130 0 0 ] [ 33 159 0 0 ] [ 4 8 0 0 ] [ 3 7 0 0 ] + */ diff --git a/tolk-tester/tests/some-tests-2.tolk b/tolk-tester/tests/some-tests-2.tolk index b3b756b199..592cf96efc 100644 --- a/tolk-tester/tests/some-tests-2.tolk +++ b/tolk-tester/tests/some-tests-2.tolk @@ -89,6 +89,15 @@ fun test95() { return (cur, next); } +struct Point { + x: int, y: int +} + +@method_id(96) +fun test96(p: Point) { + debug.print(p); // works ok with non-1 stack width, checked via codegen +} + /** method_id | in | out @testcase | 0 | 101 15 | 100 1 @@ -153,4 +162,11 @@ fun test95() { next GETGLOB // g_cur g_next }> """ + +@fif_codegen +""" + test96 PROC:<{ + s1 DUMP s0 DUMP 2 BLKDROP + }> +""" */ diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index c6e992c270..7aca836882 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -6,6 +6,8 @@ set(TOLK_SOURCE ast.cpp ast-from-tokens.cpp constant-evaluator.cpp + pack-unpack-api.cpp + pack-unpack-serializers.cpp pipe-discover-parse-sources.cpp pipe-register-symbols.cpp pipe-resolve-identifiers.cpp @@ -16,6 +18,7 @@ set(TOLK_SOURCE pipe-refine-lvalue-for-mutate.cpp pipe-check-rvalue-lvalue.cpp pipe-check-pure-impure.cpp + pipe-check-serialized-fields.cpp pipe-constant-folding.cpp pipe-optimize-boolean-expr.cpp pipe-ast-to-legacy.cpp diff --git a/tolk/abscode.cpp b/tolk/abscode.cpp index 02215d8847..462aad7b2d 100644 --- a/tolk/abscode.cpp +++ b/tolk/abscode.cpp @@ -436,4 +436,15 @@ std::vector CodeBlob::create_var(TypePtr var_type, SrcLocation loc, s return ir_idx; } +var_idx_t CodeBlob::create_int(SrcLocation loc, int64_t value, const char* desc) { + vars.emplace_back(var_cnt, TypeDataInt::create(), std::string{}, loc); +#ifdef TOLK_DEBUG + vars.back().desc = desc; +#endif + var_idx_t ir_int = var_cnt; + var_cnt++; + emplace_back(loc, Op::_IntConst, std::vector{ir_int}, td::make_refint(value)); + return ir_int; +} + } // namespace tolk diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 28f0ba182e..6f4b53b795 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -226,10 +226,6 @@ static AnyTypeV parse_type_nullable(Lexer& lex) { } static AnyTypeV parse_type_expression(Lexer& lex) { - if (lex.tok() == tok_bitwise_or) { // `type T = | T1 | T2 | ...` (each per line) (like in TypeScript) - lex.next(); - } - AnyTypeV result = parse_type_nullable(lex); if (lex.tok() == tok_bitwise_or) { // `int | slice`, `Pair2 | (Pair3 | null)` @@ -237,6 +233,9 @@ static AnyTypeV parse_type_expression(Lexer& lex) { items.emplace_back(result); while (lex.tok() == tok_bitwise_or) { lex.next(); + if (lex.tok() == tok_clpar || lex.tok() == tok_clbracket || lex.tok() == tok_semicolon) { + break; // allow trailing `|` (not leading, like in TypeScript, because of tree-sitter) + } items.emplace_back(parse_type_nullable(lex)); } result = createV(result->loc, std::move(items)); @@ -336,9 +335,6 @@ static AnyV parse_parameter(Lexer& lex, AnyTypeV self_type) { } static AnyV parse_global_var_declaration(Lexer& lex, const std::vector>& annotations) { - if (!annotations.empty()) { - lex.error("@annotations are not applicable to global var declaration"); - } SrcLocation loc = lex.cur_location(); lex.expect(tok_global, "`global`"); lex.check(tok_identifier, "global variable name"); @@ -353,13 +349,21 @@ static AnyV parse_global_var_declaration(Lexer& lex, const std::vectorkind) { + case AnnotationKind::deprecated: + case AnnotationKind::custom: + break; + default: + v_annotation->error("this annotation is not applicable to global"); + } + } + return createV(loc, v_ident, declared_type); } static AnyV parse_constant_declaration(Lexer& lex, const std::vector>& annotations) { - if (!annotations.empty()) { - lex.error("@annotations are not applicable to global var declaration"); - } SrcLocation loc = lex.cur_location(); lex.expect(tok_const, "`const`"); lex.check(tok_identifier, "constant name"); @@ -376,13 +380,21 @@ static AnyV parse_constant_declaration(Lexer& lex, const std::vectorkind) { + case AnnotationKind::deprecated: + case AnnotationKind::custom: + break; + default: + v_annotation->error("this annotation is not applicable to constant"); + } + } + return createV(loc, v_ident, declared_type, init_value); } static AnyV parse_type_alias_declaration(Lexer& lex, const std::vector>& annotations) { - if (!annotations.empty()) { - lex.error("@annotations are not applicable to type alias declaration"); - } SrcLocation loc = lex.cur_location(); lex.expect(tok_type, "`type`"); lex.check(tok_identifier, "type name"); @@ -397,6 +409,17 @@ static AnyV parse_type_alias_declaration(Lexer& lex, const std::vectorkind) { + case AnnotationKind::deprecated: + case AnnotationKind::custom: + break; + default: + v_annotation->error("this annotation is not applicable to type alias"); + } + } + return createV(loc, v_ident, genericsT_list, underlying_type); } @@ -1303,6 +1326,7 @@ static V parse_annotation(Lexer& lex) { v_arg = createV(loc, {}); break; case AnnotationKind::deprecated: + case AnnotationKind::custom: // allowed with and without arguments; it's IDE-only, the compiler doesn't analyze @deprecated break; case AnnotationKind::method_id: @@ -1310,9 +1334,15 @@ static V parse_annotation(Lexer& lex) { throw ParseError(loc, "expecting `(number)` after " + static_cast(name)); } break; + case AnnotationKind::overflow1023_policy: { + if (!v_arg || v_arg->size() != 1 || v_arg->get_item(0)->kind != ast_string_const) { + throw ParseError(loc, "expecting `(\"policy_name\")` after " + static_cast(name)); + } + break; + } } - return createV(loc, kind, v_arg); + return createV(loc, name, kind, v_arg); } static AnyV parse_function_declaration(Lexer& lex, const std::vector>& annotations) { @@ -1436,7 +1466,7 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector parse_struct_body(Lexer& lex) { static AnyV parse_struct_declaration(Lexer& lex, const std::vector>& annotations) { SrcLocation loc = lex.cur_location(); - if (!annotations.empty()) { - lex.error("@annotations are not applicable to type alias declaration"); - } lex.expect(tok_struct, "`struct`"); + + AnyExprV opcode = nullptr; + if (lex.tok() == tok_oppar) { // struct(0x0012) CounterIncrement + lex.next(); + lex.check(tok_int_const, "opcode `0x...` or `0b...`"); + std::string_view opcode_str = lex.cur_str(); + if (!opcode_str.starts_with("0x") && !opcode_str.starts_with("0b")) { + lex.unexpected("opcode `0x...` or `0b...`"); + } + opcode = createV(lex.cur_location(), parse_tok_int_const(opcode_str), opcode_str); + lex.next(); + lex.expect(tok_clpar, "`)`"); + } else { + opcode = createV(lex.cur_location()); + } + lex.check(tok_identifier, "identifier"); auto v_ident = createV(lex.cur_location(), lex.cur_str()); lex.next(); @@ -1499,7 +1542,27 @@ static AnyV parse_struct_declaration(Lexer& lex, const std::vector(loc, v_ident, genericsT_list, parse_struct_body(lex)); + StructData::Overflow1023Policy overflow1023_policy = StructData::Overflow1023Policy::not_specified; + for (auto v_annotation : annotations) { + switch (v_annotation->kind) { + case AnnotationKind::deprecated: + case AnnotationKind::custom: + break; + case AnnotationKind::overflow1023_policy: { + std::string_view str = v_annotation->get_arg()->get_item(0)->as()->str_val; + if (str == "suppress") { + overflow1023_policy = StructData::Overflow1023Policy::suppress; + } else { + v_annotation->error("incorrect value for " + static_cast(v_annotation->name)); + } + break; + } + default: + v_annotation->error("this annotation is not applicable to struct"); + } + } + + return createV(loc, v_ident, genericsT_list, overflow1023_policy, opcode, parse_struct_body(lex)); } static AnyV parse_tolk_required_version(Lexer& lex) { diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index d72513906c..73ada41000 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -345,6 +345,8 @@ class ASTReplicator final { v_orig->loc, new_name_ident, clone(v_orig->genericsT_list), + v_orig->overflow1023_policy, + v_orig->has_opcode() ? static_cast(clone(v_orig->get_opcode())) : createV(v_orig->loc), clone(v_orig->get_struct_body()) ); } diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index 8ced76a0ba..395f956811 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -107,16 +107,6 @@ class ASTStringifier final : public ASTVisitor { static_assert(std::size(name_pairs) == ast_tolk_file + 1, "name_pairs needs to be updated"); - constexpr static std::pair annotation_kinds[] = { - {AnnotationKind::inline_simple, "@inline"}, - {AnnotationKind::inline_ref, "@inline_ref"}, - {AnnotationKind::method_id, "@method_id"}, - {AnnotationKind::pure, "@pure"}, - {AnnotationKind::deprecated, "@deprecated"}, - }; - - static_assert(std::size(annotation_kinds) == static_cast(AnnotationKind::unknown), "annotation_kinds needs to be updated"); - template constexpr static const char* ast_node_kind_to_string() { return name_pairs[node_kind].second; @@ -205,7 +195,7 @@ class ASTStringifier final : public ASTVisitor { case ast_if_statement: return v->as()->is_ifnot ? "ifnot" : ""; case ast_annotation: - return annotation_kinds[static_cast(v->as()->kind)].second; + return static_cast(v->as()->name); case ast_parameter: return static_cast(v->as()->param_name) + ": " + ast_type_node_to_string(v->as()->type_node); case ast_function_declaration: { diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 6e725b7816..59a18a1631 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -66,6 +66,12 @@ AnnotationKind Vertex::parse_kind(std::string_view name) { if (name == "@deprecated") { return AnnotationKind::deprecated; } + if (name == "@custom") { + return AnnotationKind::custom; + } + if (name == "@overflow1023_policy") { + return AnnotationKind::overflow1023_policy; + } return AnnotationKind::unknown; } diff --git a/tolk/ast.h b/tolk/ast.h index 0770d9b844..96fc96937d 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -141,6 +141,8 @@ enum class AnnotationKind { method_id, pure, deprecated, + custom, + overflow1023_policy, unknown, }; @@ -1208,15 +1210,16 @@ template<> // ast_annotation is @annotation above a declaration // example: `@pure fun ...` struct Vertex final : ASTOtherVararg { + std::string_view name; AnnotationKind kind; auto get_arg() const { return children.at(0)->as(); } static AnnotationKind parse_kind(std::string_view name); - Vertex(SrcLocation loc, AnnotationKind kind, V arg_probably_empty) + Vertex(SrcLocation loc, std::string_view name, AnnotationKind kind, V arg_probably_empty) : ASTOtherVararg(ast_annotation, loc, {arg_probably_empty}) - , kind(kind) {} + , name(name), kind(kind) {} }; template<> @@ -1336,20 +1339,24 @@ struct Vertex final : ASTOtherVararg { template<> // ast_struct_declaration is declaring a struct with fields (each having declared_type), like interfaces in TypeScript // example: `struct Storage { owner: User; validUntil: int }` +// example: `struct(0x0012) CounterIncrement { byValue: int32; }` (0x0012 is opcode, len 16) // currently, Tolk doesn't have "implements" or whatever, so struct declaration contains only body struct Vertex final : ASTOtherVararg { - StructPtr struct_ref = nullptr; // filled after register + StructPtr struct_ref = nullptr; // filled after register V genericsT_list; // exists for `Wrapper`; otherwise, nullptr + StructData::Overflow1023Policy overflow1023_policy; auto get_identifier() const { return children.at(0)->as(); } - auto get_struct_body() const { return children.at(1)->as(); } + bool has_opcode() const { return children.at(1)->kind != ast_empty_expression; } + auto get_opcode() const { return children.at(1)->as(); } + auto get_struct_body() const { return children.at(2)->as(); } Vertex* mutate() const { return const_cast(this); } void assign_struct_ref(StructPtr struct_ref); - Vertex(SrcLocation loc, V name_identifier, V genericsT_list, V struct_body) - : ASTOtherVararg(ast_struct_declaration, loc, {name_identifier, struct_body}) - , genericsT_list(genericsT_list) {} + Vertex(SrcLocation loc, V name_identifier, V genericsT_list, StructData::Overflow1023Policy overflow1023_policy, AnyExprV opcode, V struct_body) + : ASTOtherVararg(ast_struct_declaration, loc, {name_identifier, opcode, struct_body}) + , genericsT_list(genericsT_list), overflow1023_policy(overflow1023_policy) {} }; template<> diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 7bed41a56c..ebcd2de926 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1022,9 +1022,9 @@ static AsmOp compile_store_int(std::vector& res, std::vector auto& z = args[2]; if (z.is_int_const() && z.int_const > 0 && z.int_const <= 256) { z.unused(); - return exec_arg_op(loc, "ST"s + (sgnd ? 'I' : 'U'), z.int_const, 2, 1); + return exec_arg_op(loc, sgnd? "STI" : "STU", z.int_const, 2, 1); } - return exec_op(loc, "ST"s + (sgnd ? "IX" : "UX"), 3, 1); + return exec_op(loc, sgnd ? "STIX" : "STUX", 3, 1); } // fun slice.loadBits (mutate self, len: int): self asm(s len -> 1 0) "LDSLICEX" @@ -1043,6 +1043,22 @@ static AsmOp compile_fetch_slice(std::vector& res, std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(args.size() == 3 && res.size() == 2); + auto& prefix = args[1]; + auto& prefix_len = args[2]; + if (prefix.is_int_const() && prefix.int_const >= 0 && prefix.int_const->signed_fits_bits(50) && + prefix_len.is_int_const() && prefix_len.int_const > 0 && prefix_len.int_const->signed_fits_bits(16)) { + prefix.unused(); + prefix_len.unused(); + StructData::PackOpcode opcode(prefix.int_const->to_long(), static_cast(prefix_len.int_const->to_long())); + return AsmOp::Custom(loc, opcode.format_as_slice() + " SDBEGINSQ", 0, 1); + } + throw ParseError(loc, "slice.tryStripPrefix can be used only with constant arguments"); +} + // fun tuple.get(t: tuple, index: int): X asm "INDEXVAR"; static AsmOp compile_tuple_get(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 2 && res.size() == 1); @@ -1076,8 +1092,31 @@ static AsmOp compile_strdump(std::vector&, std::vector&, Src } // fun debug.print(x: T): void; -static AsmOp compile_debug_print_to_string(std::vector&, std::vector&, SrcLocation loc) { - return AsmOp::Custom(loc, "s0 DUMP DROP", 1, 1); +static AsmOp compile_debug_print_to_string(std::vector&, std::vector& args, SrcLocation loc) { + int n = static_cast(args.size()); + if (n == 1) { // most common case + return AsmOp::Custom(loc, "s0 DUMP DROP", 1, 1); + } + if (n > 15) { + throw ParseError(loc, "call overflow, exceeds 15 elements"); + } + std::string cmd; + for (int i = n - 1; i >= 0; --i) { + cmd += "s" + std::to_string(i) + " DUMP "; + } + cmd += std::to_string(n); + cmd += " BLKDROP"; + return AsmOp::Custom(loc, cmd, n, n); +} + +// fun T.__toTuple(self): void; (T can be any number of slots, it works for structs and tensors) +static AsmOp compile_any_object_to_tuple(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 1); + int n = static_cast(args.size()); + if (n > 15) { + throw ParseError(loc, "call overflow, exceeds 15 elements"); + } + return exec_op(loc, std::to_string(args.size()) + " TUPLE", n, 1); } // fun ton(amount: slice): coins; ton("0.05") replaced by 50000000 at compile-time @@ -1120,6 +1159,7 @@ void define_builtins() { TypePtr typeT = TypeDataGenericT::create("T"); const GenericsDeclaration* declGenericT = new GenericsDeclaration(std::vector{{"T", nullptr}}, 0); + const GenericsDeclaration* declReceiverT = new GenericsDeclaration(std::vector{{"T", nullptr}}, 1); std::vector ParamsInt1 = {Int}; std::vector ParamsInt2 = {Int, Int}; @@ -1129,6 +1169,7 @@ void define_builtins() { // these types are defined in stdlib, currently unknown // see patch_builtins_after_stdlib_loaded() below TypePtr debug = TypeDataUnknown::create(); + TypePtr CellT = TypeDataUnknown::create(); // builtin operators // they are internally stored as functions, because at IR level, there is no difference @@ -1247,28 +1288,28 @@ void define_builtins() { // note their parameter being `unknown`: in order to `ton(1)` pass type inferring but fire a more gentle error later define_builtin_func("ton", {TypeDataUnknown::create()}, TypeDataCoins::create(), nullptr, compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); define_builtin_func("stringCrc32", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); define_builtin_func("stringCrc16", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); define_builtin_func("stringSha256", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); define_builtin_func("stringSha256_32", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); define_builtin_func("stringToBase256", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); define_builtin_func("stringHexToSlice", {TypeDataUnknown::create()}, TypeDataSlice::create(), nullptr, compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); define_builtin_func("address", {TypeDataUnknown::create()}, TypeDataAddress::create(), nullptr, compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); // functions from stdlib marked as `builtin`, implemented at compiler level for optimizations // (for example, `loadInt(1)` is `1 LDI`, but `loadInt(n)` for non-constant requires it be on a stack and `LDIX`) @@ -1305,6 +1346,9 @@ void define_builtins() { define_builtin_method("slice.preloadBits", Slice, ParamsSliceInt, Slice, nullptr, std::bind(compile_fetch_slice, _1, _2, _3, false), FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); + define_builtin_method("slice.tryStripPrefix", Slice, {Slice, Int, Int}, Bool, nullptr, + compile_slice_sdbeginsq, + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf); define_builtin_method("builder.storeInt", Builder, {Builder, Int, Int}, Unit, nullptr, std::bind(compile_store_int, _1, _2, _3, true), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, @@ -1321,7 +1365,7 @@ void define_builtins() { FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf); define_builtin_method("debug.print", debug, {typeT}, Unit, declGenericT, compile_debug_print_to_string, - 0); + FunctionData::flagAllowAnyWidthT); define_builtin_method("debug.printString", debug, {typeT}, Unit, declGenericT, compile_strdump, 0); @@ -1329,12 +1373,42 @@ void define_builtins() { compile_dumpstk, 0); + // serialization/deserialization methods to/from cells (or, more low-level, slices/builders) + // they work with structs (or, more low-level, with arbitrary types) + define_builtin_method("T.toCell", typeT, {typeT}, CellT, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("T.fromCell", typeT, {TypeDataCell::create()}, typeT, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + define_builtin_method("T.fromSlice", typeT, {Slice}, typeT, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + define_builtin_method("T.estimatePackSize", typeT, {}, TypeDataBrackets::create({TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create()}), declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + define_builtin_method("Cell.load", CellT, {CellT}, typeT, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("slice.loadAny", Slice, {Slice}, typeT, declGenericT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("slice.skipAny", Slice, {Slice}, Slice, declGenericT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("builder.storeAny", Builder, {Builder, typeT}, Builder, declGenericT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + // functions not presented in stdlib at all // used in tolk-tester to check/expose internal compiler state // each of them is handled in a special way, search by its name define_builtin_func("__expect_type", {TypeDataUnknown::create(), Slice}, Unit, nullptr, compile_expect_type, FunctionData::flagMarkedAsPure); + define_builtin_method("T.__toTuple", typeT, {typeT}, TypeDataTuple::create(), declReceiverT, + compile_any_object_to_tuple, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); } // there are some built-in functions that operate on types declared in stdlib (like Cell) @@ -1342,12 +1416,20 @@ void define_builtins() { // after all files have been loaded, symbols have been registered, and aliases exist, // we patch that earlier registered built-in functions providing types that now exist void patch_builtins_after_stdlib_loaded() { + TypePtr typeT = TypeDataGenericT::create("T"); StructPtr struct_debug = lookup_global_symbol("debug")->try_as(); TypePtr debug = TypeDataStruct::create(struct_debug); lookup_function("debug.print")->mutate()->receiver_type = debug; lookup_function("debug.printString")->mutate()->receiver_type = debug; lookup_function("debug.dumpStack")->mutate()->receiver_type = debug; + + StructPtr struct_ref_CellT = lookup_global_symbol("Cell")->try_as(); + TypePtr CellT = TypeDataGenericTypeWithTs::create(struct_ref_CellT, nullptr, {typeT}); + + lookup_function("Cell.load")->mutate()->parameters[0].declared_type = CellT; + lookup_function("Cell.load")->mutate()->receiver_type = CellT; + lookup_function("T.toCell")->mutate()->declared_return_type = CellT; } } // namespace tolk diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index a68de6c7c8..c6e2f43313 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -252,7 +252,7 @@ struct ConstantExpressionChecker { // `const a = ton("0.05")`, we met `ton("0.05")` static void handle_function_call(V v) { - if (v->fun_maybe && v->fun_maybe->is_compile_time_only()) { + if (v->fun_maybe && v->fun_maybe->is_compile_time_const_val()) { // `ton(local_var)` is denied; it's validated not here, but when replacing its value with a calculated one return; } @@ -318,7 +318,7 @@ std::string eval_string_const_standalone(AnyExprV v_string) { CompileTimeFunctionResult eval_call_to_compile_time_function(AnyExprV v_call) { auto v = v_call->try_as(); - tolk_assert(v && v->fun_maybe->is_compile_time_only()); + tolk_assert(v && v->fun_maybe->is_compile_time_const_val()); return parse_vertex_call_to_compile_time_function(v, v->fun_maybe->name); } diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp new file mode 100644 index 0000000000..14a20d9222 --- /dev/null +++ b/tolk/pack-unpack-api.cpp @@ -0,0 +1,257 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "pack-unpack-api.h" +#include "generics-helpers.h" +#include "type-system.h" +#include + +/* + * This module provides high-level (de)serialization functions to be used from outer code: + * - pack to cell/builder + * - unpack from cell/slice + * - etc. + * + * For the implementation of packing primitives, consider `pack-unpack-serializers.cpp`. + */ + +namespace tolk { + + +// -------------------------------------------- +// checking serialization availability +// +// for every call `obj.toCell()` and similar, checks are executed to ensure that `obj` can be serialized +// if it can, the compilation process continues +// if not, a detailed explanation is shown +// + + +struct CantSerializeBecause { + std::string because_msg; + + explicit CantSerializeBecause(std::string because_msg) + : because_msg(std::move(because_msg)) {} + explicit CantSerializeBecause(const std::string& because_msg, const CantSerializeBecause& why) + : because_msg(because_msg + "\n" + why.because_msg) {} +}; + +class PackUnpackAvailabilityChecker { +public: + static std::optional detect_why_cant_serialize(TypePtr any_type, bool is_pack) { + if (any_type->try_as()) { + return {}; + } + if (any_type->try_as()) { + return {}; + } + if (any_type == TypeDataCoins::create()) { + return {}; + } + if (any_type == TypeDataBool::create()) { + return {}; + } + if (any_type == TypeDataCell::create()) { + return {}; + } + if (any_type == TypeDataAddress::create()) { + return {}; + } + if (any_type == TypeDataNever::create()) { + return {}; + } + + if (const auto* t_struct = any_type->try_as()) { + StructPtr struct_ref = t_struct->struct_ref; + for (StructFieldPtr field_ref : struct_ref->fields) { + if (auto why = detect_why_cant_serialize(field_ref->declared_type, is_pack)) { + return CantSerializeBecause("because field `" + struct_ref->name + "." + field_ref->name + "` of type `" + field_ref->declared_type->as_human_readable() + "` can't be serialized", why.value()); + } + } + if (is_type_cellT(t_struct)) { + TypePtr cellT = struct_ref->substitutedTs->typeT_at(0); + if (auto why = detect_why_cant_serialize(cellT, is_pack)) { + return CantSerializeBecause("because type `" + cellT->as_human_readable() + "` can't be serialized", why.value()); + } + } + return {}; + } + + if (const auto* t_union = any_type->try_as()) { + // a union can almost always be serialized if every of its variants can: + // - `T?` is TL/B `(Maybe T)` + // - `T1 | T2` is TL/B `(Either T1 T2)` (or, if opcodes manually set, just by opcodes) + // - `T1 | T2 | ...` is either by manual opcodes, or the compiler implicitly defines them + // so, even `int32 | int64 | int128` or `A | B | C | null` are serializable + // (unless corner cases occur, like duplicated opcodes, etc.) + for (int i = 0; i < t_union->size(); ++i) { + TypePtr variant = t_union->variants[i]; + if (variant == TypeDataNullLiteral::create()) { + continue; + } + if (auto why = detect_why_cant_serialize(variant, is_pack)) { + return CantSerializeBecause("because variant #" + std::to_string(i + 1) + " of type `" + variant->as_human_readable() + "` can't be serialized", why.value()); + } + } + if (!t_union->or_null) { + std::string because_msg; + auto_generate_opcodes_for_union(t_union, because_msg); + if (!because_msg.empty()) { + return CantSerializeBecause("because could not automatically generate serialization prefixes for a union\n" + because_msg); + } + } + return {}; + } + + if (const auto* t_tensor = any_type->try_as()) { + for (int i = 0; i < t_tensor->size(); ++i) { + if (auto why = detect_why_cant_serialize(t_tensor->items[i], is_pack)) { + return CantSerializeBecause("because element `tensor." + std::to_string(i) + "` of type `" + t_tensor->items[i]->as_human_readable() + "` can't be serialized", why.value()); + } + } + return {}; + } + + if (const auto* t_alias = any_type->try_as()) { + if (t_alias->alias_ref->name == "RemainingBitsAndRefs") { // it's built-in RemainingBitsAndRefs (slice) + return {}; + } + if (auto why = detect_why_cant_serialize(t_alias->underlying_type, is_pack)) { + return CantSerializeBecause("because alias `" + t_alias->as_human_readable() + "` expands to `" + t_alias->underlying_type->as_human_readable() + "`", why.value()); + } + return {}; + } + + // `builder` can be used for writing, but not for reading + if (any_type == TypeDataBuilder::create()) { + if (is_pack) { + return {}; + } + return CantSerializeBecause("because type `builder` can not be used for reading, only for writing\nhint: use `bitsN` or `RemainingBitsAndRefs` for reading\nhint: using generics, you can substitute `builder` for writing and something other for reading"); + } + + // serialization not available + // for common types, make a detailed explanation with a hint how to fix + + if (any_type == TypeDataInt::create()) { + return CantSerializeBecause("because type `int` is not serializable, it doesn't define binary width\nhint: replace `int` with `int32` / `uint64` / `coins` / etc."); + } + if (any_type == TypeDataSlice::create()) { + return CantSerializeBecause("because type `slice` is not serializable, it doesn't define binary width\nhint: replace `slice` with `address` if it's an address, actually\nhint: replace `slice` with `bits128` and similar if it represents fixed-width data without refs"); + } + if (any_type == TypeDataNullLiteral::create()) { + return CantSerializeBecause("because type `null` is not serializable\nhint: `int32?` and other nullable types will work"); + } + if (any_type == TypeDataTuple::create() || any_type->try_as()) { + return CantSerializeBecause("because tuples are not serializable\nhint: use tensors instead of tuples, they will work"); + } + + return CantSerializeBecause("because type `" + any_type->as_human_readable() + "` is not serializable"); + } +}; + +bool check_struct_can_be_packed_or_unpacked(TypePtr any_type, bool is_pack, std::string& because_msg) { + if (auto why = PackUnpackAvailabilityChecker::detect_why_cant_serialize(any_type, is_pack)) { + because_msg = why.value().because_msg; + return false; + } + return true; +} + + +// -------------------------------------------- +// high-level API for outer code +// + + +std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_obj) { + FunctionPtr f_beginCell = lookup_function("beginCell"); + FunctionPtr f_endCell = lookup_function("builder.endCell"); + std::vector rvect_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); + code.emplace_back(loc, Op::_Call, rvect_builder, std::vector{}, f_beginCell); + + PackContext ctx(code, loc, rvect_builder); + ctx.generate_pack_any(any_type, std::move(ir_obj)); + + std::vector rvect_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(cell)"); + code.emplace_back(loc, Op::_Call, rvect_cell, std::move(rvect_builder), f_endCell); + + return rvect_cell; +} + +std::vector generate_pack_struct_to_builder(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_builder, std::vector&& ir_obj) { + PackContext ctx(code, loc, ir_builder); // mutate this builder + ctx.generate_pack_any(any_type, std::move(ir_obj)); + + return ir_builder; // return mutated builder +} + +std::vector generate_unpack_struct_from_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, bool mutate_slice) { + if (!mutate_slice) { + std::vector slice_copy = code.create_var(TypeDataSlice::create(), loc, "s"); + code.emplace_back(loc, Op::_Let, slice_copy, std::move(ir_slice)); + ir_slice = std::move(slice_copy); + } + + UnpackContext ctx(code, loc, std::move(ir_slice)); + std::vector rvect_struct = ctx.generate_unpack_any(any_type); + tolk_assert(any_type->get_width_on_stack() == static_cast(rvect_struct.size())); + + return rvect_struct; +} + +std::vector generate_unpack_struct_from_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_cell) { + FunctionPtr f_beginParse = lookup_function("cell.beginParse"); + std::vector ir_slice = code.create_var(TypeDataSlice::create(), loc, "s"); + code.emplace_back(loc, Op::_Call, ir_slice, std::move(ir_cell), f_beginParse); + + UnpackContext ctx(code, loc, std::move(ir_slice)); + std::vector rvect_struct = ctx.generate_unpack_any(any_type); + tolk_assert(any_type->get_width_on_stack() == static_cast(rvect_struct.size())); + + return rvect_struct; +} + +std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice) { + UnpackContext ctx(code, loc, ir_slice); // mutate this slice + ctx.generate_skip_any(any_type); + + return ir_slice; // return mutated slice +} + +PackSize estimate_serialization_size(TypePtr any_type) { + EstimateContext ctx; + return ctx.estimate_any(any_type); +} + +std::vector generate_estimate_size_call(CodeBlob& code, SrcLocation loc, TypePtr any_type) { + EstimateContext ctx; + PackSize pack_size = ctx.estimate_any(any_type); + + std::vector ir_tensor = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create()}), loc, "(result-tensor)"); + code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[0]}, td::make_refint(pack_size.min_bits)); + code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[1]}, td::make_refint(pack_size.max_bits)); + code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[2]}, td::make_refint(pack_size.min_refs)); + code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[3]}, td::make_refint(pack_size.max_refs)); + + FunctionPtr f_toTuple = lookup_function("T.__toTuple"); + std::vector ir_tuple = code.create_tmp_var(TypeDataTuple::create(), loc, "(result-tuple)"); + code.emplace_back(loc, Op::_Call, ir_tuple, ir_tensor, f_toTuple); + + return ir_tuple; +} + +} // namespace tolk diff --git a/tolk/pack-unpack-api.h b/tolk/pack-unpack-api.h new file mode 100644 index 0000000000..b0a4b29efe --- /dev/null +++ b/tolk/pack-unpack-api.h @@ -0,0 +1,35 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "pack-unpack-serializers.h" +#include "tolk.h" + +namespace tolk { + +bool check_struct_can_be_packed_or_unpacked(TypePtr any_type, bool is_pack, std::string& because_msg); + +std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_obj); +std::vector generate_pack_struct_to_builder(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_builder, std::vector&& ir_obj); +std::vector generate_unpack_struct_from_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, bool mutate_slice); +std::vector generate_unpack_struct_from_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_cell); +std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice); + +PackSize estimate_serialization_size(TypePtr any_type); +std::vector generate_estimate_size_call(CodeBlob& code, SrcLocation loc, TypePtr any_type); + +} // namespace tolk diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp new file mode 100644 index 0000000000..8be3d99952 --- /dev/null +++ b/tolk/pack-unpack-serializers.cpp @@ -0,0 +1,1031 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "pack-unpack-serializers.h" +#include "tolk.h" +#include "type-system.h" +#include "td/utils/crypto.h" + +/* + * This module implements serializing different types to/from cells. + * For any serializable TypePtr, we detect ISerializer, which can pack/unpack/skip/estimate size. + * See `get_serializer_for_type()`. + * Example: given an object of `struct A { f: int32 }` its type is TypeDataStruct(A), its serializer is + * "custom struct", which iterates fields, for field `f` its serializer is "intN" with N=32. + * + * Serializing compound types is complicated, involving transitioning IR variables. For example, to serialize + * `int8 | A` (it's Either), we have input rvect of size = 1 + width(A), generate dynamic IF ELSE, and in each branch, + * transition rvect slots to a narrowed type. Operating with transitions and runtime type checking are implemented + * in IR generation, here we just reference those prototypes. + * + * For high-level (de)serialization API, consider `pack-unpack-api.cpp`. + */ + +namespace tolk { + + +std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc); +std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc); + +constexpr int ERR_CODEGEN_ASSERT = 9; + +bool is_type_cellT(TypePtr any_type) { + if (const TypeDataStruct* t_struct = any_type->try_as()) { + StructPtr struct_ref = t_struct->struct_ref; + return struct_ref->is_instantiation_of_generic_struct() && struct_ref->base_struct_ref->name == "Cell"; + } + return false; +} + + +// -------------------------------------------- +// options, context, common helpers +// +// some of the referenced functions are built-in, some are declared in stdlib +// serialization assumes that stdlib exists and is loaded correctly +// + + +PackContext::PackContext(CodeBlob& code, SrcLocation loc, std::vector ir_builder) + : code(code) + , loc(loc) + , f_storeInt(lookup_function("builder.storeInt")) + , f_storeUint(lookup_function("builder.storeUint")) + , ir_builder(std::move(ir_builder)) + , ir_builder0(this->ir_builder[0]) { +} + +void PackContext::storeInt(var_idx_t ir_idx, int len) const { + std::vector args = { ir_builder0, ir_idx, code.create_int(loc, len, "(storeW)") }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), f_storeInt); +} + +void PackContext::storeUint(var_idx_t ir_idx, int len) const { + std::vector args = { ir_builder0, ir_idx, code.create_int(loc, len, "(storeW)") }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), f_storeUint); +} + +void PackContext::storeBool(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeBool")); +} + +void PackContext::storeCoins(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeCoins")); +} + +void PackContext::storeRef(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeRef")); +} + +void PackContext::storeMaybeRef(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeMaybeRef")); +} + +void PackContext::storeAddress(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeAddress")); +} + +void PackContext::storeBuilder(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeBuilder")); +} + +void PackContext::storeSlice(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeSlice")); +} + +void PackContext::storeOpcode(PackOpcode opcode) const { + std::vector args = { ir_builder0, code.create_int(loc, opcode.pack_prefix, "(struct-prefix)"), code.create_int(loc, opcode.prefix_len, "(storeW)") }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), f_storeUint); +} + + +UnpackContext::UnpackContext(CodeBlob& code, SrcLocation loc, std::vector ir_slice) + : code(code) + , loc(loc) + , f_loadInt(lookup_function("slice.loadInt")) + , f_loadUint(lookup_function("slice.loadUint")) + , f_skipBits(lookup_function("slice.skipBits")) + , ir_slice(std::move(ir_slice)) + , ir_slice0(this->ir_slice[0]) { +} + +std::vector UnpackContext::loadInt(int len, const char* debug_desc) const { + std::vector args = { ir_slice0, code.create_int(loc, len, "(loadW)") }; + std::vector result = code.create_tmp_var(TypeDataInt::create(), loc, debug_desc); + code.emplace_back(loc, Op::_Call, std::vector{ir_slice0, result[0]}, std::move(args), f_loadInt); + return result; +} + +std::vector UnpackContext::loadUint(int len, const char* debug_desc) const { + std::vector args = { ir_slice0, code.create_int(loc, len, "(loadW)") }; + std::vector result = code.create_tmp_var(TypeDataInt::create(), loc, debug_desc); + code.emplace_back(loc, Op::_Call, std::vector{ir_slice0, result[0]}, std::move(args), f_loadUint); + return result; +} + +void UnpackContext::loadAndCheckOpcode(PackOpcode opcode) const { + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); + std::vector args = { ir_slice0, code.create_int(loc, opcode.pack_prefix, "(pack-prefix)"), code.create_int(loc, opcode.prefix_len, "(prefix-len)") }; + code.emplace_back(loc, Op::_Call, std::vector{ir_slice0, ir_prefix_eq[0]}, std::move(args), lookup_function("slice.tryStripPrefix")); + + std::vector args_assert = { code.create_int(loc, ERR_CODEGEN_ASSERT, "(excno)"), ir_prefix_eq[0], code.create_int(loc, 0, "") }; + Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); + op_assert.set_impure_flag(); +} + +void UnpackContext::skipBits(int len) const { + std::vector args = { ir_slice0, code.create_int(loc, len, "(skipW)") }; + code.emplace_back(loc, Op::_Call, ir_slice, std::move(args), f_skipBits); +} + +void UnpackContext::skipBits_var(var_idx_t ir_len) const { + std::vector args = { ir_slice0, ir_len }; + code.emplace_back(loc, Op::_Call, ir_slice, std::move(args), f_skipBits); +} + + +// -------------------------------------------- +// serializers with pack/unpack/skip/estimate +// +// for every struct field, for every atomic type, a corresponding (de)serialization instruction is generated +// we generate IR code (Ops), not ASM directly; so, all later IR analysis will later take place +// some of them are straightforward, e.g., call a predefined function for intN and coins +// some are complicated, e.g., for Either we should check a union type at runtime while packing, +// and while unpacking, read a prefix, follow different branches, and construct a resulting union +// + + +struct ISerializer { + virtual ~ISerializer() = default; + + virtual void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) = 0; + virtual std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) = 0; + + virtual void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) = 0; + virtual PackSize estimate(const EstimateContext* ctx) = 0; +}; + +struct S_IntN final : ISerializer { + const int n_bits; + const bool is_unsigned; + + explicit S_IntN(int n_bits, bool is_unsigned) + : n_bits(n_bits), is_unsigned(is_unsigned) {} + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + if (is_unsigned) { + ctx->storeUint(rvect[0], n_bits); + } else { + ctx->storeInt(rvect[0], n_bits); + } + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + if (is_unsigned) { + return ctx->loadUint(n_bits, "(loaded-uint)"); + } else { + return ctx->loadInt(n_bits, "(loaded-int)"); + } + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + ctx->skipBits(n_bits); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(n_bits); + } +}; + +struct S_BytesN final : ISerializer { + const int n_bits; + + explicit S_BytesN(int n_width, bool is_bits) + : n_bits(is_bits ? n_width : n_width * 8) {} + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + + // bitsN/bytesN aren't checked upon writing (to have requested number of bits and 0 refs), probably to expose an option + if (false) { + FunctionPtr f_assert = lookup_function("__throw_if_unless"); + FunctionPtr f_equals = lookup_function("_==_"); + FunctionPtr f_bitAnd = lookup_function("_&_"); + FunctionPtr f_getCounts = lookup_function("slice.remainingBitsAndRefsCount"); + + std::vector ir_counts = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create()}), loc, "(slice-size)"); + code.emplace_back(loc, Op::_Call, ir_counts, rvect, f_getCounts); + std::vector ir_is_n = code.create_tmp_var(TypeDataInt::create(), loc, "(n-bits)"); + code.emplace_back(loc, Op::_Call, ir_is_n, std::vector{ir_counts[0], code.create_int(loc, n_bits, "(expected-bits)")}, f_equals); + std::vector ir_is_0 = code.create_tmp_var(TypeDataInt::create(), loc, "(n-refs)"); + code.emplace_back(loc, Op::_Call, ir_is_0, std::vector{ir_counts[1], code.create_int(loc, 0, "(expected-refs)")}, f_equals); + std::vector ir_both_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(both-eq)"); + code.emplace_back(loc, Op::_Call, ir_both_eq, std::vector{ir_is_n[0], ir_is_0[0]}, f_bitAnd); + std::vector args_assert = { code.create_int(loc, ERR_CODEGEN_ASSERT, "(excno)"), ir_both_eq[0], code.create_int(loc, 0, "") }; + Op& op1 = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), f_assert); + op1.set_impure_flag(); + } + + ctx->storeSlice(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadBits = lookup_function("slice.loadBits"); + std::vector args = { ctx->ir_slice0, code.create_int(loc, n_bits, "(loadW)") }; + std::vector ir_result = code.create_tmp_var(TypeDataSlice::create(), loc, "(loaded-slice)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadBits); + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + ctx->skipBits(n_bits); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(n_bits); + } +}; + +struct S_Bool final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeBool(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + return ctx->loadInt(1, "(loaded-bool)"); + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + ctx->skipBits(1); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(1); + } +}; + +struct S_RawTVMcell final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeRef(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadRef = lookup_function("slice.loadRef"); + std::vector args = ctx->ir_slice; + std::vector ir_result = code.create_tmp_var(TypeDataCell::create(), loc, "(loaded-cell)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadRef); + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadRef = lookup_function("slice.loadRef"); + std::vector args = ctx->ir_slice; + std::vector dummy_loaded = code.create_tmp_var(TypeDataCell::create(), loc, "(loaded-cell)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, dummy_loaded[0]}, std::move(args), f_loadRef); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(0, 0, 1, 1); + } +}; + +struct S_RawTVMcellOrNull final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeMaybeRef(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadMaybeRef = lookup_function("slice.loadMaybeRef"); + std::vector args = ctx->ir_slice; + std::vector ir_result = code.create_tmp_var(TypeDataCell::create(), loc, "(loaded-cell)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadMaybeRef); + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_skipMaybeRef = lookup_function("slice.skipMaybeRef"); + code.emplace_back(loc, Op::_Call, ctx->ir_slice, ctx->ir_slice, f_skipMaybeRef); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(1, 1, 0, 1); + } +}; + +struct S_Coins final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeCoins(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadCoins = lookup_function("slice.loadCoins"); + std::vector args = ctx->ir_slice; + std::vector ir_result = code.create_tmp_var(TypeDataInt::create(), loc, "(loaded-coins)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadCoins); + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadCoins = lookup_function("slice.loadCoins"); + std::vector args = ctx->ir_slice; + std::vector dummy_loaded = code.create_tmp_var(TypeDataInt::create(), loc, "(loaded-coins)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, dummy_loaded[0]}, std::move(args), f_loadCoins); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(4, 124); + } +}; + +struct S_Address final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeAddress(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadAddress = lookup_function("slice.loadAddress"); + std::vector ir_address = code.create_tmp_var(TypeDataSlice::create(), loc, "(loaded-addr)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_address[0]}, ctx->ir_slice, f_loadAddress); + return ir_address; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + // we can't do just + // ctx->skipBits(2 + 1 + 8 + 256); + // because it may be addr_none or addr_extern; there is no "skip address" in TVM, so just load it + FunctionPtr f_loadAddress = lookup_function("slice.loadAddress"); + std::vector ir_address = code.create_tmp_var(TypeDataSlice::create(), loc, "(tmp-addr)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_address[0]}, ctx->ir_slice, f_loadAddress); + } + + PackSize estimate(const EstimateContext* ctx) override { + // we can't do just + // return PackSize(2 + 1 + 8 + 256); + // because it may be addr_none or addr_extern; but since addr_extern is very-very uncommon, don't consider it + return PackSize(2, 2 + 1 + 8 + 256); + } +}; + +struct S_RemainingBitsAndRefs final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeSlice(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + return ctx->ir_slice; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_beginCell = lookup_function("beginCell"); + FunctionPtr f_endCell = lookup_function("builder.endCell"); + FunctionPtr f_beginParse = lookup_function("cell.beginParse"); + + std::vector ir_builder = code.create_tmp_var(TypeDataBuilder::create(), loc, "(tmp-builder)"); + std::vector ir_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(tmp-cell)"); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, f_beginCell); + code.emplace_back(loc, Op::_Call, ir_cell, ir_builder, f_endCell); + code.emplace_back(loc, Op::_Call, ctx->ir_slice, ir_cell, f_beginParse); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(0, 0, 0, 0); + } +}; + +struct S_Builder final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + std::vector args = { ctx->ir_builder0, rvect[0] }; + code.emplace_back(loc, Op::_Call, ctx->ir_builder, std::move(args), lookup_function("builder.storeBuilder")); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + tolk_assert(false); // `builder` can only be used for writing, checked earlier + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + tolk_assert(false); // `builder` can only be used for writing, checked earlier + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(0, 0, 0, 0); + } +}; + +struct S_Null final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + // while `null` itself is not serializable, it may be contained inside a union: + // `int32 | int64 | null`, for example; + // then the compiler generates prefixes for every variant, and `null` variant does nothing + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector ir_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null)"); + code.emplace_back(loc, Op::_Call, ir_null, std::vector{}, lookup_function("__null")); + return ir_null; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(0); + } +}; + +struct S_Never final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.empty()); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + return {}; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(0); + } +}; + +struct S_Maybe final : ISerializer { + const TypeDataUnion* t_union; + TypePtr or_null; + + explicit S_Maybe(const TypeDataUnion* t_union) + : t_union(t_union) + , or_null(t_union->or_null) { + } + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + std::vector ir_is_null = pre_compile_is_type(code, t_union, TypeDataNullLiteral::create(), rvect, loc, "(is-null)"); + Op& if_op = code.emplace_back(loc, Op::_If, ir_is_null); + { + code.push_set_cur(if_op.block0); + ctx->storeUint(code.create_int(loc, 0, "(maybeBit)"), 1); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + ctx->storeUint(code.create_int(loc, 1, "(maybeBit)"), 1); + rvect = transition_to_target_type(std::move(rvect), code, t_union, t_union->or_null, loc); + ctx->generate_pack_any(t_union->or_null, std::move(rvect)); + code.close_pop_cur(loc); + } + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector ir_result = code.create_tmp_var(t_union, loc, "(loaded-maybe)"); + std::vector ir_not_null = { ctx->loadUint(1, "(maybeBit)") }; + Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_not_null)); + { + code.push_set_cur(if_op.block0); + std::vector rvect_maybe = ctx->generate_unpack_any(t_union->or_null); + rvect_maybe = transition_to_target_type(std::move(rvect_maybe), code, t_union->or_null, t_union, loc); + code.emplace_back(loc, Op::_Let, ir_result, std::move(rvect_maybe)); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + std::vector rvect_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(maybe-null)"); + code.emplace_back(loc, Op::_Call, rvect_null, std::vector{}, lookup_function("__null")); + rvect_null = transition_to_target_type(std::move(rvect_null), code, TypeDataNullLiteral::create(), t_union, loc); + code.emplace_back(loc, Op::_Let, ir_result, std::move(rvect_null)); + code.close_pop_cur(loc); + } + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector ir_not_null = { ctx->loadUint(1, "(maybeBit)") }; + Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_not_null)); + { + code.push_set_cur(if_op.block0); + ctx->generate_skip_any(t_union->or_null); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + code.close_pop_cur(loc); + } + } + + PackSize estimate(const EstimateContext* ctx) override { + PackSize maybe_size = ctx->estimate_any(t_union->or_null); + return PackSize(1, 1 + maybe_size.max_bits, 0, maybe_size.max_refs); + } +}; + +struct S_Either final : ISerializer { + const TypeDataUnion* t_union; + TypePtr t_left; + TypePtr t_right; + + explicit S_Either(const TypeDataUnion* t_union) + : t_union(t_union) + , t_left(t_union->variants[0]) + , t_right(t_union->variants[1]) { + } + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + std::vector ir_is_right = pre_compile_is_type(code, t_union, t_right, rvect, loc, "(is-right)"); + Op& if_op = code.emplace_back(loc, Op::_If, ir_is_right); + { + code.push_set_cur(if_op.block0); + ctx->storeUint(code.create_int(loc, 1, "(eitherBit)"), 1); + std::vector rvect_right = transition_to_target_type(std::vector(rvect), code, t_union, t_right, loc); + ctx->generate_pack_any(t_right, std::move(rvect_right)); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + ctx->storeUint(code.create_int(loc, 0, "(eitherBit)"), 1); + std::vector rvect_left = transition_to_target_type(std::move(rvect), code, t_union, t_left, loc); + ctx->generate_pack_any(t_left, std::move(rvect_left)); + code.close_pop_cur(loc); + } + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector ir_result = code.create_tmp_var(t_union, loc, "(loaded-either)"); + std::vector ir_is_right = ctx->loadUint(1, "(eitherBit)"); + Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_is_right)); + { + code.push_set_cur(if_op.block0); + std::vector rvect_right = ctx->generate_unpack_any(t_right); + rvect_right = transition_to_target_type(std::move(rvect_right), code, t_right, t_union, loc); + code.emplace_back(loc, Op::_Let, ir_result, std::move(rvect_right)); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + std::vector rvect_left = ctx->generate_unpack_any(t_left); + rvect_left = transition_to_target_type(std::move(rvect_left), code, t_left, t_union, loc); + code.emplace_back(loc, Op::_Let, ir_result, std::move(rvect_left)); + code.close_pop_cur(loc); + } + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector ir_is_right = ctx->loadUint(1, "(eitherBit)"); + Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_is_right)); + { + code.push_set_cur(if_op.block0); + ctx->generate_skip_any(t_right); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + ctx->generate_skip_any(t_left); + code.close_pop_cur(loc); + } + } + + PackSize estimate(const EstimateContext* ctx) override { + PackSize either_size = EstimateContext::minmax(ctx->estimate_any(t_left), ctx->estimate_any(t_right)); + return EstimateContext::sum(PackSize(1), either_size); + } +}; + +struct S_MultipleConstructors final : ISerializer { + const TypeDataUnion* t_union; + std::vector opcodes; + + explicit S_MultipleConstructors(const TypeDataUnion* t_union, std::vector&& opcodes) + : t_union(t_union) + , opcodes(std::move(opcodes)) { + tolk_assert(this->opcodes.size() == t_union->variants.size()); + } + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + for (int i = 0; i < t_union->size() - 1; ++i) { + TypePtr variant = t_union->variants[i]; + std::vector ir_eq_ith = pre_compile_is_type(code, t_union, variant, rvect, loc, "(arm-cond-eq)"); + Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_eq_ith)); + code.push_set_cur(if_op.block0); + std::vector ith_rvect = transition_to_target_type(std::vector(rvect), code, t_union, variant, loc); + ctx->storeUint(code.create_int(loc, opcodes[i].pack_prefix, "(ith-prefix)"), opcodes[i].prefix_len); + ctx->generate_pack_any(variant, std::move(ith_rvect), PrefixWriteMode::DoNothingAlreadyWritten); + code.close_pop_cur(loc); + code.push_set_cur(if_op.block1); // open ELSE + } + + // we're inside the last ELSE + TypePtr last_variant = t_union->variants.back(); + std::vector last_rvect = transition_to_target_type(std::move(rvect), code, t_union, last_variant, loc); + ctx->storeUint(code.create_int(loc, opcodes.back().pack_prefix, "(ith-prefix)"), opcodes.back().prefix_len); + ctx->generate_pack_any(last_variant, std::move(last_rvect), PrefixWriteMode::DoNothingAlreadyWritten); + for (int i = 0; i < t_union->size() - 1; ++i) { + code.close_pop_cur(loc); // close all outer IFs + } + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + // assume that opcodes (either automatically generated or manually specified) + // form a valid prefix tree, and the order of reading does not matter; we'll definitely match the one; + FunctionPtr f_tryStripPrefix = lookup_function("slice.tryStripPrefix"); + + std::vector ir_result = code.create_tmp_var(t_union, loc, "(loaded-union)"); + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); + + for (int i = 0; i < t_union->size(); ++i) { + TypePtr variant = t_union->variants[i]; + std::vector args = { ctx->ir_slice0, code.create_int(loc, opcodes[i].pack_prefix, "(pack-prefix)"), code.create_int(loc, opcodes[i].prefix_len, "(prefix-len)") }; + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), f_tryStripPrefix); + if (i != t_union->size() - 1) { + Op& if_op = code.emplace_back(loc, Op::_If, ir_prefix_eq); + code.push_set_cur(if_op.block0); + std::vector ith_rvect = ctx->generate_unpack_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); + ith_rvect = transition_to_target_type(std::move(ith_rvect), code, variant, t_union, loc); + code.emplace_back(loc, Op::_Let, ir_result, std::move(ith_rvect)); + code.close_pop_cur(loc); + code.push_set_cur(if_op.block1); // open ELSE + } else { // we're inside the last ELSE + if (!are_prefixes_exhaustive()) { + std::vector args_assert = { code.create_int(loc, ERR_CODEGEN_ASSERT, "(excno)"), ir_prefix_eq[0], code.create_int(loc, 0, "") }; + Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); + op_assert.set_impure_flag(); + } + std::vector last_rvect = ctx->generate_unpack_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); + last_rvect = transition_to_target_type(std::move(last_rvect), code, variant, t_union, loc); + code.emplace_back(loc, Op::_Let, ir_result, std::move(last_rvect)); + for (int j = 0; j < t_union->size() - 1; ++j) { + code.close_pop_cur(loc); // close all outer IFs + } + } + } + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_tryStripPrefix = lookup_function("slice.tryStripPrefix"); + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); + + for (int i = 0; i < t_union->size(); ++i) { + TypePtr variant = t_union->variants[i]; + std::vector args = { ctx->ir_slice0, code.create_int(loc, opcodes[i].pack_prefix, "(pack-prefix)"), code.create_int(loc, opcodes[i].prefix_len, "(prefix-len)") }; + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), f_tryStripPrefix); + if (i != t_union->size() - 1) { + Op& if_op = code.emplace_back(loc, Op::_If, ir_prefix_eq); + code.push_set_cur(if_op.block0); + ctx->generate_skip_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); + code.close_pop_cur(loc); + code.push_set_cur(if_op.block1); // open ELSE + } else { // we're inside the last ELSE + if (!are_prefixes_exhaustive()) { + std::vector args_assert = { code.create_int(loc, ERR_CODEGEN_ASSERT, "(excno)"), ir_prefix_eq[0], code.create_int(loc, 0, "") }; + Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); + op_assert.set_impure_flag(); + } + ctx->generate_skip_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); + for (int j = 0; j < t_union->size() - 1; ++j) { + code.close_pop_cur(loc); // close all outer IFs + } + } + } + } + + PackSize estimate(const EstimateContext* ctx) override { + PackSize variants_size = ctx->estimate_any(t_union->variants[0], PrefixEstimateMode::DoNothingAlreadyIncluded); + PackSize prefix_size(opcodes[0].prefix_len); + + for (int i = 1; i < t_union->size(); ++i) { + variants_size = EstimateContext::minmax(variants_size, ctx->estimate_any(t_union->variants[i], PrefixEstimateMode::DoNothingAlreadyIncluded)); + prefix_size = EstimateContext::minmax(prefix_size, PackSize(opcodes[i].prefix_len)); + } + + return EstimateContext::sum(variants_size, prefix_size); + } + + bool are_prefixes_exhaustive() const { + bool all_prefix_len_eq = true; + for (PackOpcode opcode : opcodes) { + all_prefix_len_eq &= opcode.prefix_len == opcodes[0].prefix_len; + } + return all_prefix_len_eq && t_union->size() == (1 << opcodes[0].prefix_len); + } +}; + +struct S_Tensor final : ISerializer { + const TypeDataTensor* t_tensor; + + explicit S_Tensor(const TypeDataTensor* t_tensor) + : t_tensor(t_tensor) { + } + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + int stack_offset = 0; + for (TypePtr item : t_tensor->items) { + int stack_width = item->get_width_on_stack(); + std::vector item_vars(rvect.begin() + stack_offset, rvect.begin() + stack_offset + stack_width); + ctx->generate_pack_any(item, std::move(item_vars)); + stack_offset += stack_width; + } + tolk_assert(stack_offset == t_tensor->get_width_on_stack()); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector tensor_vars; + tensor_vars.reserve(t_tensor->get_width_on_stack()); + for (TypePtr item : t_tensor->items) { + std::vector item_vars = ctx->generate_unpack_any(item); + tensor_vars.insert(tensor_vars.end(), item_vars.begin(), item_vars.end()); + } + tolk_assert(static_cast(tensor_vars.size()) == t_tensor->get_width_on_stack()); + return tensor_vars; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + for (TypePtr item : t_tensor->items) { + ctx->generate_skip_any(item); + } + } + + PackSize estimate(const EstimateContext* ctx) override { + PackSize sum = PackSize(0); + for (TypePtr item : t_tensor->items) { + sum = EstimateContext::sum(sum, ctx->estimate_any(item)); + } + return sum; + } +}; + +struct S_CustomStruct final : ISerializer { + StructPtr struct_ref; + + explicit S_CustomStruct(StructPtr struct_ref) + : struct_ref(struct_ref) { + } + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + if (struct_ref->opcode.exists() && ctx->get_prefix_mode() == PrefixWriteMode::WritePrefixOfStruct) { + ctx->storeOpcode(struct_ref->opcode); + } + + int stack_offset = 0; + for (StructFieldPtr field_ref : struct_ref->fields) { + int stack_width = field_ref->declared_type->get_width_on_stack(); + std::vector field_vars(rvect.begin() + stack_offset, rvect.begin() + stack_offset + stack_width); + ctx->generate_pack_any(field_ref->declared_type, std::move(field_vars)); + stack_offset += stack_width; + } + tolk_assert(stack_offset == TypeDataStruct::create(struct_ref)->get_width_on_stack()); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + if (struct_ref->opcode.exists() && ctx->get_prefix_mode() == PrefixReadMode::LoadAndCheck) { + ctx->loadAndCheckOpcode(struct_ref->opcode); + } + + int total_stack_w = TypeDataStruct::create(struct_ref)->get_width_on_stack(); + std::vector ir_struct; + ir_struct.reserve(total_stack_w); + for (StructFieldPtr field_ref : struct_ref->fields) { + std::vector field_vars = ctx->generate_unpack_any(field_ref->declared_type); + ir_struct.insert(ir_struct.end(), field_vars.begin(), field_vars.end()); + } + tolk_assert(static_cast(ir_struct.size()) == total_stack_w); + return ir_struct; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + if (struct_ref->opcode.exists() && ctx->get_prefix_mode() == PrefixReadMode::LoadAndCheck) { + ctx->loadAndCheckOpcode(struct_ref->opcode); + } + + for (StructFieldPtr field_ref : struct_ref->fields) { + ctx->generate_skip_any(field_ref->declared_type); + } + } + + PackSize estimate(const EstimateContext* ctx) override { + PackSize sum(0); + + if (struct_ref->opcode.exists() && ctx->get_prefix_mode() == PrefixEstimateMode::IncludePrefixOfStruct) { + sum = EstimateContext::sum(sum, PackSize(struct_ref->opcode.prefix_len)); + } + + for (StructFieldPtr field_ref : struct_ref->fields) { + sum = EstimateContext::sum(sum, ctx->estimate_any(field_ref->declared_type)); + } + return sum; + } +}; + + +// -------------------------------------------- +// automatically generate opcodes +// +// for union types like `T1 | T2 | ...`, if prefixes for structs are not manually specified, +// the compiler generates a valid prefix tree: for `int32 | int64 | int128` it's '00' '01' '10'; +// it works both for structs (with unspecified prefixes) and primitives: `int32 | A | B` is ok; +// but if some prefixes are specified, some not — it's an error +// + + +std::vector auto_generate_opcodes_for_union(TypePtr union_type, std::string& because_msg) { + const TypeDataUnion* t_union = union_type->try_as(); + std::vector result; + result.reserve(t_union->size()); + + int n_have_opcode = 0; + bool has_null = false; + StructPtr last_struct_with_opcode = nullptr; // for error message + StructPtr last_struct_no_opcode = nullptr; + for (TypePtr variant : t_union->variants) { + if (const TypeDataStruct* variant_struct = variant->unwrap_alias()->try_as()) { + if (variant_struct->struct_ref->opcode.exists()) { + n_have_opcode++; + last_struct_with_opcode = variant_struct->struct_ref; + } else { + last_struct_no_opcode = variant_struct->struct_ref; + } + } else if (variant == TypeDataNullLiteral::create()) { + has_null = true; + } + } + + // `A | B | C`, all of them have opcodes — just use them; + // for instance, `A | B` is not either (0/1 + data), but uses manual opcodes + if (n_have_opcode == t_union->size()) { + for (TypePtr variant : t_union->variants) { + result.push_back(variant->unwrap_alias()->try_as()->struct_ref->opcode); + } + return result; + } + + // invalid: `A | B | C`, some of them have opcodes, some not; + // example: `A | B` if A has opcode, B not; + // example: `int32 | A` if A has opcode; + // example: `int32 | int64 | A` if A has opcode; + if (n_have_opcode) { + if (last_struct_with_opcode && last_struct_no_opcode) { + because_msg = "because struct `" + last_struct_with_opcode->as_human_readable() + "` has opcode, but `" + last_struct_no_opcode->as_human_readable() + "` does not\nhint: manually specify opcodes to all structures"; + } else { + because_msg = "because of mixing primitives and struct `" + last_struct_with_opcode->as_human_readable() + "` with serialization prefix\nhint: extract primitives to single-field structs and provide prefixes"; + } + return result; + } + + // okay, none of the opcodes are specified, generate a prefix tree; + // examples: `int32 | int64 | int128` / `int32 | A | null` / `A | B` / `A | B | C`; + // create prefixes `0b00 0b01 0b10` / `0b01 0b10 0b00` (use 0b00 for null if exists): + // for 3/4 variants — two bits, for 5 — three + int prefix_len = static_cast(std::ceil(std::log2(t_union->size()))); + int cur_prefix = has_null ? 1 : 0; // will use 0b00 for null, so start with 0b01 + for (TypePtr variant : t_union->variants) { + if (variant == TypeDataNullLiteral::create()) { + result.emplace_back(0, prefix_len); + } else { + result.emplace_back(cur_prefix++, prefix_len); + } + } + return result; +} + + +// -------------------------------------------- +// detect serializer by TypePtr +// +// note that at earlier compilation steps there already passed a check that any_type is serializable; +// see `check_struct_can_be_packed_or_unpacked()`, its structure reminds this function +// + + +static std::unique_ptr get_serializer_for_type(TypePtr any_type) { + if (const auto* t_intN = any_type->try_as()) { + return std::make_unique(t_intN->n_bits, t_intN->is_unsigned); + } + if (const auto* t_bytesN = any_type->try_as()) { + return std::make_unique(t_bytesN->n_width, t_bytesN->is_bits); + } + if (any_type == TypeDataCoins::create()) { + return std::make_unique(); + } + if (any_type == TypeDataBool::create()) { + return std::make_unique(); + } + if (any_type == TypeDataCell::create()) { + return std::make_unique(); + } + if (any_type == TypeDataAddress::create()) { + return std::make_unique(); + } + if (any_type == TypeDataBuilder::create()) { + return std::make_unique(); + } + if (any_type == TypeDataNullLiteral::create()) { + return std::make_unique(); + } + if (any_type == TypeDataNever::create()) { + return std::make_unique(); + } + + if (const auto* t_struct = any_type->try_as()) { + return std::make_unique(t_struct->struct_ref); + } + + if (const auto* t_union = any_type->try_as()) { + // `T?` is always `(Maybe T)`, even if T has custom opcode (opcode will follow bit '1') + if (t_union->or_null) { + TypePtr or_null = t_union->or_null->unwrap_alias(); + if (or_null == TypeDataCell::create() || is_type_cellT(or_null)) { + return std::make_unique(); + } + return std::make_unique(t_union); + } + + // `T1 | T2` is `(Either T1 T2)` (0/1 + contents) unless they both have custom prefixes + bool all_have_opcode = true; + for (TypePtr variant : t_union->variants) { + const TypeDataStruct* variant_struct = variant->unwrap_alias()->try_as(); + all_have_opcode &= variant_struct && variant_struct->struct_ref->opcode.exists(); + } + if (t_union->size() == 2 && !all_have_opcode) { + return std::make_unique(t_union); + } + // `T1 | T2 | T3`, probably nullable, probably with primitives, probably with custom opcodes; + // compiler is able to generate serialization prefixes automatically; + // and this type is valid, it was checked earlier + std::string err_msg; + std::vector opcodes = auto_generate_opcodes_for_union(t_union, err_msg); + tolk_assert(err_msg.empty()); + return std::make_unique(t_union, std::move(opcodes)); + } + + if (const auto* t_tensor = any_type->try_as()) { + return std::make_unique(t_tensor); + } + + if (const auto* t_alias = any_type->try_as()) { + if (t_alias->alias_ref->name == "RemainingBitsAndRefs") { + return std::make_unique(); + } + return get_serializer_for_type(t_alias->underlying_type); + } + + // this should not be reachable, serialization availability is checked earlier + throw Fatal("type `" + any_type->as_human_readable() + "` can not be serialized"); +} + + +void PackContext::generate_pack_any(TypePtr any_type, std::vector&& rvect, PrefixWriteMode prefix_mode) const { + PrefixWriteMode backup = this->prefix_mode; + this->prefix_mode = prefix_mode; + get_serializer_for_type(any_type)->pack(this, code, loc, std::move(rvect)); + this->prefix_mode = backup; +} + +std::vector UnpackContext::generate_unpack_any(TypePtr any_type, PrefixReadMode prefix_mode) const { + PrefixReadMode backup = this->prefix_mode; + this->prefix_mode = prefix_mode; + std::vector result = get_serializer_for_type(any_type)->unpack(this, code, loc); + this->prefix_mode = backup; + return result; +} + +void UnpackContext::generate_skip_any(TypePtr any_type, PrefixReadMode prefix_mode) const { + PrefixReadMode backup = this->prefix_mode; + this->prefix_mode = prefix_mode; + get_serializer_for_type(any_type)->skip(this, code, loc); + this->prefix_mode = backup; +} + +PackSize EstimateContext::estimate_any(TypePtr any_type, PrefixEstimateMode prefix_mode) const { + PrefixEstimateMode backup = this->prefix_mode; + this->prefix_mode = prefix_mode; + PackSize result = get_serializer_for_type(any_type)->estimate(this); + this->prefix_mode = backup; + return result; +} + +} // namespace tolk diff --git a/tolk/pack-unpack-serializers.h b/tolk/pack-unpack-serializers.h new file mode 100644 index 0000000000..47629a7dbd --- /dev/null +++ b/tolk/pack-unpack-serializers.h @@ -0,0 +1,137 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "fwd-declarations.h" +#include "tolk.h" + +namespace tolk { + +using PackOpcode = StructData::PackOpcode; + +struct PackSize { + int min_bits; + int max_bits; + int min_refs; + int max_refs; + + explicit PackSize(int exact_bits) + : min_bits(exact_bits), max_bits(exact_bits), min_refs(0), max_refs(0) { + } + PackSize(int min_bits, int max_bits) + : min_bits(min_bits), max_bits(max_bits), min_refs(0), max_refs() { + } + PackSize(int min_bits, int max_bits, int min_refs, int max_refs) + : min_bits(min_bits), max_bits(max_bits), min_refs(min_refs), max_refs(max_refs) { + } +}; + + +enum class PrefixWriteMode { + WritePrefixOfStruct, + DoNothingAlreadyWritten, +}; + +class PackContext { + CodeBlob& code; + SrcLocation loc; + const FunctionPtr f_storeInt; + const FunctionPtr f_storeUint; + mutable PrefixWriteMode prefix_mode = PrefixWriteMode::WritePrefixOfStruct; + +public: + const std::vector ir_builder; + const var_idx_t ir_builder0; + + PackContext(CodeBlob& code, SrcLocation loc, std::vector ir_builder); + + PrefixWriteMode get_prefix_mode() const { return prefix_mode; } + + void storeInt(var_idx_t ir_idx, int len) const; + void storeUint(var_idx_t ir_idx, int len) const; + void storeBool(var_idx_t ir_idx) const; + void storeCoins(var_idx_t ir_idx) const; + void storeRef(var_idx_t ir_idx) const; + void storeMaybeRef(var_idx_t ir_idx) const; + void storeAddress(var_idx_t ir_idx) const; + void storeBuilder(var_idx_t ir_idx) const; + void storeSlice(var_idx_t ir_idx) const; + void storeOpcode(PackOpcode opcode) const; + + void generate_pack_any(TypePtr any_type, std::vector&& rvect, PrefixWriteMode prefix_mode = PrefixWriteMode::WritePrefixOfStruct) const; +}; + + +enum class PrefixReadMode { + LoadAndCheck, + DoNothingAlreadyLoaded, +}; + +class UnpackContext { + CodeBlob& code; + SrcLocation loc; + const FunctionPtr f_loadInt; + const FunctionPtr f_loadUint; + const FunctionPtr f_skipBits; + mutable PrefixReadMode prefix_mode = PrefixReadMode::LoadAndCheck; + +public: + const std::vector ir_slice; + const var_idx_t ir_slice0; + + UnpackContext(CodeBlob& code, SrcLocation loc, std::vector ir_slice); + + PrefixReadMode get_prefix_mode() const { return prefix_mode; } + + std::vector loadInt(int len, const char* debug_desc) const; + std::vector loadUint(int len, const char* debug_desc) const; + void loadAndCheckOpcode(PackOpcode opcode) const; + void skipBits(int len) const; + void skipBits_var(var_idx_t ir_len) const; + + std::vector generate_unpack_any(TypePtr any_type, PrefixReadMode prefix_mode = PrefixReadMode::LoadAndCheck) const; + void generate_skip_any(TypePtr any_type, PrefixReadMode prefix_mode = PrefixReadMode::LoadAndCheck) const; +}; + + +enum class PrefixEstimateMode { + IncludePrefixOfStruct, + DoNothingAlreadyIncluded, +}; + +class EstimateContext { + mutable PrefixEstimateMode prefix_mode = PrefixEstimateMode::IncludePrefixOfStruct; + +public: + + PrefixEstimateMode get_prefix_mode() const { return prefix_mode; } + + static PackSize minmax(PackSize a, PackSize b) { + return PackSize(std::min(a.min_bits, b.min_bits), std::max(a.max_bits, b.max_bits), std::min(a.min_refs, b.min_refs), std::max(a.max_refs, b.max_refs)); + } + static PackSize sum(PackSize a, PackSize b) { + return PackSize(a.min_bits + b.min_bits, a.max_bits + b.max_bits, a.min_refs + b.min_refs, a.max_refs + b.max_refs); + } + + PackSize estimate_any(TypePtr any_type, PrefixEstimateMode prefix_mode = PrefixEstimateMode::IncludePrefixOfStruct) const; +}; + + +bool is_type_cellT(TypePtr any_type); +std::vector auto_generate_opcodes_for_union(TypePtr union_type, std::string& because_msg); + +} // namespace tolk diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 31bccca150..9d58a7faac 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -22,6 +22,8 @@ #include "type-system.h" #include "common/refint.h" #include "smart-casts-cfg.h" +#include "pack-unpack-api.h" +#include "generics-helpers.h" #include /* @@ -205,7 +207,7 @@ class LValContext { code.emplace_back(loc, Op::_IntConst, index_ir_idx, td::make_refint(index_at)); vars_modification_watcher.trigger_callbacks(tuple_ir_idx, loc); - FunctionPtr builtin_sym = lookup_global_symbol("tuple.set")->try_as(); + FunctionPtr builtin_sym = lookup_function("tuple.set"); code.emplace_back(loc, Op::_Call, std::vector{tuple_ir_idx}, std::vector{tuple_ir_idx[0], lval_ir_idx[0], index_ir_idx[0]}, builtin_sym); local_lval.after_let(std::move(tuple_ir_idx), code, loc); } @@ -491,10 +493,10 @@ static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyE return right; } -static std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc) { - FunctionPtr eq_sym = lookup_global_symbol("_==_")->try_as(); - FunctionPtr isnull_sym = lookup_global_symbol("__isNull")->try_as(); - FunctionPtr not_sym = lookup_global_symbol("!b_")->try_as(); +std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc) { + FunctionPtr eq_sym = lookup_function("_==_"); + FunctionPtr isnull_sym = lookup_function("__isNull"); + FunctionPtr not_sym = lookup_function("!b_"); std::vector result_ir_idx = code.create_tmp_var(TypeDataBool::create(), loc, debug_desc); const TypeDataUnion* lhs_union = expr_type->try_as(); @@ -538,6 +540,66 @@ static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcL return rvect; } +static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob& code, SrcLocation loc, std::vector&& rvect, FunctionPtr called_f) { + if (called_f->is_instantiation_of_generic_function()) { + std::string_view f_name = called_f->base_fun_ref->name; + TypePtr typeT = called_f->substitutedTs->typeT_at(0); + int n_rvect = static_cast(rvect.size()); + + if (f_name == "T.toCell") { + // in: object T, out: Cell (just a cell, wrapped) + tolk_assert(n_rvect == typeT->get_width_on_stack()); + std::vector ir_obj = std::move(rvect); + return generate_pack_struct_to_cell(code, loc, typeT, std::move(ir_obj)); + } + if (f_name == "T.fromCell") { + // in: cell, out: object T + tolk_assert(n_rvect == 1); + std::vector ir_cell = std::move(rvect); + return generate_unpack_struct_from_cell(code, loc, typeT, std::move(ir_cell)); + } + if (f_name == "T.fromSlice") { + // in: slice, out: object T, input slice NOT mutated + tolk_assert(n_rvect == 1); + std::vector ir_slice = std::move(rvect); + return generate_unpack_struct_from_slice(code, loc, typeT, std::move(ir_slice), false); + } + if (f_name == "Cell.load") { + // in: cell, out: object T + tolk_assert(n_rvect == 1); + std::vector ir_cell = std::move(rvect); + return generate_unpack_struct_from_cell(code, loc, typeT, std::move(ir_cell)); + } + if (f_name == "slice.loadAny") { + // in: slice, out: object T, input slice is mutated, so prepend self before an object + var_idx_t ir_self = rvect[0]; + std::vector ir_obj = generate_unpack_struct_from_slice(code, loc, typeT, std::move(rvect), true); + std::vector ir_result = {ir_self}; + ir_result.insert(ir_result.end(), ir_obj.begin(), ir_obj.end()); + return ir_result; + } + if (f_name == "slice.skipAny") { + // in: slice, out: the same slice, with a shifted pointer + tolk_assert(n_rvect == 1); + std::vector ir_slice = std::move(rvect); + return generate_skip_struct_in_slice(code, loc, typeT, std::move(ir_slice)); + } + if (f_name == "builder.storeAny") { + // in: builder and object T, out: mutated builder + tolk_assert(n_rvect == 1 + typeT->get_width_on_stack()); + std::vector ir_builder = {rvect[0]}; + std::vector ir_obj(rvect.begin() + 1, rvect.end()); + return generate_pack_struct_to_builder(code, loc, typeT, std::move(ir_builder), std::vector(ir_obj)); + } + if (f_name == "T.estimatePackSize") { + tolk_assert(rvect.empty()); + return generate_estimate_size_call(code, loc, typeT); + } + } + + tolk_assert(false); +} + // "Transition to target (runtime) type" is the following process. // Imagine `fun analyze(t: (int,int)?)` and a call `analyze((1,2))`. // `(1,2)` (inferred_type) is 2 stack slots, but `t` (target_type) is 3 (one for null-flag). @@ -615,7 +677,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector 1 && original_type == TypeDataNullLiteral::create()) { tolk_assert(t_union->has_null()); - FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); + FunctionPtr null_sym = lookup_function("__null"); rvect.reserve(target_w); // keep rvect[0], it's already null for (int i = 1; i < target_w - 1; ++i) { std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); @@ -657,7 +719,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectorhas_null()); - FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); + FunctionPtr null_sym = lookup_function("__null"); std::vector new_rvect = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); code.emplace_back(loc, Op::_Call, new_rvect, std::vector{}, null_sym); return new_rvect; @@ -674,7 +736,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as(); + FunctionPtr null_sym = lookup_function("__null"); std::vector new_rvect; new_rvect.resize(target_w); for (int i = 0; i < target_w - 2; ++i) { // N-1 nulls @@ -686,7 +748,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector eq_null_cond = code.create_tmp_var(TypeDataBool::create(), loc, "(value-is-null)"); - FunctionPtr isnull_sym = lookup_global_symbol("__isNull")->try_as(); + FunctionPtr isnull_sym = lookup_function("__isNull"); code.emplace_back(loc, Op::_Call, eq_null_cond, rvect, isnull_sym); Op& if_op = code.emplace_back(loc, Op::_If, eq_null_cond); code.push_set_cur(if_op.block0); @@ -712,7 +774,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector prepend_nulls; prepend_nulls.reserve(target_w - t_subtype->get_width_on_stack() - 1); for (int i = 0; i < target_w - t_subtype->get_width_on_stack() - 1; ++i) { - FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); + FunctionPtr null_sym = lookup_function("__null"); std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(UVar.null)"); prepend_nulls.push_back(ith_null[0]); code.emplace_back(loc, Op::_Call, std::move(ith_null), std::vector{}, null_sym); @@ -747,7 +809,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector prepend_nulls; prepend_nulls.reserve(target_w - orig_w); for (int i = 0; i < target_w - orig_w; ++i) { - FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); + FunctionPtr null_sym = lookup_function("__null"); std::vector ith_null_ir_idx = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(UVar.null)"); prepend_nulls.push_back(ith_null_ir_idx[0]); code.emplace_back(loc, Op::_Call, std::move(ith_null_ir_idx), std::vector{}, null_sym); @@ -910,6 +972,17 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector` to `cell`, e.g. `setContractData(obj.toCell())` + if (target_type == TypeDataCell::create() && original_type->try_as()) { + tolk_assert(orig_w == 1 && original_type->try_as()->struct_ref->is_instantiation_of_generic_struct()); + return rvect; + } + // and vice versa, `cell as Cell` + if (original_type == TypeDataCell::create() && target_type->try_as()) { + tolk_assert(target_w == 1 && target_type->try_as()->struct_ref->is_instantiation_of_generic_struct()); + return rvect; + } + throw Fatal("unhandled transition_expr_to_runtime_type_impl() combination"); } @@ -929,7 +1002,7 @@ static std::vector transition_to_target_type(std::vector&& #ifndef TOLK_DEBUG GNU_ATTRIBUTE_ALWAYS_INLINE #endif -static std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc) { +std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc) { if (target_type != original_type) { rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, target_type, loc); } @@ -1037,7 +1110,7 @@ static std::vector process_binary_operator(V v, v_1->mutate()->assign_inferred_type(TypeDataInt::create()); auto v_b_ne_0 = createV(v->loc, "!=", tok_neq, v->get_rhs(), v_0); v_b_ne_0->mutate()->assign_inferred_type(TypeDataInt::create()); - v_b_ne_0->mutate()->assign_fun_ref(lookup_global_symbol("_!=_")->try_as()); + v_b_ne_0->mutate()->assign_fun_ref(lookup_function("_!=_")); std::vector cond = pre_compile_expr(v->get_lhs(), code, nullptr); tolk_assert(cond.size() == 1); std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(ternary)"); @@ -1052,13 +1125,13 @@ static std::vector process_binary_operator(V v, } if (t == tok_eq || t == tok_neq) { if (v->get_lhs()->inferred_type->unwrap_alias() == TypeDataAddress::create() && v->get_rhs()->inferred_type->unwrap_alias() == TypeDataAddress::create()) { - FunctionPtr f_sliceEq = lookup_global_symbol("slice.bitsEqual")->try_as(); + FunctionPtr f_sliceEq = lookup_function("slice.bitsEqual"); std::vector ir_lhs_slice = pre_compile_expr(v->get_lhs(), code); std::vector ir_rhs_slice = pre_compile_expr(v->get_rhs(), code); std::vector rvect = code.create_tmp_var(TypeDataBool::create(), v->loc, "(addr-eq)"); code.emplace_back(v->loc, Op::_Call, rvect, std::vector{ir_lhs_slice[0], ir_rhs_slice[0]}, f_sliceEq); if (t == tok_neq) { - FunctionPtr not_sym = lookup_global_symbol("!b_")->try_as(); + FunctionPtr not_sym = lookup_function("!b_"); code.emplace_back(v->loc, Op::_Call, rvect, rvect, not_sym); } return transition_to_target_type(std::move(rvect), code, target_type, v); @@ -1112,7 +1185,7 @@ static std::vector process_is_type_operator(V v std::vector result_ir_idx = pre_compile_is_type(code, lhs_type, cmp_type, expr_ir_idx, v->loc, is_null_check ? "(is-null)" : "(is-type)"); if (v->is_negated) { - FunctionPtr not_sym = lookup_global_symbol("!b_")->try_as(); + FunctionPtr not_sym = lookup_function("!b_"); code.emplace_back(v->loc, Op::_Call, result_ir_idx, result_ir_idx, not_sym); } return transition_to_target_type(std::move(result_ir_idx), code, target_type, v); @@ -1152,7 +1225,7 @@ static std::vector process_match_expression(V v // construct nested IFs: IF is int { ... } ELSE { IF is slice { ... } ELSE { ... } } // example 3 (not exhaustive): `match (v) { -1 => ... 0 => ... 1 => ... }` // construct nested IFs: IF == -1 { ... } ELSE { IF == 0 { ... } ELSE { IF == 1 { ... } } } - FunctionPtr eq_sym = lookup_global_symbol("_==_")->try_as(); + FunctionPtr eq_sym = lookup_function("_==_"); for (int i = 0; i < n_arms - is_exhaustive; ++i) { auto v_ith_arm = v->get_arm(i); std::vector eq_ith_ir_idx; @@ -1259,7 +1332,7 @@ static std::vector process_dot_access(V v, CodeBlob& code.emplace_back(v->loc, Op::_IntConst, index_ir_idx, td::make_refint(index_at)); std::vector field_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(tuple-field)"); tolk_assert(tuple_ir_idx.size() == 1 && field_ir_idx.size() == 1); // tuples contain only 1-slot values - FunctionPtr builtin_sym = lookup_global_symbol("tuple.get")->try_as(); + FunctionPtr builtin_sym = lookup_function("tuple.get"); code.emplace_back(v->loc, Op::_Call, field_ir_idx, std::vector{tuple_ir_idx[0], index_ir_idx[0]}, builtin_sym); if (lval_ctx && calc_sink_leftmost_obj(v)) { // `tupleVar.0.1 = rhs`, then `tupleVar.0` is rval inside lval lval_ctx->capture_tuple_index_modification(v->get_obj(), index_at, field_ir_idx); @@ -1371,7 +1444,9 @@ static std::vector process_function_call(V v, Code for (const std::vector& list : vars_per_arg) { args_vars.insert(args_vars.end(), list.cbegin(), list.cend()); } - std::vector rvect_apply = gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)", arg_order_already_equals_asm); + std::vector rvect_apply = fun_ref->is_compile_time_special_gen() + ? gen_compile_time_code_instead_of_fun_call(code, v->loc, std::move(args_vars), fun_ref) + : gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)", arg_order_already_equals_asm); if (fun_ref->has_mutate_params()) { LValContext local_lval; @@ -1553,13 +1628,13 @@ static std::vector process_string_const(V v, CodeBl } static std::vector process_bool_const(V v, CodeBlob& code, TypePtr target_type) { - FunctionPtr builtin_sym = lookup_global_symbol(v->bool_val ? "__true" : "__false")->try_as(); + FunctionPtr builtin_sym = lookup_function(v->bool_val ? "__true" : "__false"); std::vector rvect = gen_op_call(code, v->inferred_type, v->loc, {}, builtin_sym, "(bool-const)"); return transition_to_target_type(std::move(rvect), code, target_type, v); } static std::vector process_null_keyword(V v, CodeBlob& code, TypePtr target_type) { - FunctionPtr builtin_sym = lookup_global_symbol("__null")->try_as(); + FunctionPtr builtin_sym = lookup_function("__null"); std::vector rvect = gen_op_call(code, v->inferred_type, v->loc, {}, builtin_sym, "(null-literal)"); return transition_to_target_type(std::move(rvect), code, target_type, v); } @@ -1685,7 +1760,7 @@ static void process_assert_statement(V v, CodeBlob& code) args[2]->mutate()->assign_inferred_type(TypeDataInt::create()); } - FunctionPtr builtin_sym = lookup_global_symbol("__throw_if_unless")->try_as(); + FunctionPtr builtin_sym = lookup_function("__throw_if_unless"); std::vector args_vars = pre_compile_tensor(code, args); gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); } @@ -1781,9 +1856,9 @@ static void process_do_while_statement(V v, CodeBlob& co } until_cond->mutate()->assign_inferred_type(TypeDataInt::create()); if (auto v_bin = until_cond->try_as(); v_bin && !v_bin->fun_ref) { - v_bin->mutate()->assign_fun_ref(lookup_global_symbol("_" + static_cast(v_bin->operator_name) + "_")->try_as()); + v_bin->mutate()->assign_fun_ref(lookup_function("_" + static_cast(v_bin->operator_name) + "_")); } else if (auto v_un = until_cond->try_as(); v_un && !v_un->fun_ref) { - v_un->mutate()->assign_fun_ref(lookup_global_symbol(static_cast(v_un->operator_name) + "_")->try_as()); + v_un->mutate()->assign_fun_ref(lookup_function(static_cast(v_un->operator_name) + "_")); } until_op.left = pre_compile_expr(until_cond, code, nullptr); @@ -1804,11 +1879,11 @@ static void process_while_statement(V v, CodeBlob& code) { static void process_throw_statement(V v, CodeBlob& code) { if (v->has_thrown_arg()) { - FunctionPtr builtin_sym = lookup_global_symbol("__throw_arg")->try_as(); + FunctionPtr builtin_sym = lookup_function("__throw_arg"); std::vector args_vars = pre_compile_tensor(code, {v->get_thrown_arg(), v->get_thrown_code()}); gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); } else { - FunctionPtr builtin_sym = lookup_global_symbol("__throw")->try_as(); + FunctionPtr builtin_sym = lookup_function("__throw"); std::vector args_vars = pre_compile_tensor(code, {v->get_thrown_code()}); gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); } diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index dc17fa6523..8500ec59a4 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -65,6 +65,10 @@ static void fire_error_cannot_apply_operator(FunctionPtr cur_f, SrcLocation loc, GNU_ATTRIBUTE_NOINLINE static void warning_condition_always_true_or_false(FunctionPtr cur_f, SrcLocation loc, AnyExprV cond, const char* operator_name) { + bool no_warning = cond->kind == ast_bool_const || cond->kind == ast_int_const; + if (no_warning) { // allow `while(true)` without a warning + return; + } loc.show_warning("condition of " + static_cast(operator_name) + " is always " + (cond->is_always_true ? "true" : "false")); } diff --git a/tolk/pipe-check-serialized-fields.cpp b/tolk/pipe-check-serialized-fields.cpp new file mode 100644 index 0000000000..c49b3dc349 --- /dev/null +++ b/tolk/pipe-check-serialized-fields.cpp @@ -0,0 +1,115 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-visitor.h" +#include "pack-unpack-api.h" +#include "generics-helpers.h" +#include "type-system.h" + +namespace tolk { + +// fire an error on overflow 1023 bits +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_theoretical_overflow_1023(StructPtr struct_ref, PackSize size) { + throw ParseError(struct_ref->ast_root->loc, + "struct `" + struct_ref->as_human_readable() + "` can exceed 1023 bits in serialization (estimated size: " + std::to_string(size.min_bits) + ".." + std::to_string(size.max_bits) + " bits)\n\n" + "1) either suppress it by adding an annotation:\n" + "> @overflow1023_policy(\"suppress\")\n" + "> struct " + struct_ref->name + " {\n" + "> ...\n" + "> }\n" + " then, if limit exceeds, it will fail at runtime: you've manually agreed to ignore this\n\n" + "2) or place some fields into a separate struct (e.g. ExtraFields), and create a ref:\n" + "> struct " + struct_ref->name + " {\n" + "> ...\n" + "> more: Cell;\n" + "> }\n" + ); +} + + +class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; + + static void check_type_fits_cell_or_has_policy(TypePtr serialized_type) { + if (const TypeDataStruct* s_struct = serialized_type->unwrap_alias()->try_as()) { + check_struct_fits_cell_or_has_policy(s_struct); + } else if (const TypeDataUnion* s_union = serialized_type->unwrap_alias()->try_as()) { + for (TypePtr variant : s_union->variants) { + check_type_fits_cell_or_has_policy(variant); + } + } + } + + static void check_struct_fits_cell_or_has_policy(const TypeDataStruct* t_struct) { + StructPtr struct_ref = t_struct->struct_ref; + PackSize size = estimate_serialization_size(t_struct); + if (size.max_bits > 1023) { + if (struct_ref->overflow1023_policy == StructData::Overflow1023Policy::not_specified) { + fire_error_theoretical_overflow_1023(struct_ref, size); + } + } + for (StructFieldPtr field_ref : struct_ref->fields) { + if (is_type_cellT(field_ref->declared_type)) { + const TypeDataStruct* f_struct = field_ref->declared_type->try_as(); + check_type_fits_cell_or_has_policy(f_struct->struct_ref->substitutedTs->typeT_at(0)); + } + } + } + + void visit(V v) override { + FunctionPtr fun_ref = v->fun_maybe; + if (!fun_ref || !fun_ref->is_compile_time_special_gen() || !fun_ref->is_instantiation_of_generic_function()) { + return; + } + + std::string_view f_name = fun_ref->base_fun_ref->name; + TypePtr serialized_type = nullptr; + bool is_pack = false; + if (f_name == "Cell.load" || f_name == "T.fromSlice" || f_name == "T.fromCell" || f_name == "T.toCell" || + f_name == "T.loadAny" || f_name == "slice.skipAny" || f_name == "slice.storeAny" || f_name == "T.estimatePackSize") { + serialized_type = fun_ref->substitutedTs->typeT_at(0); + is_pack = f_name == "T.toCell" || f_name == "slice.storeAny"; + } else { + return; // not a serialization function + } + + std::string because_msg; + if (!check_struct_can_be_packed_or_unpacked(serialized_type, is_pack, because_msg)) { + fire(cur_f, v->loc, "auto-serialization via " + fun_ref->method_name + "() is not available for type `" + serialized_type->as_human_readable() + "`\n" + because_msg); + } + + check_type_fits_cell_or_has_policy(serialized_type); + } + + public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + } +}; + +void pipeline_check_serialized_fields() { + visit_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index 32b7c28c2a..f16bf90a5e 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -58,9 +58,9 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { return inner; } - static V create_string_const(SrcLocation loc, std::string&& literal_value) { + static V create_string_const(SrcLocation loc, std::string&& literal_value, TypePtr inferred_type) { auto v_string = createV(loc, literal_value); - v_string->assign_inferred_type(TypeDataSlice::create()); + v_string->assign_inferred_type(inferred_type); v_string->assign_literal_value(std::move(literal_value)); v_string->assign_rvalue_true(); return v_string; @@ -112,12 +112,12 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { parent::replace(v); // replace `ton("0.05")` with 50000000 / `stringCrc32("some_str")` with calculated value / etc. - if (v->fun_maybe && v->fun_maybe->is_compile_time_only()) { + if (v->fun_maybe && v->fun_maybe->is_compile_time_const_val()) { CompileTimeFunctionResult value = eval_call_to_compile_time_function(v); if (std::holds_alternative(value)) { return create_int_const(v->loc, std::move(std::get(value))); } else { - return create_string_const(v->loc, std::move(std::get(value))); + return create_string_const(v->loc, std::move(std::get(value)), v->fun_maybe->declared_return_type); } } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 8e56e62b6a..dd83a8dc26 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -577,7 +577,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, lhs); - FunctionPtr builtin_sym = lookup_global_symbol("_" + static_cast(builtin_func) + "_")->try_as(); + FunctionPtr builtin_sym = lookup_function("_" + static_cast(builtin_func) + "_"); v->mutate()->assign_fun_ref(builtin_sym); return ExprFlow(std::move(after_rhs.out_flow), used_as_condition); @@ -607,7 +607,7 @@ class InferTypesAndCallsAndFieldsVisitor final { tolk_assert(false); } - FunctionPtr builtin_sym = lookup_global_symbol(static_cast(builtin_func) + "_")->try_as(); + FunctionPtr builtin_sym = lookup_function(static_cast(builtin_func) + "_"); v->mutate()->assign_fun_ref(builtin_sym); return after_rhs; @@ -685,7 +685,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } if (!builtin_func.empty()) { - FunctionPtr builtin_sym = lookup_global_symbol("_" + static_cast(builtin_func) + "_")->try_as(); + FunctionPtr builtin_sym = lookup_function("_" + static_cast(builtin_func) + "_"); v->mutate()->assign_fun_ref(builtin_sym); } @@ -849,7 +849,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // T for asm function must be a TVM primitive (width 1), otherwise, asm would act incorrectly if (fun_ref->is_asm_function() || fun_ref->is_builtin_function()) { for (int i = 0; i < substitutedTs.size(); ++i) { - if (substitutedTs.typeT_at(i)->get_width_on_stack() != 1) { + if (substitutedTs.typeT_at(i)->get_width_on_stack() != 1 && !fun_ref->is_variadic_width_T_allowed()) { fire_error_calling_asm_function_with_non1_stack_width_arg(cur_f, loc, fun_ref, substitutedTs, i); } } @@ -908,7 +908,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (out_f_called) { // so, it's `globalF()` / `genericFn()` / `genericFn()` *out_f_called = fun_ref; // (it's still may be a generic one, then Ts will be deduced from arguments) } else { // so, it's `globalF` / `genericFn` as a reference - if (fun_ref->is_compile_time_only()) { + if (fun_ref->is_compile_time_const_val() || fun_ref->is_compile_time_special_gen()) { fire(cur_f, v->loc, "can not get reference to this function, it's compile-time only"); } fun_ref->mutate()->assign_is_used_as_noncall(); @@ -1071,7 +1071,7 @@ class InferTypesAndCallsAndFieldsVisitor final { *out_f_called = fun_ref; // (it's still may be a generic one, then Ts will be deduced from arguments) *out_dot_obj = dot_obj; } else { // so, it's `user.method` / `t.tupleAt` as a reference - if (fun_ref->is_compile_time_only()) { + if (fun_ref->is_compile_time_const_val() || fun_ref->is_compile_time_special_gen()) { fire(cur_f, v->get_identifier()->loc, "can not get reference to this method, it's compile-time only"); } fun_ref->mutate()->assign_is_used_as_noncall(); @@ -1186,7 +1186,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (fun_ref->is_generic_function()) { // if `f(args)` was called, Ts were inferred; check that all of them are known std::string_view nameT_unknown = deducingTs.get_first_not_deduced_nameT(); - if (!nameT_unknown.empty() && hint && fun_ref->declared_return_type) { + if (!nameT_unknown.empty() && hint && !hint->has_genericT_inside() && fun_ref->declared_return_type) { // example: `t.tupleFirst()`, T doesn't depend on arguments, but is determined by return type // if used like `var x: int = t.tupleFirst()` / `t.tupleFirst() as int` / etc., use hint deducingTs.auto_deduce_from_argument(cur_f, v->loc, fun_ref->declared_return_type, hint); diff --git a/tolk/pipe-optimize-boolean-expr.cpp b/tolk/pipe-optimize-boolean-expr.cpp index ef6c8208cb..6b053915f7 100644 --- a/tolk/pipe-optimize-boolean-expr.cpp +++ b/tolk/pipe-optimize-boolean-expr.cpp @@ -53,7 +53,7 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { auto v_not = createV(loc, "!", tok_logical_not, rhs); v_not->assign_inferred_type(TypeDataBool::create()); v_not->assign_rvalue_true(); - v_not->assign_fun_ref(lookup_global_symbol("!b_")->try_as()); + v_not->assign_fun_ref(lookup_function("!b_")); return v_not; } @@ -98,7 +98,7 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { auto v_neq = createV(v->loc, "!=", tok_neq, cond_not_not, v_zero); v_neq->mutate()->assign_rvalue_true(); v_neq->mutate()->assign_inferred_type(TypeDataBool::create()); - v_neq->mutate()->assign_fun_ref(lookup_global_symbol("_!=_")->try_as()); + v_neq->mutate()->assign_fun_ref(lookup_function("_!=_")); return v_neq; } } diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index bbb105e428..71b2d3bed7 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -20,6 +20,7 @@ #include "ast.h" #include "compiler-state.h" #include "generics-helpers.h" +#include "pack-unpack-serializers.h" #include "td/utils/crypto.h" #include @@ -141,12 +142,30 @@ static StructPtr register_struct(V v, StructPtr base_str fields.emplace_back(new StructFieldData(static_cast(v_field->get_identifier()->name), v_field->loc, i, v_field->type_node, default_value)); } + PackOpcode opcode(0, 0); + if (v->has_opcode()) { + auto v_opcode = v->get_opcode()->as(); + if (v_opcode->intval < 0 || v_opcode->intval > (1ULL << 48)) { + v->error("opcode must not exceed 2^48"); + } + opcode.pack_prefix = v_opcode->intval->to_long(); + + std::string_view prefix_str = v_opcode->orig_str; + if (prefix_str.starts_with("0x")) { + opcode.prefix_len = static_cast(prefix_str.size() - 2) * 4; + } else if (prefix_str.starts_with("0b")) { + opcode.prefix_len = static_cast(prefix_str.size() - 2); + } else { + tolk_assert(false); + } + } + std::string name = std::move(override_name); if (name.empty()) { name = v->get_identifier()->name; } const GenericsDeclaration* genericTs = nullptr; // at registering it's null; will be assigned after types resolving - StructData* s_sym = new StructData(std::move(name), v->loc, std::move(fields), genericTs, substitutedTs, v); + StructData* s_sym = new StructData(std::move(name), v->loc, std::move(fields), opcode, v->overflow1023_policy, genericTs, substitutedTs, v); s_sym->base_struct_ref = base_struct_ref; // for `Container`, here is `Container` G.symtable.add_struct(s_sym); diff --git a/tolk/pipeline.h b/tolk/pipeline.h index a46e3e0375..4e5beca20e 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -41,6 +41,7 @@ void pipeline_check_inferred_types(); void pipeline_refine_lvalue_for_mutate_arguments(); void pipeline_check_rvalue_lvalue(); void pipeline_check_pure_impure_operations(); +void pipeline_check_serialized_fields(); void pipeline_constant_folding(); void pipeline_optimize_boolean_expressions(); void pipeline_convert_ast_to_legacy_Expr_Op(); diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 7efccc75ee..a441919347 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -174,6 +174,24 @@ StructFieldPtr StructData::find_field(std::string_view field_name) const { return nullptr; } +// formats opcode as "x{...}" or "b{...}" +std::string StructData::PackOpcode::format_as_slice() const { + const int base = prefix_len % 4 == 0 ? 16 : 2; + const int s_len = base == 16 ? prefix_len / 4 : prefix_len; + const char* digits = "0123456789abcdef"; + + std::string result(s_len + 3, '0'); + result[0] = base == 16 ? 'x' : 'b'; + result[1] = '{'; + result[s_len + 3 - 1] = '}'; + int64_t opcode = pack_prefix; + for (int i = s_len - 1; i >= 0 && opcode != 0; --i) { + result[2 + i] = digits[opcode % base]; + opcode /= base; + } + return result; +} + GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_redefinition_of_symbol(SrcLocation loc, const Symbol* previous) { SrcLocation prev_loc = previous->loc; @@ -226,6 +244,12 @@ void GlobalSymbolTable::add_struct(StructPtr s_sym) { } } +void GlobalSymbolTable::replace_function(FunctionPtr f_sym) { + auto key = key_hash(f_sym->name); + assert(entries.contains(key)); + entries[key] = f_sym; +} + const Symbol* lookup_global_symbol(std::string_view name) { return G.symtable.lookup(name); } diff --git a/tolk/symtable.h b/tolk/symtable.h index ccc8bc5279..6bcba2aa73 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -111,7 +111,9 @@ struct FunctionData final : Symbol { flagAcceptsSelf = 512, // is a member function (has `self` first parameter) flagReturnsSelf = 1024, // return type is `self` (returns the mutated 1st argument), calls can be chainable flagReallyUsed = 2048, // calculated via dfs from used functions; declared but unused functions are not codegenerated - flagCompileTimeOnly = 4096, // calculated only at compile-time for constant arguments: `ton("0.05")`, `stringCrc32`, and others + flagCompileTimeVal = 4096, // calculated only at compile-time for constant arguments: `ton("0.05")`, `stringCrc32`, and others + flagCompileTimeGen = 8192, // at compile-time it's handled specially, not as a regular function: `T.toCell`, etc. + flagAllowAnyWidthT = 16384, // for built-in generic functions that is not restricted to be 1-slot type }; int tvm_method_id = EMPTY_TVM_METHOD_ID; @@ -196,7 +198,9 @@ struct FunctionData final : Symbol { bool does_return_self() const { return flags & flagReturnsSelf; } bool does_mutate_self() const { return (flags & flagAcceptsSelf) && parameters[0].is_mutate_parameter(); } bool is_really_used() const { return flags & flagReallyUsed; } - bool is_compile_time_only() const { return flags & flagCompileTimeOnly; } + bool is_compile_time_const_val() const { return flags & flagCompileTimeVal; } + bool is_compile_time_special_gen() const { return flags & flagCompileTimeGen; } + bool is_variadic_width_T_allowed() const { return flags & flagAllowAnyWidthT; } bool does_need_codegen() const; @@ -299,7 +303,26 @@ struct StructFieldData final : Symbol { }; struct StructData final : Symbol { + enum class Overflow1023Policy { // annotation @overflow1023_policy above a struct + not_specified, + suppress, + }; + + struct PackOpcode { + int64_t pack_prefix; + int prefix_len; + + PackOpcode(int64_t pack_prefix, int prefix_len) + : pack_prefix(pack_prefix), prefix_len(prefix_len) {} + + bool exists() const { return prefix_len != 0; } + + std::string format_as_slice() const; // "x{...}" (or "b{...}") + }; + std::vector fields; + PackOpcode opcode; + Overflow1023Policy overflow1023_policy; const GenericsDeclaration* genericTs; const GenericsSubstitutions* substitutedTs; @@ -316,9 +339,11 @@ struct StructData final : Symbol { StructData* mutate() const { return const_cast(this); } void assign_resolved_genericTs(const GenericsDeclaration* genericTs); - StructData(std::string name, SrcLocation loc, std::vector&& fields, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, AnyV ast_root) + StructData(std::string name, SrcLocation loc, std::vector&& fields, PackOpcode opcode, Overflow1023Policy overflow1023_policy, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, AnyV ast_root) : Symbol(std::move(name), loc) , fields(std::move(fields)) + , opcode(opcode) + , overflow1023_policy(overflow1023_policy) , genericTs(genericTs) , substitutedTs(substitutedTs) , ast_root(ast_root) { @@ -350,6 +375,8 @@ class GlobalSymbolTable { void add_type_alias(AliasDefPtr a_sym); void add_struct(StructPtr s_sym); + void replace_function(FunctionPtr f_sym); + const Symbol* lookup(std::string_view name) const { const auto it = entries.find(key_hash(name)); return it == entries.end() ? nullptr : it->second; diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp index fc38571bc4..6ead2b149e 100644 --- a/tolk/tolk.cpp +++ b/tolk/tolk.cpp @@ -64,6 +64,7 @@ int tolk_proceed(const std::string &entrypoint_filename) { pipeline_refine_lvalue_for_mutate_arguments(); pipeline_check_rvalue_lvalue(); pipeline_check_pure_impure_operations(); + pipeline_check_serialized_fields(); pipeline_constant_folding(); pipeline_optimize_boolean_expressions(); pipeline_convert_ast_to_legacy_Expr_Op(); diff --git a/tolk/tolk.h b/tolk/tolk.h index ad2a5898d3..6da5c1ff83 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -1100,6 +1100,7 @@ struct CodeBlob { #endif return ir_idx; } + var_idx_t create_int(SrcLocation loc, int64_t value, const char* desc); bool compute_used_code_vars(); bool compute_used_code_vars(std::unique_ptr& ops, const VarDescrList& var_info, bool edit) const; void print(std::ostream& os, int flags = 0) const; diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index cd249a5c37..d37a657784 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -721,6 +721,11 @@ bool TypeDataCell::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataStruct* rhs_struct = rhs->try_as()) { + if (rhs_struct->struct_ref->is_instantiation_of_generic_struct() && rhs_struct->struct_ref->base_struct_ref->name == "Cell") { + return true; // Cell to cell, e.g. `contract.setData(obj.toCell())` + } + } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { return can_rhs_be_assigned(rhs_alias->underlying_type); } @@ -995,6 +1000,9 @@ bool TypeDataCell::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataUnion* to_union = cast_to->try_as()) { return can_be_casted_to_union(this, to_union); } + if (const TypeDataStruct* to_struct = cast_to->try_as()) { // cell as Cell + return to_struct->struct_ref->is_instantiation_of_generic_struct() && to_struct->struct_ref->base_struct_ref->name == "Cell"; + } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } @@ -1107,6 +1115,9 @@ bool TypeDataStruct::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataUnion* to_union = cast_to->try_as()) { return can_be_casted_to_union(this, to_union); } + if (cast_to == TypeDataCell::create()) { // Cell as cell + return struct_ref->is_instantiation_of_generic_struct() && struct_ref->base_struct_ref->name == "Cell"; + } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } From 89fcd7b845968555700b83daaeb64cdb21fa09e8 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 25 May 2025 21:26:26 +0300 Subject: [PATCH 269/524] [Tolk] Compile-time `sizeof()` function for any variable --- crypto/smartcont/tolk-stdlib/common.tolk | 8 ++++++++ tolk-tester/tests/indexed-access.tolk | 2 +- tolk-tester/tests/invalid-semantics/err-4320.tolk | 13 +++++++++++++ tolk-tester/tests/struct-tests.tolk | 1 + tolk/builtins.cpp | 14 ++++++++++++++ 5 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 tolk-tester/tests/invalid-semantics/err-4320.tolk diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index db3dfc0ff2..29b298cd95 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -516,6 +516,14 @@ fun slice.depth(self): int fun builder.depth(self): int asm "BDEPTH"; +/// Returns the number of stack slots anyVariable occupies (works at compile-time). +/// Example: sizeof(nullableInt) = 1, because `int?` is 1 TVM slot holding either NULL or a value. +/// Example: sizeof(somePoint) = 2 for `struct Point { x:int, y: int }`: two fields one slot per each. +/// Useful for debugging or when preparing stack contents for RUNVM. +@pure +fun sizeof(anyVariable: T): int + builtin; + /** Debug primitives. diff --git a/tolk-tester/tests/indexed-access.tolk b/tolk-tester/tests/indexed-access.tolk index f84d2d93ed..113124dd54 100644 --- a/tolk-tester/tests/indexed-access.tolk +++ b/tolk-tester/tests/indexed-access.tolk @@ -239,7 +239,7 @@ fun getT() { return (1, 2); } @method_id(120) fun test120() { - return (getT().0 = 3, getT().0 = 4, [getT().0 = 5, getT().0 = 6]); + return (getT().0 = 3, getT().0 = sizeof(getT()) * 2, [getT().0 = 5, getT().0 = 6]); } @method_id(121) diff --git a/tolk-tester/tests/invalid-semantics/err-4320.tolk b/tolk-tester/tests/invalid-semantics/err-4320.tolk new file mode 100644 index 0000000000..f57df590e2 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4320.tolk @@ -0,0 +1,13 @@ +struct Point { + x: int; + y: int; +} + +fun main() { + return sizeof(Point); +} + +/** +@compilation_should_fail +@stderr `Point` only refers to a type, but is being used as a value here + */ diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 4bd407f59e..46da0207c0 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -98,6 +98,7 @@ fun test3() { @method_id(104) fun test4() { var p: Point = { x: 10, y: 20 }; + assert(sizeof(p) == 2, 100); return (p == null, p, p = {x:30,y:40}, p != null); } diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index ebcd2de926..6cee0985d8 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1119,6 +1119,17 @@ static AsmOp compile_any_object_to_tuple(std::vector& res, std::vector return exec_op(loc, std::to_string(args.size()) + " TUPLE", n, 1); } +// fun sizeof(anything: T): int; // (returns the number of stack elements) +static AsmOp compile_any_object_sizeof(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 1); + int n = static_cast(args.size()); + res[0].set_const(n); + for (int i = 0; i < n; ++i) { + args[i].unused(); + } + return AsmOp::IntConst(loc, td::make_refint(n)); +} + // fun ton(amount: slice): coins; ton("0.05") replaced by 50000000 at compile-time // same for stringCrc32(constString: slice) and others AsmOp compile_time_only_function(std::vector&, std::vector&, SrcLocation loc) { @@ -1372,6 +1383,9 @@ void define_builtins() { define_builtin_method("debug.dumpStack", debug, {}, Unit, nullptr, compile_dumpstk, 0); + define_builtin_func("sizeof", {typeT}, TypeDataInt::create(), declGenericT, + compile_any_object_sizeof, + FunctionData::flagMarkedAsPure | FunctionData::flagAllowAnyWidthT); // serialization/deserialization methods to/from cells (or, more low-level, slices/builders) // they work with structs (or, more low-level, with arbitrary types) From 5ae66704d8b797ef8dc1d4c691cc4b16b9f9299b Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Wed, 21 May 2025 17:39:19 +0300 Subject: [PATCH 270/524] [Tolk] Support default values for function parameters Now this works: > fun inc(x: int, by: int = 1) Will be used for serialization options. --- tolk-tester/tests/calls-tests.tolk | 164 ++++++++++++++++++ .../tests/invalid-declaration/err-1055.tolk | 19 ++ .../tests/invalid-declaration/err-1665.tolk | 8 + .../tests/invalid-declaration/err-1764.tolk | 9 + .../tests/invalid-semantics/err-4509.tolk | 14 ++ .../tests/invalid-typing/err-6179.tolk | 7 + .../tests/invalid-typing/err-6928.tolk | 7 + tolk-tester/tests/var-apply-tests.tolk | 12 ++ tolk/ast-from-tokens.cpp | 14 +- tolk/ast-replacer.h | 2 +- tolk/ast-replicator.h | 4 +- tolk/ast.h | 13 +- tolk/builtins.cpp | 2 +- tolk/constant-evaluator.cpp | 10 ++ tolk/generics-helpers.cpp | 2 +- tolk/pipe-ast-to-legacy.cpp | 11 +- tolk/pipe-check-inferred-types.cpp | 12 ++ tolk/pipe-constant-folding.cpp | 13 ++ tolk/pipe-infer-types-and-calls.cpp | 46 +++-- tolk/pipe-refine-lvalue-for-mutate.cpp | 2 +- tolk/pipe-register-symbols.cpp | 7 +- tolk/pipe-resolve-identifiers.cpp | 8 +- tolk/pipe-resolve-types.cpp | 9 +- tolk/symtable.cpp | 4 + tolk/symtable.h | 9 +- 25 files changed, 369 insertions(+), 39 deletions(-) create mode 100644 tolk-tester/tests/calls-tests.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1055.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1665.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1764.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4509.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6179.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6928.tolk diff --git a/tolk-tester/tests/calls-tests.tolk b/tolk-tester/tests/calls-tests.tolk new file mode 100644 index 0000000000..de508a2d5d --- /dev/null +++ b/tolk-tester/tests/calls-tests.tolk @@ -0,0 +1,164 @@ +const ZERO = 0; + +fun sumABDef0(a: int, b: int = 0) { + return a + b; +} + +fun sumADef1BDef2(a: int = ((1)), b: int = 2+ZERO) { + return a + b; +} + +fun getAOrB(a: T, b: T, getA: bool = false): T { + return getA ? a : b; +} + +fun pushToTuple(mutate t: tuple, v: T? = null) { + t.push(v); +} + +struct Point { + x: int; + y: int; +} + +fun Point.create(x: int = 0, y: int = 0): Point { + return {x,y}; +} + +fun Point.incrementX(mutate self, deltaX: int = 1) { + self.x += deltaX; +} + +fun makeCost(cost: coins = ton("0.05")) { + return cost + ton("0.05"); +} + +fun makeUnion(v: int | slice = 0) { + return v; +} + +struct MyOptions { + negate: bool = false, + mulBy: int = 0, +} + +global t105: tuple; + +fun log105(a: int, options: MyOptions = {}) { + if (options.negate) { + a = -a; + } + if (options.mulBy) { + a *= options.mulBy; + } + t105.push(a); +} + +struct AnotherOptions { + c1: int; + leaveSign: bool = true; +} + +fun helper106(a: int, options: AnotherOptions = { c1: 1 }) { + a *= options.c1; + return options.leaveSign ? +a : -a; +} + + +@method_id(101) +fun test1(a: int) { + return ( + sumABDef0(a), + sumABDef0(a, 5), + sumABDef0(100), + sumADef1BDef2(a), + sumADef1BDef2(), + sumADef1BDef2(a, 100), + sumADef1BDef2(200, 100), + ) +} + +@method_id(102) +fun test2(a: int) { + var t = createEmptyTuple(); + pushToTuple(mutate t, getAOrB(sumADef1BDef2(a), sumADef1BDef2(a, a))); + pushToTuple(mutate t, true); + pushToTuple(mutate t, null); + pushToTuple(mutate t, null); + pushToTuple(mutate t, null); + return t; +} + +@method_id(103) +fun test3() { + var p = Point.create(); + p.incrementX(); + var p2 = Point.create(8); + p2.incrementX(p.x += 1); + return (p, p2); +} + +@method_id(104) +fun test4() { + return ( + makeCost(), + makeCost(ton("0.1")), + makeUnion() is int, + makeUnion(), + makeUnion(makeCost()), + ) +} + +@method_id(105) +fun test5() { + t105 = createEmptyTuple(); + log105(1); + log105(1, { negate: true }); + log105(1, { mulBy: 100 }); + log105(1, { mulBy: 100, negate: true }); + log105(1, {}); + return t105; +} + +@method_id(106) +fun test6(l4: bool) { + return ( + helper106(5), + helper106(5, {leaveSign: false, c1: 1}), + helper106(5, {c1: 10}), + helper106(5, {leaveSign: l4, c1: 10}), + helper106(5, {c1: 0}), + ); +} + +fun int.plus(self, v: int = 1) { + return self + v; +} + +@method_id(107) +fun test7() { + return (10.plus(5.plus()), int.plus(4)); +} + +fun createTFrom(v: (int, (U, V)) = (1, (2, 3))) { + return [v.0, v.1.0, v.1.1]; +} + +@method_id(108) +fun test8() { + __expect_type(createTFrom, "((int, (coins, int8))) -> [int, coins, int8]"); + return (createTFrom(), createTFrom((5, (8, createTFrom().0)))); +} + +fun main() {} + +/** +@testcase | 101 | 10 | 10 15 100 12 3 110 300 +@testcase | 102 | 10 | [ 20 -1 (null) (null) (null) ] +@testcase | 103 | | 2 0 10 0 +@testcase | 104 | | 100000000 150000000 -1 0 1 100000000 1 +@testcase | 105 | | [ 1 -1 100 -100 1 ] +@testcase | 106 | 0 | 5 -5 50 -50 0 +@testcase | 107 | | 16 5 +@testcase | 108 | | [ 1 2 3 ] [ 5 8 1 ] + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1055.tolk b/tolk-tester/tests/invalid-declaration/err-1055.tolk new file mode 100644 index 0000000000..3b89f53e0b --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1055.tolk @@ -0,0 +1,19 @@ + +struct Options { + o1: bool; + o2: bool; +} + +fun getBool() { return true; } + +fun f(x: Options = { + o1: true, + o2: getBool(), +}) { +} + +/** +@compilation_should_fail +@stderr not a constant expression +@stderr o2: getBool() + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1665.tolk b/tolk-tester/tests/invalid-declaration/err-1665.tolk new file mode 100644 index 0000000000..4ba5297b9a --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1665.tolk @@ -0,0 +1,8 @@ +fun f(a: int, b: int = a) { + +} + +/** +@compilation_should_fail +@stderr symbol `a` is not a constant + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1764.tolk b/tolk-tester/tests/invalid-declaration/err-1764.tolk new file mode 100644 index 0000000000..6afaa18498 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1764.tolk @@ -0,0 +1,9 @@ + +fun increment(mutate x: int = 0) { + x += 1; +} + +/** +@compilation_should_fail +@stderr `mutate` parameter can't have a default value + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4509.tolk b/tolk-tester/tests/invalid-semantics/err-4509.tolk new file mode 100644 index 0000000000..d168ddb968 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4509.tolk @@ -0,0 +1,14 @@ +fun f(a: int = 0, b: int) { + +} + +fun main() { + f(1, 2); // ok + return f(); +} + +/** +@compilation_should_fail +@stderr too few arguments in call to `f`, expected 2, have 0 +@stderr f(); + */ diff --git a/tolk-tester/tests/invalid-typing/err-6179.tolk b/tolk-tester/tests/invalid-typing/err-6179.tolk new file mode 100644 index 0000000000..73148590b2 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6179.tolk @@ -0,0 +1,7 @@ +fun f(x: slice = ton("0.04")) { +} + +/** +@compilation_should_fail +@stderr can not assign `coins` to `slice` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6928.tolk b/tolk-tester/tests/invalid-typing/err-6928.tolk new file mode 100644 index 0000000000..a335647ace --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6928.tolk @@ -0,0 +1,7 @@ +fun int.f(self, v: (int, int) = (0, 1+"")) { +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int` and `slice` + */ diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index 1c0f164cba..825b48b96b 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -314,6 +314,17 @@ fun testMethodsOfGenericStruct() { return (eq(w1, w2), w2.value = w1.value as int, eq(w1, w2), creator3()); } +fun add3WithDefaults(a: int, b: int = 0, c: int = 0) { + return a + b + c; +} + +@method_id(119) +fun testSavingFunWithDefaults() { + var cc = add3WithDefaults; + __expect_type(cc, "(int, int, int) -> int"); + return cc(1, 2, 3); +} + fun main() {} @@ -343,4 +354,5 @@ fun main() {} @testcase | 116 | | 7 11 80 @testcase | 117 | | 6 8 @testcase | 118 | | 0 10 -1 (null) +@testcase | 119 | | 6 */ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 6f4b53b795..0ca8b3aebe 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -331,7 +331,17 @@ static AnyV parse_parameter(Lexer& lex, AnyTypeV self_type) { lex.error("`self` parameter should not have a type"); } - return createV(loc, param_name, param_type, declared_as_mutate); + // optional default value + AnyExprV default_value = nullptr; + if (lex.tok() == tok_assign && !is_self) { // `a: int = 0` + if (declared_as_mutate) { + lex.error("`mutate` parameter can't have a default value"); + } + lex.next(); + default_value = parse_expr(lex); + } + + return createV(loc, param_name, param_type, default_value, declared_as_mutate); } static AnyV parse_global_var_declaration(Lexer& lex, const std::vector>& annotations) { @@ -1489,8 +1499,6 @@ static AnyV parse_struct_field(Lexer& lex) { if (lex.tok() == tok_assign) { // `id: int = 3` lex.next(); default_value = parse_expr(lex); - } else { - default_value = createV(lex.cur_location()); } return createV(loc, v_ident, default_value, declared_type); diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 382f2bd715..03e7d43206 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -200,7 +200,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { public: virtual bool should_visit_function(FunctionPtr fun_ref) = 0; - void start_replacing_in_function(FunctionPtr fun_ref, V v_function) { + virtual void start_replacing_in_function(FunctionPtr fun_ref, V v_function) { replace(v_function->get_body()); } }; diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 73ada41000..03a9b0a53b 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -223,13 +223,13 @@ class ASTReplicator final { return createV(v->loc, clone(v->get_items())); } static V clone(V v) { - return createV(v->loc, v->param_name, clone(v->type_node), v->declared_as_mutate); + return createV(v->loc, v->param_name, clone(v->type_node), v->default_value ? clone(v->default_value) : nullptr, v->declared_as_mutate); } static V clone(V v) { return createV(v->loc, clone(v->get_params())); } static V clone(V v) { - return createV(v->loc, clone(v->get_identifier()), clone(v->get_default_value()), clone(v->type_node)); + return createV(v->loc, clone(v->get_identifier()), v->default_value ? clone(v->default_value) : nullptr, clone(v->type_node)); } static V clone(V v) { return createV(v->loc, clone(v->get_all_fields())); diff --git a/tolk/ast.h b/tolk/ast.h index 96fc96937d..cf82a0a178 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -1179,16 +1179,18 @@ struct Vertex final : ASTOtherVararg { template<> // ast_parameter is a parameter of a function in its declaration // example: `fun f(a: int, mutate b: slice)` has 2 parameters +// example: `fun f(a: int = 0)` has 1 parameter with default value struct Vertex final : ASTOtherLeaf { std::string_view param_name; AnyTypeV type_node; // always exists, typing parameters is mandatory + AnyExprV default_value; // default value of the parameter or nullptr bool declared_as_mutate; // declared as `mutate param_name` bool is_underscore() const { return param_name.empty(); } - Vertex(SrcLocation loc, std::string_view param_name, AnyTypeV type_node, bool declared_as_mutate) + Vertex(SrcLocation loc, std::string_view param_name, AnyTypeV type_node, AnyExprV default_value, bool declared_as_mutate) : ASTOtherLeaf(ast_parameter, loc) - , param_name(param_name), type_node(type_node), declared_as_mutate(declared_as_mutate) {} + , param_name(param_name), type_node(type_node), default_value(default_value), declared_as_mutate(declared_as_mutate) {} }; template<> @@ -1314,14 +1316,13 @@ template<> // example: `struct Point { x: int, y: int }` is struct declaration, its body contains 2 fields struct Vertex final : ASTOtherVararg { AnyTypeV type_node; // always exists, typing struct fields is mandatory + AnyExprV default_value; // nullptr if no default auto get_identifier() const { return children.at(0)->as(); } - bool has_default_value() const { return children.at(1)->kind != ast_empty_expression; } - auto get_default_value() const { return child_as_expr(1); } Vertex(SrcLocation loc, V name_identifier, AnyExprV default_value, AnyTypeV type_node) - : ASTOtherVararg(ast_struct_field, loc, {name_identifier, default_value}) - , type_node(type_node) {} + : ASTOtherVararg(ast_struct_field, loc, {name_identifier}) + , type_node(type_node), default_value(default_value) {} }; template<> diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 6cee0985d8..1f205f2ca1 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -32,7 +32,7 @@ static std::vector define_builtin_parameters(const std::vector(params_types.size()); ++i) { - LocalVarData p_sym("", {}, params_types[i], (i == 0 && is_mutate_self) * LocalVarData::flagMutateParameter, i); + LocalVarData p_sym("", {}, params_types[i], nullptr, (i == 0 && is_mutate_self) * LocalVarData::flagMutateParameter, i); parameters.push_back(std::move(p_sym)); } diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index c6e2f43313..953cc317cd 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -266,6 +266,13 @@ struct ConstantExpressionChecker { } } + // `a: Options = {}`, object literal may occur as a default value for parameters + static void handle_object_literal(V v) { + for (int i = 0; i < v->get_num_fields(); ++i) { + visit(v->get_field(i)->get_init_val()); + } + } + static void visit(AnyExprV v) { if (v->try_as() || v->try_as() || v->try_as() || v->try_as()) { return; @@ -285,6 +292,9 @@ struct ConstantExpressionChecker { if (auto v_tensor = v->try_as()) { return handle_tensor(v_tensor); } + if (auto v_obj = v->try_as()) { + return handle_object_literal(v_obj->get_body()); + } if (auto v_par = v->try_as()) { return visit(v_par->get_expr()); } diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index 34d2f8f333..c2c7a9b546 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -366,7 +366,7 @@ FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstituti new_parameters.reserve(fun_ref->get_num_params()); for (const LocalVarData& orig_p : fun_ref->parameters) { TypePtr new_param_type = replace_genericT_with_deduced(orig_p.declared_type, allocatedTs); - new_parameters.emplace_back(orig_p.name, orig_p.loc, new_param_type, orig_p.flags, orig_p.param_idx); + new_parameters.emplace_back(orig_p.name, orig_p.loc, new_param_type, orig_p.default_value, orig_p.flags, orig_p.param_idx); } TypePtr new_return_type = replace_genericT_with_deduced(fun_ref->declared_return_type, allocatedTs); TypePtr new_receiver_type = replace_genericT_with_deduced(fun_ref->receiver_type, allocatedTs); diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 9d58a7faac..773d67eccb 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1380,10 +1380,11 @@ static std::vector process_function_call(V v, Code return transition_to_target_type(std::move(rvect), code, target_type, v); } + // fill args for evaluation: dot object + passed arguments + parameters defaults if not all passed AnyExprV obj_leftmost = v->get_self_obj(); int delta_self = obj_leftmost != nullptr; std::vector args; - args.reserve(delta_self + v->get_num_args()); + args.reserve(fun_ref->get_num_params()); if (delta_self) { args.push_back(obj_leftmost); while (obj_leftmost->kind == ast_function_call && obj_leftmost->as()->get_self_obj() && obj_leftmost->as()->fun_maybe && obj_leftmost->as()->fun_maybe->does_return_self()) { @@ -1393,6 +1394,14 @@ static std::vector process_function_call(V v, Code for (int i = 0; i < v->get_num_args(); ++i) { args.push_back(v->get_arg(i)->get_expr()); } + for (int i = delta_self + v->get_num_args(); i < fun_ref->get_num_params(); ++i) { + LocalVarPtr param_ref = &fun_ref->get_param(i); + tolk_assert(param_ref->has_default_value()); + SrcLocation last_loc = args.empty() ? v->loc : args.back()->loc; + ASTAuxData *aux_data = new AuxData_ForceFiftLocation(last_loc); + auto v_force_loc = createV(last_loc, param_ref->default_value, aux_data, param_ref->declared_type); + args.push_back(v_force_loc); + } // the purpose of tensor_tt ("tensor target type") is to transition `null` to `(int, int)?` and so on // the purpose of calling `pre_compile_tensor_inner` is to have 0-th IR vars to handle return self diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 8500ec59a4..ba15825592 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -730,6 +730,18 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { fire(fun_ref, v_function->get_body()->as()->loc_end, "missing return"); } } + + // visit default values of parameters + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + if (LocalVarPtr param_ref = &fun_ref->get_param(i); param_ref->has_default_value()) { + parent::visit(param_ref->default_value); + + TypePtr inferred_type = param_ref->default_value->inferred_type; + if (!param_ref->declared_type->can_rhs_be_assigned(inferred_type)) { + throw ParseError(param_ref->loc, "can not assign " + to_string(inferred_type) + " to " + to_string(param_ref->declared_type)); + } + } + } } // given `const a = 2 + 3` check types within its init_value diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index f16bf90a5e..74fbb3453a 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -150,6 +150,19 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } + void start_replacing_in_function(FunctionPtr fun_ref, V v_function) override { + // visit default values of parameters + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + if (LocalVarPtr param_ref = &fun_ref->get_param(i); param_ref->has_default_value()) { + check_expression_is_constant(param_ref->default_value); + AnyExprV replaced = replace_in_expression(param_ref->default_value); + param_ref->mutate()->assign_default_value(replaced); + } + } + + parent::replace(v_function->get_body()); + } + // used to replace `ton("0.05")` and other compile-time functions inside fields defaults, etc. AnyExprV replace_in_expression(AnyExprV init_value) { return parent::replace(init_value); diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index dd83a8dc26..0345ac0aa1 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -254,6 +254,28 @@ static MethodCallCandidate choose_only_method_to_call(FunctionPtr cur_f, SrcLoca fire(cur_f, loc, msg.str()); } +// given fun `f` and a call `f(a,b,c)`, check that argument count is expected; +// (parameters may have default values, so it's not as trivial as to compare params and args size) +void check_arguments_count_at_fun_call(FunctionPtr cur_f, V v, FunctionPtr called_f, AnyExprV self_obj) { + int delta_self = self_obj != nullptr; + int n_arguments = v->get_num_args() + delta_self; + int n_max_params = called_f->get_num_params(); + int n_min_params = n_max_params; + while (n_min_params && called_f->get_param(n_min_params - 1).has_default_value()) { + n_min_params--; + } + + if (!called_f->does_accept_self() && self_obj) { // static method `Point.create(...)` called as `p.create()` + fire(cur_f, v->loc, "method " + to_string(called_f) + " can not be called via dot\n(it's a static method, it does not accept `self`)"); + } + if (n_max_params < n_arguments) { + fire(cur_f, v->loc, "too many arguments in call to " + to_string(called_f) + ", expected " + std::to_string(n_max_params - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); + } + if (n_arguments < n_min_params) { + fire(cur_f, v->loc, "too few arguments in call to " + to_string(called_f) + ", expected " + std::to_string(n_min_params - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); + } +} + /* * This class handles all types of AST vertices and traverses them, filling all AnyExprV::inferred_type. * Note, that it isn't derived from ASTVisitor, it has manual `switch` over all existing vertex types. @@ -1119,19 +1141,9 @@ class InferTypesAndCallsAndFieldsVisitor final { // so, we have a call `f(args)` or `obj.f(args)`, f is fun_ref (function / method) (code / asm / builtin) // we're going to iterate over passed arguments, and (if generic) infer substitutedTs - // at first, check arguments count (Tolk doesn't have optional parameters, so just compare counts) + // at first, check argument count int delta_self = self_obj != nullptr; - int n_arguments = v->get_num_args() + delta_self; - int n_parameters = fun_ref->get_num_params(); - if (!fun_ref->does_accept_self() && self_obj) { // static method `Point.create(...)` called as `p.create()` - fire(cur_f, v->loc, "method " + to_string(fun_ref) + " can not be called via dot\n(it's a static method, it does not accept `self`)"); - } - if (n_parameters < n_arguments) { - fire(cur_f, v->loc, "too many arguments in call to " + to_string(fun_ref) + ", expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); - } - if (n_arguments < n_parameters) { - fire(cur_f, v->loc, "too few arguments in call to " + to_string(fun_ref) + ", expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); - } + check_arguments_count_at_fun_call(cur_f, v, fun_ref, self_obj); // for every passed argument, we need to infer its type // for generic functions, we need to infer type arguments (substitutedTs) on the fly @@ -1638,6 +1650,16 @@ class InferTypesAndCallsAndFieldsVisitor final { tolk_assert(fun_ref->declared_return_type); } + // visit default values of parameters; to correctly track symbols in `fun f(a: int, b: int = a)`, use flow context + FlowContext params_flow; + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + LocalVarPtr param_ref = &fun_ref->get_param(i); + if (param_ref->has_default_value()) { + params_flow = infer_any_expr(param_ref->default_value, std::move(params_flow), false, param_ref->declared_type).out_flow; + } + params_flow.register_known_type(SinkExpression(param_ref), param_ref->declared_type); + } + assign_fun_full_type(fun_ref, inferred_return_type); fun_ref->mutate()->assign_is_type_inferring_done(); } diff --git a/tolk/pipe-refine-lvalue-for-mutate.cpp b/tolk/pipe-refine-lvalue-for-mutate.cpp index 7692b0a2b2..3e02cf7738 100644 --- a/tolk/pipe-refine-lvalue-for-mutate.cpp +++ b/tolk/pipe-refine-lvalue-for-mutate.cpp @@ -65,7 +65,7 @@ class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBod } int delta_self = v->get_self_obj() != nullptr; - tolk_assert(fun_ref->get_num_params() == delta_self + v->get_num_args()); + tolk_assert(fun_ref->get_num_params() >= delta_self + v->get_num_args()); if (delta_self && fun_ref->does_mutate_self()) { // for `b.storeInt()`, `b` should become lvalue, since `storeInt` is a method mutating self diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 71b2d3bed7..7eaa6ac91b 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -132,14 +132,13 @@ static StructPtr register_struct(V v, StructPtr base_str for (int i = 0; i < v_body->get_num_fields(); ++i) { auto v_field = v_body->get_field(i); std::string field_name = static_cast(v_field->get_identifier()->name); - AnyExprV default_value = v_field->has_default_value() ? v_field->get_default_value() : nullptr; for (StructFieldPtr prev : fields) { if (UNLIKELY(prev->name == field_name)) { v_field->error("redeclaration of field `" + field_name + "`"); } } - fields.emplace_back(new StructFieldData(static_cast(v_field->get_identifier()->name), v_field->loc, i, v_field->type_node, default_value)); + fields.emplace_back(new StructFieldData(field_name, v_field->loc, i, v_field->type_node, v_field->default_value)); } PackOpcode opcode(0, 0); @@ -176,7 +175,7 @@ static StructPtr register_struct(V v, StructPtr base_str static LocalVarData register_parameter(V v, int idx) { if (v->is_underscore()) { - return LocalVarData{"", v->loc, v->type_node, 0, idx}; + return LocalVarData{"", v->loc, v->type_node, v->default_value, 0, idx}; } int flags = 0; @@ -186,7 +185,7 @@ static LocalVarData register_parameter(V v, int idx) { if (!v->declared_as_mutate && idx == 0 && v->param_name == "self") { flags |= LocalVarData::flagImmutable; } - return LocalVarData(static_cast(v->param_name), v->loc, v->type_node, flags, idx); + return LocalVarData(static_cast(v->param_name), v->loc, v->type_node, v->default_value, flags, idx); } static FunctionPtr register_function(V v, FunctionPtr base_fun_ref = nullptr, std::string override_name = {}, const GenericsSubstitutions* substitutedTs = nullptr) { diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index ab17c5b42c..2f6694469e 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -133,7 +133,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { static FunctionPtr cur_f; static LocalVarPtr create_local_var_sym(std::string_view name, SrcLocation loc, AnyTypeV declared_type_node, bool immutable, bool lateinit) { - LocalVarData* v_sym = new LocalVarData(static_cast(name), loc, declared_type_node, immutable * LocalVarData::flagImmutable + lateinit * LocalVarData::flagLateInit, -1); + LocalVarData* v_sym = new LocalVarData(static_cast(name), loc, declared_type_node, nullptr, immutable * LocalVarData::flagImmutable + lateinit * LocalVarData::flagLateInit, -1); current_scope.add_local_var(v_sym); return v_sym; } @@ -274,7 +274,11 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { auto v_block = v->get_body()->as(); current_scope.open_scope(v->loc); for (int i = 0; i < fun_ref->get_num_params(); ++i) { - current_scope.add_local_var(&fun_ref->parameters[i]); + LocalVarPtr param_ref = &fun_ref->parameters[i]; + current_scope.add_local_var(param_ref); + if (param_ref->has_default_value()) { + parent::visit(param_ref->default_value); + } } parent::visit(v_block); current_scope.close_scope(v_block->loc_end); diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 5a2aa0a4af..25d98bb46a 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -537,9 +537,12 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { type_nodes_visitor = TypeNodesVisitorResolver(fun_ref, fun_ref->genericTs, fun_ref->substitutedTs, false); for (int i = 0; i < fun_ref->get_num_params(); ++i) { - const LocalVarData& param_var = fun_ref->parameters[i]; - TypePtr declared_type = finalize_type_node(param_var.type_node); - param_var.mutate()->assign_resolved_type(declared_type); + LocalVarPtr param_ref = &fun_ref->parameters[i]; + TypePtr declared_type = finalize_type_node(param_ref->type_node); + param_ref->mutate()->assign_resolved_type(declared_type); + if (param_ref->has_default_value()) { + parent::visit(param_ref->default_value); + } } if (fun_ref->return_type_node) { TypePtr declared_return_type = finalize_type_node(fun_ref->return_type_node); diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index a441919347..46d3949383 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -141,6 +141,10 @@ void LocalVarData::assign_inferred_type(TypePtr inferred_type) { this->declared_type = inferred_type; } +void LocalVarData::assign_default_value(AnyExprV default_value) { + this->default_value = default_value; +} + void AliasDefData::assign_resolved_genericTs(const GenericsDeclaration* genericTs) { if (this->substitutedTs == nullptr) { this->genericTs = genericTs; diff --git a/tolk/symtable.h b/tolk/symtable.h index 6bcba2aa73..15a52d148b 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -54,20 +54,23 @@ struct LocalVarData final : Symbol { AnyTypeV type_node; // either at declaration `var x:int`, or if omitted, from assigned value `var x=2` TypePtr declared_type = nullptr; // = resolved type_node + AnyExprV default_value = nullptr; // for function parameters, if it has a default value int flags; int param_idx; // 0...N for function parameters, -1 for local vars std::vector ir_idx; - LocalVarData(std::string name, SrcLocation loc, AnyTypeV type_node, int flags, int param_idx) + LocalVarData(std::string name, SrcLocation loc, AnyTypeV type_node, AnyExprV default_value, int flags, int param_idx) : Symbol(std::move(name), loc) , type_node(type_node) + , default_value(default_value) , flags(flags) , param_idx(param_idx) { } - LocalVarData(std::string name, SrcLocation loc, TypePtr declared_type, int flags, int param_idx) + LocalVarData(std::string name, SrcLocation loc, TypePtr declared_type, AnyExprV default_value, int flags, int param_idx) : Symbol(std::move(name), loc) , type_node(nullptr) // for built-in functions (their parameters) , declared_type(declared_type) + , default_value(default_value) , flags(flags) , param_idx(param_idx) { } @@ -77,11 +80,13 @@ struct LocalVarData final : Symbol { bool is_immutable() const { return flags & flagImmutable; } bool is_lateinit() const { return flags & flagLateInit; } bool is_mutate_parameter() const { return flags & flagMutateParameter; } + bool has_default_value() const { return default_value != nullptr; } LocalVarData* mutate() const { return const_cast(this); } void assign_ir_idx(std::vector&& ir_idx); void assign_resolved_type(TypePtr declared_type); void assign_inferred_type(TypePtr inferred_type); + void assign_default_value(AnyExprV default_value); }; struct FunctionBodyCode; From b13bcb4bbc98ba4a82d98dbba89750d5082b8baa Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Wed, 28 May 2025 14:48:17 +0300 Subject: [PATCH 271/524] [Tolk] Struct PackOptions and UnpackOptions Allow to control fromCell/toCell behavior --- crypto/smartcont/tolk-stdlib/common.tolk | 54 ++++- .../tests/invalid-symbol/err-2833.tolk | 13 + tolk-tester/tests/pack-unpack-1.tolk | 8 +- tolk-tester/tests/pack-unpack-4.tolk | 42 ++-- tolk-tester/tests/pack-unpack-5.tolk | 24 +- tolk-tester/tests/pack-unpack-6.tolk | 223 ++++++++++++++++++ tolk-tester/tests/try-catch-tests.tolk | 21 +- tolk/builtins.cpp | 56 ++++- tolk/constant-evaluator.cpp | 16 +- tolk/optimize.cpp | 30 +++ tolk/pack-unpack-api.cpp | 31 ++- tolk/pack-unpack-api.h | 10 +- tolk/pack-unpack-serializers.cpp | 140 ++++++----- tolk/pack-unpack-serializers.h | 18 +- tolk/pipe-ast-to-legacy.cpp | 43 ++-- tolk/pipe-check-serialized-fields.cpp | 4 +- tolk/tolk.h | 3 + 17 files changed, 575 insertions(+), 161 deletions(-) create mode 100644 tolk-tester/tests/invalid-symbol/err-2833.tolk create mode 100644 tolk-tester/tests/pack-unpack-6.tolk diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 29b298cd95..366345385e 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -219,6 +219,32 @@ fun commitContractDataAndActions(): void Auto packing structures to/from cells. */ +/// PackOptions allows you to control behavior of `obj.toCell()` and similar functions. +struct PackOptions { + /// when a struct has a field of type `bits128` and similar (it's a slice under the hood), + /// by default, compiler inserts runtime checks (get bits/refs count + compare with 128 + compare with 0); + /// these checks ensure that serialized binary data will be correct, but they cost gas; + /// however, if you guarantee that a slice is valid (for example, it comes from trusted sources), + /// set this option to true to disable runtime checks; + /// note: `int32` and other are always validated for overflow without any extra gas, + /// so this flag controls only rarely used `bytesN` / `bitsN` types + skipBitsNFieldsValidation: bool = false, +} + +/// UnpackOptions allows you to control behavior of `MyStruct.fromCell(c)` and similar functions. +struct UnpackOptions { + // after finished reading all fields from a cell/slice, call [slice.assertEnd] to ensure no remaining data left; + // it's the default behavior, it ensures that you've fully described data you're reading with a struct; + // example: `struct Point { x: int8; y: int8 }`, input "0102" is ok, "0102FF" will throw excno 9; + // note: setting this to false does not decrease gas (DROP from a stack and ENDS cost the same); + // note: this option controls [T.fromCell] and [T.fromSlice], but is ignored by [slice.loadAny] + assertEndAfterReading: bool = true, + + /// this excNo is thrown if a prefix doesn't match, e.g. for `struct (0x01) A` given input "88..."; + /// similarly, for a union type, this is thrown when none of the opcodes match + throwIfOpcodeDoesNotMatch: int = 63, +} + /// Convert anything to a cell (most likely, you'll call it for structures). /// Example: /// ``` @@ -228,7 +254,7 @@ fun commitContractDataAndActions(): void /// Internally, a builder is created, all fields are serialized one by one, and a builder is flushed /// (beginCell() + serialize fields + endCell()). @pure -fun T.toCell(self): Cell +fun T.toCell(self, options: PackOptions = {}): Cell builtin; /// Parse anything from a cell (most likely, you'll call it for structures). @@ -239,7 +265,7 @@ fun T.toCell(self): Cell /// Internally, a cell is unpacked to a slice, and that slice is parsed /// (packedCell.beginParse() + read from slice). @pure -fun T.fromCell(packedCell: cell): T +fun T.fromCell(packedCell: cell, options: UnpackOptions = {}): T builtin; /// Parse anything from a slice (most likely, you'll call it for structures). @@ -252,7 +278,7 @@ fun T.fromCell(packedCell: cell): T /// Note, that a passed slice is NOT mutated, its internal pointer is NOT shifted. /// If you need to mutate it, like `cs.loadInt()`, consider calling `cs.loadAny()`. @pure -fun T.fromSlice(rawSlice: slice): T +fun T.fromSlice(rawSlice: slice, options: UnpackOptions = {}): T builtin; /// Parse anything from a slice, shifting its internal pointer. @@ -262,8 +288,10 @@ fun T.fromSlice(rawSlice: slice): T /// var st: MyStorage = cs.loadAny(); // or cs.loadAny() /// ``` /// Similar to `MyStorage.fromSlice(cs)`, but called as a slice method and mutates the slice. +/// Note: [options.assertEndAfterReading] is ignored by this function, because it's actually intended +/// to read data from the middle. @pure -fun slice.loadAny(mutate self): T +fun slice.loadAny(mutate self, options: UnpackOptions = {}): T builtin; /// Skip anything in a slice, shifting its internal pointer. @@ -274,7 +302,7 @@ fun slice.loadAny(mutate self): T /// cs.skipAny(); // skips 64 bits /// ``` @pure -fun slice.skipAny(mutate self): self +fun slice.skipAny(mutate self, options: UnpackOptions = {}): self builtin; /// Store anything to a builder. @@ -284,7 +312,19 @@ fun slice.skipAny(mutate self): self /// var b = beginCell().storeUint(32).storeAny(msgBody).endCell(); /// ``` @pure -fun builder.storeAny(mutate self, v: T): self +fun builder.storeAny(mutate self, v: T, options: PackOptions = {}): self + builtin; + +/// Returns serialization prefix of a struct. Works at compile-time. +/// Example: for `struct (0xF0) AssetRegular { ... }` will return `240`. +@pure +fun T.getDeclaredPackPrefix(): int + builtin; + +/// Returns serialization prefix length of a struct. Works at compile-time. +/// Example: for `struct (0xF0) AssetRegular { ... }` will return `16`. +@pure +fun T.getDeclaredPackPrefixLen(): int builtin; /// Cell represents a typed cell reference (as opposed to untyped `cell`). @@ -316,7 +356,7 @@ struct Cell { /// var extra = st.extra.load(); // it's ExtraData, unpacked from loaded ref /// ``` @pure -fun Cell.load(self): T +fun Cell.load(self, options: UnpackOptions = {}): T builtin; /// Converts a typed cell into a slice. diff --git a/tolk-tester/tests/invalid-symbol/err-2833.tolk b/tolk-tester/tests/invalid-symbol/err-2833.tolk new file mode 100644 index 0000000000..8a7d249ff7 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2833.tolk @@ -0,0 +1,13 @@ +struct Point { + x: int8; + y: int8; +} + +fun main() { + return Point.getDeclaredPackPrefix(); +} + +/** +@compilation_should_fail +@stderr type `Point` does not have a serialization prefix + */ diff --git a/tolk-tester/tests/pack-unpack-1.tolk b/tolk-tester/tests/pack-unpack-1.tolk index 00767042c6..002df87ddc 100644 --- a/tolk-tester/tests/pack-unpack-1.tolk +++ b/tolk-tester/tests/pack-unpack-1.tolk @@ -48,7 +48,7 @@ fun test3() { @method_id(104) fun test4() { var s = stringHexToSlice("000000010000000200000003"); - var p = Point.fromSlice(s); // does not mutate s + var p = Point.fromSlice(s, {assertEndAfterReading: false}); // does not mutate s return (p, s.remainingBitsCount()); } @@ -114,11 +114,11 @@ fun main(c: cell) { 32 STI // b ENDC // c CTOS // s - 32 PUSHINT // s '12=32 + 32 PUSHINT // s '15=32 SDSKIPFIRST // s - 32 PUSHINT // s '13=32 + 32 PUSHINT // s '16=32 SDSKIPFIRST // s - SBITS // '14 + SBITS // '17 }> """ diff --git a/tolk-tester/tests/pack-unpack-4.tolk b/tolk-tester/tests/pack-unpack-4.tolk index 4839032dce..992a0664aa 100644 --- a/tolk-tester/tests/pack-unpack-4.tolk +++ b/tolk-tester/tests/pack-unpack-4.tolk @@ -92,7 +92,7 @@ fun invalid105_slice(): slice asm "b{1100} PUSHSLICE"; // prefix 0b11 doesn't e @method_id(105) fun test5() { try { - var u = U105.fromSlice(invalid105_slice()); + var u = U105.fromSlice(invalid105_slice(), {throwIfOpcodeDoesNotMatch: 9}); return (u is int8) ? 8 : -8; } catch (excode) { return excode; @@ -125,37 +125,37 @@ fun main() {} """ test6 PROC:<{ // b{010000000000001111} PUSHSLICE // s - b{00} SDBEGINSQ // s '6 + b{00} SDBEGINSQ // s '8 IF:<{ // s - 8 LDI // '10 s - DROP // '10 - 42 PUSHINT // 'USlot1 'UTag=42 + 8 LDI // '12 s + 42 PUSHINT // 'USlot1 s 'UTag=42 }>ELSE<{ // s - b{01} SDBEGINSQ // s '6 + b{01} SDBEGINSQ // s '8 IF:<{ // s - 16 LDI // '15 s - DROP // '15 - 44 PUSHINT // 'USlot1 'UTag=44 + 16 LDI // '17 s + 44 PUSHINT // 'USlot1 s 'UTag=44 }>ELSE<{ // s - b{10} SDBEGINSQ // s '6 + b{10} SDBEGINSQ // s '8 IF:<{ // s - 32 LDI // '20 s - DROP // '20 - 46 PUSHINT // 'USlot1 'UTag=46 + 32 LDI // '22 s + 46 PUSHINT // 'USlot1 s 'UTag=46 }>ELSE<{ // s - b{11} SDBEGINSQ // s '6 - DROP // s - 64 LDI // '25 s - DROP // '25 - 48 PUSHINT // 'USlot1 'UTag=48 + b{11} SDBEGINSQ // s '8 + IFNOTJMP:<{ // s + 63 THROW + }> + 64 LDI // '27 s + 48 PUSHINT // 'USlot1 s 'UTag=48 }> }> - }> // u.USlot1 u.UTag - 44 EQINT // u.USlot1 '27 + }> + SWAP // 'USlot1 'UTag s + ENDS // u.USlot1 u.UTag + 44 EQINT // u.USlot1 '29 IFJMP:<{ // u.USlot1 }> // u.USlot1 DROP // - -1 PUSHINT // '29=-1 + -1 PUSHINT // '31=-1 }> """ */ diff --git a/tolk-tester/tests/pack-unpack-5.tolk b/tolk-tester/tests/pack-unpack-5.tolk index 068d031f23..9644d7331f 100644 --- a/tolk-tester/tests/pack-unpack-5.tolk +++ b/tolk-tester/tests/pack-unpack-5.tolk @@ -166,18 +166,38 @@ fun test9() { return (Test9_f1.estimatePackSize(), Test9_f2.estimatePackSize(), Test9_f3.estimatePackSize(), Test9_f4.estimatePackSize()); } +struct Test10_1 { + a: int32; + b: builder; // unpredictable +} + +type Test10_2 = (Test10_1, bool?, RemainingBitsAndRefs); + +@method_id(110) +fun test10() { + return (Test10_1.estimatePackSize(), Test10_2.estimatePackSize()); +} + +@method_id(120) +fun test20() { + return (Test7_1 .getDeclaredPackPrefixLen(), Test7_1 .getDeclaredPackPrefix(), + CellInner8_1.getDeclaredPackPrefixLen(), CellInner8_1.getDeclaredPackPrefix()); +} + fun main() { __expect_type(int8.estimatePackSize(), "[int, int, int, int]"); } /** -@testcase | 101 | | [ 32 32 0 0 ] [ 64 64 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 0 0 0 0 ] [ 4 124 0 0 ] [ 6 6 0 0 ] [ 64 64 0 0 ] +@testcase | 101 | | [ 32 32 0 0 ] [ 64 64 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 0 9999 0 4 ] [ 4 124 0 0 ] [ 6 6 0 0 ] [ 64 64 0 0 ] @testcase | 102 | | [ 32 32 0 0 ] [ 1 33 0 0 ] [ 1 33 0 0 ] [ 17 33 0 0 ] @testcase | 103 | | [ 67 199 0 0 ] [ 69 599 0 0 ] @testcase | 104 | | [ 4 534 0 0 ] [ 2 536 0 0 ] [ 6 1070 0 0 ] -@testcase | 105 | | [ 2 2 2 4 ] [ 5 5 0 5 ] [ 4 152 1 4 ] +@testcase | 105 | | [ 2 2 2 4 ] [ 5 9999 0 9 ] [ 4 152 1 4 ] @testcase | 106 | | [ 64 64 0 0 ] [ 5 37 1 1 ] [ 5 65 0 1 ] @testcase | 107 | | [ 16 16 0 0 ] [ 16 16 0 0 ] @testcase | 108 | | [ 10 275 0 0 ] [ 34 158 0 0 ] [ 47 1143 0 1 ] @testcase | 109 | | [ 34 130 0 0 ] [ 33 159 0 0 ] [ 4 8 0 0 ] [ 3 7 0 0 ] +@testcase | 110 | | [ 32 9999 0 4 ] [ 33 9999 0 8 ] +@testcase | 120 | | 16 4128 1 1 */ diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk new file mode 100644 index 0000000000..049109ad78 --- /dev/null +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -0,0 +1,223 @@ +struct (0x01) CounterIncrement { byValue: int8; } +struct (0x03) CounterDecrementBy1 {} +type CounterMsg = CounterIncrement | CounterDecrementBy1; + +struct SomeBytesFields { + f1: bytes1; +} + +@method_id(101) +fun test1() { + try { + return CounterIncrement.fromSlice(stringHexToSlice("880f"), {throwIfOpcodeDoesNotMatch: 101}).byValue as int; + } catch (excno) { + return excno; + } +} + +@method_id(102) +fun test2() { + try { + return CounterIncrement.fromSlice(stringHexToSlice("890f")).byValue as int; + } catch (excno) { + return excno; + } +} + +@method_id(103) +fun test3() { + var cc: Cell = { + tvmCell: beginCell().storeSlice(stringHexToSlice("0109ab")).endCell() + }; + return cc.load({assertEndAfterReading: false}); +} + +@method_id(104) +fun test4() { + try { + var msg = CounterMsg.fromSlice(stringHexToSlice("88"), {throwIfOpcodeDoesNotMatch: 104, assertEndAfterReading: false}); + __expect_type(msg, "CounterMsg"); + return (-1, msg is CounterIncrement); + } catch (excno) { + return (excno, null); + } +} + +@method_id(105) +fun test5() { + return SomeBytesFields { f1: stringHexToSlice("11") as bytes1 }.toCell().hash() & 0xFFFF; +} + +@method_id(106) +fun test6() { + return SomeBytesFields { f1: stringHexToSlice("11") as bytes1 }.toCell({skipBitsNFieldsValidation: true}).hash() & 0xFFFF; +} + +@method_id(107) +fun test7(believe: bool) { + try { + return SomeBytesFields { f1: stringHexToSlice("ffff") as bytes1 }.toCell({skipBitsNFieldsValidation: believe}).hash() & 0xFFFF; + } catch (excno) { + return excno; + } +} + +@method_id(108) +fun test8() { + return CounterIncrement.fromSlice(stringHexToSlice("010f"), { + throwIfOpcodeDoesNotMatch: 0xFFFF + }).byValue; +} + +@method_id(109) +fun test9() { + return CounterMsg.fromSlice(stringHexToSlice("010f"), { + throwIfOpcodeDoesNotMatch: 0xFFFF, + assertEndAfterReading: false, + }) is CounterIncrement; +} + +@method_id(110) +fun test10() { + var c = beginCell().storeUint(123, 64).endCell() as Cell; + try { + var b = c.load(); + return -(b.f1 as slice).remainingBitsCount(); + } catch (excno) { + return excno; + } +} + +@method_id(111) +fun test11() { + var c = beginCell().storeUint(123, 64).endCell() as Cell; + var b = c.load({assertEndAfterReading: false}); + return -(b.f1 as slice).remainingBitsCount(); +} + +fun main(){} + +/** +@testcase | 101 | | 101 +@testcase | 102 | | 63 +@testcase | 103 | | 9 +@testcase | 104 | | 104 (null) +@testcase | 105 | | 36896 +@testcase | 106 | | 36896 +@testcase | 107 | -1 | 2142 +@testcase | 107 | 0 | 9 +@testcase | 108 | | 15 +@testcase | 109 | | -1 +@testcase | 110 | | 9 +@testcase | 111 | | -8 + +@fif_codegen +""" +x{880f} PUSHSLICE +x{01} SDBEGINSQ +101 THROWIFNOT +""" + +@fif_codegen +""" +x{890f} PUSHSLICE +x{01} SDBEGINSQ +63 THROWIFNOT +8 LDI +ENDS +RETALT +""" + +@fif_codegen +""" + test3 PROC:<{ + NEWC + x{0109ab} PUSHSLICE + STSLICER + ENDC + CTOS + x{01} SDBEGINSQ + 63 THROWIFNOT + 8 LDI + DROP + }> +""" + +@fif_codegen +""" +IF:<{ + DROP + 129 PUSHINT +}>ELSE<{ + x{03} SDBEGINSQ + NIP + IFNOTJMP:<{ + 104 THROW + }> + 130 PUSHINT +}> +""" + +@fif_codegen +""" + test5 PROC:<{ + x{11} PUSHSLICE + NEWC + OVER + SBITREFS + 9 THROWIF + 8 EQINT + 9 THROWIFNOT + SWAP + STSLICER +""" + +@fif_codegen +""" + test6 PROC:<{ + x{11} PUSHSLICE + NEWC + SWAP + STSLICER + ENDC + HASHCU +""" + +@fif_codegen +""" + test8 PROC:<{ // + x{010f} PUSHSLICE // '0 + 16 PUSHPOW2DEC // s '2=65535 + SWAP // '2=65535 s + x{01} SDBEGINSQ // '2=65535 s '4 + s1 s2 XCHG // s '2=65535 '4 + THROWANYIFNOT // s + 8 LDI // '9 s + ENDS // '9 + }> +""" + +@fif_codegen +""" + test9 PROC:<{ // + x{010f} PUSHSLICE // s + x{01} SDBEGINSQ // s '6 + IF:<{ // s + DROP // + 129 PUSHINT // 'UTag=129 + }>ELSE<{ // s + x{03} SDBEGINSQ // s '6 + NIP // '6 + IFNOTJMP:<{ // + 16 PUSHPOW2DEC + THROWANY + }> + 130 PUSHINT // 'UTag=130 + }> + 129 PUSHINT // 'UTag '17=129 + SWAP // '17=129 'UTag + EQUAL // '16 + }> +""" + + */ diff --git a/tolk-tester/tests/try-catch-tests.tolk b/tolk-tester/tests/try-catch-tests.tolk index b0a50aacb3..be0362e395 100644 --- a/tolk-tester/tests/try-catch-tests.tolk +++ b/tolk-tester/tests/try-catch-tests.tolk @@ -236,6 +236,16 @@ fun testCodegen3(numberId: int, paramVal: cell) { paramVal.beginParse(); } +@method_id(113) +fun testBigExcno() { + try { + throw 2048; + return 10; + } catch (excno) { + return excno; + } +} + fun main() { } @@ -264,8 +274,9 @@ fun main() { @testcase | 110 | 0 | 5 @testcase | 111 | -1 | 123 @testcase | 111 | 0 | 456 +@testcase | 113 | | 2048 -@code_hash 57361460846265694653029920796509802052573595128418810728101968091567195330515 +@code_hash 26411074751358281917876479601714698321674833208179417883888954015536273820035 @fif_codegen """ @@ -317,4 +328,12 @@ fun main() { DROP // }> """ + +@fif_codegen +""" +<{ + 11 PUSHPOW2 + THROWANY +}>CONT +""" */ diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 1f205f2ca1..39075452de 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -18,6 +18,7 @@ #include "compiler-state.h" #include "type-system.h" #include "generics-helpers.h" +#include "ast.h" namespace tolk { using namespace std::literals::string_literals; @@ -938,7 +939,9 @@ static AsmOp compile_cmp_int(std::vector& res, std::vector& static AsmOp compile_throw(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.empty() && args.size() == 1); VarDescr& x = args[0]; - if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { + if (x.is_int_const() && x.int_const >= 0) { + // in Fift assembler, "N THROW" is valid if N < 2048; for big N (particularly, widely used 0xFFFF) + // we now still generate "N THROW", and later, in optimizer, transform it to "PUSHINT" + "THROWANY" x.unused(); return exec_arg_op(loc, "THROW", x.int_const, 0, 0); } else { @@ -1181,6 +1184,8 @@ void define_builtins() { // see patch_builtins_after_stdlib_loaded() below TypePtr debug = TypeDataUnknown::create(); TypePtr CellT = TypeDataUnknown::create(); + TypePtr PackOptions = TypeDataUnknown::create(); + TypePtr UnpackOptions = TypeDataUnknown::create(); // builtin operators // they are internally stored as functions, because at IR level, there is no difference @@ -1389,28 +1394,34 @@ void define_builtins() { // serialization/deserialization methods to/from cells (or, more low-level, slices/builders) // they work with structs (or, more low-level, with arbitrary types) - define_builtin_method("T.toCell", typeT, {typeT}, CellT, declReceiverT, + define_builtin_method("T.toCell", typeT, {typeT, PackOptions}, CellT, declReceiverT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); - define_builtin_method("T.fromCell", typeT, {TypeDataCell::create()}, typeT, declReceiverT, + define_builtin_method("T.fromCell", typeT, {TypeDataCell::create(), UnpackOptions}, typeT, declReceiverT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); - define_builtin_method("T.fromSlice", typeT, {Slice}, typeT, declReceiverT, + define_builtin_method("T.fromSlice", typeT, {Slice, UnpackOptions}, typeT, declReceiverT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); define_builtin_method("T.estimatePackSize", typeT, {}, TypeDataBrackets::create({TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create()}), declReceiverT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); - define_builtin_method("Cell.load", CellT, {CellT}, typeT, declReceiverT, + define_builtin_method("T.getDeclaredPackPrefix", typeT, {}, Int, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal | FunctionData::flagAllowAnyWidthT); + define_builtin_method("T.getDeclaredPackPrefixLen", typeT, {}, Int, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal | FunctionData::flagAllowAnyWidthT); + define_builtin_method("Cell.load", CellT, {CellT, UnpackOptions}, typeT, declReceiverT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); - define_builtin_method("slice.loadAny", Slice, {Slice}, typeT, declGenericT, + define_builtin_method("slice.loadAny", Slice, {Slice, UnpackOptions}, typeT, declGenericT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); - define_builtin_method("slice.skipAny", Slice, {Slice}, Slice, declGenericT, + define_builtin_method("slice.skipAny", Slice, {Slice, UnpackOptions}, Slice, declGenericT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); - define_builtin_method("builder.storeAny", Builder, {Builder, typeT}, Builder, declGenericT, + define_builtin_method("builder.storeAny", Builder, {Builder, typeT, PackOptions}, Builder, declGenericT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); @@ -1439,11 +1450,38 @@ void patch_builtins_after_stdlib_loaded() { lookup_function("debug.dumpStack")->mutate()->receiver_type = debug; StructPtr struct_ref_CellT = lookup_global_symbol("Cell")->try_as(); + StructPtr struct_ref_PackOptions = lookup_global_symbol("PackOptions")->try_as(); + StructPtr struct_ref_UnpackOptions = lookup_global_symbol("UnpackOptions")->try_as(); TypePtr CellT = TypeDataGenericTypeWithTs::create(struct_ref_CellT, nullptr, {typeT}); + TypePtr PackOptions = TypeDataStruct::create(struct_ref_PackOptions); + TypePtr UnpackOptions = TypeDataStruct::create(struct_ref_UnpackOptions); + + // in stdlib, there is a default parameter `options = {}`; since default parameters are evaluated with AST, + // emulate its presence in built-in functions; it looks ugly, but currently I don't have a better solution + auto v_empty_PackOptions = createV({}, nullptr, createV({}, {})); + v_empty_PackOptions->assign_struct_ref(struct_ref_PackOptions); + v_empty_PackOptions->assign_inferred_type(PackOptions); + auto v_empty_UnpackOptions = createV({}, nullptr, createV({}, {})); + v_empty_UnpackOptions->assign_struct_ref(struct_ref_UnpackOptions); + v_empty_UnpackOptions->assign_inferred_type(UnpackOptions); + lookup_function("T.toCell")->mutate()->declared_return_type = CellT; + lookup_function("T.toCell")->mutate()->parameters[1].declared_type = PackOptions; + lookup_function("T.toCell")->mutate()->parameters[1].default_value = v_empty_PackOptions; + lookup_function("T.fromCell")->mutate()->parameters[1].declared_type = UnpackOptions; + lookup_function("T.fromCell")->mutate()->parameters[1].default_value = v_empty_UnpackOptions; + lookup_function("T.fromSlice")->mutate()->parameters[1].declared_type = UnpackOptions; + lookup_function("T.fromSlice")->mutate()->parameters[1].default_value = v_empty_UnpackOptions; lookup_function("Cell.load")->mutate()->parameters[0].declared_type = CellT; + lookup_function("Cell.load")->mutate()->parameters[1].declared_type = UnpackOptions; + lookup_function("Cell.load")->mutate()->parameters[1].default_value = v_empty_UnpackOptions; lookup_function("Cell.load")->mutate()->receiver_type = CellT; - lookup_function("T.toCell")->mutate()->declared_return_type = CellT; + lookup_function("slice.loadAny")->mutate()->parameters[1].declared_type = UnpackOptions; + lookup_function("slice.loadAny")->mutate()->parameters[1].default_value = v_empty_UnpackOptions; + lookup_function("slice.skipAny")->mutate()->parameters[1].declared_type = UnpackOptions; + lookup_function("slice.skipAny")->mutate()->parameters[1].default_value = v_empty_UnpackOptions; + lookup_function("builder.storeAny")->mutate()->parameters[2].declared_type = PackOptions; + lookup_function("builder.storeAny")->mutate()->parameters[2].default_value = v_empty_PackOptions; } } // namespace tolk diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index 953cc317cd..20c316fdb1 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -17,6 +17,7 @@ #include "constant-evaluator.h" #include "ast.h" #include "tolk.h" +#include "type-system.h" #include "openssl/digest.hpp" #include "crypto/common/util.h" #include "td/utils/crypto.h" @@ -161,8 +162,21 @@ static td::RefInt256 parse_nanotons_as_floating_string(SrcLocation loc, std::str // given `ton("0.05")` evaluate it to 50000000 // given `stringCrc32("some_str")` evaluate it // etc. -// currently, all compile-time functions accept 1 argument, a literal string static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(V v, std::string_view f_name) { + // most functions accept 1 argument, but static compile-time methods like `MyStruct.getDeclaredPackPrefix()` have 0 args + if (v->get_num_args() == 0) { + TypePtr receiver = v->fun_maybe->receiver_type; + f_name = v->fun_maybe->method_name; + + if (f_name == "getDeclaredPackPrefix" || f_name == "getDeclaredPackPrefixLen") { + const TypeDataStruct* t_struct = receiver->try_as(); + if (!t_struct || !t_struct->struct_ref->opcode.exists()) { + throw ParseError(v->loc, "type `" + receiver->as_human_readable() + "` does not have a serialization prefix"); + } + return td::make_refint(f_name.ends_with('x') ? t_struct->struct_ref->opcode.pack_prefix : t_struct->struct_ref->opcode.prefix_len); + } + } + tolk_assert(v->get_num_args() == 1); // checked by type inferring AnyExprV v_arg = v->get_arg(0)->get_expr(); diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index 332069c03f..e6d6b070db 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -135,6 +135,35 @@ bool Optimizer::find_const_op(int* op_idx, int cst) { return false; } +// purpose: transform `65535 THROW` to `PUSHINT` + `THROWANY`; +// such a technique allows pushing a number onto a stack just before THROW, even if a variable is created in advance; +// used for `T.fromSlice(s, {code:0xFFFF})`, where `tmp = 0xFFFF` + serialization match + `else throw tmp` is generated; +// but since it's constant, it transforms to (unused 0xFFFF) + ... + else "65535 THROW", unwrapped here +bool Optimizer::detect_rewrite_big_THROW() { + bool is_throw = op_[0]->is_custom() && op_[0]->op.ends_with(" THROW"); + if (!is_throw) { + return false; + } + + std::string_view s_num_throw = op_[0]->op; + size_t sp = s_num_throw.find(' '); + if (sp != s_num_throw.rfind(' ') || s_num_throw[0] < '1' || s_num_throw[0] > '9') { + return false; + } + + std::string s_number(s_num_throw.substr(0, sp)); + uint64_t excno = std::stoul(s_number); + if (excno < 2048) { // "9 THROW" left as is, but "N THROW" where N>=2^11 is invalid for Fift + return false; + } + + p_ = 1; + q_ = 2; + oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, td::make_refint(excno))); + oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, "THROWANY", 1, 0)); + return true; +} + bool Optimizer::is_push_const(int* i, int* c) const { return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_push_const(i, c); } @@ -553,6 +582,7 @@ bool Optimizer::find_at_least(int pb) { (is_xchg(&i, &j) && rewrite(AsmOp::Xchg(loc, i, j))) || (is_push(&i) && rewrite(AsmOp::Push(loc, i))) || (is_pop(&i) && rewrite(AsmOp::Pop(loc, i))) || (is_pop_pop(&i, &j) && rewrite(AsmOp::Pop(loc, i), AsmOp::Pop(loc, j))) || (is_xchg_xchg(&i, &j, &k, &l) && rewrite(AsmOp::Xchg(loc, i, j), AsmOp::Xchg(loc, k, l))) || + detect_rewrite_big_THROW() || (!(mode_ & 1) && ((is_rot() && rewrite(AsmOp::Custom(loc, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(loc, "-ROT", 3, 3))) || (is_2dup() && rewrite(AsmOp::Custom(loc, "2DUP", 2, 4))) || diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index 14a20d9222..6af6650f49 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -177,13 +177,14 @@ bool check_struct_can_be_packed_or_unpacked(TypePtr any_type, bool is_pack, std: // -std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_obj) { +std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_obj, const std::vector& ir_options) { FunctionPtr f_beginCell = lookup_function("beginCell"); FunctionPtr f_endCell = lookup_function("builder.endCell"); std::vector rvect_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); code.emplace_back(loc, Op::_Call, rvect_builder, std::vector{}, f_beginCell); - PackContext ctx(code, loc, rvect_builder); + tolk_assert(ir_options.size() == 1); // struct PackOptions + PackContext ctx(code, loc, rvect_builder, ir_options); ctx.generate_pack_any(any_type, std::move(ir_obj)); std::vector rvect_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(cell)"); @@ -192,41 +193,51 @@ std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation return rvect_cell; } -std::vector generate_pack_struct_to_builder(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_builder, std::vector&& ir_obj) { - PackContext ctx(code, loc, ir_builder); // mutate this builder +std::vector generate_pack_struct_to_builder(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_builder, std::vector&& ir_obj, const std::vector& ir_options) { + PackContext ctx(code, loc, ir_builder, ir_options); // mutate this builder ctx.generate_pack_any(any_type, std::move(ir_obj)); return ir_builder; // return mutated builder } -std::vector generate_unpack_struct_from_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, bool mutate_slice) { +std::vector generate_unpack_struct_from_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, bool mutate_slice, const std::vector& ir_options) { if (!mutate_slice) { std::vector slice_copy = code.create_var(TypeDataSlice::create(), loc, "s"); code.emplace_back(loc, Op::_Let, slice_copy, std::move(ir_slice)); ir_slice = std::move(slice_copy); } - UnpackContext ctx(code, loc, std::move(ir_slice)); + tolk_assert(ir_options.size() == 2); // struct UnpackOptions + UnpackContext ctx(code, loc, std::move(ir_slice), ir_options); std::vector rvect_struct = ctx.generate_unpack_any(any_type); tolk_assert(any_type->get_width_on_stack() == static_cast(rvect_struct.size())); + // slice.loadAny() ignores options.assertEndAfterReading, because it's intended to read data in the middle + if (!mutate_slice && !estimate_serialization_size(any_type).is_unpredictable_infinity()) { + ctx.assertEndIfOption(); + } return rvect_struct; } -std::vector generate_unpack_struct_from_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_cell) { +std::vector generate_unpack_struct_from_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_cell, const std::vector& ir_options) { FunctionPtr f_beginParse = lookup_function("cell.beginParse"); std::vector ir_slice = code.create_var(TypeDataSlice::create(), loc, "s"); code.emplace_back(loc, Op::_Call, ir_slice, std::move(ir_cell), f_beginParse); - UnpackContext ctx(code, loc, std::move(ir_slice)); + tolk_assert(ir_options.size() == 2); // struct UnpackOptions + UnpackContext ctx(code, loc, std::move(ir_slice), ir_options); std::vector rvect_struct = ctx.generate_unpack_any(any_type); tolk_assert(any_type->get_width_on_stack() == static_cast(rvect_struct.size())); + // if a struct has RemainingBitsAndRefs, don't test it for assertEnd + if (!estimate_serialization_size(any_type).is_unpredictable_infinity()) { + ctx.assertEndIfOption(); + } return rvect_struct; } -std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice) { - UnpackContext ctx(code, loc, ir_slice); // mutate this slice +std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, const std::vector& ir_options) { + UnpackContext ctx(code, loc, ir_slice, ir_options); // mutate this slice ctx.generate_skip_any(any_type); return ir_slice; // return mutated slice diff --git a/tolk/pack-unpack-api.h b/tolk/pack-unpack-api.h index b0a4b29efe..7737125d40 100644 --- a/tolk/pack-unpack-api.h +++ b/tolk/pack-unpack-api.h @@ -23,11 +23,11 @@ namespace tolk { bool check_struct_can_be_packed_or_unpacked(TypePtr any_type, bool is_pack, std::string& because_msg); -std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_obj); -std::vector generate_pack_struct_to_builder(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_builder, std::vector&& ir_obj); -std::vector generate_unpack_struct_from_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, bool mutate_slice); -std::vector generate_unpack_struct_from_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_cell); -std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice); +std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_obj, const std::vector& ir_options); +std::vector generate_pack_struct_to_builder(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_builder, std::vector&& ir_obj, const std::vector& ir_options); +std::vector generate_unpack_struct_from_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, bool mutate_slice, const std::vector& ir_options); +std::vector generate_unpack_struct_from_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_cell, const std::vector& ir_options); +std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, const std::vector& ir_options); PackSize estimate_serialization_size(TypePtr any_type); std::vector generate_estimate_size_call(CodeBlob& code, SrcLocation loc, TypePtr any_type); diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index 8be3d99952..c0490c5c76 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -40,8 +40,6 @@ namespace tolk { std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc); std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc); -constexpr int ERR_CODEGEN_ASSERT = 9; - bool is_type_cellT(TypePtr any_type) { if (const TypeDataStruct* t_struct = any_type->try_as()) { StructPtr struct_ref = t_struct->struct_ref; @@ -59,13 +57,14 @@ bool is_type_cellT(TypePtr any_type) { // -PackContext::PackContext(CodeBlob& code, SrcLocation loc, std::vector ir_builder) +PackContext::PackContext(CodeBlob& code, SrcLocation loc, std::vector ir_builder, const std::vector& ir_options) : code(code) , loc(loc) , f_storeInt(lookup_function("builder.storeInt")) , f_storeUint(lookup_function("builder.storeUint")) , ir_builder(std::move(ir_builder)) - , ir_builder0(this->ir_builder[0]) { + , ir_builder0(this->ir_builder[0]) + , option_skipBitsNFieldsValidation(ir_options[0]) { } void PackContext::storeInt(var_idx_t ir_idx, int len) const { @@ -119,14 +118,16 @@ void PackContext::storeOpcode(PackOpcode opcode) const { } -UnpackContext::UnpackContext(CodeBlob& code, SrcLocation loc, std::vector ir_slice) +UnpackContext::UnpackContext(CodeBlob& code, SrcLocation loc, std::vector ir_slice, const std::vector& ir_options) : code(code) , loc(loc) , f_loadInt(lookup_function("slice.loadInt")) , f_loadUint(lookup_function("slice.loadUint")) , f_skipBits(lookup_function("slice.skipBits")) , ir_slice(std::move(ir_slice)) - , ir_slice0(this->ir_slice[0]) { + , ir_slice0(this->ir_slice[0]) + , option_assertEndAfterReading(ir_options[0]) + , option_throwIfOpcodeDoesNotMatch(ir_options[1]) { } std::vector UnpackContext::loadInt(int len, const char* debug_desc) const { @@ -147,8 +148,7 @@ void UnpackContext::loadAndCheckOpcode(PackOpcode opcode) const { std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); std::vector args = { ir_slice0, code.create_int(loc, opcode.pack_prefix, "(pack-prefix)"), code.create_int(loc, opcode.prefix_len, "(prefix-len)") }; code.emplace_back(loc, Op::_Call, std::vector{ir_slice0, ir_prefix_eq[0]}, std::move(args), lookup_function("slice.tryStripPrefix")); - - std::vector args_assert = { code.create_int(loc, ERR_CODEGEN_ASSERT, "(excno)"), ir_prefix_eq[0], code.create_int(loc, 0, "") }; + std::vector args_assert = { option_throwIfOpcodeDoesNotMatch, ir_prefix_eq[0], code.create_int(loc, 0, "") }; Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); op_assert.set_impure_flag(); } @@ -163,6 +163,19 @@ void UnpackContext::skipBits_var(var_idx_t ir_len) const { code.emplace_back(loc, Op::_Call, ir_slice, std::move(args), f_skipBits); } +void UnpackContext::assertEndIfOption() const { + Op& if_assertEnd = code.emplace_back(loc, Op::_If, std::vector{option_assertEndAfterReading}); + { + code.push_set_cur(if_assertEnd.block0); + Op& op_ends = code.emplace_back(loc, Op::_Call, std::vector{}, ir_slice, lookup_function("slice.assertEnd")); + op_ends.set_impure_flag(); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_assertEnd.block1); + code.close_pop_cur(loc); + } +} // -------------------------------------------- // serializers with pack/unpack/skip/estimate @@ -226,24 +239,27 @@ struct S_BytesN final : ISerializer { void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { tolk_assert(rvect.size() == 1); - // bitsN/bytesN aren't checked upon writing (to have requested number of bits and 0 refs), probably to expose an option - if (false) { + Op& if_disabled_by_user = code.emplace_back(loc, Op::_If, std::vector{ctx->option_skipBitsNFieldsValidation}); + { + code.push_set_cur(if_disabled_by_user.block0); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_disabled_by_user.block1); FunctionPtr f_assert = lookup_function("__throw_if_unless"); - FunctionPtr f_equals = lookup_function("_==_"); - FunctionPtr f_bitAnd = lookup_function("_&_"); - FunctionPtr f_getCounts = lookup_function("slice.remainingBitsAndRefsCount"); + constexpr int EXCNO = 9; std::vector ir_counts = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create()}), loc, "(slice-size)"); - code.emplace_back(loc, Op::_Call, ir_counts, rvect, f_getCounts); - std::vector ir_is_n = code.create_tmp_var(TypeDataInt::create(), loc, "(n-bits)"); - code.emplace_back(loc, Op::_Call, ir_is_n, std::vector{ir_counts[0], code.create_int(loc, n_bits, "(expected-bits)")}, f_equals); - std::vector ir_is_0 = code.create_tmp_var(TypeDataInt::create(), loc, "(n-refs)"); - code.emplace_back(loc, Op::_Call, ir_is_0, std::vector{ir_counts[1], code.create_int(loc, 0, "(expected-refs)")}, f_equals); - std::vector ir_both_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(both-eq)"); - code.emplace_back(loc, Op::_Call, ir_both_eq, std::vector{ir_is_n[0], ir_is_0[0]}, f_bitAnd); - std::vector args_assert = { code.create_int(loc, ERR_CODEGEN_ASSERT, "(excno)"), ir_both_eq[0], code.create_int(loc, 0, "") }; - Op& op1 = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), f_assert); - op1.set_impure_flag(); + code.emplace_back(loc, Op::_Call, ir_counts, rvect, lookup_function("slice.remainingBitsAndRefsCount")); + std::vector args_assert0 = { code.create_int(loc, EXCNO, "(excno)"), ir_counts[1], code.create_int(loc, 1, "") }; + Op& op_assert0 = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert0), f_assert); + op_assert0.set_impure_flag(); + std::vector ir_eq_n = code.create_tmp_var(TypeDataInt::create(), loc, "(eq-n)"); + code.emplace_back(loc, Op::_Call, ir_eq_n, std::vector{ir_counts[0], code.create_int(loc, n_bits, "(n-bits)")}, lookup_function("_==_")); + std::vector args_assertN = { code.create_int(loc, EXCNO, "(excno)"), ir_eq_n[0], code.create_int(loc, 0, "") }; + Op& op_assertN = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assertN), f_assert); + op_assertN.set_impure_flag(); + code.close_pop_cur(loc); } ctx->storeSlice(rvect[0]); @@ -414,7 +430,7 @@ struct S_RemainingBitsAndRefs final : ISerializer { } PackSize estimate(const EstimateContext* ctx) override { - return PackSize(0, 0, 0, 0); + return PackSize::unpredictable_infinity(); } }; @@ -434,7 +450,7 @@ struct S_Builder final : ISerializer { } PackSize estimate(const EstimateContext* ctx) override { - return PackSize(0, 0, 0, 0); + return PackSize::unpredictable_infinity(); } }; @@ -661,27 +677,21 @@ struct S_MultipleConstructors final : ISerializer { TypePtr variant = t_union->variants[i]; std::vector args = { ctx->ir_slice0, code.create_int(loc, opcodes[i].pack_prefix, "(pack-prefix)"), code.create_int(loc, opcodes[i].prefix_len, "(prefix-len)") }; code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), f_tryStripPrefix); - if (i != t_union->size() - 1) { - Op& if_op = code.emplace_back(loc, Op::_If, ir_prefix_eq); - code.push_set_cur(if_op.block0); - std::vector ith_rvect = ctx->generate_unpack_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); - ith_rvect = transition_to_target_type(std::move(ith_rvect), code, variant, t_union, loc); - code.emplace_back(loc, Op::_Let, ir_result, std::move(ith_rvect)); - code.close_pop_cur(loc); - code.push_set_cur(if_op.block1); // open ELSE - } else { // we're inside the last ELSE - if (!are_prefixes_exhaustive()) { - std::vector args_assert = { code.create_int(loc, ERR_CODEGEN_ASSERT, "(excno)"), ir_prefix_eq[0], code.create_int(loc, 0, "") }; - Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); - op_assert.set_impure_flag(); - } - std::vector last_rvect = ctx->generate_unpack_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); - last_rvect = transition_to_target_type(std::move(last_rvect), code, variant, t_union, loc); - code.emplace_back(loc, Op::_Let, ir_result, std::move(last_rvect)); - for (int j = 0; j < t_union->size() - 1; ++j) { - code.close_pop_cur(loc); // close all outer IFs - } - } + Op& if_prefix_eq = code.emplace_back(loc, Op::_If, ir_prefix_eq); + code.push_set_cur(if_prefix_eq.block0); + std::vector ith_rvect = ctx->generate_unpack_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); + ith_rvect = transition_to_target_type(std::move(ith_rvect), code, variant, t_union, loc); + code.emplace_back(loc, Op::_Let, ir_result, std::move(ith_rvect)); + code.close_pop_cur(loc); + code.push_set_cur(if_prefix_eq.block1); // open ELSE + } + + // we're inside last ELSE + std::vector args_throw = { ctx->option_throwIfOpcodeDoesNotMatch }; + Op& op_throw = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_throw), lookup_function("__throw")); + op_throw.set_impure_flag(); + for (int j = 0; j < t_union->size(); ++j) { + code.close_pop_cur(loc); // close all outer IFs } return ir_result; } @@ -694,23 +704,19 @@ struct S_MultipleConstructors final : ISerializer { TypePtr variant = t_union->variants[i]; std::vector args = { ctx->ir_slice0, code.create_int(loc, opcodes[i].pack_prefix, "(pack-prefix)"), code.create_int(loc, opcodes[i].prefix_len, "(prefix-len)") }; code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), f_tryStripPrefix); - if (i != t_union->size() - 1) { - Op& if_op = code.emplace_back(loc, Op::_If, ir_prefix_eq); - code.push_set_cur(if_op.block0); - ctx->generate_skip_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); - code.close_pop_cur(loc); - code.push_set_cur(if_op.block1); // open ELSE - } else { // we're inside the last ELSE - if (!are_prefixes_exhaustive()) { - std::vector args_assert = { code.create_int(loc, ERR_CODEGEN_ASSERT, "(excno)"), ir_prefix_eq[0], code.create_int(loc, 0, "") }; - Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); - op_assert.set_impure_flag(); - } - ctx->generate_skip_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); - for (int j = 0; j < t_union->size() - 1; ++j) { - code.close_pop_cur(loc); // close all outer IFs - } - } + Op& if_prefix_eq = code.emplace_back(loc, Op::_If, ir_prefix_eq); + code.push_set_cur(if_prefix_eq.block0); + ctx->generate_skip_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); + code.close_pop_cur(loc); + code.push_set_cur(if_prefix_eq.block1); // open ELSE + } + + // we're inside last ELSE + std::vector args_throw = { ctx->option_throwIfOpcodeDoesNotMatch }; + Op& op_throw = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_throw), lookup_function("__throw")); + op_throw.set_impure_flag(); + for (int j = 0; j < t_union->size(); ++j) { + code.close_pop_cur(loc); // close all outer IFs } } @@ -725,14 +731,6 @@ struct S_MultipleConstructors final : ISerializer { return EstimateContext::sum(variants_size, prefix_size); } - - bool are_prefixes_exhaustive() const { - bool all_prefix_len_eq = true; - for (PackOpcode opcode : opcodes) { - all_prefix_len_eq &= opcode.prefix_len == opcodes[0].prefix_len; - } - return all_prefix_len_eq && t_union->size() == (1 << opcodes[0].prefix_len); - } }; struct S_Tensor final : ISerializer { diff --git a/tolk/pack-unpack-serializers.h b/tolk/pack-unpack-serializers.h index 47629a7dbd..c2c99ea2ae 100644 --- a/tolk/pack-unpack-serializers.h +++ b/tolk/pack-unpack-serializers.h @@ -29,6 +29,10 @@ struct PackSize { int min_refs; int max_refs; + bool is_unpredictable_infinity() const { + return max_bits >= 9999; + } + explicit PackSize(int exact_bits) : min_bits(exact_bits), max_bits(exact_bits), min_refs(0), max_refs(0) { } @@ -38,6 +42,10 @@ struct PackSize { PackSize(int min_bits, int max_bits, int min_refs, int max_refs) : min_bits(min_bits), max_bits(max_bits), min_refs(min_refs), max_refs(max_refs) { } + + static PackSize unpredictable_infinity() { + return PackSize(0, 9999, 0, 4); + } }; @@ -56,8 +64,9 @@ class PackContext { public: const std::vector ir_builder; const var_idx_t ir_builder0; + const var_idx_t option_skipBitsNFieldsValidation; - PackContext(CodeBlob& code, SrcLocation loc, std::vector ir_builder); + PackContext(CodeBlob& code, SrcLocation loc, std::vector ir_builder, const std::vector& ir_options); PrefixWriteMode get_prefix_mode() const { return prefix_mode; } @@ -92,8 +101,10 @@ class UnpackContext { public: const std::vector ir_slice; const var_idx_t ir_slice0; + const var_idx_t option_assertEndAfterReading; + const var_idx_t option_throwIfOpcodeDoesNotMatch; - UnpackContext(CodeBlob& code, SrcLocation loc, std::vector ir_slice); + UnpackContext(CodeBlob& code, SrcLocation loc, std::vector ir_slice, const std::vector& ir_options); PrefixReadMode get_prefix_mode() const { return prefix_mode; } @@ -102,6 +113,7 @@ class UnpackContext { void loadAndCheckOpcode(PackOpcode opcode) const; void skipBits(int len) const; void skipBits_var(var_idx_t ir_len) const; + void assertEndIfOption() const; std::vector generate_unpack_any(TypePtr any_type, PrefixReadMode prefix_mode = PrefixReadMode::LoadAndCheck) const; void generate_skip_any(TypePtr any_type, PrefixReadMode prefix_mode = PrefixReadMode::LoadAndCheck) const; @@ -124,7 +136,7 @@ class EstimateContext { return PackSize(std::min(a.min_bits, b.min_bits), std::max(a.max_bits, b.max_bits), std::min(a.min_refs, b.min_refs), std::max(a.max_refs, b.max_refs)); } static PackSize sum(PackSize a, PackSize b) { - return PackSize(a.min_bits + b.min_bits, a.max_bits + b.max_bits, a.min_refs + b.min_refs, a.max_refs + b.max_refs); + return PackSize(a.min_bits + b.min_bits, std::min(9999, a.max_bits + b.max_bits), a.min_refs + b.min_refs, a.max_refs + b.max_refs); } PackSize estimate_any(TypePtr any_type, PrefixEstimateMode prefix_mode = PrefixEstimateMode::IncludePrefixOfStruct) const; diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 773d67eccb..91891f8c16 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -540,59 +540,52 @@ static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcL return rvect; } -static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob& code, SrcLocation loc, std::vector&& rvect, FunctionPtr called_f) { +static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob& code, SrcLocation loc, const std::vector>& vars_per_arg, FunctionPtr called_f) { if (called_f->is_instantiation_of_generic_function()) { std::string_view f_name = called_f->base_fun_ref->name; TypePtr typeT = called_f->substitutedTs->typeT_at(0); - int n_rvect = static_cast(rvect.size()); if (f_name == "T.toCell") { // in: object T, out: Cell (just a cell, wrapped) - tolk_assert(n_rvect == typeT->get_width_on_stack()); - std::vector ir_obj = std::move(rvect); - return generate_pack_struct_to_cell(code, loc, typeT, std::move(ir_obj)); + std::vector ir_obj = vars_per_arg[0]; + return generate_pack_struct_to_cell(code, loc, typeT, std::move(ir_obj), vars_per_arg[1]); } if (f_name == "T.fromCell") { // in: cell, out: object T - tolk_assert(n_rvect == 1); - std::vector ir_cell = std::move(rvect); - return generate_unpack_struct_from_cell(code, loc, typeT, std::move(ir_cell)); + std::vector ir_cell = vars_per_arg[0]; + return generate_unpack_struct_from_cell(code, loc, typeT, std::move(ir_cell), vars_per_arg[1]); } if (f_name == "T.fromSlice") { // in: slice, out: object T, input slice NOT mutated - tolk_assert(n_rvect == 1); - std::vector ir_slice = std::move(rvect); - return generate_unpack_struct_from_slice(code, loc, typeT, std::move(ir_slice), false); + std::vector ir_slice = vars_per_arg[0]; + return generate_unpack_struct_from_slice(code, loc, typeT, std::move(ir_slice), false, vars_per_arg[1]); } if (f_name == "Cell.load") { // in: cell, out: object T - tolk_assert(n_rvect == 1); - std::vector ir_cell = std::move(rvect); - return generate_unpack_struct_from_cell(code, loc, typeT, std::move(ir_cell)); + std::vector ir_cell = vars_per_arg[0]; + return generate_unpack_struct_from_cell(code, loc, typeT, std::move(ir_cell), vars_per_arg[1]); } if (f_name == "slice.loadAny") { // in: slice, out: object T, input slice is mutated, so prepend self before an object - var_idx_t ir_self = rvect[0]; - std::vector ir_obj = generate_unpack_struct_from_slice(code, loc, typeT, std::move(rvect), true); + var_idx_t ir_self = vars_per_arg[0][0]; + std::vector ir_slice = vars_per_arg[0]; + std::vector ir_obj = generate_unpack_struct_from_slice(code, loc, typeT, std::move(ir_slice), true, vars_per_arg[1]); std::vector ir_result = {ir_self}; ir_result.insert(ir_result.end(), ir_obj.begin(), ir_obj.end()); return ir_result; } if (f_name == "slice.skipAny") { // in: slice, out: the same slice, with a shifted pointer - tolk_assert(n_rvect == 1); - std::vector ir_slice = std::move(rvect); - return generate_skip_struct_in_slice(code, loc, typeT, std::move(ir_slice)); + std::vector ir_slice = vars_per_arg[0]; + return generate_skip_struct_in_slice(code, loc, typeT, std::move(ir_slice), vars_per_arg[1]); } if (f_name == "builder.storeAny") { // in: builder and object T, out: mutated builder - tolk_assert(n_rvect == 1 + typeT->get_width_on_stack()); - std::vector ir_builder = {rvect[0]}; - std::vector ir_obj(rvect.begin() + 1, rvect.end()); - return generate_pack_struct_to_builder(code, loc, typeT, std::move(ir_builder), std::vector(ir_obj)); + std::vector ir_builder = vars_per_arg[0]; + std::vector ir_obj = vars_per_arg[1]; + return generate_pack_struct_to_builder(code, loc, typeT, std::move(ir_builder), std::move(ir_obj), vars_per_arg[2]); } if (f_name == "T.estimatePackSize") { - tolk_assert(rvect.empty()); return generate_estimate_size_call(code, loc, typeT); } } @@ -1454,7 +1447,7 @@ static std::vector process_function_call(V v, Code args_vars.insert(args_vars.end(), list.cbegin(), list.cend()); } std::vector rvect_apply = fun_ref->is_compile_time_special_gen() - ? gen_compile_time_code_instead_of_fun_call(code, v->loc, std::move(args_vars), fun_ref) + ? gen_compile_time_code_instead_of_fun_call(code, v->loc, vars_per_arg, fun_ref) : gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)", arg_order_already_equals_asm); if (fun_ref->has_mutate_params()) { diff --git a/tolk/pipe-check-serialized-fields.cpp b/tolk/pipe-check-serialized-fields.cpp index c49b3dc349..3ed8fc33a2 100644 --- a/tolk/pipe-check-serialized-fields.cpp +++ b/tolk/pipe-check-serialized-fields.cpp @@ -59,7 +59,7 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody static void check_struct_fits_cell_or_has_policy(const TypeDataStruct* t_struct) { StructPtr struct_ref = t_struct->struct_ref; PackSize size = estimate_serialization_size(t_struct); - if (size.max_bits > 1023) { + if (size.max_bits > 1023 && !size.is_unpredictable_infinity()) { if (struct_ref->overflow1023_policy == StructData::Overflow1023Policy::not_specified) { fire_error_theoretical_overflow_1023(struct_ref, size); } @@ -84,7 +84,7 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody if (f_name == "Cell.load" || f_name == "T.fromSlice" || f_name == "T.fromCell" || f_name == "T.toCell" || f_name == "T.loadAny" || f_name == "slice.skipAny" || f_name == "slice.storeAny" || f_name == "T.estimatePackSize") { serialized_type = fun_ref->substitutedTs->typeT_at(0); - is_pack = f_name == "T.toCell" || f_name == "slice.storeAny"; + is_pack = f_name == "T.toCell" || f_name == "slice.storeAny" || f_name == "T.estimatePackSize"; } else { return; // not a serialization function } diff --git a/tolk/tolk.h b/tolk/tolk.h index 6da5c1ff83..6e5af87ca7 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -924,6 +924,9 @@ struct Optimizer { bool is_nip_seq(int* i, int* j); bool is_pop_blkdrop(int* i, int* k); bool is_2pop_blkdrop(int* i, int* j, int* k); + + bool detect_rewrite_big_THROW(); + AsmOpConsList extract_code(); }; From e4d0ab7157486a1fd9720cfceb0556ad0e67a6e2 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Wed, 28 May 2025 18:22:51 +0300 Subject: [PATCH 272/524] [Tolk] Bump version to v0.13 It's considered as "release candidate" (although there's still a lot of work ahead) --- crypto/smartcont/tolk-stdlib/common.tolk | 2 +- crypto/smartcont/tolk-stdlib/gas-payments.tolk | 2 +- crypto/smartcont/tolk-stdlib/lisp-lists.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk | 2 +- tolk/tolk-version.h | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 366345385e..c9e116860d 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -1,7 +1,7 @@ // Standard library for Tolk (LGPL licence). // It contains common functions that are available out of the box, the user doesn't have to import anything. // More specific functions are required to be imported explicitly, like "@stdlib/tvm-dicts". -tolk 0.12 +tolk 0.13 /// In Tolk v1.x there would be a type `map`. /// Currently, working with dictionaries is still low-level, with raw cells. diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 89366cf4ff..8fde5f3c32 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.12 +tolk 0.13 /** Gas and payment related primitives. diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk index ada7a72bc7..14d8f835b8 100644 --- a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.12 +tolk 0.13 /** Lisp-style lists are nested 2-elements tuples: `[1, [2, [3, null]]]` represents list `[1, 2, 3]`. diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index 90b392f615..9d678e83d8 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.12 +tolk 0.13 /** Dictionaries are represented as `cell` data type (cells can store anything, dicts in particular). diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index 47b4a300ee..7281038d64 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.12 +tolk 0.13 /// Usually `c3` has a continuation initialized by the whole code of the contract. It is used for function calls. /// The primitive returns the current value of `c3`. diff --git a/tolk/tolk-version.h b/tolk/tolk-version.h index dd9134f453..5da1582d25 100644 --- a/tolk/tolk-version.h +++ b/tolk/tolk-version.h @@ -18,6 +18,6 @@ namespace tolk { -constexpr const char* TOLK_VERSION = "0.12.0"; +constexpr const char* TOLK_VERSION = "0.13.0"; } // namespace tolk From a13be2ed61d8d06dff0f3611a258c0a164c7a17e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 30 May 2025 10:47:33 +0300 Subject: [PATCH 273/524] Session stats: "got_block_by", "shard_configuration" --- tl/generate/scheme/ton_api.tl | 4 ++-- tl/generate/scheme/ton_api.tlo | Bin 113716 -> 113828 bytes validator-session/validator-session-types.h | 6 ++++-- validator-session/validator-session.cpp | 18 ++++++++++++++---- validator-session/validator-session.hpp | 4 ++-- validator/impl/collator.cpp | 6 ++++++ validator/interfaces/validator-manager.h | 7 ++++++- 7 files changed, 34 insertions(+), 11 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 5d42796ec0..e441551ee3 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -913,7 +913,7 @@ validatorStats.stats.producer flags:# validator_id:int256 block_status:int candidate_id:flags.0?int256 block_id:flags.0?tonNode.blockIdExt collated_data_hash:flags.0?int256 is_accepted:flags.0?Bool is_ours:flags.0?Bool - got_block_at:flags.0?double got_submit_at:flags.0?double gen_utime:flags.0?int comment:flags.0?string + got_block_at:flags.0?double got_block_by:flags.0?int got_submit_at:flags.0?double gen_utime:flags.0?int comment:flags.0?string collation_time:flags.1?double collated_at:flags.1?double collation_cached:flags.1?Bool self_collated:flags.1?Bool collator_node_id:flags.2?int256 validation_time:flags.3?double validated_at:flags.3?double validation_cached:flags.3?Bool approved_weight:flags.0?long approved_33pct_at:flags.0?double approved_66pct_at:flags.0?double approvers:flags.0?string @@ -937,7 +937,7 @@ validatorStats.blockLimitsStatus validatorStats.extMsgsStats ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validatorStats.ExtMsgsStats; validatorStats.blockStats - ext_msgs:validatorStats.extMsgsStats transactions:int = validatorStats.BlockStats; + ext_msgs:validatorStats.extMsgsStats transactions:int shard_configuration:(vector tonNode.blockIdExt) = validatorStats.BlockStats; validatorStats.collatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 cc_seqno:int collated_at:double diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 7b0b3750ac301801cd4b6efe14317c597d86146d..5c7285d51655490215647cf2954abcf83c0c7ef0 100644 GIT binary patch delta 203 zcmdn;gKf!AwhbTbSSBdR9oziHPKAl3w@59&S;ld@j3Z-?2v~f3Pbi}clO{-sWLaWP zW=dj7eo=5qVo9-HQciwyHkir40Fu9XOvs0M`p63kO2zY6O$SH8Tok9^Go8vHpC}YGE9D# as3ZnuGM2ko!V+WYEeL*JUlpp{bH%fQ_ delta 159 zcmZ4TlWofnwhbTbSZ*)5(!KeMoeC2RFPqBVW*NurGLDQnB4F|DJ)w*)Oqw7il4XfG znJI}S`9;Aci6zB)Njdq+*( flags, validator_id.bits256_value(), block_status, candidate_id, create_tl_block_id(block_id), - collated_data_hash, is_accepted, is_ours, got_block_at, got_submit_at, gen_utime, comment, collation_time, - collated_at, collation_cached, self_collated, collator_node_id, validation_time, validated_at, + collated_data_hash, is_accepted, is_ours, got_block_at, got_block_by, got_submit_at, gen_utime, comment, + collation_time, collated_at, collation_cached, self_collated, collator_node_id, validation_time, validated_at, validation_cached, approved_weight, approved_33pct_at, approved_66pct_at, std::move(approvers_str), signed_weight, signed_33pct_at, signed_66pct_at, std::move(signers_str), serialize_time, deserialize_time, serialized_size); diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 79233d6990..9ce7e54045 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -222,7 +222,7 @@ bool ValidatorSessionImpl::ensure_candidate_unique(td::uint32 src_idx, td::uint3 void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice data, td::optional expected_id, - bool is_overlay_broadcast) { + bool is_overlay_broadcast, bool is_startup) { // Note: src is not necessarily equal to the sender of this message: // If requested using get_broadcast_p2p, src is the creator of the block, sender possibly is some other node. auto src_idx = description().get_source_idx(src); @@ -269,6 +269,13 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice } if (stat->got_block_at <= 0.0) { stat->got_block_at = td::Clocks::system(); + if (is_overlay_broadcast) { + stat->got_block_by = ValidatorSessionStats::recv_broadcast; + } else if (is_startup) { + stat->got_block_by = ValidatorSessionStats::recv_startup; + } else { + stat->got_block_by = ValidatorSessionStats::recv_query; + } } stat->deserialize_time = deserialize_time; stat->serialized_size = data.size(); @@ -470,6 +477,7 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, GeneratedCandidate stat->collation_time = collation_time; stat->collated_at = td::Clocks::system(); stat->got_block_at = td::Clocks::system(); + stat->got_block_by = ValidatorSessionStats::recv_collated; stat->collation_cached = c.is_cached; stat->self_collated = c.self_collated; stat->collator_node_id = c.collator_node_id; @@ -610,6 +618,7 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { } if (stat->got_block_at <= 0.0) { stat->got_block_at = td::Clocks::system(); + stat->got_block_by = ValidatorSessionStats::recv_cached; } stat->block_id.root_hash = B->root_hash_; stat->block_id.file_hash = td::sha256_bits256(B->data_); @@ -656,8 +665,9 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { VLOG(VALIDATOR_SESSION_WARNING) << print_id << ": failed to get candidate " << hash << " from " << id << ": " << R.move_as_error(); } else { + LOG(ERROR) << "QQQQQ Got block " << R.ok().size(); td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src_id, R.move_as_ok(), - candidate_id, false); + candidate_id, false, false); } }); @@ -971,8 +981,8 @@ void ValidatorSessionImpl::on_catchain_started() { auto broadcast = create_tl_object( src.tl(), round, root_hash, std::move(B.data), std::move(B.collated_data)); td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src, - serialize_candidate(broadcast, compress).move_as_ok(), td::optional(), - false); + serialize_candidate(broadcast, compress).move_as_ok(), + td::optional(), false, true); } }); callback_->get_approved_candidate(description().get_source_public_key(x->get_src_idx()), x->get_root_hash(), diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 2d996101ad..04fdc875c5 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -116,7 +116,7 @@ class ValidatorSessionImpl : public ValidatorSession { } void process_broadcast(const PublicKeyHash &src, td::BufferSlice data) override { td::actor::send_closure(id_, &ValidatorSessionImpl::process_broadcast, src, std::move(data), - td::optional(), true); + td::optional(), true, false); } void process_message(const PublicKeyHash &src, td::BufferSlice data) override { td::actor::send_closure(id_, &ValidatorSessionImpl::process_message, src, std::move(data)); @@ -204,7 +204,7 @@ class ValidatorSessionImpl : public ValidatorSession { void preprocess_block(catchain::CatChainBlock *block); bool ensure_candidate_unique(td::uint32 src_idx, td::uint32 round, ValidatorSessionCandidateId block_id); void process_broadcast(PublicKeyHash src, td::BufferSlice data, td::optional expected_id, - bool is_overlay_broadcast); + bool is_overlay_broadcast, bool is_startup); void process_message(PublicKeyHash src, td::BufferSlice data); void process_query(PublicKeyHash src, td::BufferSlice data, td::Promise promise); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 93fbaa1984..33719c2f9a 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -6374,6 +6374,12 @@ void Collator::finalize_stats() { stats_.work_time = work_time; stats_.cpu_work_time = cpu_work_time; stats_.time_stats = (PSTRING() << perf_log_); + if (is_masterchain() && shard_conf_) { + shard_conf_->process_shard_hashes([&](const block::McShardHash& shard) { + stats_.shard_configuration.push_back(shard.top_block_id()); + return 0; + }); + } } /** diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index d1394c942c..534e5ea92b 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -70,6 +70,7 @@ struct CollationStats { int cat_bytes = 0, cat_gas = 0, cat_lt_delta = 0, cat_collated_data_bytes = 0; std::string limits_log; td::uint32 transactions = 0; + std::vector shard_configuration; td::uint32 ext_msgs_total = 0; td::uint32 ext_msgs_filtered = 0; td::uint32 ext_msgs_accepted = 0; @@ -78,10 +79,14 @@ struct CollationStats { std::string time_stats; tl_object_ptr tl() const { + std::vector> shards; + for (const BlockIdExt& block_id : shard_configuration) { + shards.push_back(create_tl_block_id(block_id)); + } auto block_stats = create_tl_object( create_tl_object(ext_msgs_total, ext_msgs_filtered, ext_msgs_accepted, ext_msgs_rejected), - transactions); + transactions, std::move(shards)); return create_tl_object( create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes, actual_collated_data_bytes, attempt, self.bits256_value(), is_validator, total_time, work_time, cpu_work_time, From c89081d4bdf279af4d6cd4ab11f7d8f6363cc786 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sun, 1 Jun 2025 15:52:52 +0300 Subject: [PATCH 274/524] Fix start_lt of tick transactions --- validator/impl/collator.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 805240b3eb..cc9ee709c1 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -3007,12 +3007,12 @@ bool Collator::update_last_proc_int_msg(const std::pairtick * 2 + found->tock : config_->get_smc_tick_tock(smc_addr.cbits())); From 220890f55c07bb88db501ceb9ed734e0e8f8b038 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 2 Jun 2025 13:13:46 +0300 Subject: [PATCH 275/524] Add msg import info to collatedBlock session stats --- crypto/block/block.cpp | 9 ++++ crypto/block/block.h | 4 ++ crypto/smartcont/CreateState.fif | 8 +++ tl/generate/scheme/ton_api.tl | 11 +++- tl/generate/scheme/ton_api.tlo | Bin 113828 -> 114688 bytes validator/impl/collator-impl.h | 3 +- validator/impl/collator.cpp | 60 +++++++++++++++------ validator/impl/out-msg-queue-proof.cpp | 6 +-- validator/interfaces/out-msg-queue-proof.h | 4 +- validator/interfaces/validator-manager.h | 52 ++++++++++++++---- 10 files changed, 124 insertions(+), 33 deletions(-) diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index d06e7a40cb..0893d630f9 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -768,6 +768,15 @@ bool BlockLimitStatus::would_fit(unsigned cls, ton::LogicalTime end_lt, td::uint limits.collated_data.fits(cls, collated_data_size_estimate)); } +double BlockLimitStatus::load_fraction(unsigned cls) const { + if (cls >= ParamLimits::limits_cnt) { + return 0.0; + } + return std::max({(double)estimate_block_size() / (double)limits.bytes.limit(cls), + (double)gas_used / (double)limits.gas.limit(cls), + (double)collated_data_size_estimate / (double)limits.collated_data.limit(cls)}); +} + // SETS: account_dict, shard_libraries_, mc_state_extra // total_balance{,_extra}, total_validator_fees // SETS: out_msg_queue, processed_upto_, ihr_pending (via unpack_out_msg_queue_info) diff --git a/crypto/block/block.h b/crypto/block/block.h index 65834d79e9..c81dfdefa2 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -242,6 +242,9 @@ struct ParamLimits { td::uint32 hard() const { return limits_[3]; } + td::uint32 limit(unsigned cls) const { + return limits_[cls]; + } bool compute_medium_limit() { limits_[2] = soft() + ((hard() - soft()) >> 1); return true; @@ -299,6 +302,7 @@ struct BlockLimitStatus { bool fits(unsigned cls) const; bool would_fit(unsigned cls, ton::LogicalTime end_lt, td::uint64 more_gas, const vm::NewCellStorageStat::Stat* extra = nullptr) const; + double load_fraction(unsigned cls) const; bool add_cell(Ref cell) { st_stat.add_cell(std::move(cell)); return true; diff --git a/crypto/smartcont/CreateState.fif b/crypto/smartcont/CreateState.fif index 6cb9dbcb5a..25873eefe6 100644 --- a/crypto/smartcont/CreateState.fif +++ b/crypto/smartcont/CreateState.fif @@ -179,8 +179,16 @@ variable special-dict // bytes-limits gas-limits lt-limits -- c { } : make-block-limits +// bytes-limits gas-limits lt-limits collated-data-limits imported_queue_bytes imported_queue_msgs -- c +{ +} : make-block-limits-v2 + { make-block-limits 22 config! } : config.mc_block_limits! { make-block-limits 23 config! } : config.block_limits! +{ make-block-limits-v2 22 config! } : config.mc_block_limits_v2! +{ make-block-limits-v2 23 config! } : config.block_limits_v2! // mc-block-create-fee bc-block-create-fee { } : make-block-create-fees diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index e441551ee3..56f3145b82 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -933,11 +933,18 @@ validatorStats.stats flags:# validatorStats.blockLimitsStatus bytes:int gas:int lt_delta:int collated_data_bytes:int cat_bytes:int cat_gas:int cat_lt_delta:int cat_collated_data_bytes:int + load_fraction_queue_cleanup:double load_fraction_dispatch:double + load_fraction_internals:double load_fraction_externals:double + load_fraction_new_msgs:double limits_log:string = validatorStats.BlockLimitsStatus; -validatorStats.extMsgsStats +validatorStats.blockStats.extMsgsStats ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validatorStats.ExtMsgsStats; +validatorStats.blockStats.neighborStats shard:tonNode.shardId is_trivial:Bool is_local:Bool msg_limit:int + processed_msgs:int skipped_msgs:int limit_reached:Bool = validatorStats.blockStats.NeighborStats; validatorStats.blockStats - ext_msgs:validatorStats.extMsgsStats transactions:int shard_configuration:(vector tonNode.blockIdExt) = validatorStats.BlockStats; + ext_msgs:validatorStats.blockStats.extMsgsStats transactions:int shard_configuration:(vector tonNode.blockIdExt) + old_out_msg_queue_size:long new_out_msg_queue_size:long msg_queue_cleaned:int + neighbors:(vector validatorStats.blockStats.neighborStats) = validatorStats.BlockStats; validatorStats.collatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 cc_seqno:int collated_at:double diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 5c7285d51655490215647cf2954abcf83c0c7ef0..76d27f04a2f05c3b7cd3bb76a2c4c1b57c83158c 100644 GIT binary patch delta 749 zcmZ4TldYkFo%hjfeJchiaNfwf*p8)m^Nlx?x7gXRJV;ARoBYLY3k%QH--{=&v45h& z@FMSwdRbylW=dj7eo=5qVo9-HQciwyHiYSynwg%F1eKV$L1OX*haZd=n>!u1cRDiO z76zL0a$5HXuDMiowwy*G~u`S4Bu8_cM_eL@%`2cwL1PJUuad|FXr za!F=>UVLF`YH4bGa!zVuUTMMf1+I*eVyFsIGK&imOOi7t=jTaEh@;A7=9Q!t* is_special = nullptr); bool process_inbound_internal_messages(); bool precheck_inbound_message(Ref msg, ton::LogicalTime lt); - bool process_inbound_message(Ref msg, ton::LogicalTime lt, td::ConstBitPtr key, - const block::McShardDescr& src_nb); + bool process_inbound_message(Ref msg, ton::LogicalTime lt, td::ConstBitPtr key, int src_nb_idx); bool process_inbound_external_messages(); int process_external_message(Ref msg); bool process_dispatch_queue(); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 33719c2f9a..1d292fd7bf 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -919,6 +919,7 @@ void Collator::got_neighbor_msg_queues(td::Result res) { if (res->block_state_proof_.not_null() && !block_id.is_masterchain()) { block_state_proofs_.emplace(block_id.root_hash, res->block_state_proof_); } + + auto &neighbor_stats = stats_.neighbors.at(i); + neighbor_stats.shard = block_id.shard_full(); + neighbor_stats.is_trivial = shard_intersects(block_id.shard_full(), shard_); + neighbor_stats.is_local = res->is_local_; + neighbor_stats.msg_limit = res->msg_count_; + Ref state_root; if (block_id.is_masterchain()) { state_root = res->state_root_; @@ -1000,18 +1008,16 @@ void Collator::got_neighbor_msg_queue(unsigned i, Ref res) { return; } outq_descr.clear(); - do { - // require masterchain blocks referred to in ProcessedUpto - // TODO: perform this only if there are messages for this shard in our output queue - // .. (have to check the above condition and perform a `break` here) .. - // .. - for (const auto& entry : descr.processed_upto->list) { - Ref state; - if (!request_aux_mc_state(entry.mc_seqno, state)) { - return; - } + // require masterchain blocks referred to in ProcessedUpto + // TODO: perform this only if there are messages for this shard in our output queue + // .. (have to check the above condition and perform a `break` here) .. + // .. + for (const auto& entry : descr.processed_upto->list) { + Ref state; + if (!request_aux_mc_state(entry.mc_seqno, state)) { + return; } - } while (false); + } } /** @@ -1940,7 +1946,7 @@ bool Collator::try_collate() { last_proc_int_msg_.second.set_zero(); first_unproc_int_msg_.first = ~0ULL; first_unproc_int_msg_.second.set_ones(); - old_out_msg_queue_size_ = out_msg_queue_size_; + stats_.old_out_msg_queue_size = old_out_msg_queue_size_ = out_msg_queue_size_; if (is_masterchain()) { LOG(DEBUG) << "getting the list of special smart contracts"; auto res = config_->get_special_smartcontracts(); @@ -2407,6 +2413,9 @@ bool Collator::dequeue_message(Ref msg_envelope, ton::LogicalTime deli * @returns True if the cleanup operation was successful, false otherwise. */ bool Collator::out_msg_queue_cleanup() { + SCOPE_EXIT { + stats_.load_fraction_queue_cleanup = block_limit_status_->load_fraction(block::ParamLimits::cl_normal); + }; LOG(INFO) << "cleaning outbound queue from messages already imported by neighbors"; if (verbosity >= 2) { FLOG(INFO) { @@ -2492,6 +2501,7 @@ bool Collator::out_msg_queue_cleanup() { } return true; }); + stats_.msg_queue_cleaned = deleted; LOG(WARNING) << "deleted " << deleted << " messages from out_msg_queue after merge, remaining queue size is " << out_msg_queue_size_; if (!ok) { @@ -2572,6 +2582,7 @@ bool Collator::out_msg_queue_cleanup() { std::swap(queue_parts[i], queue_parts.back()); queue_parts.pop_back(); } + stats_.msg_queue_cleaned = deleted; LOG(WARNING) << "deleted " << deleted << " messages from out_msg_queue, remaining queue size is " << out_msg_queue_size_; } @@ -3712,12 +3723,13 @@ bool Collator::precheck_inbound_message(Ref enq_msg, ton::Logical * @param enq_msg The inbound message serialized using EnqueuedMsg TLB-scheme. * @param lt The logical time of the message. * @param key The 32+64+256-bit key of the message. - * @param src_nb The description of the source neighbor shard. + * @param src_nb_idx The index of the source neighbor shard. * * @returns True if the message was processed successfully, false otherwise. */ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalTime lt, td::ConstBitPtr key, - const block::McShardDescr& src_nb) { + int src_nb_idx) { + const auto& src_nb = neighbors_.at(src_nb_idx); ton::LogicalTime enqueued_lt = enq_msg->prefetch_ulong(64); auto msg_env = enq_msg->prefetch_ref(); // 1. unpack MsgEnvelope @@ -3815,6 +3827,8 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT << " enqueued_lt=" << enq_msg_descr.enqueued_lt_ << " has been already processed by us before, skipping"; // should we dequeue the message if it is ours (after a merge?) // (it should have been dequeued by out_msg_queue_cleanup() before) + auto &neighbor_stats = stats_.neighbors.at(src_nb_idx); + ++neighbor_stats.skipped_msgs; return true; } // 6.1. check whether we have already processed this message by IHR @@ -3901,12 +3915,17 @@ static std::string block_full_comment(const block::BlockLimitStatus& block_limit * @returns True if the processing was successful, false otherwise. */ bool Collator::process_inbound_internal_messages() { + SCOPE_EXIT { + stats_.load_fraction_internals = block_limit_status_->load_fraction(block::ParamLimits::cl_normal); + }; while (!nb_out_msgs_->is_eof()) { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); auto kv = nb_out_msgs_->extract_cur(); CHECK(kv && kv->msg.not_null()); + auto &neighbor_stats = stats_.neighbors.at(kv->source); if (kv->limit_exceeded) { LOG(INFO) << "limit for imported messages is reached, stop processing inbound internal messages"; + neighbor_stats.limit_reached = true; block::EnqueuedMsgDescr enq; enq.unpack(kv->msg.write()); // Visit cells to include it in proof break; @@ -3941,13 +3960,14 @@ bool Collator::process_inbound_internal_messages() { } LOG(DEBUG) << "processing inbound message with (lt,hash)=(" << kv->lt << "," << kv->key.to_hex() << ") from neighbor #" << kv->source; + ++neighbor_stats.processed_msgs; if (verbosity > 2) { FLOG(INFO) { sb << "inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg="; block::gen::t_EnqueuedMsg.print(sb, kv->msg); }; } - if (!process_inbound_message(kv->msg, kv->lt, kv->key.cbits(), neighbors_.at(kv->source))) { + if (!process_inbound_message(kv->msg, kv->lt, kv->key.cbits(), kv->source)) { if (verbosity > 1) { FLOG(INFO) { sb << "invalid inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() @@ -3970,6 +3990,9 @@ bool Collator::process_inbound_internal_messages() { * @returns True if the processing was successful, false otherwise. */ bool Collator::process_inbound_external_messages() { + SCOPE_EXIT { + stats_.load_fraction_externals = block_limit_status_->load_fraction(block::ParamLimits::cl_soft); + }; if (skip_extmsg_) { LOG(INFO) << "skipping processing of inbound external messages"; return true; @@ -4089,6 +4112,9 @@ int Collator::process_external_message(Ref msg) { * @returns True if the processing was successful, false otherwise. */ bool Collator::process_dispatch_queue() { + SCOPE_EXIT { + stats_.load_fraction_dispatch = block_limit_status_->load_fraction(block::ParamLimits::cl_normal); + }; if (out_msg_queue_size_ > defer_out_queue_size_limit_ && old_out_msg_queue_size_ > hard_defer_out_queue_size_limit_) { return true; } @@ -4548,6 +4574,9 @@ bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_rema * @returns True if all new messages were processed successfully, false otherwise. */ bool Collator::process_new_messages(bool enqueue_only) { + SCOPE_EXIT { + stats_.load_fraction_new_msgs = block_limit_status_->load_fraction(block::ParamLimits::cl_normal); + }; while (!new_msgs.empty()) { block::NewOutMsg msg = new_msgs.top(); new_msgs.pop(); @@ -6380,6 +6409,7 @@ void Collator::finalize_stats() { return 0; }); } + stats_.new_out_msg_queue_size = out_msg_queue_size_; } /** diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index edd20a8d21..d5183d59c4 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -196,7 +196,7 @@ td::Result>> OutMsgQueueProof::fetch(Shard if (state_root->get_hash().as_slice() != state_root_hash.as_slice()) { return td::Status::Error("state root hash mismatch"); } - res.emplace_back(true, blocks[i], state_root, block_state_proof, f.msg_counts_[i]); + res.emplace_back(true, blocks[i], state_root, block_state_proof, false, f.msg_counts_[i]); data[i].first = blocks[i]; TRY_RESULT(state, ShardStateQ::fetch(blocks[i], {}, state_root)); @@ -346,7 +346,7 @@ void OutMsgQueueImporter::get_proof_local(std::shared_ptr entry, Blo auto state = R.move_as_ok(); if (block.seqno() == 0) { std::vector> proof = { - td::Ref(true, block, state->root_cell(), td::Ref{})}; + td::Ref(true, block, state->root_cell(), td::Ref{}, true)}; td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, std::move(proof), ProofSource::Local); return; } @@ -362,7 +362,7 @@ void OutMsgQueueImporter::get_proof_local(std::shared_ptr entry, Blo } Ref block_state_proof = create_block_state_proof(R.ok()->root_cell()).move_as_ok(); std::vector> proof = { - td::Ref(true, block, state->root_cell(), std::move(block_state_proof))}; + td::Ref(true, block, state->root_cell(), std::move(block_state_proof), true)}; td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, std::move(proof), ProofSource::Local); }); diff --git a/validator/interfaces/out-msg-queue-proof.h b/validator/interfaces/out-msg-queue-proof.h index c0aa56106b..564b040cfa 100644 --- a/validator/interfaces/out-msg-queue-proof.h +++ b/validator/interfaces/out-msg-queue-proof.h @@ -26,17 +26,19 @@ namespace validator { using td::Ref; struct OutMsgQueueProof : public td::CntObject { - OutMsgQueueProof(BlockIdExt block_id, Ref state_root, Ref block_state_proof, + OutMsgQueueProof(BlockIdExt block_id, Ref state_root, Ref block_state_proof, bool is_local, td::int32 msg_count = -1) : block_id_(block_id) , state_root_(std::move(state_root)) , block_state_proof_(std::move(block_state_proof)) + , is_local_(is_local) , msg_count_(msg_count) { } BlockIdExt block_id_; Ref state_root_; Ref block_state_proof_; + bool is_local_ = false; td::int32 msg_count_; // -1 - no limit static td::Result>> fetch(ShardIdFull dst_shard, std::vector blocks, diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 534e5ea92b..995afa1869 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -69,31 +69,63 @@ struct CollationStats { td::uint32 estimated_bytes = 0, gas = 0, lt_delta = 0, estimated_collated_data_bytes = 0; int cat_bytes = 0, cat_gas = 0, cat_lt_delta = 0, cat_collated_data_bytes = 0; std::string limits_log; + double total_time = 0.0, work_time = 0.0, cpu_work_time = 0.0; + std::string time_stats; + td::uint32 transactions = 0; std::vector shard_configuration; td::uint32 ext_msgs_total = 0; td::uint32 ext_msgs_filtered = 0; td::uint32 ext_msgs_accepted = 0; td::uint32 ext_msgs_rejected = 0; - double total_time = 0.0, work_time = 0.0, cpu_work_time = 0.0; - std::string time_stats; + + td::uint64 old_out_msg_queue_size = 0; + td::uint64 new_out_msg_queue_size = 0; + td::uint32 msg_queue_cleaned = 0; + struct NeighborStats { + ShardIdFull shard; + bool is_trivial = false; + bool is_local = false; + int msg_limit = -1; + td::uint32 processed_msgs = 0; + td::uint32 skipped_msgs = 0; + bool limit_reached = false; + + tl_object_ptr tl() const { + return create_tl_object( + create_tl_shard_id(shard), is_trivial, is_local, msg_limit, processed_msgs, skipped_msgs, limit_reached); + } + }; + std::vector neighbors; + + double load_fraction_queue_cleanup = -1.0; + double load_fraction_dispatch = -1.0; + double load_fraction_internals = -1.0; + double load_fraction_externals = -1.0; + double load_fraction_new_msgs = -1.0; tl_object_ptr tl() const { - std::vector> shards; + std::vector> shards_obj; for (const BlockIdExt& block_id : shard_configuration) { - shards.push_back(create_tl_block_id(block_id)); + shards_obj.push_back(create_tl_block_id(block_id)); + } + std::vector> neighbors_obj; + for (const NeighborStats& neighbor : neighbors) { + neighbors_obj.push_back(neighbor.tl()); } auto block_stats = create_tl_object( - create_tl_object(ext_msgs_total, ext_msgs_filtered, ext_msgs_accepted, - ext_msgs_rejected), - transactions, std::move(shards)); + create_tl_object(ext_msgs_total, ext_msgs_filtered, + ext_msgs_accepted, ext_msgs_rejected), + transactions, std::move(shards_obj), old_out_msg_queue_size, new_out_msg_queue_size, msg_queue_cleaned, + std::move(neighbors_obj)); return create_tl_object( create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes, actual_collated_data_bytes, attempt, self.bits256_value(), is_validator, total_time, work_time, cpu_work_time, time_stats, - create_tl_object(estimated_bytes, gas, lt_delta, - estimated_collated_data_bytes, cat_bytes, cat_gas, - cat_lt_delta, cat_collated_data_bytes, limits_log), + create_tl_object( + estimated_bytes, gas, lt_delta, estimated_collated_data_bytes, cat_bytes, cat_gas, cat_lt_delta, + cat_collated_data_bytes, load_fraction_queue_cleanup, load_fraction_dispatch, load_fraction_internals, + load_fraction_externals, load_fraction_new_msgs, limits_log), std::move(block_stats)); } }; From 4e02ba69a2763c50ef851b2b0a7e6d3c632da356 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 3 Jun 2025 12:02:51 +0300 Subject: [PATCH 276/524] Check limits from config in BuildOutMsgQueueProof --- validator/full-node-shard.cpp | 8 ------ validator/impl/out-msg-queue-proof.cpp | 37 ++++++++++++++++++++++---- validator/impl/out-msg-queue-proof.hpp | 1 + 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index fd3e587f40..60125a1c86 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -706,14 +706,6 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod return; } block::ImportedMsgQueueLimits limits{(td::uint32)query.limits_->max_bytes_, (td::uint32)query.limits_->max_msgs_}; - if (limits.max_msgs > 512) { - promise.set_error(td::Status::Error("max_msgs is too big")); - return; - } - if (limits.max_bytes > (1 << 21)) { - promise.set_error(td::Status::Error("max_bytes is too big")); - return; - } FLOG(DEBUG) { sb << "Got query getOutMsgQueueProof to shard " << dst_shard.to_str() << " from blocks"; for (const BlockIdExt &id : blocks) { diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index d5183d59c4..e3330ea42b 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -528,11 +528,11 @@ void OutMsgQueueImporter::alarm() { td::remove_if(it->second.pending_entries, [](const std::shared_ptr& entry) { return entry->done || entry->promises.empty(); }); if (it->second.timeout.is_in_past()) { - if (it->second.pending_entries.empty()) { - it = small_cache_.erase(it); - } else { - ++it; - } + if (it->second.pending_entries.empty()) { + it = small_cache_.erase(it); + } else { + ++it; + } } else { alarm_timestamp().relax(it->second.timeout); ++it; @@ -571,6 +571,33 @@ void BuildOutMsgQueueProof::abort_query(td::Status reason) { } void BuildOutMsgQueueProof::start_up() { + if (blocks_.size() > 16) { + abort_query(td::Status::Error("too many blocks")); + return; + } + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_top_masterchain_state, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, + R.move_as_error_prefix("failed to get masterchain state: ")); + } else { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_masterchain_state, + R.move_as_ok()); + } + }); +} + +void BuildOutMsgQueueProof::got_masterchain_state(Ref mc_state) { + auto config_limits = mc_state->get_imported_msg_queue_limits(dst_shard_.is_masterchain()); + if ((td::uint64)config_limits.max_msgs * blocks_.size() < limits_.max_msgs) { + abort_query(td::Status::Error("too big max_msgs")); + return; + } + if ((td::uint64)config_limits.max_bytes * blocks_.size() < limits_.max_bytes) { + abort_query(td::Status::Error("too big max_bytes")); + return; + } + for (size_t i = 0; i < blocks_.size(); ++i) { BlockIdExt id = blocks_[i].id; ++pending; diff --git a/validator/impl/out-msg-queue-proof.hpp b/validator/impl/out-msg-queue-proof.hpp index 6f4c08d1de..b537f31a91 100644 --- a/validator/impl/out-msg-queue-proof.hpp +++ b/validator/impl/out-msg-queue-proof.hpp @@ -105,6 +105,7 @@ class BuildOutMsgQueueProof : public td::actor::Actor { void abort_query(td::Status reason); void start_up() override; + void got_masterchain_state(Ref mc_state); void got_state_root(size_t i, Ref root); void got_block_root(size_t i, Ref root); void build_proof(); From 99ac9ad1a173b42f5c8ca9378eaae4805adadba9 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 3 Jun 2025 12:05:07 +0300 Subject: [PATCH 277/524] Prepare to use rldp2 in validator group --- validator/manager.cpp | 4 ++-- validator/validator-group.cpp | 1 + validator/validator-group.hpp | 7 +++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index 8cf85a9dc0..db1a49efc3 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2633,8 +2633,8 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group descr->addr.is_zero() ? ValidatorFullId{descr->key}.compute_short_id().bits256_value() : descr->addr}; auto G = td::actor::create_actor( PSTRING() << "valgroup" << shard.to_str(), shard, validator_id, session_id, validator_set, key_seqno, opts, - keyring_, adnl_, rldp_, overlays_, db_root_, actor_id(this), get_collation_manager(adnl_id), init_session, - opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_, + keyring_, adnl_, rldp_, rldp2_, overlays_, db_root_, actor_id(this), get_collation_manager(adnl_id), + init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_, opts_->need_monitor(shard, last_masterchain_state_)); return G; } diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index f9a8f27325..d2db0ae992 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -409,6 +409,7 @@ void ValidatorGroup::create_session() { } td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_adnl_id_); + td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_adnl_id_); } void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterchain_block_id) { diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index c7f9404128..a7456cf9f0 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -24,6 +24,7 @@ #include "validator-session/validator-session.h" #include "rldp/rldp.h" +#include "rldp2/rldp.h" #include @@ -77,8 +78,8 @@ class ValidatorGroup : public td::actor::Actor { td::Ref validator_set, BlockSeqno last_key_block_seqno, validatorsession::ValidatorSessionOptions config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId overlays, std::string db_root, - td::actor::ActorId validator_manager, + td::actor::ActorId rldp2, td::actor::ActorId overlays, + std::string db_root, td::actor::ActorId validator_manager, td::actor::ActorId collation_manager, bool create_session, bool allow_unsafe_self_blocks_resync, td::Ref opts, bool monitoring_shard) : shard_(shard) @@ -90,6 +91,7 @@ class ValidatorGroup : public td::actor::Actor { , keyring_(keyring) , adnl_(adnl) , rldp_(rldp) + , rldp2_(rldp2) , overlays_(overlays) , db_root_(std::move(db_root)) , manager_(validator_manager) @@ -131,6 +133,7 @@ class ValidatorGroup : public td::actor::Actor { td::actor::ActorId keyring_; td::actor::ActorId adnl_; td::actor::ActorId rldp_; + td::actor::ActorId rldp2_; td::actor::ActorId overlays_; std::string db_root_; td::actor::ActorId manager_; From 75c8ddd9bc4c88305cdbfed8b3187a3422d59253 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 4 Jun 2025 11:34:57 +0300 Subject: [PATCH 278/524] Increase default imported msg queue limit --- crypto/block/block.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/block/block.h b/crypto/block/block.h index c81dfdefa2..c83fb67018 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -219,7 +219,7 @@ static inline std::ostream& operator<<(std::ostream& os, const MsgProcessedUptoC struct ImportedMsgQueueLimits { // Default values td::uint32 max_bytes = 1 << 16; - td::uint32 max_msgs = 30; + td::uint32 max_msgs = 100; bool deserialize(vm::CellSlice& cs); ImportedMsgQueueLimits operator*(td::uint32 x) const { return {max_bytes * x, max_msgs * x}; From faff9865e9da58570ad0a1dcfb55ab3d962ea7eb Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 4 Jun 2025 17:13:02 +0300 Subject: [PATCH 279/524] Fix error processing in CellDbIn::store_block_state_permanent_bulk --- validator/db/celldb.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 675859bd14..f4a3a084bf 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -561,10 +561,13 @@ void CellDbIn::store_block_state_permanent_bulk(std::vector> new_blocks, async_executor, [this, SelfId = actor_id(this), timer = std::move(timer), timer_prepare = std::move(timer_prepare), promise = std::move(promise)](td::Result> R) mutable { - TRY_RESULT_PROMISE(promise, updates, std::move(R)); td::actor::send_lambda_later( - SelfId, [=, this, timer = std::move(timer), timer_prepare = std::move(timer_prepare), - updates = std::move(updates), promise = std::move(promise)]() mutable { + SelfId, [=, this, timer = std::move(timer), timer_prepare = std::move(timer_prepare), R = std::move(R), + promise = std::move(promise)]() mutable { + SCOPE_EXIT { + release_db(); + }; + TRY_RESULT_PROMISE(promise, updates, std::move(R)); TD_PERF_COUNTER(celldb_store_cell_multi); timer_prepare.pause(); td::Timer timer_write; @@ -608,7 +611,6 @@ void CellDbIn::store_block_state_permanent_bulk(std::vector> cell_db_statistics_.store_cell_bulk_queries_++; cell_db_statistics_.store_cell_bulk_total_blocks_ += updates.size(); } - release_db(); promise.set_result(td::Unit()); }); }); From dcde63acc9caae32a8ab3543cc715199641c384b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 6 Jun 2025 14:08:29 +0300 Subject: [PATCH 280/524] Fix parsing shard block verifier config --- validator/validator-options.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index 4768d64bfb..a4a61373c0 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -84,14 +84,14 @@ td::Status ShardBlockVerifierConfig::unpack(const ton_api::engine_validator_shar return td::Status::Error(PSTRING() << "duplicate node " << node_id); } shard.trusted_nodes.push_back(node_id); - if (shard_obj->required_confirms_ < 0 || shard_obj->required_confirms_ > (int)shard.trusted_nodes.size()) { - return td::Status::Error(PSTRING() - << "invalid required_confirms " << shard_obj->required_confirms_ << " for shard " - << shard.shard_id.to_str() << " (nodes: " << shard.trusted_nodes.size() << ")"); - } - shard.required_confirms = shard_obj->required_confirms_; - shards.push_back(std::move(shard)); } + if (shard_obj->required_confirms_ < 0 || shard_obj->required_confirms_ > (int)shard.trusted_nodes.size()) { + return td::Status::Error(PSTRING() + << "invalid required_confirms " << shard_obj->required_confirms_ << " for shard " + << shard.shard_id.to_str() << " (nodes: " << shard.trusted_nodes.size() << ")"); + } + shard.required_confirms = shard_obj->required_confirms_; + shards.push_back(std::move(shard)); } return td::Status::OK(); } From b4cf0ff2953a7a50f5759757403a8ba17b54ae51 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 6 Jun 2025 17:32:57 +0300 Subject: [PATCH 281/524] Bugfix in ext-client.cpp --- lite-client/ext-client.cpp | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/lite-client/ext-client.cpp b/lite-client/ext-client.cpp index a0e48e64a4..fd624ff8cc 100644 --- a/lite-client/ext-client.cpp +++ b/lite-client/ext-client.cpp @@ -99,7 +99,7 @@ class ExtClientImpl : public ExtClient { promise = std::move(promise)](td::Result R) mutable { if (R.is_error() && (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { - td::actor::send_closure(SelfId, &ExtClientImpl::on_server_error, server_idx); + td::actor::send_closure(SelfId, &ExtClientImpl::on_server_status, server_idx, false); } promise.set_result(std::move(R)); }; @@ -163,9 +163,10 @@ class ExtClientImpl : public ExtClient { explicit Callback(td::actor::ActorId parent, size_t idx) : parent_(std::move(parent)), idx_(idx) { } void on_ready() override { + td::actor::send_closure(parent_, &ExtClientImpl::on_server_status, idx_, true); } void on_stop_ready() override { - td::actor::send_closure(parent_, &ExtClientImpl::on_server_error, idx_); + td::actor::send_closure(parent_, &ExtClientImpl::on_server_status, idx_, false); } private: @@ -199,19 +200,32 @@ class ExtClientImpl : public ExtClient { return; } for (Server& server : servers_) { - if (server.timeout && server.timeout.is_in_past()) { + if (!server.timeout) { + continue; + } + if (server.timeout.is_in_past()) { LOG(INFO) << "Closing connection to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; server.client.reset(); server.alive = false; server.ignore_until = {}; + server.timeout = {}; + } else { + alarm_timestamp().relax(server.timeout); } } } - void on_server_error(size_t idx) { - servers_[idx].alive = false; - servers_[idx].ignore_until = td::Timestamp::in(BAD_SERVER_TIMEOUT); + void on_server_status(size_t idx, bool ok) { + if (ok) { + if (connect_to_all_) { + servers_[idx].alive = true; + servers_[idx].ignore_until = td::Timestamp::never(); + } + } else { + servers_[idx].alive = false; + servers_[idx].ignore_until = td::Timestamp::in(BAD_SERVER_TIMEOUT); + } } }; From b788322dd4a0df61eb6a635675437c2cc71cf211 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 10 Jun 2025 11:12:15 +0300 Subject: [PATCH 282/524] Fix setting block hashes in session stats --- validator-session/validator-session.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 9ce7e54045..e63b0e5692 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -1240,6 +1240,9 @@ void ValidatorSessionImpl::stats_process_action(td::uint32 node_id, ton_api::val obj.round_, description().get_source_id(node_id), candidate_id); if (stat && stat->got_submit_at <= 0.0) { stat->got_submit_at = td::Clocks::system(); + stat->block_id.root_hash = obj.root_hash_; + stat->block_id.file_hash = obj.file_hash_; + stat->collated_data_hash = obj.collated_data_file_hash_; } }, [&](const ton_api::validatorSession_message_approvedBlock &obj) { From 52bcce61a2e71de4e81f934c6737d23546db3a8c Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 30 Mar 2025 10:44:56 +0300 Subject: [PATCH 283/524] [Tolk] Optimize storeUint: merge consecutive constant stores Before: b.storeUint(0x18, 6) Now: b.storeUint(0,1).storeUint(1,1)... The compiler is able to join consecutive stores of const values. It works at the IR level, not at AST. That's why it works after constant folding: > x = 0; > b.storeUint(x, 10); > x = 1; > if (x==1) b.storeUint(x, 10); Merged to "store 1, 20." Same for skip bits, also merged. --- crypto/smartcont/tolk-stdlib/common.tolk | 6 +- tolk-tester/tests/cells-slices.tolk | 196 +++++++++++++++++++++-- tolk-tester/tests/pack-unpack-1.tolk | 12 +- tolk-tester/tests/strings-tests.tolk | 2 +- tolk/analyzer.cpp | 16 ++ tolk/builtins.cpp | 70 ++++++++ tolk/codegen.cpp | 6 + tolk/optimize.cpp | 97 +++++++++++ tolk/stack-transform.cpp | 14 +- tolk/tolk.h | 31 +--- 10 files changed, 387 insertions(+), 63 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index c9e116860d..76305743a9 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -664,7 +664,7 @@ fun slice.loadBool(mutate self): bool /// Shifts a slice pointer to [len] bits forward, mutating the slice. @pure fun slice.skipBits(mutate self, len: int): self - asm "SDSKIPFIRST"; + builtin; /// Returns the first `0 ≤ len ≤ 1023` bits of a slice. @pure @@ -758,13 +758,13 @@ fun builder.storeAddress(mutate self, addr: address): self /// Stores amount of Toncoins into a builder. @pure fun builder.storeCoins(mutate self, x: coins): self - asm "STGRAMS"; + builtin; /// Stores bool (-1 or 0) into a builder. /// Attention: true value is `-1`, not 1! If you pass `1` here, TVM will throw an exception. @pure fun builder.storeBool(mutate self, x: bool): self - asm(x self) "1 STI"; + builtin; /// Stores dictionary (represented by TVM `cell` or `null`) into a builder. /// In other words, stores a `1`-bit and a reference to [c] if [c] is not `null` and `0`-bit otherwise. diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 253ebfff67..6a14578fdf 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -77,7 +77,8 @@ fun test5(): [int,int] { @method_id(106) fun test6() { - return beginCell().storeUint(1, 32).storeUint(2, 32).storeUint(3, 32); + var st = beginCell().storeUint(1, 32).storeUint(2, 32).storeUint(3, 32).endCell().beginParse(); + return st.loadUint(96) == ((1 << 64) + (2 << 32) + 3); } @method_id(107) @@ -229,6 +230,96 @@ fun test16() { } +@method_id(117) +fun test17() { + var b = beginCell().storeUint(1, 4).storeCoins(0).storeInt(123, 8); + var s = b.endCell().beginParse(); + return (s.loadUint(4), s.loadCoins(), s.loadUint(8)); +} + +@method_id(118) +fun test18() { + var x = 0; + var b = beginCell(); + b = b.storeUint(x, 14); + x += 12; + if (10 > 3) { x += x; } + if (true) { + b.storeInt(x + 2, 8).storeUint(x = match (x) { 24 => 5, else => 0 }, 4); + } + var s = b.endCell().beginParse(); + return (s.loadUint(14), s.loadInt(8), s.loadUint(4)); +} + +fun test19() { + // numbers with potential overflow for STU are not joined, check via codegen + var b = beginCell(); + b.storeInt(123, 4).storeUint(0xFF, 8).storeUint(0xFF, 8).storeInt(-1, 8); + return b; +} + +@method_id(120) +fun test20() { + var x = false; + var n = 4; + var b = true ? beginCell() : null; + b.storeBool(true).storeBool(x); + b = b.storeBool(true).storeUint(0, n *= 2).storeBool(!!true).storeCoins(0); + var s = b.endCell().beginParse(); + return (s.loadBool(), s.loadBool(), s.loadBool(), s.loadUint(8), s.loadBool(), s.loadCoins()); +} + +fun test21(s: slice) { + // successive skipBits are also joined + var x = 8; + s.skipBits(x); + x -= 4; + s = s.skipBits(x).skipBits(2); + x *= 0; + s = s.skipBits(x); + return s; +} + +@method_id(122) +fun test22() { + // different builders aren't mixed, store inside them are joined independently + var (b1, b2) = (beginCell(), beginCell()); + b1.storeUint(8, 16).storeUint(8, 8); + b2.storeUint(8, 32).storeUint(1<<88, 100); + return ( + b1.endCell().beginParse().remainingBitsCount(), + b2.endCell().beginParse().skipBits(32).loadUint(100), + ); +} + +@method_id(123) +fun test23(uns: bool) { + // corner values, signed/unsigned 255/256 + var b = beginCell(); + if (uns) { + b.storeUint(1, 100).storeUint(2, 100).storeInt(3, 55).storeInt(0, 1); + b.storeUint(8, 256); + } else { + b.storeInt(1, 10).storeUint(2, 190).storeInt(3, 54).storeUint(1, 1); + } + return b.bitsCount(); +} + +@method_id(124) +fun test24(uns: bool) { + // doesn't fit into a single STI/STU instruction, is splitted + var b = beginCell(); + if (uns) { + b.storeUint(1, 100).storeUint(2, 100) + .storeInt(3, 100).storeInt(8, 19); + return b.endCell().beginParse().skipBits(200+100).loadInt(19); + } else { + b.storeInt(1, 20).storeUint(2, 200).storeInt(3, 35) + .storeUint(1, 1).storeUint(5, 5).storeUint(10, 10); + return b.endCell().beginParse().skipBits(255+6).loadUint(10); + } +} + fun main(): int { return 0; } @@ -239,6 +330,7 @@ fun main(): int { @testcase | 103 | 103 | 103 @testcase | 104 | | [ 1 3 ] @testcase | 105 | | [ 210 1 ] +@testcase | 106 | | -1 @testcase | 107 | | 72 40 72 @testcase | 108 | | 0 40 32 @testcase | 110 | | 64 3 0 0 -1 0 100 -1 @@ -249,20 +341,102 @@ fun main(): int { @testcase | 114 | 0 | 0 0 0 @testcase | 115 | | 123 456 123 456 @testcase | 116 | | BC{00140008000000ff00000008} +@testcase | 117 | | 1 0 123 +@testcase | 118 | | 0 26 5 +@testcase | 120 | | -1 0 -1 0 -1 0 +@testcase | 122 | | 24 309485009821345068724781056 +@testcase | 123 | -1 | 512 +@testcase | 123 | 0 | 255 +@testcase | 124 | -1 | 8 +@testcase | 124 | 0 | 10 + +We test that consequtive storeInt/storeUint with constants are joined into a single number -Note, that since 'compute-asm-ltr' became on be default, chaining methods codegen is not quite optimal. @fif_codegen """ test6 PROC:<{ - 1 PUSHINT // '0=1 - NEWC // '0=1 '1 - 32 STU // '1 - 2 PUSHINT // '1 '4=2 - SWAP // '4=2 '1 - 32 STU // '1 - 3 PUSHINT // '1 '7=3 - SWAP // '7=3 '1 - 32 STU // '1 + 18446744082299486211 PUSHINT + NEWC + 96 STU // '2 + ENDC // '11 +""" + +@fif_codegen +""" + test17 PROC:<{ + 4219 PUSHINT + NEWC + 16 STU +""" + +@fif_codegen +""" + test18 PROC:<{ + 421 PUSHINT + NEWC + 26 STU +""" + +@fif_codegen +""" + test19 PROC:<{ + NEWC + 123 PUSHINT + SWAP + 4 STI + 16 PUSHPOW2DEC + 16 STUR + -1 PUSHINT + SWAP + 8 STI }> """ + +@fif_codegen +""" + test20 PROC:<{ + 40976 PUSHINT + NEWC + 16 STU +""" + +@fif_codegen +""" + test21 PROC:<{ + 14 PUSHINT + SDSKIPFIRST + }> +""" + +@fif_codegen +""" + test22 PROC:<{ + NEWC // '2 + NEWC // b1 b2 + SWAP // b2 b1 + 2056 PUSHINT + 24 STUR // b2 b1 + SWAP // b1 b2 + 10141514286835656557042350424064 PUSHINTX + 132 STUR // b1 b2 +""" + +@fif_codegen +""" + test23 PROC:<{ + NEWC + SWAP + IF:<{ + 91343852333181432387730302044911803916571639814 PUSHINT + 256 STUR + 8 PUSHINT + 256 STUR + }>ELSE<{ + 56539106072908298546665520023773392506479484700019806659963456035401760775 PUSHINT + 255 STIR + }> + BBITS + }> +""" + */ diff --git a/tolk-tester/tests/pack-unpack-1.tolk b/tolk-tester/tests/pack-unpack-1.tolk index 002df87ddc..379055c46f 100644 --- a/tolk-tester/tests/pack-unpack-1.tolk +++ b/tolk-tester/tests/pack-unpack-1.tolk @@ -107,16 +107,12 @@ fun main(c: cell) { @fif_codegen """ test3 PROC:<{ - 20 PUSHINT // yy=20 - 10 PUSHINT // p.y=20 p.x=10 - NEWC // p.y=20 p.x=10 b - 32 STI // p.y=20 b - 32 STI // b + 42949672980 PUSHINT + NEWC + 64 STI // b ENDC // c CTOS // s - 32 PUSHINT // s '15=32 - SDSKIPFIRST // s - 32 PUSHINT // s '16=32 + 64 PUSHINT SDSKIPFIRST // s SBITS // '17 }> diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index 287dc0df3c..bbe4583266 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -78,5 +78,5 @@ fun test1() { @testcase | 0 | | 0 @testcase | 101 | | [ 65 66 67 68 ] -@code_hash 58447050269190289721084012139099481162782788646785441106022886746601529758643 +@code_hash 61963905046482786665931036224265588524206004864815957521622605110240907698574 */ diff --git a/tolk/analyzer.cpp b/tolk/analyzer.cpp index f9721f5f1f..8e9fe914e7 100644 --- a/tolk/analyzer.cpp +++ b/tolk/analyzer.cpp @@ -678,6 +678,16 @@ void Op::prepare_args(VarDescrList values) { } } +void Op::maybe_swap_builtin_args_to_compile() { + // in builtins.cpp, where optimizing constants are done, implementations assume that args are passed ltr (as declared); + // if a function has arg_order, called arguments might have been put on a stack not ltr, but in asm order; + // here we swap them back before calling FunctionBodyBuiltin compile, and also swap after + tolk_assert(arg_order_already_equals_asm()); + if (f_sym->method_name == "storeUint" || f_sym->method_name == "storeInt" || f_sym->method_name == "storeBool") { + std::swap(args[0], args[1]); + } +} + VarDescrList Op::fwd_analyze(VarDescrList values) { var_info.import_values(values); switch (cl) { @@ -705,7 +715,13 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { } AsmOpList tmp; if (!f_sym->is_asm_function()) { + if (arg_order_already_equals_asm()) { + maybe_swap_builtin_args_to_compile(); + } std::get(f_sym->body)->compile(tmp, res, args, loc); + if (arg_order_already_equals_asm()) { + maybe_swap_builtin_args_to_compile(); + } } int j = 0; for (var_idx_t i : left) { diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 39075452de..6a82b432f2 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1022,7 +1022,22 @@ static AsmOp compile_fetch_int(std::vector& res, std::vector // fun builder.storeUint (mutate self, x: int, len: int): self asm(x b len) "STUX"; static AsmOp compile_store_int(std::vector& res, std::vector& args, SrcLocation loc, bool sgnd) { tolk_assert(args.size() == 3 && res.size() == 1); + auto& x = args[1]; auto& z = args[2]; + // purpose: to merge consecutive `b.storeUint(0, 1).storeUint(1, 1)` into one "1 PUSHINT + 2 STU", + // when constant arguments are passed, keep them as a separate (fake) instruction, to be handled by optimizer later + bool value_and_len_is_const = z.is_int_const() && x.is_int_const(); + if (value_and_len_is_const && G.settings.optimization_level >= 2) { + // don't handle negative numbers or potential overflow, merging them is incorrect + bool value_is_safe = sgnd + ? x.int_const >= 0 && z.int_const < 64 && x.int_const < (1ULL << (z.int_const->to_long() - 1)) + : x.int_const >= 0; + if (value_is_safe && z.int_const > 0 && z.int_const <= (255 + !sgnd)) { + z.unused(); + x.unused(); + return AsmOp::Custom(loc, "MY_store_int"s + (sgnd ? "I " : "U ") + x.int_const->to_dec_string() + " " + z.int_const->to_dec_string(), 1); + } + } if (z.is_int_const() && z.int_const > 0 && z.int_const <= 256) { z.unused(); return exec_arg_op(loc, sgnd? "STI" : "STU", z.int_const, 2, 1); @@ -1030,6 +1045,36 @@ static AsmOp compile_store_int(std::vector& res, std::vector return exec_op(loc, sgnd ? "STIX" : "STUX", 3, 1); } +// fun builder.storeBool(mutate self, value: bool): self asm( -> 1 0) "1 STI"; +static AsmOp compile_store_bool(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(args.size() == 2 && res.size() == 1); + auto& v = args[1]; + // same purpose as for storeInt/storeUint above + // (particularly, `b.storeUint(const_int,32).storeBool(const_bool)` will be joined) + if (v.is_int_const() && v.int_const == 0 && G.settings.optimization_level >= 2) { + v.unused(); + return AsmOp::Custom(loc, "MY_store_intU 0 1", 1); + } + if (v.is_int_const() && v.int_const == -1 && G.settings.optimization_level >= 2) { + v.unused(); + return AsmOp::Custom(loc, "MY_store_intU 1 1", 1); + } + return exec_op(loc, "1 STI", 2, 1); +} + +// fun builder.storeCoins(mutate self, value: coins): self asm "STGRAMS"; +static AsmOp compile_store_coins(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(args.size() == 2 && res.size() == 1); + auto& v = args[1]; + // same purpose as for storeInt/storeUint above + // (particularly, `b.storeUint(const_int,32).storeCoins(const_zero)` will be joined) + if (v.is_int_const() && v.int_const == 0 && G.settings.optimization_level >= 2) { + v.unused(); + return AsmOp::Custom(loc, "MY_store_intU 0 4", 1); + } + return exec_op(loc, "STGRAMS", 2, 1); +} + // fun slice.loadBits (mutate self, len: int): self asm(s len -> 1 0) "LDSLICEX" // fun slice.preloadBits(self, len: int): slice asm(s len -> 1 0) "PLDSLICEX" static AsmOp compile_fetch_slice(std::vector& res, std::vector& args, SrcLocation loc, bool fetch) { @@ -1062,6 +1107,21 @@ AsmOp compile_slice_sdbeginsq(std::vector& res, std::vector& throw ParseError(loc, "slice.tryStripPrefix can be used only with constant arguments"); } +// fun slice.skipBits(mutate self, len: int): self "SDSKIPFIRST" +AsmOp compile_skip_bits_in_slice(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(args.size() == 2 && res.size() == 1); + auto& len = args[1]; + // same technique as for storeUint: + // consecutive `s.skipBits(8).skipBits(const_var_16)` will be joined into a single 24 + // to track this, represent it as a separate fake instruction to be detected by optimizer later + if (len.is_int_const() && len.int_const >= 0 && G.settings.optimization_level >= 2) { + len.unused(); + return AsmOp::Custom(loc, "MY_skip_bits " + len.int_const->to_dec_string(), 1); + } + return exec_op(loc, "SDSKIPFIRST", 2, 1); +} + + // fun tuple.get(t: tuple, index: int): X asm "INDEXVAR"; static AsmOp compile_tuple_get(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 2 && res.size() == 1); @@ -1353,6 +1413,9 @@ void define_builtins() { std::bind(compile_fetch_slice, _1, _2, _3, true), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, {}, {1, 0}); + define_builtin_method("slice.skipBits", Slice, ParamsSliceInt, Slice, nullptr, + compile_skip_bits_in_slice, + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf); define_builtin_method("slice.preloadInt", Slice, ParamsSliceInt, Int, nullptr, std::bind(compile_fetch_int, _1, _2, _3, false, true), FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); @@ -1373,6 +1436,13 @@ void define_builtins() { std::bind(compile_store_int, _1, _2, _3, false), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, {1, 0, 2}, {}); + define_builtin_method("builder.storeBool", Builder, {Builder, Bool}, Unit, nullptr, + compile_store_bool, + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, + {1, 0}, {}); + define_builtin_method("builder.storeCoins", Builder, {Builder, TypeDataCoins::create()}, Unit, nullptr, + compile_store_coins, + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf); define_builtin_method("tuple.get", Tuple, {Tuple, Int}, typeT, declGenericT, compile_tuple_get, FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp index ff6c39676b..a7c84d27d9 100644 --- a/tolk/codegen.cpp +++ b/tolk/codegen.cpp @@ -508,7 +508,13 @@ bool Op::generate_code_step(Stack& stack) { if (f_sym->is_asm_function()) { std::get(f_sym->body)->compile(stack.o, loc); // compile res := f (args) } else { + if (arg_order_already_equals_asm()) { + maybe_swap_builtin_args_to_compile(); + } std::get(f_sym->body)->compile(stack.o, res, args, loc); // compile res := f (args) + if (arg_order_already_equals_asm()) { + maybe_swap_builtin_args_to_compile(); + } } } else { if (f_sym->is_inline() || f_sym->is_inline_ref()) { diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index e6d6b070db..c8fd8fc967 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -164,6 +164,102 @@ bool Optimizer::detect_rewrite_big_THROW() { return true; } +// purpose 1: for one constant b.storeInt(123, 32) generate not "123 PUSHINT; SWAP; STI", but "123 PUSHINT; STIR" +// purpose 2: consecutive b.storeUint(ff, 16).storeUint(ff, 16) generate one "00ff00ff" STU +// (since it works at IR level, it also works for const variables and auto-serialization) +bool Optimizer::detect_rewrite_MY_store_int() { + bool first_my_store = op_[0]->is_custom() && op_[0]->op.starts_with("MY_store_int"); + if (!first_my_store) { + return false; + } + bool first_unsigned = op_[0]->op[12] == 'U'; + + int n_merged = 0; + td::RefInt256 total_number = td::make_refint(0); + int total_len = 0; + for (int i = 0; i < pb_; ++i) { + std::string_view s_op_number_len = op_[i]->op; // "MY_store_intU 123 32" + if (!s_op_number_len.starts_with("MY_store_int")) { + break; + } + + size_t sp = s_op_number_len.rfind(' '); + std::string s_number(s_op_number_len.substr(13 + 1, sp - 13 - 1)); + int len = std::stoi(std::string(s_op_number_len.substr(sp + 1))); + + if (total_len + len > (255 + first_unsigned)) { + break; + } + if (total_number != 0) { + total_number <<= len; + } + total_number += td::string_to_int256(s_number); + total_len += len; + n_merged++; + } + + p_ = n_merged; + q_ = 2; + oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, total_number)); + if (total_number == 0 && total_len == 4 && first_unsigned) { // "STGRAMS" stores four 0-bits cheaper than "4 STUR" + oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, "STGRAMS", 1, 1)); + } else { + oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, std::to_string(total_len) + (first_unsigned ? " STUR" : " STIR"), 1, 1)); + } + return true; +} + +// purpose: consecutive `s.skipBits(8).skipBits(const_var_16)` will be joined into a single 24 +bool Optimizer::detect_rewrite_MY_skip_bits() { + bool first_my_skip = op_[0]->is_custom() && op_[0]->op.starts_with("MY_skip_bits"); + if (!first_my_skip) { + return false; + } + + int n_merged = 0; + int total_skip_bits = 0; + for (int i = 0; i < pb_; ++i) { + std::string_view s_op_len = op_[i]->op; // "MY_skip_bits 32" + if (!s_op_len.starts_with("MY_skip_bits")) { + break; + } + + std::string s_number(s_op_len.substr(s_op_len.find(' ') + 1)); + total_skip_bits += std::stoi(s_number); + n_merged++; + } + + p_ = n_merged; + q_ = 2; + oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, td::make_refint(total_skip_bits))); + oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, "SDSKIPFIRST")); + return true; +} + +// pattern `NEWC` + `xxx PUSHINT` + `32 STUR` -> `xxx PUSHINT` + `NEWC` + `32 STU`, it's a bit cheaper +bool Optimizer::detect_rewrite_NEWC_PUSH_STUR() { + bool first_newc = op_[0]->is_custom() && op_[0]->op == "NEWC"; + if (!first_newc || pb_ < 3) { + return false; + } + bool next_push = op_[1]->is_const() && op_[1]->op.ends_with(" PUSHINT"); // actually there can be PUSHPOWDEC2, but ok + if (!next_push) { + return false; + } + bool next_stu_r = op_[2]->is_custom() && (op_[2]->op.ends_with(" STUR") || op_[2]->op.ends_with(" STIR")); + if (!next_stu_r) { + return false; + } + + p_ = 3; + q_ = 3; + oq_[0] = std::move(op_[1]); + oq_[1] = std::move(op_[0]); + oq_[2] = std::make_unique(AsmOp::Custom(oq_[0]->loc, op_[2]->op.substr(0, op_[2]->op.size() - 1), 1, 1)); + return true; +} + + bool Optimizer::is_push_const(int* i, int* c) const { return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_push_const(i, c); } @@ -583,6 +679,7 @@ bool Optimizer::find_at_least(int pb) { (is_pop(&i) && rewrite(AsmOp::Pop(loc, i))) || (is_pop_pop(&i, &j) && rewrite(AsmOp::Pop(loc, i), AsmOp::Pop(loc, j))) || (is_xchg_xchg(&i, &j, &k, &l) && rewrite(AsmOp::Xchg(loc, i, j), AsmOp::Xchg(loc, k, l))) || detect_rewrite_big_THROW() || + detect_rewrite_MY_store_int() || detect_rewrite_MY_skip_bits() || detect_rewrite_NEWC_PUSH_STUR() || (!(mode_ & 1) && ((is_rot() && rewrite(AsmOp::Custom(loc, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(loc, "-ROT", 3, 3))) || (is_2dup() && rewrite(AsmOp::Custom(loc, "2DUP", 2, 4))) || diff --git a/tolk/stack-transform.cpp b/tolk/stack-transform.cpp index fe5735e5c0..d86afaec68 100644 --- a/tolk/stack-transform.cpp +++ b/tolk/stack-transform.cpp @@ -254,7 +254,7 @@ bool StackTransform::compose(const StackTransform &a, const StackTransform &b, S x1 = a.try_load(i); } int y = b.A[j - 1].second; - if (!c.try_store(x2, a(y))) { + if (!c.try_store(x2, a.get(y))) { return false; } x2 = b.try_load(j, a.d); @@ -360,18 +360,6 @@ StackTransform StackTransform::Xchg(int i, int j, bool relaxed) { return t; } -StackTransform StackTransform::Push(int i) { - StackTransform t; - t.apply_push(i); - return t; -} - -StackTransform StackTransform::Pop(int i) { - StackTransform t; - t.apply_pop(i); - return t; -} - bool StackTransform::is_xchg(int i, int j) const { if (i == j) { return is_id(); diff --git a/tolk/tolk.h b/tolk/tolk.h index 6e5af87ca7..a329b1d557 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -362,6 +362,7 @@ struct Op { bool set_var_info_except(const VarDescrList& new_var_info, const std::vector& var_list); bool set_var_info_except(VarDescrList&& new_var_info, const std::vector& var_list); void prepare_args(VarDescrList values); + void maybe_swap_builtin_args_to_compile(); VarDescrList fwd_analyze(VarDescrList values); bool mark_noreturn(); bool is_empty() const { @@ -707,9 +708,6 @@ struct StackTransform { bool almost_equal(const StackTransform& other) const { return equal(other, true); } - bool operator==(const StackTransform& other) const { - return dp == other.dp && almost_equal(other); - } bool operator<=(const StackTransform& other) const { return dp <= other.dp && almost_equal(other); } @@ -724,28 +722,6 @@ struct StackTransform { return get(i); } bool set(int i, int v, bool relaxed = false); - int operator()(int i) const { - return get(i); - } - class Pos { - StackTransform& t_; - int p_; - - public: - Pos(StackTransform& t, int p) : t_(t), p_(p) { - } - Pos& operator=(const Pos& other) = delete; - operator int() const { - return t_.get(p_); - } - const Pos& operator=(int v) const { - t_.set(p_, v); - return *this; - } - }; - Pos operator[](int i) { - return Pos(*this, i); - } static const StackTransform rot; static const StackTransform rot_rev; bool is_id() const { @@ -814,8 +790,6 @@ struct StackTransform { void show(std::ostream& os, int mode = 0) const; static StackTransform Xchg(int i, int j, bool relaxed = false); - static StackTransform Push(int i); - static StackTransform Pop(int i); private: int try_load(int& i, int offs = 0) const; // returns A[i++].first + offs or inf_x @@ -926,6 +900,9 @@ struct Optimizer { bool is_2pop_blkdrop(int* i, int* j, int* k); bool detect_rewrite_big_THROW(); + bool detect_rewrite_MY_store_int(); + bool detect_rewrite_MY_skip_bits(); + bool detect_rewrite_NEWC_PUSH_STUR(); AsmOpConsList extract_code(); }; From 3f4eca40651ecebed6e6ef55cd0688cbfb4f2c90 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 1 Jun 2025 11:48:49 +0300 Subject: [PATCH 284/524] [Tolk] Better resolve receivers for conflicting names Allow `fun address.staticMethod`. Before, it was not allowed due to a global `address` function. Now, the type has more precedence. --- tolk-tester/tests/calls-tests.tolk | 10 ++++++++++ tolk-tester/tests/parse-address.tolk | 6 +++++- tolk/pipe-resolve-types.cpp | 3 ++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tolk-tester/tests/calls-tests.tolk b/tolk-tester/tests/calls-tests.tolk index de508a2d5d..a03cd5a301 100644 --- a/tolk-tester/tests/calls-tests.tolk +++ b/tolk-tester/tests/calls-tests.tolk @@ -150,6 +150,15 @@ fun test8() { return (createTFrom(), createTFrom((5, (8, createTFrom().0)))); } +fun int(initial: int) { return initial } +fun int.create0() { return 0 } +fun int.plus1(self) { return self + 1 } + +@method_id(109) +fun test9() { + return int(10) + int(5).plus1() + int.create0() + int.create0().plus1(); +} + fun main() {} /** @@ -161,4 +170,5 @@ fun main() {} @testcase | 106 | 0 | 5 -5 50 -50 0 @testcase | 107 | | 16 5 @testcase | 108 | | [ 1 2 3 ] [ 5 8 1 ] +@testcase | 109 | | 17 */ diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index cf74af3f10..3f1151f7a4 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -21,6 +21,10 @@ fun verifyAddr(addr: address, workchain: int, number: int) { assert (s.loadUint(256) == number) throw 111; } +fun address.createNone() { + return createAddressNone() +} + fun check0(addr: address) { assert (addr.getWorkchain() == 0) throw 111; } @@ -134,7 +138,7 @@ fun main() { cc2 != cc1, (cc1 as slice).bitsEqual(((cc2 as slice) as address) as slice), createAddressNone() == cc1, - createAddressNone() == createAddressNone(), + createAddressNone() == address.createNone(), check0(address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff")) ); } diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 25d98bb46a..194cc4998c 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -467,7 +467,8 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { // for static method calls, like "int.zero()" or "Point.create()", dot obj symbol is unresolved for now // so, resolve it as a type and store as a "type reference symbol" if (auto obj_ref = v->get_obj()->try_as()) { - if (obj_ref->sym == nullptr) { + // also, `someFn.prop` doesn't make any sense, show "unknown type"; it also forces `address.staticMethod()` to work + if (obj_ref->sym == nullptr || obj_ref->sym->try_as()) { std::string_view obj_type_name = obj_ref->get_identifier()->name; AnyTypeV obj_type_node = createV(obj_ref->loc, obj_type_name); if (obj_ref->has_instantiationTs()) { // Container.create From ba0f9835f6e614b00ec547d764c458873826c6d1 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 5 Jun 2025 14:18:05 +0700 Subject: [PATCH 285/524] [Tolk] Optimization: generate `IFJMP` inside `match` arms Then `match` is the last statement, or is immediately followed by `return`, generate not `IF...ELSE` to Fift output, but `IFJMP...`. Same for `if/else` statement. --- tolk-tester/tests/if-else-tests.tolk | 171 +++++++++++++++++++++++++++ tolk/pipe-ast-to-legacy.cpp | 42 ++++++- 2 files changed, 208 insertions(+), 5 deletions(-) diff --git a/tolk-tester/tests/if-else-tests.tolk b/tolk-tester/tests/if-else-tests.tolk index 159a86bd57..1656519af0 100644 --- a/tolk-tester/tests/if-else-tests.tolk +++ b/tolk-tester/tests/if-else-tests.tolk @@ -1,3 +1,5 @@ +global t: tuple; + @method_id(101) fun test1(x: int): int { if (x > 200) { @@ -100,6 +102,86 @@ fun test8(op: int) { return (withNoElse(op), withElse(op), withMatch(op)); } +fun demo9_1(x: int) { + if (x > 0) { t.push(123); } + else { t.push(456) } +} + +fun demo9_2(x: int) { + if (x > 0) { t.push(123); } +} + +fun demo9_3(x: int) { + if (x > 0) { t.push(123); } + return; +} + +fun demo9_4(x: int) { + if (x != -100) { + if (x > 0) { t.push(123); } + return; + } +} + +@method_id(109) +fun test9(x: int) { + t = createEmptyTuple(); + demo9_1(x); demo9_2(x); demo9_3(x); demo9_4(x); + return t; +} + +fun demo10_1(x: int) { + match(x) { + 0 => t.push(123), + 1 => t.push(456), + } +} + +fun demo10_2(x: int) { + if (x != -100) { + match (x) { + 0 => t.push(123), + else => t.push(456) + } + return; + } + t.push(789) +} + +@method_id(110) +fun test10(x: int) { + t = createEmptyTuple(); + demo10_1(x); demo10_2(x); + return t; +} + +fun demo_neg_11_1(mutate x: int) { + match (x) { + -1 => { x = 0; } + else => { x = 1; } + } +} + +fun justVoidPush() { + t.push(100); +} + +fun demo_neg_11_2(x: int) { + match (x) { + -1 => t.push(123), + else => t.push(456) + } + return justVoidPush(); +} + +@method_id(111) +fun test11(x: int) { + t = createEmptyTuple(); + demo_neg_11_1(mutate x); + demo_neg_11_2(x); + return (x, t); +} + fun main() { } @@ -125,6 +207,11 @@ fun main() { @testcase | 106 | 5 | 5000 @testcase | 108 | 123 | 100 100 100 @testcase | 108 | 345 | 300 300 300 +@testcase | 109 | 10 | [ 123 123 123 123 ] +@testcase | 109 | -10 | [ 456 ] +@testcase | 110 | 0 | [ 123 123 ] +@testcase | 110 | 5 | [ 456 ] +@testcase | 111 | 5 | 1 [ 456 100 ] @fif_codegen """ @@ -214,4 +301,88 @@ fun main() { 255 THROW }> """ + +@fif_codegen +""" + demo9_1 PROC:<{ + 0 GTINT + IFJMP:<{ + t GETGLOB + 123 PUSHINT + TPUSH + t SETGLOB + }> + t GETGLOB + 456 PUSHINT + TPUSH + t SETGLOB + }> +""" + +@fif_codegen +""" + demo9_2 PROC:<{ + 0 GTINT + IFJMP:<{ + t GETGLOB + 123 PUSHINT + TPUSH + t SETGLOB + }> + }> +""" + +@fif_codegen +""" + demo9_3 PROC:<{ + 0 GTINT + IFJMP:<{ + t GETGLOB + 123 PUSHINT + TPUSH + t SETGLOB + }> + }> +""" + +@fif_codegen +""" + demo9_4 PROC:<{ + DUP + -100 NEQINT + IFJMP:<{ + 0 GTINT + IFJMP:<{ + t GETGLOB + 123 PUSHINT + TPUSH + t SETGLOB + }> + }> + DROP + }> +""" + +@fif_codegen +""" + demo10_1 PROC:<{ + DUP + 0 EQINT + IFJMP:<{ + DROP + t GETGLOB + 123 PUSHINT + TPUSH + t SETGLOB + }> + 1 EQINT + IFJMP:<{ + t GETGLOB + 456 PUSHINT + TPUSH + t SETGLOB + }> + }> +""" + */ diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 91891f8c16..1f5025918f 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -54,6 +54,7 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, CodeBlob& code, LValContext* lval_ctx); void process_any_statement(AnyV v, CodeBlob& code); +static AnyV stmt_before_immediate_return = nullptr; // The goal of VarsModificationWatcher is to detect such cases: `return (x, x += y, x)`. // Without any changes, ops will be { _Call $2 = +($0_x, $1_y); _Return $0_x, $2, $0_x } - incorrect @@ -1236,6 +1237,9 @@ static std::vector process_match_expression(V v code.push_set_cur(if_op.block0); if (v->is_statement()) { pre_compile_expr(v_ith_arm->get_body(), code); + if (v == stmt_before_immediate_return) { + code.emplace_back(v_ith_arm->loc, Op::_Return); + } } else { std::vector arm_ir_idx = pre_compile_expr(v_ith_arm->get_body(), code, v->inferred_type); code.emplace_back(v->loc, Op::_Let, result_ir_idx, std::move(arm_ir_idx)); @@ -1249,6 +1253,9 @@ static std::vector process_match_expression(V v auto v_last_arm = v->get_arm(n_arms - 1); if (v->is_statement()) { pre_compile_expr(v_last_arm->get_body(), code); + if (v == stmt_before_immediate_return) { + code.emplace_back(v_last_arm->loc, Op::_Return); + } } else { std::vector arm_ir_idx = pre_compile_expr(v_last_arm->get_body(), code, v->inferred_type); code.emplace_back(v->loc, Op::_Let, result_ir_idx, std::move(arm_ir_idx)); @@ -1743,9 +1750,30 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ static void process_block_statement(V v, CodeBlob& code) { - for (AnyV item : v->get_items()) { - process_any_statement(item, code); + if (v->empty()) { + return; } + + FunctionPtr cur_f = code.fun_ref; + bool does_f_return_nothing = cur_f->inferred_return_type == TypeDataVoid::create() && !cur_f->does_return_self() && !cur_f->has_mutate_params(); + bool is_toplevel_block = v == cur_f->ast_root->as()->get_body(); + + // we want to optimize `match` and `if/else`: if it's the last statement, implicitly add "return" to every branch + // (to generate IFJMP instead of nested IF ELSE); + // a competent way is to do it at the IR level (building CST, etc.), it's impossible to tweak Ops for now; + // so, for every `f() { here }` of `... here; return;`, save it into a global, and handle within match/if + AnyV backup = stmt_before_immediate_return; + for (int i = 0; i < v->size() - 1; ++i) { + AnyV stmt = v->get_item(i); + AnyV next_stmt = v->get_item(i + 1); + bool next_is_empty_return = next_stmt->kind == ast_return_statement && !next_stmt->as()->has_return_value(); + stmt_before_immediate_return = next_is_empty_return && does_f_return_nothing ? stmt : nullptr; + process_any_statement(stmt, code); + } + AnyV last_stmt = v->get_item(v->size() - 1); + stmt_before_immediate_return = is_toplevel_block && does_f_return_nothing ? last_stmt : nullptr; + process_any_statement(last_stmt, code); + stmt_before_immediate_return = backup; } static void process_assert_statement(V v, CodeBlob& code) { @@ -1817,9 +1845,15 @@ static void process_if_statement(V v, CodeBlob& code) { Op& if_op = code.emplace_back(v->loc, Op::_If, std::move(cond)); code.push_set_cur(if_op.block0); process_any_statement(v->get_if_body(), code); + if (v == stmt_before_immediate_return) { + code.emplace_back(v->get_if_body()->loc_end, Op::_Return); + } code.close_pop_cur(v->get_if_body()->loc_end); code.push_set_cur(if_op.block1); process_any_statement(v->get_else_body(), code); + if (v == stmt_before_immediate_return) { + code.emplace_back(v->get_else_body()->loc_end, Op::_Return); + } code.close_pop_cur(v->get_else_body()->loc_end); if (v->is_ifnot) { std::swap(if_op.block0, if_op.block1); @@ -1978,9 +2012,7 @@ static void convert_function_body_to_CodeBlob(FunctionPtr fun_ref, FunctionBodyC blob->in_var_cnt = blob->var_cnt; tolk_assert(blob->var_cnt == total_arg_width); - for (AnyV item : v_body->get_items()) { - process_any_statement(item, *blob); - } + process_block_statement(v_body, *blob); append_implicit_return_statement(v_body->loc_end, *blob); blob->close_blk(v_body->loc_end); From 3feba11dfd47e04e9c5097b7f676ec39ebc6ac6c Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Tue, 10 Jun 2025 17:48:43 +0700 Subject: [PATCH 286/524] [Tolk] Universal createMessage and createExternalLogMessage Features: - extra currencies - stateInit (code+data) with automatic address computation - different workchains - sharding (formerly splitDepth) - auto-serialized body - auto-detect "body ref or not" - more efficient than handwritten code --- crypto/smartcont/tolk-stdlib/common.tolk | 373 +++++++-- tolk-tester/tests/cells-slices.tolk | 5 +- tolk-tester/tests/generics-2.tolk | 12 +- tolk-tester/tests/generics-3.tolk | 2 +- tolk-tester/tests/generics-4.tolk | 2 +- tolk-tester/tests/intN-tests.tolk | 2 +- .../tests/invalid-serialization/err-7114.tolk | 8 +- .../tests/invalid-serialization/err-7604.tolk | 21 + .../tests/invalid-serialization/err-7605.tolk | 19 + tolk-tester/tests/nullable-tensors.tolk | 42 +- tolk-tester/tests/pack-unpack-2.tolk | 17 +- tolk-tester/tests/pack-unpack-3.tolk | 2 +- tolk-tester/tests/pack-unpack-6.tolk | 32 +- tolk-tester/tests/parse-address.tolk | 94 ++- tolk-tester/tests/send-msg-1.tolk | 733 ++++++++++++++++++ tolk-tester/tests/send-msg-2.tolk | 337 ++++++++ tolk-tester/tests/send-msg-3.tolk | 233 ++++++ tolk-tester/tests/smart-cast-tests.tolk | 8 +- tolk-tester/tests/struct-tests.tolk | 26 +- tolk-tester/tests/union-types-tests.tolk | 38 +- tolk-tester/tests/var-apply-tests.tolk | 4 +- tolk-tester/tolk-tester.js | 13 +- tolk-tester/tolk-tester.py | 12 +- tolk/CMakeLists.txt | 1 + tolk/builtins.cpp | 33 + tolk/pack-unpack-api.cpp | 11 +- tolk/pack-unpack-serializers.cpp | 27 + tolk/pack-unpack-serializers.h | 1 + tolk/pipe-ast-to-legacy.cpp | 17 +- tolk/pipe-check-serialized-fields.cpp | 13 +- tolk/send-message-api.cpp | 496 ++++++++++++ tolk/send-message-api.h | 28 + 32 files changed, 2489 insertions(+), 173 deletions(-) create mode 100644 tolk-tester/tests/invalid-serialization/err-7604.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7605.tolk create mode 100644 tolk-tester/tests/send-msg-1.tolk create mode 100644 tolk-tester/tests/send-msg-2.tolk create mode 100644 tolk-tester/tests/send-msg-3.tolk create mode 100644 tolk/send-message-api.cpp create mode 100644 tolk/send-message-api.h diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 76305743a9..e522bf2c07 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -364,6 +364,11 @@ fun Cell.load(self, options: UnpackOptions = {}): T fun Cell.beginParse(self): slice asm "CTOS"; +/// Returns hash of a typed cell, same as [cell.hash]. +@pure +fun Cell.hash(self): slice + asm "HASHCU"; + /// RemainingBitsAndRefs is a special built-in type to get "all the rest" slice tail on reading. /// Example: /// ``` @@ -681,6 +686,12 @@ fun slice.removeLastBits(mutate self, len: int): self fun slice.getLastBits(self, len: int): slice asm "SDCUTLAST"; +/// Returns `0 ≤ len ≤ 1023` bits of a slice starting from `0 ≤ offset ≤ 1023`. +/// (in other words, extracts a bit substring `[offset, len)` out of the slice) +@pure +fun slice.getMiddleBits(self, offset: int, len: int): slice + asm "SDSUBSTR"; + /// Loads a dictionary (TL HashMapE structure, represented as TVM cell) from a slice. /// Returns `null` if `nothing` constructor is used. @pure @@ -884,7 +895,7 @@ fun builder.bitsCount(self): int fun address(stdAddress: slice): address builtin; -/// Creates a slice representing TL addr_none$00 (two `0` bits). +/// Creates a slice representing "none address" (TL addr_none$00 — two zero bits). @pure fun createAddressNone(): address asm "b{00} PUSHSLICE"; @@ -965,62 +976,14 @@ fun reserveExtraCurrenciesOnBalance(nanoTonCoins: coins, extraAmount: dict, rese /** - Messages sending and parsing primitives. - Working with messages is low-level right now, but still, every contract should do that. - - `Message` structure, its header and so on are specified in TL-B scheme, particularly: - int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool ... = CommonMsgInfo; + Creating and sending messages. + Basic scenario: + ``` + val outMsg = createMessage({ ... options }); // you get OutMessage + outMsg.send(mode); + ``` */ -/// 0b011000 tag - 0, ihr_disabled - 1, bounce - 1, bounced - 0, src = adr_none$00 -const BOUNCEABLE = 0x18; -/// 0b010000 tag - 0, ihr_disabled - 1, bounce - 0, bounced - 0, src = adr_none$00 -const NON_BOUNCEABLE = 0x10; - -/// Load msgFlags from incoming message body (4 bits). -@pure -fun slice.loadMessageFlags(mutate self): int - asm( -> 1 0) "4 LDU"; - -/// Having msgFlags (4 bits), check that a message is bounced. -/// Effectively, it's `msgFlags & 1` (the lowest bit present). -@pure -fun isMessageBounced(msgFlags: int): bool - asm "2 PUSHINT" "MODR"; - -/// Skip 0xFFFFFFFF prefix (when a message is bounced). -@pure -fun slice.skipBouncedPrefix(mutate self): self - asm "32 PUSHINT" "SDSKIPFIRST"; - -/// The guideline recommends to start the body of an internal message with uint32 `op` and uint64 `queryId`. -@pure -fun slice.loadMessageOp(mutate self): int - asm( -> 1 0) "32 LDU"; - -@pure -fun slice.skipMessageOp(mutate self): self - asm "32 PUSHINT" "SDSKIPFIRST"; - -@pure -fun builder.storeMessageOp(mutate self, op: int): self - asm(op self) "32 STU"; - -/// The guideline recommends that uint64 `queryId` should follow uint32 `op`. -@pure -fun slice.loadMessageQueryId(mutate self): int - asm( -> 1 0) "64 LDU"; - -@pure -fun slice.skipMessageQueryId(mutate self): self - asm "64 PUSHINT" "SDSKIPFIRST"; - -@pure -fun builder.storeMessageQueryId(mutate self, queryId: int): self - asm(queryId self) "64 STU"; - -/// SEND MODES - https://docs.ton.org/tvm.pdf page 137, SENDRAWMSG - /// mode = 0 is used for ordinary messages; the gas fees are deducted from the senging amount; action phaes should NOT be ignored. const SEND_MODE_REGULAR = 0; /// +1 means that the sender wants to pay transfer fees separately. @@ -1041,13 +1004,301 @@ const SEND_MODE_ESTIMATE_FEE_ONLY = 1024; /// +64 substitutes the entire balance of the incoming message as an outcoming value (slightly inaccurate, gas expenses that cannot be estimated before the computation is completed are not taken into account). /// +128 substitutes the value of the entire balance of the contract before the start of the computation phase (slightly inaccurate, since gas expenses that cannot be estimated before the completion of the computation phase are not taken into account). +type ExtraCurrenciesDict = dict; + +/// ContractState is "code + data" of a contract. +/// Used in outgoing messages (StateInit) to initialize a destination contract. +struct ContractState { + code: cell; + data: cell; +} + +/// AddressShardingOptions provides settings to calculate an address in another shard. +/// Consider [createMessage] and [address.buildSameAddressInAnotherShard] for usage. +struct AddressShardingOptions { + fixedPrefixLength: uint5; // shard depth, formerly splitDepth + closeTo: address; +} + +/// AutoDeployAddress is a destination that initializes a receiver contract if it does not exist yet. +/// In order to do this, it contains StateInit — and calculates an address by a hash of StateInit. +/// Example: +/// ``` +/// createMessage({ +/// dest: { +/// workchain: 0, +/// stateInit: { +/// code: jettonWalletCode, +/// data: jettonWalletEmptyStorage.toCell() +/// } +/// }, +/// ... +/// ``` +/// It's often called "deployment", but it's inaccurate. TON Blockchain doesn't have a dedicated +/// deployment mechanism. You just send a message to some address — and if it doesn't exist, but you've +/// attached a way to initialize it (code+data) — it's initialized immediately, and accepts your message. +/// You just provide code+data, and the compiler automatically calculates the destination address, +/// because in TON, the address of a contract, by definition, is a hash of its initial state. +struct AutoDeployAddress { + workchain: int8 = BASECHAIN; + stateInit: ContractState | cell; + toShard: AddressShardingOptions? = null; +} + +/// Options for creating an outgoing message. +/// Consider [createMessage] for examples. +struct CreateMessageOptions { + /// whether a message will bounce back on error + bounce: bool; + /// message value: attached tons (or tons + extra currencies) + value: coins | (coins, ExtraCurrenciesDict); + /// destination is either a provided address, or is auto-calculated by stateInit + dest: address | // either just send a message to some address + builder | // ... or a manually constructed builder with a valid address + (int8, uint256) | // ... or to workchain + hash (also known as accountID) + AutoDeployAddress; // ... or "send to stateInit" aka deploy (address auto-calculated) + /// body is any serializable object (or just miss this field for empty body) + body: TBody; +} + +/// Creates a message (`OutMessage`) — a well-formatted message cell. +/// Typically, you just send it. In advanced scenarios, you can estimate fees or even postpone sending. +/// Example: +/// ``` +/// val reply = createMessage({ +/// bounce: false, +/// value: ton("0.05"), +/// dest: senderAddress, +/// body: RequestedInfo { ... } // note: no `toCell`! just pass an object +/// }); +/// reply.send(SEND_MODE_REGULAR); +/// ``` +/// Hint: don't call `body.toCell()`, pass `body` directly! +/// (if body is small, it will be inlined without an expensive cell creation) +/// (if body is large, the compiler will automatically wrap it into a cell) +/// If you need an empty body, just miss the field `body`, fill other 3 fields. +@pure +fun createMessage(options: CreateMessageOptions): OutMessage + builtin; + + +/// ExtOutLogBucket is a variant of a custom external address for emitting logs "to the outer world". +/// It includes some "topic" (arbitrary number), that determines the format of the message body. +/// For example, you emit "deposit event" (reserving topic "deposit" = 123): +/// > `dest: ExtOutLogBucket { topic: 123 }, body: DepositData { ... }` +/// and external indexers can index your emitted logs by destination address without parsing body. +/// Currently, external messages are used only for emitting logs (for viewing them in indexers). +/// In the future, there might be +/// > 0x01 ExtOutOffchainContract { adnl: uint256 | bits256 } +/// Serialization details: '01' (addr_extern) + 256 (len) + 0x00 (prefix) + 248 bits = 267 in total +struct (0x00) ExtOutLogBucket { + topic: uint248 | bits248; +} + +/// Options for creating an external outgoing message. +/// Consider [createExternalLogMessage] for examples. +struct createExternalLogMessageOptions { + /// destination is either an external address or a pattern to calculate it + dest: address | // either some valid external/none address (not internal!) + builder | // ... or a manually constructed builder with a valid external address + ExtOutLogBucket; // ... or encode topic/eventID in destination + /// body is any serializable object (or just miss this field for empty body) + body: TBody; +} + +/// Creates an external message (`OutMessage`) — a well-formatted message cell. +/// Typically, you just send it. In advanced scenarios, you can estimate fees or even postpone sending. +/// Example: +/// ``` +/// val emitMsg = createExternalLogMessage({ +/// dest: createAddressNone(), +/// body: DepositEvent { ... } // note: no `toCell`! just pass an object +/// }); +/// emitMsg.send(SEND_MODE_REGULAR); +/// ``` +/// Note: `createMessage` also returns `OutMessage`, it's okay: a composed message cell is universal. +/// Hint: don't call `body.toCell()`, pass `body` directly! +/// (if body is small, it will be inlined without an expensive cell creation) +/// (if body is large, the compiler will automatically wrap it into a cell) +@pure +fun createExternalLogMessage(options: createExternalLogMessageOptions): OutMessage + builtin; + +/// UnsafeBodyNoRef is used to prevent default behavior: when message body is potentially large, +/// it's packed into a separate ref. Wrapping body with this struct tells the compiler: +/// "inline message body in-place, I guarantee it will fit". +/// Example: +/// ``` +/// struct ProbablyLarge { a: (coins, coins, coins, coins, coins) } // max 620 bits +/// +/// val contents: ProbablyLarge = { ... }; // you are sure: values are small +/// createMessage({ +/// // body: contents, // by default, it will be stored a ref (it's large) +/// body: UnsafeBodyNoRef { +/// bodyForceNoRef: contents, // but wrapping forces the compiler to inline it +/// } +/// ``` +/// Another example: your body contains `builder` or `RemainingBitsAndRefs`: unpredictable size. +/// If you guarantee it's small and refs won't clash with code/data, avoid creating a cell. +struct UnsafeBodyNoRef { + bodyForceNoRef: T; +} + +/// OutMessage is a result of [createMessage]. +/// Essentially, it's a composed message cell, ready to be sent. +struct OutMessage { + messageCell: cell; +} + +/// Sends a ready message cell. +/// For `sendMode`, see the constants above (SEND_MODE_*). +fun OutMessage.send(self, sendMode: int): void + asm "SENDRAWMSG"; + +fun OutMessage.sendAndEstimateFee(self, sendMode: int): coins + asm "SENDMSG"; + +@pure +fun OutMessage.estimateFeeWithoutSending(self, sendMode: int): coins + asm "10 PUSHPOW2" "OR" "SENDMSG"; + +@pure +fun OutMessage.hash(self): uint256 + asm "HASHCU"; + + +/// StateInit is a "canonical TL/B representation from block.tlb" of a contract initial state. +/// But for everyday tasks, it's too complicated. It is not used in practice. +/// To represent code+data, consider [ContractState]. +/// To represent sharding (fixedPrefixLength, formerly splitDepth), consider [AutoDeployAddress]. +struct StateInit { + fixedPrefixLength: uint5?; + special: (bool, bool)?; + code: cell?; + data: cell?; + library: cell?; +} + +/// Calculates a hash of StateInit if only code+data are set. +@pure +fun StateInit.calcHashCodeData(code: cell, data: cell): uint256 asm +""" // code data + DUP2 // code data code data + HASHCU + SWAP + HASHCU // code data dataHash codeHash + SWAP2 // dataHash codeHash code data + CDEPTH + SWAP + CDEPTH // dataHash codeHash dataDepth codeDepth + + NEWC + x{020134} STSLICECONST // store refs_descriptor | bits_descriptor | data + 16 STU // store codeDepth + 16 STU // store dataDepth + 256 STU // store codeHash + 256 STU // store dataHash + + ONE HASHEXT_SHA256 +"""; + +/// Calculates a hash of StateInit if fixedPrefixLength+code+data are set. +@pure +fun StateInit.calcHashPrefixCodeData(fixedPrefixLength: uint5, code: cell, data: cell): uint256 asm +""" // pDepth code data + DUP2 // pDepth code data code data + HASHCU + SWAP + HASHCU // pDepth code data dataHash codeHash + SWAP2 // pDepth dataHash codeHash code data + CDEPTH + SWAP + CDEPTH // pDepth dataHash codeHash dataDepth codeDepth + + s4 PUSH // pDepth dataHash codeHash dataDepth codeDepth pDepth + 10 LSHIFT# + 0x020381A0 PUSHINT + ADD // calc refs_descriptor | bits_descriptor | data + + NEWC + 32 STU + 16 STU + 16 STU + 256 STU + 256 STU + + ONE HASHEXT_SHA256 // pDepth hash + NIP +"""; + + +/// Given an internal address A="aaaa...a" returns "bbaa...a" (D bits from address B, 256-D from A). +/// Example for fixedPrefixLength (shard depth) = 8: +/// | self (A) | aaaaaaaaaaa...aaa | +/// | closeTo (B) | 01010101bbb...bbb | shardPrefix = 01010101 (depth 8) +/// | result | 01010101aaa...aaa | address of A in same shard as B +/// More precisely, self (input) is 267 bits: '100' (std addr no anycast) + workchainA + "aaaa...a". +/// The result is also 267 bits: '100' + workchainB + "bb" (D bits) + "aa...a" (256-D bits). +/// Note: returns `builder`, not `address`! Because builder is cheap, and you can send a message to it: +/// ``` +/// createMessage({ +/// dest: resBuilder, // a builder containing a valid address can be passed to createMessage() +/// ... +/// ``` +/// If you really need `address`, use `address.fromValidBuilder(resBuilder)`. +@pure +fun address.buildSameAddressInAnotherShard(self, options: AddressShardingOptions): builder + builtin; + +/// Converts a builder containing a valid address (internal/external/none) to `address` (slice under the hood). +/// Gas-expensive! Because or a cell creation: essentially, it's `b.endCell().beginParse()`. +/// Example: you've manually built a 267-bit address: '100' (std addr no anycast) + workchain + hash +/// and you want to return `address` (TVM slice) from a contract getter: +/// ``` +/// get calcWalletAddress(ownerAddress: address): address { +/// val b = calculate(...); // manually calculate into builder +/// return address.fromValidBuilder(b) // but really need address +/// } +/// ``` +/// Hint: if you want just to send a message, don't call this function, use a builder directly: +/// ``` +/// createMessage({ +/// dest: someBuilder, // a builder containing a valid address can be passed to createMessage() +/// ... // so, don't call this expensive function in this case +/// ``` +@pure +fun address.fromValidBuilder(b: builder): address + asm "ENDC" "CTOS"; + + /// Sends a raw message — a correctly serialized TL object `Message X`. -/// For `mode`, see constants above (except SEND_MODE_ESTIMATE_FEE_ONLY). +/// In practice, you'll use a high-level wrapper [createMessage]. fun sendRawMessage(msg: cell, mode: int): void asm "SENDRAWMSG"; -/// Creates an output action and returns a fee for creating a message. -/// Mode has the same effect as in the case of SENDRAWMSG. -/// For mode including SEND_MODE_ESTIMATE_FEE_ONLY it just returns estimated fee without sending a message. -fun sendMessage(msg: cell, mode: int): int - asm "SENDMSG"; + + +/** + Receiving and handling messages. +*/ + +/// Load msgFlags from incoming message body (4 bits). +@pure +fun slice.loadMessageFlags(mutate self): int + asm( -> 1 0) "4 LDU"; + +/// Skip 0xFFFFFFFF prefix (when a message is bounced). +@pure +fun slice.skipBouncedPrefix(mutate self): self + asm "32 PUSHINT" "SDSKIPFIRST"; + +/// The guideline recommends to start the body of an internal message with uint32 `op` and uint64 `queryId`. +@pure +fun slice.loadMessageOp(mutate self): int + asm( -> 1 0) "32 LDU"; + +/// The guideline recommends that uint64 `queryId` should follow uint32 `op`. +@pure +fun slice.loadMessageQueryId(mutate self): int + asm( -> 1 0) "64 LDU"; + diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 6a14578fdf..2cbb786470 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -1,6 +1,10 @@ fun builder.store_u32(mutate self, value: int): self { return self.storeUint(value, 32); } +fun builder.storeMessageOp(mutate self, op: int): self + asm(op self) "32 STU"; +fun builder.storeMessageQueryId(mutate self, queryId: int): self + asm(queryId self) "64 STU"; fun slice.load_u32(mutate self): int { return self.loadUint(32); @@ -195,7 +199,6 @@ fun test111() { var q2 = s.loadMessageQueryId(); s.skipBits(64); s.assertEnd(); - assert(isMessageBounced(0x001) && !isMessageBounced(0x002)) throw 444; return (op1, q1, op2, q2); } diff --git a/tolk-tester/tests/generics-2.tolk b/tolk-tester/tests/generics-2.tolk index 8e95d884b0..2ee2851ae8 100644 --- a/tolk-tester/tests/generics-2.tolk +++ b/tolk-tester/tests/generics-2.tolk @@ -451,15 +451,15 @@ fun main(c: Wrapper, d: WrappedInt) { @testcase | 104 | | 11 11 13 13 13 2 @testcase | 105 | 0 | 80 @testcase | 105 | -1 | (null) -@testcase | 106 | | (null) (null) 0 777 1 2 142 +@testcase | 106 | | (null) (null) 0 777 1 2 typeid-14 @testcase | 107 | 5 | 20 20 5 5 20 777 (null) (null) (null) 0 -@testcase | 108 | 0 | 777 143 777 143 +@testcase | 108 | 0 | 777 typeid-15 777 typeid-15 @testcase | 108 | -1 | 777 0 777 0 @testcase | 109 | | 40 40 70 @testcase | 110 | | 20 1 20 42 -@testcase | 111 | | 5 1 132 5 1 132 5 1 132 5 1 132 -@testcase | 112 | -1 | 10 1 777 10 1 133 -@testcase | 112 | 0 | 20 1 777 (null) 0 134 +@testcase | 111 | | 5 1 typeid-4 5 1 typeid-4 5 1 typeid-4 5 1 typeid-4 +@testcase | 112 | -1 | 10 1 777 10 1 typeid-5 +@testcase | 112 | 0 | 20 1 777 (null) 0 typeid-6 @testcase | 113 | | 30 -1 @testcase | 114 | | 999 (null) 2 @testcase | 115 | | 10 0 200 1 -1 200 1 -1 @@ -467,7 +467,7 @@ fun main(c: Wrapper, d: WrappedInt) { @testcase | 117 | | 100 123 @testcase | 118 | | 123 1 777 123 1 -1 2 @testcase | 119 | | 40 40 -1 -1 -1 -1 -@testcase | 120 | | (null) 137 777 (null) (null) 0 139 -1 -1 +@testcase | 120 | | (null) typeid-9 777 (null) (null) 0 typeid-11 -1 -1 @testcase | 123 | | 10 10 10 10 10 diff --git a/tolk-tester/tests/generics-3.tolk b/tolk-tester/tests/generics-3.tolk index 4539d61446..a6fc748852 100644 --- a/tolk-tester/tests/generics-3.tolk +++ b/tolk-tester/tests/generics-3.tolk @@ -130,7 +130,7 @@ fun main() { @testcase | 101 | -1 | 10 @testcase | 101 | 0 | -1 @testcase | 103 | | 10 -1 0 -@testcase | 104 | | 12 129 132 -1 -1 -1 0 0 -1 0 0 +@testcase | 104 | | 12 typeid-1 typeid-4 -1 -1 -1 0 0 -1 0 0 @testcase | 105 | | -1 -1 -1 -1 0 0 @testcase | 108 | | 50000000 (null) (null) 100000000 12 12 24 diff --git a/tolk-tester/tests/generics-4.tolk b/tolk-tester/tests/generics-4.tolk index cfe5714ff1..70c70818d4 100644 --- a/tolk-tester/tests/generics-4.tolk +++ b/tolk-tester/tests/generics-4.tolk @@ -172,7 +172,7 @@ fun main() { /** @testcase | 103 | | 10 20 30 777 40 40 @testcase | 104 | | (null) (null) (null) -@testcase | 105 | | -1 (null) 0 (null) 134 777 0 123 0 456 133 +@testcase | 105 | | -1 (null) 0 (null) typeid-6 777 0 123 0 456 typeid-5 @testcase | 106 | | -1 0 -1 -1 @testcase | 107 | | 1 10 110 117 @testcase | 108 | | 10 (null) 20 diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index 6f44ed0305..5b5d8563f0 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -246,7 +246,7 @@ fun main() { @testcase | 110 | | 50000000 50000100 1234000000 51000000 @testcase | 111 | | [ 1000000000 1000000000 1000000000 -321123456789 321123456789 1100000000 ] @testcase | 114 | 5 | (null) (null) 0 -@testcase | 114 | 15 | 15 2 130 +@testcase | 114 | 15 | 15 2 typeid-2 @fif_codegen DECLPROC assign0 @fif_codegen DECLPROC assign0 diff --git a/tolk-tester/tests/invalid-serialization/err-7114.tolk b/tolk-tester/tests/invalid-serialization/err-7114.tolk index fe228d6019..2507e81f25 100644 --- a/tolk-tester/tests/invalid-serialization/err-7114.tolk +++ b/tolk-tester/tests/invalid-serialization/err-7114.tolk @@ -6,17 +6,17 @@ struct Demo { c: NotSerializableTensor; } -fun main(p: Demo?) { - p.toCell(); +fun main() { + Demo.fromSlice(""); } /** @compilation_should_fail -@stderr auto-serialization via toCell() is not available for type `Demo?` +@stderr auto-serialization via fromSlice() is not available for type `Demo` @stderr because field `Demo.c` of type `NotSerializableTensor` can't be serialized @stderr because alias `NotSerializableTensor` expands to `(int8, slice)` @stderr because element `tensor.1` of type `slice` can't be serialized -@stderr because type `slice` is not serializable, it doesn't define binary width +@stderr because type `slice` can not be used for reading, it doesn't define binary width @stderr hint: replace `slice` with `address` if it's an address, actually @stderr hint: replace `slice` with `bits128` and similar if it represents fixed-width data without refs */ diff --git a/tolk-tester/tests/invalid-serialization/err-7604.tolk b/tolk-tester/tests/invalid-serialization/err-7604.tolk new file mode 100644 index 0000000000..ea6edfe7ae --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7604.tolk @@ -0,0 +1,21 @@ +struct CCC { + v: int; +} + +fun main() { + var options = CreateMessageOptions { + bounce: true, + dest: createAddressNone(), + value: 0, + body: CCC { v: 10 } + }; + createMessage(options); +} + +/** +@compilation_should_fail +@stderr error: auto-serialization via createMessage() is not available for type `CCC` +@stderr because field `CCC.v` of type `int` can't be serialized +@stderr because type `int` is not serializable, it doesn't define binary width +@stderr hint: replace `int` with `int32` / `uint64` / `coins` / etc. + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7605.tolk b/tolk-tester/tests/invalid-serialization/err-7605.tolk new file mode 100644 index 0000000000..087e56457b --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7605.tolk @@ -0,0 +1,19 @@ +struct BigBodyMessage { + v1: int256; + v2: int256; + v3: int256; + a1: address; + a2: address; +} + +fun testFail(body: BigBodyMessage) { + createExternalLogMessage({ + dest: createAddressNone(), + body, + }); +} + +/** +@compilation_should_fail +@stderr struct `BigBodyMessage` can exceed 1023 bits in serialization (estimated size: 772..1302 bits) + */ diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index 4811c41447..d6934324e5 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -494,59 +494,59 @@ fun test143(setBNull: bool, setANullMid: bool) { fun main(){} /** -@testcase | 101 | | 1 2 129 -@testcase | 102 | | 1 2 129 (null) (null) 0 +@testcase | 101 | | 1 2 typeid-1 +@testcase | 102 | | 1 2 typeid-1 (null) (null) 0 @testcase | 103 | 1 2 | 3 3 0 1 2 @testcase | 104 | | 1 2 (null) (null) 0 -@testcase | 105 | | (null) (null) (null) 0 1 2 3 131 +@testcase | 105 | | (null) (null) (null) 0 1 2 3 typeid-3 @testcase | 106 | | 1 2 @testcase | 107 | | 0 0 -1 0 0 -1 -@testcase | 108 | 5 6 | 7 8 10 11 129 (null) (null) 0 +@testcase | 108 | 5 6 | 7 8 10 11 typeid-1 (null) (null) 0 @testcase | 109 | | 0 0 -1 0 -1 0 0 -1 -1 -@testcase | 110 | | 3 4 (null) (null) 0 6 7 129 +@testcase | 110 | | 3 4 (null) (null) 0 6 7 typeid-1 @testcase | 111 | | 50 30 70 90 100 @testcase | 112 | | 12 22 @testcase | 113 | | -1 @testcase | 114 | | (null) (null) (null) 0 (null) (null) (null) 0 -@testcase | 115 | | 2 3 7 (null) (null) 0 5 0 129 0 +@testcase | 115 | | 2 3 7 (null) (null) 0 5 0 typeid-1 0 @testcase | 116 | -1 | (null) (null) 0 (null) (null) 0 -@testcase | 116 | 0 | 1 2 129 1 2 129 +@testcase | 116 | 0 | 1 2 typeid-1 1 2 typeid-1 @testcase | 117 | | (null) 1 3 -@testcase | 118 | 5 | 5 10 129 +@testcase | 118 | 5 | 5 10 typeid-1 @testcase | 118 | null | (null) (null) 0 -@testcase | 119 | | (null) (null) 1 2 129 100 +@testcase | 119 | | (null) (null) 1 2 typeid-1 100 @testcase | 120 | -1 | (null) (null) 0 -@testcase | 120 | 0 | 1 2 129 +@testcase | 120 | 0 | 1 2 typeid-1 @testcase | 121 | | [ 1 [ 3 4 ] ] @testcase | 122 | 0 | [ 1 [ 3 4 ] 4 (null) ] @testcase | 122 | -1 | [ 1 (null) 4 (null) ] -@testcase | 123 | | 1 3 4 132 -@testcase | 124 | 0 | 1 3 4 132 4 (null) (null) 0 +@testcase | 123 | | 1 3 4 typeid-4 +@testcase | 124 | 0 | 1 3 4 typeid-4 4 (null) (null) 0 @testcase | 124 | -1 | 1 (null) (null) 0 4 (null) (null) 0 @testcase | 125 | | 3 @testcase | 126 | | 1 (null) 2 @testcase | 127 | 1 | 1 (null) (null) 0 2 -@testcase | 127 | 2 | 1 2 3 129 4 +@testcase | 127 | 2 | 1 2 3 typeid-1 4 @testcase | 127 | 3 | 1 (null) (null) 0 5 -@testcase | 128 | 1 | 1 (null) (null) 0 2 139 +@testcase | 128 | 1 | 1 (null) (null) 0 2 typeid-11 @testcase | 128 | 2 | (null) (null) (null) (null) (null) 0 -@testcase | 128 | 3 | 1 2 3 129 4 139 +@testcase | 128 | 3 | 1 2 3 typeid-1 4 typeid-11 @testcase | 129 | 0 | 5 5 0 -1 1 2 0 -1 @testcase | 129 | -1 | 5 5 0 -1 (null) (null) 0 -1 -@testcase | 130 | 0 | 1 2 3 129 +@testcase | 130 | 0 | 1 2 3 typeid-1 @testcase | 130 | -1 | 1 (null) (null) 0 -@testcase | 131 | | 140 777 0 777 777 777 0 0 140 140 777 140 140 141 777 +@testcase | 131 | | typeid-12 777 0 777 777 777 0 0 typeid-12 typeid-12 777 typeid-12 typeid-12 typeid-13 777 @testcase | 132 | | -1 0 -1 0 777 (null) (null) -1 0 0 @testcase | 133 | | 60 -@testcase | 134 | | 11 21 129 -@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 143 (null) 143 (null) 0 777 10 144 (null) 144 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 +@testcase | 134 | | 11 21 typeid-1 +@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 typeid-15 (null) typeid-15 (null) 0 777 10 typeid-16 (null) typeid-16 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 @testcase | 136 | 9 | 9 0 @testcase | 136 | null | (null) -1 -@testcase | 140 | 8 9 | 8 9 145 (null) (null) 0 +@testcase | 140 | 8 9 | 8 9 typeid-17 (null) (null) 0 @testcase | 141 | | (null) 10 @testcase | 142 | | 3 3 1 2 @testcase | 143 | -1 0 | 1 2 1 3 777 (null) (null) (null) (null) 0 -@testcase | 143 | 0 -1 | 1 (null) 0 3 777 1 2 1 3 146 +@testcase | 143 | 0 -1 | 1 (null) 0 3 777 1 2 1 3 typeid-18 @fif_codegen """ diff --git a/tolk-tester/tests/pack-unpack-2.tolk b/tolk-tester/tests/pack-unpack-2.tolk index 75f20a5fdd..2a17f7ca19 100644 --- a/tolk-tester/tests/pack-unpack-2.tolk +++ b/tolk-tester/tests/pack-unpack-2.tolk @@ -343,6 +343,11 @@ struct WriteWithBuilder { rest: builder; } +struct WriteWithSlice { + f1: int32; + rest: slice; +} + struct ReadWrittenWithBuilder { f1: int32; someInt: uint32; @@ -607,6 +612,8 @@ fun test_DifferentMix3() { fun test_WriteWithBuilderReadWithOther() { var b = beginCell().storeUint(55, 32).storeMaybeRef(null); var w: WriteWithBuilder = { f1: 10, rest: b }; + var w2: WriteWithSlice = { f1: 10, rest: stringHexToSlice("FFFF") }; + assert(w2.toCell().beginParse().skipBits(32).loadUint(16) == 0xFFFF, 100); return ReadWrittenWithBuilder.fromCell(w.toCell()); } @@ -664,18 +671,18 @@ fun main() { @testcase | 208 | | -1 @testcase | 209 | | -1 @testcase | 210 | | 123 555 46 (null) -@testcase | 211 | | (null) 132 123 -@testcase | 212 | | (null) 133 123 +@testcase | 211 | | (null) typeid-4 123 +@testcase | 212 | | (null) typeid-5 123 @testcase | 213 | | -1 @testcase | 214 | | -1 @testcase | 215 | | -1 @testcase | 216 | | -1 @testcase | 217 | | -1 @testcase | 218 | | -1 -@testcase | 219 | | 44 (null) (null) 46 143 -@testcase | 220 | | 0 0 (null) 99 136 137 99 1234 (null) 889129 14 +@testcase | 219 | | 44 (null) (null) 46 typeid-15 +@testcase | 220 | | 0 0 (null) 99 typeid-8 typeid-9 99 1234 (null) 889129 14 @testcase | 221 | | -1 -@testcase | 222 | | 510 567 9392843922 146 81923 81923 147 777 0 -1 (null) (null) 0 100000 100000 147 +@testcase | 222 | | 510 567 9392843922 typeid-18 81923 81923 typeid-19 777 0 -1 (null) (null) 0 100000 100000 typeid-19 @testcase | 223 | | 10 55 (null) @testcase | 224 | | 60 50000000 5 9 123 -1 @testcase | 225 | | 5 40 65535 8 50000000 diff --git a/tolk-tester/tests/pack-unpack-3.tolk b/tolk-tester/tests/pack-unpack-3.tolk index 74fa1b8e92..2e7d6ab2bc 100644 --- a/tolk-tester/tests/pack-unpack-3.tolk +++ b/tolk-tester/tests/pack-unpack-3.tolk @@ -278,7 +278,7 @@ fun main() {} /** @testcase | 201 | | 80 800000000 @testcase | 202 | | 800000000 17 777 -1 0 -@testcase | 203 | | (null) 0 131 777 -1 0 +@testcase | 203 | | (null) 0 typeid-3 777 -1 0 @testcase | 204 | | -1 0 -1 @testcase | 205 | | 0 -1 0 -1 */ diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk index 049109ad78..07ec41f852 100644 --- a/tolk-tester/tests/pack-unpack-6.tolk +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -147,14 +147,14 @@ RETALT """ IF:<{ DROP - 129 PUSHINT + 139 PUSHINT }>ELSE<{ x{03} SDBEGINSQ NIP IFNOTJMP:<{ 104 THROW }> - 130 PUSHINT + 140 PUSHINT }> """ @@ -199,24 +199,24 @@ IF:<{ @fif_codegen """ - test9 PROC:<{ // - x{010f} PUSHSLICE // s - x{01} SDBEGINSQ // s '6 - IF:<{ // s - DROP // - 129 PUSHINT // 'UTag=129 - }>ELSE<{ // s - x{03} SDBEGINSQ // s '6 - NIP // '6 - IFNOTJMP:<{ // + test9 PROC:<{ + x{010f} PUSHSLICE + x{01} SDBEGINSQ + IF:<{ + DROP + 139 PUSHINT + }>ELSE<{ + x{03} SDBEGINSQ + NIP + IFNOTJMP:<{ 16 PUSHPOW2DEC THROWANY }> - 130 PUSHINT // 'UTag=130 + 140 PUSHINT }> - 129 PUSHINT // 'UTag '17=129 - SWAP // '17=129 'UTag - EQUAL // '16 + 139 PUSHINT + SWAP + EQUAL }> """ diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index 3f1151f7a4..0f8ad02d29 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -35,6 +35,91 @@ fun codegenAddrEq(a: address, b: address) { return 3; } +@inline_ref +fun buildAddrInShard_manual(a: address, options: AddressShardingOptions) { + var sb = options.closeTo as slice; + sb.skipBits(3); // addr_std$10 + anycast 0 + val wc_b = sb.loadInt(8); + val shardPrefix = sb.loadUint(options.fixedPrefixLength); + + var sa = a as slice; + sa.skipBits(3 + 8 + options.fixedPrefixLength); + + return beginCell() + .storeUint(0b100, 3) // addr_std$10 + anycast 0 + .storeInt(wc_b, 8) + .storeUint(shardPrefix, options.fixedPrefixLength) + .storeSlice(sa); +} + +@method_id(101) +fun test1() { + val a = address("0:00000000000000000000000000000000000000000000000000000000000000FF"); + val b = address("1:1100000000000000000000000000000000000000000000000000000000000000"); + val dd1 = buildAddrInShard_manual(a, {fixedPrefixLength: 8, closeTo: b}); + val dd2 = a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: b}); + assert(dd1.endCell().hash() == dd2.endCell().hash(), 400); + return dd1.endCell().beginParse() as address + == address("1:11000000000000000000000000000000000000000000000000000000000000FF"); +} + +@method_id(102) +fun test2() { + val a = address("0:1234aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + val b = address("0:FFFFbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + val dd1 = buildAddrInShard_manual(a, {closeTo: b, fixedPrefixLength: 16}); + val dd2 = a.buildSameAddressInAnotherShard({fixedPrefixLength: 16, closeTo: b}); + assert((a as slice).remainingBitsCount() == 267, 267); + assert((b as slice).remainingBitsCount() == 267, 267); + assert(dd1.endCell().hash() == dd2.endCell().hash(), 400); + return address.fromValidBuilder(dd2) + == address("0:FFFFaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); +} + +@method_id(103) +fun test3() { + var t1_a = address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"); + var t1_b = address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); + var t2_a = address("EQCe4AYIBce1pAk2qJJPSs1OzyZRlKjkfq8zuC8D7erv6DUP"); + var t2_b = address("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"); + var t3_a = address("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"); + var t3_b = address("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"); + return ( + t1_a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: t1_b}).endCell().hash() & 0xFFFF, + t1_a.buildSameAddressInAnotherShard({fixedPrefixLength: 4, closeTo: t1_b}).endCell().hash() & 0xFFFF, + t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 2, closeTo: t2_b}).endCell().hash() & 0xFFFF, + t3_a.buildSameAddressInAnotherShard({fixedPrefixLength: 30, closeTo: t2_b}).endCell().hash() & 0xFFFF, + t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 1, closeTo: t3_b}).endCell().hash() & 0xFFFF, + t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 0, closeTo: t3_b}).endCell().hash() & 0xFFFF, + ) +} + +@method_id(104) +fun test4(shardDepth: int) { + var a = address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"); + var b = address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); + return a.buildSameAddressInAnotherShard({fixedPrefixLength: shardDepth, closeTo: b}).endCell().hash() & 0xFFFF; +} + +@method_id(105) +fun test5() { + var b = beginCell() + .storeUint(0b100, 3) // std addr no anycast + .storeInt(MASTERCHAIN, 8) + .storeUint(0xFFFF, 256); + val a = address("-1:000000000000000000000000000000000000000000000000000000000000FFFF"); + return address.fromValidBuilder(b) == a; +} + +@method_id(106) +fun test6() { + val a = address("0:00000000000000000000000000000000000000000000000000000000000000FF"); + val b = address("1:1100000000000000000000000000000000000000000000000000000000000000"); + return address.fromValidBuilder(a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: b})) == + address("1:11000000000000000000000000000000000000000000000000000000000000FF"); +} + + fun main() { __expect_type(cc1, "address"); @@ -144,7 +229,14 @@ fun main() { } /** -@testcase | 0 | | -1 0 -1 0 -1 +@testcase | 0 | | -1 0 -1 0 -1 +@testcase | 101 | | -1 +@testcase | 102 | | -1 +@testcase | 103 | | 39876 24338 15241 50719 11252 15241 +@testcase | 104 | 2 | 24338 +@testcase | 104 | 9 | 39876 +@testcase | 105 | | -1 +@testcase | 106 | | -1 @fif_codegen """ diff --git a/tolk-tester/tests/send-msg-1.tolk b/tolk-tester/tests/send-msg-1.tolk new file mode 100644 index 0000000000..07f736e207 --- /dev/null +++ b/tolk-tester/tests/send-msg-1.tolk @@ -0,0 +1,733 @@ +import "@stdlib/tvm-dicts" +import "@stdlib/tvm-lowlevel" + +/* +int_msg_info$0 1 + ihr_disabled:Bool // always 0, not implemented 1 + bounce:Bool // parameter 1 + bounced:Bool // always 0 on send 1 + src:MsgAddress // always 00 on send 2 + dest:MsgAddressInt // parameter 267 + value:CurrencyCollection // parameter 124 + 1 + ihr_fee:Grams // always 0, not implemented 4 + fwd_fee:Grams // always 0 on send 4 + created_lt:uint64 // always 0 on send 64 + created_at:uint32 // always 0 on send 32 + = CommonMsgInfoRelaxed; +_ split_depth:(Maybe (## 5)) 1 + 5 + special:(Maybe TickTock) 1 + 2 + code:(Maybe ^Cell) 1 + data:(Maybe ^Cell) 1 + library:(Maybe ^Cell) 1 + = StateInit; +message$_ {X:Type} + info:CommonMsgInfoRelaxed 502 + init:(Maybe (Either StateInit ^StateInit)) 12 + body:(Either X ^X) // body is either embedded or stored as ref + = MessageRelaxed X; + */ + +fun getMyAddressDev(): address + asm "x{80194DC6438F99D3D9DBE151944925D90B2492954BF6B9C070FBFF2DDED5F30547D_} PUSHSLICE"; + +@inline +fun calculateNftItemStateInitData(itemIndex: int): cell { + return beginCell() + .storeUint(itemIndex, 64) + .storeAddress(getMyAddressDev()) + .endCell(); +} + +@inline +fun calculateStateInitCell(code: cell, data: cell): cell { + return beginCell() + .storeUint(0, 2) // 0 split_depth, 0 special + .storeDict(code) + .storeDict(data) + .storeUint(0, 1) // 0 library + .endCell(); +} + +fun calculateNftItemAddress(workchain: int, stateInitCell: cell): address { + return beginCell() + .storeUint(0b100, 3) // addr_std$10 + 0 split_depth + .storeInt(workchain, 8) + .storeUint(stateInitCell.hash(), 256) + .endCell() + .beginParse() as address; +} + +struct(0x12345678) MyBody { + queryId: uint64; +} + +fun test1_manual() { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA")) + .storeCoins(123) // value.grams + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + // MyBody: op + queryId + .storeUint(0x12345678, 32) + .storeUint(800, 64) + .endCell(); +} + +@method_id(101) +fun test1() { + val body: MyBody = { queryId: 800 }; + var b = createMessage({ + body: body, + bounce: true, + dest: address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), + value: 123, + }); + assert(b.hash() == test1_manual().hash(), 101); + return b.hash(); +} + +fun test2_manual() { + return beginCell() + .storeUint(0x10, 6) // no bounce + .storeAddress(address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8")) + .storeCoins(90) // value.grams + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + // body: op + queryId + .storeUint(0x12345678, 32) + .storeUint(127493264572, 64) + .endCell(); +} + +@method_id(102) +fun test2() { + val body: MyBody = { queryId: 127493264572 }; + var b = createMessage({ + body, + bounce: false, + dest: address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), + value: 90, + }); + assert(b.hash() == test2_manual().hash(), 102); + return b.hash(); +} + +fun test3_manual() { + val body_ref = beginCell() + .storeUint(0x12345678, 32) + .storeUint(127493264572, 64) + .endCell(); + return beginCell() + .storeUint(0x10, 6) // no bounce + .storeAddress(address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8")) + .storeCoins(90) // value.grams + .storeUint(1, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 1 body (ref) + .storeRef(body_ref) + .endCell(); +} + +@method_id(103) +fun test3() { + var b = createMessage({ + body: MyBody{ queryId: 127493264572 }.toCell(), + bounce: false, + dest: address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), + value: 90, + }); + assert(b.hash() == test3_manual().hash(), 103); + return b.hash(); +} + +fun test4_manual(bodyCell: cell, dest: address, value: coins) { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(dest) + .storeCoins(value) + .storeUint(1, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 1 body (ref) + .storeRef(bodyCell) + .endCell(); +} + +@method_id(104) +fun test4(value: coins) { + val bodyCell = beginCell() + .storeUint(0x03738FA9, 32) + .storeBool(true).storeBool(false).storeCoins(123) + .endCell(); + val dest = address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"); + var b = createMessage({ + body: bodyCell, + value, + bounce: true, + dest: dest, + }); + assert(b.hash() == test4_manual(bodyCell, dest, value).hash(), 104); + return b.hash(); +} + +fun test5_manual() { + var ec_dict = createEmptyDict(); + ec_dict.iDictSet(32, 1, "ec1"); + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA")) + .storeCoins(123) // value.grams + .storeMaybeRef(ec_dict) // value.extra + .storeUint(0, 0 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + // body: op + queryId + .storeUint(0x12345678, 32) + .storeUint(800, 64) + .endCell(); +} + +@method_id(105) +fun test5() { + var ec_dict = createEmptyDict(); + ec_dict.iDictSet(32, 1, "ec1"); + val body: MyBody = { queryId: 800 }; + var b = createMessage({ + body, + bounce: true, + dest: address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), + value: (123, ec_dict), + }); + assert(b.hash() == test5_manual().hash(), 105); + return b.hash(); +} + +fun test6_manual(value: coins, ec_dict: dict) { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA")) + .storeCoins(value) // value.grams + .storeMaybeRef(ec_dict) // value.extra + .storeUint(0, 0 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + // body: op + queryId + .storeUint(0x12345678, 32) + .storeUint(800, 64) + .endCell(); +} + +@method_id(106) +fun test6(value: coins, dictKey: int?) { + var ec_dict = createEmptyDict(); + if (dictKey != null) { + ec_dict.iDictSet(32, dictKey, "ec1"); + } + val body: MyBody = { queryId: 800 }; + var b = createMessage({ + bounce: true, + dest: address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), + value: (value, ec_dict), + body, + }); + assert(b.hash() == test6_manual(value, ec_dict).hash(), 106); + return b.hash(); +} + +struct MyNftBody { + nftContent: cell; +} + +fun test7_manual(nftItemCode: cell, amount: coins) { + var (itemIndex: int, nftContent: cell) = (10, beginCell().endCell()); + val nftItemData = calculateNftItemStateInitData(itemIndex); + val nftAddress = calculateNftItemAddress(BASECHAIN, calculateStateInitCell(nftItemCode, nftItemData)); + + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(nftAddress) + .storeCoins(amount) + // 1 state init exists, 0 either left (state init embedded), 00110 (code and data), 0 either left (body inline) + .storeUint(0b10001100, 1 + 4 + 4 + 64 + 32 + (1 + 1 + 5 + 1)) + .storeRef(nftItemCode) + .storeRef(nftItemData) + .storeRef(nftContent) + .endCell(); +} + +@method_id(107) +fun test7(amount: coins) { + val nftItemCode: cell = beginCell().storeInt(0x273849723892, 94).endCell(); + var (itemIndex: int, nftContent: cell) = (10, beginCell().endCell()); + val stateInitData = calculateNftItemStateInitData(itemIndex); + + val body: MyNftBody = { nftContent }; + var b = createMessage({ + bounce: true, + body, + dest: { workchain: BASECHAIN, stateInit: { code: nftItemCode, data: stateInitData } }, + value: amount, + }); + assert(b.hash() == test7_manual(nftItemCode, amount).hash(), 107); + return b.hash(); +} + +struct(0x706c7567) RequestPaymentMessage { + queryId: uint64; + amount: coins; + someDict: dict; +} + +fun test8_manual(destAddr: address, requestedAmount: coins) { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(destAddr) + .storeCoins(123) // value.grams + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + .storeUint(0x706c7567, 32) + .storeUint(0, 64) + .storeCoins(requestedAmount) + .storeDict(null) + .endCell(); +} + +@method_id(108) +fun test8(requestedAmount: coins) { + val destAddr = address("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"); + var b = createMessage({ + bounce: true, + dest: destAddr, + value: 123, + body: RequestPaymentMessage { + queryId: 0, + amount: requestedAmount, + someDict: null, + } + }); + assert(b.hash() == test8_manual(destAddr, requestedAmount).hash(), 108); + return b.hash(); +} + +fun test9_manual(bounceable: bool) { + return beginCell() + .storeUint(0b01, 2).storeBool(bounceable).storeUint(0b000, 3) + .storeAddress(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA")) + .storeCoins(0) // value.grams + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + .storeUint(0x12345678, 32) + .storeUint(800, 64) + .storeInt(-777, 13) + .endCell(); +} + +@method_id(109) +fun test9(bounceable: bool) { + var b = createMessage({ + bounce: bounceable, + dest: address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), + value: 0, + body: (0x12345678 as uint32, 800 as uint64, -777 as int13), + }); + assert(b.hash() == test9_manual(bounceable).hash(), 109); + return b.hash(); +} + +type Body500Bits = (uint250, uint250); // this body guaranteely fits into cell + +fun test10_manual(bd: Body500Bits) { + return beginCell() + .storeUint(0x10, 6) + .storeAddress(address("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6")) + .storeCoins(1 << 118) + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + .storeUint(bd.0, 250) + .storeUint(bd.1, 250) + .endCell(); +} + +@method_id(110) +fun test10(bd: Body500Bits) { + var b = createMessage({ + bounce: false, + body: bd, + dest: address("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"), + value: 1 << 118, + }); + assert(b.hash() == test10_manual(bd).hash(), 110); + return b.hash(); +} + +type Body750Bits = (uint250, uint250, uint250); // this body is auto-ref + +fun test11_manual(bd: Body750Bits) { + val bodyRef = beginCell().storeUint(bd.0,250).storeUint(bd.1,250).storeUint(bd.2,250).endCell(); + return beginCell() + .storeUint(0x10, 6) + .storeAddress(address("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6")) + .storeCoins(1 << 110) + .storeUint(1, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 1 body (ref) + .storeRef(bodyRef) + .endCell(); +} + +@method_id(111) +fun test11(bd: Body750Bits) { + var b = createMessage({ + bounce: false, + body: bd, + dest: address("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"), + value: 1 << 110, + }); + assert(b.hash() == test11_manual(bd).hash(), 111); + return b.hash(); +} + +fun test12_manual(data32: uint32) { + var init = ContractState { + data: beginCell().storeUint(data32,32).endCell(), + code: beginCell().endCell(), + }; + + return beginCell() + .storeUint(0x18, 6) // bounce + .storeUint(0b100, 3).storeInt(MASTERCHAIN, 8).storeUint(StateInit.calcHashCodeData(init.code, init.data), 256) + .storeCoins(ton("0.05")) + // 1 state init exists, 0 either left (state init embedded), 00110 (code and data), 0 either left (body inline) + .storeUint(0b10001100, 1 + 4 + 4 + 64 + 32 + (1 + 1 + 5 + 1)) + .storeRef(init.code) + .storeRef(init.data) + .storeUint(data32, 32) + .endCell(); +} + +@method_id(112) +fun test12(data32: uint32) { + var init = ContractState { + data: beginCell().storeUint(data32,32).endCell(), + code: beginCell().endCell(), + }; + + var b = createMessage({ + bounce: true, + body: data32, + dest: { workchain: MASTERCHAIN, stateInit: init }, + value: ton("0.05"), + }); + assert(b.hash() == test12_manual(data32).hash(), 112); + return b.hash(); +} + +fun test13_manual() { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(address("EQDE0HBgfkOiqHezLtExBGTvOs8eitthHQosBjW3BmDy1y2K")) + .storeCoins(52 << 78) // value.grams + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + .endCell(); // no body at all +} + +@method_id(113) +fun test13() { + var b = createMessage({ + bounce: true, + dest: address("EQDE0HBgfkOiqHezLtExBGTvOs8eitthHQosBjW3BmDy1y2K"), + value: 52 << 78, + }); + assert(b.hash() == test13_manual().hash(), 113); + return b.hash(); +} + +fun test14_manual(stateInitCell: cell, amount: coins) { + val nftAddress = calculateNftItemAddress(BASECHAIN, stateInitCell); + + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(nftAddress) + .storeCoins(amount) + // 1 state init exists, 1 either right (state init ref), 0 either left (body inline) + .storeUint(0b110, 1 + 4 + 4 + 64 + 32 + (1 + 1 + 1)) + .storeRef(stateInitCell) + .endCell(); // no body +} + +@method_id(114) +fun test14(amount: coins) { + val stateInitCode = beginCell().storeInt(0x273849723892, 94).endCell(); + val stateInitData = calculateNftItemStateInitData(10); + val stateInitCell = calculateStateInitCell(stateInitCode, stateInitData); + + var b = createMessage({ + bounce: true, + dest: { stateInit: stateInitCell }, + value: amount, + }); + assert(b.hash() == test14_manual(stateInitCell, amount).hash(), 114); + return b.hash(); +} + +struct(0x1234) Body15 { + tens: Body750Bits; + more: int32; +} + +fun test15_manual(stateInitCell: cell, bd: Body15) { + val bodyRef = beginCell().storeUint(0x1234,16).storeUint(bd.tens.0,250).storeUint(bd.tens.1,250).storeUint(bd.tens.2,250).storeInt(bd.more,32).endCell(); + val nftAddress = calculateNftItemAddress(MASTERCHAIN, stateInitCell); + + return beginCell() + .storeUint(0x10, 6) // no bounce + .storeAddress(nftAddress) + .storeCoins(ton("100.0004")) + // 1 state init exists, 1 either right (state init ref), 1 either right (body ref) + .storeUint(0b111, 1 + 4 + 4 + 64 + 32 + (1 + 1 + 1)) + .storeRef(stateInitCell) + .storeRef(bodyRef) + .endCell(); // no body +} + +@method_id(115) +fun test15(tens0: int, tens1: int) { + val stateInitCode = beginCell().storeInt(0x273849723892, 94).endCell(); + val stateInitData = calculateNftItemStateInitData(10); + val stateInitCell = beginCell() + .storeUint(1, 1) // has split_depth + .storeUint(3, 5) // split_depth + .storeUint(0, 1) // 0 special + .storeDict(stateInitCode) + .storeDict(stateInitData) + .storeUint(0, 1) // 0 library + .endCell(); + var bd: Body15 = { + tens: (1 << 88, tens0, tens1), + more: max(tens0, tens1), + }; + + var b = createMessage({ + bounce: false, + dest: { workchain: MASTERCHAIN, stateInit: stateInitCell }, + body: bd, + value: ton("100.0004"), + }); + assert(b.hash() == test15_manual(stateInitCell, bd).hash(), 115); + return b.hash(); +} + +fun test18_manual(dest: builder, queryId: uint64) { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeBuilder(dest) + .storeCoins(123) // value.grams + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + // MyBody: op + queryId + .storeUint(0x12345678, 32) + .storeUint(queryId, 64) + .endCell(); +} + +@method_id(118) +fun test18(queryId: uint64) { + val body: MyBody = { queryId }; + var b = createMessage({ + body: body, + bounce: true, + dest: beginCell().storeUint(0, 2), + value: 123, + }); + assert(b.hash() == test18_manual(beginCell().storeUint(0, 2), queryId).hash(), 118); + return b.hash(); +} + +fun test19_manual(bd: uint64) { + var destB = beginCell().storeUint(0, 2); + return beginCell() + .storeUint(0x10, 6) // no bounce + .storeBuilder(destB) + .storeCoins(ton("0.059")) + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + .storeUint(bd, 64) + .endCell(); +} + +@method_id(119) +fun test19(bd: uint64) { + var destB = beginCell().storeUint(0, 2); + var b = createMessage({ + bounce: 10 < 3, + body: bd, + dest: destB, + value: ton("0.059"), + }); + assert(b.hash() == test19_manual(bd).hash(), 119); + return b.hash(); +} + +fun test20_manual() { + var bodyB = beginCell().storeUint(0x12345678, 32).storeUint(800, 64); + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA")) + .storeCoins(123) // value.grams + .storeUint(1, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 1 body (ref) + .storeRef(bodyB.endCell()) + .endCell(); +} + +@method_id(120) +fun test20() { + var bodyB = beginCell().storeUint(0x12345678, 32).storeUint(800, 64); + var b = createMessage({ + body: bodyB, + bounce: true, + dest: address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), + value: 123, + }); + assert(b.hash() == test20_manual().hash(), 120); + return b.hash(); +} + +struct(0x12345678) Body21 { + queryId: uint64; + payload: RemainingBitsAndRefs; +} + +fun test21_manual(bd: Body21) { + return beginCell() + .storeUint(0x10, 6) // no bounce + .storeAddress(address("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi")) + .storeCoins(ton("0.1")) // value.grams + .storeDict(createEmptyDict()) + .storeUint(1, 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 1 body (ref) + .storeRef(bd.toCell()) // stored as ref due to remainder of unpredictable size + .endCell(); +} + +@method_id(121) +fun test21(queryId: uint64) { + var payloadSlice = beginCell().storeUint(0x12345678, 32).storeUint(800, 64).storeRef(createEmptyCell()).endCell().beginParse(); + var body: Body21 = { queryId, payload: payloadSlice }; + var b = createMessage({ + bounce: false, + dest: address("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi"), + value: (ton("0.1"), createEmptyDict()), + body, + }); + assert(b.hash() == test21_manual(body).hash(), 121); + return b.hash(); +} + +fun test22_manual(unsafeB: builder) { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff")) + .storeCoins(ton("0.08")) + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (no ref) + .storeBuilder(unsafeB) + .endCell(); +} + +@method_id(122) +fun test22(op: uint32, queryId: uint64) { + var bodyB = beginCell().storeUint(op, 32).storeUint(queryId, 64); + var b = createMessage({ + bounce: true, + dest: address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"), + value: ton("0.08"), + body: UnsafeBodyNoRef { + bodyForceNoRef: bodyB + }, + }); + assert(b.hash() == test22_manual(bodyB).hash(), 122); + return b.hash(); +} + +struct Body23Unlimited { + any1: (address, builder), + any2: RemainingBitsAndRefs; +} + +fun test23_manual(state32: uint32, body: Body23Unlimited) { + var init = ContractState { + data: beginCell().storeUint(state32,32).endCell(), + code: beginCell().endCell(), + }; + + return beginCell() + .storeUint(0x18, 6) // bounce + .storeUint(0b100, 3).storeInt(9, 8).storeUint(StateInit.calcHashCodeData(init.code, init.data), 256) + .storeCoins(ton("0.10009")) + // 1 state init exists, 0 either left (state init embedded), 00110 (code and data), 0 either left (body inline) + .storeUint(0b10001100, 1 + 4 + 4 + 64 + 32 + (1 + 1 + 5 + 1)) + .storeRef(init.code) + .storeRef(init.data) + .storeAddress(body.any1.0) + .storeBuilder(body.any1.1) + .storeSlice(body.any2) + .endCell(); +} + +@method_id(123) +fun test23(state32: uint32) { + var init = ContractState { + data: beginCell().storeUint(state32,32).endCell(), + code: beginCell().endCell(), + }; + var bd: Body23Unlimited = { + any1: (createAddressNone(), beginCell().storeRef(init.code)), + any2: beginCell().storeUint(80, 80).endCell().beginParse(), + }; + + var b = createMessage({ + bounce: true, + body: UnsafeBodyNoRef { + bodyForceNoRef: bd, + }, + dest: { workchain: 9, stateInit: init }, + value: ton("0.10009"), + }); + assert(b.hash() == test23_manual(state32, bd).hash(), 123); + return b.hash(); +} + +fun test24_manual(addrHash: uint256) { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeUint(0b100, 3).storeInt(MASTERCHAIN, 8).storeUint(addrHash, 256) + .storeCoins(ton("0.6")) + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + .storeUint(0x12345678, 32) + .storeUint(800, 64) + .endCell(); +} + +@method_id(124) +fun test24() { + val addrHash = address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA").getWorkchainAndHash().1; + var b = createMessage({ + body: MyBody { queryId: 800 }, + bounce: true, + dest: (MASTERCHAIN, addrHash), + value: ton("0.6"), + }); + assert(b.hash() == test24_manual(addrHash).hash(), 124); + return b.hash(); +} + +fun main() { +} + +/** +@testcase | 101 | | 104134380371273907780196393444164225714229635235007677971195651971972203592811 +@testcase | 102 | | 20192939504955637835708665496574868659039935190188156593169026529135727309085 +@testcase | 103 | | 97654287980681401436727082042373998183264988661375688119718809500301413968039 +@testcase | 104 | 600000 | 755636693689039391990782995030008885663781699175339033402019272057515711062 +@testcase | 105 | | 24816341499673835567887890014278904436471582322802948121781418622643959482495 +@testcase | 106 | 123 null | 104134380371273907780196393444164225714229635235007677971195651971972203592811 +@testcase | 106 | 456 8888 | 39337586036945718311402746340438400160817844833530971545330721291986281100430 +@testcase | 107 | 1000 | 55093441331748625324828489600632232039914212774002148634088483962817636598198 +@testcase | 108 | 50000000 | 95023796475113775225029817428715936488418545169963429399979521091689824066088 +@testcase | 109 | 0 | 55999621586681214992294941423256376619779969729861696464321825639854258502733 +@testcase | 109 | -1 | 84087871798432599249441213206223143701565541307347047545146076475041341315422 +@testcase | 110 | 250 250 | 97468400996544929599099493087921300963923138788231489050737873840992619823773 +@testcase | 111 | 1 2 3 | 35151166799433266221446406287469019610025742512503320058804207122452431754243 +@testcase | 112 | 32 | 40256061350602595831367445407067573081836468277788226383346273736379122699330 +@testcase | 113 | | 8212062468046185391185852622213582155366804215840270337189205672457136520017 +@testcase | 114 | 100500 | 69207815800109735757177433421533576767812185821226447066044060358661780329301 +@testcase | 115 | 66 77 | 77952119695754644819736002369288963466111166174255607433142955344110346202253 +@testcase | 118 | 888 | 75462675913935779917745192355822465171309245151518256862408373999119088535160 +@testcase | 119 | 999 | 20451206881650273118327988889219529836875916996856547550069532964562229905067 +@testcase | 120 | | 86341416901030824925289599533989709413619468614523233983159539599569269903295 +@testcase | 121 | 2983742 | 29682465216061902145511914581895871811826194319877849112468723187093436476183 +@testcase | 122 | 88 12892 | 104095783372529353117379287284649790470404208891518326338200063605256169961669 +@testcase | 123 | 999 | 93691386126134034953606952841435771108648354412078750046821446829279390964584 +@testcase | 124 | | 21886688052816798288463190773103865772534937765373272039597499398551023701577 + */ diff --git a/tolk-tester/tests/send-msg-2.tolk b/tolk-tester/tests/send-msg-2.tolk new file mode 100644 index 0000000000..b1e3af99cb --- /dev/null +++ b/tolk-tester/tests/send-msg-2.tolk @@ -0,0 +1,337 @@ +/* +int_msg_info$0 1 + ihr_disabled:Bool // always 0, not implemented 1 + bounce:Bool // parameter 1 + bounced:Bool // always 0 on send 1 + src:MsgAddress // always 00 on send 2 + dest:MsgAddressInt // parameter 267 + value:CurrencyCollection // parameter 124 + 1 + ihr_fee:Grams // always 0, not implemented 4 + fwd_fee:Grams // always 0 on send 4 + created_lt:uint64 // always 0 on send 64 + created_at:uint32 // always 0 on send 32 + = CommonMsgInfoRelaxed; +_ split_depth:(Maybe (## 5)) 1 + 5 + special:(Maybe TickTock) 1 + 2 + code:(Maybe ^Cell) 1 + data:(Maybe ^Cell) 1 + library:(Maybe ^Cell) 1 + = StateInit; +message$_ {X:Type} + info:CommonMsgInfoRelaxed 502 + init:(Maybe (Either StateInit ^StateInit)) 12 + body:(Either X ^X) // body is either embedded or stored as ref + = MessageRelaxed X; + */ + +const SHARD_DEPTH = 8; + +fun getMyAddressDev(): address + asm "x{80194DC6438F99D3D9DBE151944925D90B2492954BF6B9C070FBFF2DDED5F30547D_} PUSHSLICE"; + +@inline +fun getAddressShard(address: address, shardLen: int): int { + // Skip workchain, load shard prefix + (address as slice).skipBits(3 + 8); + return (address as slice).loadUint(shardLen); +} + +@inline +fun packJettonWalletData(status: int, balance: int, ownerAddress: address, jettonMasterAddress: address): cell { + return beginCell() + .storeUint(status, 8) + .storeCoins(balance) + .storeAddress(ownerAddress) + .storeAddress(jettonMasterAddress) + .endCell(); +} + +@inline +fun calculateJettonWalletStateInitWithShard(ownerAddress: address, jettonMasterAddress: address, jettonWalletCode: cell): cell { + /* + https://github.com/ton-blockchain/ton/blob/8a9ff339927b22b72819c5125428b70c406da631/crypto/block/block.tlb#L144 + _ split_depth:(Maybe (## 5)) special:(Maybe TickTock) + code:(Maybe ^Cell) data:(Maybe ^Cell) + library:(Maybe ^Cell) = StateInit; + */ + return beginCell() + .storeUint(1, 1) + .storeUint(SHARD_DEPTH, 5) + .storeUint(0, 1) + .storeMaybeRef(jettonWalletCode) + .storeMaybeRef( + packJettonWalletData( + 0, // status + 0, // balance + ownerAddress, + jettonMasterAddress) + ) + .storeUint(0, 1) // Empty libraries + .endCell(); +} + +@inline +fun calculateJettonWalletAddress(shardPrefix: int, stateInitCell: cell): address { + var mask = (1 << (256 - SHARD_DEPTH)) - 1; + var prefixLess = stateInitCell.hash() & mask; + return beginCell() + .storeUint(4, 3) // addr_std$10 + anycast 0 + .storeInt(BASECHAIN, 8) + .storeUint(shardPrefix, SHARD_DEPTH) + .storeUint(prefixLess, 256 - SHARD_DEPTH) + .endCell() + .beginParse() as address; +} + +@inline +fun calculateAddressInAnotherShard(pivotAddress: address, shardPrefixLen: uint5, code: cell, data: cell): builder { + val stateInitCell = beginCell() + .storeUint(1, 1) + .storeUint(shardPrefixLen, 5) // shard depth + .storeUint(0, 1) + .storeMaybeRef(code) + .storeMaybeRef(data) + .storeUint(0, 1) // Empty libraries + .endCell(); + + val shardPrefix = getAddressShard(pivotAddress, shardPrefixLen); + var mask = (1 << (256 - shardPrefixLen)) - 1; + var prefixLess = stateInitCell.hash() & mask; + return beginCell() + .storeUint(4, 3) // addr_std$10 + anycast 0 + .storeInt(BASECHAIN, 8) + .storeUint(shardPrefix, shardPrefixLen) + .storeUint(prefixLess, 256 - shardPrefixLen); +} + +fun test1_manual(toAddress: address, masterMsg: cell, jettonWalletCode: cell) { + var shardPrefix = getAddressShard(toAddress, SHARD_DEPTH); + var stateInitCell = calculateJettonWalletStateInitWithShard(toAddress, getMyAddressDev(), jettonWalletCode); + var toWalletAddress = calculateJettonWalletAddress(shardPrefix, stateInitCell); + var jettonWalletData = packJettonWalletData(0, 0, toAddress, getMyAddressDev()); + + return beginCell() + .storeUint(0x18, 6) // bounceable + .storeAddress(toWalletAddress) // dest + .storeCoins(ton("0.05")) + // 1 state init exists, 0 either left (state init embedded), 1 either left (fixed_prefix_length exists) + .storeUint(0b101, 1 + 4 + 4 + 64 + 32 + 1 + 1 + 1) + .storeUint(SHARD_DEPTH, 5) + .storeUint(0b01101, 4 + 1) // 0 ticktock + 1 code + 1 data + 0 library + 1 body ref + .storeRef(jettonWalletCode) + .storeRef(jettonWalletData) + .storeRef(masterMsg) + .endCell(); +} + +@method_id(101) +fun test1() { + val toAddress = address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"); + val masterMsg = beginCell().storeUint(1<<14, 32).endCell(); + val jettonWalletCode = beginCell().storeInt(0x273849723892, 94).endCell(); + + var b = createMessage({ + bounce: true, + dest: { + stateInit: { + code: jettonWalletCode, + data: packJettonWalletData(0, 0, toAddress, getMyAddressDev()), + }, + toShard: { + fixedPrefixLength: SHARD_DEPTH, + closeTo: toAddress, + } + }, + body: masterMsg, + value: ton("0.05"), + }); + assert(b.hash() == test1_manual(toAddress, masterMsg, jettonWalletCode).hash(), 101); + return b.hash() +} + +fun test2_manual(toAddress: address, masterMsg: cell, jettonWalletCode: cell) { + var shardPrefix = getAddressShard(toAddress, SHARD_DEPTH); + var stateInit = calculateJettonWalletStateInitWithShard(toAddress, getMyAddressDev(), jettonWalletCode); + var toWalletAddress = calculateJettonWalletAddress(shardPrefix, stateInit); + + return beginCell() + .storeUint(0x10, 6) // not bounceable + .storeAddress(toWalletAddress) // dest + .storeCoins(ton("0.05")) + .storeUint(0b111, 1 + 4 + 4 + 64 + 32 + 1 + 1 + 1) // 1 state init exists + 1 state init ref + 1 body ref + .storeRef(stateInit) + .storeRef(masterMsg) + .endCell(); +} + +@method_id(102) +fun test2() { + val toAddress = address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"); + val masterMsg = beginCell().endCell(); + val jettonWalletCode = beginCell().storeInt(0x273849723892, 94).endCell(); + var stateInitCell = calculateJettonWalletStateInitWithShard(toAddress, getMyAddressDev(), jettonWalletCode); + + var b = createMessage({ + bounce: false, + dest: { + workchain: BASECHAIN, + stateInit: stateInitCell, + toShard: { + fixedPrefixLength: SHARD_DEPTH, + closeTo: toAddress, + } + }, + body: masterMsg, + value: ton("0.05"), + }); + assert(b.hash() == test2_manual(toAddress, masterMsg, jettonWalletCode).hash(), 102); + return b.hash() +} + +struct Test3Body { + bigData: bits800; +} + +fun test3_manual(myCode: cell, myData: cell, msgBody: Test3Body) { + var addrInShard = calculateAddressInAnotherShard(getMyAddressDev(), 20, myCode, myData); + + return beginCell() + .storeUint(0x18, 6) // bounceable + .storeBuilder(addrInShard) // dest + .storeCoins(ton("0.001")) + .storeUint(0b10, 1 + 4 + 4 + 64 + 32 + 1 + 1) // 1 state init exists + 1 state init inline + .storeUint(1, 1) + .storeUint(20, 5) // shard depth + .storeUint(0b0110, 4) // code + data exist + .storeUint(1, 1) // body ref + .storeRef(myCode) + .storeRef(myData) + .storeRef(msgBody.toCell()) + .endCell(); +} + +@method_id(103) +fun test3() { + val myCode = beginCell().storeInt(0x273849723892, 94).endCell(); + val myData = beginCell().storeInt(0x273849723892, 200).storeMaybeRef(myCode).endCell(); + val builder800 = beginCell().storeInt(1, 250).storeInt(1, 250).storeInt(1, 250).storeInt(1, 50); + val body: Test3Body = { bigData: builder800.endCell().beginParse() as bits800 }; + + var b = createMessage({ + bounce: true, + dest: { + stateInit: { + code: myCode, + data: myData, + }, + toShard: { + fixedPrefixLength: 20, + closeTo: getMyAddressDev(), + } + }, + body, + value: ton("0.001"), + }); + assert(b.hash() == test3_manual(myCode, myData, body).hash(), 103); + return b.hash() +} + +struct Test4Body { + f1: int64; + f2: int64; +} + +@method_id(104) +fun test4() { + val myCode = beginCell().storeInt(0x273849723892, 94).endCell(); + val myData = beginCell().storeInt(0x273849723892, 200).endCell(); + val body: Test4Body = { f1: 123, f2: 456 }; + + var b = createMessage({ + bounce: true, + dest: { + stateInit: { + code: myCode, + data: myData, + }, + toShard: { + fixedPrefixLength: 30, + closeTo: getMyAddressDev(), + } + }, + body, + value: ton("0.1"), + }); + return b.hash() +} + +@overflow1023_policy("suppress") +struct Test5Body { + c1: coins; c2: coins; c3: coins; + c4: coins; c5: coins; c6: coins; + c7: coins; c8: coins; c9: coins; +} + +@method_id(105) +fun test5() { + val pivot = address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); + val myCode = beginCell().storeInt(0x273849723892, 94).endCell(); + val myData = beginCell().storeInt(0x273849723892, 200).endCell(); + val body: Test5Body = { c1: 1, c2: 12, c3: 13, c4: 41, c5: 15, c6: 66, c7: 777, c8: 8888, c9: 0x9999 }; + + var b = createMessage({ + bounce: true, + dest: { + stateInit: { + code: myCode, + data: myData, + }, + toShard: { + fixedPrefixLength: 2, + closeTo: pivot, + } + }, + body: UnsafeBodyNoRef { bodyForceNoRef: body }, + value: ton("0.1"), + }); + return b.hash() +} + +@method_id(106) +fun test6(shardLen: int) { + val pivot = address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); + val myCode = beginCell().storeInt(0x273849723892, 94).endCell(); + val myData = beginCell().storeInt(0x273849723892, 200).endCell(); + val body: Test5Body = { c1: 1, c2: 12, c3: 13, c4: 41, c5: 15, c6: 66, c7: 777, c8: 8888, c9: 0x9999 }; + + var b = createMessage({ + bounce: true, + dest: { + workchain: BASECHAIN, + stateInit: { + code: myCode, + data: myData, + }, + toShard: { + fixedPrefixLength: shardLen, + closeTo: pivot, + } + }, + body: body, + value: ton("0.1"), + }); + return b.hash() +} + +fun main() {} + + +/** +@testcase | 101 | | 114692697425779687591180352274328586758984823673144676907237399062035787598876 +@testcase | 102 | | 19141316219057792078125032524380927286847294424548966584483011017258286475706 +@testcase | 103 | | 33184074158306258754479533493449661031201063538320283251640171344508567641003 +@testcase | 104 | | 52060388210984960140400107426752119783720473036191582802152450095687953630545 +@testcase | 105 | | 18248582963085351579261363895401647616194364201057112283341027605548475235397 +@testcase | 106 | 2 | 73367500160949078368695742399322272058735198813159066885436981754975296689213 +@testcase | 106 | 9 | 26260034723231226694451786564335585031223880314377072827334911071856417531179 + */ diff --git a/tolk-tester/tests/send-msg-3.tolk b/tolk-tester/tests/send-msg-3.tolk new file mode 100644 index 0000000000..69afed4cd5 --- /dev/null +++ b/tolk-tester/tests/send-msg-3.tolk @@ -0,0 +1,233 @@ +/* +ext_out_msg_info$11 + src:MsgAddress // 00 on send + dest:MsgAddressExt // parameter + created_lt:uint64 // 0 on send + created_at:uint32 // 0 on send + = CommonMsgInfoRelaxed; +message$_ {X:Type} + info:CommonMsgInfoRelaxed + init:(Maybe (Either StateInit ^StateInit)) // 0 on send + body:(Either X ^X) // body is either embedded or stored as ref + = MessageRelaxed X; +*/ + +struct(0x12345678) MyBody { + queryId: uint64; +} + +fun test1_manual(topic: int) { + return beginCell() + .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () + .storeUint(1, 2) // addr_extern$01 + .storeUint(256, 9) // len:(## 9) + .storeUint(topic, 256) // external_address:(bits len) (assume it fits 248 bits) + .storeUint(0, 64 + 32 + 2) // created_lt, created_at, init:Maybe (0), body:Either left (inline) + // MyBody: op + queryId + .storeUint(0x12345678, 32) + .storeUint(800, 64) + .endCell(); +} + +@method_id(101) +fun test1(topic: int) { + val body: MyBody = { queryId: 800 }; + var b = createExternalLogMessage({ + body: body, + dest: ExtOutLogBucket { topic }, + }); + assert(b.hash() == test1_manual(topic).hash(), 101); + return b.hash(); +} + +type Body750Bits = (uint250, uint250, uint250); // this body is auto-ref + +fun test2_manual(bd: Body750Bits) { + val bodyRef = beginCell().storeUint(bd.0,250).storeUint(bd.1,250).storeUint(bd.2,250).endCell(); + return beginCell() + .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () + .storeUint(1, 2) // addr_extern$01 + .storeUint(256, 9) // len:(## 9) + .storeUint(1 << 102, 256) // external_address:(bits len) + .storeUint(1, 64 + 32 + 2) // created_lt, created_at, init:Maybe (0), body:Either right (ref) + .storeRef(bodyRef) + .endCell(); +} + +@method_id(102) +fun test2(bd: Body750Bits) { + var b = createExternalLogMessage({ + body: bd, + dest: { topic: 1 << 102 }, + }); + assert(b.hash() == test2_manual(bd).hash(), 102); + return b.hash(); +} + +struct Body3 { + bodyAddress: address; + amount: coins; + someRef: cell; +} + +@method_id(103) +fun test3(eventId: int) { + val bodyAddress = address("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"); + val someRef = MyBody { queryId: 800 }.toCell(); + var b = createExternalLogMessage({ + body: Body3 { bodyAddress, someRef, amount: ton("0.05") }, + dest: { topic: eventId }, + }); + return b.hash(); +} + +fun test4_manual(dest: address, body: cell) { + return beginCell() + .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () + .storeAddress(dest) // dest (manually created external address) + .storeUint(1, 64 + 32 + 2) // created_lt, created_at, init:Maybe (0), body:Either right (ref) + .storeRef(body) + .endCell(); +} + +@method_id(104) +fun test4() { + val body: cell = Body3 { bodyAddress: createAddressNone(), amount: 0, someRef: createEmptyCell() }.toCell(); + val dest = beginCell().storeUint(0b01, 2).storeUint(12, 9).storeUint(88, 12).endCell() + .beginParse() as address; + var b = createExternalLogMessage({ + body, + dest, + }); + assert(b.hash() == test4_manual(dest, body).hash(), 104); + return b.hash(); +} + +@method_id(105) +fun test5(topic: int) { + var b = createExternalLogMessage({ + dest: { topic }, + }); + return b.hash(); +} + +@method_id(106) +fun test6() { + val dest = beginCell().storeUint(0b01, 2).storeUint(12, 9).storeUint(88, 12).endCell() + .beginParse() as address; + var b = createExternalLogMessage({ + dest, + }); + return b.hash(); +} + +fun test7_manual(dest: builder, queryId: uint64) { + return beginCell() + .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () + .storeBuilder(dest) // manually created bits representing external address + .storeUint(0, 64 + 32 + 2) // created_lt, created_at, init:Maybe (0), body:Either left (inline) + // MyBody: op + queryId + .storeUint(0x12345678, 32) + .storeUint(queryId, 64) + .endCell(); +} + +@method_id(107) +fun test7(queryId: uint64) { + val body: MyBody = { queryId }; + val dest = beginCell().storeUint(0b01, 2).storeUint(12, 9).storeUint(88, 12); + var b = createExternalLogMessage({ + body, + dest, + }); + assert(b.hash() == test7_manual(dest, queryId).hash(), 107); + return b.hash(); +} + +@method_id(108) +fun test8(topic: int) { + var bodyB = beginCell().storeUint(0x12345678, 32).storeUint(800, 64); + var b = createExternalLogMessage({ + body: bodyB, + dest: { topic }, + }); + return b.hash(); +} + +struct Body9Unlimited { + any1: cell?, + any2: builder, +} + +@method_id(109) +fun test9(topic: int) { + var bd: Body9Unlimited = { + any1: null, + any2: beginCell().storeUint(80, 80).storeMaybeRef(null).storeRef(createEmptyCell()), + }; + var b = createExternalLogMessage({ + body: UnsafeBodyNoRef { + bodyForceNoRef: bd, + }, + dest: ExtOutLogBucket { topic: topic }, + }); + return b.hash(); +} + +fun test10_manual(topic: slice, queryIdRef: uint64) { + return beginCell() + .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () + .storeUint(1, 2) // addr_extern$01 + .storeUint(256, 9) // len:(## 9) + .storeUint(0x00, 8) // prefix of ExtOutLogBucket + .storeSlice(topic) // assume this slice is bits248 + .storeUint(0, 64 + 32 + 2) // created_lt, created_at, init:Maybe (0), body:Either left (inline) + .storeRef(beginCell().storeUint(queryIdRef, 64).endCell()) + .endCell(); +} + +struct Body10 { + queryIdRef: Cell; +} + +@method_id(110) +fun test10() { + val queryIdRef = 123; + val topic = stringHexToSlice("012345678901234567890123456789012345678901234567890123456789FF"); + var b = createExternalLogMessage({ + body: Body10 { queryIdRef: (queryIdRef as uint64).toCell() }, + dest: ExtOutLogBucket { topic: topic as bits248 }, + }); + assert(b.hash() == test10_manual(topic, queryIdRef).hash(), 110); + return b.hash(); +} + +@method_id(111) +fun test11() { + try { + createExternalLogMessage({ + dest: ExtOutLogBucket { topic: "asdf" as bits248 } // wrong bits count + }); + return 0; + } catch (excno) { + return excno; + } +} + +fun main() { + +} + +/** +@testcase | 101 | 100500 | 8399954633429803564068357060778961153860921657197909539205508789211284886940 +@testcase | 102 | 1 2 3 | 64371641017422166665232911624005896033238822098768040861805756890339950064655 +@testcase | 103 | 829999 | 33039908261421267188072304726227928987040300641614103065684678216731595546507 +@testcase | 104 | | 94681566274361007141065727833953679134419126606639224444485202401996571399154 +@testcase | 105 | 329889 | 24202801754361933063453004029355483858376498707977868951519201997233775688005 +@testcase | 106 | | 17838299862930869444234264325786950440464929864630237676886309244018701054708 +@testcase | 107 | 777 | 77493748753370371314291958955179535305777482888906702570602071501642726442389 +@testcase | 108 | 0 | 69287572703823759594480986943855745035158339431740068559880301501812484359455 +@testcase | 109 | 12 | 72694183679327534355706107347199645597776533707531061585024123910830113048519 +@testcase | 110 | | 33186381366677843958628736306577552541950719844529663083708985644217867888324 +@testcase | 111 | | 9 + */ diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 106d93840b..7790953022 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -733,16 +733,16 @@ fun main(x: int?): int { @testcase | 139 | | 16 @testcase | 140 | 5 | 25 @testcase | 141 | | 1 2 -@testcase | 142 | | 5 3 (null) (null) 0 133 3 (null) (null) 0 +@testcase | 142 | | 5 3 (null) (null) 0 typeid-5 3 (null) (null) 0 @testcase | 143 | | 10 11 (null) 10 11 (null) (null) 0 @testcase | 144 | | 10 11 (null) 10 11 (null) (null) 0 -@testcase | 145 | | 5 3 (null) (null) 0 133 3 (null) (null) (null) (null) 0 3 (null) (null) 0 -@testcase | 146 | | 3 4 5 3 4 5 132 +@testcase | 145 | | 5 3 (null) (null) 0 typeid-5 3 (null) (null) (null) (null) 0 3 (null) (null) 0 +@testcase | 146 | | 3 4 5 3 4 5 typeid-4 @testcase | 147 | | (null) (null) 100 (null) 100 (null) (null) 0 @testcase | 158 | | 123 10 123 5 @testcase | 160 | | 101 109 @testcase | 161 | 9 9 | (null) (null) 0 (null) (null) -@testcase | 161 | 19 0 | 19 0 129 19 0 +@testcase | 161 | 19 0 | 19 0 typeid-1 19 0 @stderr warning: expression of type `int` can never be `null`, this condition is always true @stderr warning: unreachable code diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 46da0207c0..c7f81e4c3f 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -581,37 +581,37 @@ type PointAlias = Point; @testcase | 105 | | 35 30 45 @testcase | 106 | | 10 20 @testcase | 107 | | 5 5 10 20 15 -@testcase | 108 | | 777 131 777 0 777 777 777 131 777 132 777 777 +@testcase | 108 | | 777 typeid-3 777 0 777 777 777 typeid-3 777 typeid-4 777 777 @testcase | 109 | | 70 30 20 20 -80 @testcase | 110 | 0 | (null) (null) 0 -@testcase | 110 | -1 | 10 20 133 +@testcase | 110 | -1 | 10 20 typeid-5 @testcase | 111 | 0 | 0 2 10 20 30 @testcase | 111 | -1 | 0 0 3 4 7 @testcase | 112 | 0 | (null) (null) 0 -@testcase | 112 | -1 | 1 2 133 +@testcase | 112 | -1 | 1 2 typeid-5 @testcase | 113 | | (null) (null) 0 @testcase | 114 | | 1 2 @testcase | 115 | | (null) (null) (null) (null) 0 -@testcase | 116 | | -1 -4 [ 8 4 ] 7 (null) 0 [ 10 11 ] 135 +@testcase | 116 | | -1 -4 [ 8 4 ] 7 (null) 0 [ 10 11 ] typeid-7 @testcase | 117 | 5 | 5 5 5 5 5 @testcase | 117 | null | (null) (null) (null) -1 (null) @testcase | 118 | | 17 -@testcase | 119 | | 10 20 9 10 133 9 -@testcase | 120 | 0 | 80 9 8 80 133 +@testcase | 119 | | 10 20 9 10 typeid-5 9 +@testcase | 120 | 0 | 80 9 8 80 typeid-5 @testcase | 120 | -1 | 8 14 (null) (null) 0 @testcase | 121 | 0 | 17 17 8 136 @testcase | 121 | -1 | 8 (null) 8 8 @testcase | 122 | 100 | 101 50 106 212 100 101 101 @testcase | 124 | 66 | 66 -@testcase | 125 | | (null) (null) 40 60 133 +@testcase | 125 | | (null) (null) 40 60 typeid-5 @testcase | 126 | | 118 -@testcase | 127 | | 0 0 131 131 777 -1 -1 0 0 777 0 0 138 777 0 0 777 (null) (null) 0 -@testcase | 128 | -1 | (null) (null) 131 -1 0 0 777 (null) (null) 131 -1 0 0 -@testcase | 128 | 0 | 1 2 130 0 -1 0 777 0 131 138 0 -1 0 +@testcase | 127 | | 0 0 typeid-3 typeid-3 777 -1 -1 0 0 777 0 0 typeid-10 777 0 0 777 (null) (null) 0 +@testcase | 128 | -1 | (null) (null) typeid-3 -1 0 0 777 (null) (null) typeid-3 -1 0 0 +@testcase | 128 | 0 | 1 2 typeid-2 0 -1 0 777 0 typeid-3 typeid-10 0 -1 0 @testcase | 129 | | (null) -@testcase | 130 | -1 | 140 777 (null) 140 777 0 777 0 0 -@testcase | 130 | 0 | 139 777 4 1 777 139 777 0 0 -@testcase | 131 | | 5 141 (null) 141 (null) 0 777 0 0 -1 777 0 -1 +@testcase | 130 | -1 | typeid-12 777 (null) typeid-12 777 0 777 0 0 +@testcase | 130 | 0 | typeid-11 777 4 1 777 typeid-11 777 0 0 +@testcase | 131 | | 5 typeid-13 (null) typeid-13 (null) 0 777 0 0 -1 777 0 -1 @testcase | 132 | | 10 20 10 0 0 20 0 0 0 0 @testcase | 133 | | -1 0 5 (null) (null) (null) 0 0 46 @testcase | 134 | | 10 20 diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 692c06e1cd..422b08bdb2 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -791,11 +791,11 @@ fun main() { @testcase | 105 | | 5 5 1 5 1 5 @testcase | 106 | 1 | (null) 0 @testcase | 106 | 2 | 2 1 -@testcase | 107 | 1 | (null) 2 3 129 (null) 2 3 129 -@testcase | 107 | 3 | 6 7 8 130 6 7 8 130 -@testcase | 108 | | (null) 2 3 129 (null) (null) (null) 0 (null) 2 3 129 (null) 2 3 129 (null) (null) (null) 0 -@testcase | 109 | | 6 7 8 130 0 (null) -1 (null) (null) (null) 0 -1 (null) (null) (null) 0 -@testcase | 110 | | 1 (null) (null) 0 (null) (null) 0 135 +@testcase | 107 | 1 | (null) 2 3 typeid-1 (null) 2 3 typeid-1 +@testcase | 107 | 3 | 6 7 8 typeid-2 6 7 8 typeid-2 +@testcase | 108 | | (null) 2 3 typeid-1 (null) (null) (null) 0 (null) 2 3 typeid-1 (null) 2 3 typeid-1 (null) (null) (null) 0 +@testcase | 109 | | 6 7 8 typeid-2 0 (null) -1 (null) (null) (null) 0 -1 (null) (null) (null) 0 +@testcase | 110 | | 1 (null) (null) 0 (null) (null) 0 typeid-7 @testcase | 120 | | 5 @testcase | 121 | | -1 0 0 -1 -1 0 @testcase | 122 | 0 0 1 | 36 @@ -807,21 +807,21 @@ fun main() { @testcase | 125 | 1 1 | 42 @testcase | 126 | | 42 @testcase | 127 | | 1 2 5 1 -@testcase | 128 | | (null) (null) 0 777 (null) 2 1 777 3 4 129 +@testcase | 128 | | (null) (null) 0 777 (null) 2 1 777 3 4 typeid-1 @testcase | 129 | | (null) (null) 0 (null) 5 1 -@testcase | 130 | 0 | (null) 5 1 777 (null) (null) 0 777 1 2 129 777 (null) [ 6 ] 136 777 (null) (null) 0 -@testcase | 130 | -1 | (null) (null) 0 777 (null) (null) 0 777 1 2 129 777 (null) [ 6 ] 136 777 (null) (null) 0 -@testcase | 131 | | (null) 777 5 777 1 2 777 (null) 777 5 777 (null) (null) 0 777 1 2 129 -@testcase | 132 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 129 777 (null) (null) (null) 0 -@testcase | 133 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 129 777 (null) (null) (null) 0 -@testcase | 134 | | (null) 5 1 777 (null) 5 1 777 1 2 129 777 1 2 129 777 (null) 5 42 777 5 42 777 5 42 -@testcase | 135 | | (null) 5 1 777 (null) (null) 1 2 129 777 (null) (null) 1 2 129 777 (null) (null) 0 -@testcase | 136 | | 1 2 129 777 1 2 140 777 1 2 141 777 (null) 1 2 142 +@testcase | 130 | 0 | (null) 5 1 777 (null) (null) 0 777 1 2 typeid-1 777 (null) [ 6 ] typeid-8 777 (null) (null) 0 +@testcase | 130 | -1 | (null) (null) 0 777 (null) (null) 0 777 1 2 typeid-1 777 (null) [ 6 ] typeid-8 777 (null) (null) 0 +@testcase | 131 | | (null) 777 5 777 1 2 777 (null) 777 5 777 (null) (null) 0 777 1 2 typeid-1 +@testcase | 132 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 typeid-1 777 (null) (null) (null) 0 +@testcase | 133 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 typeid-1 777 (null) (null) (null) 0 +@testcase | 134 | | (null) 5 1 777 (null) 5 1 777 1 2 typeid-1 777 1 2 typeid-1 777 (null) 5 42 777 5 42 777 5 42 +@testcase | 135 | | (null) 5 1 777 (null) (null) 1 2 typeid-1 777 (null) (null) 1 2 typeid-1 777 (null) (null) 0 +@testcase | 136 | | 1 2 typeid-1 777 1 2 typeid-12 777 1 2 typeid-13 777 (null) 1 2 typeid-14 @testcase | 137 | 1 2 | 1 2 777 1 2 777 1 2 777 [ 1 2 ] -@testcase | 138 | | 1 (null) (null) (null) (null) 0 2 146 777 1 (null) 0 2 147 777 1 5 1 2 147 777 1 (null) 5 1 2 148 +@testcase | 138 | | 1 (null) (null) (null) (null) 0 2 typeid-18 777 1 (null) 0 2 typeid-19 777 1 5 1 2 typeid-19 777 1 (null) 5 1 2 typeid-20 @testcase | 139 | | 1 (null) (null) (null) (null) 0 2 777 1 (null) 0 2 777 1 5 1 2 777 1 (null) 5 1 2 @testcase | 140 | 5 | 5 42 -@testcase | 140 | 15 | 15 131 +@testcase | 140 | 15 | 15 typeid-3 @testcase | 140 | 90 | 90 49 @testcase | 141 | 7 8 | 7 7 1 8 8 1 @testcase | 141 | null null | (null) (null) 0 (null) (null) 0 @@ -842,9 +842,9 @@ fun main() { @testcase | 154 | | 100 1 @testcase | 155 | | 5 1 5 1 @testcase | 156 | 1 2 -1 | 2 44 2 2 44 2 2 44 2 44 2 2 2 -@testcase | 157 | 1 | (null) 0 (null) 149 777 1 2 129 777 149 149 777 0 0 -1 0 -1 0 0 777 -1 0 -@testcase | 157 | 0 | (null) 0 (null) 149 777 (null) (null) 149 777 0 0 777 0 0 -1 0 0 -1 0 777 0 -1 -@testcase | 158 | | (null) (null) 149 (null) (null) 149 (null) 0 (null) +@testcase | 157 | 1 | (null) 0 (null) typeid-21 777 1 2 typeid-1 777 typeid-21 typeid-21 777 0 0 -1 0 -1 0 0 777 -1 0 +@testcase | 157 | 0 | (null) 0 (null) typeid-21 777 (null) (null) typeid-21 777 0 0 777 0 0 -1 0 0 -1 0 777 0 -1 +@testcase | 158 | | (null) (null) typeid-21 (null) (null) typeid-21 (null) 0 (null) @testcase | 159 | 0 4 | 456 @testcase | 159 | null 0 | 123 @testcase | 160 | | 10 0 diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index 825b48b96b..d1361bd0c8 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -338,7 +338,7 @@ fun main() {} @testcase | 107 | | 65537 @testcase | 108 | | 4 @testcase | 109 | | 0 3 0 7 -@testcase | 110 | 5 | 5 10 100 100 100 134 +@testcase | 110 | 5 | 5 10 100 100 100 typeid-6 @testcase | 110 | 0 | (null) (null) (null) (null) (null) 0 @testcase | 111 | 2 3 9 | -1 @testcase | 111 | 11 22 44 | -1 @@ -348,7 +348,7 @@ fun main() {} @testcase | 112 | -1 -10 -20 | -1 @testcase | 113 | 0 | 12 12 @testcase | 113 | -1 | 12 -1 -@testcase | 114 | -1 | 2 3 131 +@testcase | 114 | -1 | 2 3 typeid-3 @testcase | 114 | 0 | (null) 12 1 @testcase | 115 | | 7 6 7 @testcase | 116 | | 7 11 80 diff --git a/tolk-tester/tolk-tester.js b/tolk-tester/tolk-tester.js index 8b70007246..dcd667e6ec 100644 --- a/tolk-tester/tolk-tester.js +++ b/tolk-tester/tolk-tester.js @@ -130,9 +130,12 @@ class TolkTestCaseInputOutput { this.expected_output = output_str } - check(/**string[]*/ stdout_lines, /**number*/ line_idx) { - if (stdout_lines[line_idx] !== this.expected_output) - throw new CompareOutputError(`error on case #${line_idx + 1} (${this.method_id} | ${this.input}):\n expect: ${this.expected_output}\n actual: ${stdout_lines[line_idx]}`, stdout_lines.join("\n")) + check(/**string[]*/ stdout_lines, /**number*/ line_idx, /**number*/ pivot_typeid) { + let expected_str = this.expected_output + if (expected_str.includes("typeid")) + expected_str = expected_str.replace(/typeid-(\d+)/g, (match, p1) => pivot_typeid + (+p1)) + if (stdout_lines[line_idx] !== expected_str) + throw new CompareOutputError(`error on case #${line_idx + 1} (${this.method_id} | ${this.input}):\n expect: ${expected_str}\n actual: ${stdout_lines[line_idx]}`, stdout_lines.join("\n")) } } @@ -275,6 +278,8 @@ class TolkTestFile { this.experimental_options = null /** @type {boolean} */ this.enable_tolk_lines_comments = false + /** @type {number} */ + this.pivot_typeid = 138 // may be changed when stdlib introduces new union types } parse_input_from_tolk_file() { @@ -394,7 +399,7 @@ class TolkTestFile { throw new CompareOutputError(`unexpected number of fift output: ${stdout_lines.length} lines, but ${this.input_output.length} testcases`, stdout) for (let i = 0; i < stdout_lines.length; ++i) - this.input_output[i].check(stdout_lines, i) + this.input_output[i].check(stdout_lines, i, this.pivot_typeid) if (this.fif_codegen.length) { const fif_output = fs.readFileSync(this.get_compiled_fif_filename(), 'utf-8').split(/\r?\n/) diff --git a/tolk-tester/tolk-tester.py b/tolk-tester/tolk-tester.py index 126b079308..f39d2d3ab8 100644 --- a/tolk-tester/tolk-tester.py +++ b/tolk-tester/tolk-tester.py @@ -127,9 +127,12 @@ def __init__(self, method_id_str: str, input_str: str, output_str: str): self.input = " ".join(processed_inputs) self.expected_output = output_str - def check(self, stdout_lines: List[str], line_idx: int): - if stdout_lines[line_idx] != self.expected_output: - raise CompareOutputError("error on case #%d (%d | %s):\n expect: %s\n actual: %s" % (line_idx + 1, self.method_id, self.input, self.expected_output, stdout_lines[line_idx]), "\n".join(stdout_lines)) + def check(self, stdout_lines: List[str], line_idx: int, pivot_typeid: int): + expected_str = self.expected_output + if expected_str.find("typeid") != -1: + expected_str = re.sub(r'typeid-(\d+)', lambda m: str(pivot_typeid + int(m.group(1))), expected_str) + if stdout_lines[line_idx] != expected_str: + raise CompareOutputError("error on case #%d (%d | %s):\n expect: %s\n actual: %s" % (line_idx + 1, self.method_id, self.input, expected_str, stdout_lines[line_idx]), "\n".join(stdout_lines)) class TolkTestCaseStderr: @@ -261,6 +264,7 @@ def __init__(self, tolk_filename: str, artifacts_folder: str): self.expected_hash: TolkTestCaseExpectedHash | None = None self.experimental_options: str | None = None self.enable_tolk_lines_comments = False + self.pivot_typeid = 138 # may be changed when stdlib introduces new union types def parse_input_from_tolk_file(self): with open(self.tolk_filename, "r") as fd: @@ -374,7 +378,7 @@ def run_and_check(self): raise CompareOutputError("unexpected number of fift output: %d lines, but %d testcases" % (len(stdout_lines), len(self.input_output)), stdout) for i in range(len(stdout_lines)): - self.input_output[i].check(stdout_lines, i) + self.input_output[i].check(stdout_lines, i, self.pivot_typeid) if len(self.fif_codegen): with open(self.get_compiled_fif_filename()) as fd: diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index 7aca836882..2e0bf0cad3 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -8,6 +8,7 @@ set(TOLK_SOURCE constant-evaluator.cpp pack-unpack-api.cpp pack-unpack-serializers.cpp + send-message-api.cpp pipe-discover-parse-sources.cpp pipe-register-symbols.cpp pipe-resolve-identifiers.cpp diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 6a82b432f2..3696d550e6 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1228,6 +1228,7 @@ void define_builtins() { TypePtr Bool = TypeDataBool::create(); TypePtr Slice = TypeDataSlice::create(); TypePtr Builder = TypeDataBuilder::create(); + TypePtr Address = TypeDataAddress::create(); TypePtr Tuple = TypeDataTuple::create(); TypePtr Never = TypeDataNever::create(); @@ -1246,6 +1247,11 @@ void define_builtins() { TypePtr CellT = TypeDataUnknown::create(); TypePtr PackOptions = TypeDataUnknown::create(); TypePtr UnpackOptions = TypeDataUnknown::create(); + TypePtr CreateMessageOptions = TypeDataUnknown::create(); + TypePtr createExternalLogMessageOptions = TypeDataUnknown::create(); + TypePtr OutMessage = TypeDataUnknown::create(); + TypePtr AddressShardingOptions = TypeDataUnknown::create(); + const GenericsDeclaration* declTBody = new GenericsDeclaration(std::vector{{"TBody", nullptr}}, 0); // builtin operators // they are internally stored as functions, because at IR level, there is no difference @@ -1449,6 +1455,9 @@ void define_builtins() { define_builtin_method("tuple.set", Tuple, {Tuple, typeT, Int}, Unit, declGenericT, compile_tuple_set_at, FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf); + define_builtin_method("address.buildSameAddressInAnotherShard", Address, {Address, AddressShardingOptions}, Builder, nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagCompileTimeGen); define_builtin_method("debug.print", debug, {typeT}, Unit, declGenericT, compile_debug_print_to_string, FunctionData::flagAllowAnyWidthT); @@ -1495,6 +1504,13 @@ void define_builtins() { compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_func("createMessage", {CreateMessageOptions}, OutMessage, declTBody, + compile_time_only_function, + FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + define_builtin_func("createExternalLogMessage", {createExternalLogMessageOptions}, OutMessage, declTBody, + compile_time_only_function, + FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + // functions not presented in stdlib at all // used in tolk-tester to check/expose internal compiler state // each of them is handled in a special way, search by its name @@ -1519,6 +1535,11 @@ void patch_builtins_after_stdlib_loaded() { lookup_function("debug.printString")->mutate()->receiver_type = debug; lookup_function("debug.dumpStack")->mutate()->receiver_type = debug; + StructPtr struct_ref_AddressShardingOptions = lookup_global_symbol("AddressShardingOptions")->try_as(); + TypePtr AddressShardingOptions = TypeDataStruct::create(struct_ref_AddressShardingOptions); + + lookup_function("address.buildSameAddressInAnotherShard")->mutate()->parameters[1].declared_type = AddressShardingOptions; + StructPtr struct_ref_CellT = lookup_global_symbol("Cell")->try_as(); StructPtr struct_ref_PackOptions = lookup_global_symbol("PackOptions")->try_as(); StructPtr struct_ref_UnpackOptions = lookup_global_symbol("UnpackOptions")->try_as(); @@ -1552,6 +1573,18 @@ void patch_builtins_after_stdlib_loaded() { lookup_function("slice.skipAny")->mutate()->parameters[1].default_value = v_empty_UnpackOptions; lookup_function("builder.storeAny")->mutate()->parameters[2].declared_type = PackOptions; lookup_function("builder.storeAny")->mutate()->parameters[2].default_value = v_empty_PackOptions; + + StructPtr struct_ref_CreateMessageOptions = lookup_global_symbol("CreateMessageOptions")->try_as(); + StructPtr struct_ref_createExternalLogMessageOptions = lookup_global_symbol("createExternalLogMessageOptions")->try_as(); + StructPtr struct_ref_OutMessage = lookup_global_symbol("OutMessage")->try_as(); + TypePtr CreateMessageOptions = TypeDataGenericTypeWithTs::create(struct_ref_CreateMessageOptions, nullptr, {TypeDataGenericT::create("TBody")}); + TypePtr createExternalLogMessageOptions = TypeDataGenericTypeWithTs::create(struct_ref_createExternalLogMessageOptions, nullptr, {TypeDataGenericT::create("TBody")}); + TypePtr OutMessage = TypeDataStruct::create(struct_ref_OutMessage); + + lookup_function("createMessage")->mutate()->parameters[0].declared_type = CreateMessageOptions; + lookup_function("createMessage")->mutate()->declared_return_type = OutMessage; + lookup_function("createExternalLogMessage")->mutate()->parameters[0].declared_type = createExternalLogMessageOptions; + lookup_function("createExternalLogMessage")->mutate()->declared_return_type = OutMessage; } } // namespace tolk diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index 6af6650f49..227a070660 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -135,13 +135,19 @@ class PackUnpackAvailabilityChecker { return {}; } - // `builder` can be used for writing, but not for reading + // `builder` and `slice` can be used for writing, but not for reading if (any_type == TypeDataBuilder::create()) { if (is_pack) { return {}; } return CantSerializeBecause("because type `builder` can not be used for reading, only for writing\nhint: use `bitsN` or `RemainingBitsAndRefs` for reading\nhint: using generics, you can substitute `builder` for writing and something other for reading"); } + if (any_type == TypeDataSlice::create()) { + if (is_pack) { + return {}; + } + return CantSerializeBecause("because type `slice` can not be used for reading, it doesn't define binary width\nhint: replace `slice` with `address` if it's an address, actually\nhint: replace `slice` with `bits128` and similar if it represents fixed-width data without refs"); + } // serialization not available // for common types, make a detailed explanation with a hint how to fix @@ -149,9 +155,6 @@ class PackUnpackAvailabilityChecker { if (any_type == TypeDataInt::create()) { return CantSerializeBecause("because type `int` is not serializable, it doesn't define binary width\nhint: replace `int` with `int32` / `uint64` / `coins` / etc."); } - if (any_type == TypeDataSlice::create()) { - return CantSerializeBecause("because type `slice` is not serializable, it doesn't define binary width\nhint: replace `slice` with `address` if it's an address, actually\nhint: replace `slice` with `bits128` and similar if it represents fixed-width data without refs"); - } if (any_type == TypeDataNullLiteral::create()) { return CantSerializeBecause("because type `null` is not serializable\nhint: `int32?` and other nullable types will work"); } diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index c0490c5c76..0ffce5fa23 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -77,6 +77,11 @@ void PackContext::storeUint(var_idx_t ir_idx, int len) const { code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), f_storeUint); } +void PackContext::storeUint_var(var_idx_t ir_idx, var_idx_t ir_len) const { + std::vector args = { ir_builder0, ir_idx, ir_len }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), f_storeUint); +} + void PackContext::storeBool(var_idx_t ir_idx) const { std::vector args = { ir_builder0, ir_idx }; code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeBool")); @@ -454,6 +459,25 @@ struct S_Builder final : ISerializer { } }; +struct S_Slice final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeSlice(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + tolk_assert(false); // `slice` can only be used for writing, checked earlier + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + tolk_assert(false); // `slice` can only be used for writing, checked earlier + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize::unpredictable_infinity(); + } +}; + struct S_Null final : ISerializer { void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { // while `null` itself is not serializable, it may be contained inside a union: @@ -941,6 +965,9 @@ static std::unique_ptr get_serializer_for_type(TypePtr any_type) { if (any_type == TypeDataBuilder::create()) { return std::make_unique(); } + if (any_type == TypeDataSlice::create()) { + return std::make_unique(); + } if (any_type == TypeDataNullLiteral::create()) { return std::make_unique(); } diff --git a/tolk/pack-unpack-serializers.h b/tolk/pack-unpack-serializers.h index c2c99ea2ae..06163153ba 100644 --- a/tolk/pack-unpack-serializers.h +++ b/tolk/pack-unpack-serializers.h @@ -72,6 +72,7 @@ class PackContext { void storeInt(var_idx_t ir_idx, int len) const; void storeUint(var_idx_t ir_idx, int len) const; + void storeUint_var(var_idx_t ir_idx, var_idx_t ir_len) const; void storeBool(var_idx_t ir_idx) const; void storeCoins(var_idx_t ir_idx) const; void storeRef(var_idx_t ir_idx) const; diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 1f5025918f..8a6c2911b6 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -24,7 +24,7 @@ #include "smart-casts-cfg.h" #include "pack-unpack-api.h" #include "generics-helpers.h" -#include +#include "send-message-api.h" /* * This pipe is the last one operating AST: it transforms AST to IR. @@ -589,6 +589,21 @@ static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob if (f_name == "T.estimatePackSize") { return generate_estimate_size_call(code, loc, typeT); } + + if (f_name == "createMessage") { + std::vector ir_msg_params = vars_per_arg[0]; + return generate_createMessage(code, loc, typeT->unwrap_alias(), std::move(ir_msg_params)); + } + if (f_name == "createExternalLogMessage") { + std::vector ir_msg_params = vars_per_arg[0]; + return generate_createExternalLogMessage(code, loc, typeT->unwrap_alias(), std::move(ir_msg_params)); + } + } + + if (called_f->name == "address.buildSameAddressInAnotherShard") { + std::vector ir_self_address = vars_per_arg[0]; + std::vector ir_shard_options = vars_per_arg[1]; + return generate_address_buildInAnotherShard(code, loc, std::move(ir_self_address), std::move(ir_shard_options)); } tolk_assert(false); diff --git a/tolk/pipe-check-serialized-fields.cpp b/tolk/pipe-check-serialized-fields.cpp index 3ed8fc33a2..8f436c9571 100644 --- a/tolk/pipe-check-serialized-fields.cpp +++ b/tolk/pipe-check-serialized-fields.cpp @@ -58,6 +58,11 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody static void check_struct_fits_cell_or_has_policy(const TypeDataStruct* t_struct) { StructPtr struct_ref = t_struct->struct_ref; + bool avoid_check = struct_ref->is_instantiation_of_generic_struct() && struct_ref->base_struct_ref->name == "UnsafeBodyNoRef"; + if (avoid_check) { + return; + } + PackSize size = estimate_serialization_size(t_struct); if (size.max_bits > 1023 && !size.is_unpredictable_infinity()) { if (struct_ref->overflow1023_policy == StructData::Overflow1023Policy::not_specified) { @@ -82,16 +87,18 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody TypePtr serialized_type = nullptr; bool is_pack = false; if (f_name == "Cell.load" || f_name == "T.fromSlice" || f_name == "T.fromCell" || f_name == "T.toCell" || - f_name == "T.loadAny" || f_name == "slice.skipAny" || f_name == "slice.storeAny" || f_name == "T.estimatePackSize") { + f_name == "T.loadAny" || f_name == "slice.skipAny" || f_name == "slice.storeAny" || f_name == "T.estimatePackSize" || + f_name == "createMessage" || f_name == "createExternalLogMessage") { serialized_type = fun_ref->substitutedTs->typeT_at(0); - is_pack = f_name == "T.toCell" || f_name == "slice.storeAny" || f_name == "T.estimatePackSize"; + is_pack = f_name == "T.toCell" || f_name == "slice.storeAny" || f_name == "T.estimatePackSize" || f_name == "createMessage" || f_name == "createExternalLogMessage"; } else { return; // not a serialization function } std::string because_msg; if (!check_struct_can_be_packed_or_unpacked(serialized_type, is_pack, because_msg)) { - fire(cur_f, v->loc, "auto-serialization via " + fun_ref->method_name + "() is not available for type `" + serialized_type->as_human_readable() + "`\n" + because_msg); + std::string via_name = fun_ref->is_method() ? fun_ref->method_name : fun_ref->base_fun_ref->name; + fire(cur_f, v->loc, "auto-serialization via " + via_name + "() is not available for type `" + serialized_type->as_human_readable() + "`\n" + because_msg); } check_type_fits_cell_or_has_policy(serialized_type); diff --git a/tolk/send-message-api.cpp b/tolk/send-message-api.cpp new file mode 100644 index 0000000000..4fd8ab8465 --- /dev/null +++ b/tolk/send-message-api.cpp @@ -0,0 +1,496 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "send-message-api.h" +#include "pack-unpack-serializers.h" +#include "type-system.h" + +namespace tolk { + +std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc); +std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc); + +static bool is_type_UnsafeBodyNoRef_T(TypePtr bodyT) { + if (const TypeDataStruct* t_struct = bodyT->unwrap_alias()->try_as()) { + if (t_struct->struct_ref->is_instantiation_of_generic_struct() && t_struct->struct_ref->base_struct_ref->name == "UnsafeBodyNoRef") { + return true; + } + } + return false; +} + +// currently, there is no way to pass custom pack options to createMessage, using hardcoded ones +static std::vector create_default_PackOptions(CodeBlob& code, SrcLocation loc) { + StructPtr s_PackOptions = lookup_global_symbol("PackOptions")->try_as(); + std::vector ir_options = code.create_tmp_var(TypeDataStruct::create(s_PackOptions), loc, "(pack-options)"); + tolk_assert(ir_options.size() == 1); + + var_idx_t ir_zero = code.create_int(loc, 0, "(zero)"); + code.emplace_back(loc, Op::_Let, std::vector{ir_options[0]}, std::vector{ir_zero}); // skipBitsNFieldsValidation + return ir_options; +} + +std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect) { + StructPtr s_Options = lookup_global_symbol("CreateMessageOptions")->try_as(); + StructPtr s_AutoDeployAddress = lookup_global_symbol("AutoDeployAddress")->try_as(); + + const TypeDataBool* t_bounce = s_Options->find_field("bounce")->declared_type->try_as(); + const TypeDataUnion* t_dest = s_Options->find_field("dest")->declared_type->try_as(); + const TypeDataUnion* t_value = s_Options->find_field("value")->declared_type->try_as(); + const TypeDataUnion* t_stateInit = s_AutoDeployAddress->find_field("stateInit")->declared_type->try_as(); + const TypeDataUnion* t_toShard = s_AutoDeployAddress->find_field("toShard")->declared_type->try_as(); + tolk_assert(t_bounce); + tolk_assert(t_dest && t_dest->get_width_on_stack() == (1+3+3+1) && t_dest->size() == 4); + tolk_assert(t_value && t_value->get_width_on_stack() == (2+1) && t_value->size() == 2); + tolk_assert(t_stateInit && t_stateInit->get_width_on_stack() == (2+1) && t_stateInit->size() == 2); + tolk_assert(t_toShard && t_toShard->get_width_on_stack() == (2+1) && t_toShard->or_null); + + int offset = 0; + auto next_slice = [&rvect, &offset](int width) -> std::vector { + int start = offset; + offset += width; + return std::vector(rvect.begin() + start, rvect.begin() + start + width); + }; + + std::vector ir_bounce = next_slice(t_bounce->get_width_on_stack()); + std::vector ir_value = next_slice(t_value->get_width_on_stack()); + std::vector ir_dest = next_slice(t_dest->get_width_on_stack()); + std::vector ir_body = next_slice(bodyT->get_width_on_stack()); + tolk_assert(offset == static_cast(rvect.size())); + + // field `dest` is `dest: address | AutoDeployAddress | (int8, uint256) | builder`; + // struct AutoDeployAddress { workchain: int8; stateInit: ContractState | cell; toShard: AddressShardingOptions?; } + // struct ContractState { code: cell; data: cell; } + // struct AddressShardingOptions { fixedPrefixLength: uint5; closeTo: address; } + std::vector ir_dest_is_address = pre_compile_is_type(code, t_dest, TypeDataAddress::create(), ir_dest, loc, "(is-address)"); + std::vector ir_dest_is_AutoDeploy = pre_compile_is_type(code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), ir_dest, loc, "(is-address)"); + std::vector ir_dest_is_builder = pre_compile_is_type(code, t_dest, TypeDataBuilder::create(), ir_dest, loc, "(is-builder)"); + std::vector ir_dest_AutoDeployAddress = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), loc); + // dest.workchain + std::vector ir_dest_workchain(ir_dest_AutoDeployAddress.begin(), ir_dest_AutoDeployAddress.begin() + 1); + // dest.stateInit (code+data or cell) + std::vector ir_dest_stateInitUnion(ir_dest_AutoDeployAddress.begin() + 1, ir_dest_AutoDeployAddress.begin() + 1 + 3); + std::vector ir_is_ContractState = pre_compile_is_type(code, t_stateInit, t_stateInit->variants[0], ir_dest_stateInitUnion, loc, "(is-ContractState)"); + std::vector ir_ContractState = transition_to_target_type(std::vector(ir_dest_stateInitUnion), code, t_stateInit, t_stateInit->variants[0], loc); + std::vector ir_StateInitCell = transition_to_target_type(std::vector(ir_dest_stateInitUnion), code, t_stateInit, t_stateInit->variants[1], loc); + // dest.toShard (shardPrefix+closeTo or null) + std::vector ir_dest_toShardOrNull(ir_dest_AutoDeployAddress.begin() + 1 + 3, ir_dest_AutoDeployAddress.begin() + 1 + 3 + 3); + std::vector ir_is_AddressSharding = pre_compile_is_type(code, t_toShard, t_toShard->or_null, ir_dest_toShardOrNull, loc, "(is-AddressSharding)"); + std::vector ir_AddressSharding = transition_to_target_type(std::vector(ir_dest_toShardOrNull), code, t_toShard, t_toShard->or_null, loc); + + // currently, there is no way to pass PackOptions, defaults are used + std::vector ir_options = create_default_PackOptions(code, loc); + + FunctionPtr f_beginCell = lookup_function("beginCell"); + FunctionPtr f_endCell = lookup_function("builder.endCell"); + + // detect whether to store `body: (Either X ^X)` inline or as ref + // if it's small (guaranteed to fit), store it inside the same builder, without creating a cell + PackSize body_size = EstimateContext().estimate_any(bodyT); + // if `body` is already `cell` / `Cell` + bool body_already_ref = bodyT == TypeDataCell::create() || is_type_cellT(bodyT); + // if `body` is `UnsafeBodyNoRef` + bool body_force_no_ref = is_type_UnsafeBodyNoRef_T(bodyT); + // max size of all fields before body = 514 (502 CommonMsgInfoRelaxed + 12 StateInit), so 500 bits will fit + bool body_100p_fits_no_ref = body_size.max_bits <= 500 && body_size.max_refs < 2; + // final decision: 1 (^X) or 0 (X) + bool body_store_as_ref = body_already_ref || (!body_100p_fits_no_ref && !body_force_no_ref); + + // if we need to store body ref, convert it to a cell here, before creating a builder for the message; + // it's more optimal, since the `body` field is the topmost at the stack + if (body_store_as_ref && !body_already_ref) { + std::vector ir_ref_builder = code.create_var(TypeDataBuilder::create(), loc, "refb"); + code.emplace_back(loc, Op::_Call, ir_ref_builder, std::vector{}, f_beginCell); + PackContext ref_ctx(code, loc, ir_ref_builder, ir_options); + ref_ctx.generate_pack_any(bodyT, std::move(ir_body)); + std::vector ir_ref_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(ref-cell)"); + code.emplace_back(loc, Op::_Call, ir_ref_cell, std::move(ir_ref_builder), f_endCell); + ir_body = std::move(ir_ref_cell); + } + + std::vector ir_builder = code.create_var(TypeDataSlice::create(), loc, "b"); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, f_beginCell); + PackContext ctx(code, loc, ir_builder, ir_options); + var_idx_t ir_zero = code.create_int(loc, 0, "(zero)"); + var_idx_t ir_one = code.create_int(loc, 1, "(one)"); + + // '0' prefix int_msg_info + ctx.storeUint(ir_zero, 1); + // fill `ihr_disabled:Bool` always 1 + ctx.storeUint(ir_one, 1); + // fill `bounce:Bool` from p.bounce (if it's constant (most likely), it will be concatenated with prev and next) + ctx.storeBool(ir_bounce[0]); + // fill `bounced:Bool` + `src:MsgAddress` 00 + ctx.storeUint(ir_zero, 1 + 2); + + // fill `dest:MsgAddressInt` from p.dest (complex union) + Op& if_address = code.emplace_back(loc, Op::_If, ir_dest_is_address); + { + // input is `dest: someAddress` + code.push_set_cur(if_address.block0); + std::vector ir_dest_address = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataAddress::create(), loc); + ctx.storeAddress(ir_dest_address[0]); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_address.block1); + Op& if_AutoDeploy = code.emplace_back(loc, Op::_If, ir_dest_is_AutoDeploy); + { + // input is `dest: { workchain, stateInit, [toShard] }`; + // then calculate hash equal to StateInit cell would be and fill "addr_std$10 + 0 anycast + workchain + hash"; + // and, if toShard, take first D bits from dest.toShard.closeTo and mix with 256-D bits of hash + code.push_set_cur(if_AutoDeploy.block0); + ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); // addr_std$10 + 0 anycast + ctx.storeInt(ir_dest_workchain[0], 8); + std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_is_ContractState); + { + // input is `dest: { ... stateInit: { code, data } }` + code.push_set_cur(if_ContractState.block0); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_is_AddressSharding); + { + // input is `dest: { ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } }; + // then stateInitHash = (hash of StateInit = 0b1(depth)0110 (prefix + code + data)) + code.push_set_cur(if_sharded.block0); + std::vector ir_args_depth_code_data = { ir_AddressSharding[0], ir_ContractState[0], ir_ContractState[1] }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(ir_args_depth_code_data), lookup_function("StateInit.calcHashPrefixCodeData")); + code.close_pop_cur(loc); + } + { + // input is: `dest: { ... stateInit: { code, data } }` (toShard is null); + // then hash = (hash of StateInit = 0b00110 (only code + data)) + code.push_set_cur(if_sharded.block1); + code.emplace_back(loc, Op::_Call, ir_hash, ir_ContractState, lookup_function("StateInit.calcHashCodeData")); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + { + // input is `dest: { ... stateInit: cell }` + code.push_set_cur(if_ContractState.block1); + code.emplace_back(loc, Op::_Call, ir_hash, ir_StateInitCell, lookup_function("cell.hash")); + code.close_pop_cur(loc); + } + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_is_AddressSharding); + { + // input is `dest: { ... toShard: { fixedPrefixLength, closeTo } }` + // we already calculated stateInitHash (ir_hash): either cell.hash() or based on prefix+code+data; + // now, we need: hash = (first D bits from dest.toShard.closeTo) + (last 256-D bits from stateInitHash); + // example for fixedPrefixLength (shard depth) = 8: + // | closeTo | 01010101...xxx | given as input, by user (it's address, internally slice) + // | shardPrefix | 01010101 | first 8 bits of closeTo + // | stateInitHash | yyyyyyyy...yyy | mask = (1 << (256-D)) - 1 = 00000000111...111 (8 zeroes) + // | hash (result) | 01010101...yyy | + // remember, that closeTo is addr_std$10 + 0 + workchain + xxx...xxx, so skip 11 bits and read 8 + code.push_set_cur(if_sharded.block0); + std::vector ir_shardPrefix = code.create_tmp_var(TypeDataSlice::create(), loc, "(shardPrefix)"); + std::vector args_subslice = { ir_AddressSharding[1], code.create_int(loc, 3+8, ""), ir_AddressSharding[0] }; + code.emplace_back(loc, Op::_Call, ir_shardPrefix, std::move(args_subslice), lookup_function("slice.getMiddleBits")); + ctx.storeSlice(ir_shardPrefix[0]); // first 8 bits of closeTo hash + std::vector ir_mask = code.create_tmp_var(TypeDataInt::create(), loc, "(mask)"); + code.emplace_back(loc, Op::_Call, ir_mask, std::vector{code.create_int(loc, 256, ""), ir_AddressSharding[0]}, lookup_function("_-_")); + code.emplace_back(loc, Op::_Call, ir_mask, std::vector{ir_one, ir_mask[0]}, lookup_function("_<<_")); + code.emplace_back(loc, Op::_Call, ir_mask, std::vector{ir_mask[0], ir_one}, lookup_function("_-_")); + code.emplace_back(loc, Op::_Call, ir_hash, std::vector{ir_hash[0], ir_mask[0]}, lookup_function("_&_")); + code.emplace_back(loc, Op::_Call, ir_mask, std::vector{code.create_int(loc, 256, ""), ir_AddressSharding[0]}, lookup_function("_-_")); + ctx.storeUint_var(ir_hash[0], ir_mask[0]); // 248 STU (stateInitHash & mask) + code.close_pop_cur(loc); + } + { + // input is `dest: { workchain, stateInit }` (toShard is null); + // we already calculated stateInitHash: either cell.hash() or based on code+data + code.push_set_cur(if_sharded.block1); + ctx.storeUint(ir_hash[0], 256); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_AutoDeploy.block1); + Op& if_builder = code.emplace_back(loc, Op::_If, ir_dest_is_builder); + { + // input is `dest: someBuilder` + code.push_set_cur(if_builder.block0); + std::vector ir_dest_builder = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataBuilder::create(), loc); + ctx.storeBuilder(ir_dest_builder[0]); + code.close_pop_cur(loc); + } + { + // input is `dest: (workchain, hash)` + code.push_set_cur(if_builder.block1); + std::vector ir_dest_wc_hash = transition_to_target_type(std::vector(ir_dest), code, t_dest, t_dest->variants[2], loc); + ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); + ctx.storeInt(ir_dest_wc_hash[0], 8); // most likely, it's 0 (basechain), will be merged with above + ctx.storeUint(ir_dest_wc_hash[1], 256); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + + // fill `value:CurrencyCollection` from p.value `coins | (coins, dict)` + std::vector ir_is_coins = pre_compile_is_type(code, t_value, TypeDataCoins::create(), ir_value, loc, "(is-coins)"); + Op& if_coins = code.emplace_back(loc, Op::_If, ir_is_coins); + { + code.push_set_cur(if_coins.block0); + std::vector ir_coins = transition_to_target_type(std::vector(ir_value), code, t_value, TypeDataCoins::create(), loc); + ctx.storeCoins(ir_coins[0]); + ctx.storeUint(ir_zero, 1); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_coins.block1); + std::vector ir_coins_dict = transition_to_target_type(std::move(ir_value), code, t_value, t_value->variants[1], loc); + ctx.storeCoins(ir_coins_dict[0]); + ctx.storeMaybeRef(ir_coins_dict[1]); + code.close_pop_cur(loc); + } + + // tail of CommonMsgInfoRelaxed: 4*0 ihr_fee + 4*0 fwd_fee + 64*0 created_lt + 32*0 created_at + ctx.storeUint(ir_zero, 4 + 4 + 64 + 32); + + // fill `init: (Maybe (Either StateInit ^StateInit))` + // it's present only if p.dest contains StateInit + // also fill the either bit of `body: (Either X ^X)` + Op& if_no_init = code.emplace_back(loc, Op::_If, ir_dest_is_AutoDeploy); + { + // when it's known at compile-time (always in practice), this `if` is simplified, and bits join with above + code.push_set_cur(if_no_init.block1); + ctx.storeUint(body_store_as_ref ? ir_one : ir_zero, 1 + 1); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_no_init.block0); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_is_ContractState); + { + // input is `dest: { ... stateInit: { code, data } }` and need to compose TL/B StateInit; + // it's either just code+data OR (if `toShard: { ... }` is set) fixedPrefixLength+code+data + code.push_set_cur(if_ContractState.block0); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_is_AddressSharding); + { + // 1 (maybe true) + 0 (either left) + 1 (maybe true of StateInit) + fixedPrefixLength + 0110 + body ref or not + code.push_set_cur(if_sharded.block0); + ctx.storeUint(code.create_int(loc, 0b101, ""), 1 + 1 + 1); + ctx.storeUint(ir_AddressSharding[0], 5); // fixedPrefixLength (shard depth) + ctx.storeUint(code.create_int(loc, 0b01100 + body_store_as_ref, ""), 4 + 1); + code.close_pop_cur(loc); + // also, we used dest.toShard to fill CommonMsgInfoRelaxed.dest.address (with a mask for stateInitHash, see above) + } + { + // 1 (maybe true) + 0 (either left) + 00110 (only code and data from StateInit) + body ref or not + code.push_set_cur(if_sharded.block1); + var_idx_t ir_rest_bits = code.create_int(loc, 0b10001100 + body_store_as_ref, "(rest-bits)"); + ctx.storeUint(ir_rest_bits, 1 + 1 + 5 + 1); + code.close_pop_cur(loc); + } + ctx.storeRef(ir_ContractState[0]); // dest.stateInit.code + ctx.storeRef(ir_ContractState[1]); // dest.stateInit.data + code.close_pop_cur(loc); + } + { + // so, we have `dest: { stateInit: someCell }`, store it as ref + // 1 (maybe true) + 1 (either right) + body ref or not + code.push_set_cur(if_ContractState.block1); + var_idx_t ir_rest_bits = code.create_int(loc, 0b110 + body_store_as_ref, "(rest-bits)"); + ctx.storeUint(ir_rest_bits, 1 + 1 + 1); + ctx.storeRef(ir_StateInitCell[0]); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + + // store body; previously, we've calculated whether to store is as a ref or not + if (body_size.max_bits == 0 && body_size.max_refs == 0) { + tolk_assert(ir_body.empty()); + } else if (body_store_as_ref) { + tolk_assert(ir_body.size() == 1); // it was either an input cell or a automatically created one + ctx.storeRef(ir_body[0]); + } else { + ctx.generate_pack_any(bodyT, std::move(ir_body)); + } + + std::vector ir_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(msg-cell)"); + code.emplace_back(loc, Op::_Call, ir_cell, std::move(ir_builder), f_endCell); + return ir_cell; +} + +std::vector generate_createExternalLogMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect) { + StructPtr s_Options = lookup_global_symbol("createExternalLogMessageOptions")->try_as(); + StructPtr s_ExtOutLogBucket = lookup_global_symbol("ExtOutLogBucket")->try_as(); + + const TypeDataUnion* t_dest = s_Options->find_field("dest")->declared_type->try_as(); + const TypeDataUnion* t_topic = s_ExtOutLogBucket->find_field("topic")->declared_type->try_as(); + tolk_assert(t_dest && t_dest->get_width_on_stack() == (2+1) && t_dest->size() == 3); + tolk_assert(t_topic && t_topic->get_width_on_stack() == (1+1) && t_topic->size() == 2); + + int offset = 0; + auto next_slice = [&rvect, &offset](int width) -> std::vector { + int start = offset; + offset += width; + return std::vector(rvect.begin() + start, rvect.begin() + start + width); + }; + + std::vector ir_dest = next_slice(t_dest->get_width_on_stack()); + std::vector ir_body = next_slice(bodyT->get_width_on_stack()); + tolk_assert(offset == static_cast(rvect.size())); + + // field `dest` is `dest: address | builder | ExtOutLogBucket`; + // struct ExtOutLogBucket { topic: uint248 | bits248; } + std::vector ir_dest_is_address = pre_compile_is_type(code, t_dest, TypeDataAddress::create(), ir_dest, loc, "(is-address)"); + std::vector ir_dest_is_builder = pre_compile_is_type(code, t_dest, TypeDataBuilder::create(), ir_dest, loc, "(is-builder)"); + std::vector ir_dest_ExtOutLogBucket = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataStruct::create(s_ExtOutLogBucket), loc); + // dest.topic (it's the only field in a struct) + std::vector ir_dest_topic = std::move(ir_dest_ExtOutLogBucket); + + std::vector ir_options = create_default_PackOptions(code, loc); + + FunctionPtr f_beginCell = lookup_function("beginCell"); + FunctionPtr f_endCell = lookup_function("builder.endCell"); + + // detect whether to store `body: (Either X ^X)` inline or as ref + // if it's small (guaranteed to fit), store it inside the same builder, without creating a cell + PackSize body_size = EstimateContext().estimate_any(bodyT); + // if `body` is already `cell` / `Cell` + bool body_already_ref = bodyT == TypeDataCell::create() || is_type_cellT(bodyT); + // if `body` is `UnsafeBodyNoRef` + bool body_force_no_ref = is_type_UnsafeBodyNoRef_T(bodyT); + // max size of all fields before body = 622 (621 CommonMsgInfoRelaxed + 1 StateInit), so 400 bits will fit + bool body_100p_fits_no_ref = body_size.max_bits < 400; + // final decision: 1 (^X) or 0 (X) + bool body_store_as_ref = body_already_ref || (!body_100p_fits_no_ref && !body_force_no_ref); + + // same as for createMessage: `body` field is the topmost at the stack, convert it to a cell before creating a builder + if (body_store_as_ref && !body_already_ref) { + std::vector ir_ref_builder = code.create_var(TypeDataBuilder::create(), loc, "refb"); + code.emplace_back(loc, Op::_Call, ir_ref_builder, std::vector{}, f_beginCell); + PackContext ref_ctx(code, loc, ir_ref_builder, ir_options); + ref_ctx.generate_pack_any(bodyT, std::move(ir_body)); + std::vector ir_ref_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(ref-cell)"); + code.emplace_back(loc, Op::_Call, ir_ref_cell, std::move(ir_ref_builder), f_endCell); + ir_body = std::move(ir_ref_cell); + } + + std::vector ir_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, f_beginCell); + PackContext ctx(code, loc, ir_builder, ir_options); + var_idx_t ir_zero = code.create_int(loc, 0, "(zero)"); + var_idx_t ir_one = code.create_int(loc, 1, "(one)"); + + // '11' prefix ext_out_msg_info + '00' src + ctx.storeUint(code.create_int(loc, 0b1100, "(out-prefix)"), 4); + + // fill `dest:MsgAddressExt` from p.dest (complex union) + Op& if_address = code.emplace_back(loc, Op::_If, ir_dest_is_address); + { + // input is `dest: someAddress` + code.push_set_cur(if_address.block0); + std::vector ir_dest_address = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataAddress::create(), loc); + ctx.storeAddress(ir_dest_address[0]); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_address.block1); + Op& if_builder = code.emplace_back(loc, Op::_If, ir_dest_is_builder); + { + // input is `dest: someBuilder` + code.push_set_cur(if_builder.block0); + std::vector ir_dest_builder = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataBuilder::create(), loc); + ctx.storeBuilder(ir_dest_builder[0]); + code.close_pop_cur(loc); + } + { + // input is `dest: ExtOutLogBucket`; + // fill addr_extern$01 + 256 (len 9 bit) + 0x00 (prefix) + 248 bits + code.push_set_cur(if_builder.block1); + ctx.storeUint(ir_one, 2); // addr_extern$01 + ctx.storeUint(code.create_int(loc, 256, "(addr-len)"), 9); // len:(## 9) = 256 + ctx.storeOpcode(s_ExtOutLogBucket->opcode); + std::vector ir_if_topic_uint = pre_compile_is_type(code, t_topic, t_topic->variants[0], ir_dest_topic, loc, "(topic-is-uint)"); + Op& if_topic_uint = code.emplace_back(loc, Op::_If, ir_if_topic_uint); + { + // input is `dest: ExtOutLogBucket { topic: uint248 }` + code.push_set_cur(if_topic_uint.block0); + ctx.storeUint(ir_dest_topic[0], 248); + code.close_pop_cur(loc); + } + { + // input is `dest: ExtOutLogBucket { topic: bits248 }` + // for this field, generate runtime check to ensure its length + code.push_set_cur(if_topic_uint.block1); + ctx.generate_pack_any(t_topic->variants[1], std::vector{ir_dest_topic[0]}); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + + // tail of CommonMsgInfoRelaxed: 64*0 created_lt + 32*0 created_at + // plus, StateInit is empty (0 maybe bit) for external messages + ctx.storeUint(ir_zero, 64 + 32 + 1); + + // fill bit `body: (Either X ^X)` and store body + if (body_size.max_bits == 0 && body_size.max_refs == 0) { + // missing body of type `never` + tolk_assert(ir_body.empty()); + ctx.storeUint(ir_zero, 1); + } else if (body_store_as_ref) { + tolk_assert(ir_body.size() == 1); + ctx.storeUint(ir_one, 1); + ctx.storeRef(ir_body[0]); + } else { + ctx.storeUint(ir_zero, 1); + ctx.generate_pack_any(bodyT, std::move(ir_body)); + } + + std::vector ir_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(msg-cell)"); + code.emplace_back(loc, Op::_Call, ir_cell, std::move(ir_builder), f_endCell); + return ir_cell; +} + +std::vector generate_address_buildInAnotherShard(CodeBlob& code, SrcLocation loc, std::vector&& ir_self_address, std::vector&& ir_shard_options) { + tolk_assert(ir_shard_options.size() == 2); + + // example for fixedPrefixLength (shard depth) = 8: + // | self (A) | aaaaaaaaaaa...aaa | + // | closeTo (B) | 01010101bbb...bbb | shardPrefix = 01010101 (depth 8) + // | result | 01010101aaa...aaa | address of A in same shard as B + + // the most effective way is not to calculate shardPrefix, but to: + // - take first 3+8+D bits of B: we'll have '100' (std addr no anycast) + workchainB + shardPrefix + // - take last 256-D bits of A: we'll have "aa...a" + // - concatenate: we'll result in '100' + workchainB + "bbaa...a" + + std::vector ir_offsetB = {code.create_int(loc, 3 + 8, "(offset-addrB)")}; + code.emplace_back(loc, Op::_Call, ir_offsetB, std::vector{ir_offsetB[0], ir_shard_options[0]}, lookup_function("_+_")); + std::vector ir_headB = code.create_tmp_var(TypeDataSlice::create(), loc, "(headB)"); + code.emplace_back(loc, Op::_Call, ir_headB, std::vector{ir_shard_options[1], ir_offsetB[0]}, lookup_function("slice.getFirstBits")); + + std::vector ir_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{ir_builder[0], ir_headB[0]}, lookup_function("builder.storeSlice")); + + std::vector ir_restLenA = {code.create_int(loc, 256, "(last-addrA)")}; + code.emplace_back(loc, Op::_Call, ir_restLenA, std::vector{ir_restLenA[0], ir_shard_options[0]}, lookup_function("_-_")); + std::vector ir_tailA = code.create_tmp_var(TypeDataSlice::create(), loc, "(tailA)"); + code.emplace_back(loc, Op::_Call, ir_tailA, std::vector{ir_self_address[0], ir_restLenA[0]}, lookup_function("slice.getLastBits")); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{ir_builder[0], ir_tailA[0]}, lookup_function("builder.storeSlice")); + + return ir_builder; +} + +} // namespace tolk diff --git a/tolk/send-message-api.h b/tolk/send-message-api.h new file mode 100644 index 0000000000..1af7389d49 --- /dev/null +++ b/tolk/send-message-api.h @@ -0,0 +1,28 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "tolk.h" + +namespace tolk { + +std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect); +std::vector generate_createExternalLogMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect); + +std::vector generate_address_buildInAnotherShard(CodeBlob& code, SrcLocation loc, std::vector&& ir_self_address, std::vector&& ir_shard_options); + +} // namespace tolk From c8a5b1ba8d72943c4ace58910ce7660135caeba8 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Tue, 10 Jun 2025 18:24:07 +0700 Subject: [PATCH 287/524] [Tolk] Fire an error on using `++/--` operators Before, `++i` was valid (and did nothing, --1 is 1), probably unexpected --- .../tests/invalid-syntax/err-3498.tolk | 12 ++++++++++ .../tests/invalid-syntax/err-3810.tolk | 14 +++++++++++ tolk-tester/tests/no-spaces.tolk | 14 +++++------ tolk-tester/tests/numbers-tests.tolk | 2 +- tolk/ast-from-tokens.cpp | 24 ++++++++++++++++--- tolk/lexer.cpp | 2 ++ tolk/lexer.h | 2 ++ 7 files changed, 59 insertions(+), 11 deletions(-) create mode 100644 tolk-tester/tests/invalid-syntax/err-3498.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3810.tolk diff --git a/tolk-tester/tests/invalid-syntax/err-3498.tolk b/tolk-tester/tests/invalid-syntax/err-3498.tolk new file mode 100644 index 0000000000..204a750cc7 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3498.tolk @@ -0,0 +1,12 @@ + +fun main() { + var z = 5; + ++z; + return z; +} + +/** +@compilation_should_fail +@stderr Tolk has no increment operator +@stderr hint: use `i += 1`, not `i++` + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3810.tolk b/tolk-tester/tests/invalid-syntax/err-3810.tolk new file mode 100644 index 0000000000..a70ae569bb --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3810.tolk @@ -0,0 +1,14 @@ + +struct A { + f: int; +} + +fun main(a: A) { + a.f--; +} + +/** +@compilation_should_fail +@stderr Tolk has no decrement operator +@stderr hint: use `i -= 1`, not `i--` + */ diff --git a/tolk-tester/tests/no-spaces.tolk b/tolk-tester/tests/no-spaces.tolk index 50b2761d7f..857b71e3b2 100644 --- a/tolk-tester/tests/no-spaces.tolk +++ b/tolk-tester/tests/no-spaces.tolk @@ -4,27 +4,27 @@ fun just10(): int { return int10; } fun eq(v: int): int { return`v` } @method_id(101,) fun `get_-1` (): int {return-1} -@method_id(102,) fun `get_--1` (): int {return--1} -@method_id(103,) fun `get_---1`(): int {return---1} -@method_id(104,) fun `get_+++1`(): int {return+++1} +@method_id(102,) fun `get_--1` (): int {return- -1} +@method_id(103,) fun `get_---1`(): int {return-(-(-1))} +@method_id(104,) fun `get_+++1`(): int {return+(+(+1))} @method_id(105,) fun `get_+-+1`(): int {return+-+1} global `some()var`:int; @method_id(110) fun `some_math`(): int { - `some()var`=--6; - return 1*-2*-3*-4*just10()*-5+-`some()var`+--`some()var`---`some()var`; + `some()var`=- -6; + return 1*-2*-3*-4*just10()*-5+-`some()var`+-(-`some()var`)-(-(-`some()var`)); } @method_id(111) fun `negative_nums`(a:int):int { var m$0:int=1; var m1:int=-(+0x1)*m$0; - return `a`*-1*-(1)*---(1)*+just10()+-`just10`()*m1*-m1+-eq(m1)----0x1; + return `a`*-1*-(1)*-(1)*+just10()+-`just10`()*m1*-m1+-eq(m1)-(-0x1); } @method_id(112) fun `bitwise~ops`(flags:int):[bool,bool] { return[ - (just10()-3==just10()-(4)--1)|((2==2)&(eq(eq(10)) -3==just10()--13)), + (just10()-3==just10()-(4)- -1)|((2==2)&(eq(eq(10)) -3==just10()-(-13))), ((flags&0xFF)!=0) ] } diff --git a/tolk-tester/tests/numbers-tests.tolk b/tolk-tester/tests/numbers-tests.tolk index 2dfdee8fdc..7a08740bab 100644 --- a/tolk-tester/tests/numbers-tests.tolk +++ b/tolk-tester/tests/numbers-tests.tolk @@ -26,7 +26,7 @@ fun main() { 0b0 == 0, -0b1010100001010101010-0b1+0b1 == -344746, 0b00001 == 1, - --0b00101111101100 == 3052 + -(-0b00101111101100) == 3052 ); } diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 0ca8b3aebe..9a3a4e6fb0 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -69,6 +69,16 @@ static void fire_error_mix_and_or_no_parenthesis(SrcLocation loc, std::string_vi "Use parenthesis to emphasize operator precedence."); } +// fire an error "Tolk does not have ++i operator" +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_no_increment_operator(SrcLocation loc, bool is_increment) { + std::string op_name = is_increment ? "increment" : "decrement"; + std::string op_wrong = is_increment ? "++" : "--"; + std::string op_right = is_increment ? "+=" : "-="; + throw ParseError(loc, std::string("Tolk has no ") + op_name + " operator\n" + + "hint: use `i " + op_right + " 1`, not `i" + op_wrong + "`"); +} + // diagnose when bitwise operators are used in a probably wrong way due to tricky precedence // example: "flags & 0xFF != 0" is equivalent to "flags & 1", most likely it's unexpected // the only way to suppress this error for the programmer is to use parenthesis @@ -847,7 +857,7 @@ static AnyExprV parse_expr100(Lexer& lex) { } } -// parse E(...) and E! having parsed E already (left-to-right) +// parse E(...) / E! / E++ / E-- having parsed E already (left-to-right) static AnyExprV parse_fun_call_postfix(Lexer& lex, AnyExprV lhs) { while (true) { if (lex.tok() == tok_oppar) { @@ -855,6 +865,8 @@ static AnyExprV parse_fun_call_postfix(Lexer& lex, AnyExprV lhs) { } else if (lex.tok() == tok_logical_not) { lex.next(); lhs = createV(lhs->loc, lhs); + } else if (lex.tok() == tok_double_plus || lex.tok() == tok_double_minus) { + fire_error_no_increment_operator(lex.cur_location(), lex.tok() == tok_double_plus); } else { break; } @@ -865,7 +877,7 @@ static AnyExprV parse_fun_call_postfix(Lexer& lex, AnyExprV lhs) { // parse E(...) and E! (left-to-right) static AnyExprV parse_expr90(Lexer& lex) { AnyExprV res = parse_expr100(lex); - if (lex.tok() == tok_oppar || lex.tok() == tok_logical_not) { + if (lex.tok() == tok_oppar || lex.tok() == tok_logical_not || lex.tok() == tok_double_plus || lex.tok() == tok_double_minus) { res = parse_fun_call_postfix(lex, res); } return res; @@ -892,7 +904,7 @@ static AnyExprV parse_expr80(Lexer& lex) { lex.unexpected("method name"); } lhs = createV(loc, lhs, v_ident, v_instantiationTs); - if (lex.tok() == tok_oppar || lex.tok() == tok_logical_not) { + if (lex.tok() == tok_oppar || lex.tok() == tok_logical_not || lex.tok() == tok_double_plus || lex.tok() == tok_double_minus) { lhs = parse_fun_call_postfix(lex, lhs); } } @@ -909,6 +921,12 @@ static AnyExprV parse_expr75(Lexer& lex) { AnyExprV rhs = parse_expr75(lex); return createV(loc, operator_name, t, rhs); } + if (t == tok_double_minus || t == tok_double_plus) { + SrcLocation loc = lex.cur_location(); + lex.next(); + parse_expr75(lex); + fire_error_no_increment_operator(loc, t == tok_double_plus); + } return parse_expr80(lex); } diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index c31f48762c..94620dc25e 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -523,6 +523,8 @@ struct TolkLanguageGrammar { register_token("^=", 2, tok_set_bitwise_xor); register_token("->", 2, tok_arrow); register_token("=>", 2, tok_double_arrow); + register_token("++", 2, tok_double_plus); + register_token("--", 2, tok_double_minus); register_token("<=>", 3, tok_spaceship); register_token("~>>", 3, tok_rshiftR); register_token("^>>", 3, tok_rshiftC); diff --git a/tolk/lexer.h b/tolk/lexer.h index 19ccb5c105..dbe0f34b71 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -64,6 +64,8 @@ enum TokenType { tok_set_div, tok_mod, tok_set_mod, + tok_double_plus, + tok_double_minus, tok_lshift, tok_set_lshift, tok_rshift, From 79bf8ebcda05ac888e7f0886dfd266e226aac577 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Wed, 11 Jun 2025 18:18:20 +0700 Subject: [PATCH 288/524] [Tolk] Fix unpacking RemainingBitsAndRefs when it's not the last --- tolk-tester/tests/pack-unpack-2.tolk | 22 ++++++++++++++++++++++ tolk/pack-unpack-serializers.cpp | 5 ++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/tolk-tester/tests/pack-unpack-2.tolk b/tolk-tester/tests/pack-unpack-2.tolk index 2a17f7ca19..789ed664f3 100644 --- a/tolk-tester/tests/pack-unpack-2.tolk +++ b/tolk-tester/tests/pack-unpack-2.tolk @@ -375,6 +375,12 @@ struct ReadWriteMid { f3: coins; } +struct WithTwoRestFields { + i32: int32; + rest1: RemainingBitsAndRefs; + rest2: RemainingBitsAndRefs; +} + // --------------------------------------------- @@ -649,6 +655,20 @@ fun test_MidIsBuilderOrBitsN() { return (r.f1, mid.remainingBitsCount(), mid.loadAny(), mid.remainingBitsCount(), r.f3); } +@method_id(226) +fun test_MultipleRemainers() { + val m = WithTwoRestFields.fromSlice(stringHexToSlice("00000001FFFF")); + return (m.i32, m.rest1.remainingBitsCount(), m.rest2.remainingBitsCount()); +} + +@method_id(227) +fun test_mutatingRemainder() { + var cs = stringHexToSlice("00000001FFFF"); + val obj = cs.loadAny(); + cs.assertEnd(); + return obj.rest.remainingBitsCount(); +} + fun main() { var t: JustInt32 = { value: 10 }; @@ -686,4 +706,6 @@ fun main() { @testcase | 223 | | 10 55 (null) @testcase | 224 | | 60 50000000 5 9 123 -1 @testcase | 225 | | 5 40 65535 8 50000000 +@testcase | 226 | | 1 16 0 +@testcase | 227 | | 16 */ diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index 0ffce5fa23..61962615a6 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -419,7 +419,10 @@ struct S_RemainingBitsAndRefs final : ISerializer { } std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { - return ctx->ir_slice; + std::vector ir_rem_slice = code.create_tmp_var(TypeDataSlice::create(), loc, "(remainder)"); + code.emplace_back(loc, Op::_Let, ir_rem_slice, ctx->ir_slice); + code.emplace_back(loc, Op::_Call, ctx->ir_slice, std::vector{}, lookup_function("createEmptySlice")); + return ir_rem_slice; } void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { From 973be6b5f6f5228d40a0778ebc06f19225271de0 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Wed, 11 Jun 2025 18:19:13 +0700 Subject: [PATCH 289/524] [Tolk] Bump version to v0.99 A pre-release one (although many optimizations are yet to come) --- crypto/smartcont/tolk-stdlib/common.tolk | 2 +- crypto/smartcont/tolk-stdlib/gas-payments.tolk | 2 +- crypto/smartcont/tolk-stdlib/lisp-lists.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk | 2 +- tolk/tolk-version.h | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index e522bf2c07..30aabed3a7 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -1,7 +1,7 @@ // Standard library for Tolk (LGPL licence). // It contains common functions that are available out of the box, the user doesn't have to import anything. // More specific functions are required to be imported explicitly, like "@stdlib/tvm-dicts". -tolk 0.13 +tolk 0.99 /// In Tolk v1.x there would be a type `map`. /// Currently, working with dictionaries is still low-level, with raw cells. diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 8fde5f3c32..21de803caa 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.13 +tolk 0.99 /** Gas and payment related primitives. diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk index 14d8f835b8..13625c5caf 100644 --- a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.13 +tolk 0.99 /** Lisp-style lists are nested 2-elements tuples: `[1, [2, [3, null]]]` represents list `[1, 2, 3]`. diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index 9d678e83d8..dc2d3fd6b5 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.13 +tolk 0.99 /** Dictionaries are represented as `cell` data type (cells can store anything, dicts in particular). diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index 7281038d64..724464b706 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.13 +tolk 0.99 /// Usually `c3` has a continuation initialized by the whole code of the contract. It is used for function calls. /// The primitive returns the current value of `c3`. diff --git a/tolk/tolk-version.h b/tolk/tolk-version.h index 5da1582d25..aaf1255d20 100644 --- a/tolk/tolk-version.h +++ b/tolk/tolk-version.h @@ -18,6 +18,6 @@ namespace tolk { -constexpr const char* TOLK_VERSION = "0.13.0"; +constexpr const char* TOLK_VERSION = "0.99.0"; } // namespace tolk From d6eb770f6269ac8292adb1b367071a05d64c08d2 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 13 Jun 2025 11:18:41 +0300 Subject: [PATCH 290/524] Fix initializing neighbor stats in collator --- validator/impl/collator.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index c165b7191a..1c7ad1f276 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1245,6 +1245,7 @@ bool Collator::add_trivial_neighbor_after_merge() { nb.set_queue_root(out_msg_queue_->get_root_cell()); nb.processed_upto = processed_upto_; nb.blk_.id.shard = get_shard(); + stats_.neighbors[i].shard = nb.blk_.shard_full(); LOG(DEBUG) << "adjusted neighbor #" << i << " : " << nb.blk_.to_str() << " with shard expansion (immediate after-merge adjustment)"; } else { @@ -1320,10 +1321,13 @@ bool Collator::add_trivial_neighbor() { CHECK(sibling_out_msg_queue_); CHECK(sibling_processed_upto_); neighbors_.emplace_back(*descr_ref); + stats_.neighbors.push_back(CollationStats::NeighborStats{ + .shard = descr_ref->shard(), .is_trivial = true, .is_local = true, .msg_limit = -1}); auto& nb2 = neighbors_.at(i); nb2.set_queue_root(sibling_out_msg_queue_->get_root_cell()); nb2.processed_upto = sibling_processed_upto_; nb2.blk_.id.shard = ton::shard_sibling(get_shard()); + stats_.neighbors[i].shard = nb2.blk_.shard_full(); LOG(DEBUG) << "adjusted neighbor #" << i << " : " << nb2.blk_.to_str() << " with shard shrinking to our sibling (immediate after-split adjustment)"; auto& nb1 = neighbors_.at(n); @@ -1343,6 +1347,8 @@ bool Collator::add_trivial_neighbor() { CHECK(!sibling_out_msg_queue_); CHECK(!sibling_processed_upto_); neighbors_.emplace_back(*descr_ref); + stats_.neighbors.push_back(CollationStats::NeighborStats{ + .shard = descr_ref->shard(), .is_trivial = true, .is_local = true, .msg_limit = -1}); auto& nb2 = neighbors_.at(i); auto sib_shard = ton::shard_sibling(shard_); // compute the part of virtual sibling's OutMsgQueue with destinations in our shard @@ -1362,6 +1368,7 @@ bool Collator::add_trivial_neighbor() { return fatal_error("error splitting ProcessedUpto for our virtual sibling"); } nb2.blk_.id.shard = ton::shard_sibling(get_shard()); + stats_.neighbors[i].shard = nb2.blk_.shard_full(); LOG(DEBUG) << "adjusted neighbor #" << i << " : " << nb2.blk_.to_str() << " with shard shrinking to our sibling (continued after-split adjustment)"; auto& nb1 = neighbors_.at(n); @@ -6410,6 +6417,14 @@ void Collator::finalize_stats() { }); } stats_.new_out_msg_queue_size = out_msg_queue_size_; + + auto neighbors_stats = std::move(stats_.neighbors); + stats_.neighbors.clear(); + for (size_t i = 0; i < neighbors_stats.size(); ++i) { + if (!neighbors_.at(i).is_disabled()) { + stats_.neighbors.push_back(std::move(neighbors_stats[i])); + } + } } /** From 6a7b2102cca1ca707675541dd3da77c8c3b615af Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Fri, 30 May 2025 16:38:16 -0400 Subject: [PATCH 291/524] Delete unused validator/db/archive-mover.{hpp,cpp} --- validator/db/archive-mover.cpp | 493 --------------------------------- validator/db/archive-mover.hpp | 166 ----------- 2 files changed, 659 deletions(-) delete mode 100644 validator/db/archive-mover.cpp delete mode 100644 validator/db/archive-mover.hpp diff --git a/validator/db/archive-mover.cpp b/validator/db/archive-mover.cpp deleted file mode 100644 index aa89570d96..0000000000 --- a/validator/db/archive-mover.cpp +++ /dev/null @@ -1,493 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "archive-mover.hpp" -#include "td/actor/MultiPromise.h" -#include "validator/fabric.h" - -namespace ton { - -namespace validator { - -void ArchiveFileMover::start_up() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_handle1, std::move(R)); - }); - td::actor::send_closure(archive_manager_, &ArchiveManager::get_handle, block_id_, std::move(P)); -} - -void ArchiveFileMover::got_block_handle0(td::Result R) { - if (R.is_ok()) { - handle_ = R.move_as_ok(); - CHECK(handle_->moved_to_archive()); - CHECK(handle_->handle_moved_to_archive()); - finish_query(); - return; - } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_handle1, std::move(R)); - }); - td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read_handle, block_id_, std::move(P)); -} - -void ArchiveFileMover::got_block_handle1(td::Result R) { - if (R.is_ok()) { - handle_ = R.move_as_ok(); - got_block_handle(); - return; - } - - if (R.error().code() != ErrorCode::notready) { - abort_query(R.move_as_error()); - return; - } - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_handle1, std::move(R)); - }); - td::actor::send_closure(block_db_, &BlockDb::get_block_handle, std::move(P)); -} - -void ArchiveFileMover::got_block_handle2(td::Result R) { - if (R.is_ok()) { - handle_ = R.move_as_ok(); - got_block_handle(); - return; - } - - if (R.error().code() != ErrorCode::notready) { - abort_query(R.move_as_error()); - return; - } - - finish_query(); -} - -void ArchiveFileMover::got_block_handle() { - if (!handle_->is_applied()) { - finish_query(); - return; - } - if (handle_->id().seqno() == 0) { - processed_all_children(); - return; - } - - CHECK(handle_->inited_prev()); - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &ArchiveFileMover::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &ArchiveFileMover::processed_child); - } - }); - - td::actor::create_actor("mover", handle_->one_prev(left_), mode_, block_db_, file_db_, - old_archive_db_, old_archive_manager_, archive_manager_, std::move(P)) - .release(); -} - -void ArchiveFileMover::processed_child() { - if (!left_ || !handle_->merge_before()) { - processed_all_children(); - return; - } - left_ = false; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &ArchiveFileMover::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &ArchiveFileMover::processed_child); - } - }); - - td::actor::create_actor("mover", handle_->one_prev(left_), mode_, block_db_, file_db_, - old_archive_db_, old_archive_manager_, archive_manager_, std::move(P)) - .release(); -} - -void ArchiveFileMover::processed_all_children() { - if (!handle_->received()) { - got_block_data(td::BufferSlice{}); - } else { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_data, std::move(R)); - }); - - if (handle_->moved_to_archive()) { - CHECK(handle_->inited_unix_time()); - td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, handle_->unix_time(), - handle_->is_key_block(), FileDb::RefId{fileref::Block{handle_->id()}}, std::move(P)); - } else { - td::actor::send_closure(handle_->moved_to_storage() ? old_archive_db_ : file_db_, &FileDb::load_file, - FileDb::RefId{fileref::Block{handle_->id()}}, std::move(P)); - } - } -} - -void ArchiveFileMover::got_block_data(td::Result R) { - if (R.is_error()) { - if (R.error().code() != ErrorCode::notready) { - abort_query(R.move_as_error()); - return; - } - } else { - data_ = R.move_as_ok(); - } - if (!handle_->inited_proof()) { - got_block_proof(td::BufferSlice{}); - } else { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_proof, std::move(R)); - }); - - if (handle_->moved_to_archive()) { - CHECK(handle_->inited_unix_time()); - td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, handle_->unix_time(), - handle_->is_key_block(), FileDb::RefId{fileref::Proof{handle_->id()}}, std::move(P)); - } else { - td::actor::send_closure(handle_->moved_to_storage() ? old_archive_db_ : file_db_, &FileDb::load_file, - FileDb::RefId{fileref::Proof{handle_->id()}}, std::move(P)); - } - } -} - -void ArchiveFileMover::got_block_proof(td::Result R) { - if (R.is_error()) { - if (R.error().code() != ErrorCode::notready) { - abort_query(R.move_as_error()); - return; - } - } else { - proof_ = R.move_as_ok(); - } - if (!handle_->inited_proof_link()) { - got_block_proof_link(td::BufferSlice{}); - } else { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_proof_link, std::move(R)); - }); - - if (handle_->moved_to_archive()) { - CHECK(handle_->inited_unix_time()); - td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, handle_->unix_time(), - handle_->is_key_block(), FileDb::RefId{fileref::ProofLink{handle_->id()}}, std::move(P)); - } else { - td::actor::send_closure(handle_->moved_to_storage() ? old_archive_db_ : file_db_, &FileDb::load_file, - FileDb::RefId{fileref::ProofLink{handle_->id()}}, std::move(P)); - } - } -} - -void ArchiveFileMover::got_block_proof_link(td::Result R) { - if (R.is_error()) { - if (R.error().code() != ErrorCode::notready) { - abort_query(R.move_as_error()); - return; - } - } else { - proof_link_ = R.move_as_ok(); - } - - td::MultiPromise mp; - auto ig = mp.init_guard(); - ig.add_promise([SelfId = actor_id(this)](td::Result R) { - R.ensure(); - td::actor::send_closure(SelfId, &ArchiveFileMover::written_data); - }); - - if (data_.size() > 0) { - td::actor::send_closure(archive_manager_, &ArchiveManager::add_file, handle_, fileref::Block{block_id_}, - std::move(data_), ig.get_promise()); - } - if (proof_.size() > 0) { - td::actor::send_closure(archive_manager_, &ArchiveManager::add_file, handle_, fileref::Proof{block_id_}, - std::move(proof_), ig.get_promise()); - } - if (proof_link_.size() > 0) { - td::actor::send_closure(archive_manager_, &ArchiveManager::add_file, handle_, fileref::ProofLink{block_id_}, - std::move(proof_link_), ig.get_promise()); - } -} - -void ArchiveFileMover::written_data() { - handle_->set_moved_to_archive(); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); - td::actor::send_closure(SelfId, &ArchiveFileMover::written_handle); - }); - td::actor::send_closure(archive_manager_, &ArchiveManager::add_handle, handle_, std::move(P)); -} - -void ArchiveFileMover::written_handle() { - CHECK(handle_->handle_moved_to_archive()); - finish_query(); -} - -void ArchiveFileMover::abort_query(td::Status error) { - if (promise_) { - promise_.set_error(std::move(error)); - } - stop(); -} - -void ArchiveFileMover::finish_query() { - if (promise_) { - promise_.set_value(td::Unit()); - } - stop(); -} - -void ArchiveKeyBlockMover::start_up() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_ok()) { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::skip_block_proof, R.move_as_ok()); - } else { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof0); - } - }); - if (proof_link_) { - td::actor::send_closure(archive_manager_, &ArchiveManager::get_file_short, fileref::ProofLink{block_id_}, - std::move(P)); - } else { - td::actor::send_closure(archive_manager_, &ArchiveManager::get_file_short, fileref::Proof{block_id_}, std::move(P)); - } -} - -void ArchiveKeyBlockMover::failed_to_get_proof0() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_ok()) { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::got_block_proof, R.move_as_ok()); - } else { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof1); - } - }); - if (proof_link_) { - td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, fileref::ProofLink{block_id_}, - std::move(P)); - } else { - td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, fileref::Proof{block_id_}, std::move(P)); - } -} - -void ArchiveKeyBlockMover::failed_to_get_proof1() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_ok()) { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::got_block_proof, R.move_as_ok()); - } else { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof2); - } - }); - if (proof_link_) { - td::actor::send_closure(old_archive_db_, &FileDb::load_file, fileref::ProofLink{block_id_}, std::move(P)); - } else { - td::actor::send_closure(old_archive_db_, &FileDb::load_file, fileref::Proof{block_id_}, std::move(P)); - } -} - -void ArchiveKeyBlockMover::failed_to_get_proof2() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_ok()) { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::got_block_proof, R.move_as_ok()); - } else { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof3); - } - }); - if (proof_link_) { - td::actor::send_closure(file_db_, &FileDb::load_file, fileref::ProofLink{block_id_}, std::move(P)); - } else { - td::actor::send_closure(file_db_, &FileDb::load_file, fileref::Proof{block_id_}, std::move(P)); - } -} - -void ArchiveKeyBlockMover::failed_to_get_proof3() { - if (proof_link_) { - written_data(); - } else { - proof_link_ = true; - start_up(); - } -} - -void ArchiveKeyBlockMover::got_block_proof(td::BufferSlice data) { - data_ = std::move(data); - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::written_data); - }); - - if (proof_link_) { - auto p = create_proof_link(block_id_, data_.clone()).move_as_ok(); - auto h = p->get_basic_header_info().move_as_ok(); - td::actor::send_closure(archive_manager_, &ArchiveManager::add_key_block_proof, h.utime, - fileref::ProofLink{block_id_}, std::move(data_), std::move(P)); - } else { - auto p = create_proof(block_id_, data_.clone()).move_as_ok(); - auto h = p->get_basic_header_info().move_as_ok(); - td::actor::send_closure(archive_manager_, &ArchiveManager::add_key_block_proof, h.utime, fileref::Proof{block_id_}, - std::move(data_), std::move(P)); - } -} - -void ArchiveKeyBlockMover::skip_block_proof(td::BufferSlice data) { - data_ = std::move(data); - written_data(); -} - -void ArchiveKeyBlockMover::written_data() { - td::Ref proof_link; - if (proof_link_) { - auto p = create_proof_link(block_id_, data_.clone()).move_as_ok(); - proof_link = std::move(p); - } else { - auto p = create_proof(block_id_, data_.clone()).move_as_ok(); - proof_link = std::move(p); - } - auto ts = proof_link->get_basic_header_info().move_as_ok().utime; - auto te = ValidatorManager::persistent_state_ttl(ts); - if (te < td::Clocks::system()) { - finish_query(); - return; - } -} - -void ArchiveKeyBlockMover::abort_query(td::Status error) { - if (promise_) { - promise_.set_error(std::move(error)); - } - stop(); -} - -void ArchiveKeyBlockMover::finish_query() { - if (promise_) { - promise_.set_value(td::Unit()); - } - stop(); -} - -void ArchiveMover::start_up() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &ArchiveMover::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &ArchiveMover::moved_blocks); - } - }); - td::actor::create_actor("fmover", masterchain_block_id_, block_db_.get(), file_db_.get(), - old_archive_db_.get(), old_archive_manager_.get(), archive_manager_.get(), - std::move(P)) - .release(); -} - -void ArchiveMover::moved_blocks() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); - td::actor::send_closure(SelfId, &ArchiveMover::got_handle, R.move_as_ok()); - }); - td::actor::send_closure(archive_manager_, &ArchiveManager::get_handle, masterchain_block_id_, std::move(P)); -} - -void ArchiveMover::got_handle(BlockHandle handle) { - handle_ = std::move(handle); - CHECK(handle_->is_applied()); - CHECK(handle_->inited_state_boc()); - CHECK(!handle_->deleted_state_boc()); - auto P = td::PromiseCreator::lambda( - [handle = handle_, SelfId = actor_id(this)](td::Result> R) mutable { - R.ensure(); - auto S = create_shard_state(handle->id(), R.move_as_ok()); - S.ensure(); - td::actor::send_closure(SelfId, &ArchiveMover::got_state, td::Ref{S.move_as_ok()}); - }); - td::actor::send_closure(cell_db_, &CellDb::load_cell, handle_->state(), std::move(P)); -} - -void ArchiveMover::got_state(td::Ref state) { - state_ = std::move(state); - - td::MultiPromise mp; - auto ig = mp.init_guard(); - ig.add_promise([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &ArchiveMover::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &ArchiveMover::moved_key_blocks); - } - }); - - auto k = state_->prev_key_block_id(std::numeric_limits::max()); - while (k.is_valid() && k.seqno() > 0) { - td::actor::create_actor("keymover", k, block_db_.get(), file_db_.get(), old_archive_db_.get(), - old_archive_manager_.get(), archive_manager_.get(), ig.get_promise()) - .release(); - k = state_->prev_key_block_id(k.seqno()); - } -} - -void ArchiveMover::moved_key_blocks() { - td::MultiPromise mp; - auto ig = mp.init_guard(); - ig.add_promise([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &ArchiveMover::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &ArchiveMover::moved_key_blocks); - } - }); - - auto k = state_->prev_key_block_id(std::numeric_limits::max()); - while (k.is_valid() && k.seqno() > 0) { - td::actor::create_actor("keymover", k, block_db_.get(), file_db_.get(), old_archive_db_.get(), - old_archive_manager_.get(), archive_manager_.get(), ig.get_promise()) - .release(); - k = state_->prev_key_block_id(k.seqno()); - } -} - -void ArchiveMover::run() { - if (to_move_.empty() && to_check_.empty()) { - completed(); - return; - } - if (!to_check_.empty()) { - auto B = to_check_.back(); - CHECK(to_check_set_.count(B) == 1); - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveMover::got_to_check_handle, std::move(R)); - }); - td::actor::send_closure(block_db_, &BlockDb::get_block_handle, B, std::move(P)); - return; - } - CHECK(!to_move_.empty()); -} - -void ArchiveMover::got_to_check_handle(td::Result R) { - if (R.is_error()) { - CHECK(R.error().code() == ErrorCode::notready); - run(); - return; - } - auto handle = R.move_as_ok(); -} - -} // namespace validator - -} // namespace ton diff --git a/validator/db/archive-mover.hpp b/validator/db/archive-mover.hpp deleted file mode 100644 index be0d8dc754..0000000000 --- a/validator/db/archive-mover.hpp +++ /dev/null @@ -1,166 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -#include "td/actor/actor.h" -#include "filedb.hpp" -#include "blockdb.hpp" -#include "statedb.hpp" -#include "celldb.hpp" -#include "archive-db.hpp" -#include "archive-manager.hpp" - -#include -#include - -namespace ton { - -namespace validator { - -class ArchiveFileMover : public td::actor::Actor { - public: - ArchiveFileMover(BlockIdExt block_id, td::actor::ActorId block_db, td::actor::ActorId file_db, - td::actor::ActorId old_archive_db, td::actor::ActorId old_archive_manager, - td::actor::ActorId archive_manager, td::Promise promise) - : block_id_(block_id) - , block_db_(block_db) - , file_db_(file_db) - , old_archive_db_(old_archive_db) - , old_archive_manager_(old_archive_manager) - , archive_manager_(archive_manager) - , promise_(std::move(promise)) { - } - void start_up() override; - void got_block_handle0(td::Result R); - void got_block_handle1(td::Result R); - void got_block_handle2(td::Result R); - void got_block_handle(); - - void processed_child(); - void processed_all_children(); - - void got_block_data(td::Result R); - void got_block_proof(td::Result R); - void got_block_proof_link(td::Result R); - - void written_data(); - void written_handle(); - - void abort_query(td::Status error); - void finish_query(); - - private: - BlockIdExt block_id_; - BlockHandle handle_; - td::BufferSlice data_; - td::BufferSlice proof_; - td::BufferSlice proof_link_; - bool left_ = true; - - td::actor::ActorId block_db_; - td::actor::ActorId file_db_; - td::actor::ActorId old_archive_db_; - td::actor::ActorId old_archive_manager_; - td::actor::ActorId archive_manager_; - - td::Promise promise_; -}; - -class ArchiveKeyBlockMover : public td::actor::Actor { - public: - ArchiveKeyBlockMover(BlockIdExt block_id, td::actor::ActorId block_db, td::actor::ActorId file_db, - td::actor::ActorId old_archive_db, - td::actor::ActorId old_archive_manager, - td::actor::ActorId archive_manager, td::Promise promise) - : block_id_(block_id) - , block_db_(block_db) - , file_db_(file_db) - , old_archive_db_(old_archive_db) - , old_archive_manager_(old_archive_manager) - , archive_manager_(archive_manager) - , promise_(std::move(promise)) { - } - - void start_up() override; - void failed_to_get_proof0(); - void failed_to_get_proof1(); - void failed_to_get_proof2(); - void failed_to_get_proof3(); - void got_block_proof(td::BufferSlice data); - void skip_block_proof(td::BufferSlice data); - - void written_data(); - - void abort_query(td::Status error); - void finish_query(); - - private: - BlockIdExt block_id_; - td::BufferSlice data_; - bool proof_link_ = false; - - td::actor::ActorId block_db_; - td::actor::ActorId file_db_; - td::actor::ActorId old_archive_db_; - td::actor::ActorId old_archive_manager_; - td::actor::ActorId archive_manager_; - - td::Promise promise_; -}; - -class ArchiveMover : public td::actor::Actor { - public: - ArchiveMover(std::string db_root, BlockIdExt masterchain_block_id, BlockIdExt shard_block_id, - BlockIdExt key_block_id); - - void start_up() override; - void moved_blocks(); - void got_handle(BlockHandle handle); - void got_state(td::Ref state); - void moved_key_blocks(); - void run(); - void completed(); - void add_to_move(BlockIdExt block_id); - void add_to_check(BlockIdExt block_id); - - void got_to_check_handle(td::Result R); - - void abort_query(td::Status error); - void finish_query(); - - private: - std::string db_root_; - BlockHandle handle_; - td::Ref state_; - - td::actor::ActorOwn block_db_; - td::actor::ActorOwn file_db_; - td::actor::ActorOwn old_archive_db_; - td::actor::ActorOwn old_archive_manager_; - td::actor::ActorOwn archive_manager_; - td::actor::ActorOwn cell_db_; - - BlockIdExt masterchain_block_id_; - BlockIdExt shard_block_id_; - BlockIdExt key_block_id_; -}; - -} // namespace validator - -} // namespace ton From af42e201786d5f385501a96eeef94fc721056562 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Tue, 3 Jun 2025 22:24:19 -0400 Subject: [PATCH 292/524] Remove unused FileReference constructor --- validator/db/fileref.cpp | 31 ------------------------------- validator/db/fileref.hpp | 1 - 2 files changed, 32 deletions(-) diff --git a/validator/db/fileref.cpp b/validator/db/fileref.cpp index feefd1f9c6..72aa32cfbd 100644 --- a/validator/db/fileref.cpp +++ b/validator/db/fileref.cpp @@ -258,37 +258,6 @@ std::string BlockInfoShort::filename_short() const { } // namespace fileref -FileReference::FileReference(tl_object_ptr key) { - ton_api::downcast_call( - *key.get(), - td::overloaded( - [&](const ton_api::db_filedb_key_empty& key) { ref_ = fileref::Empty{}; }, - [&](const ton_api::db_filedb_key_blockFile& key) { ref_ = fileref::Block{create_block_id(key.block_id_)}; }, - [&](const ton_api::db_filedb_key_zeroStateFile& key) { - ref_ = fileref::ZeroState{create_block_id(key.block_id_)}; - }, - [&](const ton_api::db_filedb_key_persistentStateFile& key) { - ref_ = fileref::PersistentState{create_block_id(key.block_id_), create_block_id(key.masterchain_block_id_)}; - }, - [&](const ton_api::db_filedb_key_proof& key) { ref_ = fileref::Proof{create_block_id(key.block_id_)}; }, - [&](const ton_api::db_filedb_key_proofLink& key) { - ref_ = fileref::ProofLink{create_block_id(key.block_id_)}; - }, - [&](const ton_api::db_filedb_key_signatures& key) { - ref_ = fileref::Signatures{create_block_id(key.block_id_)}; - }, - [&](const ton_api::db_filedb_key_candidate& key) { - ref_ = fileref::Candidate{PublicKey{key.id_->source_}, create_block_id(key.id_->id_), - key.id_->collated_data_file_hash_}; - }, - [&](const ton_api::db_filedb_key_candidateRef& key) { - ref_ = fileref::CandidateRef{create_block_id(key.id_)}; - }, - [&](const ton_api::db_filedb_key_blockInfo& key) { - ref_ = fileref::BlockInfo{create_block_id(key.block_id_)}; - })); -} - FileReferenceShort FileReference::shortref() const { FileReferenceShort h; ref_.visit([&](const auto& obj) { h = obj.shortref(); }); diff --git a/validator/db/fileref.hpp b/validator/db/fileref.hpp index 6710cc0638..14060fb453 100644 --- a/validator/db/fileref.hpp +++ b/validator/db/fileref.hpp @@ -382,7 +382,6 @@ class FileReference { } FileReference() : ref_(fileref::Empty{}) { } - FileReference(tl_object_ptr key); static td::Result create(std::string filename); From 774371bdc9f6107fd05106c1fd559e8903e0513d Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Thu, 29 May 2025 19:29:43 -0400 Subject: [PATCH 293/524] Store `archive_split_depth` in WorkchainDescr in config The parameter will be used in later commits to denote number of parts we split archived shard states into. --- crypto/block/block.tlb | 1 + crypto/block/mc-config.cpp | 1 + crypto/block/mc-config.h | 2 ++ validator/impl/shard.cpp | 8 ++++++++ validator/impl/shard.hpp | 1 + validator/interfaces/shard.h | 1 + 6 files changed, 14 insertions(+) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 99dfdb5223..93b1f3ad76 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -670,6 +670,7 @@ workchain_v2#a7 enabled_since:uint32 monitor_min_split:(## 8) zerostate_root_hash:bits256 zerostate_file_hash:bits256 version:uint32 format:(WorkchainFormat basic) split_merge_timings:WcSplitMergeTimings + persistent_state_split_depth:(## 8) { persistent_state_split_depth <= 63 } = WorkchainDescr; _ workchains:(HashmapE 32 WorkchainDescr) = ConfigParam 12; diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 78e2294f88..35c3df0055 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2199,6 +2199,7 @@ bool WorkchainInfo::unpack(ton::WorkchainId wc, vm::CellSlice& cs) { split_merge_interval = rec.split_merge_interval; min_split_merge_interval = rec.min_split_merge_interval; max_split_merge_delay = rec.max_split_merge_delay; + persistent_state_split_depth = info.persistent_state_split_depth; return true; }; block::gen::WorkchainDescr::Record_workchain info_v1; diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 95e107352b..34fab4df40 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -438,6 +438,8 @@ struct WorkchainInfo : public td::CntObject { unsigned min_split_merge_interval = 30; // split/merge interval must be at least 30 seconds unsigned max_split_merge_delay = 1000; // end of split/merge interval must be at most 1000 seconds in the future + td::uint32 persistent_state_split_depth = 0; + bool is_valid() const { return workchain != ton::workchainInvalid; } diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index d2d402dd66..d9a78b2de2 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -524,6 +524,14 @@ bool MasterchainStateQ::check_old_mc_block_id(const ton::BlockIdExt& blkid, bool return config_ && config_->check_old_mc_block_id(blkid, strict); } +td::uint32 MasterchainStateQ::persistent_state_split_depth(WorkchainId workchain_id) const { + if (!config_) { + return 0; + } + auto wc_info = config_->get_workchain_info(workchain_id); + return wc_info.not_null() ? wc_info->persistent_state_split_depth : 0; +} + td::uint32 MasterchainStateQ::monitor_min_split_depth(WorkchainId workchain_id) const { if (!config_) { return 0; diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index fa36e1e668..a896f3eb0a 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -124,6 +124,7 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { bool has_workchain(WorkchainId workchain) const { return config_ && config_->has_workchain(workchain); } + td::uint32 persistent_state_split_depth(WorkchainId workchain_id) const override; td::uint32 monitor_min_split_depth(WorkchainId workchain_id) const override; td::uint32 min_split_depth(WorkchainId workchain_id) const override; BlockSeqno min_ref_masterchain_seqno() const override; diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 64aea9b62b..96767cbf9d 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -71,6 +71,7 @@ class MasterchainState : virtual public ShardState { virtual std::vector> get_shards() const = 0; virtual td::Ref get_shard_from_config(ShardIdFull shard) const = 0; virtual bool workchain_is_active(WorkchainId workchain_id) const = 0; + virtual td::uint32 persistent_state_split_depth(WorkchainId workchain_id) const = 0; virtual td::uint32 monitor_min_split_depth(WorkchainId workchain_id) const = 0; virtual td::uint32 min_split_depth(WorkchainId workchain_id) const = 0; virtual BlockSeqno min_ref_masterchain_seqno() const = 0; From 7fefaa9a627da246cc2684b113a438c15b60991e Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Tue, 3 Jun 2025 22:30:13 -0400 Subject: [PATCH 294/524] Add initial support for split persistent states in ArchiveManager For now, we only make implementation ready for them without changing public API in any way. In particular, it is not possible to create a split state or request it. --- tl/generate/scheme/ton_api.tl | 4 +- tl/generate/scheme/ton_api.tlo | Bin 103024 -> 103388 bytes validator/db/archive-manager.cpp | 63 ++++++++++++++----- validator/db/archive-manager.hpp | 3 +- validator/db/fileref.cpp | 57 +++++++++++++++++ validator/db/fileref.hpp | 78 +++++++++++++++++++++++- validator/interfaces/persistent-state.h | 38 ++++++++++++ 7 files changed, 223 insertions(+), 20 deletions(-) create mode 100644 validator/interfaces/persistent-state.h diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 580d74861c..06f202a7f2 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -542,7 +542,9 @@ db.candidate.id source:PublicKey id:tonNode.blockIdExt collated_data_file_hash:i db.filedb.key.empty = db.filedb.Key; db.filedb.key.blockFile block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.zeroStateFile block_id:tonNode.blockIdExt = db.filedb.Key; -db.filedb.key.persistentStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt = db.filedb.Key; +db.filedb.key.persistentStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt = db.filedb.Key; +db.filedb.key.splitAccountStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt effective_shard:long = db.filedb.Key; +db.filedb.key.splitPersistentStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.proof block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.proofLink block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.signatures block_id:tonNode.blockIdExt = db.filedb.Key; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index ae1785ebb566de1406a4792679287455daaccfc3..f30418f7bd78ff6a6f1b8519fe1c506e3f5999ad 100644 GIT binary patch delta 419 zcmeycgze69whac{q6_wz)bXYy>7`}nqyj1L)Jg^ho{5g)n`5|-=reBHtZ3bDF9A|? z^O%qi0}vEBv*~cywjBhCO+Hbj!2)I+s4;;sbgFgGM5a_jrOs5RID!;|OaXzba$GTK zOLLAXBdpI(t<)`*4-fNp%e@rK!ISd0%yEh^3|E=fhQfdT9fpc^61H>puU_eTx7Uw_n~ u`!%5!s>YyPV)Bw&5r`3cYN6^3%ApLII&`xd>TJM@n-A1&KTyZGK^OoTf~5ig delta 254 zcmcb!obAIBwhac{qBFH#DD$Qy>7`}nqyj1L)Jg^hu8EG~n`5|-=reBFtZ3bDF9A|? z^O%qi0}vEBv*~cywjBhCO+Hbj!2)K)RGUB;I@LO8B2!>eXR1>mQVVKTP}NMTLpLU+ u4&9hDFf|6{5|ah$MIa_<)I-%7ltURE_2_23skZ?uZq{kouG7G{K^Oqe_g{Yi diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 9e6ea797e3..926e7c3b5e 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -311,9 +311,6 @@ void ArchiveManager::get_file(ConstBlockHandle handle, FileReference ref_id, td: } void ArchiveManager::register_perm_state(FileReferenceShort id) { - BlockSeqno masterchain_seqno = 0; - id.ref().visit(td::overloaded( - [&](const fileref::PersistentStateShort &x) { masterchain_seqno = x.masterchain_seqno; }, [&](const auto &) {})); td::uint64 size; auto r_stat = td::stat(db_root_ + "/archive/states/" + id.filename_short()); if (r_stat.is_error()) { @@ -322,7 +319,7 @@ void ArchiveManager::register_perm_state(FileReferenceShort id) { } else { size = r_stat.ok().size_; } - perm_states_[{masterchain_seqno, id.hash()}] = {.id = id, .size = size}; + perm_states_[{id.seqno_of_persistent_state(), id.hash()}] = {.id = id, .size = size}; } void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise promise) { @@ -347,6 +344,23 @@ void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, t .release(); } +namespace { + +FileReferenceShort create_persistent_state_id(BlockIdExt block_id, BlockIdExt mc_block_id, PersistentStateType type) { + FileReferenceShort result; + type.visit(td::overloaded( + [&](UnsplitStateType const &) { result = fileref::PersistentStateShort::create(block_id, mc_block_id); }, + [&](SplitAccountStateType const &account_state) { + result = fileref::SplitAccountState::create(block_id, mc_block_id, account_state.effective_shard_id); + }, + [&](SplitPersistentStateType const &persistent_state) { + result = fileref::SplitPersistentState::create(block_id, mc_block_id); + })); + return result; +} + +} // namespace + void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data, td::Promise promise) { auto create_writer = [&](std::string path, td::Promise P) { @@ -354,7 +368,9 @@ void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt master std::move(P)) .release(); }; - add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer)); + // TODO: Allow specifying state type. + add_persistent_state_impl(create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}), + std::move(promise), std::move(create_writer)); } void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, @@ -365,23 +381,22 @@ void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt ma std::move(write_state), std::move(P)) .release(); }; - add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer)); + // TODO: Allow specifying state type. + add_persistent_state_impl(create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}), + std::move(promise), std::move(create_writer)); } void ArchiveManager::add_persistent_state_impl( - BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise, + FileReferenceShort const &id, td::Promise promise, std::function)> create_writer) { - auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; - BlockSeqno masterchain_seqno = masterchain_block_id.seqno(); - auto hash = id.hash(); - if (perm_states_.find({masterchain_seqno, hash}) != perm_states_.end()) { + if (perm_states_.find({id.seqno_of_persistent_state(), id.hash()}) != perm_states_.end()) { promise.set_value(td::Unit()); return; } auto path = db_root_ + "/archive/states/" + id.filename_short(); auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), id = id.shortref(), promise = std::move(promise)](td::Result R) mutable { + [SelfId = actor_id(this), id = id, promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); } else { @@ -436,7 +451,8 @@ void ArchiveManager::get_previous_persistent_state_files( void ArchiveManager::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) { - auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; + // TODO: Allow specifying state type. + auto id = create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}); auto hash = id.hash(); if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db")); @@ -449,7 +465,8 @@ void ArchiveManager::get_persistent_state(BlockIdExt block_id, BlockIdExt master void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, td::int64 max_size, td::Promise promise) { - auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; + // TODO: Allow specifying state type. + auto id = create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}); auto hash = id.hash(); if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db")); @@ -462,7 +479,8 @@ void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt void ArchiveManager::get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) { - auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; + // TODO: Allow specifying state type. + auto id = create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}); auto hash = id.hash(); auto it = perm_states_.find({masterchain_block_id.seqno(), hash}); if (it == perm_states_.end()) { @@ -918,6 +936,11 @@ void ArchiveManager::start_up() { R = FileReferenceShort::create(newfname); R.ensure(); } + if (!R.ok().is_state_like()) { + LOG(ERROR) << "deleting file that is not state-like '" << fname << "'"; + td::unlink(db_root_ + "/archive/states/" + fname.str()).ignore(); + return; + } register_perm_state(R.move_as_ok()); } }).ensure(); @@ -1037,6 +1060,14 @@ void ArchiveManager::persistent_state_gc(std::pair last) { res = 0; seqno = x.masterchain_seqno; }, + [&](const fileref::SplitAccountState &x) { + res = 0; + seqno = x.masterchain_seqno; + }, + [&](const fileref::SplitPersistentState &x) { + res = 0; + seqno = x.masterchain_seqno; + }, [&](const auto &obj) { res = -1; })); if (res == -1) { @@ -1315,6 +1346,8 @@ void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle han it->second.id.ref().visit(td::overloaded( [&](const fileref::ZeroStateShort &x) { res = -1; }, [&](const fileref::PersistentStateShort &x) { res = x.masterchain_seqno <= masterchain_seqno ? -1 : 1; }, + [&](const fileref::SplitPersistentState &x) { res = x.masterchain_seqno <= masterchain_seqno ? -1 : 1; }, + [&](const fileref::SplitAccountState &x) { res = x.masterchain_seqno <= masterchain_seqno ? -1 : 1; }, [&](const auto &obj) { res = 1; })); if (res <= 0) { it++; diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index 33dd1f3d6f..361eaf5162 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -19,6 +19,7 @@ #pragma once #include "archive-slice.hpp" +#include "interfaces/persistent-state.h" namespace ton { @@ -219,7 +220,7 @@ class ArchiveManager : public td::actor::Actor { PackageId get_max_temp_file_desc_idx(); PackageId get_prev_temp_file_desc_idx(PackageId id); - void add_persistent_state_impl(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise, + void add_persistent_state_impl(FileReferenceShort const &id, td::Promise promise, std::function)> create_writer); void register_perm_state(FileReferenceShort id); diff --git a/validator/db/fileref.cpp b/validator/db/fileref.cpp index 72aa32cfbd..4e87327dc6 100644 --- a/validator/db/fileref.cpp +++ b/validator/db/fileref.cpp @@ -106,6 +106,16 @@ std::string ZeroStateShort::filename_short() const { return PSTRING() << "zerostate_" << workchain << "_" << hash().to_hex(); } +std::string SplitAccountState::filename_short() const { + return PSTRING() << "stateaccount_" << masterchain_seqno << "_" << shard_id.workchain << "_" + << shard_to_str(shard_id.shard) << "_" << shard_to_str(effective_shard) << "_" << hash().to_hex(); +} + +std::string SplitPersistentState::filename_short() const { + return PSTRING() << "statesplit_" << masterchain_seqno << "_" << shard_id.workchain << "_" + << shard_to_str(shard_id.shard) << "_" << hash().to_hex(); +} + PersistentStateShort PersistentState::shortref() const { return PersistentStateShort{block_id.shard_full(), masterchain_block_id.seqno(), hash()}; } @@ -288,6 +298,24 @@ ShardIdFull FileReferenceShort::shard() const { return h; } +BlockSeqno FileReferenceShort::seqno_of_persistent_state() const { + BlockSeqno result; + ref_.visit(td::overloaded([&](const fileref::PersistentStateShort& x) { result = x.masterchain_seqno; }, + [&](const fileref::SplitAccountState& x) { result = x.masterchain_seqno; }, + [&](const fileref::SplitPersistentState& x) { result = x.masterchain_seqno; }, + [&](const fileref::ZeroStateShort) { result = 0; }, [&](const auto&) { CHECK(false); })); + return result; +} + +bool FileReferenceShort::is_state_like() const { + bool result = false; + ref_.visit(td::overloaded([&](const fileref::PersistentStateShort& x) { result = true; }, + [&](const fileref::SplitAccountState& x) { result = true; }, + [&](const fileref::SplitPersistentState& x) { result = true; }, + [&](const fileref::ZeroStateShort) { result = true; }, [&](const auto&) {})); + return result; +} + std::string FileReference::filename() const { std::string h; ref_.visit([&](const auto& obj) { h = obj.filename(); }); @@ -420,6 +448,35 @@ td::Result FileReferenceShort::create(std::string filename) } else { return td::Status::Error(ErrorCode::protoviolation, "too big file name"); } + } else if (token == "stateaccount") { + std::getline(ss, token, '_'); + TRY_RESULT(masterchain_seqno, td::to_integer_safe(token)); + std::getline(ss, token, '_'); + TRY_RESULT(workchain, td::to_integer_safe(token)); + std::getline(ss, token, '_'); + TRY_RESULT(shard, td::hex_to_integer_safe(token)); + std::getline(ss, token, '_'); + TRY_RESULT(effective_shard, td::hex_to_integer_safe(token)); + TRY_RESULT(vhash, get_token_hash(ss)); + if (!ss.eof()) { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + if (!shard_is_proper_ancestor(shard, effective_shard)) { + return td::Status::Error(ErrorCode::protoviolation, "shard is not a proper ancestor of effective shard"); + } + return fileref::SplitAccountState{ShardIdFull{workchain, shard}, effective_shard, masterchain_seqno, vhash}; + } else if (token == "statesplit") { + std::getline(ss, token, '_'); + TRY_RESULT(masterchain_seqno, td::to_integer_safe(token)); + std::getline(ss, token, '_'); + TRY_RESULT(workchain, td::to_integer_safe(token)); + std::getline(ss, token, '_'); + TRY_RESULT(shard, td::hex_to_integer_safe(token)); + TRY_RESULT(vhash, get_token_hash(ss)); + if (!ss.eof()) { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + return fileref::SplitPersistentState{ShardIdFull{workchain, shard}, masterchain_seqno, vhash}; } else if (token == "state") { std::getline(ss, token, '_'); TRY_RESULT(masterchain_seqno, td::to_integer_safe(token)); diff --git a/validator/db/fileref.hpp b/validator/db/fileref.hpp index 14060fb453..df368f759c 100644 --- a/validator/db/fileref.hpp +++ b/validator/db/fileref.hpp @@ -110,8 +110,77 @@ class ZeroState { BlockIdExt block_id; }; +class SplitAccountState { + public: + static SplitAccountState create(BlockIdExt block_id, BlockIdExt masterchain_block_id, ShardId effective_shard) { + auto hash = create_hash_tl_object( + create_tl_block_id(block_id), create_tl_block_id(masterchain_block_id), effective_shard); + + return { + .shard_id = block_id.shard_full(), + .effective_shard = effective_shard, + .masterchain_seqno = masterchain_block_id.seqno(), + .hashv = hash, + }; + } + + FileHash hash() const { + return hashv; + } + + ShardIdFull shard() const { + return {shard_id.workchain, effective_shard}; + } + + std::string filename_short() const; + + ShardIdFull shard_id; + ShardId effective_shard; + BlockSeqno masterchain_seqno; + FileHash hashv; +}; + +class SplitPersistentState { + public: + static SplitPersistentState create(BlockIdExt block_id, BlockIdExt masterchain_block_id) { + auto hash = create_hash_tl_object( + create_tl_block_id(block_id), create_tl_block_id(masterchain_block_id)); + + return { + .shard_id = block_id.shard_full(), + .masterchain_seqno = masterchain_block_id.seqno(), + .hashv = hash, + }; + } + + FileHash hash() const { + return hashv; + } + + ShardIdFull shard() const { + return shard_id; + } + + std::string filename_short() const; + + ShardIdFull shard_id; + BlockSeqno masterchain_seqno; + FileHash hashv; +}; + class PersistentStateShort { public: + static PersistentStateShort create(BlockIdExt block_id, BlockIdExt masterchain_block_id) { + auto hash = create_hash_tl_object( + create_tl_block_id(block_id), create_tl_block_id(masterchain_block_id)); + + return { + .shard_id = block_id.shard_full(), + .masterchain_seqno = masterchain_block_id.seqno(), + .hashv = hash, + }; + } + FileHash hash() const { return hashv; } @@ -346,9 +415,10 @@ class BlockInfo { class FileReferenceShort { private: - td::Variant + td::Variant ref_; public: @@ -366,6 +436,8 @@ class FileReferenceShort { td::Bits256 hash() const; ShardIdFull shard() const; + BlockSeqno seqno_of_persistent_state() const; + bool is_state_like() const; std::string filename_short() const; }; diff --git a/validator/interfaces/persistent-state.h b/validator/interfaces/persistent-state.h new file mode 100644 index 0000000000..4a03a8f2b9 --- /dev/null +++ b/validator/interfaces/persistent-state.h @@ -0,0 +1,38 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "td/utils/Variant.h" +#include "ton/ton-types.h" + +namespace ton { + +namespace validator { + +struct UnsplitStateType {}; + +struct SplitAccountStateType { + ShardId effective_shard_id; +}; + +struct SplitPersistentStateType {}; + +using PersistentStateType = td::Variant; + +} // namespace validator + +} // namespace ton From f3c43560de8e06b473a2ad36aba75e69dff66877 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Wed, 4 Jun 2025 09:52:36 -0400 Subject: [PATCH 295/524] Expose PersistentStateType and pass it through a million actors What are we, an enterprise? More seriously, the commit makes it a responsibility of (indirect) users of ArchiveManager to pass PersistentStateType. For now, we still hardcode UnsplitStateType everywhere. --- validator/db/archive-manager.cpp | 32 +++++++++++------------- validator/db/archive-manager.hpp | 18 ++++++------- validator/db/rootdb.cpp | 24 ++++++++++-------- validator/db/rootdb.hpp | 15 +++++------ validator/downloaders/download-state.cpp | 2 +- validator/full-node-master.cpp | 8 +++--- validator/full-node-shard.cpp | 8 +++--- validator/interfaces/db.h | 14 +++++++---- validator/interfaces/validator-manager.h | 4 ++- validator/manager-disk.cpp | 19 ++++++++------ validator/manager-disk.hpp | 17 +++++++------ validator/manager-hardfork.hpp | 17 +++++++------ validator/manager.cpp | 25 ++++++++++-------- validator/manager.hpp | 17 +++++++------ validator/net/download-state.cpp | 2 +- validator/state-serializer.cpp | 4 +-- validator/validator.h | 10 +++++--- 17 files changed, 127 insertions(+), 109 deletions(-) diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 926e7c3b5e..f3d8b94253 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -361,19 +361,20 @@ FileReferenceShort create_persistent_state_id(BlockIdExt block_id, BlockIdExt mc } // namespace -void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data, +void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::BufferSlice data, td::Promise promise) { auto create_writer = [&](std::string path, td::Promise P) { td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", std::move(path), std::move(data), std::move(P)) .release(); }; - // TODO: Allow specifying state type. - add_persistent_state_impl(create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}), - std::move(promise), std::move(create_writer)); + add_persistent_state_impl(create_persistent_state_id(block_id, masterchain_block_id, type), std::move(promise), + std::move(create_writer)); } void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, std::function write_state, td::Promise promise) { auto create_writer = [&](std::string path, td::Promise P) { @@ -381,9 +382,8 @@ void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt ma std::move(write_state), std::move(P)) .release(); }; - // TODO: Allow specifying state type. - add_persistent_state_impl(create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}), - std::move(promise), std::move(create_writer)); + add_persistent_state_impl(create_persistent_state_id(block_id, masterchain_block_id, type), std::move(promise), + std::move(create_writer)); } void ArchiveManager::add_persistent_state_impl( @@ -450,9 +450,8 @@ void ArchiveManager::get_previous_persistent_state_files( } void ArchiveManager::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - // TODO: Allow specifying state type. - auto id = create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}); + PersistentStateType type, td::Promise promise) { + auto id = create_persistent_state_id(block_id, masterchain_block_id, type); auto hash = id.hash(); if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db")); @@ -463,10 +462,10 @@ void ArchiveManager::get_persistent_state(BlockIdExt block_id, BlockIdExt master td::actor::create_actor("readfile", path, 0, -1, 0, std::move(promise)).release(); } -void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_size, td::Promise promise) { - // TODO: Allow specifying state type. - auto id = create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}); +void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::int64 offset, td::int64 max_size, + td::Promise promise) { + auto id = create_persistent_state_id(block_id, masterchain_block_id, type); auto hash = id.hash(); if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db")); @@ -478,9 +477,8 @@ void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt } void ArchiveManager::get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - // TODO: Allow specifying state type. - auto id = create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}); + PersistentStateType type, td::Promise promise) { + auto id = create_persistent_state_id(block_id, masterchain_block_id, type); auto hash = id.hash(); auto it = perm_states_.find({masterchain_block_id.seqno(), hash}); if (it == perm_states_.end()) { diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index 361eaf5162..0d9a59df39 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -44,16 +44,16 @@ class ArchiveManager : public td::actor::Actor { void get_file(ConstBlockHandle handle, FileReference ref_id, td::Promise promise); void add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise promise); - void add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data, - td::Promise promise); - void add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_state, - td::Promise promise); + void add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::BufferSlice data, td::Promise promise); + void add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + std::function write_state, td::Promise promise); void get_zero_state(BlockIdExt block_id, td::Promise promise); - void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); - void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_size, td::Promise promise); - void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::Promise promise); + void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::int64 offset, td::int64 max_size, td::Promise promise); + void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise); void check_zero_state(BlockIdExt block_id, td::Promise promise); void get_previous_persistent_state_files(BlockSeqno cur_mc_seqno, diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 90546fbef9..d041228c06 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -285,35 +285,37 @@ void RootDb::get_cell_db_reader(td::Promise> p td::actor::send_closure(cell_db_, &CellDb::get_cell_db_reader, std::move(promise)); } -void RootDb::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, - td::Promise promise) { - td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state, block_id, masterchain_block_id, +void RootDb::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::BufferSlice state, td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state, block_id, masterchain_block_id, type, std::move(state), std::move(promise)); } void RootDb::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, std::function write_data, td::Promise promise) { - td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state_gen, block_id, masterchain_block_id, + td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state_gen, block_id, masterchain_block_id, type, std::move(write_data), std::move(promise)); } -void RootDb::get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, +void RootDb::get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) { - td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state, block_id, masterchain_block_id, + td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state, block_id, masterchain_block_id, type, std::move(promise)); } -void RootDb::get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_size, td::Promise promise) { +void RootDb::get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::int64 offset, td::int64 max_size, + td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state_slice, block_id, masterchain_block_id, - offset, max_size, std::move(promise)); + type, offset, max_size, std::move(promise)); } void RootDb::get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { + PersistentStateType type, td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state_file_size, block_id, masterchain_block_id, - std::move(promise)); + type, std::move(promise)); } void RootDb::store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) { diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 2ccd7f942d..57e560e1c4 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -71,16 +71,17 @@ class RootDb : public Db { td::actor::send_closure(validator_manager_, &ValidatorManager::get_block_handle, id, force, std::move(promise)); } - void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, - td::Promise promise) override; - void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::BufferSlice state, td::Promise promise) override; + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, std::function write_data, td::Promise promise) override; - void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override; - void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_length, td::Promise promise) override; - void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::int64 offset, td::int64 max_length, + td::Promise promise) override; + void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; void get_zero_state_file(BlockIdExt block_id, td::Promise promise) override; diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index 8473cb228b..4335a29568 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -189,7 +189,7 @@ void DownloadShardState::checked_shard_state() { std::move(P)); } else { td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file, block_id_, masterchain_block_id_, - std::move(data_), std::move(P)); + UnsplitStateType{}, std::move(data_), std::move(P)); } } diff --git a/validator/full-node-master.cpp b/validator/full-node-master.cpp index 8dc2a6815b..996ce2219e 100644 --- a/validator/full-node-master.cpp +++ b/validator/full-node-master.cpp @@ -288,7 +288,7 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, - masterchain_block_id, std::move(P)); + masterchain_block_id, UnsplitStateType{}, std::move(P)); } void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, @@ -301,7 +301,7 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, - masterchain_block_id, std::move(P)); + masterchain_block_id, UnsplitStateType{}, std::move(P)); } void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, @@ -361,7 +361,7 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id, - masterchain_block_id, std::move(P)); + masterchain_block_id, UnsplitStateType{}, std::move(P)); } void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, @@ -378,7 +378,7 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, - masterchain_block_id, query.offset_, query.max_size_, std::move(P)); + masterchain_block_id, UnsplitStateType{}, query.offset_, query.max_size_, std::move(P)); } void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 2543431a81..db629c8981 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -523,7 +523,7 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod VLOG(FULL_NODE_DEBUG) << "Got query preparePersistentState " << block_id.to_str() << " " << masterchain_block_id.to_str() << " from " << src; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, - masterchain_block_id, std::move(P)); + masterchain_block_id, UnsplitStateType{}, std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, @@ -538,7 +538,7 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod VLOG(FULL_NODE_DEBUG) << "Got query getPersistentStateSize " << block_id.to_str() << " " << masterchain_block_id.to_str() << " from " << src; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, - masterchain_block_id, std::move(P)); + masterchain_block_id, UnsplitStateType{}, std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, @@ -607,7 +607,7 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod VLOG(FULL_NODE_DEBUG) << "Got query downloadPersistentState " << block_id.to_str() << " " << masterchain_block_id.to_str() << " from " << src; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, - masterchain_block_id, 0, max_size + 1, std::move(P)); + masterchain_block_id, UnsplitStateType{}, 0, max_size + 1, std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, @@ -631,7 +631,7 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_value(R.move_as_ok()); }); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, - masterchain_block_id, query.offset_, query.max_size_, std::move(P)); + masterchain_block_id, UnsplitStateType{}, query.offset_, query.max_size_, std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index f21c6f9028..afd087549f 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -20,6 +20,7 @@ #include "ton/ton-types.h" #include "validator/interfaces/block-handle.h" +#include "validator/interfaces/persistent-state.h" #include "validator/interfaces/validator-manager.h" namespace ton { @@ -53,17 +54,20 @@ class Db : public td::actor::Actor { virtual void get_block_state(ConstBlockHandle handle, td::Promise> promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; - virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, + virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::BufferSlice state, td::Promise promise) = 0; virtual void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, std::function write_data, td::Promise promise) = 0; - virtual void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, + virtual void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) = 0; - virtual void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_length, td::Promise promise) = 0; + virtual void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::int64 offset, td::int64 max_length, + td::Promise promise) = 0; virtual void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) = 0; + PersistentStateType type, td::Promise promise) = 0; virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) = 0; virtual void get_zero_state_file(BlockIdExt block_id, td::Promise promise) = 0; virtual void check_zero_state_file_exists(BlockIdExt block_id, td::Promise promise) = 0; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 077ce7849a..2aebaca747 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -71,9 +71,11 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; - virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, + virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::BufferSlice state, td::Promise promise) = 0; virtual void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, std::function write_data, td::Promise promise) = 0; virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 82081cd11a..c48528b48c 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -201,13 +201,14 @@ void ValidatorManagerImpl::get_zero_state(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, + PersistentStateType type, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, type, std::move(promise)); } void ValidatorManagerImpl::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - td::actor::send_closure(db_, &Db::get_persistent_state_file, block_id, masterchain_block_id, std::move(promise)); + PersistentStateType type, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file, block_id, masterchain_block_id, type, + std::move(promise)); } void ValidatorManagerImpl::get_block_proof(BlockHandle handle, td::Promise promise) { @@ -692,15 +693,17 @@ void ValidatorManagerImpl::get_cell_db_reader(td::Promise promise) { - td::actor::send_closure(db_, &Db::store_persistent_state_file, block_id, masterchain_block_id, std::move(state), + PersistentStateType type, td::BufferSlice state, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_persistent_state_file, block_id, masterchain_block_id, type, std::move(state), std::move(promise)); } void ValidatorManagerImpl::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + PersistentStateType type, + std::function write_data, td::Promise promise) { - td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id, + td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id, type, std::move(write_data), std::move(promise)); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index cdf6c1caef..ee83fc850b 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -108,12 +108,13 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_data(BlockHandle handle, td::Promise promise) override; void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override; void get_zero_state(BlockIdExt block_id, td::Promise promise) override; - void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override; - void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override; - void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_length, td::Promise promise) override { + void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::int64 offset, td::int64 max_length, + td::Promise promise) override { UNREACHABLE(); } void get_previous_persistent_state_files( @@ -150,10 +151,10 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; void get_cell_db_reader(td::Promise> promise) override; - void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, - td::Promise promise) override; - void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::BufferSlice state, td::Promise promise) override; + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + std::function write_data, td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index fbb40901f5..ecfc8de7ed 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -128,16 +128,17 @@ class ValidatorManagerImpl : public ValidatorManager { void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override { UNREACHABLE(); } - void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override { UNREACHABLE(); } - void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override { UNREACHABLE(); } - void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_length, td::Promise promise) override { + void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::int64 offset, td::int64 max_length, + td::Promise promise) override { UNREACHABLE(); } void get_previous_persistent_state_files( @@ -176,12 +177,12 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } void get_cell_db_reader(td::Promise> promise) override; - void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, - td::Promise promise) override { + void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::BufferSlice state, td::Promise promise) override { UNREACHABLE(); } - void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + std::function write_data, td::Promise promise) override { UNREACHABLE(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index a94f9c07ca..86d794bc87 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -305,20 +305,21 @@ void ValidatorManagerImpl::get_zero_state(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, + PersistentStateType type, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, type, std::move(promise)); } void ValidatorManagerImpl::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - td::actor::send_closure(db_, &Db::get_persistent_state_file, block_id, masterchain_block_id, std::move(promise)); + PersistentStateType type, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file, block_id, masterchain_block_id, type, + std::move(promise)); } void ValidatorManagerImpl::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::int64 offset, td::int64 max_length, + PersistentStateType type, td::int64 offset, td::int64 max_length, td::Promise promise) { - td::actor::send_closure(db_, &Db::get_persistent_state_file_slice, block_id, masterchain_block_id, offset, max_length, - std::move(promise)); + td::actor::send_closure(db_, &Db::get_persistent_state_file_slice, block_id, masterchain_block_id, type, offset, + max_length, std::move(promise)); } void ValidatorManagerImpl::get_previous_persistent_state_files( @@ -1237,15 +1238,17 @@ void ValidatorManagerImpl::get_cell_db_reader(td::Promise promise) { - td::actor::send_closure(db_, &Db::store_persistent_state_file, block_id, masterchain_block_id, std::move(state), + PersistentStateType type, td::BufferSlice state, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_persistent_state_file, block_id, masterchain_block_id, type, std::move(state), std::move(promise)); } void ValidatorManagerImpl::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + PersistentStateType type, + std::function write_data, td::Promise promise) { - td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id, + td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id, type, std::move(write_data), std::move(promise)); } diff --git a/validator/manager.hpp b/validator/manager.hpp index a321684b77..ab86d2d372 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -378,12 +378,13 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_data(BlockHandle handle, td::Promise promise) override; void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override; void get_zero_state(BlockIdExt block_id, td::Promise promise) override; - void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override; - void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override; - void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_length, td::Promise promise) override; + void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::int64 offset, td::int64 max_length, + td::Promise promise) override; void get_previous_persistent_state_files( BlockSeqno cur_mc_seqno, td::Promise>> promise) override; void get_block_proof(BlockHandle handle, td::Promise promise) override; @@ -410,10 +411,10 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; void get_cell_db_reader(td::Promise> promise) override; - void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, - td::Promise promise) override; - void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::BufferSlice state, td::Promise promise) override; + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + std::function write_data, td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index c481adbbfb..3824fa01c3 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -74,7 +74,7 @@ void DownloadState::start_up() { alarm_timestamp() = timeout_; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id_, - masterchain_block_id_, + masterchain_block_id_, UnsplitStateType{}, [SelfId = actor_id(this), block_id = block_id_](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &DownloadState::get_block_handle); diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index b8fd84d171..73ce440d36 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -418,7 +418,7 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, masterchain_handle_->id(), - masterchain_handle_->id(), write_data, std::move(P)); + masterchain_handle_->id(), UnsplitStateType{}, write_data, std::move(P)); current_status_ = PSTRING() << "serializing masterchain state " << state->get_block_id().id.to_str(); current_status_ts_ = td::Timestamp::now(); @@ -488,7 +488,7 @@ void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Refid(), - masterchain_handle_->id(), write_data, std::move(P)); + masterchain_handle_->id(), UnsplitStateType{}, write_data, std::move(P)); current_status_ = PSTRING() << "serializing shard state " << next_idx_ << "/" << shards_.size() << " " << state->get_block_id().id.to_str(); current_status_ts_ = td::Timestamp::now(); diff --git a/validator/validator.h b/validator/validator.h index dc0f5761ad..d988261a97 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -22,6 +22,7 @@ #include #include +#include "interfaces/persistent-state.h" #include "td/actor/actor.h" #include "ton/ton-types.h" @@ -231,12 +232,13 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_data(BlockHandle handle, td::Promise promise) = 0; virtual void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) = 0; virtual void get_zero_state(BlockIdExt block_id, td::Promise promise) = 0; - virtual void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + virtual void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) = 0; - virtual void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + virtual void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) = 0; - virtual void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_length, td::Promise promise) = 0; + virtual void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::int64 offset, td::int64 max_length, + td::Promise promise) = 0; virtual void get_previous_persistent_state_files( BlockSeqno cur_mc_seqno, td::Promise>> promise) = 0; virtual void get_block_proof(BlockHandle handle, td::Promise promise) = 0; From 0ed9c4a9c250affff99eb9389eabdbd11f6f8036 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Wed, 4 Jun 2025 15:46:48 -0400 Subject: [PATCH 296/524] Move around request handlers in validator/full-node-{master,shard}.cpp No functional change. This will make next commit easier to review. --- validator/full-node-master.cpp | 60 +++++++++++++------------- validator/full-node-shard.cpp | 78 +++++++++++++++++----------------- 2 files changed, 69 insertions(+), 69 deletions(-) diff --git a/validator/full-node-master.cpp b/validator/full-node-master.cpp index 996ce2219e..8ca0aab090 100644 --- a/validator/full-node-master.cpp +++ b/validator/full-node-master.cpp @@ -291,19 +291,6 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo masterchain_block_id, UnsplitStateType{}, std::move(P)); } -void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, - td::Promise promise) { - auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - TRY_RESULT_PROMISE(promise, size, std::move(R)); - promise.set_value(create_serialize_tl_object(size)); - }); - auto block_id = create_block_id(query.block_); - auto masterchain_block_id = create_block_id(query.masterchain_block_); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, - masterchain_block_id, UnsplitStateType{}, std::move(P)); -} - void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise) { auto cnt = static_cast(query.max_size_); @@ -364,23 +351,6 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo masterchain_block_id, UnsplitStateType{}, std::move(P)); } -void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, - td::Promise promise) { - auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error_prefix("failed to get state from db: ")); - return; - } - - promise.set_value(R.move_as_ok()); - }); - auto block_id = create_block_id(query.block_); - auto masterchain_block_id = create_block_id(query.masterchain_block_); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, - masterchain_block_id, UnsplitStateType{}, query.offset_, query.max_size_, std::move(P)); -} - void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise) { promise.set_value( @@ -432,6 +402,36 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo promise.set_value(create_serialize_tl_object()); } +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("failed to get state from db: ")); + return; + } + + promise.set_value(R.move_as_ok()); + }); + auto block_id = create_block_id(query.block_); + auto masterchain_block_id = create_block_id(query.masterchain_block_); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, + masterchain_block_id, UnsplitStateType{}, query.offset_, query.max_size_, std::move(P)); +} + +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, size, std::move(R)); + promise.set_value(create_serialize_tl_object(size)); + }); + auto block_id = create_block_id(query.block_); + auto masterchain_block_id = create_block_id(query.masterchain_block_); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, + masterchain_block_id, UnsplitStateType{}, std::move(P)); +} + void FullNodeMasterImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { auto BX = fetch_tl_prefix(query, true); diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index db629c8981..baf55f052e 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -526,21 +526,6 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod masterchain_block_id, UnsplitStateType{}, std::move(P)); } -void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, - td::Promise promise) { - auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - TRY_RESULT_PROMISE(promise, size, std::move(R)); - promise.set_value(create_serialize_tl_object(size)); - }); - auto block_id = create_block_id(query.block_); - auto masterchain_block_id = create_block_id(query.masterchain_block_); - VLOG(FULL_NODE_DEBUG) << "Got query getPersistentStateSize " << block_id.to_str() << " " - << masterchain_block_id.to_str() << " from " << src; - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, - masterchain_block_id, UnsplitStateType{}, std::move(P)); -} - void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise) { auto cnt = static_cast(query.max_size_); @@ -610,30 +595,6 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod masterchain_block_id, UnsplitStateType{}, 0, max_size + 1, std::move(P)); } -void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, - td::Promise promise) { - auto block_id = create_block_id(query.block_); - auto masterchain_block_id = create_block_id(query.masterchain_block_); - VLOG(FULL_NODE_DEBUG) << "Got query downloadPersistentStateSlice " << block_id.to_str() << " " - << masterchain_block_id.to_str() << " " << query.offset_ << " " << query.max_size_ << " from " - << src; - if (query.max_size_ < 0 || query.max_size_ > (1 << 24)) { - promise.set_error(td::Status::Error(ErrorCode::protoviolation, "invalid max_size")); - return; - } - auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error_prefix("failed to get state from db: ")); - return; - } - - promise.set_value(R.move_as_ok()); - }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, - masterchain_block_id, UnsplitStateType{}, query.offset_, query.max_size_, std::move(P)); -} - void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise) { VLOG(FULL_NODE_DEBUG) << "Got query getCapabilities from " << src; @@ -741,6 +702,45 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod }); } +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, + td::Promise promise) { + auto block_id = create_block_id(query.block_); + auto masterchain_block_id = create_block_id(query.masterchain_block_); + VLOG(FULL_NODE_DEBUG) << "Got query downloadPersistentStateSlice " << block_id.to_str() << " " + << masterchain_block_id.to_str() << " " << query.offset_ << " " << query.max_size_ << " from " + << src; + if (query.max_size_ < 0 || query.max_size_ > (1 << 24)) { + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "invalid max_size")); + return; + } + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("failed to get state from db: ")); + return; + } + + promise.set_value(R.move_as_ok()); + }); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, + masterchain_block_id, UnsplitStateType{}, query.offset_, query.max_size_, std::move(P)); +} + +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, size, std::move(R)); + promise.set_value(create_serialize_tl_object(size)); + }); + auto block_id = create_block_id(query.block_); + auto masterchain_block_id = create_block_id(query.masterchain_block_); + VLOG(FULL_NODE_DEBUG) << "Got query getPersistentStateSize " << block_id.to_str() << " " + << masterchain_block_id.to_str() << " from " << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, + masterchain_block_id, UnsplitStateType{}, std::move(P)); +} + void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { if (!active_) { From 27a4b367c98fcc47016d7b214cc48c5a3120fab9 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Sat, 14 Jun 2025 14:03:01 -0400 Subject: [PATCH 297/524] Report error if state is not found in getPersistentStateSize query This will allow us to omit preparePersistentState query in the future. --- tl/generate/scheme/ton_api.tl | 1 + tl/generate/scheme/ton_api.tlo | Bin 103388 -> 103464 bytes validator/full-node-master.cpp | 7 +++++-- validator/full-node-shard.cpp | 7 +++++-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 06f202a7f2..dacf442d82 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -410,6 +410,7 @@ tonNode.preparedProofLink = tonNode.PreparedProof; tonNode.preparedState = tonNode.PreparedState; tonNode.notFoundState = tonNode.PreparedState; tonNode.persistentStateSize size:long = tonNode.PersistentStateSize; +tonNode.persistentStateSizeNotFound = tonNode.PersistentStateSize; tonNode.prepared = tonNode.Prepared; tonNode.notFound = tonNode.Prepared; tonNode.data data:bytes = tonNode.Data; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index f30418f7bd78ff6a6f1b8519fe1c506e3f5999ad..96aee7ac06af55f9aa09e7eb9f4e00f65c4234a5 100644 GIT binary patch delta 129 zcmcb!oNdJlwhdQ|S#}B+aZJ8pEHlwTVl$742s`8U&5G9BA40?vb{mL-6fi)+RXMJh z>;qxHl&9~NVU$+z%P(=uFU?BQ promise) { auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - TRY_RESULT_PROMISE(promise, size, std::move(R)); - promise.set_value(create_serialize_tl_object(size)); + if (R.is_error()) { + promise.set_value(create_serialize_tl_object()); + } else { + promise.set_value(create_serialize_tl_object(R.move_as_ok())); + } }); auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index baf55f052e..480c24b64a 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -730,8 +730,11 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod td::Promise promise) { auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - TRY_RESULT_PROMISE(promise, size, std::move(R)); - promise.set_value(create_serialize_tl_object(size)); + if (R.is_error()) { + promise.set_value(create_serialize_tl_object()); + } else { + promise.set_value(create_serialize_tl_object(R.move_as_ok())); + } }); auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); From 8d2551ae289a3caaa0a9a0efa8d7ef8e7516889d Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Wed, 4 Jun 2025 15:40:48 -0400 Subject: [PATCH 298/524] Teach full node to response with split persistent states We introduce two additional ADNL requests that can be made between nodes: downloadPersistentStateSliceV2 and getPersistentStateSizeV2. They differ from "V1" versions by inclusion of "effective shard" parameter. The parameter denotes a split account state if block.shard is a proper ancestor of effective_shard. If block.shard == effective_shard, we treat this as a request for split state header, and if it is 0 (or invalid), we serve an unsplit state. Old requests are wired to call V2 with effective shard set to 0. --- tl/generate/scheme/ton_api.tl | 13 ++++--- tl/generate/scheme/ton_api.tlo | Bin 103464 -> 103992 bytes validator/full-node-master.cpp | 28 ++++++++++----- validator/full-node-master.hpp | 6 ++++ validator/full-node-shard.cpp | 38 +++++++++++++------- validator/full-node-shard.hpp | 4 +++ validator/interfaces/persistent-state.h | 44 ++++++++++++++++++++++++ 7 files changed, 108 insertions(+), 25 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index dacf442d82..9b535ba0e6 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -409,6 +409,7 @@ tonNode.preparedProof = tonNode.PreparedProof; tonNode.preparedProofLink = tonNode.PreparedProof; tonNode.preparedState = tonNode.PreparedState; tonNode.notFoundState = tonNode.PreparedState; +tonNode.persistentStateIdV2 block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt effective_shard:long = tonNode.PersistentStateIdV2; tonNode.persistentStateSize size:long = tonNode.PersistentStateSize; tonNode.persistentStateSizeNotFound = tonNode.PersistentStateSize; tonNode.prepared = tonNode.Prepared; @@ -476,15 +477,11 @@ tonNode.prepareBlockProofs blocks:(vector tonNode.blockIdExt) allow_partial:Bool tonNode.prepareKeyBlockProofs blocks:(vector tonNode.blockIdExt) allow_partial:Bool = tonNode.PreparedProof; tonNode.prepareBlock block:tonNode.blockIdExt = tonNode.Prepared; tonNode.prepareBlocks blocks:(vector tonNode.blockIdExt) = tonNode.Prepared; -tonNode.preparePersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PreparedState; -tonNode.getPersistentStateSize block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PersistentStateSize; tonNode.prepareZeroState block:tonNode.blockIdExt = tonNode.PreparedState; tonNode.getNextKeyBlockIds block:tonNode.blockIdExt max_size:int = tonNode.KeyBlocks; tonNode.downloadNextBlockFull prev_block:tonNode.blockIdExt = tonNode.DataFull; tonNode.downloadBlockFull block:tonNode.blockIdExt = tonNode.DataFull; tonNode.downloadBlock block:tonNode.blockIdExt = tonNode.Data; -tonNode.downloadPersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.Data; -tonNode.downloadPersistentStateSlice block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt offset:long max_size:long = tonNode.Data; tonNode.downloadZeroState block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadBlockProof block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadKeyBlockProof block:tonNode.blockIdExt = tonNode.Data; @@ -496,6 +493,14 @@ tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data; tonNode.getOutMsgQueueProof dst_shard:tonNode.shardId blocks:(vector tonNode.blockIdExt) limits:tonNode.importedMsgQueueLimits = tonNode.OutMsgQueueProof; +tonNode.downloadPersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.Data; +tonNode.downloadPersistentStateSlice block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt offset:long max_size:long = tonNode.Data; +tonNode.getPersistentStateSize block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PersistentStateSize; +tonNode.preparePersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PreparedState; + +tonNode.downloadPersistentStateSliceV2 state:tonNode.persistentStateIdV2 offset:long max_size:long = tonNode.Data; +tonNode.getPersistentStateSizeV2 state:tonNode.persistentStateIdV2 = tonNode.PersistentStateSize; + tonNode.getCapabilities = tonNode.Capabilities; tonNode.slave.sendExtMessage message:tonNode.externalMessage = tonNode.Success; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 96aee7ac06af55f9aa09e7eb9f4e00f65c4234a5..27bc598a75eb3a54807bba7b2ef4f942a1e368ad 100644 GIT binary patch delta 227 zcmZ3nf^EkdHr_|G^{p77KyxGSabtz#fH$1dCHZ-N`6;P-0jWjBnZ+fkc_qOmi6yC? zDPcwvH%M&0V(i1txMQ=T_4J3VVD-}#Wf{eJLDC@Z&0|77%+o)%GX_sSQ=`KI6575- zhEYJDv3L7kZ^rvJtO6}PZEDjGwlk{mgc&iksBndW^-L~&Ee=*beM2Xs!(@jqJd+P3 zaZGOzXB5~zr-QMFg;iqtB5#H1GkX~2_<$N27 promise) { + auto query_v2 = create_tl_object(persistent_state_id_from_v1_query(query)); + return process_query(src, *query_v2, std::move(promise)); +} + void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise) { auto cnt = static_cast(query.max_size_); @@ -351,6 +357,13 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo masterchain_block_id, UnsplitStateType{}, std::move(P)); } +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, + td::Promise promise) { + auto query_v2 = create_tl_object( + persistent_state_id_from_v1_query(query), query.offset_, query.max_size_); + return process_query(src, *query_v2, std::move(promise)); +} + void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise) { promise.set_value( @@ -402,7 +415,8 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo promise.set_value(create_serialize_tl_object()); } -void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, + ton_api::tonNode_downloadPersistentStateSliceV2 &query, td::Promise promise) { auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { @@ -413,13 +427,12 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo promise.set_value(R.move_as_ok()); }); - auto block_id = create_block_id(query.block_); - auto masterchain_block_id = create_block_id(query.masterchain_block_); + auto [block_id, mc_block_id, state_type] = persistent_state_from_v2_query(query); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, - masterchain_block_id, UnsplitStateType{}, query.offset_, query.max_size_, std::move(P)); + mc_block_id, state_type, query.offset_, query.max_size_, std::move(P)); } -void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSizeV2 &query, td::Promise promise) { auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { @@ -429,10 +442,9 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo promise.set_value(create_serialize_tl_object(R.move_as_ok())); } }); - auto block_id = create_block_id(query.block_); - auto masterchain_block_id = create_block_id(query.masterchain_block_); + auto [block_id, mc_block_id, state_type] = persistent_state_from_v2_query(query); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, - masterchain_block_id, UnsplitStateType{}, std::move(P)); + mc_block_id, state_type, std::move(P)); } void FullNodeMasterImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, diff --git a/validator/full-node-master.hpp b/validator/full-node-master.hpp index d6160a94ee..82815b48f3 100644 --- a/validator/full-node-master.hpp +++ b/validator/full-node-master.hpp @@ -86,6 +86,12 @@ class FullNodeMasterImpl : public FullNodeMaster { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise); + + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSliceV2 &query, + td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSizeV2 &query, + td::Promise promise); + // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query, // td::Promise promise); void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise); diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 480c24b64a..64d0d81210 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -526,6 +526,12 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod masterchain_block_id, UnsplitStateType{}, std::move(P)); } +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise) { + auto query_v2 = create_tl_object(persistent_state_id_from_v1_query(query)); + return process_query(src, *query_v2, std::move(promise)); +} + void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise) { auto cnt = static_cast(query.max_size_); @@ -595,6 +601,13 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod masterchain_block_id, UnsplitStateType{}, 0, max_size + 1, std::move(P)); } +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, + td::Promise promise) { + auto query_v2 = create_tl_object( + persistent_state_id_from_v1_query(query), query.offset_, query.max_size_); + return process_query(src, *query_v2, std::move(promise)); +} + void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise) { VLOG(FULL_NODE_DEBUG) << "Got query getCapabilities from " << src; @@ -702,13 +715,12 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod }); } -void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSliceV2 &query, td::Promise promise) { - auto block_id = create_block_id(query.block_); - auto masterchain_block_id = create_block_id(query.masterchain_block_); - VLOG(FULL_NODE_DEBUG) << "Got query downloadPersistentStateSlice " << block_id.to_str() << " " - << masterchain_block_id.to_str() << " " << query.offset_ << " " << query.max_size_ << " from " - << src; + auto [block_id, mc_block_id, state_type] = persistent_state_from_v2_query(query); + VLOG(FULL_NODE_DEBUG) << "Got query downloadPersistentStateSlice " << block_id.to_str() << " " << mc_block_id.to_str() + << " (" << persistent_state_type_to_string(block_id.shard_full(), state_type) << ") " + << query.offset_ << " " << query.max_size_ << " from " << src; if (query.max_size_ < 0 || query.max_size_ > (1 << 24)) { promise.set_error(td::Status::Error(ErrorCode::protoviolation, "invalid max_size")); return; @@ -723,10 +735,10 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_value(R.move_as_ok()); }); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, - masterchain_block_id, UnsplitStateType{}, query.offset_, query.max_size_, std::move(P)); + mc_block_id, state_type, query.offset_, query.max_size_, std::move(P)); } -void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSizeV2 &query, td::Promise promise) { auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { @@ -736,12 +748,12 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_value(create_serialize_tl_object(R.move_as_ok())); } }); - auto block_id = create_block_id(query.block_); - auto masterchain_block_id = create_block_id(query.masterchain_block_); - VLOG(FULL_NODE_DEBUG) << "Got query getPersistentStateSize " << block_id.to_str() << " " - << masterchain_block_id.to_str() << " from " << src; + auto [block_id, mc_block_id, state_type] = persistent_state_from_v2_query(query); + VLOG(FULL_NODE_DEBUG) << "Got query getPersistentStateSize " << block_id.to_str() << " " << mc_block_id.to_str() + << " (" << persistent_state_type_to_string(block_id.shard_full(), state_type) << ") from " + << src; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, - masterchain_block_id, UnsplitStateType{}, std::move(P)); + mc_block_id, state_type, std::move(P)); } void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 8cbd614aa1..5be6b6a5bf 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -143,6 +143,10 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getOutMsgQueueProof &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSliceV2 &query, + td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSizeV2 &query, + td::Promise promise); void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise); void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data); diff --git a/validator/interfaces/persistent-state.h b/validator/interfaces/persistent-state.h index 4a03a8f2b9..8e501398ad 100644 --- a/validator/interfaces/persistent-state.h +++ b/validator/interfaces/persistent-state.h @@ -16,8 +16,11 @@ */ #pragma once +#include "auto/tl/ton_api.h" #include "td/utils/Variant.h" +#include "td/utils/overloaded.h" #include "ton/ton-types.h" +#include "ton/ton-shard.h" namespace ton { @@ -33,6 +36,47 @@ struct SplitPersistentStateType {}; using PersistentStateType = td::Variant; +auto persistent_state_id_from_v1_query(auto const &query) { + auto block = create_tl_block_id(create_block_id(query.block_)); + auto mc_block = create_tl_block_id(create_block_id(query.masterchain_block_)); + return create_tl_object(std::move(block), std::move(mc_block), 0); +} + +auto persistent_state_from_v2_query(auto const &query) { + auto block = create_block_id(query.state_->block_); + auto mc_block = create_block_id(query.state_->masterchain_block_); + ShardId effective_shard = static_cast(query.state_->effective_shard_); + + if (effective_shard == 0 || !shard_is_ancestor(block.shard_full().shard, effective_shard)) { + // The second condition is technically a "protocol" violation but since we don't really validate + // stuff here regardless, let's just map it to an unsplit state. + return std::tuple{block, mc_block, PersistentStateType{UnsplitStateType{}}}; + } + + if (effective_shard == block.shard_full().shard) { + return std::tuple{block, mc_block, PersistentStateType{SplitPersistentStateType{}}}; + } + + CHECK(shard_is_proper_ancestor(block.shard_full().shard, effective_shard)); + return std::tuple{block, mc_block, PersistentStateType{SplitAccountStateType{effective_shard}}}; +} + +inline std::string persistent_state_type_to_string(ShardIdFull const &shard, PersistentStateType const &state) { + std::string result; + state.visit(td::overloaded([&](UnsplitStateType) { result = "unsplit"; }, + [&](SplitAccountStateType type) { + int real_pfx_len = shard_prefix_length(shard.shard); + int effective_pfx_len = shard_prefix_length(type.effective_shard_id); + td::uint64 parts_count = 1 << (effective_pfx_len - real_pfx_len); + td::uint64 part_idx = + type.effective_shard_id >> (64 - effective_pfx_len) & (parts_count - 1); + result = + "part " + std::to_string(part_idx + 1) + " out of " + std::to_string(parts_count); + }, + [&](SplitPersistentStateType) { result = "split header"; })); + return result; +} + } // namespace validator } // namespace ton From d54065abe58c8ab9bbcf9cffa81f70da4dd16773 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Thu, 19 Jun 2025 10:20:54 +0300 Subject: [PATCH 299/524] Update Windows github action (#1714) * not always gh mac runners have nproc utility * deprecate Ubuntu 20.04 * fix linking issues of portable tonlibjson.dylib * Upgrade gh action win-2019->win-2022 * adjust gh path to MSVC --- .github/workflows/ton-x86-64-windows.yml | 10 +- assembly/native/build-windows-2022.bat | 207 ++++++++++++++++++ assembly/native/build-windows-github-2022.bat | 2 + 3 files changed, 214 insertions(+), 5 deletions(-) create mode 100644 assembly/native/build-windows-2022.bat create mode 100644 assembly/native/build-windows-github-2022.bat diff --git a/.github/workflows/ton-x86-64-windows.yml b/.github/workflows/ton-x86-64-windows.yml index b930d25a96..8de9e11be9 100644 --- a/.github/workflows/ton-x86-64-windows.yml +++ b/.github/workflows/ton-x86-64-windows.yml @@ -9,7 +9,7 @@ defaults: jobs: build: - runs-on: windows-2019 + runs-on: windows-2022 steps: - name: Get Current OS version @@ -30,15 +30,15 @@ jobs: uses: actions/cache@v4 with: path: third_libs - key: ${{ runner.os }}-${{ runner.arch }}-windows-2019-3pp-${{ hashFiles('**/assembly/native/build-windows-2019.bat') }} + key: ${{ runner.os }}-${{ runner.arch }}-windows-2022-3pp-${{ hashFiles('**/assembly/native/build-windows-2022.bat') }} - name: Build TON run: | git submodule sync --recursive git submodule update - copy assembly\native\build-windows-github-2019.bat . - copy assembly\native\build-windows-2019.bat . - build-windows-github-2019.bat Enterprise + copy assembly\native\build-windows-github-2022.bat . + copy assembly\native\build-windows-2022.bat . + build-windows-github-2022.bat Enterprise ccache -sp # - name: Run Tests diff --git a/assembly/native/build-windows-2022.bat b/assembly/native/build-windows-2022.bat new file mode 100644 index 0000000000..6ff7166af3 --- /dev/null +++ b/assembly/native/build-windows-2022.bat @@ -0,0 +1,207 @@ +REM execute this script inside elevated (Run as Administrator) console "x64 Native Tools Command Prompt for VS 2022" + +echo off + +echo Installing chocolatey windows package manager... +@"%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell.exe" -NoProfile -InputFormat None -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" && SET "PATH=%PATH%;%ALLUSERSPROFILE%\chocolatey\bin" +choco -? +IF %errorlevel% NEQ 0 ( + echo Can't install chocolatey + exit /b %errorlevel% +) + +choco feature enable -n allowEmptyChecksums + +echo Installing pkgconfiglite... +choco install -y pkgconfiglite +IF %errorlevel% NEQ 0 ( + echo Can't install pkgconfiglite + exit /b %errorlevel% +) + +echo Installing ninja... +choco install -y ninja +IF %errorlevel% NEQ 0 ( + echo Can't install ninja + exit /b %errorlevel% +) + +echo Installing ccache... +choco install -y ccache +IF %errorlevel% NEQ 0 ( + echo Can't install ccache + exit /b %errorlevel% +) + +echo Installing nasm... +choco install -y nasm +where nasm +SET PATH=%PATH%;C:\Program Files\NASM +IF %errorlevel% NEQ 0 ( + echo Can't install nasm + exit /b %errorlevel% +) + +if not exist "third_libs" ( + mkdir "third_libs" +) +cd third_libs + +set third_libs=%cd% +echo %third_libs% + +if not exist "zlib" ( + git clone https://github.com/madler/zlib.git + cd zlib + git checkout v1.3.1 + cd contrib\vstudio\vc14 + msbuild zlibstat.vcxproj /p:Configuration=ReleaseWithoutAsm /p:platform=x64 -p:PlatformToolset=v143 + cd ..\..\..\.. +) else ( + echo Using zlib... +) + +if not exist "lz4" ( + git clone https://github.com/lz4/lz4.git + cd lz4 + git checkout v1.9.4 + cd build\VS2022\liblz4 + msbuild liblz4.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v143 + cd ..\..\..\.. +) else ( + echo Using lz4... +) + +if not exist "libsodium" ( + git clone https://github.com/jedisct1/libsodium + cd libsodium + git checkout 1.0.18-RELEASE + msbuild libsodium.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v143 + cd .. +) else ( + echo Using libsodium... +) + +if not exist "openssl" ( + git clone https://github.com/openssl/openssl.git + cd openssl + git checkout openssl-3.1.4 + where perl + perl Configure VC-WIN64A + IF %errorlevel% NEQ 0 ( + echo Can't configure openssl + exit /b %errorlevel% + ) + nmake + cd .. +) else ( + echo Using openssl... +) + +if not exist "libmicrohttpd" ( + git clone https://github.com/Karlson2k/libmicrohttpd.git + cd libmicrohttpd + git checkout v1.0.1 + cd w32\VS2022 + msbuild libmicrohttpd.vcxproj /p:Configuration=Release-static /p:platform=x64 -p:PlatformToolset=v143 + IF %errorlevel% NEQ 0 ( + echo Can't compile libmicrohttpd + exit /b %errorlevel% + ) + cd ../../.. +) else ( + echo Using libmicrohttpd... +) + +cd .. +echo Current dir %cd% + +mkdir build +cd build +cmake -GNinja -DCMAKE_BUILD_TYPE=Release ^ +-DPORTABLE=1 ^ +-DSODIUM_USE_STATIC_LIBS=1 ^ +-DSODIUM_LIBRARY_RELEASE=%third_libs%\libsodium\Build\Release\x64\libsodium.lib ^ +-DSODIUM_LIBRARY_DEBUG=%third_libs%\libsodium\Build\Release\x64\libsodium.lib ^ +-DSODIUM_INCLUDE_DIR=%third_libs%\libsodium\src\libsodium\include ^ +-DLZ4_FOUND=1 ^ +-DLZ4_INCLUDE_DIRS=%third_libs%\lz4\lib ^ +-DLZ4_LIBRARIES=%third_libs%\lz4\build\VS2022\liblz4\bin\x64_Release\liblz4_static.lib ^ +-DMHD_FOUND=1 ^ +-DMHD_LIBRARY=%third_libs%\libmicrohttpd\w32\VS2022\Output\x64\libmicrohttpd.lib ^ +-DMHD_INCLUDE_DIR=%third_libs%\libmicrohttpd\src\include ^ +-DZLIB_FOUND=1 ^ +-DZLIB_INCLUDE_DIR=%third_libs%\zlib ^ +-DZLIB_LIBRARIES=%third_libs%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib ^ +-DOPENSSL_FOUND=1 ^ +-DOPENSSL_INCLUDE_DIR=%third_libs%\openssl\include ^ +-DOPENSSL_CRYPTO_LIBRARY=%third_libs%\openssl\libcrypto_static.lib ^ +-DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj" .. + +IF %errorlevel% NEQ 0 ( + echo Can't configure TON + exit /b %errorlevel% +) + +IF "%1"=="-t" ( +ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ +json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ +test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^ +test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ +test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver +IF %errorlevel% NEQ 0 ( + echo Can't compile TON + exit /b %errorlevel% +) +) else ( +ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ +json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver +IF %errorlevel% NEQ 0 ( + echo Can't compile TON + exit /b %errorlevel% +) +) + +copy validator-engine\validator-engine.exe test +IF %errorlevel% NEQ 0 ( + echo validator-engine.exe does not exist + exit /b %errorlevel% +) + +echo Strip and copy artifacts +cd .. +echo where strip +where strip +mkdir artifacts +mkdir artifacts\smartcont +mkdir artifacts\lib + +for %%I in (build\storage\storage-daemon\storage-daemon.exe ^ + build\storage\storage-daemon\storage-daemon-cli.exe ^ + build\blockchain-explorer\blockchain-explorer.exe ^ + build\crypto\fift.exe ^ + build\crypto\tlbc.exe ^ + build\crypto\func.exe ^ + build\tolk\tolk.exe ^ + build\crypto\create-state.exe ^ + build\validator-engine-console\validator-engine-console.exe ^ + build\tonlib\tonlib-cli.exe ^ + build\tonlib\tonlibjson.dll ^ + build\http\http-proxy.exe ^ + build\rldp-http-proxy\rldp-http-proxy.exe ^ + build\dht-server\dht-server.exe ^ + build\lite-client\lite-client.exe ^ + build\validator-engine\validator-engine.exe ^ + build\utils\generate-random-id.exe ^ + build\utils\json2tlo.exe ^ + build\utils\proxy-liteserver.exe ^ + build\adnl\adnl-proxy.exe ^ + build\emulator\emulator.dll) do ( + echo strip -s %%I & copy %%I artifacts\ + strip -s %%I & copy %%I artifacts\ +) + +xcopy /e /k /h /i crypto\smartcont artifacts\smartcont +xcopy /e /k /h /i crypto\fift\lib artifacts\lib diff --git a/assembly/native/build-windows-github-2022.bat b/assembly/native/build-windows-github-2022.bat new file mode 100644 index 0000000000..f1e4b884a5 --- /dev/null +++ b/assembly/native/build-windows-github-2022.bat @@ -0,0 +1,2 @@ +call "C:\Program Files\Microsoft Visual Studio\2022\%1\VC\Auxiliary\Build\vcvars64.bat" +call build-windows-2022.bat -t From 5a1d271b9b0dd2bbffea555561cdc385ed2672a6 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Sat, 7 Jun 2025 13:13:14 -0400 Subject: [PATCH 300/524] Add support for split persistent states in StateSerializer --- crypto/vm/dict.cpp | 6 ++ crypto/vm/dict.h | 1 + validator/state-serializer.cpp | 168 ++++++++++++++++++++++++++++----- validator/state-serializer.hpp | 16 +++- 4 files changed, 168 insertions(+), 23 deletions(-) diff --git a/crypto/vm/dict.cpp b/crypto/vm/dict.cpp index e1e683fe36..0450932a2f 100644 --- a/crypto/vm/dict.cpp +++ b/crypto/vm/dict.cpp @@ -2790,6 +2790,12 @@ Ref AugmentedDictionary::get_root() const { return root; } +Ref AugmentedDictionary::get_wrapped_dict_root() const { + vm::CellBuilder cb; + cb.append_cellslice(get_root()); + return cb.finalize(); +} + Ref AugmentedDictionary::extract_root() && { if (!(flags & f_root_cached) && !compute_root()) { return {}; diff --git a/crypto/vm/dict.h b/crypto/vm/dict.h index e22b909465..4da56acd97 100644 --- a/crypto/vm/dict.h +++ b/crypto/vm/dict.h @@ -572,6 +572,7 @@ class AugmentedDictionary final : public DictionaryFixed { AugmentedDictionary(DictNonEmpty, Ref _root, int _n, const AugmentationData& _aug, bool validate = true); Ref get_empty_dictionary() const; Ref get_root() const; + Ref get_wrapped_dict_root() const; Ref extract_root() &&; bool append_dict_to_bool(CellBuilder& cb) &&; bool append_dict_to_bool(CellBuilder& cb) const &; diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index 73ce440d36..890d7e23f5 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -17,11 +17,15 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "state-serializer.hpp" +#include "crypto/block/block-auto.h" +#include "crypto/block/block-parse.h" #include "td/utils/Random.h" +#include "td/utils/overloaded.h" #include "ton/ton-io.hpp" #include "common/delay.h" #include "td/utils/filesystem.h" #include "td/utils/HashSet.h" +#include "vm/cells/MerkleProof.h" namespace ton { @@ -199,7 +203,7 @@ void AsyncStateSerializer::next_iteration() { } if (next_idx_ < shards_.size()) { running_ = true; - request_shard_state(shards_[next_idx_]); + request_shard_state(shards_[next_idx_].block_id); return; } LOG(ERROR) << "finished serializing persistent state for " << masterchain_handle_->id().id.to_str(); @@ -324,7 +328,16 @@ class CachedCellDbReader : public vm::CellDbReader { td::uint64 bulk_reqs_ = 0; }; -void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard) { +void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard, PersistentStateType type) { + if (type.get_offset() == type.offset()) { + // Header of a split state is small, so not caching it is fine. + return; + } + + if (type.get_offset() == type.offset()) { + shard = {shard.workchain, type.get().effective_shard_id}; + } + std::vector prev_shards; for (const auto& [_, prev_shard] : state_files) { if (shard_intersects(shard, prev_shard)) { @@ -335,7 +348,6 @@ void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard) return; } cur_shards = std::move(prev_shards); - cache = {}; if (cur_shards.empty()) { return; } @@ -375,6 +387,25 @@ void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard) cache = std::make_shared(std::move(cells)); } +void AsyncStateSerializer::PreviousStateCache::add_new_cells(vm::CellDbReader& reader, Ref const& cell) { + if (!cell->is_loaded()) { + return; + } + if (reader.load_cell(cell->get_hash().as_slice()).is_ok()) { + return; + } + + auto [_, inserted] = cache->insert(cell); + if (!inserted) { + return; + } + + vm::CellSlice cs{vm::NoVm{}, cell}; + for (unsigned i = 0; i < cs.size_refs(); ++i) { + add_new_cells(reader, cs.prefetch_ref(i)); + } +} + void AsyncStateSerializer::got_masterchain_state(td::Ref state, std::shared_ptr cell_db_reader) { if (!opts_->get_state_serializer_enabled() || auto_disabled_) { @@ -389,7 +420,10 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state auto vec = state->get_shards(); for (auto &v : vec) { if (opts_->need_monitor(v->shard(), state)) { - shards_.push_back(v->top_block_id()); + shards_.push_back({ + .block_id = v->top_block_id(), + .split_depth = state->persistent_state_split_depth(v->shard().workchain), + }); } } @@ -401,7 +435,7 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state return vm::std_boc_serialize_to_file(root, fd, 31, std::move(cancellation_token)); } if (fast_serializer_enabled) { - previous_state_cache->prepare_cache(shard); + previous_state_cache->prepare_cache(shard, UnsplitStateType{}); } auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache->cache); auto res = vm::boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); @@ -455,43 +489,135 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) { td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle, std::move(P)); } +namespace { + +// Expects `ShardStateUnsplit` as `shard_state_cell`. +std::vector split_shard_state(ShardId shard_id, td::Ref shard_state_cell, int split_depth) { + CHECK(split_depth <= 63); + int shard_prefix_length = shard_pfx_len(shard_id); + if (shard_prefix_length >= static_cast(split_depth)) { + return {{UnsplitStateType{}, std::move(shard_state_cell)}}; + } + + block::gen::ShardStateUnsplit::Record unsplit_shard_state; + bool rc = tlb::unpack_cell(shard_state_cell, unsplit_shard_state); + CHECK(rc); + + std::vector result; + + auto unwrapped_accounts_root = unsplit_shard_state.accounts; + auto accounts_cut = std::make_shared(); + auto accounts_root = vm::UsageCell::create(unwrapped_accounts_root, accounts_cut->root_ptr()); + + // NOTE: Ref constructor expects caller to unwrap HashMapAugE. + vm::AugmentedDictionary accounts{ + vm::load_cell_slice_ref(accounts_root), + 256, + block::tlb::aug_ShardAccounts, + false, + }; + + // Build account dict parts + ShardId effective_shard = shard_id ^ (1ULL << (63 - shard_prefix_length)) ^ (1ULL << (63 - split_depth)); + ShardId increment = 1ULL << (64 - split_depth); + + for (int i = 0; i < (1 << (split_depth - shard_prefix_length)); ++i, effective_shard += increment) { + td::BitArray<64> prefix; + prefix.store_ulong(effective_shard); + auto account_dict_part = accounts; + account_dict_part.cut_prefix_subdict(prefix.bits(), split_depth); + + if (!account_dict_part.is_empty()) { + result.push_back({SplitAccountStateType{effective_shard}, account_dict_part.get_wrapped_dict_root()}); + } + } + + auto accounts_proof = vm::MerkleProof::generate_raw(unwrapped_accounts_root, accounts_cut.get()); + + // Build header + unsplit_shard_state.accounts = accounts_proof; + vm::CellBuilder unsplit_shard_state_cb; + rc = tlb::pack(unsplit_shard_state_cb, unsplit_shard_state); + CHECK(rc); + + auto header = unsplit_shard_state_cb.finalize(); + CHECK(header->get_level() <= 1 && header->get_hash(0) == shard_state_cell->get_hash()); + + result.push_back({SplitPersistentStateType{}, vm::CellBuilder::create_merkle_proof(header)}); + + return result; +} + +} // namespace + void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref state, std::shared_ptr cell_db_reader) { + int archive_split_depth = shards_[next_idx_].split_depth; next_idx_++; + if (!opts_->get_state_serializer_enabled() || auto_disabled_) { success_handler(); return; } LOG(ERROR) << "serializing shard state " << handle->id().id.to_str(); - auto write_data = [shard = state->get_shard(), root = state->root_cell(), cell_db_reader, - previous_state_cache = previous_state_cache_, - fast_serializer_enabled = opts_->get_fast_state_serializer_enabled(), + + auto parts = split_shard_state(state->get_shard().shard, state->root_cell(), archive_split_depth); + CHECK(!parts.empty()); + + write_shard_state(handle, state->get_shard(), cell_db_reader, + std::make_shared>(std::move(parts)), 0); + + current_status_ = PSTRING() << "serializing shard state " << next_idx_ << "/" << shards_.size() << " " + << state->get_block_id().id.to_str(); + current_status_ts_ = td::Timestamp::now(); +} + +void AsyncStateSerializer::write_shard_state(BlockHandle handle, ShardIdFull shard, + std::shared_ptr cell_db_reader, + std::shared_ptr> parts, size_t idx) { + auto [type, cell] = parts->at(idx); + + auto write_data = [=, this, cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { + CHECK(running_); + + LOG(ERROR) << "serializing shard state " << handle->id().id.to_str() << " (" + << persistent_state_type_to_string(shard, type) << ")"; if (!cell_db_reader) { - return vm::std_boc_serialize_to_file(root, fd, 31, std::move(cancellation_token)); + return vm::std_boc_serialize_to_file(cell, fd, 31, std::move(cancellation_token)); } - if (fast_serializer_enabled) { - previous_state_cache->prepare_cache(shard); + if (opts_->get_fast_state_serializer_enabled()) { + previous_state_cache_->prepare_cache(shard, type); } - auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache->cache); - auto res = vm::boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); + if (!previous_state_cache_->cache) { + previous_state_cache_->cache = std::make_shared(); + } + previous_state_cache_->add_new_cells(*cell_db_reader, cell); + auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache_->cache); + auto res = + vm::boc_serialize_to_file_large(new_cell_db_reader, cell->get_hash(), fd, 31, std::move(cancellation_token)); new_cell_db_reader->print_stats(); return res; }; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { + auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this)](td::Result R) { if (R.is_error() && R.error().code() == cancelled) { LOG(ERROR) << "Persistent state serialization cancelled"; + td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); + return; + } + + R.ensure(); + LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str() << " (" + << persistent_state_type_to_string(shard, type) << ")"; + if (idx + 1 == parts->size()) { + td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); } else { - R.ensure(); - LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str(); + td::actor::send_closure(SelfId, &AsyncStateSerializer::write_shard_state, handle, shard, cell_db_reader, parts, + idx + 1); } - td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(), - masterchain_handle_->id(), UnsplitStateType{}, write_data, std::move(P)); - current_status_ = PSTRING() << "serializing shard state " << next_idx_ << "/" << shards_.size() << " " - << state->get_block_id().id.to_str(); - current_status_ts_ = td::Timestamp::now(); + masterchain_handle_->id(), type, write_data, std::move(P)); } void AsyncStateSerializer::fail_handler(td::Status reason) { diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index 406ac350a2..baa9268db0 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -27,6 +27,11 @@ namespace ton { namespace validator { +struct SerializablePart { + PersistentStateType type; + td::Ref cell; +}; + class AsyncStateSerializer : public td::actor::Actor { private: td::uint32 attempt_ = 0; @@ -52,13 +57,18 @@ class AsyncStateSerializer : public td::actor::Actor { bool stored_persistent_state_description_ = false; bool have_masterchain_state_ = false; - std::vector shards_; + struct ShardSerializationConfig { + BlockIdExt block_id; + td::uint32 split_depth; + }; + std::vector shards_; struct PreviousStateCache { std::vector> state_files; std::shared_ptr cache; std::vector cur_shards; - void prepare_cache(ShardIdFull shard); + void prepare_cache(ShardIdFull shard, PersistentStateType type); + void add_new_cells(vm::CellDbReader& reader, Ref const& cell); }; std::shared_ptr previous_state_cache_; @@ -93,6 +103,8 @@ class AsyncStateSerializer : public td::actor::Actor { void stored_masterchain_state(); void got_shard_handle(BlockHandle handle); void got_shard_state(BlockHandle handle, td::Ref state, std::shared_ptr cell_db_reader); + void write_shard_state(BlockHandle handle, ShardIdFull shard, std::shared_ptr cell_db_reader, + std::shared_ptr> parts, size_t idx); void get_masterchain_seqno(td::Promise promise) { promise.set_result(last_block_id_.id.seqno); From 3a93b225e2887d83f167f5886343990edef418c2 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Wed, 11 Jun 2025 19:22:00 -0400 Subject: [PATCH 301/524] Teach DownloadState how to download split states This way, we propagate hardcoding UnsplitStateType one layer up. --- validator/full-node-shard.cpp | 4 +- validator/interfaces/persistent-state.h | 8 +++ validator/net/download-state.cpp | 94 +++++++++++++++++++------ validator/net/download-state.hpp | 10 ++- 4 files changed, 91 insertions(+), 25 deletions(-) diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 64d0d81210..a6dd2346cd 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -990,8 +990,8 @@ void FullNodeShardImpl::download_persistent_state(BlockIdExt id, BlockIdExt mast td::Timestamp timeout, td::Promise promise) { auto &b = choose_neighbour(); td::actor::create_actor(PSTRING() << "downloadstatereq" << id.id.to_str(), id, masterchain_block_id, - adnl_id_, overlay_id_, b.adnl_id, priority, timeout, validator_manager_, - rldp2_, overlays_, adnl_, client_, std::move(promise)) + UnsplitStateType{}, adnl_id_, overlay_id_, b.adnl_id, priority, timeout, + validator_manager_, rldp2_, overlays_, adnl_, client_, std::move(promise)) .release(); } diff --git a/validator/interfaces/persistent-state.h b/validator/interfaces/persistent-state.h index 8e501398ad..36d9ef0bd4 100644 --- a/validator/interfaces/persistent-state.h +++ b/validator/interfaces/persistent-state.h @@ -61,6 +61,14 @@ auto persistent_state_from_v2_query(auto const &query) { return std::tuple{block, mc_block, PersistentStateType{SplitAccountStateType{effective_shard}}}; } +inline ShardId persistent_state_to_effective_shard(ShardIdFull const &shard, PersistentStateType const &type) { + ShardId result = 0; + type.visit(td::overloaded([](UnsplitStateType) {}, + [&](SplitAccountStateType type) { result = type.effective_shard_id; }, + [&](SplitPersistentStateType) { result = shard.shard; })); + return result; +} + inline std::string persistent_state_type_to_string(ShardIdFull const &shard, PersistentStateType const &state) { std::string result; state.visit(td::overloaded([&](UnsplitStateType) { result = "unsplit"; }, diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index 3824fa01c3..16d619b8e7 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -28,15 +28,17 @@ namespace validator { namespace fullnode { -DownloadState::DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_id, adnl::AdnlNodeIdShort local_id, - overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, - td::uint32 priority, td::Timestamp timeout, +DownloadState::DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, + adnl::AdnlNodeIdShort download_from, td::uint32 priority, td::Timestamp timeout, td::actor::ActorId validator_manager, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId adnl, td::actor::ActorId client, td::Promise promise) : block_id_(block_id) , masterchain_block_id_(masterchain_block_id) + , type_(type) + , effective_shard_(persistent_state_to_effective_shard(block_id_.shard_full(), type)) , local_id_(local_id) , overlay_id_(overlay_id) , download_from_(download_from) @@ -48,6 +50,7 @@ DownloadState::DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_i , adnl_(adnl) , client_(client) , promise_(std::move(promise)) { + CHECK(masterchain_block_id_.is_valid() || effective_shard_ == 0); } void DownloadState::abort_query(td::Status reason) { @@ -74,7 +77,7 @@ void DownloadState::start_up() { alarm_timestamp() = timeout_; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id_, - masterchain_block_id_, UnsplitStateType{}, + masterchain_block_id_, type_, [SelfId = actor_id(this), block_id = block_id_](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &DownloadState::get_block_handle); @@ -124,22 +127,39 @@ void DownloadState::got_block_handle(BlockHandle handle) { void DownloadState::got_node_to_download(adnl::AdnlNodeIdShort node) { download_from_ = node; - LOG(WARNING) << "downloading state " << block_id_.to_str() << " from " << download_from_; + LOG(WARNING) << "downloading state " << block_id_.to_str() << " (" + << persistent_state_type_to_string(block_id_.shard_full(), type_) << ") from " << download_from_; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) mutable { - if (R.is_error()) { - td::actor::send_closure(SelfId, &DownloadState::abort_query, R.move_as_error()); + td::Promise P; + td::BufferSlice query; + + if (effective_shard_ == 0) { + P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadState::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &DownloadState::got_block_state_description, R.move_as_ok()); + } + }); + + if (masterchain_block_id_.is_valid()) { + query = create_serialize_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_)); } else { - td::actor::send_closure(SelfId, &DownloadState::got_block_state_description, R.move_as_ok()); + query = create_serialize_tl_object(create_tl_block_id(block_id_)); } - }); - - td::BufferSlice query; - if (masterchain_block_id_.is_valid()) { - query = create_serialize_tl_object( - create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_)); } else { - query = create_serialize_tl_object(create_tl_block_id(block_id_)); + P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadState::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &DownloadState::got_state_size, R.move_as_ok()); + } + }); + + query = create_serialize_tl_object( + create_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_), effective_shard_)); } if (client_.empty()) { @@ -195,6 +215,25 @@ void DownloadState::got_block_state_description(td::BufferSlice data) { })); } +void DownloadState::got_state_size(td::BufferSlice size_or_not_found) { + auto F = fetch_tl_object(std::move(size_or_not_found), true); + if (F.is_error()) { + abort_query(F.move_as_error()); + return; + } + prev_logged_timer_ = td::Timer(); + + ton_api::downcast_call(*F.move_as_ok().get(), + td::overloaded( + [&](ton_api::tonNode_persistentStateSizeNotFound &f) { + abort_query(td::Status::Error(ErrorCode::notready, "state not found")); + }, + [&](ton_api::tonNode_persistentStateSize &f) { + total_size_ = f.size_; + got_block_state_part(td::BufferSlice{}, 0); + })); +} + void DownloadState::request_total_size() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { @@ -207,8 +246,15 @@ void DownloadState::request_total_size() { td::actor::send_closure(SelfId, &DownloadState::got_total_size, res.ok()->size_); }); - td::BufferSlice query = create_serialize_tl_object( - create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_)); + td::BufferSlice query; + if (effective_shard_ == 0) { + query = create_serialize_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_)); + } else { + query = create_serialize_tl_object( + create_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_), effective_shard_)); + } if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, "get size", std::move(P), td::Timestamp::in(3.0), std::move(query), @@ -275,8 +321,16 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques } }); - td::BufferSlice query = create_serialize_tl_object( - create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_), sum_, part_size); + td::BufferSlice query; + if (effective_shard_ == 0) { + query = create_serialize_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_), sum_, part_size); + } else { + query = create_serialize_tl_object( + create_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_), effective_shard_), + sum_, part_size); + } if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, "download state", std::move(P), td::Timestamp::in(20.0), std::move(query), diff --git a/validator/net/download-state.hpp b/validator/net/download-state.hpp index 29854ce273..ebf119bd37 100644 --- a/validator/net/download-state.hpp +++ b/validator/net/download-state.hpp @@ -33,9 +33,10 @@ namespace fullnode { class DownloadState : public td::actor::Actor { public: - DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_id, adnl::AdnlNodeIdShort local_id, - overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::uint32 priority, - td::Timestamp timeout, td::actor::ActorId validator_manager, + DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, + td::uint32 priority, td::Timestamp timeout, + td::actor::ActorId validator_manager, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId adnl, td::actor::ActorId client, td::Promise promise); @@ -49,6 +50,7 @@ class DownloadState : public td::actor::Actor { void got_block_handle(BlockHandle handle); void got_node_to_download(adnl::AdnlNodeIdShort node); void got_block_state_description(td::BufferSlice data_description); + void got_state_size(td::BufferSlice size_or_not_found); void request_total_size(); void got_total_size(td::uint64 size); void got_block_state_part(td::BufferSlice data, td::uint32 requested_size); @@ -57,6 +59,8 @@ class DownloadState : public td::actor::Actor { private: BlockIdExt block_id_; BlockIdExt masterchain_block_id_; + PersistentStateType type_; + ShardId effective_shard_; adnl::AdnlNodeIdShort local_id_; overlay::OverlayIdShort overlay_id_; From ca80ed7ea2bc61286e27997a006bfbcfbf72b9fc Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Thu, 12 Jun 2025 09:28:54 -0400 Subject: [PATCH 302/524] Expose PersistentStateType to DownloadShardState This is the actor which will merge separate parts of a persistent state into a single one in the future commits. --- create-hardfork/create-hardfork.cpp | 4 ++-- test/test-ton-collator.cpp | 4 ++-- validator/downloaders/download-state.cpp | 2 +- validator/full-node-shard.cpp | 22 ++++++++++++---------- validator/full-node-shard.h | 5 +++-- validator/full-node-shard.hpp | 5 +++-- validator/full-node.cpp | 16 +++++++++------- validator/full-node.hpp | 4 ++-- validator/interfaces/validator-manager.h | 3 ++- validator/manager-disk.cpp | 2 +- validator/manager-disk.hpp | 4 ++-- validator/manager-hardfork.hpp | 4 ++-- validator/manager.cpp | 4 ++-- validator/manager.hpp | 4 ++-- validator/validator.h | 5 +++-- 15 files changed, 48 insertions(+), 40 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 20e58f006b..e2f77dd86c 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -258,8 +258,8 @@ class HardforkCreator : public td::actor::Actor { td::Promise promise) override { } void download_persistent_state(ton::BlockIdExt block_id, ton::BlockIdExt masterchain_block_id, - td::uint32 priority, td::Timestamp timeout, - td::Promise promise) override { + ton::validator::PersistentStateType type, td::uint32 priority, + td::Timestamp timeout, td::Promise promise) override { } void download_block_proof(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override { diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index a062ddbe5b..89be586e62 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -359,8 +359,8 @@ class TestNode : public td::actor::Actor { td::Promise promise) override { } void download_persistent_state(ton::BlockIdExt block_id, ton::BlockIdExt masterchain_block_id, - td::uint32 priority, td::Timestamp timeout, - td::Promise promise) override { + ton::validator::PersistentStateType type, td::uint32 priority, + td::Timestamp timeout, td::Promise promise) override { } void download_block_proof(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override { diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index 4335a29568..cc942d1ec6 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -123,7 +123,7 @@ void DownloadShardState::checked_proof_link() { CHECK(masterchain_block_id_.is_valid()); CHECK(masterchain_block_id_.is_masterchain()); td::actor::send_closure(manager_, &ValidatorManager::send_get_persistent_state_request, block_id_, - masterchain_block_id_, priority_, std::move(P)); + masterchain_block_id_, UnsplitStateType{}, priority_, std::move(P)); } status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading state"); } diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index a6dd2346cd..608a5ec8be 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -980,18 +980,20 @@ void FullNodeShardImpl::download_block(BlockIdExt id, td::uint32 priority, td::T void FullNodeShardImpl::download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { - td::actor::create_actor(PSTRING() << "downloadstatereq" << id.id.to_str(), id, BlockIdExt{}, adnl_id_, - overlay_id_, adnl::AdnlNodeIdShort::zero(), priority, timeout, - validator_manager_, rldp_, overlays_, adnl_, client_, std::move(promise)) + td::actor::create_actor(PSTRING() << "downloadstatereq" << id.id.to_str(), id, BlockIdExt{}, + UnsplitStateType{}, adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), + priority, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, + std::move(promise)) .release(); } -void FullNodeShardImpl::download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise) { +void FullNodeShardImpl::download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::uint32 priority, td::Timestamp timeout, + td::Promise promise) { auto &b = choose_neighbour(); td::actor::create_actor(PSTRING() << "downloadstatereq" << id.id.to_str(), id, masterchain_block_id, - UnsplitStateType{}, adnl_id_, overlay_id_, b.adnl_id, priority, timeout, - validator_manager_, rldp2_, overlays_, adnl_, client_, std::move(promise)) + type, adnl_id_, overlay_id_, b.adnl_id, priority, timeout, validator_manager_, + rldp2_, overlays_, adnl_, client_, std::move(promise)) .release(); } @@ -1007,9 +1009,9 @@ void FullNodeShardImpl::download_block_proof(BlockIdExt block_id, td::uint32 pri void FullNodeShardImpl::download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { auto &b = choose_neighbour(); - td::actor::create_actor("downloadproofreq", block_id, true, false, adnl_id_, overlay_id_, - b.adnl_id, priority, timeout, validator_manager_, rldp_, - overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) + td::actor::create_actor("downloadproofreq", block_id, true, false, adnl_id_, overlay_id_, b.adnl_id, + priority, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, + create_neighbour_promise(b, std::move(promise))) .release(); } diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 5898db80da..0c6aa441b8 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -55,8 +55,9 @@ class FullNodeShard : public td::actor::Actor { td::Promise promise) = 0; virtual void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; - virtual void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise) = 0; + virtual void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Timestamp timeout, + td::Promise promise) = 0; virtual void download_block_proof(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 5be6b6a5bf..891d8edb5c 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -178,8 +178,9 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise) override; void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override; - void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise) override; + void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Timestamp timeout, + td::Promise promise) override; void download_block_proof(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 953e0c599c..d871c024b5 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -374,16 +374,17 @@ void FullNodeImpl::download_zero_state(BlockIdExt id, td::uint32 priority, td::T td::actor::send_closure(shard, &FullNodeShard::download_zero_state, id, priority, timeout, std::move(promise)); } -void FullNodeImpl::download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise) { +void FullNodeImpl::download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Timestamp timeout, + td::Promise promise) { auto shard = get_shard(id.shard_full()); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping download state diff query to unknown shard"; promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); return; } - td::actor::send_closure(shard, &FullNodeShard::download_persistent_state, id, masterchain_block_id, priority, timeout, - std::move(promise)); + td::actor::send_closure(shard, &FullNodeShard::download_persistent_state, id, masterchain_block_id, type, priority, + timeout, std::move(promise)); } void FullNodeImpl::download_block_proof(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, @@ -647,9 +648,10 @@ void FullNodeImpl::start_up() { td::Promise promise) override { td::actor::send_closure(id_, &FullNodeImpl::download_zero_state, id, priority, timeout, std::move(promise)); } - void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise) override { - td::actor::send_closure(id_, &FullNodeImpl::download_persistent_state, id, masterchain_block_id, priority, + void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Timestamp timeout, + td::Promise promise) override { + td::actor::send_closure(id_, &FullNodeImpl::download_persistent_state, id, masterchain_block_id, type, priority, timeout, std::move(promise)); } void download_block_proof(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, diff --git a/validator/full-node.hpp b/validator/full-node.hpp index e2d5c72b08..e501e1e996 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -71,8 +71,8 @@ class FullNodeImpl : public FullNode { void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); - void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise); + void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_block_proof(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 2aebaca747..e18a7f18f6 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -135,7 +135,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) = 0; virtual void send_get_zero_state_request(BlockIdExt id, td::uint32 priority, td::Promise promise) = 0; - virtual void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, + virtual void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::uint32 priority, td::Promise promise) = 0; virtual void send_get_block_proof_request(BlockIdExt block_id, td::uint32 priority, td::Promise promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index c48528b48c..ed6f2ddae8 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -911,7 +911,7 @@ void ValidatorManagerImpl::send_get_zero_state_request(BlockIdExt id, td::uint32 } void ValidatorManagerImpl::send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, - td::uint32 priority, + PersistentStateType type, td::uint32 priority, td::Promise promise) { UNREACHABLE(); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index ee83fc850b..3729f909af 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -242,8 +242,8 @@ class ValidatorManagerImpl : public ValidatorManager { void send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override; void send_get_zero_state_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override; - void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Promise promise) override; + void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Promise promise) override; void send_get_block_proof_request(BlockIdExt block_id, td::uint32 priority, td::Promise promise) override { UNREACHABLE(); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index ecfc8de7ed..b2c8de6174 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -308,8 +308,8 @@ class ValidatorManagerImpl : public ValidatorManager { void send_get_zero_state_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override { UNREACHABLE(); } - void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Promise promise) override { + void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Promise promise) override { UNREACHABLE(); } void send_get_block_proof_request(BlockIdExt block_id, td::uint32 priority, diff --git a/validator/manager.cpp b/validator/manager.cpp index 86d794bc87..5ca1581fcb 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1609,9 +1609,9 @@ void ValidatorManagerImpl::send_get_zero_state_request(BlockIdExt id, td::uint32 } void ValidatorManagerImpl::send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, - td::uint32 priority, + PersistentStateType type, td::uint32 priority, td::Promise promise) { - callback_->download_persistent_state(id, masterchain_block_id, priority, td::Timestamp::in(3600 * 3), + callback_->download_persistent_state(id, masterchain_block_id, type, priority, td::Timestamp::in(3600 * 3), std::move(promise)); } diff --git a/validator/manager.hpp b/validator/manager.hpp index ab86d2d372..bee59dff77 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -502,8 +502,8 @@ class ValidatorManagerImpl : public ValidatorManager { void send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override; void send_get_zero_state_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override; - void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Promise promise) override; + void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Promise promise) override; void send_get_block_proof_request(BlockIdExt block_id, td::uint32 priority, td::Promise promise) override; void send_get_block_proof_link_request(BlockIdExt block_id, td::uint32 priority, diff --git a/validator/validator.h b/validator/validator.h index d988261a97..943c6e9c52 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -185,8 +185,9 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise promise) = 0; virtual void download_zero_state(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; - virtual void download_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise) = 0; + virtual void download_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::uint32 priority, td::Timestamp timeout, + td::Promise promise) = 0; virtual void download_block_proof(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; virtual void download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, From 716475342f243237f91c14583d59e53a8861425b Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Thu, 12 Jun 2025 19:49:28 -0400 Subject: [PATCH 303/524] Store state split depth in persistent state descriptions --- tl/generate/scheme/ton_api.tl | 2 ++ tl/generate/scheme/ton_api.tlo | Bin 103992 -> 104360 bytes ton/ton-types.h | 7 +++- validator/db/statedb.cpp | 57 ++++++++++++++++++++++++++------- validator/manager.cpp | 4 +-- validator/state-serializer.cpp | 5 ++- 6 files changed, 60 insertions(+), 15 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 9b535ba0e6..42281547f5 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -568,6 +568,8 @@ db.state.asyncSerializer block:tonNode.blockIdExt last:tonNode.blockIdExt last_t db.state.hardforks blocks:(vector tonNode.blockIdExt) = db.state.Hardforks; db.state.dbVersion version:int = db.state.DbVersion; db.state.persistentStateDescriptionShards shard_blocks:(vector tonNode.blockIdExt) = db.state.PersistentStateDescriptionShards; +db.state.persistentStateDescriptionShard block:tonNode.blockIdExt split_depth:int = db.state.PersistentStateDescriptionShard; +db.state.persistentStateDescriptionShardsV2 shard_blocks:(vector db.state.PersistentStateDescriptionShard) = db.state.PersistentStateDescriptionShards; db.state.persistentStateDescriptionHeader masterchain_id:tonNode.blockIdExt start_time:int end_time:int = db.state.PersistentStateDescriptionHeader; db.state.persistentStateDescriptionsList list:(vector db.state.persistentStateDescriptionHeader) = db.state.PersistentStateDescriptionsList; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 27bc598a75eb3a54807bba7b2ef4f942a1e368ad..5c47336ef5bd9b69ea119d49cddd58431303ec1f 100644 GIT binary patch delta 199 zcmdn7hHb@qHr_|G^{p77Kx-p!G(W3n*sn<%lXLksCQsy%W|3#&u$r91uQSm>V)F+6 z721ruHb+{wheA~;PEejafuEHfXaGptbcYT`9UkuDf}G5f_>|Owl8ng*>}6O$Qj@Pt zuz?sRGSNpAWGc+ot8!d1{?jb(XitWi#>N(A1h!RVqJ;!VD@FZQDVRWOK^I L?I{x(udo0BR`EIo delta 77 zcmZ3no^8h(Hr_|G^{p77KyxE+G(T&=6oY4)lXLksCT|p%-u#1qg*M}k%?qvDLnj}Y ZXutxJo&05@517$>WzzO5lNhhC008_L9pwN3 diff --git a/ton/ton-types.h b/ton/ton-types.h index c7aff64480..bbc4b2a247 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -515,8 +515,13 @@ struct ValidatorSessionConfig { }; struct PersistentStateDescription : public td::CntObject { + struct ShardBlock { + BlockIdExt block; + td::uint32 split_depth; + }; + BlockIdExt masterchain_id; - std::vector shard_blocks; + std::vector shard_blocks; UnixTime start_time, end_time; virtual CntObject* make_copy() const { diff --git a/validator/db/statedb.cpp b/validator/db/statedb.cpp index fe3b9d7351..753ed7e646 100644 --- a/validator/db/statedb.cpp +++ b/validator/db/statedb.cpp @@ -277,16 +277,36 @@ void StateDb::add_persistent_state_description(td::Reflist_.resize(new_size); - std::vector> shard_blocks; - for (const BlockIdExt& block_id : desc->shard_blocks) { - shard_blocks.push_back(create_tl_block_id(block_id)); + bool can_be_stored_as_v1 = true; + for (auto const& [block_id, split_depth] : desc->shard_blocks) { + if (split_depth != 0) { + can_be_stored_as_v1 = false; + break; + } } + + td::BufferSlice serialized_shards; + + if (can_be_stored_as_v1) { + std::vector> shard_blocks; + for (auto const& [block_id, _] : desc->shard_blocks) { + shard_blocks.push_back(create_tl_block_id(block_id)); + } + serialized_shards = + create_serialize_tl_object(std::move(shard_blocks)); + } else { + std::vector> shard_blocks; + for (auto const& [block_id, split_depth] : desc->shard_blocks) { + shard_blocks.push_back(create_tl_object( + create_tl_block_id(block_id), static_cast(split_depth))); + } + serialized_shards = + create_serialize_tl_object(std::move(shard_blocks)); + } + auto key = create_hash_tl_object(desc->masterchain_id.seqno()); - kv_->set(key.as_slice(), - create_serialize_tl_object(std::move(shard_blocks)) - .as_slice()) - .ensure(); + kv_->set(key.as_slice(), serialized_shards.as_slice()).ensure(); list->list_.push_back(create_tl_object( create_tl_block_id(desc->masterchain_id), desc->start_time, desc->end_time)); @@ -325,11 +345,26 @@ void StateDb::get_persistent_state_descriptions(td::Promise(value, true); + auto F2 = fetch_tl_object(value, true); F2.ensure(); - for (const auto& block_id : F2.ok()->shard_blocks_) { - desc.shard_blocks.push_back(create_block_id(block_id)); - } + ton_api::downcast_call(*F2.ok().get(), + td::overloaded( + [&](ton_api::db_state_persistentStateDescriptionShards const& shards) { + for (auto const& block : shards.shard_blocks_) { + desc.shard_blocks.push_back({ + .block = create_block_id(block), + .split_depth = 0, + }); + } + }, + [&](ton_api::db_state_persistentStateDescriptionShardsV2 const& shards) { + for (auto const& shard_description : shards.shard_blocks_) { + desc.shard_blocks.push_back({ + .block = create_block_id(shard_description->block_), + .split_depth = static_cast(shard_description->split_depth_), + }); + } + })); result.push_back(td::Ref(true, std::move(desc))); } promise.set_result(std::move(result)); diff --git a/validator/manager.cpp b/validator/manager.cpp index 5ca1581fcb..1ff686dbd4 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3254,7 +3254,7 @@ void ValidatorManagerImpl::add_persistent_state_description(td::Refsecond; if (prev_desc->end_time <= now) { - for (const BlockIdExt &block_id : prev_desc->shard_blocks) { + for (auto const &[block_id, _] : prev_desc->shard_blocks) { persistent_state_blocks_.erase(block_id); } it = persistent_state_descriptions_.erase(it); @@ -3271,7 +3271,7 @@ void ValidatorManagerImpl::add_persistent_state_description_impl(td::Refmasterchain_id.to_str() << " start_time=" << desc->start_time << " end_time=" << desc->end_time; - for (const BlockIdExt &block_id : desc->shard_blocks) { + for (auto const &[block_id, _] : desc->shard_blocks) { persistent_state_blocks_[block_id] = desc; LOG(DEBUG) << "Persistent state description: shard block " << block_id.to_str(); } diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index 890d7e23f5..e65bad4c00 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -251,7 +251,10 @@ void AsyncStateSerializer::store_persistent_state_description(td::Refget_unix_time(); desc.end_time = ValidatorManager::persistent_state_ttl(desc.start_time); for (const auto &v : state->get_shards()) { - desc.shard_blocks.push_back(v->top_block_id()); + desc.shard_blocks.push_back({ + .block = v->top_block_id(), + .split_depth = state->persistent_state_split_depth(v->shard().workchain), + }); } td::actor::send_closure(manager_, &ValidatorManager::add_persistent_state_description, td::Ref(true, std::move(desc))); From 09dbf7250fbf6557460af9fbd73e9c61918840b1 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Thu, 12 Jun 2025 20:01:15 -0400 Subject: [PATCH 304/524] Pass correct state split depth to DownloadShardState ... and let DownloadShardState do nothing with it. With this commit, we are expecting DownloadShardState to pass correct PersistentStateType from one side and are passing split depth from the other side. So, only state merging in that single actor remains to be done. --- validator/downloaders/download-state.cpp | 13 ++++++++++--- validator/downloaders/download-state.hpp | 3 ++- validator/downloaders/wait-block-state.cpp | 22 +++++++++++++++++++--- validator/import-db-slice.cpp | 6 ++++-- validator/manager-init.cpp | 6 +++--- validator/shard-client.cpp | 17 +++++++++++------ validator/shard-client.hpp | 7 ++++++- 7 files changed, 55 insertions(+), 19 deletions(-) diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index cc942d1ec6..bebbe9e517 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -26,15 +26,22 @@ namespace ton { namespace validator { -DownloadShardState::DownloadShardState(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise> promise) +DownloadShardState::DownloadShardState(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::uint32 split_depth, + td::uint32 priority, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise> promise) : block_id_(block_id) , masterchain_block_id_(masterchain_block_id) + , split_depth_(split_depth) , priority_(priority) , manager_(manager) , timeout_(timeout) , promise_(std::move(promise)) { + CHECK(masterchain_block_id_.is_valid() || split_depth_ == 0); + + int shard_prefix_length = shard_pfx_len(block_id_.shard_full().shard); + if (shard_prefix_length >= static_cast(split_depth_)) { + split_depth_ = 0; + } } void DownloadShardState::start_up() { diff --git a/validator/downloaders/download-state.hpp b/validator/downloaders/download-state.hpp index bde80aae15..9d3a495cb2 100644 --- a/validator/downloaders/download-state.hpp +++ b/validator/downloaders/download-state.hpp @@ -27,7 +27,7 @@ namespace validator { class DownloadShardState : public td::actor::Actor { public: - DownloadShardState(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::uint32 priority, + DownloadShardState(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::uint32 split_depth, td::uint32 priority, td::actor::ActorId manager, td::Timestamp timeout, td::Promise> promise); @@ -58,6 +58,7 @@ class DownloadShardState : public td::actor::Actor { private: BlockIdExt block_id_; BlockIdExt masterchain_block_id_; + td::uint32 split_depth_; BlockHandle handle_; td::uint32 priority_; diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index c80e7d8966..00085e9606 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -117,10 +117,26 @@ void WaitBlockState::start() { td::actor::send_closure(SelfId, &WaitBlockState::written_state, R.move_as_ok()); } }); + BlockIdExt masterchain_id = persistent_state_desc_->masterchain_id; - td::actor::create_actor("downloadstate", handle_->id(), masterchain_id, priority_, manager_, - timeout_, std::move(P)) - .release(); + td::uint32 split_depth = 0; + bool block_found = false; + for (auto const& [block, block_split_depth] : persistent_state_desc_->shard_blocks) { + if (block == handle_->id()) { + split_depth = block_split_depth; + block_found = true; + break; + } + } + if (!block_found) { + LOG(ERROR) << "invalid persistent state description passed to WaitBlockState for block " + << handle_->id().to_str(); + P.set_error(td::Status::Error("invalid persistent state description")); + } else { + td::actor::create_actor("downloadstate", handle_->id(), masterchain_id, split_depth, + priority_, manager_, timeout_, std::move(P)) + .release(); + } } else if (!handle_->inited_prev() || (!handle_->inited_proof() && !handle_->inited_proof_link())) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { if (R.is_error()) { diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index c0d502cb2a..80fe141bbd 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -418,8 +418,10 @@ void ArchiveImporter::apply_shard_block_cont1(BlockHandle handle, BlockIdExt mas if (handle->id().seqno() == 0) { auto P = td::PromiseCreator::lambda( [promise = std::move(promise)](td::Result>) mutable { promise.set_value(td::Unit()); }); - td::actor::create_actor("downloadstate", handle->id(), masterchain_block_id, 2, manager_, - td::Timestamp::in(3600), std::move(P)) + td::actor::create_actor( + "downloadstate", handle->id(), masterchain_block_id, + start_state_->persistent_state_split_depth(handle->id().shard_full().workchain), 2, manager_, + td::Timestamp::in(3600), std::move(P)) .release(); return; } diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index 6f304680ba..004e5b120d 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -65,7 +65,7 @@ void ValidatorManagerMasterchainReiniter::got_masterchain_handle(BlockHandle han R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_masterchain_state); }); - td::actor::create_actor("downloadstate", handle_->id(), BlockIdExt{}, 2, manager_, + td::actor::create_actor("downloadstate", handle_->id(), BlockIdExt{}, 0, 2, manager_, td::Timestamp::in(3600), std::move(P)) .release(); return; @@ -80,7 +80,7 @@ void ValidatorManagerMasterchainReiniter::download_proof_link() { R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::downloaded_zero_state); }); - td::actor::create_actor("downloadstate", handle_->id(), BlockIdExt{}, 2, manager_, + td::actor::create_actor("downloadstate", handle_->id(), BlockIdExt{}, 0, 2, manager_, td::Timestamp::in(3600), std::move(P)) .release(); } else { @@ -263,7 +263,7 @@ void ValidatorManagerMasterchainReiniter::download_masterchain_state() { R.move_as_ok()); } }); - td::actor::create_actor("downloadstate", block_id_, block_id_, 2, manager_, + td::actor::create_actor("downloadstate", block_id_, block_id_, 0, 2, manager_, td::Timestamp::in(3600 * 3), std::move(P)) .release(); } diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index ac86cf37d4..e5fd0a2ac9 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -74,24 +74,29 @@ void ShardClient::got_init_state_from_db(td::Ref state) { } void ShardClient::start_up_init_mode() { - std::vector shards; - for (const auto& s : masterchain_state_->get_shards()) { + std::vector shards; + for (const auto &s : masterchain_state_->get_shards()) { if (opts_->need_monitor(s->shard(), masterchain_state_)) { - shards.push_back(s->top_block_id()); + auto shard = s->top_block_id(); + shards.push_back({ + .shard = shard, + .split_depth = masterchain_state_->persistent_state_split_depth(shard.shard_full().workchain), + }); } } download_shard_states(masterchain_block_handle_->id(), std::move(shards), 0); } -void ShardClient::download_shard_states(BlockIdExt masterchain_block_id, std::vector shards, size_t idx) { +void ShardClient::download_shard_states(BlockIdExt masterchain_block_id, std::vector shards, + size_t idx) { if (idx >= shards.size()) { LOG(WARNING) << "downloaded all shard states"; applied_all_shards(); return; } - BlockIdExt block_id = shards[idx]; + auto [block_id, split_depth] = shards[idx]; td::actor::create_actor( - "downloadstate", block_id, masterchain_block_handle_->id(), 2, manager_, td::Timestamp::in(3600 * 5), + "downloadstate", block_id, masterchain_block_handle_->id(), split_depth, 2, manager_, td::Timestamp::in(3600 * 5), [=, SelfId = actor_id(this), shards = std::move(shards)](td::Result> R) { R.ensure(); td::actor::send_closure(SelfId, &ShardClient::download_shard_states, masterchain_block_id, std::move(shards), diff --git a/validator/shard-client.hpp b/validator/shard-client.hpp index 7c2c978c2b..c13b625aa8 100644 --- a/validator/shard-client.hpp +++ b/validator/shard-client.hpp @@ -27,6 +27,11 @@ namespace validator { class ShardClient : public td::actor::Actor { private: + struct DownloadableShard { + BlockIdExt shard; + td::uint32 split_depth; + }; + td::Ref opts_; BlockHandle masterchain_block_handle_; @@ -64,7 +69,7 @@ class ShardClient : public td::actor::Actor { void start_up() override; void start_up_init_mode(); - void download_shard_states(BlockIdExt masterchain_block_id, std::vector shards, size_t idx); + void download_shard_states(BlockIdExt masterchain_block_id, std::vector shards, size_t idx); void start(); void got_state_from_db(BlockIdExt masterchain_block_id); void got_init_handle_from_db(BlockHandle handle); From d3cec64d803ee29f48ae2ad90d4f365456aca69f Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Sun, 15 Jun 2025 10:13:37 -0400 Subject: [PATCH 305/524] Actually deserialize split states We don't yet store state parts in DB as this requires yet another instance of "change every single actor in the observable universe". --- crypto/vm/cells/MerkleProof.cpp | 6 + crypto/vm/cells/MerkleProof.h | 1 + validator/downloaders/download-state.cpp | 241 ++++++++++++++++++++++- validator/downloaders/download-state.hpp | 18 +- validator/fabric.h | 2 +- validator/impl/fabric.cpp | 2 +- 6 files changed, 257 insertions(+), 13 deletions(-) diff --git a/crypto/vm/cells/MerkleProof.cpp b/crypto/vm/cells/MerkleProof.cpp index 26dff7879d..4c1fcbdb7c 100644 --- a/crypto/vm/cells/MerkleProof.cpp +++ b/crypto/vm/cells/MerkleProof.cpp @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "vm/cells/MerkleProof.h" +#include "td/utils/Status.h" #include "vm/cells/CellBuilder.h" #include "vm/cells/CellSlice.h" #include "vm/boc.h" @@ -152,6 +153,11 @@ Ref MerkleProof::virtualize(Ref cell, int virtualization) { return virtualize_raw(r_raw.move_as_ok(), {0 /*level*/, static_cast(virtualization)}); } +td::Result> MerkleProof::try_virtualize(Ref cell, int virtualization) { + TRY_RESULT(unpacked_cell, unpack_proof(std::move(cell))); + return unpacked_cell->virtualize({0 /*level*/, static_cast(virtualization)}); +} + class MerkleProofCombineFast { public: MerkleProofCombineFast(Ref a, Ref b) : a_(std::move(a)), b_(std::move(b)) { diff --git a/crypto/vm/cells/MerkleProof.h b/crypto/vm/cells/MerkleProof.h index fc2cb6ebde..0f0eaef096 100644 --- a/crypto/vm/cells/MerkleProof.h +++ b/crypto/vm/cells/MerkleProof.h @@ -36,6 +36,7 @@ class MerkleProof { // cell must have zero level and must be a MerkleProof static Ref virtualize(Ref cell, int virtualization); + static td::Result> try_virtualize(Ref cell, int virtualization = 1); static Ref combine(Ref a, Ref b); static td::Result> combine_status(Ref a, Ref b); diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index bebbe9e517..42f2c5c40f 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -21,11 +21,103 @@ #include "common/checksum.h" #include "common/delay.h" #include "ton/ton-io.hpp" +#include "vm/cells/MerkleProof.h" +#include "crypto/block/block-auto.h" +#include "crypto/block/block-parse.h" namespace ton { namespace validator { +class SplitStateDeserializer { + public: + td::Result> get_effective_shards_from_header(ShardId shard_id, RootHash root_hash, + td::Ref wrapped_header, + td::uint32 split_depth) { + int shard_prefix_length = shard_pfx_len(shard_id); + CHECK(split_depth <= 63 && shard_prefix_length < static_cast(split_depth)); + + try { + TRY_RESULT(header, vm::MerkleProof::try_virtualize(wrapped_header)); + + if (RootHash{header->get_hash().bits()} != root_hash) { + return td::Status::Error("Hash mismatch in split state header"); + } + + auto shard_state_cs = vm::load_cell_slice(header); + bool rc = block::gen::t_ShardStateUnsplit.unpack(shard_state_cs, shard_state_); + if (!rc) { + return td::Status::Error("Cannot deserialize ShardStateUnsplit"); + } + + vm::AugmentedDictionary accounts{ + vm::load_cell_slice_ref(shard_state_.accounts), + 256, + block::tlb::aug_ShardAccounts, + false, + }; + + std::vector parts; + + // The following loop is the same as in state-serializer.cpp. + ShardId effective_shard = shard_id ^ (1ULL << (63 - shard_prefix_length)) ^ (1ULL << (63 - split_depth)); + ShardId increment = 1ULL << (64 - split_depth); + + for (int i = 0; i < (1 << (split_depth - shard_prefix_length)); ++i, effective_shard += increment) { + td::BitArray<64> prefix; + prefix.store_ulong(effective_shard); + auto account_dict_part = accounts; + account_dict_part.cut_prefix_subdict(prefix.bits(), split_depth); + + if (!account_dict_part.is_empty()) { + parts.push_back({effective_shard, account_dict_part.get_wrapped_dict_root()->get_hash()}); + } + } + + // Now check that header does not contain pruned cells outside of accounts dict. For that, we + // just replace account dict with an empty cell and see if header remains virtualized or not. + shard_state_.accounts = vm::DataCell::create("", 0, {}, false).move_as_ok(); + + vm::CellBuilder cb; + block::gen::t_ShardStateUnsplit.pack(cb, shard_state_); + if (cb.finalize()->get_virtualization() > 0) { + return td::Status::Error("State headers is pruned outside of account dict"); + } + + return parts; + } catch (vm::VmVirtError const&) { + return td::Status::Error("Insufficient number of cells in split state header"); + } + } + + td::Ref merge(std::vector> const& parts) { + vm::AugmentedDictionary accounts{256, block::tlb::aug_ShardAccounts}; + for (auto const& part_root : parts) { + vm::AugmentedDictionary part{ + vm::load_cell_slice_ref(part_root), + 256, + block::tlb::aug_ShardAccounts, + false, + }; + bool rc = accounts.combine_with(part); + LOG_CHECK(rc) << "Split state parts have been validated but merging them still resulted in a conflict"; + } + + CHECK(accounts.is_valid()); + + shard_state_.accounts = accounts.get_wrapped_dict_root(); + + vm::CellBuilder cb; + block::gen::t_ShardStateUnsplit.pack(cb, shard_state_); + auto state_root = cb.finalize(); + CHECK(state_root->get_virtualization() == 0); + return state_root; + } + + private: + block::gen::ShardStateUnsplit::Record shard_state_; +}; + DownloadShardState::DownloadShardState(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::uint32 split_depth, td::uint32 priority, td::actor::ActorId manager, td::Timestamp timeout, td::Promise> promise) @@ -42,8 +134,13 @@ DownloadShardState::DownloadShardState(BlockIdExt block_id, BlockIdExt mastercha if (shard_prefix_length >= static_cast(split_depth_)) { split_depth_ = 0; } + + LOG(INFO) << "requested to download state of " << block_id.to_str() << " referenced by " + << masterchain_block_id.to_str() << " with split depth " << split_depth; } +DownloadShardState::~DownloadShardState() = default; + void DownloadShardState::start_up() { status_ = ProcessStatus(manager_, "process.download_state"); alarm_timestamp() = timeout_; @@ -71,6 +168,8 @@ void DownloadShardState::got_block_handle(BlockHandle handle) { } void DownloadShardState::retry() { + deserializer_ = {}; + parts_.clear(); download_state(); } @@ -119,20 +218,35 @@ void DownloadShardState::checked_proof_link() { } }); td::actor::send_closure(manager_, &ValidatorManager::try_get_static_file, block_id_.file_hash, std::move(P)); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading zero state"); } else { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - fail_handler(SelfId, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &DownloadShardState::downloaded_shard_state, R.move_as_ok()); - } - }); CHECK(masterchain_block_id_.is_valid()); CHECK(masterchain_block_id_.is_masterchain()); - td::actor::send_closure(manager_, &ValidatorManager::send_get_persistent_state_request, block_id_, - masterchain_block_id_, UnsplitStateType{}, priority_, std::move(P)); + + if (split_depth_ == 0) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + fail_handler(SelfId, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &DownloadShardState::downloaded_shard_state, R.move_as_ok()); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::send_get_persistent_state_request, block_id_, + masterchain_block_id_, UnsplitStateType{}, priority_, std::move(P)); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading state"); + } else { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + fail_handler(SelfId, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &DownloadShardState::downloaded_split_state_header, R.move_as_ok()); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::send_get_persistent_state_request, block_id_, + masterchain_block_id_, SplitPersistentStateType{}, priority_, std::move(P)); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading state header"); + } } - status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading state"); } void DownloadShardState::download_zero_state() { @@ -200,6 +314,113 @@ void DownloadShardState::checked_shard_state() { } } +void DownloadShardState::downloaded_split_state_header(td::BufferSlice data) { + LOG(INFO) << "processing state header"; + status_.set_status(PSTRING() << block_id_.id.to_str() << " : processing state header"); + + deserializer_ = std::make_unique(); + + auto maybe_header = vm::std_boc_deserialize(data); + if (maybe_header.is_error()) { + fail_handler(actor_id(this), maybe_header.move_as_error()); + return; + } + + auto maybe_parts = deserializer_->get_effective_shards_from_header(block_id_.shard_full().shard, handle_->state(), + maybe_header.move_as_ok(), split_depth_); + if (maybe_parts.is_error()) { + fail_handler(actor_id(this), maybe_parts.move_as_error()); + return; + } + + parts_ = maybe_parts.move_as_ok(); + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &DownloadShardState::written_split_state_file); + }); + td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file, block_id_, masterchain_block_id_, + SplitPersistentStateType{}, std::move(data), std::move(P)); +} + +namespace { + +void retry_part_download(td::actor::ActorId SelfId, td::Status error) { + LOG(WARNING) << "failed to download state part : " << error; + delay_action([=]() { td::actor::send_closure(SelfId, &DownloadShardState::written_split_state_file); }, + td::Timestamp::in(1.0)); +} + +} // namespace + +void DownloadShardState::written_split_state_file() { + if (stored_parts_.size() == parts_.size()) { + auto state_root = deserializer_->merge(stored_parts_); + auto maybe_state = create_shard_state(block_id_, state_root); + + // We cannot rollback database changes here without significant elbow grease. + maybe_state.ensure(); + state_ = maybe_state.move_as_ok(); + CHECK(state_->root_hash() == handle_->state()); + + written_shard_state_file(); + return; + } + + size_t idx = stored_parts_.size(); + + LOG(INFO) << "downloading state part " << idx + 1 << " out of " << parts_.size(); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading state part (part " << idx + 1 << " out of " + << parts_.size() << ")"); + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + retry_part_download(SelfId, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &DownloadShardState::downloaded_state_part, R.move_as_ok()); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::send_get_persistent_state_request, block_id_, + masterchain_block_id_, SplitAccountStateType{parts_[idx].effective_shard}, priority_, + std::move(P)); +} + +void DownloadShardState::downloaded_state_part(td::BufferSlice data) { + size_t idx = stored_parts_.size(); + + LOG(INFO) << "processing state part " << idx + 1 << " out of " << parts_.size(); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : processing state part (part " << idx + 1 << " out of " + << parts_.size() << ")"); + + auto maybe_part = vm::std_boc_deserialize(data); + if (maybe_part.is_error()) { + retry_part_download(actor_id(this), maybe_part.move_as_error()); + return; + } + + auto root = maybe_part.move_as_ok(); + if (root->get_hash() != parts_[idx].root_hash) { + auto error_message = + "Hash mismatch for part " + + persistent_state_type_to_string(block_id_.shard_full(), SplitAccountStateType{parts_[idx].effective_shard}); + retry_part_download(actor_id(this), td::Status::Error(error_message)); + return; + } + + stored_parts_.push_back(root); + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &DownloadShardState::written_split_state_file); + }); + td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file, block_id_, masterchain_block_id_, + SplitAccountStateType{parts_[idx].effective_shard}, std::move(data), std::move(P)); + + LOG(INFO) << "storing state part to file " << idx + 1 << " out of " << parts_.size(); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : storing state part to file (part " << idx + 1 + << " out of " << parts_.size() << ")"); +} + void DownloadShardState::written_shard_state_file() { status_.set_status(PSTRING() << block_id_.id.to_str() << " : storing state to celldb"); LOG(WARNING) << "written shard state file " << block_id_.to_str(); diff --git a/validator/downloaders/download-state.hpp b/validator/downloaders/download-state.hpp index 9d3a495cb2..0f66bad4b5 100644 --- a/validator/downloaders/download-state.hpp +++ b/validator/downloaders/download-state.hpp @@ -25,11 +25,19 @@ namespace ton { namespace validator { +class SplitStateDeserializer; + +struct SplitStatePart { + ShardId effective_shard; + vm::CellHash root_hash; +}; + class DownloadShardState : public td::actor::Actor { public: DownloadShardState(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::uint32 split_depth, td::uint32 priority, td::actor::ActorId manager, td::Timestamp timeout, td::Promise> promise); + ~DownloadShardState(); void start_up() override; void got_block_handle(BlockHandle handle); @@ -43,8 +51,12 @@ class DownloadShardState : public td::actor::Actor { void downloaded_zero_state(td::BufferSlice data); void downloaded_shard_state(td::BufferSlice data); - void checked_shard_state(); + + void downloaded_split_state_header(td::BufferSlice data); + void written_split_state_file(); + void downloaded_state_part(td::BufferSlice data); + void written_shard_state_file(); void written_shard_state(td::Ref state); void written_block_handle(); @@ -67,6 +79,10 @@ class DownloadShardState : public td::actor::Actor { td::Timestamp timeout_; td::Promise> promise_; + std::unique_ptr deserializer_; + std::vector parts_; + std::vector> stored_parts_; + td::BufferSlice data_; td::Ref state_; diff --git a/validator/fabric.h b/validator/fabric.h index 2c39acebed..7776c8e891 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -39,7 +39,7 @@ td::Result> create_proof(BlockIdExt masterchain_block_id, td::Buf td::Result> create_proof_link(BlockIdExt block_id, td::BufferSlice proof); td::Result> create_signature_set(td::BufferSlice sig_set); td::Result> create_shard_state(BlockIdExt block_id, td::BufferSlice data); -td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell); +td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell); td::Result create_block_handle(td::BufferSlice data); td::Result create_block_handle(td::Slice data); td::Result create_temp_block_handle(td::BufferSlice data); diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 65b922624c..55b416859a 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -84,7 +84,7 @@ td::Result> create_shard_state(BlockIdExt block_id, td::Buff } } -td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell) { +td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell) { auto res = ShardStateQ::fetch(block_id, {}, std::move(root_cell)); if (res.is_error()) { return res.move_as_error(); From 39d5519c97bd6ac58741f6ac09bd6a7d5de0ae3a Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Sun, 15 Jun 2025 14:55:54 -0400 Subject: [PATCH 306/524] Save intermediate split state parts in DB This way, we don't need to keep full shard state at any point during deserialization. --- validator/db/rootdb.cpp | 5 +++++ validator/db/rootdb.hpp | 2 ++ validator/downloaders/download-state.cpp | 28 ++++++++++++++++++++---- validator/downloaders/download-state.hpp | 4 +++- validator/interfaces/db.h | 2 ++ validator/interfaces/validator-manager.h | 2 ++ validator/manager-disk.cpp | 5 +++++ validator/manager-disk.hpp | 2 ++ validator/manager-hardfork.hpp | 5 +++++ validator/manager.cpp | 5 +++++ validator/manager.hpp | 2 ++ 11 files changed, 57 insertions(+), 5 deletions(-) diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index d041228c06..8eb2b76a93 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -281,6 +281,11 @@ void RootDb::get_block_state(ConstBlockHandle handle, td::Promise cell, + td::Promise> promise) { + td::actor::send_closure(cell_db_, &CellDb::store_cell, BlockIdExt{effective_block}, cell, std::move(promise)); +} + void RootDb::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(cell_db_, &CellDb::get_cell_db_reader, std::move(promise)); } diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 57e560e1c4..782e48532c 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -63,6 +63,8 @@ class RootDb : public Db { void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; void get_block_state(ConstBlockHandle handle, td::Promise> promise) override; + void store_block_state_part(BlockId effective_block, td::Ref cell, + td::Promise> promise) override; void get_cell_db_reader(td::Promise> promise) override; void store_block_handle(BlockHandle handle, td::Promise promise) override; diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index 42f2c5c40f..8e4cded06e 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -337,7 +337,7 @@ void DownloadShardState::downloaded_split_state_header(td::BufferSlice data) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); - td::actor::send_closure(SelfId, &DownloadShardState::written_split_state_file); + td::actor::send_closure(SelfId, &DownloadShardState::download_next_part_or_finish); }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file, block_id_, masterchain_block_id_, SplitPersistentStateType{}, std::move(data), std::move(P)); @@ -347,13 +347,13 @@ namespace { void retry_part_download(td::actor::ActorId SelfId, td::Status error) { LOG(WARNING) << "failed to download state part : " << error; - delay_action([=]() { td::actor::send_closure(SelfId, &DownloadShardState::written_split_state_file); }, + delay_action([=]() { td::actor::send_closure(SelfId, &DownloadShardState::download_next_part_or_finish); }, td::Timestamp::in(1.0)); } } // namespace -void DownloadShardState::written_split_state_file() { +void DownloadShardState::download_next_part_or_finish() { if (stored_parts_.size() == parts_.size()) { auto state_root = deserializer_->merge(stored_parts_); auto maybe_state = create_shard_state(block_id_, state_root); @@ -411,7 +411,7 @@ void DownloadShardState::downloaded_state_part(td::BufferSlice data) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); - td::actor::send_closure(SelfId, &DownloadShardState::written_split_state_file); + td::actor::send_closure(SelfId, &DownloadShardState::written_state_part_file); }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file, block_id_, masterchain_block_id_, SplitAccountStateType{parts_[idx].effective_shard}, std::move(data), std::move(P)); @@ -421,6 +421,26 @@ void DownloadShardState::downloaded_state_part(td::BufferSlice data) { << " out of " << parts_.size() << ")"); } +void DownloadShardState::written_state_part_file() { + size_t idx = stored_parts_.size() - 1; + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + R.ensure(); + td::actor::send_closure(SelfId, &DownloadShardState::saved_state_part_into_celldb, R.move_as_ok()); + }); + td::actor::send_closure(manager_, &ValidatorManager::store_block_state_part, + BlockId{block_id_.shard_full().workchain, parts_[idx].effective_shard, block_id_.seqno()}, + stored_parts_.back(), std::move(P)); + LOG(INFO) << "saving to celldb state part " << idx + 1 << " out of " << parts_.size(); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : saving state part to celldb (part " << idx + 1 + << " out of " << parts_.size() << ")"); +} + +void DownloadShardState::saved_state_part_into_celldb(td::Ref cell) { + stored_parts_.back() = cell; + download_next_part_or_finish(); +} + void DownloadShardState::written_shard_state_file() { status_.set_status(PSTRING() << block_id_.id.to_str() << " : storing state to celldb"); LOG(WARNING) << "written shard state file " << block_id_.to_str(); diff --git a/validator/downloaders/download-state.hpp b/validator/downloaders/download-state.hpp index 0f66bad4b5..2ba30e922f 100644 --- a/validator/downloaders/download-state.hpp +++ b/validator/downloaders/download-state.hpp @@ -54,8 +54,10 @@ class DownloadShardState : public td::actor::Actor { void checked_shard_state(); void downloaded_split_state_header(td::BufferSlice data); - void written_split_state_file(); + void download_next_part_or_finish(); void downloaded_state_part(td::BufferSlice data); + void written_state_part_file(); + void saved_state_part_into_celldb(td::Ref cell); void written_shard_state_file(); void written_shard_state(td::Ref state); diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index afd087549f..f8fce7ba9b 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -52,6 +52,8 @@ class Db : public td::actor::Actor { virtual void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; virtual void get_block_state(ConstBlockHandle handle, td::Promise> promise) = 0; + virtual void store_block_state_part(BlockId effective_block, td::Ref cell, + td::Promise> promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index e18a7f18f6..34d7d57c77 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -70,6 +70,8 @@ class ValidatorManager : public ValidatorManagerInterface { } virtual void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; + virtual void store_block_state_part(BlockId effective_block, td::Ref cell, + td::Promise> promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::BufferSlice state, diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index ed6f2ddae8..3cc463a654 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -688,6 +688,11 @@ void ValidatorManagerImpl::set_block_state(BlockHandle handle, td::Ref cell, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::store_block_state_part, effective_block, cell, std::move(promise)); +} + void ValidatorManagerImpl::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise)); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 3729f909af..ebbdd88877 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -150,6 +150,8 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; + void store_block_state_part(BlockId effective_block, td::Ref cell, + td::Promise> promise) override; void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::BufferSlice state, td::Promise promise) override; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index b2c8de6174..20430d8042 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -176,6 +176,11 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { UNREACHABLE(); } + void store_block_state_part(BlockId effective_block, td::Ref cell, + td::Promise> promise) override { + UNREACHABLE(); + } + void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::BufferSlice state, td::Promise promise) override { diff --git a/validator/manager.cpp b/validator/manager.cpp index 1ff686dbd4..31e7573622 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1233,6 +1233,11 @@ void ValidatorManagerImpl::set_block_state(BlockHandle handle, td::Ref cell, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::store_block_state_part, effective_block, cell, std::move(promise)); +} + void ValidatorManagerImpl::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise)); } diff --git a/validator/manager.hpp b/validator/manager.hpp index bee59dff77..e90d5b3472 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -410,6 +410,8 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; + void store_block_state_part(BlockId effective_block, td::Ref cell, + td::Promise> promise) override; void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::BufferSlice state, td::Promise promise) override; From f42d13e4402b80fdf19c770779490785a0b36a05 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 20 Jun 2025 17:50:29 +0300 Subject: [PATCH 307/524] Various fixes in node, add tests for p256 signature (#1716) --- crypto/Ed25519.cpp | 2 +- crypto/ellcurve/p256.cpp | 5 ++++- crypto/test/fift.cpp | 4 ++++ crypto/test/fift/p256.fif | 25 +++++++++++++++++++++++++ lite-client/lite-client.cpp | 7 ++++++- storage/MicrochunkTree.cpp | 3 ++- tdutils/td/utils/BigNum.cpp | 7 +++++-- tdutils/td/utils/BigNum.h | 2 +- tdutils/td/utils/format.h | 5 +++-- test/regression-tests.ans | 1 + validator/impl/external-message.cpp | 1 + validator/manager.cpp | 2 -- 12 files changed, 53 insertions(+), 11 deletions(-) create mode 100644 crypto/test/fift/p256.fif diff --git a/crypto/Ed25519.cpp b/crypto/Ed25519.cpp index fa5b4f2f6d..20fb8a3ca2 100644 --- a/crypto/Ed25519.cpp +++ b/crypto/Ed25519.cpp @@ -237,7 +237,7 @@ Result Ed25519::compute_shared_secret(const PublicKey &public_key, BigNum::mod_sub(y2, p, y2, p, context); BigNum inverse_y_plus_1; - BigNum::mod_inverse(inverse_y_plus_1, y2, p, context); + TRY_STATUS(BigNum::mod_inverse(inverse_y_plus_1, y2, p, context)); BigNum u; BigNum::mod_mul(u, y, inverse_y_plus_1, p, context); diff --git a/crypto/ellcurve/p256.cpp b/crypto/ellcurve/p256.cpp index de5393723d..684ac8ca08 100644 --- a/crypto/ellcurve/p256.cpp +++ b/crypto/ellcurve/p256.cpp @@ -58,7 +58,7 @@ td::Status p256_check_signature(td::Slice data, td::Slice public_key, td::Slice SCOPE_EXIT { EVP_MD_CTX_free(md_ctx); }; - if (EVP_DigestVerifyInit(md_ctx, nullptr, nullptr, nullptr, pkey) <= 0) { + if (EVP_DigestVerifyInit(md_ctx, nullptr, EVP_sha256(), nullptr, pkey) <= 0) { return td::Status::Error("Can't init DigestVerify"); } ECDSA_SIG* sig = ECDSA_SIG_new(); @@ -71,7 +71,10 @@ td::Status p256_check_signature(td::Slice data, td::Slice public_key, td::Slice BIGNUM* r = BN_bin2bn(buf, 33, nullptr); std::copy(signature.ubegin() + 32, signature.ubegin() + 64, buf + 1); BIGNUM* s = BN_bin2bn(buf, 33, nullptr); + CHECK(r != nullptr && s != nullptr); if (ECDSA_SIG_set0(sig, r, s) != 1) { + BN_free(r); + BN_free(s); return td::Status::Error("Invalid signature"); } unsigned char* signature_encoded = nullptr; diff --git a/crypto/test/fift.cpp b/crypto/test/fift.cpp index 38a9ddd92b..55a6ded755 100644 --- a/crypto/test/fift.cpp +++ b/crypto/test/fift.cpp @@ -175,3 +175,7 @@ TEST(Fift, test_secp256k1) { TEST(Fift, test_get_extra_balance) { run_fift("get_extra_balance.fif"); } + +TEST(Fift, test_p256) { + run_fift("p256.fif"); +} diff --git a/crypto/test/fift/p256.fif b/crypto/test/fift/p256.fif new file mode 100644 index 0000000000..ecd43d19b9 --- /dev/null +++ b/crypto/test/fift/p256.fif @@ -0,0 +1,25 @@ +"Asm.fif" include +"FiftExt.fif" include + +// Test vectors from https://datatracker.ietf.org/doc/html/rfc6979#appendix-A.2.5 + +x{73616d706c65} // "sample" +x{EFD48B2AACB6A8FD1140DD9CD45E81D69D2C877B56AAF991C34D0EA84EAF3716F7CB1C942D657C41D436C7A1B6E29F65F3E900DBB9AFF4064DC4AB2F843ACDA8} +x{0360FED4BA255A9D31C961EB74C6356D68C049B8923B61FA6CE669622E60F29FB6} + +<{ P256_CHKSIGNS }>s 64 runvmx .s // -1 0 +drop drop + +x{74657374} // "test" +x{F1ABB023518351CD71D881567B1EA663ED3EFCF6C5132B354F28D3B0B7D38367019F4113742A2B14BD25926B49C649155F267E60D3814B4C0CC84250E46F0083} +x{0360FED4BA255A9D31C961EB74C6356D68C049B8923B61FA6CE669622E60F29FB6} + +<{ P256_CHKSIGNS }>s 64 runvmx .s // -1 0 +drop drop + +x{74657374} // "test" +x{83910E8B48BB0C74244EBDF7F07A1C5413D61472BD941EF3920E623FBCCEBEB68DDBEC54CF8CD5874883841D712142A56A8D0F218F5003CB0296B6B509619F2C} +x{0360FED4BA255A9D31C961EB74C6356D68C049B8923B61FA6CE669622E60F29FB6} + +<{ P256_CHKSIGNS }>s 64 runvmx .s // 0 0 +drop drop diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 1050e6d270..e97b42c807 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -422,7 +422,12 @@ void TestNode::got_server_mc_block_id(ton::BlockIdExt blkid, ton::ZeroStateIdExt } td::TerminalIO::out() << "latest masterchain block known to server is " << blkid.to_str(); if (created > 0) { - td::TerminalIO::out() << " created at " << created << " (" << now() - created << " seconds ago)\n"; + auto time = now(); + if (time >= created) { + td::TerminalIO::out() << " created at " << created << " (" << time - created << " seconds ago)\n"; + } else { + td::TerminalIO::out() << " created at " << created << " (" << created - time << " seconds in the future)\n"; + } } else { td::TerminalIO::out() << "\n"; } diff --git a/storage/MicrochunkTree.cpp b/storage/MicrochunkTree.cpp index 3249d25b42..1c73593363 100644 --- a/storage/MicrochunkTree.cpp +++ b/storage/MicrochunkTree.cpp @@ -188,7 +188,8 @@ td::Result> MicrochunkTree::get_proof(td::uint64 l, td::uint64 if (!torrent.inited_info()) { return td::Status::Error("Torrent info is not ready"); } - if (!torrent.get_info().piece_size % MICROCHUNK_SIZE != 0) { + // piece_size must be an exact multiple of MICROCHUNK_SIZE + if ((torrent.get_info().piece_size % MICROCHUNK_SIZE) != 0) { return td::Status::Error("Invalid piece size in torrent"); } td::Ref root_raw = vm::CellSlice(vm::NoVm(), root_proof_).prefetch_ref(); diff --git a/tdutils/td/utils/BigNum.cpp b/tdutils/td/utils/BigNum.cpp index 9de11fcae9..22f5244652 100644 --- a/tdutils/td/utils/BigNum.cpp +++ b/tdutils/td/utils/BigNum.cpp @@ -291,9 +291,12 @@ void BigNum::mod_mul(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumCon LOG_IF(FATAL, result != 1); } -void BigNum::mod_inverse(BigNum &r, BigNum &a, const BigNum &m, BigNumContext &context) { +td::Status BigNum::mod_inverse(BigNum &r, BigNum &a, const BigNum &m, BigNumContext &context) { auto result = BN_mod_inverse(r.impl_->big_num, a.impl_->big_num, m.impl_->big_num, context.impl_->big_num_context); - LOG_IF(FATAL, result != r.impl_->big_num); + if (result != r.impl_->big_num) { + return td::Status::Error("Failed to compute modulo inverse"); + } + return td::Status::OK(); } void BigNum::div(BigNum *quotient, BigNum *remainder, const BigNum ÷nd, const BigNum &divisor, diff --git a/tdutils/td/utils/BigNum.h b/tdutils/td/utils/BigNum.h index b0094659ce..682b441ab9 100644 --- a/tdutils/td/utils/BigNum.h +++ b/tdutils/td/utils/BigNum.h @@ -109,7 +109,7 @@ class BigNum { static void mod_mul(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context); - static void mod_inverse(BigNum &r, BigNum &a, const BigNum &m, BigNumContext &context); + static td::Status mod_inverse(BigNum &r, BigNum &a, const BigNum &m, BigNumContext &context); static void div(BigNum *quotient, BigNum *remainder, const BigNum ÷nd, const BigNum &divisor, BigNumContext &context); diff --git a/tdutils/td/utils/format.h b/tdutils/td/utils/format.h index 30d183abe3..cf4447186a 100644 --- a/tdutils/td/utils/format.h +++ b/tdutils/td/utils/format.h @@ -179,11 +179,12 @@ inline StringBuilder &operator<<(StringBuilder &logger, Time t) { double value; }; - static constexpr NamedValue durations[] = {{"ns", 1e-9}, {"us", 1e-6}, {"ms", 1e-3}, {"s", 1}}; + static constexpr NamedValue durations[] = {{"ns", 1e-9}, {"us", 1e-6}, {"ms", 1e-3}, + {"s", 1}, {"h", 3600}, {"d", 86400}}; static constexpr size_t durations_n = sizeof(durations) / sizeof(NamedValue); size_t i = 0; - while (i + 1 < durations_n && t.seconds_ > 10 * durations[i + 1].value) { + while (i + 1 < durations_n && std::abs(t.seconds_) > 10 * durations[i + 1].value) { i++; } logger << StringBuilder::FixedDouble(t.seconds_ / durations[i].value, 1) << durations[i].name; diff --git a/test/regression-tests.ans b/test/regression-tests.ans index 5831ebe740..16db1c9f1f 100644 --- a/test/regression-tests.ans +++ b/test/regression-tests.ans @@ -21,6 +21,7 @@ Test_Fift_test_hash_ext_default 686fc5680feca5b3bb207768215b27f6872a95128762dee0 Test_Fift_test_hmap_default c269246882039824bb5822e896c3e6e82ef8e1251b6b251f5af8ea9fb8d05067 Test_Fift_test_levels_default 9fba4a7c98aec9000f42846d6e5fd820343ba61d68f9139dd16c88ccda757cf3 Test_Fift_test_namespaces_default e6419619c51332fb5e8bf22043ef415db686c47fe24f03061e5ad831014e7c6c +Test_Fift_test_p256_default e1948ddd3d2686baa9f70fdf376ffcebbc2ec5f20eeb366cd856254e61fbfa31 Test_Fift_test_rist255_default f4d7558f200a656934f986145c19b1dedbe2ad029292a5a975576d6891e25fc4 Test_Fift_test_secp256k1_default 3118450dace6af05fcdbd54a87d9446162ce11ac6ef6dfc57998cf113587d602 Test_Fift_test_sort2_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index 8b1f5eb7f9..54acb105e8 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -112,6 +112,7 @@ void ExtMessageQ::run_message(td::Ref message, td::actor::ActorIdclone()}); } - LOG(ERROR) << "XXXX download " << id.id.to_str(); callback_->download_block(id, priority, td::Timestamp::in(10.0), std::move(promise)); } @@ -1632,7 +1631,6 @@ void ValidatorManagerImpl::send_get_block_proof_link_request(BlockIdExt block_id return; } } - LOG(ERROR) << "XXXX download link " << block_id.id.to_str(); callback_->download_block_proof_link(block_id, priority, td::Timestamp::in(10.0), std::move(promise)); } From 2e0378df313931c73c45011de197fa5938a7ada3 Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Fri, 20 Jun 2025 10:52:54 -0400 Subject: [PATCH 308/524] Add more CMake options (#1715) * Enable colored diagnostics A well-known limitation of using ninja is that it requires the user to pass -fcolor-diagnostics=always flag to the compiler to get colored diagnostics. Starting from version 3.24, CMake can do this for us if we set CMAKE_COLOR_DIAGNOSTICS flag to a truthful value. * Add CMake option to disable GDB backtrace on fatal signal Backtraces we produce are very verbose and somewhat useless. They also are sometimes so long that they fill all of terminal scrollback and hide the actual error. * Add an off-by-default CMake option for using LLD as a linker Using LLD gives a 3-fold link-time improvement on my machine. --- CMakeLists.txt | 12 ++++++++++++ tdutils/CMakeLists.txt | 4 ++++ tdutils/td/utils/port/signals.cpp | 4 +++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5cc0a32da9..c37f351d8c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,6 +33,8 @@ set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED TRUE) set(CMAKE_CXX_EXTENSIONS FALSE) +set(CMAKE_COLOR_DIAGNOSTICS TRUE) + #BEGIN internal option(BUILD_SHARED_LIBS "Use \"ON\" to build shared libraries instead of static where it's not specified (not recommended)" OFF) option(USE_EMSCRIPTEN "Use \"ON\" for config building wasm." OFF) @@ -56,6 +58,16 @@ option(TON_USE_TSAN "Use \"ON\" to enable ThreadSanitizer." OFF) option(TON_USE_UBSAN "Use \"ON\" to enable UndefinedBehaviorSanitizer." OFF) set(TON_ARCH "native" CACHE STRING "Architecture, will be passed to -march=") +option(TON_PRINT_BACKTRACE_ON_CRASH "Attempt to print a backtrace when a fatal signal is caught" ON) + +option(TON_USE_LLD "Use LLD for linking" OFF) + +if (TON_USE_LLD) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld") + set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=lld") +endif() + #BEGIN M1 support EXECUTE_PROCESS( COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE ARCHITECTURE ) diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index b4a7fec211..e1d0c87b80 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -319,6 +319,10 @@ set(TDUTILS_TEST_SOURCE PARENT_SCOPE ) +if (NOT TON_PRINT_BACKTRACE_ON_CRASH) + set_source_files_properties(td/utils/port/signals.cpp PROPERTIES COMPILE_DEFINITIONS TON_DISABLE_BACKTRACE) +endif() + #RULES #LIBRARIES add_library(tdutils STATIC ${TDUTILS_SOURCE}) diff --git a/tdutils/td/utils/port/signals.cpp b/tdutils/td/utils/port/signals.cpp index 1d60b2fa67..12962f29e3 100644 --- a/tdutils/td/utils/port/signals.cpp +++ b/tdutils/td/utils/port/signals.cpp @@ -317,7 +317,7 @@ static void block_stdin() { #endif } -static void default_failure_signal_handler(int sig) { +[[maybe_unused]] static void default_failure_signal_handler(int sig) { Stacktrace::init(); signal_safe_write_signal_number(sig); @@ -334,9 +334,11 @@ Status set_default_failure_signal_handler() { Stdin(); // init static variables before atexit #endif std::atexit(block_stdin); +#ifndef TON_DISABLE_BACKTRACE TRY_STATUS(setup_signals_alt_stack()); TRY_STATUS(set_signal_handler(SignalType::Abort, default_failure_signal_handler)); TRY_STATUS(set_signal_handler(SignalType::Error, default_failure_signal_handler)); +#endif return Status::OK(); } From eb45c4849f4d1584b1070155e56d8bdaf01e7e46 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Fri, 20 Jun 2025 17:55:23 +0300 Subject: [PATCH 309/524] Smart cont try fetch config (#1717) * SmartContract: try fetch config from c7 if it's not set * feat: Added tvm_emulator_emulate_run_method clone with logs available * free * fix cast warning * bf --------- Co-authored-by: Marat S Co-authored-by: Yma Het --- crypto/smc-envelope/SmartContract.cpp | 42 ++++++++++++++++++++++++++ emulator/emulator-extern.cpp | 43 ++++++++++++++++++++++----- emulator/emulator-extern.h | 30 +++++++++++++++++++ emulator/emulator_export_list | 3 ++ 4 files changed, 111 insertions(+), 7 deletions(-) diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index db2ab74ab8..265c602d79 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -191,6 +191,42 @@ td::Ref prepare_vm_c7(SmartContract::Args args, td::Ref cod return vm::make_tuple_ref(std::move(tuple_ref)); } +std::shared_ptr try_fetch_config_from_c7(td::Ref c7) { + if (c7.is_null() || c7->size() < 1) { + return nullptr; + } + auto c7_tuple = c7->at(0).as_tuple(); + if (c7_tuple.is_null() || c7_tuple->size() < 10) { + return nullptr; + } + auto config_cell = c7_tuple->at(9).as_cell(); + if (config_cell.is_null()) { + return nullptr; + } + auto config_dict = std::make_unique(config_cell, 32); + auto config_addr_cell = config_dict->lookup_ref(td::BitArray<32>::zero()); + ton::StdSmcAddress config_addr; + if (config_addr_cell.is_null()) { + config_addr = ton::StdSmcAddress::zero(); + } else { + auto config_addr_cs = vm::load_cell_slice(std::move(config_addr_cell)); + if (config_addr_cs.size() != 0x100) { + LOG(WARNING) << "Config parameter 0 with config address has wrong size"; + config_addr = ton::StdSmcAddress::zero(); + } else { + config_addr_cs.fetch_bits_to(config_addr); + } + } + auto global_config = block::Config(config_cell, std::move(config_addr), + block::Config::needWorkchainInfo | block::Config::needSpecialSmc | block::Config::needCapabilities); + auto unpack_res = global_config.unpack(); + if (unpack_res.is_error()) { + LOG(ERROR) << "Failed to unpack config: " << unpack_res.error(); + return nullptr; + } + return std::make_shared(std::move(global_config)); +} + SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref stack, td::Ref c7, vm::GasLimits gas, bool ignore_chksig, td::Ref libraries, int vm_log_verbosity, bool debug_enabled, @@ -314,6 +350,9 @@ td::Ref SmartContract::get_init_state() const { } SmartContract::Answer SmartContract::run_method(Args args) { + if (args.c7 && !args.config) { + args.config = try_fetch_config_from_c7(args.c7.value()); + } if (!args.c7) { args.c7 = prepare_vm_c7(args, state_.code); } @@ -335,6 +374,9 @@ SmartContract::Answer SmartContract::run_method(Args args) { } SmartContract::Answer SmartContract::run_get_method(Args args) const { + if (args.c7 && !args.config) { + args.config = try_fetch_config_from_c7(args.c7.value()); + } if (!args.c7) { args.c7 = prepare_vm_c7(args, state_.code); } diff --git a/emulator/emulator-extern.cpp b/emulator/emulator-extern.cpp index c5f3c04e00..2addd353cb 100644 --- a/emulator/emulator-extern.cpp +++ b/emulator/emulator-extern.cpp @@ -639,10 +639,16 @@ const char *tvm_emulator_run_get_method(void *tvm_emulator, int method_id, const return strdup(jb.string_builder().as_cslice().c_str()); } -const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc, int64_t gas_limit) { +struct TvmEulatorEmulateRunMethodResponse +{ + const char *response; + const char *log; +}; + +TvmEulatorEmulateRunMethodResponse emulate_run_method(uint32_t len, const char *params_boc, int64_t gas_limit) { auto params_cell = vm::std_boc_deserialize(td::Slice(params_boc, len)); if (params_cell.is_error()) { - return nullptr; + return { nullptr, nullptr }; } auto params_cs = vm::load_cell_slice(params_cell.move_as_ok()); auto code = params_cs.fetch_ref(); @@ -657,12 +663,12 @@ const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc td::Ref stack; if (!vm::Stack::deserialize_to(stack_cs, stack)) { - return nullptr; + return { nullptr, nullptr }; } td::Ref c7; if (!vm::Stack::deserialize_to(c7_cs, c7)) { - return nullptr; + return { nullptr, nullptr }; } auto emulator = new emulator::TvmEmulator(code, data); @@ -677,7 +683,7 @@ const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc vm::CellBuilder stack_cb; if (!result.stack->serialize(stack_cb)) { - return nullptr; + return { nullptr, nullptr }; } vm::CellBuilder cb; @@ -687,7 +693,7 @@ const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc auto ser = vm::std_boc_serialize(cb.finalize()); if (!ser.is_ok()) { - return nullptr; + return { nullptr, nullptr }; } auto sok = ser.move_as_ok(); @@ -696,7 +702,24 @@ const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc memcpy(rn, &sz, 4); memcpy(rn+4, sok.data(), sz); - return rn; + return { rn, strdup(result.vm_log.data()) }; +} + +const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc, int64_t gas_limit) { + auto result = emulate_run_method(len, params_boc, gas_limit); + return result.response; +} + +void *tvm_emulator_emulate_run_method_detailed(uint32_t len, const char *params_boc, int64_t gas_limit) { + auto result = emulate_run_method(len, params_boc, gas_limit); + return new TvmEulatorEmulateRunMethodResponse(result); +} + +void run_method_detailed_result_destroy(void *detailed_result) { + auto result = static_cast(detailed_result); + free(const_cast(result->response)); + free(const_cast(result->log)); + delete result; } const char *tvm_emulator_send_external_message(void *tvm_emulator, const char *message_body_boc) { @@ -773,6 +796,12 @@ void emulator_config_destroy(void *config) { delete static_cast(config); } +void string_destroy(const char *str) { + if (str != nullptr) { + free(const_cast(str)); + } +} + const char* emulator_version() { auto version_json = td::JsonBuilder(); auto obj = version_json.enter_object(); diff --git a/emulator/emulator-extern.h b/emulator/emulator-extern.h index 14879e1ec9..2da7674565 100644 --- a/emulator/emulator-extern.h +++ b/emulator/emulator-extern.h @@ -254,6 +254,27 @@ EMULATOR_EXPORT const char *tvm_emulator_run_get_method(void *tvm_emulator, int */ EMULATOR_EXPORT const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc, int64_t gas_limit); +/** + * @brief Optimized version of "run get method" with all passed parameters in a single call. Also returns log. + * @param len Length of params_boc buffer + * @param params_boc BoC serialized parameters, scheme: request$_ code:^Cell data:^Cell stack:^VmStack params:^[c7:^VmStack libs:^Cell] method_id:(## 32) + * @param gas_limit Gas limit + * @return Pointer to struct with two fields: + * - response: Char* with first 4 bytes defining length, and the rest BoC serialized result + * Scheme: result$_ exit_code:(## 32) gas_used:(## 32) stack:^VmStack + * - log: Char* with VM log string + */ +EMULATOR_EXPORT void *tvm_emulator_emulate_run_method_detailed(uint32_t len, const char *params_boc, int64_t gas_limit); + +/** + * @brief Destroy detailed result of "tvm_emulator_emulate_run_method_detailed" + * @param detailed_result Pointer to detailed result struct returned by "tvm_emulator_emulate_run_method_detailed" + * + * Caller should not use string_destroy() for fields of this struct, + * as they are already freed in this function. + */ +EMULATOR_EXPORT void run_method_detailed_result_destroy(void *detailed_result); + /** * @brief Send external message * @param tvm_emulator Pointer to TVM emulator @@ -315,6 +336,15 @@ EMULATOR_EXPORT void tvm_emulator_destroy(void *tvm_emulator); */ EMULATOR_EXPORT void emulator_config_destroy(void *config); +/** + * @brief Destroy string created by emulator library + * @param string Pointer to string to destroy + * + * This function should be used to free strings returned by emulator library functions. + * It is not safe to use caller's free() on them, as they may have been allocated using a different allocator. + */ +EMULATOR_EXPORT void string_destroy(const char *string); + /** * @brief Get git commit hash and date of the library */ diff --git a/emulator/emulator_export_list b/emulator/emulator_export_list index bd991cd73f..2cec6fed9f 100644 --- a/emulator/emulator_export_list +++ b/emulator/emulator_export_list @@ -27,4 +27,7 @@ _tvm_emulator_send_external_message _tvm_emulator_send_internal_message _tvm_emulator_destroy _tvm_emulator_emulate_run_method +_tvm_emulator_emulate_run_method_detailed +_run_method_detailed_result_destroy +_string_destroy _emulator_version From a174fb1044355f31ea0f3c5dcc3ae416cb97cf31 Mon Sep 17 00:00:00 2001 From: Kirill <43584399+Sumrachek@users.noreply.github.com> Date: Fri, 20 Jun 2025 21:42:45 +0400 Subject: [PATCH 310/524] Replaced hashmap by b-tree map in large boc serializer (#1686) --- crypto/test/test-db.cpp | 53 ++++++++++++++++++++++-------- crypto/vm/large-boc-serializer.cpp | 4 +-- tdutils/td/utils/HashMap.h | 6 ++++ 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/crypto/test/test-db.cpp b/crypto/test/test-db.cpp index f4dde8df71..c70738c7b3 100644 --- a/crypto/test/test-db.cpp +++ b/crypto/test/test-db.cpp @@ -1691,6 +1691,31 @@ TEST(TonDb, BocDeserializerSimpleThreads) { test_boc_deserializer_threads(); } +class RandomTree { + public: + RandomTree(size_t size, td::Random::Xorshift128plus rnd) : rnd_(rnd) { + root_ = create(size); + } + Ref root() const { + return root_; + } + private: + Ref root_; + td::Random::Xorshift128plus rnd_; + Ref create(size_t size) { + CellBuilder cb; + cb.store_long(rnd_(), rnd_() % 63 + 1); + if (size > 0) { + td::uint64 rc = (rnd_() % 4) + 1; + for (td::uint64 i = 0; i < rc; i++) { + auto ref = create(size / rc); + cb.store_ref(std::move(ref)); + } + } + return cb.finalize(); + } +}; + class CompactArray { public: CompactArray(size_t size) { @@ -2987,11 +3012,8 @@ TEST(TonDb, DynamicBocRespectsUsageCell) { TEST(TonDb, LargeBocSerializer) { td::Random::Xorshift128plus rnd{123}; - size_t n = 1000000; - std::vector data(n); - std::iota(data.begin(), data.end(), 0); - vm::CompactArray arr(data); - auto root = arr.root(); + vm::RandomTree tree(100000, rnd); + auto root = tree.root(); std::string path = "serialization"; td::unlink(path).ignore(); auto fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write) @@ -3009,15 +3031,20 @@ TEST(TonDb, LargeBocSerializer) { dboc->commit(cell_storer); dboc->set_loader(std::make_unique(kv)); td::unlink(path).ignore(); - fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write) - .move_as_ok(); - boc_serialize_to_file_large(dboc->get_cell_db_reader(), root->get_hash(), fd, 31); - fd.close(); - auto b = td::read_file_str(path).move_as_ok(); - + td::string prev_b(""); auto a_cell = vm::deserialize_boc(td::BufferSlice(a)); - auto b_cell = vm::deserialize_boc(td::BufferSlice(b)); - ASSERT_EQ(a_cell->get_hash(), b_cell->get_hash()); + for (int i = 0; i < 4; i++) { + fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write) + .move_as_ok(); + boc_serialize_to_file_large(dboc->get_cell_db_reader(), root->get_hash(), fd, 31); + fd.close(); + auto b = td::read_file_str(path).move_as_ok(); + + auto b_cell = vm::deserialize_boc(td::BufferSlice(b)); + ASSERT_EQ(a_cell->get_hash(), b_cell->get_hash()); + if (i > 0) ASSERT_EQ(prev_b, b); + prev_b = b; + } } TEST(TonDb, DoNotMakeListsPrunned) { diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp index 839e6235ad..f6e35c6fe8 100644 --- a/crypto/vm/large-boc-serializer.cpp +++ b/crypto/vm/large-boc-serializer.cpp @@ -115,7 +115,7 @@ td::Status LargeBocSerializer::import_cells() { td::Result LargeBocSerializer::import_cell(Hash root_hash, int root_depth) { const int start_ind = cell_count; - td::HashMap> current_depth_hashes; + td::BTreeMap> current_depth_hashes; auto existing_it = cells.find(root_hash); if (existing_it != cells.end()) { @@ -131,7 +131,7 @@ td::Result LargeBocSerializer::import_cell(Hash root_hash, int root_depth) } cell_list.resize(cell_list.size() + current_depth_hashes.size()); - td::HashMap> next_depth_hashes; + td::BTreeMap> next_depth_hashes; auto batch_start = current_depth_hashes.begin(); while (batch_start != current_depth_hashes.end()) { std::vector batch_hashes; diff --git a/tdutils/td/utils/HashMap.h b/tdutils/td/utils/HashMap.h index 1380f0291c..d40006785d 100644 --- a/tdutils/td/utils/HashMap.h +++ b/tdutils/td/utils/HashMap.h @@ -23,8 +23,10 @@ #if TD_HAVE_ABSL #include #include +#include #else #include +#include #endif namespace td { @@ -34,11 +36,15 @@ template > using HashMap = absl::flat_hash_map; template , class E = std::equal_to<>> using NodeHashMap = absl::node_hash_map; +template +using BTreeMap = absl::btree_map; #else template > using HashMap = std::unordered_map; template > using NodeHashMap = std::unordered_map; +template +using BTreeMap = std::map; #endif } // namespace td From 6c9de4626c4fdf73ff2d85e7ccd60c2c059e0a8a Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 20 Jun 2025 20:44:29 +0300 Subject: [PATCH 311/524] Optimized celldb and package import for archive nodes (#1653) * celldb: version 2 - thread safe cache - parallel commit - multiple optimizations - support of key-value merge operations - improved tests and benchmarks - in-memory version won't read from key value after start - uses vector in-memory table now - use rocksdb::WALRecoveryMode::kTolerateCorruptedTailRecords - do not silently ignore errors during recovery * celldb: add test for load nonexisting cell, test thread safeness of CellUsageTree, fixes * Cache validator sets for create_shard_state * Permanent celldb, improve importing archive slices, new flags for validator-engine * Fix persistent state lookup * Tool for generating liteserver desc for global config * Import proof for initial block * Perf timer for archive import * Do not recompute validator set in MasterchainStateQ::mc_reinit This does not improve performance (as unpack_validator_set result is cached regardless) but makes the code clearer. * Remove debug logs * Fix downloading initial proof * Fix error processing in CellDbIn::store_block_state_permanent_bulk * Improve some logs --------- Co-authored-by: birydrad <> Co-authored-by: Dan Klishch --- crypto/block/mc-config.h | 4 +- crypto/vm/db/CellStorage.cpp | 33 +- crypto/vm/db/CellStorage.h | 3 +- utils/CMakeLists.txt | 3 + utils/prepare-ls-slice-config.cpp | 266 ++++++++ validator-engine/validator-engine.cpp | 30 +- validator-engine/validator-engine.hpp | 12 + validator/CMakeLists.txt | 4 + validator/db/celldb.cpp | 209 +++++- validator/db/celldb.hpp | 8 + .../permanent-celldb-utils.cpp | 83 +++ .../permanent-celldb/permanent-celldb-utils.h | 37 + validator/db/rootdb.cpp | 35 + validator/db/rootdb.hpp | 4 + validator/downloaders/download-state.cpp | 16 +- validator/downloaders/download-state.hpp | 1 + validator/downloaders/wait-block-state.cpp | 28 +- validator/downloaders/wait-block-state.hpp | 11 +- validator/full-node-shard.cpp | 6 +- validator/impl/shard.cpp | 7 +- validator/import-db-slice-local.cpp | 640 ++++++++++++++++++ validator/import-db-slice-local.hpp | 103 +++ validator/import-db-slice.cpp | 8 +- validator/import-db-slice.hpp | 2 + validator/interfaces/db.h | 4 + validator/interfaces/validator-manager.h | 9 + validator/manager-disk.cpp | 14 +- validator/manager-disk.hpp | 4 + validator/manager-hardfork.cpp | 4 +- validator/manager-hardfork.hpp | 7 + validator/manager-init.cpp | 57 +- validator/manager-init.hpp | 2 +- validator/manager.cpp | 124 +++- validator/manager.hpp | 7 + validator/net/download-state.cpp | 24 +- validator/validator-options.cpp | 15 +- validator/validator-options.hpp | 18 +- validator/validator.h | 6 +- 38 files changed, 1739 insertions(+), 109 deletions(-) create mode 100644 utils/prepare-ls-slice-config.cpp create mode 100644 validator/db/permanent-celldb/permanent-celldb-utils.cpp create mode 100644 validator/db/permanent-celldb/permanent-celldb-utils.h create mode 100644 validator/import-db-slice-local.cpp create mode 100644 validator/import-db-slice-local.hpp diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 34fab4df40..d32a9945d7 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -650,8 +650,8 @@ class Config { const WorkchainSet& get_workchain_list() const { return workchains_; } - const ValidatorSet* get_cur_validator_set() const { - return cur_validators_.get(); + std::shared_ptr const& get_cur_validator_set() const& { + return cur_validators_; } std::pair get_validator_set_start_stop(int next = 0) const; ton::ValidatorSessionConfig get_consensus_config() const; diff --git a/crypto/vm/db/CellStorage.cpp b/crypto/vm/db/CellStorage.cpp index f93e3fa59c..33c18ab3f9 100644 --- a/crypto/vm/db/CellStorage.cpp +++ b/crypto/vm/db/CellStorage.cpp @@ -32,8 +32,9 @@ namespace { class RefcntCellStorer { public: - RefcntCellStorer(td::int32 refcnt, const td::Ref &cell, bool as_boc) - : refcnt_(refcnt), cell_(cell), as_boc_(as_boc) { + RefcntCellStorer(td::int32 refcnt, const td::Ref &cell, bool as_boc, int max_level = vm::Cell::max_level) + : refcnt_(refcnt), cell_(cell), as_boc_(as_boc), max_level_(max_level) { + CHECK(!as_boc_ || max_level_ == vm::Cell::max_level); } template @@ -51,10 +52,25 @@ class RefcntCellStorer { CHECK(refcnt_ > 0); store(refcnt_, storer); CHECK(cell_.not_null()) - store(*cell_, storer); + if (max_level_ == vm::Cell::max_level) { + store(*cell_, storer); + } else { + auto level_mask = cell_->get_level_mask().apply(max_level_); + storer.template store_binary( + static_cast(cell_->get_refs_cnt() + 8 * cell_->is_special() + 32 * level_mask.get_mask())); + auto d2 = static_cast((cell_->get_bits() / 8) * 2); + if ((cell_->get_bits() & 7) != 0) { + d2 = static_cast(d2 + 1); + } + storer.template store_binary(d2); + storer.store_slice(td::Slice(cell_->get_data(), (cell_->get_bits() + 7) / 8)); + } + for (unsigned i = 0; i < cell_->size_refs(); i++) { auto cell = cell_->get_ref(i); - auto level_mask = cell->get_level_mask(); + auto level_mask = + cell->get_level_mask().apply(max_level_ + (cell_->special_type() == CellTraits::SpecialType::MerkleProof || + cell_->special_type() == CellTraits::SpecialType::MerkleUpdate)); auto level = level_mask.get_level(); td::uint8 x = static_cast(level_mask.get_mask()); storer.store_slice(td::Slice(&x, 1)); @@ -79,6 +95,7 @@ class RefcntCellStorer { td::int32 refcnt_; td::Ref cell_; bool as_boc_; + int max_level_; }; class RefcntCellParser { @@ -120,13 +137,13 @@ class RefcntCellParser { Ref refs[Cell::max_refs]; for (int i = 0; i < info.refs_cnt; i++) { if (data.size() < 1) { - return td::Status::Error("Not enought data"); + return td::Status::Error("Not enough data"); } Cell::LevelMask level_mask(data[0]); auto n = level_mask.get_hashes_count(); auto end_offset = 1 + n * (Cell::hash_bytes + Cell::depth_bytes); if (data.size() < end_offset) { - return td::Status::Error("Not enought data"); + return td::Status::Error("Not enough data"); } TRY_RESULT(ext_cell, ext_cell_creator.ext_cell(level_mask, data.substr(1, n * Cell::hash_bytes), @@ -243,8 +260,8 @@ td::Status CellStorer::erase(td::Slice hash) { return kv_.erase(hash); } -std::string CellStorer::serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc) { - return td::serialize(RefcntCellStorer(refcnt, cell, as_boc)); +std::string CellStorer::serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc, int max_level) { + return td::serialize(RefcntCellStorer(refcnt, cell, as_boc, max_level)); } td::Status CellStorer::set(td::int32 refcnt, const td::Ref &cell, bool as_boc) { diff --git a/crypto/vm/db/CellStorage.h b/crypto/vm/db/CellStorage.h index 7ae5867933..e17fab5f22 100644 --- a/crypto/vm/db/CellStorage.h +++ b/crypto/vm/db/CellStorage.h @@ -72,7 +72,8 @@ class CellStorer { static void merge_refcnt_diffs(std::string &left, td::Slice right); static std::string serialize_refcnt_diffs(td::int32 refcnt_diff); - static std::string serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc); + static std::string serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc, + int max_level = vm::Cell::max_level); struct Diff { enum Type { Set, Erase, Merge } type{Set}; diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 99a4e92df5..0afdf7389f 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -22,4 +22,7 @@ target_include_directories(pack-viewer PUBLIC $. +*/ +#include "td/utils/filesystem.h" +#include "td/actor/actor.h" +#include "td/actor/MultiPromise.h" +#include "td/utils/OptionParser.h" +#include "td/utils/port/path.h" +#include "td/utils/port/signals.h" +#include "td/utils/port/IPAddress.h" +#include "td/utils/Random.h" +#include "td/utils/FileLog.h" +#include "git.h" +#include "auto/tl/ton_api.h" +#include "auto/tl/lite_api.h" +#include "tl-utils/lite-utils.hpp" +#include "auto/tl/ton_api_json.h" +#include "adnl/adnl.h" +#include "lite-client/ext-client.h" +#include "ton/lite-tl.hpp" + +#include "td/utils/overloaded.h" + +#include +#include "td/utils/tl_storers.h" +#include "vm/boc.h" +#include "vm/cells/MerkleProof.h" + +#include +#include "block/block-auto.h" +#include "block/mc-config.h" + +using namespace ton; + +std::string global_config_file; +td::optional start_mc_seqno, end_mc_seqno; +std::vector shards; + +class PrepareLsSliceConfig : public td::actor::Actor { + public: + void start_up() override { + if (start_mc_seqno && end_mc_seqno && start_mc_seqno.value() > end_mc_seqno.value()) { + LOG(FATAL) << "from-seqno is greater than to-seqno"; + } + + if (!start_mc_seqno && !end_mc_seqno) { + auto slice = create_tl_object(); + if (shards.empty()) { + slice->shards_.push_back(create_tl_shard_id(ShardIdFull{basechainId, shardIdAll})); + } else { + for (const ShardIdFull& shard : shards) { + if (!shard.is_masterchain()) { + slice->shards_.push_back(create_tl_shard_id(shard)); + } + } + } + print_result(*slice); + return; + } + + auto gc_s = td::read_file(global_config_file).move_as_ok(); + auto gc_j = td::json_decode(gc_s.as_slice()).move_as_ok(); + ton_api::liteclient_config_global gc; + ton_api::from_json(gc, gc_j.get_object()).ensure(); + auto r_servers = liteclient::LiteServerConfig::parse_global_config(gc); + r_servers.ensure(); + client_ = liteclient::ExtClient::create(r_servers.move_as_ok(), nullptr); + + slice_timed_ = create_tl_object(); + ++pending_; + request_shards_info(start_mc_seqno, true); + request_shards_info(end_mc_seqno, false); + dec_pending(); + } + + template + static td::BufferSlice create_query(Args&&... args) { + Type object(std::forward(args)...); + return create_serialize_tl_object(serialize_tl_object(&object, true)); + } + + template + static tl_object_ptr parse_response(const td::Result& R) { + R.ensure(); + auto err = fetch_tl_object(R.ok(), true); + if (err.is_ok()) { + LOG(FATAL) << "liteserver error: " << err.ok()->message_; + } + auto res = fetch_tl_object(R.ok(), true); + res.ensure(); + return res.move_as_ok(); + } + + void request_shards_info(td::optional seqno, bool is_start) { + if (!seqno) { + return; + } + ++pending_; + td::actor::send_closure( + client_, &liteclient::ExtClient::send_query, "q", + create_query( + 1, create_tl_object(masterchainId, shardIdAll, seqno.value()), 0, 0), + td::Timestamp::in(5.0), [=, client = client_.get(), SelfId = actor_id(this)](td::Result R) { + auto mc_header = parse_response(std::move(R)); + auto block_id = create_block_id(mc_header->id_); + td::actor::send_closure( + client, &liteclient::ExtClient::send_query, "q", + create_query(create_tl_lite_block_id(block_id)), + td::Timestamp::in(5.0), [=, mc_header = std::move(mc_header)](td::Result R) mutable { + auto shards_info = parse_response(std::move(R)); + td::actor::send_closure(SelfId, &PrepareLsSliceConfig::got_shards_info, std::move(mc_header), + std::move(shards_info), is_start); + }); + }); + } + + static tl_object_ptr parse_header(const lite_api::liteServer_blockHeader& obj, + bool is_start) { + auto res = create_tl_object(); + + BlockIdExt block_id = create_block_id(obj.id_); + res->shard_id_ = create_tl_shard_id(block_id.shard_full()); + res->seqno_ = block_id.seqno(); + + auto root = vm::std_boc_deserialize(obj.header_proof_).move_as_ok(); + root = vm::MerkleProof::virtualize(root, 1); + block::gen::Block::Record blk; + block::gen::BlockInfo::Record info; + CHECK(tlb::unpack_cell(root, blk) && tlb::unpack_cell(blk.info, info)); + res->utime_ = info.gen_utime; + res->lt_ = (is_start ? info.start_lt : info.end_lt); + + return res; + } + + void got_shards_info(tl_object_ptr mc_header, + tl_object_ptr shards_info, bool is_start) { + (is_start ? slice_timed_->shards_from_ : slice_timed_->shards_to_).push_back(parse_header(*mc_header, is_start)); + + auto root = vm::std_boc_deserialize(shards_info->data_).move_as_ok(); + block::ShardConfig sh_conf; + CHECK(sh_conf.unpack(vm::load_cell_slice_ref(root))); + auto ids = sh_conf.get_shard_hash_ids(true); + for (auto id : ids) { + BlockIdExt block_id = sh_conf.get_shard_hash(ton::ShardIdFull(id))->top_block_id(); + bool ok = shards.empty(); + for (const auto& our_shard : shards) { + if (shard_intersects(our_shard, block_id.shard_full())) { + ok = true; + break; + } + } + if (ok) { + ++pending_; + td::actor::send_closure( + client_, &liteclient::ExtClient::send_query, "q", + create_query(create_tl_lite_block_id(block_id), 0xffff), + td::Timestamp::in(5.0), [=, SelfId = actor_id(this)](td::Result R) mutable { + auto header = parse_response(std::move(R)); + td::actor::send_closure(SelfId, &PrepareLsSliceConfig::got_block_header, std::move(header), is_start); + }); + } + } + + dec_pending(); + } + + void got_block_header(tl_object_ptr header, bool is_start) { + (is_start ? slice_timed_->shards_from_ : slice_timed_->shards_to_).push_back(parse_header(*header, is_start)); + dec_pending(); + } + + void print_result(const ton_api::liteserver_descV2_Slice& result) { + auto s = td::json_encode(td::ToJson(result), true); + std::cout << s << "\n"; + std::cout.flush(); + exit(0); + } + + private: + td::actor::ActorOwn client_; + tl_object_ptr slice_timed_; + size_t pending_ = 0; + + void dec_pending() { + --pending_; + if (pending_ == 0) { + auto cmp = [](const tl_object_ptr& a, + const tl_object_ptr& b) { + return create_shard_id(a->shard_id_) < create_shard_id(b->shard_id_); + }; + std::sort(slice_timed_->shards_from_.begin(), slice_timed_->shards_from_.end(), cmp); + std::sort(slice_timed_->shards_to_.begin(), slice_timed_->shards_to_.end(), cmp); + print_result(*slice_timed_); + } + } +}; + +int main(int argc, char* argv[]) { + SET_VERBOSITY_LEVEL(verbosity_INFO); + + td::unique_ptr logger_; + SCOPE_EXIT { + td::log_interface = td::default_log_interface; + }; + + td::OptionParser p; + p.set_description( + "Generate liteserver.descV2.Slice for global-config.json from given shards and masterchain seqnos\n"); + p.add_option('v', "verbosity", "set verbosity level", [&](td::Slice arg) { + int v = VERBOSITY_NAME(FATAL) + (td::to_integer(arg)); + SET_VERBOSITY_LEVEL(v); + }); + p.add_option('V', "version", "show build information", [&]() { + std::cout << "prepare-ls-slice-config build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::exit(0); + }); + p.add_option('h', "help", "print help", [&]() { + char b[10240]; + td::StringBuilder sb(td::MutableSlice{b, 10000}); + sb << p; + std::cout << sb.as_cslice().c_str(); + std::exit(2); + }); + p.add_option('C', "global-config", "global TON configuration file (used to fetch shard configuration)", + [&](td::Slice arg) { global_config_file = arg.str(); }); + p.add_checked_option('f', "from-seqno", "starting masterchain seqno (default: none)", + [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(start_mc_seqno, td::to_integer_safe(arg)) + return td::Status::OK(); + }); + p.add_checked_option('t', "to-seqno", "ending masterchain seqno (default: none)", [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(end_mc_seqno, td::to_integer_safe(arg)) + return td::Status::OK(); + }); + p.add_checked_option('s', "shard", "shard in format 0:8000000000000000 (default: all shards)", + [&](td::Slice arg) -> td::Status { + TRY_RESULT(shard, ShardIdFull::parse(arg)); + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << arg); + } + shards.push_back(shard); + return td::Status::OK(); + }); + + p.run(argc, argv).ensure(); + td::actor::Scheduler scheduler({3}); + + scheduler.run_in_context([&] { td::actor::create_actor("main").release(); }); + while (scheduler.run(1)) { + } +} diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 5829027829..b8a4f715a7 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1489,6 +1489,8 @@ td::Status ValidatorEngine::load_global_config() { if (catchain_max_block_delay_slow_) { validator_options_.write().set_catchain_max_block_delay_slow(catchain_max_block_delay_slow_.value()); } + validator_options_.write().set_permanent_celldb(permanent_celldb_); + validator_options_.write().set_initial_sync_disabled(skip_key_sync_); std::vector h; for (auto &x : conf.validator_->hardforks_) { @@ -1516,16 +1518,23 @@ td::Status ValidatorEngine::load_global_config() { void ValidatorEngine::set_shard_check_function() { if (!not_all_shards_) { - validator_options_.write().set_shard_check_function([](ton::ShardIdFull shard) -> bool { return true; }); + validator_options_.write().set_shard_check_function( + [sync_shards_upto = sync_shards_upto_](ton::ShardIdFull shard, ton::BlockSeqno mc_seqno) -> bool { + return shard.is_masterchain() || !sync_shards_upto || mc_seqno <= sync_shards_upto.value(); + }); } else { std::vector shards = {ton::ShardIdFull(ton::masterchainId)}; - for (const auto& s : config_.shards_to_monitor) { + for (const auto &s : config_.shards_to_monitor) { shards.push_back(s); } std::sort(shards.begin(), shards.end()); shards.erase(std::unique(shards.begin(), shards.end()), shards.end()); validator_options_.write().set_shard_check_function( - [shards = std::move(shards)](ton::ShardIdFull shard) -> bool { + [shards = std::move(shards), sync_shards_upto = sync_shards_upto_](ton::ShardIdFull shard, + ton::BlockSeqno mc_seqno) -> bool { + if (!shard.is_masterchain() && sync_shards_upto && mc_seqno > sync_shards_upto.value()) { + return false; + } for (auto s : shards) { if (shard_intersects(shard, s)) { return true; @@ -4622,6 +4631,21 @@ int main(int argc, char *argv[]) { [&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_broadcast_speed_multiplier_private, v); }); return td::Status::OK(); }); + p.add_option( + '\0', "permanent-celldb", + "disable garbage collection in CellDb. This improves performance on archival nodes (once enabled, this option " + "cannot be disabled)", + [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_permanent_celldb, true); }); }); + p.add_option('\0', "skip-key-sync", + "don't select the best persistent state on initial sync, start on init_block from global config", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_skip_key_sync, true); }); + }); + p.add_checked_option( + '\0', "sync-shards-upto", "stop syncing shards on this masterchain seqno", [&](td::Slice s) -> td::Status { + TRY_RESULT(v, td::to_integer_safe(s)); + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_sync_shards_upto, v); }); + return td::Status::OK(); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index de6c3ce193..61bbb72ff4 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -234,6 +234,9 @@ class ValidatorEngine : public td::actor::Actor { double broadcast_speed_multiplier_catchain_ = 3.33; double broadcast_speed_multiplier_public_ = 3.33; double broadcast_speed_multiplier_private_ = 3.33; + bool permanent_celldb_ = false; + bool skip_key_sync_ = false; + td::optional sync_shards_upto_; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -349,6 +352,15 @@ class ValidatorEngine : public td::actor::Actor { void set_broadcast_speed_multiplier_private(double value) { broadcast_speed_multiplier_private_ = value; } + void set_permanent_celldb(bool value) { + permanent_celldb_ = value; + } + void set_skip_key_sync(bool value) { + skip_key_sync_ = value; + } + void set_sync_shards_upto(ton::BlockSeqno seqno) { + sync_shards_upto_ = seqno; + } void start_up() override; ValidatorEngine() { diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index f7a4ce1d11..9fb0d190fa 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -14,6 +14,8 @@ set(VALIDATOR_DB_SOURCE db/archive-slice.hpp db/celldb.cpp db/celldb.hpp + db/permanent-celldb/permanent-celldb-utils.cpp + db/permanent-celldb/permanent-celldb-utils.h db/files-async.hpp db/fileref.hpp db/fileref.cpp @@ -54,6 +56,7 @@ set(VALIDATOR_HEADERS invariants.hpp import-db-slice.hpp + import-db-slice-local.hpp queue-size-counter.hpp validator-telemetry.hpp @@ -74,6 +77,7 @@ set(VALIDATOR_SOURCE block-handle.cpp get-next-key-blocks.cpp import-db-slice.cpp + import-db-slice-local.cpp shard-client.cpp state-serializer.cpp token-manager.cpp diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index fc8735d318..f4a3a084bf 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -27,6 +27,12 @@ #include "ton/ton-tl.hpp" #include "ton/ton-io.hpp" #include "common/delay.h" +#include "block/block-auto.h" +#include "permanent-celldb/permanent-celldb-utils.h" +#include "td/actor/MultiPromise.h" + +#include +#include #include #include @@ -331,6 +337,30 @@ void CellDbIn::start_up() { last_deleted_mc_state_ = r_value.move_as_ok(); } } + { + std::string key = "opts.permanent_mode", value; + auto R = boc_->meta_get(td::as_slice(key), value); + R.ensure(); + bool stored_permanent_mode = R.ok() == td::KeyValue::GetStatus::Ok; + permanent_mode_ = stored_permanent_mode || opts_->get_permanent_celldb(); + if (permanent_mode_) { + LOG(WARNING) << "Celldb is in permanent mode"; + if (!stored_permanent_mode) { + cell_db_->begin_write_batch().ensure(); + value = "1"; + vm::CellStorer stor{*cell_db_}; + boc_->meta_set(td::as_slice(key), td::as_slice(value)); + boc_->commit(stor).ensure(); + cell_db_->commit_write_batch().ensure(); + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + } + } + } + cell_db_statistics_.permanent_mode_ = permanent_mode_; + LOG_IF(FATAL, permanent_mode_ && opts_->get_celldb_in_memory()) + << "celldb permanent_mode and in_memory_mode are not compatible"; + } } void CellDbIn::load_cell(RootHash hash, td::Promise> promise) { @@ -366,7 +396,9 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi auto R = get_block(key_hash); // duplicate if (R.is_ok()) { - promise.set_result(boc_->load_cell(cell->get_hash().as_slice())); + delay_action([cell = boc_->load_cell(cell->get_hash().as_slice()), + promise = std::move(promise)]() mutable { promise.set_result(std::move(cell)); }, + td::Timestamp::now()); return; } @@ -377,7 +409,7 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi cell = std::move(cell)](td::Result Res) mutable { Res.ensure(); timer_prepare.pause(); - td::actor::send_lambda( + td::actor::send_lambda_later( SelfId, [=, this, timer = std::move(timer), promise = std::move(promise), cell = std::move(cell)]() mutable { TD_PERF_COUNTER(celldb_store_cell); auto empty = get_empty_key_hash(); @@ -414,7 +446,9 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); } - promise.set_result(boc_->load_cell(cell->get_hash().as_slice())); + delay_action([cell = boc_->load_cell(cell->get_hash().as_slice()), + promise = std::move(promise)]() mutable { promise.set_result(std::move(cell)); }, + td::Timestamp::now()); if (!opts_->get_disable_rocksdb_stats()) { cell_db_statistics_.store_cell_time_.insert(timer.elapsed() * 1e6); cell_db_statistics_.store_cell_prepare_time_.insert(timer_prepare.elapsed() * 1e6); @@ -437,6 +471,151 @@ void CellDbIn::get_cell_db_reader(td::Promise> promise.set_result(boc_->get_cell_db_reader()); } +void CellDbIn::store_block_state_permanent(td::Ref block, td::Promise> promise) { + if (!permanent_mode_) { + promise.set_error(td::Status::Error("celldb is not in permanent mode")); + return; + } + if (db_busy_) { + action_queue_.push( + [self = this, block = std::move(block), promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + self->store_block_state_permanent(std::move(block), std::move(promise)); + }); + return; + } + auto key_hash = get_key_hash(block->block_id()); + auto R = get_block(key_hash); + // duplicate + if (R.is_ok()) { + delay_action([cell = boc_->load_cell(R.ok().root_hash.as_slice()), + promise = std::move(promise)]() mutable { promise.set_result(std::move(cell)); }, + td::Timestamp::now()); + return; + } + store_block_state_permanent_bulk( + {block}, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + block::gen::Block::Record rec; + if (!block::gen::unpack_cell(block->root_cell(), rec)) { + promise.set_error(td::Status::Error("cannot unpack Block record")); + return; + } + bool spec; + vm::CellSlice update_cs = vm::load_cell_slice_special(rec.state_update, spec); + if (update_cs.special_type() != vm::CellTraits::SpecialType::MerkleUpdate) { + promise.set_error(td::Status::Error("invalid Merkle update in block")); + return; + } + td::Ref new_state_root = update_cs.prefetch_ref(1); + RootHash state_root_hash = new_state_root->get_hash(0).bits(); + td::actor::send_closure(SelfId, &CellDbIn::load_cell, state_root_hash, std::move(promise)); + }); +} + +void CellDbIn::store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise) { + if (!permanent_mode_) { + promise.set_error(td::Status::Error("celldb is not in permanent mode")); + return; + } + if (db_busy_) { + action_queue_.push( + [self = this, blocks = std::move(blocks), promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + self->store_block_state_permanent_bulk(std::move(blocks), std::move(promise)); + }); + return; + } + td::PerfWarningTimer timer{"storecellbulk", 0.1}; + td::Timer timer_prepare; + std::map> new_blocks; + for (auto& block : blocks) { + BlockIdExt block_id = block->block_id(); + if (new_blocks.contains(block_id)) { + continue; + } + if (get_block(get_key_hash(block_id)).is_ok()) { + continue; + } + new_blocks[block_id] = std::move(block); + } + if (new_blocks.empty()) { + promise.set_value(td::Unit{}); + return; + } + for (auto& [block_id, block] : new_blocks) { + std::vector prev; + BlockIdExt mc_blkid; + bool after_split; + TRY_STATUS_PROMISE(promise, + block::unpack_block_prev_blk_try(block->root_cell(), block_id, prev, mc_blkid, after_split)); + for (const BlockIdExt& prev_id : prev) { + if (!new_blocks.contains(prev_id) && get_block(get_key_hash(prev_id)).is_error()) { + promise.set_error(td::Status::Error("cannot store block state: previous block is not in db")); + return; + } + } + } + db_busy_ = true; + calculate_permanent_celldb_update( + new_blocks, async_executor, + [this, SelfId = actor_id(this), timer = std::move(timer), timer_prepare = std::move(timer_prepare), + promise = std::move(promise)](td::Result> R) mutable { + td::actor::send_lambda_later( + SelfId, [=, this, timer = std::move(timer), timer_prepare = std::move(timer_prepare), R = std::move(R), + promise = std::move(promise)]() mutable { + SCOPE_EXIT { + release_db(); + }; + TRY_RESULT_PROMISE(promise, updates, std::move(R)); + TD_PERF_COUNTER(celldb_store_cell_multi); + timer_prepare.pause(); + td::Timer timer_write; + vm::CellStorer stor{*cell_db_}; + cell_db_->begin_write_batch().ensure(); + + for (auto& update : updates) { + for (auto& [k, v] : update.to_store) { + cell_db_->set(k.as_slice(), v).ensure(); + } + } + + CHECK(!updates.empty()); + auto empty = get_empty_key_hash(); + auto E = get_block(empty).move_as_ok(); + for (size_t i = 0; i < updates.size(); ++i) { + KeyHash prev = i == 0 ? empty : get_key_hash(updates[i - 1].block_id); + KeyHash next = i + 1 == updates.size() ? E.next : get_key_hash(updates[i + 1].block_id); + DbEntry entry{updates[i].block_id, prev, next, updates[i].state_root_hash}; + set_block(get_key_hash(updates[i].block_id), std::move(entry)); + } + E.next = get_key_hash(updates[0].block_id); + if (E.prev == empty) { + E.prev = get_key_hash(updates.back().block_id); + } + set_block(empty, std::move(E)); + + boc_->commit(stor).ensure(); // Save meta + cell_db_->commit_write_batch().ensure(); + timer_write.pause(); + + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + } + + if (!opts_->get_disable_rocksdb_stats()) { + cell_db_statistics_.store_cell_time_.insert(timer.elapsed() * 1e6); + cell_db_statistics_.store_cell_prepare_time_.insert(timer_prepare.elapsed() * 1e6); + cell_db_statistics_.store_cell_write_time_.insert(timer_write.elapsed() * 1e6); + cell_db_statistics_.store_cell_bulk_queries_++; + cell_db_statistics_.store_cell_bulk_total_blocks_ += updates.size(); + } + promise.set_result(td::Unit()); + }); + }); +} + std::vector> CellDbIn::prepare_stats() { TD_PERF_COUNTER(celldb_prepare_stats); auto r_boc_stats = boc_->get_stats(); @@ -504,6 +683,7 @@ void CellDbIn::flush_db_stats() { } td::RocksDb::reset_statistics(statistics_); cell_db_statistics_.clear(); + cell_db_statistics_.permanent_mode_ = permanent_mode_; } void CellDbIn::alarm() { @@ -522,6 +702,10 @@ void CellDbIn::alarm() { << " queue_size=" << cells_to_migrate_.size(); migration_stats_ = {}; } + if (permanent_mode_) { + skip_gc(); + return; + } auto E = get_block(get_empty_key_hash()).move_as_ok(); auto N = get_block(E.next).move_as_ok(); if (N.is_empty()) { @@ -575,6 +759,7 @@ void CellDbIn::gc_cont2(BlockHandle handle) { }); return; } + CHECK(!permanent_mode_); td::PerfWarningTimer timer{"gccell", 0.1}; td::PerfWarningTimer timer_all{"gccell_all", 0.05}; @@ -614,7 +799,7 @@ void CellDbIn::gc_cont2(BlockHandle handle) { P = std::move(P), N = std::move(N), cell = std::move(cell), timer = std::move(timer), timer_all = std::move(timer_all), handle](td::Result R) mutable { R.ensure(); - td::actor::send_lambda(SelfId, [this, timer_boc = std::move(timer_boc), F = std::move(F), key_hash, + td::actor::send_lambda_later(SelfId, [this, timer_boc = std::move(timer_boc), F = std::move(F), key_hash, P = std::move(P), N = std::move(N), cell = std::move(cell), timer = std::move(timer), timer_all = std::move(timer_all), handle]() mutable { TD_PERF_COUNTER(celldb_gc_cell); @@ -714,6 +899,9 @@ void CellDbIn::set_block(KeyHash key_hash, DbEntry e) { } void CellDbIn::migrate_cell(td::Bits256 hash) { + if (permanent_mode_) { + return; + } cells_to_migrate_.insert(hash); if (!migration_active_) { migration_active_ = true; @@ -826,6 +1014,14 @@ void CellDb::store_cell(BlockIdExt block_id, td::Ref cell, td::Promise td::actor::send_closure(cell_db_, &CellDbIn::store_cell, block_id, std::move(cell), std::move(promise)); } +void CellDb::store_block_state_permanent(td::Ref block, td::Promise> promise) { + td::actor::send_closure(cell_db_, &CellDbIn::store_block_state_permanent, std::move(block), std::move(promise)); +} + +void CellDb::store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise) { + td::actor::send_closure(cell_db_, &CellDbIn::store_block_state_permanent_bulk, std::move(blocks), std::move(promise)); +} + void CellDb::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(cell_db_, &CellDbIn::get_cell_db_reader, std::move(promise)); } @@ -862,9 +1058,14 @@ td::BufferSlice CellDbIn::DbEntry::release() { std::vector> CellDbIn::CellDbStatistics::prepare_stats() { std::vector> stats; + stats.emplace_back("permanent_mode", PSTRING() << permanent_mode_); stats.emplace_back("store_cell.micros", PSTRING() << store_cell_time_.to_string()); stats.emplace_back("store_cell.prepare.micros", PSTRING() << store_cell_prepare_time_.to_string()); stats.emplace_back("store_cell.write.micros", PSTRING() << store_cell_write_time_.to_string()); + if (permanent_mode_) { + stats.emplace_back("store_cell.bulk.queries", PSTRING() << store_cell_bulk_queries_); + stats.emplace_back("store_cell.bulk.total_blocks", PSTRING() << store_cell_bulk_total_blocks_); + } stats.emplace_back("gc_cell.micros", PSTRING() << gc_cell_time_.to_string()); stats.emplace_back("total_time.micros", PSTRING() << (td::Timestamp::now().at() - stats_start_time_.at()) * 1e6); stats.emplace_back("in_memory", PSTRING() << bool(in_memory_load_time_)); diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 1e1ccddabf..f70f6aef82 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -66,6 +66,8 @@ class CellDbIn : public CellDbBase { void load_cell(RootHash hash, td::Promise> promise); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); void get_cell_db_reader(td::Promise> promise); + void store_block_state_permanent(td::Ref block, td::Promise> promise); + void store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise); void migrate_cell(td::Bits256 hash); @@ -137,9 +139,12 @@ class CellDbIn : public CellDbBase { std::unique_ptr migration_stats_; struct CellDbStatistics { + bool permanent_mode_; PercentileStats store_cell_time_; PercentileStats store_cell_prepare_time_; PercentileStats store_cell_write_time_; + size_t store_cell_bulk_queries_ = 0; + size_t store_cell_bulk_total_blocks_ = 0; PercentileStats gc_cell_time_; td::Timestamp stats_start_time_ = td::Timestamp::now(); std::optional in_memory_load_time_; @@ -156,6 +161,7 @@ class CellDbIn : public CellDbBase { CellDbStatistics cell_db_statistics_; td::Timestamp statistics_flush_at_ = td::Timestamp::never(); BlockSeqno last_deleted_mc_state_ = 0; + bool permanent_mode_ = false; bool db_busy_ = false; std::queue> action_queue_; @@ -188,6 +194,8 @@ class CellDb : public CellDbBase { void prepare_stats(td::Promise>> promise); void load_cell(RootHash hash, td::Promise> promise); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); + void store_block_state_permanent(td::Ref block, td::Promise> promise); + void store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise); void update_snapshot(std::unique_ptr snapshot) { CHECK(!opts_->get_celldb_in_memory()); if (!started_) { diff --git a/validator/db/permanent-celldb/permanent-celldb-utils.cpp b/validator/db/permanent-celldb/permanent-celldb-utils.cpp new file mode 100644 index 0000000000..b1d19ceb4b --- /dev/null +++ b/validator/db/permanent-celldb/permanent-celldb-utils.cpp @@ -0,0 +1,83 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "permanent-celldb-utils.h" +#include "block/block-auto.h" +#include "td/actor/MultiPromise.h" +#include "td/utils/HashMap.h" +#include "vm/db/CellStorage.h" + +namespace ton::validator { + +void calculate_permanent_celldb_update(const std::map>& blocks, + std::shared_ptr executor, + td::Promise> promise) { + td::MultiPromise mp; + auto ig = mp.init_guard(); + auto updates = std::make_shared>(); + updates->reserve(blocks.size()); + for (auto& [_, block] : blocks) { + executor->execute_async([block = block, updates, executor, + promise = std::make_shared>(ig.get_promise())]() mutable { + block::gen::Block::Record rec; + if (!block::gen::unpack_cell(block->root_cell(), rec)) { + promise->set_error(td::Status::Error("cannot unpack Block record")); + return; + } + bool spec; + vm::CellSlice update_cs = vm::load_cell_slice_special(rec.state_update, spec); + if (update_cs.special_type() != vm::CellTraits::SpecialType::MerkleUpdate) { + promise->set_error(td::Status::Error("invalid Merkle update in block")); + return; + } + td::Ref new_state_root = update_cs.prefetch_ref(1); + td::HashMap visited; + PermanentCellDbUpdate update{.block_id = block->block_id(), + .state_root_hash = new_state_root->get_hash(0).bits()}; + std::function&, int)> dfs = [&](const td::Ref& cell, int merkle_depth) { + int& vis = visited[cell->get_hash()]; + if (vis & (1 << merkle_depth)) { + return; + } + vis |= (1 << merkle_depth); + vm::CellSlice cs{vm::NoVm(), cell}; + if (cs.special_type() == vm::CellTraits::SpecialType::PrunnedBranch && cell->get_level() == merkle_depth + 1) { + return; + } + update.to_store.emplace_back( + cell->get_hash(merkle_depth), + vm::CellStorer::serialize_value(1 << 29, cell->load_cell().move_as_ok().data_cell, false, merkle_depth)); + merkle_depth = cs.child_merkle_depth(merkle_depth); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + dfs(cs.prefetch_ref(i), merkle_depth); + } + }; + dfs(new_state_root, 0); + executor->execute_sync( + [update = std::move(update), updates = std::move(updates), promise = std::move(promise)]() mutable { + updates->push_back(std::move(update)); + promise->set_result(td::Unit()); + }); + }); + } + + ig.add_promise([updates, promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + promise.set_value(std::move(*updates)); + }); +} + +} // namespace ton::validator diff --git a/validator/db/permanent-celldb/permanent-celldb-utils.h b/validator/db/permanent-celldb/permanent-celldb-utils.h new file mode 100644 index 0000000000..fad2ddfcdd --- /dev/null +++ b/validator/db/permanent-celldb/permanent-celldb-utils.h @@ -0,0 +1,37 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "vm/cells.h" +#include "ton/ton-types.h" +#include "interfaces/block.h" +#include "vm/db/DynamicBagOfCellsDb.h" + +#include +#include + +namespace ton::validator { + +struct PermanentCellDbUpdate { + BlockIdExt block_id; + RootHash state_root_hash; + std::vector> to_store; +}; +void calculate_permanent_celldb_update(const std::map>& blocks, + std::shared_ptr executor, + td::Promise> promise); + +} // namespace ton::validator diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 8eb2b76a93..dfdbc72d0f 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -259,6 +259,41 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref state, } } +void RootDb::store_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) { + if (handle->id() != block->block_id()) { + promise.set_error(td::Status::Error("block id mismatch")); + return; + } + if (handle->moved_to_archive() || handle->inited_state_boc()) { + get_block_state(handle, std::move(promise)); + return; + } + auto P = td::PromiseCreator::lambda( + [b = archive_db_.get(), handle, promise = std::move(promise)](td::Result> R) mutable { + TRY_RESULT_PROMISE(promise, root, std::move(R)); + handle->set_state_root_hash(root->get_hash().bits()); + handle->set_state_boc(); + + auto S = create_shard_state(handle->id(), std::move(root)); + S.ensure(); + + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise), state = S.move_as_ok()](td::Result R) mutable { + R.ensure(); + promise.set_value(std::move(state)); + }); + + td::actor::send_closure(b, &ArchiveManager::update_handle, std::move(handle), std::move(P)); + }); + td::actor::send_closure(cell_db_, &CellDb::store_block_state_permanent, std::move(block), std::move(P)); +} + +void RootDb::store_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) { + td::actor::send_closure(cell_db_, &CellDb::store_block_state_permanent_bulk, std::move(blocks), std::move(promise)); +} + void RootDb::get_block_state(ConstBlockHandle handle, td::Promise> promise) { if (handle->inited_state_boc()) { if (handle->deleted_state_boc()) { diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 782e48532c..05d5cc027f 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -62,6 +62,10 @@ class RootDb : public Db { void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; + void store_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override; + void store_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) override; void get_block_state(ConstBlockHandle handle, td::Promise> promise) override; void store_block_state_part(BlockId effective_block, td::Ref cell, td::Promise> promise) override; diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index 8e4cded06e..d60811d59d 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -178,7 +178,22 @@ void DownloadShardState::download_state() { checked_proof_link(); return; } + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading proof"); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = block_id_](td::Result R) { + if (R.is_error()) { + LOG(DEBUG) << "Cannot get proof link from import: " << R.move_as_error(); + td::actor::send_closure(SelfId, &DownloadShardState::download_proof_link); + } else { + LOG(INFO) << "Got proof link for " << block_id.to_str() << " from import"; + td::actor::send_closure(SelfId, &DownloadShardState::downloaded_proof_link, R.move_as_ok()); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::get_block_proof_link_from_import, block_id_, + masterchain_block_id_, std::move(P)); +} + +void DownloadShardState::download_proof_link() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { fail_handler(SelfId, R.move_as_error()); @@ -188,7 +203,6 @@ void DownloadShardState::download_state() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, block_id_, priority_, std::move(P)); - status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading proof"); } void DownloadShardState::downloaded_proof_link(td::BufferSlice data) { diff --git a/validator/downloaders/download-state.hpp b/validator/downloaders/download-state.hpp index 2ba30e922f..1c1be04c47 100644 --- a/validator/downloaders/download-state.hpp +++ b/validator/downloaders/download-state.hpp @@ -46,6 +46,7 @@ class DownloadShardState : public td::actor::Actor { void checked_proof_link(); void download_state(); + void download_proof_link(); void download_zero_state(); void downloaded_zero_state(td::BufferSlice data); diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index 00085e9606..92a0720bc7 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -68,6 +68,8 @@ void WaitBlockState::start() { return; } bool inited_proof = handle_->id().is_masterchain() ? handle_->inited_proof() : handle_->inited_proof_link(); + bool allow_download = + last_masterchain_state_.is_null() || opts_->need_monitor(handle_->id().shard_full(), last_masterchain_state_); if (handle_->received_state() && inited_proof) { reading_from_db_ = true; @@ -81,8 +83,8 @@ void WaitBlockState::start() { td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle_, std::move(P)); } else if (handle_->id().id.seqno == 0 && next_static_file_attempt_.is_in_past()) { next_static_file_attempt_ = td::Timestamp::in(60.0); - // id.file_hash contrains correct file hash of zero state - // => if file with this sha256 is found it is garanteed to be correct + // id.file_hash contains correct file hash of zero state + // => if file with this sha256 is found it is guaranteed to be correct // => if it is not, this error is permanent auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id = handle_->id()](td::Result R) { if (R.is_error()) { @@ -108,7 +110,7 @@ void WaitBlockState::start() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_zero_state_request, handle_->id(), priority_, std::move(P)); - } else if (check_persistent_state_desc() && !handle_->received_state()) { + } else if (check_persistent_state_desc() && !handle_->received_state() && allow_download) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { LOG(WARNING) << "failed to get persistent state: " << R.move_as_error(); @@ -138,7 +140,11 @@ void WaitBlockState::start() { .release(); } } else if (!handle_->inited_prev() || (!handle_->inited_proof() && !handle_->inited_proof_link())) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { + if (!allow_download) { + abort_query(td::Status::Error(PSTRING() << "not monitoring shard " << handle_->id().shard_full())); + return; + } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); }, td::Timestamp::in(0.1)); @@ -164,6 +170,10 @@ void WaitBlockState::start() { td::actor::send_closure(manager_, &ValidatorManager::wait_prev_block_state, handle_, priority_, timeout_, std::move(P)); } else if (handle_->id().is_masterchain() && !handle_->inited_proof()) { + if (!allow_download) { + abort_query(td::Status::Error(PSTRING() << "not monitoring shard " << handle_->id().shard_full())); + return; + } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { if (R.is_error()) { delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof); }, @@ -177,6 +187,10 @@ void WaitBlockState::start() { td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_request, handle_->id(), priority_, std::move(P)); } else if (block_.is_null()) { + if (!allow_download && !handle_->received()) { + abort_query(td::Status::Error(PSTRING() << "not monitoring shard " << handle_->id().shard_full())); + return; + } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &WaitBlockState::failed_to_get_block_data, @@ -261,6 +275,12 @@ void WaitBlockState::got_block_data(td::Ref data) { } void WaitBlockState::apply() { + if (opts_->get_permanent_celldb()) { + td::actor::send_closure(manager_, &ValidatorManager::set_block_state_from_data, handle_, block_, + std::move(promise_)); + stop(); + return; + } TD_PERF_COUNTER(apply_block_to_state); td::PerfWarningTimer t{"applyblocktostate", 0.1}; auto S = prev_state_.write().apply_block(handle_->id(), block_); diff --git a/validator/downloaders/wait-block-state.hpp b/validator/downloaders/wait-block-state.hpp index 6a14d909fc..a9317381c6 100644 --- a/validator/downloaders/wait-block-state.hpp +++ b/validator/downloaders/wait-block-state.hpp @@ -26,18 +26,21 @@ namespace validator { class WaitBlockState : public td::actor::Actor { public: - WaitBlockState(BlockHandle handle, td::uint32 priority, td::actor::ActorId manager, + WaitBlockState(BlockHandle handle, td::uint32 priority, td::Ref opts, + td::Ref last_masterchain_state, td::actor::ActorId manager, td::Timestamp timeout, td::Promise> promise, td::Ref persistent_state_desc = {}) : handle_(std::move(handle)) , priority_(priority) + , opts_(opts) + , last_masterchain_state_(last_masterchain_state) , manager_(manager) , timeout_(timeout) , promise_(std::move(promise)) , persistent_state_desc_(std::move(persistent_state_desc)) , perf_timer_("waitstate", 1.0, [manager](double duration) { - send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitstate", duration); - }) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitstate", duration); + }) { } void abort_query(td::Status reason); @@ -89,6 +92,8 @@ class WaitBlockState : public td::actor::Actor { td::uint32 priority_; + td::Ref opts_; + td::Ref last_masterchain_state_; td::actor::ActorId manager_; td::Timestamp timeout_; td::Promise> promise_; diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 608a5ec8be..00f6d0f255 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -541,7 +541,11 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod auto P = td::PromiseCreator::lambda([promise = std::move(promise), cnt](td::Result> R) mutable { if (R.is_error()) { - LOG(WARNING) << "getnextkey: " << R.move_as_error(); + if (R.error().code() == ErrorCode::notready) { + LOG(DEBUG) << "getnextkey: " << R.move_as_error(); + } else { + LOG(WARNING) << "getnextkey: " << R.move_as_error(); + } auto x = create_serialize_tl_object( std::vector>{}, false, true); promise.set_value(std::move(x)); diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index d9a78b2de2..dd038c6085 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -388,11 +388,8 @@ td::Status MasterchainStateQ::mc_reinit() { CHECK(config_); CHECK(config_->set_block_id_ext(get_block_id())); - auto cv_root = config_->get_config_param(35, 34); - if (cv_root.not_null()) { - TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(cv_root), true)); - cur_validators_ = std::move(validators); - } + cur_validators_ = config_->get_cur_validator_set(); + auto nv_root = config_->get_config_param(37, 36); if (nv_root.not_null()) { TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root), true)); diff --git a/validator/import-db-slice-local.cpp b/validator/import-db-slice-local.cpp new file mode 100644 index 0000000000..8c8db4ffb1 --- /dev/null +++ b/validator/import-db-slice-local.cpp @@ -0,0 +1,640 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "import-db-slice-local.hpp" + +#include "validator/db/fileref.hpp" +#include "td/utils/overloaded.h" +#include "validator/fabric.h" +#include "td/actor/MultiPromise.h" +#include "common/checksum.h" +#include "td/utils/port/path.h" +#include "ton/ton-io.hpp" +#include "downloaders/download-state.hpp" +#include "block/block-auto.h" + +#include + +namespace ton { + +namespace validator { + +ArchiveImporterLocal::ArchiveImporterLocal(std::string db_root, td::Ref state, + BlockSeqno shard_client_seqno, td::Ref opts, + td::actor::ActorId manager, + std::vector to_import_files, + td::Promise> promise) + : db_root_(std::move(db_root)) + , last_masterchain_state_(std::move(state)) + , shard_client_seqno_(shard_client_seqno) + , opts_(std::move(opts)) + , manager_(manager) + , to_import_files_(std::move(to_import_files)) + , promise_(std::move(promise)) + , perf_timer_("import-slice-local", 10.0, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "import-slice-local", duration); + }) { +} + +void ArchiveImporterLocal::start_up() { + LOG(WARNING) << "Importing archive for masterchain seqno #" << shard_client_seqno_ + 1 << " from disk"; + for (const std::string &path : to_import_files_) { + LOG(INFO) << "Importing file from disk " << path; + td::Status S = process_package(path); + if (S.is_error()) { + LOG(WARNING) << "Error processing package " << path << ": " << S; + } + } + + process_masterchain_blocks(); +} + +td::Status ArchiveImporterLocal::process_package(std::string path) { + LOG(DEBUG) << "Processing package " << path; + TRY_RESULT(p, Package::open(path, false, false)); + auto package = std::make_shared(std::move(p)); + + td::Status S = td::Status::OK(); + package->iterate([&](std::string filename, td::BufferSlice data, td::uint64) -> bool { + auto F = FileReference::create(filename); + if (F.is_error()) { + S = F.move_as_error(); + return false; + } + auto f = F.move_as_ok(); + + BlockIdExt block_id; + bool is_proof = false; + bool ignore = true; + + f.ref().visit(td::overloaded( + [&](const fileref::Proof &p) { + block_id = p.block_id; + ignore = !block_id.is_masterchain(); + is_proof = true; + }, + [&](const fileref::ProofLink &p) { + block_id = p.block_id; + ignore = block_id.is_masterchain(); + is_proof = true; + }, + [&](const fileref::Block &p) { + block_id = p.block_id; + ignore = false; + is_proof = false; + }, + [&](const auto &) { ignore = true; })); + + if (ignore || block_id.is_masterchain() && block_id.seqno() <= last_masterchain_state_->get_seqno()) { + return true; + } + + if (is_proof) { + if (block_id.is_masterchain()) { + auto R = create_proof(block_id, std::move(data)); + if (R.is_error()) { + S = R.move_as_error(); + return false; + } + blocks_[block_id].proof = R.move_as_ok(); + } else { + auto R = create_proof_link(block_id, std::move(data)); + if (R.is_error()) { + S = R.move_as_error(); + return false; + } + blocks_[block_id].proof_link = R.move_as_ok(); + } + } else { + if (td::sha256_bits256(data) != block_id.file_hash) { + S = td::Status::Error(ErrorCode::protoviolation, "bad block file hash"); + return false; + } + auto R = create_block(block_id, std::move(data)); + if (R.is_error()) { + S = R.move_as_error(); + return false; + } + blocks_[block_id].block = R.move_as_ok(); + } + if (block_id.is_masterchain()) { + masterchain_blocks_[block_id.seqno()] = block_id; + } + return true; + }); + return S; +} + +void ArchiveImporterLocal::process_masterchain_blocks() { + if (masterchain_blocks_.empty()) { + LOG(INFO) << "No masterchain blocks in the archive"; + checked_masterchain_proofs(); + return; + } + + if (masterchain_blocks_.begin()->first != last_masterchain_state_->get_seqno() + 1) { + abort_query(td::Status::Error(ErrorCode::notready, PSTRING() << "expected masterchain seqno " + << last_masterchain_state_->get_seqno() + 1 + << ", found " << masterchain_blocks_.begin()->first)); + return; + } + { + BlockSeqno expected_seqno = last_masterchain_state_->get_seqno() + 1; + for (auto &[seqno, _] : masterchain_blocks_) { + if (seqno != expected_seqno) { + abort_query( + td::Status::Error(ErrorCode::protoviolation, "non-consecutive masterchain blocks in the archive")); + return; + } + ++expected_seqno; + } + } + BlockInfo &first_block = blocks_[masterchain_blocks_.begin()->second]; + if (first_block.proof.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block proof")); + return; + } + if (first_block.block.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block data")); + return; + } + block::gen::Block::Record rec; + block::gen::BlockInfo::Record info; + if (!(block::gen::unpack_cell(first_block.block->root_cell(), rec) && block::gen::unpack_cell(rec.info, info))) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "cannot unpack masterchain block info")); + return; + } + if (info.key_block) { + import_first_key_block(); + return; + } + + process_masterchain_blocks_cont(); +} + +void ArchiveImporterLocal::import_first_key_block() { + BlockIdExt block_id = masterchain_blocks_.begin()->second; + BlockInfo &first_block = blocks_[block_id]; + LOG(INFO) << "First block in archive is key block : " << block_id.id.to_str(); + + auto P = + td::PromiseCreator::lambda([SelfId = actor_id(this), prev_block_id = last_masterchain_state_->get_block_id()]( + td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + auto handle = R.move_as_ok(); + CHECK(!handle->merge_before()); + if (handle->one_prev(true) != prev_block_id) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + td::Status::Error(ErrorCode::protoviolation, "prev block mismatch")); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::checked_key_block_proof, std::move(handle)); + }); + + run_check_proof_query(block_id, first_block.proof, manager_, td::Timestamp::in(10.0), std::move(P), + last_masterchain_state_, opts_->is_hardfork(block_id)); +} + +void ArchiveImporterLocal::checked_key_block_proof(BlockHandle handle) { + BlockIdExt block_id = masterchain_blocks_.begin()->second; + CHECK(block_id == handle->id()); + BlockInfo &first_block = blocks_[block_id]; + run_apply_block_query( + handle->id(), first_block.block, handle->id(), manager_, td::Timestamp::in(600.0), + [SelfId = actor_id(this), manager = manager_, handle](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure( + manager, &ValidatorManager::get_shard_state_from_db, handle, [=](td::Result> R2) { + if (R2.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R2.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::applied_key_block, + td::Ref{R2.move_as_ok()}); + }); + }); +} + +void ArchiveImporterLocal::applied_key_block(td::Ref state) { + CHECK(state->get_block_id() == masterchain_blocks_.begin()->second); + last_masterchain_state_ = state; + imported_any_ = true; + masterchain_blocks_.erase(masterchain_blocks_.begin()); + blocks_.erase(state->get_block_id()); + LOG(INFO) << "Imported key block " << state->get_block_id().id.to_str(); + if (masterchain_blocks_.empty()) { + LOG(INFO) << "No more masterchain blocks in the archive"; + checked_masterchain_proofs(); + return; + } + process_masterchain_blocks_cont(); +} + +void ArchiveImporterLocal::process_masterchain_blocks_cont() { + LOG(INFO) << "Importing masterchain blocks from " << masterchain_blocks_.begin()->first << " to " + << masterchain_blocks_.rbegin()->first; + + td::MultiPromise mp; + auto ig = mp.init_guard(); + + BlockIdExt prev_block_id = last_masterchain_state_->get_block_id(); + for (auto &[_, block_id] : masterchain_blocks_) { + auto &info = blocks_[block_id]; + info.import = true; + if (info.proof.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block proof")); + return; + } + if (info.block.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block data")); + return; + } + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), prev_block_id, promise = ig.get_promise()](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + CHECK(!handle->merge_before()); + if (handle->one_prev(true) != prev_block_id) { + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "prev block mismatch")); + return; + } + promise.set_result(td::Unit()); + }); + run_check_proof_query(block_id, info.proof, manager_, td::Timestamp::in(10.0), std::move(P), + last_masterchain_state_, opts_->is_hardfork(block_id)); + prev_block_id = block_id; + } + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + } else { + LOG(INFO) << "Checked proofs for masterchain blocks"; + td::actor::send_closure(SelfId, &ArchiveImporterLocal::checked_masterchain_proofs); + } + }); +} + +void ArchiveImporterLocal::checked_masterchain_proofs() { + if (shard_client_seqno_ == last_masterchain_state_->get_seqno()) { + got_shard_client_state(last_masterchain_state_); + } else { + CHECK(shard_client_seqno_ < last_masterchain_state_->get_seqno()); + BlockIdExt block_id; + if (!last_masterchain_state_->get_old_mc_block_id(shard_client_seqno_, block_id)) { + abort_query(td::Status::Error("failed to get shard client block id")); + return; + } + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db_short, block_id, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + R.move_as_error_prefix("failed to get shard client state: ")); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::got_shard_client_state, + td::Ref{R.move_as_ok()}); + }); + } +} + +void ArchiveImporterLocal::got_shard_client_state(td::Ref state) { + CHECK(state->get_seqno() == shard_client_seqno_); + LOG(DEBUG) << "got_shard_client_state " << shard_client_seqno_; + shard_client_state_ = state; + new_shard_client_seqno_ = shard_client_seqno_; + current_shard_client_seqno_ = shard_client_seqno_; + for (auto &shard : state->get_shards()) { + visited_shard_blocks_.insert(shard->top_block_id()); + } + try_advance_shard_client_seqno(); +} + +void ArchiveImporterLocal::try_advance_shard_client_seqno() { + BlockSeqno seqno = new_shard_client_seqno_ + 1; + auto it = masterchain_blocks_.find(seqno); + if (it != masterchain_blocks_.end()) { + try_advance_shard_client_seqno_cont(blocks_[it->second].block); + return; + } + if (seqno > last_masterchain_state_->get_seqno()) { + processed_shard_blocks(); + return; + } + BlockIdExt block_id; + if (!last_masterchain_state_->get_old_mc_block_id(seqno, block_id)) { + abort_query(td::Status::Error("failed to get old mc block id")); + return; + } + td::actor::send_closure(manager_, &ValidatorManager::get_block_data_from_db_short, block_id, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + R.move_as_error_prefix("failed to get block data: ")); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::try_advance_shard_client_seqno_cont, + R.move_as_ok()); + }); +} + +void ArchiveImporterLocal::try_advance_shard_client_seqno_cont(td::Ref mc_block) { + CHECK(mc_block.not_null()); + CHECK(mc_block->block_id().seqno() == new_shard_client_seqno_ + 1); + LOG(DEBUG) << "try_advance_shard_client_seqno " << new_shard_client_seqno_ + 1; + + block::gen::Block::Record rec; + block::gen::BlockExtra::Record extra; + block::gen::McBlockExtra::Record mc_extra; + CHECK(block::gen::unpack_cell(mc_block->root_cell(), rec) && block::gen::unpack_cell(rec.extra, extra) && + block::gen::unpack_cell(extra.custom->prefetch_ref(), mc_extra)); + auto shard_config = std::make_unique(mc_extra.shard_hashes->prefetch_ref()); + + std::vector blocks_to_import; + std::function dfs = [&](const BlockIdExt &block_id) -> td::Status { + if (visited_shard_blocks_.contains(block_id)) { + return td::Status::OK(); + } + if (block_id.seqno() == 0) { + new_zerostates_.insert(block_id); + return td::Status::OK(); + } + visited_shard_blocks_.insert(block_id); + auto &info = blocks_[block_id]; + if (info.block.is_null()) { + return td::Status::Error(PSTRING() << "no shard block " << block_id.to_str()); + } + blocks_to_import.push_back(block_id); + + std::vector prev; + BlockIdExt mc_blkid; + bool after_split; + TRY_STATUS(block::unpack_block_prev_blk_try(info.block->root_cell(), block_id, prev, mc_blkid, after_split)); + for (const BlockIdExt &prev_block_id : prev) { + TRY_STATUS(dfs(prev_block_id)); + } + + return td::Status::OK(); + }; + td::Status S = td::Status::OK(); + std::vector top_shard_blocks; + shard_config->process_shard_hashes([&](block::McShardHash &shard) { + if (!opts_->need_monitor(shard.shard(), shard_client_state_)) { + return 0; + } + S = dfs(shard.top_block_id()); + top_shard_blocks.push_back(shard.top_block_id()); + if (S.is_error()) { + return -1; + } + return 0; + }); + if (S.is_error()) { + LOG(DEBUG) << "Cannot advance shard client seqno to " << new_shard_client_seqno_ + 1 << " : " << S; + processed_shard_blocks(); + return; + } + shard_configs_[mc_block->block_id().seqno()] = {mc_block->block_id(), std::move(top_shard_blocks)}; + ++new_shard_client_seqno_; + LOG(DEBUG) << "Advancing shard client seqno to " << new_shard_client_seqno_; + for (const BlockIdExt &block_id : blocks_to_import) { + blocks_[block_id].import = true; + } + td::actor::send_closure(actor_id(this), &ArchiveImporterLocal::try_advance_shard_client_seqno); +} + +void ArchiveImporterLocal::processed_shard_blocks() { + if (new_shard_client_seqno_ == shard_client_seqno_) { + LOG(INFO) << "No new shard blocks"; + } else { + LOG(INFO) << "New shard client seqno = " << new_shard_client_seqno_; + } + + td::MultiPromise mp; + auto ig = mp.init_guard(); + for (const BlockIdExt &block_id : new_zerostates_) { + LOG(INFO) << "Downloading zerostate " << block_id.to_str(); + td::actor::create_actor( + "downloadstate", block_id, shard_client_state_->get_block_id(), + shard_client_state_->persistent_state_split_depth(block_id.id.workchain), 2, manager_, td::Timestamp::in(3600), + ig.get_promise().wrap([](td::Ref &&) { return td::Unit(); })) + .release(); + } + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::store_data); + } + }); +} + +void ArchiveImporterLocal::store_data() { + td::MultiPromise mp; + auto ig = mp.init_guard(); + + if (opts_->get_permanent_celldb()) { + std::vector> blocks; + for (auto &[_, info] : blocks_) { + if (info.import) { + blocks.push_back(info.block); + } + } + td::actor::send_closure(manager_, &ValidatorManager::set_block_state_from_data_preliminary, std::move(blocks), + ig.get_promise()); + } + for (auto &[block_id, info] : blocks_) { + if (info.import) { + td::actor::send_closure( + manager_, &ValidatorManager::get_block_handle, block_id, true, + [promise = ig.get_promise(), block = info.block, manager = manager_](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + td::actor::send_closure(manager, &ValidatorManager::set_block_data, handle, std::move(block), + std::move(promise)); + }); + if (info.proof_link.not_null()) { + run_check_proof_link_query(block_id, info.proof_link, manager_, td::Timestamp::in(60.0), + ig.get_promise().wrap([](BlockHandle &&) { return td::Unit(); })); + } + } + } + + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::apply_next_masterchain_block); + } + }); +} + +void ArchiveImporterLocal::apply_next_masterchain_block() { + auto it = masterchain_blocks_.find(last_masterchain_state_->get_seqno() + 1); + if (it == masterchain_blocks_.end()) { + LOG(INFO) << "Applied masterchain blocks, last seqno = " << last_masterchain_state_->get_seqno(); + apply_shard_blocks(); + return; + } + BlockIdExt block_id = it->second; + LOG(DEBUG) << "Applying masterchain block " << block_id.to_str(); + BlockInfo &info = blocks_[block_id]; + run_apply_block_query(block_id, info.block, block_id, manager_, td::Timestamp::in(600.0), + [=, SelfId = actor_id(this), manager = manager_](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure( + manager, &ValidatorManager::get_shard_state_from_db_short, block_id, + [=](td::Result> R2) { + if (R2.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + R2.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::applied_next_masterchain_block, + td::Ref{R2.move_as_ok()}); + }); + }); +} + +void ArchiveImporterLocal::applied_next_masterchain_block(td::Ref state) { + last_masterchain_state_ = state; + imported_any_ = true; + LOG(DEBUG) << "Applied masterchain block " << state->get_block_id().to_str(); + apply_next_masterchain_block(); +} + +void ArchiveImporterLocal::apply_shard_blocks() { + if (current_shard_client_seqno_ == new_shard_client_seqno_) { + finish_query(); + return; + } + auto it = shard_configs_.find(current_shard_client_seqno_ + 1); + if (it == shard_configs_.end()) { + abort_query(td::Status::Error("no shard config for the next shard client seqno")); + return; + } + + td::MultiPromise mp; + auto ig = mp.init_guard(); + BlockIdExt mc_block_id = it->second.first; + LOG(DEBUG) << "Applying top shard blocks from " << current_shard_client_seqno_ + 1; + for (const BlockIdExt &block_id : it->second.second) { + apply_shard_block(block_id, mc_block_id, ig.get_promise()); + } + + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::applied_shard_blocks); + }); +} + +void ArchiveImporterLocal::applied_shard_blocks() { + LOG(DEBUG) << "Applied top shard blocks from " << current_shard_client_seqno_ + 1; + ++current_shard_client_seqno_; + imported_any_ = true; + apply_shard_blocks(); +} + +void ArchiveImporterLocal::apply_shard_block(BlockIdExt block_id, BlockIdExt mc_block_id, + td::Promise promise) { + td::actor::send_closure( + manager_, &ValidatorManager::get_block_handle, block_id, true, + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + td::actor::send_closure(SelfId, &ArchiveImporterLocal::apply_shard_block_cont1, R.move_as_ok(), mc_block_id, + std::move(promise)); + }); +} + +void ArchiveImporterLocal::apply_shard_block_cont1(BlockHandle handle, BlockIdExt mc_block_id, + td::Promise promise) { + if (handle->is_applied()) { + promise.set_value(td::Unit()); + return; + } + + promise = [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + td::actor::send_closure(SelfId, &ArchiveImporterLocal::apply_shard_block_cont2, handle, mc_block_id, + std::move(promise)); + }; + + if (!handle->merge_before() && handle->one_prev(true).shard_full() == handle->id().shard_full()) { + apply_shard_block(handle->one_prev(true), mc_block_id, std::move(promise)); + } else { + td::MultiPromise mp; + auto ig = mp.init_guard(); + ig.add_promise(std::move(promise)); + check_shard_block_applied(handle->one_prev(true), ig.get_promise()); + if (handle->merge_before()) { + check_shard_block_applied(handle->one_prev(false), ig.get_promise()); + } + } +} + +void ArchiveImporterLocal::apply_shard_block_cont2(BlockHandle handle, BlockIdExt mc_block_id, + td::Promise promise) { + td::Ref block = blocks_[handle->id()].block; + CHECK(block.not_null()); + LOG(DEBUG) << "Applying shard block " << handle->id().to_str(); + run_apply_block_query(handle->id(), std::move(block), mc_block_id, manager_, td::Timestamp::in(600.0), + std::move(promise)); +} + +void ArchiveImporterLocal::check_shard_block_applied(BlockIdExt block_id, td::Promise promise) { + td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, block_id, false, + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + if (!handle->is_applied()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "not applied")); + } else { + promise.set_value(td::Unit()); + } + }); +} + +void ArchiveImporterLocal::abort_query(td::Status error) { + if (!imported_any_) { + LOG(ERROR) << "Archive import: " << error; + promise_.set_error(std::move(error)); + stop(); + } else { + LOG(WARNING) << "Archive import: " << error; + finish_query(); + } +} + +void ArchiveImporterLocal::finish_query() { + LOG(WARNING) << "Imported archive in " << perf_timer_.elapsed() + << "s : mc_seqno=" << last_masterchain_state_->get_seqno() + << " shard_seqno=" << current_shard_client_seqno_; + promise_.set_value({last_masterchain_state_->get_seqno(), + std::min(last_masterchain_state_->get_seqno(), current_shard_client_seqno_)}); + stop(); +} + +} // namespace validator + +} // namespace ton diff --git a/validator/import-db-slice-local.hpp b/validator/import-db-slice-local.hpp new file mode 100644 index 0000000000..e9e6efdb1e --- /dev/null +++ b/validator/import-db-slice-local.hpp @@ -0,0 +1,103 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "td/actor/actor.h" +#include "td/utils/port/path.h" +#include "validator/interfaces/validator-manager.h" +#include "validator/db/package.hpp" + +namespace ton { + +namespace validator { + +class ArchiveImporterLocal : public td::actor::Actor { + public: + ArchiveImporterLocal(std::string db_root, td::Ref state, BlockSeqno shard_client_seqno, + td::Ref opts, td::actor::ActorId manager, + std::vector to_import_files, + td::Promise> promise); + void start_up() override; + + void abort_query(td::Status error); + void finish_query(); + + td::Status process_package(std::string path); + + void process_masterchain_blocks(); + void process_masterchain_blocks_cont(); + + void import_first_key_block(); + void checked_key_block_proof(BlockHandle handle); + void applied_key_block(td::Ref state); + + void checked_masterchain_proofs(); + void got_shard_client_state(td::Ref state); + + void try_advance_shard_client_seqno(); + void try_advance_shard_client_seqno_cont(td::Ref mc_block); + + void processed_shard_blocks(); + void store_data(); + void apply_next_masterchain_block(); + void applied_next_masterchain_block(td::Ref state); + + void apply_shard_blocks(); + void applied_shard_blocks(); + + void apply_shard_block(BlockIdExt block_id, BlockIdExt mc_block_id, td::Promise promise); + void apply_shard_block_cont1(BlockHandle handle, BlockIdExt mc_block_id, td::Promise promise); + void apply_shard_block_cont2(BlockHandle handle, BlockIdExt mc_block_id, td::Promise promise); + void check_shard_block_applied(BlockIdExt block_id, td::Promise promise); + + private: + std::string db_root_; + td::Ref last_masterchain_state_; + BlockSeqno shard_client_seqno_; + + td::Ref opts_; + + td::actor::ActorId manager_; + + std::vector to_import_files_; + td::Promise> promise_; + + struct BlockInfo { + td::Ref block; + td::Ref proof; + td::Ref proof_link; + bool import = false; + }; + std::map blocks_; + std::map masterchain_blocks_; + + td::Ref shard_client_state_; + BlockSeqno new_shard_client_seqno_; + BlockSeqno current_shard_client_seqno_; + std::set visited_shard_blocks_; + std::set new_zerostates_; + + std::map>> shard_configs_; + + bool imported_any_ = false; + + td::PerfWarningTimer perf_timer_; +}; + +} // namespace validator + +} // namespace ton diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index 80fe141bbd..58f77afb68 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -45,7 +45,10 @@ ArchiveImporter::ArchiveImporter(std::string db_root, td::Ref , manager_(manager) , to_import_files_(std::move(to_import_files)) , use_imported_files_(!to_import_files_.empty()) - , promise_(std::move(promise)) { + , promise_(std::move(promise)) + , perf_timer_("import-slice", 10.0, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "import-slice", duration); + }) { } void ArchiveImporter::start_up() { @@ -85,7 +88,7 @@ void ArchiveImporter::downloaded_mc_archive(std::string path) { void ArchiveImporter::processed_mc_archive() { if (masterchain_blocks_.empty()) { - LOG(DEBUG) << "No masterhchain blocks in archive"; + LOG(DEBUG) << "No masterchain blocks in archive"; last_masterchain_seqno_ = last_masterchain_state_->get_seqno(); checked_all_masterchain_blocks(); return; @@ -527,6 +530,7 @@ void ArchiveImporter::finish_query() { td::unlink(f).ignore(); } if (promise_) { + LOG(INFO) << "Imported archive in " << perf_timer_.elapsed(); promise_.set_value({last_masterchain_state_->get_seqno(), std::min(last_masterchain_state_->get_seqno(), shard_client_seqno_)}); } diff --git a/validator/import-db-slice.hpp b/validator/import-db-slice.hpp index 04f22642dc..4cf6ce40ac 100644 --- a/validator/import-db-slice.hpp +++ b/validator/import-db-slice.hpp @@ -91,6 +91,8 @@ class ArchiveImporter : public td::actor::Actor { bool imported_any_ = false; bool have_shard_blocks_ = false; std::vector files_to_cleanup_; + + td::PerfWarningTimer perf_timer_; }; } // namespace validator diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index f8fce7ba9b..00e7f31f88 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -51,6 +51,10 @@ class Db : public td::actor::Actor { virtual void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; + virtual void store_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) = 0; + virtual void store_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) = 0; virtual void get_block_state(ConstBlockHandle handle, td::Promise> promise) = 0; virtual void store_block_state_part(BlockId effective_block, td::Ref cell, td::Promise> promise) = 0; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 34d7d57c77..abab4e6c97 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -72,6 +72,10 @@ class ValidatorManager : public ValidatorManagerInterface { td::Promise> promise) = 0; virtual void store_block_state_part(BlockId effective_block, td::Ref cell, td::Promise> promise) = 0; + virtual void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) = 0; + virtual void set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::BufferSlice state, @@ -157,6 +161,11 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; + virtual void get_block_proof_link_from_import(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + promise.set_error(td::Status::Error("not supported")); + } + virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 3cc463a654..f1be05ca34 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -323,8 +323,8 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle->id(), std::move(R)); }); - auto id = td::actor::create_actor("waitstate", handle, 0, actor_id(this), td::Timestamp::in(10.0), - std::move(P)) + auto id = td::actor::create_actor("waitstate", handle, 0, opts_, last_masterchain_state_, + actor_id(this), td::Timestamp::in(10.0), std::move(P)) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); @@ -693,6 +693,16 @@ void ValidatorManagerImpl::store_block_state_part(BlockId effective_block, td::R td::actor::send_closure(db_, &Db::store_block_state_part, effective_block, cell, std::move(promise)); } +void ValidatorManagerImpl::set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data, handle, block, std::move(promise)); +} + +void ValidatorManagerImpl::set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data_preliminary, std::move(blocks), std::move(promise)); +} + void ValidatorManagerImpl::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise)); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index ebbdd88877..dc08bf28b5 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -152,6 +152,10 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void store_block_state_part(BlockId effective_block, td::Ref cell, td::Promise> promise) override; + void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override; + void set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) override; void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::BufferSlice state, td::Promise promise) override; diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index 91c598aa2e..c3d6267bef 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -171,8 +171,8 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle->id(), std::move(R)); }); - auto id = td::actor::create_actor("waitstate", handle, 0, actor_id(this), td::Timestamp::in(10.0), - std::move(P)) + auto id = td::actor::create_actor("waitstate", handle, 0, opts_, last_masterchain_state_, + actor_id(this), td::Timestamp::in(10.0), std::move(P)) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 20430d8042..0f25a5cfa7 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -181,6 +181,13 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override { + UNREACHABLE(); + } + void set_block_state_from_data_preliminary(std::vector> blocks, td::Promise promise) { + UNREACHABLE(); + } void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::BufferSlice state, td::Promise promise) override { diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index 004e5b120d..f9a55c6ae3 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -59,7 +59,7 @@ void ValidatorManagerMasterchainReiniter::got_masterchain_handle(BlockHandle han handle_ = std::move(handle); key_blocks_.push_back(handle_); - if (opts_->initial_sync_disabled()) { + if (opts_->initial_sync_disabled() && handle_->id().seqno() == 0) { status_.set_status(PSTRING() << "downloading masterchain state " << handle_->id().seqno()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); @@ -74,7 +74,7 @@ void ValidatorManagerMasterchainReiniter::got_masterchain_handle(BlockHandle han download_proof_link(); } -void ValidatorManagerMasterchainReiniter::download_proof_link() { +void ValidatorManagerMasterchainReiniter::download_proof_link(bool try_local) { if (handle_->id().id.seqno == 0) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); @@ -84,45 +84,55 @@ void ValidatorManagerMasterchainReiniter::download_proof_link() { td::Timestamp::in(3600), std::move(P)) .release(); } else { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { - LOG(WARNING) << "failed to download proof link: " << R.move_as_error(); + if (try_local) { + LOG(DEBUG) << "failed to get proof link from local import: " << R.move_as_error(); + } else { + LOG(WARNING) << "failed to download proof link: " << R.move_as_error(); + } delay_action( - [SelfId]() { td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link); }, + [SelfId]() { + td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link, false); + }, td::Timestamp::in(1.0)); } else { td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::downloaded_proof_link, R.move_as_ok()); } }); - td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, handle_->id(), 2, - std::move(P)); + if (try_local) { + td::actor::send_closure(manager_, &ValidatorManager::get_block_proof_link_from_import, handle_->id(), + handle_->id(), std::move(P)); + } else { + td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, handle_->id(), 2, + std::move(P)); + } } } -void ValidatorManagerMasterchainReiniter::downloaded_proof_link(td::BufferSlice proof) { - auto pp = create_proof_link(handle_->id(), std::move(proof)); - if (pp.is_error()) { - LOG(WARNING) << "bad proof link: " << pp.move_as_error(); +void ValidatorManagerMasterchainReiniter::downloaded_proof_link(td::BufferSlice data) { + auto r_proof = create_proof(handle_->id(), std::move(data)); + if (r_proof.is_error()) { + LOG(WARNING) << "bad proof link: " << r_proof.move_as_error(); download_proof_link(); return; } + auto proof = r_proof.move_as_ok(); - auto proof_link = pp.move_as_ok(); - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), db = db_, proof_link](td::Result R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), db = db_, proof](td::Result R) { if (R.is_error()) { LOG(WARNING) << "downloaded proof link failed: " << R.move_as_error(); - td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link); + td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link, false); } else { auto P = td::PromiseCreator::lambda([SelfId, handle = R.move_as_ok()](td::Result R) { R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::try_download_key_blocks, false); }); - td::actor::send_closure(db, &Db::add_key_block_proof_link, proof_link, std::move(P)); + td::actor::send_closure(db, &Db::add_key_block_proof_link, proof, std::move(P)); } }); - - run_check_proof_link_query(handle_->id(), proof_link, manager_, td::Timestamp::in(60.0), std::move(P)); + run_check_proof_query(handle_->id(), proof, manager_, td::Timestamp::in(60.0), std::move(P), + /* skip_check_signatures = */ true); } void ValidatorManagerMasterchainReiniter::downloaded_zero_state() { @@ -130,6 +140,10 @@ void ValidatorManagerMasterchainReiniter::downloaded_zero_state() { } void ValidatorManagerMasterchainReiniter::try_download_key_blocks(bool try_start) { + if (opts_->initial_sync_disabled()) { + download_masterchain_state(); + return; + } if (!download_new_key_blocks_until_) { if (opts_->allow_blockchain_init()) { download_new_key_blocks_until_ = td::Timestamp::in(60.0); @@ -183,8 +197,6 @@ void ValidatorManagerMasterchainReiniter::got_next_key_blocks(std::vector(key_blocks_.size()); key_blocks_.resize(key_blocks_.size() + vec.size(), nullptr); @@ -204,6 +216,11 @@ void ValidatorManagerMasterchainReiniter::got_key_block_handle(td::uint32 idx, B CHECK(!key_blocks_[idx]); CHECK(handle->inited_proof()); CHECK(handle->is_key_block()); + if (idx + 1 == key_blocks_.size()) { + int ago = (int)td::Clocks::system() - (int)handle->unix_time(); + LOG(WARNING) << "last key block is " << handle->id().to_str() << ", " << ago << "s ago"; + status_.set_status(PSTRING() << "last key block is " << handle->id().seqno() << ", " << ago << " s ago"); + } key_blocks_[idx] = std::move(handle); CHECK(pending_ > 0); if (!--pending_) { diff --git a/validator/manager-init.hpp b/validator/manager-init.hpp index 901b826bdb..90dc5ccc97 100644 --- a/validator/manager-init.hpp +++ b/validator/manager-init.hpp @@ -44,7 +44,7 @@ class ValidatorManagerMasterchainReiniter : public td::actor::Actor { void start_up() override; void written_hardforks(); void got_masterchain_handle(BlockHandle handle); - void download_proof_link(); + void download_proof_link(bool try_local = true); void downloaded_proof_link(td::BufferSlice data); void downloaded_zero_state(); diff --git a/validator/manager.cpp b/validator/manager.cpp index 578b913233..7a2d63b7d8 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -31,6 +31,7 @@ #include "state-serializer.hpp" #include "get-next-key-blocks.h" #include "import-db-slice.hpp" +#include "import-db-slice-local.hpp" #include "auto/tl/lite_api.h" #include "tl-utils/lite-utils.hpp" @@ -42,6 +43,7 @@ #include "td/utils/JsonBuilder.h" #include "common/delay.h" +#include "db/fileref.hpp" #include "td/utils/filesystem.h" #include "validator/stats-merger.h" @@ -708,10 +710,6 @@ void ValidatorManagerImpl::run_ext_query(td::BufferSlice data, td::Promise> promise) { - if (last_masterchain_state_.not_null() && !opts_->need_monitor(handle->id().shard_full(), last_masterchain_state_)) { - return promise.set_error( - td::Status::Error(PSTRING() << "not monitoring shard " << handle->id().shard_full().to_str())); - } auto it0 = block_state_cache_.find(handle->id()); if (it0 != block_state_cache_.end()) { it0->second.ttl_ = td::Timestamp::in(30.0); @@ -723,10 +721,11 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); }); - auto id = td::actor::create_actor("waitstate", handle, priority, actor_id(this), - td::Timestamp::at(timeout.at() + 10.0), std::move(P), - get_block_persistent_state_to_download(handle->id())) - .release(); + auto id = + td::actor::create_actor("waitstate", handle, priority, opts_, last_masterchain_state_, + actor_id(this), td::Timestamp::at(timeout.at() + 10.0), std::move(P), + get_block_persistent_state_to_download(handle->id())) + .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); } @@ -785,10 +784,6 @@ void ValidatorManagerImpl::wait_block_data_short(BlockIdExt block_id, td::uint32 void ValidatorManagerImpl::wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) { - if (last_masterchain_state_.not_null() && !opts_->need_monitor(left_id.shard_full(), last_masterchain_state_)) { - return promise.set_error( - td::Status::Error(PSTRING() << "not monitoring shard " << left_id.shard_full().to_str())); - } td::actor::create_actor("merge", left_id, right_id, priority, actor_id(this), timeout, std::move(promise)) .release(); @@ -1171,10 +1166,10 @@ void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); }); - auto id = - td::actor::create_actor("waitstate", handle, X.second, actor_id(this), X.first, - std::move(P), get_block_persistent_state_to_download(handle->id())) - .release(); + auto id = td::actor::create_actor("waitstate", handle, X.second, opts_, last_masterchain_state_, + actor_id(this), X.first, std::move(P), + get_block_persistent_state_to_download(handle->id())) + .release(); it->second.actor_ = id; return; } @@ -1238,6 +1233,16 @@ void ValidatorManagerImpl::store_block_state_part(BlockId effective_block, td::R td::actor::send_closure(db_, &Db::store_block_state_part, effective_block, cell, std::move(promise)); } +void ValidatorManagerImpl::set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data, handle, block, std::move(promise)); +} + +void ValidatorManagerImpl::set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data_preliminary, std::move(blocks), std::move(promise)); +} + void ValidatorManagerImpl::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise)); } @@ -1693,6 +1698,67 @@ void ValidatorManagerImpl::send_download_archive_request(BlockSeqno mc_seqno, Sh callback_->download_archive(mc_seqno, shard_prefix, std::move(tmp_dir), timeout, std::move(promise)); } +void ValidatorManagerImpl::get_block_proof_link_from_import(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + auto it = to_import_all_.upper_bound(masterchain_block_id.seqno() + 1); + while (true) { + if (it == to_import_all_.begin()) { + promise.set_error(td::Status::Error("proof not found")); + return; + } + --it; + bool stop = false; + for (const std::string &path : it->second) { + td::BufferSlice result; + auto r_package = Package::open(path, false, false); + if (r_package.is_error()) { + LOG(WARNING) << "Cannot open package " << path << " : " << r_package.move_as_error(); + continue; + } + auto package = r_package.move_as_ok(); + package.iterate([&](std::string filename, td::BufferSlice data, td::uint64) -> bool { + auto F = FileReference::create(filename); + if (F.is_error()) { + return true; + } + auto f = F.move_as_ok(); + BlockIdExt id; + bool is_proof = false; + f.ref().visit(td::overloaded( + [&](const fileref::Block &p) { + id = p.block_id; + is_proof = false; + }, + [&](const fileref::Proof &p) { + id = p.block_id; + is_proof = true; + }, + [&](const fileref::ProofLink &p) { + id = p.block_id; + is_proof = true; + }, + [&](const auto &) {})); + if (is_proof && id == block_id) { + result = std::move(data); + return false; + } + if (shard_intersects(id.shard_full(), block_id.shard_full()) && id.seqno() < block_id.seqno()) { + stop = true; + } + return true; + }); + if (!result.empty()) { + promise.set_result(std::move(result)); + return; + } + } + if (block_id.is_masterchain() || stop) { + promise.set_error(td::Status::Error("proof not found")); + return; + } + } +} + void ValidatorManagerImpl::start_up() { db_ = create_db_actor(actor_id(this), db_root_, opts_); actor_stats_ = td::actor::create_actor("actor_stats"); @@ -1714,6 +1780,7 @@ void ValidatorManagerImpl::start_up() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::started, R.move_as_ok()); }); + size_t to_import_files = 0; auto to_import_dir = db_root_ + "/import"; auto S = td::WalkPath::run(to_import_dir, [&](td::CSlice cfname, td::WalkPath::Type t) -> void { auto fname = td::Slice(cfname); @@ -1733,26 +1800,31 @@ void ValidatorManagerImpl::start_up() { } fname = fname.substr(8); - while (fname.size() > 1 && fname[0] == '0') { - fname.remove_prefix(1); - } auto i = fname.find('.'); if (i == td::Slice::npos) { return; } fname = fname.substr(0, i); + while (fname.size() > 1 && fname[0] == '0') { + fname.remove_prefix(1); + } auto v = td::to_integer_safe(fname); if (v.is_error()) { return; } auto seqno = v.move_as_ok(); - LOG(INFO) << "found archive slice '" << cfname << "' for seqno " << seqno; + LOG(DEBUG) << "found archive slice '" << cfname << "' for seqno " << seqno; to_import_[seqno].push_back(cfname.str()); + ++to_import_files; } }); + to_import_all_ = to_import_; if (S.is_error()) { LOG(INFO) << "failed to load blocks from import dir: " << S; } + if (to_import_files > 0) { + LOG(INFO) << "found " << to_import_files << " files to import"; + } validator_manager_init(opts_, actor_id(this), db_.get(), std::move(P)); @@ -1935,9 +2007,15 @@ void ValidatorManagerImpl::download_next_archive() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::checked_archive_slice, R.ok().first, R.ok().second); } }); - td::actor::create_actor("archiveimport", db_root_, last_masterchain_state_, seqno, opts_, - actor_id(this), std::move(to_import_files), std::move(P)) - .release(); + if (to_import_files.empty()) { + td::actor::create_actor("archiveimport", db_root_, last_masterchain_state_, seqno, opts_, + actor_id(this), std::move(to_import_files), std::move(P)) + .release(); + } else { + td::actor::create_actor("archiveimport", db_root_, last_masterchain_state_, seqno, opts_, + actor_id(this), std::move(to_import_files), std::move(P)) + .release(); + } } void ValidatorManagerImpl::checked_archive_slice(BlockSeqno new_last_mc_seqno, BlockSeqno new_shard_client_seqno) { diff --git a/validator/manager.hpp b/validator/manager.hpp index e90d5b3472..59ee1ff63e 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -412,6 +412,9 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void store_block_state_part(BlockId effective_block, td::Ref cell, td::Promise> promise) override; + void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override; + void set_block_state_from_data_preliminary(std::vector> blocks, td::Promise promise) override; void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::BufferSlice state, td::Promise promise) override; @@ -523,6 +526,9 @@ class ValidatorManagerImpl : public ValidatorManager { void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) override; + void get_block_proof_link_from_import(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override; + void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; @@ -685,6 +691,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::actor::ActorOwn serializer_; std::map> to_import_; + std::map> to_import_all_; private: std::unique_ptr callback_; diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index 16d619b8e7..da6363f0d6 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -76,16 +76,20 @@ void DownloadState::start_up() { status_ = ProcessStatus(validator_manager_, "process.download_state_net"); alarm_timestamp() = timeout_; - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id_, - masterchain_block_id_, type_, - [SelfId = actor_id(this), block_id = block_id_](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &DownloadState::get_block_handle); - } else { - LOG(WARNING) << "got block state from disk: " << block_id.to_str(); - td::actor::send_closure(SelfId, &DownloadState::got_block_state, R.move_as_ok()); - } - }); + td::Promise P = [SelfId = actor_id(this), block_id = block_id_](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadState::get_block_handle); + } else { + LOG(WARNING) << "got block state from disk: " << block_id.to_str(); + td::actor::send_closure(SelfId, &DownloadState::got_block_state, R.move_as_ok()); + } + }; + if (block_id_.seqno() == 0) { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_zero_state, block_id_, std::move(P)); + } else { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id_, + masterchain_block_id_, type_, std::move(P)); + } } void DownloadState::get_block_handle() { diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index cb26fe44d6..a87562377f 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -24,14 +24,13 @@ namespace ton { namespace validator { -td::Ref ValidatorManagerOptions::create( - BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, bool allow_blockchain_init, - double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, - double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) { - return td::make_ref(zero_block_id, init_block_id, std::move(check_shard), - allow_blockchain_init, sync_blocks_before, block_ttl, state_ttl, - max_mempool_num, +td::Ref ValidatorManagerOptions::create(BlockIdExt zero_block_id, BlockIdExt init_block_id, + bool allow_blockchain_init, double sync_blocks_before, + double block_ttl, double state_ttl, + double max_mempool_num, double archive_ttl, + double key_proof_ttl, bool initial_sync_disabled) { + return td::make_ref(zero_block_id, init_block_id, allow_blockchain_init, + sync_blocks_before, block_ttl, state_ttl, max_mempool_num, archive_ttl, key_proof_ttl, initial_sync_disabled); } diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 55552635c0..1fba6091d0 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -34,7 +34,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { } bool need_monitor(ShardIdFull shard, const td::Ref& state) const override { td::uint32 min_split = state->monitor_min_split_depth(shard.workchain); - return check_shard_((td::uint32)shard.pfx_len() <= min_split ? shard : shard_prefix(shard, min_split)); + return check_shard_((td::uint32)shard.pfx_len() <= min_split ? shard : shard_prefix(shard, min_split), + state->get_seqno()); } bool allow_blockchain_init() const override { return allow_blockchain_init_; @@ -163,6 +164,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { double get_catchain_broadcast_speed_multiplier() const override { return catchain_broadcast_speed_multipliers_; } + bool get_permanent_celldb() const override { + return permanent_celldb_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -170,7 +174,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_init_block_id(BlockIdExt block_id) override { init_block_id_ = block_id; } - void set_shard_check_function(std::function check_shard) override { + void set_shard_check_function(std::function check_shard) override { check_shard_ = std::move(check_shard); } void set_allow_blockchain_init(bool value) override { @@ -267,18 +271,19 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_catchain_broadcast_speed_multiplier(double value) override { catchain_broadcast_speed_multipliers_ = value; } + void set_permanent_celldb(bool value) override { + permanent_celldb_ = value; + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); } - ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, bool allow_blockchain_init, + ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, bool allow_blockchain_init, double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) : zero_block_id_(zero_block_id) , init_block_id_(init_block_id) - , check_shard_(std::move(check_shard)) , allow_blockchain_init_(allow_blockchain_init) , sync_blocks_before_(sync_blocks_before) , block_ttl_(block_ttl) @@ -292,7 +297,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { private: BlockIdExt zero_block_id_; BlockIdExt init_block_id_; - std::function check_shard_; + std::function check_shard_ = [](ShardIdFull, BlockSeqno) { return true; }; bool allow_blockchain_init_; double sync_blocks_before_; double block_ttl_; @@ -323,6 +328,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { td::Ref collator_options_{true}; bool fast_state_serializer_enabled_ = false; double catchain_broadcast_speed_multipliers_; + bool permanent_celldb_ = false; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 943c6e9c52..afeb86cefa 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -120,10 +120,11 @@ struct ValidatorManagerOptions : public td::CntObject { virtual td::Ref get_collator_options() const = 0; virtual bool get_fast_state_serializer_enabled() const = 0; virtual double get_catchain_broadcast_speed_multiplier() const = 0; + virtual bool get_permanent_celldb() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; - virtual void set_shard_check_function(std::function check_shard) = 0; + virtual void set_shard_check_function(std::function check_shard) = 0; virtual void set_allow_blockchain_init(bool value) = 0; virtual void set_sync_blocks_before(double value) = 0; virtual void set_block_ttl(double value) = 0; @@ -155,11 +156,10 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_collator_options(td::Ref value) = 0; virtual void set_fast_state_serializer_enabled(bool value) = 0; virtual void set_catchain_broadcast_speed_multiplier(double value) = 0; + virtual void set_permanent_celldb(bool value) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, - - std::function check_shard = [](ShardIdFull) { return true; }, bool allow_blockchain_init = false, double sync_blocks_before = 3600, double block_ttl = 86400, double state_ttl = 86400, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650, double max_mempool_num = 999999, bool initial_sync_disabled = false); From 5a0f54df45f913557f03bc2bc52bad4667ac241a Mon Sep 17 00:00:00 2001 From: neodix42 Date: Fri, 20 Jun 2025 20:45:11 +0300 Subject: [PATCH 312/524] Improve linux portable binaries (#1693) * not always gh mac runners have nproc utility * deprecate Ubuntu 20.04 * fix linking issues of portable tonlibjson.dylib * bind libgsl and libblas into portable binaries --- .github/workflows/build-ton-linux-x86-64-appimage.yml | 3 +-- assembly/appimage/create-appimages.sh | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-ton-linux-x86-64-appimage.yml b/.github/workflows/build-ton-linux-x86-64-appimage.yml index 43d9892432..7ad873d22e 100644 --- a/.github/workflows/build-ton-linux-x86-64-appimage.yml +++ b/.github/workflows/build-ton-linux-x86-64-appimage.yml @@ -21,8 +21,7 @@ jobs: - name: Install system libraries run: | sudo apt update - sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache - sudo apt remove libgsl-dev + sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache libgsl-dev libblas-dev mkdir ~/.ccache 3pp - name: Install clang-16 diff --git a/assembly/appimage/create-appimages.sh b/assembly/appimage/create-appimages.sh index 2a8cd0ec6c..7bd68b6fbf 100644 --- a/assembly/appimage/create-appimages.sh +++ b/assembly/appimage/create-appimages.sh @@ -35,6 +35,8 @@ for file in ../artifacts/*; do /lib/$ARCH-linux-gnu/libmicrohttpd.so.12 \ /lib/$ARCH-linux-gnu/libreadline.so.8 \ /lib/$ARCH-linux-gnu/libstdc++.so.6 \ + /lib/$ARCH-linux-gnu/libgsl.so.27 \ + /lib/$ARCH-linux-gnu/libblas.so.3 \ $appName.AppDir/usr/lib/ chmod +x ./$appName.AppDir/usr/bin/$appName From 34cbbbb90fb9fab4348f0f8aa13a7e8364ab5136 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Fri, 20 Jun 2025 20:45:40 +0300 Subject: [PATCH 313/524] Add dht utils for troubleshooting complex deployments (#1685) * not always gh mac runners have nproc utility * deprecate Ubuntu 20.04 * fix linking issues of portable tonlibjson.dylib * add dht-ping-servers and dht-resolve utilities to artifacts * test gh builds * minor win fix * minor win fix --- Dockerfile | 4 +++- assembly/native/build-macos-portable.sh | 6 ++++-- assembly/native/build-macos-shared.sh | 19 +++++++++++++++++-- assembly/native/build-ubuntu-appimages.sh | 5 +++-- assembly/native/build-ubuntu-portable.sh | 5 +++-- assembly/native/build-ubuntu-shared.sh | 5 +++-- assembly/native/build-windows-2019.bat | 6 ++++-- assembly/native/build-windows.bat | 6 ++++-- 8 files changed, 41 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index f1b836bf90..7a53500db6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,7 @@ RUN mkdir build && \ cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DTON_USE_JEMALLOC=ON .. && \ ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console \ generate-random-id dht-server lite-client tolk rldp-http-proxy dht-server proxy-liteserver create-state \ - blockchain-explorer emulator tonlibjson http-proxy adnl-proxy + blockchain-explorer emulator tonlibjson http-proxy adnl-proxy dht-ping-servers dht-resolve FROM ubuntu:22.04 ARG DEBIAN_FRONTEND=noninteractive @@ -50,6 +50,8 @@ COPY --from=builder /ton/build/blockchain-explorer/blockchain-explorer /usr/loca COPY --from=builder /ton/build/crypto/create-state /usr/local/bin/ COPY --from=builder /ton/build/utils/proxy-liteserver /usr/local/bin/ COPY --from=builder /ton/build/dht-server/dht-server /usr/local/bin/ +COPY --from=builder /ton/build/dht/dht-ping-servers /usr/local/bin/ +COPY --from=builder /ton/build/dht/dht-resolve /usr/local/bin/ COPY --from=builder /ton/build/rldp-http-proxy/rldp-http-proxy /usr/local/bin/ COPY --from=builder /ton/build/http/http-proxy /usr/local/bin/ COPY --from=builder /ton/build/adnl/adnl-proxy /usr/local/bin/ diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index e112ab499e..351c4e7fb6 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -150,7 +150,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont \ test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ @@ -159,7 +159,7 @@ if [ "$with_tests" = true ]; then else ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi @@ -184,6 +184,8 @@ if [ "$with_artifacts" = true ]; then cp build/http/http-proxy artifacts/ cp build/rldp-http-proxy/rldp-http-proxy artifacts/ cp build/dht-server/dht-server artifacts/ + cp build/dht/dht-ping-servers artifacts/ + cp build/dht/dht-resolve artifacts/ cp build/lite-client/lite-client artifacts/ cp build/validator-engine/validator-engine artifacts/ cp build/utils/generate-random-id artifacts/ diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index b21f21f893..f5aa15b4ac 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -61,6 +61,19 @@ else echo "Using compiled lz4" fi +if [ ! -d "zlib" ]; then + git clone https://github.com/madler/zlib.git + cd zlib + zlibPath=`pwd` + ./configure --static + make -j4 + test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } + cd .. +else + zlibPath=$(pwd)/zlib + echo "Using compiled zlib" +fi + brew unlink openssl@1.1 brew install openssl@3 brew unlink openssl@3 && brew link --overwrite openssl@3 @@ -75,7 +88,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont \ test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ @@ -84,7 +97,7 @@ if [ "$with_tests" = true ]; then else ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi @@ -109,6 +122,8 @@ if [ "$with_artifacts" = true ]; then cp build/http/http-proxy artifacts/ cp build/rldp-http-proxy/rldp-http-proxy artifacts/ cp build/dht-server/dht-server artifacts/ + cp build/dht/dht-ping-servers artifacts/ + cp build/dht/dht-resolve artifacts/ cp build/lite-client/lite-client artifacts/ cp build/validator-engine/validator-engine artifacts/ cp build/utils/generate-random-id artifacts/ diff --git a/assembly/native/build-ubuntu-appimages.sh b/assembly/native/build-ubuntu-appimages.sh index 015b33cb78..2cdc6c6521 100644 --- a/assembly/native/build-ubuntu-appimages.sh +++ b/assembly/native/build-ubuntu-appimages.sh @@ -61,7 +61,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ - generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ + generate-random-id json2tlo dht-server http-proxy rldp-http-proxy dht-ping-servers dht-resolve \ adnl-proxy create-state emulator test-ed25519 test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ @@ -71,7 +71,7 @@ else ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator proxy-liteserver + adnl-proxy create-state emulator proxy-liteserver dht-ping-servers dht-resolve test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi @@ -104,6 +104,7 @@ if [ "$with_artifacts" = true ]; then build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \ build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \ build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \ + build/dht/dht-ping-servers build/dht/dht-resolve \ artifacts test $? -eq 0 || { echo "Can't copy final binaries"; exit 1; } cp -R crypto/smartcont artifacts diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh index 3a65ddd2bb..a4f0bb233a 100644 --- a/assembly/native/build-ubuntu-portable.sh +++ b/assembly/native/build-ubuntu-portable.sh @@ -135,7 +135,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ - generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ + generate-random-id json2tlo dht-server http-proxy rldp-http-proxy dht-ping-servers dht-resolve \ adnl-proxy create-state emulator test-ed25519 test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ @@ -145,7 +145,7 @@ else ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator proxy-liteserver + adnl-proxy create-state emulator proxy-liteserver dht-ping-servers dht-resolve test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi @@ -167,6 +167,7 @@ if [ "$with_artifacts" = true ]; then build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \ build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \ build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \ + build/dht/dht-ping-servers build/dht/dht-resolve \ artifacts test $? -eq 0 || { echo "Can't copy final binaries"; exit 1; } cp -R crypto/smartcont artifacts diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index 0bdf4072ba..6ddbaf8856 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -64,7 +64,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ - generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ + generate-random-id json2tlo dht-server http-proxy rldp-http-proxy dht-ping-servers dht-resolve \ adnl-proxy create-state emulator test-ed25519 test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ @@ -74,7 +74,7 @@ else ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator proxy-liteserver + adnl-proxy create-state emulator proxy-liteserver dht-ping-servers dht-resolve test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi @@ -99,6 +99,7 @@ if [ "$with_artifacts" = true ]; then build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \ build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \ build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \ + build/dht/dht-ping-servers build/dht/dht-resolve \ artifacts test $? -eq 0 || { echo "Can't copy final binaries"; exit 1; } cp -R crypto/smartcont artifacts diff --git a/assembly/native/build-windows-2019.bat b/assembly/native/build-windows-2019.bat index 2c635d3a63..e671d277c8 100644 --- a/assembly/native/build-windows-2019.bat +++ b/assembly/native/build-windows-2019.bat @@ -149,14 +149,14 @@ tonlib-cli validator-engine lite-client validator-engine-console generate-random json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^ test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ -test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver +test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver dht-ping-servers dht-resolve IF %errorlevel% NEQ 0 ( echo Can't compile TON exit /b %errorlevel% ) ) else ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ -tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id dht-ping-servers dht-resolve ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver IF %errorlevel% NEQ 0 ( echo Can't compile TON @@ -192,6 +192,8 @@ for %%I in (build\storage\storage-daemon\storage-daemon.exe ^ build\http\http-proxy.exe ^ build\rldp-http-proxy\rldp-http-proxy.exe ^ build\dht-server\dht-server.exe ^ + build\dht\dht-ping-servers.exe ^ + build\dht\dht-resolve.exe ^ build\lite-client\lite-client.exe ^ build\validator-engine\validator-engine.exe ^ build\utils\generate-random-id.exe ^ diff --git a/assembly/native/build-windows.bat b/assembly/native/build-windows.bat index 6ff7166af3..f0a65670b4 100644 --- a/assembly/native/build-windows.bat +++ b/assembly/native/build-windows.bat @@ -149,14 +149,14 @@ tonlib-cli validator-engine lite-client validator-engine-console generate-random json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^ test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ -test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver +test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver dht-ping-servers dht-resolve IF %errorlevel% NEQ 0 ( echo Can't compile TON exit /b %errorlevel% ) ) else ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ -tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id dht-ping-servers dht-resolve ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver IF %errorlevel% NEQ 0 ( echo Can't compile TON @@ -192,6 +192,8 @@ for %%I in (build\storage\storage-daemon\storage-daemon.exe ^ build\http\http-proxy.exe ^ build\rldp-http-proxy\rldp-http-proxy.exe ^ build\dht-server\dht-server.exe ^ + build\dht\dht-ping-servers.exe ^ + build\dht\dht-resolve.exe ^ build\lite-client\lite-client.exe ^ build\validator-engine\validator-engine.exe ^ build\utils\generate-random-id.exe ^ From 95acdb1a4a625a62d8d599fdd0388f49083e4724 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Fri, 20 Jun 2025 22:09:28 +0300 Subject: [PATCH 314/524] Fix structured binding in StateSerializer lambda --- validator/state-serializer.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index e65bad4c00..b024dc1009 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -578,7 +578,9 @@ void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref cell_db_reader, std::shared_ptr> parts, size_t idx) { - auto [type, cell] = parts->at(idx); + auto part = parts->at(idx); + auto type = part.type; + auto cell = part.cell; auto write_data = [=, this, cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { From dfe997ee3254782cc2f06a0d23f1fe3c03c18d98 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Wed, 25 Jun 2025 21:54:55 +0300 Subject: [PATCH 315/524] Storage stat cache (#1724) Co-authored-by: SpyCheese --- crypto/block/account-storage-stat.h | 4 ++ crypto/block/transaction.cpp | 38 ++++++++++++++ crypto/block/transaction.h | 1 + tdutils/td/utils/LRUCache.h | 24 +++++---- validator/CMakeLists.txt | 2 + validator/impl/collator-impl.h | 4 ++ validator/impl/collator.cpp | 46 ++++++++++++++++- validator/impl/validate-query.cpp | 49 ++++++++++++++++++ validator/impl/validate-query.hpp | 4 ++ validator/interfaces/validator-manager.h | 7 +++ validator/manager.cpp | 1 + validator/manager.hpp | 10 ++++ validator/storage-stat-cache.cpp | 40 +++++++++++++++ validator/storage-stat-cache.hpp | 63 ++++++++++++++++++++++++ 14 files changed, 282 insertions(+), 11 deletions(-) create mode 100644 validator/storage-stat-cache.cpp create mode 100644 validator/storage-stat-cache.hpp diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h index e939a0ff51..4ada076b88 100644 --- a/crypto/block/account-storage-stat.h +++ b/crypto/block/account-storage-stat.h @@ -57,6 +57,10 @@ class AccountStorageStat { return root.is_null() ? td::Bits256::zero() : td::Bits256{root->get_hash().bits()}; } + bool is_dict_ready() const { + return dict_up_to_date_; + } + void apply_child_stat(AccountStorageStat &&child); static constexpr int errorcode_limits_exceeded = 999; diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 8a72f8e48e..7cc0813cdc 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -561,6 +561,44 @@ bool Account::init_new(ton::UnixTime now) { return true; } +/** + * Initializes account_storage_stat of the account using the existing dict_root. + * This is not strictly necessary, as the storage stat is recomputed in Transaction. + * However, it can be used to optimize cell usage. + * This requires storage_dict_hash to be set, as it guarantees that the stored storage_used was computed recently + * (in older versions it included extra currency balance, in newer versions it does not). + * + * @param dict_root Root of the storage dictionary. + * + * @returns Status of the operation. + */ +td::Status Account::init_account_storage_stat(Ref dict_root) { + if (storage.is_null()) { + if (dict_root.not_null()) { + return td::Status::Error("storage is null, but dict_root is not null"); + } + account_storage_stat = {}; + return td::Status::OK(); + } + if (!storage_dict_hash) { + return td::Status::Error("cannot init storage dict: storage_dict_hash is not set"); + } + // Root of AccountStorage is not counted in AccountStorageStat + if (storage_used.cells < 1 || storage_used.bits < storage->size()) { + return td::Status::Error(PSTRING() << "storage_used is too small: cells=" << storage_used.cells + << " bits=" << storage_used.bits << " storage_root_bits=" << storage->size()); + } + AccountStorageStat new_stat(std::move(dict_root), storage->prefetch_all_refs(), storage_used.cells - 1, + storage_used.bits - storage->size()); + TRY_RESULT(root_hash, new_stat.get_dict_hash()); + if (storage_dict_hash.value() != root_hash) { + return td::Status::Error(PSTRING() << "invalid storage dict hash: computed " << root_hash.to_hex() << ", found " + << storage_dict_hash.value().to_hex()); + } + account_storage_stat = std::move(new_stat); + return td::Status::OK(); +} + /** * Resets the fixed prefix length of the account. * diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 0b3a4358d7..2f26ea3fb0 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -298,6 +298,7 @@ struct Account { bool set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr); bool unpack(Ref account, ton::UnixTime now, bool special); bool init_new(ton::UnixTime now); + td::Status init_account_storage_stat(Ref dict_root); bool deactivate(); bool recompute_tmp_addr(Ref& tmp_addr, int fixed_prefix_length, td::ConstBitPtr orig_addr_rewrite) const; td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector& pricing) const; diff --git a/tdutils/td/utils/LRUCache.h b/tdutils/td/utils/LRUCache.h index 5c30f66938..d8f51525ba 100644 --- a/tdutils/td/utils/LRUCache.h +++ b/tdutils/td/utils/LRUCache.h @@ -26,8 +26,7 @@ namespace td { template class LRUCache { public: - explicit LRUCache(size_t max_size) : max_size_(max_size) { - CHECK(max_size_ > 0); + explicit LRUCache(uint64 max_size) : max_size_(max_size) { } LRUCache(const LRUCache&) = delete; LRUCache& operator=(const LRUCache&) = delete; @@ -49,13 +48,14 @@ class LRUCache { return cache_.contains(key); } - bool put(const K& key, V value, bool update = true) { + bool put(const K& key, V value, bool update = true, uint64 weight = 1) { bool added = false; auto it = cache_.find(key); if (it == cache_.end()) { update = true; - it = cache_.insert(std::make_unique(key, std::move(value))).first; + it = cache_.insert(std::make_unique(key, std::move(value), weight)).first; added = true; + total_weight_ += weight; } else { (*it)->value = std::move(value); if (update) { @@ -69,11 +69,12 @@ class LRUCache { return added; } - V& get(const K& key, bool update = true) { + V& get(const K& key, bool update = true, uint64 weight = 1) { auto it = cache_.find(key); if (it == cache_.end()) { update = true; - it = cache_.insert(std::make_unique(key)).first; + it = cache_.insert(std::make_unique(key, weight)).first; + total_weight_ += weight; } else if (update) { (*it)->remove(); } @@ -87,12 +88,13 @@ class LRUCache { private: struct Entry : ListNode { - explicit Entry(K key) : key(std::move(key)) { + Entry(K key, uint64 weight) : key(std::move(key)), weight(weight) { } - Entry(K key, V value) : key(std::move(key)), value(std::move(value)) { + Entry(K key, V value, uint64 weight) : key(std::move(key)), value(std::move(value)), weight(weight) { } K key; V value; + uint64 weight; }; struct Cmp { using is_transparent = void; @@ -108,13 +110,15 @@ class LRUCache { }; std::set, Cmp> cache_; ListNode lru_; - size_t max_size_; + uint64 max_size_; + uint64 total_weight_ = 0; void cleanup() { - while (cache_.size() > max_size_) { + while (total_weight_ > max_size_ && cache_.size() > 1) { auto to_remove = (Entry*)lru_.get(); CHECK(to_remove); to_remove->remove(); + total_weight_ -= to_remove->weight; cache_.erase(cache_.find(to_remove->key)); } } diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 9fb0d190fa..78909f34fe 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -59,6 +59,7 @@ set(VALIDATOR_HEADERS import-db-slice-local.hpp queue-size-counter.hpp validator-telemetry.hpp + storage-stat-cache.hpp manager-disk.h manager-disk.hpp @@ -87,6 +88,7 @@ set(VALIDATOR_SOURCE validator-options.cpp queue-size-counter.cpp validator-telemetry.cpp + storage-stat-cache.cpp downloaders/wait-block-data.cpp downloaders/wait-block-state.cpp diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 340e3a4010..8b113c74e0 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -227,6 +227,9 @@ class Collator final : public td::actor::Actor { bool deferring_messages_enabled_ = false; bool store_out_msg_queue_size_ = false; + std::function(const td::Bits256&)> storage_stat_cache_; + std::vector, td::uint32>> storage_stat_cache_update_; + td::PerfWarningTimer perf_timer_; // block::Account* lookup_account(td::ConstBitPtr addr) const; @@ -247,6 +250,7 @@ class Collator final : public td::actor::Actor { void after_get_shard_state(int idx, td::Result> res); void after_get_block_data(int idx, td::Result> res); void after_get_shard_blocks(td::Result>> res); + void after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res); bool preprocess_prev_mc_state(); bool register_mc_state(Ref other_mc_state); bool request_aux_mc_state(BlockSeqno seqno, Ref& state); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index cc9ee709c1..614e0ecdbe 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -277,7 +277,16 @@ void Collator::start_up() { td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_blocks, std::move(res)); }); } - // 6. set timeout + // 6. get storage stat cache + ++pending; + LOG(DEBUG) << "sending get_storage_stat_cache() query to Manager"; + td::actor::send_closure_later( + manager, &ValidatorManager::get_storage_stat_cache, + [self = get_self()](td::Result(const td::Bits256&)>> res) { + LOG(DEBUG) << "got answer to get_storage_stat_cache() query"; + td::actor::send_closure_later(std::move(self), &Collator::after_get_storage_stat_cache, std::move(res)); + }); + // 7. set timeout alarm_timestamp() = timeout; CHECK(pending); } @@ -686,6 +695,22 @@ void Collator::after_get_shard_blocks(td::Result(const td::Bits256&)>> res) { + --pending; + if (res.is_error()) { + LOG(INFO) << "after_get_storage_stat_cache : " << res.error(); + } else { + LOG(DEBUG) << "after_get_storage_stat_cache : OK"; + storage_stat_cache_ = res.move_as_ok(); + } + check_pending(); +} + /** * Unpacks the last masterchain state and initializes the Collator object with the extracted configuration. * @@ -2448,6 +2473,20 @@ std::unique_ptr Collator::make_account_from(td::ConstBitPtr addr return nullptr; } ptr->block_lt = start_lt; + if (storage_stat_cache_ && ptr->storage_dict_hash) { + auto dict_root = storage_stat_cache_(ptr->storage_dict_hash.value()); + if (dict_root.not_null()) { + auto S = ptr->init_account_storage_stat(dict_root); + if (S.is_error()) { + fatal_error(S.move_as_error_prefix(PSTRING() << "failed to init storage stat from cache for account " + << addr.to_hex(256) << ": ")); + return nullptr; + } + LOG(DEBUG) << "Inited storage stat from cache for account " << addr.to_hex(256) << " (" << ptr->storage_used.cells + << " cells)"; + storage_stat_cache_update_.emplace_back(dict_root, ptr->storage_used.cells); + } + } return ptr; } @@ -2589,6 +2628,10 @@ bool Collator::combine_account_transactions() { } } } + if (acc.storage_dict_hash && acc.account_storage_stat && acc.account_storage_stat.value().is_dict_ready()) { + storage_stat_cache_update_.emplace_back(acc.account_storage_stat.value().get_dict_root().move_as_ok(), + acc.storage_used.cells); + } } else { if (acc.total_state->get_hash() != acc.orig_total_state->get_hash()) { return fatal_error(std::string{"total state of account "} + z.first.to_hex() + @@ -5685,6 +5728,7 @@ bool Collator::create_block_candidate() { stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, block_candidate->id, work_time, cpu_work_time, std::move(stats_)); + td::actor::send_closure(manager, &ValidatorManager::update_storage_stat_cache, std::move(storage_stat_cache_update_)); return true; } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 8c974b431b..a191b02001 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -381,6 +381,15 @@ void ValidateQuery::start_up() { return; } } + // 5. get storage stat cache + ++pending; + LOG(DEBUG) << "sending get_storage_stat_cache() query to Manager"; + td::actor::send_closure_later( + manager, &ValidatorManager::get_storage_stat_cache, + [self = get_self()](td::Result(const td::Bits256&)>> res) { + LOG(DEBUG) << "got answer to get_storage_stat_cache() query"; + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_storage_stat_cache, std::move(res)); + }); // ... CHECK(pending); } @@ -749,6 +758,26 @@ void ValidateQuery::got_mc_handle(td::Result res) { }); } +/** + * Callback function called after retrieving storage stat cache. + * + * @param res The retrieved storage stat cache. + */ +void ValidateQuery::after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res) { + --pending; + if (res.is_error()) { + LOG(INFO) << "after_get_storage_stat_cache : " << res.error(); + } else { + LOG(DEBUG) << "after_get_storage_stat_cache : OK"; + storage_stat_cache_ = res.move_as_ok(); + } + if (!pending) { + if (!try_validate()) { + fatal_error("cannot validate new block"); + } + } +} + /** * Callback function called after retrieving the shard state for a previous block. * @@ -5133,6 +5162,20 @@ std::unique_ptr ValidateQuery::make_account_from(td::ConstBitPtr return nullptr; } ptr->block_lt = start_lt_; + if (storage_stat_cache_ && ptr->storage_dict_hash) { + auto dict_root = storage_stat_cache_(ptr->storage_dict_hash.value()); + if (dict_root.not_null()) { + auto S = ptr->init_account_storage_stat(dict_root); + if (S.is_error()) { + fatal_error(S.move_as_error_prefix(PSTRING() << "failed to init storage stat from cache for account " + << addr.to_hex(256) << ": ")); + return nullptr; + } + LOG(DEBUG) << "Inited storage stat from cache for account " << addr.to_hex(256) << " (" << ptr->storage_used.cells + << " cells)"; + storage_stat_cache_update_.emplace_back(dict_root, ptr->storage_used.cells); + } + } return ptr; } @@ -5746,6 +5789,11 @@ bool ValidateQuery::check_account_transactions(const StdSmcAddress& acc_addr, Re })) { return reject_query("at least one Transaction of account "s + acc_addr.to_hex() + " is invalid"); } + if (account.storage_dict_hash && account.account_storage_stat && + account.account_storage_stat.value().is_dict_ready()) { + storage_stat_cache_update_.emplace_back(account.account_storage_stat.value().get_dict_root().move_as_ok(), + account.storage_used.cells); + } if (is_masterchain() && account.libraries_changed()) { return scan_account_libraries(account.orig_library, account.library, acc_addr); } else { @@ -6923,6 +6971,7 @@ bool ValidateQuery::save_candidate() { td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, id_, block_candidate.clone(), validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), std::move(P)); + td::actor::send_closure(manager, &ValidatorManager::update_storage_stat_cache, std::move(storage_stat_cache_update_)); return true; } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 60f0cc8a49..14aee1a8ec 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -236,6 +236,9 @@ class ValidateQuery : public td::actor::Actor { std::set account_expected_defer_all_messages_; td::uint64 old_out_msg_queue_size_ = 0, new_out_msg_queue_size_ = 0; + std::function(const td::Bits256&)> storage_stat_cache_; + std::vector, td::uint32>> storage_stat_cache_update_; + bool msg_metadata_enabled_ = false; bool deferring_messages_enabled_ = false; bool store_out_msg_queue_size_ = false; @@ -289,6 +292,7 @@ class ValidateQuery : public td::actor::Actor { void after_get_latest_mc_state(td::Result, BlockIdExt>> res); void after_get_mc_state(td::Result> res); void got_mc_handle(td::Result res); + void after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res); void after_get_shard_state(int idx, td::Result> res); bool process_mc_state(Ref mc_state); bool try_unpack_mc_state(); diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index abab4e6c97..44c74f07a4 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -221,6 +221,13 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void add_persistent_state_description(td::Ref desc) = 0; + virtual void get_storage_stat_cache(td::Promise(const td::Bits256&)>> promise) { + promise.set_error(td::Status::Error("not implemented")); + } + virtual void update_storage_stat_cache(std::vector, td::uint32>> data) { + // not implemented + } + static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 7a2d63b7d8..5acb2928ab 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1764,6 +1764,7 @@ void ValidatorManagerImpl::start_up() { actor_stats_ = td::actor::create_actor("actor_stats"); lite_server_cache_ = create_liteserver_cache_actor(actor_id(this), db_root_); token_manager_ = td::actor::create_actor("tokenmanager"); + storage_stat_cache_ = td::actor::create_actor("storagestatcache"); td::mkdir(db_root_ + "/tmp/").ensure(); td::mkdir(db_root_ + "/catchains/").ensure(); diff --git a/validator/manager.hpp b/validator/manager.hpp index 59ee1ff63e..4ea0d24f47 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -34,6 +34,7 @@ #include "rldp/rldp.h" #include "token-manager.h" #include "queue-size-counter.hpp" +#include "storage-stat-cache.hpp" #include "validator-telemetry.hpp" #include "impl/candidates-buffer.hpp" #include "td/utils/LRUCache.h" @@ -648,6 +649,13 @@ class ValidatorManagerImpl : public ValidatorManager { ++(success ? total_ls_queries_ok_ : total_ls_queries_error_)[lite_query_id]; } + void get_storage_stat_cache(td::Promise(const td::Bits256&)>> promise) override { + td::actor::send_closure(storage_stat_cache_, &StorageStatCache::get_cache, std::move(promise)); + } + void update_storage_stat_cache(std::vector, td::uint32>> data) override { + td::actor::send_closure(storage_stat_cache_, &StorageStatCache::update, std::move(data)); + } + private: td::Timestamp resend_shard_blocks_at_; td::Timestamp check_waiters_at_; @@ -773,6 +781,8 @@ class ValidatorManagerImpl : public ValidatorManager { std::map>>)>>> stats_providers_; + + td::actor::ActorOwn storage_stat_cache_; }; } // namespace validator diff --git a/validator/storage-stat-cache.cpp b/validator/storage-stat-cache.cpp new file mode 100644 index 0000000000..6a8049b0dc --- /dev/null +++ b/validator/storage-stat-cache.cpp @@ -0,0 +1,40 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "storage-stat-cache.hpp" + +namespace ton::validator { + +void StorageStatCache::get_cache(td::Promise(const td::Bits256&)>> promise) { + LOG(DEBUG) << "StorageStatCache::get_cache"; + promise.set_value( + [cache = cache_](const td::Bits256& hash) mutable -> td::Ref { return cache.lookup_ref(hash); }); +} + +void StorageStatCache::update(std::vector, td::uint32>> data) { + for (auto &[cell, size] : data) { + if (size < 4000) { + continue; + } + td::Bits256 hash = cell->get_hash().bits(); + LOG(DEBUG) << "StorageStatCache::update " << hash.to_hex() << " " << size; + cache_.set_ref(hash, cell); + lru_.put(hash, Deleter{hash, &cache_}, true, size); + } +} + +} // namespace ton::validator diff --git a/validator/storage-stat-cache.hpp b/validator/storage-stat-cache.hpp new file mode 100644 index 0000000000..d32611fcd7 --- /dev/null +++ b/validator/storage-stat-cache.hpp @@ -0,0 +1,63 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "interfaces/validator-manager.h" +#include "td/utils/ConcurrentHashTable.h" +#include "td/utils/LRUCache.h" + +#include + +namespace ton::validator { + +class StorageStatCache : public td::actor::Actor { + public: + void get_cache(td::Promise(const td::Bits256&)>> promise); + + // (storage dict root, account total cells) + void update(std::vector, td::uint32>> data); + + private: + vm::Dictionary cache_{256}; + + struct Deleter { + Deleter(const td::Bits256& hash, vm::Dictionary* cache) : hash(hash), cache(cache) { + } + Deleter(const Deleter&) = delete; + Deleter(Deleter&& other) noexcept : hash(other.hash), cache(other.cache) { + other.cache = nullptr; + } + Deleter& operator=(const Deleter&) = delete; + Deleter& operator=(Deleter&& other) noexcept { + hash = other.hash; + cache = other.cache; + other.cache = nullptr; + return *this; + } + ~Deleter() { + if (cache) { + CHECK(cache->lookup_delete_ref(hash).not_null()); + LOG(DEBUG) << "StorageStatCache remove " << hash.to_hex(); + } + } + + td::Bits256 hash = td::Bits256::zero(); + vm::Dictionary* cache; + }; + td::LRUCache lru_{1 << 24}; +}; + +} // namespace ton::validator From 283186bf1453e00f4b206b3916240bc5b6c1d35d Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Thu, 26 Jun 2025 11:58:53 +0300 Subject: [PATCH 316/524] Update changelog --- Changelog.md | 14 +++++++++++++- recent_changelog.md | 17 ++++++++--------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/Changelog.md b/Changelog.md index 344b26cd3e..d2f940c00c 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,15 @@ +## 2025.06 Update + +1. ADNL and candidate broadcast optimization +2. [TVM version v11](./doc/GlobalVersions.md): new opcodes, and `c7` entry to improve developer experience. It also activates storage stats and `ihr_fee` nullification. +3. Fixed `start_lt` of tick transactions [see details on 01.06.2025 incident](https://telegra.ph/Report-on-June-1-2025-Operation-Incident-06-02). +4. Introduction of persistent state sharding, as well as making serialization of large BOCs more deterministic +5. Emulator improvements: in get methods, set config from provided `c7`; allow retrieval of logs from emulator runs for get methods +6. Optimized package import for archive nodes + +Besides the work of the core team, this update is based on the efforts of the RSquad team (deterministic large BOC serialization); AArayz, wy666444, Robinlzw, Lucian-code233 from TonBit (early discovery of the TVM 11 bug); @Skydev0h (uninitialized `BLOCKLT` in get methods); and @yma-het from TONWhales (emulator improvements). + + ## 2025.04 Update 1. Introduced substantial improvements of CellDB performance: celldb-v2, bloom filters. @@ -8,7 +20,7 @@ 6. [Added normalized hash](https://github.com/ton-blockchain/TEPs/pull/467) 7. Fix SDBEGINS(Q) in Asm.fif -Besides the work of the core team, this update is based on the efforts of @Stanislav-Povolotsky (tonlib fixes), @ice-charon (tonlib fixes), RSquad team (due payments improvements in v10), Arayz @ TonBit (improvements in RUNVM), @Skydev0h and @pyAndr3w (Asm.fif). +Besides the work of the core team, this update is based on the efforts of @Stanislav-Povolotsky (tonlib fixes); @ice-charon (tonlib fixes); RSquad team (due payments improvements in v10); Arayz, Robinlzw, @wy666444 @Lucian-code233 from TonBit (improvements in RUNVM); @Skydev0h and @pyAndr3w (Asm.fif). ## 2025.03 Update 1. New extracurrency behavior introduced, check [GlobalVersions.md](./doc/GlobalVersions.md#version-10) diff --git a/recent_changelog.md b/recent_changelog.md index 5589603184..a178f38fa6 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,12 +1,11 @@ -## 2025.04 Update +## 2025.06 Update -1. Introduced substantial improvements of CellDB performance: celldb-v2, bloom filters. -2. Accelerated a number of intrinsic node operations: SHA256, cell operations, large boc serialization, validator set checks. -3. [TVM version v10](./doc/GlobalVersions.md) -4. Overlay broadcast speed up and improved network stats. -5. Fixed some issues in tonlib -6. [Added normalized hash](https://github.com/ton-blockchain/TEPs/pull/467) -7. Fix SDBEGINS(Q) in Asm.fif +1. ADNL and candidate broadcast optimization +2. [TVM version v11](./doc/GlobalVersions.md): new opcodes, and `c7` entry to improve developer experience. It also activates storage stats and `ihr_fee` nullification. +3. Fixed `start_lt` of tick transactions [see details on 01.06.2025 incident](https://telegra.ph/Report-on-June-1-2025-Operation-Incident-06-02). +4. Introduction of persistent state sharding, as well as making serialization of large BOCs more deterministic +5. Emulator improvements: in get methods, set config from provided `c7`; allow retrieval of logs from emulator runs for get methods +6. Optimized package import for archive nodes -Besides the work of the core team, this update is based on the efforts of @Stanislav-Povolotsky (tonlib fixes), @ice-charon (tonlib fixes), RSquad team (due payments improvements in v10), Arayz @ TonBit (improvements in RUNVM), @Skydev0h and @pyAndr3w (Asm.fif). +Besides the work of the core team, this update is based on the efforts of the RSquad team (deterministic large BOC serialization); AArayz, wy666444, Robinlzw, Lucian-code233 from TonBit (early discovery of the TVM 11 bug); @Skydev0h (uninitialized `BLOCKLT` in get methods); and @yma-het from TONWhales (emulator improvements). From 75a75eb1f9283a958f6760f8b717f2eb49774a88 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Thu, 26 Jun 2025 16:38:30 +0400 Subject: [PATCH 317/524] use zero-autotools approach when compiling libsodium on mac (#1725) * Update build-macos-portable.sh update macro names * Update build-macos-portable.sh Make config.sub executable * Update build-macos-portable.sh Let's use zero-autotools approach when compiling libsodium on mac * Update build-macos-portable.sh refactor libsodium directory --- assembly/native/build-macos-portable.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index 351c4e7fb6..feb5bf72f2 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -65,19 +65,19 @@ else echo "Using compiled lz4" fi -if [ ! -d "../3pp/libsodium" ]; then +if [ ! -d "../3pp/libsodium-1.0.18" ]; then export LIBSODIUM_FULL_BUILD=1 - git clone https://github.com/jedisct1/libsodium.git ../3pp/libsodium - cd ../3pp/libsodium + cd ../3pp + curl -LO https://download.libsodium.org/libsodium/releases/libsodium-1.0.18.tar.gz + tar -xzf libsodium-1.0.18.tar.gz + cd libsodium-1.0.18 sodiumPath=`pwd` - git checkout 1.0.18 - ./autogen.sh ./configure --with-pic --enable-static make -j4 test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } cd ../../build else - sodiumPath=$(pwd)/../3pp/libsodium + sodiumPath=$(pwd)/../3pp/libsodium-1.0.18 echo "Using compiled libsodium" fi From 2bfe25a9ba20f1bff5d61a0f98e6e2a594487635 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 27 Jun 2025 12:00:07 +0300 Subject: [PATCH 318/524] Increase timeouts in ArchiveImporterLocal --- validator/import-db-slice-local.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validator/import-db-slice-local.cpp b/validator/import-db-slice-local.cpp index 8c8db4ffb1..945f41a6b5 100644 --- a/validator/import-db-slice-local.cpp +++ b/validator/import-db-slice-local.cpp @@ -207,7 +207,7 @@ void ArchiveImporterLocal::import_first_key_block() { td::actor::send_closure(SelfId, &ArchiveImporterLocal::checked_key_block_proof, std::move(handle)); }); - run_check_proof_query(block_id, first_block.proof, manager_, td::Timestamp::in(10.0), std::move(P), + run_check_proof_query(block_id, first_block.proof, manager_, td::Timestamp::in(600.0), std::move(P), last_masterchain_state_, opts_->is_hardfork(block_id)); } @@ -278,7 +278,7 @@ void ArchiveImporterLocal::process_masterchain_blocks_cont() { } promise.set_result(td::Unit()); }); - run_check_proof_query(block_id, info.proof, manager_, td::Timestamp::in(10.0), std::move(P), + run_check_proof_query(block_id, info.proof, manager_, td::Timestamp::in(600.0), std::move(P), last_masterchain_state_, opts_->is_hardfork(block_id)); prev_block_id = block_id; } @@ -470,7 +470,7 @@ void ArchiveImporterLocal::store_data() { std::move(promise)); }); if (info.proof_link.not_null()) { - run_check_proof_link_query(block_id, info.proof_link, manager_, td::Timestamp::in(60.0), + run_check_proof_link_query(block_id, info.proof_link, manager_, td::Timestamp::in(600.0), ig.get_promise().wrap([](BlockHandle &&) { return td::Unit(); })); } } From aff4956078a0342f400bd4ab4bbab4a432ab9c49 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 1 Jul 2025 17:15:43 +0300 Subject: [PATCH 319/524] Storage stat cache for accelerator --- crypto/block/account-storage-stat.cpp | 6 +- crypto/block/account-storage-stat.h | 3 +- crypto/block/transaction.cpp | 6 +- crypto/block/transaction.h | 2 +- crypto/vm/cells/CellUsageTree.cpp | 4 +- crypto/vm/cells/CellUsageTree.h | 4 +- crypto/vm/cells/MerkleProof.h | 3 + validator/impl/collator-impl.h | 6 +- validator/impl/collator.cpp | 198 +++++++++++++++++++------- validator/impl/validate-query.cpp | 59 ++++---- validator/storage-stat-cache.cpp | 3 +- validator/storage-stat-cache.hpp | 7 +- 12 files changed, 206 insertions(+), 95 deletions(-) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index e03341d4e6..a6159c2245 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -36,8 +36,7 @@ AccountStorageStat::AccountStorageStat(const AccountStorageStat* parent) CHECK(parent_->parent_ == nullptr); } -td::Status AccountStorageStat::replace_roots(std::vector> new_roots, bool check_merkle_depth, - td::HashSet* store_added) { +td::Status AccountStorageStat::replace_roots(std::vector> new_roots, bool check_merkle_depth) { std::erase_if(new_roots, [](const Ref& c) { return c.is_null(); }); if (new_roots.empty()) { roots_.clear(); @@ -74,9 +73,6 @@ td::Status AccountStorageStat::replace_roots(std::vector> new_root roots_ = std::move(new_roots); dict_up_to_date_ = false; for (auto& [_, e] : cache_) { - if (store_added && !e.exists && e.refcnt_diff > 0) { - store_added->insert(e.hash); - } TRY_STATUS(finalize_entry(e)); } return td::Status::OK(); diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h index 12e922af11..4ada076b88 100644 --- a/crypto/block/account-storage-stat.h +++ b/crypto/block/account-storage-stat.h @@ -39,8 +39,7 @@ class AccountStorageStat { AccountStorageStat &operator=(const AccountStorageStat &other) = delete; AccountStorageStat &operator=(AccountStorageStat &&other) = default; - td::Status replace_roots(std::vector> new_roots, bool check_merkle_depth = false, - td::HashSet *store_added = nullptr); + td::Status replace_roots(std::vector> new_roots, bool check_merkle_depth = false); void add_hint(const td::HashSet &visited); td::uint64 get_total_cells() const { diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 8e98f052e9..ca34e14af8 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -602,11 +602,9 @@ static td::Ref storage_without_extra_currencies(td::Ref> Account::compute_account_storage_dict(td::HashSet* storage_cells) const { +td::Result> Account::compute_account_storage_dict() const { if (storage.is_null()) { return td::Status::Error("cannot compute storage dict: empty storage"); } @@ -618,7 +616,7 @@ td::Result> Account::compute_account_storage_dict(td::HashSetprefetch_all_refs(), false, storage_cells)); + TRY_STATUS(stat.replace_roots(storage_for_stat->prefetch_all_refs())); // Root of AccountStorage is not counted in AccountStorageStat td::uint64 expected_cells = stat.get_total_cells() + 1; td::uint64 expected_bits = stat.get_total_bits() + storage->size(); diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 528df4a003..1a7af2d125 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -301,7 +301,7 @@ struct Account { bool set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr); bool unpack(Ref account, ton::UnixTime now, bool special); bool init_new(ton::UnixTime now); - td::Result> compute_account_storage_dict(td::HashSet* storage_cells = nullptr) const; + td::Result> compute_account_storage_dict() const; td::Status init_account_storage_stat(Ref dict_root); bool deactivate(); bool recompute_tmp_addr(Ref& tmp_addr, int fixed_prefix_length, td::ConstBitPtr orig_addr_rewrite) const; diff --git a/crypto/vm/cells/CellUsageTree.cpp b/crypto/vm/cells/CellUsageTree.cpp index 301ee7da49..05a4f0c452 100644 --- a/crypto/vm/cells/CellUsageTree.cpp +++ b/crypto/vm/cells/CellUsageTree.cpp @@ -99,11 +99,11 @@ void CellUsageTree::mark_path(NodeId node_id) { } } -CellUsageTree::NodeId CellUsageTree::get_parent(NodeId node_id) { +CellUsageTree::NodeId CellUsageTree::get_parent(NodeId node_id) const { return nodes_[node_id].parent; } -CellUsageTree::NodeId CellUsageTree::get_child(NodeId node_id, unsigned ref_id) { +CellUsageTree::NodeId CellUsageTree::get_child(NodeId node_id, unsigned ref_id) const { DCHECK(ref_id < CellTraits::max_refs); return nodes_[node_id].children[ref_id]; } diff --git a/crypto/vm/cells/CellUsageTree.h b/crypto/vm/cells/CellUsageTree.h index 902b98d751..cd2470ddae 100644 --- a/crypto/vm/cells/CellUsageTree.h +++ b/crypto/vm/cells/CellUsageTree.h @@ -59,8 +59,8 @@ class CellUsageTree : public std::enable_shared_from_this { bool has_mark(NodeId node_id) const; void set_mark(NodeId node_id, bool mark = true); void mark_path(NodeId node_id); - NodeId get_parent(NodeId node_id); - NodeId get_child(NodeId node_id, unsigned ref_id); + NodeId get_parent(NodeId node_id) const; + NodeId get_child(NodeId node_id, unsigned ref_id) const; void set_use_mark_for_is_loaded(bool use_mark = true); NodeId create_child(NodeId node_id, unsigned ref_id); diff --git a/crypto/vm/cells/MerkleProof.h b/crypto/vm/cells/MerkleProof.h index f63683a59e..68f0be7b97 100644 --- a/crypto/vm/cells/MerkleProof.h +++ b/crypto/vm/cells/MerkleProof.h @@ -74,6 +74,9 @@ class MerkleProofBuilder { void set_cell_load_callback(std::function f) { usage_tree->set_cell_load_callback(std::move(f)); } + const CellUsageTree &get_usage_tree() const { + return *usage_tree; + } }; } // namespace vm diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 91f3cd81c0..fcc95caade 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -227,7 +227,6 @@ class Collator final : public td::actor::Actor { vm::ProofStorageStat proof_stat; bool add_to_collated_data = false; std::vector> storage_stat_updates; - td::HashSet original_storage_cells; }; std::map account_storage_dicts_; @@ -275,7 +274,8 @@ class Collator final : public td::actor::Actor { void after_get_shard_state(int idx, td::Result> res, td::PerfLogAction token); void after_get_block_data(int idx, td::Result> res, td::PerfLogAction token); void after_get_shard_blocks(td::Result>> res, td::PerfLogAction token); - void after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res); + void after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res, + td::PerfLogAction token); bool preprocess_prev_mc_state(); bool register_mc_state(Ref other_mc_state); bool request_aux_mc_state(BlockSeqno seqno, Ref& state); @@ -360,7 +360,7 @@ class Collator final : public td::actor::Actor { bool update_account_dict_estimation(const block::transaction::Transaction& trans); void update_account_storage_dict_info(const block::transaction::Transaction& trans); bool update_min_mc_seqno(ton::BlockSeqno some_mc_seqno); - bool process_account_storage_dict(const block::Account& account); + bool process_account_storage_dict(block::Account& account); bool combine_account_transactions(); bool update_public_libraries(); bool update_account_public_libraries(Ref orig_libs, Ref final_libs, const td::Bits256& addr); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index f693d95547..acf9075b4a 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -32,6 +32,7 @@ #include #include #include "fabric.h" +#include "storage-stat-cache.hpp" #include "validator-set.hpp" #include "top-shard-descr.hpp" #include @@ -298,12 +299,14 @@ void Collator::start_up() { // 6. get storage stat cache ++pending; LOG(DEBUG) << "sending get_storage_stat_cache() query to Manager"; - td::actor::send_closure_later( - manager, &ValidatorManager::get_storage_stat_cache, - [self = get_self()](td::Result(const td::Bits256&)>> res) { - LOG(DEBUG) << "got answer to get_storage_stat_cache() query"; - td::actor::send_closure_later(std::move(self), &Collator::after_get_storage_stat_cache, std::move(res)); - }); + td::actor::send_closure_later(manager, &ValidatorManager::get_storage_stat_cache, + [self = get_self(), token = perf_log_.start_action("get_storage_stat_cache")]( + td::Result(const td::Bits256&)>> res) mutable { + LOG(DEBUG) << "got answer to get_storage_stat_cache() query"; + td::actor::send_closure_later(std::move(self), + &Collator::after_get_storage_stat_cache, std::move(res), + std::move(token)); + }); // 7. set timeout alarm_timestamp() = timeout; CHECK(pending); @@ -741,8 +744,10 @@ void Collator::after_get_shard_blocks(td::Result(const td::Bits256&)>> res) { +void Collator::after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res, + td::PerfLogAction token) { --pending; + token.finish(res); if (res.is_error()) { LOG(INFO) << "after_get_storage_stat_cache : " << res.error(); } else { @@ -2655,20 +2660,6 @@ std::unique_ptr Collator::make_account_from(td::ConstBitPtr addr if (!init_account_storage_dict(*ptr)) { return nullptr; } - if (storage_stat_cache_ && ptr->storage_dict_hash) { - auto dict_root = storage_stat_cache_(ptr->storage_dict_hash.value()); - if (dict_root.not_null()) { - auto S = ptr->init_account_storage_stat(dict_root); - if (S.is_error()) { - fatal_error(S.move_as_error_prefix(PSTRING() << "failed to init storage stat from cache for account " - << addr.to_hex(256) << ": ")); - return nullptr; - } - LOG(DEBUG) << "Inited storage stat from cache for account " << addr.to_hex(256) << " (" << ptr->storage_used.cells - << " cells)"; - storage_stat_cache_update_.emplace_back(dict_root, ptr->storage_used.cells); - } - } return ptr; } @@ -2679,29 +2670,54 @@ std::unique_ptr Collator::make_account_from(td::ConstBitPtr addr * @return True on success, False on failure */ bool Collator::init_account_storage_dict(block::Account& account) { - if (!full_collated_data_ || is_masterchain() || !account.storage_dict_hash || account.storage.is_null()) { + if (!account.storage_dict_hash || account.storage.is_null()) { return true; } td::Bits256 storage_dict_hash = account.storage_dict_hash.value(); if (storage_dict_hash.is_zero()) { return true; } + td::Ref cached_dict_root = + storage_stat_cache_ ? storage_stat_cache_(storage_dict_hash) : td::Ref{}; + if (cached_dict_root.not_null()) { + CHECK(td::Bits256{cached_dict_root->get_hash().bits()} == storage_dict_hash); + LOG(DEBUG) << "Inited storage stat from cache for account " << account.addr.to_hex() << " (" + << account.storage_used.cells << " cells)"; + storage_stat_cache_update_.emplace_back(cached_dict_root, account.storage_used.cells); + } + if (!full_collated_data_ || is_masterchain()) { + if (cached_dict_root.not_null()) { + auto S = account.init_account_storage_stat(cached_dict_root); + if (S.is_error()) { + return fatal_error(S.move_as_error_prefix(PSTRING() << "failed to init storage stat from cache for account " + << account.addr.to_hex() << ": ")); + } + } + return true; + } + AccountStorageDict& dict = account_storage_dicts_[storage_dict_hash]; if (!dict.inited) { dict.inited = true; - // don't mark cells in account state as loaded during compute_account_storage_dict - state_usage_tree_->set_ignore_loads(true); - auto res = account.compute_account_storage_dict(&dict.original_storage_cells); - state_usage_tree_->set_ignore_loads(false); - if (res.is_error()) { - return fatal_error(res.move_as_error_prefix(PSTRING() << "Failed to init account storage dict for " - << account.addr.to_hex() << ": ")); - } - if (res.ok().is_null()) { // Impossible if storage_dict_hash is not zero - return fatal_error(PSTRING() << "Failed to init account storage dict for " << account.addr.to_hex() - << ": dict is empty"); + td::Ref dict_root; + if (cached_dict_root.not_null()) { + dict_root = cached_dict_root; + } else { + // don't mark cells in account state as loaded during compute_account_storage_dict + state_usage_tree_->set_ignore_loads(true); + auto res = account.compute_account_storage_dict(); + state_usage_tree_->set_ignore_loads(false); + if (res.is_error()) { + return fatal_error(res.move_as_error_prefix(PSTRING() << "Failed to init account storage dict for " + << account.addr.to_hex() << ": ")); + } + dict_root = res.move_as_ok(); + if (dict_root.is_null()) { // Impossible if storage_dict_hash is not zero + return fatal_error(PSTRING() << "Failed to init account storage dict for " << account.addr.to_hex() + << ": dict is empty"); + } } - dict.mpb = vm::MerkleProofBuilder(res.move_as_ok()); + dict.mpb = vm::MerkleProofBuilder(std::move(dict_root)); dict.mpb.set_cell_load_callback([&](const vm::LoadedCell& cell) { on_cell_loaded(cell); }); @@ -2764,6 +2780,65 @@ td::Result Collator::make_account(td::ConstBitPtr addr, bool fo return ins.first->second.get(); } +/** + * Removes UsageCell's from new_root's subtree, replacing them with regular DataCell's + * + * @param old_root Original root (not wrapped in UsageCell) + * @param new_root Cell to remove UsageCell's from + * @param usage_tree CellUsageTree for old_root + * + * @returns new_root without UsageCell's + */ +static td::Ref clean_usage_cells(td::Ref old_root, td::Ref new_root, + const vm::CellUsageTree& usage_tree) { + if (new_root.is_null()) { + return {}; + } + + td::HashMap> visited_cells; + std::function, vm::CellUsageTree::NodeId)> dfs_old = [&](td::Ref cell, + vm::CellUsageTree::NodeId node_id) { + visited_cells[cell->get_hash()] = cell; + if (!usage_tree.is_loaded(node_id)) { + return; + } + vm::CellSlice cs{vm::NoVm(), cell}; + for (unsigned i = 0; i < cs.size_refs(); i++) { + dfs_old(cs.prefetch_ref(i), usage_tree.get_child(node_id, i)); + } + }; + if (old_root.not_null()) { + dfs_old(old_root, usage_tree.root_id()); + } + + std::function(td::Ref)> dfs = [&](td::Ref cell) -> td::Ref { + auto it = visited_cells.find(cell->get_hash()); + if (it != visited_cells.end()) { + return it->second; + } + auto loaded_cell = cell->load_cell().move_as_ok(); + CHECK(loaded_cell.virt.get_virtualization() == 0); + td::Ref data_cell = std::move(loaded_cell.data_cell); + td::Ref children[vm::Cell::max_refs]; + bool changed = false; + for (unsigned i = 0; i < data_cell->size_refs(); ++i) { + td::Ref child = data_cell->get_ref(i); + children[i] = dfs(child); + if (children[i] != child) { + changed = true; + } + } + if (changed) { + data_cell = vm::DataCell::create(td::Slice{data_cell->get_data(), (data_cell->size() + 7) / 8}, data_cell->size(), + {children, data_cell->size_refs()}, data_cell->is_special()) + .move_as_ok(); + CHECK(data_cell->get_hash() == cell->get_hash()); + } + return visited_cells[cell->get_hash()] = data_cell; + }; + return dfs(new_root); +} + /** * Decides whether to include storage dict proof to collated data for this account or not. * @@ -2771,17 +2846,34 @@ td::Result Collator::make_account(td::ConstBitPtr addr, bool fo * * @returns True if the operation is successful, false otherwise. */ -bool Collator::process_account_storage_dict(const block::Account& account) { +bool Collator::process_account_storage_dict(block::Account& account) { + bool store_dict_to_cache = account.storage_dict_hash && account.account_storage_stat && + account.account_storage_stat.value().is_dict_ready() && + account.storage_used.cells >= StorageStatCache::MIN_ACCOUNT_CELLS; if (!account.orig_storage_dict_hash) { + if (store_dict_to_cache) { + td::Ref dict_root = account.account_storage_stat.value().get_dict_root().move_as_ok(); + storage_stat_cache_update_.emplace_back(dict_root, account.storage_used.cells); + } return true; } td::Bits256 storage_dict_hash = account.orig_storage_dict_hash.value(); auto it = account_storage_dicts_.find(storage_dict_hash); if (it == account_storage_dicts_.end()) { + if (store_dict_to_cache) { + td::Ref dict_root = account.account_storage_stat.value().get_dict_root().move_as_ok(); + storage_stat_cache_update_.emplace_back(dict_root, account.storage_used.cells); + } return true; } CHECK(full_collated_data_ && !is_masterchain()); AccountStorageDict& dict = it->second; + td::Ref original_dict_root = dict.mpb.original_root(); + if (store_dict_to_cache) { + td::Ref dict_root = account.account_storage_stat.value().get_dict_root().move_as_ok(); + dict_root = clean_usage_cells(original_dict_root, dict_root, dict.mpb.get_usage_tree()); + storage_stat_cache_update_.emplace_back(dict_root, account.storage_used.cells); + } if (dict.add_to_collated_data) { LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() << " : already included"; return true; @@ -2794,12 +2886,13 @@ bool Collator::process_account_storage_dict(const block::Account& account) { td::HashSet visited; bool calculate_proof_size_diff = true; td::int64 proof_size_diff = 0; - std::function&)> dfs = [&](const Ref& cell) { + vm::Dictionary original_dict{original_dict_root, 256}; + std::function&)> dfs = [&](const Ref& cell) -> bool { if (cell.is_null() || !visited.emplace(cell->get_hash()).second) { - return; + return true; } auto loaded_cell = cell->load_cell().move_as_ok(); - if (dict.original_storage_cells.contains(cell->get_hash())) { + if (original_dict.lookup(cell->get_hash().bits(), 256).not_null()) { if (calculate_proof_size_diff) { switch (collated_data_stat.get_cell_status(cell->get_hash())) { case vm::ProofStorageStat::c_none: @@ -2812,26 +2905,36 @@ bool Collator::process_account_storage_dict(const block::Account& account) { case vm::ProofStorageStat::c_loaded: break; } + if (proof_size_diff > (td::int64)dict.proof_stat.estimate_proof_size()) { + return false; + } } else { collated_data_stat.add_loaded_cell(loaded_cell.data_cell, loaded_cell.virt.get_level()); } } vm::CellSlice cs{std::move(loaded_cell.data_cell)}; for (unsigned i = 0; i < cs.size_refs(); ++i) { - dfs(cs.prefetch_ref(i)); + if (!dfs(cs.prefetch_ref(i))) { + return false; + } } + return true; }; // Visit cells that were used in storage stat computation to calculate collated data increase + bool account_proof_too_big = false; state_usage_tree_->set_ignore_loads(true); for (const auto& cell : dict.storage_stat_updates) { - dfs(cell); + if (!dfs(cell)) { + account_proof_too_big = true; + break; + } } state_usage_tree_->set_ignore_loads(false); - if (proof_size_diff > (td::int64)dict.proof_stat.estimate_proof_size()) { + if (account_proof_too_big) { LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() - << " : account_proof_size=" << proof_size_diff + << " : account_proof_size>=" << proof_size_diff << ", dict_proof_size=" << dict.proof_stat.estimate_proof_size() << ", include dict in collated data"; dict.add_to_collated_data = true; collated_data_stat.add_loaded_cells(dict.proof_stat); @@ -2844,7 +2947,7 @@ bool Collator::process_account_storage_dict(const block::Account& account) { calculate_proof_size_diff = false; visited.clear(); for (const auto& cell : dict.storage_stat_updates) { - dfs(cell); + CHECK(dfs(cell)); } } @@ -2943,10 +3046,6 @@ bool Collator::combine_account_transactions() { if (!process_account_storage_dict(acc)) { return false; } - if (acc.storage_dict_hash && acc.account_storage_stat && acc.account_storage_stat.value().is_dict_ready()) { - storage_stat_cache_update_.emplace_back(acc.account_storage_stat.value().get_dict_root().move_as_ok(), - acc.storage_used.cells); - } } else { if (acc.total_state->get_hash() != acc.orig_total_state->get_hash()) { return fatal_error(std::string{"total state of account "} + z.first.to_hex() + @@ -6270,7 +6369,10 @@ bool Collator::create_block_candidate() { td::actor::send_closure_later(manager, &ValidatorManager::complete_external_messages, std::move(delay_ext_msgs_), std::move(bad_ext_msgs_)); } - td::actor::send_closure(manager, &ValidatorManager::update_storage_stat_cache, std::move(storage_stat_cache_update_)); + if (!storage_stat_cache_update_.empty()) { + td::actor::send_closure(manager, &ValidatorManager::update_storage_stat_cache, + std::move(storage_stat_cache_update_)); + } return true; } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 7570c78996..ac7f400f34 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -32,6 +32,8 @@ #include "vm/cells/MerkleUpdate.h" #include "common/errorlog.h" #include "fabric.h" +#include "storage-stat-cache.hpp" + #include namespace ton { @@ -657,6 +659,7 @@ bool ValidateQuery::extract_collated_data_from(Ref croot, int idx) { if (!virt_account_storage_dicts_.emplace(virt_root->get_hash().bits(), virt_root).second) { return reject_query("duplicate AccountStorageDictProof"); } + full_collated_data_ = true; return true; } LOG(WARNING) << "collated datum # " << idx << " has unknown type (magic " << cs.prefetch_ulong(32) << "), ignoring"; @@ -5316,20 +5319,6 @@ std::unique_ptr ValidateQuery::make_account_from(td::ConstBitPtr return nullptr; } ptr->block_lt = start_lt_; - if (storage_stat_cache_ && ptr->storage_dict_hash) { - auto dict_root = storage_stat_cache_(ptr->storage_dict_hash.value()); - if (dict_root.not_null()) { - auto S = ptr->init_account_storage_stat(dict_root); - if (S.is_error()) { - fatal_error(S.move_as_error_prefix(PSTRING() << "failed to init storage stat from cache for account " - << addr.to_hex(256) << ": ")); - return nullptr; - } - LOG(DEBUG) << "Inited storage stat from cache for account " << addr.to_hex(256) << " (" << ptr->storage_used.cells - << " cells)"; - storage_stat_cache_update_.emplace_back(dict_root, ptr->storage_used.cells); - } - } return ptr; } @@ -5356,14 +5345,30 @@ std::unique_ptr ValidateQuery::unpack_account(td::ConstBitPtr ad return {}; } if (new_acc->storage_dict_hash) { - auto it = virt_account_storage_dicts_.find(new_acc->storage_dict_hash.value()); - if (it != virt_account_storage_dicts_.end()) { - LOG(DEBUG) << "Using account storage dict proof for account " << addr.to_hex(256) - << ", hash=" << it->second->get_hash().to_hex(); - auto S = new_acc->init_account_storage_stat(it->second); - if (S.is_error()) { - reject_query(PSTRING() << "Failed to init account storage stat for account " << addr.to_hex(256), std::move(S)); - return {}; + if (full_collated_data_ && !is_masterchain()) { + auto it = virt_account_storage_dicts_.find(new_acc->storage_dict_hash.value()); + if (it != virt_account_storage_dicts_.end()) { + LOG(DEBUG) << "Using account storage dict proof for account " << addr.to_hex(256) + << ", hash=" << it->second->get_hash().to_hex(); + auto S = new_acc->init_account_storage_stat(it->second); + if (S.is_error()) { + reject_query(PSTRING() << "Failed to init account storage stat for account " << addr.to_hex(256), + std::move(S)); + return {}; + } + } + } else if (storage_stat_cache_ && new_acc->storage_dict_hash) { + auto dict_root = storage_stat_cache_(new_acc->storage_dict_hash.value()); + if (dict_root.not_null()) { + auto S = new_acc->init_account_storage_stat(dict_root); + if (S.is_error()) { + fatal_error(S.move_as_error_prefix(PSTRING() << "failed to init storage stat from cache for account " + << addr.to_hex(256) << ": ")); + return {}; + } + LOG(DEBUG) << "Inited storage stat from cache for account " << addr.to_hex(256) << " (" + << new_acc->storage_used.cells << " cells)"; + storage_stat_cache_update_.emplace_back(dict_root, new_acc->storage_used.cells); } } } @@ -5955,8 +5960,9 @@ bool ValidateQuery::check_account_transactions(const StdSmcAddress& acc_addr, Re })) { return reject_query("at least one Transaction of account "s + acc_addr.to_hex() + " is invalid"); } - if (account.storage_dict_hash && account.account_storage_stat && - account.account_storage_stat.value().is_dict_ready()) { + if ((!full_collated_data_ || is_masterchain()) && account.storage_dict_hash && account.account_storage_stat && + account.account_storage_stat.value().is_dict_ready() && + account.storage_used.cells >= StorageStatCache::MIN_ACCOUNT_CELLS) { storage_stat_cache_update_.emplace_back(account.account_storage_stat.value().get_dict_root().move_as_ok(), account.storage_used.cells); } @@ -7156,7 +7162,10 @@ bool ValidateQuery::save_candidate() { td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, id_, block_candidate.clone(), validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), std::move(P)); - td::actor::send_closure(manager, &ValidatorManager::update_storage_stat_cache, std::move(storage_stat_cache_update_)); + if (!storage_stat_cache_update_.empty()) { + td::actor::send_closure(manager, &ValidatorManager::update_storage_stat_cache, + std::move(storage_stat_cache_update_)); + } return true; } diff --git a/validator/storage-stat-cache.cpp b/validator/storage-stat-cache.cpp index 6a8049b0dc..a81d958f5a 100644 --- a/validator/storage-stat-cache.cpp +++ b/validator/storage-stat-cache.cpp @@ -14,7 +14,6 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#pragma once #include "storage-stat-cache.hpp" namespace ton::validator { @@ -27,7 +26,7 @@ void StorageStatCache::get_cache(td::Promise(con void StorageStatCache::update(std::vector, td::uint32>> data) { for (auto &[cell, size] : data) { - if (size < 4000) { + if (size < MIN_ACCOUNT_CELLS) { continue; } td::Bits256 hash = cell->get_hash().bits(); diff --git a/validator/storage-stat-cache.hpp b/validator/storage-stat-cache.hpp index d32611fcd7..194a0b79ab 100644 --- a/validator/storage-stat-cache.hpp +++ b/validator/storage-stat-cache.hpp @@ -57,7 +57,12 @@ class StorageStatCache : public td::actor::Actor { td::Bits256 hash = td::Bits256::zero(); vm::Dictionary* cache; }; - td::LRUCache lru_{1 << 24}; + td::LRUCache lru_{MAX_CACHE_TOTAL_CELLS}; + + static constexpr td::uint64 MAX_CACHE_TOTAL_CELLS = 1 << 24; + + public: + static constexpr td::uint64 MIN_ACCOUNT_CELLS = 4000; }; } // namespace ton::validator From 21c191d2648c7a6d216afb4263a14a890d538d6d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 1 Jul 2025 17:26:00 +0300 Subject: [PATCH 320/524] Improve TL scheme for overlay nodes V2 --- tl/generate/scheme/ton_api.tl | 4 ++-- tl/generate/scheme/ton_api.tlo | Bin 116024 -> 116024 bytes 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 30e8262d4a..4da1ee301c 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -232,7 +232,7 @@ overlay.node.toSignEx id:adnl.id.short overlay:int256 flags:int version:int = ov overlay.node id:PublicKey overlay:int256 version:int signature:bytes = overlay.Node; overlay.nodeV2 id:PublicKey overlay:int256 flags:int version:int signature:bytes certificate:overlay.MemberCertificate = overlay.NodeV2; overlay.nodes nodes:(vector overlay.node) = overlay.Nodes; -overlay.nodesV2 nodes:(vector overlay.NodeV2) = overlay.NodesV2; +overlay.nodesV2 nodes:(vector overlay.nodeV2) = overlay.NodesV2; overlay.pong = overlay.Pong; @@ -271,7 +271,7 @@ overlay.broadcastNotFound = overlay.Broadcast; ---functions--- overlay.getRandomPeers peers:overlay.nodes = overlay.Nodes; -overlay.getRandomPeersV2 peers:overlay.NodesV2 = overlay.NodesV2; +overlay.getRandomPeersV2 peers:overlay.nodesV2 = overlay.NodesV2; overlay.ping = overlay.Pong; overlay.query overlay:int256 = True; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 3caa8b14674e96e891645db8579517e543cebd5d..9959470322712e8a29e37b912ad8211c0c60d9d7 100644 GIT binary patch delta 167 zcmdnd#lEA9eZvp6iCaV_^Qh~vI7zTS*=(a;aZ&=r;?FNjEy_u()XU3HNi7aD0*Os7 zm~SAleC_PLj0_M^HFt0%7SgfX5jXSd5jO#0DE6JaR2}S delta 169 zcmdnd#lEA9eZvp6$w!z(CiAH4uvk31D7D!}z2c+ Date: Wed, 2 Jul 2025 11:50:28 +0300 Subject: [PATCH 321/524] Remove duplicate validator-engine options --- validator-engine/validator-engine.cpp | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index b28f349f9a..c78e0acc36 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -5578,21 +5578,6 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_sync_shards_upto, v); }); return td::Status::OK(); }); - p.add_option( - '\0', "permanent-celldb", - "disable garbage collection in CellDb. This improves performance on archival nodes (once enabled, this option " - "cannot be disabled)", - [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_permanent_celldb, true); }); }); - p.add_option('\0', "skip-key-sync", - "don't select the best persistent state on initial sync, start on init_block from global config", [&]() { - acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_skip_key_sync, true); }); - }); - p.add_checked_option( - '\0', "sync-shards-upto", "stop syncing shards on this masterchain seqno", [&](td::Slice s) -> td::Status { - TRY_RESULT(v, td::to_integer_safe(s)); - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_sync_shards_upto, v); }); - return td::Status::OK(); - }); p.add_checked_option('\0', "shard-block-retainer", "adnl id for shard block retainer (hex), or \"fullnode\" for full node id", [&](td::Slice arg) -> td::Status { From ee58f04c015dc971020c8470649d317dab347be1 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 4 Jul 2025 13:36:17 +0300 Subject: [PATCH 322/524] Properly add non-light validators to fast-sync overlays --- tl/generate/scheme/ton_api.tl | 2 +- tl/generate/scheme/ton_api.tlo | Bin 116024 -> 116068 bytes validator/full-node-fast-sync-overlays.cpp | 3 ++- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 4da1ee301c..7db6ac0e73 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -768,7 +768,7 @@ engine.validator.shardOverlayStats.neighbour id:string verison_major:int version engine.validator.shardOverlayStats shard:string active:Bool neighbours:(vector engine.validator.shardOverlayStats.neighbour) = engine.validator.ShardOverlayStats; engine.validator.fastSyncOverlayStats shard:string validators_adnl:(vector int256) root_public_keys:(vector int256) - member_certificate:overlay.MemberCertificate = engine.validator.FastSyncOverlayStats; + member_certificate:overlay.MemberCertificate receive_broadcasts:Bool = engine.validator.FastSyncOverlayStats; engine.validator.onePerfTimerStat time:int min:double avg:double max:double = engine.validator.OnePerfTimerStat; engine.validator.perfTimerStatsByName name:string stats:(vector engine.validator.OnePerfTimerStat) = engine.validator.PerfTimerStatsByName; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 9959470322712e8a29e37b912ad8211c0c60d9d7..b4d84dd96804cee1ac61c758da3f1a1307cfb32f 100644 GIT binary patch delta 99 zcmdnd#r~v=eZveX*6f#m4y#TUl#<-MO={l~C^vAQ promise) if (!member_certificate_.empty()) { res->member_certificate_ = member_certificate_.tl(); } + res->receive_broadcasts_ = receive_broadcasts_; promise.set_result(td::json_encode(td::ToJson(*res), true)); } @@ -525,7 +526,7 @@ void FullNodeFastSyncOverlays::update_overlays(td::Ref state, // Update shard overlays for (ShardIdFull shard : all_shards) { - bool receive_broadcasts = overlays_info.is_validator_ ? shard.is_masterchain() : monitoring_shards.count(shard); + bool receive_broadcasts = monitoring_shards.contains(shard); auto &overlay = overlays_info.overlays_[shard]; if (overlay.empty()) { overlay = td::actor::create_actor( From 75ff7e6288d678ef209317f7b497dd64b2343bd5 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 4 Jul 2025 16:06:56 +0300 Subject: [PATCH 323/524] Enable fast-sync overlays by proto_version 5 --- validator/full-node.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 27278c3ee4..df91237619 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -262,7 +262,11 @@ void FullNodeImpl::on_new_masterchain_block(td::Ref state, std } } + int proto_version = state->get_consensus_config().proto_version; + use_old_private_overlays_ = (proto_version < 5); + if (!use_old_private_overlays_) { + private_block_overlays_.clear(); std::set my_adnl_ids; my_adnl_ids.insert(adnl_id_); for (const auto &[adnl_id, _] : local_collator_nodes_) { @@ -690,14 +694,6 @@ void FullNodeImpl::update_validator_telemetry_collector() { } void FullNodeImpl::start_up() { - // TODO: enable fast sync overlays by other means (e.g. some config param) - // TODO: in the future - remove the old private overlay entirely - // This env var is for testing - auto fast_sync_env = getenv("TON_FAST_SYNC_OVERLAYS"); - if (fast_sync_env && !strcmp(fast_sync_env, "1")) { - use_old_private_overlays_ = false; - } - update_shard_actor(ShardIdFull{masterchainId}, true); if (local_id_.is_zero()) { if (adnl_id_.is_zero()) { From 0f4849b6671050581c48d43e624f3ba3d4ff9a39 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 4 Jul 2025 16:40:52 +0300 Subject: [PATCH 324/524] Set max fast sync overlay clients to 5 --- validator/full-node.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/full-node.h b/validator/full-node.h index 2c51038eb4..dc29f67ca2 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -117,7 +117,7 @@ class FullNode : public td::actor::Actor { } enum { broadcast_mode_public = 1, broadcast_mode_private_block = 2, broadcast_mode_custom = 4 }; - static constexpr td::int32 MAX_FAST_SYNC_OVERLAY_CLIENTS = 5000; // TODO: set lower limit (high limit for testing) + static constexpr td::int32 MAX_FAST_SYNC_OVERLAY_CLIENTS = 5; static td::actor::ActorOwn create( ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeOptions opts, From cbe79eb848ab554b5eb96e70c08e777cfa3b2297 Mon Sep 17 00:00:00 2001 From: krigga <25533192+krigga@users.noreply.github.com> Date: Sun, 6 Jul 2025 12:18:22 +0300 Subject: [PATCH 325/524] feat: add prev blocks info param to emscripten tx emulator (#1726) --- emulator/emulator-emscripten.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/emulator/emulator-emscripten.cpp b/emulator/emulator-emscripten.cpp index efb14eff94..7b8e9dd1ea 100644 --- a/emulator/emulator-emscripten.cpp +++ b/emulator/emulator-emscripten.cpp @@ -11,6 +11,7 @@ struct TransactionEmulationParams { uint32_t utime; uint64_t lt; td::optional rand_seed_hex; + td::optional prev_blocks_info; bool ignore_chksig; bool is_tick_tock; bool is_tock; @@ -49,6 +50,11 @@ td::Result decode_transaction_emulation_params(const TRY_RESULT(is_tock, td::get_json_object_bool_field(obj, "is_tock", true, false)); params.is_tock = is_tock; + TRY_RESULT(prev_blocks_info_str, td::get_json_object_string_field(obj, "prev_blocks_info", true)); + if (prev_blocks_info_str.size() > 0) { + params.prev_blocks_info = prev_blocks_info_str; + } + if (is_tock && !is_tick_tock) { return td::Status::Error("Inconsistent parameters is_tick_tock=false, is_tock=true"); } @@ -200,12 +206,18 @@ const char *emulate_with_emulator(void* em, const char* libs, const char* accoun rand_seed_set = transaction_emulator_set_rand_seed(em, decoded_params.rand_seed_hex.unwrap().c_str()); } + bool prev_blocks_set = true; + if (decoded_params.prev_blocks_info) { + prev_blocks_set = transaction_emulator_set_prev_blocks_info(em, decoded_params.prev_blocks_info.unwrap().c_str()); + } + if (!transaction_emulator_set_libs(em, libs) || !transaction_emulator_set_lt(em, decoded_params.lt) || !transaction_emulator_set_unixtime(em, decoded_params.utime) || !transaction_emulator_set_ignore_chksig(em, decoded_params.ignore_chksig) || !transaction_emulator_set_debug_enabled(em, decoded_params.debug_enabled) || - !rand_seed_set) { + !rand_seed_set || + !prev_blocks_set) { transaction_emulator_destroy(em); return strdup(R"({"fail":true,"message":"Can't set params"})"); } From bd89b252ca5bee5fa9dee2f53accc0e8221e0c2f Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 24 Apr 2025 18:30:43 +0400 Subject: [PATCH 326/524] [Tolk] Optimize try-catch codegen with SETCONTMANY instruction --- tolk-tester/tests/try-catch-tests.tolk | 14 +++++++++++++- tolk/codegen.cpp | 11 +---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/tolk-tester/tests/try-catch-tests.tolk b/tolk-tester/tests/try-catch-tests.tolk index be0362e395..8acb6f2c78 100644 --- a/tolk-tester/tests/try-catch-tests.tolk +++ b/tolk-tester/tests/try-catch-tests.tolk @@ -236,6 +236,17 @@ fun testCodegen3(numberId: int, paramVal: cell) { paramVal.beginParse(); } +@method_id(112) +fun testCatchUsesVars(x1: int) { + var x2 = x1 * 2; + try { alwaysThrow123(); } + catch (excno) { + var result = x1 + x2 + excno; + return result; + } + return -1; +} + @method_id(113) fun testBigExcno() { try { @@ -274,9 +285,10 @@ fun main() { @testcase | 110 | 0 | 5 @testcase | 111 | -1 | 123 @testcase | 111 | 0 | 456 +@testcase | 112 | 5 | 138 @testcase | 113 | | 2048 -@code_hash 26411074751358281917876479601714698321674833208179417883888954015536273820035 +@code_hash 100812963733861852648745958727639570706776197131971804296829155862118445447691 @fif_codegen """ diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp index a7c84d27d9..c635c7483a 100644 --- a/tolk/codegen.cpp +++ b/tolk/codegen.cpp @@ -843,11 +843,6 @@ bool Op::generate_code_step(Stack& stack) { catch_stack.push_new_var(left[1]); stack.rearrange_top(loc, catch_vars, catch_last); stack.opt_show(); - stack.o << AsmOp::Custom(loc, "c1 PUSH"); - stack.o << AsmOp::Custom(loc, "c3 PUSH"); - stack.o << AsmOp::Custom(loc, "c4 PUSH"); - stack.o << AsmOp::Custom(loc, "c5 PUSH"); - stack.o << AsmOp::Custom(loc, "c7 PUSH"); stack.o << AsmOp::Custom(loc, "<{"); stack.o.indent(); if (block1->noreturn()) { @@ -858,11 +853,7 @@ bool Op::generate_code_step(Stack& stack) { catch_stack.opt_show(); stack.o.undent(); stack.o << AsmOp::Custom({}, "}>CONT"); - stack.o << AsmOp::Custom(loc, "c7 SETCONT"); - stack.o << AsmOp::Custom(loc, "c5 SETCONT"); - stack.o << AsmOp::Custom(loc, "c4 SETCONT"); - stack.o << AsmOp::Custom(loc, "c3 SETCONT"); - stack.o << AsmOp::Custom(loc, "c1 SETCONT"); + stack.o << AsmOp::Custom(loc, "0b10111010 SETCONTMANY"); for (size_t begin = catch_vars.size(), end = begin; end > 0; end = begin) { begin = end >= block_size ? end - block_size : 0; stack.o << AsmOp::Custom(loc, std::to_string(end - begin) + " PUSHINT"); From 487034a5a2e59c1e4ef8fd4ab0f742296f271234 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 15 Jun 2025 14:53:55 +0700 Subject: [PATCH 327/524] [Tolk] More accurate serialization estimation for snake structs This allows express snake nesting like struct A { next: Cell? } --- .../tests/invalid-serialization/err-7563.tolk | 2 +- tolk-tester/tests/pack-unpack-5.tolk | 16 ++++++++++++++++ tolk/pack-unpack-api.cpp | 12 ++++++++++-- tolk/pack-unpack-serializers.cpp | 2 +- tolk/pipe-check-serialized-fields.cpp | 7 +------ 5 files changed, 29 insertions(+), 10 deletions(-) diff --git a/tolk-tester/tests/invalid-serialization/err-7563.tolk b/tolk-tester/tests/invalid-serialization/err-7563.tolk index cc83a0cd59..9934ebb6e4 100644 --- a/tolk-tester/tests/invalid-serialization/err-7563.tolk +++ b/tolk-tester/tests/invalid-serialization/err-7563.tolk @@ -11,7 +11,7 @@ struct B { } fun main(b: B) { - b.toCell(); + B.fromSlice("").a.load(); } /** diff --git a/tolk-tester/tests/pack-unpack-5.tolk b/tolk-tester/tests/pack-unpack-5.tolk index 9644d7331f..46b2a77b54 100644 --- a/tolk-tester/tests/pack-unpack-5.tolk +++ b/tolk-tester/tests/pack-unpack-5.tolk @@ -178,6 +178,21 @@ fun test10() { return (Test10_1.estimatePackSize(), Test10_2.estimatePackSize()); } +struct Test11_1 { + data: bits1022; + next: Cell?; +} + +struct Test11_2 { + self1: Cell; + self2: Cell; +} + +@method_id(111) +fun test11() { + return (Test11_1.estimatePackSize(), Test11_2.estimatePackSize()); +} + @method_id(120) fun test20() { return (Test7_1 .getDeclaredPackPrefixLen(), Test7_1 .getDeclaredPackPrefix(), @@ -199,5 +214,6 @@ fun main() { @testcase | 108 | | [ 10 275 0 0 ] [ 34 158 0 0 ] [ 47 1143 0 1 ] @testcase | 109 | | [ 34 130 0 0 ] [ 33 159 0 0 ] [ 4 8 0 0 ] [ 3 7 0 0 ] @testcase | 110 | | [ 32 9999 0 4 ] [ 33 9999 0 8 ] +@testcase | 111 | | [ 1023 1023 0 1 ] [ 0 0 2 2 ] @testcase | 120 | | 16 4128 1 1 */ diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index 227a070660..107130d598 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -50,8 +50,10 @@ struct CantSerializeBecause { }; class PackUnpackAvailabilityChecker { + std::vector already_checked; + public: - static std::optional detect_why_cant_serialize(TypePtr any_type, bool is_pack) { + std::optional detect_why_cant_serialize(TypePtr any_type, bool is_pack) { if (any_type->try_as()) { return {}; } @@ -76,6 +78,11 @@ class PackUnpackAvailabilityChecker { if (const auto* t_struct = any_type->try_as()) { StructPtr struct_ref = t_struct->struct_ref; + if (std::find(already_checked.begin(), already_checked.end(), struct_ref) != already_checked.end()) { + return {}; + } + already_checked.push_back(struct_ref); // prevent recursion and visiting one struct multiple times + for (StructFieldPtr field_ref : struct_ref->fields) { if (auto why = detect_why_cant_serialize(field_ref->declared_type, is_pack)) { return CantSerializeBecause("because field `" + struct_ref->name + "." + field_ref->name + "` of type `" + field_ref->declared_type->as_human_readable() + "` can't be serialized", why.value()); @@ -167,7 +174,8 @@ class PackUnpackAvailabilityChecker { }; bool check_struct_can_be_packed_or_unpacked(TypePtr any_type, bool is_pack, std::string& because_msg) { - if (auto why = PackUnpackAvailabilityChecker::detect_why_cant_serialize(any_type, is_pack)) { + PackUnpackAvailabilityChecker checker; + if (auto why = checker.detect_why_cant_serialize(any_type, is_pack)) { because_msg = why.value().because_msg; return false; } diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index 61962615a6..d139a45266 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -959,7 +959,7 @@ static std::unique_ptr get_serializer_for_type(TypePtr any_type) { if (any_type == TypeDataBool::create()) { return std::make_unique(); } - if (any_type == TypeDataCell::create()) { + if (any_type == TypeDataCell::create() || is_type_cellT(any_type)) { return std::make_unique(); } if (any_type == TypeDataAddress::create()) { diff --git a/tolk/pipe-check-serialized-fields.cpp b/tolk/pipe-check-serialized-fields.cpp index 8f436c9571..a13032e372 100644 --- a/tolk/pipe-check-serialized-fields.cpp +++ b/tolk/pipe-check-serialized-fields.cpp @@ -69,12 +69,7 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody fire_error_theoretical_overflow_1023(struct_ref, size); } } - for (StructFieldPtr field_ref : struct_ref->fields) { - if (is_type_cellT(field_ref->declared_type)) { - const TypeDataStruct* f_struct = field_ref->declared_type->try_as(); - check_type_fits_cell_or_has_policy(f_struct->struct_ref->substitutedTs->typeT_at(0)); - } - } + // don't check Cell fields for overflow of T: it would be checked on load() or other interaction with T } void visit(V v) override { From f825c353e0093067a5b7857d5ba23a092c951bfa Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 6 Jun 2025 16:07:18 +0700 Subject: [PATCH 328/524] [Tolk] Auto-detect and inline functions at AST level Instead of generating PROCINLINE to Fift, perform compile-time inlining when possible --- tolk-tester/tests/a-tests.tolk | 2 +- .../tests/allow-post-modification.tolk | 8 + tolk-tester/tests/asm-arg-order.tolk | 6 +- tolk-tester/tests/bit-operators.tolk | 1 + tolk-tester/tests/constants-tests.tolk | 2 +- tolk-tester/tests/generics-1.tolk | 6 +- tolk-tester/tests/generics-2.tolk | 2 + tolk-tester/tests/generics-3.tolk | 4 +- tolk-tester/tests/generics-4.tolk | 3 + tolk-tester/tests/if-else-tests.tolk | 41 ++ tolk-tester/tests/indexed-access.tolk | 1 + tolk-tester/tests/inline-tests.tolk | 612 +++++++++++++++--- tolk-tester/tests/intN-tests.tolk | 1 + .../tests/invalid-semantics/err-4188.tolk | 13 + tolk-tester/tests/match-by-expr-tests.tolk | 52 ++ tolk-tester/tests/methods-tests.tolk | 4 + tolk-tester/tests/mutate-methods.tolk | 5 + tolk-tester/tests/no-spaces.tolk | 1 + tolk-tester/tests/nullable-tensors.tolk | 5 + tolk-tester/tests/op-priority.tolk | 3 + tolk-tester/tests/pack-unpack-2.tolk | 3 +- tolk-tester/tests/pack-unpack-3.tolk | 3 +- tolk-tester/tests/pack-unpack-4.tolk | 1 + tolk-tester/tests/parse-address.tolk | 1 + .../tests/remove-unused-functions.tolk | 1 + tolk-tester/tests/self-keyword.tolk | 4 + tolk-tester/tests/send-msg-1.tolk | 24 +- tolk-tester/tests/send-msg-2.tolk | 3 + tolk-tester/tests/send-msg-3.tolk | 5 + tolk-tester/tests/smart-cast-tests.tolk | 1 + tolk-tester/tests/some-tests-2.tolk | 2 +- tolk-tester/tests/some-tests-3.tolk | 1 + tolk-tester/tests/strings-tests.tolk | 2 +- tolk-tester/tests/struct-tests.tolk | 9 +- tolk-tester/tests/try-catch-tests.tolk | 10 +- tolk-tester/tests/union-types-tests.tolk | 1 + tolk/CMakeLists.txt | 1 + tolk/ast-from-tokens.cpp | 11 +- tolk/ast-replicator.h | 3 +- tolk/ast.cpp | 3 + tolk/ast.h | 6 +- tolk/builtins.cpp | 7 +- tolk/codegen.cpp | 2 +- tolk/fwd-declarations.h | 8 + tolk/generics-helpers.cpp | 2 +- tolk/pipe-ast-to-legacy.cpp | 126 +++- tolk/pipe-check-rvalue-lvalue.cpp | 21 +- tolk/pipe-detect-inline-in-place.cpp | 299 +++++++++ tolk/pipe-find-unused-symbols.cpp | 2 +- tolk/pipe-generate-fif-output.cpp | 29 +- tolk/pipe-register-symbols.cpp | 2 +- tolk/pipeline.h | 1 + tolk/symtable.cpp | 12 + tolk/symtable.h | 17 +- tolk/tolk.cpp | 1 + tolk/tolk.h | 4 +- 56 files changed, 1236 insertions(+), 164 deletions(-) create mode 100644 tolk-tester/tests/invalid-semantics/err-4188.tolk create mode 100644 tolk/pipe-detect-inline-in-place.cpp diff --git a/tolk-tester/tests/a-tests.tolk b/tolk-tester/tests/a-tests.tolk index 077303d648..a122ca39dc 100644 --- a/tolk-tester/tests/a-tests.tolk +++ b/tolk-tester/tests/a-tests.tolk @@ -79,5 +79,5 @@ fun main(): int { method_id | in | out @testcase | 0 | | 31415926535897932384626433832795028841971693993751058209749445923078164 -@code_hash 84337043972311674339187056298873613816389434478842780265748859098303774481976 +@code_hash 103119459254804391561326789266022754176632268076661379738543746273668564328534 */ diff --git a/tolk-tester/tests/allow-post-modification.tolk b/tolk-tester/tests/allow-post-modification.tolk index 6d3abf0573..0f8603cc66 100644 --- a/tolk-tester/tests/allow-post-modification.tolk +++ b/tolk-tester/tests/allow-post-modification.tolk @@ -1,18 +1,24 @@ fun unsafe_tuple(x: X): tuple asm "NOP"; +@noinline fun inc(x: int, y: int): (int, int) { return (x + y, y * 10); } +@noinline fun int.`~inc`(mutate self, y: int): int { val (newX, newY) = inc(self, y); self = newX; return newY; } +@noinline fun eq(v: X): X { return v; } +@noinline fun eq2(v: (int, int)) { return v; } +@noinline fun mul2(mutate dest: int, v: int): int { dest = v*2; return dest; } +@noinline fun (int, int).multens(mutate self, v: (int, int)): (int, int) { var (f, s) = self; var (m1, m2) = v; self = (f*m1, s*m2); return self; } @method_id(11) @@ -38,6 +44,7 @@ fun test_tuple_assign(x: int): (int, int, int, int, int, int, int) { return (x1, x2, x3, x4, x5, x6, x7); } +@noinline fun foo1(x1: int, x2: int, x3: int, x4: int, x5: int, x6: int, x7: int): (int, int, int, int, int, int, int) { return (x1, x2, x3, x4, x5, x6, x7); } @@ -47,6 +54,7 @@ fun test_call_1(x: int): (int, int, int, int, int, int, int) { return foo1(x, x.`~inc`(x / 20), x, x = x * 2, x, x += 1, x); } +@noinline fun foo2(x1: int, x2: int, x3456: (int, int, int, int), x7: int, ): (int, int, int, int, int, int, int) { var (x3: int, x4: int, x5: int, x6: int) = x3456; return (x1, x2, x3, x4, x5, x6, x7); diff --git a/tolk-tester/tests/asm-arg-order.tolk b/tolk-tester/tests/asm-arg-order.tolk index cb7b6f1e30..b8622060a2 100644 --- a/tolk-tester/tests/asm-arg-order.tolk +++ b/tolk-tester/tests/asm-arg-order.tolk @@ -129,11 +129,15 @@ fun int.plus1TimesB(self, b: int): int asm(b self) "1 ADDCONST MUL"; @pure +@noinline fun get2Pure() { return 2; } @pure +@noinline fun get10Pure() { return 10; } +@noinline fun get2Impure() { return 2; } +@noinline fun get10Impure() { return 10; } global g2: int; @@ -272,5 +276,5 @@ fun main() { }> """ -@code_hash 93297578247504549901423325446957614083213743835412767087757830691482809332788 +@code_hash 88343225778124778743440967192570181022747908733235053033508198051414837052492 */ diff --git a/tolk-tester/tests/bit-operators.tolk b/tolk-tester/tests/bit-operators.tolk index 9ae04090c2..692275bf64 100644 --- a/tolk-tester/tests/bit-operators.tolk +++ b/tolk-tester/tests/bit-operators.tolk @@ -88,6 +88,7 @@ fun testBoolNegateOptimized(x: bool) { return (x, !x, !!x, !!!x, !!!!true); } +@noinline fun eqX(x: bool) { return x; } @method_id(19) diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index 838a7dae33..607cccf0a9 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -127,5 +127,5 @@ fun main() { @testcase | 105 | | -1 0 7 48 @testcase | 106 | | -1 0 0 -1 -@code_hash 85012002134196298607946339234948178079079623823648671175958399073436861460061 +@code_hash 39394999917297360167950120811797040756088634052063904555277102239104815314650 */ diff --git a/tolk-tester/tests/generics-1.tolk b/tolk-tester/tests/generics-1.tolk index 0984dcd515..39ddb2cdf3 100644 --- a/tolk-tester/tests/generics-1.tolk +++ b/tolk-tester/tests/generics-1.tolk @@ -2,6 +2,7 @@ type MInt = int; type Tensor2Int = (int, int); type Tensor2IntN = (int, int)?; +@noinline fun eq1(value: X): X { return value; } fun eq2(value: X) { return value; } fun eq3(value: X): X { var cp: [X] = [eq1(value)]; var ((([v: X]))) = cp; return v; } @@ -240,7 +241,10 @@ fun main(x: int): (int, [Tup2Int]) { @fif_codegen DECLPROC eq1 @fif_codegen DECLPROC eq1 @fif_codegen DECLPROC eq1>> -@fif_codegen DECLPROC getTwo + +// was inlined +@fif_codegen_avoid DECLPROC getTwo +@fif_codegen_avoid getTwo @fif_codegen_avoid DECLPROC eq1 @fif_codegen_avoid DECLPROC eq2 diff --git a/tolk-tester/tests/generics-2.tolk b/tolk-tester/tests/generics-2.tolk index 2ee2851ae8..65d76cebfe 100644 --- a/tolk-tester/tests/generics-2.tolk +++ b/tolk-tester/tests/generics-2.tolk @@ -61,6 +61,7 @@ fun test2(c: Wrapper) { } @pure +@noinline fun getWrappervalue1(c: Wrapper) { return c.value; } @@ -71,6 +72,7 @@ fun Wrapper.getWrappervalue1(self) { } @pure +@noinline fun getWrappervalue2(c: T) { return c.value; } diff --git a/tolk-tester/tests/generics-3.tolk b/tolk-tester/tests/generics-3.tolk index a6fc748852..691b878356 100644 --- a/tolk-tester/tests/generics-3.tolk +++ b/tolk-tester/tests/generics-3.tolk @@ -138,8 +138,8 @@ fun main() { """ test3 PROC:<{ 10 PUSHINT // r.USlot1=10 - -1 PUSHINT // r.USlot1=10 '11=-1 - FALSE // r.USlot1=10 '11=-1 '13 + -1 PUSHINT // r.USlot1=10 '22=-1 + FALSE // r.USlot1=10 '22=-1 '24 }> """ */ diff --git a/tolk-tester/tests/generics-4.tolk b/tolk-tester/tests/generics-4.tolk index 70c70818d4..2869ad1f57 100644 --- a/tolk-tester/tests/generics-4.tolk +++ b/tolk-tester/tests/generics-4.tolk @@ -1,4 +1,6 @@ +@noinline fun eqUnusedT(v: int) { return v; } +@noinline fun eqUnusedU(v: T) { if (v is int123) { __expect_type(v as U, "never"); } return v; } fun dup(x: T1, y: T2): (T1, T2) { return (x, y) } @@ -45,6 +47,7 @@ fun createParameters(bounce: bool, body: TBody, data return { bounce, body, init: { value: ton("0"), data } }; } +@noinline fun mySend(p: Parameters): int { var total = 0; if (p.bounce) { diff --git a/tolk-tester/tests/if-else-tests.tolk b/tolk-tester/tests/if-else-tests.tolk index 1656519af0..d34dc178b2 100644 --- a/tolk-tester/tests/if-else-tests.tolk +++ b/tolk-tester/tests/if-else-tests.tolk @@ -102,20 +102,24 @@ fun test8(op: int) { return (withNoElse(op), withElse(op), withMatch(op)); } +@noinline fun demo9_1(x: int) { if (x > 0) { t.push(123); } else { t.push(456) } } +@noinline fun demo9_2(x: int) { if (x > 0) { t.push(123); } } +@noinline fun demo9_3(x: int) { if (x > 0) { t.push(123); } return; } +@noinline fun demo9_4(x: int) { if (x != -100) { if (x > 0) { t.push(123); } @@ -130,6 +134,7 @@ fun test9(x: int) { return t; } +@noinline fun demo10_1(x: int) { match(x) { 0 => t.push(123), @@ -137,6 +142,7 @@ fun demo10_1(x: int) { } } +@noinline fun demo10_2(x: int) { if (x != -100) { match (x) { @@ -155,6 +161,7 @@ fun test10(x: int) { return t; } +@noinline fun demo_neg_11_1(mutate x: int) { match (x) { -1 => { x = 0; } @@ -182,6 +189,38 @@ fun test11(x: int) { return (x, t); } +fun demo12_1(x: int) { + __expect_inline(true); + if (x > 0) { t.push(123); } + else { t.push(456) } +} + +fun demo12_2(x: int) { + __expect_inline(true); + if (x > 0) { t.push(123); } +} + +fun demo12_3(x: int) { + __expect_inline(true); + if (x > 0) { t.push(123); } + return; +} + +fun demo12_4(x: int) { + __expect_inline(false); + if (x != -100) { + if (x > 0) { t.push(123); } + return; + } +} + +@method_id(112) +fun test12(x: int) { + t = createEmptyTuple(); + demo12_1(x); demo12_2(x); demo12_3(x); demo12_4(x); + return t; +} + fun main() { } @@ -212,6 +251,8 @@ fun main() { @testcase | 110 | 0 | [ 123 123 ] @testcase | 110 | 5 | [ 456 ] @testcase | 111 | 5 | 1 [ 456 100 ] +@testcase | 112 | 10 | [ 123 123 123 123 ] +@testcase | 112 | -10 | [ 456 ] @fif_codegen """ diff --git a/tolk-tester/tests/indexed-access.tolk b/tolk-tester/tests/indexed-access.tolk index 113124dd54..8cd4a74d1c 100644 --- a/tolk-tester/tests/indexed-access.tolk +++ b/tolk-tester/tests/indexed-access.tolk @@ -174,6 +174,7 @@ fun test112(f: int, s: int) { } @pure +@noinline fun getConstTuple(): Tup2Int { return [1,2]; } diff --git a/tolk-tester/tests/inline-tests.tolk b/tolk-tester/tests/inline-tests.tolk index 13add753e3..0645546a60 100644 --- a/tolk-tester/tests/inline-tests.tolk +++ b/tolk-tester/tests/inline-tests.tolk @@ -1,140 +1,560 @@ fun foo1(x: int): int { - if (x == 1) { - return 1; - } - return 2; + __expect_inline(false); // returns in the middle, leave it for Fift + if (x == 1) { + return 1; + } + return 2; } @inline fun foo2(x: int): int { - if (x == 1) { - return 11; - } - return 22; + __expect_inline(false); // returns in the middle, leave it for Fift + if (x == 1) { + return 11; + } + return 22; } @inline_ref fun foo3(x: int): int { - if (x == 1) { - return 111; - } - return 222; + if (x == 1) { + return 111; + } + return 222; +} + +@inline_ref +fun foo_light_but_ref() { + __expect_inline(false); + return 0; +} + +@noinline +fun foo_light_but_no() { + __expect_inline(false); + return 0; } @method_id(101) fun test1(x: int): (int, int, int) { - return (foo1(x)+1, foo2(x)+1, foo3(x)+1); + __expect_inline(false); // @method_id + return (foo1(x) + 1, foo2(x) + 1, foo3(x) + 1 + foo_light_but_ref() + foo_light_but_no()); } global g: int; @inline fun foo_repeat() { - g = 1; - repeat(5) { - g *= 2; - } + __expect_inline(true); + g = 1; + repeat (5) { + g *= 2; + } } @inline fun foo_until(): int { - g = 1; - var i: int = 0; - do { - g *= 2; - i += 1; - } while (i < 8); - return i; + __expect_inline(true); + g = 1; + var i: int = 0; + do { + g *= 2; + i += 1; + } while (i < 8); + return i; } @inline fun foo_while(): int { - g = 1; - var i: int = 0; - while (i < 10) { - g *= 2; - i += 1; - } - return i; + __expect_inline(true); + g = 1; + var i: int = 0; + while (i < 10) { + g *= 2; + i += 1; + } + return i; } @method_id(102) fun test2() { - foo_repeat(); - var x: int = g; - foo_until(); - var y: int = g; - foo_while(); - var z: int = g; - return (x, y, z); + foo_repeat(); + var x: int = g; + foo_until(); + var y: int = g; + foo_while(); + var z: int = g; + return (x, y, z); } @inline fun foo_big(x: int): int { - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - return x; + __expect_inline(true); + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + return x; +} + +@noinline +fun foo_big_not_annotated_called_once(x: int): int { + __expect_inline(true); // called only once, inline + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + return x; +} + +fun foo_big_not_annotated_called_twice(x: int): int { + __expect_inline(false); // too big for in-place auto-inlining when called multiple times + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + return x; } @method_id(103) fun test3(x: int): int { - return foo_big(x) * 10 + 5; + return foo_big(x) * 10 + 5; +} + +@method_id(104) +fun test4(x: int) { + return (foo_big_not_annotated_called_once(x) * 10, foo_big_not_annotated_called_twice(x) * 10, foo_big_not_annotated_called_twice(x)); +} + +struct Storage { + owner: address; + lastCall: int64; + extra: Cell<(int8, uint256)>; +} + +fun Storage.load() { + __expect_inline(true); + return Storage.fromCell(contract.getData()); +} + +fun Storage.save(self) { + __expect_inline(true); + contract.setData(self.toCell()); +} + +@method_id(105) +fun test5() { + var st = Storage.load(); + st.lastCall = 0; + st.save(); +} + +@inline +fun factorial(x: int): int { + __expect_inline(false); // due to recursion + if (x <= 1) { + return 1; + } + return x * factorial(x - 1); +} + +fun recurse1(x: int): int { __expect_inline(false); return recurse2(x + 1); } +fun recurse2(x: int): int { __expect_inline(false); return recurse3(x + 1); } +fun recurse3(x: int): int { __expect_inline(false); return recurse1(x + 1); } + +global g10: int; + +fun startCellWith(x: int) { + __expect_inline(true); + return beginCell().storeUint(x, 32); +} + +fun startCellWithG10() { + __expect_inline(true); + return beginCell().storeUint(g10, 32); +} + +fun builder.myBitsCount(self): int { + __expect_inline(true); + return self.bitsCount(); +} + +fun builder.store32(mutate self, v: int32): self { + __expect_inline(true); + return self.storeUint(v, 32); +} + +fun sum(a: int, b: int) { + __expect_inline(true); + return a + b; +} + +fun eq(v: T): T { + __expect_inline(true); + return v; +} + +fun get20() { return 20; } + +struct Point { + x: int; + y: int; +} + +fun Point.getX(self) { + __expect_inline(true); + return self.x; +} + +fun Point.create(x: int, y: int): Point { + __expect_inline(true); + return {x, y} +} + +fun increment(mutate x: int) { + __expect_inline(true); + x += 1; +} + +@method_id(106) +fun test6() { + var p = Point.create(10, 20); + increment(mutate p.x); + p.y = p.getX(); + return p; +} + +@method_id(107) +fun test7() { + g10 = 10; + return startCellWithG10().store32(20).myBitsCount(); +} + +@method_id(108) +fun test8() { + var x = 0; + return (startCellWith(x = 5).store32(x += 20).store32(x), x); +} + +@method_id(109) +fun test9(x: int) { + var p1: Point = {x, y: 0}; + var p2: Point = {x: 0, y: p1.getX()}; + var t = createEmptyTuple(); + t.pushPoint(p1); + t.pushPoint(p2); + return (t.popPoint(), t.popPoint(), t); +} + +fun tuple.pushPoint(mutate self, p: Point) { + __expect_inline(true); + var x = p.x; + self.push(x); + self.push(p.y); +} + +fun tuple.popPoint(mutate self): Point { + __expect_inline(true); + var y = self.pop(); + return {x: self.pop(), y}; +} + +@method_id(110) +fun test10() { + var (a, b, c) = usedIn10ButDeclaredBelow(5); + return (a, b, c.bitsCount()); +} + +global g11_1: int; +global g11_2: int; + +fun mutateGlobals(mutate v: int, inc11_2: int) { + __expect_inline(true); + v += 1; + g11_2 += inc11_2; +} + +@method_id(111) +fun test11() { + g11_1 = 0; + g11_2 = 0; + mutateGlobals(mutate g11_1, 5); // 1 5 + mutateGlobals(mutate g11_1, g11_2); // 2 10 + mutateGlobals(mutate g11_2, g11_1); // 2 11 + return (g11_1, g11_2); +} + +fun nested1(x: int) { + var y = x + 2; + x += 7; + return y; +} + +fun nested2(x: int) { + var y = x + 2; + var z = nested1(y); + y += z; + return y + z; +} + +@method_id(112) +fun test12(x: int) { + var r1 = nested2(10); + var r2 = nested2(x); + return (r1, r2); +} + +fun anotherMath(arg: int) { + __expect_inline(true); + var cp_x = arg; + increment(mutate arg); + return cp_x + arg; +} + +@method_id(113) +fun test13(p: Point) { + anotherMath(p.x); + var r = anotherMath(p.y); + return (p, r); +} + +fun evalPoint(p: Point) { + __expect_inline(true); + increment(mutate p.x); + p.y *= 2; + return max(p.x, p.y); +} + +@method_id(114) +fun test14(p2: Point) { + var p1: Point = { x: 0, y: 0 }; + var r1 = evalPoint(p1); + return (r1, p1, evalPoint(p2), p2); +} + +global t: tuple; + +fun logT(a: int) { + __expect_inline(true); + t.push(a); +} + +fun demoInlined1(x: int) { + if (x == -1) { + logT(10); + } else if (x == 0) { + logT(20); + } +} + +fun demoInlined2(x: int) { + __expect_inline(true); + match (x) { + -1 => logT(10), + else => logT(20), + } +} + +fun wrapperNotInlined(x: int?) { + if (x != null) { + demoInlined1(x); + return; + } + logT(100); +} + +fun someUsages(x: int?) { + __expect_inline(true); + wrapperNotInlined(x); + demoInlined2(x!); + return; +} + +@method_id(115) +fun test15(x: int?) { + t = createEmptyTuple(); + someUsages(x); + return t; +} + +struct (0x01) A16 { a: int8; } +struct (0x02) B16 {} +struct (0x03) C16 {} +type U16 = A16 | B16 | C16; + +fun check16(msg: U16) { + match (msg) { + A16 => { assert(msg.a > 0, 100); } + B16 => {} + C16 => {} + } +} + +fun wrap16(msg: U16, check: bool) { + if (check) { + check16(msg); + return; + } + throw 123; +} + +@method_id(116) +fun test16() { + wrap16(U16.fromSlice(stringHexToSlice("0170")), true); + return -1; } -fun main() {} + +fun main() { + // regardless of number of calls, it will be inlined, since it's lightweight in AST terms + Storage.load(); Storage.load(); Storage.load(); Storage.load(); Storage.load(); + Storage.load(); Storage.load(); Storage.load(); Storage.load(); Storage.load(); + Storage.load(); Storage.load(); Storage.load(); Storage.load(); Storage.load(); + Storage.load(); Storage.load(); Storage.load(); Storage.load(); Storage.load(); + Storage.load(); Storage.load(); Storage.load(); Storage.load(); Storage.load(); + usedIn10ButDeclaredBelow(10); +} + +fun usedIn10ButDeclaredBelow(x: int) { + __expect_inline(true); + var y = eq(x); + return (x, y, eq(startCellWith(x))); +} + + /** - method_id | in | out -@testcase | 101 | 1 | 2 12 112 -@testcase | 101 | 2 | 3 23 223 -@testcase | 102 | | 32 256 1024 -@testcase | 103 | 9 | 9111111111111111111111111111111111111111111111111115 + method_id | in | out +@testcase | 101 | 1 | 2 12 112 +@testcase | 101 | 2 | 3 23 223 +@testcase | 102 | | 32 256 1024 +@testcase | 103 | 9 | 911111111111111111111111111111111111111111111111115 +@testcase | 104 | 9 | 911111111111111111111111111110 911111111111111111111111111110 91111111111111111111111111111 +@testcase | 106 | | 11 11 +@testcase | 107 | | 64 +@testcase | 108 | | BC{0018000000050000001900000019} 25 +@testcase | 109 | 10 | 0 10 10 0 [] +@testcase | 110 | | 5 5 32 +@testcase | 111 | | 2 11 +@testcase | 112 | 10 | 40 40 +@testcase | 113 | 10 20 | 10 20 41 +@testcase | 114 | 10 20 | 1 0 0 40 10 20 +@testcase | 115 | -1 | [ 10 10 ] +@testcase | 115 | 1 | [ 20 ] +@testcase | 116 | | -1 + +@fif_codegen_avoid Storage.load +@fif_codegen_avoid Storage.save +@fif_codegen +""" + test5 PROC:<{ // + c4 PUSH // '7 + CTOS // s + LDMSGADDR // '11 s +""" + +@fif_codegen +""" + test6 PROC:<{ + 11 PUSHINT + DUP + }> +""" + +@fif_codegen +""" + test8 PROC:<{ // + 5 PUSHINT + NEWC + 32 STU // '3 + 25 PUSHINT // '3 '9 + SWAP // x self + 107374182425 PUSHINT + 64 STUR // x '3 + SWAP // '3 x + }> +""" + +@fif_codegen +""" + test9 PROC:<{ // x + 0 PUSHINT // p1.x p1.y=0 + s0 s1 PUSH2 // p1.x p1.y=0 p2.x=0 p2.y + NIL // x p1.y=0 p2.x=0 p2.y self + s0 s4 XCHG2 // p2.y p1.y=0 p2.x=0 self x + TPUSH // p2.y p1.y=0 p2.x=0 self + ROT // p2.y p2.x=0 self p1.y=0 + TPUSH // p2.y x=0 self + SWAP // p2.y self x=0 + TPUSH // p2.y self + SWAP // self p2.y + TPUSH // self + TPOP // self y + SWAP // y self + TPOP // '36 self '35 + SWAP // '36 '35 self + TPOP // '36 '35 self y + SWAP // '36 '35 y self + TPOP // '36 '35 '50 t '49 + s3 s4 XCHG // '35 '36 '50 t '49 + -ROT // '35 '36 '49 '50 t + }> +""" + +@fif_codegen +""" + test10 PROC:<{ + 5 PUSHINT // '3=5 + DUP // '3=5 y=5 + 5 PUSHINT + NEWC + 32 STU // a=5 b=5 c + BBITS // a=5 b=5 '18 + }> +""" + +@fif_codegen +""" + test12 PROC:<{ // x + 40 PUSHINT // x r1 + SWAP // r1 x + 2 ADDCONST // r1 y + ... +""" + +@fif_codegen +""" + wrapperNotInlined PROC:<{ // x + DUP // x x + ISNULL // x '1 + IFNOTJMP:<{ // x + DUP // x x + -1 EQINT // x '3 + IFJMP:<{ // x +""" + +@fif_codegen +""" + wrap16 PROC:<{ + IFJMP:<{ + 139 PUSHINT + OVER + EQUAL + IFJMP:<{ + DROP + 0 GTINT + 100 THROWIFNOT + }> + 140 PUSHINT + s2 POP + EQUAL + IFJMP:<{ + }> + }> + 123 THROW + }> +""" */ diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index 5b5d8563f0..bf39811125 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -215,6 +215,7 @@ fun test14(firstComponent: int8?): (int, int16)? { return firstComponent! < 10 ? null : (firstComponent! as int, 2 as int16); } +@noinline fun assign0(mutate v: T) { v = 0; } fun main() { diff --git a/tolk-tester/tests/invalid-semantics/err-4188.tolk b/tolk-tester/tests/invalid-semantics/err-4188.tolk new file mode 100644 index 0000000000..c908ed107f --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4188.tolk @@ -0,0 +1,13 @@ +fun recursive1(): int { + return recursive2(); +} + +fun recursive2(): int { + __expect_inline(true); + return recursive1(); +} + +/** +@compilation_should_fail +@stderr __expect_inline failed + */ diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk index 5599193532..ca3e21d839 100644 --- a/tolk-tester/tests/match-by-expr-tests.tolk +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -22,6 +22,7 @@ fun test2() { } } +@noinline fun isGt10(x: int) { return x > 10; } @method_id(103) @@ -116,6 +117,26 @@ fun test11(x: bool) { } } +global g12: int; + +@noinline +fun helper12(x: int) { + // via codegen, check that `return` is added implicitly to every case, and `IFJMP` is produced + match (x) { + 1 => g12 = 1, + 2 => g12 = 2, + 3 => g12 = 3, + else => g12 = x, + } +} + +@method_id(112) +fun test12(x: int) { + g12 = 0; + helper12(x); + return g12 +} + type asdf = int; @@ -131,6 +152,7 @@ fun main() { } /** +@testcase | 0 | | 2 @testcase | 101 | 1 | 100 @testcase | 101 | 2 | 200 @testcase | 101 | 3 | 300 @@ -154,6 +176,8 @@ fun main() { @testcase | 110 | 50 | 100 @testcase | 111 | 0 | -1 @testcase | 111 | -1 | 0 +@testcase | 112 | 2 | 2 +@testcase | 112 | 20 | 20 @fif_codegen """ @@ -209,4 +233,32 @@ fun main() { }> """ +@fif_codegen +""" + helper12 PROC:<{ + DUP + 1 EQINT + IFJMP:<{ + DROP + 1 PUSHINT + g12 SETGLOB + }> + DUP + 2 EQINT + IFJMP:<{ + DROP + 2 PUSHINT + g12 SETGLOB + }> + DUP + 3 EQINT + IFJMP:<{ + DROP + 3 PUSHINT + g12 SETGLOB + }> + g12 SETGLOB + }> +""" + */ diff --git a/tolk-tester/tests/methods-tests.tolk b/tolk-tester/tests/methods-tests.tolk index a54fe94157..83118ef369 100644 --- a/tolk-tester/tests/methods-tests.tolk +++ b/tolk-tester/tests/methods-tests.tolk @@ -33,12 +33,14 @@ fun Pair.create0(): Pair { fun Pair.create0(): Pair { return { first: 0, second: 0 }; } +@noinline fun Pair.createFrom(first: U, second: V): Pair { return { first: first as A, second: second as B }; } fun Pair.is0(self) { return self.first == 0 && self.second == 0; } +@noinline fun Pair.compareWith(self, f: U, s: V) { return self.first == f && self.second == s; } @@ -243,9 +245,11 @@ struct CounterIncrement { inc_by: int; } struct CounterReset { initial_value: int; } type CounterMsg = CounterIncrement | CounterReset; +@noinline fun CounterReset.onInternalMessage(self) { throw 123; } +@noinline fun CounterMsg.onInternalMessage(self) { return match (self) { CounterIncrement => self.inc_by, diff --git a/tolk-tester/tests/mutate-methods.tolk b/tolk-tester/tests/mutate-methods.tolk index 18cb644045..a53b2c15c0 100644 --- a/tolk-tester/tests/mutate-methods.tolk +++ b/tolk-tester/tests/mutate-methods.tolk @@ -1,17 +1,21 @@ +@noinline fun incrementInPlace(mutate x: int, byValue: int): void { x = x + byValue; } +@noinline fun int.incrementInPlace(mutate self, byValue: int): void { self = self + byValue; } +@noinline fun incrementTwoInPlace(mutate x: int, mutate y: int, byValue: int): int { x.incrementInPlace(byValue); y += byValue; return x + y; } +@noinline fun int.incrementTwoInPlace(mutate self, mutate y: int, byValue: int): int { self.incrementInPlace(byValue); y += byValue; @@ -40,6 +44,7 @@ fun testIncrement2() { } +@noinline fun load_next(mutate cs: slice): int { return cs.loadInt(32); } diff --git a/tolk-tester/tests/no-spaces.tolk b/tolk-tester/tests/no-spaces.tolk index 857b71e3b2..016cf478a5 100644 --- a/tolk-tester/tests/no-spaces.tolk +++ b/tolk-tester/tests/no-spaces.tolk @@ -37,6 +37,7 @@ global `some()var`:int; return [~-~~+-c3, ~+c3-~`c9`, -(-~+-c20-~c10+c3+~c38&39)]; } +@noinline fun add3(a: int, b: int, c: int) { return a+b+c; } @method_id(115) fun unary_const_check(): [int,int] { diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index d6934324e5..56e111b827 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -9,14 +9,17 @@ fun sumOfNullableTensorComponents(t: (int, int)?): int { return t!.0 + t!.1; } +@noinline fun isTensorNull(t: (int, int)?) { return t == null; } +@noinline fun (int, int)?.isTensorNull(self) { return self == null; } +@noinline fun incrementNullableTensorComponents(mutate t: (int, int)?) { if (t != null) { t!.0 += 1; @@ -37,12 +40,14 @@ fun incrementTensorComponents(mutate t: (int, int)) { t.1 += 1; } +@noinline fun (int, int).incrementTensorComponents(mutate self): self { self.0 += 1; self.1 += 1; return self; } +@noinline fun assignFirstComponent(mutate t: (int, int), first: int) { t!.0 = first; } diff --git a/tolk-tester/tests/op-priority.tolk b/tolk-tester/tests/op-priority.tolk index 3a1dee21e0..27b30c988d 100644 --- a/tolk-tester/tests/op-priority.tolk +++ b/tolk-tester/tests/op-priority.tolk @@ -1,7 +1,10 @@ fun justTrue(): bool { return true; } +@noinline fun unary_minus_1(a: int, b: int, c: int): int{return -(a+b) *c;} +@noinline fun unary_minus_2(a: int, b: int, c: int): int{return(-(a+b))*c;} +@noinline fun unary_minus_3(a: int, b: int, c: int): int{return-((a+b) *c);} diff --git a/tolk-tester/tests/pack-unpack-2.tolk b/tolk-tester/tests/pack-unpack-2.tolk index 789ed664f3..51432d0b4d 100644 --- a/tolk-tester/tests/pack-unpack-2.tolk +++ b/tolk-tester/tests/pack-unpack-2.tolk @@ -41,11 +41,12 @@ fun assert_slice_is_44_and_ref45(s: slice) { s.assertEnd(); } -@inline +@inline_ref fun slice.appendRef(self, refSlice: slice): slice { return beginCell().storeSlice(self).storeRef(beginCell().storeSlice(refSlice).endCell()).endCell().beginParse(); } +@noinline fun run(input: TInputStruct, ans: slice) { repeat (2) { var s = input.toCell().beginParse(); diff --git a/tolk-tester/tests/pack-unpack-3.tolk b/tolk-tester/tests/pack-unpack-3.tolk index 2e7d6ab2bc..310a3ed6cc 100644 --- a/tolk-tester/tests/pack-unpack-3.tolk +++ b/tolk-tester/tests/pack-unpack-3.tolk @@ -19,7 +19,7 @@ fun slice.assertEqDeeply(self, rhs: slice): slice { return self; } -@inline +@inline_ref fun slice.appendRef(self, refSlice: slice): slice { return beginCell().storeSlice(self).storeRef(beginCell().storeSlice(refSlice).endCell()).endCell().beginParse(); } @@ -35,6 +35,7 @@ fun generateCell_44_with_ref45(): cell { } +@noinline fun run(input: TInputStruct, ans: slice) { repeat (2) { var s = input.toCell().beginParse(); diff --git a/tolk-tester/tests/pack-unpack-4.tolk b/tolk-tester/tests/pack-unpack-4.tolk index 992a0664aa..d2bcfb6187 100644 --- a/tolk-tester/tests/pack-unpack-4.tolk +++ b/tolk-tester/tests/pack-unpack-4.tolk @@ -10,6 +10,7 @@ fun slice.assertEqDeeply(self, rhs: slice): slice { } +@noinline fun run(input: TInputStruct, ans: slice) { repeat (2) { var s = input.toCell().beginParse(); diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index 0f8ad02d29..36fb2ec2b9 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -25,6 +25,7 @@ fun address.createNone() { return createAddressNone() } +@noinline fun check0(addr: address) { assert (addr.getWorkchain() == 0) throw 111; } diff --git a/tolk-tester/tests/remove-unused-functions.tolk b/tolk-tester/tests/remove-unused-functions.tolk index 8e748ecf1f..cd72fd08c0 100644 --- a/tolk-tester/tests/remove-unused-functions.tolk +++ b/tolk-tester/tests/remove-unused-functions.tolk @@ -15,6 +15,7 @@ global used_gv: int; fun receiveGetter(): () -> int { return used_as_noncall2; } @pure +@noinline fun usedButOptimizedOut(x: int): int { return x + 2; } fun main(): (int, int, int) { diff --git a/tolk-tester/tests/self-keyword.tolk b/tolk-tester/tests/self-keyword.tolk index e504b5c484..e5941dc81f 100644 --- a/tolk-tester/tests/self-keyword.tolk +++ b/tolk-tester/tests/self-keyword.tolk @@ -1,17 +1,21 @@ +@noinline fun int.incChained(mutate self): self { self = self + 1; return self; } +@noinline fun int.incChained2(mutate self): self { return self.incChained(); } +@noinline fun int.incChained3(mutate self): self { self.incChained(); return self; } +@noinline fun int.incChained4(mutate self): self { self.incChained(); return self; diff --git a/tolk-tester/tests/send-msg-1.tolk b/tolk-tester/tests/send-msg-1.tolk index 07f736e207..a7be4ec058 100644 --- a/tolk-tester/tests/send-msg-1.tolk +++ b/tolk-tester/tests/send-msg-1.tolk @@ -30,7 +30,7 @@ message$_ {X:Type} fun getMyAddressDev(): address asm "x{80194DC6438F99D3D9DBE151944925D90B2492954BF6B9C070FBFF2DDED5F30547D_} PUSHSLICE"; -@inline +@inline_ref fun calculateNftItemStateInitData(itemIndex: int): cell { return beginCell() .storeUint(itemIndex, 64) @@ -61,6 +61,7 @@ struct(0x12345678) MyBody { queryId: uint64; } +@noinline fun test1_manual() { return beginCell() .storeUint(0x18, 6) // bounce @@ -86,6 +87,7 @@ fun test1() { return b.hash(); } +@noinline fun test2_manual() { return beginCell() .storeUint(0x10, 6) // no bounce @@ -111,6 +113,7 @@ fun test2() { return b.hash(); } +@noinline fun test3_manual() { val body_ref = beginCell() .storeUint(0x12345678, 32) @@ -137,6 +140,7 @@ fun test3() { return b.hash(); } +@noinline fun test4_manual(bodyCell: cell, dest: address, value: coins) { return beginCell() .storeUint(0x18, 6) // bounce @@ -164,6 +168,7 @@ fun test4(value: coins) { return b.hash(); } +@noinline fun test5_manual() { var ec_dict = createEmptyDict(); ec_dict.iDictSet(32, 1, "ec1"); @@ -194,6 +199,7 @@ fun test5() { return b.hash(); } +@noinline fun test6_manual(value: coins, ec_dict: dict) { return beginCell() .storeUint(0x18, 6) // bounce @@ -228,6 +234,7 @@ struct MyNftBody { nftContent: cell; } +@noinline fun test7_manual(nftItemCode: cell, amount: coins) { var (itemIndex: int, nftContent: cell) = (10, beginCell().endCell()); val nftItemData = calculateNftItemStateInitData(itemIndex); @@ -268,6 +275,7 @@ struct(0x706c7567) RequestPaymentMessage { someDict: dict; } +@noinline fun test8_manual(destAddr: address, requestedAmount: coins) { return beginCell() .storeUint(0x18, 6) // bounce @@ -298,6 +306,7 @@ fun test8(requestedAmount: coins) { return b.hash(); } +@noinline fun test9_manual(bounceable: bool) { return beginCell() .storeUint(0b01, 2).storeBool(bounceable).storeUint(0b000, 3) @@ -324,6 +333,7 @@ fun test9(bounceable: bool) { type Body500Bits = (uint250, uint250); // this body guaranteely fits into cell +@noinline fun test10_manual(bd: Body500Bits) { return beginCell() .storeUint(0x10, 6) @@ -349,6 +359,7 @@ fun test10(bd: Body500Bits) { type Body750Bits = (uint250, uint250, uint250); // this body is auto-ref +@noinline fun test11_manual(bd: Body750Bits) { val bodyRef = beginCell().storeUint(bd.0,250).storeUint(bd.1,250).storeUint(bd.2,250).endCell(); return beginCell() @@ -372,6 +383,7 @@ fun test11(bd: Body750Bits) { return b.hash(); } +@noinline fun test12_manual(data32: uint32) { var init = ContractState { data: beginCell().storeUint(data32,32).endCell(), @@ -407,6 +419,7 @@ fun test12(data32: uint32) { return b.hash(); } +@noinline fun test13_manual() { return beginCell() .storeUint(0x18, 6) // bounce @@ -427,6 +440,7 @@ fun test13() { return b.hash(); } +@noinline fun test14_manual(stateInitCell: cell, amount: coins) { val nftAddress = calculateNftItemAddress(BASECHAIN, stateInitCell); @@ -460,6 +474,7 @@ struct(0x1234) Body15 { more: int32; } +@noinline fun test15_manual(stateInitCell: cell, bd: Body15) { val bodyRef = beginCell().storeUint(0x1234,16).storeUint(bd.tens.0,250).storeUint(bd.tens.1,250).storeUint(bd.tens.2,250).storeInt(bd.more,32).endCell(); val nftAddress = calculateNftItemAddress(MASTERCHAIN, stateInitCell); @@ -502,6 +517,7 @@ fun test15(tens0: int, tens1: int) { return b.hash(); } +@noinline fun test18_manual(dest: builder, queryId: uint64) { return beginCell() .storeUint(0x18, 6) // bounce @@ -527,6 +543,7 @@ fun test18(queryId: uint64) { return b.hash(); } +@noinline fun test19_manual(bd: uint64) { var destB = beginCell().storeUint(0, 2); return beginCell() @@ -551,6 +568,7 @@ fun test19(bd: uint64) { return b.hash(); } +@noinline fun test20_manual() { var bodyB = beginCell().storeUint(0x12345678, 32).storeUint(800, 64); return beginCell() @@ -580,6 +598,7 @@ struct(0x12345678) Body21 { payload: RemainingBitsAndRefs; } +@noinline fun test21_manual(bd: Body21) { return beginCell() .storeUint(0x10, 6) // no bounce @@ -605,6 +624,7 @@ fun test21(queryId: uint64) { return b.hash(); } +@noinline fun test22_manual(unsafeB: builder) { return beginCell() .storeUint(0x18, 6) // bounce @@ -635,6 +655,7 @@ struct Body23Unlimited { any2: RemainingBitsAndRefs; } +@noinline fun test23_manual(state32: uint32, body: Body23Unlimited) { var init = ContractState { data: beginCell().storeUint(state32,32).endCell(), @@ -678,6 +699,7 @@ fun test23(state32: uint32) { return b.hash(); } +@noinline fun test24_manual(addrHash: uint256) { return beginCell() .storeUint(0x18, 6) // bounce diff --git a/tolk-tester/tests/send-msg-2.tolk b/tolk-tester/tests/send-msg-2.tolk index b1e3af99cb..1d11752db3 100644 --- a/tolk-tester/tests/send-msg-2.tolk +++ b/tolk-tester/tests/send-msg-2.tolk @@ -104,6 +104,7 @@ fun calculateAddressInAnotherShard(pivotAddress: address, shardPrefixLen: uint5, .storeUint(prefixLess, 256 - shardPrefixLen); } +@noinline fun test1_manual(toAddress: address, masterMsg: cell, jettonWalletCode: cell) { var shardPrefix = getAddressShard(toAddress, SHARD_DEPTH); var stateInitCell = calculateJettonWalletStateInitWithShard(toAddress, getMyAddressDev(), jettonWalletCode); @@ -149,6 +150,7 @@ fun test1() { return b.hash() } +@noinline fun test2_manual(toAddress: address, masterMsg: cell, jettonWalletCode: cell) { var shardPrefix = getAddressShard(toAddress, SHARD_DEPTH); var stateInit = calculateJettonWalletStateInitWithShard(toAddress, getMyAddressDev(), jettonWalletCode); @@ -192,6 +194,7 @@ struct Test3Body { bigData: bits800; } +@noinline fun test3_manual(myCode: cell, myData: cell, msgBody: Test3Body) { var addrInShard = calculateAddressInAnotherShard(getMyAddressDev(), 20, myCode, myData); diff --git a/tolk-tester/tests/send-msg-3.tolk b/tolk-tester/tests/send-msg-3.tolk index 69afed4cd5..07265ec965 100644 --- a/tolk-tester/tests/send-msg-3.tolk +++ b/tolk-tester/tests/send-msg-3.tolk @@ -16,6 +16,7 @@ struct(0x12345678) MyBody { queryId: uint64; } +@noinline fun test1_manual(topic: int) { return beginCell() .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () @@ -42,6 +43,7 @@ fun test1(topic: int) { type Body750Bits = (uint250, uint250, uint250); // this body is auto-ref +@noinline fun test2_manual(bd: Body750Bits) { val bodyRef = beginCell().storeUint(bd.0,250).storeUint(bd.1,250).storeUint(bd.2,250).endCell(); return beginCell() @@ -81,6 +83,7 @@ fun test3(eventId: int) { return b.hash(); } +@noinline fun test4_manual(dest: address, body: cell) { return beginCell() .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () @@ -121,6 +124,7 @@ fun test6() { return b.hash(); } +@noinline fun test7_manual(dest: builder, queryId: uint64) { return beginCell() .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () @@ -174,6 +178,7 @@ fun test9(topic: int) { return b.hash(); } +@noinline fun test10_manual(topic: slice, queryIdRef: uint64) { return beginCell() .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 7790953022..247431eb0c 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -600,6 +600,7 @@ fun test54() { __expect_type(x3, "int?"); } +@noinline fun eq55(v: T) { return v; } fun test55() { diff --git a/tolk-tester/tests/some-tests-2.tolk b/tolk-tester/tests/some-tests-2.tolk index 592cf96efc..3568da56ca 100644 --- a/tolk-tester/tests/some-tests-2.tolk +++ b/tolk-tester/tests/some-tests-2.tolk @@ -44,7 +44,7 @@ fun test89(last: int): (int, int, int, int) { return (t.get(0), t.get(t.size() - 1), t.first(), t.last()); } -@pure fun get10() { return 10; } +@pure @noinline fun get10() { return 10; } @method_id(91) fun touchCodegen2() { diff --git a/tolk-tester/tests/some-tests-3.tolk b/tolk-tester/tests/some-tests-3.tolk index 728b18d3f8..8c53e33bf3 100644 --- a/tolk-tester/tests/some-tests-3.tolk +++ b/tolk-tester/tests/some-tests-3.tolk @@ -15,6 +15,7 @@ fun main(cs: slice) { return (cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8)); } +@noinline fun f(cs: slice) { return (cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index bbe4583266..ab2facf8a5 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -78,5 +78,5 @@ fun test1() { @testcase | 0 | | 0 @testcase | 101 | | [ 65 66 67 68 ] -@code_hash 61963905046482786665931036224265588524206004864815957521622605110240907698574 +@code_hash 15952581052628192793433894459656689089299796554419234117286198843831925928625 */ diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index c7f81e4c3f..4a9a431e7e 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -264,6 +264,7 @@ fun test17(x: JustInt?) { return (x, x!, x!.value, w1, w2); } +@noinline fun sumXY